[
  {
    "path": ".asf.yaml",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\ngithub:\n  description: \"Apache RocketMQ is a cloud native messaging and streaming platform, making it simple to build event-driven applications.\"\n  homepage: https://rocketmq.apache.org/\n  labels:\n    - messaging\n    - streaming\n    - eventing\n    - cloud-native\n    - rocketmq\n    - java\n    - hacktoberfest\n  enabled_merge_buttons:\n    # Enable squash button\n    squash: true\n    # Disable merge button\n    merge: false\n    # Disable rebase button\n    rebase: false\n  protected_branches:\n    master: {}\n    develop:\n      required_pull_request_reviews:\n        dismiss_stale_reviews: true\n        require_code_owner_reviews: false\n        required_approving_review_count: 1\n      required_status_checks:\n        contexts:\n          - misspell-check\n          - check-license\n          - maven-compile (ubuntu-latest, JDK-8)\n          - maven-compile (windows-latest, JDK-8)\n          - maven-compile (macos-latest, JDK-8)\nnotifications:\n  commits:      commits@rocketmq.apache.org\n  issues:       commits@rocketmq.apache.org\n  pullrequests: commits@rocketmq.apache.org\n  jobs:         commits@rocketmq.apache.org\n  discussions:  dev@rocketmq.apache.org\n"
  },
  {
    "path": ".bazelrc",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\nstartup --host_jvm_args=-Xmx2g\n\nrun --color=yes\n\nbuild --color=yes\nbuild --enable_platform_specific_config\n\ntest --action_env=TEST_TMPDIR=/tmp\n\ntest  --experimental_strict_java_deps=warn\ntest  --experimental_ui_max_stdouterr_bytes=10485760\nbuild --experimental_strict_java_deps=warn\n\ntest --test_output=errors\n\n\n# This .bazelrc file contains all of the flags required for the provided\n# toolchain with Remote Build Execution.\n# Note your WORKSPACE must contain an rbe_autoconfig target with\n# name=\"rbe_default\" to use these flags as-is.\n\n# Depending on how many machines are in the remote execution instance, setting\n# this higher can make builds faster by allowing more jobs to run in parallel.\n# Setting it too high can result in jobs that timeout, however, while waiting\n# for a remote machine to execute them.\nbuild:remote --jobs=150\n\nbuild:remote --remote_executor=grpcs://remote.buildbuddy.io\nbuild:remote --host_platform=@buildbuddy_toolchain//:platform\nbuild:remote --platforms=@buildbuddy_toolchain//:platform\nbuild:remote --extra_execution_platforms=@buildbuddy_toolchain//:platform\nbuild:remote --crosstool_top=@buildbuddy_toolchain//:toolchain\nbuild:remote --extra_toolchains=@buildbuddy_toolchain//:cc_toolchain\nbuild:remote --java_language_version=8\nbuild:remote --java_runtime_version=8\nbuild:remote --define=EXECUTOR=remote\n\n# Enable remote execution so actions are performed on the remote systems.\nbuild:remote --remote_executor=grpcs://remote.buildbuddy.io\n\n# Enforce stricter environment rules, which eliminates some non-hermetic\n# behavior and therefore improves both the remote cache hit rate and the\n# correctness and repeatability of the build.\nbuild:remote --incompatible_strict_action_env=true\n\n# Set a higher timeout value, just in case.\nbuild:remote --remote_timeout=3600\n\n# Use a pre-configured account, such that we may have pull-request replacing pull-request-target\nbuild:remote --remote_header=x-buildbuddy-api-key=FD819nUEY7WjvqmoufsU\ntest:remote --remote_header=x-buildbuddy-api-key=FD819nUEY7WjvqmoufsU\n"
  },
  {
    "path": ".bazelversion",
    "content": "6.5.0"
  },
  {
    "path": ".github/ISSUE_TEMPLATE/bug_report.yml",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nname: Bug Report\ntitle: \"[Bug] Bug title \"\ndescription: Create a report to help us identify any unintended flaws, errors, or faults.\nbody:\n  - type: checkboxes\n    attributes:\n      label: Before Creating the Bug Report\n      options:\n        - label: >\n            I found a bug, not just asking a question, which should be created in [GitHub Discussions](https://github.com/apache/rocketmq/discussions).\n          required: true\n        - label: >\n            I have searched the [GitHub Issues](https://github.com/apache/rocketmq/issues) and [GitHub Discussions](https://github.com/apache/rocketmq/discussions)  of this repository and believe that this is not a duplicate.\n          required: true\n        - label: >\n            I have confirmed that this bug belongs to the current repository, not other repositories of RocketMQ.\n          required: true\n\n  - type: textarea\n    attributes:\n      label: Runtime platform environment\n      description: Describe the runtime platform environment.\n      placeholder: >\n        OS: (e.g., \"Ubuntu 20.04\")\n        OS: (e.g., \"Windows Server 2019\")\n    validations:\n      required: true\n\n  - type: textarea\n    attributes:\n      label: RocketMQ version\n      description: Describe the RocketMQ version.\n      placeholder: >\n        branch: (e.g develop|4.9.x)\n        version: (e.g. 5.1.0|4.9.5)\n        Git commit id: (e.g. c88b5cfa72e204962929eea105687647146112c6)\n    validations:\n      required: true\n\n  - type: textarea\n    attributes:\n      label: JDK Version\n      description: Run or Compiler version.\n      placeholder: >\n        Compiler: (e.g., \"Oracle JDK 11.0.17\")\n        OS: (e.g., \"Ubuntu 20.04\")\n        Runtime (if different from JDK above): (e.g., \"Oracle JRE 8u251\")\n        OS (if different from OS compiled on): (e.g., \"Windows Server 2019\")\n    validations:\n      required: false\n\n  - type: textarea\n    attributes:\n      label: Describe the Bug\n      description: Describe what happened.\n      placeholder: >\n        A clear and concise description of what the bug is.\n    validations:\n      required: true\n\n  - type: textarea\n    attributes:\n      label: Steps to Reproduce\n      description: Describe the steps to reproduce the bug here.\n      placeholder: >\n        If possible, provide a recipe for reproducing the error.\n    validations:\n      required: true\n\n  - type: textarea\n    attributes:\n      label: What Did You Expect to See?\n      description: You expect to see result.\n      placeholder: >\n        A clear and concise description of what you expected to see.\n    validations:\n      required: true\n\n  - type: textarea\n    attributes:\n      label: What Did You See Instead?\n      description: You instead to see result.\n      placeholder: >\n        A clear and concise description of what you saw instead.\n    validations:\n      required: true\n\n  - type: textarea\n    attributes:\n      label: Additional Context\n      description: Additional context.\n      placeholder: >\n        Add any other context about the problem here.\n    validations:\n      required: false\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE/config.yml",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nblank_issues_enabled: false\ncontact_links:\n  - name: Ask Question\n    url: https://github.com/apache/rocketmq/discussions\n    about: Please go to GitHub Disccusions to ask questions\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE/doc.yml",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nname: Documentation Related\ntitle: \"[Doc] Documentation Related \"\ndescription: I find some issues related to the documentation.\nlabels: [ \"module/doc\" ]\nbody:\n  - type: checkboxes\n    attributes:\n      label: Search before creation\n      description: >\n        Please make sure to search in the [issues](https://github.com/apache/rocketmq/issues)\n        first to see whether the same issue was reported already.\n      options:\n        - label: >\n            I had searched in the [issues](https://github.com/apache/rocketmq/issues) and found\n            no similar issues.\n          required: true\n\n  - type: textarea\n    attributes:\n      label: Documentation Related\n      description: Describe the suggestion about document.\n      placeholder: >\n        e.g There is a typo\n    validations:\n      required: true\n\n  - type: checkboxes\n    attributes:\n      label: Are you willing to submit PR?\n      description: >\n        This is absolutely not required, but we are happy to guide you in the contribution process\n        especially if you already have a good understanding of how to implement the fix.\n      options:\n        - label: Yes I am willing to submit a PR!\n\n  - type: markdown\n    attributes:\n      value: \"Thanks for completing our form!\"\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE/enhancement_request.yml",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nname: Enhancement Request\ntitle: \"[Enhancement] Enhancement title\"\ndescription: Suggest an enhancement for this project\nlabels: [ \"type/enhancement\" ]\nbody:\n  - type: checkboxes\n    attributes:\n      label: Before Creating the Enhancement Request\n      description: >\n        Most of issues should be classified as bug or feature request. An issue should be considered as an enhancement when it proposes improvements to\n        existing functionality or user experience, without necessarily introducing new features or fixing existing bugs.\n      options:\n        - label: >\n            I have confirmed that this should be classified as an enhancement rather than a bug/feature.\n          required: true\n\n  - type: textarea\n    attributes:\n      label: Summary\n      placeholder: >\n        A clear and concise description of the enhancement you would like to see in the project.\n    validations:\n      required: true\n\n  - type: textarea\n    attributes:\n      label: Motivation\n      placeholder: >\n        Explain why you believe this enhancement is necessary, and how it benefits the project and community.\n        Include any specific use cases that you have in mind.\n    validations:\n      required: true\n\n  - type: textarea\n    attributes:\n      label: Describe the Solution You'd Like\n      placeholder: >\n        Describe the enhancement you propose, detailing the change and implementation steps involved.\n        If you have multiple solutions, please list them separately.\n    validations:\n      required: true\n\n  - type: textarea\n    attributes:\n      label: Describe Alternatives You've Considered\n      placeholder: >\n        List any alternative enhancements or implementations you have considered, and explain why they may not be as effective or appropriate.\n    validations:\n      required: true\n\n  - type: textarea\n    attributes:\n      label: Additional Context\n      placeholder: >\n        Add any relevant context, screenshots, prototypes, or other supplementary information to help illustrate the enhancement.\n    validations:\n      required: false\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE/feature_request.yml",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nname: Feature Request\ntitle: \"[Feature] New feature title\"\ndescription: Suggest an idea for this project.\nlabels: [ \"type/new feature\" ]\nbody:\n  - type: textarea\n    attributes:\n      label: Is Your Feature Request Related to a Problem?\n      description: Please Describe It.\n      placeholder: >\n        A clear and concise description of what the problem is.\n    validations:\n      required: true\n\n  - type: textarea\n    attributes:\n      label: Describe the Solution You'd Like\n      description: Describe how you solved it.\n      placeholder: >\n        A clear and concise description of what you want to happen.\n    validations:\n      required: true\n\n  - type: textarea\n    attributes:\n      label: Describe Alternatives You've Considered\n      description: Describe your solution\n      placeholder: >\n        A clear and concise description of any alternative solutions or features you've considered.\n    validations:\n      required: true\n\n  - type: textarea\n    attributes:\n      label: Additional Context\n      description: Additional context.\n      placeholder: >\n        Add any other context about the problem here.\n    validations:\n      required: false\n"
  },
  {
    "path": ".github/PULL_REQUEST_TEMPLATE.md",
    "content": "<!-- Please make sure the target branch is right. In most case, the target branch should be `develop`. -->\n\n### Which Issue(s) This PR Fixes\n\n<!-- Please ensure that the related issue has already been created, and [link this pull request to that issue using keywords](<https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword>) to ensure automatic closure. -->\n\n- Fixes #issue_id\n\n### Brief Description\n\n<!-- Write a brief description for your pull request to help the maintainer understand the reasons behind your changes. -->\n\n### How Did You Test This Change?\n\n<!-- In order to ensure the code quality of Apache RocketMQ, we expect every pull request to have undergone thorough testing. -->\n"
  },
  {
    "path": ".github/asf-deploy-settings.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one or more\n  ~ contributor license agreements.  See the NOTICE file distributed with\n  ~ this work for additional information regarding copyright ownership.\n  ~ The ASF licenses this file to You under the Apache License, Version 2.0\n  ~ (the \"License\"); you may not use this file except in compliance with\n  ~ the License.  You may obtain a copy of the License at\n  ~\n  ~      http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing, software\n  ~ distributed under the License is distributed on an \"AS IS\" BASIS,\n  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  ~ See the License for the specific language governing permissions and\n  ~ limitations under the License.\n  -->\n\n<settings xmlns=\"http://maven.apache.org/SETTINGS/1.0.0\"\n          xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n          xsi:schemaLocation=\"\n            http://maven.apache.org/SETTINGS/1.0.0\n            http://maven.apache.org/xsd/settings-1.0.0.xsd\">\n\n    <servers>\n        <server>\n            <id>apache.snapshots.https</id>\n            <username>${env.NEXUS_DEPLOY_USERNAME}</username>\n            <password>${env.NEXUS_DEPLOY_PASSWORD}</password>\n            <configuration>\n                <snapshotRepository>\n                    <maxUniqueSnapshots>60</maxUniqueSnapshots>\n                </snapshotRepository>\n            </configuration>\n        </server>\n    </servers>\n\n</settings>"
  },
  {
    "path": ".github/workflows/bazel.yml",
    "content": "name: Build and Run Tests by Bazel\non:\n  pull_request:\n    types: [opened, reopened, synchronize]\n  push:\n    branches:\n      - master\n      - develop\n      - bazel\n\njobs:\n  build:\n    name: \"bazel-compile (${{ matrix.os }})\"\n    runs-on: ${{ matrix.os }}\n    strategy:\n      matrix:\n        os: [ubuntu-latest]\n    steps:\n      - uses: actions/checkout@v2\n      - name: Build\n        run: bazel build --config=remote //...\n      - name: Run Tests\n        run: bazel test --config=remote //..."
  },
  {
    "path": ".github/workflows/codeql_analysis.yml",
    "content": "name: CodeQL Analysis\n\non:\n  pull_request:\n    types: [opened, reopened, synchronize]\n  push:\n    branches:\n      - master\n\njobs:\n  CodeQL-Build:\n    runs-on: ubuntu-latest\n\n    steps:\n      - name: Checkout repository\n        uses: actions/checkout@v3\n      - name: Cache Maven packages\n        uses: actions/cache@v3\n        with:\n          path: ~/.m2/repository\n          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}\n          restore-keys: |\n            ${{ runner.os }}-maven-\n      # Initializes the CodeQL tools for scanning.\n      - name: Initialize CodeQL\n        uses: github/codeql-action/init@v2\n        with:\n          languages: java\n      - name: Autobuild\n        uses: github/codeql-action/autobuild@v2\n      - name: Perform CodeQL Analysis\n        uses: github/codeql-action/analyze@v2\n"
  },
  {
    "path": ".github/workflows/coverage.yml",
    "content": "name: Coverage\non:\n  pull_request:\n    types: [opened, reopened, synchronize]\n  push:\n    branches: [master, develop]\njobs:\n  calculate-coverage:\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@master\n      - name: Set up JDK 8\n        uses: actions/setup-java@v4\n        with:\n          java-version: \"8\"\n          distribution: \"corretto\"\n          cache: \"maven\"\n      - name: Generate coverage report\n        run: mvn -B test -T 2C --file pom.xml\n      - name: Upload to Codecov\n        uses: codecov/codecov-action@v3\n        with:\n          fail_ci_if_error: true\n          verbose: true\n          token: cf0cba0a-22f8-4580-89ab-4f1dec3bda6f\n"
  },
  {
    "path": ".github/workflows/integration-test.yml",
    "content": "name: Run Integration Tests\non:\n  pull_request:\n    types: [opened, reopened, synchronize]\n  push:\n    branches: [master, develop]\n\njobs:\n  it-test:\n    name: \"maven-compile (${{ matrix.os }}, JDK-${{ matrix.jdk }})\"\n    runs-on: ${{ matrix.os }}\n    strategy:\n      matrix:\n        os: [ubuntu-latest]\n        jdk: [8]\n    steps:\n      - name: Cache Maven Repos\n        uses: actions/cache@v4\n        with:\n          path: ~/.m2/repository\n          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}\n          restore-keys: |\n            ${{ runner.os }}-maven-\n\n      - name: Checkout\n        uses: actions/checkout@v2\n\n      - name: Set up JDK ${{ matrix.jdk }}\n        uses: actions/setup-java@v4\n        with:\n          java-version: ${{ matrix.jdk }}\n          distribution: \"corretto\"\n          cache: \"maven\"\n\n      - name: Run integration tests with Maven\n        run: mvn clean verify -Pit-test -Pskip-unit-tests\n\n      - name: Publish Test Report\n        uses: mikepenz/action-junit-report@v3\n        if: always()\n        with:\n          report_paths: 'test/target/failsafe-reports/TEST-*.xml'\n          annotate_only: true\n          include_passed: true\n          detailed_summary: true\n"
  },
  {
    "path": ".github/workflows/license-checker.yaml",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nname: License checker\n\non:\n  pull_request:\n    branches:\n      - develop\n      - master\n\njobs:\n  check-license:\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@v3\n      - name: Check License Header\n        uses: apache/skywalking-eyes@v0.2.0\n        with:\n          log: info\n          config: .licenserc.yaml\n"
  },
  {
    "path": ".github/workflows/maven.yaml",
    "content": "name: Build and Run Tests by Maven\non:\n  pull_request:\n    types: [opened, reopened, synchronize]\n  push:\n    branches: [master, develop, bazel]\n\njobs:\n  java_build:\n    name: \"maven-compile (${{ matrix.os }}, JDK-${{ matrix.jdk }})\"\n    runs-on: ${{ matrix.os }}\n    strategy:\n      fail-fast: false\n      matrix:\n        # see https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources\n        os: [ubuntu-latest, windows-latest, macos-latest]\n        jdk: [8]\n    steps:\n      - name: Checkout\n        uses: actions/checkout@v2\n      - name: Set up JDK ${{ matrix.jdk }}\n        uses: actions/setup-java@v4\n        with:\n          java-version: ${{ matrix.jdk }}\n          # See https://github.com/actions/setup-java?tab=readme-ov-file#supported-distributions\n          # AdoptOpenJDK got moved to Eclipse Temurin and won't be updated anymore.\n          distribution: \"corretto\"\n          cache: \"maven\"\n      - name: Build with Maven\n        run: mvn -B package --file pom.xml\n\n      - name: Upload Auth JVM crash logs\n        if: failure()\n        uses: actions/upload-artifact@v4\n        with:\n          name: jvm-crash-logs\n          path: /Users/runner/work/rocketmq/rocketmq/auth/hs_err_pid*.log\n          retention-days: 1\n\n      - name: Upload broker JVM crash logs\n        if: failure()\n        uses: actions/upload-artifact@v4\n        with:\n          name: jvm-crash-logs\n          path: /Users/runner/work/rocketmq/rocketmq/broker/hs_err_pid*.log\n          retention-days: 1\n"
  },
  {
    "path": ".github/workflows/misspell_check.yml",
    "content": "name: Misspell Check\non:\n  pull_request:\n    types: [opened, reopened, synchronize]\n  push:\n    branches: [master, develop]\njobs:\n  misspell-check:\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@v3\n      - name: Install misspell\n        run: |\n          curl -L -o ./install-misspell.sh https://git.io/misspell\n          sh ./install-misspell.sh\n      - name: Run misspell\n        run: find . -type f -print0 | xargs -0 bin/misspell -error -i transfered,derivate\n"
  },
  {
    "path": ".github/workflows/pr-ci.yml",
    "content": "name: PR-CI\n\non:\n  pull_request:\n    types: [opened, reopened, synchronize]\n\njobs:\n  dist-tar:\n    name: Build distribution tar\n    runs-on: ubuntu-latest\n    timeout-minutes: 120\n    steps:\n      - uses: actions/checkout@v3\n        with:\n          submodules: true\n      - uses: actions/setup-java@v3\n        with:\n          distribution: \"temurin\"\n          java-version: \"8\"\n          cache: \"maven\"\n      - name: Build distribution tar\n        run: |\n          mvn -Prelease-all -DskipTests -Dspotbugs.skip=true clean install -U\n      - uses: actions/upload-artifact@v4\n        name: Upload distribution tar\n        with:\n          name: rocketmq\n          path: distribution/target/rocketmq*/rocketmq*\n      - name: Save PR number\n        run: |\n          mkdir -p ./pr\n          echo ${{ github.event.number }} > ./pr/NR\n      - uses: actions/upload-artifact@v4\n        with:\n          name: pr\n          path: pr/\n"
  },
  {
    "path": ".github/workflows/pr-e2e-test.yml",
    "content": "name: E2E test for pull request\n\n# read-write repo token\n# access to secrets\non:\n  workflow_run:\n    workflows: [\"PR-CI\"]\n    types:\n      - completed\n\nenv:\n  DOCKER_REPO: apache/rocketmq-ci\n\njobs:\n  docker:\n    if: >\n      github.repository == 'apache/rocketmq' &&\n      github.event.workflow_run.event == 'pull_request' &&\n      github.event.workflow_run.conclusion == 'success'\n    runs-on: ubuntu-latest\n    timeout-minutes: 30\n    strategy:\n      matrix:\n        base-image: [\"ubuntu\"]\n        java-version: [\"8\"]\n    steps:\n      - name: 'Download artifact'\n        uses: actions/github-script@v6\n        with:\n          script: |\n            let artifacts = await github.rest.actions.listWorkflowRunArtifacts({\n               owner: context.repo.owner,\n               repo: context.repo.repo,\n               run_id: ${{github.event.workflow_run.id }},\n            });\n            let matchArtifactRmq = artifacts.data.artifacts.filter((artifact) => {\n              return artifact.name == \"rocketmq\"\n            })[0];\n            let download = await github.rest.actions.downloadArtifact({\n               owner: context.repo.owner,\n               repo: context.repo.repo,\n               artifact_id: matchArtifactRmq.id,\n               archive_format: 'zip',\n            });\n            var fs = require('fs');\n            fs.writeFileSync('${{github.workspace}}/rocketmq.zip', Buffer.from(download.data));\n      - run: |\n          unzip rocketmq.zip\n          mkdir rocketmq\n          cp -r rocketmq-* rocketmq/\n          ls\n      - uses: actions/checkout@v3\n        with:\n          repository: apache/rocketmq-docker.git\n          ref: master\n          path: rocketmq-docker\n      - name: docker-login\n        uses: docker/login-action@v2\n        with:\n          registry: ${{ env.REGISTRY }}\n          username: ${{ secrets.DOCKERHUB_USER }}\n          password: ${{ secrets.DOCKERHUB_TOKEN }}\n      - name: Build and save docker images\n        id: build-images\n        run: |\n          cd rocketmq-docker/image-build-ci\n          version=${{ github.event.pull_request.number || github.ref_name }}-$(uuidgen)\n          mkdir versionlist\n          touch versionlist/\"${version}-`echo ${{ matrix.base-image }} | sed -e \"s/:/-/g\"`\"\n          sh ./build-image-local.sh ${version} ${{ matrix.base-image }} ${{ matrix.java-version }} ${DOCKER_REPO}\n      - uses: actions/upload-artifact@v4\n        name: Upload distribution tar\n        with:\n          name: versionlist\n          path: rocketmq-docker/image-build-ci/versionlist/*\n  \n  list-version:\n    if: >\n      github.repository == 'apache/rocketmq' &&\n      always()\n    name: List version\n    needs: [docker]\n    runs-on: ubuntu-latest\n    timeout-minutes: 30\n    outputs:\n      version-json: ${{ steps.show_versions.outputs.version-json }}\n    steps:\n      - uses: actions/download-artifact@v4\n        name: Download versionlist\n        with:\n          name: versionlist\n          path: versionlist\n      - name: Show versions\n        id: show_versions\n        run: | \n          a=(`ls versionlist`)\n          printf '%s\\n' \"${a[@]}\" | jq -R . | jq -s .\n          echo version-json=`printf '%s\\n' \"${a[@]}\" | jq -R . | jq -s .` >> $GITHUB_OUTPUT\n\n  deploy:\n    if: ${{ success() }}\n    name: Deploy RocketMQ\n    needs: [list-version,docker]\n    runs-on: ubuntu-latest\n    timeout-minutes: 60\n    strategy:\n      matrix:\n        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}\n    steps:\n      - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288\n        name: Deploy rocketmq\n        with:\n          action: \"deploy\"\n          ask-config: \"${{ secrets.ASK_CONFIG_VIRGINA }}\"\n          test-version: \"${{ matrix.version }}\"\n          chart-git: \"https://ghproxy.com/https://github.com/apache/rocketmq-docker.git\"\n          chart-branch: \"master\"\n          chart-path: \"./rocketmq-k8s-helm\"\n          job-id: ${{ strategy.job-index }}\n          helm-values: |\n            nameserver:\n              image:\n                repository: ${{env.DOCKER_REPO}}\n                tag: ${{ matrix.version }}\n            broker:\n              image:\n                repository: ${{env.DOCKER_REPO}}\n                tag: ${{ matrix.version }}\n            proxy:\n              image:\n                repository: ${{env.DOCKER_REPO}}\n                tag: ${{ matrix.version }}\n  test-e2e-grpc-java:\n    if: ${{ success() }}\n    name: Test E2E grpc java\n    needs: [list-version, deploy]\n    runs-on: ubuntu-latest\n    timeout-minutes: 60\n    strategy:\n      matrix:\n        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}\n    steps:\n      - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288\n        name: e2e test\n        with:\n          action: \"test\"\n          ask-config: \"${{ secrets.ASK_CONFIG_VIRGINA }}\"\n          test-version: \"${{ matrix.version }}\"\n          test-code-git: \"https://ghproxy.com/https://github.com/apache/rocketmq-e2e\"\n          test-code-branch: \"master\"\n          test-code-path: java/e2e\n          test-cmd: \"mvn -B test\"\n          job-id: 0\n      - name: Publish Test Report\n        uses: mikepenz/action-junit-report@v3\n        if: always() # always run even if the previous step fails\n        with:\n          report_paths: '**/test_report/TEST-*.xml'\n          annotate_only: true\n          include_passed: true\n          detailed_summary: true\n      - uses: actions/upload-artifact@v4\n        if: always()\n        name: Upload test log\n        with:\n          name: test-e2e-grpc-java-log.txt\n          path: testlog.txt\n          \n  test-e2e-golang:\n    if: ${{ success() }}\n    name: Test E2E golang\n    needs: [list-version, deploy]\n    runs-on: ubuntu-latest\n    timeout-minutes: 60\n    strategy:\n      matrix:\n        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}\n    steps:\n      - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288\n        name: e2e test\n        with:\n          action: \"test\"\n          ask-config: \"${{ secrets.ASK_CONFIG_VIRGINA }}\"\n          test-version: \"${{ matrix.version }}\"\n          test-code-git: \"https://ghproxy.com/https://github.com/apache/rocketmq-e2e\"\n          test-code-branch: \"master\"\n          test-code-path: golang\n          test-cmd: |\n            cd ../common &&  mvn -Prelease -DskipTests clean package -U\n            cd ../rocketmq-admintools && source bin/env.sh\n            LATEST_GO_VERSION=$(curl -s https://go.dev/VERSION?m=text | awk 'NR==1')\n            wget \"https://go.dev/dl/${LATEST_GO_VERSION}.linux-amd64.tar.gz\" && \\\n            rm -rf /usr/local/go && tar -C /usr/local -xzf ${LATEST_GO_VERSION}.linux-amd64.tar.gz\n            cd ../golang && go get -u github.com/apache/rocketmq-clients/golang && gotestsum --junitfile ./target/surefire-reports/TEST-report.xml ./mqgotest/... -timeout 2m  -v\n          job-id: 0\n      - name: Publish Test Report\n        uses: mikepenz/action-junit-report@v3\n        if: always() # always run even if the previous step fails\n        with:\n          report_paths: '**/test_report/TEST-*.xml'\n          annotate_only: true\n          include_passed: true\n          detailed_summary: true\n      - uses: actions/upload-artifact@v4\n        if: always()\n        name: Upload test log\n        with:\n          name: test-e2e-golang-log.txt\n          path: testlog.txt\n          \n  test-e2e-remoting-java:\n    if: ${{ success() }}\n    name: Test E2E remoting java\n    needs: [ list-version, deploy ]\n    runs-on: ubuntu-latest\n    timeout-minutes: 60\n    strategy:\n      matrix:\n        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}\n    steps:\n      - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288\n        name: e2e test\n        with:\n          action: \"test\"\n          ask-config: \"${{ secrets.ASK_CONFIG_VIRGINA }}\"\n          test-version: \"${{ matrix.version }}\"\n          test-code-git: \"https://ghproxy.com/https://github.com/apache/rocketmq-e2e\"\n          test-code-branch: \"master\"\n          test-code-path: java/e2e-v4\n          test-cmd: \"mvn -B test\"\n          job-id: 0\n      - name: Publish Test Report\n        uses: mikepenz/action-junit-report@v3\n        if: always() # always run even if the previous step fails\n        with:\n          report_paths: '**/test_report/TEST-*.xml'\n          annotate_only: true\n          include_passed: true\n          detailed_summary: true\n      - uses: actions/upload-artifact@v4\n        if: always()\n        name: Upload test log\n        with:\n          name: test-e2e-remoting-java-log.txt\n          path: testlog.txt\n\n  clean:\n    if: always()\n    name: Clean\n    needs: [list-version, test-e2e-grpc-java, test-e2e-golang, test-e2e-remoting-java]\n    runs-on: ubuntu-latest\n    timeout-minutes: 60\n    strategy:\n      matrix:\n        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}\n    steps:\n      - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288\n        name: clean\n        with:\n          action: \"clean\"\n          ask-config: \"${{ secrets.ASK_CONFIG_VIRGINA }}\"\n          test-version: \"${{ matrix.version }}\"\n          job-id: ${{ strategy.job-index }}"
  },
  {
    "path": ".github/workflows/push-ci.yml",
    "content": "name: PUSH-CI\n\non:\n  push:\n    branches: [master, develop]\n  #schedule:\n  #  - cron: \"0 18 * * *\" # TimeZone: UTC 0\n\nconcurrency:\n  group: rocketmq-${{ github.ref }}\n\nenv:\n  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120\n  DOCKER_REPO: apache/rocketmq-ci\n\njobs:\n  dist-tar:\n    if: github.repository == 'apache/rocketmq'\n    name: Build dist tar\n    runs-on: ubuntu-latest\n    timeout-minutes: 30\n    steps:\n      - uses: actions/checkout@v3\n        with:\n          submodules: true\n      - uses: actions/setup-java@v3\n        with:\n          distribution: \"temurin\"\n          java-version: \"8\"\n          cache: \"maven\"\n      - name: Build distribution tar\n        run: |\n          mvn -Prelease-all -DskipTests -Dspotbugs.skip=true clean install -U\n      - uses: actions/upload-artifact@v4\n        name: Upload distribution tar\n        with:\n          name: rocketmq\n          path: distribution/target/rocketmq*/rocketmq*\n\n  docker:\n    if: ${{ success() }}\n    name: Docker images\n    needs: [dist-tar]\n    runs-on: ubuntu-latest\n    timeout-minutes: 30\n    strategy:\n      matrix:\n        base-image: [\"ubuntu\"]\n        java-version: [\"8\"]\n    steps:\n      - uses: actions/checkout@v3\n        with:\n          repository: apache/rocketmq-docker.git\n          ref: master\n          path: rocketmq-docker\n      - uses: actions/download-artifact@v4\n        name: Download distribution tar\n        with:\n          name: rocketmq\n          path: rocketmq\n      - name: docker-login\n        uses: docker/login-action@v2\n        with:\n          registry: ${{ env.REGISTRY }}\n          username: ${{ secrets.DOCKERHUB_USER }}\n          password: ${{ secrets.DOCKERHUB_TOKEN }}\n      - name: Build and save docker images\n        id: build-images\n        run: |\n          cd rocketmq-docker/image-build-ci\n          version=${{ github.event.pull_request.number || github.ref_name }}-$(uuidgen)\n          mkdir versionlist\n          touch versionlist/\"${version}-`echo ${{ matrix.base-image }} | sed -e \"s/:/-/g\"`\"\n          sh ./build-image-local.sh ${version} ${{ matrix.base-image }} ${{ matrix.java-version }} ${DOCKER_REPO}\n      - uses: actions/upload-artifact@v4\n        name: Upload distribution tar\n        with:\n          name: versionlist\n          path: rocketmq-docker/image-build-ci/versionlist/*\n\n  list-version:\n    if: >\n      github.repository == 'apache/rocketmq' && \n      always()\n    name: List version\n    needs: [docker]\n    runs-on: ubuntu-latest\n    timeout-minutes: 30\n    outputs:\n      version-json: ${{ steps.show_versions.outputs.version-json }}\n    steps:\n      - uses: actions/download-artifact@v4\n        name: Download versionlist\n        with:\n          name: versionlist\n          path: versionlist\n      - name: Show versions\n        id: show_versions\n        run: | \n          a=(`ls versionlist`)\n          printf '%s\\n' \"${a[@]}\" | jq -R . | jq -s .\n          echo version-json=`printf '%s\\n' \"${a[@]}\" | jq -R . | jq -s .` >> $GITHUB_OUTPUT\n\n  deploy-e2e:\n    if: ${{ success() }}\n    name: Deploy RocketMQ For E2E\n    needs: [list-version,docker]\n    runs-on: ubuntu-latest\n    timeout-minutes: 60\n    strategy:\n      matrix:\n        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}\n    steps:\n      - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288\n        name: Deploy rocketmq\n        with:\n          action: \"deploy\"\n          ask-config: \"${{ secrets.ASK_CONFIG_VIRGINA }}\"\n          test-version: \"${{ matrix.version }}\"\n          chart-git: \"https://ghproxy.com/https://github.com/apache/rocketmq-docker.git\"\n          chart-branch: \"master\"\n          chart-path: \"./rocketmq-k8s-helm\"\n          job-id: ${{ strategy.job-index }}\n          helm-values: |\n            nameserver:\n              image:\n                repository: ${{env.DOCKER_REPO}}\n                tag: ${{ matrix.version }}\n            broker:\n              image:\n                repository: ${{env.DOCKER_REPO}}\n                tag: ${{ matrix.version }}\n            proxy:\n              image:\n                repository: ${{env.DOCKER_REPO}}\n                tag: ${{ matrix.version }}\n\n  deploy-benchmark:\n    if: ${{ success() }}\n    name: Deploy RocketMQ For Benchmarking\n    needs: [list-version,docker]\n    runs-on: ubuntu-latest\n    timeout-minutes: 60\n    strategy:\n      matrix:\n        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}\n    steps:\n      - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288\n        name: Deploy rocketmq\n        with:\n          action: \"deploy\"\n          ask-config: \"${{ secrets.ASK_CONFIG_VIRGINA }}\"\n          test-version: \"${{ matrix.version }}\"\n          chart-git: \"https://ghproxy.com/https://github.com/apache/rocketmq-docker.git\"\n          chart-branch: \"master\"\n          chart-path: \"./rocketmq-k8s-helm\"\n          job-id: \"001-${{ strategy.job-index }}\"\n          helm-values: |\n            nameserver:\n              image:\n                repository: ${{env.DOCKER_REPO}}\n                tag: ${{ matrix.version }}\n            broker:\n              image:\n                repository: ${{env.DOCKER_REPO}}\n                tag: ${{ matrix.version }}\n            proxy:\n              image:\n                repository: ${{env.DOCKER_REPO}}\n                tag: ${{ matrix.version }}\n\n  test-e2e-grpc-java:\n    if: ${{ success() }}\n    name: Test E2E grpc java\n    needs: [list-version, deploy-e2e]\n    runs-on: ubuntu-latest\n    timeout-minutes: 60\n    strategy:\n      matrix:\n        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}\n    steps:\n      - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288\n        name: e2e test\n        with:\n          action: \"test\"\n          ask-config: \"${{ secrets.ASK_CONFIG_VIRGINA }}\"\n          test-version: \"${{ matrix.version }}\"\n          test-code-git: \"https://ghproxy.com/https://github.com/apache/rocketmq-e2e\"\n          test-code-branch: \"master\"\n          test-code-path: java/e2e\n          test-cmd: \"mvn -B test\"\n          job-id: 0\n      - name: Publish Test Report\n        uses: mikepenz/action-junit-report@v3\n        if: always() # always run even if the previous step fails\n        with:\n          report_paths: '**/test_report/TEST-*.xml'\n          annotate_only: true\n          include_passed: true\n          detailed_summary: true\n      - uses: actions/upload-artifact@v4\n        if: always()\n        name: Upload test log\n        with:\n          name: test-e2e-grpc-java-log.txt\n          path: testlog.txt\n          \n  test-e2e-golang:\n    if: ${{ success() }}\n    name: Test E2E golang\n    needs: [list-version, deploy-e2e]\n    runs-on: ubuntu-latest\n    timeout-minutes: 60\n    strategy:\n      matrix:\n        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}\n    steps:\n      - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288\n        name: e2e test\n        with:\n          action: \"test\"\n          ask-config: \"${{ secrets.ASK_CONFIG_VIRGINA }}\"\n          test-version: \"${{ matrix.version }}\"\n          test-code-git: \"https://ghproxy.com/https://github.com/apache/rocketmq-e2e\"\n          test-code-branch: \"master\"\n          test-code-path: golang\n          test-cmd: |\n            cd ../common &&  mvn -Prelease -DskipTests clean package -U\n            cd ../rocketmq-admintools && source bin/env.sh\n            LATEST_GO_VERSION=$(curl -s https://go.dev/VERSION?m=text | awk 'NR==1')\n            wget \"https://go.dev/dl/${LATEST_GO_VERSION}.linux-amd64.tar.gz\" && \\\n            rm -rf /usr/local/go && tar -C /usr/local -xzf ${LATEST_GO_VERSION}.linux-amd64.tar.gz\n            cd ../golang && go get -u github.com/apache/rocketmq-clients/golang && gotestsum --junitfile ./target/surefire-reports/TEST-report.xml ./mqgotest/... -timeout 2m  -v\n          job-id: 0\n      - name: Publish Test Report\n        uses: mikepenz/action-junit-report@v3\n        if: always() # always run even if the previous step fails\n        with:\n          report_paths: '**/test_report/TEST-*.xml'\n          annotate_only: true\n          include_passed: true\n          detailed_summary: true\n      - uses: actions/upload-artifact@v4\n        if: always()\n        name: Upload test log\n        with:\n          name: test-e2e-golang-log.txt\n          path: testlog.txt\n          \n  test-e2e-remoting-java:\n    if: ${{ success() }}\n    name: Test E2E remoting java\n    needs: [ list-version, deploy-e2e ]\n    runs-on: ubuntu-latest\n    timeout-minutes: 60\n    strategy:\n      matrix:\n        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}\n    steps:\n      - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288\n        name: e2e test\n        with:\n          action: \"test\"\n          ask-config: \"${{ secrets.ASK_CONFIG_VIRGINA }}\"\n          test-version: \"${{ matrix.version }}\"\n          test-code-git: \"https://ghproxy.com/https://github.com/apache/rocketmq-e2e\"\n          test-code-branch: \"master\"\n          test-code-path: java/e2e-v4\n          test-cmd: \"mvn -B test\"\n          job-id: 0\n      - name: Publish Test Report\n        uses: mikepenz/action-junit-report@v3\n        if: always() # always run even if the previous step fails\n        with:\n          report_paths: '**/test_report/TEST-*.xml'\n          annotate_only: true\n          include_passed: true\n          detailed_summary: true\n      - uses: actions/upload-artifact@v4\n        if: always()\n        name: Upload test log\n        with:\n          name: test-e2e-remoting-java-log.txt\n          path: testlog.txt\n\n  benchmark-test:\n    if: ${{ success() }}\n    runs-on: ubuntu-latest\n    name: Performance benchmark test\n    needs: [ list-version, deploy-benchmark ]\n    timeout-minutes: 60\n    steps:\n      - uses: apache/rocketmq-test-tool/benchmark-runner@ce372e5f3906ca1891e4918b05be14608eae608e\n        name: Performance benchmark\n        with:\n          action: \"performance-benchmark\"\n          ask-config: \"${{ secrets.ASK_CONFIG_VIRGINA }}\"\n          job-id: \"001-${{ strategy.job-index }}\"\n          # The time to run the test, 15 minutes\n          test-time: \"900\"\n          # Some thresholds set in advance\n          min-send-tps-threshold: \"12000\"\n          max-rt-ms-threshold: \"500\"\n          avg-rt-ms-threshold: \"10\"\n          max-2c-rt-ms-threshold: \"150\"\n          avg-2c-rt-ms-threshold: \"10\"\n      - name: Upload test report\n        if: always()\n        uses: actions/upload-artifact@v4\n        with:\n          name: benchmark-report\n          path: benchmark/\n\n  clean-e2e:\n    if: always()\n    name: Clean E2E\n    needs: [ list-version, test-e2e-grpc-java, test-e2e-golang, test-e2e-remoting-java ]\n    runs-on: ubuntu-latest\n    timeout-minutes: 60\n    strategy:\n      matrix:\n        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}\n    steps:\n      - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288\n        name: clean\n        with:\n          action: \"clean\"\n          ask-config: \"${{ secrets.ASK_CONFIG_VIRGINA }}\"\n          test-version: \"${{ matrix.version }}\"\n          job-id: ${{ strategy.job-index }}\n          \n  clean-benchmark:\n    if: always()\n    name: Clean Benchmarking\n    needs: [ list-version, benchmark-test ]\n    runs-on: ubuntu-latest\n    timeout-minutes: 60\n    strategy:\n      matrix:\n        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}\n    steps:\n      - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288\n        name: clean\n        with:\n          action: \"clean\"\n          ask-config: \"${{ secrets.ASK_CONFIG_VIRGINA }}\"\n          test-version: \"${{ matrix.version }}\"\n          job-id: \"001-${{ strategy.job-index }}\""
  },
  {
    "path": ".github/workflows/rerun-workflow.yml",
    "content": "name: Rerun workflow\non:\n  workflow_run:\n    workflows: [\"Build and Run Tests by Maven\" , \"Build and Run Tests by Bazel\"]\n    types:\n      - completed\n\npermissions:\n  actions: write\n\njobs:\n  rerun:\n    if: github.event.workflow_run.conclusion == 'failure' && fromJSON(github.event.workflow_run.run_attempt) < 3\n    runs-on: ubuntu-latest\n    steps:\n      - name: rerun ${{  github.event.workflow_run.id }}\n        env:\n          GH_REPO:  ${{ github.repository }}\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n        run: |\n          gh run watch ${{ github.event.workflow_run.id }} > /dev/null 2>&1\n          gh run rerun ${{ github.event.workflow_run.id }} --failed"
  },
  {
    "path": ".github/workflows/snapshot-automation.yml",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nname: Snapshot Daily Release Automation\non:\n  schedule: # schedule the job to run at 12 a.m. daily\n    - cron: \"0 0 * * *\"\n  workflow_dispatch:\n    inputs:\n      branch:\n        description: 'The branch to trigger the workflow, The default branch is \"develop\" when both branch and commit_id are empty'\n        required: false\n      commit_id:\n        description: 'The commit id to trigger the workflow. Do not set branch and commit_id together'\n        required: false\n      rocketmq_version:\n        description: 'Name of the SNAPSHOT version to be generated. The default version is \"$VERSION-stable-SNAPSHOT\"'\n        required: false\n\nenv:\n  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120\n  DOCKER_REPO: apache/rocketmq-ci\n\njobs:\n  dist-tar:\n    if: github.repository == 'apache/rocketmq'\n    name: Build dist tar\n    runs-on: ubuntu-latest\n    timeout-minutes: 30\n    steps:\n      - name: Checkout develop\n        if: github.event.inputs.branch == '' && github.event.inputs.commit_id == ''\n        uses: actions/checkout@v3\n        with:\n          ref: develop\n\n      - name: Checkout specific commit\n        if: github.event.inputs.branch == '' && github.event.inputs.commit_id != ''\n        uses: actions/checkout@v3\n        with:\n          ref: ${{ github.event.inputs.commit_id }}\n\n      - name: Checkout specific branch\n        if: github.event.inputs.branch != '' && github.event.inputs.commit_id == ''\n        uses: actions/checkout@v3\n        with:\n          ref: ${{ github.event.inputs.branch }}\n\n      - uses: actions/setup-java@v4\n        with:\n          distribution: \"corretto\"\n          java-version: \"8\"\n          cache: \"maven\"\n      - name: Build distribution tar\n        env:\n          MAVEN_SETTINGS: ${{ github.workspace }}/.github/asf-deploy-settings.xml\n        run: |\n          mvn -Prelease-all -DskipTests -Dspotbugs.skip=true clean install -U\n      - uses: actions/upload-artifact@v4\n        name: Upload distribution tar\n        with:\n          name: rocketmq\n          path: distribution/target/rocketmq*/rocketmq*\n\n  docker-build:\n    if: ${{ success() }}\n    name: Docker images\n    needs: [ dist-tar ]\n    runs-on: ubuntu-latest\n    timeout-minutes: 30\n    strategy:\n      matrix:\n        base-image: [ \"ubuntu\" ]\n        java-version: [ \"8\" ]\n    steps:\n      - uses: actions/checkout@v3\n        with:\n          repository: apache/rocketmq-docker.git\n          ref: master\n          path: rocketmq-docker\n      - uses: actions/download-artifact@v4\n        name: Download distribution tar\n        with:\n          name: rocketmq\n          path: rocketmq\n      - name: docker-login\n        uses: docker/login-action@v2\n        with:\n          registry: ${{ env.REGISTRY }}\n          username: ${{ secrets.DOCKERHUB_USER }}\n          password: ${{ secrets.DOCKERHUB_TOKEN }}\n      - name: Build and save docker images\n        id: build-images\n        run: |\n          cd rocketmq-docker/image-build-ci\n          version=${{ github.event.pull_request.number || github.ref_name }}-$(uuidgen)\n          mkdir versionlist\n          touch versionlist/\"${version}-`echo ${{ matrix.base-image }} | sed -e \"s/:/-/g\"`\"\n          sh ./build-image-local.sh ${version} ${{ matrix.base-image }} ${{ matrix.java-version }} ${DOCKER_REPO}\n      - uses: actions/upload-artifact@v4\n        name: Upload distribution tar\n        with:\n          name: versionlist\n          path: rocketmq-docker/image-build-ci/versionlist/*\n\n  list-version:\n    if: >\n      github.repository == 'apache/rocketmq' && \n      always()\n    name: List version\n    needs: [ docker-build ]\n    runs-on: ubuntu-latest\n    timeout-minutes: 30\n    outputs:\n      version-json: ${{ steps.show_versions.outputs.version-json }}\n    steps:\n      - uses: actions/download-artifact@v4\n        name: Download versionlist\n        with:\n          name: versionlist\n          path: versionlist\n      - name: Show versions\n        id: show_versions\n        run: |\n          a=(`ls versionlist`)\n          printf '%s\\n' \"${a[@]}\" | jq -R . | jq -s .\n          echo version-json=`printf '%s\\n' \"${a[@]}\" | jq -R . | jq -s .` >> $GITHUB_OUTPUT\n\n  deploy-rocketmq:\n    if: ${{ success() }}\n    name: Deploy RocketMQ\n    needs: [ list-version,docker-build ]\n    runs-on: ubuntu-latest\n    timeout-minutes: 60\n    strategy:\n      matrix:\n        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}\n    steps:\n      - uses: apache/rocketmq-test-tool@1a646589accad17070423eabf0f54925e52b0666\n        name: Deploy rocketmq\n        with:\n          action: \"deploy\"\n          ask-config: \"${{ secrets.ASK_CONFIG_VIRGINA }}\"\n          test-version: \"${{ matrix.version }}\"\n          chart-git: \"https://github.com/apache/rocketmq-docker.git\"\n          chart-branch: \"master\"\n          chart-path: \"./rocketmq-k8s-helm\"\n          job-id: ${{ strategy.job-index }}\n          helm-values: |\n            nameserver:\n              image:\n                repository: ${{env.DOCKER_REPO}}\n                tag: ${{ matrix.version }}\n            broker:\n              image:\n                repository: ${{env.DOCKER_REPO}}\n                tag: ${{ matrix.version }}\n            proxy:\n              image:\n                repository: ${{env.DOCKER_REPO}}\n                tag: ${{ matrix.version }}\n\n  java-grpc-e2e-test:\n    if: ${{ success() }}\n    name: E2E Test\n    needs: [ list-version, deploy-rocketmq ]\n    runs-on: ubuntu-latest\n    timeout-minutes: 60\n    strategy:\n      matrix:\n        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}\n    steps:\n      - uses: apache/rocketmq-test-tool@1a646589accad17070423eabf0f54925e52b0666\n        name: e2e test\n        with:\n          action: \"test\"\n          ask-config: \"${{ secrets.ASK_CONFIG_VIRGINA }}\"\n          test-version: \"${{ matrix.version }}\"\n          test-code-git: \"https://github.com/apache/rocketmq-e2e.git\"\n          test-code-branch: \"master\"\n          test-code-path: java/e2e\n          test-cmd: \"mvn -B test\"\n          job-id: ${{ strategy.job-index }}\n      - name: Publish Test Report\n        uses: mikepenz/action-junit-report@v3\n        if: always() # always run even if the previous step fails\n        with:\n          report_paths: '**/test_report/TEST-*.xml'\n          annotate_only: true\n          include_passed: true\n          detailed_summary: true\n      - uses: actions/upload-artifact@v4\n        if: always()\n        name: Upload test log\n        with:\n          name: testlog.txt\n          path: testlog.txt\n\n  clean:\n    if: always()\n    name: Clean\n    needs: [ list-version, java-grpc-e2e-test ]\n    runs-on: ubuntu-latest\n    timeout-minutes: 60\n    strategy:\n      matrix:\n        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}\n    steps:\n      - uses: apache/rocketmq-test-tool@1a646589accad17070423eabf0f54925e52b0666\n        name: clean\n        with:\n          action: \"clean\"\n          ask-config: \"${{ secrets.ASK_CONFIG_VIRGINA }}\"\n          test-version: \"${{ matrix.version }}\"\n          job-id: ${{ strategy.job-index }}\n\n  snapshot:\n    runs-on: ubuntu-latest\n    needs: [ java-grpc-e2e-test ]\n    env:\n      NEXUS_DEPLOY_USERNAME: ${{ secrets.NEXUS_USER }}\n      NEXUS_DEPLOY_PASSWORD: ${{ secrets.NEXUS_PW }}\n    steps:\n      - name: Checkout\n        uses: actions/checkout@v3\n        with:\n          ref: develop\n          persist-credentials: false\n      - name: Set up JDK\n        uses: actions/setup-java@v4\n        with:\n          java-version: 8\n          distribution: \"corretto\"\n          cache: \"maven\"\n      - name: Update default pom version\n        if: github.event.inputs.rocketmq_version == ''\n        run: |\n          VERSION=$(mvn -q -Dexec.executable='echo' -Dexec.args='${project.version}' --non-recursive exec:exec)\n          VERSION=$(echo $VERSION | awk -F '-SNAPSHOT' '{print $1}')\n          VERSION=$VERSION-stable-SNAPSHOT\n          mvn versions:set -DnewVersion=$VERSION\n      - name: Update User-defined pom version\n        if: github.event.inputs.rocketmq_version != ''\n        run: |\n          mvn versions:set -DnewVersion=${{ github.event.inputs.rocketmq_version }}\n      - name: Deploy to ASF Snapshots Repository\n        timeout-minutes: 40\n        run: mvn clean deploy -DskipTests=true --settings .github/asf-deploy-settings.xml\n"
  },
  {
    "path": ".github/workflows/stale.yml",
    "content": "name: Close Stale Issues/PRs\n\npermissions:\n  issues: write\n  pull-requests: write\n\non:\n  workflow_dispatch:\n  schedule:\n    - cron: \"0 0 * * *\"\n\njobs:\n  stale:\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/stale@v5\n        with:\n          operations-per-run: 128\n          days-before-issue-stale: 365\n          days-before-issue-close: 3\n          exempt-issue-labels: \"no stale\"\n          stale-issue-label: \"stale\"\n          stale-issue-message: \"This issue is stale because it has been open for 365 days with no activity. It will be closed in 3 days if no further activity occurs.\"\n          close-issue-message: \"This issue was closed because it has been inactive for 3 days since being marked as stale.\"\n          remove-issue-stale-when-updated: true\n          days-before-pr-stale: 365\n          days-before-pr-close: 3\n          exempt-pr-labels: \"no stale\"\n          stale-pr-label: \"stale\"\n          stale-pr-message: \"This PR is stale because it has been open for 365 days with no activity. It will be closed in 3 days if no further activity occurs. If you wish not to mark it as stale, please leave a comment in this PR.\"\n          close-pr-message: \"This PR was closed because it has been inactive for 3 days since being marked as stale.\"\n          remove-pr-stale-when-updated: true\n"
  },
  {
    "path": ".gitignore",
    "content": "*dependency-reduced-pom.xml\r\n.classpath\r\n.project\r\n.settings/\r\ntarget/\r\ndevenv\r\n*.log.*\r\n*.iml\r\n.idea/\r\n*.versionsBackup\r\n!NOTICE-BIN\r\n!LICENSE-BIN\r\n.DS_Store\r\nlocalbin\r\nnohup.out\r\nbazel-out\r\nbazel-bin\r\nbazel-rocketmq\r\nbazel-testlogs\r\n.vscode\r\nMODULE.bazel.lock\r\n*.flattened-pom.xml\r\n"
  },
  {
    "path": ".licenserc.yaml",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\nheader:\n  license:\n    spdx-id: Apache-2.0\n    copyright-owner: Apache Software Foundation\n\n  paths-ignore:\n    - '.gitignore'\n    - '.travis.yml'\n    - 'CONTRIBUTING.md'\n    - 'LICENSE'\n    - 'NOTICE'\n    - '**/*.md'\n    - 'BUILDING'\n    - '.github/**'\n    - '*/src/test/resources/certs/*'\n    - 'src/test/**/*.log'\n    - '*/src/test/resources/META-INF/service/*'\n    - '*/src/main/resources/META-INF/service/*'\n    - '*/src/test/resources/rmq-proxy-home/conf/rmq-proxy.json'\n    - '*/src/test/resources/mockito-extensions/*'\n    - '**/target/**'\n    - '**/*.iml'\n    - 'docs/**'\n    - 'localbin/**'\n    - 'distribution/LICENSE-BIN'\n    - 'distribution/NOTICE-BIN'\n    - 'distribution/conf/rmq-proxy.json'\n    - '.bazelversion'\n    - 'common/src/main/resources/META-INF/services/org.apache.rocketmq.logging.ch.qos.logback.classic.spi.Configurator'\n\n\n  comment: on-failure"
  },
  {
    "path": "BUILD.bazel",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\njava_library(\n    name = \"test_deps\",\n    visibility = [\"//visibility:public\"],\n    exports = [\n        \"@maven//:junit_junit\",\n        \"@maven//:org_assertj_assertj_core\",\n        \"@maven//:org_hamcrest_hamcrest_library\",\n        \"@maven//:org_mockito_mockito_core\",\n        \"@maven//:org_powermock_powermock_module_junit4\",\n        \"@maven//:org_powermock_powermock_api_mockito2\",\n        \"@maven//:org_hamcrest_hamcrest_core\",\n        \"@maven//:ch_qos_logback_logback_classic\",\n        \"@maven//:org_awaitility_awaitility\",\n        \"@maven//:org_openjdk_jmh_jmh_core\",\n        \"@maven//:org_openjdk_jmh_jmh_generator_annprocess\",\n        \"@maven//:org_mockito_mockito_junit_jupiter\",\n    ],\n)\n"
  },
  {
    "path": "BUILDING",
    "content": "Build Instructions for Apache RocketMQ\n\n====================================================\n\n(1) Prerequisites\n\n    JDK 1.8+ is required in order to compile and run RocketMQ.\n\n    RocketMQ utilizes Maven as a distribution management and packaging tool. Version 3.0.3 or later is required.\n    Maven installation and configuration instructions can be found here:\n\n    http://maven.apache.org/run-maven/index.html\n\n\n(2) Run test cases\n\n    Execute the following command in order to compile and run test cases of each components:\n\n    $ mvn test\n\n\n(3) Import projects to Eclipse IDE\n\n    First, generate eclipse project files:\n\n    $ mvn -U eclipse:eclipse\n\n    Then, import to eclipse by specifying the root directory of the project via:\n\n    [File] > [Import] > [Existing Projects into Workspace].\n\n\n(4) Build distribution packages\n\n    Execute the following command in order to build the tar.gz packages and install JAR into local repository:\n\n    $ mvn -Prelease-all -DskipTests clean install -U"
  },
  {
    "path": "CONTRIBUTING.md",
    "content": "## How To Contribute\n\nWe are always very happy to have contributions, whether for trivial cleanups or big new features.\nWe want to have high quality, well documented codes for each programming language, as well as the surrounding [ecosystem](https://github.com/apache/rocketmq-externals) of integration tools that people use with RocketMQ.\n\nNor is code the only way to contribute to the project. We strongly value documentation, integration with other projects, and gladly accept improvements for these aspects.\n\nRecommend reading:\n * [Contributors Tech Guide](http://www.apache.org/dev/contributors)\n * [Get involved!](http://www.apache.org/foundation/getinvolved.html)\n\n## Contributing code\n\nTo submit a change for inclusion, please do the following:\n\n#### If the change is non-trivial please include some unit tests that cover the new functionality.\n#### If you are introducing a completely new feature or API it is a good idea to start a [RIP](https://github.com/apache/rocketmq/wiki/RocketMQ-Improvement-Proposal) and get consensus on the basic design first.\n#### It is our job to follow up on patches in a timely fashion. Nag us if we aren't doing our job (sometimes we drop things).\n\n### Squash commits\n\nIf you have a pull request on GitHub, and updated more than once, it's better to squash all commits.\n\n1. Identify how many commits you made since you began: ``git log``.\n2. Squash these commits by N: ``git rebase -i HEAD~N`` .\n3. Leave \"pick\" tag in the first line.\n4. Change all other commits from \"pick\" to \"fixup\".\n5. Then do \"force push\" to overwrite remote history: ``git push -u origin ROCKETMQ-9999 --force``\n6. All your changes are now in a single commit, that would be much better for review.\n\nMore details of squash can be found at [stackoverflow](https://stackoverflow.com/questions/5189560/squash-my-last-x-commits-together-using-git).\n\n## Becoming a Committer\n\nWe are always interested in adding new contributors. What we look for are series of contributions, good taste and ongoing interest in the project. If you are interested in becoming a committer, please let one of the existing committers know and they can help you walk through the process.\n\nNowadays, we have several important contribution points:\n#### Wiki & JavaDoc\n#### RocketMQ SDK(C++\\.Net\\Php\\Python\\Go\\Node.js)\n#### RocketMQ Connectors\n\n##### Prerequisites\nIf you want to contribute to the above listed points, you must abide by the following prerequisites:\n\n###### Readability - API must have Javadoc, and some very important methods must also have Javadoc\n###### Testability - Above 80% unit test coverage for the main process\n###### Maintainability - Comply with our [checkstyle spec](style/rmq_checkstyle.xml), and at least a 3-month update frequency\n###### Deployability - We encourage you to deploy into [maven repository](http://search.maven.org/)\n"
  },
  {
    "path": "LICENSE",
    "content": "                                 Apache License\n                           Version 2.0, January 2004\n                        http://www.apache.org/licenses/\n\n   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n   1. Definitions.\n\n      \"License\" shall mean the terms and conditions for use, reproduction,\n      and distribution as defined by Sections 1 through 9 of this document.\n\n      \"Licensor\" shall mean the copyright owner or entity authorized by\n      the copyright owner that is granting the License.\n\n      \"Legal Entity\" shall mean the union of the acting entity and all\n      other entities that control, are controlled by, or are under common\n      control with that entity. For the purposes of this definition,\n      \"control\" means (i) the power, direct or indirect, to cause the\n      direction or management of such entity, whether by contract or\n      otherwise, or (ii) ownership of fifty percent (50%) or more of the\n      outstanding shares, or (iii) beneficial ownership of such entity.\n\n      \"You\" (or \"Your\") shall mean an individual or Legal Entity\n      exercising permissions granted by this License.\n\n      \"Source\" form shall mean the preferred form for making modifications,\n      including but not limited to software source code, documentation\n      source, and configuration files.\n\n      \"Object\" form shall mean any form resulting from mechanical\n      transformation or translation of a Source form, including but\n      not limited to compiled object code, generated documentation,\n      and conversions to other media types.\n\n      \"Work\" shall mean the work of authorship, whether in Source or\n      Object form, made available under the License, as indicated by a\n      copyright notice that is included in or attached to the work\n      (an example is provided in the Appendix below).\n\n      \"Derivative Works\" shall mean any work, whether in Source or Object\n      form, that is based on (or derived from) the Work and for which the\n      editorial revisions, annotations, elaborations, or other modifications\n      represent, as a whole, an original work of authorship. For the purposes\n      of this License, Derivative Works shall not include works that remain\n      separable from, or merely link (or bind by name) to the interfaces of,\n      the Work and Derivative Works thereof.\n\n      \"Contribution\" shall mean any work of authorship, including\n      the original version of the Work and any modifications or additions\n      to that Work or Derivative Works thereof, that is intentionally\n      submitted to Licensor for inclusion in the Work by the copyright owner\n      or by an individual or Legal Entity authorized to submit on behalf of\n      the copyright owner. For the purposes of this definition, \"submitted\"\n      means any form of electronic, verbal, or written communication sent\n      to the Licensor or its representatives, including but not limited to\n      communication on electronic mailing lists, source code control systems,\n      and issue tracking systems that are managed by, or on behalf of, the\n      Licensor for the purpose of discussing and improving the Work, but\n      excluding communication that is conspicuously marked or otherwise\n      designated in writing by the copyright owner as \"Not a Contribution.\"\n\n      \"Contributor\" shall mean Licensor and any individual or Legal Entity\n      on behalf of whom a Contribution has been received by Licensor and\n      subsequently incorporated within the Work.\n\n   2. Grant of Copyright License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      copyright license to reproduce, prepare Derivative Works of,\n      publicly display, publicly perform, sublicense, and distribute the\n      Work and such Derivative Works in Source or Object form.\n\n   3. Grant of Patent License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      (except as stated in this section) patent license to make, have made,\n      use, offer to sell, sell, import, and otherwise transfer the Work,\n      where such license applies only to those patent claims licensable\n      by such Contributor that are necessarily infringed by their\n      Contribution(s) alone or by combination of their Contribution(s)\n      with the Work to which such Contribution(s) was submitted. If You\n      institute patent litigation against any entity (including a\n      cross-claim or counterclaim in a lawsuit) alleging that the Work\n      or a Contribution incorporated within the Work constitutes direct\n      or contributory patent infringement, then any patent licenses\n      granted to You under this License for that Work shall terminate\n      as of the date such litigation is filed.\n\n   4. Redistribution. You may reproduce and distribute copies of the\n      Work or Derivative Works thereof in any medium, with or without\n      modifications, and in Source or Object form, provided that You\n      meet the following conditions:\n\n      (a) You must give any other recipients of the Work or\n          Derivative Works a copy of this License; and\n\n      (b) You must cause any modified files to carry prominent notices\n          stating that You changed the files; and\n\n      (c) You must retain, in the Source form of any Derivative Works\n          that You distribute, all copyright, patent, trademark, and\n          attribution notices from the Source form of the Work,\n          excluding those notices that do not pertain to any part of\n          the Derivative Works; and\n\n      (d) If the Work includes a \"NOTICE\" text file as part of its\n          distribution, then any Derivative Works that You distribute must\n          include a readable copy of the attribution notices contained\n          within such NOTICE file, excluding those notices that do not\n          pertain to any part of the Derivative Works, in at least one\n          of the following places: within a NOTICE text file distributed\n          as part of the Derivative Works; within the Source form or\n          documentation, if provided along with the Derivative Works; or,\n          within a display generated by the Derivative Works, if and\n          wherever such third-party notices normally appear. The contents\n          of the NOTICE file are for informational purposes only and\n          do not modify the License. You may add Your own attribution\n          notices within Derivative Works that You distribute, alongside\n          or as an addendum to the NOTICE text from the Work, provided\n          that such additional attribution notices cannot be construed\n          as modifying the License.\n\n      You may add Your own copyright statement to Your modifications and\n      may provide additional or different license terms and conditions\n      for use, reproduction, or distribution of Your modifications, or\n      for any such Derivative Works as a whole, provided Your use,\n      reproduction, and distribution of the Work otherwise complies with\n      the conditions stated in this License.\n\n   5. Submission of Contributions. Unless You explicitly state otherwise,\n      any Contribution intentionally submitted for inclusion in the Work\n      by You to the Licensor shall be under the terms and conditions of\n      this License, without any additional terms or conditions.\n      Notwithstanding the above, nothing herein shall supersede or modify\n      the terms of any separate license agreement you may have executed\n      with Licensor regarding such Contributions.\n\n   6. Trademarks. This License does not grant permission to use the trade\n      names, trademarks, service marks, or product names of the Licensor,\n      except as required for reasonable and customary use in describing the\n      origin of the Work and reproducing the content of the NOTICE file.\n\n   7. Disclaimer of Warranty. Unless required by applicable law or\n      agreed to in writing, Licensor provides the Work (and each\n      Contributor provides its Contributions) on an \"AS IS\" BASIS,\n      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n      implied, including, without limitation, any warranties or conditions\n      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n      PARTICULAR PURPOSE. You are solely responsible for determining the\n      appropriateness of using or redistributing the Work and assume any\n      risks associated with Your exercise of permissions under this License.\n\n   8. Limitation of Liability. In no event and under no legal theory,\n      whether in tort (including negligence), contract, or otherwise,\n      unless required by applicable law (such as deliberate and grossly\n      negligent acts) or agreed to in writing, shall any Contributor be\n      liable to You for damages, including any direct, indirect, special,\n      incidental, or consequential damages of any character arising as a\n      result of this License or out of the use or inability to use the\n      Work (including but not limited to damages for loss of goodwill,\n      work stoppage, computer failure or malfunction, or any and all\n      other commercial damages or losses), even if such Contributor\n      has been advised of the possibility of such damages.\n\n   9. Accepting Warranty or Additional Liability. While redistributing\n      the Work or Derivative Works thereof, You may choose to offer,\n      and charge a fee for, acceptance of support, warranty, indemnity,\n      or other liability obligations and/or rights consistent with this\n      License. However, in accepting such obligations, You may act only\n      on Your own behalf and on Your sole responsibility, not on behalf\n      of any other Contributor, and only if You agree to indemnify,\n      defend, and hold each Contributor harmless for any liability\n      incurred by, or claims asserted against, such Contributor by reason\n      of your accepting any such warranty or additional liability.\n\n   END OF TERMS AND CONDITIONS\n\n   APPENDIX: How to apply the Apache License to your work.\n\n      To apply the Apache License to your work, attach the following\n      boilerplate notice, with the fields enclosed by brackets \"[]\"\n      replaced with your own identifying information. (Don't include\n      the brackets!)  The text should be enclosed in the appropriate\n      comment syntax for the file format. We also recommend that a\n      file or class name and description of purpose be included on the\n      same \"printed page\" as the copyright notice for easier\n      identification within third-party archives.\n\n   Copyright [yyyy] [name of copyright owner]\n\n   Licensed under the Apache License, Version 2.0 (the \"License\");\n   you may not use this file except in compliance with the License.\n   You may obtain a copy of the License at\n\n       http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing, software\n   distributed under the License is distributed on an \"AS IS\" BASIS,\n   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n   See the License for the specific language governing permissions and\n   limitations under the License."
  },
  {
    "path": "MODULE.bazel",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n###############################################################################\n# Bazel now uses Bzlmod by default to manage external dependencies.\n# Please consider migrating your external dependencies from WORKSPACE to MODULE.bazel.\n#\n# For more details, please check https://github.com/bazelbuild/bazel/issues/18958\n###############################################################################\n"
  },
  {
    "path": "NOTICE",
    "content": "Apache RocketMQ\nCopyright 2016-2026 The Apache Software Foundation\n\nThis product includes software developed at\nThe Apache Software Foundation (http://www.apache.org/).\n"
  },
  {
    "path": "README.md",
    "content": "## Apache RocketMQ\n\n[![Build Status][maven-build-image]][maven-build-url]\n[![CodeCov][codecov-image]][codecov-url]\n[![Maven Central][maven-central-image]][maven-central-url]\n[![Release][release-image]][release-url]\n[![License][license-image]][license-url]\n[![Average Time to Resolve An Issue][average-time-to-resolve-an-issue-image]][average-time-to-resolve-an-issue-url]\n[![Percentage of Issues Still Open][percentage-of-issues-still-open-image]][percentage-of-issues-still-open-url]\n[![Twitter Follow][twitter-follow-image]][twitter-follow-url]\n\n**[Apache RocketMQ](https://rocketmq.apache.org) is a distributed messaging and streaming platform with low latency, high performance and reliability, trillion-level capacity and flexible scalability.**\n\n\nIt offers a variety of features:\n\n* Messaging patterns including publish/subscribe, request/reply and streaming\n* Financial grade transactional message\n* Built-in fault tolerance and high availability configuration options based on [DLedger Controller](docs/en/controller/quick_start.md)\n* Built-in message tracing capability, also supports opentracing\n* Versatile big-data and streaming ecosystem integration\n* Message retroactivity by time or offset\n* Reliable FIFO and strict ordered messaging in the same queue\n* Efficient pull and push consumption model\n* Million-level message accumulation capacity in a single queue\n* Multiple messaging protocols like gRPC, MQTT, JMS and OpenMessaging\n* Flexible distributed scale-out deployment architecture\n* Lightning-fast batch message exchange system\n* Various message filter mechanics such as SQL and Tag\n* Docker images for isolated testing and cloud isolated clusters\n* Feature-rich administrative dashboard for configuration, metrics and monitoring\n* Authentication and authorization\n* Free open source connectors, for both sources and sinks\n* Lightweight real-time computing\n----------\n\n\n## Quick Start\n\nThis paragraph guides you through steps of installing RocketMQ in different ways.\nFor local development and testing, only one instance will be created for each component.\n\n### Run RocketMQ locally\n\nRocketMQ runs on all major operating systems and requires only a Java JDK version 8 or higher to be installed.\nTo check, run `java -version`:\n```shell\n$ java -version\njava version \"1.8.0_121\"\n```\n\nFor Windows users, click [here](https://dist.apache.org/repos/dist/release/rocketmq/5.4.0/rocketmq-all-5.4.0-bin-release.zip) to download the 5.4.0 RocketMQ binary release,\nunpack it to your local disk, such as `D:\\rocketmq`.\nFor macOS and Linux users, execute following commands:\n\n```shell\n# Download release from the Apache mirror\n$ wget https://dist.apache.org/repos/dist/release/rocketmq/5.4.0/rocketmq-all-5.4.0-bin-release.zip\n\n# Unpack the release\n$ unzip rocketmq-all-5.4.0-bin-release.zip\n```\n\nPrepare a terminal and change to the extracted `bin` directory:\n```shell\n$ cd rocketmq-all-5.4.0-bin-release/bin\n```\n\n**1) Start NameServer**\n\nNameServer will be listening at `0.0.0.0:9876`, make sure that the port is not used by others on the local machine, and then do as follows.\n\nFor macOS and Linux users:\n```shell\n### start Name Server\n$ nohup sh mqnamesrv &\n\n### check whether Name Server is successfully started\n$ tail -f ~/logs/rocketmqlogs/namesrv.log\nThe Name Server boot success...\n```\n\nFor Windows users, you need to set environment variables first:\n- From the desktop, right click the Computer icon.\n- Choose Properties from the context menu.\n- Click the Advanced system settings link.\n- Click Environment Variables.\n- Add Environment `ROCKETMQ_HOME=\"D:\\rocketmq\"`. \n\nThen change directory to rocketmq, type and run:\n```shell\n$ mqnamesrv.cmd\nThe Name Server boot success...\n```\n\n**2) Start Broker**\n\nFor macOS and Linux users:\n```shell\n### start Broker\n$ nohup sh mqbroker -n localhost:9876 &\n\n### check whether Broker is successfully started, eg: Broker's IP is 192.168.1.2, Broker's name is broker-a\n$ tail -f ~/logs/rocketmqlogs/broker.log\nThe broker[broker-a, 192.168.1.2:10911] boot success...\n```\n\nFor Windows users:\n```shell\n$ mqbroker.cmd -n localhost:9876\nThe broker[broker-a, 192.168.1.2:10911] boot success...\n```\n\n### Run RocketMQ in Docker\n\nYou can run RocketMQ on your own machine within Docker containers,\n`host` network will be used to expose listening port in the container.\n\n**1) Start NameServer**\n\n```shell\n$ docker run -it --net=host apache/rocketmq ./mqnamesrv\n```\n\n**2) Start Broker**\n\n```shell\n$ docker run -it --net=host --mount type=bind,source=/tmp/store,target=/home/rocketmq/store apache/rocketmq ./mqbroker -n localhost:9876\n```\n\n### Run RocketMQ in Kubernetes\n\nYou can also run a RocketMQ cluster within a Kubernetes cluster using [RocketMQ Operator](https://github.com/apache/rocketmq-operator).\nBefore your operations, make sure that `kubectl` and related kubeconfig file installed on your machine.\n\n**1) Install CRDs**\n```shell\n### install CRDs\n$ git clone https://github.com/apache/rocketmq-operator\n$ cd rocketmq-operator && make deploy\n\n### check whether CRDs are successfully installed\n$ kubectl get crd | grep rocketmq.apache.org\nbrokers.rocketmq.apache.org                 2022-05-12T09:23:18Z\nconsoles.rocketmq.apache.org                2022-05-12T09:23:19Z\nnameservices.rocketmq.apache.org            2022-05-12T09:23:18Z\ntopictransfers.rocketmq.apache.org          2022-05-12T09:23:19Z\n\n### check whether operator is running\n$ kubectl get pods | grep rocketmq-operator\nrocketmq-operator-6f65c77c49-8hwmj   1/1     Running   0          93s\n```\n\n**2) Create Cluster Instance**\n```shell\n### create RocketMQ cluster resource\n$ cd example && kubectl create -f rocketmq_v1alpha1_rocketmq_cluster.yaml\n\n### check whether cluster resources are running\n$ kubectl get sts\nNAME                 READY   AGE\nbroker-0-master      1/1     107m\nbroker-0-replica-1   1/1     107m\nname-service         1/1     107m\n```\n\n---\n## Apache RocketMQ Community\n* [RocketMQ Streams](https://github.com/apache/rocketmq-streams): A lightweight stream computing engine based on Apache RocketMQ.\n* [RocketMQ Flink](https://github.com/apache/rocketmq-flink): The Apache RocketMQ connector of Apache Flink that supports source and sink connector in data stream and Table.\n* [RocketMQ APIs](https://github.com/apache/rocketmq-apis): RocketMQ protobuf protocol.\n* [RocketMQ Clients](https://github.com/apache/rocketmq-clients): gRPC/protobuf-based RocketMQ clients.\n* RocketMQ Remoting-based Clients\n\t - [RocketMQ Client CPP](https://github.com/apache/rocketmq-client-cpp)\n\t - [RocketMQ Client Go](https://github.com/apache/rocketmq-client-go)\n\t - [RocketMQ Client Python](https://github.com/apache/rocketmq-client-python)\n\t - [RocketMQ Client Nodejs](https://github.com/apache/rocketmq-client-nodejs)\n* [RocketMQ Spring](https://github.com/apache/rocketmq-spring): A project which helps developers quickly integrate Apache RocketMQ with Spring Boot.\n* [RocketMQ Exporter](https://github.com/apache/rocketmq-exporter): An Apache RocketMQ exporter for Prometheus.\n* [RocketMQ Operator](https://github.com/apache/rocketmq-operator): Providing a way to run an Apache RocketMQ cluster on Kubernetes.\n* [RocketMQ Docker](https://github.com/apache/rocketmq-docker): The Git repo of the Docker Image for Apache RocketMQ.\n* [RocketMQ Dashboard](https://github.com/apache/rocketmq-dashboard): Operation and maintenance console of Apache RocketMQ.\n* [RocketMQ Connect](https://github.com/apache/rocketmq-connect): A tool for scalably and reliably streaming data between Apache RocketMQ and other systems.\n* [RocketMQ MQTT](https://github.com/apache/rocketmq-mqtt): A new MQTT protocol architecture model, based on which Apache RocketMQ can better support messages from terminals such as IoT devices and Mobile APP.\n* [RocketMQ EventBridge](https://github.com/apache/rocketmq-eventbridge): EventBridge makes it easier to build an event-driven application.\n* [RocketMQ Incubating Community Projects](https://github.com/apache/rocketmq-externals): Incubator community projects of Apache RocketMQ, including [logappender](https://github.com/apache/rocketmq-externals/tree/master/logappender), [rocketmq-ansible](https://github.com/apache/rocketmq-externals/tree/master/rocketmq-ansible), [rocketmq-beats-integration](https://github.com/apache/rocketmq-externals/tree/master/rocketmq-beats-integration), [rocketmq-cloudevents-binding](https://github.com/apache/rocketmq-externals/tree/master/rocketmq-cloudevents-binding), etc.\n* [RocketMQ Site](https://github.com/apache/rocketmq-site): The repository for Apache RocketMQ website.\n* [RocketMQ E2E](https://github.com/apache/rocketmq-e2e): A project for testing Apache RocketMQ, including end-to-end, performance, compatibility tests.\n\n\n----------\n## Learn it & Contact us\n* Mailing Lists: <https://rocketmq.apache.org/about/contact/>\n* Home: <https://rocketmq.apache.org>\n* Docs: <https://rocketmq.apache.org/docs/quick-start/>\n* Issues: <https://github.com/apache/rocketmq/issues>\n* Rips: <https://github.com/apache/rocketmq/wiki/RocketMQ-Improvement-Proposal>\n* Ask: <https://stackoverflow.com/questions/tagged/rocketmq>\n* Slack: <https://rocketmq-invite-automation.herokuapp.com/>\n\n\n----------\n\n\n\n## Contributing\nWe always welcome new contributions, whether for trivial cleanups, [big new features](https://github.com/apache/rocketmq/wiki/RocketMQ-Improvement-Proposal) or other material rewards, more details see [here](http://rocketmq.apache.org/docs/how-to-contribute/).\n\n----------\n## License\n[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html) Copyright (C) Apache Software Foundation\n\n\n----------\n## Export Control Notice\nThis distribution includes cryptographic software. The country in which you currently reside may have\nrestrictions on the import, possession, use, and/or re-export to another country, of encryption software.\nBEFORE using any encryption software, please check your country's laws, regulations and policies concerning\nthe import, possession, or use, and re-export of encryption software, to see if this is permitted. See\n<http://www.wassenaar.org/> for more information.\n\nThe U.S. Government Department of Commerce, Bureau of Industry and Security (BIS), has classified this\nsoftware as Export Commodity Control Number (ECCN) 5D002.C.1, which includes information security software\nusing or performing cryptographic functions with asymmetric algorithms. The form and manner of this Apache\nSoftware Foundation distribution makes it eligible for export under the License Exception ENC Technology\nSoftware Unrestricted (TSU) exception (see the BIS Export Administration Regulations, Section 740.13) for\nboth object code and source code.\n\nThe following provides more details on the included cryptographic software:\n\nThis software uses Apache Commons Crypto (https://commons.apache.org/proper/commons-crypto/) to\nsupport authentication, and encryption and decryption of data sent across the network between\nservices.\n\n[maven-build-image]: https://github.com/apache/rocketmq/actions/workflows/maven.yaml/badge.svg\n[maven-build-url]: https://github.com/apache/rocketmq/actions/workflows/maven.yaml\n[codecov-image]: https://codecov.io/gh/apache/rocketmq/branch/master/graph/badge.svg\n[codecov-url]: https://codecov.io/gh/apache/rocketmq\n[maven-central-image]: https://maven-badges.herokuapp.com/maven-central/org.apache.rocketmq/rocketmq-all/badge.svg\n[maven-central-url]: http://search.maven.org/#search%7Cga%7C1%7Corg.apache.rocketmq\n[release-image]: https://img.shields.io/badge/release-download-orange.svg\n[release-url]: https://rocketmq.apache.org/download/\n[license-image]: https://img.shields.io/badge/license-Apache%202-4EB1BA.svg\n[license-url]: https://www.apache.org/licenses/LICENSE-2.0.html\n[average-time-to-resolve-an-issue-image]: http://isitmaintained.com/badge/resolution/apache/rocketmq.svg\n[average-time-to-resolve-an-issue-url]: http://isitmaintained.com/project/apache/rocketmq\n[percentage-of-issues-still-open-image]: http://isitmaintained.com/badge/open/apache/rocketmq.svg\n[percentage-of-issues-still-open-url]: http://isitmaintained.com/project/apache/rocketmq\n[twitter-follow-image]: https://img.shields.io/twitter/follow/ApacheRocketMQ?style=social\n[twitter-follow-url]: https://twitter.com/intent/follow?screen_name=ApacheRocketMQ\n"
  },
  {
    "path": "WORKSPACE",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\nload(\"@bazel_tools//tools/build_defs/repo:http.bzl\", \"http_archive\")\n\nRULES_JVM_EXTERNAL_TAG = \"4.2\"\n\nRULES_JVM_EXTERNAL_SHA = \"cd1a77b7b02e8e008439ca76fd34f5b07aecb8c752961f9640dea15e9e5ba1ca\"\n\nhttp_archive(\n    name = \"rules_jvm_external\",\n    sha256 = RULES_JVM_EXTERNAL_SHA,\n    strip_prefix = \"rules_jvm_external-%s\" % RULES_JVM_EXTERNAL_TAG,\n    url = \"https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip\" % RULES_JVM_EXTERNAL_TAG,\n)\n\nload(\"@rules_jvm_external//:repositories.bzl\", \"rules_jvm_external_deps\")\n\nrules_jvm_external_deps()\n\nload(\"@rules_jvm_external//:setup.bzl\", \"rules_jvm_external_setup\")\n\nrules_jvm_external_setup()\n\nload(\"@rules_jvm_external//:defs.bzl\", \"maven_install\")\n\nmaven_install(\n    artifacts = [\n        \"junit:junit:4.13.2\",\n        \"com.alibaba:fastjson:1.2.76\",\n        \"com.alibaba.fastjson2:fastjson2:2.0.59\",\n        \"org.hamcrest:hamcrest-library:1.3\",\n        \"io.netty:netty-all:4.1.65.Final\",\n        \"org.assertj:assertj-core:3.22.0\",\n        \"org.mockito:mockito-core:3.10.0\",\n        \"org.powermock:powermock-module-junit4:2.0.9\",\n        \"org.powermock:powermock-api-mockito2:2.0.9\",\n        \"org.powermock:powermock-core:2.0.9\",\n        \"com.github.luben:zstd-jni:1.5.2-2\",\n        \"org.lz4:lz4-java:1.8.0\",\n        \"commons-validator:commons-validator:1.7\",\n        \"org.apache.commons:commons-lang3:3.12.0\",\n        \"org.hamcrest:hamcrest-core:1.3\",\n        \"io.openmessaging.storage:dledger:0.3.2\",\n        \"net.java.dev.jna:jna:4.2.2\",\n        \"ch.qos.logback:logback-classic:1.2.10\",\n        \"ch.qos.logback:logback-core:1.2.10\",\n        \"io.opentracing:opentracing-api:0.33.0\",\n        \"io.opentracing:opentracing-mock:0.33.0\",\n        \"commons-collections:commons-collections:3.2.2\",\n        \"org.awaitility:awaitility:4.1.0\",\n        \"commons-cli:commons-cli:1.5.0\",\n        \"com.google.guava:guava:31.0.1-jre\",\n        \"org.yaml:snakeyaml:2.0\",\n        \"commons-codec:commons-codec:1.13\",\n        \"commons-io:commons-io:2.7\",\n        \"com.google.truth:truth:0.30\",\n        \"org.bouncycastle:bcpkix-jdk15on:1.69\",\n        \"com.google.code.gson:gson:2.8.9\",\n        \"com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru:1.4.2\",\n        \"org.apache.rocketmq:rocketmq-proto:2.1.1\",\n        \"com.google.protobuf:protobuf-java:3.20.1\",\n        \"com.google.protobuf:protobuf-java-util:3.20.1\",\n        \"com.conversantmedia:disruptor:1.2.10\",\n        \"org.apache.tomcat:annotations-api:6.0.53\",\n        \"com.google.code.findbugs:jsr305:3.0.2\",\n        \"org.checkerframework:checker-qual:3.12.0\",\n        \"org.reflections:reflections:0.9.11\",\n        \"org.openjdk.jmh:jmh-core:1.19\",\n        \"org.openjdk.jmh:jmh-generator-annprocess:1.19\",\n        \"com.github.ben-manes.caffeine:caffeine:2.9.3\",\n        \"io.grpc:grpc-services:1.47.0\",\n        \"io.grpc:grpc-netty-shaded:1.47.0\",\n        \"io.grpc:grpc-context:1.47.0\",\n        \"io.grpc:grpc-stub:1.47.0\",\n        \"io.grpc:grpc-api:1.47.0\",\n        \"io.grpc:grpc-testing:1.47.0\",\n        \"org.springframework:spring-core:5.3.26\",\n        \"io.opentelemetry:opentelemetry-exporter-otlp:1.29.0\",\n        \"io.opentelemetry:opentelemetry-exporter-prometheus:1.29.0-alpha\",\n        \"io.opentelemetry:opentelemetry-exporter-logging:1.29.0\",\n        \"io.opentelemetry:opentelemetry-sdk:1.29.0\",\n        \"io.opentelemetry:opentelemetry-exporter-logging-otlp:1.29.0\",\n        \"com.squareup.okio:okio-jvm:3.0.0\",\n        \"io.opentelemetry:opentelemetry-api:1.29.0\",\n        \"io.opentelemetry:opentelemetry-sdk-metrics:1.29.0\",\n        \"io.opentelemetry:opentelemetry-sdk-common:1.29.0\",\n        \"io.github.aliyunmq:rocketmq-slf4j-api:1.0.0\",\n        \"io.github.aliyunmq:rocketmq-logback-classic:1.0.0\",\n        \"org.slf4j:jul-to-slf4j:2.0.6\",\n    \t\"org.jetbrains:annotations:23.1.0\",\n        \"io.github.aliyunmq:rocketmq-shaded-slf4j-api-bridge:1.0.0\",\n        \"software.amazon.awssdk:s3:2.20.29\",\n        \"com.fasterxml.jackson.core:jackson-databind:2.13.4.2\",\n        \"com.adobe.testing:s3mock-junit4:2.11.0\",\n        \"io.github.aliyunmq:rocketmq-grpc-netty-codec-haproxy:1.0.0\",\n        \"org.apache.rocketmq:rocketmq-rocksdb:1.0.6\",\n        \"com.alipay.sofa:jraft-core:1.3.14\",\n        \"com.alipay.sofa:hessian:3.3.6\",\n        \"io.netty:netty-tcnative-boringssl-static:2.0.48.Final\",\n        \"org.mockito:mockito-junit-jupiter:4.11.0\",\n        \"com.alibaba.fastjson2:fastjson2:2.0.59\",\n        \"org.junit.jupiter:junit-jupiter-api:5.9.1\",\n    ],\n    fetch_sources = True,\n    repositories = [\n        # Private repositories are supported through HTTP Basic auth\n        \"https://repo1.maven.org/maven2\",\n    ],\n)\n\nhttp_archive(\n    name = \"io_buildbuddy_buildbuddy_toolchain\",\n    sha256 = \"b12273608db627eb14051eb75f8a2134590172cd69392086d392e25f3954ea6e\",\n    strip_prefix = \"buildbuddy-toolchain-8d5d18373adfca9d8e33b4812915abc9b132f1ee\",\n    urls = [\"https://github.com/buildbuddy-io/buildbuddy-toolchain/archive/8d5d18373adfca9d8e33b4812915abc9b132f1ee.tar.gz\"],\n)\nload(\"@io_buildbuddy_buildbuddy_toolchain//:deps.bzl\", \"buildbuddy_deps\")\nbuildbuddy_deps()\nload(\"@io_buildbuddy_buildbuddy_toolchain//:rules.bzl\", \"buildbuddy\")\nbuildbuddy(name = \"buildbuddy_toolchain\")\n\nhttp_archive(\n    name = \"bazel_skylib\",\n    sha256 = \"51b5105a760b353773f904d2bbc5e664d0987fbaf22265164de65d43e910d8ac\",\n    urls = [\n        \"https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.8.1/bazel-skylib-1.8.1.tar.gz\",\n        \"https://github.com/bazelbuild/bazel-skylib/releases/download/1.8.1/bazel-skylib-1.8.1.tar.gz\",\n    ],\n)\n\nload(\"@bazel_skylib//:workspace.bzl\", \"bazel_skylib_workspace\")\nbazel_skylib_workspace()\n\nhttp_archive(\n    name = \"rules_java\",\n    urls = [\n        \"https://github.com/bazelbuild/rules_java/releases/download/7.12.5/rules_java-7.12.5.tar.gz\",\n    ],\n    sha256 = \"17b18cb4f92ab7b94aa343ce78531b73960b1bed2ba166e5b02c9fdf0b0ac270\",\n)\nload(\"@rules_java//java:repositories.bzl\", \"rules_java_dependencies\", \"rules_java_toolchains\")\nrules_java_dependencies()\nrules_java_toolchains()\n\nload(\"@rules_java//toolchains:local_java_repository.bzl\", \"local_java_repository\")\nlocal_java_repository(\n  name = \"jdk8\",\n  version = \"8\",\n  java_home = \"/usr/lib/jvm/java-8-openjdk-amd64\",\n)\n"
  },
  {
    "path": "auth/BUILD.bazel",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\nload(\"//bazel:GenTestRules.bzl\", \"GenTestRules\")\n\njava_library(\n    name = \"auth\",\n    srcs = glob([\"src/main/java/**/*.java\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \"//common\",\n        \"//remoting\",\n        \"//client\",\n        \"@maven//:commons_codec_commons_codec\",\n        \"@maven//:org_apache_commons_commons_lang3\",\n        \"@maven//:commons_collections_commons_collections\",\n        \"@maven//:com_alibaba_fastjson2_fastjson2\",\n        \"@maven//:org_apache_rocketmq_rocketmq_proto\",\n        \"@maven//:org_slf4j_slf4j_api\",\n        \"@maven//:com_github_ben_manes_caffeine_caffeine\",\n        \"@maven//:io_grpc_grpc_api\",\n        \"@maven//:com_google_protobuf_protobuf_java\",\n        \"@maven//:com_google_protobuf_protobuf_java_util\",\n        \"@maven//:io_netty_netty_all\",\n        \"@maven//:com_google_guava_guava\",\n        \"@maven//:org_apache_rocketmq_rocketmq_rocksdb\",\n    ],\n)\n\njava_library(\n    name = \"tests\",\n    srcs = glob([\"src/test/java/**/*.java\"]),\n    resources = glob([\"src/test/resources/**/*.yml\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \":auth\",\n        \"//:test_deps\",\n        \"//common\",\n        \"//remoting\",\n        \"//client\",\n        \"@maven//:commons_codec_commons_codec\",\n        \"@maven//:org_apache_commons_commons_lang3\",\n        \"@maven//:commons_collections_commons_collections\",\n        \"@maven//:com_alibaba_fastjson2_fastjson2\",\n        \"@maven//:org_apache_rocketmq_rocketmq_proto\",\n        \"@maven//:org_slf4j_slf4j_api\",\n        \"@maven//:com_github_ben_manes_caffeine_caffeine\",\n        \"@maven//:io_grpc_grpc_api\",\n        \"@maven//:com_google_protobuf_protobuf_java\",\n        \"@maven//:com_google_protobuf_protobuf_java_util\",\n        \"@maven//:io_netty_netty_all\",\n        \"@maven//:com_google_guava_guava\",\n        \"@maven//:org_apache_rocketmq_rocketmq_rocksdb\",\n    ],\n)\n\nGenTestRules(\n    name = \"GeneratedTestRules\",\n    test_files = glob([\"src/test/java/**/*Test.java\"]),\n    deps = [\n        \":tests\",\n    ],\n)\n"
  },
  {
    "path": "auth/pom.xml",
    "content": "<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor\n\tlicense agreements. See the NOTICE file distributed with this work for additional\n\tinformation regarding copyright ownership. The ASF licenses this file to\n\tYou under the Apache License, Version 2.0 (the \"License\"); you may not use\n\tthis file except in compliance with the License. You may obtain a copy of\n\tthe License at http://www.apache.org/licenses/LICENSE-2.0 Unless required\n\tby applicable law or agreed to in writing, software distributed under the\n\tLicense is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS\n\tOF ANY KIND, either express or implied. See the License for the specific\n\tlanguage governing permissions and limitations under the License. -->\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n    <modelVersion>4.0.0</modelVersion>\n    <parent>\n        <groupId>org.apache.rocketmq</groupId>\n        <artifactId>rocketmq-all</artifactId>\n        <version>${revision}</version>\n    </parent>\n    <artifactId>rocketmq-auth</artifactId>\n    <name>rocketmq-auth ${project.version}</name>\n\n    <properties>\n        <project.root>${basedir}/..</project.root>\n    </properties>\n\n    <dependencies>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-proto</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-client</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>commons-codec</groupId>\n            <artifactId>commons-codec</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.apache.commons</groupId>\n            <artifactId>commons-lang3</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>com.google.protobuf</groupId>\n            <artifactId>protobuf-java-util</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.slf4j</groupId>\n            <artifactId>slf4j-api</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>com.github.ben-manes.caffeine</groupId>\n            <artifactId>caffeine</artifactId>\n            <exclusions>\n                <exclusion>\n                    <groupId>org.checkerframework</groupId>\n                    <artifactId>checker-qual</artifactId>\n                </exclusion>\n            </exclusions>\n        </dependency>\n        <dependency>\n            <groupId>junit</groupId>\n            <artifactId>junit</artifactId>\n        </dependency>\n    </dependencies>\n\n    <build>\n        <plugins>\n            <plugin>\n                <artifactId>maven-surefire-plugin</artifactId>\n                <version>${maven-surefire-plugin.version}</version>\n                <configuration>\n                    <forkCount>1</forkCount>\n                    <reuseForks>false</reuseForks>\n                </configuration>\n            </plugin>\n        </plugins>\n    </build>\n</project>\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/AuthenticationEvaluator.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication;\n\nimport java.util.function.Supplier;\nimport org.apache.rocketmq.auth.authentication.context.AuthenticationContext;\nimport org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;\nimport org.apache.rocketmq.auth.authentication.strategy.AuthenticationStrategy;\nimport org.apache.rocketmq.auth.config.AuthConfig;\n\npublic class AuthenticationEvaluator {\n\n    private final AuthenticationStrategy authenticationStrategy;\n\n    public AuthenticationEvaluator(AuthConfig authConfig) {\n        this(authConfig, null);\n    }\n\n    public AuthenticationEvaluator(AuthConfig authConfig, Supplier<?> metadataService) {\n        this.authenticationStrategy = AuthenticationFactory.getStrategy(authConfig, metadataService);\n    }\n\n    public void evaluate(AuthenticationContext context) {\n        if (context == null) {\n            return;\n        }\n        this.authenticationStrategy.evaluate(context);\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/builder/AuthenticationContextBuilder.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.builder;\n\nimport com.google.protobuf.GeneratedMessageV3;\nimport io.grpc.Metadata;\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic interface AuthenticationContextBuilder<AuthenticationContext> {\n\n    AuthenticationContext build(Metadata metadata, GeneratedMessageV3 request);\n\n    AuthenticationContext build(ChannelHandlerContext context, RemotingCommand request);\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/builder/DefaultAuthenticationContextBuilder.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.builder;\n\nimport com.google.protobuf.GeneratedMessageV3;\nimport io.grpc.Metadata;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.nio.charset.StandardCharsets;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.SortedMap;\nimport java.util.TreeMap;\nimport org.apache.commons.codec.DecoderException;\nimport org.apache.commons.codec.binary.Base64;\nimport org.apache.commons.codec.binary.Hex;\nimport org.apache.commons.collections.MapUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.acl.common.AclUtils;\nimport org.apache.rocketmq.acl.common.SessionCredentials;\nimport org.apache.rocketmq.auth.authentication.context.DefaultAuthenticationContext;\nimport org.apache.rocketmq.auth.authentication.exception.AuthenticationException;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.CommonConstants;\nimport org.apache.rocketmq.common.constant.GrpcConstants;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic class DefaultAuthenticationContextBuilder implements AuthenticationContextBuilder<DefaultAuthenticationContext> {\n\n    private static final String CREDENTIAL = \"Credential\";\n    private static final String SIGNATURE = \"Signature\";\n\n    @Override\n    public DefaultAuthenticationContext build(Metadata metadata, GeneratedMessageV3 request) {\n        try {\n            DefaultAuthenticationContext context = new DefaultAuthenticationContext();\n            context.setChannelId(metadata.get(GrpcConstants.CHANNEL_ID));\n            context.setRpcCode(request.getDescriptorForType().getFullName());\n            String authorization = metadata.get(GrpcConstants.AUTHORIZATION);\n            if (StringUtils.isEmpty(authorization)) {\n                return context;\n            }\n            String datetime = metadata.get(GrpcConstants.DATE_TIME);\n            if (StringUtils.isEmpty(datetime)) {\n                throw new AuthenticationException(\"datetime is null.\");\n            }\n\n            String[] result = authorization.split(CommonConstants.SPACE, 2);\n            if (result.length != 2) {\n                throw new AuthenticationException(\"authentication header is incorrect.\");\n            }\n            String[] keyValues = result[1].split(CommonConstants.COMMA);\n            for (String keyValue : keyValues) {\n                String[] kv = keyValue.trim().split(CommonConstants.EQUAL, 2);\n                int kvLength = kv.length;\n                if (kv.length != 2) {\n                    throw new AuthenticationException(\"authentication keyValues length is incorrect, actual length={}.\", kvLength);\n                }\n                String authItem = kv[0];\n                if (CREDENTIAL.equals(authItem)) {\n                    String[] credential = kv[1].split(CommonConstants.SLASH);\n                    int credentialActualLength = credential.length;\n                    if (credentialActualLength == 0) {\n                        throw new AuthenticationException(\"authentication credential length is incorrect, actual length={}.\", credentialActualLength);\n                    }\n                    context.setUsername(credential[0]);\n                    continue;\n                }\n                if (SIGNATURE.equals(authItem)) {\n                    context.setSignature(this.hexToBase64(kv[1]));\n                }\n            }\n\n            context.setContent(datetime.getBytes(StandardCharsets.UTF_8));\n\n            return context;\n        } catch (AuthenticationException e) {\n            throw e;\n        } catch (Throwable e) {\n            throw new AuthenticationException(\"create authentication context error.\", e);\n        }\n    }\n\n    @Override\n    public DefaultAuthenticationContext build(ChannelHandlerContext context, RemotingCommand request) {\n        HashMap<String, String> fields = request.getExtFields();\n        DefaultAuthenticationContext result = new DefaultAuthenticationContext();\n        result.setChannelId(context.channel().id().asLongText());\n        result.setRpcCode(String.valueOf(request.getCode()));\n        if (MapUtils.isEmpty(fields)) {\n            return result;\n        }\n        if (!fields.containsKey(SessionCredentials.ACCESS_KEY)) {\n            return result;\n        }\n        result.setUsername(fields.get(SessionCredentials.ACCESS_KEY));\n        result.setSignature(fields.get(SessionCredentials.SIGNATURE));\n        // Content\n        SortedMap<String, String> map = new TreeMap<>();\n        for (Map.Entry<String, String> entry : fields.entrySet()) {\n            if (request.getVersion() <= MQVersion.Version.V4_9_3.ordinal() &&\n                MixAll.UNIQUE_MSG_QUERY_FLAG.equals(entry.getKey())) {\n                continue;\n            }\n            if (!SessionCredentials.SIGNATURE.equals(entry.getKey())) {\n                map.put(entry.getKey(), entry.getValue());\n            }\n        }\n        result.setContent(AclUtils.combineRequestContent(request, map));\n        return result;\n    }\n\n    public String hexToBase64(String input) throws DecoderException {\n        byte[] bytes = Hex.decodeHex(input);\n        return Base64.encodeBase64String(bytes);\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/chain/DefaultAuthenticationHandler.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.chain;\n\nimport java.security.MessageDigest;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.function.Supplier;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.acl.common.AclSigner;\nimport org.apache.rocketmq.auth.authentication.context.DefaultAuthenticationContext;\nimport org.apache.rocketmq.auth.authentication.enums.UserStatus;\nimport org.apache.rocketmq.auth.authentication.exception.AuthenticationException;\nimport org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.auth.authentication.provider.AuthenticationMetadataProvider;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.chain.Handler;\nimport org.apache.rocketmq.common.chain.HandlerChain;\n\npublic class DefaultAuthenticationHandler implements Handler<DefaultAuthenticationContext, CompletableFuture<Void>> {\n\n    private final AuthenticationMetadataProvider authenticationMetadataProvider;\n\n    public DefaultAuthenticationHandler(AuthConfig config, Supplier<?> metadataService) {\n        this.authenticationMetadataProvider = AuthenticationFactory.getMetadataProvider(config, metadataService);\n    }\n\n    @Override\n    public CompletableFuture<Void> handle(DefaultAuthenticationContext context,\n        HandlerChain<DefaultAuthenticationContext, CompletableFuture<Void>> chain) {\n        return getUser(context).thenAccept(user -> doAuthenticate(context, user));\n    }\n\n    protected CompletableFuture<User> getUser(DefaultAuthenticationContext context) {\n        if (this.authenticationMetadataProvider == null) {\n            throw new AuthenticationException(\"The authenticationMetadataProvider is not configured\");\n        }\n        if (StringUtils.isEmpty(context.getUsername())) {\n            throw new AuthenticationException(\"username cannot be null.\");\n        }\n        return this.authenticationMetadataProvider.getUser(context.getUsername());\n    }\n\n    protected void doAuthenticate(DefaultAuthenticationContext context, User user) {\n        if (user == null) {\n            throw new AuthenticationException(\"User:{} is not found.\", context.getUsername());\n        }\n        if (user.getUserStatus() == UserStatus.DISABLE) {\n            throw new AuthenticationException(\"User:{} is disabled.\", context.getUsername());\n        }\n        String signature = AclSigner.calSignature(context.getContent(), user.getPassword());\n        if (context.getSignature() == null\n            || !MessageDigest.isEqual(signature.getBytes(AclSigner.DEFAULT_CHARSET), context.getSignature().getBytes(AclSigner.DEFAULT_CHARSET))) {\n            throw new AuthenticationException(\"check signature failed.\");\n        }\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/context/AuthenticationContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.context;\n\nimport java.util.HashMap;\nimport java.util.Map;\nimport org.apache.commons.lang3.StringUtils;\n\npublic abstract class AuthenticationContext {\n\n    private String channelId;\n\n    private String rpcCode;\n\n    private Map<String, Object> extInfo;\n\n    public String getChannelId() {\n        return channelId;\n    }\n\n    public void setChannelId(String channelId) {\n        this.channelId = channelId;\n    }\n\n    public String getRpcCode() {\n        return rpcCode;\n    }\n\n    public void setRpcCode(String rpcCode) {\n        this.rpcCode = rpcCode;\n    }\n\n    @SuppressWarnings(\"unchecked\")\n    public <T> T getExtInfo(String key) {\n        if (StringUtils.isBlank(key)) {\n            return null;\n        }\n        if (this.extInfo == null) {\n            return null;\n        }\n        Object value = this.extInfo.get(key);\n        if (value == null) {\n            return null;\n        }\n        return (T) value;\n    }\n\n    public void setExtInfo(String key, Object value) {\n        if (StringUtils.isBlank(key) || value == null) {\n            return;\n        }\n        if (this.extInfo == null) {\n            this.extInfo = new HashMap<>();\n        }\n        this.extInfo.put(key, value);\n    }\n\n    public boolean hasExtInfo(String key) {\n        Object value = getExtInfo(key);\n        return value != null;\n    }\n\n    public Map<String, Object> getExtInfo() {\n        return extInfo;\n    }\n\n    public void setExtInfo(Map<String, Object> extInfo) {\n        this.extInfo = extInfo;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/context/DefaultAuthenticationContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.context;\n\npublic class DefaultAuthenticationContext extends AuthenticationContext {\n\n    private String username;\n\n    private byte[] content;\n\n    private String signature;\n\n    public String getUsername() {\n        return username;\n    }\n\n    public void setUsername(String username) {\n        this.username = username;\n    }\n\n    public byte[] getContent() {\n        return content;\n    }\n\n    public void setContent(byte[] content) {\n        this.content = content;\n    }\n\n    public String getSignature() {\n        return signature;\n    }\n\n    public void setSignature(String signature) {\n        this.signature = signature;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/enums/SubjectType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.enums;\n\nimport com.alibaba.fastjson2.annotation.JSONField;\nimport org.apache.commons.lang3.StringUtils;\n\npublic enum SubjectType {\n\n    USER((byte) 1, \"User\");\n\n    @JSONField(value = true)\n    private final byte code;\n    private final String name;\n\n    SubjectType(byte code, String name) {\n        this.code = code;\n        this.name = name;\n    }\n\n    public static SubjectType getByName(String name) {\n        for (SubjectType subjectType : SubjectType.values()) {\n            if (StringUtils.equalsIgnoreCase(subjectType.getName(), name)) {\n                return subjectType;\n            }\n        }\n        return null;\n    }\n\n    public byte getCode() {\n        return code;\n    }\n\n    public String getName() {\n        return name;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/enums/UserStatus.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.enums;\n\nimport com.alibaba.fastjson2.annotation.JSONField;\nimport org.apache.commons.lang3.StringUtils;\n\npublic enum UserStatus {\n\n    ENABLE((byte) 1, \"enable\"),\n\n    DISABLE((byte) 2, \"disable\");\n\n    @JSONField(value = true)\n    private final byte code;\n\n    private final String name;\n\n    UserStatus(byte code, String name) {\n        this.code = code;\n        this.name = name;\n    }\n\n    public static UserStatus getByName(String name) {\n        for (UserStatus subjectType : UserStatus.values()) {\n            if (StringUtils.equalsIgnoreCase(subjectType.getName(), name)) {\n                return subjectType;\n            }\n        }\n        return null;\n    }\n\n    public byte getCode() {\n        return code;\n    }\n\n    public String getName() {\n        return name;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/enums/UserType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.enums;\n\nimport com.alibaba.fastjson2.annotation.JSONField;\nimport org.apache.commons.lang3.StringUtils;\n\npublic enum UserType {\n\n    SUPER((byte) 1, \"Super\"),\n\n    NORMAL((byte) 2, \"Normal\");\n\n    @JSONField(value = true)\n    private final byte code;\n\n    private final String name;\n\n    UserType(byte code, String name) {\n        this.code = code;\n        this.name = name;\n    }\n\n    public static UserType getByName(String name) {\n        for (UserType subjectType : UserType.values()) {\n            if (StringUtils.equalsIgnoreCase(subjectType.getName(), name)) {\n                return subjectType;\n            }\n        }\n        return null;\n    }\n\n    public byte getCode() {\n        return code;\n    }\n\n    public String getName() {\n        return name;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/exception/AuthenticationException.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.exception;\n\nimport org.slf4j.helpers.MessageFormatter;\n\npublic class AuthenticationException extends RuntimeException {\n\n    public AuthenticationException(String message) {\n        super(message);\n    }\n\n    public AuthenticationException(String message, Throwable cause) {\n        super(message, cause);\n    }\n\n    public AuthenticationException(String messagePattern, Object... argArray) {\n        super(MessageFormatter.arrayFormat(messagePattern, argArray).getMessage());\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/factory/AuthenticationFactory.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.factory;\n\nimport com.google.protobuf.GeneratedMessageV3;\nimport io.grpc.Metadata;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.function.Function;\nimport java.util.function.Supplier;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.auth.authentication.AuthenticationEvaluator;\nimport org.apache.rocketmq.auth.authentication.context.AuthenticationContext;\nimport org.apache.rocketmq.auth.authentication.manager.AuthenticationMetadataManager;\nimport org.apache.rocketmq.auth.authentication.manager.AuthenticationMetadataManagerImpl;\nimport org.apache.rocketmq.auth.authentication.provider.AuthenticationMetadataProvider;\nimport org.apache.rocketmq.auth.authentication.provider.AuthenticationProvider;\nimport org.apache.rocketmq.auth.authentication.provider.DefaultAuthenticationProvider;\nimport org.apache.rocketmq.auth.authentication.strategy.AuthenticationStrategy;\nimport org.apache.rocketmq.auth.authentication.strategy.StatelessAuthenticationStrategy;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic class AuthenticationFactory {\n\n    private static final Map<String, Object> INSTANCE_MAP = new HashMap<>();\n    private static final String PROVIDER_PREFIX = \"PROVIDER_\";\n    private static final String METADATA_PROVIDER_PREFIX = \"METADATA_PROVIDER_\";\n    private static final String EVALUATOR_PREFIX = \"EVALUATOR_\";\n\n    @SuppressWarnings(\"unchecked\")\n    public static AuthenticationProvider<AuthenticationContext> getProvider(AuthConfig config) {\n        if (config == null) {\n            return null;\n        }\n        return computeIfAbsent(PROVIDER_PREFIX + config.getConfigName(), key -> {\n            try {\n                Class<? extends AuthenticationProvider<? extends AuthenticationContext>> clazz =\n                    DefaultAuthenticationProvider.class;\n                if (StringUtils.isNotBlank(config.getAuthenticationProvider())) {\n                    clazz = (Class<? extends AuthenticationProvider<? extends AuthenticationContext>>) Class.forName(config.getAuthenticationProvider());\n                }\n                return (AuthenticationProvider<AuthenticationContext>) clazz.getDeclaredConstructor().newInstance();\n            } catch (Exception e) {\n                throw new RuntimeException(\"Failed to load the authentication provider.\", e);\n            }\n        });\n    }\n\n    public static AuthenticationMetadataProvider getMetadataProvider(AuthConfig config) {\n        return getMetadataProvider(config, null);\n    }\n\n    public static AuthenticationMetadataManager getMetadataManager(AuthConfig config) {\n        return new AuthenticationMetadataManagerImpl(config);\n    }\n\n    @SuppressWarnings(\"unchecked\")\n    public static AuthenticationMetadataProvider getMetadataProvider(AuthConfig config, Supplier<?> metadataService) {\n        if (config == null) {\n            return null;\n        }\n        return computeIfAbsent(METADATA_PROVIDER_PREFIX + config.getConfigName(), key -> {\n            try {\n                if (StringUtils.isBlank(config.getAuthenticationMetadataProvider())) {\n                    return null;\n                }\n                Class<? extends AuthenticationMetadataProvider> clazz = (Class<? extends AuthenticationMetadataProvider>)\n                    Class.forName(config.getAuthenticationMetadataProvider());\n                AuthenticationMetadataProvider result = clazz.getDeclaredConstructor().newInstance();\n                result.initialize(config, metadataService);\n                return result;\n            } catch (Exception e) {\n                throw new RuntimeException(\"Failed to load the authentication metadata provider\", e);\n            }\n        });\n    }\n\n    public static AuthenticationEvaluator getEvaluator(AuthConfig config) {\n        return computeIfAbsent(EVALUATOR_PREFIX + config.getConfigName(), key -> new AuthenticationEvaluator(config));\n    }\n\n    public static AuthenticationEvaluator getEvaluator(AuthConfig config, Supplier<?> metadataService) {\n        return computeIfAbsent(EVALUATOR_PREFIX + config.getConfigName(), key -> new AuthenticationEvaluator(config, metadataService));\n    }\n\n    @SuppressWarnings(\"unchecked\")\n    public static AuthenticationStrategy getStrategy(AuthConfig config, Supplier<?> metadataService) {\n        try {\n            Class<? extends AuthenticationStrategy> clazz = StatelessAuthenticationStrategy.class;\n            if (StringUtils.isNotBlank(config.getAuthenticationStrategy())) {\n                clazz = (Class<? extends AuthenticationStrategy>) Class.forName(config.getAuthenticationStrategy());\n            }\n            return clazz.getDeclaredConstructor(AuthConfig.class, Supplier.class).newInstance(config, metadataService);\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    public static AuthenticationContext newContext(AuthConfig config, Metadata metadata, GeneratedMessageV3 request) {\n        AuthenticationProvider<AuthenticationContext> authenticationProvider = getProvider(config);\n        if (authenticationProvider == null) {\n            return null;\n        }\n        return authenticationProvider.newContext(metadata, request);\n    }\n\n    public static AuthenticationContext newContext(AuthConfig config, ChannelHandlerContext context,\n        RemotingCommand command) {\n        AuthenticationProvider<AuthenticationContext> authenticationProvider = getProvider(config);\n        if (authenticationProvider == null) {\n            return null;\n        }\n        return authenticationProvider.newContext(context, command);\n    }\n\n    @SuppressWarnings(\"unchecked\")\n    private static <V> V computeIfAbsent(String key, Function<String, ? extends V> function) {\n        Object result = null;\n        if (INSTANCE_MAP.containsKey(key)) {\n            result = INSTANCE_MAP.get(key);\n        }\n        if (result == null) {\n            synchronized (INSTANCE_MAP) {\n                if (INSTANCE_MAP.containsKey(key)) {\n                    result = INSTANCE_MAP.get(key);\n                }\n                if (result == null) {\n                    result = function.apply(key);\n                    if (result != null) {\n                        INSTANCE_MAP.put(key, result);\n                    }\n                }\n            }\n        }\n        return result != null ? (V) result : null;\n    }\n}"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/manager/AuthenticationMetadataManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.manager;\n\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.auth.config.AuthConfig;\n\npublic interface AuthenticationMetadataManager {\n\n    void shutdown();\n\n    void initUser(AuthConfig authConfig);\n\n    CompletableFuture<Void> createUser(User user);\n\n    CompletableFuture<Void> updateUser(User user);\n\n    CompletableFuture<Void> deleteUser(String username);\n\n    CompletableFuture<User> getUser(String username);\n\n    CompletableFuture<List<User>> listUser(String filter);\n\n    CompletableFuture<Boolean> isSuperUser(String username);\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/manager/AuthenticationMetadataManagerImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.manager;\n\nimport com.alibaba.fastjson2.JSON;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.acl.common.SessionCredentials;\nimport org.apache.rocketmq.auth.authentication.enums.UserStatus;\nimport org.apache.rocketmq.auth.authentication.enums.UserType;\nimport org.apache.rocketmq.auth.authentication.exception.AuthenticationException;\nimport org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.auth.authentication.provider.AuthenticationMetadataProvider;\nimport org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;\nimport org.apache.rocketmq.auth.authorization.provider.AuthorizationMetadataProvider;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.utils.ExceptionUtils;\n\npublic class AuthenticationMetadataManagerImpl implements AuthenticationMetadataManager {\n\n    private final AuthenticationMetadataProvider authenticationMetadataProvider;\n\n    private final AuthorizationMetadataProvider authorizationMetadataProvider;\n\n    public AuthenticationMetadataManagerImpl(AuthConfig authConfig) {\n        this.authenticationMetadataProvider = AuthenticationFactory.getMetadataProvider(authConfig);\n        this.authorizationMetadataProvider = AuthorizationFactory.getMetadataProvider(authConfig);\n        this.initUser(authConfig);\n    }\n\n    @Override\n    public void shutdown() {\n        if (this.authenticationMetadataProvider != null) {\n            this.authenticationMetadataProvider.shutdown();\n        }\n        if (this.authorizationMetadataProvider != null) {\n            this.authorizationMetadataProvider.shutdown();\n        }\n    }\n\n    @Override\n    public void initUser(AuthConfig authConfig) {\n        if (authConfig == null) {\n            return;\n        }\n        if (StringUtils.isNotBlank(authConfig.getInitAuthenticationUser())) {\n            try {\n                User initUser = JSON.parseObject(authConfig.getInitAuthenticationUser(), User.class);\n                initUser.setUserType(UserType.SUPER);\n                this.getUser(initUser.getUsername()).thenCompose(user -> {\n                    if (user != null) {\n                        return CompletableFuture.completedFuture(null);\n                    }\n                    return this.createUser(initUser);\n                }).join();\n            } catch (Exception e) {\n                throw new AuthenticationException(\"Init authentication user error.\", e);\n            }\n        }\n        if (StringUtils.isNotBlank(authConfig.getInnerClientAuthenticationCredentials())) {\n            try {\n                SessionCredentials credentials = JSON.parseObject(authConfig.getInnerClientAuthenticationCredentials(), SessionCredentials.class);\n                User innerUser = User.of(credentials.getAccessKey(), credentials.getSecretKey(), UserType.SUPER);\n                this.getUser(innerUser.getUsername()).thenCompose(user -> {\n                    if (user != null) {\n                        return CompletableFuture.completedFuture(null);\n                    }\n                    return this.createUser(innerUser);\n                }).join();\n            } catch (Exception e) {\n                throw new AuthenticationException(\"Init inner client authentication credentials error\", e);\n            }\n        }\n    }\n\n    @Override\n    public CompletableFuture<Void> createUser(User user) {\n        CompletableFuture<Void> result = new CompletableFuture<>();\n        try {\n            this.validate(user, true);\n            if (user.getUserType() == null) {\n                user.setUserType(UserType.NORMAL);\n            }\n            if (user.getUserStatus() == null) {\n                user.setUserStatus(UserStatus.ENABLE);\n            }\n            result = this.getAuthenticationMetadataProvider().getUser(user.getUsername()).thenCompose(old -> {\n                if (old != null) {\n                    throw new AuthenticationException(\"The user is existed\");\n                }\n                return this.getAuthenticationMetadataProvider().createUser(user);\n            });\n        } catch (Exception e) {\n            this.handleException(e, result);\n        }\n        return result;\n    }\n\n    @Override\n    public CompletableFuture<Void> updateUser(User user) {\n        CompletableFuture<Void> result = new CompletableFuture<>();\n        try {\n            this.validate(user, false);\n            result = this.getAuthenticationMetadataProvider().getUser(user.getUsername()).thenCompose(old -> {\n                if (old == null) {\n                    throw new AuthenticationException(\"The user is not exist\");\n                }\n                if (StringUtils.isNotBlank(user.getPassword())) {\n                    old.setPassword(user.getPassword());\n                }\n                if (user.getUserType() != null) {\n                    old.setUserType(user.getUserType());\n                }\n                if (user.getUserStatus() != null) {\n                    old.setUserStatus(user.getUserStatus());\n                }\n                return this.getAuthenticationMetadataProvider().updateUser(old);\n            });\n        } catch (Exception e) {\n            this.handleException(e, result);\n        }\n        return result;\n    }\n\n    @Override\n    public CompletableFuture<Void> deleteUser(String username) {\n        CompletableFuture<Void> result = new CompletableFuture<>();\n        try {\n            if (StringUtils.isBlank(username)) {\n                throw new AuthenticationException(\"username can not be blank\");\n            }\n            CompletableFuture<Void> deleteUser = this.getAuthenticationMetadataProvider().deleteUser(username);\n            CompletableFuture<Void> deleteAcl = this.getAuthorizationMetadataProvider().deleteAcl(User.of(username));\n            return CompletableFuture.allOf(deleteUser, deleteAcl);\n        } catch (Exception e) {\n            this.handleException(e, result);\n        }\n        return result;\n    }\n\n    @Override\n    public CompletableFuture<User> getUser(String username) {\n        CompletableFuture<User> result = new CompletableFuture<>();\n        try {\n            if (StringUtils.isBlank(username)) {\n                throw new AuthenticationException(\"username can not be blank\");\n            }\n            result = this.getAuthenticationMetadataProvider().getUser(username);\n        } catch (Exception e) {\n            this.handleException(e, result);\n        }\n        return result;\n    }\n\n    @Override\n    public CompletableFuture<List<User>> listUser(String filter) {\n        CompletableFuture<List<User>> result = new CompletableFuture<>();\n        try {\n            result = this.getAuthenticationMetadataProvider().listUser(filter);\n        } catch (Exception e) {\n            this.handleException(e, result);\n        }\n        return result;\n    }\n\n    @Override\n    public CompletableFuture<Boolean> isSuperUser(String username) {\n        return this.getUser(username).thenApply(user -> {\n            if (user == null) {\n                throw new AuthenticationException(\"User:{} is not found\", username);\n            }\n            return user.getUserType() == UserType.SUPER;\n        });\n    }\n\n    private void validate(User user, boolean isCreate) {\n        if (user == null) {\n            throw new AuthenticationException(\"user can not be null\");\n        }\n        if (StringUtils.isBlank(user.getUsername())) {\n            throw new AuthenticationException(\"username can not be blank\");\n        }\n        if (isCreate && StringUtils.isBlank(user.getPassword())) {\n            throw new AuthenticationException(\"password can not be blank\");\n        }\n    }\n\n    private void handleException(Exception e, CompletableFuture<?> result) {\n        Throwable throwable = ExceptionUtils.getRealException(e);\n        result.completeExceptionally(throwable);\n    }\n\n    private AuthenticationMetadataProvider getAuthenticationMetadataProvider() {\n        if (authenticationMetadataProvider == null) {\n            throw new IllegalStateException(\"The authenticationMetadataProvider is not configured.\");\n        }\n        return authenticationMetadataProvider;\n    }\n\n    private AuthorizationMetadataProvider getAuthorizationMetadataProvider() {\n        if (authorizationMetadataProvider == null) {\n            throw new IllegalStateException(\"The authorizationMetadataProvider is not configured.\");\n        }\n        return authorizationMetadataProvider;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/model/Subject.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.model;\n\nimport com.alibaba.fastjson2.annotation.JSONField;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.auth.authentication.enums.SubjectType;\nimport org.apache.rocketmq.common.constant.CommonConstants;\n\npublic interface Subject {\n\n    @JSONField(serialize = false)\n    String getSubjectKey();\n\n    SubjectType getSubjectType();\n\n    default boolean isSubject(SubjectType subjectType) {\n        return subjectType == this.getSubjectType();\n    }\n\n    @SuppressWarnings(\"unchecked\")\n    static <T extends Subject> T of(String subjectKey) {\n        String type = StringUtils.substringBefore(subjectKey, CommonConstants.COLON);\n        SubjectType subjectType = SubjectType.getByName(type);\n        if (subjectType == null) {\n            return null;\n        }\n        if (subjectType == SubjectType.USER) {\n            return (T) User.of(StringUtils.substringAfter(subjectKey, CommonConstants.COLON));\n        }\n        return null;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/model/User.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.model;\n\nimport org.apache.rocketmq.auth.authentication.enums.SubjectType;\nimport org.apache.rocketmq.auth.authentication.enums.UserStatus;\nimport org.apache.rocketmq.auth.authentication.enums.UserType;\nimport org.apache.rocketmq.common.constant.CommonConstants;\n\npublic class User implements Subject {\n\n    private String username;\n\n    private String password;\n\n    private UserType userType;\n\n    private UserStatus userStatus;\n\n    public static User of(String username) {\n        User user = new User();\n        user.setUsername(username);\n        return user;\n    }\n\n    public static User of(String username, String password) {\n        User user = new User();\n        user.setUsername(username);\n        user.setPassword(password);\n        return user;\n    }\n\n    public static User of(String username, String password, UserType userType) {\n        User user = new User();\n        user.setUsername(username);\n        user.setPassword(password);\n        user.setUserType(userType);\n        return user;\n    }\n\n    @Override\n    public String getSubjectKey() {\n        return this.getSubjectType().getName() + CommonConstants.COLON + this.username;\n    }\n\n    @Override\n    public SubjectType getSubjectType() {\n        return SubjectType.USER;\n    }\n\n    public String getUsername() {\n        return username;\n    }\n\n    public void setUsername(String username) {\n        this.username = username;\n    }\n\n    public String getPassword() {\n        return password;\n    }\n\n    public void setPassword(String password) {\n        this.password = password;\n    }\n\n    public UserType getUserType() {\n        return userType;\n    }\n\n    public void setUserType(UserType userType) {\n        this.userType = userType;\n    }\n\n    public UserStatus getUserStatus() {\n        return userStatus;\n    }\n\n    public void setUserStatus(UserStatus userStatus) {\n        this.userStatus = userStatus;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/provider/AuthenticationMetadataProvider.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.provider;\n\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.function.Supplier;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.auth.config.AuthConfig;\n\npublic interface AuthenticationMetadataProvider {\n\n    void initialize(AuthConfig authConfig, Supplier<?> metadataService);\n\n    void shutdown();\n\n    CompletableFuture<Void> createUser(User user);\n\n    CompletableFuture<Void> deleteUser(String username);\n\n    CompletableFuture<Void> updateUser(User user);\n\n    CompletableFuture<User> getUser(String username);\n\n    CompletableFuture<List<User>> listUser(String filter);\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/provider/AuthenticationProvider.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.provider;\n\nimport com.google.protobuf.GeneratedMessageV3;\nimport io.grpc.Metadata;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.function.Supplier;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic interface AuthenticationProvider<AuthenticationContext> {\n\n    void initialize(AuthConfig config, Supplier<?> metadataService);\n\n    CompletableFuture<Void> authenticate(AuthenticationContext context);\n\n    AuthenticationContext newContext(Metadata metadata, GeneratedMessageV3 request);\n\n    AuthenticationContext newContext(ChannelHandlerContext context, RemotingCommand command);\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/provider/DefaultAuthenticationProvider.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.provider;\n\nimport com.google.protobuf.GeneratedMessageV3;\nimport io.grpc.Metadata;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.function.Supplier;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.auth.authentication.builder.AuthenticationContextBuilder;\nimport org.apache.rocketmq.auth.authentication.builder.DefaultAuthenticationContextBuilder;\nimport org.apache.rocketmq.auth.authentication.context.DefaultAuthenticationContext;\nimport org.apache.rocketmq.auth.authentication.chain.DefaultAuthenticationHandler;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.chain.HandlerChain;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\npublic class DefaultAuthenticationProvider implements AuthenticationProvider<DefaultAuthenticationContext> {\n\n    protected final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_AUTH_AUDIT_LOGGER_NAME);\n    protected AuthConfig authConfig;\n    protected Supplier<?> metadataService;\n    protected AuthenticationContextBuilder<DefaultAuthenticationContext> authenticationContextBuilder;\n\n    @Override\n    public void initialize(AuthConfig config, Supplier<?> metadataService) {\n        this.authConfig = config;\n        this.metadataService = metadataService;\n        this.authenticationContextBuilder = new DefaultAuthenticationContextBuilder();\n    }\n\n    @Override\n    public CompletableFuture<Void> authenticate(DefaultAuthenticationContext context) {\n        return this.newHandlerChain().handle(context)\n            .whenComplete((nil, ex) -> doAuditLog(context, ex));\n    }\n\n    @Override\n    public DefaultAuthenticationContext newContext(Metadata metadata, GeneratedMessageV3 request) {\n        return this.authenticationContextBuilder.build(metadata, request);\n    }\n\n    @Override\n    public DefaultAuthenticationContext newContext(ChannelHandlerContext context, RemotingCommand command) {\n        return this.authenticationContextBuilder.build(context, command);\n    }\n\n    protected HandlerChain<DefaultAuthenticationContext, CompletableFuture<Void>> newHandlerChain() {\n        return HandlerChain.<DefaultAuthenticationContext, CompletableFuture<Void>>create()\n            .addNext(new DefaultAuthenticationHandler(this.authConfig, metadataService));\n    }\n\n    protected void doAuditLog(DefaultAuthenticationContext context, Throwable ex) {\n        if (StringUtils.isBlank(context.getUsername())) {\n            return;\n        }\n        if (ex != null) {\n            log.info(\"[AUTHENTICATION] User:{} is authenticated failed with Signature = {}.\", context.getUsername(), context.getSignature());\n        } else {\n            log.debug(\"[AUTHENTICATION] User:{} is authenticated success with Signature = {}.\", context.getUsername(), context.getSignature());\n        }\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/provider/LocalAuthenticationMetadataProvider.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.provider;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.github.benmanes.caffeine.cache.CacheLoader;\nimport com.github.benmanes.caffeine.cache.Caffeine;\nimport com.github.benmanes.caffeine.cache.LoadingCache;\nimport java.io.File;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport java.util.function.Supplier;\nimport org.apache.commons.lang3.ArrayUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.auth.authentication.exception.AuthenticationException;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.config.ConfigRocksDBStorage;\nimport org.apache.rocketmq.common.thread.ThreadPoolMonitor;\nimport org.rocksdb.RocksDB;\n\npublic class LocalAuthenticationMetadataProvider implements AuthenticationMetadataProvider {\n\n    private final static String AUTH_METADATA_COLUMN_FAMILY = new String(RocksDB.DEFAULT_COLUMN_FAMILY,\n        StandardCharsets.UTF_8);\n\n    private ConfigRocksDBStorage storage;\n\n    private LoadingCache<String, User> userCache;\n\n    protected ThreadPoolExecutor cacheRefreshExecutor;\n\n    @Override\n    public void initialize(AuthConfig authConfig, Supplier<?> metadataService) {\n        this.storage = ConfigRocksDBStorage.getStore(authConfig.getAuthConfigPath() + File.separator + \"users\", false);\n        if (!this.storage.start()) {\n            throw new RuntimeException(\"Failed to load rocksdb for auth_user, please check whether it is occupied\");\n        }\n\n        this.cacheRefreshExecutor = ThreadPoolMonitor.createAndMonitor(\n            1,\n            1,\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            \"UserCacheRefresh\",\n            100000\n        );\n\n        this.userCache = Caffeine.newBuilder()\n            .maximumSize(authConfig.getUserCacheMaxNum())\n            .expireAfterAccess(authConfig.getUserCacheExpiredSecond(), TimeUnit.SECONDS)\n            .refreshAfterWrite(authConfig.getUserCacheRefreshSecond(), TimeUnit.SECONDS)\n            .executor(cacheRefreshExecutor)\n            .build(new UserCacheLoader(this.storage));\n    }\n\n    @Override\n    public CompletableFuture<Void> createUser(User user) {\n        try {\n            byte[] keyBytes = user.getUsername().getBytes(StandardCharsets.UTF_8);\n            byte[] valueBytes = JSON.toJSONBytes(user);\n            this.storage.put(AUTH_METADATA_COLUMN_FAMILY, keyBytes, keyBytes.length, valueBytes);\n            this.storage.flushWAL();\n            this.userCache.invalidate(user.getUsername());\n        } catch (Exception e) {\n            throw new AuthenticationException(\"create user to RocksDB failed\", e);\n        }\n        return CompletableFuture.completedFuture(null);\n    }\n\n    @Override\n    public CompletableFuture<Void> deleteUser(String username) {\n        try {\n            this.storage.delete(AUTH_METADATA_COLUMN_FAMILY, username.getBytes(StandardCharsets.UTF_8));\n            this.storage.flushWAL();\n            this.userCache.invalidate(username);\n        } catch (Exception e) {\n            throw new AuthenticationException(\"delete user from RocksDB failed\", e);\n        }\n        return CompletableFuture.completedFuture(null);\n    }\n\n    @Override\n    public CompletableFuture<Void> updateUser(User user) {\n        try {\n            byte[] keyBytes = user.getUsername().getBytes(StandardCharsets.UTF_8);\n            byte[] valueBytes = JSON.toJSONBytes(user);\n            this.storage.put(AUTH_METADATA_COLUMN_FAMILY, keyBytes, keyBytes.length, valueBytes);\n            this.storage.flushWAL();\n            this.userCache.invalidate(user.getUsername());\n        } catch (Exception e) {\n            throw new AuthenticationException(\"update user to RocksDB failed\", e);\n        }\n        return CompletableFuture.completedFuture(null);\n    }\n\n    @Override\n    public CompletableFuture<User> getUser(String username) {\n        User user = this.userCache.get(username);\n        if (user == UserCacheLoader.EMPTY_USER) {\n            return CompletableFuture.completedFuture(null);\n        }\n        return CompletableFuture.completedFuture(user);\n    }\n\n    @Override\n    public CompletableFuture<List<User>> listUser(String filter) {\n        List<User> result = new ArrayList<>();\n        CompletableFuture<List<User>> future = new CompletableFuture<>();\n        try {\n            this.storage.iterate(AUTH_METADATA_COLUMN_FAMILY, (key, value) -> {\n                String username = new String(key, StandardCharsets.UTF_8);\n                if (StringUtils.isNotBlank(filter) && !username.contains(filter)) {\n                    return;\n                }\n                User user = JSON.parseObject(new String(value, StandardCharsets.UTF_8), User.class);\n                result.add(user);\n            });\n        } catch (Exception e) {\n            future.completeExceptionally(e);\n        }\n        future.complete(result);\n        return future;\n    }\n\n    @Override\n    public void shutdown() {\n        if (this.storage != null) {\n            this.storage.shutdown();\n        }\n        if (this.cacheRefreshExecutor != null) {\n            this.cacheRefreshExecutor.shutdown();\n        }\n    }\n\n    private static class UserCacheLoader implements CacheLoader<String, User> {\n        private final ConfigRocksDBStorage storage;\n        public static final User EMPTY_USER = new User();\n\n        public UserCacheLoader(ConfigRocksDBStorage storage) {\n            this.storage = storage;\n        }\n\n        @Override\n        public User load(String username) {\n            try {\n                byte[] keyBytes = username.getBytes(StandardCharsets.UTF_8);\n                byte[] valueBytes = storage.get(AUTH_METADATA_COLUMN_FAMILY, keyBytes);\n                if (ArrayUtils.isEmpty(valueBytes)) {\n                    return EMPTY_USER;\n                }\n                return JSON.parseObject(new String(valueBytes, StandardCharsets.UTF_8), User.class);\n            } catch (Exception e) {\n                throw new AuthenticationException(\"Get user from RocksDB failed.\", e);\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/strategy/AbstractAuthenticationStrategy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.strategy;\n\nimport java.util.HashSet;\nimport java.util.Set;\nimport java.util.function.Supplier;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.auth.authentication.context.AuthenticationContext;\nimport org.apache.rocketmq.auth.authentication.exception.AuthenticationException;\nimport org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;\nimport org.apache.rocketmq.auth.authentication.provider.AuthenticationProvider;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.utils.ExceptionUtils;\n\npublic abstract class AbstractAuthenticationStrategy implements AuthenticationStrategy {\n\n    protected final AuthConfig authConfig;\n    protected final Set<String> authenticationWhiteSet = new HashSet<>();\n    protected final AuthenticationProvider<AuthenticationContext> authenticationProvider;\n\n    public AbstractAuthenticationStrategy(AuthConfig authConfig, Supplier<?> metadataService) {\n        this.authConfig = authConfig;\n        this.authenticationProvider = AuthenticationFactory.getProvider(authConfig);\n        if (this.authenticationProvider != null) {\n            this.authenticationProvider.initialize(authConfig, metadataService);\n        }\n        if (StringUtils.isNotBlank(authConfig.getAuthenticationWhitelist())) {\n            String[] whitelist = StringUtils.split(authConfig.getAuthenticationWhitelist(), \",\");\n            for (String rpcCode : whitelist) {\n                this.authenticationWhiteSet.add(StringUtils.trim(rpcCode));\n            }\n        }\n    }\n\n    protected void doEvaluate(AuthenticationContext context) {\n        if (context == null) {\n            return;\n        }\n        if (!authConfig.isAuthenticationEnabled()) {\n            return;\n        }\n        if (this.authenticationProvider == null) {\n            return;\n        }\n        if (this.authenticationWhiteSet.contains(context.getRpcCode())) {\n            return;\n        }\n        try {\n            this.authenticationProvider.authenticate(context).join();\n        } catch (AuthenticationException ex) {\n            throw ex;\n        } catch (Throwable ex) {\n            Throwable exception = ExceptionUtils.getRealException(ex);\n            if (exception instanceof AuthenticationException) {\n                throw (AuthenticationException) exception;\n            }\n            throw new AuthenticationException(\"Authentication failed. Please verify the credentials and try again.\", exception);\n        }\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/strategy/AuthenticationStrategy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.strategy;\n\nimport org.apache.rocketmq.auth.authentication.context.AuthenticationContext;\n\npublic interface AuthenticationStrategy {\n\n    void evaluate(AuthenticationContext context);\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/strategy/StatefulAuthenticationStrategy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.strategy;\n\nimport com.github.benmanes.caffeine.cache.Cache;\nimport com.github.benmanes.caffeine.cache.Caffeine;\nimport java.util.concurrent.TimeUnit;\nimport java.util.function.Supplier;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.auth.authentication.context.AuthenticationContext;\nimport org.apache.rocketmq.auth.authentication.context.DefaultAuthenticationContext;\nimport org.apache.rocketmq.auth.authentication.exception.AuthenticationException;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.constant.CommonConstants;\n\npublic class StatefulAuthenticationStrategy extends AbstractAuthenticationStrategy {\n\n    protected Cache<String, Pair<Boolean, AuthenticationException>> authCache;\n\n    public StatefulAuthenticationStrategy(AuthConfig authConfig, Supplier<?> metadataService) {\n        super(authConfig, metadataService);\n        this.authCache = Caffeine.newBuilder()\n            .expireAfterWrite(authConfig.getStatefulAuthenticationCacheExpiredSecond(), TimeUnit.SECONDS)\n            .maximumSize(authConfig.getStatefulAuthenticationCacheMaxNum())\n            .build();\n    }\n\n    @Override\n    public void evaluate(AuthenticationContext context) {\n        if (StringUtils.isBlank(context.getChannelId())) {\n            this.doEvaluate(context);\n            return;\n        }\n        Pair<Boolean, AuthenticationException> result = this.authCache.get(buildKey(context), key -> {\n            try {\n                this.doEvaluate(context);\n                return Pair.of(true, null);\n            } catch (AuthenticationException ex) {\n                return Pair.of(false, ex);\n            }\n        });\n        if (result != null && result.getObject1() == Boolean.FALSE) {\n            throw result.getObject2();\n        }\n    }\n\n    private String buildKey(AuthenticationContext context) {\n        if (context instanceof DefaultAuthenticationContext) {\n            DefaultAuthenticationContext ctx = (DefaultAuthenticationContext) context;\n            if (StringUtils.isBlank(ctx.getUsername())) {\n                return ctx.getChannelId();\n            }\n            return ctx.getChannelId() + CommonConstants.POUND + ctx.getUsername();\n        }\n        throw new AuthenticationException(\"The request of {} is not support.\", context.getClass().getSimpleName());\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authentication/strategy/StatelessAuthenticationStrategy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.strategy;\n\nimport java.util.function.Supplier;\nimport org.apache.rocketmq.auth.authentication.context.AuthenticationContext;\nimport org.apache.rocketmq.auth.config.AuthConfig;\n\npublic class StatelessAuthenticationStrategy extends AbstractAuthenticationStrategy {\n\n    public StatelessAuthenticationStrategy(AuthConfig authConfig, Supplier<?> metadataService) {\n        super(authConfig, metadataService);\n    }\n\n    @Override\n    public void evaluate(AuthenticationContext context) {\n        super.doEvaluate(context);\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/AuthorizationEvaluator.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization;\n\nimport java.util.List;\nimport java.util.function.Supplier;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.rocketmq.auth.authorization.context.AuthorizationContext;\nimport org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;\nimport org.apache.rocketmq.auth.authorization.strategy.AuthorizationStrategy;\nimport org.apache.rocketmq.auth.config.AuthConfig;\n\npublic class AuthorizationEvaluator {\n\n    private final AuthorizationStrategy authorizationStrategy;\n\n    public AuthorizationEvaluator(AuthConfig authConfig) {\n        this(authConfig, null);\n    }\n\n    public AuthorizationEvaluator(AuthConfig authConfig, Supplier<?> metadataService) {\n        this.authorizationStrategy = AuthorizationFactory.getStrategy(authConfig, metadataService);\n    }\n\n    public void evaluate(List<AuthorizationContext> contexts) {\n        if (CollectionUtils.isEmpty(contexts)) {\n            return;\n        }\n        contexts.forEach(this.authorizationStrategy::evaluate);\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/AuthorizationContextBuilder.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.builder;\n\nimport com.google.protobuf.GeneratedMessageV3;\nimport io.grpc.Metadata;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.List;\nimport org.apache.rocketmq.auth.authorization.context.DefaultAuthorizationContext;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic interface AuthorizationContextBuilder {\n\n    List<DefaultAuthorizationContext> build(Metadata metadata, GeneratedMessageV3 message);\n\n    List<DefaultAuthorizationContext> build(ChannelHandlerContext context, RemotingCommand command);\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.builder;\n\nimport apache.rocketmq.v2.AckMessageRequest;\nimport apache.rocketmq.v2.ChangeInvisibleDurationRequest;\nimport apache.rocketmq.v2.ClientType;\nimport apache.rocketmq.v2.EndTransactionRequest;\nimport apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest;\nimport apache.rocketmq.v2.HeartbeatRequest;\nimport apache.rocketmq.v2.NotifyClientTerminationRequest;\nimport apache.rocketmq.v2.QueryAssignmentRequest;\nimport apache.rocketmq.v2.QueryRouteRequest;\nimport apache.rocketmq.v2.RecallMessageRequest;\nimport apache.rocketmq.v2.ReceiveMessageRequest;\nimport apache.rocketmq.v2.SendMessageRequest;\nimport apache.rocketmq.v2.Subscription;\nimport apache.rocketmq.v2.SubscriptionEntry;\nimport apache.rocketmq.v2.TelemetryCommand;\nimport apache.rocketmq.v2.SyncLiteSubscriptionRequest;\nimport com.google.protobuf.GeneratedMessageV3;\nimport io.grpc.Metadata;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.lang.reflect.Field;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.EnumSet;\nimport java.util.HashMap;\nimport java.util.List;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.collections.MapUtils;\nimport org.apache.commons.lang3.ArrayUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.acl.common.AclException;\nimport org.apache.rocketmq.acl.common.SessionCredentials;\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.auth.authorization.context.DefaultAuthorizationContext;\nimport org.apache.rocketmq.auth.authorization.exception.AuthorizationException;\nimport org.apache.rocketmq.auth.authorization.model.Resource;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.constant.CommonConstants;\nimport org.apache.rocketmq.common.constant.GrpcConstants;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.resource.ResourcePattern;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.RequestHeaderRegistry;\nimport org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UnregisterClientRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumerData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\npublic class DefaultAuthorizationContextBuilder implements AuthorizationContextBuilder {\n\n    private static final String TOPIC = \"topic\";\n    private static final String GROUP = \"group\";\n    private static final String A = \"a\";\n    private static final String B = \"b\";\n    private static final String CONSUMER_GROUP = \"consumerGroup\";\n    private final AuthConfig authConfig;\n    private static final EnumSet<ClientType> CONSUMER_CLIENT_TYPES =\n            EnumSet.of(ClientType.PUSH_CONSUMER, ClientType.SIMPLE_CONSUMER, ClientType.PULL_CONSUMER);\n\n    private final RequestHeaderRegistry requestHeaderRegistry;\n\n    public DefaultAuthorizationContextBuilder(AuthConfig authConfig) {\n        this.authConfig = authConfig;\n        this.requestHeaderRegistry = RequestHeaderRegistry.getInstance();\n    }\n\n    @Override\n    public List<DefaultAuthorizationContext> build(Metadata metadata, GeneratedMessageV3 message) {\n        List<DefaultAuthorizationContext> result = null;\n        if (message instanceof SendMessageRequest) {\n            SendMessageRequest request = (SendMessageRequest) message;\n            if (request.getMessagesCount() <= 0) {\n                throw new AuthorizationException(\"message is null.\");\n            }\n            result = newPubContext(metadata, request.getMessages(0).getTopic());\n        }\n        if (message instanceof RecallMessageRequest) {\n            RecallMessageRequest request = (RecallMessageRequest) message;\n            result = newPubContext(metadata, request.getTopic());\n        }\n        if (message instanceof EndTransactionRequest) {\n            EndTransactionRequest request = (EndTransactionRequest) message;\n            result = newPubContext(metadata, request.getTopic());\n        }\n        if (message instanceof HeartbeatRequest) {\n            HeartbeatRequest request = (HeartbeatRequest) message;\n            if (!isConsumerClientType(request.getClientType())) {\n                return null;\n            }\n            result = newGroupSubContexts(metadata, request.getGroup());\n        }\n        if (message instanceof ReceiveMessageRequest) {\n            ReceiveMessageRequest request = (ReceiveMessageRequest) message;\n            if (!request.hasMessageQueue()) {\n                throw new AuthorizationException(\"messageQueue is null.\");\n            }\n            result = newSubContexts(metadata, request.getGroup(), request.getMessageQueue().getTopic());\n        }\n        if (message instanceof SyncLiteSubscriptionRequest) {\n            SyncLiteSubscriptionRequest request = (SyncLiteSubscriptionRequest) message;\n            if (request.getLiteTopicSetCount() <= 0) {\n                return null;\n            }\n            result = newSubContexts(metadata, request.getGroup(), request.getTopic());\n        }\n        if (message instanceof AckMessageRequest) {\n            AckMessageRequest request = (AckMessageRequest) message;\n            result = newSubContexts(metadata, request.getGroup(), request.getTopic());\n        }\n        if (message instanceof ForwardMessageToDeadLetterQueueRequest) {\n            ForwardMessageToDeadLetterQueueRequest request = (ForwardMessageToDeadLetterQueueRequest) message;\n            result = newSubContexts(metadata, request.getGroup(), request.getTopic());\n        }\n        if (message instanceof NotifyClientTerminationRequest) {\n            NotifyClientTerminationRequest request = (NotifyClientTerminationRequest) message;\n            if (StringUtils.isNotBlank(request.getGroup().getName())) {\n                result = newGroupSubContexts(metadata, request.getGroup());\n            }\n        }\n        if (message instanceof ChangeInvisibleDurationRequest) {\n            ChangeInvisibleDurationRequest request = (ChangeInvisibleDurationRequest) message;\n            result = newGroupSubContexts(metadata, request.getGroup());\n        }\n        if (message instanceof QueryRouteRequest) {\n            QueryRouteRequest request = (QueryRouteRequest) message;\n            result = newContext(metadata, request);\n        }\n        if (message instanceof QueryAssignmentRequest) {\n            QueryAssignmentRequest request = (QueryAssignmentRequest) message;\n            result = newSubContexts(metadata, request.getGroup(), request.getTopic());\n        }\n        if (message instanceof TelemetryCommand) {\n            TelemetryCommand request = (TelemetryCommand) message;\n            result = newContext(metadata, request);\n        }\n        if (CollectionUtils.isNotEmpty(result)) {\n            result.forEach(context -> {\n                context.setChannelId(metadata.get(GrpcConstants.CHANNEL_ID));\n                context.setRpcCode(message.getDescriptorForType().getFullName());\n            });\n        }\n        return result;\n    }\n\n    @Override\n    public List<DefaultAuthorizationContext> build(ChannelHandlerContext context, RemotingCommand command) {\n        List<DefaultAuthorizationContext> result = new ArrayList<>();\n        try {\n            HashMap<String, String> fields = command.getExtFields();\n            if (MapUtils.isEmpty(fields)) {\n                return result;\n            }\n            Subject subject = null;\n            if (fields.containsKey(SessionCredentials.ACCESS_KEY)) {\n                subject = User.of(fields.get(SessionCredentials.ACCESS_KEY));\n            }\n            String remoteAddr = RemotingHelper.parseChannelRemoteAddr(context.channel());\n            String sourceIp = StringUtils.substringBeforeLast(remoteAddr, CommonConstants.COLON);\n\n            Resource topic;\n            Resource group;\n            switch (command.getCode()) {\n                case RequestCode.GET_ROUTEINFO_BY_TOPIC:\n                    if (NamespaceUtil.isRetryTopic(fields.get(TOPIC))) {\n                        group = Resource.ofGroup(fields.get(TOPIC));\n                        result.add(DefaultAuthorizationContext.of(subject, group, Arrays.asList(Action.SUB, Action.GET), sourceIp));\n                    } else {\n                        topic = Resource.ofTopic(fields.get(TOPIC));\n                        result.add(DefaultAuthorizationContext.of(subject, topic, Arrays.asList(Action.PUB, Action.SUB, Action.GET), sourceIp));\n                    }\n                    break;\n                case RequestCode.SEND_MESSAGE:\n                    if (NamespaceUtil.isRetryTopic(fields.get(TOPIC))) {\n                        group = Resource.ofGroup(fields.get(TOPIC));\n                        result.add(DefaultAuthorizationContext.of(subject, group, Action.SUB, sourceIp));\n                    } else {\n                        topic = Resource.ofTopic(fields.get(TOPIC));\n                        result.add(DefaultAuthorizationContext.of(subject, topic, Action.PUB, sourceIp));\n                    }\n                    break;\n                case RequestCode.SEND_MESSAGE_V2:\n                case RequestCode.SEND_BATCH_MESSAGE:\n                    if (NamespaceUtil.isRetryTopic(fields.get(B))) {\n                        group = Resource.ofGroup(fields.get(B));\n                        result.add(DefaultAuthorizationContext.of(subject, group, Action.SUB, sourceIp));\n                    } else {\n                        topic = Resource.ofTopic(fields.get(B));\n                        result.add(DefaultAuthorizationContext.of(subject, topic, Action.PUB, sourceIp));\n                    }\n                    break;\n                case RequestCode.RECALL_MESSAGE:\n                    topic = Resource.ofTopic(fields.get(TOPIC));\n                    result.add(DefaultAuthorizationContext.of(subject, topic, Action.PUB, sourceIp));\n                    break;\n                case RequestCode.END_TRANSACTION:\n                    if (StringUtils.isNotBlank(fields.get(TOPIC))) {\n                        topic = Resource.ofTopic(fields.get(TOPIC));\n                        result.add(DefaultAuthorizationContext.of(subject, topic, Action.PUB, sourceIp));\n                    }\n                    break;\n                case RequestCode.CONSUMER_SEND_MSG_BACK:\n                    group = Resource.ofGroup(fields.get(GROUP));\n                    result.add(DefaultAuthorizationContext.of(subject, group, Action.SUB, sourceIp));\n                    break;\n                case RequestCode.PULL_MESSAGE:\n                    if (!NamespaceUtil.isRetryTopic(fields.get(TOPIC))) {\n                        topic = Resource.ofTopic(fields.get(TOPIC));\n                        result.add(DefaultAuthorizationContext.of(subject, topic, Action.SUB, sourceIp));\n                    }\n                    group = Resource.ofGroup(fields.get(CONSUMER_GROUP));\n                    result.add(DefaultAuthorizationContext.of(subject, group, Action.SUB, sourceIp));\n                    break;\n                case RequestCode.QUERY_MESSAGE:\n                    topic = Resource.ofTopic(fields.get(TOPIC));\n                    result.add(DefaultAuthorizationContext.of(subject, topic, Arrays.asList(Action.SUB, Action.GET), sourceIp));\n                    break;\n                case RequestCode.HEART_BEAT:\n                    HeartbeatData heartbeatData = HeartbeatData.decode(command.getBody(), HeartbeatData.class);\n                    for (ConsumerData data : heartbeatData.getConsumerDataSet()) {\n                        group = Resource.ofGroup(data.getGroupName());\n                        result.add(DefaultAuthorizationContext.of(subject, group, Action.SUB, sourceIp));\n                        for (SubscriptionData subscriptionData : data.getSubscriptionDataSet()) {\n                            if (NamespaceUtil.isRetryTopic(subscriptionData.getTopic())) {\n                                continue;\n                            }\n                            topic = Resource.ofTopic(subscriptionData.getTopic());\n                            result.add(DefaultAuthorizationContext.of(subject, topic, Action.SUB, sourceIp));\n                        }\n                    }\n                    break;\n                case RequestCode.UNREGISTER_CLIENT:\n                    final UnregisterClientRequestHeader unregisterClientRequestHeader =\n                        command.decodeCommandCustomHeader(UnregisterClientRequestHeader.class);\n                    if (StringUtils.isNotBlank(unregisterClientRequestHeader.getConsumerGroup())) {\n                        group = Resource.ofGroup(unregisterClientRequestHeader.getConsumerGroup());\n                        result.add(DefaultAuthorizationContext.of(subject, group, Action.SUB, sourceIp));\n                    }\n                    break;\n                case RequestCode.GET_CONSUMER_LIST_BY_GROUP:\n                    final GetConsumerListByGroupRequestHeader getConsumerListByGroupRequestHeader =\n                        command.decodeCommandCustomHeader(GetConsumerListByGroupRequestHeader.class);\n                    group = Resource.ofGroup(getConsumerListByGroupRequestHeader.getConsumerGroup());\n                    result.add(DefaultAuthorizationContext.of(subject, group, Arrays.asList(Action.SUB, Action.GET), sourceIp));\n                    break;\n                case RequestCode.QUERY_CONSUMER_OFFSET:\n                    final QueryConsumerOffsetRequestHeader queryConsumerOffsetRequestHeader =\n                        command.decodeCommandCustomHeader(QueryConsumerOffsetRequestHeader.class);\n                    if (!NamespaceUtil.isRetryTopic(queryConsumerOffsetRequestHeader.getTopic())) {\n                        topic = Resource.ofTopic(queryConsumerOffsetRequestHeader.getTopic());\n                        result.add(DefaultAuthorizationContext.of(subject, topic, Arrays.asList(Action.SUB, Action.GET), sourceIp));\n                    }\n                    group = Resource.ofGroup(queryConsumerOffsetRequestHeader.getConsumerGroup());\n                    result.add(DefaultAuthorizationContext.of(subject, group, Arrays.asList(Action.SUB, Action.GET), sourceIp));\n                    break;\n                case RequestCode.UPDATE_CONSUMER_OFFSET:\n                    final UpdateConsumerOffsetRequestHeader updateConsumerOffsetRequestHeader =\n                        command.decodeCommandCustomHeader(UpdateConsumerOffsetRequestHeader.class);\n                    if (!NamespaceUtil.isRetryTopic(updateConsumerOffsetRequestHeader.getTopic())) {\n                        topic = Resource.ofTopic(updateConsumerOffsetRequestHeader.getTopic());\n                        result.add(DefaultAuthorizationContext.of(subject, topic, Arrays.asList(Action.SUB, Action.UPDATE), sourceIp));\n                    }\n                    group = Resource.ofGroup(updateConsumerOffsetRequestHeader.getConsumerGroup());\n                    result.add(DefaultAuthorizationContext.of(subject, group, Arrays.asList(Action.SUB, Action.UPDATE), sourceIp));\n                    break;\n                case RequestCode.LOCK_BATCH_MQ:\n                    LockBatchRequestBody lockBatchRequestBody = LockBatchRequestBody.decode(command.getBody(), LockBatchRequestBody.class);\n                    group = Resource.ofGroup(lockBatchRequestBody.getConsumerGroup());\n                    result.add(DefaultAuthorizationContext.of(subject, group, Action.SUB, sourceIp));\n                    if (CollectionUtils.isNotEmpty(lockBatchRequestBody.getMqSet())) {\n                        for (MessageQueue messageQueue : lockBatchRequestBody.getMqSet()) {\n                            if (NamespaceUtil.isRetryTopic(messageQueue.getTopic())) {\n                                continue;\n                            }\n                            topic = Resource.ofTopic(messageQueue.getTopic());\n                            result.add(DefaultAuthorizationContext.of(subject, topic, Action.SUB, sourceIp));\n                        }\n                    }\n                    break;\n                case RequestCode.UNLOCK_BATCH_MQ:\n                    UnlockBatchRequestBody unlockBatchRequestBody = UnlockBatchRequestBody.decode(command.getBody(), UnlockBatchRequestBody.class);\n                    group = Resource.ofGroup(unlockBatchRequestBody.getConsumerGroup());\n                    result.add(DefaultAuthorizationContext.of(subject, group, Action.SUB, sourceIp));\n                    if (CollectionUtils.isNotEmpty(unlockBatchRequestBody.getMqSet())) {\n                        for (MessageQueue messageQueue : unlockBatchRequestBody.getMqSet()) {\n                            if (NamespaceUtil.isRetryTopic(messageQueue.getTopic())) {\n                                continue;\n                            }\n                            topic = Resource.ofTopic(messageQueue.getTopic());\n                            result.add(DefaultAuthorizationContext.of(subject, topic, Action.SUB, sourceIp));\n                        }\n                    }\n                    break;\n                default:\n                    result = buildContextByAnnotation(subject, command, sourceIp);\n                    break;\n            }\n            if (CollectionUtils.isNotEmpty(result)) {\n                result.forEach(r -> {\n                    r.setChannelId(context.channel().id().asLongText());\n                    r.setRpcCode(String.valueOf(command.getCode()));\n                });\n            }\n        } catch (AuthorizationException ex) {\n            throw ex;\n        } catch (Throwable t) {\n            throw new AuthorizationException(\"parse authorization context error.\", t);\n        }\n        return result;\n    }\n\n    private List<DefaultAuthorizationContext> buildContextByAnnotation(Subject subject, RemotingCommand request,\n        String sourceIp) throws Exception {\n        List<DefaultAuthorizationContext> result = new ArrayList<>();\n\n        Class<? extends CommandCustomHeader> clazz = this.requestHeaderRegistry.getRequestHeader(request.getCode());\n        if (clazz == null) {\n            return result;\n        }\n        CommandCustomHeader header = request.decodeCommandCustomHeader(clazz);\n\n        RocketMQAction rocketMQAction = clazz.getAnnotation(RocketMQAction.class);\n        ResourceType resourceType = rocketMQAction.resource();\n        Action[] actions = rocketMQAction.action();\n        Resource resource = null;\n        if (resourceType == ResourceType.CLUSTER) {\n            resource = Resource.ofCluster(authConfig.getClusterName());\n        }\n\n        Field[] fields = clazz.getDeclaredFields();\n        if (ArrayUtils.isNotEmpty(fields)) {\n            for (Field field : fields) {\n                RocketMQResource rocketMQResource = field.getAnnotation(RocketMQResource.class);\n                if (rocketMQResource == null) {\n                    continue;\n                }\n                field.setAccessible(true);\n                try {\n                    resourceType = rocketMQResource.value();\n                    String splitter = rocketMQResource.splitter();\n                    Object value = field.get(header);\n                    if (value == null) {\n                        continue;\n                    }\n                    String[] resourceValues;\n                    if (StringUtils.isNotBlank(splitter)) {\n                        resourceValues = StringUtils.split(value.toString(), splitter);\n                    } else {\n                        resourceValues = new String[] {value.toString()};\n                    }\n                    for (String resourceValue : resourceValues) {\n                        if (resourceType == ResourceType.TOPIC && NamespaceUtil.isRetryTopic(resourceValue)) {\n                            resource = Resource.ofGroup(resourceValue);\n                            result.add(DefaultAuthorizationContext.of(subject, resource, Arrays.asList(actions), sourceIp));\n                        } else {\n                            resource = Resource.of(resourceType, resourceValue, ResourcePattern.LITERAL);\n                            result.add(DefaultAuthorizationContext.of(subject, resource, Arrays.asList(actions), sourceIp));\n                        }\n                    }\n                } finally {\n                    field.setAccessible(false);\n                }\n            }\n        }\n\n        if (CollectionUtils.isEmpty(result) && resource != null) {\n            result.add(DefaultAuthorizationContext.of(subject, resource, Arrays.asList(actions), sourceIp));\n        }\n\n        return result;\n    }\n\n    private List<DefaultAuthorizationContext> newContext(Metadata metadata, QueryRouteRequest request) {\n        apache.rocketmq.v2.Resource topic = request.getTopic();\n        if (StringUtils.isBlank(topic.getName())) {\n            throw new AuthorizationException(\"topic is null.\");\n        }\n        Subject subject = null;\n        if (metadata.containsKey(GrpcConstants.AUTHORIZATION_AK)) {\n            subject = User.of(metadata.get(GrpcConstants.AUTHORIZATION_AK));\n        }\n        Resource resource = Resource.ofTopic(topic.getName());\n        String sourceIp = StringUtils.substringBeforeLast(metadata.get(GrpcConstants.REMOTE_ADDRESS), CommonConstants.COLON);\n        DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, Arrays.asList(Action.PUB, Action.SUB), sourceIp);\n        return Collections.singletonList(context);\n    }\n\n    private static List<DefaultAuthorizationContext> newContext(Metadata metadata, TelemetryCommand request) {\n        if (request.getCommandCase() != TelemetryCommand.CommandCase.SETTINGS) {\n            return null;\n        }\n        if (!request.getSettings().hasPublishing() && !request.getSettings().hasSubscription()) {\n            throw new AclException(\"settings command doesn't have publishing or subscription.\");\n        }\n        List<DefaultAuthorizationContext> result = new ArrayList<>();\n        if (request.getSettings().hasPublishing()) {\n            List<apache.rocketmq.v2.Resource> topicList = request.getSettings().getPublishing().getTopicsList();\n            for (apache.rocketmq.v2.Resource topic : topicList) {\n                result.addAll(newPubContext(metadata, topic));\n            }\n        }\n        if (request.getSettings().hasSubscription()) {\n            Subscription subscription = request.getSettings().getSubscription();\n            result.addAll(newSubContexts(metadata, ResourceType.GROUP, subscription.getGroup()));\n            for (SubscriptionEntry entry : subscription.getSubscriptionsList()) {\n                result.addAll(newSubContexts(metadata, ResourceType.TOPIC, entry.getTopic()));\n            }\n        }\n        return result;\n    }\n\n    private boolean isConsumerClientType(ClientType clientType) {\n        return CONSUMER_CLIENT_TYPES.contains(clientType);\n    }\n\n    private static List<DefaultAuthorizationContext> newPubContext(Metadata metadata, apache.rocketmq.v2.Resource topic) {\n        if (topic == null || StringUtils.isBlank(topic.getName())) {\n            throw new AuthorizationException(\"topic is null.\");\n        }\n        Subject subject = null;\n        if (metadata.containsKey(GrpcConstants.AUTHORIZATION_AK)) {\n            subject = User.of(metadata.get(GrpcConstants.AUTHORIZATION_AK));\n        }\n        Resource resource = Resource.ofTopic(topic.getName());\n        String sourceIp = StringUtils.substringBeforeLast(metadata.get(GrpcConstants.REMOTE_ADDRESS), CommonConstants.COLON);\n        DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, Action.PUB, sourceIp);\n        return Collections.singletonList(context);\n    }\n\n    private List<DefaultAuthorizationContext> newSubContexts(Metadata metadata, apache.rocketmq.v2.Resource group,\n        apache.rocketmq.v2.Resource topic) {\n        List<DefaultAuthorizationContext> result = new ArrayList<>();\n        result.addAll(newGroupSubContexts(metadata, group));\n        result.addAll(newTopicSubContexts(metadata, topic));\n        return result;\n    }\n\n    private static List<DefaultAuthorizationContext> newTopicSubContexts(Metadata metadata,\n        apache.rocketmq.v2.Resource resource) {\n        return newSubContexts(metadata, ResourceType.TOPIC, resource);\n    }\n\n    private static List<DefaultAuthorizationContext> newGroupSubContexts(Metadata metadata,\n        apache.rocketmq.v2.Resource resource) {\n        return newSubContexts(metadata, ResourceType.GROUP, resource);\n    }\n\n    private static List<DefaultAuthorizationContext> newSubContexts(Metadata metadata, ResourceType resourceType,\n        apache.rocketmq.v2.Resource resource) {\n        if (resourceType == ResourceType.GROUP) {\n            if (resource == null || StringUtils.isBlank(resource.getName())) {\n                throw new AuthorizationException(\"group is null.\");\n            }\n            return newSubContexts(metadata, Resource.ofGroup(resource.getName()));\n        }\n        if (resourceType == ResourceType.TOPIC) {\n            if (resource == null || StringUtils.isBlank(resource.getName())) {\n                throw new AuthorizationException(\"topic is null.\");\n            }\n            return newSubContexts(metadata, Resource.ofTopic(resource.getName()));\n        }\n        throw new AuthorizationException(\"unknown resource type.\");\n    }\n\n    private static List<DefaultAuthorizationContext> newSubContexts(Metadata metadata, Resource resource) {\n        List<DefaultAuthorizationContext> result = new ArrayList<>();\n        Subject subject = null;\n        if (metadata.containsKey(GrpcConstants.AUTHORIZATION_AK)) {\n            subject = User.of(metadata.get(GrpcConstants.AUTHORIZATION_AK));\n        }\n        String sourceIp = StringUtils.substringBeforeLast(metadata.get(GrpcConstants.REMOTE_ADDRESS), CommonConstants.COLON);\n        result.add(DefaultAuthorizationContext.of(subject, resource, Action.SUB, sourceIp));\n        return result;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/chain/AclAuthorizationHandler.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.chain;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.function.Supplier;\nimport java.util.stream.Collectors;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.rocketmq.auth.authorization.context.DefaultAuthorizationContext;\nimport org.apache.rocketmq.auth.authorization.enums.Decision;\nimport org.apache.rocketmq.auth.authorization.enums.PolicyType;\nimport org.apache.rocketmq.auth.authorization.exception.AuthorizationException;\nimport org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;\nimport org.apache.rocketmq.auth.authorization.model.Acl;\nimport org.apache.rocketmq.auth.authorization.model.Environment;\nimport org.apache.rocketmq.auth.authorization.model.Policy;\nimport org.apache.rocketmq.auth.authorization.model.PolicyEntry;\nimport org.apache.rocketmq.auth.authorization.model.Resource;\nimport org.apache.rocketmq.auth.authorization.provider.AuthorizationMetadataProvider;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.chain.Handler;\nimport org.apache.rocketmq.common.chain.HandlerChain;\nimport org.apache.rocketmq.common.resource.ResourcePattern;\nimport org.apache.rocketmq.common.resource.ResourceType;\n\npublic class AclAuthorizationHandler implements Handler<DefaultAuthorizationContext, CompletableFuture<Void>> {\n\n    private final AuthorizationMetadataProvider authorizationMetadataProvider;\n\n    public AclAuthorizationHandler(AuthConfig config) {\n        this.authorizationMetadataProvider = AuthorizationFactory.getMetadataProvider(config);\n    }\n\n    public AclAuthorizationHandler(AuthConfig config, Supplier<?> metadataService) {\n        this.authorizationMetadataProvider = AuthorizationFactory.getMetadataProvider(config, metadataService);\n    }\n\n    @Override\n    public CompletableFuture<Void> handle(DefaultAuthorizationContext context,\n        HandlerChain<DefaultAuthorizationContext, CompletableFuture<Void>> chain) {\n        if (this.authorizationMetadataProvider == null) {\n            throw new AuthorizationException(\"The authorizationMetadataProvider is not configured\");\n        }\n        return this.authorizationMetadataProvider.getAcl(context.getSubject()).thenAccept(acl -> {\n            if (acl == null) {\n                throwException(context, \"no matched policies.\");\n            }\n\n            // 1. get the defined acl entries which match the request.\n            PolicyEntry matchedEntry = matchPolicyEntries(context, acl);\n\n            // 2. if no matched acl entries, return deny\n            if (matchedEntry == null) {\n                throwException(context, \"no matched policies.\");\n            }\n\n            // 3. judge is the entries has denied decision.\n            if (matchedEntry.getDecision() == Decision.DENY) {\n                throwException(context, \"the decision is deny.\");\n            }\n        });\n    }\n\n    private PolicyEntry matchPolicyEntries(DefaultAuthorizationContext context, Acl acl) {\n        List<PolicyEntry> policyEntries = new ArrayList<>();\n\n        Policy policy = acl.getPolicy(PolicyType.CUSTOM);\n        if (policy != null) {\n            List<PolicyEntry> entries = matchPolicyEntries(context, policy.getEntries());\n            if (CollectionUtils.isNotEmpty(entries)) {\n                policyEntries.addAll(entries);\n            }\n        }\n\n        if (CollectionUtils.isEmpty(policyEntries)) {\n            policy = acl.getPolicy(PolicyType.DEFAULT);\n            if (policy != null) {\n                List<PolicyEntry> entries = matchPolicyEntries(context, policy.getEntries());\n                if (CollectionUtils.isNotEmpty(entries)) {\n                    policyEntries.addAll(entries);\n                }\n            }\n        }\n\n        if (CollectionUtils.isEmpty(policyEntries)) {\n            return null;\n        }\n\n        policyEntries.sort(this::comparePolicyEntries);\n\n        return policyEntries.get(0);\n    }\n\n    private List<PolicyEntry> matchPolicyEntries(DefaultAuthorizationContext context, List<PolicyEntry> entries) {\n        if (CollectionUtils.isEmpty(entries)) {\n            return null;\n        }\n        return entries.stream()\n            .filter(entry -> entry.isMatchResource(context.getResource()))\n            .filter(entry -> entry.isMatchAction(context.getActions()))\n            .filter(entry -> entry.isMatchEnvironment(Environment.of(context.getSourceIp())))\n            .collect(Collectors.toList());\n    }\n\n    private int comparePolicyEntries(PolicyEntry o1, PolicyEntry o2) {\n        int compare = 0;\n        Resource r1 = o1.getResource();\n        Resource r2 = o2.getResource();\n        if (r1.getResourceType() != r2.getResourceType()) {\n            if (r1.getResourceType() == ResourceType.ANY) {\n                compare = 1;\n            }\n            if (r2.getResourceType() == ResourceType.ANY) {\n                compare = -1;\n            }\n        } else if (r1.getResourcePattern() == r2.getResourcePattern()) {\n            if (r1.getResourcePattern() == ResourcePattern.PREFIXED) {\n                String n1 = r1.getResourceName();\n                String n2 = r2.getResourceName();\n                compare = -1 * Integer.compare(n1.length(), n2.length());\n            }\n        } else {\n            if (r1.getResourcePattern() == ResourcePattern.LITERAL) {\n                compare = -1;\n            } else if (r2.getResourcePattern() == ResourcePattern.LITERAL) {\n                compare = 1;\n            } else if (r1.getResourcePattern() == ResourcePattern.PREFIXED) {\n                compare = -1;\n            } else if (r2.getResourcePattern() == ResourcePattern.PREFIXED) {\n                compare = 1;\n            }\n        }\n\n        if (compare != 0) {\n            return compare;\n        }\n\n        // the decision deny has higher priority\n        Decision d1 = o1.getDecision();\n        Decision d2 = o2.getDecision();\n\n        if (d1 != d2) {\n            return d1 == Decision.DENY ? -1 : 1;\n        }\n        return 0;\n    }\n\n    private static void throwException(DefaultAuthorizationContext context, String detail) {\n        throw new AuthorizationException(\"{} has no permission to access {} from {}, \" + detail,\n            context.getSubject().getSubjectKey(), context.getResource().getResourceKey(), context.getSourceIp());\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/chain/UserAuthorizationHandler.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.chain;\n\nimport java.util.concurrent.CompletableFuture;\nimport java.util.function.Supplier;\nimport org.apache.rocketmq.auth.authentication.enums.SubjectType;\nimport org.apache.rocketmq.auth.authentication.enums.UserStatus;\nimport org.apache.rocketmq.auth.authentication.enums.UserType;\nimport org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.auth.authentication.provider.AuthenticationMetadataProvider;\nimport org.apache.rocketmq.auth.authorization.context.DefaultAuthorizationContext;\nimport org.apache.rocketmq.auth.authorization.exception.AuthorizationException;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.chain.Handler;\nimport org.apache.rocketmq.common.chain.HandlerChain;\n\npublic class UserAuthorizationHandler implements Handler<DefaultAuthorizationContext, CompletableFuture<Void>> {\n\n    private final AuthenticationMetadataProvider authenticationMetadataProvider;\n\n    public UserAuthorizationHandler(AuthConfig config, Supplier<?> metadataService) {\n        this.authenticationMetadataProvider = AuthenticationFactory.getMetadataProvider(config, metadataService);\n    }\n\n    @Override\n    public CompletableFuture<Void> handle(DefaultAuthorizationContext context, HandlerChain<DefaultAuthorizationContext, CompletableFuture<Void>> chain) {\n        if (!context.getSubject().isSubject(SubjectType.USER)) {\n            return chain.handle(context);\n        }\n        return this.getUser(context.getSubject()).thenCompose(user -> {\n            if (user.getUserType() == UserType.SUPER) {\n                return CompletableFuture.completedFuture(null);\n            }\n            return chain.handle(context);\n        });\n    }\n\n    private CompletableFuture<User> getUser(Subject subject) {\n        if (this.authenticationMetadataProvider == null) {\n            throw new AuthorizationException(\"The authenticationMetadataProvider is not configured\");\n        }\n        User user = (User) subject;\n        return authenticationMetadataProvider.getUser(user.getUsername()).thenApply(result -> {\n            if (result == null) {\n                throw new AuthorizationException(\"User:{} not found.\", user.getUsername());\n            }\n            if (result.getUserStatus() == UserStatus.DISABLE) {\n                throw new AuthorizationException(\"User:{} is disabled.\", result.getUsername());\n            }\n            return result;\n        });\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/context/AuthorizationContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.context;\n\nimport java.util.HashMap;\nimport java.util.Map;\nimport org.apache.commons.lang3.StringUtils;\n\npublic abstract class AuthorizationContext {\n\n    private String channelId;\n\n    private String rpcCode;\n\n    private Map<String, Object> extInfo;\n\n    @SuppressWarnings(\"unchecked\")\n    public <T> T getExtInfo(String key) {\n        if (StringUtils.isBlank(key)) {\n            return null;\n        }\n        if (this.extInfo == null) {\n            return null;\n        }\n        Object value = this.extInfo.get(key);\n        if (value == null) {\n            return null;\n        }\n        return (T) value;\n    }\n\n    public void setExtInfo(String key, Object value) {\n        if (StringUtils.isBlank(key) || value == null) {\n            return;\n        }\n        if (this.extInfo == null) {\n            this.extInfo = new HashMap<>();\n        }\n        this.extInfo.put(key, value);\n    }\n\n    public boolean hasExtInfo(String key) {\n        Object value = getExtInfo(key);\n        return value != null;\n    }\n\n    public String getChannelId() {\n        return channelId;\n    }\n\n    public void setChannelId(String channelId) {\n        this.channelId = channelId;\n    }\n\n    public String getRpcCode() {\n        return rpcCode;\n    }\n\n    public void setRpcCode(String rpcCode) {\n        this.rpcCode = rpcCode;\n    }\n\n    public Map<String, Object> getExtInfo() {\n        return extInfo;\n    }\n\n    public void setExtInfo(Map<String, Object> extInfo) {\n        this.extInfo = extInfo;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/context/DefaultAuthorizationContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.context;\n\nimport java.util.Collections;\nimport java.util.List;\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.auth.authorization.model.Resource;\nimport org.apache.rocketmq.common.action.Action;\n\npublic class DefaultAuthorizationContext extends AuthorizationContext {\n\n    private Subject subject;\n\n    private Resource resource;\n\n    private List<Action> actions;\n\n    private String sourceIp;\n\n    public static DefaultAuthorizationContext of(Subject subject, Resource resource, Action action, String sourceIp) {\n        DefaultAuthorizationContext context = new DefaultAuthorizationContext();\n        context.setSubject(subject);\n        context.setResource(resource);\n        context.setActions(Collections.singletonList(action));\n        context.setSourceIp(sourceIp);\n        return context;\n    }\n\n    public static DefaultAuthorizationContext of(Subject subject, Resource resource, List<Action> actions, String sourceIp) {\n        DefaultAuthorizationContext context = new DefaultAuthorizationContext();\n        context.setSubject(subject);\n        context.setResource(resource);\n        context.setActions(actions);\n        context.setSourceIp(sourceIp);\n        return context;\n    }\n\n    public String getSubjectKey() {\n        return this.subject != null ? this.subject.getSubjectKey() : null;\n    }\n\n    public String getResourceKey() {\n        return this.resource != null ? this.resource.getResourceKey() : null;\n    }\n\n    public Subject getSubject() {\n        return subject;\n    }\n\n    public void setSubject(Subject subject) {\n        this.subject = subject;\n    }\n\n    public Resource getResource() {\n        return resource;\n    }\n\n    public void setResource(Resource resource) {\n        this.resource = resource;\n    }\n\n    public List<Action> getActions() {\n        return actions;\n    }\n\n    public void setActions(List<Action> actions) {\n        this.actions = actions;\n    }\n\n    public String getSourceIp() {\n        return sourceIp;\n    }\n\n    public void setSourceIp(String sourceIp) {\n        this.sourceIp = sourceIp;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/enums/Decision.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.enums;\n\nimport com.alibaba.fastjson2.annotation.JSONField;\nimport org.apache.commons.lang3.StringUtils;\n\npublic enum Decision {\n\n    ALLOW((byte) 1, \"Allow\"),\n\n    DENY((byte) 2, \"Deny\");\n\n    @JSONField(value = true)\n    private final byte code;\n    private final String name;\n\n    Decision(byte code, String name) {\n        this.code = code;\n        this.name = name;\n    }\n\n    public static Decision getByName(String name) {\n        for (Decision decision : Decision.values()) {\n            if (StringUtils.equalsIgnoreCase(decision.getName(), name)) {\n                return decision;\n            }\n        }\n        return null;\n    }\n\n    public byte getCode() {\n        return code;\n    }\n\n    public String getName() {\n        return name;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/enums/PolicyType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.enums;\n\nimport com.alibaba.fastjson2.annotation.JSONField;\nimport org.apache.commons.lang3.StringUtils;\n\npublic enum PolicyType {\n\n    CUSTOM((byte) 1, \"Custom\"),\n\n    DEFAULT((byte) 2, \"Default\");\n\n    @JSONField(value = true)\n    private final byte code;\n    private final String name;\n\n    PolicyType(byte code, String name) {\n        this.code = code;\n        this.name = name;\n    }\n\n    public static PolicyType getByName(String name) {\n        for (PolicyType policyType : PolicyType.values()) {\n            if (StringUtils.equalsIgnoreCase(policyType.getName(), name)) {\n                return policyType;\n            }\n        }\n        return null;\n    }\n\n    public byte getCode() {\n        return code;\n    }\n\n    public String getName() {\n        return name;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/exception/AuthorizationException.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.exception;\n\nimport org.slf4j.helpers.MessageFormatter;\n\npublic class AuthorizationException extends RuntimeException {\n\n    public AuthorizationException(String message) {\n        super(message);\n    }\n\n    public AuthorizationException(String message, Throwable cause) {\n        super(message, cause);\n    }\n\n    public AuthorizationException(String messagePattern, Object... argArray) {\n        super(MessageFormatter.arrayFormat(messagePattern, argArray).getMessage());\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/factory/AuthorizationFactory.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.factory;\n\nimport com.google.protobuf.GeneratedMessageV3;\nimport io.grpc.Metadata;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.function.Function;\nimport java.util.function.Supplier;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.auth.authorization.AuthorizationEvaluator;\nimport org.apache.rocketmq.auth.authorization.context.AuthorizationContext;\nimport org.apache.rocketmq.auth.authorization.manager.AuthorizationMetadataManager;\nimport org.apache.rocketmq.auth.authorization.manager.AuthorizationMetadataManagerImpl;\nimport org.apache.rocketmq.auth.authorization.provider.AuthorizationMetadataProvider;\nimport org.apache.rocketmq.auth.authorization.provider.AuthorizationProvider;\nimport org.apache.rocketmq.auth.authorization.provider.DefaultAuthorizationProvider;\nimport org.apache.rocketmq.auth.authorization.strategy.AuthorizationStrategy;\nimport org.apache.rocketmq.auth.authorization.strategy.StatelessAuthorizationStrategy;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic class AuthorizationFactory {\n\n    private static final Map<String, Object> INSTANCE_MAP = new HashMap<>();\n    private static final String PROVIDER_PREFIX = \"PROVIDER_\";\n    private static final String METADATA_PROVIDER_PREFIX = \"METADATA_PROVIDER_\";\n    private static final String EVALUATOR_PREFIX = \"EVALUATOR_\";\n\n    @SuppressWarnings(\"unchecked\")\n    public static AuthorizationProvider<AuthorizationContext> getProvider(AuthConfig config) {\n        if (config == null) {\n            return null;\n        }\n        return computeIfAbsent(PROVIDER_PREFIX + config.getConfigName(), key -> {\n            try {\n                Class<? extends AuthorizationProvider<? extends AuthorizationContext>> clazz =\n                    DefaultAuthorizationProvider.class;\n                if (StringUtils.isNotBlank(config.getAuthorizationProvider())) {\n                    clazz = (Class<? extends AuthorizationProvider<? extends AuthorizationContext>>) Class.forName(config.getAuthorizationProvider());\n                }\n                return (AuthorizationProvider<AuthorizationContext>) clazz\n                    .getDeclaredConstructor().newInstance();\n            } catch (Exception e) {\n                throw new RuntimeException(\"Failed to load the authorization provider.\", e);\n            }\n        });\n    }\n\n    public static AuthorizationMetadataProvider getMetadataProvider(AuthConfig config) {\n        return getMetadataProvider(config, null);\n    }\n\n    public static AuthorizationMetadataManager getMetadataManager(AuthConfig config) {\n        return new AuthorizationMetadataManagerImpl(config);\n    }\n\n    @SuppressWarnings(\"unchecked\")\n    public static AuthorizationMetadataProvider getMetadataProvider(AuthConfig config, Supplier<?> metadataService) {\n        if (config == null) {\n            return null;\n        }\n        return computeIfAbsent(METADATA_PROVIDER_PREFIX + config.getConfigName(), key -> {\n            try {\n                if (StringUtils.isBlank(config.getAuthorizationMetadataProvider())) {\n                    return null;\n                }\n                Class<? extends AuthorizationMetadataProvider> clazz = (Class<? extends AuthorizationMetadataProvider>)\n                    Class.forName(config.getAuthorizationMetadataProvider());\n                AuthorizationMetadataProvider result = clazz.getDeclaredConstructor().newInstance();\n                result.initialize(config, metadataService);\n                return result;\n            } catch (Exception e) {\n                throw new RuntimeException(\"Failed to load the authorization metadata provider.\", e);\n            }\n        });\n    }\n\n    public static AuthorizationEvaluator getEvaluator(AuthConfig config) {\n        return computeIfAbsent(EVALUATOR_PREFIX + config.getConfigName(), key -> new AuthorizationEvaluator(config));\n    }\n\n    public static AuthorizationEvaluator getEvaluator(AuthConfig config, Supplier<?> metadataService) {\n        return computeIfAbsent(EVALUATOR_PREFIX + config.getConfigName(), key -> new AuthorizationEvaluator(config, metadataService));\n    }\n\n    @SuppressWarnings(\"unchecked\")\n    public static AuthorizationStrategy getStrategy(AuthConfig config, Supplier<?> metadataService) {\n        try {\n            Class<? extends AuthorizationStrategy> clazz = StatelessAuthorizationStrategy.class;\n            if (StringUtils.isNotBlank(config.getAuthorizationStrategy())) {\n                clazz = (Class<? extends AuthorizationStrategy>) Class.forName(config.getAuthorizationStrategy());\n            }\n            return clazz.getDeclaredConstructor(AuthConfig.class, Supplier.class).newInstance(config, metadataService);\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    public static List<AuthorizationContext> newContexts(AuthConfig config, Metadata metadata,\n        GeneratedMessageV3 message) {\n        AuthorizationProvider<AuthorizationContext> authorizationProvider = getProvider(config);\n        if (authorizationProvider == null) {\n            return null;\n        }\n        return authorizationProvider.newContexts(metadata, message);\n    }\n\n    public static List<AuthorizationContext> newContexts(AuthConfig config, ChannelHandlerContext context,\n        RemotingCommand command) {\n        AuthorizationProvider<AuthorizationContext> authorizationProvider = getProvider(config);\n        if (authorizationProvider == null) {\n            return null;\n        }\n        return authorizationProvider.newContexts(context, command);\n    }\n\n    @SuppressWarnings(\"unchecked\")\n    private static <V> V computeIfAbsent(String key, Function<String, ? extends V> function) {\n        Object result = null;\n        if (INSTANCE_MAP.containsKey(key)) {\n            result = INSTANCE_MAP.get(key);\n        }\n        if (result == null) {\n            synchronized (INSTANCE_MAP) {\n                if (INSTANCE_MAP.containsKey(key)) {\n                    result = INSTANCE_MAP.get(key);\n                }\n                if (result == null) {\n                    result = function.apply(key);\n                    if (result != null) {\n                        INSTANCE_MAP.put(key, result);\n                    }\n                }\n            }\n        }\n        return result != null ? (V) result : null;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/manager/AuthorizationMetadataManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.manager;\n\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.auth.authorization.enums.PolicyType;\nimport org.apache.rocketmq.auth.authorization.model.Acl;\nimport org.apache.rocketmq.auth.authorization.model.Resource;\n\npublic interface AuthorizationMetadataManager {\n\n    void shutdown();\n\n    CompletableFuture<Void> createAcl(Acl acl);\n\n    CompletableFuture<Void> updateAcl(Acl acl);\n\n    CompletableFuture<Void> deleteAcl(Subject subject);\n\n    CompletableFuture<Void> deleteAcl(Subject subject, PolicyType policyType, Resource resource);\n\n    CompletableFuture<Acl> getAcl(Subject subject);\n\n    CompletableFuture<List<Acl>> listAcl(String subjectFilter, String resourceFilter);\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/manager/AuthorizationMetadataManagerImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.manager;\n\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.auth.authentication.enums.SubjectType;\nimport org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.auth.authentication.provider.AuthenticationMetadataProvider;\nimport org.apache.rocketmq.auth.authorization.enums.PolicyType;\nimport org.apache.rocketmq.auth.authorization.exception.AuthorizationException;\nimport org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;\nimport org.apache.rocketmq.auth.authorization.model.Acl;\nimport org.apache.rocketmq.auth.authorization.model.Environment;\nimport org.apache.rocketmq.auth.authorization.model.Policy;\nimport org.apache.rocketmq.auth.authorization.model.PolicyEntry;\nimport org.apache.rocketmq.auth.authorization.model.Resource;\nimport org.apache.rocketmq.auth.authorization.provider.AuthorizationMetadataProvider;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.utils.ExceptionUtils;\nimport org.apache.rocketmq.common.utils.IPAddressUtils;\n\npublic class AuthorizationMetadataManagerImpl implements AuthorizationMetadataManager {\n\n    private final AuthorizationMetadataProvider authorizationMetadataProvider;\n\n    private final AuthenticationMetadataProvider authenticationMetadataProvider;\n\n    public AuthorizationMetadataManagerImpl(AuthConfig authConfig) {\n        this.authorizationMetadataProvider = AuthorizationFactory.getMetadataProvider(authConfig);\n        this.authenticationMetadataProvider = AuthenticationFactory.getMetadataProvider(authConfig);\n    }\n\n    @Override\n    public void shutdown() {\n        if (this.authenticationMetadataProvider != null) {\n            this.authenticationMetadataProvider.shutdown();\n        }\n        if (this.authorizationMetadataProvider != null) {\n            this.authorizationMetadataProvider.shutdown();\n        }\n    }\n\n    @Override\n    public CompletableFuture<Void> createAcl(Acl acl) {\n        try {\n            validate(acl);\n\n            initAcl(acl);\n\n            CompletableFuture<? extends Subject> subjectFuture;\n            if (acl.getSubject().isSubject(SubjectType.USER)) {\n                User user = (User) acl.getSubject();\n                subjectFuture = this.getAuthenticationMetadataProvider().getUser(user.getUsername());\n            } else {\n                subjectFuture = CompletableFuture.completedFuture(acl.getSubject());\n            }\n\n            return subjectFuture.thenCompose(subject -> {\n                if (subject == null) {\n                    throw new AuthorizationException(\"The subject of {} is not exist.\", acl.getSubject().getSubjectKey());\n                }\n                return this.getAuthorizationMetadataProvider().getAcl(acl.getSubject());\n            }).thenCompose(oldAcl -> {\n                if (oldAcl == null) {\n                    return this.getAuthorizationMetadataProvider().createAcl(acl);\n                }\n                oldAcl.updatePolicy(acl.getPolicies());\n                return this.getAuthorizationMetadataProvider().updateAcl(oldAcl);\n            });\n\n        } catch (Exception e) {\n            return this.handleException(e);\n        }\n    }\n\n    @Override\n    public CompletableFuture<Void> updateAcl(Acl acl) {\n        try {\n            validate(acl);\n\n            initAcl(acl);\n\n            CompletableFuture<? extends Subject> subjectFuture;\n            if (acl.getSubject().isSubject(SubjectType.USER)) {\n                User user = (User) acl.getSubject();\n                subjectFuture = this.getAuthenticationMetadataProvider().getUser(user.getUsername());\n            } else {\n                subjectFuture = CompletableFuture.completedFuture(acl.getSubject());\n            }\n\n            return subjectFuture.thenCompose(subject -> {\n                if (subject == null) {\n                    throw new AuthorizationException(\"The subject of {} is not exist.\", acl.getSubject().getSubjectKey());\n                }\n                return this.getAuthorizationMetadataProvider().getAcl(acl.getSubject());\n            }).thenCompose(oldAcl -> {\n                if (oldAcl == null) {\n                    return this.getAuthorizationMetadataProvider().createAcl(acl);\n                }\n                oldAcl.updatePolicy(acl.getPolicies());\n                return this.getAuthorizationMetadataProvider().updateAcl(oldAcl);\n            });\n\n        } catch (Exception e) {\n            return this.handleException(e);\n        }\n    }\n\n    @Override\n    public CompletableFuture<Void> deleteAcl(Subject subject) {\n        return this.deleteAcl(subject, null, null);\n    }\n\n    @Override\n    public CompletableFuture<Void> deleteAcl(Subject subject, PolicyType policyType, Resource resource) {\n        try {\n            if (subject == null) {\n                throw new AuthorizationException(\"The subject is null.\");\n            }\n            if (policyType == null) {\n                policyType = PolicyType.CUSTOM;\n            }\n\n            CompletableFuture<? extends Subject> subjectFuture;\n            if (subject.isSubject(SubjectType.USER)) {\n                User user = (User) subject;\n                subjectFuture = this.getAuthenticationMetadataProvider().getUser(user.getUsername());\n            } else {\n                subjectFuture = CompletableFuture.completedFuture(subject);\n            }\n            CompletableFuture<Acl> aclFuture = this.getAuthorizationMetadataProvider().getAcl(subject);\n\n            PolicyType finalPolicyType = policyType;\n            return subjectFuture.thenCombine(aclFuture, (sub, oldAcl) -> {\n                if (sub == null) {\n                    throw new AuthorizationException(\"The subject is not exist.\");\n                }\n                if (oldAcl == null) {\n                    throw new AuthorizationException(\"The acl is not exist.\");\n                }\n                return oldAcl;\n            }).thenCompose(oldAcl -> {\n                if (resource != null) {\n                    oldAcl.deletePolicy(finalPolicyType, resource);\n                }\n                if (resource == null || CollectionUtils.isEmpty(oldAcl.getPolicies())) {\n                    return this.getAuthorizationMetadataProvider().deleteAcl(subject);\n                }\n                return this.getAuthorizationMetadataProvider().updateAcl(oldAcl);\n            });\n\n        } catch (Exception e) {\n            return this.handleException(e);\n        }\n    }\n\n    @Override\n    public CompletableFuture<Acl> getAcl(Subject subject) {\n        try {\n            if (subject == null) {\n                throw new AuthorizationException(\"The subject is null.\");\n            }\n            CompletableFuture<? extends Subject> subjectFuture;\n            if (subject.isSubject(SubjectType.USER)) {\n                User user = (User) subject;\n                subjectFuture = this.getAuthenticationMetadataProvider().getUser(user.getUsername());\n            } else {\n                subjectFuture = CompletableFuture.completedFuture(subject);\n            }\n            return subjectFuture.thenCompose(sub -> {\n                if (sub == null) {\n                    throw new AuthorizationException(\"The subject is not exist.\");\n                }\n                return this.getAuthorizationMetadataProvider().getAcl(sub);\n            });\n        } catch (Exception e) {\n            return this.handleException(e);\n        }\n    }\n\n    @Override\n    public CompletableFuture<List<Acl>> listAcl(String subjectFilter, String resourceFilter) {\n        return this.getAuthorizationMetadataProvider().listAcl(subjectFilter, resourceFilter);\n    }\n\n    private static void initAcl(Acl acl) {\n        acl.getPolicies().forEach(policy -> {\n            if (policy.getPolicyType() == null) {\n                policy.setPolicyType(PolicyType.CUSTOM);\n            }\n        });\n    }\n\n    private void validate(Acl acl) {\n        Subject subject = acl.getSubject();\n        if (subject.getSubjectType() == null) {\n            throw new AuthorizationException(\"The subject type is null.\");\n        }\n        List<Policy> policies = acl.getPolicies();\n        if (CollectionUtils.isEmpty(policies)) {\n            throw new AuthorizationException(\"The policies is empty.\");\n        }\n        for (Policy policy : policies) {\n            this.validate(policy);\n        }\n    }\n\n    private void validate(Policy policy) {\n        List<PolicyEntry> policyEntries = policy.getEntries();\n        if (CollectionUtils.isEmpty(policyEntries)) {\n            throw new AuthorizationException(\"The policy entries is empty.\");\n        }\n        for (PolicyEntry policyEntry : policyEntries) {\n            this.validate(policyEntry);\n        }\n    }\n\n    private void validate(PolicyEntry entry) {\n        Resource resource = entry.getResource();\n        if (resource == null) {\n            throw new AuthorizationException(\"The resource is null.\");\n        }\n        if (resource.getResourceType() == null) {\n            throw new AuthorizationException(\"The resource type is null.\");\n        }\n        if (resource.getResourcePattern() == null) {\n            throw new AuthorizationException(\"The resource pattern is null.\");\n        }\n        if (CollectionUtils.isEmpty(entry.getActions())) {\n            throw new AuthorizationException(\"The actions is empty.\");\n        }\n        if (entry.getActions().contains(Action.ANY)) {\n            throw new AuthorizationException(\"The actions can not be Any.\");\n        }\n        Environment environment = entry.getEnvironment();\n        if (environment != null && CollectionUtils.isNotEmpty(environment.getSourceIps())) {\n            for (String sourceIp : environment.getSourceIps()) {\n                if (StringUtils.isBlank(sourceIp)) {\n                    throw new AuthorizationException(\"The source ip is empty.\");\n                }\n                if (!IPAddressUtils.isValidIPOrCidr(sourceIp)) {\n                    throw new AuthorizationException(\"The source ip is invalid.\");\n                }\n            }\n        }\n        if (entry.getDecision() == null) {\n            throw new AuthorizationException(\"The decision is null or illegal.\");\n        }\n    }\n\n    private <T> CompletableFuture<T> handleException(Exception e) {\n        CompletableFuture<T> result = new CompletableFuture<>();\n        Throwable throwable = ExceptionUtils.getRealException(e);\n        result.completeExceptionally(throwable);\n        return result;\n    }\n\n    private AuthenticationMetadataProvider getAuthenticationMetadataProvider() {\n        if (authenticationMetadataProvider == null) {\n            throw new IllegalStateException(\"The authenticationMetadataProvider is not configured.\");\n        }\n        return authenticationMetadataProvider;\n    }\n\n    private AuthorizationMetadataProvider getAuthorizationMetadataProvider() {\n        if (authorizationMetadataProvider == null) {\n            throw new IllegalStateException(\"The authorizationMetadataProvider is not configured.\");\n        }\n        return authorizationMetadataProvider;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/model/Acl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.model;\n\nimport com.google.common.collect.Lists;\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.auth.authorization.enums.Decision;\nimport org.apache.rocketmq.auth.authorization.enums.PolicyType;\nimport org.apache.rocketmq.common.action.Action;\n\npublic class Acl {\n\n    private Subject subject;\n\n    private List<Policy> policies;\n\n    public static Acl of(Subject subject, Policy policy) {\n        return of(subject, Lists.newArrayList(policy));\n    }\n\n    public static Acl of(Subject subject, List<Policy> policies) {\n        Acl acl = new Acl();\n        acl.setSubject(subject);\n        acl.setPolicies(policies);\n        return acl;\n    }\n\n    public static Acl of(Subject subject, List<Resource> resources, List<Action> actions, Environment environment,\n        Decision decision) {\n        Acl acl = new Acl();\n        acl.setSubject(subject);\n        Policy policy = Policy.of(resources, actions, environment, decision);\n        acl.setPolicies(Lists.newArrayList(policy));\n        return acl;\n    }\n\n    public void updatePolicy(Policy policy) {\n        this.updatePolicy(Lists.newArrayList(policy));\n    }\n\n    public void updatePolicy(List<Policy> policies) {\n        if (this.policies == null) {\n            this.policies = new ArrayList<>();\n        }\n        policies.forEach(newPolicy -> {\n            Policy oldPolicy = this.getPolicy(newPolicy.getPolicyType());\n            if (oldPolicy == null) {\n                this.policies.add(newPolicy);\n            } else {\n                oldPolicy.updateEntry(newPolicy.getEntries());\n            }\n        });\n    }\n\n    public void deletePolicy(PolicyType policyType, Resource resource) {\n        Policy policy = getPolicy(policyType);\n        if (policy == null) {\n            return;\n        }\n        policy.deleteEntry(resource);\n        if (CollectionUtils.isEmpty(policy.getEntries())) {\n            this.policies.remove(policy);\n        }\n    }\n\n    public Policy getPolicy(PolicyType policyType) {\n        if (CollectionUtils.isEmpty(this.policies)) {\n            return null;\n        }\n        for (Policy policy : this.policies) {\n            if (policy.getPolicyType() == policyType) {\n                return policy;\n            }\n        }\n        return null;\n    }\n\n    public Subject getSubject() {\n        return subject;\n    }\n\n    public void setSubject(Subject subject) {\n        this.subject = subject;\n    }\n\n    public List<Policy> getPolicies() {\n        return policies;\n    }\n\n    public void setPolicies(List<Policy> policies) {\n        this.policies = policies;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/model/Environment.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.model;\n\nimport java.util.Collections;\nimport java.util.List;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.utils.IPAddressUtils;\n\npublic class Environment {\n\n    private List<String> sourceIps;\n\n    public static Environment of(String sourceIp) {\n        if (StringUtils.isEmpty(sourceIp)) {\n            return null;\n        }\n        return of(Collections.singletonList(sourceIp));\n    }\n\n    public static Environment of(List<String> sourceIps) {\n        if (CollectionUtils.isEmpty(sourceIps)) {\n            return null;\n        }\n        Environment environment = new Environment();\n        environment.setSourceIps(sourceIps);\n        return environment;\n    }\n\n    public boolean isMatch(Environment environment) {\n        if (CollectionUtils.isEmpty(this.sourceIps)) {\n            return true;\n        }\n        if (CollectionUtils.isEmpty(environment.getSourceIps())) {\n            return false;\n        }\n        String targetIp = environment.getSourceIps().get(0);\n        for (String sourceIp : this.sourceIps) {\n            if (IPAddressUtils.isIPInRange(targetIp, sourceIp)) {\n                return true;\n            }\n        }\n        return false;\n    }\n\n    public List<String> getSourceIps() {\n        return sourceIps;\n    }\n\n    public void setSourceIps(List<String> sourceIps) {\n        this.sourceIps = sourceIps;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/model/Policy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.model;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.stream.Collectors;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.auth.authorization.enums.Decision;\nimport org.apache.rocketmq.auth.authorization.enums.PolicyType;\n\npublic class Policy {\n\n    private PolicyType policyType;\n\n    private List<PolicyEntry> entries;\n\n    public static Policy of(List<Resource> resources, List<Action> actions, Environment environment,\n        Decision decision) {\n        return of(PolicyType.CUSTOM, resources, actions, environment, decision);\n    }\n\n    public static Policy of(PolicyType policyType, List<Resource> resources, List<Action> actions,\n        Environment environment,\n        Decision decision) {\n        Policy policy = new Policy();\n        policy.setPolicyType(policyType);\n        List<PolicyEntry> entries = resources.stream()\n            .map(resource -> PolicyEntry.of(resource, actions, environment, decision))\n            .collect(Collectors.toList());\n        policy.setEntries(entries);\n        return policy;\n    }\n\n    public static Policy of(PolicyType type, List<PolicyEntry> entries) {\n        Policy policy = new Policy();\n        policy.setPolicyType(type);\n        policy.setEntries(entries);\n        return policy;\n    }\n\n    public void updateEntry(List<PolicyEntry> newEntries) {\n        if (this.entries == null) {\n            this.entries = new ArrayList<>();\n        }\n        newEntries.forEach(newEntry -> {\n            PolicyEntry entry = getEntry(newEntry.getResource());\n            if (entry == null) {\n                this.entries.add(newEntry);\n            } else {\n                entry.updateEntry(newEntry.getActions(), newEntry.getEnvironment(), newEntry.getDecision());\n            }\n        });\n    }\n\n    public void deleteEntry(Resource resources) {\n        PolicyEntry entry = getEntry(resources);\n        if (entry != null) {\n            this.entries.remove(entry);\n        }\n    }\n\n    private PolicyEntry getEntry(Resource resource) {\n        if (CollectionUtils.isEmpty(this.entries)) {\n            return null;\n        }\n        for (PolicyEntry entry : this.entries) {\n            if (Objects.equals(entry.getResource(), resource)) {\n                return entry;\n            }\n        }\n        return null;\n    }\n\n    public PolicyType getPolicyType() {\n        return policyType;\n    }\n\n    public void setPolicyType(PolicyType policyType) {\n        this.policyType = policyType;\n    }\n\n    public List<PolicyEntry> getEntries() {\n        return entries;\n    }\n\n    public void setEntries(List<PolicyEntry> entries) {\n        this.entries = entries;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/model/PolicyEntry.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.model;\n\nimport java.util.List;\nimport java.util.stream.Collectors;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.auth.authorization.enums.Decision;\n\npublic class PolicyEntry {\n\n    private Resource resource;\n\n    private List<Action> actions;\n\n    private Environment environment;\n\n    private Decision decision;\n\n    public static PolicyEntry of(Resource resource, List<Action> actions, Environment environment, Decision decision) {\n        PolicyEntry policyEntry = new PolicyEntry();\n        policyEntry.setResource(resource);\n        policyEntry.setActions(actions);\n        policyEntry.setEnvironment(environment);\n        policyEntry.setDecision(decision);\n        return policyEntry;\n    }\n\n    public void updateEntry(List<Action> actions, Environment environment,\n        Decision decision) {\n        this.setActions(actions);\n        this.setEnvironment(environment);\n        this.setDecision(decision);\n    }\n\n    public boolean isMatchResource(Resource resource) {\n        return this.resource.isMatch(resource);\n    }\n\n    public boolean isMatchAction(List<Action> actions) {\n        if (CollectionUtils.isEmpty(this.actions)) {\n            return false;\n        }\n        if (actions.contains(Action.ANY)) {\n            return true;\n        }\n        return actions.stream()\n            .anyMatch(action -> this.actions.contains(action)\n                || this.actions.contains(Action.ALL));\n    }\n\n    public boolean isMatchEnvironment(Environment environment) {\n        if (this.environment == null) {\n            return true;\n        }\n        return this.environment.isMatch(environment);\n    }\n\n    public String toResourceStr() {\n        if (resource == null) {\n            return null;\n        }\n        return resource.getResourceKey();\n    }\n\n    public List<String> toActionsStr() {\n        if (CollectionUtils.isEmpty(actions)) {\n            return null;\n        }\n        return actions.stream().map(Action::getName)\n            .collect(Collectors.toList());\n    }\n\n    public Resource getResource() {\n        return resource;\n    }\n\n    public void setResource(Resource resource) {\n        this.resource = resource;\n    }\n\n    public List<Action> getActions() {\n        return actions;\n    }\n\n    public void setActions(List<Action> actions) {\n        this.actions = actions;\n    }\n\n    public Environment getEnvironment() {\n        return environment;\n    }\n\n    public void setEnvironment(Environment environment) {\n        this.environment = environment;\n    }\n\n    public Decision getDecision() {\n        return decision;\n    }\n\n    public void setDecision(Decision decision) {\n        this.decision = decision;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/model/RequestContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.model;\n\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.common.action.Action;\n\npublic class RequestContext {\n\n    private Subject subject;\n\n    private Resource resource;\n\n    private Action action;\n\n    private String sourceIp;\n\n    public Subject getSubject() {\n        return subject;\n    }\n\n    public void setSubject(Subject subject) {\n        this.subject = subject;\n    }\n\n    public Resource getResource() {\n        return resource;\n    }\n\n    public void setResource(Resource resource) {\n        this.resource = resource;\n    }\n\n    public Action getAction() {\n        return action;\n    }\n\n    public void setAction(Action action) {\n        this.action = action;\n    }\n\n    public String getSourceIp() {\n        return sourceIp;\n    }\n\n    public void setSourceIp(String sourceIp) {\n        this.sourceIp = sourceIp;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/model/Resource.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.model;\n\nimport com.alibaba.fastjson2.annotation.JSONField;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.stream.Collectors;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.ResourcePattern;\nimport org.apache.rocketmq.common.constant.CommonConstants;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\n\npublic class Resource {\n\n    private ResourceType resourceType;\n\n    private String resourceName;\n\n    private ResourcePattern resourcePattern;\n\n    public static Resource ofCluster(String clusterName) {\n        return of(ResourceType.CLUSTER, clusterName, ResourcePattern.LITERAL);\n    }\n\n    public static Resource ofTopic(String topicName) {\n        return of(ResourceType.TOPIC, topicName, ResourcePattern.LITERAL);\n    }\n\n    public static Resource ofGroup(String groupName) {\n        if (NamespaceUtil.isRetryTopic(groupName)) {\n            groupName = NamespaceUtil.withOutRetryAndDLQ(groupName);\n        }\n        return of(ResourceType.GROUP, groupName, ResourcePattern.LITERAL);\n    }\n\n    public static Resource of(ResourceType resourceType, String resourceName, ResourcePattern resourcePattern) {\n        Resource resource = new Resource();\n        resource.resourceType = resourceType;\n        resource.resourceName = resourceName;\n        resource.resourcePattern = resourcePattern;\n        return resource;\n    }\n\n    public static List<Resource> of(List<String> resourceKeys) {\n        if (CollectionUtils.isEmpty(resourceKeys)) {\n            return null;\n        }\n        return resourceKeys.stream().map(Resource::of).collect(Collectors.toList());\n    }\n\n    public static Resource of(String resourceKey) {\n        if (StringUtils.isBlank(resourceKey)) {\n            return null;\n        }\n        if (StringUtils.equals(resourceKey, CommonConstants.ASTERISK)) {\n            return of(ResourceType.ANY, null, ResourcePattern.ANY);\n        }\n        String type = StringUtils.substringBefore(resourceKey, CommonConstants.COLON);\n        ResourceType resourceType = ResourceType.getByName(type);\n        if (resourceType == null) {\n            return null;\n        }\n        String resourceName = StringUtils.substringAfter(resourceKey, CommonConstants.COLON);\n        ResourcePattern resourcePattern = ResourcePattern.LITERAL;\n        if (StringUtils.equals(resourceName, CommonConstants.ASTERISK)) {\n            resourceName = null;\n            resourcePattern = ResourcePattern.ANY;\n        } else if (StringUtils.endsWith(resourceName, CommonConstants.ASTERISK)) {\n            resourceName = StringUtils.substringBefore(resourceName, CommonConstants.ASTERISK);\n            resourcePattern = ResourcePattern.PREFIXED;\n        }\n        return of(resourceType, resourceName, resourcePattern);\n    }\n\n    @JSONField(serialize = false)\n    public String getResourceKey() {\n        if (resourceType == ResourceType.ANY) {\n            return CommonConstants.ASTERISK;\n        }\n        switch (resourcePattern) {\n            case ANY:\n                return resourceType.getName() + CommonConstants.COLON + CommonConstants.ASTERISK;\n            case LITERAL:\n                return resourceType.getName() + CommonConstants.COLON + resourceName;\n            case PREFIXED:\n                return resourceType.getName() + CommonConstants.COLON + resourceName + CommonConstants.ASTERISK;\n            default:\n                return null;\n        }\n    }\n\n    public boolean isMatch(Resource resource) {\n        if (this.resourceType == ResourceType.ANY) {\n            return true;\n        }\n        if (this.resourceType != resource.resourceType) {\n            return false;\n        }\n        switch (resourcePattern) {\n            case ANY:\n                return true;\n            case LITERAL:\n                return StringUtils.equals(resource.resourceName, this.resourceName);\n            case PREFIXED:\n                return StringUtils.startsWith(resource.resourceName, this.resourceName);\n            default:\n                return false;\n        }\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o)\n            return true;\n        if (o == null || getClass() != o.getClass())\n            return false;\n        Resource resource = (Resource) o;\n        return resourceType == resource.resourceType\n            && Objects.equals(resourceName, resource.resourceName)\n            && resourcePattern == resource.resourcePattern;\n    }\n\n    @Override\n    public int hashCode() {\n        return Objects.hash(resourceType, resourceName, resourcePattern);\n    }\n\n    public ResourceType getResourceType() {\n        return resourceType;\n    }\n\n    public void setResourceType(ResourceType resourceType) {\n        this.resourceType = resourceType;\n    }\n\n    public String getResourceName() {\n        return resourceName;\n    }\n\n    public void setResourceName(String resourceName) {\n        this.resourceName = resourceName;\n    }\n\n    public ResourcePattern getResourcePattern() {\n        return resourcePattern;\n    }\n\n    public void setResourcePattern(ResourcePattern resourcePattern) {\n        this.resourcePattern = resourcePattern;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/provider/AuthorizationMetadataProvider.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.provider;\n\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.function.Supplier;\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.auth.authorization.model.Acl;\nimport org.apache.rocketmq.auth.config.AuthConfig;\n\npublic interface AuthorizationMetadataProvider {\n\n    void initialize(AuthConfig authConfig, Supplier<?> metadataService);\n\n    void shutdown();\n\n    CompletableFuture<Void> createAcl(Acl acl);\n\n    CompletableFuture<Void> deleteAcl(Subject subject);\n\n    CompletableFuture<Void> updateAcl(Acl acl);\n\n    CompletableFuture<Acl> getAcl(Subject subject);\n\n    CompletableFuture<List<Acl>> listAcl(String subjectFilter, String resourceFilter);\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/provider/AuthorizationProvider.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.provider;\n\nimport com.google.protobuf.GeneratedMessageV3;\nimport io.grpc.Metadata;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.function.Supplier;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic interface AuthorizationProvider<AuthorizationContext> {\n\n    void initialize(AuthConfig config);\n\n    void initialize(AuthConfig config, Supplier<?> metadataService);\n\n    CompletableFuture<Void> authorize(AuthorizationContext context);\n\n    List<AuthorizationContext> newContexts(Metadata metadata, GeneratedMessageV3 message);\n\n    List<AuthorizationContext> newContexts(ChannelHandlerContext context, RemotingCommand command);\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/provider/DefaultAuthorizationProvider.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.provider;\n\nimport com.google.protobuf.GeneratedMessageV3;\nimport io.grpc.Metadata;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.function.Supplier;\nimport java.util.stream.Collectors;\nimport org.apache.rocketmq.auth.authorization.builder.AuthorizationContextBuilder;\nimport org.apache.rocketmq.auth.authorization.builder.DefaultAuthorizationContextBuilder;\nimport org.apache.rocketmq.auth.authorization.chain.AclAuthorizationHandler;\nimport org.apache.rocketmq.auth.authorization.chain.UserAuthorizationHandler;\nimport org.apache.rocketmq.auth.authorization.context.DefaultAuthorizationContext;\nimport org.apache.rocketmq.auth.authorization.enums.Decision;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.chain.HandlerChain;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\npublic class DefaultAuthorizationProvider implements AuthorizationProvider<DefaultAuthorizationContext> {\n\n    protected final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_AUTH_AUDIT_LOGGER_NAME);\n    protected AuthConfig authConfig;\n    protected Supplier<?> metadataService;\n    protected AuthorizationContextBuilder authorizationContextBuilder;\n\n    @Override\n    public void initialize(AuthConfig config) {\n        this.initialize(config, null);\n    }\n\n    @Override\n    public void initialize(AuthConfig config, Supplier<?> metadataService) {\n        this.authConfig = config;\n        this.metadataService = metadataService;\n        this.authorizationContextBuilder = new DefaultAuthorizationContextBuilder(config);\n    }\n\n    @Override\n    public CompletableFuture<Void> authorize(DefaultAuthorizationContext context) {\n        return this.newHandlerChain().handle(context)\n            .whenComplete((nil, ex) -> doAuditLog(context, ex));\n    }\n\n    @Override\n    public List<DefaultAuthorizationContext> newContexts(Metadata metadata, GeneratedMessageV3 message) {\n        return this.authorizationContextBuilder.build(metadata, message);\n    }\n\n    @Override\n    public List<DefaultAuthorizationContext> newContexts(ChannelHandlerContext context, RemotingCommand command) {\n        return this.authorizationContextBuilder.build(context, command);\n    }\n\n    protected HandlerChain<DefaultAuthorizationContext, CompletableFuture<Void>> newHandlerChain() {\n        return HandlerChain.<DefaultAuthorizationContext, CompletableFuture<Void>>create()\n            .addNext(new UserAuthorizationHandler(authConfig, metadataService))\n            .addNext(new AclAuthorizationHandler(authConfig, metadataService));\n    }\n\n    protected void doAuditLog(DefaultAuthorizationContext context, Throwable ex) {\n        if (context.getSubject() == null) {\n            return;\n        }\n        Decision decision = Decision.ALLOW;\n        if (ex != null) {\n            decision = Decision.DENY;\n        }\n        String subject = context.getSubject().getSubjectKey();\n        String actions = context.getActions().stream().map(Action::getName)\n            .collect(Collectors.joining(\",\"));\n        String sourceIp = context.getSourceIp();\n        String resource = context.getResource().getResourceKey();\n        String request = context.getRpcCode();\n        String format = \"[AUTHORIZATION] Subject = {} is {} Action = {} from sourceIp = {} on resource = {} for request = {}.\";\n        if (decision == Decision.ALLOW) {\n            log.debug(format, subject, decision.getName(), actions, sourceIp, resource, request);\n        } else {\n            log.info(format, subject, decision.getName(), actions, sourceIp, resource, request);\n        }\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/provider/LocalAuthorizationMetadataProvider.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.provider;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.github.benmanes.caffeine.cache.CacheLoader;\nimport com.github.benmanes.caffeine.cache.Caffeine;\nimport com.github.benmanes.caffeine.cache.LoadingCache;\nimport java.io.File;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport java.util.function.Supplier;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.ArrayUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.auth.authorization.exception.AuthorizationException;\nimport org.apache.rocketmq.auth.authorization.model.Acl;\nimport org.apache.rocketmq.auth.authorization.model.Policy;\nimport org.apache.rocketmq.auth.authorization.model.PolicyEntry;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.config.ConfigRocksDBStorage;\nimport org.apache.rocketmq.common.thread.ThreadPoolMonitor;\nimport org.rocksdb.RocksDB;\n\npublic class LocalAuthorizationMetadataProvider implements AuthorizationMetadataProvider {\n\n    private final static String AUTH_METADATA_COLUMN_FAMILY = new String(RocksDB.DEFAULT_COLUMN_FAMILY,\n        StandardCharsets.UTF_8);\n\n    private ConfigRocksDBStorage storage;\n\n    private LoadingCache<String, Acl> aclCache;\n\n    protected ThreadPoolExecutor cacheRefreshExecutor;\n\n    @Override\n    public void initialize(AuthConfig authConfig, Supplier<?> metadataService) {\n        this.storage = ConfigRocksDBStorage.getStore(authConfig.getAuthConfigPath() + File.separator + \"acls\", false);\n        if (!this.storage.start()) {\n            throw new RuntimeException(\"Failed to load rocksdb for auth_acl, please check whether it is occupied.\");\n        }\n        this.cacheRefreshExecutor = ThreadPoolMonitor.createAndMonitor(\n            1,\n            1,\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            \"AclCacheRefresh\",\n            100000\n        );\n\n        this.aclCache = Caffeine.newBuilder()\n            .maximumSize(authConfig.getAclCacheMaxNum())\n            .expireAfterAccess(authConfig.getAclCacheExpiredSecond(), TimeUnit.SECONDS)\n            .refreshAfterWrite(authConfig.getAclCacheRefreshSecond(), TimeUnit.SECONDS)\n            .executor(cacheRefreshExecutor)\n            .build(new AclCacheLoader(this.storage));\n    }\n\n    @Override\n    public CompletableFuture<Void> createAcl(Acl acl) {\n        try {\n            Subject subject = acl.getSubject();\n            byte[] keyBytes = subject.getSubjectKey().getBytes(StandardCharsets.UTF_8);\n            byte[] valueBytes = JSON.toJSONBytes(acl);\n            this.storage.put(AUTH_METADATA_COLUMN_FAMILY, keyBytes, keyBytes.length, valueBytes);\n            this.storage.flushWAL();\n            this.aclCache.invalidate(subject.getSubjectKey());\n        } catch (Exception e) {\n            throw new AuthorizationException(\"create Acl to RocksDB failed.\", e);\n        }\n        return CompletableFuture.completedFuture(null);\n    }\n\n    @Override\n    public CompletableFuture<Void> deleteAcl(Subject subject) {\n        try {\n            byte[] keyBytes = subject.getSubjectKey().getBytes(StandardCharsets.UTF_8);\n            this.storage.delete(AUTH_METADATA_COLUMN_FAMILY, keyBytes);\n            this.storage.flushWAL();\n            this.aclCache.invalidate(subject.getSubjectKey());\n        } catch (Exception e) {\n            throw new AuthorizationException(\"delete Acl from RocksDB failed.\", e);\n        }\n        return CompletableFuture.completedFuture(null);\n    }\n\n    @Override\n    public CompletableFuture<Void> updateAcl(Acl acl) {\n        try {\n            Subject subject = acl.getSubject();\n            byte[] keyBytes = subject.getSubjectKey().getBytes(StandardCharsets.UTF_8);\n            byte[] valueBytes = JSON.toJSONBytes(acl);\n            this.storage.put(AUTH_METADATA_COLUMN_FAMILY, keyBytes, keyBytes.length, valueBytes);\n            this.storage.flushWAL();\n            this.aclCache.invalidate(subject.getSubjectKey());\n        } catch (Exception e) {\n            throw new AuthorizationException(\"update Acl to RocksDB failed.\", e);\n        }\n        return CompletableFuture.completedFuture(null);\n    }\n\n    @Override\n    public CompletableFuture<Acl> getAcl(Subject subject) {\n        Acl acl = aclCache.get(subject.getSubjectKey());\n        if (acl == AclCacheLoader.EMPTY_ACL) {\n            return CompletableFuture.completedFuture(null);\n        }\n        return CompletableFuture.completedFuture(acl);\n    }\n\n    @Override\n    public CompletableFuture<List<Acl>> listAcl(String subjectFilter, String resourceFilter) {\n        List<Acl> result = new ArrayList<>();\n        CompletableFuture<List<Acl>> future = new CompletableFuture<>();\n        try {\n            this.storage.iterate(AUTH_METADATA_COLUMN_FAMILY, (key, value) -> {\n                String subjectKey = new String(key, StandardCharsets.UTF_8);\n                if (StringUtils.isNotBlank(subjectFilter) && !subjectKey.contains(subjectFilter)) {\n                    return;\n                }\n                Subject subject = Subject.of(subjectKey);\n                Acl acl = JSON.parseObject(new String(value, StandardCharsets.UTF_8), Acl.class);\n                List<Policy> policies = acl.getPolicies();\n                if (!CollectionUtils.isNotEmpty(policies)) {\n                    return;\n                }\n                Iterator<Policy> policyIterator = policies.iterator();\n                while (policyIterator.hasNext()) {\n                    Policy policy = policyIterator.next();\n                    List<PolicyEntry> entries = policy.getEntries();\n                    if (CollectionUtils.isEmpty(entries)) {\n                        continue;\n                    }\n                    if (StringUtils.isNotBlank(resourceFilter)) {\n                        entries.removeIf(entry -> !entry.toResourceStr().contains(resourceFilter));\n                    }\n                    if (CollectionUtils.isEmpty(entries)) {\n                        policyIterator.remove();\n                    }\n                }\n                if (CollectionUtils.isNotEmpty(policies)) {\n                    result.add(Acl.of(subject, policies));\n                }\n            });\n        } catch (Exception e) {\n            future.completeExceptionally(e);\n        }\n        future.complete(result);\n        return future;\n    }\n\n    @Override\n    public void shutdown() {\n        if (this.storage != null) {\n            this.storage.shutdown();\n        }\n        if (this.cacheRefreshExecutor != null) {\n            this.cacheRefreshExecutor.shutdown();\n        }\n    }\n\n    private static class AclCacheLoader implements CacheLoader<String, Acl> {\n        private final ConfigRocksDBStorage storage;\n        public static final Acl EMPTY_ACL = new Acl();\n\n        public AclCacheLoader(ConfigRocksDBStorage storage) {\n            this.storage = storage;\n        }\n\n        @Override\n        public Acl load(String subjectKey) {\n            try {\n                byte[] keyBytes = subjectKey.getBytes(StandardCharsets.UTF_8);\n                Subject subject = Subject.of(subjectKey);\n\n                byte[] valueBytes = this.storage.get(AUTH_METADATA_COLUMN_FAMILY, keyBytes);\n                if (ArrayUtils.isEmpty(valueBytes)) {\n                    return EMPTY_ACL;\n                }\n                Acl acl = JSON.parseObject(valueBytes, Acl.class);\n                return Acl.of(subject, acl.getPolicies());\n            } catch (Exception e) {\n                throw new AuthorizationException(\"get Acl from RocksDB failed.\", e);\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/strategy/AbstractAuthorizationStrategy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.strategy;\n\nimport java.util.HashSet;\nimport java.util.Set;\nimport java.util.function.Supplier;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.auth.authorization.context.AuthorizationContext;\nimport org.apache.rocketmq.auth.authorization.exception.AuthorizationException;\nimport org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;\nimport org.apache.rocketmq.auth.authorization.provider.AuthorizationProvider;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.utils.ExceptionUtils;\n\npublic abstract class AbstractAuthorizationStrategy implements AuthorizationStrategy {\n\n    protected final AuthConfig authConfig;\n    protected final Set<String> authorizationWhiteSet = new HashSet<>();\n    protected final AuthorizationProvider<AuthorizationContext> authorizationProvider;\n\n    public AbstractAuthorizationStrategy(AuthConfig authConfig, Supplier<?> metadataService) {\n        this.authConfig = authConfig;\n        this.authorizationProvider = AuthorizationFactory.getProvider(authConfig);\n        if (this.authorizationProvider != null) {\n            this.authorizationProvider.initialize(authConfig, metadataService);\n        }\n        if (StringUtils.isNotBlank(authConfig.getAuthorizationWhitelist())) {\n            String[] whitelist = StringUtils.split(authConfig.getAuthorizationWhitelist(), \",\");\n            for (String rpcCode : whitelist) {\n                this.authorizationWhiteSet.add(StringUtils.trim(rpcCode));\n            }\n        }\n    }\n\n    public void doEvaluate(AuthorizationContext context) {\n        if (context == null) {\n            return;\n        }\n        if (!this.authConfig.isAuthorizationEnabled()) {\n            return;\n        }\n        if (this.authorizationProvider == null) {\n            return;\n        }\n        if (this.authorizationWhiteSet.contains(context.getRpcCode())) {\n            return;\n        }\n        try {\n            this.authorizationProvider.authorize(context).join();\n        } catch (AuthorizationException ex) {\n            throw ex;\n        } catch (Throwable ex) {\n            Throwable exception = ExceptionUtils.getRealException(ex);\n            if (exception instanceof AuthorizationException) {\n                throw (AuthorizationException) exception;\n            }\n            throw new AuthorizationException(\"Authorization failed. Please verify your access rights and try again.\", exception);\n        }\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/strategy/AuthorizationStrategy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.strategy;\n\nimport org.apache.rocketmq.auth.authorization.context.AuthorizationContext;\n\npublic interface AuthorizationStrategy {\n\n    void evaluate(AuthorizationContext context);\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/strategy/StatefulAuthorizationStrategy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.strategy;\n\nimport com.github.benmanes.caffeine.cache.Cache;\nimport com.github.benmanes.caffeine.cache.Caffeine;\nimport java.util.concurrent.TimeUnit;\nimport java.util.function.Supplier;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.auth.authorization.context.AuthorizationContext;\nimport org.apache.rocketmq.auth.authorization.context.DefaultAuthorizationContext;\nimport org.apache.rocketmq.auth.authorization.exception.AuthorizationException;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.constant.CommonConstants;\n\npublic class StatefulAuthorizationStrategy extends AbstractAuthorizationStrategy {\n\n    protected Cache<String, Pair<Boolean, AuthorizationException>> authCache;\n\n    public StatefulAuthorizationStrategy(AuthConfig authConfig, Supplier<?> metadataService) {\n        super(authConfig, metadataService);\n        this.authCache = Caffeine.newBuilder()\n            .expireAfterWrite(authConfig.getStatefulAuthorizationCacheExpiredSecond(), TimeUnit.SECONDS)\n            .maximumSize(authConfig.getStatefulAuthorizationCacheMaxNum())\n            .build();\n    }\n\n    @Override\n    public void evaluate(AuthorizationContext context) {\n        if (StringUtils.isBlank(context.getChannelId())) {\n            this.doEvaluate(context);\n            return;\n        }\n        Pair<Boolean, AuthorizationException> result = this.authCache.get(buildKey(context), key -> {\n            try {\n                this.doEvaluate(context);\n                return Pair.of(true, null);\n            } catch (AuthorizationException ex) {\n                return Pair.of(false, ex);\n            }\n        });\n        if (result != null && result.getObject1() == Boolean.FALSE) {\n            throw result.getObject2();\n        }\n    }\n\n    private String buildKey(AuthorizationContext context) {\n        if (context instanceof DefaultAuthorizationContext) {\n            DefaultAuthorizationContext ctx = (DefaultAuthorizationContext) context;\n            return ctx.getChannelId()\n                + (ctx.getSubject() != null ? CommonConstants.POUND + ctx.getSubjectKey() : \"\")\n                + CommonConstants.POUND + ctx.getResourceKey()\n                + CommonConstants.POUND + StringUtils.join(ctx.getActions(), CommonConstants.COMMA)\n                + CommonConstants.POUND + ctx.getSourceIp();\n        }\n        throw new AuthorizationException(\"The request of {} is not support.\", context.getClass().getSimpleName());\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/authorization/strategy/StatelessAuthorizationStrategy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.strategy;\n\nimport java.util.function.Supplier;\nimport org.apache.rocketmq.auth.authorization.context.AuthorizationContext;\nimport org.apache.rocketmq.auth.config.AuthConfig;\n\npublic class StatelessAuthorizationStrategy extends AbstractAuthorizationStrategy {\n\n    public StatelessAuthorizationStrategy(AuthConfig authConfig, Supplier<?> metadataService) {\n        super(authConfig, metadataService);\n    }\n\n    @Override\n    public void evaluate(AuthorizationContext context) {\n        super.doEvaluate(context);\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/config/AuthConfig.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.config;\n\npublic class AuthConfig implements Cloneable {\n\n    private String configName;\n\n    private String clusterName;\n\n    private String authConfigPath;\n\n    private boolean authenticationEnabled = false;\n\n    private String authenticationProvider;\n\n    private String authenticationMetadataProvider;\n\n    private String authenticationStrategy;\n\n    private String authenticationWhitelist;\n\n    private String initAuthenticationUser;\n\n    private String innerClientAuthenticationCredentials;\n\n    private boolean authorizationEnabled = false;\n\n    private String authorizationProvider;\n\n    private String authorizationMetadataProvider;\n\n    private String authorizationStrategy;\n\n    private String authorizationWhitelist;\n\n    private boolean migrateAuthFromV1Enabled = false;\n\n    private int userCacheMaxNum = 1000;\n\n    private int userCacheExpiredSecond = 600;\n\n    private int userCacheRefreshSecond = 60;\n\n    private int aclCacheMaxNum = 1000;\n\n    private int aclCacheExpiredSecond = 600;\n\n    private int aclCacheRefreshSecond = 60;\n\n    private int statefulAuthenticationCacheMaxNum = 10000;\n\n    private int statefulAuthenticationCacheExpiredSecond = 60;\n\n    private int statefulAuthorizationCacheMaxNum = 10000;\n\n    private int statefulAuthorizationCacheExpiredSecond = 60;\n\n    @Override\n    public AuthConfig clone() {\n        try {\n            return (AuthConfig) super.clone();\n        } catch (CloneNotSupportedException e) {\n            throw new AssertionError();\n        }\n    }\n\n    public String getConfigName() {\n        return configName;\n    }\n\n    public void setConfigName(String configName) {\n        this.configName = configName;\n    }\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    public void setClusterName(String clusterName) {\n        this.clusterName = clusterName;\n    }\n\n    public String getAuthConfigPath() {\n        return authConfigPath;\n    }\n\n    public void setAuthConfigPath(String authConfigPath) {\n        this.authConfigPath = authConfigPath;\n    }\n\n    public boolean isAuthenticationEnabled() {\n        return authenticationEnabled;\n    }\n\n    public void setAuthenticationEnabled(boolean authenticationEnabled) {\n        this.authenticationEnabled = authenticationEnabled;\n    }\n\n    public String getAuthenticationProvider() {\n        return authenticationProvider;\n    }\n\n    public void setAuthenticationProvider(String authenticationProvider) {\n        this.authenticationProvider = authenticationProvider;\n    }\n\n    public String getAuthenticationMetadataProvider() {\n        return authenticationMetadataProvider;\n    }\n\n    public void setAuthenticationMetadataProvider(String authenticationMetadataProvider) {\n        this.authenticationMetadataProvider = authenticationMetadataProvider;\n    }\n\n    public String getAuthenticationStrategy() {\n        return authenticationStrategy;\n    }\n\n    public void setAuthenticationStrategy(String authenticationStrategy) {\n        this.authenticationStrategy = authenticationStrategy;\n    }\n\n    public String getAuthenticationWhitelist() {\n        return authenticationWhitelist;\n    }\n\n    public void setAuthenticationWhitelist(String authenticationWhitelist) {\n        this.authenticationWhitelist = authenticationWhitelist;\n    }\n\n    public String getInitAuthenticationUser() {\n        return initAuthenticationUser;\n    }\n\n    public void setInitAuthenticationUser(String initAuthenticationUser) {\n        this.initAuthenticationUser = initAuthenticationUser;\n    }\n\n    public String getInnerClientAuthenticationCredentials() {\n        return innerClientAuthenticationCredentials;\n    }\n\n    public void setInnerClientAuthenticationCredentials(String innerClientAuthenticationCredentials) {\n        this.innerClientAuthenticationCredentials = innerClientAuthenticationCredentials;\n    }\n\n    public boolean isAuthorizationEnabled() {\n        return authorizationEnabled;\n    }\n\n    public void setAuthorizationEnabled(boolean authorizationEnabled) {\n        this.authorizationEnabled = authorizationEnabled;\n    }\n\n    public String getAuthorizationProvider() {\n        return authorizationProvider;\n    }\n\n    public void setAuthorizationProvider(String authorizationProvider) {\n        this.authorizationProvider = authorizationProvider;\n    }\n\n    public String getAuthorizationMetadataProvider() {\n        return authorizationMetadataProvider;\n    }\n\n    public void setAuthorizationMetadataProvider(String authorizationMetadataProvider) {\n        this.authorizationMetadataProvider = authorizationMetadataProvider;\n    }\n\n    public String getAuthorizationStrategy() {\n        return authorizationStrategy;\n    }\n\n    public void setAuthorizationStrategy(String authorizationStrategy) {\n        this.authorizationStrategy = authorizationStrategy;\n    }\n\n    public String getAuthorizationWhitelist() {\n        return authorizationWhitelist;\n    }\n\n    public void setAuthorizationWhitelist(String authorizationWhitelist) {\n        this.authorizationWhitelist = authorizationWhitelist;\n    }\n\n    public boolean isMigrateAuthFromV1Enabled() {\n        return migrateAuthFromV1Enabled;\n    }\n\n    public void setMigrateAuthFromV1Enabled(boolean migrateAuthFromV1Enabled) {\n        this.migrateAuthFromV1Enabled = migrateAuthFromV1Enabled;\n    }\n\n    public int getUserCacheMaxNum() {\n        return userCacheMaxNum;\n    }\n\n    public void setUserCacheMaxNum(int userCacheMaxNum) {\n        this.userCacheMaxNum = userCacheMaxNum;\n    }\n\n    public int getUserCacheExpiredSecond() {\n        return userCacheExpiredSecond;\n    }\n\n    public void setUserCacheExpiredSecond(int userCacheExpiredSecond) {\n        this.userCacheExpiredSecond = userCacheExpiredSecond;\n    }\n\n    public int getUserCacheRefreshSecond() {\n        return userCacheRefreshSecond;\n    }\n\n    public void setUserCacheRefreshSecond(int userCacheRefreshSecond) {\n        this.userCacheRefreshSecond = userCacheRefreshSecond;\n    }\n\n    public int getAclCacheMaxNum() {\n        return aclCacheMaxNum;\n    }\n\n    public void setAclCacheMaxNum(int aclCacheMaxNum) {\n        this.aclCacheMaxNum = aclCacheMaxNum;\n    }\n\n    public int getAclCacheExpiredSecond() {\n        return aclCacheExpiredSecond;\n    }\n\n    public void setAclCacheExpiredSecond(int aclCacheExpiredSecond) {\n        this.aclCacheExpiredSecond = aclCacheExpiredSecond;\n    }\n\n    public int getAclCacheRefreshSecond() {\n        return aclCacheRefreshSecond;\n    }\n\n    public void setAclCacheRefreshSecond(int aclCacheRefreshSecond) {\n        this.aclCacheRefreshSecond = aclCacheRefreshSecond;\n    }\n\n    public int getStatefulAuthenticationCacheMaxNum() {\n        return statefulAuthenticationCacheMaxNum;\n    }\n\n    public void setStatefulAuthenticationCacheMaxNum(int statefulAuthenticationCacheMaxNum) {\n        this.statefulAuthenticationCacheMaxNum = statefulAuthenticationCacheMaxNum;\n    }\n\n    public int getStatefulAuthenticationCacheExpiredSecond() {\n        return statefulAuthenticationCacheExpiredSecond;\n    }\n\n    public void setStatefulAuthenticationCacheExpiredSecond(int statefulAuthenticationCacheExpiredSecond) {\n        this.statefulAuthenticationCacheExpiredSecond = statefulAuthenticationCacheExpiredSecond;\n    }\n\n    public int getStatefulAuthorizationCacheMaxNum() {\n        return statefulAuthorizationCacheMaxNum;\n    }\n\n    public void setStatefulAuthorizationCacheMaxNum(int statefulAuthorizationCacheMaxNum) {\n        this.statefulAuthorizationCacheMaxNum = statefulAuthorizationCacheMaxNum;\n    }\n\n    public int getStatefulAuthorizationCacheExpiredSecond() {\n        return statefulAuthorizationCacheExpiredSecond;\n    }\n\n    public void setStatefulAuthorizationCacheExpiredSecond(int statefulAuthorizationCacheExpiredSecond) {\n        this.statefulAuthorizationCacheExpiredSecond = statefulAuthorizationCacheExpiredSecond;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/migration/AuthMigrator.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.migration;\n\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.acl.common.AclConstants;\nimport org.apache.rocketmq.auth.authentication.enums.UserType;\nimport org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;\nimport org.apache.rocketmq.auth.authentication.manager.AuthenticationMetadataManager;\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.auth.authorization.enums.Decision;\nimport org.apache.rocketmq.auth.authorization.enums.PolicyType;\nimport org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;\nimport org.apache.rocketmq.auth.authorization.manager.AuthorizationMetadataManager;\nimport org.apache.rocketmq.auth.authorization.model.Acl;\nimport org.apache.rocketmq.auth.authorization.model.Policy;\nimport org.apache.rocketmq.auth.authorization.model.PolicyEntry;\nimport org.apache.rocketmq.auth.authorization.model.Resource;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.auth.migration.v1.PlainPermissionManager;\nimport org.apache.rocketmq.auth.migration.v1.AclConfig;\nimport org.apache.rocketmq.auth.migration.v1.PlainAccessConfig;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.constant.CommonConstants;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.resource.ResourcePattern;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.concurrent.CompletableFuture;\n\npublic class AuthMigrator {\n\n    protected static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n\n    private final AuthConfig authConfig;\n\n    private final PlainPermissionManager plainPermissionManager;\n\n    private final AuthenticationMetadataManager authenticationMetadataManager;\n\n    private final AuthorizationMetadataManager authorizationMetadataManager;\n\n    public AuthMigrator(AuthConfig authConfig) {\n        this.authConfig = authConfig;\n        this.plainPermissionManager = new PlainPermissionManager();\n        this.authenticationMetadataManager = AuthenticationFactory.getMetadataManager(authConfig);\n        this.authorizationMetadataManager = AuthorizationFactory.getMetadataManager(authConfig);\n    }\n\n    public void migrate() {\n        if (!authConfig.isMigrateAuthFromV1Enabled()) {\n            return;\n        }\n\n        AclConfig aclConfig = this.plainPermissionManager.getAllAclConfig();\n        List<PlainAccessConfig> accessConfigs = aclConfig.getPlainAccessConfigs();\n        if (CollectionUtils.isEmpty(accessConfigs)) {\n            return;\n        }\n\n        for (PlainAccessConfig accessConfig : accessConfigs) {\n            doMigrate(accessConfig);\n        }\n    }\n\n    private void doMigrate(PlainAccessConfig accessConfig) {\n        this.isUserExisted(accessConfig.getAccessKey()).thenCompose(existed -> {\n            if (existed) {\n                return CompletableFuture.completedFuture(null);\n            }\n            return createUserAndAcl(accessConfig);\n        }).exceptionally(ex -> {\n            LOG.error(\"[ACL MIGRATE] An error occurred while migrating ACL configurations for AccessKey:{}.\", accessConfig.getAccessKey(), ex);\n            return null;\n        }).join();\n    }\n\n    private CompletableFuture<Void> createUserAndAcl(PlainAccessConfig accessConfig) {\n        return createUser(accessConfig).thenCompose(nil -> createAcl(accessConfig));\n    }\n\n    private CompletableFuture<Void> createUser(PlainAccessConfig accessConfig) {\n        User user = new User();\n        user.setUsername(accessConfig.getAccessKey());\n        user.setPassword(accessConfig.getSecretKey());\n        if (accessConfig.isAdmin()) {\n            user.setUserType(UserType.SUPER);\n        } else {\n            user.setUserType(UserType.NORMAL);\n        }\n        return this.authenticationMetadataManager.createUser(user);\n    }\n\n    private CompletableFuture<Void> createAcl(PlainAccessConfig config) {\n        Subject subject = User.of(config.getAccessKey());\n        List<Policy> policies = new ArrayList<>();\n\n        Policy customPolicy = null;\n        if (CollectionUtils.isNotEmpty(config.getTopicPerms())) {\n            for (String topicPerm : config.getTopicPerms()) {\n                String[] temp = StringUtils.split(topicPerm, CommonConstants.EQUAL);\n                if (temp.length != 2) {\n                    continue;\n                }\n                String topicName = StringUtils.trim(temp[0]);\n                String perm = StringUtils.trim(temp[1]);\n                Resource resource = Resource.ofTopic(topicName);\n                List<Action> actions = parseActions(perm);\n                Decision decision = parseDecision(perm);\n                PolicyEntry policyEntry = PolicyEntry.of(resource, actions, null, decision);\n                if (customPolicy == null) {\n                    customPolicy = Policy.of(PolicyType.CUSTOM, new ArrayList<>());\n                }\n                customPolicy.getEntries().add(policyEntry);\n            }\n        }\n        if (CollectionUtils.isNotEmpty(config.getGroupPerms())) {\n            for (String groupPerm : config.getGroupPerms()) {\n                String[] temp = StringUtils.split(groupPerm, CommonConstants.EQUAL);\n                if (temp.length != 2) {\n                    continue;\n                }\n                String groupName = StringUtils.trim(temp[0]);\n                String perm = StringUtils.trim(temp[1]);\n                Resource resource = Resource.ofGroup(groupName);\n                List<Action> actions = parseActions(perm);\n                Decision decision = parseDecision(perm);\n                PolicyEntry policyEntry = PolicyEntry.of(resource, actions, null, decision);\n                if (customPolicy == null) {\n                    customPolicy = Policy.of(PolicyType.CUSTOM, new ArrayList<>());\n                }\n                customPolicy.getEntries().add(policyEntry);\n            }\n        }\n        if (customPolicy != null) {\n            policies.add(customPolicy);\n        }\n\n        Policy defaultPolicy = null;\n        if (StringUtils.isNotBlank(config.getDefaultTopicPerm())) {\n            String topicPerm = StringUtils.trim(config.getDefaultTopicPerm());\n            Resource resource = Resource.of(ResourceType.TOPIC, null, ResourcePattern.ANY);\n            List<Action> actions = parseActions(topicPerm);\n            Decision decision = parseDecision(topicPerm);\n            PolicyEntry policyEntry = PolicyEntry.of(resource, actions, null, decision);\n            defaultPolicy = Policy.of(PolicyType.DEFAULT, new ArrayList<>());\n            defaultPolicy.getEntries().add(policyEntry);\n        }\n        if (StringUtils.isNotBlank(config.getDefaultGroupPerm())) {\n            String groupPerm = StringUtils.trim(config.getDefaultGroupPerm());\n            Resource resource = Resource.of(ResourceType.GROUP, null, ResourcePattern.ANY);\n            List<Action> actions = parseActions(groupPerm);\n            Decision decision = parseDecision(groupPerm);\n            PolicyEntry policyEntry = PolicyEntry.of(resource, actions, null, decision);\n            if (defaultPolicy == null) {\n                defaultPolicy = Policy.of(PolicyType.DEFAULT, new ArrayList<>());\n            }\n            defaultPolicy.getEntries().add(policyEntry);\n        }\n        if (defaultPolicy != null) {\n            policies.add(defaultPolicy);\n        }\n\n        if (CollectionUtils.isEmpty(policies)) {\n            return CompletableFuture.completedFuture(null);\n        }\n\n        Acl acl = Acl.of(subject, policies);\n        return this.authorizationMetadataManager.createAcl(acl);\n    }\n\n    private Decision parseDecision(String str) {\n        if (StringUtils.isBlank(str)) {\n            return Decision.DENY;\n        }\n        return StringUtils.equals(str, AclConstants.DENY) ? Decision.DENY : Decision.ALLOW;\n    }\n\n    private List<Action> parseActions(String str) {\n        List<Action> result = new ArrayList<>();\n        if (StringUtils.isBlank(str)) {\n            result.add(Action.ALL);\n        }\n        switch (StringUtils.trim(str)) {\n            case AclConstants.PUB:\n                result.add(Action.PUB);\n                break;\n            case AclConstants.SUB:\n                result.add(Action.SUB);\n                break;\n            case AclConstants.PUB_SUB:\n            case AclConstants.SUB_PUB:\n                result.add(Action.PUB);\n                result.add(Action.SUB);\n                break;\n            case AclConstants.DENY:\n                result.add(Action.ALL);\n                break;\n            default:\n                result.add(Action.ALL);\n                break;\n        }\n        return result;\n    }\n\n    private CompletableFuture<Boolean> isUserExisted(String username) {\n        return this.authenticationMetadataManager.getUser(username).thenApply(Objects::nonNull);\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/migration/v1/AccessResource.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.auth.migration.v1;\n\npublic interface AccessResource {\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/migration/v1/AclConfig.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.migration.v1;\n\nimport java.util.List;\n\npublic class AclConfig {\n\n    private List<String> globalWhiteAddrs;\n\n    private List<PlainAccessConfig> plainAccessConfigs;\n\n\n    public List<String> getGlobalWhiteAddrs() {\n        return globalWhiteAddrs;\n    }\n\n    public void setGlobalWhiteAddrs(List<String> globalWhiteAddrs) {\n        this.globalWhiteAddrs = globalWhiteAddrs;\n    }\n\n    public List<PlainAccessConfig> getPlainAccessConfigs() {\n        return plainAccessConfigs;\n    }\n\n    public void setPlainAccessConfigs(List<PlainAccessConfig> plainAccessConfigs) {\n        this.plainAccessConfigs = plainAccessConfigs;\n    }\n\n    @Override\n    public String toString() {\n        return \"AclConfig{\" +\n            \"globalWhiteAddrs=\" + globalWhiteAddrs +\n            \", plainAccessConfigs=\" + plainAccessConfigs +\n            '}';\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/migration/v1/PlainAccessConfig.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.migration.v1;\n\nimport java.io.Serializable;\nimport java.util.List;\nimport java.util.Objects;\n\npublic class PlainAccessConfig  implements Serializable {\n    private static final long serialVersionUID = -4517357000307227637L;\n\n    private String accessKey;\n\n    private String secretKey;\n\n    private String whiteRemoteAddress;\n\n    private boolean admin;\n\n    private String defaultTopicPerm;\n\n    private String defaultGroupPerm;\n\n    private List<String> topicPerms;\n\n    private List<String> groupPerms;\n\n    public String getAccessKey() {\n        return accessKey;\n    }\n\n    public void setAccessKey(String accessKey) {\n        this.accessKey = accessKey;\n    }\n\n    public String getSecretKey() {\n        return secretKey;\n    }\n\n    public void setSecretKey(String secretKey) {\n        this.secretKey = secretKey;\n    }\n\n    public String getWhiteRemoteAddress() {\n        return whiteRemoteAddress;\n    }\n\n    public void setWhiteRemoteAddress(String whiteRemoteAddress) {\n        this.whiteRemoteAddress = whiteRemoteAddress;\n    }\n\n    public boolean isAdmin() {\n        return admin;\n    }\n\n    public void setAdmin(boolean admin) {\n        this.admin = admin;\n    }\n\n    public String getDefaultTopicPerm() {\n        return defaultTopicPerm;\n    }\n\n    public void setDefaultTopicPerm(String defaultTopicPerm) {\n        this.defaultTopicPerm = defaultTopicPerm;\n    }\n\n    public String getDefaultGroupPerm() {\n        return defaultGroupPerm;\n    }\n\n    public void setDefaultGroupPerm(String defaultGroupPerm) {\n        this.defaultGroupPerm = defaultGroupPerm;\n    }\n\n    public List<String> getTopicPerms() {\n        return topicPerms;\n    }\n\n    public void setTopicPerms(List<String> topicPerms) {\n        this.topicPerms = topicPerms;\n    }\n\n    public List<String> getGroupPerms() {\n        return groupPerms;\n    }\n\n    public void setGroupPerms(List<String> groupPerms) {\n        this.groupPerms = groupPerms;\n    }\n\n    @Override\n    public String toString() {\n        return \"PlainAccessConfig{\" +\n            \"accessKey='\" + accessKey + '\\'' +\n            \", whiteRemoteAddress='\" + whiteRemoteAddress + '\\'' +\n            \", admin=\" + admin +\n            \", defaultTopicPerm='\" + defaultTopicPerm + '\\'' +\n            \", defaultGroupPerm='\" + defaultGroupPerm + '\\'' +\n            \", topicPerms=\" + topicPerms +\n            \", groupPerms=\" + groupPerms +\n            '}';\n    }\n\n    @Override public boolean equals(Object o) {\n        if (this == o)\n            return true;\n        if (o == null || getClass() != o.getClass())\n            return false;\n        PlainAccessConfig config = (PlainAccessConfig) o;\n        return admin == config.admin && Objects.equals(accessKey, config.accessKey) && Objects.equals(secretKey, config.secretKey) && Objects.equals(whiteRemoteAddress, config.whiteRemoteAddress) && Objects.equals(defaultTopicPerm, config.defaultTopicPerm) && Objects.equals(defaultGroupPerm, config.defaultGroupPerm) && Objects.equals(topicPerms, config.topicPerms) && Objects.equals(groupPerms, config.groupPerms);\n    }\n\n    @Override public int hashCode() {\n        return Objects.hash(accessKey, secretKey, whiteRemoteAddress, admin, defaultTopicPerm, defaultGroupPerm, topicPerms, groupPerms);\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/migration/v1/PlainAccessData.java",
    "content": "/*\r\n * Licensed to the Apache Software Foundation (ASF) under one or more\r\n * contributor license agreements.  See the NOTICE file distributed with\r\n * this work for additional information regarding copyright ownership.\r\n * The ASF licenses this file to You under the Apache License, Version 2.0\r\n * (the \"License\"); you may not use this file except in compliance with\r\n * the License.  You may obtain a copy of the License at\r\n *\r\n *     http://www.apache.org/licenses/LICENSE-2.0\r\n *\r\n * Unless required by applicable law or agreed to in writing, software\r\n * distributed under the License is distributed on an \"AS IS\" BASIS,\r\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r\n * See the License for the specific language governing permissions and\r\n * limitations under the License.\r\n */\r\npackage org.apache.rocketmq.auth.migration.v1;\r\n\r\nimport java.io.Serializable;\r\nimport java.util.ArrayList;\r\nimport java.util.List;\r\nimport java.util.Objects;\r\n\r\npublic class PlainAccessData implements Serializable {\r\n    private static final long serialVersionUID = -7971775135605117152L;\r\n\r\n    private List<String> globalWhiteRemoteAddresses = new ArrayList<>();\r\n    private List<PlainAccessConfig> accounts = new ArrayList<>();\r\n    private List<DataVersion> dataVersion = new ArrayList<>();\r\n\r\n    public List<String> getGlobalWhiteRemoteAddresses() {\r\n        return globalWhiteRemoteAddresses;\r\n    }\r\n\r\n    public void setGlobalWhiteRemoteAddresses(List<String> globalWhiteRemoteAddresses) {\r\n        this.globalWhiteRemoteAddresses = globalWhiteRemoteAddresses;\r\n    }\r\n\r\n    public List<PlainAccessConfig> getAccounts() {\r\n        return accounts;\r\n    }\r\n\r\n    public void setAccounts(List<PlainAccessConfig> accounts) {\r\n        this.accounts = accounts;\r\n    }\r\n\r\n    public List<DataVersion> getDataVersion() {\r\n        return dataVersion;\r\n    }\r\n\r\n    public void setDataVersion(List<DataVersion> dataVersion) {\r\n        this.dataVersion = dataVersion;\r\n    }\r\n\r\n    public static class DataVersion implements Serializable {\r\n        private static final long serialVersionUID = 6437361970079056954L;\r\n        private long timestamp;\r\n        private long counter;\r\n\r\n        public long getTimestamp() {\r\n            return timestamp;\r\n        }\r\n\r\n        public void setTimestamp(long timestamp) {\r\n            this.timestamp = timestamp;\r\n        }\r\n\r\n        public long getCounter() {\r\n            return counter;\r\n        }\r\n\r\n        public void setCounter(long counter) {\r\n            this.counter = counter;\r\n        }\r\n\r\n        @Override\r\n        public boolean equals(Object o) {\r\n            if (this == o) return true;\r\n            if (o == null || getClass() != o.getClass()) return false;\r\n            DataVersion that = (DataVersion) o;\r\n            return timestamp == that.timestamp && counter == that.counter;\r\n        }\r\n\r\n        @Override\r\n        public int hashCode() {\r\n            return Objects.hash(timestamp, counter);\r\n        }\r\n    }\r\n\r\n    @Override\r\n    public boolean equals(Object o) {\r\n        if (this == o) return true;\r\n        if (o == null || getClass() != o.getClass()) return false;\r\n        PlainAccessData that = (PlainAccessData) o;\r\n        return Objects.equals(globalWhiteRemoteAddresses, that.globalWhiteRemoteAddresses) && Objects.equals(accounts, that.accounts) && Objects.equals(dataVersion, that.dataVersion);\r\n    }\r\n\r\n    @Override\r\n    public int hashCode() {\r\n        return Objects.hash(globalWhiteRemoteAddresses, accounts, dataVersion);\r\n    }\r\n}\r\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/migration/v1/PlainAccessResource.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.migration.v1;\n\nimport org.apache.commons.lang3.builder.ToStringBuilder;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.MixAll;\n\nimport java.util.HashMap;\nimport java.util.Map;\n\npublic class PlainAccessResource implements AccessResource {\n\n    // Identify the user\n    private String accessKey;\n\n    private String secretKey;\n\n    private String whiteRemoteAddress;\n\n    private boolean admin;\n\n    private byte defaultTopicPerm = 1;\n\n    private byte defaultGroupPerm = 1;\n\n    private Map<String, Byte> resourcePermMap;\n\n    private int requestCode;\n\n    // The content to calculate the content\n    private byte[] content;\n\n    private String signature;\n\n    private String secretToken;\n\n    private String recognition;\n\n    public PlainAccessResource() {\n    }\n\n    public static String getGroupFromRetryTopic(String retryTopic) {\n        if (retryTopic == null) {\n            return null;\n        }\n        return KeyBuilder.parseGroup(retryTopic);\n    }\n\n    public static String getRetryTopic(String group) {\n        if (group == null) {\n            return null;\n        }\n        return MixAll.getRetryTopic(group);\n    }\n\n    public void addResourceAndPerm(String resource, byte perm) {\n        if (resource == null) {\n            return;\n        }\n        if (resourcePermMap == null) {\n            resourcePermMap = new HashMap<>();\n        }\n        resourcePermMap.put(resource, perm);\n    }\n\n    public String getAccessKey() {\n        return accessKey;\n    }\n\n    public void setAccessKey(String accessKey) {\n        this.accessKey = accessKey;\n    }\n\n    public String getSecretKey() {\n        return secretKey;\n    }\n\n    public void setSecretKey(String secretKey) {\n        this.secretKey = secretKey;\n    }\n\n    public String getWhiteRemoteAddress() {\n        return whiteRemoteAddress;\n    }\n\n    public void setWhiteRemoteAddress(String whiteRemoteAddress) {\n        this.whiteRemoteAddress = whiteRemoteAddress;\n    }\n\n    public boolean isAdmin() {\n        return admin;\n    }\n\n    public void setAdmin(boolean admin) {\n        this.admin = admin;\n    }\n\n    public byte getDefaultTopicPerm() {\n        return defaultTopicPerm;\n    }\n\n    public void setDefaultTopicPerm(byte defaultTopicPerm) {\n        this.defaultTopicPerm = defaultTopicPerm;\n    }\n\n    public byte getDefaultGroupPerm() {\n        return defaultGroupPerm;\n    }\n\n    public void setDefaultGroupPerm(byte defaultGroupPerm) {\n        this.defaultGroupPerm = defaultGroupPerm;\n    }\n\n    public Map<String, Byte> getResourcePermMap() {\n        return resourcePermMap;\n    }\n\n    public String getRecognition() {\n        return recognition;\n    }\n\n    public void setRecognition(String recognition) {\n        this.recognition = recognition;\n    }\n\n    public int getRequestCode() {\n        return requestCode;\n    }\n\n    public void setRequestCode(int requestCode) {\n        this.requestCode = requestCode;\n    }\n\n    public String getSecretToken() {\n        return secretToken;\n    }\n\n    public void setSecretToken(String secretToken) {\n        this.secretToken = secretToken;\n    }\n\n    public String getSignature() {\n        return signature;\n    }\n\n    public void setSignature(String signature) {\n        this.signature = signature;\n    }\n\n    @Override\n    public String toString() {\n        return ToStringBuilder.reflectionToString(this);\n    }\n\n    public byte[] getContent() {\n        return content;\n    }\n\n    public void setContent(byte[] content) {\n        this.content = content;\n    }\n}\n"
  },
  {
    "path": "auth/src/main/java/org/apache/rocketmq/auth/migration/v1/PlainPermissionManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.migration.v1;\n\nimport org.apache.rocketmq.acl.common.AclException;\nimport org.apache.rocketmq.acl.common.AclUtils;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.nio.file.FileAlreadyExistsException;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\n\npublic class PlainPermissionManager {\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);\n\n    private String fileHome = MixAll.ROCKETMQ_HOME_DIR;\n\n    private String defaultAclDir;\n\n    private String defaultAclFile;\n\n    private List<String> fileList = new ArrayList<>();\n\n\n    public PlainPermissionManager() {\n        this.defaultAclDir = MixAll.dealFilePath(fileHome + File.separator + \"conf\" + File.separator + \"acl\");\n        this.defaultAclFile = MixAll.dealFilePath(fileHome + File.separator + System.getProperty(\"rocketmq.acl.plain.file\", \"conf\" + File.separator + \"plain_acl.yml\"));\n        load();\n    }\n\n    public List<String> getAllAclFiles(String path) {\n        if (!new File(path).exists()) {\n            log.info(\"The default acl dir {} is not exist\", path);\n            return new ArrayList<>();\n        }\n        List<String> allAclFileFullPath = new ArrayList<>();\n        File file = new File(path);\n        File[] files = file.listFiles();\n        for (int i = 0; files != null && i < files.length; i++) {\n            String fileName = files[i].getAbsolutePath();\n            File f = new File(fileName);\n            if (fileName.equals(fileHome + MixAll.ACL_CONF_TOOLS_FILE)) {\n                continue;\n            } else if (fileName.endsWith(\".yml\") || fileName.endsWith(\".yaml\")) {\n                allAclFileFullPath.add(fileName);\n            } else if (f.isDirectory()) {\n                allAclFileFullPath.addAll(getAllAclFiles(fileName));\n            }\n        }\n        return allAclFileFullPath;\n    }\n\n    public void load() {\n        if (fileHome == null || fileHome.isEmpty()) {\n            return;\n        }\n\n        assureAclConfigFilesExist();\n\n        fileList = getAllAclFiles(defaultAclDir);\n        if (new File(defaultAclFile).exists() && !fileList.contains(defaultAclFile)) {\n            fileList.add(defaultAclFile);\n        }\n    }\n\n    /**\n     * Currently GlobalWhiteAddress is defined in {@link #defaultAclFile}, so make sure it exists.\n     */\n    private void assureAclConfigFilesExist() {\n        final Path defaultAclFilePath = Paths.get(this.defaultAclFile);\n        if (!Files.exists(defaultAclFilePath)) {\n            try {\n                Files.createFile(defaultAclFilePath);\n            } catch (FileAlreadyExistsException e) {\n                // Maybe created by other threads\n            } catch (IOException e) {\n                log.error(\"Error in creating \" + this.defaultAclFile, e);\n                throw new AclException(e.getMessage());\n            }\n        }\n    }\n\n    public AclConfig getAllAclConfig() {\n        AclConfig aclConfig = new AclConfig();\n        List<PlainAccessConfig> configs = new ArrayList<>();\n        List<String> whiteAddrs = new ArrayList<>();\n        Set<String> accessKeySets = new HashSet<>();\n\n        for (String path : fileList) {\n            PlainAccessData plainAclConfData = AclUtils.getYamlDataObject(path, PlainAccessData.class);\n            if (plainAclConfData == null) {\n                continue;\n            }\n            List<String> globalWhiteAddrs = plainAclConfData.getGlobalWhiteRemoteAddresses();\n            if (globalWhiteAddrs != null && !globalWhiteAddrs.isEmpty()) {\n                whiteAddrs.addAll(globalWhiteAddrs);\n            }\n\n            List<PlainAccessConfig> plainAccessConfigs = plainAclConfData.getAccounts();\n            if (plainAccessConfigs != null && !plainAccessConfigs.isEmpty()) {\n                for (PlainAccessConfig accessConfig : plainAccessConfigs) {\n                    if (!accessKeySets.contains(accessConfig.getAccessKey())) {\n                        accessKeySets.add(accessConfig.getAccessKey());\n                        PlainAccessConfig plainAccessConfig = new PlainAccessConfig();\n                        plainAccessConfig.setGroupPerms(accessConfig.getGroupPerms());\n                        plainAccessConfig.setDefaultTopicPerm(accessConfig.getDefaultTopicPerm());\n                        plainAccessConfig.setDefaultGroupPerm(accessConfig.getDefaultGroupPerm());\n                        plainAccessConfig.setAccessKey(accessConfig.getAccessKey());\n                        plainAccessConfig.setSecretKey(accessConfig.getSecretKey());\n                        plainAccessConfig.setAdmin(accessConfig.isAdmin());\n                        plainAccessConfig.setTopicPerms(accessConfig.getTopicPerms());\n                        plainAccessConfig.setWhiteRemoteAddress(accessConfig.getWhiteRemoteAddress());\n                        configs.add(plainAccessConfig);\n                    }\n                }\n            }\n        }\n        aclConfig.setPlainAccessConfigs(configs);\n        aclConfig.setGlobalWhiteAddrs(whiteAddrs);\n        return aclConfig;\n    }\n}\n"
  },
  {
    "path": "auth/src/test/java/org/apache/rocketmq/auth/authentication/AuthenticationEvaluatorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.List;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.rocketmq.auth.authentication.context.DefaultAuthenticationContext;\nimport org.apache.rocketmq.auth.authentication.exception.AuthenticationException;\nimport org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;\nimport org.apache.rocketmq.auth.authentication.manager.AuthenticationMetadataManager;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.auth.helper.AuthTestHelper;\nimport org.apache.rocketmq.common.MixAll;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\npublic class AuthenticationEvaluatorTest {\n\n    private AuthConfig authConfig;\n    private AuthenticationEvaluator evaluator;\n    private AuthenticationMetadataManager authenticationMetadataManager;\n\n    @Before\n    public void setUp() throws Exception {\n        if (MixAll.isMac()) {\n            return;\n        }\n        this.authConfig = AuthTestHelper.createDefaultConfig();\n        this.evaluator = new AuthenticationEvaluator(authConfig);\n        this.authenticationMetadataManager = AuthenticationFactory.getMetadataManager(authConfig);\n        this.clearAllUsers();\n    }\n\n    @After\n    public void tearDown() throws Exception {\n        if (MixAll.isMac()) {\n            return;\n        }\n        this.clearAllUsers();\n        this.authenticationMetadataManager.shutdown();\n    }\n\n    @Test\n    public void evaluate1() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"test\", \"test\");\n        this.authenticationMetadataManager.createUser(user);\n\n        DefaultAuthenticationContext context = new DefaultAuthenticationContext();\n        context.setRpcCode(\"11\");\n        context.setUsername(\"test\");\n        context.setContent(\"test\".getBytes(StandardCharsets.UTF_8));\n        context.setSignature(\"DJRRXBXlCVuKh6ULoN87847QX+Y=\");\n        this.evaluator.evaluate(context);\n    }\n\n    @Test\n    public void evaluate2() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        DefaultAuthenticationContext context = new DefaultAuthenticationContext();\n        context.setRpcCode(\"11\");\n        context.setUsername(\"test\");\n        context.setContent(\"test\".getBytes(StandardCharsets.UTF_8));\n        context.setSignature(\"DJRRXBXlCVuKh6ULoN87847QX+Y=\");\n        Assert.assertThrows(AuthenticationException.class, () -> this.evaluator.evaluate(context));\n    }\n\n    @Test\n    public void evaluate3() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"test\", \"test\");\n        this.authenticationMetadataManager.createUser(user);\n\n        DefaultAuthenticationContext context = new DefaultAuthenticationContext();\n        context.setRpcCode(\"11\");\n        context.setUsername(\"test\");\n        context.setContent(\"test\".getBytes(StandardCharsets.UTF_8));\n        context.setSignature(\"test\");\n        Assert.assertThrows(AuthenticationException.class, () -> this.evaluator.evaluate(context));\n    }\n\n    @Test\n    public void evaluate4() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        this.authConfig.setAuthenticationWhitelist(\"11\");\n        this.evaluator = new AuthenticationEvaluator(authConfig);\n\n        DefaultAuthenticationContext context = new DefaultAuthenticationContext();\n        context.setRpcCode(\"11\");\n        context.setUsername(\"test\");\n        context.setContent(\"test\".getBytes(StandardCharsets.UTF_8));\n        context.setSignature(\"test\");\n        this.evaluator.evaluate(context);\n    }\n\n    @Test\n    public void evaluate5() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        this.authConfig.setAuthenticationEnabled(false);\n        this.evaluator = new AuthenticationEvaluator(authConfig);\n\n        DefaultAuthenticationContext context = new DefaultAuthenticationContext();\n        context.setRpcCode(\"11\");\n        context.setUsername(\"test\");\n        context.setContent(\"test\".getBytes(StandardCharsets.UTF_8));\n        context.setSignature(\"test\");\n        this.evaluator.evaluate(context);\n    }\n\n    private void clearAllUsers() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        List<User> users = this.authenticationMetadataManager.listUser(null).join();\n        if (CollectionUtils.isEmpty(users)) {\n            return;\n        }\n        users.forEach(user -> this.authenticationMetadataManager.deleteUser(user.getUsername()).join());\n    }\n}"
  },
  {
    "path": "auth/src/test/java/org/apache/rocketmq/auth/authentication/builder/DefaultAuthenticationContextBuilderTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.builder;\n\nimport apache.rocketmq.v2.Message;\nimport apache.rocketmq.v2.Resource;\nimport apache.rocketmq.v2.SendMessageRequest;\nimport com.google.protobuf.ByteString;\nimport io.grpc.Metadata;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.channel.ChannelId;\nimport java.nio.charset.StandardCharsets;\nimport org.apache.rocketmq.auth.authentication.context.DefaultAuthenticationContext;\nimport org.apache.rocketmq.common.constant.GrpcConstants;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.Silent.class)\npublic class DefaultAuthenticationContextBuilderTest {\n\n    private DefaultAuthenticationContextBuilder builder;\n\n    @Mock\n    private ChannelHandlerContext channelHandlerContext;\n\n    @Mock\n    private Channel channel;\n\n    @Before\n    public void setUp() throws Exception {\n        builder = new DefaultAuthenticationContextBuilder();\n    }\n\n    @Test\n    public void build1() {\n        Resource topic = Resource.newBuilder().setName(\"topic-test\").build();\n        {\n            SendMessageRequest request = SendMessageRequest.newBuilder()\n                .addMessages(Message.newBuilder().setTopic(topic)\n                    .setBody(ByteString.copyFromUtf8(\"message-body\"))\n                    .build())\n                .build();\n            Metadata metadata = new Metadata();\n            metadata.put(GrpcConstants.AUTHORIZATION, \"MQv2-HMAC-SHA1 Credential=abc, SignedHeaders=x-mq-date-time, Signature=D18A9CBCDDBA9041D6693268FEF15A989E64430B\");\n            metadata.put(GrpcConstants.DATE_TIME, \"20231227T194619Z\");\n            DefaultAuthenticationContext context = builder.build(metadata, request);\n            Assert.assertNotNull(context);\n            Assert.assertEquals(\"abc\", context.getUsername());\n            Assert.assertEquals(\"0YqcvN26kEHWaTJo/vFamJ5kQws=\", context.getSignature());\n            Assert.assertEquals(\"20231227T194619Z\", new String(context.getContent(), StandardCharsets.UTF_8));\n        }\n    }\n\n    @Test\n    public void build2() {\n        when(channel.id()).thenReturn(mockChannelId(\"channel-id\"));\n        when(channelHandlerContext.channel()).thenReturn(channel);\n        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();\n        requestHeader.setTopic(\"topic-test\");\n        requestHeader.setQueueId(0);\n        requestHeader.setBornTimestamp(117036786441330L);\n        requestHeader.setBname(\"brokerName-1\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, requestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"abc\");\n        request.addExtField(\"Signature\", \"ZG26exJ5u9q1fwZlO4DCmz2Rs88=\");\n        request.makeCustomHeaderToNet();\n        DefaultAuthenticationContext context = builder.build(channelHandlerContext, request);\n        Assert.assertNotNull(context);\n        Assert.assertEquals(\"abc\", context.getUsername());\n        Assert.assertEquals(\"ZG26exJ5u9q1fwZlO4DCmz2Rs88=\", context.getSignature());\n        Assert.assertEquals(\"abcbrokerName-11170367864413300topic-test\", new String(context.getContent(), StandardCharsets.UTF_8));\n    }\n\n    private ChannelId mockChannelId(String channelId) {\n        return new ChannelId() {\n            @Override\n            public String asShortText() {\n                return channelId;\n            }\n\n            @Override\n            public String asLongText() {\n                return channelId;\n            }\n\n            @Override\n            public int compareTo(ChannelId o) {\n                return 0;\n            }\n        };\n    }\n}"
  },
  {
    "path": "auth/src/test/java/org/apache/rocketmq/auth/authentication/manager/AuthenticationMetadataManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.manager;\n\nimport java.util.List;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.rocketmq.auth.authentication.enums.UserType;\nimport org.apache.rocketmq.auth.authentication.exception.AuthenticationException;\nimport org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.auth.helper.AuthTestHelper;\nimport org.apache.rocketmq.common.MixAll;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\npublic class AuthenticationMetadataManagerTest {\n\n    private AuthConfig authConfig;\n    private AuthenticationMetadataManager authenticationMetadataManager;\n\n    @Before\n    public void setUp() throws Exception {\n        if (MixAll.isMac()) {\n            return;\n        }\n        this.authConfig = AuthTestHelper.createDefaultConfig();\n        this.authenticationMetadataManager = AuthenticationFactory.getMetadataManager(this.authConfig);\n        this.clearAllUsers();\n    }\n\n    @After\n    public void tearDown() throws Exception {\n        if (MixAll.isMac()) {\n            return;\n        }\n        this.clearAllUsers();\n        this.authenticationMetadataManager.shutdown();\n    }\n\n    @Test\n    public void createUser() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"test\", \"test\");\n        this.authenticationMetadataManager.createUser(user).join();\n        user = this.authenticationMetadataManager.getUser(\"test\").join();\n        Assert.assertNotNull(user);\n        Assert.assertEquals(user.getUsername(), \"test\");\n        Assert.assertEquals(user.getPassword(), \"test\");\n        Assert.assertEquals(user.getUserType(), UserType.NORMAL);\n\n        user = User.of(\"super\", \"super\", UserType.SUPER);\n        this.authenticationMetadataManager.createUser(user).join();\n        user = this.authenticationMetadataManager.getUser(\"super\").join();\n        Assert.assertNotNull(user);\n        Assert.assertEquals(user.getUsername(), \"super\");\n        Assert.assertEquals(user.getPassword(), \"super\");\n        Assert.assertEquals(user.getUserType(), UserType.SUPER);\n\n        Assert.assertThrows(AuthenticationException.class, () -> {\n            try {\n                User user2 = User.of(\"test\", \"test\");\n                this.authenticationMetadataManager.createUser(user2).join();\n            } catch (Exception e) {\n                AuthTestHelper.handleException(e);\n            }\n        });\n    }\n\n    @Test\n    public void updateUser() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"test\", \"test\");\n        this.authenticationMetadataManager.createUser(user).join();\n        user = this.authenticationMetadataManager.getUser(\"test\").join();\n        Assert.assertNotNull(user);\n        Assert.assertEquals(user.getUsername(), \"test\");\n        Assert.assertEquals(user.getPassword(), \"test\");\n        Assert.assertEquals(user.getUserType(), UserType.NORMAL);\n\n        user.setPassword(\"123\");\n        this.authenticationMetadataManager.updateUser(user).join();\n        user = this.authenticationMetadataManager.getUser(\"test\").join();\n        Assert.assertNotNull(user);\n        Assert.assertEquals(user.getUsername(), \"test\");\n        Assert.assertEquals(user.getPassword(), \"123\");\n        Assert.assertEquals(user.getUserType(), UserType.NORMAL);\n\n        user.setUserType(UserType.SUPER);\n        this.authenticationMetadataManager.updateUser(user).join();\n        user = this.authenticationMetadataManager.getUser(\"test\").join();\n        Assert.assertNotNull(user);\n        Assert.assertEquals(user.getUsername(), \"test\");\n        Assert.assertEquals(user.getPassword(), \"123\");\n        Assert.assertEquals(user.getUserType(), UserType.SUPER);\n\n        Assert.assertThrows(AuthenticationException.class, () -> {\n            try {\n                User user2 = User.of(\"no_user\", \"no_user\");\n                this.authenticationMetadataManager.updateUser(user2).join();\n            } catch (Exception e) {\n                AuthTestHelper.handleException(e);\n            }\n        });\n    }\n\n    @Test\n    public void deleteUser() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"test\", \"test\");\n        this.authenticationMetadataManager.createUser(user).join();\n        user = this.authenticationMetadataManager.getUser(\"test\").join();\n        Assert.assertNotNull(user);\n        this.authenticationMetadataManager.deleteUser(\"test\").join();\n        user = this.authenticationMetadataManager.getUser(\"test\").join();\n        Assert.assertNull(user);\n\n        this.authenticationMetadataManager.deleteUser(\"no_user\").join();\n    }\n\n    @Test\n    public void getUser() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"test\", \"test\");\n        this.authenticationMetadataManager.createUser(user).join();\n        user = this.authenticationMetadataManager.getUser(\"test\").join();\n        Assert.assertNotNull(user);\n        Assert.assertEquals(user.getUsername(), \"test\");\n        Assert.assertEquals(user.getPassword(), \"test\");\n        Assert.assertEquals(user.getUserType(), UserType.NORMAL);\n\n        user = this.authenticationMetadataManager.getUser(\"no_user\").join();\n        Assert.assertNull(user);\n    }\n\n    @Test\n    public void listUser() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        List<User> users = this.authenticationMetadataManager.listUser(null).join();\n        Assert.assertTrue(CollectionUtils.isEmpty(users));\n\n        User user = User.of(\"test-1\", \"test-1\");\n        this.authenticationMetadataManager.createUser(user).join();\n        users = this.authenticationMetadataManager.listUser(null).join();\n        Assert.assertEquals(users.size(), 1);\n\n        user = User.of(\"test-2\", \"test-2\");\n        this.authenticationMetadataManager.createUser(user).join();\n        users = this.authenticationMetadataManager.listUser(\"test\").join();\n        Assert.assertEquals(users.size(), 2);\n    }\n\n    private void clearAllUsers() {\n        List<User> users = this.authenticationMetadataManager.listUser(null).join();\n        if (CollectionUtils.isEmpty(users)) {\n            return;\n        }\n        users.forEach(user -> this.authenticationMetadataManager.deleteUser(user.getUsername()).join());\n    }\n}"
  },
  {
    "path": "auth/src/test/java/org/apache/rocketmq/auth/authentication/provider/LocalAuthenticationMetadataProviderTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authentication.provider;\n\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.auth.helper.AuthTestHelper;\nimport org.junit.Assert;\nimport org.junit.Rule;\nimport org.junit.Test;\nimport org.junit.rules.TemporaryFolder;\n\npublic class LocalAuthenticationMetadataProviderTest {\n\n    @Rule\n    public TemporaryFolder tempFolder = new TemporaryFolder();\n\n    @Test\n    public void testShutdownReleasesCacheExecutor() throws Exception {\n        AuthConfig authConfig = AuthTestHelper.createDefaultConfig();\n        authConfig.setAuthConfigPath(tempFolder.newFolder(\"auth-test\").getAbsolutePath());\n\n        LocalAuthenticationMetadataProvider provider = new LocalAuthenticationMetadataProvider();\n        // Initialize provider to create the internal cache refresh executor\n        provider.initialize(authConfig, () -> null);\n\n        // After initialization, the executor should exist and not be shutdown\n        Assert.assertNotNull(provider.cacheRefreshExecutor);\n        Assert.assertFalse(provider.cacheRefreshExecutor.isShutdown());\n\n        // Shutdown provider should also shutdown its executor to release resources\n        provider.shutdown();\n\n        // Verify that the cache refresh executor has been shutdown\n        Assert.assertTrue(provider.cacheRefreshExecutor.isShutdown());\n    }\n}\n"
  },
  {
    "path": "auth/src/test/java/org/apache/rocketmq/auth/authorization/AuthorizationEvaluatorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;\nimport org.apache.rocketmq.auth.authentication.manager.AuthenticationMetadataManager;\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.auth.authorization.context.AuthorizationContext;\nimport org.apache.rocketmq.auth.authorization.context.DefaultAuthorizationContext;\nimport org.apache.rocketmq.auth.authorization.enums.Decision;\nimport org.apache.rocketmq.auth.authorization.enums.PolicyType;\nimport org.apache.rocketmq.auth.authorization.exception.AuthorizationException;\nimport org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;\nimport org.apache.rocketmq.auth.authorization.manager.AuthorizationMetadataManager;\nimport org.apache.rocketmq.auth.authorization.model.Acl;\nimport org.apache.rocketmq.auth.authorization.model.Resource;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.auth.helper.AuthTestHelper;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.action.Action;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\npublic class AuthorizationEvaluatorTest {\n\n    private AuthConfig authConfig;\n    private AuthorizationEvaluator evaluator;\n    private AuthenticationMetadataManager authenticationMetadataManager;\n    private AuthorizationMetadataManager authorizationMetadataManager;\n\n    @Before\n    public void setUp() throws Exception {\n        if (MixAll.isMac()) {\n            return;\n        }\n        this.authConfig = AuthTestHelper.createDefaultConfig();\n        this.evaluator = new AuthorizationEvaluator(authConfig);\n        this.authenticationMetadataManager = AuthenticationFactory.getMetadataManager(authConfig);\n        this.authorizationMetadataManager = AuthorizationFactory.getMetadataManager(authConfig);\n        this.clearAllAcls();\n        this.clearAllUsers();\n    }\n\n    @After\n    public void tearDown() throws Exception {\n        if (MixAll.isMac()) {\n            return;\n        }\n        this.clearAllAcls();\n        this.clearAllUsers();\n        this.authenticationMetadataManager.shutdown();\n    }\n\n    @Test\n    public void evaluate1() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"test\", \"test\");\n        this.authenticationMetadataManager.createUser(user).join();\n\n        Acl acl = AuthTestHelper.buildAcl(\"User:test\", \"Topic:test*\", \"Pub\", \"192.168.0.0/24\", Decision.ALLOW);\n        this.authorizationMetadataManager.createAcl(acl).join();\n\n        Subject subject = Subject.of(\"User:test\");\n        Resource resource = Resource.ofTopic(\"test\");\n        Action action = Action.PUB;\n        String sourceIp = \"192.168.0.1\";\n        DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);\n        context.setRpcCode(\"10\");\n        this.evaluator.evaluate(Collections.singletonList(context));\n\n        // acl sourceIp is null\n        acl = AuthTestHelper.buildAcl(\"User:test\", \"Topic:test*\", \"Pub\", null, Decision.ALLOW);\n        this.authorizationMetadataManager.updateAcl(acl).join();\n\n        subject = Subject.of(\"User:test\");\n        resource = Resource.ofTopic(\"test\");\n        action = Action.PUB;\n        sourceIp = \"192.168.0.1\";\n        context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);\n        context.setRpcCode(\"10\");\n        this.evaluator.evaluate(Collections.singletonList(context));\n    }\n\n    @Test\n    public void evaluate2() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"test\", \"test\");\n        this.authenticationMetadataManager.createUser(user).join();\n\n        Acl acl = AuthTestHelper.buildAcl(\"User:test\", \"Topic:test*,Group:test*\", \"Sub\", \"192.168.0.0/24\", Decision.ALLOW);\n        this.authorizationMetadataManager.createAcl(acl).join();\n\n        List<AuthorizationContext> contexts = new ArrayList<>();\n\n        Subject subject = Subject.of(\"User:test\");\n        Resource resource = Resource.ofTopic(\"test\");\n        Action action = Action.SUB;\n        String sourceIp = \"192.168.0.1\";\n        DefaultAuthorizationContext context1 = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);\n        context1.setRpcCode(\"11\");\n        contexts.add(context1);\n\n        subject = Subject.of(\"User:test\");\n        resource = Resource.ofGroup(\"test\");\n        action = Action.SUB;\n        sourceIp = \"192.168.0.1\";\n        DefaultAuthorizationContext context2 = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);\n        context2.setRpcCode(\"11\");\n        contexts.add(context2);\n\n        this.evaluator.evaluate(contexts);\n    }\n\n    @Test\n    public void evaluate4() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"test\", \"test\");\n        this.authenticationMetadataManager.createUser(user).join();\n\n        Acl acl = AuthTestHelper.buildAcl(\"User:test\", \"Topic:test*\", \"Pub\", \"192.168.0.0/24\", Decision.ALLOW);\n        this.authorizationMetadataManager.createAcl(acl).join();\n\n        // user not exist\n        Assert.assertThrows(AuthorizationException.class, () -> {\n            Subject subject = Subject.of(\"User:abc\");\n            Resource resource = Resource.ofTopic(\"test\");\n            Action action = Action.PUB;\n            String sourceIp = \"192.168.0.1\";\n            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);\n            context.setRpcCode(\"10\");\n            this.evaluator.evaluate(Collections.singletonList(context));\n        });\n\n        // resource not match\n        Assert.assertThrows(AuthorizationException.class, () -> {\n            Subject subject = Subject.of(\"User:test\");\n            Resource resource = Resource.ofTopic(\"abc\");\n            Action action = Action.PUB;\n            String sourceIp = \"192.168.0.1\";\n            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);\n            context.setRpcCode(\"10\");\n            this.evaluator.evaluate(Collections.singletonList(context));\n        });\n\n        // action not match\n        Assert.assertThrows(AuthorizationException.class, () -> {\n            Subject subject = Subject.of(\"User:test\");\n            Resource resource = Resource.ofTopic(\"test\");\n            Action action = Action.SUB;\n            String sourceIp = \"192.168.0.1\";\n            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);\n            context.setRpcCode(\"10\");\n            this.evaluator.evaluate(Collections.singletonList(context));\n        });\n\n        // sourceIp not match\n        Assert.assertThrows(AuthorizationException.class, () -> {\n            Subject subject = Subject.of(\"User:test\");\n            Resource resource = Resource.ofTopic(\"test\");\n            Action action = Action.PUB;\n            String sourceIp = \"10.10.0.1\";\n            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);\n            context.setRpcCode(\"10\");\n            this.evaluator.evaluate(Collections.singletonList(context));\n        });\n\n        // decision is deny\n        acl = AuthTestHelper.buildAcl(\"User:test\", \"Topic:test*\", \"Pub\", \"192.168.0.0/24\", Decision.DENY);\n        this.authorizationMetadataManager.updateAcl(acl).join();\n        Assert.assertThrows(AuthorizationException.class, () -> {\n            Subject subject = Subject.of(\"User:test\");\n            Resource resource = Resource.ofTopic(\"test\");\n            Action action = Action.PUB;\n            String sourceIp = \"192.168.0.1\";\n            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);\n            context.setRpcCode(\"10\");\n            this.evaluator.evaluate(Collections.singletonList(context));\n        });\n    }\n\n    @Test\n    public void evaluate5() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"test\", \"test\");\n        this.authenticationMetadataManager.createUser(user).join();\n\n        Acl acl = AuthTestHelper.buildAcl(\"User:test\", \"*\", \"Pub,Sub\", \"192.168.0.0/24\", Decision.ALLOW);\n        this.authorizationMetadataManager.createAcl(acl).join();\n\n        acl = AuthTestHelper.buildAcl(\"User:test\", \"Topic:*\", \"Pub,Sub\", \"192.168.0.0/24\", Decision.DENY);\n        this.authorizationMetadataManager.updateAcl(acl).join();\n\n        acl = AuthTestHelper.buildAcl(\"User:test\", \"Topic:test*\", \"Pub,Sub\", \"192.168.0.0/24\", Decision.ALLOW);\n        this.authorizationMetadataManager.updateAcl(acl).join();\n\n        acl = AuthTestHelper.buildAcl(\"User:test\", \"Topic:test-1\", \"Pub,Sub\", \"192.168.0.0/24\", Decision.DENY);\n        this.authorizationMetadataManager.updateAcl(acl).join();\n\n        Assert.assertThrows(AuthorizationException.class, () -> {\n            Subject subject = Subject.of(\"User:test\");\n            Resource resource = Resource.ofTopic(\"test-1\");\n            Action action = Action.PUB;\n            String sourceIp = \"192.168.0.1\";\n            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);\n            context.setRpcCode(\"10\");\n            this.evaluator.evaluate(Collections.singletonList(context));\n        });\n\n        {\n            Subject subject = Subject.of(\"User:test\");\n            Resource resource = Resource.ofTopic(\"test-2\");\n            Action action = Action.PUB;\n            String sourceIp = \"192.168.0.1\";\n            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);\n            context.setRpcCode(\"10\");\n            this.evaluator.evaluate(Collections.singletonList(context));\n        }\n\n        Assert.assertThrows(AuthorizationException.class, () -> {\n            Subject subject = Subject.of(\"User:test\");\n            Resource resource = Resource.ofTopic(\"abc\");\n            Action action = Action.PUB;\n            String sourceIp = \"192.168.0.1\";\n            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);\n            context.setRpcCode(\"10\");\n            this.evaluator.evaluate(Collections.singletonList(context));\n        });\n\n        {\n            Subject subject = Subject.of(\"User:test\");\n            Resource resource = Resource.ofGroup(\"test-2\");\n            Action action = Action.SUB;\n            String sourceIp = \"192.168.0.1\";\n            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);\n            context.setRpcCode(\"10\");\n            this.evaluator.evaluate(Collections.singletonList(context));\n        }\n    }\n\n    @Test\n    public void evaluate6() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        this.authConfig.setAuthorizationWhitelist(\"10\");\n        this.evaluator = new AuthorizationEvaluator(this.authConfig);\n\n        Subject subject = Subject.of(\"User:test\");\n        Resource resource = Resource.ofTopic(\"test\");\n        Action action = Action.PUB;\n        String sourceIp = \"192.168.0.1\";\n        DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);\n        context.setRpcCode(\"10\");\n        this.evaluator.evaluate(Collections.singletonList(context));\n    }\n\n    @Test\n    public void evaluate7() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        this.authConfig.setAuthorizationEnabled(false);\n        this.evaluator = new AuthorizationEvaluator(this.authConfig);\n\n        Subject subject = Subject.of(\"User:test\");\n        Resource resource = Resource.ofTopic(\"test\");\n        Action action = Action.PUB;\n        String sourceIp = \"192.168.0.1\";\n        DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);\n        context.setRpcCode(\"10\");\n        this.evaluator.evaluate(Collections.singletonList(context));\n    }\n\n    @Test\n    public void evaluate8() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"test\", \"test\");\n        this.authenticationMetadataManager.createUser(user).join();\n\n        Acl acl = AuthTestHelper.buildAcl(\"User:test\", \"Topic:test*\", \"Pub\", \"192.168.0.0/24\", Decision.DENY);\n        this.authorizationMetadataManager.createAcl(acl).join();\n\n        Assert.assertThrows(AuthorizationException.class, () -> {\n            Subject subject = Subject.of(\"User:test\");\n            Resource resource = Resource.ofTopic(\"test\");\n            Action action = Action.PUB;\n            String sourceIp = \"192.168.0.1\";\n            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);\n            context.setRpcCode(\"10\");\n            this.evaluator.evaluate(Collections.singletonList(context));\n        });\n\n        Assert.assertThrows(AuthorizationException.class, () -> {\n            Subject subject = Subject.of(\"User:test\");\n            Resource resource = Resource.ofTopic(\"abc\");\n            Action action = Action.PUB;\n            String sourceIp = \"192.168.0.1\";\n            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);\n            context.setRpcCode(\"10\");\n            this.evaluator.evaluate(Collections.singletonList(context));\n        });\n\n        acl = AuthTestHelper.buildAcl(\"User:test\", PolicyType.DEFAULT, \"Topic:*\", \"Pub\", null, Decision.ALLOW);\n        this.authorizationMetadataManager.updateAcl(acl).join();\n        {\n            Subject subject = Subject.of(\"User:test\");\n            Resource resource = Resource.ofTopic(\"abc\");\n            Action action = Action.PUB;\n            String sourceIp = \"192.168.0.1\";\n            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);\n            context.setRpcCode(\"10\");\n            this.evaluator.evaluate(Collections.singletonList(context));\n        }\n    }\n\n    @Test\n    public void evaluate9() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"test\", \"test\");\n        this.authenticationMetadataManager.createUser(user).join();\n\n        Acl acl0 = AuthTestHelper.buildAcl(\"User:test\", \"*\", \"Pub\", \"192.168.0.0/24\", Decision.ALLOW);\n        this.authorizationMetadataManager.createAcl(acl0).join();\n        Acl acl1 = AuthTestHelper.buildAcl(\"User:test\", \"Topic:*\", \"Pub\", \"192.168.0.0/24\", Decision.ALLOW);\n        this.authorizationMetadataManager.createAcl(acl1).join();\n        Acl acl2 = AuthTestHelper.buildAcl(\"User:test\", \"Topic:test*\", \"Pub\", \"192.168.0.0/24\", Decision.ALLOW);\n        this.authorizationMetadataManager.createAcl(acl2).join();\n        Acl acl3 = AuthTestHelper.buildAcl(\"User:test\", \"Topic:test_*\", \"Pub\", \"192.168.0.0/24\", Decision.DENY);\n        this.authorizationMetadataManager.createAcl(acl3).join();\n        Acl acl4 = AuthTestHelper.buildAcl(\"User:test\", \"Topic:test_001\", \"Pub\", \"192.168.0.0/24\", Decision.DENY);\n        this.authorizationMetadataManager.createAcl(acl4).join();\n\n        Assert.assertThrows(AuthorizationException.class, () -> {\n            Subject subject = Subject.of(\"User:test\");\n            Resource resource = Resource.ofTopic(\"test_001\");\n            Action action = Action.PUB;\n            String sourceIp = \"192.168.0.1\";\n            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);\n            context.setRpcCode(\"10\");\n            this.evaluator.evaluate(Collections.singletonList(context));\n        });\n\n        Assert.assertThrows(AuthorizationException.class, () -> {\n            Subject subject = Subject.of(\"User:test\");\n            Resource resource = Resource.ofTopic(\"test_002\");\n            Action action = Action.PUB;\n            String sourceIp = \"192.168.0.1\";\n            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);\n            context.setRpcCode(\"10\");\n            this.evaluator.evaluate(Collections.singletonList(context));\n        });\n    }\n\n    private void clearAllUsers() {\n        List<User> users = this.authenticationMetadataManager.listUser(null).join();\n        if (CollectionUtils.isEmpty(users)) {\n            return;\n        }\n        users.forEach(user -> this.authenticationMetadataManager.deleteUser(user.getUsername()).join());\n    }\n\n    private void clearAllAcls() {\n        List<Acl> acls = this.authorizationMetadataManager.listAcl(null, null).join();\n        if (CollectionUtils.isEmpty(acls)) {\n            return;\n        }\n        acls.forEach(acl -> this.authorizationMetadataManager.deleteAcl(acl.getSubject(), null, null).join());\n    }\n}"
  },
  {
    "path": "auth/src/test/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilderTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.builder;\n\nimport apache.rocketmq.v2.AckMessageRequest;\nimport apache.rocketmq.v2.ChangeInvisibleDurationRequest;\nimport apache.rocketmq.v2.ClientType;\nimport apache.rocketmq.v2.EndTransactionRequest;\nimport apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest;\nimport apache.rocketmq.v2.HeartbeatRequest;\nimport apache.rocketmq.v2.Message;\nimport apache.rocketmq.v2.MessageQueue;\nimport apache.rocketmq.v2.NotifyClientTerminationRequest;\nimport apache.rocketmq.v2.Publishing;\nimport apache.rocketmq.v2.QueryAssignmentRequest;\nimport apache.rocketmq.v2.QueryRouteRequest;\nimport apache.rocketmq.v2.RecallMessageRequest;\nimport apache.rocketmq.v2.ReceiveMessageRequest;\nimport apache.rocketmq.v2.Resource;\nimport apache.rocketmq.v2.SendMessageRequest;\nimport apache.rocketmq.v2.Settings;\nimport apache.rocketmq.v2.Subscription;\nimport apache.rocketmq.v2.SubscriptionEntry;\nimport apache.rocketmq.v2.TelemetryCommand;\nimport com.alibaba.fastjson2.JSON;\nimport com.google.common.collect.Sets;\nimport com.google.protobuf.GeneratedMessageV3;\nimport io.grpc.Metadata;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.channel.ChannelId;\nimport io.netty.util.Attribute;\nimport io.netty.util.AttributeKey;\nimport java.util.Arrays;\nimport java.util.List;\nimport org.apache.rocketmq.auth.authorization.context.DefaultAuthorizationContext;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.TopicFilterType;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.constant.GrpcConstants;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.netty.AttributeKeys;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.RequestHeaderRegistry;\nimport org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.CreateTopicRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.CreateUserRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.HeartbeatRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2;\nimport org.apache.rocketmq.remoting.protocol.header.UnregisterClientRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumerData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.Silent.class)\npublic class DefaultAuthorizationContextBuilderTest {\n\n    private AuthorizationContextBuilder builder;\n\n    @Mock\n    private ChannelHandlerContext channelHandlerContext;\n\n    @Mock\n    private Channel channel;\n\n    @Before\n    public void setUp() throws Exception {\n        AuthConfig authConfig = new AuthConfig();\n        authConfig.setClusterName(\"DefaultCluster\");\n        builder = new DefaultAuthorizationContextBuilder(authConfig);\n        RequestHeaderRegistry.getInstance().initialize();\n    }\n\n    @Test\n    public void buildGrpc() {\n        Metadata metadata = new Metadata();\n        metadata.put(GrpcConstants.AUTHORIZATION_AK, \"rocketmq\");\n        metadata.put(GrpcConstants.REMOTE_ADDRESS, \"192.168.0.1\");\n        metadata.put(GrpcConstants.CHANNEL_ID, \"channel-id\");\n\n        GeneratedMessageV3 request = SendMessageRequest.newBuilder()\n            .addMessages(Message.newBuilder()\n                .setTopic(Resource.newBuilder().setName(\"topic\").build())\n                .build())\n            .build();\n        List<DefaultAuthorizationContext> result = builder.build(metadata, request);\n        Assert.assertEquals(1, result.size());\n        Assert.assertEquals(result.get(0).getSubject().getSubjectKey(), \"User:rocketmq\");\n        Assert.assertEquals(result.get(0).getResource().getResourceKey(), \"Topic:topic\");\n        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.PUB)));\n        Assert.assertEquals(result.get(0).getSourceIp(), \"192.168.0.1\");\n        Assert.assertEquals(result.get(0).getChannelId(), \"channel-id\");\n        Assert.assertEquals(result.get(0).getRpcCode(), SendMessageRequest.getDescriptor().getFullName());\n\n        request = RecallMessageRequest.newBuilder()\n            .setTopic(Resource.newBuilder().setName(\"topic\").build())\n            .setRecallHandle(\"handle\")\n            .build();\n        result = builder.build(metadata, request);\n        Assert.assertEquals(1, result.size());\n        Assert.assertEquals(result.get(0).getSubject().getSubjectKey(), \"User:rocketmq\");\n        Assert.assertEquals(result.get(0).getResource().getResourceKey(), \"Topic:topic\");\n        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.PUB)));\n        Assert.assertEquals(result.get(0).getSourceIp(), \"192.168.0.1\");\n        Assert.assertEquals(result.get(0).getChannelId(), \"channel-id\");\n        Assert.assertEquals(result.get(0).getRpcCode(), RecallMessageRequest.getDescriptor().getFullName());\n\n        request = EndTransactionRequest.newBuilder()\n            .setTopic(Resource.newBuilder().setName(\"topic\").build())\n            .build();\n        result = builder.build(metadata, request);\n        Assert.assertEquals(1, result.size());\n        Assert.assertEquals(result.get(0).getSubject().getSubjectKey(), \"User:rocketmq\");\n        Assert.assertEquals(result.get(0).getResource().getResourceKey(), \"Topic:topic\");\n        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.PUB)));\n\n        request = HeartbeatRequest.newBuilder()\n            .setClientType(ClientType.PUSH_CONSUMER)\n            .setGroup(Resource.newBuilder().setName(\"group\").build())\n            .build();\n        result = builder.build(metadata, request);\n        Assert.assertEquals(1, result.size());\n        Assert.assertEquals(result.get(0).getSubject().getSubjectKey(), \"User:rocketmq\");\n        Assert.assertEquals(result.get(0).getResource().getResourceKey(), \"Group:group\");\n        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.SUB)));\n\n        request = ReceiveMessageRequest.newBuilder()\n            .setMessageQueue(MessageQueue.newBuilder()\n                .setTopic(Resource.newBuilder().setName(\"topic\").build())\n                .build())\n            .setGroup(Resource.newBuilder().setName(\"group\").build())\n            .build();\n        result = builder.build(metadata, request);\n        Assert.assertEquals(2, result.size());\n        Assert.assertEquals(getContext(result, ResourceType.GROUP).getSubject().getSubjectKey(), \"User:rocketmq\");\n        Assert.assertEquals(getContext(result, ResourceType.GROUP).getResource().getResourceKey(), \"Group:group\");\n        Assert.assertTrue(getContext(result, ResourceType.GROUP).getActions().containsAll(Arrays.asList(Action.SUB)));\n        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getSubject().getSubjectKey(), \"User:rocketmq\");\n        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getResource().getResourceKey(), \"Topic:topic\");\n        Assert.assertTrue(getContext(result, ResourceType.TOPIC).getActions().containsAll(Arrays.asList(Action.SUB)));\n\n        request = AckMessageRequest.newBuilder()\n            .setTopic(Resource.newBuilder().setName(\"topic\").build())\n            .setGroup(Resource.newBuilder().setName(\"group\").build())\n            .build();\n        result = builder.build(metadata, request);\n        Assert.assertEquals(2, result.size());\n        Assert.assertEquals(getContext(result, ResourceType.GROUP).getSubject().getSubjectKey(), \"User:rocketmq\");\n        Assert.assertEquals(getContext(result, ResourceType.GROUP).getResource().getResourceKey(), \"Group:group\");\n        Assert.assertTrue(getContext(result, ResourceType.GROUP).getActions().containsAll(Arrays.asList(Action.SUB)));\n        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getSubject().getSubjectKey(), \"User:rocketmq\");\n        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getResource().getResourceKey(), \"Topic:topic\");\n        Assert.assertTrue(getContext(result, ResourceType.TOPIC).getActions().containsAll(Arrays.asList(Action.SUB)));\n\n        request = ForwardMessageToDeadLetterQueueRequest.newBuilder()\n            .setTopic(Resource.newBuilder().setName(\"topic\").build())\n            .setGroup(Resource.newBuilder().setName(\"group\").build())\n            .build();\n        result = builder.build(metadata, request);\n        Assert.assertEquals(2, result.size());\n        Assert.assertEquals(getContext(result, ResourceType.GROUP).getSubject().getSubjectKey(), \"User:rocketmq\");\n        Assert.assertEquals(getContext(result, ResourceType.GROUP).getResource().getResourceKey(), \"Group:group\");\n        Assert.assertTrue(getContext(result, ResourceType.GROUP).getActions().containsAll(Arrays.asList(Action.SUB)));\n        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getSubject().getSubjectKey(), \"User:rocketmq\");\n        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getResource().getResourceKey(), \"Topic:topic\");\n        Assert.assertTrue(getContext(result, ResourceType.TOPIC).getActions().containsAll(Arrays.asList(Action.SUB)));\n\n        request = NotifyClientTerminationRequest.newBuilder()\n            .setGroup(Resource.newBuilder().setName(\"group\").build())\n            .build();\n        result = builder.build(metadata, request);\n        Assert.assertEquals(1, result.size());\n        Assert.assertEquals(result.get(0).getSubject().getSubjectKey(), \"User:rocketmq\");\n        Assert.assertEquals(result.get(0).getResource().getResourceKey(), \"Group:group\");\n        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.SUB)));\n\n        request = ChangeInvisibleDurationRequest.newBuilder()\n            .setGroup(Resource.newBuilder().setName(\"group\").build())\n            .build();\n        result = builder.build(metadata, request);\n        Assert.assertEquals(1, result.size());\n        Assert.assertEquals(result.get(0).getSubject().getSubjectKey(), \"User:rocketmq\");\n        Assert.assertEquals(result.get(0).getResource().getResourceKey(), \"Group:group\");\n        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.SUB)));\n\n        request = QueryRouteRequest.newBuilder()\n            .setTopic(Resource.newBuilder().setName(\"topic\").build())\n            .build();\n        result = builder.build(metadata, request);\n        Assert.assertEquals(1, result.size());\n        Assert.assertEquals(result.get(0).getSubject().getSubjectKey(), \"User:rocketmq\");\n        Assert.assertEquals(result.get(0).getResource().getResourceKey(), \"Topic:topic\");\n        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.PUB, Action.SUB)));\n\n        request = QueryAssignmentRequest.newBuilder()\n            .setTopic(Resource.newBuilder().setName(\"topic\").build())\n            .setGroup(Resource.newBuilder().setName(\"group\").build())\n            .build();\n        result = builder.build(metadata, request);\n        Assert.assertEquals(2, result.size());\n        Assert.assertEquals(getContext(result, ResourceType.GROUP).getSubject().getSubjectKey(), \"User:rocketmq\");\n        Assert.assertEquals(getContext(result, ResourceType.GROUP).getResource().getResourceKey(), \"Group:group\");\n        Assert.assertTrue(getContext(result, ResourceType.GROUP).getActions().containsAll(Arrays.asList(Action.SUB)));\n        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getSubject().getSubjectKey(), \"User:rocketmq\");\n        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getResource().getResourceKey(), \"Topic:topic\");\n        Assert.assertTrue(getContext(result, ResourceType.TOPIC).getActions().containsAll(Arrays.asList(Action.SUB)));\n\n        request = TelemetryCommand.newBuilder()\n            .setSettings(Settings.newBuilder()\n                .setPublishing(Publishing.newBuilder()\n                    .addTopics(Resource.newBuilder().setName(\"topic\").build())\n                    .build())\n                .build())\n            .build();\n        result = builder.build(metadata, request);\n        Assert.assertEquals(1, result.size());\n        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getSubject().getSubjectKey(), \"User:rocketmq\");\n        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getResource().getResourceKey(), \"Topic:topic\");\n        Assert.assertTrue(getContext(result, ResourceType.TOPIC).getActions().containsAll(Arrays.asList(Action.PUB)));\n\n        request = TelemetryCommand.newBuilder()\n            .setSettings(Settings.newBuilder()\n                .setSubscription(Subscription.newBuilder()\n                    .setGroup(Resource.newBuilder().setName(\"group\").build())\n                    .addSubscriptions(SubscriptionEntry.newBuilder()\n                        .setTopic(Resource.newBuilder().setName(\"topic\").build())\n                        .build())\n                    .build())\n                .build())\n            .build();\n        result = builder.build(metadata, request);\n        Assert.assertEquals(2, result.size());\n        Assert.assertEquals(getContext(result, ResourceType.GROUP).getSubject().getSubjectKey(), \"User:rocketmq\");\n        Assert.assertEquals(getContext(result, ResourceType.GROUP).getResource().getResourceKey(), \"Group:group\");\n        Assert.assertTrue(getContext(result, ResourceType.GROUP).getActions().containsAll(Arrays.asList(Action.SUB)));\n        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getSubject().getSubjectKey(), \"User:rocketmq\");\n        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getResource().getResourceKey(), \"Topic:topic\");\n        Assert.assertTrue(getContext(result, ResourceType.TOPIC).getActions().containsAll(Arrays.asList(Action.SUB)));\n    }\n\n    @Test\n    public void buildRemoting() {\n        when(channel.id()).thenReturn(mockChannelId(\"channel-id\"));\n        when(channel.hasAttr(eq(AttributeKeys.PROXY_PROTOCOL_ADDR))).thenReturn(true);\n        when(channel.attr(eq(AttributeKeys.PROXY_PROTOCOL_ADDR))).thenReturn(mockAttribute(\"192.168.0.1\"));\n        when(channel.hasAttr(eq(AttributeKeys.PROXY_PROTOCOL_PORT))).thenReturn(true);\n        when(channel.attr(eq(AttributeKeys.PROXY_PROTOCOL_PORT))).thenReturn(mockAttribute(\"1234\"));\n        when(channelHandlerContext.channel()).thenReturn(channel);\n\n        SendMessageRequestHeader sendMessageRequestHeader = new SendMessageRequestHeader();\n        sendMessageRequestHeader.setTopic(\"topic\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, sendMessageRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        List<DefaultAuthorizationContext> result = builder.build(channelHandlerContext, request);\n        Assert.assertEquals(1, result.size());\n        Assert.assertEquals(\"User:rocketmq\", result.get(0).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Topic:topic\", result.get(0).getResource().getResourceKey());\n        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.PUB)));\n        Assert.assertEquals(\"192.168.0.1\", result.get(0).getSourceIp());\n        Assert.assertEquals(\"channel-id\", result.get(0).getChannelId());\n        Assert.assertEquals(RequestCode.SEND_MESSAGE + \"\", result.get(0).getRpcCode());\n\n        sendMessageRequestHeader = new SendMessageRequestHeader();\n        sendMessageRequestHeader.setTopic(\"%RETRY%group\");\n        request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, sendMessageRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        result = builder.build(channelHandlerContext, request);\n        Assert.assertEquals(1, result.size());\n        Assert.assertEquals(\"User:rocketmq\", result.get(0).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Group:group\", result.get(0).getResource().getResourceKey());\n        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.SUB)));\n\n        SendMessageRequestHeaderV2 sendMessageRequestHeaderV2 = new SendMessageRequestHeaderV2();\n        sendMessageRequestHeaderV2.setTopic(\"topic\");\n        request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, sendMessageRequestHeaderV2);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        result = builder.build(channelHandlerContext, request);\n        Assert.assertEquals(1, result.size());\n        Assert.assertEquals(\"User:rocketmq\", result.get(0).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Topic:topic\", result.get(0).getResource().getResourceKey());\n        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.PUB)));\n\n        sendMessageRequestHeaderV2 = new SendMessageRequestHeaderV2();\n        sendMessageRequestHeaderV2.setTopic(\"%RETRY%group\");\n        request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, sendMessageRequestHeaderV2);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        result = builder.build(channelHandlerContext, request);\n        Assert.assertEquals(1, result.size());\n        Assert.assertEquals(\"User:rocketmq\", result.get(0).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Group:group\", result.get(0).getResource().getResourceKey());\n        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.SUB)));\n\n        RecallMessageRequestHeader recallMessageRequestHeader = new RecallMessageRequestHeader();\n        recallMessageRequestHeader.setTopic(\"topic\");\n        recallMessageRequestHeader.setRecallHandle(\"handle\");\n        request = RemotingCommand.createRequestCommand(RequestCode.RECALL_MESSAGE, recallMessageRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        result = builder.build(channelHandlerContext, request);\n        Assert.assertEquals(1, result.size());\n        Assert.assertEquals(\"User:rocketmq\", result.get(0).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Topic:topic\", result.get(0).getResource().getResourceKey());\n        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.PUB)));\n        Assert.assertEquals(\"192.168.0.1\", result.get(0).getSourceIp());\n        Assert.assertEquals(\"channel-id\", result.get(0).getChannelId());\n        Assert.assertEquals(RequestCode.RECALL_MESSAGE + \"\", result.get(0).getRpcCode());\n\n        EndTransactionRequestHeader endTransactionRequestHeader = new EndTransactionRequestHeader();\n        endTransactionRequestHeader.setTopic(\"topic\");\n        request = RemotingCommand.createRequestCommand(RequestCode.END_TRANSACTION, endTransactionRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        result = builder.build(channelHandlerContext, request);\n        Assert.assertEquals(1, result.size());\n        Assert.assertEquals(\"User:rocketmq\", result.get(0).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Topic:topic\", result.get(0).getResource().getResourceKey());\n        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.PUB)));\n\n        endTransactionRequestHeader = new EndTransactionRequestHeader();\n        request = RemotingCommand.createRequestCommand(RequestCode.END_TRANSACTION, endTransactionRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        result = builder.build(channelHandlerContext, request);\n        Assert.assertEquals(0, result.size());\n\n        ConsumerSendMsgBackRequestHeader consumerSendMsgBackRequestHeader = new ConsumerSendMsgBackRequestHeader();\n        consumerSendMsgBackRequestHeader.setGroup(\"group\");\n        request = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, consumerSendMsgBackRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        result = builder.build(channelHandlerContext, request);\n        Assert.assertEquals(1, result.size());\n        Assert.assertEquals(\"User:rocketmq\", result.get(0).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Group:group\", result.get(0).getResource().getResourceKey());\n        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.SUB)));\n\n        PullMessageRequestHeader pullMessageRequestHeader = new PullMessageRequestHeader();\n        pullMessageRequestHeader.setTopic(\"topic\");\n        pullMessageRequestHeader.setConsumerGroup(\"group\");\n        request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, pullMessageRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        result = builder.build(channelHandlerContext, request);\n        Assert.assertEquals(2, result.size());\n        Assert.assertEquals(\"User:rocketmq\", getContext(result, ResourceType.GROUP).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Group:group\", getContext(result, ResourceType.GROUP).getResource().getResourceKey());\n        Assert.assertTrue(getContext(result, ResourceType.GROUP).getActions().containsAll(Arrays.asList(Action.SUB)));\n        Assert.assertEquals(\"User:rocketmq\", getContext(result, ResourceType.TOPIC).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Topic:topic\", getContext(result, ResourceType.TOPIC).getResource().getResourceKey());\n        Assert.assertTrue(getContext(result, ResourceType.TOPIC).getActions().containsAll(Arrays.asList(Action.SUB)));\n\n        QueryMessageRequestHeader queryMessageRequestHeader = new QueryMessageRequestHeader();\n        queryMessageRequestHeader.setTopic(\"topic\");\n        request = RemotingCommand.createRequestCommand(RequestCode.QUERY_MESSAGE, queryMessageRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        result = builder.build(channelHandlerContext, request);\n        Assert.assertEquals(1, result.size());\n        Assert.assertEquals(\"User:rocketmq\", result.get(0).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Topic:topic\", result.get(0).getResource().getResourceKey());\n        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.SUB, Action.GET)));\n\n        HeartbeatRequestHeader heartbeatRequestHeader = new HeartbeatRequestHeader();\n        request = RemotingCommand.createRequestCommand(RequestCode.HEART_BEAT, heartbeatRequestHeader);\n        HeartbeatData heartbeatData = new HeartbeatData();\n        ConsumerData consumerData = new ConsumerData();\n        consumerData.setGroupName(\"group\");\n        SubscriptionData subscriptionData = new SubscriptionData();\n        subscriptionData.setTopic(\"topic\");\n        consumerData.setSubscriptionDataSet(Sets.newHashSet(subscriptionData));\n        heartbeatData.setConsumerDataSet(Sets.newHashSet(consumerData));\n        request.setBody(JSON.toJSONBytes(heartbeatData));\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        result = builder.build(channelHandlerContext, request);\n        Assert.assertEquals(2, result.size());\n        Assert.assertEquals(\"User:rocketmq\", getContext(result, ResourceType.GROUP).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Group:group\", getContext(result, ResourceType.GROUP).getResource().getResourceKey());\n        Assert.assertTrue(getContext(result, ResourceType.GROUP).getActions().containsAll(Arrays.asList(Action.SUB)));\n        Assert.assertEquals(\"User:rocketmq\", getContext(result, ResourceType.TOPIC).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Topic:topic\", getContext(result, ResourceType.TOPIC).getResource().getResourceKey());\n        Assert.assertTrue(getContext(result, ResourceType.TOPIC).getActions().containsAll(Arrays.asList(Action.SUB)));\n\n        UnregisterClientRequestHeader unregisterClientRequestHeader = new UnregisterClientRequestHeader();\n        unregisterClientRequestHeader.setConsumerGroup(\"group\");\n        request = RemotingCommand.createRequestCommand(RequestCode.UNREGISTER_CLIENT, unregisterClientRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        result = builder.build(channelHandlerContext, request);\n        Assert.assertEquals(1, result.size());\n        Assert.assertEquals(\"User:rocketmq\", result.get(0).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Group:group\", result.get(0).getResource().getResourceKey());\n        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.SUB)));\n\n        GetConsumerListByGroupRequestHeader getConsumerListByGroupRequestHeader = new GetConsumerListByGroupRequestHeader();\n        getConsumerListByGroupRequestHeader.setConsumerGroup(\"group\");\n        request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_LIST_BY_GROUP, getConsumerListByGroupRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        result = builder.build(channelHandlerContext, request);\n        Assert.assertEquals(1, result.size());\n        Assert.assertEquals(\"User:rocketmq\", result.get(0).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Group:group\", result.get(0).getResource().getResourceKey());\n        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.SUB, Action.GET)));\n\n        QueryConsumerOffsetRequestHeader queryConsumerOffsetRequestHeader = new QueryConsumerOffsetRequestHeader();\n        queryConsumerOffsetRequestHeader.setTopic(\"topic\");\n        queryConsumerOffsetRequestHeader.setConsumerGroup(\"group\");\n        request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, queryConsumerOffsetRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        result = builder.build(channelHandlerContext, request);\n        Assert.assertEquals(2, result.size());\n        Assert.assertEquals(\"User:rocketmq\", getContext(result, ResourceType.GROUP).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Group:group\", getContext(result, ResourceType.GROUP).getResource().getResourceKey());\n        Assert.assertTrue(getContext(result, ResourceType.GROUP).getActions().containsAll(Arrays.asList(Action.SUB)));\n        Assert.assertEquals(\"User:rocketmq\", getContext(result, ResourceType.TOPIC).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Topic:topic\", getContext(result, ResourceType.TOPIC).getResource().getResourceKey());\n        Assert.assertTrue(getContext(result, ResourceType.TOPIC).getActions().containsAll(Arrays.asList(Action.SUB)));\n\n        UpdateConsumerOffsetRequestHeader updateConsumerOffsetRequestHeader = new UpdateConsumerOffsetRequestHeader();\n        updateConsumerOffsetRequestHeader.setTopic(\"topic\");\n        updateConsumerOffsetRequestHeader.setConsumerGroup(\"group\");\n        request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_CONSUMER_OFFSET, updateConsumerOffsetRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        result = builder.build(channelHandlerContext, request);\n        Assert.assertEquals(2, result.size());\n        Assert.assertEquals(\"User:rocketmq\", getContext(result, ResourceType.GROUP).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Group:group\", getContext(result, ResourceType.GROUP).getResource().getResourceKey());\n        Assert.assertTrue(getContext(result, ResourceType.GROUP).getActions().containsAll(Arrays.asList(Action.SUB, Action.UPDATE)));\n        Assert.assertEquals(\"User:rocketmq\", getContext(result, ResourceType.TOPIC).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Topic:topic\", getContext(result, ResourceType.TOPIC).getResource().getResourceKey());\n        Assert.assertTrue(getContext(result, ResourceType.TOPIC).getActions().containsAll(Arrays.asList(Action.SUB, Action.UPDATE)));\n\n        CreateTopicRequestHeader createTopicRequestHeader = new CreateTopicRequestHeader();\n        createTopicRequestHeader.setTopic(\"topic\");\n        createTopicRequestHeader.setTopicFilterType(TopicFilterType.SINGLE_TAG.name());\n        request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_TOPIC, createTopicRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        result = builder.build(channelHandlerContext, request);\n        Assert.assertEquals(1, result.size());\n        Assert.assertEquals(\"User:rocketmq\", result.get(0).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Topic:topic\", result.get(0).getResource().getResourceKey());\n        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.CREATE)));\n\n        CreateUserRequestHeader createUserRequestHeader = new CreateUserRequestHeader();\n        createUserRequestHeader.setUsername(\"abc\");\n        request = RemotingCommand.createRequestCommand(RequestCode.AUTH_CREATE_USER, createUserRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        result = builder.build(channelHandlerContext, request);\n        Assert.assertEquals(1, result.size());\n        Assert.assertEquals(\"User:rocketmq\", result.get(0).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Cluster:DefaultCluster\", result.get(0).getResource().getResourceKey());\n        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.UPDATE)));\n\n        LockBatchRequestBody lockBatchRequestBody = new LockBatchRequestBody();\n        lockBatchRequestBody.setConsumerGroup(\"group\");\n        java.util.Set<org.apache.rocketmq.common.message.MessageQueue> lockMqSet = new java.util.HashSet<>();\n\n        lockMqSet.add(new org.apache.rocketmq.common.message.MessageQueue(\"topic\", \"broker-a\", 0));\n        // retry topic, should be skipped\n        lockMqSet.add(new org.apache.rocketmq.common.message.MessageQueue(\"%RETRY%group\", \"broker-a\", 1));\n        lockBatchRequestBody.setMqSet(lockMqSet);\n\n        request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, null);\n        request.setBody(JSON.toJSONBytes(lockBatchRequestBody));\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n\n        result = builder.build(channelHandlerContext, request);\n        Assert.assertEquals(2, result.size());\n\n        Assert.assertEquals(\"User:rocketmq\", getContext(result, ResourceType.GROUP).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Group:group\", getContext(result, ResourceType.GROUP).getResource().getResourceKey());\n        Assert.assertTrue(getContext(result, ResourceType.GROUP).getActions().containsAll(Arrays.asList(Action.SUB)));\n\n        Assert.assertEquals(\"User:rocketmq\", getContext(result, ResourceType.TOPIC).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Topic:topic\", getContext(result, ResourceType.TOPIC).getResource().getResourceKey());\n        Assert.assertTrue(getContext(result, ResourceType.TOPIC).getActions().containsAll(Arrays.asList(Action.SUB)));\n\n        Assert.assertEquals(\"192.168.0.1\", getContext(result, ResourceType.TOPIC).getSourceIp());\n        Assert.assertEquals(\"channel-id\", getContext(result, ResourceType.TOPIC).getChannelId());\n        Assert.assertEquals(String.valueOf(RequestCode.LOCK_BATCH_MQ), getContext(result, ResourceType.TOPIC).getRpcCode());\n\n        UnlockBatchRequestBody unlockBatchRequestBody = new UnlockBatchRequestBody();\n        unlockBatchRequestBody.setConsumerGroup(\"group\");\n        java.util.Set<org.apache.rocketmq.common.message.MessageQueue> unlockMqSet = new java.util.HashSet<>();\n        unlockMqSet.add(new org.apache.rocketmq.common.message.MessageQueue(\"topic\", \"broker-a\", 0));\n        // retry topic, should be skipped\n        unlockMqSet.add(new org.apache.rocketmq.common.message.MessageQueue(\"%RETRY%group\", \"broker-a\", 1));\n        unlockBatchRequestBody.setMqSet(unlockMqSet);\n\n        request = RemotingCommand.createRequestCommand(RequestCode.UNLOCK_BATCH_MQ, null);\n        request.setBody(JSON.toJSONBytes(unlockBatchRequestBody));\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n\n        result = builder.build(channelHandlerContext, request);\n        Assert.assertEquals(2, result.size());\n\n        Assert.assertEquals(\"User:rocketmq\", getContext(result, ResourceType.GROUP).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Group:group\", getContext(result, ResourceType.GROUP).getResource().getResourceKey());\n        Assert.assertTrue(getContext(result, ResourceType.GROUP).getActions().containsAll(Arrays.asList(Action.SUB)));\n\n        Assert.assertEquals(\"User:rocketmq\", getContext(result, ResourceType.TOPIC).getSubject().getSubjectKey());\n        Assert.assertEquals(\"Topic:topic\", getContext(result, ResourceType.TOPIC).getResource().getResourceKey());\n        Assert.assertTrue(getContext(result, ResourceType.TOPIC).getActions().containsAll(Arrays.asList(Action.SUB)));\n\n        Assert.assertEquals(\"192.168.0.1\", getContext(result, ResourceType.TOPIC).getSourceIp());\n        Assert.assertEquals(\"channel-id\", getContext(result, ResourceType.TOPIC).getChannelId());\n        Assert.assertEquals(String.valueOf(RequestCode.UNLOCK_BATCH_MQ), getContext(result, ResourceType.TOPIC).getRpcCode());\n\n    }\n\n    private DefaultAuthorizationContext getContext(List<DefaultAuthorizationContext> contexts,\n        ResourceType resourceType) {\n        return contexts.stream().filter(context -> context.getResource().getResourceType() == resourceType)\n            .findFirst().orElse(null);\n    }\n\n    private ChannelId mockChannelId(String channelId) {\n        return new ChannelId() {\n            @Override\n            public String asShortText() {\n                return channelId;\n            }\n\n            @Override\n            public String asLongText() {\n                return channelId;\n            }\n\n            @Override\n            public int compareTo(ChannelId o) {\n                return 0;\n            }\n        };\n    }\n\n    private Attribute<String> mockAttribute(String value) {\n        return new Attribute<String>() {\n            @Override\n            public AttributeKey<String> key() {\n                return null;\n            }\n\n            @Override\n            public String get() {\n                return value;\n            }\n\n            @Override\n            public void set(String value) {\n            }\n\n            @Override\n            public String getAndSet(String value) {\n                return null;\n            }\n\n            @Override\n            public String setIfAbsent(String value) {\n                return null;\n            }\n\n            @Override\n            public String getAndRemove() {\n                return null;\n            }\n\n            @Override\n            public boolean compareAndSet(String oldValue, String newValue) {\n                return false;\n            }\n\n            @Override\n            public void remove() {\n\n            }\n        };\n    }\n}"
  },
  {
    "path": "auth/src/test/java/org/apache/rocketmq/auth/authorization/chain/AclAuthorizationHandlerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.chain;\n\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;\nimport org.apache.rocketmq.auth.authentication.manager.AuthenticationMetadataManager;\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.auth.authorization.context.DefaultAuthorizationContext;\nimport org.apache.rocketmq.auth.authorization.enums.Decision;\nimport org.apache.rocketmq.auth.authorization.enums.PolicyType;\nimport org.apache.rocketmq.auth.authorization.exception.AuthorizationException;\nimport org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;\nimport org.apache.rocketmq.auth.authorization.manager.AuthorizationMetadataManager;\nimport org.apache.rocketmq.auth.authorization.model.Acl;\nimport org.apache.rocketmq.auth.authorization.model.Policy;\nimport org.apache.rocketmq.auth.authorization.model.PolicyEntry;\nimport org.apache.rocketmq.auth.authorization.model.Resource;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.auth.helper.AuthTestHelper;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.chain.HandlerChain;\nimport org.apache.rocketmq.common.resource.ResourcePattern;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\n\nimport static org.mockito.Mockito.mock;\n\npublic class AclAuthorizationHandlerTest {\n\n    private AuthConfig authConfig;\n    private AuthenticationMetadataManager authenticationMetadataManager;\n    private AuthorizationMetadataManager authorizationMetadataManager;\n    private AclAuthorizationHandler handler;\n    private HandlerChain<DefaultAuthorizationContext, CompletableFuture<Void>> nextChain;\n\n    @Before\n    public void setUp() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        this.authConfig = AuthTestHelper.createDefaultConfig();\n        this.authenticationMetadataManager = AuthenticationFactory.getMetadataManager(this.authConfig);\n        this.authorizationMetadataManager = AuthorizationFactory.getMetadataManager(this.authConfig);\n        this.handler = new AclAuthorizationHandler(this.authConfig);\n        this.nextChain = mock(HandlerChain.class);\n        clearAllAcls();\n        clearAllUsers();\n    }\n\n    @After\n    public void tearDown() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        clearAllAcls();\n        clearAllUsers();\n        this.authenticationMetadataManager.shutdown();\n        this.authorizationMetadataManager.shutdown();\n    }\n\n    @Test\n    public void testNoAclThrows() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        // Create a user with no ACL entries.\n        User user = User.of(\"noacl\", \"pwd\");\n        authenticationMetadataManager.createUser(user).join();\n\n        DefaultAuthorizationContext ctx = buildContext(user, Resource.ofTopic(\"t1\"), Action.SUB, \"127.0.0.1\");\n\n        AuthorizationException authorizationException = Assert.assertThrows(AuthorizationException.class, () -> {\n            try {\n                handler.handle(ctx, nextChain).join();\n            } catch (Exception e) {\n                AuthTestHelper.handleException(e);\n            }\n        });\n        Assert.assertEquals(\"User:noacl has no permission to access Topic:t1 from 127.0.0.1, no matched policies.\",\n                authorizationException.getMessage());\n    }\n\n    @Test\n    public void testNoMatchedPolicyThrows() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"no_match_acl\", \"pwd\");\n        authenticationMetadataManager.createUser(user).join();\n\n        Acl acl = AuthTestHelper.buildAcl(\"User:no_match_acl\", \"Topic:abc\", Action.SUB.getName(), null, Decision.ALLOW);\n        authorizationMetadataManager.createAcl(acl).join();\n\n        // Ensure an ACL has been created.\n        List<Acl> acls = authorizationMetadataManager.listAcl(null, null).join();\n        Assert.assertEquals(1, acls.size());\n\n        // The requested resource does not match any ACL entry.\n        DefaultAuthorizationContext ctx = buildContext(user, Resource.ofTopic(\"t1\"), Action.SUB, \"127.0.0.1\");\n\n        AuthorizationException authorizationException = Assert.assertThrows(AuthorizationException.class, () -> {\n            try {\n                handler.handle(ctx, nextChain).join();\n            } catch (Exception e) {\n                AuthTestHelper.handleException(e);\n            }\n        });\n        Assert.assertEquals(\"User:no_match_acl has no permission to access Topic:t1 from 127.0.0.1, no matched policies.\",\n                authorizationException.getMessage());\n    }\n\n    @Test\n    public void testDecisionDenyThrows() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"deny\", \"pwd\");\n        authenticationMetadataManager.createUser(user).join();\n\n        // The ACL entry matches, but the decision is DENY.\n        Acl acl = AuthTestHelper.buildAcl(\"User:deny\", \"Topic:t1\", Action.SUB.getName(), null, Decision.DENY);\n        authorizationMetadataManager.createAcl(acl).join();\n\n        List<Acl> acls = authorizationMetadataManager.listAcl(null, null).join();\n        Assert.assertEquals(1, acls.size());\n\n        DefaultAuthorizationContext ctx = buildContext(user, Resource.ofTopic(\"t1\"), Action.SUB, \"127.0.0.1\");\n\n        AuthorizationException authorizationException = Assert.assertThrows(AuthorizationException.class, () -> {\n            try {\n                handler.handle(ctx, nextChain).join();\n            } catch (Exception e) {\n                AuthTestHelper.handleException(e);\n            }\n        });\n        Assert.assertEquals(\"User:deny has no permission to access Topic:t1 from 127.0.0.1, the decision is deny.\",\n                authorizationException.getMessage());\n    }\n\n    @Test\n    public void testAllowDoesNotThrow() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"allow\", \"pwd\");\n        authenticationMetadataManager.createUser(user).join();\n\n        // The ACL matches and the decision is ALLOW.\n        Acl acl = AuthTestHelper.buildAcl(\"User:allow\", \"Topic:t1\", Action.SUB.getName(), null, Decision.ALLOW);\n        authorizationMetadataManager.createAcl(acl).join();\n\n        DefaultAuthorizationContext ctx = buildContext(user, Resource.ofTopic(\"t1\"), Action.SUB, \"127.0.0.1\");\n\n        handler.handle(ctx, nextChain).join();\n    }\n\n    @Test\n    public void testDenyBeatsAllow() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"user\", \"pwd\");\n        authenticationMetadataManager.createUser(user).join();\n\n        // Set up policy entries with both ALLOW and DENY for the same resource.\n        Resource resource = Resource.of(ResourceType.TOPIC, \"t1\", ResourcePattern.LITERAL);\n        PolicyEntry allowLiteral = PolicyEntry.of(resource, Collections.singletonList(Action.SUB), null, Decision.ALLOW);\n        PolicyEntry denyLiteral = PolicyEntry.of(resource, Collections.singletonList(Action.SUB), null, Decision.DENY);\n\n        // Include both entries in the policy to verify precedence.\n        Policy policy = Policy.of(PolicyType.CUSTOM, new ArrayList<>(Arrays.asList(allowLiteral, denyLiteral, allowLiteral)));\n        authorizationMetadataManager.createAcl(Acl.of(user, policy)).join();\n\n        DefaultAuthorizationContext ctx = buildContext(user, resource, Action.SUB, \"127.0.0.1\");\n\n        AuthorizationException authorizationException = Assert.assertThrows(AuthorizationException.class, () -> {\n            try {\n                handler.handle(ctx, nextChain).join();\n            } catch (Throwable e) {\n                AuthTestHelper.handleException(e);\n            }\n        });\n        // DENY should take precedence.\n        Assert.assertEquals(\"User:user has no permission to access Topic:t1 from 127.0.0.1, the decision is deny.\", authorizationException.getMessage());\n    }\n\n    @Test\n    public void testPrefixedLongerDenyBeatsPrefixedShorterAllow() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"user\", \"pwd\");\n        authenticationMetadataManager.createUser(user).join();\n\n        // The longer PREFIXED DENY policy entry should take precedence over the shorter PREFIXED ALLOW policy entry.\n        PolicyEntry denyLonger = PolicyEntry.of(\n                Resource.of(ResourceType.TOPIC, \"t1-abc\", ResourcePattern.PREFIXED),\n                Collections.singletonList(Action.SUB), null, Decision.DENY);\n        PolicyEntry allowShorter = PolicyEntry.of(\n                Resource.of(ResourceType.TOPIC, \"t1-\", ResourcePattern.PREFIXED),\n                Collections.singletonList(Action.SUB), null, Decision.ALLOW);\n\n        Policy policy = Policy.of(PolicyType.CUSTOM, new ArrayList<>(Arrays.asList(allowShorter, denyLonger)));\n        authorizationMetadataManager.createAcl(Acl.of(user, policy)).join();\n\n        DefaultAuthorizationContext ctx = buildContext(user, Resource.ofTopic(\"t1-abcd\"), Action.SUB, \"127.0.0.1\");\n        AuthorizationException authorizationException = Assert.assertThrows(AuthorizationException.class, () -> {\n            try {\n                handler.handle(ctx, nextChain).join();\n            } catch (Throwable e) {\n                AuthTestHelper.handleException(e);\n            }\n        });\n        Assert.assertEquals(\"User:user has no permission to access Topic:t1-abcd from 127.0.0.1, the decision is deny.\", authorizationException.getMessage());\n    }\n\n    @Test\n    public void testLiteralAllowBeatsPrefixedDeny() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"user\", \"pwd\");\n        authenticationMetadataManager.createUser(user).join();\n\n        // The LITERAL ALLOW policy entry should take precedence over the PREFIXED DENY policy entry.\n        PolicyEntry allowLiteral = PolicyEntry.of(\n                Resource.of(ResourceType.TOPIC, \"t1\", ResourcePattern.LITERAL),\n                Collections.singletonList(Action.SUB), null, Decision.ALLOW);\n        PolicyEntry denyPrefixed = PolicyEntry.of(\n                Resource.of(ResourceType.TOPIC, \"t\", ResourcePattern.PREFIXED),\n                Collections.singletonList(Action.SUB), null, Decision.DENY);\n\n        Policy policy = Policy.of(PolicyType.CUSTOM, new ArrayList<>(Arrays.asList(denyPrefixed, allowLiteral)));\n        authorizationMetadataManager.createAcl(Acl.of(user, policy)).join();\n\n        DefaultAuthorizationContext ctx = buildContext(user, Resource.ofTopic(\"t1\"), Action.SUB, \"127.0.0.1\");\n        handler.handle(ctx, nextChain).join();\n    }\n\n    @Test\n    public void testTopicTypeAllowBeatsAnyTypeDeny() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"user\", \"pwd\");\n        authenticationMetadataManager.createUser(user).join();\n\n        // The ALLOW policy entry with resource type TOPIC should take precedence over the DENY policy entry with resource type ANY.\n        PolicyEntry denyAnyType = PolicyEntry.of(\n                Resource.of(ResourceType.ANY, \"t1\", ResourcePattern.LITERAL),\n                Collections.singletonList(Action.SUB), null, Decision.DENY);\n        PolicyEntry allowTopicType = PolicyEntry.of(\n                Resource.of(ResourceType.TOPIC, \"t1\", ResourcePattern.LITERAL),\n                Collections.singletonList(Action.SUB), null, Decision.ALLOW);\n\n        Policy policy = Policy.of(PolicyType.CUSTOM, new ArrayList<>(Arrays.asList(allowTopicType, denyAnyType)));\n        authorizationMetadataManager.createAcl(Acl.of(user, policy)).join();\n\n        DefaultAuthorizationContext ctx = buildContext(user, Resource.ofTopic(\"t1\"), Action.SUB, \"127.0.0.1\");\n        handler.handle(ctx, nextChain).join();\n    }\n\n    @Test\n    public void testPrefixedPatternAllowBeatsAnyPatternDeny() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"user\", \"pwd\");\n        authenticationMetadataManager.createUser(user).join();\n\n        // The PREFIXED pattern ALLOW policy entry should take precedence over the ANY pattern DENY policy entry.\n        PolicyEntry denyAny = PolicyEntry.of(\n                Resource.of(ResourceType.TOPIC, null, ResourcePattern.ANY),\n                Collections.singletonList(Action.SUB), null, Decision.DENY);\n        PolicyEntry allowPrefixed = PolicyEntry.of(\n                Resource.of(ResourceType.TOPIC, \"t1\", ResourcePattern.PREFIXED),\n                Collections.singletonList(Action.SUB), null, Decision.ALLOW);\n\n        Policy policy = Policy.of(PolicyType.CUSTOM, new ArrayList<>(Arrays.asList(allowPrefixed, denyAny)));\n        authorizationMetadataManager.createAcl(Acl.of(user, policy)).join();\n\n        DefaultAuthorizationContext ctx = buildContext(user, Resource.ofTopic(\"t1\"), Action.SUB, \"127.0.0.1\");\n        handler.handle(ctx, nextChain).join();\n    }\n\n    @Test\n    public void testLiteralPatternDenyBeatsAnyPatternAllow() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"user\", \"pwd\");\n        authenticationMetadataManager.createUser(user).join();\n\n        // The LITERAL pattern DENY policy entry should take precedence over the ANY pattern ALLOW policy entry.\n        PolicyEntry allowAny = PolicyEntry.of(\n                Resource.of(ResourceType.TOPIC, null, ResourcePattern.ANY),\n                Collections.singletonList(Action.SUB), null, Decision.ALLOW);\n        PolicyEntry denyLiteral = PolicyEntry.of(\n                Resource.of(ResourceType.TOPIC, \"t1\", ResourcePattern.LITERAL),\n                Collections.singletonList(Action.SUB), null, Decision.DENY);\n\n\n        Policy policy = Policy.of(PolicyType.CUSTOM, new ArrayList<>(Arrays.asList(allowAny, denyLiteral)));\n        authorizationMetadataManager.createAcl(Acl.of(user, policy)).join();\n\n        DefaultAuthorizationContext ctx = buildContext(user, Resource.ofTopic(\"t1\"), Action.SUB, \"127.0.0.1\");\n        AuthorizationException authorizationException = Assert.assertThrows(AuthorizationException.class, () -> {\n            try {\n                handler.handle(ctx, nextChain).join();\n            } catch (Throwable e) {\n                AuthTestHelper.handleException(e);\n            }\n        });\n        Assert.assertEquals(\"User:user has no permission to access Topic:t1 from 127.0.0.1, the decision is deny.\", authorizationException.getMessage());\n    }\n\n    private DefaultAuthorizationContext buildContext(Subject subject, Resource resource, Action action, String sourceIp) {\n        return DefaultAuthorizationContext.of(subject, resource, action, sourceIp);\n    }\n\n    private void clearAllUsers() {\n        List<User> users = this.authenticationMetadataManager.listUser(null).join();\n        if (CollectionUtils.isEmpty(users)) {\n            return;\n        }\n        users.forEach(user -> this.authenticationMetadataManager.deleteUser(user.getUsername()).join());\n    }\n\n    private void clearAllAcls() {\n        List<Acl> acls = this.authorizationMetadataManager.listAcl(null, null).join();\n        if (CollectionUtils.isEmpty(acls)) {\n            return;\n        }\n        acls.forEach(acl -> this.authorizationMetadataManager.deleteAcl(acl.getSubject(), null, null).join());\n    }\n}\n"
  },
  {
    "path": "auth/src/test/java/org/apache/rocketmq/auth/authorization/chain/UserAuthorizationHandlerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.chain;\n\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\n\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.rocketmq.auth.authentication.enums.UserStatus;\nimport org.apache.rocketmq.auth.authentication.enums.UserType;\nimport org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;\nimport org.apache.rocketmq.auth.authentication.manager.AuthenticationMetadataManager;\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.auth.authorization.context.DefaultAuthorizationContext;\nimport org.apache.rocketmq.auth.authorization.exception.AuthorizationException;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.auth.helper.AuthTestHelper;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.chain.HandlerChain;\nimport org.apache.rocketmq.auth.authorization.model.Resource;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.never;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n\npublic class UserAuthorizationHandlerTest {\n\n    private AuthConfig authConfig;\n    private AuthenticationMetadataManager authenticationMetadataManager;\n    private UserAuthorizationHandler handler;\n    private HandlerChain<DefaultAuthorizationContext, CompletableFuture<Void>> nextChain;\n\n    @Before\n    public void setUp() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        this.authConfig = AuthTestHelper.createDefaultConfig();\n        this.authenticationMetadataManager = AuthenticationFactory.getMetadataManager(this.authConfig);\n        this.handler = new UserAuthorizationHandler(this.authConfig, null);\n        this.nextChain = mock(HandlerChain.class);\n        clearAllUsers();\n    }\n\n    @After\n    public void tearDown() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        clearAllUsers();\n        this.authenticationMetadataManager.shutdown();\n    }\n\n    @Test\n    public void testUserNotFoundThrows() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User noSuchUser = User.of(\"no_such_user\", \"pwd\");\n        DefaultAuthorizationContext ctx = buildContext(noSuchUser, Resource.ofTopic(\"t1\"), Action.SUB, \"127.0.0.1\");\n\n        AuthorizationException authorizationException = Assert.assertThrows(AuthorizationException.class, () -> {\n            try {\n                handler.handle(ctx, nextChain).join();\n            } catch (Exception e) {\n                AuthTestHelper.handleException(e);\n            }\n        });\n        Assert.assertEquals(\"User:no_such_user not found.\", authorizationException.getMessage());\n    }\n\n    @Test\n    public void testUserDisabledThrows() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"disabled\", \"pwd\");\n        authenticationMetadataManager.createUser(user).join();\n        User saved = authenticationMetadataManager.getUser(\"disabled\").join();\n        saved.setUserStatus(UserStatus.DISABLE);\n        authenticationMetadataManager.updateUser(saved).join();\n\n        DefaultAuthorizationContext ctx = buildContext(user, Resource.ofTopic(\"t1\"), Action.SUB, \"127.0.0.1\");\n\n        AuthorizationException authorizationException = Assert.assertThrows(AuthorizationException.class, () -> {\n            try {\n                handler.handle(ctx, nextChain).join();\n            } catch (Exception e) {\n                AuthTestHelper.handleException(e);\n            }\n        });\n\n        Assert.assertEquals(\"User:disabled is disabled.\", authorizationException.getMessage());\n        verify(nextChain, never()).handle(any());\n    }\n\n    @Test\n    public void testSuperUserBypassNextChain() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User superUser = User.of(\"super\", \"pwd\", UserType.SUPER);\n        authenticationMetadataManager.createUser(superUser).join();\n\n        DefaultAuthorizationContext ctx = buildContext(superUser, Resource.ofTopic(\"t1\"), Action.SUB, \"127.0.0.1\");\n\n        handler.handle(ctx, nextChain).join();\n        // super user should bypass the next chain\n        verify(nextChain, never()).handle(any());\n    }\n\n    @Test\n    public void testNormalUserGoesToNextChain() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User normalUser = User.of(\"normal\", \"pwd\", UserType.NORMAL);\n        authenticationMetadataManager.createUser(normalUser).join();\n\n        DefaultAuthorizationContext ctx = buildContext(normalUser, Resource.ofTopic(\"t1\"), Action.SUB, \"127.0.0.1\");\n\n        when(nextChain.handle(any())).thenReturn(CompletableFuture.completedFuture(null));\n        handler.handle(ctx, nextChain).join();\n        // normal user should go to the next chain\n        verify(nextChain, times(1)).handle(any());\n    }\n\n    private DefaultAuthorizationContext buildContext(Subject subject, Resource resource, Action action, String sourceIp) {\n        return DefaultAuthorizationContext.of(subject, resource, action, sourceIp);\n    }\n\n    private void clearAllUsers() {\n        List<User> users = this.authenticationMetadataManager.listUser(null).join();\n        if (CollectionUtils.isEmpty(users)) {\n            return;\n        }\n        users.forEach(user -> this.authenticationMetadataManager.deleteUser(user.getUsername()).join());\n    }\n}"
  },
  {
    "path": "auth/src/test/java/org/apache/rocketmq/auth/authorization/manager/AuthorizationMetadataManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.manager;\n\nimport java.util.List;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;\nimport org.apache.rocketmq.auth.authentication.manager.AuthenticationMetadataManager;\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.auth.authorization.enums.Decision;\nimport org.apache.rocketmq.auth.authorization.enums.PolicyType;\nimport org.apache.rocketmq.auth.authorization.exception.AuthorizationException;\nimport org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;\nimport org.apache.rocketmq.auth.authorization.model.Acl;\nimport org.apache.rocketmq.auth.authorization.model.Policy;\nimport org.apache.rocketmq.auth.authorization.model.PolicyEntry;\nimport org.apache.rocketmq.auth.authorization.model.Resource;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.auth.helper.AuthTestHelper;\nimport org.apache.rocketmq.common.MixAll;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\npublic class AuthorizationMetadataManagerTest {\n\n    private AuthConfig authConfig;\n\n    private AuthenticationMetadataManager authenticationMetadataManager;\n\n    private AuthorizationMetadataManager authorizationMetadataManager;\n\n    @Before\n    public void setUp() throws Exception {\n        if (MixAll.isMac()) {\n            return;\n        }\n        this.authConfig = AuthTestHelper.createDefaultConfig();\n        this.authenticationMetadataManager = AuthenticationFactory.getMetadataManager(this.authConfig);\n        this.authorizationMetadataManager = AuthorizationFactory.getMetadataManager(this.authConfig);\n        this.clearAllAcls();\n        this.clearAllUsers();\n    }\n\n    @After\n    public void tearDown() throws Exception {\n        if (MixAll.isMac()) {\n            return;\n        }\n        this.clearAllAcls();\n        this.clearAllUsers();\n        this.authenticationMetadataManager.shutdown();\n        this.authorizationMetadataManager.shutdown();\n    }\n\n    @Test\n    public void createAcl() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"test\", \"test\");\n        this.authenticationMetadataManager.createUser(user).join();\n\n        Acl acl1 = AuthTestHelper.buildAcl(\"User:test\", \"Topic:test,Group:test\", \"PUB,SUB\",\n            \"192.168.0.0/24,10.10.0.0/24\", Decision.ALLOW);\n        this.authorizationMetadataManager.createAcl(acl1).join();\n        Acl acl2 = this.authorizationMetadataManager.getAcl(Subject.of(\"User:test\")).join();\n        Assert.assertTrue(AuthTestHelper.isEquals(acl1, acl2));\n\n        user = User.of(\"abc\", \"abc\");\n        this.authenticationMetadataManager.createUser(user).join();\n\n        acl1 = AuthTestHelper.buildAcl(\"User:abc\", PolicyType.DEFAULT, \"Topic:*,Group:*\", \"PUB,SUB\",\n            null, Decision.DENY);\n        this.authorizationMetadataManager.createAcl(acl1).join();\n        acl2 = this.authorizationMetadataManager.getAcl(Subject.of(\"User:abc\")).join();\n        Assert.assertTrue(AuthTestHelper.isEquals(acl1, acl2));\n\n        Acl acl3 = AuthTestHelper.buildAcl(\"User:test\", \"Topic:test,Group:test\", \"PUB,SUB\",\n            \"192.168.0.0/24,10.10.0.0/24\", Decision.ALLOW);\n        this.authorizationMetadataManager.createAcl(acl3).join();\n        Acl acl4 = this.authorizationMetadataManager.getAcl(Subject.of(\"User:test\")).join();\n        Assert.assertTrue(AuthTestHelper.isEquals(acl3, acl4));\n\n        Assert.assertThrows(AuthorizationException.class, () -> {\n            try {\n                Acl acl5 = AuthTestHelper.buildAcl(\"User:ddd\", \"Topic:test,Group:test\", \"PUB,SUB\",\n                    \"192.168.0.0/24,10.10.0.0/24\", Decision.ALLOW);\n                this.authorizationMetadataManager.createAcl(acl5).join();\n            } catch (Exception e) {\n                AuthTestHelper.handleException(e);\n            }\n        });\n    }\n\n    @Test\n    public void updateAcl() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"test\", \"test\");\n        this.authenticationMetadataManager.createUser(user).join();\n\n        Acl acl1 = AuthTestHelper.buildAcl(\"User:test\", \"Topic:test,Group:test\", \"PUB,SUB\",\n            \"192.168.0.0/24,10.10.0.0/24\", Decision.ALLOW);\n        this.authorizationMetadataManager.createAcl(acl1).join();\n\n        Acl acl2 = AuthTestHelper.buildAcl(\"User:test\", \"Topic:abc,Group:abc\", \"PUB,SUB\",\n            \"192.168.0.0/24,10.10.0.0/24\", Decision.ALLOW);\n        this.authorizationMetadataManager.updateAcl(acl2).join();\n\n        Acl acl3 = AuthTestHelper.buildAcl(\"User:test\", \"Topic:test,Group:test,Topic:abc,Group:abc\", \"PUB,SUB\",\n            \"192.168.0.0/24,10.10.0.0/24\", Decision.ALLOW);\n        Acl acl4 = this.authorizationMetadataManager.getAcl(Subject.of(\"User:test\")).join();\n        Assert.assertTrue(AuthTestHelper.isEquals(acl3, acl4));\n\n        Policy policy = AuthTestHelper.buildPolicy(\"Topic:test,Group:test\", \"PUB,SUB,Create\", \"192.168.0.0/24\", Decision.DENY);\n        acl4.updatePolicy(policy);\n        this.authorizationMetadataManager.updateAcl(acl4);\n        Acl acl5 = this.authorizationMetadataManager.getAcl(Subject.of(\"User:test\")).join();\n        Assert.assertTrue(AuthTestHelper.isEquals(acl4, acl5));\n\n        User user2 = User.of(\"abc\", \"abc\");\n        this.authenticationMetadataManager.createUser(user2).join();\n        Acl acl6 = AuthTestHelper.buildAcl(\"User:abc\", \"Topic:test,Group:test\", \"PUB,SUB\",\n            \"192.168.0.0/24,10.10.0.0/24\", Decision.ALLOW);\n        this.authorizationMetadataManager.updateAcl(acl6).join();\n        Acl acl7 = this.authorizationMetadataManager.getAcl(Subject.of(\"User:abc\")).join();\n        Assert.assertTrue(AuthTestHelper.isEquals(acl6, acl7));\n    }\n\n    @Test\n    public void deleteAcl() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"test\", \"test\");\n        this.authenticationMetadataManager.createUser(user).join();\n\n        Acl acl1 = AuthTestHelper.buildAcl(\"User:test\", \"Topic:test,Group:test\", \"PUB,SUB\",\n            \"192.168.0.0/24,10.10.0.0/24\", Decision.ALLOW);\n        this.authorizationMetadataManager.createAcl(acl1).join();\n\n        this.authorizationMetadataManager.deleteAcl(Subject.of(\"User:test\"), PolicyType.CUSTOM, Resource.ofTopic(\"abc\")).join();\n        Acl acl2 = this.authorizationMetadataManager.getAcl(Subject.of(\"User:test\")).join();\n        Assert.assertTrue(AuthTestHelper.isEquals(acl1, acl2));\n\n        this.authorizationMetadataManager.deleteAcl(Subject.of(\"User:test\"), PolicyType.CUSTOM, Resource.ofTopic(\"test\")).join();\n        Acl acl3 = AuthTestHelper.buildAcl(\"User:test\", \"Group:test\", \"PUB,SUB\",\n            \"192.168.0.0/24,10.10.0.0/24\", Decision.ALLOW);\n        Acl acl4 = this.authorizationMetadataManager.getAcl(Subject.of(\"User:test\")).join();\n        Assert.assertTrue(AuthTestHelper.isEquals(acl3, acl4));\n\n        this.authorizationMetadataManager.deleteAcl(Subject.of(\"User:test\"));\n        Acl acl5 = this.authorizationMetadataManager.getAcl(Subject.of(\"User:test\")).join();\n        Assert.assertNull(acl5);\n\n        Assert.assertThrows(AuthorizationException.class, () -> {\n            try {\n                this.authorizationMetadataManager.deleteAcl(Subject.of(\"User:abc\")).join();\n            } catch (Exception e) {\n                AuthTestHelper.handleException(e);\n            }\n        });\n    }\n\n    @Test\n    public void getAcl() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user = User.of(\"test\", \"test\");\n        this.authenticationMetadataManager.createUser(user).join();\n\n        Acl acl1 = AuthTestHelper.buildAcl(\"User:test\", \"Topic:test,Group:test\", \"PUB,SUB\",\n            \"192.168.0.0/24,10.10.0.0/24\", Decision.ALLOW);\n        this.authorizationMetadataManager.createAcl(acl1).join();\n        Acl acl2 = this.authorizationMetadataManager.getAcl(Subject.of(\"User:test\")).join();\n        Assert.assertTrue(AuthTestHelper.isEquals(acl1, acl2));\n\n        Assert.assertThrows(AuthorizationException.class, () -> {\n            try {\n                this.authorizationMetadataManager.getAcl(Subject.of(\"User:abc\")).join();\n            } catch (Exception e) {\n                AuthTestHelper.handleException(e);\n            }\n        });\n    }\n\n    @Test\n    public void testGetAclWithNullSubject() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        AuthorizationException authorizationException = Assert.assertThrows(AuthorizationException.class, () -> {\n            try {\n                this.authorizationMetadataManager.getAcl(null).join();\n            } catch (Exception e) {\n                AuthTestHelper.handleException(e);\n            }\n        });\n        Assert.assertEquals(\"The subject is null.\", authorizationException.getMessage());\n    }\n\n    @Test\n    public void listAcl() {\n        if (MixAll.isMac()) {\n            return;\n        }\n        User user1 = User.of(\"test-1\", \"test-1\");\n        this.authenticationMetadataManager.createUser(user1).join();\n        User user2 = User.of(\"test-2\", \"test-2\");\n        this.authenticationMetadataManager.createUser(user2).join();\n\n        Acl acl1 = AuthTestHelper.buildAcl(\"User:test-1\", \"Topic:test-1,Group:test-1\", \"PUB,SUB\",\n            \"192.168.0.0/24,10.10.0.0/24\", Decision.ALLOW);\n        this.authorizationMetadataManager.createAcl(acl1).join();\n\n        Acl acl2 = AuthTestHelper.buildAcl(\"User:test-2\", \"Topic:test-2,Group:test-2\", \"PUB,SUB\",\n            \"192.168.0.0/24,10.10.0.0/24\", Decision.ALLOW);\n        this.authorizationMetadataManager.createAcl(acl2).join();\n\n        Acl acl3 = AuthTestHelper.buildAcl(\"User:test-2\", \"Topic:acl-2,Group:acl-2\", \"PUB,SUB\",\n            \"192.168.0.0/24,10.10.0.0/24\", Decision.ALLOW);\n        this.authorizationMetadataManager.createAcl(acl3).join();\n\n        List<Acl> acls1 = this.authorizationMetadataManager.listAcl(null, null).join();\n        Assert.assertEquals(acls1.size(), 2);\n\n        List<Acl> acls2 = this.authorizationMetadataManager.listAcl(\"User:test-1\", null).join();\n        Assert.assertEquals(acls2.size(), 1);\n\n        List<Acl> acls3 = this.authorizationMetadataManager.listAcl(\"test\", null).join();\n        Assert.assertEquals(acls3.size(), 2);\n\n        List<Acl> acls4 = this.authorizationMetadataManager.listAcl(null, \"Topic:test-1\").join();\n        Assert.assertEquals(acls4.size(), 1);\n        Assert.assertEquals(acls4.get(0).getPolicy(PolicyType.CUSTOM).getEntries().size(), 1);\n\n        List<Acl> acls5 = this.authorizationMetadataManager.listAcl(null, \"test-1\").join();\n        Assert.assertEquals(acls5.size(), 1);\n        Assert.assertEquals(acls5.get(0).getPolicy(PolicyType.CUSTOM).getEntries().size(), 2);\n\n        List<Acl> acls6 = this.authorizationMetadataManager.listAcl(\"User:abc\", null).join();\n        Assert.assertTrue(CollectionUtils.isEmpty(acls6));\n\n        List<Acl> acls7 = this.authorizationMetadataManager.listAcl(null, \"Topic:abc\").join();\n        Assert.assertTrue(CollectionUtils.isEmpty(acls7));\n\n        List<Acl> acls8 = this.authorizationMetadataManager.listAcl(\"test-2\", \"test-2\").join();\n        Assert.assertEquals(acls8.size(), 1);\n        List<PolicyEntry> policyEntries = acls8.get(0).getPolicy(PolicyType.CUSTOM).getEntries();\n        Assert.assertEquals(policyEntries.size(), 2);\n        for (PolicyEntry policyEntry : policyEntries) {\n            Assert.assertTrue(policyEntry.toResourceStr().contains(\"test-2\"));\n        }\n    }\n\n    private void clearAllUsers() {\n        List<User> users = this.authenticationMetadataManager.listUser(null).join();\n        if (CollectionUtils.isEmpty(users)) {\n            return;\n        }\n        users.forEach(user -> this.authenticationMetadataManager.deleteUser(user.getUsername()).join());\n    }\n\n    private void clearAllAcls() {\n        List<Acl> acls = this.authorizationMetadataManager.listAcl(null, null).join();\n        if (CollectionUtils.isEmpty(acls)) {\n            return;\n        }\n        acls.forEach(acl -> this.authorizationMetadataManager.deleteAcl(acl.getSubject(), null, null).join());\n    }\n}"
  },
  {
    "path": "auth/src/test/java/org/apache/rocketmq/auth/authorization/model/ResourceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.model;\n\nimport org.apache.rocketmq.common.resource.ResourcePattern;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class ResourceTest {\n\n    @Test\n    public void parseResource() {\n        Resource resource = Resource.of(\"*\");\n        Assert.assertEquals(resource.getResourceType(), ResourceType.ANY);\n        Assert.assertNull(resource.getResourceName());\n        Assert.assertEquals(resource.getResourcePattern(), ResourcePattern.ANY);\n\n        resource = Resource.of(\"Topic:*\");\n        Assert.assertEquals(resource.getResourceType(), ResourceType.TOPIC);\n        Assert.assertNull(resource.getResourceName());\n        Assert.assertEquals(resource.getResourcePattern(), ResourcePattern.ANY);\n\n        resource = Resource.of(\"Topic:test-*\");\n        Assert.assertEquals(resource.getResourceType(), ResourceType.TOPIC);\n        Assert.assertEquals(resource.getResourceName(), \"test-\");\n        Assert.assertEquals(resource.getResourcePattern(), ResourcePattern.PREFIXED);\n\n        resource = Resource.of(\"Topic:test-1\");\n        Assert.assertEquals(resource.getResourceType(), ResourceType.TOPIC);\n        Assert.assertEquals(resource.getResourceName(), \"test-1\");\n        Assert.assertEquals(resource.getResourcePattern(), ResourcePattern.LITERAL);\n    }\n\n    @Test\n    public void isMatch() {\n\n    }\n}"
  },
  {
    "path": "auth/src/test/java/org/apache/rocketmq/auth/authorization/provider/LocalAuthorizationMetadataProviderTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.provider;\n\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.auth.helper.AuthTestHelper;\nimport org.junit.Assert;\nimport org.junit.Rule;\nimport org.junit.Test;\nimport org.junit.rules.TemporaryFolder;\n\npublic class LocalAuthorizationMetadataProviderTest {\n\n    @Rule\n    public TemporaryFolder tempFolder = new TemporaryFolder();\n\n    @Test\n    public void testShutdownReleasesCacheExecutor() throws Exception {\n        AuthConfig authConfig = AuthTestHelper.createDefaultConfig();\n        authConfig.setAuthConfigPath(tempFolder.newFolder(\"auth-test\").getAbsolutePath());\n\n        LocalAuthorizationMetadataProvider provider = new LocalAuthorizationMetadataProvider();\n        // Initialize provider to create the internal cache refresh executor\n        provider.initialize(authConfig, () -> null);\n\n        // After initialization, the executor should exist and not be shutdown\n        Assert.assertNotNull(provider.cacheRefreshExecutor);\n        Assert.assertFalse(provider.cacheRefreshExecutor.isShutdown());\n\n        // Shutdown provider should also shutdown its executor to release resources\n        provider.shutdown();\n\n        // Verify that the cache refresh executor has been shutdown\n        Assert.assertTrue(provider.cacheRefreshExecutor.isShutdown());\n    }\n}\n"
  },
  {
    "path": "auth/src/test/java/org/apache/rocketmq/auth/authorization/strategy/StatefulAuthorizationStrategyTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.authorization.strategy;\n\nimport com.github.benmanes.caffeine.cache.Cache;\nimport com.github.benmanes.caffeine.cache.Caffeine;\nimport org.apache.commons.lang3.reflect.MethodUtils;\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.auth.authorization.context.AuthorizationContext;\nimport org.apache.rocketmq.auth.authorization.context.DefaultAuthorizationContext;\nimport org.apache.rocketmq.auth.authorization.exception.AuthorizationException;\nimport org.apache.rocketmq.auth.authorization.model.Resource;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.action.Action;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.lang.reflect.InvocationTargetException;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.concurrent.TimeUnit;\nimport java.util.function.Supplier;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.fail;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.never;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class StatefulAuthorizationStrategyTest {\n\n    @Mock\n    private AuthConfig authConfig;\n\n    private StatefulAuthorizationStrategy statefulAuthorizationStrategy;\n\n    @Before\n    public void setUp() {\n        when(authConfig.getStatefulAuthorizationCacheExpiredSecond()).thenReturn(60);\n        when(authConfig.getStatefulAuthorizationCacheMaxNum()).thenReturn(100);\n        Supplier<?> metadataService = mock(Supplier.class);\n        statefulAuthorizationStrategy = spy(new StatefulAuthorizationStrategy(authConfig, metadataService));\n    }\n\n    @Test\n    public void testEvaluateChannelIdBlankDoesNotUseCache() {\n        AuthorizationContext context = mock(AuthorizationContext.class);\n        when(context.getChannelId()).thenReturn(null);\n        statefulAuthorizationStrategy.evaluate(context);\n        verify(statefulAuthorizationStrategy, times(1)).doEvaluate(context);\n    }\n\n    @Test\n    public void testEvaluateChannelIdNotNullCacheHit() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException {\n        DefaultAuthorizationContext context = new DefaultAuthorizationContext();\n        context.setChannelId(\"channelId\");\n        context.setSubject(Subject.of(\"User\"));\n        context.setResource(Resource.of(\"Cluster\"));\n        context.setActions(new ArrayList<>());\n        context.setSourceIp(\"sourceIp\");\n        Pair<Boolean, AuthorizationException> pair = Pair.of(true, null);\n        Cache<String, Pair<Boolean, AuthorizationException>> authCache = Caffeine.newBuilder()\n                .expireAfterWrite(60, TimeUnit.SECONDS)\n                .maximumSize(100)\n                .build();\n        authCache.put(buildKey(context), pair);\n        statefulAuthorizationStrategy.authCache = authCache;\n        statefulAuthorizationStrategy.evaluate(context);\n        verify(statefulAuthorizationStrategy, never()).doEvaluate(context);\n    }\n\n    @Test\n    public void testEvaluateChannelIdNotNullCacheMiss() {\n        DefaultAuthorizationContext context = new DefaultAuthorizationContext();\n        context.setChannelId(\"channelId\");\n        context.setSubject(Subject.of(\"User\"));\n        context.setResource(Resource.of(\"Cluster\"));\n        context.setActions(Collections.singletonList(Action.PUB));\n        context.setSourceIp(\"sourceIp\");\n        statefulAuthorizationStrategy.authCache = Caffeine.newBuilder()\n                .expireAfterWrite(60, TimeUnit.SECONDS)\n                .maximumSize(100)\n                .build();\n        statefulAuthorizationStrategy.evaluate(context);\n        verify(statefulAuthorizationStrategy, times(1)).doEvaluate(context);\n    }\n\n    @Test\n    public void testEvaluateChannelIdNotNullCacheException() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException {\n        DefaultAuthorizationContext context = new DefaultAuthorizationContext();\n        context.setChannelId(\"channelId\");\n        context.setSubject(Subject.of(\"subjectKey\"));\n        context.setResource(Resource.of(\"resourceKey\"));\n        context.setActions(Collections.singletonList(Action.PUB));\n        context.setSourceIp(\"sourceIp\");\n        AuthorizationException exception = new AuthorizationException(\"test\");\n        Pair<Boolean, AuthorizationException> pair = Pair.of(false, exception);\n        Cache<String, Pair<Boolean, AuthorizationException>> authCache = Caffeine.newBuilder()\n                .expireAfterWrite(60, TimeUnit.SECONDS)\n                .maximumSize(100)\n                .build();\n        authCache.put(buildKey(context), pair);\n        statefulAuthorizationStrategy.authCache = authCache;\n        try {\n            statefulAuthorizationStrategy.evaluate(context);\n            fail(\"Expected AuthorizationException to be thrown\");\n        } catch (final AuthorizationException ex) {\n            assertEquals(exception, ex);\n        }\n        verify(statefulAuthorizationStrategy, never()).doEvaluate(context);\n    }\n\n    private String buildKey(AuthorizationContext context) throws InvocationTargetException, NoSuchMethodException, IllegalAccessException {\n        return (String) MethodUtils.invokeMethod(statefulAuthorizationStrategy, true, \"buildKey\", context);\n    }\n}\n"
  },
  {
    "path": "auth/src/test/java/org/apache/rocketmq/auth/helper/AuthTestHelper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.helper;\n\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.stream.Collectors;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.auth.authentication.exception.AuthenticationException;\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.auth.authentication.provider.DefaultAuthenticationProvider;\nimport org.apache.rocketmq.auth.authentication.provider.LocalAuthenticationMetadataProvider;\nimport org.apache.rocketmq.auth.authorization.enums.Decision;\nimport org.apache.rocketmq.auth.authorization.enums.PolicyType;\nimport org.apache.rocketmq.auth.authorization.exception.AuthorizationException;\nimport org.apache.rocketmq.auth.authorization.model.Acl;\nimport org.apache.rocketmq.auth.authorization.model.Environment;\nimport org.apache.rocketmq.auth.authorization.model.Policy;\nimport org.apache.rocketmq.auth.authorization.model.PolicyEntry;\nimport org.apache.rocketmq.auth.authorization.model.Resource;\nimport org.apache.rocketmq.auth.authorization.provider.DefaultAuthorizationProvider;\nimport org.apache.rocketmq.auth.authorization.provider.LocalAuthorizationMetadataProvider;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.utils.ExceptionUtils;\n\npublic class AuthTestHelper {\n\n    public static AuthConfig createDefaultConfig() {\n        AuthConfig authConfig = new AuthConfig();\n        authConfig.setConfigName(\"test-\" + System.nanoTime());\n        authConfig.setAuthConfigPath(\"~/config\");\n        authConfig.setAuthenticationEnabled(true);\n        authConfig.setAuthenticationProvider(DefaultAuthenticationProvider.class.getName());\n        authConfig.setAuthenticationMetadataProvider(LocalAuthenticationMetadataProvider.class.getName());\n        authConfig.setAuthorizationEnabled(true);\n        authConfig.setAuthorizationProvider(DefaultAuthorizationProvider.class.getName());\n        authConfig.setAuthorizationMetadataProvider(LocalAuthorizationMetadataProvider.class.getName());\n        return authConfig;\n    }\n\n    public static Acl buildAcl(String subjectKey, String resources, String actions, String sourceIps,\n        Decision decision) {\n        return buildAcl(subjectKey, null, resources, actions, sourceIps, decision);\n    }\n\n    public static Acl buildAcl(String subjectKey, PolicyType policyType, String resources, String actions,\n        String sourceIps, Decision decision) {\n        Subject subject = Subject.of(subjectKey);\n        Policy policy = buildPolicy(policyType, resources, actions, sourceIps, decision);\n        return Acl.of(subject, policy);\n    }\n\n    public static Policy buildPolicy(String resources, String actions, String sourceIps,\n        Decision decision) {\n        return buildPolicy(null, resources, actions, sourceIps, decision);\n    }\n\n    public static Policy buildPolicy(PolicyType policyType, String resources, String actions, String sourceIps,\n        Decision decision) {\n        List<Resource> resourceList = Arrays.stream(StringUtils.split(resources, \",\"))\n            .map(Resource::of).collect(Collectors.toList());\n        List<Action> actionList = Arrays.stream(StringUtils.split(actions, \",\"))\n            .map(Action::getByName).collect(Collectors.toList());\n        Environment environment = null;\n        if (StringUtils.isNotBlank(sourceIps)) {\n            environment = Environment.of(Arrays.stream(StringUtils.split(sourceIps, \",\"))\n                .collect(Collectors.toList()));\n        }\n        return Policy.of(policyType, resourceList, actionList, environment, decision);\n    }\n\n    public static boolean isEquals(Acl acl1, Acl acl2) {\n        if (acl1 == null && acl2 == null) {\n            return true;\n        }\n        if (acl1 == null || acl2 == null) {\n            return false;\n        }\n        Subject subject1 = acl1.getSubject();\n        Subject subject2 = acl2.getSubject();\n        if (!isEquals(subject1, subject2)) {\n            return false;\n        }\n        Map<PolicyType, Policy> policyMap1 = new HashMap<>();\n        Map<PolicyType, Policy> policyMap2 = new HashMap<>();\n        if (CollectionUtils.isNotEmpty(acl1.getPolicies())) {\n            acl1.getPolicies().forEach(policy -> {\n                if (policy.getPolicyType() == null) {\n                    policy.setPolicyType(PolicyType.CUSTOM);\n                }\n                policyMap1.put(policy.getPolicyType(), policy);\n            });\n        }\n        if (CollectionUtils.isNotEmpty(acl2.getPolicies())) {\n            acl2.getPolicies().forEach(policy -> {\n                if (policy.getPolicyType() == null) {\n                    policy.setPolicyType(PolicyType.CUSTOM);\n                }\n                policyMap2.put(policy.getPolicyType(), policy);\n            });\n        }\n        if (policyMap1.size() != policyMap2.size()) {\n            return false;\n        }\n        Policy customPolicy1 = policyMap1.get(PolicyType.CUSTOM);\n        Policy customPolicy2 = policyMap2.get(PolicyType.CUSTOM);\n        if (!isEquals(customPolicy1, customPolicy2)) {\n            return false;\n        }\n\n        Policy defaultPolicy1 = policyMap1.get(PolicyType.DEFAULT);\n        Policy defaultPolicy2 = policyMap2.get(PolicyType.DEFAULT);\n        if (!isEquals(defaultPolicy1, defaultPolicy2)) {\n            return false;\n        }\n\n        return true;\n    }\n\n    private static boolean isEquals(Policy policy1, Policy policy2) {\n        if (policy1 == null && policy2 == null) {\n            return true;\n        }\n        if (policy1 == null || policy2 == null) {\n            return false;\n        }\n        if (policy1.getPolicyType() != policy2.getPolicyType()) {\n            return false;\n        }\n        Map<String, PolicyEntry> policyEntryMap1 = new HashMap<>();\n        Map<String, PolicyEntry> policyEntryMap2 = new HashMap<>();\n        if (CollectionUtils.isNotEmpty(policy1.getEntries())) {\n            policy1.getEntries().forEach(policyEntry -> {\n                policyEntryMap1.put(policyEntry.getResource().getResourceKey(), policyEntry);\n            });\n        }\n        if (CollectionUtils.isNotEmpty(policy2.getEntries())) {\n            policy2.getEntries().forEach(policyEntry -> {\n                policyEntryMap2.put(policyEntry.getResource().getResourceKey(), policyEntry);\n            });\n        }\n        if (policyEntryMap1.size() != policyEntryMap2.size()) {\n            return false;\n        }\n\n        for (String resourceKey : policyEntryMap1.keySet()) {\n            if (!isEquals(policyEntryMap1.get(resourceKey), policyEntryMap2.get(resourceKey))) {\n                return false;\n            }\n        }\n\n        for (String resourceKey : policyEntryMap2.keySet()) {\n            if (!isEquals(policyEntryMap1.get(resourceKey), policyEntryMap2.get(resourceKey))) {\n                return false;\n            }\n        }\n\n        return true;\n    }\n\n    private static boolean isEquals(PolicyEntry entry1, PolicyEntry entry2) {\n        if (entry1 == null && entry2 == null) {\n            return true;\n        }\n        if (entry1 == null || entry2 == null) {\n            return false;\n        }\n        Resource resource1 = entry1.getResource();\n        Resource resource2 = entry2.getResource();\n        if (!isEquals(resource1, resource2)) {\n            return false;\n        }\n        List<Action> actions1 = entry1.getActions();\n        List<Action> actions2 = entry2.getActions();\n        if (CollectionUtils.isEmpty(actions1) && CollectionUtils.isNotEmpty(actions2)) {\n            return false;\n        }\n        if (CollectionUtils.isNotEmpty(actions1) && CollectionUtils.isEmpty(actions2)) {\n            return false;\n        }\n        if (CollectionUtils.isNotEmpty(actions1) && CollectionUtils.isNotEmpty(actions2)\n            && !CollectionUtils.isEqualCollection(actions1, actions2)) {\n            return false;\n        }\n        Environment environment1 = entry1.getEnvironment();\n        Environment environment2 = entry2.getEnvironment();\n        if (!isEquals(environment1, environment2)) {\n            return false;\n        }\n        return entry1.getDecision() == entry2.getDecision();\n    }\n\n    private static boolean isEquals(Resource resource1, Resource resource2) {\n        if (resource1 == null && resource2 == null) {\n            return true;\n        }\n        if (resource1 == null || resource2 == null) {\n            return false;\n        }\n        return Objects.equals(resource1, resource2);\n    }\n\n    private static boolean isEquals(Environment environment1, Environment environment2) {\n        if (environment1 == null && environment2 == null) {\n            return true;\n        }\n        if (environment1 == null || environment2 == null) {\n            return false;\n        }\n        List<String> sourceIp1 = environment1.getSourceIps();\n        List<String> sourceIp2 = environment2.getSourceIps();\n        if (CollectionUtils.isEmpty(sourceIp1) && CollectionUtils.isEmpty(sourceIp2)) {\n            return true;\n        }\n        if (CollectionUtils.isEmpty(sourceIp1) || CollectionUtils.isEmpty(sourceIp2)) {\n            return false;\n        }\n        return CollectionUtils.isEqualCollection(sourceIp1, sourceIp2);\n    }\n\n    private static boolean isEquals(Subject subject1, Subject subject2) {\n        if (subject1 == null && subject2 == null) {\n            return true;\n        }\n        if (subject1 == null || subject2 == null) {\n            return false;\n        }\n        return subject1.getSubjectType() == subject2.getSubjectType()\n            && StringUtils.equals(subject1.getSubjectKey(), subject2.getSubjectKey());\n    }\n\n    public static void handleException(Throwable e) {\n        Throwable throwable = ExceptionUtils.getRealException(e);\n        if (throwable instanceof AuthenticationException) {\n            throw (AuthenticationException) throwable;\n        }\n        if (throwable instanceof AuthorizationException) {\n            throw (AuthorizationException) throwable;\n        }\n        throw new RuntimeException(e);\n    }\n}\n"
  },
  {
    "path": "auth/src/test/java/org/apache/rocketmq/auth/migration/AuthMigratorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.auth.migration;\n\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.auth.authentication.manager.AuthenticationMetadataManager;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.auth.authorization.manager.AuthorizationMetadataManager;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.auth.migration.v1.PlainPermissionManager;\nimport org.apache.rocketmq.auth.migration.v1.AclConfig;\nimport org.apache.rocketmq.auth.migration.v1.PlainAccessConfig;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.never;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class AuthMigratorTest {\n\n    @Mock\n    private AuthenticationMetadataManager authenticationMetadataManager;\n\n    @Mock\n    private AuthorizationMetadataManager authorizationMetadataManager;\n\n    @Mock\n    private PlainPermissionManager plainPermissionManager;\n\n    @Mock\n    private AuthConfig authConfig;\n\n    private AuthMigrator authMigrator;\n\n    @Before\n    public void setUp() throws IllegalAccessException {\n        when(authConfig.isMigrateAuthFromV1Enabled()).thenReturn(true);\n        authMigrator = new AuthMigrator(authConfig);\n        FieldUtils.writeDeclaredField(authMigrator, \"authenticationMetadataManager\", authenticationMetadataManager, true);\n        FieldUtils.writeDeclaredField(authMigrator, \"authorizationMetadataManager\", authorizationMetadataManager, true);\n        FieldUtils.writeDeclaredField(authMigrator, \"plainPermissionManager\", plainPermissionManager, true);\n    }\n\n    @Test\n    public void testMigrateNoAclConfigDoesNothing() {\n        AclConfig aclConfig = mock(AclConfig.class);\n        when(aclConfig.getPlainAccessConfigs()).thenReturn(new ArrayList<>());\n        when(plainPermissionManager.getAllAclConfig()).thenReturn(aclConfig);\n        authMigrator.migrate();\n        verify(authConfig, times(1)).isMigrateAuthFromV1Enabled();\n        verify(plainPermissionManager, times(1)).getAllAclConfig();\n        verify(authenticationMetadataManager, never()).createUser(any());\n        verify(authorizationMetadataManager, never()).createAcl(any());\n    }\n\n    @Test\n    public void testMigrateWithAclConfigCreatesUserAndAcl() {\n        AclConfig aclConfig = mock(AclConfig.class);\n        List<PlainAccessConfig> accessConfigs = new ArrayList<>();\n        accessConfigs.add(createPlainAccessConfig());\n        when(aclConfig.getPlainAccessConfigs()).thenReturn(accessConfigs);\n        when(plainPermissionManager.getAllAclConfig()).thenReturn(aclConfig);\n        when(authenticationMetadataManager.getUser(anyString()))\n                .thenReturn(CompletableFuture.completedFuture(null));\n        when(authenticationMetadataManager.createUser(any()))\n                .thenReturn(CompletableFuture.completedFuture(null));\n        authMigrator.migrate();\n        verify(authConfig, times(1)).isMigrateAuthFromV1Enabled();\n        verify(plainPermissionManager, times(1)).getAllAclConfig();\n        verify(authenticationMetadataManager, times(1)).createUser(any());\n        verify(authorizationMetadataManager, times(1)).createAcl(any());\n    }\n\n    @Test\n    public void testMigrateExceptionInMigrateLogsError() {\n        PlainAccessConfig accessConfig = mock(PlainAccessConfig.class);\n        when(accessConfig.getAccessKey()).thenReturn(\"testAk\");\n        when(authenticationMetadataManager.createUser(any(User.class)))\n                .thenThrow(new RuntimeException(\"Test Exception\"));\n        AclConfig aclConfig = mock(AclConfig.class);\n        List<PlainAccessConfig> accessConfigs = new ArrayList<>();\n        accessConfigs.add(accessConfig);\n        when(aclConfig.getPlainAccessConfigs()).thenReturn(accessConfigs);\n        when(plainPermissionManager.getAllAclConfig()).thenReturn(aclConfig);\n        when(authenticationMetadataManager.getUser(anyString()))\n                .thenReturn(CompletableFuture.completedFuture(null));\n        try {\n            authMigrator.migrate();\n            verify(authConfig, times(1)).isMigrateAuthFromV1Enabled();\n            verify(plainPermissionManager, times(1)).getAllAclConfig();\n            verify(authenticationMetadataManager, times(1)).createUser(any());\n            verify(authorizationMetadataManager, never()).createAcl(any());\n        } catch (final RuntimeException ex) {\n            assertEquals(\"Test Exception\", ex.getMessage());\n        }\n    }\n\n    private PlainAccessConfig createPlainAccessConfig() {\n        PlainAccessConfig result = mock(PlainAccessConfig.class);\n        when(result.getAccessKey()).thenReturn(\"testAk\");\n        when(result.getSecretKey()).thenReturn(\"testSk\");\n        when(result.isAdmin()).thenReturn(false);\n        when(result.getTopicPerms()).thenReturn(new ArrayList<>());\n        when(result.getGroupPerms()).thenReturn(new ArrayList<>());\n        when(result.getDefaultTopicPerm()).thenReturn(\"PUB\");\n        when(result.getDefaultGroupPerm()).thenReturn(null);\n        return result;\n    }\n}\n"
  },
  {
    "path": "bazel/BUILD.bazel",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#"
  },
  {
    "path": "bazel/GenTestRules.bzl",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\"\"\"Generate java test rules from given test_files.\n\nInstead of having to create one test rule per test in the BUILD file, this rule\nprovides a handy way to create a bunch of test rules for the specified test\nfiles.\n\n\"\"\"\n\ndef GenTestRules(\n        name,\n        test_files,\n        deps,\n        exclude_tests = [],\n        default_test_size = \"small\",\n        small_tests = [],\n        medium_tests = [],\n        large_tests = [],\n        enormous_tests = [],\n        resources = [],\n        data = [],\n        flaky_tests = [],\n        tags = [],\n        prefix = \"\",\n        jvm_flags = [],\n        args = [],\n        visibility = None,\n        shard_count = 1):\n    for test in _get_test_names(test_files):\n        if test in exclude_tests:\n            continue\n        test_size = default_test_size\n        if test in small_tests:\n            test_size = \"small\"\n        if test in medium_tests:\n            test_size = \"medium\"\n        if test in large_tests:\n            test_size = \"large\"\n        if test in enormous_tests:\n            test_size = \"enormous\"\n        flaky = 0\n        if (test in flaky_tests) or (\"flaky\" in tags):\n            flaky = 1\n        java_class = _package_from_path(\n            native.package_name() + \"/\" + _strip_right(test, \".java\"),\n        )\n        package = java_class[:java_class.rfind(\".\")]\n        native.java_test(\n            name = prefix + test,\n            runtime_deps = deps,\n            resources = resources,\n            size = test_size,\n            jvm_flags = jvm_flags + [\"-Dbuild.bazel=true\"],\n            args = args,\n            flaky = flaky,\n            tags = tags,\n            test_class = java_class,\n            visibility = visibility,\n            shard_count = shard_count,\n            data = data,\n        )\n\ndef _get_test_names(test_files):\n    test_names = []\n    for test_file in test_files:\n        if not test_file.endswith(\"Test.java\") and not test_file.endswith(\"IT.java\"):\n            continue\n        test_names += [test_file[:-5]]\n    return test_names\n\ndef _package_from_path(package_path, src_impls = None):\n    src_impls = src_impls or [\"javatests/\", \"java/\"]\n    for src_impl in src_impls:\n        if not src_impl.endswith(\"/\"):\n            src_impl += \"/\"\n        index = _index_of_end(package_path, src_impl)\n        if index >= 0:\n            package_path = package_path[index:]\n            break\n    return package_path.replace(\"/\", \".\")\n\ndef _strip_right(str, suffix):\n    \"\"\"Returns str without the suffix if it ends with suffix.\"\"\"\n    if str.endswith(suffix):\n        return str[0:len(str) - len(suffix)]\n    else:\n        return str\n\ndef _index_of_end(str, part):\n    \"\"\"If part is in str, return the index of the first character after part.\n    Return -1 if part is not in str.\"\"\"\n    index = str.find(part)\n    if index >= 0:\n        return index + len(part)\n    return -1\n"
  },
  {
    "path": "broker/BUILD.bazel",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\nload(\"//bazel:GenTestRules.bzl\", \"GenTestRules\")\n\njava_library(\n    name = \"broker\",\n    srcs = glob([\"src/main/java/**/*.java\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \"//auth\",\n        \"//client\",\n        \"//common\",\n        \"//filter\",\n        \"//remoting\",\n        \"//srvutil\",\n        \"//store\",\n        \"//tieredstore\",\n        \"@maven//:org_slf4j_slf4j_api\",\n        \"@maven//:ch_qos_logback_logback_classic\",\n        \"@maven//:com_alibaba_fastjson2_fastjson2\",\n        \"@maven//:com_github_luben_zstd_jni\",\n        \"@maven//:com_google_guava_guava\",\n        \"@maven//:com_googlecode_concurrentlinkedhashmap_concurrentlinkedhashmap_lru\",\n        \"@maven//:commons_cli_commons_cli\",\n        \"@maven//:commons_collections_commons_collections\",\n        \"@maven//:commons_io_commons_io\",\n        \"@maven//:commons_validator_commons_validator\",\n        \"@maven//:io_netty_netty_all\",\n        \"@maven//:io_openmessaging_storage_dledger\",\n        \"@maven//:io_opentelemetry_opentelemetry_api\",\n        \"@maven//:io_opentelemetry_opentelemetry_context\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_otlp\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_prometheus\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_logging\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk_common\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk_metrics\",\n        \"@maven//:org_apache_commons_commons_lang3\",\n        \"@maven//:commons_codec_commons_codec\",\n        \"@maven//:org_lz4_lz4_java\",\n        \"@maven//:io_github_aliyunmq_rocketmq_slf4j_api\",\n        \"@maven//:io_github_aliyunmq_rocketmq_logback_classic\",\n        \"@maven//:org_slf4j_jul_to_slf4j\",\n        \"@maven//:io_github_aliyunmq_rocketmq_shaded_slf4j_api_bridge\",\n        \"@maven//:org_apache_rocketmq_rocketmq_rocksdb\",\n        \"@maven//:net_java_dev_jna_jna\",\n        \"@maven//:com_github_ben_manes_caffeine_caffeine\",\n    ],\n)\n\njava_library(\n    name = \"tests\",\n    srcs = glob([\"src/test/java/**/*.java\"]),\n    resources = [\n        \"src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener\",\n        \"src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.TransactionalMessageService\",\n        \"src/test/resources/rmq.logback-test.xml\",\n    ],\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \":broker\",\n        \"//:test_deps\",\n        \"//auth\",\n        \"//client\",\n        \"//common\",\n        \"//filter\",\n        \"//remoting\",\n        \"//store\",\n        \"//tieredstore\",\n        \"@maven//:com_alibaba_fastjson2_fastjson2\",\n        \"@maven//:org_slf4j_slf4j_api\",\n        \"@maven//:com_google_guava_guava\",\n        \"@maven//:io_netty_netty_all\",\n        \"@maven//:org_apache_commons_commons_lang3\",\n        \"@maven//:commons_codec_commons_codec\",\n        \"@maven//:commons_io_commons_io\",\n        \"@maven//:io_github_aliyunmq_rocketmq_slf4j_api\",\n        \"@maven//:org_powermock_powermock_core\",\n        \"@maven//:io_opentelemetry_opentelemetry_api\",\n        \"@maven//:com_googlecode_concurrentlinkedhashmap_concurrentlinkedhashmap_lru\",\n        \"@maven//:org_apache_rocketmq_rocketmq_rocksdb\",\n        \"@maven//:commons_collections_commons_collections\",\n        \"@maven//:org_junit_jupiter_junit_jupiter_api\",\n        \"@maven//:com_github_ben_manes_caffeine_caffeine\",\n    ],\n)\n\nGenTestRules(\n    name = \"GeneratedTestRules\",\n    test_files = glob([\"src/test/java/**/*Test.java\"]),\n    exclude_tests = [\n            # These tests are extremely slow and flaky, exclude them before they are properly fixed.\n            \"src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerRegisterTest\",\n            \"src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest\",\n        ],\n    deps = [\n        \":tests\",\n    ],\n)\n"
  },
  {
    "path": "broker/pom.xml",
    "content": "<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor\n\tlicense agreements. See the NOTICE file distributed with this work for additional\n\tinformation regarding copyright ownership. The ASF licenses this file to\n\tYou under the Apache License, Version 2.0 (the \"License\"); you may not use\n\tthis file except in compliance with the License. You may obtain a copy of\n\tthe License at http://www.apache.org/licenses/LICENSE-2.0 Unless required\n\tby applicable law or agreed to in writing, software distributed under the\n\tLicense is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS\n\tOF ANY KIND, either express or implied. See the License for the specific\n\tlanguage governing permissions and limitations under the License. -->\n\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n    <parent>\n        <groupId>org.apache.rocketmq</groupId>\n        <artifactId>rocketmq-all</artifactId>\n        <version>${revision}</version>\n    </parent>\n\n    <modelVersion>4.0.0</modelVersion>\n    <packaging>jar</packaging>\n    <artifactId>rocketmq-broker</artifactId>\n    <name>rocketmq-broker ${project.version}</name>\n\n    <properties>\n        <project.root>${basedir}/..</project.root>\n    </properties>\n\n    <dependencies>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-remoting</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-store</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-tiered-store</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.github.aliyunmq</groupId>\n            <artifactId>rocketmq-slf4j-api</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.github.aliyunmq</groupId>\n            <artifactId>rocketmq-logback-classic</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-client</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-srvutil</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-filter</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.apache.rocketmq</groupId>\n            <artifactId>rocketmq-auth</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>commons-io</groupId>\n            <artifactId>commons-io</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.javassist</groupId>\n            <artifactId>javassist</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.bouncycastle</groupId>\n            <artifactId>bcpkix-jdk18on</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>com.googlecode.concurrentlinkedhashmap</groupId>\n            <artifactId>concurrentlinkedhashmap-lru</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.github.aliyunmq</groupId>\n            <artifactId>rocketmq-shaded-slf4j-api-bridge</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.slf4j</groupId>\n            <artifactId>jul-to-slf4j</artifactId>\n        </dependency>\n    </dependencies>\n\n    <build>\n        <plugins>\n            <plugin>\n                <artifactId>maven-surefire-plugin</artifactId>\n                <version>${maven-surefire-plugin.version}</version>\n                <configuration>\n                    <forkCount>1</forkCount>\n                    <reuseForks>false</reuseForks>\n                </configuration>\n            </plugin>\n        </plugins>\n    </build>\n</project>\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker;\n\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.collect.Lists;\nimport java.net.InetSocketAddress;\nimport java.util.AbstractMap;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Optional;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ScheduledFuture;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.locks.Lock;\nimport java.util.concurrent.locks.ReentrantLock;\nimport java.util.function.Function;\nimport java.util.function.Supplier;\nimport java.util.stream.Collectors;\nimport org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;\nimport org.apache.rocketmq.auth.authentication.manager.AuthenticationMetadataManager;\nimport org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;\nimport org.apache.rocketmq.auth.authorization.manager.AuthorizationMetadataManager;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.auth.migration.AuthMigrator;\nimport org.apache.rocketmq.broker.auth.pipeline.AuthenticationPipeline;\nimport org.apache.rocketmq.broker.auth.pipeline.AuthorizationPipeline;\nimport org.apache.rocketmq.broker.client.ClientHousekeepingService;\nimport org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;\nimport org.apache.rocketmq.broker.client.ConsumerManager;\nimport org.apache.rocketmq.broker.client.DefaultConsumerIdsChangeListener;\nimport org.apache.rocketmq.broker.client.ProducerManager;\nimport org.apache.rocketmq.broker.client.net.Broker2Client;\nimport org.apache.rocketmq.broker.client.rebalance.RebalanceLockManager;\nimport org.apache.rocketmq.broker.coldctr.ColdDataCgCtrService;\nimport org.apache.rocketmq.broker.coldctr.ColdDataPullRequestHoldService;\nimport org.apache.rocketmq.broker.config.v1.RocksDBConsumerOffsetManager;\nimport org.apache.rocketmq.broker.config.v1.RocksDBLmqSubscriptionGroupManager;\nimport org.apache.rocketmq.broker.config.v1.RocksDBLmqTopicConfigManager;\nimport org.apache.rocketmq.broker.config.v1.RocksDBSubscriptionGroupManager;\nimport org.apache.rocketmq.broker.config.v1.RocksDBTopicConfigManager;\nimport org.apache.rocketmq.broker.config.v2.ConfigStorage;\nimport org.apache.rocketmq.broker.config.v2.ConsumerOffsetManagerV2;\nimport org.apache.rocketmq.broker.config.v2.SubscriptionGroupManagerV2;\nimport org.apache.rocketmq.broker.config.v2.TopicConfigManagerV2;\nimport org.apache.rocketmq.broker.controller.ReplicasManager;\nimport org.apache.rocketmq.broker.dledger.DLedgerRoleChangeHandler;\nimport org.apache.rocketmq.broker.failover.EscapeBridge;\nimport org.apache.rocketmq.broker.filter.CommitLogDispatcherCalcBitMap;\nimport org.apache.rocketmq.broker.filter.ConsumerFilterManager;\nimport org.apache.rocketmq.broker.latency.BrokerFastFailure;\nimport org.apache.rocketmq.broker.lite.AbstractLiteLifecycleManager;\nimport org.apache.rocketmq.broker.lite.LiteEventDispatcher;\nimport org.apache.rocketmq.broker.lite.LiteSubscriptionRegistry;\nimport org.apache.rocketmq.broker.lite.LiteSubscriptionRegistryImpl;\nimport org.apache.rocketmq.broker.lite.LiteLifecycleManager;\nimport org.apache.rocketmq.broker.lite.LiteSharding;\nimport org.apache.rocketmq.broker.lite.LiteShardingImpl;\nimport org.apache.rocketmq.broker.lite.RocksDBLiteLifecycleManager;\nimport org.apache.rocketmq.broker.longpolling.LmqPullRequestHoldService;\nimport org.apache.rocketmq.broker.longpolling.NotifyMessageArrivingListener;\nimport org.apache.rocketmq.broker.longpolling.PullRequestHoldService;\nimport org.apache.rocketmq.broker.metrics.BrokerMetricsManager;\nimport org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook;\nimport org.apache.rocketmq.broker.mqtrace.SendMessageHook;\nimport org.apache.rocketmq.broker.offset.BroadcastOffsetManager;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.broker.offset.LmqConsumerOffsetManager;\nimport org.apache.rocketmq.broker.out.BrokerOuterAPI;\nimport org.apache.rocketmq.broker.plugin.BrokerAttachedPlugin;\nimport org.apache.rocketmq.broker.pop.PopConsumerService;\nimport org.apache.rocketmq.broker.pop.orderly.ConsumerOrderInfoManager;\nimport org.apache.rocketmq.broker.pop.orderly.QueueLevelConsumerManager;\nimport org.apache.rocketmq.broker.processor.AckMessageProcessor;\nimport org.apache.rocketmq.broker.processor.AdminBrokerProcessor;\nimport org.apache.rocketmq.broker.processor.ChangeInvisibleTimeProcessor;\nimport org.apache.rocketmq.broker.processor.ClientManageProcessor;\nimport org.apache.rocketmq.broker.processor.ConsumerManageProcessor;\nimport org.apache.rocketmq.broker.processor.EndTransactionProcessor;\nimport org.apache.rocketmq.broker.processor.LiteManagerProcessor;\nimport org.apache.rocketmq.broker.processor.LiteSubscriptionCtlProcessor;\nimport org.apache.rocketmq.broker.processor.NotificationProcessor;\nimport org.apache.rocketmq.broker.processor.PeekMessageProcessor;\nimport org.apache.rocketmq.broker.processor.PollingInfoProcessor;\nimport org.apache.rocketmq.broker.processor.PopInflightMessageCounter;\nimport org.apache.rocketmq.broker.processor.PopLiteMessageProcessor;\nimport org.apache.rocketmq.broker.processor.PopMessageProcessor;\nimport org.apache.rocketmq.broker.processor.PullMessageProcessor;\nimport org.apache.rocketmq.broker.processor.QueryAssignmentProcessor;\nimport org.apache.rocketmq.broker.processor.QueryMessageProcessor;\nimport org.apache.rocketmq.broker.processor.RecallMessageProcessor;\nimport org.apache.rocketmq.broker.processor.ReplyMessageProcessor;\nimport org.apache.rocketmq.broker.processor.SendMessageProcessor;\nimport org.apache.rocketmq.broker.schedule.ScheduleMessageService;\nimport org.apache.rocketmq.broker.slave.SlaveSynchronize;\nimport org.apache.rocketmq.broker.subscription.LmqSubscriptionGroupManager;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.broker.topic.LmqTopicConfigManager;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.broker.topic.TopicQueueMappingCleanService;\nimport org.apache.rocketmq.broker.topic.TopicQueueMappingManager;\nimport org.apache.rocketmq.broker.topic.TopicRouteInfoManager;\nimport org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener;\nimport org.apache.rocketmq.broker.transaction.TransactionMetricsFlushService;\nimport org.apache.rocketmq.broker.transaction.TransactionalMessageCheckService;\nimport org.apache.rocketmq.broker.transaction.TransactionalMessageService;\nimport org.apache.rocketmq.broker.transaction.rocksdb.TransactionalMessageRocksDBService;\nimport org.apache.rocketmq.broker.transaction.queue.DefaultTransactionalMessageCheckListener;\nimport org.apache.rocketmq.broker.transaction.queue.TransactionalMessageBridge;\nimport org.apache.rocketmq.broker.transaction.queue.TransactionalMessageServiceImpl;\nimport org.apache.rocketmq.broker.util.HookUtils;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.BrokerIdentity;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.config.ConfigManagerVersion;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.stats.MomentStatsItem;\nimport org.apache.rocketmq.common.utils.ServiceProvider;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.Configuration;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.RemotingServer;\nimport org.apache.rocketmq.remoting.common.TlsMode;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingServer;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.netty.RequestTask;\nimport org.apache.rocketmq.remoting.netty.TlsSystemConfig;\nimport org.apache.rocketmq.remoting.pipeline.RequestPipeline;\nimport org.apache.rocketmq.remoting.protocol.BrokerSyncInfo;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.RequestHeaderRegistry;\nimport org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.namesrv.RegisterBrokerResult;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingDetail;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo;\nimport org.apache.rocketmq.srvutil.FileWatchService;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.MessageArrivingListener;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.RocksDBMessageStore;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.dledger.DLedgerCommitLog;\nimport org.apache.rocketmq.store.hook.PutMessageHook;\nimport org.apache.rocketmq.store.hook.SendMessageBackHook;\nimport org.apache.rocketmq.store.plugin.MessageStoreFactory;\nimport org.apache.rocketmq.store.plugin.MessageStorePluginContext;\nimport org.apache.rocketmq.store.stats.BrokerStats;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.apache.rocketmq.store.stats.LmqBrokerStatsManager;\nimport org.apache.rocketmq.store.timer.TimerCheckpoint;\nimport org.apache.rocketmq.store.timer.TimerMessageStore;\nimport org.apache.rocketmq.store.timer.TimerMetrics;\nimport org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStore;\nimport org.apache.rocketmq.store.transaction.TransMessageRocksDBStore;\n\npublic class BrokerController {\n    protected static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private static final Logger LOG_PROTECTION = LoggerFactory.getLogger(LoggerName.PROTECTION_LOGGER_NAME);\n    private static final Logger LOG_WATER_MARK = LoggerFactory.getLogger(LoggerName.WATER_MARK_LOGGER_NAME);\n    protected static final int HA_ADDRESS_MIN_LENGTH = 6;\n\n    protected final BrokerConfig brokerConfig;\n    private final NettyServerConfig nettyServerConfig;\n    private final NettyClientConfig nettyClientConfig;\n    protected final MessageStoreConfig messageStoreConfig;\n    private final AuthConfig authConfig;\n    protected ConsumerOffsetManager consumerOffsetManager;\n    protected final BroadcastOffsetManager broadcastOffsetManager;\n    protected final ConsumerManager consumerManager;\n    protected final ConsumerFilterManager consumerFilterManager;\n    protected final ConsumerOrderInfoManager consumerOrderInfoManager;\n    protected final PopInflightMessageCounter popInflightMessageCounter;\n    protected final PopConsumerService popConsumerService;\n    protected final ProducerManager producerManager;\n    protected final ScheduleMessageService scheduleMessageService;\n    protected final ClientHousekeepingService clientHousekeepingService;\n    protected final PullMessageProcessor pullMessageProcessor;\n    protected final PeekMessageProcessor peekMessageProcessor;\n    protected final PopMessageProcessor popMessageProcessor;\n    protected final PopLiteMessageProcessor popLiteMessageProcessor;\n    protected final AckMessageProcessor ackMessageProcessor;\n    protected final ChangeInvisibleTimeProcessor changeInvisibleTimeProcessor;\n    protected final NotificationProcessor notificationProcessor;\n    protected final PollingInfoProcessor pollingInfoProcessor;\n    protected final QueryAssignmentProcessor queryAssignmentProcessor;\n    protected final ClientManageProcessor clientManageProcessor;\n    protected final LiteSubscriptionCtlProcessor liteSubscriptionCtlProcessor;\n    protected final LiteSharding liteSharding;\n    protected final AbstractLiteLifecycleManager liteLifecycleManager;\n    protected final LiteSubscriptionRegistry liteSubscriptionRegistry;\n    protected final LiteEventDispatcher liteEventDispatcher;\n    protected final LiteManagerProcessor liteManagerProcessor;\n    protected final SendMessageProcessor sendMessageProcessor;\n    protected final RecallMessageProcessor recallMessageProcessor;\n    protected final ReplyMessageProcessor replyMessageProcessor;\n    protected final PullRequestHoldService pullRequestHoldService;\n    protected final MessageArrivingListener messageArrivingListener;\n    protected final Broker2Client broker2Client;\n    protected final ConsumerIdsChangeListener consumerIdsChangeListener;\n    protected final EndTransactionProcessor endTransactionProcessor;\n    private final RebalanceLockManager rebalanceLockManager = new RebalanceLockManager();\n    private final TopicRouteInfoManager topicRouteInfoManager;\n    protected BrokerOuterAPI brokerOuterAPI;\n    protected ScheduledExecutorService scheduledExecutorService;\n    protected ScheduledExecutorService syncBrokerMemberGroupExecutorService;\n    protected ScheduledExecutorService brokerHeartbeatExecutorService;\n    protected final SlaveSynchronize slaveSynchronize;\n    protected final BlockingQueue<Runnable> sendThreadPoolQueue;\n    protected final BlockingQueue<Runnable> putThreadPoolQueue;\n    protected final BlockingQueue<Runnable> ackThreadPoolQueue;\n    protected final BlockingQueue<Runnable> pullThreadPoolQueue;\n    protected final BlockingQueue<Runnable> litePullThreadPoolQueue;\n    protected final BlockingQueue<Runnable> replyThreadPoolQueue;\n    protected final BlockingQueue<Runnable> queryThreadPoolQueue;\n    protected final BlockingQueue<Runnable> clientManagerThreadPoolQueue;\n    protected final BlockingQueue<Runnable> heartbeatThreadPoolQueue;\n    protected final BlockingQueue<Runnable> consumerManagerThreadPoolQueue;\n    protected final BlockingQueue<Runnable> endTransactionThreadPoolQueue;\n    protected final BlockingQueue<Runnable> adminBrokerThreadPoolQueue;\n    protected final BlockingQueue<Runnable> loadBalanceThreadPoolQueue;\n    protected BrokerStatsManager brokerStatsManager;\n    protected final List<SendMessageHook> sendMessageHookList = new ArrayList<>();\n    protected final List<ConsumeMessageHook> consumeMessageHookList = new ArrayList<>();\n    protected MessageStore messageStore;\n    protected static final String TCP_REMOTING_SERVER = \"TCP_REMOTING_SERVER\";\n    protected static final String FAST_REMOTING_SERVER = \"FAST_REMOTING_SERVER\";\n    protected final Map<String, RemotingServer> remotingServerMap = new ConcurrentHashMap<>();\n    protected CountDownLatch remotingServerStartLatch;\n    /**\n     * If {Topic, SubscriptionGroup, Offset}ManagerV2 are used, config entries are stored in RocksDB.\n     */\n    protected ConfigStorage configStorage;\n    protected TopicConfigManager topicConfigManager;\n    protected SubscriptionGroupManager subscriptionGroupManager;\n    protected TopicQueueMappingManager topicQueueMappingManager;\n    protected ExecutorService sendMessageExecutor;\n    protected ExecutorService pullMessageExecutor;\n    protected ExecutorService litePullMessageExecutor;\n    protected ExecutorService putMessageFutureExecutor;\n    protected ExecutorService ackMessageExecutor;\n    protected ExecutorService replyMessageExecutor;\n    protected ExecutorService queryMessageExecutor;\n    protected ExecutorService adminBrokerExecutor;\n    protected ExecutorService clientManageExecutor;\n    protected ExecutorService heartbeatExecutor;\n    protected ExecutorService consumerManageExecutor;\n    protected ExecutorService loadBalanceExecutor;\n    protected ExecutorService endTransactionExecutor;\n    protected boolean updateMasterHAServerAddrPeriodically = false;\n    private BrokerStats brokerStats;\n    private InetSocketAddress storeHost;\n    private TimerMessageStore timerMessageStore;\n    private TimerMessageRocksDBStore timerMessageRocksDBStore;\n    private TransMessageRocksDBStore transMessageRocksDBStore;\n    private TimerCheckpoint timerCheckpoint;\n    protected BrokerFastFailure brokerFastFailure;\n    private Configuration configuration;\n    protected TopicQueueMappingCleanService topicQueueMappingCleanService;\n    protected FileWatchService fileWatchService;\n    protected TransactionalMessageCheckService transactionalMessageCheckService;\n    protected TransactionalMessageService transactionalMessageService;\n    protected AbstractTransactionalMessageCheckListener transactionalMessageCheckListener;\n    protected TransactionalMessageRocksDBService transactionalMessageRocksDBService;\n    protected volatile boolean shutdown = false;\n    protected ShutdownHook shutdownHook;\n    private volatile boolean isScheduleServiceStart = false;\n    private volatile boolean isTransactionCheckServiceStart = false;\n    protected volatile BrokerMemberGroup brokerMemberGroup;\n    protected EscapeBridge escapeBridge;\n    protected List<BrokerAttachedPlugin> brokerAttachedPlugins = new ArrayList<>();\n    protected volatile long shouldStartTime;\n    private BrokerPreOnlineService brokerPreOnlineService;\n    protected volatile boolean isIsolated = false;\n    protected volatile long minBrokerIdInGroup = 0;\n    protected volatile String minBrokerAddrInGroup = null;\n    private final Lock lock = new ReentrantLock();\n    protected final List<ScheduledFuture<?>> scheduledFutures = new ArrayList<>();\n    protected ReplicasManager replicasManager;\n    private long lastSyncTimeMs = System.currentTimeMillis();\n    protected BrokerMetricsManager brokerMetricsManager;\n    private ColdDataPullRequestHoldService coldDataPullRequestHoldService;\n    private ColdDataCgCtrService coldDataCgCtrService;\n    private TransactionMetricsFlushService transactionMetricsFlushService;\n    private AuthenticationMetadataManager authenticationMetadataManager;\n    private AuthorizationMetadataManager authorizationMetadataManager;\n\n    private ConfigContext configContext;\n\n    public BrokerController(\n        final BrokerConfig brokerConfig,\n        final NettyServerConfig nettyServerConfig,\n        final NettyClientConfig nettyClientConfig,\n        final MessageStoreConfig messageStoreConfig,\n        final AuthConfig authConfig,\n        final ShutdownHook shutdownHook\n    ) {\n        this(brokerConfig, nettyServerConfig, nettyClientConfig, messageStoreConfig, authConfig);\n        this.shutdownHook = shutdownHook;\n    }\n\n    public BrokerController(\n        final BrokerConfig brokerConfig,\n        final MessageStoreConfig messageStoreConfig\n    ) {\n        this(brokerConfig, null, null, messageStoreConfig, null);\n    }\n\n    public BrokerController(\n        final BrokerConfig brokerConfig,\n        final MessageStoreConfig messageStoreConfig,\n        final AuthConfig authConfig\n    ) {\n        this(brokerConfig, null, null, messageStoreConfig, authConfig);\n    }\n\n    public BrokerController(\n        final BrokerConfig brokerConfig,\n        final NettyServerConfig nettyServerConfig,\n        final NettyClientConfig nettyClientConfig,\n        final MessageStoreConfig messageStoreConfig\n    ) {\n        this(brokerConfig, nettyServerConfig, nettyClientConfig, messageStoreConfig, null);\n    }\n\n    public BrokerController(\n        final BrokerConfig brokerConfig,\n        final NettyServerConfig nettyServerConfig,\n        final NettyClientConfig nettyClientConfig,\n        final MessageStoreConfig messageStoreConfig,\n        final AuthConfig authConfig\n    ) {\n        this.brokerConfig = brokerConfig;\n        this.nettyServerConfig = nettyServerConfig;\n        this.nettyClientConfig = nettyClientConfig;\n        this.messageStoreConfig = messageStoreConfig;\n        this.authConfig = authConfig;\n        this.setStoreHost(new InetSocketAddress(this.getBrokerConfig().getBrokerIP1(), getListenPort()));\n        this.brokerStatsManager = messageStoreConfig.isEnableLmq() ? new LmqBrokerStatsManager(this.brokerConfig) : new BrokerStatsManager(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.isEnableDetailStat());\n        this.broadcastOffsetManager = new BroadcastOffsetManager(this);\n        if (ConfigManagerVersion.V2.getVersion().equals(brokerConfig.getConfigManagerVersion())) {\n            this.configStorage = new ConfigStorage(messageStoreConfig);\n            this.topicConfigManager = new TopicConfigManagerV2(this, configStorage);\n            this.subscriptionGroupManager = new SubscriptionGroupManagerV2(this, configStorage);\n            this.consumerOffsetManager = new ConsumerOffsetManagerV2(this, configStorage);\n        } else if (this.messageStoreConfig.isEnableRocksDBStore()) {\n            this.topicConfigManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqTopicConfigManager(this) : new RocksDBTopicConfigManager(this);\n            this.subscriptionGroupManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqSubscriptionGroupManager(this) : new RocksDBSubscriptionGroupManager(this);\n            this.consumerOffsetManager = new RocksDBConsumerOffsetManager(this);\n        } else {\n            this.topicConfigManager = messageStoreConfig.isEnableLmq() ? new LmqTopicConfigManager(this) : new TopicConfigManager(this);\n            this.subscriptionGroupManager = messageStoreConfig.isEnableLmq() ? new LmqSubscriptionGroupManager(this) : new SubscriptionGroupManager(this);\n            this.consumerOffsetManager = messageStoreConfig.isEnableLmq() ? new LmqConsumerOffsetManager(this) : new ConsumerOffsetManager(this);\n        }\n        this.topicQueueMappingManager = new TopicQueueMappingManager(this);\n        this.authenticationMetadataManager = AuthenticationFactory.getMetadataManager(this.authConfig);\n        this.authorizationMetadataManager = AuthorizationFactory.getMetadataManager(this.authConfig);\n        this.topicRouteInfoManager = new TopicRouteInfoManager(this);\n        this.liteSharding = new LiteShardingImpl(this, this.topicRouteInfoManager);\n        this.liteLifecycleManager = this.messageStoreConfig.isEnableRocksDBStore() ?\n            new RocksDBLiteLifecycleManager(this, this.liteSharding) : new LiteLifecycleManager(this, this.liteSharding);\n        this.liteSubscriptionRegistry = new LiteSubscriptionRegistryImpl(this, liteLifecycleManager);\n        this.liteSubscriptionCtlProcessor = new LiteSubscriptionCtlProcessor(this, liteSubscriptionRegistry);\n        this.liteEventDispatcher = new LiteEventDispatcher(this, this.liteSubscriptionRegistry, this.liteLifecycleManager);\n        this.liteManagerProcessor = new LiteManagerProcessor(this, liteLifecycleManager, liteSharding);\n        this.pullMessageProcessor = new PullMessageProcessor(this);\n        this.peekMessageProcessor = new PeekMessageProcessor(this);\n        this.pullRequestHoldService = messageStoreConfig.isEnableLmq() ? new LmqPullRequestHoldService(this) : new PullRequestHoldService(this);\n        this.popMessageProcessor = new PopMessageProcessor(this);\n        this.popLiteMessageProcessor = new PopLiteMessageProcessor(this, this.liteEventDispatcher);\n        this.notificationProcessor = new NotificationProcessor(this);\n        this.pollingInfoProcessor = new PollingInfoProcessor(this);\n        this.ackMessageProcessor = new AckMessageProcessor(this);\n        this.changeInvisibleTimeProcessor = new ChangeInvisibleTimeProcessor(this);\n        this.sendMessageProcessor = new SendMessageProcessor(this);\n        this.recallMessageProcessor = new RecallMessageProcessor(this);\n        this.replyMessageProcessor = new ReplyMessageProcessor(this);\n        this.messageArrivingListener = new NotifyMessageArrivingListener(this.pullRequestHoldService, this.popMessageProcessor, this.notificationProcessor, this.liteEventDispatcher);\n        this.consumerIdsChangeListener = new DefaultConsumerIdsChangeListener(this);\n        this.consumerManager = new ConsumerManager(this.consumerIdsChangeListener, this.brokerStatsManager, this.brokerConfig);\n        this.producerManager = new ProducerManager(this.brokerStatsManager);\n        this.consumerFilterManager = new ConsumerFilterManager(this);\n        this.consumerOrderInfoManager = new QueueLevelConsumerManager(this);\n        this.popInflightMessageCounter = new PopInflightMessageCounter(this);\n        this.popConsumerService = brokerConfig.isPopConsumerKVServiceInit() ? new PopConsumerService(this) : null;\n        this.clientHousekeepingService = new ClientHousekeepingService(this);\n        this.broker2Client = new Broker2Client(this);\n        this.scheduleMessageService = new ScheduleMessageService(this);\n        this.coldDataPullRequestHoldService = new ColdDataPullRequestHoldService(this);\n        this.coldDataCgCtrService = new ColdDataCgCtrService(this);\n\n        if (nettyClientConfig != null) {\n            this.brokerOuterAPI = new BrokerOuterAPI(nettyClientConfig, authConfig);\n        }\n\n        this.queryAssignmentProcessor = new QueryAssignmentProcessor(this);\n        this.clientManageProcessor = new ClientManageProcessor(this);\n        this.slaveSynchronize = new SlaveSynchronize(this);\n        this.endTransactionProcessor = new EndTransactionProcessor(this);\n\n        this.sendThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getSendThreadPoolQueueCapacity());\n        this.putThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getPutThreadPoolQueueCapacity());\n        this.pullThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getPullThreadPoolQueueCapacity());\n        this.litePullThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getLitePullThreadPoolQueueCapacity());\n\n        this.ackThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getAckThreadPoolQueueCapacity());\n        this.replyThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getReplyThreadPoolQueueCapacity());\n        this.queryThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getQueryThreadPoolQueueCapacity());\n        this.clientManagerThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getClientManagerThreadPoolQueueCapacity());\n        this.consumerManagerThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getConsumerManagerThreadPoolQueueCapacity());\n        this.heartbeatThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getHeartbeatThreadPoolQueueCapacity());\n        this.endTransactionThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getEndTransactionPoolQueueCapacity());\n        this.adminBrokerThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getAdminBrokerThreadPoolQueueCapacity());\n        this.loadBalanceThreadPoolQueue = new LinkedBlockingQueue<>(this.brokerConfig.getLoadBalanceThreadPoolQueueCapacity());\n\n        this.brokerFastFailure = new BrokerFastFailure(this);\n\n        String brokerConfigPath;\n        if (brokerConfig.getBrokerConfigPath() != null && !brokerConfig.getBrokerConfigPath().isEmpty()) {\n            brokerConfigPath = brokerConfig.getBrokerConfigPath();\n        } else {\n            brokerConfigPath = BrokerPathConfigHelper.getBrokerConfigPath();\n        }\n        this.configuration = new Configuration(\n            LOG,\n            brokerConfigPath,\n            this.brokerConfig, this.nettyServerConfig, this.nettyClientConfig, this.messageStoreConfig\n        );\n\n        this.brokerStatsManager.setProducerStateGetter(new BrokerStatsManager.StateGetter() {\n            @Override\n            public boolean online(String instanceId, String group, String topic) {\n                if (getTopicConfigManager().getTopicConfigTable().containsKey(NamespaceUtil.wrapNamespace(instanceId, topic))) {\n                    return getProducerManager().groupOnline(NamespaceUtil.wrapNamespace(instanceId, group));\n                } else {\n                    return getProducerManager().groupOnline(group);\n                }\n            }\n        });\n        this.brokerStatsManager.setConsumerStateGetter(new BrokerStatsManager.StateGetter() {\n            @Override\n            public boolean online(String instanceId, String group, String topic) {\n                String topicFullName = NamespaceUtil.wrapNamespace(instanceId, topic);\n                if (getTopicConfigManager().getTopicConfigTable().containsKey(topicFullName)) {\n                    return getConsumerManager().findSubscriptionData(NamespaceUtil.wrapNamespace(instanceId, group), topicFullName) != null;\n                } else {\n                    return getConsumerManager().findSubscriptionData(group, topic) != null;\n                }\n            }\n        });\n\n        this.brokerMemberGroup = new BrokerMemberGroup(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.getBrokerName());\n        this.brokerMemberGroup.getBrokerAddrs().put(this.brokerConfig.getBrokerId(), this.getBrokerAddr());\n\n        this.escapeBridge = new EscapeBridge(this);\n\n        if (this.brokerConfig.isEnableSlaveActingMaster() && !this.brokerConfig.isSkipPreOnline()) {\n            this.brokerPreOnlineService = new BrokerPreOnlineService(this);\n        }\n\n        if (this.authConfig != null && this.authConfig.isMigrateAuthFromV1Enabled()) {\n            new AuthMigrator(this.authConfig).migrate();\n        }\n    }\n\n    public AuthConfig getAuthConfig() {\n        return authConfig;\n    }\n\n    public BrokerConfig getBrokerConfig() {\n        return brokerConfig;\n    }\n\n    public NettyServerConfig getNettyServerConfig() {\n        return nettyServerConfig;\n    }\n\n    public NettyClientConfig getNettyClientConfig() {\n        return nettyClientConfig;\n    }\n\n    public BlockingQueue<Runnable> getPullThreadPoolQueue() {\n        return pullThreadPoolQueue;\n    }\n\n    public BlockingQueue<Runnable> getQueryThreadPoolQueue() {\n        return queryThreadPoolQueue;\n    }\n\n    public BrokerMetricsManager getBrokerMetricsManager() {\n        return brokerMetricsManager;\n    }\n\n    public void setBrokerMetricsManager(BrokerMetricsManager brokerMetricsManager) {\n        this.brokerMetricsManager = brokerMetricsManager;\n    }\n\n    protected void initializeRemotingServer() throws CloneNotSupportedException {\n        NettyRemotingServer tcpRemotingServer = new NettyRemotingServer(this.nettyServerConfig, this.clientHousekeepingService);\n        NettyServerConfig fastConfig = (NettyServerConfig) this.nettyServerConfig.clone();\n\n        int listeningPort = nettyServerConfig.getListenPort() - 2;\n        if (listeningPort < 0) {\n            listeningPort = 0;\n        }\n        fastConfig.setListenPort(listeningPort);\n\n        NettyRemotingServer fastRemotingServer = new NettyRemotingServer(fastConfig, this.clientHousekeepingService);\n\n        // Set RemotingMetricsManager on both remoting servers\n        if (this.brokerMetricsManager != null) {\n            tcpRemotingServer.setRemotingMetricsManager(this.brokerMetricsManager.getRemotingMetricsManager());\n            fastRemotingServer.setRemotingMetricsManager(this.brokerMetricsManager.getRemotingMetricsManager());\n        }\n\n        remotingServerMap.put(TCP_REMOTING_SERVER, tcpRemotingServer);\n        remotingServerMap.put(FAST_REMOTING_SERVER, fastRemotingServer);\n    }\n\n    /**\n     * Initialize resources including remoting server and thread executors.\n     */\n    protected void initializeResources() {\n        this.scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1,\n            new ThreadFactoryImpl(\"BrokerControllerScheduledThread\", true, getBrokerIdentity()));\n\n        this.sendMessageExecutor = ThreadUtils.newThreadPoolExecutor(\n            this.brokerConfig.getSendMessageThreadPoolNums(),\n            this.brokerConfig.getSendMessageThreadPoolNums(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            this.sendThreadPoolQueue,\n            new ThreadFactoryImpl(\"SendMessageThread_\", getBrokerIdentity()));\n\n        this.pullMessageExecutor = ThreadUtils.newThreadPoolExecutor(\n            this.brokerConfig.getPullMessageThreadPoolNums(),\n            this.brokerConfig.getPullMessageThreadPoolNums(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            this.pullThreadPoolQueue,\n            new ThreadFactoryImpl(\"PullMessageThread_\", getBrokerIdentity()));\n\n        this.litePullMessageExecutor = ThreadUtils.newThreadPoolExecutor(\n            this.brokerConfig.getLitePullMessageThreadPoolNums(),\n            this.brokerConfig.getLitePullMessageThreadPoolNums(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            this.litePullThreadPoolQueue,\n            new ThreadFactoryImpl(\"LitePullMessageThread_\", getBrokerIdentity()));\n\n        this.putMessageFutureExecutor = ThreadUtils.newThreadPoolExecutor(\n            this.brokerConfig.getPutMessageFutureThreadPoolNums(),\n            this.brokerConfig.getPutMessageFutureThreadPoolNums(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            this.putThreadPoolQueue,\n            new ThreadFactoryImpl(\"PutMessageThread_\", getBrokerIdentity()));\n\n        this.ackMessageExecutor = ThreadUtils.newThreadPoolExecutor(\n            this.brokerConfig.getAckMessageThreadPoolNums(),\n            this.brokerConfig.getAckMessageThreadPoolNums(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            this.ackThreadPoolQueue,\n            new ThreadFactoryImpl(\"AckMessageThread_\", getBrokerIdentity()));\n\n        this.queryMessageExecutor = ThreadUtils.newThreadPoolExecutor(\n            this.brokerConfig.getQueryMessageThreadPoolNums(),\n            this.brokerConfig.getQueryMessageThreadPoolNums(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            this.queryThreadPoolQueue,\n            new ThreadFactoryImpl(\"QueryMessageThread_\", getBrokerIdentity()));\n\n        this.adminBrokerExecutor = ThreadUtils.newThreadPoolExecutor(\n            this.brokerConfig.getAdminBrokerThreadPoolNums(),\n            this.brokerConfig.getAdminBrokerThreadPoolNums(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            this.adminBrokerThreadPoolQueue,\n            new ThreadFactoryImpl(\"AdminBrokerThread_\", getBrokerIdentity()));\n\n        this.clientManageExecutor = ThreadUtils.newThreadPoolExecutor(\n            this.brokerConfig.getClientManageThreadPoolNums(),\n            this.brokerConfig.getClientManageThreadPoolNums(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            this.clientManagerThreadPoolQueue,\n            new ThreadFactoryImpl(\"ClientManageThread_\", getBrokerIdentity()));\n\n        this.heartbeatExecutor = ThreadUtils.newThreadPoolExecutor(\n            this.brokerConfig.getHeartbeatThreadPoolNums(),\n            this.brokerConfig.getHeartbeatThreadPoolNums(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            this.heartbeatThreadPoolQueue,\n            new ThreadFactoryImpl(\"HeartbeatThread_\", true, getBrokerIdentity()));\n\n        this.consumerManageExecutor = ThreadUtils.newThreadPoolExecutor(\n            this.brokerConfig.getConsumerManageThreadPoolNums(),\n            this.brokerConfig.getConsumerManageThreadPoolNums(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            this.consumerManagerThreadPoolQueue,\n            new ThreadFactoryImpl(\"ConsumerManageThread_\", true, getBrokerIdentity()));\n\n        this.replyMessageExecutor = ThreadUtils.newThreadPoolExecutor(\n            this.brokerConfig.getProcessReplyMessageThreadPoolNums(),\n            this.brokerConfig.getProcessReplyMessageThreadPoolNums(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            this.replyThreadPoolQueue,\n            new ThreadFactoryImpl(\"ProcessReplyMessageThread_\", getBrokerIdentity()));\n\n        this.endTransactionExecutor = ThreadUtils.newThreadPoolExecutor(\n            this.brokerConfig.getEndTransactionThreadPoolNums(),\n            this.brokerConfig.getEndTransactionThreadPoolNums(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            this.endTransactionThreadPoolQueue,\n            new ThreadFactoryImpl(\"EndTransactionThread_\", getBrokerIdentity()));\n\n        this.loadBalanceExecutor = ThreadUtils.newThreadPoolExecutor(\n            this.brokerConfig.getLoadBalanceProcessorThreadPoolNums(),\n            this.brokerConfig.getLoadBalanceProcessorThreadPoolNums(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            this.loadBalanceThreadPoolQueue,\n            new ThreadFactoryImpl(\"LoadBalanceProcessorThread_\", getBrokerIdentity()));\n\n        this.syncBrokerMemberGroupExecutorService = ThreadUtils.newScheduledThreadPool(1,\n            new ThreadFactoryImpl(\"BrokerControllerSyncBrokerScheduledThread\", getBrokerIdentity()));\n        this.brokerHeartbeatExecutorService = ThreadUtils.newScheduledThreadPool(1,\n            new ThreadFactoryImpl(\"BrokerControllerHeartbeatScheduledThread\", getBrokerIdentity()));\n\n        this.topicQueueMappingCleanService = new TopicQueueMappingCleanService(this);\n    }\n\n    protected void initializeBrokerScheduledTasks() {\n        final long initialDelay = UtilAll.computeNextMorningTimeMillis() - System.currentTimeMillis();\n        final long period = TimeUnit.DAYS.toMillis(1);\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    BrokerController.this.getBrokerStats().record();\n                } catch (Throwable e) {\n                    LOG.error(\"BrokerController: failed to record broker stats\", e);\n                }\n            }\n        }, initialDelay, period, TimeUnit.MILLISECONDS);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    BrokerController.this.consumerOffsetManager.persist();\n                } catch (Throwable e) {\n                    LOG.error(\n                        \"BrokerController: failed to persist config file of consumerOffset\", e);\n                }\n            }\n        }, 1000 * 10, this.brokerConfig.getFlushConsumerOffsetInterval(), TimeUnit.MILLISECONDS);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    BrokerController.this.consumerFilterManager.persist();\n                    BrokerController.this.consumerOrderInfoManager.persist();\n                } catch (Throwable e) {\n                    LOG.error(\n                        \"BrokerController: failed to persist config file of consumerFilter or consumerOrderInfo\",\n                        e);\n                }\n            }\n        }, 1000 * 10, 1000 * 10, TimeUnit.MILLISECONDS);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    BrokerController.this.protectBroker();\n                } catch (Throwable e) {\n                    LOG.error(\"BrokerController: failed to protectBroker\", e);\n                }\n            }\n        }, 3, 3, TimeUnit.MINUTES);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    BrokerController.this.printWaterMark();\n                } catch (Throwable e) {\n                    LOG.error(\"BrokerController: failed to print broker watermark\", e);\n                }\n            }\n        }, 10, 1, TimeUnit.SECONDS);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    BrokerController.this.messageStore.getTimerMessageStore().getTimerMetrics()\n                            .cleanMetrics(BrokerController.this.topicConfigManager.getTopicConfigTable().keySet());\n                } catch (Throwable e) {\n                    LOG.error(\"BrokerController: failed to clean unused timer metrics.\", e);\n                }\n            }\n        }, 3, 3, TimeUnit.MINUTES);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n\n            @Override\n            public void run() {\n                try {\n                    LOG.info(\"Dispatch task fall behind commit log {}bytes\",\n                        BrokerController.this.getMessageStore().dispatchBehindBytes());\n                } catch (Throwable e) {\n                    LOG.error(\"Failed to print dispatchBehindBytes\", e);\n                }\n            }\n        }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);\n\n        if (!messageStoreConfig.isEnableDLegerCommitLog() && !messageStoreConfig.isDuplicationEnable() && !brokerConfig.isEnableControllerMode()) {\n            if (BrokerRole.SLAVE == this.messageStoreConfig.getBrokerRole()) {\n                if (this.messageStoreConfig.getHaMasterAddress() != null && this.messageStoreConfig.getHaMasterAddress().length() >= HA_ADDRESS_MIN_LENGTH) {\n                    this.messageStore.updateHaMasterAddress(this.messageStoreConfig.getHaMasterAddress());\n                    this.updateMasterHAServerAddrPeriodically = false;\n                } else {\n                    this.updateMasterHAServerAddrPeriodically = true;\n                }\n\n                this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n\n                    @Override\n                    public void run() {\n                        try {\n                            if (System.currentTimeMillis() - lastSyncTimeMs > 60 * 1000) {\n                                BrokerController.this.getSlaveSynchronize().syncAll();\n                                lastSyncTimeMs = System.currentTimeMillis();\n                            }\n\n                            //timer checkpoint, latency-sensitive, so sync it more frequently\n                            if (messageStoreConfig.isTimerWheelEnable()) {\n                                BrokerController.this.getSlaveSynchronize().syncTimerCheckPoint();\n                            }\n                        } catch (Throwable e) {\n                            LOG.error(\"Failed to sync all config for slave.\", e);\n                        }\n                    }\n                }, 1000 * 10, 3 * 1000, TimeUnit.MILLISECONDS);\n\n            } else {\n                this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n\n                    @Override\n                    public void run() {\n                        try {\n                            BrokerController.this.printMasterAndSlaveDiff();\n                        } catch (Throwable e) {\n                            LOG.error(\"Failed to print diff of master and slave.\", e);\n                        }\n                    }\n                }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);\n            }\n        }\n\n        if (this.brokerConfig.isEnableControllerMode()) {\n            this.updateMasterHAServerAddrPeriodically = true;\n        }\n    }\n\n    protected void initializeScheduledTasks() {\n\n        initializeBrokerScheduledTasks();\n\n        if (this.brokerConfig.getNamesrvAddr() != null) {\n            this.updateNamesrvAddr();\n            LOG.info(\"Set user specified name server address: {}\", this.brokerConfig.getNamesrvAddr());\n            // also auto update namesrv if specify\n            this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n                @Override\n                public void run() {\n                    try {\n                        BrokerController.this.updateNamesrvAddr();\n                    } catch (Throwable e) {\n                        LOG.error(\"Failed to update nameServer address list\", e);\n                    }\n                }\n            }, 1000 * 10, this.brokerConfig.getUpdateNameServerAddrPeriod(), TimeUnit.MILLISECONDS);\n        } else if (this.brokerConfig.isFetchNamesrvAddrByAddressServer()) {\n            this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n\n                @Override\n                public void run() {\n                    try {\n                        BrokerController.this.brokerOuterAPI.fetchNameServerAddr();\n                    } catch (Throwable e) {\n                        LOG.error(\"Failed to fetch nameServer address\", e);\n                    }\n                }\n            }, 1000 * 10, this.brokerConfig.getFetchNamesrvAddrInterval(), TimeUnit.MILLISECONDS);\n        }\n    }\n\n    private void updateNamesrvAddr() {\n        if (this.brokerConfig.isFetchNameSrvAddrByDnsLookup()) {\n            this.brokerOuterAPI.updateNameServerAddressListByDnsLookup(this.brokerConfig.getNamesrvAddr());\n        } else {\n            this.brokerOuterAPI.updateNameServerAddressList(this.brokerConfig.getNamesrvAddr());\n        }\n    }\n\n    public boolean initializeMetadata() {\n        boolean result = true;\n        if (null != configStorage) {\n            result = configStorage.start();\n        }\n        result = result && this.topicConfigManager.load();\n        result = result && this.topicQueueMappingManager.load();\n        result = result && this.consumerOffsetManager.load();\n        result = result && this.subscriptionGroupManager.load();\n        result = result && this.consumerFilterManager.load();\n        result = result && this.consumerOrderInfoManager.load();\n        return result;\n    }\n\n    public boolean initializeMessageStore() {\n        boolean result = true;\n        try {\n            DefaultMessageStore defaultMessageStore;\n            if (this.messageStoreConfig.isEnableRocksDBStore()) {\n                defaultMessageStore = new RocksDBMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable());\n            } else {\n                defaultMessageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable());\n            }\n\n            if (messageStoreConfig.isEnableDLegerCommitLog()) {\n                DLedgerRoleChangeHandler roleChangeHandler =\n                    new DLedgerRoleChangeHandler(this, defaultMessageStore);\n                ((DLedgerCommitLog) defaultMessageStore.getCommitLog())\n                    .getdLedgerServer().getDLedgerLeaderElector().addRoleChangeHandler(roleChangeHandler);\n            }\n\n            this.brokerStats = new BrokerStats(defaultMessageStore);\n\n            // Load store plugin\n            MessageStorePluginContext context = new MessageStorePluginContext(\n                messageStoreConfig, brokerStatsManager, messageArrivingListener, brokerConfig, configuration);\n            this.messageStore = MessageStoreFactory.build(context, defaultMessageStore);\n            this.messageStore.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(this.brokerConfig, this.consumerFilterManager));\n            if (messageStoreConfig.isTimerWheelEnable()) {\n                this.timerCheckpoint = new TimerCheckpoint(BrokerPathConfigHelper.getTimerCheckPath(messageStoreConfig.getStorePathRootDir()));\n                TimerMetrics timerMetrics = new TimerMetrics(BrokerPathConfigHelper.getTimerMetricsPath(messageStoreConfig.getStorePathRootDir()));\n                this.timerMessageStore = new TimerMessageStore(messageStore, messageStoreConfig, timerCheckpoint, timerMetrics, brokerStatsManager);\n                this.timerMessageStore.registerEscapeBridgeHook(msg -> escapeBridge.putMessage(msg));\n                this.messageStore.setTimerMessageStore(this.timerMessageStore);\n                if (messageStoreConfig.isTimerRocksDBEnable()) {\n                    this.timerMessageRocksDBStore = new TimerMessageRocksDBStore(messageStore, timerMetrics, brokerStatsManager);\n                    this.messageStore.setTimerMessageRocksDBStore(timerMessageRocksDBStore);\n                }\n            }\n            if (messageStoreConfig.isTransRocksDBEnable()) {\n                this.transMessageRocksDBStore = new TransMessageRocksDBStore(messageStore, brokerStatsManager, new InetSocketAddress(this.getBrokerConfig().getBrokerIP1(), this.getNettyServerConfig().getListenPort()));\n                this.messageStore.setTransMessageRocksDBStore(transMessageRocksDBStore);\n            }\n        } catch (Exception e) {\n            result = false;\n            LOG.error(\"BrokerController#initialize: unexpected error occurs\", e);\n        }\n        return result;\n    }\n\n    public boolean initialize() throws CloneNotSupportedException {\n\n        boolean result = this.initializeMetadata();\n        if (!result) {\n            return false;\n        }\n\n        result = this.initializeMessageStore();\n        if (!result) {\n            return false;\n        }\n\n        return this.recoverAndInitService();\n    }\n\n    public boolean recoverAndInitService() throws CloneNotSupportedException {\n\n        boolean result = true;\n\n        if (this.brokerConfig.isEnableControllerMode()) {\n            this.replicasManager = new ReplicasManager(this);\n            this.replicasManager.setFenced(true);\n        }\n\n        if (messageStore != null) {\n            registerMessageStoreHook();\n            result = this.messageStore.load();\n        }\n\n        if (messageStoreConfig.isTimerWheelEnable()) {\n            result = result && this.timerMessageStore.load();\n            if (messageStoreConfig.isTimerRocksDBEnable()) {\n                result = result && this.timerMessageRocksDBStore.load();\n            }\n        }\n\n        //scheduleMessageService load after messageStore load success\n        result = result && this.scheduleMessageService.load();\n\n        for (BrokerAttachedPlugin brokerAttachedPlugin : brokerAttachedPlugins) {\n            if (brokerAttachedPlugin != null) {\n                result = result && brokerAttachedPlugin.load();\n            }\n        }\n\n        this.brokerMetricsManager = new BrokerMetricsManager(this);\n\n        if (result) {\n\n            initializeRemotingServer();\n\n            initializeResources();\n\n            registerProcessor();\n\n            initializeScheduledTasks();\n\n            initialTransaction();\n\n            initialRpcHooks();\n\n            initialRequestPipeline();\n\n            initLiteService();\n\n            if (TlsSystemConfig.tlsMode != TlsMode.DISABLED) {\n                // Register a listener to reload SslContext\n                try {\n                    fileWatchService = new FileWatchService(\n                        new String[] {\n                            TlsSystemConfig.tlsServerCertPath,\n                            TlsSystemConfig.tlsServerKeyPath,\n                            TlsSystemConfig.tlsServerTrustCertPath\n                        },\n                        new FileWatchService.Listener() {\n                            boolean certChanged, keyChanged = false;\n\n                            @Override\n                            public void onChanged(String path) {\n                                if (path.equals(TlsSystemConfig.tlsServerTrustCertPath)) {\n                                    LOG.info(\"The trust certificate changed, reload the ssl context\");\n                                    reloadServerSslContext();\n                                }\n                                if (path.equals(TlsSystemConfig.tlsServerCertPath)) {\n                                    certChanged = true;\n                                }\n                                if (path.equals(TlsSystemConfig.tlsServerKeyPath)) {\n                                    keyChanged = true;\n                                }\n                                if (certChanged && keyChanged) {\n                                    LOG.info(\"The certificate and private key changed, reload the ssl context\");\n                                    certChanged = keyChanged = false;\n                                    reloadServerSslContext();\n                                }\n                            }\n\n                            private void reloadServerSslContext() {\n                                for (Map.Entry<String, RemotingServer> entry : remotingServerMap.entrySet()) {\n                                    RemotingServer remotingServer = entry.getValue();\n                                    if (remotingServer instanceof NettyRemotingServer) {\n                                        ((NettyRemotingServer) remotingServer).loadSslContext();\n                                    }\n                                }\n                            }\n                        });\n                } catch (Exception e) {\n                    result = false;\n                    LOG.warn(\"FileWatchService created error, can't load the certificate dynamically\");\n                }\n            }\n        }\n\n        return result;\n    }\n\n    public void registerMessageStoreHook() {\n        List<PutMessageHook> putMessageHookList = messageStore.getPutMessageHookList();\n\n        putMessageHookList.add(new PutMessageHook() {\n            @Override\n            public String hookName() {\n                return \"checkBeforePutMessage\";\n            }\n\n            @Override\n            public PutMessageResult executeBeforePutMessage(MessageExt msg) {\n                return HookUtils.checkBeforePutMessage(BrokerController.this, msg);\n            }\n        });\n\n        putMessageHookList.add(new PutMessageHook() {\n            @Override\n            public String hookName() {\n                return \"innerBatchChecker\";\n            }\n\n            @Override\n            public PutMessageResult executeBeforePutMessage(MessageExt msg) {\n                if (msg instanceof MessageExtBrokerInner) {\n                    return HookUtils.checkInnerBatch(BrokerController.this, msg);\n                }\n                return null;\n            }\n        });\n\n        putMessageHookList.add(new PutMessageHook() {\n            @Override\n            public String hookName() {\n                return \"handleScheduleMessage\";\n            }\n\n            @Override\n            public PutMessageResult executeBeforePutMessage(MessageExt msg) {\n                if (msg instanceof MessageExtBrokerInner) {\n                    return HookUtils.handleScheduleMessage(BrokerController.this, (MessageExtBrokerInner) msg);\n                }\n                return null;\n            }\n        });\n\n        putMessageHookList.add(new PutMessageHook() {\n            @Override\n            public String hookName() {\n                return \"handleLmqQuota\";\n            }\n\n            @Override\n            public PutMessageResult executeBeforePutMessage(MessageExt msg) {\n                if (msg instanceof MessageExtBrokerInner) {\n                    return HookUtils.handleLmqQuota(BrokerController.this, (MessageExtBrokerInner) msg);\n                }\n                return null;\n            }\n        });\n\n        SendMessageBackHook sendMessageBackHook = new SendMessageBackHook() {\n            @Override\n            public boolean executeSendMessageBack(List<MessageExt> msgList, String brokerName, String brokerAddr) {\n                return HookUtils.sendMessageBack(BrokerController.this, msgList, brokerName, brokerAddr);\n            }\n        };\n\n        if (messageStore != null) {\n            messageStore.setSendMessageBackHook(sendMessageBackHook);\n        }\n    }\n\n    private void initialTransaction() {\n        this.transactionalMessageService = ServiceProvider.loadClass(TransactionalMessageService.class);\n        if (null == this.transactionalMessageService) {\n            this.transactionalMessageService = new TransactionalMessageServiceImpl(\n                new TransactionalMessageBridge(this, this.getMessageStore()));\n            LOG.warn(\"Load default transaction message hook service: {}\",\n                TransactionalMessageServiceImpl.class.getSimpleName());\n        }\n        this.transactionalMessageCheckListener = ServiceProvider.loadClass(\n            AbstractTransactionalMessageCheckListener.class);\n        if (null == this.transactionalMessageCheckListener) {\n            this.transactionalMessageCheckListener = new DefaultTransactionalMessageCheckListener();\n            LOG.warn(\"Load default discard message hook service: {}\",\n                DefaultTransactionalMessageCheckListener.class.getSimpleName());\n        }\n        this.transactionalMessageCheckListener.setBrokerController(this);\n        this.transactionalMessageCheckService = new TransactionalMessageCheckService(this);\n        this.transactionMetricsFlushService = new TransactionMetricsFlushService(this);\n        this.transactionMetricsFlushService.start();\n\n        if (messageStoreConfig.isTransRocksDBEnable()) {\n            this.transactionalMessageRocksDBService = new TransactionalMessageRocksDBService(messageStore, this);\n            this.transactionalMessageRocksDBService.start();\n        }\n    }\n\n    private void initialRpcHooks() {\n\n        List<RPCHook> rpcHooks = ServiceProvider.load(RPCHook.class);\n        if (rpcHooks == null || rpcHooks.isEmpty()) {\n            return;\n        }\n        for (RPCHook rpcHook : rpcHooks) {\n            this.registerServerRPCHook(rpcHook);\n        }\n    }\n\n    private void initialRequestPipeline() {\n        if (this.authConfig == null) {\n            return;\n        }\n        RequestPipeline pipeline = (ctx, request) -> {\n        };\n        // add pipeline\n        // the last pipe add will execute at the first\n        try {\n            pipeline = pipeline.pipe(new AuthorizationPipeline(authConfig))\n                .pipe(new AuthenticationPipeline(authConfig));\n            this.setRequestPipeline(pipeline);\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    private void initLiteService() {\n        this.liteEventDispatcher.init();\n        this.liteLifecycleManager.init();\n    }\n\n    public void registerProcessor() {\n        RemotingServer remotingServer = remotingServerMap.get(TCP_REMOTING_SERVER);\n        RemotingServer fastRemotingServer = remotingServerMap.get(FAST_REMOTING_SERVER);\n\n        /*\n         * SendMessageProcessor\n         */\n        sendMessageProcessor.registerSendMessageHook(sendMessageHookList);\n        sendMessageProcessor.registerConsumeMessageHook(consumeMessageHookList);\n\n        remotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendMessageProcessor, this.sendMessageExecutor);\n        remotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendMessageProcessor, this.sendMessageExecutor);\n        remotingServer.registerProcessor(RequestCode.SEND_BATCH_MESSAGE, sendMessageProcessor, this.sendMessageExecutor);\n        remotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendMessageProcessor, this.sendMessageExecutor);\n        remotingServer.registerProcessor(RequestCode.RECALL_MESSAGE, recallMessageProcessor, this.sendMessageExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendMessageProcessor, this.sendMessageExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendMessageProcessor, this.sendMessageExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.SEND_BATCH_MESSAGE, sendMessageProcessor, this.sendMessageExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendMessageProcessor, this.sendMessageExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.RECALL_MESSAGE, recallMessageProcessor, this.sendMessageExecutor);\n        /**\n         * PullMessageProcessor\n         */\n        remotingServer.registerProcessor(RequestCode.PULL_MESSAGE, this.pullMessageProcessor, this.pullMessageExecutor);\n        remotingServer.registerProcessor(RequestCode.LITE_PULL_MESSAGE, this.pullMessageProcessor, this.litePullMessageExecutor);\n        this.pullMessageProcessor.registerConsumeMessageHook(consumeMessageHookList);\n        /**\n         * PeekMessageProcessor\n         */\n        remotingServer.registerProcessor(RequestCode.PEEK_MESSAGE, this.peekMessageProcessor, this.pullMessageExecutor);\n        /**\n         * PopMessageProcessor\n         */\n        remotingServer.registerProcessor(RequestCode.POP_MESSAGE, this.popMessageProcessor, this.pullMessageExecutor);\n        remotingServer.registerProcessor(RequestCode.POP_LITE_MESSAGE, this.popLiteMessageProcessor, this.pullMessageExecutor);\n\n        /**\n         * AckMessageProcessor\n         */\n        remotingServer.registerProcessor(RequestCode.ACK_MESSAGE, this.ackMessageProcessor, this.ackMessageExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.ACK_MESSAGE, this.ackMessageProcessor, this.ackMessageExecutor);\n\n        remotingServer.registerProcessor(RequestCode.BATCH_ACK_MESSAGE, this.ackMessageProcessor, this.ackMessageExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.BATCH_ACK_MESSAGE, this.ackMessageProcessor, this.ackMessageExecutor);\n        /**\n         * ChangeInvisibleTimeProcessor\n         */\n        remotingServer.registerProcessor(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, this.changeInvisibleTimeProcessor, this.ackMessageExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, this.changeInvisibleTimeProcessor, this.ackMessageExecutor);\n        /**\n         * notificationProcessor\n         */\n        remotingServer.registerProcessor(RequestCode.NOTIFICATION, this.notificationProcessor, this.pullMessageExecutor);\n\n        /**\n         * pollingInfoProcessor\n         */\n        remotingServer.registerProcessor(RequestCode.POLLING_INFO, this.pollingInfoProcessor, this.pullMessageExecutor);\n\n        /**\n         * ReplyMessageProcessor\n         */\n\n        replyMessageProcessor.registerSendMessageHook(sendMessageHookList);\n\n        remotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE, replyMessageProcessor, replyMessageExecutor);\n        remotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE_V2, replyMessageProcessor, replyMessageExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE, replyMessageProcessor, replyMessageExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE_V2, replyMessageProcessor, replyMessageExecutor);\n\n        /**\n         * QueryMessageProcessor\n         */\n        NettyRequestProcessor queryProcessor = new QueryMessageProcessor(this);\n        remotingServer.registerProcessor(RequestCode.QUERY_MESSAGE, queryProcessor, this.queryMessageExecutor);\n        remotingServer.registerProcessor(RequestCode.VIEW_MESSAGE_BY_ID, queryProcessor, this.queryMessageExecutor);\n\n        fastRemotingServer.registerProcessor(RequestCode.QUERY_MESSAGE, queryProcessor, this.queryMessageExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.VIEW_MESSAGE_BY_ID, queryProcessor, this.queryMessageExecutor);\n\n        /**\n         * ClientManageProcessor\n         */\n        remotingServer.registerProcessor(RequestCode.HEART_BEAT, clientManageProcessor, this.heartbeatExecutor);\n        remotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientManageProcessor, this.clientManageExecutor);\n        remotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientManageProcessor, this.clientManageExecutor);\n        remotingServer.registerProcessor(RequestCode.LITE_SUBSCRIPTION_CTL, liteSubscriptionCtlProcessor, this.clientManageExecutor);\n\n        fastRemotingServer.registerProcessor(RequestCode.HEART_BEAT, clientManageProcessor, this.heartbeatExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientManageProcessor, this.clientManageExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientManageProcessor, this.clientManageExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.LITE_SUBSCRIPTION_CTL, liteSubscriptionCtlProcessor, this.clientManageExecutor);\n\n        /**\n         * ConsumerManageProcessor\n         */\n        ConsumerManageProcessor consumerManageProcessor = new ConsumerManageProcessor(this);\n        remotingServer.registerProcessor(RequestCode.GET_CONSUMER_LIST_BY_GROUP, consumerManageProcessor, this.consumerManageExecutor);\n        remotingServer.registerProcessor(RequestCode.UPDATE_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);\n        remotingServer.registerProcessor(RequestCode.QUERY_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);\n\n        fastRemotingServer.registerProcessor(RequestCode.GET_CONSUMER_LIST_BY_GROUP, consumerManageProcessor, this.consumerManageExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.UPDATE_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.QUERY_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);\n\n        /**\n         * QueryAssignmentProcessor\n         */\n        remotingServer.registerProcessor(RequestCode.QUERY_ASSIGNMENT, queryAssignmentProcessor, loadBalanceExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.QUERY_ASSIGNMENT, queryAssignmentProcessor, loadBalanceExecutor);\n        remotingServer.registerProcessor(RequestCode.SET_MESSAGE_REQUEST_MODE, queryAssignmentProcessor, loadBalanceExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.SET_MESSAGE_REQUEST_MODE, queryAssignmentProcessor, loadBalanceExecutor);\n\n        /**\n         * EndTransactionProcessor\n         */\n        remotingServer.registerProcessor(RequestCode.END_TRANSACTION, endTransactionProcessor, this.endTransactionExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.END_TRANSACTION, endTransactionProcessor, this.endTransactionExecutor);\n\n        /*\n         * lite admin\n         */\n        remotingServer.registerProcessor(RequestCode.GET_BROKER_LITE_INFO, liteManagerProcessor, adminBrokerExecutor);\n        remotingServer.registerProcessor(RequestCode.GET_PARENT_TOPIC_INFO, liteManagerProcessor, adminBrokerExecutor);\n        remotingServer.registerProcessor(RequestCode.GET_LITE_TOPIC_INFO, liteManagerProcessor, adminBrokerExecutor);\n        remotingServer.registerProcessor(RequestCode.GET_LITE_CLIENT_INFO, liteManagerProcessor, adminBrokerExecutor);\n        remotingServer.registerProcessor(RequestCode.GET_LITE_GROUP_INFO, liteManagerProcessor, adminBrokerExecutor);\n        remotingServer.registerProcessor(RequestCode.TRIGGER_LITE_DISPATCH, liteManagerProcessor, adminBrokerExecutor);\n\n        fastRemotingServer.registerProcessor(RequestCode.GET_BROKER_LITE_INFO, liteManagerProcessor, adminBrokerExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.GET_PARENT_TOPIC_INFO, liteManagerProcessor, adminBrokerExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.GET_LITE_TOPIC_INFO, liteManagerProcessor, adminBrokerExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.GET_LITE_CLIENT_INFO, liteManagerProcessor, adminBrokerExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.GET_LITE_GROUP_INFO, liteManagerProcessor, adminBrokerExecutor);\n        fastRemotingServer.registerProcessor(RequestCode.TRIGGER_LITE_DISPATCH, liteManagerProcessor, adminBrokerExecutor);\n\n        /*\n         * Default\n         */\n        AdminBrokerProcessor adminProcessor = new AdminBrokerProcessor(this);\n        remotingServer.registerDefaultProcessor(adminProcessor, this.adminBrokerExecutor);\n        fastRemotingServer.registerDefaultProcessor(adminProcessor, this.adminBrokerExecutor);\n\n        /*\n         * Initialize the mapping of request codes to request headers.\n         */\n        RequestHeaderRegistry.getInstance().initialize();\n    }\n\n    public BrokerStats getBrokerStats() {\n        return brokerStats;\n    }\n\n    public void setBrokerStats(BrokerStats brokerStats) {\n        this.brokerStats = brokerStats;\n    }\n\n    public void protectBroker() {\n        if (this.brokerConfig.isDisableConsumeIfConsumerReadSlowly()) {\n            for (Map.Entry<String, MomentStatsItem> next : this.brokerStatsManager.getMomentStatsItemSetFallSize().getStatsItemTable().entrySet()) {\n                final long fallBehindBytes = next.getValue().getValue().get();\n                if (fallBehindBytes > this.brokerConfig.getConsumerFallbehindThreshold()) {\n                    final String[] split = next.getValue().getStatsKey().split(\"@\");\n                    final String group = split[2];\n                    LOG_PROTECTION.info(\"[PROTECT_BROKER] the consumer[{}] consume slowly, {} bytes, disable it\", group, fallBehindBytes);\n                    this.subscriptionGroupManager.disableConsume(group);\n                }\n            }\n        }\n    }\n\n    public long headSlowTimeMills(BlockingQueue<Runnable> q) {\n        long slowTimeMills = 0;\n        final Runnable peek = q.peek();\n        if (peek != null) {\n            RequestTask rt = BrokerFastFailure.castRunnable(peek);\n            slowTimeMills = rt == null ? 0 : this.messageStore.now() - rt.getCreateTimestamp();\n        }\n\n        if (slowTimeMills < 0) {\n            slowTimeMills = 0;\n        }\n\n        return slowTimeMills;\n    }\n\n    public long headSlowTimeMills4SendThreadPoolQueue() {\n        return this.headSlowTimeMills(this.sendThreadPoolQueue);\n    }\n\n    public long headSlowTimeMills4PullThreadPoolQueue() {\n        return this.headSlowTimeMills(this.pullThreadPoolQueue);\n    }\n\n    public long headSlowTimeMills4LitePullThreadPoolQueue() {\n        return this.headSlowTimeMills(this.litePullThreadPoolQueue);\n    }\n\n    public long headSlowTimeMills4QueryThreadPoolQueue() {\n        return this.headSlowTimeMills(this.queryThreadPoolQueue);\n    }\n\n    public long headSlowTimeMills4AckThreadPoolQueue() {\n        return this.headSlowTimeMills(this.ackThreadPoolQueue);\n    }\n\n    public long headSlowTimeMills4EndTransactionThreadPoolQueue() {\n        return this.headSlowTimeMills(this.endTransactionThreadPoolQueue);\n    }\n\n    public long headSlowTimeMills4ClientManagerThreadPoolQueue() {\n        return this.headSlowTimeMills(this.clientManagerThreadPoolQueue);\n    }\n\n    public long headSlowTimeMills4HeartbeatThreadPoolQueue() {\n        return this.headSlowTimeMills(this.heartbeatThreadPoolQueue);\n    }\n\n    public long headSlowTimeMills4AdminBrokerThreadPoolQueue() {\n        return this.headSlowTimeMills(this.adminBrokerThreadPoolQueue);\n    }\n\n    public void printWaterMark() {\n        logWaterMarkQueueInfo(\"Send\", this.sendThreadPoolQueue, this::headSlowTimeMills4SendThreadPoolQueue);\n        logWaterMarkQueueInfo(\"Pull\", this.pullThreadPoolQueue, this::headSlowTimeMills4PullThreadPoolQueue);\n        logWaterMarkQueueInfo(\"Query\", this.queryThreadPoolQueue, this::headSlowTimeMills4QueryThreadPoolQueue);\n        logWaterMarkQueueInfo(\"Lite Pull\", this.litePullThreadPoolQueue, this::headSlowTimeMills4LitePullThreadPoolQueue);\n        logWaterMarkQueueInfo(\"Transaction\", this.endTransactionThreadPoolQueue, this::headSlowTimeMills4EndTransactionThreadPoolQueue);\n        logWaterMarkQueueInfo(\"ClientManager\", this.clientManagerThreadPoolQueue, this::headSlowTimeMills4ClientManagerThreadPoolQueue);\n        logWaterMarkQueueInfo(\"Heartbeat\", this.heartbeatThreadPoolQueue, this::headSlowTimeMills4HeartbeatThreadPoolQueue);\n        logWaterMarkQueueInfo(\"Ack\", this.ackThreadPoolQueue, this::headSlowTimeMills4AckThreadPoolQueue);\n        logWaterMarkQueueInfo(\"Admin\", this.adminBrokerThreadPoolQueue, this::headSlowTimeMills4AdminBrokerThreadPoolQueue);\n    }\n\n    private void logWaterMarkQueueInfo(String queueName, BlockingQueue<?> queue, Supplier<Long> slowTimeSupplier) {\n        LOG_WATER_MARK.info(\"[WATERMARK] {} Queue Size: {} SlowTimeMills: {}\", queueName, queue.size(), slowTimeSupplier.get());\n    }\n\n    public MessageStore getMessageStore() {\n        return messageStore;\n    }\n\n    public void setMessageStore(MessageStore messageStore) {\n        this.messageStore = messageStore;\n    }\n\n    protected void printMasterAndSlaveDiff() {\n        if (messageStore.getHaService() != null && messageStore.getHaService().getConnectionCount().get() > 0) {\n            long diff = this.messageStore.slaveFallBehindMuch();\n            LOG.info(\"CommitLog: slave fall behind master {}bytes\", diff);\n        }\n    }\n\n    public Broker2Client getBroker2Client() {\n        return broker2Client;\n    }\n\n    public ConsumerManager getConsumerManager() {\n        return consumerManager;\n    }\n\n    public ConsumerFilterManager getConsumerFilterManager() {\n        return consumerFilterManager;\n    }\n\n    public ConsumerOrderInfoManager getConsumerOrderInfoManager() {\n        return consumerOrderInfoManager;\n    }\n\n    public PopInflightMessageCounter getPopInflightMessageCounter() {\n        return popInflightMessageCounter;\n    }\n\n    public PopConsumerService getPopConsumerService() {\n        return popConsumerService;\n    }\n\n    public ConsumerOffsetManager getConsumerOffsetManager() {\n        return consumerOffsetManager;\n    }\n\n    public void setConsumerOffsetManager(ConsumerOffsetManager consumerOffsetManager) {\n        this.consumerOffsetManager = consumerOffsetManager;\n    }\n\n\n    public BroadcastOffsetManager getBroadcastOffsetManager() {\n        return broadcastOffsetManager;\n    }\n\n    public MessageStoreConfig getMessageStoreConfig() {\n        return messageStoreConfig;\n    }\n\n    public ProducerManager getProducerManager() {\n        return producerManager;\n    }\n\n    public PullMessageProcessor getPullMessageProcessor() {\n        return pullMessageProcessor;\n    }\n\n    public PullRequestHoldService getPullRequestHoldService() {\n        return pullRequestHoldService;\n    }\n\n    public void setSubscriptionGroupManager(SubscriptionGroupManager subscriptionGroupManager) {\n        this.subscriptionGroupManager = subscriptionGroupManager;\n    }\n\n    public SubscriptionGroupManager getSubscriptionGroupManager() {\n        return subscriptionGroupManager;\n    }\n\n    public PopMessageProcessor getPopMessageProcessor() {\n        return popMessageProcessor;\n    }\n\n    public PopLiteMessageProcessor getPopLiteMessageProcessor() {\n        return popLiteMessageProcessor;\n    }\n\n    public NotificationProcessor getNotificationProcessor() {\n        return notificationProcessor;\n    }\n\n    public TimerMessageStore getTimerMessageStore() {\n        return timerMessageStore;\n    }\n\n    public void setTimerMessageStore(TimerMessageStore timerMessageStore) {\n        this.timerMessageStore = timerMessageStore;\n    }\n\n    public TimerMessageRocksDBStore getTimerMessageRocksDBStore() {\n        return timerMessageRocksDBStore;\n    }\n\n    public void setTimerMessageRocksDBStore(TimerMessageRocksDBStore timerMessageRocksDBStore) {\n        this.timerMessageRocksDBStore = timerMessageRocksDBStore;\n    }\n\n    public AckMessageProcessor getAckMessageProcessor() {\n        return ackMessageProcessor;\n    }\n\n    public ChangeInvisibleTimeProcessor getChangeInvisibleTimeProcessor() {\n        return changeInvisibleTimeProcessor;\n    }\n\n    public LiteSubscriptionRegistry getLiteSubscriptionRegistry() {\n        return liteSubscriptionRegistry;\n    }\n\n    public AbstractLiteLifecycleManager getLiteLifecycleManager() {\n        return liteLifecycleManager;\n    }\n\n    protected void shutdownBasicService() {\n\n        shutdown = true;\n\n        this.unregisterBrokerAll();\n\n        if (this.shutdownHook != null) {\n            this.shutdownHook.beforeShutdown(this);\n        }\n\n        for (Map.Entry<String, RemotingServer> entry : remotingServerMap.entrySet()) {\n            RemotingServer remotingServer = entry.getValue();\n            if (remotingServer != null) {\n                remotingServer.shutdown();\n            }\n        }\n\n        if (this.brokerMetricsManager != null) {\n            this.brokerMetricsManager.shutdown();\n        }\n\n        if (this.brokerStatsManager != null) {\n            this.brokerStatsManager.shutdown();\n        }\n\n        if (this.clientHousekeepingService != null) {\n            this.clientHousekeepingService.shutdown();\n        }\n\n        if (this.pullRequestHoldService != null) {\n            this.pullRequestHoldService.shutdown();\n        }\n\n        if (this.popMessageProcessor.getPopLongPollingService() != null) {\n            this.popMessageProcessor.getPopLongPollingService().shutdown();\n        }\n\n        if (this.popLiteMessageProcessor != null) {\n            this.popLiteMessageProcessor.stopPopLiteLockManager();\n            if (this.popLiteMessageProcessor.getPopLiteLongPollingService() != null) {\n                this.popLiteMessageProcessor.getPopLiteLongPollingService().shutdown();\n            }\n        }\n\n        if (this.popMessageProcessor.getQueueLockManager() != null) {\n            this.popMessageProcessor.getQueueLockManager().shutdown();\n        }\n\n        if (this.popMessageProcessor.getPopBufferMergeService() != null) {\n            this.popMessageProcessor.getPopBufferMergeService().shutdown();\n        }\n\n        if (this.ackMessageProcessor.getPopReviveServices() != null) {\n            this.ackMessageProcessor.shutdownPopReviveService();\n        }\n\n        if (this.transactionalMessageService != null) {\n            this.transactionalMessageService.close();\n        }\n\n        if (this.transactionalMessageCheckListener != null) {\n            this.transactionalMessageCheckListener.shutdown();\n        }\n\n        if (transactionalMessageCheckService != null) {\n            this.transactionalMessageCheckService.shutdown();\n        }\n\n        if (transactionMetricsFlushService != null) {\n            this.transactionMetricsFlushService.shutdown();\n        }\n\n        if (this.transactionalMessageRocksDBService != null) {\n            this.transactionalMessageRocksDBService.shutdown();\n        }\n\n        if (this.notificationProcessor != null) {\n            this.notificationProcessor.getPopLongPollingService().shutdown();\n        }\n\n        if (this.consumerIdsChangeListener != null) {\n            this.consumerIdsChangeListener.shutdown();\n        }\n\n        if (this.topicQueueMappingCleanService != null) {\n            this.topicQueueMappingCleanService.shutdown();\n        }\n        //it is better to make sure the timerMessageStore shutdown firstly\n        if (this.timerMessageStore != null) {\n            this.timerMessageStore.shutdown();\n        }\n\n        if (this.timerMessageRocksDBStore != null) {\n            this.timerMessageRocksDBStore.shutdown();\n        }\n\n        if (this.transMessageRocksDBStore != null) {\n            this.transMessageRocksDBStore.shutdown();\n        }\n\n        if (this.fileWatchService != null) {\n            this.fileWatchService.shutdown();\n        }\n\n        if (this.broadcastOffsetManager != null) {\n            this.broadcastOffsetManager.shutdown();\n        }\n\n        if (this.replicasManager != null) {\n            this.replicasManager.shutdown();\n        }\n\n        shutdownScheduledExecutorService(this.scheduledExecutorService);\n\n        if (this.sendMessageExecutor != null) {\n            this.sendMessageExecutor.shutdown();\n        }\n\n        if (this.litePullMessageExecutor != null) {\n            this.litePullMessageExecutor.shutdown();\n        }\n\n        if (this.pullMessageExecutor != null) {\n            this.pullMessageExecutor.shutdown();\n        }\n\n        if (this.replyMessageExecutor != null) {\n            this.replyMessageExecutor.shutdown();\n        }\n\n        if (this.putMessageFutureExecutor != null) {\n            this.putMessageFutureExecutor.shutdown();\n        }\n\n        if (this.ackMessageExecutor != null) {\n            this.ackMessageExecutor.shutdown();\n        }\n\n        if (this.adminBrokerExecutor != null) {\n            this.adminBrokerExecutor.shutdown();\n        }\n\n        if (this.brokerFastFailure != null) {\n            this.brokerFastFailure.shutdown();\n        }\n\n        if (this.consumerFilterManager != null) {\n            this.consumerFilterManager.persist();\n        }\n\n        if (this.scheduleMessageService != null) {\n            this.scheduleMessageService.persist();\n            this.scheduleMessageService.shutdown();\n        }\n\n        if (this.clientManageExecutor != null) {\n            this.clientManageExecutor.shutdown();\n        }\n\n        if (this.queryMessageExecutor != null) {\n            this.queryMessageExecutor.shutdown();\n        }\n\n        if (this.heartbeatExecutor != null) {\n            this.heartbeatExecutor.shutdown();\n        }\n\n        if (this.consumerManageExecutor != null) {\n            this.consumerManageExecutor.shutdown();\n        }\n\n        if (this.transactionalMessageCheckService != null) {\n            this.transactionalMessageCheckService.shutdown(false);\n        }\n\n        if (this.loadBalanceExecutor != null) {\n            this.loadBalanceExecutor.shutdown();\n        }\n\n        if (this.endTransactionExecutor != null) {\n            this.endTransactionExecutor.shutdown();\n        }\n\n        if (this.transactionMetricsFlushService != null) {\n            this.transactionMetricsFlushService.shutdown();\n        }\n\n        if (this.escapeBridge != null) {\n            this.escapeBridge.shutdown();\n        }\n\n        if (this.topicRouteInfoManager != null) {\n            this.topicRouteInfoManager.shutdown();\n        }\n\n        if (this.brokerPreOnlineService != null && !this.brokerPreOnlineService.isStopped()) {\n            this.brokerPreOnlineService.shutdown();\n        }\n\n        if (this.coldDataPullRequestHoldService != null) {\n            this.coldDataPullRequestHoldService.shutdown();\n        }\n\n        if (this.coldDataCgCtrService != null) {\n            this.coldDataCgCtrService.shutdown();\n        }\n\n        if (this.liteEventDispatcher != null) {\n            this.liteEventDispatcher.shutdown();\n        }\n\n        if (this.liteLifecycleManager != null) {\n            this.liteLifecycleManager.shutdown();\n        }\n\n        if (this.liteSubscriptionRegistry != null) {\n            this.liteSubscriptionRegistry.shutdown();\n        }\n\n        shutdownScheduledExecutorService(this.syncBrokerMemberGroupExecutorService);\n        shutdownScheduledExecutorService(this.brokerHeartbeatExecutorService);\n\n        if (this.topicConfigManager != null) {\n            this.topicConfigManager.persist();\n            this.topicConfigManager.stop();\n        }\n\n        if (this.subscriptionGroupManager != null) {\n            this.subscriptionGroupManager.persist();\n            this.subscriptionGroupManager.stop();\n        }\n\n        if (this.consumerOffsetManager != null) {\n            this.consumerOffsetManager.persist();\n            this.consumerOffsetManager.stop();\n        }\n\n        if (this.consumerOrderInfoManager != null) {\n            this.consumerOrderInfoManager.persist();\n            this.consumerOrderInfoManager.shutdown();\n        }\n\n        if (this.configStorage != null) {\n            this.configStorage.shutdown();\n        }\n\n        if (this.authenticationMetadataManager != null) {\n            this.authenticationMetadataManager.shutdown();\n        }\n\n        if (this.authorizationMetadataManager != null) {\n            this.authorizationMetadataManager.shutdown();\n        }\n\n        for (BrokerAttachedPlugin brokerAttachedPlugin : brokerAttachedPlugins) {\n            if (brokerAttachedPlugin != null) {\n                brokerAttachedPlugin.shutdown();\n            }\n        }\n\n        if (this.popConsumerService != null) {\n            this.popConsumerService.shutdown();\n        }\n\n        if (this.messageStore != null) {\n            this.messageStore.shutdown();\n        }\n    }\n\n    public void shutdown() {\n\n        shutdownBasicService();\n\n        for (ScheduledFuture<?> scheduledFuture : scheduledFutures) {\n            scheduledFuture.cancel(true);\n        }\n\n        if (this.brokerOuterAPI != null) {\n            this.brokerOuterAPI.shutdown();\n        }\n    }\n\n    protected void shutdownScheduledExecutorService(ScheduledExecutorService scheduledExecutorService) {\n        if (scheduledExecutorService == null) {\n            return;\n        }\n        scheduledExecutorService.shutdown();\n        try {\n            scheduledExecutorService.awaitTermination(5000, TimeUnit.MILLISECONDS);\n        } catch (InterruptedException ignore) {\n            BrokerController.LOG.warn(\"shutdown ScheduledExecutorService was Interrupted!  \", ignore);\n            Thread.currentThread().interrupt();\n        }\n    }\n\n    protected void unregisterBrokerAll() {\n        this.brokerOuterAPI.unregisterBrokerAll(\n            this.brokerConfig.getBrokerClusterName(),\n            this.getBrokerAddr(),\n            this.brokerConfig.getBrokerName(),\n            this.brokerConfig.getBrokerId());\n    }\n\n    public String getBrokerAddr() {\n        return this.brokerConfig.getBrokerIP1() + \":\" + this.nettyServerConfig.getListenPort();\n    }\n\n    protected void startBasicService() throws Exception {\n\n        if (this.messageStore != null) {\n            this.messageStore.start();\n        }\n\n        if (this.timerMessageStore != null) {\n            this.timerMessageStore.start();\n        }\n\n        if (this.timerMessageRocksDBStore != null && this.messageStoreConfig.isTimerRocksDBEnable()) {\n            this.timerMessageRocksDBStore.start();\n        }\n\n        if (this.replicasManager != null) {\n            this.replicasManager.start();\n        }\n\n        if (remotingServerStartLatch != null) {\n            remotingServerStartLatch.await();\n        }\n\n        for (Map.Entry<String, RemotingServer> entry : remotingServerMap.entrySet()) {\n            RemotingServer remotingServer = entry.getValue();\n            if (remotingServer != null) {\n                remotingServer.start();\n\n                if (TCP_REMOTING_SERVER.equals(entry.getKey())) {\n                    // In test scenarios where it is up to OS to pick up an available port, set the listening port back to config\n                    if (null != nettyServerConfig && 0 == nettyServerConfig.getListenPort()) {\n                        nettyServerConfig.setListenPort(remotingServer.localListenPort());\n                    }\n                }\n            }\n        }\n\n        this.storeHost = new InetSocketAddress(this.getBrokerConfig().getBrokerIP1(), this.getNettyServerConfig().getListenPort());\n\n        for (BrokerAttachedPlugin brokerAttachedPlugin : brokerAttachedPlugins) {\n            if (brokerAttachedPlugin != null) {\n                brokerAttachedPlugin.start();\n            }\n        }\n\n        if (this.popMessageProcessor != null) {\n            this.popMessageProcessor.getPopLongPollingService().start();\n            if (brokerConfig.isPopConsumerFSServiceInit()) {\n                this.popMessageProcessor.getPopBufferMergeService().start();\n            }\n            this.popMessageProcessor.getQueueLockManager().start();\n        }\n\n        if (this.popLiteMessageProcessor != null) {\n            this.popLiteMessageProcessor.startPopLiteLockManager();\n            if (this.popLiteMessageProcessor.getPopLiteLongPollingService() != null) {\n                this.popLiteMessageProcessor.getPopLiteLongPollingService().start();\n            }\n        }\n\n        if (this.ackMessageProcessor != null) {\n            if (brokerConfig.isPopConsumerFSServiceInit()) {\n                this.ackMessageProcessor.startPopReviveService();\n            }\n        }\n\n        if (this.notificationProcessor != null) {\n            this.notificationProcessor.getPopLongPollingService().start();\n        }\n\n        if (this.popConsumerService != null) {\n            this.popConsumerService.start();\n        }\n\n        if (this.topicQueueMappingCleanService != null) {\n            this.topicQueueMappingCleanService.start();\n        }\n\n        if (this.fileWatchService != null) {\n            this.fileWatchService.start();\n        }\n\n        if (this.pullRequestHoldService != null) {\n            this.pullRequestHoldService.start();\n        }\n\n        if (this.clientHousekeepingService != null) {\n            this.clientHousekeepingService.start();\n        }\n\n        if (this.brokerStatsManager != null) {\n            this.brokerStatsManager.start();\n        }\n\n        if (this.brokerFastFailure != null) {\n            this.brokerFastFailure.start();\n        }\n\n        if (this.broadcastOffsetManager != null) {\n            this.broadcastOffsetManager.start();\n        }\n\n        if (this.escapeBridge != null) {\n            this.escapeBridge.start();\n        }\n\n        if (this.topicRouteInfoManager != null) {\n            this.topicRouteInfoManager.start();\n        }\n\n        if (this.brokerPreOnlineService != null) {\n            this.brokerPreOnlineService.start();\n        }\n\n        if (this.coldDataPullRequestHoldService != null) {\n            this.coldDataPullRequestHoldService.start();\n        }\n\n        if (this.coldDataCgCtrService != null) {\n            this.coldDataCgCtrService.start();\n        }\n\n        if (this.liteEventDispatcher != null) {\n            this.liteEventDispatcher.start();\n        }\n\n        if (this.liteLifecycleManager != null) {\n            this.liteLifecycleManager.start();\n        }\n\n        if (this.liteSubscriptionRegistry != null) {\n            this.liteSubscriptionRegistry.start();\n        }\n    }\n\n    public void start() throws Exception {\n\n        this.shouldStartTime = System.currentTimeMillis() + messageStoreConfig.getDisappearTimeAfterStart();\n\n        if (messageStoreConfig.getTotalReplicas() > 1 && this.brokerConfig.isEnableSlaveActingMaster()) {\n            isIsolated = true;\n        }\n\n        if (this.brokerOuterAPI != null) {\n            this.brokerOuterAPI.start();\n        }\n\n        startBasicService();\n\n        if (!isIsolated && !this.messageStoreConfig.isEnableDLegerCommitLog() && !this.messageStoreConfig.isDuplicationEnable()) {\n            changeSpecialServiceStatus(this.brokerConfig.getBrokerId() == MixAll.MASTER_ID);\n            this.registerBrokerAll(true, false, true);\n        }\n\n        scheduledFutures.add(this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    if (System.currentTimeMillis() < shouldStartTime) {\n                        BrokerController.LOG.info(\"Register to namesrv after {}\", shouldStartTime);\n                        return;\n                    }\n                    if (isIsolated) {\n                        BrokerController.LOG.info(\"Skip register for broker is isolated\");\n                        return;\n                    }\n                    BrokerController.this.registerBrokerAll(true, false, brokerConfig.isForceRegister());\n                } catch (Throwable e) {\n                    BrokerController.LOG.error(\"registerBrokerAll Exception\", e);\n                }\n            }\n        }, 1000 * 10, Math.max(10000, Math.min(brokerConfig.getRegisterNameServerPeriod(), 60000)), TimeUnit.MILLISECONDS));\n\n        if (this.brokerConfig.isEnableSlaveActingMaster()) {\n            scheduleSendHeartbeat();\n\n            scheduledFutures.add(this.syncBrokerMemberGroupExecutorService.scheduleAtFixedRate(new Runnable() {\n                @Override\n                public void run() {\n                    try {\n                        BrokerController.this.syncBrokerMemberGroup();\n                    } catch (Throwable e) {\n                        BrokerController.LOG.error(\"sync BrokerMemberGroup error. \", e);\n                    }\n                }\n            }, 1000, this.brokerConfig.getSyncBrokerMemberGroupPeriod(), TimeUnit.MILLISECONDS));\n        }\n\n        if (this.brokerConfig.isEnableControllerMode()) {\n            scheduleSendHeartbeat();\n        }\n\n        if (brokerConfig.isSkipPreOnline()) {\n            startServiceWithoutCondition();\n        }\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    BrokerController.this.brokerOuterAPI.refreshMetadata();\n                } catch (Exception e) {\n                    LOG.error(\"ScheduledTask refresh metadata exception\", e);\n                }\n            }\n        }, 10, 5, TimeUnit.SECONDS);\n    }\n\n    protected void scheduleSendHeartbeat() {\n        scheduledFutures.add(this.brokerHeartbeatExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                if (isIsolated) {\n                    return;\n                }\n                try {\n                    BrokerController.this.sendHeartbeat();\n                } catch (Exception e) {\n                    BrokerController.LOG.error(\"sendHeartbeat Exception\", e);\n                }\n\n            }\n        }, 1000, brokerConfig.getBrokerHeartbeatInterval(), TimeUnit.MILLISECONDS));\n    }\n\n    public synchronized void registerSingleTopicAll(final TopicConfig topicConfig) {\n        TopicConfig tmpTopic = topicConfig;\n        if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission())\n            || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) {\n            // Copy the topic config and modify the perm\n            tmpTopic = new TopicConfig(topicConfig);\n            tmpTopic.setPerm(topicConfig.getPerm() & this.brokerConfig.getBrokerPermission());\n        }\n        this.brokerOuterAPI.registerSingleTopicAll(this.brokerConfig.getBrokerName(), tmpTopic, 3000);\n    }\n\n    public synchronized void registerIncrementBrokerData(TopicConfig topicConfig, DataVersion dataVersion) {\n        this.registerIncrementBrokerData(Collections.singletonList(topicConfig), dataVersion);\n    }\n\n    public synchronized void registerIncrementBrokerData(List<TopicConfig> topicConfigList, DataVersion dataVersion) {\n        if (topicConfigList == null || topicConfigList.isEmpty()) {\n            return;\n        }\n\n        TopicConfigAndMappingSerializeWrapper topicConfigSerializeWrapper = new TopicConfigAndMappingSerializeWrapper();\n        topicConfigSerializeWrapper.setDataVersion(dataVersion);\n\n        ConcurrentMap<String, TopicConfig> topicConfigTable = topicConfigList.stream()\n            .map(topicConfig -> {\n                TopicConfig registerTopicConfig;\n                if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission())\n                    || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) {\n                    registerTopicConfig =\n                        new TopicConfig(topicConfig.getTopicName(),\n                            topicConfig.getReadQueueNums(),\n                            topicConfig.getWriteQueueNums(),\n                                topicConfig.getPerm()\n                                        & this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag());\n                } else {\n                    registerTopicConfig = new TopicConfig(topicConfig);\n                }\n                return registerTopicConfig;\n            })\n            .collect(Collectors.toConcurrentMap(TopicConfig::getTopicName, Function.identity()));\n        topicConfigSerializeWrapper.setTopicConfigTable(topicConfigTable);\n\n        Map<String, TopicQueueMappingInfo> topicQueueMappingInfoMap = topicConfigList.stream()\n            .map(TopicConfig::getTopicName)\n            .map(topicName -> Optional.ofNullable(this.topicQueueMappingManager.getTopicQueueMapping(topicName))\n                .map(info -> new AbstractMap.SimpleImmutableEntry<>(topicName, TopicQueueMappingDetail.cloneAsMappingInfo(info)))\n                .orElse(null))\n            .filter(Objects::nonNull)\n            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));\n        if (!topicQueueMappingInfoMap.isEmpty()) {\n            topicConfigSerializeWrapper.setTopicQueueMappingInfoMap(topicQueueMappingInfoMap);\n        }\n\n        doRegisterBrokerAll(true, false, topicConfigSerializeWrapper);\n    }\n\n    public synchronized void registerBrokerAll(final boolean checkOrderConfig, boolean oneway, boolean forceRegister) {\n        ConcurrentMap<String, TopicConfig> topicConfigMap = this.getTopicConfigManager().getTopicConfigTable();\n        ConcurrentHashMap<String, TopicConfig> topicConfigTable = new ConcurrentHashMap<>();\n\n        for (TopicConfig topicConfig : topicConfigMap.values()) {\n            if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission())\n                || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) {\n                topicConfigTable.put(topicConfig.getTopicName(),\n                    new TopicConfig(topicConfig.getTopicName(), topicConfig.getReadQueueNums(), topicConfig.getWriteQueueNums(),\n                        topicConfig.getPerm() & getBrokerConfig().getBrokerPermission()));\n            } else {\n                topicConfigTable.put(topicConfig.getTopicName(), topicConfig);\n            }\n\n            if (this.brokerConfig.isEnableSplitRegistration()\n                && topicConfigTable.size() >= this.brokerConfig.getSplitRegistrationSize()) {\n                TopicConfigAndMappingSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().buildSerializeWrapper(topicConfigTable);\n                doRegisterBrokerAll(checkOrderConfig, oneway, topicConfigWrapper);\n                topicConfigTable.clear();\n            }\n        }\n\n        Map<String, TopicQueueMappingInfo> topicQueueMappingInfoMap = this.getTopicQueueMappingManager().getTopicQueueMappingTable().entrySet().stream()\n            .map(entry -> new AbstractMap.SimpleImmutableEntry<>(entry.getKey(), TopicQueueMappingDetail.cloneAsMappingInfo(entry.getValue())))\n            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));\n\n        TopicConfigAndMappingSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().\n            buildSerializeWrapper(topicConfigTable, topicQueueMappingInfoMap);\n        if (this.brokerConfig.isEnableSplitRegistration() || forceRegister || needRegister(this.brokerConfig.getBrokerClusterName(),\n            this.getBrokerAddr(),\n            this.brokerConfig.getBrokerName(),\n            this.brokerConfig.getBrokerId(),\n            this.brokerConfig.getRegisterBrokerTimeoutMills(),\n            this.brokerConfig.isInBrokerContainer())) {\n            doRegisterBrokerAll(checkOrderConfig, oneway, topicConfigWrapper);\n        }\n    }\n\n    protected void doRegisterBrokerAll(boolean checkOrderConfig, boolean oneway,\n        TopicConfigSerializeWrapper topicConfigWrapper) {\n\n        if (shutdown) {\n            BrokerController.LOG.info(\"BrokerController#doRegisterBrokerAll: broker has shutdown, no need to register any more.\");\n            return;\n        }\n        List<RegisterBrokerResult> registerBrokerResultList = this.brokerOuterAPI.registerBrokerAll(\n            this.brokerConfig.getBrokerClusterName(),\n            this.getBrokerAddr(),\n            this.brokerConfig.getBrokerName(),\n            this.brokerConfig.getBrokerId(),\n            this.getHAServerAddr(),\n            topicConfigWrapper,\n            Lists.newArrayList(),\n            oneway,\n            this.brokerConfig.getRegisterBrokerTimeoutMills(),\n            this.brokerConfig.isEnableSlaveActingMaster(),\n            this.brokerConfig.isCompressedRegister(),\n            this.brokerConfig.isEnableSlaveActingMaster() ? this.brokerConfig.getBrokerNotActiveTimeoutMillis() : null,\n            this.getBrokerIdentity());\n\n        handleRegisterBrokerResult(registerBrokerResultList, checkOrderConfig);\n    }\n\n    protected void sendHeartbeat() {\n        if (this.brokerConfig.isEnableControllerMode()) {\n            this.replicasManager.sendHeartbeatToController();\n        }\n\n        if (this.brokerConfig.isEnableSlaveActingMaster()) {\n            if (this.brokerConfig.isCompatibleWithOldNameSrv()) {\n                this.brokerOuterAPI.sendHeartbeatViaDataVersion(\n                    this.brokerConfig.getBrokerClusterName(),\n                    this.getBrokerAddr(),\n                    this.brokerConfig.getBrokerName(),\n                    this.brokerConfig.getBrokerId(),\n                    this.brokerConfig.getSendHeartbeatTimeoutMillis(),\n                    this.getTopicConfigManager().getDataVersion(),\n                    this.brokerConfig.isInBrokerContainer());\n            } else {\n                this.brokerOuterAPI.sendHeartbeat(\n                    this.brokerConfig.getBrokerClusterName(),\n                    this.getBrokerAddr(),\n                    this.brokerConfig.getBrokerName(),\n                    this.brokerConfig.getBrokerId(),\n                    this.brokerConfig.getSendHeartbeatTimeoutMillis(),\n                    this.brokerConfig.isInBrokerContainer());\n            }\n        }\n    }\n\n    protected void syncBrokerMemberGroup() {\n        try {\n            brokerMemberGroup = this.getBrokerOuterAPI()\n                .syncBrokerMemberGroup(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.getBrokerName(), this.brokerConfig.isCompatibleWithOldNameSrv());\n        } catch (Exception e) {\n            BrokerController.LOG.error(\"syncBrokerMemberGroup from namesrv failed, \", e);\n            return;\n        }\n        if (brokerMemberGroup == null || brokerMemberGroup.getBrokerAddrs().size() == 0) {\n            BrokerController.LOG.warn(\"Couldn't find any broker member from namesrv in {}/{}\", this.brokerConfig.getBrokerClusterName(), this.brokerConfig.getBrokerName());\n            return;\n        }\n        this.messageStore.setAliveReplicaNumInGroup(calcAliveBrokerNumInGroup(brokerMemberGroup.getBrokerAddrs()));\n\n        if (!this.isIsolated) {\n            long minBrokerId = brokerMemberGroup.minimumBrokerId();\n            this.updateMinBroker(minBrokerId, brokerMemberGroup.getBrokerAddrs().get(minBrokerId));\n        }\n    }\n\n    private int calcAliveBrokerNumInGroup(Map<Long, String> brokerAddrTable) {\n        if (brokerAddrTable.containsKey(this.brokerConfig.getBrokerId())) {\n            return brokerAddrTable.size();\n        } else {\n            return brokerAddrTable.size() + 1;\n        }\n    }\n\n    protected void handleRegisterBrokerResult(List<RegisterBrokerResult> registerBrokerResultList,\n        boolean checkOrderConfig) {\n        for (RegisterBrokerResult registerBrokerResult : registerBrokerResultList) {\n            if (registerBrokerResult != null) {\n                if (this.updateMasterHAServerAddrPeriodically && registerBrokerResult.getHaServerAddr() != null) {\n                    this.messageStore.updateHaMasterAddress(registerBrokerResult.getHaServerAddr());\n                    this.messageStore.updateMasterAddress(registerBrokerResult.getMasterAddr());\n                }\n\n                this.slaveSynchronize.setMasterAddr(registerBrokerResult.getMasterAddr());\n                if (checkOrderConfig) {\n                    this.getTopicConfigManager().updateOrderTopicConfig(registerBrokerResult.getKvTable());\n                }\n                break;\n            }\n        }\n    }\n\n    private boolean needRegister(final String clusterName,\n        final String brokerAddr,\n        final String brokerName,\n        final long brokerId,\n        final int timeoutMills,\n        final boolean isInBrokerContainer) {\n\n        TopicConfigSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().buildTopicConfigSerializeWrapper();\n        List<Boolean> changeList = brokerOuterAPI.needRegister(clusterName, brokerAddr, brokerName, brokerId, topicConfigWrapper, timeoutMills, isInBrokerContainer);\n        boolean needRegister = false;\n        for (Boolean changed : changeList) {\n            if (changed) {\n                needRegister = true;\n                break;\n            }\n        }\n        return needRegister;\n    }\n\n    public void startService(long minBrokerId, String minBrokerAddr) {\n        BrokerController.LOG.info(\"{} start service, min broker id is {}, min broker addr: {}\",\n            this.brokerConfig.getCanonicalName(), minBrokerId, minBrokerAddr);\n        this.minBrokerIdInGroup = minBrokerId;\n        this.minBrokerAddrInGroup = minBrokerAddr;\n\n        this.changeSpecialServiceStatus(this.brokerConfig.getBrokerId() == minBrokerId);\n        this.registerBrokerAll(true, false, brokerConfig.isForceRegister());\n\n        isIsolated = false;\n    }\n\n    public void startServiceWithoutCondition() {\n        BrokerController.LOG.info(\"{} start service\", this.brokerConfig.getCanonicalName());\n\n        this.changeSpecialServiceStatus(this.brokerConfig.getBrokerId() == MixAll.MASTER_ID);\n        this.registerBrokerAll(true, false, brokerConfig.isForceRegister());\n\n        isIsolated = false;\n    }\n\n    public void stopService() {\n        BrokerController.LOG.info(\"{} stop service\", this.getBrokerConfig().getCanonicalName());\n        isIsolated = true;\n        this.changeSpecialServiceStatus(false);\n    }\n\n    public boolean isSpecialServiceRunning() {\n        if (isScheduleServiceStart() && isTransactionCheckServiceStart()) {\n            return true;\n        }\n\n        return this.ackMessageProcessor != null && this.ackMessageProcessor.isPopReviveServiceRunning();\n    }\n\n    private void onMasterOffline() {\n        // close channels with master broker\n        String masterAddr = this.slaveSynchronize.getMasterAddr();\n        if (masterAddr != null) {\n            this.brokerOuterAPI.getRemotingClient().closeChannels(\n                Arrays.asList(masterAddr, MixAll.brokerVIPChannel(true, masterAddr)));\n        }\n        // master not available, stop sync\n        this.slaveSynchronize.setMasterAddr(null);\n        this.messageStore.updateHaMasterAddress(null);\n    }\n\n    private void onMasterOnline(String masterAddr, String masterHaAddr) {\n        boolean needSyncMasterFlushOffset = this.messageStore.getMasterFlushedOffset() == 0\n            && this.messageStoreConfig.isSyncMasterFlushOffsetWhenStartup();\n        if (masterHaAddr == null || needSyncMasterFlushOffset) {\n            try {\n                BrokerSyncInfo brokerSyncInfo = this.brokerOuterAPI.retrieveBrokerHaInfo(masterAddr);\n\n                if (needSyncMasterFlushOffset) {\n                    LOG.info(\"Set master flush offset in slave to {}\", brokerSyncInfo.getMasterFlushOffset());\n                    this.messageStore.setMasterFlushedOffset(brokerSyncInfo.getMasterFlushOffset());\n                }\n\n                if (masterHaAddr == null) {\n                    this.messageStore.updateHaMasterAddress(brokerSyncInfo.getMasterHaAddress());\n                    this.messageStore.updateMasterAddress(brokerSyncInfo.getMasterAddress());\n                }\n            } catch (Exception e) {\n                LOG.error(\"retrieve master ha info exception, {}\", e);\n            }\n        }\n\n        // set master HA address.\n        if (masterHaAddr != null) {\n            this.messageStore.updateHaMasterAddress(masterHaAddr);\n        }\n\n        // wakeup HAClient\n        this.messageStore.wakeupHAClient();\n    }\n\n    private void onMinBrokerChange(long minBrokerId, String minBrokerAddr, String offlineBrokerAddr,\n        String masterHaAddr) {\n        LOG.info(\"Min broker changed, old: {}-{}, new {}-{}\",\n            this.minBrokerIdInGroup, this.minBrokerAddrInGroup, minBrokerId, minBrokerAddr);\n\n        this.minBrokerIdInGroup = minBrokerId;\n        this.minBrokerAddrInGroup = minBrokerAddr;\n\n        this.changeSpecialServiceStatus(this.brokerConfig.getBrokerId() == this.minBrokerIdInGroup);\n\n        if (offlineBrokerAddr != null && offlineBrokerAddr.equals(this.slaveSynchronize.getMasterAddr())) {\n            // master offline\n            onMasterOffline();\n        }\n\n        if (minBrokerId == MixAll.MASTER_ID && minBrokerAddr != null) {\n            // master online\n            onMasterOnline(minBrokerAddr, masterHaAddr);\n        }\n\n        // notify PullRequest on hold to pull from master.\n        if (this.minBrokerIdInGroup == MixAll.MASTER_ID) {\n            this.pullRequestHoldService.notifyMasterOnline();\n        }\n    }\n\n    public void updateMinBroker(long minBrokerId, String minBrokerAddr) {\n        if (brokerConfig.isEnableSlaveActingMaster() && brokerConfig.getBrokerId() != MixAll.MASTER_ID) {\n            if (lock.tryLock()) {\n                try {\n                    if (minBrokerId != this.minBrokerIdInGroup) {\n                        String offlineBrokerAddr = null;\n                        if (minBrokerId > this.minBrokerIdInGroup) {\n                            offlineBrokerAddr = this.minBrokerAddrInGroup;\n                        }\n                        onMinBrokerChange(minBrokerId, minBrokerAddr, offlineBrokerAddr, null);\n                    }\n                } finally {\n                    lock.unlock();\n                }\n            }\n        }\n    }\n\n    public void updateMinBroker(long minBrokerId, String minBrokerAddr, String offlineBrokerAddr,\n        String masterHaAddr) {\n        if (brokerConfig.isEnableSlaveActingMaster() && brokerConfig.getBrokerId() != MixAll.MASTER_ID) {\n            try {\n                if (lock.tryLock(3000, TimeUnit.MILLISECONDS)) {\n                    try {\n                        if (minBrokerId != this.minBrokerIdInGroup) {\n                            onMinBrokerChange(minBrokerId, minBrokerAddr, offlineBrokerAddr, masterHaAddr);\n                        }\n                    } finally {\n                        lock.unlock();\n                    }\n\n                }\n            } catch (InterruptedException e) {\n                LOG.error(\"Update min broker error, {}\", e);\n            }\n        }\n    }\n\n    public void changeSpecialServiceStatus(boolean shouldStart) {\n\n        for (BrokerAttachedPlugin brokerAttachedPlugin : brokerAttachedPlugins) {\n            if (brokerAttachedPlugin != null) {\n                brokerAttachedPlugin.statusChanged(shouldStart);\n            }\n        }\n\n        changeScheduleServiceStatus(shouldStart);\n\n        changeTransactionCheckServiceStatus(shouldStart);\n\n        if (this.ackMessageProcessor != null) {\n            LOG.info(\"Set PopReviveService Status to {}\", shouldStart);\n            this.ackMessageProcessor.setPopReviveServiceStatus(shouldStart);\n        }\n    }\n\n    private synchronized void changeTransactionCheckServiceStatus(boolean shouldStart) {\n        if (isTransactionCheckServiceStart != shouldStart) {\n            LOG.info(\"TransactionCheckService status changed to {}\", shouldStart);\n            if (shouldStart) {\n                this.transactionalMessageCheckService.start();\n            } else {\n                this.transactionalMessageCheckService.shutdown(true);\n            }\n            isTransactionCheckServiceStart = shouldStart;\n        }\n    }\n\n    public synchronized void changeScheduleServiceStatus(boolean shouldStart) {\n        if (isScheduleServiceStart != shouldStart) {\n            LOG.info(\"ScheduleServiceStatus changed to {}\", shouldStart);\n            if (shouldStart) {\n                this.scheduleMessageService.start();\n            } else {\n                this.scheduleMessageService.stop();\n            }\n            isScheduleServiceStart = shouldStart;\n\n            if (timerMessageStore != null) {\n                timerMessageStore.syncLastReadTimeMs();\n                timerMessageStore.setShouldRunningDequeue(shouldStart);\n            }\n        }\n    }\n\n    public MessageStore getMessageStoreByBrokerName(String brokerName) {\n        if (this.brokerConfig.getBrokerName().equals(brokerName)) {\n            return this.getMessageStore();\n        }\n        return null;\n    }\n\n    public BrokerIdentity getBrokerIdentity() {\n        if (messageStoreConfig.isEnableDLegerCommitLog()) {\n            return new BrokerIdentity(\n                brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName(),\n                Integer.parseInt(messageStoreConfig.getdLegerSelfId().substring(1)), brokerConfig.isInBrokerContainer());\n        } else {\n            return new BrokerIdentity(\n                brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName(),\n                brokerConfig.getBrokerId(), brokerConfig.isInBrokerContainer());\n        }\n    }\n\n    public TopicConfigManager getTopicConfigManager() {\n        return topicConfigManager;\n    }\n\n    public void setTopicConfigManager(TopicConfigManager topicConfigManager) {\n        this.topicConfigManager = topicConfigManager;\n    }\n\n    public TopicQueueMappingManager getTopicQueueMappingManager() {\n        return topicQueueMappingManager;\n    }\n\n    public AuthenticationMetadataManager getAuthenticationMetadataManager() {\n        return authenticationMetadataManager;\n    }\n\n    @VisibleForTesting\n    public void setAuthenticationMetadataManager(\n        AuthenticationMetadataManager authenticationMetadataManager) {\n        this.authenticationMetadataManager = authenticationMetadataManager;\n    }\n\n    public AuthorizationMetadataManager getAuthorizationMetadataManager() {\n        return authorizationMetadataManager;\n    }\n\n    @VisibleForTesting\n    public void setAuthorizationMetadataManager(\n        AuthorizationMetadataManager authorizationMetadataManager) {\n        this.authorizationMetadataManager = authorizationMetadataManager;\n    }\n\n    public String getHAServerAddr() {\n        return this.brokerConfig.getBrokerIP2() + \":\" + this.messageStoreConfig.getHaListenPort();\n    }\n\n    public RebalanceLockManager getRebalanceLockManager() {\n        return rebalanceLockManager;\n    }\n\n    public SlaveSynchronize getSlaveSynchronize() {\n        return slaveSynchronize;\n    }\n\n    public ScheduledExecutorService getScheduledExecutorService() {\n        return scheduledExecutorService;\n    }\n\n    public ExecutorService getPullMessageExecutor() {\n        return pullMessageExecutor;\n    }\n\n    public ExecutorService getPutMessageFutureExecutor() {\n        return putMessageFutureExecutor;\n    }\n\n    public void setPullMessageExecutor(ExecutorService pullMessageExecutor) {\n        this.pullMessageExecutor = pullMessageExecutor;\n    }\n\n    public BlockingQueue<Runnable> getSendThreadPoolQueue() {\n        return sendThreadPoolQueue;\n    }\n\n    public BlockingQueue<Runnable> getAckThreadPoolQueue() {\n        return ackThreadPoolQueue;\n    }\n\n    public BrokerStatsManager getBrokerStatsManager() {\n        return brokerStatsManager;\n    }\n\n    public void setBrokerStatsManager(BrokerStatsManager brokerStatsManager) {\n        this.brokerStatsManager = brokerStatsManager;\n    }\n\n    public List<SendMessageHook> getSendMessageHookList() {\n        return sendMessageHookList;\n    }\n\n    public void registerSendMessageHook(final SendMessageHook hook) {\n        this.sendMessageHookList.add(hook);\n        LOG.info(\"register SendMessageHook Hook, {}\", hook.hookName());\n    }\n\n    public List<ConsumeMessageHook> getConsumeMessageHookList() {\n        return consumeMessageHookList;\n    }\n\n    public void registerConsumeMessageHook(final ConsumeMessageHook hook) {\n        this.consumeMessageHookList.add(hook);\n        LOG.info(\"register ConsumeMessageHook Hook, {}\", hook.hookName());\n    }\n\n    public void registerServerRPCHook(RPCHook rpcHook) {\n        for (Map.Entry<String, RemotingServer> entry : remotingServerMap.entrySet()) {\n            RemotingServer remotingServer = entry.getValue();\n            if (remotingServer != null) {\n                remotingServer.registerRPCHook(rpcHook);\n            }\n        }\n    }\n\n    public void setRequestPipeline(RequestPipeline pipeline) {\n        for (Map.Entry<String, RemotingServer> entry : remotingServerMap.entrySet()) {\n            RemotingServer remotingServer = entry.getValue();\n            if (remotingServer != null) {\n                remotingServer.setRequestPipeline(pipeline);\n            }\n        }\n    }\n\n    public RemotingServer getRemotingServer() {\n        return remotingServerMap.get(TCP_REMOTING_SERVER);\n    }\n\n    public void setRemotingServer(RemotingServer remotingServer) {\n        remotingServerMap.put(TCP_REMOTING_SERVER, remotingServer);\n    }\n\n    public RemotingServer getFastRemotingServer() {\n        return remotingServerMap.get(FAST_REMOTING_SERVER);\n    }\n\n    public void setFastRemotingServer(RemotingServer fastRemotingServer) {\n        remotingServerMap.put(FAST_REMOTING_SERVER, fastRemotingServer);\n    }\n\n    public RemotingServer getRemotingServerByName(String name) {\n        return remotingServerMap.get(name);\n    }\n\n    public void setRemotingServerByName(String name, RemotingServer remotingServer) {\n        remotingServerMap.put(name, remotingServer);\n    }\n\n    public ClientHousekeepingService getClientHousekeepingService() {\n        return clientHousekeepingService;\n    }\n\n    public CountDownLatch getRemotingServerStartLatch() {\n        return remotingServerStartLatch;\n    }\n\n    public void setRemotingServerStartLatch(CountDownLatch remotingServerStartLatch) {\n        this.remotingServerStartLatch = remotingServerStartLatch;\n    }\n\n    public void registerClientRPCHook(RPCHook rpcHook) {\n        this.getBrokerOuterAPI().registerRPCHook(rpcHook);\n    }\n\n    public BrokerOuterAPI getBrokerOuterAPI() {\n        return brokerOuterAPI;\n    }\n\n    public InetSocketAddress getStoreHost() {\n        return storeHost;\n    }\n\n    public void setStoreHost(InetSocketAddress storeHost) {\n        this.storeHost = storeHost;\n    }\n\n    public Configuration getConfiguration() {\n        return this.configuration;\n    }\n\n    public BlockingQueue<Runnable> getHeartbeatThreadPoolQueue() {\n        return heartbeatThreadPoolQueue;\n    }\n\n    public TransactionalMessageCheckService getTransactionalMessageCheckService() {\n        return transactionalMessageCheckService;\n    }\n\n    public void setTransactionalMessageCheckService(\n        TransactionalMessageCheckService transactionalMessageCheckService) {\n        this.transactionalMessageCheckService = transactionalMessageCheckService;\n    }\n\n    public TransactionalMessageService getTransactionalMessageService() {\n        return transactionalMessageService;\n    }\n\n    public void setTransactionalMessageService(TransactionalMessageService transactionalMessageService) {\n        this.transactionalMessageService = transactionalMessageService;\n    }\n\n    public AbstractTransactionalMessageCheckListener getTransactionalMessageCheckListener() {\n        return transactionalMessageCheckListener;\n    }\n\n    public void setTransactionalMessageCheckListener(\n        AbstractTransactionalMessageCheckListener transactionalMessageCheckListener) {\n        this.transactionalMessageCheckListener = transactionalMessageCheckListener;\n    }\n\n    public BlockingQueue<Runnable> getEndTransactionThreadPoolQueue() {\n        return endTransactionThreadPoolQueue;\n\n    }\n\n    public ExecutorService getSendMessageExecutor() {\n        return sendMessageExecutor;\n    }\n\n    public SendMessageProcessor getSendMessageProcessor() {\n        return sendMessageProcessor;\n    }\n\n    public RecallMessageProcessor getRecallMessageProcessor() {\n        return recallMessageProcessor;\n    }\n\n    public QueryAssignmentProcessor getQueryAssignmentProcessor() {\n        return queryAssignmentProcessor;\n    }\n\n    public TopicQueueMappingCleanService getTopicQueueMappingCleanService() {\n        return topicQueueMappingCleanService;\n    }\n\n    public ExecutorService getAdminBrokerExecutor() {\n        return adminBrokerExecutor;\n    }\n\n    public BlockingQueue<Runnable> getLitePullThreadPoolQueue() {\n        return litePullThreadPoolQueue;\n    }\n\n    public ShutdownHook getShutdownHook() {\n        return shutdownHook;\n    }\n\n    public void setShutdownHook(ShutdownHook shutdownHook) {\n        this.shutdownHook = shutdownHook;\n    }\n\n    public long getMinBrokerIdInGroup() {\n        return this.brokerConfig.getBrokerId();\n    }\n\n    public BrokerController peekMasterBroker() {\n        return brokerConfig.getBrokerId() == MixAll.MASTER_ID ? this : null;\n    }\n\n    public BrokerMemberGroup getBrokerMemberGroup() {\n        return this.brokerMemberGroup;\n    }\n\n    public int getListenPort() {\n        return this.nettyServerConfig.getListenPort();\n    }\n\n    public List<BrokerAttachedPlugin> getBrokerAttachedPlugins() {\n        return brokerAttachedPlugins;\n    }\n\n    public EscapeBridge getEscapeBridge() {\n        return escapeBridge;\n    }\n\n    public long getShouldStartTime() {\n        return shouldStartTime;\n    }\n\n    public BrokerPreOnlineService getBrokerPreOnlineService() {\n        return brokerPreOnlineService;\n    }\n\n    public EndTransactionProcessor getEndTransactionProcessor() {\n        return endTransactionProcessor;\n    }\n\n    public boolean isScheduleServiceStart() {\n        return isScheduleServiceStart;\n    }\n\n    public boolean isTransactionCheckServiceStart() {\n        return isTransactionCheckServiceStart;\n    }\n\n    public ScheduleMessageService getScheduleMessageService() {\n        return scheduleMessageService;\n    }\n\n    public ReplicasManager getReplicasManager() {\n        return replicasManager;\n    }\n\n    public void setIsolated(boolean isolated) {\n        isIsolated = isolated;\n    }\n\n    public boolean isIsolated() {\n        return this.isIsolated;\n    }\n\n    public TimerCheckpoint getTimerCheckpoint() {\n        return timerCheckpoint;\n    }\n\n    public TopicRouteInfoManager getTopicRouteInfoManager() {\n        return this.topicRouteInfoManager;\n    }\n\n    public BlockingQueue<Runnable> getClientManagerThreadPoolQueue() {\n        return clientManagerThreadPoolQueue;\n    }\n\n    public BlockingQueue<Runnable> getConsumerManagerThreadPoolQueue() {\n        return consumerManagerThreadPoolQueue;\n    }\n\n    public BlockingQueue<Runnable> getAsyncPutThreadPoolQueue() {\n        return putThreadPoolQueue;\n    }\n\n    public BlockingQueue<Runnable> getReplyThreadPoolQueue() {\n        return replyThreadPoolQueue;\n    }\n\n    public BlockingQueue<Runnable> getAdminBrokerThreadPoolQueue() {\n        return adminBrokerThreadPoolQueue;\n    }\n\n    public ColdDataPullRequestHoldService getColdDataPullRequestHoldService() {\n        return coldDataPullRequestHoldService;\n    }\n\n    public void setColdDataPullRequestHoldService(\n        ColdDataPullRequestHoldService coldDataPullRequestHoldService) {\n        this.coldDataPullRequestHoldService = coldDataPullRequestHoldService;\n    }\n\n    public ColdDataCgCtrService getColdDataCgCtrService() {\n        return coldDataCgCtrService;\n    }\n\n    public void setColdDataCgCtrService(ColdDataCgCtrService coldDataCgCtrService) {\n        this.coldDataCgCtrService = coldDataCgCtrService;\n    }\n\n    public ConfigContext getConfigContext() {\n        return configContext;\n    }\n\n    public void setConfigContext(ConfigContext configContext) {\n        this.configContext = configContext;\n    }\n\n    public LiteEventDispatcher getLiteEventDispatcher() {\n        return liteEventDispatcher;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker;\n\nimport java.io.File;\n\npublic class BrokerPathConfigHelper {\n    private static String brokerConfigPath = System.getProperty(\"user.home\") + File.separator + \"store\"\n        + File.separator + \"config\" + File.separator + \"broker.properties\";\n\n    public static String getBrokerConfigPath() {\n        return brokerConfigPath;\n    }\n\n    public static void setBrokerConfigPath(String path) {\n        brokerConfigPath = path;\n    }\n\n    public static String getTopicConfigPath(final String rootDir) {\n        return getConfigDir(rootDir) + \"topics.json\";\n    }\n\n    public static String getTopicQueueMappingPath(final String rootDir) {\n        return getConfigDir(rootDir) + \"topicQueueMapping.json\";\n    }\n\n    public static String getConsumerOffsetPath(final String rootDir) {\n        return getConfigDir(rootDir) + \"consumerOffset.json\";\n    }\n\n    public static String getLmqConsumerOffsetPath(final String rootDir) {\n        return getConfigDir(rootDir) + \"lmqConsumerOffset.json\";\n    }\n\n    public static String getConsumerOrderInfoPath(final String rootDir) {\n        return getConfigDir(rootDir) + \"consumerOrderInfo.json\";\n    }\n\n    public static String getSubscriptionGroupPath(final String rootDir) {\n        return getConfigDir(rootDir) + \"subscriptionGroup.json\";\n    }\n    public static String getTimerCheckPath(final String rootDir) {\n        return getConfigDir(rootDir) + \"timercheck\";\n    }\n    public static String getTimerMetricsPath(final String rootDir) {\n        return getConfigDir(rootDir) + \"timermetrics\";\n    }\n    public static String getTransactionMetricsPath(final String rootDir) {\n        return getConfigDir(rootDir) + \"transactionMetrics\";\n    }\n\n    public static String getConsumerFilterPath(final String rootDir) {\n        return getConfigDir(rootDir) + \"consumerFilter.json\";\n    }\n\n    public static String getMessageRequestModePath(final String rootDir) {\n        return getConfigDir(rootDir) + \"messageRequestMode.json\";\n    }\n\n    private static String getConfigDir(final String rootDir) {\n        return rootDir + File.separator + \"config\" + File.separator;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/BrokerPreOnlineService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker;\n\nimport java.io.IOException;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.broker.plugin.BrokerAttachedPlugin;\nimport org.apache.rocketmq.broker.schedule.DelayOffsetSerializeWrapper;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.protocol.BrokerSyncInfo;\nimport org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerOffsetSerializeWrapper;\nimport org.apache.rocketmq.store.config.StorePathConfigHelper;\nimport org.apache.rocketmq.store.ha.HAConnectionState;\nimport org.apache.rocketmq.store.ha.HAConnectionStateNotificationRequest;\nimport org.apache.rocketmq.store.timer.TimerCheckpoint;\n\npublic class BrokerPreOnlineService extends ServiceThread {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private final BrokerController brokerController;\n\n    private int waitBrokerIndex = 0;\n\n    public BrokerPreOnlineService(BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    @Override\n    public String getServiceName() {\n        if (this.brokerController != null && this.brokerController.getBrokerConfig().isInBrokerContainer()) {\n            return brokerController.getBrokerIdentity().getIdentifier() + BrokerPreOnlineService.class.getSimpleName();\n        }\n        return BrokerPreOnlineService.class.getSimpleName();\n    }\n\n    @Override\n    public void run() {\n        LOGGER.info(this.getServiceName() + \" service started\");\n\n        while (!this.isStopped()) {\n            if (!this.brokerController.isIsolated()) {\n                LOGGER.info(\"broker {} is online\", this.brokerController.getBrokerConfig().getCanonicalName());\n                break;\n            }\n            try {\n                boolean isSuccess = this.prepareForBrokerOnline();\n                if (!isSuccess) {\n                    this.waitForRunning(1000);\n                } else {\n                    break;\n                }\n            } catch (Exception e) {\n                LOGGER.error(\"Broker preOnline error, \", e);\n            }\n        }\n\n        LOGGER.info(this.getServiceName() + \" service end\");\n    }\n\n    CompletableFuture<Boolean> waitForHaHandshakeComplete(String brokerAddr) {\n        LOGGER.info(\"wait for handshake completion with {}\", brokerAddr);\n        HAConnectionStateNotificationRequest request =\n            new HAConnectionStateNotificationRequest(HAConnectionState.TRANSFER, RemotingHelper.parseHostFromAddress(brokerAddr), true);\n        if (this.brokerController.getMessageStore().getHaService() != null) {\n            this.brokerController.getMessageStore().getHaService().putGroupConnectionStateRequest(request);\n        } else {\n            LOGGER.error(\"HAService is null, maybe broker config is wrong. For example, duplicationEnable is true\");\n            request.getRequestFuture().complete(false);\n        }\n        return request.getRequestFuture();\n    }\n\n    private boolean futureWaitAction(boolean result, BrokerMemberGroup brokerMemberGroup) {\n        if (!result) {\n            LOGGER.error(\"wait for handshake completion failed, HA connection lost\");\n            return false;\n        }\n        if (this.brokerController.getBrokerConfig().getBrokerId() != MixAll.MASTER_ID) {\n            LOGGER.info(\"slave preOnline complete, start service\");\n            long minBrokerId = getMinBrokerId(brokerMemberGroup.getBrokerAddrs());\n            this.brokerController.startService(minBrokerId, brokerMemberGroup.getBrokerAddrs().get(minBrokerId));\n        }\n        return true;\n    }\n\n    private boolean prepareForMasterOnline(BrokerMemberGroup brokerMemberGroup) {\n        List<Long> brokerIdList = new ArrayList<>(brokerMemberGroup.getBrokerAddrs().keySet());\n        Collections.sort(brokerIdList);\n        while (true) {\n            if (waitBrokerIndex >= brokerIdList.size()) {\n                LOGGER.info(\"master preOnline complete, start service\");\n                this.brokerController.startService(MixAll.MASTER_ID, this.brokerController.getBrokerAddr());\n                return true;\n            }\n\n            String brokerAddrToWait = brokerMemberGroup.getBrokerAddrs().get(brokerIdList.get(waitBrokerIndex));\n\n            try {\n                this.brokerController.getBrokerOuterAPI().\n                    sendBrokerHaInfo(brokerAddrToWait, this.brokerController.getHAServerAddr(),\n                        this.brokerController.getMessageStore().getBrokerInitMaxOffset(), this.brokerController.getBrokerAddr());\n            } catch (Exception e) {\n                LOGGER.error(\"send ha address to {} exception, {}\", brokerAddrToWait, e);\n                return false;\n            }\n\n            CompletableFuture<Boolean> haHandshakeFuture = waitForHaHandshakeComplete(brokerAddrToWait)\n                .thenApply(result -> futureWaitAction(result, brokerMemberGroup));\n\n            try {\n                if (!haHandshakeFuture.get()) {\n                    return false;\n                }\n            } catch (Exception e) {\n                LOGGER.error(\"Wait handshake completion exception, {}\", e);\n                return false;\n            }\n\n            if (syncMetadataReverse(brokerAddrToWait)) {\n                waitBrokerIndex++;\n            } else {\n                return false;\n            }\n        }\n    }\n\n    private boolean syncMetadataReverse(String brokerAddr) {\n        try {\n            LOGGER.info(\"Get metadata reverse from {}\", brokerAddr);\n\n            String delayOffset = this.brokerController.getBrokerOuterAPI().getAllDelayOffset(brokerAddr);\n            DelayOffsetSerializeWrapper delayOffsetSerializeWrapper =\n                DelayOffsetSerializeWrapper.fromJson(delayOffset, DelayOffsetSerializeWrapper.class);\n\n            ConsumerOffsetSerializeWrapper consumerOffsetSerializeWrapper = this.brokerController.getBrokerOuterAPI().getAllConsumerOffset(brokerAddr);\n\n            TimerCheckpoint timerCheckpoint = this.brokerController.getBrokerOuterAPI().getTimerCheckPoint(brokerAddr);\n\n            if (null != consumerOffsetSerializeWrapper && brokerController.getConsumerOffsetManager().getDataVersion().compare(consumerOffsetSerializeWrapper.getDataVersion()) <= 0) {\n                LOGGER.info(\"{}'s consumerOffset data version is larger than master broker, {}'s consumerOffset will be used.\", brokerAddr, brokerAddr);\n                this.brokerController.getConsumerOffsetManager().getOffsetTable()\n                    .putAll(consumerOffsetSerializeWrapper.getOffsetTable());\n                this.brokerController.getConsumerOffsetManager().getDataVersion().assignNewOne(consumerOffsetSerializeWrapper.getDataVersion());\n                this.brokerController.getConsumerOffsetManager().persist();\n            }\n\n            if (null != delayOffset && brokerController.getScheduleMessageService().getDataVersion().compare(delayOffsetSerializeWrapper.getDataVersion()) <= 0) {\n                LOGGER.info(\"{}'s scheduleMessageService data version is larger than master broker, {}'s delayOffset will be used.\", brokerAddr, brokerAddr);\n                String fileName =\n                    StorePathConfigHelper.getDelayOffsetStorePath(this.brokerController\n                        .getMessageStoreConfig().getStorePathRootDir());\n                try {\n                    MixAll.string2File(delayOffset, fileName);\n                    this.brokerController.getScheduleMessageService().load();\n                } catch (IOException e) {\n                    LOGGER.error(\"Persist file Exception, {}\", fileName, e);\n                }\n            }\n\n            if (null != this.brokerController.getTimerCheckpoint() && this.brokerController.getTimerCheckpoint().getDataVersion().compare(timerCheckpoint.getDataVersion()) <= 0) {\n                LOGGER.info(\"{}'s timerCheckpoint data version is larger than master broker, {}'s timerCheckpoint will be used.\", brokerAddr, brokerAddr);\n                this.brokerController.getTimerCheckpoint().setLastReadTimeMs(timerCheckpoint.getLastReadTimeMs());\n                this.brokerController.getTimerCheckpoint().setMasterTimerQueueOffset(timerCheckpoint.getMasterTimerQueueOffset());\n                this.brokerController.getTimerCheckpoint().getDataVersion().assignNewOne(timerCheckpoint.getDataVersion());\n                this.brokerController.getTimerCheckpoint().flush();\n            }\n\n            for (BrokerAttachedPlugin brokerAttachedPlugin : brokerController.getBrokerAttachedPlugins()) {\n                if (brokerAttachedPlugin != null) {\n                    brokerAttachedPlugin.syncMetadataReverse(brokerAddr);\n                }\n            }\n\n        } catch (Exception e) {\n            LOGGER.error(\"GetMetadataReverse Failed\", e);\n            return false;\n        }\n\n        return true;\n    }\n\n    private boolean prepareForSlaveOnline(BrokerMemberGroup brokerMemberGroup) {\n        BrokerSyncInfo brokerSyncInfo;\n        try {\n            brokerSyncInfo = this.brokerController.getBrokerOuterAPI()\n                .retrieveBrokerHaInfo(brokerMemberGroup.getBrokerAddrs().get(MixAll.MASTER_ID));\n        } catch (Exception e) {\n            LOGGER.error(\"retrieve master ha info exception, {}\", e);\n            return false;\n        }\n\n        if (this.brokerController.getMessageStore().getMasterFlushedOffset() == 0\n            && this.brokerController.getMessageStoreConfig().isSyncMasterFlushOffsetWhenStartup()) {\n            LOGGER.info(\"Set master flush offset in slave to {}\", brokerSyncInfo.getMasterFlushOffset());\n            this.brokerController.getMessageStore().setMasterFlushedOffset(brokerSyncInfo.getMasterFlushOffset());\n        }\n\n        if (brokerSyncInfo.getMasterHaAddress() != null) {\n            this.brokerController.getMessageStore().updateHaMasterAddress(brokerSyncInfo.getMasterHaAddress());\n            this.brokerController.getMessageStore().updateMasterAddress(brokerSyncInfo.getMasterAddress());\n        } else {\n            LOGGER.info(\"fetch master ha address return null, start service directly\");\n            long minBrokerId = getMinBrokerId(brokerMemberGroup.getBrokerAddrs());\n            this.brokerController.startService(minBrokerId, brokerMemberGroup.getBrokerAddrs().get(minBrokerId));\n            return true;\n        }\n\n        CompletableFuture<Boolean> haHandshakeFuture = waitForHaHandshakeComplete(brokerSyncInfo.getMasterHaAddress())\n            .thenApply(result -> futureWaitAction(result, brokerMemberGroup));\n\n        try {\n            if (!haHandshakeFuture.get()) {\n                return false;\n            }\n        } catch (Exception e) {\n            LOGGER.error(\"Wait handshake completion exception, {}\", e);\n            return false;\n        }\n\n        return true;\n    }\n\n    private boolean prepareForBrokerOnline() {\n        BrokerMemberGroup brokerMemberGroup;\n        try {\n            brokerMemberGroup = this.brokerController.getBrokerOuterAPI().syncBrokerMemberGroup(\n                this.brokerController.getBrokerConfig().getBrokerClusterName(),\n                this.brokerController.getBrokerConfig().getBrokerName(),\n                this.brokerController.getBrokerConfig().isCompatibleWithOldNameSrv());\n        } catch (Exception e) {\n            LOGGER.error(\"syncBrokerMemberGroup from namesrv error, start service failed, will try later, \", e);\n            return false;\n        }\n\n        if (brokerMemberGroup != null && !brokerMemberGroup.getBrokerAddrs().isEmpty()) {\n            long minBrokerId = getMinBrokerId(brokerMemberGroup.getBrokerAddrs());\n\n            if (this.brokerController.getBrokerConfig().getBrokerId() == MixAll.MASTER_ID) {\n                return prepareForMasterOnline(brokerMemberGroup);\n            } else if (minBrokerId == MixAll.MASTER_ID) {\n                return prepareForSlaveOnline(brokerMemberGroup);\n            } else {\n                LOGGER.info(\"no master online, start service directly\");\n                this.brokerController.startService(minBrokerId, brokerMemberGroup.getBrokerAddrs().get(minBrokerId));\n            }\n        } else {\n            LOGGER.info(\"no other broker online, will start service directly\");\n            this.brokerController.startService(this.brokerController.getBrokerConfig().getBrokerId(), this.brokerController.getBrokerAddr());\n        }\n\n        return true;\n    }\n\n    private long getMinBrokerId(Map<Long, String> brokerAddrMap) {\n        Map<Long, String> brokerAddrMapCopy = new HashMap<>(brokerAddrMap);\n        brokerAddrMapCopy.remove(this.brokerController.getBrokerConfig().getBrokerId());\n        if (!brokerAddrMapCopy.isEmpty()) {\n            return Collections.min(brokerAddrMapCopy.keySet());\n        }\n        return this.brokerController.getBrokerConfig().getBrokerId();\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker;\n\nimport java.io.BufferedInputStream;\nimport java.io.File;\nimport java.io.InputStream;\nimport java.nio.file.Files;\nimport java.nio.file.Paths;\nimport java.util.Properties;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.commons.cli.CommandLine;\nimport org.apache.commons.cli.DefaultParser;\nimport org.apache.commons.cli.Option;\nimport org.apache.commons.cli.Options;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.srvutil.ServerUtil;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\n\npublic class BrokerStartup {\n\n    public static Logger log;\n\n    public static void main(String[] args) {\n        start(createBrokerController(args));\n    }\n\n    public static BrokerController start(BrokerController controller) {\n        try {\n            controller.start();\n\n            String tip = String.format(\"The broker[%s, %s] boot success. serializeType=%s\",\n                controller.getBrokerConfig().getBrokerName(), controller.getBrokerAddr(),\n                RemotingCommand.getSerializeTypeConfigInThisServer());\n\n            if (null != controller.getBrokerConfig().getNamesrvAddr()) {\n                tip += \" and name server is \" + controller.getBrokerConfig().getNamesrvAddr();\n            }\n\n            log.info(tip);\n            System.out.printf(\"%s%n\", tip);\n            return controller;\n        } catch (Throwable e) {\n            e.printStackTrace();\n            System.exit(-1);\n        }\n\n        return null;\n    }\n\n    public static void shutdown(final BrokerController controller) {\n        if (null != controller) {\n            controller.shutdown();\n        }\n    }\n\n    public static ConfigContext parseCmdLine(String[] args) throws Exception {\n        Options options = ServerUtil.buildCommandlineOptions(new Options());\n        CommandLine commandLine = ServerUtil.parseCmdLine(\n            \"mqbroker\", args, buildCommandlineOptions(options), new DefaultParser());\n        if (null == commandLine) {\n            System.exit(-1);\n        }\n\n        ConfigContext configContext;\n        String filePath = null;\n        if (commandLine.hasOption('c')) {\n            filePath = commandLine.getOptionValue('c');\n        }\n\n        configContext = configFileToConfigContext(filePath);\n\n        if (commandLine.hasOption('p') && configContext != null) {\n            Logger console = LoggerFactory.getLogger(LoggerName.BROKER_CONSOLE_NAME);\n            MixAll.printObjectProperties(console, configContext.getBrokerConfig());\n            MixAll.printObjectProperties(console, configContext.getNettyServerConfig());\n            MixAll.printObjectProperties(console, configContext.getNettyClientConfig());\n            MixAll.printObjectProperties(console, configContext.getAuthConfig());\n            System.exit(0);\n        } else if (commandLine.hasOption('m') && configContext != null) {\n            Logger console = LoggerFactory.getLogger(LoggerName.BROKER_CONSOLE_NAME);\n            MixAll.printObjectProperties(console, configContext.getBrokerConfig(), true);\n            MixAll.printObjectProperties(console, configContext.getNettyServerConfig(), true);\n            MixAll.printObjectProperties(console, configContext.getNettyClientConfig(), true);\n            MixAll.printObjectProperties(console, configContext.getAuthConfig(), true);\n            System.exit(0);\n        }\n\n        assert configContext != null;\n        MixAll.properties2Object(ServerUtil.commandLine2Properties(commandLine), configContext.getBrokerConfig());\n\n        return configContext;\n    }\n\n    public static ConfigContext configFileToConfigContext(String filePath) throws Exception {\n        SystemConfigFileHelper systemConfigFileHelper = new SystemConfigFileHelper();\n        BrokerConfig brokerConfig = new BrokerConfig();\n        NettyServerConfig nettyServerConfig = new NettyServerConfig();\n        NettyClientConfig nettyClientConfig = new NettyClientConfig();\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        AuthConfig authConfig = new AuthConfig();\n\n        nettyServerConfig.setListenPort(10911);\n        messageStoreConfig.setHaListenPort(0);\n\n        Properties properties = new Properties();\n        if (StringUtils.isNotBlank(filePath)) {\n            systemConfigFileHelper.setFile(filePath);\n            BrokerPathConfigHelper.setBrokerConfigPath(filePath);\n            properties = systemConfigFileHelper.loadConfig();\n        }\n\n        if (properties != null) {\n            properties2SystemEnv(properties);\n            MixAll.properties2Object(properties, brokerConfig);\n            MixAll.properties2Object(properties, nettyServerConfig);\n            MixAll.properties2Object(properties, nettyClientConfig);\n            MixAll.properties2Object(properties, messageStoreConfig);\n            MixAll.properties2Object(properties, authConfig);\n        }\n\n        return new ConfigContext.Builder()\n            .configFilePath(filePath)\n            .properties(properties)\n            .brokerConfig(brokerConfig)\n            .messageStoreConfig(messageStoreConfig)\n            .nettyServerConfig(nettyServerConfig)\n            .nettyClientConfig(nettyClientConfig)\n            .authConfig(authConfig)\n            .build();\n    }\n\n    public static BrokerController buildBrokerController(ConfigContext configContext) {\n        System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION));\n\n        BrokerConfig brokerConfig = configContext.getBrokerConfig();\n        MessageStoreConfig messageStoreConfig = configContext.getMessageStoreConfig();\n        NettyClientConfig nettyClientConfig = configContext.getNettyClientConfig();\n        NettyServerConfig nettyServerConfig = configContext.getNettyServerConfig();\n        AuthConfig authConfig = configContext.getAuthConfig();\n        Properties properties = configContext.getProperties();\n\n        if (null == brokerConfig.getRocketmqHome()) {\n            System.out.printf(\"Please set the %s variable in your environment \" +\n                \"to match the location of the RocketMQ installation\", MixAll.ROCKETMQ_HOME_ENV);\n            System.exit(-2);\n        }\n\n        // Validate namesrvAddr\n        String namesrvAddr = brokerConfig.getNamesrvAddr();\n        if (StringUtils.isNotBlank(namesrvAddr)) {\n            try {\n                String[] addrArray = namesrvAddr.split(\";\");\n                for (String addr : addrArray) {\n                    NetworkUtil.string2SocketAddress(addr);\n                }\n            } catch (Exception e) {\n                System.out.printf(\"The Name Server Address[%s] illegal, please set it as follows, \" +\n                    \"\\\"127.0.0.1:9876;192.168.0.1:9876\\\"%n\", namesrvAddr);\n                System.exit(-3);\n            }\n        }\n\n        // Set broker role according to ha config\n        if (!brokerConfig.isEnableControllerMode()) {\n            switch (messageStoreConfig.getBrokerRole()) {\n                case ASYNC_MASTER:\n                case SYNC_MASTER:\n                    brokerConfig.setBrokerId(MixAll.MASTER_ID);\n                    break;\n                case SLAVE:\n                    if (brokerConfig.getBrokerId() <= MixAll.MASTER_ID) {\n                        System.out.printf(\"Slave's brokerId must be > 0%n\");\n                        System.exit(-3);\n                    }\n                    break;\n                default:\n                    break;\n            }\n        }\n\n        if (messageStoreConfig.isEnableDLegerCommitLog()) {\n            brokerConfig.setBrokerId(-1);\n        }\n\n        if (brokerConfig.isEnableControllerMode() && messageStoreConfig.isEnableDLegerCommitLog()) {\n            System.out.printf(\"The config enableControllerMode and enableDLegerCommitLog cannot both be true.%n\");\n            System.exit(-4);\n        }\n\n        if (messageStoreConfig.getHaListenPort() <= 0) {\n            messageStoreConfig.setHaListenPort(nettyServerConfig.getListenPort() + 1);\n        }\n\n        brokerConfig.setInBrokerContainer(false);\n\n        System.setProperty(\"brokerLogDir\", \"\");\n        if (brokerConfig.isIsolateLogEnable()) {\n            System.setProperty(\"brokerLogDir\", brokerConfig.getBrokerName() + \"_\" + brokerConfig.getBrokerId());\n        }\n        if (brokerConfig.isIsolateLogEnable() && messageStoreConfig.isEnableDLegerCommitLog()) {\n            System.setProperty(\"brokerLogDir\", brokerConfig.getBrokerName() + \"_\" + messageStoreConfig.getdLegerSelfId());\n        }\n\n        log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n        MixAll.printObjectProperties(log, brokerConfig);\n        MixAll.printObjectProperties(log, nettyServerConfig);\n        MixAll.printObjectProperties(log, nettyClientConfig);\n        MixAll.printObjectProperties(log, messageStoreConfig);\n\n        authConfig.setConfigName(brokerConfig.getBrokerName());\n        authConfig.setClusterName(brokerConfig.getBrokerClusterName());\n        authConfig.setAuthConfigPath(messageStoreConfig.getStorePathRootDir() + File.separator + \"config\");\n\n        final BrokerController controller = new BrokerController(\n            brokerConfig, nettyServerConfig, nettyClientConfig, messageStoreConfig, authConfig);\n\n        // Remember all configs to prevent discard\n        controller.getConfiguration().registerConfig(properties);\n\n        controller.setConfigContext(configContext);\n\n        return controller;\n    }\n\n    public static Runnable buildShutdownHook(BrokerController brokerController) {\n        return new Runnable() {\n            private volatile boolean hasShutdown = false;\n            private final AtomicInteger shutdownTimes = new AtomicInteger(0);\n\n            @Override\n            public void run() {\n                synchronized (this) {\n                    log.info(\"Shutdown hook was invoked, {}\", this.shutdownTimes.incrementAndGet());\n                    if (!this.hasShutdown) {\n                        this.hasShutdown = true;\n                        long beginTime = System.currentTimeMillis();\n                        brokerController.shutdown();\n                        long consumingTimeTotal = System.currentTimeMillis() - beginTime;\n                        log.info(\"Shutdown hook over, consuming total time(ms): {}\", consumingTimeTotal);\n                    }\n                }\n            }\n        };\n    }\n\n    public static BrokerController createBrokerController(String[] args) {\n        try {\n            ConfigContext configContext = parseCmdLine(args);\n            BrokerController controller = buildBrokerController(configContext);\n            boolean initResult = controller.initialize();\n            if (!initResult) {\n                controller.shutdown();\n                System.exit(-3);\n            }\n            Runtime.getRuntime().addShutdownHook(new Thread(buildShutdownHook(controller)));\n            return controller;\n        } catch (Throwable e) {\n            e.printStackTrace();\n            System.exit(-1);\n        }\n        return null;\n    }\n\n    private static void properties2SystemEnv(Properties properties) {\n        if (properties == null) {\n            return;\n        }\n        String rmqAddressServerDomain = properties.getProperty(\"rmqAddressServerDomain\", MixAll.WS_DOMAIN_NAME);\n        String rmqAddressServerSubGroup = properties.getProperty(\"rmqAddressServerSubGroup\", MixAll.WS_DOMAIN_SUBGROUP);\n        System.setProperty(\"rocketmq.namesrv.domain\", rmqAddressServerDomain);\n        System.setProperty(\"rocketmq.namesrv.domain.subgroup\", rmqAddressServerSubGroup);\n    }\n\n    private static Options buildCommandlineOptions(final Options options) {\n        Option opt = new Option(\"c\", \"configFile\", true, \"Broker config properties file\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"p\", \"printConfigItem\", false, \"Print all config item\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"m\", \"printImportantConfig\", false, \"Print important config item\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        return options;\n    }\n\n    public static class SystemConfigFileHelper {\n        private static final Logger LOGGER = LoggerFactory.getLogger(SystemConfigFileHelper.class);\n\n        private String file;\n\n        public SystemConfigFileHelper() {\n        }\n\n        public Properties loadConfig() throws Exception {\n            Properties properties = new Properties();\n            try (InputStream in = new BufferedInputStream(Files.newInputStream(Paths.get(file)))) {\n                properties.load(in);\n            }\n            return properties;\n        }\n\n        public void update(Properties properties) throws Exception {\n            LOGGER.error(\"[SystemConfigFileHelper] update no thing.\");\n        }\n\n        public void setFile(String file) {\n            this.file = file;\n        }\n\n        public String getFile() {\n            return file;\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/ConfigContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker;\n\nimport java.util.Properties;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\n\npublic class ConfigContext {\n    private String configFilePath;\n    private Properties properties;\n\n    private BrokerConfig brokerConfig;\n    private NettyServerConfig nettyServerConfig;\n    private NettyClientConfig nettyClientConfig;\n    private MessageStoreConfig messageStoreConfig;\n    private AuthConfig authConfig;\n\n    private ConfigContext(Builder builder) {\n        this.configFilePath = builder.configFilePath;\n        this.properties = builder.properties;\n        this.brokerConfig = builder.brokerConfig;\n        this.nettyServerConfig = builder.nettyServerConfig;\n        this.nettyClientConfig = builder.nettyClientConfig;\n        this.messageStoreConfig = builder.messageStoreConfig;\n        this.authConfig = builder.authConfig;\n    }\n\n    public String getConfigFilePath() {\n        return configFilePath;\n    }\n\n    public Properties getProperties() {\n        return properties;\n    }\n\n    public BrokerConfig getBrokerConfig() {\n        return brokerConfig;\n    }\n\n    public NettyServerConfig getNettyServerConfig() {\n        return nettyServerConfig;\n    }\n\n    public NettyClientConfig getNettyClientConfig() {\n        return nettyClientConfig;\n    }\n\n    public MessageStoreConfig getMessageStoreConfig() {\n        return messageStoreConfig;\n    }\n\n    public AuthConfig getAuthConfig() {\n        return authConfig;\n    }\n\n    public static class Builder {\n        private String configFilePath;\n        private Properties properties;\n\n        private BrokerConfig brokerConfig;\n        private NettyServerConfig nettyServerConfig;\n        private NettyClientConfig nettyClientConfig;\n        private MessageStoreConfig messageStoreConfig;\n        private AuthConfig authConfig;\n\n        public Builder() {\n        }\n\n        public Builder configFilePath(String configFilePath) {\n            this.configFilePath = configFilePath;\n            return this;\n        }\n\n        public Builder properties(Properties properties) {\n            this.properties = properties;\n            return this;\n        }\n\n        public Builder brokerConfig(BrokerConfig brokerConfig) {\n            this.brokerConfig = brokerConfig;\n            return this;\n        }\n\n        public Builder nettyServerConfig(NettyServerConfig nettyServerConfig) {\n            this.nettyServerConfig = nettyServerConfig;\n            return this;\n        }\n\n        public Builder nettyClientConfig(NettyClientConfig nettyClientConfig) {\n            this.nettyClientConfig = nettyClientConfig;\n            return this;\n        }\n\n        public Builder messageStoreConfig(MessageStoreConfig messageStoreConfig) {\n            this.messageStoreConfig = messageStoreConfig;\n            return this;\n        }\n\n        public Builder authConfig(AuthConfig authConfig) {\n            this.authConfig = authConfig;\n            return this;\n        }\n\n        public ConfigContext build() {\n            return new ConfigContext(this);\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/ShutdownHook.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker;\n\npublic interface ShutdownHook {\n    /**\n     * Code to execute before broker shutdown.\n     *\n     * @param controller broker to shutdown\n     */\n    void beforeShutdown(BrokerController controller);\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/auth/converter/AclConverter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.auth.converter;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.stream.Collectors;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.auth.authorization.enums.Decision;\nimport org.apache.rocketmq.auth.authorization.enums.PolicyType;\nimport org.apache.rocketmq.auth.authorization.model.Acl;\nimport org.apache.rocketmq.auth.authorization.model.Environment;\nimport org.apache.rocketmq.auth.authorization.model.Policy;\nimport org.apache.rocketmq.auth.authorization.model.PolicyEntry;\nimport org.apache.rocketmq.auth.authorization.model.Resource;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.remoting.protocol.body.AclInfo;\n\npublic class AclConverter {\n\n    public static Acl convertAcl(AclInfo aclInfo) {\n        if (aclInfo == null) {\n            return null;\n        }\n        Subject subject = Subject.of(aclInfo.getSubject());\n        List<Policy> policies = new ArrayList<>();\n        for (AclInfo.PolicyInfo policy : aclInfo.getPolicies()) {\n            PolicyType policyType = PolicyType.getByName(policy.getPolicyType());\n\n            List<AclInfo.PolicyEntryInfo> entryInfos = policy.getEntries();\n            if (CollectionUtils.isEmpty(entryInfos)) {\n                continue;\n            }\n            List<PolicyEntry> entries = new ArrayList<>();\n            for (AclInfo.PolicyEntryInfo entryInfo : entryInfos) {\n                Resource resource = Resource.of(entryInfo.getResource());\n\n                List<Action> actions = new ArrayList<>();\n                for (String a : entryInfo.getActions()) {\n                    Action action = Action.getByName(a);\n                    if (action == null) {\n                        continue;\n                    }\n                    actions.add(action);\n                }\n\n                Environment environment = new Environment();\n                if (CollectionUtils.isNotEmpty(entryInfo.getSourceIps())) {\n                    environment.setSourceIps(entryInfo.getSourceIps());\n                }\n\n                Decision decision = Decision.getByName(entryInfo.getDecision());\n\n                entries.add(PolicyEntry.of(resource, actions, environment, decision));\n            }\n\n            policies.add(Policy.of(policyType, entries));\n        }\n\n        return Acl.of(subject, policies);\n    }\n\n    public static List<AclInfo> convertAcls(List<Acl> acls) {\n        if (CollectionUtils.isEmpty(acls)) {\n            return null;\n        }\n        return acls.stream().map(AclConverter::convertAcl)\n            .collect(Collectors.toList());\n    }\n\n    public static AclInfo convertAcl(Acl acl) {\n        if (acl == null) {\n            return null;\n        }\n        AclInfo aclInfo = new AclInfo();\n        aclInfo.setSubject(acl.getSubject().getSubjectKey());\n        if (CollectionUtils.isEmpty(acl.getPolicies())) {\n            return aclInfo;\n        }\n        List<AclInfo.PolicyInfo> policyInfos = acl.getPolicies().stream()\n            .map(AclConverter::convertPolicy)\n            .collect(Collectors.toList());\n        aclInfo.setPolicies(policyInfos);\n        return aclInfo;\n    }\n\n    private static AclInfo.PolicyInfo convertPolicy(Policy policy) {\n        AclInfo.PolicyInfo policyInfo = new AclInfo.PolicyInfo();\n        if (policy.getPolicyType() != null) {\n            policyInfo.setPolicyType(policy.getPolicyType().getName());\n        }\n        if (CollectionUtils.isEmpty(policy.getEntries())) {\n            return policyInfo;\n        }\n        List<AclInfo.PolicyEntryInfo> entryInfos = policy.getEntries().stream()\n            .map(AclConverter::convertPolicyEntry).collect(Collectors.toList());\n        policyInfo.setEntries(entryInfos);\n        return policyInfo;\n    }\n\n    private static AclInfo.PolicyEntryInfo convertPolicyEntry(PolicyEntry entry) {\n        AclInfo.PolicyEntryInfo entryInfo = new AclInfo.PolicyEntryInfo();\n        entryInfo.setResource(entry.toResourceStr());\n        entryInfo.setActions(entry.toActionsStr());\n        if (entry.getEnvironment() != null) {\n            entryInfo.setSourceIps(entry.getEnvironment().getSourceIps());\n        }\n        entryInfo.setDecision(entry.getDecision().getName());\n        return entryInfo;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/auth/converter/UserConverter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.auth.converter;\n\nimport java.util.List;\nimport java.util.stream.Collectors;\nimport org.apache.rocketmq.auth.authentication.enums.UserStatus;\nimport org.apache.rocketmq.auth.authentication.enums.UserType;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.remoting.protocol.body.UserInfo;\n\npublic class UserConverter {\n\n    public static List<UserInfo> convertUsers(List<User> users) {\n        return users.stream().map(UserConverter::convertUser)\n            .collect(Collectors.toList());\n    }\n\n    public static UserInfo convertUser(User user) {\n        UserInfo result = new UserInfo();\n        result.setUsername(user.getUsername());\n        result.setPassword(user.getPassword());\n        if (user.getUserType() != null) {\n            result.setUserType(user.getUserType().getName());\n        }\n        if (user.getUserStatus() != null) {\n            result.setUserStatus(user.getUserStatus().getName());\n        }\n        return result;\n    }\n\n    public static User convertUser(UserInfo userInfo) {\n        User result = new User();\n        result.setUsername(userInfo.getUsername());\n        result.setPassword(userInfo.getPassword());\n        result.setUserType(UserType.getByName(userInfo.getUserType()));\n        result.setUserStatus(UserStatus.getByName(userInfo.getUserStatus()));\n        return result;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/auth/pipeline/AuthenticationPipeline.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.auth.pipeline;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.auth.authentication.AuthenticationEvaluator;\nimport org.apache.rocketmq.auth.authentication.context.AuthenticationContext;\nimport org.apache.rocketmq.auth.authentication.exception.AuthenticationException;\nimport org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.AbortProcessException;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.pipeline.RequestPipeline;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\n\npublic class AuthenticationPipeline implements RequestPipeline {\n    protected static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private final AuthConfig authConfig;\n    private final AuthenticationEvaluator evaluator;\n\n    public AuthenticationPipeline(AuthConfig authConfig) {\n        this.authConfig = authConfig;\n        this.evaluator = AuthenticationFactory.getEvaluator(authConfig);\n    }\n\n    @Override\n    public void execute(ChannelHandlerContext ctx, RemotingCommand request) throws Exception {\n        if (!authConfig.isAuthenticationEnabled()) {\n            return;\n        }\n        try {\n            AuthenticationContext authenticationContext = newContext(ctx, request);\n            evaluator.evaluate(authenticationContext);\n        } catch (AuthenticationException ex) {\n            throw new AbortProcessException(ResponseCode.NO_PERMISSION, ex.getMessage());\n        } catch (Throwable ex) {\n            LOGGER.error(\"authenticate failed, request:{}\", request, ex);\n            throw ex;\n        }\n    }\n\n    protected AuthenticationContext newContext(ChannelHandlerContext ctx, RemotingCommand request) {\n        return AuthenticationFactory.newContext(authConfig, ctx, request);\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/auth/pipeline/AuthorizationPipeline.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.auth.pipeline;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.List;\nimport org.apache.rocketmq.auth.authentication.exception.AuthenticationException;\nimport org.apache.rocketmq.auth.authorization.AuthorizationEvaluator;\nimport org.apache.rocketmq.auth.authorization.context.AuthorizationContext;\nimport org.apache.rocketmq.auth.authorization.exception.AuthorizationException;\nimport org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.AbortProcessException;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.pipeline.RequestPipeline;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\n\npublic class AuthorizationPipeline implements RequestPipeline {\n    protected static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private final AuthConfig authConfig;\n    private final AuthorizationEvaluator evaluator;\n\n    public AuthorizationPipeline(AuthConfig authConfig) {\n        this.authConfig = authConfig;\n        this.evaluator = AuthorizationFactory.getEvaluator(authConfig);\n    }\n\n    @Override\n    public void execute(ChannelHandlerContext ctx, RemotingCommand request) throws Exception {\n        if (!authConfig.isAuthorizationEnabled()) {\n            return;\n        }\n        try {\n            List<AuthorizationContext> contexts = newContexts(ctx, request);\n            evaluator.evaluate(contexts);\n        } catch (AuthorizationException | AuthenticationException ex) {\n            throw new AbortProcessException(ResponseCode.NO_PERMISSION, ex.getMessage());\n        }  catch (Throwable ex) {\n            LOGGER.error(\"authorization failed, request:{}\", request, ex);\n            throw ex;\n        }\n    }\n\n    protected List<AuthorizationContext> newContexts(ChannelHandlerContext ctx, RemotingCommand request) {\n        return AuthorizationFactory.newContexts(authConfig, ctx, request);\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/client/ClientChannelAttributeHelper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.client;\n\nimport io.netty.channel.Channel;\nimport io.netty.util.AttributeKey;\n\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.List;\n\npublic class ClientChannelAttributeHelper {\n    private static final AttributeKey<String> ATTR_CG = AttributeKey.valueOf(\"CHANNEL_CONSUMER_GROUP\");\n    private static final AttributeKey<String> ATTR_PG = AttributeKey.valueOf(\"CHANNEL_PRODUCER_GROUP\");\n    private static final String SEPARATOR = \"|\";\n\n    public static void addProducerGroup(Channel channel, String group) {\n        addGroup(channel, group, ATTR_PG);\n    }\n\n    public static void addConsumerGroup(Channel channel, String group) {\n        addGroup(channel, group, ATTR_CG);\n    }\n\n    public static List<String> getProducerGroups(Channel channel) {\n        return getGroups(channel, ATTR_PG);\n    }\n\n    public static List<String> getConsumerGroups(Channel channel) {\n        return getGroups(channel, ATTR_CG);\n    }\n\n    private static void addGroup(Channel channel, String group, AttributeKey<String> key) {\n        if (null == channel || !channel.isActive()) {  // no side effect if check active status.\n            return;\n        }\n        if (null == group || group.length() == 0 || null == key) {\n            return;\n        }\n        String groups = channel.attr(key).get();\n        if (null == groups) {\n            channel.attr(key).set(group + SEPARATOR);\n        } else {\n            if (groups.contains(SEPARATOR + group + SEPARATOR)) {\n                return;\n            } else {\n                channel.attr(key).compareAndSet(groups, groups + group + SEPARATOR);\n            }\n        }\n    }\n\n    private static List<String> getGroups(Channel channel, AttributeKey<String> key) {\n        if (null == channel) {\n            return Collections.emptyList();\n        }\n        if (null == key) {\n            return Collections.emptyList();\n        }\n        String groups = channel.attr(key).get();\n        return null == groups ? Collections.<String>emptyList() : Arrays.asList(groups.split(\"\\\\|\"));\n    }\n\n}"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/client/ClientChannelInfo.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.client;\n\nimport io.netty.channel.Channel;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\n\npublic class ClientChannelInfo {\n    private final Channel channel;\n    private final String clientId;\n    private final LanguageCode language;\n    private final int version;\n    private volatile long lastUpdateTimestamp = System.currentTimeMillis();\n\n    public ClientChannelInfo(Channel channel) {\n        this(channel, null, null, 0);\n    }\n\n    public ClientChannelInfo(Channel channel, String clientId, LanguageCode language, int version) {\n        this.channel = channel;\n        this.clientId = clientId;\n        this.language = language;\n        this.version = version;\n    }\n\n    public Channel getChannel() {\n        return channel;\n    }\n\n    public String getClientId() {\n        return clientId;\n    }\n\n    public LanguageCode getLanguage() {\n        return language;\n    }\n\n    public int getVersion() {\n        return version;\n    }\n\n    public long getLastUpdateTimestamp() {\n        return lastUpdateTimestamp;\n    }\n\n    public void setLastUpdateTimestamp(long lastUpdateTimestamp) {\n        this.lastUpdateTimestamp = lastUpdateTimestamp;\n    }\n\n    @Override\n    public int hashCode() {\n        final int prime = 31;\n        int result = 1;\n        result = prime * result + ((channel == null) ? 0 : channel.hashCode());\n        result = prime * result + ((clientId == null) ? 0 : clientId.hashCode());\n        result = prime * result + ((language == null) ? 0 : language.hashCode());\n        result = prime * result + (int) (lastUpdateTimestamp ^ (lastUpdateTimestamp >>> 32));\n        result = prime * result + version;\n        return result;\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj)\n            return true;\n        if (obj == null)\n            return false;\n        if (getClass() != obj.getClass())\n            return false;\n        ClientChannelInfo other = (ClientChannelInfo) obj;\n        if (channel == null) {\n            if (other.channel != null)\n                return false;\n        } else if (this.channel != other.channel) {\n            return false;\n        }\n\n        return true;\n    }\n\n    @Override\n    public String toString() {\n        return \"ClientChannelInfo [channel=\" + channel + \", clientId=\" + clientId + \", language=\" + language\n            + \", version=\" + version + \", lastUpdateTimestamp=\" + lastUpdateTimestamp + \"]\";\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.client;\n\nimport io.netty.channel.Channel;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.ChannelEventListener;\n\npublic class ClientHousekeepingService implements ChannelEventListener {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private final BrokerController brokerController;\n\n    private ScheduledExecutorService scheduledExecutorService;\n\n    public ClientHousekeepingService(final BrokerController brokerController) {\n        this.brokerController = brokerController;\n        scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1,\n            new ThreadFactoryImpl(\"ClientHousekeepingScheduledThread\", brokerController.getBrokerIdentity()));\n    }\n\n    public void start() {\n\n        this.scheduledExecutorService.scheduleAtFixedRate(() -> {\n            try {\n                ClientHousekeepingService.this.scanExceptionChannel();\n            } catch (Throwable e) {\n                log.error(\"Error occurred when scan not active client channels.\", e);\n            }\n        }, 1000 * 10, 1000 * 10, TimeUnit.MILLISECONDS);\n    }\n\n    private void scanExceptionChannel() {\n        this.brokerController.getProducerManager().scanNotActiveChannel();\n        this.brokerController.getConsumerManager().scanNotActiveChannel();\n    }\n\n    public void shutdown() {\n        this.scheduledExecutorService.shutdown();\n    }\n\n    @Override\n    public void onChannelConnect(String remoteAddr, Channel channel) {\n        this.brokerController.getBrokerStatsManager().incChannelConnectNum();\n    }\n\n    @Override\n    public void onChannelClose(String remoteAddr, Channel channel) {\n        this.brokerController.getProducerManager().doChannelCloseEvent(remoteAddr, channel);\n        this.brokerController.getConsumerManager().doChannelCloseEvent(remoteAddr, channel);\n        this.brokerController.getBrokerStatsManager().incChannelCloseNum();\n    }\n\n    @Override\n    public void onChannelException(String remoteAddr, Channel channel) {\n        this.brokerController.getProducerManager().doChannelCloseEvent(remoteAddr, channel);\n        this.brokerController.getConsumerManager().doChannelCloseEvent(remoteAddr, channel);\n        this.brokerController.getBrokerStatsManager().incChannelExceptionNum();\n    }\n\n    @Override\n    public void onChannelIdle(String remoteAddr, Channel channel) {\n        this.brokerController.getProducerManager().doChannelCloseEvent(remoteAddr, channel);\n        this.brokerController.getConsumerManager().doChannelCloseEvent(remoteAddr, channel);\n        this.brokerController.getBrokerStatsManager().incChannelIdleNum();\n    }\n\n    @Override\n    public void onChannelActive(String remoteAddr, Channel channel) {\n\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupEvent.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.client;\n\npublic enum ConsumerGroupEvent {\n\n    /**\n     * Some consumers in the group are changed.\n     */\n    CHANGE,\n    /**\n     * The group of consumer is unregistered.\n     */\n    UNREGISTER,\n    /**\n     * The group of consumer is registered.\n     */\n    REGISTER,\n    /**\n     * The client of this consumer is new registered.\n     */\n    CLIENT_REGISTER,\n    /**\n     * The client of this consumer is unregistered.\n     */\n    CLIENT_UNREGISTER\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.client;\n\nimport io.netty.channel.Channel;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map.Entry;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\npublic class ConsumerGroupInfo {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private final String groupName;\n    private final ConcurrentMap<String/* Topic */, SubscriptionData> subscriptionTable =\n        new ConcurrentHashMap<>();\n    private final ConcurrentMap<Channel, ClientChannelInfo> channelInfoTable =\n        new ConcurrentHashMap<>(16);\n    private volatile ConsumeType consumeType;\n    private volatile MessageModel messageModel;\n    private volatile ConsumeFromWhere consumeFromWhere;\n    private volatile long lastUpdateTimestamp = System.currentTimeMillis();\n\n    public ConsumerGroupInfo(String groupName, ConsumeType consumeType, MessageModel messageModel,\n        ConsumeFromWhere consumeFromWhere) {\n        this.groupName = groupName;\n        this.consumeType = consumeType;\n        this.messageModel = messageModel;\n        this.consumeFromWhere = consumeFromWhere;\n    }\n\n    public ConsumerGroupInfo(String groupName) {\n        this.groupName = groupName;\n    }\n\n    public ClientChannelInfo findChannel(final String clientId) {\n        Iterator<Entry<Channel, ClientChannelInfo>> it = this.channelInfoTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<Channel, ClientChannelInfo> next = it.next();\n            if (next.getValue().getClientId().equals(clientId)) {\n                return next.getValue();\n            }\n        }\n\n        return null;\n    }\n\n    public ConcurrentMap<String, SubscriptionData> getSubscriptionTable() {\n        return subscriptionTable;\n    }\n\n    public ClientChannelInfo findChannel(final Channel channel) {\n        return this.channelInfoTable.get(channel);\n    }\n\n    public ConcurrentMap<Channel, ClientChannelInfo> getChannelInfoTable() {\n        return channelInfoTable;\n    }\n\n    public List<Channel> getAllChannel() {\n        List<Channel> result = new ArrayList<>();\n\n        result.addAll(this.channelInfoTable.keySet());\n\n        return result;\n    }\n\n    public List<String> getAllClientId() {\n        List<String> result = new ArrayList<>();\n\n        Iterator<Entry<Channel, ClientChannelInfo>> it = this.channelInfoTable.entrySet().iterator();\n\n        while (it.hasNext()) {\n            Entry<Channel, ClientChannelInfo> entry = it.next();\n            ClientChannelInfo clientChannelInfo = entry.getValue();\n            result.add(clientChannelInfo.getClientId());\n        }\n\n        return result;\n    }\n\n    public boolean unregisterChannel(final ClientChannelInfo clientChannelInfo) {\n        ClientChannelInfo old = this.channelInfoTable.remove(clientChannelInfo.getChannel());\n        if (old != null) {\n            log.info(\"unregister a consumer[{}] from consumerGroupInfo {}\", this.groupName, old.toString());\n            return true;\n        }\n        return false;\n    }\n\n    public ClientChannelInfo doChannelCloseEvent(final String remoteAddr, final Channel channel) {\n        final ClientChannelInfo info = this.channelInfoTable.remove(channel);\n        if (info != null) {\n            log.warn(\n                \"NETTY EVENT: remove not active channel[{}] from ConsumerGroupInfo groupChannelTable, consumer group: {}\",\n                info.toString(), groupName);\n        }\n\n        return info;\n    }\n\n    /**\n     * Update {@link #channelInfoTable} in {@link ConsumerGroupInfo}\n     *\n     * @param infoNew Channel info of new client.\n     * @param consumeType consume type of new client.\n     * @param messageModel message consuming model (CLUSTERING/BROADCASTING) of new client.\n     * @param consumeFromWhere indicate the position when the client consume message firstly.\n     * @return the result that if new connector is connected or not.\n     */\n    public boolean updateChannel(final ClientChannelInfo infoNew, ConsumeType consumeType,\n        MessageModel messageModel, ConsumeFromWhere consumeFromWhere) {\n        boolean updated = false;\n        this.consumeType = consumeType;\n        this.messageModel = messageModel;\n        this.consumeFromWhere = consumeFromWhere;\n\n        ClientChannelInfo infoOld = this.channelInfoTable.get(infoNew.getChannel());\n        if (null == infoOld) {\n            ClientChannelInfo prev = this.channelInfoTable.put(infoNew.getChannel(), infoNew);\n            if (null == prev) {\n                log.info(\"new consumer connected, group: {} {} {} channel: {}\", this.groupName, consumeType,\n                    messageModel, infoNew.toString());\n                updated = true;\n            }\n\n            infoOld = infoNew;\n        } else {\n            if (!infoOld.getClientId().equals(infoNew.getClientId())) {\n                log.error(\n                    \"ConsumerGroupInfo: consumer channel exists in broker, but clientId is not the same one, \"\n                        + \"group={}, old clientChannelInfo={}, new clientChannelInfo={}\", groupName, infoOld.toString(),\n                    infoNew.toString());\n                this.channelInfoTable.put(infoNew.getChannel(), infoNew);\n            }\n        }\n\n        this.lastUpdateTimestamp = System.currentTimeMillis();\n        infoOld.setLastUpdateTimestamp(this.lastUpdateTimestamp);\n\n        return updated;\n    }\n\n    /**\n     * Update subscription.\n     *\n     * @param subList set of {@link SubscriptionData}\n     * @return the boolean indicates the subscription has changed or not.\n     */\n    public boolean updateSubscription(final Set<SubscriptionData> subList) {\n        boolean updated = false;\n        Set<String> topicSet = new HashSet<>();\n        for (SubscriptionData sub : subList) {\n            SubscriptionData old = this.subscriptionTable.get(sub.getTopic());\n            if (old == null) {\n                SubscriptionData prev = this.subscriptionTable.putIfAbsent(sub.getTopic(), sub);\n                if (null == prev) {\n                    updated = true;\n                    log.info(\"subscription changed, add new topic, group: {} {}\",\n                        this.groupName,\n                        sub.toString());\n                }\n            } else if (sub.getSubVersion() > old.getSubVersion()) {\n                if (this.consumeType == ConsumeType.CONSUME_PASSIVELY) {\n                    log.info(\"subscription changed, group: {} OLD: {} NEW: {}\",\n                        this.groupName,\n                        old.toString(),\n                        sub.toString()\n                    );\n                }\n\n                this.subscriptionTable.put(sub.getTopic(), sub);\n            }\n            // Add all new topics to the HashSet\n            topicSet.add(sub.getTopic());\n        }\n\n        Iterator<Entry<String, SubscriptionData>> it = this.subscriptionTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, SubscriptionData> next = it.next();\n            String oldTopic = next.getKey();\n            // Check HashSet with O(1) time complexity\n            if (!topicSet.contains(oldTopic)) {\n                log.warn(\"subscription changed, group: {} remove topic {} {}\",\n                    this.groupName,\n                    oldTopic,\n                    next.getValue().toString()\n                );\n\n                it.remove();\n                updated = true;\n            }\n        }\n\n        this.lastUpdateTimestamp = System.currentTimeMillis();\n\n        return updated;\n    }\n\n    public Set<String> getSubscribeTopics() {\n        return subscriptionTable.keySet();\n    }\n\n    public SubscriptionData findSubscriptionData(final String topic) {\n        return this.subscriptionTable.get(topic);\n    }\n\n    public ConsumeType getConsumeType() {\n        return consumeType;\n    }\n\n    public void setConsumeType(ConsumeType consumeType) {\n        this.consumeType = consumeType;\n    }\n\n    public MessageModel getMessageModel() {\n        return messageModel;\n    }\n\n    public void setMessageModel(MessageModel messageModel) {\n        this.messageModel = messageModel;\n    }\n\n    public String getGroupName() {\n        return groupName;\n    }\n\n    public long getLastUpdateTimestamp() {\n        return lastUpdateTimestamp;\n    }\n\n    public void setLastUpdateTimestamp(long lastUpdateTimestamp) {\n        this.lastUpdateTimestamp = lastUpdateTimestamp;\n    }\n\n    public ConsumeFromWhere getConsumeFromWhere() {\n        return consumeFromWhere;\n    }\n\n    public void setConsumeFromWhere(ConsumeFromWhere consumeFromWhere) {\n        this.consumeFromWhere = consumeFromWhere;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerIdsChangeListener.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.client;\n\npublic interface ConsumerIdsChangeListener {\n\n    void handle(ConsumerGroupEvent event, String group, Object... args);\n\n    void shutdown();\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.client;\n\nimport io.netty.channel.Channel;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map.Entry;\nimport java.util.Optional;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.CopyOnWriteArrayList;\nimport java.util.stream.Collectors;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\n\npublic class ConsumerManager {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private final ConcurrentMap<String, ConsumerGroupInfo> consumerTable =\n        new ConcurrentHashMap<>(1024);\n    private final ConcurrentMap<String, Set<String>> topicGroupTable =\n            new ConcurrentHashMap<>(1024);\n    private final ConcurrentMap<String, ConsumerGroupInfo> consumerCompensationTable =\n        new ConcurrentHashMap<>(1024);\n    private final List<ConsumerIdsChangeListener> consumerIdsChangeListenerList = new CopyOnWriteArrayList<>();\n    protected final BrokerStatsManager brokerStatsManager;\n    private final long channelExpiredTimeout;\n    private final long subscriptionExpiredTimeout;\n    private final BrokerConfig brokerConfig;\n\n    public ConsumerManager(final ConsumerIdsChangeListener consumerIdsChangeListener, long expiredTimeout) {\n        this.consumerIdsChangeListenerList.add(consumerIdsChangeListener);\n        this.brokerStatsManager = null;\n        this.channelExpiredTimeout = expiredTimeout;\n        this.subscriptionExpiredTimeout = expiredTimeout;\n        this.brokerConfig = null;\n    }\n\n    public ConsumerManager(final ConsumerIdsChangeListener consumerIdsChangeListener,\n        final BrokerStatsManager brokerStatsManager, BrokerConfig brokerConfig) {\n        this.consumerIdsChangeListenerList.add(consumerIdsChangeListener);\n        this.brokerStatsManager = brokerStatsManager;\n        this.channelExpiredTimeout = brokerConfig.getChannelExpiredTimeout();\n        this.subscriptionExpiredTimeout = brokerConfig.getSubscriptionExpiredTimeout();\n        this.brokerConfig = brokerConfig;\n    }\n\n    public ClientChannelInfo findChannel(final String group, final String clientId) {\n        ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group);\n        if (consumerGroupInfo != null) {\n            return consumerGroupInfo.findChannel(clientId);\n        }\n        return null;\n    }\n\n    public ClientChannelInfo findChannel(final String group, final Channel channel) {\n        ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group);\n        if (consumerGroupInfo != null) {\n            return consumerGroupInfo.findChannel(channel);\n        }\n        return null;\n    }\n\n    public SubscriptionData findSubscriptionData(final String group, final String topic) {\n        return findSubscriptionData(group, topic, true);\n    }\n\n    public SubscriptionData findSubscriptionData(final String group, final String topic,\n        boolean fromCompensationTable) {\n        ConsumerGroupInfo consumerGroupInfo = getConsumerGroupInfo(group, false);\n        if (consumerGroupInfo != null) {\n            SubscriptionData subscriptionData = consumerGroupInfo.findSubscriptionData(topic);\n            if (subscriptionData != null) {\n                return subscriptionData;\n            }\n        }\n\n        if (fromCompensationTable) {\n            ConsumerGroupInfo consumerGroupCompensationInfo = consumerCompensationTable.get(group);\n            if (consumerGroupCompensationInfo != null) {\n                return consumerGroupCompensationInfo.findSubscriptionData(topic);\n            }\n        }\n        return null;\n    }\n\n    public ConcurrentMap<String, ConsumerGroupInfo> getConsumerTable() {\n        return this.consumerTable;\n    }\n\n    public ConsumerGroupInfo getConsumerGroupInfo(final String group) {\n        return getConsumerGroupInfo(group, false);\n    }\n\n    public ConsumerGroupInfo getConsumerGroupInfo(String group, boolean fromCompensationTable) {\n        ConsumerGroupInfo consumerGroupInfo = consumerTable.get(group);\n        if (consumerGroupInfo == null && fromCompensationTable) {\n            consumerGroupInfo = consumerCompensationTable.get(group);\n        }\n        return consumerGroupInfo;\n    }\n\n    public int findSubscriptionDataCount(final String group) {\n        ConsumerGroupInfo consumerGroupInfo = this.getConsumerGroupInfo(group);\n        if (consumerGroupInfo != null) {\n            return consumerGroupInfo.getSubscriptionTable().size();\n        }\n\n        return 0;\n    }\n\n    public boolean doChannelCloseEvent(final String remoteAddr, final Channel channel) {\n        boolean removed = false;\n        if (this.brokerConfig != null && this.brokerConfig.isEnableFastChannelEventProcess()) {\n            List<String> groups = ClientChannelAttributeHelper.getConsumerGroups(channel);\n            if (this.brokerConfig.isPrintChannelGroups() && groups.size() >= 5 && groups.size() >= this.brokerConfig.getPrintChannelGroupsMinNum()) {\n                LOGGER.warn(\"channel close event, too many consumer groups one channel, {}, {}, {}\", groups.size(), remoteAddr, groups);\n            }\n            for (String group : groups) {\n                if (null == group || group.length() == 0) {\n                    continue;\n                }\n                ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group);\n                if (null == consumerGroupInfo) {\n                    continue;\n                }\n                ClientChannelInfo clientChannelInfo = consumerGroupInfo.doChannelCloseEvent(remoteAddr, channel);\n                if (clientChannelInfo != null) {\n                    removed = true;\n                    callConsumerIdsChangeListener(ConsumerGroupEvent.CLIENT_UNREGISTER, group, clientChannelInfo, consumerGroupInfo.getSubscribeTopics());\n                    if (consumerGroupInfo.getChannelInfoTable().isEmpty()) {\n                        ConsumerGroupInfo remove = this.consumerTable.remove(group);\n                        if (remove != null) {\n                            LOGGER.info(\"unregister consumer ok, no any connection, and remove consumer group, {}\",\n                                    group);\n                            callConsumerIdsChangeListener(ConsumerGroupEvent.UNREGISTER, group);\n                            clearTopicGroupTable(remove);\n                        }\n                    }\n                    callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel());\n                }\n            }\n            return removed;\n        }\n        Iterator<Entry<String, ConsumerGroupInfo>> it = this.consumerTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, ConsumerGroupInfo> next = it.next();\n            ConsumerGroupInfo info = next.getValue();\n            ClientChannelInfo clientChannelInfo = info.doChannelCloseEvent(remoteAddr, channel);\n            if (clientChannelInfo != null) {\n                removed = true;\n                callConsumerIdsChangeListener(ConsumerGroupEvent.CLIENT_UNREGISTER, next.getKey(), clientChannelInfo, info.getSubscribeTopics());\n                if (info.getChannelInfoTable().isEmpty()) {\n                    ConsumerGroupInfo remove = this.consumerTable.remove(next.getKey());\n                    if (remove != null) {\n                        LOGGER.info(\"unregister consumer ok, no any connection, and remove consumer group, {}\",\n                            next.getKey());\n                        callConsumerIdsChangeListener(ConsumerGroupEvent.UNREGISTER, next.getKey());\n                        clearTopicGroupTable(remove);\n                    }\n                }\n                if (!isBroadcastMode(info.getMessageModel())) {\n                    callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, next.getKey(), info.getAllChannel());\n                }\n            }\n        }\n        return removed;\n    }\n\n    private void clearTopicGroupTable(final ConsumerGroupInfo groupInfo) {\n        for (String subscribeTopic : groupInfo.getSubscribeTopics()) {\n            Set<String> groups = this.topicGroupTable.get(subscribeTopic);\n            if (groups != null) {\n                groups.remove(groupInfo.getGroupName());\n            }\n            if (groups != null && groups.isEmpty()) {\n                this.topicGroupTable.remove(subscribeTopic);\n            }\n        }\n    }\n\n    // compensate consumer info for consumer without heartbeat\n    public void compensateBasicConsumerInfo(String group, ConsumeType consumeType, MessageModel messageModel) {\n        ConsumerGroupInfo consumerGroupInfo = consumerCompensationTable.computeIfAbsent(group, ConsumerGroupInfo::new);\n        consumerGroupInfo.setConsumeType(consumeType);\n        consumerGroupInfo.setMessageModel(messageModel);\n    }\n\n    // compensate subscription for pull consumer and consumer via proxy\n    public void compensateSubscribeData(String group, String topic, SubscriptionData subscriptionData) {\n        ConsumerGroupInfo consumerGroupInfo = consumerCompensationTable.computeIfAbsent(group, ConsumerGroupInfo::new);\n        consumerGroupInfo.getSubscriptionTable().put(topic, subscriptionData);\n    }\n\n    public boolean registerConsumer(final String group, final ClientChannelInfo clientChannelInfo,\n        ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,\n        final Set<SubscriptionData> subList, boolean isNotifyConsumerIdsChangedEnable) {\n        return registerConsumer(group, clientChannelInfo, consumeType, messageModel, consumeFromWhere, subList,\n            isNotifyConsumerIdsChangedEnable, true);\n    }\n\n    public boolean registerConsumer(final String group, final ClientChannelInfo clientChannelInfo,\n        ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,\n        final Set<SubscriptionData> subList, boolean isNotifyConsumerIdsChangedEnable, boolean updateSubscription) {\n        long start = System.currentTimeMillis();\n        ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group);\n        if (null == consumerGroupInfo) {\n            ConsumerGroupInfo tmp = new ConsumerGroupInfo(group, consumeType, messageModel, consumeFromWhere);\n            ConsumerGroupInfo prev = this.consumerTable.putIfAbsent(group, tmp);\n            consumerGroupInfo = prev != null ? prev : tmp;\n        }\n\n        for (SubscriptionData subscriptionData : subList) {\n            Set<String> groups = this.topicGroupTable.get(subscriptionData.getTopic());\n            if (groups == null) {\n                Set<String> tmp = new HashSet<>();\n                Set<String> prev = this.topicGroupTable.putIfAbsent(subscriptionData.getTopic(), tmp);\n                groups = prev != null ? prev : tmp;\n            }\n            groups.add(group);\n        }\n\n        boolean r1 =\n            consumerGroupInfo.updateChannel(clientChannelInfo, consumeType, messageModel,\n                consumeFromWhere);\n        if (r1) {\n            callConsumerIdsChangeListener(ConsumerGroupEvent.CLIENT_REGISTER, group, clientChannelInfo,\n                subList.stream().map(SubscriptionData::getTopic).collect(Collectors.toSet()));\n        }\n        boolean r2 = false;\n        if (updateSubscription) {\n            r2 = consumerGroupInfo.updateSubscription(subList);\n        }\n\n        if (r1 || r2) {\n            if (isNotifyConsumerIdsChangedEnable && !isBroadcastMode(consumerGroupInfo.getMessageModel())) {\n                callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel());\n            }\n        }\n\n        if (this.brokerConfig != null && this.brokerConfig.isEnableFastChannelEventProcess() && r1) {\n            ClientChannelAttributeHelper.addConsumerGroup(clientChannelInfo.getChannel(), group);\n        }\n\n        if (null != this.brokerStatsManager) {\n            this.brokerStatsManager.incConsumerRegisterTime((int) (System.currentTimeMillis() - start));\n        }\n\n        callConsumerIdsChangeListener(ConsumerGroupEvent.REGISTER, group, subList, clientChannelInfo);\n\n        return r1 || r2;\n    }\n\n    public boolean registerConsumerWithoutSub(final String group, final ClientChannelInfo clientChannelInfo,\n        ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere, boolean isNotifyConsumerIdsChangedEnable) {\n        long start = System.currentTimeMillis();\n        ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group);\n        if (null == consumerGroupInfo) {\n            ConsumerGroupInfo tmp = new ConsumerGroupInfo(group, consumeType, messageModel, consumeFromWhere);\n            ConsumerGroupInfo prev = this.consumerTable.putIfAbsent(group, tmp);\n            consumerGroupInfo = prev != null ? prev : tmp;\n        }\n\n        for (SubscriptionData subscriptionData : consumerGroupInfo.getSubscriptionTable().values()) {\n            Set<String> groups = this.topicGroupTable.get(subscriptionData.getTopic());\n            if (groups == null) {\n                Set<String> tmp = new HashSet<>();\n                Set<String> prev = this.topicGroupTable.putIfAbsent(subscriptionData.getTopic(), tmp);\n                groups = prev != null ? prev : tmp;\n            }\n            groups.add(group);\n        }\n\n        boolean updateChannelRst = consumerGroupInfo.updateChannel(clientChannelInfo, consumeType, messageModel, consumeFromWhere);\n        if (updateChannelRst && isNotifyConsumerIdsChangedEnable && !isBroadcastMode(consumerGroupInfo.getMessageModel())) {\n            callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel());\n        }\n        if (null != this.brokerStatsManager) {\n            this.brokerStatsManager.incConsumerRegisterTime((int) (System.currentTimeMillis() - start));\n        }\n        return updateChannelRst;\n    }\n\n    public void unregisterConsumer(final String group, final ClientChannelInfo clientChannelInfo,\n        boolean isNotifyConsumerIdsChangedEnable) {\n        ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group);\n        if (null != consumerGroupInfo) {\n            boolean removed = consumerGroupInfo.unregisterChannel(clientChannelInfo);\n            if (removed) {\n                callConsumerIdsChangeListener(ConsumerGroupEvent.CLIENT_UNREGISTER, group, clientChannelInfo, consumerGroupInfo.getSubscribeTopics());\n            }\n            if (consumerGroupInfo.getChannelInfoTable().isEmpty()) {\n                ConsumerGroupInfo remove = this.consumerTable.remove(group);\n                if (remove != null) {\n                    LOGGER.info(\"unregister consumer ok, no any connection, and remove consumer group, {}\", group);\n\n                    callConsumerIdsChangeListener(ConsumerGroupEvent.UNREGISTER, group);\n                    clearTopicGroupTable(remove);\n                }\n            }\n            if (isNotifyConsumerIdsChangedEnable && !isBroadcastMode(consumerGroupInfo.getMessageModel())) {\n                callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel());\n            }\n        }\n    }\n\n    public void removeExpireConsumerGroupInfo() {\n        List<String> removeList = new ArrayList<>();\n        consumerCompensationTable.forEach((group, consumerGroupInfo) -> {\n            List<String> removeTopicList = new ArrayList<>();\n            ConcurrentMap<String, SubscriptionData> subscriptionTable = consumerGroupInfo.getSubscriptionTable();\n            subscriptionTable.forEach((topic, subscriptionData) -> {\n                long diff = System.currentTimeMillis() - subscriptionData.getSubVersion();\n                if (diff > subscriptionExpiredTimeout) {\n                    removeTopicList.add(topic);\n                }\n            });\n            for (String topic : removeTopicList) {\n                subscriptionTable.remove(topic);\n                if (subscriptionTable.isEmpty()) {\n                    removeList.add(group);\n                }\n            }\n        });\n        for (String group : removeList) {\n            consumerCompensationTable.remove(group);\n        }\n    }\n\n    public void scanNotActiveChannel() {\n        Iterator<Entry<String, ConsumerGroupInfo>> it = this.consumerTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, ConsumerGroupInfo> next = it.next();\n            String group = next.getKey();\n            ConsumerGroupInfo consumerGroupInfo = next.getValue();\n            ConcurrentMap<Channel, ClientChannelInfo> channelInfoTable =\n                consumerGroupInfo.getChannelInfoTable();\n\n            Iterator<Entry<Channel, ClientChannelInfo>> itChannel = channelInfoTable.entrySet().iterator();\n            while (itChannel.hasNext()) {\n                Entry<Channel, ClientChannelInfo> nextChannel = itChannel.next();\n                ClientChannelInfo clientChannelInfo = nextChannel.getValue();\n                long diff = System.currentTimeMillis() - clientChannelInfo.getLastUpdateTimestamp();\n                if (diff > channelExpiredTimeout) {\n                    LOGGER.warn(\n                        \"SCAN: remove expired channel from ConsumerManager consumerTable. channel={}, consumerGroup={}\",\n                        RemotingHelper.parseChannelRemoteAddr(clientChannelInfo.getChannel()), group);\n                    callConsumerIdsChangeListener(ConsumerGroupEvent.CLIENT_UNREGISTER, group, clientChannelInfo, consumerGroupInfo.getSubscribeTopics());\n                    RemotingHelper.closeChannel(clientChannelInfo.getChannel());\n                    itChannel.remove();\n                }\n            }\n\n            if (channelInfoTable.isEmpty()) {\n                LOGGER.warn(\n                    \"SCAN: remove expired channel from ConsumerManager consumerTable, all clear, consumerGroup={}\",\n                    group);\n                it.remove();\n            }\n        }\n        removeExpireConsumerGroupInfo();\n    }\n\n    public HashSet<String> queryTopicConsumeByWho(final String topic) {\n        return new HashSet<>(Optional.ofNullable(topicGroupTable.get(topic)).orElseGet(HashSet::new));\n    }\n\n    public void appendConsumerIdsChangeListener(ConsumerIdsChangeListener listener) {\n        consumerIdsChangeListenerList.add(listener);\n    }\n\n    protected void callConsumerIdsChangeListener(ConsumerGroupEvent event, String group, Object... args) {\n        for (ConsumerIdsChangeListener listener : consumerIdsChangeListenerList) {\n            try {\n                listener.handle(event, group, args);\n            } catch (Throwable t) {\n                LOGGER.error(\"err when call consumerIdsChangeListener\", t);\n            }\n        }\n    }\n\n    private boolean isBroadcastMode(final MessageModel messageModel) {\n        return MessageModel.BROADCASTING.equals(messageModel);\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.client;\n\nimport io.netty.channel.Channel;\nimport java.util.Collection;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicBoolean;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\npublic class DefaultConsumerIdsChangeListener implements ConsumerIdsChangeListener {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private final BrokerController brokerController;\n    private final int cacheSize = 8096;\n\n    private final ScheduledExecutorService scheduledExecutorService =  ThreadUtils.newScheduledThreadPool(1,\n        ThreadUtils.newGenericThreadFactory(\"DefaultConsumerIdsChangeListener\", true));\n\n    private ConcurrentHashMap<String,List<Channel>> consumerChannelMap = new ConcurrentHashMap<>(cacheSize);\n\n    private final ConcurrentHashMap<String, NotifyTaskControl> activeGroupNotifyMap = new ConcurrentHashMap<>();\n\n    public DefaultConsumerIdsChangeListener(BrokerController brokerController) {\n        this.brokerController = brokerController;\n\n        scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    notifyConsumerChange();\n                } catch (Exception e) {\n                    log.error(\n                        \"DefaultConsumerIdsChangeListen#notifyConsumerChange: unexpected error occurs\", e);\n                }\n            }\n        }, 30, 15, TimeUnit.SECONDS);\n    }\n\n    @Override\n    public void handle(ConsumerGroupEvent event, String group, Object... args) {\n        if (event == null) {\n            return;\n        }\n        switch (event) {\n            case CHANGE:\n                if (args == null || args.length < 1) {\n                    return;\n                }\n                List<Channel> channels = (List<Channel>) args[0];\n                if (channels != null && brokerController.getBrokerConfig().isNotifyConsumerIdsChangedEnable()) {\n                    if (this.brokerController.getBrokerConfig().isRealTimeNotifyConsumerChange()) {\n                        NotifyTaskControl currentNotifyTaskControl = new NotifyTaskControl(channels);\n                        activeGroupNotifyMap.compute(group, (k, oldVal) -> {\n                            if (null != oldVal) {\n                                oldVal.interrupt();\n                            }\n                            return currentNotifyTaskControl;\n                        });\n\n                        boolean isNormalCompletion = true;\n                        for (Channel chl : currentNotifyTaskControl.getChannels()) {\n                            if (currentNotifyTaskControl.isInterrupted()) {\n                                isNormalCompletion = false;\n                                break;\n                            }\n                            this.brokerController.getBroker2Client().notifyConsumerIdsChanged(chl, group);\n                        }\n                        if (isNormalCompletion) {\n                            activeGroupNotifyMap.computeIfPresent(group, (k, val) -> val == currentNotifyTaskControl ? null : val);\n                        }\n                    } else {\n                        consumerChannelMap.put(group, channels);\n                    }\n                }\n                break;\n            case UNREGISTER:\n                this.brokerController.getConsumerFilterManager().unRegister(group);\n                break;\n            case REGISTER:\n                if (args == null || args.length < 1) {\n                    return;\n                }\n                Collection<SubscriptionData> subscriptionDataList = (Collection<SubscriptionData>) args[0];\n                this.brokerController.getConsumerFilterManager().register(group, subscriptionDataList);\n                break;\n            case CLIENT_REGISTER:\n            case CLIENT_UNREGISTER:\n                break;\n            default:\n                throw new RuntimeException(\"Unknown event \" + event);\n        }\n    }\n\n    private void notifyConsumerChange() {\n\n        if (consumerChannelMap.isEmpty()) {\n            return;\n        }\n\n        ConcurrentHashMap<String, List<Channel>> processMap = new ConcurrentHashMap<>(consumerChannelMap);\n        consumerChannelMap = new ConcurrentHashMap<>(cacheSize);\n\n        for (Map.Entry<String, List<Channel>> entry : processMap.entrySet()) {\n            String consumerId = entry.getKey();\n            List<Channel> channelList = entry.getValue();\n            try {\n                if (channelList != null && brokerController.getBrokerConfig().isNotifyConsumerIdsChangedEnable()) {\n                    for (Channel chl : channelList) {\n                        this.brokerController.getBroker2Client().notifyConsumerIdsChanged(chl, consumerId);\n                    }\n                }\n            } catch (Exception e) {\n                log.error(\"Failed to notify consumer when some consumers changed, consumerId to notify: {}\",\n                    consumerId, e);\n            }\n        }\n    }\n\n    @Override\n    public void shutdown() {\n        this.scheduledExecutorService.shutdown();\n    }\n\n    private static class NotifyTaskControl {\n\n        private final AtomicBoolean interrupted = new AtomicBoolean(false);\n\n        private final List<Channel> channels;\n\n        public NotifyTaskControl(List<Channel> channels) {\n            this.channels = channels;\n        }\n\n        public boolean isInterrupted() {\n            return interrupted.get();\n        }\n\n        public void interrupt() {\n            interrupted.set(true);\n        }\n\n        public List<Channel> getChannels() {\n            return channels;\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/client/ProducerChangeListener.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.client;\n\n/**\n * producer manager will call this listener when something happen\n * <p>\n * event type: {@link ProducerGroupEvent}\n */\npublic interface ProducerChangeListener {\n\n    void handle(ProducerGroupEvent event, String group, ClientChannelInfo clientChannelInfo);\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/client/ProducerGroupEvent.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.client;\n\npublic enum ProducerGroupEvent {\n    /**\n     * The group of producer is unregistered.\n     */\n    GROUP_UNREGISTER,\n    /**\n     * The client of this producer is unregistered.\n     */\n    CLIENT_UNREGISTER\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.client;\n\nimport io.netty.channel.Channel;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Map.Entry;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.CopyOnWriteArrayList;\nimport org.apache.rocketmq.broker.util.PositiveAtomicCounter;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.protocol.body.ProducerInfo;\nimport org.apache.rocketmq.remoting.protocol.body.ProducerTableInfo;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\n\npublic class ProducerManager {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private static final long CHANNEL_EXPIRED_TIMEOUT = 1000 * 120;\n    private static final int GET_AVAILABLE_CHANNEL_RETRY_COUNT = 3;\n    private final ConcurrentMap<String /* group name */, ConcurrentMap<Channel, ClientChannelInfo>> groupChannelTable =\n        new ConcurrentHashMap<>();\n    private final ConcurrentMap<String, Channel> clientChannelTable = new ConcurrentHashMap<>();\n    protected final BrokerStatsManager brokerStatsManager;\n    private final BrokerConfig brokerConfig;\n    private final PositiveAtomicCounter positiveAtomicCounter = new PositiveAtomicCounter();\n    private final List<ProducerChangeListener> producerChangeListenerList = new CopyOnWriteArrayList<>();\n\n    public ProducerManager() {\n        this.brokerStatsManager = null;\n        this.brokerConfig = null;\n    }\n\n    public ProducerManager(final BrokerStatsManager brokerStatsManager) {\n        this.brokerStatsManager = brokerStatsManager;\n        this.brokerConfig = null;\n    }\n\n    public ProducerManager(final BrokerStatsManager brokerStatsManager, final BrokerConfig brokerConfig) {\n        this.brokerStatsManager = brokerStatsManager;\n        this.brokerConfig = brokerConfig;\n    }\n\n    public int groupSize() {\n        return this.groupChannelTable.size();\n    }\n\n    public boolean groupOnline(String group) {\n        Map<Channel, ClientChannelInfo> channels = this.groupChannelTable.get(group);\n        return channels != null && !channels.isEmpty();\n    }\n\n    public ConcurrentMap<String, ConcurrentMap<Channel, ClientChannelInfo>> getGroupChannelTable() {\n        return groupChannelTable;\n    }\n\n    public ProducerTableInfo getProducerTable() {\n        Map<String, List<ProducerInfo>> map = new HashMap<>();\n        for (String group : this.groupChannelTable.keySet()) {\n            for (Entry<Channel, ClientChannelInfo> entry : this.groupChannelTable.get(group).entrySet()) {\n                ClientChannelInfo clientChannelInfo = entry.getValue();\n                if (map.containsKey(group)) {\n                    map.get(group).add(new ProducerInfo(\n                        clientChannelInfo.getClientId(),\n                        clientChannelInfo.getChannel().remoteAddress().toString(),\n                        clientChannelInfo.getLanguage(),\n                        clientChannelInfo.getVersion(),\n                        clientChannelInfo.getLastUpdateTimestamp()\n                    ));\n                } else {\n                    map.put(group, new ArrayList<>(Collections.singleton(new ProducerInfo(\n                        clientChannelInfo.getClientId(),\n                        clientChannelInfo.getChannel().remoteAddress().toString(),\n                        clientChannelInfo.getLanguage(),\n                        clientChannelInfo.getVersion(),\n                        clientChannelInfo.getLastUpdateTimestamp()\n                    ))));\n                }\n            }\n        }\n        return new ProducerTableInfo(map);\n    }\n\n    public void scanNotActiveChannel() {\n        Iterator<Map.Entry<String, ConcurrentMap<Channel, ClientChannelInfo>>> iterator = this.groupChannelTable.entrySet().iterator();\n\n        while (iterator.hasNext()) {\n            Map.Entry<String, ConcurrentMap<Channel, ClientChannelInfo>> entry = iterator.next();\n\n            final String group = entry.getKey();\n            final ConcurrentMap<Channel, ClientChannelInfo> chlMap = entry.getValue();\n\n            Iterator<Entry<Channel, ClientChannelInfo>> it = chlMap.entrySet().iterator();\n            while (it.hasNext()) {\n                Entry<Channel, ClientChannelInfo> item = it.next();\n                // final Integer id = item.getKey();\n                final ClientChannelInfo info = item.getValue();\n\n                long diff = System.currentTimeMillis() - info.getLastUpdateTimestamp();\n                if (diff > CHANNEL_EXPIRED_TIMEOUT) {\n                    it.remove();\n                    Channel channelInClientTable = clientChannelTable.get(info.getClientId());\n                    if (channelInClientTable != null && channelInClientTable.equals(info.getChannel())) {\n                        clientChannelTable.remove(info.getClientId());\n                    }\n                    log.warn(\n                        \"ProducerManager#scanNotActiveChannel: remove expired channel[{}] from ProducerManager groupChannelTable, producer group name: {}\",\n                        RemotingHelper.parseChannelRemoteAddr(info.getChannel()), group);\n                    callProducerChangeListener(ProducerGroupEvent.CLIENT_UNREGISTER, group, info);\n                    RemotingHelper.closeChannel(info.getChannel());\n                }\n            }\n\n            if (chlMap.isEmpty()) {\n                log.warn(\"SCAN: remove expired channel from ProducerManager groupChannelTable, all clear, group={}\", group);\n                iterator.remove();\n                callProducerChangeListener(ProducerGroupEvent.GROUP_UNREGISTER, group, null);\n            }\n        }\n    }\n\n    public boolean doChannelCloseEvent(final String remoteAddr, final Channel channel) {\n        boolean removed = false;\n        if (channel != null) {\n            if (this.brokerConfig != null && this.brokerConfig.isEnableFastChannelEventProcess()) {\n                List<String> groups = ClientChannelAttributeHelper.getProducerGroups(channel);\n                if (this.brokerConfig.isPrintChannelGroups() && groups.size() >= 5 && groups.size() >= this.brokerConfig.getPrintChannelGroupsMinNum()) {\n                    log.warn(\"channel close event, too many producer groups one channel, {}, {}, {}\", groups.size(), remoteAddr, groups);\n                }\n                for (String group : groups) {\n                    if (null == group || group.length() == 0) {\n                        continue;\n                    }\n                    ConcurrentMap<Channel, ClientChannelInfo> clientChannelInfoTable = this.groupChannelTable.get(group);\n                    if (null == clientChannelInfoTable) {\n                        continue;\n                    }\n                    final ClientChannelInfo clientChannelInfo =\n                            clientChannelInfoTable.remove(channel);\n                    if (clientChannelInfo != null) {\n                        clientChannelTable.remove(clientChannelInfo.getClientId());\n                        removed = true;\n                        log.info(\n                                \"NETTY EVENT: remove channel[{}][{}] from ProducerManager groupChannelTable, producer group: {}\",\n                                clientChannelInfo.toString(), remoteAddr, group);\n                        callProducerChangeListener(ProducerGroupEvent.CLIENT_UNREGISTER, group, clientChannelInfo);\n                        if (clientChannelInfoTable.isEmpty()) {\n                            ConcurrentMap<Channel, ClientChannelInfo> oldGroupTable = this.groupChannelTable.remove(group);\n                            if (oldGroupTable != null) {\n                                log.info(\"unregister a producer group[{}] from groupChannelTable\", group);\n                                callProducerChangeListener(ProducerGroupEvent.GROUP_UNREGISTER, group, null);\n                            }\n                        }\n                    }\n                }\n                return removed;  // must return here, degrade to scanNotActiveChannel at worst.\n            }\n            for (final Map.Entry<String, ConcurrentMap<Channel, ClientChannelInfo>> entry : this.groupChannelTable.entrySet()) {\n                final String group = entry.getKey();\n                final ConcurrentMap<Channel, ClientChannelInfo> clientChannelInfoTable = entry.getValue();\n                final ClientChannelInfo clientChannelInfo = clientChannelInfoTable.remove(channel);\n                if (clientChannelInfo != null) {\n                    clientChannelTable.remove(clientChannelInfo.getClientId());\n                    removed = true;\n                    log.info(\n                        \"NETTY EVENT: remove channel[{}][{}] from ProducerManager groupChannelTable, producer group: {}\",\n                        clientChannelInfo.toString(), remoteAddr, group);\n                    callProducerChangeListener(ProducerGroupEvent.CLIENT_UNREGISTER, group, clientChannelInfo);\n                    if (clientChannelInfoTable.isEmpty()) {\n                        ConcurrentMap<Channel, ClientChannelInfo> oldGroupTable = this.groupChannelTable.remove(group);\n                        if (oldGroupTable != null) {\n                            log.info(\"unregister a producer group[{}] from groupChannelTable\", group);\n                            callProducerChangeListener(ProducerGroupEvent.GROUP_UNREGISTER, group, null);\n                        }\n                    }\n                }\n\n            }\n        }\n        return removed;\n    }\n\n    public void registerProducer(final String group, final ClientChannelInfo clientChannelInfo) {\n\n        long start = System.currentTimeMillis();\n        ClientChannelInfo clientChannelInfoFound;\n\n        ConcurrentMap<Channel, ClientChannelInfo> channelTable = this.groupChannelTable.get(group);\n        // note that we must take care of the exist groups and channels,\n        // only can return when groups or channels not exist.\n        if (this.brokerConfig != null\n                && !this.brokerConfig.isEnableRegisterProducer()\n                && this.brokerConfig.isRejectTransactionMessage()) {\n            boolean needRegister = true;\n            if (null == channelTable) {\n                needRegister = false;\n            } else {\n                clientChannelInfoFound = channelTable.get(clientChannelInfo.getChannel());\n                if (null == clientChannelInfoFound) {\n                    needRegister = false;\n                }\n            }\n            if (!needRegister) {\n                if (null != this.brokerStatsManager) {\n                    this.brokerStatsManager.incProducerRegisterTime((int) (System.currentTimeMillis() - start));\n                }\n                return;\n            }\n        }\n\n        if (null == channelTable) {\n            channelTable = new ConcurrentHashMap<>();\n            ConcurrentMap<Channel, ClientChannelInfo> prev = this.groupChannelTable.putIfAbsent(group, channelTable);\n            channelTable = prev != null ? prev : channelTable;\n        }\n\n        clientChannelInfoFound = channelTable.get(clientChannelInfo.getChannel());\n        // Add client-channel info to existing producer group\n        if (null == clientChannelInfoFound) {\n            channelTable.put(clientChannelInfo.getChannel(), clientChannelInfo);\n            clientChannelTable.put(clientChannelInfo.getClientId(), clientChannelInfo.getChannel());\n            log.info(\"new producer connected, group: {} channel: {}\", group, clientChannelInfo.toString());\n            if (this.brokerConfig != null && this.brokerConfig.isEnableFastChannelEventProcess()) {\n                ClientChannelAttributeHelper.addProducerGroup(clientChannelInfo.getChannel(), group);\n            }\n        }\n\n        // Refresh existing client-channel-info update-timestamp\n        if (clientChannelInfoFound != null) {\n            clientChannelInfoFound.setLastUpdateTimestamp(System.currentTimeMillis());\n        }\n\n        if (null != this.brokerStatsManager) {\n            this.brokerStatsManager.incProducerRegisterTime((int) (System.currentTimeMillis() - start));\n        }\n    }\n\n    public void unregisterProducer(final String group, final ClientChannelInfo clientChannelInfo) {\n        ConcurrentMap<Channel, ClientChannelInfo> channelTable = this.groupChannelTable.get(group);\n        if (null != channelTable && !channelTable.isEmpty()) {\n            ClientChannelInfo old = channelTable.remove(clientChannelInfo.getChannel());\n            clientChannelTable.remove(clientChannelInfo.getClientId());\n            if (old != null) {\n                log.info(\"unregister a producer[{}] from groupChannelTable {}\", group, clientChannelInfo.toString());\n                callProducerChangeListener(ProducerGroupEvent.CLIENT_UNREGISTER, group, clientChannelInfo);\n            }\n\n            if (channelTable.isEmpty()) {\n                this.groupChannelTable.remove(group);\n                callProducerChangeListener(ProducerGroupEvent.GROUP_UNREGISTER, group, null);\n                log.info(\"unregister a producer group[{}] from groupChannelTable\", group);\n            }\n        }\n    }\n\n    public Channel getAvailableChannel(String groupId) {\n        if (groupId == null) {\n            return null;\n        }\n        List<Channel> channelList;\n        ConcurrentMap<Channel, ClientChannelInfo> channelClientChannelInfoHashMap = groupChannelTable.get(groupId);\n        if (channelClientChannelInfoHashMap != null) {\n            channelList = new ArrayList<>(channelClientChannelInfoHashMap.keySet());\n        } else {\n            log.warn(\"Check transaction failed, channel table is empty. groupId={}\", groupId);\n            return null;\n        }\n\n        int size = channelList.size();\n        if (0 == size) {\n            log.warn(\"Channel list is empty. groupId={}\", groupId);\n            return null;\n        }\n\n        Channel lastActiveChannel = null;\n\n        int index = positiveAtomicCounter.incrementAndGet() % size;\n        Channel channel = channelList.get(index);\n        int count = 0;\n        boolean isOk = channel.isActive() && channel.isWritable();\n        while (count++ < GET_AVAILABLE_CHANNEL_RETRY_COUNT) {\n            if (isOk) {\n                return channel;\n            }\n            if (channel.isActive()) {\n                lastActiveChannel = channel;\n            }\n            index = (++index) % size;\n            channel = channelList.get(index);\n            isOk = channel.isActive() && channel.isWritable();\n        }\n\n        return lastActiveChannel;\n    }\n\n    public Channel findChannel(String clientId) {\n        return clientChannelTable.get(clientId);\n    }\n\n    private void callProducerChangeListener(ProducerGroupEvent event, String group,\n        ClientChannelInfo clientChannelInfo) {\n        for (ProducerChangeListener listener : producerChangeListenerList) {\n            try {\n                listener.handle(event, group, clientChannelInfo);\n            } catch (Throwable t) {\n                log.error(\"err when call producerChangeListener\", t);\n            }\n        }\n    }\n\n    public void appendProducerChangeListener(ProducerChangeListener producerChangeListener) {\n        producerChangeListenerList.add(producerChangeListener);\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.client.net;\n\nimport io.netty.channel.Channel;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Map.Entry;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ConsumerGroupInfo;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.message.MessageQueueForC;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.GetConsumerStatusBody;\nimport org.apache.rocketmq.remoting.protocol.body.ResetOffsetBody;\nimport org.apache.rocketmq.remoting.protocol.body.ResetOffsetBodyForC;\nimport org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerStatusRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.NotifyConsumerIdsChangedRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.NotifyUnsubscribeLiteRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\n\npublic class Broker2Client {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private final BrokerController brokerController;\n\n    public Broker2Client(BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    public void notifyUnsubscribeLite(Channel channel, NotifyUnsubscribeLiteRequestHeader requestHeader) {\n        RemotingCommand request =\n            RemotingCommand.createRequestCommand(RequestCode.NOTIFY_UNSUBSCRIBE_LITE, requestHeader);\n        try {\n            this.brokerController.getRemotingServer().invokeOneway(channel, request, 100);\n        } catch (Exception e) {\n            log.error(\"notifyUnsubscribeLite failed. header={}, error={}\", requestHeader, e.toString());\n        }\n    }\n\n    public void checkProducerTransactionState(\n        final String group,\n        final Channel channel,\n        final CheckTransactionStateRequestHeader requestHeader,\n        final MessageExt messageExt) throws Exception {\n        RemotingCommand request =\n            RemotingCommand.createRequestCommand(RequestCode.CHECK_TRANSACTION_STATE, requestHeader);\n        request.setBody(MessageDecoder.encode(messageExt, false));\n        try {\n            this.brokerController.getRemotingServer().invokeOneway(channel, request, 10);\n        } catch (Exception e) {\n            log.error(\"Check transaction failed because invoke producer exception. group={}, msgId={}, error={}\",\n                    group, messageExt.getMsgId(), e.toString());\n        }\n    }\n\n    public RemotingCommand callClient(final Channel channel,\n        final RemotingCommand request\n    ) throws RemotingSendRequestException, RemotingTimeoutException, InterruptedException {\n        return this.brokerController.getRemotingServer().invokeSync(channel, request, 10000);\n    }\n\n    public void notifyConsumerIdsChanged(\n        final Channel channel,\n        final String consumerGroup) {\n        if (null == consumerGroup) {\n            log.error(\"notifyConsumerIdsChanged consumerGroup is null\");\n            return;\n        }\n\n        NotifyConsumerIdsChangedRequestHeader requestHeader = new NotifyConsumerIdsChangedRequestHeader();\n        requestHeader.setConsumerGroup(consumerGroup);\n        RemotingCommand request =\n            RemotingCommand.createRequestCommand(RequestCode.NOTIFY_CONSUMER_IDS_CHANGED, requestHeader);\n\n        try {\n            this.brokerController.getRemotingServer().invokeOneway(channel, request, 10);\n        } catch (Exception e) {\n            log.error(\"notifyConsumerIdsChanged exception. group={}, error={}\", consumerGroup, e.toString());\n        }\n    }\n\n    public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce) throws RemotingCommandException {\n        return resetOffset(topic, group, timeStamp, isForce, false);\n    }\n\n    public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce,\n        boolean isC) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);\n        if (null == topicConfig) {\n            log.error(\"[reset-offset] reset offset failed, no topic in this broker. topic={}\", topic);\n            response.setCode(ResponseCode.TOPIC_NOT_EXIST);\n            response.setRemark(\"[reset-offset] reset offset failed, no topic in this broker. topic=\" + topic);\n            return response;\n        }\n\n        Map<MessageQueue, Long> offsetTable = new HashMap<>();\n\n        for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) {\n            MessageQueue mq = new MessageQueue();\n            mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());\n            mq.setTopic(topic);\n            mq.setQueueId(i);\n\n            long consumerOffset =\n                this.brokerController.getConsumerOffsetManager().queryOffset(group, topic, i);\n            if (-1 == consumerOffset) {\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(String.format(\"THe consumer group <%s> not exist\", group));\n                return response;\n            }\n\n            long timeStampOffset;\n            if (timeStamp == -1) {\n                try {\n                    timeStampOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);\n                } catch (ConsumeQueueException e) {\n                    throw new RemotingCommandException(\"Failed to get max offset in queue\", e);\n                }\n            } else {\n                timeStampOffset = this.brokerController.getMessageStore().getOffsetInQueueByTime(topic, i, timeStamp);\n            }\n\n            if (timeStampOffset < 0) {\n                log.warn(\"reset offset is invalid. topic={}, queueId={}, timeStampOffset={}\", topic, i, timeStampOffset);\n                timeStampOffset = 0;\n            }\n\n            if (isForce || timeStampOffset < consumerOffset) {\n                offsetTable.put(mq, timeStampOffset);\n            } else {\n                offsetTable.put(mq, consumerOffset);\n            }\n        }\n\n        ResetOffsetRequestHeader requestHeader = new ResetOffsetRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setGroup(group);\n        requestHeader.setTimestamp(timeStamp);\n        RemotingCommand request =\n            RemotingCommand.createRequestCommand(RequestCode.RESET_CONSUMER_CLIENT_OFFSET, requestHeader);\n        if (isC) {\n            // c++ language\n            ResetOffsetBodyForC body = new ResetOffsetBodyForC();\n            List<MessageQueueForC> offsetList = convertOffsetTable2OffsetList(offsetTable);\n            body.setOffsetTable(offsetList);\n            request.setBody(body.encode());\n        } else {\n            // other language\n            ResetOffsetBody body = new ResetOffsetBody();\n            body.setOffsetTable(offsetTable);\n            request.setBody(body.encode());\n        }\n\n        ConsumerGroupInfo consumerGroupInfo =\n            this.brokerController.getConsumerManager().getConsumerGroupInfo(group);\n\n        if (consumerGroupInfo != null && !consumerGroupInfo.getAllChannel().isEmpty()) {\n            ConcurrentMap<Channel, ClientChannelInfo> channelInfoTable =\n                consumerGroupInfo.getChannelInfoTable();\n            for (Map.Entry<Channel, ClientChannelInfo> entry : channelInfoTable.entrySet()) {\n                int version = entry.getValue().getVersion();\n                if (version >= MQVersion.Version.V3_0_7_SNAPSHOT.ordinal()) {\n                    try {\n                        this.brokerController.getRemotingServer().invokeOneway(entry.getKey(), request, 5000);\n                        log.info(\"[reset-offset] reset offset success. topic={}, group={}, clientId={}\",\n                            topic, group, entry.getValue().getClientId());\n                    } catch (Exception e) {\n                        log.error(\"[reset-offset] reset offset exception. topic={}, group={} ,error={}\",\n                            topic, group, e.toString());\n                    }\n                } else {\n                    response.setCode(ResponseCode.SYSTEM_ERROR);\n                    response.setRemark(\"the client does not support this feature. version=\"\n                        + MQVersion.getVersionDesc(version));\n                    log.warn(\"[reset-offset] the client does not support this feature. channel={}, version={}\",\n                        RemotingHelper.parseChannelRemoteAddr(entry.getKey()), MQVersion.getVersionDesc(version));\n                    return response;\n                }\n            }\n        } else {\n            String errorInfo =\n                String.format(\"Consumer not online, so can not reset offset, Group: %s Topic: %s Timestamp: %d\",\n                    requestHeader.getGroup(),\n                    requestHeader.getTopic(),\n                    requestHeader.getTimestamp());\n            log.error(errorInfo);\n            response.setCode(ResponseCode.CONSUMER_NOT_ONLINE);\n            response.setRemark(errorInfo);\n            return response;\n        }\n        response.setCode(ResponseCode.SUCCESS);\n        ResetOffsetBody resBody = new ResetOffsetBody();\n        resBody.setOffsetTable(offsetTable);\n        response.setBody(resBody.encode());\n        return response;\n    }\n\n    private List<MessageQueueForC> convertOffsetTable2OffsetList(Map<MessageQueue, Long> table) {\n        List<MessageQueueForC> list = new ArrayList<>();\n        for (Entry<MessageQueue, Long> entry : table.entrySet()) {\n            MessageQueue mq = entry.getKey();\n            MessageQueueForC tmp =\n                new MessageQueueForC(mq.getTopic(), mq.getBrokerName(), mq.getQueueId(), entry.getValue());\n            list.add(tmp);\n        }\n        return list;\n    }\n\n    public RemotingCommand getConsumeStatus(String topic, String group, String originClientId) {\n        final RemotingCommand result = RemotingCommand.createResponseCommand(null);\n\n        GetConsumerStatusRequestHeader requestHeader = new GetConsumerStatusRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setGroup(group);\n        RemotingCommand request =\n            RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_STATUS_FROM_CLIENT,\n                requestHeader);\n\n        Map<String, Map<MessageQueue, Long>> consumerStatusTable = new HashMap<>();\n        ConcurrentMap<Channel, ClientChannelInfo> channelInfoTable =\n            this.brokerController.getConsumerManager().getConsumerGroupInfo(group).getChannelInfoTable();\n        if (null == channelInfoTable || channelInfoTable.isEmpty()) {\n            result.setCode(ResponseCode.SYSTEM_ERROR);\n            result.setRemark(String.format(\"No Any Consumer online in the consumer group: [%s]\", group));\n            return result;\n        }\n\n        for (Map.Entry<Channel, ClientChannelInfo> entry : channelInfoTable.entrySet()) {\n            int version = entry.getValue().getVersion();\n            String clientId = entry.getValue().getClientId();\n            if (version < MQVersion.Version.V3_0_7_SNAPSHOT.ordinal()) {\n                result.setCode(ResponseCode.SYSTEM_ERROR);\n                result.setRemark(\"the client does not support this feature. version=\"\n                    + MQVersion.getVersionDesc(version));\n                log.warn(\"[get-consumer-status] the client does not support this feature. channel={}, version={}\",\n                    RemotingHelper.parseChannelRemoteAddr(entry.getKey()), MQVersion.getVersionDesc(version));\n                return result;\n            } else if (UtilAll.isBlank(originClientId) || originClientId.equals(clientId)) {\n                try {\n                    RemotingCommand response =\n                        this.brokerController.getRemotingServer().invokeSync(entry.getKey(), request, 5000);\n                    assert response != null;\n                    switch (response.getCode()) {\n                        case ResponseCode.SUCCESS: {\n                            if (response.getBody() != null) {\n                                GetConsumerStatusBody body =\n                                    GetConsumerStatusBody.decode(response.getBody(),\n                                        GetConsumerStatusBody.class);\n\n                                consumerStatusTable.put(clientId, body.getMessageQueueTable());\n                                log.info(\n                                    \"[get-consumer-status] get consumer status success. topic={}, group={}, channelRemoteAddr={}\",\n                                    topic, group, clientId);\n                            }\n                        }\n                        default:\n                            break;\n                    }\n                } catch (Exception e) {\n                    log.error(\n                        \"[get-consumer-status] get consumer status exception. topic={}, group={}, error={}\",\n                        topic, group, e.toString());\n                }\n\n                if (!UtilAll.isBlank(originClientId) && originClientId.equals(clientId)) {\n                    break;\n                }\n            }\n        }\n\n        result.setCode(ResponseCode.SUCCESS);\n        GetConsumerStatusBody resBody = new GetConsumerStatusBody();\n        resBody.setConsumerTable(consumerStatusTable);\n        result.setBody(resBody.encode());\n        return result;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.client.rebalance;\n\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\nimport java.util.HashSet;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.locks.Lock;\nimport java.util.concurrent.locks.ReentrantLock;\n\npublic class RebalanceLockManager {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.REBALANCE_LOCK_LOGGER_NAME);\n    private final static long REBALANCE_LOCK_MAX_LIVE_TIME = Long.parseLong(System.getProperty(\n        \"rocketmq.broker.rebalance.lockMaxLiveTime\", \"60000\"));\n    private final Lock lock = new ReentrantLock();\n    private final ConcurrentMap<String/* group */, ConcurrentHashMap<MessageQueue, LockEntry>> mqLockTable =\n        new ConcurrentHashMap<>(1024);\n\n    public boolean isLockAllExpired(final String group) {\n        final ConcurrentHashMap<MessageQueue, LockEntry> lockEntryMap = mqLockTable.get(group);\n        if (null == lockEntryMap) {\n            return true;\n        }\n        for (LockEntry entry : lockEntryMap.values()) {\n            if (!entry.isExpired()) {\n                return false;\n            }\n        }\n        return true;\n    }\n\n    public boolean tryLock(final String group, final MessageQueue mq, final String clientId) {\n\n        if (!this.isLocked(group, mq, clientId)) {\n            try {\n                this.lock.lockInterruptibly();\n                try {\n                    ConcurrentHashMap<MessageQueue, LockEntry> groupValue = this.mqLockTable.get(group);\n                    if (null == groupValue) {\n                        groupValue = new ConcurrentHashMap<>(32);\n                        this.mqLockTable.put(group, groupValue);\n                    }\n\n                    LockEntry lockEntry = groupValue.get(mq);\n                    if (null == lockEntry) {\n                        lockEntry = new LockEntry();\n                        lockEntry.setClientId(clientId);\n                        groupValue.put(mq, lockEntry);\n                        log.info(\n                            \"RebalanceLockManager#tryLock: lock a message queue which has not been locked yet, \"\n                                + \"group={}, clientId={}, mq={}\", group, clientId, mq);\n                    }\n\n                    if (lockEntry.isLocked(clientId)) {\n                        lockEntry.setLastUpdateTimestamp(System.currentTimeMillis());\n                        return true;\n                    }\n\n                    String oldClientId = lockEntry.getClientId();\n\n                    if (lockEntry.isExpired()) {\n                        lockEntry.setClientId(clientId);\n                        lockEntry.setLastUpdateTimestamp(System.currentTimeMillis());\n                        log.warn(\n                            \"RebalanceLockManager#tryLock: try to lock a expired message queue, group={}, mq={}, old \"\n                                + \"client id={}, new client id={}\", group, mq, oldClientId, clientId);\n                        return true;\n                    }\n\n                    log.warn(\n                        \"RebalanceLockManager#tryLock: message queue has been locked by other client, group={}, \"\n                            + \"mq={}, locked client id={}, current client id={}\", group, mq, oldClientId, clientId);\n                    return false;\n                } finally {\n                    this.lock.unlock();\n                }\n            } catch (InterruptedException e) {\n                log.error(\"RebalanceLockManager#tryLock: unexpected error, group={}, mq={}, clientId={}\", group, mq,\n                    clientId, e);\n            }\n        }\n\n        return true;\n    }\n\n    private boolean isLocked(final String group, final MessageQueue mq, final String clientId) {\n        ConcurrentHashMap<MessageQueue, LockEntry> groupValue = this.mqLockTable.get(group);\n        if (groupValue != null) {\n            LockEntry lockEntry = groupValue.get(mq);\n            if (lockEntry != null) {\n                boolean locked = lockEntry.isLocked(clientId);\n                if (locked) {\n                    lockEntry.setLastUpdateTimestamp(System.currentTimeMillis());\n                }\n\n                return locked;\n            }\n        }\n\n        return false;\n    }\n\n    public Set<MessageQueue> tryLockBatch(final String group, final Set<MessageQueue> mqs,\n        final String clientId) {\n        Set<MessageQueue> lockedMqs = new HashSet<>(mqs.size());\n        Set<MessageQueue> notLockedMqs = new HashSet<>(mqs.size());\n\n        for (MessageQueue mq : mqs) {\n            if (this.isLocked(group, mq, clientId)) {\n                lockedMqs.add(mq);\n            } else {\n                notLockedMqs.add(mq);\n            }\n        }\n\n        if (!notLockedMqs.isEmpty()) {\n            try {\n                this.lock.lockInterruptibly();\n                try {\n                    ConcurrentHashMap<MessageQueue, LockEntry> groupValue = this.mqLockTable.get(group);\n                    if (null == groupValue) {\n                        groupValue = new ConcurrentHashMap<>(32);\n                        this.mqLockTable.put(group, groupValue);\n                    }\n\n                    for (MessageQueue mq : notLockedMqs) {\n                        LockEntry lockEntry = groupValue.get(mq);\n                        if (null == lockEntry) {\n                            lockEntry = new LockEntry();\n                            lockEntry.setClientId(clientId);\n                            groupValue.put(mq, lockEntry);\n                            log.info(\n                                \"RebalanceLockManager#tryLockBatch: lock a message which has not been locked yet, \"\n                                    + \"group={}, clientId={}, mq={}\", group, clientId, mq);\n                        }\n\n                        if (lockEntry.isLocked(clientId)) {\n                            lockEntry.setLastUpdateTimestamp(System.currentTimeMillis());\n                            lockedMqs.add(mq);\n                            continue;\n                        }\n\n                        String oldClientId = lockEntry.getClientId();\n\n                        if (lockEntry.isExpired()) {\n                            lockEntry.setClientId(clientId);\n                            lockEntry.setLastUpdateTimestamp(System.currentTimeMillis());\n                            log.warn(\n                                \"RebalanceLockManager#tryLockBatch: try to lock a expired message queue, group={}, \"\n                                    + \"mq={}, old client id={}, new client id={}\", group, mq, oldClientId, clientId);\n                            lockedMqs.add(mq);\n                            continue;\n                        }\n\n                        log.warn(\n                            \"RebalanceLockManager#tryLockBatch: message queue has been locked by other client, \"\n                                + \"group={}, mq={}, locked client id={}, current client id={}\", group, mq, oldClientId,\n                            clientId);\n                    }\n                } finally {\n                    this.lock.unlock();\n                }\n            } catch (InterruptedException e) {\n                log.error(\"RebalanceLockManager#tryBatch: unexpected error, group={}, mqs={}, clientId={}\", group, mqs,\n                    clientId, e);\n            }\n        }\n\n        return lockedMqs;\n    }\n\n    public void unlockBatch(final String group, final Set<MessageQueue> mqs, final String clientId) {\n        try {\n            this.lock.lockInterruptibly();\n            try {\n                ConcurrentHashMap<MessageQueue, LockEntry> groupValue = this.mqLockTable.get(group);\n                if (null != groupValue) {\n                    for (MessageQueue mq : mqs) {\n                        LockEntry lockEntry = groupValue.get(mq);\n                        if (null != lockEntry) {\n                            if (lockEntry.getClientId().equals(clientId)) {\n                                groupValue.remove(mq);\n                                log.info(\"RebalanceLockManager#unlockBatch: unlock mq, group={}, clientId={}, mqs={}\",\n                                    group, clientId, mq);\n                            } else {\n                                log.warn(\n                                    \"RebalanceLockManager#unlockBatch: mq locked by other client, group={}, locked \"\n                                        + \"clientId={}, current clientId={}, mqs={}\", group, lockEntry.getClientId(),\n                                    clientId, mq);\n                            }\n                        } else {\n                            log.warn(\"RebalanceLockManager#unlockBatch: mq not locked, group={}, clientId={}, mq={}\",\n                                group, clientId, mq);\n                        }\n                    }\n                } else {\n                    log.warn(\"RebalanceLockManager#unlockBatch: group not exist, group={}, clientId={}, mqs={}\", group,\n                        clientId, mqs);\n                }\n            } finally {\n                this.lock.unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"RebalanceLockManager#unlockBatch: unexpected error, group={}, mqs={}, clientId={}\", group, mqs,\n                clientId);\n        }\n    }\n\n    static class LockEntry {\n        private String clientId;\n        private volatile long lastUpdateTimestamp = System.currentTimeMillis();\n\n        public String getClientId() {\n            return clientId;\n        }\n\n        public void setClientId(String clientId) {\n            this.clientId = clientId;\n        }\n\n        public long getLastUpdateTimestamp() {\n            return lastUpdateTimestamp;\n        }\n\n        public void setLastUpdateTimestamp(long lastUpdateTimestamp) {\n            this.lastUpdateTimestamp = lastUpdateTimestamp;\n        }\n\n        public boolean isLocked(final String clientId) {\n            boolean eq = this.clientId.equals(clientId);\n            return eq && !this.isExpired();\n        }\n\n        public boolean isExpired() {\n            boolean expired =\n                (System.currentTimeMillis() - this.lastUpdateTimestamp) > REBALANCE_LOCK_MAX_LIVE_TIME;\n\n            return expired;\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/coldctr/ColdCtrStrategy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.coldctr;\n\npublic interface ColdCtrStrategy {\n    /**\n     * Calculate the determining factor about whether to accelerate or decelerate\n     * @return\n     */\n    Double decisionFactor();\n    /**\n     * Promote the speed for consumerGroup to read cold data\n     * @param consumerGroup\n     * @param currentThreshold\n     */\n    void promote(String consumerGroup, Long currentThreshold);\n    /**\n     * Decelerate the speed for consumerGroup to read cold data\n     * @param consumerGroup\n     * @param currentThreshold\n     */\n    void decelerate(String consumerGroup, Long currentThreshold);\n    /**\n     * Collect the total number of cold read data in the system\n     * @param globalAcc\n     */\n    void collect(Long globalAcc);\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/coldctr/ColdDataCgCtrService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.coldctr;\n\nimport java.util.ArrayList;\nimport java.util.Comparator;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map.Entry;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.atomic.AtomicLong;\n\nimport com.alibaba.fastjson2.JSONObject;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.SystemClock;\nimport org.apache.rocketmq.common.coldctr.AccAndTimeStamp;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\n\n/**\n * store the cg cold read ctr table and acc the size of the cold\n * reading msg, timing to clear the table and set acc to zero\n */\npublic class ColdDataCgCtrService extends ServiceThread {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_COLDCTR_LOGGER_NAME);\n    private final SystemClock systemClock = new SystemClock();\n    private final long cgColdAccResideTimeoutMills = 60 * 1000;\n    private static final AtomicLong GLOBAL_ACC = new AtomicLong(0L);\n    private static final String ADAPTIVE = \"||adaptive\";\n    /**\n     * as soon as the consumerGroup read the cold data then it will be put into @code cgColdThresholdMapRuntime,\n     * and it also will be removed when does not read cold data in @code cgColdAccResideTimeoutMills later;\n     */\n    private final ConcurrentHashMap<String, AccAndTimeStamp> cgColdThresholdMapRuntime = new ConcurrentHashMap<>();\n    /**\n     * if the system admin wants to set the special cold read threshold for some consumerGroup, the configuration will\n     * be putted into @code cgColdThresholdMapConfig\n     */\n    private final ConcurrentHashMap<String, Long> cgColdThresholdMapConfig = new ConcurrentHashMap<>();\n    private final BrokerConfig brokerConfig;\n    private final MessageStoreConfig messageStoreConfig;\n    private final ColdCtrStrategy coldCtrStrategy;\n\n    public ColdDataCgCtrService(BrokerController brokerController) {\n        this.brokerConfig = brokerController.getBrokerConfig();\n        this.messageStoreConfig = brokerController.getMessageStoreConfig();\n        this.coldCtrStrategy = brokerConfig.isUsePIDColdCtrStrategy() ? new PIDAdaptiveColdCtrStrategy(this, (long)(brokerConfig.getGlobalColdReadThreshold() * 0.8)) : new SimpleColdCtrStrategy(this);\n    }\n\n    @Override\n    public String getServiceName() {\n        return ColdDataCgCtrService.class.getSimpleName();\n    }\n\n    @Override\n    public void run() {\n        log.info(\"{} service started\", this.getServiceName());\n        while (!this.isStopped()) {\n            try {\n                if (messageStoreConfig.isColdDataFlowControlEnable()) {\n                    this.waitForRunning(5 * 1000);\n                } else {\n                    this.waitForRunning(180 * 1000);\n                }\n                long beginLockTimestamp = this.systemClock.now();\n                clearDataAcc();\n                if (!brokerConfig.isColdCtrStrategyEnable()) {\n                    clearAdaptiveConfig();\n                }\n                long costTime = this.systemClock.now() - beginLockTimestamp;\n                log.info(\"[{}] clearTheDataAcc-cost {} ms.\", costTime > 3 * 1000 ? \"NOTIFYME\" : \"OK\", costTime);\n            } catch (Throwable e) {\n                log.warn(this.getServiceName() + \" service has exception\", e);\n            }\n        }\n        log.info(\"{} service end\", this.getServiceName());\n    }\n\n    public String getColdDataFlowCtrInfo() {\n        JSONObject result = new JSONObject();\n        result.put(\"runtimeTable\", this.cgColdThresholdMapRuntime);\n        result.put(\"configTable\", this.cgColdThresholdMapConfig);\n        result.put(\"cgColdReadThreshold\", this.brokerConfig.getCgColdReadThreshold());\n        result.put(\"globalColdReadThreshold\", this.brokerConfig.getGlobalColdReadThreshold());\n        result.put(\"globalAcc\", GLOBAL_ACC.get());\n        return result.toJSONString();\n    }\n\n    /**\n     * clear the long time no cold read cg in the table;\n     * update the acc to zero for the cg in the table;\n     * use the strategy to promote or decelerate the cg;\n     */\n    private void clearDataAcc() {\n        log.info(\"clearDataAcc cgColdThresholdMapRuntime key size: {}\", cgColdThresholdMapRuntime.size());\n        if (brokerConfig.isColdCtrStrategyEnable()) {\n            coldCtrStrategy.collect(GLOBAL_ACC.get());\n        }\n        Iterator<Entry<String, AccAndTimeStamp>> iterator = cgColdThresholdMapRuntime.entrySet().iterator();\n        while (iterator.hasNext()) {\n            Entry<String, AccAndTimeStamp> next = iterator.next();\n            if (System.currentTimeMillis() >= cgColdAccResideTimeoutMills + next.getValue().getLastColdReadTimeMills()) {\n                if (brokerConfig.isColdCtrStrategyEnable()) {\n                    cgColdThresholdMapConfig.remove(buildAdaptiveKey(next.getKey()));\n                }\n                iterator.remove();\n            } else if (next.getValue().getColdAcc().get() >= getThresholdByConsumerGroup(next.getKey())) {\n                log.info(\"Coldctr consumerGroup: {}, acc: {}, threshold: {}\", next.getKey(), next.getValue().getColdAcc().get(), getThresholdByConsumerGroup(next.getKey()));\n                if (brokerConfig.isColdCtrStrategyEnable() && !isGlobalColdCtr() && !isAdminConfig(next.getKey())) {\n                    coldCtrStrategy.promote(buildAdaptiveKey(next.getKey()), getThresholdByConsumerGroup(next.getKey()));\n                }\n            }\n            next.getValue().getColdAcc().set(0L);\n        }\n        if (isGlobalColdCtr()) {\n            log.info(\"Coldctr global acc: {}, threshold: {}\", GLOBAL_ACC.get(), this.brokerConfig.getGlobalColdReadThreshold());\n        }\n        if (brokerConfig.isColdCtrStrategyEnable()) {\n            sortAndDecelerate();\n        }\n        GLOBAL_ACC.set(0L);\n    }\n\n    private void sortAndDecelerate() {\n        List<Entry<String, Long>> configMapList = new ArrayList<Entry<String, Long>>(cgColdThresholdMapConfig.entrySet());\n        configMapList.sort(new Comparator<Entry<String, Long>>() {\n            @Override\n            public int compare(Entry<String, Long> o1, Entry<String, Long> o2) {\n                return (int)(o2.getValue() - o1.getValue());\n            }\n        });\n        Iterator<Entry<String, Long>> iterator = configMapList.iterator();\n        int maxDecelerate = 3;\n        while (iterator.hasNext() && maxDecelerate > 0) {\n            Entry<String, Long> next = iterator.next();\n            if (!isAdminConfig(next.getKey())) {\n                coldCtrStrategy.decelerate(next.getKey(), getThresholdByConsumerGroup(next.getKey()));\n                maxDecelerate --;\n            }\n        }\n    }\n\n    public void coldAcc(String consumerGroup, long coldDataToAcc) {\n        if (coldDataToAcc <= 0) {\n            return;\n        }\n        GLOBAL_ACC.addAndGet(coldDataToAcc);\n        AccAndTimeStamp atomicAcc = cgColdThresholdMapRuntime.get(consumerGroup);\n        if (null == atomicAcc) {\n            atomicAcc = new AccAndTimeStamp(new AtomicLong(coldDataToAcc));\n            atomicAcc = cgColdThresholdMapRuntime.putIfAbsent(consumerGroup, atomicAcc);\n        }\n        if (null != atomicAcc) {\n            atomicAcc.getColdAcc().addAndGet(coldDataToAcc);\n            atomicAcc.setLastColdReadTimeMills(System.currentTimeMillis());\n        }\n    }\n\n    public void addOrUpdateGroupConfig(String consumerGroup, Long threshold) {\n        cgColdThresholdMapConfig.put(consumerGroup, threshold);\n    }\n\n    public void removeGroupConfig(String consumerGroup) {\n        cgColdThresholdMapConfig.remove(consumerGroup);\n    }\n\n    public boolean isCgNeedColdDataFlowCtr(String consumerGroup) {\n        if (!this.messageStoreConfig.isColdDataFlowControlEnable()) {\n            return false;\n        }\n        if (MixAll.isSysConsumerGroupPullMessage(consumerGroup)) {\n            return false;\n        }\n        AccAndTimeStamp accAndTimeStamp = cgColdThresholdMapRuntime.get(consumerGroup);\n        if (null == accAndTimeStamp) {\n            return false;\n        }\n\n        Long threshold = getThresholdByConsumerGroup(consumerGroup);\n        if (accAndTimeStamp.getColdAcc().get() >= threshold) {\n            return true;\n        }\n        return GLOBAL_ACC.get() >= this.brokerConfig.getGlobalColdReadThreshold();\n    }\n\n    public boolean isGlobalColdCtr() {\n        return GLOBAL_ACC.get() > this.brokerConfig.getGlobalColdReadThreshold();\n    }\n\n    public BrokerConfig getBrokerConfig() {\n        return brokerConfig;\n    }\n\n    private Long getThresholdByConsumerGroup(String consumerGroup) {\n        if (isAdminConfig(consumerGroup)) {\n            if (consumerGroup.endsWith(ADAPTIVE)) {\n                return cgColdThresholdMapConfig.get(consumerGroup.split(ADAPTIVE)[0]);\n            }\n            return cgColdThresholdMapConfig.get(consumerGroup);\n        }\n        Long threshold = null;\n        if (brokerConfig.isColdCtrStrategyEnable()) {\n            if (consumerGroup.endsWith(ADAPTIVE)) {\n                threshold = cgColdThresholdMapConfig.get(consumerGroup);\n            } else {\n                threshold = cgColdThresholdMapConfig.get(buildAdaptiveKey(consumerGroup));\n            }\n        }\n        if (null == threshold) {\n            threshold = this.brokerConfig.getCgColdReadThreshold();\n        }\n        return threshold;\n    }\n\n    private String buildAdaptiveKey(String consumerGroup) {\n        return consumerGroup + ADAPTIVE;\n    }\n\n    private boolean isAdminConfig(String consumerGroup) {\n        if (consumerGroup.endsWith(ADAPTIVE)) {\n            consumerGroup = consumerGroup.split(ADAPTIVE)[0];\n        }\n        return cgColdThresholdMapConfig.containsKey(consumerGroup);\n    }\n\n    private void clearAdaptiveConfig() {\n        cgColdThresholdMapConfig.entrySet().removeIf(next -> next.getKey().endsWith(ADAPTIVE));\n    }\n\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/coldctr/ColdDataPullRequestHoldService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.coldctr;\n\nimport java.util.Iterator;\nimport java.util.concurrent.LinkedBlockingQueue;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.longpolling.PullRequest;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.SystemClock;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\n/**\n * just requests are type of pull have the qualification to be put into this hold queue.\n * if the pull request is reading cold data and that request will be cold at the first time,\n * then the pull request will be cold in this @code pullRequestLinkedBlockingQueue,\n * in @code coldTimeoutMillis later the pull request will be warm and marked holded\n */\npublic class ColdDataPullRequestHoldService extends ServiceThread {\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_COLDCTR_LOGGER_NAME);\n    public static final String NO_SUSPEND_KEY = \"_noSuspend_\";\n\n    private final long coldHoldTimeoutMillis = 3000;\n    private final SystemClock systemClock = new SystemClock();\n    private final BrokerController brokerController;\n    private final LinkedBlockingQueue<PullRequest> pullRequestColdHoldQueue = new LinkedBlockingQueue<>(10000);\n\n    public void suspendColdDataReadRequest(PullRequest pullRequest) {\n        if (this.brokerController.getMessageStoreConfig().isColdDataFlowControlEnable()) {\n            pullRequestColdHoldQueue.offer(pullRequest);\n        }\n    }\n\n    public ColdDataPullRequestHoldService(BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    @Override\n    public String getServiceName() {\n        return ColdDataPullRequestHoldService.class.getSimpleName();\n    }\n\n    @Override\n    public void run() {\n        log.info(\"{} service started\", this.getServiceName());\n        while (!this.isStopped()) {\n            try {\n                if (!this.brokerController.getMessageStoreConfig().isColdDataFlowControlEnable()) {\n                    this.waitForRunning(20 * 1000);\n                } else {\n                    this.waitForRunning(5 * 1000);\n                }\n                long beginClockTimestamp = this.systemClock.now();\n                this.checkColdDataPullRequest();\n                long costTime = this.systemClock.now() - beginClockTimestamp;\n                log.info(\"[{}] checkColdDataPullRequest-cost {} ms.\", costTime > 5 * 1000 ? \"NOTIFYME\" : \"OK\", costTime);\n            } catch (Throwable e) {\n                log.warn(this.getServiceName() + \" service has exception\", e);\n            }\n        }\n        log.info(\"{} service end\", this.getServiceName());\n    }\n\n    private void checkColdDataPullRequest() {\n        int succTotal = 0, errorTotal = 0, queueSize = pullRequestColdHoldQueue.size() ;\n        Iterator<PullRequest> iterator = pullRequestColdHoldQueue.iterator();\n        while (iterator.hasNext()) {\n            PullRequest pullRequest = iterator.next();\n            if (System.currentTimeMillis() >= pullRequest.getSuspendTimestamp() + coldHoldTimeoutMillis) {\n                try {\n                    pullRequest.getRequestCommand().addExtField(NO_SUSPEND_KEY, \"1\");\n                    this.brokerController.getPullMessageProcessor().executeRequestWhenWakeup(\n                        pullRequest.getClientChannel(), pullRequest.getRequestCommand());\n                    succTotal++;\n                } catch (Exception e) {\n                    log.error(\"PullRequestColdHoldService checkColdDataPullRequest error\", e);\n                    errorTotal++;\n                }\n                //remove the timeout request from the iterator\n                iterator.remove();\n            }\n        }\n        log.info(\"checkColdPullRequest-info-finish, queueSize: {} successTotal: {} errorTotal: {}\",\n            queueSize, succTotal, errorTotal);\n    }\n\n}"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/coldctr/PIDAdaptiveColdCtrStrategy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.coldctr;\n\nimport java.util.ArrayList;\nimport java.util.Iterator;\nimport java.util.List;\n\npublic class PIDAdaptiveColdCtrStrategy implements ColdCtrStrategy {\n    /**\n     * Stores the maximum number of recent et val\n     */\n    private static final int MAX_STORE_NUMS = 10;\n    /**\n     * The weights of the three modules of the PID formula\n     */\n    private static final Double KP = 0.5, KI = 0.3, KD = 0.2;\n    private final List<Long> historyEtValList = new ArrayList<>();\n    private final ColdDataCgCtrService coldDataCgCtrService;\n    private final Long expectGlobalVal;\n    private long et = 0L;\n\n    public PIDAdaptiveColdCtrStrategy(ColdDataCgCtrService coldDataCgCtrService, Long expectGlobalVal) {\n        this.coldDataCgCtrService = coldDataCgCtrService;\n        this.expectGlobalVal = expectGlobalVal;\n    }\n\n    @Override\n    public Double decisionFactor() {\n        if (historyEtValList.size() < MAX_STORE_NUMS) {\n            return 0.0;\n        }\n        Long et1 = historyEtValList.get(historyEtValList.size() - 1);\n        Long et2 = historyEtValList.get(historyEtValList.size() - 2);\n        Long differential = et1 - et2;\n        Double integration = 0.0;\n        for (Long item: historyEtValList) {\n            integration += item;\n        }\n        return  KP * et + KI * integration + KD * differential;\n    }\n\n    @Override\n    public void promote(String consumerGroup, Long currentThreshold) {\n        if (decisionFactor() > 0) {\n            coldDataCgCtrService.addOrUpdateGroupConfig(consumerGroup, (long)(currentThreshold * 1.5));\n        }\n    }\n\n    @Override\n    public void decelerate(String consumerGroup, Long currentThreshold) {\n        if (decisionFactor() < 0) {\n            long changedThresholdVal = (long)(currentThreshold * 0.8);\n            if (changedThresholdVal < coldDataCgCtrService.getBrokerConfig().getCgColdReadThreshold()) {\n                changedThresholdVal = coldDataCgCtrService.getBrokerConfig().getCgColdReadThreshold();\n            }\n            coldDataCgCtrService.addOrUpdateGroupConfig(consumerGroup, changedThresholdVal);\n        }\n    }\n\n    @Override\n    public void collect(Long globalAcc) {\n        et = expectGlobalVal - globalAcc;\n        historyEtValList.add(et);\n        Iterator<Long> iterator = historyEtValList.iterator();\n        while (historyEtValList.size() > MAX_STORE_NUMS && iterator.hasNext()) {\n            iterator.next();\n            iterator.remove();\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/coldctr/SimpleColdCtrStrategy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.coldctr;\n\npublic class SimpleColdCtrStrategy implements ColdCtrStrategy {\n    private final ColdDataCgCtrService coldDataCgCtrService;\n\n    public SimpleColdCtrStrategy(ColdDataCgCtrService coldDataCgCtrService) {\n        this.coldDataCgCtrService = coldDataCgCtrService;\n    }\n\n    @Override\n    public Double decisionFactor() {\n        return null;\n    }\n\n    @Override\n    public void promote(String consumerGroup, Long currentThreshold) {\n        coldDataCgCtrService.addOrUpdateGroupConfig(consumerGroup, (long)(currentThreshold * 1.5));\n    }\n\n    @Override\n    public void decelerate(String consumerGroup, Long currentThreshold) {\n        if (!coldDataCgCtrService.isGlobalColdCtr()) {\n            return;\n        }\n        long changedThresholdVal = (long)(currentThreshold * 0.8);\n        if (changedThresholdVal < coldDataCgCtrService.getBrokerConfig().getCgColdReadThreshold()) {\n            changedThresholdVal = coldDataCgCtrService.getBrokerConfig().getCgColdReadThreshold();\n        }\n        coldDataCgCtrService.addOrUpdateGroupConfig(consumerGroup, changedThresholdVal);\n    }\n\n    @Override\n    public void collect(Long globalAcc) {\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBConfigManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.config.v1;\n\nimport com.alibaba.fastjson2.JSON;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.config.ConfigRocksDBStorage;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.rocksdb.CompressionType;\nimport org.rocksdb.FlushOptions;\nimport org.rocksdb.RocksDB;\nimport org.rocksdb.RocksDBException;\nimport org.rocksdb.Statistics;\nimport org.rocksdb.WriteBatch;\n\nimport java.nio.charset.Charset;\nimport java.nio.charset.StandardCharsets;\nimport java.util.function.BiConsumer;\n\npublic class RocksDBConfigManager {\n    protected static final Logger BROKER_LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n\n    public static final Charset CHARSET = StandardCharsets.UTF_8;\n\n    public ConfigRocksDBStorage configRocksDBStorage = null;\n    private FlushOptions flushOptions = null;\n    private volatile long lastFlushMemTableMicroSecond = 0;\n    private final String filePath;\n    private final long memTableFlushInterval;\n    private final CompressionType compressionType;\n    private DataVersion kvDataVersion = new DataVersion();\n\n    public static final byte[] KV_DATA_VERSION_COLUMN_FAMILY_NAME = \"kvDataVersion\".getBytes(CHARSET);\n    public static final byte[] KV_DATA_VERSION_KEY = \"kvDataVersionKey\".getBytes(CHARSET);\n\n    private final String defaultCF;\n    private final String versionCF;\n\n\n    public RocksDBConfigManager(String filePath, long memTableFlushInterval, CompressionType compressionType,\n        String defaultCF, String versionCF) {\n        this.filePath = filePath;\n        this.memTableFlushInterval = memTableFlushInterval;\n        this.compressionType = compressionType;\n        this.defaultCF = defaultCF;\n        this.versionCF = versionCF;\n    }\n\n    public RocksDBConfigManager(String filePath, long memTableFlushInterval, CompressionType compressionType) {\n        this.filePath = filePath;\n        this.memTableFlushInterval = memTableFlushInterval;\n        this.compressionType = compressionType;\n        this.defaultCF = new String(RocksDB.DEFAULT_COLUMN_FAMILY, CHARSET);\n        this.versionCF = new String(KV_DATA_VERSION_COLUMN_FAMILY_NAME, CHARSET);\n    }\n\n    public boolean init(boolean readOnly) {\n        this.configRocksDBStorage = ConfigRocksDBStorage.getStore(filePath, readOnly, compressionType);\n        return this.configRocksDBStorage.start();\n    }\n\n    public boolean isLoaded() {\n        return this.configRocksDBStorage != null && this.configRocksDBStorage.isLoaded();\n    }\n\n    public boolean init() {\n        return this.init(false);\n    }\n\n    public boolean loadDataVersion() {\n        String currDataVersionString = null;\n        try {\n            byte[] dataVersion = this.configRocksDBStorage.get(versionCF, KV_DATA_VERSION_KEY);\n            if (dataVersion != null && dataVersion.length > 0) {\n                currDataVersionString = new String(dataVersion, StandardCharsets.UTF_8);\n            }\n            kvDataVersion = StringUtils.isNotBlank(currDataVersionString) ? JSON.parseObject(currDataVersionString, DataVersion.class) : new DataVersion();\n            return true;\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    public boolean loadData(BiConsumer<byte[], byte[]> biConsumer) {\n        try {\n            configRocksDBStorage.iterate(this.defaultCF, biConsumer);\n        } catch (Exception e) {\n            BROKER_LOG.error(\"RocksDBConfigManager loadData failed\", e);\n            return false;\n        }\n\n        this.flushOptions = new FlushOptions();\n        this.flushOptions.setWaitForFlush(false);\n        this.flushOptions.setAllowWriteStall(false);\n        return true;\n    }\n\n    public void start() {\n    }\n\n    public boolean stop() {\n        ConfigRocksDBStorage.shutdown(filePath);\n        if (this.flushOptions != null) {\n            this.flushOptions.close();\n        }\n        return true;\n    }\n\n    public void flushWAL() {\n        try {\n            if (!isLoaded()) {\n                return;\n            }\n            if (this.configRocksDBStorage != null) {\n                this.configRocksDBStorage.flushWAL();\n\n                long now = System.currentTimeMillis();\n                if (now > this.lastFlushMemTableMicroSecond + this.memTableFlushInterval) {\n                    this.configRocksDBStorage.flush(this.flushOptions);\n                    this.lastFlushMemTableMicroSecond = now;\n                }\n            }\n        } catch (Exception e) {\n            BROKER_LOG.error(\"kv flush WAL Failed.\", e);\n        }\n    }\n\n    public void put(final byte[] keyBytes, final byte[] valueBytes) throws Exception {\n        this.configRocksDBStorage.put(defaultCF, keyBytes, keyBytes.length, valueBytes);\n    }\n\n    public void put(String cf, String key, String value) throws Exception {\n        byte[] keyBytes = key.getBytes(CHARSET);\n        this.configRocksDBStorage.put(cf, keyBytes, keyBytes.length, value.getBytes(CHARSET));\n    }\n\n    public void put(String cf, final byte[] keyBytes, final byte[] valueBytes) throws Exception {\n        this.configRocksDBStorage.put(cf, keyBytes, keyBytes.length, valueBytes);\n    }\n\n    public void delete(final byte[] keyBytes) throws Exception {\n        this.configRocksDBStorage.delete(defaultCF, keyBytes);\n    }\n\n    public void updateKvDataVersion() throws Exception {\n        kvDataVersion.nextVersion();\n        this.configRocksDBStorage.put(versionCF, KV_DATA_VERSION_KEY, KV_DATA_VERSION_KEY.length,\n            JSON.toJSONString(kvDataVersion).getBytes(StandardCharsets.UTF_8));\n    }\n\n    public DataVersion getKvDataVersion() {\n        return kvDataVersion;\n    }\n\n    // batch operations\n    public void writeBatchPutOperation(WriteBatch writeBatch, final byte[] key, final byte[] value) throws RocksDBException {\n        configRocksDBStorage.writeBatchPutOperation(defaultCF, writeBatch, key, value);\n    }\n\n    public void batchPutWithWal(final WriteBatch batch) throws Exception {\n        this.configRocksDBStorage.batchPutWithWal(batch);\n    }\n\n    public Statistics getStatistics() {\n        if (this.configRocksDBStorage == null) {\n            return null;\n        }\n\n        return configRocksDBStorage.getStatistics();\n    }\n\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBConsumerOffsetManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.config.v1;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.alibaba.fastjson2.JSONWriter;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.util.Map.Entry;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.BrokerPathConfigHelper;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.DataConverter;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.rocksdb.CompressionType;\nimport org.rocksdb.WriteBatch;\n\npublic class RocksDBConsumerOffsetManager extends ConsumerOffsetManager {\n\n    protected static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n\n    private static final String VERSION_COLUMN_FAMILY = \"consumerOffsetVersion\";\n    private static final String OFFSET_COLUMN_FAMILY = \"consumerOffset\";\n\n    protected transient RocksDBConfigManager rocksDBConfigManager;\n    private final boolean useSingleRocksDBForAllConfigs;\n    private final String storePathRootDir;\n\n    public RocksDBConsumerOffsetManager(BrokerController brokerController, boolean useSingleRocksDB,\n        String storePathRootDir) {\n        super(brokerController);\n\n        this.useSingleRocksDBForAllConfigs = useSingleRocksDB;\n        this.storePathRootDir = StringUtils.isBlank(storePathRootDir) ?\n            brokerController.getMessageStoreConfig().getStorePathRootDir() : storePathRootDir;\n\n        long flushInterval = brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs();\n        CompressionType compressionType =\n            CompressionType.getCompressionType(brokerController.getMessageStoreConfig().getRocksdbCompressionType());\n        String rocksDBPath = rocksdbConfigFilePath(storePathRootDir, useSingleRocksDB);\n\n        this.rocksDBConfigManager = useSingleRocksDB ? new RocksDBConfigManager(rocksDBPath, flushInterval,\n            compressionType, OFFSET_COLUMN_FAMILY, VERSION_COLUMN_FAMILY) : new RocksDBConfigManager(rocksDBPath,\n            flushInterval, compressionType);\n    }\n\n    public RocksDBConsumerOffsetManager(BrokerController brokerController, boolean useSingleRocksDBForAllConfigs) {\n        this(brokerController, useSingleRocksDBForAllConfigs, null);\n    }\n\n    public RocksDBConsumerOffsetManager(BrokerController brokerController) {\n        this(brokerController, brokerController.getBrokerConfig().isUseSingleRocksDBForAllConfigs(), null);\n    }\n\n    @Override\n    public boolean load() {\n        if (!rocksDBConfigManager.init()) {\n            return false;\n        }\n        if (!loadDataVersion() || !loadConsumerOffset()) {\n            return false;\n        }\n        if (useSingleRocksDBForAllConfigs) {\n            migrateFromSeparateRocksDBs();\n        }\n        return true;\n    }\n\n    public boolean loadConsumerOffset() {\n        return this.rocksDBConfigManager.loadData(this::decodeOffset) && merge();\n    }\n\n    private boolean merge() {\n        if (!UtilAll.isPathExists(this.configFilePath()) && !UtilAll.isPathExists(this.configFilePath() + \".bak\")) {\n            log.info(\"consumerOffset json file does not exist, so skip merge\");\n            return true;\n        }\n        if (!super.loadDataVersion()) {\n            log.error(\"load json consumerOffset dataVersion error, startup will exit\");\n            return false;\n        }\n\n        final DataVersion dataVersion = super.getDataVersion();\n        final DataVersion kvDataVersion = this.getDataVersion();\n        if (dataVersion.getCounter().get() > kvDataVersion.getCounter().get()) {\n            if (!super.load()) {\n                log.error(\"load json consumerOffset info failed, startup will exit\");\n                return false;\n            }\n            this.persist();\n            this.getDataVersion().assignNewOne(dataVersion);\n            updateDataVersion();\n            log.info(\"update offset from json, dataVersion:{}, offsetTable: {} \", this.getDataVersion(), JSON.toJSONString(this.getOffsetTable()));\n        }\n        return true;\n    }\n\n\n    @Override\n    public boolean stop() {\n        return this.rocksDBConfigManager.stop();\n    }\n\n    @Override\n    public void removeConsumerOffset(String topicAtGroup) {\n        try {\n            byte[] keyBytes = topicAtGroup.getBytes(DataConverter.CHARSET_UTF8);\n            this.rocksDBConfigManager.delete(keyBytes);\n        } catch (Exception e) {\n            log.error(\"kv remove consumerOffset Failed, {}\", topicAtGroup);\n        }\n    }\n\n    protected void decodeOffset(final byte[] key, final byte[] body) {\n        String topicAtGroup = new String(key, DataConverter.CHARSET_UTF8);\n        RocksDBOffsetSerializeWrapper wrapper = JSON.parseObject(body, RocksDBOffsetSerializeWrapper.class);\n\n        this.offsetTable.put(topicAtGroup, wrapper.getOffsetTable());\n        log.info(\"load exist local offset, {}, {}\", topicAtGroup, wrapper.getOffsetTable());\n    }\n\n    public String rocksdbConfigFilePath(String storePathRootDir, boolean useSingleRocksDBForAllConfigs) {\n        if (StringUtils.isBlank(storePathRootDir)) {\n            storePathRootDir = brokerController.getMessageStoreConfig().getStorePathRootDir();\n        }\n        Path rootPath = Paths.get(storePathRootDir);\n        if (useSingleRocksDBForAllConfigs) {\n            return rootPath.resolve(\"config\").resolve(\"metadata\").toString();\n        }\n        return rootPath.resolve(\"config\").resolve(\"consumerOffsets\").toString();\n    }\n\n    @Override\n    public String configFilePath() {\n        return BrokerPathConfigHelper.getConsumerOffsetPath(this.storePathRootDir);\n    }\n\n    @Override\n    public synchronized void persist() {\n        if (rocksDBConfigManager.isLoaded()) {\n            if (brokerController.getBrokerConfig().isPersistConsumerOffsetIncrementally()) {\n                updateDataVersion();\n                this.rocksDBConfigManager.flushWAL();\n                return;\n            }\n            try (WriteBatch writeBatch = new WriteBatch()) {\n                for (Entry<String, ConcurrentMap<Integer, Long>> entry : this.offsetTable.entrySet()) {\n                    putWriteBatch(writeBatch, entry.getKey(), entry.getValue());\n                    if (writeBatch.getDataSize() >= 4 * 1024) {\n                        this.rocksDBConfigManager.batchPutWithWal(writeBatch);\n                    }\n                }\n                this.rocksDBConfigManager.batchPutWithWal(writeBatch);\n                this.rocksDBConfigManager.flushWAL();\n            } catch (Exception e) {\n                log.error(\"consumer offset persist Failed\", e);\n            }\n        } else {\n            log.warn(\"RocksDBConsumerOffsetManager has been stopped, persist fail\");\n        }\n    }\n\n    @Override\n    public void commitOffset(String clientHost, String group, String topic, int queueId, long offset) {\n        String key = topic + TOPIC_GROUP_SEPARATOR + group;\n        ConcurrentMap<Integer, Long> map = this.offsetTable.get(key);\n        if (null == map) {\n            map = MixAll.isLmq(topic) ? new ConcurrentHashMap<>(1, 1.0F) : new ConcurrentHashMap<>();\n            map.put(queueId, offset);\n            this.offsetTable.put(key, map);\n        } else {\n            Long storeOffset = map.put(queueId, offset);\n            if (storeOffset != null && offset < storeOffset) {\n                LOG.warn(\"[NOTIFYME]update consumer offset less than store. clientHost={}, key={}, queueId={}, requestOffset={}, storeOffset={}\", clientHost, key, queueId, offset, storeOffset);\n            }\n        }\n        if (versionChangeCounter.incrementAndGet() % brokerController.getBrokerConfig().getConsumerOffsetUpdateVersionStep() == 0) {\n            updateDataVersion();\n        }\n        if (!brokerController.getBrokerConfig().isPersistConsumerOffsetIncrementally()) {\n            return;\n        }\n\n        try (WriteBatch writeBatch = new WriteBatch()) {\n            putWriteBatch(writeBatch, key, map);\n            this.rocksDBConfigManager.batchPutWithWal(writeBatch);\n        } catch (Exception e) {\n            log.error(\"consumer offset persist Failed\", e);\n        }\n    }\n\n    public synchronized void exportToJson() {\n        log.info(\"RocksDBConsumerOffsetManager export consumer offset to json file\");\n        super.persist();\n    }\n\n    private void putWriteBatch(final WriteBatch writeBatch, final String topicGroupName, final ConcurrentMap<Integer, Long> offsetMap) throws Exception {\n        byte[] keyBytes = topicGroupName.getBytes(DataConverter.CHARSET_UTF8);\n        RocksDBOffsetSerializeWrapper wrapper = new RocksDBOffsetSerializeWrapper();\n        wrapper.setOffsetTable(offsetMap);\n        byte[] valueBytes = JSON.toJSONBytes(wrapper, JSONWriter.Feature.BrowserCompatible);\n        rocksDBConfigManager.writeBatchPutOperation(writeBatch, keyBytes, valueBytes);\n    }\n\n    @Override\n    public boolean loadDataVersion() {\n        return this.rocksDBConfigManager.loadDataVersion();\n    }\n\n    @Override\n    public DataVersion getDataVersion() {\n        return rocksDBConfigManager.getKvDataVersion();\n    }\n\n    @Override\n    public void updateDataVersion() {\n        try {\n            rocksDBConfigManager.updateKvDataVersion();\n        } catch (Exception e) {\n            log.error(\"update consumer offset dataVersion error\", e);\n            throw new RuntimeException(e);\n        }\n    }\n\n    /**\n     * Migrate data from separate RocksDB instances to the unified RocksDB when useSingleRocksDBForAllConfigs is\n     * enabled.\n     * This method will only be called when switching from separate RocksDB mode to unified mode.\n     * It opens the separate RocksDB in read-only mode, compares versions, and imports data if needed.\n     */\n    private void migrateFromSeparateRocksDBs() {\n        String separateRocksDBPath = rocksdbConfigFilePath(this.storePathRootDir, false);\n\n        // Check if separate RocksDB exists\n        if (!UtilAll.isPathExists(separateRocksDBPath)) {\n            log.info(\"Separate RocksDB for consumer offsets does not exist at {}, no migration needed\",\n                separateRocksDBPath);\n            return;\n        }\n\n        log.info(\"Starting migration from separate RocksDB at {} to unified RocksDB\", separateRocksDBPath);\n\n        // Open separate RocksDB in read-only mode\n        RocksDBConfigManager separateRocksDBConfigManager = null;\n        try {\n            long memTableFlushIntervalMs = brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs();\n            org.rocksdb.CompressionType compressionType =\n                org.rocksdb.CompressionType.getCompressionType(brokerController.getMessageStoreConfig().getRocksdbCompressionType());\n\n            separateRocksDBConfigManager = new RocksDBConfigManager(separateRocksDBPath, memTableFlushIntervalMs,\n                compressionType);\n\n            // Initialize in read-only mode\n            if (!separateRocksDBConfigManager.init(true)) {\n                log.error(\"Failed to initialize separate RocksDB in read-only mode\");\n                return;\n            }\n\n            // Load data version from separate RocksDB\n            if (!separateRocksDBConfigManager.loadDataVersion()) {\n                log.error(\"Failed to load data version from separate RocksDB\");\n                return;\n            }\n\n            DataVersion separateDataVersion = separateRocksDBConfigManager.getKvDataVersion();\n            DataVersion unifiedDataVersion = this.getDataVersion();\n\n            log.info(\"Comparing data versions - Separate: {}, Unified: {}\", separateDataVersion, unifiedDataVersion);\n\n            // Compare versions and import if separate version is newer\n            if (separateDataVersion.getCounter().get() > unifiedDataVersion.getCounter().get()) {\n                log.info(\"Separate RocksDB has newer data, importing...\");\n\n                // Load consumer offsets from separate RocksDB\n                if (separateRocksDBConfigManager.loadData(this::importConsumerOffset)) {\n                    log.info(\"Successfully imported consumer offsets from separate RocksDB\");\n\n                    // Update unified data version to be newer than separate one\n                    this.getDataVersion().assignNewOne(separateDataVersion);\n                    this.getDataVersion().nextVersion(); // Make it one version higher\n                    updateDataVersion();\n\n                    log.info(\"Updated unified data version to {}\", this.getDataVersion());\n                } else {\n                    log.error(\"Failed to import consumer offsets from separate RocksDB\");\n                }\n            } else {\n                log.info(\"Unified RocksDB is already up-to-date, no migration needed\");\n            }\n        } catch (Exception e) {\n            log.error(\"Error during migration from separate RocksDB\", e);\n        } finally {\n            // Clean up resources\n            if (separateRocksDBConfigManager != null) {\n                try {\n                    separateRocksDBConfigManager.stop();\n                } catch (Exception e) {\n                    log.warn(\"Error stopping separate RocksDB config manager\", e);\n                }\n            }\n        }\n    }\n\n    /**\n     * Import a consumer offset from the separate RocksDB during migration\n     *\n     * @param key  The topic@group name bytes\n     * @param body The consumer offset data bytes\n     */\n    private void importConsumerOffset(final byte[] key, final byte[] body) {\n        try {\n            decodeOffset(key, body);\n            this.rocksDBConfigManager.put(key, body);\n        } catch (Exception e) {\n            log.error(\"Error importing consumer offset\", e);\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBLmqSubscriptionGroupManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.config.v1;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\n\npublic class RocksDBLmqSubscriptionGroupManager extends RocksDBSubscriptionGroupManager {\n\n    public RocksDBLmqSubscriptionGroupManager(BrokerController brokerController) {\n        super(brokerController);\n    }\n\n    @Override\n    public SubscriptionGroupConfig findSubscriptionGroupConfig(final String group) {\n        if (MixAll.isLmq(group)) {\n            SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n            subscriptionGroupConfig.setGroupName(group);\n            return subscriptionGroupConfig;\n        }\n        return super.findSubscriptionGroupConfig(group);\n    }\n\n    @Override\n    public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config) {\n        if (config == null || MixAll.isLmq(config.getGroupName())) {\n            return;\n        }\n        super.updateSubscriptionGroupConfig(config);\n    }\n\n    @Override\n    public boolean containsSubscriptionGroup(String group) {\n        if (MixAll.isLmq(group)) {\n            return true;\n        } else {\n            return super.containsSubscriptionGroup(group);\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBLmqTopicConfigManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.config.v1;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.PermName;\n\npublic class RocksDBLmqTopicConfigManager extends RocksDBTopicConfigManager {\n\n    public RocksDBLmqTopicConfigManager(BrokerController brokerController) {\n        super(brokerController);\n    }\n\n    @Override\n    public TopicConfig selectTopicConfig(final String topic) {\n        if (MixAll.isLmq(topic)) {\n            return simpleLmqTopicConfig(topic);\n        }\n        return super.selectTopicConfig(topic);\n    }\n\n    @Override\n    public void updateTopicConfig(final TopicConfig topicConfig) {\n        if (topicConfig == null || MixAll.isLmq(topicConfig.getTopicName())) {\n            return;\n        }\n        super.updateTopicConfig(topicConfig);\n    }\n\n    @Override\n    public boolean containsTopic(String topic) {\n        if (MixAll.isLmq(topic)) {\n            return true;\n        }\n        return super.containsTopic(topic);\n    }\n\n    private TopicConfig simpleLmqTopicConfig(String topic) {\n        return new TopicConfig(topic, 1, 1, PermName.PERM_READ | PermName.PERM_WRITE);\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBOffsetSerializeWrapper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.config.v1;\n\nimport java.util.concurrent.ConcurrentMap;\n\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class RocksDBOffsetSerializeWrapper extends RemotingSerializable {\n    private ConcurrentMap<Integer, Long> offsetTable = null;\n\n    public ConcurrentMap<Integer, Long> getOffsetTable() {\n        return offsetTable;\n    }\n\n    public void setOffsetTable(ConcurrentMap<Integer, Long> offsetTable) {\n        this.offsetTable = offsetTable;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBSubscriptionGroupManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.config.v1;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.alibaba.fastjson2.JSONObject;\nimport com.alibaba.fastjson2.JSONWriter;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.BrokerPathConfigHelper;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.rocksdb.CompressionType;\n\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.function.BiConsumer;\n\npublic class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager {\n\n    protected transient RocksDBConfigManager rocksDBConfigManager;\n\n    private static final String VERSION_COLUMN_FAMILY = \"subscriptionGroupVersion\";\n    private static final String GROUP_COLUMN_FAMILY = \"subscriptionGroup\";\n    private static final String FORBIDDEN_COLUMN_FAMILY_NAME = \"forbidden\";\n\n    private final boolean useSingleRocksDBForAllConfigs;\n    private final String storePathRootDir;\n\n    public RocksDBSubscriptionGroupManager(BrokerController brokerController, boolean useSingleRocksDB,\n        String storePathRootDir) {\n        super(brokerController, false);\n\n        this.useSingleRocksDBForAllConfigs = useSingleRocksDB;\n        this.storePathRootDir = StringUtils.isBlank(storePathRootDir) ?\n            brokerController.getMessageStoreConfig().getStorePathRootDir() : storePathRootDir;\n\n        long flushInterval = brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs();\n        CompressionType compressionType =\n            CompressionType.getCompressionType(brokerController.getMessageStoreConfig().getRocksdbCompressionType());\n        String rocksDBPath = rocksdbConfigFilePath(storePathRootDir, useSingleRocksDB);\n\n        this.rocksDBConfigManager = useSingleRocksDB ? new RocksDBConfigManager(rocksDBPath, flushInterval,\n            compressionType, GROUP_COLUMN_FAMILY, VERSION_COLUMN_FAMILY) : new RocksDBConfigManager(rocksDBPath,\n            flushInterval, compressionType);\n    }\n\n    public RocksDBSubscriptionGroupManager(BrokerController brokerController, boolean useSingleRocksDBForAllConfigs) {\n        this(brokerController, useSingleRocksDBForAllConfigs, null);\n    }\n\n    public RocksDBSubscriptionGroupManager(BrokerController brokerController) {\n        this(brokerController, brokerController.getBrokerConfig().isUseSingleRocksDBForAllConfigs(), null);\n    }\n\n    @Override\n    public boolean load() {\n        if (!rocksDBConfigManager.init()) {\n            return false;\n        }\n        if (!loadDataVersion() || !loadSubscriptionGroupAndForbidden()) {\n            return false;\n        }\n        if (useSingleRocksDBForAllConfigs) {\n            migrateFromSeparateRocksDBs();\n        }\n        this.init();\n        return true;\n    }\n\n    public boolean loadDataVersion() {\n        return this.rocksDBConfigManager.loadDataVersion();\n    }\n\n    public boolean loadSubscriptionGroupAndForbidden() {\n        return this.rocksDBConfigManager.loadData(this::decodeSubscriptionGroup)\n                && this.loadForbidden(this::decodeForbidden)\n                && merge();\n    }\n\n    public boolean loadForbidden(BiConsumer<byte[], byte[]> biConsumer) {\n        try {\n            this.rocksDBConfigManager.configRocksDBStorage.iterate(FORBIDDEN_COLUMN_FAMILY_NAME, biConsumer);\n            return true;\n        } catch (Exception e) {\n            log.error(\"loadForbidden exception\", e);\n        }\n        return false;\n    }\n\n    private boolean merge() {\n        if (!UtilAll.isPathExists(this.configFilePath()) && !UtilAll.isPathExists(this.configFilePath() + \".bak\")) {\n            log.info(\"subGroup json file does not exist, so skip merge\");\n            return true;\n        }\n        if (!super.loadDataVersion()) {\n            log.error(\"load json subGroup dataVersion error, startup will exit\");\n            return false;\n        }\n        final DataVersion dataVersion = super.getDataVersion();\n        final DataVersion kvDataVersion = this.getDataVersion();\n        if (dataVersion.getCounter().get() > kvDataVersion.getCounter().get()) {\n            if (!super.load()) {\n                log.error(\"load group and forbidden info from json file error, startup will exit\");\n                return false;\n            }\n            final ConcurrentMap<String, SubscriptionGroupConfig> groupTable = this.getSubscriptionGroupTable();\n            for (Map.Entry<String, SubscriptionGroupConfig> entry : groupTable.entrySet()) {\n                putSubscriptionGroupConfig(entry.getValue());\n                log.info(\"import subscription config to rocksdb, group={}\", entry.getValue());\n            }\n            final ConcurrentMap<String, ConcurrentMap<String, Integer>> forbiddenTable = this.getForbiddenTable();\n            for (Map.Entry<String, ConcurrentMap<String, Integer>> entry : forbiddenTable.entrySet()) {\n                try {\n                    this.rocksDBConfigManager.put(FORBIDDEN_COLUMN_FAMILY_NAME, entry.getKey(),\n                        JSON.toJSONString(entry.getValue()));\n                    log.info(\"import forbidden config to rocksdb, group={}\", entry.getValue());\n                } catch (Exception e) {\n                    log.error(\"import forbidden config to rocksdb failed, group={}\", entry.getValue());\n                    return false;\n                }\n            }\n            this.getDataVersion().assignNewOne(dataVersion);\n            updateDataVersion();\n        } else {\n            log.info(\"dataVersion is not greater than kvDataVersion, no need to merge group metaData, dataVersion={}, kvDataVersion={}\", dataVersion, kvDataVersion);\n        }\n        log.info(\"finish merge subscription config from json file and merge to rocksdb\");\n        this.persist();\n\n        return true;\n    }\n\n    @Override\n    public boolean stop() {\n        return this.rocksDBConfigManager.stop();\n    }\n\n    @Override\n    public SubscriptionGroupConfig putSubscriptionGroupConfig(SubscriptionGroupConfig subscriptionGroupConfig) {\n        String groupName = subscriptionGroupConfig.getGroupName();\n        SubscriptionGroupConfig oldConfig = this.subscriptionGroupTable.put(groupName, subscriptionGroupConfig);\n\n        try {\n            byte[] keyBytes = groupName.getBytes(RocksDBConfigManager.CHARSET);\n            byte[] valueBytes = JSON.toJSONBytes(subscriptionGroupConfig, JSONWriter.Feature.BrowserCompatible);\n            this.rocksDBConfigManager.put(keyBytes, valueBytes);\n        } catch (Exception e) {\n            log.error(\"kv put sub Failed, {}\", subscriptionGroupConfig.toString());\n        }\n        return oldConfig;\n    }\n\n    @Override\n    protected SubscriptionGroupConfig putSubscriptionGroupConfigIfAbsent(SubscriptionGroupConfig subscriptionGroupConfig) {\n        String groupName = subscriptionGroupConfig.getGroupName();\n        SubscriptionGroupConfig oldConfig = this.subscriptionGroupTable.putIfAbsent(groupName, subscriptionGroupConfig);\n        if (oldConfig == null) {\n            try {\n                byte[] keyBytes = groupName.getBytes(RocksDBConfigManager.CHARSET);\n                byte[] valueBytes = JSON.toJSONBytes(subscriptionGroupConfig, JSONWriter.Feature.BrowserCompatible);\n                this.rocksDBConfigManager.put(keyBytes, valueBytes);\n            } catch (Exception e) {\n                log.error(\"kv put sub Failed, {}\", subscriptionGroupConfig.toString());\n            }\n        }\n        return oldConfig;\n    }\n\n    @Override\n    protected SubscriptionGroupConfig removeSubscriptionGroupConfig(String groupName) {\n        SubscriptionGroupConfig subscriptionGroupConfig = this.subscriptionGroupTable.remove(groupName);\n        try {\n            this.rocksDBConfigManager.delete(groupName.getBytes(RocksDBConfigManager.CHARSET));\n        } catch (Exception e) {\n            log.error(\"kv delete sub Failed, {}\", subscriptionGroupConfig.toString());\n        }\n        return subscriptionGroupConfig;\n    }\n\n\n    protected void decodeSubscriptionGroup(byte[] key, byte[] body) {\n        String groupName = new String(key, RocksDBConfigManager.CHARSET);\n        SubscriptionGroupConfig subscriptionGroupConfig = JSON.parseObject(body, SubscriptionGroupConfig.class);\n\n        this.subscriptionGroupTable.put(groupName, subscriptionGroupConfig);\n        log.info(\"load exist local sub, {}\", subscriptionGroupConfig.toString());\n    }\n\n    @Override\n    public synchronized void persist() {\n        if (brokerController.getMessageStoreConfig().isRealTimePersistRocksDBConfig()) {\n            this.rocksDBConfigManager.flushWAL();\n        }\n    }\n\n    public synchronized void exportToJson() {\n        log.info(\"RocksDBSubscriptionGroupManager export subscription group to json file\");\n        super.persist();\n    }\n\n    public String rocksdbConfigFilePath(String storePathRootDir, boolean useSingleRocksDBForAllConfigs) {\n        if (StringUtils.isBlank(storePathRootDir)) {\n            storePathRootDir = brokerController.getMessageStoreConfig().getStorePathRootDir();\n        }\n        Path rootPath = Paths.get(storePathRootDir);\n        if (useSingleRocksDBForAllConfigs) {\n            return rootPath.resolve(\"config\").resolve(\"metadata\").toString();\n        }\n        return rootPath.resolve(\"config\").resolve(\"subscriptionGroups\").toString();\n    }\n\n    @Override\n    public String configFilePath() {\n        return BrokerPathConfigHelper.getSubscriptionGroupPath(this.storePathRootDir);\n    }\n\n    @Override\n    public DataVersion getDataVersion() {\n        return rocksDBConfigManager.getKvDataVersion();\n    }\n\n    @Override\n    public void updateDataVersion() {\n        try {\n            rocksDBConfigManager.updateKvDataVersion();\n        } catch (Exception e) {\n            log.error(\"update group config dataVersion error\", e);\n            throw new RuntimeException(e);\n        }\n    }\n\n    protected void decodeForbidden(byte[] key, byte[] body) {\n        String forbiddenGroupName = new String(key, RocksDBConfigManager.CHARSET);\n        JSONObject jsonObject = JSON.parseObject(new String(body, RocksDBConfigManager.CHARSET));\n        Set<Map.Entry<String, Object>> entries = jsonObject.entrySet();\n        ConcurrentMap<String, Integer> forbiddenGroup = new ConcurrentHashMap<>(entries.size());\n        for (Map.Entry<String, Object> entry : entries) {\n            forbiddenGroup.put(entry.getKey(), (Integer) entry.getValue());\n        }\n        this.getForbiddenTable().put(forbiddenGroupName, forbiddenGroup);\n        log.info(\"load forbidden,{} value {}\", forbiddenGroupName, forbiddenGroup.toString());\n    }\n\n    @Override\n    public void updateForbidden(String group, String topic, int forbiddenIndex, boolean setOrClear) {\n        try {\n            super.updateForbidden(group, topic, forbiddenIndex, setOrClear);\n            this.rocksDBConfigManager.put(FORBIDDEN_COLUMN_FAMILY_NAME, group,\n                JSON.toJSONString(this.getForbiddenTable().get(group)));\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    @Override\n    public void setForbidden(String group, String topic, int forbiddenIndex) {\n        try {\n            super.setForbidden(group, topic, forbiddenIndex);\n            this.rocksDBConfigManager.put(FORBIDDEN_COLUMN_FAMILY_NAME, group,\n                JSON.toJSONString(this.getForbiddenTable().get(group)));\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    @Override\n    public void clearForbidden(String group, String topic, int forbiddenIndex) {\n        try {\n            super.clearForbidden(group, topic, forbiddenIndex);\n            this.rocksDBConfigManager.put(FORBIDDEN_COLUMN_FAMILY_NAME, group,\n                JSON.toJSONString(this.getForbiddenTable().get(group)));\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    /**\n     * Migrate data from separate RocksDB instances to the unified RocksDB when useSingleRocksDBForAllConfigs is\n     * enabled.\n     * This method will only be called when switching from separate RocksDB mode to unified mode.\n     * It opens the separate RocksDB in read-only mode, compares versions, and imports data if needed.\n     */\n    private void migrateFromSeparateRocksDBs() {\n        String separateRocksDBPath = rocksdbConfigFilePath(this.storePathRootDir, false);\n\n        // Check if separate RocksDB exists\n        if (!org.apache.rocketmq.common.UtilAll.isPathExists(separateRocksDBPath)) {\n            log.info(\"Separate RocksDB for subscription groups does not exist at {}, no migration needed\",\n                separateRocksDBPath);\n            return;\n        }\n\n        log.info(\"Starting migration from separate RocksDB at {} to unified RocksDB\", separateRocksDBPath);\n\n        // Open separate RocksDB in read-only mode\n        RocksDBConfigManager separateRocksDBConfigManager = null;\n        try {\n            long memTableFlushIntervalMs = brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs();\n            org.rocksdb.CompressionType compressionType =\n                org.rocksdb.CompressionType.getCompressionType(brokerController.getMessageStoreConfig().getRocksdbCompressionType());\n\n            separateRocksDBConfigManager = new RocksDBConfigManager(separateRocksDBPath, memTableFlushIntervalMs,\n                compressionType);\n\n            // Initialize in read-only mode\n            if (!separateRocksDBConfigManager.init(true)) {\n                log.error(\"Failed to initialize separate RocksDB in read-only mode\");\n                return;\n            }\n\n            // Load data version from separate RocksDB\n            if (!separateRocksDBConfigManager.loadDataVersion()) {\n                log.error(\"Failed to load data version from separate RocksDB\");\n                return;\n            }\n\n            org.apache.rocketmq.remoting.protocol.DataVersion separateDataVersion =\n                separateRocksDBConfigManager.getKvDataVersion();\n            org.apache.rocketmq.remoting.protocol.DataVersion unifiedDataVersion = this.getDataVersion();\n\n            log.info(\"Comparing data versions - Separate: {}, Unified: {}\", separateDataVersion, unifiedDataVersion);\n\n            // Compare versions and import if separate version is newer\n            if (separateDataVersion.getCounter().get() > unifiedDataVersion.getCounter().get()) {\n                log.info(\"Separate RocksDB has newer data, importing...\");\n\n                // Load subscription groups from separate RocksDB\n                boolean success = separateRocksDBConfigManager.loadData(this::importSubscriptionGroup);\n                if (success) {\n                    // Load forbidden data directly using the storage\n                    try {\n                        separateRocksDBConfigManager.configRocksDBStorage.iterate(FORBIDDEN_COLUMN_FAMILY_NAME,\n                            this::importForbidden);\n                        log.info(\"Successfully imported subscription groups and forbidden data from separate RocksDB\");\n\n                        // Update unified data version to be newer than separate one\n                        this.getDataVersion().assignNewOne(separateDataVersion);\n                        this.getDataVersion().nextVersion(); // Make it one version higher\n                        updateDataVersion();\n\n                        log.info(\"Updated unified data version to {}\", this.getDataVersion());\n                    } catch (Exception e) {\n                        log.error(\"Failed to import forbidden data from separate RocksDB\", e);\n                        success = false;\n                    }\n                }\n\n                if (!success) {\n                    log.error(\"Failed to import subscription groups or forbidden data from separate RocksDB\");\n                }\n            } else {\n                log.info(\"Unified RocksDB is already up-to-date, no migration needed\");\n            }\n        } catch (Exception e) {\n            log.error(\"Error during migration from separate RocksDB\", e);\n        } finally {\n            // Clean up resources\n            if (separateRocksDBConfigManager != null) {\n                try {\n                    separateRocksDBConfigManager.stop();\n                } catch (Exception e) {\n                    log.warn(\"Error stopping separate RocksDB config manager\", e);\n                }\n            }\n        }\n    }\n\n    /**\n     * Import a subscription group from the separate RocksDB during migration\n     *\n     * @param key  The group name bytes\n     * @param body The subscription group data bytes\n     */\n    private void importSubscriptionGroup(byte[] key, byte[] body) {\n        try {\n            decodeSubscriptionGroup(key, body);\n            this.rocksDBConfigManager.put(key, body);\n        } catch (Exception e) {\n            log.error(\"Error importing subscription group\", e);\n        }\n    }\n\n    /**\n     * Import forbidden data from the separate RocksDB during migration\n     *\n     * @param key  The group name bytes\n     * @param body The forbidden data bytes\n     */\n    private void importForbidden(byte[] key, byte[] body) {\n        try {\n            decodeForbidden(key, body);\n            this.rocksDBConfigManager.put(FORBIDDEN_COLUMN_FAMILY_NAME, key, body);\n        } catch (Exception e) {\n            log.error(\"Error importing forbidden data\", e);\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBTopicConfigManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.config.v1;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.alibaba.fastjson2.JSONWriter;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.BrokerPathConfigHelper;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.utils.DataConverter;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.rocksdb.CompressionType;\n\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentMap;\n\npublic class RocksDBTopicConfigManager extends TopicConfigManager {\n    private static final String VERSION_COLUMN_FAMILY = \"topicVersion\";\n    private static final String TOPIC_COLUMN_FAMILY = \"topic\";\n\n    protected transient RocksDBConfigManager rocksDBConfigManager;\n    private final boolean useSingleRocksDBForAllConfigs;\n    private final String storePathRootDir;\n\n    public RocksDBTopicConfigManager(BrokerController brokerController, boolean useSingleRocksDB,\n        String storePathRootDir) {\n        super(brokerController, false);\n\n        this.useSingleRocksDBForAllConfigs = useSingleRocksDB;\n        this.storePathRootDir = StringUtils.isBlank(storePathRootDir) ?\n            brokerController.getMessageStoreConfig().getStorePathRootDir() : storePathRootDir;\n\n        long flushInterval = brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs();\n        CompressionType compressionType =\n            CompressionType.getCompressionType(brokerController.getMessageStoreConfig().getRocksdbCompressionType());\n        String rocksDBPath = rocksdbConfigFilePath(storePathRootDir, useSingleRocksDB);\n\n        this.rocksDBConfigManager = useSingleRocksDB ? new RocksDBConfigManager(rocksDBPath, flushInterval,\n            compressionType, TOPIC_COLUMN_FAMILY, VERSION_COLUMN_FAMILY) : new RocksDBConfigManager(rocksDBPath,\n            flushInterval, compressionType);\n    }\n\n    public RocksDBTopicConfigManager(BrokerController brokerController, boolean useSingleRocksDBForAllConfigs) {\n        this(brokerController, useSingleRocksDBForAllConfigs, null);\n    }\n\n    public RocksDBTopicConfigManager(BrokerController brokerController) {\n        this(brokerController, brokerController.getBrokerConfig().isUseSingleRocksDBForAllConfigs(), null);\n    }\n\n    @Override\n    public boolean load() {\n        if (!rocksDBConfigManager.init()) {\n            return false;\n        }\n        if (!loadDataVersion() || !loadTopicConfig()) {\n            return false;\n        }\n        if (useSingleRocksDBForAllConfigs) {\n            migrateFromSeparateRocksDBs();\n        }\n        this.init();\n        return true;\n    }\n\n    public boolean loadTopicConfig() {\n        return this.rocksDBConfigManager.loadData(this::decodeTopicConfig) && merge();\n    }\n\n    public boolean loadDataVersion() {\n        return this.rocksDBConfigManager.loadDataVersion();\n    }\n\n    private boolean merge() {\n        if (!UtilAll.isPathExists(this.configFilePath()) && !UtilAll.isPathExists(this.configFilePath() + \".bak\")) {\n            log.info(\"topic json file does not exist, so skip merge\");\n            return true;\n        }\n\n        if (!super.loadDataVersion()) {\n            log.error(\"load json topic dataVersion error, startup will exit\");\n            return false;\n        }\n\n        final DataVersion dataVersion = super.getDataVersion();\n        final DataVersion kvDataVersion = this.getDataVersion();\n        if (dataVersion.getCounter().get() > kvDataVersion.getCounter().get()) {\n            if (!super.load()) {\n                log.error(\"load topic config from json file error, startup will exit\");\n                return false;\n            }\n            final ConcurrentMap<String, TopicConfig> topicConfigTable = this.getTopicConfigTable();\n            for (Map.Entry<String, TopicConfig> entry : topicConfigTable.entrySet()) {\n                putTopicConfig(entry.getValue());\n                log.info(\"import topic config to rocksdb, topic={}\", entry.getValue());\n            }\n            this.getDataVersion().assignNewOne(dataVersion);\n            updateDataVersion();\n        } else {\n            log.info(\"dataVersion is not greater than kvDataVersion, no need to merge topic metaData, dataVersion={}, kvDataVersion={}\", dataVersion, kvDataVersion);\n        }\n        log.info(\"finish read topic config from json file and merge to rocksdb\");\n        this.persist();\n        return true;\n    }\n\n\n    @Override\n    public boolean stop() {\n        return this.rocksDBConfigManager.stop();\n    }\n\n    protected void decodeTopicConfig(byte[] key, byte[] body) {\n        String topicName = new String(key, DataConverter.CHARSET_UTF8);\n        TopicConfig topicConfig = JSON.parseObject(body, TopicConfig.class);\n\n        this.topicConfigTable.put(topicName, topicConfig);\n        log.info(\"load exist local topic, {}\", topicConfig.toString());\n    }\n\n    @Override\n    public TopicConfig putTopicConfig(TopicConfig topicConfig) {\n        String topicName = topicConfig.getTopicName();\n        TopicConfig oldTopicConfig = this.topicConfigTable.put(topicName, topicConfig);\n        try {\n            byte[] keyBytes = topicName.getBytes(DataConverter.CHARSET_UTF8);\n            byte[] valueBytes = JSON.toJSONBytes(topicConfig, JSONWriter.Feature.BrowserCompatible);\n            this.rocksDBConfigManager.put(keyBytes, valueBytes);\n        } catch (Exception e) {\n            log.error(\"kv put topic Failed, {}\", topicConfig.toString(), e);\n        }\n        return oldTopicConfig;\n    }\n\n    @Override\n    protected TopicConfig removeTopicConfig(String topicName) {\n        TopicConfig topicConfig = this.topicConfigTable.remove(topicName);\n        try {\n            this.rocksDBConfigManager.delete(topicName.getBytes(DataConverter.CHARSET_UTF8));\n        } catch (Exception e) {\n            log.error(\"kv remove topic Failed, {}\", topicConfig.toString());\n        }\n        return topicConfig;\n    }\n\n    @Override\n    public synchronized void persist() {\n        if (brokerController.getMessageStoreConfig().isRealTimePersistRocksDBConfig()) {\n            this.rocksDBConfigManager.flushWAL();\n        }\n    }\n\n    public synchronized void exportToJson() {\n        log.info(\"RocksDBTopicConfigManager export topic config to json file\");\n        super.persist();\n    }\n\n    public String rocksdbConfigFilePath(String storePathRootDir, boolean useSingleRocksDBForAllConfigs) {\n        if (StringUtils.isBlank(storePathRootDir)) {\n            storePathRootDir = brokerController.getMessageStoreConfig().getStorePathRootDir();\n        }\n        Path rootPath = Paths.get(storePathRootDir);\n        if (useSingleRocksDBForAllConfigs) {\n            return rootPath.resolve(\"config\").resolve(\"metadata\").toString();\n        }\n        return rootPath.resolve(\"config\").resolve(\"topics\").toString();\n    }\n\n    @Override\n    public String configFilePath() {\n        return BrokerPathConfigHelper.getTopicConfigPath(this.storePathRootDir);\n    }\n\n    @Override\n    public DataVersion getDataVersion() {\n        return rocksDBConfigManager.getKvDataVersion();\n    }\n\n    @Override\n    public void updateDataVersion() {\n        try {\n            rocksDBConfigManager.updateKvDataVersion();\n        } catch (Exception e) {\n            log.error(\"update topic config dataVersion error\", e);\n            throw new RuntimeException(e);\n        }\n    }\n\n    /**\n     * Migrate data from separate RocksDB instances to the unified RocksDB when useSingleRocksDBForAllConfigs is\n     * enabled.\n     * This method will only be called when switching from separate RocksDB mode to unified mode.\n     * It opens the separate RocksDB in read-only mode, compares versions, and imports data if needed.\n     */\n    private void migrateFromSeparateRocksDBs() {\n        String separateRocksDBPath = rocksdbConfigFilePath(this.storePathRootDir, false);\n\n        // Check if separate RocksDB exists\n        if (!UtilAll.isPathExists(separateRocksDBPath)) {\n            log.info(\"Separate RocksDB for topics does not exist at {}, no migration needed\", separateRocksDBPath);\n            return;\n        }\n\n        log.info(\"Starting migration from separate RocksDB at {} to unified RocksDB\", separateRocksDBPath);\n\n        // Open separate RocksDB in read-only mode\n        RocksDBConfigManager separateRocksDBConfigManager = null;\n        try {\n            long memTableFlushIntervalMs = brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs();\n            org.rocksdb.CompressionType compressionType =\n                org.rocksdb.CompressionType.getCompressionType(brokerController.getMessageStoreConfig().getRocksdbCompressionType());\n\n            separateRocksDBConfigManager = new RocksDBConfigManager(separateRocksDBPath, memTableFlushIntervalMs,\n                compressionType);\n\n            // Initialize in read-only mode\n            if (!separateRocksDBConfigManager.init(true)) {\n                log.error(\"Failed to initialize separate RocksDB in read-only mode\");\n                return;\n            }\n\n            // Load data version from separate RocksDB\n            if (!separateRocksDBConfigManager.loadDataVersion()) {\n                log.error(\"Failed to load data version from separate RocksDB\");\n                return;\n            }\n\n            DataVersion separateDataVersion = separateRocksDBConfigManager.getKvDataVersion();\n            DataVersion unifiedDataVersion = this.getDataVersion();\n\n            log.info(\"Comparing data versions - Separate: {}, Unified: {}\", separateDataVersion, unifiedDataVersion);\n\n            // Compare versions and import if separate version is newer\n            if (separateDataVersion.getCounter().get() > unifiedDataVersion.getCounter().get()) {\n                log.info(\"Separate RocksDB has newer data, importing...\");\n\n                // Load topic configs from separate RocksDB\n                if (separateRocksDBConfigManager.loadData(this::importTopicConfig)) {\n                    log.info(\"Successfully imported topic configs from separate RocksDB\");\n\n                    this.getDataVersion().assignNewOne(separateDataVersion);\n                    this.getDataVersion().nextVersion(); // Make it one version higher\n                    updateDataVersion();\n                    log.info(\"Updated unified data version to {}\", this.getDataVersion());\n                } else {\n                    log.error(\"Failed to import topic configs from separate RocksDB\");\n                }\n            } else {\n                log.info(\"Unified RocksDB is already up-to-date, no migration needed\");\n            }\n        } catch (Exception e) {\n            log.error(\"Error during migration from separate RocksDB\", e);\n        } finally {\n            if (separateRocksDBConfigManager != null) {\n                try {\n                    separateRocksDBConfigManager.stop();\n                } catch (Exception e) {\n                    log.warn(\"Error stopping separate RocksDB config manager\", e);\n                }\n            }\n        }\n    }\n\n    /**\n     * Import a topic config from the separate RocksDB during migration\n     *\n     * @param key  The topic name bytes\n     * @param body The topic config data bytes\n     */\n    private void importTopicConfig(byte[] key, byte[] body) {\n        try {\n            decodeTopicConfig(key, body);\n            this.rocksDBConfigManager.put(key, body);\n        } catch (Exception e) {\n            log.error(\"Error importing topic config\", e);\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigHelper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.config.v2;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.google.common.base.Preconditions;\nimport io.netty.buffer.ByteBuf;\nimport io.netty.buffer.Unpooled;\nimport java.nio.charset.StandardCharsets;\nimport java.util.Optional;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.common.config.AbstractRocksDBStorage;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.rocksdb.RocksDBException;\nimport org.rocksdb.WriteBatch;\n\npublic class ConfigHelper {\n\n    /**\n     * <p>\n     * Layout of data version key:\n     * [table-prefix, 1 byte][table-id, 2 byte][record-prefix, 1 byte][data-version-bytes]\n     * </p>\n     *\n     * <p>\n     * Layout of data version value:\n     * [state-machine-version, 8 bytes][timestamp, 8 bytes][sequence counter, 8 bytes]\n     * </p>\n     *\n     * @throws RocksDBException if RocksDB raises an error\n     */\n    public static Optional<ByteBuf> loadDataVersion(ConfigStorage configStorage, TableId tableId)\n        throws RocksDBException {\n        int keyLen = 1 /* table-prefix */ + Short.BYTES /* table-id */ + 1 /* record-prefix */\n            + ConfigStorage.DATA_VERSION_KEY_BYTES.length;\n        ByteBuf keyBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);\n        try {\n            keyBuf.writeByte(TablePrefix.TABLE.getValue());\n            keyBuf.writeShort(tableId.getValue());\n            keyBuf.writeByte(RecordPrefix.DATA_VERSION.getValue());\n            keyBuf.writeBytes(ConfigStorage.DATA_VERSION_KEY_BYTES);\n            byte[] valueByes = configStorage.get(keyBuf.nioBuffer());\n            if (null != valueByes) {\n                ByteBuf valueBuf = Unpooled.wrappedBuffer(valueByes);\n                return Optional.of(valueBuf);\n            }\n        } finally {\n            keyBuf.release();\n        }\n        return Optional.empty();\n    }\n\n    public static void stampDataVersion(WriteBatch writeBatch, TableId table, DataVersion dataVersion, long stateMachineVersion)\n        throws RocksDBException {\n        // Increase data version\n        dataVersion.nextVersion(stateMachineVersion);\n\n        int keyLen = 1 /* table-prefix */ + Short.BYTES /* table-id */ + 1 /* record-prefix */\n            + ConfigStorage.DATA_VERSION_KEY_BYTES.length;\n        ByteBuf keyBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);\n        ByteBuf valueBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(Long.BYTES * 3);\n        try {\n            keyBuf.writeByte(TablePrefix.TABLE.getValue());\n            keyBuf.writeShort(table.getValue());\n            keyBuf.writeByte(RecordPrefix.DATA_VERSION.getValue());\n            keyBuf.writeBytes(ConfigStorage.DATA_VERSION_KEY_BYTES);\n            valueBuf.writeLong(dataVersion.getStateVersion());\n            valueBuf.writeLong(dataVersion.getTimestamp());\n            valueBuf.writeLong(dataVersion.getCounter().get());\n            writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer());\n        } finally {\n            keyBuf.release();\n            valueBuf.release();\n        }\n    }\n\n    public static void onDataVersionLoad(ByteBuf buf, DataVersion dataVersion) {\n        if (buf.readableBytes() == 8 /* state machine version */ + 8 /* timestamp */ + 8 /* counter */) {\n            long stateMachineVersion = buf.readLong();\n            long timestamp = buf.readLong();\n            long counter = buf.readLong();\n            dataVersion.setStateVersion(stateMachineVersion);\n            dataVersion.setTimestamp(timestamp);\n            dataVersion.setCounter(new AtomicLong(counter));\n        }\n        buf.release();\n    }\n\n    public static ByteBuf keyBufOf(TableId tableId, final String name) {\n        Preconditions.checkNotNull(name);\n        byte[] bytes = name.getBytes(StandardCharsets.UTF_8);\n        int keyLen = 1 /* table-prefix */ + 2 /* table-id */ + 1 /* record-type-prefix */ + 2 /* name-length */ + bytes.length;\n        ByteBuf keyBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);\n        keyBuf.writeByte(TablePrefix.TABLE.getValue());\n        keyBuf.writeShort(tableId.getValue());\n        keyBuf.writeByte(RecordPrefix.DATA.getValue());\n        keyBuf.writeShort(bytes.length);\n        keyBuf.writeBytes(bytes);\n        return keyBuf;\n    }\n\n    public static ByteBuf valueBufOf(final Object config, SerializationType serializationType) {\n        if (SerializationType.JSON == serializationType) {\n            byte[] payload = JSON.toJSONBytes(config);\n            ByteBuf valueBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(1 + payload.length);\n            valueBuf.writeByte(SerializationType.JSON.getValue());\n            valueBuf.writeBytes(payload);\n            return valueBuf;\n        }\n        throw new RuntimeException(\"Unsupported serialization type: \" + serializationType);\n    }\n\n    public static byte[] readBytes(final ByteBuf buf) {\n        byte[] bytes = new byte[buf.readableBytes()];\n        buf.readBytes(bytes);\n        return bytes;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigStorage.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.config.v2;\n\nimport com.google.common.base.Stopwatch;\nimport com.google.common.util.concurrent.ThreadFactoryBuilder;\nimport io.netty.buffer.PooledByteBufAllocatorMetric;\nimport io.netty.util.internal.PlatformDependent;\nimport java.io.File;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ScheduledThreadPoolExecutor;\nimport java.util.concurrent.ThreadFactory;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.config.AbstractRocksDBStorage;\nimport org.apache.rocketmq.common.config.ConfigHelper;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.rocksdb.ColumnFamilyDescriptor;\nimport org.rocksdb.ColumnFamilyOptions;\nimport org.rocksdb.FlushOptions;\nimport org.rocksdb.ReadOptions;\nimport org.rocksdb.RocksDB;\nimport org.rocksdb.RocksDBException;\nimport org.rocksdb.RocksIterator;\nimport org.rocksdb.Slice;\nimport org.rocksdb.WriteBatch;\nimport org.rocksdb.WriteOptions;\n\n/**\n * https://book.tidb.io/session1/chapter3/tidb-kv-to-relation.html\n */\npublic class ConfigStorage extends AbstractRocksDBStorage {\n\n    public static final String DATA_VERSION_KEY = \"data_version\";\n    public static final byte[] DATA_VERSION_KEY_BYTES = DATA_VERSION_KEY.getBytes(StandardCharsets.UTF_8);\n\n    private final ScheduledExecutorService scheduledExecutorService;\n\n    /**\n     * Number of write ops since previous flush.\n     */\n    private final AtomicInteger writeOpsCounter;\n\n    private final AtomicLong estimateWalFileSize = new AtomicLong(0L);\n\n    private final MessageStoreConfig messageStoreConfig;\n\n    private final FlushSyncService flushSyncService;\n\n    public ConfigStorage(MessageStoreConfig messageStoreConfig) {\n        super(messageStoreConfig.getStorePathRootDir() + File.separator + \"config\" + File.separator + \"rdb\");\n        this.messageStoreConfig = messageStoreConfig;\n        ThreadFactory threadFactory = new ThreadFactoryBuilder()\n            .setDaemon(true)\n            .setNameFormat(\"config-storage-%d\")\n            .build();\n        scheduledExecutorService = new ScheduledThreadPoolExecutor(1, threadFactory);\n        writeOpsCounter = new AtomicInteger(0);\n        this.flushSyncService = new FlushSyncService();\n        this.flushSyncService.setDaemon(true);\n    }\n\n    private void statNettyMemory() {\n        PooledByteBufAllocatorMetric metric = AbstractRocksDBStorage.POOLED_ALLOCATOR.metric();\n        LOGGER.info(\"Netty Memory Usage: {}\", metric);\n    }\n\n    @Override\n    public synchronized boolean start() {\n        boolean started = super.start();\n        if (started) {\n            scheduledExecutorService.scheduleWithFixedDelay(() -> statRocksdb(LOGGER), 1, 10, TimeUnit.SECONDS);\n            scheduledExecutorService.scheduleWithFixedDelay(this::statNettyMemory, 10, 10, TimeUnit.SECONDS);\n            this.flushSyncService.start();\n        } else {\n            LOGGER.error(\"Failed to start config storage\");\n        }\n        return started;\n    }\n\n    @Override\n    protected boolean postLoad() {\n        if (!PlatformDependent.hasUnsafe()) {\n            LOGGER.error(\"Unsafe not available and POOLED_ALLOCATOR cannot work correctly\");\n            return false;\n        }\n        try {\n            UtilAll.ensureDirOK(this.dbPath);\n            initOptions();\n            List<ColumnFamilyDescriptor> cfDescriptors = new ArrayList<>();\n\n            ColumnFamilyOptions defaultOptions = ConfigHelper.createConfigColumnFamilyOptions();\n            this.cfOptions.add(defaultOptions);\n            cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, defaultOptions));\n\n            // Start RocksDB instance\n            open(cfDescriptors);\n\n            this.defaultCFHandle = cfHandles.get(0);\n        } catch (Exception e) {\n            AbstractRocksDBStorage.LOGGER.error(\"postLoad Failed. {}\", this.dbPath, e);\n            return false;\n        }\n        return true;\n    }\n\n    @Override\n    protected void preShutdown() {\n        scheduledExecutorService.shutdown();\n        flushSyncService.shutdown();\n    }\n\n    protected void initOptions() {\n        this.options = ConfigHelper.createConfigDBOptions();\n        super.initOptions();\n    }\n\n    @Override\n    protected void initAbleWalWriteOptions() {\n        this.ableWalWriteOptions = new WriteOptions();\n\n        // Given that fdatasync is kind of expensive, sync-WAL for every write cannot be afforded.\n        this.ableWalWriteOptions.setSync(false);\n\n        // We need WAL for config changes\n        this.ableWalWriteOptions.setDisableWAL(false);\n\n        // No fast failure on block, wait synchronously even if there is wait for the write request\n        this.ableWalWriteOptions.setNoSlowdown(false);\n    }\n\n    public byte[] get(ByteBuffer key) throws RocksDBException {\n        byte[] keyBytes = new byte[key.remaining()];\n        key.get(keyBytes);\n        return super.get(getDefaultCFHandle(), totalOrderReadOptions, keyBytes);\n    }\n\n    public void write(WriteBatch writeBatch) throws RocksDBException {\n        db.write(ableWalWriteOptions, writeBatch);\n        accountWriteOps(writeBatch.getDataSize());\n    }\n\n    private void accountWriteOps(long dataSize) {\n        writeOpsCounter.incrementAndGet();\n        estimateWalFileSize.addAndGet(dataSize);\n    }\n\n    public RocksIterator iterate(ByteBuffer beginKey, ByteBuffer endKey) {\n        try (ReadOptions readOptions = new ReadOptions()) {\n            readOptions.setTotalOrderSeek(true);\n            readOptions.setTailing(false);\n            readOptions.setAutoPrefixMode(true);\n            // Use DirectSlice till the follow issue is fixed:\n            // https://github.com/facebook/rocksdb/issues/13098\n            //\n            // readOptions.setIterateUpperBound(new DirectSlice(endKey));\n            byte[] buf = new byte[endKey.remaining()];\n            endKey.slice().get(buf);\n            readOptions.setIterateUpperBound(new Slice(buf));\n\n            RocksIterator iterator = db.newIterator(defaultCFHandle, readOptions);\n            iterator.seek(beginKey.slice());\n            return iterator;\n        }\n    }\n\n    /**\n     * RocksDB writes contain 3 stages: application memory buffer --> OS Page Cache --> Disk.\n     * Given that we are having DBOptions::manual_wal_flush, we need to manually call DB::FlushWAL and DB::SyncWAL\n     * Note: DB::FlushWAL(true) will internally call DB::SyncWAL.\n     * <p>\n     * See <a href=\"https://rocksdb.org/blog/2017/08/25/flushwal.html\">Flush And Sync WAL</a>\n     */\n    class FlushSyncService extends ServiceThread {\n\n        private long lastSyncTime = 0;\n\n        private static final long MAX_SYNC_INTERVAL_IN_MILLIS = 100;\n\n        private final Stopwatch stopwatch = Stopwatch.createUnstarted();\n\n        private final FlushOptions flushOptions = new FlushOptions();\n\n        @Override\n        public String getServiceName() {\n            return \"FlushSyncService\";\n        }\n\n        @Override\n        public void run() {\n            flushOptions.setAllowWriteStall(false);\n            flushOptions.setWaitForFlush(true);\n            log.info(\"{} service started\", this.getServiceName());\n            while (!this.isStopped()) {\n                try {\n                    this.waitForRunning(10);\n                    this.flushAndSyncWAL(false);\n                } catch (Exception e) {\n                    log.warn(\"{} service has exception. \", this.getServiceName(), e);\n                }\n            }\n            try {\n                flushAndSyncWAL(true);\n            } catch (Exception e) {\n                log.warn(\"{} raised an exception while performing flush-and-sync WAL on exit\",\n                    this.getServiceName(), e);\n            }\n            flushOptions.close();\n            log.info(\"{} service end\", this.getServiceName());\n        }\n\n        private void flushAndSyncWAL(boolean onExit) throws RocksDBException {\n            int writeOps = writeOpsCounter.get();\n            if (0 == writeOps) {\n                // No write ops to flush\n                return;\n            }\n\n            /*\n             * Normally, when MemTables become full then immutable, RocksDB threads will automatically flush them to L0\n             * SST files. The use case here is different: the MemTable may never get full and immutable given that the\n             * volume of data involved is relatively small. Further, we are constantly modifying the key-value pairs and\n             * generating WAL entries. The WAL file size can grow up to dozens of gigabytes without manual triggering of\n             * flush.\n             */\n            if (ConfigStorage.this.estimateWalFileSize.get() >= messageStoreConfig.getRocksdbWalFileRollingThreshold()) {\n                ConfigStorage.this.flush(flushOptions);\n                estimateWalFileSize.set(0L);\n            }\n\n            // Flush and Sync WAL if we have committed enough writes\n            if (writeOps >= messageStoreConfig.getRocksdbFlushWalFrequency() || onExit) {\n                stopwatch.reset().start();\n                ConfigStorage.this.db.flushWal(true);\n                long elapsed = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS);\n                writeOpsCounter.getAndAdd(-writeOps);\n                lastSyncTime = System.currentTimeMillis();\n                LOGGER.debug(\"Flush and Sync WAL of RocksDB[{}] costs {}ms, write-ops={}\", dbPath, elapsed, writeOps);\n                return;\n            }\n            // Flush and Sync WAL if some writes are out there for a period of time\n            long elapsedTime = System.currentTimeMillis() - lastSyncTime;\n            if (elapsedTime > MAX_SYNC_INTERVAL_IN_MILLIS) {\n                stopwatch.reset().start();\n                ConfigStorage.this.db.flushWal(true);\n                long elapsed = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS);\n                LOGGER.debug(\"Flush and Sync WAL of RocksDB[{}] costs {}ms, write-ops={}\", dbPath, elapsed, writeOps);\n                writeOpsCounter.getAndAdd(-writeOps);\n                lastSyncTime = System.currentTimeMillis();\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.config.v2;\n\nimport com.google.common.base.Strings;\nimport io.netty.buffer.ByteBuf;\nimport io.netty.util.internal.PlatformDependent;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.config.AbstractRocksDBStorage;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.rocksdb.RocksDBException;\nimport org.rocksdb.RocksIterator;\nimport org.rocksdb.WriteBatch;\n\n/**\n * <p>\n * Layout of consumer offset key:\n * [table-prefix, 1 byte][table-id, 2 bytes][record-prefix, 1 byte][group-len, 2 bytes][group bytes][CTRL_1, 1 byte]\n * [topic-len, 2 bytes][topic bytes][CTRL_1, 1 byte][queue-id, 4 bytes]\n * </p>\n *\n * <p>\n * Layout of consumer offset value: [offset, 8 bytes]\n * </p>\n */\npublic class ConsumerOffsetManagerV2 extends ConsumerOffsetManager {\n\n    private final ConfigStorage configStorage;\n\n    public ConsumerOffsetManagerV2(BrokerController brokerController, ConfigStorage configStorage) {\n        super(brokerController);\n        this.configStorage = configStorage;\n    }\n\n    @Override\n    public void removeConsumerOffset(String topicAtGroup) {\n        if (!MixAll.isLmq(topicAtGroup)) {\n            super.removeConsumerOffset(topicAtGroup);\n        }\n\n        String[] topicGroup = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);\n        if (topicGroup.length != 2) {\n            LOG.error(\"Invalid topic group: {}\", topicAtGroup);\n            return;\n        }\n\n        byte[] topicBytes = topicGroup[0].getBytes(StandardCharsets.UTF_8);\n        byte[] groupBytes = topicGroup[1].getBytes(StandardCharsets.UTF_8);\n\n        int keyLen = 1 /* table-prefix */ + Short.BYTES /* table-id */ + 1 /* record-prefix */\n            + Short.BYTES /* group-len */ + groupBytes.length + 1 /* CTRL_1 */\n            + Short.BYTES + topicBytes.length + 1;\n        // [table-prefix, 1 byte][table-id, 2 bytes][record-prefix, 1 byte][group-len, 2 bytes][group-bytes][CTRL_1, 1 byte]\n        // [topic-len, 2 bytes][topic-bytes][CTRL_1]\n        ByteBuf beginKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);\n        beginKey.writeByte(TablePrefix.TABLE.getValue());\n        beginKey.writeShort(TableId.CONSUMER_OFFSET.getValue());\n        beginKey.writeByte(RecordPrefix.DATA.getValue());\n        beginKey.writeShort(groupBytes.length);\n        beginKey.writeBytes(groupBytes);\n        beginKey.writeByte(AbstractRocksDBStorage.CTRL_1);\n        beginKey.writeShort(topicBytes.length);\n        beginKey.writeBytes(topicBytes);\n        beginKey.writeByte(AbstractRocksDBStorage.CTRL_1);\n\n        ByteBuf endKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);\n        endKey.writeByte(TablePrefix.TABLE.getValue());\n        endKey.writeShort(TableId.CONSUMER_OFFSET.getValue());\n        endKey.writeByte(RecordPrefix.DATA.getValue());\n        endKey.writeShort(groupBytes.length);\n        endKey.writeBytes(groupBytes);\n        endKey.writeByte(AbstractRocksDBStorage.CTRL_1);\n        endKey.writeShort(topicBytes.length);\n        endKey.writeBytes(topicBytes);\n        endKey.writeByte(AbstractRocksDBStorage.CTRL_2);\n\n        try (WriteBatch writeBatch = new WriteBatch()) {\n            // TODO: we have to make a copy here as WriteBatch lacks ByteBuffer API here\n            writeBatch.deleteRange(ConfigHelper.readBytes(beginKey), ConfigHelper.readBytes(endKey));\n            long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;\n            ConfigHelper.stampDataVersion(writeBatch, TableId.CONSUMER_OFFSET, dataVersion, stateMachineVersion);\n            configStorage.write(writeBatch);\n        } catch (RocksDBException e) {\n            LOG.error(\"Failed to removeConsumerOffset, topicAtGroup={}\", topicAtGroup, e);\n        } finally {\n            beginKey.release();\n            endKey.release();\n        }\n    }\n\n    @Override\n    public void removeOffset(String group) {\n        if (!MixAll.isLmq(group)) {\n            super.removeOffset(group);\n        }\n\n        byte[] groupBytes = group.getBytes(StandardCharsets.UTF_8);\n        int keyLen = 1 /* table-prefix */ + Short.BYTES /* table-id */ + 1 /* record-prefix */\n            + Short.BYTES /* group-len */ + groupBytes.length + 1 /* CTRL_1 */;\n\n        // [table-prefix, 1 byte][table-id, 2 bytes][record-prefix, 1 byte][group-len, 2 bytes][group bytes][CTRL_1, 1 byte]\n        ByteBuf consumerOffsetBeginKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);\n        consumerOffsetBeginKey.writeByte(TablePrefix.TABLE.getValue());\n        consumerOffsetBeginKey.writeShort(TableId.CONSUMER_OFFSET.getValue());\n        consumerOffsetBeginKey.writeByte(RecordPrefix.DATA.getValue());\n        consumerOffsetBeginKey.writeShort(groupBytes.length);\n        consumerOffsetBeginKey.writeBytes(groupBytes);\n        consumerOffsetBeginKey.writeByte(AbstractRocksDBStorage.CTRL_1);\n\n        ByteBuf consumerOffsetEndKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);\n        consumerOffsetEndKey.writeByte(TablePrefix.TABLE.getValue());\n        consumerOffsetEndKey.writeShort(TableId.CONSUMER_OFFSET.getValue());\n        consumerOffsetEndKey.writeByte(RecordPrefix.DATA.getValue());\n        consumerOffsetEndKey.writeShort(groupBytes.length);\n        consumerOffsetEndKey.writeBytes(groupBytes);\n        consumerOffsetEndKey.writeByte(AbstractRocksDBStorage.CTRL_2);\n\n        // [table-prefix, 1 byte][table-id, 2 bytes][record-prefix, 1 byte][group-len, 2 bytes][group bytes][CTRL_1, 1 byte]\n        ByteBuf pullOffsetBeginKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);\n        pullOffsetBeginKey.writeByte(TablePrefix.TABLE.getValue());\n        pullOffsetBeginKey.writeShort(TableId.PULL_OFFSET.getValue());\n        pullOffsetBeginKey.writeByte(RecordPrefix.DATA.getValue());\n        pullOffsetBeginKey.writeShort(groupBytes.length);\n        pullOffsetBeginKey.writeBytes(groupBytes);\n        pullOffsetBeginKey.writeByte(AbstractRocksDBStorage.CTRL_1);\n\n        ByteBuf pullOffsetEndKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);\n        pullOffsetEndKey.writeByte(TablePrefix.TABLE.getValue());\n        pullOffsetEndKey.writeShort(TableId.PULL_OFFSET.getValue());\n        pullOffsetEndKey.writeByte(RecordPrefix.DATA.getValue());\n        pullOffsetEndKey.writeShort(groupBytes.length);\n        pullOffsetEndKey.writeBytes(groupBytes);\n        pullOffsetEndKey.writeByte(AbstractRocksDBStorage.CTRL_2);\n        try (WriteBatch writeBatch = new WriteBatch()) {\n            // TODO: we have to make a copy here as WriteBatch lacks ByteBuffer API here\n            writeBatch.deleteRange(ConfigHelper.readBytes(consumerOffsetBeginKey), ConfigHelper.readBytes(consumerOffsetEndKey));\n            writeBatch.deleteRange(ConfigHelper.readBytes(pullOffsetBeginKey), ConfigHelper.readBytes(pullOffsetEndKey));\n            MessageStore messageStore = brokerController.getMessageStore();\n            long stateMachineVersion = messageStore != null ? messageStore.getStateMachineVersion() : 0;\n            ConfigHelper.stampDataVersion(writeBatch, TableId.CONSUMER_OFFSET, dataVersion, stateMachineVersion);\n            ConfigHelper.stampDataVersion(writeBatch, TableId.PULL_OFFSET, dataVersion, stateMachineVersion);\n            configStorage.write(writeBatch);\n        } catch (RocksDBException e) {\n            LOG.error(\"Failed to consumer offsets by group={}\", group, e);\n        } finally {\n            consumerOffsetBeginKey.release();\n            consumerOffsetEndKey.release();\n            pullOffsetBeginKey.release();\n            pullOffsetEndKey.release();\n        }\n    }\n\n    /**\n     * <p>\n     * Layout of consumer offset key:\n     * [table-prefix, 1 byte][table-id, 2 bytes][record-prefix, 1 byte][group-len, 2 bytes][group bytes][CTRL_1, 1 byte]\n     * [topic-len, 2 bytes][topic bytes][CTRL_1, 1 byte][queue-id, 4 bytes]\n     * </p>\n     *\n     * <p>\n     * Layout of consumer offset value:\n     * [offset, 8 bytes]\n     * </p>\n     *\n     * @param clientHost The client that submits consumer offsets\n     * @param group      Group name\n     * @param topic      Topic name\n     * @param queueId    Queue ID\n     * @param offset     Consumer offset of the specified queue\n     */\n    @Override\n    public void commitOffset(String clientHost, String group, String topic, int queueId, long offset) {\n        String key = topic + TOPIC_GROUP_SEPARATOR + group;\n\n        // We maintain a copy of classic consumer offset table in memory as they take very limited memory footprint.\n        // For LMQ offsets, given the volume and number of these type of records, they are maintained in RocksDB\n        // directly. Frequently used LMQ consumer offsets should reside either in block-cache or MemTable, so read/write\n        // should be blazingly fast.\n        if (!MixAll.isLmq(topic)) {\n            if (offsetTable.containsKey(key)) {\n                offsetTable.get(key).put(queueId, offset);\n            } else {\n                ConcurrentMap<Integer, Long> map = new ConcurrentHashMap<>();\n                ConcurrentMap<Integer, Long> prev = offsetTable.putIfAbsent(key, map);\n                if (null != prev) {\n                    map = prev;\n                }\n                map.put(queueId, offset);\n            }\n        }\n\n        ByteBuf keyBuf = keyOfConsumerOffset(group, topic, queueId);\n        ByteBuf valueBuf = ConfigStorage.POOLED_ALLOCATOR.buffer(Long.BYTES);\n        try (WriteBatch writeBatch = new WriteBatch()) {\n            valueBuf.writeLong(offset);\n            writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer());\n            MessageStore messageStore = brokerController.getMessageStore();\n            long stateMachineVersion = messageStore != null ? messageStore.getStateMachineVersion() : 0;\n            ConfigHelper.stampDataVersion(writeBatch, TableId.CONSUMER_OFFSET, dataVersion, stateMachineVersion);\n            configStorage.write(writeBatch);\n        } catch (RocksDBException e) {\n            LOG.error(\"Failed to commit consumer offset\", e);\n        } finally {\n            keyBuf.release();\n            valueBuf.release();\n        }\n    }\n\n    private ByteBuf keyOfConsumerOffset(String group, String topic, int queueId) {\n        byte[] groupBytes = group.getBytes(StandardCharsets.UTF_8);\n        byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);\n        int keyLen = 1 /*table prefix*/ + Short.BYTES /*table-id*/ + 1 /*record-prefix*/\n            + Short.BYTES /*group-len*/ + groupBytes.length + 1 /*CTRL_1*/\n            + 2 /*topic-len*/ + topicBytes.length + 1 /* CTRL_1*/\n            + Integer.BYTES /*queue-id*/;\n        ByteBuf keyBuf = ConfigStorage.POOLED_ALLOCATOR.buffer(keyLen);\n        keyBuf.writeByte(TablePrefix.TABLE.getValue());\n        keyBuf.writeShort(TableId.CONSUMER_OFFSET.getValue());\n        keyBuf.writeByte(RecordPrefix.DATA.getValue());\n        keyBuf.writeShort(groupBytes.length);\n        keyBuf.writeBytes(groupBytes);\n        keyBuf.writeByte(AbstractRocksDBStorage.CTRL_1);\n        keyBuf.writeShort(topicBytes.length);\n        keyBuf.writeBytes(topicBytes);\n        keyBuf.writeByte(AbstractRocksDBStorage.CTRL_1);\n        keyBuf.writeInt(queueId);\n        return keyBuf;\n    }\n\n    private ByteBuf keyOfPullOffset(String group, String topic, int queueId) {\n        byte[] groupBytes = group.getBytes(StandardCharsets.UTF_8);\n        byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);\n        int keyLen = 1 /*table prefix*/ + Short.BYTES /*table-id*/ + 1 /*record-prefix*/\n            + Short.BYTES /*group-len*/ + groupBytes.length + 1 /*CTRL_1*/\n            + 2 /*topic-len*/ + topicBytes.length + 1 /* CTRL_1*/\n            + Integer.BYTES /*queue-id*/;\n        ByteBuf keyBuf = ConfigStorage.POOLED_ALLOCATOR.buffer(keyLen);\n        keyBuf.writeByte(TablePrefix.TABLE.getValue());\n        keyBuf.writeShort(TableId.PULL_OFFSET.getValue());\n        keyBuf.writeByte(RecordPrefix.DATA.getValue());\n        keyBuf.writeShort(groupBytes.length);\n        keyBuf.writeBytes(groupBytes);\n        keyBuf.writeByte(AbstractRocksDBStorage.CTRL_1);\n        keyBuf.writeShort(topicBytes.length);\n        keyBuf.writeBytes(topicBytes);\n        keyBuf.writeByte(AbstractRocksDBStorage.CTRL_1);\n        keyBuf.writeInt(queueId);\n        return keyBuf;\n    }\n\n    @Override\n    public boolean load() {\n        return loadDataVersion() && loadConsumerOffsets();\n    }\n\n    @Override\n    public synchronized void persist() {\n        try {\n            configStorage.flushWAL();\n        } catch (RocksDBException e) {\n            LOG.error(\"Failed to flush RocksDB config instance WAL\", e);\n        }\n    }\n\n    /**\n     * <p>\n     * Layout of data version key:\n     * [table-prefix, 1 byte][table-id, 2 byte][record-prefix, 1 byte][data-version-bytes]\n     * </p>\n     *\n     * <p>\n     * Layout of data version value:\n     * [state-machine-version, 8 bytes][timestamp, 8 bytes][sequence counter, 8 bytes]\n     * </p>\n     */\n    public boolean loadDataVersion() {\n        try {\n            ConfigHelper.loadDataVersion(configStorage, TableId.CONSUMER_OFFSET)\n                .ifPresent(buf -> ConfigHelper.onDataVersionLoad(buf, dataVersion));\n        } catch (RocksDBException e) {\n            LOG.error(\"Failed to load RocksDB config\", e);\n            return false;\n        }\n        return true;\n    }\n\n    private boolean loadConsumerOffsets() {\n        // [table-prefix, 1 byte][table-id, 2 bytes][record-prefix, 1 byte]\n        ByteBuf beginKeyBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(4);\n        beginKeyBuf.writeByte(TablePrefix.TABLE.getValue());\n        beginKeyBuf.writeShort(TableId.CONSUMER_OFFSET.getValue());\n        beginKeyBuf.writeByte(RecordPrefix.DATA.getValue());\n\n        ByteBuf endKeyBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(4);\n        endKeyBuf.writeByte(TablePrefix.TABLE.getValue());\n        endKeyBuf.writeShort(TableId.CONSUMER_OFFSET.getValue());\n        endKeyBuf.writeByte(RecordPrefix.DATA.getValue() + 1);\n\n        try (RocksIterator iterator = configStorage.iterate(beginKeyBuf.nioBuffer(), endKeyBuf.nioBuffer())) {\n            int keyCapacity = 256;\n            // We may iterate millions of LMQ consumer offsets here, use direct byte buffers here to avoid memory\n            // fragment\n            ByteBuffer keyBuffer = ByteBuffer.allocateDirect(keyCapacity);\n            ByteBuffer valueBuffer = ByteBuffer.allocateDirect(Long.BYTES);\n            while (iterator.isValid()) {\n                keyBuffer.clear();\n                valueBuffer.clear();\n\n                int len = iterator.key(keyBuffer);\n                if (len > keyCapacity) {\n                    keyCapacity = len;\n                    PlatformDependent.freeDirectBuffer(keyBuffer);\n                    // Reserve more space for key\n                    keyBuffer = ByteBuffer.allocateDirect(keyCapacity);\n                    continue;\n                }\n                len = iterator.value(valueBuffer);\n                assert len == Long.BYTES;\n\n                // skip table-prefix, table-id, record-prefix\n                keyBuffer.position(1 + 2 + 1);\n                short groupLen = keyBuffer.getShort();\n                byte[] groupBytes = new byte[groupLen];\n                keyBuffer.get(groupBytes);\n                byte ctrl = keyBuffer.get();\n                assert ctrl == AbstractRocksDBStorage.CTRL_1;\n\n                short topicLen = keyBuffer.getShort();\n                byte[] topicBytes = new byte[topicLen];\n                keyBuffer.get(topicBytes);\n                String topic = new String(topicBytes, StandardCharsets.UTF_8);\n                ctrl = keyBuffer.get();\n                assert ctrl == AbstractRocksDBStorage.CTRL_1;\n\n                int queueId = keyBuffer.getInt();\n\n                long offset = valueBuffer.getLong();\n\n                if (!MixAll.isLmq(topic)) {\n                    String group = new String(groupBytes, StandardCharsets.UTF_8);\n                    onConsumerOffsetRecordLoad(topic, group, queueId, offset);\n                }\n                iterator.next();\n            }\n            PlatformDependent.freeDirectBuffer(keyBuffer);\n            PlatformDependent.freeDirectBuffer(valueBuffer);\n        } finally {\n            beginKeyBuf.release();\n            endKeyBuf.release();\n        }\n        return true;\n    }\n\n    private void onConsumerOffsetRecordLoad(String topic, String group, int queueId, long offset) {\n        if (MixAll.isLmq(topic)) {\n            return;\n        }\n        String key = topic + TOPIC_GROUP_SEPARATOR + group;\n        if (!offsetTable.containsKey(key)) {\n            ConcurrentMap<Integer, Long> map = new ConcurrentHashMap<>();\n            offsetTable.putIfAbsent(key, map);\n        }\n        offsetTable.get(key).put(queueId, offset);\n    }\n\n    @Override\n    public long queryOffset(String group, String topic, int queueId) {\n        if (!MixAll.isLmq(topic)) {\n            return super.queryOffset(group, topic, queueId);\n        }\n\n        ByteBuf keyBuf = keyOfConsumerOffset(group, topic, queueId);\n        try {\n            byte[] slice = configStorage.get(keyBuf.nioBuffer());\n            if (null == slice) {\n                return -1;\n            }\n            assert slice.length == Long.BYTES;\n            return ByteBuffer.wrap(slice).getLong();\n        } catch (RocksDBException e) {\n            throw new RuntimeException(e);\n        } finally {\n            keyBuf.release();\n        }\n    }\n\n    @Override\n    public void commitPullOffset(String clientHost, String group, String topic, int queueId, long offset) {\n        if (!MixAll.isLmq(topic)) {\n            super.commitPullOffset(clientHost, group, topic, queueId, offset);\n        }\n\n        ByteBuf keyBuf = keyOfPullOffset(group, topic, queueId);\n        ByteBuf valueBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(8);\n        valueBuf.writeLong(offset);\n        try (WriteBatch writeBatch = new WriteBatch()) {\n            writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer());\n            long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;\n            ConfigHelper.stampDataVersion(writeBatch, TableId.PULL_OFFSET, dataVersion, stateMachineVersion);\n            configStorage.write(writeBatch);\n        } catch (RocksDBException e) {\n            LOG.error(\"Failed to commit pull offset. group={}, topic={}, queueId={}, offset={}\",\n                group, topic, queueId, offset);\n        } finally {\n            keyBuf.release();\n            valueBuf.release();\n        }\n    }\n\n    @Override\n    public long queryPullOffset(String group, String topic, int queueId) {\n        if (!MixAll.isLmq(topic)) {\n            return super.queryPullOffset(group, topic, queueId);\n        }\n\n        ByteBuf keyBuf = keyOfPullOffset(group, topic, queueId);\n        try {\n            byte[] valueBytes = configStorage.get(keyBuf.nioBuffer());\n            if (null == valueBytes) {\n                return -1;\n            }\n            return ByteBuffer.wrap(valueBytes).getLong();\n        } catch (RocksDBException e) {\n            LOG.error(\"Failed to queryPullOffset. group={}, topic={}, queueId={}\", group, topic, queueId);\n        } finally {\n            keyBuf.release();\n        }\n        return -1;\n    }\n\n    @Override\n    public void assignResetOffset(String topic, String group, int queueId, long offset) {\n        if (Strings.isNullOrEmpty(topic) || Strings.isNullOrEmpty(group) || queueId < 0 || offset < 0) {\n            LOG.warn(\"Illegal arguments when assigning reset offset. Topic={}, group={}, queueId={}, offset={}\",\n                    topic, group, queueId, offset);\n            return;\n        }\n        if (!MixAll.isLmq(topic) || !MixAll.isLmq(group)) {\n            super.assignResetOffset(topic, group, queueId, offset);\n        } else {\n            String key = topic + TOPIC_GROUP_SEPARATOR + group;\n            ConcurrentMap<Integer, Long> map = resetOffsetTable.get(key);\n            if (null == map) {\n                map = new ConcurrentHashMap<>();\n                ConcurrentMap<Integer, Long> previous = resetOffsetTable.putIfAbsent(key, map);\n                if (null != previous) {\n                    map = previous;\n                }\n            }\n            map.put(queueId, offset);\n        }\n\n        this.commitOffset(null, topic, group, queueId, offset);\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/config/v2/RecordPrefix.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.config.v2;\n\npublic enum RecordPrefix {\n    UNSPECIFIED((byte)0),\n    DATA_VERSION((byte)1),\n    DATA((byte)2);\n\n    private final byte value;\n\n    RecordPrefix(byte value) {\n        this.value = value;\n    }\n\n    public byte getValue() {\n        return value;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/config/v2/SerializationType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.config.v2;\n\npublic enum SerializationType {\n    UNSPECIFIED((byte) 0),\n\n    JSON((byte) 1),\n\n    PROTOBUF((byte) 2),\n\n    FLAT_BUFFERS((byte) 3);\n\n    private final byte value;\n\n    SerializationType(byte value) {\n        this.value = value;\n    }\n\n    public byte getValue() {\n        return value;\n    }\n\n    public static SerializationType valueOf(byte value) {\n        for (SerializationType type : SerializationType.values()) {\n            if (type.getValue() == value) {\n                return type;\n            }\n        }\n        return SerializationType.UNSPECIFIED;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.config.v2;\n\nimport com.alibaba.fastjson2.JSON;\nimport io.netty.buffer.ByteBuf;\nimport io.netty.buffer.Unpooled;\nimport java.nio.charset.StandardCharsets;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.config.AbstractRocksDBStorage;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.rocksdb.RocksDBException;\nimport org.rocksdb.RocksIterator;\nimport org.rocksdb.WriteBatch;\n\npublic class SubscriptionGroupManagerV2 extends SubscriptionGroupManager {\n\n    private final ConfigStorage configStorage;\n\n    public SubscriptionGroupManagerV2(BrokerController brokerController, ConfigStorage configStorage) {\n        super(brokerController);\n        this.configStorage = configStorage;\n    }\n\n    @Override\n    public boolean load() {\n        return loadDataVersion() && loadSubscriptions();\n    }\n\n    public boolean loadDataVersion() {\n        try {\n            ConfigHelper.loadDataVersion(configStorage, TableId.SUBSCRIPTION_GROUP)\n                .ifPresent(buf -> {\n                    ConfigHelper.onDataVersionLoad(buf, dataVersion);\n                });\n        } catch (RocksDBException e) {\n            log.error(\"loadDataVersion error\", e);\n            return false;\n        }\n        return true;\n    }\n\n    private boolean loadSubscriptions() {\n        int keyLen = 1 /* table prefix */ + 2 /* table-id */ + 1 /* record-type-prefix */;\n        ByteBuf beginKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);\n        beginKey.writeByte(TablePrefix.TABLE.getValue());\n        beginKey.writeShort(TableId.SUBSCRIPTION_GROUP.getValue());\n        beginKey.writeByte(RecordPrefix.DATA.getValue());\n\n        ByteBuf endKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);\n        endKey.writeByte(TablePrefix.TABLE.getValue());\n        endKey.writeShort(TableId.SUBSCRIPTION_GROUP.getValue());\n        endKey.writeByte(RecordPrefix.DATA.getValue() + 1);\n\n        try (RocksIterator iterator = configStorage.iterate(beginKey.nioBuffer(), endKey.nioBuffer())) {\n            while (iterator.isValid()) {\n                SubscriptionGroupConfig subscriptionGroupConfig = parseSubscription(iterator.key(), iterator.value());\n                if (null != subscriptionGroupConfig) {\n                    super.putSubscriptionGroupConfig(subscriptionGroupConfig);\n                }\n                iterator.next();\n            }\n        } finally {\n            beginKey.release();\n            endKey.release();\n        }\n        return true;\n    }\n\n    private SubscriptionGroupConfig parseSubscription(byte[] key, byte[] value) {\n        ByteBuf keyBuf = Unpooled.wrappedBuffer(key);\n        ByteBuf valueBuf = Unpooled.wrappedBuffer(value);\n        try {\n            // Skip table-prefix, table-id, record-type-prefix\n            keyBuf.readerIndex(4);\n            short groupNameLen = keyBuf.readShort();\n            assert groupNameLen == keyBuf.readableBytes();\n            CharSequence groupName = keyBuf.readCharSequence(groupNameLen, StandardCharsets.UTF_8);\n            assert null != groupName;\n            byte serializationType = valueBuf.readByte();\n            if (SerializationType.JSON == SerializationType.valueOf(serializationType)) {\n                CharSequence json = valueBuf.readCharSequence(valueBuf.readableBytes(), StandardCharsets.UTF_8);\n                SubscriptionGroupConfig subscriptionGroupConfig = JSON.parseObject(json.toString(), SubscriptionGroupConfig.class);\n                assert subscriptionGroupConfig != null;\n                assert groupName.equals(subscriptionGroupConfig.getGroupName());\n                return subscriptionGroupConfig;\n            }\n        } finally {\n            keyBuf.release();\n            valueBuf.release();\n        }\n        return null;\n    }\n\n    @Override\n    public synchronized void persist() {\n        try {\n            configStorage.flushWAL();\n        } catch (RocksDBException e) {\n            log.error(\"Failed to flush RocksDB WAL\", e);\n        }\n    }\n\n    @Override\n    public SubscriptionGroupConfig findSubscriptionGroupConfig(final String group) {\n        if (MixAll.isLmq(group)) {\n            SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n            subscriptionGroupConfig.setGroupName(group);\n            return subscriptionGroupConfig;\n        }\n        return super.findSubscriptionGroupConfig(group);\n    }\n\n    @Override\n    public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config) {\n        if (config == null || MixAll.isLmq(config.getGroupName())) {\n            return;\n        }\n        super.updateSubscriptionGroupConfigWithoutPersist(config);\n        ByteBuf keyBuf = ConfigHelper.keyBufOf(TableId.SUBSCRIPTION_GROUP, config.getGroupName());\n        ByteBuf valueBuf = ConfigHelper.valueBufOf(config, SerializationType.JSON);\n        try (WriteBatch writeBatch = new WriteBatch()) {\n            writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer());\n            long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;\n            ConfigHelper.stampDataVersion(writeBatch, TableId.SUBSCRIPTION_GROUP, dataVersion, stateMachineVersion);\n            configStorage.write(writeBatch);\n            // fdatasync on core metadata change\n            persist();\n        } catch (RocksDBException e) {\n            log.error(\"update subscription group config error\", e);\n        } finally {\n            keyBuf.release();\n            valueBuf.release();\n        }\n    }\n\n    @Override\n    public boolean containsSubscriptionGroup(String group) {\n        if (MixAll.isLmq(group)) {\n            return true;\n        } else {\n            return super.containsSubscriptionGroup(group);\n        }\n    }\n\n    @Override\n    protected SubscriptionGroupConfig removeSubscriptionGroupConfig(String groupName) {\n        ByteBuf keyBuf = ConfigHelper.keyBufOf(TableId.SUBSCRIPTION_GROUP, groupName);\n        try (WriteBatch writeBatch = new WriteBatch()) {\n            writeBatch.delete(ConfigHelper.readBytes(keyBuf));\n            long stateMachineVersion = brokerController.getMessageStore().getStateMachineVersion();\n            ConfigHelper.stampDataVersion(writeBatch, TableId.SUBSCRIPTION_GROUP, dataVersion, stateMachineVersion);\n            configStorage.write(writeBatch);\n        } catch (RocksDBException e) {\n            log.error(\"Failed to remove subscription group config by group-name={}\", groupName, e);\n        }\n        return super.removeSubscriptionGroupConfig(groupName);\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/config/v2/TableId.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.config.v2;\n\n/**\n * See <a href=\"https://book.tidb.io/session1/chapter3/tidb-kv-to-relation.html\">Table, Key Value Mapping</a>\n */\npublic enum TableId {\n    UNSPECIFIED((short) 0),\n    CONSUMER_OFFSET((short) 1),\n    PULL_OFFSET((short) 2),\n    TOPIC((short) 3),\n    SUBSCRIPTION_GROUP((short) 4);\n\n    private final short value;\n\n    TableId(short value) {\n        this.value = value;\n    }\n\n    public short getValue() {\n        return value;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/config/v2/TablePrefix.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.config.v2;\n\npublic enum TablePrefix {\n    UNSPECIFIED((byte) 0),\n    TABLE((byte) 1);\n\n    private final byte value;\n\n    TablePrefix(byte value) {\n        this.value = value;\n    }\n\n    public byte getValue() {\n        return value;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.config.v2;\n\nimport com.alibaba.fastjson2.JSON;\nimport io.netty.buffer.ByteBuf;\nimport io.netty.buffer.Unpooled;\nimport java.nio.charset.StandardCharsets;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.config.AbstractRocksDBStorage;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.rocksdb.RocksDBException;\nimport org.rocksdb.RocksIterator;\nimport org.rocksdb.WriteBatch;\n\n/**\n * Key layout: [table-prefix, 1 byte][table-id, 2 bytes][record-type-prefix, 1 byte][topic-len, 2 bytes][topic-bytes]\n * Value layout: [serialization-type, 1 byte][topic-config-bytes]\n */\npublic class TopicConfigManagerV2 extends TopicConfigManager {\n    private final ConfigStorage configStorage;\n\n    public TopicConfigManagerV2(BrokerController brokerController, ConfigStorage configStorage) {\n        super(brokerController);\n        this.configStorage = configStorage;\n    }\n\n    @Override\n    public boolean load() {\n        return loadDataVersion() && loadTopicConfig();\n    }\n\n    public boolean loadDataVersion() {\n        try {\n            ConfigHelper.loadDataVersion(configStorage, TableId.TOPIC)\n                .ifPresent(buf -> ConfigHelper.onDataVersionLoad(buf, dataVersion));\n        } catch (RocksDBException e) {\n            log.error(\"Failed to load data version of topic\", e);\n            return false;\n        }\n        return true;\n    }\n\n    private boolean loadTopicConfig() {\n        int keyLen = 1 /* table-prefix */ + 2 /* table-id */ + 1 /* record-type-prefix */;\n        ByteBuf beginKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);\n        beginKey.writeByte(TablePrefix.TABLE.getValue());\n        beginKey.writeShort(TableId.TOPIC.getValue());\n        beginKey.writeByte(RecordPrefix.DATA.getValue());\n\n        ByteBuf endKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);\n        endKey.writeByte(TablePrefix.TABLE.getValue());\n        endKey.writeShort(TableId.TOPIC.getValue());\n        endKey.writeByte(RecordPrefix.DATA.getValue() + 1);\n\n        try (RocksIterator iterator = configStorage.iterate(beginKey.nioBuffer(), endKey.nioBuffer())) {\n            while (iterator.isValid()) {\n                byte[] key = iterator.key();\n                byte[] value = iterator.value();\n                TopicConfig topicConfig = parseTopicConfig(key, value);\n                if (null != topicConfig) {\n                    super.putTopicConfig(topicConfig);\n                }\n                iterator.next();\n            }\n        } finally {\n            beginKey.release();\n            endKey.release();\n        }\n        return true;\n    }\n\n    /**\n     * Key layout: [table-prefix, 1 byte][table-id, 2 bytes][record-type-prefix, 1 byte][topic-len, 2 bytes][topic-bytes]\n     * Value layout: [serialization-type, 1 byte][topic-config-bytes]\n     *\n     * @param key   Topic config key representation in RocksDB\n     * @param value Topic config value representation in RocksDB\n     * @return decoded topic config\n     */\n    private TopicConfig parseTopicConfig(byte[] key, byte[] value) {\n        ByteBuf keyBuf = Unpooled.wrappedBuffer(key);\n        ByteBuf valueBuf = Unpooled.wrappedBuffer(value);\n        try {\n            // Skip table-prefix, table-id, record-type-prefix\n            keyBuf.readerIndex(4);\n            short topicLen = keyBuf.readShort();\n            assert topicLen == keyBuf.readableBytes();\n            CharSequence topic = keyBuf.readCharSequence(topicLen, StandardCharsets.UTF_8);\n            assert null != topic;\n\n            byte serializationType = valueBuf.readByte();\n            if (SerializationType.JSON == SerializationType.valueOf(serializationType)) {\n                CharSequence json = valueBuf.readCharSequence(valueBuf.readableBytes(), StandardCharsets.UTF_8);\n                TopicConfig topicConfig = JSON.parseObject(json.toString(), TopicConfig.class);\n                assert topicConfig != null;\n                assert topic.equals(topicConfig.getTopicName());\n                return topicConfig;\n            }\n        } finally {\n            keyBuf.release();\n            valueBuf.release();\n        }\n\n        return null;\n    }\n\n    @Override\n    public synchronized void persist() {\n        try {\n            configStorage.flushWAL();\n        } catch (RocksDBException e) {\n            log.error(\"Failed to flush WAL\", e);\n        }\n    }\n\n    @Override\n    public TopicConfig selectTopicConfig(final String topic) {\n        if (MixAll.isLmq(topic)) {\n            return simpleLmqTopicConfig(topic);\n        }\n        return super.selectTopicConfig(topic);\n    }\n\n    @Override\n    public void updateTopicConfig(final TopicConfig topicConfig) {\n        if (topicConfig == null || MixAll.isLmq(topicConfig.getTopicName())) {\n            return;\n        }\n        super.updateSingleTopicConfigWithoutPersist(topicConfig);\n\n        ByteBuf keyBuf = ConfigHelper.keyBufOf(TableId.TOPIC, topicConfig.getTopicName());\n        ByteBuf valueBuf = ConfigHelper.valueBufOf(topicConfig, SerializationType.JSON);\n        try (WriteBatch writeBatch = new WriteBatch()) {\n            writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer());\n            long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;\n            ConfigHelper.stampDataVersion(writeBatch, TableId.TOPIC, dataVersion, stateMachineVersion);\n            configStorage.write(writeBatch);\n            // fdatasync on core metadata change\n            this.persist();\n        } catch (RocksDBException e) {\n            log.error(\"Failed to update topic config\", e);\n        } finally {\n            keyBuf.release();\n            valueBuf.release();\n        }\n    }\n\n    @Override\n    protected TopicConfig removeTopicConfig(String topicName) {\n        ByteBuf keyBuf = ConfigHelper.keyBufOf(TableId.TOPIC, topicName);\n        try (WriteBatch writeBatch = new WriteBatch()) {\n            writeBatch.delete(keyBuf.nioBuffer());\n            long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;\n            ConfigHelper.stampDataVersion(writeBatch, TableId.TOPIC, dataVersion, stateMachineVersion);\n            configStorage.write(writeBatch);\n        } catch (RocksDBException e) {\n            log.error(\"Failed to delete topic config by topicName={}\", topicName, e);\n        } finally {\n            keyBuf.release();\n        }\n        return super.removeTopicConfig(topicName);\n    }\n\n    @Override\n    public boolean containsTopic(String topic) {\n        if (MixAll.isLmq(topic)) {\n            return true;\n        }\n        return super.containsTopic(topic);\n    }\n\n    private TopicConfig simpleLmqTopicConfig(String topic) {\n        return new TopicConfig(topic, 1, 1, PermName.PERM_READ | PermName.PERM_WRITE);\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/config/v2/package-info.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.config.v2;\n\n/*\n * <strong>Endian</strong>: we use network byte order for all integrals, aka, always big endian.\n *\n * Unlike v1 config managers, implementations in this package prioritize data integrity and reliability.\n * As a result,RocksDB write-ahead-log is always on and changes are immediately flushed. Another significant\n * difference is that heap-based cache is removed because it is not necessary and duplicated to RocksDB\n * MemTable/BlockCache.\n */\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.controller;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Random;\nimport java.util.Set;\nimport java.util.concurrent.ArrayBlockingQueue;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ScheduledFuture;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.out.BrokerOuterAPI;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.EpochEntry;\nimport org.apache.rocketmq.remoting.protocol.body.SyncStateSet;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerResponseHeader;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService;\nimport org.apache.rocketmq.store.ha.autoswitch.BrokerMetadata;\nimport org.apache.rocketmq.store.ha.autoswitch.TempBrokerMetadata;\n\nimport static org.apache.rocketmq.remoting.protocol.ResponseCode.CONTROLLER_BROKER_METADATA_NOT_EXIST;\n\n/**\n * The manager of broker replicas, including: 0.regularly syncing controller metadata, change controller leader address,\n * both master and slave will start this timed task. 1.regularly syncing metadata from controllers, and changing broker\n * roles and master if needed, both master and slave will start this timed task. 2.regularly expanding and Shrinking\n * syncStateSet, only master will start this timed task.\n */\npublic class ReplicasManager {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n\n    private static final int RETRY_INTERVAL_SECOND = 5;\n\n    private final ScheduledExecutorService scheduledService;\n    private final ExecutorService executorService;\n    private final ExecutorService scanExecutor;\n    private final BrokerController brokerController;\n    private final AutoSwitchHAService haService;\n    private final BrokerConfig brokerConfig;\n    private final String brokerAddress;\n    private final BrokerOuterAPI brokerOuterAPI;\n    private List<String> controllerAddresses;\n    private final ConcurrentMap<String, Boolean> availableControllerAddresses;\n\n    private volatile String controllerLeaderAddress = \"\";\n    private volatile State state = State.INITIAL;\n\n    private volatile RegisterState registerState = RegisterState.INITIAL;\n\n    private ScheduledFuture<?> checkSyncStateSetTaskFuture;\n    private ScheduledFuture<?> slaveSyncFuture;\n\n    private Long brokerControllerId;\n\n    private Long masterBrokerId;\n\n    private BrokerMetadata brokerMetadata;\n\n    private TempBrokerMetadata tempBrokerMetadata;\n\n    private Set<Long> syncStateSet;\n    private int syncStateSetEpoch = 0;\n    private String masterAddress = \"\";\n    private int masterEpoch = 0;\n    private long lastSyncTimeMs = System.currentTimeMillis();\n    private Random random = new Random();\n\n    public ReplicasManager(final BrokerController brokerController) {\n        this.brokerController = brokerController;\n        this.brokerOuterAPI = brokerController.getBrokerOuterAPI();\n        this.scheduledService = ThreadUtils.newScheduledThreadPool(3, new ThreadFactoryImpl(\"ReplicasManager_ScheduledService_\", brokerController.getBrokerIdentity()));\n        this.executorService = ThreadUtils.newThreadPoolExecutor(3, new ThreadFactoryImpl(\"ReplicasManager_ExecutorService_\", brokerController.getBrokerIdentity()));\n        this.scanExecutor = ThreadUtils.newThreadPoolExecutor(4, 10, 60, TimeUnit.SECONDS,\n            new ArrayBlockingQueue<>(32), new ThreadFactoryImpl(\"ReplicasManager_scan_thread_\", brokerController.getBrokerIdentity()));\n        this.haService = (AutoSwitchHAService) brokerController.getMessageStore().getHaService();\n        this.brokerConfig = brokerController.getBrokerConfig();\n        this.availableControllerAddresses = new ConcurrentHashMap<>();\n        this.syncStateSet = new HashSet<>();\n        this.brokerAddress = brokerController.getBrokerAddr();\n        this.brokerMetadata = new BrokerMetadata(this.brokerController.getMessageStoreConfig().getStorePathBrokerIdentity());\n        this.tempBrokerMetadata = new TempBrokerMetadata(this.brokerController.getMessageStoreConfig().getStorePathBrokerIdentity() + \"-temp\");\n    }\n\n    enum State {\n        INITIAL,\n        FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE,\n        REGISTER_TO_CONTROLLER_DONE,\n        RUNNING,\n        SHUTDOWN,\n    }\n\n    enum RegisterState {\n        INITIAL,\n        CREATE_TEMP_METADATA_FILE_DONE,\n        CREATE_METADATA_FILE_DONE,\n        REGISTERED\n    }\n\n    public void start() {\n        this.state = State.INITIAL;\n        updateControllerAddr();\n        scanAvailableControllerAddresses();\n        this.scheduledService.scheduleAtFixedRate(this::updateControllerAddr, 2 * 60 * 1000, 2 * 60 * 1000, TimeUnit.MILLISECONDS);\n        this.scheduledService.scheduleAtFixedRate(this::scanAvailableControllerAddresses, 3 * 1000, 3 * 1000, TimeUnit.MILLISECONDS);\n        if (!startBasicService()) {\n            LOGGER.error(\"Failed to start replicasManager\");\n            this.executorService.submit(() -> {\n                int retryTimes = 0;\n                do {\n                    try {\n                        TimeUnit.SECONDS.sleep(RETRY_INTERVAL_SECOND);\n                    } catch (InterruptedException ignored) {\n\n                    }\n                    retryTimes++;\n                    LOGGER.warn(\"Failed to start replicasManager, retry times:{}, current state:{}, try it again\", retryTimes, this.state);\n                }\n                while (!startBasicService());\n\n                LOGGER.info(\"Start replicasManager success, retry times:{}\", retryTimes);\n            });\n        }\n    }\n\n    private boolean startBasicService() {\n        if (this.state == State.SHUTDOWN)\n            return false;\n        if (this.state == State.INITIAL) {\n            if (schedulingSyncControllerMetadata()) {\n                this.state = State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE;\n                LOGGER.info(\"First time sync controller metadata success, change state to: {}\", this.state);\n            } else {\n                return false;\n            }\n        }\n\n        if (this.state == State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE) {\n            for (int retryTimes = 0; retryTimes < 5; retryTimes++) {\n                if (register()) {\n                    this.state = State.REGISTER_TO_CONTROLLER_DONE;\n                    LOGGER.info(\"First time register broker success, change state to: {}\", this.state);\n                    break;\n                }\n\n                // Try to avoid registration concurrency conflicts in random sleep\n                try {\n                    Thread.sleep(random.nextInt(1000));\n                } catch (Exception ignore) {\n\n                }\n            }\n            // register 5 times but still unsuccessful\n            if (this.state != State.REGISTER_TO_CONTROLLER_DONE) {\n                LOGGER.error(\"Register to broker failed 5 times\");\n                return false;\n            }\n        }\n\n        if (this.state == State.REGISTER_TO_CONTROLLER_DONE) {\n            // The scheduled task for heartbeat sending is not starting now, so we should manually send heartbeat request\n            this.sendHeartbeatToController();\n            if (this.masterBrokerId != null || brokerElect()) {\n                LOGGER.info(\"Master in this broker set is elected, masterBrokerId: {}, masterBrokerAddr: {}\", this.masterBrokerId, this.masterAddress);\n                this.state = State.RUNNING;\n                setFenced(false);\n                LOGGER.info(\"All register process has been done, change state to: {}\", this.state);\n            } else {\n                return false;\n            }\n        }\n\n        schedulingSyncBrokerMetadata();\n\n        // Register syncStateSet changed listener.\n        this.haService.registerSyncStateSetChangedListener(this::doReportSyncStateSetChanged);\n        return true;\n    }\n\n    public void shutdown() {\n        this.state = State.SHUTDOWN;\n        this.registerState = RegisterState.INITIAL;\n        this.executorService.shutdownNow();\n        this.scheduledService.shutdownNow();\n        this.scanExecutor.shutdownNow();\n    }\n\n    public synchronized void changeBrokerRole(final Long newMasterBrokerId, final String newMasterAddress,\n        final Integer newMasterEpoch,\n        final Integer syncStateSetEpoch, final Set<Long> syncStateSet) throws Exception {\n        if (newMasterBrokerId != null && newMasterEpoch > this.masterEpoch) {\n            if (newMasterBrokerId.equals(this.brokerControllerId)) {\n                changeToMaster(newMasterEpoch, syncStateSetEpoch, syncStateSet);\n            } else {\n                changeToSlave(newMasterAddress, newMasterEpoch, newMasterBrokerId);\n            }\n        }\n    }\n\n    public void changeToMaster(final int newMasterEpoch, final int syncStateSetEpoch, final Set<Long> syncStateSet) throws Exception {\n        synchronized (this) {\n            if (newMasterEpoch > this.masterEpoch) {\n                LOGGER.info(\"Begin to change to master, brokerName:{}, replicas:{}, new Epoch:{}\", this.brokerConfig.getBrokerName(), this.brokerAddress, newMasterEpoch);\n                this.masterEpoch = newMasterEpoch;\n                if (this.masterBrokerId != null && this.masterBrokerId.equals(this.brokerControllerId) && this.brokerController.getBrokerConfig().getBrokerId() == MixAll.MASTER_ID) {\n                    // Change SyncStateSet\n                    final HashSet<Long> newSyncStateSet = new HashSet<>(syncStateSet);\n                    changeSyncStateSet(newSyncStateSet, syncStateSetEpoch);\n                    // if master doesn't change\n                    this.haService.changeToMasterWhenLastRoleIsMaster(newMasterEpoch);\n                    this.brokerController.getTopicConfigManager().getDataVersion().nextVersion(newMasterEpoch);\n                    this.executorService.submit(this::checkSyncStateSetAndDoReport);\n                    registerBrokerWhenRoleChange();\n                    return;\n                }\n\n                // Change SyncStateSet\n                final HashSet<Long> newSyncStateSet = new HashSet<>(syncStateSet);\n                changeSyncStateSet(newSyncStateSet, syncStateSetEpoch);\n\n                // Handle the slave synchronise\n                handleSlaveSynchronize(BrokerRole.SYNC_MASTER);\n\n                // Notify ha service, change to master\n                this.haService.changeToMaster(newMasterEpoch);\n\n                this.brokerController.getBrokerConfig().setBrokerId(MixAll.MASTER_ID);\n                this.brokerController.getMessageStoreConfig().setBrokerRole(BrokerRole.SYNC_MASTER);\n                this.brokerController.changeSpecialServiceStatus(true);\n\n                // Change record\n                this.masterAddress = this.brokerAddress;\n                this.masterBrokerId = this.brokerControllerId;\n\n                schedulingCheckSyncStateSet();\n\n                this.brokerController.getTopicConfigManager().getDataVersion().nextVersion(newMasterEpoch);\n                this.executorService.submit(this::checkSyncStateSetAndDoReport);\n                registerBrokerWhenRoleChange();\n            }\n        }\n    }\n\n    public void changeToSlave(final String newMasterAddress, final int newMasterEpoch, Long newMasterBrokerId) {\n        synchronized (this) {\n            if (newMasterEpoch > this.masterEpoch) {\n                LOGGER.info(\"Begin to change to slave, brokerName={}, brokerId={}, newMasterBrokerId={}, newMasterAddress={}, newMasterEpoch={}\",\n                    this.brokerConfig.getBrokerName(), this.brokerControllerId, newMasterBrokerId, newMasterAddress, newMasterEpoch);\n\n                this.masterEpoch = newMasterEpoch;\n                if (newMasterBrokerId.equals(this.masterBrokerId)) {\n                    // if master doesn't change\n                    this.haService.changeToSlaveWhenMasterNotChange(newMasterAddress, newMasterEpoch);\n                    this.brokerController.getTopicConfigManager().getDataVersion().nextVersion(newMasterEpoch);\n                    registerBrokerWhenRoleChange();\n                    return;\n                }\n\n                // Stop checking syncStateSet because only master is able to check\n                stopCheckSyncStateSet();\n\n                // Change config(compatibility problem)\n                this.brokerController.getMessageStoreConfig().setBrokerRole(BrokerRole.SLAVE);\n                this.brokerController.changeSpecialServiceStatus(false);\n                // The brokerId in brokerConfig just means its role(master[0] or slave[>=1])\n                this.brokerConfig.setBrokerId(brokerControllerId);\n\n                // Change record\n                this.masterAddress = newMasterAddress;\n                this.masterBrokerId = newMasterBrokerId;\n\n                // Handle the slave synchronise\n                handleSlaveSynchronize(BrokerRole.SLAVE);\n\n                // Notify ha service, change to slave\n                this.haService.changeToSlave(newMasterAddress, newMasterEpoch, brokerControllerId);\n\n                this.brokerController.getTopicConfigManager().getDataVersion().nextVersion(newMasterEpoch);\n                registerBrokerWhenRoleChange();\n            }\n        }\n    }\n\n    public void registerBrokerWhenRoleChange() {\n\n        this.executorService.submit(() -> {\n            // Register broker to name-srv\n            try {\n                this.brokerController.registerBrokerAll(true, false, this.brokerController.getBrokerConfig().isForceRegister());\n            } catch (final Throwable e) {\n                LOGGER.error(\"Error happen when register broker to name-srv, Failed to change broker to {}\", this.brokerController.getMessageStoreConfig().getBrokerRole(), e);\n                return;\n            }\n            LOGGER.info(\"Change broker [id:{}][address:{}] to {}, newMasterBrokerId:{}, newMasterAddress:{}, newMasterEpoch:{}, syncStateSetEpoch:{}\",\n                this.brokerControllerId, this.brokerAddress, this.brokerController.getMessageStoreConfig().getBrokerRole(), this.masterBrokerId, this.masterAddress, this.masterEpoch, this.syncStateSetEpoch);\n        });\n\n    }\n\n    private void changeSyncStateSet(final Set<Long> newSyncStateSet, final int newSyncStateSetEpoch) {\n        synchronized (this) {\n            if (newSyncStateSetEpoch > this.syncStateSetEpoch) {\n                LOGGER.info(\"SyncStateSet changed from {} to {}\", this.syncStateSet, newSyncStateSet);\n                this.syncStateSetEpoch = newSyncStateSetEpoch;\n                this.syncStateSet = new HashSet<>(newSyncStateSet);\n                this.haService.setSyncStateSet(newSyncStateSet);\n            }\n        }\n    }\n\n    private void handleSlaveSynchronize(final BrokerRole role) {\n        if (role == BrokerRole.SLAVE) {\n            if (this.slaveSyncFuture != null) {\n                this.slaveSyncFuture.cancel(false);\n            }\n            this.brokerController.getSlaveSynchronize().setMasterAddr(this.masterAddress);\n            slaveSyncFuture = this.brokerController.getScheduledExecutorService().scheduleAtFixedRate(() -> {\n                try {\n                    if (System.currentTimeMillis() - lastSyncTimeMs > 10 * 1000) {\n                        brokerController.getSlaveSynchronize().syncAll();\n                        lastSyncTimeMs = System.currentTimeMillis();\n                    }\n                    //timer checkpoint, latency-sensitive, so sync it more frequently\n                    brokerController.getSlaveSynchronize().syncTimerCheckPoint();\n                } catch (final Throwable e) {\n                    LOGGER.error(\"ScheduledTask SlaveSynchronize syncAll error.\", e);\n                }\n            }, 1000 * 3, 1000 * 3, TimeUnit.MILLISECONDS);\n\n        } else {\n            if (this.slaveSyncFuture != null) {\n                this.slaveSyncFuture.cancel(false);\n            }\n            this.brokerController.getSlaveSynchronize().setMasterAddr(null);\n        }\n    }\n\n    private boolean brokerElect() {\n        // Broker try to elect itself as a master in broker set.\n        try {\n            Pair<ElectMasterResponseHeader, Set<Long>> tryElectResponsePair = this.brokerOuterAPI.brokerElect(this.controllerLeaderAddress, this.brokerConfig.getBrokerClusterName(),\n                this.brokerConfig.getBrokerName(), this.brokerControllerId);\n            ElectMasterResponseHeader tryElectResponse = tryElectResponsePair.getObject1();\n            Set<Long> syncStateSet = tryElectResponsePair.getObject2();\n            final String masterAddress = tryElectResponse.getMasterAddress();\n            final Long masterBrokerId = tryElectResponse.getMasterBrokerId();\n            if (StringUtils.isEmpty(masterAddress) || masterBrokerId == null) {\n                LOGGER.warn(\"Now no master in broker set\");\n                return false;\n            }\n\n            if (masterBrokerId.equals(this.brokerControllerId)) {\n                changeToMaster(tryElectResponse.getMasterEpoch(), tryElectResponse.getSyncStateSetEpoch(), syncStateSet);\n            } else {\n                changeToSlave(masterAddress, tryElectResponse.getMasterEpoch(), tryElectResponse.getMasterBrokerId());\n            }\n            return true;\n        } catch (Exception e) {\n            LOGGER.error(\"Failed to try elect\", e);\n            return false;\n        }\n    }\n\n    public void sendHeartbeatToController() {\n        final List<String> controllerAddresses = this.getAvailableControllerAddresses();\n        for (String controllerAddress : controllerAddresses) {\n            if (StringUtils.isNotEmpty(controllerAddress)) {\n                this.brokerOuterAPI.sendHeartbeatToController(\n                    controllerAddress,\n                    this.brokerConfig.getBrokerClusterName(),\n                    this.brokerAddress,\n                    this.brokerConfig.getBrokerName(),\n                    this.brokerControllerId,\n                    this.brokerConfig.getSendHeartbeatTimeoutMillis(),\n                    this.brokerConfig.isInBrokerContainer(), this.getLastEpoch(),\n                    this.brokerController.getMessageStore().getMaxPhyOffset(),\n                    this.brokerController.getMessageStore().getConfirmOffset(),\n                    this.brokerConfig.getControllerHeartBeatTimeoutMills(),\n                    this.brokerConfig.getBrokerElectionPriority()\n                );\n            }\n        }\n    }\n\n    /**\n     * Register broker to controller, and persist the metadata to file\n     *\n     * @return whether registering process succeeded\n     */\n    private boolean register() {\n        try {\n            // 1. confirm now registering state\n            confirmNowRegisteringState();\n            LOGGER.info(\"Confirm now register state: {}\", this.registerState);\n            // 2. check metadata/tempMetadata if valid\n            if (!checkMetadataValid()) {\n                LOGGER.error(\"Check and find that metadata/tempMetadata invalid, you can modify the broker config to make them valid\");\n                return false;\n            }\n            // 2. get next assigning brokerId, and create temp metadata file\n            if (this.registerState == RegisterState.INITIAL) {\n                Long nextBrokerId = getNextBrokerId();\n                if (nextBrokerId == null || !createTempMetadataFile(nextBrokerId)) {\n                    LOGGER.error(\"Failed to create temp metadata file, nextBrokerId: {}\", nextBrokerId);\n                    return false;\n                }\n                this.registerState = RegisterState.CREATE_TEMP_METADATA_FILE_DONE;\n                LOGGER.info(\"Register state change to {}, temp metadata: {}\", this.registerState, this.tempBrokerMetadata);\n            }\n            // 3. apply brokerId to controller, and create metadata file\n            if (this.registerState == RegisterState.CREATE_TEMP_METADATA_FILE_DONE) {\n                if (!applyBrokerId()) {\n                    // apply broker id failed, means that this brokerId has been used\n                    // delete temp metadata file\n                    this.tempBrokerMetadata.clear();\n                    // back to the first step\n                    this.registerState = RegisterState.INITIAL;\n                    LOGGER.info(\"Register state change to: {}\", this.registerState);\n                    return false;\n                }\n                if (!createMetadataFileAndDeleteTemp()) {\n                    LOGGER.error(\"Failed to create metadata file and delete temp metadata file, temp metadata: {}\", this.tempBrokerMetadata);\n                    return false;\n                }\n                this.registerState = RegisterState.CREATE_METADATA_FILE_DONE;\n                LOGGER.info(\"Register state change to: {}, metadata: {}\", this.registerState, this.brokerMetadata);\n            }\n            // 4. register\n            if (this.registerState == RegisterState.CREATE_METADATA_FILE_DONE) {\n                if (!registerBrokerToController()) {\n                    LOGGER.error(\"Failed to register broker to controller\");\n                    return false;\n                }\n                this.registerState = RegisterState.REGISTERED;\n                LOGGER.info(\"Register state change to: {}, masterBrokerId: {}, masterBrokerAddr: {}\", this.registerState, this.masterBrokerId, this.masterAddress);\n            }\n            return true;\n        } catch (final Exception e) {\n            LOGGER.error(\"Failed to register broker to controller\", e);\n            return false;\n        }\n    }\n\n    /**\n     * Send GetNextBrokerRequest to controller for getting next assigning brokerId in this broker-set\n     *\n     * @return next brokerId in this broker-set\n     */\n    private Long getNextBrokerId() {\n        try {\n            GetNextBrokerIdResponseHeader nextBrokerIdResp = this.brokerOuterAPI.getNextBrokerId(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.getBrokerName(), this.controllerLeaderAddress);\n            return nextBrokerIdResp.getNextBrokerId();\n        } catch (Exception e) {\n            LOGGER.error(\"fail to get next broker id from controller\", e);\n            return null;\n        }\n    }\n\n    /**\n     * Create temp metadata file in local file system, records the brokerId and registerCheckCode\n     *\n     * @param brokerId the brokerId that is expected to be assigned\n     * @return whether the temp meta file is created successfully\n     */\n\n    private boolean createTempMetadataFile(Long brokerId) {\n        // generate register check code, format like that: $ipAddress;$timestamp\n        String registerCheckCode = this.brokerAddress + \";\" + System.currentTimeMillis();\n        try {\n            this.tempBrokerMetadata.updateAndPersist(brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName(), brokerId, registerCheckCode);\n            return true;\n        } catch (Exception e) {\n            LOGGER.error(\"update and persist temp broker metadata file failed\", e);\n            this.tempBrokerMetadata.clear();\n            return false;\n        }\n    }\n\n    /**\n     * Send applyBrokerId request to controller\n     *\n     * @return whether controller has assigned this brokerId for this broker\n     */\n    private boolean applyBrokerId() {\n        try {\n            ApplyBrokerIdResponseHeader response = this.brokerOuterAPI.applyBrokerId(brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName(),\n                tempBrokerMetadata.getBrokerId(), tempBrokerMetadata.getRegisterCheckCode(), this.controllerLeaderAddress);\n            return true;\n\n        } catch (Exception e) {\n            LOGGER.error(\"fail to apply broker id: {}\", tempBrokerMetadata.getBrokerId(), e);\n            return false;\n        }\n    }\n\n    /**\n     * Create metadata file and delete temp metadata file\n     *\n     * @return whether process success\n     */\n    private boolean createMetadataFileAndDeleteTemp() {\n        // create metadata file and delete temp metadata file\n        try {\n            this.brokerMetadata.updateAndPersist(brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName(), tempBrokerMetadata.getBrokerId());\n            this.tempBrokerMetadata.clear();\n            this.brokerControllerId = this.brokerMetadata.getBrokerId();\n            this.haService.setLocalBrokerId(this.brokerControllerId);\n            return true;\n        } catch (Exception e) {\n            LOGGER.error(\"fail to create metadata file\", e);\n            this.brokerMetadata.clear();\n            return false;\n        }\n    }\n\n    /**\n     * Send registerBrokerToController request to inform controller that now broker has been registered successfully and\n     * controller should update broker ipAddress if changed\n     *\n     * @return whether request success\n     */\n    private boolean registerBrokerToController() {\n        try {\n            Pair<RegisterBrokerToControllerResponseHeader, Set<Long>> responsePair = this.brokerOuterAPI.registerBrokerToController(brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName(), brokerControllerId, brokerAddress, controllerLeaderAddress);\n            if (responsePair == null)\n                return false;\n            RegisterBrokerToControllerResponseHeader response = responsePair.getObject1();\n            Set<Long> syncStateSet = responsePair.getObject2();\n            final Long masterBrokerId = response.getMasterBrokerId();\n            final String masterAddress = response.getMasterAddress();\n            if (masterBrokerId == null) {\n                return true;\n            }\n            if (this.brokerControllerId.equals(masterBrokerId)) {\n                changeToMaster(response.getMasterEpoch(), response.getSyncStateSetEpoch(), syncStateSet);\n            } else {\n                changeToSlave(masterAddress, response.getMasterEpoch(), masterBrokerId);\n            }\n            return true;\n        } catch (Exception e) {\n            LOGGER.error(\"fail to send registerBrokerToController request to controller\", e);\n            return false;\n        }\n    }\n\n    /**\n     * Confirm the registering state now\n     */\n    private void confirmNowRegisteringState() {\n        // 1. check if metadata exist\n        try {\n            this.brokerMetadata.readFromFile();\n        } catch (Exception e) {\n            LOGGER.error(\"Read metadata file failed\", e);\n        }\n        if (this.brokerMetadata.isLoaded()) {\n            this.registerState = RegisterState.CREATE_METADATA_FILE_DONE;\n            this.brokerControllerId = brokerMetadata.getBrokerId();\n            this.haService.setLocalBrokerId(this.brokerControllerId);\n            return;\n        }\n        // 2. check if temp metadata exist\n        try {\n            this.tempBrokerMetadata.readFromFile();\n        } catch (Exception e) {\n            LOGGER.error(\"Read temp metadata file failed\", e);\n        }\n        if (this.tempBrokerMetadata.isLoaded()) {\n            this.registerState = RegisterState.CREATE_TEMP_METADATA_FILE_DONE;\n        }\n    }\n\n    private boolean checkMetadataValid() {\n        if (this.registerState == RegisterState.CREATE_TEMP_METADATA_FILE_DONE) {\n            if (this.tempBrokerMetadata.getClusterName() == null || !this.tempBrokerMetadata.getClusterName().equals(this.brokerConfig.getBrokerClusterName())) {\n                LOGGER.error(\"The clusterName: {} in broker temp metadata is different from the clusterName: {} in broker config\",\n                    this.tempBrokerMetadata.getClusterName(), this.brokerConfig.getBrokerClusterName());\n                return false;\n            }\n            if (this.tempBrokerMetadata.getBrokerName() == null || !this.tempBrokerMetadata.getBrokerName().equals(this.brokerConfig.getBrokerName())) {\n                LOGGER.error(\"The brokerName: {} in broker temp metadata is different from the brokerName: {} in broker config\",\n                    this.tempBrokerMetadata.getBrokerName(), this.brokerConfig.getBrokerName());\n                return false;\n            }\n        }\n        if (this.registerState == RegisterState.CREATE_METADATA_FILE_DONE) {\n            if (this.brokerMetadata.getClusterName() == null || !this.brokerMetadata.getClusterName().equals(this.brokerConfig.getBrokerClusterName())) {\n                LOGGER.error(\"The clusterName: {} in broker metadata is different from the clusterName: {} in broker config\",\n                    this.brokerMetadata.getClusterName(), this.brokerConfig.getBrokerClusterName());\n                return false;\n            }\n            if (this.brokerMetadata.getBrokerName() == null || !this.brokerMetadata.getBrokerName().equals(this.brokerConfig.getBrokerName())) {\n                LOGGER.error(\"The brokerName: {} in broker metadata is different from the brokerName: {} in broker config\",\n                    this.brokerMetadata.getBrokerName(), this.brokerConfig.getBrokerName());\n                return false;\n            }\n        }\n        return true;\n    }\n\n    /**\n     * Scheduling sync broker metadata form controller.\n     */\n    private void schedulingSyncBrokerMetadata() {\n        this.scheduledService.scheduleAtFixedRate(() -> {\n            try {\n                final Pair<GetReplicaInfoResponseHeader, SyncStateSet> result = this.brokerOuterAPI.getReplicaInfo(this.controllerLeaderAddress, this.brokerConfig.getBrokerName());\n                final GetReplicaInfoResponseHeader info = result.getObject1();\n                final SyncStateSet syncStateSet = result.getObject2();\n                final String newMasterAddress = info.getMasterAddress();\n                final int newMasterEpoch = info.getMasterEpoch();\n                final Long masterBrokerId = info.getMasterBrokerId();\n                synchronized (this) {\n                    // Check if master changed\n                    if (newMasterEpoch > this.masterEpoch) {\n                        if (StringUtils.isNoneEmpty(newMasterAddress) && masterBrokerId != null) {\n                            if (masterBrokerId.equals(this.brokerControllerId)) {\n                                // If this broker is now the master\n                                changeToMaster(newMasterEpoch, syncStateSet.getSyncStateSetEpoch(), syncStateSet.getSyncStateSet());\n                            } else {\n                                // If this broker is now the slave, and master has been changed\n                                changeToSlave(newMasterAddress, newMasterEpoch, masterBrokerId);\n                            }\n                        } else {\n                            // In this case, the master in controller is null, try elect in controller, this will trigger the electMasterEvent in controller.\n                            brokerElect();\n                        }\n                    } else if (newMasterEpoch == this.masterEpoch) {\n                        // Check if SyncStateSet changed\n                        if (isMasterState()) {\n                            changeSyncStateSet(syncStateSet.getSyncStateSet(), syncStateSet.getSyncStateSetEpoch());\n                        }\n                    }\n                }\n            } catch (final MQBrokerException exception) {\n                LOGGER.warn(\"Error happen when get broker {}'s metadata\", this.brokerConfig.getBrokerName(), exception);\n                if (exception.getResponseCode() == CONTROLLER_BROKER_METADATA_NOT_EXIST) {\n                    try {\n                        registerBrokerToController();\n                        TimeUnit.SECONDS.sleep(2);\n                    } catch (InterruptedException ignore) {\n\n                    }\n                }\n            } catch (final Exception e) {\n                LOGGER.warn(\"Error happen when get broker {}'s metadata\", this.brokerConfig.getBrokerName(), e);\n            }\n        }, 3 * 1000, this.brokerConfig.getSyncBrokerMetadataPeriod(), TimeUnit.MILLISECONDS);\n    }\n\n    /**\n     * Scheduling sync controller metadata.\n     */\n    private boolean schedulingSyncControllerMetadata() {\n        // Get controller metadata first.\n        int tryTimes = 0;\n        while (tryTimes < 3) {\n            boolean flag = updateControllerMetadata();\n            if (flag) {\n                this.scheduledService.scheduleAtFixedRate(this::updateControllerMetadata, 1000 * 3, this.brokerConfig.getSyncControllerMetadataPeriod(), TimeUnit.MILLISECONDS);\n                return true;\n            }\n            try {\n                TimeUnit.SECONDS.sleep(1);\n            } catch (InterruptedException ignore) {\n\n            }\n            tryTimes++;\n        }\n        LOGGER.error(\"Failed to init controller metadata, maybe the controllers in {} is not available\", this.controllerAddresses);\n        return false;\n    }\n\n    /**\n     * Update controller leader address by rpc.\n     */\n    private boolean updateControllerMetadata() {\n        for (String address : this.availableControllerAddresses.keySet()) {\n            try {\n                final GetMetaDataResponseHeader responseHeader = this.brokerOuterAPI.getControllerMetaData(address);\n                if (responseHeader != null && StringUtils.isNoneEmpty(responseHeader.getControllerLeaderAddress())) {\n                    this.controllerLeaderAddress = responseHeader.getControllerLeaderAddress();\n                    LOGGER.info(\"Update controller leader address to {}\", this.controllerLeaderAddress);\n                    return true;\n                }\n            } catch (final Exception e) {\n                LOGGER.error(\"Failed to update controller metadata\", e);\n            }\n        }\n        return false;\n    }\n\n    /**\n     * Scheduling check syncStateSet.\n     */\n    private void schedulingCheckSyncStateSet() {\n        if (this.checkSyncStateSetTaskFuture != null) {\n            this.checkSyncStateSetTaskFuture.cancel(false);\n        }\n        this.checkSyncStateSetTaskFuture = this.scheduledService.scheduleAtFixedRate(this::checkSyncStateSetAndDoReport, 3 * 1000,\n            this.brokerConfig.getCheckSyncStateSetPeriod(), TimeUnit.MILLISECONDS);\n    }\n\n    private void checkSyncStateSetAndDoReport() {\n        try {\n            final Set<Long> newSyncStateSet = this.haService.maybeShrinkSyncStateSet();\n            newSyncStateSet.add(this.brokerControllerId);\n            synchronized (this) {\n                if (this.syncStateSet != null) {\n                    // Check if syncStateSet changed\n                    if (this.syncStateSet.size() == newSyncStateSet.size() && this.syncStateSet.containsAll(newSyncStateSet)) {\n                        return;\n                    }\n                }\n            }\n            doReportSyncStateSetChanged(newSyncStateSet);\n        } catch (Exception e) {\n            LOGGER.error(\"Check syncStateSet error\", e);\n        }\n    }\n\n    private void doReportSyncStateSetChanged(Set<Long> newSyncStateSet) {\n        try {\n            final SyncStateSet result = this.brokerOuterAPI.alterSyncStateSet(this.controllerLeaderAddress, this.brokerConfig.getBrokerName(), this.brokerControllerId, this.masterEpoch, newSyncStateSet, this.syncStateSetEpoch);\n            if (result != null) {\n                changeSyncStateSet(result.getSyncStateSet(), result.getSyncStateSetEpoch());\n            }\n        } catch (final Exception e) {\n            LOGGER.error(\"Error happen when change SyncStateSet, broker:{}, masterAddress:{}, masterEpoch:{}, oldSyncStateSet:{}, newSyncStateSet:{}, syncStateSetEpoch:{}\",\n                this.brokerConfig.getBrokerName(), this.masterAddress, this.masterEpoch, this.syncStateSet, newSyncStateSet, this.syncStateSetEpoch, e);\n        }\n    }\n\n    private void stopCheckSyncStateSet() {\n        if (this.checkSyncStateSetTaskFuture != null) {\n            this.checkSyncStateSetTaskFuture.cancel(false);\n        }\n    }\n\n    private void scanAvailableControllerAddresses() {\n        try {\n            if (controllerAddresses == null) {\n                LOGGER.warn(\"scanAvailableControllerAddresses addresses of controller is null!\");\n                return;\n            }\n\n            for (String address : availableControllerAddresses.keySet()) {\n                if (!controllerAddresses.contains(address)) {\n                    LOGGER.warn(\"scanAvailableControllerAddresses remove invalid address {}\", address);\n                    availableControllerAddresses.remove(address);\n                }\n            }\n\n            for (String address : controllerAddresses) {\n                scanExecutor.submit(() -> {\n                    if (brokerOuterAPI.checkAddressReachable(address)) {\n                        availableControllerAddresses.putIfAbsent(address, true);\n                    } else {\n                        Boolean value = availableControllerAddresses.remove(address);\n                        if (value != null) {\n                            LOGGER.warn(\"scanAvailableControllerAddresses remove unconnected address {}\", address);\n                        }\n                    }\n                });\n            }\n        } catch (final Throwable t) {\n            LOGGER.error(\"scanAvailableControllerAddresses unexpected exception\", t);\n        }\n    }\n\n    private void updateControllerAddr() {\n        if (brokerConfig.isFetchControllerAddrByDnsLookup()) {\n            List<String> adders = brokerOuterAPI.dnsLookupAddressByDomain(this.brokerConfig.getControllerAddr());\n            if (CollectionUtils.isNotEmpty(adders)) {\n                this.controllerAddresses = adders;\n            }\n        } else {\n            final String controllerPaths = this.brokerConfig.getControllerAddr();\n            final String[] controllers = controllerPaths.split(\";\");\n            assert controllers.length > 0;\n            this.controllerAddresses = Arrays.asList(controllers);\n        }\n    }\n\n    public int getLastEpoch() {\n        return this.haService.getLastEpoch();\n    }\n\n    public BrokerRole getBrokerRole() {\n        return this.brokerController.getMessageStoreConfig().getBrokerRole();\n    }\n\n    public boolean isMasterState() {\n        return getBrokerRole() == BrokerRole.SYNC_MASTER;\n    }\n\n    public SyncStateSet getSyncStateSet() {\n        return new SyncStateSet(this.syncStateSet, this.syncStateSetEpoch);\n    }\n\n    public String getBrokerAddress() {\n        return brokerAddress;\n    }\n\n    public String getMasterAddress() {\n        return masterAddress;\n    }\n\n    public int getMasterEpoch() {\n        return masterEpoch;\n    }\n\n    public List<String> getControllerAddresses() {\n        return controllerAddresses;\n    }\n\n    public List<EpochEntry> getEpochEntries() {\n        return this.haService.getEpochEntries();\n    }\n\n    public List<String> getAvailableControllerAddresses() {\n        return new ArrayList<>(availableControllerAddresses.keySet());\n    }\n\n    public Long getBrokerControllerId() {\n        return brokerControllerId;\n    }\n\n    public RegisterState getRegisterState() {\n        return registerState;\n    }\n\n    public State getState() {\n        return state;\n    }\n\n    public BrokerMetadata getBrokerMetadata() {\n        return brokerMetadata;\n    }\n\n    public TempBrokerMetadata getTempBrokerMetadata() {\n        return tempBrokerMetadata;\n    }\n\n    public void setFenced(boolean fenced) {\n        this.brokerController.setIsolated(fenced);\n        this.brokerController.getMessageStore().getRunningFlags().makeFenced(fenced);\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.dledger;\n\nimport io.openmessaging.storage.dledger.DLedgerLeaderElector;\nimport io.openmessaging.storage.dledger.DLedgerServer;\nimport io.openmessaging.storage.dledger.MemberState;\nimport io.openmessaging.storage.dledger.utils.DLedgerUtils;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Future;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.dledger.DLedgerCommitLog;\n\npublic class DLedgerRoleChangeHandler implements DLedgerLeaderElector.RoleChangeHandler {\n\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private ExecutorService executorService;\n    private BrokerController brokerController;\n    private DefaultMessageStore messageStore;\n    private DLedgerCommitLog dLedgerCommitLog;\n    private DLedgerServer dLegerServer;\n    private Future<?> slaveSyncFuture;\n    private long lastSyncTimeMs = System.currentTimeMillis();\n\n    public DLedgerRoleChangeHandler(BrokerController brokerController, DefaultMessageStore messageStore) {\n        this.brokerController = brokerController;\n        this.messageStore = messageStore;\n        this.dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog();\n        this.dLegerServer = dLedgerCommitLog.getdLedgerServer();\n        this.executorService = ThreadUtils.newSingleThreadExecutor(\n            new ThreadFactoryImpl(\"DLegerRoleChangeHandler_\", brokerController.getBrokerIdentity()));\n    }\n\n    @Override\n    public void handle(long term, MemberState.Role role) {\n        Runnable runnable = new Runnable() {\n            @Override\n            public void run() {\n                long start = System.currentTimeMillis();\n                try {\n                    boolean succ = true;\n                    LOGGER.info(\"Begin handling broker role change term={} role={} currStoreRole={}\", term, role, messageStore.getMessageStoreConfig().getBrokerRole());\n                    switch (role) {\n                        case CANDIDATE:\n                            if (messageStore.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE) {\n                                changeToSlave(dLedgerCommitLog.getId());\n                            }\n                            break;\n                        case FOLLOWER:\n                            changeToSlave(dLedgerCommitLog.getId());\n                            break;\n                        case LEADER:\n                            while (true) {\n                                if (!dLegerServer.getMemberState().isLeader()) {\n                                    succ = false;\n                                    break;\n                                }\n                                if (dLegerServer.getDLedgerStore().getLedgerEndIndex() == -1) {\n                                    break;\n                                }\n                                if (dLegerServer.getDLedgerStore().getLedgerEndIndex() == dLegerServer.getDLedgerStore().getCommittedIndex()\n                                    && messageStore.dispatchBehindBytes() == 0) {\n                                    break;\n                                }\n                                Thread.sleep(100);\n                            }\n                            if (succ) {\n                                messageStore.recoverTopicQueueTable();\n                                changeToMaster(BrokerRole.SYNC_MASTER);\n                            }\n                            break;\n                        default:\n                            break;\n                    }\n                    LOGGER.info(\"Finish handling broker role change succ={} term={} role={} currStoreRole={} cost={}\", succ, term, role, messageStore.getMessageStoreConfig().getBrokerRole(), DLedgerUtils.elapsed(start));\n                } catch (Throwable t) {\n                    LOGGER.info(\"[MONITOR]Failed handling broker role change term={} role={} currStoreRole={} cost={}\", term, role, messageStore.getMessageStoreConfig().getBrokerRole(), DLedgerUtils.elapsed(start), t);\n                }\n            }\n        };\n        executorService.submit(runnable);\n    }\n\n    private void handleSlaveSynchronize(BrokerRole role) {\n        if (role == BrokerRole.SLAVE) {\n            if (null != slaveSyncFuture) {\n                slaveSyncFuture.cancel(false);\n            }\n            this.brokerController.getSlaveSynchronize().setMasterAddr(null);\n            slaveSyncFuture = this.brokerController.getScheduledExecutorService().scheduleAtFixedRate(new Runnable() {\n                @Override\n                public void run() {\n                    try {\n                        if (System.currentTimeMillis() - lastSyncTimeMs > 10 * 1000) {\n                            brokerController.getSlaveSynchronize().syncAll();\n                            lastSyncTimeMs = System.currentTimeMillis();\n                        }\n                        //timer checkpoint, latency-sensitive, so sync it more frequently\n                        brokerController.getSlaveSynchronize().syncTimerCheckPoint();\n                    } catch (Throwable e) {\n                        LOGGER.error(\"ScheduledTask SlaveSynchronize syncAll error.\", e);\n                    }\n                }\n            }, 1000 * 3, 1000 * 3, TimeUnit.MILLISECONDS);\n        } else {\n            //handle the slave synchronise\n            if (null != slaveSyncFuture) {\n                slaveSyncFuture.cancel(false);\n            }\n            this.brokerController.getSlaveSynchronize().setMasterAddr(null);\n        }\n    }\n\n    public void changeToSlave(int brokerId) {\n        LOGGER.info(\"Begin to change to slave brokerName={} brokerId={}\", this.brokerController.getBrokerConfig().getBrokerName(), brokerId);\n\n        //change the role\n        this.brokerController.getBrokerConfig().setBrokerId(brokerId == 0 ? 1 : brokerId); //TO DO check\n        this.brokerController.getMessageStoreConfig().setBrokerRole(BrokerRole.SLAVE);\n\n        this.brokerController.changeSpecialServiceStatus(false);\n\n        //handle the slave synchronise\n        handleSlaveSynchronize(BrokerRole.SLAVE);\n\n        try {\n            this.brokerController.registerBrokerAll(true, true, this.brokerController.getBrokerConfig().isForceRegister());\n        } catch (Throwable ignored) {\n\n        }\n        LOGGER.info(\"Finish to change to slave brokerName={} brokerId={}\", this.brokerController.getBrokerConfig().getBrokerName(), brokerId);\n    }\n\n    public void changeToMaster(BrokerRole role) {\n        if (role == BrokerRole.SLAVE) {\n            return;\n        }\n        LOGGER.info(\"Begin to change to master brokerName={}\", this.brokerController.getBrokerConfig().getBrokerName());\n\n        //handle the slave synchronise\n        handleSlaveSynchronize(role);\n\n        this.brokerController.changeSpecialServiceStatus(true);\n\n        //if the operations above are totally successful, we change to master\n        this.brokerController.getBrokerConfig().setBrokerId(0); //TO DO check\n        this.brokerController.getMessageStoreConfig().setBrokerRole(role);\n\n        try {\n            this.brokerController.registerBrokerAll(true, true, this.brokerController.getBrokerConfig().isForceRegister());\n        } catch (Throwable ignored) {\n\n        }\n        LOGGER.info(\"Finish to change to master brokerName={}\", this.brokerController.getBrokerConfig().getBrokerName());\n    }\n\n    @Override\n    public void startup() {\n\n    }\n\n    @Override\n    public void shutdown() {\n        executorService.shutdown();\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.failover;\n\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.commons.lang3.tuple.Triple;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.transaction.queue.TransactionalMessageUtil;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.impl.producer.TopicPublishInfo;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.tieredstore.TieredMessageStore;\n\npublic class EscapeBridge {\n    protected static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private static final long SEND_TIMEOUT = 3000L;\n    private static final long DEFAULT_PULL_TIMEOUT_MILLIS = 1000 * 10L;\n    private final String innerProducerGroupName;\n    private final String innerConsumerGroupName;\n\n    private final BrokerController brokerController;\n\n    private ExecutorService defaultAsyncSenderExecutor;\n\n    public EscapeBridge(BrokerController brokerController) {\n        this.brokerController = brokerController;\n        this.innerProducerGroupName = \"InnerProducerGroup_\" + brokerController.getBrokerConfig().getBrokerName() + \"_\" + brokerController.getBrokerConfig().getBrokerId();\n        this.innerConsumerGroupName = \"InnerConsumerGroup_\" + brokerController.getBrokerConfig().getBrokerName() + \"_\" + brokerController.getBrokerConfig().getBrokerId();\n    }\n\n    public void start() throws Exception {\n        if (brokerController.getBrokerConfig().isEnableSlaveActingMaster() && brokerController.getBrokerConfig().isEnableRemoteEscape()) {\n            final BlockingQueue<Runnable> asyncSenderThreadPoolQueue = new LinkedBlockingQueue<>(50000);\n            this.defaultAsyncSenderExecutor = ThreadUtils.newThreadPoolExecutor(\n                Runtime.getRuntime().availableProcessors(),\n                Runtime.getRuntime().availableProcessors(),\n                1000 * 60,\n                TimeUnit.MILLISECONDS,\n                asyncSenderThreadPoolQueue,\n                new ThreadFactoryImpl(\"AsyncEscapeBridgeExecutor_\", this.brokerController.getBrokerIdentity())\n            );\n            LOG.info(\"init executor for escaping messages asynchronously success.\");\n        }\n    }\n\n    public void shutdown() {\n        if (null != this.defaultAsyncSenderExecutor) {\n            this.defaultAsyncSenderExecutor.shutdown();\n        }\n    }\n\n    public PutMessageResult putMessage(MessageExtBrokerInner messageExt) {\n        BrokerController masterBroker = this.brokerController.peekMasterBroker();\n        if (masterBroker != null) {\n            return masterBroker.getMessageStore().putMessage(messageExt);\n        } else if (this.brokerController.getBrokerConfig().isEnableSlaveActingMaster()\n            && this.brokerController.getBrokerConfig().isEnableRemoteEscape()) {\n\n            try {\n                messageExt.setWaitStoreMsgOK(false);\n                final SendResult sendResult = putMessageToRemoteBroker(messageExt, null);\n                return transformSendResult2PutResult(sendResult);\n            } catch (Exception e) {\n                LOG.error(\"sendMessageInFailover to remote failed\", e);\n                return new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true);\n            }\n        } else {\n            LOG.warn(\"Put message failed, enableSlaveActingMaster={}, enableRemoteEscape={}.\",\n                this.brokerController.getBrokerConfig().isEnableSlaveActingMaster(), this.brokerController.getBrokerConfig().isEnableRemoteEscape());\n            return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null);\n        }\n    }\n\n    public SendResult putMessageToRemoteBroker(MessageExtBrokerInner messageExt, String brokerNameToSend) {\n        if (this.brokerController.getBrokerConfig().getBrokerName().equals(brokerNameToSend)) { // not remote broker\n            return null;\n        }\n        final boolean isTransHalfMessage = TransactionalMessageUtil.buildHalfTopic().equals(messageExt.getTopic());\n        MessageExtBrokerInner messageToPut = messageExt;\n        if (isTransHalfMessage) {\n            messageToPut = TransactionalMessageUtil.buildTransactionalMessageFromHalfMessage(messageExt);\n        }\n        final TopicPublishInfo topicPublishInfo = this.brokerController.getTopicRouteInfoManager().tryToFindTopicPublishInfo(messageToPut.getTopic());\n        if (null == topicPublishInfo || !topicPublishInfo.ok()) {\n            LOG.warn(\"putMessageToRemoteBroker: no route info of topic {} when escaping message, msgId={}\",\n                messageToPut.getTopic(), messageToPut.getMsgId());\n            return null;\n        }\n\n        final MessageQueue mqSelected;\n        if (StringUtils.isEmpty(brokerNameToSend)) {\n            mqSelected = topicPublishInfo.selectOneMessageQueue(this.brokerController.getBrokerConfig().getBrokerName());\n            messageToPut.setQueueId(mqSelected.getQueueId());\n            brokerNameToSend = mqSelected.getBrokerName();\n            if (this.brokerController.getBrokerConfig().getBrokerName().equals(brokerNameToSend)) {\n                LOG.warn(\"putMessageToRemoteBroker failed, remote broker not found. Topic: {}, MsgId: {}, Broker: {}\",\n                    messageExt.getTopic(), messageExt.getMsgId(), brokerNameToSend);\n                return null;\n            }\n        } else {\n            mqSelected = new MessageQueue(messageExt.getTopic(), brokerNameToSend, messageExt.getQueueId());\n        }\n\n        final String brokerAddrToSend = this.brokerController.getTopicRouteInfoManager().findBrokerAddressInPublish(brokerNameToSend);\n        if (null == brokerAddrToSend) {\n            LOG.warn(\"putMessageToRemoteBroker failed, remote broker address not found. Topic: {}, MsgId: {}, Broker: {}\",\n                messageExt.getTopic(), messageExt.getMsgId(), brokerNameToSend);\n            return null;\n        }\n\n        final long beginTimestamp = System.currentTimeMillis();\n        try {\n            final SendResult sendResult = this.brokerController.getBrokerOuterAPI().sendMessageToSpecificBroker(\n                brokerAddrToSend, brokerNameToSend,\n                messageToPut, this.getProducerGroup(messageToPut), SEND_TIMEOUT);\n            if (null != sendResult && SendStatus.SEND_OK.equals(sendResult.getSendStatus())) {\n                return sendResult;\n            } else {\n                LOG.error(\"Escaping failed! cost {}ms, Topic: {}, MsgId: {}, Broker: {}\",\n                    System.currentTimeMillis() - beginTimestamp, messageExt.getTopic(),\n                    messageExt.getMsgId(), brokerNameToSend);\n            }\n        } catch (RemotingException | MQBrokerException e) {\n            LOG.error(String.format(\"putMessageToRemoteBroker exception, MsgId: %s, RT: %sms, Broker: %s\",\n                messageToPut.getMsgId(), System.currentTimeMillis() - beginTimestamp, mqSelected), e);\n        } catch (InterruptedException e) {\n            LOG.error(String.format(\"putMessageToRemoteBroker interrupted, MsgId: %s, RT: %sms, Broker: %s\",\n                messageToPut.getMsgId(), System.currentTimeMillis() - beginTimestamp, mqSelected), e);\n            Thread.currentThread().interrupt();\n        }\n\n        return null;\n    }\n\n    public CompletableFuture<PutMessageResult> asyncPutMessage(MessageExtBrokerInner messageExt) {\n        BrokerController masterBroker = this.brokerController.peekMasterBroker();\n        if (masterBroker != null) {\n            return masterBroker.getMessageStore().asyncPutMessage(messageExt);\n        } else if (this.brokerController.getBrokerConfig().isEnableSlaveActingMaster()\n            && this.brokerController.getBrokerConfig().isEnableRemoteEscape()) {\n            try {\n                messageExt.setWaitStoreMsgOK(false);\n\n                final TopicPublishInfo topicPublishInfo = this.brokerController.getTopicRouteInfoManager().tryToFindTopicPublishInfo(messageExt.getTopic());\n                final String producerGroup = getProducerGroup(messageExt);\n\n                final MessageQueue mqSelected = topicPublishInfo.selectOneMessageQueue();\n                messageExt.setQueueId(mqSelected.getQueueId());\n\n                final String brokerNameToSend = mqSelected.getBrokerName();\n                final String brokerAddrToSend = this.brokerController.getTopicRouteInfoManager().findBrokerAddressInPublish(brokerNameToSend);\n                final CompletableFuture<SendResult> future = this.brokerController.getBrokerOuterAPI().sendMessageToSpecificBrokerAsync(brokerAddrToSend,\n                    brokerNameToSend, messageExt,\n                    producerGroup, SEND_TIMEOUT);\n\n                return future.exceptionally(throwable -> null)\n                    .thenApplyAsync(this::transformSendResult2PutResult, this.defaultAsyncSenderExecutor)\n                    .exceptionally(throwable -> transformSendResult2PutResult(null));\n\n            } catch (Exception e) {\n                LOG.error(\"sendMessageInFailover to remote failed\", e);\n                return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true));\n            }\n        } else {\n            LOG.warn(\"Put message failed, enableSlaveActingMaster={}, enableRemoteEscape={}.\",\n                this.brokerController.getBrokerConfig().isEnableSlaveActingMaster(), this.brokerController.getBrokerConfig().isEnableRemoteEscape());\n            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null));\n        }\n    }\n\n    private String getProducerGroup(MessageExtBrokerInner messageExt) {\n        if (null == messageExt) {\n            return this.innerProducerGroupName;\n        }\n        String producerGroup = messageExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP);\n        if (StringUtils.isEmpty(producerGroup)) {\n            producerGroup = this.innerProducerGroupName;\n        }\n        return producerGroup;\n    }\n\n    public PutMessageResult putMessageToSpecificQueue(MessageExtBrokerInner messageExt) {\n        BrokerController masterBroker = this.brokerController.peekMasterBroker();\n        if (masterBroker != null) {\n            return masterBroker.getMessageStore().putMessage(messageExt);\n        }\n        try {\n            return asyncRemotePutMessageToSpecificQueue(messageExt).get(SEND_TIMEOUT, TimeUnit.MILLISECONDS);\n        } catch (Exception e) {\n            LOG.error(\"Put message to specific queue error\", e);\n            return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, null, true);\n        }\n    }\n\n    public CompletableFuture<PutMessageResult> asyncPutMessageToSpecificQueue(MessageExtBrokerInner messageExt) {\n        BrokerController masterBroker = this.brokerController.peekMasterBroker();\n        if (masterBroker != null) {\n            return masterBroker.getMessageStore().asyncPutMessage(messageExt);\n        }\n        return asyncRemotePutMessageToSpecificQueue(messageExt);\n    }\n\n    public CompletableFuture<PutMessageResult> asyncRemotePutMessageToSpecificQueue(MessageExtBrokerInner messageExt) {\n        if (this.brokerController.getBrokerConfig().isEnableSlaveActingMaster()\n            && this.brokerController.getBrokerConfig().isEnableRemoteEscape()) {\n            try {\n                messageExt.setWaitStoreMsgOK(false);\n\n                final TopicPublishInfo topicPublishInfo = this.brokerController.getTopicRouteInfoManager().tryToFindTopicPublishInfo(messageExt.getTopic());\n                List<MessageQueue> mqs = topicPublishInfo.getMessageQueueList();\n\n                if (null == mqs || mqs.isEmpty()) {\n                    return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true));\n                }\n\n                String id = messageExt.getTopic() + messageExt.getStoreHost();\n                final int index = Math.floorMod(id.hashCode(), mqs.size());\n\n                MessageQueue mq = mqs.get(index);\n                messageExt.setQueueId(mq.getQueueId());\n\n                String brokerNameToSend = mq.getBrokerName();\n                String brokerAddrToSend = this.brokerController.getTopicRouteInfoManager().findBrokerAddressInPublish(brokerNameToSend);\n                return this.brokerController.getBrokerOuterAPI().sendMessageToSpecificBrokerAsync(\n                    brokerAddrToSend, brokerNameToSend,\n                    messageExt, this.getProducerGroup(messageExt), SEND_TIMEOUT).thenCompose(sendResult -> CompletableFuture.completedFuture(transformSendResult2PutResult(sendResult)));\n            } catch (Exception e) {\n                LOG.error(\"sendMessageInFailover to remote failed\", e);\n                return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true));\n            }\n        } else {\n            LOG.warn(\"Put message to specific queue failed, enableSlaveActingMaster={}, enableRemoteEscape={}.\",\n                this.brokerController.getBrokerConfig().isEnableSlaveActingMaster(), this.brokerController.getBrokerConfig().isEnableRemoteEscape());\n            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null));\n        }\n    }\n\n    private PutMessageResult transformSendResult2PutResult(SendResult sendResult) {\n        if (sendResult == null) {\n            return new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true);\n        }\n        switch (sendResult.getSendStatus()) {\n            case SEND_OK:\n                return new PutMessageResult(PutMessageStatus.PUT_OK, null, true);\n            case SLAVE_NOT_AVAILABLE:\n                return new PutMessageResult(PutMessageStatus.SLAVE_NOT_AVAILABLE, null, true);\n            case FLUSH_DISK_TIMEOUT:\n                return new PutMessageResult(PutMessageStatus.FLUSH_DISK_TIMEOUT, null, true);\n            case FLUSH_SLAVE_TIMEOUT:\n                return new PutMessageResult(PutMessageStatus.FLUSH_SLAVE_TIMEOUT, null, true);\n            default:\n                return new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true);\n        }\n    }\n\n    public Triple<MessageExt, String, Boolean> getMessage(String topic, long offset, int queueId, String brokerName,\n        boolean deCompressBody) {\n        return getMessageAsync(topic, offset, queueId, brokerName, deCompressBody).join();\n    }\n\n    // Triple<MessageExt, info, needRetry>, check info and retry if and only if MessageExt is null\n    public CompletableFuture<Triple<MessageExt, String, Boolean>> getMessageAsync(String topic, long offset,\n        int queueId, String brokerName, boolean deCompressBody) {\n        MessageStore messageStore = brokerController.getMessageStoreByBrokerName(brokerName);\n        if (messageStore != null) {\n            return messageStore.getMessageAsync(innerConsumerGroupName, topic, queueId, offset, 1, null)\n                .thenApply(result -> {\n                    if (result == null) {\n                        LOG.warn(\"getMessageResult is null , innerConsumerGroupName {}, topic {}, offset {}, queueId {}\", innerConsumerGroupName, topic, offset, queueId);\n                        return Triple.of(null, \"getMessageResult is null\", false); // local store, so no retry\n                    }\n                    List<MessageExt> list = decodeMsgList(result, deCompressBody);\n                    if (list == null || list.isEmpty()) {\n                        // OFFSET_FOUND_NULL returned by TieredMessageStore indicates exception occurred\n                        boolean needRetry = GetMessageStatus.OFFSET_FOUND_NULL.equals(result.getStatus())\n                            && messageStore instanceof TieredMessageStore;\n                        LOG.warn(\"Can not get msg , topic {}, offset {}, queueId {}, needRetry {}, result is {}\",\n                            topic, offset, queueId, needRetry, result);\n                        return Triple.of(null, \"Can not get msg\", needRetry);\n                    }\n                    return Triple.of(list.get(0), \"\", false);\n                });\n        } else {\n            return getMessageFromRemoteAsync(topic, offset, queueId, brokerName);\n        }\n    }\n\n    protected List<MessageExt> decodeMsgList(GetMessageResult getMessageResult, boolean deCompressBody) {\n        List<MessageExt> foundList = new ArrayList<>();\n        try {\n            List<ByteBuffer> messageBufferList = getMessageResult.getMessageBufferList();\n            if (messageBufferList != null) {\n                for (int i = 0; i < messageBufferList.size(); i++) {\n                    ByteBuffer bb = messageBufferList.get(i);\n                    if (bb == null) {\n                        LOG.error(\"bb is null {}\", getMessageResult);\n                        continue;\n                    }\n                    MessageExt msgExt = MessageDecoder.decode(bb, true, deCompressBody);\n                    if (msgExt == null) {\n                        LOG.error(\"decode msgExt is null {}\", getMessageResult);\n                        continue;\n                    }\n                    // use CQ offset, not offset in Message\n                    msgExt.setQueueOffset(getMessageResult.getMessageQueueOffset().get(i));\n                    foundList.add(msgExt);\n                }\n            }\n        } finally {\n            getMessageResult.release();\n        }\n\n        return foundList;\n    }\n\n    protected Triple<MessageExt, String, Boolean> getMessageFromRemote(String topic, long offset, int queueId,\n        String brokerName) {\n        return getMessageFromRemoteAsync(topic, offset, queueId, brokerName).join();\n    }\n\n    // Triple<MessageExt, info, needRetry>, check info and retry if and only if MessageExt is null\n    protected CompletableFuture<Triple<MessageExt, String, Boolean>> getMessageFromRemoteAsync(String topic,\n        long offset, int queueId, String brokerName) {\n        try {\n            String brokerAddr = this.brokerController.getTopicRouteInfoManager().findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, false);\n            if (null == brokerAddr) {\n                this.brokerController.getTopicRouteInfoManager().updateTopicRouteInfoFromNameServer(topic, true, false);\n                brokerAddr = this.brokerController.getTopicRouteInfoManager().findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, false);\n\n                if (null == brokerAddr) {\n                    LOG.warn(\"can't find broker address for topic {}, {}\", topic, brokerName);\n                    return CompletableFuture.completedFuture(Triple.of(null, \"brokerAddress not found\", true)); // maybe offline temporarily, so need retry\n                }\n            }\n\n            return this.brokerController.getBrokerOuterAPI().pullMessageFromSpecificBrokerAsync(brokerName,\n                    brokerAddr, this.innerConsumerGroupName, topic, queueId, offset, 1, DEFAULT_PULL_TIMEOUT_MILLIS)\n                .thenApply(pullResult -> {\n                    if (pullResult.getLeft() != null\n                        && PullStatus.FOUND.equals(pullResult.getLeft().getPullStatus())\n                        && CollectionUtils.isNotEmpty(pullResult.getLeft().getMsgFoundList())) {\n                        return Triple.of(pullResult.getLeft().getMsgFoundList().get(0), \"\", false);\n                    }\n                    return Triple.of(null, pullResult.getMiddle(), pullResult.getRight());\n                });\n        } catch (Exception e) {\n            LOG.error(\"Get message from remote failed. {}, {}, {}, {}\", topic, offset, queueId, brokerName, e);\n        }\n\n        return CompletableFuture.completedFuture(Triple.of(null, \"Get message from remote failed\", true)); // need retry\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/filter/CommitLogDispatcherCalcBitMap.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.filter;\n\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.filter.util.BitsArray;\nimport org.apache.rocketmq.store.CommitLogDispatcher;\nimport org.apache.rocketmq.store.DispatchRequest;\n\nimport java.util.Collection;\nimport java.util.Iterator;\n\n/**\n * Calculate bit map of filter.\n */\npublic class CommitLogDispatcherCalcBitMap implements CommitLogDispatcher {\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.FILTER_LOGGER_NAME);\n\n    protected final BrokerConfig brokerConfig;\n    protected final ConsumerFilterManager consumerFilterManager;\n\n    public CommitLogDispatcherCalcBitMap(BrokerConfig brokerConfig, ConsumerFilterManager consumerFilterManager) {\n        this.brokerConfig = brokerConfig;\n        this.consumerFilterManager = consumerFilterManager;\n    }\n\n    @Override\n    public void dispatch(DispatchRequest request) {\n        if (!this.brokerConfig.isEnableCalcFilterBitMap()) {\n            return;\n        }\n\n        try {\n\n            Collection<ConsumerFilterData> filterDatas = consumerFilterManager.get(request.getTopic());\n\n            if (filterDatas == null || filterDatas.isEmpty()) {\n                return;\n            }\n\n            Iterator<ConsumerFilterData> iterator = filterDatas.iterator();\n            BitsArray filterBitMap = BitsArray.create(\n                this.consumerFilterManager.getBloomFilter().getM()\n            );\n\n            long startTime = System.currentTimeMillis();\n            while (iterator.hasNext()) {\n                ConsumerFilterData filterData = iterator.next();\n\n                if (filterData.getCompiledExpression() == null) {\n                    log.error(\"[BUG] Consumer in filter manager has no compiled expression! {}\", filterData);\n                    continue;\n                }\n\n                if (filterData.getBloomFilterData() == null) {\n                    log.error(\"[BUG] Consumer in filter manager has no bloom data! {}\", filterData);\n                    continue;\n                }\n\n                Object ret = null;\n                try {\n                    MessageEvaluationContext context = new MessageEvaluationContext(request.getPropertiesMap());\n\n                    ret = filterData.getCompiledExpression().evaluate(context);\n                } catch (Throwable e) {\n                    log.error(\"Calc filter bit map error!commitLogOffset={}, consumer={}, {}\", request.getCommitLogOffset(), filterData, e);\n                }\n\n                log.debug(\"Result of Calc bit map:ret={}, data={}, props={}, offset={}\", ret, filterData, request.getPropertiesMap(), request.getCommitLogOffset());\n\n                // eval true\n                if (ret != null && ret instanceof Boolean && (Boolean) ret) {\n                    consumerFilterManager.getBloomFilter().hashTo(\n                        filterData.getBloomFilterData(),\n                        filterBitMap\n                    );\n                }\n            }\n\n            request.setBitMap(filterBitMap.bytes());\n\n            long elapsedTime = UtilAll.computeElapsedTimeMilliseconds(startTime);\n            // 1ms\n            if (elapsedTime >= 1) {\n                log.warn(\"Spend {} ms to calc bit map, consumerNum={}, topic={}\", elapsedTime, filterDatas.size(), request.getTopic());\n            }\n        } catch (Throwable e) {\n            log.error(\"Calc bit map error! topic={}, offset={}, queueId={}, {}\", request.getTopic(), request.getCommitLogOffset(), request.getQueueId(), e);\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/filter/ConsumerFilterData.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.filter;\n\nimport org.apache.commons.lang3.builder.EqualsBuilder;\nimport org.apache.commons.lang3.builder.HashCodeBuilder;\nimport org.apache.commons.lang3.builder.ReflectionToStringBuilder;\nimport org.apache.commons.lang3.builder.ToStringStyle;\nimport org.apache.rocketmq.filter.expression.Expression;\nimport org.apache.rocketmq.filter.util.BloomFilterData;\n\nimport java.util.Collections;\n\n/**\n * Filter data of consumer.\n */\npublic class ConsumerFilterData {\n\n    private String consumerGroup;\n    private String topic;\n    private String expression;\n    private String expressionType;\n    private transient Expression compiledExpression;\n    private long bornTime;\n    private long deadTime = 0;\n    private BloomFilterData bloomFilterData;\n    private long clientVersion;\n\n    public boolean isDead() {\n        return this.deadTime >= this.bornTime;\n    }\n\n    public long howLongAfterDeath() {\n        if (isDead()) {\n            return System.currentTimeMillis() - getDeadTime();\n        }\n        return -1;\n    }\n\n    /**\n     * Check this filter data has been used to calculate bit map when msg was stored in server.\n     */\n    public boolean isMsgInLive(long msgStoreTime) {\n        return msgStoreTime > getBornTime();\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(final String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(final String topic) {\n        this.topic = topic;\n    }\n\n    public String getExpression() {\n        return expression;\n    }\n\n    public void setExpression(final String expression) {\n        this.expression = expression;\n    }\n\n    public String getExpressionType() {\n        return expressionType;\n    }\n\n    public void setExpressionType(final String expressionType) {\n        this.expressionType = expressionType;\n    }\n\n    public Expression getCompiledExpression() {\n        return compiledExpression;\n    }\n\n    public void setCompiledExpression(final Expression compiledExpression) {\n        this.compiledExpression = compiledExpression;\n    }\n\n    public long getBornTime() {\n        return bornTime;\n    }\n\n    public void setBornTime(final long bornTime) {\n        this.bornTime = bornTime;\n    }\n\n    public long getDeadTime() {\n        return deadTime;\n    }\n\n    public void setDeadTime(final long deadTime) {\n        this.deadTime = deadTime;\n    }\n\n    public BloomFilterData getBloomFilterData() {\n        return bloomFilterData;\n    }\n\n    public void setBloomFilterData(final BloomFilterData bloomFilterData) {\n        this.bloomFilterData = bloomFilterData;\n    }\n\n    public long getClientVersion() {\n        return clientVersion;\n    }\n\n    public void setClientVersion(long clientVersion) {\n        this.clientVersion = clientVersion;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        return EqualsBuilder.reflectionEquals(this, o, Collections.<String>emptyList());\n    }\n\n    @Override\n    public int hashCode() {\n        return HashCodeBuilder.reflectionHashCode(this, Collections.<String>emptyList());\n    }\n\n    @Override\n    public String toString() {\n        return ReflectionToStringBuilder.toString(this, ToStringStyle.SHORT_PREFIX_STYLE);\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/filter/ConsumerFilterManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.filter;\n\nimport java.util.Collection;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.Map;\nimport java.util.Map.Entry;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.BrokerPathConfigHelper;\nimport org.apache.rocketmq.common.ConfigManager;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.filter.FilterFactory;\nimport org.apache.rocketmq.filter.util.BloomFilter;\nimport org.apache.rocketmq.filter.util.BloomFilterData;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\n/**\n * Consumer filter data manager.Just manage the consumers use expression filter.\n */\npublic class ConsumerFilterManager extends ConfigManager {\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.FILTER_LOGGER_NAME);\n\n    private static final long MS_24_HOUR = 24 * 3600 * 1000;\n\n    private ConcurrentMap<String/*Topic*/, FilterDataMapByTopic>\n        filterDataByTopic = new ConcurrentHashMap<>(256);\n\n    private transient BrokerController brokerController;\n    private transient BloomFilter bloomFilter;\n\n    public ConsumerFilterManager() {\n        // just for test\n        this.bloomFilter = BloomFilter.createByFn(20, 64);\n    }\n\n    public ConsumerFilterManager(BrokerController brokerController) {\n        this.brokerController = brokerController;\n        this.bloomFilter = BloomFilter.createByFn(\n            brokerController.getBrokerConfig().getMaxErrorRateOfBloomFilter(),\n            brokerController.getBrokerConfig().getExpectConsumerNumUseFilter()\n        );\n        // then set bit map length of store config.\n        brokerController.getMessageStoreConfig().setBitMapLengthConsumeQueueExt(\n            this.bloomFilter.getM()\n        );\n    }\n\n    /**\n     * Build consumer filter data.Be care, bloom filter data is not included.\n     *\n     * @return maybe null\n     */\n    public static ConsumerFilterData build(final String topic, final String consumerGroup,\n        final String expression, final String type,\n        final long clientVersion) {\n        if (ExpressionType.isTagType(type)) {\n            return null;\n        }\n\n        ConsumerFilterData consumerFilterData = new ConsumerFilterData();\n        consumerFilterData.setTopic(topic);\n        consumerFilterData.setConsumerGroup(consumerGroup);\n        consumerFilterData.setBornTime(System.currentTimeMillis());\n        consumerFilterData.setDeadTime(0);\n        consumerFilterData.setExpression(expression);\n        consumerFilterData.setExpressionType(type);\n        consumerFilterData.setClientVersion(clientVersion);\n        try {\n            consumerFilterData.setCompiledExpression(\n                FilterFactory.INSTANCE.get(type).compile(expression)\n            );\n        } catch (Throwable e) {\n            log.error(\"parse error: expr={}, topic={}, group={}, error={}\", expression, topic, consumerGroup, e.getMessage());\n            return null;\n        }\n\n        return consumerFilterData;\n    }\n\n    public void register(final String consumerGroup, final Collection<SubscriptionData> subList) {\n        for (SubscriptionData subscriptionData : subList) {\n            register(\n                subscriptionData.getTopic(),\n                consumerGroup,\n                subscriptionData.getSubString(),\n                subscriptionData.getExpressionType(),\n                subscriptionData.getSubVersion()\n            );\n        }\n\n        // make illegal topic dead.\n        Collection<ConsumerFilterData> groupFilterData = getByGroup(consumerGroup);\n\n        Iterator<ConsumerFilterData> iterator = groupFilterData.iterator();\n        while (iterator.hasNext()) {\n            ConsumerFilterData filterData = iterator.next();\n\n            boolean exist = false;\n            for (SubscriptionData subscriptionData : subList) {\n                if (subscriptionData.getTopic().equals(filterData.getTopic())) {\n                    exist = true;\n                    break;\n                }\n            }\n\n            if (!exist && !filterData.isDead()) {\n                filterData.setDeadTime(System.currentTimeMillis());\n                log.info(\"Consumer filter changed: {}, make illegal topic dead:{}\", consumerGroup, filterData);\n            }\n        }\n    }\n\n    public boolean register(final String topic, final String consumerGroup, final String expression,\n        final String type, final long clientVersion) {\n        if (ExpressionType.isTagType(type)) {\n            return false;\n        }\n\n        if (expression == null || expression.length() == 0) {\n            return false;\n        }\n\n        FilterDataMapByTopic filterDataMapByTopic = this.filterDataByTopic.get(topic);\n\n        if (filterDataMapByTopic == null) {\n            FilterDataMapByTopic temp = new FilterDataMapByTopic(topic);\n            FilterDataMapByTopic prev = this.filterDataByTopic.putIfAbsent(topic, temp);\n            filterDataMapByTopic = prev != null ? prev : temp;\n        }\n\n        BloomFilterData bloomFilterData = bloomFilter.generate(consumerGroup + \"#\" + topic);\n\n        return filterDataMapByTopic.register(consumerGroup, expression, type, bloomFilterData, clientVersion);\n    }\n\n    public void unRegister(final String consumerGroup) {\n        for (Entry<String, FilterDataMapByTopic> entry : filterDataByTopic.entrySet()) {\n            entry.getValue().unRegister(consumerGroup);\n        }\n    }\n\n    public ConsumerFilterData get(final String topic, final String consumerGroup) {\n        if (!this.filterDataByTopic.containsKey(topic)) {\n            return null;\n        }\n        if (this.filterDataByTopic.get(topic).getGroupFilterData().isEmpty()) {\n            return null;\n        }\n\n        return this.filterDataByTopic.get(topic).getGroupFilterData().get(consumerGroup);\n    }\n\n    public Collection<ConsumerFilterData> getByGroup(final String consumerGroup) {\n        Collection<ConsumerFilterData> ret = new HashSet<>();\n\n        Iterator<FilterDataMapByTopic> topicIterator = this.filterDataByTopic.values().iterator();\n        while (topicIterator.hasNext()) {\n            FilterDataMapByTopic filterDataMapByTopic = topicIterator.next();\n\n            Iterator<ConsumerFilterData> filterDataIterator = filterDataMapByTopic.getGroupFilterData().values().iterator();\n\n            while (filterDataIterator.hasNext()) {\n                ConsumerFilterData filterData = filterDataIterator.next();\n\n                if (filterData.getConsumerGroup().equals(consumerGroup)) {\n                    ret.add(filterData);\n                }\n            }\n        }\n\n        return ret;\n    }\n\n    public final Collection<ConsumerFilterData> get(final String topic) {\n        if (!this.filterDataByTopic.containsKey(topic)) {\n            return null;\n        }\n        if (this.filterDataByTopic.get(topic).getGroupFilterData().isEmpty()) {\n            return null;\n        }\n\n        return this.filterDataByTopic.get(topic).getGroupFilterData().values();\n    }\n\n    public BloomFilter getBloomFilter() {\n        return bloomFilter;\n    }\n\n    @Override\n    public String encode() {\n        return encode(false);\n    }\n\n    @Override\n    public String configFilePath() {\n        if (this.brokerController != null) {\n            return BrokerPathConfigHelper.getConsumerFilterPath(\n                this.brokerController.getMessageStoreConfig().getStorePathRootDir()\n            );\n        }\n        return BrokerPathConfigHelper.getConsumerFilterPath(\"./unit_test\");\n    }\n\n    @Override\n    public void decode(final String jsonString) {\n        ConsumerFilterManager load = RemotingSerializable.fromJson(jsonString, ConsumerFilterManager.class);\n        if (load != null && load.filterDataByTopic != null) {\n            boolean bloomChanged = false;\n            for (Entry<String, FilterDataMapByTopic> entry : load.filterDataByTopic.entrySet()) {\n                FilterDataMapByTopic dataMapByTopic = entry.getValue();\n                if (dataMapByTopic == null) {\n                    continue;\n                }\n\n                for (Entry<String, ConsumerFilterData> groupEntry : dataMapByTopic.getGroupFilterData().entrySet()) {\n\n                    ConsumerFilterData filterData = groupEntry.getValue();\n\n                    if (filterData == null) {\n                        continue;\n                    }\n\n                    try {\n                        filterData.setCompiledExpression(\n                                FilterFactory.INSTANCE.get(filterData.getExpressionType()).compile(filterData.getExpression())\n                        );\n                    } catch (Exception e) {\n                        log.error(\"load filter data error, \" + filterData, e);\n                    }\n\n                    // check whether bloom filter is changed\n                    // if changed, ignore the bit map calculated before.\n                    if (!this.bloomFilter.isValid(filterData.getBloomFilterData())) {\n                        bloomChanged = true;\n                        log.info(\"Bloom filter is changed!So ignore all filter data persisted! {}, {}\", this.bloomFilter, filterData.getBloomFilterData());\n                        break;\n                    }\n\n                    log.info(\"load exist consumer filter data: {}\", filterData);\n\n                    if (filterData.getDeadTime() == 0) {\n                        // we think all consumers are dead when load\n                        long deadTime = System.currentTimeMillis() - 30 * 1000;\n                        filterData.setDeadTime(\n                                deadTime <= filterData.getBornTime() ? filterData.getBornTime() : deadTime\n                        );\n                    }\n                }\n            }\n\n            if (!bloomChanged) {\n                this.filterDataByTopic = load.filterDataByTopic;\n            }\n        }\n    }\n\n    @Override\n    public String encode(final boolean prettyFormat) {\n        // clean\n        {\n            clean();\n        }\n        return RemotingSerializable.toJson(this, prettyFormat);\n    }\n\n    public void clean() {\n        Iterator<Map.Entry<String, FilterDataMapByTopic>> topicIterator = this.filterDataByTopic.entrySet().iterator();\n        while (topicIterator.hasNext()) {\n            Map.Entry<String, FilterDataMapByTopic> filterDataMapByTopic = topicIterator.next();\n\n            Iterator<Map.Entry<String, ConsumerFilterData>> filterDataIterator\n                = filterDataMapByTopic.getValue().getGroupFilterData().entrySet().iterator();\n\n            while (filterDataIterator.hasNext()) {\n                Map.Entry<String, ConsumerFilterData> filterDataByGroup = filterDataIterator.next();\n\n                ConsumerFilterData filterData = filterDataByGroup.getValue();\n                if (filterData.howLongAfterDeath() >= (this.brokerController == null ? MS_24_HOUR : this.brokerController.getBrokerConfig().getFilterDataCleanTimeSpan())) {\n                    log.info(\"Remove filter consumer {}, died too long!\", filterDataByGroup.getValue());\n                    filterDataIterator.remove();\n                }\n            }\n\n            if (filterDataMapByTopic.getValue().getGroupFilterData().isEmpty()) {\n                log.info(\"Topic has no consumer, remove it! {}\", filterDataMapByTopic.getKey());\n                topicIterator.remove();\n            }\n        }\n    }\n\n    public ConcurrentMap<String, FilterDataMapByTopic> getFilterDataByTopic() {\n        return filterDataByTopic;\n    }\n\n    public void setFilterDataByTopic(final ConcurrentHashMap<String, FilterDataMapByTopic> filterDataByTopic) {\n        this.filterDataByTopic = filterDataByTopic;\n    }\n\n    public static class FilterDataMapByTopic {\n\n        private ConcurrentMap<String/*consumer group*/, ConsumerFilterData>\n            groupFilterData = new ConcurrentHashMap<>();\n\n        private String topic;\n\n        public FilterDataMapByTopic() {\n        }\n\n        public FilterDataMapByTopic(String topic) {\n            this.topic = topic;\n        }\n\n        public void unRegister(String consumerGroup) {\n            if (!this.groupFilterData.containsKey(consumerGroup)) {\n                return;\n            }\n\n            ConsumerFilterData data = this.groupFilterData.get(consumerGroup);\n\n            if (data == null || data.isDead()) {\n                return;\n            }\n\n            long now = System.currentTimeMillis();\n\n            log.info(\"Unregister consumer filter: {}, deadTime: {}\", data, now);\n\n            data.setDeadTime(now);\n        }\n\n        public boolean register(String consumerGroup, String expression, String type, BloomFilterData bloomFilterData,\n            long clientVersion) {\n            ConsumerFilterData old = this.groupFilterData.get(consumerGroup);\n\n            if (old == null) {\n                ConsumerFilterData consumerFilterData = build(topic, consumerGroup, expression, type, clientVersion);\n                if (consumerFilterData == null) {\n                    return false;\n                }\n                consumerFilterData.setBloomFilterData(bloomFilterData);\n\n                old = this.groupFilterData.putIfAbsent(consumerGroup, consumerFilterData);\n                if (old == null) {\n                    log.info(\"New consumer filter registered: {}\", consumerFilterData);\n                    return true;\n                } else {\n                    if (clientVersion <= old.getClientVersion()) {\n                        if (!type.equals(old.getExpressionType()) || !expression.equals(old.getExpression())) {\n                            log.warn(\"Ignore consumer({} : {}) filter(concurrent), because of version {} <= {}, but maybe info changed!old={}:{}, ignored={}:{}\",\n                                consumerGroup, topic,\n                                clientVersion, old.getClientVersion(),\n                                old.getExpressionType(), old.getExpression(),\n                                type, expression);\n                        }\n                        if (clientVersion == old.getClientVersion() && old.isDead()) {\n                            reAlive(old);\n                            return true;\n                        }\n\n                        return false;\n                    } else {\n                        this.groupFilterData.put(consumerGroup, consumerFilterData);\n                        log.info(\"New consumer filter registered(concurrent): {}, old: {}\", consumerFilterData, old);\n                        return true;\n                    }\n                }\n            } else {\n                if (clientVersion <= old.getClientVersion()) {\n                    if (!type.equals(old.getExpressionType()) || !expression.equals(old.getExpression())) {\n                        log.info(\"Ignore consumer({}:{}) filter, because of version {} <= {}, but maybe info changed!old={}:{}, ignored={}:{}\",\n                            consumerGroup, topic,\n                            clientVersion, old.getClientVersion(),\n                            old.getExpressionType(), old.getExpression(),\n                            type, expression);\n                    }\n                    if (clientVersion == old.getClientVersion() && old.isDead()) {\n                        reAlive(old);\n                        return true;\n                    }\n\n                    return false;\n                }\n\n                boolean change = !old.getExpression().equals(expression) || !old.getExpressionType().equals(type);\n                if (old.getBloomFilterData() == null && bloomFilterData != null) {\n                    change = true;\n                }\n                if (old.getBloomFilterData() != null && !old.getBloomFilterData().equals(bloomFilterData)) {\n                    change = true;\n                }\n\n                // if subscribe data is changed, or consumer is died too long.\n                if (change) {\n                    ConsumerFilterData consumerFilterData = build(topic, consumerGroup, expression, type, clientVersion);\n                    if (consumerFilterData == null) {\n                        // new expression compile error, remove old, let client report error.\n                        this.groupFilterData.remove(consumerGroup);\n                        return false;\n                    }\n                    consumerFilterData.setBloomFilterData(bloomFilterData);\n\n                    this.groupFilterData.put(consumerGroup, consumerFilterData);\n\n                    log.info(\"Consumer filter info change, old: {}, new: {}, change: {}\",\n                        old, consumerFilterData, change);\n\n                    return true;\n                } else {\n                    old.setClientVersion(clientVersion);\n                    if (old.isDead()) {\n                        reAlive(old);\n                    }\n                    return true;\n                }\n            }\n        }\n\n        protected void reAlive(ConsumerFilterData filterData) {\n            long oldDeadTime = filterData.getDeadTime();\n            filterData.setDeadTime(0);\n            log.info(\"Re alive consumer filter: {}, oldDeadTime: {}\", filterData, oldDeadTime);\n        }\n\n        public final ConsumerFilterData get(String consumerGroup) {\n            return this.groupFilterData.get(consumerGroup);\n        }\n\n        public final ConcurrentMap<String, ConsumerFilterData> getGroupFilterData() {\n            return this.groupFilterData;\n        }\n\n        public void setGroupFilterData(final ConcurrentHashMap<String, ConsumerFilterData> groupFilterData) {\n            this.groupFilterData = groupFilterData;\n        }\n\n        public String getTopic() {\n            return topic;\n        }\n\n        public void setTopic(final String topic) {\n            this.topic = topic;\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/filter/ExpressionForRetryMessageFilter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.filter;\n\nimport java.nio.ByteBuffer;\nimport java.util.Map;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\n/**\n * Support filter to retry topic.\n * <br>It will decode properties first in order to get real topic.\n */\npublic class ExpressionForRetryMessageFilter extends ExpressionMessageFilter {\n    public ExpressionForRetryMessageFilter(SubscriptionData subscriptionData, ConsumerFilterData consumerFilterData,\n        ConsumerFilterManager consumerFilterManager) {\n        super(subscriptionData, consumerFilterData, consumerFilterManager);\n    }\n\n    @Override\n    public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map<String, String> properties) {\n        if (subscriptionData == null) {\n            return true;\n        }\n\n        if (subscriptionData.isClassFilterMode()) {\n            return true;\n        }\n\n        if (ExpressionType.isTagType(subscriptionData.getExpressionType())) {\n            return true;\n        }\n\n        boolean isRetryTopic = subscriptionData.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX);\n\n        ConsumerFilterData realFilterData = this.consumerFilterData;\n        Map<String, String> tempProperties = properties;\n        boolean decoded = false;\n        if (isRetryTopic) {\n            // retry topic, use original filter data.\n            // poor performance to support retry filter.\n            if (tempProperties == null && msgBuffer != null) {\n                decoded = true;\n                tempProperties = MessageDecoder.decodeProperties(msgBuffer);\n            }\n            String realTopic = tempProperties.get(MessageConst.PROPERTY_RETRY_TOPIC);\n            String group = KeyBuilder.parseGroup(subscriptionData.getTopic());\n            realFilterData = this.consumerFilterManager.get(realTopic, group);\n        }\n\n        // no expression\n        if (realFilterData == null || realFilterData.getExpression() == null\n            || realFilterData.getCompiledExpression() == null) {\n            return true;\n        }\n\n        if (!decoded && tempProperties == null && msgBuffer != null) {\n            tempProperties = MessageDecoder.decodeProperties(msgBuffer);\n        }\n\n        Object ret = null;\n        try {\n            MessageEvaluationContext context = new MessageEvaluationContext(tempProperties);\n\n            ret = realFilterData.getCompiledExpression().evaluate(context);\n        } catch (Throwable e) {\n            log.error(\"Message Filter error, \" + realFilterData + \", \" + tempProperties, e);\n        }\n\n        log.debug(\"Pull eval result: {}, {}, {}\", ret, realFilterData, tempProperties);\n\n        if (ret == null || !(ret instanceof Boolean)) {\n            return false;\n        }\n\n        return (Boolean) ret;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/filter/ExpressionMessageFilter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.filter;\n\nimport java.nio.ByteBuffer;\nimport java.util.Map;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.filter.util.BitsArray;\nimport org.apache.rocketmq.filter.util.BloomFilter;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.store.ConsumeQueueExt;\nimport org.apache.rocketmq.store.MessageFilter;\n\npublic class ExpressionMessageFilter implements MessageFilter {\n\n    protected static final Logger log = LoggerFactory.getLogger(LoggerName.FILTER_LOGGER_NAME);\n\n    protected final SubscriptionData subscriptionData;\n    protected final ConsumerFilterData consumerFilterData;\n    protected final ConsumerFilterManager consumerFilterManager;\n    protected final boolean bloomDataValid;\n\n    public ExpressionMessageFilter(SubscriptionData subscriptionData, ConsumerFilterData consumerFilterData,\n        ConsumerFilterManager consumerFilterManager) {\n        this.subscriptionData = subscriptionData;\n        this.consumerFilterData = consumerFilterData;\n        this.consumerFilterManager = consumerFilterManager;\n        if (consumerFilterData == null) {\n            bloomDataValid = false;\n            return;\n        }\n        BloomFilter bloomFilter = this.consumerFilterManager.getBloomFilter();\n        if (bloomFilter != null && bloomFilter.isValid(consumerFilterData.getBloomFilterData())) {\n            bloomDataValid = true;\n        } else {\n            bloomDataValid = false;\n        }\n    }\n\n    @Override\n    public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {\n        if (null == subscriptionData) {\n            return true;\n        }\n\n        if (subscriptionData.isClassFilterMode()) {\n            return true;\n        }\n\n        // by tags code.\n        if (ExpressionType.isTagType(subscriptionData.getExpressionType())) {\n\n            if (tagsCode == null) {\n                return true;\n            }\n\n            if (subscriptionData.getSubString().equals(SubscriptionData.SUB_ALL)) {\n                return true;\n            }\n\n            return subscriptionData.getCodeSet().contains(tagsCode.intValue());\n        } else {\n            // no expression or no bloom\n            if (consumerFilterData == null || consumerFilterData.getExpression() == null\n                || consumerFilterData.getCompiledExpression() == null || consumerFilterData.getBloomFilterData() == null) {\n                return true;\n            }\n\n            // message is before consumer\n            if (cqExtUnit == null || !consumerFilterData.isMsgInLive(cqExtUnit.getMsgStoreTime())) {\n                log.debug(\"Pull matched because not in live: {}, {}\", consumerFilterData, cqExtUnit);\n                return true;\n            }\n\n            byte[] filterBitMap = cqExtUnit.getFilterBitMap();\n            BloomFilter bloomFilter = this.consumerFilterManager.getBloomFilter();\n            if (filterBitMap == null || !this.bloomDataValid\n                || filterBitMap.length * Byte.SIZE != consumerFilterData.getBloomFilterData().getBitNum()) {\n                return true;\n            }\n\n            BitsArray bitsArray = null;\n            try {\n                bitsArray = BitsArray.create(filterBitMap);\n                boolean ret = bloomFilter.isHit(consumerFilterData.getBloomFilterData(), bitsArray);\n                log.debug(\"Pull {} by bit map:{}, {}, {}\", ret, consumerFilterData, bitsArray, cqExtUnit);\n                return ret;\n            } catch (Throwable e) {\n                log.error(\"bloom filter error, sub=\" + subscriptionData\n                    + \", filter=\" + consumerFilterData + \", bitMap=\" + bitsArray, e);\n            }\n        }\n\n        return true;\n    }\n\n    @Override\n    public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map<String, String> properties) {\n        if (subscriptionData == null) {\n            return true;\n        }\n\n        if (subscriptionData.isClassFilterMode()) {\n            return true;\n        }\n\n        if (ExpressionType.isTagType(subscriptionData.getExpressionType())) {\n            return true;\n        }\n\n        ConsumerFilterData realFilterData = this.consumerFilterData;\n        Map<String, String> tempProperties = properties;\n\n        // no expression\n        if (realFilterData == null || realFilterData.getExpression() == null\n            || realFilterData.getCompiledExpression() == null) {\n            return true;\n        }\n\n        if (tempProperties == null && msgBuffer != null) {\n            tempProperties = MessageDecoder.decodeProperties(msgBuffer);\n        }\n\n        Object ret = null;\n        try {\n            MessageEvaluationContext context = new MessageEvaluationContext(tempProperties);\n\n            ret = realFilterData.getCompiledExpression().evaluate(context);\n        } catch (Throwable e) {\n            log.error(\"Message Filter error, \" + realFilterData + \", \" + tempProperties, e);\n        }\n\n        log.debug(\"Pull eval result: {}, {}, {}\", ret, realFilterData, tempProperties);\n\n        if (ret == null || !(ret instanceof Boolean)) {\n            return false;\n        }\n\n        return (Boolean) ret;\n    }\n\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/filter/MessageEvaluationContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.filter;\n\nimport org.apache.rocketmq.filter.expression.EvaluationContext;\n\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.Map.Entry;\n\n/**\n * Evaluation context from message.\n */\npublic class MessageEvaluationContext implements EvaluationContext {\n\n    private Map<String, String> properties;\n\n    public MessageEvaluationContext(Map<String, String> properties) {\n        this.properties = properties;\n    }\n\n    @Override\n    public Object get(final String name) {\n        if (this.properties == null) {\n            return null;\n        }\n        return this.properties.get(name);\n    }\n\n    @Override\n    public Map<String, Object> keyValues() {\n        if (properties == null) {\n            return null;\n        }\n\n        Map<String, Object> copy = new HashMap<>(properties.size(), 1);\n\n        for (Entry<String, String> entry : properties.entrySet()) {\n            copy.put(entry.getKey(), entry.getValue());\n        }\n\n        return copy;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.latency;\n\nimport java.util.List;\nimport java.util.ArrayList;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport java.util.function.Supplier;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.future.FutureTaskExt;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.netty.RequestTask;\nimport org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode;\n\n/**\n * BrokerFastFailure will cover {@link BrokerController#getSendThreadPoolQueue()} and {@link\n * BrokerController#getPullThreadPoolQueue()}\n */\npublic class BrokerFastFailure {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private final ScheduledExecutorService scheduledExecutorService;\n    private final BrokerController brokerController;\n\n    private volatile long jstackTime = System.currentTimeMillis();\n\n    private final List<Pair<BlockingQueue<Runnable>, Supplier<Long>>> cleanExpiredRequestQueueList = new ArrayList<>();\n\n    public BrokerFastFailure(final BrokerController brokerController) {\n        this.brokerController = brokerController;\n        initCleanExpiredRequestQueueList();\n        this.scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1,\n            new ThreadFactoryImpl(\"BrokerFastFailureScheduledThread\", true,\n                brokerController == null ? null : brokerController.getBrokerConfig()));\n    }\n\n    private void initCleanExpiredRequestQueueList() {\n        cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getSendThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInSendQueue()));\n        cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getPullThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInPullQueue()));\n        cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getLitePullThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInLitePullQueue()));\n        cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getHeartbeatThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInHeartbeatQueue()));\n        cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getEndTransactionThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInTransactionQueue()));\n        cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getAckThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInAckQueue()));\n        cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getAdminBrokerThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInAdminBrokerQueue()));\n    }\n\n    public static RequestTask castRunnable(final Runnable runnable) {\n        try {\n            if (runnable instanceof FutureTaskExt) {\n                FutureTaskExt object = (FutureTaskExt) runnable;\n                return (RequestTask) object.getRunnable();\n            }\n        } catch (Throwable e) {\n            LOGGER.error(String.format(\"castRunnable exception, %s\", runnable.getClass().getName()), e);\n        }\n\n        return null;\n    }\n\n    public void start() {\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                if (brokerController.getBrokerConfig().isBrokerFastFailureEnable()) {\n                    cleanExpiredRequest();\n                }\n            }\n        }, 1000, 10, TimeUnit.MILLISECONDS);\n    }\n\n    private void cleanExpiredRequest() {\n\n        while (this.brokerController.getMessageStore().isOSPageCacheBusy()) {\n            try {\n                if (!this.brokerController.getSendThreadPoolQueue().isEmpty()) {\n                    final Runnable runnable = this.brokerController.getSendThreadPoolQueue().poll(0, TimeUnit.SECONDS);\n                    if (null == runnable) {\n                        break;\n                    }\n\n                    final RequestTask rt = castRunnable(runnable);\n                    if (rt != null) {\n                        rt.returnResponse(RemotingSysResponseCode.SYSTEM_BUSY, String.format(\n                            \"[PCBUSY_CLEAN_QUEUE]broker busy, start flow control for a while, period in queue: %sms, \"\n                                + \"size of queue: %d\",\n                            System.currentTimeMillis() - rt.getCreateTimestamp(),\n                            this.brokerController.getSendThreadPoolQueue().size()));\n                    }\n                } else {\n                    break;\n                }\n            } catch (Throwable ignored) {\n            }\n        }\n\n        for (Pair<BlockingQueue<Runnable>, Supplier<Long>> pair : cleanExpiredRequestQueueList) {\n            cleanExpiredRequestInQueue(pair.getObject1(), pair.getObject2().get());\n        }\n    }\n\n    void cleanExpiredRequestInQueue(final BlockingQueue<Runnable> blockingQueue, final long maxWaitTimeMillsInQueue) {\n        while (true) {\n            try {\n                if (!blockingQueue.isEmpty()) {\n                    final Runnable runnable = blockingQueue.peek();\n                    if (null == runnable) {\n                        break;\n                    }\n                    final RequestTask rt = castRunnable(runnable);\n                    if (rt == null || rt.isStopRun()) {\n                        break;\n                    }\n\n                    final long behind = System.currentTimeMillis() - rt.getCreateTimestamp();\n                    if (behind >= maxWaitTimeMillsInQueue) {\n                        if (blockingQueue.remove(runnable)) {\n                            rt.setStopRun(true);\n                            rt.returnResponse(RemotingSysResponseCode.SYSTEM_BUSY, String.format(\"[TIMEOUT_CLEAN_QUEUE]broker busy, start flow control for a while, period in queue: %sms, size of queue: %d\", behind, blockingQueue.size()));\n                            if (System.currentTimeMillis() - jstackTime > 15000) {\n                                jstackTime = System.currentTimeMillis();\n                                LOGGER.warn(\"broker jstack \\n \" + UtilAll.jstack());\n                            }\n                        }\n                    } else {\n                        break;\n                    }\n                } else {\n                    break;\n                }\n            } catch (Throwable ignored) {\n            }\n        }\n    }\n\n    public synchronized void addCleanExpiredRequestQueue(BlockingQueue<Runnable> cleanExpiredRequestQueue,\n        Supplier<Long> maxWaitTimeMillsInQueueSupplier) {\n        cleanExpiredRequestQueueList.add(new Pair<>(cleanExpiredRequestQueue, maxWaitTimeMillsInQueueSupplier));\n    }\n\n    public void shutdown() {\n        this.scheduledExecutorService.shutdown();\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/lite/AbstractLiteLifecycleManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.lite;\n\nimport com.google.common.collect.Sets;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.MessageStore;\n\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\n\nimport static org.apache.rocketmq.broker.offset.ConsumerOffsetManager.TOPIC_GROUP_SEPARATOR;\n\n/**\n * Abstract class of lite lifecycle manager, which is used to manage the TTL of lite topics\n * and the validity of subscription. The subclasses provide file CQ and rocksdb CQ implementations.\n */\npublic abstract class AbstractLiteLifecycleManager extends ServiceThread {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LITE_LOGGER_NAME);\n\n    protected final BrokerController brokerController;\n    protected final String brokerName;\n    protected final LiteSharding liteSharding;\n    protected MessageStore messageStore;\n    protected Map<String, Integer> ttlMap = Collections.emptyMap();\n    protected Map<String, Set<String>> subscriberGroupMap = Collections.emptyMap();\n\n    public AbstractLiteLifecycleManager(BrokerController brokerController, LiteSharding liteSharding) {\n        this.brokerController = brokerController;\n        this.brokerName = brokerController.getBrokerConfig().getBrokerName();\n        this.liteSharding = liteSharding;\n    }\n\n    public void init() {\n        this.messageStore = brokerController.getMessageStore();\n        assert messageStore != null;\n    }\n\n    /**\n     * This method actually returns NEXT slot index to use, starting from 0\n     */\n    public abstract long getMaxOffsetInQueue(String lmqName);\n\n    /**\n     * Collect expired LMQ of lite topic, and also attach its parent topic name\n     * return Pair of parent topic and lmq name, not null\n     */\n    public abstract List<Pair<String, String>> collectExpiredLiteTopic();\n\n    /**\n     * Collect LMQ by parent topic\n     * return lmq name list, not null\n     */\n    public abstract List<String> collectByParentTopic(String parentTopic);\n\n    /**\n     * Check if the subscription for the given LMQ is active.\n     * A subscription is considered active if either:\n     * - the current broker is responsible for this LMQ according to the sharding strategy\n     * - the LMQ exists (has messages) in the message store\n     */\n    public boolean isSubscriptionActive(String parentTopic, String lmqName) {\n        return brokerName.equals(liteSharding.shardingByLmqName(parentTopic, lmqName)) || isLmqExist(lmqName);\n    }\n\n    public int getLiteTopicCount(String parentTopic) {\n        if (!LiteMetadataUtil.isLiteMessageType(parentTopic, brokerController)) {\n            return 0;\n        }\n        return collectByParentTopic(parentTopic).size();\n    }\n\n    public boolean isLmqExist(String lmqName) {\n        return getMaxOffsetInQueue(lmqName) > 0;\n    }\n\n    public void cleanExpiredLiteTopic() {\n        try {\n            updateMetadata(); // necessary\n            List<Pair<String, String>> lmqToDelete = collectExpiredLiteTopic();\n            LOGGER.info(\"collect expired topic, size:{}\", lmqToDelete.size());\n            lmqToDelete.forEach(pair -> deleteLmq(pair.getObject1(), pair.getObject2()));\n            if (!lmqToDelete.isEmpty()) {\n                brokerController.getMessageStore().getQueueStore().flush();\n            }\n        } catch (Exception e) {\n            LOGGER.error(\"cleanExpiredLiteTopic error\", e);\n        }\n    }\n\n    public void cleanByParentTopic(String parentTopic) {\n        try {\n            if (!LiteMetadataUtil.isLiteMessageType(parentTopic, brokerController)) {\n                return;\n            }\n            updateMetadata(); // necessary\n            List<String> lmqToDelete = collectByParentTopic(parentTopic);\n            LOGGER.info(\"clean by parent topic, {}, size:{}\", parentTopic, lmqToDelete.size());\n            lmqToDelete.forEach(lmqName -> deleteLmq(parentTopic, lmqName));\n        } catch (Exception e) {\n            LOGGER.error(\"cleanByParentTopic error\", e);\n        }\n    }\n\n    @Override\n    public void run() {\n        LOGGER.info(\"Start checking lite ttl.\");\n        while (!this.isStopped()) {\n            long runningTime = System.currentTimeMillis() - brokerController.getShouldStartTime();\n            if (runningTime < brokerController.getBrokerConfig().getMinLiteTTl()) { // base protection for restart\n                this.waitForRunning(20 * 1000);\n                continue;\n            }\n\n            cleanExpiredLiteTopic();\n            long checkInterval = brokerController.getBrokerConfig().getLiteTtlCheckInterval();\n            this.waitForRunning(checkInterval);\n        }\n        LOGGER.info(\"End checking lite ttl.\");\n    }\n\n    public void updateMetadata() {\n        ttlMap = LiteMetadataUtil.getTopicTtlMap(brokerController);\n        subscriberGroupMap = LiteMetadataUtil.getSubscriberGroupMap(brokerController);\n    }\n\n    public boolean isLiteTopicExpired(String parentTopic, String lmqName, long maxOffset) {\n        if (!LiteUtil.isLiteTopicQueue(lmqName)) {\n            return false;\n        }\n        if (maxOffset <= 0) {\n            LOGGER.warn(\"unexpected condition, max offset <= 0, {}, {}\", lmqName, maxOffset);\n            return false;\n        }\n        long latestStoreTime =\n            this.brokerController.getMessageStore().getMessageStoreTimeStamp(lmqName, 0, maxOffset - 1);\n        long inactiveTime = System.currentTimeMillis() - latestStoreTime;\n        if (inactiveTime < brokerController.getBrokerConfig().getMinLiteTTl()) {\n            return false;\n        }\n        Integer minutes = ttlMap.get(parentTopic);\n        if (null == minutes) {\n            LOGGER.warn(\"unexpected condition, topic ttl not found. {}\", lmqName);\n            return false;\n        }\n        if (minutes <= 0) {\n            return false;\n        }\n        if (hasConsumerLag(lmqName, maxOffset, latestStoreTime, parentTopic)) {\n            return false;\n        }\n        return inactiveTime > minutes * 60 * 1000;\n    }\n\n    public void deleteLmq(String parentTopic, String lmqName) {\n        try {\n            Set<String> groups = subscriberGroupMap.getOrDefault(parentTopic, Collections.emptySet());\n            groups.forEach(group -> {\n                String topicAtGroup = lmqName + TOPIC_GROUP_SEPARATOR + group;\n                brokerController.getConsumerOffsetManager().getOffsetTable().remove(topicAtGroup);\n                brokerController.getConsumerOffsetManager().removeConsumerOffset(topicAtGroup); // no iteration\n                brokerController.getPopLiteMessageProcessor().getConsumerOrderInfoManager().remove(lmqName, group);\n            });\n            brokerController.getMessageStore().deleteTopics(Sets.newHashSet(lmqName));\n            boolean sharding = brokerName.equals(liteSharding.shardingByLmqName(parentTopic, lmqName));\n            brokerController.getLiteSubscriptionRegistry().cleanSubscription(lmqName, false);\n            brokerController.getConsumerOffsetManager().getPullOffsetTable().remove(\n                lmqName + TOPIC_GROUP_SEPARATOR + MixAll.TOOLS_CONSUMER_GROUP);\n            LOGGER.info(\"delete lmq finish. {}, sharding:{}\", lmqName, sharding);\n        } catch (Exception e) {\n            LOGGER.error(\"delete lmq error. {}\", lmqName, e);\n        }\n    }\n\n    /**\n     * Maybe we can check all subscriber groups, but currently consumer lag checking is not performed.\n     * Only inactive time of message sending is considered for TTL expiration.\n     */\n    public boolean hasConsumerLag(String lmqName, long maxOffset, long latestStoreTime, String parentTopic) {\n        return false;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/lite/LiteCtlListener.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.lite;\n\npublic interface LiteCtlListener {\n\n    void onRegister(String clientId, String group, String lmqName);\n\n    void onUnregister(String clientId, String group, String lmqName);\n\n    void onRemoveAll(String clientId, String group);\n\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/lite/LiteEventDispatcher.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.lite;\n\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.cache.Cache;\nimport com.google.common.cache.CacheBuilder;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.broker.pop.orderly.ConsumerOrderInfoManager;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.entity.ClientGroup;\nimport org.apache.rocketmq.common.lite.LiteSubscription;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.Comparator;\nimport java.util.HashMap;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Random;\nimport java.util.Set;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.ConcurrentSkipListSet;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.ThreadLocalRandom;\nimport java.util.concurrent.TimeUnit;\n\npublic class LiteEventDispatcher extends ServiceThread {\n\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LITE_LOGGER_NAME);\n    private static final Object PRESENT = new Object();\n    private static final long CLIENT_INACTIVE_INTERVAL = 10 * 1000; // inactive time when it has unprocessed events\n    private static final long CLIENT_LONG_POLLING_INTERVAL = 30 * 1000 + 5000; // at least a period of long polling as 30s\n    private static final long ACTIVE_CONSUMING_WINDOW = 5000;\n    private static final double LOW_WATER_MARK = 0.2;\n    private static final int BLACKLIST_EXPIRE_SECONDS = 10;\n    private static final int SCAN_LOG_INTERVAL = 10000;\n\n    private final BrokerController brokerController;\n    private final LiteSubscriptionRegistry liteSubscriptionRegistry;\n    private final AbstractLiteLifecycleManager liteLifecycleManager;\n    private final ConsumerOffsetManager consumerOffsetManager;\n    private ConsumerOrderInfoManager consumerOrderInfoManager;\n\n    private final ConcurrentMap<String, ClientEventSet> clientEventMap = new ConcurrentHashMap<>();\n    private final ConcurrentSkipListSet<FullDispatchRequest> fullDispatchSet = new ConcurrentSkipListSet<>(COMPARATOR);\n    private final ConcurrentMap<String, Object> fullDispatchMap = new ConcurrentHashMap<>(); // deduplication\n    private final Cache<String, Object> blacklist =\n        CacheBuilder.newBuilder().expireAfterWrite(BLACKLIST_EXPIRE_SECONDS, TimeUnit.SECONDS).build();\n    private final Random random = ThreadLocalRandom.current();\n    private long lastLogTime = System.currentTimeMillis();\n\n    public LiteEventDispatcher(BrokerController brokerController,\n        LiteSubscriptionRegistry liteSubscriptionRegistry, AbstractLiteLifecycleManager liteLifecycleManager) {\n        this.brokerController = brokerController;\n        this.liteSubscriptionRegistry = liteSubscriptionRegistry;\n        this.liteLifecycleManager = liteLifecycleManager;\n        this.consumerOffsetManager = brokerController.getConsumerOffsetManager();\n    }\n\n    public void init() {\n        this.consumerOrderInfoManager = brokerController.getPopLiteMessageProcessor().getConsumerOrderInfoManager();\n        this.liteSubscriptionRegistry.addListener(new LiteCtlListenerImpl());\n    }\n\n    /**\n     * If event mode is enabled, try to dispatch event to one client when message arriving or available.\n     * In most cases, there is only one subscriber for a LMQ under a consumer group,\n     * but also supports multiple clients consuming in share mode.\n     * When group is null, dispatch to all subscribers regardless of their group,\n     * when group is specified, only dispatch to subscribers belonging to this group.\n     * <p>\n     * If the expected number of subscriptions by each client is small, disabling event mode can be a choice.\n     */\n    public void dispatch(String group, String lmqName, int queueId, long offset, long msgStoreTime) {\n        if (!this.brokerController.getBrokerConfig().isEnableLiteEventMode()) {\n            return;\n        }\n        if (queueId != 0 || !LiteUtil.isLiteTopicQueue(lmqName)) {\n            return;\n        }\n        doDispatch(group, lmqName, null);\n    }\n\n    @SuppressWarnings(\"unchecked\")\n    private void doDispatch(String group, String lmqName, String excludeClientId) {\n        if (!this.brokerController.getBrokerConfig().isEnableLiteEventMode()) {\n            return;\n        }\n        Object subscribers = getAllSubscriber(group, lmqName);\n        if (null == subscribers) {\n            return;\n        }\n        if (subscribers instanceof List) {\n            selectAndDispatch(lmqName, (List<ClientGroup>) subscribers, excludeClientId);\n        }\n        if (subscribers instanceof Map) {\n            Map<String, List<ClientGroup>> map = (Map<String, List<ClientGroup>>) subscribers;\n            map.forEach((key, value) -> selectAndDispatch(lmqName, value, excludeClientId));\n        }\n    }\n\n    /**\n     * Select an appropriate client from the client list and try to dispatch the event to it.\n     * If there's only one client, dispatch directly to it.\n     * If there are multiple clients, randomly select one and consider fallback options\n     * Try to avoid dispatching to the excluded one but fallback if no other choice.\n     *\n     * @param clients all clients of one group\n     * @param excludeClientId the client ID to exclude from selection, probably consuming blocked.\n     */\n    @VisibleForTesting\n    public void selectAndDispatch(String lmqName, List<ClientGroup> clients, String excludeClientId) {\n        if (!this.brokerController.getBrokerConfig().isEnableLiteEventMode()) {\n            return;\n        }\n        if (CollectionUtils.isEmpty(clients)) {\n            return;\n        }\n\n        String clientId = null; // the selected one\n        if (clients.size() == 1) {\n            clientId = clients.get(0).clientId;\n            if (brokerController.getBrokerConfig().isEnableLitePopLog() && clientId.equals(excludeClientId)) {\n                LOGGER.info(\"no others, still dispatch to {}, {}\", clientId, lmqName);\n            }\n            if (!tryDispatchToClient(lmqName, clientId, clients.get(0).group)) {\n                clientId = null;\n            }\n        } else {\n            int start = random.nextInt(clients.size());\n            boolean dispatched = false;\n            List<ClientGroup> fallbackList = new ArrayList<>(clients.size());\n            for (int i = 0; i < clients.size(); i++) {\n                int index = (start + i) % clients.size();\n                clientId = clients.get(index).clientId;\n                if (clientId.equals(excludeClientId)) {\n                    fallbackList.add(clients.get(index));\n                    continue;\n                }\n                if (blacklist.getIfPresent(clientId) != null) {\n                    fallbackList.add(clients.get(index));\n                    continue;\n                }\n                if (tryDispatchToClient(lmqName, clientId, clients.get(index).group)) {\n                    dispatched = true;\n                    break;\n                }\n            }\n            if (!dispatched) {\n                clientId = null;\n                for (ClientGroup clientGroup : fallbackList) {\n                    if (tryDispatchToClient(lmqName, clientGroup.clientId, clientGroup.group)) {\n                        clientId = clientGroup.clientId;\n                        break;\n                    }\n                }\n            }\n        }\n        if (clientId != null) {\n            this.brokerController.getPopLiteMessageProcessor().getPopLiteLongPollingService()\n                .notifyMessageArriving(clientId, true, 0, clients.get(0).group);\n        }\n    }\n\n    /**\n     * Try to dispatch an event to a selected client by adding it to the client's event queue.\n     * If the event queue is full, mark a full dispatch for retry later.\n     */\n    @VisibleForTesting\n    public boolean tryDispatchToClient(String lmqName, String clientId, String group) {\n        ClientEventSet eventSet = clientEventMap.computeIfAbsent(clientId, key -> new ClientEventSet(group));\n        if (eventSet.offer(lmqName)) {\n            return true;\n        }\n        scheduleFullDispatch(clientId, group, blacklist.getIfPresent(clientId) != null);\n        LOGGER.warn(\"client event set is full. {}\", clientId);\n        return false;\n    }\n\n    /**\n     * Get an iterator for iterating over events for a specific client.\n     * In lite event mode, returns events from the client's event queue,\n     * or else returns topics from the client's subscription.\n     */\n    public Iterator<String> getEventIterator(String clientId) {\n        if (this.brokerController.getBrokerConfig().isEnableLiteEventMode()) {\n            return new EventSetIterator(clientEventMap.get(clientId));\n        } else {\n            LiteSubscription liteSubscription = liteSubscriptionRegistry.getLiteSubscription(clientId);\n            return liteSubscription != null && liteSubscription.getLiteTopicSet() != null ?\n                new LiteSubscriptionIterator(liteSubscription.getTopic(), liteSubscription.getLiteTopicSet().iterator())\n                    : Collections.emptyIterator();\n        }\n    }\n\n    /**\n     * Perform a full dispatch for a client which was previously marked for a delayed full dispatch.\n     * This always happens when a client's event queue is full or re-dispatching is needed.\n     * It iterates through all LMQ topics subscribed by the client and dispatches events for those\n     * with available messages.\n     */\n    public void doFullDispatch(String clientId, String group) {\n        if (!this.brokerController.getBrokerConfig().isEnableLiteEventMode()) {\n            return;\n        }\n        LiteSubscription subscription = liteSubscriptionRegistry.getLiteSubscription(clientId);\n        if (null == subscription || CollectionUtils.isEmpty(subscription.getLiteTopicSet())) {\n            LOGGER.info(\"client full dispatch, but no subscription. {}\", clientId);\n            return;\n        }\n        ClientEventSet eventSet = clientEventMap.computeIfAbsent(clientId, key -> new ClientEventSet(group));\n        if (eventSet.maybeBlock()) {\n            LOGGER.warn(\"client may block for a while, wait another period. {}\", clientId);\n            scheduleFullDispatch(clientId, group, true);\n            return;\n        }\n        boolean isActiveConsuming = eventSet.isActiveConsuming();\n        if (!eventSet.isLowWaterMark()) {\n            LOGGER.warn(\"client event set high water mark, wait another period. {}, {}\", clientId, isActiveConsuming);\n            scheduleFullDispatch(clientId, group, !isActiveConsuming);\n            return;\n        }\n        LOGGER.info(\"client full dispatch, {}, total:{}\", clientId, subscription.getLiteTopicSet().size());\n        int count = 0;\n        for (String lmqName : subscription.getLiteTopicSet()) {\n            long maxOffset = liteLifecycleManager.getMaxOffsetInQueue(lmqName);\n            if (maxOffset <= 0) {\n                continue;\n            }\n            long consumerOffset = consumerOffsetManager.queryOffset(group, lmqName, 0);\n            if (consumerOffset >= maxOffset) {\n                continue;\n            }\n            if (eventSet.offer(lmqName)) {\n                if (count++ % 10 == 0) {\n                    brokerController.getPopLiteMessageProcessor().getPopLiteLongPollingService()\n                        .notifyMessageArriving(clientId, true, 0, group);\n                }\n            } else {\n                LOGGER.warn(\"client event set full again, wait another period. {}, {}\", clientId, isActiveConsuming);\n                scheduleFullDispatch(clientId, group, !isActiveConsuming);\n                break;\n            }\n        }\n        brokerController.getPopLiteMessageProcessor().getPopLiteLongPollingService()\n            .notifyMessageArriving(clientId, true, 0, group);\n        LOGGER.info(\"client full dispatch finish. {}, dispatch:{}\", clientId, count);\n    }\n\n    /**\n     * Perform a full dispatch for all clients under a specific group, only invoked by admin for now.\n     */\n    public void doFullDispatchByGroup(String group) {\n        List<String> clientIds = liteSubscriptionRegistry.getAllClientIdByGroup(group);\n        LOGGER.info(\"do full dispatch by group, {}, size:{}\", group, clientIds.size());\n        for (String clientId : clientIds) {\n            doFullDispatch(clientId, group);\n        }\n    }\n\n    public void scheduleFullDispatch(String clientId, String group, boolean reentry) {\n        if (fullDispatchMap.putIfAbsent(clientId, PRESENT) != null) {\n            return;\n        }\n        int randomDelay = reentry ? random.nextInt(25 * 1000) : 0;\n        fullDispatchSet.add(new FullDispatchRequest(clientId, group,\n            brokerController.getBrokerConfig().getLiteEventFullDispatchDelayTime() + randomDelay));\n    }\n\n    /**\n     * Get all subscribers for a specific LMQ, with optional group filtering.\n     * To avoid unnecessary comparisons and wrapping, Object is used as the return type here.\n     * This method returns different types based on the subscription scenario:\n     * 1. When there's only one subscriber, return List<ClientGroup>\n     * 2. When group is specified, return List<ClientGroup> containing subscribers of that group\n     * 3. When group is null and multiple groups exist, return Map<String, List<ClientGroup>>\n     *    mapping each group to its subscribers\n     *\n     * @return Object that can be either List<ClientGroup> or Map<String, List<ClientGroup>> or null if not found\n     */\n    @VisibleForTesting\n    public Object getAllSubscriber(String group, String lmqName) {\n        Set<ClientGroup> observers = liteSubscriptionRegistry.getSubscriber(lmqName);\n        if (null == observers || observers.isEmpty()) {\n            return null;\n        }\n        if (observers.size() == 1) {\n            if (null == group || group.equals(observers.iterator().next().group)) {\n                return new ArrayList<>(observers);\n            }\n            return null;\n        }\n        if (group != null) {\n            List<ClientGroup> result = new ArrayList<>(4);\n            for (ClientGroup ele : observers) {\n                if (group.equals(ele.group)) {\n                    result.add(ele);\n                }\n            }\n            return !result.isEmpty() ? result : null;\n        }\n\n        Map<String, List<ClientGroup>> group2Clients = new HashMap<>(4);\n        for (ClientGroup ele : observers) {\n            group2Clients.computeIfAbsent(ele.group, k -> new ArrayList<>(2)).add(ele);\n        }\n        return group2Clients;\n    }\n\n    /**\n     * Get the last access time of a client's event set.\n     *\n     * @param clientId the client id\n     * @return the last access time in milliseconds, or -1 if client not found\n     */\n    public long getClientLastAccessTime(String clientId) {\n        ClientEventSet eventSet = clientEventMap.get(clientId);\n        if (eventSet != null) {\n            return eventSet.lastAccessTime;\n        }\n        return -1;\n    }\n\n    @Override\n    public String getServiceName() {\n        if (brokerController.getBrokerConfig().isInBrokerContainer()) {\n            return brokerController.getBrokerIdentity().getIdentifier() + LiteEventDispatcher.class.getSimpleName();\n        }\n        return LiteEventDispatcher.class.getSimpleName();\n    }\n\n    @Override\n    public void run() {\n        while (!this.isStopped()) {\n            long checkInterval = brokerController.getBrokerConfig().getLiteEventCheckInterval();\n            this.waitForRunning(checkInterval);\n            try {\n                scan();\n            } catch (Exception e) {\n                LOGGER.error(\"LiteEventDispatcher-scan error.\", e);\n            }\n        }\n    }\n\n    /**\n     * Due to the event pre-allocation mechanism, it is necessary to perform\n     * two main tasks to check inactive event queues and do full dispatch to reduce potential delivery latency.\n     * 1. Check client event set for inactive clients and re-dispatches their events\n     * 2. Process delayed full dispatch requests that are ready to be executed\n     */\n    public void scan() {\n        boolean needLog = System.currentTimeMillis() - lastLogTime > SCAN_LOG_INTERVAL;\n\n        // 1. check all client event set\n        if (needLog) {\n            LOGGER.info(\"Check client event set. size:{}\", clientEventMap.size());\n            lastLogTime = System.currentTimeMillis();\n        }\n        Iterator<Map.Entry<String, ClientEventSet>> iterator = clientEventMap.entrySet().iterator();\n        while (iterator.hasNext()) {\n            Map.Entry<String, ClientEventSet> entry = iterator.next();\n            ClientEventSet eventSet = entry.getValue();\n            if (!eventSet.maybeBlock()) {\n                continue;\n            }\n            String clientId = entry.getKey();\n            LOGGER.warn(\"remove inactive client and re-dispatch. {}, {}\", clientId, eventSet.events.size());\n            iterator.remove();\n            blacklist.put(clientId, PRESENT);\n            String event;\n            while ((event = eventSet.poll()) != null) {\n                doDispatch(eventSet.group, event, clientId); // may still dispatch to current client\n            }\n        }\n\n        // 2. perform full dispatch\n        if (needLog) {\n            LOGGER.info(\"Begin to trigger full dispatch. size:{}, mapSize:{}\", fullDispatchSet.size(), fullDispatchMap.size());\n            lastLogTime = System.currentTimeMillis();\n        }\n        FullDispatchRequest request;\n        while ((request = fullDispatchSet.pollFirst()) != null) {\n            if (request.timestamp > System.currentTimeMillis()) {\n                fullDispatchSet.add(request);\n                break;\n            }\n            fullDispatchMap.remove(request.clientId);\n            doFullDispatch(request.clientId, request.group);\n        }\n    }\n\n    public int getEventMapSize() {\n        return clientEventMap.size();\n    }\n\n    /**\n     * We use dual data structure to maintain the event queue for each client\n     * and ensure event deduplication to avoid duplicate events, although it\n     * has a bit more memory usage than a single concurrent set.\n     */\n    class ClientEventSet {\n        private final BlockingQueue<String> events;\n        private final ConcurrentMap<String, Object> map = new ConcurrentHashMap<>();\n        private final String group;\n        private volatile long lastAccessTime = System.currentTimeMillis();\n        private volatile long lastConsumeTime = System.currentTimeMillis();\n\n        public ClientEventSet(String group) {\n            this.group = group;\n            events = new LinkedBlockingQueue<>(LiteMetadataUtil.getMaxClientEventCount(group, brokerController));\n        }\n\n        // return false if and only if the queue is full, has race condition with poll(), but no side effect.\n        public boolean offer(String event) {\n            if (events.remainingCapacity() == 0) {\n                return false;\n            }\n            boolean rst;\n            if (map.putIfAbsent(event, PRESENT) == null) {\n                rst = events.offer(event);\n                if (!rst) {\n                    map.remove(event);\n                }\n            } else {\n                rst = true;\n            }\n            return rst;\n        }\n\n        public String poll() {\n            lastAccessTime = System.currentTimeMillis();\n            String event = events.poll();\n            if (event != null) {\n                map.remove(event);\n                lastConsumeTime = System.currentTimeMillis();\n            }\n            return event;\n        }\n\n        public boolean maybeBlock() {\n            long inactiveTime = System.currentTimeMillis() - lastAccessTime;\n            return inactiveTime > CLIENT_LONG_POLLING_INTERVAL\n                || !events.isEmpty() && inactiveTime > CLIENT_INACTIVE_INTERVAL;\n        }\n\n        public boolean isLowWaterMark() {\n            int used = events.size();\n            return (double) used / (used + events.remainingCapacity()) < LOW_WATER_MARK;\n        }\n\n        public boolean isActiveConsuming() {\n            return System.currentTimeMillis() - lastAccessTime < ACTIVE_CONSUMING_WINDOW;\n        }\n\n        public int size() {\n            return events.size();\n        }\n    }\n\n    class LiteCtlListenerImpl implements LiteCtlListener {\n\n        @Override\n        public void onRegister(String clientId, String group, String lmqName) {\n            if (liteLifecycleManager.isLmqExist(lmqName)) {\n                doDispatch(group, lmqName, null);\n            }\n        }\n\n        @Override\n        public void onUnregister(String clientId, String group, String lmqName) {\n        }\n\n        /**\n         *  Mostly triggered when client channel closed, ensure that lite subscriptions is cleared before.\n         */\n        @Override\n        public void onRemoveAll(String clientId, String group) {\n            ClientEventSet eventSet = clientEventMap.remove(clientId);\n            if (null == eventSet) {\n                return;\n            }\n            LOGGER.warn(\"Maybe client offline. {}\", clientId);\n            String event;\n            while ((event = eventSet.poll()) != null) {\n                doDispatch(eventSet.group, event, clientId);\n            }\n        }\n    }\n\n    static class EventSetIterator implements Iterator<String> {\n        private final ClientEventSet eventSet;\n\n        public EventSetIterator(ClientEventSet eventSet) {\n            this.eventSet = eventSet;\n        }\n\n        @Override\n        public boolean hasNext() {\n            return eventSet != null && !eventSet.events.isEmpty();\n        }\n\n        @Override\n        public String next() {\n            return eventSet.poll();\n        }\n    }\n\n    static class LiteSubscriptionIterator implements Iterator<String> {\n        private final Iterator<String> iterator;\n        private final String parentTopic;\n        public LiteSubscriptionIterator(String parentTopic, Iterator<String> iterator) {\n            this.parentTopic = parentTopic;\n            this.iterator = iterator;\n        }\n        @Override\n        public boolean hasNext() {\n            return iterator.hasNext();\n        }\n\n        @Override\n        public String next() {\n            return iterator.next();\n        }\n    }\n\n    static class FullDispatchRequest {\n        private final String clientId;\n        private final String group;\n        private final long timestamp;\n        public FullDispatchRequest(String clientId, String group, long delayMillis) {\n            this.clientId = clientId;\n            this.group = group;\n            this.timestamp = System.currentTimeMillis() + delayMillis;\n        }\n    }\n\n    // no need to compare group\n    static final Comparator<FullDispatchRequest> COMPARATOR = (r1, r2) -> {\n        if (null == r1 || null == r2 || null == r1.clientId || null == r2.clientId) {\n            return 0;\n        }\n        if (r1.clientId.equals(r2.clientId)) {\n            return 0;\n        }\n        int ret = Long.compare(r1.timestamp, r2.timestamp);\n        if (ret != 0) {\n            return ret;\n        }\n        return r1.clientId.compareTo(r2.clientId);\n    };\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/lite/LiteLifecycleManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.lite;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.queue.ConsumeQueueInterface;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentMap;\n\npublic class LiteLifecycleManager extends AbstractLiteLifecycleManager {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LITE_LOGGER_NAME);\n\n    public LiteLifecycleManager(BrokerController brokerController, LiteSharding liteSharding) {\n        super(brokerController, liteSharding);\n    }\n\n    @Override\n    public long getMaxOffsetInQueue(String lmqName) {\n        ConsumeQueueInterface consumeQueue = messageStore.getConsumeQueue(lmqName, 0);\n        return consumeQueue != null ? consumeQueue.getMaxOffsetInQueue() : 0L;\n    }\n\n    @Override\n    public List<String> collectByParentTopic(String parentTopic) {\n        if (StringUtils.isEmpty(parentTopic)) {\n            return Collections.emptyList();\n        }\n        List<String> resultList = new ArrayList<>();\n        Iterator<Map.Entry<String, ConcurrentMap<Integer, ConsumeQueueInterface>>> iterator =\n            messageStore.getQueueStore().getConsumeQueueTable().entrySet().iterator();\n        while (iterator.hasNext()) {\n            Map.Entry<String, ConcurrentMap<Integer, ConsumeQueueInterface>> entry = iterator.next();\n            if (LiteUtil.belongsTo(entry.getKey(), parentTopic)) {\n                resultList.add(entry.getKey());\n            }\n        }\n        return resultList;\n    }\n\n    @Override\n    public List<Pair<String, String>> collectExpiredLiteTopic() {\n        List<Pair<String, String>> lmqToDelete = new ArrayList<>();\n        Iterator<Map.Entry<String, ConcurrentMap<Integer, ConsumeQueueInterface>>> iterator =\n            messageStore.getQueueStore().getConsumeQueueTable().entrySet().iterator();\n        while (iterator.hasNext()) {\n            Map.Entry<String, ConcurrentMap<Integer, ConsumeQueueInterface>> entry = iterator.next();\n            String lmqName =  entry.getKey();\n            String parentTopic = LiteUtil.getParentTopic(lmqName);\n            if (null == parentTopic) {\n                continue;\n            }\n            Map<Integer, ConsumeQueueInterface> map = entry.getValue();\n            if (map.size() != 1 || null == map.get(0)) {\n                LOGGER.warn(\"unexpected lmq count. {}\", lmqName);\n                continue;\n            }\n            if (isLiteTopicExpired(parentTopic, entry.getKey(), map.get(0).getMaxOffsetInQueue())) {\n                lmqToDelete.add(new Pair<>(parentTopic, lmqName));\n            }\n        }\n        return lmqToDelete;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/lite/LiteMetadataUtil.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.lite;\n\nimport java.util.Collections;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.stream.Collectors;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\n\npublic class LiteMetadataUtil {\n\n    public static boolean isConsumeEnable(String group, BrokerController brokerController) {\n        if (null == group || null == brokerController) {\n            return false;\n        }\n        SubscriptionGroupConfig groupConfig =\n            brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(group);\n        return null != groupConfig && groupConfig.isConsumeEnable();\n    }\n\n    public static boolean isLiteMessageType(String parentTopic, BrokerController brokerController) {\n        if (null == parentTopic || null == brokerController) {\n            return false;\n        }\n        TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(parentTopic);\n        return topicConfig != null && TopicMessageType.LITE.equals(topicConfig.getTopicMessageType());\n    }\n\n    public static boolean isLiteGroupType(String group, BrokerController brokerController) {\n        if (null == group || null == brokerController) {\n            return false;\n        }\n        SubscriptionGroupConfig groupConfig =\n            brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(group);\n        return null != groupConfig && groupConfig.getLiteBindTopic() != null;\n    }\n\n    public static String getLiteBindTopic(String group, BrokerController brokerController) {\n        if (null == group || null == brokerController) {\n            return null;\n        }\n        SubscriptionGroupConfig groupConfig =\n            brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(group);\n        return null != groupConfig ? groupConfig.getLiteBindTopic() : null;\n    }\n\n    public static boolean isSubLiteExclusive(String group, BrokerController brokerController) {\n        if (null == group || null == brokerController) {\n            return false;\n        }\n        SubscriptionGroupConfig groupConfig =\n            brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(group);\n        return null != groupConfig && groupConfig.isLiteSubExclusive();\n    }\n\n    public static boolean isResetOffsetInExclusiveMode(String group, BrokerController brokerController) {\n        if (null == group || null == brokerController) {\n            return false;\n        }\n        SubscriptionGroupConfig groupConfig =\n            brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(group);\n        return null != groupConfig && groupConfig.isResetOffsetInExclusiveMode();\n    }\n\n    public static boolean isResetOffsetOnUnsubscribe(String group, BrokerController brokerController) {\n        if (null == group || null == brokerController) {\n            return false;\n        }\n        SubscriptionGroupConfig groupConfig =\n            brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(group);\n        return null != groupConfig && groupConfig.isResetOffsetOnUnsubscribe();\n    }\n\n    public static int getMaxClientEventCount(String group, BrokerController brokerController) {\n        if (null == group || null == brokerController) {\n            return -1;\n        }\n        SubscriptionGroupConfig groupConfig =\n            brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(group);\n        if (null == groupConfig || groupConfig.getMaxClientEventCount() <= 0) {\n            return brokerController.getBrokerConfig().getMaxClientEventCount();\n        }\n        return groupConfig.getMaxClientEventCount();\n    }\n\n    public static Map<String, Integer> getTopicTtlMap(BrokerController brokerController) {\n        if (null == brokerController) {\n            return Collections.emptyMap();\n        }\n        ConcurrentMap<String, TopicConfig> topicConfigTable =\n            brokerController.getTopicConfigManager().getTopicConfigTable();\n\n        return topicConfigTable.entrySet().stream()\n            .filter(entry -> entry.getValue().getTopicMessageType().equals(TopicMessageType.LITE))\n            .collect(Collectors.toMap(\n                entry -> entry.getKey(),\n                entry -> entry.getValue().getLiteTopicExpiration()\n            ));\n    }\n\n    public static Map<String, Set<String>> getSubscriberGroupMap(BrokerController brokerController) {\n        if (null == brokerController) {\n            return Collections.emptyMap();\n        }\n        ConcurrentMap<String, SubscriptionGroupConfig> groupTable =\n            brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable();\n\n        return groupTable.entrySet().stream()\n            .filter(entry -> entry.getValue().getLiteBindTopic() != null)\n            .collect(Collectors.groupingBy(\n                entry -> entry.getValue().getLiteBindTopic(),\n                Collectors.mapping(Map.Entry::getKey, Collectors.toSet())\n            ));\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/lite/LiteQuotaException.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.lite;\n\npublic class LiteQuotaException extends RuntimeException {\n    public LiteQuotaException(String message) {\n        super(message);\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/lite/LiteSharding.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.lite;\n\npublic interface LiteSharding {\n\n    String shardingByLmqName(String parentTopic, String lmqName);\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/lite/LiteShardingImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.lite;\n\nimport com.google.common.hash.Hashing;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.topic.TopicRouteInfoManager;\nimport org.apache.rocketmq.client.impl.producer.TopicPublishInfo;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\nimport java.util.List;\n\npublic class LiteShardingImpl implements LiteSharding {\n\n    private final BrokerController brokerController;\n    private final TopicRouteInfoManager topicRouteInfoManager;\n\n    public LiteShardingImpl(BrokerController brokerController, TopicRouteInfoManager topicRouteInfoManager) {\n        this.brokerController = brokerController;\n        this.topicRouteInfoManager = topicRouteInfoManager;\n    }\n\n    @Override\n    public String shardingByLmqName(String parentTopic, String lmqName) {\n        TopicPublishInfo topicPublishInfo = topicRouteInfoManager.tryToFindTopicPublishInfo(parentTopic);\n        if (topicPublishInfo == null) {\n            // if topic not exist, return current broker\n            return brokerController.getBrokerConfig().getBrokerName();\n        }\n        List<MessageQueue> writeQueues = topicPublishInfo.getMessageQueueList();\n        if (CollectionUtils.isEmpty(writeQueues)) {\n            return brokerController.getBrokerConfig().getBrokerName();\n        }\n        String liteTopic = LiteUtil.getLiteTopic(lmqName);\n        if (StringUtils.isEmpty(liteTopic)) {\n            return brokerController.getBrokerConfig().getBrokerName();\n        }\n        int bucket = Hashing.consistentHash(liteTopic.hashCode(), writeQueues.size());\n        MessageQueue targetQueue = writeQueues.get(bucket);\n        return targetQueue.getBrokerName();\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/lite/LiteSubscriptionRegistry.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.lite;\n\nimport io.netty.channel.Channel;\n\nimport java.util.List;\nimport java.util.Set;\nimport org.apache.rocketmq.common.entity.ClientGroup;\nimport org.apache.rocketmq.common.lite.LiteSubscription;\nimport org.apache.rocketmq.common.lite.OffsetOption;\n\npublic interface LiteSubscriptionRegistry {\n\n    void updateClientChannel(String clientId, Channel channel);\n\n    LiteSubscription getLiteSubscription(String clientId);\n\n    int getActiveSubscriptionNum();\n\n    void addPartialSubscription(String clientId, String group, String topic, Set<String> lmqNameSet, OffsetOption offsetOption);\n\n    void removePartialSubscription(String clientId, String group, String topic, Set<String> lmqNameSet);\n\n    void addCompleteSubscription(String clientId, String group, String topic, Set<String> newLmqNameSet, long version);\n\n    void removeCompleteSubscription(String clientId);\n\n    void addListener(LiteCtlListener listener);\n\n    Set<ClientGroup> getSubscriber(String lmqName);\n\n    List<String> getAllClientIdByGroup(String group);\n\n    void cleanSubscription(String lmqName, boolean notifyClient);\n\n    void start();\n\n    void shutdown();\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/lite/LiteSubscriptionRegistryImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.lite;\n\nimport com.google.common.annotations.VisibleForTesting;\nimport io.netty.channel.Channel;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.stream.Collectors;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.entity.ClientGroup;\nimport org.apache.rocketmq.common.lite.LiteSubscription;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.apache.rocketmq.common.lite.OffsetOption;\nimport org.apache.rocketmq.common.utils.ConcurrentHashMapUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.header.NotifyUnsubscribeLiteRequestHeader;\n\npublic class LiteSubscriptionRegistryImpl extends ServiceThread implements LiteSubscriptionRegistry {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LITE_LOGGER_NAME);\n\n    protected final ConcurrentMap<String/*clientId*/, Channel> clientChannels = new ConcurrentHashMap<>();\n    protected final ConcurrentMap<String/*clientId*/, LiteSubscription> client2Subscription = new ConcurrentHashMap<>();\n    protected final ConcurrentMap<String/*lmqName*/, Set<ClientGroup>> liteTopic2Group = new ConcurrentHashMap<>();\n\n    private final List<LiteCtlListener> listeners = new ArrayList<>();\n    private final BrokerController brokerController;\n    private final AbstractLiteLifecycleManager liteLifecycleManager;\n\n    public LiteSubscriptionRegistryImpl(BrokerController brokerController,\n        AbstractLiteLifecycleManager liteLifecycleManager) {\n        this.brokerController = brokerController;\n        this.liteLifecycleManager = liteLifecycleManager;\n    }\n\n    // Number of active liteTopic references.\n    // [(client1, liteTopic1), (client2, liteTopic1)] counts as two active references.\n    protected final AtomicInteger activeNum = new AtomicInteger(0);\n\n    @Override\n    public void updateClientChannel(String clientId, Channel channel) {\n        clientChannels.put(clientId, channel);\n    }\n\n    @Override\n    public void addPartialSubscription(String clientId, String group, String topic, Set<String> lmqNameSet,\n        OffsetOption offsetOption) {\n        long maxCount = brokerController.getBrokerConfig().getMaxLiteSubscriptionCount();\n        if (getActiveSubscriptionNum() >= maxCount) {\n            // No need to check existence, if reach here, it must be new.\n            throw new LiteQuotaException(\"lite subscription quota exceeded \" + maxCount);\n        }\n\n        LiteSubscription thisSub = getOrCreateLiteSubscription(clientId, group, topic);\n        // Utilize existing string object\n        final ClientGroup clientGroup = new ClientGroup(clientId, thisSub.getGroup());\n        for (String lmqName : lmqNameSet) {\n            if (!liteLifecycleManager.isSubscriptionActive(topic, lmqName)) {\n                continue;\n            }\n            thisSub.addLiteTopic(lmqName);\n            // First remove the old subscription\n            if (LiteMetadataUtil.isSubLiteExclusive(group, brokerController)) {\n                excludeClientByLmqName(clientId, group, lmqName);\n            }\n            resetOffset(lmqName, group, clientId, offsetOption);\n            addTopicGroup(clientGroup, lmqName);\n        }\n    }\n\n    @Override\n    public void removePartialSubscription(String clientId, String group, String topic, Set<String> lmqNameSet) {\n        LiteSubscription thisSub = getOrCreateLiteSubscription(clientId, group, topic);\n        ClientGroup clientGroup = new ClientGroup(clientId, thisSub.getGroup());\n        boolean isResetOffsetOnUnsubscribe = LiteMetadataUtil.isResetOffsetOnUnsubscribe(group, brokerController);\n        for (String lmqName : lmqNameSet) {\n            thisSub.removeLiteTopic(lmqName);\n            removeTopicGroup(clientGroup, lmqName, isResetOffsetOnUnsubscribe);\n        }\n    }\n\n    @Override\n    public void addCompleteSubscription(String clientId, String group, String topic, Set<String> lmqNameAll, long version) {\n        Set<String> lmqNameNew = lmqNameAll.stream()\n            .filter(lmqName -> liteLifecycleManager.isSubscriptionActive(topic, lmqName))\n            .collect(Collectors.toSet());\n\n        LiteSubscription thisSub = getOrCreateLiteSubscription(clientId, group, topic);\n        Set<String> lmqNamePrev = thisSub.getLiteTopicSet();\n        // Find topics to remove (in current set but not in new set)\n        Set<String> lmqNameRemove = lmqNamePrev.stream()\n            .filter(lmqName -> !lmqNameNew.contains(lmqName))\n            .collect(Collectors.toSet());\n\n        ClientGroup clientGroup = new ClientGroup(clientId, thisSub.getGroup());\n        lmqNameRemove.forEach(lmqName -> {\n            thisSub.removeLiteTopic(lmqName);\n            removeTopicGroup(clientGroup, lmqName, false);\n        });\n        lmqNameNew.forEach(lmqName -> {\n            thisSub.addLiteTopic(lmqName);\n            addTopicGroup(clientGroup, lmqName);\n        });\n    }\n\n    @Override\n    public void removeCompleteSubscription(String clientId) {\n        clientChannels.remove(clientId);\n        LiteSubscription thisSub = client2Subscription.remove(clientId);\n        if (thisSub == null) {\n            return;\n        }\n        LOGGER.info(\"removeCompleteSubscription, topic:{}, group:{}, clientId:{}\", thisSub.getTopic(), thisSub.getGroup(), clientId);\n        ClientGroup clientGroup = new ClientGroup(clientId, thisSub.getGroup());\n        thisSub.getLiteTopicSet().forEach(lmqName -> {\n            removeTopicGroup(clientGroup, lmqName, false);\n        });\n        for (LiteCtlListener listener : listeners) {\n            listener.onRemoveAll(clientId, thisSub.getGroup());\n        }\n    }\n\n    @Override\n    public void addListener(LiteCtlListener listener) {\n        listeners.add(listener);\n    }\n\n    @Override\n    public Set<ClientGroup> getSubscriber(String lmqName) {\n        return liteTopic2Group.get(lmqName);\n    }\n\n    /**\n     * Cleans up subscription for the given LMQ name.\n     * Removes all related client subscriptions and notifies listeners.\n     *\n     * @param lmqName the LMQ name to clean up\n     */\n    @Override\n    public void cleanSubscription(String lmqName, boolean notifyClient) {\n        Set<ClientGroup> topicGroupSet = liteTopic2Group.remove(lmqName);\n        if (CollectionUtils.isEmpty(topicGroupSet)) {\n            return;\n        }\n        for (ClientGroup topicGroup : topicGroupSet) {\n            LiteSubscription liteSubscription = client2Subscription.get(topicGroup.clientId);\n            if (liteSubscription == null) {\n                continue;\n            }\n            if (liteSubscription.removeLiteTopic(lmqName)) {\n                if (notifyClient) {\n                    notifyUnsubscribeLite(topicGroup.clientId, topicGroup.group, lmqName);\n                }\n                activeNum.decrementAndGet();\n            }\n        }\n    }\n\n    protected void addTopicGroup(ClientGroup clientGroup, String lmqName) {\n        Set<ClientGroup> topicGroupSet = liteTopic2Group\n            .computeIfAbsent(lmqName, k -> ConcurrentHashMap.newKeySet());\n        if (topicGroupSet.add(clientGroup)) {\n            activeNum.incrementAndGet();\n            for (LiteCtlListener listener : listeners) {\n                listener.onRegister(clientGroup.clientId, clientGroup.group, lmqName);\n            }\n        }\n    }\n\n    protected void removeTopicGroup(ClientGroup clientGroup, String lmqName, boolean resetOffset) {\n        Set<ClientGroup> topicGroupSet = liteTopic2Group.get(lmqName);\n        if (topicGroupSet == null) {\n            return;\n        }\n        if (topicGroupSet.remove(clientGroup)) {\n            activeNum.decrementAndGet();\n            for (LiteCtlListener listener : listeners) {\n                listener.onUnregister(clientGroup.clientId, clientGroup.group, lmqName);\n            }\n            if (resetOffset) {\n                resetOffset(lmqName, clientGroup.group, clientGroup.clientId,\n                    new OffsetOption(OffsetOption.Type.POLICY, OffsetOption.POLICY_MIN_VALUE));\n            }\n        }\n        if (topicGroupSet.isEmpty()) {\n            liteTopic2Group.remove(lmqName);\n        }\n    }\n\n    /**\n     * Remove clients that subscribe to the same liteTopic under the same group\n     */\n    protected void excludeClientByLmqName(String newClientId, String group, String lmqName) {\n        Set<ClientGroup> clientSet = liteTopic2Group.get(lmqName);\n        if (CollectionUtils.isEmpty(clientSet)) {\n            return;\n        }\n        List<ClientGroup> toRemove = clientSet.stream()\n            .filter(clientGroup -> Objects.equals(group, clientGroup.group))\n            .collect(Collectors.toList());\n\n        toRemove.forEach(clientGroup -> {\n            LiteSubscription liteSubscription = client2Subscription.get(clientGroup.clientId);\n            if (liteSubscription != null) {\n                liteSubscription.removeLiteTopic(lmqName);\n            }\n            notifyUnsubscribeLite(clientGroup.clientId, clientGroup.group, lmqName);\n            boolean resetOffset = LiteMetadataUtil.isResetOffsetInExclusiveMode(group, brokerController);\n            LOGGER.info(\"excludeClientByLmqName group:{}, lmqName:{}, resetOffset:{}, clientId:{} -> {}\",\n                group, lmqName, resetOffset, clientGroup.clientId, newClientId);\n            removeTopicGroup(clientGroup, lmqName, resetOffset);\n        });\n    }\n\n    /**\n     * Notify the client to remove the liteTopic subscription from its local memory\n     */\n    private void notifyUnsubscribeLite(String clientId, String group, String lmqName) {\n        String topic = LiteUtil.getParentTopic(lmqName);\n        String liteTopic = LiteUtil.getLiteTopic(lmqName);\n        Channel channel = clientChannels.get(clientId);\n        if (channel == null) {\n            LOGGER.warn(\"notifyUnsubscribeLite but channel is null, liteTopic:{}, group:{}, topic:{}, clientId:{},\",\n                liteTopic, group, topic, clientId);\n            return;\n        }\n\n        NotifyUnsubscribeLiteRequestHeader header = new NotifyUnsubscribeLiteRequestHeader();\n        header.setClientId(clientId);\n        header.setConsumerGroup(group);\n        header.setLiteTopic(liteTopic);\n        brokerController.getBroker2Client().notifyUnsubscribeLite(channel, header);\n        LOGGER.info(\"notifyUnsubscribeLite liteTopic:{}, group:{}, topic:{}, clientId:{}\", liteTopic, group, topic, clientId);\n    }\n\n    @Override\n    public LiteSubscription getLiteSubscription(String clientId) {\n        return client2Subscription.get(clientId);\n    }\n\n    @Override\n    public int getActiveSubscriptionNum() {\n        return activeNum.get();\n    }\n\n    @Override\n    public List<String> getAllClientIdByGroup(String group) {\n        return client2Subscription.entrySet().stream()\n            .filter(entry -> entry.getValue().getGroup().equals(group))\n            .map(Map.Entry::getKey)\n            .collect(Collectors.toList());\n    }\n\n    protected void resetOffset(String lmqName, String group, String clientId, OffsetOption offsetOption) {\n        if (null == offsetOption) {\n            return;\n        }\n        Long targetOffset = null;\n        long currentOffset = brokerController.getConsumerOffsetManager().queryOffset(group, lmqName, 0);\n        switch (offsetOption.getType()) {\n            case POLICY:\n                if (OffsetOption.POLICY_MIN_VALUE == offsetOption.getValue()) {\n                    targetOffset = 0L;\n                } else if (OffsetOption.POLICY_MAX_VALUE == offsetOption.getValue()) {\n                    targetOffset = liteLifecycleManager.getMaxOffsetInQueue(lmqName);\n                }\n                break;\n            case OFFSET:\n                targetOffset = offsetOption.getValue();\n                break;\n            case TAIL_N:\n                if (currentOffset >= 0) { // only when consumer offset exists\n                    targetOffset = Math.max(0L, currentOffset - offsetOption.getValue());\n                }\n                break;\n            case TIMESTAMP:\n                // timestamp option is disabled silently for now\n                break;\n        }\n\n        LOGGER.info(\"try to reset lite offset. {}, {}, {}, {}, current:{}, target:{}\",\n            group, lmqName, clientId, offsetOption, currentOffset, targetOffset);\n        if (targetOffset != null && currentOffset != targetOffset) {\n            brokerController.getConsumerOffsetManager().assignResetOffset(lmqName, group, 0, targetOffset);\n            brokerController.getPopLiteMessageProcessor().getConsumerOrderInfoManager().remove(lmqName, group);\n        }\n    }\n\n    private LiteSubscription getOrCreateLiteSubscription(String clientId, String group, String topic) {\n        LiteSubscription curLiteSubscription = ConcurrentHashMapUtils.computeIfAbsent(client2Subscription, clientId,\n            k -> new LiteSubscription().setGroup(group).setTopic(topic));\n        assert curLiteSubscription != null;\n        return curLiteSubscription;\n    }\n\n    @Override\n    public void run() {\n        LOGGER.info(\"Start checking lite subscription.\");\n        while (!this.isStopped()) {\n            long checkInterval = brokerController.getBrokerConfig().getLiteSubscriptionCheckInterval();\n            this.waitForRunning(checkInterval);\n\n            long checkTimeout = brokerController.getBrokerConfig().getLiteSubscriptionCheckTimeoutMills();\n            cleanupExpiredSubscriptions(checkTimeout);\n        }\n        LOGGER.info(\"End checking lite subscription.\");\n    }\n\n    /**\n     * Cleans up expired client subscriptions based on the provided timeout.\n     *\n     * @param checkTimeout the timeout in milliseconds to determine if a subscription is expired\n     */\n    @VisibleForTesting\n    protected void cleanupExpiredSubscriptions(long checkTimeout) {\n        // Step 1: Find expired clients and their subscription information\n        long currentTime = System.currentTimeMillis();\n        List<Map.Entry<String, LiteSubscription>> expiredEntries = client2Subscription.entrySet()\n            .stream()\n            .filter(entry -> currentTime - entry.getValue().getUpdateTime() > checkTimeout)\n            .collect(Collectors.toList());\n\n        // Step 2: Remove expired clients and their subscriptions\n        expiredEntries.forEach(expiredEntry -> {\n            String clientId = expiredEntry.getKey();\n            LiteSubscription liteSubscription = expiredEntry.getValue();\n            String group = liteSubscription.getGroup();\n            String topic = liteSubscription.getTopic();\n            removeCompleteSubscription(clientId);\n            LOGGER.info(\"Remove expired LiteSubscription, topic: {}, group: {}, clientId: {}, timeout: {}ms, expired: {}ms\",\n                topic, group, clientId, checkTimeout, System.currentTimeMillis() - liteSubscription.getUpdateTime());\n        });\n    }\n\n}"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/lite/RocksDBLiteLifecycleManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.lite;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.RocksDBMessageStore;\nimport org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable;\nimport org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore;\nimport org.apache.rocketmq.tieredstore.TieredMessageStore;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentMap;\n\npublic class RocksDBLiteLifecycleManager extends AbstractLiteLifecycleManager {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LITE_LOGGER_NAME);\n\n    private Map<String, Long> maxCqOffsetTable;\n\n    public RocksDBLiteLifecycleManager(BrokerController brokerController, LiteSharding liteSharding) {\n        super(brokerController, liteSharding);\n    }\n\n    @Override\n    public long getMaxOffsetInQueue(String lmqName) {\n        return maxCqOffsetTable.getOrDefault(lmqName + \"-0\", -1L) + 1;\n    }\n\n    @Override\n    public List<String> collectByParentTopic(String parentTopic) {\n        if (StringUtils.isEmpty(parentTopic)) {\n            return Collections.emptyList();\n        }\n        List<String> resultList = new ArrayList<>();\n        Iterator<Map.Entry<String, Long>> iterator = maxCqOffsetTable.entrySet().iterator();\n        while (iterator.hasNext()) {\n            Map.Entry<String, Long> entry = iterator.next();\n            String queueAndQid = entry.getKey();\n            String lmqName = queueAndQid.substring(0, queueAndQid.lastIndexOf(\"-\"));\n            if (LiteUtil.belongsTo(lmqName, parentTopic)) {\n                resultList.add(lmqName);\n            }\n        }\n        return resultList;\n    }\n\n    @Override\n    public List<Pair<String, String>> collectExpiredLiteTopic() {\n        List<Pair<String, String>> lmqToDelete = new ArrayList<>();\n        Iterator<Map.Entry<String, Long>> iterator = maxCqOffsetTable.entrySet().iterator();\n        while (iterator.hasNext()) {\n            Map.Entry<String, Long> entry = iterator.next();\n            String queueAndQid = entry.getKey();\n            String lmqName = queueAndQid.substring(0, queueAndQid.lastIndexOf(\"-\"));\n            String parentTopic = LiteUtil.getParentTopic(lmqName);\n            if (null == parentTopic) {\n                continue;\n            }\n            if (isLiteTopicExpired(parentTopic, lmqName, entry.getValue() + 1)) {\n                lmqToDelete.add(new Pair<>(parentTopic, lmqName));\n            }\n        }\n        return lmqToDelete;\n    }\n\n    @Override\n    public void init() {\n        super.init();\n        if (messageStore instanceof TieredMessageStore) { // only support TieredMessageStore plugin\n            messageStore = ((TieredMessageStore) messageStore).getDefaultStore();\n        }\n        if (!(messageStore instanceof RocksDBMessageStore)) {\n            LOGGER.warn(\"init failed, not a RocksDB store. {}\", messageStore.getClass());\n            return; // startup with lite feature disabled\n        }\n        try {\n            RocksDBConsumeQueueStore queueStore = (RocksDBConsumeQueueStore) messageStore.getQueueStore();\n            RocksDBConsumeQueueOffsetTable cqOffsetTable = (RocksDBConsumeQueueOffsetTable) FieldUtils.readField(\n                FieldUtils.getField(RocksDBConsumeQueueStore.class, \"rocksDBConsumeQueueOffsetTable\", true), queueStore);\n            @SuppressWarnings(\"unchecked\")\n            ConcurrentMap<String, Long> innerMaxCqOffsetTable = (ConcurrentMap<String, Long>) FieldUtils.readField(\n                FieldUtils.getField(RocksDBConsumeQueueOffsetTable.class, \"topicQueueMaxCqOffset\", true), cqOffsetTable);\n            maxCqOffsetTable = Collections.unmodifiableMap(innerMaxCqOffsetTable);\n        } catch (Exception e) {\n            LOGGER.error(\"LiteLifecycleManager-init error\", e);\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/loadbalance/MessageRequestModeManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.loadbalance;\n\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.BrokerPathConfigHelper;\nimport org.apache.rocketmq.common.ConfigManager;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.body.SetMessageRequestModeRequestBody;\n\npublic class MessageRequestModeManager extends ConfigManager {\n\n    private transient BrokerController brokerController;\n\n    private ConcurrentHashMap<String/*topic*/, ConcurrentHashMap<String/*consumerGroup*/, SetMessageRequestModeRequestBody>>\n        messageRequestModeMap = new ConcurrentHashMap<>();\n\n    public MessageRequestModeManager() {\n        // empty construct for decode\n    }\n\n    public MessageRequestModeManager(BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    public void setMessageRequestMode(String topic, String consumerGroup, SetMessageRequestModeRequestBody requestBody) {\n        ConcurrentHashMap<String, SetMessageRequestModeRequestBody> consumerGroup2ModeMap = messageRequestModeMap.get(topic);\n        if (consumerGroup2ModeMap == null) {\n            consumerGroup2ModeMap = new ConcurrentHashMap<>();\n            ConcurrentHashMap<String, SetMessageRequestModeRequestBody> pre =\n                messageRequestModeMap.putIfAbsent(topic, consumerGroup2ModeMap);\n            if (pre != null) {\n                consumerGroup2ModeMap = pre;\n            }\n        }\n        consumerGroup2ModeMap.put(consumerGroup, requestBody);\n    }\n\n    public SetMessageRequestModeRequestBody getMessageRequestMode(String topic, String consumerGroup) {\n        ConcurrentHashMap<String, SetMessageRequestModeRequestBody> consumerGroup2ModeMap = messageRequestModeMap.get(topic);\n        if (consumerGroup2ModeMap != null) {\n            return consumerGroup2ModeMap.get(consumerGroup);\n        }\n\n        return null;\n    }\n\n    public ConcurrentHashMap<String, ConcurrentHashMap<String, SetMessageRequestModeRequestBody>> getMessageRequestModeMap() {\n        return this.messageRequestModeMap;\n    }\n\n    public void setMessageRequestModeMap(ConcurrentHashMap<String, ConcurrentHashMap<String, SetMessageRequestModeRequestBody>> messageRequestModeMap) {\n        this.messageRequestModeMap = messageRequestModeMap;\n    }\n\n    @Override\n    public String encode() {\n        return this.encode(false);\n    }\n\n    @Override\n    public String configFilePath() {\n        return BrokerPathConfigHelper.getMessageRequestModePath(this.brokerController.getMessageStoreConfig().getStorePathRootDir());\n    }\n\n    @Override\n    public void decode(String jsonString) {\n        if (jsonString != null) {\n            MessageRequestModeManager obj = RemotingSerializable.fromJson(jsonString, MessageRequestModeManager.class);\n            if (obj != null) {\n                this.messageRequestModeMap = obj.messageRequestModeMap;\n            }\n        }\n    }\n\n    @Override\n    public String encode(boolean prettyFormat) {\n        return RemotingSerializable.toJson(this, prettyFormat);\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/longpolling/LmqPullRequestHoldService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.longpolling;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class LmqPullRequestHoldService extends PullRequestHoldService {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n\n    public LmqPullRequestHoldService(BrokerController brokerController) {\n        super(brokerController);\n    }\n\n    @Override\n    public String getServiceName() {\n        if (brokerController != null && brokerController.getBrokerConfig().isInBrokerContainer()) {\n            return this.brokerController.getBrokerIdentity().getIdentifier() + LmqPullRequestHoldService.class.getSimpleName();\n        }\n        return LmqPullRequestHoldService.class.getSimpleName();\n    }\n\n    @Override\n    public void checkHoldRequest() {\n        for (String key : pullRequestTable.keySet()) {\n            int idx = key.lastIndexOf(TOPIC_QUEUEID_SEPARATOR);\n            if (idx <= 0 || idx >= key.length() - 1) {\n                pullRequestTable.remove(key);\n                continue;\n            }\n            String topic = key.substring(0, idx);\n            int queueId = Integer.parseInt(key.substring(idx + 1));\n            try {\n                final long offset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);\n                this.notifyMessageArriving(topic, queueId, offset);\n            } catch (Throwable e) {\n                LOGGER.error(\"check hold request failed. topic={}, queueId={}\", topic, queueId, e);\n            }\n            if (MixAll.isLmq(topic)) {\n                ManyPullRequest mpr = pullRequestTable.get(key);\n                if (mpr == null || mpr.getPullRequestList() == null || mpr.getPullRequestList().isEmpty()) {\n                    pullRequestTable.remove(key);\n                }\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/longpolling/ManyPullRequest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.longpolling;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\npublic class ManyPullRequest {\n    private final ArrayList<PullRequest> pullRequestList = new ArrayList<>();\n\n    public synchronized void addPullRequest(final PullRequest pullRequest) {\n        this.pullRequestList.add(pullRequest);\n    }\n\n    public synchronized void addPullRequest(final List<PullRequest> many) {\n        this.pullRequestList.addAll(many);\n    }\n\n    public synchronized List<PullRequest> cloneListAndClear() {\n        if (!this.pullRequestList.isEmpty()) {\n            List<PullRequest> result = (ArrayList<PullRequest>) this.pullRequestList.clone();\n            this.pullRequestList.clear();\n            return result;\n        }\n\n        return null;\n    }\n\n    public ArrayList<PullRequest> getPullRequestList() {\n        return pullRequestList;\n    }\n\n    public synchronized boolean isEmpty() {\n        return this.pullRequestList.isEmpty();\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotificationRequest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.longpolling;\n\nimport java.util.concurrent.atomic.AtomicBoolean;\n\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\nimport io.netty.channel.Channel;\n\npublic class NotificationRequest {\n    private RemotingCommand remotingCommand;\n    private Channel channel;\n    private long expired;\n    private AtomicBoolean complete = new AtomicBoolean(false);\n\n    public NotificationRequest(RemotingCommand remotingCommand, Channel channel, long expired) {\n        this.channel = channel;\n        this.remotingCommand = remotingCommand;\n        this.expired = expired;\n    }\n\n    public Channel getChannel() {\n        return channel;\n    }\n\n    public RemotingCommand getRemotingCommand() {\n        return remotingCommand;\n    }\n\n    public boolean isTimeout() {\n        return System.currentTimeMillis() > (expired - 500);\n    }\n\n    public boolean complete() {\n        return complete.compareAndSet(false, true);\n    }\n\n    @Override\n    public String toString() {\n        return remotingCommand.toString();\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.longpolling;\n\nimport java.util.Map;\n\nimport org.apache.rocketmq.broker.lite.LiteEventDispatcher;\nimport org.apache.rocketmq.broker.processor.NotificationProcessor;\nimport org.apache.rocketmq.broker.processor.PopMessageProcessor;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.apache.rocketmq.store.MessageArrivingListener;\n\npublic class NotifyMessageArrivingListener implements MessageArrivingListener {\n    private final PullRequestHoldService pullRequestHoldService;\n    private final PopMessageProcessor popMessageProcessor;\n    private final NotificationProcessor notificationProcessor;\n    private final LiteEventDispatcher liteEventDispatcher;\n\n    public NotifyMessageArrivingListener(final PullRequestHoldService pullRequestHoldService, final PopMessageProcessor popMessageProcessor, final NotificationProcessor notificationProcessor, final LiteEventDispatcher liteEventDispatcher) {\n        this.pullRequestHoldService = pullRequestHoldService;\n        this.popMessageProcessor = popMessageProcessor;\n        this.notificationProcessor = notificationProcessor;\n        this.liteEventDispatcher = liteEventDispatcher;\n    }\n\n    @Override\n    public void arriving(String topic, int queueId, long logicOffset, long tagsCode,\n                         long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {\n        if (LiteUtil.isLiteTopicQueue(topic)) {\n            this.liteEventDispatcher.dispatch(null, topic, queueId, logicOffset, msgStoreTime);\n            return;\n        }\n        this.pullRequestHoldService.notifyMessageArriving(\n            topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties);\n        this.popMessageProcessor.notifyMessageArriving(\n            topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties);\n        this.notificationProcessor.notifyMessageArriving(\n            topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties);\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/longpolling/PollingHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.longpolling;\n\nimport org.apache.rocketmq.remoting.protocol.header.NotificationRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;\n\npublic class PollingHeader {\n    private final String consumerGroup;\n    private final String topic;\n    private final int queueId;\n    private final long bornTime;\n    private final long pollTime;\n\n    public PollingHeader(PopMessageRequestHeader requestHeader) {\n        this.consumerGroup = requestHeader.getConsumerGroup();\n        this.topic = requestHeader.getTopic();\n        this.queueId = requestHeader.getQueueId();\n        this.bornTime = requestHeader.getBornTime();\n        this.pollTime = requestHeader.getPollTime();\n    }\n\n    public PollingHeader(NotificationRequestHeader requestHeader) {\n        this.consumerGroup = requestHeader.getConsumerGroup();\n        this.topic = requestHeader.getTopic();\n        this.queueId = requestHeader.getQueueId();\n        this.bornTime = requestHeader.getBornTime();\n        this.pollTime = requestHeader.getPollTime();\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public int getQueueId() {\n        return queueId;\n    }\n\n    public long getBornTime() {\n        return bornTime;\n    }\n\n    public long getPollTime() {\n        return pollTime;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/longpolling/PollingResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.longpolling;\n\npublic enum PollingResult {\n    POLLING_SUC,\n    POLLING_FULL,\n    POLLING_TIMEOUT,\n    NOT_POLLING;\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopCommandCallback.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.longpolling;\n\nimport java.util.concurrent.CompletableFuture;\nimport java.util.function.BiConsumer;\nimport org.apache.rocketmq.broker.metrics.ConsumerLagCalculator;\nimport org.apache.rocketmq.remoting.CommandCallback;\n\npublic class PopCommandCallback implements CommandCallback {\n\n    private final BiConsumer<ConsumerLagCalculator.ProcessGroupInfo,\n        CompletableFuture<ConsumerLagCalculator.CalculateLagResult>> biConsumer;\n    private final ConsumerLagCalculator.ProcessGroupInfo info;\n    private final CompletableFuture<ConsumerLagCalculator.CalculateLagResult> future;\n\n    public PopCommandCallback(\n        BiConsumer<ConsumerLagCalculator.ProcessGroupInfo,\n            CompletableFuture<ConsumerLagCalculator.CalculateLagResult>> biConsumer,\n        ConsumerLagCalculator.ProcessGroupInfo info,\n        CompletableFuture<ConsumerLagCalculator.CalculateLagResult> future) {\n\n        this.biConsumer = biConsumer;\n        this.info = info;\n        this.future = future;\n    }\n\n    @Override\n    public void accept() {\n        biConsumer.accept(info, future);\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopLiteLongPollingService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.longpolling;\n\nimport com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentSkipListSet;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingAbstract;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.netty.RequestTask;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\nimport static org.apache.rocketmq.broker.longpolling.PollingResult.NOT_POLLING;\nimport static org.apache.rocketmq.broker.longpolling.PollingResult.POLLING_FULL;\nimport static org.apache.rocketmq.broker.longpolling.PollingResult.POLLING_SUC;\nimport static org.apache.rocketmq.broker.longpolling.PollingResult.POLLING_TIMEOUT;\n\n/**\n * Long polling service specifically designed for lite consumption.\n * Stores pending requests in memory using clientId as the key instead of topic@cid@qid.\n * Notification and resource checking mechanisms are identical to those in PopLongPollingService.\n */\npublic class PopLiteLongPollingService extends ServiceThread {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LITE_LOGGER_NAME);\n\n    private final BrokerController brokerController;\n    private final NettyRequestProcessor processor;\n    private final ConcurrentLinkedHashMap<String, ConcurrentSkipListSet<PopRequest>> pollingMap;\n    private long lastCleanTime = 0;\n\n    private final AtomicLong totalPollingNum = new AtomicLong(0);\n    private final boolean notifyLast;\n\n    public PopLiteLongPollingService(BrokerController brokerController, NettyRequestProcessor processor, boolean notifyLast) {\n        this.brokerController = brokerController;\n        this.processor = processor;\n        this.pollingMap = new ConcurrentLinkedHashMap.Builder<String, ConcurrentSkipListSet<PopRequest>>()\n            .maximumWeightedCapacity(this.brokerController.getBrokerConfig().getPopPollingMapSize()).build();\n        this.notifyLast = notifyLast;\n    }\n\n    @Override\n    public String getServiceName() {\n        if (brokerController.getBrokerConfig().isInBrokerContainer()) {\n            return brokerController.getBrokerIdentity().getIdentifier() + PopLiteLongPollingService.class.getSimpleName();\n        }\n        return PopLiteLongPollingService.class.getSimpleName();\n    }\n\n    @Override\n    public void run() {\n        int i = 0;\n        while (!this.stopped) {\n            try {\n                this.waitForRunning(20);\n                i++;\n                if (pollingMap.isEmpty()) {\n                    continue;\n                }\n                long tmpTotalPollingNum = 0;\n                for (Map.Entry<String, ConcurrentSkipListSet<PopRequest>> entry : pollingMap.entrySet()) {\n                    String key = entry.getKey();\n                    ConcurrentSkipListSet<PopRequest> popQ = entry.getValue();\n                    if (popQ == null) {\n                        continue;\n                    }\n                    PopRequest first;\n                    do {\n                        first = popQ.pollFirst();\n                        if (first == null) {\n                            break;\n                        }\n                        if (!first.isTimeout()) {\n                            if (popQ.add(first)) {\n                                break;\n                            } else {\n                                LOGGER.info(\"lite polling, add back again but failed. {}\", first);\n                            }\n                        }\n                        if (brokerController.getBrokerConfig().isEnablePopLog()) {\n                            LOGGER.info(\"timeout , wakeUp lite polling : {}\", first);\n                        }\n                        totalPollingNum.decrementAndGet();\n                        wakeUp(first);\n                    }\n                    while (true);\n                    if (i >= 100) {\n                        long tmpPollingNum = popQ.size();\n                        tmpTotalPollingNum = tmpTotalPollingNum + tmpPollingNum;\n                        if (tmpPollingNum > 20) {\n                            LOGGER.info(\"lite polling queue {} , size={} \", key, tmpPollingNum);\n                        }\n                    }\n                }\n\n                if (i >= 100) {\n                    LOGGER.info(\"litePollingMapSize={}, tmpTotalSize={}, atomicTotalSize={}, diffSize={}\",\n                        pollingMap.size(), tmpTotalPollingNum, totalPollingNum.get(),\n                        Math.abs(totalPollingNum.get() - tmpTotalPollingNum));\n                    totalPollingNum.set(tmpTotalPollingNum);\n                    i = 0;\n                }\n\n                // clean unused\n                if (lastCleanTime == 0 || System.currentTimeMillis() - lastCleanTime > 3 * 60 * 1000) {\n                    cleanUnusedResource();\n                }\n            } catch (Throwable e) {\n                LOGGER.error(\"checkLitePolling error\", e);\n            }\n        }\n        // clean all;\n        try {\n            for (Map.Entry<String, ConcurrentSkipListSet<PopRequest>> entry : pollingMap.entrySet()) {\n                ConcurrentSkipListSet<PopRequest> popQ = entry.getValue();\n                PopRequest first;\n                while ((first = popQ.pollFirst()) != null) {\n                    wakeUp(first);\n                }\n            }\n        } catch (Throwable ignored) {\n        }\n    }\n\n    public boolean notifyMessageArriving(final String clientId, boolean force, long msgStoreTime, String group) {\n        String pollingKey = getPollingKey(clientId, group);\n        ConcurrentSkipListSet<PopRequest> remotingCommands = pollingMap.get(pollingKey);\n        if (remotingCommands == null || remotingCommands.isEmpty()) {\n            return false;\n        }\n        PopRequest popRequest = pollRemotingCommands(remotingCommands);\n        if (popRequest == null) {\n            return false;\n        }\n\n        if (brokerController.getBrokerConfig().isEnableLitePopLog()) {\n            LOGGER.info(\"notify lite polling, wakeUp: {}\", popRequest);\n        }\n        return wakeUp(popRequest);\n    }\n\n    public boolean wakeUp(final PopRequest request) {\n        if (request == null || !request.complete()) {\n            return false;\n        }\n        if (!request.getCtx().channel().isActive()) {\n            return false;\n        }\n\n        Runnable run = () -> {\n            try {\n                final RemotingCommand response = processor.processRequest(request.getCtx(), request.getRemotingCommand());\n                if (response != null) {\n                    response.setOpaque(request.getRemotingCommand().getOpaque());\n                    response.markResponseType();\n                    NettyRemotingAbstract.writeResponse(request.getChannel(), request.getRemotingCommand(), response, future -> {\n                        if (!future.isSuccess()) {\n                            LOGGER.error(\"ProcessRequestWrapper response to {} failed\", request.getChannel().remoteAddress(), future.cause());\n                            LOGGER.error(request.toString());\n                            LOGGER.error(response.toString());\n                        }\n                    }, brokerController.getBrokerMetricsManager().getRemotingMetricsManager());\n                }\n            } catch (Exception e) {\n                LOGGER.error(\"ExecuteRequestWhenWakeup error.\", e);\n            }\n        };\n\n        this.brokerController.getPullMessageExecutor().submit(\n            new RequestTask(run, request.getChannel(), request.getRemotingCommand()));\n        return true;\n    }\n\n    public PollingResult polling(final ChannelHandlerContext ctx, RemotingCommand remotingCommand,\n        long bornTime, long pollTime, String clientId, String group) {\n        if (pollTime <= 0 || this.isStopped()) {\n            return NOT_POLLING;\n        }\n        long expired = bornTime + pollTime;\n        final PopRequest request = new PopRequest(remotingCommand, ctx, expired, null, null);\n        boolean isFull = totalPollingNum.get() >= this.brokerController.getBrokerConfig().getMaxPopPollingSize();\n        if (isFull) {\n            LOGGER.info(\"lite polling {}, result POLLING_FULL, total:{}\", remotingCommand, totalPollingNum.get());\n            return POLLING_FULL;\n        }\n        boolean isTimeout = request.isTimeout();\n        if (isTimeout) {\n            if (brokerController.getBrokerConfig().isEnablePopLog()) {\n                LOGGER.info(\"lite polling {}, result POLLING_TIMEOUT\", remotingCommand);\n            }\n            return POLLING_TIMEOUT;\n        }\n\n        String pollingKey = getPollingKey(clientId, group);\n        ConcurrentSkipListSet<PopRequest> queue = pollingMap.get(pollingKey);\n        if (queue == null) {\n            queue = new ConcurrentSkipListSet<>(PopRequest.COMPARATOR);\n            ConcurrentSkipListSet<PopRequest> old = pollingMap.putIfAbsent(pollingKey, queue);\n            if (old != null) {\n                queue = old;\n            }\n        } else {\n            // check size\n            int size = queue.size();\n            if (size > brokerController.getBrokerConfig().getPopPollingSize()) {\n                LOGGER.info(\"lite polling {}, result POLLING_FULL, singleSize:{}\", remotingCommand, size);\n                return POLLING_FULL;\n            }\n        }\n        if (queue.add(request)) {\n            remotingCommand.setSuspended(true);\n            totalPollingNum.incrementAndGet();\n            if (brokerController.getBrokerConfig().isEnableLitePopLog()) {\n                LOGGER.info(\"lite polling {}, result POLLING_SUC\", remotingCommand);\n            }\n            return POLLING_SUC;\n        } else {\n            LOGGER.info(\"lite polling {}, result POLLING_FULL, add fail, {}\", request, queue);\n            return POLLING_FULL;\n        }\n    }\n\n    private void cleanUnusedResource() {\n        try {\n            pollingMap.entrySet().removeIf(entry -> {\n                if (CollectionUtils.isEmpty(entry.getValue())) {\n                    LOGGER.info(\"clean polling structure of {}\", entry.getKey()); // see getPollingKey()\n                    return true;\n                }\n                return false;\n            });\n        } catch (Throwable ignored) {\n        }\n        lastCleanTime = System.currentTimeMillis();\n    }\n\n    private PopRequest pollRemotingCommands(ConcurrentSkipListSet<PopRequest> remotingCommands) {\n        if (remotingCommands == null || remotingCommands.isEmpty()) {\n            return null;\n        }\n\n        PopRequest popRequest;\n        do {\n            if (notifyLast) {\n                popRequest = remotingCommands.pollLast();\n            } else {\n                popRequest = remotingCommands.pollFirst();\n            }\n            if (popRequest != null) {\n                totalPollingNum.decrementAndGet();\n            }\n        } while (popRequest != null && !popRequest.getChannel().isActive());\n\n        return popRequest;\n    }\n\n    // Assume that clientId is unique, so we use it as the key for now.\n    private String getPollingKey(String clientId, String group) {\n        return clientId;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopLongPollingService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.longpolling;\n\nimport com.github.benmanes.caffeine.cache.Cache;\nimport com.github.benmanes.caffeine.cache.Caffeine;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.ArrayList;\nimport java.util.Iterator;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentSkipListSet;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.PopAckConstants;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.CommandCallback;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingAbstract;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.netty.RequestTask;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.store.ConsumeQueueExt;\nimport org.apache.rocketmq.store.MessageFilter;\n\nimport static org.apache.rocketmq.broker.longpolling.PollingResult.NOT_POLLING;\nimport static org.apache.rocketmq.broker.longpolling.PollingResult.POLLING_FULL;\nimport static org.apache.rocketmq.broker.longpolling.PollingResult.POLLING_SUC;\nimport static org.apache.rocketmq.broker.longpolling.PollingResult.POLLING_TIMEOUT;\n\npublic class PopLongPollingService extends ServiceThread {\n\n    private static final Logger POP_LOGGER =\n        LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);\n    private final BrokerController brokerController;\n    private final NettyRequestProcessor processor;\n    private final Cache<String, ConcurrentHashMap<String, Byte>> topicCidMap;\n    private final Cache<String, ConcurrentSkipListSet<PopRequest>> pollingMap;\n    private long lastCleanTime = 0;\n\n    private final AtomicLong totalPollingNum = new AtomicLong(0);\n    private final boolean notifyLast;\n\n    public PopLongPollingService(BrokerController brokerController, NettyRequestProcessor processor,\n        boolean notifyLast) {\n        this.brokerController = brokerController;\n        this.processor = processor;\n        // 100000 topic default,  100000 lru topic + cid + qid\n        this.topicCidMap = Caffeine.newBuilder()\n            .maximumSize(this.brokerController.getBrokerConfig().getPopPollingMapSize() * 2L)\n            .expireAfterAccess(this.brokerController.getBrokerConfig().getPopPollingMapExpireTimeSeconds(), TimeUnit.SECONDS)\n            .build();\n\n        this.pollingMap = Caffeine.newBuilder()\n            .maximumSize(this.brokerController.getBrokerConfig().getPopPollingMapSize())\n            .expireAfterAccess(this.brokerController.getBrokerConfig().getPopPollingMapExpireTimeSeconds(), TimeUnit.SECONDS)\n            .build();\n        this.notifyLast = notifyLast;\n    }\n\n    @Override\n    public String getServiceName() {\n        if (brokerController.getBrokerConfig().isInBrokerContainer()) {\n            return brokerController.getBrokerIdentity().getIdentifier() + PopLongPollingService.class.getSimpleName();\n        }\n        return PopLongPollingService.class.getSimpleName();\n    }\n\n    @Override\n    public void run() {\n        int i = 0;\n        while (!this.stopped) {\n            try {\n                this.waitForRunning(20);\n                i++;\n                if (pollingMap.estimatedSize() == 0) {\n                    continue;\n                }\n                long tmpTotalPollingNum = 0;\n                for (Map.Entry<String, ConcurrentSkipListSet<PopRequest>> entry : pollingMap.asMap().entrySet()) {\n                    String key = entry.getKey();\n                    ConcurrentSkipListSet<PopRequest> popQ = entry.getValue();\n                    if (popQ == null) {\n                        continue;\n                    }\n                    PopRequest first;\n                    do {\n                        first = popQ.pollFirst();\n                        if (first == null) {\n                            break;\n                        }\n                        if (!first.isTimeout()) {\n                            if (popQ.add(first)) {\n                                break;\n                            } else {\n                                POP_LOGGER.info(\"polling, add fail again: {}\", first);\n                            }\n                        }\n                        if (brokerController.getBrokerConfig().isEnablePopLog()) {\n                            POP_LOGGER.info(\"timeout , wakeUp polling : {}\", first);\n                        }\n                        totalPollingNum.decrementAndGet();\n                        wakeUp(first);\n                    }\n                    while (true);\n                    if (i >= 100) {\n                        long tmpPollingNum = popQ.size();\n                        tmpTotalPollingNum = tmpTotalPollingNum + tmpPollingNum;\n                        if (tmpPollingNum > 100) {\n                            POP_LOGGER.info(\"polling queue {} , size={} \", key, tmpPollingNum);\n                        }\n                    }\n                }\n\n                if (i >= 100) {\n                    POP_LOGGER.info(\"pollingMapSize={},tmpTotalSize={},atomicTotalSize={},diffSize={}\",\n                        pollingMap.estimatedSize(), tmpTotalPollingNum, totalPollingNum.get(),\n                        Math.abs(totalPollingNum.get() - tmpTotalPollingNum));\n                    totalPollingNum.set(tmpTotalPollingNum);\n                    i = 0;\n                }\n\n                // clean unused\n                if (lastCleanTime == 0 || System.currentTimeMillis() - lastCleanTime > 5 * 60 * 1000) {\n                    cleanUnusedResource();\n                }\n            } catch (Throwable e) {\n                POP_LOGGER.error(\"checkPolling error\", e);\n            }\n        }\n        // clean all;\n        try {\n            for (Map.Entry<String, ConcurrentSkipListSet<PopRequest>> entry : pollingMap.asMap().entrySet()) {\n                ConcurrentSkipListSet<PopRequest> popQ = entry.getValue();\n                PopRequest first;\n                while ((first = popQ.pollFirst()) != null) {\n                    wakeUp(first);\n                }\n            }\n        } catch (Throwable e) {\n        }\n    }\n\n    public void notifyMessageArrivingWithRetryTopic(final String topic, final int queueId) {\n        this.notifyMessageArrivingWithRetryTopic(topic, queueId, -1L, null, 0L, null, null);\n    }\n\n    public void notifyMessageArrivingWithRetryTopic(final String topic, final int queueId, long offset,\n        Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {\n        if (NamespaceUtil.isRetryTopic(topic)) {\n            notifyMessageArrivingFromRetry(topic, queueId, tagsCode, msgStoreTime, filterBitMap, properties);\n        } else {\n            notifyMessageArriving(topic, queueId, offset, tagsCode, msgStoreTime, filterBitMap, properties);\n        }\n    }\n\n    private void notifyMessageArrivingFromRetry(String topic, int queueId, Long tagsCode, long msgStoreTime, byte[] filterBitMap,\n        Map<String, String> properties) {\n        String prefix = MixAll.RETRY_GROUP_TOPIC_PREFIX;\n        String originGroup = properties.get(MessageConst.PROPERTY_ORIGIN_GROUP);\n        // In the case of pop consumption, there is no long polling hanging on the retry topic, so the wake-up is skipped.\n        if (StringUtils.isBlank(originGroup)) {\n            return;\n        }\n        // %RETRY%GROUP is used for pull mode, so the wake-up is skipped.\n        int originTopicStartIndex = prefix.length() + originGroup.length() + 1;\n        if (topic.length() <= originTopicStartIndex) {\n            return;\n        }\n        String originTopic = topic.substring(originTopicStartIndex);\n        if (queueId >= 0) {\n            notifyMessageArriving(originTopic, -1, originGroup, true, tagsCode, msgStoreTime, filterBitMap, properties);\n        }\n        notifyMessageArriving(originTopic, queueId, originGroup, true, tagsCode, msgStoreTime, filterBitMap, properties);\n    }\n\n    public void notifyMessageArriving(final String topic, final int queueId, long offset,\n        Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {\n        ConcurrentHashMap<String, Byte> cids = topicCidMap.getIfPresent(topic);\n        if (cids == null) {\n            return;\n        }\n        long interval = brokerController.getBrokerConfig().getPopLongPollingForceNotifyInterval();\n        boolean force = interval > 0L && offset % interval == 0L;\n        for (Map.Entry<String, Byte> cid : cids.entrySet()) {\n            if (queueId >= 0) {\n                notifyMessageArriving(topic, -1, cid.getKey(), force, tagsCode, msgStoreTime, filterBitMap, properties);\n            }\n            notifyMessageArriving(topic, queueId, cid.getKey(), force, tagsCode, msgStoreTime, filterBitMap, properties);\n        }\n    }\n\n    public boolean notifyMessageArriving(final String topic, final int queueId, final String cid,\n        Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {\n        return notifyMessageArriving(topic, queueId, cid, false, tagsCode, msgStoreTime, filterBitMap, properties, null);\n    }\n\n    public boolean notifyMessageArriving(final String topic, final int queueId, final String cid, boolean force,\n        Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {\n        return notifyMessageArriving(topic, queueId, cid, force, tagsCode, msgStoreTime, filterBitMap, properties, null);\n    }\n\n    public boolean notifyMessageArriving(final String topic, final int queueId, final String cid, boolean force,\n        Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map<String, String> properties, CommandCallback callback) {\n        ConcurrentSkipListSet<PopRequest> remotingCommands = pollingMap.getIfPresent(KeyBuilder.buildPollingKey(topic, cid, queueId));\n        if (remotingCommands == null || remotingCommands.isEmpty()) {\n            return false;\n        }\n\n        PopRequest popRequest = pollRemotingCommands(remotingCommands);\n        if (popRequest == null) {\n            return false;\n        }\n\n        if (!force && popRequest.getMessageFilter() != null && popRequest.getSubscriptionData() != null) {\n            boolean match = popRequest.getMessageFilter().isMatchedByConsumeQueue(tagsCode,\n                new ConsumeQueueExt.CqExtUnit(tagsCode, msgStoreTime, filterBitMap));\n            if (match && properties != null) {\n                match = popRequest.getMessageFilter().isMatchedByCommitLog(null, properties);\n            }\n            if (!match) {\n                remotingCommands.add(popRequest);\n                totalPollingNum.incrementAndGet();\n                return false;\n            }\n        }\n\n        if (brokerController.getBrokerConfig().isEnablePopLog()) {\n            POP_LOGGER.info(\"lock release, new msg arrive, wakeUp: {}\", popRequest);\n        }\n\n        return wakeUp(popRequest, callback);\n    }\n\n    public boolean wakeUp(final PopRequest request) {\n        return wakeUp(request, null);\n    }\n\n    public boolean wakeUp(final PopRequest request, CommandCallback callback) {\n        if (request == null || !request.complete()) {\n            return false;\n        }\n\n        if (callback != null && request.getRemotingCommand() != null) {\n            if (request.getRemotingCommand().getCallbackList() == null) {\n                request.getRemotingCommand().setCallbackList(new ArrayList<>());\n            }\n            request.getRemotingCommand().getCallbackList().add(callback);\n        }\n\n        if (!request.getCtx().channel().isActive()) {\n            return false;\n        }\n\n        Runnable run = () -> {\n            try {\n                final RemotingCommand response = processor.processRequest(request.getCtx(), request.getRemotingCommand());\n                if (response != null) {\n                    response.setOpaque(request.getRemotingCommand().getOpaque());\n                    response.markResponseType();\n                    NettyRemotingAbstract.writeResponse(request.getChannel(), request.getRemotingCommand(), response, future -> {\n                        if (!future.isSuccess()) {\n                            POP_LOGGER.error(\"ProcessRequestWrapper response to {} failed\", request.getChannel().remoteAddress(), future.cause());\n                            POP_LOGGER.error(request.toString());\n                            POP_LOGGER.error(response.toString());\n                        }\n                    }, brokerController.getBrokerMetricsManager().getRemotingMetricsManager());\n                }\n            } catch (Exception e1) {\n                POP_LOGGER.error(\"ExecuteRequestWhenWakeup run\", e1);\n            }\n        };\n\n        this.brokerController.getPullMessageExecutor().submit(\n            new RequestTask(run, request.getChannel(), request.getRemotingCommand()));\n        return true;\n    }\n\n    /**\n     * @param ctx\n     * @param remotingCommand\n     * @param requestHeader\n     * @return\n     */\n    public PollingResult polling(final ChannelHandlerContext ctx, RemotingCommand remotingCommand,\n        final PollingHeader requestHeader) {\n        return this.polling(ctx, remotingCommand, requestHeader, null, null);\n    }\n\n    public PollingResult polling(final ChannelHandlerContext ctx, RemotingCommand remotingCommand,\n        final PollingHeader requestHeader, SubscriptionData subscriptionData, MessageFilter messageFilter) {\n        if (requestHeader.getPollTime() <= 0 || this.isStopped()) {\n            return NOT_POLLING;\n        }\n        ConcurrentHashMap<String, Byte> cids = topicCidMap.get(requestHeader.getTopic(), key -> new ConcurrentHashMap<>());\n        cids.putIfAbsent(requestHeader.getConsumerGroup(), Byte.MIN_VALUE);\n        long expired = requestHeader.getBornTime() + requestHeader.getPollTime();\n        final PopRequest request = new PopRequest(remotingCommand, ctx, expired, subscriptionData, messageFilter);\n        boolean isFull = totalPollingNum.get() >= this.brokerController.getBrokerConfig().getMaxPopPollingSize();\n        if (isFull) {\n            POP_LOGGER.info(\"polling {}, result POLLING_FULL, total:{}\", remotingCommand, totalPollingNum.get());\n            return POLLING_FULL;\n        }\n        boolean isTimeout = request.isTimeout();\n        if (isTimeout) {\n            if (brokerController.getBrokerConfig().isEnablePopLog()) {\n                POP_LOGGER.info(\"polling {}, result POLLING_TIMEOUT\", remotingCommand);\n            }\n            return POLLING_TIMEOUT;\n        }\n        String key = KeyBuilder.buildPollingKey(requestHeader.getTopic(), requestHeader.getConsumerGroup(),\n            requestHeader.getQueueId());\n        ConcurrentSkipListSet<PopRequest> queue = pollingMap.get(key, k -> new ConcurrentSkipListSet<>(PopRequest.COMPARATOR));\n        int size = queue.size();\n        if (size > brokerController.getBrokerConfig().getPopPollingSize()) {\n            POP_LOGGER.info(\"polling {}, result POLLING_FULL, singleSize:{}\", remotingCommand, size);\n            return POLLING_FULL;\n        }\n\n        if (queue.add(request)) {\n            remotingCommand.setSuspended(true);\n            totalPollingNum.incrementAndGet();\n            if (brokerController.getBrokerConfig().isEnablePopLog()) {\n                POP_LOGGER.info(\"polling {}, result POLLING_SUC\", remotingCommand);\n            }\n            return POLLING_SUC;\n        } else {\n            POP_LOGGER.info(\"polling {}, result POLLING_FULL, add fail, {}\", request, queue);\n            return POLLING_FULL;\n        }\n    }\n\n    public Cache<String, ConcurrentSkipListSet<PopRequest>> getPollingMap() {\n        return pollingMap;\n    }\n\n    public Cache<String, ConcurrentHashMap<String, Byte>> getTopicCidMap() {\n        return topicCidMap;\n    }\n\n    private void cleanUnusedResource() {\n        try {\n            {\n                Iterator<Map.Entry<String, ConcurrentHashMap<String, Byte>>> topicCidMapIter = topicCidMap.asMap().entrySet().iterator();\n                while (topicCidMapIter.hasNext()) {\n                    Map.Entry<String, ConcurrentHashMap<String, Byte>> entry = topicCidMapIter.next();\n                    String topic = entry.getKey();\n                    if (brokerController.getTopicConfigManager().selectTopicConfig(topic) == null) {\n                        POP_LOGGER.info(\"remove nonexistent topic {} in topicCidMap!\", topic);\n                        topicCidMapIter.remove();\n                        continue;\n                    }\n                    Iterator<Map.Entry<String, Byte>> cidMapIter = entry.getValue().entrySet().iterator();\n                    while (cidMapIter.hasNext()) {\n                        Map.Entry<String, Byte> cidEntry = cidMapIter.next();\n                        String cid = cidEntry.getKey();\n                        if (!brokerController.getSubscriptionGroupManager().containsSubscriptionGroup(cid)) {\n                            POP_LOGGER.info(\"remove nonexistent subscription group {} of topic {} in topicCidMap!\", cid, topic);\n                            cidMapIter.remove();\n                        }\n                    }\n                }\n            }\n\n            {\n                Iterator<Map.Entry<String, ConcurrentSkipListSet<PopRequest>>> pollingMapIter = pollingMap.asMap().entrySet().iterator();\n                while (pollingMapIter.hasNext()) {\n                    Map.Entry<String, ConcurrentSkipListSet<PopRequest>> entry = pollingMapIter.next();\n                    if (entry.getKey() == null) {\n                        continue;\n                    }\n                    String[] keyArray = entry.getKey().split(PopAckConstants.SPLIT);\n                    if (keyArray.length != 3) {\n                        continue;\n                    }\n                    String topic = keyArray[0];\n                    String cid = keyArray[1];\n                    if (brokerController.getTopicConfigManager().selectTopicConfig(topic) == null) {\n                        POP_LOGGER.info(\"remove nonexistent topic {} in pollingMap!\", topic);\n                        pollingMapIter.remove();\n                        continue;\n                    }\n                    if (!brokerController.getSubscriptionGroupManager().containsSubscriptionGroup(cid)) {\n                        POP_LOGGER.info(\"remove nonexistent subscription group {} of topic {} in pollingMap!\", cid, topic);\n                        pollingMapIter.remove();\n                    }\n                }\n            }\n        } catch (Throwable e) {\n            POP_LOGGER.error(\"cleanUnusedResource\", e);\n        }\n\n        lastCleanTime = System.currentTimeMillis();\n    }\n\n    private PopRequest pollRemotingCommands(ConcurrentSkipListSet<PopRequest> remotingCommands) {\n        if (remotingCommands == null || remotingCommands.isEmpty()) {\n            return null;\n        }\n\n        PopRequest popRequest;\n        do {\n            if (notifyLast) {\n                popRequest = remotingCommands.pollLast();\n            } else {\n                popRequest = remotingCommands.pollFirst();\n            }\n            totalPollingNum.decrementAndGet();\n        } while (popRequest != null && !popRequest.getChannel().isActive());\n\n        return popRequest;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopRequest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.longpolling;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.Comparator;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.store.MessageFilter;\n\npublic class PopRequest {\n    private static final AtomicLong COUNTER = new AtomicLong(Long.MIN_VALUE);\n\n    private final RemotingCommand remotingCommand;\n    private final ChannelHandlerContext ctx;\n    private final AtomicBoolean complete = new AtomicBoolean(false);\n    private final long op = COUNTER.getAndIncrement();\n\n    private final long expired;\n    private final SubscriptionData subscriptionData;\n    private final MessageFilter messageFilter;\n\n    public PopRequest(RemotingCommand remotingCommand, ChannelHandlerContext ctx,\n        long expired, SubscriptionData subscriptionData, MessageFilter messageFilter) {\n\n        this.ctx = ctx;\n        this.remotingCommand = remotingCommand;\n        this.expired = expired;\n        this.subscriptionData = subscriptionData;\n        this.messageFilter = messageFilter;\n    }\n\n    public Channel getChannel() {\n        return ctx.channel();\n    }\n\n    public ChannelHandlerContext getCtx() {\n        return ctx;\n    }\n\n    public RemotingCommand getRemotingCommand() {\n        return remotingCommand;\n    }\n\n    public boolean isTimeout() {\n        return System.currentTimeMillis() > (expired - 50);\n    }\n\n    public boolean complete() {\n        return complete.compareAndSet(false, true);\n    }\n\n    public long getExpired() {\n        return expired;\n    }\n\n    public SubscriptionData getSubscriptionData() {\n        return subscriptionData;\n    }\n\n    public MessageFilter getMessageFilter() {\n        return messageFilter;\n    }\n\n    @Override\n    public String toString() {\n        final StringBuilder sb = new StringBuilder(\"PopRequest{\");\n        sb.append(\"cmd=\").append(remotingCommand);\n        sb.append(\", ctx=\").append(ctx);\n        sb.append(\", expired=\").append(expired);\n        sb.append(\", complete=\").append(complete);\n        sb.append(\", op=\").append(op);\n        sb.append('}');\n        return sb.toString();\n    }\n\n    public static final Comparator<PopRequest> COMPARATOR = (o1, o2) -> {\n        int ret = (int) (o1.getExpired() - o2.getExpired());\n\n        if (ret != 0) {\n            return ret;\n        }\n        ret = (int) (o1.op - o2.op);\n        if (ret != 0) {\n            return ret;\n        }\n        return -1;\n    };\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.longpolling;\n\nimport io.netty.channel.Channel;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.store.MessageFilter;\n\npublic class PullRequest {\n    private final RemotingCommand requestCommand;\n    private final Channel clientChannel;\n    private final long timeoutMillis;\n    private final long suspendTimestamp;\n    private final long pullFromThisOffset;\n    private final SubscriptionData subscriptionData;\n    private final MessageFilter messageFilter;\n\n    public PullRequest(RemotingCommand requestCommand, Channel clientChannel, long timeoutMillis, long suspendTimestamp,\n        long pullFromThisOffset, SubscriptionData subscriptionData,\n        MessageFilter messageFilter) {\n        this.requestCommand = requestCommand;\n        this.clientChannel = clientChannel;\n        this.timeoutMillis = timeoutMillis;\n        this.suspendTimestamp = suspendTimestamp;\n        this.pullFromThisOffset = pullFromThisOffset;\n        this.subscriptionData = subscriptionData;\n        this.messageFilter = messageFilter;\n    }\n\n    public RemotingCommand getRequestCommand() {\n        return requestCommand;\n    }\n\n    public Channel getClientChannel() {\n        return clientChannel;\n    }\n\n    public long getTimeoutMillis() {\n        return timeoutMillis;\n    }\n\n    public long getSuspendTimestamp() {\n        return suspendTimestamp;\n    }\n\n    public long getPullFromThisOffset() {\n        return pullFromThisOffset;\n    }\n\n    public SubscriptionData getSubscriptionData() {\n        return subscriptionData;\n    }\n\n    public MessageFilter getMessageFilter() {\n        return messageFilter;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.longpolling;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.SystemClock;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.ConsumeQueueExt;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\n\npublic class PullRequestHoldService extends ServiceThread {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    protected static final String TOPIC_QUEUEID_SEPARATOR = \"@\";\n    protected final BrokerController brokerController;\n    private final SystemClock systemClock = new SystemClock();\n    protected ConcurrentMap<String/* topic@queueId */, ManyPullRequest> pullRequestTable =\n        new ConcurrentHashMap<>(1024);\n\n    public PullRequestHoldService(final BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    public void suspendPullRequest(final String topic, final int queueId, final PullRequest pullRequest) {\n        String key = this.buildKey(topic, queueId);\n        ManyPullRequest mpr = this.pullRequestTable.get(key);\n        if (null == mpr) {\n            mpr = new ManyPullRequest();\n            ManyPullRequest prev = this.pullRequestTable.putIfAbsent(key, mpr);\n            if (prev != null) {\n                mpr = prev;\n            }\n        }\n\n        pullRequest.getRequestCommand().setSuspended(true);\n        mpr.addPullRequest(pullRequest);\n    }\n\n    private String buildKey(final String topic, final int queueId) {\n        StringBuilder sb = new StringBuilder(topic.length() + 5);\n        sb.append(topic);\n        sb.append(TOPIC_QUEUEID_SEPARATOR);\n        sb.append(queueId);\n        return sb.toString();\n    }\n\n    @Override\n    public void run() {\n        log.info(\"{} service started\", this.getServiceName());\n        while (!this.isStopped()) {\n            try {\n                if (this.brokerController.getBrokerConfig().isLongPollingEnable()) {\n                    this.waitForRunning(5 * 1000);\n                } else {\n                    this.waitForRunning(this.brokerController.getBrokerConfig().getShortPollingTimeMills());\n                }\n\n                long beginLockTimestamp = this.systemClock.now();\n                this.checkHoldRequest();\n                long costTime = this.systemClock.now() - beginLockTimestamp;\n                if (costTime > 5 * 1000) {\n                    log.warn(\"PullRequestHoldService: check hold pull request cost {}ms\", costTime);\n                }\n            } catch (Throwable e) {\n                log.warn(this.getServiceName() + \" service has exception. \", e);\n            }\n        }\n\n        log.info(\"{} service end\", this.getServiceName());\n    }\n\n    @Override\n    public String getServiceName() {\n        if (brokerController != null && brokerController.getBrokerConfig().isInBrokerContainer()) {\n            return this.brokerController.getBrokerIdentity().getIdentifier() + PullRequestHoldService.class.getSimpleName();\n        }\n        return PullRequestHoldService.class.getSimpleName();\n    }\n\n    protected void checkHoldRequest() {\n        for (String key : this.pullRequestTable.keySet()) {\n            String[] kArray = key.split(TOPIC_QUEUEID_SEPARATOR);\n            if (2 == kArray.length) {\n                String topic = kArray[0];\n                int queueId = Integer.parseInt(kArray[1]);\n                try {\n                    final long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);\n                    this.notifyMessageArriving(topic, queueId, offset);\n                } catch (Throwable e) {\n                    log.error(\n                        \"PullRequestHoldService: failed to check hold request failed, topic={}, queueId={}\", topic,\n                        queueId, e);\n                }\n            }\n        }\n    }\n\n    public void notifyMessageArriving(final String topic, final int queueId, final long maxOffset) {\n        notifyMessageArriving(topic, queueId, maxOffset, null, 0, null, null);\n    }\n\n    public void notifyMessageArriving(final String topic, final int queueId, final long maxOffset, final Long tagsCode,\n        long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {\n        String key = this.buildKey(topic, queueId);\n        ManyPullRequest mpr = this.pullRequestTable.get(key);\n        if (mpr != null) {\n            List<PullRequest> requestList = mpr.cloneListAndClear();\n            if (requestList != null) {\n                List<PullRequest> replayList = new ArrayList<>();\n\n                for (PullRequest request : requestList) {\n                    long newestOffset = maxOffset;\n                    if (newestOffset <= request.getPullFromThisOffset()) {\n                        try {\n                            newestOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);\n                        } catch (ConsumeQueueException e) {\n                            log.error(\"Failed tp get max offset in queue\", e);\n                            continue;\n                        }\n                    }\n\n                    if (newestOffset > request.getPullFromThisOffset()) {\n                        boolean match = request.getMessageFilter().isMatchedByConsumeQueue(tagsCode,\n                            new ConsumeQueueExt.CqExtUnit(tagsCode, msgStoreTime, filterBitMap));\n                        // match by bit map, need eval again when properties is not null.\n                        if (match && properties != null) {\n                            match = request.getMessageFilter().isMatchedByCommitLog(null, properties);\n                        }\n\n                        if (match) {\n                            try {\n                                this.brokerController.getPullMessageProcessor().executeRequestWhenWakeup(request.getClientChannel(),\n                                    request.getRequestCommand());\n                            } catch (Throwable e) {\n                                log.error(\n                                    \"PullRequestHoldService#notifyMessageArriving: failed to execute request when \"\n                                        + \"message matched, topic={}, queueId={}\", topic, queueId, e);\n                            }\n                            continue;\n                        }\n                    }\n\n                    if (System.currentTimeMillis() >= (request.getSuspendTimestamp() + request.getTimeoutMillis())) {\n                        try {\n                            this.brokerController.getPullMessageProcessor().executeRequestWhenWakeup(request.getClientChannel(),\n                                request.getRequestCommand());\n                        } catch (Throwable e) {\n                            log.error(\n                                \"PullRequestHoldService#notifyMessageArriving: failed to execute request when time's \"\n                                    + \"up, topic={}, queueId={}\", topic, queueId, e);\n                        }\n                        continue;\n                    }\n\n                    replayList.add(request);\n                }\n\n                if (!replayList.isEmpty()) {\n                    mpr.addPullRequest(replayList);\n                }\n            }\n        }\n    }\n\n    public void notifyMasterOnline() {\n        for (ManyPullRequest mpr : this.pullRequestTable.values()) {\n            if (mpr == null || mpr.isEmpty()) {\n                continue;\n            }\n            for (PullRequest request : mpr.cloneListAndClear()) {\n                try {\n                    log.info(\"notify master online, wakeup {} {}\", request.getClientChannel(), request.getRequestCommand());\n                    this.brokerController.getPullMessageProcessor().executeRequestWhenWakeup(request.getClientChannel(),\n                        request.getRequestCommand());\n                } catch (Throwable e) {\n                    log.error(\"execute request when master online failed.\", e);\n                }\n            }\n        }\n\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsConstant.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.metrics;\n\npublic class BrokerMetricsConstant {\n    public static final String OPEN_TELEMETRY_METER_NAME = \"broker-meter\";\n\n    public static final String GAUGE_PROCESSOR_WATERMARK = \"rocketmq_processor_watermark\";\n    public static final String GAUGE_BROKER_PERMISSION = \"rocketmq_broker_permission\";\n    public static final String GAUGE_TOPIC_NUM = \"rocketmq_topic_number\";\n    public static final String GAUGE_CONSUMER_GROUP_NUM = \"rocketmq_consumer_group_number\";\n\n    public static final String COUNTER_MESSAGES_IN_TOTAL = \"rocketmq_messages_in_total\";\n    public static final String COUNTER_MESSAGES_OUT_TOTAL = \"rocketmq_messages_out_total\";\n    public static final String COUNTER_THROUGHPUT_IN_TOTAL = \"rocketmq_throughput_in_total\";\n    public static final String COUNTER_THROUGHPUT_OUT_TOTAL = \"rocketmq_throughput_out_total\";\n    public static final String HISTOGRAM_MESSAGE_SIZE = \"rocketmq_message_size\";\n    public static final String HISTOGRAM_TOPIC_CREATE_EXECUTE_TIME = \"rocketmq_topic_create_execution_time\";\n    public static final String HISTOGRAM_CONSUMER_GROUP_CREATE_EXECUTE_TIME = \"rocketmq_consumer_group_create_execution_time\";\n\n    public static final String GAUGE_PRODUCER_CONNECTIONS = \"rocketmq_producer_connections\";\n    public static final String GAUGE_CONSUMER_CONNECTIONS = \"rocketmq_consumer_connections\";\n\n    public static final String GAUGE_CONSUMER_LAG_MESSAGES = \"rocketmq_consumer_lag_messages\";\n    public static final String GAUGE_CONSUMER_LAG_LATENCY = \"rocketmq_consumer_lag_latency\";\n    public static final String GAUGE_CONSUMER_INFLIGHT_MESSAGES = \"rocketmq_consumer_inflight_messages\";\n    public static final String GAUGE_CONSUMER_QUEUEING_LATENCY = \"rocketmq_consumer_queueing_latency\";\n    public static final String GAUGE_CONSUMER_READY_MESSAGES = \"rocketmq_consumer_ready_messages\";\n    public static final String COUNTER_CONSUMER_SEND_TO_DLQ_MESSAGES_TOTAL = \"rocketmq_send_to_dlq_messages_total\";\n\n    public static final String COUNTER_COMMIT_MESSAGES_TOTAL = \"rocketmq_commit_messages_total\";\n    public static final String COUNTER_ROLLBACK_MESSAGES_TOTAL = \"rocketmq_rollback_messages_total\";\n    public static final String HISTOGRAM_FINISH_MSG_LATENCY = \"rocketmq_finish_message_latency\";\n    public static final String GAUGE_HALF_MESSAGES = \"rocketmq_half_messages\";\n\n    public static final String LABEL_CLUSTER_NAME = \"cluster\";\n    public static final String LABEL_NODE_TYPE = \"node_type\";\n    public static final String NODE_TYPE_BROKER = \"broker\";\n    public static final String LABEL_NODE_ID = \"node_id\";\n    public static final String LABEL_AGGREGATION = \"aggregation\";\n    public static final String AGGREGATION_DELTA = \"delta\";\n    public static final String LABEL_PROCESSOR = \"processor\";\n\n    public static final String LABEL_TOPIC = \"topic\";\n    public static final String LABEL_INVOCATION_STATUS = \"invocation_status\";\n    public static final String LABEL_IS_RETRY = \"is_retry\";\n    public static final String LABEL_IS_SYSTEM = \"is_system\";\n    public static final String LABEL_CONSUMER_GROUP = \"consumer_group\";\n    public static final String LABEL_MESSAGE_TYPE = \"message_type\";\n    public static final String LABEL_LANGUAGE = \"language\";\n    public static final String LABEL_VERSION = \"version\";\n    public static final String LABEL_CONSUME_MODE = \"consume_mode\";\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.metrics;\n\nimport com.google.common.base.Splitter;\nimport io.opentelemetry.api.common.Attributes;\nimport io.opentelemetry.api.common.AttributesBuilder;\nimport io.opentelemetry.api.metrics.LongCounter;\nimport io.opentelemetry.api.metrics.LongHistogram;\nimport io.opentelemetry.api.metrics.Meter;\nimport io.opentelemetry.api.metrics.ObservableLongGauge;\nimport io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter;\nimport io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter;\nimport io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporterBuilder;\nimport io.opentelemetry.exporter.prometheus.PrometheusHttpServer;\nimport io.opentelemetry.sdk.OpenTelemetrySdk;\nimport io.opentelemetry.sdk.metrics.Aggregation;\nimport io.opentelemetry.sdk.metrics.InstrumentSelector;\nimport io.opentelemetry.sdk.metrics.InstrumentType;\nimport io.opentelemetry.sdk.metrics.SdkMeterProvider;\nimport io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder;\nimport io.opentelemetry.sdk.metrics.View;\nimport io.opentelemetry.sdk.metrics.ViewBuilder;\nimport io.opentelemetry.sdk.metrics.data.AggregationTemporality;\nimport io.opentelemetry.sdk.metrics.export.MetricExporter;\nimport io.opentelemetry.sdk.metrics.export.PeriodicMetricReader;\nimport io.opentelemetry.sdk.metrics.internal.SdkMeterProviderUtil;\nimport io.opentelemetry.sdk.resources.Resource;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.client.ConsumerManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.metrics.MetricsExporterType;\nimport org.apache.rocketmq.common.metrics.NopLongCounter;\nimport org.apache.rocketmq.common.metrics.NopLongHistogram;\nimport org.apache.rocketmq.common.metrics.NopObservableLongGauge;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.metrics.RemotingMetricsManager;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant;\nimport org.slf4j.bridge.SLF4JBridgeHandler;\n\nimport java.time.Duration;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.TimeUnit;\nimport java.util.function.Supplier;\n\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.AGGREGATION_DELTA;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.COUNTER_COMMIT_MESSAGES_TOTAL;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.COUNTER_CONSUMER_SEND_TO_DLQ_MESSAGES_TOTAL;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.COUNTER_MESSAGES_IN_TOTAL;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.COUNTER_MESSAGES_OUT_TOTAL;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.COUNTER_ROLLBACK_MESSAGES_TOTAL;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.COUNTER_THROUGHPUT_IN_TOTAL;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.COUNTER_THROUGHPUT_OUT_TOTAL;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_TOPIC_NUM;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_CONSUMER_GROUP_NUM;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_BROKER_PERMISSION;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_CONSUMER_CONNECTIONS;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_CONSUMER_INFLIGHT_MESSAGES;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_CONSUMER_LAG_LATENCY;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_CONSUMER_LAG_MESSAGES;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_CONSUMER_QUEUEING_LATENCY;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_CONSUMER_READY_MESSAGES;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_HALF_MESSAGES;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_PROCESSOR_WATERMARK;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_PRODUCER_CONNECTIONS;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.HISTOGRAM_FINISH_MSG_LATENCY;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.HISTOGRAM_MESSAGE_SIZE;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.HISTOGRAM_TOPIC_CREATE_EXECUTE_TIME;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.HISTOGRAM_CONSUMER_GROUP_CREATE_EXECUTE_TIME;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_AGGREGATION;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CLUSTER_NAME;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUME_MODE;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_RETRY;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_LANGUAGE;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_NODE_ID;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_NODE_TYPE;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_PROCESSOR;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_TOPIC;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_VERSION;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.NODE_TYPE_BROKER;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.OPEN_TELEMETRY_METER_NAME;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_PROTOCOL_TYPE;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.PROTOCOL_TYPE_REMOTING;\n\npublic class BrokerMetricsManager {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n\n    private final BrokerConfig brokerConfig;\n    private final MessageStore messageStore;\n    private final BrokerController brokerController;\n    private final ConsumerLagCalculator consumerLagCalculator;\n    private final LiteConsumerLagCalculator liteConsumerLagCalculator;\n    private final Map<String, String> labelMap = new HashMap<>();\n    private OtlpGrpcMetricExporter metricExporter;\n    private PeriodicMetricReader periodicMetricReader;\n    private PrometheusHttpServer prometheusHttpServer;\n    private MetricExporter loggingMetricExporter;\n    private Meter brokerMeter;\n\n    private Supplier<AttributesBuilder> attributesBuilderSupplier = Attributes::builder;\n\n    // broker stats metrics\n    private ObservableLongGauge processorWatermark = new NopObservableLongGauge();\n    private ObservableLongGauge brokerPermission = new NopObservableLongGauge();\n    private ObservableLongGauge topicNum = new NopObservableLongGauge();\n    private ObservableLongGauge consumerGroupNum = new NopObservableLongGauge();\n\n    // request metrics\n    private LongCounter messagesInTotal = new NopLongCounter();\n    private LongCounter messagesOutTotal = new NopLongCounter();\n    private LongCounter throughputInTotal = new NopLongCounter();\n    private LongCounter throughputOutTotal = new NopLongCounter();\n    private LongHistogram messageSize = new NopLongHistogram();\n    private LongHistogram topicCreateExecuteTime = new NopLongHistogram();\n    private LongHistogram consumerGroupCreateExecuteTime = new NopLongHistogram();\n\n    // client connection metrics\n    private ObservableLongGauge producerConnection = new NopObservableLongGauge();\n    private ObservableLongGauge consumerConnection = new NopObservableLongGauge();\n\n    // Lag metrics\n    private ObservableLongGauge consumerLagMessages = new NopObservableLongGauge();\n    private ObservableLongGauge consumerLagLatency = new NopObservableLongGauge();\n    private ObservableLongGauge consumerInflightMessages = new NopObservableLongGauge();\n    private ObservableLongGauge consumerQueueingLatency = new NopObservableLongGauge();\n    private ObservableLongGauge consumerReadyMessages = new NopObservableLongGauge();\n    private LongCounter sendToDlqMessages = new NopLongCounter();\n    private ObservableLongGauge halfMessages = new NopObservableLongGauge();\n    private LongCounter commitMessagesTotal = new NopLongCounter();\n    private LongCounter rollBackMessagesTotal = new NopLongCounter();\n    private LongHistogram transactionFinishLatency = new NopLongHistogram();\n\n    private final RemotingMetricsManager remotingMetricsManager;\n    private final PopMetricsManager popMetricsManager;\n\n    @SuppressWarnings(\"DoubleBraceInitialization\")\n    public static final List<String> SYSTEM_GROUP_PREFIX_LIST = new ArrayList<String>() {\n        {\n            add(MixAll.CID_RMQ_SYS_PREFIX.toLowerCase());\n        }\n    };\n\n    public BrokerMetricsManager(BrokerController brokerController) {\n        this.brokerController = brokerController;\n        brokerConfig = brokerController.getBrokerConfig();\n        this.messageStore = brokerController.getMessageStore();\n        this.consumerLagCalculator = new ConsumerLagCalculator(brokerController);\n        this.remotingMetricsManager = new RemotingMetricsManager();\n        this.popMetricsManager = new PopMetricsManager();\n        this.liteConsumerLagCalculator = new LiteConsumerLagCalculator(brokerController);\n        init();\n    }\n\n    public AttributesBuilder newAttributesBuilder() {\n        AttributesBuilder attributesBuilder;\n        if (attributesBuilderSupplier == null) {\n            attributesBuilderSupplier = Attributes::builder;\n        }\n        attributesBuilder = attributesBuilderSupplier.get();\n        labelMap.forEach(attributesBuilder::put);\n        return attributesBuilder;\n    }\n\n    private Attributes buildLagAttributes(ConsumerLagCalculator.BaseCalculateResult result) {\n        AttributesBuilder attributesBuilder = newAttributesBuilder();\n        attributesBuilder.put(LABEL_CONSUMER_GROUP, result.group);\n        attributesBuilder.put(LABEL_TOPIC, result.topic);\n        attributesBuilder.put(LABEL_IS_RETRY, result.isRetry);\n        attributesBuilder.put(LABEL_IS_SYSTEM, isSystem(result.topic, result.group));\n        return attributesBuilder.build();\n    }\n\n    public static boolean isRetryOrDlqTopic(String topic) {\n        if (StringUtils.isBlank(topic)) {\n            return false;\n        }\n        return topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) || topic.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX);\n    }\n\n    public static boolean isSystemGroup(String group) {\n        if (StringUtils.isBlank(group)) {\n            return false;\n        }\n        String groupInLowerCase = group.toLowerCase();\n        for (String prefix : SYSTEM_GROUP_PREFIX_LIST) {\n            if (groupInLowerCase.startsWith(prefix)) {\n                return true;\n            }\n        }\n        return false;\n    }\n\n    public static boolean isSystem(String topic, String group) {\n        return TopicValidator.isSystemTopic(topic) || isSystemGroup(group);\n    }\n\n    public static TopicMessageType getMessageType(SendMessageRequestHeader requestHeader) {\n        Map<String, String> properties = MessageDecoder.string2messageProperties(requestHeader.getProperties());\n        String traFlag = properties.get(MessageConst.PROPERTY_TRANSACTION_PREPARED);\n        TopicMessageType topicMessageType = TopicMessageType.NORMAL;\n        if (Boolean.parseBoolean(traFlag)) {\n            topicMessageType = TopicMessageType.TRANSACTION;\n        } else if (properties.containsKey(MessageConst.PROPERTY_SHARDING_KEY)) {\n            topicMessageType = TopicMessageType.FIFO;\n        } else if (properties.get(\"__STARTDELIVERTIME\") != null\n            || properties.get(MessageConst.PROPERTY_DELAY_TIME_LEVEL) != null\n            || properties.get(MessageConst.PROPERTY_TIMER_DELIVER_MS) != null\n            || properties.get(MessageConst.PROPERTY_TIMER_DELAY_SEC) != null\n            || properties.get(MessageConst.PROPERTY_TIMER_DELAY_MS) != null) {\n            topicMessageType = TopicMessageType.DELAY;\n        }\n        return topicMessageType;\n    }\n\n    public Meter getBrokerMeter() {\n        return brokerMeter;\n    }\n\n    // Getter methods for metrics variables\n    public LongCounter getMessagesInTotal() {\n        return messagesInTotal;\n    }\n\n    public LongCounter getMessagesOutTotal() {\n        return messagesOutTotal;\n    }\n\n    public LongCounter getThroughputInTotal() {\n        return throughputInTotal;\n    }\n\n    public LongCounter getThroughputOutTotal() {\n        return throughputOutTotal;\n    }\n\n    public LongHistogram getMessageSize() {\n        return messageSize;\n    }\n\n    public LongCounter getSendToDlqMessages() {\n        return sendToDlqMessages;\n    }\n\n    public LongCounter getCommitMessagesTotal() {\n        return commitMessagesTotal;\n    }\n\n    public LongCounter getRollBackMessagesTotal() {\n        return rollBackMessagesTotal;\n    }\n\n    public LongHistogram getTransactionFinishLatency() {\n        return transactionFinishLatency;\n    }\n\n    public LongHistogram getTopicCreateExecuteTime() {\n        return topicCreateExecuteTime;\n    }\n\n    public LongHistogram getConsumerGroupCreateExecuteTime() {\n        return consumerGroupCreateExecuteTime;\n    }\n\n    // Setter method for testing purposes\n    public void setAttributesBuilderSupplier(Supplier<AttributesBuilder> attributesBuilderSupplier) {\n        this.attributesBuilderSupplier = attributesBuilderSupplier;\n    }\n\n    private boolean checkConfig() {\n        if (brokerConfig == null) {\n            return false;\n        }\n        MetricsExporterType exporterType = brokerConfig.getMetricsExporterType();\n        if (!exporterType.isEnable()) {\n            return false;\n        }\n\n        switch (exporterType) {\n            case OTLP_GRPC:\n                return StringUtils.isNotBlank(brokerConfig.getMetricsGrpcExporterTarget());\n            case PROM:\n                return true;\n            case LOG:\n                return true;\n        }\n        return false;\n    }\n\n    private void init() {\n        MetricsExporterType metricsExporterType = brokerConfig.getMetricsExporterType();\n        if (metricsExporterType == MetricsExporterType.DISABLE) {\n            return;\n        }\n\n        if (!checkConfig()) {\n            LOGGER.error(\"check metrics config failed, will not export metrics\");\n            return;\n        }\n\n        String labels = brokerConfig.getMetricsLabel();\n        if (StringUtils.isNotBlank(labels)) {\n            List<String> kvPairs = Splitter.on(',').omitEmptyStrings().splitToList(labels);\n            for (String item : kvPairs) {\n                String[] split = item.split(\":\");\n                if (split.length != 2) {\n                    LOGGER.warn(\"metricsLabel is not valid: {}\", labels);\n                    continue;\n                }\n                labelMap.put(split[0], split[1]);\n            }\n        }\n        if (brokerConfig.isMetricsInDelta()) {\n            labelMap.put(LABEL_AGGREGATION, AGGREGATION_DELTA);\n        }\n        labelMap.put(LABEL_NODE_TYPE, NODE_TYPE_BROKER);\n        labelMap.put(LABEL_CLUSTER_NAME, brokerConfig.getBrokerClusterName());\n        labelMap.put(LABEL_NODE_ID, brokerConfig.getBrokerName());\n\n        SdkMeterProviderBuilder providerBuilder = SdkMeterProvider.builder()\n            .setResource(Resource.empty());\n\n        if (metricsExporterType == MetricsExporterType.OTLP_GRPC) {\n            String endpoint = brokerConfig.getMetricsGrpcExporterTarget();\n            if (!endpoint.startsWith(\"http\")) {\n                endpoint = \"https://\" + endpoint;\n            }\n            OtlpGrpcMetricExporterBuilder metricExporterBuilder = OtlpGrpcMetricExporter.builder()\n                .setEndpoint(endpoint)\n                .setTimeout(brokerConfig.getMetricGrpcExporterTimeOutInMills(), TimeUnit.MILLISECONDS)\n                .setAggregationTemporalitySelector(type -> {\n                    if (brokerConfig.isMetricsInDelta() &&\n                        (type == InstrumentType.COUNTER || type == InstrumentType.OBSERVABLE_COUNTER || type == InstrumentType.HISTOGRAM)) {\n                        return AggregationTemporality.DELTA;\n                    }\n                    return AggregationTemporality.CUMULATIVE;\n                });\n\n            String headers = brokerConfig.getMetricsGrpcExporterHeader();\n            if (StringUtils.isNotBlank(headers)) {\n                Map<String, String> headerMap = new HashMap<>();\n                List<String> kvPairs = Splitter.on(',').omitEmptyStrings().splitToList(headers);\n                for (String item : kvPairs) {\n                    String[] split = item.split(\":\");\n                    if (split.length != 2) {\n                        LOGGER.warn(\"metricsGrpcExporterHeader is not valid: {}\", headers);\n                        continue;\n                    }\n                    headerMap.put(split[0], split[1]);\n                }\n                headerMap.forEach(metricExporterBuilder::addHeader);\n            }\n\n            metricExporter = metricExporterBuilder.build();\n\n            periodicMetricReader = PeriodicMetricReader.builder(metricExporter)\n                .setInterval(brokerConfig.getMetricGrpcExporterIntervalInMills(), TimeUnit.MILLISECONDS)\n                .build();\n\n            providerBuilder.registerMetricReader(periodicMetricReader);\n        }\n\n        if (metricsExporterType == MetricsExporterType.PROM) {\n            String promExporterHost = brokerConfig.getMetricsPromExporterHost();\n            if (StringUtils.isBlank(promExporterHost)) {\n                promExporterHost = brokerConfig.getBrokerIP1();\n            }\n            prometheusHttpServer = PrometheusHttpServer.builder()\n                .setHost(promExporterHost)\n                .setPort(brokerConfig.getMetricsPromExporterPort())\n                .build();\n            providerBuilder.registerMetricReader(prometheusHttpServer);\n        }\n\n        if (metricsExporterType == MetricsExporterType.LOG) {\n            SLF4JBridgeHandler.removeHandlersForRootLogger();\n            SLF4JBridgeHandler.install();\n            loggingMetricExporter = OtlpJsonLoggingMetricExporter.create(brokerConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE);\n            java.util.logging.Logger.getLogger(OtlpJsonLoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST);\n            periodicMetricReader = PeriodicMetricReader.builder(loggingMetricExporter)\n                .setInterval(brokerConfig.getMetricLoggingExporterIntervalInMills(), TimeUnit.MILLISECONDS)\n                .build();\n            providerBuilder.registerMetricReader(periodicMetricReader);\n        }\n\n        registerMetricsView(providerBuilder);\n\n        brokerMeter = OpenTelemetrySdk.builder()\n            .setMeterProvider(providerBuilder.build())\n            .build()\n            .getMeter(OPEN_TELEMETRY_METER_NAME);\n\n        initStatsMetrics();\n        initRequestMetrics();\n        initConnectionMetrics();\n        initLagAndDlqMetrics();\n        initTransactionMetrics();\n        initOtherMetrics();\n    }\n\n    private void registerMetricsView(SdkMeterProviderBuilder providerBuilder) {\n        // message size buckets, 1k, 4k, 512k, 1M, 2M, 4M\n        List<Double> messageSizeBuckets = Arrays.asList(\n            1d * 1024, //1KB\n            4d * 1024, //4KB\n            512d * 1024, //512KB\n            1d * 1024 * 1024, //1MB\n            2d * 1024 * 1024, //2MB\n            4d * 1024 * 1024 //4MB\n        );\n\n        List<Double> commitLatencyBuckets = Arrays.asList(\n            1d * 1 * 1 * 5, //5s\n            1d * 1 * 1 * 60, //1min\n            1d * 1 * 10 * 60, //10min\n            1d * 1 * 60 * 60, //1h\n            1d * 12 * 60 * 60, //12h\n            1d * 24 * 60 * 60 //24h\n        );\n\n        List<Double> createTimeBuckets = Arrays.asList(\n            (double) Duration.ofMillis(10).toMillis(), //10ms\n            (double) Duration.ofMillis(100).toMillis(), //100ms\n            (double) Duration.ofSeconds(1).toMillis(), //1s\n            (double) Duration.ofSeconds(3).toMillis(), //3s\n            (double) Duration.ofSeconds(5).toMillis() //5s\n        );\n        InstrumentSelector messageSizeSelector = InstrumentSelector.builder()\n            .setType(InstrumentType.HISTOGRAM)\n            .setName(HISTOGRAM_MESSAGE_SIZE)\n            .build();\n        ViewBuilder messageSizeViewBuilder = View.builder()\n            .setAggregation(Aggregation.explicitBucketHistogram(messageSizeBuckets));\n        // To config the cardinalityLimit for openTelemetry metrics exporting.\n        SdkMeterProviderUtil.setCardinalityLimit(messageSizeViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());\n        providerBuilder.registerView(messageSizeSelector, messageSizeViewBuilder.build());\n\n        InstrumentSelector commitLatencySelector = InstrumentSelector.builder()\n            .setType(InstrumentType.HISTOGRAM)\n            .setName(HISTOGRAM_FINISH_MSG_LATENCY)\n            .build();\n        ViewBuilder commitLatencyViewBuilder = View.builder()\n            .setAggregation(Aggregation.explicitBucketHistogram(commitLatencyBuckets));\n        // To config the cardinalityLimit for openTelemetry metrics exporting.\n        SdkMeterProviderUtil.setCardinalityLimit(commitLatencyViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());\n        providerBuilder.registerView(commitLatencySelector, commitLatencyViewBuilder.build());\n\n        InstrumentSelector createTopicTimeSelector = InstrumentSelector.builder()\n            .setType(InstrumentType.HISTOGRAM)\n            .setName(HISTOGRAM_TOPIC_CREATE_EXECUTE_TIME)\n            .build();\n        InstrumentSelector createSubGroupTimeSelector = InstrumentSelector.builder()\n            .setType(InstrumentType.HISTOGRAM)\n            .setName(HISTOGRAM_CONSUMER_GROUP_CREATE_EXECUTE_TIME)\n            .build();\n        ViewBuilder createTopicTimeViewBuilder = View.builder()\n            .setAggregation(Aggregation.explicitBucketHistogram(createTimeBuckets));\n        ViewBuilder createSubGroupTimeViewBuilder = View.builder()\n            .setAggregation(Aggregation.explicitBucketHistogram(createTimeBuckets));\n        // To config the cardinalityLimit for openTelemetry metrics exporting.\n        SdkMeterProviderUtil.setCardinalityLimit(createTopicTimeViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());\n        providerBuilder.registerView(createTopicTimeSelector, createTopicTimeViewBuilder.build());\n        SdkMeterProviderUtil.setCardinalityLimit(createSubGroupTimeViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());\n        providerBuilder.registerView(createSubGroupTimeSelector, createSubGroupTimeViewBuilder.build());\n\n        for (Pair<InstrumentSelector, ViewBuilder> selectorViewPair : this.remotingMetricsManager.getMetricsView()) {\n            ViewBuilder viewBuilder = selectorViewPair.getObject2();\n            SdkMeterProviderUtil.setCardinalityLimit(viewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());\n            providerBuilder.registerView(selectorViewPair.getObject1(), viewBuilder.build());\n        }\n\n        for (Pair<InstrumentSelector, ViewBuilder> selectorViewPair : messageStore.getMetricsView()) {\n            ViewBuilder viewBuilder = selectorViewPair.getObject2();\n            SdkMeterProviderUtil.setCardinalityLimit(viewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());\n            providerBuilder.registerView(selectorViewPair.getObject1(), viewBuilder.build());\n        }\n\n        for (Pair<InstrumentSelector, ViewBuilder> selectorViewPair : this.popMetricsManager.getMetricsView()) {\n            ViewBuilder viewBuilder = selectorViewPair.getObject2();\n            SdkMeterProviderUtil.setCardinalityLimit(viewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());\n            providerBuilder.registerView(selectorViewPair.getObject1(), viewBuilder.build());\n        }\n\n        // default view builder for all counter.\n        InstrumentSelector defaultCounterSelector = InstrumentSelector.builder()\n            .setType(InstrumentType.COUNTER)\n            .build();\n        ViewBuilder defaultCounterViewBuilder = View.builder().setDescription(\"default view for counter.\");\n        SdkMeterProviderUtil.setCardinalityLimit(defaultCounterViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());\n        providerBuilder.registerView(defaultCounterSelector, defaultCounterViewBuilder.build());\n\n        //default view builder for all observable gauge.\n        InstrumentSelector defaultGaugeSelector = InstrumentSelector.builder()\n            .setType(InstrumentType.OBSERVABLE_GAUGE)\n            .build();\n        ViewBuilder defaultGaugeViewBuilder = View.builder().setDescription(\"default view for gauge.\");\n        SdkMeterProviderUtil.setCardinalityLimit(defaultGaugeViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());\n        providerBuilder.registerView(defaultGaugeSelector, defaultGaugeViewBuilder.build());\n    }\n\n    private void initStatsMetrics() {\n        if (!brokerConfig.isEnableStatsMetrics()) {\n            return;\n        }\n\n        processorWatermark = brokerMeter.gaugeBuilder(GAUGE_PROCESSOR_WATERMARK)\n            .setDescription(\"Request processor watermark\")\n            .ofLongs()\n            .buildWithCallback(measurement -> {\n                measurement.record(brokerController.getSendThreadPoolQueue().size(), newAttributesBuilder().put(LABEL_PROCESSOR, \"send\").build());\n                measurement.record(brokerController.getAsyncPutThreadPoolQueue().size(), newAttributesBuilder().put(LABEL_PROCESSOR, \"async_put\").build());\n                measurement.record(brokerController.getPullThreadPoolQueue().size(), newAttributesBuilder().put(LABEL_PROCESSOR, \"pull\").build());\n                measurement.record(brokerController.getAckThreadPoolQueue().size(), newAttributesBuilder().put(LABEL_PROCESSOR, \"ack\").build());\n                measurement.record(brokerController.getQueryThreadPoolQueue().size(), newAttributesBuilder().put(LABEL_PROCESSOR, \"query_message\").build());\n                measurement.record(brokerController.getClientManagerThreadPoolQueue().size(), newAttributesBuilder().put(LABEL_PROCESSOR, \"client_manager\").build());\n                measurement.record(brokerController.getHeartbeatThreadPoolQueue().size(), newAttributesBuilder().put(LABEL_PROCESSOR, \"heartbeat\").build());\n                measurement.record(brokerController.getLitePullThreadPoolQueue().size(), newAttributesBuilder().put(LABEL_PROCESSOR, \"lite_pull\").build());\n                measurement.record(brokerController.getEndTransactionThreadPoolQueue().size(), newAttributesBuilder().put(LABEL_PROCESSOR, \"transaction\").build());\n                measurement.record(brokerController.getConsumerManagerThreadPoolQueue().size(), newAttributesBuilder().put(LABEL_PROCESSOR, \"consumer_manager\").build());\n                measurement.record(brokerController.getAdminBrokerThreadPoolQueue().size(), newAttributesBuilder().put(LABEL_PROCESSOR, \"admin\").build());\n                measurement.record(brokerController.getReplyThreadPoolQueue().size(), newAttributesBuilder().put(LABEL_PROCESSOR, \"reply\").build());\n            });\n\n        brokerPermission = brokerMeter.gaugeBuilder(GAUGE_BROKER_PERMISSION)\n            .setDescription(\"Broker permission\")\n            .ofLongs()\n            .buildWithCallback(measurement -> measurement.record(brokerConfig.getBrokerPermission(), newAttributesBuilder().build()));\n\n        topicNum = brokerMeter.gaugeBuilder(GAUGE_TOPIC_NUM)\n            .setDescription(\"Active topic number\")\n            .ofLongs()\n            .buildWithCallback(measurement -> measurement.record(brokerController.getTopicConfigManager().getTopicConfigTable().size(), newAttributesBuilder().build()));\n\n        consumerGroupNum = brokerMeter.gaugeBuilder(GAUGE_CONSUMER_GROUP_NUM)\n            .setDescription(\"Active subscription group number\")\n            .ofLongs()\n            .buildWithCallback(measurement -> measurement.record(brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().size(), newAttributesBuilder().build()));\n    }\n\n    private void initRequestMetrics() {\n        if (!brokerConfig.isEnableRequestMetrics()) {\n            return;\n        }\n\n        messagesInTotal = brokerMeter.counterBuilder(COUNTER_MESSAGES_IN_TOTAL)\n            .setDescription(\"Total number of incoming messages\")\n            .build();\n\n        messagesOutTotal = brokerMeter.counterBuilder(COUNTER_MESSAGES_OUT_TOTAL)\n            .setDescription(\"Total number of outgoing messages\")\n            .build();\n\n        throughputInTotal = brokerMeter.counterBuilder(COUNTER_THROUGHPUT_IN_TOTAL)\n            .setDescription(\"Total traffic of incoming messages\")\n            .build();\n\n        throughputOutTotal = brokerMeter.counterBuilder(COUNTER_THROUGHPUT_OUT_TOTAL)\n            .setDescription(\"Total traffic of outgoing messages\")\n            .build();\n\n        messageSize = brokerMeter.histogramBuilder(HISTOGRAM_MESSAGE_SIZE)\n            .setDescription(\"Incoming messages size\")\n            .ofLongs()\n            .build();\n\n        topicCreateExecuteTime = brokerMeter.histogramBuilder(HISTOGRAM_TOPIC_CREATE_EXECUTE_TIME)\n            .setDescription(\"The distribution of create topic time\")\n            .ofLongs()\n            .setUnit(\"milliseconds\")\n            .build();\n\n        consumerGroupCreateExecuteTime = brokerMeter.histogramBuilder(HISTOGRAM_CONSUMER_GROUP_CREATE_EXECUTE_TIME)\n            .setDescription(\"The distribution of create subscription time\")\n            .ofLongs()\n            .setUnit(\"milliseconds\")\n            .build();\n    }\n\n    private void initConnectionMetrics() {\n        if (!brokerConfig.isEnableConnectionMetrics()) {\n            return;\n        }\n\n        producerConnection = brokerMeter.gaugeBuilder(GAUGE_PRODUCER_CONNECTIONS)\n            .setDescription(\"Producer connections\")\n            .ofLongs()\n            .buildWithCallback(measurement -> {\n                Map<ProducerAttr, Integer> metricsMap = new HashMap<>();\n                brokerController.getProducerManager()\n                    .getGroupChannelTable()\n                    .values()\n                    .stream()\n                    .flatMap(map -> map.values().stream())\n                    .forEach(info -> {\n                        ProducerAttr attr = new ProducerAttr(info.getLanguage(), info.getVersion());\n                        Integer count = metricsMap.computeIfAbsent(attr, k -> 0);\n                        metricsMap.put(attr, count + 1);\n                    });\n                metricsMap.forEach((attr, count) -> {\n                    Attributes attributes = newAttributesBuilder()\n                        .put(LABEL_LANGUAGE, attr.language.name().toLowerCase())\n                        .put(LABEL_VERSION, MQVersion.getVersionDesc(attr.version).toLowerCase())\n                        .put(LABEL_PROTOCOL_TYPE, PROTOCOL_TYPE_REMOTING)\n                        .build();\n                    measurement.record(count, attributes);\n                });\n            });\n\n        consumerConnection = brokerMeter.gaugeBuilder(GAUGE_CONSUMER_CONNECTIONS)\n            .setDescription(\"Consumer connections\")\n            .ofLongs()\n            .buildWithCallback(measurement -> {\n                Map<ConsumerAttr, Integer> metricsMap = new HashMap<>();\n                ConsumerManager consumerManager = brokerController.getConsumerManager();\n                consumerManager.getConsumerTable()\n                    .forEach((group, groupInfo) -> {\n                        if (groupInfo != null) {\n                            groupInfo.getChannelInfoTable().values().forEach(info -> {\n                                ConsumerAttr attr = new ConsumerAttr(group, info.getLanguage(), info.getVersion(), groupInfo.getConsumeType());\n                                Integer count = metricsMap.computeIfAbsent(attr, k -> 0);\n                                metricsMap.put(attr, count + 1);\n                            });\n                        }\n                    });\n                metricsMap.forEach((attr, count) -> {\n                    Attributes attributes = newAttributesBuilder()\n                        .put(LABEL_CONSUMER_GROUP, attr.group)\n                        .put(LABEL_LANGUAGE, attr.language.name().toLowerCase())\n                        .put(LABEL_VERSION, MQVersion.getVersionDesc(attr.version).toLowerCase())\n                        .put(LABEL_CONSUME_MODE, attr.consumeMode.getTypeCN().toLowerCase())\n                        .put(LABEL_PROTOCOL_TYPE, PROTOCOL_TYPE_REMOTING)\n                        .put(LABEL_IS_SYSTEM, isSystemGroup(attr.group))\n                        .build();\n                    measurement.record(count, attributes);\n                });\n            });\n    }\n\n    private void initLagAndDlqMetrics() {\n        if (!brokerConfig.isEnableLagAndDlqMetrics()) {\n            return;\n        }\n\n        consumerLagMessages = brokerMeter.gaugeBuilder(GAUGE_CONSUMER_LAG_MESSAGES)\n            .setDescription(\"Consumer lag messages\")\n            .ofLongs()\n            .buildWithCallback(measurement -> {\n                consumerLagCalculator.calculateLag(result ->\n                    measurement.record(result.lag, buildLagAttributes(result))\n                );\n\n                liteConsumerLagCalculator.calculateLiteLagCount(result ->\n                    measurement.record(result.lag, buildLagAttributes(result))\n                );\n            });\n\n        consumerLagLatency = brokerMeter.gaugeBuilder(GAUGE_CONSUMER_LAG_LATENCY)\n            .setDescription(\"Consumer lag time\")\n            .setUnit(\"milliseconds\")\n            .ofLongs()\n            .buildWithCallback(measurement -> {\n                consumerLagCalculator.calculateLag(lagResult ->\n                    measurement.record(lagResult.getLagLatency(), buildLagAttributes(lagResult)));\n\n                liteConsumerLagCalculator.calculateLiteLagLatency(lagResult ->\n                    measurement.record(lagResult.getLagLatency(), buildLagAttributes(lagResult)));\n            });\n\n        consumerInflightMessages = brokerMeter.gaugeBuilder(GAUGE_CONSUMER_INFLIGHT_MESSAGES)\n            .setDescription(\"Consumer inflight messages\")\n            .ofLongs()\n            .buildWithCallback(measurement ->\n                consumerLagCalculator.calculateInflight(result -> measurement.record(result.inFlight, buildLagAttributes(result))));\n\n        consumerQueueingLatency = brokerMeter.gaugeBuilder(GAUGE_CONSUMER_QUEUEING_LATENCY)\n            .setDescription(\"Consumer queueing time\")\n            .setUnit(\"milliseconds\")\n            .ofLongs()\n            .buildWithCallback(measurement -> consumerLagCalculator.calculateInflight(result -> {\n                long latency = 0;\n                long curTimeStamp = System.currentTimeMillis();\n                if (result.earliestUnPulledTimestamp != 0) {\n                    latency = curTimeStamp - result.earliestUnPulledTimestamp;\n                }\n                measurement.record(latency, buildLagAttributes(result));\n            }));\n\n        consumerReadyMessages = brokerMeter.gaugeBuilder(GAUGE_CONSUMER_READY_MESSAGES)\n            .setDescription(\"Consumer ready messages\")\n            .ofLongs()\n            .buildWithCallback(measurement -> {\n                consumerLagCalculator.calculateAvailable(result ->\n                    measurement.record(result.available, buildLagAttributes(result)));\n\n                // for lite, ready == lag\n                liteConsumerLagCalculator.calculateLiteLagCount(result ->\n                    measurement.record(result.lag, buildLagAttributes(result)));\n            });\n\n        sendToDlqMessages = brokerMeter.counterBuilder(COUNTER_CONSUMER_SEND_TO_DLQ_MESSAGES_TOTAL)\n            .setDescription(\"Consumer send to DLQ messages\")\n            .build();\n    }\n\n    private void initTransactionMetrics() {\n        if (!brokerController.getBrokerConfig().isEnableTransactionMetrics()) {\n            return;\n        }\n\n        commitMessagesTotal = brokerMeter.counterBuilder(COUNTER_COMMIT_MESSAGES_TOTAL)\n            .setDescription(\"Total number of commit messages\")\n            .build();\n\n        rollBackMessagesTotal = brokerMeter.counterBuilder(COUNTER_ROLLBACK_MESSAGES_TOTAL)\n            .setDescription(\"Total number of rollback messages\")\n            .build();\n\n        transactionFinishLatency = brokerMeter.histogramBuilder(HISTOGRAM_FINISH_MSG_LATENCY)\n            .setDescription(\"Transaction finish latency\")\n            .ofLongs()\n            .setUnit(\"ms\")\n            .build();\n\n        halfMessages = brokerMeter.gaugeBuilder(GAUGE_HALF_MESSAGES)\n            .setDescription(\"Half messages of all topics\")\n            .ofLongs()\n            .buildWithCallback(measurement -> {\n                brokerController.getTransactionalMessageService().getTransactionMetrics().getTransactionCounts()\n                    .forEach((topic, metric) -> {\n                        measurement.record(\n                            metric.getCount().get(),\n                            newAttributesBuilder().put(DefaultStoreMetricsConstant.LABEL_TOPIC, topic).build()\n                        );\n                    });\n            });\n    }\n\n    private void initOtherMetrics() {\n        if (brokerConfig.isEnableRemotingMetrics()) {\n            this.remotingMetricsManager.initMetrics(brokerMeter, this::newAttributesBuilder);\n        }\n        if (brokerConfig.isEnableMessageStoreMetrics()) {\n            messageStore.initMetrics(brokerMeter, this::newAttributesBuilder);\n        }\n        if (brokerConfig.isEnablePopMetrics()) {\n            this.popMetricsManager.initMetrics(brokerMeter, brokerController, this::newAttributesBuilder);\n        }\n    }\n\n    public LiteConsumerLagCalculator getLiteConsumerLagCalculator() {\n        return liteConsumerLagCalculator;\n    }\n\n    public void shutdown() {\n        if (brokerConfig.isInBrokerContainer()) {\n            // only rto need\n            if (brokerConfig.getMetricsExporterType() == MetricsExporterType.OTLP_GRPC) {\n                while (!periodicMetricReader.forceFlush().join(60, TimeUnit.SECONDS).isDone()) {\n                }\n                while (!periodicMetricReader.shutdown().join(60, TimeUnit.SECONDS).isSuccess()) {\n                }\n                while (!metricExporter.shutdown().join(60, TimeUnit.SECONDS).isSuccess()) {\n                }\n            }\n            if (brokerConfig.getMetricsExporterType() == MetricsExporterType.PROM) {\n                while (!prometheusHttpServer.forceFlush().join(60, TimeUnit.SECONDS).isDone()) {\n                }\n                while (!prometheusHttpServer.shutdown().join(60, TimeUnit.SECONDS).isSuccess()) {\n                }\n            }\n            if (brokerConfig.getMetricsExporterType() == MetricsExporterType.LOG) {\n                while (!periodicMetricReader.forceFlush().join(60, TimeUnit.SECONDS).isDone()) {\n                }\n                while (!periodicMetricReader.shutdown().join(60, TimeUnit.SECONDS).isSuccess()) {\n                }\n                while (!loggingMetricExporter.shutdown().join(60, TimeUnit.SECONDS).isSuccess()) {\n                }\n            }\n        } else {\n            if (brokerConfig.getMetricsExporterType() == MetricsExporterType.OTLP_GRPC) {\n                periodicMetricReader.forceFlush();\n                periodicMetricReader.shutdown();\n                metricExporter.shutdown();\n            }\n            if (brokerConfig.getMetricsExporterType() == MetricsExporterType.PROM) {\n                prometheusHttpServer.forceFlush();\n                prometheusHttpServer.shutdown();\n            }\n            if (brokerConfig.getMetricsExporterType() == MetricsExporterType.LOG) {\n                periodicMetricReader.forceFlush();\n                periodicMetricReader.shutdown();\n                loggingMetricExporter.shutdown();\n            }\n        }\n    }\n\n    public RemotingMetricsManager getRemotingMetricsManager() {\n        return remotingMetricsManager;\n    }\n\n    public PopMetricsManager getPopMetricsManager() {\n        return popMetricsManager;\n    }\n\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerAttr.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.metrics;\n\nimport com.google.common.base.Objects;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\n\npublic class ConsumerAttr {\n    String group;\n    LanguageCode language;\n    int version;\n    ConsumeType consumeMode;\n\n    public ConsumerAttr(String group, LanguageCode language, int version, ConsumeType consumeMode) {\n        this.group = group;\n        this.language = language;\n        this.version = version;\n        this.consumeMode = consumeMode;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o)\n            return true;\n        if (o == null || getClass() != o.getClass())\n            return false;\n        ConsumerAttr attr = (ConsumerAttr) o;\n        return version == attr.version && Objects.equal(group, attr.group) && language == attr.language && consumeMode == attr.consumeMode;\n    }\n\n    @Override\n    public int hashCode() {\n        return Objects.hashCode(group, language, version, consumeMode);\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerLagCalculator.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.metrics;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.TimeUnit;\nimport java.util.function.BiConsumer;\nimport java.util.function.Consumer;\nimport java.util.stream.Collectors;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.client.ConsumerGroupInfo;\nimport org.apache.rocketmq.broker.client.ConsumerManager;\nimport org.apache.rocketmq.broker.filter.ConsumerFilterData;\nimport org.apache.rocketmq.broker.filter.ConsumerFilterManager;\nimport org.apache.rocketmq.broker.filter.ExpressionMessageFilter;\nimport org.apache.rocketmq.broker.longpolling.PopCommandCallback;\nimport org.apache.rocketmq.broker.longpolling.PopLongPollingService;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.broker.processor.PopBufferMergeService;\nimport org.apache.rocketmq.broker.processor.PopInflightMessageCounter;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.filter.FilterAPI;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.subscription.SimpleSubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.DefaultMessageFilter;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\n\npublic class ConsumerLagCalculator {\n\n    private final BrokerConfig brokerConfig;\n    private final TopicConfigManager topicConfigManager;\n    private final ConsumerManager consumerManager;\n    private final ConsumerOffsetManager offsetManager;\n    private final ConsumerFilterManager consumerFilterManager;\n    private final SubscriptionGroupManager subscriptionGroupManager;\n    private final MessageStore messageStore;\n    private final PopBufferMergeService popBufferMergeService;\n    private final PopLongPollingService popLongPollingService;\n    private final PopInflightMessageCounter popInflightMessageCounter;\n\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n\n    public ConsumerLagCalculator(BrokerController brokerController) {\n        this.brokerConfig = brokerController.getBrokerConfig();\n        this.topicConfigManager = brokerController.getTopicConfigManager();\n        this.consumerManager = brokerController.getConsumerManager();\n        this.offsetManager = brokerController.getConsumerOffsetManager();\n        this.consumerFilterManager = brokerController.getConsumerFilterManager();\n        this.subscriptionGroupManager = brokerController.getSubscriptionGroupManager();\n        this.messageStore = brokerController.getMessageStore();\n        this.popBufferMergeService = brokerController.getPopMessageProcessor().getPopBufferMergeService();\n        this.popLongPollingService = brokerController.getPopMessageProcessor().getPopLongPollingService();\n        this.popInflightMessageCounter = brokerController.getPopInflightMessageCounter();\n    }\n\n    public static class ProcessGroupInfo {\n        public String group;\n        public String topic;\n        public boolean isPop;\n        public String retryTopic;\n\n        public ProcessGroupInfo(String group, String topic, boolean isPop,\n            String retryTopic) {\n            this.group = group;\n            this.topic = topic;\n            this.isPop = isPop;\n            this.retryTopic = retryTopic;\n        }\n    }\n\n    public static class BaseCalculateResult {\n        public String group;\n        public String topic;\n        public boolean isRetry;\n\n        public BaseCalculateResult(String group, String topic, boolean isRetry) {\n            this.group = group;\n            this.topic = topic;\n            this.isRetry = isRetry;\n        }\n    }\n\n    public static class CalculateLagResult extends BaseCalculateResult {\n        public long lag;\n        public long earliestUnconsumedTimestamp;\n\n        public CalculateLagResult(String group, String topic, boolean isRetry) {\n            super(group, topic, isRetry);\n        }\n\n        public long getLagLatency() {\n            return earliestUnconsumedTimestamp == 0 ? 0 : System.currentTimeMillis() - earliestUnconsumedTimestamp;\n        }\n    }\n\n    public static class CalculateInflightResult extends BaseCalculateResult {\n        public long inFlight;\n        public long earliestUnPulledTimestamp;\n\n        public CalculateInflightResult(String group, String topic, boolean isRetry) {\n            super(group, topic, isRetry);\n        }\n    }\n\n    public static class CalculateAvailableResult extends BaseCalculateResult {\n        public long available;\n\n        public CalculateAvailableResult(String group, String topic, boolean isRetry) {\n            super(group, topic, isRetry);\n        }\n    }\n\n    private void processAllGroup(Consumer<ProcessGroupInfo> consumer) {\n        for (Map.Entry<String, SubscriptionGroupConfig> subscriptionEntry :\n            subscriptionGroupManager.getSubscriptionGroupTable().entrySet()) {\n            String group = subscriptionEntry.getKey();\n            SubscriptionGroupConfig subscriptionGroupConfig = subscriptionEntry.getValue();\n            ConsumerGroupInfo consumerGroupInfo = consumerManager.getConsumerGroupInfo(group, true);\n\n            boolean isLite = StringUtils.isNotEmpty(subscriptionGroupConfig.getLiteBindTopic());\n            if (isLite) {\n                // lite consumer metrics are calculated by LiteConsumerLagCalculator\n                continue;\n            }\n\n            boolean isPop = false;\n            if (consumerGroupInfo != null) {\n                isPop = consumerGroupInfo.getConsumeType() == ConsumeType.CONSUME_POP;\n            }\n            Set<String> topics;\n            if (brokerConfig.isUseStaticSubscription()) {\n                if (subscriptionGroupConfig.getSubscriptionDataSet() == null ||\n                    subscriptionGroupConfig.getSubscriptionDataSet().isEmpty()) {\n                    continue;\n                }\n                topics = subscriptionGroupConfig.getSubscriptionDataSet()\n                    .stream()\n                    .map(SimpleSubscriptionData::getTopic)\n                    .collect(Collectors.toSet());\n            } else {\n                if (consumerGroupInfo == null) {\n                    continue;\n                }\n                topics = consumerGroupInfo.getSubscribeTopics();\n            }\n\n            if (null == topics || topics.isEmpty()) {\n                continue;\n            }\n            for (String topic : topics) {\n                // skip retry topic\n                if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n                    continue;\n                }\n\n                TopicConfig topicConfig = topicConfigManager.selectTopicConfig(topic);\n                if (topicConfig == null) {\n                    continue;\n                }\n\n                // skip no perm topic\n                int topicPerm = topicConfig.getPerm() & brokerConfig.getBrokerPermission();\n                if (!PermName.isReadable(topicPerm) && !PermName.isWriteable(topicPerm)) {\n                    continue;\n                }\n\n                if (isPop) {\n                    String retryTopic = KeyBuilder.buildPopRetryTopic(topic, group, brokerConfig.isEnableRetryTopicV2());\n                    TopicConfig retryTopicConfig = topicConfigManager.selectTopicConfig(retryTopic);\n                    if (retryTopicConfig != null) {\n                        int retryTopicPerm = retryTopicConfig.getPerm() & brokerConfig.getBrokerPermission();\n                        if (PermName.isReadable(retryTopicPerm) || PermName.isWriteable(retryTopicPerm)) {\n                            consumer.accept(new ProcessGroupInfo(group, topic, true, retryTopic));\n                            continue;\n                        }\n                    }\n                    if (brokerConfig.isEnableRetryTopicV2() && brokerConfig.isRetrieveMessageFromPopRetryTopicV1()) {\n                        String retryTopicV1 = KeyBuilder.buildPopRetryTopicV1(topic, group);\n                        TopicConfig retryTopicConfigV1 = topicConfigManager.selectTopicConfig(retryTopicV1);\n                        if (retryTopicConfigV1 != null) {\n                            int retryTopicPerm = retryTopicConfigV1.getPerm() & brokerConfig.getBrokerPermission();\n                            if (PermName.isReadable(retryTopicPerm) || PermName.isWriteable(retryTopicPerm)) {\n                                consumer.accept(new ProcessGroupInfo(group, topic, true, retryTopicV1));\n                                continue;\n                            }\n                        }\n                    }\n                    consumer.accept(new ProcessGroupInfo(group, topic, true, null));\n                } else {\n                    consumer.accept(new ProcessGroupInfo(group, topic, false, null));\n                }\n            }\n        }\n    }\n\n    public void calculateLag(Consumer<CalculateLagResult> lagRecorder) {\n\n        List<CompletableFuture<CalculateLagResult>> futures = new ArrayList<>();\n\n        BiConsumer<ConsumerLagCalculator.ProcessGroupInfo,\n            CompletableFuture<ConsumerLagCalculator.CalculateLagResult>> biConsumer =\n                (info, future) -> calculate(info, future::complete);\n\n        processAllGroup(info -> {\n            if (info.group == null || info.topic == null) {\n                return;\n            }\n            CompletableFuture<CalculateLagResult> future = new CompletableFuture<>();\n            if (info.isPop && brokerConfig.isEnableNotifyBeforePopCalculateLag()) {\n                if (popLongPollingService.notifyMessageArriving(info.topic, -1, info.group,\n                    true, null, 0, null, null,\n                    new PopCommandCallback(biConsumer, info, future))) {\n                    futures.add(future);\n                    return;\n                }\n            }\n            calculate(info, lagRecorder);\n        });\n\n        // Set the maximum wait time to 10 seconds to avoid indefinite blocking\n        // in case of a fast fail that causes the future to not complete its execution.\n        try {\n            CompletableFuture.allOf(futures.toArray(\n                new CompletableFuture[0])).get(10, TimeUnit.SECONDS);\n\n            futures.forEach(future -> {\n                if (future.isDone() && !future.isCompletedExceptionally()) {\n                    lagRecorder.accept(future.join());\n                }\n            });\n        } catch (Exception e) {\n            LOGGER.error(\"Calculate lag timeout after 10 seconds\", e);\n        }\n    }\n\n    public void calculate(ProcessGroupInfo info, Consumer<CalculateLagResult> lagRecorder) {\n        CalculateLagResult result = new CalculateLagResult(info.group, info.topic, false);\n        try {\n            Pair<Long, Long> lag = getConsumerLagStats(info.group, info.topic, info.isPop);\n            if (lag != null) {\n                result.lag = lag.getObject1();\n                result.earliestUnconsumedTimestamp = lag.getObject2();\n            }\n            lagRecorder.accept(result);\n        } catch (ConsumeQueueException e) {\n            LOGGER.error(\"Failed to get lag stats\", e);\n        }\n\n        if (info.isPop) {\n            try {\n                Pair<Long, Long> retryLag = getConsumerLagStats(info.group, info.retryTopic, true);\n\n                result = new CalculateLagResult(info.group, info.topic, true);\n                if (retryLag != null) {\n                    result.lag = retryLag.getObject1();\n                    result.earliestUnconsumedTimestamp = retryLag.getObject2();\n                }\n                lagRecorder.accept(result);\n            } catch (ConsumeQueueException e) {\n                LOGGER.error(\"Failed to get lag stats\", e);\n            }\n        }\n    }\n\n    public void calculateInflight(Consumer<CalculateInflightResult> inflightRecorder) {\n        processAllGroup(info -> {\n            CalculateInflightResult result = new CalculateInflightResult(info.group, info.topic, false);\n            try {\n                Pair<Long, Long> inFlight = getInFlightMsgStats(info.group, info.topic, info.isPop);\n                if (inFlight != null) {\n                    result.inFlight = inFlight.getObject1();\n                    result.earliestUnPulledTimestamp = inFlight.getObject2();\n                }\n                inflightRecorder.accept(result);\n            } catch (ConsumeQueueException e) {\n                LOGGER.error(\"Failed to get inflight message stats\", e);\n            }\n\n            if (info.isPop) {\n                try {\n                    Pair<Long, Long> retryInFlight = getInFlightMsgStats(info.group, info.retryTopic, true);\n\n                    result = new CalculateInflightResult(info.group, info.topic, true);\n                    if (retryInFlight != null) {\n                        result.inFlight = retryInFlight.getObject1();\n                        result.earliestUnPulledTimestamp = retryInFlight.getObject2();\n                    }\n                    inflightRecorder.accept(result);\n                } catch (ConsumeQueueException e) {\n                    LOGGER.error(\"Failed to get inflight message stats\", e);\n                }\n            }\n        });\n    }\n\n    public void calculateAvailable(Consumer<CalculateAvailableResult> availableRecorder) {\n        processAllGroup(info -> {\n            CalculateAvailableResult result = new CalculateAvailableResult(info.group, info.topic, false);\n\n            try {\n                result.available = getAvailableMsgCount(info.group, info.topic, info.isPop);\n                availableRecorder.accept(result);\n            } catch (ConsumeQueueException e) {\n                LOGGER.error(\"Failed to get available message count\", e);\n            }\n\n\n            if (info.isPop) {\n                try {\n                    long retryAvailable = getAvailableMsgCount(info.group, info.retryTopic, true);\n                    result = new CalculateAvailableResult(info.group, info.topic, true);\n                    result.available = retryAvailable;\n                    availableRecorder.accept(result);\n                } catch (ConsumeQueueException e) {\n                    LOGGER.error(\"Failed to get available message count\", e);\n                }\n            }\n        });\n    }\n\n    public Pair<Long, Long> getConsumerLagStats(String group, String topic, boolean isPop) throws ConsumeQueueException {\n        long total = 0L;\n        long earliestUnconsumedTimestamp = Long.MAX_VALUE;\n\n        if (group == null || topic == null) {\n            return new Pair<>(total, earliestUnconsumedTimestamp);\n        }\n\n        TopicConfig topicConfig = topicConfigManager.selectTopicConfig(topic);\n        if (topicConfig != null) {\n            for (int queueId = 0; queueId < topicConfig.getWriteQueueNums(); queueId++) {\n                Pair<Long, Long> pair = getConsumerLagStats(group, topic, queueId, isPop);\n                total += pair.getObject1();\n                earliestUnconsumedTimestamp = Math.min(earliestUnconsumedTimestamp, pair.getObject2());\n            }\n        } else {\n            LOGGER.warn(\"failed to get config of topic {}\", topic);\n        }\n\n        if (earliestUnconsumedTimestamp < 0 || earliestUnconsumedTimestamp == Long.MAX_VALUE) {\n            earliestUnconsumedTimestamp = 0L;\n        }\n\n        LOGGER.debug(\"GetConsumerLagStats, topic={}, group={}, lag={}, latency={}\", topic, group, total,\n            earliestUnconsumedTimestamp > 0 ? System.currentTimeMillis() - earliestUnconsumedTimestamp : 0);\n\n        return new Pair<>(total, earliestUnconsumedTimestamp);\n    }\n\n    public Pair<Long, Long> getConsumerLagStats(String group, String topic, int queueId, boolean isPop)\n        throws ConsumeQueueException {\n        long brokerOffset = messageStore.getMaxOffsetInQueue(topic, queueId);\n        if (brokerOffset < 0) {\n            brokerOffset = 0;\n        }\n\n        if (isPop && !brokerConfig.isPopConsumerKVServiceEnable()) {\n            long pullOffset = popBufferMergeService.getLatestOffset(topic, group, queueId);\n            if (pullOffset < 0) {\n                pullOffset = offsetManager.queryOffset(group, topic, queueId);\n            }\n            if (pullOffset < 0) {\n                pullOffset = brokerOffset;\n            }\n            long inFlightNum = popInflightMessageCounter.getGroupPopInFlightMessageNum(topic, group, queueId);\n            long lag = calculateMessageCount(group, topic, queueId, pullOffset, brokerOffset) + inFlightNum;\n            long consumerOffset = pullOffset - inFlightNum;\n            long consumerStoreTimeStamp = getStoreTimeStamp(topic, queueId, consumerOffset);\n            return new Pair<>(lag, consumerStoreTimeStamp);\n        }\n\n        long consumerOffset = offsetManager.queryOffset(group, topic, queueId);\n        if (consumerOffset < 0) {\n            consumerOffset = brokerOffset;\n        }\n\n        long lag = calculateMessageCount(group, topic, queueId, consumerOffset, brokerOffset);\n        long consumerStoreTimeStamp = getStoreTimeStamp(topic, queueId, consumerOffset);\n        return new Pair<>(lag, consumerStoreTimeStamp);\n    }\n\n    public Pair<Long, Long> getInFlightMsgStats(String group, String topic, boolean isPop) throws ConsumeQueueException {\n        long total = 0L;\n        long earliestUnPulledTimestamp = Long.MAX_VALUE;\n\n        if (group == null || topic == null) {\n            return new Pair<>(total, earliestUnPulledTimestamp);\n        }\n\n        TopicConfig topicConfig = topicConfigManager.selectTopicConfig(topic);\n        if (topicConfig != null) {\n            for (int queueId = 0; queueId < topicConfig.getWriteQueueNums(); queueId++) {\n                Pair<Long, Long> pair = getInFlightMsgStats(group, topic, queueId, isPop);\n                total += pair.getObject1();\n                earliestUnPulledTimestamp = Math.min(earliestUnPulledTimestamp, pair.getObject2());\n            }\n        } else {\n            LOGGER.warn(\"failed to get config of topic {}\", topic);\n        }\n\n        if (earliestUnPulledTimestamp < 0 || earliestUnPulledTimestamp == Long.MAX_VALUE) {\n            earliestUnPulledTimestamp = 0L;\n        }\n\n        return new Pair<>(total, earliestUnPulledTimestamp);\n    }\n\n    public Pair<Long, Long> getInFlightMsgStats(String group, String topic, int queueId, boolean isPop)\n        throws ConsumeQueueException {\n        if (isPop && !brokerConfig.isPopConsumerKVServiceEnable()) {\n            long inflight = popInflightMessageCounter.getGroupPopInFlightMessageNum(topic, group, queueId);\n            long pullOffset = popBufferMergeService.getLatestOffset(topic, group, queueId);\n            if (pullOffset < 0) {\n                pullOffset = offsetManager.queryOffset(group, topic, queueId);\n            }\n            if (pullOffset < 0) {\n                pullOffset = messageStore.getMaxOffsetInQueue(topic, queueId);\n            }\n            long pullStoreTimeStamp = getStoreTimeStamp(topic, queueId, pullOffset);\n            return new Pair<>(inflight, pullStoreTimeStamp);\n        }\n\n        long pullOffset = offsetManager.queryPullOffset(group, topic, queueId);\n        if (pullOffset < 0) {\n            pullOffset = 0;\n        }\n\n        long commitOffset = offsetManager.queryOffset(group, topic, queueId);\n        if (commitOffset < 0) {\n            commitOffset = pullOffset;\n        }\n\n        long inflight = calculateMessageCount(group, topic, queueId, commitOffset, pullOffset);\n        long pullStoreTimeStamp = getStoreTimeStamp(topic, queueId, pullOffset);\n        return new Pair<>(inflight, pullStoreTimeStamp);\n    }\n\n    public long getAvailableMsgCount(String group, String topic, boolean isPop) throws ConsumeQueueException {\n        long total = 0L;\n\n        if (group == null || topic == null) {\n            return total;\n        }\n\n        TopicConfig topicConfig = topicConfigManager.selectTopicConfig(topic);\n        if (topicConfig != null) {\n            for (int queueId = 0; queueId < topicConfig.getWriteQueueNums(); queueId++) {\n                total += getAvailableMsgCount(group, topic, queueId, isPop);\n            }\n        } else {\n            LOGGER.warn(\"failed to get config of topic {}\", topic);\n        }\n\n        return total;\n    }\n\n    public long getAvailableMsgCount(String group, String topic, int queueId, boolean isPop)\n        throws ConsumeQueueException {\n        long brokerOffset = messageStore.getMaxOffsetInQueue(topic, queueId);\n        if (brokerOffset < 0) {\n            brokerOffset = 0;\n        }\n\n        long pullOffset;\n        if (isPop && !brokerConfig.isPopConsumerKVServiceEnable()) {\n            pullOffset = popBufferMergeService.getLatestOffset(topic, group, queueId);\n            if (pullOffset < 0) {\n                pullOffset = offsetManager.queryOffset(group, topic, queueId);\n            }\n        } else {\n            pullOffset = offsetManager.queryPullOffset(group, topic, queueId);\n        }\n        if (pullOffset < 0) {\n            pullOffset = brokerOffset;\n        }\n\n        return calculateMessageCount(group, topic, queueId, pullOffset, brokerOffset);\n    }\n\n    public long getStoreTimeStamp(String topic, int queueId, long offset) {\n        long storeTimeStamp = Long.MAX_VALUE;\n        if (offset >= 0) {\n            storeTimeStamp = messageStore.getMessageStoreTimeStamp(topic, queueId, offset);\n            storeTimeStamp = storeTimeStamp > 0 ? storeTimeStamp : Long.MAX_VALUE;\n        }\n        return storeTimeStamp;\n    }\n\n    public long calculateMessageCount(String group, String topic, int queueId, long from, long to) {\n        long count = to - from;\n\n        if (brokerConfig.isEstimateAccumulation() && to > from) {\n            SubscriptionData subscriptionData = null;\n            if (brokerConfig.isUseStaticSubscription()) {\n                SubscriptionGroupConfig subscriptionGroupConfig = subscriptionGroupManager.findSubscriptionGroupConfig(group);\n                if (subscriptionGroupConfig != null) {\n                    for (SimpleSubscriptionData simpleSubscriptionData : subscriptionGroupConfig.getSubscriptionDataSet()) {\n                        if (topic.equals(simpleSubscriptionData.getTopic())) {\n                            try {\n                                subscriptionData = FilterAPI.buildSubscriptionData(simpleSubscriptionData.getTopic(),\n                                    simpleSubscriptionData.getExpression(), simpleSubscriptionData.getExpressionType());\n                            } catch (Exception e) {\n                                LOGGER.error(\"Try to build subscription for group:{}, topic:{} exception.\", group, topic, e);\n                            }\n                            break;\n                        }\n                    }\n                }\n            } else {\n                ConsumerGroupInfo consumerGroupInfo = consumerManager.getConsumerGroupInfo(group, true);\n                if (consumerGroupInfo != null) {\n                    subscriptionData = consumerGroupInfo.findSubscriptionData(topic);\n                }\n            }\n\n            if (null != subscriptionData) {\n                if (ExpressionType.TAG.equalsIgnoreCase(subscriptionData.getExpressionType())\n                    && !SubscriptionData.SUB_ALL.equals(subscriptionData.getSubString())) {\n                    count = messageStore.estimateMessageCount(topic, queueId, from, to,\n                        new DefaultMessageFilter(subscriptionData));\n                } else if (ExpressionType.SQL92.equalsIgnoreCase(subscriptionData.getExpressionType())) {\n                    ConsumerFilterData consumerFilterData = consumerFilterManager.get(topic, group);\n                    count = messageStore.estimateMessageCount(topic, queueId, from, to,\n                        new ExpressionMessageFilter(subscriptionData,\n                            consumerFilterData,\n                            consumerFilterManager));\n                }\n            }\n\n        }\n        return count < 0 ? 0 : count;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/metrics/InvocationStatus.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.metrics;\n\npublic enum InvocationStatus {\n    SUCCESS(\"success\"),\n    FAILURE(\"failure\");\n\n    private final String name;\n\n    InvocationStatus(String name) {\n        this.name = name;\n    }\n\n    public String getName() {\n        return name;\n    }\n}"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/metrics/LiteConsumerLagCalculator.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.metrics;\n\nimport com.google.common.annotations.VisibleForTesting;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.Comparator;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.PriorityQueue;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.PriorityBlockingQueue;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.function.BiConsumer;\nimport java.util.function.Consumer;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.entity.TopicGroup;\nimport org.apache.rocketmq.common.lite.LiteLagInfo;\nimport org.apache.rocketmq.common.lite.LiteUtil;\n\npublic class LiteConsumerLagCalculator {\n\n    protected static final long INIT_CONSUME_TIMESTAMP = -1L;\n\n    @VisibleForTesting\n    protected final ConcurrentHashMap<TopicGroup, PriorityBlockingQueue<LagTimeInfo>> topicGroupLagTimeMap =\n        new ConcurrentHashMap<>();\n\n    private final BrokerController brokerController;\n\n    public LiteConsumerLagCalculator(BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    public void removeLagInfo(String group, String bindTopic, String lmqName) {\n        PriorityBlockingQueue<LagTimeInfo> lagHeap = topicGroupLagTimeMap.get(new TopicGroup(bindTopic, group));\n        if (lagHeap != null) {\n            lagHeap.removeIf(info -> info.getLmqName().equals(lmqName));\n        }\n    }\n\n    public void updateLagInfo(String group, String bindTopic, String lmqName, long storeTimestamp) {\n        PriorityBlockingQueue<LagTimeInfo> lagHeap = topicGroupLagTimeMap.computeIfAbsent(\n            new TopicGroup(bindTopic, group),\n            k -> new PriorityBlockingQueue<>(8, Comparator.comparingLong(LagTimeInfo::getLagTimestamp).reversed()));\n        lagHeap.removeIf(info -> info.getLmqName().equals(lmqName));\n        lagHeap.offer(new LagTimeInfo(lmqName, storeTimestamp));\n        int topK = brokerController.getBrokerConfig().getLiteLagLatencyTopK();\n        if (lagHeap.size() > topK) {\n            lagHeap.remove();\n        }\n    }\n\n    @VisibleForTesting\n    protected long getStoreTimestamp(String lmqName, long offset) {\n        return this.brokerController.getMessageStore().getMessageStoreTimeStamp(lmqName, 0, offset);\n    }\n\n    @VisibleForTesting\n    protected long getOffset(String group, String topic) {\n        return brokerController.getConsumerOffsetManager().queryOffset(group, topic, 0);\n    }\n\n    @VisibleForTesting\n    protected long getMaxOffset(String lmqName) {\n        return brokerController.getLiteLifecycleManager().getMaxOffsetInQueue(lmqName);\n    }\n\n    private long offsetDiff(Long offset, String lmqName) {\n        long consumerOffset = offset == null ? -1L : offset;\n        if (consumerOffset < 0) {\n            return 0L;\n        }\n        long maxOffset = getMaxOffset(lmqName);\n        return Math.max(0L, maxOffset - consumerOffset);\n    }\n\n    public void calculateLiteLagCount(Consumer<ConsumerLagCalculator.CalculateLagResult> lagRecorder) {\n        if (!brokerController.getBrokerConfig().isLiteLagCountMetricsEnable()) {\n            return;\n        }\n\n        Map<TopicGroup, Long> counter = new HashMap<>();\n\n        offsetTableForEachByGroup(null, (topicGroup, consumerOffset) -> {\n            String lmqName = topicGroup.topic;\n            String group = topicGroup.group;\n            String parentTopic = LiteUtil.getParentTopic(lmqName);\n            long diff = offsetDiff(consumerOffset, lmqName);\n            if (diff > 0) {\n                TopicGroup key = new TopicGroup(parentTopic, group);\n                counter.merge(key, diff, Long::sum);\n            }\n        });\n\n        counter.forEach((topicGroup, totalCount) -> {\n            ConsumerLagCalculator.CalculateLagResult lagResult =\n                new ConsumerLagCalculator.CalculateLagResult(topicGroup.group, topicGroup.topic, false);\n            lagResult.lag = totalCount;\n            lagRecorder.accept(lagResult);\n        });\n    }\n\n    public void calculateLiteLagLatency(Consumer<ConsumerLagCalculator.CalculateLagResult> lagRecorder) {\n        if (!brokerController.getBrokerConfig().isLiteLagLatencyMetricsEnable()) {\n            return;\n        }\n\n        topicGroupLagTimeMap.forEach((topicGroup, lagHeap) -> {\n            if (CollectionUtils.isEmpty(lagHeap)) {\n                return;\n            }\n\n            // Find the minimum storeTimestamp in the heap\n            long minTimestamp = lagHeap.stream()\n                .mapToLong(LagTimeInfo::getLagTimestamp)\n                .min()\n                .orElse(0L);\n\n            ConsumerLagCalculator.CalculateLagResult lagResult =\n                new ConsumerLagCalculator.CalculateLagResult(topicGroup.group, topicGroup.topic, false);\n            lagResult.earliestUnconsumedTimestamp = minTimestamp;\n            lagRecorder.accept(lagResult);\n        });\n    }\n\n    /**\n     * Get top K LiteLagInfo entries with the smallest lag timestamps for a topic group.\n     *\n     * @param group       consumer group name\n     * @param parentTopic parent topic name\n     * @param topK        max number of entries to retrieve\n     * @return Pair containing:\n     * - Left: list of at most topK LiteLagInfo entries sorted by timestamp\n     * - Right: minimum lag timestamp (or initial consume timestamp if no data)\n     */\n    public Pair<List<LiteLagInfo>/*topK*/, Long/*timestamp*/> getLagTimestampTopK(\n        String group,\n        String parentTopic,\n        int topK\n    ) {\n        TopicGroup key = new TopicGroup(parentTopic, group);\n        PriorityBlockingQueue<LagTimeInfo> lagHeap = topicGroupLagTimeMap.get(key);\n        if (CollectionUtils.isEmpty(lagHeap)) {\n            return Pair.of(Collections.emptyList(), INIT_CONSUME_TIMESTAMP);\n        }\n\n        // Evict the largest timestamp when heap is full, keeping smallest topK timestamps\n        PriorityQueue<LagTimeInfo> maxHeap = new PriorityQueue<>(topK, Comparator.comparingLong(LagTimeInfo::getLagTimestamp).reversed());\n        for (LagTimeInfo lagInfo : lagHeap) {\n            if (maxHeap.size() < topK) {\n                maxHeap.offer(lagInfo);\n            } else if (maxHeap.peek() != null && lagInfo.getLagTimestamp() < maxHeap.peek().getLagTimestamp()) {\n                maxHeap.poll();\n                maxHeap.offer(lagInfo);\n            }\n        }\n\n        // Convert results to LiteLagInfo list and sort by timestamp\n        List<LiteLagInfo> topList = new ArrayList<>(maxHeap.size());\n        for (LagTimeInfo lagInfo : maxHeap) {\n            String lmqName = lagInfo.getLmqName();\n            LiteLagInfo liteLagInfo = new LiteLagInfo();\n            liteLagInfo.setLiteTopic(LiteUtil.getLiteTopic(lmqName));\n            liteLagInfo.setEarliestUnconsumedTimestamp(lagInfo.getLagTimestamp());\n            liteLagInfo.setLagCount(offsetDiff(getOffset(group, lmqName), lmqName));\n            topList.add(liteLagInfo);\n        }\n\n        // Sort by timestamp in ascending order\n        topList.sort(Comparator.comparingLong(LiteLagInfo::getEarliestUnconsumedTimestamp));\n        long minLagTimestamp = topList.isEmpty() ? INIT_CONSUME_TIMESTAMP :\n            topList.get(0).getEarliestUnconsumedTimestamp();\n\n        return Pair.of(topList, minLagTimestamp);\n    }\n\n    /**\n     * Get top K LiteLagInfo entries with the largest lag counts for a topic group.\n     *\n     * @param group consumer group name\n     * @param topK  max number of entries to retrieve\n     * @return Pair containing:\n     * - Left: list of at most topK LiteLagInfo entries sorted by lag count\n     * - Right: total lag count\n     */\n    public Pair<List<LiteLagInfo>, Long> getLagCountTopK(\n        String group,\n        int topK\n    ) {\n        // Use a min heap to maintain the largest topK lag counts\n        PriorityQueue<LiteLagInfo> minHeap = new PriorityQueue<>(topK, Comparator.comparingLong(LiteLagInfo::getLagCount));\n        AtomicLong totalLagCount = new AtomicLong(0L);\n\n        offsetTableForEachByGroup(group, (topicGroup, consumerOffset) -> {\n            String topic = topicGroup.topic;\n\n            long diff = offsetDiff(consumerOffset, topic);\n            if (diff > 0) {\n                totalLagCount.addAndGet(diff);\n                LiteLagInfo liteLagInfo = new LiteLagInfo();\n                liteLagInfo.setLiteTopic(LiteUtil.getLiteTopic(topic));\n                liteLagInfo.setLagCount(diff);\n                liteLagInfo.setEarliestUnconsumedTimestamp(getStoreTimestamp(topic, consumerOffset));\n\n                if (minHeap.size() < topK) {\n                    minHeap.offer(liteLagInfo);\n                } else if (minHeap.peek() != null && liteLagInfo.getLagCount() > minHeap.peek().getLagCount()) {\n                    minHeap.poll();\n                    minHeap.offer(liteLagInfo);\n                }\n            }\n        });\n\n        // Convert heap elements to list and sort by lag count in descending order\n        List<LiteLagInfo> topList = new ArrayList<>(minHeap);\n        topList.sort(Comparator.comparingLong(LiteLagInfo::getLagCount).reversed());\n\n        return Pair.of(topList, totalLagCount.get());\n    }\n\n    /**\n     * Filters the lite group offset by the specified group and processes each entry via BiConsumer.\n     *\n     * @param group    The specified consumer group. If null, all offset information is processed.\n     * @param consumer The BiConsumer used to process each entry.\n     */\n    protected void offsetTableForEachByGroup(\n        String group,\n        BiConsumer<TopicGroup, Long> consumer\n    ) {\n        ConcurrentMap<String, ConcurrentMap<Integer, Long>> offsetTable =\n            brokerController.getConsumerOffsetManager().getOffsetTable();\n        offsetTable.forEach((topicAtGroup, queueOffset) -> {\n            String[] topicGroup = topicAtGroup.split(ConsumerOffsetManager.TOPIC_GROUP_SEPARATOR);\n            if (topicGroup.length == 2) {\n                if (!LiteUtil.isLiteTopicQueue(topicGroup[0])) {\n                    return;\n                }\n                // If group specified, only process the matching group\n                if (StringUtils.isEmpty(group) || group.equals(topicGroup[1])) {\n                    TopicGroup tg = new TopicGroup(topicGroup[0], topicGroup[1]);\n                    Long consumerOffset = queueOffset.get(0);\n                    if (consumerOffset == null) {\n                        return;\n                    }\n                    consumer.accept(tg, consumerOffset);\n                }\n            }\n        });\n    }\n\n    protected static class LagTimeInfo {\n        private final String lmqName;\n        // earliest unconsumed timestamp\n        private final long lagTimestamp;\n\n        public LagTimeInfo(String lmqName, long lagTimestamp) {\n            this.lmqName = lmqName;\n            this.lagTimestamp = lagTimestamp;\n        }\n\n        public String getLmqName() {\n            return lmqName;\n        }\n\n        public long getLagTimestamp() {\n            return lagTimestamp;\n        }\n\n        @Override\n        public boolean equals(Object o) {\n            if (o == null || getClass() != o.getClass()) {\n                return false;\n            }\n            LagTimeInfo lagInfo = (LagTimeInfo) o;\n            return Objects.equals(lmqName, lagInfo.lmqName);\n        }\n\n        @Override\n        public int hashCode() {\n            return Objects.hashCode(lmqName);\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsConstant.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.metrics;\n\npublic class PopMetricsConstant {\n    public static final String HISTOGRAM_POP_BUFFER_SCAN_TIME_CONSUME = \"rocketmq_pop_buffer_scan_time_consume\";\n    public static final String COUNTER_POP_REVIVE_IN_MESSAGE_TOTAL = \"rocketmq_pop_revive_in_message_total\";\n    public static final String COUNTER_POP_REVIVE_OUT_MESSAGE_TOTAL = \"rocketmq_pop_revive_out_message_total\";\n    public static final String COUNTER_POP_REVIVE_RETRY_MESSAGES_TOTAL = \"rocketmq_pop_revive_retry_messages_total\";\n\n    public static final String GAUGE_POP_REVIVE_LAG = \"rocketmq_pop_revive_lag\";\n    public static final String GAUGE_POP_REVIVE_LATENCY = \"rocketmq_pop_revive_latency\";\n    public static final String GAUGE_POP_OFFSET_BUFFER_SIZE = \"rocketmq_pop_offset_buffer_size\";\n    public static final String GAUGE_POP_CHECKPOINT_BUFFER_SIZE = \"rocketmq_pop_checkpoint_buffer_size\";\n\n    public static final String LABEL_REVIVE_MESSAGE_TYPE = \"revive_message_type\";\n    public static final String LABEL_PUT_STATUS = \"put_status\";\n    public static final String LABEL_QUEUE_ID = \"queue_id\";\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.metrics;\n\nimport com.google.common.collect.Lists;\nimport io.opentelemetry.api.common.Attributes;\nimport io.opentelemetry.api.common.AttributesBuilder;\nimport io.opentelemetry.api.metrics.LongCounter;\nimport io.opentelemetry.api.metrics.LongHistogram;\nimport io.opentelemetry.api.metrics.Meter;\nimport io.opentelemetry.api.metrics.ObservableLongMeasurement;\nimport io.opentelemetry.sdk.metrics.Aggregation;\nimport io.opentelemetry.sdk.metrics.InstrumentSelector;\nimport io.opentelemetry.sdk.metrics.InstrumentType;\nimport io.opentelemetry.sdk.metrics.View;\nimport io.opentelemetry.sdk.metrics.ViewBuilder;\nimport java.time.Duration;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.function.Supplier;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.processor.PopBufferMergeService;\nimport org.apache.rocketmq.broker.processor.PopReviveService;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.metrics.NopLongCounter;\nimport org.apache.rocketmq.common.metrics.NopLongHistogram;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.pop.AckMsg;\nimport org.apache.rocketmq.store.pop.PopCheckPoint;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_TOPIC;\nimport static org.apache.rocketmq.broker.metrics.PopMetricsConstant.COUNTER_POP_REVIVE_IN_MESSAGE_TOTAL;\nimport static org.apache.rocketmq.broker.metrics.PopMetricsConstant.COUNTER_POP_REVIVE_OUT_MESSAGE_TOTAL;\nimport static org.apache.rocketmq.broker.metrics.PopMetricsConstant.COUNTER_POP_REVIVE_RETRY_MESSAGES_TOTAL;\nimport static org.apache.rocketmq.broker.metrics.PopMetricsConstant.GAUGE_POP_CHECKPOINT_BUFFER_SIZE;\nimport static org.apache.rocketmq.broker.metrics.PopMetricsConstant.GAUGE_POP_OFFSET_BUFFER_SIZE;\nimport static org.apache.rocketmq.broker.metrics.PopMetricsConstant.GAUGE_POP_REVIVE_LAG;\nimport static org.apache.rocketmq.broker.metrics.PopMetricsConstant.GAUGE_POP_REVIVE_LATENCY;\nimport static org.apache.rocketmq.broker.metrics.PopMetricsConstant.HISTOGRAM_POP_BUFFER_SCAN_TIME_CONSUME;\nimport static org.apache.rocketmq.broker.metrics.PopMetricsConstant.LABEL_PUT_STATUS;\nimport static org.apache.rocketmq.broker.metrics.PopMetricsConstant.LABEL_QUEUE_ID;\nimport static org.apache.rocketmq.broker.metrics.PopMetricsConstant.LABEL_REVIVE_MESSAGE_TYPE;\n\npublic class PopMetricsManager {\n    private static final Logger log = LoggerFactory.getLogger(PopMetricsManager.class);\n    \n    private Supplier<AttributesBuilder> attributesBuilderSupplier;\n\n    private LongHistogram popBufferScanTimeConsume = new NopLongHistogram();\n    private LongCounter popRevivePutTotal = new NopLongCounter();\n    private LongCounter popReviveGetTotal = new NopLongCounter();\n    private LongCounter popReviveRetryMessageTotal = new NopLongCounter();\n\n    public PopMetricsManager() {\n    }\n\n    public List<Pair<InstrumentSelector, ViewBuilder>> getMetricsView() {\n        List<Double> rpcCostTimeBuckets = Arrays.asList(\n            (double) Duration.ofMillis(1).toMillis(),\n            (double) Duration.ofMillis(10).toMillis(),\n            (double) Duration.ofMillis(100).toMillis(),\n            (double) Duration.ofSeconds(1).toMillis(),\n            (double) Duration.ofSeconds(2).toMillis(),\n            (double) Duration.ofSeconds(3).toMillis()\n        );\n        InstrumentSelector popBufferScanTimeConsumeSelector = InstrumentSelector.builder()\n            .setType(InstrumentType.HISTOGRAM)\n            .setName(HISTOGRAM_POP_BUFFER_SCAN_TIME_CONSUME)\n            .build();\n        ViewBuilder popBufferScanTimeConsumeViewBuilder = View.builder()\n            .setAggregation(Aggregation.explicitBucketHistogram(rpcCostTimeBuckets));\n\n        return Lists.newArrayList(new Pair<>(popBufferScanTimeConsumeSelector, popBufferScanTimeConsumeViewBuilder));\n    }\n\n    public void initMetrics(Meter meter, BrokerController brokerController,\n        Supplier<AttributesBuilder> attributesBuilderSupplier) {\n        this.attributesBuilderSupplier = attributesBuilderSupplier;\n\n        this.popBufferScanTimeConsume = meter.histogramBuilder(HISTOGRAM_POP_BUFFER_SCAN_TIME_CONSUME)\n            .setDescription(\"Time consuming of pop buffer scan\")\n            .setUnit(\"milliseconds\")\n            .ofLongs()\n            .build();\n        this.popRevivePutTotal = meter.counterBuilder(COUNTER_POP_REVIVE_IN_MESSAGE_TOTAL)\n            .setDescription(\"Total number of put message to revive topic\")\n            .build();\n        this.popReviveGetTotal = meter.counterBuilder(COUNTER_POP_REVIVE_OUT_MESSAGE_TOTAL)\n            .setDescription(\"Total number of get message from revive topic\")\n            .build();\n        this.popReviveRetryMessageTotal = meter.counterBuilder(COUNTER_POP_REVIVE_RETRY_MESSAGES_TOTAL)\n            .setDescription(\"Total number of put message to pop retry topic\")\n            .build();\n\n        meter.gaugeBuilder(GAUGE_POP_OFFSET_BUFFER_SIZE)\n            .setDescription(\"Time number of buffered offset\")\n            .ofLongs()\n            .buildWithCallback(measurement -> calculatePopBufferOffsetSize(brokerController, measurement));\n        meter.gaugeBuilder(GAUGE_POP_CHECKPOINT_BUFFER_SIZE)\n            .setDescription(\"The number of buffered checkpoint\")\n            .ofLongs()\n            .buildWithCallback(measurement -> calculatePopBufferCkSize(brokerController, measurement));\n        meter.gaugeBuilder(GAUGE_POP_REVIVE_LAG)\n            .setDescription(\"The processing lag of revive topic\")\n            .setUnit(\"milliseconds\")\n            .ofLongs()\n            .buildWithCallback(measurement -> calculatePopReviveLag(brokerController, measurement));\n        meter.gaugeBuilder(GAUGE_POP_REVIVE_LATENCY)\n            .setDescription(\"The processing latency of revive topic\")\n            .setUnit(\"milliseconds\")\n            .ofLongs()\n            .buildWithCallback(measurement -> calculatePopReviveLatency(brokerController, measurement));\n    }\n\n    private void calculatePopBufferOffsetSize(BrokerController brokerController,\n        ObservableLongMeasurement measurement) {\n        PopBufferMergeService popBufferMergeService = brokerController.getPopMessageProcessor().getPopBufferMergeService();\n        measurement.record(popBufferMergeService.getOffsetTotalSize(), this.newAttributesBuilder().build());\n    }\n\n    private void calculatePopBufferCkSize(BrokerController brokerController,\n        ObservableLongMeasurement measurement) {\n        PopBufferMergeService popBufferMergeService = brokerController.getPopMessageProcessor().getPopBufferMergeService();\n        measurement.record(popBufferMergeService.getBufferedCKSize(), this.newAttributesBuilder().build());\n    }\n\n    private void calculatePopReviveLatency(BrokerController brokerController,\n        ObservableLongMeasurement measurement) {\n        PopReviveService[] popReviveServices = brokerController.getAckMessageProcessor().getPopReviveServices();\n        for (PopReviveService popReviveService : popReviveServices) {\n            try {\n                measurement.record(popReviveService.getReviveBehindMillis(), this.newAttributesBuilder()\n                    .put(LABEL_QUEUE_ID, popReviveService.getQueueId())\n                    .build());\n            } catch (ConsumeQueueException e) {\n                log.error(\"Failed to get revive behind duration\", e);\n            }\n        }\n    }\n\n    private void calculatePopReviveLag(BrokerController brokerController,\n        ObservableLongMeasurement measurement) {\n        PopReviveService[] popReviveServices = brokerController.getAckMessageProcessor().getPopReviveServices();\n        for (PopReviveService popReviveService : popReviveServices) {\n            try {\n                measurement.record(popReviveService.getReviveBehindMessages(), this.newAttributesBuilder()\n                    .put(LABEL_QUEUE_ID, popReviveService.getQueueId())\n                    .build());\n            } catch (ConsumeQueueException e) {\n                log.error(\"Failed to get revive behind message count\", e);\n            }\n        }\n    }\n\n    public void incPopReviveAckPutCount(AckMsg ackMsg, PutMessageStatus status) {\n        incPopRevivePutCount(ackMsg.getConsumerGroup(), ackMsg.getTopic(), PopReviveMessageType.ACK, status, 1);\n    }\n\n    public void incPopReviveCkPutCount(PopCheckPoint checkPoint, PutMessageStatus status) {\n        incPopRevivePutCount(checkPoint.getCId(), checkPoint.getTopic(), PopReviveMessageType.CK, status, 1);\n    }\n\n    public void incPopRevivePutCount(String group, String topic, PopReviveMessageType messageType,\n        PutMessageStatus status, int num) {\n        Attributes attributes = this.newAttributesBuilder()\n            .put(LABEL_CONSUMER_GROUP, group)\n            .put(LABEL_TOPIC, topic)\n            .put(LABEL_REVIVE_MESSAGE_TYPE, messageType.name())\n            .put(LABEL_PUT_STATUS, status.name())\n            .build();\n        this.popRevivePutTotal.add(num, attributes);\n    }\n\n    public void incPopReviveAckGetCount(AckMsg ackMsg, int queueId) {\n        incPopReviveGetCount(ackMsg.getConsumerGroup(), ackMsg.getTopic(), PopReviveMessageType.ACK, queueId, 1);\n    }\n\n    public void incPopReviveCkGetCount(PopCheckPoint checkPoint, int queueId) {\n        incPopReviveGetCount(checkPoint.getCId(), checkPoint.getTopic(), PopReviveMessageType.CK, queueId, 1);\n    }\n\n    public void incPopReviveGetCount(String group, String topic, PopReviveMessageType messageType, int queueId,\n        int num) {\n        AttributesBuilder builder = this.newAttributesBuilder();\n        Attributes attributes = builder\n            .put(LABEL_CONSUMER_GROUP, group)\n            .put(LABEL_TOPIC, topic)\n            .put(LABEL_QUEUE_ID, queueId)\n            .put(LABEL_REVIVE_MESSAGE_TYPE, messageType.name())\n            .build();\n        this.popReviveGetTotal.add(num, attributes);\n    }\n\n    public void incPopReviveRetryMessageCount(PopCheckPoint checkPoint, PutMessageStatus status) {\n        AttributesBuilder builder = this.newAttributesBuilder();\n        Attributes attributes = builder\n            .put(LABEL_CONSUMER_GROUP, checkPoint.getCId())\n            .put(LABEL_TOPIC, checkPoint.getTopic())\n            .put(LABEL_PUT_STATUS, status.name())\n            .build();\n        this.popReviveRetryMessageTotal.add(1, attributes);\n    }\n\n    public void recordPopBufferScanTimeConsume(long time) {\n        this.popBufferScanTimeConsume.record(time, this.newAttributesBuilder().build());\n    }\n\n    public AttributesBuilder newAttributesBuilder() {\n        return this.attributesBuilderSupplier != null ? this.attributesBuilderSupplier.get() : Attributes.builder();\n    }\n\n    // Getter methods for external access\n    public LongHistogram getPopBufferScanTimeConsume() {\n        return popBufferScanTimeConsume;\n    }\n\n    public LongCounter getPopRevivePutTotal() {\n        return popRevivePutTotal;\n    }\n\n    public LongCounter getPopReviveGetTotal() {\n        return popReviveGetTotal;\n    }\n\n    public LongCounter getPopReviveRetryMessageTotal() {\n        return popReviveRetryMessageTotal;\n    }\n\n    public Supplier<AttributesBuilder> getAttributesBuilderSupplier() {\n        return attributesBuilderSupplier;\n    }\n\n    // Setter methods for testing\n    public void setAttributesBuilderSupplier(Supplier<AttributesBuilder> attributesBuilderSupplier) {\n        this.attributesBuilderSupplier = attributesBuilderSupplier;\n    }\n\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/metrics/PopReviveMessageType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.metrics;\n\npublic enum PopReviveMessageType {\n    CK,\n    ACK\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/metrics/ProducerAttr.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.metrics;\n\nimport com.google.common.base.Objects;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\n\npublic class ProducerAttr {\n    LanguageCode language;\n    int version;\n\n    public ProducerAttr(LanguageCode language, int version) {\n        this.language = language;\n        this.version = version;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o)\n            return true;\n        if (o == null || getClass() != o.getClass())\n            return false;\n        ProducerAttr attr = (ProducerAttr) o;\n        return version == attr.version && language == attr.language;\n    }\n\n    @Override\n    public int hashCode() {\n        return Objects.hashCode(language, version);\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/mqtrace/ConsumeMessageContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.mqtrace;\n\nimport java.util.Map;\n\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\n\npublic class ConsumeMessageContext {\n    private String consumerGroup;\n    private String topic;\n    private Integer queueId;\n    private String clientHost;\n    private String storeHost;\n    private Map<String, Long> messageIds;\n    private int bodyLength;\n    private boolean success;\n    private String status;\n    private Object mqTraceContext;\n    private TopicConfig topicConfig;\n\n    private String accountAuthType;\n    private String accountOwnerParent;\n    private String accountOwnerSelf;\n    private int rcvMsgNum;\n    private int rcvMsgSize;\n    private BrokerStatsManager.StatsType rcvStat;\n    private int commercialRcvMsgNum;\n\n    private String commercialOwner;\n    private BrokerStatsManager.StatsType commercialRcvStats;\n    private int commercialRcvTimes;\n    private int commercialRcvSize;\n    private int filterMessageCount;\n\n    private String namespace;\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public Integer getQueueId() {\n        return queueId;\n    }\n\n    public void setQueueId(Integer queueId) {\n        this.queueId = queueId;\n    }\n\n    public String getClientHost() {\n        return clientHost;\n    }\n\n    public void setClientHost(String clientHost) {\n        this.clientHost = clientHost;\n    }\n\n    public String getStoreHost() {\n        return storeHost;\n    }\n\n    public void setStoreHost(String storeHost) {\n        this.storeHost = storeHost;\n    }\n\n    public Map<String, Long> getMessageIds() {\n        return messageIds;\n    }\n\n    public void setMessageIds(Map<String, Long> messageIds) {\n        this.messageIds = messageIds;\n    }\n\n    public boolean isSuccess() {\n        return success;\n    }\n\n    public void setSuccess(boolean success) {\n        this.success = success;\n    }\n\n    public String getStatus() {\n        return status;\n    }\n\n    public void setStatus(String status) {\n        this.status = status;\n    }\n\n    public Object getMqTraceContext() {\n        return mqTraceContext;\n    }\n\n    public void setMqTraceContext(Object mqTraceContext) {\n        this.mqTraceContext = mqTraceContext;\n    }\n\n    public TopicConfig getTopicConfig() {\n        return topicConfig;\n    }\n\n    public void setTopicConfig(TopicConfig topicConfig) {\n        this.topicConfig = topicConfig;\n    }\n\n    public int getBodyLength() {\n        return bodyLength;\n    }\n\n    public void setBodyLength(int bodyLength) {\n        this.bodyLength = bodyLength;\n    }\n\n    public String getAccountAuthType() {\n        return accountAuthType;\n    }\n\n    public void setAccountAuthType(String accountAuthType) {\n        this.accountAuthType = accountAuthType;\n    }\n\n    public String getAccountOwnerParent() {\n        return accountOwnerParent;\n    }\n\n    public void setAccountOwnerParent(String accountOwnerParent) {\n        this.accountOwnerParent = accountOwnerParent;\n    }\n\n    public String getAccountOwnerSelf() {\n        return accountOwnerSelf;\n    }\n\n    public void setAccountOwnerSelf(String accountOwnerSelf) {\n        this.accountOwnerSelf = accountOwnerSelf;\n    }\n\n    public int getRcvMsgNum() {\n        return rcvMsgNum;\n    }\n\n    public void setRcvMsgNum(int rcvMsgNum) {\n        this.rcvMsgNum = rcvMsgNum;\n    }\n\n    public int getRcvMsgSize() {\n        return rcvMsgSize;\n    }\n\n    public void setRcvMsgSize(int rcvMsgSize) {\n        this.rcvMsgSize = rcvMsgSize;\n    }\n\n    public BrokerStatsManager.StatsType getRcvStat() {\n        return rcvStat;\n    }\n\n    public void setRcvStat(BrokerStatsManager.StatsType rcvStat) {\n        this.rcvStat = rcvStat;\n    }\n\n    public int getCommercialRcvMsgNum() {\n        return commercialRcvMsgNum;\n    }\n\n    public void setCommercialRcvMsgNum(int commercialRcvMsgNum) {\n        this.commercialRcvMsgNum = commercialRcvMsgNum;\n    }\n\n    public String getCommercialOwner() {\n        return commercialOwner;\n    }\n\n    public void setCommercialOwner(final String commercialOwner) {\n        this.commercialOwner = commercialOwner;\n    }\n\n    public BrokerStatsManager.StatsType getCommercialRcvStats() {\n        return commercialRcvStats;\n    }\n\n    public void setCommercialRcvStats(final BrokerStatsManager.StatsType commercialRcvStats) {\n        this.commercialRcvStats = commercialRcvStats;\n    }\n\n    public int getCommercialRcvTimes() {\n        return commercialRcvTimes;\n    }\n\n    public void setCommercialRcvTimes(final int commercialRcvTimes) {\n        this.commercialRcvTimes = commercialRcvTimes;\n    }\n\n    public int getCommercialRcvSize() {\n        return commercialRcvSize;\n    }\n\n    public void setCommercialRcvSize(final int commercialRcvSize) {\n        this.commercialRcvSize = commercialRcvSize;\n    }\n\n    public String getNamespace() {\n        return namespace;\n    }\n\n    public void setNamespace(String namespace) {\n        this.namespace = namespace;\n    }\n\n    public int getFilterMessageCount() {\n        return filterMessageCount;\n    }\n\n    public void setFilterMessageCount(int filterMessageCount) {\n        this.filterMessageCount = filterMessageCount;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/mqtrace/ConsumeMessageHook.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.mqtrace;\n\npublic interface ConsumeMessageHook {\n    String hookName();\n\n    void consumeMessageBefore(final ConsumeMessageContext context);\n\n    void consumeMessageAfter(final ConsumeMessageContext context);\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/mqtrace/SendMessageContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.mqtrace;\n\nimport java.util.Properties;\n\nimport org.apache.rocketmq.common.message.MessageType;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\n\npublic class SendMessageContext {\n    /** namespace */\n    private String namespace;\n    /** producer group without namespace. */\n    private String producerGroup;\n    /** topic without namespace. */\n    private String topic;\n    private String msgId;\n    private String originMsgId;\n    private Integer queueId;\n    private Long queueOffset;\n    private String brokerAddr;\n    private String bornHost;\n    private int bodyLength;\n    private int code;\n    private String errorMsg;\n    private String msgProps;\n    private Object mqTraceContext;\n    private Properties extProps;\n    private String brokerRegionId;\n    private String msgUniqueKey;\n    private long bornTimeStamp;\n    private long requestTimeStamp;\n    private MessageType msgType = MessageType.Trans_msg_Commit;\n\n    private boolean isSuccess = false;\n\n    /**\n     * Account Statistics\n     */\n    private String accountAuthType;\n    private String accountOwnerParent;\n    private String accountOwnerSelf;\n    private int sendMsgNum;\n    private int sendMsgSize;\n    private BrokerStatsManager.StatsType sendStat;\n    private int commercialSendMsgNum;\n\n    /**\n     * For Commercial\n     */\n    private String commercialOwner;\n    private BrokerStatsManager.StatsType commercialSendStats;\n    private int commercialSendSize;\n    private int commercialSendTimes;\n\n    public String getNamespace() {\n        return namespace;\n    }\n\n    public void setNamespace(String namespace) {\n        this.namespace = namespace;\n    }\n\n    public boolean isSuccess() {\n        return isSuccess;\n    }\n\n    public void setSuccess(final boolean success) {\n        isSuccess = success;\n    }\n\n    public MessageType getMsgType() {\n        return msgType;\n    }\n\n    public void setMsgType(final MessageType msgType) {\n        this.msgType = msgType;\n    }\n\n    public String getMsgUniqueKey() {\n        return msgUniqueKey;\n    }\n\n    public void setMsgUniqueKey(final String msgUniqueKey) {\n        this.msgUniqueKey = msgUniqueKey;\n    }\n\n    public long getBornTimeStamp() {\n        return bornTimeStamp;\n    }\n\n    public void setBornTimeStamp(final long bornTimeStamp) {\n        this.bornTimeStamp = bornTimeStamp;\n    }\n\n    public long getRequestTimeStamp() {\n        return requestTimeStamp;\n    }\n\n    public void setRequestTimeStamp(long requestTimeStamp) {\n        this.requestTimeStamp = requestTimeStamp;\n    }\n\n    public String getBrokerRegionId() {\n        return brokerRegionId;\n    }\n\n    public void setBrokerRegionId(final String brokerRegionId) {\n        this.brokerRegionId = brokerRegionId;\n    }\n\n    public String getProducerGroup() {\n        return producerGroup;\n    }\n\n    public void setProducerGroup(String producerGroup) {\n        this.producerGroup = producerGroup;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getMsgId() {\n        return msgId;\n    }\n\n    public void setMsgId(String msgId) {\n        this.msgId = msgId;\n    }\n\n    public String getOriginMsgId() {\n        return originMsgId;\n    }\n\n    public void setOriginMsgId(String originMsgId) {\n        this.originMsgId = originMsgId;\n    }\n\n    public Integer getQueueId() {\n        return queueId;\n    }\n\n    public void setQueueId(Integer queueId) {\n        this.queueId = queueId;\n    }\n\n    public Long getQueueOffset() {\n        return queueOffset;\n    }\n\n    public void setQueueOffset(Long queueOffset) {\n        this.queueOffset = queueOffset;\n    }\n\n    public String getBrokerAddr() {\n        return brokerAddr;\n    }\n\n    public void setBrokerAddr(String brokerAddr) {\n        this.brokerAddr = brokerAddr;\n    }\n\n    public String getBornHost() {\n        return bornHost;\n    }\n\n    public void setBornHost(String bornHost) {\n        this.bornHost = bornHost;\n    }\n\n    public int getBodyLength() {\n        return bodyLength;\n    }\n\n    public void setBodyLength(int bodyLength) {\n        this.bodyLength = bodyLength;\n    }\n\n    public int getCode() {\n        return code;\n    }\n\n    public void setCode(int code) {\n        this.code = code;\n    }\n\n    public String getErrorMsg() {\n        return errorMsg;\n    }\n\n    public void setErrorMsg(String errorMsg) {\n        this.errorMsg = errorMsg;\n    }\n\n    public String getMsgProps() {\n        return msgProps;\n    }\n\n    public void setMsgProps(String msgProps) {\n        this.msgProps = msgProps;\n    }\n\n    public Object getMqTraceContext() {\n        return mqTraceContext;\n    }\n\n    public void setMqTraceContext(Object mqTraceContext) {\n        this.mqTraceContext = mqTraceContext;\n    }\n\n    public Properties getExtProps() {\n        return extProps;\n    }\n\n    public void setExtProps(Properties extProps) {\n        this.extProps = extProps;\n    }\n\n    public String getCommercialOwner() {\n        return commercialOwner;\n    }\n\n    public void setCommercialOwner(final String commercialOwner) {\n        this.commercialOwner = commercialOwner;\n    }\n\n    public String getAccountAuthType() {\n        return accountAuthType;\n    }\n\n    public void setAccountAuthType(String accountAuthType) {\n        this.accountAuthType = accountAuthType;\n    }\n\n    public String getAccountOwnerParent() {\n        return accountOwnerParent;\n    }\n\n    public void setAccountOwnerParent(String accountOwnerParent) {\n        this.accountOwnerParent = accountOwnerParent;\n    }\n\n    public String getAccountOwnerSelf() {\n        return accountOwnerSelf;\n    }\n\n    public void setAccountOwnerSelf(String accountOwnerSelf) {\n        this.accountOwnerSelf = accountOwnerSelf;\n    }\n\n    public int getSendMsgNum() {\n        return sendMsgNum;\n    }\n\n    public void setSendMsgNum(int sendMsgNum) {\n        this.sendMsgNum = sendMsgNum;\n    }\n\n    public int getSendMsgSize() {\n        return sendMsgSize;\n    }\n\n    public void setSendMsgSize(int sendMsgSize) {\n        this.sendMsgSize = sendMsgSize;\n    }\n\n    public BrokerStatsManager.StatsType getSendStat() {\n        return sendStat;\n    }\n\n    public void setSendStat(BrokerStatsManager.StatsType sendStat) {\n        this.sendStat = sendStat;\n    }\n\n    public BrokerStatsManager.StatsType getCommercialSendStats() {\n        return commercialSendStats;\n    }\n\n    public int getCommercialSendMsgNum() {\n        return commercialSendMsgNum;\n    }\n\n    public void setCommercialSendMsgNum(int commercialSendMsgNum) {\n        this.commercialSendMsgNum = commercialSendMsgNum;\n    }\n\n    public void setCommercialSendStats(final BrokerStatsManager.StatsType commercialSendStats) {\n        this.commercialSendStats = commercialSendStats;\n    }\n\n    public int getCommercialSendSize() {\n        return commercialSendSize;\n    }\n\n    public void setCommercialSendSize(final int commercialSendSize) {\n        this.commercialSendSize = commercialSendSize;\n    }\n\n    public int getCommercialSendTimes() {\n        return commercialSendTimes;\n    }\n\n    public void setCommercialSendTimes(final int commercialSendTimes) {\n        this.commercialSendTimes = commercialSendTimes;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/mqtrace/SendMessageHook.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.mqtrace;\n\npublic interface SendMessageHook {\n    String hookName();\n\n    void sendMessageBefore(final SendMessageContext context);\n\n    void sendMessageAfter(final SendMessageContext context);\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.offset;\n\nimport java.time.Duration;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\n\n/**\n * manage the offset of broadcast.\n * now, use this to support switch remoting client between proxy and broker\n */\npublic class BroadcastOffsetManager extends ServiceThread {\n    private static final String TOPIC_GROUP_SEPARATOR = \"@\";\n    private final BrokerController brokerController;\n    private final BrokerConfig brokerConfig;\n\n    /**\n     * k: topic@groupId\n     * v: the pull offset of all client of all queue\n     */\n    protected final ConcurrentHashMap<String /* topic@groupId */, BroadcastOffsetData> offsetStoreMap =\n        new ConcurrentHashMap<>();\n\n    public BroadcastOffsetManager(BrokerController brokerController) {\n        this.brokerController = brokerController;\n        this.brokerConfig = brokerController.getBrokerConfig();\n    }\n\n    public void updateOffset(String topic, String group, int queueId, long offset, String clientId, boolean fromProxy) {\n        BroadcastOffsetData broadcastOffsetData = offsetStoreMap.computeIfAbsent(\n            buildKey(topic, group), key -> new BroadcastOffsetData(topic, group));\n\n        broadcastOffsetData.clientOffsetStore.compute(clientId, (clientIdKey, broadcastTimedOffsetStore) -> {\n            if (broadcastTimedOffsetStore == null) {\n                broadcastTimedOffsetStore = new BroadcastTimedOffsetStore(fromProxy);\n            }\n\n            broadcastTimedOffsetStore.timestamp = System.currentTimeMillis();\n            broadcastTimedOffsetStore.fromProxy = fromProxy;\n            broadcastTimedOffsetStore.offsetStore.updateOffset(queueId, offset, true);\n            return broadcastTimedOffsetStore;\n        });\n    }\n\n    /**\n     * the time need init offset\n     * 1. client connect to proxy -> client connect to broker\n     * 2. client connect to broker -> client connect to proxy\n     * 3. client connect to proxy at the first time\n     *\n     * @return -1 means no init offset, use the queueOffset in pullRequestHeader\n     */\n    public Long queryInitOffset(String topic, String groupId, int queueId, String clientId, long requestOffset,\n        boolean fromProxy) throws ConsumeQueueException {\n\n        BroadcastOffsetData broadcastOffsetData = offsetStoreMap.get(buildKey(topic, groupId));\n        if (broadcastOffsetData == null) {\n            if (fromProxy && requestOffset < 0) {\n                return getOffset(null, topic, groupId, queueId);\n            } else {\n                return -1L;\n            }\n        }\n\n        final AtomicLong offset = new AtomicLong(-1L);\n        BroadcastTimedOffsetStore offsetStore = broadcastOffsetData.clientOffsetStore.get(clientId);\n        if (offsetStore == null) {\n            offsetStore = new BroadcastTimedOffsetStore(fromProxy);\n            broadcastOffsetData.clientOffsetStore.put(clientId, offsetStore);\n        }\n\n        if (offsetStore.fromProxy && requestOffset < 0) {\n            // when from proxy and requestOffset is -1\n            // means proxy need a init offset to pull message\n            offset.set(getOffset(offsetStore, topic, groupId, queueId));\n        } else {\n            if (offsetStore.fromProxy != fromProxy) {\n                offset.set(getOffset(offsetStore, topic, groupId, queueId));\n            }\n        }\n        return offset.get();\n    }\n\n    private long getOffset(BroadcastTimedOffsetStore offsetStore, String topic, String groupId, int queueId)\n        throws ConsumeQueueException {\n        long storeOffset = -1;\n        if (offsetStore != null) {\n            storeOffset = offsetStore.offsetStore.readOffset(queueId);\n        }\n        if (storeOffset < 0) {\n            storeOffset =\n                brokerController.getConsumerOffsetManager().queryOffset(broadcastGroupId(groupId), topic, queueId);\n        }\n        if (storeOffset < 0) {\n            if (this.brokerController.getMessageStore().checkInMemByConsumeOffset(topic, queueId, 0, 1)) {\n                storeOffset = 0;\n            } else {\n                storeOffset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId, true);\n            }\n        }\n        return storeOffset;\n    }\n\n    /**\n     * 1. scan expire offset\n     * 2. calculate the min offset of all client of one topic@group,\n     * and then commit consumer offset by group@broadcast\n     */\n    protected void scanOffsetData() {\n        for (String k : offsetStoreMap.keySet()) {\n            BroadcastOffsetData broadcastOffsetData = offsetStoreMap.get(k);\n            if (broadcastOffsetData == null) {\n                continue;\n            }\n\n            Map<Integer, Long> queueMinOffset = new HashMap<>();\n\n            for (String clientId : broadcastOffsetData.clientOffsetStore.keySet()) {\n                broadcastOffsetData.clientOffsetStore\n                    .computeIfPresent(clientId, (clientIdKey, broadcastTimedOffsetStore) -> {\n                        long interval = System.currentTimeMillis() - broadcastTimedOffsetStore.timestamp;\n                        boolean clientIsOnline = brokerController.getConsumerManager().findChannel(broadcastOffsetData.group, clientId) != null;\n                        if (clientIsOnline || interval < Duration.ofSeconds(brokerConfig.getBroadcastOffsetExpireSecond()).toMillis()) {\n                            Set<Integer> queueSet = broadcastTimedOffsetStore.offsetStore.queueList();\n                            for (Integer queue : queueSet) {\n                                long offset = broadcastTimedOffsetStore.offsetStore.readOffset(queue);\n                                offset = Math.min(queueMinOffset.getOrDefault(queue, offset), offset);\n                                queueMinOffset.put(queue, offset);\n                            }\n                        }\n                        if (clientIsOnline && interval >= Duration.ofSeconds(brokerConfig.getBroadcastOffsetExpireMaxSecond()).toMillis()) {\n                            return null;\n                        }\n                        if (!clientIsOnline && interval >= Duration.ofSeconds(brokerConfig.getBroadcastOffsetExpireSecond()).toMillis()) {\n                            return null;\n                        }\n                        return broadcastTimedOffsetStore;\n                    });\n            }\n\n            offsetStoreMap.computeIfPresent(k, (key, broadcastOffsetDataVal) -> {\n                if (broadcastOffsetDataVal.clientOffsetStore.isEmpty()) {\n                    return null;\n                }\n                return broadcastOffsetDataVal;\n            });\n\n            queueMinOffset.forEach((queueId, offset) ->\n                this.brokerController.getConsumerOffsetManager().commitOffset(\"BroadcastOffset\",\n                broadcastGroupId(broadcastOffsetData.group), broadcastOffsetData.topic, queueId, offset));\n        }\n    }\n\n    private String buildKey(String topic, String group) {\n        return topic + TOPIC_GROUP_SEPARATOR + group;\n    }\n\n    /**\n     * @param group group of users\n     * @return the groupId used to commit offset\n     */\n    private static String broadcastGroupId(String group) {\n        return group + TOPIC_GROUP_SEPARATOR + \"broadcast\";\n    }\n\n    @Override\n    public String getServiceName() {\n        return \"BroadcastOffsetManager\";\n    }\n\n    @Override\n    public void run() {\n        while (!this.isStopped()) {\n            this.waitForRunning(Duration.ofSeconds(5).toMillis());\n        }\n    }\n\n    @Override\n    protected void onWaitEnd() {\n        this.scanOffsetData();\n    }\n\n    public static class BroadcastOffsetData {\n        private final String topic;\n        private final String group;\n        private final ConcurrentHashMap<String /* clientId */, BroadcastTimedOffsetStore> clientOffsetStore;\n\n        public BroadcastOffsetData(String topic, String group) {\n            this.topic = topic;\n            this.group = group;\n            this.clientOffsetStore = new ConcurrentHashMap<>();\n        }\n    }\n\n    public static class BroadcastTimedOffsetStore {\n\n        /**\n         * the timeStamp of last update occurred\n         */\n        private volatile long timestamp;\n\n        /**\n         * mark the offset of this client is updated by proxy or not\n         */\n        private volatile boolean fromProxy;\n\n        /**\n         * the pulled offset of each queue\n         */\n        private final BroadcastOffsetStore offsetStore;\n\n        public BroadcastTimedOffsetStore(boolean fromProxy) {\n            this.timestamp = System.currentTimeMillis();\n            this.fromProxy = fromProxy;\n            this.offsetStore = new BroadcastOffsetStore();\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/offset/BroadcastOffsetStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.offset;\n\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.common.MixAll;\n\npublic class BroadcastOffsetStore {\n\n    private final ConcurrentMap<Integer, AtomicLong> offsetTable = new ConcurrentHashMap<>();\n\n    public void updateOffset(int queueId, long offset, boolean increaseOnly) {\n        AtomicLong offsetOld = this.offsetTable.get(queueId);\n        if (null == offsetOld) {\n            offsetOld = this.offsetTable.putIfAbsent(queueId, new AtomicLong(offset));\n        }\n\n        if (null != offsetOld) {\n            if (increaseOnly) {\n                MixAll.compareAndIncreaseOnly(offsetOld, offset);\n            } else {\n                offsetOld.set(offset);\n            }\n        }\n    }\n\n    public long readOffset(int queueId) {\n        AtomicLong offset = this.offsetTable.get(queueId);\n        if (offset != null) {\n            return offset.get();\n        }\n        return -1L;\n    }\n\n    public Set<Integer> queueList() {\n        return offsetTable.keySet();\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.offset;\n\nimport com.google.common.base.Strings;\nimport com.google.common.collect.Maps;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.Map;\nimport java.util.Map.Entry;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.function.Function;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.BrokerPathConfigHelper;\nimport org.apache.rocketmq.common.ConfigManager;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class ConsumerOffsetManager extends ConfigManager {\n    protected static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    public static final String TOPIC_GROUP_SEPARATOR = \"@\";\n\n    protected DataVersion dataVersion = new DataVersion();\n\n    protected ConcurrentMap<String/* topic@group */, ConcurrentMap<Integer, Long>> offsetTable =\n        new ConcurrentHashMap<>(512);\n\n    protected final ConcurrentMap<String, ConcurrentMap<Integer, Long>> resetOffsetTable =\n        new ConcurrentHashMap<>(512);\n\n    private final ConcurrentMap<String/* topic@group */, ConcurrentMap<Integer, Long>> pullOffsetTable =\n        new ConcurrentHashMap<>(512);\n\n    protected transient BrokerController brokerController;\n\n    protected final transient AtomicLong versionChangeCounter = new AtomicLong(0);\n\n    public ConsumerOffsetManager() {\n    }\n\n    public ConsumerOffsetManager(BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    public void removeConsumerOffset(String topicAtGroup) {\n\n    }\n\n    public void cleanOffset(String group) {\n        Iterator<Entry<String, ConcurrentMap<Integer, Long>>> it = this.offsetTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, ConcurrentMap<Integer, Long>> next = it.next();\n            String topicAtGroup = next.getKey();\n            if (topicAtGroup.contains(group)) {\n                String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);\n                if (arrays.length == 2 && group.equals(arrays[1])) {\n                    it.remove();\n                    removeConsumerOffset(topicAtGroup);\n                    LOG.warn(\"Clean group's offset, {}, {}\", topicAtGroup, next.getValue());\n                }\n            }\n        }\n    }\n\n    public void cleanOffsetByTopic(String topic) {\n        Iterator<Entry<String, ConcurrentMap<Integer, Long>>> it = this.offsetTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, ConcurrentMap<Integer, Long>> next = it.next();\n            String topicAtGroup = next.getKey();\n            if (topicAtGroup.contains(topic)) {\n                String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);\n                if (arrays.length == 2 && topic.equals(arrays[0])) {\n                    it.remove();\n                    removeConsumerOffset(topicAtGroup);\n                    pullOffsetTable.remove(topicAtGroup);\n                    resetOffsetTable.remove(topicAtGroup);\n                    LOG.warn(\"Clean topic's offset, {}, {}\", topicAtGroup, next.getValue());\n                }\n            }\n        }\n    }\n\n    public void scanUnsubscribedTopic() {\n        Iterator<Entry<String, ConcurrentMap<Integer, Long>>> it = this.offsetTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, ConcurrentMap<Integer, Long>> next = it.next();\n            String topicAtGroup = next.getKey();\n            String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);\n            if (arrays.length == 2) {\n                String topic = arrays[0];\n                String group = arrays[1];\n\n                if (null == brokerController.getConsumerManager().findSubscriptionData(group, topic)\n                    && this.offsetBehindMuchThanData(topic, next.getValue())) {\n                    it.remove();\n                    removeConsumerOffset(topicAtGroup);\n                    LOG.warn(\"remove topic offset, {}\", topicAtGroup);\n                }\n            }\n        }\n    }\n\n    private boolean offsetBehindMuchThanData(final String topic, ConcurrentMap<Integer, Long> table) {\n        Iterator<Entry<Integer, Long>> it = table.entrySet().iterator();\n        boolean result = !table.isEmpty();\n\n        while (it.hasNext() && result) {\n            Entry<Integer, Long> next = it.next();\n            long minOffsetInStore = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, next.getKey());\n            long offsetInPersist = next.getValue();\n            result = offsetInPersist <= minOffsetInStore;\n        }\n\n        return result;\n    }\n\n    public Set<String> whichTopicByConsumer(final String group) {\n        Set<String> topics = new HashSet<>();\n\n        Iterator<Entry<String, ConcurrentMap<Integer, Long>>> it = this.offsetTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, ConcurrentMap<Integer, Long>> next = it.next();\n            String topicAtGroup = next.getKey();\n            String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);\n            if (arrays.length == 2) {\n                if (group.equals(arrays[1])) {\n                    topics.add(arrays[0]);\n                }\n            }\n        }\n\n        return topics;\n    }\n\n    public Set<String> whichGroupByTopic(final String topic) {\n        Set<String> groups = new HashSet<>();\n\n        Iterator<Entry<String, ConcurrentMap<Integer, Long>>> it = this.offsetTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, ConcurrentMap<Integer, Long>> next = it.next();\n            String topicAtGroup = next.getKey();\n            String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);\n            if (arrays.length == 2) {\n                if (topic.equals(arrays[0])) {\n                    groups.add(arrays[1]);\n                }\n            }\n        }\n\n        return groups;\n    }\n\n    public Map<String, Set<String>> getGroupTopicMap() {\n        Map<String, Set<String>> retMap = new HashMap<>(128);\n\n        for (String key : this.offsetTable.keySet()) {\n            String[] arr = key.split(TOPIC_GROUP_SEPARATOR);\n            if (arr.length == 2) {\n                String topic = arr[0];\n                String group = arr[1];\n\n                Set<String> topics = retMap.get(group);\n                if (topics == null) {\n                    topics = new HashSet<>(8);\n                    retMap.put(group, topics);\n                }\n\n                topics.add(topic);\n            }\n        }\n\n        return retMap;\n    }\n\n    public void commitOffset(final String clientHost, final String group, final String topic, final int queueId,\n        final long offset) {\n        // topic@group\n        String key = topic + TOPIC_GROUP_SEPARATOR + group;\n        this.commitOffset(clientHost, key, queueId, offset);\n    }\n\n    private void commitOffset(final String clientHost, final String key, final int queueId, final long offset) {\n        ConcurrentMap<Integer, Long> map = this.offsetTable.get(key);\n        if (null == map) {\n            map = new ConcurrentHashMap<>(2);\n            map.put(queueId, offset);\n            this.offsetTable.put(key, map);\n        } else {\n            Long storeOffset = map.put(queueId, offset);\n            if (storeOffset != null && offset < storeOffset) {\n                LOG.warn(\"[NOTIFYME]update consumer offset less than store. clientHost={}, key={}, queueId={}, requestOffset={}, storeOffset={}\", clientHost, key, queueId, offset, storeOffset);\n            }\n        }\n        if (versionChangeCounter.incrementAndGet() % brokerController.getBrokerConfig().getConsumerOffsetUpdateVersionStep() == 0) {\n            updateDataVersion();\n        }\n    }\n\n    public void commitPullOffset(final String clientHost, final String group, final String topic, final int queueId,\n        final long offset) {\n        // topic@group\n        String key = topic + TOPIC_GROUP_SEPARATOR + group;\n        ConcurrentMap<Integer, Long> map = this.pullOffsetTable.computeIfAbsent(\n            key, k -> new ConcurrentHashMap<>(32));\n        map.put(queueId, offset);\n    }\n\n    /**\n     * If the target queue has temporary reset offset, return the reset-offset.\n     * Otherwise, return the current consume offset in the offset store.\n     * @param group Consumer group\n     * @param topic Topic\n     * @param queueId Queue ID\n     * @return current consume offset or reset offset if there were one.\n     */\n    public long queryOffset(final String group, final String topic, final int queueId) {\n        // topic@group\n        String key = topic + TOPIC_GROUP_SEPARATOR + group;\n\n        if (this.brokerController.getBrokerConfig().isUseServerSideResetOffset()) {\n            Map<Integer, Long> reset = resetOffsetTable.get(key);\n            if (null != reset && reset.containsKey(queueId)) {\n                return reset.get(queueId);\n            }\n        }\n\n        ConcurrentMap<Integer, Long> map = this.offsetTable.get(key);\n        if (null != map) {\n            Long offset = map.get(queueId);\n            if (offset != null) {\n                return offset;\n            }\n        }\n\n        return -1L;\n    }\n\n    /**\n     * Query pull offset in pullOffsetTable\n     * @param group Consumer group\n     * @param topic Topic\n     * @param queueId Queue ID\n     * @return latest pull offset of consumer group\n     */\n    public long queryPullOffset(final String group, final String topic, final int queueId) {\n        // topic@group\n        String key = topic + TOPIC_GROUP_SEPARATOR + group;\n        Long offset = null;\n\n        ConcurrentMap<Integer, Long> map = this.pullOffsetTable.get(key);\n        if (null != map) {\n            offset = map.get(queueId);\n        }\n\n        if (offset == null) {\n            offset = queryOffset(group, topic, queueId);\n        }\n\n        return offset;\n    }\n\n    public void clearPullOffset(final String group, final String topic) {\n        this.pullOffsetTable.remove(topic + TOPIC_GROUP_SEPARATOR + group);\n    }\n\n    @Override\n    public String encode() {\n        return this.encode(false);\n    }\n\n    @Override\n    public String configFilePath() {\n        return BrokerPathConfigHelper.getConsumerOffsetPath(this.brokerController.getMessageStoreConfig().getStorePathRootDir());\n    }\n\n    @Override\n    public void decode(String jsonString) {\n        if (jsonString != null) {\n            ConsumerOffsetManager obj = RemotingSerializable.fromJson(jsonString, ConsumerOffsetManager.class);\n            if (obj != null) {\n                this.setOffsetTable(obj.getOffsetTable());\n                this.dataVersion = obj.dataVersion;\n            }\n        }\n    }\n\n    @Override\n    public String encode(final boolean prettyFormat) {\n        return RemotingSerializable.toJson(this, prettyFormat);\n    }\n\n    public ConcurrentMap<String, ConcurrentMap<Integer, Long>> getOffsetTable() {\n        return offsetTable;\n    }\n\n    public void setOffsetTable(ConcurrentMap<String, ConcurrentMap<Integer, Long>> offsetTable) {\n        this.offsetTable = offsetTable;\n    }\n\n    public ConcurrentMap<String, ConcurrentMap<Integer, Long>> getPullOffsetTable() {\n        return pullOffsetTable;\n    }\n\n    public Map<Integer, Long> queryMinOffsetInAllGroup(final String topic, final String filterGroups) {\n\n        Map<Integer, Long> queueMinOffset = new HashMap<>();\n        Set<String> topicGroups = this.offsetTable.keySet();\n        if (!UtilAll.isBlank(filterGroups)) {\n            for (String group : filterGroups.split(\",\")) {\n                Iterator<String> it = topicGroups.iterator();\n                while (it.hasNext()) {\n                    String topicAtGroup = it.next();\n                    if (group.equals(topicAtGroup.split(TOPIC_GROUP_SEPARATOR)[1])) {\n                        it.remove();\n                        removeConsumerOffset(topicAtGroup);\n                    }\n                }\n            }\n        }\n\n        for (Map.Entry<String, ConcurrentMap<Integer, Long>> offSetEntry : this.offsetTable.entrySet()) {\n            String topicGroup = offSetEntry.getKey();\n            String[] topicGroupArr = topicGroup.split(TOPIC_GROUP_SEPARATOR);\n            if (topic.equals(topicGroupArr[0])) {\n                for (Entry<Integer, Long> entry : offSetEntry.getValue().entrySet()) {\n                    long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, entry.getKey());\n                    if (entry.getValue() >= minOffset) {\n                        Long offset = queueMinOffset.get(entry.getKey());\n                        if (offset == null) {\n                            queueMinOffset.put(entry.getKey(), Math.min(Long.MAX_VALUE, entry.getValue()));\n                        } else {\n                            queueMinOffset.put(entry.getKey(), Math.min(entry.getValue(), offset));\n                        }\n                    }\n                }\n            }\n\n        }\n        return queueMinOffset;\n    }\n\n    public Map<Integer, Long> queryOffset(final String group, final String topic) {\n        // topic@group\n        String key = topic + TOPIC_GROUP_SEPARATOR + group;\n        return this.offsetTable.get(key);\n    }\n\n    public void cloneOffset(final String srcGroup, final String destGroup, final String topic) {\n        ConcurrentMap<Integer, Long> offsets = this.offsetTable.get(topic + TOPIC_GROUP_SEPARATOR + srcGroup);\n        if (offsets != null) {\n            this.offsetTable.put(topic + TOPIC_GROUP_SEPARATOR + destGroup, new ConcurrentHashMap<>(offsets));\n        }\n    }\n\n    public DataVersion getDataVersion() {\n        return dataVersion;\n    }\n\n    public void updateDataVersion() {\n        long stateMachineVersion = brokerController.getMessageStore() != null ?\n            brokerController.getMessageStore().getStateMachineVersion() : 0;\n        dataVersion.nextVersion(stateMachineVersion);\n    }\n\n    public void setDataVersion(DataVersion dataVersion) {\n        this.dataVersion = dataVersion;\n    }\n\n    public boolean loadDataVersion() {\n        String fileName = null;\n        try {\n            fileName = this.configFilePath();\n            String jsonString = MixAll.file2String(fileName);\n            if (jsonString != null) {\n                ConsumerOffsetManager obj = RemotingSerializable.fromJson(jsonString, ConsumerOffsetManager.class);\n                if (obj != null) {\n                    this.dataVersion = obj.dataVersion;\n                }\n                LOG.info(\"load consumer offset dataVersion success,{},{} \", fileName, jsonString);\n            }\n            return true;\n        } catch (Exception e) {\n            LOG.error(\"load consumer offset dataVersion failed \" + fileName, e);\n            return false;\n        }\n    }\n\n    public void removeOffset(final String group) {\n        Function<Iterator<Entry<String, ConcurrentMap<Integer, Long>>>, Boolean> deleteFunction = it -> {\n            boolean removed = false;\n            while (it.hasNext()) {\n                Entry<String, ConcurrentMap<Integer, Long>> entry = it.next();\n                String topicAtGroup = entry.getKey();\n                if (topicAtGroup.contains(group)) {\n                    String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);\n                    if (arrays.length == 2 && group.equals(arrays[1])) {\n                        it.remove();\n                        removeConsumerOffset(topicAtGroup);\n                        removed = true;\n                    }\n                }\n            }\n            return removed;\n        };\n\n        boolean clearOffset = deleteFunction.apply(this.offsetTable.entrySet().iterator());\n        boolean clearReset = deleteFunction.apply(this.resetOffsetTable.entrySet().iterator());\n        boolean clearPull = deleteFunction.apply(this.pullOffsetTable.entrySet().iterator());\n\n        LOG.info(\"Consumer offset manager clean group offset, groupName={}, \" +\n            \"offsetTable={}, resetOffsetTable={}, pullOffsetTable={}\", group, clearOffset, clearReset, clearPull);\n    }\n\n    public void assignResetOffset(String topic, String group, int queueId, long offset) {\n        if (Strings.isNullOrEmpty(topic) || Strings.isNullOrEmpty(group) || queueId < 0 || offset < 0) {\n            LOG.warn(\"Illegal arguments when assigning reset offset. Topic={}, group={}, queueId={}, offset={}\",\n                topic, group, queueId, offset);\n            return;\n        }\n\n        String key = topic + TOPIC_GROUP_SEPARATOR + group;\n        resetOffsetTable.computeIfAbsent(key, k -> Maps.newConcurrentMap()).put(queueId, offset);\n        LOG.debug(\"Reset offset OK. Topic={}, group={}, queueId={}, resetOffset={}\", topic, group, queueId, offset);\n\n        // Two things are important here:\n        // 1, currentOffsetMap might be null if there is no previous records;\n        // 2, Our overriding here may get overridden by the client instantly in concurrent cases; But it still makes\n        // sense in cases like clients are offline.\n        offsetTable.computeIfAbsent(key, k -> Maps.newConcurrentMap()).put(queueId, offset);\n    }\n\n    public boolean hasOffsetReset(String topic, String group, int queueId) {\n        String key = topic + TOPIC_GROUP_SEPARATOR + group;\n        ConcurrentMap<Integer, Long> map = resetOffsetTable.get(key);\n        if (null == map) {\n            return false;\n        }\n        return map.containsKey(queueId);\n    }\n\n    public Long queryThenEraseResetOffset(String topic, String group, Integer queueId) {\n        String key = topic + TOPIC_GROUP_SEPARATOR + group;\n        ConcurrentMap<Integer, Long> map = resetOffsetTable.get(key);\n        if (null == map) {\n            return null;\n        } else {\n            return map.remove(queueId);\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/offset/LmqConsumerOffsetManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.offset;\n\nimport java.util.HashMap;\nimport java.util.Iterator;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\n\nimport com.google.common.base.Strings;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.BrokerPathConfigHelper;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class LmqConsumerOffsetManager extends ConsumerOffsetManager {\n    private ConcurrentHashMap<String, Long> lmqOffsetTable = new ConcurrentHashMap<>(512);\n\n    public LmqConsumerOffsetManager() {\n\n    }\n\n    public LmqConsumerOffsetManager(BrokerController brokerController) {\n        super(brokerController);\n    }\n\n    @Override\n    public long queryOffset(final String group, final String topic, final int queueId) {\n        if (!MixAll.isLmq(group)) {\n            return super.queryOffset(group, topic, queueId);\n        }\n        // topic@group\n        String key = topic + TOPIC_GROUP_SEPARATOR + group;\n        Long offset = lmqOffsetTable.get(key);\n        if (offset != null) {\n            return offset;\n        }\n        return -1;\n    }\n\n    @Override\n    public Map<Integer, Long> queryOffset(final String group, final String topic) {\n        if (!MixAll.isLmq(group)) {\n            return super.queryOffset(group, topic);\n        }\n        Map<Integer, Long> map = new HashMap<>();\n        // topic@group\n        String key = topic + TOPIC_GROUP_SEPARATOR + group;\n        Long offset = lmqOffsetTable.get(key);\n        if (offset != null) {\n            map.put(0, offset);\n        }\n        return map;\n    }\n\n    @Override\n    public void commitOffset(final String clientHost, final String group, final String topic, final int queueId,\n        final long offset) {\n        if (!MixAll.isLmq(group)) {\n            super.commitOffset(clientHost, group, topic, queueId, offset);\n            return;\n        }\n        // topic@group\n        String key = topic + TOPIC_GROUP_SEPARATOR + group;\n        lmqOffsetTable.put(key, offset);\n    }\n\n    @Override\n    public String encode() {\n        return this.encode(false);\n    }\n\n    @Override\n    public String configFilePath() {\n        return BrokerPathConfigHelper.getLmqConsumerOffsetPath(brokerController.getMessageStoreConfig().getStorePathRootDir());\n    }\n\n    @Override\n    public void decode(String jsonString) {\n        if (jsonString != null) {\n            LmqConsumerOffsetManager obj = RemotingSerializable.fromJson(jsonString, LmqConsumerOffsetManager.class);\n            if (obj != null) {\n                super.setOffsetTable(obj.getOffsetTable());\n                this.lmqOffsetTable = obj.lmqOffsetTable;\n            }\n        }\n    }\n\n    @Override\n    public String encode(final boolean prettyFormat) {\n        return RemotingSerializable.toJson(this, prettyFormat);\n    }\n\n    public ConcurrentHashMap<String, Long> getLmqOffsetTable() {\n        return lmqOffsetTable;\n    }\n\n    public void setLmqOffsetTable(ConcurrentHashMap<String, Long> lmqOffsetTable) {\n        this.lmqOffsetTable = lmqOffsetTable;\n    }\n\n    @Override\n    public void removeOffset(String group) {\n        if (!MixAll.isLmq(group)) {\n            super.removeOffset(group);\n            return;\n        }\n        Iterator<Map.Entry<String, Long>> it = this.lmqOffsetTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Map.Entry<String, Long> next = it.next();\n            String topicAtGroup = next.getKey();\n            if (topicAtGroup.contains(group)) {\n                String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);\n                if (arrays.length == 2 && group.equals(arrays[1])) {\n                    it.remove();\n                    removeConsumerOffset(topicAtGroup);\n                    LOG.warn(\"clean lmq group offset {}\", topicAtGroup);\n                }\n            }\n        }\n    }\n\n    @Override\n    public void assignResetOffset(String topic, String group, int queueId, long offset) {\n        if (Strings.isNullOrEmpty(topic) || Strings.isNullOrEmpty(group) || queueId < 0 || offset < 0) {\n            LOG.warn(\"Illegal arguments when assigning reset offset. Topic={}, group={}, queueId={}, offset={}\",\n                    topic, group, queueId, offset);\n            return;\n        }\n        if (!MixAll.isLmq(topic) || !MixAll.isLmq(group)) {\n            super.assignResetOffset(topic, group, queueId, offset);\n            return;\n        }\n\n        String key = topic + TOPIC_GROUP_SEPARATOR + group;\n        ConcurrentMap<Integer, Long> map = resetOffsetTable.get(key);\n        if (null == map) {\n            map = new ConcurrentHashMap<>();\n            ConcurrentMap<Integer, Long> previous = resetOffsetTable.putIfAbsent(key, map);\n            if (null != previous) {\n                map = previous;\n            }\n        }\n        map.put(queueId, offset);\n\n        lmqOffsetTable.computeIfPresent(key, (k, oldValue) -> offset);\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/offset/MemoryConsumerOrderInfoManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.offset;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.pop.orderly.QueueLevelConsumerManager;\n\n/**\n * Memory-based Consumer Order Information Manager for Lite Topics\n * Trade-off considerations::\n * 1. Lite Topics are primarily used for lightweight consumption where\n *    strict ordering requirements are relatively low\n * 2. Considering compatibility with traditional PushConsumer,\n *    a certain degree of ordering control failure is acceptable\n * 3. Avoiding I/O overhead from persistence operations\n * <p>\n * We may make structural adjustments and optimizations to reduce overhead and memory footprint.\n */\npublic class MemoryConsumerOrderInfoManager extends QueueLevelConsumerManager {\n\n    public MemoryConsumerOrderInfoManager(BrokerController brokerController) {\n        super(brokerController);\n    }\n\n    @Override\n    protected void updateLockFreeTimestamp(String topic, String group, int queueId, OrderInfo orderInfo) {\n        if (this.getConsumerOrderInfoLockManager() != null) {\n            // use max lock free time to prevent unexpected blocking\n            this.getConsumerOrderInfoLockManager().updateLockFreeTimestamp(\n                topic, group, queueId, orderInfo.getMaxLockFreeTimestamp());\n        }\n    }\n\n    @Override\n    public void persist() {\n        // MemoryConsumerOrderInfoManager persist, do nothing.\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.out;\n\nimport com.alibaba.fastjson2.JSON;\nimport java.io.UnsupportedEncodingException;\nimport java.net.InetAddress;\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Optional;\nimport java.util.Set;\nimport java.util.concurrent.ArrayBlockingQueue;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.CopyOnWriteArrayList;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.commons.lang3.tuple.Triple;\nimport org.apache.rocketmq.acl.common.AclClientRPCHook;\nimport org.apache.rocketmq.acl.common.SessionCredentials;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.impl.consumer.PullResultExt;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.BrokerIdentity;\nimport org.apache.rocketmq.common.LockCallback;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.UnlockCallback;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageBatch;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.namesrv.DefaultTopAddressing;\nimport org.apache.rocketmq.common.namesrv.TopAddressing;\nimport org.apache.rocketmq.common.sysflag.PullSysFlag;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.InvokeCallback;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.RemotingClient;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.exception.RemotingConnectException;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingClient;\nimport org.apache.rocketmq.remoting.netty.ResponseFuture;\nimport org.apache.rocketmq.remoting.protocol.BrokerSyncInfo;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;\nimport org.apache.rocketmq.remoting.protocol.body.ClusterInfo;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerOffsetSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.ElectMasterResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.GetBrokerMemberGroupResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.KVTable;\nimport org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.LockBatchResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.MessageRequestModeSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.RegisterBrokerBody;\nimport org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.SyncStateSet;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.header.ExchangeHAInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ExchangeHAInfoResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetAllSubscriptionGroupRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetAllSubscriptionGroupResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetAllTopicConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetAllTopicConfigResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetBrokerMemberGroupRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMinOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMinOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.LockBatchMqRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UnlockBatchMqRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.BrokerHeartbeatRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.GetRouteInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.QueryDataVersionRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.QueryDataVersionResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterTopicRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.UnRegisterBrokerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.namesrv.RegisterBrokerResult;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingDetail;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.remoting.rpc.ClientMetadata;\nimport org.apache.rocketmq.remoting.rpc.RpcClient;\nimport org.apache.rocketmq.remoting.rpc.RpcClientImpl;\nimport org.apache.rocketmq.remoting.rpchook.DynamicalExtFieldRPCHook;\nimport org.apache.rocketmq.store.timer.TimerCheckpoint;\nimport org.apache.rocketmq.store.timer.TimerMetrics;\n\nimport static org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode.SUCCESS;\nimport static org.apache.rocketmq.remoting.protocol.ResponseCode.CONTROLLER_MASTER_STILL_EXIST;\n\npublic class BrokerOuterAPI {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private final RemotingClient remotingClient;\n    private final TopAddressing topAddressing = new DefaultTopAddressing(MixAll.getWSAddr());\n    private final ExecutorService brokerOuterExecutor = ThreadUtils.newThreadPoolExecutor(4, 10, 1, TimeUnit.MINUTES,\n            new ArrayBlockingQueue<>(32), new ThreadFactoryImpl(\"brokerOutApi_thread_\", true));\n    private final ClientMetadata clientMetadata;\n    private final RpcClient rpcClient;\n    private String nameSrvAddr = null;\n\n    public BrokerOuterAPI(final NettyClientConfig nettyClientConfig, AuthConfig authConfig) {\n        this(nettyClientConfig, authConfig, new DynamicalExtFieldRPCHook(), new ClientMetadata());\n    }\n\n    private BrokerOuterAPI(final NettyClientConfig nettyClientConfig, AuthConfig authConfig, RPCHook rpcHook, ClientMetadata clientMetadata) {\n        this.remotingClient = new NettyRemotingClient(nettyClientConfig);\n        this.clientMetadata = clientMetadata;\n        this.remotingClient.registerRPCHook(rpcHook);\n        this.remotingClient.registerRPCHook(newAclRPCHook(authConfig));\n        this.rpcClient = new RpcClientImpl(this.clientMetadata, this.remotingClient);\n    }\n\n    private RPCHook newAclRPCHook(AuthConfig config) {\n        if (config == null || StringUtils.isBlank(config.getInnerClientAuthenticationCredentials())) {\n            return null;\n        }\n        SessionCredentials sessionCredentials =\n            JSON.parseObject(config.getInnerClientAuthenticationCredentials(), SessionCredentials.class);\n        if (StringUtils.isBlank(sessionCredentials.getAccessKey()) || StringUtils.isBlank(sessionCredentials.getSecretKey())) {\n            return null;\n        }\n        return new AclClientRPCHook(sessionCredentials);\n    }\n\n    public void start() {\n        this.remotingClient.start();\n    }\n\n    public void shutdown() {\n        this.remotingClient.shutdown();\n        this.brokerOuterExecutor.shutdown();\n    }\n\n    public List<String> getNameServerAddressList() {\n        return this.remotingClient.getNameServerAddressList();\n    }\n\n    public String fetchNameServerAddr() {\n        try {\n            String addrs = this.topAddressing.fetchNSAddr();\n            if (!UtilAll.isBlank(addrs)) {\n                if (!addrs.equals(this.nameSrvAddr)) {\n                    LOGGER.info(\"name server address changed, old: {} new: {}\", this.nameSrvAddr, addrs);\n                    this.updateNameServerAddressList(addrs);\n                    this.nameSrvAddr = addrs;\n                    return nameSrvAddr;\n                }\n            }\n        } catch (Exception e) {\n            LOGGER.error(\"fetchNameServerAddr Exception\", e);\n        }\n        return nameSrvAddr;\n    }\n\n    public List<String> dnsLookupAddressByDomain(String domain) {\n        List<String> addressList = new ArrayList<>();\n        try {\n            java.security.Security.setProperty(\"networkaddress.cache.ttl\", \"10\");\n            int index = domain.indexOf(\":\");\n            String portStr = domain.substring(index);\n            String domainStr = domain.substring(0, index);\n            InetAddress[] addresses = InetAddress.getAllByName(domainStr);\n            for (InetAddress address : addresses) {\n                addressList.add(address.getHostAddress() + portStr);\n            }\n            LOGGER.info(\"dns lookup address by domain success, domain={}, result={}\", domain, addressList);\n        } catch (Exception e) {\n            LOGGER.error(\"dns lookup address by domain error, domain={}\", domain, e);\n        }\n        return addressList;\n    }\n\n    public boolean checkAddressReachable(String address) {\n        return this.remotingClient.isAddressReachable(address);\n    }\n\n    public void updateNameServerAddressList(final String addrs) {\n        String[] addrArray = addrs.split(\";\");\n        List<String> lst = new ArrayList<String>(Arrays.asList(addrArray));\n        this.remotingClient.updateNameServerAddressList(lst);\n    }\n\n    public void updateNameServerAddressListByDnsLookup(final String domain) {\n        List<String> lst = this.dnsLookupAddressByDomain(domain);\n        this.remotingClient.updateNameServerAddressList(lst);\n    }\n\n    public BrokerMemberGroup syncBrokerMemberGroup(String clusterName, String brokerName)\n        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {\n        return syncBrokerMemberGroup(clusterName, brokerName, false);\n    }\n\n    public BrokerMemberGroup syncBrokerMemberGroup(String clusterName, String brokerName,\n        boolean isCompatibleWithOldNameSrv)\n        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {\n        if (isCompatibleWithOldNameSrv) {\n            return getBrokerMemberGroupCompatible(clusterName, brokerName);\n        } else {\n            return getBrokerMemberGroup(clusterName, brokerName);\n        }\n    }\n\n    public BrokerMemberGroup getBrokerMemberGroup(String clusterName, String brokerName)\n        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {\n        BrokerMemberGroup brokerMemberGroup = new BrokerMemberGroup(clusterName, brokerName);\n\n        GetBrokerMemberGroupRequestHeader requestHeader = new GetBrokerMemberGroupRequestHeader();\n        requestHeader.setClusterName(clusterName);\n        requestHeader.setBrokerName(brokerName);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_MEMBER_GROUP, requestHeader);\n\n        RemotingCommand response = null;\n        response = this.remotingClient.invokeSync(null, request, 3000);\n        assert response != null;\n\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                byte[] body = response.getBody();\n                if (body != null) {\n                    GetBrokerMemberGroupResponseBody brokerMemberGroupResponseBody =\n                        GetBrokerMemberGroupResponseBody.decode(body, GetBrokerMemberGroupResponseBody.class);\n\n                    return brokerMemberGroupResponseBody.getBrokerMemberGroup();\n                }\n            }\n            default:\n                break;\n        }\n\n        return brokerMemberGroup;\n    }\n\n    public BrokerMemberGroup getBrokerMemberGroupCompatible(String clusterName, String brokerName)\n        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {\n        BrokerMemberGroup brokerMemberGroup = new BrokerMemberGroup(clusterName, brokerName);\n\n        GetRouteInfoRequestHeader requestHeader = new GetRouteInfoRequestHeader();\n        requestHeader.setTopic(TopicValidator.SYNC_BROKER_MEMBER_GROUP_PREFIX + brokerName);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC, requestHeader);\n\n        RemotingCommand response;\n        response = this.remotingClient.invokeSync(null, request, 3000);\n        assert response != null;\n\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                byte[] body = response.getBody();\n                if (body != null) {\n                    TopicRouteData topicRouteData = TopicRouteData.decode(body, TopicRouteData.class);\n                    for (BrokerData brokerData : topicRouteData.getBrokerDatas()) {\n                        if (brokerData != null\n                            && brokerData.getBrokerName().equals(brokerName)\n                            && brokerData.getCluster().equals(clusterName)) {\n                            brokerMemberGroup.getBrokerAddrs().putAll(brokerData.getBrokerAddrs());\n                            break;\n                        }\n                    }\n                    return brokerMemberGroup;\n                }\n            }\n            default:\n                break;\n        }\n\n        return brokerMemberGroup;\n    }\n\n    public void sendHeartbeatViaDataVersion(\n        final String clusterName,\n        final String brokerAddr,\n        final String brokerName,\n        final Long brokerId,\n        final int timeoutMillis,\n        final DataVersion dataVersion,\n        final boolean isInBrokerContainer) {\n        List<String> nameServerAddressList = this.remotingClient.getAvailableNameSrvList();\n        if (nameServerAddressList != null && !nameServerAddressList.isEmpty()) {\n            final QueryDataVersionRequestHeader requestHeader = new QueryDataVersionRequestHeader();\n            requestHeader.setBrokerAddr(brokerAddr);\n            requestHeader.setBrokerName(brokerName);\n            requestHeader.setBrokerId(brokerId);\n            requestHeader.setClusterName(clusterName);\n\n            for (final String namesrvAddr : nameServerAddressList) {\n                brokerOuterExecutor.execute(() -> {\n                    RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_DATA_VERSION, requestHeader);\n                    request.setBody(dataVersion.encode());\n\n                    try {\n                        BrokerOuterAPI.this.remotingClient.invokeOneway(namesrvAddr, request, timeoutMillis);\n                    } catch (Exception e) {\n                        LOGGER.error(\"sendHeartbeat Exception \" + namesrvAddr, e);\n                    }\n                });\n            }\n        }\n    }\n\n    public void sendHeartbeat(final String clusterName,\n        final String brokerAddr,\n        final String brokerName,\n        final Long brokerId,\n        final int timeoutMills,\n        final boolean isInBrokerContainer) {\n        List<String> nameServerAddressList = this.remotingClient.getAvailableNameSrvList();\n\n        final BrokerHeartbeatRequestHeader requestHeader = new BrokerHeartbeatRequestHeader();\n        requestHeader.setClusterName(clusterName);\n        requestHeader.setBrokerAddr(brokerAddr);\n        requestHeader.setBrokerName(brokerName);\n\n        if (nameServerAddressList != null && nameServerAddressList.size() > 0) {\n            for (final String namesrvAddr : nameServerAddressList) {\n                brokerOuterExecutor.execute(new Runnable() {\n                    @Override\n                    public void run() {\n                        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.BROKER_HEARTBEAT, requestHeader);\n\n                        try {\n                            BrokerOuterAPI.this.remotingClient.invokeOneway(namesrvAddr, request, timeoutMills);\n                        } catch (Exception e) {\n                            LOGGER.error(\"sendHeartbeat Exception \" + namesrvAddr, e);\n                        }\n                    }\n                });\n            }\n        }\n    }\n\n    public BrokerSyncInfo retrieveBrokerHaInfo(String masterBrokerAddr)\n        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException,\n        MQBrokerException, RemotingCommandException {\n        ExchangeHAInfoRequestHeader requestHeader = new ExchangeHAInfoRequestHeader();\n        requestHeader.setMasterHaAddress(null);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.EXCHANGE_BROKER_HA_INFO, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(masterBrokerAddr, request, 3000);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                ExchangeHAInfoResponseHeader responseHeader = response.decodeCommandCustomHeader(ExchangeHAInfoResponseHeader.class);\n                return new BrokerSyncInfo(responseHeader.getMasterHaAddress(), responseHeader.getMasterFlushOffset(), responseHeader.getMasterAddress());\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public void sendBrokerHaInfo(String brokerAddr, String masterHaAddr, long brokerInitMaxOffset, String masterAddr)\n        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {\n        ExchangeHAInfoRequestHeader requestHeader = new ExchangeHAInfoRequestHeader();\n        requestHeader.setMasterHaAddress(masterHaAddr);\n        requestHeader.setMasterFlushOffset(brokerInitMaxOffset);\n        requestHeader.setMasterAddress(masterAddr);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.EXCHANGE_BROKER_HA_INFO, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, 3000);\n\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return;\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public List<RegisterBrokerResult> registerBrokerAll(\n        final String clusterName,\n        final String brokerAddr,\n        final String brokerName,\n        final long brokerId,\n        final String haServerAddr,\n        final TopicConfigSerializeWrapper topicConfigWrapper,\n        final List<String> filterServerList,\n        final boolean oneway,\n        final int timeoutMills,\n        final boolean enableActingMaster,\n        final boolean compressed,\n        final BrokerIdentity brokerIdentity) {\n        return registerBrokerAll(clusterName,\n            brokerAddr,\n            brokerName,\n            brokerId,\n            haServerAddr,\n            topicConfigWrapper,\n            filterServerList,\n            oneway, timeoutMills,\n            enableActingMaster,\n            compressed,\n            null,\n            brokerIdentity);\n    }\n\n    /**\n     * Considering compression brings much CPU overhead to name server, stream API will not support compression and\n     * compression feature is deprecated.\n     *\n     * @param clusterName\n     * @param brokerAddr\n     * @param brokerName\n     * @param brokerId\n     * @param haServerAddr\n     * @param topicConfigWrapper\n     * @param filterServerList\n     * @param oneway\n     * @param timeoutMills\n     * @param compressed         default false\n     * @return\n     */\n    public List<RegisterBrokerResult> registerBrokerAll(\n        final String clusterName,\n        final String brokerAddr,\n        final String brokerName,\n        final long brokerId,\n        final String haServerAddr,\n        final TopicConfigSerializeWrapper topicConfigWrapper,\n        final List<String> filterServerList,\n        final boolean oneway,\n        final int timeoutMills,\n        final boolean enableActingMaster,\n        final boolean compressed,\n        final Long heartbeatTimeoutMillis,\n        final BrokerIdentity brokerIdentity) {\n\n        final List<RegisterBrokerResult> registerBrokerResultList = new CopyOnWriteArrayList<>();\n        List<String> nameServerAddressList = this.remotingClient.getAvailableNameSrvList();\n        if (nameServerAddressList != null && nameServerAddressList.size() > 0) {\n\n            final RegisterBrokerRequestHeader requestHeader = new RegisterBrokerRequestHeader();\n            requestHeader.setBrokerAddr(brokerAddr);\n            requestHeader.setBrokerId(brokerId);\n            requestHeader.setBrokerName(brokerName);\n            requestHeader.setClusterName(clusterName);\n            requestHeader.setHaServerAddr(haServerAddr);\n            requestHeader.setEnableActingMaster(enableActingMaster);\n            requestHeader.setCompressed(false);\n            if (heartbeatTimeoutMillis != null) {\n                requestHeader.setHeartbeatTimeoutMillis(heartbeatTimeoutMillis);\n            }\n\n            RegisterBrokerBody requestBody = new RegisterBrokerBody();\n            requestBody.setTopicConfigSerializeWrapper(TopicConfigAndMappingSerializeWrapper.from(topicConfigWrapper));\n            requestBody.setFilterServerList(filterServerList);\n            final byte[] body = requestBody.encode(compressed);\n            final int bodyCrc32 = UtilAll.crc32(body);\n            requestHeader.setBodyCrc32(bodyCrc32);\n            final CountDownLatch countDownLatch = new CountDownLatch(nameServerAddressList.size());\n            for (final String namesrvAddr : nameServerAddressList) {\n                brokerOuterExecutor.execute(new Runnable() {\n                    @Override\n                    public void run() {\n                        try {\n                            RegisterBrokerResult result = registerBroker(namesrvAddr, oneway, timeoutMills, requestHeader, body);\n                            if (result != null) {\n                                registerBrokerResultList.add(result);\n                            }\n\n                            LOGGER.info(\"Registering current broker to name server completed. TargetHost={}\", namesrvAddr);\n                        } catch (Exception e) {\n                            LOGGER.error(\"Failed to register current broker to name server. TargetHost={}\", namesrvAddr, e);\n                        } finally {\n                            countDownLatch.countDown();\n                        }\n                    }\n                });\n            }\n\n            try {\n                if (!countDownLatch.await(timeoutMills, TimeUnit.MILLISECONDS)) {\n                    LOGGER.warn(\"Registration to one or more name servers does NOT complete within deadline. Timeout threshold: {}ms\", timeoutMills);\n                }\n            } catch (InterruptedException ignore) {\n            }\n        }\n\n        return registerBrokerResultList;\n    }\n\n    private RegisterBrokerResult registerBroker(\n        final String namesrvAddr,\n        final boolean oneway,\n        final int timeoutMills,\n        final RegisterBrokerRequestHeader requestHeader,\n        final byte[] body\n    ) throws RemotingCommandException, MQBrokerException, RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException,\n        InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.REGISTER_BROKER, requestHeader);\n        request.setBody(body);\n\n        if (oneway) {\n            try {\n                this.remotingClient.invokeOneway(namesrvAddr, request, timeoutMills);\n            } catch (RemotingTooMuchRequestException e) {\n                // Ignore\n            }\n            return null;\n        }\n\n        RemotingCommand response = this.remotingClient.invokeSync(namesrvAddr, request, timeoutMills);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                RegisterBrokerResponseHeader responseHeader = response.decodeCommandCustomHeader(RegisterBrokerResponseHeader.class);\n                RegisterBrokerResult result = new RegisterBrokerResult();\n                result.setMasterAddr(responseHeader.getMasterAddr());\n                result.setHaServerAddr(responseHeader.getHaServerAddr());\n                if (response.getBody() != null) {\n                    result.setKvTable(KVTable.decode(response.getBody(), KVTable.class));\n                }\n                return result;\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), requestHeader == null ? null : requestHeader.getBrokerAddr());\n    }\n\n    public void unregisterBrokerAll(\n        final String clusterName,\n        final String brokerAddr,\n        final String brokerName,\n        final long brokerId\n    ) {\n        List<String> nameServerAddressList = this.remotingClient.getNameServerAddressList();\n        if (nameServerAddressList != null) {\n            for (String namesrvAddr : nameServerAddressList) {\n                try {\n                    this.unregisterBroker(namesrvAddr, clusterName, brokerAddr, brokerName, brokerId);\n                    LOGGER.info(\"unregisterBroker OK, NamesrvAddr: {}\", namesrvAddr);\n                } catch (Exception e) {\n                    LOGGER.warn(\"unregisterBroker Exception, NamesrvAddr: {}\", namesrvAddr, e);\n                }\n            }\n        }\n    }\n\n    public void unregisterBroker(\n        final String namesrvAddr,\n        final String clusterName,\n        final String brokerAddr,\n        final String brokerName,\n        final long brokerId\n    ) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {\n        UnRegisterBrokerRequestHeader requestHeader = new UnRegisterBrokerRequestHeader();\n        requestHeader.setBrokerAddr(brokerAddr);\n        requestHeader.setBrokerId(brokerId);\n        requestHeader.setBrokerName(brokerName);\n        requestHeader.setClusterName(clusterName);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UNREGISTER_BROKER, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(namesrvAddr, request, 3000);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return;\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr);\n    }\n\n    /**\n     * Register the topic route info of single topic to all name server nodes.\n     * This method is used to replace incremental broker registration feature.\n     */\n    public void registerSingleTopicAll(\n        final String brokerName,\n        final TopicConfig topicConfig,\n        final int timeoutMills) {\n        String topic = topicConfig.getTopicName();\n        RegisterTopicRequestHeader requestHeader = new RegisterTopicRequestHeader();\n        requestHeader.setTopic(topic);\n\n        TopicRouteData topicRouteData = new TopicRouteData();\n        List<QueueData> queueDatas = new ArrayList<>();\n        topicRouteData.setQueueDatas(queueDatas);\n\n        final QueueData queueData = new QueueData();\n        queueData.setBrokerName(brokerName);\n        queueData.setPerm(topicConfig.getPerm());\n        queueData.setReadQueueNums(topicConfig.getReadQueueNums());\n        queueData.setWriteQueueNums(topicConfig.getWriteQueueNums());\n        queueData.setTopicSysFlag(topicConfig.getTopicSysFlag());\n        queueDatas.add(queueData);\n        final byte[] topicRouteBody = topicRouteData.encode();\n\n        List<String> nameServerAddressList = this.remotingClient.getNameServerAddressList();\n        final CountDownLatch countDownLatch = new CountDownLatch(nameServerAddressList.size());\n        for (final String namesrvAddr : nameServerAddressList) {\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.REGISTER_TOPIC_IN_NAMESRV, requestHeader);\n            request.setBody(topicRouteBody);\n\n            try {\n                brokerOuterExecutor.execute(() -> {\n                    try {\n                        RemotingCommand response = BrokerOuterAPI.this.remotingClient.invokeSync(namesrvAddr, request, timeoutMills);\n                        assert response != null;\n                        LOGGER.info(\"Register single topic {} to broker {} with response code {}\", topic, brokerName, response.getCode());\n                    } catch (Exception e) {\n                        LOGGER.warn(\"Register single topic {} to broker {} exception\", topic, brokerName, e);\n                    } finally {\n                        countDownLatch.countDown();\n                    }\n                });\n            } catch (Exception e) {\n                LOGGER.warn(\"Execute single topic registration task failed, topic {}, broker name {}\", topic, brokerName);\n                countDownLatch.countDown();\n            }\n\n        }\n\n        try {\n            if (!countDownLatch.await(timeoutMills, TimeUnit.MILLISECONDS)) {\n                LOGGER.warn(\"Registration single topic to one or more name servers timeout. Timeout threshold: {}ms\", timeoutMills);\n            }\n        } catch (InterruptedException ignore) {\n        }\n    }\n\n    public List<Boolean> needRegister(\n        final String clusterName,\n        final String brokerAddr,\n        final String brokerName,\n        final long brokerId,\n        final TopicConfigSerializeWrapper topicConfigWrapper,\n        final int timeoutMills,\n        final boolean isInBrokerContainer) {\n        final List<Boolean> changedList = new CopyOnWriteArrayList<>();\n        List<String> nameServerAddressList = this.remotingClient.getNameServerAddressList();\n        if (nameServerAddressList != null && nameServerAddressList.size() > 0) {\n            final CountDownLatch countDownLatch = new CountDownLatch(nameServerAddressList.size());\n            for (final String namesrvAddr : nameServerAddressList) {\n                brokerOuterExecutor.execute(new Runnable() {\n                    @Override\n                    public void run() {\n                        try {\n                            QueryDataVersionRequestHeader requestHeader = new QueryDataVersionRequestHeader();\n                            requestHeader.setBrokerAddr(brokerAddr);\n                            requestHeader.setBrokerId(brokerId);\n                            requestHeader.setBrokerName(brokerName);\n                            requestHeader.setClusterName(clusterName);\n                            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_DATA_VERSION, requestHeader);\n                            request.setBody(topicConfigWrapper.getDataVersion().encode());\n                            RemotingCommand response = remotingClient.invokeSync(namesrvAddr, request, timeoutMills);\n                            DataVersion nameServerDataVersion = null;\n                            Boolean changed = false;\n                            switch (response.getCode()) {\n                                case ResponseCode.SUCCESS: {\n                                    QueryDataVersionResponseHeader queryDataVersionResponseHeader =\n                                        response.decodeCommandCustomHeader(QueryDataVersionResponseHeader.class);\n                                    changed = queryDataVersionResponseHeader.getChanged();\n                                    byte[] body = response.getBody();\n                                    if (body != null) {\n                                        nameServerDataVersion = DataVersion.decode(body, DataVersion.class);\n                                        if (!topicConfigWrapper.getDataVersion().equals(nameServerDataVersion)) {\n                                            changed = true;\n                                        }\n                                    }\n                                    if (changed == null || changed) {\n                                        changedList.add(Boolean.TRUE);\n                                    }\n                                }\n                                default:\n                                    break;\n                            }\n                            LOGGER.warn(\"Query data version from name server {} OK, changed {}, broker {}, name server {}\", namesrvAddr, changed, topicConfigWrapper.getDataVersion(), nameServerDataVersion == null ? \"\" : nameServerDataVersion);\n                        } catch (Exception e) {\n                            changedList.add(Boolean.TRUE);\n                            LOGGER.error(\"Query data version from name server {} exception\", namesrvAddr, e);\n                        } finally {\n                            countDownLatch.countDown();\n                        }\n                    }\n                });\n\n            }\n            try {\n                countDownLatch.await(timeoutMills, TimeUnit.MILLISECONDS);\n            } catch (InterruptedException e) {\n                LOGGER.error(\"query dataversion from nameserver countDownLatch await Exception\", e);\n            }\n        }\n        return changedList;\n    }\n\n    public TopicConfigAndMappingSerializeWrapper getAllTopicConfig(final String addr)\n        throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException,\n        InterruptedException, MQBrokerException, RemotingCommandException {\n\n        DataVersion topicConfigDataVersion = null;\n        DataVersion mappingDataVersion = null;\n        long timeoutMills = getTimeoutMillis();\n        int topicSeq = 0;\n        long beginTime = System.nanoTime();\n        ConcurrentHashMap<String, TopicConfig> topicConfigTable = new ConcurrentHashMap<>();\n        Map<String, TopicQueueMappingDetail> topicQueueMappingDetailMap = new ConcurrentHashMap<>();\n        while (true) {\n            long leftTime = timeoutMills - TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - beginTime);\n            if (leftTime < 0) {\n                throw new RemotingTimeoutException(\"invokeSync call timeout\");\n            }\n\n            GetAllTopicConfigRequestHeader requestHeader = new GetAllTopicConfigRequestHeader();\n            requestHeader.setTopicSeq(topicSeq);\n            requestHeader.setMaxTopicNum(getMaxPageSize());\n            requestHeader.setDataVersion(Optional.ofNullable(topicConfigDataVersion).\n                map(DataVersion::toJson).orElse(StringUtils.EMPTY));\n            LOGGER.info(\"getAllTopicConfig from seq {}, max {}, dataVersion {}\",\n                    topicSeq, requestHeader.getMaxTopicNum(), requestHeader.getDataVersion());\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_TOPIC_CONFIG, requestHeader);\n\n            RemotingCommand response = this.remotingClient.invokeSync(\n                MixAll.brokerVIPChannel(true, addr), request, 30000);\n\n            assert response != null;\n            if (response.getCode() == SUCCESS) {\n                TopicConfigAndMappingSerializeWrapper topicConfigSerializeWrapper =\n                    TopicConfigAndMappingSerializeWrapper.decode(response.getBody(), TopicConfigAndMappingSerializeWrapper.class);\n                topicConfigTable.putAll(topicConfigSerializeWrapper.getTopicConfigTable());\n                topicQueueMappingDetailMap.putAll(topicConfigSerializeWrapper.getTopicQueueMappingDetailMap());\n                topicSeq += topicConfigSerializeWrapper.getTopicConfigTable().size();\n\n\n                DataVersion newDataVersion = topicConfigSerializeWrapper.getDataVersion();\n                if (topicConfigDataVersion == null) {\n                    // fill dataVersion before break the loop to compatible with old version server\n                    topicConfigDataVersion = newDataVersion;\n                    mappingDataVersion = topicConfigSerializeWrapper.getMappingDataVersion();\n                }\n\n                GetAllTopicConfigResponseHeader responseHeader =\n                    response.decodeCommandCustomHeader(GetAllTopicConfigResponseHeader.class);\n                Integer totalTopicNum = Optional.ofNullable(responseHeader)\n                    .map(GetAllTopicConfigResponseHeader::getTotalTopicNum).orElse(null);\n\n                if (Objects.isNull(totalTopicNum)) {       // compatible with old version server\n                    // the server side don't support totalTopicNum, all data is returned\n                    break;\n                }\n\n                if (!Objects.equals(topicConfigDataVersion, newDataVersion)) {\n                    LOGGER.error(\"dataVersion changed, currentDataVersion: {}, newDataVersion: {}\", topicConfigDataVersion, newDataVersion);\n                    topicConfigDataVersion = newDataVersion;\n                    mappingDataVersion = topicConfigSerializeWrapper.getMappingDataVersion();\n                    topicSeq = 0;\n                    topicConfigTable.clear();\n                    continue;\n                }\n\n                if (topicSeq >= totalTopicNum - 1) {\n                    LOGGER.info(\"get all topic config, totalTopicNum: {}\", totalTopicNum);\n                    break;\n                }\n            } else {\n                throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n            }\n\n        }\n\n        TopicConfigAndMappingSerializeWrapper topicConfigSerializeWrapper = new TopicConfigAndMappingSerializeWrapper();\n        topicConfigSerializeWrapper.setDataVersion(topicConfigDataVersion);\n        topicConfigSerializeWrapper.setTopicConfigTable(topicConfigTable);\n        topicConfigSerializeWrapper.setMappingDataVersion(mappingDataVersion);\n        topicConfigSerializeWrapper.setTopicQueueMappingDetailMap(topicQueueMappingDetailMap);\n        return topicConfigSerializeWrapper;\n    }\n\n    public TimerCheckpoint getTimerCheckPoint(\n        final String addr) throws RemotingConnectException, RemotingSendRequestException,\n        RemotingTimeoutException, InterruptedException, MQBrokerException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TIMER_CHECK_POINT, null);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(true, addr), request, 3000);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return TimerCheckpoint.decode(ByteBuffer.wrap(response.getBody()));\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public TimerMetrics.TimerMetricsSerializeWrapper getTimerMetrics(\n        final String addr) throws RemotingConnectException, RemotingSendRequestException,\n        RemotingTimeoutException, InterruptedException, MQBrokerException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TIMER_METRICS, null);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(true, addr), request, 3000);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return TimerMetrics.TimerMetricsSerializeWrapper.decode(response.getBody(), TimerMetrics.TimerMetricsSerializeWrapper.class);\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public ConsumerOffsetSerializeWrapper getAllConsumerOffset(\n        final String addr) throws InterruptedException, RemotingTimeoutException,\n        RemotingSendRequestException, RemotingConnectException, MQBrokerException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_CONSUMER_OFFSET, null);\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, 3000);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return ConsumerOffsetSerializeWrapper.decode(response.getBody(), ConsumerOffsetSerializeWrapper.class);\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public String getAllDelayOffset(\n        final String addr) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException,\n        RemotingConnectException, MQBrokerException, UnsupportedEncodingException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_DELAY_OFFSET, null);\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, 3000);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return new String(response.getBody(), MixAll.DEFAULT_CHARSET);\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public SubscriptionGroupWrapper getAllSubscriptionGroupConfig(final String addr)\n        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException,\n        RemotingConnectException, MQBrokerException, RemotingCommandException {\n\n        long timeoutMills = getTimeoutMillis();\n        DataVersion currentDataVersion = null;\n        int groupSeq = 0;\n        long beginTime = System.nanoTime();\n        ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroupTable = new ConcurrentHashMap<>();\n        ConcurrentMap<String, ConcurrentMap<String, Integer>> forbiddenTable = new ConcurrentHashMap<>();\n\n        while (true) {\n            long leftTime = timeoutMills - TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - beginTime);\n            if (leftTime < 0) {\n                throw new RemotingTimeoutException(\"invokeSync call timeout\");\n            }\n\n            GetAllSubscriptionGroupRequestHeader requestHeader = new GetAllSubscriptionGroupRequestHeader();\n            requestHeader.setGroupSeq(groupSeq);\n            requestHeader.setMaxGroupNum(getMaxPageSize());\n            requestHeader.setDataVersion(Optional.ofNullable(currentDataVersion)\n                .map(DataVersion::toJson).orElse(StringUtils.EMPTY));\n            LOGGER.info(\"getAllSubscriptionGroup from seq {}, max {}, dataVersion {}\",\n                    groupSeq, requestHeader.getMaxGroupNum(), requestHeader.getDataVersion());\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_SUBSCRIPTIONGROUP_CONFIG, requestHeader);\n            RemotingCommand response = this.remotingClient.invokeSync(addr, request, 30000);\n\n            assert response != null;\n            if (response.getCode() == SUCCESS) {\n                SubscriptionGroupWrapper subscriptionGroupWrapper =\n                    SubscriptionGroupWrapper.decode(response.getBody(), SubscriptionGroupWrapper.class);\n                subscriptionGroupTable.putAll(subscriptionGroupWrapper.getSubscriptionGroupTable());\n                forbiddenTable.putAll(subscriptionGroupWrapper.getForbiddenTable());\n\n                DataVersion newDataVersion = subscriptionGroupWrapper.getDataVersion();\n                if (currentDataVersion == null) {\n                    // fill dataVersion before break the loop to compatible with old version server\n                    currentDataVersion = newDataVersion;\n                }\n\n                groupSeq += subscriptionGroupWrapper.getSubscriptionGroupTable().size();\n\n                GetAllSubscriptionGroupResponseHeader responseHeader =\n                    response.decodeCommandCustomHeader(GetAllSubscriptionGroupResponseHeader.class);\n                Integer totalGroupNum = Optional.ofNullable(responseHeader)\n                    .map(GetAllSubscriptionGroupResponseHeader::getTotalGroupNum).orElse(null);\n\n                if (Objects.isNull(totalGroupNum)) {\n                    // the server side don't support totalGroupNum, all data is returned\n                    break;\n                }\n\n                if (!Objects.equals(currentDataVersion, newDataVersion)) {\n                    LOGGER.error(\"dataVersion changed, currentDataVersion: {}, newDataVersion: {}\",\n                        currentDataVersion, newDataVersion);\n                    currentDataVersion = newDataVersion;\n                    groupSeq = 0;\n                    subscriptionGroupTable.clear();\n                    forbiddenTable.clear();\n                    continue;\n                }\n\n                if (groupSeq >= totalGroupNum - 1) {\n                    LOGGER.info(\"get all subscription group config, totalGroupNum: {}\", totalGroupNum);\n                    break;\n                }\n            } else {\n                throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n            }\n        }\n\n        SubscriptionGroupWrapper allSubscriptionGroup = new SubscriptionGroupWrapper();\n        allSubscriptionGroup.setSubscriptionGroupTable(subscriptionGroupTable);\n        allSubscriptionGroup.setForbiddenTable(forbiddenTable);\n        allSubscriptionGroup.setDataVersion(currentDataVersion);\n        return allSubscriptionGroup;\n    }\n\n    public void registerRPCHook(RPCHook rpcHook) {\n        remotingClient.registerRPCHook(rpcHook);\n    }\n\n    public void clearRPCHook() {\n        remotingClient.clearRPCHook();\n    }\n\n    public long getMaxOffset(final String addr, final String topic, final int queueId, final boolean committed,\n        final boolean isOnlyThisBroker)\n        throws RemotingException, MQBrokerException, InterruptedException {\n        GetMaxOffsetRequestHeader requestHeader = new GetMaxOffsetRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setQueueId(queueId);\n        requestHeader.setCommitted(committed);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_MAX_OFFSET, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, 3000);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                GetMaxOffsetResponseHeader responseHeader = response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class);\n\n                return responseHeader.getOffset();\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public long getMinOffset(final String addr, final String topic, final int queueId, final boolean isOnlyThisBroker)\n        throws RemotingException, MQBrokerException, InterruptedException {\n        GetMinOffsetRequestHeader requestHeader = new GetMinOffsetRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setQueueId(queueId);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_MIN_OFFSET, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, 3000);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                GetMinOffsetResponseHeader responseHeader = response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class);\n\n                return responseHeader.getOffset();\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public void lockBatchMQAsync(\n        final String addr,\n        final LockBatchRequestBody requestBody,\n        final long timeoutMillis,\n        final LockCallback callback) throws RemotingException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, new LockBatchMqRequestHeader());\n\n        request.setBody(requestBody.encode());\n        this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {\n            @Override\n            public void operationComplete(ResponseFuture responseFuture) {\n\n            }\n\n            @Override\n            public void operationSucceed(RemotingCommand response) {\n                if (callback == null) {\n                    return;\n                }\n                if (response.getCode() == ResponseCode.SUCCESS) {\n                    LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(),\n                        LockBatchResponseBody.class);\n                    Set<MessageQueue> messageQueues = responseBody.getLockOKMQSet();\n                    callback.onSuccess(messageQueues);\n                } else {\n                    callback.onException(new MQBrokerException(response.getCode(), response.getRemark()));\n                }\n            }\n\n            @Override\n            public void operationFail(Throwable throwable) {\n                if (callback == null) {\n                    return;\n                }\n                callback.onException(throwable);\n            }\n        });\n    }\n\n    public void unlockBatchMQAsync(\n        final String addr,\n        final UnlockBatchRequestBody requestBody,\n        final long timeoutMillis,\n        final UnlockCallback callback) throws RemotingException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UNLOCK_BATCH_MQ, new UnlockBatchMqRequestHeader());\n\n        request.setBody(requestBody.encode());\n\n        this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {\n            @Override\n            public void operationComplete(ResponseFuture responseFuture) {\n\n            }\n\n            @Override\n            public void operationSucceed(RemotingCommand response) {\n                if (callback == null) {\n                    return;\n                }\n                if (response.getCode() == ResponseCode.SUCCESS) {\n                    callback.onSuccess();\n                } else {\n                    callback.onException(new MQBrokerException(response.getCode(), response.getRemark()));\n                }\n            }\n\n            @Override\n            public void operationFail(Throwable throwable) {\n                if (callback == null) {\n                    return;\n                }\n                callback.onException(throwable);\n            }\n        });\n    }\n\n    public RemotingClient getRemotingClient() {\n        return this.remotingClient;\n    }\n\n    public SendResult sendMessageToSpecificBroker(String brokerAddr, final String brokerName,\n        final MessageExt msg, String group,\n        long timeoutMillis) throws RemotingException, MQBrokerException, InterruptedException {\n\n        RemotingCommand request = buildSendMessageRequest(msg, group);\n        RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis);\n        return this.processSendResponse(brokerName, msg, response);\n    }\n\n    public CompletableFuture<SendResult> sendMessageToSpecificBrokerAsync(String brokerAddr, final String brokerName,\n        final MessageExt msg, String group,\n        long timeoutMillis) {\n        RemotingCommand request = buildSendMessageRequest(msg, group);\n\n        CompletableFuture<SendResult> cf = new CompletableFuture<>();\n        final String msgId = msg.getMsgId();\n        try {\n            this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() {\n                @Override\n                public void operationComplete(ResponseFuture responseFuture) {\n\n                }\n\n                @Override\n                public void operationSucceed(RemotingCommand response) {\n                    try {\n                        SendResult sendResult = processSendResponse(brokerName, msg, response);\n                        cf.complete(sendResult);\n                    } catch (MQBrokerException | RemotingCommandException e) {\n                        LOGGER.error(\"processSendResponse in sendMessageToSpecificBrokerAsync failed, msgId=\" + msgId, e);\n                        cf.completeExceptionally(e);\n                    }\n                }\n\n                @Override\n                public void operationFail(Throwable throwable) {\n                    cf.completeExceptionally(throwable);\n                }\n            });\n        } catch (Throwable t) {\n            LOGGER.error(\"invokeAsync failed in sendMessageToSpecificBrokerAsync, msgId=\" + msgId, t);\n            cf.completeExceptionally(t);\n        }\n        return cf;\n    }\n\n    private static RemotingCommand buildSendMessageRequest(MessageExt msg, String group) {\n        SendMessageRequestHeaderV2 requestHeaderV2 = buildSendMessageRequestHeaderV2(msg, group);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, requestHeaderV2);\n\n        request.setBody(msg.getBody());\n        return request;\n    }\n\n    private static SendMessageRequestHeaderV2 buildSendMessageRequestHeaderV2(MessageExt msg, String group) {\n        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();\n        requestHeader.setProducerGroup(group);\n        requestHeader.setTopic(msg.getTopic());\n        requestHeader.setDefaultTopic(TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC);\n        requestHeader.setDefaultTopicQueueNums(8);\n        requestHeader.setQueueId(msg.getQueueId());\n        requestHeader.setSysFlag(msg.getSysFlag());\n        requestHeader.setBornTimestamp(msg.getBornTimestamp());\n        requestHeader.setFlag(msg.getFlag());\n        requestHeader.setProperties(MessageDecoder.messageProperties2String(msg.getProperties()));\n        requestHeader.setReconsumeTimes(msg.getReconsumeTimes());\n        requestHeader.setBatch(false);\n\n        SendMessageRequestHeaderV2 requestHeaderV2 = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(requestHeader);\n        return requestHeaderV2;\n    }\n\n    private SendResult processSendResponse(\n        final String brokerName,\n        final Message msg,\n        final RemotingCommand response\n    ) throws MQBrokerException, RemotingCommandException {\n        SendStatus sendStatus = null;\n        switch (response.getCode()) {\n            case ResponseCode.FLUSH_DISK_TIMEOUT:\n                sendStatus = SendStatus.FLUSH_DISK_TIMEOUT;\n                break;\n            case ResponseCode.FLUSH_SLAVE_TIMEOUT:\n                sendStatus = SendStatus.FLUSH_SLAVE_TIMEOUT;\n                break;\n            case ResponseCode.SLAVE_NOT_AVAILABLE:\n                sendStatus = SendStatus.SLAVE_NOT_AVAILABLE;\n                break;\n            case ResponseCode.SUCCESS: {\n                sendStatus = SendStatus.SEND_OK;\n                break;\n            }\n            default:\n                break;\n        }\n        if (sendStatus != null) {\n            SendMessageResponseHeader responseHeader = response.decodeCommandCustomHeader(SendMessageResponseHeader.class);\n\n            //If namespace not null , reset Topic without namespace.\n            String topic = msg.getTopic();\n\n            MessageQueue messageQueue = new MessageQueue(topic, brokerName, responseHeader.getQueueId());\n\n            String uniqMsgId = MessageClientIDSetter.getUniqID(msg);\n            if (msg instanceof MessageBatch) {\n                StringBuilder sb = new StringBuilder();\n                for (Message message : (MessageBatch) msg) {\n                    sb.append(sb.length() == 0 ? \"\" : \",\").append(MessageClientIDSetter.getUniqID(message));\n                }\n                uniqMsgId = sb.toString();\n            }\n            SendResult sendResult = new SendResult(sendStatus,\n                uniqMsgId,\n                responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset());\n            sendResult.setTransactionId(responseHeader.getTransactionId());\n            String regionId = response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION);\n            String traceOn = response.getExtFields().get(MessageConst.PROPERTY_TRACE_SWITCH);\n            if (regionId == null || regionId.isEmpty()) {\n                regionId = MixAll.DEFAULT_TRACE_REGION_ID;\n            }\n            if (traceOn != null && traceOn.equals(\"false\")) {\n                sendResult.setTraceOn(false);\n            } else {\n                sendResult.setTraceOn(true);\n            }\n            sendResult.setRegionId(regionId);\n            return sendResult;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public ExecutorService getBrokerOuterExecutor() {\n        return brokerOuterExecutor;\n    }\n\n    public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final long timeoutMillis)\n        throws RemotingException, MQBrokerException, InterruptedException {\n        return getTopicRouteInfoFromNameServer(topic, timeoutMillis, true);\n    }\n\n    public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final long timeoutMillis,\n        boolean allowTopicNotExist) throws MQBrokerException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {\n        GetRouteInfoRequestHeader requestHeader = new GetRouteInfoRequestHeader();\n        requestHeader.setTopic(topic);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.TOPIC_NOT_EXIST: {\n                if (allowTopicNotExist) {\n                    LOGGER.warn(\"get Topic [{}] RouteInfoFromNameServer is not exist value\", topic);\n                }\n\n                break;\n            }\n            case ResponseCode.SUCCESS: {\n                byte[] body = response.getBody();\n                if (body != null) {\n                    return TopicRouteData.decode(body, TopicRouteData.class);\n                }\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public ClusterInfo getBrokerClusterInfo() throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_CLUSTER_INFO, null);\n        RemotingCommand response = this.remotingClient.invokeSync(null, request, 3_000);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return ClusterInfo.decode(response.getBody(), ClusterInfo.class);\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public void forwardRequest(String brokerAddr, RemotingCommand request, long timeoutMillis,\n        InvokeCallback invokeCallback) throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException, RemotingTooMuchRequestException, RemotingConnectException {\n        this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, invokeCallback);\n    }\n\n    public void refreshMetadata() throws Exception {\n        ClusterInfo brokerClusterInfo = getBrokerClusterInfo();\n        clientMetadata.refreshClusterInfo(brokerClusterInfo);\n    }\n\n    public ClientMetadata getClientMetadata() {\n        return clientMetadata;\n    }\n\n    public RpcClient getRpcClient() {\n        return rpcClient;\n    }\n\n    public MessageRequestModeSerializeWrapper getAllMessageRequestMode(\n        final String addr) throws RemotingSendRequestException, RemotingConnectException,\n        MQBrokerException, RemotingTimeoutException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_MESSAGE_REQUEST_MODE, null);\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, 3000);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return MessageRequestModeSerializeWrapper.decode(response.getBody(), MessageRequestModeSerializeWrapper.class);\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public GetMetaDataResponseHeader getControllerMetaData(final String controllerAddress) throws Exception {\n        final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_GET_METADATA_INFO, null);\n        final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000);\n        assert response != null;\n        if (response.getCode() == SUCCESS) {\n            return (GetMetaDataResponseHeader) response.decodeCommandCustomHeader(GetMetaDataResponseHeader.class);\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    /**\n     * Alter syncStateSet\n     */\n    public SyncStateSet alterSyncStateSet(\n        final String controllerAddress,\n        final String brokerName,\n        final Long masterBrokerId, final int masterEpoch,\n        final Set<Long> newSyncStateSet, final int syncStateSetEpoch) throws Exception {\n\n        final AlterSyncStateSetRequestHeader requestHeader = new AlterSyncStateSetRequestHeader(brokerName, masterBrokerId, masterEpoch);\n        final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_ALTER_SYNC_STATE_SET, requestHeader);\n        request.setBody(new SyncStateSet(newSyncStateSet, syncStateSetEpoch).encode());\n        final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000);\n        assert response != null;\n        switch (response.getCode()) {\n            case SUCCESS: {\n                assert response.getBody() != null;\n                return RemotingSerializable.decode(response.getBody(), SyncStateSet.class);\n            }\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    /**\n     * Broker try to elect itself as a master in broker set\n     */\n    public Pair<ElectMasterResponseHeader, Set<Long>> brokerElect(String controllerAddress, String clusterName,\n        String brokerName,\n        Long brokerId) throws Exception {\n\n        final ElectMasterRequestHeader requestHeader = ElectMasterRequestHeader.ofBrokerTrigger(clusterName, brokerName, brokerId);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_ELECT_MASTER, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000);\n        assert response != null;\n        switch (response.getCode()) {\n            // Only record success response.\n            case CONTROLLER_MASTER_STILL_EXIST:\n            case SUCCESS:\n                final ElectMasterResponseHeader responseHeader = response.decodeCommandCustomHeader(ElectMasterResponseHeader.class);\n                final ElectMasterResponseBody responseBody = RemotingSerializable.decode(response.getBody(), ElectMasterResponseBody.class);\n                return new Pair<>(responseHeader, responseBody.getSyncStateSet());\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public GetNextBrokerIdResponseHeader getNextBrokerId(final String clusterName, final String brokerName,\n        final String controllerAddress) throws Exception {\n        final GetNextBrokerIdRequestHeader requestHeader = new GetNextBrokerIdRequestHeader(clusterName, brokerName);\n        final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_GET_NEXT_BROKER_ID, requestHeader);\n        final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000);\n        assert response != null;\n        if (response.getCode() == SUCCESS) {\n            return response.decodeCommandCustomHeader(GetNextBrokerIdResponseHeader.class);\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public ApplyBrokerIdResponseHeader applyBrokerId(final String clusterName, final String brokerName,\n        final Long brokerId, final String registerCheckCode, final String controllerAddress) throws Exception {\n        final ApplyBrokerIdRequestHeader requestHeader = new ApplyBrokerIdRequestHeader(clusterName, brokerName, brokerId, registerCheckCode);\n        final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_APPLY_BROKER_ID, requestHeader);\n        final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000);\n        assert response != null;\n        if (response.getCode() == SUCCESS) {\n            return response.decodeCommandCustomHeader(ApplyBrokerIdResponseHeader.class);\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public Pair<RegisterBrokerToControllerResponseHeader, Set<Long>> registerBrokerToController(\n        final String clusterName, final String brokerName, final Long brokerId, final String brokerAddress,\n        final String controllerAddress) throws Exception {\n        final RegisterBrokerToControllerRequestHeader requestHeader = new RegisterBrokerToControllerRequestHeader(clusterName, brokerName, brokerId, brokerAddress);\n        final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_REGISTER_BROKER, requestHeader);\n        final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000);\n        assert response != null;\n        if (response.getCode() == SUCCESS) {\n            RegisterBrokerToControllerResponseHeader responseHeader = response.decodeCommandCustomHeader(RegisterBrokerToControllerResponseHeader.class);\n            Set<Long> syncStateSet = RemotingSerializable.decode(response.getBody(), SyncStateSet.class).getSyncStateSet();\n            return new Pair<>(responseHeader, syncStateSet);\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    /**\n     * Get broker replica info\n     */\n    public Pair<GetReplicaInfoResponseHeader, SyncStateSet> getReplicaInfo(final String controllerAddress,\n        final String brokerName) throws Exception {\n        final GetReplicaInfoRequestHeader requestHeader = new GetReplicaInfoRequestHeader(brokerName);\n        final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_GET_REPLICA_INFO, requestHeader);\n        final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000);\n        assert response != null;\n        switch (response.getCode()) {\n            case SUCCESS: {\n                final GetReplicaInfoResponseHeader header = response.decodeCommandCustomHeader(GetReplicaInfoResponseHeader.class);\n                assert response.getBody() != null;\n                final SyncStateSet stateSet = RemotingSerializable.decode(response.getBody(), SyncStateSet.class);\n                return new Pair<>(header, stateSet);\n            }\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    /**\n     * Send heartbeat to controller\n     */\n    public void sendHeartbeatToController(final String controllerAddress,\n        final String clusterName,\n        final String brokerAddr,\n        final String brokerName,\n        final Long brokerId,\n        final int sendHeartBeatTimeoutMills,\n        final boolean isInBrokerContainer,\n        final int epoch,\n        final long maxOffset,\n        final long confirmOffset,\n        final long controllerHeartBeatTimeoutMills,\n        final int electionPriority) {\n        if (StringUtils.isEmpty(controllerAddress)) {\n            return;\n        }\n\n        final BrokerHeartbeatRequestHeader requestHeader = new BrokerHeartbeatRequestHeader();\n        requestHeader.setClusterName(clusterName);\n        requestHeader.setBrokerAddr(brokerAddr);\n        requestHeader.setBrokerName(brokerName);\n        requestHeader.setEpoch(epoch);\n        requestHeader.setMaxOffset(maxOffset);\n        requestHeader.setConfirmOffset(confirmOffset);\n        requestHeader.setHeartbeatTimeoutMills(controllerHeartBeatTimeoutMills);\n        requestHeader.setElectionPriority(electionPriority);\n        requestHeader.setBrokerId(brokerId);\n        brokerOuterExecutor.execute(new Runnable() {\n            @Override\n            public void run() {\n                RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.BROKER_HEARTBEAT, requestHeader);\n\n                try {\n                    BrokerOuterAPI.this.remotingClient.invokeOneway(controllerAddress, request, sendHeartBeatTimeoutMills);\n                } catch (Exception e) {\n                    LOGGER.error(\"Error happen when send heartbeat to controller {}\", controllerAddress, e);\n                }\n            }\n        });\n    }\n\n    // Triple<PullResult, info, needRetry>, should check info and retry if and only if PullResult is null\n    public CompletableFuture<Triple<PullResult, String, Boolean>> pullMessageFromSpecificBrokerAsync(String brokerName, String brokerAddr,\n        String consumerGroup, String topic, int queueId, long offset,\n        int maxNums, long timeoutMillis) throws RemotingException, InterruptedException {\n        PullMessageRequestHeader requestHeader = new PullMessageRequestHeader();\n        requestHeader.setConsumerGroup(consumerGroup);\n        requestHeader.setTopic(topic);\n        requestHeader.setQueueId(queueId);\n        requestHeader.setQueueOffset(offset);\n        requestHeader.setMaxMsgNums(maxNums);\n        requestHeader.setSysFlag(PullSysFlag.buildSysFlag(false, false, true, false));\n        requestHeader.setCommitOffset(0L);\n        requestHeader.setSuspendTimeoutMillis(0L);\n        requestHeader.setSubscription(SubscriptionData.SUB_ALL);\n        requestHeader.setSubVersion(System.currentTimeMillis());\n        requestHeader.setMaxMsgBytes(Integer.MAX_VALUE);\n        requestHeader.setExpressionType(ExpressionType.TAG);\n        requestHeader.setBrokerName(brokerName);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, requestHeader);\n        CompletableFuture<Triple<PullResult, String, Boolean>> pullResultFuture = new CompletableFuture<>();\n        this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() {\n            @Override\n            public void operationComplete(ResponseFuture responseFuture) {\n\n            }\n\n            @Override\n            public void operationSucceed(RemotingCommand response) {\n                try {\n                    PullResultExt pullResultExt = processPullResponse(response, brokerAddr);\n                    processPullResult(pullResultExt, brokerName, queueId);\n                    pullResultFuture.complete(Triple.of(pullResultExt, pullResultExt.getPullStatus().name(), false)); // found or not found really, so no retry\n                } catch (Exception e) {\n                    // retry when NO_PERMISSION, SUBSCRIPTION_GROUP_NOT_EXIST etc. even when TOPIC_NOT_EXIST\n                    pullResultFuture.complete(Triple.of(null, \"Response Code:\" + response.getCode(), true));\n                }\n            }\n\n            @Override\n            public void operationFail(Throwable throwable) {\n                pullResultFuture.complete(Triple.of(null, throwable.getMessage(), true));\n            }\n        });\n        return pullResultFuture;\n    }\n\n    private PullResultExt processPullResponse(\n        final RemotingCommand response,\n        final String addr) throws MQBrokerException, RemotingCommandException {\n        PullStatus pullStatus = PullStatus.NO_NEW_MSG;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS:\n                pullStatus = PullStatus.FOUND;\n                break;\n            case ResponseCode.PULL_NOT_FOUND:\n                pullStatus = PullStatus.NO_NEW_MSG;\n                break;\n            case ResponseCode.PULL_RETRY_IMMEDIATELY:\n                pullStatus = PullStatus.NO_MATCHED_MSG;\n                break;\n            case ResponseCode.PULL_OFFSET_MOVED:\n                pullStatus = PullStatus.OFFSET_ILLEGAL;\n                break;\n\n            default:\n                throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n        }\n\n        PullMessageResponseHeader responseHeader = response.decodeCommandCustomHeader(PullMessageResponseHeader.class);\n\n        return new PullResultExt(pullStatus, responseHeader.getNextBeginOffset(), responseHeader.getMinOffset(),\n            responseHeader.getMaxOffset(), null, responseHeader.getSuggestWhichBrokerId(), response.getBody(), responseHeader.getOffsetDelta());\n\n    }\n\n    private PullResult processPullResult(final PullResultExt pullResult, String brokerName, int queueId) {\n\n        if (PullStatus.FOUND == pullResult.getPullStatus()) {\n            ByteBuffer byteBuffer = ByteBuffer.wrap(pullResult.getMessageBinary());\n            List<MessageExt> msgList = MessageDecoder.decodesBatch(\n                byteBuffer,\n                true,\n                true,\n                true\n            );\n\n            // Currently batch messages are not supported\n            for (MessageExt msg : msgList) {\n                String traFlag = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED);\n                if (Boolean.parseBoolean(traFlag)) {\n                    msg.setTransactionId(msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));\n                }\n                MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MIN_OFFSET,\n                    Long.toString(pullResult.getMinOffset()));\n                MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MAX_OFFSET,\n                    Long.toString(pullResult.getMaxOffset()));\n                msg.setBrokerName(brokerName);\n                msg.setQueueId(queueId);\n                if (pullResult.getOffsetDelta() != null) {\n                    msg.setQueueOffset(pullResult.getOffsetDelta() + msg.getQueueOffset());\n                }\n            }\n\n            pullResult.setMsgFoundList(msgList);\n        }\n\n        pullResult.setMessageBinary(null);\n\n        return pullResult;\n    }\n\n    private int getMaxPageSize() {\n        return 2000;\n    }\n\n    private long getTimeoutMillis() {\n        return TimeUnit.SECONDS.toMillis(60);\n    }\n\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/pagecache/ManyMessageTransfer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.pagecache;\n\nimport io.netty.channel.FileRegion;\nimport io.netty.util.AbstractReferenceCounted;\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport java.nio.channels.WritableByteChannel;\nimport java.util.List;\nimport org.apache.rocketmq.store.GetMessageResult;\n\npublic class ManyMessageTransfer extends AbstractReferenceCounted implements FileRegion {\n    private final ByteBuffer byteBufferHeader;\n    private final GetMessageResult getMessageResult;\n\n    /**\n     * Bytes which were transferred already.\n     */\n    private long transferred;\n\n    public ManyMessageTransfer(ByteBuffer byteBufferHeader, GetMessageResult getMessageResult) {\n        this.byteBufferHeader = byteBufferHeader;\n        this.getMessageResult = getMessageResult;\n    }\n\n    @Override\n    public long position() {\n        int pos = byteBufferHeader.position();\n        List<ByteBuffer> messageBufferList = this.getMessageResult.getMessageBufferList();\n        for (ByteBuffer bb : messageBufferList) {\n            pos += bb.position();\n        }\n        return pos;\n    }\n\n    @Override\n    public long transfered() {\n        return transferred;\n    }\n\n    @Override\n    public long transferred() {\n        return transferred;\n    }\n\n    @Override\n    public long count() {\n        return byteBufferHeader.limit() + this.getMessageResult.getBufferTotalSize();\n    }\n\n    @Override\n    public long transferTo(WritableByteChannel target, long position) throws IOException {\n        if (this.byteBufferHeader.hasRemaining()) {\n            transferred += target.write(this.byteBufferHeader);\n            return transferred;\n        } else {\n            List<ByteBuffer> messageBufferList = this.getMessageResult.getMessageBufferList();\n            for (ByteBuffer bb : messageBufferList) {\n                if (bb.hasRemaining()) {\n                    transferred += target.write(bb);\n                    return transferred;\n                }\n            }\n        }\n\n        return 0;\n    }\n\n    @Override\n    public FileRegion retain() {\n        super.retain();\n        return this;\n    }\n\n    @Override\n    public FileRegion retain(int increment) {\n        super.retain(increment);\n        return this;\n    }\n\n    @Override\n    public FileRegion touch() {\n        return this;\n    }\n\n    @Override\n    public FileRegion touch(Object hint) {\n        return this;\n    }\n\n    public void close() {\n        this.deallocate();\n    }\n\n    @Override\n    protected void deallocate() {\n        this.getMessageResult.release();\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/pagecache/OneMessageTransfer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.pagecache;\n\nimport io.netty.channel.FileRegion;\nimport io.netty.util.AbstractReferenceCounted;\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport java.nio.channels.WritableByteChannel;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\n\npublic class OneMessageTransfer extends AbstractReferenceCounted implements FileRegion {\n    private final ByteBuffer byteBufferHeader;\n    private final SelectMappedBufferResult selectMappedBufferResult;\n\n    /**\n     * Bytes which were transferred already.\n     */\n    private long transferred;\n\n    public OneMessageTransfer(ByteBuffer byteBufferHeader, SelectMappedBufferResult selectMappedBufferResult) {\n        this.byteBufferHeader = byteBufferHeader;\n        this.selectMappedBufferResult = selectMappedBufferResult;\n    }\n\n    @Override\n    public long position() {\n        return this.byteBufferHeader.position() + this.selectMappedBufferResult.getByteBuffer().position();\n    }\n\n    @Override\n    public long transfered() {\n        return transferred;\n    }\n\n    @Override\n    public long transferred() {\n        return transferred;\n    }\n\n    @Override\n    public long count() {\n        return this.byteBufferHeader.limit() + this.selectMappedBufferResult.getSize();\n    }\n\n    @Override\n    public long transferTo(WritableByteChannel target, long position) throws IOException {\n        if (this.byteBufferHeader.hasRemaining()) {\n            transferred += target.write(this.byteBufferHeader);\n            return transferred;\n        } else if (this.selectMappedBufferResult.getByteBuffer().hasRemaining()) {\n            transferred += target.write(this.selectMappedBufferResult.getByteBuffer());\n            return transferred;\n        }\n\n        return 0;\n    }\n\n\n    @Override\n    public FileRegion retain() {\n        super.retain();\n        return this;\n    }\n\n    @Override\n    public FileRegion retain(int increment) {\n        super.retain(increment);\n        return this;\n    }\n\n    @Override\n    public FileRegion touch() {\n        return this;\n    }\n\n    @Override\n    public FileRegion touch(Object hint) {\n        return this;\n    }\n\n    public void close() {\n        this.deallocate();\n    }\n\n    @Override\n    protected void deallocate() {\n        this.selectMappedBufferResult.release();\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/pagecache/QueryMessageTransfer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.pagecache;\n\nimport io.netty.channel.FileRegion;\nimport io.netty.util.AbstractReferenceCounted;\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport java.nio.channels.WritableByteChannel;\nimport java.util.List;\nimport org.apache.rocketmq.store.QueryMessageResult;\n\npublic class QueryMessageTransfer extends AbstractReferenceCounted implements FileRegion {\n    private final ByteBuffer byteBufferHeader;\n    private final QueryMessageResult queryMessageResult;\n\n    /**\n     * Bytes which were transferred already.\n     */\n    private long transferred;\n\n    public QueryMessageTransfer(ByteBuffer byteBufferHeader, QueryMessageResult queryMessageResult) {\n        this.byteBufferHeader = byteBufferHeader;\n        this.queryMessageResult = queryMessageResult;\n    }\n\n    @Override\n    public long position() {\n        int pos = byteBufferHeader.position();\n        List<ByteBuffer> messageBufferList = this.queryMessageResult.getMessageBufferList();\n        for (ByteBuffer bb : messageBufferList) {\n            pos += bb.position();\n        }\n        return pos;\n    }\n\n    @Override\n    public long transfered() {\n        return transferred;\n    }\n\n    @Override\n    public long transferred() {\n        return transferred;\n    }\n\n    @Override\n    public long count() {\n        return byteBufferHeader.limit() + this.queryMessageResult.getBufferTotalSize();\n    }\n\n    @Override\n    public long transferTo(WritableByteChannel target, long position) throws IOException {\n        if (this.byteBufferHeader.hasRemaining()) {\n            transferred += target.write(this.byteBufferHeader);\n            return transferred;\n        } else {\n            List<ByteBuffer> messageBufferList = this.queryMessageResult.getMessageBufferList();\n            for (ByteBuffer bb : messageBufferList) {\n                if (bb.hasRemaining()) {\n                    transferred += target.write(bb);\n                    return transferred;\n                }\n            }\n        }\n\n        return 0;\n    }\n\n    @Override\n    public FileRegion retain() {\n        super.retain();\n        return this;\n    }\n\n    @Override\n    public FileRegion retain(int increment) {\n        super.retain(increment);\n        return this;\n    }\n\n    @Override\n    public FileRegion touch() {\n        return this;\n    }\n\n    @Override\n    public FileRegion touch(Object hint) {\n        return this;\n    }\n\n    public void close() {\n        this.deallocate();\n    }\n\n    @Override\n    protected void deallocate() {\n        this.queryMessageResult.release();\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/plugin/BrokerAttachedPlugin.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.plugin;\n\nimport java.util.Map;\n\npublic interface BrokerAttachedPlugin {\n\n    /**\n     * Get plugin name\n     *\n     * @return plugin name\n     */\n    String pluginName();\n\n    /**\n     * Load broker attached plugin.\n     *\n     * @return load success or failed\n     */\n    boolean load();\n\n    /**\n     * Start broker attached plugin.\n     */\n    void start();\n\n    /**\n     * Shutdown broker attached plugin.\n     */\n    void shutdown();\n\n    /**\n     * Sync metadata from master.\n     */\n    void syncMetadata();\n\n    /**\n     * Sync metadata reverse from slave\n     *\n     * @param brokerAddr\n     */\n    void syncMetadataReverse(String brokerAddr) throws Exception;\n\n    /**\n     * Some plugin need build runningInfo when prepare runtime info.\n     *\n     * @param runtimeInfo\n     */\n    void buildRuntimeInfo(Map<String, String> runtimeInfo);\n\n    /**\n     * Some plugin need do something when status changed. For example, brokerRole change to master or slave.\n     *\n     * @param shouldStart\n     */\n    void statusChanged(boolean shouldStart);\n\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/plugin/PullMessageResultHandler.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.plugin;\n\nimport io.netty.channel.Channel;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingContext;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.MessageFilter;\n\npublic interface PullMessageResultHandler {\n\n    /**\n     * Handle result of get message from store.\n     *\n     * @param getMessageResult store result\n     * @param request request\n     * @param requestHeader request header\n     * @param channel channel\n     * @param subscriptionData sub data\n     * @param subscriptionGroupConfig sub config\n     * @param brokerAllowSuspend brokerAllowSuspend\n     * @param messageFilter store message filter\n     * @param response response\n     * @return response or null\n     */\n    RemotingCommand handle(final GetMessageResult getMessageResult,\n                           final RemotingCommand request,\n                           final PullMessageRequestHeader requestHeader,\n                           final Channel channel,\n                           final SubscriptionData subscriptionData,\n                           final SubscriptionGroupConfig subscriptionGroupConfig,\n                           final boolean brokerAllowSuspend,\n                           final MessageFilter messageFilter,\n                           final RemotingCommand response,\n                           final TopicQueueMappingContext mappingContext,\n                           final long beginTimeMills);\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerCache.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.pop;\n\nimport java.util.ArrayList;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.ConcurrentSkipListMap;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.function.Consumer;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.ConcurrentHashMapUtils;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\npublic class PopConsumerCache extends ServiceThread {\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);\n    private static final long OFFSET_NOT_EXIST = -1L;\n\n    private final BrokerController brokerController;\n    private final PopConsumerKVStore consumerRecordStore;\n    private final PopConsumerLockService consumerLockService;\n    private final Consumer<PopConsumerRecord> reviveConsumer;\n\n    private final AtomicInteger estimateCacheSize;\n    private final ConcurrentMap<String, ConsumerRecords> consumerRecordTable;\n\n    public PopConsumerCache(BrokerController brokerController, PopConsumerKVStore consumerRecordStore,\n        PopConsumerLockService popConsumerLockService, Consumer<PopConsumerRecord> reviveConsumer) {\n\n        this.reviveConsumer = reviveConsumer;\n        this.brokerController = brokerController;\n        this.consumerRecordStore = consumerRecordStore;\n        this.consumerLockService = popConsumerLockService;\n        this.estimateCacheSize = new AtomicInteger();\n        this.consumerRecordTable = new ConcurrentHashMap<>();\n    }\n\n    public String getKey(String groupId, String topicId, int queueId) {\n        return groupId + \"@\" + topicId + \"@\" + queueId;\n    }\n\n    public String getKey(PopConsumerRecord consumerRecord) {\n        return consumerRecord.getGroupId() + \"@\" + consumerRecord.getTopicId() + \"@\" + consumerRecord.getQueueId();\n    }\n\n    public int getCacheKeySize() {\n        return this.consumerRecordTable.size();\n    }\n\n    public int getCacheSize() {\n        return this.estimateCacheSize.intValue();\n    }\n\n    public boolean isCacheFull() {\n        return this.estimateCacheSize.intValue() > brokerController.getBrokerConfig().getPopCkMaxBufferSize();\n    }\n\n    public long getMinOffsetInCache(String groupId, String topicId, int queueId) {\n        ConsumerRecords consumerRecords = consumerRecordTable.get(this.getKey(groupId, topicId, queueId));\n        return consumerRecords != null ? consumerRecords.getMinOffsetInBuffer() : OFFSET_NOT_EXIST;\n    }\n\n    public long getPopInFlightMessageCount(String groupId, String topicId, int queueId) {\n        ConsumerRecords consumerRecords = consumerRecordTable.get(this.getKey(groupId, topicId, queueId));\n        return consumerRecords != null ? consumerRecords.getInFlightRecordCount() : 0L;\n    }\n\n    public void writeRecords(List<PopConsumerRecord> consumerRecordList) {\n        this.estimateCacheSize.addAndGet(consumerRecordList.size());\n        consumerRecordList.forEach(consumerRecord -> {\n            ConsumerRecords consumerRecords = ConcurrentHashMapUtils.computeIfAbsent(consumerRecordTable,\n                this.getKey(consumerRecord), k -> new ConsumerRecords(brokerController.getBrokerConfig(),\n                    consumerRecord.getGroupId(), consumerRecord.getTopicId(), consumerRecord.getQueueId()));\n            assert consumerRecords != null;\n            consumerRecords.write(consumerRecord);\n        });\n    }\n\n    /**\n     * Remove the record from the input list then return the content that has not been deleted\n     */\n    public List<PopConsumerRecord> deleteRecords(List<PopConsumerRecord> consumerRecordList) {\n        int total = consumerRecordList.size();\n        List<PopConsumerRecord> remain = new ArrayList<>();\n        consumerRecordList.forEach(consumerRecord -> {\n            ConsumerRecords consumerRecords = consumerRecordTable.get(this.getKey(consumerRecord));\n            if (consumerRecords == null || !consumerRecords.delete(consumerRecord)) {\n                remain.add(consumerRecord);\n            }\n        });\n        this.estimateCacheSize.addAndGet(remain.size() - total);\n        return remain;\n    }\n\n    public int cleanupRecords(Consumer<PopConsumerRecord> consumer) {\n        int remain = 0;\n        Iterator<Map.Entry<String, ConsumerRecords>> iterator = consumerRecordTable.entrySet().iterator();\n        while (iterator.hasNext()) {\n            // revive or write record to store\n            ConsumerRecords records = iterator.next().getValue();\n            boolean timeout = consumerLockService.isLockTimeout(\n                records.getGroupId(), records.getTopicId());\n\n            if (timeout) {\n                records.stageExpiredRecords(Long.MAX_VALUE);\n                List<PopConsumerRecord> writeConsumerRecords =\n                    new ArrayList<>(records.getRemoveTreeMap().values());\n                if (!writeConsumerRecords.isEmpty()) {\n                    consumerRecordStore.writeRecords(writeConsumerRecords);\n                }\n                records.clearStagedRecords();\n                log.info(\"PopConsumerOffline, so clean expire records, groupId={}, topic={}, queueId={}, records={}\",\n                    records.getGroupId(), records.getTopicId(), records.getQueueId(), records.getInFlightRecordCount());\n                iterator.remove();\n                continue;\n            }\n\n            long currentTime = System.currentTimeMillis();\n            records.stageExpiredRecords(currentTime);\n            List<PopConsumerRecord> writeConsumerRecords = new ArrayList<>();\n            records.getRemoveTreeMap().values().forEach(record -> {\n                if (record.getVisibilityTimeout() <= currentTime) {\n                    consumer.accept(record);\n                } else {\n                    writeConsumerRecords.add(record);\n                }\n            });\n\n            // write to store and handle it later\n            consumerRecordStore.writeRecords(writeConsumerRecords);\n            records.clearStagedRecords();\n\n            // commit min offset in buffer to offset store\n            long offset = records.getMinOffsetInBuffer();\n            if (offset > OFFSET_NOT_EXIST) {\n                this.commitOffset(\"PopConsumerCache\",\n                    records.getGroupId(), records.getTopicId(), records.getQueueId(), offset);\n            }\n\n            remain += records.getInFlightRecordCount();\n        }\n        return remain;\n    }\n\n    public void commitOffset(String clientHost, String groupId, String topicId, int queueId, long offset) {\n        if (!consumerLockService.tryLock(groupId, topicId)) {\n            return;\n        }\n        try {\n            ConsumerOffsetManager consumerOffsetManager = brokerController.getConsumerOffsetManager();\n            long commit = consumerOffsetManager.queryOffset(groupId, topicId, queueId);\n            if (commit != OFFSET_NOT_EXIST && offset < commit) {\n                log.info(\"PopConsumerCache, consumer offset less than store, \" +\n                    \"groupId={}, topicId={}, queueId={}, offset={}\", groupId, topicId, queueId, offset);\n            }\n            consumerOffsetManager.commitOffset(clientHost, groupId, topicId, queueId, offset);\n        } finally {\n            consumerLockService.unlock(groupId, topicId);\n        }\n    }\n\n    public void removeRecords(String groupId, String topicId, int queueId) {\n        this.consumerRecordTable.remove(this.getKey(groupId, topicId, queueId));\n    }\n\n    @Override\n    public String getServiceName() {\n        return PopConsumerCache.class.getSimpleName();\n    }\n\n    @Override\n    public void run() {\n        while (!this.isStopped()) {\n            try {\n                this.waitForRunning(TimeUnit.SECONDS.toMillis(1));\n                int cacheSize = this.cleanupRecords(reviveConsumer);\n                this.estimateCacheSize.set(cacheSize);\n            } catch (Exception e) {\n                log.error(\"PopConsumerCacheService revive error\", e);\n            }\n        }\n    }\n\n    protected static class ConsumerRecords {\n\n        private final String groupId;\n        private final String topicId;\n        private final int queueId;\n        private final BrokerConfig brokerConfig;\n        private final ConcurrentSkipListMap<Long /* offset */, PopConsumerRecord> removeTreeMap;\n        private final ConcurrentSkipListMap<Long /* offset */, PopConsumerRecord> recordTreeMap;\n\n        public ConsumerRecords(BrokerConfig brokerConfig, String groupId, String topicId, int queueId) {\n            this.groupId = groupId;\n            this.topicId = topicId;\n            this.queueId = queueId;\n            this.brokerConfig = brokerConfig;\n            this.removeTreeMap = new ConcurrentSkipListMap<>();\n            this.recordTreeMap = new ConcurrentSkipListMap<>();\n        }\n\n        public void write(PopConsumerRecord record) {\n            recordTreeMap.put(record.getOffset(), record);\n        }\n\n        public boolean delete(PopConsumerRecord record) {\n            return recordTreeMap.remove(record.getOffset()) != null;\n        }\n\n        public long getMinOffsetInBuffer() {\n            Map.Entry<Long, PopConsumerRecord> entry = removeTreeMap.firstEntry();\n            if (entry != null) {\n                return entry.getKey();\n            }\n            entry = recordTreeMap.firstEntry();\n            return entry != null ? entry.getKey() : OFFSET_NOT_EXIST;\n        }\n\n        public int getInFlightRecordCount() {\n            return removeTreeMap.size() + recordTreeMap.size();\n        }\n\n        public void stageExpiredRecords(long currentTime) {\n            Iterator<Map.Entry<Long, PopConsumerRecord>>\n                iterator = recordTreeMap.entrySet().iterator();\n\n            // refer: org.apache.rocketmq.broker.processor.PopBufferMergeService.scan\n            while (iterator.hasNext()) {\n                Map.Entry<Long, PopConsumerRecord> entry = iterator.next();\n                if (entry.getValue().getVisibilityTimeout() <= currentTime ||\n                    entry.getValue().getPopTime() + brokerConfig.getPopCkStayBufferTime() <= currentTime) {\n                    removeTreeMap.put(entry.getKey(), entry.getValue());\n                    iterator.remove();\n                }\n            }\n        }\n\n        public void clearStagedRecords() {\n            removeTreeMap.clear();\n        }\n\n        public ConcurrentSkipListMap<Long, PopConsumerRecord> getRemoveTreeMap() {\n            return removeTreeMap;\n        }\n\n        public String getGroupId() {\n            return groupId;\n        }\n\n        public String getTopicId() {\n            return topicId;\n        }\n\n        public int getQueueId() {\n            return queueId;\n        }\n\n        @Override\n        public String toString() {\n            return \"ConsumerRecords{\" +\n                \"topicId=\" + topicId +\n                \", groupId=\" + groupId +\n                \", queueId=\" + queueId +\n                \", recordTreeMap=\" + recordTreeMap.size() +\n                '}';\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.pop;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\n\npublic class PopConsumerContext {\n\n    private final String clientHost;\n\n    private final long popTime;\n\n    private final long invisibleTime;\n\n    private final String groupId;\n\n    private final boolean fifo;\n\n    private final int initMode;\n\n    private final String attemptId;\n\n    private final AtomicLong restCount;\n\n    private final StringBuilder startOffsetInfo;\n\n    private final StringBuilder msgOffsetInfo;\n\n    private final StringBuilder orderCountInfo;\n\n    private List<GetMessageResult> getMessageResultList;\n\n    private List<PopConsumerRecord> popConsumerRecordList;\n\n    public PopConsumerContext(String clientHost,\n        long popTime, long invisibleTime, String groupId, boolean fifo, int initMode, String attemptId) {\n\n        this.clientHost = clientHost;\n        this.popTime = popTime;\n        this.invisibleTime = invisibleTime;\n        this.groupId = groupId;\n        this.fifo = fifo;\n        this.initMode = initMode;\n        this.attemptId = attemptId;\n        this.restCount = new AtomicLong(0);\n        this.startOffsetInfo = new StringBuilder();\n        this.msgOffsetInfo = new StringBuilder();\n        this.orderCountInfo = new StringBuilder();\n    }\n\n    public boolean isFound() {\n        return getMessageResultList != null && !getMessageResultList.isEmpty();\n    }\n\n    // offset is consumer last request offset\n    public void addGetMessageResult(GetMessageResult result,\n        String topicId, int queueId, PopConsumerRecord.RetryType retryType, long offset) {\n\n        if (result.getStatus() != GetMessageStatus.FOUND || result.getMessageQueueOffset().isEmpty()) {\n            return;\n        }\n\n        if (this.getMessageResultList == null) {\n            this.getMessageResultList = new ArrayList<>();\n        }\n\n        if (this.popConsumerRecordList == null) {\n            this.popConsumerRecordList = new ArrayList<>();\n        }\n\n        this.getMessageResultList.add(result);\n        this.addRestCount(result.getMaxOffset() - result.getNextBeginOffset());\n\n        for (int i = 0; i < result.getMessageQueueOffset().size(); i++) {\n            this.popConsumerRecordList.add(new PopConsumerRecord(popTime, groupId, topicId, queueId,\n                retryType.getCode(), invisibleTime, result.getMessageQueueOffset().get(i), attemptId));\n        }\n\n        ExtraInfoUtil.buildStartOffsetInfo(startOffsetInfo, topicId, queueId, offset);\n        ExtraInfoUtil.buildMsgOffsetInfo(msgOffsetInfo, topicId, queueId, result.getMessageQueueOffset());\n    }\n\n    public String getClientHost() {\n        return clientHost;\n    }\n\n    public String getGroupId() {\n        return groupId;\n    }\n\n    public void addRestCount(long delta) {\n        this.restCount.addAndGet(delta);\n    }\n\n    public long getRestCount() {\n        return restCount.get();\n    }\n\n    public long getPopTime() {\n        return popTime;\n    }\n\n    public boolean isFifo() {\n        return fifo;\n    }\n\n    public int getInitMode() {\n        return initMode;\n    }\n\n    public long getInvisibleTime() {\n        return invisibleTime;\n    }\n\n    public String getAttemptId() {\n        return attemptId;\n    }\n\n    public int getMessageCount() {\n        return getMessageResultList != null ?\n            getMessageResultList.stream().mapToInt(GetMessageResult::getMessageCount).sum() : 0;\n    }\n\n    public String getStartOffsetInfo() {\n        return startOffsetInfo.toString();\n    }\n\n    public String getMsgOffsetInfo() {\n        return msgOffsetInfo.toString();\n    }\n\n    public StringBuilder getOrderCountInfoBuilder() {\n        return orderCountInfo;\n    }\n\n    public String getOrderCountInfo() {\n        return orderCountInfo.toString();\n    }\n\n    public List<GetMessageResult> getGetMessageResultList() {\n        return getMessageResultList;\n    }\n\n    public List<PopConsumerRecord> getPopConsumerRecordList() {\n        return popConsumerRecordList;\n    }\n\n    @Override\n    public String toString() {\n        return \"PopConsumerContext{\" +\n            \"clientHost=\" + clientHost +\n            \", popTime=\" + popTime +\n            \", invisibleTime=\" + invisibleTime +\n            \", groupId=\" + groupId +\n            \", isFifo=\" + fifo +\n            \", attemptId=\" + attemptId +\n            \", restCount=\" + restCount +\n            \", startOffsetInfo=\" + startOffsetInfo +\n            \", msgOffsetInfo=\" + msgOffsetInfo +\n            \", orderCountInfo=\" + orderCountInfo +\n            \", getMessageResultList=\" + (getMessageResultList != null ? getMessageResultList.size() : 0) +\n            \", popConsumerRecordList=\" + (popConsumerRecordList != null ? popConsumerRecordList.size() : 0) +\n            '}';\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerKVStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.pop;\n\nimport java.util.List;\n\npublic interface PopConsumerKVStore {\n\n    /**\n     * Starts the storage service.\n     */\n    boolean start();\n\n    /**\n     * Shutdown the storage service.\n     */\n    boolean shutdown();\n\n    /**\n     * Gets the file path of the storage.\n     * @return The file path of the storage.\n     */\n    String getFilePath();\n\n    /**\n     * Writes a list of consumer records to the storage.\n     * @param consumerRecordList The list of consumer records to be written.\n     */\n    void writeRecords(List<PopConsumerRecord> consumerRecordList);\n\n    /**\n     * Deletes a list of consumer records from the storage.\n     * @param consumerRecordList The list of consumer records to be deleted.\n     */\n    void deleteRecords(List<PopConsumerRecord> consumerRecordList);\n\n    /**\n     * Scans and returns a list of expired consumer records within the specified time range.\n     * @param lowerTime The start time (inclusive) of the time range to search, in milliseconds.\n     * @param upperTime The end time (exclusive) of the time range to search, in milliseconds.\n     * @param maxCount The maximum number of records to return.\n     *                 Even if more records match the criteria, only this many will be returned.\n     * @return A list of expired consumer records within the specified time range.\n     *         If no matching records are found, an empty list is returned.\n     */\n    List<PopConsumerRecord> scanExpiredRecords(long lowerTime, long upperTime, int maxCount);\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerLockService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.pop;\n\nimport java.util.Iterator;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.PopAckConstants;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.ConcurrentHashMapUtils;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\npublic class PopConsumerLockService {\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);\n\n    private final long timeout;\n    private final ConcurrentMap<String /* groupId@topicId */, TimedLock> lockTable;\n\n    public PopConsumerLockService(long timeout) {\n        this.timeout = timeout;\n        this.lockTable = new ConcurrentHashMap<>();\n    }\n\n    public boolean tryLock(String key) {\n        return Objects.requireNonNull(ConcurrentHashMapUtils.computeIfAbsent(lockTable,\n            key, s -> new TimedLock())).tryLock();\n    }\n\n    public boolean tryLock(String groupId, String topicId) {\n        return tryLock(groupId + PopAckConstants.SPLIT + topicId);\n    }\n\n    public void unlock(String key) {\n        TimedLock lock = lockTable.get(key);\n        if (lock != null) {\n            lock.unlock();\n        }\n    }\n\n    public void unlock(String groupId, String topicId) {\n        unlock(groupId + PopAckConstants.SPLIT + topicId);\n    }\n\n    // For retry topics, should lock origin group and topic\n    public boolean isLockTimeout(String groupId, String topicId) {\n        topicId = KeyBuilder.parseNormalTopic(topicId, groupId);\n        TimedLock lock = lockTable.get(groupId + PopAckConstants.SPLIT + topicId);\n        return lock == null || System.currentTimeMillis() - lock.getLockTime() > timeout;\n    }\n\n    public void removeTimeout() {\n        Iterator<Map.Entry<String, TimedLock>> iterator = lockTable.entrySet().iterator();\n        while (iterator.hasNext()) {\n            Map.Entry<String, TimedLock> entry = iterator.next();\n            if (System.currentTimeMillis() - entry.getValue().getLockTime() > timeout) {\n                log.info(\"PopConsumerLockService remove timeout lock, \" +\n                    \"key={}, locked={}\", entry.getKey(), entry.getValue().lock.get());\n                iterator.remove();\n            }\n        }\n    }\n\n    static class TimedLock {\n        private volatile long lockTime;\n        private final AtomicBoolean lock;\n\n        public TimedLock() {\n            this.lockTime = System.currentTimeMillis();\n            this.lock = new AtomicBoolean(false);\n        }\n\n        public boolean tryLock() {\n            if (lock.compareAndSet(false, true)) {\n                this.lockTime = System.currentTimeMillis();\n                return true;\n            }\n            return false;\n        }\n\n        public void unlock() {\n            lock.set(false);\n        }\n\n        public long getLockTime() {\n            return lockTime;\n        }\n    }\n}"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerRecord.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.pop;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.alibaba.fastjson2.annotation.JSONField;\n\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\n\npublic class PopConsumerRecord {\n\n    public enum RetryType {\n\n        NORMAL_TOPIC(0),\n\n        RETRY_TOPIC_V1(1),\n\n        RETRY_TOPIC_V2(2);\n\n        private final int code;\n\n        RetryType(int code) {\n            this.code = code;\n        }\n\n        public int getCode() {\n            return code;\n        }\n    }\n\n    @JSONField()\n    private long popTime;\n\n    @JSONField(ordinal = 1)\n    private String groupId;\n\n    @JSONField(ordinal = 2)\n    private String topicId;\n\n    @JSONField(ordinal = 3)\n    private int queueId;\n\n    @JSONField(ordinal = 4)\n    private int retryFlag;\n\n    @JSONField(ordinal = 5)\n    private long invisibleTime;\n\n    @JSONField(ordinal = 6)\n    private long offset;\n\n    @JSONField(ordinal = 7)\n    private int attemptTimes;\n\n    @JSONField(ordinal = 8)\n    private String attemptId;\n\n    @JSONField(ordinal = 9)\n    private boolean suspend;\n\n    // used for test and fastjson\n    public PopConsumerRecord() {\n    }\n\n    public PopConsumerRecord(long popTime, String groupId, String topicId, int queueId,\n        int retryFlag, long invisibleTime, long offset, String attemptId) {\n        this(popTime, groupId, topicId, queueId, retryFlag, invisibleTime, offset, attemptId, false);\n    }\n\n    public PopConsumerRecord(long popTime, String groupId, String topicId, int queueId, int retryFlag,\n                             long invisibleTime, long offset, String attemptId, boolean suspend) {\n\n        this.popTime = popTime;\n        this.groupId = groupId;\n        this.topicId = topicId;\n        this.queueId = queueId;\n        this.retryFlag = retryFlag;\n        this.invisibleTime = invisibleTime;\n        this.offset = offset;\n        this.attemptId = attemptId;\n        this.suspend = suspend;\n    }\n\n    @JSONField(serialize = false)\n    public long getVisibilityTimeout() {\n        return popTime + invisibleTime;\n    }\n\n    /**\n     * Key: timestamp(8) + groupId + topicId + queueId + offset\n     */\n    @JSONField(serialize = false)\n    public byte[] getKeyBytes() {\n        int length = Long.BYTES + groupId.length() + 1 + topicId.length() + 1 + Integer.BYTES + 1 + Long.BYTES;\n        byte[] bytes = new byte[length];\n        ByteBuffer buffer = ByteBuffer.wrap(bytes);\n        buffer.putLong(this.getVisibilityTimeout());\n        buffer.put(groupId.getBytes(StandardCharsets.UTF_8)).put((byte) '@');\n        buffer.put(topicId.getBytes(StandardCharsets.UTF_8)).put((byte) '@');\n        buffer.putInt(queueId).put((byte) '@');\n        buffer.putLong(offset);\n        return bytes;\n    }\n\n    @JSONField(serialize = false)\n    public boolean isRetry() {\n        return retryFlag != 0;\n    }\n\n    @JSONField(serialize = false)\n    public byte[] getValueBytes() {\n        return JSON.toJSONBytes(this);\n    }\n\n    public static PopConsumerRecord decode(byte[] body) {\n        return JSON.parseObject(body, PopConsumerRecord.class);\n    }\n\n    public long getPopTime() {\n        return popTime;\n    }\n\n    public void setPopTime(long popTime) {\n        this.popTime = popTime;\n    }\n\n    public String getGroupId() {\n        return groupId;\n    }\n\n    public void setGroupId(String groupId) {\n        this.groupId = groupId;\n    }\n\n    public String getTopicId() {\n        return topicId;\n    }\n\n    public void setTopicId(String topicId) {\n        this.topicId = topicId;\n    }\n\n    public int getQueueId() {\n        return queueId;\n    }\n\n    public void setQueueId(int queueId) {\n        this.queueId = queueId;\n    }\n\n    public int getRetryFlag() {\n        return retryFlag;\n    }\n\n    public void setRetryFlag(int retryFlag) {\n        this.retryFlag = retryFlag;\n    }\n\n    public long getInvisibleTime() {\n        return invisibleTime;\n    }\n\n    public void setInvisibleTime(long invisibleTime) {\n        this.invisibleTime = invisibleTime;\n    }\n\n    public long getOffset() {\n        return offset;\n    }\n\n    public void setOffset(long offset) {\n        this.offset = offset;\n    }\n\n    public int getAttemptTimes() {\n        return attemptTimes;\n    }\n\n    public void setAttemptTimes(int attemptTimes) {\n        this.attemptTimes = attemptTimes;\n    }\n\n    public String getAttemptId() {\n        return attemptId;\n    }\n\n    public void setAttemptId(String attemptId) {\n        this.attemptId = attemptId;\n    }\n\n    public boolean isSuspend() {\n        return suspend;\n    }\n\n    public void setSuspend(boolean suspend) {\n        this.suspend = suspend;\n    }\n\n    @Override\n    public String toString() {\n        return \"PopDeliveryRecord{\" +\n            \"popTime=\" + popTime +\n            \", groupId='\" + groupId + '\\'' +\n            \", topicId='\" + topicId + '\\'' +\n            \", queueId=\" + queueId +\n            \", retryFlag=\" + retryFlag +\n            \", invisibleTime=\" + invisibleTime +\n            \", offset=\" + offset +\n            \", attemptTimes=\" + attemptTimes +\n            \", attemptId='\" + attemptId + '\\'' +\n            \", suspend=\" + suspend +\n            '}';\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerRocksdbStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.pop;\n\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.config.AbstractRocksDBStorage;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.store.rocksdb.RocksDBOptionsFactory;\nimport org.rocksdb.ColumnFamilyDescriptor;\nimport org.rocksdb.ColumnFamilyHandle;\nimport org.rocksdb.ColumnFamilyOptions;\nimport org.rocksdb.CompactRangeOptions;\nimport org.rocksdb.ReadOptions;\nimport org.rocksdb.RocksDB;\nimport org.rocksdb.RocksDBException;\nimport org.rocksdb.RocksIterator;\nimport org.rocksdb.Slice;\nimport org.rocksdb.WriteBatch;\nimport org.rocksdb.WriteOptions;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\npublic class PopConsumerRocksdbStore extends AbstractRocksDBStorage implements PopConsumerKVStore {\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);\n    private static final byte[] COLUMN_FAMILY_NAME = \"popState\".getBytes(StandardCharsets.UTF_8);\n\n    private WriteOptions writeOptions;\n    private WriteOptions deleteOptions;\n    protected ColumnFamilyHandle columnFamilyHandle;\n\n    public PopConsumerRocksdbStore(String filePath) {\n        super(filePath);\n    }\n\n    // https://www.cnblogs.com/renjc/p/rocksdb-class-db.html\n    // https://github.com/johnzeng/rocksdb-doc-cn/blob/master/doc/RocksDB-Tuning-Guide.md\n    protected void initOptions() {\n        this.options = RocksDBOptionsFactory.createDBOptions();\n\n        this.writeOptions = new WriteOptions();\n        this.writeOptions.setSync(true);\n        this.writeOptions.setDisableWAL(false);\n        this.writeOptions.setNoSlowdown(false);\n\n        this.deleteOptions = new WriteOptions();\n        this.deleteOptions.setSync(true);\n        this.deleteOptions.setDisableWAL(false);\n        this.deleteOptions.setNoSlowdown(false);\n\n        this.compactRangeOptions = new CompactRangeOptions();\n        this.compactRangeOptions.setBottommostLevelCompaction(\n            CompactRangeOptions.BottommostLevelCompaction.kForce);\n        this.compactRangeOptions.setAllowWriteStall(true);\n        this.compactRangeOptions.setExclusiveManualCompaction(false);\n        this.compactRangeOptions.setChangeLevel(true);\n        this.compactRangeOptions.setTargetLevel(-1);\n        this.compactRangeOptions.setMaxSubcompactions(4);\n    }\n\n    @Override\n    protected boolean postLoad() {\n        try {\n            UtilAll.ensureDirOK(this.dbPath);\n            initOptions();\n\n            // init column family here\n            ColumnFamilyOptions defaultOptions = RocksDBOptionsFactory.createPopCFOptions();\n            ColumnFamilyOptions popStateOptions = RocksDBOptionsFactory.createPopCFOptions();\n            this.cfOptions.add(defaultOptions);\n            this.cfOptions.add(popStateOptions);\n\n            List<ColumnFamilyDescriptor> cfDescriptors = new ArrayList<>();\n            cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, defaultOptions));\n            cfDescriptors.add(new ColumnFamilyDescriptor(COLUMN_FAMILY_NAME, popStateOptions));\n            this.open(cfDescriptors);\n            this.defaultCFHandle = cfHandles.get(0);\n            this.columnFamilyHandle = cfHandles.get(1);\n\n            log.debug(\"PopConsumerRocksdbStore init, filePath={}\", this.dbPath);\n        } catch (final Exception e) {\n            log.error(\"PopConsumerRocksdbStore init error, filePath={}\", this.dbPath, e);\n            return false;\n        }\n        return true;\n    }\n\n    public String getFilePath() {\n        return this.dbPath;\n    }\n\n    @Override\n    public void writeRecords(List<PopConsumerRecord> consumerRecordList) {\n        if (!consumerRecordList.isEmpty()) {\n            try (WriteBatch writeBatch = new WriteBatch()) {\n                for (PopConsumerRecord record : consumerRecordList) {\n                    writeBatch.put(columnFamilyHandle, record.getKeyBytes(), record.getValueBytes());\n                }\n                this.db.write(writeOptions, writeBatch);\n            } catch (RocksDBException e) {\n                throw new RuntimeException(\"Write record error\", e);\n            }\n        }\n    }\n\n    @Override\n    public void deleteRecords(List<PopConsumerRecord> consumerRecordList) {\n        if (!consumerRecordList.isEmpty()) {\n            try (WriteBatch writeBatch = new WriteBatch()) {\n                for (PopConsumerRecord record : consumerRecordList) {\n                    writeBatch.delete(columnFamilyHandle, record.getKeyBytes());\n                }\n                this.db.write(deleteOptions, writeBatch);\n            } catch (RocksDBException e) {\n                throw new RuntimeException(\"Delete record error\", e);\n            }\n        }\n    }\n\n    @Override\n    // https://github.com/facebook/rocksdb/issues/10300\n    public List<PopConsumerRecord> scanExpiredRecords(long lower, long upper, int maxCount) {\n        // In RocksDB, we can use SstPartitionerFixedPrefixFactory in cfOptions\n        // and new ColumnFamilyOptions().useFixedLengthPrefixExtractor() to\n        // configure prefix indexing to improve the performance of scans.\n        // However, in the current implementation, this is not the bottleneck.\n        List<PopConsumerRecord> consumerRecordList = new ArrayList<>();\n        try (ReadOptions scanOptions = new ReadOptions()\n            .setIterateLowerBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(lower).array()))\n            .setIterateUpperBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(upper).array()));\n             RocksIterator iterator = db.newIterator(this.columnFamilyHandle, scanOptions)) {\n            iterator.seek(ByteBuffer.allocate(Long.BYTES).putLong(lower).array());\n            while (iterator.isValid() && consumerRecordList.size() < maxCount) {\n                consumerRecordList.add(PopConsumerRecord.decode(iterator.value()));\n                iterator.next();\n            }\n        }\n        return consumerRecordList;\n    }\n\n    @Override\n    protected void preShutdown() {\n        if (this.writeOptions != null) {\n            this.writeOptions.close();\n        }\n        if (this.deleteOptions != null) {\n            this.deleteOptions.close();\n        }\n        if (this.columnFamilyHandle != null) {\n            this.columnFamilyHandle.close();\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.pop;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.base.Stopwatch;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.commons.lang3.tuple.Triple;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.PopAckConstants;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.TopicFilterType;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.ConsumeInitMode;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.utils.ConcurrentHashMapUtils;\nimport org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.AppendMessageStatus;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.MessageFilter;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.pop.PopCheckPoint;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.nio.ByteBuffer;\nimport java.nio.file.Paths;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.Queue;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.CompletionException;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.Semaphore;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.concurrent.atomic.AtomicLong;\n\npublic class PopConsumerService extends ServiceThread {\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);\n    private static final long OFFSET_NOT_EXIST = -1L;\n    private static final String ROCKSDB_DIRECTORY = \"kvStore\";\n    private static final int[] REWRITE_INTERVALS_IN_SECONDS =\n        new int[] {10, 30, 60, 120, 180, 240, 300, 360, 420, 480, 540, 600, 1200, 1800, 3600, 7200};\n\n    private final AtomicBoolean consumerRunning;\n    private final BrokerConfig brokerConfig;\n    private final BrokerController brokerController;\n    private final AtomicLong currentTime;\n    private final AtomicLong lastCleanupLockTime;\n    private final PopConsumerCache popConsumerCache;\n    private final PopConsumerKVStore popConsumerStore;\n    private final PopConsumerLockService consumerLockService;\n    private final ConcurrentMap<String /* groupId@topicId*/, AtomicLong> requestCountTable;\n\n    public PopConsumerService(BrokerController brokerController) {\n\n        this.brokerController = brokerController;\n        this.brokerConfig = brokerController.getBrokerConfig();\n\n        this.consumerRunning = new AtomicBoolean(false);\n        this.requestCountTable = new ConcurrentHashMap<>();\n        this.currentTime = new AtomicLong(TimeUnit.SECONDS.toMillis(3));\n        this.lastCleanupLockTime = new AtomicLong(System.currentTimeMillis());\n        this.consumerLockService = new PopConsumerLockService(TimeUnit.MINUTES.toMillis(2));\n        this.popConsumerStore = new PopConsumerRocksdbStore(Paths.get(\n            brokerController.getMessageStoreConfig().getStorePathRootDir(), ROCKSDB_DIRECTORY).toString());\n        this.popConsumerCache = brokerConfig.isEnablePopBufferMerge() ? new PopConsumerCache(\n            brokerController, this.popConsumerStore, this.consumerLockService, this::revive) : null;\n\n        log.info(\"PopConsumerService init, buffer={}, rocksdb filePath={}\",\n            brokerConfig.isEnablePopBufferMerge(), this.popConsumerStore.getFilePath());\n    }\n\n    /**\n     * In-flight messages are those that have been received from a queue\n     * by a consumer but have not yet been deleted. For standard queues,\n     * there is a limit on the number of in-flight messages, depending on queue traffic and message backlog.\n     */\n    public boolean isPopShouldStop(String group, String topic, int queueId) {\n        return brokerConfig.isEnablePopMessageThreshold() && popConsumerCache != null &&\n            popConsumerCache.getPopInFlightMessageCount(group, topic, queueId) >=\n                brokerConfig.getPopInflightMessageThreshold();\n    }\n\n    public long getPendingFilterCount(String groupId, String topicId, int queueId) {\n        try {\n            long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topicId, queueId);\n            long consumeOffset = this.brokerController.getConsumerOffsetManager().queryOffset(groupId, topicId, queueId);\n            return maxOffset - consumeOffset;\n        } catch (ConsumeQueueException e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    public GetMessageResult recodeRetryMessage(GetMessageResult getMessageResult,\n        String topicId, long offset, long popTime, long invisibleTime) {\n\n        if (getMessageResult.getMessageCount() == 0 ||\n            getMessageResult.getMessageMapedList().isEmpty()) {\n            return getMessageResult;\n        }\n\n        GetMessageResult result = new GetMessageResult(getMessageResult.getMessageCount());\n        result.setStatus(GetMessageStatus.FOUND);\n        String brokerName = brokerConfig.getBrokerName();\n\n        for (SelectMappedBufferResult bufferResult : getMessageResult.getMessageMapedList()) {\n            List<MessageExt> messageExtList = MessageDecoder.decodesBatch(\n                bufferResult.getByteBuffer(), true, false, true);\n            bufferResult.release();\n            for (MessageExt messageExt : messageExtList) {\n                try {\n                    // When override retry message topic to origin topic,\n                    // need clear message store size to recode\n                    String ckInfo = ExtraInfoUtil.buildExtraInfo(offset, popTime, invisibleTime, 0,\n                        messageExt.getTopic(), brokerName, messageExt.getQueueId(), messageExt.getQueueOffset());\n                    messageExt.getProperties().putIfAbsent(MessageConst.PROPERTY_POP_CK, ckInfo);\n                    messageExt.setTopic(topicId);\n                    messageExt.setStoreSize(0);\n                    byte[] encode = MessageDecoder.encode(messageExt, false);\n                    ByteBuffer buffer = ByteBuffer.wrap(encode);\n                    SelectMappedBufferResult tmpResult = new SelectMappedBufferResult(\n                        bufferResult.getStartOffset(), buffer, encode.length, null);\n                    result.addMessage(tmpResult);\n                } catch (Exception e) {\n                    log.error(\"PopConsumerService exception in recode retry message, topic={}\", topicId, e);\n                }\n            }\n        }\n\n        return result;\n    }\n\n    public PopConsumerContext handleGetMessageResult(PopConsumerContext context, GetMessageResult result,\n        String topicId, int queueId, PopConsumerRecord.RetryType retryType, long offset) {\n\n        if (GetMessageStatus.FOUND.equals(result.getStatus()) && !result.getMessageQueueOffset().isEmpty()) {\n            if (context.isFifo()) {\n                this.setFifoBlocked(context, context.getGroupId(), topicId, queueId, result.getMessageQueueOffset(), result);\n            }\n            // build response header here\n            context.addGetMessageResult(result, topicId, queueId, retryType, offset);\n            if (brokerConfig.isPopConsumerKVServiceLog()) {\n                log.info(\"PopConsumerService pop, time={}, invisible={}, \" +\n                        \"groupId={}, topic={}, queueId={}, offset={}, attemptId={}\",\n                    context.getPopTime(), context.getInvisibleTime(), context.getGroupId(),\n                    topicId, queueId, result.getMessageQueueOffset(), context.getAttemptId());\n            }\n        }\n\n        long commitOffset = offset;\n        if (context.isFifo()) {\n            if (!GetMessageStatus.FOUND.equals(result.getStatus())) {\n                commitOffset = result.getNextBeginOffset();\n            }\n        } else {\n            this.brokerController.getConsumerOffsetManager().commitPullOffset(\n                context.getClientHost(), context.getGroupId(), topicId, queueId, result.getNextBeginOffset());\n            if (brokerConfig.isEnablePopBufferMerge() && popConsumerCache != null) {\n                long minOffset = popConsumerCache.getMinOffsetInCache(context.getGroupId(), topicId, queueId);\n                if (minOffset != OFFSET_NOT_EXIST) {\n                    commitOffset = minOffset;\n                }\n            }\n        }\n        this.brokerController.getConsumerOffsetManager().commitOffset(\n            context.getClientHost(), context.getGroupId(), topicId, queueId, commitOffset);\n        return context;\n    }\n\n    public long getPopOffset(String groupId, String topicId, int queueId, int initMode, boolean fifo) {\n\n        // For FIFO messages, the pull offset is not used.\n        // This preserves compatibility when switching from pull consumer to pop consumer.\n        long offset = fifo ?\n            this.brokerController.getConsumerOffsetManager().queryOffset(groupId, topicId, queueId) :\n            this.brokerController.getConsumerOffsetManager().queryPullOffset(groupId, topicId, queueId);\n\n        if (offset < 0L) {\n            try {\n                offset = this.brokerController.getPopMessageProcessor()\n                    .getInitOffset(topicId, groupId, queueId, initMode, true);\n                log.info(\"PopConsumerService init offset, groupId={}, topicId={}, queueId={}, init={}, offset={}\",\n                    groupId, topicId, queueId, ConsumeInitMode.MIN == initMode ? \"min\" : \"max\", offset);\n            } catch (ConsumeQueueException e) {\n                throw new RuntimeException(e);\n            }\n        }\n        Long resetOffset =\n            this.brokerController.getConsumerOffsetManager().queryThenEraseResetOffset(topicId, groupId, queueId);\n        if (resetOffset != null) {\n            this.clearCache(groupId, topicId, queueId);\n            this.brokerController.getConsumerOrderInfoManager().clearBlock(topicId, groupId, queueId);\n            this.brokerController.getConsumerOffsetManager()\n                .commitOffset(\"ResetPopOffset\", groupId, topicId, queueId, resetOffset);\n        }\n        return resetOffset != null ? resetOffset : offset;\n    }\n\n    public CompletableFuture<GetMessageResult> getMessageAsync(String clientHost,\n        String groupId, String topicId, int queueId, long offset, int batchSize, MessageFilter filter) {\n\n        log.debug(\"PopConsumerService getMessageAsync, groupId={}, topicId={}, queueId={}, offset={}, batchSize={}, filter={}\",\n            groupId, topicId, offset, queueId, batchSize, filter != null);\n\n        CompletableFuture<GetMessageResult> getMessageFuture =\n            brokerController.getMessageStore().getMessageAsync(groupId, topicId, queueId, offset, batchSize, filter);\n\n        // refer org.apache.rocketmq.broker.processor.PopMessageProcessor#popMsgFromQueue\n        return getMessageFuture.thenCompose(result -> {\n            if (result == null) {\n                return CompletableFuture.completedFuture(null);\n            }\n\n            // maybe store offset is not correct.\n            if (GetMessageStatus.OFFSET_TOO_SMALL.equals(result.getStatus()) ||\n                GetMessageStatus.OFFSET_OVERFLOW_BADLY.equals(result.getStatus()) ||\n                GetMessageStatus.OFFSET_FOUND_NULL.equals(result.getStatus())) {\n\n                // commit offset, because the offset is not correct\n                // If offset in store is greater than cq offset, it will cause duplicate messages,\n                // because offset in PopBuffer is not committed.\n                this.brokerController.getConsumerOffsetManager().commitOffset(\n                    clientHost, groupId, topicId, queueId, result.getNextBeginOffset());\n\n                log.warn(\"PopConsumerService getMessageAsync, initial offset because store is no correct, \" +\n                        \"groupId={}, topicId={}, queueId={}, batchSize={}, offset={}->{}\",\n                    groupId, topicId, queueId, batchSize, offset, result.getNextBeginOffset());\n\n                return brokerController.getMessageStore().getMessageAsync(\n                    groupId, topicId, queueId, result.getNextBeginOffset(), batchSize, filter);\n            }\n\n            return CompletableFuture.completedFuture(result);\n\n        }).whenComplete((result, throwable) -> {\n            if (throwable != null) {\n                log.error(\"Pop getMessageAsync error\", throwable);\n            }\n        });\n    }\n\n    /**\n     * Fifo message does not have retry feature in broker\n     */\n    public void setFifoBlocked(PopConsumerContext context,\n        String groupId, String topicId, int queueId, List<Long> queueOffsetList, GetMessageResult getMessageResult) {\n        brokerController.getConsumerOrderInfoManager().update(\n            context.getAttemptId(), false, topicId, groupId, queueId,\n            context.getPopTime(), context.getInvisibleTime(), queueOffsetList, context.getOrderCountInfoBuilder(), getMessageResult);\n    }\n\n    public boolean isFifoBlocked(PopConsumerContext context, String groupId, String topicId, int queueId) {\n        return brokerController.getConsumerOrderInfoManager().checkBlock(\n            context.getAttemptId(), topicId, groupId, queueId, context.getInvisibleTime());\n    }\n\n    protected CompletableFuture<PopConsumerContext> getMessageAsync(CompletableFuture<PopConsumerContext> future,\n        String clientHost, String groupId, String topicId, int queueId, int batchSize, MessageFilter filter,\n        PopConsumerRecord.RetryType retryType) {\n\n        return future.thenCompose(result -> {\n\n            // pop request too much, should not add rest count here\n            if (isPopShouldStop(groupId, topicId, queueId)) {\n                return CompletableFuture.completedFuture(result);\n            }\n\n            // Current requests would calculate the total number of messages\n            // waiting to be filtered for new message arrival notifications in\n            // the long-polling service, need disregarding the backlog in order\n            // consumption scenario. If rest message num including the blocked\n            // queue accumulation would lead to frequent unnecessary wake-ups\n            // of long-polling requests, resulting unnecessary CPU usage.\n            // When client ack message, long-polling request would be notifications\n            // by AckMessageProcessor.ackOrderly() and message will not be delayed.\n            if (result.isFifo() && isFifoBlocked(result, groupId, topicId, queueId)) {\n                // should not add accumulation(max offset - consumer offset) here\n                return CompletableFuture.completedFuture(result);\n            }\n\n            int remain = batchSize - result.getMessageCount();\n            if (remain <= 0) {\n                result.addRestCount(this.getPendingFilterCount(groupId, topicId, queueId));\n                return CompletableFuture.completedFuture(result);\n            } else {\n                final long consumeOffset = this.getPopOffset(groupId, topicId, queueId, result.getInitMode(), result.isFifo());\n                return getMessageAsync(clientHost, groupId, topicId, queueId, consumeOffset, remain, filter)\n                    .thenApply(getMessageResult -> handleGetMessageResult(\n                        result, getMessageResult, topicId, queueId, retryType, consumeOffset));\n            }\n        });\n    }\n\n    protected CompletableFuture<PopConsumerContext> getMessageFromTopicAsync(CompletableFuture<PopConsumerContext> future,\n        String clientHost, String groupId, String topicId, long requestCount, int batchSize, MessageFilter filter,\n        PopConsumerRecord.RetryType retryType) {\n        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topicId);\n        if (null == topicConfig) {\n            return future;\n        }\n        for (int i = 0; i < topicConfig.getReadQueueNums(); i++) {\n            long index = (brokerController.getBrokerConfig().isPriorityOrderAsc() ?\n                topicConfig.getReadQueueNums() - 1 - i : i) + requestCount;\n            int current = (int) index % topicConfig.getReadQueueNums();\n            future = this.getMessageAsync(future, clientHost, groupId,\n                topicId, current, batchSize, filter, retryType);\n        }\n        return future;\n    }\n\n    public CompletableFuture<PopConsumerContext> popAsync(String clientHost, long popTime, long invisibleTime,\n        String groupId, String topicId, int queueId, int batchSize, boolean fifo, String attemptId, int initMode,\n        MessageFilter filter) {\n\n        PopConsumerContext popConsumerContext =\n            new PopConsumerContext(clientHost, popTime, invisibleTime, groupId, fifo, initMode, attemptId);\n\n        TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(topicId);\n        if (topicConfig == null || !consumerLockService.tryLock(groupId, topicId)) {\n            return CompletableFuture.completedFuture(popConsumerContext);\n        }\n\n        SubscriptionGroupConfig subscriptionGroupConfig =\n            this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(groupId);\n        if (null == subscriptionGroupConfig || !subscriptionGroupConfig.isConsumeEnable()) {\n            return CompletableFuture.completedFuture(popConsumerContext);\n        }\n\n        log.debug(\"PopConsumerService popAsync, groupId={}, topicId={}, queueId={}, \" +\n                \"batchSize={}, invisibleTime={}, fifo={}, attemptId={}, filter={}\",\n            groupId, topicId, queueId, batchSize, invisibleTime, fifo, attemptId, filter);\n\n        String requestKey = groupId + \"@\" + topicId;\n        String retryTopicV1 = KeyBuilder.buildPopRetryTopicV1(topicId, groupId);\n        String retryTopicV2 = KeyBuilder.buildPopRetryTopicV2(topicId, groupId);\n        long requestCount = Objects.requireNonNull(ConcurrentHashMapUtils.computeIfAbsent(\n            requestCountTable, requestKey, k -> new AtomicLong(0L))).getAndIncrement();\n        boolean usePriorityMode = TopicMessageType.PRIORITY.equals(topicConfig.getTopicMessageType())\n            && !fifo && requestCount % 100L < subscriptionGroupConfig.getPriorityFactor();\n        int probability = usePriorityMode ?\n            brokerConfig.getPopFromRetryProbabilityForPriority() : brokerConfig.getPopFromRetryProbability();\n        probability = Math.max(0, Math.min(100, probability)); // [51, 100] means always\n        boolean preferRetry = probability > 0 && requestCount % (100 / probability) == 0L;\n        requestCount = usePriorityMode ? 0 : requestCount; // use requestCount as randomQ\n\n        CompletableFuture<PopConsumerContext> getMessageFuture =\n            CompletableFuture.completedFuture(popConsumerContext);\n\n        try {\n            if (!fifo && preferRetry) {\n                if (brokerConfig.isRetrieveMessageFromPopRetryTopicV1()) {\n                    getMessageFuture = this.getMessageFromTopicAsync(getMessageFuture, clientHost, groupId,\n                        retryTopicV1, requestCount, batchSize, filter, PopConsumerRecord.RetryType.RETRY_TOPIC_V1);\n                }\n\n                if (brokerConfig.isEnableRetryTopicV2()) {\n                    getMessageFuture = this.getMessageFromTopicAsync(getMessageFuture, clientHost, groupId,\n                        retryTopicV2, requestCount, batchSize, filter, PopConsumerRecord.RetryType.RETRY_TOPIC_V2);\n                }\n            }\n\n            if (queueId != -1) {\n                getMessageFuture = this.getMessageAsync(getMessageFuture, clientHost, groupId,\n                    topicId, queueId, batchSize, filter, PopConsumerRecord.RetryType.NORMAL_TOPIC);\n            } else {\n                getMessageFuture = this.getMessageFromTopicAsync(getMessageFuture, clientHost, groupId,\n                    topicId, requestCount, batchSize, filter, PopConsumerRecord.RetryType.NORMAL_TOPIC);\n\n                if (!fifo && !preferRetry) {\n                    if (brokerConfig.isRetrieveMessageFromPopRetryTopicV1()) {\n                        getMessageFuture = this.getMessageFromTopicAsync(getMessageFuture, clientHost, groupId,\n                            retryTopicV1, requestCount, batchSize, filter, PopConsumerRecord.RetryType.RETRY_TOPIC_V1);\n                    }\n\n                    if (brokerConfig.isEnableRetryTopicV2()) {\n                        getMessageFuture = this.getMessageFromTopicAsync(getMessageFuture, clientHost, groupId,\n                            retryTopicV2, requestCount, batchSize, filter, PopConsumerRecord.RetryType.RETRY_TOPIC_V2);\n                    }\n                }\n            }\n\n            return getMessageFuture.thenCompose(result -> {\n                if (result.isFound() && !result.isFifo()) {\n                    if (brokerConfig.isEnablePopBufferMerge() &&\n                        popConsumerCache != null && !popConsumerCache.isCacheFull()) {\n                        this.popConsumerCache.writeRecords(result.getPopConsumerRecordList());\n                    } else {\n                        this.popConsumerStore.writeRecords(result.getPopConsumerRecordList());\n                    }\n\n                    for (int i = 0; i < result.getGetMessageResultList().size(); i++) {\n                        GetMessageResult getMessageResult = result.getGetMessageResultList().get(i);\n                        PopConsumerRecord popConsumerRecord = result.getPopConsumerRecordList().get(i);\n\n                        // If the buffer belong retries message, the message needs to be re-encoded.\n                        // The buffer should not be re-encoded when popResponseReturnActualRetryTopic\n                        // is true or the current topic is not a retry topic.\n                        boolean recode = brokerConfig.isPopResponseReturnActualRetryTopic();\n                        if (recode && popConsumerRecord.isRetry()) {\n                            result.getGetMessageResultList().set(i, this.recodeRetryMessage(\n                                getMessageResult, popConsumerRecord.getTopicId(),\n                                popConsumerRecord.getQueueId(), result.getPopTime(), invisibleTime));\n                        }\n                    }\n                }\n                return CompletableFuture.completedFuture(result);\n            }).whenComplete((result, throwable) -> {\n                try {\n                    if (throwable != null) {\n                        log.error(\"PopConsumerService popAsync get message error\",\n                            throwable instanceof CompletionException ? throwable.getCause() : throwable);\n                    }\n                    if (result.getMessageCount() > 0) {\n                        log.debug(\"PopConsumerService popAsync result, found={}, groupId={}, topicId={}, queueId={}, \" +\n                                \"batchSize={}, invisibleTime={}, fifo={}, attemptId={}, filter={}\", result.getMessageCount(),\n                            groupId, topicId, queueId, batchSize, invisibleTime, fifo, attemptId, filter);\n                    }\n                } finally {\n                    consumerLockService.unlock(groupId, topicId);\n                }\n            });\n        } catch (Throwable t) {\n            log.error(\"PopConsumerService popAsync error\", t);\n        }\n\n        return getMessageFuture;\n    }\n\n    // Notify polling request when receive orderly ack\n    public CompletableFuture<Boolean> ackAsync(\n        long popTime, long invisibleTime, String groupId, String topicId, int queueId, long offset) {\n\n        if (brokerConfig.isPopConsumerKVServiceLog()) {\n            log.info(\"PopConsumerService ack, time={}, invisible={}, groupId={}, topic={}, queueId={}, offset={}\",\n                popTime, invisibleTime, groupId, topicId, queueId, offset);\n        }\n\n        PopConsumerRecord record = new PopConsumerRecord(\n            popTime, groupId, topicId, queueId, 0, invisibleTime, offset, null);\n\n        if (brokerConfig.isEnablePopBufferMerge() && popConsumerCache != null) {\n            if (popConsumerCache.deleteRecords(Collections.singletonList(record)).isEmpty()) {\n                return CompletableFuture.completedFuture(true);\n            }\n        }\n\n        this.popConsumerStore.deleteRecords(Collections.singletonList(record));\n        return CompletableFuture.completedFuture(true);\n    }\n\n    // refer ChangeInvisibleTimeProcessor.appendCheckPointThenAckOrigin\n    public void changeInvisibilityDuration(long popTime, long invisibleTime, long changedPopTime,\n                                           long changedInvisibleTime, String groupId, String topicId,\n                                           int queueId, long offset, boolean suspend) {\n\n        if (brokerConfig.isPopConsumerKVServiceLog()) {\n            log.info(\"PopConsumerService change, time={}, invisible={}, \" +\n                    \"groupId={}, topic={}, queueId={}, offset={}, new time={}, new invisible={}\",\n                popTime, invisibleTime, groupId, topicId, queueId, offset, changedPopTime, changedInvisibleTime);\n        }\n\n        PopConsumerRecord ckRecord = new PopConsumerRecord(\n            changedPopTime, groupId, topicId, queueId, 0, changedInvisibleTime, offset, null, suspend);\n\n        PopConsumerRecord ackRecord = new PopConsumerRecord(\n            popTime, groupId, topicId, queueId, 0, invisibleTime, offset, null, suspend);\n\n        // No need to generate new records when the group does not exist,\n        // because these retry messages will not be consumed by anyone.\n        if (brokerConfig.isPopReviveSkipIfGroupAbsent() &&\n            !brokerController.getSubscriptionGroupManager().containsSubscriptionGroup(groupId)) {\n            log.info(\"PopConsumerService change invisibility skip, time={}, \" +\n                \"groupId={}, topicId={}, queueId={}, offset={}\", popTime, groupId, topicId, queueId, offset);\n        } else {\n            this.popConsumerStore.writeRecords(Collections.singletonList(ckRecord));\n        }\n\n        if (brokerConfig.isEnablePopBufferMerge() && popConsumerCache != null) {\n            if (popConsumerCache.deleteRecords(Collections.singletonList(ackRecord)).isEmpty()) {\n                return;\n            }\n        }\n\n        this.popConsumerStore.deleteRecords(Collections.singletonList(ackRecord));\n    }\n\n    // Use broker escape bridge to support remote read\n    public CompletableFuture<Triple<MessageExt, String, Boolean>> getMessageAsync(PopConsumerRecord consumerRecord) {\n        return this.brokerController.getEscapeBridge().getMessageAsync(consumerRecord.getTopicId(),\n            consumerRecord.getOffset(), consumerRecord.getQueueId(), brokerConfig.getBrokerName(), false);\n    }\n\n    public CompletableFuture<Boolean> revive(PopConsumerRecord record) {\n\n        if (brokerConfig.isPopReviveSkipIfGroupAbsent() &&\n            !brokerController.getSubscriptionGroupManager().containsSubscriptionGroup(record.getGroupId())) {\n            log.info(\"PopConsumerService skip revive message, record={}\", record);\n            return CompletableFuture.completedFuture(true);\n        }\n\n        return this.getMessageAsync(record)\n            .thenCompose(result -> {\n                if (result == null) {\n                    log.error(\"PopConsumerService revive error, message may be lost, record={}\", record);\n                    return CompletableFuture.completedFuture(false);\n                }\n                // true in triple right means get message needs to be retried\n                if (result.getLeft() == null) {\n                    log.info(\"PopConsumerService revive no need retry, record={}\", record);\n                    return CompletableFuture.completedFuture(!result.getRight());\n                }\n                return CompletableFuture.completedFuture(this.reviveRetry(record, result.getLeft()));\n            });\n    }\n\n    @SuppressWarnings(\"StatementWithEmptyBody\")\n    public void clearCache(String groupId, String topicId, int queueId) {\n        while (consumerLockService.tryLock(groupId, topicId)) {\n        }\n        try {\n            if (popConsumerCache != null) {\n                popConsumerCache.removeRecords(groupId, topicId, queueId);\n            }\n        } finally {\n            consumerLockService.unlock(groupId, topicId);\n        }\n    }\n\n    public long revive(AtomicLong currentTime, int maxCount) {\n        Stopwatch stopwatch = Stopwatch.createStarted();\n        long upperTime = System.currentTimeMillis() - 50L;\n        List<PopConsumerRecord> consumerRecords = this.popConsumerStore.scanExpiredRecords(\n                currentTime.get() - TimeUnit.SECONDS.toMillis(3), upperTime, maxCount);\n        long scanCostTime = stopwatch.elapsed(TimeUnit.MILLISECONDS);\n\n        // When reading messages from local storage, the current thread is used\n        // directly for data retrieval. When reading original messages from remote\n        // storage (such as distributed file systems), so concurrency needs to be\n        // controlled via semaphore.\n        Semaphore semaphore = new Semaphore(brokerConfig.getPopReviveConcurrency());\n        Queue<PopConsumerRecord> failureList = new LinkedBlockingQueue<>();\n        List<CompletableFuture<?>> futureList = new ArrayList<>(consumerRecords.size());\n\n        // could merge read operation here\n        for (PopConsumerRecord record : consumerRecords) {\n            CompletableFuture<Boolean> future;\n            try {\n                semaphore.acquire();\n                future = this.revive(record);\n            } catch (Exception e) {\n                semaphore.release();\n                throw new RuntimeException(e);\n            }\n            futureList.add(future.thenAccept(result -> {\n                if (!result) {\n                    if (record.getAttemptTimes() < brokerConfig.getPopReviveMaxAttemptTimes()) {\n                        long backoffInterval = 1000L * REWRITE_INTERVALS_IN_SECONDS[\n                            Math.min(REWRITE_INTERVALS_IN_SECONDS.length, record.getAttemptTimes())];\n                        long nextInvisibleTime = record.getInvisibleTime() + backoffInterval;\n                        PopConsumerRecord retryRecord = new PopConsumerRecord(System.currentTimeMillis(),\n                            record.getGroupId(), record.getTopicId(), record.getQueueId(),\n                            record.getRetryFlag(), nextInvisibleTime, record.getOffset(), record.getAttemptId());\n                        retryRecord.setAttemptTimes(record.getAttemptTimes() + 1);\n                        failureList.add(retryRecord);\n                        log.warn(\"PopConsumerService revive backoff retry, record={}\", retryRecord);\n                    } else {\n                        log.error(\"PopConsumerService drop record, message may be lost, record={}\", record);\n                    }\n                }\n            }).whenComplete((result, ex) -> semaphore.release()));\n        }\n\n        CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).join();\n        this.popConsumerStore.writeRecords(new ArrayList<>(failureList));\n        this.popConsumerStore.deleteRecords(consumerRecords);\n        currentTime.set(consumerRecords.isEmpty() ?\n            upperTime : consumerRecords.get(consumerRecords.size() - 1).getVisibilityTimeout());\n\n        if (brokerConfig.isEnablePopBufferMerge()) {\n            log.info(\"PopConsumerService, key size={}, cache size={}, revive count={}, failure count={}, \" +\n                    \"behindInMillis={}, scanInMillis={}, costInMillis={}\",\n                popConsumerCache.getCacheKeySize(), popConsumerCache.getCacheSize(),\n                consumerRecords.size(), failureList.size(), upperTime - currentTime.get(),\n                scanCostTime, stopwatch.elapsed(TimeUnit.MILLISECONDS));\n        } else {\n            log.info(\"PopConsumerService, revive count={}, failure count={}, \" +\n                    \"behindInMillis={}, scanInMillis={}, costInMillis={}\",\n                consumerRecords.size(), failureList.size(), upperTime - currentTime.get(),\n                scanCostTime, stopwatch.elapsed(TimeUnit.MILLISECONDS));\n        }\n\n        return consumerRecords.size();\n    }\n\n    public void createRetryTopicIfNeeded(String groupId, String retryTopic) {\n        TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(retryTopic);\n        if (topicConfig != null && !brokerController.getBrokerConfig().isUseSeparateRetryQueue()) {\n            return;\n        }\n\n        int retryQueueNum = PopAckConstants.retryQueueNum;\n        if (brokerController.getBrokerConfig().isUseSeparateRetryQueue()) {\n            String normalTopic = KeyBuilder.parseNormalTopic(retryTopic, groupId);\n            TopicConfig normalConfig = brokerController.getTopicConfigManager().selectTopicConfig(normalTopic); // always exists\n            retryQueueNum = normalConfig.getWriteQueueNums();\n            if (topicConfig != null && topicConfig.getWriteQueueNums() == normalConfig.getWriteQueueNums()) {\n                return;\n            }\n        }\n\n        topicConfig = new TopicConfig(retryTopic, retryQueueNum, retryQueueNum,\n            PermName.PERM_READ | PermName.PERM_WRITE, 0);\n        topicConfig.setTopicFilterType(TopicFilterType.SINGLE_TAG);\n        brokerController.getTopicConfigManager().updateTopicConfig(topicConfig);\n\n        for (int i = 0; i < retryQueueNum; i++) {\n            long offset = this.brokerController.getConsumerOffsetManager().queryOffset(groupId, retryTopic, i);\n            if (offset < 0) {\n                this.brokerController.getConsumerOffsetManager().commitOffset(\n                    \"InitPopOffset\", groupId, retryTopic, i, 0);\n            }\n        }\n    }\n\n    @SuppressWarnings(\"DuplicatedCode\")\n    // org.apache.rocketmq.broker.processor.PopReviveService#reviveRetry\n    public boolean reviveRetry(PopConsumerRecord record, MessageExt messageExt) {\n\n        if (brokerConfig.isPopConsumerKVServiceLog()) {\n            log.info(\"PopConsumerService revive, time={}, invisible={}, groupId={}, topic={}, queueId={}, offset={}\",\n                record.getPopTime(), record.getInvisibleTime(), record.getGroupId(), record.getTopicId(),\n                record.getQueueId(), record.getOffset());\n        }\n\n        boolean retry = StringUtils.startsWith(record.getTopicId(), MixAll.RETRY_GROUP_TOPIC_PREFIX);\n        String retryTopic = retry ? record.getTopicId() : KeyBuilder.buildPopRetryTopic(\n            record.getTopicId(), record.getGroupId(), brokerConfig.isEnableRetryTopicV2());\n        this.createRetryTopicIfNeeded(record.getGroupId(), retryTopic);\n\n        // deep copy here\n        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n        msgInner.setTopic(retryTopic);\n        msgInner.setBody(messageExt.getBody() != null ? messageExt.getBody() : new byte[] {});\n        msgInner.setQueueId(getRetryQueueId(retryTopic, messageExt));\n        if (messageExt.getTags() != null) {\n            msgInner.setTags(messageExt.getTags());\n        } else {\n            MessageAccessor.setProperties(msgInner, new HashMap<>());\n        }\n\n        msgInner.setBornTimestamp(messageExt.getBornTimestamp());\n        msgInner.setFlag(messageExt.getFlag());\n        msgInner.setSysFlag(messageExt.getSysFlag());\n        msgInner.setBornHost(brokerController.getStoreHost());\n        msgInner.setStoreHost(brokerController.getStoreHost());\n        if (record.isSuspend()) {\n            msgInner.setReconsumeTimes(messageExt.getReconsumeTimes());\n        } else {\n            msgInner.setReconsumeTimes(messageExt.getReconsumeTimes() + 1);\n        }\n\n        msgInner.getProperties().putAll(messageExt.getProperties());\n\n        // set first pop time here\n        if (messageExt.getReconsumeTimes() == 0 ||\n            msgInner.getProperties().get(MessageConst.PROPERTY_FIRST_POP_TIME) == null) {\n            msgInner.getProperties().put(MessageConst.PROPERTY_FIRST_POP_TIME, String.valueOf(record.getPopTime()));\n        }\n        msgInner.getProperties().put(MessageConst.PROPERTY_ORIGIN_GROUP, record.getGroupId());\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));\n\n        PutMessageResult putMessageResult =\n            brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);\n\n        if (putMessageResult.getAppendMessageResult() == null ||\n            putMessageResult.getAppendMessageResult().getStatus() != AppendMessageStatus.PUT_OK) {\n            log.error(\"PopConsumerService revive retry msg error, put status={}, ck={}, delay={}ms\",\n                putMessageResult, JSON.toJSONString(record), System.currentTimeMillis() - record.getVisibilityTimeout());\n            return false;\n        }\n\n        if (this.brokerController.getBrokerStatsManager() != null) {\n            this.brokerController.getBrokerStatsManager().incBrokerPutNums(msgInner.getTopic(), 1);\n            this.brokerController.getBrokerStatsManager().incTopicPutNums(msgInner.getTopic());\n            this.brokerController.getBrokerStatsManager().incTopicPutSize(\n                msgInner.getTopic(), putMessageResult.getAppendMessageResult().getWroteBytes());\n        }\n        return true;\n    }\n\n    private int getRetryQueueId(String retryTopic, MessageExt oriMsg) {\n        if (!brokerController.getBrokerConfig().isUseSeparateRetryQueue()) {\n            return 0;\n        }\n        int oriQueueId = oriMsg.getQueueId(); // original qid of normal or retry topic\n        if (oriQueueId > brokerController.getTopicConfigManager().selectTopicConfig(retryTopic).getWriteQueueNums() - 1) {\n            log.warn(\"not expected, {}, {}, {}\", retryTopic, oriQueueId, oriMsg.getMsgId());\n            return 0; // fallback\n        }\n        return oriQueueId;\n    }\n\n    // Export kv store record to revive topic\n    @SuppressWarnings(\"ExtractMethodRecommender\")\n    public synchronized void transferToFsStore() {\n        Stopwatch stopwatch = Stopwatch.createStarted();\n        while (true) {\n            try {\n                List<PopConsumerRecord> consumerRecords = this.popConsumerStore.scanExpiredRecords(\n                    0, Long.MAX_VALUE, brokerConfig.getPopReviveMaxReturnSizePerRead());\n                if (consumerRecords == null || consumerRecords.isEmpty()) {\n                    break;\n                }\n                for (PopConsumerRecord record : consumerRecords) {\n                    PopCheckPoint ck = new PopCheckPoint();\n                    ck.setBitMap(0);\n                    ck.setNum((byte) 1);\n                    ck.setPopTime(record.getPopTime());\n                    ck.setInvisibleTime(record.getInvisibleTime());\n                    ck.setStartOffset(record.getOffset());\n                    ck.setCId(record.getGroupId());\n                    ck.setTopic(record.getTopicId());\n                    ck.setQueueId(record.getQueueId());\n                    ck.setBrokerName(brokerConfig.getBrokerName());\n                    ck.addDiff(0);\n                    ck.setRePutTimes(ck.getRePutTimes());\n                    int reviveQueueId = (int) record.getOffset() % brokerConfig.getReviveQueueNum();\n                    MessageExtBrokerInner ckMsg =\n                        brokerController.getPopMessageProcessor().buildCkMsg(ck, reviveQueueId);\n                    brokerController.getMessageStore().asyncPutMessage(ckMsg).join();\n                }\n                log.info(\"PopConsumerStore transfer from kvStore to fsStore, count={}\", consumerRecords.size());\n                this.popConsumerStore.deleteRecords(consumerRecords);\n                this.waitForRunning(1);\n            } catch (Throwable t) {\n                log.error(\"PopConsumerStore transfer from kvStore to fsStore failure\", t);\n            }\n        }\n        log.info(\"PopConsumerStore transfer to fsStore finish, cost={}ms\", stopwatch.elapsed(TimeUnit.MILLISECONDS));\n    }\n\n    @Override\n    public String getServiceName() {\n        return PopConsumerService.class.getSimpleName();\n    }\n\n    @VisibleForTesting\n    protected PopConsumerKVStore getPopConsumerStore() {\n        return popConsumerStore;\n    }\n\n    public PopConsumerLockService getConsumerLockService() {\n        return consumerLockService;\n    }\n\n    @Override\n    public void start() {\n        if (!this.popConsumerStore.start()) {\n            throw new RuntimeException(\"PopConsumerStore init error\");\n        }\n        if (this.popConsumerCache != null) {\n            this.popConsumerCache.start();\n        }\n        super.start();\n    }\n\n    @Override\n    public void shutdown() {\n        // Block shutdown thread until write records finish\n        super.shutdown();\n        do {\n            this.waitForRunning(10);\n        }\n        while (consumerRunning.get());\n        if (this.popConsumerCache != null) {\n            this.popConsumerCache.shutdown();\n        }\n        if (this.popConsumerStore != null) {\n            this.popConsumerStore.shutdown();\n        }\n    }\n\n    @Override\n    public void run() {\n        this.consumerRunning.set(true);\n        while (!isStopped()) {\n            try {\n                // to prevent concurrency issues during read and write operations\n                long reviveCount = this.revive(this.currentTime,\n                    brokerConfig.getPopReviveMaxReturnSizePerRead());\n\n                long current = System.currentTimeMillis();\n                if (lastCleanupLockTime.get() + TimeUnit.MINUTES.toMillis(1) < current) {\n                    this.consumerLockService.removeTimeout();\n                    this.lastCleanupLockTime.set(current);\n                }\n\n                if (reviveCount < brokerConfig.getPopReviveMaxReturnSizePerRead()) {\n                    this.waitForRunning(500);\n                }\n            } catch (Exception e) {\n                log.error(\"PopConsumerService revive error\", e);\n                this.waitForRunning(500);\n            }\n        }\n        this.consumerRunning.set(false);\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/pop/orderly/ConsumerOrderInfoManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.pop.orderly;\n\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.common.OrderedConsumptionLevel;\nimport org.apache.rocketmq.store.GetMessageResult;\n\n/**\n *\n * Ordered Consumption Controller Interface\n * This is the top-level interface that encapsulates complete ordered consumption management functionality,\n * supporting different concurrency strategy implementations\n * <p>\n * Design Goals:\n * 1. Support queue-level ordered consumption (existing implementation)\n * 2. Support message group-level ordered consumption (improve concurrency)\n * 3. Support custom ordered consumption strategies\n * </p>\n */\npublic interface ConsumerOrderInfoManager {\n\n    /**\n     * Update the reception status of message list\n     * Called by handleGetMessageResult when consumer POPs messages, used to record message status and build consumption information\n     *\n     * @param attemptId          Distinguish different pop requests\n     * @param isRetry            Whether it is a retry topic\n     * @param topic              Topic name\n     * @param group              Consumer group name\n     * @param queueId            Queue ID\n     * @param popTime            Time when messages are popped\n     * @param invisibleTime      Message invisible time\n     * @param msgQueueOffsetList List of message queue offsets\n     * @param orderInfoBuilder   String builder for constructing order information\n     * @param getMessageResult   Return new result\n     */\n    void update(String attemptId, boolean isRetry, String topic, String group, int queueId,\n        long popTime, long invisibleTime, List<Long> msgQueueOffsetList,\n        StringBuilder orderInfoBuilder, GetMessageResult getMessageResult);\n\n    /**\n     * Check whether the current POP request needs to be blocked\n     * Used to ensure ordered consumption of ordered messages\n     * Called when consumer POPs messages\n     *\n     * @param attemptId     Attempt ID\n     * @param topic         Topic name\n     * @param group         Consumer group name\n     * @param queueId       Queue ID\n     * @param invisibleTime Invisible time\n     * @return true indicates blocking is needed, false indicates can proceed\n     */\n    boolean checkBlock(String attemptId, String topic, String group, int queueId, long invisibleTime);\n\n    /**\n     * Remove the specified topic and group\n     * Usually called during topic deletion\n     *\n     * @param topic Topic name\n     * @param group Consumer group name\n     */\n    void remove(String topic, String group);\n\n    /**\n     * Get order info count\n     */\n    int getOrderInfoCount();\n\n    /**\n     * Commit message and calculate next consumption offset\n     * Called when consumer ACKs messages\n     *\n     * @param topic       Topic name\n     * @param group       Consumer group name\n     * @param queueId     Queue ID\n     * @param queueOffset Message queue offset\n     * @param popTime     Pop time, used for validation\n     * @return -1: invalid, -2: no need to commit, >=0: offset that needs to be committed (indicates messages below this offset have been consumed)\n     */\n    long commitAndNext(String topic, String group, int queueId, long queueOffset, long popTime);\n\n    /**\n     * Update the next visible time of message\n     * Used for delayed message re-consumption\n     *\n     * @param topic           Topic name\n     * @param group           Consumer group name\n     * @param queueId         Queue ID\n     * @param queueOffset     Message offset\n     * @param popTime         Pop time, used for validation\n     * @param nextVisibleTime Next visible time\n     */\n    void updateNextVisibleTime(String topic, String group, int queueId, long queueOffset,\n        long popTime, long nextVisibleTime);\n\n    /**\n     * Clear the blocking status of specified queue\n     * Usually called during consumer rebalancing or queue reassignment\n     *\n     * @param topic   Topic name\n     * @param group   Consumer group name\n     * @param queueId Queue ID\n     */\n    void clearBlock(String topic, String group, int queueId);\n\n    /**\n     * Get ordered consumption level\n     * Used to distinguish different implementation strategies\n     *\n     * @return Ordered consumption level, such as: QUEUE, MESSAGE_GROUP, etc.\n     */\n    OrderedConsumptionLevel getOrderedConsumptionLevel();\n\n    /**\n     * Start the controller\n     * Initialize necessary resources, such as timers, thread pools, etc.\n     */\n    void start();\n\n    /**\n     * Shutdown the controller\n     * Release resources, clean up scheduled tasks, etc.\n     */\n    void shutdown();\n\n    /**\n     * Persist the controller\n     * Persist the controller's data\n     */\n    void persist();\n\n    boolean load();\n\n    /**\n     * Get available message result\n     * Used to retrieve messages from cache\n     */\n    CompletableFuture<GetMessageResult> getAvailableMessageResult(String attemptId, long popTime, long invisibleTime,\n        String groupId,\n        String topicId, int queueId, int batchSize, StringBuilder orderCountInfoBuilder);\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/pop/orderly/QueueLevelConsumerManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.pop.orderly;\n\nimport com.alibaba.fastjson2.annotation.JSONField;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.base.MoreObjects;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.BrokerPathConfigHelper;\nimport org.apache.rocketmq.common.ConfigManager;\nimport org.apache.rocketmq.common.OrderedConsumptionLevel;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;\nimport org.apache.rocketmq.store.GetMessageResult;\n\npublic class QueueLevelConsumerManager extends ConfigManager implements ConsumerOrderInfoManager {\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private static final String TOPIC_GROUP_SEPARATOR = \"@\";\n    private static final long CLEAN_SPAN_FROM_LAST = 24 * 3600 * 1000;\n\n    private ConcurrentHashMap<String/* topic@group*/, ConcurrentHashMap<Integer/*queueId*/, OrderInfo>> table =\n        new ConcurrentHashMap<>(128);\n\n    private transient QueueLevelConsumerOrderInfoLockManager queueLevelConsumerOrderInfoLockManager;\n    private transient BrokerController brokerController;\n\n    public QueueLevelConsumerManager() {\n    }\n\n    public QueueLevelConsumerManager(BrokerController brokerController) {\n        this.brokerController = brokerController;\n        this.queueLevelConsumerOrderInfoLockManager = new QueueLevelConsumerOrderInfoLockManager(brokerController);\n    }\n\n    public ConcurrentHashMap<String, ConcurrentHashMap<Integer, OrderInfo>> getTable() {\n        return table;\n    }\n\n    public void setTable(ConcurrentHashMap<String, ConcurrentHashMap<Integer, OrderInfo>> table) {\n        this.table = table;\n    }\n\n    protected static String buildKey(String topic, String group) {\n        return topic + TOPIC_GROUP_SEPARATOR + group;\n    }\n\n    protected static String[] decodeKey(String key) {\n        return key.split(TOPIC_GROUP_SEPARATOR);\n    }\n\n    protected void updateLockFreeTimestamp(String topic, String group, int queueId, OrderInfo orderInfo) {\n        if (queueLevelConsumerOrderInfoLockManager != null) {\n            queueLevelConsumerOrderInfoLockManager.updateLockFreeTimestamp(topic, group, queueId, orderInfo);\n        }\n    }\n\n    /**\n     * update the message list received\n     *\n     * @param isRetry is retry topic or not\n     * @param topic topic\n     * @param group group\n     * @param queueId queue id of message\n     * @param popTime the time of pop message\n     * @param invisibleTime invisible time\n     * @param msgQueueOffsetList the queue offsets of messages\n     * @param orderInfoBuilder will append order info to this builder\n     */\n    public void update(String attemptId, boolean isRetry, String topic, String group, int queueId, long popTime,\n        long invisibleTime,\n        List<Long> msgQueueOffsetList, StringBuilder orderInfoBuilder) {\n        String key = buildKey(topic, group);\n        ConcurrentHashMap<Integer/*queueId*/, OrderInfo> qs = table.get(key);\n        if (qs == null) {\n            qs = new ConcurrentHashMap<>(16);\n            ConcurrentHashMap<Integer/*queueId*/, OrderInfo> old = table.putIfAbsent(key, qs);\n            if (old != null) {\n                qs = old;\n            }\n        }\n\n        OrderInfo orderInfo = qs.get(queueId);\n\n        if (orderInfo != null) {\n            OrderInfo newOrderInfo = new OrderInfo(attemptId, popTime, invisibleTime, msgQueueOffsetList, System.currentTimeMillis(), 0);\n            newOrderInfo.mergeOffsetConsumedCount(orderInfo.attemptId, orderInfo.offsetList, orderInfo.offsetConsumedCount);\n\n            orderInfo = newOrderInfo;\n        } else {\n            orderInfo = new OrderInfo(attemptId, popTime, invisibleTime, msgQueueOffsetList, System.currentTimeMillis(), 0);\n        }\n        qs.put(queueId, orderInfo);\n\n        Map<Long, Integer> offsetConsumedCount = orderInfo.offsetConsumedCount;\n        int minConsumedTimes = Integer.MAX_VALUE;\n        if (offsetConsumedCount != null) {\n            Set<Long> offsetSet = offsetConsumedCount.keySet();\n            for (Long offset : offsetSet) {\n                Integer consumedTimes = offsetConsumedCount.getOrDefault(offset, 0);\n                ExtraInfoUtil.buildQueueOffsetOrderCountInfo(orderInfoBuilder, topic, queueId, offset, consumedTimes);\n                minConsumedTimes = Math.min(minConsumedTimes, consumedTimes);\n            }\n\n            if (offsetConsumedCount.size() != orderInfo.offsetList.size()) {\n                // offsetConsumedCount only save messages which consumed count is greater than 0\n                // if size not equal, means there are some new messages\n                minConsumedTimes = 0;\n            }\n        } else {\n            minConsumedTimes = 0;\n        }\n\n        // for compatibility\n        // the old pop sdk use queueId to get consumedTimes from orderCountInfo\n        ExtraInfoUtil.buildQueueIdOrderCountInfo(orderInfoBuilder, topic, queueId, minConsumedTimes);\n        updateLockFreeTimestamp(topic, group, queueId, orderInfo);\n    }\n\n    @Override\n    public void update(String attemptId, boolean isRetry, String topic, String group, int queueId, long popTime,\n        long invisibleTime,\n        List<Long> msgQueueOffsetList, StringBuilder orderInfoBuilder, GetMessageResult getMessageResult) {\n        update(attemptId, isRetry, topic, group, queueId, popTime, invisibleTime, msgQueueOffsetList, orderInfoBuilder);\n    }\n\n    @Override\n    public boolean checkBlock(String attemptId, String topic, String group, int queueId, long invisibleTime) {\n        String key = buildKey(topic, group);\n        ConcurrentHashMap<Integer/*queueId*/, OrderInfo> qs = table.get(key);\n        if (qs == null) {\n            qs = new ConcurrentHashMap<>(16);\n            ConcurrentHashMap<Integer/*queueId*/, OrderInfo> old = table.putIfAbsent(key, qs);\n            if (old != null) {\n                qs = old;\n            }\n        }\n\n        OrderInfo orderInfo = qs.get(queueId);\n\n        if (orderInfo == null) {\n            return false;\n        }\n        return orderInfo.needBlock(attemptId, invisibleTime);\n    }\n\n    @Override\n    public void clearBlock(String topic, String group, int queueId) {\n        table.computeIfPresent(buildKey(topic, group), (key, val) -> {\n            val.remove(queueId);\n            return val;\n        });\n    }\n\n    @Override\n    public void remove(String topic, String group) {\n        table.remove(buildKey(topic, group));\n    }\n\n    @Override\n    public int getOrderInfoCount() {\n        return table.size();\n    }\n\n    @Override\n    public OrderedConsumptionLevel getOrderedConsumptionLevel() {\n        return OrderedConsumptionLevel.QUEUE;\n    }\n\n    @Override\n    public void start() {\n    }\n\n    /**\n     * mark message is consumed finished. return the consumer offset\n     *\n     * @param topic topic\n     * @param group group\n     * @param queueId queue id of message\n     * @param queueOffset queue offset of message\n     * @return -1 : illegal, -2 : no need commit, >= 0 : commit\n     */\n    @Override\n    public long commitAndNext(String topic, String group, int queueId, long queueOffset, long popTime) {\n        String key = buildKey(topic, group);\n        ConcurrentHashMap<Integer/*queueId*/, OrderInfo> qs = table.get(key);\n\n        if (qs == null) {\n            return queueOffset + 1;\n        }\n        OrderInfo orderInfo = qs.get(queueId);\n        if (orderInfo == null) {\n            log.warn(\"OrderInfo is null, {}, {}, {}\", key, queueOffset, orderInfo);\n            return queueOffset + 1;\n        }\n\n        List<Long> o = orderInfo.offsetList;\n        if (o == null || o.isEmpty()) {\n            log.warn(\"OrderInfo is empty, {}, {}, {}\", key, queueOffset, orderInfo);\n            return -1;\n        }\n\n        if (popTime != orderInfo.popTime) {\n            log.warn(\"popTime is not equal to orderInfo saved. key: {}, offset: {}, orderInfo: {}, popTime: {}\", key, queueOffset, orderInfo, popTime);\n            return -2;\n        }\n\n        Long first = o.get(0);\n        int i = 0, size = o.size();\n        for (; i < size; i++) {\n            long temp;\n            if (i == 0) {\n                temp = first;\n            } else {\n                temp = first + o.get(i);\n            }\n            if (queueOffset == temp) {\n                break;\n            }\n        }\n        // not found\n        if (i >= size) {\n            log.warn(\"OrderInfo not found commit offset, {}, {}, {}\", key, queueOffset, orderInfo);\n            return -1;\n        }\n        //set bit\n        orderInfo.setCommitOffsetBit(orderInfo.commitOffsetBit | (1L << i));\n        long nextOffset = orderInfo.getNextOffset();\n\n        updateLockFreeTimestamp(topic, group, queueId, orderInfo);\n        return nextOffset;\n    }\n\n    /**\n     * update next visible time of this message\n     *\n     * @param topic topic\n     * @param group group\n     * @param queueId queue id of message\n     * @param queueOffset queue offset of message\n     * @param nextVisibleTime nex visible time\n     */\n    @Override\n    public void updateNextVisibleTime(String topic, String group, int queueId, long queueOffset, long popTime,\n        long nextVisibleTime) {\n        String key = buildKey(topic, group);\n        ConcurrentHashMap<Integer/*queueId*/, OrderInfo> qs = table.get(key);\n\n        if (qs == null) {\n            log.warn(\"orderInfo of queueId is null. key: {}, queueOffset: {}, queueId: {}\", key, queueOffset, queueId);\n            return;\n        }\n        OrderInfo orderInfo = qs.get(queueId);\n        if (orderInfo == null) {\n            log.warn(\"orderInfo is null, key: {}, queueOffset: {}, queueId: {}\", key, queueOffset, queueId);\n            return;\n        }\n        if (popTime != orderInfo.popTime) {\n            log.warn(\"popTime is not equal to orderInfo saved. key: {}, queueOffset: {}, orderInfo: {}, popTime: {}\", key, queueOffset, orderInfo, popTime);\n            return;\n        }\n\n        orderInfo.updateOffsetNextVisibleTime(queueOffset, nextVisibleTime);\n        updateLockFreeTimestamp(topic, group, queueId, orderInfo);\n    }\n\n    @VisibleForTesting\n    protected void autoClean() {\n        if (brokerController == null) {\n            return;\n        }\n        Iterator<Map.Entry<String/* topic@group*/, ConcurrentHashMap<Integer/*queueId*/, OrderInfo>>> iterator =\n            this.table.entrySet().iterator();\n        while (iterator.hasNext()) {\n            Map.Entry<String/* topic@group*/, ConcurrentHashMap<Integer/*queueId*/, OrderInfo>> entry =\n                iterator.next();\n            String topicAtGroup = entry.getKey();\n            ConcurrentHashMap<Integer/*queueId*/, OrderInfo> qs = entry.getValue();\n            String[] arrays = decodeKey(topicAtGroup);\n            if (arrays.length != 2) {\n                continue;\n            }\n            String topic = arrays[0];\n            String group = arrays[1];\n\n            TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);\n            if (topicConfig == null) {\n                iterator.remove();\n                log.info(\"Topic not exist, Clean order info, {}:{}\", topicAtGroup, qs);\n                continue;\n            }\n\n            if (!this.brokerController.getSubscriptionGroupManager().containsSubscriptionGroup(group)) {\n                iterator.remove();\n                log.info(\"Group not exist, Clean order info, {}:{}\", topicAtGroup, qs);\n                continue;\n            }\n\n            if (qs.isEmpty()) {\n                iterator.remove();\n                log.info(\"Order table is empty, Clean order info, {}:{}\", topicAtGroup, qs);\n                continue;\n            }\n\n            Iterator<Map.Entry<Integer/*queueId*/, OrderInfo>> qsIterator = qs.entrySet().iterator();\n            while (qsIterator.hasNext()) {\n                Map.Entry<Integer/*queueId*/, OrderInfo> qsEntry = qsIterator.next();\n\n                if (qsEntry.getKey() >= topicConfig.getReadQueueNums()) {\n                    qsIterator.remove();\n                    log.info(\"Queue not exist, Clean order info, {}:{}, {}\", topicAtGroup, entry.getValue(), topicConfig);\n                    continue;\n                }\n\n                if (System.currentTimeMillis() - qsEntry.getValue().getLastConsumeTimestamp() > CLEAN_SPAN_FROM_LAST) {\n                    qsIterator.remove();\n                    log.info(\"Not consume long time, Clean order info, {}:{}, {}\", topicAtGroup, entry.getValue(), topicConfig);\n                }\n            }\n        }\n    }\n\n    @Override\n    public String encode() {\n        return this.encode(false);\n    }\n\n    @Override\n    public String configFilePath() {\n        if (brokerController != null) {\n            return BrokerPathConfigHelper.getConsumerOrderInfoPath(this.brokerController.getMessageStoreConfig().getStorePathRootDir());\n        } else {\n            return BrokerPathConfigHelper.getConsumerOrderInfoPath(\"~\");\n        }\n    }\n\n    @Override\n    public void decode(String jsonString) {\n        if (jsonString != null) {\n            QueueLevelConsumerManager obj = RemotingSerializable.fromJson(jsonString, QueueLevelConsumerManager.class);\n            if (obj != null) {\n                this.table = obj.table;\n                if (this.queueLevelConsumerOrderInfoLockManager != null) {\n                    this.queueLevelConsumerOrderInfoLockManager.recover(this.table);\n                }\n            }\n        }\n    }\n\n    @Override\n    public String encode(boolean prettyFormat) {\n        this.autoClean();\n        return RemotingSerializable.toJson(this, prettyFormat);\n    }\n\n    public void shutdown() {\n        if (this.queueLevelConsumerOrderInfoLockManager != null) {\n            this.queueLevelConsumerOrderInfoLockManager.shutdown();\n        }\n    }\n\n    @Override\n    public CompletableFuture<GetMessageResult> getAvailableMessageResult(String attemptId, long popTime, long invisibleTime,\n        String groupId, String topicId, int queueId, int batchSize, StringBuilder orderCountInfoBuilder) {\n        return CompletableFuture.completedFuture(null);\n    }\n\n    @VisibleForTesting\n    protected QueueLevelConsumerOrderInfoLockManager getConsumerOrderInfoLockManager() {\n        return queueLevelConsumerOrderInfoLockManager;\n    }\n\n    public static class OrderInfo {\n        private long popTime;\n        /**\n         * the invisibleTime when pop message\n         */\n        @JSONField(name = \"i\")\n        private Long invisibleTime;\n        /**\n         * offset\n         * offsetList[0] is the queue offset of message\n         * offsetList[i] (i > 0) is the distance between current message and offsetList[0]\n         */\n        @JSONField(name = \"o\")\n        private List<Long> offsetList;\n        /**\n         * next visible timestamp for message\n         * key: message queue offset\n         */\n        @JSONField(name = \"ot\")\n        private Map<Long, Long> offsetNextVisibleTime;\n        /**\n         * message consumed count for offset\n         * key: message queue offset\n         */\n        @JSONField(name = \"oc\")\n        private Map<Long, Integer> offsetConsumedCount;\n        /**\n         * last consume timestamp\n         */\n        @JSONField(name = \"l\")\n        private long lastConsumeTimestamp;\n        /**\n         * commit offset bit\n         */\n        @JSONField(name = \"cm\")\n        private long commitOffsetBit;\n        @JSONField(name = \"a\")\n        private String attemptId;\n\n        public OrderInfo() {\n        }\n\n        public OrderInfo(String attemptId, long popTime, long invisibleTime, List<Long> queueOffsetList,\n            long lastConsumeTimestamp,\n            long commitOffsetBit) {\n            this.popTime = popTime;\n            this.invisibleTime = invisibleTime;\n            this.offsetList = buildOffsetList(queueOffsetList);\n            this.lastConsumeTimestamp = lastConsumeTimestamp;\n            this.commitOffsetBit = commitOffsetBit;\n            this.attemptId = attemptId;\n        }\n\n        public List<Long> getOffsetList() {\n            return offsetList;\n        }\n\n        public void setOffsetList(List<Long> offsetList) {\n            this.offsetList = offsetList;\n        }\n\n        public long getLastConsumeTimestamp() {\n            return lastConsumeTimestamp;\n        }\n\n        public void setLastConsumeTimestamp(long lastConsumeTimestamp) {\n            this.lastConsumeTimestamp = lastConsumeTimestamp;\n        }\n\n        public long getCommitOffsetBit() {\n            return commitOffsetBit;\n        }\n\n        public void setCommitOffsetBit(long commitOffsetBit) {\n            this.commitOffsetBit = commitOffsetBit;\n        }\n\n        public long getPopTime() {\n            return popTime;\n        }\n\n        public void setPopTime(long popTime) {\n            this.popTime = popTime;\n        }\n\n        public Long getInvisibleTime() {\n            return invisibleTime;\n        }\n\n        public void setInvisibleTime(Long invisibleTime) {\n            this.invisibleTime = invisibleTime;\n        }\n\n        public Map<Long, Long> getOffsetNextVisibleTime() {\n            return offsetNextVisibleTime;\n        }\n\n        public void setOffsetNextVisibleTime(Map<Long, Long> offsetNextVisibleTime) {\n            this.offsetNextVisibleTime = offsetNextVisibleTime;\n        }\n\n        public Map<Long, Integer> getOffsetConsumedCount() {\n            return offsetConsumedCount;\n        }\n\n        public void setOffsetConsumedCount(Map<Long, Integer> offsetConsumedCount) {\n            this.offsetConsumedCount = offsetConsumedCount;\n        }\n\n        public String getAttemptId() {\n            return attemptId;\n        }\n\n        public void setAttemptId(String attemptId) {\n            this.attemptId = attemptId;\n        }\n\n        public static List<Long> buildOffsetList(List<Long> queueOffsetList) {\n            List<Long> simple = new ArrayList<>();\n            if (queueOffsetList.size() == 1) {\n                simple.addAll(queueOffsetList);\n                return simple;\n            }\n            Long first = queueOffsetList.get(0);\n            simple.add(first);\n            for (int i = 1; i < queueOffsetList.size(); i++) {\n                simple.add(queueOffsetList.get(i) - first);\n            }\n            return simple;\n        }\n\n        @JSONField(serialize = false, deserialize = false)\n        public boolean needBlock(String attemptId, long currentInvisibleTime) {\n            if (offsetList == null || offsetList.isEmpty()) {\n                return false;\n            }\n            if (this.attemptId != null && this.attemptId.equals(attemptId)) {\n                return false;\n            }\n            int num = offsetList.size();\n            int i = 0;\n            if (this.invisibleTime == null || this.invisibleTime <= 0) {\n                this.invisibleTime = currentInvisibleTime;\n            }\n            long currentTime = System.currentTimeMillis();\n            for (; i < num; i++) {\n                if (isNotAck(i)) {\n                    long nextVisibleTime = popTime + invisibleTime;\n                    if (offsetNextVisibleTime != null) {\n                        Long time = offsetNextVisibleTime.get(this.getQueueOffset(i));\n                        if (time != null) {\n                            nextVisibleTime = time;\n                        }\n                    }\n                    if (currentTime < nextVisibleTime) {\n                        return true;\n                    }\n                }\n            }\n            return false;\n        }\n\n        @JSONField(serialize = false, deserialize = false)\n        public Long getLockFreeTimestamp() {\n            if (offsetList == null || offsetList.isEmpty()) {\n                return null;\n            }\n            int num = offsetList.size();\n            int i = 0;\n            long currentTime = System.currentTimeMillis();\n            for (; i < num; i++) {\n                if (isNotAck(i)) {\n                    if (invisibleTime == null || invisibleTime <= 0) {\n                        return null;\n                    }\n                    long nextVisibleTime = popTime + invisibleTime;\n                    if (offsetNextVisibleTime != null) {\n                        Long time = offsetNextVisibleTime.get(this.getQueueOffset(i));\n                        if (time != null) {\n                            nextVisibleTime = time;\n                        }\n                    }\n                    if (currentTime < nextVisibleTime) {\n                        return nextVisibleTime;\n                    }\n                }\n            }\n            return currentTime;\n        }\n\n        @JSONField(serialize = false, deserialize = false)\n        public Long getMaxLockFreeTimestamp() {\n            if (offsetList == null || offsetList.isEmpty()) {\n                return null;\n            }\n            int num = offsetList.size();\n            long maxTime = System.currentTimeMillis();\n            for (int i = 0; i < num; i++) {\n                if (isNotAck(i)) {\n                    if (invisibleTime == null || invisibleTime <= 0) {\n                        return null;\n                    }\n                    long nextVisibleTime = popTime + invisibleTime;\n                    if (offsetNextVisibleTime != null) {\n                        Long time = offsetNextVisibleTime.get(this.getQueueOffset(i));\n                        if (time != null) {\n                            nextVisibleTime = time;\n                        }\n                    }\n                    if (maxTime < nextVisibleTime) {\n                        maxTime = nextVisibleTime;\n                    }\n                }\n            }\n            return maxTime;\n        }\n\n        @JSONField(serialize = false, deserialize = false)\n        public void updateOffsetNextVisibleTime(long queueOffset, long nextVisibleTime) {\n            if (this.offsetNextVisibleTime == null) {\n                this.offsetNextVisibleTime = new HashMap<>();\n            }\n            this.offsetNextVisibleTime.put(queueOffset, nextVisibleTime);\n        }\n\n        @JSONField(serialize = false, deserialize = false)\n        public long getNextOffset() {\n            if (offsetList == null || offsetList.isEmpty()) {\n                return -2;\n            }\n            int num = offsetList.size();\n            int i = 0;\n            for (; i < num; i++) {\n                if (isNotAck(i)) {\n                    break;\n                }\n            }\n            if (i == num) {\n                // all ack\n                return getQueueOffset(num - 1) + 1;\n            }\n            return getQueueOffset(i);\n        }\n\n        /**\n         * convert the offset at the index of offsetList to queue offset\n         *\n         * @param offsetIndex the index of offsetList\n         * @return queue offset of message\n         */\n        @JSONField(serialize = false, deserialize = false)\n        public long getQueueOffset(int offsetIndex) {\n            return getQueueOffset(this.offsetList, offsetIndex);\n        }\n\n        protected static long getQueueOffset(List<Long> offsetList, int offsetIndex) {\n            if (offsetIndex == 0) {\n                return offsetList.get(0);\n            }\n            return offsetList.get(0) + offsetList.get(offsetIndex);\n        }\n\n        @JSONField(serialize = false, deserialize = false)\n        public boolean isNotAck(int offsetIndex) {\n            return (commitOffsetBit & (1L << offsetIndex)) == 0;\n        }\n\n        /**\n         * calculate message consumed count of each message, and put nonzero value into offsetConsumedCount\n         *\n         * @param prevOffsetConsumedCount the offset list of message\n         */\n        @JSONField(serialize = false, deserialize = false)\n        public void mergeOffsetConsumedCount(String preAttemptId, List<Long> preOffsetList,\n            Map<Long, Integer> prevOffsetConsumedCount) {\n            Map<Long, Integer> offsetConsumedCount = new HashMap<>();\n            if (prevOffsetConsumedCount == null) {\n                prevOffsetConsumedCount = new HashMap<>();\n            }\n            if (preAttemptId != null && preAttemptId.equals(this.attemptId)) {\n                this.offsetConsumedCount = prevOffsetConsumedCount;\n                return;\n            }\n            Set<Long> preQueueOffsetSet = new HashSet<>();\n            for (int i = 0; i < preOffsetList.size(); i++) {\n                preQueueOffsetSet.add(getQueueOffset(preOffsetList, i));\n            }\n            for (int i = 0; i < offsetList.size(); i++) {\n                long queueOffset = this.getQueueOffset(i);\n                if (preQueueOffsetSet.contains(queueOffset)) {\n                    int count = 1;\n                    Integer preCount = prevOffsetConsumedCount.get(queueOffset);\n                    if (preCount != null) {\n                        count = preCount + 1;\n                    }\n                    offsetConsumedCount.put(queueOffset, count);\n                }\n            }\n            this.offsetConsumedCount = offsetConsumedCount;\n        }\n\n        @Override\n        public String toString() {\n            return MoreObjects.toStringHelper(this)\n                .add(\"popTime\", popTime)\n                .add(\"invisibleTime\", invisibleTime)\n                .add(\"offsetList\", offsetList)\n                .add(\"offsetNextVisibleTime\", offsetNextVisibleTime)\n                .add(\"offsetConsumedCount\", offsetConsumedCount)\n                .add(\"lastConsumeTimestamp\", lastConsumeTimestamp)\n                .add(\"commitOffsetBit\", commitOffsetBit)\n                .add(\"attemptId\", attemptId)\n                .toString();\n        }\n    }\n}"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/pop/orderly/QueueLevelConsumerOrderInfoLockManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.pop.orderly;\n\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.base.MoreObjects;\nimport com.google.common.base.Objects;\nimport io.netty.util.HashedWheelTimer;\nimport io.netty.util.Timeout;\nimport io.netty.util.Timer;\nimport io.netty.util.TimerTask;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class QueueLevelConsumerOrderInfoLockManager {\n    private static final Logger POP_LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);\n    private ConsumerOrderInfoManager consumerOrderInfoManager;\n\n    private final BrokerController brokerController;\n    private final Map<Key, Timeout> timeoutMap = new ConcurrentHashMap<>();\n    private final Timer timer;\n    private static final int TIMER_TICK_MS = 100;\n\n    public QueueLevelConsumerOrderInfoLockManager(BrokerController brokerController) {\n        this.brokerController = brokerController;\n        this.timer = new HashedWheelTimer(\n            new ThreadFactoryImpl(\"ConsumerOrderInfoLockManager_\"),\n            TIMER_TICK_MS, TimeUnit.MILLISECONDS);\n    }\n\n    /**\n     * when QueueLevelConsumerManager load from disk, recover data\n     */\n    public void recover(Map<String/* topic@group*/, ConcurrentHashMap<Integer/*queueId*/, QueueLevelConsumerManager.OrderInfo>> table) {\n        if (!this.brokerController.getBrokerConfig().isEnableNotifyAfterPopOrderLockRelease()) {\n            return;\n        }\n        for (Map.Entry<String, ConcurrentHashMap<Integer, QueueLevelConsumerManager.OrderInfo>> entry : table.entrySet()) {\n            String topicAtGroup = entry.getKey();\n            ConcurrentHashMap<Integer/*queueId*/, QueueLevelConsumerManager.OrderInfo> qs = entry.getValue();\n            String[] arrays = QueueLevelConsumerManager.decodeKey(topicAtGroup);\n            if (arrays.length != 2) {\n                continue;\n            }\n            String topic = arrays[0];\n            String group = arrays[1];\n            for (Map.Entry<Integer, QueueLevelConsumerManager.OrderInfo> qsEntry : qs.entrySet()) {\n                Long lockFreeTimestamp = qsEntry.getValue().getLockFreeTimestamp();\n                if (lockFreeTimestamp == null || lockFreeTimestamp <= System.currentTimeMillis()) {\n                    continue;\n                }\n                this.updateLockFreeTimestamp(topic, group, qsEntry.getKey(), lockFreeTimestamp);\n            }\n        }\n    }\n\n    public void updateLockFreeTimestamp(String topic, String group, int queueId, QueueLevelConsumerManager.OrderInfo orderInfo) {\n        this.updateLockFreeTimestamp(topic, group, queueId, orderInfo.getLockFreeTimestamp());\n    }\n\n    public void updateLockFreeTimestamp(String topic, String group, int queueId, Long lockFreeTimestamp) {\n        if (!this.brokerController.getBrokerConfig().isEnableNotifyAfterPopOrderLockRelease()) {\n            return;\n        }\n        if (lockFreeTimestamp == null) {\n            return;\n        }\n        try {\n            this.timeoutMap.compute(new Key(topic, group, queueId), (key, oldTimeout) -> {\n                try {\n                    long delay = lockFreeTimestamp - System.currentTimeMillis();\n                    Timeout newTimeout = this.timer.newTimeout(new NotifyLockFreeTimerTask(key), delay, TimeUnit.MILLISECONDS);\n                    if (oldTimeout != null) {\n                        // cancel prev timerTask\n                        oldTimeout.cancel();\n                    }\n                    return newTimeout;\n                } catch (Exception e) {\n                    POP_LOGGER.warn(\"add timeout task failed. key:{}, lockFreeTimestamp:{}\", key, lockFreeTimestamp, e);\n                    return oldTimeout;\n                }\n            });\n        } catch (Exception e) {\n            POP_LOGGER.error(\"unexpect error when updateLockFreeTimestamp. topic:{}, group:{}, queueId:{}, lockFreeTimestamp:{}\",\n                topic, group, queueId, lockFreeTimestamp, e);\n        }\n    }\n\n    protected void notifyLockIsFree(Key key) {\n        try {\n            if (LiteUtil.isLiteTopicQueue(key.topic)) {\n                this.brokerController.getLiteEventDispatcher().dispatch(key.group, key.topic, key.queueId, -1, -1);\n                return;\n            }\n            this.brokerController.getPopMessageProcessor().notifyLongPollingRequestIfNeed(key.topic, key.group, key.queueId);\n        } catch (Exception e) {\n            POP_LOGGER.error(\"unexpect error when notifyLockIsFree. key:{}\", key, e);\n        }\n    }\n\n    public void shutdown() {\n        this.timer.stop();\n    }\n\n    @VisibleForTesting\n    protected Map<Key, Timeout> getTimeoutMap() {\n        return timeoutMap;\n    }\n\n    private class NotifyLockFreeTimerTask implements TimerTask {\n\n        private final Key key;\n\n        private NotifyLockFreeTimerTask(Key key) {\n            this.key = key;\n        }\n\n        @Override\n        public void run(Timeout timeout) throws Exception {\n            if (timeout.isCancelled() || !brokerController.getBrokerConfig().isEnableNotifyAfterPopOrderLockRelease()) {\n                return;\n            }\n            notifyLockIsFree(key);\n            timeoutMap.computeIfPresent(key, (key1, curTimeout) -> {\n                if (curTimeout == timeout) {\n                    // remove from map\n                    return null;\n                }\n                return curTimeout;\n            });\n        }\n    }\n\n    private static class Key {\n        private final String topic;\n        private final String group;\n        private final int queueId;\n\n        public Key(String topic, String group, int queueId) {\n            this.topic = topic;\n            this.group = group;\n            this.queueId = queueId;\n        }\n\n        @Override\n        public boolean equals(Object o) {\n            if (this == o) {\n                return true;\n            }\n            if (o == null || getClass() != o.getClass()) {\n                return false;\n            }\n            Key key = (Key) o;\n            return queueId == key.queueId && Objects.equal(topic, key.topic) && Objects.equal(group, key.group);\n        }\n\n        @Override\n        public int hashCode() {\n            return Objects.hashCode(topic, group, queueId);\n        }\n\n        @Override\n        public String toString() {\n            return MoreObjects.toStringHelper(this)\n                .add(\"topic\", topic)\n                .add(\"group\", group)\n                .add(\"queueId\", queueId)\n                .toString();\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport io.opentelemetry.api.common.Attributes;\nimport java.net.SocketAddress;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Optional;\nimport java.util.Random;\nimport java.util.concurrent.ThreadLocalRandom;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.metrics.BrokerMetricsManager;\nimport org.apache.rocketmq.broker.mqtrace.ConsumeMessageContext;\nimport org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook;\nimport org.apache.rocketmq.broker.mqtrace.SendMessageContext;\nimport org.apache.rocketmq.broker.mqtrace.SendMessageHook;\nimport org.apache.rocketmq.common.AbortProcessException;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.TopicFilterType;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.DBMsgConstants;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.help.FAQUrl;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.message.MessageType;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.common.sysflag.TopicSysFlag;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingAbstract;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\n\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_TOPIC;\n\npublic abstract class AbstractSendMessageProcessor implements NettyRequestProcessor {\n    protected static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    protected static final Logger DLQ_LOG = LoggerFactory.getLogger(LoggerName.DLQ_LOGGER_NAME);\n\n    protected List<ConsumeMessageHook> consumeMessageHookList;\n\n    protected final static int DLQ_NUMS_PER_GROUP = 1;\n    protected final BrokerController brokerController;\n    protected final Random random = new Random(System.currentTimeMillis());\n    private List<SendMessageHook> sendMessageHookList;\n\n    public AbstractSendMessageProcessor(final BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    public void registerConsumeMessageHook(List<ConsumeMessageHook> consumeMessageHookList) {\n        this.consumeMessageHookList = consumeMessageHookList;\n    }\n\n    protected RemotingCommand consumerSendMsgBack(final ChannelHandlerContext ctx, final RemotingCommand request)\n        throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final ConsumerSendMsgBackRequestHeader requestHeader =\n            (ConsumerSendMsgBackRequestHeader) request.decodeCommandCustomHeader(ConsumerSendMsgBackRequestHeader.class);\n\n        // The send back requests sent to SlaveBroker will be forwarded to the master broker beside\n        final BrokerController masterBroker = this.brokerController.peekMasterBroker();\n        if (null == masterBroker) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"no master available along with \" + brokerController.getBrokerConfig().getBrokerIP1());\n            return response;\n        }\n\n        // The broker that received the request.\n        // It may be a master broker or a slave broker\n        final BrokerController currentBroker = this.brokerController;\n\n        SubscriptionGroupConfig subscriptionGroupConfig =\n            masterBroker.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getGroup());\n        if (null == subscriptionGroupConfig) {\n            response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);\n            response.setRemark(\"subscription group not exist, \" + requestHeader.getGroup() + \" \"\n                + FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST));\n            return response;\n        }\n\n        BrokerConfig masterBrokerConfig = masterBroker.getBrokerConfig();\n        if (!PermName.isWriteable(masterBrokerConfig.getBrokerPermission())) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            response.setRemark(\"the broker[\" + masterBrokerConfig.getBrokerIP1() + \"] sending message is forbidden\");\n            return response;\n        }\n\n        if (subscriptionGroupConfig.getRetryQueueNums() <= 0) {\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n            return response;\n        }\n\n        String newTopic = MixAll.getRetryTopic(requestHeader.getGroup());\n        int queueIdInt = this.random.nextInt(subscriptionGroupConfig.getRetryQueueNums());\n\n        int topicSysFlag = 0;\n        if (requestHeader.isUnitMode()) {\n            topicSysFlag = TopicSysFlag.buildSysFlag(false, true);\n        }\n\n        // Create retry topic to master broker\n        TopicConfig topicConfig = masterBroker.getTopicConfigManager().createTopicInSendMessageBackMethod(\n            newTopic,\n            subscriptionGroupConfig.getRetryQueueNums(),\n            PermName.PERM_WRITE | PermName.PERM_READ, topicSysFlag);\n        if (null == topicConfig) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"topic[\" + newTopic + \"] not exist\");\n            return response;\n        }\n\n        if (!PermName.isWriteable(topicConfig.getPerm())) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            response.setRemark(String.format(\"the topic[%s] sending message is forbidden\", newTopic));\n            return response;\n        }\n\n        // Look message from the origin message store\n        MessageExt msgExt = currentBroker.getMessageStore().lookMessageByOffset(requestHeader.getOffset());\n        if (null == msgExt) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"look message by offset failed, \" + requestHeader.getOffset());\n            return response;\n        }\n\n        final String retryTopic = msgExt.getProperty(MessageConst.PROPERTY_RETRY_TOPIC);\n        if (null == retryTopic) {\n            MessageAccessor.putProperty(msgExt, MessageConst.PROPERTY_RETRY_TOPIC, msgExt.getTopic());\n        }\n        msgExt.setWaitStoreMsgOK(false);\n\n        int delayLevel = requestHeader.getDelayLevel();\n\n        int maxReconsumeTimes = subscriptionGroupConfig.getRetryMaxTimes();\n        if (request.getVersion() >= MQVersion.Version.V3_4_9.ordinal()) {\n            Integer times = requestHeader.getMaxReconsumeTimes();\n            if (times != null) {\n                maxReconsumeTimes = times;\n            }\n        }\n\n        boolean isDLQ = false;\n        if (msgExt.getReconsumeTimes() >= maxReconsumeTimes\n            || delayLevel < 0) {\n\n            Attributes attributes = this.brokerController.getBrokerMetricsManager().newAttributesBuilder()\n                .put(LABEL_CONSUMER_GROUP, requestHeader.getGroup())\n                .put(LABEL_TOPIC, requestHeader.getOriginTopic())\n                .put(LABEL_IS_SYSTEM, BrokerMetricsManager.isSystem(requestHeader.getOriginTopic(), requestHeader.getGroup()))\n                .build();\n            this.brokerController.getBrokerMetricsManager().getSendToDlqMessages().add(1, attributes);\n\n            isDLQ = true;\n            newTopic = MixAll.getDLQTopic(requestHeader.getGroup());\n            queueIdInt = randomQueueId(DLQ_NUMS_PER_GROUP);\n\n            // Create DLQ topic to master broker\n            topicConfig = masterBroker.getTopicConfigManager().createTopicInSendMessageBackMethod(newTopic,\n                DLQ_NUMS_PER_GROUP,\n                PermName.PERM_WRITE | PermName.PERM_READ, 0);\n\n            if (null == topicConfig) {\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"topic[\" + newTopic + \"] not exist\");\n                return response;\n            }\n            msgExt.setDelayTimeLevel(0);\n        } else {\n            if (0 == delayLevel) {\n                delayLevel = 3 + msgExt.getReconsumeTimes();\n            }\n\n            msgExt.setDelayTimeLevel(delayLevel);\n        }\n\n        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n        msgInner.setTopic(newTopic);\n        msgInner.setBody(msgExt.getBody());\n        msgInner.setFlag(msgExt.getFlag());\n        MessageAccessor.setProperties(msgInner, msgExt.getProperties());\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties()));\n        msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(null, msgExt.getTags()));\n\n        msgInner.setQueueId(queueIdInt);\n        msgInner.setSysFlag(msgExt.getSysFlag());\n        msgInner.setBornTimestamp(msgExt.getBornTimestamp());\n        msgInner.setBornHost(msgExt.getBornHost());\n        msgInner.setStoreHost(this.getStoreHost());\n        msgInner.setReconsumeTimes(msgExt.getReconsumeTimes() + 1);\n\n        String originMsgId = MessageAccessor.getOriginMessageId(msgExt);\n        MessageAccessor.setOriginMessageId(msgInner, UtilAll.isBlank(originMsgId) ? msgExt.getMsgId() : originMsgId);\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties()));\n\n        boolean succeeded = false;\n\n        // Put retry topic to master message store\n        PutMessageResult putMessageResult = masterBroker.getMessageStore().putMessage(msgInner);\n        if (putMessageResult != null) {\n            String commercialOwner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER);\n\n            switch (putMessageResult.getPutMessageStatus()) {\n                case PUT_OK:\n                    String backTopic = msgExt.getTopic();\n                    String correctTopic = msgExt.getProperty(MessageConst.PROPERTY_RETRY_TOPIC);\n                    if (correctTopic != null) {\n                        backTopic = correctTopic;\n                    }\n                    if (TopicValidator.RMQ_SYS_SCHEDULE_TOPIC.equals(msgInner.getTopic())) {\n                        masterBroker.getBrokerStatsManager().incTopicPutNums(msgInner.getTopic());\n                        masterBroker.getBrokerStatsManager().incTopicPutSize(msgInner.getTopic(), putMessageResult.getAppendMessageResult().getWroteBytes());\n                        masterBroker.getBrokerStatsManager().incQueuePutNums(msgInner.getTopic(), msgInner.getQueueId());\n                        masterBroker.getBrokerStatsManager().incQueuePutSize(msgInner.getTopic(), msgInner.getQueueId(), putMessageResult.getAppendMessageResult().getWroteBytes());\n                    }\n                    masterBroker.getBrokerStatsManager().incSendBackNums(requestHeader.getGroup(), backTopic);\n\n                    if (isDLQ) {\n                        masterBroker.getBrokerStatsManager().incDLQStatValue(\n                            BrokerStatsManager.SNDBCK2DLQ_TIMES,\n                            commercialOwner,\n                            requestHeader.getGroup(),\n                            requestHeader.getOriginTopic(),\n                            BrokerStatsManager.StatsType.SEND_BACK_TO_DLQ.name(),\n                            1);\n\n                        String uniqKey = msgInner.getProperties().get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX);\n                        DLQ_LOG.info(\"send msg to DLQ {}, owner={}, originalTopic={}, consumerId={}, msgUniqKey={}, storeTimestamp={}\",\n                            newTopic,\n                            commercialOwner,\n                            requestHeader.getOriginTopic(),\n                            requestHeader.getGroup(),\n                            uniqKey,\n                            putMessageResult.getAppendMessageResult().getStoreTimestamp());\n                    }\n\n                    response.setCode(ResponseCode.SUCCESS);\n                    response.setRemark(null);\n\n                    succeeded = true;\n                    break;\n                default:\n                    break;\n            }\n\n            if (!succeeded) {\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(putMessageResult.getPutMessageStatus().name());\n            }\n        } else {\n            if (isDLQ) {\n                String owner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER);\n                String uniqKey = msgInner.getProperties().get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX);\n                DLQ_LOG.info(\"failed to send msg to DLQ {}, owner={}, originalTopic={}, consumerId={}, msgUniqKey={}, result={}\",\n                    newTopic,\n                    owner,\n                    requestHeader.getOriginTopic(),\n                    requestHeader.getGroup(),\n                    uniqKey,\n                    \"null\");\n            }\n\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"putMessageResult is null\");\n        }\n\n        if (this.hasConsumeMessageHook() && !UtilAll.isBlank(requestHeader.getOriginMsgId())) {\n            String namespace = NamespaceUtil.getNamespaceFromResource(requestHeader.getGroup());\n            ConsumeMessageContext context = new ConsumeMessageContext();\n            context.setNamespace(namespace);\n            context.setTopic(requestHeader.getOriginTopic());\n            context.setConsumerGroup(requestHeader.getGroup());\n            context.setCommercialRcvStats(BrokerStatsManager.StatsType.SEND_BACK);\n            context.setCommercialRcvTimes(1);\n            context.setCommercialOwner(request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER));\n\n            context.setAccountAuthType(request.getExtFields().get(BrokerStatsManager.ACCOUNT_AUTH_TYPE));\n            context.setAccountOwnerParent(request.getExtFields().get(BrokerStatsManager.ACCOUNT_OWNER_PARENT));\n            context.setAccountOwnerSelf(request.getExtFields().get(BrokerStatsManager.ACCOUNT_OWNER_SELF));\n            context.setRcvStat(isDLQ ? BrokerStatsManager.StatsType.SEND_BACK_TO_DLQ : BrokerStatsManager.StatsType.SEND_BACK);\n            context.setSuccess(succeeded);\n            context.setRcvMsgNum(1);\n            //Set msg body size 0 when sent back by consumer.\n            context.setRcvMsgSize(0);\n            context.setCommercialRcvMsgNum(succeeded ? 1 : 0);\n\n            try {\n                this.executeConsumeMessageHookAfter(context);\n            } catch (AbortProcessException e) {\n                response.setCode(e.getResponseCode());\n                response.setRemark(e.getErrorMessage());\n            }\n        }\n\n        return response;\n    }\n\n    public boolean hasConsumeMessageHook() {\n        return consumeMessageHookList != null && !this.consumeMessageHookList.isEmpty();\n    }\n\n    public void executeConsumeMessageHookAfter(final ConsumeMessageContext context) {\n        if (hasConsumeMessageHook()) {\n            for (ConsumeMessageHook hook : this.consumeMessageHookList) {\n                try {\n                    hook.consumeMessageAfter(context);\n                } catch (Throwable e) {\n                    // Ignore\n                }\n            }\n        }\n    }\n\n    protected SendMessageContext buildMsgContext(ChannelHandlerContext ctx,\n        SendMessageRequestHeader requestHeader, RemotingCommand request) {\n        String namespace = NamespaceUtil.getNamespaceFromResource(requestHeader.getTopic());\n\n        SendMessageContext sendMessageContext;\n        sendMessageContext = new SendMessageContext();\n        sendMessageContext.setNamespace(namespace);\n        sendMessageContext.setProducerGroup(requestHeader.getProducerGroup());\n        sendMessageContext.setTopic(requestHeader.getTopic());\n        sendMessageContext.setBodyLength(request.getBody().length);\n        sendMessageContext.setMsgProps(requestHeader.getProperties());\n        sendMessageContext.setBornHost(RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n        sendMessageContext.setBrokerAddr(this.brokerController.getBrokerAddr());\n        sendMessageContext.setQueueId(requestHeader.getQueueId());\n        sendMessageContext.setBrokerRegionId(this.brokerController.getBrokerConfig().getRegionId());\n        sendMessageContext.setBornTimeStamp(requestHeader.getBornTimestamp());\n        sendMessageContext.setRequestTimeStamp(System.currentTimeMillis());\n\n        String owner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER);\n        sendMessageContext.setCommercialOwner(owner);\n\n        Map<String, String> properties = MessageDecoder.string2messageProperties(requestHeader.getProperties());\n        properties.put(MessageConst.PROPERTY_MSG_REGION, this.brokerController.getBrokerConfig().getRegionId());\n        properties.put(MessageConst.PROPERTY_TRACE_SWITCH, String.valueOf(this.brokerController.getBrokerConfig().isTraceOn()));\n        requestHeader.setProperties(MessageDecoder.messageProperties2String(properties));\n\n        String uniqueKey = properties.get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX);\n        sendMessageContext.setMsgUniqueKey(Optional.ofNullable(uniqueKey).orElse(\"\"));\n\n        if (properties.containsKey(MessageConst.PROPERTY_SHARDING_KEY)) {\n            sendMessageContext.setMsgType(MessageType.Order_Msg);\n        } else if (properties.containsKey(MessageConst.PROPERTY_DELAY_TIME_LEVEL)\n                || properties.containsKey(MessageConst.PROPERTY_TIMER_DELIVER_MS)\n                || properties.containsKey(MessageConst.PROPERTY_TIMER_DELAY_SEC)\n                || properties.containsKey(MessageConst.PROPERTY_TIMER_DELAY_MS)) {\n            sendMessageContext.setMsgType(MessageType.Delay_Msg);\n        } else if (Boolean.parseBoolean(properties.get(MessageConst.PROPERTY_TRANSACTION_PREPARED))) {\n            sendMessageContext.setMsgType(MessageType.Trans_Msg_Half);\n        } else {\n            sendMessageContext.setMsgType(MessageType.Normal_Msg);\n        }\n        return sendMessageContext;\n    }\n\n    public boolean hasSendMessageHook() {\n        return sendMessageHookList != null && !this.sendMessageHookList.isEmpty();\n    }\n\n    protected MessageExtBrokerInner buildInnerMsg(final ChannelHandlerContext ctx,\n        final SendMessageRequestHeader requestHeader, final byte[] body, TopicConfig topicConfig) {\n        int queueIdInt = requestHeader.getQueueId();\n        if (queueIdInt < 0) {\n            queueIdInt = randomQueueId(topicConfig.getWriteQueueNums());\n        }\n        int sysFlag = requestHeader.getSysFlag();\n\n        if (TopicFilterType.MULTI_TAG == topicConfig.getTopicFilterType()) {\n            sysFlag |= MessageSysFlag.MULTI_TAGS_FLAG;\n        }\n\n        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n        msgInner.setTopic(requestHeader.getTopic());\n        msgInner.setBody(body);\n        msgInner.setFlag(requestHeader.getFlag());\n        MessageAccessor.setProperties(msgInner,\n            MessageDecoder.string2messageProperties(requestHeader.getProperties()));\n        msgInner.setPropertiesString(requestHeader.getProperties());\n        msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(topicConfig.getTopicFilterType(),\n            msgInner.getTags()));\n\n        msgInner.setQueueId(queueIdInt);\n        msgInner.setSysFlag(sysFlag);\n        msgInner.setBornTimestamp(requestHeader.getBornTimestamp());\n        msgInner.setBornHost(ctx.channel().remoteAddress());\n        msgInner.setStoreHost(this.getStoreHost());\n        msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader\n            .getReconsumeTimes());\n        return msgInner;\n    }\n\n    public SocketAddress getStoreHost() {\n        return brokerController.getStoreHost();\n    }\n\n    protected RemotingCommand msgContentCheck(final ChannelHandlerContext ctx,\n        final SendMessageRequestHeader requestHeader, RemotingCommand request,\n        final RemotingCommand response) {\n        String topic = requestHeader.getTopic();\n        if (topic.length() > Byte.MAX_VALUE) {\n            LOGGER.warn(\"msgContentCheck: message topic length is too long, topic={}, topic length={}, threshold={}\",\n                topic, topic.length(), Byte.MAX_VALUE);\n            response.setCode(ResponseCode.MESSAGE_ILLEGAL);\n            return response;\n        }\n        if (requestHeader.getProperties() != null && requestHeader.getProperties().length() > Short.MAX_VALUE) {\n            LOGGER.warn(\n                \"msgContentCheck: message properties length is too long, topic={}, properties length={}, threshold={}\",\n                topic, requestHeader.getProperties().length(), Short.MAX_VALUE);\n            response.setCode(ResponseCode.MESSAGE_ILLEGAL);\n            return response;\n        }\n        if (request.getBody().length > DBMsgConstants.MAX_BODY_SIZE) {\n            LOGGER.warn(\n                \"msgContentCheck: message body size exceeds the threshold, topic={}, body size={}, threshold={}bytes\",\n                topic, request.getBody().length, DBMsgConstants.MAX_BODY_SIZE);\n            response.setRemark(\"msg body must be less 64KB\");\n            response.setCode(ResponseCode.MESSAGE_ILLEGAL);\n            return response;\n        }\n        return response;\n    }\n\n    protected RemotingCommand msgCheck(final ChannelHandlerContext ctx,\n        final SendMessageRequestHeader requestHeader, final RemotingCommand request,\n        final RemotingCommand response) {\n        if (!PermName.isWriteable(this.brokerController.getBrokerConfig().getBrokerPermission())\n            && this.brokerController.getTopicConfigManager().isOrderTopic(requestHeader.getTopic())) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            response.setRemark(\"the broker[\" + this.brokerController.getBrokerConfig().getBrokerIP1()\n                + \"] sending message is forbidden\");\n            return response;\n        }\n\n        TopicValidator.ValidateResult result = TopicValidator.validateTopic(requestHeader.getTopic());\n        if (!result.isValid()) {\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(result.getRemark());\n            return response;\n        }\n        if (TopicValidator.isNotAllowedSendTopic(requestHeader.getTopic())) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            response.setRemark(\"Sending message to topic[\" + requestHeader.getTopic() + \"] is forbidden.\");\n            return response;\n        }\n\n        TopicConfig topicConfig =\n            this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());\n        if (null == topicConfig) {\n            int topicSysFlag = 0;\n            if (requestHeader.isUnitMode()) {\n                if (requestHeader.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n                    topicSysFlag = TopicSysFlag.buildSysFlag(false, true);\n                } else {\n                    topicSysFlag = TopicSysFlag.buildSysFlag(true, false);\n                }\n            }\n\n            LOGGER.warn(\"the topic {} not exist, producer: {}\", requestHeader.getTopic(), ctx.channel().remoteAddress());\n            topicConfig = this.brokerController.getTopicConfigManager().createTopicInSendMessageMethod(\n                requestHeader.getTopic(),\n                requestHeader.getDefaultTopic(),\n                RemotingHelper.parseChannelRemoteAddr(ctx.channel()),\n                requestHeader.getDefaultTopicQueueNums(), topicSysFlag);\n\n            if (null == topicConfig) {\n                if (requestHeader.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n                    topicConfig =\n                        this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod(\n                            requestHeader.getTopic(), 1, PermName.PERM_WRITE | PermName.PERM_READ,\n                            topicSysFlag);\n                }\n            }\n\n            if (null == topicConfig) {\n                response.setCode(ResponseCode.TOPIC_NOT_EXIST);\n                response.setRemark(\"topic[\" + requestHeader.getTopic() + \"] not exist, apply first please!\"\n                    + FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL));\n                return response;\n            }\n        }\n\n        int queueIdInt = requestHeader.getQueueId();\n        int idValid = Math.max(topicConfig.getWriteQueueNums(), topicConfig.getReadQueueNums());\n        if (queueIdInt >= idValid) {\n            String errorInfo = String.format(\"request queueId[%d] is illegal, %s Producer: %s\",\n                queueIdInt,\n                topicConfig,\n                RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n\n            LOGGER.warn(errorInfo);\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(errorInfo);\n\n            return response;\n        }\n        return response;\n    }\n\n    public void registerSendMessageHook(List<SendMessageHook> sendMessageHookList) {\n        this.sendMessageHookList = sendMessageHookList;\n    }\n\n    protected void doResponse(ChannelHandlerContext ctx, RemotingCommand request,\n        final RemotingCommand response) {\n        NettyRemotingAbstract.writeResponse(ctx.channel(), request, response, null, brokerController.getBrokerMetricsManager().getRemotingMetricsManager());\n    }\n\n    public void executeSendMessageHookBefore(SendMessageContext context) {\n        if (hasSendMessageHook()) {\n            for (SendMessageHook hook : this.sendMessageHookList) {\n                try {\n                    hook.sendMessageBefore(context);\n                } catch (AbortProcessException e) {\n                    throw e;\n                } catch (Throwable e) {\n                    //ignore\n                }\n            }\n        }\n    }\n\n    protected SendMessageRequestHeader parseRequestHeader(RemotingCommand request) throws RemotingCommandException {\n        return SendMessageRequestHeader.parseRequestHeader(request);\n    }\n\n    protected int randomQueueId(int writeQueueNums) {\n        return ThreadLocalRandom.current().nextInt(99999999) % writeQueueNums;\n    }\n\n    public void executeSendMessageHookAfter(final RemotingCommand response, final SendMessageContext context) {\n        if (hasSendMessageHook()) {\n            for (SendMessageHook hook : this.sendMessageHookList) {\n                try {\n                    if (response != null) {\n                        final SendMessageResponseHeader responseHeader =\n                            (SendMessageResponseHeader) response.readCustomHeader();\n                        context.setMsgId(responseHeader.getMsgId());\n                        context.setQueueId(responseHeader.getQueueId());\n                        context.setQueueOffset(responseHeader.getQueueOffset());\n                        context.setCode(response.getCode());\n                        context.setErrorMsg(response.getRemark());\n                    }\n                    hook.sendMessageAfter(context);\n                } catch (Throwable e) {\n                    //ignore\n                }\n            }\n        }\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport com.alibaba.fastjson2.JSON;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.nio.charset.StandardCharsets;\nimport java.util.BitSet;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.lite.LiteMetadataUtil;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.broker.pop.PopConsumerLockService;\nimport org.apache.rocketmq.broker.pop.orderly.ConsumerOrderInfoManager;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.PopAckConstants;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.help.FAQUrl;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.BatchAck;\nimport org.apache.rocketmq.remoting.protocol.body.BatchAckMessageRequestBody;\nimport org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.pop.AckMsg;\nimport org.apache.rocketmq.store.pop.BatchAckMsg;\n\npublic class AckMessageProcessor implements NettyRequestProcessor {\n\n    private static final Logger POP_LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);\n    private final BrokerController brokerController;\n    private final String reviveTopic;\n    private final PopReviveService[] popReviveServices;\n\n    public AckMessageProcessor(final BrokerController brokerController) {\n        this.brokerController = brokerController;\n        this.reviveTopic = PopAckConstants.buildClusterReviveTopic(\n            this.brokerController.getBrokerConfig().getBrokerClusterName());\n        this.popReviveServices = new PopReviveService[this.brokerController.getBrokerConfig().getReviveQueueNum()];\n        for (int i = 0; i < this.brokerController.getBrokerConfig().getReviveQueueNum(); i++) {\n            this.popReviveServices[i] = new PopReviveService(brokerController, reviveTopic, i);\n            this.popReviveServices[i].setShouldRunPopRevive(brokerController.getBrokerConfig().getBrokerId() == 0);\n        }\n    }\n\n    public PopReviveService[] getPopReviveServices() {\n        return popReviveServices;\n    }\n\n    public void shutdown() throws Exception {\n        for (PopReviveService popReviveService : popReviveServices) {\n            popReviveService.shutdown();\n        }\n    }\n\n    public void startPopReviveService() {\n        for (PopReviveService popReviveService : popReviveServices) {\n            popReviveService.start();\n        }\n    }\n\n    public void shutdownPopReviveService() {\n        for (PopReviveService popReviveService : popReviveServices) {\n            popReviveService.shutdown();\n        }\n    }\n\n    public void setPopReviveServiceStatus(boolean shouldStart) {\n        for (PopReviveService popReviveService : popReviveServices) {\n            popReviveService.setShouldRunPopRevive(shouldStart);\n        }\n    }\n\n    public boolean isPopReviveServiceRunning() {\n        for (PopReviveService popReviveService : popReviveServices) {\n            if (popReviveService.isShouldRunPopRevive()) {\n                return true;\n            }\n        }\n\n        return false;\n    }\n\n    @Override\n    public RemotingCommand processRequest(final ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        return this.processRequest(ctx.channel(), request, true);\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n\n    private RemotingCommand processRequest(final Channel channel, RemotingCommand request,\n        boolean brokerAllowSuspend) throws RemotingCommandException {\n        AckMessageRequestHeader requestHeader;\n        BatchAckMessageRequestBody reqBody = null;\n        final RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, null);\n        response.setOpaque(request.getOpaque());\n        if (request.getCode() == RequestCode.ACK_MESSAGE) {\n            requestHeader = (AckMessageRequestHeader) request.decodeCommandCustomHeader(AckMessageRequestHeader.class);\n\n            TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());\n            if (null == topicConfig) {\n                POP_LOGGER.error(\"The topic {} not exist, consumer: {} \", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel));\n                response.setCode(ResponseCode.TOPIC_NOT_EXIST);\n                response.setRemark(String.format(\"topic[%s] not exist, apply first please! %s\", requestHeader.getTopic(), FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)));\n                return response;\n            }\n\n            if (requestHeader.getQueueId() >= topicConfig.getReadQueueNums() || requestHeader.getQueueId() < 0) {\n                String errorInfo = String.format(\"queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]\",\n                    requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());\n                POP_LOGGER.warn(errorInfo);\n                response.setCode(ResponseCode.MESSAGE_ILLEGAL);\n                response.setRemark(errorInfo);\n                return response;\n            }\n\n            RemotingCommand ackLiteResponse = ackLite(requestHeader, null, response, channel);\n            if (ackLiteResponse != null) {\n                return ackLiteResponse;\n            }\n\n            long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());\n            long maxOffset;\n            try {\n                maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());\n            } catch (ConsumeQueueException e) {\n                throw new RemotingCommandException(\"Failed to get max offset\", e);\n            }\n            if (requestHeader.getOffset() < minOffset || requestHeader.getOffset() > maxOffset) {\n                String errorInfo = String.format(\"offset is illegal, key:%s@%d, commit:%d, store:%d~%d\",\n                    requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getOffset(), minOffset, maxOffset);\n                POP_LOGGER.warn(errorInfo);\n                response.setCode(ResponseCode.NO_MESSAGE);\n                response.setRemark(errorInfo);\n                return response;\n            }\n            if (brokerController.getBrokerConfig().isPopConsumerKVServiceEnable()) {\n                appendAckNew(requestHeader, null, response, channel, null);\n            } else {\n                appendAck(requestHeader, null, response, channel, null);\n            }\n        } else if (request.getCode() == RequestCode.BATCH_ACK_MESSAGE) {\n            if (request.getBody() != null) {\n                reqBody = BatchAckMessageRequestBody.decode(request.getBody(), BatchAckMessageRequestBody.class);\n            }\n            if (reqBody == null || reqBody.getAcks() == null || reqBody.getAcks().isEmpty()) {\n                response.setCode(ResponseCode.NO_MESSAGE);\n                return response;\n            }\n            for (BatchAck bAck : reqBody.getAcks()) {\n                if (brokerController.getBrokerConfig().isPopConsumerKVServiceEnable()) {\n                    appendAckNew(null, bAck, response, channel, reqBody.getBrokerName());\n                } else {\n                    appendAck(null, bAck, response, channel, reqBody.getBrokerName());\n                }\n            }\n        } else {\n            POP_LOGGER.error(\"AckMessageProcessor failed to process RequestCode: {}, consumer: {} \", request.getCode(), RemotingHelper.parseChannelRemoteAddr(channel));\n            response.setCode(ResponseCode.MESSAGE_ILLEGAL);\n            response.setRemark(String.format(\"AckMessageProcessor failed to process RequestCode: %d\", request.getCode()));\n            return response;\n        }\n        return response;\n    }\n\n    private void appendAck(final AckMessageRequestHeader requestHeader, final BatchAck batchAck,\n        final RemotingCommand response, final Channel channel, String brokerName) throws RemotingCommandException {\n        String[] extraInfo;\n        String consumeGroup, topic;\n        int qId, rqId;\n        long startOffset, ackOffset;\n        long popTime, invisibleTime;\n        AckMsg ackMsg;\n        int ackCount = 0;\n        if (batchAck == null) {\n            // single ack\n            extraInfo = ExtraInfoUtil.split(requestHeader.getExtraInfo());\n            brokerName = ExtraInfoUtil.getBrokerName(extraInfo);\n            consumeGroup = requestHeader.getConsumerGroup();\n            topic = requestHeader.getTopic();\n            qId = requestHeader.getQueueId();\n            rqId = ExtraInfoUtil.getReviveQid(extraInfo);\n            startOffset = ExtraInfoUtil.getCkQueueOffset(extraInfo);\n            ackOffset = requestHeader.getOffset();\n            popTime = ExtraInfoUtil.getPopTime(extraInfo);\n            invisibleTime = ExtraInfoUtil.getInvisibleTime(extraInfo);\n\n            if (rqId == KeyBuilder.POP_ORDER_REVIVE_QUEUE) {\n                ackOrderly(topic, consumeGroup, qId, ackOffset, popTime, invisibleTime, channel, response);\n                return;\n            }\n\n            ackMsg = new AckMsg();\n            ackCount = 1;\n        } else {\n            // batch ack\n            consumeGroup = batchAck.getConsumerGroup();\n            topic = ExtraInfoUtil.getRealTopic(batchAck.getTopic(), batchAck.getConsumerGroup(), batchAck.getRetry());\n            qId = batchAck.getQueueId();\n            rqId = batchAck.getReviveQueueId();\n            startOffset = batchAck.getStartOffset();\n            ackOffset = -1;\n            popTime = batchAck.getPopTime();\n            invisibleTime = batchAck.getInvisibleTime();\n\n            long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, qId);\n            long maxOffset;\n            try {\n                maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, qId);\n            } catch (ConsumeQueueException e) {\n                throw new RemotingCommandException(\"Failed to get max offset in queue\", e);\n            }\n            if (minOffset == -1 || maxOffset == -1) {\n                POP_LOGGER.error(\"Illegal topic or queue found when batch ack {}\", batchAck);\n                return;\n            }\n\n            BatchAckMsg batchAckMsg = new BatchAckMsg();\n            BitSet bitSet = batchAck.getBitSet();\n            for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) {\n                if (i == Integer.MAX_VALUE) {\n                    break;\n                }\n                long offset = startOffset + i;\n                if (offset < minOffset || offset > maxOffset) {\n                    continue;\n                }\n                if (rqId == KeyBuilder.POP_ORDER_REVIVE_QUEUE) {\n                    ackOrderly(topic, consumeGroup, qId, offset, popTime, invisibleTime, channel, response);\n                } else {\n                    batchAckMsg.getAckOffsetList().add(offset);\n                }\n            }\n            if (rqId == KeyBuilder.POP_ORDER_REVIVE_QUEUE || batchAckMsg.getAckOffsetList().isEmpty()) {\n                return;\n            }\n\n            ackMsg = batchAckMsg;\n            ackCount = batchAckMsg.getAckOffsetList().size();\n        }\n\n        this.brokerController.getBrokerStatsManager().incBrokerAckNums(ackCount);\n        this.brokerController.getBrokerStatsManager().incGroupAckNums(consumeGroup, topic, ackCount);\n\n        ackMsg.setConsumerGroup(consumeGroup);\n        ackMsg.setTopic(topic);\n        ackMsg.setQueueId(qId);\n        ackMsg.setStartOffset(startOffset);\n        ackMsg.setAckOffset(ackOffset);\n        ackMsg.setPopTime(popTime);\n        ackMsg.setBrokerName(brokerName);\n\n        if (this.brokerController.getPopMessageProcessor().getPopBufferMergeService().addAk(rqId, ackMsg)) {\n            brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(topic, consumeGroup, popTime, qId, ackCount);\n            return;\n        }\n\n        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n        msgInner.setTopic(reviveTopic);\n        msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(StandardCharsets.UTF_8));\n        msgInner.setQueueId(rqId);\n        if (ackMsg instanceof BatchAckMsg) {\n            msgInner.setTags(PopAckConstants.BATCH_ACK_TAG);\n            msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genBatchAckUniqueId((BatchAckMsg) ackMsg));\n        } else {\n            msgInner.setTags(PopAckConstants.ACK_TAG);\n            msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genAckUniqueId(ackMsg));\n        }\n        msgInner.setBornTimestamp(System.currentTimeMillis());\n        msgInner.setBornHost(this.brokerController.getStoreHost());\n        msgInner.setStoreHost(this.brokerController.getStoreHost());\n        msgInner.setDeliverTimeMs(popTime + invisibleTime);\n        msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genAckUniqueId(ackMsg));\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));\n        if (brokerController.getBrokerConfig().isAppendAckAsync()) {\n            int finalAckCount = ackCount;\n            this.brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenAccept(putMessageResult -> {\n                handlePutMessageResult(putMessageResult, ackMsg, topic, consumeGroup, popTime, qId, finalAckCount);\n            }).exceptionally(throwable -> {\n                handlePutMessageResult(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, null, false),\n                    ackMsg, topic, consumeGroup, popTime, qId, finalAckCount);\n                POP_LOGGER.error(\"put ack msg error \", throwable);\n                return null;\n            });\n        } else {\n            PutMessageResult putMessageResult = this.brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);\n            handlePutMessageResult(putMessageResult, ackMsg, topic, consumeGroup, popTime, qId, ackCount);\n        }\n    }\n\n    private void appendAckNew(final AckMessageRequestHeader requestHeader, final BatchAck batchAck,\n        final RemotingCommand response, final Channel channel, String brokerName) throws RemotingCommandException {\n\n        if (requestHeader != null && batchAck == null) {\n            String[] extraInfo = ExtraInfoUtil.split(requestHeader.getExtraInfo());\n            String groupId = requestHeader.getConsumerGroup();\n            String topicId = requestHeader.getTopic();\n            int queueId = requestHeader.getQueueId();\n            long ackOffset = requestHeader.getOffset();\n            long popTime = ExtraInfoUtil.getPopTime(extraInfo);\n            long invisibleTime = ExtraInfoUtil.getInvisibleTime(extraInfo);\n\n            int reviveQueueId = ExtraInfoUtil.getReviveQid(extraInfo);\n            if (reviveQueueId == KeyBuilder.POP_ORDER_REVIVE_QUEUE) {\n                ackOrderlyNew(topicId, groupId, queueId, ackOffset, popTime, invisibleTime, channel, response);\n            } else {\n                this.brokerController.getPopConsumerService().ackAsync(\n                    popTime, invisibleTime, groupId, topicId, queueId, ackOffset);\n            }\n\n            this.brokerController.getBrokerStatsManager().incBrokerAckNums(1);\n            this.brokerController.getBrokerStatsManager().incGroupAckNums(groupId, topicId, 1);\n        } else {\n            String groupId = batchAck.getConsumerGroup();\n            String topicId = ExtraInfoUtil.getRealTopic(\n                batchAck.getTopic(), batchAck.getConsumerGroup(), batchAck.getRetry());\n            int queueId = batchAck.getQueueId();\n            int reviveQueueId = batchAck.getReviveQueueId();\n            long startOffset = batchAck.getStartOffset();\n            long popTime = batchAck.getPopTime();\n            long invisibleTime = batchAck.getInvisibleTime();\n\n            try {\n                long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(topicId, queueId);\n                long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topicId, queueId);\n                if (minOffset == -1 || maxOffset == -1) {\n                    POP_LOGGER.error(\"Illegal topic or queue found when batch ack {}\", batchAck);\n                    return;\n                }\n\n                int ackCount = 0;\n                // Maintain consistency with the old implementation code style\n                BitSet bitSet = batchAck.getBitSet();\n                for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) {\n                    if (i == Integer.MAX_VALUE) {\n                        break;\n                    }\n                    long offset = startOffset + i;\n                    if (offset < minOffset || offset > maxOffset) {\n                        continue;\n                    }\n                    if (reviveQueueId == KeyBuilder.POP_ORDER_REVIVE_QUEUE) {\n                        ackOrderlyNew(topicId, groupId, queueId, offset, popTime, invisibleTime, channel, response);\n                    } else {\n                        this.brokerController.getPopConsumerService().ackAsync(\n                            popTime, invisibleTime, groupId, topicId, queueId, offset);\n                    }\n                    ackCount++;\n                }\n\n                this.brokerController.getBrokerStatsManager().incBrokerAckNums(ackCount);\n                this.brokerController.getBrokerStatsManager().incGroupAckNums(groupId, topicId, ackCount);\n            } catch (ConsumeQueueException e) {\n                throw new RemotingCommandException(\"Failed to ack message\", e);\n            }\n        }\n    }\n\n    private void handlePutMessageResult(PutMessageResult putMessageResult, AckMsg ackMsg, String topic,\n        String consumeGroup, long popTime, int qId, int ackCount) {\n        if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK\n            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT\n            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT\n            && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {\n            POP_LOGGER.error(\"put ack msg error:\" + putMessageResult);\n        }\n        brokerController.getBrokerMetricsManager().getPopMetricsManager().incPopReviveAckPutCount(ackMsg, putMessageResult.getPutMessageStatus());\n        brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(topic, consumeGroup, popTime, qId, ackCount);\n    }\n\n    protected void ackOrderly(String topic, String consumeGroup, int qId, long ackOffset, long popTime,\n        long invisibleTime, Channel channel, RemotingCommand response) {\n        String lockKey = topic + PopAckConstants.SPLIT + consumeGroup + PopAckConstants.SPLIT + qId;\n        long oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId);\n        if (ackOffset < oldOffset) {\n            return;\n        }\n        while (!this.brokerController.getPopMessageProcessor().getQueueLockManager().tryLock(lockKey)) {\n        }\n        try {\n            oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId);\n            if (ackOffset < oldOffset) {\n                return;\n            }\n            long nextOffset = brokerController.getConsumerOrderInfoManager().commitAndNext(\n                topic, consumeGroup, qId, ackOffset, popTime);\n            if (nextOffset > -1) {\n                if (!this.brokerController.getConsumerOffsetManager().hasOffsetReset(topic, consumeGroup, qId)) {\n                    this.brokerController.getConsumerOffsetManager().commitOffset(\n                        channel.remoteAddress().toString(), consumeGroup, topic, qId, nextOffset);\n                }\n                if (!this.brokerController.getConsumerOrderInfoManager().checkBlock(null, topic, consumeGroup, qId, invisibleTime)) {\n                    this.brokerController.getPopMessageProcessor().notifyMessageArriving(topic, qId, consumeGroup);\n                }\n            } else if (nextOffset == -1) {\n                String errorInfo = String.format(\"offset is illegal, key:%s, old:%d, commit:%d, next:%d, %s\",\n                    lockKey, oldOffset, ackOffset, nextOffset, channel.remoteAddress());\n                POP_LOGGER.warn(errorInfo);\n                response.setCode(ResponseCode.MESSAGE_ILLEGAL);\n                response.setRemark(errorInfo);\n                return;\n            }\n        } finally {\n            this.brokerController.getPopMessageProcessor().getQueueLockManager().unLock(lockKey);\n        }\n        brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(topic, consumeGroup, popTime, qId, 1);\n    }\n\n    protected void ackOrderlyNew(String topic, String consumeGroup, int qId, long ackOffset, long popTime,\n        long invisibleTime, Channel channel, RemotingCommand response) {\n\n        ConsumerOffsetManager consumerOffsetManager = this.brokerController.getConsumerOffsetManager();\n        ConsumerOrderInfoManager consumerOrderInfoManager = brokerController.getConsumerOrderInfoManager();\n        PopConsumerLockService consumerLockService = this.brokerController.getPopConsumerService().getConsumerLockService();\n\n        long oldOffset = consumerOffsetManager.queryOffset(consumeGroup, topic, qId);\n        if (ackOffset < oldOffset) {\n            return;\n        }\n\n        while (!consumerLockService.tryLock(consumeGroup, topic)) {\n        }\n\n        try {\n            // double check\n            oldOffset = consumerOffsetManager.queryOffset(consumeGroup, topic, qId);\n            if (ackOffset < oldOffset) {\n                return;\n            }\n\n            long nextOffset = consumerOrderInfoManager.commitAndNext(topic, consumeGroup, qId, ackOffset, popTime);\n            if (brokerController.getBrokerConfig().isPopConsumerKVServiceLog()) {\n                POP_LOGGER.info(\"PopConsumerService ack orderly, time={}, topicId={}, groupId={}, queueId={}, \" +\n                    \"offset={}, next={}\", popTime, topic, consumeGroup, qId, ackOffset, nextOffset);\n            }\n\n            if (nextOffset > -1L) {\n                if (!consumerOffsetManager.hasOffsetReset(topic, consumeGroup, qId)) {\n                    String remoteAddress = RemotingHelper.parseSocketAddressAddr(channel.remoteAddress());\n                    consumerOffsetManager.commitOffset(remoteAddress, consumeGroup, topic, qId, nextOffset);\n                }\n                if (!consumerOrderInfoManager.checkBlock(null, topic, consumeGroup, qId, invisibleTime)) {\n                    this.brokerController.getPopMessageProcessor().notifyMessageArriving(topic, qId, consumeGroup);\n                }\n                return;\n            }\n\n            if (nextOffset == -1) {\n                String errorInfo = String.format(\"offset is illegal, key:%s %s %s, old:%d, commit:%d, next:%d, %s\",\n                    consumeGroup, topic, qId, oldOffset, ackOffset, nextOffset, channel.remoteAddress());\n                POP_LOGGER.warn(errorInfo);\n                response.setCode(ResponseCode.MESSAGE_ILLEGAL);\n                response.setRemark(errorInfo);\n            }\n        } finally {\n            consumerLockService.unlock(consumeGroup, topic);\n        }\n    }\n\n    /**\n     * Currently, batch ack for lite messages is not supported, so we should ensure that all acknowledgements are individual.\n     */\n    protected RemotingCommand ackLite(AckMessageRequestHeader requestHeader, BatchAckMessageRequestBody batchAckBody,\n        final RemotingCommand response, final Channel channel) {\n        if (batchAckBody != null) {\n            POP_LOGGER.warn(\"bad request, batch ack lite, {}\", batchAckBody);\n            response.setCode(ResponseCode.ILLEGAL_OPERATION);\n            response.setRemark(\"batch ack lite is not supported.\");\n            return response;\n        }\n        if (StringUtils.isBlank(requestHeader.getLiteTopic())) {\n            return null;\n        }\n        String group = requestHeader.getConsumerGroup();\n        if (!requestHeader.getTopic().equals(LiteMetadataUtil.getLiteBindTopic(group, brokerController))) {\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(\"group type or bind topic not match.\");\n            return response;\n        }\n\n        String lmqName = LiteUtil.toLmqName(requestHeader.getTopic(), requestHeader.getLiteTopic());\n        long ackOffset = requestHeader.getOffset();\n        long maxOffset = this.brokerController.getLiteLifecycleManager().getMaxOffsetInQueue(lmqName);\n        if (ackOffset > maxOffset) {\n            POP_LOGGER.warn(\"ack lite offset illegal, {}, {}, {}\", lmqName, ackOffset, maxOffset);\n            response.setCode(ResponseCode.NO_MESSAGE);\n            response.setRemark(\"ack offset illegal.\");\n            return response;\n        }\n        String[] extraInfo = ExtraInfoUtil.split(requestHeader.getExtraInfo());\n        if (requestHeader.getQueueId() != 0\n            || ExtraInfoUtil.getReviveQid(extraInfo) != KeyBuilder.POP_ORDER_REVIVE_QUEUE) {\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(\"ack queue illegal.\");\n            return response;\n        }\n\n        long popTime = ExtraInfoUtil.getPopTime(extraInfo);\n        long invisibleTime = ExtraInfoUtil.getInvisibleTime(extraInfo);\n\n        ConsumerOffsetManager consumerOffsetManager = this.brokerController.getConsumerOffsetManager();\n        ConsumerOrderInfoManager consumerOrderInfoManager =\n            brokerController.getPopLiteMessageProcessor().getConsumerOrderInfoManager();\n        PopConsumerLockService consumerLockService = this.brokerController.getPopLiteMessageProcessor().getLockService();\n\n        long oldOffset = consumerOffsetManager.queryOffset(group, lmqName, 0);\n        if (ackOffset < oldOffset) {\n            return response;\n        }\n        String lockKey = KeyBuilder.buildPopLiteLockKey(group, lmqName);\n        while (!consumerLockService.tryLock(lockKey)) {\n        }\n\n        try {\n            oldOffset = consumerOffsetManager.queryOffset(group, lmqName, 0);\n            if (ackOffset < oldOffset) {\n                return response;\n            }\n            long nextOffset = consumerOrderInfoManager.commitAndNext(lmqName, group, 0, ackOffset, popTime);\n            if (nextOffset > -1L) {\n                if (!consumerOffsetManager.hasOffsetReset(lmqName, group, 0)) {\n                    consumerOffsetManager.commitOffset(\"AckLiteHost\", group, lmqName, 0, nextOffset);\n                }\n                if (!consumerOrderInfoManager.checkBlock(null, lmqName, group, 0, invisibleTime)) {\n                    this.brokerController.getLiteEventDispatcher().dispatch(group, lmqName, 0, nextOffset, -1);\n                }\n            }\n            if (nextOffset == -1) {\n                POP_LOGGER.warn(\"ack lite, nextOffset illegal. lmq:{}, old:{}, commit:{}\", lmqName, oldOffset, ackOffset);\n                response.setCode(ResponseCode.MESSAGE_ILLEGAL);\n                response.setRemark(\"ack offset illegal.\");\n                return response;\n            }\n        } finally {\n            consumerLockService.unlock(lockKey);\n        }\n\n        this.brokerController.getBrokerStatsManager().incBrokerAckNums(1);\n        this.brokerController.getBrokerStatsManager().incGroupAckNums(group, requestHeader.getTopic(), 1);\n        response.setCode(ResponseCode.SUCCESS);\n        return response;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.alibaba.fastjson2.JSONObject;\nimport com.alibaba.fastjson2.JSONWriter;\nimport com.google.common.collect.Sets;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.opentelemetry.api.common.Attributes;\nimport java.io.UnsupportedEncodingException;\nimport java.net.UnknownHostException;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Optional;\nimport java.util.Properties;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.CompletionException;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.SynchronousQueue;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.ObjectUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.auth.authentication.enums.UserType;\nimport org.apache.rocketmq.auth.authentication.exception.AuthenticationException;\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.auth.authorization.enums.PolicyType;\nimport org.apache.rocketmq.auth.authorization.exception.AuthorizationException;\nimport org.apache.rocketmq.auth.authorization.model.Acl;\nimport org.apache.rocketmq.auth.authorization.model.Resource;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.auth.converter.AclConverter;\nimport org.apache.rocketmq.broker.auth.converter.UserConverter;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ConsumerGroupInfo;\nimport org.apache.rocketmq.broker.config.v1.RocksDBConsumerOffsetManager;\nimport org.apache.rocketmq.broker.config.v1.RocksDBSubscriptionGroupManager;\nimport org.apache.rocketmq.broker.config.v1.RocksDBTopicConfigManager;\nimport org.apache.rocketmq.broker.controller.ReplicasManager;\nimport org.apache.rocketmq.broker.filter.ConsumerFilterData;\nimport org.apache.rocketmq.broker.filter.ExpressionMessageFilter;\nimport org.apache.rocketmq.broker.lite.LiteMetadataUtil;\nimport org.apache.rocketmq.broker.metrics.InvocationStatus;\nimport org.apache.rocketmq.broker.plugin.BrokerAttachedPlugin;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.broker.topic.TopicQueueMappingManager;\nimport org.apache.rocketmq.broker.transaction.queue.TransactionalMessageUtil;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.CheckRocksdbCqWriteResult;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.LockCallback;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicAttributes;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.UnlockCallback;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.attribute.AttributeParser;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.ConsumeInitMode;\nimport org.apache.rocketmq.common.constant.FIleReadaheadMode;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.message.MessageId;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.stats.StatsItem;\nimport org.apache.rocketmq.common.stats.StatsSnapshot;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.common.utils.ExceptionUtils;\nimport org.apache.rocketmq.filter.util.BitsArray;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.admin.ConsumeStats;\nimport org.apache.rocketmq.remoting.protocol.admin.OffsetWrapper;\nimport org.apache.rocketmq.remoting.protocol.admin.TopicOffset;\nimport org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;\nimport org.apache.rocketmq.remoting.protocol.body.AclInfo;\nimport org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;\nimport org.apache.rocketmq.remoting.protocol.body.BrokerStatsData;\nimport org.apache.rocketmq.remoting.protocol.body.BrokerStatsItem;\nimport org.apache.rocketmq.remoting.protocol.body.Connection;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeQueueData;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerConnection;\nimport org.apache.rocketmq.remoting.protocol.body.CreateTopicListRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.EpochEntryCache;\nimport org.apache.rocketmq.remoting.protocol.body.GroupList;\nimport org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;\nimport org.apache.rocketmq.remoting.protocol.body.KVTable;\nimport org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.LockBatchResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.ProducerConnection;\nimport org.apache.rocketmq.remoting.protocol.body.ProducerTableInfo;\nimport org.apache.rocketmq.remoting.protocol.body.QueryConsumeQueueResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.QueryConsumeTimeSpanBody;\nimport org.apache.rocketmq.remoting.protocol.body.QueryCorrectionOffsetBody;\nimport org.apache.rocketmq.remoting.protocol.body.QuerySubscriptionResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan;\nimport org.apache.rocketmq.remoting.protocol.body.ResetOffsetBody;\nimport org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupList;\nimport org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.SyncStateSet;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.TopicList;\nimport org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.UserInfo;\nimport org.apache.rocketmq.remoting.protocol.header.CheckRocksdbCqWriteProgressRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.CloneGroupOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.CreateAclRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.CreateTopicRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.CreateUserRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.DeleteAclRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.DeleteSubscriptionGroupRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.DeleteTopicRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.DeleteUserRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ExchangeHAInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ExchangeHAInfoResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ExportRocksDBConfigToJsonRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetAclRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetAllProducerInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetAllSubscriptionGroupRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetAllSubscriptionGroupResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetAllTopicConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetAllTopicConfigResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetBrokerConfigResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumeStatsInBrokerHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumeStatsRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerConnectionListRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerStatusRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetEarliestMsgStoretimeRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetEarliestMsgStoretimeResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMinOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMinOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetProducerConnectionListRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetSubscriptionGroupConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetTopicConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetTopicStatsInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetUserRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ListAclsRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ListUsersRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.NotifyBrokerRoleChangedRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.NotifyMinBrokerIdChangeRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumeQueueRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumeTimeSpanRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryCorrectionOffsetHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QuerySubscriptionByConsumerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryTopicConsumeByWhoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryTopicsByConsumerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ResetMasterFlushOffsetHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ResumeCheckHalfMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SearchOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SearchOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateAclRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateGroupForbiddenRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateUserRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ViewBrokerStatsDataRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.statictopic.LogicQueueMappingItem;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicConfigAndQueueMapping;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingContext;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingDetail;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingUtils;\nimport org.apache.rocketmq.remoting.protocol.subscription.GroupForbidden;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.remoting.rpc.RpcClientUtils;\nimport org.apache.rocketmq.remoting.rpc.RpcException;\nimport org.apache.rocketmq.remoting.rpc.RpcRequest;\nimport org.apache.rocketmq.remoting.rpc.RpcResponse;\nimport org.apache.rocketmq.store.ConsumeQueueExt;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.MessageFilter;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.StoreType;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.plugin.AbstractPluginMessageStore;\nimport org.apache.rocketmq.store.queue.CombineConsumeQueueStore;\nimport org.apache.rocketmq.store.queue.ConsumeQueueInterface;\nimport org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface;\nimport org.apache.rocketmq.store.queue.CqUnit;\nimport org.apache.rocketmq.store.queue.ReferredIterator;\nimport org.apache.rocketmq.store.timer.TimerCheckpoint;\nimport org.apache.rocketmq.store.timer.TimerMessageStore;\nimport org.apache.rocketmq.store.util.LibC;\n\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_INVOCATION_STATUS;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM;\nimport static org.apache.rocketmq.common.message.MessageConst.TIMER_ENGINE_TYPE;\nimport static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse;\n\npublic class AdminBrokerProcessor implements NettyRequestProcessor {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    protected final BrokerController brokerController;\n    protected Set<String> configBlackList = new HashSet<>();\n    private final ExecutorService asyncExecuteWorker = new ThreadPoolExecutor(0, 4, 60L, TimeUnit.SECONDS, new SynchronousQueue<>());\n\n    public AdminBrokerProcessor(final BrokerController brokerController) {\n        this.brokerController = brokerController;\n        initConfigBlackList();\n    }\n\n    private void initConfigBlackList() {\n        configBlackList.add(\"brokerConfigPath\");\n        configBlackList.add(\"rocketmqHome\");\n        configBlackList.add(\"configBlackList\");\n        String[] configArray = brokerController.getBrokerConfig().getConfigBlackList().split(\";\");\n        configBlackList.addAll(Arrays.asList(configArray));\n    }\n\n    @Override\n    public RemotingCommand processRequest(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        switch (request.getCode()) {\n            case RequestCode.UPDATE_AND_CREATE_TOPIC:\n                return this.updateAndCreateTopic(ctx, request);\n            case RequestCode.UPDATE_AND_CREATE_TOPIC_LIST:\n                return this.updateAndCreateTopicList(ctx, request);\n            case RequestCode.DELETE_TOPIC_IN_BROKER:\n                return this.deleteTopic(ctx, request);\n            case RequestCode.GET_ALL_TOPIC_CONFIG:\n                return this.getAllTopicConfig(ctx, request);\n            case RequestCode.GET_TIMER_CHECK_POINT:\n                return this.getTimerCheckPoint(ctx, request);\n            case RequestCode.GET_TIMER_METRICS:\n                return this.getTimerMetrics(ctx, request);\n            case RequestCode.UPDATE_BROKER_CONFIG:\n                return this.updateBrokerConfig(ctx, request);\n            case RequestCode.GET_BROKER_CONFIG:\n                return this.getBrokerConfig(ctx, request);\n            case RequestCode.UPDATE_COLD_DATA_FLOW_CTR_CONFIG:\n                return this.updateColdDataFlowCtrGroupConfig(ctx, request);\n            case RequestCode.REMOVE_COLD_DATA_FLOW_CTR_CONFIG:\n                return this.removeColdDataFlowCtrGroupConfig(ctx, request);\n            case RequestCode.GET_COLD_DATA_FLOW_CTR_INFO:\n                return this.getColdDataFlowCtrInfo(ctx);\n            case RequestCode.SET_COMMITLOG_READ_MODE:\n                return this.setCommitLogReadaheadMode(ctx, request);\n            case RequestCode.SEARCH_OFFSET_BY_TIMESTAMP:\n                return this.searchOffsetByTimestamp(ctx, request);\n            case RequestCode.GET_MAX_OFFSET:\n                return this.getMaxOffset(ctx, request);\n            case RequestCode.GET_MIN_OFFSET:\n                return this.getMinOffset(ctx, request);\n            case RequestCode.GET_EARLIEST_MSG_STORETIME:\n                return this.getEarliestMsgStoretime(ctx, request);\n            case RequestCode.GET_BROKER_RUNTIME_INFO:\n                return this.getBrokerRuntimeInfo(ctx, request);\n            case RequestCode.LOCK_BATCH_MQ:\n                return this.lockBatchMQ(ctx, request);\n            case RequestCode.UNLOCK_BATCH_MQ:\n                return this.unlockBatchMQ(ctx, request);\n            case RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP:\n                return this.updateAndCreateSubscriptionGroup(ctx, request);\n            case RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP_LIST:\n                return this.updateAndCreateSubscriptionGroupList(ctx, request);\n            case RequestCode.GET_ALL_SUBSCRIPTIONGROUP_CONFIG:\n                return this.getAllSubscriptionGroup(ctx, request);\n            case RequestCode.DELETE_SUBSCRIPTIONGROUP:\n                return this.deleteSubscriptionGroup(ctx, request);\n            case RequestCode.GET_TOPIC_STATS_INFO:\n                return this.getTopicStatsInfo(ctx, request);\n            case RequestCode.GET_CONSUMER_CONNECTION_LIST:\n                return this.getConsumerConnectionList(ctx, request);\n            case RequestCode.GET_PRODUCER_CONNECTION_LIST:\n                return this.getProducerConnectionList(ctx, request);\n            case RequestCode.GET_ALL_PRODUCER_INFO:\n                return this.getAllProducerInfo(ctx, request);\n            case RequestCode.GET_CONSUME_STATS:\n                return this.getConsumeStats(ctx, request);\n            case RequestCode.GET_ALL_CONSUMER_OFFSET:\n                return this.getAllConsumerOffset(ctx, request);\n            case RequestCode.GET_ALL_DELAY_OFFSET:\n                return this.getAllDelayOffset(ctx, request);\n            case RequestCode.GET_ALL_MESSAGE_REQUEST_MODE:\n                return this.getAllMessageRequestMode(ctx, request);\n            case RequestCode.INVOKE_BROKER_TO_RESET_OFFSET:\n                return this.resetOffset(ctx, request);\n            case RequestCode.INVOKE_BROKER_TO_GET_CONSUMER_STATUS:\n                return this.getConsumerStatus(ctx, request);\n            case RequestCode.QUERY_TOPIC_CONSUME_BY_WHO:\n                return this.queryTopicConsumeByWho(ctx, request);\n            case RequestCode.QUERY_TOPICS_BY_CONSUMER:\n                return this.queryTopicsByConsumer(ctx, request);\n            case RequestCode.QUERY_SUBSCRIPTION_BY_CONSUMER:\n                return this.querySubscriptionByConsumer(ctx, request);\n            case RequestCode.QUERY_CONSUME_TIME_SPAN:\n                return this.queryConsumeTimeSpan(ctx, request);\n            case RequestCode.GET_SYSTEM_TOPIC_LIST_FROM_BROKER:\n                return this.getSystemTopicListFromBroker(ctx, request);\n            case RequestCode.CLEAN_EXPIRED_CONSUMEQUEUE:\n                return this.cleanExpiredConsumeQueue();\n            case RequestCode.DELETE_EXPIRED_COMMITLOG:\n                return this.deleteExpiredCommitLog();\n            case RequestCode.CLEAN_UNUSED_TOPIC:\n                return this.cleanUnusedTopic();\n            case RequestCode.GET_CONSUMER_RUNNING_INFO:\n                return this.getConsumerRunningInfo(ctx, request);\n            case RequestCode.QUERY_CORRECTION_OFFSET:\n                return this.queryCorrectionOffset(ctx, request);\n            case RequestCode.CONSUME_MESSAGE_DIRECTLY:\n                return this.consumeMessageDirectly(ctx, request);\n            case RequestCode.CLONE_GROUP_OFFSET:\n                return this.cloneGroupOffset(ctx, request);\n            case RequestCode.VIEW_BROKER_STATS_DATA:\n                return ViewBrokerStatsData(ctx, request);\n            case RequestCode.GET_BROKER_CONSUME_STATS:\n                return fetchAllConsumeStatsInBroker(ctx, request);\n            case RequestCode.QUERY_CONSUME_QUEUE:\n                return queryConsumeQueue(ctx, request);\n            case RequestCode.CHECK_ROCKSDB_CQ_WRITE_PROGRESS:\n                return this.checkRocksdbCqWriteProgress(ctx, request);\n            case RequestCode.EXPORT_ROCKSDB_CONFIG_TO_JSON:\n                return this.exportRocksDBConfigToJson(ctx, request);\n            case RequestCode.UPDATE_AND_GET_GROUP_FORBIDDEN:\n                return this.updateAndGetGroupForbidden(ctx, request);\n            case RequestCode.GET_SUBSCRIPTIONGROUP_CONFIG:\n                return this.getSubscriptionGroup(ctx, request);\n            case RequestCode.RESUME_CHECK_HALF_MESSAGE:\n                return resumeCheckHalfMessage(ctx, request);\n            case RequestCode.GET_TOPIC_CONFIG:\n                return getTopicConfig(ctx, request);\n            case RequestCode.UPDATE_AND_CREATE_STATIC_TOPIC:\n                return this.updateAndCreateStaticTopic(ctx, request);\n            case RequestCode.NOTIFY_MIN_BROKER_ID_CHANGE:\n                return this.notifyMinBrokerIdChange(ctx, request);\n            case RequestCode.EXCHANGE_BROKER_HA_INFO:\n                return this.updateBrokerHaInfo(ctx, request);\n            case RequestCode.GET_BROKER_HA_STATUS:\n                return this.getBrokerHaStatus(ctx, request);\n            case RequestCode.RESET_MASTER_FLUSH_OFFSET:\n                return this.resetMasterFlushOffset(ctx, request);\n            case RequestCode.GET_BROKER_EPOCH_CACHE:\n                return this.getBrokerEpochCache(ctx, request);\n            case RequestCode.NOTIFY_BROKER_ROLE_CHANGED:\n                return this.notifyBrokerRoleChanged(ctx, request);\n            case RequestCode.AUTH_CREATE_USER:\n                return this.createUser(ctx, request);\n            case RequestCode.AUTH_UPDATE_USER:\n                return this.updateUser(ctx, request);\n            case RequestCode.AUTH_DELETE_USER:\n                return this.deleteUser(ctx, request);\n            case RequestCode.AUTH_GET_USER:\n                return this.getUser(ctx, request);\n            case RequestCode.AUTH_LIST_USER:\n                return this.listUser(ctx, request);\n            case RequestCode.AUTH_CREATE_ACL:\n                return this.createAcl(ctx, request);\n            case RequestCode.AUTH_UPDATE_ACL:\n                return this.updateAcl(ctx, request);\n            case RequestCode.AUTH_DELETE_ACL:\n                return this.deleteAcl(ctx, request);\n            case RequestCode.AUTH_GET_ACL:\n                return this.getAcl(ctx, request);\n            case RequestCode.AUTH_LIST_ACL:\n                return this.listAcl(ctx, request);\n            case RequestCode.POP_ROLLBACK:\n                return this.transferPopToFsStore(ctx, request);\n            case RequestCode.SWITCH_TIMER_ENGINE:\n                return this.switchTimerEngine(ctx, request);\n            default:\n                return getUnknownCmdResponse(ctx, request);\n        }\n    }\n\n    /**\n     * @param ctx\n     * @param request\n     * @return\n     * @throws RemotingCommandException\n     */\n    private RemotingCommand getSubscriptionGroup(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        GetSubscriptionGroupConfigRequestHeader requestHeader = (GetSubscriptionGroupConfigRequestHeader) request.decodeCommandCustomHeader(GetSubscriptionGroupConfigRequestHeader.class);\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        SubscriptionGroupConfig groupConfig = this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getGroup());\n        if (groupConfig == null) {\n            LOGGER.error(\"No group in this broker, client: {} group: {}\", ctx.channel().remoteAddress(), requestHeader.getGroup());\n            response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);\n            response.setRemark(\"No group in this broker\");\n            return response;\n        }\n        String content = JSONObject.toJSONString(groupConfig);\n        try {\n            response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET));\n        } catch (UnsupportedEncodingException e) {\n            LOGGER.error(\"UnsupportedEncodingException getSubscriptionGroup: group=\" + groupConfig.getGroupName(), e);\n\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"UnsupportedEncodingException \" + e.getMessage());\n            return response;\n        }\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n\n        return response;\n    }\n\n    /**\n     * @param ctx\n     * @param request\n     * @return\n     */\n    private RemotingCommand updateAndGetGroupForbidden(ChannelHandlerContext ctx, RemotingCommand request)\n        throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        UpdateGroupForbiddenRequestHeader requestHeader = (UpdateGroupForbiddenRequestHeader) //\n            request.decodeCommandCustomHeader(UpdateGroupForbiddenRequestHeader.class);\n        String group = requestHeader.getGroup();\n        String topic = requestHeader.getTopic();\n        LOGGER.info(\"updateAndGetGroupForbidden called by {} for object {}@{} readable={}\",//\n            RemotingHelper.parseChannelRemoteAddr(ctx.channel()), group, //\n            topic, requestHeader.getReadable());\n        SubscriptionGroupManager groupManager = this.brokerController.getSubscriptionGroupManager();\n        if (requestHeader.getReadable() != null) {\n            groupManager.updateForbidden(group, topic, PermName.INDEX_PERM_READ, !requestHeader.getReadable());\n        }\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(\"\");\n        GroupForbidden groupForbidden = new GroupForbidden();\n        groupForbidden.setGroup(group);\n        groupForbidden.setTopic(topic);\n        groupForbidden.setReadable(!groupManager.getForbidden(group, topic, PermName.INDEX_PERM_READ));\n        response.setBody(groupForbidden.toJson().getBytes(StandardCharsets.UTF_8));\n        return response;\n    }\n\n    private RemotingCommand checkRocksdbCqWriteProgress(ChannelHandlerContext ctx, RemotingCommand request) {\n        CheckRocksdbCqWriteResult result = new CheckRocksdbCqWriteResult();\n        result.setCheckStatus(CheckRocksdbCqWriteResult.CheckStatus.CHECK_IN_PROGRESS.getValue());\n        Runnable runnable = () -> {\n            try {\n                CheckRocksdbCqWriteResult checkResult = doCheckRocksdbCqWriteProgress(ctx, request);\n                LOGGER.info(\"checkRocksdbCqWriteProgress result: {}\", JSON.toJSONString(checkResult));\n            } catch (Exception e) {\n                LOGGER.error(\"checkRocksdbCqWriteProgress error\", e);\n            }\n        };\n        asyncExecuteWorker.submit(runnable);\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setBody(JSON.toJSONBytes(result));\n        return response;\n    }\n\n    private RemotingCommand exportRocksDBConfigToJson(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        ExportRocksDBConfigToJsonRequestHeader requestHeader = request.decodeCommandCustomHeader(ExportRocksDBConfigToJsonRequestHeader.class);\n        List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> configTypes = requestHeader.fetchConfigType();\n        List<CompletableFuture<Void>> futureList = new ArrayList<>(configTypes.size());\n        for (ExportRocksDBConfigToJsonRequestHeader.ConfigType type : configTypes) {\n            switch (type) {\n                case TOPICS:\n                    if (this.brokerController.getTopicConfigManager() instanceof RocksDBTopicConfigManager) {\n                        RocksDBTopicConfigManager rocksDBTopicConfigManager = (RocksDBTopicConfigManager) this.brokerController.getTopicConfigManager();\n                        futureList.add(CompletableFuture.runAsync(rocksDBTopicConfigManager::exportToJson, asyncExecuteWorker));\n                    }\n                    break;\n                case SUBSCRIPTION_GROUPS:\n                    if (this.brokerController.getSubscriptionGroupManager() instanceof RocksDBSubscriptionGroupManager) {\n                        RocksDBSubscriptionGroupManager rocksDBSubscriptionGroupManager = (RocksDBSubscriptionGroupManager) this.brokerController.getSubscriptionGroupManager();\n                        futureList.add(CompletableFuture.runAsync(rocksDBSubscriptionGroupManager::exportToJson, asyncExecuteWorker));\n                    }\n                    break;\n                case CONSUMER_OFFSETS:\n                    if (this.brokerController.getConsumerOffsetManager() instanceof RocksDBConsumerOffsetManager) {\n                        RocksDBConsumerOffsetManager rocksDBConsumerOffsetManager = (RocksDBConsumerOffsetManager) this.brokerController.getConsumerOffsetManager();\n                        futureList.add(CompletableFuture.runAsync(rocksDBConsumerOffsetManager::exportToJson, asyncExecuteWorker));\n                    }\n                    break;\n                default:\n                    break;\n            }\n        }\n\n        try {\n            CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).join();\n        } catch (CompletionException e) {\n            RemotingCommand response = RemotingCommand.createResponseCommand(null);\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(String.valueOf(e));\n            return response;\n        }\n\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(\"export done.\");\n        return response;\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n\n    private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        long startTime = System.currentTimeMillis();\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final CreateTopicRequestHeader requestHeader =\n            (CreateTopicRequestHeader) request.decodeCommandCustomHeader(CreateTopicRequestHeader.class);\n\n        LOGGER.info(\"Broker receive request to update or create topic={}, caller address={}\",\n            requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n\n        String topic = requestHeader.getTopic();\n\n        long executionTime;\n        try {\n            TopicValidator.ValidateResult result = TopicValidator.validateTopic(topic);\n            if (!result.isValid()) {\n                response.setCode(ResponseCode.INVALID_PARAMETER);\n                response.setRemark(result.getRemark());\n                return response;\n            }\n            if (brokerController.getBrokerConfig().isValidateSystemTopicWhenUpdateTopic()) {\n                if (TopicValidator.isSystemTopic(topic)) {\n                    response.setCode(ResponseCode.INVALID_PARAMETER);\n                    response.setRemark(\"The topic[\" + topic + \"] is conflict with system topic.\");\n                    return response;\n                }\n            }\n\n            TopicConfig topicConfig = new TopicConfig(topic);\n            topicConfig.setReadQueueNums(requestHeader.getReadQueueNums());\n            topicConfig.setWriteQueueNums(requestHeader.getWriteQueueNums());\n            topicConfig.setTopicFilterType(requestHeader.getTopicFilterTypeEnum());\n            topicConfig.setPerm(requestHeader.getPerm());\n            topicConfig.setTopicSysFlag(requestHeader.getTopicSysFlag() == null ? 0 : requestHeader.getTopicSysFlag());\n            topicConfig.setOrder(requestHeader.getOrder());\n            String attributesModification = requestHeader.getAttributes();\n            topicConfig.setAttributes(AttributeParser.parseToMap(attributesModification));\n\n            if (!brokerController.getBrokerConfig().isEnableMixedMessageType() && topicConfig.getAttributes() != null) {\n                // Get attribute by key with prefix sign\n                String msgTypeAttrKey = AttributeParser.ATTR_ADD_PLUS_SIGN + TopicAttributes.TOPIC_MESSAGE_TYPE_ATTRIBUTE.getName();\n                String msgTypeAttrValue = topicConfig.getAttributes().get(msgTypeAttrKey);\n                if (msgTypeAttrValue != null && msgTypeAttrValue.equals(TopicMessageType.MIXED.getValue())) {\n                    response.setCode(ResponseCode.INVALID_PARAMETER);\n                    response.setRemark(\"MIXED message type is not supported.\");\n                    return response;\n                }\n            }\n\n            if (topicConfig.equals(this.brokerController.getTopicConfigManager().getTopicConfigTable().get(topic))) {\n                LOGGER.info(\"Broker receive request to update or create topic={}, but topicConfig has  no changes , so idempotent, caller address={}\",\n                    requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n                response.setCode(ResponseCode.SUCCESS);\n                return response;\n            }\n\n            this.brokerController.getTopicConfigManager().updateTopicConfig(topicConfig);\n            if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) {\n                this.brokerController.registerSingleTopicAll(topicConfig);\n            } else {\n                this.brokerController.registerIncrementBrokerData(topicConfig, this.brokerController.getTopicConfigManager().getDataVersion());\n            }\n            response.setCode(ResponseCode.SUCCESS);\n        } catch (Exception e) {\n            LOGGER.error(\"Update / create topic failed for [{}]\", request, e);\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(e.getMessage());\n            return response;\n        } finally {\n            executionTime = System.currentTimeMillis() - startTime;\n            InvocationStatus status = response.getCode() == ResponseCode.SUCCESS ?\n                InvocationStatus.SUCCESS : InvocationStatus.FAILURE;\n            Attributes attributes = this.brokerController.getBrokerMetricsManager().newAttributesBuilder()\n                .put(LABEL_INVOCATION_STATUS, status.getName())\n                .put(LABEL_IS_SYSTEM, TopicValidator.isSystemTopic(topic))\n                .build();\n            this.brokerController.getBrokerMetricsManager().getTopicCreateExecuteTime().record(executionTime, attributes);\n        }\n        LOGGER.info(\"executionTime of create topic:{} is {} ms\", topic, executionTime);\n        return response;\n    }\n\n    private synchronized RemotingCommand updateAndCreateTopicList(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        long startTime = System.currentTimeMillis();\n\n        final CreateTopicListRequestBody requestBody = CreateTopicListRequestBody.decode(request.getBody(), CreateTopicListRequestBody.class);\n        List<TopicConfig> topicConfigList = requestBody.getTopicConfigList();\n\n        StringBuilder builder = new StringBuilder();\n        for (TopicConfig topicConfig : topicConfigList) {\n            builder.append(topicConfig.getTopicName()).append(\";\");\n        }\n        String topicNames = builder.toString();\n        LOGGER.info(\"AdminBrokerProcessor#updateAndCreateTopicList: topicNames: {}, called by {}\", topicNames, RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        long executionTime;\n\n        try {\n            // Valid topics\n            for (TopicConfig topicConfig : topicConfigList) {\n                String topic = topicConfig.getTopicName();\n                TopicValidator.ValidateResult result = TopicValidator.validateTopic(topic);\n                if (!result.isValid()) {\n                    response.setCode(ResponseCode.INVALID_PARAMETER);\n                    response.setRemark(result.getRemark());\n                    return response;\n                }\n                if (brokerController.getBrokerConfig().isValidateSystemTopicWhenUpdateTopic()) {\n                    if (TopicValidator.isSystemTopic(topic)) {\n                        response.setCode(ResponseCode.INVALID_PARAMETER);\n                        response.setRemark(\"The topic[\" + topic + \"] is conflict with system topic.\");\n                        return response;\n                    }\n                }\n                if (!brokerController.getBrokerConfig().isEnableMixedMessageType() && topicConfig.getAttributes() != null) {\n                    // Get attribute by key with prefix sign\n                    String msgTypeAttrKey = AttributeParser.ATTR_ADD_PLUS_SIGN + TopicAttributes.TOPIC_MESSAGE_TYPE_ATTRIBUTE.getName();\n                    String msgTypeAttrValue = topicConfig.getAttributes().get(msgTypeAttrKey);\n                    if (msgTypeAttrValue != null && msgTypeAttrValue.equals(TopicMessageType.MIXED.getValue())) {\n                        response.setCode(ResponseCode.INVALID_PARAMETER);\n                        response.setRemark(\"MIXED message type is not supported.\");\n                        return response;\n                    }\n                }\n                if (topicConfig.equals(this.brokerController.getTopicConfigManager().getTopicConfigTable().get(topic))) {\n                    LOGGER.info(\"Broker receive request to update or create topic={}, but topicConfig has  no changes , so idempotent, caller address={}\",\n                        topic, RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n                    response.setCode(ResponseCode.SUCCESS);\n                    return response;\n                }\n            }\n\n            this.brokerController.getTopicConfigManager().updateTopicConfigList(topicConfigList);\n            if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) {\n                for (TopicConfig topicConfig : topicConfigList) {\n                    this.brokerController.registerSingleTopicAll(topicConfig);\n                }\n            } else {\n                this.brokerController.registerIncrementBrokerData(topicConfigList, this.brokerController.getTopicConfigManager().getDataVersion());\n            }\n            response.setCode(ResponseCode.SUCCESS);\n        } catch (Exception e) {\n            LOGGER.error(\"Update / create topic failed for [{}]\", request, e);\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(e.getMessage());\n            return response;\n        } finally {\n            executionTime = System.currentTimeMillis() - startTime;\n            InvocationStatus status = response.getCode() == ResponseCode.SUCCESS ?\n                InvocationStatus.SUCCESS : InvocationStatus.FAILURE;\n            Attributes attributes = this.brokerController.getBrokerMetricsManager().newAttributesBuilder()\n                .put(LABEL_INVOCATION_STATUS, status.getName())\n                .put(LABEL_IS_SYSTEM, TopicValidator.isSystemTopic(topicNames))\n                .build();\n            this.brokerController.getBrokerMetricsManager().getTopicCreateExecuteTime().record(executionTime, attributes);\n        }\n        LOGGER.info(\"executionTime of all topics:{} is {} ms\", topicNames, executionTime);\n        return response;\n    }\n\n    private synchronized RemotingCommand updateAndCreateStaticTopic(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final CreateTopicRequestHeader requestHeader =\n            (CreateTopicRequestHeader) request.decodeCommandCustomHeader(CreateTopicRequestHeader.class);\n        LOGGER.info(\"Broker receive request to update or create static topic={}, caller address={}\", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n\n        final TopicQueueMappingDetail topicQueueMappingDetail = RemotingSerializable.decode(request.getBody(), TopicQueueMappingDetail.class);\n\n        String topic = requestHeader.getTopic();\n\n        TopicValidator.ValidateResult result = TopicValidator.validateTopic(topic);\n        if (!result.isValid()) {\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(result.getRemark());\n            return response;\n        }\n        if (brokerController.getBrokerConfig().isValidateSystemTopicWhenUpdateTopic()) {\n            if (TopicValidator.isSystemTopic(topic)) {\n                response.setCode(ResponseCode.INVALID_PARAMETER);\n                response.setRemark(\"The topic[\" + topic + \"] is conflict with system topic.\");\n                return response;\n            }\n        }\n        boolean force = requestHeader.getForce() != null && requestHeader.getForce();\n\n        TopicConfig topicConfig = new TopicConfig(topic);\n        topicConfig.setReadQueueNums(requestHeader.getReadQueueNums());\n        topicConfig.setWriteQueueNums(requestHeader.getWriteQueueNums());\n        topicConfig.setTopicFilterType(requestHeader.getTopicFilterTypeEnum());\n        topicConfig.setPerm(requestHeader.getPerm());\n        topicConfig.setTopicSysFlag(requestHeader.getTopicSysFlag() == null ? 0 : requestHeader.getTopicSysFlag());\n\n        try {\n            this.brokerController.getTopicConfigManager().updateTopicConfig(topicConfig);\n\n            this.brokerController.getTopicQueueMappingManager().updateTopicQueueMapping(topicQueueMappingDetail, force, false, true);\n\n            this.brokerController.registerIncrementBrokerData(topicConfig, this.brokerController.getTopicConfigManager().getDataVersion());\n            response.setCode(ResponseCode.SUCCESS);\n        } catch (Exception e) {\n            LOGGER.error(\"Update static topic failed for [{}]\", request, e);\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(e.getMessage());\n        }\n        return response;\n    }\n\n    private synchronized RemotingCommand deleteTopic(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        DeleteTopicRequestHeader requestHeader =\n            (DeleteTopicRequestHeader) request.decodeCommandCustomHeader(DeleteTopicRequestHeader.class);\n\n        LOGGER.info(\"AdminBrokerProcessor#deleteTopic: broker receive request to delete topic={}, caller={}\",\n            requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n\n        String topic = requestHeader.getTopic();\n\n        if (UtilAll.isBlank(topic)) {\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(\"The specified topic is blank.\");\n            return response;\n        }\n\n        if (brokerController.getBrokerConfig().isValidateSystemTopicWhenUpdateTopic()) {\n            if (TopicValidator.isSystemTopic(topic)) {\n                response.setCode(ResponseCode.INVALID_PARAMETER);\n                response.setRemark(\"The topic[\" + topic + \"] is conflict with system topic.\");\n                return response;\n            }\n        }\n\n        List<String> topicsToClean = new ArrayList<>();\n        topicsToClean.add(topic);\n\n        if (brokerController.getBrokerConfig().isClearRetryTopicWhenDeleteTopic()) {\n            final Set<String> groups = this.brokerController.getConsumerOffsetManager().whichGroupByTopic(topic);\n            for (String group : groups) {\n                final String popRetryTopicV2 = KeyBuilder.buildPopRetryTopic(topic, group, true);\n                if (brokerController.getTopicConfigManager().selectTopicConfig(popRetryTopicV2) != null) {\n                    topicsToClean.add(popRetryTopicV2);\n                }\n                final String popRetryTopicV1 = KeyBuilder.buildPopRetryTopicV1(topic, group);\n                if (brokerController.getTopicConfigManager().selectTopicConfig(popRetryTopicV1) != null) {\n                    topicsToClean.add(popRetryTopicV1);\n                }\n            }\n        }\n\n        try {\n            if (LiteMetadataUtil.isLiteMessageType(topic, brokerController)) {\n                brokerController.getLiteLifecycleManager().cleanByParentTopic(topic);\n            }\n            for (String topicToClean : topicsToClean) {\n                // delete topic\n                deleteTopicInBroker(topicToClean);\n            }\n        } catch (Throwable t) {\n            return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());\n        }\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private void deleteTopicInBroker(String topic) {\n        this.brokerController.getTopicConfigManager().deleteTopicConfig(topic);\n        this.brokerController.getTopicQueueMappingManager().delete(topic);\n        this.brokerController.getConsumerOffsetManager().cleanOffsetByTopic(topic);\n        this.brokerController.getPopInflightMessageCounter().clearInFlightMessageNumByTopicName(topic);\n        this.brokerController.getMessageStore().deleteTopics(Sets.newHashSet(topic));\n        this.brokerController.getMessageStore().getTimerMessageStore().getTimerMetrics().removeTimingCount(topic);\n    }\n\n    private RemotingCommand getUnknownCmdResponse(ChannelHandlerContext ctx, RemotingCommand request) {\n        String error = \" request type \" + request.getCode() + \" not supported\";\n        final RemotingCommand response =\n            RemotingCommand.createResponseCommand(RemotingSysResponseCode.REQUEST_CODE_NOT_SUPPORTED, error);\n        return response;\n    }\n\n    private RemotingCommand getAllTopicConfig(ChannelHandlerContext ctx, RemotingCommand request)\n        throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(GetAllTopicConfigResponseHeader.class);\n        final GetAllTopicConfigResponseHeader responseHeader =\n            (GetAllTopicConfigResponseHeader) response.readCustomHeader();\n        final GetAllTopicConfigRequestHeader requestHeader =\n            request.decodeCommandCustomHeader(GetAllTopicConfigRequestHeader.class);\n\n        String dataVersionStr = requestHeader.getDataVersion();\n        Integer topicSeq = requestHeader.getTopicSeq();\n        Integer maxTopicNum = requestHeader.getMaxTopicNum();\n\n        TopicConfigManager tcManager = brokerController.getTopicConfigManager();\n        TopicQueueMappingManager tqmManager = brokerController.getTopicQueueMappingManager();\n\n        TopicConfigAndMappingSerializeWrapper topicConfigAndMappingSerializeWrapper = new TopicConfigAndMappingSerializeWrapper();\n        if (!brokerController.getBrokerConfig().isEnableSplitMetadata()\n            || ObjectUtils.allNull(dataVersionStr, topicSeq, maxTopicNum)) {  // old client, return all topic config\n\n            topicConfigAndMappingSerializeWrapper.setDataVersion(tcManager.getDataVersion());\n            topicConfigAndMappingSerializeWrapper.setTopicConfigTable(tcManager.getTopicConfigTable());\n\n            topicConfigAndMappingSerializeWrapper.setMappingDataVersion(tqmManager.getDataVersion());\n            topicConfigAndMappingSerializeWrapper.setTopicQueueMappingDetailMap(tqmManager.getTopicQueueMappingTable());\n        } else {\n            int topicNum = Math.min(brokerController.getBrokerConfig().getSplitMetadataSize(),\n                Optional.ofNullable(maxTopicNum).orElse(Integer.MAX_VALUE));  // use smaller value\n            ConcurrentHashMap<String, TopicConfig> subTopicConfigTable =\n                tcManager.subTopicConfigTable(dataVersionStr, topicSeq, topicNum);\n            topicConfigAndMappingSerializeWrapper.setTopicConfigTable(subTopicConfigTable);\n            topicConfigAndMappingSerializeWrapper.setDataVersion(tcManager.getDataVersion());\n\n            topicConfigAndMappingSerializeWrapper.setMappingDataVersion(tqmManager.getDataVersion());\n            topicConfigAndMappingSerializeWrapper.setTopicQueueMappingDetailMap(\n                tqmManager.subTopicQueueMappingTable(subTopicConfigTable.keySet()));\n        }\n\n        responseHeader.setTotalTopicNum(tcManager.getTopicConfigTable().size());\n        String content = topicConfigAndMappingSerializeWrapper.toJson();\n        if (StringUtils.isNotBlank(content)) {\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n            response.setBody(content.getBytes(StandardCharsets.UTF_8));\n        } else {\n            LOGGER.error(\"No topic in this broker, client: {}\", ctx.channel().remoteAddress());\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"No topic in this broker\");\n        }\n        return response;\n    }\n\n    private RemotingCommand getTimerCheckPoint(ChannelHandlerContext ctx, RemotingCommand request) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, \"Unknown\");\n        TimerCheckpoint timerCheckpoint = this.brokerController.getTimerCheckpoint();\n        if (null == timerCheckpoint) {\n            LOGGER.error(\"AdminBrokerProcessor#getTimerCheckPoint: checkpoint is null, caller={}\", ctx.channel().remoteAddress());\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"The checkpoint is null\");\n            return response;\n        }\n        response.setBody(TimerCheckpoint.encode(timerCheckpoint).array());\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand getTimerMetrics(ChannelHandlerContext ctx, RemotingCommand request) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, \"Unknown\");\n        TimerMessageStore timerMessageStore = this.brokerController.getMessageStore().getTimerMessageStore();\n        if (null == timerMessageStore) {\n            LOGGER.error(\"The timer message store is null, client: {}\", ctx.channel().remoteAddress());\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"The timer message store is null\");\n            return response;\n        }\n        response.setBody(timerMessageStore.getTimerMetrics().encode().getBytes(StandardCharsets.UTF_8));\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private synchronized RemotingCommand updateColdDataFlowCtrGroupConfig(ChannelHandlerContext ctx,\n        RemotingCommand request) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        LOGGER.info(\"updateColdDataFlowCtrGroupConfig called by {}\", RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n\n        byte[] body = request.getBody();\n        if (body != null) {\n            try {\n                String bodyStr = new String(body, MixAll.DEFAULT_CHARSET);\n                Properties properties = MixAll.string2Properties(bodyStr);\n                if (properties != null) {\n                    LOGGER.info(\"updateColdDataFlowCtrGroupConfig new config: {}, client: {}\", properties, ctx.channel().remoteAddress());\n                    properties.forEach((key, value) -> {\n                        try {\n                            String consumerGroup = String.valueOf(key);\n                            Long threshold = Long.valueOf(String.valueOf(value));\n                            this.brokerController.getColdDataCgCtrService()\n                                .addOrUpdateGroupConfig(consumerGroup, threshold);\n                        } catch (Exception e) {\n                            LOGGER.error(\"updateColdDataFlowCtrGroupConfig properties on entry error, key: {}, val: {}\",\n                                key, value, e);\n                        }\n                    });\n                } else {\n                    LOGGER.error(\"updateColdDataFlowCtrGroupConfig string2Properties error\");\n                    response.setCode(ResponseCode.SYSTEM_ERROR);\n                    response.setRemark(\"string2Properties error\");\n                    return response;\n                }\n            } catch (UnsupportedEncodingException e) {\n                LOGGER.error(\"updateColdDataFlowCtrGroupConfig UnsupportedEncodingException\", e);\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"UnsupportedEncodingException \" + e);\n                return response;\n            }\n        }\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private synchronized RemotingCommand removeColdDataFlowCtrGroupConfig(ChannelHandlerContext ctx,\n        RemotingCommand request) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        LOGGER.info(\"removeColdDataFlowCtrGroupConfig called by {}\", RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n\n        byte[] body = request.getBody();\n        if (body != null) {\n            try {\n                String consumerGroup = new String(body, MixAll.DEFAULT_CHARSET);\n                if (consumerGroup != null) {\n                    LOGGER.info(\"removeColdDataFlowCtrGroupConfig, consumerGroup: {} client: {}\", consumerGroup, ctx.channel().remoteAddress());\n                    this.brokerController.getColdDataCgCtrService().removeGroupConfig(consumerGroup);\n                } else {\n                    LOGGER.error(\"removeColdDataFlowCtrGroupConfig string parse error\");\n                    response.setCode(ResponseCode.SYSTEM_ERROR);\n                    response.setRemark(\"string parse error\");\n                    return response;\n                }\n            } catch (UnsupportedEncodingException e) {\n                LOGGER.error(\"removeColdDataFlowCtrGroupConfig UnsupportedEncodingException\", e);\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"UnsupportedEncodingException \" + e);\n                return response;\n            }\n        }\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand getColdDataFlowCtrInfo(ChannelHandlerContext ctx) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        LOGGER.info(\"getColdDataFlowCtrInfo called by {}\", RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n\n        String content = this.brokerController.getColdDataCgCtrService().getColdDataFlowCtrInfo();\n        if (content != null) {\n            try {\n                response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET));\n            } catch (UnsupportedEncodingException e) {\n                LOGGER.error(\"getColdDataFlowCtrInfo UnsupportedEncodingException\", e);\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"UnsupportedEncodingException \" + e);\n                return response;\n            }\n        }\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand setCommitLogReadaheadMode(ChannelHandlerContext ctx, RemotingCommand request) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        LOGGER.info(\"setCommitLogReadaheadMode called by {}\", RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n\n        try {\n            HashMap<String, String> extFields = request.getExtFields();\n            if (null == extFields) {\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"set commitlog readahead mode param error\");\n                return response;\n            }\n            int mode = Integer.parseInt(extFields.get(FIleReadaheadMode.READ_AHEAD_MODE));\n            if (mode != LibC.MADV_RANDOM && mode != LibC.MADV_NORMAL) {\n                response.setCode(ResponseCode.INVALID_PARAMETER);\n                response.setRemark(\"set commitlog readahead mode param value error\");\n                return response;\n            }\n            MessageStore messageStore = this.brokerController.getMessageStore();\n            if (messageStore instanceof DefaultMessageStore) {\n                DefaultMessageStore defaultMessageStore = (DefaultMessageStore) messageStore;\n                if (mode == LibC.MADV_NORMAL) {\n                    defaultMessageStore.getMessageStoreConfig().setDataReadAheadEnable(true);\n                } else {\n                    defaultMessageStore.getMessageStoreConfig().setDataReadAheadEnable(false);\n                }\n                defaultMessageStore.getCommitLog().scanFileAndSetReadMode(mode);\n            }\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(\"set commitlog readahead mode success, mode: \" + mode);\n        } catch (Exception e) {\n            LOGGER.error(\"set commitlog readahead mode failed\", e);\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"set commitlog readahead mode failed\");\n        }\n        return response;\n    }\n\n    private synchronized RemotingCommand updateBrokerConfig(ChannelHandlerContext ctx, RemotingCommand request) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        final String callerAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());\n        LOGGER.info(\"Broker receive request to update config, caller address={}\", callerAddress);\n\n        byte[] body = request.getBody();\n        if (body != null) {\n            try {\n                String bodyStr = new String(body, MixAll.DEFAULT_CHARSET);\n                Properties properties = MixAll.string2Properties(bodyStr);\n                if (properties != null) {\n                    LOGGER.info(\"updateBrokerConfig, new config: [{}] client: {} \", properties, callerAddress);\n                    if (validateBlackListConfigExist(properties)) {\n                        response.setCode(ResponseCode.NO_PERMISSION);\n                        response.setRemark(\"Can not update config in black list.\");\n                        return response;\n                    }\n\n                    this.brokerController.getConfiguration().update(properties);\n                    if (properties.containsKey(\"brokerPermission\")) {\n                        long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;\n                        this.brokerController.getTopicConfigManager().getDataVersion().nextVersion(stateMachineVersion);\n                        this.brokerController.registerBrokerAll(false, false, true);\n                    }\n\n                } else {\n                    LOGGER.error(\"string2Properties error\");\n                    response.setCode(ResponseCode.SYSTEM_ERROR);\n                    response.setRemark(\"string2Properties error\");\n                    return response;\n                }\n            } catch (UnsupportedEncodingException e) {\n                LOGGER.error(\"AdminBrokerProcessor#updateBrokerConfig: unexpected error, caller={}\",\n                    callerAddress, e);\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"UnsupportedEncodingException \" + e);\n                return response;\n            }\n        }\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand getBrokerConfig(ChannelHandlerContext ctx, RemotingCommand request) {\n\n        final RemotingCommand response = RemotingCommand.createResponseCommand(GetBrokerConfigResponseHeader.class);\n        final GetBrokerConfigResponseHeader responseHeader = (GetBrokerConfigResponseHeader) response.readCustomHeader();\n\n        String content = this.brokerController.getConfiguration().getAllConfigsFormatString();\n        if (content != null && content.length() > 0) {\n            try {\n                content = MixAll.adjustConfigForPlatform(content);\n                response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET));\n            } catch (UnsupportedEncodingException e) {\n                LOGGER.error(\"AdminBrokerProcessor#getBrokerConfig: unexpected error, caller={}\",\n                    RemotingHelper.parseChannelRemoteAddr(ctx.channel()), e);\n\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"UnsupportedEncodingException \" + e);\n                return response;\n            }\n        }\n\n        responseHeader.setVersion(this.brokerController.getConfiguration().getDataVersionJson());\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand rewriteRequestForStaticTopic(SearchOffsetRequestHeader requestHeader,\n        TopicQueueMappingContext mappingContext) {\n        try {\n            if (mappingContext.getMappingDetail() == null) {\n                return null;\n            }\n\n            TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();\n            List<LogicQueueMappingItem> mappingItems = mappingContext.getMappingItemList();\n            if (!mappingContext.isLeader()) {\n                return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format(\"%s-%d does not exit in request process of current broker %s\", mappingContext.getTopic(), mappingContext.getGlobalId(), mappingDetail.getBname()));\n            }\n            //TO DO should make sure the timestampOfOffset is equal or bigger than the searched timestamp\n            Long timestamp = requestHeader.getTimestamp();\n            long offset = -1;\n            for (int i = 0; i < mappingItems.size(); i++) {\n                LogicQueueMappingItem item = mappingItems.get(i);\n                if (!item.checkIfLogicoffsetDecided()) {\n                    continue;\n                }\n                if (mappingDetail.getBname().equals(item.getBname())) {\n                    offset = this.brokerController.getMessageStore().getOffsetInQueueByTime(mappingContext.getTopic(), item.getQueueId(), timestamp, requestHeader.getBoundaryType());\n                    if (offset > 0) {\n                        offset = item.computeStaticQueueOffsetStrictly(offset);\n                        break;\n                    }\n                } else {\n                    requestHeader.setLo(false);\n                    requestHeader.setTimestamp(timestamp);\n                    requestHeader.setQueueId(item.getQueueId());\n                    requestHeader.setBrokerName(item.getBname());\n                    RpcRequest rpcRequest = new RpcRequest(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, requestHeader, null);\n                    RpcResponse rpcResponse = this.brokerController.getBrokerOuterAPI().getRpcClient().invoke(rpcRequest, this.brokerController.getBrokerConfig().getForwardTimeout()).get();\n                    if (rpcResponse.getException() != null) {\n                        throw rpcResponse.getException();\n                    }\n                    SearchOffsetResponseHeader offsetResponseHeader = (SearchOffsetResponseHeader) rpcResponse.getHeader();\n                    if (offsetResponseHeader.getOffset() < 0\n                        || item.checkIfEndOffsetDecided() && offsetResponseHeader.getOffset() >= item.getEndOffset()) {\n                        continue;\n                    } else {\n                        offset = item.computeStaticQueueOffsetStrictly(offsetResponseHeader.getOffset());\n                    }\n\n                }\n            }\n            final RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class);\n            final SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader();\n            responseHeader.setOffset(offset);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n            return response;\n        } catch (Throwable t) {\n            return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());\n        }\n    }\n\n    private RemotingCommand searchOffsetByTimestamp(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class);\n        final SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader();\n        final SearchOffsetRequestHeader requestHeader =\n            (SearchOffsetRequestHeader) request.decodeCommandCustomHeader(SearchOffsetRequestHeader.class);\n\n        TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader);\n\n        RemotingCommand rewriteResult = rewriteRequestForStaticTopic(requestHeader, mappingContext);\n        if (rewriteResult != null) {\n            return rewriteResult;\n        }\n\n        boolean queryOffset = true;\n        String topic = requestHeader.getTopic();\n        int queueId = requestHeader.getQueueId();\n        String liteTopic = requestHeader.getLiteTopic();\n        if (StringUtils.isNotBlank(liteTopic)) {\n            topic = LiteUtil.toLmqName(topic, liteTopic);\n            long maxOffset = 0;\n            if (queueId == 0) {\n                maxOffset = this.brokerController.getLiteLifecycleManager().getMaxOffsetInQueue(topic);\n            }\n            // lite topic check max offset first\n            if (maxOffset <= 0) {\n                queryOffset = false;\n            }\n        }\n\n        long offset = 0L;\n        if (queryOffset) {\n            offset = this.brokerController.getMessageStore().getOffsetInQueueByTime(topic, queueId,\n                requestHeader.getTimestamp(), requestHeader.getBoundaryType());\n        }\n        responseHeader.setOffset(offset);\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand rewriteRequestForStaticTopic(GetMaxOffsetRequestHeader requestHeader,\n        TopicQueueMappingContext mappingContext) {\n        if (mappingContext.getMappingDetail() == null) {\n            return null;\n        }\n\n        TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();\n        LogicQueueMappingItem mappingItem = mappingContext.getLeaderItem();\n        if (!mappingContext.isLeader()) {\n            return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format(\"%s-%d does not exit in request process of current broker %s\", mappingContext.getTopic(), mappingContext.getGlobalId(), mappingDetail.getBname()));\n        }\n\n        try {\n            LogicQueueMappingItem maxItem = TopicQueueMappingUtils.findLogicQueueMappingItem(mappingContext.getMappingItemList(), Long.MAX_VALUE, true);\n            assert maxItem != null;\n            assert maxItem.getLogicOffset() >= 0;\n            requestHeader.setBrokerName(maxItem.getBname());\n            requestHeader.setLo(false);\n            requestHeader.setQueueId(mappingItem.getQueueId());\n\n            long maxPhysicalOffset = Long.MAX_VALUE;\n            if (maxItem.getBname().equals(mappingDetail.getBname())) {\n                //current broker\n                maxPhysicalOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(mappingContext.getTopic(), mappingItem.getQueueId());\n            } else {\n                RpcRequest rpcRequest = new RpcRequest(RequestCode.GET_MAX_OFFSET, requestHeader, null);\n                RpcResponse rpcResponse = this.brokerController.getBrokerOuterAPI().getRpcClient().invoke(rpcRequest, this.brokerController.getBrokerConfig().getForwardTimeout()).get();\n                if (rpcResponse.getException() != null) {\n                    throw rpcResponse.getException();\n                }\n                GetMaxOffsetResponseHeader offsetResponseHeader = (GetMaxOffsetResponseHeader) rpcResponse.getHeader();\n                maxPhysicalOffset = offsetResponseHeader.getOffset();\n            }\n\n            final RemotingCommand response = RemotingCommand.createResponseCommand(GetMaxOffsetResponseHeader.class);\n            final GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.readCustomHeader();\n            responseHeader.setOffset(maxItem.computeStaticQueueOffsetStrictly(maxPhysicalOffset));\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n            return response;\n        } catch (Throwable t) {\n            return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());\n        }\n    }\n\n    private RemotingCommand getMaxOffset(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(GetMaxOffsetResponseHeader.class);\n        final GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.readCustomHeader();\n        final GetMaxOffsetRequestHeader requestHeader = request.decodeCommandCustomHeader(GetMaxOffsetRequestHeader.class);\n\n        TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader);\n        RemotingCommand rewriteResult = rewriteRequestForStaticTopic(requestHeader, mappingContext);\n        if (rewriteResult != null) {\n            return rewriteResult;\n        }\n\n        try {\n            long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());\n            responseHeader.setOffset(offset);\n        } catch (ConsumeQueueException e) {\n            throw new RemotingCommandException(\"Failed to get max offset in queue\", e);\n        }\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private CompletableFuture<RpcResponse> handleGetMinOffsetForStaticTopic(RpcRequest request,\n        TopicQueueMappingContext mappingContext) {\n        if (mappingContext.getMappingDetail() == null) {\n            return null;\n        }\n        TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();\n        if (!mappingContext.isLeader()) {\n            //this may not\n            return CompletableFuture.completedFuture(new RpcResponse(new RpcException(ResponseCode.NOT_LEADER_FOR_QUEUE,\n                String.format(\"%s-%d is not leader in broker %s, request code %d\", mappingContext.getTopic(), mappingContext.getGlobalId(), mappingDetail.getBname(), request.getCode()))));\n        }\n        GetMinOffsetRequestHeader requestHeader = (GetMinOffsetRequestHeader) request.getHeader();\n        LogicQueueMappingItem mappingItem = TopicQueueMappingUtils.findLogicQueueMappingItem(mappingContext.getMappingItemList(), 0L, true);\n        assert mappingItem != null;\n        try {\n            requestHeader.setBrokerName(mappingItem.getBname());\n            requestHeader.setLo(false);\n            requestHeader.setQueueId(mappingItem.getQueueId());\n            long physicalOffset;\n            //run in local\n            if (mappingItem.getBname().equals(mappingDetail.getBname())) {\n                physicalOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(mappingDetail.getTopic(), mappingItem.getQueueId());\n            } else {\n                RpcRequest rpcRequest = new RpcRequest(RequestCode.GET_MIN_OFFSET, requestHeader, null);\n                RpcResponse rpcResponse = this.brokerController.getBrokerOuterAPI().getRpcClient().invoke(rpcRequest, this.brokerController.getBrokerConfig().getForwardTimeout()).get();\n                if (rpcResponse.getException() != null) {\n                    throw rpcResponse.getException();\n                }\n                GetMinOffsetResponseHeader offsetResponseHeader = (GetMinOffsetResponseHeader) rpcResponse.getHeader();\n                physicalOffset = offsetResponseHeader.getOffset();\n            }\n            long offset = mappingItem.computeStaticQueueOffsetLoosely(physicalOffset);\n\n            final GetMinOffsetResponseHeader responseHeader = new GetMinOffsetResponseHeader();\n            responseHeader.setOffset(offset);\n            return CompletableFuture.completedFuture(new RpcResponse(ResponseCode.SUCCESS, responseHeader, null));\n        } catch (Throwable t) {\n            LOGGER.error(\"rewriteRequestForStaticTopic failed\", t);\n            return CompletableFuture.completedFuture(new RpcResponse(new RpcException(ResponseCode.SYSTEM_ERROR, t.getMessage(), t)));\n        }\n    }\n\n    private CompletableFuture<RpcResponse> handleGetMinOffset(RpcRequest request) {\n        assert request.getCode() == RequestCode.GET_MIN_OFFSET;\n        GetMinOffsetRequestHeader requestHeader = (GetMinOffsetRequestHeader) request.getHeader();\n        TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader, false);\n        CompletableFuture<RpcResponse> rewriteResult = handleGetMinOffsetForStaticTopic(request, mappingContext);\n        if (rewriteResult != null) {\n            return rewriteResult;\n        }\n        final GetMinOffsetResponseHeader responseHeader = new GetMinOffsetResponseHeader();\n        long offset = this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());\n        responseHeader.setOffset(offset);\n        return CompletableFuture.completedFuture(new RpcResponse(ResponseCode.SUCCESS, responseHeader, null));\n    }\n\n    private RemotingCommand getMinOffset(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final GetMinOffsetRequestHeader requestHeader =\n            (GetMinOffsetRequestHeader) request.decodeCommandCustomHeader(GetMinOffsetRequestHeader.class);\n        try {\n            CompletableFuture<RpcResponse> responseFuture = handleGetMinOffset(new RpcRequest(RequestCode.GET_MIN_OFFSET, requestHeader, null));\n            RpcResponse rpcResponse = responseFuture.get();\n            return RpcClientUtils.createCommandForRpcResponse(rpcResponse);\n        } catch (Throwable t) {\n            return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());\n        }\n    }\n\n    private RemotingCommand rewriteRequestForStaticTopic(GetEarliestMsgStoretimeRequestHeader requestHeader,\n        TopicQueueMappingContext mappingContext) {\n        if (mappingContext.getMappingDetail() == null) {\n            return null;\n        }\n\n        TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();\n        if (!mappingContext.isLeader()) {\n            return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format(\"%s-%d does not exit in request process of current broker %s\", mappingContext.getTopic(), mappingContext.getGlobalId(), mappingDetail.getBname()));\n        }\n        LogicQueueMappingItem mappingItem = TopicQueueMappingUtils.findLogicQueueMappingItem(mappingContext.getMappingItemList(), 0L, true);\n        assert mappingItem != null;\n        try {\n            requestHeader.setBrokerName(mappingItem.getBname());\n            requestHeader.setLo(false);\n            RpcRequest rpcRequest = new RpcRequest(RequestCode.GET_EARLIEST_MSG_STORETIME, requestHeader, null);\n            //TO DO check if it is in current broker\n            RpcResponse rpcResponse = this.brokerController.getBrokerOuterAPI().getRpcClient().invoke(rpcRequest, this.brokerController.getBrokerConfig().getForwardTimeout()).get();\n            if (rpcResponse.getException() != null) {\n                throw rpcResponse.getException();\n            }\n            GetEarliestMsgStoretimeResponseHeader offsetResponseHeader = (GetEarliestMsgStoretimeResponseHeader) rpcResponse.getHeader();\n\n            final RemotingCommand response = RemotingCommand.createResponseCommand(GetEarliestMsgStoretimeResponseHeader.class);\n            final GetEarliestMsgStoretimeResponseHeader responseHeader = (GetEarliestMsgStoretimeResponseHeader) response.readCustomHeader();\n            responseHeader.setTimestamp(offsetResponseHeader.getTimestamp());\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n            return response;\n        } catch (Throwable t) {\n            return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());\n        }\n    }\n\n    private RemotingCommand getEarliestMsgStoretime(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(GetEarliestMsgStoretimeResponseHeader.class);\n        final GetEarliestMsgStoretimeResponseHeader responseHeader = (GetEarliestMsgStoretimeResponseHeader) response.readCustomHeader();\n        final GetEarliestMsgStoretimeRequestHeader requestHeader =\n            (GetEarliestMsgStoretimeRequestHeader) request.decodeCommandCustomHeader(GetEarliestMsgStoretimeRequestHeader.class);\n\n        TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader, false);\n        RemotingCommand rewriteResult = rewriteRequestForStaticTopic(requestHeader, mappingContext);\n        if (rewriteResult != null) {\n            return rewriteResult;\n        }\n\n        long timestamp =\n            this.brokerController.getMessageStore().getEarliestMessageTime(requestHeader.getTopic(), requestHeader.getQueueId());\n\n        responseHeader.setTimestamp(timestamp);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand getBrokerRuntimeInfo(ChannelHandlerContext ctx, RemotingCommand request)\n        throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        HashMap<String, String> runtimeInfo = this.prepareRuntimeInfo();\n        KVTable kvTable = new KVTable();\n        kvTable.setTable(runtimeInfo);\n\n        byte[] body = kvTable.encode();\n        response.setBody(body);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand lockBatchMQ(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        LockBatchRequestBody requestBody = LockBatchRequestBody.decode(request.getBody(), LockBatchRequestBody.class);\n\n        Set<MessageQueue> lockOKMQSet = new HashSet<>();\n        Set<MessageQueue> selfLockOKMQSet = this.brokerController.getRebalanceLockManager().tryLockBatch(\n            requestBody.getConsumerGroup(),\n            requestBody.getMqSet(),\n            requestBody.getClientId());\n        if (requestBody.isOnlyThisBroker() || !brokerController.getBrokerConfig().isLockInStrictMode()) {\n            lockOKMQSet = selfLockOKMQSet;\n        } else {\n            requestBody.setOnlyThisBroker(true);\n            int replicaSize = this.brokerController.getMessageStoreConfig().getTotalReplicas();\n\n            int quorum = replicaSize / 2 + 1;\n\n            if (quorum <= 1) {\n                lockOKMQSet = selfLockOKMQSet;\n            } else {\n                final ConcurrentMap<MessageQueue, Integer> mqLockMap = new ConcurrentHashMap<>();\n                for (MessageQueue mq : selfLockOKMQSet) {\n                    if (!mqLockMap.containsKey(mq)) {\n                        mqLockMap.put(mq, 0);\n                    }\n                    mqLockMap.put(mq, mqLockMap.get(mq) + 1);\n                }\n\n                BrokerMemberGroup memberGroup = this.brokerController.getBrokerMemberGroup();\n\n                if (memberGroup != null) {\n                    Map<Long, String> addrMap = new HashMap<>(memberGroup.getBrokerAddrs());\n                    addrMap.remove(this.brokerController.getBrokerConfig().getBrokerId());\n                    final CountDownLatch countDownLatch = new CountDownLatch(addrMap.size());\n                    requestBody.setMqSet(selfLockOKMQSet);\n                    requestBody.setOnlyThisBroker(true);\n                    for (Long brokerId : addrMap.keySet()) {\n                        try {\n                            this.brokerController.getBrokerOuterAPI().lockBatchMQAsync(addrMap.get(brokerId),\n                                requestBody, 1000, new LockCallback() {\n                                    @Override\n                                    public void onSuccess(Set<MessageQueue> lockOKMQSet) {\n                                        for (MessageQueue mq : lockOKMQSet) {\n                                            if (!mqLockMap.containsKey(mq)) {\n                                                mqLockMap.put(mq, 0);\n                                            }\n                                            mqLockMap.put(mq, mqLockMap.get(mq) + 1);\n                                        }\n                                        countDownLatch.countDown();\n                                    }\n\n                                    @Override\n                                    public void onException(Throwable e) {\n                                        LOGGER.warn(\"lockBatchMQAsync on {} failed, {}\", addrMap.get(brokerId), e);\n                                        countDownLatch.countDown();\n                                    }\n                                });\n                        } catch (Exception e) {\n                            LOGGER.warn(\"lockBatchMQAsync on {} failed, {}\", addrMap.get(brokerId), e);\n                            countDownLatch.countDown();\n                        }\n                    }\n                    try {\n                        countDownLatch.await(2000, TimeUnit.MILLISECONDS);\n                    } catch (InterruptedException e) {\n                        LOGGER.warn(\"lockBatchMQ exception on {}, {}\", this.brokerController.getBrokerConfig().getBrokerName(), e);\n                    }\n                }\n\n                for (MessageQueue mq : mqLockMap.keySet()) {\n                    if (mqLockMap.get(mq) >= quorum) {\n                        lockOKMQSet.add(mq);\n                    }\n                }\n            }\n        }\n\n        LockBatchResponseBody responseBody = new LockBatchResponseBody();\n        responseBody.setLockOKMQSet(lockOKMQSet);\n\n        response.setBody(responseBody.encode());\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand unlockBatchMQ(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        UnlockBatchRequestBody requestBody = UnlockBatchRequestBody.decode(request.getBody(), UnlockBatchRequestBody.class);\n\n        if (requestBody.isOnlyThisBroker() || !this.brokerController.getBrokerConfig().isLockInStrictMode()) {\n            this.brokerController.getRebalanceLockManager().unlockBatch(\n                requestBody.getConsumerGroup(),\n                requestBody.getMqSet(),\n                requestBody.getClientId());\n        } else {\n            requestBody.setOnlyThisBroker(true);\n            BrokerMemberGroup memberGroup = this.brokerController.getBrokerMemberGroup();\n\n            if (memberGroup != null) {\n                Map<Long, String> addrMap = memberGroup.getBrokerAddrs();\n                for (Long brokerId : addrMap.keySet()) {\n                    try {\n                        this.brokerController.getBrokerOuterAPI().unlockBatchMQAsync(addrMap.get(brokerId), requestBody, 1000, new UnlockCallback() {\n                            @Override\n                            public void onSuccess() {\n\n                            }\n\n                            @Override\n                            public void onException(Throwable e) {\n                                LOGGER.warn(\"unlockBatchMQ exception on {}, {}\", addrMap.get(brokerId), e);\n                            }\n                        });\n                    } catch (Exception e) {\n                        LOGGER.warn(\"unlockBatchMQ exception on {}, {}\", addrMap.get(brokerId), e);\n                    }\n                }\n            }\n        }\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand updateAndCreateSubscriptionGroup(ChannelHandlerContext ctx, RemotingCommand request)\n        throws RemotingCommandException {\n        long startTime = System.currentTimeMillis();\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        LOGGER.info(\"AdminBrokerProcessor#updateAndCreateSubscriptionGroup called by {}\",\n            RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n\n        SubscriptionGroupConfig config = RemotingSerializable.decode(request.getBody(), SubscriptionGroupConfig.class);\n        if (null != config) {\n            TopicValidator.ValidateResult result = TopicValidator.validateGroup(config.getGroupName());\n            if (!result.isValid()) {\n                response.setCode(ResponseCode.INVALID_PARAMETER);\n                response.setRemark(result.getRemark());\n                return response;\n            }\n\n            this.brokerController.getSubscriptionGroupManager().updateSubscriptionGroupConfig(config);\n        }\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        long executionTime = System.currentTimeMillis() - startTime;\n        if (null != config) {\n            LOGGER.info(\"executionTime of create subscriptionGroup:{} is {} ms\", config.getGroupName(), executionTime);\n        }\n        InvocationStatus status = response.getCode() == ResponseCode.SUCCESS ?\n            InvocationStatus.SUCCESS : InvocationStatus.FAILURE;\n        Attributes attributes = this.brokerController.getBrokerMetricsManager().newAttributesBuilder()\n            .put(LABEL_INVOCATION_STATUS, status.getName())\n            .build();\n        this.brokerController.getBrokerMetricsManager().getConsumerGroupCreateExecuteTime().record(executionTime, attributes);\n        return response;\n    }\n\n    private RemotingCommand updateAndCreateSubscriptionGroupList(ChannelHandlerContext ctx, RemotingCommand request) {\n        final long startTime = System.nanoTime();\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        final SubscriptionGroupList subscriptionGroupList = SubscriptionGroupList.decode(request.getBody(), SubscriptionGroupList.class);\n        final List<SubscriptionGroupConfig> groupConfigList = subscriptionGroupList.getGroupConfigList();\n\n        final StringBuilder builder = new StringBuilder();\n        for (SubscriptionGroupConfig config : groupConfigList) {\n            TopicValidator.ValidateResult result = TopicValidator.validateGroup(config.getGroupName());\n            if (!result.isValid()) {\n                response.setCode(ResponseCode.INVALID_PARAMETER);\n                response.setRemark(result.getRemark());\n                return response;\n            }\n            builder.append(config.getGroupName()).append(\";\");\n        }\n        final String groupNames = builder.toString();\n        LOGGER.info(\"AdminBrokerProcessor#updateAndCreateSubscriptionGroupList: groupNames: {}, called by {}\",\n            groupNames,\n            RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n\n        try {\n            this.brokerController.getSubscriptionGroupManager().updateSubscriptionGroupConfigList(groupConfigList);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n        } finally {\n            long executionTime = (System.nanoTime() - startTime) / 1000000L;\n            LOGGER.info(\"executionTime of create updateAndCreateSubscriptionGroupList: {} is {} ms\", groupNames, executionTime);\n            InvocationStatus status = response.getCode() == ResponseCode.SUCCESS ?\n                InvocationStatus.SUCCESS : InvocationStatus.FAILURE;\n            Attributes attributes = this.brokerController.getBrokerMetricsManager().newAttributesBuilder()\n                .put(LABEL_INVOCATION_STATUS, status.getName())\n                .build();\n            this.brokerController.getBrokerMetricsManager().getConsumerGroupCreateExecuteTime().record(executionTime, attributes);\n        }\n\n        return response;\n    }\n\n    private void initConsumerOffset(String clientHost, String groupName, int mode, TopicConfig topicConfig)\n        throws ConsumeQueueException {\n        String topic = topicConfig.getTopicName();\n        for (int queueId = 0; queueId < topicConfig.getReadQueueNums(); queueId++) {\n            if (this.brokerController.getConsumerOffsetManager().queryOffset(groupName, topic, queueId) > -1) {\n                continue;\n            }\n            long offset = 0;\n            if (this.brokerController.getMessageStore().getConsumeQueue(topic, queueId) != null) {\n                if (ConsumeInitMode.MAX == mode) {\n                    offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);\n                } else if (ConsumeInitMode.MIN == mode) {\n                    offset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId);\n                }\n            }\n            this.brokerController.getConsumerOffsetManager().commitOffset(clientHost, groupName, topic, queueId, offset);\n            LOGGER.info(\"AdminBrokerProcessor#initConsumerOffset: consumerGroup={}, topic={}, queueId={}, offset={}\",\n                groupName, topic, queueId, offset);\n        }\n    }\n\n    private RemotingCommand getAllSubscriptionGroup(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(GetAllSubscriptionGroupResponseHeader.class);\n        final GetAllSubscriptionGroupResponseHeader responseHeader =\n            (GetAllSubscriptionGroupResponseHeader) response.readCustomHeader();\n        final GetAllSubscriptionGroupRequestHeader requestHeader =\n            request.decodeCommandCustomHeader(GetAllSubscriptionGroupRequestHeader.class);\n\n        String dataVersionStr = requestHeader.getDataVersion();\n        Integer groupSeq = requestHeader.getGroupSeq();\n        Integer maxGroupNum = requestHeader.getMaxGroupNum();\n\n        SubscriptionGroupManager sgManager = this.brokerController.getSubscriptionGroupManager();\n\n        SubscriptionGroupWrapper subscriptionGroupWrapper = new SubscriptionGroupWrapper();\n        if (!brokerController.getBrokerConfig().isEnableSplitMetadata()\n            || ObjectUtils.allNull(dataVersionStr, groupSeq, maxGroupNum)) {\n            subscriptionGroupWrapper.setSubscriptionGroupTable(sgManager.getSubscriptionGroupTable());\n            subscriptionGroupWrapper.setForbiddenTable(sgManager.getForbiddenTable());\n            subscriptionGroupWrapper.setDataVersion(sgManager.getDataVersion());\n        } else {\n            int groupNum = Math.min(brokerController.getBrokerConfig().getSplitMetadataSize(),\n                Optional.ofNullable(maxGroupNum).orElse(Integer.MAX_VALUE));\n            ConcurrentMap<String, SubscriptionGroupConfig> subGroupTable =\n                sgManager.subGroupTable(dataVersionStr, groupSeq, groupNum);\n            subscriptionGroupWrapper.setSubscriptionGroupTable(subGroupTable);\n            subscriptionGroupWrapper.setDataVersion(sgManager.getDataVersion());\n            subscriptionGroupWrapper.setForbiddenTable(sgManager.subForbiddenTable(subGroupTable.keySet()));\n        }\n\n        responseHeader.setTotalGroupNum(sgManager.getSubscriptionGroupTable().size());\n        String content = subscriptionGroupWrapper.toJson();\n        if (StringUtils.isNotBlank(content)) {\n            response.setBody(content.getBytes(StandardCharsets.UTF_8));\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n        } else {\n            LOGGER.error(\"No subscription group in this broker, client:{} \", ctx.channel().remoteAddress());\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"No subscription group in this broker\");\n        }\n        return response;\n    }\n\n    private RemotingCommand deleteSubscriptionGroup(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        DeleteSubscriptionGroupRequestHeader requestHeader =\n            (DeleteSubscriptionGroupRequestHeader) request.decodeCommandCustomHeader(DeleteSubscriptionGroupRequestHeader.class);\n\n        LOGGER.info(\"AdminBrokerProcessor#deleteSubscriptionGroup, caller={}\",\n            RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n\n        this.brokerController.getSubscriptionGroupManager().deleteSubscriptionGroupConfig(requestHeader.getGroupName());\n\n        if (requestHeader.isCleanOffset()\n            || LiteMetadataUtil.isLiteGroupType(requestHeader.getGroupName(), this.brokerController)) {\n            this.brokerController.getConsumerOffsetManager().removeOffset(requestHeader.getGroupName());\n            this.brokerController.getPopInflightMessageCounter().clearInFlightMessageNumByGroupName(requestHeader.getGroupName());\n        }\n\n        if (this.brokerController.getBrokerConfig().isAutoDeleteUnusedStats()) {\n            this.brokerController.getBrokerStatsManager().onGroupDeleted(requestHeader.getGroupName());\n        }\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand getTopicStatsInfo(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final GetTopicStatsInfoRequestHeader requestHeader = request.decodeCommandCustomHeader(GetTopicStatsInfoRequestHeader.class);\n\n        final String topic = requestHeader.getTopic();\n        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);\n        if (null == topicConfig) {\n            response.setCode(ResponseCode.TOPIC_NOT_EXIST);\n            response.setRemark(\"topic[\" + topic + \"] not exist\");\n            return response;\n        }\n\n        TopicStatsTable topicStatsTable = new TopicStatsTable();\n\n        int maxQueueNums = Math.max(topicConfig.getWriteQueueNums(), topicConfig.getReadQueueNums());\n        try {\n            for (int i = 0; i < maxQueueNums; i++) {\n                MessageQueue mq = new MessageQueue();\n                mq.setTopic(topic);\n                mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());\n                mq.setQueueId(i);\n\n                TopicOffset topicOffset = new TopicOffset();\n                long min = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, i);\n                if (min < 0) {\n                    min = 0;\n                }\n\n                long max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);\n                if (max < 0) {\n                    max = 0;\n                }\n\n                long timestamp = 0;\n                if (max > 0) {\n                    timestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, max - 1);\n                }\n\n                topicOffset.setMinOffset(min);\n                topicOffset.setMaxOffset(max);\n                topicOffset.setLastUpdateTimestamp(timestamp);\n\n                topicStatsTable.getOffsetTable().put(mq, topicOffset);\n            }\n\n            topicStatsTable.setTopicPutTps(this.brokerController.getBrokerStatsManager().tpsTopicPutNums(requestHeader.getTopic()));\n            byte[] body = topicStatsTable.encode();\n            response.setBody(body);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n        } catch (ConsumeQueueException e) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(e.getMessage());\n        }\n\n        return response;\n    }\n\n    private RemotingCommand getConsumerConnectionList(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final GetConsumerConnectionListRequestHeader requestHeader =\n            (GetConsumerConnectionListRequestHeader) request.decodeCommandCustomHeader(GetConsumerConnectionListRequestHeader.class);\n\n        ConsumerGroupInfo consumerGroupInfo =\n            this.brokerController.getConsumerManager().getConsumerGroupInfo(requestHeader.getConsumerGroup());\n        if (consumerGroupInfo != null) {\n            ConsumerConnection bodydata = new ConsumerConnection();\n            bodydata.setConsumeFromWhere(consumerGroupInfo.getConsumeFromWhere());\n            bodydata.setConsumeType(consumerGroupInfo.getConsumeType());\n            bodydata.setMessageModel(consumerGroupInfo.getMessageModel());\n            bodydata.getSubscriptionTable().putAll(consumerGroupInfo.getSubscriptionTable());\n\n            Iterator<Map.Entry<Channel, ClientChannelInfo>> it = consumerGroupInfo.getChannelInfoTable().entrySet().iterator();\n            while (it.hasNext()) {\n                ClientChannelInfo info = it.next().getValue();\n                Connection connection = new Connection();\n                connection.setClientId(info.getClientId());\n                connection.setLanguage(info.getLanguage());\n                connection.setVersion(info.getVersion());\n                connection.setClientAddr(RemotingHelper.parseChannelRemoteAddr(info.getChannel()));\n\n                bodydata.getConnectionSet().add(connection);\n            }\n\n            byte[] body = bodydata.encode();\n            response.setBody(body);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n\n            return response;\n        }\n\n        response.setCode(ResponseCode.CONSUMER_NOT_ONLINE);\n        response.setRemark(\"the consumer group[\" + requestHeader.getConsumerGroup() + \"] not online\");\n        return response;\n    }\n\n    private RemotingCommand getAllProducerInfo(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final GetAllProducerInfoRequestHeader requestHeader =\n            (GetAllProducerInfoRequestHeader) request.decodeCommandCustomHeader(GetAllProducerInfoRequestHeader.class);\n\n        ProducerTableInfo producerTable = this.brokerController.getProducerManager().getProducerTable();\n        if (producerTable != null) {\n            byte[] body = producerTable.encode();\n            response.setBody(body);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n            return response;\n        }\n\n        response.setCode(ResponseCode.SYSTEM_ERROR);\n        return response;\n    }\n\n    private RemotingCommand getProducerConnectionList(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final GetProducerConnectionListRequestHeader requestHeader =\n            (GetProducerConnectionListRequestHeader) request.decodeCommandCustomHeader(GetProducerConnectionListRequestHeader.class);\n\n        ProducerConnection bodydata = new ProducerConnection();\n        Map<Channel, ClientChannelInfo> channelInfoHashMap =\n            this.brokerController.getProducerManager().getGroupChannelTable().get(requestHeader.getProducerGroup());\n        if (channelInfoHashMap != null) {\n            Iterator<Map.Entry<Channel, ClientChannelInfo>> it = channelInfoHashMap.entrySet().iterator();\n            while (it.hasNext()) {\n                ClientChannelInfo info = it.next().getValue();\n                Connection connection = new Connection();\n                connection.setClientId(info.getClientId());\n                connection.setLanguage(info.getLanguage());\n                connection.setVersion(info.getVersion());\n                connection.setClientAddr(RemotingHelper.parseChannelRemoteAddr(info.getChannel()));\n\n                bodydata.getConnectionSet().add(connection);\n            }\n\n            byte[] body = bodydata.encode();\n            response.setBody(body);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n            return response;\n        }\n\n        response.setCode(ResponseCode.SYSTEM_ERROR);\n        response.setRemark(\"the producer group[\" + requestHeader.getProducerGroup() + \"] not exist\");\n        return response;\n    }\n\n    private RemotingCommand getConsumeStats(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        try {\n            final GetConsumeStatsRequestHeader requestHeader = request.decodeCommandCustomHeader(GetConsumeStatsRequestHeader.class);\n            List<String> topicListProvided = requestHeader.fetchTopicList();\n            String topicProvided = requestHeader.getTopic();\n            String group = requestHeader.getConsumerGroup();\n\n            ConsumeStats consumeStats = new ConsumeStats();\n            Set<String> topicsForCollecting = getTopicsForCollectingConsumeStats(topicListProvided, topicProvided, group);\n\n            for (String topic : topicsForCollecting) {\n                TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);\n                if (null == topicConfig) {\n                    LOGGER.warn(\"AdminBrokerProcessor#getConsumeStats: topic config does not exist, topic={}\", topic);\n                    continue;\n                }\n\n                TopicQueueMappingDetail mappingDetail = this.brokerController.getTopicQueueMappingManager().getTopicQueueMapping(topic);\n                for (int i = 0; i < topicConfig.getReadQueueNums(); i++) {\n                    MessageQueue mq = new MessageQueue();\n                    mq.setTopic(topic);\n                    mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());\n                    mq.setQueueId(i);\n\n                    OffsetWrapper offsetWrapper = new OffsetWrapper();\n\n                    long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);\n                    if (brokerOffset < 0) {\n                        brokerOffset = 0;\n                    }\n\n                    long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset(\n                        requestHeader.getConsumerGroup(), topic, i);\n\n                    // the consumerOffset cannot be zero for static topic because of the \"double read check\" strategy\n                    // just remain the logic for dynamic topic\n                    // maybe we should remove it in the future\n                    if (mappingDetail == null) {\n                        if (consumerOffset < 0) {\n                            consumerOffset = 0;\n                        }\n                    }\n\n                    long pullOffset = this.brokerController.getConsumerOffsetManager().queryPullOffset(\n                        requestHeader.getConsumerGroup(), topic, i);\n\n                    offsetWrapper.setBrokerOffset(brokerOffset);\n                    offsetWrapper.setConsumerOffset(consumerOffset);\n                    offsetWrapper.setPullOffset(Math.max(consumerOffset, pullOffset));\n\n                    long timeOffset = consumerOffset - 1;\n                    if (timeOffset >= 0) {\n                        long lastTimestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, timeOffset);\n                        if (lastTimestamp > 0) {\n                            offsetWrapper.setLastTimestamp(lastTimestamp);\n                        }\n                    }\n\n                    consumeStats.getOffsetTable().put(mq, offsetWrapper);\n                }\n\n                double consumeTps = this.brokerController.getBrokerStatsManager().tpsGroupGetNums(requestHeader.getConsumerGroup(), topic);\n\n                consumeTps += consumeStats.getConsumeTps();\n                consumeStats.setConsumeTps(consumeTps);\n            }\n\n            byte[] body = consumeStats.encode();\n            response.setBody(body);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n        } catch (ConsumeQueueException e) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(e.getMessage());\n        }\n        return response;\n    }\n\n    private Set<String> getTopicsForCollectingConsumeStats(List<String> topicListProvided, String topicProvided,\n        String group) {\n        Set<String> topicsForCollecting = new HashSet<>();\n        if (!topicListProvided.isEmpty()) {\n            // if topic list is provided, only collect the topics in the list\n            // and ignore subscription check\n            topicsForCollecting.addAll(topicListProvided);\n        } else {\n            // In order to be compatible with the old logic,\n            // even if the topic has been provided here, the subscription will be checked.\n            if (UtilAll.isBlank(topicProvided)) {\n                topicsForCollecting.addAll(\n                    this.brokerController.getConsumerOffsetManager().whichTopicByConsumer(group));\n            } else {\n                topicsForCollecting.add(topicProvided);\n            }\n            int subscriptionCount = this.brokerController.getConsumerManager().findSubscriptionDataCount(group);\n            Iterator<String> iterator = topicsForCollecting.iterator();\n            while (iterator.hasNext()) {\n                String topic = iterator.next();\n                SubscriptionData findSubscriptionData = this.brokerController.getConsumerManager().findSubscriptionData(group, topic);\n                if (findSubscriptionData == null && subscriptionCount > 0) {\n                    LOGGER.warn(\n                        \"AdminBrokerProcessor#getConsumeStats: topic does not exist in consumer group's subscription, topic={}, consumer group={}\",\n                        topic, group);\n                    iterator.remove();\n                }\n            }\n        }\n        return topicsForCollecting;\n    }\n\n    private RemotingCommand getAllConsumerOffset(ChannelHandlerContext ctx, RemotingCommand request) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        String content = this.brokerController.getConsumerOffsetManager().encode();\n        if (content != null && content.length() > 0) {\n            try {\n                response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET));\n            } catch (UnsupportedEncodingException e) {\n                LOGGER.error(\"get all consumer offset from master error.\", e);\n\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"UnsupportedEncodingException \" + e.getMessage());\n                return response;\n            }\n        } else {\n            LOGGER.error(\"No consumer offset in this broker, client: {} \", ctx.channel().remoteAddress());\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"No consumer offset in this broker\");\n            return response;\n        }\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n\n        return response;\n    }\n\n    private RemotingCommand getAllDelayOffset(ChannelHandlerContext ctx, RemotingCommand request) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        String content = this.brokerController.getScheduleMessageService().encode();\n        if (content != null && content.length() > 0) {\n            try {\n                response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET));\n            } catch (UnsupportedEncodingException e) {\n                LOGGER.error(\"AdminBrokerProcessor#getAllDelayOffset: unexpected error, caller={}.\",\n                    RemotingHelper.parseChannelRemoteAddr(ctx.channel()), e);\n\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"UnsupportedEncodingException \" + e);\n                return response;\n            }\n        } else {\n            LOGGER.error(\"AdminBrokerProcessor#getAllDelayOffset: no delay offset in this broker, caller={}\",\n                RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"No delay offset in this broker\");\n            return response;\n        }\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n\n        return response;\n    }\n\n    private RemotingCommand getAllMessageRequestMode(ChannelHandlerContext ctx, RemotingCommand request) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        String content = this.brokerController.getQueryAssignmentProcessor().getMessageRequestModeManager().encode();\n        if (content != null && !content.isEmpty()) {\n            try {\n                response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET));\n            } catch (UnsupportedEncodingException e) {\n                LOGGER.error(\"get all message request mode from master error.\", e);\n\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"UnsupportedEncodingException \" + e);\n                return response;\n            }\n        } else {\n            LOGGER.error(\"No message request mode in this broker, client: {} \", ctx.channel().remoteAddress());\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"No message request mode in this broker\");\n            return response;\n        }\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n\n        return response;\n    }\n\n    public RemotingCommand resetOffset(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final ResetOffsetRequestHeader requestHeader =\n            (ResetOffsetRequestHeader) request.decodeCommandCustomHeader(ResetOffsetRequestHeader.class);\n        LOGGER.info(\"[reset-offset] reset offset started by {}. topic={}, group={}, timestamp={}, isForce={}\",\n            RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.getTopic(), requestHeader.getGroup(),\n            requestHeader.getTimestamp(), requestHeader.isForce());\n\n        if (this.brokerController.getBrokerConfig().isUseServerSideResetOffset()) {\n            String topic = requestHeader.getTopic();\n            String group = requestHeader.getGroup();\n            int queueId = requestHeader.getQueueId();\n            long timestamp = requestHeader.getTimestamp();\n            Long offset = requestHeader.getOffset();\n            return resetOffsetInner(topic, group, queueId, timestamp, offset);\n        }\n\n        boolean isC = false;\n        LanguageCode language = request.getLanguage();\n        switch (language) {\n            case CPP:\n                isC = true;\n                break;\n        }\n        return this.brokerController.getBroker2Client().resetOffset(requestHeader.getTopic(), requestHeader.getGroup(),\n            requestHeader.getTimestamp(), requestHeader.isForce(), isC);\n    }\n\n    private Long searchOffsetByTimestamp(String topic, int queueId, long timestamp) throws ConsumeQueueException {\n        if (timestamp < 0) {\n            return brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);\n        } else {\n            return brokerController.getMessageStore().getOffsetInQueueByTime(topic, queueId, timestamp);\n        }\n    }\n\n    /**\n     * Reset consumer offset.\n     *\n     * @param topic     Required, not null.\n     * @param group     Required, not null.\n     * @param queueId   if target queue ID is negative, all message queues will be reset; otherwise, only the target queue\n     *                  would get reset.\n     * @param timestamp if timestamp is negative, offset would be reset to broker offset at the time being; otherwise,\n     *                  binary search is performed to locate target offset.\n     * @param offset    Target offset to reset to if target queue ID is properly provided.\n     * @return Affected queues and their new offset\n     */\n    private RemotingCommand resetOffsetInner(String topic, String group, int queueId, long timestamp, Long offset) {\n        RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, null);\n\n        if (BrokerRole.SLAVE == brokerController.getMessageStoreConfig().getBrokerRole()) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"Can not reset offset in slave broker\");\n            return response;\n        }\n\n        Map<Integer, Long> queueOffsetMap = new HashMap<>();\n\n        // Reset offset for all queues belonging to the specified topic\n        TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(topic);\n        if (null == topicConfig) {\n            response.setCode(ResponseCode.TOPIC_NOT_EXIST);\n            response.setRemark(\"Topic \" + topic + \" does not exist\");\n            LOGGER.warn(\"Reset offset failed, topic does not exist. topic={}, group={}\", topic, group);\n            return response;\n        }\n\n        if (!brokerController.getSubscriptionGroupManager().containsSubscriptionGroup(group)) {\n            response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);\n            response.setRemark(\"Group \" + group + \" does not exist\");\n            LOGGER.warn(\"Reset offset failed, group does not exist. topic={}, group={}\", topic, group);\n            return response;\n        }\n\n        try {\n            if (queueId >= 0) {\n                if (null != offset && -1 != offset) {\n                    long min = brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId);\n                    long max = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);\n                    if (min >= 0 && offset < min || offset > max + 1) {\n                        response.setCode(ResponseCode.SYSTEM_ERROR);\n                        response.setRemark(\n                            String.format(\"Target offset %d not in consume queue range [%d-%d]\", offset, min, max));\n                        return response;\n                    }\n                } else {\n                    offset = searchOffsetByTimestamp(topic, queueId, timestamp);\n                }\n                queueOffsetMap.put(queueId, offset);\n            } else {\n                for (int index = 0; index < topicConfig.getReadQueueNums(); index++) {\n                    offset = searchOffsetByTimestamp(topic, index, timestamp);\n                    queueOffsetMap.put(index, offset);\n                }\n            }\n        } catch (ConsumeQueueException e) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(e.getMessage());\n            return response;\n        }\n\n        if (queueOffsetMap.isEmpty()) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"No queues to reset.\");\n            LOGGER.warn(\"Reset offset aborted: no queues to reset\");\n            return response;\n        }\n\n        for (Map.Entry<Integer, Long> entry : queueOffsetMap.entrySet()) {\n            brokerController.getConsumerOffsetManager()\n                .assignResetOffset(topic, group, entry.getKey(), entry.getValue());\n        }\n\n        // Prepare reset result.\n        ResetOffsetBody body = new ResetOffsetBody();\n        String brokerName = brokerController.getBrokerConfig().getBrokerName();\n        for (Map.Entry<Integer, Long> entry : queueOffsetMap.entrySet()) {\n            if (brokerController.getPopInflightMessageCounter() != null) {\n                brokerController.getPopInflightMessageCounter().clearInFlightMessageNum(topic, group, entry.getKey());\n            }\n            if (brokerController.getBrokerConfig().isPopConsumerKVServiceEnable()) {\n                brokerController.getPopConsumerService().clearCache(group, topic, entry.getKey());\n                brokerController.getConsumerOffsetManager().clearPullOffset(group, topic);\n            }\n            body.getOffsetTable().put(new MessageQueue(topic, brokerName, entry.getKey()), entry.getValue());\n        }\n\n        LOGGER.info(\"Reset offset, topic={}, group={}, queues={}\", topic, group, body.toJson(false));\n        response.setBody(body.encode());\n        return response;\n    }\n\n    public RemotingCommand getConsumerStatus(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final GetConsumerStatusRequestHeader requestHeader =\n            (GetConsumerStatusRequestHeader) request.decodeCommandCustomHeader(GetConsumerStatusRequestHeader.class);\n\n        LOGGER.info(\"[get-consumer-status] get consumer status by {}. topic={}, group={}\",\n            RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.getTopic(), requestHeader.getGroup());\n\n        return this.brokerController.getBroker2Client().getConsumeStatus(requestHeader.getTopic(), requestHeader.getGroup(),\n            requestHeader.getClientAddr());\n    }\n\n    private RemotingCommand queryTopicConsumeByWho(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        QueryTopicConsumeByWhoRequestHeader requestHeader =\n            (QueryTopicConsumeByWhoRequestHeader) request.decodeCommandCustomHeader(QueryTopicConsumeByWhoRequestHeader.class);\n\n        HashSet<String> groups = this.brokerController.getConsumerManager().queryTopicConsumeByWho(requestHeader.getTopic());\n\n        Set<String> groupInOffset = this.brokerController.getConsumerOffsetManager().whichGroupByTopic(requestHeader.getTopic());\n        if (groupInOffset != null && !groupInOffset.isEmpty()) {\n            groups.addAll(groupInOffset);\n        }\n\n        GroupList groupList = new GroupList();\n        groupList.setGroupList(groups);\n        byte[] body = groupList.encode();\n\n        response.setBody(body);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand queryTopicsByConsumer(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        QueryTopicsByConsumerRequestHeader requestHeader =\n            (QueryTopicsByConsumerRequestHeader) request.decodeCommandCustomHeader(QueryTopicsByConsumerRequestHeader.class);\n\n        Set<String> topics = this.brokerController.getConsumerOffsetManager().whichTopicByConsumer(requestHeader.getGroup());\n\n        TopicList topicList = new TopicList();\n        topicList.setTopicList(topics);\n        topicList.setBrokerAddr(brokerController.getBrokerAddr());\n        byte[] body = topicList.encode();\n\n        response.setBody(body);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand querySubscriptionByConsumer(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        QuerySubscriptionByConsumerRequestHeader requestHeader =\n            (QuerySubscriptionByConsumerRequestHeader) request.decodeCommandCustomHeader(QuerySubscriptionByConsumerRequestHeader.class);\n\n        SubscriptionData subscriptionData = this.brokerController.getConsumerManager()\n            .findSubscriptionData(requestHeader.getGroup(), requestHeader.getTopic());\n\n        QuerySubscriptionResponseBody responseBody = new QuerySubscriptionResponseBody();\n        responseBody.setGroup(requestHeader.getGroup());\n        responseBody.setTopic(requestHeader.getTopic());\n        responseBody.setSubscriptionData(subscriptionData);\n        byte[] body = responseBody.encode();\n\n        response.setBody(body);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n\n    }\n\n    private RemotingCommand queryConsumeTimeSpan(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        QueryConsumeTimeSpanRequestHeader requestHeader = request.decodeCommandCustomHeader(QueryConsumeTimeSpanRequestHeader.class);\n\n        final String topic = requestHeader.getTopic();\n        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);\n        if (null == topicConfig) {\n            response.setCode(ResponseCode.TOPIC_NOT_EXIST);\n            response.setRemark(\"topic[\" + topic + \"] not exist\");\n            return response;\n        }\n\n        List<QueueTimeSpan> timeSpanSet = new ArrayList<>();\n        for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) {\n            QueueTimeSpan timeSpan = new QueueTimeSpan();\n            MessageQueue mq = new MessageQueue();\n            mq.setTopic(topic);\n            mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());\n            mq.setQueueId(i);\n            timeSpan.setMessageQueue(mq);\n\n            long minTime = this.brokerController.getMessageStore().getEarliestMessageTime(topic, i);\n            timeSpan.setMinTimeStamp(minTime);\n\n            long max;\n            try {\n                max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);\n            } catch (ConsumeQueueException e) {\n                throw new RemotingCommandException(\"Failed to get max offset in queue\", e);\n            }\n            long maxTime = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, max - 1);\n            timeSpan.setMaxTimeStamp(maxTime);\n\n            long consumeTime;\n            long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset(\n                requestHeader.getGroup(), topic, i);\n            if (consumerOffset > 0) {\n                consumeTime = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, consumerOffset - 1);\n            } else {\n                consumeTime = minTime;\n            }\n            timeSpan.setConsumeTimeStamp(consumeTime);\n\n            long maxBrokerOffset;\n            try {\n                maxBrokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), i);\n            } catch (ConsumeQueueException e) {\n                throw new RemotingCommandException(\"Failed to get max offset in queue\", e);\n            }\n            if (consumerOffset < maxBrokerOffset) {\n                long nextTime = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, consumerOffset);\n                timeSpan.setDelayTime(System.currentTimeMillis() - nextTime);\n            }\n            timeSpanSet.add(timeSpan);\n        }\n\n        QueryConsumeTimeSpanBody queryConsumeTimeSpanBody = new QueryConsumeTimeSpanBody();\n        queryConsumeTimeSpanBody.setConsumeTimeSpanSet(timeSpanSet);\n        response.setBody(queryConsumeTimeSpanBody.encode());\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand getSystemTopicListFromBroker(ChannelHandlerContext ctx, RemotingCommand request)\n        throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        Set<String> topics = TopicValidator.getSystemTopicSet();\n        TopicList topicList = new TopicList();\n        topicList.setTopicList(topics);\n        response.setBody(topicList.encode());\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    public RemotingCommand cleanExpiredConsumeQueue() {\n        LOGGER.info(\"AdminBrokerProcessor#cleanExpiredConsumeQueue: start.\");\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        try {\n            brokerController.getMessageStore().cleanExpiredConsumerQueue();\n        } catch (Throwable t) {\n            return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());\n        }\n        LOGGER.info(\"AdminBrokerProcessor#cleanExpiredConsumeQueue: end.\");\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    public RemotingCommand deleteExpiredCommitLog() {\n        LOGGER.warn(\"invoke deleteExpiredCommitLog start.\");\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        brokerController.getMessageStore().executeDeleteFilesManually();\n        LOGGER.warn(\"invoke deleteExpiredCommitLog end.\");\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    public RemotingCommand cleanUnusedTopic() {\n        LOGGER.warn(\"invoke cleanUnusedTopic start.\");\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        brokerController.getMessageStore().cleanUnusedTopic(brokerController.getTopicConfigManager().getTopicConfigTable().keySet());\n        LOGGER.warn(\"invoke cleanUnusedTopic end.\");\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand getConsumerRunningInfo(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final GetConsumerRunningInfoRequestHeader requestHeader =\n            (GetConsumerRunningInfoRequestHeader) request.decodeCommandCustomHeader(GetConsumerRunningInfoRequestHeader.class);\n\n        return this.callConsumer(RequestCode.GET_CONSUMER_RUNNING_INFO, request, requestHeader.getConsumerGroup(),\n            requestHeader.getClientId());\n    }\n\n    private RemotingCommand queryCorrectionOffset(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        QueryCorrectionOffsetHeader requestHeader =\n            (QueryCorrectionOffsetHeader) request.decodeCommandCustomHeader(QueryCorrectionOffsetHeader.class);\n\n        Map<Integer, Long> correctionOffset = this.brokerController.getConsumerOffsetManager()\n            .queryMinOffsetInAllGroup(requestHeader.getTopic(), requestHeader.getFilterGroups());\n\n        Map<Integer, Long> compareOffset =\n            this.brokerController.getConsumerOffsetManager().queryOffset(requestHeader.getCompareGroup(), requestHeader.getTopic());\n\n        if (compareOffset != null && !compareOffset.isEmpty()) {\n            for (Map.Entry<Integer, Long> entry : compareOffset.entrySet()) {\n                Integer queueId = entry.getKey();\n                correctionOffset.put(queueId,\n                    correctionOffset.get(queueId) > entry.getValue() ? Long.MAX_VALUE : correctionOffset.get(queueId));\n            }\n        }\n\n        QueryCorrectionOffsetBody body = new QueryCorrectionOffsetBody();\n        body.setCorrectionOffsets(correctionOffset);\n        response.setBody(body.encode());\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand consumeMessageDirectly(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final ConsumeMessageDirectlyResultRequestHeader requestHeader = (ConsumeMessageDirectlyResultRequestHeader) request\n            .decodeCommandCustomHeader(ConsumeMessageDirectlyResultRequestHeader.class);\n\n        // brokerName\n        request.getExtFields().put(\"brokerName\", this.brokerController.getBrokerConfig().getBrokerName());\n        // topicSysFlag\n        if (StringUtils.isNotEmpty(requestHeader.getTopic())) {\n            TopicConfig topicConfig = this.brokerController.getTopicConfigManager().getTopicConfigTable().get(requestHeader.getTopic());\n            if (topicConfig != null) {\n                request.addExtField(\"topicSysFlag\", String.valueOf(topicConfig.getTopicSysFlag()));\n            }\n        }\n        // groupSysFlag\n        if (StringUtils.isNotEmpty(requestHeader.getConsumerGroup())) {\n            SubscriptionGroupConfig groupConfig = brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getConsumerGroup());\n            if (groupConfig != null) {\n                request.addExtField(\"groupSysFlag\", String.valueOf(groupConfig.getGroupSysFlag()));\n            }\n        }\n        SelectMappedBufferResult selectMappedBufferResult = null;\n        try {\n            MessageId messageId = MessageDecoder.decodeMessageId(requestHeader.getMsgId());\n            selectMappedBufferResult = this.brokerController.getMessageStore().selectOneMessageByOffset(messageId.getOffset());\n\n            byte[] body = new byte[selectMappedBufferResult.getSize()];\n            selectMappedBufferResult.getByteBuffer().get(body);\n            request.setBody(body);\n        } catch (UnknownHostException e) {\n        } finally {\n            if (selectMappedBufferResult != null) {\n                selectMappedBufferResult.release();\n            }\n        }\n\n        return this.callConsumer(RequestCode.CONSUME_MESSAGE_DIRECTLY, request, requestHeader.getConsumerGroup(),\n            requestHeader.getClientId());\n    }\n\n    private RemotingCommand cloneGroupOffset(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        CloneGroupOffsetRequestHeader requestHeader =\n            (CloneGroupOffsetRequestHeader) request.decodeCommandCustomHeader(CloneGroupOffsetRequestHeader.class);\n\n        Set<String> topics;\n        if (UtilAll.isBlank(requestHeader.getTopic())) {\n            topics = this.brokerController.getConsumerOffsetManager().whichTopicByConsumer(requestHeader.getSrcGroup());\n        } else {\n            topics = new HashSet<>();\n            topics.add(requestHeader.getTopic());\n        }\n\n        for (String topic : topics) {\n            TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);\n            if (null == topicConfig) {\n                LOGGER.warn(\"[cloneGroupOffset], topic config not exist, {}\", topic);\n                continue;\n            }\n\n            if (!requestHeader.isOffline()) {\n\n                SubscriptionData findSubscriptionData =\n                    this.brokerController.getConsumerManager().findSubscriptionData(requestHeader.getSrcGroup(), topic);\n                if (this.brokerController.getConsumerManager().findSubscriptionDataCount(requestHeader.getSrcGroup()) > 0\n                    && findSubscriptionData == null) {\n                    LOGGER.warn(\n                        \"AdminBrokerProcessor#cloneGroupOffset: topic does not exist in consumer group's \"\n                            + \"subscription, topic={}, consumer group={}\", topic, requestHeader.getSrcGroup());\n                    continue;\n                }\n            }\n\n            this.brokerController.getConsumerOffsetManager().cloneOffset(requestHeader.getSrcGroup(), requestHeader.getDestGroup(),\n                requestHeader.getTopic());\n        }\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand ViewBrokerStatsData(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final ViewBrokerStatsDataRequestHeader requestHeader =\n            (ViewBrokerStatsDataRequestHeader) request.decodeCommandCustomHeader(ViewBrokerStatsDataRequestHeader.class);\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        MessageStore messageStore = this.brokerController.getMessageStore();\n\n        StatsItem statsItem = messageStore.getBrokerStatsManager().getStatsItem(requestHeader.getStatsName(), requestHeader.getStatsKey());\n        if (null == statsItem) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(String.format(\"The stats <%s> <%s> not exist\", requestHeader.getStatsName(), requestHeader.getStatsKey()));\n            return response;\n        }\n\n        BrokerStatsData brokerStatsData = new BrokerStatsData();\n\n        {\n            BrokerStatsItem it = new BrokerStatsItem();\n            StatsSnapshot ss = statsItem.getStatsDataInMinute();\n            it.setSum(ss.getSum());\n            it.setTps(ss.getTps());\n            it.setAvgpt(ss.getAvgpt());\n            brokerStatsData.setStatsMinute(it);\n        }\n\n        {\n            BrokerStatsItem it = new BrokerStatsItem();\n            StatsSnapshot ss = statsItem.getStatsDataInHour();\n            it.setSum(ss.getSum());\n            it.setTps(ss.getTps());\n            it.setAvgpt(ss.getAvgpt());\n            brokerStatsData.setStatsHour(it);\n        }\n\n        {\n            BrokerStatsItem it = new BrokerStatsItem();\n            StatsSnapshot ss = statsItem.getStatsDataInDay();\n            it.setSum(ss.getSum());\n            it.setTps(ss.getTps());\n            it.setAvgpt(ss.getAvgpt());\n            brokerStatsData.setStatsDay(it);\n        }\n\n        response.setBody(brokerStatsData.encode());\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand fetchAllConsumeStatsInBroker(ChannelHandlerContext ctx, RemotingCommand request)\n        throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        GetConsumeStatsInBrokerHeader requestHeader = request.decodeCommandCustomHeader(GetConsumeStatsInBrokerHeader.class);\n        boolean isOrder = requestHeader.isOrder();\n        ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroups =\n            brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable();\n\n        List<Map<String/* subscriptionGroupName */, List<ConsumeStats>>> brokerConsumeStatsList =\n            new ArrayList<>();\n\n        long totalDiff = 0L;\n        long totalInflightDiff = 0L;\n        for (String group : subscriptionGroups.keySet()) {\n            Map<String, List<ConsumeStats>> subscripTopicConsumeMap = new HashMap<>();\n            Set<String> topics = this.brokerController.getConsumerOffsetManager().whichTopicByConsumer(group);\n            List<ConsumeStats> consumeStatsList = new ArrayList<>();\n            for (String topic : topics) {\n                ConsumeStats consumeStats = new ConsumeStats();\n                TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);\n                if (null == topicConfig) {\n                    LOGGER.warn(\n                        \"AdminBrokerProcessor#fetchAllConsumeStatsInBroker: topic config does not exist, topic={}\",\n                        topic);\n                    continue;\n                }\n\n                if (isOrder && !topicConfig.isOrder()) {\n                    continue;\n                }\n\n                {\n                    SubscriptionData findSubscriptionData = this.brokerController.getConsumerManager().findSubscriptionData(group, topic);\n\n                    if (null == findSubscriptionData\n                        && this.brokerController.getConsumerManager().findSubscriptionDataCount(group) > 0) {\n                        LOGGER.warn(\n                            \"AdminBrokerProcessor#fetchAllConsumeStatsInBroker: topic does not exist in consumer \"\n                                + \"group's subscription, topic={}, consumer group={}\", topic, group);\n                        continue;\n                    }\n                }\n\n                for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) {\n                    MessageQueue mq = new MessageQueue();\n                    mq.setTopic(topic);\n                    mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());\n                    mq.setQueueId(i);\n                    OffsetWrapper offsetWrapper = new OffsetWrapper();\n                    long brokerOffset;\n                    try {\n                        brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);\n                    } catch (ConsumeQueueException e) {\n                        throw new RemotingCommandException(\"Failed to get max offset\", e);\n                    }\n                    if (brokerOffset < 0) {\n                        brokerOffset = 0;\n                    }\n                    long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset(\n                        group,\n                        topic,\n                        i);\n                    if (consumerOffset < 0)\n                        consumerOffset = 0;\n\n                    offsetWrapper.setBrokerOffset(brokerOffset);\n                    offsetWrapper.setConsumerOffset(consumerOffset);\n\n                    long timeOffset = consumerOffset - 1;\n                    if (timeOffset >= 0) {\n                        long lastTimestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, timeOffset);\n                        if (lastTimestamp > 0) {\n                            offsetWrapper.setLastTimestamp(lastTimestamp);\n                        }\n                    }\n                    consumeStats.getOffsetTable().put(mq, offsetWrapper);\n                }\n                double consumeTps = this.brokerController.getBrokerStatsManager().tpsGroupGetNums(group, topic);\n                consumeTps += consumeStats.getConsumeTps();\n                consumeStats.setConsumeTps(consumeTps);\n                totalDiff += consumeStats.computeTotalDiff();\n                totalInflightDiff += consumeStats.computeInflightTotalDiff();\n                consumeStatsList.add(consumeStats);\n            }\n            subscripTopicConsumeMap.put(group, consumeStatsList);\n            brokerConsumeStatsList.add(subscripTopicConsumeMap);\n        }\n        ConsumeStatsList consumeStats = new ConsumeStatsList();\n        consumeStats.setBrokerAddr(brokerController.getBrokerAddr());\n        consumeStats.setConsumeStatsList(brokerConsumeStatsList);\n        consumeStats.setTotalDiff(totalDiff);\n        consumeStats.setTotalInflightDiff(totalInflightDiff);\n        response.setBody(consumeStats.encode());\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private HashMap<String, String> prepareRuntimeInfo() throws RemotingCommandException {\n        HashMap<String, String> runtimeInfo = this.brokerController.getMessageStore().getRuntimeInfo();\n\n        for (BrokerAttachedPlugin brokerAttachedPlugin : brokerController.getBrokerAttachedPlugins()) {\n            if (brokerAttachedPlugin != null) {\n                brokerAttachedPlugin.buildRuntimeInfo(runtimeInfo);\n            }\n        }\n\n        try {\n            this.brokerController.getScheduleMessageService().buildRunningStats(runtimeInfo);\n        } catch (ConsumeQueueException e) {\n            throw new RemotingCommandException(\"Failed to get max offset in queue\", e);\n        }\n        runtimeInfo.put(\"brokerActive\", String.valueOf(this.brokerController.isSpecialServiceRunning()));\n        runtimeInfo.put(\"brokerVersionDesc\", MQVersion.getVersionDesc(MQVersion.CURRENT_VERSION));\n\n        runtimeInfo.put(\"msgPutTotalYesterdayMorning\",\n            String.valueOf(this.brokerController.getBrokerStats().getMsgPutTotalYesterdayMorning()));\n        runtimeInfo.put(\"msgPutTotalTodayMorning\", String.valueOf(this.brokerController.getBrokerStats().getMsgPutTotalTodayMorning()));\n        runtimeInfo.put(\"msgPutTotalTodayNow\", String.valueOf(this.brokerController.getBrokerStats().getMsgPutTotalTodayNow()));\n\n        runtimeInfo.put(\"msgGetTotalYesterdayMorning\",\n            String.valueOf(this.brokerController.getBrokerStats().getMsgGetTotalYesterdayMorning()));\n        runtimeInfo.put(\"msgGetTotalTodayMorning\", String.valueOf(this.brokerController.getBrokerStats().getMsgGetTotalTodayMorning()));\n        runtimeInfo.put(\"msgGetTotalTodayNow\", String.valueOf(this.brokerController.getBrokerStats().getMsgGetTotalTodayNow()));\n\n        runtimeInfo.put(\"dispatchBehindBytes\", String.valueOf(this.brokerController.getMessageStore().dispatchBehindBytes()));\n        runtimeInfo.put(\"pageCacheLockTimeMills\", String.valueOf(this.brokerController.getMessageStore().lockTimeMills()));\n\n        runtimeInfo.put(\"earliestMessageTimeStamp\", String.valueOf(this.brokerController.getMessageStore().getEarliestMessageTime()));\n        runtimeInfo.put(\"startAcceptSendRequestTimeStamp\", String.valueOf(this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp()));\n\n        if (this.brokerController.getMessageStoreConfig().isTimerWheelEnable()) {\n            runtimeInfo.put(\"timerReadBehind\", String.valueOf(this.brokerController.getMessageStore().getTimerMessageStore().getDequeueBehind()));\n            runtimeInfo.put(\"timerOffsetBehind\", String.valueOf(this.brokerController.getMessageStore().getTimerMessageStore().getEnqueueBehindMessages()));\n            runtimeInfo.put(\"timerCongestNum\", String.valueOf(this.brokerController.getMessageStore().getTimerMessageStore().getAllCongestNum()));\n            runtimeInfo.put(\"timerEnqueueTps\", String.valueOf(this.brokerController.getMessageStore().getTimerMessageStore().getEnqueueTps()));\n            runtimeInfo.put(\"timerDequeueTps\", String.valueOf(this.brokerController.getMessageStore().getTimerMessageStore().getDequeueTps()));\n        } else {\n            runtimeInfo.put(\"timerReadBehind\", \"0\");\n            runtimeInfo.put(\"timerOffsetBehind\", \"0\");\n            runtimeInfo.put(\"timerCongestNum\", \"0\");\n            runtimeInfo.put(\"timerEnqueueTps\", \"0.0\");\n            runtimeInfo.put(\"timerDequeueTps\", \"0.0\");\n        }\n        MessageStore messageStore = this.brokerController.getMessageStore();\n        runtimeInfo.put(\"remainTransientStoreBufferNumbs\", String.valueOf(messageStore.remainTransientStoreBufferNumbs()));\n        if (this.brokerController.getMessageStore() instanceof DefaultMessageStore && ((DefaultMessageStore) this.brokerController.getMessageStore()).isTransientStorePoolEnable()) {\n            runtimeInfo.put(\"remainHowManyDataToCommit\", MixAll.humanReadableByteCount(messageStore.remainHowManyDataToCommit(), false));\n        }\n        runtimeInfo.put(\"remainHowManyDataToFlush\", MixAll.humanReadableByteCount(messageStore.remainHowManyDataToFlush(), false));\n\n        java.io.File commitLogDir = new java.io.File(this.brokerController.getMessageStoreConfig().getStorePathRootDir());\n        if (commitLogDir.exists()) {\n            runtimeInfo.put(\"commitLogDirCapacity\", String.format(\"Total : %s, Free : %s.\", MixAll.humanReadableByteCount(commitLogDir.getTotalSpace(), false), MixAll.humanReadableByteCount(commitLogDir.getFreeSpace(), false)));\n        }\n\n        runtimeInfo.put(\"sendThreadPoolQueueSize\", String.valueOf(this.brokerController.getSendThreadPoolQueue().size()));\n        runtimeInfo.put(\"sendThreadPoolQueueCapacity\",\n            String.valueOf(this.brokerController.getBrokerConfig().getSendThreadPoolQueueCapacity()));\n\n        runtimeInfo.put(\"pullThreadPoolQueueSize\", String.valueOf(this.brokerController.getPullThreadPoolQueue().size()));\n        runtimeInfo.put(\"pullThreadPoolQueueCapacity\",\n            String.valueOf(this.brokerController.getBrokerConfig().getPullThreadPoolQueueCapacity()));\n\n        runtimeInfo.put(\"litePullThreadPoolQueueSize\", String.valueOf(brokerController.getLitePullThreadPoolQueue().size()));\n        runtimeInfo.put(\"litePullThreadPoolQueueCapacity\",\n            String.valueOf(this.brokerController.getBrokerConfig().getLitePullThreadPoolQueueCapacity()));\n\n        runtimeInfo.put(\"queryThreadPoolQueueSize\", String.valueOf(this.brokerController.getQueryThreadPoolQueue().size()));\n        runtimeInfo.put(\"queryThreadPoolQueueCapacity\",\n            String.valueOf(this.brokerController.getBrokerConfig().getQueryThreadPoolQueueCapacity()));\n\n        runtimeInfo.put(\"ackThreadPoolQueueSize\", String.valueOf(this.brokerController.getAckThreadPoolQueue().size()));\n        runtimeInfo.put(\"ackThreadPoolQueueCapacity\",\n            String.valueOf(this.brokerController.getBrokerConfig().getAckThreadPoolQueueCapacity()));\n\n        runtimeInfo.put(\"sendThreadPoolQueueHeadWaitTimeMills\", String.valueOf(this.brokerController.headSlowTimeMills4SendThreadPoolQueue()));\n        runtimeInfo.put(\"pullThreadPoolQueueHeadWaitTimeMills\", String.valueOf(brokerController.headSlowTimeMills4PullThreadPoolQueue()));\n        runtimeInfo.put(\"queryThreadPoolQueueHeadWaitTimeMills\", String.valueOf(this.brokerController.headSlowTimeMills4QueryThreadPoolQueue()));\n        runtimeInfo.put(\"litePullThreadPoolQueueHeadWaitTimeMills\", String.valueOf(brokerController.headSlowTimeMills4LitePullThreadPoolQueue()));\n        runtimeInfo.put(\"ackThreadPoolQueueHeadWaitTimeMills\", String.valueOf(brokerController.headSlowTimeMills4AckThreadPoolQueue()));\n\n        runtimeInfo.put(\"EndTransactionQueueSize\", String.valueOf(this.brokerController.getEndTransactionThreadPoolQueue().size()));\n        runtimeInfo.put(\"EndTransactionThreadPoolQueueCapacity\",\n            String.valueOf(this.brokerController.getBrokerConfig().getEndTransactionPoolQueueCapacity()));\n\n        return runtimeInfo;\n    }\n\n    private RemotingCommand callConsumer(\n        final int requestCode,\n        final RemotingCommand request,\n        final String consumerGroup,\n        final String clientId) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        ClientChannelInfo clientChannelInfo = this.brokerController.getConsumerManager().findChannel(consumerGroup, clientId);\n\n        if (null == clientChannelInfo) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(String.format(\"The Consumer <%s> <%s> not online\", consumerGroup, clientId));\n            return response;\n        }\n\n        if (clientChannelInfo.getVersion() < MQVersion.Version.V3_1_8_SNAPSHOT.ordinal()) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(String.format(\"The Consumer <%s> Version <%s> too low to finish, please upgrade it to V3_1_8_SNAPSHOT\",\n                clientId,\n                MQVersion.getVersionDesc(clientChannelInfo.getVersion())));\n            return response;\n        }\n\n        try {\n            RemotingCommand newRequest = RemotingCommand.createRequestCommand(requestCode, null);\n            newRequest.setExtFields(request.getExtFields());\n            newRequest.setBody(request.getBody());\n\n            return this.brokerController.getBroker2Client().callClient(clientChannelInfo.getChannel(), newRequest);\n        } catch (RemotingTimeoutException e) {\n            response.setCode(ResponseCode.CONSUME_MSG_TIMEOUT);\n            response\n                .setRemark(String.format(\"consumer <%s> <%s> Timeout: %s\", consumerGroup, clientId, UtilAll.exceptionSimpleDesc(e)));\n            return response;\n        } catch (Exception e) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\n                String.format(\"invoke consumer <%s> <%s> Exception: %s\", consumerGroup, clientId, UtilAll.exceptionSimpleDesc(e)));\n            return response;\n        }\n    }\n\n    private RemotingCommand queryConsumeQueue(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        QueryConsumeQueueRequestHeader requestHeader =\n            (QueryConsumeQueueRequestHeader) request.decodeCommandCustomHeader(QueryConsumeQueueRequestHeader.class);\n\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        ConsumeQueueInterface consumeQueue = this.brokerController.getMessageStore().getConsumeQueue(requestHeader.getTopic(),\n            requestHeader.getQueueId());\n        if (consumeQueue == null) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(String.format(\"%d@%s is not exist!\", requestHeader.getQueueId(), requestHeader.getTopic()));\n            return response;\n        }\n        response.setCode(ResponseCode.SUCCESS);\n\n        QueryConsumeQueueResponseBody body = new QueryConsumeQueueResponseBody();\n        body.setMaxQueueIndex(consumeQueue.getMaxOffsetInQueue());\n        body.setMinQueueIndex(consumeQueue.getMinOffsetInQueue());\n\n        MessageFilter messageFilter = null;\n        if (requestHeader.getConsumerGroup() != null) {\n            SubscriptionData subscriptionData = this.brokerController.getConsumerManager().findSubscriptionData(\n                requestHeader.getConsumerGroup(), requestHeader.getTopic()\n            );\n            body.setSubscriptionData(subscriptionData);\n            if (subscriptionData == null) {\n                body.setFilterData(String.format(\"%s@%s is not online!\", requestHeader.getConsumerGroup(), requestHeader.getTopic()));\n            } else {\n                ConsumerFilterData filterData = this.brokerController.getConsumerFilterManager()\n                    .get(requestHeader.getTopic(), requestHeader.getConsumerGroup());\n                body.setFilterData(JSON.toJSONString(filterData, JSONWriter.Feature.PrettyFormat));\n\n                messageFilter = new ExpressionMessageFilter(subscriptionData, filterData,\n                    this.brokerController.getConsumerFilterManager());\n            }\n        }\n\n        ReferredIterator<CqUnit> result = consumeQueue.iterateFrom(requestHeader.getIndex());\n        if (result == null) {\n            response.setRemark(String.format(\"Index %d of %d@%s is not exist!\", requestHeader.getIndex(), requestHeader.getQueueId(), requestHeader.getTopic()));\n            return response;\n        }\n        try {\n            List<ConsumeQueueData> queues = new ArrayList<>();\n            while (result.hasNext()) {\n                CqUnit cqUnit = result.next();\n                if (cqUnit.getQueueOffset() - requestHeader.getIndex() >= requestHeader.getCount()) {\n                    break;\n                }\n\n                ConsumeQueueData one = new ConsumeQueueData();\n                one.setPhysicOffset(cqUnit.getPos());\n                one.setPhysicSize(cqUnit.getSize());\n                one.setTagsCode(cqUnit.getTagsCode());\n\n                if (cqUnit.getCqExtUnit() == null && cqUnit.isTagsCodeValid()) {\n                    queues.add(one);\n                    continue;\n                }\n\n                if (cqUnit.getCqExtUnit() != null) {\n                    ConsumeQueueExt.CqExtUnit cqExtUnit = cqUnit.getCqExtUnit();\n                    one.setExtendDataJson(JSON.toJSONString(cqExtUnit));\n                    if (cqExtUnit.getFilterBitMap() != null) {\n                        one.setBitMap(BitsArray.create(cqExtUnit.getFilterBitMap()).toString());\n                    }\n                    if (messageFilter != null) {\n                        one.setEval(messageFilter.isMatchedByConsumeQueue(cqExtUnit.getTagsCode(), cqExtUnit));\n                    }\n                } else {\n                    one.setMsg(\"Cq extend not exist!addr: \" + one.getTagsCode());\n                }\n\n                queues.add(one);\n            }\n            body.setQueueData(queues);\n        } finally {\n            result.release();\n        }\n        response.setBody(body.encode());\n        return response;\n    }\n\n    private RemotingCommand resumeCheckHalfMessage(ChannelHandlerContext ctx,\n        RemotingCommand request)\n        throws RemotingCommandException {\n        final ResumeCheckHalfMessageRequestHeader requestHeader = (ResumeCheckHalfMessageRequestHeader) request\n            .decodeCommandCustomHeader(ResumeCheckHalfMessageRequestHeader.class);\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        SelectMappedBufferResult selectMappedBufferResult = null;\n        try {\n            MessageId messageId = MessageDecoder.decodeMessageId(requestHeader.getMsgId());\n            selectMappedBufferResult = this.brokerController.getMessageStore()\n                .selectOneMessageByOffset(messageId.getOffset());\n            MessageExt msg = MessageDecoder.decode(selectMappedBufferResult.getByteBuffer(), true, false);\n            msg.putUserProperty(MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES, String.valueOf(0));\n            PutMessageResult putMessageResult = this.brokerController.getMessageStore()\n                .putMessage(toMessageExtBrokerInner(msg));\n            if (putMessageResult != null\n                && putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK) {\n                LOGGER.info(\n                    \"Put message back to RMQ_SYS_TRANS_HALF_TOPIC. real topic={}\",\n                    msg.getUserProperty(MessageConst.PROPERTY_REAL_TOPIC));\n                response.setCode(ResponseCode.SUCCESS);\n                response.setRemark(null);\n            } else {\n                LOGGER.error(\"Put message back to RMQ_SYS_TRANS_HALF_TOPIC failed.\");\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"Put message back to RMQ_SYS_TRANS_HALF_TOPIC failed.\");\n            }\n        } catch (Exception e) {\n            LOGGER.error(\"Exception was thrown when putting message back to RMQ_SYS_TRANS_HALF_TOPIC.\");\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"Exception was thrown when putting message back to RMQ_SYS_TRANS_HALF_TOPIC.\");\n        } finally {\n            if (selectMappedBufferResult != null) {\n                selectMappedBufferResult.release();\n            }\n        }\n        return response;\n    }\n\n    private MessageExtBrokerInner toMessageExtBrokerInner(MessageExt msgExt) {\n        MessageExtBrokerInner inner = new MessageExtBrokerInner();\n        if (brokerController.getMessageStoreConfig().isTransRocksDBEnable() && !brokerController.getMessageStoreConfig().isTransWriteOriginTransHalfEnable()) {\n            inner.setTopic(TransactionalMessageUtil.buildHalfTopicForRocksDB());\n        } else {\n            inner.setTopic(TransactionalMessageUtil.buildHalfTopic());\n        }\n        inner.setBody(msgExt.getBody());\n        inner.setFlag(msgExt.getFlag());\n        MessageAccessor.setProperties(inner, msgExt.getProperties());\n        inner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties()));\n        inner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(msgExt.getTags()));\n        inner.setQueueId(0);\n        inner.setSysFlag(msgExt.getSysFlag());\n        inner.setBornHost(msgExt.getBornHost());\n        inner.setBornTimestamp(msgExt.getBornTimestamp());\n        inner.setStoreHost(msgExt.getStoreHost());\n        inner.setReconsumeTimes(msgExt.getReconsumeTimes());\n        inner.setMsgId(msgExt.getMsgId());\n        inner.setWaitStoreMsgOK(false);\n        return inner;\n    }\n\n    private RemotingCommand getTopicConfig(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        GetTopicConfigRequestHeader requestHeader = (GetTopicConfigRequestHeader) request.decodeCommandCustomHeader(GetTopicConfigRequestHeader.class);\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());\n        if (topicConfig == null) {\n            LOGGER.error(\"No topic in this broker, client: {} topic: {}\", ctx.channel().remoteAddress(), requestHeader.getTopic());\n            //be care of the response code, should set \"not-exist\" explicitly\n            response.setCode(ResponseCode.TOPIC_NOT_EXIST);\n            response.setRemark(\"No topic in this broker. topic: \" + requestHeader.getTopic());\n            return response;\n        }\n        TopicQueueMappingDetail topicQueueMappingDetail = null;\n        if (Boolean.TRUE.equals(requestHeader.getLo())) {\n            topicQueueMappingDetail = this.brokerController.getTopicQueueMappingManager().getTopicQueueMapping(requestHeader.getTopic());\n        }\n        String content = JSONObject.toJSONString(new TopicConfigAndQueueMapping(topicConfig, topicQueueMappingDetail));\n        try {\n            response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET));\n        } catch (UnsupportedEncodingException e) {\n            LOGGER.error(\"UnsupportedEncodingException getTopicConfig: topic=\" + topicConfig.getTopicName(), e);\n\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"UnsupportedEncodingException \" + e.getMessage());\n            return response;\n        }\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n\n        return response;\n    }\n\n    private RemotingCommand notifyMinBrokerIdChange(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        NotifyMinBrokerIdChangeRequestHeader requestHeader = (NotifyMinBrokerIdChangeRequestHeader) request.decodeCommandCustomHeader(NotifyMinBrokerIdChangeRequestHeader.class);\n\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        LOGGER.warn(\"min broker id changed, prev {}, new {}\", this.brokerController.getMinBrokerIdInGroup(), requestHeader.getMinBrokerId());\n\n        this.brokerController.updateMinBroker(requestHeader.getMinBrokerId(), requestHeader.getMinBrokerAddr(),\n            requestHeader.getOfflineBrokerAddr(),\n            requestHeader.getHaBrokerAddr());\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n\n        return response;\n    }\n\n    private RemotingCommand updateBrokerHaInfo(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        RemotingCommand response = RemotingCommand.createResponseCommand(ExchangeHAInfoResponseHeader.class);\n\n        ExchangeHAInfoRequestHeader requestHeader = (ExchangeHAInfoRequestHeader) request.decodeCommandCustomHeader(ExchangeHAInfoRequestHeader.class);\n        if (requestHeader.getMasterHaAddress() != null) {\n            this.brokerController.getMessageStore().updateHaMasterAddress(requestHeader.getMasterHaAddress());\n            this.brokerController.getMessageStore().updateMasterAddress(requestHeader.getMasterAddress());\n            if (this.brokerController.getMessageStore().getMasterFlushedOffset() == 0\n                && this.brokerController.getMessageStoreConfig().isSyncMasterFlushOffsetWhenStartup()) {\n                LOGGER.info(\"Set master flush offset in slave to {}\", requestHeader.getMasterFlushOffset());\n                this.brokerController.getMessageStore().setMasterFlushedOffset(requestHeader.getMasterFlushOffset());\n            }\n        } else if (this.brokerController.getBrokerConfig().getBrokerId() == MixAll.MASTER_ID) {\n            final ExchangeHAInfoResponseHeader responseHeader = (ExchangeHAInfoResponseHeader) response.readCustomHeader();\n            responseHeader.setMasterHaAddress(this.brokerController.getHAServerAddr());\n            responseHeader.setMasterFlushOffset(this.brokerController.getMessageStore().getBrokerInitMaxOffset());\n            responseHeader.setMasterAddress(this.brokerController.getBrokerAddr());\n        }\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n\n        return response;\n    }\n\n    private RemotingCommand getBrokerHaStatus(ChannelHandlerContext ctx, RemotingCommand request) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        HARuntimeInfo runtimeInfo = this.brokerController.getMessageStore().getHARuntimeInfo();\n\n        if (runtimeInfo != null) {\n            byte[] body = runtimeInfo.encode();\n            response.setBody(body);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n        } else {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"Can not get HARuntimeInfo, may be duplicationEnable is true\");\n        }\n\n        return response;\n    }\n\n    private RemotingCommand getBrokerEpochCache(ChannelHandlerContext ctx, RemotingCommand request) {\n        final ReplicasManager replicasManager = this.brokerController.getReplicasManager();\n        assert replicasManager != null;\n        final BrokerConfig brokerConfig = this.brokerController.getBrokerConfig();\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        if (!brokerConfig.isEnableControllerMode()) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"this request only for controllerMode \");\n            return response;\n        }\n        final EpochEntryCache entryCache = new EpochEntryCache(brokerConfig.getBrokerClusterName(),\n            brokerConfig.getBrokerName(), brokerConfig.getBrokerId(), replicasManager.getEpochEntries(), this.brokerController.getMessageStore().getMaxPhyOffset());\n\n        response.setBody(entryCache.encode());\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand resetMasterFlushOffset(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        if (this.brokerController.getBrokerConfig().getBrokerId() != MixAll.MASTER_ID) {\n\n            ResetMasterFlushOffsetHeader requestHeader = (ResetMasterFlushOffsetHeader) request.decodeCommandCustomHeader(ResetMasterFlushOffsetHeader.class);\n\n            if (requestHeader.getMasterFlushOffset() != null) {\n                this.brokerController.getMessageStore().setMasterFlushedOffset(requestHeader.getMasterFlushOffset());\n            }\n        }\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand notifyBrokerRoleChanged(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        NotifyBrokerRoleChangedRequestHeader requestHeader = (NotifyBrokerRoleChangedRequestHeader) request.decodeCommandCustomHeader(NotifyBrokerRoleChangedRequestHeader.class);\n        SyncStateSet syncStateSetInfo = RemotingSerializable.decode(request.getBody(), SyncStateSet.class);\n\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        LOGGER.info(\"Receive notifyBrokerRoleChanged request, try to change brokerRole, request:{}\", requestHeader);\n\n        final ReplicasManager replicasManager = this.brokerController.getReplicasManager();\n        if (replicasManager != null) {\n            try {\n                replicasManager.changeBrokerRole(requestHeader.getMasterBrokerId(), requestHeader.getMasterAddress(), requestHeader.getMasterEpoch(), requestHeader.getSyncStateSetEpoch(), syncStateSetInfo.getSyncStateSet());\n            } catch (Exception e) {\n                throw new RemotingCommandException(e.getMessage());\n            }\n        }\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n\n        return response;\n    }\n\n    private RemotingCommand createUser(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        CreateUserRequestHeader requestHeader = request.decodeCommandCustomHeader(CreateUserRequestHeader.class);\n        if (StringUtils.isEmpty(requestHeader.getUsername())) {\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(\"The username is blank\");\n            return response;\n        }\n\n        UserInfo userInfo = RemotingSerializable.decode(request.getBody(), UserInfo.class);\n        userInfo.setUsername(requestHeader.getUsername());\n        User user = UserConverter.convertUser(userInfo);\n\n        if (user.getUserType() == UserType.SUPER && isNotSuperUserLogin(request)) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"The super user can only be create by super user\");\n            return response;\n        }\n\n        this.brokerController.getAuthenticationMetadataManager().createUser(user)\n            .thenAccept(nil -> response.setCode(ResponseCode.SUCCESS))\n            .exceptionally(ex -> {\n                LOGGER.error(\"create user {} error\", user.getUsername(), ex);\n                return handleAuthException(response, ex);\n            })\n            .join();\n\n        return response;\n    }\n\n    private RemotingCommand updateUser(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        UpdateUserRequestHeader requestHeader = request.decodeCommandCustomHeader(UpdateUserRequestHeader.class);\n        if (StringUtils.isEmpty(requestHeader.getUsername())) {\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(\"The username is blank\");\n            return response;\n        }\n\n        UserInfo userInfo = RemotingSerializable.decode(request.getBody(), UserInfo.class);\n        userInfo.setUsername(requestHeader.getUsername());\n        User user = UserConverter.convertUser(userInfo);\n\n        if (user.getUserType() == UserType.SUPER && isNotSuperUserLogin(request)) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"The super user can only be update by super user\");\n            return response;\n        }\n\n        this.brokerController.getAuthenticationMetadataManager().getUser(requestHeader.getUsername())\n            .thenCompose(old -> {\n                if (old == null) {\n                    throw new AuthenticationException(\"The user is not exist\");\n                }\n                if (old.getUserType() == UserType.SUPER && isNotSuperUserLogin(request)) {\n                    throw new AuthenticationException(\"The super user can only be update by super user\");\n                }\n                return this.brokerController.getAuthenticationMetadataManager().updateUser(user);\n            }).thenAccept(nil -> response.setCode(ResponseCode.SUCCESS))\n            .exceptionally(ex -> {\n                LOGGER.error(\"update user {} error\", requestHeader.getUsername(), ex);\n                return handleAuthException(response, ex);\n            })\n            .join();\n        return response;\n    }\n\n    private RemotingCommand deleteUser(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        DeleteUserRequestHeader requestHeader = request.decodeCommandCustomHeader(DeleteUserRequestHeader.class);\n\n        this.brokerController.getAuthenticationMetadataManager().getUser(requestHeader.getUsername())\n            .thenCompose(user -> {\n                if (user == null) {\n                    return CompletableFuture.completedFuture(null);\n                }\n                if (user.getUserType() == UserType.SUPER && isNotSuperUserLogin(request)) {\n                    throw new AuthenticationException(\"The super user can only be update by super user\");\n                }\n                return this.brokerController.getAuthenticationMetadataManager().deleteUser(requestHeader.getUsername());\n            }).thenAccept(nil -> response.setCode(ResponseCode.SUCCESS))\n            .exceptionally(ex -> {\n                LOGGER.error(\"delete user {} error\", requestHeader.getUsername(), ex);\n                return handleAuthException(response, ex);\n            })\n            .join();\n        return response;\n    }\n\n    private RemotingCommand getUser(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        GetUserRequestHeader requestHeader = request.decodeCommandCustomHeader(GetUserRequestHeader.class);\n\n        if (StringUtils.isBlank(requestHeader.getUsername())) {\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(\"The username is blank\");\n            return response;\n        }\n\n        this.brokerController.getAuthenticationMetadataManager().getUser(requestHeader.getUsername())\n            .thenAccept(user -> {\n                response.setCode(ResponseCode.SUCCESS);\n                if (user != null) {\n                    UserInfo userInfo = UserConverter.convertUser(user);\n                    response.setBody(JSON.toJSONString(userInfo).getBytes(StandardCharsets.UTF_8));\n                }\n            })\n            .exceptionally(ex -> {\n                LOGGER.error(\"get user {} error\", requestHeader.getUsername(), ex);\n                return handleAuthException(response, ex);\n            })\n            .join();\n\n        return response;\n    }\n\n    private RemotingCommand listUser(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        ListUsersRequestHeader requestHeader = request.decodeCommandCustomHeader(ListUsersRequestHeader.class);\n\n        this.brokerController.getAuthenticationMetadataManager().listUser(requestHeader.getFilter())\n            .thenAccept(users -> {\n                response.setCode(ResponseCode.SUCCESS);\n                if (CollectionUtils.isNotEmpty(users)) {\n                    List<UserInfo> userInfos = UserConverter.convertUsers(users);\n                    response.setBody(JSON.toJSONString(userInfos).getBytes(StandardCharsets.UTF_8));\n                }\n            })\n            .exceptionally(ex -> {\n                LOGGER.error(\"list user by {} error\", requestHeader.getFilter(), ex);\n                return handleAuthException(response, ex);\n            })\n            .join();\n\n        return response;\n    }\n\n    private RemotingCommand createAcl(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        CreateAclRequestHeader requestHeader = request.decodeCommandCustomHeader(CreateAclRequestHeader.class);\n        Subject subject = Subject.of(requestHeader.getSubject());\n\n        AclInfo aclInfo = RemotingSerializable.decode(request.getBody(), AclInfo.class);\n        if (aclInfo == null || CollectionUtils.isEmpty(aclInfo.getPolicies())) {\n            throw new AuthorizationException(\"The body of acl is null\");\n        }\n\n        Acl acl = AclConverter.convertAcl(aclInfo);\n        if (acl != null && acl.getSubject() == null) {\n            acl.setSubject(subject);\n        }\n\n        this.brokerController.getAuthorizationMetadataManager().createAcl(acl)\n            .thenAccept(nil -> response.setCode(ResponseCode.SUCCESS))\n            .exceptionally(ex -> {\n                LOGGER.error(\"create acl for {} error\", requestHeader.getSubject(), ex);\n                return handleAuthException(response, ex);\n            })\n            .join();\n        return response;\n    }\n\n    private RemotingCommand updateAcl(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        UpdateAclRequestHeader requestHeader = request.decodeCommandCustomHeader(UpdateAclRequestHeader.class);\n        Subject subject = Subject.of(requestHeader.getSubject());\n\n        AclInfo aclInfo = RemotingSerializable.decode(request.getBody(), AclInfo.class);\n        if (aclInfo == null || CollectionUtils.isEmpty(aclInfo.getPolicies())) {\n            throw new AuthorizationException(\"The body of acl is null\");\n        }\n\n        Acl acl = AclConverter.convertAcl(aclInfo);\n        if (acl != null && acl.getSubject() == null) {\n            acl.setSubject(subject);\n        }\n\n        this.brokerController.getAuthorizationMetadataManager().updateAcl(acl)\n            .thenAccept(nil -> response.setCode(ResponseCode.SUCCESS))\n            .exceptionally(ex -> {\n                LOGGER.error(\"update acl for {} error\", requestHeader.getSubject(), ex);\n                return handleAuthException(response, ex);\n            })\n            .join();\n\n        return response;\n    }\n\n    private RemotingCommand deleteAcl(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        DeleteAclRequestHeader requestHeader = request.decodeCommandCustomHeader(DeleteAclRequestHeader.class);\n\n        Subject subject = Subject.of(requestHeader.getSubject());\n\n        PolicyType policyType = PolicyType.getByName(requestHeader.getPolicyType());\n\n        Resource resource = Resource.of(requestHeader.getResource());\n\n        this.brokerController.getAuthorizationMetadataManager().deleteAcl(subject, policyType, resource)\n            .thenAccept(nil -> {\n                response.setCode(ResponseCode.SUCCESS);\n            })\n            .exceptionally(ex -> {\n                LOGGER.error(\"delete acl for {} error\", requestHeader.getSubject(), ex);\n                return handleAuthException(response, ex);\n            })\n            .join();\n\n        return response;\n    }\n\n    private RemotingCommand getAcl(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        GetAclRequestHeader requestHeader = request.decodeCommandCustomHeader(GetAclRequestHeader.class);\n\n        Subject subject = Subject.of(requestHeader.getSubject());\n\n        this.brokerController.getAuthorizationMetadataManager().getAcl(subject)\n            .thenAccept(acl -> {\n                response.setCode(ResponseCode.SUCCESS);\n                if (acl != null) {\n                    AclInfo aclInfo = AclConverter.convertAcl(acl);\n                    String body = JSON.toJSONString(aclInfo);\n                    response.setBody(body.getBytes(StandardCharsets.UTF_8));\n                }\n            })\n            .exceptionally(ex -> {\n                LOGGER.error(\"get acl for {} error\", requestHeader.getSubject(), ex);\n                return handleAuthException(response, ex);\n            })\n            .join();\n\n        return response;\n    }\n\n    private RemotingCommand listAcl(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        ListAclsRequestHeader requestHeader = request.decodeCommandCustomHeader(ListAclsRequestHeader.class);\n\n        this.brokerController.getAuthorizationMetadataManager()\n            .listAcl(requestHeader.getSubjectFilter(), requestHeader.getResourceFilter())\n            .thenAccept(acls -> {\n                response.setCode(ResponseCode.SUCCESS);\n                if (CollectionUtils.isNotEmpty(acls)) {\n                    List<AclInfo> aclInfos = AclConverter.convertAcls(acls);\n                    String body = JSON.toJSONString(aclInfos);\n                    response.setBody(body.getBytes(StandardCharsets.UTF_8));\n                }\n            })\n            .exceptionally(ex -> {\n                LOGGER.error(\"list acl error, subjectFilter:{}, resourceFilter:{}\", requestHeader.getSubjectFilter(), requestHeader.getResourceFilter(), ex);\n                return handleAuthException(response, ex);\n            })\n            .join();\n\n        return response;\n    }\n\n    private boolean isNotSuperUserLogin(RemotingCommand request) {\n        String accessKey = request.getExtFields().get(\"AccessKey\");\n        // if accessKey is null, it may be authentication is not enabled.\n        if (StringUtils.isEmpty(accessKey)) {\n            return false;\n        }\n        return !this.brokerController.getAuthenticationMetadataManager()\n            .isSuperUser(accessKey).join();\n    }\n\n    private Void handleAuthException(RemotingCommand response, Throwable ex) {\n        Throwable throwable = ExceptionUtils.getRealException(ex);\n        if (throwable instanceof AuthenticationException || throwable instanceof AuthorizationException) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            response.setRemark(throwable.getMessage());\n        } else {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"An system error occurred, please try again later.\");\n            LOGGER.error(\"An system error occurred when processing auth admin request.\", ex);\n        }\n        return null;\n    }\n\n    private boolean validateSlave(RemotingCommand response) {\n        if (this.brokerController.getMessageStoreConfig().getBrokerRole().equals(BrokerRole.SLAVE)) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"Can't modify topic or subscription group from slave broker, \" +\n                \"please execute it from master broker.\");\n            return true;\n        }\n        return false;\n    }\n\n    private boolean validateBlackListConfigExist(Properties properties) {\n        for (String blackConfig : configBlackList) {\n            if (properties.containsKey(blackConfig)) {\n                return true;\n            }\n        }\n        return false;\n    }\n\n    private CheckRocksdbCqWriteResult doCheckRocksdbCqWriteProgress(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        CheckRocksdbCqWriteProgressRequestHeader requestHeader = request.decodeCommandCustomHeader(CheckRocksdbCqWriteProgressRequestHeader.class);\n        MessageStore messageStore = brokerController.getMessageStore();\n        DefaultMessageStore defaultMessageStore;\n        if (messageStore instanceof AbstractPluginMessageStore) {\n            defaultMessageStore = (DefaultMessageStore) ((AbstractPluginMessageStore) messageStore).getNext();\n        } else {\n            defaultMessageStore = (DefaultMessageStore) messageStore;\n        }\n        ConsumeQueueStoreInterface consumeQueueStore = defaultMessageStore.getQueueStore();\n\n        if (!(consumeQueueStore instanceof CombineConsumeQueueStore)) {\n            CheckRocksdbCqWriteResult result = new CheckRocksdbCqWriteResult();\n            result.setCheckResult(\"It is not CombineConsumeQueueStore, no need check\");\n            result.setCheckStatus(CheckRocksdbCqWriteResult.CheckStatus.CHECK_OK.getValue());\n            return result;\n        }\n\n        return ((CombineConsumeQueueStore) consumeQueueStore).\n            doCheckCqWriteProgress(requestHeader.getTopic(), requestHeader.getCheckStoreTime(), StoreType.DEFAULT, StoreType.DEFAULT_ROCKSDB);\n    }\n\n    private RemotingCommand transferPopToFsStore(ChannelHandlerContext ctx, RemotingCommand request) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        try {\n            if (brokerController.getPopConsumerService() != null) {\n                brokerController.getPopConsumerService().transferToFsStore();\n            }\n            response.setCode(ResponseCode.SUCCESS);\n        } catch (Exception e) {\n            LOGGER.error(\"PopConsumerStore transfer from kvStore to fsStore finish [{}]\", request, e);\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(e.getMessage());\n        }\n        return response;\n    }\n\n    private synchronized RemotingCommand switchTimerEngine(ChannelHandlerContext ctx, RemotingCommand request) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        if (!this.brokerController.getMessageStoreConfig().isTimerWheelEnable()) {\n            LOGGER.info(\"switchTimerEngine error, broker timerWheelEnable is false\");\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(\"broker timerWheelEnable is false\");\n            return response;\n        }\n        if (null == request.getExtFields()) {\n            LOGGER.info(\"switchTimerEngine extFields is null\");\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(\"param error, extFields is null\");\n            return response;\n        }\n        String engineType = request.getExtFields().get(TIMER_ENGINE_TYPE);\n        if (StringUtils.isEmpty(engineType) || !MessageConst.TIMER_ENGINE_ROCKSDB_TIMELINE.equals(engineType) && !MessageConst.TIMER_ENGINE_FILE_TIME_WHEEL.equals(engineType)) {\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(\"param error\");\n            return response;\n        }\n        try {\n            Properties properties = new Properties();\n            boolean result = false;\n            if (MessageConst.TIMER_ENGINE_ROCKSDB_TIMELINE.equals(engineType)) {\n                if (this.brokerController.getTimerMessageRocksDBStore() == null) {\n                    response.setCode(ResponseCode.INVALID_PARAMETER);\n                    response.setRemark(\"timerRocksDBEnable must be configured true when broker start\");\n                    return response;\n                }\n                result = this.brokerController.getTimerMessageRocksDBStore().restart();\n                if (result) {\n                    properties.put(\"timerStopEnqueue\", Boolean.TRUE.toString());\n                    properties.put(\"timerRocksDBEnable\", Boolean.TRUE.toString());\n                    properties.put(\"timerRocksDBStopScan\", Boolean.FALSE.toString());\n                }\n            } else {\n                result = this.brokerController.getTimerMessageStore().restart();\n                if (result) {\n                    properties.put(\"timerRocksDBStopScan\", Boolean.TRUE.toString());\n                    properties.put(\"timerStopEnqueue\", Boolean.FALSE.toString());\n                }\n            }\n            if (result) {\n                this.brokerController.getConfiguration().update(properties);\n                response.setCode(ResponseCode.SUCCESS);\n                response.setRemark(\"switch timer engine success\");\n                LOGGER.info(\"switchTimerEngine success\");\n            } else {\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"switch timer engine error\");\n                LOGGER.info(\"switchTimerEngine error\");\n            }\n        } catch (Exception e) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"switch timer engine error\");\n            LOGGER.error(\"switchTimerEngine error : {}\", e.getMessage());\n        }\n        return response;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport com.alibaba.fastjson2.JSON;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.nio.charset.StandardCharsets;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.broker.pop.PopConsumerLockService;\nimport org.apache.rocketmq.broker.pop.orderly.ConsumerOrderInfoManager;\nimport org.apache.rocketmq.common.PopAckConstants;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.help.FAQUrl;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingAbstract;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.pop.AckMsg;\nimport org.apache.rocketmq.store.pop.PopCheckPoint;\n\npublic class ChangeInvisibleTimeProcessor implements NettyRequestProcessor {\n    private static final Logger POP_LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);\n    private final BrokerController brokerController;\n    private final String reviveTopic;\n\n    public ChangeInvisibleTimeProcessor(final BrokerController brokerController) {\n        this.brokerController = brokerController;\n        this.reviveTopic = PopAckConstants.buildClusterReviveTopic(this.brokerController.getBrokerConfig().getBrokerClusterName());\n    }\n\n    @Override\n    public RemotingCommand processRequest(final ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        return this.processRequest(ctx.channel(), request, true);\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n\n    private RemotingCommand processRequest(final Channel channel, RemotingCommand request,\n        boolean brokerAllowSuspend) throws RemotingCommandException {\n\n        CompletableFuture<RemotingCommand> responseFuture = processRequestAsync(channel, request, brokerAllowSuspend);\n\n        if (brokerController.getBrokerConfig().isAppendCkAsync() && brokerController.getBrokerConfig().isAppendAckAsync()) {\n            responseFuture.thenAccept(response -> doResponse(channel, request, response)).exceptionally(throwable -> {\n                RemotingCommand response = RemotingCommand.createResponseCommand(ChangeInvisibleTimeResponseHeader.class);\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setOpaque(request.getOpaque());\n                doResponse(channel, request, response);\n                POP_LOGGER.error(\"append checkpoint or ack origin failed\", throwable);\n                return null;\n            });\n        } else {\n            RemotingCommand response;\n            try {\n                response = responseFuture.get(3000, TimeUnit.MILLISECONDS);\n            } catch (Exception e) {\n                response = RemotingCommand.createResponseCommand(ChangeInvisibleTimeResponseHeader.class);\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setOpaque(request.getOpaque());\n                POP_LOGGER.error(\"append checkpoint or ack origin failed\", e);\n            }\n            return response;\n        }\n        return null;\n    }\n\n    public CompletableFuture<RemotingCommand> processRequestAsync(final Channel channel, RemotingCommand request,\n        boolean brokerAllowSuspend) throws RemotingCommandException {\n        final ChangeInvisibleTimeRequestHeader requestHeader = (ChangeInvisibleTimeRequestHeader) request.decodeCommandCustomHeader(ChangeInvisibleTimeRequestHeader.class);\n        RemotingCommand response = RemotingCommand.createResponseCommand(ChangeInvisibleTimeResponseHeader.class);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setOpaque(request.getOpaque());\n        final ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.readCustomHeader();\n        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());\n        if (null == topicConfig) {\n            POP_LOGGER.error(\"The topic {} not exist, consumer: {} \", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel));\n            response.setCode(ResponseCode.TOPIC_NOT_EXIST);\n            response.setRemark(String.format(\"topic[%s] not exist, apply first please! %s\", requestHeader.getTopic(), FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)));\n            return CompletableFuture.completedFuture(response);\n        }\n\n        if (requestHeader.getQueueId() >= topicConfig.getReadQueueNums() || requestHeader.getQueueId() < 0) {\n            String errorInfo = String.format(\"queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]\",\n                requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());\n            POP_LOGGER.warn(errorInfo);\n            response.setCode(ResponseCode.MESSAGE_ILLEGAL);\n            response.setRemark(errorInfo);\n            return CompletableFuture.completedFuture(response);\n        }\n\n        CompletableFuture<RemotingCommand> future = processChangeInvisibleTimeForLite(requestHeader, response, responseHeader);\n        if (future != null) {\n            return future;\n        }\n\n        long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());\n        long maxOffset;\n        try {\n            maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());\n        } catch (ConsumeQueueException e) {\n            throw new RemotingCommandException(\"Failed to get max consume offset\", e);\n        }\n        if (requestHeader.getOffset() < minOffset || requestHeader.getOffset() >= maxOffset) {\n            response.setCode(ResponseCode.NO_MESSAGE);\n            return CompletableFuture.completedFuture(response);\n        }\n\n        String[] extraInfo = ExtraInfoUtil.split(requestHeader.getExtraInfo());\n        if (brokerController.getBrokerConfig().isPopConsumerKVServiceEnable()) {\n            if (ExtraInfoUtil.isOrder(extraInfo)) {\n                return this.processChangeInvisibleTimeForOrderNew(\n                    requestHeader, extraInfo, response, responseHeader);\n            }\n            try {\n                long current = System.currentTimeMillis();\n                brokerController.getPopConsumerService().changeInvisibilityDuration(\n                    ExtraInfoUtil.getPopTime(extraInfo), ExtraInfoUtil.getInvisibleTime(extraInfo), current,\n                    requestHeader.getInvisibleTime(), requestHeader.getConsumerGroup(), requestHeader.getTopic(),\n                    requestHeader.getQueueId(), requestHeader.getOffset(), requestHeader.isSuspend());\n                responseHeader.setInvisibleTime(requestHeader.getInvisibleTime());\n                responseHeader.setPopTime(current);\n                responseHeader.setReviveQid(ExtraInfoUtil.getReviveQid(extraInfo));\n            } catch (Exception e) {\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n            }\n            return CompletableFuture.completedFuture(response);\n        }\n\n        if (ExtraInfoUtil.isOrder(extraInfo)) {\n            return CompletableFuture.completedFuture(\n                processChangeInvisibleTimeForOrder(requestHeader, extraInfo, response, responseHeader));\n        }\n\n        // add new ck\n        long now = System.currentTimeMillis();\n        CompletableFuture<Boolean> futureResult = appendCheckPointThenAckOrigin(requestHeader,\n            ExtraInfoUtil.getReviveQid(extraInfo), requestHeader.getQueueId(), requestHeader.getOffset(), now, extraInfo);\n\n        return futureResult.thenCompose(result -> {\n            if (result) {\n                responseHeader.setInvisibleTime(requestHeader.getInvisibleTime());\n                responseHeader.setPopTime(now);\n                responseHeader.setReviveQid(ExtraInfoUtil.getReviveQid(extraInfo));\n            } else {\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n            }\n            return CompletableFuture.completedFuture(response);\n        });\n    }\n\n    @SuppressWarnings({\"StatementWithEmptyBody\", \"DuplicatedCode\"})\n    public CompletableFuture<RemotingCommand> processChangeInvisibleTimeForOrderNew(\n        ChangeInvisibleTimeRequestHeader requestHeader, String[] extraInfo,\n        RemotingCommand response, ChangeInvisibleTimeResponseHeader responseHeader) {\n\n        String groupId = requestHeader.getConsumerGroup();\n        String topicId = requestHeader.getTopic();\n        Integer queueId = requestHeader.getQueueId();\n        long popTime = ExtraInfoUtil.getPopTime(extraInfo);\n\n        PopConsumerLockService consumerLockService =\n            this.brokerController.getPopConsumerService().getConsumerLockService();\n        ConsumerOffsetManager consumerOffsetManager = this.brokerController.getConsumerOffsetManager();\n        ConsumerOrderInfoManager consumerOrderInfoManager = brokerController.getConsumerOrderInfoManager();\n\n        long oldOffset = consumerOffsetManager.queryOffset(groupId, topicId, queueId);\n        if (requestHeader.getOffset() < oldOffset) {\n            return CompletableFuture.completedFuture(response);\n        }\n\n        while (!consumerLockService.tryLock(groupId, topicId)) {\n        }\n\n        try {\n            // double check\n            oldOffset = consumerOffsetManager.queryOffset(groupId, topicId, queueId);\n            if (requestHeader.getOffset() < oldOffset) {\n                return CompletableFuture.completedFuture(response);\n            }\n\n            long visibilityTimeout = System.currentTimeMillis() + requestHeader.getInvisibleTime();\n            consumerOrderInfoManager.updateNextVisibleTime(\n                topicId, groupId, queueId, requestHeader.getOffset(), popTime, visibilityTimeout);\n\n            responseHeader.setInvisibleTime(visibilityTimeout - popTime);\n            responseHeader.setPopTime(popTime);\n            responseHeader.setReviveQid(ExtraInfoUtil.getReviveQid(extraInfo));\n        } finally {\n            consumerLockService.unlock(groupId, topicId);\n        }\n\n        return CompletableFuture.completedFuture(response);\n    }\n\n    protected RemotingCommand processChangeInvisibleTimeForOrder(ChangeInvisibleTimeRequestHeader requestHeader,\n        String[] extraInfo, RemotingCommand response, ChangeInvisibleTimeResponseHeader responseHeader) {\n        long popTime = ExtraInfoUtil.getPopTime(extraInfo);\n        long oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(requestHeader.getConsumerGroup(),\n            requestHeader.getTopic(), requestHeader.getQueueId());\n        if (requestHeader.getOffset() < oldOffset) {\n            return response;\n        }\n        while (!this.brokerController.getPopMessageProcessor().getQueueLockManager().tryLock(requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getQueueId())) {\n        }\n        try {\n            oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(requestHeader.getConsumerGroup(),\n                requestHeader.getTopic(), requestHeader.getQueueId());\n            if (requestHeader.getOffset() < oldOffset) {\n                return response;\n            }\n\n            long nextVisibleTime = System.currentTimeMillis() + requestHeader.getInvisibleTime();\n            this.brokerController.getConsumerOrderInfoManager().updateNextVisibleTime(\n                requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getQueueId(), requestHeader.getOffset(), popTime, nextVisibleTime);\n\n            responseHeader.setInvisibleTime(nextVisibleTime - popTime);\n            responseHeader.setPopTime(popTime);\n            responseHeader.setReviveQid(ExtraInfoUtil.getReviveQid(extraInfo));\n        } finally {\n            this.brokerController.getPopMessageProcessor().getQueueLockManager().unLock(requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getQueueId());\n        }\n        return response;\n    }\n\n    private CompletableFuture<Boolean> ackOrigin(final ChangeInvisibleTimeRequestHeader requestHeader,\n        String[] extraInfo) {\n        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n        AckMsg ackMsg = new AckMsg();\n\n        ackMsg.setAckOffset(requestHeader.getOffset());\n        ackMsg.setStartOffset(ExtraInfoUtil.getCkQueueOffset(extraInfo));\n        ackMsg.setConsumerGroup(requestHeader.getConsumerGroup());\n        ackMsg.setTopic(requestHeader.getTopic());\n        ackMsg.setQueueId(requestHeader.getQueueId());\n        ackMsg.setPopTime(ExtraInfoUtil.getPopTime(extraInfo));\n        ackMsg.setBrokerName(ExtraInfoUtil.getBrokerName(extraInfo));\n\n        int rqId = ExtraInfoUtil.getReviveQid(extraInfo);\n\n        this.brokerController.getBrokerStatsManager().incBrokerAckNums(1);\n        this.brokerController.getBrokerStatsManager().incGroupAckNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(), 1);\n\n        if (brokerController.getPopMessageProcessor().getPopBufferMergeService().addAk(rqId, ackMsg)) {\n            return CompletableFuture.completedFuture(true);\n        }\n\n        msgInner.setTopic(reviveTopic);\n        msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(StandardCharsets.UTF_8));\n        msgInner.setQueueId(rqId);\n        msgInner.setTags(PopAckConstants.ACK_TAG);\n        msgInner.setBornTimestamp(System.currentTimeMillis());\n        msgInner.setBornHost(this.brokerController.getStoreHost());\n        msgInner.setStoreHost(this.brokerController.getStoreHost());\n        msgInner.setDeliverTimeMs(ExtraInfoUtil.getPopTime(extraInfo) + ExtraInfoUtil.getInvisibleTime(extraInfo));\n        msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genAckUniqueId(ackMsg));\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));\n        return this.brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenCompose(putMessageResult -> {\n            if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK\n                && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT\n                && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT\n                && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {\n                POP_LOGGER.error(\"change Invisible, put ack msg fail: {}, {}\", ackMsg, putMessageResult);\n            }\n            brokerController.getBrokerMetricsManager().getPopMetricsManager().incPopReviveAckPutCount(ackMsg, putMessageResult.getPutMessageStatus());\n            return CompletableFuture.completedFuture(true);\n        }).exceptionally(e -> {\n            POP_LOGGER.error(\"change Invisible, put ack msg error: {}, {}\", requestHeader.getExtraInfo(), e.getMessage());\n            return false;\n        });\n    }\n\n    private CompletableFuture<Boolean> appendCheckPointThenAckOrigin(\n        final ChangeInvisibleTimeRequestHeader requestHeader,\n        int reviveQid,\n        int queueId, long offset, long popTime, String[] extraInfo) {\n        // add check point msg to revive log\n        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n        msgInner.setTopic(reviveTopic);\n        PopCheckPoint ck = new PopCheckPoint();\n        ck.setBitMap(0);\n        ck.setNum((byte) 1);\n        ck.setPopTime(popTime);\n        ck.setInvisibleTime(requestHeader.getInvisibleTime());\n        ck.setStartOffset(offset);\n        ck.setCId(requestHeader.getConsumerGroup());\n        ck.setTopic(requestHeader.getTopic());\n        ck.setQueueId(queueId);\n        ck.addDiff(0);\n        ck.setBrokerName(ExtraInfoUtil.getBrokerName(extraInfo));\n        ck.setSuspend(requestHeader.isSuspend());\n\n        msgInner.setBody(JSON.toJSONString(ck).getBytes(StandardCharsets.UTF_8));\n        msgInner.setQueueId(reviveQid);\n        msgInner.setTags(PopAckConstants.CK_TAG);\n        msgInner.setBornTimestamp(System.currentTimeMillis());\n        msgInner.setBornHost(this.brokerController.getStoreHost());\n        msgInner.setStoreHost(this.brokerController.getStoreHost());\n        msgInner.setDeliverTimeMs(ck.getReviveTime() - PopAckConstants.ackTimeInterval);\n        msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genCkUniqueId(ck));\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));\n        return this.brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenCompose(putMessageResult -> {\n            if (brokerController.getBrokerConfig().isEnablePopLog()) {\n                POP_LOGGER.info(\"change Invisible, appendCheckPoint, topic {}, queueId {},reviveId {}, cid {}, startOffset {}, rt {}, result {}\", requestHeader.getTopic(), queueId, reviveQid, requestHeader.getConsumerGroup(), offset,\n                    ck.getReviveTime(), putMessageResult);\n            }\n\n            if (putMessageResult != null) {\n                brokerController.getBrokerMetricsManager().getPopMetricsManager().incPopReviveCkPutCount(ck, putMessageResult.getPutMessageStatus());\n                if (putMessageResult.isOk()) {\n                    this.brokerController.getBrokerStatsManager().incBrokerCkNums(1);\n                    this.brokerController.getBrokerStatsManager().incGroupCkNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(), 1);\n                }\n            }\n            if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK\n                && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT\n                && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT\n                && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {\n                POP_LOGGER.error(\"change invisible, put new ck error: {}\", putMessageResult);\n                return CompletableFuture.completedFuture(false);\n            } else {\n                return ackOrigin(requestHeader, extraInfo);\n            }\n        }).exceptionally(throwable -> {\n            POP_LOGGER.error(\"change invisible, put new ck error\", throwable);\n            return null;\n        });\n    }\n\n    protected CompletableFuture<RemotingCommand> processChangeInvisibleTimeForLite(\n        ChangeInvisibleTimeRequestHeader requestHeader,\n        RemotingCommand response, ChangeInvisibleTimeResponseHeader responseHeader) {\n        if (StringUtils.isBlank(requestHeader.getLiteTopic())) {\n            return null;\n        }\n        String lmqName = LiteUtil.toLmqName(requestHeader.getTopic(), requestHeader.getLiteTopic());\n        long maxOffset = this.brokerController.getLiteLifecycleManager().getMaxOffsetInQueue(lmqName);\n        if (requestHeader.getOffset() > maxOffset) {\n            POP_LOGGER.warn(\"process lite offset illegal, {}, {}, {}\", lmqName, requestHeader.getOffset(), maxOffset);\n            response.setCode(ResponseCode.NO_MESSAGE);\n            return CompletableFuture.completedFuture(response);\n        }\n\n        String group = requestHeader.getConsumerGroup();\n        String[] extraInfo = ExtraInfoUtil.split(requestHeader.getExtraInfo());\n        long popTime = ExtraInfoUtil.getPopTime(extraInfo);\n\n        ConsumerOffsetManager consumerOffsetManager = this.brokerController.getConsumerOffsetManager();\n        ConsumerOrderInfoManager consumerOrderInfoManager =\n            brokerController.getPopLiteMessageProcessor().getConsumerOrderInfoManager();\n        PopConsumerLockService consumerLockService = this.brokerController.getPopLiteMessageProcessor().getLockService();\n\n        long oldOffset = consumerOffsetManager.queryOffset(group, lmqName, 0);\n        if (requestHeader.getOffset() < oldOffset) {\n            return CompletableFuture.completedFuture(response);\n        }\n\n        while (!consumerLockService.tryLock(group, lmqName)) {\n        }\n\n        try {\n            oldOffset = consumerOffsetManager.queryOffset(group, lmqName, 0);\n            if (requestHeader.getOffset() < oldOffset) {\n                return CompletableFuture.completedFuture(response);\n            }\n            long visibilityTimeout = System.currentTimeMillis() + requestHeader.getInvisibleTime();\n            consumerOrderInfoManager.updateNextVisibleTime(\n                lmqName, group, 0, requestHeader.getOffset(), popTime, visibilityTimeout);\n\n            responseHeader.setInvisibleTime(visibilityTimeout - popTime);\n            responseHeader.setPopTime(popTime);\n            responseHeader.setReviveQid(ExtraInfoUtil.getReviveQid(extraInfo));\n        } finally {\n            consumerLockService.unlock(group, lmqName);\n        }\n        return CompletableFuture.completedFuture(response);\n    }\n\n    protected void doResponse(Channel channel, RemotingCommand request,\n        final RemotingCommand response) {\n        NettyRemotingAbstract.writeResponse(channel, request, response, null, brokerController.getBrokerMetricsManager().getRemotingMetricsManager());\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/ClientManageProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.common.sysflag.TopicSysFlag;\nimport org.apache.rocketmq.filter.FilterFactory;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.CheckClientRequestBody;\nimport org.apache.rocketmq.remoting.protocol.header.UnregisterClientRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UnregisterClientResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumerData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ProducerData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\n\npublic class ClientManageProcessor implements NettyRequestProcessor {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private final BrokerController brokerController;\n    private final ConcurrentMap<String /* ConsumerGroup */, Integer /* HeartbeatFingerprint */> consumerGroupHeartbeatTable = new ConcurrentHashMap<>();\n\n    public ClientManageProcessor(final BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    @Override\n    public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request)\n        throws RemotingCommandException {\n        switch (request.getCode()) {\n            case RequestCode.HEART_BEAT:\n                return this.heartBeat(ctx, request);\n            case RequestCode.UNREGISTER_CLIENT:\n                return this.unregisterClient(ctx, request);\n            case RequestCode.CHECK_CLIENT_CONFIG:\n                return this.checkClientConfig(ctx, request);\n            default:\n                break;\n        }\n        return null;\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n\n    public RemotingCommand heartBeat(ChannelHandlerContext ctx, RemotingCommand request) {\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        HeartbeatData heartbeatData = HeartbeatData.decode(request.getBody(), HeartbeatData.class);\n        ClientChannelInfo clientChannelInfo = new ClientChannelInfo(\n            ctx.channel(),\n            heartbeatData.getClientID(),\n            request.getLanguage(),\n            request.getVersion()\n        );\n        int heartbeatFingerprint = heartbeatData.getHeartbeatFingerprint();\n        if (heartbeatFingerprint != 0) {\n            return heartBeatV2(ctx, heartbeatData, clientChannelInfo, response);\n        }\n        for (ConsumerData consumerData : heartbeatData.getConsumerDataSet()) {\n            //Reject the PullConsumer\n            if (brokerController.getBrokerConfig().isRejectPullConsumerEnable()) {\n                if (ConsumeType.CONSUME_ACTIVELY == consumerData.getConsumeType()) {\n                    continue;\n                }\n            }\n            consumerGroupHeartbeatTable.put(consumerData.getGroupName(), heartbeatFingerprint);\n            boolean hasOrderTopicSub = false;\n\n            for (final SubscriptionData subscriptionData : consumerData.getSubscriptionDataSet()) {\n                if (this.brokerController.getTopicConfigManager().isOrderTopic(subscriptionData.getTopic())) {\n                    hasOrderTopicSub = true;\n                    break;\n                }\n            }\n\n            SubscriptionGroupConfig subscriptionGroupConfig = this.brokerController.getSubscriptionGroupManager()\n                .findSubscriptionGroupConfig(consumerData.getGroupName());\n            boolean isNotifyConsumerIdsChangedEnable = true;\n\n            if (null == subscriptionGroupConfig) {\n                continue;\n            }\n\n            isNotifyConsumerIdsChangedEnable = subscriptionGroupConfig.isNotifyConsumerIdsChangedEnable();\n            int topicSysFlag = 0;\n            if (consumerData.isUnitMode()) {\n                topicSysFlag = TopicSysFlag.buildSysFlag(false, true);\n            }\n            String newTopic = MixAll.getRetryTopic(consumerData.getGroupName());\n            this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod(newTopic, subscriptionGroupConfig.getRetryQueueNums(),\n                PermName.PERM_WRITE | PermName.PERM_READ, hasOrderTopicSub, topicSysFlag);\n\n            boolean changed = this.brokerController.getConsumerManager().registerConsumer(\n                consumerData.getGroupName(),\n                clientChannelInfo,\n                consumerData.getConsumeType(),\n                consumerData.getMessageModel(),\n                consumerData.getConsumeFromWhere(),\n                consumerData.getSubscriptionDataSet(),\n                isNotifyConsumerIdsChangedEnable\n            );\n            if (changed) {\n                LOGGER.info(\"ClientManageProcessor: registerConsumer info changed, SDK address={}, consumerData={}\",\n                    RemotingHelper.parseChannelRemoteAddr(ctx.channel()), consumerData.toString());\n            }\n\n        }\n\n        for (ProducerData data : heartbeatData.getProducerDataSet()) {\n            this.brokerController.getProducerManager().registerProducer(data.getGroupName(),\n                clientChannelInfo);\n        }\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        response.addExtField(MixAll.IS_SUPPORT_HEART_BEAT_V2, Boolean.TRUE.toString());\n        response.addExtField(MixAll.IS_SUB_CHANGE, Boolean.TRUE.toString());\n        return response;\n    }\n\n    private RemotingCommand heartBeatV2(ChannelHandlerContext ctx, HeartbeatData heartbeatData, ClientChannelInfo clientChannelInfo, RemotingCommand response) {\n        boolean isSubChange = false;\n        for (ConsumerData consumerData : heartbeatData.getConsumerDataSet()) {\n            //Reject the PullConsumer\n            if (brokerController.getBrokerConfig().isRejectPullConsumerEnable()) {\n                if (ConsumeType.CONSUME_ACTIVELY == consumerData.getConsumeType()) {\n                    continue;\n                }\n            }\n            if (null != consumerGroupHeartbeatTable.get(consumerData.getGroupName()) && consumerGroupHeartbeatTable.get(consumerData.getGroupName()) != heartbeatData.getHeartbeatFingerprint()) {\n                isSubChange = true;\n            }\n            consumerGroupHeartbeatTable.put(consumerData.getGroupName(), heartbeatData.getHeartbeatFingerprint());\n            boolean hasOrderTopicSub = false;\n\n            for (final SubscriptionData subscriptionData : consumerData.getSubscriptionDataSet()) {\n                if (this.brokerController.getTopicConfigManager().isOrderTopic(subscriptionData.getTopic())) {\n                    hasOrderTopicSub = true;\n                    break;\n                }\n            }\n            SubscriptionGroupConfig subscriptionGroupConfig = this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(consumerData.getGroupName());\n            boolean isNotifyConsumerIdsChangedEnable = true;\n            if (null == subscriptionGroupConfig) {\n                continue;\n            }\n            isNotifyConsumerIdsChangedEnable = subscriptionGroupConfig.isNotifyConsumerIdsChangedEnable();\n            int topicSysFlag = 0;\n            if (consumerData.isUnitMode()) {\n                topicSysFlag = TopicSysFlag.buildSysFlag(false, true);\n            }\n            String newTopic = MixAll.getRetryTopic(consumerData.getGroupName());\n            this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod(newTopic, subscriptionGroupConfig.getRetryQueueNums(), PermName.PERM_WRITE | PermName.PERM_READ, hasOrderTopicSub, topicSysFlag);\n            boolean changed = false;\n            if (heartbeatData.isWithoutSub()) {\n                changed = this.brokerController.getConsumerManager().registerConsumerWithoutSub(consumerData.getGroupName(), clientChannelInfo, consumerData.getConsumeType(), consumerData.getMessageModel(), consumerData.getConsumeFromWhere(), isNotifyConsumerIdsChangedEnable);\n            } else {\n                changed = this.brokerController.getConsumerManager().registerConsumer(consumerData.getGroupName(), clientChannelInfo, consumerData.getConsumeType(), consumerData.getMessageModel(), consumerData.getConsumeFromWhere(), consumerData.getSubscriptionDataSet(), isNotifyConsumerIdsChangedEnable);\n            }\n            if (changed) {\n                LOGGER.info(\"heartBeatV2 ClientManageProcessor: registerConsumer info changed, SDK address={}, consumerData={}\",\n                        RemotingHelper.parseChannelRemoteAddr(ctx.channel()), consumerData.toString());\n            }\n\n        }\n        for (ProducerData data : heartbeatData.getProducerDataSet()) {\n            this.brokerController.getProducerManager().registerProducer(data.getGroupName(), clientChannelInfo);\n        }\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        response.addExtField(MixAll.IS_SUPPORT_HEART_BEAT_V2, Boolean.TRUE.toString());\n        response.addExtField(MixAll.IS_SUB_CHANGE, Boolean.valueOf(isSubChange).toString());\n        return response;\n    }\n\n    public RemotingCommand unregisterClient(ChannelHandlerContext ctx, RemotingCommand request)\n        throws RemotingCommandException {\n        final RemotingCommand response =\n            RemotingCommand.createResponseCommand(UnregisterClientResponseHeader.class);\n        final UnregisterClientRequestHeader requestHeader =\n            (UnregisterClientRequestHeader) request\n                .decodeCommandCustomHeader(UnregisterClientRequestHeader.class);\n\n        ClientChannelInfo clientChannelInfo = new ClientChannelInfo(\n            ctx.channel(),\n            requestHeader.getClientID(),\n            request.getLanguage(),\n            request.getVersion());\n        {\n            final String group = requestHeader.getProducerGroup();\n            if (group != null) {\n                this.brokerController.getProducerManager().unregisterProducer(group, clientChannelInfo);\n            }\n        }\n\n        {\n            final String group = requestHeader.getConsumerGroup();\n            if (group != null) {\n                SubscriptionGroupConfig subscriptionGroupConfig =\n                    this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(group);\n                boolean isNotifyConsumerIdsChangedEnable = true;\n                if (null != subscriptionGroupConfig) {\n                    isNotifyConsumerIdsChangedEnable = subscriptionGroupConfig.isNotifyConsumerIdsChangedEnable();\n                }\n                this.brokerController.getConsumerManager().unregisterConsumer(group, clientChannelInfo, isNotifyConsumerIdsChangedEnable);\n            }\n        }\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    public RemotingCommand checkClientConfig(ChannelHandlerContext ctx, RemotingCommand request)\n        throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        CheckClientRequestBody requestBody = CheckClientRequestBody.decode(request.getBody(),\n            CheckClientRequestBody.class);\n\n        if (requestBody != null && requestBody.getSubscriptionData() != null) {\n            SubscriptionData subscriptionData = requestBody.getSubscriptionData();\n\n            if (ExpressionType.isTagType(subscriptionData.getExpressionType())) {\n                response.setCode(ResponseCode.SUCCESS);\n                response.setRemark(null);\n                return response;\n            }\n\n            if (!this.brokerController.getBrokerConfig().isEnablePropertyFilter()) {\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"The broker does not support consumer to filter message by \" + subscriptionData.getExpressionType());\n                return response;\n            }\n\n            try {\n                FilterFactory.INSTANCE.get(subscriptionData.getExpressionType()).compile(subscriptionData.getSubString());\n            } catch (Exception e) {\n                LOGGER.warn(\"Client {}@{} filter message, but failed to compile expression! sub={}, error={}\",\n                    requestBody.getClientId(), requestBody.getGroup(), requestBody.getSubscriptionData(), e.getMessage());\n                response.setCode(ResponseCode.SUBSCRIPTION_PARSE_FAILED);\n                response.setRemark(e.getMessage());\n                return response;\n            }\n        }\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.List;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.client.ConsumerGroupInfo;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupResponseBody;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.statictopic.LogicQueueMappingItem;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingContext;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingDetail;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingUtils;\nimport org.apache.rocketmq.remoting.rpc.RpcClientUtils;\nimport org.apache.rocketmq.remoting.rpc.RpcRequest;\nimport org.apache.rocketmq.remoting.rpc.RpcResponse;\n\nimport static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse;\n\npublic class ConsumerManageProcessor implements NettyRequestProcessor {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private final BrokerController brokerController;\n\n    public ConsumerManageProcessor(final BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    @Override\n    public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request)\n        throws RemotingCommandException {\n        switch (request.getCode()) {\n            case RequestCode.GET_CONSUMER_LIST_BY_GROUP:\n                return this.getConsumerListByGroup(ctx, request);\n            case RequestCode.UPDATE_CONSUMER_OFFSET:\n                return this.updateConsumerOffset(ctx, request);\n            case RequestCode.QUERY_CONSUMER_OFFSET:\n                return this.queryConsumerOffset(ctx, request);\n            default:\n                break;\n        }\n        return null;\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n\n    public RemotingCommand getConsumerListByGroup(ChannelHandlerContext ctx, RemotingCommand request)\n        throws RemotingCommandException {\n        final RemotingCommand response =\n            RemotingCommand.createResponseCommand(GetConsumerListByGroupResponseHeader.class);\n        final GetConsumerListByGroupRequestHeader requestHeader =\n            (GetConsumerListByGroupRequestHeader) request\n                .decodeCommandCustomHeader(GetConsumerListByGroupRequestHeader.class);\n\n        ConsumerGroupInfo consumerGroupInfo =\n            this.brokerController.getConsumerManager().getConsumerGroupInfo(\n                requestHeader.getConsumerGroup());\n        if (consumerGroupInfo != null) {\n            List<String> clientIds = consumerGroupInfo.getAllClientId();\n            if (!clientIds.isEmpty()) {\n                GetConsumerListByGroupResponseBody body = new GetConsumerListByGroupResponseBody();\n                body.setConsumerIdList(clientIds);\n                response.setBody(body.encode());\n                response.setCode(ResponseCode.SUCCESS);\n                response.setRemark(null);\n                return response;\n            } else {\n                LOGGER.warn(\"getAllClientId failed, {} {}\", requestHeader.getConsumerGroup(),\n                    RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n            }\n        } else {\n            LOGGER.warn(\"getConsumerGroupInfo failed, {} {}\", requestHeader.getConsumerGroup(),\n                RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n        }\n\n        response.setCode(ResponseCode.SYSTEM_ERROR);\n        response.setRemark(\"no consumer for this group, \" + requestHeader.getConsumerGroup());\n        return response;\n    }\n\n    public RemotingCommand rewriteRequestForStaticTopic(final UpdateConsumerOffsetRequestHeader requestHeader,\n        final TopicQueueMappingContext mappingContext) {\n        try {\n            if (mappingContext.getMappingDetail() == null) {\n                return null;\n            }\n            TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();\n            if (!mappingContext.isLeader()) {\n                return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format(\"%s-%d does not exit in request process of current broker %s\", requestHeader.getTopic(), requestHeader.getQueueId(), mappingDetail.getBname()));\n            }\n            Long globalOffset = requestHeader.getCommitOffset();\n            LogicQueueMappingItem mappingItem = TopicQueueMappingUtils.findLogicQueueMappingItem(mappingContext.getMappingItemList(), globalOffset, true);\n            requestHeader.setQueueId(mappingItem.getQueueId());\n            requestHeader.setLo(false);\n            requestHeader.setBrokerName(mappingItem.getBname());\n            requestHeader.setCommitOffset(mappingItem.computePhysicalQueueOffset(globalOffset));\n            //leader, let it go, do not need to rewrite the response\n            if (mappingDetail.getBname().equals(mappingItem.getBname())) {\n                return null;\n            }\n            RpcRequest rpcRequest = new RpcRequest(RequestCode.UPDATE_CONSUMER_OFFSET, requestHeader, null);\n            RpcResponse rpcResponse = this.brokerController.getBrokerOuterAPI().getRpcClient().invoke(rpcRequest, this.brokerController.getBrokerConfig().getForwardTimeout()).get();\n            if (rpcResponse.getException() != null) {\n                throw rpcResponse.getException();\n            }\n            return RpcClientUtils.createCommandForRpcResponse(rpcResponse);\n        } catch (Throwable t) {\n            return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());\n        }\n    }\n\n    private RemotingCommand updateConsumerOffset(ChannelHandlerContext ctx, RemotingCommand request)\n        throws RemotingCommandException {\n\n        final RemotingCommand response =\n            RemotingCommand.createResponseCommand(UpdateConsumerOffsetResponseHeader.class);\n\n        final UpdateConsumerOffsetRequestHeader requestHeader =\n            (UpdateConsumerOffsetRequestHeader)\n                request.decodeCommandCustomHeader(UpdateConsumerOffsetRequestHeader.class);\n\n        TopicQueueMappingContext mappingContext =\n            this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader);\n\n        RemotingCommand rewriteResult = rewriteRequestForStaticTopic(requestHeader, mappingContext);\n        if (rewriteResult != null) {\n            return rewriteResult;\n        }\n\n        String topic = requestHeader.getTopic();\n        String group = requestHeader.getConsumerGroup();\n        Integer queueId = requestHeader.getQueueId();\n        Long offset = requestHeader.getCommitOffset();\n\n        if (!this.brokerController.getSubscriptionGroupManager().containsSubscriptionGroup(group)) {\n            response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);\n            response.setRemark(\"Group \" + group + \" not exist!\");\n            return response;\n        }\n\n        if (!this.brokerController.getTopicConfigManager().containsTopic(requestHeader.getTopic())) {\n            response.setCode(ResponseCode.TOPIC_NOT_EXIST);\n            response.setRemark(\"Topic \" + topic + \" not exist!\");\n            return response;\n        }\n\n        if (queueId == null) {\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(\"QueueId is null, topic is \" + topic);\n            return response;\n        }\n\n        if (offset == null) {\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(\"Offset is null, topic is \" + topic);\n            return response;\n        }\n\n        ConsumerOffsetManager consumerOffsetManager = brokerController.getConsumerOffsetManager();\n        if (this.brokerController.getBrokerConfig().isUseServerSideResetOffset()) {\n            // Note, ignoring this update offset request\n            if (consumerOffsetManager.hasOffsetReset(topic, group, queueId)) {\n                response.setCode(ResponseCode.SUCCESS);\n                response.setRemark(\"Offset has been previously reset\");\n                LOGGER.info(\"Update consumer offset is rejected because of previous offset-reset. Group={}, \" +\n                    \"Topic={}, QueueId={}, Offset={}\", group, topic, queueId, offset);\n                return response;\n            }\n        }\n\n        this.brokerController.getConsumerOffsetManager().commitOffset(\n            RemotingHelper.parseChannelRemoteAddr(ctx.channel()), group, topic, queueId, offset);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    public RemotingCommand rewriteRequestForStaticTopic(QueryConsumerOffsetRequestHeader requestHeader,\n        TopicQueueMappingContext mappingContext) {\n        try {\n            if (mappingContext.getMappingDetail() == null) {\n                return null;\n            }\n            TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();\n            if (!mappingContext.isLeader()) {\n                return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format(\"%s-%d does not exit in request process of current broker %s\", requestHeader.getTopic(), requestHeader.getQueueId(), mappingDetail.getBname()));\n            }\n            List<LogicQueueMappingItem> mappingItemList = mappingContext.getMappingItemList();\n            if (mappingItemList.size() == 1\n                && mappingItemList.get(0).getLogicOffset() == 0) {\n                //as physical, just let it go\n                mappingContext.setCurrentItem(mappingItemList.get(0));\n                requestHeader.setQueueId(mappingContext.getLeaderItem().getQueueId());\n                return null;\n            }\n            //double read check\n            List<LogicQueueMappingItem> itemList = mappingContext.getMappingItemList();\n            //by default, it is -1\n            long offset = -1;\n            //double read, first from leader, then from second leader\n            for (int i = itemList.size() - 1; i >= 0; i--) {\n                LogicQueueMappingItem mappingItem = itemList.get(i);\n                mappingContext.setCurrentItem(mappingItem);\n                if (mappingItem.getBname().equals(mappingDetail.getBname())) {\n                    offset = this.brokerController.getConsumerOffsetManager().queryOffset(requestHeader.getConsumerGroup(), requestHeader.getTopic(), mappingItem.getQueueId());\n                    if (offset >= 0) {\n                        break;\n                    } else {\n                        //not found\n                        continue;\n                    }\n                } else {\n                    //maybe we need to reconstruct an object\n                    requestHeader.setBrokerName(mappingItem.getBname());\n                    requestHeader.setQueueId(mappingItem.getQueueId());\n                    requestHeader.setLo(false);\n                    requestHeader.setSetZeroIfNotFound(false);\n                    RpcRequest rpcRequest = new RpcRequest(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader, null);\n                    RpcResponse rpcResponse = this.brokerController.getBrokerOuterAPI().getRpcClient().invoke(rpcRequest, this.brokerController.getBrokerConfig().getForwardTimeout()).get();\n                    if (rpcResponse.getException() != null) {\n                        throw rpcResponse.getException();\n                    }\n                    if (rpcResponse.getCode() == ResponseCode.SUCCESS) {\n                        offset = ((QueryConsumerOffsetResponseHeader) rpcResponse.getHeader()).getOffset();\n                        break;\n                    } else if (rpcResponse.getCode() == ResponseCode.QUERY_NOT_FOUND) {\n                        continue;\n                    } else {\n                        //this should not happen\n                        throw new RuntimeException(\"Unknown response code \" + rpcResponse.getCode());\n                    }\n                }\n            }\n            final RemotingCommand response = RemotingCommand.createResponseCommand(QueryConsumerOffsetResponseHeader.class);\n            final QueryConsumerOffsetResponseHeader responseHeader = (QueryConsumerOffsetResponseHeader) response.readCustomHeader();\n            if (offset >= 0) {\n                responseHeader.setOffset(offset);\n                response.setCode(ResponseCode.SUCCESS);\n                response.setRemark(null);\n            } else {\n                response.setCode(ResponseCode.QUERY_NOT_FOUND);\n                response.setRemark(\"Not found, maybe this group consumer boot first\");\n            }\n            RemotingCommand rewriteResponseResult = rewriteResponseForStaticTopic(requestHeader, responseHeader, mappingContext, response.getCode());\n            if (rewriteResponseResult != null) {\n                return rewriteResponseResult;\n            }\n            return response;\n        } catch (Throwable t) {\n            return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());\n        }\n    }\n\n    public RemotingCommand rewriteResponseForStaticTopic(final QueryConsumerOffsetRequestHeader requestHeader,\n        final QueryConsumerOffsetResponseHeader responseHeader,\n        final TopicQueueMappingContext mappingContext, final int code) {\n        try {\n            if (mappingContext.getMappingDetail() == null) {\n                return null;\n            }\n            if (code != ResponseCode.SUCCESS) {\n                return null;\n            }\n            LogicQueueMappingItem item = mappingContext.getCurrentItem();\n            responseHeader.setOffset(item.computeStaticQueueOffsetStrictly(responseHeader.getOffset()));\n            //no need to construct new object\n            return null;\n        } catch (Throwable t) {\n            return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());\n        }\n    }\n\n    private RemotingCommand queryConsumerOffset(ChannelHandlerContext ctx, RemotingCommand request)\n        throws RemotingCommandException {\n        final RemotingCommand response =\n            RemotingCommand.createResponseCommand(QueryConsumerOffsetResponseHeader.class);\n        final QueryConsumerOffsetResponseHeader responseHeader =\n            (QueryConsumerOffsetResponseHeader) response.readCustomHeader();\n        final QueryConsumerOffsetRequestHeader requestHeader =\n            (QueryConsumerOffsetRequestHeader) request\n                .decodeCommandCustomHeader(QueryConsumerOffsetRequestHeader.class);\n\n        TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader);\n        RemotingCommand rewriteResult = rewriteRequestForStaticTopic(requestHeader, mappingContext);\n        if (rewriteResult != null) {\n            return rewriteResult;\n        }\n\n        long offset =\n            this.brokerController.getConsumerOffsetManager().queryOffset(\n                requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId());\n\n        if (offset >= 0) {\n            responseHeader.setOffset(offset);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n        } else {\n            long minOffset =\n                this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(),\n                    requestHeader.getQueueId());\n            if (requestHeader.getSetZeroIfNotFound() != null && Boolean.FALSE.equals(requestHeader.getSetZeroIfNotFound())) {\n                response.setCode(ResponseCode.QUERY_NOT_FOUND);\n                response.setRemark(\"Not found, do not set to zero, maybe this group boot first\");\n            } else if (minOffset <= 0\n                && this.brokerController.getMessageStore().checkInMemByConsumeOffset(\n                requestHeader.getTopic(), requestHeader.getQueueId(), 0, 1)) {\n                responseHeader.setOffset(0L);\n                response.setCode(ResponseCode.SUCCESS);\n                response.setRemark(null);\n            } else {\n                response.setCode(ResponseCode.QUERY_NOT_FOUND);\n                response.setRemark(\"Not found, V3_0_6_SNAPSHOT maybe this group consumer boot first\");\n            }\n        }\n\n        RemotingCommand rewriteResponseResult = rewriteResponseForStaticTopic(requestHeader, responseHeader, mappingContext, response.getCode());\n        if (rewriteResponseResult != null) {\n            return rewriteResponseResult;\n        }\n\n        return response;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/DefaultPullMessageResultHandler.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelFutureListener;\nimport io.netty.channel.FileRegion;\nimport io.opentelemetry.api.common.Attributes;\nimport java.nio.ByteBuffer;\nimport java.util.List;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.longpolling.PullRequest;\nimport org.apache.rocketmq.broker.metrics.BrokerMetricsManager;\nimport org.apache.rocketmq.broker.pagecache.ManyMessageTransfer;\nimport org.apache.rocketmq.broker.plugin.PullMessageResultHandler;\nimport org.apache.rocketmq.common.AbortProcessException;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.TopicFilterType;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.common.sysflag.PullSysFlag;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.metrics.RemotingMetricsManager;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingContext;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.remoting.protocol.topic.OffsetMovedEvent;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.MessageFilter;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.config.BrokerRole;\n\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_TOPIC;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_REQUEST_CODE;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_RESPONSE_CODE;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_RESULT;\n\npublic class DefaultPullMessageResultHandler implements PullMessageResultHandler {\n\n    protected static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    protected final BrokerController brokerController;\n\n    public DefaultPullMessageResultHandler(final BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    @Override\n    public RemotingCommand handle(final GetMessageResult getMessageResult,\n        final RemotingCommand request,\n        final PullMessageRequestHeader requestHeader,\n        final Channel channel,\n        final SubscriptionData subscriptionData,\n        final SubscriptionGroupConfig subscriptionGroupConfig,\n        final boolean brokerAllowSuspend,\n        final MessageFilter messageFilter,\n        RemotingCommand response,\n        TopicQueueMappingContext mappingContext,\n        long beginTimeMills) {\n        PullMessageProcessor processor = brokerController.getPullMessageProcessor();\n        final String clientAddress = RemotingHelper.parseChannelRemoteAddr(channel);\n        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());\n        processor.composeResponseHeader(requestHeader, getMessageResult, topicConfig.getTopicSysFlag(),\n            subscriptionGroupConfig, response, clientAddress);\n        try {\n            processor.executeConsumeMessageHookBefore(request, requestHeader, getMessageResult, brokerAllowSuspend, response.getCode());\n        } catch (AbortProcessException e) {\n            response.setCode(e.getResponseCode());\n            response.setRemark(e.getErrorMessage());\n            return response;\n        }\n\n        //rewrite the response for the static topic\n        final PullMessageResponseHeader responseHeader = (PullMessageResponseHeader) response.readCustomHeader();\n        RemotingCommand rewriteResult = processor.rewriteResponseForStaticTopic(requestHeader, responseHeader, mappingContext, response.getCode());\n        if (rewriteResult != null) {\n            response = rewriteResult;\n        }\n\n        processor.updateBroadcastPulledOffset(requestHeader.getTopic(), requestHeader.getConsumerGroup(),\n            requestHeader.getQueueId(), requestHeader, channel, response, getMessageResult.getNextBeginOffset());\n        processor.tryCommitOffset(brokerAllowSuspend, requestHeader, getMessageResult.getNextBeginOffset(),\n            clientAddress);\n\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS:\n                this.brokerController.getBrokerStatsManager().incGroupGetNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(),\n                    getMessageResult.getMessageCount());\n\n                this.brokerController.getBrokerStatsManager().incGroupGetSize(requestHeader.getConsumerGroup(), requestHeader.getTopic(),\n                    getMessageResult.getBufferTotalSize());\n\n                this.brokerController.getBrokerStatsManager().incBrokerGetNums(requestHeader.getTopic(), getMessageResult.getMessageCount());\n\n                if (!BrokerMetricsManager.isRetryOrDlqTopic(requestHeader.getTopic())) {\n                    Attributes attributes = this.brokerController.getBrokerMetricsManager().newAttributesBuilder()\n                        .put(LABEL_TOPIC, requestHeader.getTopic())\n                        .put(LABEL_CONSUMER_GROUP, requestHeader.getConsumerGroup())\n                        .put(LABEL_IS_SYSTEM, TopicValidator.isSystemTopic(requestHeader.getTopic()) || MixAll.isSysConsumerGroup(requestHeader.getConsumerGroup()))\n                        .build();\n                    this.brokerController.getBrokerMetricsManager().getMessagesOutTotal().add(getMessageResult.getMessageCount(), attributes);\n                    this.brokerController.getBrokerMetricsManager().getThroughputOutTotal().add(getMessageResult.getBufferTotalSize(), attributes);\n                }\n\n                if (!channelIsWritable(channel, requestHeader)) {\n                    getMessageResult.release();\n                    //ignore pull request\n                    return null;\n                }\n\n                if (this.brokerController.getBrokerConfig().isTransferMsgByHeap()) {\n                    final byte[] r = this.readGetMessageResult(getMessageResult, requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId());\n                    this.brokerController.getBrokerStatsManager().incGroupGetLatency(requestHeader.getConsumerGroup(),\n                        requestHeader.getTopic(), requestHeader.getQueueId(),\n                        (int) (this.brokerController.getMessageStore().now() - beginTimeMills));\n                    response.setBody(r);\n                    return response;\n                } else {\n                    try {\n                        FileRegion fileRegion =\n                            new ManyMessageTransfer(response.encodeHeader(getMessageResult.getBufferTotalSize()), getMessageResult);\n                        RemotingCommand finalResponse = response;\n                        channel.writeAndFlush(fileRegion)\n                            .addListener((ChannelFutureListener) future -> {\n                                getMessageResult.release();\n                                RemotingMetricsManager remotingMetricsManager = brokerController.getBrokerMetricsManager().getRemotingMetricsManager();\n                                Attributes attributes = remotingMetricsManager.newAttributesBuilder()\n                                    .put(LABEL_REQUEST_CODE, RemotingHelper.getRequestCodeDesc(request.getCode()))\n                                    .put(LABEL_RESPONSE_CODE, RemotingHelper.getResponseCodeDesc(finalResponse.getCode()))\n                                    .put(LABEL_RESULT, remotingMetricsManager.getWriteAndFlushResult(future))\n                                    .build();\n                                remotingMetricsManager.getRpcLatency().record(request.getProcessTimer().elapsed(TimeUnit.MILLISECONDS), attributes);\n                                if (!future.isSuccess()) {\n                                    log.error(\"Fail to transfer messages from page cache to {}\", channel.remoteAddress(), future.cause());\n                                }\n                            });\n                    } catch (Throwable e) {\n                        log.error(\"Error occurred when transferring messages from page cache\", e);\n                        getMessageResult.release();\n                    }\n                    return null;\n                }\n            case ResponseCode.PULL_NOT_FOUND:\n                final boolean hasSuspendFlag = PullSysFlag.hasSuspendFlag(requestHeader.getSysFlag());\n                final long suspendTimeoutMillisLong = hasSuspendFlag ? requestHeader.getSuspendTimeoutMillis() : 0;\n\n                if (brokerAllowSuspend && hasSuspendFlag) {\n                    long pollingTimeMills = suspendTimeoutMillisLong;\n                    if (!this.brokerController.getBrokerConfig().isLongPollingEnable()) {\n                        pollingTimeMills = this.brokerController.getBrokerConfig().getShortPollingTimeMills();\n                    }\n\n                    String topic = requestHeader.getTopic();\n                    long offset = requestHeader.getQueueOffset();\n                    int queueId = requestHeader.getQueueId();\n                    PullRequest pullRequest = new PullRequest(request, channel, pollingTimeMills,\n                        this.brokerController.getMessageStore().now(), offset, subscriptionData, messageFilter);\n                    this.brokerController.getPullRequestHoldService().suspendPullRequest(topic, queueId, pullRequest);\n                    return null;\n                }\n            case ResponseCode.PULL_RETRY_IMMEDIATELY:\n                break;\n            case ResponseCode.PULL_OFFSET_MOVED:\n                if (this.brokerController.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE\n                    || this.brokerController.getMessageStoreConfig().isOffsetCheckInSlave()) {\n                    MessageQueue mq = new MessageQueue();\n                    mq.setTopic(requestHeader.getTopic());\n                    mq.setQueueId(requestHeader.getQueueId());\n                    mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());\n\n                    OffsetMovedEvent event = new OffsetMovedEvent();\n                    event.setConsumerGroup(requestHeader.getConsumerGroup());\n                    event.setMessageQueue(mq);\n                    event.setOffsetRequest(requestHeader.getQueueOffset());\n                    event.setOffsetNew(getMessageResult.getNextBeginOffset());\n                    log.warn(\n                        \"PULL_OFFSET_MOVED:correction offset. topic={}, groupId={}, requestOffset={}, newOffset={}, suggestBrokerId={}\",\n                        requestHeader.getTopic(), requestHeader.getConsumerGroup(), event.getOffsetRequest(), event.getOffsetNew(),\n                        responseHeader.getSuggestWhichBrokerId());\n                } else {\n                    responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getBrokerId());\n                    response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY);\n                    log.warn(\"PULL_OFFSET_MOVED:none correction. topic={}, groupId={}, requestOffset={}, suggestBrokerId={}\",\n                        requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getQueueOffset(),\n                        responseHeader.getSuggestWhichBrokerId());\n                }\n\n                break;\n            default:\n                log.warn(\"[BUG] impossible result code of get message: {}\", response.getCode());\n                assert false;\n        }\n\n        return response;\n    }\n\n    private boolean channelIsWritable(Channel channel, PullMessageRequestHeader requestHeader) {\n        if (this.brokerController.getBrokerConfig().isEnableNetWorkFlowControl()) {\n            if (!channel.isWritable()) {\n                log.warn(\"channel {} not writable ,cid {}\", channel.remoteAddress(), requestHeader.getConsumerGroup());\n                return false;\n            }\n\n        }\n        return true;\n    }\n\n    protected byte[] readGetMessageResult(final GetMessageResult getMessageResult, final String group,\n        final String topic,\n        final int queueId) {\n        final ByteBuffer byteBuffer = ByteBuffer.allocate(getMessageResult.getBufferTotalSize());\n\n        long storeTimestamp = 0;\n        try {\n            List<ByteBuffer> messageBufferList = getMessageResult.getMessageBufferList();\n            for (ByteBuffer bb : messageBufferList) {\n\n                byteBuffer.put(bb);\n                int sysFlag = bb.getInt(MessageDecoder.SYSFLAG_POSITION);\n//                bornhost has the IPv4 ip if the MessageSysFlag.BORNHOST_V6_FLAG bit of sysFlag is 0\n//                IPv4 host = ip(4 byte) + port(4 byte); IPv6 host = ip(16 byte) + port(4 byte)\n                int bornhostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20;\n                int msgStoreTimePos = 4 // 1 TOTALSIZE\n                    + 4 // 2 MAGICCODE\n                    + 4 // 3 BODYCRC\n                    + 4 // 4 QUEUEID\n                    + 4 // 5 FLAG\n                    + 8 // 6 QUEUEOFFSET\n                    + 8 // 7 PHYSICALOFFSET\n                    + 4 // 8 SYSFLAG\n                    + 8 // 9 BORNTIMESTAMP\n                    + bornhostLength; // 10 BORNHOST\n                storeTimestamp = bb.getLong(msgStoreTimePos);\n            }\n        } finally {\n            getMessageResult.release();\n        }\n\n        this.brokerController.getBrokerStatsManager().recordDiskFallBehindTime(group, topic, queueId, this.brokerController.getMessageStore().now() - storeTimestamp);\n        return byteBuffer.array();\n    }\n\n    protected void generateOffsetMovedEvent(final OffsetMovedEvent event) {\n        try {\n            MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n            msgInner.setTopic(TopicValidator.RMQ_SYS_OFFSET_MOVED_EVENT);\n            msgInner.setTags(event.getConsumerGroup());\n            msgInner.setDelayTimeLevel(0);\n            msgInner.setKeys(event.getConsumerGroup());\n            msgInner.setBody(event.encode());\n            msgInner.setFlag(0);\n            msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));\n            msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(TopicFilterType.SINGLE_TAG, msgInner.getTags()));\n\n            msgInner.setQueueId(0);\n            msgInner.setSysFlag(0);\n            msgInner.setBornTimestamp(System.currentTimeMillis());\n            msgInner.setBornHost(NetworkUtil.string2SocketAddress(this.brokerController.getBrokerAddr()));\n            msgInner.setStoreHost(msgInner.getBornHost());\n\n            msgInner.setReconsumeTimes(0);\n\n            PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner);\n        } catch (Exception e) {\n            log.warn(String.format(\"generateOffsetMovedEvent Exception, %s\", event.toString()), e);\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\n\nimport org.apache.rocketmq.broker.transaction.OperationResult;\nimport org.apache.rocketmq.broker.transaction.queue.TransactionalMessageUtil;\nimport org.apache.rocketmq.common.TopicFilterType;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.config.BrokerRole;\n\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_TOPIC;\n\n/**\n * EndTransaction processor: process commit and rollback message\n */\npublic class EndTransactionProcessor implements NettyRequestProcessor {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME);\n    private final BrokerController brokerController;\n\n    public EndTransactionProcessor(final BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    @Override\n    public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws\n        RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final EndTransactionRequestHeader requestHeader =\n            (EndTransactionRequestHeader) request.decodeCommandCustomHeader(EndTransactionRequestHeader.class);\n        LOGGER.debug(\"Transaction request:{}\", requestHeader);\n        if (BrokerRole.SLAVE == brokerController.getMessageStoreConfig().getBrokerRole()) {\n            response.setCode(ResponseCode.SLAVE_NOT_AVAILABLE);\n            LOGGER.warn(\"Message store is slave mode, so end transaction is forbidden. \");\n            return response;\n        }\n\n        if (requestHeader.getFromTransactionCheck()) {\n            switch (requestHeader.getCommitOrRollback()) {\n                case MessageSysFlag.TRANSACTION_NOT_TYPE: {\n                    LOGGER.warn(\"Check producer[{}] transaction state, but it's pending status.\"\n                            + \"RequestHeader: {} Remark: {}\",\n                        RemotingHelper.parseChannelRemoteAddr(ctx.channel()),\n                        requestHeader.toString(),\n                        request.getRemark());\n                    return null;\n                }\n\n                case MessageSysFlag.TRANSACTION_COMMIT_TYPE: {\n                    LOGGER.warn(\"Check producer[{}] transaction state, the producer commit the message.\"\n                            + \"RequestHeader: {} Remark: {}\",\n                        RemotingHelper.parseChannelRemoteAddr(ctx.channel()),\n                        requestHeader.toString(),\n                        request.getRemark());\n\n                    break;\n                }\n\n                case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: {\n                    LOGGER.warn(\"Check producer[{}] transaction state, the producer rollback the message.\"\n                            + \"RequestHeader: {} Remark: {}\",\n                        RemotingHelper.parseChannelRemoteAddr(ctx.channel()),\n                        requestHeader.toString(),\n                        request.getRemark());\n                    break;\n                }\n                default:\n                    return null;\n            }\n        } else {\n            switch (requestHeader.getCommitOrRollback()) {\n                case MessageSysFlag.TRANSACTION_NOT_TYPE: {\n                    LOGGER.warn(\"The producer[{}] end transaction in sending message,  and it's pending status.\"\n                            + \"RequestHeader: {} Remark: {}\",\n                        RemotingHelper.parseChannelRemoteAddr(ctx.channel()),\n                        requestHeader.toString(),\n                        request.getRemark());\n                    return null;\n                }\n\n                case MessageSysFlag.TRANSACTION_COMMIT_TYPE: {\n                    break;\n                }\n\n                case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: {\n                    LOGGER.warn(\"The producer[{}] end transaction in sending message, rollback the message.\"\n                            + \"RequestHeader: {} Remark: {}\",\n                        RemotingHelper.parseChannelRemoteAddr(ctx.channel()),\n                        requestHeader.toString(),\n                        request.getRemark());\n                    break;\n                }\n                default:\n                    return null;\n            }\n        }\n        OperationResult result = new OperationResult();\n        if (MessageSysFlag.TRANSACTION_COMMIT_TYPE == requestHeader.getCommitOrRollback()) {\n            result = this.brokerController.getTransactionalMessageService().commitMessage(requestHeader);\n            if (result.getResponseCode() == ResponseCode.SUCCESS) {\n                if (rejectCommitOrRollback(requestHeader, result.getPrepareMessage())) {\n                    response.setCode(ResponseCode.ILLEGAL_OPERATION);\n                    LOGGER.warn(\"Message commit fail [producer end]. currentTimeMillis - bornTime > checkImmunityTime, msgId={},commitLogOffset={}, wait check\",\n                            requestHeader.getMsgId(), requestHeader.getCommitLogOffset());\n                    return response;\n                }\n                RemotingCommand res = checkPrepareMessage(result.getPrepareMessage(), requestHeader);\n                if (res.getCode() == ResponseCode.SUCCESS) {\n                    MessageExtBrokerInner msgInner = endMessageTransaction(result.getPrepareMessage());\n                    msgInner.setSysFlag(MessageSysFlag.resetTransactionValue(msgInner.getSysFlag(), requestHeader.getCommitOrRollback()));\n                    msgInner.setQueueOffset(requestHeader.getTranStateTableOffset());\n                    msgInner.setPreparedTransactionOffset(requestHeader.getCommitLogOffset());\n                    msgInner.setStoreTimestamp(result.getPrepareMessage().getStoreTimestamp());\n                    MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED);\n                    RemotingCommand sendResult = sendFinalMessage(msgInner);\n                    if (sendResult.getCode() == ResponseCode.SUCCESS) {\n                        deletePrepareMessage(result);\n                        // successful committed, then total num of half-messages minus 1\n                        this.brokerController.getTransactionalMessageService().getTransactionMetrics().addAndGet(msgInner.getTopic(), -1);\n                        this.brokerController.getBrokerMetricsManager().getCommitMessagesTotal().add(1, this.brokerController.getBrokerMetricsManager().newAttributesBuilder()\n                                .put(LABEL_TOPIC, msgInner.getTopic())\n                                .build());\n                        // record the commit latency.\n                        Long commitLatency = (System.currentTimeMillis() - result.getPrepareMessage().getBornTimestamp()) / 1000;\n                        this.brokerController.getBrokerMetricsManager().getTransactionFinishLatency().record(commitLatency, this.brokerController.getBrokerMetricsManager().newAttributesBuilder()\n                                .put(LABEL_TOPIC, msgInner.getTopic())\n                                .build());\n                    }\n                    return sendResult;\n                }\n                return res;\n            }\n        } else if (MessageSysFlag.TRANSACTION_ROLLBACK_TYPE == requestHeader.getCommitOrRollback()) {\n            result = this.brokerController.getTransactionalMessageService().rollbackMessage(requestHeader);\n            if (result.getResponseCode() == ResponseCode.SUCCESS) {\n                if (rejectCommitOrRollback(requestHeader, result.getPrepareMessage())) {\n                    response.setCode(ResponseCode.ILLEGAL_OPERATION);\n                    LOGGER.warn(\"Message rollback fail [producer end]. currentTimeMillis - bornTime > checkImmunityTime, msgId={},commitLogOffset={}, wait check\",\n                            requestHeader.getMsgId(), requestHeader.getCommitLogOffset());\n                    return response;\n                }\n                RemotingCommand res = checkPrepareMessage(result.getPrepareMessage(), requestHeader);\n                if (res.getCode() == ResponseCode.SUCCESS) {\n                    deletePrepareMessage(result);\n                    // roll back, then total num of half-messages minus 1\n                    this.brokerController.getTransactionalMessageService().getTransactionMetrics().addAndGet(result.getPrepareMessage().getProperty(MessageConst.PROPERTY_REAL_TOPIC), -1);\n                    this.brokerController.getBrokerMetricsManager().getRollBackMessagesTotal().add(1, this.brokerController.getBrokerMetricsManager().newAttributesBuilder()\n                            .put(LABEL_TOPIC, result.getPrepareMessage().getProperty(MessageConst.PROPERTY_REAL_TOPIC))\n                            .build());\n                }\n                return res;\n            }\n        }\n        response.setCode(result.getResponseCode());\n        response.setRemark(result.getResponseRemark());\n        return response;\n    }\n\n    private void deletePrepareMessage(OperationResult result) {\n        if (null == result || null == result.getPrepareMessage()) {\n            LOGGER.error(\"deletePrepareMessage param error, result is null or prepareMessage is null\");\n            return;\n        }\n        MessageExt prepareMessage = result.getPrepareMessage();\n        String halfTopic = prepareMessage.getTopic();\n        if (StringUtils.isEmpty(halfTopic)) {\n            LOGGER.error(\"deletePrepareMessage halfTopic is empty, halfTopic: {}\", halfTopic);\n            return;\n        }\n        if (TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC.equals(halfTopic)) {\n            this.brokerController.getTransactionalMessageService().deletePrepareMessage(prepareMessage);\n        } else if (this.brokerController.getMessageStoreConfig().isTransRocksDBEnable() && TopicValidator.RMQ_SYS_ROCKSDB_TRANS_HALF_TOPIC.equals(halfTopic)) {\n            this.brokerController.getMessageStore().getTransMessageRocksDBStore().deletePrepareMessage(prepareMessage);\n        } else {\n            LOGGER.warn(\"deletePrepareMessage error, topic of half message is: {}, transRocksDBEnable: {}\", halfTopic, this.brokerController.getMessageStoreConfig().isTransRocksDBEnable());\n        }\n    }\n\n    /**\n     * If you specify a custom first check time CheckImmunityTimeInSeconds,\n     * And the commit/rollback request whose validity period exceeds CheckImmunityTimeInSeconds and is not checked back will be processed and failed\n     * returns ILLEGAL_OPERATION 604 error\n     * @param requestHeader\n     * @param messageExt\n     * @return\n     */\n    public boolean rejectCommitOrRollback(EndTransactionRequestHeader requestHeader, MessageExt messageExt) {\n        if (requestHeader.getFromTransactionCheck()) {\n            return false;\n        }\n        long transactionTimeout = brokerController.getBrokerConfig().getTransactionTimeOut();\n\n        String checkImmunityTimeStr = messageExt.getUserProperty(MessageConst.PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS);\n        if (StringUtils.isNotEmpty(checkImmunityTimeStr)) {\n            long valueOfCurrentMinusBorn = System.currentTimeMillis() - messageExt.getBornTimestamp();\n            long checkImmunityTime = TransactionalMessageUtil.getImmunityTime(checkImmunityTimeStr, transactionTimeout);\n            //Non-check requests that exceed the specified custom first check time fail to return\n            return valueOfCurrentMinusBorn > checkImmunityTime;\n        }\n        return false;\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n\n    private RemotingCommand checkPrepareMessage(MessageExt msgExt, EndTransactionRequestHeader requestHeader) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        if (msgExt != null) {\n            final String pgroupRead = msgExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP);\n            if (!pgroupRead.equals(requestHeader.getProducerGroup())) {\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"The producer group wrong\");\n                return response;\n            }\n\n            if (msgExt.getQueueOffset() != requestHeader.getTranStateTableOffset()) {\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"The transaction state table offset wrong\");\n                return response;\n            }\n\n            if (msgExt.getCommitLogOffset() != requestHeader.getCommitLogOffset()) {\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"The commit log offset wrong\");\n                return response;\n            }\n        } else {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"Find prepared transaction message failed\");\n            return response;\n        }\n        response.setCode(ResponseCode.SUCCESS);\n        return response;\n    }\n\n    private MessageExtBrokerInner endMessageTransaction(MessageExt msgExt) {\n        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n        msgInner.setTopic(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_TOPIC));\n        msgInner.setQueueId(Integer.parseInt(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_QUEUE_ID)));\n        msgInner.setBody(msgExt.getBody());\n        msgInner.setFlag(msgExt.getFlag());\n        msgInner.setBornTimestamp(msgExt.getBornTimestamp());\n        msgInner.setBornHost(msgExt.getBornHost());\n        msgInner.setStoreHost(msgExt.getStoreHost());\n        msgInner.setReconsumeTimes(msgExt.getReconsumeTimes());\n        msgInner.setWaitStoreMsgOK(false);\n        msgInner.setTransactionId(msgExt.getUserProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));\n        msgInner.setSysFlag(msgExt.getSysFlag());\n        TopicFilterType topicFilterType =\n            (msgInner.getSysFlag() & MessageSysFlag.MULTI_TAGS_FLAG) == MessageSysFlag.MULTI_TAGS_FLAG ? TopicFilterType.MULTI_TAG\n                : TopicFilterType.SINGLE_TAG;\n        long tagsCodeValue = MessageExtBrokerInner.tagsString2tagsCode(topicFilterType, msgInner.getTags());\n        msgInner.setTagsCode(tagsCodeValue);\n        String checkTimes = msgExt.getUserProperty(MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES);\n        if (StringUtils.isEmpty(checkTimes) && this.brokerController.getMessageStoreConfig().isTransRocksDBEnable() && null != this.brokerController.getMessageStore().getTransMessageRocksDBStore()) {\n            Integer checkTimesRocksDB = this.brokerController.getMessageStore().getTransMessageRocksDBStore().getCheckTimes(msgInner.getTopic(), msgInner.getTransactionId(), msgExt.getCommitLogOffset());\n            if (null != checkTimesRocksDB && checkTimesRocksDB >= 0) {\n                msgExt.putUserProperty(MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES, String.valueOf(checkTimesRocksDB));\n            }\n        }\n        MessageAccessor.setProperties(msgInner, MessageDecoder.string2messageProperties(MessageDecoder.messageProperties2String(msgExt.getProperties())));\n        MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_REAL_TOPIC);\n        MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_REAL_QUEUE_ID);\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));\n        return msgInner;\n    }\n\n    private RemotingCommand sendFinalMessage(MessageExtBrokerInner msgInner) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner);\n        if (putMessageResult != null) {\n            switch (putMessageResult.getPutMessageStatus()) {\n                // Success\n                case PUT_OK:\n                    this.brokerController.getBrokerStatsManager().incTopicPutNums(msgInner.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum(), 1);\n                    this.brokerController.getBrokerStatsManager().incTopicPutSize(msgInner.getTopic(), putMessageResult.getAppendMessageResult().getWroteBytes());\n                    this.brokerController.getBrokerStatsManager().incBrokerPutNums(msgInner.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum());\n                case FLUSH_DISK_TIMEOUT:\n                case FLUSH_SLAVE_TIMEOUT:\n                case SLAVE_NOT_AVAILABLE:\n                    response.setCode(ResponseCode.SUCCESS);\n                    response.setRemark(null);\n                    break;\n                // Failed\n                case CREATE_MAPPED_FILE_FAILED:\n                    response.setCode(ResponseCode.SYSTEM_ERROR);\n                    response.setRemark(\"Create mapped file failed.\");\n                    break;\n                case MESSAGE_ILLEGAL:\n                case PROPERTIES_SIZE_EXCEEDED:\n                    response.setCode(ResponseCode.MESSAGE_ILLEGAL);\n                    response.setRemark(String.format(\"The message is illegal, maybe msg body or properties length not matched. msg body length limit %dB, msg properties length limit 32KB.\",\n                        this.brokerController.getMessageStoreConfig().getMaxMessageSize()));\n                    break;\n                case SERVICE_NOT_AVAILABLE:\n                    response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE);\n                    response.setRemark(\"Service not available now.\");\n                    break;\n                case OS_PAGE_CACHE_BUSY:\n                    response.setCode(ResponseCode.SYSTEM_ERROR);\n                    response.setRemark(\"OS page cache busy, please try another machine\");\n                    break;\n                case WHEEL_TIMER_MSG_ILLEGAL:\n                    response.setCode(ResponseCode.MESSAGE_ILLEGAL);\n                    response.setRemark(String.format(\"timer message illegal, the delay time should not be bigger than the max delay %dms; or if set del msg, the delay time should be bigger than the current time\",\n                        this.brokerController.getMessageStoreConfig().getTimerMaxDelaySec() * 1000L));\n                    break;\n                case WHEEL_TIMER_FLOW_CONTROL:\n                    response.setCode(ResponseCode.SYSTEM_ERROR);\n                    response.setRemark(String.format(\"timer message is under flow control, max num limit is %d or the current value is greater than %d and less than %d, trigger random flow control\",\n                        this.brokerController.getMessageStoreConfig().getTimerCongestNumEachSlot() * 2L, this.brokerController.getMessageStoreConfig().getTimerCongestNumEachSlot(), this.brokerController.getMessageStoreConfig().getTimerCongestNumEachSlot() * 2L));\n                    break;\n                case WHEEL_TIMER_NOT_ENABLE:\n                    response.setCode(ResponseCode.SYSTEM_ERROR);\n                    response.setRemark(String.format(\"accurate timer message is not enabled, timerWheelEnable is %s\",\n                        this.brokerController.getMessageStoreConfig().isTimerWheelEnable()));\n                    break;\n                case UNKNOWN_ERROR:\n                    response.setCode(ResponseCode.SYSTEM_ERROR);\n                    response.setRemark(\"UNKNOWN_ERROR\");\n                    break;\n                case IN_SYNC_REPLICAS_NOT_ENOUGH:\n                    response.setCode(ResponseCode.SYSTEM_ERROR);\n                    response.setRemark(\"in-sync replicas not enough\");\n                    break;\n                case PUT_TO_REMOTE_BROKER_FAIL:\n                    response.setCode(ResponseCode.SYSTEM_ERROR);\n                    response.setRemark(\"put to remote broker fail\");\n                    break;\n                default:\n                    response.setCode(ResponseCode.SYSTEM_ERROR);\n                    response.setRemark(\"UNKNOWN_ERROR DEFAULT\");\n                    break;\n            }\n            return response;\n        } else {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"store putMessage return null\");\n        }\n        return response;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/LiteManagerProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.processor;\n\nimport com.google.common.annotations.VisibleForTesting;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.List;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.lite.AbstractLiteLifecycleManager;\nimport org.apache.rocketmq.broker.lite.LiteMetadataUtil;\nimport org.apache.rocketmq.broker.lite.LiteSharding;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.lite.LiteLagInfo;\nimport org.apache.rocketmq.common.lite.LiteSubscription;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.admin.OffsetWrapper;\nimport org.apache.rocketmq.remoting.protocol.admin.TopicOffset;\nimport org.apache.rocketmq.remoting.protocol.body.GetBrokerLiteInfoResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.GetLiteClientInfoResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.GetLiteGroupInfoResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.GetLiteTopicInfoResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.GetParentTopicInfoResponseBody;\nimport org.apache.rocketmq.remoting.protocol.header.GetLiteClientInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetLiteGroupInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetLiteTopicInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetParentTopicInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.TriggerLiteDispatchRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\n\nimport java.util.HashSet;\nimport java.util.Map;\nimport java.util.Set;\n\npublic class LiteManagerProcessor implements NettyRequestProcessor {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LITE_LOGGER_NAME);\n\n    private static final int MAX_RETURN_COUNT = 10000;\n    private final BrokerController brokerController;\n    private final AbstractLiteLifecycleManager liteLifecycleManager;\n    private final LiteSharding liteSharding;\n\n    public LiteManagerProcessor(BrokerController brokerController,\n        AbstractLiteLifecycleManager liteLifecycleManager, LiteSharding liteSharding) {\n        this.brokerController = brokerController;\n        this.liteLifecycleManager = liteLifecycleManager;\n        this.liteSharding = liteSharding;\n    }\n\n    @Override\n    public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws Exception {\n        switch (request.getCode()) {\n            case RequestCode.GET_BROKER_LITE_INFO:\n                return this.getBrokerLiteInfo(ctx, request);\n            case RequestCode.GET_PARENT_TOPIC_INFO:\n                return this.getParentTopicInfo(ctx, request);\n            case RequestCode.GET_LITE_TOPIC_INFO:\n                return this.getLiteTopicInfo(ctx, request);\n            case RequestCode.GET_LITE_CLIENT_INFO:\n                return this.getLiteClientInfo(ctx, request);\n            case RequestCode.GET_LITE_GROUP_INFO:\n                return this.getLiteGroupInfo(ctx, request);\n            case RequestCode.TRIGGER_LITE_DISPATCH:\n                return this.triggerLiteDispatch(ctx, request);\n            default:\n                break;\n        }\n        return null;\n    }\n\n    @VisibleForTesting\n    protected RemotingCommand getBrokerLiteInfo(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        GetBrokerLiteInfoResponseBody body = new GetBrokerLiteInfoResponseBody();\n        body.setStoreType(brokerController.getMessageStoreConfig().getStoreType());\n        body.setMaxLmqNum(brokerController.getMessageStoreConfig().getMaxLmqConsumeQueueNum());\n        body.setCurrentLmqNum(brokerController.getMessageStore().getQueueStore().getLmqNum());\n        body.setLiteSubscriptionCount(brokerController.getLiteSubscriptionRegistry().getActiveSubscriptionNum());\n        body.setOrderInfoCount(brokerController.getPopLiteMessageProcessor().getConsumerOrderInfoManager().getOrderInfoCount());\n        body.setCqTableSize(brokerController.getMessageStore().getQueueStore().getConsumeQueueTable().size());\n        body.setOffsetTableSize(brokerController.getConsumerOffsetManager().getOffsetTable().size());\n        body.setEventMapSize(brokerController.getLiteEventDispatcher().getEventMapSize());\n        body.setTopicMeta(LiteMetadataUtil.getTopicTtlMap(brokerController));\n        body.setGroupMeta(LiteMetadataUtil.getSubscriberGroupMap(brokerController));\n\n        response.setBody(body.encode());\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    @VisibleForTesting\n    protected RemotingCommand getParentTopicInfo(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final GetParentTopicInfoRequestHeader requestHeader =\n            request.decodeCommandCustomHeader(GetParentTopicInfoRequestHeader.class);\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        String topic = requestHeader.getTopic();\n        TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(topic);\n        if (null == topicConfig) {\n            response.setCode(ResponseCode.TOPIC_NOT_EXIST);\n            response.setRemark(String.format(\"Topic [%s] not exist.\", topic));\n            return response;\n        }\n        if (!TopicMessageType.LITE.equals(topicConfig.getTopicMessageType())) {\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(String.format(\"Topic [%s] type not match.\", topic));\n            return response;\n        }\n\n        Map<String, Set<String>> subscriberGroupMap = LiteMetadataUtil.getSubscriberGroupMap(brokerController);\n\n        GetParentTopicInfoResponseBody body = new GetParentTopicInfoResponseBody();\n        body.setTopic(topic);\n        body.setTtl(topicConfig.getLiteTopicExpiration());\n        body.setLmqNum(brokerController.getMessageStore().getQueueStore().getLmqNum());\n        body.setLiteTopicCount(liteLifecycleManager.getLiteTopicCount(topic));\n        body.setGroups(subscriberGroupMap != null ? subscriberGroupMap.get(topic) : null);\n\n        response.setBody(body.encode());\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    @VisibleForTesting\n    protected RemotingCommand getLiteTopicInfo(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final GetLiteTopicInfoRequestHeader requestHeader =\n            request.decodeCommandCustomHeader(GetLiteTopicInfoRequestHeader.class);\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        String parentTopic = requestHeader.getParentTopic();\n        String liteTopic = requestHeader.getLiteTopic();\n        TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(parentTopic);\n        if (null == topicConfig) {\n            response.setCode(ResponseCode.TOPIC_NOT_EXIST);\n            response.setRemark(String.format(\"Topic [%s] not exist.\", parentTopic));\n            return response;\n        }\n        if (!TopicMessageType.LITE.equals(topicConfig.getTopicMessageType())) {\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(String.format(\"Topic [%s] type not match.\", parentTopic));\n            return response;\n        }\n\n        String lmqName = LiteUtil.toLmqName(parentTopic, liteTopic);\n        TopicOffset topicOffset = new TopicOffset();\n        long minOffset = 0;\n        long lastUpdateTimestamp = 0;\n        long maxOffset = liteLifecycleManager.getMaxOffsetInQueue(lmqName);\n        if (maxOffset > 0) {\n            minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(lmqName, 0);\n            lastUpdateTimestamp = brokerController.getMessageStore().getMessageStoreTimeStamp(lmqName, 0, maxOffset - 1);\n        }\n        topicOffset.setMinOffset(minOffset < 0 ? 0 : minOffset);\n        topicOffset.setMaxOffset(maxOffset < 0 ? 0 : maxOffset);\n        topicOffset.setLastUpdateTimestamp(lastUpdateTimestamp);\n\n        GetLiteTopicInfoResponseBody body = new GetLiteTopicInfoResponseBody();\n        body.setParentTopic(parentTopic);\n        body.setLiteTopic(liteTopic);\n        body.setSubscriber(brokerController.getLiteSubscriptionRegistry().getSubscriber(lmqName));\n        body.setTopicOffset(topicOffset);\n        body.setShardingToBroker(brokerController.getBrokerConfig().getBrokerName().equals(\n            liteSharding.shardingByLmqName(parentTopic, lmqName)));\n\n        response.setBody(body.encode());\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    @VisibleForTesting\n    protected RemotingCommand getLiteClientInfo(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final GetLiteClientInfoRequestHeader requestHeader =\n            request.decodeCommandCustomHeader(GetLiteClientInfoRequestHeader.class);\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        String parentTopic = requestHeader.getParentTopic();\n        String group = requestHeader.getGroup();\n        TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(parentTopic);\n        if (null == topicConfig) {\n            response.setCode(ResponseCode.TOPIC_NOT_EXIST);\n            response.setRemark(String.format(\"Topic [%s] not exist.\", parentTopic));\n            return response;\n        }\n        if (!TopicMessageType.LITE.equals(topicConfig.getTopicMessageType())) {\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(String.format(\"Topic [%s] type not match.\", parentTopic));\n            return response;\n        }\n        SubscriptionGroupConfig groupConfig =\n            brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(group);\n        if (null == groupConfig) {\n            response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);\n            response.setRemark(String.format(\"Group [%s] not exist.\", group));\n            return response;\n        }\n        if (!parentTopic.equals(groupConfig.getLiteBindTopic())) {\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(String.format(\"Subscription [%s]-[%s] not match.\", group, parentTopic));\n            return response;\n        }\n\n        String clientId = requestHeader.getClientId();\n        int maxCount = Math.min(requestHeader.getMaxCount(), MAX_RETURN_COUNT);\n        Set<String> returnSet = null;\n        int liteTopicCount = 0;\n        LiteSubscription liteSubscription = brokerController.getLiteSubscriptionRegistry().getLiteSubscription(clientId);\n        if (liteSubscription != null && liteSubscription.getLiteTopicSet() != null) {\n            Set<String> liteTopicSet = liteSubscription.getLiteTopicSet();\n            liteTopicCount = liteTopicSet.size();\n            if (maxCount >= liteTopicCount) {\n                returnSet = liteTopicSet;\n            } else {\n                returnSet = new HashSet<>(maxCount);\n                int count = 0;\n                for (String topic : liteTopicSet) {\n                    if (count >= maxCount) {\n                        break;\n                    }\n                    returnSet.add(topic);\n                    count++;\n                }\n            }\n        } else {\n            liteTopicCount = -1;\n        }\n\n        GetLiteClientInfoResponseBody body = new GetLiteClientInfoResponseBody();\n        body.setParentTopic(parentTopic);\n        body.setGroup(group);\n        body.setClientId(clientId);\n        body.setLiteTopicCount(liteTopicCount);\n        body.setLiteTopicSet(returnSet);\n        body.setLastAccessTime(brokerController.getLiteEventDispatcher().getClientLastAccessTime(clientId));\n\n        response.setBody(body.encode());\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    @VisibleForTesting\n    protected RemotingCommand getLiteGroupInfo(ChannelHandlerContext ctx, RemotingCommand request)\n        throws RemotingCommandException {\n        final GetLiteGroupInfoRequestHeader requestHeader =\n            request.decodeCommandCustomHeader(GetLiteGroupInfoRequestHeader.class);\n        final String group = requestHeader.getGroup();\n        final String liteTopic = requestHeader.getLiteTopic();\n        final int topK = requestHeader.getTopK();\n        LOGGER.info(\"Broker receive request to getLiteGroupInfo, group:{}, liteTopic:{}, caller:{}\",\n            group, liteTopic, RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n\n        SubscriptionGroupConfig groupConfig =\n            brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(group);\n        if (null == groupConfig) {\n            return RemotingCommand.createResponseCommand(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST,\n                String.format(\"Group [%s] not exist.\", group));\n        }\n        if (StringUtils.isEmpty(groupConfig.getLiteBindTopic())) {\n            return RemotingCommand.createResponseCommand(ResponseCode.INVALID_PARAMETER,\n                String.format(\"Group [%s] is not a LITE group.\", group));\n        }\n        String bindTopic = groupConfig.getLiteBindTopic();\n        GetLiteGroupInfoResponseBody body = new GetLiteGroupInfoResponseBody();\n        body.setGroup(group);\n        body.setParentTopic(bindTopic);\n        body.setLiteTopic(liteTopic);\n\n        if (StringUtils.isEmpty(liteTopic)) {\n            Pair<List<LiteLagInfo>, Long> lagCountPair = brokerController.getBrokerMetricsManager()\n                .getLiteConsumerLagCalculator()\n                .getLagCountTopK(group, topK);\n\n            Pair<List<LiteLagInfo>, Long> lagTimePair = brokerController.getBrokerMetricsManager()\n                .getLiteConsumerLagCalculator()\n                .getLagTimestampTopK(group, bindTopic, topK);\n\n            body.setLagCountTopK(lagCountPair.getObject1());\n            body.setTotalLagCount(lagCountPair.getObject2());\n            body.setLagTimestampTopK(lagTimePair.getObject1());\n            body.setEarliestUnconsumedTimestamp(lagTimePair.getObject2());\n        } else {\n            String lmqName = LiteUtil.toLmqName(bindTopic, liteTopic);\n            long maxOffset = liteLifecycleManager.getMaxOffsetInQueue(lmqName);\n            if (maxOffset > 0) {\n                long commitOffset = brokerController.getConsumerOffsetManager().queryOffset(group, lmqName, 0);\n                if (commitOffset >= 0) {\n                    // lag count and unconsumedTimestamp, reuse total field\n                    body.setTotalLagCount(maxOffset - commitOffset);\n                    body.setEarliestUnconsumedTimestamp(brokerController.getMessageStore().getMessageStoreTimeStamp(\n                        lmqName, 0, commitOffset));\n\n                    OffsetWrapper offsetWrapper = new OffsetWrapper();\n                    offsetWrapper.setBrokerOffset(maxOffset);\n                    offsetWrapper.setConsumerOffset(commitOffset);\n                    if (commitOffset - 1 >= 0) {\n                        offsetWrapper.setLastTimestamp(\n                            brokerController.getMessageStore().getMessageStoreTimeStamp(lmqName, 0, commitOffset - 1));\n                    }\n                    body.setLiteTopicOffsetWrapper(offsetWrapper);\n                }\n            } else {\n                body.setTotalLagCount(-1);\n                body.setEarliestUnconsumedTimestamp(-1);\n            }\n        }\n\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setBody(body.encode());\n        response.setCode(ResponseCode.SUCCESS);\n        return response;\n    }\n\n    @VisibleForTesting\n    protected RemotingCommand triggerLiteDispatch(ChannelHandlerContext ctx, RemotingCommand request)\n        throws RemotingCommandException {\n        final TriggerLiteDispatchRequestHeader requestHeader =\n            request.decodeCommandCustomHeader(TriggerLiteDispatchRequestHeader.class);\n        final String group = requestHeader.getGroup();\n        final String clientId = requestHeader.getClientId();\n        LOGGER.info(\"Broker receive request to triggerLiteDispatch, group:{}, clientId:{}, caller:{}\",\n            group, clientId, RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n        SubscriptionGroupConfig groupConfig =\n            brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(group);\n        if (null == groupConfig) {\n            return RemotingCommand.createResponseCommand(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST,\n                String.format(\"Group [%s] not exist.\", group));\n        }\n        if (StringUtils.isEmpty(groupConfig.getLiteBindTopic())) {\n            return RemotingCommand.createResponseCommand(ResponseCode.INVALID_PARAMETER,\n                String.format(\"Group [%s] is not a LITE group.\", group));\n        }\n\n        if (StringUtils.isNotEmpty(clientId)) {\n            brokerController.getLiteEventDispatcher().doFullDispatch(clientId, group);\n        } else {\n            brokerController.getLiteEventDispatcher().doFullDispatchByGroup(group);\n        }\n\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        return response;\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/LiteSubscriptionCtlProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.Collections;\nimport java.util.Set;\nimport java.util.stream.Collectors;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.lite.LiteSubscriptionRegistry;\nimport org.apache.rocketmq.broker.lite.LiteQuotaException;\nimport org.apache.rocketmq.broker.lite.LiteMetadataUtil;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.lite.LiteSubscriptionDTO;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.LiteSubscriptionCtlRequestBody;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\npublic class LiteSubscriptionCtlProcessor implements NettyRequestProcessor {\n    protected final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LITE_LOGGER_NAME);\n\n    private final BrokerController brokerController;\n    private final LiteSubscriptionRegistry liteSubscriptionRegistry;\n\n    public LiteSubscriptionCtlProcessor(BrokerController brokerController, LiteSubscriptionRegistry liteSubscriptionRegistry) {\n        this.brokerController = brokerController;\n        this.liteSubscriptionRegistry = liteSubscriptionRegistry;\n    }\n\n    @Override\n    public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws Exception {\n        if (request.getBody() == null) {\n            return RemotingCommand.createResponseCommand(ResponseCode.ILLEGAL_OPERATION,\n                \"Request body is null.\");\n        }\n\n        final LiteSubscriptionCtlRequestBody requestBody = LiteSubscriptionCtlRequestBody\n            .decode(request.getBody(), LiteSubscriptionCtlRequestBody.class);\n\n        Set<LiteSubscriptionDTO> entrySet = requestBody.getSubscriptionSet();\n        if (CollectionUtils.isEmpty(entrySet)) {\n            return RemotingCommand.createResponseCommand(ResponseCode.ILLEGAL_OPERATION,\n                \"LiteSubscriptionCtlRequestBody is empty.\");\n        }\n\n        try {\n            for (LiteSubscriptionDTO entry : entrySet) {\n                final String clientId = entry.getClientId();\n                final String group = entry.getGroup();\n                final String topic = entry.getTopic();\n                if (StringUtils.isBlank(clientId)) {\n                    log.warn(\"clientId is blank, {}\", entry);\n                    continue;\n                }\n                if (StringUtils.isBlank(group)) {\n                    log.warn(\"group is blank, {}\", entry);\n                    continue;\n                }\n                if (StringUtils.isBlank(topic)) {\n                    log.warn(\"topic is blank, {}\", entry);\n                    continue;\n                }\n                final Set<String> lmqNameSet = toLmqNameSet(entry);\n                switch (entry.getAction()) {\n                    case PARTIAL_ADD:\n                        checkConsumeEnable(group);\n                        this.liteSubscriptionRegistry.updateClientChannel(clientId, ctx.channel());\n                        this.liteSubscriptionRegistry.addPartialSubscription(clientId, group, topic, lmqNameSet, entry.getOffsetOption());\n                        break;\n                    case PARTIAL_REMOVE:\n                        this.liteSubscriptionRegistry.removePartialSubscription(clientId, group, topic, lmqNameSet);\n                        break;\n                    case COMPLETE_ADD:\n                        checkConsumeEnable(group);\n                        this.liteSubscriptionRegistry.updateClientChannel(clientId, ctx.channel());\n                        this.liteSubscriptionRegistry.addCompleteSubscription(clientId, group, topic, lmqNameSet,\n                            entry.getVersion());\n                        break;\n                    case COMPLETE_REMOVE:\n                        this.liteSubscriptionRegistry.removeCompleteSubscription(clientId);\n                        break;\n                }\n            }\n            return RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, null);\n        } catch (LiteQuotaException e) {\n            return RemotingCommand.createResponseCommand(ResponseCode.LITE_SUBSCRIPTION_QUOTA_EXCEEDED, e.toString());\n        } catch (IllegalStateException e) {\n            return RemotingCommand.createResponseCommand(ResponseCode.ILLEGAL_OPERATION, e.toString());\n        } catch (Exception e) {\n            log.error(\"LiteSubscriptionCtlProcessor error\", e);\n            return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, e.toString());\n        }\n    }\n\n    private void checkConsumeEnable(String group) {\n        if (!LiteMetadataUtil.isConsumeEnable(group, brokerController)) {\n            throw new IllegalStateException(\"Consumer group is not allowed to consume.\");\n        }\n    }\n\n    private Set<String> toLmqNameSet(LiteSubscriptionDTO liteSubscriptionDTO) {\n        if (CollectionUtils.isEmpty(liteSubscriptionDTO.getLiteTopicSet())) {\n            return Collections.emptySet();\n        }\n        return liteSubscriptionDTO.getLiteTopicSet().stream()\n            .map(liteTopic -> LiteUtil.toLmqName(liteSubscriptionDTO.getTopic(), liteTopic))\n            .collect(Collectors.toSet());\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.Map;\nimport java.util.Random;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.filter.ConsumerFilterData;\nimport org.apache.rocketmq.broker.filter.ConsumerFilterManager;\nimport org.apache.rocketmq.broker.filter.ExpressionMessageFilter;\nimport org.apache.rocketmq.broker.longpolling.PollingHeader;\nimport org.apache.rocketmq.broker.longpolling.PollingResult;\nimport org.apache.rocketmq.broker.longpolling.PopLongPollingService;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.common.help.FAQUrl;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.filter.FilterAPI;\nimport org.apache.rocketmq.remoting.protocol.header.NotificationRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.NotificationResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.MessageFilter;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.queue.ConsumeQueueInterface;\nimport org.apache.rocketmq.store.queue.CqUnit;\nimport org.apache.rocketmq.store.queue.ReferredIterator;\nimport org.rocksdb.RocksDBException;\n\npublic class NotificationProcessor implements NettyRequestProcessor {\n    private static final Logger POP_LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);\n    private final BrokerController brokerController;\n    private final Random random = new Random(System.currentTimeMillis());\n    private final PopLongPollingService popLongPollingService;\n    private static final String BORN_TIME = \"bornTime\";\n\n    public NotificationProcessor(final BrokerController brokerController) {\n        this.brokerController = brokerController;\n        this.popLongPollingService = new PopLongPollingService(brokerController, this, true);\n    }\n\n    public void shutdown() throws Exception {\n        this.popLongPollingService.shutdown();\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n\n    // When a new message is written to CommitLog, this method would be called.\n    // Suspended long polling will receive notification and be wakeup.\n    public void notifyMessageArriving(final String topic, final int queueId, long offset,\n        Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {\n        this.popLongPollingService.notifyMessageArrivingWithRetryTopic(\n            topic, queueId, offset, tagsCode, msgStoreTime, filterBitMap, properties);\n    }\n\n    public void notifyMessageArriving(final String topic, final int queueId) {\n        this.popLongPollingService.notifyMessageArrivingWithRetryTopic(topic, queueId);\n    }\n\n    @Override\n    public RemotingCommand processRequest(final ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        Channel channel = ctx.channel();\n\n        RemotingCommand response = RemotingCommand.createResponseCommand(NotificationResponseHeader.class);\n        final NotificationResponseHeader responseHeader = (NotificationResponseHeader) response.readCustomHeader();\n        final NotificationRequestHeader requestHeader =\n            request.decodeCommandCustomHeader(NotificationRequestHeader.class, true);\n        if (requestHeader.getBornTime() == 0) {\n            final long beginTimeMills = this.brokerController.getMessageStore().now();\n            request.addExtField(BORN_TIME, String.valueOf(beginTimeMills));\n            requestHeader.setBornTime(beginTimeMills);\n        }\n\n        response.setOpaque(request.getOpaque());\n\n        if (!PermName.isReadable(this.brokerController.getBrokerConfig().getBrokerPermission())) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            response.setRemark(String.format(\"the broker[%s] peeking message is forbidden\", this.brokerController.getBrokerConfig().getBrokerIP1()));\n            return response;\n        }\n\n        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());\n        if (null == topicConfig) {\n            POP_LOGGER.error(\"The topic {} not exist, consumer: {} \", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel));\n            response.setCode(ResponseCode.TOPIC_NOT_EXIST);\n            response.setRemark(String.format(\"topic[%s] not exist, apply first please! %s\", requestHeader.getTopic(), FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)));\n            return response;\n        }\n\n        if (!PermName.isReadable(topicConfig.getPerm())) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            response.setRemark(\"the topic[\" + requestHeader.getTopic() + \"] peeking message is forbidden\");\n            return response;\n        }\n\n        if (requestHeader.getQueueId() >= topicConfig.getReadQueueNums()) {\n            String errorInfo = String.format(\"queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]\",\n                requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());\n            POP_LOGGER.warn(errorInfo);\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(errorInfo);\n            return response;\n        }\n\n        SubscriptionGroupConfig subscriptionGroupConfig = this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getConsumerGroup());\n        if (null == subscriptionGroupConfig) {\n            response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);\n            response.setRemark(String.format(\"subscription group [%s] does not exist, %s\", requestHeader.getConsumerGroup(), FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST)));\n            return response;\n        }\n\n        if (!subscriptionGroupConfig.isConsumeEnable()) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            response.setRemark(\"subscription group no permission, \" + requestHeader.getConsumerGroup());\n            return response;\n        }\n        int randomQ = random.nextInt(100);\n        boolean hasMsg = false;\n        BrokerConfig brokerConfig = brokerController.getBrokerConfig();\n\n        SubscriptionData subscriptionData = null;\n        ExpressionMessageFilter messageFilter = null;\n        if (brokerConfig.isUseMessageFilterForNotification() &&\n            StringUtils.isNotEmpty(requestHeader.getExpType()) &&\n            StringUtils.isNotEmpty(requestHeader.getExp())) {\n            try {\n                // origin topic\n                subscriptionData = FilterAPI.build(\n                    requestHeader.getTopic(), requestHeader.getExp(), requestHeader.getExpType());\n\n                ConsumerFilterData consumerFilterData = null;\n                if (!ExpressionType.isTagType(subscriptionData.getExpressionType())) {\n                    consumerFilterData = ConsumerFilterManager.build(\n                        requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getExp(),\n                        requestHeader.getExpType(), System.currentTimeMillis());\n                    if (consumerFilterData == null) {\n                        POP_LOGGER.warn(\"Parse the consumer's subscription[{}] failed, group: {}\",\n                            requestHeader.getExp(), requestHeader.getConsumerGroup());\n                        response.setCode(ResponseCode.SUBSCRIPTION_PARSE_FAILED);\n                        response.setRemark(\"parse the consumer's subscription failed\");\n                        return response;\n                    }\n                }\n                messageFilter = new ExpressionMessageFilter(\n                    subscriptionData, consumerFilterData, brokerController.getConsumerFilterManager());\n            } catch (Exception e) {\n                POP_LOGGER.warn(\"Parse the consumer's subscription[{}] error, group: {}\", requestHeader.getExp(),\n                    requestHeader.getConsumerGroup());\n                response.setCode(ResponseCode.SUBSCRIPTION_PARSE_FAILED);\n                response.setRemark(\"parse the consumer's subscription failed\");\n                return response;\n            }\n        }\n\n        if (requestHeader.getQueueId() < 0) {\n            // read all queue\n            hasMsg = hasMsgFromTopic(topicConfig, randomQ, requestHeader, subscriptionData, messageFilter);\n        } else {\n            int queueId = requestHeader.getQueueId();\n            hasMsg = hasMsgFromQueue(topicConfig.getTopicName(), requestHeader, queueId, subscriptionData, messageFilter);\n        }\n        // if it doesn't have message, fetch retry\n        if (!hasMsg) {\n            String retryTopic = KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), requestHeader.getConsumerGroup(), brokerConfig.isEnableRetryTopicV2());\n            hasMsg = hasMsgFromTopic(retryTopic, randomQ, requestHeader, null, null);\n            if (!hasMsg && brokerConfig.isEnableRetryTopicV2() && brokerConfig.isRetrieveMessageFromPopRetryTopicV1()) {\n                String retryTopicConfigV1 = KeyBuilder.buildPopRetryTopicV1(requestHeader.getTopic(), requestHeader.getConsumerGroup());\n                hasMsg = hasMsgFromTopic(retryTopicConfigV1, randomQ, requestHeader, null, null);\n            }\n        }\n\n        if (!hasMsg) {\n            PollingResult pollingResult = popLongPollingService.polling(ctx, request, new PollingHeader(requestHeader), subscriptionData, messageFilter);\n            if (pollingResult == PollingResult.POLLING_SUC) {\n                return null;\n            } else if (pollingResult == PollingResult.POLLING_FULL) {\n                responseHeader.setPollingFull(true);\n            }\n        }\n        response.setCode(ResponseCode.SUCCESS);\n        responseHeader.setHasMsg(hasMsg);\n        return response;\n    }\n\n    private boolean hasMsgFromTopic(String topicName, int randomQ, NotificationRequestHeader requestHeader, SubscriptionData subscriptionData, MessageFilter messageFilter)\n        throws RemotingCommandException {\n        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topicName);\n        return hasMsgFromTopic(topicConfig, randomQ, requestHeader, subscriptionData, messageFilter);\n    }\n\n    private boolean hasMsgFromTopic(TopicConfig topicConfig, int randomQ, NotificationRequestHeader requestHeader, SubscriptionData subscriptionData, MessageFilter messageFilter)\n        throws RemotingCommandException {\n        boolean hasMsg;\n        if (topicConfig != null) {\n            for (int i = 0; i < topicConfig.getReadQueueNums(); i++) {\n                int queueId = (randomQ + i) % topicConfig.getReadQueueNums();\n                hasMsg = hasMsgFromQueue(topicConfig.getTopicName(), requestHeader, queueId, subscriptionData, messageFilter);\n                if (hasMsg) {\n                    return true;\n                }\n            }\n        }\n        return false;\n    }\n\n    private boolean hasMsgFromQueue(String targetTopic, NotificationRequestHeader requestHeader, int queueId, SubscriptionData subscriptionData, MessageFilter messageFilter) throws RemotingCommandException {\n        if (Boolean.TRUE.equals(requestHeader.getOrder())) {\n            if (this.brokerController.getConsumerOrderInfoManager().checkBlock(requestHeader.getAttemptId(), requestHeader.getTopic(), requestHeader.getConsumerGroup(), queueId, 0)) {\n                return false;\n            }\n        }\n        long offset = getPopOffset(targetTopic, requestHeader.getConsumerGroup(), queueId);\n        try {\n            long restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(targetTopic, queueId) - offset;\n            int maxFilterMessageNum = this.brokerController.getBrokerConfig().getMaxMessageFilterNumForNotification();\n            boolean needFilter = restNum < maxFilterMessageNum &&\n                subscriptionData != null &&\n                messageFilter != null &&\n                ExpressionType.isTagType(subscriptionData.getExpressionType());\n            if (needFilter) {\n                ConsumeQueueInterface queue = this.brokerController.getMessageStore().getConsumeQueue(targetTopic, queueId);\n                // If the ConsumeQueue doesn't exist, it's not readable.\n                if (queue == null) {\n                    return false;\n                }\n                ReferredIterator<CqUnit> iterator = null;\n                try {\n                    // In order to take into account both the file CQ and the Rocksdb CQ,\n                    // the count passed here is 32.\n                    iterator = queue.iterateFrom(offset, 32);\n                    if (iterator != null) {\n                        while (iterator.hasNext()) {\n                            CqUnit cqUnit = iterator.next();\n                            if (messageFilter.isMatchedByConsumeQueue(cqUnit.getValidTagsCodeAsLong(), cqUnit.getCqExtUnit())) {\n                                return true;\n                            }\n                        }\n                        return false;\n                    }\n                } finally {\n                    if (iterator != null) {\n                        iterator.release();\n                    }\n                }\n            }\n            return restNum > 0;\n        } catch (ConsumeQueueException | RocksDBException e) {\n            throw new RemotingCommandException(\"Failed to get max offset in queue or iterate in queue\", e);\n        }\n    }\n\n    private long getPopOffset(String topic, String cid, int queueId) {\n        long offset = this.brokerController.getConsumerOffsetManager().queryOffset(cid, topic, queueId);\n        if (offset < 0) {\n            offset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId);\n        }\n\n        long bufferOffset;\n        if (brokerController.getBrokerConfig().isPopConsumerKVServiceEnable()) {\n            bufferOffset = this.brokerController.getConsumerOffsetManager().queryPullOffset(cid, topic, queueId);\n        } else {\n            bufferOffset = this.brokerController.getPopMessageProcessor()\n                .getPopBufferMergeService().getLatestOffset(topic, cid, queueId);\n        }\n\n        return bufferOffset < 0L ? offset : Math.max(bufferOffset, offset);\n    }\n\n    public PopLongPollingService getPopLongPollingService() {\n        return popLongPollingService;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelFutureListener;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.channel.FileRegion;\nimport io.opentelemetry.api.common.Attributes;\nimport java.nio.ByteBuffer;\nimport java.util.List;\nimport java.util.Random;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.broker.BrokerController;\n\nimport org.apache.rocketmq.broker.pagecache.ManyMessageTransfer;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.help.FAQUrl;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.metrics.RemotingMetricsManager;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.PeekMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\n\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_TOPIC;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_REQUEST_CODE;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_RESPONSE_CODE;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_RESULT;\n\npublic class PeekMessageProcessor implements NettyRequestProcessor {\n    private static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private final BrokerController brokerController;\n    private Random random = new Random(System.currentTimeMillis());\n\n    public PeekMessageProcessor(final BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    @Override\n    public RemotingCommand processRequest(final ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        return this.processRequest(ctx.channel(), request, true);\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n\n    private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend)\n        throws RemotingCommandException {\n        final long beginTimeMills = this.brokerController.getMessageStore().now();\n        RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class);\n        final PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader();\n        final PeekMessageRequestHeader requestHeader =\n            (PeekMessageRequestHeader) request.decodeCommandCustomHeader(PeekMessageRequestHeader.class);\n\n        response.setOpaque(request.getOpaque());\n\n        if (!PermName.isReadable(this.brokerController.getBrokerConfig().getBrokerPermission())) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            response.setRemark(String.format(\"the broker[%s] peeking message is forbidden\", this.brokerController.getBrokerConfig().getBrokerIP1()));\n            return response;\n        }\n\n        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());\n        if (null == topicConfig) {\n            LOG.error(\"The topic {} not exist, consumer: {} \", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel));\n            response.setCode(ResponseCode.TOPIC_NOT_EXIST);\n            response.setRemark(String.format(\"topic[%s] not exist, apply first please! %s\", requestHeader.getTopic(), FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)));\n            return response;\n        }\n\n        if (!PermName.isReadable(topicConfig.getPerm())) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            response.setRemark(\"the topic[\" + requestHeader.getTopic() + \"] peeking message is forbidden\");\n            return response;\n        }\n\n        if (requestHeader.getQueueId() >= topicConfig.getReadQueueNums()) {\n            String errorInfo = String.format(\"queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]\",\n                requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());\n            LOG.warn(errorInfo);\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(errorInfo);\n            return response;\n        }\n        SubscriptionGroupConfig subscriptionGroupConfig = this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getConsumerGroup());\n        if (null == subscriptionGroupConfig) {\n            response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);\n            response.setRemark(String.format(\"subscription group [%s] does not exist, %s\", requestHeader.getConsumerGroup(), FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST)));\n            return response;\n        }\n\n        if (!subscriptionGroupConfig.isConsumeEnable()) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            response.setRemark(\"subscription group no permission, \" + requestHeader.getConsumerGroup());\n            return response;\n        }\n        int randomQ = random.nextInt(100);\n        int reviveQid = randomQ % this.brokerController.getBrokerConfig().getReviveQueueNum();\n        GetMessageResult getMessageResult = new GetMessageResult(requestHeader.getMaxMsgNums());\n        boolean needRetry = randomQ % 5 == 0;\n        long popTime = System.currentTimeMillis();\n        long restNum = 0;\n        BrokerConfig brokerConfig = brokerController.getBrokerConfig();\n        if (needRetry) {\n            TopicConfig retryTopicConfig = this.brokerController.getTopicConfigManager()\n                .selectTopicConfig(KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), requestHeader.getConsumerGroup(), brokerConfig.isEnableRetryTopicV2()));\n            if (retryTopicConfig != null) {\n                for (int i = 0; i < retryTopicConfig.getReadQueueNums(); i++) {\n                    int queueId = (randomQ + i) % retryTopicConfig.getReadQueueNums();\n                    restNum = peekMsgFromQueue(true, getMessageResult, requestHeader, queueId, restNum, reviveQid, channel, popTime);\n                }\n            }\n        }\n        if (requestHeader.getQueueId() < 0) {\n            // read all queue\n            for (int i = 0; i < topicConfig.getReadQueueNums(); i++) {\n                int queueId = (randomQ + i) % topicConfig.getReadQueueNums();\n                restNum = peekMsgFromQueue(false, getMessageResult, requestHeader, queueId, restNum, reviveQid, channel, popTime);\n            }\n        } else {\n            int queueId = requestHeader.getQueueId();\n            restNum = peekMsgFromQueue(false, getMessageResult, requestHeader, queueId, restNum, reviveQid, channel, popTime);\n        }\n        // if not full , fetch retry again\n        if (!needRetry && getMessageResult.getMessageMapedList().size() < requestHeader.getMaxMsgNums()) {\n            TopicConfig retryTopicConfig = this.brokerController.getTopicConfigManager()\n                .selectTopicConfig(KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), requestHeader.getConsumerGroup(), brokerConfig.isEnableRetryTopicV2()));\n            if (retryTopicConfig != null) {\n                for (int i = 0; i < retryTopicConfig.getReadQueueNums(); i++) {\n                    int queueId = (randomQ + i) % retryTopicConfig.getReadQueueNums();\n                    restNum = peekMsgFromQueue(true, getMessageResult, requestHeader, queueId, restNum, reviveQid, channel, popTime);\n                }\n            }\n        }\n        if (!getMessageResult.getMessageBufferList().isEmpty()) {\n            response.setCode(ResponseCode.SUCCESS);\n            getMessageResult.setStatus(GetMessageStatus.FOUND);\n        } else {\n            response.setCode(ResponseCode.PULL_NOT_FOUND);\n            getMessageResult.setStatus(GetMessageStatus.NO_MESSAGE_IN_QUEUE);\n\n        }\n        responseHeader.setRestNum(restNum);\n        response.setRemark(getMessageResult.getStatus().name());\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS:\n\n                this.brokerController.getBrokerStatsManager().incGroupGetNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(),\n                    getMessageResult.getMessageCount());\n\n                this.brokerController.getBrokerStatsManager().incGroupGetSize(requestHeader.getConsumerGroup(), requestHeader.getTopic(),\n                    getMessageResult.getBufferTotalSize());\n\n                this.brokerController.getBrokerStatsManager().incBrokerGetNums(requestHeader.getTopic(), getMessageResult.getMessageCount());\n\n                if (this.brokerController.getBrokerConfig().isTransferMsgByHeap()) {\n                    final byte[] r = this.readGetMessageResult(getMessageResult, requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId());\n                    this.brokerController.getBrokerStatsManager().incGroupGetLatency(requestHeader.getConsumerGroup(),\n                        requestHeader.getTopic(), requestHeader.getQueueId(),\n                        (int) (this.brokerController.getMessageStore().now() - beginTimeMills));\n                    response.setBody(r);\n                } else {\n                    final GetMessageResult tmpGetMessageResult = getMessageResult;\n                    try {\n                        FileRegion fileRegion =\n                            new ManyMessageTransfer(response.encodeHeader(getMessageResult.getBufferTotalSize()), getMessageResult);\n                        RemotingCommand finalResponse = response;\n                        channel.writeAndFlush(fileRegion)\n                            .addListener((ChannelFutureListener) future -> {\n                                tmpGetMessageResult.release();\n                                RemotingMetricsManager remotingMetricsManager = brokerController.getBrokerMetricsManager().getRemotingMetricsManager();\n                                Attributes attributes = remotingMetricsManager.newAttributesBuilder()\n                                    .put(LABEL_REQUEST_CODE, RemotingHelper.getRequestCodeDesc(request.getCode()))\n                                    .put(LABEL_RESPONSE_CODE, RemotingHelper.getResponseCodeDesc(finalResponse.getCode()))\n                                    .put(LABEL_RESULT, remotingMetricsManager.getWriteAndFlushResult(future))\n                                    .build();\n                                remotingMetricsManager.getRpcLatency().record(request.getProcessTimer().elapsed(TimeUnit.MILLISECONDS), attributes);\n                                if (!future.isSuccess()) {\n                                    LOG.error(\"Fail to transfer messages from page cache to {}\", channel.remoteAddress(), future.cause());\n                                }\n                            });\n                    } catch (Throwable e) {\n                        LOG.error(\"Error occurred when transferring messages from page cache\", e);\n                        getMessageResult.release();\n                    }\n\n                    response = null;\n                }\n                break;\n            default:\n                assert false;\n        }\n        return response;\n    }\n\n    private long peekMsgFromQueue(boolean isRetry, GetMessageResult getMessageResult,\n        PeekMessageRequestHeader requestHeader, int queueId, long restNum, int reviveQid, Channel channel,\n        long popTime) throws RemotingCommandException {\n        String topic = isRetry ?\n            KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), requestHeader.getConsumerGroup(), brokerController.getBrokerConfig().isEnableRetryTopicV2())\n            : requestHeader.getTopic();\n        GetMessageResult getMessageTmpResult;\n        long offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId);\n        try {\n            restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;\n        } catch (ConsumeQueueException e) {\n            LOG.error(\"Failed to get max offset in queue. topic={}, queue-id={}\", topic, queueId, e);\n            throw new RemotingCommandException(\"Failed to get max offset in queue\", e);\n        }\n        if (getMessageResult.getMessageMapedList().size() >= requestHeader.getMaxMsgNums()) {\n            return restNum;\n        }\n        getMessageTmpResult = this.brokerController.getMessageStore().getMessage(requestHeader.getConsumerGroup(), topic, queueId, offset,\n            requestHeader.getMaxMsgNums() - getMessageResult.getMessageMapedList().size(), null);\n        // maybe store offset is not correct.\n        if (GetMessageStatus.OFFSET_TOO_SMALL.equals(getMessageTmpResult.getStatus()) || GetMessageStatus.OFFSET_OVERFLOW_BADLY.equals(getMessageTmpResult.getStatus())) {\n            offset = getMessageTmpResult.getNextBeginOffset();\n            getMessageTmpResult = this.brokerController.getMessageStore().getMessage(requestHeader.getConsumerGroup(), topic, queueId, offset,\n                requestHeader.getMaxMsgNums() - getMessageResult.getMessageMapedList().size(), null);\n        }\n        if (getMessageTmpResult != null) {\n            if (!getMessageTmpResult.getMessageMapedList().isEmpty() && !isRetry) {\n                Attributes attributes = this.brokerController.getBrokerMetricsManager().newAttributesBuilder()\n                    .put(LABEL_TOPIC, requestHeader.getTopic())\n                    .put(LABEL_CONSUMER_GROUP, requestHeader.getConsumerGroup())\n                    .put(LABEL_IS_SYSTEM, TopicValidator.isSystemTopic(requestHeader.getTopic()) || MixAll.isSysConsumerGroup(requestHeader.getConsumerGroup()))\n                    .build();\n                this.brokerController.getBrokerMetricsManager().getMessagesOutTotal().add(getMessageResult.getMessageCount(), attributes);\n                this.brokerController.getBrokerMetricsManager().getThroughputOutTotal().add(getMessageResult.getBufferTotalSize(), attributes);\n            }\n\n            for (SelectMappedBufferResult mappedBuffer : getMessageTmpResult.getMessageMapedList()) {\n                getMessageResult.addMessage(mappedBuffer);\n            }\n        }\n        return restNum;\n    }\n\n    private long getPopOffset(String topic, String cid, int queueId) {\n        long offset = this.brokerController.getConsumerOffsetManager().queryOffset(cid, topic, queueId);\n        if (offset < 0) {\n            offset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId);\n        }\n        long bufferOffset = this.brokerController.getPopMessageProcessor().getPopBufferMergeService()\n            .getLatestOffset(topic, cid, queueId);\n        if (bufferOffset < 0) {\n            return offset;\n        } else {\n            return bufferOffset > offset ? bufferOffset : offset;\n        }\n    }\n\n    private byte[] readGetMessageResult(final GetMessageResult getMessageResult, final String group, final String topic,\n        final int queueId) {\n        final ByteBuffer byteBuffer = ByteBuffer.allocate(getMessageResult.getBufferTotalSize());\n\n        long storeTimestamp = 0;\n        try {\n            List<ByteBuffer> messageBufferList = getMessageResult.getMessageBufferList();\n            for (ByteBuffer bb : messageBufferList) {\n\n                byteBuffer.put(bb);\n                storeTimestamp = bb.getLong(MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION);\n            }\n        } finally {\n            getMessageResult.release();\n        }\n\n        this.brokerController.getBrokerStatsManager().recordDiskFallBehindTime(group, topic, queueId, this.brokerController.getMessageStore().now() - storeTimestamp);\n        return byteBuffer.array();\n    }\n\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/PollingInfoProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.concurrent.ConcurrentSkipListSet;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.longpolling.PopRequest;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.help.FAQUrl;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.PollingInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PollingInfoResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\n\npublic class PollingInfoProcessor implements NettyRequestProcessor {\n    private static final Logger POP_LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);\n    private final BrokerController brokerController;\n\n    public PollingInfoProcessor(final BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    @Override\n    public RemotingCommand processRequest(final ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        return this.processRequest(ctx.channel(), request);\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n\n    private RemotingCommand processRequest(final Channel channel, RemotingCommand request)\n        throws RemotingCommandException {\n        RemotingCommand response = RemotingCommand.createResponseCommand(PollingInfoResponseHeader.class);\n        final PollingInfoResponseHeader responseHeader = (PollingInfoResponseHeader) response.readCustomHeader();\n        final PollingInfoRequestHeader requestHeader =\n            (PollingInfoRequestHeader) request.decodeCommandCustomHeader(PollingInfoRequestHeader.class);\n\n        response.setOpaque(request.getOpaque());\n\n        if (!PermName.isReadable(this.brokerController.getBrokerConfig().getBrokerPermission())) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            response.setRemark(String.format(\"the broker[%s] peeking message is forbidden\", this.brokerController.getBrokerConfig().getBrokerIP1()));\n            return response;\n        }\n\n        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());\n        if (null == topicConfig) {\n            POP_LOGGER.error(\"The topic {} not exist, consumer: {} \", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel));\n            response.setCode(ResponseCode.TOPIC_NOT_EXIST);\n            response.setRemark(String.format(\"topic[%s] not exist, apply first please! %s\", requestHeader.getTopic(), FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)));\n            return response;\n        }\n\n        if (!PermName.isReadable(topicConfig.getPerm())) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            response.setRemark(\"the topic[\" + requestHeader.getTopic() + \"] peeking message is forbidden\");\n            return response;\n        }\n\n        if (requestHeader.getQueueId() >= topicConfig.getReadQueueNums()) {\n            String errorInfo = String.format(\"queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]\",\n                requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());\n            POP_LOGGER.warn(errorInfo);\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(errorInfo);\n            return response;\n        }\n        SubscriptionGroupConfig subscriptionGroupConfig = this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getConsumerGroup());\n        if (null == subscriptionGroupConfig) {\n            response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);\n            response.setRemark(String.format(\"subscription group [%s] does not exist, %s\", requestHeader.getConsumerGroup(), FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST)));\n            return response;\n        }\n\n        if (!subscriptionGroupConfig.isConsumeEnable()) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            response.setRemark(\"subscription group no permission, \" + requestHeader.getConsumerGroup());\n            return response;\n        }\n        String key = KeyBuilder.buildPollingKey(requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getQueueId());\n        ConcurrentSkipListSet<PopRequest> queue = this.brokerController.getPopMessageProcessor().getPollingMap().getIfPresent(key);\n        if (queue != null) {\n            responseHeader.setPollingNum(queue.size());\n        } else {\n            responseHeader.setPollingNum(0);\n        }\n        response.setCode(ResponseCode.SUCCESS);\n        return response;\n    }\n\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport com.alibaba.fastjson2.JSON;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.PopAckConstants;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.utils.DataConverter;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.pop.AckMsg;\nimport org.apache.rocketmq.store.pop.BatchAckMsg;\nimport org.apache.rocketmq.store.pop.PopCheckPoint;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.LinkedBlockingDeque;\nimport java.util.concurrent.atomic.AtomicInteger;\n\npublic class PopBufferMergeService extends ServiceThread {\n    private static final Logger POP_LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);\n    ConcurrentHashMap<String/*mergeKey*/, PopCheckPointWrapper>\n        buffer = new ConcurrentHashMap<>(1024 * 16);\n    ConcurrentHashMap<String/*topic@cid@queueId*/, QueueWithTime<PopCheckPointWrapper>> commitOffsets =\n        new ConcurrentHashMap<>();\n    private volatile boolean serving = true;\n    private AtomicInteger counter = new AtomicInteger(0);\n    private int scanTimes = 0;\n    private final BrokerController brokerController;\n    private final PopMessageProcessor popMessageProcessor;\n    private final PopMessageProcessor.QueueLockManager queueLockManager;\n    private final long interval = 5;\n    private final long minute5 = 5 * 60 * 1000;\n    private final int countOfMinute1 = (int) (60 * 1000 / interval);\n    private final int countOfSecond1 = (int) (1000 / interval);\n    private final int countOfSecond30 = (int) (30 * 1000 / interval);\n\n    private final List<Byte> batchAckIndexList = new ArrayList<>(32);\n    private volatile boolean master = false;\n\n    public PopBufferMergeService(BrokerController brokerController, PopMessageProcessor popMessageProcessor) {\n        this.brokerController = brokerController;\n        this.popMessageProcessor = popMessageProcessor;\n        this.queueLockManager = popMessageProcessor.getQueueLockManager();\n    }\n\n    private boolean isShouldRunning() {\n        if (this.brokerController.getBrokerConfig().isEnableSlaveActingMaster()) {\n            return true;\n        }\n        this.master = brokerController.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE;\n        return this.master;\n    }\n\n    @Override\n    public String getServiceName() {\n        if (this.brokerController != null && this.brokerController.getBrokerConfig().isInBrokerContainer()) {\n            return brokerController.getBrokerIdentity().getIdentifier() + PopBufferMergeService.class.getSimpleName();\n        }\n        return PopBufferMergeService.class.getSimpleName();\n    }\n\n    @Override\n    public void run() {\n        // scan\n        while (!this.isStopped()) {\n            try {\n                if (!isShouldRunning()) {\n                    // slave\n                    this.waitForRunning(interval * 200 * 5);\n                    POP_LOGGER.info(\"Broker is {}, {}, clear all data\",\n                        brokerController.getMessageStoreConfig().getBrokerRole(), this.master);\n                    this.buffer.clear();\n                    this.commitOffsets.clear();\n                    continue;\n                }\n\n                scan();\n                if (scanTimes % countOfSecond30 == 0) {\n                    scanGarbage();\n                }\n\n                this.waitForRunning(interval);\n\n                if (!this.serving && this.buffer.size() == 0 && getOffsetTotalSize() == 0) {\n                    this.serving = true;\n                }\n            } catch (Throwable e) {\n                POP_LOGGER.error(\"PopBufferMergeService error\", e);\n                this.waitForRunning(3000);\n            }\n        }\n\n        this.serving = false;\n        try {\n            Thread.sleep(2000);\n        } catch (InterruptedException e) {\n        }\n        if (!isShouldRunning()) {\n            return;\n        }\n        if (!brokerController.getBrokerConfig().isInBrokerContainer()) {\n            while (this.buffer.size() > 0 || getOffsetTotalSize() > 0) {\n                scan();\n            }\n        }\n    }\n\n    private int scanCommitOffset() {\n        Iterator<Map.Entry<String, QueueWithTime<PopCheckPointWrapper>>> iterator = this.commitOffsets.entrySet().iterator();\n        int count = 0;\n        while (iterator.hasNext()) {\n            Map.Entry<String, QueueWithTime<PopCheckPointWrapper>> entry = iterator.next();\n            LinkedBlockingDeque<PopCheckPointWrapper> queue = entry.getValue().get();\n            PopCheckPointWrapper pointWrapper;\n            while ((pointWrapper = queue.peek()) != null) {\n                // 1. just offset & stored, not processed by scan\n                // 2. ck is buffer(acked)\n                // 3. ck is buffer(not all acked), all ak are stored and ck is stored\n                if (pointWrapper.isJustOffset() && pointWrapper.isCkStored() || isCkDone(pointWrapper)\n                    || isCkDoneForFinish(pointWrapper) && pointWrapper.isCkStored()) {\n                    if (commitOffset(pointWrapper)) {\n                        queue.poll();\n                    } else {\n                        break;\n                    }\n                } else {\n                    if (System.currentTimeMillis() - pointWrapper.getCk().getPopTime()\n                        > brokerController.getBrokerConfig().getPopCkStayBufferTime() * 2) {\n                        POP_LOGGER.warn(\"[PopBuffer] ck offset long time not commit, {}\", pointWrapper);\n                    }\n                    break;\n                }\n            }\n            final int qs = queue.size();\n            count += qs;\n            if (qs > 5000 && scanTimes % countOfSecond1 == 0) {\n                POP_LOGGER.info(\"[PopBuffer] offset queue size too long, {}, {}\",\n                    entry.getKey(), qs);\n            }\n        }\n        return count;\n    }\n\n    public long getLatestOffset(String lockKey) {\n        QueueWithTime<PopCheckPointWrapper> queue = this.commitOffsets.get(lockKey);\n        if (queue == null) {\n            return -1;\n        }\n        PopCheckPointWrapper pointWrapper = queue.get().peekLast();\n        if (pointWrapper != null) {\n            return pointWrapper.getNextBeginOffset();\n        }\n        return -1;\n    }\n\n    public long getLatestOffset(String topic, String group, int queueId) {\n        return getLatestOffset(KeyBuilder.buildPollingKey(topic, group, queueId));\n    }\n\n    private void scanGarbage() {\n        Iterator<Map.Entry<String, QueueWithTime<PopCheckPointWrapper>>> iterator = commitOffsets.entrySet().iterator();\n        while (iterator.hasNext()) {\n            Map.Entry<String, QueueWithTime<PopCheckPointWrapper>> entry = iterator.next();\n            if (entry.getKey() == null) {\n                continue;\n            }\n            String[] keyArray = entry.getKey().split(PopAckConstants.SPLIT);\n            if (keyArray == null || keyArray.length != 3) {\n                continue;\n            }\n            String topic = keyArray[0];\n            String cid = keyArray[1];\n            if (brokerController.getTopicConfigManager().selectTopicConfig(topic) == null) {\n                POP_LOGGER.info(\"[PopBuffer]remove nonexistent topic {} in buffer!\", topic);\n                iterator.remove();\n                continue;\n            }\n            if (!brokerController.getSubscriptionGroupManager().containsSubscriptionGroup(cid)) {\n                POP_LOGGER.info(\"[PopBuffer]remove nonexistent subscription group {} of topic {} in buffer!\", cid, topic);\n                iterator.remove();\n                continue;\n            }\n            if (System.currentTimeMillis() - entry.getValue().getTime() > minute5) {\n                POP_LOGGER.info(\"[PopBuffer]remove long time not used sub {} of topic {} in buffer!\", cid, topic);\n                iterator.remove();\n                continue;\n            }\n        }\n    }\n\n    private boolean isSubscriptionGroupNotExist(PopCheckPointWrapper pointWrapper) {\n        String group = pointWrapper.getCk().getCId();\n        return brokerController.getSubscriptionGroupManager()\n                .findSubscriptionGroupConfig(group) == null;\n    }\n\n\n    private void scan() {\n        long startTime = System.currentTimeMillis();\n        AtomicInteger count = new AtomicInteger(0);\n        int countCk = 0;\n        Iterator<Map.Entry<String, PopCheckPointWrapper>> iterator = buffer.entrySet().iterator();\n        while (iterator.hasNext()) {\n            Map.Entry<String, PopCheckPointWrapper> entry = iterator.next();\n            PopCheckPointWrapper pointWrapper = entry.getValue();\n\n            // Skip invalid POP records when consumer group does not exist\n            if (isSubscriptionGroupNotExist(pointWrapper)) {\n                POP_LOGGER.warn(\n                        \"[PopBuffer] skip pop record because consumer group not exist, group={}, ck={}\",\n                        pointWrapper.getCk().getCId(),\n                        pointWrapper\n                );\n                iterator.remove();\n                counter.decrementAndGet();\n                continue;\n            }\n\n\n            // just process offset(already stored at pull thread), or buffer ck(not stored and ack finish)\n            if (pointWrapper.isJustOffset() && pointWrapper.isCkStored() || isCkDone(pointWrapper)\n                || isCkDoneForFinish(pointWrapper) && pointWrapper.isCkStored()) {\n                if (brokerController.getBrokerConfig().isEnablePopLog()) {\n                    POP_LOGGER.info(\"[PopBuffer]ck done, {}\", pointWrapper);\n                }\n                iterator.remove();\n                counter.decrementAndGet();\n                continue;\n            }\n\n            PopCheckPoint point = pointWrapper.getCk();\n            long now = System.currentTimeMillis();\n\n            boolean removeCk = !this.serving;\n            // ck will be timeout\n            if (point.getReviveTime() - now < brokerController.getBrokerConfig().getPopCkStayBufferTimeOut()) {\n                removeCk = true;\n            }\n\n            // the time stayed is too long\n            if (now - point.getPopTime() > brokerController.getBrokerConfig().getPopCkStayBufferTime()) {\n                removeCk = true;\n            }\n\n            if (now - point.getPopTime() > brokerController.getBrokerConfig().getPopCkStayBufferTime() * 2L) {\n                POP_LOGGER.warn(\"[PopBuffer]ck finish fail, stay too long, {}\", pointWrapper);\n            }\n\n            // double check\n            if (isCkDone(pointWrapper)) {\n                continue;\n            } else if (pointWrapper.isJustOffset()) {\n                // just offset should be in store.\n                if (pointWrapper.getReviveQueueOffset() < 0) {\n                    putCkToStore(pointWrapper, this.brokerController.getBrokerConfig().isAppendCkAsync());\n                    countCk++;\n                }\n                continue;\n            } else if (removeCk) {\n                // put buffer ak to store\n                if (pointWrapper.getReviveQueueOffset() < 0) {\n                    putCkToStore(pointWrapper, this.brokerController.getBrokerConfig().isAppendCkAsync());\n                    countCk++;\n                }\n\n                if (!pointWrapper.isCkStored()) {\n                    continue;\n                }\n\n                if (brokerController.getBrokerConfig().isEnablePopBatchAck()) {\n                    List<Byte> indexList = this.batchAckIndexList;\n                    try {\n                        for (byte i = 0; i < point.getNum(); i++) {\n                            // reput buffer ak to store\n                            if (DataConverter.getBit(pointWrapper.getBits().get(), i)\n                                && !DataConverter.getBit(pointWrapper.getToStoreBits().get(), i)) {\n                                indexList.add(i);\n                            }\n                        }\n                        if (indexList.size() > 0) {\n                            putBatchAckToStore(pointWrapper, indexList, count);\n                        }\n                    } finally {\n                        indexList.clear();\n                    }\n                } else {\n                    for (byte i = 0; i < point.getNum(); i++) {\n                        // reput buffer ak to store\n                        if (DataConverter.getBit(pointWrapper.getBits().get(), i)\n                            && !DataConverter.getBit(pointWrapper.getToStoreBits().get(), i)) {\n                            putAckToStore(pointWrapper, i, count);\n                        }\n                    }\n                }\n\n                if (isCkDoneForFinish(pointWrapper) && pointWrapper.isCkStored()) {\n                    if (brokerController.getBrokerConfig().isEnablePopLog()) {\n                        POP_LOGGER.info(\"[PopBuffer]ck finish, {}\", pointWrapper);\n                    }\n                    iterator.remove();\n                    counter.decrementAndGet();\n                }\n            }\n        }\n\n        int offsetBufferSize = scanCommitOffset();\n\n        long eclipse = System.currentTimeMillis() - startTime;\n        if (eclipse > brokerController.getBrokerConfig().getPopCkStayBufferTimeOut() - 1000) {\n            POP_LOGGER.warn(\"[PopBuffer]scan stop, because eclipse too long, PopBufferEclipse={}, \" +\n                    \"PopBufferToStoreAck={}, PopBufferToStoreCk={}, PopBufferSize={}, PopBufferOffsetSize={}\",\n                eclipse, count.get(), countCk, counter.get(), offsetBufferSize);\n            this.serving = false;\n        } else {\n            if (scanTimes % countOfSecond1 == 0) {\n                POP_LOGGER.info(\"[PopBuffer]scan, PopBufferEclipse={}, \" +\n                        \"PopBufferToStoreAck={}, PopBufferToStoreCk={}, PopBufferSize={}, PopBufferOffsetSize={}\",\n                    eclipse, count.get(), countCk, counter.get(), offsetBufferSize);\n            }\n        }\n        brokerController.getBrokerMetricsManager().getPopMetricsManager().recordPopBufferScanTimeConsume(eclipse);\n        scanTimes++;\n\n        if (scanTimes >= countOfMinute1) {\n            counter.set(this.buffer.size());\n            scanTimes = 0;\n        }\n    }\n\n    public int getOffsetTotalSize() {\n        int count = 0;\n        Iterator<Map.Entry<String, QueueWithTime<PopCheckPointWrapper>>> iterator = this.commitOffsets.entrySet().iterator();\n        while (iterator.hasNext()) {\n            Map.Entry<String, QueueWithTime<PopCheckPointWrapper>> entry = iterator.next();\n            LinkedBlockingDeque<PopCheckPointWrapper> queue = entry.getValue().get();\n            count += queue.size();\n        }\n        return count;\n    }\n\n    public int getBufferedCKSize() {\n        return this.counter.get();\n    }\n\n    private void markBitCAS(AtomicInteger setBits, int index) {\n        while (true) {\n            int bits = setBits.get();\n            if (DataConverter.getBit(bits, index)) {\n                break;\n            }\n\n            int newBits = DataConverter.setBit(bits, index, true);\n            if (setBits.compareAndSet(bits, newBits)) {\n                break;\n            }\n        }\n    }\n\n    private boolean commitOffset(final PopCheckPointWrapper wrapper) {\n        if (wrapper.getNextBeginOffset() < 0) {\n            return true;\n        }\n\n        final PopCheckPoint popCheckPoint = wrapper.getCk();\n        final String lockKey = wrapper.getLockKey();\n\n        if (!queueLockManager.tryLock(lockKey)) {\n            return false;\n        }\n        try {\n            final long offset = brokerController.getConsumerOffsetManager().queryOffset(popCheckPoint.getCId(), popCheckPoint.getTopic(), popCheckPoint.getQueueId());\n            if (wrapper.getNextBeginOffset() > offset) {\n                if (brokerController.getBrokerConfig().isEnablePopLog()) {\n                    POP_LOGGER.info(\"Commit offset, {}, {}\", wrapper, offset);\n                }\n            } else {\n                // maybe store offset is not correct.\n                POP_LOGGER.warn(\"Commit offset, consumer offset less than store, {}, {}\", wrapper, offset);\n            }\n            brokerController.getConsumerOffsetManager().commitOffset(getServiceName(),\n                popCheckPoint.getCId(), popCheckPoint.getTopic(), popCheckPoint.getQueueId(), wrapper.getNextBeginOffset());\n        } finally {\n            queueLockManager.unLock(lockKey);\n        }\n        return true;\n    }\n\n    private boolean putOffsetQueue(PopCheckPointWrapper pointWrapper) {\n        QueueWithTime<PopCheckPointWrapper> queue = this.commitOffsets.get(pointWrapper.getLockKey());\n        if (queue == null) {\n            queue = new QueueWithTime<>();\n            QueueWithTime old = this.commitOffsets.putIfAbsent(pointWrapper.getLockKey(), queue);\n            if (old != null) {\n                queue = old;\n            }\n        }\n        queue.setTime(pointWrapper.getCk().getPopTime());\n        return queue.get().offer(pointWrapper);\n    }\n\n    private boolean checkQueueOk(PopCheckPointWrapper pointWrapper) {\n        QueueWithTime<PopCheckPointWrapper> queue = this.commitOffsets.get(pointWrapper.getLockKey());\n        if (queue == null) {\n            return true;\n        }\n        return queue.get().size() < brokerController.getBrokerConfig().getPopCkOffsetMaxQueueSize();\n    }\n\n    /**\n     * put to store && add to buffer.\n     *\n     * @param point\n     * @param reviveQueueId\n     * @param reviveQueueOffset\n     * @param nextBeginOffset\n     * @return\n     */\n    public boolean addCkJustOffset(PopCheckPoint point, int reviveQueueId, long reviveQueueOffset,\n        long nextBeginOffset) {\n        PopCheckPointWrapper pointWrapper = new PopCheckPointWrapper(reviveQueueId, reviveQueueOffset, point, nextBeginOffset, true);\n\n        if (this.buffer.containsKey(pointWrapper.getMergeKey())) {\n            // when mergeKey conflict\n            // will cause PopBufferMergeService.scanCommitOffset cannot poll PopCheckPointWrapper\n            POP_LOGGER.warn(\"[PopBuffer]mergeKey conflict when add ckJustOffset. ck:{}, mergeKey:{}\", pointWrapper, pointWrapper.getMergeKey());\n            return false;\n        }\n\n        this.putCkToStore(pointWrapper, checkQueueOk(pointWrapper));\n\n        putOffsetQueue(pointWrapper);\n        this.buffer.put(pointWrapper.getMergeKey(), pointWrapper);\n        this.counter.incrementAndGet();\n        if (brokerController.getBrokerConfig().isEnablePopLog()) {\n            POP_LOGGER.info(\"[PopBuffer]add ck just offset, {}\", pointWrapper);\n        }\n        return true;\n    }\n\n    public void addCkMock(String group, String topic, int queueId, long startOffset, long invisibleTime,\n        long popTime, int reviveQueueId, long nextBeginOffset, String brokerName) {\n        final PopCheckPoint ck = new PopCheckPoint();\n        ck.setBitMap(0);\n        ck.setNum((byte) 0);\n        ck.setPopTime(popTime);\n        ck.setInvisibleTime(invisibleTime);\n        ck.setStartOffset(startOffset);\n        ck.setCId(group);\n        ck.setTopic(topic);\n        ck.setQueueId(queueId);\n        ck.setBrokerName(brokerName);\n\n        PopCheckPointWrapper pointWrapper = new PopCheckPointWrapper(reviveQueueId, Long.MAX_VALUE, ck, nextBeginOffset, true);\n        pointWrapper.setCkStored(true);\n\n        putOffsetQueue(pointWrapper);\n        if (brokerController.getBrokerConfig().isEnablePopLog()) {\n            POP_LOGGER.info(\"[PopBuffer]add ck just offset, mocked, {}\", pointWrapper);\n        }\n    }\n\n    public boolean addCk(PopCheckPoint point, int reviveQueueId, long reviveQueueOffset, long nextBeginOffset) {\n        // key: point.getT() + point.getC() + point.getQ() + point.getSo() + point.getPt()\n        if (!brokerController.getBrokerConfig().isEnablePopBufferMerge()) {\n            return false;\n        }\n        if (!serving) {\n            return false;\n        }\n\n        long now = System.currentTimeMillis();\n        if (point.getReviveTime() - now < brokerController.getBrokerConfig().getPopCkStayBufferTimeOut() + 1500) {\n            if (brokerController.getBrokerConfig().isEnablePopLog()) {\n                POP_LOGGER.warn(\"[PopBuffer]add ck, timeout, {}, {}\", point, now);\n            }\n            return false;\n        }\n\n        if (this.counter.get() > brokerController.getBrokerConfig().getPopCkMaxBufferSize()) {\n            POP_LOGGER.warn(\"[PopBuffer]add ck, max size, {}, {}\", point, this.counter.get());\n            return false;\n        }\n\n        PopCheckPointWrapper pointWrapper = new PopCheckPointWrapper(reviveQueueId, reviveQueueOffset, point, nextBeginOffset);\n\n        if (!checkQueueOk(pointWrapper)) {\n            return false;\n        }\n\n        if (this.buffer.containsKey(pointWrapper.getMergeKey())) {\n            // when mergeKey conflict\n            // will cause PopBufferMergeService.scanCommitOffset cannot poll PopCheckPointWrapper\n            POP_LOGGER.warn(\"[PopBuffer]mergeKey conflict when add ck. ck:{}, mergeKey:{}\", pointWrapper, pointWrapper.getMergeKey());\n            return false;\n        }\n\n        putOffsetQueue(pointWrapper);\n        this.buffer.put(pointWrapper.getMergeKey(), pointWrapper);\n        this.counter.incrementAndGet();\n        if (brokerController.getBrokerConfig().isEnablePopLog()) {\n            POP_LOGGER.info(\"[PopBuffer]add ck, {}\", pointWrapper);\n        }\n        return true;\n    }\n\n    public boolean addAk(int reviveQid, AckMsg ackMsg) {\n        if (!brokerController.getBrokerConfig().isEnablePopBufferMerge()) {\n            return false;\n        }\n        if (!serving) {\n            return false;\n        }\n        try {\n            PopCheckPointWrapper pointWrapper = this.buffer.get(ackMsg.getTopic() + ackMsg.getConsumerGroup() + ackMsg.getQueueId() + ackMsg.getStartOffset() + ackMsg.getPopTime() + ackMsg.getBrokerName());\n            if (pointWrapper == null) {\n                if (brokerController.getBrokerConfig().isEnablePopLog()) {\n                    POP_LOGGER.warn(\"[PopBuffer]add ack fail, rqId={}, no ck, {}\", reviveQid, ackMsg);\n                }\n                return false;\n            }\n\n            if (pointWrapper.isJustOffset()) {\n                return false;\n            }\n\n            PopCheckPoint point = pointWrapper.getCk();\n            long now = System.currentTimeMillis();\n\n            if (point.getReviveTime() - now < brokerController.getBrokerConfig().getPopCkStayBufferTimeOut() + 1500) {\n                if (brokerController.getBrokerConfig().isEnablePopLog()) {\n                    POP_LOGGER.warn(\"[PopBuffer]add ack fail, rqId={}, almost timeout for revive, {}, {}, {}\", reviveQid, pointWrapper, ackMsg, now);\n                }\n                return false;\n            }\n\n            if (now - point.getPopTime() > brokerController.getBrokerConfig().getPopCkStayBufferTime() - 1500) {\n                if (brokerController.getBrokerConfig().isEnablePopLog()) {\n                    POP_LOGGER.warn(\"[PopBuffer]add ack fail, rqId={}, stay too long, {}, {}, {}\", reviveQid, pointWrapper, ackMsg, now);\n                }\n                return false;\n            }\n\n            if (ackMsg instanceof BatchAckMsg) {\n                for (Long ackOffset : ((BatchAckMsg) ackMsg).getAckOffsetList()) {\n                    int indexOfAck = point.indexOfAck(ackOffset);\n                    if (indexOfAck > -1) {\n                        markBitCAS(pointWrapper.getBits(), indexOfAck);\n                    } else {\n                        POP_LOGGER.error(\"[PopBuffer]Invalid index of ack, reviveQid={}, {}, {}\", reviveQid, ackMsg, point);\n                    }\n                }\n            } else {\n                int indexOfAck = point.indexOfAck(ackMsg.getAckOffset());\n                if (indexOfAck > -1) {\n                    markBitCAS(pointWrapper.getBits(), indexOfAck);\n                } else {\n                    POP_LOGGER.error(\"[PopBuffer]Invalid index of ack, reviveQid={}, {}, {}\", reviveQid, ackMsg, point);\n                    return true;\n                }\n            }\n\n            if (brokerController.getBrokerConfig().isEnablePopLog()) {\n                POP_LOGGER.info(\"[PopBuffer]add ack, rqId={}, {}, {}\", reviveQid, pointWrapper, ackMsg);\n            }\n\n//            // check ak done\n//            if (isCkDone(pointWrapper)) {\n//                // cancel ck for timer\n//                cancelCkTimer(pointWrapper);\n//            }\n            return true;\n        } catch (Throwable e) {\n            POP_LOGGER.error(\"[PopBuffer]add ack error, rqId=\" + reviveQid + \", \" + ackMsg, e);\n        }\n\n        return false;\n    }\n\n    public void clearOffsetQueue(String lockKey) {\n        this.commitOffsets.remove(lockKey);\n    }\n\n    private void putCkToStore(final PopCheckPointWrapper pointWrapper, final boolean runInCurrent) {\n        if (pointWrapper.getReviveQueueOffset() >= 0) {\n            return;\n        }\n\n        MessageExtBrokerInner msgInner = popMessageProcessor.buildCkMsg(pointWrapper.getCk(), pointWrapper.getReviveQueueId());\n\n        // Indicates that ck message is storing\n        pointWrapper.setReviveQueueOffset(Long.MAX_VALUE);\n        if (brokerController.getBrokerConfig().isAppendCkAsync() && runInCurrent) {\n            brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenAccept(putMessageResult -> {\n                handleCkMessagePutResult(putMessageResult, pointWrapper);\n            }).exceptionally(throwable -> {\n                POP_LOGGER.error(\"[PopBuffer]put ck to store fail: {}\", pointWrapper, throwable);\n                pointWrapper.setReviveQueueOffset(-1);\n                return null;\n            });\n        } else {\n            PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);\n            handleCkMessagePutResult(putMessageResult, pointWrapper);\n        }\n    }\n\n    private void handleCkMessagePutResult(PutMessageResult putMessageResult, final PopCheckPointWrapper pointWrapper) {\n        brokerController.getBrokerMetricsManager().getPopMetricsManager().incPopReviveCkPutCount(pointWrapper.getCk(), putMessageResult.getPutMessageStatus());\n        if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK\n            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT\n            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT\n            && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {\n            pointWrapper.setReviveQueueOffset(-1);\n            POP_LOGGER.error(\"[PopBuffer]put ck to store fail: {}, {}\", pointWrapper, putMessageResult);\n            return;\n        }\n        pointWrapper.setCkStored(true);\n\n        if (putMessageResult.isRemotePut()) {\n            //No AppendMessageResult when escaping remotely\n            pointWrapper.setReviveQueueOffset(0);\n        } else {\n            pointWrapper.setReviveQueueOffset(putMessageResult.getAppendMessageResult().getLogicsOffset());\n        }\n\n        if (brokerController.getBrokerConfig().isEnablePopLog()) {\n            POP_LOGGER.info(\"[PopBuffer]put ck to store ok: {}, {}\", pointWrapper, putMessageResult);\n        }\n    }\n\n    private void putAckToStore(final PopCheckPointWrapper pointWrapper, byte msgIndex, AtomicInteger count) {\n        PopCheckPoint point = pointWrapper.getCk();\n        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n        final AckMsg ackMsg = new AckMsg();\n\n        ackMsg.setAckOffset(point.ackOffsetByIndex(msgIndex));\n        ackMsg.setStartOffset(point.getStartOffset());\n        ackMsg.setConsumerGroup(point.getCId());\n        ackMsg.setTopic(point.getTopic());\n        ackMsg.setQueueId(point.getQueueId());\n        ackMsg.setPopTime(point.getPopTime());\n        ackMsg.setBrokerName(point.getBrokerName());\n        msgInner.setTopic(popMessageProcessor.getReviveTopic());\n        msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8));\n        msgInner.setQueueId(pointWrapper.getReviveQueueId());\n        msgInner.setTags(PopAckConstants.ACK_TAG);\n        msgInner.setBornTimestamp(System.currentTimeMillis());\n        msgInner.setBornHost(brokerController.getStoreHost());\n        msgInner.setStoreHost(brokerController.getStoreHost());\n        msgInner.setDeliverTimeMs(point.getReviveTime());\n        msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genAckUniqueId(ackMsg));\n\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));\n\n        if (brokerController.getBrokerConfig().isAppendAckAsync()) {\n            brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenAccept(putMessageResult -> {\n                handleAckPutMessageResult(ackMsg, putMessageResult, pointWrapper, count, msgIndex);\n            }).exceptionally(throwable -> {\n                POP_LOGGER.error(\"[PopBuffer]put ack to store fail: {}, {}\", pointWrapper, ackMsg, throwable);\n                return null;\n            });\n        } else {\n            PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);\n            handleAckPutMessageResult(ackMsg, putMessageResult, pointWrapper, count, msgIndex);\n        }\n    }\n\n    private void handleAckPutMessageResult(AckMsg ackMsg, PutMessageResult putMessageResult,\n        PopCheckPointWrapper pointWrapper, AtomicInteger count, byte msgIndex) {\n        brokerController.getBrokerMetricsManager().getPopMetricsManager().incPopReviveAckPutCount(ackMsg, putMessageResult.getPutMessageStatus());\n        if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK\n            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT\n            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT\n            && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {\n            POP_LOGGER.error(\"[PopBuffer]put ack to store fail: {}, {}, {}\", pointWrapper, ackMsg, putMessageResult);\n            return;\n        }\n        if (brokerController.getBrokerConfig().isEnablePopLog()) {\n            POP_LOGGER.info(\"[PopBuffer]put ack to store ok: {}, {}, {}\", pointWrapper, ackMsg, putMessageResult);\n        }\n        count.incrementAndGet();\n        markBitCAS(pointWrapper.getToStoreBits(), msgIndex);\n    }\n\n    private void putBatchAckToStore(final PopCheckPointWrapper pointWrapper, final List<Byte> msgIndexList,\n        AtomicInteger count) {\n        PopCheckPoint point = pointWrapper.getCk();\n        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n        final BatchAckMsg batchAckMsg = new BatchAckMsg();\n\n        for (Byte msgIndex : msgIndexList) {\n            batchAckMsg.getAckOffsetList().add(point.ackOffsetByIndex(msgIndex));\n        }\n        batchAckMsg.setStartOffset(point.getStartOffset());\n        batchAckMsg.setConsumerGroup(point.getCId());\n        batchAckMsg.setTopic(point.getTopic());\n        batchAckMsg.setQueueId(point.getQueueId());\n        batchAckMsg.setPopTime(point.getPopTime());\n        msgInner.setTopic(popMessageProcessor.getReviveTopic());\n        msgInner.setBody(JSON.toJSONString(batchAckMsg).getBytes(DataConverter.CHARSET_UTF8));\n        msgInner.setQueueId(pointWrapper.getReviveQueueId());\n        msgInner.setTags(PopAckConstants.BATCH_ACK_TAG);\n        msgInner.setBornTimestamp(System.currentTimeMillis());\n        msgInner.setBornHost(brokerController.getStoreHost());\n        msgInner.setStoreHost(brokerController.getStoreHost());\n        msgInner.setDeliverTimeMs(point.getReviveTime());\n        msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genBatchAckUniqueId(batchAckMsg));\n\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));\n        if (brokerController.getBrokerConfig().isAppendAckAsync()) {\n            brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenAccept(putMessageResult -> {\n                handleBatchAckPutMessageResult(batchAckMsg, putMessageResult, pointWrapper, count, msgIndexList);\n            }).exceptionally(throwable -> {\n                POP_LOGGER.error(\"[PopBuffer]put batchAckMsg to store fail: {}, {}\", pointWrapper, batchAckMsg, throwable);\n                return null;\n            });\n        } else {\n            PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);\n            handleBatchAckPutMessageResult(batchAckMsg, putMessageResult, pointWrapper, count, msgIndexList);\n        }\n    }\n\n    private void handleBatchAckPutMessageResult(BatchAckMsg batchAckMsg, PutMessageResult putMessageResult,\n        PopCheckPointWrapper pointWrapper, AtomicInteger count, List<Byte> msgIndexList) {\n        if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK\n            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT\n            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT\n            && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {\n            POP_LOGGER.error(\"[PopBuffer]put batch ack to store fail: {}, {}, {}\", pointWrapper, batchAckMsg, putMessageResult);\n            return;\n        }\n        if (brokerController.getBrokerConfig().isEnablePopLog()) {\n            POP_LOGGER.info(\"[PopBuffer]put batch ack to store ok: {}, {}, {}\", pointWrapper, batchAckMsg, putMessageResult);\n        }\n\n        count.addAndGet(msgIndexList.size());\n        for (Byte i : msgIndexList) {\n            markBitCAS(pointWrapper.getToStoreBits(), i);\n        }\n    }\n\n    private boolean cancelCkTimer(final PopCheckPointWrapper pointWrapper) {\n        // not stored, no need cancel\n        if (pointWrapper.getReviveQueueOffset() < 0) {\n            return true;\n        }\n        PopCheckPoint point = pointWrapper.getCk();\n        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n        msgInner.setTopic(popMessageProcessor.getReviveTopic());\n        msgInner.setBody((pointWrapper.getReviveQueueId() + \"-\" + pointWrapper.getReviveQueueOffset()).getBytes(StandardCharsets.UTF_8));\n        msgInner.setQueueId(pointWrapper.getReviveQueueId());\n        msgInner.setTags(PopAckConstants.CK_TAG);\n        msgInner.setBornTimestamp(System.currentTimeMillis());\n        msgInner.setBornHost(brokerController.getStoreHost());\n        msgInner.setStoreHost(brokerController.getStoreHost());\n\n        msgInner.setDeliverTimeMs(point.getReviveTime() - PopAckConstants.ackTimeInterval);\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));\n        PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);\n        if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK\n            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT\n            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT\n            && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {\n            POP_LOGGER.error(\"[PopBuffer]PutMessageCallback cancelCheckPoint fail, {}, {}\", pointWrapper, putMessageResult);\n            return false;\n        }\n        if (brokerController.getBrokerConfig().isEnablePopLog()) {\n            POP_LOGGER.info(\"[PopBuffer]cancelCheckPoint, {}\", pointWrapper);\n        }\n        return true;\n    }\n\n    private boolean isCkDone(PopCheckPointWrapper pointWrapper) {\n        byte num = pointWrapper.getCk().getNum();\n        for (byte i = 0; i < num; i++) {\n            if (!DataConverter.getBit(pointWrapper.getBits().get(), i)) {\n                return false;\n            }\n        }\n        return true;\n    }\n\n    private boolean isCkDoneForFinish(PopCheckPointWrapper pointWrapper) {\n        byte num = pointWrapper.getCk().getNum();\n        int bits = pointWrapper.getBits().get() ^ pointWrapper.getToStoreBits().get();\n        for (byte i = 0; i < num; i++) {\n            if (DataConverter.getBit(bits, i)) {\n                return false;\n            }\n        }\n        return true;\n    }\n\n    public class QueueWithTime<T> {\n        private final LinkedBlockingDeque<T> queue;\n        private long time;\n\n        public QueueWithTime() {\n            this.queue = new LinkedBlockingDeque<>();\n            this.time = System.currentTimeMillis();\n        }\n\n        public void setTime(long popTime) {\n            this.time = popTime;\n        }\n\n        public long getTime() {\n            return time;\n        }\n\n        public LinkedBlockingDeque<T> get() {\n            return queue;\n        }\n    }\n\n    public class PopCheckPointWrapper {\n        private final int reviveQueueId;\n        // -1: not stored, >=0: stored, Long.MAX: storing.\n        private volatile long reviveQueueOffset;\n        private final PopCheckPoint ck;\n        // bit for concurrent\n        private final AtomicInteger bits;\n        // bit for stored buffer ak\n        private final AtomicInteger toStoreBits;\n        private final long nextBeginOffset;\n        private final String lockKey;\n        private final String mergeKey;\n        private final boolean justOffset;\n        private volatile boolean ckStored = false;\n\n        public PopCheckPointWrapper(int reviveQueueId, long reviveQueueOffset, PopCheckPoint point,\n            long nextBeginOffset) {\n            this.reviveQueueId = reviveQueueId;\n            this.reviveQueueOffset = reviveQueueOffset;\n            this.ck = point;\n            this.bits = new AtomicInteger(0);\n            this.toStoreBits = new AtomicInteger(0);\n            this.nextBeginOffset = nextBeginOffset;\n            this.lockKey = ck.getTopic() + PopAckConstants.SPLIT + ck.getCId() + PopAckConstants.SPLIT + ck.getQueueId();\n            this.mergeKey = point.getTopic() + point.getCId() + point.getQueueId() + point.getStartOffset() + point.getPopTime() + point.getBrokerName();\n            this.justOffset = false;\n        }\n\n        public PopCheckPointWrapper(int reviveQueueId, long reviveQueueOffset, PopCheckPoint point,\n            long nextBeginOffset,\n            boolean justOffset) {\n            this.reviveQueueId = reviveQueueId;\n            this.reviveQueueOffset = reviveQueueOffset;\n            this.ck = point;\n            this.bits = new AtomicInteger(0);\n            this.toStoreBits = new AtomicInteger(0);\n            this.nextBeginOffset = nextBeginOffset;\n            this.lockKey = ck.getTopic() + PopAckConstants.SPLIT + ck.getCId() + PopAckConstants.SPLIT + ck.getQueueId();\n            this.mergeKey = point.getTopic() + point.getCId() + point.getQueueId() + point.getStartOffset() + point.getPopTime() + point.getBrokerName();\n            this.justOffset = justOffset;\n        }\n\n        public int getReviveQueueId() {\n            return reviveQueueId;\n        }\n\n        public long getReviveQueueOffset() {\n            return reviveQueueOffset;\n        }\n\n        public boolean isCkStored() {\n            return ckStored;\n        }\n\n        public void setReviveQueueOffset(long reviveQueueOffset) {\n            this.reviveQueueOffset = reviveQueueOffset;\n        }\n\n        public PopCheckPoint getCk() {\n            return ck;\n        }\n\n        public AtomicInteger getBits() {\n            return bits;\n        }\n\n        public AtomicInteger getToStoreBits() {\n            return toStoreBits;\n        }\n\n        public long getNextBeginOffset() {\n            return nextBeginOffset;\n        }\n\n        public String getLockKey() {\n            return lockKey;\n        }\n\n        public String getMergeKey() {\n            return mergeKey;\n        }\n\n        public boolean isJustOffset() {\n            return justOffset;\n        }\n\n        public void setCkStored(boolean ckStored) {\n            this.ckStored = ckStored;\n        }\n\n        @Override\n        public String toString() {\n            final StringBuilder sb = new StringBuilder(\"CkWrap{\");\n            sb.append(\"rq=\").append(reviveQueueId);\n            sb.append(\", rqo=\").append(reviveQueueOffset);\n            sb.append(\", ck=\").append(ck);\n            sb.append(\", bits=\").append(bits);\n            sb.append(\", sBits=\").append(toStoreBits);\n            sb.append(\", nbo=\").append(nextBeginOffset);\n            sb.append(\", cks=\").append(ckStored);\n            sb.append(\", jo=\").append(justOffset);\n            sb.append('}');\n            return sb.toString();\n        }\n    }\n\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/PopInflightMessageCounter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.pop.PopCheckPoint;\n\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.atomic.AtomicLong;\n\npublic class PopInflightMessageCounter {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n\n    private static final String TOPIC_GROUP_SEPARATOR = \"@\";\n    private final Map<String /* topic@group */, Map<Integer /* queueId */, AtomicLong>> topicInFlightMessageNum =\n        new ConcurrentHashMap<>(512);\n    private final BrokerController brokerController;\n\n    public PopInflightMessageCounter(BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    public void incrementInFlightMessageNum(String topic, String group, int queueId, int num) {\n        if (num <= 0) {\n            return;\n        }\n        topicInFlightMessageNum.compute(buildKey(topic, group), (key, queueNum) -> {\n            if (queueNum == null) {\n                queueNum = new ConcurrentHashMap<>(8);\n            }\n            queueNum.compute(queueId, (queueIdKey, counter) -> {\n                if (counter == null) {\n                    return new AtomicLong(num);\n                }\n                if (counter.addAndGet(num) <= 0) {\n                    return null;\n                }\n                return counter;\n            });\n            return queueNum;\n        });\n    }\n\n    public void decrementInFlightMessageNum(String topic, String group, long popTime, int qId, int delta) {\n        if (popTime < this.brokerController.getShouldStartTime()) {\n            return;\n        }\n        decrementInFlightMessageNum(topic, group, qId, delta);\n    }\n\n    public void decrementInFlightMessageNum(PopCheckPoint checkPoint) {\n        if (checkPoint.getPopTime() < this.brokerController.getShouldStartTime()) {\n            return;\n        }\n        decrementInFlightMessageNum(checkPoint.getTopic(), checkPoint.getCId(), checkPoint.getQueueId(), 1);\n    }\n\n    private void decrementInFlightMessageNum(String topic, String group, int queueId, int delta) {\n        topicInFlightMessageNum.computeIfPresent(buildKey(topic, group), (key, queueNum) -> {\n            queueNum.computeIfPresent(queueId, (queueIdKey, counter) -> {\n                if (counter.addAndGet(-delta) <= 0) {\n                    return null;\n                }\n                return counter;\n            });\n            if (queueNum.isEmpty()) {\n                return null;\n            }\n            return queueNum;\n        });\n    }\n\n    public void clearInFlightMessageNumByGroupName(String group) {\n        Set<String> topicGroupKey = this.topicInFlightMessageNum.keySet();\n        for (String key : topicGroupKey) {\n            if (key.contains(group)) {\n                Pair<String, String> topicAndGroup = splitKey(key);\n                if (topicAndGroup != null && topicAndGroup.getObject2().equals(group)) {\n                    this.topicInFlightMessageNum.remove(key);\n                    log.info(\"PopInflightMessageCounter#clearInFlightMessageNumByGroupName: clean by group, topic={}, group={}\",\n                        topicAndGroup.getObject1(), topicAndGroup.getObject2());\n                }\n            }\n        }\n    }\n\n    public void clearInFlightMessageNumByTopicName(String topic) {\n        Set<String> topicGroupKey = this.topicInFlightMessageNum.keySet();\n        for (String key : topicGroupKey) {\n            if (key.contains(topic)) {\n                Pair<String, String> topicAndGroup = splitKey(key);\n                if (topicAndGroup != null && topicAndGroup.getObject1().equals(topic)) {\n                    this.topicInFlightMessageNum.remove(key);\n                    log.info(\"PopInflightMessageCounter#clearInFlightMessageNumByTopicName: clean by topic, topic={}, group={}\",\n                        topicAndGroup.getObject1(), topicAndGroup.getObject2());\n                }\n            }\n        }\n    }\n\n    public void clearInFlightMessageNum(String topic, String group, int queueId) {\n        topicInFlightMessageNum.computeIfPresent(buildKey(topic, group), (key, queueNum) -> {\n            queueNum.computeIfPresent(queueId, (queueIdKey, counter) -> null);\n            if (queueNum.isEmpty()) {\n                return null;\n            }\n            return queueNum;\n        });\n    }\n\n    public long getGroupPopInFlightMessageNum(String topic, String group, int queueId) {\n        Map<Integer /* queueId */, AtomicLong> queueCounter = topicInFlightMessageNum.get(buildKey(topic, group));\n        if (queueCounter == null) {\n            return 0;\n        }\n        AtomicLong counter = queueCounter.get(queueId);\n        if (counter == null) {\n            return 0;\n        }\n        return Math.max(0, counter.get());\n    }\n\n    private static Pair<String /* topic */, String /* group */> splitKey(String key) {\n        String[] strings = key.split(TOPIC_GROUP_SEPARATOR);\n        if (strings.length != 2) {\n            return null;\n        }\n        return new Pair<>(strings[0], strings[1]);\n    }\n\n    private static String buildKey(String topic, String group) {\n        return topic + TOPIC_GROUP_SEPARATOR + group;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/PopLiteMessageProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport com.google.common.annotations.VisibleForTesting;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.opentelemetry.api.common.Attributes;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.lite.LiteEventDispatcher;\nimport org.apache.rocketmq.broker.longpolling.PollingResult;\nimport org.apache.rocketmq.broker.longpolling.PopLiteLongPollingService;\nimport org.apache.rocketmq.broker.metrics.LiteConsumerLagCalculator;\nimport org.apache.rocketmq.broker.offset.MemoryConsumerOrderInfoManager;\nimport org.apache.rocketmq.broker.pop.PopConsumerLockService;\nimport org.apache.rocketmq.broker.pop.orderly.ConsumerOrderInfoManager;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.ConsumeInitMode;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.help.FAQUrl;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingAbstract;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.PopLiteMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopLiteMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\n\nimport java.nio.ByteBuffer;\nimport java.util.Collections;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.Set;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\n\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_RETRY;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_TOPIC;\n\n/**\n * Pop lite implementation, support FIFO consuming.\n * This processor uses independent in-memory consumer order info and lock service,\n * along with a specialized long polling service.\n */\npublic class PopLiteMessageProcessor implements NettyRequestProcessor {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LITE_LOGGER_NAME);\n    private static final String BORN_TIME = \"bornTime\";\n\n    private final BrokerController brokerController;\n    private final PopLiteLongPollingService popLiteLongPollingService;\n    private final PopConsumerLockService lockService;\n    private final LiteEventDispatcher liteEventDispatcher;\n    private final ConsumerOrderInfoManager consumerOrderInfoManager;\n    private final PopLiteLockManager popLiteLockManager;\n\n    public PopLiteMessageProcessor(final BrokerController brokerController, LiteEventDispatcher liteEventDispatcher) {\n        this.brokerController = brokerController;\n        this.popLiteLongPollingService = new PopLiteLongPollingService(brokerController, this, false);\n        this.lockService = new PopConsumerLockService(TimeUnit.MINUTES.toMillis(1));\n        this.liteEventDispatcher = liteEventDispatcher;\n        this.consumerOrderInfoManager = new MemoryConsumerOrderInfoManager(brokerController);\n        this.popLiteLockManager = new PopLiteLockManager();\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n\n    @Override\n    public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingCommand request)\n        throws RemotingCommandException {\n\n        final long beginTimeMills = brokerController.getMessageStore().now();\n        Channel channel = ctx.channel();\n        request.addExtFieldIfNotExist(BORN_TIME, String.valueOf(System.currentTimeMillis()));\n        if (Objects.equals(request.getExtFields().get(BORN_TIME), \"0\")) {\n            request.addExtField(BORN_TIME, String.valueOf(System.currentTimeMillis()));\n        }\n        RemotingCommand response = RemotingCommand.createResponseCommand(PopLiteMessageResponseHeader.class);\n        response.setOpaque(request.getOpaque());\n\n        final PopLiteMessageRequestHeader requestHeader =\n            request.decodeCommandCustomHeader(PopLiteMessageRequestHeader.class, true);\n        final PopLiteMessageResponseHeader responseHeader = (PopLiteMessageResponseHeader) response.readCustomHeader();\n        RemotingCommand preCheckResponse = preCheck(ctx, requestHeader, response);\n        if (preCheckResponse != null) {\n            return preCheckResponse;\n        }\n\n        String clientId = requestHeader.getClientId();\n        String group = requestHeader.getConsumerGroup();\n        String parentTopic = requestHeader.getTopic();\n        int maxNum = requestHeader.getMaxMsgNum();\n        long popTime = System.currentTimeMillis();\n        long invisibleTime = requestHeader.getInvisibleTime();\n\n        Pair<StringBuilder, GetMessageResult> rst = popByClientId(channel.remoteAddress().toString(), parentTopic,\n            group, clientId, popTime, invisibleTime, maxNum, requestHeader.getAttemptId());\n\n        final GetMessageResult getMessageResult = rst.getObject2();\n        if (getMessageResult != null && getMessageResult.getMessageCount() > 0) {\n            final byte[] r = readGetMessageResult(getMessageResult);\n            brokerController.getBrokerStatsManager().incGroupGetLatency(group, parentTopic, 0,\n                (int) (brokerController.getMessageStore().now() - beginTimeMills));\n            brokerController.getBrokerStatsManager().incBrokerGetNums(parentTopic, getMessageResult.getMessageCount());\n            brokerController.getBrokerStatsManager().incGroupGetNums(group, parentTopic, getMessageResult.getMessageCount());\n            brokerController.getBrokerStatsManager().incGroupGetSize(group, parentTopic, getMessageResult.getBufferTotalSize());\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(GetMessageStatus.FOUND.name());\n            response.setBody(r);\n        } else {\n            response.setRemark(GetMessageStatus.NO_MESSAGE_IN_QUEUE.name());\n            PollingResult pollingResult = popLiteLongPollingService.polling(ctx, request, requestHeader.getBornTime(),\n                requestHeader.getPollTime(), clientId, group);\n            if (PollingResult.POLLING_SUC.equals(pollingResult)) {\n                return null;\n            } else if (PollingResult.POLLING_FULL.equals(pollingResult)) {\n                response.setCode(ResponseCode.POLLING_FULL);\n            } else {\n                response.setCode(ResponseCode.POLLING_TIMEOUT);\n            }\n        }\n\n        responseHeader.setPopTime(popTime);\n        responseHeader.setInvisibleTime(invisibleTime);\n        responseHeader.setReviveQid(KeyBuilder.POP_ORDER_REVIVE_QUEUE);\n        responseHeader.setOrderCountInfo(rst.getObject1().toString());\n        // Since a single read operation potentially retrieving messages from multiple LMQs,\n        // we no longer utilize startOffset and msgOffset\n        NettyRemotingAbstract.writeResponse(channel, request, response, null, brokerController.getBrokerMetricsManager().getRemotingMetricsManager());\n        return null;\n    }\n\n    @VisibleForTesting\n    public RemotingCommand preCheck(ChannelHandlerContext ctx,\n        PopLiteMessageRequestHeader requestHeader, RemotingCommand response) {\n        if (requestHeader.isTimeoutTooMuch()) {\n            response.setCode(ResponseCode.POLLING_TIMEOUT);\n            response.setRemark(String.format(\"the broker[%s] pop message is timeout too much\",\n                brokerController.getBrokerConfig().getBrokerIP1()));\n            return response;\n        }\n\n        if (!PermName.isReadable(brokerController.getBrokerConfig().getBrokerPermission())) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            response.setRemark(String.format(\"the broker[%s] pop message is forbidden\",\n                brokerController.getBrokerConfig().getBrokerIP1()));\n            return response;\n        }\n\n        if (requestHeader.getMaxMsgNum() > 32) {\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(String.format(\"the broker[%s] pop message's num is greater than 32\",\n                brokerController.getBrokerConfig().getBrokerIP1()));\n            return response;\n        }\n\n        TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());\n        if (null == topicConfig) {\n            LOGGER.error(\"The parentTopic {} not exist, consumer: {} \", requestHeader.getTopic());\n            response.setCode(ResponseCode.TOPIC_NOT_EXIST);\n            response.setRemark(String.format(\"topic [%s] not exist, apply first please! %s\", requestHeader.getTopic(),\n                FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)));\n            return response;\n        }\n\n        if (!PermName.isReadable(topicConfig.getPerm())) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            response.setRemark(String.format(\"the topic [%s] peeking message is forbidden\", requestHeader.getTopic()));\n            return response;\n        }\n\n        if (!TopicMessageType.LITE.equals(topicConfig.getTopicMessageType())) {\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(String.format(\"the topic [%s] message type not match\", requestHeader.getTopic()));\n            return response;\n        }\n\n        SubscriptionGroupConfig subscriptionGroupConfig =\n            brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getConsumerGroup());\n        if (null == subscriptionGroupConfig) {\n            response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);\n            response.setRemark(String.format(\"subscription group [%s] not exist, %s\",\n                requestHeader.getConsumerGroup(), FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST)));\n            return response;\n        }\n\n        if (!subscriptionGroupConfig.isConsumeEnable()) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            response.setRemark(\"subscription group no permission, \" + requestHeader.getConsumerGroup());\n            return response;\n        }\n\n        if (!requestHeader.getTopic().equals(subscriptionGroupConfig.getLiteBindTopic())) {\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(\"subscription bind topic not match, \" + requestHeader.getConsumerGroup());\n            return response;\n        }\n\n        return null;\n    }\n\n    private byte[] readGetMessageResult(GetMessageResult getMessageResult) {\n        final ByteBuffer byteBuffer = ByteBuffer.allocate(getMessageResult.getBufferTotalSize());\n        try {\n            List<ByteBuffer> messageBufferList = getMessageResult.getMessageBufferList();\n            for (ByteBuffer bb : messageBufferList) {\n                byteBuffer.put(bb);\n            }\n        } finally {\n            getMessageResult.release();\n        }\n        return byteBuffer.array();\n    }\n\n    public Pair<StringBuilder, GetMessageResult> popByClientId(String clientHost, String parentTopic, String group,\n        String clientId, long popTime, long invisibleTime, int maxNum, String attemptId) {\n        GetMessageResult getMessageResult = new GetMessageResult();\n        StringBuilder orderCountInfoAll = new StringBuilder();\n        AtomicLong total = new AtomicLong(0);\n\n        Set<String> processed = new HashSet<>(); // deduplication in one request\n        Iterator<String> iterator = liteEventDispatcher.getEventIterator(clientId);\n        while (total.get() < maxNum && iterator.hasNext()) {\n            String lmqName = iterator.next(); // here event represents a lmq name\n            if (null == lmqName) {\n                break;\n            }\n            if (!processed.add(lmqName)) {\n                continue; // wait for next pop request or re-fetch in current process, here prefer the former approach\n            }\n            Pair<StringBuilder, GetMessageResult> pair = popLiteTopic(parentTopic, clientHost, group, lmqName,\n                maxNum - total.get(), popTime, invisibleTime, attemptId);\n            if (null == pair || pair.getObject2().getMessageCount() <= 0) {\n                continue;\n            }\n            GetMessageResult singleResult = pair.getObject2();\n            total.addAndGet(singleResult.getMessageCount());\n            for (SelectMappedBufferResult mappedBuffer : singleResult.getMessageMapedList()) {\n                getMessageResult.addMessage(mappedBuffer);\n            }\n            if (orderCountInfoAll.length() > 0) {\n                orderCountInfoAll.append(\";\");\n            }\n            orderCountInfoAll.append(pair.getObject1());\n            collectLiteConsumerLagMetrics(group, parentTopic, lmqName, singleResult, maxNum, total);\n        }\n        return new Pair<>(orderCountInfoAll, getMessageResult);\n    }\n\n    @VisibleForTesting\n    public Pair<StringBuilder, GetMessageResult> popLiteTopic(String parentTopic, String clientHost, String group,\n        String lmqName, long maxNum, long popTime, long invisibleTime, String attemptId) {\n        if (!brokerController.getBrokerConfig().isEnableLiteEventMode()\n            && !brokerController.getLiteLifecycleManager().isLmqExist(lmqName)) {\n            return null;\n        }\n        String lockKey = KeyBuilder.buildPopLiteLockKey(group, lmqName);\n        if (!lockService.tryLock(lockKey)) {\n            return null;\n        }\n        try {\n            if (isFifoBlocked(attemptId, group, lmqName, invisibleTime)) {\n                return null;\n            }\n            final long consumeOffset = getPopOffset(group, lmqName);\n            GetMessageResult result = getMessage(clientHost, group, lmqName, consumeOffset, (int) maxNum);\n            return handleGetMessageResult(result, parentTopic, group, lmqName, popTime, invisibleTime, attemptId);\n        } catch (Throwable e) {\n            LOGGER.error(\"popLiteTopic error. {}, {}\", group, lmqName, e);\n        } finally {\n            lockService.unlock(lockKey);\n        }\n        return null;\n    }\n\n    public boolean isFifoBlocked(String attemptId, String group, String lmqName, long invisibleTime) {\n        return consumerOrderInfoManager.checkBlock(attemptId, lmqName, group, 0, invisibleTime);\n    }\n\n    public long getPopOffset(String group, String lmqName) {\n        long offset = brokerController.getConsumerOffsetManager().queryOffset(group, lmqName, 0);\n        if (offset < 0L) {\n            try {\n                offset = brokerController.getPopMessageProcessor().getInitOffset(lmqName, group, 0, ConsumeInitMode.MAX, true); // reuse code, init as max\n                LOGGER.info(\"init offset, group:{}, topic:{}, offset:{}\", group, lmqName, offset);\n            } catch (ConsumeQueueException e) {\n                throw new RuntimeException(e);\n            }\n        }\n        Long resetOffset = brokerController.getConsumerOffsetManager().queryThenEraseResetOffset(lmqName, group, 0);\n        if (resetOffset != null) {\n            consumerOrderInfoManager.clearBlock(lmqName, group, 0);\n            brokerController.getConsumerOffsetManager().commitOffset(\"ResetOffset\", group, lmqName, 0, resetOffset);\n            LOGGER.info(\"find resetOffset, group:{}, topic:{}, resetOffset:{}\", group, lmqName, resetOffset);\n            return resetOffset;\n        }\n        return offset;\n    }\n\n    public Pair<StringBuilder, GetMessageResult> handleGetMessageResult(GetMessageResult result, String parentTopic,\n        String group, String lmqName, long popTime, long invisibleTime, String attemptId) {\n        if (null == result) {\n            return null;\n        }\n\n        StringBuilder orderCountInfo = new StringBuilder();\n        if (GetMessageStatus.FOUND.equals(result.getStatus()) && !result.getMessageQueueOffset().isEmpty()) {\n            consumerOrderInfoManager.update(attemptId, false, lmqName, group, 0,\n                popTime, invisibleTime, result.getMessageQueueOffset(), orderCountInfo, null);\n            recordPopLiteMetrics(result, parentTopic, group);\n            orderCountInfo = transformOrderCountInfo(orderCountInfo, result.getMessageCount());\n        }\n        return new Pair<>(orderCountInfo, result);\n    }\n\n    /**\n     * For order count information, we use a uniform format of one consume count per offset.\n     */\n    @VisibleForTesting\n    public StringBuilder transformOrderCountInfo(StringBuilder orderCountInfo, int msgCount) {\n        if (null == orderCountInfo || orderCountInfo.length() <= 0) {\n            return new StringBuilder(String.join(\";\", Collections.nCopies(msgCount, \"0\")));\n        }\n        String infoStr = orderCountInfo.toString();\n        String[] infos = infoStr.split(\";\");\n        if (infos.length > 1) {\n            // consume count of each offset + \";\" + consume count of queueId\n            return new StringBuilder(infoStr.substring(0, infoStr.lastIndexOf(\";\")));\n        } else {\n            // just consume count of queueId, like \"0 0 N\"\n            String[] split = orderCountInfo.toString().split(MessageConst.KEY_SEPARATOR);\n            if (split.length == 3) {\n                return new StringBuilder(String.join(\";\", Collections.nCopies(msgCount, split[2])));\n            } else {\n                return new StringBuilder(String.join(\";\", Collections.nCopies(msgCount, \"0\")));\n            }\n        }\n    }\n\n    @VisibleForTesting\n    protected void recordPopLiteMetrics(GetMessageResult result, String parentTopic, String group) {\n        Attributes attributes = this.brokerController.getBrokerMetricsManager().newAttributesBuilder()\n            .put(LABEL_TOPIC, parentTopic)\n            .put(LABEL_CONSUMER_GROUP, group)\n            .put(LABEL_IS_SYSTEM, TopicValidator.isSystemTopic(parentTopic) ||\n                MixAll.isSysConsumerGroup(group))\n            .put(LABEL_IS_RETRY, false)\n            .build();\n        this.brokerController.getBrokerMetricsManager().getMessagesOutTotal().add(result.getMessageCount(), attributes);\n        this.brokerController.getBrokerMetricsManager().getThroughputOutTotal().add(result.getBufferTotalSize(), attributes);\n    }\n\n    private void collectLiteConsumerLagMetrics(String group, String topic, String liteTopic,\n        GetMessageResult getResult, long maxNum, AtomicLong total) {\n        if (!brokerController.getBrokerConfig().isLiteLagLatencyCollectEnable()) {\n            return;\n        }\n\n        try {\n            final LiteConsumerLagCalculator lagCalculator = brokerController.getBrokerMetricsManager()\n                .getLiteConsumerLagCalculator();\n\n            if (total.get() < maxNum) {\n                // Batch not full, no consume lag\n                lagCalculator.removeLagInfo(group, topic, liteTopic);\n                return;\n            }\n\n            // Batch full, check for potential consume lag\n            long storeTimestamp = brokerController.getMessageStore()\n                .getMessageStoreTimeStamp(liteTopic, 0, getResult.getNextBeginOffset());\n            if (storeTimestamp > 0) {\n                lagCalculator.updateLagInfo(group, topic, liteTopic, storeTimestamp);\n            } else {\n                // no next msg, no consume lag\n                lagCalculator.removeLagInfo(group, topic, liteTopic);\n            }\n        } catch (Exception e) {\n            LOGGER.warn(\"Failed to collect lite consumer lag metrics for group={}, topic={}, liteTopic={}\",\n                group, topic, liteTopic, e);\n        }\n    }\n\n    // tiered store ensures reading lmq from local storage\n    public GetMessageResult getMessage(String clientHost, String group, String lmqName, long offset, int batchSize) {\n        GetMessageResult result = brokerController.getMessageStore().getMessage(group, lmqName, 0, offset, batchSize, null);\n        if (null == result) {\n            return null;\n        }\n        if (GetMessageStatus.OFFSET_TOO_SMALL.equals(result.getStatus())\n            || GetMessageStatus.OFFSET_OVERFLOW_BADLY.equals(result.getStatus())\n            || GetMessageStatus.OFFSET_FOUND_NULL.equals(result.getStatus())\n            || GetMessageStatus.NO_MATCHED_MESSAGE.equals(result.getStatus())\n            || GetMessageStatus.MESSAGE_WAS_REMOVING.equals(result.getStatus())\n            || GetMessageStatus.NO_MATCHED_LOGIC_QUEUE.equals(result.getStatus())) {\n\n            long correctOffset = result.getNextBeginOffset(); // >=0\n            brokerController.getConsumerOffsetManager().commitOffset(\"CorrectOffset\", group, lmqName, 0, correctOffset);\n            LOGGER.warn(\"correct offset, {}, {}, from {} to {}\", group, lmqName, offset, correctOffset);\n            return brokerController.getMessageStore().getMessage(group, lmqName, 0, correctOffset, batchSize, null);\n        }\n        return result;\n    }\n\n    public class PopLiteLockManager extends ServiceThread {\n        @Override\n        public String getServiceName() {\n            if (brokerController.getBrokerConfig().isInBrokerContainer()) {\n                return brokerController.getBrokerIdentity().getIdentifier() + PopLiteLockManager.class.getSimpleName();\n            }\n            return PopLiteLockManager.class.getSimpleName();\n        }\n\n        @Override\n        public void run() {\n            while (!isStopped()) {\n                try {\n                    waitForRunning(60000);\n                    lockService.removeTimeout();\n                } catch (Exception ignored) {\n                }\n            }\n        }\n    }\n\n    public PopLiteLongPollingService getPopLiteLongPollingService() {\n        return popLiteLongPollingService;\n    }\n\n    public PopConsumerLockService getLockService() {\n        return lockService;\n    }\n\n    public ConsumerOrderInfoManager getConsumerOrderInfoManager() {\n        return consumerOrderInfoManager;\n    }\n\n    public void startPopLiteLockManager() {\n        popLiteLockManager.start();\n    }\n\n    public void stopPopLiteLockManager() {\n        popLiteLockManager.shutdown();\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.github.benmanes.caffeine.cache.Cache;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelFutureListener;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.channel.FileRegion;\nimport io.opentelemetry.api.common.Attributes;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.filter.ConsumerFilterData;\nimport org.apache.rocketmq.broker.filter.ConsumerFilterManager;\nimport org.apache.rocketmq.broker.filter.ExpressionMessageFilter;\nimport org.apache.rocketmq.broker.longpolling.PollingHeader;\nimport org.apache.rocketmq.broker.longpolling.PollingResult;\nimport org.apache.rocketmq.broker.longpolling.PopLongPollingService;\nimport org.apache.rocketmq.broker.longpolling.PopRequest;\nimport org.apache.rocketmq.broker.pagecache.ManyMessageTransfer;\nimport org.apache.rocketmq.broker.pop.PopConsumerContext;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.PopAckConstants;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.ConsumeInitMode;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.common.help.FAQUrl;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.common.utils.ConcurrentHashMapUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.CommandCallback;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.metrics.RemotingMetricsManager;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingAbstract;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.filter.FilterAPI;\nimport org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.pop.AckMsg;\nimport org.apache.rocketmq.store.pop.BatchAckMsg;\nimport org.apache.rocketmq.store.pop.PopCheckPoint;\n\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Map.Entry;\nimport java.util.Random;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentSkipListSet;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.concurrent.atomic.AtomicLong;\n\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_RETRY;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_TOPIC;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_REQUEST_CODE;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_RESPONSE_CODE;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_RESULT;\n\npublic class PopMessageProcessor implements NettyRequestProcessor {\n\n    private static final Logger POP_LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);\n    private static final String BORN_TIME = \"bornTime\";\n\n    private final BrokerController brokerController;\n    private final Random random = new Random(System.currentTimeMillis());\n    private final String reviveTopic;\n\n    private final PopLongPollingService popLongPollingService;\n    private final PopBufferMergeService popBufferMergeService;\n    private final QueueLockManager queueLockManager;\n    private final AtomicLong ckMessageNumber;\n\n    public PopMessageProcessor(final BrokerController brokerController) {\n        this.brokerController = brokerController;\n        this.reviveTopic = PopAckConstants.buildClusterReviveTopic(\n            this.brokerController.getBrokerConfig().getBrokerClusterName());\n        this.popLongPollingService = new PopLongPollingService(brokerController, this, false);\n        this.queueLockManager = new QueueLockManager();\n        this.popBufferMergeService = new PopBufferMergeService(this.brokerController, this);\n        this.ckMessageNumber = new AtomicLong();\n    }\n\n    public void shutdown() throws Exception {\n        popLongPollingService.shutdown();\n        queueLockManager.shutdown();\n        popBufferMergeService.shutdown();\n    }\n\n    protected String getReviveTopic() {\n        return reviveTopic;\n    }\n\n    public PopLongPollingService getPopLongPollingService() {\n        return popLongPollingService;\n    }\n\n    public PopBufferMergeService getPopBufferMergeService() {\n        return this.popBufferMergeService;\n    }\n\n    public QueueLockManager getQueueLockManager() {\n        return queueLockManager;\n    }\n\n    public static String genAckUniqueId(AckMsg ackMsg) {\n        return ackMsg.getTopic()\n            + PopAckConstants.SPLIT + ackMsg.getQueueId()\n            + PopAckConstants.SPLIT + ackMsg.getAckOffset()\n            + PopAckConstants.SPLIT + ackMsg.getConsumerGroup()\n            + PopAckConstants.SPLIT + ackMsg.getPopTime()\n            + PopAckConstants.SPLIT + ackMsg.getBrokerName()\n            + PopAckConstants.SPLIT + PopAckConstants.ACK_TAG;\n    }\n\n    public static String genBatchAckUniqueId(BatchAckMsg batchAckMsg) {\n        return batchAckMsg.getTopic()\n            + PopAckConstants.SPLIT + batchAckMsg.getQueueId()\n            + PopAckConstants.SPLIT + batchAckMsg.getAckOffsetList().toString()\n            + PopAckConstants.SPLIT + batchAckMsg.getConsumerGroup()\n            + PopAckConstants.SPLIT + batchAckMsg.getPopTime()\n            + PopAckConstants.SPLIT + PopAckConstants.BATCH_ACK_TAG;\n    }\n\n    public static String genCkUniqueId(PopCheckPoint ck) {\n        return ck.getTopic()\n            + PopAckConstants.SPLIT + ck.getQueueId()\n            + PopAckConstants.SPLIT + ck.getStartOffset()\n            + PopAckConstants.SPLIT + ck.getCId()\n            + PopAckConstants.SPLIT + ck.getPopTime()\n            + PopAckConstants.SPLIT + ck.getBrokerName()\n            + PopAckConstants.SPLIT + PopAckConstants.CK_TAG;\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n\n    public Cache<String, ConcurrentSkipListSet<PopRequest>> getPollingMap() {\n        return popLongPollingService.getPollingMap();\n    }\n\n    public void notifyLongPollingRequestIfNeed(String topic, String group, int queueId) throws ConsumeQueueException {\n        this.notifyLongPollingRequestIfNeed(\n            topic, group, queueId, null, 0L, null, null);\n    }\n\n    public void notifyLongPollingRequestIfNeed(String topic, String group, int queueId,\n        Long tagsCode, long msgStoreTime, byte[] filterBitMap,\n        Map<String, String> properties) throws ConsumeQueueException {\n        long popBufferOffset = this.brokerController.getPopMessageProcessor().getPopBufferMergeService().getLatestOffset(topic, group, queueId);\n        long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset(group, topic, queueId);\n        long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);\n        long offset = Math.max(popBufferOffset, consumerOffset);\n        if (maxOffset > offset) {\n            boolean notifySuccess = popLongPollingService.notifyMessageArriving(\n                topic, -1, group, tagsCode, msgStoreTime, filterBitMap, properties);\n            if (!notifySuccess) {\n                // notify pop queue\n                notifySuccess = popLongPollingService.notifyMessageArriving(\n                    topic, queueId, group, tagsCode, msgStoreTime, filterBitMap, properties);\n            }\n            this.brokerController.getNotificationProcessor().notifyMessageArriving(topic, queueId);\n            if (this.brokerController.getBrokerConfig().isEnablePopLog()) {\n                POP_LOGGER.info(\"notify long polling request. topic:{}, group:{}, queueId:{}, success:{}\",\n                    topic, group, queueId, notifySuccess);\n            }\n        }\n    }\n\n    public void notifyMessageArriving(final String topic, final int queueId, long offset,\n        Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {\n        popLongPollingService.notifyMessageArrivingWithRetryTopic(\n            topic, queueId, offset, tagsCode, msgStoreTime, filterBitMap, properties);\n    }\n\n    public void notifyMessageArriving(final String topic, final int queueId, final String cid) {\n        popLongPollingService.notifyMessageArriving(\n            topic, queueId, cid, false, null, 0L, null, null);\n    }\n\n    @Override\n    public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingCommand request)\n        throws RemotingCommandException {\n\n        final long beginTimeMills = this.brokerController.getMessageStore().now();\n\n        Channel channel = ctx.channel();\n        RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class);\n        response.setOpaque(request.getOpaque());\n\n        final PopMessageRequestHeader requestHeader =\n            request.decodeCommandCustomHeader(PopMessageRequestHeader.class, true);\n        if (requestHeader.getBornTime() == 0) {\n            request.addExtField(BORN_TIME, String.valueOf(beginTimeMills));\n            requestHeader.setBornTime(beginTimeMills);\n        }\n        final PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader();\n\n        // Pop mode only supports consumption in cluster load balancing mode\n        brokerController.getConsumerManager().compensateBasicConsumerInfo(\n            requestHeader.getConsumerGroup(), ConsumeType.CONSUME_POP, MessageModel.CLUSTERING);\n\n        if (brokerController.getBrokerConfig().isEnablePopLog()) {\n            POP_LOGGER.info(\"receive PopMessage request command, {}\", request);\n        }\n\n        if (requestHeader.isTimeoutTooMuch()) {\n            response.setCode(ResponseCode.POLLING_TIMEOUT);\n            response.setRemark(String.format(\"the broker[%s] pop message is timeout too much\",\n                this.brokerController.getBrokerConfig().getBrokerIP1()));\n            return response;\n        }\n\n        if (!PermName.isReadable(this.brokerController.getBrokerConfig().getBrokerPermission())) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            response.setRemark(String.format(\"the broker[%s] pop message is forbidden\",\n                this.brokerController.getBrokerConfig().getBrokerIP1()));\n            return response;\n        }\n\n        if (requestHeader.getMaxMsgNums() > 32) {\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(String.format(\"the broker[%s] pop message's num is greater than 32\",\n                this.brokerController.getBrokerConfig().getBrokerIP1()));\n            return response;\n        }\n\n        if (!brokerController.getMessageStore().getMessageStoreConfig().isTimerWheelEnable()) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(String.format(\"the broker[%s] pop message is forbidden because timerWheelEnable is false\",\n                this.brokerController.getBrokerConfig().getBrokerIP1()));\n            return response;\n        }\n\n        TopicConfig topicConfig =\n            this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());\n        if (null == topicConfig) {\n            POP_LOGGER.error(\"The topic {} not exist, consumer: {} \", requestHeader.getTopic(),\n                RemotingHelper.parseChannelRemoteAddr(channel));\n            response.setCode(ResponseCode.TOPIC_NOT_EXIST);\n            response.setRemark(String.format(\"topic[%s] not exist, apply first please! %s\", requestHeader.getTopic(),\n                FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)));\n            return response;\n        }\n\n        if (!PermName.isReadable(topicConfig.getPerm())) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            response.setRemark(\"the topic[\" + requestHeader.getTopic() + \"] peeking message is forbidden\");\n            return response;\n        }\n\n        if (requestHeader.getQueueId() >= topicConfig.getReadQueueNums()) {\n            String errorInfo = String.format(\"queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] \" +\n                    \"consumer:[%s]\",\n                requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(),\n                channel.remoteAddress());\n            POP_LOGGER.warn(errorInfo);\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(errorInfo);\n            return response;\n        }\n\n        SubscriptionGroupConfig subscriptionGroupConfig =\n            this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getConsumerGroup());\n        if (null == subscriptionGroupConfig) {\n            response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);\n            response.setRemark(String.format(\"subscription group [%s] does not exist, %s\",\n                requestHeader.getConsumerGroup(), FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST)));\n            return response;\n        }\n\n        if (!subscriptionGroupConfig.isConsumeEnable()) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            response.setRemark(\"subscription group no permission, \" + requestHeader.getConsumerGroup());\n            return response;\n        }\n\n        BrokerConfig brokerConfig = brokerController.getBrokerConfig();\n        SubscriptionData subscriptionData = null;\n        ExpressionMessageFilter messageFilter = null;\n        if (requestHeader.getExp() != null && !requestHeader.getExp().isEmpty()) {\n            try {\n                // origin topic\n                subscriptionData = FilterAPI.build(\n                    requestHeader.getTopic(), requestHeader.getExp(), requestHeader.getExpType());\n                brokerController.getConsumerManager().compensateSubscribeData(\n                    requestHeader.getConsumerGroup(), requestHeader.getTopic(), subscriptionData);\n\n                // retry topic\n                String retryTopic = KeyBuilder.buildPopRetryTopic(\n                    requestHeader.getTopic(), requestHeader.getConsumerGroup(), brokerConfig.isEnableRetryTopicV2());\n                SubscriptionData retrySubscriptionData = FilterAPI.build(\n                    retryTopic, SubscriptionData.SUB_ALL, requestHeader.getExpType());\n                brokerController.getConsumerManager().compensateSubscribeData(\n                    requestHeader.getConsumerGroup(), retryTopic, retrySubscriptionData);\n\n                ConsumerFilterData consumerFilterData = null;\n                if (!ExpressionType.isTagType(subscriptionData.getExpressionType())) {\n                    consumerFilterData = ConsumerFilterManager.build(\n                        requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getExp(),\n                        requestHeader.getExpType(), System.currentTimeMillis());\n                    if (consumerFilterData == null) {\n                        POP_LOGGER.warn(\"Parse the consumer's subscription[{}] failed, group: {}\",\n                            requestHeader.getExp(), requestHeader.getConsumerGroup());\n                        response.setCode(ResponseCode.SUBSCRIPTION_PARSE_FAILED);\n                        response.setRemark(\"parse the consumer's subscription failed\");\n                        return response;\n                    }\n                }\n                messageFilter = new ExpressionMessageFilter(\n                    subscriptionData, consumerFilterData, brokerController.getConsumerFilterManager());\n            } catch (Exception e) {\n                POP_LOGGER.warn(\"Parse the consumer's subscription[{}] error, group: {}\", requestHeader.getExp(),\n                    requestHeader.getConsumerGroup());\n                response.setCode(ResponseCode.SUBSCRIPTION_PARSE_FAILED);\n                response.setRemark(\"parse the consumer's subscription failed\");\n                return response;\n            }\n        } else {\n            try {\n                // origin topic\n                subscriptionData = FilterAPI.build(requestHeader.getTopic(), \"*\", ExpressionType.TAG);\n                brokerController.getConsumerManager().compensateSubscribeData(\n                    requestHeader.getConsumerGroup(), requestHeader.getTopic(), subscriptionData);\n\n                // retry topic\n                String retryTopic = KeyBuilder.buildPopRetryTopic(\n                    requestHeader.getTopic(), requestHeader.getConsumerGroup(), brokerConfig.isEnableRetryTopicV2());\n                SubscriptionData retrySubscriptionData = FilterAPI.build(retryTopic, \"*\", ExpressionType.TAG);\n                brokerController.getConsumerManager().compensateSubscribeData(\n                    requestHeader.getConsumerGroup(), retryTopic, retrySubscriptionData);\n            } catch (Exception e) {\n                POP_LOGGER.warn(\"Build default subscription error, group: {}\", requestHeader.getConsumerGroup());\n            }\n        }\n\n        GetMessageResult getMessageResult = new GetMessageResult(requestHeader.getMaxMsgNums());\n        ExpressionMessageFilter finalMessageFilter = messageFilter;\n        SubscriptionData finalSubscriptionData = subscriptionData;\n\n        if (brokerConfig.isPopConsumerKVServiceEnable()) {\n\n            CompletableFuture<PopConsumerContext> popAsyncFuture = brokerController.getPopConsumerService().popAsync(\n                RemotingHelper.parseChannelRemoteAddr(channel), beginTimeMills, requestHeader.getInvisibleTime(),\n                requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId(),\n                requestHeader.getMaxMsgNums(), requestHeader.isOrder(),\n                requestHeader.getAttemptId(), requestHeader.getInitMode(), messageFilter);\n\n            popAsyncFuture.thenApply(result -> {\n                try {\n                    if (request.getCallbackList() != null) {\n                        request.getCallbackList().forEach(CommandCallback::accept);\n                        request.getCallbackList().clear();\n                    }\n                } catch (Throwable t) {\n                    POP_LOGGER.error(\"PopProcessor execute callback error\", t);\n                }\n\n                if (result.isFound()) {\n                    response.setCode(ResponseCode.SUCCESS);\n                    getMessageResult.setStatus(GetMessageStatus.FOUND);\n                    // recursive processing\n                    if (result.getRestCount() > 0) {\n                        popLongPollingService.notifyMessageArriving(\n                            requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getConsumerGroup(),\n                            null, 0L, null, null);\n                    }\n                } else {\n                    POP_LOGGER.debug(\"Processor not found, polling request, popTime={}, restCount={}\",\n                        result.getPopTime(), result.getRestCount());\n\n                    PollingResult pollingResult = popLongPollingService.polling(\n                        ctx, request, new PollingHeader(requestHeader), finalSubscriptionData, finalMessageFilter);\n\n                    if (PollingResult.POLLING_SUC == pollingResult) {\n                        // recursive processing\n                        if (result.getRestCount() > 0) {\n                            popLongPollingService.notifyMessageArriving(\n                                requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getConsumerGroup(),\n                                null, 0L, null, null);\n                        }\n                        return null;\n                    } else if (PollingResult.POLLING_FULL == pollingResult) {\n                        response.setCode(ResponseCode.POLLING_FULL);\n                    } else {\n                        response.setCode(ResponseCode.POLLING_TIMEOUT);\n                    }\n                    getMessageResult.setStatus(GetMessageStatus.NO_MESSAGE_IN_QUEUE);\n                }\n\n                responseHeader.setPopTime(result.getPopTime());\n                responseHeader.setInvisibleTime(result.getInvisibleTime());\n                responseHeader.setReviveQid(\n                    requestHeader.isOrder() ? KeyBuilder.POP_ORDER_REVIVE_QUEUE : 0);\n                responseHeader.setRestNum(result.getRestCount());\n                responseHeader.setStartOffsetInfo(result.getStartOffsetInfo());\n                responseHeader.setMsgOffsetInfo(result.getMsgOffsetInfo());\n                if (requestHeader.isOrder() && !result.getOrderCountInfo().isEmpty()) {\n                    responseHeader.setOrderCountInfo(result.getOrderCountInfo());\n                }\n\n                response.setRemark(getMessageResult.getStatus().name());\n                if (response.getCode() != ResponseCode.SUCCESS) {\n                    return response;\n                }\n\n                // add message\n                result.getGetMessageResultList().forEach(temp -> {\n                    for (int i = 0; i < temp.getMessageMapedList().size(); i++) {\n                        getMessageResult.addMessage(temp.getMessageMapedList().get(i));\n                    }\n                });\n\n                if (this.brokerController.getBrokerConfig().isTransferMsgByHeap()) {\n                    final byte[] r = this.readGetMessageResult(getMessageResult,\n                        requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId());\n                    this.brokerController.getBrokerStatsManager().incGroupGetLatency(\n                        requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId(),\n                        (int) (this.brokerController.getMessageStore().now() - beginTimeMills));\n                    response.setBody(r);\n                } else {\n                    final GetMessageResult tmpGetMessageResult = getMessageResult;\n                    try {\n                        FileRegion fileRegion = new ManyMessageTransfer(\n                            response.encodeHeader(getMessageResult.getBufferTotalSize()), getMessageResult);\n                        channel.writeAndFlush(fileRegion)\n                            .addListener((ChannelFutureListener) future -> {\n                                tmpGetMessageResult.release();\n                                RemotingMetricsManager remotingMetricsManager = brokerController.getBrokerMetricsManager().getRemotingMetricsManager();\n                                Attributes attributes = remotingMetricsManager.newAttributesBuilder()\n                                    .put(LABEL_REQUEST_CODE, RemotingHelper.getRequestCodeDesc(request.getCode()))\n                                    .put(LABEL_RESPONSE_CODE, RemotingHelper.getResponseCodeDesc(response.getCode()))\n                                    .put(LABEL_RESULT, remotingMetricsManager.getWriteAndFlushResult(future))\n                                    .build();\n                                remotingMetricsManager.getRpcLatency().record(\n                                    request.getProcessTimer().elapsed(TimeUnit.MILLISECONDS), attributes);\n                                if (!future.isSuccess()) {\n                                    POP_LOGGER.error(\"Fail to transfer messages from page cache to {}\",\n                                        channel.remoteAddress(), future.cause());\n                                }\n                            });\n                    } catch (Throwable e) {\n                        POP_LOGGER.error(\"Error occurred when transferring messages from page cache\", e);\n                        getMessageResult.release();\n                    }\n                    return null;\n                }\n                return response;\n            }).thenAccept(result -> NettyRemotingAbstract.writeResponse(channel, request, result, null, brokerController.getBrokerMetricsManager().getRemotingMetricsManager()));\n            return null;\n        }\n\n        int randomQ = random.nextInt(100);\n        int reviveQid;\n        if (requestHeader.isOrder()) {\n            reviveQid = KeyBuilder.POP_ORDER_REVIVE_QUEUE;\n        } else {\n            reviveQid = (int) Math.abs(ckMessageNumber.getAndIncrement() %\n                this.brokerController.getBrokerConfig().getReviveQueueNum());\n        }\n\n        StringBuilder startOffsetInfo = new StringBuilder(64);\n        StringBuilder msgOffsetInfo = new StringBuilder(64);\n        StringBuilder orderCountInfo = requestHeader.isOrder() ? new StringBuilder(64) : null;\n\n        // Due to the design of the fields startOffsetInfo, msgOffsetInfo, and orderCountInfo,\n        // a single POP request could only invoke the popMsgFromQueue method once\n        // for either a normal topic or a retry topic's queue. Retry topics v1 and v2 are\n        // considered the same type because they share the same retry flag in previous fields.\n        // Therefore, needRetryV1 is designed as a subset of needRetry, and within a single request,\n        // only one type of retry topic is able to call popMsgFromQueue.\n        boolean usePriorityMode = TopicMessageType.PRIORITY.equals(topicConfig.getTopicMessageType())\n            && !requestHeader.isOrder() && randomQ < subscriptionGroupConfig.getPriorityFactor();\n        boolean needRetry = randomQ < (usePriorityMode ?\n            brokerConfig.getPopFromRetryProbabilityForPriority() : brokerConfig.getPopFromRetryProbability());\n        boolean needRetryV1 = false;\n        if (brokerConfig.isEnableRetryTopicV2() && brokerConfig.isRetrieveMessageFromPopRetryTopicV1()) {\n            needRetryV1 = randomQ % 2 == 0;\n        }\n        randomQ = usePriorityMode ? 0 : randomQ; // reset randomQ\n        long popTime = System.currentTimeMillis();\n        CompletableFuture<Long> getMessageFuture = CompletableFuture.completedFuture(0L);\n        if (needRetry && !requestHeader.isOrder()) {\n            if (needRetryV1) {\n                String retryTopic = KeyBuilder.buildPopRetryTopicV1(requestHeader.getTopic(), requestHeader.getConsumerGroup());\n                getMessageFuture = popMsgFromTopic(retryTopic, true, getMessageResult, requestHeader, reviveQid, channel,\n                    popTime, finalMessageFilter, startOffsetInfo, msgOffsetInfo, orderCountInfo, randomQ, getMessageFuture);\n            } else {\n                String retryTopic = KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), requestHeader.getConsumerGroup(), brokerConfig.isEnableRetryTopicV2());\n                getMessageFuture = popMsgFromTopic(retryTopic, true, getMessageResult, requestHeader, reviveQid, channel,\n                    popTime, finalMessageFilter, startOffsetInfo, msgOffsetInfo, orderCountInfo, randomQ, getMessageFuture);\n            }\n        }\n        if (requestHeader.getQueueId() < 0) {\n            // read all queue\n            getMessageFuture = popMsgFromTopic(topicConfig, false, getMessageResult, requestHeader, reviveQid, channel,\n                popTime, finalMessageFilter, startOffsetInfo, msgOffsetInfo, orderCountInfo, randomQ, getMessageFuture);\n        } else {\n            int queueId = requestHeader.getQueueId();\n            getMessageFuture = getMessageFuture.thenCompose(restNum ->\n                popMsgFromQueue(topicConfig.getTopicName(), requestHeader.getAttemptId(), false,\n                    getMessageResult, requestHeader, queueId, restNum, reviveQid, channel, popTime, finalMessageFilter,\n                    startOffsetInfo, msgOffsetInfo, orderCountInfo));\n        }\n        // if not full , fetch retry again\n        if (!needRetry && getMessageResult.getMessageMapedList().size() < requestHeader.getMaxMsgNums() && !requestHeader.isOrder()) {\n            if (needRetryV1) {\n                String retryTopicV1 = KeyBuilder.buildPopRetryTopicV1(requestHeader.getTopic(), requestHeader.getConsumerGroup());\n                getMessageFuture = popMsgFromTopic(retryTopicV1, true, getMessageResult, requestHeader, reviveQid, channel,\n                    popTime, finalMessageFilter, startOffsetInfo, msgOffsetInfo, orderCountInfo, randomQ, getMessageFuture);\n            } else {\n                String retryTopic = KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), requestHeader.getConsumerGroup(), brokerConfig.isEnableRetryTopicV2());\n                getMessageFuture = popMsgFromTopic(retryTopic, true, getMessageResult, requestHeader, reviveQid, channel,\n                    popTime, finalMessageFilter, startOffsetInfo, msgOffsetInfo, orderCountInfo, randomQ, getMessageFuture);\n            }\n        }\n\n        final RemotingCommand finalResponse = response;\n        getMessageFuture.thenApply(restNum -> {\n            try {\n                if (request.getCallbackList() != null) {\n                    request.getCallbackList().forEach(CommandCallback::accept);\n                    request.getCallbackList().clear();\n                }\n            } catch (Throwable t) {\n                POP_LOGGER.error(\"PopProcessor execute callback error\", t);\n            }\n\n            if (!getMessageResult.getMessageBufferList().isEmpty()) {\n                finalResponse.setCode(ResponseCode.SUCCESS);\n                getMessageResult.setStatus(GetMessageStatus.FOUND);\n                if (restNum > 0) {\n                    // all queue pop can not notify specified queue pop, and vice versa\n                    popLongPollingService.notifyMessageArriving(\n                        requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getConsumerGroup(),\n                        null, 0L, null, null);\n                }\n            } else {\n                PollingResult pollingResult = popLongPollingService.polling(\n                    ctx, request, new PollingHeader(requestHeader), finalSubscriptionData, finalMessageFilter);\n                if (PollingResult.POLLING_SUC == pollingResult) {\n                    if (restNum > 0) {\n                        popLongPollingService.notifyMessageArriving(\n                            requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getConsumerGroup(),\n                            null, 0L, null, null);\n                    }\n                    return null;\n                } else if (PollingResult.POLLING_FULL == pollingResult) {\n                    finalResponse.setCode(ResponseCode.POLLING_FULL);\n                } else {\n                    finalResponse.setCode(ResponseCode.POLLING_TIMEOUT);\n                }\n                getMessageResult.setStatus(GetMessageStatus.NO_MESSAGE_IN_QUEUE);\n            }\n            responseHeader.setInvisibleTime(requestHeader.getInvisibleTime());\n            responseHeader.setPopTime(popTime);\n            responseHeader.setReviveQid(reviveQid);\n            responseHeader.setRestNum(restNum);\n            responseHeader.setStartOffsetInfo(startOffsetInfo.toString());\n            responseHeader.setMsgOffsetInfo(msgOffsetInfo.toString());\n            if (requestHeader.isOrder() && orderCountInfo != null) {\n                responseHeader.setOrderCountInfo(orderCountInfo.toString());\n            }\n            finalResponse.setRemark(getMessageResult.getStatus().name());\n            switch (finalResponse.getCode()) {\n                case ResponseCode.SUCCESS:\n                    if (this.brokerController.getBrokerConfig().isTransferMsgByHeap()) {\n                        final byte[] r = this.readGetMessageResult(getMessageResult, requestHeader.getConsumerGroup(),\n                            requestHeader.getTopic(), requestHeader.getQueueId());\n                        this.brokerController.getBrokerStatsManager().incGroupGetLatency(requestHeader.getConsumerGroup(),\n                            requestHeader.getTopic(), requestHeader.getQueueId(),\n                            (int) (this.brokerController.getMessageStore().now() - beginTimeMills));\n                        finalResponse.setBody(r);\n                    } else {\n                        final GetMessageResult tmpGetMessageResult = getMessageResult;\n                        try {\n                            FileRegion fileRegion =\n                                new ManyMessageTransfer(finalResponse.encodeHeader(getMessageResult.getBufferTotalSize()),\n                                    getMessageResult);\n                            channel.writeAndFlush(fileRegion)\n                                .addListener((ChannelFutureListener) future -> {\n                                    tmpGetMessageResult.release();\n                                    RemotingMetricsManager remotingMetricsManager = brokerController.getBrokerMetricsManager().getRemotingMetricsManager();\n                                    Attributes attributes = remotingMetricsManager.newAttributesBuilder()\n                                        .put(LABEL_REQUEST_CODE, RemotingHelper.getRequestCodeDesc(request.getCode()))\n                                        .put(LABEL_RESPONSE_CODE, RemotingHelper.getResponseCodeDesc(finalResponse.getCode()))\n                                        .put(LABEL_RESULT, remotingMetricsManager.getWriteAndFlushResult(future))\n                                        .build();\n                                    remotingMetricsManager.getRpcLatency().record(request.getProcessTimer().elapsed(TimeUnit.MILLISECONDS), attributes);\n                                    if (!future.isSuccess()) {\n                                        POP_LOGGER.error(\"Fail to transfer messages from page cache to {}\",\n                                            channel.remoteAddress(), future.cause());\n                                    }\n                                });\n                        } catch (Throwable e) {\n                            POP_LOGGER.error(\"Error occurred when transferring messages from page cache\", e);\n                            getMessageResult.release();\n                        }\n\n                        return null;\n                    }\n                    break;\n                default:\n                    return finalResponse;\n            }\n            return finalResponse;\n        }).thenAccept(result -> NettyRemotingAbstract.writeResponse(channel, request, result, null, brokerController.getBrokerMetricsManager().getRemotingMetricsManager()));\n        return null;\n    }\n\n    private CompletableFuture<Long> popMsgFromTopic(TopicConfig topicConfig, boolean isRetry, GetMessageResult getMessageResult,\n        PopMessageRequestHeader requestHeader, int reviveQid, Channel channel, long popTime,\n        ExpressionMessageFilter messageFilter, StringBuilder startOffsetInfo,\n        StringBuilder msgOffsetInfo, StringBuilder orderCountInfo, int randomQ, CompletableFuture<Long> getMessageFuture) {\n        if (topicConfig != null) {\n            for (int i = 0; i < topicConfig.getReadQueueNums(); i++) {\n                int index = (brokerController.getBrokerConfig().isPriorityOrderAsc() ?\n                    topicConfig.getReadQueueNums() - 1 - i : i) + randomQ;\n                int queueId = index % topicConfig.getReadQueueNums();\n                getMessageFuture = getMessageFuture.thenCompose(restNum ->\n                    popMsgFromQueue(topicConfig.getTopicName(), requestHeader.getAttemptId(), isRetry,\n                        getMessageResult, requestHeader, queueId, restNum, reviveQid, channel, popTime, messageFilter,\n                        startOffsetInfo, msgOffsetInfo, orderCountInfo));\n            }\n        }\n        return getMessageFuture;\n    }\n\n    private CompletableFuture<Long> popMsgFromTopic(String topic, boolean isRetry, GetMessageResult getMessageResult,\n        PopMessageRequestHeader requestHeader, int reviveQid, Channel channel, long popTime,\n        ExpressionMessageFilter messageFilter, StringBuilder startOffsetInfo,\n        StringBuilder msgOffsetInfo, StringBuilder orderCountInfo, int randomQ, CompletableFuture<Long> getMessageFuture) {\n        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);\n        return popMsgFromTopic(topicConfig, isRetry, getMessageResult, requestHeader, reviveQid, channel, popTime,\n            messageFilter, startOffsetInfo, msgOffsetInfo, orderCountInfo, randomQ, getMessageFuture);\n    }\n\n    private CompletableFuture<Long> popMsgFromQueue(String topic, String attemptId, boolean isRetry,\n        GetMessageResult getMessageResult,\n        PopMessageRequestHeader requestHeader, int queueId, long restNum, int reviveQid,\n        Channel channel, long popTime, ExpressionMessageFilter messageFilter, StringBuilder startOffsetInfo,\n        StringBuilder msgOffsetInfo, StringBuilder orderCountInfo) {\n\n        String lockKey =\n            topic + PopAckConstants.SPLIT + requestHeader.getConsumerGroup() + PopAckConstants.SPLIT + queueId;\n        boolean isOrder = requestHeader.isOrder();\n        long offset;\n        try {\n            offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId, requestHeader.getInitMode(),\n                false, lockKey, false);\n        } catch (ConsumeQueueException e) {\n            CompletableFuture<Long> failure = new CompletableFuture<>();\n            failure.completeExceptionally(e);\n            return failure;\n        }\n\n        CompletableFuture<Long> future = new CompletableFuture<>();\n        if (!queueLockManager.tryLock(lockKey)) {\n            try {\n                if (!requestHeader.isOrder()) {\n                    restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;\n                }\n                future.complete(restNum);\n            } catch (ConsumeQueueException e) {\n                future.completeExceptionally(e);\n            }\n            return future;\n        }\n\n        future.whenComplete((result, throwable) -> queueLockManager.unLock(lockKey));\n        if (isPopShouldStop(topic, requestHeader.getConsumerGroup(), queueId)) {\n            POP_LOGGER.warn(\"Too much msgs unacked, then stop popping. topic={}, group={}, queueId={}\",\n                topic, requestHeader.getConsumerGroup(), queueId);\n            try {\n                restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;\n                future.complete(restNum);\n            } catch (ConsumeQueueException e) {\n                future.completeExceptionally(e);\n            }\n            return future;\n        }\n\n        try {\n            offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId, requestHeader.getInitMode(),\n                true, lockKey, true);\n\n            // Current requests would calculate the total number of messages\n            // waiting to be filtered for new message arrival notifications in\n            // the long-polling service, need disregarding the backlog in order\n            // consumption scenario. If rest message num including the blocked\n            // queue accumulation would lead to frequent unnecessary wake-ups\n            // of long-polling requests, resulting unnecessary CPU usage.\n            // When client ack message, long-polling request would be notifications\n            // by AckMessageProcessor.ackOrderly() and message will not be delayed.\n            if (isOrder) {\n                if (brokerController.getConsumerOrderInfoManager().checkBlock(\n                    attemptId, topic, requestHeader.getConsumerGroup(), queueId, requestHeader.getInvisibleTime())) {\n                    // should not add accumulation(max offset - consumer offset) here\n                    future.complete(restNum);\n                    return future;\n                }\n                this.brokerController.getPopInflightMessageCounter().clearInFlightMessageNum(\n                    topic, requestHeader.getConsumerGroup(), queueId);\n            }\n\n            if (getMessageResult.getMessageMapedList().size() >= requestHeader.getMaxMsgNums()) {\n                restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;\n                future.complete(restNum);\n                return future;\n            }\n        } catch (Exception e) {\n            POP_LOGGER.error(\"Exception in popMsgFromQueue\", e);\n            future.complete(restNum);\n            return future;\n        }\n\n        AtomicLong atomicRestNum = new AtomicLong(restNum);\n        AtomicLong atomicOffset = new AtomicLong(offset);\n        long finalOffset = offset;\n        return this.brokerController.getMessageStore()\n            .getMessageAsync(requestHeader.getConsumerGroup(), topic, queueId, offset,\n                requestHeader.getMaxMsgNums() - getMessageResult.getMessageMapedList().size(), messageFilter)\n            .thenCompose(result -> {\n                if (result == null) {\n                    return CompletableFuture.completedFuture(null);\n                }\n                // maybe store offset is not correct.\n                if (GetMessageStatus.OFFSET_TOO_SMALL.equals(result.getStatus())\n                    || GetMessageStatus.OFFSET_OVERFLOW_BADLY.equals(result.getStatus())\n                    || GetMessageStatus.OFFSET_FOUND_NULL.equals(result.getStatus())) {\n                    // commit offset, because the offset is not correct\n                    // If offset in store is greater than cq offset, it will cause duplicate messages,\n                    // because offset in PopBuffer is not committed.\n                    POP_LOGGER.warn(\"Pop initial offset, because store is no correct, {}, {}->{}\",\n                        lockKey, atomicOffset.get(), result.getNextBeginOffset());\n                    this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(), requestHeader.getConsumerGroup(), topic,\n                        queueId, result.getNextBeginOffset());\n                    atomicOffset.set(result.getNextBeginOffset());\n                    return this.brokerController.getMessageStore().getMessageAsync(requestHeader.getConsumerGroup(), topic, queueId, atomicOffset.get(),\n                        requestHeader.getMaxMsgNums() - getMessageResult.getMessageMapedList().size(), messageFilter);\n                }\n                return CompletableFuture.completedFuture(result);\n            }).thenApply(result -> {\n                if (result == null) {\n                    try {\n                        atomicRestNum.set(brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - atomicOffset.get() + atomicRestNum.get());\n                    } catch (ConsumeQueueException e) {\n                        POP_LOGGER.error(\"Failed to get max offset in queue\", e);\n                    }\n                    return atomicRestNum.get();\n                }\n                if (!result.getMessageMapedList().isEmpty()) {\n                    this.brokerController.getBrokerStatsManager().incBrokerGetNums(requestHeader.getTopic(), result.getMessageCount());\n                    this.brokerController.getBrokerStatsManager().incGroupGetNums(requestHeader.getConsumerGroup(), topic,\n                        result.getMessageCount());\n                    this.brokerController.getBrokerStatsManager().incGroupGetSize(requestHeader.getConsumerGroup(), topic,\n                        result.getBufferTotalSize());\n\n                    Attributes attributes = this.brokerController.getBrokerMetricsManager().newAttributesBuilder()\n                        .put(LABEL_TOPIC, requestHeader.getTopic())\n                        .put(LABEL_CONSUMER_GROUP, requestHeader.getConsumerGroup())\n                        .put(LABEL_IS_SYSTEM, TopicValidator.isSystemTopic(requestHeader.getTopic()) || MixAll.isSysConsumerGroup(requestHeader.getConsumerGroup()))\n                        .put(LABEL_IS_RETRY, isRetry)\n                        .build();\n                    this.brokerController.getBrokerMetricsManager().getMessagesOutTotal().add(result.getMessageCount(), attributes);\n                    this.brokerController.getBrokerMetricsManager().getThroughputOutTotal().add(result.getBufferTotalSize(), attributes);\n\n                    if (isOrder) {\n                        this.brokerController.getConsumerOrderInfoManager().update(requestHeader.getAttemptId(), isRetry, topic,\n                            requestHeader.getConsumerGroup(),\n                            queueId, popTime, requestHeader.getInvisibleTime(), result.getMessageQueueOffset(),\n                            orderCountInfo, result);\n                        this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(),\n                            requestHeader.getConsumerGroup(), topic, queueId, finalOffset);\n                    } else {\n                        if (!appendCheckPoint(requestHeader, topic, reviveQid, queueId, finalOffset, result, popTime, this.brokerController.getBrokerConfig().getBrokerName())) {\n                            return atomicRestNum.get() + result.getMessageCount();\n                        }\n                    }\n                    ExtraInfoUtil.buildStartOffsetInfo(startOffsetInfo, topic, queueId, finalOffset);\n                    ExtraInfoUtil.buildMsgOffsetInfo(msgOffsetInfo, topic, queueId,\n                        result.getMessageQueueOffset());\n                } else if ((GetMessageStatus.NO_MATCHED_MESSAGE.equals(result.getStatus())\n                    || GetMessageStatus.OFFSET_FOUND_NULL.equals(result.getStatus())\n                    || GetMessageStatus.MESSAGE_WAS_REMOVING.equals(result.getStatus())\n                    || GetMessageStatus.NO_MATCHED_LOGIC_QUEUE.equals(result.getStatus()))\n                    && result.getNextBeginOffset() > -1) {\n                    if (isOrder) {\n                        this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(), requestHeader.getConsumerGroup(), topic,\n                            queueId, result.getNextBeginOffset());\n                    } else {\n                        popBufferMergeService.addCkMock(requestHeader.getConsumerGroup(), topic, queueId, finalOffset,\n                            requestHeader.getInvisibleTime(), popTime, reviveQid, result.getNextBeginOffset(), brokerController.getBrokerConfig().getBrokerName());\n                    }\n                }\n\n                atomicRestNum.set(result.getMaxOffset() - result.getNextBeginOffset() + atomicRestNum.get());\n                String brokerName = brokerController.getBrokerConfig().getBrokerName();\n                for (SelectMappedBufferResult mapedBuffer : result.getMessageMapedList()) {\n                    // We should not recode buffer when popResponseReturnActualRetryTopic is true or topic is not retry topic\n                    if (brokerController.getBrokerConfig().isPopResponseReturnActualRetryTopic() || !isRetry) {\n                        getMessageResult.addMessage(mapedBuffer);\n                    } else {\n                        List<MessageExt> messageExtList = MessageDecoder.decodesBatch(mapedBuffer.getByteBuffer(),\n                            true, false, true);\n                        mapedBuffer.release();\n                        for (MessageExt messageExt : messageExtList) {\n                            try {\n                                String ckInfo = ExtraInfoUtil.buildExtraInfo(finalOffset, popTime, requestHeader.getInvisibleTime(),\n                                    reviveQid, messageExt.getTopic(), brokerName, messageExt.getQueueId(), messageExt.getQueueOffset());\n                                messageExt.getProperties().putIfAbsent(MessageConst.PROPERTY_POP_CK, ckInfo);\n\n                                // Set retry message topic to origin topic and clear message store size to recode\n                                messageExt.setTopic(requestHeader.getTopic());\n                                messageExt.setStoreSize(0);\n\n                                byte[] encode = MessageDecoder.encode(messageExt, false);\n                                ByteBuffer buffer = ByteBuffer.wrap(encode);\n                                SelectMappedBufferResult tmpResult =\n                                    new SelectMappedBufferResult(mapedBuffer.getStartOffset(), buffer, encode.length, null);\n                                getMessageResult.addMessage(tmpResult);\n                            } catch (Exception e) {\n                                POP_LOGGER.error(\"Exception in recode retry message buffer, topic={}\", topic, e);\n                            }\n                        }\n                    }\n                }\n                this.brokerController.getPopInflightMessageCounter().incrementInFlightMessageNum(\n                    topic,\n                    requestHeader.getConsumerGroup(),\n                    queueId,\n                    result.getMessageCount()\n                );\n                return atomicRestNum.get();\n            }).whenComplete((result, throwable) -> {\n                if (throwable != null) {\n                    POP_LOGGER.error(\"Pop message error, {}\", lockKey, throwable);\n                }\n                queueLockManager.unLock(lockKey);\n            });\n    }\n\n    private boolean isPopShouldStop(String topic, String group, int queueId) {\n        return brokerController.getBrokerConfig().isEnablePopMessageThreshold() &&\n            brokerController.getPopInflightMessageCounter().getGroupPopInFlightMessageNum(topic, group, queueId) > brokerController.getBrokerConfig().getPopInflightMessageThreshold();\n    }\n\n    private long getPopOffset(String topic, String group, int queueId, int initMode, boolean init, String lockKey,\n        boolean checkResetOffset) throws ConsumeQueueException {\n\n        long offset = this.brokerController.getConsumerOffsetManager().queryOffset(group, topic, queueId);\n        if (offset < 0) {\n            offset = this.getInitOffset(topic, group, queueId, initMode, init);\n        }\n\n        if (checkResetOffset) {\n            Long resetOffset = resetPopOffset(topic, group, queueId);\n            if (resetOffset != null) {\n                return resetOffset;\n            }\n        }\n\n        long bufferOffset = this.popBufferMergeService.getLatestOffset(lockKey);\n        if (bufferOffset < 0) {\n            return offset;\n        } else {\n            return Math.max(bufferOffset, offset);\n        }\n    }\n\n    public long getInitOffset(String topic, String group, int queueId, int initMode, boolean init)\n        throws ConsumeQueueException {\n        long offset;\n        if (ConsumeInitMode.MIN == initMode || topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n            offset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId);\n        } else {\n            if (this.brokerController.getBrokerConfig().isInitPopOffsetByCheckMsgInMem() &&\n                this.brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId) <= 0 &&\n                this.brokerController.getMessageStore().checkInMemByConsumeOffset(topic, queueId, 0, 1)) {\n                offset = 0;\n            } else {\n                // pop last one,then commit offset.\n                offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - 1;\n                // max & no consumer offset\n                if (offset < 0) {\n                    offset = 0;\n                }\n            }\n        }\n        if (init) { // whichever initMode\n            this.brokerController.getConsumerOffsetManager().commitOffset(\n                \"getPopOffset\", group, topic, queueId, offset);\n        }\n        return offset;\n    }\n\n    public MessageExtBrokerInner buildCkMsg(final PopCheckPoint ck, final int reviveQid) {\n        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n\n        msgInner.setTopic(reviveTopic);\n        msgInner.setBody(JSON.toJSONString(ck).getBytes(StandardCharsets.UTF_8));\n        msgInner.setQueueId(reviveQid);\n        msgInner.setTags(PopAckConstants.CK_TAG);\n        msgInner.setBornTimestamp(System.currentTimeMillis());\n        msgInner.setBornHost(this.brokerController.getStoreHost());\n        msgInner.setStoreHost(this.brokerController.getStoreHost());\n        msgInner.setDeliverTimeMs(ck.getReviveTime() - PopAckConstants.ackTimeInterval);\n        msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, genCkUniqueId(ck));\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));\n\n        return msgInner;\n    }\n\n    private boolean appendCheckPoint(final PopMessageRequestHeader requestHeader,\n        final String topic, final int reviveQid, final int queueId, final long offset,\n        final GetMessageResult getMessageTmpResult, final long popTime, final String brokerName) {\n        // add check point msg to revive log\n        final PopCheckPoint ck = new PopCheckPoint();\n        ck.setBitMap(0);\n        ck.setNum((byte) getMessageTmpResult.getMessageMapedList().size());\n        ck.setPopTime(popTime);\n        ck.setInvisibleTime(requestHeader.getInvisibleTime());\n        ck.setStartOffset(offset);\n        ck.setCId(requestHeader.getConsumerGroup());\n        ck.setTopic(topic);\n        ck.setQueueId(queueId);\n        ck.setBrokerName(brokerName);\n        for (Long msgQueueOffset : getMessageTmpResult.getMessageQueueOffset()) {\n            ck.addDiff((int) (msgQueueOffset - offset));\n        }\n\n        this.brokerController.getBrokerStatsManager().incBrokerCkNums(1);\n        this.brokerController.getBrokerStatsManager().incGroupCkNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(), 1);\n\n        final boolean addBufferSuc = this.popBufferMergeService.addCk(\n            ck, reviveQid, -1, getMessageTmpResult.getNextBeginOffset()\n        );\n\n        if (addBufferSuc) {\n            return true;\n        }\n        return this.popBufferMergeService.addCkJustOffset(\n            ck, reviveQid, -1, getMessageTmpResult.getNextBeginOffset()\n        );\n    }\n\n    private Long resetPopOffset(String topic, String group, int queueId) {\n        String lockKey = topic + PopAckConstants.SPLIT + group + PopAckConstants.SPLIT + queueId;\n        Long resetOffset =\n            this.brokerController.getConsumerOffsetManager().queryThenEraseResetOffset(topic, group, queueId);\n        if (resetOffset != null) {\n            this.brokerController.getConsumerOrderInfoManager().clearBlock(topic, group, queueId);\n            this.getPopBufferMergeService().clearOffsetQueue(lockKey);\n            this.brokerController.getConsumerOffsetManager()\n                .commitOffset(\"ResetPopOffset\", group, topic, queueId, resetOffset);\n        }\n        return resetOffset;\n    }\n\n    private byte[] readGetMessageResult(final GetMessageResult getMessageResult, final String group, final String topic,\n        final int queueId) {\n        final ByteBuffer byteBuffer = ByteBuffer.allocate(getMessageResult.getBufferTotalSize());\n\n        long storeTimestamp = 0;\n        try {\n            List<ByteBuffer> messageBufferList = getMessageResult.getMessageBufferList();\n            for (ByteBuffer bb : messageBufferList) {\n\n                byteBuffer.put(bb);\n                storeTimestamp = bb.getLong(MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION);\n            }\n        } finally {\n            getMessageResult.release();\n        }\n\n        this.brokerController.getBrokerStatsManager().recordDiskFallBehindTime(group, topic, queueId,\n            this.brokerController.getMessageStore().now() - storeTimestamp);\n        return byteBuffer.array();\n    }\n\n    static class TimedLock {\n        private final AtomicBoolean lock;\n        private volatile long lockTime;\n\n        public TimedLock() {\n            // init lock status, false means not locked\n            this.lock = new AtomicBoolean(false);\n            this.lockTime = System.currentTimeMillis();\n        }\n\n        public boolean tryLock() {\n            boolean ret = lock.compareAndSet(false, true);\n            if (ret) {\n                this.lockTime = System.currentTimeMillis();\n                return true;\n            } else {\n                return false;\n            }\n        }\n\n        public void unLock() {\n            lock.set(false);\n        }\n\n        public boolean isLock() {\n            return lock.get();\n        }\n\n        public long getLockTime() {\n            return lockTime;\n        }\n    }\n\n    public class QueueLockManager extends ServiceThread {\n        private final ConcurrentHashMap<String, TimedLock> expiredLocalCache = new ConcurrentHashMap<>(100000);\n\n        public String buildLockKey(String topic, String consumerGroup, int queueId) {\n            return topic + PopAckConstants.SPLIT + consumerGroup + PopAckConstants.SPLIT + queueId;\n        }\n\n        public boolean tryLock(String topic, String consumerGroup, int queueId) {\n            return tryLock(buildLockKey(topic, consumerGroup, queueId));\n        }\n\n        public boolean tryLock(String key) {\n            TimedLock timedLock = ConcurrentHashMapUtils.computeIfAbsent(expiredLocalCache, key, k -> new TimedLock());\n            return timedLock.tryLock();\n        }\n\n        /**\n         * is not thread safe, may cause duplicate lock\n         *\n         * @param usedExpireMillis the expired time in millisecond\n         * @return total numbers of TimedLock\n         */\n        public int cleanUnusedLock(final long usedExpireMillis) {\n            Iterator<Entry<String, TimedLock>> iterator = expiredLocalCache.entrySet().iterator();\n\n            int total = 0;\n            while (iterator.hasNext()) {\n                Entry<String, TimedLock> entry = iterator.next();\n\n                if (System.currentTimeMillis() - entry.getValue().getLockTime() > usedExpireMillis) {\n                    iterator.remove();\n                    POP_LOGGER.info(\"Remove unused queue lock: {}, {}, {}\", entry.getKey(),\n                        entry.getValue().getLockTime(),\n                        entry.getValue().isLock());\n                }\n\n                total++;\n            }\n\n            return total;\n        }\n\n        public void unLock(String topic, String consumerGroup, int queueId) {\n            unLock(buildLockKey(topic, consumerGroup, queueId));\n        }\n\n        public void unLock(String key) {\n            TimedLock timedLock = expiredLocalCache.get(key);\n            if (timedLock != null) {\n                timedLock.unLock();\n            }\n        }\n\n        @Override\n        public String getServiceName() {\n            if (PopMessageProcessor.this.brokerController.getBrokerConfig().isInBrokerContainer()) {\n                return PopMessageProcessor.this.brokerController.getBrokerIdentity().getIdentifier() + QueueLockManager.class.getSimpleName();\n            }\n            return QueueLockManager.class.getSimpleName();\n        }\n\n        @Override\n        public void run() {\n            while (!isStopped()) {\n                try {\n                    this.waitForRunning(60000);\n                    int count = cleanUnusedLock(60000);\n                    POP_LOGGER.info(\"QueueLockSize={}\", count);\n                } catch (Exception e) {\n                    PopMessageProcessor.POP_LOGGER.error(\"QueueLockManager run error\", e);\n                }\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport com.alibaba.fastjson2.JSON;\nimport io.opentelemetry.api.common.Attributes;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.commons.lang3.tuple.Triple;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.PopAckConstants;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.TopicFilterType;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.common.utils.DataConverter;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.AppendMessageStatus;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.pop.AckMsg;\nimport org.apache.rocketmq.store.pop.BatchAckMsg;\nimport org.apache.rocketmq.store.pop.PopCheckPoint;\n\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.NavigableMap;\nimport java.util.TreeMap;\nimport java.util.concurrent.CompletableFuture;\n\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_TOPIC;\n\npublic class PopReviveService extends ServiceThread {\n    private static final Logger POP_LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);\n    private final int[] ckRewriteIntervalsInSeconds = new int[] { 10, 20, 30, 60, 120, 180, 240, 300, 360, 420, 480, 540, 600, 1200, 1800, 3600, 7200 };\n\n    private int queueId;\n    private BrokerController brokerController;\n    private String reviveTopic;\n    private long currentReviveMessageTimestamp = -1;\n    private volatile boolean shouldRunPopRevive = false;\n\n    private final NavigableMap<PopCheckPoint/* oldCK */, Pair<Long/* timestamp */, Boolean/* result */>> inflightReviveRequestMap = Collections.synchronizedNavigableMap(new TreeMap<>());\n    private long reviveOffset;\n\n    public PopReviveService(BrokerController brokerController, String reviveTopic, int queueId) {\n        this.queueId = queueId;\n        this.brokerController = brokerController;\n        this.reviveTopic = reviveTopic;\n        this.reviveOffset = brokerController.getConsumerOffsetManager().queryOffset(PopAckConstants.REVIVE_GROUP, reviveTopic, queueId);\n    }\n\n    @Override\n    public String getServiceName() {\n        if (brokerController != null && brokerController.getBrokerConfig().isInBrokerContainer()) {\n            return brokerController.getBrokerIdentity().getIdentifier() + \"PopReviveService_\" + this.queueId;\n        }\n        return \"PopReviveService_\" + this.queueId;\n    }\n\n    public int getQueueId() {\n        return queueId;\n    }\n\n    public void setShouldRunPopRevive(final boolean shouldRunPopRevive) {\n        this.shouldRunPopRevive = shouldRunPopRevive;\n    }\n\n    public boolean isShouldRunPopRevive() {\n        return shouldRunPopRevive;\n    }\n\n    private boolean reviveRetry(PopCheckPoint popCheckPoint, MessageExt messageExt) {\n        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n        if (!popCheckPoint.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n            msgInner.setTopic(KeyBuilder.buildPopRetryTopic(popCheckPoint.getTopic(), popCheckPoint.getCId(), brokerController.getBrokerConfig().isEnableRetryTopicV2()));\n        } else {\n            msgInner.setTopic(popCheckPoint.getTopic());\n        }\n        msgInner.setBody(messageExt.getBody());\n        if (messageExt.getTags() != null) {\n            msgInner.setTags(messageExt.getTags());\n        } else {\n            MessageAccessor.setProperties(msgInner, new HashMap<>());\n        }\n        msgInner.setBornTimestamp(messageExt.getBornTimestamp());\n        msgInner.setFlag(messageExt.getFlag());\n        msgInner.setSysFlag(messageExt.getSysFlag());\n        msgInner.setBornHost(brokerController.getStoreHost());\n        msgInner.setStoreHost(brokerController.getStoreHost());\n        if (popCheckPoint.isSuspend()) {\n            msgInner.setReconsumeTimes(messageExt.getReconsumeTimes());\n        } else {\n            msgInner.setReconsumeTimes(messageExt.getReconsumeTimes() + 1);\n        }\n        msgInner.getProperties().putAll(messageExt.getProperties());\n        if (messageExt.getReconsumeTimes() == 0 || msgInner.getProperties().get(MessageConst.PROPERTY_FIRST_POP_TIME) == null) {\n            msgInner.getProperties().put(MessageConst.PROPERTY_FIRST_POP_TIME, String.valueOf(popCheckPoint.getPopTime()));\n        }\n        msgInner.getProperties().put(MessageConst.PROPERTY_ORIGIN_GROUP, popCheckPoint.getCId());\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));\n        addRetryTopicIfNotExist(msgInner.getTopic(), popCheckPoint.getCId());\n        msgInner.setQueueId(getRetryQueueId(msgInner.getTopic(), messageExt));\n        PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);\n        brokerController.getBrokerMetricsManager().getPopMetricsManager().incPopReviveRetryMessageCount(popCheckPoint, putMessageResult.getPutMessageStatus());\n        if (brokerController.getBrokerConfig().isEnablePopLog()) {\n            POP_LOGGER.info(\"reviveQueueId={},retry msg, ck={}, msg queueId {}, offset {}, reviveDelay={}, result is {} \",\n                queueId, popCheckPoint, messageExt.getQueueId(), messageExt.getQueueOffset(),\n                (System.currentTimeMillis() - popCheckPoint.getReviveTime()) / 1000, putMessageResult);\n        }\n        if (putMessageResult.getAppendMessageResult() == null ||\n            putMessageResult.getAppendMessageResult().getStatus() != AppendMessageStatus.PUT_OK) {\n            POP_LOGGER.error(\"reviveQueueId={}, revive error, msg is: {}\", queueId, msgInner);\n            return false;\n        }\n        this.brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(popCheckPoint);\n        this.brokerController.getBrokerStatsManager().incBrokerPutNums(popCheckPoint.getTopic(), 1);\n        this.brokerController.getBrokerStatsManager().incTopicPutNums(msgInner.getTopic());\n        this.brokerController.getBrokerStatsManager().incTopicPutSize(msgInner.getTopic(), putMessageResult.getAppendMessageResult().getWroteBytes());\n        return true;\n    }\n\n    private void initPopRetryOffset(String retryTopic, String consumerGroup, int retryQueueNum) {\n        for (int i = 0; i < retryQueueNum; i++) {\n            long offset = this.brokerController.getConsumerOffsetManager().queryOffset(consumerGroup, retryTopic, i);\n            if (offset < 0) {\n                this.brokerController.getConsumerOffsetManager().commitOffset(\"initPopRetryOffset\", consumerGroup, retryTopic, i, 0);\n            }\n        }\n    }\n\n    public void addRetryTopicIfNotExist(String retryTopic, String consumerGroup) {\n        if (brokerController != null) {\n            TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(retryTopic);\n            if (topicConfig != null && !brokerController.getBrokerConfig().isUseSeparateRetryQueue()) {\n                return;\n            }\n\n            int retryQueueNum = PopAckConstants.retryQueueNum;\n            if (brokerController.getBrokerConfig().isUseSeparateRetryQueue()) {\n                String normalTopic = KeyBuilder.parseNormalTopic(retryTopic, consumerGroup);\n                TopicConfig normalConfig = brokerController.getTopicConfigManager().selectTopicConfig(normalTopic); // always exists\n                retryQueueNum = normalConfig.getWriteQueueNums();\n                if (topicConfig != null && topicConfig.getWriteQueueNums() == normalConfig.getWriteQueueNums()) {\n                    return;\n                }\n            }\n\n            // create new one, or update in case of queue expansion\n            topicConfig = new TopicConfig(retryTopic);\n            topicConfig.setReadQueueNums(retryQueueNum);\n            topicConfig.setWriteQueueNums(retryQueueNum);\n            topicConfig.setTopicFilterType(TopicFilterType.SINGLE_TAG);\n            topicConfig.setPerm(6);\n            topicConfig.setTopicSysFlag(0);\n            brokerController.getTopicConfigManager().updateTopicConfig(topicConfig);\n\n            initPopRetryOffset(retryTopic, consumerGroup, retryQueueNum);\n        }\n    }\n\n    private int getRetryQueueId(String retryTopic, MessageExt messageExt) {\n        if (!brokerController.getBrokerConfig().isUseSeparateRetryQueue()) {\n            return 0;\n        }\n        int oriQueueId = messageExt.getQueueId(); // original qid of normal or retry topic\n        if (oriQueueId > brokerController.getTopicConfigManager().selectTopicConfig(retryTopic).getWriteQueueNums() - 1) {\n            POP_LOGGER.warn(\"not expected, {}, {}, {}\", retryTopic, oriQueueId, messageExt.getMsgId());\n            return 0; // fallback\n        }\n        return oriQueueId;\n    }\n\n    protected List<MessageExt> getReviveMessage(long offset, int queueId) {\n        PullResult pullResult = getMessage(PopAckConstants.REVIVE_GROUP, reviveTopic, queueId, offset, 32, true);\n        if (pullResult == null) {\n            return null;\n        }\n        if (reachTail(pullResult, offset)) {\n            if (this.brokerController.getBrokerConfig().isEnablePopLog()) {\n                POP_LOGGER.info(\"reviveQueueId={}, reach tail,offset {}\", queueId, offset);\n            }\n        } else if (pullResult.getPullStatus() == PullStatus.OFFSET_ILLEGAL || pullResult.getPullStatus() == PullStatus.NO_MATCHED_MSG) {\n            POP_LOGGER.error(\"reviveQueueId={}, OFFSET_ILLEGAL {}, result is {}\", queueId, offset, pullResult);\n            if (!shouldRunPopRevive) {\n                POP_LOGGER.info(\"slave skip offset correct topic={}, reviveQueueId={}\", reviveTopic, queueId);\n                return null;\n            }\n            this.brokerController.getConsumerOffsetManager().commitOffset(PopAckConstants.LOCAL_HOST, PopAckConstants.REVIVE_GROUP, reviveTopic, queueId, pullResult.getNextBeginOffset() - 1);\n        }\n        return pullResult.getMsgFoundList();\n    }\n\n    private boolean reachTail(PullResult pullResult, long offset) {\n        return pullResult.getPullStatus() == PullStatus.NO_NEW_MSG\n            || pullResult.getPullStatus() == PullStatus.OFFSET_ILLEGAL && offset == pullResult.getMaxOffset();\n    }\n\n    // Triple<MessageExt, info, needRetry>\n    public CompletableFuture<Triple<MessageExt, String, Boolean>> getBizMessage(PopCheckPoint popCheckPoint, long offset) {\n        return this.brokerController.getEscapeBridge().getMessageAsync(popCheckPoint.getTopic(), offset, popCheckPoint.getQueueId(), popCheckPoint.getBrokerName(), false);\n    }\n\n    public PullResult getMessage(String group, String topic, int queueId, long offset, int nums,\n        boolean deCompressBody) {\n        GetMessageResult getMessageResult = this.brokerController.getMessageStore().getMessage(group, topic, queueId, offset, nums, null);\n\n        if (getMessageResult != null) {\n            PullStatus pullStatus = PullStatus.NO_NEW_MSG;\n            List<MessageExt> foundList = null;\n            switch (getMessageResult.getStatus()) {\n                case FOUND:\n                    pullStatus = PullStatus.FOUND;\n                    foundList = decodeMsgList(getMessageResult, deCompressBody);\n                    brokerController.getBrokerStatsManager().incGroupGetNums(group, topic, getMessageResult.getMessageCount());\n                    brokerController.getBrokerStatsManager().incGroupGetSize(group, topic, getMessageResult.getBufferTotalSize());\n                    brokerController.getBrokerStatsManager().incBrokerGetNums(topic, getMessageResult.getMessageCount());\n                    brokerController.getBrokerStatsManager().recordDiskFallBehindTime(group, topic, queueId,\n                        brokerController.getMessageStore().now() - foundList.get(foundList.size() - 1).getStoreTimestamp());\n\n                    Attributes attributes = this.brokerController.getBrokerMetricsManager().newAttributesBuilder()\n                        .put(LABEL_TOPIC, topic)\n                        .put(LABEL_CONSUMER_GROUP, group)\n                        .put(LABEL_IS_SYSTEM, TopicValidator.isSystemTopic(topic) || MixAll.isSysConsumerGroup(group))\n                        .build();\n                    this.brokerController.getBrokerMetricsManager().getMessagesOutTotal().add(getMessageResult.getMessageCount(), attributes);\n                    this.brokerController.getBrokerMetricsManager().getThroughputOutTotal().add(getMessageResult.getBufferTotalSize(), attributes);\n\n                    break;\n                case NO_MATCHED_MESSAGE:\n                    pullStatus = PullStatus.NO_MATCHED_MSG;\n                    POP_LOGGER.debug(\"no matched message. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}\",\n                        getMessageResult.getStatus(), topic, group, offset);\n                    break;\n                case NO_MESSAGE_IN_QUEUE:\n                    POP_LOGGER.debug(\"no new message. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}\",\n                        getMessageResult.getStatus(), topic, group, offset);\n                    break;\n                case MESSAGE_WAS_REMOVING:\n                case NO_MATCHED_LOGIC_QUEUE:\n                case OFFSET_FOUND_NULL:\n                case OFFSET_OVERFLOW_BADLY:\n                case OFFSET_TOO_SMALL:\n                    pullStatus = PullStatus.OFFSET_ILLEGAL;\n                    POP_LOGGER.warn(\"offset illegal. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}\",\n                        getMessageResult.getStatus(), topic, group, offset);\n                    break;\n                case OFFSET_OVERFLOW_ONE:\n                    // no need to print WARN, because we use \"offset + 1\" to get the next message\n                    pullStatus = PullStatus.OFFSET_ILLEGAL;\n                    break;\n                default:\n                    assert false;\n                    break;\n            }\n\n            return new PullResult(pullStatus, getMessageResult.getNextBeginOffset(), getMessageResult.getMinOffset(),\n                getMessageResult.getMaxOffset(), foundList);\n\n        } else {\n            try {\n                long maxQueueOffset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);\n                if (maxQueueOffset > offset) {\n                    POP_LOGGER.error(\"get message from store return null. topic={}, groupId={}, requestOffset={}, maxQueueOffset={}\",\n                        topic, group, offset, maxQueueOffset);\n                }\n            } catch (ConsumeQueueException e) {\n                POP_LOGGER.error(\"Failed to get max offset in queue\", e);\n            }\n            return null;\n        }\n    }\n\n    private List<MessageExt> decodeMsgList(GetMessageResult getMessageResult, boolean deCompressBody) {\n        List<MessageExt> foundList = new ArrayList<>();\n        try {\n            List<ByteBuffer> messageBufferList = getMessageResult.getMessageBufferList();\n            if (messageBufferList != null) {\n                for (int i = 0; i < messageBufferList.size(); i++) {\n                    ByteBuffer bb = messageBufferList.get(i);\n                    if (bb == null) {\n                        POP_LOGGER.error(\"bb is null {}\", getMessageResult);\n                        continue;\n                    }\n                    MessageExt msgExt = MessageDecoder.decode(bb, true, deCompressBody);\n                    if (msgExt == null) {\n                        POP_LOGGER.error(\"decode msgExt is null {}\", getMessageResult);\n                        continue;\n                    }\n                    // use CQ offset, not offset in Message\n                    msgExt.setQueueOffset(getMessageResult.getMessageQueueOffset().get(i));\n                    foundList.add(msgExt);\n                }\n            }\n        } finally {\n            getMessageResult.release();\n        }\n\n        return foundList;\n    }\n\n    protected void consumeReviveMessage(ConsumeReviveObj consumeReviveObj) {\n        HashMap<String, PopCheckPoint> map = consumeReviveObj.map;\n        HashMap<String, PopCheckPoint> mockPointMap = new HashMap<>();\n        long startScanTime = System.currentTimeMillis();\n        long endTime = 0;\n        long consumeOffset = this.brokerController.getConsumerOffsetManager().queryOffset(PopAckConstants.REVIVE_GROUP, reviveTopic, queueId);\n        long oldOffset = Math.max(reviveOffset, consumeOffset);\n        consumeReviveObj.oldOffset = oldOffset;\n        POP_LOGGER.info(\"reviveQueueId={}, old offset is {} \", queueId, oldOffset);\n        long offset = oldOffset + 1;\n        int noMsgCount = 0;\n        long firstRt = 0;\n        // offset self amend\n        while (true) {\n            if (!shouldRunPopRevive) {\n                POP_LOGGER.info(\"slave skip scan, revive topic={}, reviveQueueId={}\", reviveTopic, queueId);\n                break;\n            }\n            List<MessageExt> messageExts = getReviveMessage(offset, queueId);\n            if (messageExts == null || messageExts.isEmpty()) {\n                long old = endTime;\n                long timerDelay = brokerController.getMessageStore().getTimerMessageStore().getDequeueBehind();\n                long commitLogDelay = brokerController.getMessageStore().getTimerMessageStore().getEnqueueBehind();\n                // move endTime\n                if (endTime != 0 && System.currentTimeMillis() - endTime > 3 * PopAckConstants.SECOND && timerDelay <= 0 && commitLogDelay <= 0) {\n                    endTime = System.currentTimeMillis();\n                }\n                POP_LOGGER.debug(\"reviveQueueId={}, offset is {}, can not get new msg, old endTime {}, new endTime {}, timerDelay={}, commitLogDelay={} \",\n                    queueId, offset, old, endTime, timerDelay, commitLogDelay);\n                if (endTime - firstRt > PopAckConstants.ackTimeInterval + PopAckConstants.SECOND) {\n                    break;\n                }\n                noMsgCount++;\n                // Fixme: why sleep is useful here?\n                try {\n                    Thread.sleep(100);\n                } catch (Throwable ignore) {\n                }\n                if (noMsgCount * 100L > 4 * PopAckConstants.SECOND) {\n                    break;\n                } else {\n                    continue;\n                }\n            } else {\n                noMsgCount = 0;\n            }\n            if (System.currentTimeMillis() - startScanTime > brokerController.getBrokerConfig().getReviveScanTime()) {\n                POP_LOGGER.info(\"reviveQueueId={}, scan timeout \", queueId);\n                break;\n            }\n            for (MessageExt messageExt : messageExts) {\n                if (PopAckConstants.CK_TAG.equals(messageExt.getTags())) {\n                    String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8);\n                    if (brokerController.getBrokerConfig().isEnablePopLog()) {\n                        POP_LOGGER.info(\"reviveQueueId={},find ck, offset:{}, raw : {}\", messageExt.getQueueId(), messageExt.getQueueOffset(), raw);\n                    }\n                    PopCheckPoint point = JSON.parseObject(raw, PopCheckPoint.class);\n                    if (point.getTopic() == null || point.getCId() == null) {\n                        continue;\n                    }\n                    map.put(point.getTopic() + point.getCId() + point.getQueueId() + point.getStartOffset() + point.getPopTime() + point.getBrokerName(), point);\n                    brokerController.getBrokerMetricsManager().getPopMetricsManager().incPopReviveCkGetCount(point, queueId);\n                    point.setReviveOffset(messageExt.getQueueOffset());\n                    if (firstRt == 0) {\n                        firstRt = point.getReviveTime();\n                    }\n                } else if (PopAckConstants.ACK_TAG.equals(messageExt.getTags())) {\n                    String raw = new String(messageExt.getBody(), StandardCharsets.UTF_8);\n                    if (brokerController.getBrokerConfig().isEnablePopLog()) {\n                        POP_LOGGER.info(\"reviveQueueId={}, find ack, offset:{}, raw : {}\", messageExt.getQueueId(), messageExt.getQueueOffset(), raw);\n                    }\n                    AckMsg ackMsg = JSON.parseObject(raw, AckMsg.class);\n                    brokerController.getBrokerMetricsManager().getPopMetricsManager().incPopReviveAckGetCount(ackMsg, queueId);\n                    String brokerName = StringUtils.isNotBlank(ackMsg.getBrokerName()) ?\n                        ackMsg.getBrokerName() : brokerController.getBrokerConfig().getBrokerName();\n                    String mergeKey = ackMsg.getTopic() + ackMsg.getConsumerGroup() + ackMsg.getQueueId() + ackMsg.getStartOffset() + ackMsg.getPopTime() + brokerName;\n                    PopCheckPoint point = map.get(mergeKey);\n                    if (point == null) {\n                        if (!brokerController.getBrokerConfig().isEnableSkipLongAwaitingAck()) {\n                            continue;\n                        }\n                        if (mockCkForAck(messageExt, ackMsg, mergeKey, mockPointMap) && firstRt == 0) {\n                            firstRt = mockPointMap.get(mergeKey).getReviveTime();\n                        }\n                    } else {\n                        int indexOfAck = point.indexOfAck(ackMsg.getAckOffset());\n                        if (indexOfAck > -1) {\n                            point.setBitMap(DataConverter.setBit(point.getBitMap(), indexOfAck, true));\n                        } else {\n                            POP_LOGGER.error(\"invalid ack index, {}, {}\", ackMsg, point);\n                        }\n                    }\n                } else if (PopAckConstants.BATCH_ACK_TAG.equals(messageExt.getTags())) {\n                    String raw = new String(messageExt.getBody(), StandardCharsets.UTF_8);\n                    if (brokerController.getBrokerConfig().isEnablePopLog()) {\n                        POP_LOGGER.info(\"reviveQueueId={}, find batch ack, offset:{}, raw : {}\", messageExt.getQueueId(), messageExt.getQueueOffset(), raw);\n                    }\n\n                    BatchAckMsg bAckMsg = JSON.parseObject(raw, BatchAckMsg.class);\n                    brokerController.getBrokerMetricsManager().getPopMetricsManager().incPopReviveAckGetCount(bAckMsg, queueId);\n                    String brokerName = StringUtils.isNotBlank(bAckMsg.getBrokerName()) ?\n                        bAckMsg.getBrokerName() : brokerController.getBrokerConfig().getBrokerName();\n                    String mergeKey = bAckMsg.getTopic() + bAckMsg.getConsumerGroup() + bAckMsg.getQueueId() + bAckMsg.getStartOffset() + bAckMsg.getPopTime() + brokerName;\n                    PopCheckPoint point = map.get(mergeKey);\n                    if (point == null) {\n                        if (!brokerController.getBrokerConfig().isEnableSkipLongAwaitingAck()) {\n                            continue;\n                        }\n                        if (mockCkForAck(messageExt, bAckMsg, mergeKey, mockPointMap) && firstRt == 0) {\n                            firstRt = mockPointMap.get(mergeKey).getReviveTime();\n                        }\n                    } else {\n                        List<Long> ackOffsetList = bAckMsg.getAckOffsetList();\n                        for (Long ackOffset : ackOffsetList) {\n                            int indexOfAck = point.indexOfAck(ackOffset);\n                            if (indexOfAck > -1) {\n                                point.setBitMap(DataConverter.setBit(point.getBitMap(), indexOfAck, true));\n                            } else {\n                                POP_LOGGER.error(\"invalid batch ack index, {}, {}\", bAckMsg, point);\n                            }\n                        }\n                    }\n                }\n                long deliverTime = messageExt.getDeliverTimeMs();\n                if (deliverTime > endTime) {\n                    endTime = deliverTime;\n                }\n            }\n            offset = offset + messageExts.size();\n        }\n        consumeReviveObj.map.putAll(mockPointMap);\n        consumeReviveObj.endTime = endTime;\n    }\n\n    private boolean mockCkForAck(MessageExt messageExt, AckMsg ackMsg, String mergeKey, HashMap<String, PopCheckPoint> mockPointMap) {\n        long ackWaitTime = System.currentTimeMillis() - messageExt.getDeliverTimeMs();\n        long reviveAckWaitMs = brokerController.getBrokerConfig().getReviveAckWaitMs();\n        if (ackWaitTime > reviveAckWaitMs) {\n            // will use the reviveOffset of popCheckPoint to commit offset in mergeAndRevive\n            PopCheckPoint mockPoint = createMockCkForAck(ackMsg, messageExt.getQueueOffset());\n            POP_LOGGER.warn(\n                    \"ack wait for {}ms cannot find ck, skip this ack. mergeKey:{}, ack:{}, mockCk:{}\",\n                    reviveAckWaitMs, mergeKey, ackMsg, mockPoint);\n            mockPointMap.put(mergeKey, mockPoint);\n            return true;\n        }\n        return false;\n    }\n\n    private PopCheckPoint createMockCkForAck(AckMsg ackMsg, long reviveOffset) {\n        PopCheckPoint point = new PopCheckPoint();\n        point.setStartOffset(ackMsg.getStartOffset());\n        point.setPopTime(ackMsg.getPopTime());\n        point.setQueueId(ackMsg.getQueueId());\n        point.setCId(ackMsg.getConsumerGroup());\n        point.setTopic(ackMsg.getTopic());\n        point.setNum((byte) 0);\n        point.setBitMap(0);\n        point.setReviveOffset(reviveOffset);\n        point.setBrokerName(ackMsg.getBrokerName());\n        return point;\n    }\n\n    protected void mergeAndRevive(ConsumeReviveObj consumeReviveObj) throws Throwable {\n        ArrayList<PopCheckPoint> sortList = consumeReviveObj.genSortList();\n        POP_LOGGER.info(\"reviveQueueId={}, ck listSize={}\", queueId, sortList.size());\n        if (sortList.size() != 0) {\n            POP_LOGGER.info(\"reviveQueueId={}, 1st ck, startOffset={}, reviveOffset={}; last ck, startOffset={}, reviveOffset={}\", queueId, sortList.get(0).getStartOffset(),\n                sortList.get(0).getReviveOffset(), sortList.get(sortList.size() - 1).getStartOffset(), sortList.get(sortList.size() - 1).getReviveOffset());\n        }\n        long newOffset = consumeReviveObj.oldOffset;\n        for (PopCheckPoint popCheckPoint : sortList) {\n            if (!shouldRunPopRevive) {\n                POP_LOGGER.info(\"slave skip ck process, revive topic={}, reviveQueueId={}\", reviveTopic, queueId);\n                break;\n            }\n            if (consumeReviveObj.endTime - popCheckPoint.getReviveTime() <= (PopAckConstants.ackTimeInterval + PopAckConstants.SECOND)) {\n                break;\n            }\n\n            // check normal topic, skip ck , if normal topic is not exist\n            String normalTopic = KeyBuilder.parseNormalTopic(popCheckPoint.getTopic(), popCheckPoint.getCId());\n            if (brokerController.getTopicConfigManager().selectTopicConfig(normalTopic) == null) {\n                POP_LOGGER.warn(\"reviveQueueId={}, can not get normal topic {}, then continue\", queueId, popCheckPoint.getTopic());\n                newOffset = popCheckPoint.getReviveOffset();\n                continue;\n            }\n            if (null == brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(popCheckPoint.getCId())) {\n                POP_LOGGER.warn(\"reviveQueueId={}, can not get cid {}, then continue\", queueId, popCheckPoint.getCId());\n                newOffset = popCheckPoint.getReviveOffset();\n                continue;\n            }\n\n            while (inflightReviveRequestMap.size() > 3) {\n                waitForRunning(100);\n                Pair<Long, Boolean> pair = inflightReviveRequestMap.firstEntry().getValue();\n                if (!pair.getObject2() && System.currentTimeMillis() - pair.getObject1() > 1000 * 30) {\n                    PopCheckPoint oldCK = inflightReviveRequestMap.firstKey();\n                    rePutCK(oldCK, pair);\n                    inflightReviveRequestMap.remove(oldCK);\n                    POP_LOGGER.warn(\"stay too long, remove from reviveRequestMap, {}, {}, {}, {}\", popCheckPoint.getTopic(),\n                            popCheckPoint.getBrokerName(), popCheckPoint.getQueueId(), popCheckPoint.getStartOffset());\n                }\n            }\n\n            reviveMsgFromCk(popCheckPoint);\n\n            newOffset = popCheckPoint.getReviveOffset();\n        }\n        if (newOffset > consumeReviveObj.oldOffset) {\n            if (!shouldRunPopRevive) {\n                POP_LOGGER.info(\"slave skip commit, revive topic={}, reviveQueueId={}\", reviveTopic, queueId);\n                return;\n            }\n            this.brokerController.getConsumerOffsetManager().commitOffset(PopAckConstants.LOCAL_HOST, PopAckConstants.REVIVE_GROUP, reviveTopic, queueId, newOffset);\n        }\n        reviveOffset = newOffset;\n        consumeReviveObj.newOffset = newOffset;\n    }\n\n    private void reviveMsgFromCk(PopCheckPoint popCheckPoint) {\n        if (!shouldRunPopRevive) {\n            POP_LOGGER.info(\"slave skip retry, revive topic={}, reviveQueueId={}\", reviveTopic, queueId);\n            return;\n        }\n        inflightReviveRequestMap.put(popCheckPoint, new Pair<>(System.currentTimeMillis(), false));\n        List<CompletableFuture<Pair<Long, Boolean>>> futureList = new ArrayList<>(popCheckPoint.getNum());\n        for (int j = 0; j < popCheckPoint.getNum(); j++) {\n            if (DataConverter.getBit(popCheckPoint.getBitMap(), j)) {\n                continue;\n            }\n\n            // retry msg\n            long msgOffset = popCheckPoint.ackOffsetByIndex((byte) j);\n            CompletableFuture<Pair<Long, Boolean>> future = getBizMessage(popCheckPoint, msgOffset)\n                .thenApply(rst -> {\n                    MessageExt message = rst.getLeft();\n                    if (message == null) {\n                        POP_LOGGER.info(\"reviveQueueId={}, can not get biz msg, topic:{}, qid:{}, offset:{}, brokerName:{}, info:{}, retry:{}, then continue\",\n                            queueId, popCheckPoint.getTopic(), popCheckPoint.getQueueId(), msgOffset, popCheckPoint.getBrokerName(), UtilAll.frontStringAtLeast(rst.getMiddle(), 60), rst.getRight());\n                        return new Pair<>(msgOffset, !rst.getRight()); // Pair.object2 means OK or not, Triple.right value means needRetry\n                    }\n                    boolean result = reviveRetry(popCheckPoint, message);\n                    return new Pair<>(msgOffset, result);\n                });\n            futureList.add(future);\n        }\n        CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0]))\n            .whenComplete((v, e) -> {\n                for (CompletableFuture<Pair<Long, Boolean>> future : futureList) {\n                    Pair<Long, Boolean> pair = future.getNow(new Pair<>(0L, false));\n                    if (!pair.getObject2()) {\n                        rePutCK(popCheckPoint, pair);\n                    }\n                }\n\n                if (inflightReviveRequestMap.containsKey(popCheckPoint)) {\n                    inflightReviveRequestMap.get(popCheckPoint).setObject2(true);\n                }\n                for (Map.Entry<PopCheckPoint, Pair<Long, Boolean>> entry : inflightReviveRequestMap.entrySet()) {\n                    PopCheckPoint oldCK = entry.getKey();\n                    Pair<Long, Boolean> pair = entry.getValue();\n                    if (pair.getObject2()) {\n                        brokerController.getConsumerOffsetManager().commitOffset(PopAckConstants.LOCAL_HOST, PopAckConstants.REVIVE_GROUP, reviveTopic, queueId, oldCK.getReviveOffset());\n                        inflightReviveRequestMap.remove(oldCK);\n                    } else {\n                        break;\n                    }\n                }\n            });\n    }\n\n    private void rePutCK(PopCheckPoint oldCK, Pair<Long, Boolean> pair) {\n        int rePutTimes = oldCK.parseRePutTimes();\n        if (rePutTimes >= ckRewriteIntervalsInSeconds.length && brokerController.getBrokerConfig().isSkipWhenCKRePutReachMaxTimes()) {\n            POP_LOGGER.warn(\"rePut CK reach max times, drop it. {}, {}, {}, {}-{}, {}, {}, {}\", oldCK.getTopic(), oldCK.getCId(),\n                    oldCK.getBrokerName(), oldCK.getQueueId(), pair.getObject1(), oldCK.getPopTime(), oldCK.getInvisibleTime(), rePutTimes);\n            return;\n        }\n\n        PopCheckPoint newCk = new PopCheckPoint();\n        newCk.setBitMap(0);\n        newCk.setNum((byte) 1);\n        newCk.setPopTime(oldCK.getPopTime());\n        newCk.setInvisibleTime(oldCK.getInvisibleTime());\n        newCk.setStartOffset(pair.getObject1());\n        newCk.setCId(oldCK.getCId());\n        newCk.setTopic(oldCK.getTopic());\n        newCk.setQueueId(oldCK.getQueueId());\n        newCk.setBrokerName(oldCK.getBrokerName());\n        newCk.addDiff(0);\n        newCk.setRePutTimes(String.valueOf(rePutTimes + 1)); // always increment even if removed from reviveRequestMap\n        if (oldCK.getReviveTime() <= System.currentTimeMillis()) {\n            // never expect an ACK matched in the future, we just use it to rewrite CK and try to revive retry message next time\n            int intervalIndex = rePutTimes >= ckRewriteIntervalsInSeconds.length ? ckRewriteIntervalsInSeconds.length - 1 : rePutTimes;\n            newCk.setInvisibleTime(oldCK.getInvisibleTime() + ckRewriteIntervalsInSeconds[intervalIndex] * 1000);\n        }\n        MessageExtBrokerInner ckMsg = brokerController.getPopMessageProcessor().buildCkMsg(newCk, queueId);\n        brokerController.getMessageStore().putMessage(ckMsg);\n    }\n\n    public long getReviveBehindMillis() throws ConsumeQueueException {\n        if (currentReviveMessageTimestamp <= 0) {\n            return 0;\n        }\n        long maxOffset = brokerController.getMessageStore().getMaxOffsetInQueue(reviveTopic, queueId);\n        if (maxOffset - reviveOffset > 1) {\n            return Math.max(0, System.currentTimeMillis() - currentReviveMessageTimestamp);\n        }\n        return 0;\n    }\n\n    public long getReviveBehindMessages() throws ConsumeQueueException {\n        if (currentReviveMessageTimestamp <= 0) {\n            return 0;\n        }\n        // the next pull offset is reviveOffset + 1\n        long diff = brokerController.getMessageStore().getMaxOffsetInQueue(reviveTopic, queueId) - reviveOffset - 1;\n        return Math.max(0, diff);\n    }\n\n    @Override\n    public void run() {\n        int slow = 1;\n        while (!this.isStopped()) {\n            try {\n                if (System.currentTimeMillis() < brokerController.getShouldStartTime()) {\n                    POP_LOGGER.info(\"PopReviveService Ready to run after {}\", brokerController.getShouldStartTime());\n                    this.waitForRunning(1000);\n                    continue;\n                }\n                this.waitForRunning(brokerController.getBrokerConfig().getReviveInterval());\n                if (!shouldRunPopRevive) {\n                    POP_LOGGER.info(\"skip start revive topic={}, reviveQueueId={}\", reviveTopic, queueId);\n                    continue;\n                }\n\n                if (!brokerController.getMessageStore().getMessageStoreConfig().isTimerWheelEnable()) {\n                    POP_LOGGER.warn(\"skip revive topic because timerWheelEnable is false\");\n                    continue;\n                }\n\n                POP_LOGGER.info(\"start revive topic={}, reviveQueueId={}\", reviveTopic, queueId);\n                ConsumeReviveObj consumeReviveObj = new ConsumeReviveObj();\n                consumeReviveMessage(consumeReviveObj);\n\n                if (!shouldRunPopRevive) {\n                    POP_LOGGER.info(\"slave skip scan, revive topic={}, reviveQueueId={}\", reviveTopic, queueId);\n                    continue;\n                }\n\n                mergeAndRevive(consumeReviveObj);\n\n                ArrayList<PopCheckPoint> sortList = consumeReviveObj.sortList;\n                long delay = 0;\n                if (sortList != null && !sortList.isEmpty()) {\n                    delay = (System.currentTimeMillis() - sortList.get(0).getReviveTime()) / 1000;\n                    currentReviveMessageTimestamp = sortList.get(0).getReviveTime();\n                    slow = 1;\n                } else {\n                    currentReviveMessageTimestamp = System.currentTimeMillis();\n                }\n\n                POP_LOGGER.info(\"reviveQueueId={}, revive finish,old offset is {}, new offset is {}, ckDelay={}  \",\n                    queueId, consumeReviveObj.oldOffset, consumeReviveObj.newOffset, delay);\n\n                if (sortList == null || sortList.isEmpty()) {\n                    POP_LOGGER.info(\"reviveQueueId={}, has no new msg, take a rest {}\", queueId, slow);\n                    this.waitForRunning(slow * brokerController.getBrokerConfig().getReviveInterval());\n                    if (slow < brokerController.getBrokerConfig().getReviveMaxSlow()) {\n                        slow++;\n                    }\n                }\n\n            } catch (Throwable e) {\n                POP_LOGGER.error(\"reviveQueueId={}, revive error\", queueId, e);\n            }\n        }\n    }\n\n    static class ConsumeReviveObj {\n        HashMap<String, PopCheckPoint> map = new HashMap<>();\n        ArrayList<PopCheckPoint> sortList;\n        long oldOffset;\n        long endTime;\n        long newOffset;\n\n        ArrayList<PopCheckPoint> genSortList() {\n            if (sortList != null) {\n                return sortList;\n            }\n            sortList = new ArrayList<>(map.values());\n            sortList.sort((o1, o2) -> (int) (o1.getReviveOffset() - o2.getReviveOffset()));\n            return sortList;\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.List;\nimport java.util.Objects;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ConsumerGroupInfo;\nimport org.apache.rocketmq.broker.client.ConsumerManager;\nimport org.apache.rocketmq.broker.coldctr.ColdDataPullRequestHoldService;\nimport org.apache.rocketmq.broker.filter.ConsumerFilterData;\nimport org.apache.rocketmq.broker.filter.ConsumerFilterManager;\nimport org.apache.rocketmq.broker.filter.ExpressionForRetryMessageFilter;\nimport org.apache.rocketmq.broker.filter.ExpressionMessageFilter;\nimport org.apache.rocketmq.broker.longpolling.PullRequest;\nimport org.apache.rocketmq.broker.mqtrace.ConsumeMessageContext;\nimport org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook;\nimport org.apache.rocketmq.broker.plugin.PullMessageResultHandler;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.common.help.FAQUrl;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.apache.rocketmq.common.sysflag.PullSysFlag;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.netty.RequestTask;\nimport org.apache.rocketmq.remoting.protocol.ForbiddenType;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.RequestSource;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.filter.FilterAPI;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingAbstract;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.statictopic.LogicQueueMappingItem;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingContext;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingDetail;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingUtils;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.remoting.rpc.RpcClientUtils;\nimport org.apache.rocketmq.remoting.rpc.RpcRequest;\nimport org.apache.rocketmq.remoting.rpc.RpcResponse;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.MessageFilter;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\n\nimport static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse;\n\npublic class PullMessageProcessor implements NettyRequestProcessor {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private List<ConsumeMessageHook> consumeMessageHookList;\n    private PullMessageResultHandler pullMessageResultHandler;\n    private final BrokerController brokerController;\n\n    public PullMessageProcessor(final BrokerController brokerController) {\n        this.brokerController = brokerController;\n        this.pullMessageResultHandler = new DefaultPullMessageResultHandler(brokerController);\n    }\n\n    private RemotingCommand rewriteRequestForStaticTopic(PullMessageRequestHeader requestHeader,\n        TopicQueueMappingContext mappingContext) {\n        try {\n            if (mappingContext.getMappingDetail() == null) {\n                return null;\n            }\n            TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();\n            String topic = mappingContext.getTopic();\n            Integer globalId = mappingContext.getGlobalId();\n            // if the leader? consider the order consumer, which will lock the mq\n            if (!mappingContext.isLeader()) {\n                return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format(\"%s-%d cannot find mapping item in request process of current broker %s\", topic, globalId, mappingDetail.getBname()));\n            }\n\n            Long globalOffset = requestHeader.getQueueOffset();\n            LogicQueueMappingItem mappingItem = TopicQueueMappingUtils.findLogicQueueMappingItem(mappingContext.getMappingItemList(), globalOffset, true);\n            mappingContext.setCurrentItem(mappingItem);\n\n            if (globalOffset < mappingItem.getLogicOffset()) {\n                //handleOffsetMoved\n                //If the physical queue is reused, we should handle the PULL_OFFSET_MOVED independently\n                //Otherwise, we could just transfer it to the physical process\n            }\n            //below are physical info\n            String bname = mappingItem.getBname();\n            Integer phyQueueId = mappingItem.getQueueId();\n            Long phyQueueOffset = mappingItem.computePhysicalQueueOffset(globalOffset);\n            requestHeader.setQueueId(phyQueueId);\n            requestHeader.setQueueOffset(phyQueueOffset);\n            if (mappingItem.checkIfEndOffsetDecided()\n                && requestHeader.getMaxMsgNums() != null) {\n                requestHeader.setMaxMsgNums((int) Math.min(mappingItem.getEndOffset() - mappingItem.getStartOffset(), requestHeader.getMaxMsgNums()));\n            }\n\n            if (mappingDetail.getBname().equals(bname)) {\n                //just let it go, do the local pull process\n                return null;\n            }\n\n            int sysFlag = requestHeader.getSysFlag();\n            requestHeader.setLo(false);\n            requestHeader.setBrokerName(bname);\n            sysFlag = PullSysFlag.clearSuspendFlag(sysFlag);\n            sysFlag = PullSysFlag.clearCommitOffsetFlag(sysFlag);\n            requestHeader.setSysFlag(sysFlag);\n            RpcRequest rpcRequest = new RpcRequest(RequestCode.PULL_MESSAGE, requestHeader, null);\n            RpcResponse rpcResponse = this.brokerController.getBrokerOuterAPI().getRpcClient().invoke(rpcRequest, this.brokerController.getBrokerConfig().getForwardTimeout()).get();\n            if (rpcResponse.getException() != null) {\n                throw rpcResponse.getException();\n            }\n\n            PullMessageResponseHeader responseHeader = (PullMessageResponseHeader) rpcResponse.getHeader();\n            {\n                RemotingCommand rewriteResult = rewriteResponseForStaticTopic(requestHeader, responseHeader, mappingContext, rpcResponse.getCode());\n                if (rewriteResult != null) {\n                    return rewriteResult;\n                }\n            }\n            return RpcClientUtils.createCommandForRpcResponse(rpcResponse);\n        } catch (Throwable t) {\n            LOGGER.warn(\"\", t);\n            return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.toString());\n        }\n    }\n\n    protected RemotingCommand rewriteResponseForStaticTopic(PullMessageRequestHeader requestHeader,\n        PullMessageResponseHeader responseHeader,\n        TopicQueueMappingContext mappingContext, final int code) {\n        try {\n            if (mappingContext.getMappingDetail() == null) {\n                return null;\n            }\n            TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();\n            LogicQueueMappingItem leaderItem = mappingContext.getLeaderItem();\n\n            LogicQueueMappingItem currentItem = mappingContext.getCurrentItem();\n\n            LogicQueueMappingItem earlistItem = TopicQueueMappingUtils.findLogicQueueMappingItem(mappingContext.getMappingItemList(), 0L, true);\n\n            assert currentItem.getLogicOffset() >= 0;\n\n            long requestOffset = requestHeader.getQueueOffset();\n            long nextBeginOffset = responseHeader.getNextBeginOffset();\n            long minOffset = responseHeader.getMinOffset();\n            long maxOffset = responseHeader.getMaxOffset();\n            int responseCode = code;\n\n            //consider the following situations\n            // 1. read from slave, currently not supported\n            // 2. the middle queue is truncated because of deleting commitlog\n            if (code != ResponseCode.SUCCESS) {\n                //note the currentItem maybe both the leader and  the earliest\n                boolean isRevised = false;\n                if (leaderItem.getGen() == currentItem.getGen()) {\n                    //read the leader\n                    if (requestOffset > maxOffset) {\n                        //actually, we need do nothing, but keep the code structure here\n                        if (code == ResponseCode.PULL_OFFSET_MOVED) {\n                            responseCode = ResponseCode.PULL_OFFSET_MOVED;\n                            nextBeginOffset = maxOffset;\n                        } else {\n                            //maybe current broker is the slave\n                            responseCode = code;\n                        }\n                    } else if (requestOffset < minOffset) {\n                        nextBeginOffset = minOffset;\n                        responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY;\n                    } else {\n                        responseCode = code;\n                    }\n                }\n                //note the currentItem maybe both the leader and  the earliest\n                if (earlistItem.getGen() == currentItem.getGen()) {\n                    //read the earliest one\n                    if (requestOffset < minOffset) {\n                        if (code == ResponseCode.PULL_OFFSET_MOVED) {\n                            responseCode = ResponseCode.PULL_OFFSET_MOVED;\n                            nextBeginOffset = minOffset;\n                        } else {\n                            //maybe read from slave, but we still set it to moved\n                            responseCode = ResponseCode.PULL_OFFSET_MOVED;\n                            nextBeginOffset = minOffset;\n                        }\n                    } else if (requestOffset >= maxOffset) {\n                        //just move to another item\n                        LogicQueueMappingItem nextItem = TopicQueueMappingUtils.findNext(mappingContext.getMappingItemList(), currentItem, true);\n                        if (nextItem != null) {\n                            isRevised = true;\n                            currentItem = nextItem;\n                            nextBeginOffset = currentItem.getStartOffset();\n                            minOffset = currentItem.getStartOffset();\n                            maxOffset = minOffset;\n                            responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY;\n                        } else {\n                            //maybe the next one's logic offset is -1\n                            responseCode = ResponseCode.PULL_NOT_FOUND;\n                        }\n                    } else {\n                        //let it go\n                        responseCode = code;\n                    }\n                }\n\n                //read from the middle item, ignore the PULL_OFFSET_MOVED\n                if (!isRevised\n                    && leaderItem.getGen() != currentItem.getGen()\n                    && earlistItem.getGen() != currentItem.getGen()) {\n                    if (requestOffset < minOffset) {\n                        nextBeginOffset = minOffset;\n                        responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY;\n                    } else if (requestOffset >= maxOffset) {\n                        //just move to another item\n                        LogicQueueMappingItem nextItem = TopicQueueMappingUtils.findNext(mappingContext.getMappingItemList(), currentItem, true);\n                        if (nextItem != null) {\n                            currentItem = nextItem;\n                            nextBeginOffset = currentItem.getStartOffset();\n                            minOffset = currentItem.getStartOffset();\n                            maxOffset = minOffset;\n                            responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY;\n                        } else {\n                            //maybe the next one's logic offset is -1\n                            responseCode = ResponseCode.PULL_NOT_FOUND;\n                        }\n                    } else {\n                        responseCode = code;\n                    }\n                }\n            }\n\n            //handle nextBeginOffset\n            //the next begin offset should no more than the end offset\n            if (currentItem.checkIfEndOffsetDecided()\n                && nextBeginOffset >= currentItem.getEndOffset()) {\n                nextBeginOffset = currentItem.getEndOffset();\n            }\n            responseHeader.setNextBeginOffset(currentItem.computeStaticQueueOffsetStrictly(nextBeginOffset));\n            //handle min offset\n            responseHeader.setMinOffset(currentItem.computeStaticQueueOffsetStrictly(Math.max(currentItem.getStartOffset(), minOffset)));\n            //handle max offset\n            responseHeader.setMaxOffset(Math.max(currentItem.computeStaticQueueOffsetStrictly(maxOffset),\n                TopicQueueMappingDetail.computeMaxOffsetFromMapping(mappingDetail, mappingContext.getGlobalId())));\n            //set the offsetDelta\n            responseHeader.setOffsetDelta(currentItem.computeOffsetDelta());\n\n            if (code != ResponseCode.SUCCESS) {\n                return RemotingCommand.createResponseCommandWithHeader(responseCode, responseHeader);\n            } else {\n                return null;\n            }\n        } catch (Throwable t) {\n            LOGGER.warn(\"\", t);\n            return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.toString());\n        }\n    }\n\n    @Override\n    public RemotingCommand processRequest(final ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        return this.processRequest(ctx.channel(), request, true, true);\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        if (!this.brokerController.getBrokerConfig().isSlaveReadEnable()\n            && this.brokerController.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) {\n            return true;\n        }\n        return false;\n    }\n\n    private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend,\n        boolean brokerAllowFlowCtrSuspend)\n        throws RemotingCommandException {\n        final long beginTimeMills = this.brokerController.getMessageStore().now();\n        RemotingCommand response = RemotingCommand.createResponseCommand(PullMessageResponseHeader.class);\n        final PullMessageResponseHeader responseHeader = (PullMessageResponseHeader) response.readCustomHeader();\n        final PullMessageRequestHeader requestHeader =\n            (PullMessageRequestHeader) request.decodeCommandCustomHeader(PullMessageRequestHeader.class);\n\n        response.setOpaque(request.getOpaque());\n\n        LOGGER.debug(\"receive PullMessage request command, {}\", request);\n\n        if (!PermName.isReadable(this.brokerController.getBrokerConfig().getBrokerPermission())) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            responseHeader.setForbiddenType(ForbiddenType.BROKER_FORBIDDEN);\n            response.setRemark(String.format(\"the broker[%s] pulling message is forbidden\",\n                this.brokerController.getBrokerConfig().getBrokerIP1()));\n            return response;\n        }\n\n        if (request.getCode() == RequestCode.LITE_PULL_MESSAGE && !this.brokerController.getBrokerConfig().isLitePullMessageEnable()) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            responseHeader.setForbiddenType(ForbiddenType.BROKER_FORBIDDEN);\n            response.setRemark(\n                \"the broker[\" + this.brokerController.getBrokerConfig().getBrokerIP1() + \"] for lite pull consumer is forbidden\");\n            return response;\n        }\n\n        SubscriptionGroupConfig subscriptionGroupConfig =\n            this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getConsumerGroup());\n        if (null == subscriptionGroupConfig) {\n            response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);\n            response.setRemark(String.format(\"subscription group [%s] does not exist, %s\", requestHeader.getConsumerGroup(), FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST)));\n            return response;\n        }\n\n        if (!subscriptionGroupConfig.isConsumeEnable()) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            responseHeader.setForbiddenType(ForbiddenType.GROUP_FORBIDDEN);\n            response.setRemark(\"subscription group no permission, \" + requestHeader.getConsumerGroup());\n            return response;\n        }\n\n        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());\n        if (null == topicConfig) {\n            LOGGER.error(\"the topic {} not exist, consumer: {}\", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel));\n            response.setCode(ResponseCode.TOPIC_NOT_EXIST);\n            response.setRemark(String.format(\"topic[%s] not exist, apply first please! %s\", requestHeader.getTopic(), FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)));\n            return response;\n        }\n\n        if (!PermName.isReadable(topicConfig.getPerm())) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            responseHeader.setForbiddenType(ForbiddenType.TOPIC_FORBIDDEN);\n            response.setRemark(\"the topic[\" + requestHeader.getTopic() + \"] pulling message is forbidden\");\n            return response;\n        }\n\n        TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader, false);\n\n        {\n            RemotingCommand rewriteResult = rewriteRequestForStaticTopic(requestHeader, mappingContext);\n            if (rewriteResult != null) {\n                return rewriteResult;\n            }\n        }\n\n        if (requestHeader.getQueueId() < 0 || requestHeader.getQueueId() >= topicConfig.getReadQueueNums()) {\n            String errorInfo = String.format(\"queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]\",\n                requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());\n            LOGGER.warn(errorInfo);\n            response.setCode(ResponseCode.INVALID_PARAMETER);\n            response.setRemark(errorInfo);\n            return response;\n        }\n\n        ConsumerManager consumerManager = brokerController.getConsumerManager();\n        switch (RequestSource.parseInteger(requestHeader.getRequestSource())) {\n            case PROXY_FOR_BROADCAST:\n                consumerManager.compensateBasicConsumerInfo(requestHeader.getConsumerGroup(), ConsumeType.CONSUME_PASSIVELY, MessageModel.BROADCASTING);\n                break;\n            case PROXY_FOR_STREAM:\n                consumerManager.compensateBasicConsumerInfo(requestHeader.getConsumerGroup(), ConsumeType.CONSUME_ACTIVELY, MessageModel.CLUSTERING);\n                break;\n            default:\n                consumerManager.compensateBasicConsumerInfo(requestHeader.getConsumerGroup(), ConsumeType.CONSUME_PASSIVELY, MessageModel.CLUSTERING);\n                break;\n        }\n\n        SubscriptionData subscriptionData = null;\n        ConsumerFilterData consumerFilterData = null;\n        final boolean hasSubscriptionFlag = PullSysFlag.hasSubscriptionFlag(requestHeader.getSysFlag());\n        if (hasSubscriptionFlag) {\n            try {\n                subscriptionData = FilterAPI.build(\n                    requestHeader.getTopic(), requestHeader.getSubscription(), requestHeader.getExpressionType()\n                );\n                consumerManager.compensateSubscribeData(requestHeader.getConsumerGroup(), requestHeader.getTopic(), subscriptionData);\n\n                if (!ExpressionType.isTagType(subscriptionData.getExpressionType())) {\n                    consumerFilterData = ConsumerFilterManager.build(\n                        requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getSubscription(),\n                        requestHeader.getExpressionType(), requestHeader.getSubVersion()\n                    );\n                    assert consumerFilterData != null;\n                }\n            } catch (Exception e) {\n                LOGGER.warn(\"Parse the consumer's subscription[{}] failed, group: {}\", requestHeader.getSubscription(),\n                    requestHeader.getConsumerGroup());\n                response.setCode(ResponseCode.SUBSCRIPTION_PARSE_FAILED);\n                response.setRemark(\"parse the consumer's subscription failed\");\n                return response;\n            }\n        } else {\n            ConsumerGroupInfo consumerGroupInfo =\n                this.brokerController.getConsumerManager().getConsumerGroupInfo(requestHeader.getConsumerGroup());\n            if (null == consumerGroupInfo) {\n                LOGGER.warn(\"the consumer's group info not exist, group: {}\", requestHeader.getConsumerGroup());\n                response.setCode(ResponseCode.SUBSCRIPTION_NOT_EXIST);\n                response.setRemark(\"the consumer's group info not exist\" + FAQUrl.suggestTodo(FAQUrl.SAME_GROUP_DIFFERENT_TOPIC));\n                return response;\n            }\n\n            if (!subscriptionGroupConfig.isConsumeBroadcastEnable()\n                && consumerGroupInfo.getMessageModel() == MessageModel.BROADCASTING) {\n                response.setCode(ResponseCode.NO_PERMISSION);\n                responseHeader.setForbiddenType(ForbiddenType.BROADCASTING_DISABLE_FORBIDDEN);\n                response.setRemark(\"the consumer group[\" + requestHeader.getConsumerGroup() + \"] can not consume by broadcast way\");\n                return response;\n            }\n\n            boolean readForbidden = this.brokerController.getSubscriptionGroupManager().getForbidden(//\n                subscriptionGroupConfig.getGroupName(), requestHeader.getTopic(), PermName.INDEX_PERM_READ);\n            if (readForbidden) {\n                response.setCode(ResponseCode.NO_PERMISSION);\n                responseHeader.setForbiddenType(ForbiddenType.SUBSCRIPTION_FORBIDDEN);\n                response.setRemark(\"the consumer group[\" + requestHeader.getConsumerGroup() + \"] is forbidden for topic[\" + requestHeader.getTopic() + \"]\");\n                return response;\n            }\n\n            subscriptionData = consumerGroupInfo.findSubscriptionData(requestHeader.getTopic());\n            if (null == subscriptionData) {\n                LOGGER.warn(\"the consumer's subscription not exist, group: {}, topic:{}\", requestHeader.getConsumerGroup(), requestHeader.getTopic());\n                response.setCode(ResponseCode.SUBSCRIPTION_NOT_EXIST);\n                response.setRemark(\"the consumer's subscription not exist\" + FAQUrl.suggestTodo(FAQUrl.SAME_GROUP_DIFFERENT_TOPIC));\n                return response;\n            }\n\n            if (subscriptionData.getSubVersion() < requestHeader.getSubVersion()) {\n                LOGGER.warn(\"The broker's subscription is not latest, group: {} {}\", requestHeader.getConsumerGroup(),\n                    subscriptionData.getSubString());\n                response.setCode(ResponseCode.SUBSCRIPTION_NOT_LATEST);\n                response.setRemark(\"the consumer's subscription not latest\");\n                return response;\n            }\n            if (!ExpressionType.isTagType(subscriptionData.getExpressionType())) {\n                consumerFilterData = this.brokerController.getConsumerFilterManager().get(requestHeader.getTopic(),\n                    requestHeader.getConsumerGroup());\n                if (consumerFilterData == null) {\n                    response.setCode(ResponseCode.FILTER_DATA_NOT_EXIST);\n                    response.setRemark(\"The broker's consumer filter data is not exist!Your expression may be wrong!\");\n                    return response;\n                }\n                if (consumerFilterData.getClientVersion() < requestHeader.getSubVersion()) {\n                    LOGGER.warn(\"The broker's consumer filter data is not latest, group: {}, topic: {}, serverV: {}, clientV: {}\",\n                        requestHeader.getConsumerGroup(), requestHeader.getTopic(), consumerFilterData.getClientVersion(), requestHeader.getSubVersion());\n                    response.setCode(ResponseCode.FILTER_DATA_NOT_LATEST);\n                    response.setRemark(\"the consumer's consumer filter data not latest\");\n                    return response;\n                }\n            }\n        }\n\n        if (!ExpressionType.isTagType(subscriptionData.getExpressionType())\n            && !this.brokerController.getBrokerConfig().isEnablePropertyFilter()) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"The broker does not support consumer to filter message by \" + subscriptionData.getExpressionType());\n            return response;\n        }\n\n        MessageFilter messageFilter;\n        if (this.brokerController.getBrokerConfig().isFilterSupportRetry()) {\n            messageFilter = new ExpressionForRetryMessageFilter(subscriptionData, consumerFilterData,\n                this.brokerController.getConsumerFilterManager());\n        } else {\n            messageFilter = new ExpressionMessageFilter(subscriptionData, consumerFilterData,\n                this.brokerController.getConsumerFilterManager());\n        }\n\n        if (brokerController.getBrokerConfig().isRejectPullConsumerEnable()) {\n            ConsumerGroupInfo consumerGroupInfo =\n                    this.brokerController.getConsumerManager().getConsumerGroupInfo(requestHeader.getConsumerGroup());\n            if (null == consumerGroupInfo || ConsumeType.CONSUME_ACTIVELY == consumerGroupInfo.getConsumeType()) {\n                if ((null == consumerGroupInfo || null == consumerGroupInfo.findChannel(channel))\n                        && !MixAll.isSysConsumerGroupPullMessage(requestHeader.getConsumerGroup())) {\n                    response.setCode(ResponseCode.SUBSCRIPTION_NOT_EXIST);\n                    response.setRemark(\"the consumer's group info not exist, or the pull consumer is rejected by server.\" + FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST));\n                    return response;\n                }\n            }\n        }\n\n        final MessageStore messageStore = brokerController.getMessageStore();\n        if (this.brokerController.getMessageStore() instanceof DefaultMessageStore) {\n            DefaultMessageStore defaultMessageStore = (DefaultMessageStore) this.brokerController.getMessageStore();\n            boolean cgNeedColdDataFlowCtr = brokerController.getColdDataCgCtrService().isCgNeedColdDataFlowCtr(requestHeader.getConsumerGroup());\n            if (cgNeedColdDataFlowCtr) {\n                boolean isMsgLogicCold = defaultMessageStore.getCommitLog()\n                    .getColdDataCheckService().isMsgInColdArea(requestHeader.getConsumerGroup(),\n                        requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getQueueOffset());\n                if (isMsgLogicCold) {\n                    ConsumeType consumeType = this.brokerController.getConsumerManager().getConsumerGroupInfo(requestHeader.getConsumerGroup()).getConsumeType();\n                    if (consumeType == ConsumeType.CONSUME_PASSIVELY) {\n                        response.setCode(ResponseCode.SYSTEM_BUSY);\n                        response.setRemark(\"This consumer group is reading cold data. It has been flow control\");\n                        return response;\n                    } else if (consumeType == ConsumeType.CONSUME_ACTIVELY) {\n                        if (brokerAllowFlowCtrSuspend) {  // second arrived, which will not be held\n                            PullRequest pullRequest = new PullRequest(request, channel, 1000,\n                                this.brokerController.getMessageStore().now(), requestHeader.getQueueOffset(), subscriptionData, messageFilter);\n                            this.brokerController.getColdDataPullRequestHoldService().suspendColdDataReadRequest(pullRequest);\n                            return null;\n                        }\n                        requestHeader.setMaxMsgNums(1);\n                    }\n                }\n            }\n        }\n\n        final boolean useResetOffsetFeature = brokerController.getBrokerConfig().isUseServerSideResetOffset();\n        String topic = requestHeader.getTopic();\n        String liteTopic = requestHeader.getLiteTopic();\n        String group = requestHeader.getConsumerGroup();\n        int queueId = requestHeader.getQueueId();\n        Long resetOffset = brokerController.getConsumerOffsetManager().queryThenEraseResetOffset(topic, group, queueId);\n\n        GetMessageResult getMessageResult = null;\n        if (useResetOffsetFeature && null != resetOffset) {\n            getMessageResult = new GetMessageResult();\n            getMessageResult.setStatus(GetMessageStatus.OFFSET_RESET);\n            getMessageResult.setNextBeginOffset(resetOffset);\n            getMessageResult.setMinOffset(messageStore.getMinOffsetInQueue(topic, queueId));\n            try {\n                getMessageResult.setMaxOffset(messageStore.getMaxOffsetInQueue(topic, queueId));\n            } catch (ConsumeQueueException e) {\n                throw new RemotingCommandException(\"Failed tp get max offset in queue\", e);\n            }\n            getMessageResult.setSuggestPullingFromSlave(false);\n        } else {\n            long broadcastInitOffset = queryBroadcastPullInitOffset(topic, group, queueId, requestHeader, channel);\n            if (broadcastInitOffset >= 0) {\n                getMessageResult = new GetMessageResult();\n                getMessageResult.setStatus(GetMessageStatus.OFFSET_RESET);\n                getMessageResult.setNextBeginOffset(broadcastInitOffset);\n            } else {\n                SubscriptionData finalSubscriptionData = subscriptionData;\n                RemotingCommand finalResponse = response;\n                String storeTopic = topic;\n                if (StringUtils.isNotBlank(liteTopic)) {\n                    storeTopic = LiteUtil.toLmqName(topic, liteTopic);\n                }\n                messageStore.getMessageAsync(group, storeTopic, queueId, requestHeader.getQueueOffset(),\n                        requestHeader.getMaxMsgNums(), messageFilter)\n                    .thenApply(result -> {\n                        if (null == result) {\n                            finalResponse.setCode(ResponseCode.SYSTEM_ERROR);\n                            finalResponse.setRemark(\"store getMessage return null\");\n                            return finalResponse;\n                        }\n                        brokerController.getColdDataCgCtrService().coldAcc(requestHeader.getConsumerGroup(), result.getColdDataSum());\n                        return pullMessageResultHandler.handle(\n                            result,\n                            request,\n                            requestHeader,\n                            channel,\n                            finalSubscriptionData,\n                            subscriptionGroupConfig,\n                            brokerAllowSuspend,\n                            messageFilter,\n                            finalResponse,\n                            mappingContext,\n                            beginTimeMills\n                        );\n                    })\n                    .thenAccept(result -> NettyRemotingAbstract.writeResponse(channel, request, result, null, brokerController.getBrokerMetricsManager().getRemotingMetricsManager()));\n            }\n        }\n\n        if (getMessageResult != null) {\n\n            return this.pullMessageResultHandler.handle(\n                getMessageResult,\n                request,\n                requestHeader,\n                channel,\n                subscriptionData,\n                subscriptionGroupConfig,\n                brokerAllowSuspend,\n                messageFilter,\n                response,\n                mappingContext,\n                beginTimeMills\n            );\n        }\n        return null;\n    }\n\n    public boolean hasConsumeMessageHook() {\n        return consumeMessageHookList != null && !this.consumeMessageHookList.isEmpty();\n    }\n\n    /**\n     * Composes the header of the response message to be sent back to the client\n     *\n     * @param requestHeader           - the header of the request message\n     * @param getMessageResult        - the result of the GetMessage request\n     * @param topicSysFlag            - the system flag of the topic\n     * @param subscriptionGroupConfig - configuration of the subscription group\n     * @param response                - the response message to be sent back to the client\n     * @param clientAddress           - the address of the client\n     */\n    protected void composeResponseHeader(PullMessageRequestHeader requestHeader, GetMessageResult getMessageResult,\n        int topicSysFlag, SubscriptionGroupConfig subscriptionGroupConfig, RemotingCommand response,\n        String clientAddress) {\n        final PullMessageResponseHeader responseHeader = (PullMessageResponseHeader) response.readCustomHeader();\n        response.setRemark(getMessageResult.getStatus().name());\n        responseHeader.setNextBeginOffset(getMessageResult.getNextBeginOffset());\n        responseHeader.setMinOffset(getMessageResult.getMinOffset());\n        // this does not need to be modified since it's not an accurate value under logical queue.\n        responseHeader.setMaxOffset(getMessageResult.getMaxOffset());\n        responseHeader.setTopicSysFlag(topicSysFlag);\n        responseHeader.setGroupSysFlag(subscriptionGroupConfig.getGroupSysFlag());\n\n        switch (getMessageResult.getStatus()) {\n            case FOUND:\n                response.setCode(ResponseCode.SUCCESS);\n                break;\n            case MESSAGE_WAS_REMOVING:\n            case NO_MATCHED_MESSAGE:\n                response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY);\n                break;\n            case NO_MATCHED_LOGIC_QUEUE:\n            case NO_MESSAGE_IN_QUEUE:\n                if (0 != requestHeader.getQueueOffset()) {\n                    response.setCode(ResponseCode.PULL_OFFSET_MOVED);\n                    // XXX: warn and notify me\n                    LOGGER.info(\"the broker stores no queue data, fix the request offset {} to {}, Topic: {} QueueId: {} Consumer Group: {}\",\n                        requestHeader.getQueueOffset(),\n                        getMessageResult.getNextBeginOffset(),\n                        requestHeader.getTopic(),\n                        requestHeader.getQueueId(),\n                        requestHeader.getConsumerGroup()\n                    );\n                } else {\n                    response.setCode(ResponseCode.PULL_NOT_FOUND);\n                }\n                break;\n            case OFFSET_FOUND_NULL:\n            case OFFSET_OVERFLOW_ONE:\n                response.setCode(ResponseCode.PULL_NOT_FOUND);\n                break;\n            case OFFSET_OVERFLOW_BADLY:\n                response.setCode(ResponseCode.PULL_OFFSET_MOVED);\n                // XXX: warn and notify me\n                LOGGER.info(\"the request offset: {} over flow badly, fix to {}, broker max offset: {}, consumer: {}\",\n                    requestHeader.getQueueOffset(), getMessageResult.getNextBeginOffset(), getMessageResult.getMaxOffset(), clientAddress);\n                break;\n            case OFFSET_RESET:\n                response.setCode(ResponseCode.PULL_OFFSET_MOVED);\n                LOGGER.info(\"The queue under pulling was previously reset to start from {}\",\n                    getMessageResult.getNextBeginOffset());\n                break;\n            case OFFSET_TOO_SMALL:\n                response.setCode(ResponseCode.PULL_OFFSET_MOVED);\n                LOGGER.info(\"the request offset too small. group={}, topic={}, requestOffset={}, brokerMinOffset={}, clientIp={}\",\n                    requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueOffset(),\n                    getMessageResult.getMinOffset(), clientAddress);\n                break;\n            default:\n                assert false;\n                break;\n        }\n\n        if (this.brokerController.getBrokerConfig().isSlaveReadEnable() && !this.brokerController.getBrokerConfig().isInBrokerContainer()) {\n            // consume too slow ,redirect to another machine\n            if (getMessageResult.isSuggestPullingFromSlave()) {\n                responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getWhichBrokerWhenConsumeSlowly());\n            }\n            // consume ok\n            else {\n                responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getBrokerId());\n            }\n        } else {\n            responseHeader.setSuggestWhichBrokerId(MixAll.MASTER_ID);\n        }\n\n        if (this.brokerController.getBrokerConfig().getBrokerId() != MixAll.MASTER_ID && !getMessageResult.isSuggestPullingFromSlave()) {\n            if (this.brokerController.getMinBrokerIdInGroup() == MixAll.MASTER_ID) {\n                LOGGER.debug(\"slave redirect pullRequest to master, topic: {}, queueId: {}, consumer group: {}, next: {}, min: {}, max: {}\",\n                    requestHeader.getTopic(),\n                    requestHeader.getQueueId(),\n                    requestHeader.getConsumerGroup(),\n                    responseHeader.getNextBeginOffset(),\n                    responseHeader.getMinOffset(),\n                    responseHeader.getMaxOffset()\n                );\n                responseHeader.setSuggestWhichBrokerId(MixAll.MASTER_ID);\n                if (!getMessageResult.getStatus().equals(GetMessageStatus.FOUND)) {\n                    response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY);\n                }\n            }\n        }\n\n    }\n\n    protected void executeConsumeMessageHookBefore(RemotingCommand request, PullMessageRequestHeader requestHeader,\n        GetMessageResult getMessageResult, boolean brokerAllowSuspend, int responseCode) {\n        if (this.hasConsumeMessageHook()) {\n            String owner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER);\n            String authType = request.getExtFields().get(BrokerStatsManager.ACCOUNT_AUTH_TYPE);\n            String ownerParent = request.getExtFields().get(BrokerStatsManager.ACCOUNT_OWNER_PARENT);\n            String ownerSelf = request.getExtFields().get(BrokerStatsManager.ACCOUNT_OWNER_SELF);\n\n            ConsumeMessageContext context = new ConsumeMessageContext();\n            context.setConsumerGroup(requestHeader.getConsumerGroup());\n            context.setTopic(requestHeader.getTopic());\n            context.setQueueId(requestHeader.getQueueId());\n            context.setAccountAuthType(authType);\n            context.setAccountOwnerParent(ownerParent);\n            context.setAccountOwnerSelf(ownerSelf);\n            context.setNamespace(NamespaceUtil.getNamespaceFromResource(requestHeader.getTopic()));\n            context.setFilterMessageCount(getMessageResult.getFilterMessageCount());\n\n            switch (responseCode) {\n                case ResponseCode.SUCCESS:\n                    int commercialBaseCount = brokerController.getBrokerConfig().getCommercialBaseCount();\n                    int incValue = getMessageResult.getMsgCount4Commercial() * commercialBaseCount;\n\n                    context.setCommercialRcvStats(BrokerStatsManager.StatsType.RCV_SUCCESS);\n                    context.setCommercialRcvTimes(incValue);\n                    context.setCommercialRcvSize(getMessageResult.getBufferTotalSize());\n                    context.setCommercialOwner(owner);\n\n                    context.setRcvStat(BrokerStatsManager.StatsType.RCV_SUCCESS);\n                    context.setRcvMsgNum(getMessageResult.getMessageCount());\n                    context.setRcvMsgSize(getMessageResult.getBufferTotalSize());\n                    context.setCommercialRcvMsgNum(getMessageResult.getMsgCount4Commercial());\n\n                    break;\n                case ResponseCode.PULL_NOT_FOUND:\n                    if (!brokerAllowSuspend) {\n\n                        context.setCommercialRcvStats(BrokerStatsManager.StatsType.RCV_EPOLLS);\n                        context.setCommercialRcvTimes(1);\n                        context.setCommercialOwner(owner);\n\n                        context.setRcvStat(BrokerStatsManager.StatsType.RCV_EPOLLS);\n                        context.setRcvMsgNum(0);\n                        context.setRcvMsgSize(0);\n                        context.setCommercialRcvMsgNum(0);\n                    }\n                    break;\n                case ResponseCode.PULL_RETRY_IMMEDIATELY:\n                case ResponseCode.PULL_OFFSET_MOVED:\n                    context.setCommercialRcvStats(BrokerStatsManager.StatsType.RCV_EPOLLS);\n                    context.setCommercialRcvTimes(1);\n                    context.setCommercialOwner(owner);\n\n                    context.setRcvStat(BrokerStatsManager.StatsType.RCV_EPOLLS);\n                    context.setRcvMsgNum(0);\n                    context.setRcvMsgSize(0);\n                    context.setCommercialRcvMsgNum(0);\n                    break;\n                default:\n                    assert false;\n                    break;\n            }\n\n            for (ConsumeMessageHook hook : this.consumeMessageHookList) {\n                try {\n                    hook.consumeMessageBefore(context);\n                } catch (Throwable ignored) {\n                }\n            }\n        }\n    }\n\n    protected void tryCommitOffset(boolean brokerAllowSuspend, PullMessageRequestHeader requestHeader,\n        long nextOffset, String clientAddress) {\n        this.brokerController.getConsumerOffsetManager().commitPullOffset(clientAddress,\n            requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId(), nextOffset);\n\n        boolean storeOffsetEnable = brokerAllowSuspend;\n        final boolean hasCommitOffsetFlag = PullSysFlag.hasCommitOffsetFlag(requestHeader.getSysFlag());\n        storeOffsetEnable = storeOffsetEnable && hasCommitOffsetFlag;\n        if (storeOffsetEnable) {\n            this.brokerController.getConsumerOffsetManager().commitOffset(clientAddress, requestHeader.getConsumerGroup(),\n                requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getCommitOffset());\n        }\n    }\n\n    public void executeRequestWhenWakeup(final Channel channel, final RemotingCommand request) {\n        Runnable run = () -> {\n            try {\n                boolean brokerAllowFlowCtrSuspend = !(request.getExtFields() != null && request.getExtFields().containsKey(ColdDataPullRequestHoldService.NO_SUSPEND_KEY));\n                final RemotingCommand response = PullMessageProcessor.this.processRequest(channel, request, false, brokerAllowFlowCtrSuspend);\n\n                if (response != null) {\n                    response.setOpaque(request.getOpaque());\n                    response.markResponseType();\n                    try {\n                        NettyRemotingAbstract.writeResponse(channel, request, response, future -> {\n                            if (!future.isSuccess()) {\n                                LOGGER.error(\"processRequestWrapper response to {} failed\", channel.remoteAddress(), future.cause());\n                                LOGGER.error(request.toString());\n                                LOGGER.error(response.toString());\n                            }\n                        }, brokerController.getBrokerMetricsManager().getRemotingMetricsManager());\n                    } catch (Throwable e) {\n                        LOGGER.error(\"processRequestWrapper process request over, but response failed\", e);\n                        LOGGER.error(request.toString());\n                        LOGGER.error(response.toString());\n                    }\n                }\n            } catch (RemotingCommandException e1) {\n                LOGGER.error(\"executeRequestWhenWakeup run\", e1);\n            }\n        };\n        this.brokerController.getPullMessageExecutor().submit(new RequestTask(run, channel, request));\n    }\n\n    public void registerConsumeMessageHook(List<ConsumeMessageHook> consumeMessageHookList) {\n        this.consumeMessageHookList = consumeMessageHookList;\n    }\n\n    public void setPullMessageResultHandler(PullMessageResultHandler pullMessageResultHandler) {\n        this.pullMessageResultHandler = pullMessageResultHandler;\n    }\n\n    private boolean isBroadcast(boolean proxyPullBroadcast, ConsumerGroupInfo consumerGroupInfo) {\n        return proxyPullBroadcast ||\n            consumerGroupInfo != null\n                && MessageModel.BROADCASTING.equals(consumerGroupInfo.getMessageModel())\n                && ConsumeType.CONSUME_PASSIVELY.equals(consumerGroupInfo.getConsumeType());\n    }\n\n    protected void updateBroadcastPulledOffset(String topic, String group, int queueId,\n        PullMessageRequestHeader requestHeader, Channel channel, RemotingCommand response, long nextBeginOffset) {\n\n        if (response == null || !this.brokerController.getBrokerConfig().isEnableBroadcastOffsetStore()) {\n            return;\n        }\n\n        boolean proxyPullBroadcast = Objects.equals(\n            RequestSource.PROXY_FOR_BROADCAST.getValue(), requestHeader.getRequestSource());\n        ConsumerGroupInfo consumerGroupInfo = this.brokerController.getConsumerManager().getConsumerGroupInfo(group);\n\n        if (isBroadcast(proxyPullBroadcast, consumerGroupInfo)) {\n            long offset = requestHeader.getQueueOffset();\n            if (ResponseCode.PULL_OFFSET_MOVED == response.getCode()) {\n                offset = nextBeginOffset;\n            }\n            String clientId;\n            if (proxyPullBroadcast) {\n                clientId = requestHeader.getProxyFrowardClientId();\n            } else {\n                ClientChannelInfo clientChannelInfo = consumerGroupInfo.findChannel(channel);\n                if (clientChannelInfo == null) {\n                    return;\n                }\n                clientId = clientChannelInfo.getClientId();\n            }\n            this.brokerController.getBroadcastOffsetManager()\n                .updateOffset(topic, group, queueId, offset, clientId, proxyPullBroadcast);\n        }\n    }\n\n    /**\n     * When pull request is not broadcast or not return -1\n     */\n    protected long queryBroadcastPullInitOffset(String topic, String group, int queueId,\n        PullMessageRequestHeader requestHeader, Channel channel) throws RemotingCommandException {\n\n        if (!this.brokerController.getBrokerConfig().isEnableBroadcastOffsetStore()) {\n            return -1L;\n        }\n\n        ConsumerGroupInfo consumerGroupInfo = this.brokerController.getConsumerManager().getConsumerGroupInfo(group);\n        boolean proxyPullBroadcast = Objects.equals(\n            RequestSource.PROXY_FOR_BROADCAST.getValue(), requestHeader.getRequestSource());\n\n        if (isBroadcast(proxyPullBroadcast, consumerGroupInfo)) {\n            String clientId;\n            if (proxyPullBroadcast) {\n                clientId = requestHeader.getProxyFrowardClientId();\n            } else {\n                ClientChannelInfo clientChannelInfo = consumerGroupInfo.findChannel(channel);\n                if (clientChannelInfo == null) {\n                    return -1;\n                }\n                clientId = clientChannelInfo.getClientId();\n            }\n\n            try {\n                return this.brokerController.getBroadcastOffsetManager()\n                    .queryInitOffset(topic, group, queueId, clientId, requestHeader.getQueueOffset(), proxyPullBroadcast);\n            } catch (ConsumeQueueException e) {\n                throw new RemotingCommandException(\"Failed to query initial offset\", e);\n            }\n        }\n        return -1L;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.client.ConsumerGroupInfo;\nimport org.apache.rocketmq.broker.loadbalance.MessageRequestModeManager;\nimport org.apache.rocketmq.broker.topic.TopicRouteInfoManager;\nimport org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;\nimport org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely;\nimport org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragelyByCircle;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.message.MessageQueueAssignment;\nimport org.apache.rocketmq.common.message.MessageRequestMode;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.QueryAssignmentRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.QueryAssignmentResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.SetMessageRequestModeRequestBody;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\n\npublic class QueryAssignmentProcessor implements NettyRequestProcessor {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n\n    private final BrokerController brokerController;\n\n    private final ConcurrentHashMap<String, AllocateMessageQueueStrategy> name2LoadStrategy = new ConcurrentHashMap<>();\n\n    private MessageRequestModeManager messageRequestModeManager;\n\n    public QueryAssignmentProcessor(final BrokerController brokerController) {\n        this.brokerController = brokerController;\n\n        //register strategy\n        //NOTE: init with broker's log instead of init with ClientLogger.getLog();\n        AllocateMessageQueueAveragely allocateMessageQueueAveragely = new AllocateMessageQueueAveragely();\n        name2LoadStrategy.put(allocateMessageQueueAveragely.getName(), allocateMessageQueueAveragely);\n        AllocateMessageQueueAveragelyByCircle allocateMessageQueueAveragelyByCircle = new AllocateMessageQueueAveragelyByCircle();\n        name2LoadStrategy.put(allocateMessageQueueAveragelyByCircle.getName(), allocateMessageQueueAveragelyByCircle);\n\n        this.messageRequestModeManager = new MessageRequestModeManager(brokerController);\n        this.messageRequestModeManager.load();\n    }\n\n    @Override\n    public RemotingCommand processRequest(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        switch (request.getCode()) {\n            case RequestCode.QUERY_ASSIGNMENT:\n                return this.queryAssignment(ctx, request);\n            case RequestCode.SET_MESSAGE_REQUEST_MODE:\n                return this.setMessageRequestMode(ctx, request);\n            default:\n                break;\n        }\n        return null;\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n\n    /**\n     *\n     */\n    private RemotingCommand queryAssignment(ChannelHandlerContext ctx, RemotingCommand request)\n        throws RemotingCommandException {\n        final QueryAssignmentRequestBody requestBody = QueryAssignmentRequestBody.decode(request.getBody(), QueryAssignmentRequestBody.class);\n        final String topic = requestBody.getTopic();\n        final String consumerGroup = requestBody.getConsumerGroup();\n        final String clientId = requestBody.getClientId();\n        final MessageModel messageModel = requestBody.getMessageModel();\n        final String strategyName = requestBody.getStrategyName();\n\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final QueryAssignmentResponseBody responseBody = new QueryAssignmentResponseBody();\n\n        SetMessageRequestModeRequestBody setMessageRequestModeRequestBody = this.messageRequestModeManager.getMessageRequestMode(topic, consumerGroup);\n\n        if (setMessageRequestModeRequestBody == null) {\n            setMessageRequestModeRequestBody = new SetMessageRequestModeRequestBody();\n            setMessageRequestModeRequestBody.setTopic(topic);\n            setMessageRequestModeRequestBody.setConsumerGroup(consumerGroup);\n\n            if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n                // retry topic must be pull mode\n                setMessageRequestModeRequestBody.setMode(MessageRequestMode.PULL);\n            } else {\n                setMessageRequestModeRequestBody.setMode(brokerController.getBrokerConfig().getDefaultMessageRequestMode());\n            }\n\n            if (setMessageRequestModeRequestBody.getMode() == MessageRequestMode.POP) {\n                setMessageRequestModeRequestBody.setPopShareQueueNum(brokerController.getBrokerConfig().getDefaultPopShareQueueNum());\n            }\n        }\n\n        Set<MessageQueue> messageQueues = doLoadBalance(topic, consumerGroup, clientId, messageModel, strategyName, setMessageRequestModeRequestBody, ctx);\n\n        Set<MessageQueueAssignment> assignments = null;\n        if (messageQueues != null) {\n            assignments = new HashSet<>();\n            for (MessageQueue messageQueue : messageQueues) {\n                MessageQueueAssignment messageQueueAssignment = new MessageQueueAssignment();\n                messageQueueAssignment.setMessageQueue(messageQueue);\n                if (setMessageRequestModeRequestBody != null) {\n                    messageQueueAssignment.setMode(setMessageRequestModeRequestBody.getMode());\n                }\n                assignments.add(messageQueueAssignment);\n            }\n        }\n\n        responseBody.setMessageQueueAssignments(assignments);\n        response.setBody(responseBody.encode());\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    /**\n     * Returns empty set means the client should clear all load assigned to it before, null means invalid result and the\n     * client should skip the update logic\n     *\n     * @param topic\n     * @param consumerGroup\n     * @param clientId\n     * @param messageModel\n     * @param strategyName\n     * @return the MessageQueues assigned to this client\n     */\n    private Set<MessageQueue> doLoadBalance(final String topic, final String consumerGroup, final String clientId,\n        final MessageModel messageModel, final String strategyName,\n        SetMessageRequestModeRequestBody setMessageRequestModeRequestBody, final ChannelHandlerContext ctx) {\n        Set<MessageQueue> assignedQueueSet = null;\n        final TopicRouteInfoManager topicRouteInfoManager = this.brokerController.getTopicRouteInfoManager();\n\n        switch (messageModel) {\n            case BROADCASTING: {\n                assignedQueueSet = topicRouteInfoManager.getTopicSubscribeInfo(topic);\n                if (assignedQueueSet == null) {\n                    log.warn(\"QueryLoad: no assignment for group[{}], the topic[{}] does not exist.\", consumerGroup, topic);\n                }\n                break;\n            }\n            case CLUSTERING: {\n                Set<MessageQueue> mqSet;\n                if (MixAll.isLmq(topic)) {\n                    mqSet = new HashSet<>();\n                    mqSet.add(new MessageQueue(\n                        topic, brokerController.getBrokerConfig().getBrokerName(), (int)MixAll.LMQ_QUEUE_ID));\n                } else {\n                    mqSet = topicRouteInfoManager.getTopicSubscribeInfo(topic);\n                }\n                if (null == mqSet) {\n                    if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n                        log.warn(\"QueryLoad: no assignment for group[{}], the topic[{}] does not exist.\", consumerGroup, topic);\n                    }\n                    return null;\n                }\n\n                if (!brokerController.getBrokerConfig().isServerLoadBalancerEnable()) {\n                    return mqSet;\n                }\n\n                List<String> cidAll = null;\n                ConsumerGroupInfo consumerGroupInfo = this.brokerController.getConsumerManager().getConsumerGroupInfo(consumerGroup);\n                if (consumerGroupInfo != null) {\n                    cidAll = consumerGroupInfo.getAllClientId();\n                }\n                if (null == cidAll) {\n                    log.warn(\"QueryLoad: no assignment for group[{}] topic[{}], get consumer id list failed\", consumerGroup, topic);\n                    return null;\n                }\n\n                List<MessageQueue> mqAll = new ArrayList<>();\n                mqAll.addAll(mqSet);\n                Collections.sort(mqAll);\n                Collections.sort(cidAll);\n                List<MessageQueue> allocateResult = null;\n\n                try {\n                    AllocateMessageQueueStrategy allocateMessageQueueStrategy = name2LoadStrategy.get(strategyName);\n                    if (null == allocateMessageQueueStrategy) {\n                        log.warn(\"QueryLoad: unsupported strategy [{}],  {}\", strategyName, RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n                        return null;\n                    }\n\n                    if (setMessageRequestModeRequestBody != null && setMessageRequestModeRequestBody.getMode() == MessageRequestMode.POP) {\n                        allocateResult = allocate4Pop(allocateMessageQueueStrategy, consumerGroup, clientId, mqAll,\n                            cidAll, setMessageRequestModeRequestBody.getPopShareQueueNum());\n\n                    } else {\n                        allocateResult = allocateMessageQueueStrategy.allocate(consumerGroup, clientId, mqAll, cidAll);\n                    }\n                } catch (Throwable e) {\n                    log.error(\"QueryLoad: no assignment for group[{}] topic[{}], allocate message queue exception. strategy name: {}, ex: {}\", consumerGroup, topic, strategyName, e);\n                    return null;\n                }\n\n                assignedQueueSet = new HashSet<>();\n                if (allocateResult != null) {\n                    assignedQueueSet.addAll(allocateResult);\n                }\n                break;\n            }\n            default:\n                break;\n        }\n        return assignedQueueSet;\n    }\n\n    public List<MessageQueue> allocate4Pop(AllocateMessageQueueStrategy allocateMessageQueueStrategy,\n        final String consumerGroup, final String clientId, List<MessageQueue> mqAll, List<String> cidAll,\n        int popShareQueueNum) {\n\n        List<MessageQueue> allocateResult;\n        if (popShareQueueNum <= 0 || popShareQueueNum >= cidAll.size() - 1) {\n            //each client pop all messagequeue\n            allocateResult = new ArrayList<>(mqAll.size());\n            for (MessageQueue mq : mqAll) {\n                //must create new MessageQueue in case of change cache in AssignmentManager\n                MessageQueue newMq = new MessageQueue(mq.getTopic(), mq.getBrokerName(), -1);\n                allocateResult.add(newMq);\n            }\n\n        } else {\n            if (cidAll.size() <= mqAll.size()) {\n                //consumer working in pop mode could share the MessageQueues assigned to the N (N = popWorkGroupSize) consumer following it in the cid list\n                allocateResult = allocateMessageQueueStrategy.allocate(consumerGroup, clientId, mqAll, cidAll);\n                int index = cidAll.indexOf(clientId);\n                if (index >= 0) {\n                    for (int i = 1; i <= popShareQueueNum; i++) {\n                        index++;\n                        index = index % cidAll.size();\n                        List<MessageQueue> tmp = allocateMessageQueueStrategy.allocate(consumerGroup, cidAll.get(index), mqAll, cidAll);\n                        allocateResult.addAll(tmp);\n                    }\n                }\n            } else {\n                //make sure each cid is assigned\n                allocateResult = allocate(consumerGroup, clientId, mqAll, cidAll);\n            }\n        }\n\n        return allocateResult;\n    }\n\n    private List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll,\n        List<String> cidAll) {\n        if (StringUtils.isBlank(currentCID)) {\n            throw new IllegalArgumentException(\"currentCID is empty\");\n        }\n\n        if (CollectionUtils.isEmpty(mqAll)) {\n            throw new IllegalArgumentException(\"mqAll is null or mqAll empty\");\n        }\n        if (CollectionUtils.isEmpty(cidAll)) {\n            throw new IllegalArgumentException(\"cidAll is null or cidAll empty\");\n        }\n\n        List<MessageQueue> result = new ArrayList<>();\n        if (!cidAll.contains(currentCID)) {\n            log.info(\"[BUG] ConsumerGroup: {} The consumerId: {} not in cidAll: {}\",\n                consumerGroup,\n                currentCID,\n                cidAll);\n            return result;\n        }\n\n        int index = cidAll.indexOf(currentCID);\n        result.add(mqAll.get(index % mqAll.size()));\n        return result;\n    }\n\n    private RemotingCommand setMessageRequestMode(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final SetMessageRequestModeRequestBody requestBody = SetMessageRequestModeRequestBody.decode(request.getBody(), SetMessageRequestModeRequestBody.class);\n\n        final String topic = requestBody.getTopic();\n        if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            response.setRemark(\"retry topic is not allowed to set mode\");\n            return response;\n        }\n        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);\n        if (null == topicConfig) {\n            response.setCode(ResponseCode.TOPIC_NOT_EXIST);\n            response.setRemark(\"topic[\" + topic + \"] not exist\");\n            return response;\n        }\n\n        final String consumerGroup = requestBody.getConsumerGroup();\n        SubscriptionGroupConfig groupConfig = this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(consumerGroup);\n        if (null == groupConfig) {\n            response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);\n            response.setRemark(\"subscription group does not exist\");\n            return response;\n        }\n\n        this.messageRequestModeManager.setMessageRequestMode(topic, consumerGroup, requestBody);\n        this.messageRequestModeManager.persist();\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    public MessageRequestModeManager getMessageRequestModeManager() {\n        return messageRequestModeManager;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/QueryMessageProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.ChannelFutureListener;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.channel.FileRegion;\nimport io.opentelemetry.api.common.Attributes;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.pagecache.OneMessageTransfer;\nimport org.apache.rocketmq.broker.pagecache.QueryMessageTransfer;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.metrics.RemotingMetricsManager;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ViewMessageRequestHeader;\nimport org.apache.rocketmq.store.QueryMessageResult;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\n\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_REQUEST_CODE;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_RESPONSE_CODE;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_RESULT;\n\npublic class QueryMessageProcessor implements NettyRequestProcessor {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private final BrokerController brokerController;\n\n    public QueryMessageProcessor(final BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    @Override\n    public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request)\n        throws RemotingCommandException {\n        switch (request.getCode()) {\n            case RequestCode.QUERY_MESSAGE:\n                return this.queryMessage(ctx, request);\n            case RequestCode.VIEW_MESSAGE_BY_ID:\n                return this.viewMessageById(ctx, request);\n            default:\n                break;\n        }\n\n        return null;\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n\n    public RemotingCommand queryMessage(ChannelHandlerContext ctx, RemotingCommand request)\n        throws RemotingCommandException {\n        final RemotingCommand response =\n            RemotingCommand.createResponseCommand(QueryMessageResponseHeader.class);\n        final QueryMessageResponseHeader responseHeader =\n            (QueryMessageResponseHeader) response.readCustomHeader();\n        final QueryMessageRequestHeader requestHeader =\n            (QueryMessageRequestHeader) request\n                .decodeCommandCustomHeader(QueryMessageRequestHeader.class);\n\n        response.setOpaque(request.getOpaque());\n        String indexType = requestHeader.getIndexType();\n        String lastKey = requestHeader.getLastKey();\n        String isUniqueKey = null;\n        if (null != request.getExtFields()) {\n            isUniqueKey = request.getExtFields().get(MixAll.UNIQUE_MSG_QUERY_FLAG);\n        }\n        if (!StringUtils.isEmpty(isUniqueKey) && Boolean.parseBoolean(isUniqueKey)) {\n            requestHeader.setMaxNum(this.brokerController.getMessageStoreConfig().getDefaultQueryMaxNum());\n            indexType = MessageConst.INDEX_UNIQUE_TYPE;\n        } else if (StringUtils.isEmpty(indexType)) {\n            indexType = MessageConst.INDEX_KEY_TYPE;\n        }\n        final QueryMessageResult queryMessageResult = this.brokerController.getMessageStore().queryMessage(requestHeader.getTopic(), requestHeader.getKey(), requestHeader.getMaxNum(), requestHeader.getBeginTimestamp(), requestHeader.getEndTimestamp(), indexType, lastKey);\n        assert queryMessageResult != null;\n\n        responseHeader.setIndexLastUpdatePhyoffset(queryMessageResult.getIndexLastUpdatePhyoffset());\n        responseHeader.setIndexLastUpdateTimestamp(queryMessageResult.getIndexLastUpdateTimestamp());\n\n        if (queryMessageResult.getBufferTotalSize() > 0) {\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n\n            try {\n                FileRegion fileRegion =\n                    new QueryMessageTransfer(response.encodeHeader(queryMessageResult\n                        .getBufferTotalSize()), queryMessageResult);\n                ctx.channel()\n                    .writeAndFlush(fileRegion)\n                    .addListener((ChannelFutureListener) future -> {\n                        queryMessageResult.release();\n                        RemotingMetricsManager remotingMetricsManager = brokerController.getBrokerMetricsManager().getRemotingMetricsManager();\n                        Attributes attributes = remotingMetricsManager.newAttributesBuilder()\n                            .put(LABEL_REQUEST_CODE, RemotingHelper.getRequestCodeDesc(request.getCode()))\n                            .put(LABEL_RESPONSE_CODE, RemotingHelper.getResponseCodeDesc(response.getCode()))\n                            .put(LABEL_RESULT, remotingMetricsManager.getWriteAndFlushResult(future))\n                            .build();\n                        remotingMetricsManager.getRpcLatency().record(request.getProcessTimer().elapsed(TimeUnit.MILLISECONDS), attributes);\n                        if (!future.isSuccess()) {\n                            LOGGER.error(\"transfer query message by page cache failed, \", future.cause());\n                        }\n                    });\n            } catch (Throwable e) {\n                LOGGER.error(\"\", e);\n                queryMessageResult.release();\n            }\n\n            return null;\n        }\n\n        response.setCode(ResponseCode.QUERY_NOT_FOUND);\n        response.setRemark(\"can not find message, maybe time range not correct\");\n        return response;\n    }\n\n    public RemotingCommand viewMessageById(ChannelHandlerContext ctx, RemotingCommand request)\n        throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final ViewMessageRequestHeader requestHeader =\n            (ViewMessageRequestHeader) request.decodeCommandCustomHeader(ViewMessageRequestHeader.class);\n\n        response.setOpaque(request.getOpaque());\n\n        final SelectMappedBufferResult selectMappedBufferResult =\n            this.brokerController.getMessageStore().selectOneMessageByOffset(requestHeader.getOffset());\n        if (selectMappedBufferResult != null) {\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n\n            try {\n                FileRegion fileRegion =\n                    new OneMessageTransfer(response.encodeHeader(selectMappedBufferResult.getSize()),\n                        selectMappedBufferResult);\n                ctx.channel()\n                    .writeAndFlush(fileRegion)\n                    .addListener((ChannelFutureListener) future -> {\n                        selectMappedBufferResult.release();\n                        RemotingMetricsManager remotingMetricsManager = brokerController.getBrokerMetricsManager().getRemotingMetricsManager();\n                        Attributes attributes = remotingMetricsManager.newAttributesBuilder()\n                            .put(LABEL_REQUEST_CODE, RemotingHelper.getRequestCodeDesc(request.getCode()))\n                            .put(LABEL_RESPONSE_CODE, RemotingHelper.getResponseCodeDesc(response.getCode()))\n                            .put(LABEL_RESULT, remotingMetricsManager.getWriteAndFlushResult(future))\n                            .build();\n                        remotingMetricsManager.getRpcLatency().record(request.getProcessTimer().elapsed(TimeUnit.MILLISECONDS), attributes);\n                        if (!future.isSuccess()) {\n                            LOGGER.error(\"Transfer one message from page cache failed, \", future.cause());\n                        }\n                    });\n            } catch (Throwable e) {\n                LOGGER.error(\"\", e);\n                selectMappedBufferResult.release();\n            }\n\n            return null;\n        } else {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"can not find message by the offset, \" + requestHeader.getOffset());\n        }\n\n        return response;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/RecallMessageProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.commons.codec.DecoderException;\nimport org.apache.commons.lang3.math.NumberUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.producer.RecallMessageHandle;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.timer.TimerMessageStore;\n\nimport java.nio.charset.StandardCharsets;\n\npublic class RecallMessageProcessor implements NettyRequestProcessor {\n    private static final String RECALL_MESSAGE_TAG = \"_RECALL_TAG_\";\n    private final BrokerController brokerController;\n\n    public RecallMessageProcessor(final BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    @Override\n    public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws\n            RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(RecallMessageResponseHeader.class);\n        response.addExtField(MessageConst.PROPERTY_MSG_REGION, this.brokerController.getBrokerConfig().getRegionId());\n        final RecallMessageRequestHeader requestHeader =\n            request.decodeCommandCustomHeader(RecallMessageRequestHeader.class);\n\n        if (!brokerController.getBrokerConfig().isRecallMessageEnable()) {\n            response.setCode(ResponseCode.NO_PERMISSION);\n            response.setRemark(\"recall failed, operation is forbidden\");\n            return response;\n        }\n\n        if (BrokerRole.SLAVE == brokerController.getMessageStoreConfig().getBrokerRole()) {\n            response.setCode(ResponseCode.SLAVE_NOT_AVAILABLE);\n            response.setRemark(\"recall failed, broker service not available\");\n            return response;\n        }\n\n        final long startTimestamp = this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp();\n        if (this.brokerController.getMessageStore().now() < startTimestamp) {\n            response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE);\n            response.setRemark(\"recall failed, broker service not available\");\n            return response;\n        }\n\n        if (!PermName.isWriteable(this.brokerController.getBrokerConfig().getBrokerPermission())\n            && !this.brokerController.getBrokerConfig().isAllowRecallWhenBrokerNotWriteable()) {\n            response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE);\n            response.setRemark(\"recall failed, broker service not available\");\n            return response;\n        }\n\n        TopicConfig topicConfig =\n            this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());\n        if (null == topicConfig) {\n            response.setCode(ResponseCode.TOPIC_NOT_EXIST);\n            response.setRemark(\"recall failed, the topic[\" + requestHeader.getTopic() + \"] not exist\");\n            return response;\n        }\n\n        RecallMessageHandle.HandleV1 handle;\n        try {\n            handle = (RecallMessageHandle.HandleV1) RecallMessageHandle.decodeHandle(requestHeader.getRecallHandle());\n        } catch (DecoderException e) {\n            response.setCode(ResponseCode.ILLEGAL_OPERATION);\n            response.setRemark(e.getMessage());\n            return response;\n        }\n\n        if (!requestHeader.getTopic().equals(handle.getTopic())) {\n            response.setCode(ResponseCode.ILLEGAL_OPERATION);\n            response.setRemark(\"recall failed, topic not match\");\n            return response;\n        }\n        if (!brokerController.getBrokerConfig().getBrokerName().equals(handle.getBrokerName())) {\n            response.setCode(ResponseCode.ILLEGAL_OPERATION);\n            response.setRemark(\"recall failed, broker service not available\");\n            return response;\n        }\n\n        long timestamp = NumberUtils.toLong(handle.getTimestampStr(), -1);\n        long timeLeft = timestamp - System.currentTimeMillis();\n        if (timeLeft <= 0\n            || timeLeft >= brokerController.getMessageStoreConfig().getTimerMaxDelaySec() * 1000L) {\n            response.setCode(ResponseCode.ILLEGAL_OPERATION);\n            response.setRemark(\"recall failed, timestamp invalid\");\n            return response;\n        }\n\n        MessageExtBrokerInner msgInner = buildMessage(ctx, requestHeader, handle);\n        long beginTimeMillis = this.brokerController.getMessageStore().now();\n        PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner);\n        handlePutMessageResult(putMessageResult, request, response, msgInner, ctx, beginTimeMillis);\n        return response;\n    }\n\n    public MessageExtBrokerInner buildMessage(ChannelHandlerContext ctx, RecallMessageRequestHeader requestHeader,\n        RecallMessageHandle.HandleV1 handle) {\n        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n        msgInner.setTopic(handle.getTopic());\n        msgInner.setBody(\"0\".getBytes(StandardCharsets.UTF_8));\n        msgInner.setTags(RECALL_MESSAGE_TAG);\n        msgInner.setTagsCode(RECALL_MESSAGE_TAG.hashCode());\n        msgInner.setQueueId(0);\n        MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TIMER_DEL_UNIQKEY, TimerMessageStore.buildDeleteKey(\n            handle.getTopic(), handle.getMessageId(), brokerController.getMessageStoreConfig().isAppendTopicForTimerDeleteKey()));\n        MessageAccessor.putProperty(msgInner,\n            MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, handle.getMessageId());\n        MessageAccessor.putProperty(msgInner,\n            MessageConst.PROPERTY_TIMER_DELIVER_MS, String.valueOf(handle.getTimestampStr()));\n        MessageAccessor.putProperty(msgInner,\n            MessageConst.PROPERTY_BORN_TIMESTAMP, String.valueOf(System.currentTimeMillis()));\n        MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRACE_CONTEXT, \"\");\n        MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_PRODUCER_GROUP, requestHeader.getProducerGroup());\n        msgInner.setBornTimestamp(System.currentTimeMillis());\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));\n        msgInner.setBornHost(ctx.channel().remoteAddress());\n        msgInner.setStoreHost(this.brokerController.getStoreHost());\n        return msgInner;\n    }\n\n    public void handlePutMessageResult(PutMessageResult putMessageResult, RemotingCommand request,\n        RemotingCommand response, MessageExt message, ChannelHandlerContext ctx, long beginTimeMillis) {\n        if (null == putMessageResult) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"recall failed, execute error\");\n            return;\n        }\n        RecallMessageResponseHeader responseHeader = (RecallMessageResponseHeader) response.readCustomHeader();\n        switch (putMessageResult.getPutMessageStatus()) {\n            case PUT_OK:\n                this.brokerController.getBrokerStatsManager().incTopicPutNums(\n                    message.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum(), 1); // system timer topic\n                this.brokerController.getBrokerStatsManager().incTopicPutSize(\n                    message.getTopic(), putMessageResult.getAppendMessageResult().getWroteBytes());\n                this.brokerController.getBrokerStatsManager().incBrokerPutNums(\n                    message.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum());\n                this.brokerController.getBrokerStatsManager().incTopicPutLatency(\n                    message.getTopic(), 0, (int) (this.brokerController.getMessageStore().now() - beginTimeMillis));\n            case FLUSH_DISK_TIMEOUT:\n            case FLUSH_SLAVE_TIMEOUT:\n            case SLAVE_NOT_AVAILABLE:\n                response.setCode(ResponseCode.SUCCESS);\n                responseHeader.setMsgId(MessageClientIDSetter.getUniqID(message));\n                break;\n            default:\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"recall failed, execute error\");\n                break;\n        }\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.opentelemetry.api.common.Attributes;\nimport java.net.InetSocketAddress;\nimport java.util.concurrent.ThreadLocalRandom;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.metrics.BrokerMetricsManager;\nimport org.apache.rocketmq.broker.mqtrace.SendMessageContext;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.ReplyMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\n\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_MESSAGE_TYPE;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_TOPIC;\n\npublic class ReplyMessageProcessor extends AbstractSendMessageProcessor {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n\n    public ReplyMessageProcessor(final BrokerController brokerController) {\n        super(brokerController);\n    }\n\n    @Override\n    public RemotingCommand processRequest(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        SendMessageContext mqtraceContext = null;\n        SendMessageRequestHeader requestHeader = parseRequestHeader(request);\n        if (requestHeader == null) {\n            return null;\n        }\n\n        mqtraceContext = buildMsgContext(ctx, requestHeader, request);\n        this.executeSendMessageHookBefore(mqtraceContext);\n\n        RemotingCommand response = this.processReplyMessageRequest(ctx, request, mqtraceContext, requestHeader);\n\n        this.executeSendMessageHookAfter(response, mqtraceContext);\n        return response;\n    }\n\n    @Override\n    protected SendMessageRequestHeader parseRequestHeader(RemotingCommand request) throws RemotingCommandException {\n        SendMessageRequestHeaderV2 requestHeaderV2 = null;\n        SendMessageRequestHeader requestHeader = null;\n        switch (request.getCode()) {\n            case RequestCode.SEND_REPLY_MESSAGE_V2:\n                requestHeaderV2 =\n                    (SendMessageRequestHeaderV2) request\n                        .decodeCommandCustomHeader(SendMessageRequestHeaderV2.class);\n            case RequestCode.SEND_REPLY_MESSAGE:\n                if (null == requestHeaderV2) {\n                    requestHeader =\n                        (SendMessageRequestHeader) request\n                            .decodeCommandCustomHeader(SendMessageRequestHeader.class);\n                } else {\n                    requestHeader = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV1(requestHeaderV2);\n                }\n            default:\n                break;\n        }\n        return requestHeader;\n    }\n\n    private RemotingCommand processReplyMessageRequest(final ChannelHandlerContext ctx,\n        final RemotingCommand request,\n        final SendMessageContext sendMessageContext,\n        final SendMessageRequestHeader requestHeader) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);\n        final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader) response.readCustomHeader();\n\n        response.setOpaque(request.getOpaque());\n\n        response.addExtField(MessageConst.PROPERTY_MSG_REGION, this.brokerController.getBrokerConfig().getRegionId());\n        response.addExtField(MessageConst.PROPERTY_TRACE_SWITCH, String.valueOf(this.brokerController.getBrokerConfig().isTraceOn()));\n\n        log.debug(\"receive SendReplyMessage request command, {}\", request);\n        final long startTimestamp = this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp();\n        if (this.brokerController.getMessageStore().now() < startTimestamp) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(String.format(\"broker unable to service, until %s\", UtilAll.timeMillisToHumanString2(startTimestamp)));\n            return response;\n        }\n\n        response.setCode(-1);\n        super.msgCheck(ctx, requestHeader, request, response);\n        if (response.getCode() != -1) {\n            return response;\n        }\n\n        final byte[] body = request.getBody();\n\n        int queueIdInt = requestHeader.getQueueId();\n        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());\n\n        if (queueIdInt < 0) {\n            queueIdInt = ThreadLocalRandom.current().nextInt(99999999) % topicConfig.getWriteQueueNums();\n        }\n\n        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n        msgInner.setTopic(requestHeader.getTopic());\n        msgInner.setQueueId(queueIdInt);\n        msgInner.setBody(body);\n        msgInner.setFlag(requestHeader.getFlag());\n        MessageAccessor.setProperties(msgInner, MessageDecoder.string2messageProperties(requestHeader.getProperties()));\n        msgInner.setPropertiesString(requestHeader.getProperties());\n        msgInner.setBornTimestamp(requestHeader.getBornTimestamp());\n        msgInner.setBornHost(ctx.channel().remoteAddress());\n        msgInner.setStoreHost(this.getStoreHost());\n        msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes());\n\n        PushReplyResult pushReplyResult = this.pushReplyMessage(ctx, requestHeader, msgInner);\n        this.handlePushReplyResult(pushReplyResult, response, responseHeader, queueIdInt);\n\n        if (this.brokerController.getBrokerConfig().isStoreReplyMessageEnable()) {\n            PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner);\n            this.handlePutMessageResult(putMessageResult, request, msgInner, responseHeader, sendMessageContext, queueIdInt, BrokerMetricsManager.getMessageType(requestHeader));\n        }\n\n        return response;\n    }\n\n    private PushReplyResult pushReplyMessage(final ChannelHandlerContext ctx,\n        final SendMessageRequestHeader requestHeader,\n        final Message msg) {\n        ReplyMessageRequestHeader replyMessageRequestHeader = new ReplyMessageRequestHeader();\n        InetSocketAddress bornAddress = (InetSocketAddress)(ctx.channel().remoteAddress());\n        replyMessageRequestHeader.setBornHost(bornAddress.getAddress().getHostAddress() + \":\" + bornAddress.getPort());\n        InetSocketAddress storeAddress = (InetSocketAddress)(this.getStoreHost());\n        replyMessageRequestHeader.setStoreHost(storeAddress.getAddress().getHostAddress() + \":\" + storeAddress.getPort());\n        replyMessageRequestHeader.setStoreTimestamp(System.currentTimeMillis());\n        replyMessageRequestHeader.setProducerGroup(requestHeader.getProducerGroup());\n        replyMessageRequestHeader.setTopic(requestHeader.getTopic());\n        replyMessageRequestHeader.setDefaultTopic(requestHeader.getDefaultTopic());\n        replyMessageRequestHeader.setDefaultTopicQueueNums(requestHeader.getDefaultTopicQueueNums());\n        replyMessageRequestHeader.setQueueId(requestHeader.getQueueId());\n        replyMessageRequestHeader.setSysFlag(requestHeader.getSysFlag());\n        replyMessageRequestHeader.setBornTimestamp(requestHeader.getBornTimestamp());\n        replyMessageRequestHeader.setFlag(requestHeader.getFlag());\n        replyMessageRequestHeader.setProperties(requestHeader.getProperties());\n        replyMessageRequestHeader.setReconsumeTimes(requestHeader.getReconsumeTimes());\n        replyMessageRequestHeader.setUnitMode(requestHeader.isUnitMode());\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PUSH_REPLY_MESSAGE_TO_CLIENT, replyMessageRequestHeader);\n        request.setBody(msg.getBody());\n\n        String senderId = msg.getProperties().get(MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT);\n        PushReplyResult pushReplyResult = new PushReplyResult(false);\n\n        if (senderId != null) {\n            Channel channel = this.brokerController.getProducerManager().findChannel(senderId);\n            if (channel != null) {\n                msg.getProperties().put(MessageConst.PROPERTY_PUSH_REPLY_TIME, String.valueOf(System.currentTimeMillis()));\n                replyMessageRequestHeader.setProperties(MessageDecoder.messageProperties2String(msg.getProperties()));\n\n                try {\n                    RemotingCommand pushResponse = this.brokerController.getBroker2Client().callClient(channel, request);\n                    assert pushResponse != null;\n                    switch (pushResponse.getCode()) {\n                        case ResponseCode.SUCCESS: {\n                            pushReplyResult.setPushOk(true);\n                            break;\n                        }\n                        default: {\n                            pushReplyResult.setPushOk(false);\n                            pushReplyResult.setRemark(\"push reply message to \" + senderId + \"fail.\");\n                            log.warn(\"push reply message to <{}> return fail, response remark: {}\", senderId, pushResponse.getRemark());\n                        }\n                    }\n                } catch (RemotingException | InterruptedException e) {\n                    pushReplyResult.setPushOk(false);\n                    pushReplyResult.setRemark(\"push reply message to \" + senderId + \"fail.\");\n                    log.warn(\"push reply message to <{}> fail. {}\", senderId, channel, e);\n                }\n            } else {\n                pushReplyResult.setPushOk(false);\n                pushReplyResult.setRemark(\"push reply message fail, channel of <\" + senderId + \"> not found.\");\n                log.warn(pushReplyResult.getRemark());\n            }\n        } else {\n            log.warn(MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT + \" is null, can not reply message\");\n            pushReplyResult.setPushOk(false);\n            pushReplyResult.setRemark(\"reply message properties[\" + MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT + \"] is null\");\n        }\n        return pushReplyResult;\n    }\n\n    private void handlePushReplyResult(PushReplyResult pushReplyResult, final RemotingCommand response,\n        final SendMessageResponseHeader responseHeader, int queueIdInt) {\n\n        if (!pushReplyResult.isPushOk()) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(pushReplyResult.getRemark());\n        } else {\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n            //set to zero to avoid client decoding exception\n            responseHeader.setMsgId(\"0\");\n            responseHeader.setQueueId(queueIdInt);\n            responseHeader.setQueueOffset(0L);\n        }\n    }\n\n    private void handlePutMessageResult(PutMessageResult putMessageResult,\n        final RemotingCommand request, final MessageExt msg,\n        final SendMessageResponseHeader responseHeader, SendMessageContext sendMessageContext,\n        int queueIdInt, TopicMessageType messageType) {\n        if (putMessageResult == null) {\n            log.warn(\"process reply message, store putMessage return null\");\n            return;\n        }\n        boolean putOk = false;\n\n        switch (putMessageResult.getPutMessageStatus()) {\n            // Success\n            case PUT_OK:\n            case FLUSH_DISK_TIMEOUT:\n            case FLUSH_SLAVE_TIMEOUT:\n            case SLAVE_NOT_AVAILABLE:\n                putOk = true;\n                break;\n\n            // Failed\n            case CREATE_MAPPED_FILE_FAILED:\n                log.warn(\"create mapped file failed, server is busy or broken.\");\n                break;\n            case MESSAGE_ILLEGAL:\n                log.warn(\n                    \"the message is illegal, maybe msg body or properties length not matched. msg body length limit {}B.\",\n                    this.brokerController.getMessageStoreConfig().getMaxMessageSize());\n                break;\n            case PROPERTIES_SIZE_EXCEEDED:\n                log.warn(\n                    \"the message is illegal, maybe msg properties length limit 32KB.\");\n                break;\n            case SERVICE_NOT_AVAILABLE:\n                log.warn(\n                    \"service not available now. It may be caused by one of the following reasons: \" +\n                        \"the broker's disk is full, messages are put to the slave, message store has been shut down, etc.\");\n                break;\n            case OS_PAGE_CACHE_BUSY:\n                log.warn(\"[PC_SYNCHRONIZED]broker busy, start flow control for a while\");\n                break;\n            case UNKNOWN_ERROR:\n                log.warn(\"UNKNOWN_ERROR\");\n                break;\n            default:\n                log.warn(\"UNKNOWN_ERROR DEFAULT\");\n                break;\n        }\n\n        String owner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER);\n        int commercialSizePerMsg = brokerController.getBrokerConfig().getCommercialSizePerMsg();\n        if (putOk) {\n            this.brokerController.getBrokerStatsManager().incTopicPutNums(msg.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum(), 1);\n            this.brokerController.getBrokerStatsManager().incTopicPutSize(msg.getTopic(),\n                putMessageResult.getAppendMessageResult().getWroteBytes());\n            this.brokerController.getBrokerStatsManager().incBrokerPutNums(msg.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum());\n\n            if (!BrokerMetricsManager.isRetryOrDlqTopic(msg.getTopic())) {\n                Attributes attributes = this.brokerController.getBrokerMetricsManager().newAttributesBuilder()\n                    .put(LABEL_TOPIC, msg.getTopic())\n                    .put(LABEL_MESSAGE_TYPE, messageType.getMetricsValue())\n                    .put(LABEL_IS_SYSTEM, TopicValidator.isSystemTopic(msg.getTopic()))\n                    .build();\n                this.brokerController.getBrokerMetricsManager().getMessagesInTotal().add(putMessageResult.getAppendMessageResult().getMsgNum(), attributes);\n                this.brokerController.getBrokerMetricsManager().getThroughputInTotal().add(putMessageResult.getAppendMessageResult().getWroteBytes(), attributes);\n                this.brokerController.getBrokerMetricsManager().getMessageSize().record(putMessageResult.getAppendMessageResult().getWroteBytes() / putMessageResult.getAppendMessageResult().getMsgNum(), attributes);\n            }\n\n            responseHeader.setMsgId(putMessageResult.getAppendMessageResult().getMsgId());\n            responseHeader.setQueueId(queueIdInt);\n            responseHeader.setQueueOffset(putMessageResult.getAppendMessageResult().getLogicsOffset());\n\n            if (hasSendMessageHook()) {\n                sendMessageContext.setMsgId(responseHeader.getMsgId());\n                sendMessageContext.setQueueId(responseHeader.getQueueId());\n                sendMessageContext.setQueueOffset(responseHeader.getQueueOffset());\n\n                int commercialBaseCount = brokerController.getBrokerConfig().getCommercialBaseCount();\n                int wroteSize = putMessageResult.getAppendMessageResult().getWroteBytes();\n                int incValue = (int) Math.ceil(wroteSize * 1.0 / commercialSizePerMsg) * commercialBaseCount;\n\n                sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_SUCCESS);\n                sendMessageContext.setCommercialSendTimes(incValue);\n                sendMessageContext.setCommercialSendSize(wroteSize);\n                sendMessageContext.setCommercialOwner(owner);\n            }\n        } else {\n            if (hasSendMessageHook()) {\n                int wroteSize = request.getBody().length;\n                int incValue = (int) Math.ceil(wroteSize * 1.0 / commercialSizePerMsg);\n\n                sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_FAILURE);\n                sendMessageContext.setCommercialSendTimes(incValue);\n                sendMessageContext.setCommercialSendSize(wroteSize);\n                sendMessageContext.setCommercialOwner(owner);\n            }\n        }\n    }\n\n    class PushReplyResult {\n        boolean pushOk;\n        String remark;\n\n        public PushReplyResult(boolean pushOk) {\n            this.pushOk = pushOk;\n            remark = \"\";\n        }\n\n        public boolean isPushOk() {\n            return pushOk;\n        }\n\n        public void setPushOk(boolean pushOk) {\n            this.pushOk = pushOk;\n        }\n\n        public String getRemark() {\n            return remark;\n        }\n\n        public void setRemark(String remark) {\n            this.remark = remark;\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageCallback.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.processor;\n\nimport org.apache.rocketmq.broker.mqtrace.SendMessageContext;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic interface SendMessageCallback {\n    /**\n     * On send complete.\n     *\n     * @param ctx send context\n     * @param response send response\n     */\n    void onComplete(SendMessageContext ctx, RemotingCommand response);\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport io.opentelemetry.api.common.Attributes;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.metrics.BrokerMetricsManager;\nimport org.apache.rocketmq.broker.mqtrace.SendMessageContext;\nimport org.apache.rocketmq.common.AbortProcessException;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.TopicFilterType;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.attribute.CleanupPolicy;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.help.FAQUrl;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBatch;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.producer.RecallMessageHandle;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.common.utils.CleanupPolicyUtils;\nimport org.apache.rocketmq.common.utils.MessageUtils;\nimport org.apache.rocketmq.common.utils.QueueTypeUtils;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.statictopic.LogicQueueMappingItem;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingContext;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingDetail;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.AppendMessageResult;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.config.StorePathConfigHelper;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\n\nimport java.nio.ByteBuffer;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Optional;\nimport java.util.concurrent.CompletableFuture;\n\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_MESSAGE_TYPE;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_TOPIC;\nimport static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse;\n\npublic class SendMessageProcessor extends AbstractSendMessageProcessor implements NettyRequestProcessor {\n\n    public SendMessageProcessor(final BrokerController brokerController) {\n        super(brokerController);\n    }\n\n    @Override\n    public RemotingCommand processRequest(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        SendMessageContext sendMessageContext;\n        switch (request.getCode()) {\n            case RequestCode.CONSUMER_SEND_MSG_BACK:\n                return this.consumerSendMsgBack(ctx, request);\n            default:\n                SendMessageRequestHeader requestHeader = parseRequestHeader(request);\n                if (requestHeader == null) {\n                    return null;\n                }\n                TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader, true);\n                RemotingCommand rewriteResult = this.brokerController.getTopicQueueMappingManager().rewriteRequestForStaticTopic(requestHeader, mappingContext);\n                if (rewriteResult != null) {\n                    return rewriteResult;\n                }\n                sendMessageContext = buildMsgContext(ctx, requestHeader, request);\n                try {\n                    this.executeSendMessageHookBefore(sendMessageContext);\n                } catch (AbortProcessException e) {\n                    final RemotingCommand errorResponse = RemotingCommand.createResponseCommand(e.getResponseCode(), e.getErrorMessage());\n                    errorResponse.setOpaque(request.getOpaque());\n                    return errorResponse;\n                }\n\n                RemotingCommand response;\n                clearReservedProperties(requestHeader);\n\n                if (requestHeader.isBatch()) {\n                    response = this.sendBatchMessage(ctx, request, sendMessageContext, requestHeader, mappingContext,\n                        (ctx1, response1) -> executeSendMessageHookAfter(response1, ctx1));\n                } else {\n                    response = this.sendMessage(ctx, request, sendMessageContext, requestHeader, mappingContext,\n                        (ctx12, response12) -> executeSendMessageHookAfter(response12, ctx12));\n                }\n\n                return response;\n        }\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        if (!this.brokerController.getBrokerConfig().isEnableSlaveActingMaster() && this.brokerController.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) {\n            return true;\n        }\n\n        if (this.brokerController.getMessageStore().isOSPageCacheBusy() || this.brokerController.getMessageStore().isTransientStorePoolDeficient()) {\n            return true;\n        }\n\n        return false;\n    }\n\n    private void clearReservedProperties(SendMessageRequestHeader requestHeader) {\n        String properties = requestHeader.getProperties();\n        properties = MessageUtils.deleteProperty(properties, MessageConst.PROPERTY_POP_CK);\n        requestHeader.setProperties(properties);\n    }\n\n    /**\n     * If the response is not null, it meets some errors\n     *\n     * @return\n     */\n\n    private RemotingCommand rewriteResponseForStaticTopic(SendMessageResponseHeader responseHeader,\n        TopicQueueMappingContext mappingContext) {\n        try {\n            if (mappingContext.getMappingDetail() == null) {\n                return null;\n            }\n            TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();\n\n            LogicQueueMappingItem mappingItem = mappingContext.getLeaderItem();\n            if (mappingItem == null) {\n                return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format(\"%s-%d does not exit in request process of current broker %s\", mappingContext.getTopic(), mappingContext.getGlobalId(), mappingDetail.getBname()));\n            }\n            //no need to care the broker name\n            long staticLogicOffset = mappingItem.computeStaticQueueOffsetLoosely(responseHeader.getQueueOffset());\n            if (staticLogicOffset < 0) {\n                //if the logic offset is -1, just let it go\n                //maybe we need a dynamic config\n                //return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format(\"%s-%d convert offset error in current broker %s\", mappingContext.getTopic(), mappingContext.getGlobalId(), mappingDetail.getBname()));\n            }\n            responseHeader.setQueueId(mappingContext.getGlobalId());\n            responseHeader.setQueueOffset(staticLogicOffset);\n        } catch (Throwable t) {\n            return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());\n        }\n        return null;\n    }\n\n    private boolean handleRetryAndDLQ(SendMessageRequestHeader requestHeader, RemotingCommand response,\n        RemotingCommand request,\n        MessageExt msg, TopicConfig topicConfig, Map<String, String> properties) {\n        String newTopic = requestHeader.getTopic();\n        if (null != newTopic && newTopic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n            String groupName = KeyBuilder.parseGroup(newTopic);\n            SubscriptionGroupConfig subscriptionGroupConfig =\n                this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(groupName);\n            if (null == subscriptionGroupConfig) {\n                response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);\n                response.setRemark(\n                    \"subscription group not exist, \" + groupName + \" \" + FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST));\n                return false;\n            }\n\n            int maxReconsumeTimes = subscriptionGroupConfig.getRetryMaxTimes();\n            if (request.getVersion() >= MQVersion.Version.V3_4_9.ordinal() && requestHeader.getMaxReconsumeTimes() != null) {\n                maxReconsumeTimes = requestHeader.getMaxReconsumeTimes();\n            }\n            int reconsumeTimes = requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes();\n\n            boolean sendRetryMessageToDeadLetterQueueDirectly = false;\n            if (!brokerController.getRebalanceLockManager().isLockAllExpired(groupName)) {\n                LOGGER.info(\"Group has unexpired lock record, which show it is ordered message, send it to DLQ \"\n                        + \"right now group={}, topic={}, reconsumeTimes={}, maxReconsumeTimes={}.\", groupName,\n                    newTopic, reconsumeTimes, maxReconsumeTimes);\n                sendRetryMessageToDeadLetterQueueDirectly = true;\n            }\n\n            if (reconsumeTimes > maxReconsumeTimes || sendRetryMessageToDeadLetterQueueDirectly) {\n                Attributes attributes = this.brokerController.getBrokerMetricsManager().newAttributesBuilder()\n                    .put(LABEL_CONSUMER_GROUP, requestHeader.getProducerGroup())\n                    .put(LABEL_TOPIC, requestHeader.getTopic())\n                    .put(LABEL_IS_SYSTEM, BrokerMetricsManager.isSystem(requestHeader.getTopic(), requestHeader.getProducerGroup()))\n                    .build();\n                this.brokerController.getBrokerMetricsManager().getSendToDlqMessages().add(1, attributes);\n\n                properties.put(MessageConst.PROPERTY_DELAY_TIME_LEVEL, \"-1\");\n                newTopic = MixAll.getDLQTopic(groupName);\n                int queueIdInt = randomQueueId(DLQ_NUMS_PER_GROUP);\n                topicConfig = this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod(newTopic,\n                    DLQ_NUMS_PER_GROUP,\n                    PermName.PERM_WRITE | PermName.PERM_READ, 0\n                );\n                msg.setTopic(newTopic);\n                msg.setQueueId(queueIdInt);\n                msg.setDelayTimeLevel(0);\n                if (null == topicConfig) {\n                    response.setCode(ResponseCode.SYSTEM_ERROR);\n                    response.setRemark(\"topic[\" + newTopic + \"] not exist\");\n                    return false;\n                }\n            }\n        }\n        int sysFlag = requestHeader.getSysFlag();\n        if (TopicFilterType.MULTI_TAG == topicConfig.getTopicFilterType()) {\n            sysFlag |= MessageSysFlag.MULTI_TAGS_FLAG;\n        }\n        msg.setSysFlag(sysFlag);\n        return true;\n    }\n\n    public RemotingCommand sendMessage(final ChannelHandlerContext ctx,\n        final RemotingCommand request,\n        final SendMessageContext sendMessageContext,\n        final SendMessageRequestHeader requestHeader,\n        final TopicQueueMappingContext mappingContext,\n        final SendMessageCallback sendMessageCallback) throws RemotingCommandException {\n\n        final RemotingCommand response = preSend(ctx, request, requestHeader);\n        if (response.getCode() != -1) {\n            return response;\n        }\n\n        final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader) response.readCustomHeader();\n\n        final byte[] body = request.getBody();\n\n        int queueIdInt = requestHeader.getQueueId();\n        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());\n\n        if (queueIdInt < 0) {\n            queueIdInt = randomQueueId(topicConfig.getWriteQueueNums());\n        }\n\n        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n        msgInner.setTopic(requestHeader.getTopic());\n        msgInner.setQueueId(queueIdInt);\n\n        Map<String, String> oriProps = MessageDecoder.string2messageProperties(requestHeader.getProperties());\n        if (!handleRetryAndDLQ(requestHeader, response, request, msgInner, topicConfig, oriProps)) {\n            return response;\n        }\n\n        msgInner.setBody(body);\n        msgInner.setFlag(requestHeader.getFlag());\n\n        String uniqKey = oriProps.get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX);\n        if (uniqKey == null || uniqKey.length() <= 0) {\n            uniqKey = MessageClientIDSetter.createUniqID();\n            oriProps.put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, uniqKey);\n        }\n\n        // liteTopic multi dispatch\n        String liteTopic = oriProps.get(MessageConst.PROPERTY_LITE_TOPIC);\n        if (StringUtils.isNotEmpty(liteTopic)) {\n            String lmqName = LiteUtil.toLmqName(requestHeader.getTopic(), liteTopic);\n            oriProps.put(MessageConst.PROPERTY_INNER_MULTI_DISPATCH, lmqName);\n        }\n\n        MessageAccessor.setProperties(msgInner, oriProps);\n        // check properties to ensure exclusive, don't check topic meta config to keep the behavior consistent\n        int msgPriority = msgInner.getPriority();\n        if (msgPriority >= 0) {\n            if (TopicMessageType.PRIORITY.equals(TopicMessageType.parseFromMessageProperty(msgInner.getProperties()))) {\n                queueIdInt = Math.min(msgPriority, topicConfig.getWriteQueueNums() - 1);\n                msgInner.setQueueId(queueIdInt);\n            } else {\n                MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_PRIORITY);\n            }\n        }\n\n        CleanupPolicy cleanupPolicy = CleanupPolicyUtils.getDeletePolicy(Optional.of(topicConfig));\n        if (Objects.equals(cleanupPolicy, CleanupPolicy.COMPACTION)) {\n            if (StringUtils.isBlank(msgInner.getKeys())) {\n                response.setCode(ResponseCode.MESSAGE_ILLEGAL);\n                response.setRemark(\"Required message key is missing\");\n                return response;\n            }\n        }\n\n        msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(topicConfig.getTopicFilterType(), msgInner.getTags()));\n        msgInner.setBornTimestamp(requestHeader.getBornTimestamp());\n        msgInner.setBornHost(ctx.channel().remoteAddress());\n        msgInner.setStoreHost(this.getStoreHost());\n        msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes());\n        String clusterName = this.brokerController.getBrokerConfig().getBrokerClusterName();\n        MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_CLUSTER, clusterName);\n\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));\n\n        // Map<String, String> oriProps = MessageDecoder.string2messageProperties(requestHeader.getProperties());\n        String traFlag = oriProps.get(MessageConst.PROPERTY_TRANSACTION_PREPARED);\n        boolean sendTransactionPrepareMessage;\n        if (Boolean.parseBoolean(traFlag)\n            && !(msgInner.getReconsumeTimes() > 0 && msgInner.getDelayTimeLevel() > 0)) { //For client under version 4.6.1\n            if (this.brokerController.getBrokerConfig().isRejectTransactionMessage()) {\n                response.setCode(ResponseCode.NO_PERMISSION);\n                response.setRemark(\n                    \"the broker[\" + this.brokerController.getBrokerConfig().getBrokerIP1()\n                        + \"] sending transaction message is forbidden\");\n                return response;\n            }\n            sendTransactionPrepareMessage = true;\n        } else {\n            sendTransactionPrepareMessage = false;\n        }\n\n        long beginTimeMillis = this.brokerController.getMessageStore().now();\n\n        if (brokerController.getBrokerConfig().isAsyncSendEnable()) {\n            CompletableFuture<PutMessageResult> asyncPutMessageFuture;\n            if (sendTransactionPrepareMessage) {\n                asyncPutMessageFuture = this.brokerController.getTransactionalMessageService().asyncPrepareMessage(msgInner);\n            } else {\n                asyncPutMessageFuture = this.brokerController.getMessageStore().asyncPutMessage(msgInner);\n            }\n\n            final int finalQueueIdInt = queueIdInt;\n            final MessageExtBrokerInner finalMsgInner = msgInner;\n            asyncPutMessageFuture.thenAcceptAsync(putMessageResult -> {\n                RemotingCommand responseFuture =\n                    handlePutMessageResult(putMessageResult, response, request, finalMsgInner, responseHeader, sendMessageContext,\n                        ctx, finalQueueIdInt, beginTimeMillis, mappingContext, BrokerMetricsManager.getMessageType(requestHeader));\n                if (responseFuture != null) {\n                    doResponse(ctx, request, responseFuture);\n                }\n\n                // record the transaction metrics, responseFuture == null means put successfully\n                if (sendTransactionPrepareMessage && (responseFuture == null || responseFuture.getCode() == ResponseCode.SUCCESS)) {\n                    this.brokerController.getTransactionalMessageService().getTransactionMetrics().addAndGet(msgInner.getProperty(MessageConst.PROPERTY_REAL_TOPIC), 1);\n                }\n\n                sendMessageCallback.onComplete(sendMessageContext, response);\n            }, this.brokerController.getPutMessageFutureExecutor());\n            // Returns null to release the send message thread\n            return null;\n        } else {\n            PutMessageResult putMessageResult = null;\n            if (sendTransactionPrepareMessage) {\n                putMessageResult = this.brokerController.getTransactionalMessageService().prepareMessage(msgInner);\n            } else {\n                putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner);\n            }\n            handlePutMessageResult(putMessageResult, response, request, msgInner, responseHeader, sendMessageContext, ctx, queueIdInt, beginTimeMillis, mappingContext, BrokerMetricsManager.getMessageType(requestHeader));\n            // record the transaction metrics\n            if (sendTransactionPrepareMessage && putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK && putMessageResult.getAppendMessageResult().isOk()) {\n                this.brokerController.getTransactionalMessageService().getTransactionMetrics().addAndGet(msgInner.getProperty(MessageConst.PROPERTY_REAL_TOPIC), 1);\n            }\n            sendMessageCallback.onComplete(sendMessageContext, response);\n            return response;\n        }\n    }\n\n    private RemotingCommand handlePutMessageResult(PutMessageResult putMessageResult, RemotingCommand response,\n        RemotingCommand request, MessageExt msg, SendMessageResponseHeader responseHeader,\n        SendMessageContext sendMessageContext, ChannelHandlerContext ctx, int queueIdInt, long beginTimeMillis,\n        TopicQueueMappingContext mappingContext, TopicMessageType messageType) {\n        if (putMessageResult == null) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"store putMessage return null\");\n            return response;\n        }\n        boolean sendOK = false;\n\n        switch (putMessageResult.getPutMessageStatus()) {\n            // Success\n            case PUT_OK:\n                sendOK = true;\n                response.setCode(ResponseCode.SUCCESS);\n                break;\n            case FLUSH_DISK_TIMEOUT:\n                response.setCode(ResponseCode.FLUSH_DISK_TIMEOUT);\n                sendOK = true;\n                break;\n            case FLUSH_SLAVE_TIMEOUT:\n                response.setCode(ResponseCode.FLUSH_SLAVE_TIMEOUT);\n                sendOK = true;\n                break;\n            case SLAVE_NOT_AVAILABLE:\n                response.setCode(ResponseCode.SLAVE_NOT_AVAILABLE);\n                sendOK = true;\n                break;\n\n            // Failed\n            case IN_SYNC_REPLICAS_NOT_ENOUGH:\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"in-sync replicas not enough\");\n                break;\n            case CREATE_MAPPED_FILE_FAILED:\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"create mapped file failed, server is busy or broken.\");\n                break;\n            case MESSAGE_ILLEGAL:\n            case PROPERTIES_SIZE_EXCEEDED:\n                response.setCode(ResponseCode.MESSAGE_ILLEGAL);\n                response.setRemark(String.format(\"the message is illegal, maybe msg body or properties length not matched. msg body length limit %dB, msg properties length limit 32KB.\",\n                    this.brokerController.getMessageStoreConfig().getMaxMessageSize()));\n                break;\n            case WHEEL_TIMER_MSG_ILLEGAL:\n                response.setCode(ResponseCode.MESSAGE_ILLEGAL);\n                response.setRemark(String.format(\"timer message illegal, the delay time should not be bigger than the max delay %dms; or if set del msg, the delay time should be bigger than the current time\",\n                    this.brokerController.getMessageStoreConfig().getTimerMaxDelaySec() * 1000L));\n                break;\n            case WHEEL_TIMER_FLOW_CONTROL:\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(String.format(\"timer message is under flow control, max num limit is %d or the current value is greater than %d and less than %d, trigger random flow control\",\n                     this.brokerController.getMessageStoreConfig().getTimerCongestNumEachSlot() * 2L, this.brokerController.getMessageStoreConfig().getTimerCongestNumEachSlot(), this.brokerController.getMessageStoreConfig().getTimerCongestNumEachSlot() * 2L));\n                break;\n            case WHEEL_TIMER_NOT_ENABLE:\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(String.format(\"accurate timer message is not enabled, timerWheelEnable is %s\",\n                     this.brokerController.getMessageStoreConfig().isTimerWheelEnable()));\n                break;\n            case SERVICE_NOT_AVAILABLE:\n                response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE);\n                response.setRemark(\n                    \"service not available now. It may be caused by one of the following reasons: \" +\n                        \"the broker's disk is full [\" + diskUtil() + \"], messages are put to the slave, message store has been shut down, etc.\");\n                break;\n            case OS_PAGE_CACHE_BUSY:\n                response.setCode(ResponseCode.SYSTEM_BUSY);\n                response.setRemark(\"[PC_SYNCHRONIZED]broker busy, start flow control for a while\");\n                break;\n            case LMQ_CONSUME_QUEUE_NUM_EXCEEDED:\n                response.setCode(ResponseCode.LMQ_QUOTA_EXCEEDED);\n                response.setRemark(\"[LMQ_CONSUME_QUEUE_NUM_EXCEEDED]lmq consume queue num exceeded.\");\n                break;\n            case UNKNOWN_ERROR:\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"UNKNOWN_ERROR\");\n                break;\n            default:\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"UNKNOWN_ERROR DEFAULT\");\n                break;\n        }\n\n        String owner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER);\n        String authType = request.getExtFields().get(BrokerStatsManager.ACCOUNT_AUTH_TYPE);\n        String ownerParent = request.getExtFields().get(BrokerStatsManager.ACCOUNT_OWNER_PARENT);\n        String ownerSelf = request.getExtFields().get(BrokerStatsManager.ACCOUNT_OWNER_SELF);\n        int commercialSizePerMsg = brokerController.getBrokerConfig().getCommercialSizePerMsg();\n        if (sendOK) {\n\n            if (TopicValidator.RMQ_SYS_SCHEDULE_TOPIC.equals(msg.getTopic())) {\n                this.brokerController.getBrokerStatsManager().incQueuePutNums(msg.getTopic(), msg.getQueueId(), putMessageResult.getAppendMessageResult().getMsgNum(), 1);\n                this.brokerController.getBrokerStatsManager().incQueuePutSize(msg.getTopic(), msg.getQueueId(), putMessageResult.getAppendMessageResult().getWroteBytes());\n            }\n\n            this.brokerController.getBrokerStatsManager().incTopicPutNums(msg.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum(), 1);\n            this.brokerController.getBrokerStatsManager().incTopicPutSize(msg.getTopic(),\n                putMessageResult.getAppendMessageResult().getWroteBytes());\n            this.brokerController.getBrokerStatsManager().incBrokerPutNums(msg.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum());\n            this.brokerController.getBrokerStatsManager().incTopicPutLatency(msg.getTopic(), queueIdInt,\n                (int) (this.brokerController.getMessageStore().now() - beginTimeMillis));\n\n            if (!BrokerMetricsManager.isRetryOrDlqTopic(msg.getTopic())) {\n                Attributes attributes = this.brokerController.getBrokerMetricsManager().newAttributesBuilder()\n                    .put(LABEL_TOPIC, msg.getTopic())\n                    .put(LABEL_MESSAGE_TYPE, messageType.getMetricsValue())\n                    .put(LABEL_IS_SYSTEM, TopicValidator.isSystemTopic(msg.getTopic()))\n                    .build();\n                this.brokerController.getBrokerMetricsManager().getMessagesInTotal().add(putMessageResult.getAppendMessageResult().getMsgNum(), attributes);\n                this.brokerController.getBrokerMetricsManager().getThroughputInTotal().add(putMessageResult.getAppendMessageResult().getWroteBytes(), attributes);\n                this.brokerController.getBrokerMetricsManager().getMessageSize().record(putMessageResult.getAppendMessageResult().getWroteBytes() / putMessageResult.getAppendMessageResult().getMsgNum(), attributes);\n            }\n\n            response.setRemark(null);\n\n            responseHeader.setMsgId(putMessageResult.getAppendMessageResult().getMsgId());\n            responseHeader.setQueueId(queueIdInt);\n            responseHeader.setQueueOffset(putMessageResult.getAppendMessageResult().getLogicsOffset());\n            responseHeader.setTransactionId(MessageClientIDSetter.getUniqID(msg));\n            attachRecallHandle(request, msg, responseHeader);\n\n            RemotingCommand rewriteResult = rewriteResponseForStaticTopic(responseHeader, mappingContext);\n            if (rewriteResult != null) {\n                return rewriteResult;\n            }\n\n            doResponse(ctx, request, response);\n\n            if (hasSendMessageHook()) {\n                sendMessageContext.setMsgId(responseHeader.getMsgId());\n                sendMessageContext.setQueueId(responseHeader.getQueueId());\n                sendMessageContext.setQueueOffset(responseHeader.getQueueOffset());\n\n                int commercialBaseCount = brokerController.getBrokerConfig().getCommercialBaseCount();\n                int wroteSize = putMessageResult.getAppendMessageResult().getWroteBytes();\n                int msgNum = putMessageResult.getAppendMessageResult().getMsgNum();\n                int commercialMsgNum = (int) Math.ceil(wroteSize / (double) commercialSizePerMsg);\n                int incValue = commercialMsgNum * commercialBaseCount;\n\n                sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_SUCCESS);\n                sendMessageContext.setCommercialSendTimes(incValue);\n                sendMessageContext.setCommercialSendSize(wroteSize);\n                sendMessageContext.setCommercialOwner(owner);\n\n                sendMessageContext.setSendStat(BrokerStatsManager.StatsType.SEND_SUCCESS);\n                sendMessageContext.setCommercialSendMsgNum(commercialMsgNum);\n                sendMessageContext.setAccountAuthType(authType);\n                sendMessageContext.setAccountOwnerParent(ownerParent);\n                sendMessageContext.setAccountOwnerSelf(ownerSelf);\n                sendMessageContext.setSendMsgSize(wroteSize);\n                sendMessageContext.setSendMsgNum(msgNum);\n            }\n            return null;\n        } else {\n            if (hasSendMessageHook()) {\n                AppendMessageResult appendMessageResult = putMessageResult.getAppendMessageResult();\n\n                // TODO process partial failures of batch message\n                int wroteSize = request.getBody().length;\n                int msgNum = Math.max(appendMessageResult != null ? appendMessageResult.getMsgNum() : 1, 1);\n                int commercialMsgNum = (int) Math.ceil(wroteSize / (double) commercialSizePerMsg);\n\n                sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_FAILURE);\n                sendMessageContext.setCommercialSendTimes(commercialMsgNum);\n                sendMessageContext.setCommercialSendSize(wroteSize);\n                sendMessageContext.setCommercialOwner(owner);\n\n                sendMessageContext.setSendStat(BrokerStatsManager.StatsType.SEND_FAILURE);\n                sendMessageContext.setCommercialSendMsgNum(commercialMsgNum);\n                sendMessageContext.setAccountAuthType(authType);\n                sendMessageContext.setAccountOwnerParent(ownerParent);\n                sendMessageContext.setAccountOwnerSelf(ownerSelf);\n                sendMessageContext.setSendMsgSize(wroteSize);\n                sendMessageContext.setSendMsgNum(msgNum);\n            }\n        }\n        return response;\n    }\n\n    private RemotingCommand sendBatchMessage(final ChannelHandlerContext ctx,\n        final RemotingCommand request,\n        final SendMessageContext sendMessageContext,\n        final SendMessageRequestHeader requestHeader,\n        TopicQueueMappingContext mappingContext,\n        final SendMessageCallback sendMessageCallback) {\n        final RemotingCommand response = preSend(ctx, request, requestHeader);\n        final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader) response.readCustomHeader();\n\n        if (response.getCode() != -1) {\n            return response;\n        }\n\n        int queueIdInt = requestHeader.getQueueId();\n        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());\n\n        if (queueIdInt < 0) {\n            queueIdInt = randomQueueId(topicConfig.getWriteQueueNums());\n        }\n\n        if (requestHeader.getTopic().length() > Byte.MAX_VALUE) {\n            response.setCode(ResponseCode.MESSAGE_ILLEGAL);\n            response.setRemark(\"message topic length too long \" + requestHeader.getTopic().length());\n            return response;\n        }\n\n        if (requestHeader.getTopic() != null && requestHeader.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n            response.setCode(ResponseCode.MESSAGE_ILLEGAL);\n            response.setRemark(\"batch request does not support retry group \" + requestHeader.getTopic());\n            return response;\n        }\n        MessageExtBatch messageExtBatch = new MessageExtBatch();\n        messageExtBatch.setTopic(requestHeader.getTopic());\n        messageExtBatch.setQueueId(queueIdInt);\n\n        int sysFlag = requestHeader.getSysFlag();\n        if (TopicFilterType.MULTI_TAG == topicConfig.getTopicFilterType()) {\n            sysFlag |= MessageSysFlag.MULTI_TAGS_FLAG;\n        }\n        messageExtBatch.setSysFlag(sysFlag);\n\n        messageExtBatch.setFlag(requestHeader.getFlag());\n        MessageAccessor.setProperties(messageExtBatch, MessageDecoder.string2messageProperties(requestHeader.getProperties()));\n        messageExtBatch.setBody(request.getBody());\n        messageExtBatch.setBornTimestamp(requestHeader.getBornTimestamp());\n        messageExtBatch.setBornHost(ctx.channel().remoteAddress());\n        messageExtBatch.setStoreHost(this.getStoreHost());\n        messageExtBatch.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes());\n        String clusterName = this.brokerController.getBrokerConfig().getBrokerClusterName();\n        MessageAccessor.putProperty(messageExtBatch, MessageConst.PROPERTY_CLUSTER, clusterName);\n\n        boolean isInnerBatch = false;\n\n        if (QueueTypeUtils.isBatchCq(Optional.of(topicConfig)) && MessageClientIDSetter.getUniqID(messageExtBatch) != null) {\n            // newly introduced inner-batch message\n            messageExtBatch.setSysFlag(messageExtBatch.getSysFlag() | MessageSysFlag.NEED_UNWRAP_FLAG);\n            messageExtBatch.setSysFlag(messageExtBatch.getSysFlag() | MessageSysFlag.INNER_BATCH_FLAG);\n            messageExtBatch.setInnerBatch(true);\n\n            int innerNum = MessageDecoder.countInnerMsgNum(ByteBuffer.wrap(messageExtBatch.getBody()));\n\n            MessageAccessor.putProperty(messageExtBatch, MessageConst.PROPERTY_INNER_NUM, String.valueOf(innerNum));\n            messageExtBatch.setPropertiesString(MessageDecoder.messageProperties2String(messageExtBatch.getProperties()));\n\n            // tell the producer that it's an inner-batch message response.\n            responseHeader.setBatchUniqId(MessageClientIDSetter.getUniqID(messageExtBatch));\n\n            isInnerBatch = true;\n        }\n\n        long beginTimeMillis = this.brokerController.getMessageStore().now();\n\n        if (this.brokerController.getBrokerConfig().isAsyncSendEnable()) {\n            CompletableFuture<PutMessageResult> asyncPutMessageFuture;\n            if (isInnerBatch) {\n                asyncPutMessageFuture = this.brokerController.getMessageStore().asyncPutMessage(messageExtBatch);\n            } else {\n                asyncPutMessageFuture = this.brokerController.getMessageStore().asyncPutMessages(messageExtBatch);\n            }\n            final int finalQueueIdInt = queueIdInt;\n            asyncPutMessageFuture.thenAcceptAsync(putMessageResult -> {\n                RemotingCommand responseFuture =\n                    handlePutMessageResult(putMessageResult, response, request, messageExtBatch, responseHeader,\n                        sendMessageContext, ctx, finalQueueIdInt, beginTimeMillis, mappingContext, BrokerMetricsManager.getMessageType(requestHeader));\n                if (responseFuture != null) {\n                    doResponse(ctx, request, responseFuture);\n                }\n                sendMessageCallback.onComplete(sendMessageContext, response);\n            }, this.brokerController.getSendMessageExecutor());\n            // Returns null to release the send message thread\n            return null;\n        } else {\n            PutMessageResult putMessageResult;\n            if (isInnerBatch) {\n                putMessageResult = this.brokerController.getMessageStore().putMessage(messageExtBatch);\n            } else {\n                putMessageResult = this.brokerController.getMessageStore().putMessages(messageExtBatch);\n            }\n            handlePutMessageResult(putMessageResult, response, request, messageExtBatch, responseHeader,\n                sendMessageContext, ctx, queueIdInt, beginTimeMillis, mappingContext, BrokerMetricsManager.getMessageType(requestHeader));\n            sendMessageCallback.onComplete(sendMessageContext, response);\n            return response;\n        }\n    }\n\n    public void attachRecallHandle(RemotingCommand request, MessageExt msg, SendMessageResponseHeader responseHeader) {\n        if (RequestCode.SEND_BATCH_MESSAGE == request.getCode()\n            || RequestCode.CONSUMER_SEND_MSG_BACK == request.getCode()) {\n            return;\n        }\n        String timestampStr = msg.getProperty(MessageConst.PROPERTY_TIMER_OUT_MS);\n        String realTopic = msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC);\n        if (timestampStr != null && realTopic != null && !realTopic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n            timestampStr = String.valueOf(Long.parseLong(timestampStr) + 1); // consider of floor\n            String recallHandle = RecallMessageHandle.HandleV1.buildHandle(realTopic,\n                brokerController.getBrokerConfig().getBrokerName(), timestampStr, MessageClientIDSetter.getUniqID(msg));\n            responseHeader.setRecallHandle(recallHandle);\n        }\n    }\n\n    private String diskUtil() {\n        double physicRatio = 100;\n        String storePath;\n        MessageStore messageStore = this.brokerController.getMessageStore();\n        if (messageStore instanceof DefaultMessageStore) {\n            storePath = ((DefaultMessageStore) messageStore).getStorePathPhysic();\n        } else {\n            storePath = this.brokerController.getMessageStoreConfig().getStorePathCommitLog();\n        }\n        String[] paths = storePath.trim().split(MixAll.MULTI_PATH_SPLITTER);\n        for (String storePathPhysic : paths) {\n            physicRatio = Math.min(physicRatio, UtilAll.getDiskPartitionSpaceUsedPercent(storePathPhysic));\n        }\n\n        String storePathLogis =\n            StorePathConfigHelper.getStorePathConsumeQueue(this.brokerController.getMessageStoreConfig().getStorePathRootDir());\n        double logisRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathLogis);\n\n        String storePathIndex =\n            StorePathConfigHelper.getStorePathIndex(this.brokerController.getMessageStoreConfig().getStorePathRootDir());\n        double indexRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathIndex);\n\n        return String.format(\"CL: %5.2f CQ: %5.2f INDEX: %5.2f\", physicRatio, logisRatio, indexRatio);\n    }\n\n    private RemotingCommand preSend(ChannelHandlerContext ctx, RemotingCommand request,\n        SendMessageRequestHeader requestHeader) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);\n\n        response.setOpaque(request.getOpaque());\n\n        response.addExtField(MessageConst.PROPERTY_MSG_REGION, this.brokerController.getBrokerConfig().getRegionId());\n        response.addExtField(MessageConst.PROPERTY_TRACE_SWITCH, String.valueOf(this.brokerController.getBrokerConfig().isTraceOn()));\n\n        LOGGER.debug(\"Receive SendMessage request command {}\", request);\n\n        final long startTimestamp = this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp();\n\n        if (this.brokerController.getMessageStore().now() < startTimestamp) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(String.format(\"broker unable to service, until %s\", UtilAll.timeMillisToHumanString2(startTimestamp)));\n            return response;\n        }\n\n        response.setCode(-1);\n        super.msgCheck(ctx, requestHeader, request, response);\n\n        return response;\n    }\n\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/schedule/DelayOffsetSerializeWrapper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.schedule;\n\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class DelayOffsetSerializeWrapper extends RemotingSerializable {\n    private ConcurrentMap<Integer /* level */, Long/* offset */> offsetTable =\n        new ConcurrentHashMap<>(32);\n\n    private DataVersion dataVersion;\n\n    public ConcurrentMap<Integer, Long> getOffsetTable() {\n        return offsetTable;\n    }\n\n    public void setOffsetTable(ConcurrentMap<Integer, Long> offsetTable) {\n        this.offsetTable = offsetTable;\n    }\n\n    public DataVersion getDataVersion() {\n        return dataVersion;\n    }\n\n    public void setDataVersion(DataVersion dataVersion) {\n        this.dataVersion = dataVersion;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.schedule;\n\nimport io.opentelemetry.api.common.Attributes;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.Queue;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.ConcurrentSkipListMap;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.broker.BrokerController;\n\nimport org.apache.rocketmq.common.ConfigManager;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.TopicFilterType;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.running.RunningStats;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.config.StorePathConfigHelper;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.queue.ConsumeQueueInterface;\nimport org.apache.rocketmq.store.queue.CqUnit;\nimport org.apache.rocketmq.store.queue.ReferredIterator;\n\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_MESSAGE_TYPE;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_TOPIC;\n\npublic class ScheduleMessageService extends ConfigManager {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n\n    private static final long FIRST_DELAY_TIME = 1000L;\n    private static final long DELAY_FOR_A_WHILE = 100L;\n    private static final long DELAY_FOR_A_PERIOD = 10000L;\n    private static final long WAIT_FOR_SHUTDOWN = 5000L;\n    private static final long DELAY_FOR_A_SLEEP = 10L;\n\n    private final ConcurrentSkipListMap<Integer /* level */, Long/* delay timeMillis */> delayLevelTable =\n        new ConcurrentSkipListMap<>();\n\n    private final ConcurrentMap<Integer /* level */, Long/* offset */> offsetTable =\n        new ConcurrentHashMap<>(32);\n    private final AtomicBoolean started = new AtomicBoolean(false);\n    private ScheduledExecutorService deliverExecutorService;\n    private int maxDelayLevel;\n    private DataVersion dataVersion = new DataVersion();\n    private boolean enableAsyncDeliver = false;\n    private ScheduledExecutorService handleExecutorService;\n    private final ScheduledExecutorService scheduledPersistService;\n    private final Map<Integer /* level */, LinkedBlockingQueue<PutResultProcess>> deliverPendingTable =\n        new ConcurrentHashMap<>(32);\n    private final BrokerController brokerController;\n    private final transient AtomicLong versionChangeCounter = new AtomicLong(0);\n\n    public ScheduleMessageService(final BrokerController brokerController) {\n        this.brokerController = brokerController;\n        this.enableAsyncDeliver = brokerController.getMessageStoreConfig().isEnableScheduleAsyncDeliver();\n        scheduledPersistService = ThreadUtils.newScheduledThreadPool(1,\n            new ThreadFactoryImpl(\"ScheduleMessageServicePersistThread\", true, brokerController.getBrokerConfig()));\n    }\n\n    public static int queueId2DelayLevel(final int queueId) {\n        return queueId + 1;\n    }\n\n    public static int delayLevel2QueueId(final int delayLevel) {\n        return delayLevel - 1;\n    }\n\n    public void buildRunningStats(HashMap<String, String> stats) throws ConsumeQueueException {\n        for (Map.Entry<Integer, Long> next : this.offsetTable.entrySet()) {\n            int queueId = delayLevel2QueueId(next.getKey());\n            long delayOffset = next.getValue();\n            long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, queueId);\n            String value = String.format(\"%d,%d\", delayOffset, maxOffset);\n            String key = String.format(\"%s_%d\", RunningStats.scheduleMessageOffset.name(), next.getKey());\n            stats.put(key, value);\n        }\n    }\n\n    private void updateOffset(int delayLevel, long offset) {\n        this.offsetTable.put(delayLevel, offset);\n        if (versionChangeCounter.incrementAndGet() % brokerController.getBrokerConfig().getDelayOffsetUpdateVersionStep() == 0) {\n            long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;\n            dataVersion.nextVersion(stateMachineVersion);\n        }\n    }\n\n    public long computeDeliverTimestamp(final int delayLevel, final long storeTimestamp) {\n        Long time = this.delayLevelTable.get(delayLevel);\n        if (time != null) {\n            return time + storeTimestamp;\n        }\n\n        return storeTimestamp + 1000;\n    }\n\n    public void start() {\n        if (started.compareAndSet(false, true)) {\n            this.load();\n            this.deliverExecutorService = ThreadUtils.newScheduledThreadPool(this.maxDelayLevel, new ThreadFactoryImpl(\"ScheduleMessageTimerThread_\"));\n            if (this.enableAsyncDeliver) {\n                this.handleExecutorService = ThreadUtils.newScheduledThreadPool(this.maxDelayLevel, new ThreadFactoryImpl(\"ScheduleMessageExecutorHandleThread_\"));\n            }\n            for (Map.Entry<Integer, Long> entry : this.delayLevelTable.entrySet()) {\n                Integer level = entry.getKey();\n                Long timeDelay = entry.getValue();\n                Long offset = this.offsetTable.get(level);\n                if (null == offset) {\n                    offset = 0L;\n                }\n\n                if (timeDelay != null) {\n                    if (this.enableAsyncDeliver) {\n                        this.handleExecutorService.schedule(new HandlePutResultTask(level), FIRST_DELAY_TIME, TimeUnit.MILLISECONDS);\n                    }\n                    this.deliverExecutorService.schedule(new DeliverDelayedMessageTimerTask(level, offset), FIRST_DELAY_TIME, TimeUnit.MILLISECONDS);\n                }\n            }\n\n            scheduledPersistService.scheduleAtFixedRate(() -> {\n                try {\n                    ScheduleMessageService.this.persist();\n                } catch (Throwable e) {\n                    log.error(\"scheduleAtFixedRate flush exception\", e);\n                }\n            }, 10000, this.brokerController.getMessageStoreConfig().getFlushDelayOffsetInterval(), TimeUnit.MILLISECONDS);\n        }\n    }\n\n    public void shutdown() {\n        stop();\n        ThreadUtils.shutdown(scheduledPersistService);\n    }\n\n    public boolean stop() {\n        if (this.started.compareAndSet(true, false) && null != this.deliverExecutorService) {\n            this.deliverExecutorService.shutdown();\n            try {\n                this.deliverExecutorService.awaitTermination(WAIT_FOR_SHUTDOWN, TimeUnit.MILLISECONDS);\n            } catch (InterruptedException e) {\n                log.error(\"deliverExecutorService awaitTermination error\", e);\n            }\n\n            if (this.handleExecutorService != null) {\n                this.handleExecutorService.shutdown();\n                try {\n                    this.handleExecutorService.awaitTermination(WAIT_FOR_SHUTDOWN, TimeUnit.MILLISECONDS);\n                } catch (InterruptedException e) {\n                    log.error(\"handleExecutorService awaitTermination error\", e);\n                }\n            }\n\n            for (int i = 1; i <= this.deliverPendingTable.size(); i++) {\n                log.warn(\"deliverPendingTable level: {}, size: {}\", i, this.deliverPendingTable.get(i).size());\n            }\n\n            this.persist();\n        }\n        return true;\n    }\n\n    public boolean isStarted() {\n        return started.get();\n    }\n\n    public int getMaxDelayLevel() {\n        return maxDelayLevel;\n    }\n\n    public DataVersion getDataVersion() {\n        return dataVersion;\n    }\n\n    public void setDataVersion(DataVersion dataVersion) {\n        this.dataVersion = dataVersion;\n    }\n\n    @Override\n    public String encode() {\n        return this.encode(false);\n    }\n\n    @Override\n    public boolean load() {\n        boolean result = super.load();\n        result = result && this.parseDelayLevel();\n        result = result && this.correctDelayOffset();\n        return result;\n    }\n\n    public boolean loadWhenSyncDelayOffset() {\n        boolean result = super.load();\n        result = result && this.parseDelayLevel();\n        return result;\n    }\n\n    public boolean correctDelayOffset() {\n        try {\n            for (int delayLevel : delayLevelTable.keySet()) {\n                ConsumeQueueInterface cq =\n                    brokerController.getMessageStore().findConsumeQueue(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC,\n                        delayLevel2QueueId(delayLevel));\n                Long currentDelayOffset = offsetTable.get(delayLevel);\n                if (currentDelayOffset == null || cq == null) {\n                    continue;\n                }\n                long correctDelayOffset = currentDelayOffset;\n                long cqMinOffset = cq.getMinOffsetInQueue();\n                long cqMaxOffset = cq.getMaxOffsetInQueue();\n                if (currentDelayOffset < cqMinOffset) {\n                    correctDelayOffset = cqMinOffset;\n                    log.error(\"schedule CQ offset invalid. offset={}, cqMinOffset={}, cqMaxOffset={}, queueId={}\",\n                        currentDelayOffset, cqMinOffset, cqMaxOffset, cq.getQueueId());\n                }\n\n                if (currentDelayOffset > cqMaxOffset) {\n                    correctDelayOffset = cqMaxOffset;\n                    log.error(\"schedule CQ offset invalid. offset={}, cqMinOffset={}, cqMaxOffset={}, queueId={}\",\n                        currentDelayOffset, cqMinOffset, cqMaxOffset, cq.getQueueId());\n                }\n                if (correctDelayOffset != currentDelayOffset) {\n                    log.error(\"correct delay offset [ delayLevel {} ] from {} to {}\", delayLevel, currentDelayOffset, correctDelayOffset);\n                    offsetTable.put(delayLevel, correctDelayOffset);\n                }\n            }\n        } catch (Exception e) {\n            log.error(\"correctDelayOffset exception\", e);\n            return false;\n        }\n        return true;\n    }\n\n    @Override\n    public String configFilePath() {\n        return StorePathConfigHelper.getDelayOffsetStorePath(this.brokerController.getMessageStore().getMessageStoreConfig()\n            .getStorePathRootDir());\n    }\n\n    @Override\n    public void decode(String jsonString) {\n        if (jsonString != null) {\n            DelayOffsetSerializeWrapper delayOffsetSerializeWrapper =\n                DelayOffsetSerializeWrapper.fromJson(jsonString, DelayOffsetSerializeWrapper.class);\n            if (delayOffsetSerializeWrapper != null) {\n                this.offsetTable.putAll(delayOffsetSerializeWrapper.getOffsetTable());\n                // For compatible\n                if (delayOffsetSerializeWrapper.getDataVersion() != null) {\n                    this.dataVersion.assignNewOne(delayOffsetSerializeWrapper.getDataVersion());\n                }\n            }\n        }\n    }\n\n    @Override\n    public String encode(final boolean prettyFormat) {\n        DelayOffsetSerializeWrapper delayOffsetSerializeWrapper = new DelayOffsetSerializeWrapper();\n        delayOffsetSerializeWrapper.setOffsetTable(this.offsetTable);\n        delayOffsetSerializeWrapper.setDataVersion(this.dataVersion);\n        return delayOffsetSerializeWrapper.toJson(prettyFormat);\n    }\n\n    public boolean parseDelayLevel() {\n        HashMap<String, Long> timeUnitTable = new HashMap<>();\n        timeUnitTable.put(\"s\", 1000L);\n        timeUnitTable.put(\"m\", 1000L * 60);\n        timeUnitTable.put(\"h\", 1000L * 60 * 60);\n        timeUnitTable.put(\"d\", 1000L * 60 * 60 * 24);\n\n        String levelString = this.brokerController.getMessageStoreConfig().getMessageDelayLevel();\n        try {\n            String[] levelArray = levelString.split(\" \");\n            for (int i = 0; i < levelArray.length; i++) {\n                String value = levelArray[i];\n                String ch = value.substring(value.length() - 1);\n                Long tu = timeUnitTable.get(ch);\n\n                int level = i + 1;\n                if (level > this.maxDelayLevel) {\n                    this.maxDelayLevel = level;\n                }\n                long num = Long.parseLong(value.substring(0, value.length() - 1));\n                long delayTimeMillis = tu * num;\n                this.delayLevelTable.put(level, delayTimeMillis);\n                if (this.enableAsyncDeliver) {\n                    this.deliverPendingTable.put(level, new LinkedBlockingQueue<>());\n                }\n            }\n        } catch (Exception e) {\n            log.error(\"parse message delay level failed. messageDelayLevel = {}\", levelString, e);\n            return false;\n        }\n\n        return true;\n    }\n\n    private MessageExtBrokerInner messageTimeUp(MessageExt msgExt) {\n        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n        msgInner.setBody(msgExt.getBody());\n        msgInner.setFlag(msgExt.getFlag());\n        MessageAccessor.setProperties(msgInner, msgExt.getProperties());\n\n        TopicFilterType topicFilterType = MessageExt.parseTopicFilterType(msgInner.getSysFlag());\n        long tagsCodeValue =\n            MessageExtBrokerInner.tagsString2tagsCode(topicFilterType, msgInner.getTags());\n        msgInner.setTagsCode(tagsCodeValue);\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties()));\n\n        msgInner.setSysFlag(msgExt.getSysFlag());\n        msgInner.setBornTimestamp(msgExt.getBornTimestamp());\n        msgInner.setBornHost(msgExt.getBornHost());\n        msgInner.setStoreHost(msgExt.getStoreHost());\n        msgInner.setReconsumeTimes(msgExt.getReconsumeTimes());\n\n        msgInner.setWaitStoreMsgOK(false);\n        MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_DELAY_TIME_LEVEL);\n        MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_TIMER_DELIVER_MS);\n        MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_TIMER_DELAY_SEC);\n\n        msgInner.setTopic(msgInner.getProperty(MessageConst.PROPERTY_REAL_TOPIC));\n\n        String queueIdStr = msgInner.getProperty(MessageConst.PROPERTY_REAL_QUEUE_ID);\n        int queueId = Integer.parseInt(queueIdStr);\n        msgInner.setQueueId(queueId);\n\n        return msgInner;\n    }\n\n    class DeliverDelayedMessageTimerTask implements Runnable {\n        private final int delayLevel;\n        private final long offset;\n\n        public DeliverDelayedMessageTimerTask(int delayLevel, long offset) {\n            this.delayLevel = delayLevel;\n            this.offset = offset;\n        }\n\n        @Override\n        public void run() {\n            try {\n                if (isStarted()) {\n                    this.executeOnTimeUp();\n                }\n            } catch (Throwable e) {\n                // XXX: warn and notify me\n                log.error(\"ScheduleMessageService, executeOnTimeUp exception\", e);\n                this.scheduleNextTimerTask(this.offset, DELAY_FOR_A_PERIOD);\n            }\n        }\n\n        private long correctDeliverTimestamp(final long now, final long deliverTimestamp) {\n\n            long result = deliverTimestamp;\n\n            long maxTimestamp = now + ScheduleMessageService.this.delayLevelTable.get(this.delayLevel);\n            if (deliverTimestamp > maxTimestamp) {\n                result = now;\n            }\n\n            return result;\n        }\n\n        public void executeOnTimeUp() {\n            ConsumeQueueInterface cq =\n                ScheduleMessageService.this.brokerController.getMessageStore().getConsumeQueue(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC,\n                    delayLevel2QueueId(delayLevel));\n\n            if (cq == null) {\n                this.scheduleNextTimerTask(this.offset, DELAY_FOR_A_WHILE);\n                return;\n            }\n\n            ReferredIterator<CqUnit> bufferCQ = cq.iterateFrom(this.offset);\n            if (bufferCQ == null) {\n                long resetOffset;\n                if ((resetOffset = cq.getMinOffsetInQueue()) > this.offset) {\n                    log.error(\"schedule CQ offset invalid. offset={}, cqMinOffset={}, queueId={}\",\n                        this.offset, resetOffset, cq.getQueueId());\n                } else if ((resetOffset = cq.getMaxOffsetInQueue()) < this.offset) {\n                    log.error(\"schedule CQ offset invalid. offset={}, cqMaxOffset={}, queueId={}\",\n                        this.offset, resetOffset, cq.getQueueId());\n                } else {\n                    resetOffset = this.offset;\n                }\n\n                this.scheduleNextTimerTask(resetOffset, DELAY_FOR_A_WHILE);\n                return;\n            }\n\n            long nextOffset = this.offset;\n            try {\n                while (bufferCQ.hasNext() && isStarted()) {\n                    CqUnit cqUnit = bufferCQ.next();\n                    long offsetPy = cqUnit.getPos();\n                    int sizePy = cqUnit.getSize();\n                    long tagsCode = cqUnit.getTagsCode();\n\n                    if (!cqUnit.isTagsCodeValid()) {\n                        //can't find ext content.So re compute tags code.\n                        log.error(\"[BUG] can't find consume queue extend file content!addr={}, offsetPy={}, sizePy={}\",\n                            tagsCode, offsetPy, sizePy);\n                        long msgStoreTime = ScheduleMessageService.this.brokerController.getMessageStore().getCommitLog().pickupStoreTimestamp(offsetPy, sizePy);\n                        tagsCode = computeDeliverTimestamp(delayLevel, msgStoreTime);\n                    }\n\n                    long now = System.currentTimeMillis();\n                    long deliverTimestamp = this.correctDeliverTimestamp(now, tagsCode);\n\n                    long currOffset = cqUnit.getQueueOffset();\n                    assert cqUnit.getBatchNum() == 1;\n                    nextOffset = currOffset + cqUnit.getBatchNum();\n\n                    long countdown = deliverTimestamp - now;\n                    if (countdown > 0) {\n                        this.scheduleNextTimerTask(currOffset, DELAY_FOR_A_WHILE);\n                        ScheduleMessageService.this.updateOffset(this.delayLevel, currOffset);\n                        return;\n                    }\n\n                    MessageExt msgExt = ScheduleMessageService.this.brokerController.getMessageStore().lookMessageByOffset(offsetPy, sizePy);\n                    if (msgExt == null) {\n                        continue;\n                    }\n\n                    MessageExtBrokerInner msgInner = ScheduleMessageService.this.messageTimeUp(msgExt);\n                    if (TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC.equals(msgInner.getTopic())) {\n                        log.error(\"[BUG] the real topic of schedule msg is {}, discard the msg. msg={}\",\n                            msgInner.getTopic(), msgInner);\n                        continue;\n                    }\n\n                    boolean deliverSuc;\n                    if (ScheduleMessageService.this.enableAsyncDeliver) {\n                        deliverSuc = this.asyncDeliver(msgInner, msgExt.getMsgId(), currOffset, offsetPy, sizePy);\n                    } else {\n                        deliverSuc = this.syncDeliver(msgInner, msgExt.getMsgId(), currOffset, offsetPy, sizePy);\n                    }\n\n                    if (!deliverSuc) {\n                        this.scheduleNextTimerTask(currOffset, DELAY_FOR_A_WHILE);\n                        return;\n                    }\n                }\n            } catch (Exception e) {\n                log.error(\"ScheduleMessageService, messageTimeUp execute error, offset = {}\", nextOffset, e);\n            } finally {\n                bufferCQ.release();\n            }\n\n            this.scheduleNextTimerTask(nextOffset, DELAY_FOR_A_WHILE);\n        }\n\n        public void scheduleNextTimerTask(long offset, long delay) {\n            ScheduleMessageService.this.deliverExecutorService.schedule(new DeliverDelayedMessageTimerTask(\n                this.delayLevel, offset), delay, TimeUnit.MILLISECONDS);\n        }\n\n        private boolean syncDeliver(MessageExtBrokerInner msgInner, String msgId, long offset, long offsetPy,\n            int sizePy) {\n            PutResultProcess resultProcess = deliverMessage(msgInner, msgId, offset, offsetPy, sizePy, false);\n            PutMessageResult result = resultProcess.get();\n            boolean sendStatus = result != null && result.getPutMessageStatus() == PutMessageStatus.PUT_OK;\n            if (sendStatus) {\n                ScheduleMessageService.this.updateOffset(this.delayLevel, resultProcess.getNextOffset());\n            }\n            return sendStatus;\n        }\n\n        private boolean asyncDeliver(MessageExtBrokerInner msgInner, String msgId, long offset, long offsetPy,\n            int sizePy) {\n            Queue<PutResultProcess> processesQueue = ScheduleMessageService.this.deliverPendingTable.get(this.delayLevel);\n\n            //Flow Control\n            int currentPendingNum = processesQueue.size();\n            int maxPendingLimit = brokerController.getMessageStoreConfig()\n                .getScheduleAsyncDeliverMaxPendingLimit();\n            if (currentPendingNum > maxPendingLimit) {\n                log.warn(\"Asynchronous deliver triggers flow control, \" +\n                    \"currentPendingNum={}, maxPendingLimit={}\", currentPendingNum, maxPendingLimit);\n                return false;\n            }\n\n            //Blocked\n            PutResultProcess firstProcess = processesQueue.peek();\n            if (firstProcess != null && firstProcess.need2Blocked()) {\n                log.warn(\"Asynchronous deliver block. info={}\", firstProcess.toString());\n                return false;\n            }\n\n            PutResultProcess resultProcess = deliverMessage(msgInner, msgId, offset, offsetPy, sizePy, true);\n            processesQueue.add(resultProcess);\n            return true;\n        }\n\n        private PutResultProcess deliverMessage(MessageExtBrokerInner msgInner, String msgId, long offset,\n            long offsetPy, int sizePy, boolean autoResend) {\n            CompletableFuture<PutMessageResult> future =\n                brokerController.getEscapeBridge().asyncPutMessage(msgInner);\n            return new PutResultProcess()\n                .setTopic(msgInner.getTopic())\n                .setDelayLevel(this.delayLevel)\n                .setOffset(offset)\n                .setPhysicOffset(offsetPy)\n                .setPhysicSize(sizePy)\n                .setMsgId(msgId)\n                .setAutoResend(autoResend)\n                .setFuture(future)\n                .thenProcess();\n        }\n    }\n\n    class HandlePutResultTask implements Runnable {\n        private final int delayLevel;\n\n        public HandlePutResultTask(int delayLevel) {\n            this.delayLevel = delayLevel;\n        }\n\n        @Override\n        public void run() {\n            LinkedBlockingQueue<PutResultProcess> pendingQueue =\n                ScheduleMessageService.this.deliverPendingTable.get(this.delayLevel);\n\n            // Check if the queue exists for the given level\n            if (pendingQueue == null) {\n                log.warn(\"No pending queue found for delay level: {}\", this.delayLevel);\n                return;\n            }\n\n            PutResultProcess putResultProcess;\n            while ((putResultProcess = pendingQueue.peek()) != null) {\n                try {\n                    switch (putResultProcess.getStatus()) {\n                        case SUCCESS:\n                            ScheduleMessageService.this.updateOffset(this.delayLevel, putResultProcess.getNextOffset());\n                            pendingQueue.remove();\n                            break;\n                        case RUNNING:\n                            scheduleNextTask();\n                            return;\n                        case EXCEPTION:\n                            if (!isStarted()) {\n                                log.warn(\"HandlePutResultTask shutdown, info={}\", putResultProcess.toString());\n                                return;\n                            }\n                            log.warn(\"putResultProcess error, info={}\", putResultProcess.toString());\n                            putResultProcess.doResend();\n                            break;\n                        case SKIP:\n                            log.warn(\"putResultProcess skip, info={}\", putResultProcess.toString());\n                            pendingQueue.remove();\n                            break;\n                    }\n                } catch (Exception e) {\n                    log.error(\"HandlePutResultTask exception. info={}\", putResultProcess.toString(), e);\n                    putResultProcess.doResend();\n                }\n            }\n\n            scheduleNextTask();\n        }\n\n        private void scheduleNextTask() {\n            if (isStarted()) {\n                ScheduleMessageService.this.handleExecutorService\n                    .schedule(new HandlePutResultTask(this.delayLevel), DELAY_FOR_A_SLEEP, TimeUnit.MILLISECONDS);\n            }\n        }\n    }\n\n    class PutResultProcess {\n        private String topic;\n        private long offset;\n        private long physicOffset;\n        private int physicSize;\n        private int delayLevel;\n        private String msgId;\n        private boolean autoResend = false;\n        private CompletableFuture<PutMessageResult> future;\n\n        private volatile AtomicInteger resendCount = new AtomicInteger(0);\n        private volatile ProcessStatus status = ProcessStatus.RUNNING;\n\n        public PutResultProcess setTopic(String topic) {\n            this.topic = topic;\n            return this;\n        }\n\n        public PutResultProcess setOffset(long offset) {\n            this.offset = offset;\n            return this;\n        }\n\n        public PutResultProcess setPhysicOffset(long physicOffset) {\n            this.physicOffset = physicOffset;\n            return this;\n        }\n\n        public PutResultProcess setPhysicSize(int physicSize) {\n            this.physicSize = physicSize;\n            return this;\n        }\n\n        public PutResultProcess setDelayLevel(int delayLevel) {\n            this.delayLevel = delayLevel;\n            return this;\n        }\n\n        public PutResultProcess setMsgId(String msgId) {\n            this.msgId = msgId;\n            return this;\n        }\n\n        public PutResultProcess setAutoResend(boolean autoResend) {\n            this.autoResend = autoResend;\n            return this;\n        }\n\n        public PutResultProcess setFuture(CompletableFuture<PutMessageResult> future) {\n            this.future = future;\n            return this;\n        }\n\n        public String getTopic() {\n            return topic;\n        }\n\n        public long getOffset() {\n            return offset;\n        }\n\n        public long getNextOffset() {\n            return offset + 1;\n        }\n\n        public long getPhysicOffset() {\n            return physicOffset;\n        }\n\n        public int getPhysicSize() {\n            return physicSize;\n        }\n\n        public Integer getDelayLevel() {\n            return delayLevel;\n        }\n\n        public String getMsgId() {\n            return msgId;\n        }\n\n        public boolean isAutoResend() {\n            return autoResend;\n        }\n\n        public CompletableFuture<PutMessageResult> getFuture() {\n            return future;\n        }\n\n        public AtomicInteger getResendCount() {\n            return resendCount;\n        }\n\n        public PutResultProcess thenProcess() {\n            this.future.thenAccept(this::handleResult);\n\n            this.future.exceptionally(e -> {\n                log.error(\"ScheduleMessageService put message exceptionally, info: {}\",\n                    PutResultProcess.this.toString(), e);\n\n                onException();\n                return null;\n            });\n            return this;\n        }\n\n        private void handleResult(PutMessageResult result) {\n            if (result != null && result.getPutMessageStatus() == PutMessageStatus.PUT_OK) {\n                onSuccess(result);\n            } else {\n                log.warn(\"ScheduleMessageService put message failed. info: {}.\", result);\n                onException();\n            }\n        }\n\n        public void onSuccess(PutMessageResult result) {\n            this.status = ProcessStatus.SUCCESS;\n            if (ScheduleMessageService.this.brokerController.getMessageStore().getMessageStoreConfig().isEnableScheduleMessageStats() && !result.isRemotePut()) {\n                ScheduleMessageService.this.brokerController.getBrokerStatsManager().incQueueGetNums(MixAll.SCHEDULE_CONSUMER_GROUP, TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, delayLevel - 1, result.getAppendMessageResult().getMsgNum());\n                ScheduleMessageService.this.brokerController.getBrokerStatsManager().incQueueGetSize(MixAll.SCHEDULE_CONSUMER_GROUP, TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, delayLevel - 1, result.getAppendMessageResult().getWroteBytes());\n                ScheduleMessageService.this.brokerController.getBrokerStatsManager().incGroupGetNums(MixAll.SCHEDULE_CONSUMER_GROUP, TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, result.getAppendMessageResult().getMsgNum());\n                ScheduleMessageService.this.brokerController.getBrokerStatsManager().incGroupGetSize(MixAll.SCHEDULE_CONSUMER_GROUP, TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, result.getAppendMessageResult().getWroteBytes());\n\n                Attributes attributes = ScheduleMessageService.this.brokerController.getBrokerMetricsManager().newAttributesBuilder()\n                    .put(LABEL_TOPIC, TopicValidator.RMQ_SYS_SCHEDULE_TOPIC)\n                    .put(LABEL_CONSUMER_GROUP, MixAll.SCHEDULE_CONSUMER_GROUP)\n                    .put(LABEL_IS_SYSTEM, true)\n                    .build();\n                ScheduleMessageService.this.brokerController.getBrokerMetricsManager().getMessagesOutTotal().add(result.getAppendMessageResult().getMsgNum(), attributes);\n                ScheduleMessageService.this.brokerController.getBrokerMetricsManager().getThroughputOutTotal().add(result.getAppendMessageResult().getWroteBytes(), attributes);\n\n                ScheduleMessageService.this.brokerController.getBrokerStatsManager().incTopicPutNums(this.topic, result.getAppendMessageResult().getMsgNum(), 1);\n                ScheduleMessageService.this.brokerController.getBrokerStatsManager().incTopicPutSize(this.topic, result.getAppendMessageResult().getWroteBytes());\n                ScheduleMessageService.this.brokerController.getBrokerStatsManager().incBrokerPutNums(this.topic, result.getAppendMessageResult().getMsgNum());\n\n                attributes = ScheduleMessageService.this.brokerController.getBrokerMetricsManager().newAttributesBuilder()\n                    .put(LABEL_TOPIC, topic)\n                    .put(LABEL_MESSAGE_TYPE, TopicMessageType.DELAY.getMetricsValue())\n                    .put(LABEL_IS_SYSTEM, TopicValidator.isSystemTopic(topic))\n                    .build();\n                ScheduleMessageService.this.brokerController.getBrokerMetricsManager().getMessagesInTotal().add(result.getAppendMessageResult().getMsgNum(), attributes);\n                ScheduleMessageService.this.brokerController.getBrokerMetricsManager().getThroughputInTotal().add(result.getAppendMessageResult().getWroteBytes(), attributes);\n                ScheduleMessageService.this.brokerController.getBrokerMetricsManager().getMessageSize().record(result.getAppendMessageResult().getWroteBytes() / result.getAppendMessageResult().getMsgNum(), attributes);\n            }\n        }\n\n        public void onException() {\n            log.warn(\"ScheduleMessageService onException, info: {}\", this.toString());\n            if (this.autoResend) {\n                this.status = ProcessStatus.EXCEPTION;\n            } else {\n                this.status = ProcessStatus.SKIP;\n            }\n        }\n\n        public ProcessStatus getStatus() {\n            return this.status;\n        }\n\n        public PutMessageResult get() {\n            try {\n                return this.future.get();\n            } catch (InterruptedException | ExecutionException e) {\n                return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, null);\n            }\n        }\n\n        public void doResend() {\n            log.info(\"Resend message, info: {}\", this.toString());\n\n            // Gradually increase the resend interval.\n            try {\n                Thread.sleep(Math.min(this.resendCount.incrementAndGet() * 100, 60 * 1000));\n            } catch (InterruptedException e) {\n                e.printStackTrace();\n            }\n\n            try {\n                MessageExt msgExt = ScheduleMessageService.this.brokerController.getMessageStore().lookMessageByOffset(this.physicOffset, this.physicSize);\n                if (msgExt == null) {\n                    log.warn(\"ScheduleMessageService resend not found message. info: {}\", this.toString());\n                    this.status = need2Skip() ? ProcessStatus.SKIP : ProcessStatus.EXCEPTION;\n                    return;\n                }\n\n                MessageExtBrokerInner msgInner = ScheduleMessageService.this.messageTimeUp(msgExt);\n                PutMessageResult result = ScheduleMessageService.this.brokerController.getEscapeBridge().putMessage(msgInner);\n                this.handleResult(result);\n                if (result != null && result.getPutMessageStatus() == PutMessageStatus.PUT_OK) {\n                    log.info(\"Resend message success, info: {}\", this.toString());\n                }\n            } catch (Exception e) {\n                this.status = ProcessStatus.EXCEPTION;\n                log.error(\"Resend message error, info: {}\", this.toString(), e);\n            }\n        }\n\n        public boolean need2Blocked() {\n            int maxResendNum2Blocked = ScheduleMessageService.this.brokerController.getMessageStore().getMessageStoreConfig()\n                .getScheduleAsyncDeliverMaxResendNum2Blocked();\n            return this.resendCount.get() > maxResendNum2Blocked;\n        }\n\n        public boolean need2Skip() {\n            int maxResendNum2Blocked = ScheduleMessageService.this.brokerController.getMessageStore().getMessageStoreConfig()\n                .getScheduleAsyncDeliverMaxResendNum2Blocked();\n            return this.resendCount.get() > maxResendNum2Blocked * 2;\n        }\n\n        @Override\n        public String toString() {\n            return \"PutResultProcess{\" +\n                \"topic='\" + topic + '\\'' +\n                \", offset=\" + offset +\n                \", physicOffset=\" + physicOffset +\n                \", physicSize=\" + physicSize +\n                \", delayLevel=\" + delayLevel +\n                \", msgId='\" + msgId + '\\'' +\n                \", autoResend=\" + autoResend +\n                \", resendCount=\" + resendCount +\n                \", status=\" + status +\n                '}';\n        }\n    }\n\n    enum ProcessStatus {\n        /**\n         * In process, the processing result has not yet been returned.\n         */\n        RUNNING,\n\n        /**\n         * Put message success.\n         */\n        SUCCESS,\n\n        /**\n         * Put message exception. When autoResend is true, the message will be resend.\n         */\n        EXCEPTION,\n\n        /**\n         * Skip put message. When the message cannot be looked, the message will be skipped.\n         */\n        SKIP,\n    }\n\n    public ConcurrentMap<Integer, Long> getOffsetTable() {\n        return offsetTable;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.slave;\n\nimport java.io.IOException;\nimport java.util.Iterator;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.loadbalance.MessageRequestModeManager;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerOffsetSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.MessageRequestModeSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.SetMessageRequestModeRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.config.StorePathConfigHelper;\nimport org.apache.rocketmq.store.timer.TimerCheckpoint;\nimport org.apache.rocketmq.store.timer.TimerMetrics;\n\npublic class SlaveSynchronize {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private final BrokerController brokerController;\n    private volatile String masterAddr = null;\n\n    public SlaveSynchronize(BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    public String getMasterAddr() {\n        return masterAddr;\n    }\n\n    public void setMasterAddr(String masterAddr) {\n        if (!StringUtils.equals(this.masterAddr, masterAddr)) {\n            LOGGER.info(\"Update master address from {} to {}\", this.masterAddr, masterAddr);\n            this.masterAddr = masterAddr;\n        }\n    }\n\n    public void syncAll() {\n        this.syncTopicConfig();\n        this.syncConsumerOffset();\n        this.syncDelayOffset();\n        this.syncSubscriptionGroupConfig();\n        this.syncMessageRequestMode();\n\n        if (brokerController.getMessageStoreConfig().isTimerWheelEnable()) {\n            this.syncTimerMetrics();\n        }\n    }\n\n    private void syncTopicConfig() {\n        String masterAddrBak = this.masterAddr;\n        if (masterAddrBak != null && !masterAddrBak.equals(brokerController.getBrokerAddr())) {\n            try {\n                TopicConfigAndMappingSerializeWrapper topicWrapper =\n                        this.brokerController.getBrokerOuterAPI().getAllTopicConfig(masterAddrBak);\n                TopicConfigManager topicConfigManager = this.brokerController.getTopicConfigManager();\n                if (!topicConfigManager.getDataVersion().equals(topicWrapper.getDataVersion())) {\n\n                    topicConfigManager.getDataVersion().assignNewOne(topicWrapper.getDataVersion());\n\n                    ConcurrentMap<String, TopicConfig> newTopicConfigTable = topicWrapper.getTopicConfigTable();\n                    ConcurrentMap<String, TopicConfig> topicConfigTable = topicConfigManager.getTopicConfigTable();\n\n                    //delete\n                    Iterator<Map.Entry<String, TopicConfig>> iterator = topicConfigTable.entrySet().iterator();\n                    while (iterator.hasNext()) {\n                        Map.Entry<String, TopicConfig> entry = iterator.next();\n                        if (!newTopicConfigTable.containsKey(entry.getKey())) {\n                            iterator.remove();\n                        }\n                        topicConfigManager.deleteTopicConfig(entry.getKey());\n                    }\n\n                    //update\n                    newTopicConfigTable.values().forEach(topicConfigManager::putTopicConfig);\n                    topicConfigManager.updateDataVersion();\n                    topicConfigManager.persist();\n                }\n                if (topicWrapper.getTopicQueueMappingDetailMap() != null\n                        && !topicWrapper.getMappingDataVersion().equals(this.brokerController.getTopicQueueMappingManager().getDataVersion())) {\n                    this.brokerController.getTopicQueueMappingManager().getDataVersion()\n                            .assignNewOne(topicWrapper.getMappingDataVersion());\n\n                    ConcurrentMap<String, TopicConfig> newTopicConfigTable = topicWrapper.getTopicConfigTable();\n                    //delete\n                    ConcurrentMap<String, TopicConfig> topicConfigTable = this.brokerController.getTopicConfigManager().getTopicConfigTable();\n                    topicConfigTable.entrySet().removeIf(item -> !newTopicConfigTable.containsKey(item.getKey()));\n                    //update\n                    topicConfigTable.putAll(newTopicConfigTable);\n\n                    this.brokerController.getTopicQueueMappingManager().persist();\n                }\n                LOGGER.info(\"Update slave topic config from master, {}\", masterAddrBak);\n            } catch (Exception e) {\n                LOGGER.error(\"SyncTopicConfig Exception, {}\", masterAddrBak, e);\n            }\n        }\n    }\n\n    private void syncConsumerOffset() {\n        String masterAddrBak = this.masterAddr;\n        if (masterAddrBak != null && !masterAddrBak.equals(brokerController.getBrokerAddr())) {\n            try {\n                ConsumerOffsetSerializeWrapper offsetWrapper =\n                        this.brokerController.getBrokerOuterAPI().getAllConsumerOffset(masterAddrBak);\n                this.brokerController.getConsumerOffsetManager().getOffsetTable()\n                        .putAll(offsetWrapper.getOffsetTable());\n                this.brokerController.getConsumerOffsetManager().getDataVersion().assignNewOne(offsetWrapper.getDataVersion());\n                this.brokerController.getConsumerOffsetManager().persist();\n                LOGGER.info(\"Update slave consumer offset from master, {}\", masterAddrBak);\n            } catch (Exception e) {\n                LOGGER.error(\"SyncConsumerOffset Exception, {}\", masterAddrBak, e);\n            }\n        }\n    }\n\n    private void syncDelayOffset() {\n        String masterAddrBak = this.masterAddr;\n        if (masterAddrBak != null && !masterAddrBak.equals(brokerController.getBrokerAddr())) {\n            try {\n                String delayOffset =\n                        this.brokerController.getBrokerOuterAPI().getAllDelayOffset(masterAddrBak);\n                if (delayOffset != null) {\n\n                    String fileName =\n                            StorePathConfigHelper.getDelayOffsetStorePath(this.brokerController\n                                    .getMessageStoreConfig().getStorePathRootDir());\n                    try {\n                        MixAll.string2File(delayOffset, fileName);\n                        this.brokerController.getScheduleMessageService().loadWhenSyncDelayOffset();\n                    } catch (IOException e) {\n                        LOGGER.error(\"Persist file Exception, {}\", fileName, e);\n                    }\n                }\n                LOGGER.info(\"Update slave delay offset from master, {}\", masterAddrBak);\n            } catch (Exception e) {\n                LOGGER.error(\"SyncDelayOffset Exception, {}\", masterAddrBak, e);\n            }\n        }\n    }\n\n    private void syncSubscriptionGroupConfig() {\n        String masterAddrBak = this.masterAddr;\n        if (masterAddrBak != null && !masterAddrBak.equals(brokerController.getBrokerAddr())) {\n            try {\n                SubscriptionGroupWrapper subscriptionWrapper =\n                        this.brokerController.getBrokerOuterAPI()\n                                .getAllSubscriptionGroupConfig(masterAddrBak);\n\n                if (!this.brokerController.getSubscriptionGroupManager().getDataVersion()\n                        .equals(subscriptionWrapper.getDataVersion())) {\n                    SubscriptionGroupManager subscriptionGroupManager = this.brokerController.getSubscriptionGroupManager();\n                    subscriptionGroupManager.getDataVersion().assignNewOne(subscriptionWrapper.getDataVersion());\n\n                    ConcurrentMap<String, SubscriptionGroupConfig> curSubscriptionGroupTable =\n                            subscriptionGroupManager.getSubscriptionGroupTable();\n                    ConcurrentMap<String, SubscriptionGroupConfig> newSubscriptionGroupTable =\n                            subscriptionWrapper.getSubscriptionGroupTable();\n                    // delete\n                    Iterator<Map.Entry<String, SubscriptionGroupConfig>> iterator = curSubscriptionGroupTable.entrySet().iterator();\n                    while (iterator.hasNext()) {\n                        Map.Entry<String, SubscriptionGroupConfig> configEntry = iterator.next();\n                        if (!newSubscriptionGroupTable.containsKey(configEntry.getKey())) {\n                            iterator.remove();\n                        }\n                        subscriptionGroupManager.deleteSubscriptionGroupConfig(configEntry.getKey());\n                    }\n                    // update\n                    newSubscriptionGroupTable.values().forEach(subscriptionGroupManager::putSubscriptionGroupConfig);\n                    subscriptionGroupManager.updateDataVersion();\n                    // persist\n                    subscriptionGroupManager.persist();\n                    LOGGER.info(\"Update slave Subscription Group from master, {}\", masterAddrBak);\n                }\n            } catch (Exception e) {\n                LOGGER.error(\"SyncSubscriptionGroup Exception, {}\", masterAddrBak, e);\n            }\n        }\n    }\n\n    private void syncMessageRequestMode() {\n        String masterAddrBak = this.masterAddr;\n        if (masterAddrBak != null && !masterAddrBak.equals(brokerController.getBrokerAddr())) {\n            try {\n                MessageRequestModeSerializeWrapper messageRequestModeSerializeWrapper =\n                        this.brokerController.getBrokerOuterAPI().getAllMessageRequestMode(masterAddrBak);\n\n                MessageRequestModeManager messageRequestModeManager =\n                        this.brokerController.getQueryAssignmentProcessor().getMessageRequestModeManager();\n                ConcurrentHashMap<String, ConcurrentHashMap<String, SetMessageRequestModeRequestBody>> curMessageRequestModeMap =\n                        messageRequestModeManager.getMessageRequestModeMap();\n                ConcurrentHashMap<String, ConcurrentHashMap<String, SetMessageRequestModeRequestBody>> newMessageRequestModeMap =\n                        messageRequestModeSerializeWrapper.getMessageRequestModeMap();\n\n                // delete\n                curMessageRequestModeMap.entrySet().removeIf(e -> !newMessageRequestModeMap.containsKey(e.getKey()));\n                // update\n                curMessageRequestModeMap.putAll(newMessageRequestModeMap);\n                // persist\n                messageRequestModeManager.persist();\n                LOGGER.info(\"Update slave Message Request Mode from master, {}\", masterAddrBak);\n            } catch (Exception e) {\n                LOGGER.error(\"SyncMessageRequestMode Exception, {}\", masterAddrBak, e);\n            }\n        }\n    }\n\n    public void syncTimerCheckPoint() {\n        String masterAddrBak = this.masterAddr;\n        if (masterAddrBak != null) {\n            try {\n                if (null != brokerController.getMessageStore().getTimerMessageStore() &&\n                        !brokerController.getTimerMessageStore().isShouldRunningDequeue()) {\n                    TimerCheckpoint checkpoint = this.brokerController.getBrokerOuterAPI().getTimerCheckPoint(masterAddrBak);\n                    if (null != this.brokerController.getTimerCheckpoint()) {\n                        this.brokerController.getTimerCheckpoint().setLastReadTimeMs(checkpoint.getLastReadTimeMs());\n                        this.brokerController.getTimerCheckpoint().setMasterTimerQueueOffset(checkpoint.getMasterTimerQueueOffset());\n                        this.brokerController.getTimerCheckpoint().getDataVersion().assignNewOne(checkpoint.getDataVersion());\n                    }\n                }\n            } catch (Exception e) {\n                LOGGER.error(\"syncTimerCheckPoint Exception, {}\", masterAddrBak, e);\n            }\n        }\n    }\n\n    private void syncTimerMetrics() {\n        String masterAddrBak = this.masterAddr;\n        if (masterAddrBak != null) {\n            try {\n                if (null != brokerController.getMessageStore().getTimerMessageStore()) {\n                    TimerMetrics.TimerMetricsSerializeWrapper metricsSerializeWrapper =\n                            this.brokerController.getBrokerOuterAPI().getTimerMetrics(masterAddrBak);\n                    if (!brokerController.getMessageStore().getTimerMessageStore().getTimerMetrics().getDataVersion().equals(metricsSerializeWrapper.getDataVersion())) {\n                        this.brokerController.getMessageStore().getTimerMessageStore().getTimerMetrics().getDataVersion().assignNewOne(metricsSerializeWrapper.getDataVersion());\n                        this.brokerController.getMessageStore().getTimerMessageStore().getTimerMetrics().getTimingCount().clear();\n                        this.brokerController.getMessageStore().getTimerMessageStore().getTimerMetrics().getTimingCount().putAll(metricsSerializeWrapper.getTimingCount());\n                        this.brokerController.getMessageStore().getTimerMessageStore().getTimerMetrics().persist();\n                    }\n                }\n            } catch (Exception e) {\n                LOGGER.error(\"SyncTimerMetrics Exception, {}\", masterAddrBak, e);\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/subscription/LmqSubscriptionGroupManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.subscription;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\n\npublic class LmqSubscriptionGroupManager extends SubscriptionGroupManager {\n\n    public LmqSubscriptionGroupManager(BrokerController brokerController) {\n        super(brokerController);\n    }\n\n    @Override\n    public SubscriptionGroupConfig findSubscriptionGroupConfig(final String group) {\n        if (MixAll.isLmq(group)) {\n            SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n            subscriptionGroupConfig.setGroupName(group);\n            return subscriptionGroupConfig;\n        }\n        return super.findSubscriptionGroupConfig(group);\n    }\n\n    @Override\n    public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config) {\n        if (config == null || MixAll.isLmq(config.getGroupName())) {\n            return;\n        }\n        super.updateSubscriptionGroupConfig(config);\n    }\n\n    @Override\n    public boolean containsSubscriptionGroup(String group) {\n        if (MixAll.isLmq(group)) {\n            return true;\n        } else {\n            return super.containsSubscriptionGroup(group);\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.subscription;\n\nimport com.google.common.collect.ImmutableMap;\nimport com.google.common.collect.ImmutableSortedMap;\nimport com.google.common.collect.Maps;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Map.Entry;\nimport java.util.Objects;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.stream.Collectors;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.collections.MapUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.BrokerPathConfigHelper;\nimport org.apache.rocketmq.common.ConfigManager;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.SubscriptionGroupAttributes;\nimport org.apache.rocketmq.common.attribute.AttributeUtil;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\n\n@SuppressWarnings(\"Duplicates\")\npublic class SubscriptionGroupManager extends ConfigManager {\n    protected static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n\n    protected ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroupTable =\n        new ConcurrentHashMap<>(1024);\n\n    private ConcurrentMap<String, ConcurrentMap<String, Integer>> forbiddenTable =\n        new ConcurrentHashMap<>(4);\n\n    protected final DataVersion dataVersion = new DataVersion();\n    protected transient BrokerController brokerController;\n\n    public SubscriptionGroupManager() {\n        this.init();\n    }\n\n    public SubscriptionGroupManager(BrokerController brokerController) {\n        this(brokerController, true);\n    }\n\n    public SubscriptionGroupManager(BrokerController brokerController, boolean init) {\n        this.brokerController = brokerController;\n        if (init) {\n            init();\n        }\n    }\n\n    protected void init() {\n        {\n            SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n            subscriptionGroupConfig.setGroupName(MixAll.TOOLS_CONSUMER_GROUP);\n            putSubscriptionGroupConfig(subscriptionGroupConfig);\n        }\n\n        {\n            SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n            subscriptionGroupConfig.setGroupName(MixAll.FILTERSRV_CONSUMER_GROUP);\n            putSubscriptionGroupConfig(subscriptionGroupConfig);\n        }\n\n        {\n            SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n            subscriptionGroupConfig.setGroupName(MixAll.SELF_TEST_CONSUMER_GROUP);\n            putSubscriptionGroupConfig(subscriptionGroupConfig);\n        }\n\n        {\n            SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n            subscriptionGroupConfig.setGroupName(MixAll.ONS_HTTP_PROXY_GROUP);\n            subscriptionGroupConfig.setConsumeBroadcastEnable(true);\n            putSubscriptionGroupConfig(subscriptionGroupConfig);\n        }\n\n        {\n            SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n            subscriptionGroupConfig.setGroupName(MixAll.CID_ONSAPI_PULL_GROUP);\n            subscriptionGroupConfig.setConsumeBroadcastEnable(true);\n            putSubscriptionGroupConfig(subscriptionGroupConfig);\n        }\n\n        {\n            SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n            subscriptionGroupConfig.setGroupName(MixAll.CID_ONSAPI_PERMISSION_GROUP);\n            subscriptionGroupConfig.setConsumeBroadcastEnable(true);\n            putSubscriptionGroupConfig(subscriptionGroupConfig);\n        }\n\n        {\n            SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n            subscriptionGroupConfig.setGroupName(MixAll.CID_ONSAPI_OWNER_GROUP);\n            subscriptionGroupConfig.setConsumeBroadcastEnable(true);\n            putSubscriptionGroupConfig(subscriptionGroupConfig);\n        }\n\n        {\n            SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n            subscriptionGroupConfig.setGroupName(MixAll.CID_SYS_RMQ_TRANS);\n            subscriptionGroupConfig.setConsumeBroadcastEnable(true);\n            putSubscriptionGroupConfig(subscriptionGroupConfig);\n        }\n    }\n\n    public SubscriptionGroupConfig putSubscriptionGroupConfig(SubscriptionGroupConfig subscriptionGroupConfig) {\n        return this.subscriptionGroupTable.put(subscriptionGroupConfig.getGroupName(), subscriptionGroupConfig);\n    }\n\n    protected SubscriptionGroupConfig putSubscriptionGroupConfigIfAbsent(SubscriptionGroupConfig subscriptionGroupConfig) {\n        return this.subscriptionGroupTable.putIfAbsent(subscriptionGroupConfig.getGroupName(), subscriptionGroupConfig);\n    }\n\n    protected SubscriptionGroupConfig getSubscriptionGroupConfig(String groupName) {\n        return this.subscriptionGroupTable.get(groupName);\n    }\n\n    protected SubscriptionGroupConfig removeSubscriptionGroupConfig(String groupName) {\n        return this.subscriptionGroupTable.remove(groupName);\n    }\n\n    public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config) {\n        updateSubscriptionGroupConfigWithoutPersist(config);\n        this.persist();\n    }\n\n    public void updateSubscriptionGroupConfigWithoutPersist(SubscriptionGroupConfig config) {\n        Map<String, String> newAttributes = request(config);\n        Map<String, String> currentAttributes = current(config.getGroupName());\n\n        Map<String, String> finalAttributes = AttributeUtil.alterCurrentAttributes(\n            this.subscriptionGroupTable.get(config.getGroupName()) == null,\n            SubscriptionGroupAttributes.ALL,\n            ImmutableMap.copyOf(currentAttributes),\n            ImmutableMap.copyOf(newAttributes));\n\n        config.setAttributes(finalAttributes);\n\n        SubscriptionGroupConfig old = putSubscriptionGroupConfig(config);\n        if (old != null) {\n            log.info(\"update subscription group config, old: {} new: {}\", old, config);\n        } else {\n            log.info(\"create new subscription group, {}\", config);\n        }\n\n        updateDataVersion();\n    }\n\n    public void updateSubscriptionGroupConfigList(List<SubscriptionGroupConfig> configList) {\n        if (null == configList || configList.isEmpty()) {\n            return;\n        }\n        configList.forEach(this::updateSubscriptionGroupConfigWithoutPersist);\n        this.persist();\n    }\n\n    public void updateForbidden(String group, String topic, int forbiddenIndex, boolean setOrClear) {\n        if (setOrClear) {\n            setForbidden(group, topic, forbiddenIndex);\n        } else {\n            clearForbidden(group, topic, forbiddenIndex);\n        }\n    }\n\n    /**\n     * set the bit value to 1 at the specific index (from 0)\n     */\n    public void setForbidden(String group, String topic, int forbiddenIndex) {\n        int topicForbidden = getForbidden(group, topic);\n        topicForbidden |= 1 << forbiddenIndex;\n        updateForbiddenValue(group, topic, topicForbidden);\n    }\n\n    /**\n     * clear the bit value to 0 at the specific index (from 0)\n     */\n    public void clearForbidden(String group, String topic, int forbiddenIndex) {\n        int topicForbidden = getForbidden(group, topic);\n        topicForbidden &= ~(1 << forbiddenIndex);\n        updateForbiddenValue(group, topic, topicForbidden);\n    }\n\n    public boolean getForbidden(String group, String topic, int forbiddenIndex) {\n        int topicForbidden = getForbidden(group, topic);\n        int bitForbidden = 1 << forbiddenIndex;\n        return (topicForbidden & bitForbidden) == bitForbidden;\n    }\n\n    public int getForbidden(String group, String topic) {\n        ConcurrentMap<String, Integer> topicForbiddens = this.forbiddenTable.get(group);\n        if (topicForbiddens == null) {\n            return 0;\n        }\n        Integer topicForbidden = topicForbiddens.get(topic);\n        if (topicForbidden == null || topicForbidden < 0) {\n            topicForbidden = 0;\n        }\n        return topicForbidden;\n    }\n\n    protected void updateForbiddenValue(String group, String topic, Integer forbidden) {\n        if (forbidden == null || forbidden <= 0) {\n            this.forbiddenTable.remove(group);\n            log.info(\"clear group forbidden, {}@{} \", group, topic);\n            return;\n        }\n\n        ConcurrentMap<String, Integer> topicsPermMap = this.forbiddenTable.get(group);\n        if (topicsPermMap == null) {\n            this.forbiddenTable.putIfAbsent(group, new ConcurrentHashMap<>());\n            topicsPermMap = this.forbiddenTable.get(group);\n        }\n        Integer old = topicsPermMap.put(topic, forbidden);\n        if (old != null) {\n            log.info(\"set group forbidden, {}@{} old: {} new: {}\", group, topic, old, forbidden);\n        } else {\n            log.info(\"set group forbidden, {}@{} old: {} new: {}\", group, topic, 0, forbidden);\n        }\n\n        updateDataVersion();\n\n        this.persist();\n    }\n\n    public void disableConsume(final String groupName) {\n        SubscriptionGroupConfig old = getSubscriptionGroupConfig(groupName);\n        if (old != null) {\n            old.setConsumeEnable(false);\n            updateDataVersion();\n        }\n    }\n\n    public SubscriptionGroupConfig findSubscriptionGroupConfig(final String group) {\n        SubscriptionGroupConfig subscriptionGroupConfig = getSubscriptionGroupConfig(group);\n        if (null == subscriptionGroupConfig) {\n            if (brokerController.getBrokerConfig().isAutoCreateSubscriptionGroup()\n                    || MixAll.isSysConsumerGroupAndEnableCreate(group, brokerController.getBrokerConfig().isEnableCreateSysGroup())) {\n                TopicValidator.ValidateResult result = TopicValidator.validateGroup(group);\n                if (!result.isValid()) {\n                    return null;\n                }\n                subscriptionGroupConfig = new SubscriptionGroupConfig();\n                subscriptionGroupConfig.setGroupName(group);\n                SubscriptionGroupConfig preConfig = putSubscriptionGroupConfigIfAbsent(subscriptionGroupConfig);\n                if (null == preConfig) {\n                    log.info(\"auto create a subscription group, {}\", subscriptionGroupConfig.toString());\n                }\n                updateDataVersion();\n                this.persist();\n            }\n        }\n\n        return subscriptionGroupConfig;\n    }\n\n    @Override\n    public String encode() {\n        return this.encode(false);\n    }\n\n    @Override\n    public String configFilePath() {\n        return BrokerPathConfigHelper.getSubscriptionGroupPath(this.brokerController.getMessageStoreConfig()\n            .getStorePathRootDir());\n    }\n\n    @Override\n    public void decode(String jsonString) {\n        if (jsonString != null) {\n            SubscriptionGroupManager obj = RemotingSerializable.fromJson(jsonString, SubscriptionGroupManager.class);\n            if (obj != null) {\n                this.subscriptionGroupTable.putAll(obj.subscriptionGroupTable);\n                if (obj.forbiddenTable != null) {\n                    this.forbiddenTable.putAll(obj.forbiddenTable);\n                }\n                this.dataVersion.assignNewOne(obj.dataVersion);\n                this.printLoadDataWhenFirstBoot(obj);\n            }\n        }\n    }\n\n    @Override\n    public String encode(final boolean prettyFormat) {\n        return RemotingSerializable.toJson(this, prettyFormat);\n    }\n\n    private void printLoadDataWhenFirstBoot(final SubscriptionGroupManager sgm) {\n        for (Entry<String, SubscriptionGroupConfig> next : sgm.getSubscriptionGroupTable().entrySet()) {\n            log.info(\"load exist subscription group, {}\", next.getValue().toString());\n        }\n    }\n\n    public ConcurrentMap<String, SubscriptionGroupConfig> getSubscriptionGroupTable() {\n        return subscriptionGroupTable;\n    }\n\n    public ConcurrentHashMap<String, SubscriptionGroupConfig> subGroupTable(String dataVersion, int groupSeq,\n        int maxGroupNum) {\n        // [groupSeq, groupSeq + maxGroupNum)\n        int beginIndex = groupSeq;\n        if (StringUtils.isBlank(dataVersion) || !Objects.equals(DataVersion.fromJson(dataVersion, DataVersion.class), this.dataVersion)) {\n            beginIndex = 0;\n            log.info(\"get sub subscription group table from {} due to {}\", beginIndex,\n                StringUtils.isBlank(dataVersion) ? \"DataVersion Empty\" : \"DataVersion Changed\");\n        }\n\n        ConcurrentHashMap<String, SubscriptionGroupConfig> subGroupTable = new ConcurrentHashMap<>();\n        if (beginIndex < subscriptionGroupTable.size()) {\n            int endIndex = Math.min(beginIndex + maxGroupNum, subscriptionGroupTable.size());\n\n            ImmutableSortedMap<String, SubscriptionGroupConfig> sortedMap = ImmutableSortedMap.copyOf(subscriptionGroupTable);\n            subGroupTable.putAll(sortedMap.subMap(sortedMap.keySet().asList().get(beginIndex),true,\n                sortedMap.keySet().asList().get(endIndex - 1),true));\n        }\n\n        return subGroupTable;\n    }\n\n    public ConcurrentMap<String, ConcurrentMap<String, Integer>> getForbiddenTable() {\n        return forbiddenTable;\n    }\n\n    public ConcurrentMap<String, ConcurrentMap<String, Integer>> subForbiddenTable(Set<String> groupSet) {\n        if (MapUtils.isEmpty(forbiddenTable) || CollectionUtils.isEmpty(groupSet)) {\n            return Maps.newConcurrentMap();\n        }\n\n        return forbiddenTable.entrySet().stream()\n            .filter(e -> groupSet.contains(e.getKey()))\n            .collect(Collectors.toConcurrentMap(Map.Entry::getKey, Map.Entry::getValue));\n    }\n\n    public void setForbiddenTable(\n        ConcurrentMap<String, ConcurrentMap<String, Integer>> forbiddenTable) {\n        this.forbiddenTable = forbiddenTable;\n    }\n\n    public DataVersion getDataVersion() {\n        return dataVersion;\n    }\n\n    public boolean loadDataVersion() {\n        String fileName = null;\n        try {\n            fileName = this.configFilePath();\n            String jsonString = MixAll.file2String(fileName);\n            if (jsonString != null) {\n                SubscriptionGroupManager obj = RemotingSerializable.fromJson(jsonString, SubscriptionGroupManager.class);\n                if (obj != null) {\n                    this.dataVersion.assignNewOne(obj.dataVersion);\n                    this.printLoadDataWhenFirstBoot(obj);\n                    log.info(\"load subGroup dataVersion success,{},{}\", fileName, obj.dataVersion);\n                }\n            }\n            return true;\n        } catch (Exception e) {\n            log.error(\"load subGroup dataVersion failed\" + fileName, e);\n            return false;\n        }\n    }\n\n    public void deleteSubscriptionGroupConfig(final String groupName) {\n        SubscriptionGroupConfig old = removeSubscriptionGroupConfig(groupName);\n        this.forbiddenTable.remove(groupName);\n        if (old != null) {\n            log.info(\"delete subscription group OK, subscription group:{}\", old);\n            updateDataVersion();\n            this.persist();\n        } else {\n            log.warn(\"delete subscription group failed, subscription groupName: {} not exist\", groupName);\n        }\n    }\n\n\n    public void setSubscriptionGroupTable(ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroupTable) {\n        this.subscriptionGroupTable = subscriptionGroupTable;\n    }\n\n    public boolean containsSubscriptionGroup(String group) {\n        if (StringUtils.isBlank(group)) {\n            return false;\n        }\n\n        return subscriptionGroupTable.containsKey(group);\n    }\n\n    private Map<String, String> request(SubscriptionGroupConfig subscriptionGroupConfig) {\n        return subscriptionGroupConfig.getAttributes() == null ? new HashMap<>() : subscriptionGroupConfig.getAttributes();\n    }\n\n    private Map<String, String> current(String groupName) {\n        SubscriptionGroupConfig subscriptionGroupConfig = this.subscriptionGroupTable.get(groupName);\n        if (subscriptionGroupConfig == null) {\n            return new HashMap<>();\n        } else {\n            Map<String, String> attributes = subscriptionGroupConfig.getAttributes();\n            if (attributes == null) {\n                return new HashMap<>();\n            } else {\n                return attributes;\n            }\n        }\n    }\n\n    public void setDataVersion(DataVersion dataVersion) {\n        this.dataVersion.assignNewOne(dataVersion);\n    }\n\n    public void updateDataVersion() {\n        long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;\n        dataVersion.nextVersion(stateMachineVersion);\n    }\n\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/topic/LmqTopicConfigManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.topic;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.PermName;\n\npublic class LmqTopicConfigManager extends TopicConfigManager {\n    public LmqTopicConfigManager(BrokerController brokerController) {\n        super(brokerController);\n    }\n\n    @Override\n    public TopicConfig selectTopicConfig(final String topic) {\n        if (MixAll.isLmq(topic)) {\n            return simpleLmqTopicConfig(topic);\n        }\n        return super.selectTopicConfig(topic);\n    }\n\n    @Override\n    public void updateTopicConfig(final TopicConfig topicConfig) {\n        if (topicConfig == null || MixAll.isLmq(topicConfig.getTopicName())) {\n            return;\n        }\n        super.updateTopicConfig(topicConfig);\n    }\n\n    private TopicConfig simpleLmqTopicConfig(String topic) {\n        return new TopicConfig(topic, 1, 1, PermName.PERM_READ | PermName.PERM_WRITE);\n    }\n\n    @Override\n    public boolean containsTopic(String topic) {\n        if (MixAll.isLmq(topic)) {\n            return true;\n        }\n        return super.containsTopic(topic);\n    }\n\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.topic;\n\nimport java.util.HashMap;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Map.Entry;\nimport java.util.Objects;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.locks.Lock;\nimport java.util.concurrent.locks.ReentrantLock;\n\nimport com.google.common.collect.ImmutableMap;\nimport com.google.common.collect.ImmutableSortedMap;\nimport com.google.common.collect.Maps;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.BrokerPathConfigHelper;\nimport org.apache.rocketmq.common.ConfigManager;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.PopAckConstants;\nimport org.apache.rocketmq.common.TopicAttributes;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.attribute.Attribute;\nimport org.apache.rocketmq.common.attribute.AttributeUtil;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.sysflag.TopicSysFlag;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.body.KVTable;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo;\nimport org.apache.rocketmq.store.timer.TimerMessageStore;\nimport org.apache.rocketmq.tieredstore.TieredMessageStore;\nimport org.apache.rocketmq.tieredstore.metadata.MetadataStore;\nimport org.apache.rocketmq.tieredstore.metadata.entity.TopicMetadata;\n\nimport static com.google.common.base.Preconditions.checkNotNull;\n\npublic class TopicConfigManager extends ConfigManager {\n    protected static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private static final long LOCK_TIMEOUT_MILLIS = 3000;\n    private static final int SCHEDULE_TOPIC_QUEUE_NUM = 18;\n\n    private transient final Lock topicConfigTableLock = new ReentrantLock();\n    protected ConcurrentMap<String, TopicConfig> topicConfigTable = new ConcurrentHashMap<>(1024);\n    protected DataVersion dataVersion = new DataVersion();\n    protected transient BrokerController brokerController;\n\n    public TopicConfigManager() {\n\n    }\n\n    public TopicConfigManager(BrokerController brokerController) {\n        this(brokerController, true);\n    }\n\n    public TopicConfigManager(BrokerController brokerController, boolean init) {\n        this.brokerController = brokerController;\n        if (init) {\n            init();\n        }\n    }\n\n    protected void init() {\n        {\n            String topic = TopicValidator.RMQ_SYS_SELF_TEST_TOPIC;\n            TopicConfig topicConfig = new TopicConfig(topic);\n            TopicValidator.addSystemTopic(topic);\n            topicConfig.setReadQueueNums(1);\n            topicConfig.setWriteQueueNums(1);\n            putTopicConfig(topicConfig);\n        }\n        {\n            if (this.brokerController.getBrokerConfig().isAutoCreateTopicEnable()) {\n                String topic = TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC;\n                TopicConfig topicConfig = new TopicConfig(topic);\n                TopicValidator.addSystemTopic(topic);\n                topicConfig.setReadQueueNums(this.brokerController.getBrokerConfig()\n                    .getDefaultTopicQueueNums());\n                topicConfig.setWriteQueueNums(this.brokerController.getBrokerConfig()\n                    .getDefaultTopicQueueNums());\n                int perm = PermName.PERM_INHERIT | PermName.PERM_READ | PermName.PERM_WRITE;\n                topicConfig.setPerm(perm);\n                putTopicConfig(topicConfig);\n            }\n        }\n        {\n            String topic = TopicValidator.RMQ_SYS_BENCHMARK_TOPIC;\n            TopicConfig topicConfig = new TopicConfig(topic);\n            TopicValidator.addSystemTopic(topic);\n            topicConfig.setReadQueueNums(1024);\n            topicConfig.setWriteQueueNums(1024);\n            putTopicConfig(topicConfig);\n        }\n        {\n            String topic = this.brokerController.getBrokerConfig().getBrokerClusterName();\n            TopicConfig topicConfig = new TopicConfig(topic);\n            TopicValidator.addSystemTopic(topic);\n            int perm = PermName.PERM_INHERIT;\n            if (this.brokerController.getBrokerConfig().isClusterTopicEnable()) {\n                perm |= PermName.PERM_READ | PermName.PERM_WRITE;\n            }\n            topicConfig.setPerm(perm);\n            putTopicConfig(topicConfig);\n        }\n        {\n\n            String topic = this.brokerController.getBrokerConfig().getBrokerName();\n            TopicConfig topicConfig = new TopicConfig(topic);\n            TopicValidator.addSystemTopic(topic);\n            int perm = PermName.PERM_INHERIT;\n            if (this.brokerController.getBrokerConfig().isBrokerTopicEnable()) {\n                perm |= PermName.PERM_READ | PermName.PERM_WRITE;\n            }\n            topicConfig.setReadQueueNums(1);\n            topicConfig.setWriteQueueNums(1);\n            topicConfig.setPerm(perm);\n            putTopicConfig(topicConfig);\n        }\n        {\n            String topic = TopicValidator.RMQ_SYS_OFFSET_MOVED_EVENT;\n            TopicConfig topicConfig = new TopicConfig(topic);\n            TopicValidator.addSystemTopic(topic);\n            topicConfig.setReadQueueNums(1);\n            topicConfig.setWriteQueueNums(1);\n            putTopicConfig(topicConfig);\n        }\n        {\n            String topic = TopicValidator.RMQ_SYS_SCHEDULE_TOPIC;\n            TopicConfig topicConfig = new TopicConfig(topic);\n            TopicValidator.addSystemTopic(topic);\n            topicConfig.setReadQueueNums(SCHEDULE_TOPIC_QUEUE_NUM);\n            topicConfig.setWriteQueueNums(SCHEDULE_TOPIC_QUEUE_NUM);\n            putTopicConfig(topicConfig);\n        }\n        {\n            if (this.brokerController.getBrokerConfig().isTraceTopicEnable()) {\n                String topic = this.brokerController.getBrokerConfig().getMsgTraceTopicName();\n                TopicConfig topicConfig = new TopicConfig(topic);\n                TopicValidator.addSystemTopic(topic);\n                topicConfig.setReadQueueNums(1);\n                topicConfig.setWriteQueueNums(1);\n                putTopicConfig(topicConfig);\n            }\n        }\n        {\n            String topic = this.brokerController.getBrokerConfig().getBrokerClusterName() + \"_\" + MixAll.REPLY_TOPIC_POSTFIX;\n            TopicConfig topicConfig = new TopicConfig(topic);\n            TopicValidator.addSystemTopic(topic);\n            topicConfig.setReadQueueNums(1);\n            topicConfig.setWriteQueueNums(1);\n            putTopicConfig(topicConfig);\n        }\n        {\n            // PopAckConstants.REVIVE_TOPIC\n            String topic = PopAckConstants.buildClusterReviveTopic(this.brokerController.getBrokerConfig().getBrokerClusterName());\n            TopicConfig topicConfig = new TopicConfig(topic);\n            TopicValidator.addSystemTopic(topic);\n            topicConfig.setReadQueueNums(this.brokerController.getBrokerConfig().getReviveQueueNum());\n            topicConfig.setWriteQueueNums(this.brokerController.getBrokerConfig().getReviveQueueNum());\n            putTopicConfig(topicConfig);\n        }\n        {\n            // sync broker member group topic\n            String topic = TopicValidator.SYNC_BROKER_MEMBER_GROUP_PREFIX + this.brokerController.getBrokerConfig().getBrokerName();\n            TopicConfig topicConfig = new TopicConfig(topic);\n            TopicValidator.addSystemTopic(topic);\n            topicConfig.setReadQueueNums(1);\n            topicConfig.setWriteQueueNums(1);\n            topicConfig.setPerm(PermName.PERM_INHERIT);\n            putTopicConfig(topicConfig);\n        }\n        {\n            // TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC\n            String topic = TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC;\n            TopicConfig topicConfig = new TopicConfig(topic);\n            TopicValidator.addSystemTopic(topic);\n            topicConfig.setReadQueueNums(1);\n            topicConfig.setWriteQueueNums(1);\n            putTopicConfig(topicConfig);\n        }\n\n        {\n            // TopicValidator.RMQ_SYS_TRANS_OP_HALF_TOPIC\n            String topic = TopicValidator.RMQ_SYS_TRANS_OP_HALF_TOPIC;\n            TopicConfig topicConfig = new TopicConfig(topic);\n            TopicValidator.addSystemTopic(topic);\n            topicConfig.setReadQueueNums(1);\n            topicConfig.setWriteQueueNums(1);\n            putTopicConfig(topicConfig);\n        }\n\n        {\n            // TopicValidator.RMQ_SYS_ROCKSDB_TRANS_HALF_TOPIC\n            String topic = TopicValidator.RMQ_SYS_ROCKSDB_TRANS_HALF_TOPIC;\n            TopicConfig topicConfig = new TopicConfig(topic);\n            TopicValidator.addSystemTopic(topic);\n            topicConfig.setReadQueueNums(1);\n            topicConfig.setWriteQueueNums(1);\n            putTopicConfig(topicConfig);\n        }\n\n        {\n            // TopicValidator.RMQ_SYS_ROCKSDB_TRANS_OP_HALF_TOPIC\n            String topic = TopicValidator.RMQ_SYS_ROCKSDB_TRANS_OP_HALF_TOPIC;\n            TopicConfig topicConfig = new TopicConfig(topic);\n            TopicValidator.addSystemTopic(topic);\n            topicConfig.setReadQueueNums(1);\n            topicConfig.setWriteQueueNums(1);\n            putTopicConfig(topicConfig);\n        }\n\n        {\n            if (this.brokerController.getMessageStoreConfig().isTimerWheelEnable()) {\n                String topic = TimerMessageStore.TIMER_TOPIC;\n                TopicConfig topicConfig = new TopicConfig(topic);\n                TopicValidator.addSystemTopic(topic);\n                topicConfig.setReadQueueNums(1);\n                topicConfig.setWriteQueueNums(1);\n                this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);\n            }\n        }\n    }\n\n    public TopicConfig putTopicConfig(TopicConfig topicConfig) {\n        return this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);\n    }\n\n    protected TopicConfig getTopicConfig(String topicName) {\n        return this.topicConfigTable.get(topicName);\n    }\n\n    protected TopicConfig removeTopicConfig(String topicName) {\n        return this.topicConfigTable.remove(topicName);\n    }\n\n    public TopicConfig selectTopicConfig(final String topic) {\n        return getTopicConfig(topic);\n    }\n\n    public TopicConfig createTopicInSendMessageMethod(final String topic, final String defaultTopic,\n        final String remoteAddress, final int clientDefaultTopicQueueNums, final int topicSysFlag) {\n        TopicConfig topicConfig = null;\n        boolean createNew = false;\n\n        try {\n            if (this.topicConfigTableLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {\n                try {\n                    topicConfig = getTopicConfig(topic);\n                    if (topicConfig != null) {\n                        return topicConfig;\n                    }\n\n                    TopicConfig defaultTopicConfig = getTopicConfig(defaultTopic);\n                    if (defaultTopicConfig != null) {\n                        if (defaultTopic.equals(TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC)) {\n                            if (!this.brokerController.getBrokerConfig().isAutoCreateTopicEnable()) {\n                                defaultTopicConfig.setPerm(PermName.PERM_READ | PermName.PERM_WRITE);\n                            }\n                        }\n\n                        if (PermName.isInherited(defaultTopicConfig.getPerm())) {\n                            topicConfig = new TopicConfig(topic);\n\n                            int queueNums = Math.min(clientDefaultTopicQueueNums, defaultTopicConfig.getWriteQueueNums());\n\n                            if (queueNums < 0) {\n                                queueNums = 0;\n                            }\n\n                            topicConfig.setReadQueueNums(queueNums);\n                            topicConfig.setWriteQueueNums(queueNums);\n                            int perm = defaultTopicConfig.getPerm();\n                            perm &= ~PermName.PERM_INHERIT;\n                            topicConfig.setPerm(perm);\n                            topicConfig.setTopicSysFlag(topicSysFlag);\n                            topicConfig.setTopicFilterType(defaultTopicConfig.getTopicFilterType());\n                        } else {\n                            log.warn(\"Create new topic failed, because the default topic[{}] has no perm [{}] producer:[{}]\",\n                                defaultTopic, defaultTopicConfig.getPerm(), remoteAddress);\n                        }\n                    } else {\n                        log.warn(\"Create new topic failed, because the default topic[{}] not exist. producer:[{}]\",\n                            defaultTopic, remoteAddress);\n                    }\n\n                    if (topicConfig != null) {\n                        log.info(\"Create new topic by default topic:[{}] config:[{}] producer:[{}]\",\n                            defaultTopic, topicConfig, remoteAddress);\n\n                        putTopicConfig(topicConfig);\n\n                        updateDataVersion();\n\n                        createNew = true;\n\n                        this.persist();\n                    }\n                } finally {\n                    this.topicConfigTableLock.unlock();\n                }\n            }\n        } catch (InterruptedException e) {\n            log.error(\"createTopicInSendMessageMethod exception\", e);\n        }\n\n        if (createNew) {\n            registerBrokerData(topicConfig);\n        }\n\n        return topicConfig;\n    }\n\n    public TopicConfig createTopicIfAbsent(TopicConfig topicConfig) {\n        return createTopicIfAbsent(topicConfig, true);\n    }\n\n    public TopicConfig createTopicIfAbsent(TopicConfig topicConfig, boolean register) {\n        boolean createNew = false;\n        if (topicConfig == null) {\n            throw new NullPointerException(\"TopicConfig\");\n        }\n        if (StringUtils.isEmpty(topicConfig.getTopicName())) {\n            throw new IllegalArgumentException(\"TopicName\");\n        }\n\n        try {\n            if (this.topicConfigTableLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {\n                try {\n                    TopicConfig existedTopicConfig = getTopicConfig(topicConfig.getTopicName());\n                    if (existedTopicConfig != null) {\n                        return existedTopicConfig;\n                    }\n                    log.info(\"Create new topic [{}] config:[{}]\", topicConfig.getTopicName(), topicConfig);\n                    putTopicConfig(topicConfig);\n                    updateDataVersion();\n                    createNew = true;\n                    this.persist();\n                } finally {\n                    this.topicConfigTableLock.unlock();\n                }\n            }\n        } catch (InterruptedException e) {\n            log.error(\"createTopicIfAbsent \", e);\n        }\n        if (createNew && register) {\n            registerBrokerData(topicConfig);\n        }\n        return getTopicConfig(topicConfig.getTopicName());\n    }\n\n    public TopicConfig createTopicInSendMessageBackMethod(\n        final String topic,\n        final int clientDefaultTopicQueueNums,\n        final int perm,\n        final int topicSysFlag) {\n        return createTopicInSendMessageBackMethod(topic, clientDefaultTopicQueueNums, perm, false, topicSysFlag);\n    }\n\n    public TopicConfig createTopicInSendMessageBackMethod(\n        final String topic,\n        final int clientDefaultTopicQueueNums,\n        final int perm,\n        final boolean isOrder,\n        final int topicSysFlag) {\n        TopicConfig topicConfig = getTopicConfig(topic);\n        if (topicConfig != null) {\n            if (isOrder != topicConfig.isOrder()) {\n                topicConfig.setOrder(isOrder);\n                this.updateTopicConfig(topicConfig);\n            }\n            return topicConfig;\n        }\n\n        boolean createNew = false;\n\n        try {\n            if (this.topicConfigTableLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {\n                try {\n                    topicConfig = getTopicConfig(topic);\n                    if (topicConfig != null) {\n                        return topicConfig;\n                    }\n\n                    topicConfig = new TopicConfig(topic);\n                    topicConfig.setReadQueueNums(clientDefaultTopicQueueNums);\n                    topicConfig.setWriteQueueNums(clientDefaultTopicQueueNums);\n                    topicConfig.setPerm(perm);\n                    topicConfig.setTopicSysFlag(topicSysFlag);\n                    topicConfig.setOrder(isOrder);\n\n                    log.info(\"create new topic {}\", topicConfig);\n                    putTopicConfig(topicConfig);\n                    createNew = true;\n                    updateDataVersion();\n                    this.persist();\n                } finally {\n                    this.topicConfigTableLock.unlock();\n                }\n            }\n        } catch (InterruptedException e) {\n            log.error(\"createTopicInSendMessageBackMethod exception\", e);\n        }\n\n        if (createNew) {\n            registerBrokerData(topicConfig);\n        }\n\n        return topicConfig;\n    }\n\n    public TopicConfig createTopicOfTranCheckMaxTime(final int clientDefaultTopicQueueNums, final int perm) {\n        TopicConfig topicConfig = getTopicConfig(TopicValidator.RMQ_SYS_TRANS_CHECK_MAX_TIME_TOPIC);\n        if (topicConfig != null)\n            return topicConfig;\n\n        boolean createNew = false;\n\n        try {\n            if (this.topicConfigTableLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {\n                try {\n                    topicConfig = getTopicConfig(TopicValidator.RMQ_SYS_TRANS_CHECK_MAX_TIME_TOPIC);\n                    if (topicConfig != null)\n                        return topicConfig;\n\n                    topicConfig = new TopicConfig(TopicValidator.RMQ_SYS_TRANS_CHECK_MAX_TIME_TOPIC);\n                    topicConfig.setReadQueueNums(clientDefaultTopicQueueNums);\n                    topicConfig.setWriteQueueNums(clientDefaultTopicQueueNums);\n                    topicConfig.setPerm(perm);\n                    topicConfig.setTopicSysFlag(0);\n\n                    log.info(\"create new topic {}\", topicConfig);\n                    putTopicConfig(topicConfig);\n                    createNew = true;\n                    updateDataVersion();\n                    this.persist();\n                } finally {\n                    this.topicConfigTableLock.unlock();\n                }\n            }\n        } catch (InterruptedException e) {\n            log.error(\"create TRANS_CHECK_MAX_TIME_TOPIC exception\", e);\n        }\n\n        if (createNew) {\n            registerBrokerData(topicConfig);\n        }\n\n        return topicConfig;\n    }\n\n    public void updateTopicUnitFlag(final String topic, final boolean unit) {\n\n        TopicConfig topicConfig = getTopicConfig(topic);\n        if (topicConfig != null) {\n            int oldTopicSysFlag = topicConfig.getTopicSysFlag();\n            if (unit) {\n                topicConfig.setTopicSysFlag(TopicSysFlag.setUnitFlag(oldTopicSysFlag));\n            } else {\n                topicConfig.setTopicSysFlag(TopicSysFlag.clearUnitFlag(oldTopicSysFlag));\n            }\n\n            log.info(\"update topic sys flag. oldTopicSysFlag={}, newTopicSysFlag={}\", oldTopicSysFlag,\n                topicConfig.getTopicSysFlag());\n\n            putTopicConfig(topicConfig);\n\n            updateDataVersion();\n\n            this.persist();\n            registerBrokerData(topicConfig);\n        }\n    }\n\n    public void updateTopicUnitSubFlag(final String topic, final boolean hasUnitSub) {\n        TopicConfig topicConfig = getTopicConfig(topic);\n        if (topicConfig != null) {\n            int oldTopicSysFlag = topicConfig.getTopicSysFlag();\n            if (hasUnitSub) {\n                topicConfig.setTopicSysFlag(TopicSysFlag.setUnitSubFlag(oldTopicSysFlag));\n            } else {\n                topicConfig.setTopicSysFlag(TopicSysFlag.clearUnitSubFlag(oldTopicSysFlag));\n            }\n\n            log.info(\"update topic sys flag. oldTopicSysFlag={}, newTopicSysFlag={}\", oldTopicSysFlag,\n                topicConfig.getTopicSysFlag());\n\n            putTopicConfig(topicConfig);\n\n            updateDataVersion();\n\n            this.persist();\n            registerBrokerData(topicConfig);\n        }\n    }\n\n    public void updateSingleTopicConfigWithoutPersist(final TopicConfig topicConfig) {\n        checkNotNull(topicConfig, \"topicConfig shouldn't be null\");\n\n        Map<String, String> newAttributes = request(topicConfig);\n        Map<String, String> currentAttributes = current(topicConfig.getTopicName());\n\n        Map<String, String> finalAttributes = AttributeUtil.alterCurrentAttributes(\n            this.topicConfigTable.get(topicConfig.getTopicName()) == null,\n            TopicAttributes.ALL,\n            ImmutableMap.copyOf(currentAttributes),\n            ImmutableMap.copyOf(newAttributes));\n\n        topicConfig.setAttributes(finalAttributes);\n        updateTieredStoreTopicMetadata(topicConfig, newAttributes);\n\n        TopicConfig old = putTopicConfig(topicConfig);\n        if (old != null) {\n            log.info(\"update topic config, old:[{}] new:[{}]\", old, topicConfig);\n        } else {\n            log.info(\"create new topic [{}]\", topicConfig);\n        }\n\n        updateDataVersion();\n    }\n\n    public void updateTopicConfig(final TopicConfig topicConfig) {\n        updateSingleTopicConfigWithoutPersist(topicConfig);\n        this.persist(topicConfig.getTopicName(), topicConfig);\n    }\n\n    public void updateTopicConfigList(final List<TopicConfig> topicConfigList) {\n        topicConfigList.forEach(this::updateSingleTopicConfigWithoutPersist);\n        this.persist();\n    }\n\n    private synchronized void updateTieredStoreTopicMetadata(final TopicConfig topicConfig, Map<String, String> newAttributes) {\n        if (!(brokerController.getMessageStore() instanceof TieredMessageStore)) {\n            if (newAttributes.get(TopicAttributes.TOPIC_RESERVE_TIME_ATTRIBUTE.getName()) != null) {\n                throw new IllegalArgumentException(\"Update topic reserveTime not supported\");\n            }\n            return;\n        }\n\n        String topic = topicConfig.getTopicName();\n        long reserveTime = TopicAttributes.TOPIC_RESERVE_TIME_ATTRIBUTE.getDefaultValue();\n        String attr = topicConfig.getAttributes().get(TopicAttributes.TOPIC_RESERVE_TIME_ATTRIBUTE.getName());\n        if (attr != null) {\n            reserveTime = Long.parseLong(attr);\n        }\n\n        log.info(\"Update tiered storage metadata, topic {}, reserveTime {}\", topic, reserveTime);\n        TieredMessageStore tieredMessageStore = (TieredMessageStore) brokerController.getMessageStore();\n        MetadataStore metadataStore = tieredMessageStore.getMetadataStore();\n        TopicMetadata topicMetadata = metadataStore.getTopic(topic);\n        if (topicMetadata == null) {\n            metadataStore.addTopic(topic, reserveTime);\n        } else if (topicMetadata.getReserveTime() != reserveTime) {\n            topicMetadata.setReserveTime(reserveTime);\n            metadataStore.updateTopic(topicMetadata);\n        }\n    }\n\n    public void updateOrderTopicConfig(final KVTable orderKVTableFromNs) {\n\n        if (orderKVTableFromNs != null && orderKVTableFromNs.getTable() != null) {\n            boolean isChange = false;\n            Set<String> orderTopics = orderKVTableFromNs.getTable().keySet();\n            for (String topic : orderTopics) {\n                TopicConfig topicConfig = getTopicConfig(topic);\n                if (topicConfig != null && !topicConfig.isOrder()) {\n                    topicConfig.setOrder(true);\n                    isChange = true;\n                    log.info(\"update order topic config, topic={}, order={}\", topic, true);\n                }\n            }\n\n            if (isChange) {\n                updateDataVersion();\n                this.persist();\n            }\n        }\n    }\n\n    // make it testable\n    public Map<String, Attribute> allAttributes() {\n        return TopicAttributes.ALL;\n    }\n\n    public boolean isOrderTopic(final String topic) {\n        TopicConfig topicConfig = getTopicConfig(topic);\n        if (topicConfig == null) {\n            return false;\n        } else {\n            return topicConfig.isOrder();\n        }\n    }\n\n    public void deleteTopicConfig(final String topic) {\n        TopicConfig old = removeTopicConfig(topic);\n        if (old != null) {\n            log.info(\"delete topic config OK, topic: {}\", old);\n            updateDataVersion();\n            this.persist();\n        } else {\n            log.warn(\"delete topic config failed, topic: {} not exists\", topic);\n        }\n    }\n\n    public TopicConfigSerializeWrapper buildTopicConfigSerializeWrapper() {\n        TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();\n        topicConfigSerializeWrapper.setTopicConfigTable(this.topicConfigTable);\n        DataVersion dataVersionCopy = new DataVersion();\n        dataVersionCopy.assignNewOne(this.dataVersion);\n        topicConfigSerializeWrapper.setDataVersion(dataVersionCopy);\n        return topicConfigSerializeWrapper;\n    }\n\n    public TopicConfigAndMappingSerializeWrapper buildSerializeWrapper(final ConcurrentMap<String, TopicConfig> topicConfigTable) {\n        return buildSerializeWrapper(topicConfigTable, Maps.newHashMap());\n    }\n\n    public TopicConfigAndMappingSerializeWrapper buildSerializeWrapper(\n        final ConcurrentMap<String, TopicConfig> topicConfigTable,\n        final Map<String, TopicQueueMappingInfo> topicQueueMappingInfoMap\n    ) {\n        TopicConfigAndMappingSerializeWrapper topicConfigWrapper = new TopicConfigAndMappingSerializeWrapper();\n        topicConfigWrapper.setTopicConfigTable(topicConfigTable);\n        topicConfigWrapper.setTopicQueueMappingInfoMap(topicQueueMappingInfoMap);\n        topicConfigWrapper.setDataVersion(this.getDataVersion());\n        return topicConfigWrapper;\n    }\n\n    @Override\n    public String encode() {\n        return encode(false);\n    }\n\n    public boolean loadDataVersion() {\n        String fileName = null;\n        try {\n            fileName = this.configFilePath();\n            String jsonString = MixAll.file2String(fileName);\n            if (jsonString != null) {\n                TopicConfigSerializeWrapper topicConfigSerializeWrapper =\n                    TopicConfigSerializeWrapper.fromJson(jsonString, TopicConfigSerializeWrapper.class);\n                if (topicConfigSerializeWrapper != null) {\n                    this.dataVersion.assignNewOne(topicConfigSerializeWrapper.getDataVersion());\n                    log.info(\"load topic metadata dataVersion success {}, {}\", fileName, topicConfigSerializeWrapper.getDataVersion());\n                }\n            }\n            return true;\n        } catch (Exception e) {\n            log.error(\"load topic metadata dataVersion failed\" + fileName, e);\n            return false;\n        }\n    }\n\n    @Override\n    public String configFilePath() {\n        return BrokerPathConfigHelper.getTopicConfigPath(this.brokerController.getMessageStoreConfig().getStorePathRootDir());\n    }\n\n    @Override\n    public void decode(String jsonString) {\n        if (jsonString != null) {\n            TopicConfigSerializeWrapper topicConfigSerializeWrapper =\n                TopicConfigSerializeWrapper.fromJson(jsonString, TopicConfigSerializeWrapper.class);\n            if (topicConfigSerializeWrapper != null) {\n                this.topicConfigTable.putAll(topicConfigSerializeWrapper.getTopicConfigTable());\n                this.dataVersion.assignNewOne(topicConfigSerializeWrapper.getDataVersion());\n                this.printLoadDataWhenFirstBoot(topicConfigSerializeWrapper);\n            }\n        }\n    }\n\n    public String encode(final boolean prettyFormat) {\n        TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();\n        topicConfigSerializeWrapper.setTopicConfigTable(this.topicConfigTable);\n        topicConfigSerializeWrapper.setDataVersion(getDataVersion());\n        return topicConfigSerializeWrapper.toJson(prettyFormat);\n    }\n\n    private void printLoadDataWhenFirstBoot(final TopicConfigSerializeWrapper tcs) {\n        Iterator<Entry<String, TopicConfig>> it = tcs.getTopicConfigTable().entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, TopicConfig> next = it.next();\n            log.info(\"load exist local topic, {}\", next.getValue().toString());\n        }\n    }\n\n    public DataVersion getDataVersion() {\n        return dataVersion;\n    }\n\n    public void setTopicConfigTable(\n        ConcurrentMap<String, TopicConfig> topicConfigTable) {\n        this.topicConfigTable = topicConfigTable;\n    }\n\n    public ConcurrentMap<String, TopicConfig> getTopicConfigTable() {\n        return topicConfigTable;\n    }\n\n    public ConcurrentHashMap<String, TopicConfig> subTopicConfigTable(String dataVersion, int topicSeq,\n        int maxTopicNum) {\n        // [topicSeq, topicSeq + maxTopicNum)\n        int beginIndex = topicSeq;\n        if (StringUtils.isBlank(dataVersion) || !Objects.equals(DataVersion.fromJson(dataVersion, DataVersion.class), this.dataVersion)) {\n            beginIndex = 0;\n            log.info(\"get sub topic config table from {} due to {}\", beginIndex,\n                StringUtils.isBlank(dataVersion) ? \"DataVersion Empty\" : \"DataVersion Changed\");\n        }\n\n        ConcurrentHashMap<String, TopicConfig> subTopicConfigTable = new ConcurrentHashMap<>();\n        if (beginIndex < topicConfigTable.size()) {\n            int endIndex = Math.min(beginIndex + maxTopicNum, topicConfigTable.size());\n\n            ImmutableSortedMap<String, TopicConfig> sortedMap = ImmutableSortedMap.copyOf(topicConfigTable);\n            subTopicConfigTable.putAll(sortedMap.subMap(sortedMap.keySet().asList().get(beginIndex),true,\n                sortedMap.keySet().asList().get(endIndex - 1),true));\n        }\n\n        return subTopicConfigTable;\n    }\n\n    private Map<String, String> request(TopicConfig topicConfig) {\n        return topicConfig.getAttributes() == null ? new HashMap<>() : topicConfig.getAttributes();\n    }\n\n    private Map<String, String> current(String topic) {\n        TopicConfig topicConfig = getTopicConfig(topic);\n        if (topicConfig == null) {\n            return new HashMap<>();\n        } else {\n            Map<String, String> attributes = topicConfig.getAttributes();\n            if (attributes == null) {\n                return new HashMap<>();\n            } else {\n                return attributes;\n            }\n        }\n    }\n\n    private void registerBrokerData(TopicConfig topicConfig) {\n        if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) {\n            this.brokerController.registerSingleTopicAll(topicConfig);\n        } else {\n            this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion);\n        }\n    }\n\n    public boolean containsTopic(String topic) {\n        return topicConfigTable.containsKey(topic);\n    }\n\n    public void updateDataVersion() {\n        long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;\n        dataVersion.nextVersion(stateMachineVersion);\n    }\n\n\n\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/topic/TopicQueueMappingCleanService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.topic;\n\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.out.BrokerOuterAPI;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.admin.TopicOffset;\nimport org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;\nimport org.apache.rocketmq.remoting.protocol.header.GetTopicConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetTopicStatsInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.apache.rocketmq.remoting.protocol.statictopic.LogicQueueMappingItem;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicConfigAndQueueMapping;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingDetail;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingUtils;\nimport org.apache.rocketmq.remoting.rpc.ClientMetadata;\nimport org.apache.rocketmq.remoting.rpc.RpcClient;\nimport org.apache.rocketmq.remoting.rpc.RpcRequest;\nimport org.apache.rocketmq.remoting.rpc.RpcResponse;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\n\npublic class TopicQueueMappingCleanService extends ServiceThread {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n\n    private TopicQueueMappingManager topicQueueMappingManager;\n    private BrokerOuterAPI brokerOuterAPI;\n    private RpcClient rpcClient;\n    private MessageStoreConfig messageStoreConfig;\n    private BrokerConfig brokerConfig;\n    private BrokerController brokerController;\n\n    public TopicQueueMappingCleanService(BrokerController brokerController) {\n        this.brokerController = brokerController;\n        this.topicQueueMappingManager = brokerController.getTopicQueueMappingManager();\n        this.rpcClient = brokerController.getBrokerOuterAPI().getRpcClient();\n        this.messageStoreConfig = brokerController.getMessageStoreConfig();\n        this.brokerConfig = brokerController.getBrokerConfig();\n        this.brokerOuterAPI = brokerController.getBrokerOuterAPI();\n    }\n\n    @Override\n    public String getServiceName() {\n        if (this.brokerConfig.isInBrokerContainer()) {\n            return this.brokerController.getBrokerIdentity().getIdentifier() + TopicQueueMappingCleanService.class.getSimpleName();\n        }\n        return TopicQueueMappingCleanService.class.getSimpleName();\n    }\n\n    @Override\n    public void run() {\n        log.info(\"Start topic queue mapping clean service thread!\");\n        while (!this.isStopped()) {\n            try {\n                this.waitForRunning(5L * 60 * 1000);\n            } catch (Throwable ignored) {\n            }\n            try {\n                cleanItemExpired();\n            } catch (Throwable t) {\n                log.error(\"topic queue mapping cleanItemExpired failed\", t);\n            }\n            try {\n                cleanItemListMoreThanSecondGen();\n            } catch (Throwable t) {\n                log.error(\"topic queue mapping cleanItemListMoreThanSecondGen failed\", t);\n            }\n\n        }\n        log.info(\"End topic queue mapping clean service  thread!\");\n    }\n\n\n\n    public void cleanItemExpired() {\n        String when = messageStoreConfig.getDeleteWhen();\n        if (!UtilAll.isItTimeToDo(when)) {\n            return;\n        }\n        boolean changed = false;\n        long start = System.currentTimeMillis();\n        try {\n            for (String topic : this.topicQueueMappingManager.getTopicQueueMappingTable().keySet()) {\n                try {\n                    if (isStopped()) {\n                        break;\n                    }\n                    TopicQueueMappingDetail mappingDetail = this.topicQueueMappingManager.getTopicQueueMappingTable().get(topic);\n                    if (mappingDetail == null\n                            || mappingDetail.getHostedQueues().isEmpty()) {\n                        continue;\n                    }\n                    if (!mappingDetail.getBname().equals(brokerConfig.getBrokerName())) {\n                        log.warn(\"The TopicQueueMappingDetail [{}] should not exist in this broker\", mappingDetail);\n                        continue;\n                    }\n                    Set<String> brokers = new HashSet<>();\n                    for (List<LogicQueueMappingItem> items: mappingDetail.getHostedQueues().values()) {\n                        if (items.size() <= 1) {\n                            continue;\n                        }\n                        if (!TopicQueueMappingUtils.checkIfLeader(items, mappingDetail)) {\n                            continue;\n                        }\n                        LogicQueueMappingItem earlistItem = items.get(0);\n                        brokers.add(earlistItem.getBname());\n                    }\n                    Map<String, TopicStatsTable> statsTable = new HashMap<>();\n                    for (String broker: brokers) {\n                        GetTopicStatsInfoRequestHeader header = new GetTopicStatsInfoRequestHeader();\n                        header.setTopic(topic);\n                        header.setBrokerName(broker);\n                        header.setLo(false);\n                        try {\n                            RpcRequest rpcRequest = new RpcRequest(RequestCode.GET_TOPIC_STATS_INFO, header, null);\n                            RpcResponse rpcResponse = rpcClient.invoke(rpcRequest, brokerConfig.getForwardTimeout()).get();\n                            if (rpcResponse.getException() != null) {\n                                throw rpcResponse.getException();\n                            }\n                            statsTable.put(broker, (TopicStatsTable) rpcResponse.getBody());\n                        } catch (Throwable rt) {\n                            log.error(\"Get remote topic {} state info failed from broker {}\", topic, broker, rt);\n                        }\n                    }\n                    Map<Integer, List<LogicQueueMappingItem>> newHostedQueues = new HashMap<>();\n                    boolean changedForTopic = false;\n                    for (Map.Entry<Integer, List<LogicQueueMappingItem>> entry : mappingDetail.getHostedQueues().entrySet()) {\n                        Integer qid = entry.getKey();\n                        List<LogicQueueMappingItem> items = entry.getValue();\n                        if (items.size() <= 1) {\n                            continue;\n                        }\n                        if (!TopicQueueMappingUtils.checkIfLeader(items, mappingDetail)) {\n                            continue;\n                        }\n                        LogicQueueMappingItem earlistItem = items.get(0);\n                        TopicStatsTable topicStats = statsTable.get(earlistItem.getBname());\n                        if (topicStats == null) {\n                            continue;\n                        }\n                        TopicOffset topicOffset = topicStats.getOffsetTable().get(new MessageQueue(topic, earlistItem.getBname(), earlistItem.getQueueId()));\n                        if (topicOffset == null) {\n                            //this may should not happen\n                            log.error(\"Get null topicOffset for {} {}\",topic,  earlistItem);\n                            continue;\n                        }\n                        //ignore the maxOffset < 0, which may in case of some error\n                        if (topicOffset.getMaxOffset() == topicOffset.getMinOffset()\n                            || topicOffset.getMaxOffset() == 0) {\n                            List<LogicQueueMappingItem> newItems = new ArrayList<>(items);\n                            boolean result = newItems.remove(earlistItem);\n                            if (result) {\n                                changedForTopic = true;\n                                newHostedQueues.put(qid, newItems);\n                            }\n                            log.info(\"The logic queue item {} {} is removed {} because of {}\", topic, earlistItem, result, topicOffset);\n                        }\n                    }\n                    if (changedForTopic) {\n                        TopicQueueMappingDetail newMappingDetail = new TopicQueueMappingDetail(mappingDetail.getTopic(), mappingDetail.getTotalQueues(), mappingDetail.getBname(), mappingDetail.getEpoch());\n                        newMappingDetail.getHostedQueues().putAll(mappingDetail.getHostedQueues());\n                        newMappingDetail.getHostedQueues().putAll(newHostedQueues);\n                        this.topicQueueMappingManager.updateTopicQueueMapping(newMappingDetail, false, true, false);\n                        changed = true;\n                    }\n                } catch (Throwable tt) {\n                    log.error(\"Try CleanItemExpired failed for {}\", topic, tt);\n                } finally {\n                    UtilAll.sleep(10);\n                }\n            }\n        } catch (Throwable t) {\n            log.error(\"Try cleanItemExpired failed\", t);\n        } finally {\n            if (changed) {\n                this.topicQueueMappingManager.getDataVersion().nextVersion();\n                this.topicQueueMappingManager.persist();\n                log.info(\"CleanItemExpired changed\");\n            }\n            log.info(\"cleanItemExpired cost {} ms\", System.currentTimeMillis() - start);\n        }\n    }\n\n    public void cleanItemListMoreThanSecondGen() {\n        String when = messageStoreConfig.getDeleteWhen();\n        if (!UtilAll.isItTimeToDo(when)) {\n            return;\n        }\n        boolean changed = false;\n        long start = System.currentTimeMillis();\n        try {\n            ClientMetadata clientMetadata = new ClientMetadata();\n            for (String topic : this.topicQueueMappingManager.getTopicQueueMappingTable().keySet()) {\n                try {\n                    if (isStopped()) {\n                        break;\n                    }\n                    TopicQueueMappingDetail mappingDetail = this.topicQueueMappingManager.getTopicQueueMappingTable().get(topic);\n                    if (mappingDetail == null\n                            || mappingDetail.getHostedQueues().isEmpty()) {\n                        continue;\n                    }\n                    if (!mappingDetail.getBname().equals(brokerConfig.getBrokerName())) {\n                        log.warn(\"The TopicQueueMappingDetail [{}] should not exist in this broker\", mappingDetail);\n                        continue;\n                    }\n                    Map<Integer, String> qid2CurrLeaderBroker = new HashMap<>();\n                    for (Map.Entry<Integer, List<LogicQueueMappingItem>> entry : mappingDetail.getHostedQueues().entrySet()) {\n                        Integer qId = entry.getKey();\n                        List<LogicQueueMappingItem> items = entry.getValue();\n                        if (items.isEmpty()) {\n                            continue;\n                        }\n                        LogicQueueMappingItem leaderItem = items.get(items.size() - 1);\n                        if (!leaderItem.getBname().equals(mappingDetail.getBname())) {\n                            qid2CurrLeaderBroker.put(qId, leaderItem.getBname());\n                        }\n                    }\n                    if (qid2CurrLeaderBroker.isEmpty()) {\n                        continue;\n                    }\n                    //find the topic route\n                    TopicRouteData topicRouteData = brokerOuterAPI.getTopicRouteInfoFromNameServer(topic, brokerConfig.getForwardTimeout());\n                    clientMetadata.freshTopicRoute(topic, topicRouteData);\n                    Map<Integer, String> qid2RealLeaderBroker = new HashMap<>();\n                    //fine the real leader\n                    for (Map.Entry<Integer, String> entry : qid2CurrLeaderBroker.entrySet()) {\n                        qid2RealLeaderBroker.put(entry.getKey(), clientMetadata.getBrokerNameFromMessageQueue(new MessageQueue(topic, TopicQueueMappingUtils.getMockBrokerName(mappingDetail.getScope()), entry.getKey())));\n                    }\n\n                    //find the mapping detail of real leader\n                    Map<String, TopicQueueMappingDetail> mappingDetailMap = new HashMap<>();\n                    for (Map.Entry<Integer, String> entry : qid2RealLeaderBroker.entrySet()) {\n                        if (entry.getValue().startsWith(MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX)) {\n                            continue;\n                        }\n                        String broker = entry.getValue();\n                        GetTopicConfigRequestHeader header = new GetTopicConfigRequestHeader();\n                        header.setTopic(topic);\n                        header.setBrokerName(broker);\n                        header.setLo(true);\n                        try {\n                            RpcRequest rpcRequest = new RpcRequest(RequestCode.GET_TOPIC_CONFIG, header, null);\n                            RpcResponse rpcResponse = rpcClient.invoke(rpcRequest, brokerConfig.getForwardTimeout()).get();\n                            if (rpcResponse.getException() != null) {\n                                throw rpcResponse.getException();\n                            }\n                            TopicQueueMappingDetail mappingDetailRemote = ((TopicConfigAndQueueMapping) rpcResponse.getBody()).getMappingDetail();\n                            if (broker.equals(mappingDetailRemote.getBname())) {\n                                mappingDetailMap.put(broker, mappingDetailRemote);\n                            }\n                        } catch (Throwable rt) {\n                            log.error(\"Get remote topic {} state info failed from broker {}\", topic, broker, rt);\n                        }\n                    }\n                    //check all the info\n                    Set<Integer> ids2delete = new HashSet<>();\n                    for (Map.Entry<Integer, String> entry : qid2CurrLeaderBroker.entrySet()) {\n                        Integer qId = entry.getKey();\n                        String currLeaderBroker = entry.getValue();\n                        String realLeaderBroker = qid2RealLeaderBroker.get(qId);\n                        TopicQueueMappingDetail remoteMappingDetail = mappingDetailMap.get(realLeaderBroker);\n                        if (remoteMappingDetail == null\n                                || remoteMappingDetail.getTotalQueues() != mappingDetail.getTotalQueues()\n                                || remoteMappingDetail.getEpoch() != mappingDetail.getEpoch()) {\n                            continue;\n                        }\n                        List<LogicQueueMappingItem> items = remoteMappingDetail.getHostedQueues().get(qId);\n                        if (items.isEmpty()) {\n                            continue;\n                        }\n                        LogicQueueMappingItem leaderItem = items.get(items.size() - 1);\n                        if (!realLeaderBroker.equals(leaderItem.getBname())) {\n                            continue;\n                        }\n                        //all the check is ok\n                        if (!realLeaderBroker.equals(currLeaderBroker)) {\n                            ids2delete.add(qId);\n                        }\n                    }\n                    for (Integer qid : ids2delete) {\n                        List<LogicQueueMappingItem> items = mappingDetail.getHostedQueues().remove(qid);\n                        changed =  true;\n                        if (items != null) {\n                            log.info(\"Remove the ItemListMoreThanSecondGen topic {} qid {} items {}\", topic, qid, items);\n                        }\n                    }\n                } catch (Throwable tt) {\n                    log.error(\"Try cleanItemListMoreThanSecondGen failed for topic {}\", topic, tt);\n                } finally {\n                    UtilAll.sleep(10);\n                }\n            }\n        } catch (Throwable t) {\n            log.error(\"Try cleanItemListMoreThanSecondGen failed\", t);\n        } finally {\n            if (changed) {\n                this.topicQueueMappingManager.getDataVersion().nextVersion();\n                this.topicQueueMappingManager.persist();\n            }\n            log.info(\"Try cleanItemListMoreThanSecondGen cost {} ms\", System.currentTimeMillis() - start);\n        }\n    }\n\n\n\n\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/topic/TopicQueueMappingManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.topic;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.alibaba.fastjson2.JSONWriter;\nimport com.google.common.collect.Maps;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.locks.Lock;\nimport java.util.concurrent.locks.ReentrantLock;\nimport java.util.stream.Collectors;\n\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.collections.MapUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.BrokerPathConfigHelper;\nimport org.apache.rocketmq.common.ConfigManager;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.TopicQueueMappingSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.statictopic.LogicQueueMappingItem;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingContext;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingDetail;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingUtils;\nimport org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;\nimport org.apache.rocketmq.remoting.rpc.TopicRequestHeader;\n\nimport static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse;\n\npublic class TopicQueueMappingManager extends ConfigManager {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private static final long LOCK_TIMEOUT_MILLIS = 3000;\n    private transient final Lock lock = new ReentrantLock();\n\n    //this data version should be equal to the TopicConfigManager\n    private final DataVersion dataVersion = new DataVersion();\n    private transient BrokerController brokerController;\n\n    private final ConcurrentMap<String, TopicQueueMappingDetail> topicQueueMappingTable = new ConcurrentHashMap<>();\n\n\n    public TopicQueueMappingManager(BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    public void updateTopicQueueMapping(TopicQueueMappingDetail newDetail, boolean force, boolean isClean, boolean flush) throws Exception {\n        boolean locked = false;\n        boolean updated = false;\n        TopicQueueMappingDetail oldDetail = null;\n        try {\n\n            if (lock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {\n                locked = true;\n            } else {\n                return;\n            }\n            if (newDetail == null) {\n                return;\n            }\n            assert newDetail.getBname().equals(this.brokerController.getBrokerConfig().getBrokerName());\n\n            newDetail.getHostedQueues().forEach((queueId, items) -> {\n                TopicQueueMappingUtils.checkLogicQueueMappingItemOffset(items);\n            });\n\n            oldDetail = topicQueueMappingTable.get(newDetail.getTopic());\n            if (oldDetail == null) {\n                topicQueueMappingTable.put(newDetail.getTopic(), newDetail);\n                updated = true;\n                return;\n            }\n            if (force) {\n                //bakeup the old items\n                oldDetail.getHostedQueues().forEach((queueId, items) -> {\n                    newDetail.getHostedQueues().putIfAbsent(queueId, items);\n                });\n                topicQueueMappingTable.put(newDetail.getTopic(), newDetail);\n                updated = true;\n                return;\n            }\n            //do more check\n            if (newDetail.getEpoch() < oldDetail.getEpoch()) {\n                throw new RuntimeException(String.format(\"Can't accept data with small epoch %d < %d\", newDetail.getEpoch(), oldDetail.getEpoch()));\n            }\n            if (!newDetail.getScope().equals(oldDetail.getScope())) {\n                throw new RuntimeException(String.format(\"Can't accept data with unmatched scope %s != %s\", newDetail.getScope(), oldDetail.getScope()));\n            }\n            boolean epochEqual = newDetail.getEpoch() == oldDetail.getEpoch();\n            for (Integer globalId : oldDetail.getHostedQueues().keySet()) {\n                List<LogicQueueMappingItem> oldItems = oldDetail.getHostedQueues().get(globalId);\n                List<LogicQueueMappingItem> newItems = newDetail.getHostedQueues().get(globalId);\n                if (newItems == null) {\n                    if (epochEqual) {\n                        throw new RuntimeException(\"Cannot accept equal epoch with null data\");\n                    } else {\n                        newDetail.getHostedQueues().put(globalId, oldItems);\n                    }\n                } else {\n                    TopicQueueMappingUtils.makeSureLogicQueueMappingItemImmutable(oldItems, newItems, epochEqual, isClean);\n                }\n            }\n            topicQueueMappingTable.put(newDetail.getTopic(), newDetail);\n            updated = true;\n        }  finally {\n            if (locked) {\n                this.lock.unlock();\n            }\n            if (updated && flush) {\n                this.dataVersion.nextVersion();\n                this.persist();\n                log.info(\"Update topic queue mapping from [{}] to [{}], force {}\", oldDetail, newDetail, force);\n            }\n        }\n\n    }\n\n    public void delete(final String topic) {\n        TopicQueueMappingDetail old = this.topicQueueMappingTable.remove(topic);\n        if (old != null) {\n            log.info(\"delete topic queue mapping OK, static topic queue mapping: {}\", old);\n            this.dataVersion.nextVersion();\n            this.persist();\n        } else {\n            log.warn(\"delete topic queue mapping failed, static topic: {} not exists\", topic);\n        }\n    }\n\n    public TopicQueueMappingDetail getTopicQueueMapping(String topic) {\n        return topicQueueMappingTable.get(topic);\n    }\n\n    @Override\n    public String encode(boolean pretty) {\n        TopicQueueMappingSerializeWrapper wrapper = new TopicQueueMappingSerializeWrapper();\n        wrapper.setTopicQueueMappingInfoMap(topicQueueMappingTable);\n        wrapper.setDataVersion(this.dataVersion);\n        if (pretty) {\n            return JSON.toJSONString(wrapper, JSONWriter.Feature.PrettyFormat);\n        }\n        return JSON.toJSONString(wrapper);\n    }\n\n    @Override\n    public String encode() {\n        return encode(false);\n    }\n\n    @Override\n    public String configFilePath() {\n        return BrokerPathConfigHelper.getTopicQueueMappingPath(this.brokerController.getMessageStoreConfig()\n            .getStorePathRootDir());\n    }\n\n    @Override\n    public void decode(String jsonString) {\n        if (jsonString != null) {\n            TopicQueueMappingSerializeWrapper wrapper = TopicQueueMappingSerializeWrapper.fromJson(jsonString, TopicQueueMappingSerializeWrapper.class);\n            if (wrapper != null) {\n                this.topicQueueMappingTable.putAll(wrapper.getTopicQueueMappingInfoMap());\n                this.dataVersion.assignNewOne(wrapper.getDataVersion());\n            }\n        }\n    }\n\n    public ConcurrentMap<String, TopicQueueMappingDetail> getTopicQueueMappingTable() {\n        return topicQueueMappingTable;\n    }\n\n    public ConcurrentMap<String, TopicQueueMappingDetail> subTopicQueueMappingTable(Set<String> topicSet) {\n        if (MapUtils.isEmpty(topicQueueMappingTable) || CollectionUtils.isEmpty(topicSet)) {\n            return Maps.newConcurrentMap();\n        }\n\n        return topicQueueMappingTable.entrySet().stream()\n                .filter(e -> topicSet.contains(e.getKey()))\n                .collect(Collectors.toConcurrentMap(Map.Entry::getKey, Map.Entry::getValue));\n    }\n\n    public DataVersion getDataVersion() {\n        return dataVersion;\n    }\n\n    public TopicQueueMappingContext buildTopicQueueMappingContext(TopicRequestHeader requestHeader) {\n        return buildTopicQueueMappingContext(requestHeader, false);\n    }\n\n    //Do not return a null context\n    public TopicQueueMappingContext buildTopicQueueMappingContext(TopicRequestHeader requestHeader, boolean selectOneWhenMiss) {\n        // if lo is set to false explicitly, it maybe the forwarded request\n        if (requestHeader.getLo() != null\n                && Boolean.FALSE.equals(requestHeader.getLo())) {\n            return new TopicQueueMappingContext(requestHeader.getTopic(), null, null, null, null);\n        }\n        String topic = requestHeader.getTopic();\n        Integer globalId = null;\n        if (requestHeader instanceof  TopicQueueRequestHeader) {\n            globalId = ((TopicQueueRequestHeader) requestHeader).getQueueId();\n        }\n\n        TopicQueueMappingDetail mappingDetail = getTopicQueueMapping(topic);\n        if (mappingDetail == null) {\n            //it is not static topic\n            return new TopicQueueMappingContext(topic, null, null, null, null);\n        }\n        assert mappingDetail.getBname().equals(this.brokerController.getBrokerConfig().getBrokerName());\n\n        if (globalId == null) {\n            return new TopicQueueMappingContext(topic, null, mappingDetail, null, null);\n        }\n\n        //If not find mappingItem, it encounters some errors\n        if (globalId < 0 && !selectOneWhenMiss) {\n            return new TopicQueueMappingContext(topic, globalId, mappingDetail, null, null);\n        }\n\n        if (globalId < 0) {\n            try {\n                if (!mappingDetail.getHostedQueues().isEmpty()) {\n                    //do not check\n                    globalId = mappingDetail.getHostedQueues().keySet().iterator().next();\n                }\n            } catch (Throwable ignored) {\n            }\n        }\n        if (globalId < 0) {\n            return new TopicQueueMappingContext(topic, globalId,  mappingDetail, null, null);\n        }\n\n        List<LogicQueueMappingItem> mappingItemList = TopicQueueMappingDetail.getMappingInfo(mappingDetail, globalId);\n        LogicQueueMappingItem leaderItem = null;\n        if (mappingItemList != null\n                && mappingItemList.size() > 0) {\n            leaderItem = mappingItemList.get(mappingItemList.size() - 1);\n        }\n        return new TopicQueueMappingContext(topic, globalId, mappingDetail, mappingItemList, leaderItem);\n    }\n\n\n    public  RemotingCommand rewriteRequestForStaticTopic(TopicQueueRequestHeader requestHeader, TopicQueueMappingContext mappingContext) {\n        try {\n            if (mappingContext.getMappingDetail() == null) {\n                return null;\n            }\n            TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();\n            if (!mappingContext.isLeader()) {\n                return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format(\"%s-%d does not exit in request process of current broker %s\", requestHeader.getTopic(), requestHeader.getQueueId(), mappingDetail.getBname()));\n            }\n            LogicQueueMappingItem mappingItem = mappingContext.getLeaderItem();\n            requestHeader.setQueueId(mappingItem.getQueueId());\n            return null;\n        } catch (Throwable t) {\n            return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());\n        }\n    }\n\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/topic/TopicRouteInfoManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.topic;\n\nimport com.google.common.collect.Sets;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.locks.Lock;\nimport java.util.concurrent.locks.ReentrantLock;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.client.impl.producer.TopicPublishInfo;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\n\npublic class TopicRouteInfoManager {\n\n    private static final long GET_TOPIC_ROUTE_TIMEOUT = 3000L;\n    private static final long LOCK_TIMEOUT_MILLIS = 3000L;\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n\n    private final Lock lockNamesrv = new ReentrantLock();\n    private final ConcurrentMap<String/* Topic */, TopicRouteData> topicRouteTable = new ConcurrentHashMap<>();\n    private final ConcurrentMap<String/* Broker Name */, HashMap<Long/* brokerId */, String/* address */>> brokerAddrTable =\n        new ConcurrentHashMap<>();\n    private final ConcurrentMap<String/* topic */, TopicPublishInfo> topicPublishInfoTable = new ConcurrentHashMap<>();\n\n    private final ConcurrentHashMap<String, Set<MessageQueue>> topicSubscribeInfoTable = new ConcurrentHashMap<>();\n\n    private ScheduledExecutorService scheduledExecutorService;\n    private BrokerController brokerController;\n\n    public TopicRouteInfoManager(BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    public void start() {\n        this.scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(\"TopicRouteInfoManagerScheduledThread\"));\n\n        this.scheduledExecutorService.scheduleAtFixedRate(() -> {\n            try {\n                updateTopicRouteInfoFromNameServer();\n            } catch (Exception e) {\n                log.error(\"ScheduledTask: failed to pull TopicRouteData from NameServer\", e);\n            }\n        }, 1000, this.brokerController.getBrokerConfig().getLoadBalancePollNameServerInterval(), TimeUnit.MILLISECONDS);\n    }\n\n    private void updateTopicRouteInfoFromNameServer() {\n        final Set<String> topicSetForPopAssignment = this.topicSubscribeInfoTable.keySet();\n        final Set<String> topicSetForEscapeBridge = this.topicRouteTable.keySet();\n        final Set<String> topicsAll = Sets.union(topicSetForPopAssignment, topicSetForEscapeBridge);\n\n        for (String topic : topicsAll) {\n            boolean isNeedUpdatePublishInfo = topicSetForEscapeBridge.contains(topic);\n            boolean isNeedUpdateSubscribeInfo = topicSetForPopAssignment.contains(topic);\n            updateTopicRouteInfoFromNameServer(topic, isNeedUpdatePublishInfo, isNeedUpdateSubscribeInfo);\n        }\n    }\n\n    public void updateTopicRouteInfoFromNameServer(String topic, boolean isNeedUpdatePublishInfo,\n        boolean isNeedUpdateSubscribeInfo) {\n        try {\n            if (this.lockNamesrv.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {\n                try {\n                    final TopicRouteData topicRouteData = this.brokerController.getBrokerOuterAPI()\n                        .getTopicRouteInfoFromNameServer(topic, GET_TOPIC_ROUTE_TIMEOUT);\n                    if (null == topicRouteData) {\n                        log.warn(\"TopicRouteInfoManager: updateTopicRouteInfoFromNameServer, getTopicRouteInfoFromNameServer return null, Topic: {}.\", topic);\n                        return;\n                    }\n\n                    if (isNeedUpdateSubscribeInfo) {\n                        this.updateSubscribeInfoTable(topicRouteData, topic);\n                    }\n\n                    if (isNeedUpdatePublishInfo) {\n                        this.updateTopicRouteTable(topic, topicRouteData);\n                    }\n                } catch (RemotingException e) {\n                    log.error(\"updateTopicRouteInfoFromNameServer Exception\", e);\n                } catch (MQBrokerException e) {\n                    log.error(\"updateTopicRouteInfoFromNameServer Exception\", e);\n                    if (!NamespaceUtil.isRetryTopic(topic)\n                        && ResponseCode.TOPIC_NOT_EXIST == e.getResponseCode()) {\n                        // clean no used topic\n                        cleanNoneRouteTopic(topic);\n                    }\n                } finally {\n                    this.lockNamesrv.unlock();\n                }\n            }\n        } catch (InterruptedException e) {\n            log.warn(\"updateTopicRouteInfoFromNameServer Exception\", e);\n        }\n    }\n\n    private boolean updateTopicRouteTable(String topic, TopicRouteData topicRouteData) {\n        TopicRouteData old = this.topicRouteTable.get(topic);\n        boolean changed = topicRouteData.topicRouteDataChanged(old);\n        if (!changed) {\n            if (!this.isNeedUpdateTopicRouteInfo(topic)) {\n                return false;\n            }\n        } else {\n            log.info(\"the topic[{}] route info changed, old[{}] ,new[{}]\", topic, old, topicRouteData);\n        }\n\n        for (BrokerData bd : topicRouteData.getBrokerDatas()) {\n            this.brokerAddrTable.put(bd.getBrokerName(), bd.getBrokerAddrs());\n        }\n\n        TopicPublishInfo publishInfo = MQClientInstance.topicRouteData2TopicPublishInfo(topic, topicRouteData);\n        publishInfo.setHaveTopicRouterInfo(true);\n        this.updateTopicPublishInfo(topic, publishInfo);\n\n        TopicRouteData cloneTopicRouteData = new TopicRouteData(topicRouteData);\n        log.info(\"topicRouteTable.put. Topic = {}, TopicRouteData[{}]\", topic, cloneTopicRouteData);\n        this.topicRouteTable.put(topic, cloneTopicRouteData);\n\n        return true;\n    }\n\n    private boolean updateSubscribeInfoTable(TopicRouteData topicRouteData, String topic) {\n        final TopicRouteData tmp = new TopicRouteData(topicRouteData);\n        tmp.setTopicQueueMappingByBroker(null);\n        Set<MessageQueue> newSubscribeInfo = MQClientInstance.topicRouteData2TopicSubscribeInfo(topic, tmp);\n        Set<MessageQueue> oldSubscribeInfo = topicSubscribeInfoTable.get(topic);\n\n        if (Objects.equals(newSubscribeInfo, oldSubscribeInfo)) {\n            return false;\n        }\n\n        log.info(\"the topic[{}] subscribe message queue changed, old[{}] ,new[{}]\", topic, oldSubscribeInfo, newSubscribeInfo);\n        topicSubscribeInfoTable.put(topic, newSubscribeInfo);\n        return true;\n\n    }\n\n    private boolean isNeedUpdateTopicRouteInfo(final String topic) {\n        final TopicPublishInfo prev = this.topicPublishInfoTable.get(topic);\n        return null == prev || !prev.ok();\n    }\n\n    private void cleanNoneRouteTopic(String topic) {\n        // clean no used topic\n        topicSubscribeInfoTable.remove(topic);\n    }\n\n    private void updateTopicPublishInfo(final String topic, final TopicPublishInfo info) {\n        if (info != null && topic != null) {\n            TopicPublishInfo prev = this.topicPublishInfoTable.put(topic, info);\n            if (prev != null) {\n                log.info(\"updateTopicPublishInfo prev is not null, \" + prev);\n            }\n        }\n    }\n\n    public void shutdown() {\n        if (null != this.scheduledExecutorService) {\n            this.scheduledExecutorService.shutdown();\n        }\n    }\n\n    public TopicPublishInfo tryToFindTopicPublishInfo(final String topic) {\n        TopicPublishInfo topicPublishInfo = this.topicPublishInfoTable.get(topic);\n        if (null == topicPublishInfo || !topicPublishInfo.ok()) {\n            this.updateTopicRouteInfoFromNameServer(topic, true, false);\n            topicPublishInfo = this.topicPublishInfoTable.get(topic);\n        }\n        return topicPublishInfo;\n    }\n\n    public String findBrokerAddressInPublish(String brokerName) {\n        if (brokerName == null) {\n            return null;\n        }\n        Map<Long/* brokerId */, String/* address */> map = this.brokerAddrTable.get(brokerName);\n        if (map != null && !map.isEmpty()) {\n            return map.get(MixAll.MASTER_ID);\n        }\n\n        return null;\n    }\n\n    public String findBrokerAddressInSubscribe(\n        final String brokerName,\n        final long brokerId,\n        final boolean onlyThisBroker\n    ) {\n        if (brokerName == null) {\n            return null;\n        }\n        String brokerAddr = null;\n        boolean found = false;\n\n        Map<Long/* brokerId */, String/* address */> map = this.brokerAddrTable.get(brokerName);\n        if (map != null && !map.isEmpty()) {\n            brokerAddr = map.get(brokerId);\n            boolean slave = brokerId != MixAll.MASTER_ID;\n            found = brokerAddr != null;\n\n            if (!found && slave) {\n                brokerAddr = map.get(brokerId + 1);\n                found = brokerAddr != null;\n            }\n\n            if (!found && !onlyThisBroker) {\n                Map.Entry<Long, String> entry = map.entrySet().iterator().next();\n                brokerAddr = entry.getValue();\n                found = true;\n            }\n        }\n\n        return brokerAddr;\n\n    }\n\n    public Set<MessageQueue> getTopicSubscribeInfo(String topic) {\n        Set<MessageQueue> queues = topicSubscribeInfoTable.get(topic);\n        if (null == queues || queues.isEmpty()) {\n            this.updateTopicRouteInfoFromNameServer(topic, false, true);\n            queues = this.topicSubscribeInfoTable.get(topic);\n        }\n        return queues;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.transaction;\n\nimport io.netty.channel.Channel;\nimport java.util.concurrent.ArrayBlockingQueue;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.ThreadPoolExecutor.CallerRunsPolicy;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;\n\npublic abstract class AbstractTransactionalMessageCheckListener {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME);\n\n    private BrokerController brokerController;\n\n    //queue nums of topic TRANS_CHECK_MAX_TIME_TOPIC\n    protected final static int TCMT_QUEUE_NUMS = 1;\n\n    private volatile ExecutorService executorService;\n\n    public AbstractTransactionalMessageCheckListener() {\n    }\n\n    public AbstractTransactionalMessageCheckListener(BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    public void sendCheckMessage(MessageExt msgExt) throws Exception {\n        CheckTransactionStateRequestHeader checkTransactionStateRequestHeader = new CheckTransactionStateRequestHeader();\n        checkTransactionStateRequestHeader.setTopic(msgExt.getTopic());\n        checkTransactionStateRequestHeader.setCommitLogOffset(msgExt.getCommitLogOffset());\n        checkTransactionStateRequestHeader.setOffsetMsgId(msgExt.getMsgId());\n        checkTransactionStateRequestHeader.setMsgId(msgExt.getUserProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));\n        checkTransactionStateRequestHeader.setTransactionId(checkTransactionStateRequestHeader.getMsgId());\n        checkTransactionStateRequestHeader.setTranStateTableOffset(msgExt.getQueueOffset());\n        checkTransactionStateRequestHeader.setBrokerName(brokerController.getBrokerConfig().getBrokerName());\n        msgExt.setTopic(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_TOPIC));\n        msgExt.setQueueId(Integer.parseInt(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_QUEUE_ID)));\n        msgExt.setStoreSize(0);\n        String groupId = msgExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP);\n        Channel channel = brokerController.getProducerManager().getAvailableChannel(groupId);\n        if (channel != null) {\n            brokerController.getBroker2Client().checkProducerTransactionState(groupId, channel, checkTransactionStateRequestHeader, msgExt);\n        } else {\n            LOGGER.warn(\"Check transaction failed, channel is null. groupId={}\", groupId);\n        }\n    }\n\n    public void resolveHalfMsg(final MessageExt msgExt) {\n        if (executorService != null) {\n            executorService.execute(new Runnable() {\n                @Override\n                public void run() {\n                    try {\n                        sendCheckMessage(msgExt);\n                    } catch (Exception e) {\n                        LOGGER.error(\"Send check message error!\", e);\n                    }\n                }\n            });\n        } else {\n            LOGGER.error(\"TransactionalMessageCheckListener not init\");\n        }\n    }\n\n    public BrokerController getBrokerController() {\n        return brokerController;\n    }\n\n    public void shutdown() {\n        if (executorService != null) {\n            executorService.shutdown();\n        }\n    }\n\n    public synchronized void initExecutorService() {\n        if (executorService == null) {\n            executorService = ThreadUtils.newThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<>(2000),\n                new ThreadFactoryImpl(\"Transaction-msg-check-thread\", brokerController.getBrokerIdentity()), new CallerRunsPolicy());\n        }\n    }\n\n    /**\n     * Inject brokerController for this listener\n     *\n     * @param brokerController\n     */\n    public void setBrokerController(BrokerController brokerController) {\n        this.brokerController = brokerController;\n        initExecutorService();\n    }\n\n    /**\n     * In order to avoid check back unlimited, we will discard the message that have been checked more than a certain\n     * number of times.\n     *\n     * @param msgExt Message to be discarded.\n     */\n    public abstract void resolveDiscardMsg(MessageExt msgExt);\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/transaction/OperationResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.transaction;\n\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic class OperationResult {\n    private MessageExt prepareMessage;\n\n    private int responseCode;\n\n    private String responseRemark;\n\n    public void setPrepareMessage(MessageExt prepareMessage) {\n        this.prepareMessage = prepareMessage;\n    }\n\n    public int getResponseCode() {\n        return responseCode;\n    }\n\n    public void setResponseCode(int responseCode) {\n        this.responseCode = responseCode;\n    }\n\n    public String getResponseRemark() {\n        return responseRemark;\n    }\n\n    public void setResponseRemark(String responseRemark) {\n        this.responseRemark = responseRemark;\n    }\n\n    public MessageExt getPrepareMessage() {\n        return prepareMessage;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionMetrics.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.transaction;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.alibaba.fastjson2.JSONWriter;\nimport org.apache.rocketmq.common.ConfigManager;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.io.RandomAccessFile;\nimport java.io.StringWriter;\nimport java.io.Writer;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.nio.file.Paths;\nimport java.nio.file.StandardCopyOption;\nimport java.util.Iterator;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.atomic.AtomicLong;\n\n\npublic class TransactionMetrics extends ConfigManager {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n\n    private ConcurrentMap<String, Metric> transactionCounts =\n            new ConcurrentHashMap<>(1024);\n\n    private DataVersion dataVersion = new DataVersion();\n\n    private final String configPath;\n\n    public TransactionMetrics(String configPath) {\n        this.configPath = configPath;\n    }\n\n    public long addAndGet(String topic, int value) {\n        Metric pair = getTopicPair(topic);\n        getDataVersion().nextVersion();\n        pair.setTimeStamp(System.currentTimeMillis());\n        return pair.getCount().addAndGet(value);\n    }\n\n    public Metric getTopicPair(String topic) {\n        Metric pair = transactionCounts.get(topic);\n        if (null != pair) {\n            return pair;\n        }\n        pair = new Metric();\n        final Metric previous = transactionCounts.putIfAbsent(topic, pair);\n        if (null != previous) {\n            return previous;\n        }\n        return pair;\n    }\n    public long getTransactionCount(String topic) {\n        Metric pair = transactionCounts.get(topic);\n        if (null == pair) {\n            return 0;\n        } else {\n            return pair.getCount().get();\n        }\n    }\n\n    public Map<String, Metric> getTransactionCounts() {\n        return transactionCounts;\n    }\n    public void setTransactionCounts(ConcurrentMap<String, Metric> transactionCounts) {\n        this.transactionCounts = transactionCounts;\n    }\n\n    protected void write0(Writer writer) throws IOException {\n        TransactionMetricsSerializeWrapper wrapper = new TransactionMetricsSerializeWrapper();\n        wrapper.setTransactionCount(transactionCounts);\n        wrapper.setDataVersion(dataVersion);\n        writer.write(JSON.toJSONString(wrapper, JSONWriter.Feature.BrowserCompatible));\n    }\n\n    @Override\n    public String encode() {\n        return encode(false);\n    }\n\n    @Override\n    public String configFilePath() {\n        return configPath;\n    }\n\n    @Override\n    public void decode(String jsonString) {\n        if (jsonString != null) {\n            TransactionMetricsSerializeWrapper transactionMetricsSerializeWrapper =\n                    TransactionMetricsSerializeWrapper.fromJson(jsonString, TransactionMetricsSerializeWrapper.class);\n            if (transactionMetricsSerializeWrapper != null) {\n                this.transactionCounts.putAll(transactionMetricsSerializeWrapper.getTransactionCount());\n                this.dataVersion.assignNewOne(transactionMetricsSerializeWrapper.getDataVersion());\n            }\n        }\n    }\n\n    @Override\n    public String encode(boolean prettyFormat) {\n        TransactionMetricsSerializeWrapper metricsSerializeWrapper = new TransactionMetricsSerializeWrapper();\n        metricsSerializeWrapper.setDataVersion(this.dataVersion);\n        metricsSerializeWrapper.setTransactionCount(this.transactionCounts);\n        return metricsSerializeWrapper.toJson(prettyFormat);\n    }\n\n    public DataVersion getDataVersion() {\n        return dataVersion;\n    }\n\n    public void setDataVersion(DataVersion dataVersion) {\n        this.dataVersion = dataVersion;\n    }\n\n    public void cleanMetrics(Set<String> topics) {\n        if (topics == null || topics.isEmpty()) {\n            return;\n        }\n        Iterator<Map.Entry<String, Metric>> iterator = transactionCounts.entrySet().iterator();\n        while (iterator.hasNext()) {\n            Map.Entry<String, Metric> entry = iterator.next();\n            final String topic = entry.getKey();\n            if (topic.startsWith(TopicValidator.SYSTEM_TOPIC_PREFIX)) {\n                continue;\n            }\n            if (!topics.contains(topic)) {\n                continue;\n            }\n            // in the input topics set, then remove it.\n            iterator.remove();\n        }\n    }\n\n    public static class TransactionMetricsSerializeWrapper extends RemotingSerializable {\n        private ConcurrentMap<String, Metric> transactionCount =\n                new ConcurrentHashMap<>(1024);\n        private DataVersion dataVersion = new DataVersion();\n\n        public ConcurrentMap<String, Metric> getTransactionCount() {\n            return transactionCount;\n        }\n\n        public void setTransactionCount(\n                ConcurrentMap<String, Metric> transactionCount) {\n            this.transactionCount = transactionCount;\n        }\n\n        public DataVersion getDataVersion() {\n            return dataVersion;\n        }\n\n        public void setDataVersion(DataVersion dataVersion) {\n            this.dataVersion = dataVersion;\n        }\n    }\n\n    @Override\n    public synchronized void persist() {\n        try {\n            // bak metrics file\n            String config = configFilePath();\n            String backup = config + \".bak\";\n            File configFile = new File(config);\n            File bakFile = new File(backup);\n\n            if (configFile.exists()) {\n                // atomic move\n                Files.move(configFile.toPath(), bakFile.toPath(), StandardCopyOption.ATOMIC_MOVE);\n\n                // sync the directory, ensure that the bak file is visible\n                MixAll.fsyncDirectory(Paths.get(bakFile.getParent()));\n            }\n\n            File dir = new File(configFile.getParent());\n            if (!dir.exists()) {\n                Files.createDirectories(dir.toPath());\n            }\n\n            // persist metrics file\n            StringWriter stringWriter = new StringWriter();\n            write0(stringWriter);\n            try (RandomAccessFile randomAccessFile = new RandomAccessFile(config, \"rw\")) {\n                randomAccessFile.write(stringWriter.toString().getBytes(StandardCharsets.UTF_8));\n                randomAccessFile.getChannel().force(true);\n                // sync the directory, ensure that the config file is visible\n                MixAll.fsyncDirectory(Paths.get(configFile.getParent()));\n            }\n        } catch (Throwable t) {\n            log.error(\"Failed to persist\", t);\n        }\n    }\n\n    public static class Metric {\n        private AtomicLong count;\n        private long timeStamp;\n\n        public Metric() {\n            count = new AtomicLong(0);\n            timeStamp = System.currentTimeMillis();\n        }\n\n        public AtomicLong getCount() {\n            return count;\n        }\n\n        public void setCount(AtomicLong count) {\n            this.count = count;\n        }\n\n        public long getTimeStamp() {\n            return timeStamp;\n        }\n\n        public void setTimeStamp(long timeStamp) {\n            this.timeStamp = timeStamp;\n        }\n\n        @Override\n        public String toString() {\n            return String.format(\"[%d,%d]\", count.get(), timeStamp);\n        }\n    }\n\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionMetricsFlushService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.transaction;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class TransactionMetricsFlushService extends ServiceThread {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME);\n    private BrokerController brokerController;\n    public TransactionMetricsFlushService(BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    @Override\n    public String getServiceName() {\n        return \"TransactionFlushService\";\n    }\n\n    @Override\n    public void run() {\n        log.info(this.getServiceName() + \" service start\");\n        long start = System.currentTimeMillis();\n        while (!this.isStopped()) {\n            try {\n                if (System.currentTimeMillis() - start > brokerController.getBrokerConfig().getTransactionMetricFlushInterval()) {\n                    start = System.currentTimeMillis();\n                    brokerController.getTransactionalMessageService().getTransactionMetrics().persist();\n                    waitForRunning(brokerController.getBrokerConfig().getTransactionMetricFlushInterval());\n                }\n            } catch (Throwable e) {\n                log.error(\"Error occurred in \" + getServiceName(), e);\n            }\n        }\n        log.info(this.getServiceName() + \" service end\");\n    }\n}"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.transaction;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class TransactionalMessageCheckService extends ServiceThread {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME);\n\n    private BrokerController brokerController;\n\n    public TransactionalMessageCheckService(BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    @Override\n    public String getServiceName() {\n        if (brokerController != null && brokerController.getBrokerConfig().isInBrokerContainer()) {\n            return brokerController.getBrokerIdentity().getIdentifier() + TransactionalMessageCheckService.class.getSimpleName();\n        }\n        return TransactionalMessageCheckService.class.getSimpleName();\n    }\n\n    @Override\n    public void run() {\n        log.info(\"Start transaction check service thread!\");\n        while (!this.isStopped()) {\n            long checkInterval = brokerController.getBrokerConfig().getTransactionCheckInterval();\n            this.waitForRunning(checkInterval);\n        }\n        log.info(\"End transaction check service thread!\");\n    }\n\n    @Override\n    protected void onWaitEnd() {\n        long timeout = brokerController.getBrokerConfig().getTransactionTimeOut();\n        int checkMax = brokerController.getBrokerConfig().getTransactionCheckMax();\n        long begin = System.currentTimeMillis();\n        log.info(\"Begin to check prepare message, begin time:{}\", begin);\n        this.brokerController.getTransactionalMessageService().check(timeout, checkMax, this.brokerController.getTransactionalMessageCheckListener());\n        log.info(\"End to check prepare message, consumed time:{}\", System.currentTimeMillis() - begin);\n    }\n\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.transaction;\n\nimport java.util.concurrent.CompletableFuture;\n\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;\nimport org.apache.rocketmq.store.PutMessageResult;\n\npublic interface TransactionalMessageService {\n\n    /**\n     * Process prepare message, in common, we should put this message to storage service.\n     *\n     * @param messageInner Prepare(Half) message.\n     * @return Prepare message storage result.\n     */\n    PutMessageResult prepareMessage(MessageExtBrokerInner messageInner);\n\n    /**\n     * Process prepare message in async manner, we should put this message to storage service\n     *\n     * @param messageInner Prepare(Half) message.\n     * @return CompletableFuture of put result, will be completed at put success(flush and replica done)\n     */\n    CompletableFuture<PutMessageResult> asyncPrepareMessage(MessageExtBrokerInner messageInner);\n\n    /**\n     * Delete prepare message when this message has been committed or rolled back.\n     *\n     * @param messageExt\n     */\n    boolean deletePrepareMessage(MessageExt messageExt);\n\n    /**\n     * Invoked to process commit prepare message.\n     *\n     * @param requestHeader Commit message request header.\n     * @return Operate result contains prepare message and relative error code.\n     */\n    OperationResult commitMessage(EndTransactionRequestHeader requestHeader);\n\n    /**\n     * Invoked to roll back prepare message.\n     *\n     * @param requestHeader Prepare message request header.\n     * @return Operate result contains prepare message and relative error code.\n     */\n    OperationResult rollbackMessage(EndTransactionRequestHeader requestHeader);\n\n    /**\n     * Traverse uncommitted/unroll back half message and send check back request to producer to obtain transaction\n     * status.\n     *\n     * @param transactionTimeout The minimum time of the transactional message to be checked firstly, one message only\n     * exceed this time interval that can be checked.\n     * @param transactionCheckMax The maximum number of times the message was checked, if exceed this value, this\n     * message will be discarded.\n     * @param listener When the message is considered to be checked or discarded, the relative method of this class will\n     * be invoked.\n     */\n    void check(long transactionTimeout, int transactionCheckMax, AbstractTransactionalMessageCheckListener listener);\n\n    /**\n     * Open transaction service.\n     *\n     * @return If open success, return true.\n     */\n    boolean open();\n\n    /**\n     * Close transaction service.\n     */\n    void close();\n\n    TransactionMetrics getTransactionMetrics();\n\n    void setTransactionMetrics(TransactionMetrics transactionMetrics);\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/DefaultTransactionalMessageCheckListener.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.transaction.queue;\n\nimport org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\n\npublic class DefaultTransactionalMessageCheckListener extends AbstractTransactionalMessageCheckListener {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME);\n\n    public DefaultTransactionalMessageCheckListener() {\n        super();\n    }\n\n    @Override\n    public void resolveDiscardMsg(MessageExt msgExt) {\n        log.error(\"MsgExt:{} has been checked too many times, so discard it by moving it to system topic TRANS_CHECK_MAXTIME_TOPIC\", msgExt);\n\n        try {\n            MessageExtBrokerInner brokerInner = toMessageExtBrokerInner(msgExt);\n            PutMessageResult putMessageResult = this.getBrokerController().getMessageStore().putMessage(brokerInner);\n            if (putMessageResult != null && putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK) {\n                log.info(\"Put checked-too-many-time half message to TRANS_CHECK_MAXTIME_TOPIC OK. Restored in queueOffset={}, \" +\n                    \"commitLogOffset={}, real topic={}\", msgExt.getQueueOffset(), msgExt.getCommitLogOffset(), msgExt.getUserProperty(MessageConst.PROPERTY_REAL_TOPIC));\n                // discarded, then the num of half-messages minus 1\n                this.getBrokerController().getTransactionalMessageService().getTransactionMetrics().addAndGet(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_TOPIC), -1);\n            } else {\n                log.error(\"Put checked-too-many-time half message to TRANS_CHECK_MAXTIME_TOPIC failed, real topic={}, msgId={}\", msgExt.getTopic(), msgExt.getMsgId());\n            }\n        } catch (Exception e) {\n            log.warn(\"Put checked-too-many-time message to TRANS_CHECK_MAXTIME_TOPIC error. {}\", e);\n        }\n\n    }\n\n    private MessageExtBrokerInner toMessageExtBrokerInner(MessageExt msgExt) {\n        TopicConfig topicConfig = this.getBrokerController().getTopicConfigManager().createTopicOfTranCheckMaxTime(TCMT_QUEUE_NUMS, PermName.PERM_READ | PermName.PERM_WRITE);\n        MessageExtBrokerInner inner = new MessageExtBrokerInner();\n        inner.setTopic(topicConfig.getTopicName());\n        inner.setBody(msgExt.getBody());\n        inner.setFlag(msgExt.getFlag());\n        MessageAccessor.setProperties(inner, msgExt.getProperties());\n        inner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties()));\n        inner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(msgExt.getTags()));\n        inner.setQueueId(0);\n        inner.setSysFlag(msgExt.getSysFlag());\n        inner.setBornHost(msgExt.getBornHost());\n        inner.setBornTimestamp(msgExt.getBornTimestamp());\n        inner.setStoreHost(msgExt.getStoreHost());\n        inner.setReconsumeTimes(msgExt.getReconsumeTimes());\n        inner.setMsgId(msgExt.getMsgId());\n        inner.setWaitStoreMsgOK(false);\n        return inner;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/GetResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.transaction.queue;\n\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic class GetResult {\n    private MessageExt msg;\n    private PullResult pullResult;\n\n    public MessageExt getMsg() {\n        return msg;\n    }\n\n    public void setMsg(MessageExt msg) {\n        this.msg = msg;\n    }\n\n    public PullResult getPullResult() {\n        return pullResult;\n    }\n\n    public void setPullResult(PullResult pullResult) {\n        this.pullResult = pullResult;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/MessageQueueOpContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.transaction.queue;\n\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.atomic.AtomicInteger;\n\npublic class MessageQueueOpContext {\n    private AtomicInteger totalSize = new AtomicInteger(0);\n    private volatile long lastWriteTimestamp;\n    private LinkedBlockingQueue<String> contextQueue;\n\n    public MessageQueueOpContext(long timestamp, int queueLength) {\n        this.lastWriteTimestamp = timestamp;\n        contextQueue = new LinkedBlockingQueue<String>(queueLength);\n    }\n\n    public LinkedBlockingQueue<String> getContextQueue() {\n        return contextQueue;\n    }\n\n\n    public AtomicInteger getTotalSize() {\n        return totalSize;\n    }\n\n\n    public long getLastWriteTimestamp() {\n        return lastWriteTimestamp;\n    }\n\n\n    public void setLastWriteTimestamp(long lastWriteTimestamp) {\n        this.lastWriteTimestamp = lastWriteTimestamp;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.transaction.queue;\n\nimport io.opentelemetry.api.common.Attributes;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.broker.BrokerController;\n\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\n\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_TOPIC;\n\npublic class TransactionalMessageBridge {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME);\n\n    private final ConcurrentHashMap<Integer, MessageQueue> opQueueMap = new ConcurrentHashMap<>();\n    private final BrokerController brokerController;\n    private final MessageStore store;\n    private final SocketAddress storeHost;\n\n    public TransactionalMessageBridge(BrokerController brokerController, MessageStore store) {\n        try {\n            this.brokerController = brokerController;\n            this.store = store;\n            this.storeHost =\n                new InetSocketAddress(brokerController.getBrokerConfig().getBrokerIP1(),\n                    brokerController.getNettyServerConfig().getListenPort());\n        } catch (Exception e) {\n            LOGGER.error(\"Init TransactionBridge error\", e);\n            throw new RuntimeException(e);\n        }\n\n    }\n\n    public long fetchConsumeOffset(MessageQueue mq) {\n        long offset = brokerController.getConsumerOffsetManager().queryOffset(TransactionalMessageUtil.buildConsumerGroup(),\n            mq.getTopic(), mq.getQueueId());\n        if (offset == -1) {\n            offset = store.getMinOffsetInQueue(mq.getTopic(), mq.getQueueId());\n        }\n        return offset;\n    }\n\n    public Set<MessageQueue> fetchMessageQueues(String topic) {\n        Set<MessageQueue> mqSet = new HashSet<>();\n        TopicConfig topicConfig = selectTopicConfig(topic);\n        if (topicConfig != null && topicConfig.getReadQueueNums() > 0) {\n            for (int i = 0; i < topicConfig.getReadQueueNums(); i++) {\n                MessageQueue mq = new MessageQueue();\n                mq.setTopic(topic);\n                mq.setBrokerName(brokerController.getBrokerConfig().getBrokerName());\n                mq.setQueueId(i);\n                mqSet.add(mq);\n            }\n        }\n        return mqSet;\n    }\n\n    public void updateConsumeOffset(MessageQueue mq, long offset) {\n        this.brokerController.getConsumerOffsetManager().commitOffset(\n            RemotingHelper.parseSocketAddressAddr(this.storeHost), TransactionalMessageUtil.buildConsumerGroup(), mq.getTopic(),\n            mq.getQueueId(), offset);\n    }\n\n    public PullResult getHalfMessage(int queueId, long offset, int nums) {\n        String group = TransactionalMessageUtil.buildConsumerGroup();\n        String topic = TransactionalMessageUtil.buildHalfTopic();\n        SubscriptionData sub = new SubscriptionData(topic, \"*\");\n        return getMessage(group, topic, queueId, offset, nums, sub);\n    }\n\n    public PullResult getOpMessage(int queueId, long offset, int nums) {\n        String group = TransactionalMessageUtil.buildConsumerGroup();\n        String topic = TransactionalMessageUtil.buildOpTopic();\n        SubscriptionData sub = new SubscriptionData(topic, \"*\");\n        return getMessage(group, topic, queueId, offset, nums, sub);\n    }\n\n    private PullResult getMessage(String group, String topic, int queueId, long offset, int nums,\n        SubscriptionData sub) {\n        GetMessageResult getMessageResult = store.getMessage(group, topic, queueId, offset, nums, null);\n\n        if (getMessageResult != null) {\n            PullStatus pullStatus = PullStatus.NO_NEW_MSG;\n            List<MessageExt> foundList = null;\n            switch (getMessageResult.getStatus()) {\n                case FOUND:\n                    pullStatus = PullStatus.FOUND;\n                    foundList = decodeMsgList(getMessageResult);\n                    this.brokerController.getBrokerStatsManager().incGroupGetNums(group, topic,\n                        getMessageResult.getMessageCount());\n                    this.brokerController.getBrokerStatsManager().incGroupGetSize(group, topic,\n                        getMessageResult.getBufferTotalSize());\n                    this.brokerController.getBrokerStatsManager().incBrokerGetNums(topic, getMessageResult.getMessageCount());\n                    if (foundList == null || foundList.size() == 0) {\n                        break;\n                    }\n                    this.brokerController.getBrokerStatsManager().recordDiskFallBehindTime(group, topic, queueId,\n                        this.brokerController.getMessageStore().now() - foundList.get(foundList.size() - 1)\n                            .getStoreTimestamp());\n\n                    Attributes attributes = this.brokerController.getBrokerMetricsManager().newAttributesBuilder()\n                        .put(LABEL_TOPIC, topic)\n                        .put(LABEL_CONSUMER_GROUP, group)\n                        .put(LABEL_IS_SYSTEM, TopicValidator.isSystemTopic(topic) || MixAll.isSysConsumerGroup(group))\n                        .build();\n                    this.brokerController.getBrokerMetricsManager().getMessagesOutTotal().add(getMessageResult.getMessageCount(), attributes);\n                    this.brokerController.getBrokerMetricsManager().getThroughputOutTotal().add(getMessageResult.getBufferTotalSize(), attributes);\n\n                    break;\n                case NO_MATCHED_MESSAGE:\n                    pullStatus = PullStatus.NO_MATCHED_MSG;\n                    LOGGER.warn(\"No matched message. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}\",\n                        getMessageResult.getStatus(), topic, group, offset);\n                    break;\n                case NO_MESSAGE_IN_QUEUE:\n                case OFFSET_OVERFLOW_ONE:\n                    pullStatus = PullStatus.NO_NEW_MSG;\n                    LOGGER.warn(\"No new message. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}\",\n                        getMessageResult.getStatus(), topic, group, offset);\n                    break;\n                case MESSAGE_WAS_REMOVING:\n                case NO_MATCHED_LOGIC_QUEUE:\n                case OFFSET_FOUND_NULL:\n                case OFFSET_OVERFLOW_BADLY:\n                case OFFSET_TOO_SMALL:\n                    pullStatus = PullStatus.OFFSET_ILLEGAL;\n                    LOGGER.warn(\"Offset illegal. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}\",\n                        getMessageResult.getStatus(), topic, group, offset);\n                    break;\n                default:\n                    assert false;\n                    break;\n            }\n\n            return new PullResult(pullStatus, getMessageResult.getNextBeginOffset(), getMessageResult.getMinOffset(),\n                getMessageResult.getMaxOffset(), foundList);\n\n        } else {\n            LOGGER.error(\"Get message from store return null. topic={}, groupId={}, requestOffset={}\", topic, group,\n                offset);\n            return null;\n        }\n    }\n\n    private List<MessageExt> decodeMsgList(GetMessageResult getMessageResult) {\n        List<MessageExt> foundList = new ArrayList<>();\n        try {\n            List<ByteBuffer> messageBufferList = getMessageResult.getMessageBufferList();\n            for (ByteBuffer bb : messageBufferList) {\n                MessageExt msgExt = MessageDecoder.decode(bb, true, false);\n                if (msgExt != null) {\n                    foundList.add(msgExt);\n                }\n            }\n\n        } finally {\n            getMessageResult.release();\n        }\n\n        return foundList;\n    }\n\n    public PutMessageResult putHalfMessage(MessageExtBrokerInner messageInner) {\n        return store.putMessage(parseHalfMessageInner(messageInner));\n    }\n\n    public CompletableFuture<PutMessageResult> asyncPutHalfMessage(MessageExtBrokerInner messageInner) {\n        return store.asyncPutMessage(parseHalfMessageInner(messageInner));\n    }\n\n    private MessageExtBrokerInner parseHalfMessageInner(MessageExtBrokerInner msgInner) {\n        String uniqId = msgInner.getUserProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX);\n        if (uniqId != null && !uniqId.isEmpty()) {\n            MessageAccessor.putProperty(msgInner, TransactionalMessageUtil.TRANSACTION_ID, uniqId);\n        }\n        MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_REAL_TOPIC, msgInner.getTopic());\n        MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_REAL_QUEUE_ID,\n            String.valueOf(msgInner.getQueueId()));\n        msgInner.setSysFlag(\n            MessageSysFlag.resetTransactionValue(msgInner.getSysFlag(), MessageSysFlag.TRANSACTION_NOT_TYPE));\n        if (null != store.getMessageStoreConfig() && store.getMessageStoreConfig().isTransRocksDBEnable() && !store.getMessageStoreConfig().isTransWriteOriginTransHalfEnable()) {\n            msgInner.setTopic(TransactionalMessageUtil.buildHalfTopicForRocksDB());\n        } else {\n            msgInner.setTopic(TransactionalMessageUtil.buildHalfTopic());\n        }\n        msgInner.setQueueId(0);\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));\n        return msgInner;\n    }\n\n    public PutMessageResult putMessageReturnResult(MessageExtBrokerInner messageInner) {\n        LOGGER.debug(\"[BUG-TO-FIX] Thread:{} msgID:{}\", Thread.currentThread().getName(), messageInner.getMsgId());\n        PutMessageResult result = store.putMessage(messageInner);\n        if (result != null && result.getPutMessageStatus() == PutMessageStatus.PUT_OK) {\n            this.brokerController.getBrokerStatsManager().incTopicPutNums(messageInner.getTopic());\n            this.brokerController.getBrokerStatsManager().incTopicPutSize(messageInner.getTopic(),\n                    result.getAppendMessageResult().getWroteBytes());\n            this.brokerController.getBrokerStatsManager().incBrokerPutNums();\n        }\n        return result;\n    }\n\n    public boolean putMessage(MessageExtBrokerInner messageInner) {\n        PutMessageResult putMessageResult = store.putMessage(messageInner);\n        if (putMessageResult != null\n            && putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK) {\n            return true;\n        } else {\n            LOGGER.error(\"Put message failed, topic: {}, queueId: {}, msgId: {}\",\n                messageInner.getTopic(), messageInner.getQueueId(), messageInner.getMsgId());\n            return false;\n        }\n    }\n\n    public MessageExtBrokerInner renewImmunityHalfMessageInner(MessageExt msgExt) {\n        MessageExtBrokerInner msgInner = renewHalfMessageInner(msgExt);\n        String queueOffsetFromPrepare = msgExt.getUserProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET);\n        if (null != queueOffsetFromPrepare) {\n            MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET,\n                    queueOffsetFromPrepare);\n        } else {\n            MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET,\n                String.valueOf(msgExt.getQueueOffset()));\n        }\n\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));\n\n        return msgInner;\n    }\n\n    public MessageExtBrokerInner renewHalfMessageInner(MessageExt msgExt) {\n        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n        msgInner.setTopic(msgExt.getTopic());\n        msgInner.setBody(msgExt.getBody());\n        msgInner.setQueueId(msgExt.getQueueId());\n        msgInner.setMsgId(msgExt.getMsgId());\n        msgInner.setSysFlag(msgExt.getSysFlag());\n        msgInner.setTags(msgExt.getTags());\n        msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(msgInner.getTags()));\n        MessageAccessor.setProperties(msgInner, msgExt.getProperties());\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties()));\n        msgInner.setBornTimestamp(msgExt.getBornTimestamp());\n        msgInner.setBornHost(msgExt.getBornHost());\n        msgInner.setStoreHost(msgExt.getStoreHost());\n        msgInner.setWaitStoreMsgOK(false);\n        return msgInner;\n    }\n\n    private MessageExtBrokerInner makeOpMessageInner(Message message, MessageQueue messageQueue) {\n        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n        msgInner.setTopic(message.getTopic());\n        msgInner.setBody(message.getBody());\n        msgInner.setQueueId(messageQueue.getQueueId());\n        msgInner.setTags(message.getTags());\n        msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(msgInner.getTags()));\n        msgInner.setSysFlag(0);\n        MessageAccessor.setProperties(msgInner, message.getProperties());\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(message.getProperties()));\n        msgInner.setBornTimestamp(System.currentTimeMillis());\n        msgInner.setBornHost(this.storeHost);\n        msgInner.setStoreHost(this.storeHost);\n        msgInner.setWaitStoreMsgOK(false);\n        MessageClientIDSetter.setUniqID(msgInner);\n        return msgInner;\n    }\n\n    private TopicConfig selectTopicConfig(String topic) {\n        TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(topic);\n        if (topicConfig == null) {\n            topicConfig = this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod(\n                topic, 1, PermName.PERM_WRITE | PermName.PERM_READ, 0);\n        }\n        return topicConfig;\n    }\n\n    public boolean writeOp(Integer queueId,Message message) {\n        MessageQueue opQueue = opQueueMap.get(queueId);\n        if (opQueue == null) {\n            opQueue = getOpQueueByHalf(queueId, this.brokerController.getBrokerConfig().getBrokerName());\n            MessageQueue oldQueue = opQueueMap.putIfAbsent(queueId, opQueue);\n            if (oldQueue != null) {\n                opQueue = oldQueue;\n            }\n        }\n\n        PutMessageResult result = putMessageReturnResult(makeOpMessageInner(message, opQueue));\n        if (result != null && result.getPutMessageStatus() == PutMessageStatus.PUT_OK) {\n            return true;\n        }\n\n        return false;\n    }\n\n    private MessageQueue getOpQueueByHalf(Integer queueId, String brokerName) {\n        MessageQueue opQueue = new MessageQueue();\n        opQueue.setTopic(TransactionalMessageUtil.buildOpTopic());\n        opQueue.setBrokerName(brokerName);\n        opQueue.setQueueId(queueId);\n        return opQueue;\n    }\n\n    public MessageExt lookMessageByOffset(final long commitLogOffset) {\n        return this.store.lookMessageByOffset(commitLogOffset);\n    }\n\n    public BrokerController getBrokerController() {\n        return brokerController;\n    }\n\n    public boolean escapeMessage(MessageExtBrokerInner messageInner) {\n        PutMessageResult putMessageResult = this.brokerController.getEscapeBridge().putMessage(messageInner);\n        if (putMessageResult != null && putMessageResult.isOk()) {\n            return true;\n        } else {\n            LOGGER.error(\"Escaping message failed, topic: {}, queueId: {}, msgId: {}\",\n                messageInner.getTopic(), messageInner.getQueueId(), messageInner.getMsgId());\n            return false;\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.transaction.queue;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.Date;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.TimeUnit;\n\nimport org.apache.rocketmq.broker.BrokerPathConfigHelper;\nimport org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener;\nimport org.apache.rocketmq.broker.transaction.OperationResult;\nimport org.apache.rocketmq.broker.transaction.TransactionMetrics;\nimport org.apache.rocketmq.broker.transaction.TransactionalMessageService;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.config.BrokerRole;\n\npublic class TransactionalMessageServiceImpl implements TransactionalMessageService {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME);\n\n    private TransactionalMessageBridge transactionalMessageBridge;\n\n    private static final int PULL_MSG_RETRY_NUMBER = 1;\n\n    private static final int MAX_PROCESS_TIME_LIMIT = 60000;\n    private static final int MAX_RETRY_TIMES_FOR_ESCAPE = 10;\n\n    private static final int MAX_RETRY_COUNT_WHEN_HALF_NULL = 1;\n\n    private static final int OP_MSG_PULL_NUMS = 32;\n\n    private static final int SLEEP_WHILE_NO_OP = 1000;\n\n    private final ConcurrentHashMap<Integer, MessageQueueOpContext> deleteContext = new ConcurrentHashMap<>();\n\n    private ServiceThread transactionalOpBatchService;\n\n    private ConcurrentHashMap<MessageQueue, MessageQueue> opQueueMap = new ConcurrentHashMap<>();\n\n    private TransactionMetrics transactionMetrics;\n\n    public TransactionalMessageServiceImpl(TransactionalMessageBridge transactionBridge) {\n        this.transactionalMessageBridge = transactionBridge;\n        transactionalOpBatchService = new TransactionalOpBatchService(transactionalMessageBridge.getBrokerController(), this);\n        transactionalOpBatchService.start();\n        transactionMetrics = new TransactionMetrics(BrokerPathConfigHelper.getTransactionMetricsPath(\n                transactionalMessageBridge.getBrokerController().getMessageStoreConfig().getStorePathRootDir()));\n        transactionMetrics.load();\n    }\n\n    @Override\n    public TransactionMetrics getTransactionMetrics() {\n        return transactionMetrics;\n    }\n\n    @Override\n    public void setTransactionMetrics(TransactionMetrics transactionMetrics) {\n        this.transactionMetrics = transactionMetrics;\n    }\n\n\n    @Override\n    public CompletableFuture<PutMessageResult> asyncPrepareMessage(MessageExtBrokerInner messageInner) {\n        return transactionalMessageBridge.asyncPutHalfMessage(messageInner);\n    }\n\n    @Override\n    public PutMessageResult prepareMessage(MessageExtBrokerInner messageInner) {\n        return transactionalMessageBridge.putHalfMessage(messageInner);\n    }\n\n    private boolean needDiscard(MessageExt msgExt, int transactionCheckMax) {\n        String checkTimes = msgExt.getProperty(MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES);\n        int checkTime = 1;\n        if (null != checkTimes) {\n            checkTime = getInt(checkTimes);\n            if (checkTime >= transactionCheckMax) {\n                return true;\n            } else {\n                checkTime++;\n            }\n        }\n        msgExt.putUserProperty(MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES, String.valueOf(checkTime));\n        return false;\n    }\n\n    private boolean needSkip(MessageExt msgExt) {\n        long valueOfCurrentMinusBorn = System.currentTimeMillis() - msgExt.getBornTimestamp();\n        if (valueOfCurrentMinusBorn\n            > transactionalMessageBridge.getBrokerController().getMessageStoreConfig().getFileReservedTime()\n            * 3600L * 1000) {\n            log.info(\"Half message exceed file reserved time ,so skip it.messageId {},bornTime {}\",\n                msgExt.getMsgId(), msgExt.getBornTimestamp());\n            return true;\n        }\n        return false;\n    }\n\n    private boolean putBackHalfMsgQueue(MessageExt msgExt, long offset) {\n        PutMessageResult putMessageResult = putBackToHalfQueueReturnResult(msgExt);\n        if (putMessageResult != null\n            && putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK) {\n            msgExt.setQueueOffset(\n                putMessageResult.getAppendMessageResult().getLogicsOffset());\n            msgExt.setCommitLogOffset(\n                putMessageResult.getAppendMessageResult().getWroteOffset());\n            msgExt.setMsgId(putMessageResult.getAppendMessageResult().getMsgId());\n            log.debug(\n                \"Send check message, the offset={} restored in queueOffset={} \"\n                    + \"commitLogOffset={} \"\n                    + \"newMsgId={} realMsgId={} topic={}\",\n                offset, msgExt.getQueueOffset(), msgExt.getCommitLogOffset(), msgExt.getMsgId(),\n                msgExt.getUserProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX),\n                msgExt.getTopic());\n            return true;\n        } else {\n            log.error(\n                \"PutBackToHalfQueueReturnResult write failed, topic: {}, queueId: {}, \"\n                    + \"msgId: {}\",\n                msgExt.getTopic(), msgExt.getQueueId(), msgExt.getMsgId());\n            return false;\n        }\n    }\n\n    @Override\n    public void check(long transactionTimeout, int transactionCheckMax,\n        AbstractTransactionalMessageCheckListener listener) {\n        try {\n            String topic = TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC;\n            Set<MessageQueue> msgQueues = transactionalMessageBridge.fetchMessageQueues(topic);\n            if (msgQueues == null || msgQueues.size() == 0) {\n                log.warn(\"The queue of topic is empty :\" + topic);\n                return;\n            }\n            log.debug(\"Check topic={}, queues={}\", topic, msgQueues);\n            for (MessageQueue messageQueue : msgQueues) {\n                long startTime = System.currentTimeMillis();\n                MessageQueue opQueue = getOpQueue(messageQueue);\n                long halfOffset = transactionalMessageBridge.fetchConsumeOffset(messageQueue);\n                long opOffset = transactionalMessageBridge.fetchConsumeOffset(opQueue);\n                log.info(\"Before check, the queue={} msgOffset={} opOffset={}\", messageQueue, halfOffset, opOffset);\n                if (halfOffset < 0 || opOffset < 0) {\n                    log.error(\"MessageQueue: {} illegal offset read: {}, op offset: {},skip this queue\", messageQueue,\n                        halfOffset, opOffset);\n                    continue;\n                }\n\n                List<Long> doneOpOffset = new ArrayList<>();\n                HashMap<Long, Long> removeMap = new HashMap<>();\n                HashMap<Long, HashSet<Long>> opMsgMap = new HashMap<Long, HashSet<Long>>();\n                PullResult pullResult = fillOpRemoveMap(removeMap, opQueue, opOffset, halfOffset, opMsgMap, doneOpOffset);\n                if (null == pullResult) {\n                    log.error(\"The queue={} check msgOffset={} with opOffset={} failed, pullResult is null\",\n                        messageQueue, halfOffset, opOffset);\n                    continue;\n                }\n                // single thread\n                int getMessageNullCount = 1;\n                long newOffset = halfOffset;\n                long i = halfOffset;\n                long nextOpOffset = pullResult.getNextBeginOffset();\n                int putInQueueCount = 0;\n                int escapeFailCnt = 0;\n\n                while (true) {\n                    if (System.currentTimeMillis() - startTime > MAX_PROCESS_TIME_LIMIT) {\n                        log.info(\"Queue={} process time reach max={}\", messageQueue, MAX_PROCESS_TIME_LIMIT);\n                        break;\n                    }\n                    Long removedOpOffset;\n                    if ((removedOpOffset = removeMap.remove(i)) != null) {\n                        log.debug(\"Half offset {} has been committed/rolled back\", i);\n                        opMsgMap.get(removedOpOffset).remove(i);\n                        if (opMsgMap.get(removedOpOffset).size() == 0) {\n                            opMsgMap.remove(removedOpOffset);\n                            doneOpOffset.add(removedOpOffset);\n                        }\n                    } else {\n                        GetResult getResult = getHalfMsg(messageQueue, i);\n                        MessageExt msgExt = getResult.getMsg();\n                        if (msgExt == null) {\n                            if (getMessageNullCount++ > MAX_RETRY_COUNT_WHEN_HALF_NULL) {\n                                break;\n                            }\n                            if (getResult.getPullResult().getPullStatus() == PullStatus.NO_NEW_MSG) {\n                                log.debug(\"No new msg, the miss offset={} in={}, continue check={}, pull result={}\", i,\n                                    messageQueue, getMessageNullCount, getResult.getPullResult());\n                                break;\n                            } else {\n                                log.info(\"Illegal offset, the miss offset={} in={}, continue check={}, pull result={}\",\n                                    i, messageQueue, getMessageNullCount, getResult.getPullResult());\n                                i = getResult.getPullResult().getNextBeginOffset();\n                                newOffset = i;\n                                continue;\n                            }\n                        }\n\n                        if (this.transactionalMessageBridge.getBrokerController().getBrokerConfig().isEnableSlaveActingMaster()\n                            && this.transactionalMessageBridge.getBrokerController().getMinBrokerIdInGroup()\n                            == this.transactionalMessageBridge.getBrokerController().getBrokerIdentity().getBrokerId()\n                            && BrokerRole.SLAVE.equals(this.transactionalMessageBridge.getBrokerController().getMessageStoreConfig().getBrokerRole())\n                        ) {\n                            final MessageExtBrokerInner msgInner = this.transactionalMessageBridge.renewHalfMessageInner(msgExt);\n                            final boolean isSuccess = this.transactionalMessageBridge.escapeMessage(msgInner);\n\n                            if (isSuccess) {\n                                escapeFailCnt = 0;\n                                newOffset = i + 1;\n                                i++;\n                            } else {\n                                log.warn(\"Escaping transactional message failed {} times! msgId(offsetId)={}, UNIQ_KEY(transactionId)={}\",\n                                    escapeFailCnt + 1,\n                                    msgExt.getMsgId(),\n                                    msgExt.getUserProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));\n                                if (escapeFailCnt < MAX_RETRY_TIMES_FOR_ESCAPE) {\n                                    escapeFailCnt++;\n                                    Thread.sleep(100L * (2 ^ escapeFailCnt));\n                                } else {\n                                    escapeFailCnt = 0;\n                                    newOffset = i + 1;\n                                    i++;\n                                }\n                            }\n                            continue;\n                        }\n\n                        if (needDiscard(msgExt, transactionCheckMax) || needSkip(msgExt)) {\n                            listener.resolveDiscardMsg(msgExt);\n                            newOffset = i + 1;\n                            i++;\n                            continue;\n                        }\n                        if (msgExt.getStoreTimestamp() >= startTime) {\n                            log.debug(\"Fresh stored. the miss offset={}, check it later, store={}\", i,\n                                new Date(msgExt.getStoreTimestamp()));\n                            break;\n                        }\n\n                        long valueOfCurrentMinusBorn = System.currentTimeMillis() - msgExt.getBornTimestamp();\n                        long checkImmunityTime = transactionTimeout;\n                        String checkImmunityTimeStr = msgExt.getUserProperty(MessageConst.PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS);\n                        if (null != checkImmunityTimeStr) {\n                            checkImmunityTime = getImmunityTime(checkImmunityTimeStr, transactionTimeout);\n                            if (valueOfCurrentMinusBorn <= checkImmunityTime) {\n                                if (checkPrepareQueueOffset(removeMap, doneOpOffset, msgExt, checkImmunityTimeStr)) {\n                                    newOffset = i + 1;\n                                    i++;\n                                    continue;\n                                }\n                            }\n                        } else {\n                            if (0 <= valueOfCurrentMinusBorn && valueOfCurrentMinusBorn <= checkImmunityTime) {\n                                log.debug(\"New arrived, the miss offset={}, check it later checkImmunity={}, born={}\", i,\n                                    checkImmunityTime, new Date(msgExt.getBornTimestamp()));\n                                break;\n                            }\n                        }\n                        List<MessageExt> opMsg = pullResult == null ? null : pullResult.getMsgFoundList();\n                        boolean isNeedCheck = opMsg == null && valueOfCurrentMinusBorn > checkImmunityTime\n                            || opMsg != null && opMsg.get(opMsg.size() - 1).getBornTimestamp() - startTime > transactionTimeout\n                            || valueOfCurrentMinusBorn <= -1;\n\n                        if (isNeedCheck) {\n\n                            if (!putBackHalfMsgQueue(msgExt, i)) {\n                                continue;\n                            }\n                            putInQueueCount++;\n                            log.info(\"Check transaction. real_topic={},uniqKey={},offset={},commitLogOffset={}\",\n                                    msgExt.getUserProperty(MessageConst.PROPERTY_REAL_TOPIC),\n                                    msgExt.getUserProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX),\n                                    msgExt.getQueueOffset(), msgExt.getCommitLogOffset());\n                            listener.resolveHalfMsg(msgExt);\n                        } else {\n                            nextOpOffset = pullResult != null ? pullResult.getNextBeginOffset() : nextOpOffset;\n                            pullResult = fillOpRemoveMap(removeMap, opQueue, nextOpOffset,\n                                    halfOffset, opMsgMap, doneOpOffset);\n                            if (pullResult == null || pullResult.getPullStatus() == PullStatus.NO_NEW_MSG\n                                    || pullResult.getPullStatus() == PullStatus.OFFSET_ILLEGAL\n                                    || pullResult.getPullStatus() == PullStatus.NO_MATCHED_MSG) {\n\n                                try {\n                                    Thread.sleep(SLEEP_WHILE_NO_OP);\n                                } catch (Throwable ignored) {\n                                }\n\n                            } else {\n                                log.info(\"The miss message offset:{}, pullOffsetOfOp:{}, miniOffset:{} get more opMsg.\", i, nextOpOffset, halfOffset);\n                            }\n\n                            continue;\n                        }\n                    }\n                    newOffset = i + 1;\n                    i++;\n                }\n                if (newOffset != halfOffset) {\n                    transactionalMessageBridge.updateConsumeOffset(messageQueue, newOffset);\n                }\n                long newOpOffset = calculateOpOffset(doneOpOffset, opOffset);\n                if (newOpOffset != opOffset) {\n                    transactionalMessageBridge.updateConsumeOffset(opQueue, newOpOffset);\n                }\n                GetResult getResult = getHalfMsg(messageQueue, newOffset);\n                pullResult = pullOpMsg(opQueue, newOpOffset, 1);\n                long maxMsgOffset = getResult.getPullResult() == null ? newOffset : getResult.getPullResult().getMaxOffset();\n                long maxOpOffset = pullResult == null ? newOpOffset : pullResult.getMaxOffset();\n                long msgTime = getResult.getMsg() == null ? System.currentTimeMillis() : getResult.getMsg().getStoreTimestamp();\n\n                log.info(\"After check, {} opOffset={} opOffsetDiff={} msgOffset={} msgOffsetDiff={} msgTime={} msgTimeDelayInMs={} putInQueueCount={}\",\n                        messageQueue, newOpOffset, maxOpOffset - newOpOffset, newOffset, maxMsgOffset - newOffset, new Date(msgTime),\n                        System.currentTimeMillis() - msgTime, putInQueueCount);\n            }\n        } catch (Throwable e) {\n            log.error(\"Check error\", e);\n        }\n\n    }\n\n    private long getImmunityTime(String checkImmunityTimeStr, long transactionTimeout) {\n        long checkImmunityTime;\n\n        checkImmunityTime = getLong(checkImmunityTimeStr);\n        if (-1 == checkImmunityTime) {\n            checkImmunityTime = transactionTimeout;\n        } else {\n            checkImmunityTime *= 1000;\n        }\n        return checkImmunityTime;\n    }\n\n    /**\n     * Read op message, parse op message, and fill removeMap\n     *\n     * @param removeMap Half message to be remove, key:halfOffset, value: opOffset.\n     * @param opQueue Op message queue.\n     * @param pullOffsetOfOp The begin offset of op message queue.\n     * @param miniOffset The current minimum offset of half message queue.\n     * @param opMsgMap Half message offset in op message\n     * @param doneOpOffset Stored op messages that have been processed.\n     * @return Op message result.\n     */\n    private PullResult fillOpRemoveMap(HashMap<Long, Long> removeMap, MessageQueue opQueue,\n                                       long pullOffsetOfOp, long miniOffset, Map<Long, HashSet<Long>> opMsgMap, List<Long> doneOpOffset) {\n        PullResult pullResult = pullOpMsg(opQueue, pullOffsetOfOp, OP_MSG_PULL_NUMS);\n        if (null == pullResult) {\n            return null;\n        }\n        if (pullResult.getPullStatus() == PullStatus.OFFSET_ILLEGAL\n            || pullResult.getPullStatus() == PullStatus.NO_MATCHED_MSG) {\n            log.warn(\"The miss op offset={} in queue={} is illegal, pullResult={}\", pullOffsetOfOp, opQueue,\n                pullResult);\n            transactionalMessageBridge.updateConsumeOffset(opQueue, pullResult.getNextBeginOffset());\n            return pullResult;\n        } else if (pullResult.getPullStatus() == PullStatus.NO_NEW_MSG) {\n            log.warn(\"The miss op offset={} in queue={} is NO_NEW_MSG, pullResult={}\", pullOffsetOfOp, opQueue,\n                pullResult);\n            return pullResult;\n        }\n        List<MessageExt> opMsg = pullResult.getMsgFoundList();\n        if (opMsg == null) {\n            log.warn(\"The miss op offset={} in queue={} is empty, pullResult={}\", pullOffsetOfOp, opQueue, pullResult);\n            return pullResult;\n        }\n        for (MessageExt opMessageExt : opMsg) {\n            if (opMessageExt.getBody() == null) {\n                log.error(\"op message body is null. queueId={}, offset={}\", opMessageExt.getQueueId(),\n                        opMessageExt.getQueueOffset());\n                doneOpOffset.add(opMessageExt.getQueueOffset());\n                continue;\n            }\n            HashSet<Long> set = new HashSet<Long>();\n            String queueOffsetBody = new String(opMessageExt.getBody(), TransactionalMessageUtil.CHARSET);\n\n            log.debug(\"Topic: {} tags: {}, OpOffset: {}, HalfOffset: {}\", opMessageExt.getTopic(),\n                    opMessageExt.getTags(), opMessageExt.getQueueOffset(), queueOffsetBody);\n            if (TransactionalMessageUtil.REMOVE_TAG.equals(opMessageExt.getTags())) {\n                String[] offsetArray = queueOffsetBody.split(TransactionalMessageUtil.OFFSET_SEPARATOR);\n                for (String offset : offsetArray) {\n                    Long offsetValue = getLong(offset);\n                    if (offsetValue < miniOffset) {\n                        continue;\n                    }\n\n                    removeMap.put(offsetValue, opMessageExt.getQueueOffset());\n                    set.add(offsetValue);\n                }\n            } else {\n                log.error(\"Found a illegal tag in opMessageExt= {} \", opMessageExt);\n            }\n\n            if (set.size() > 0) {\n                opMsgMap.put(opMessageExt.getQueueOffset(), set);\n            } else {\n                doneOpOffset.add(opMessageExt.getQueueOffset());\n            }\n        }\n\n        log.debug(\"Remove map: {}\", removeMap);\n        log.debug(\"Done op list: {}\", doneOpOffset);\n        log.debug(\"opMsg map: {}\", opMsgMap);\n        return pullResult;\n    }\n\n    /**\n     * If return true, skip this msg\n     *\n     * @param removeMap Op message map to determine whether a half message was responded by producer.\n     * @param doneOpOffset Op Message which has been checked.\n     * @param msgExt Half message\n     * @return Return true if put success, otherwise return false.\n     */\n    private boolean checkPrepareQueueOffset(HashMap<Long, Long> removeMap, List<Long> doneOpOffset,\n        MessageExt msgExt, String checkImmunityTimeStr) {\n        String prepareQueueOffsetStr = msgExt.getUserProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET);\n        if (null == prepareQueueOffsetStr) {\n            return putImmunityMsgBackToHalfQueue(msgExt);\n        } else {\n            long prepareQueueOffset = getLong(prepareQueueOffsetStr);\n            if (-1 == prepareQueueOffset) {\n                return false;\n            } else {\n                Long tmpOpOffset;\n                if ((tmpOpOffset = removeMap.remove(prepareQueueOffset)) != null) {\n                    doneOpOffset.add(tmpOpOffset);\n                    log.info(\"removeMap contain prepareQueueOffset. real_topic={},uniqKey={},immunityTime={},offset={}\",\n                            msgExt.getUserProperty(MessageConst.PROPERTY_REAL_TOPIC),\n                            msgExt.getUserProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX),\n                            checkImmunityTimeStr,\n                            msgExt.getQueueOffset());\n                    return true;\n                } else {\n                    return putImmunityMsgBackToHalfQueue(msgExt);\n                }\n            }\n        }\n    }\n\n    /**\n     * Write messageExt to Half topic again\n     *\n     * @param messageExt Message will be write back to queue\n     * @return Put result can used to determine the specific results of storage.\n     */\n    private PutMessageResult putBackToHalfQueueReturnResult(MessageExt messageExt) {\n        PutMessageResult putMessageResult = null;\n        try {\n            MessageExtBrokerInner msgInner = transactionalMessageBridge.renewHalfMessageInner(messageExt);\n            putMessageResult = transactionalMessageBridge.putMessageReturnResult(msgInner);\n        } catch (Exception e) {\n            log.warn(\"PutBackToHalfQueueReturnResult error\", e);\n        }\n        return putMessageResult;\n    }\n\n    private boolean putImmunityMsgBackToHalfQueue(MessageExt messageExt) {\n        MessageExtBrokerInner msgInner = transactionalMessageBridge.renewImmunityHalfMessageInner(messageExt);\n        return transactionalMessageBridge.putMessage(msgInner);\n    }\n\n    /**\n     * Read half message from Half Topic\n     *\n     * @param mq Target message queue, in this method, it means the half message queue.\n     * @param offset Offset in the message queue.\n     * @param nums Pull message number.\n     * @return Messages pulled from half message queue.\n     */\n    private PullResult pullHalfMsg(MessageQueue mq, long offset, int nums) {\n        return transactionalMessageBridge.getHalfMessage(mq.getQueueId(), offset, nums);\n    }\n\n    /**\n     * Read op message from Op Topic\n     *\n     * @param mq Target Message Queue\n     * @param offset Offset in the message queue\n     * @param nums Pull message number\n     * @return Messages pulled from operate message queue.\n     */\n    private PullResult pullOpMsg(MessageQueue mq, long offset, int nums) {\n        return transactionalMessageBridge.getOpMessage(mq.getQueueId(), offset, nums);\n    }\n\n    private Long getLong(String s) {\n        long v = -1;\n        try {\n            v = Long.parseLong(s);\n        } catch (Exception e) {\n            log.error(\"GetLong error\", e);\n        }\n        return v;\n\n    }\n\n    private Integer getInt(String s) {\n        int v = -1;\n        try {\n            v = Integer.parseInt(s);\n        } catch (Exception e) {\n            log.error(\"GetInt error\", e);\n        }\n        return v;\n\n    }\n\n    private long calculateOpOffset(List<Long> doneOffset, long oldOffset) {\n        Collections.sort(doneOffset);\n        long newOffset = oldOffset;\n        for (int i = 0; i < doneOffset.size(); i++) {\n            if (doneOffset.get(i) == newOffset) {\n                newOffset++;\n            } else {\n                break;\n            }\n        }\n        return newOffset;\n\n    }\n\n    private MessageQueue getOpQueue(MessageQueue messageQueue) {\n        MessageQueue opQueue = opQueueMap.get(messageQueue);\n        if (opQueue == null) {\n            opQueue = new MessageQueue(TransactionalMessageUtil.buildOpTopic(), messageQueue.getBrokerName(),\n                messageQueue.getQueueId());\n            opQueueMap.put(messageQueue, opQueue);\n        }\n        return opQueue;\n\n    }\n\n    private GetResult getHalfMsg(MessageQueue messageQueue, long offset) {\n        GetResult getResult = new GetResult();\n\n        PullResult result = pullHalfMsg(messageQueue, offset, PULL_MSG_RETRY_NUMBER);\n        if (result != null) {\n            getResult.setPullResult(result);\n            List<MessageExt> messageExts = result.getMsgFoundList();\n            if (messageExts == null || messageExts.size() == 0) {\n                return getResult;\n            }\n            getResult.setMsg(messageExts.get(0));\n        }\n        return getResult;\n    }\n\n    private OperationResult getHalfMessageByOffset(long commitLogOffset) {\n        OperationResult response = new OperationResult();\n        MessageExt messageExt = this.transactionalMessageBridge.lookMessageByOffset(commitLogOffset);\n        if (messageExt != null) {\n            response.setPrepareMessage(messageExt);\n            response.setResponseCode(ResponseCode.SUCCESS);\n        } else {\n            response.setResponseCode(ResponseCode.SYSTEM_ERROR);\n            response.setResponseRemark(\"Find prepared transaction message failed\");\n        }\n        return response;\n    }\n\n    @Override\n    public boolean deletePrepareMessage(MessageExt messageExt) {\n        Integer queueId = messageExt.getQueueId();\n        MessageQueueOpContext mqContext = deleteContext.get(queueId);\n        if (mqContext == null) {\n            mqContext = new MessageQueueOpContext(System.currentTimeMillis(), 20000);\n            MessageQueueOpContext old = deleteContext.putIfAbsent(queueId, mqContext);\n            if (old != null) {\n                mqContext = old;\n            }\n        }\n\n        String data = messageExt.getQueueOffset() + TransactionalMessageUtil.OFFSET_SEPARATOR;\n        try {\n            boolean res = mqContext.getContextQueue().offer(data, 100, TimeUnit.MILLISECONDS);\n            if (res) {\n                int totalSize = mqContext.getTotalSize().addAndGet(data.length());\n                if (totalSize > transactionalMessageBridge.getBrokerController().getBrokerConfig().getTransactionOpMsgMaxSize()) {\n                    this.transactionalOpBatchService.wakeup();\n                }\n                return true;\n            } else {\n                this.transactionalOpBatchService.wakeup();\n            }\n        } catch (InterruptedException ignore) {\n        }\n\n        Message msg = getOpMessage(queueId, data);\n        if (this.transactionalMessageBridge.writeOp(queueId, msg)) {\n            log.warn(\"Force add remove op data. queueId={}\", queueId);\n            return true;\n        } else {\n            log.error(\"Transaction op message write failed. messageId is {}, queueId is {}\", messageExt.getMsgId(), messageExt.getQueueId());\n            return false;\n        }\n    }\n\n    @Override\n    public OperationResult commitMessage(EndTransactionRequestHeader requestHeader) {\n        return getHalfMessageByOffset(requestHeader.getCommitLogOffset());\n    }\n\n    @Override\n    public OperationResult rollbackMessage(EndTransactionRequestHeader requestHeader) {\n        return getHalfMessageByOffset(requestHeader.getCommitLogOffset());\n    }\n\n    @Override\n    public boolean open() {\n        return true;\n    }\n\n    @Override\n    public void close() {\n        if (this.transactionalOpBatchService != null) {\n            this.transactionalOpBatchService.shutdown();\n        }\n        this.getTransactionMetrics().persist();\n    }\n\n    public Message getOpMessage(int queueId, String moreData) {\n        String opTopic = TransactionalMessageUtil.buildOpTopic();\n        MessageQueueOpContext mqContext = deleteContext.get(queueId);\n\n        int moreDataLength = moreData != null ? moreData.length() : 0;\n        int length = moreDataLength;\n        int maxSize = transactionalMessageBridge.getBrokerController().getBrokerConfig().getTransactionOpMsgMaxSize();\n        if (length < maxSize) {\n            int sz = mqContext.getTotalSize().get();\n            if (sz > maxSize || length + sz > maxSize) {\n                length = maxSize + 100;\n            } else {\n                length += sz;\n            }\n        }\n\n        StringBuilder sb = new StringBuilder(length);\n\n        if (moreData != null) {\n            sb.append(moreData);\n        }\n\n        while (!mqContext.getContextQueue().isEmpty()) {\n            if (sb.length() >= maxSize) {\n                break;\n            }\n            String data = mqContext.getContextQueue().poll();\n            if (data != null) {\n                sb.append(data);\n            }\n        }\n\n        if (sb.length() == 0) {\n            return null;\n        }\n\n        int l = sb.length() - moreDataLength;\n        mqContext.getTotalSize().addAndGet(-l);\n        mqContext.setLastWriteTimestamp(System.currentTimeMillis());\n        return new Message(opTopic, TransactionalMessageUtil.REMOVE_TAG,\n                sb.toString().getBytes(TransactionalMessageUtil.CHARSET));\n    }\n    public long batchSendOpMessage() {\n        long startTime = System.currentTimeMillis();\n        try {\n            long firstTimestamp = startTime;\n            Map<Integer, Message> sendMap = null;\n            long interval = transactionalMessageBridge.getBrokerController().getBrokerConfig().getTransactionOpBatchInterval();\n            int maxSize = transactionalMessageBridge.getBrokerController().getBrokerConfig().getTransactionOpMsgMaxSize();\n            boolean overSize = false;\n            for (Map.Entry<Integer, MessageQueueOpContext> entry : deleteContext.entrySet()) {\n                MessageQueueOpContext mqContext = entry.getValue();\n                //no msg in contextQueue\n                if (mqContext.getTotalSize().get() <= 0 || mqContext.getContextQueue().size() == 0 ||\n                        // wait for the interval\n                        mqContext.getTotalSize().get() < maxSize &&\n                                startTime - mqContext.getLastWriteTimestamp() < interval) {\n                    continue;\n                }\n\n                if (sendMap == null) {\n                    sendMap = new HashMap<>();\n                }\n\n                Message opMsg = getOpMessage(entry.getKey(), null);\n                if (opMsg == null) {\n                    continue;\n                }\n                sendMap.put(entry.getKey(), opMsg);\n                firstTimestamp = Math.min(firstTimestamp, mqContext.getLastWriteTimestamp());\n                if (mqContext.getTotalSize().get() >= maxSize) {\n                    overSize = true;\n                }\n            }\n\n            if (sendMap != null) {\n                for (Map.Entry<Integer, Message> entry : sendMap.entrySet()) {\n                    if (!this.transactionalMessageBridge.writeOp(entry.getKey(), entry.getValue())) {\n                        log.error(\"Transaction batch op message write failed. body is {}, queueId is {}\",\n                                new String(entry.getValue().getBody(), TransactionalMessageUtil.CHARSET), entry.getKey());\n                    }\n                }\n            }\n\n            log.debug(\"Send op message queueIds={}\", sendMap == null ? null : sendMap.keySet());\n\n            //wait for next batch remove\n            long wakeupTimestamp = firstTimestamp + interval;\n            if (!overSize && wakeupTimestamp > startTime) {\n                return wakeupTimestamp;\n            }\n        } catch (Throwable t) {\n            log.error(\"batchSendOp error.\", t);\n        }\n\n        return 0L;\n    }\n\n    public Map<Integer, MessageQueueOpContext> getDeleteContext() {\n        return this.deleteContext;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageUtil.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.transaction.queue;\n\nimport java.nio.charset.Charset;\nimport java.nio.charset.StandardCharsets;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.common.topic.TopicValidator;\n\npublic class TransactionalMessageUtil {\n    public static final String REMOVE_TAG = \"d\";\n    public static final Charset CHARSET = StandardCharsets.UTF_8;\n    public static final String OFFSET_SEPARATOR = \",\";\n    public static final String TRANSACTION_ID = \"__transactionId__\";\n\n    public static String buildOpTopic() {\n        return TopicValidator.RMQ_SYS_TRANS_OP_HALF_TOPIC;\n    }\n\n    public static String buildOpTopicForRocksDB() {\n        return TopicValidator.RMQ_SYS_ROCKSDB_TRANS_OP_HALF_TOPIC;\n    }\n\n    public static String buildHalfTopic() {\n        return TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC;\n    }\n\n    public static String buildHalfTopicForRocksDB() {\n        return TopicValidator.RMQ_SYS_ROCKSDB_TRANS_HALF_TOPIC;\n    }\n\n    public static String buildConsumerGroup() {\n        return MixAll.CID_SYS_RMQ_TRANS;\n    }\n\n    public static MessageExtBrokerInner buildTransactionalMessageFromHalfMessage(MessageExt msgExt) {\n        final MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n        msgInner.setWaitStoreMsgOK(false);\n        msgInner.setMsgId(msgExt.getMsgId());\n        msgInner.setTopic(msgExt.getProperty(MessageConst.PROPERTY_REAL_TOPIC));\n        msgInner.setBody(msgExt.getBody());\n        final String realQueueIdStr = msgExt.getProperty(MessageConst.PROPERTY_REAL_QUEUE_ID);\n        if (StringUtils.isNumeric(realQueueIdStr)) {\n            msgInner.setQueueId(Integer.parseInt(realQueueIdStr));\n        }\n        msgInner.setFlag(msgExt.getFlag());\n        msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(msgInner.getTags()));\n        msgInner.setBornTimestamp(msgExt.getBornTimestamp());\n        msgInner.setBornHost(msgExt.getBornHost());\n        msgInner.setTransactionId(msgExt.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));\n\n        MessageAccessor.setProperties(msgInner, msgExt.getProperties());\n        MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED, \"true\");\n        MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET);\n        MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_REAL_QUEUE_ID);\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));\n\n        int sysFlag = msgExt.getSysFlag();\n        sysFlag |= MessageSysFlag.TRANSACTION_PREPARED_TYPE;\n        msgInner.setSysFlag(sysFlag);\n\n        return msgInner;\n    }\n\n    public static long getImmunityTime(String checkImmunityTimeStr, long transactionTimeout) {\n        long checkImmunityTime = 0;\n\n        try {\n            checkImmunityTime = Long.parseLong(checkImmunityTimeStr) * 1000;\n        } catch (Throwable ignored) {\n        }\n\n        //If a custom first check time is set, the minimum check time;\n        //The default check protection period is transactionTimeout\n        if (checkImmunityTime < transactionTimeout) {\n            checkImmunityTime = transactionTimeout;\n        }\n        return checkImmunityTime;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalOpBatchService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.transaction.queue;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class TransactionalOpBatchService extends ServiceThread {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME);\n\n    private BrokerController brokerController;\n    private TransactionalMessageServiceImpl transactionalMessageService;\n\n    private long wakeupTimestamp = 0;\n\n\n    public TransactionalOpBatchService(BrokerController brokerController,\n                                       TransactionalMessageServiceImpl transactionalMessageService) {\n        this.brokerController = brokerController;\n        this.transactionalMessageService = transactionalMessageService;\n    }\n\n    @Override\n    public String getServiceName() {\n        return TransactionalOpBatchService.class.getSimpleName();\n    }\n\n    @Override\n    public void run() {\n        LOGGER.info(\"Start transaction op batch thread!\");\n        long checkInterval = brokerController.getBrokerConfig().getTransactionOpBatchInterval();\n        wakeupTimestamp = System.currentTimeMillis() + checkInterval;\n        while (!this.isStopped()) {\n            long interval = wakeupTimestamp - System.currentTimeMillis();\n            if (interval <= 0) {\n                interval = 0;\n                wakeup();\n            }\n            this.waitForRunning(interval);\n        }\n        LOGGER.info(\"End transaction op batch thread!\");\n    }\n\n    @Override\n    protected void onWaitEnd() {\n        wakeupTimestamp = transactionalMessageService.batchSendOpMessage();\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/transaction/rocksdb/TransactionalMessageRocksDBService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.transaction.rocksdb;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.ArrayBlockingQueue;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.ThreadPoolExecutor.CallerRunsPolicy;\nimport java.util.concurrent.TimeUnit;\nimport io.netty.channel.Channel;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.rocksdb.MessageRocksDBStorage;\nimport org.apache.rocketmq.store.transaction.TransRocksDBRecord;\nimport org.apache.rocketmq.store.transaction.TransMessageRocksDBStore;\nimport static org.apache.rocketmq.store.rocksdb.MessageRocksDBStorage.TRANS_COLUMN_FAMILY;\n\npublic class TransactionalMessageRocksDBService {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME);\n    private static final int MAX_BATCH_SIZE_FROM_ROCKSDB = 2000;\n    private static final int INITIAL = 0, RUNNING = 1, SHUTDOWN = 2;\n    private volatile int state = INITIAL;\n\n    private final MessageRocksDBStorage messageRocksDBStorage;\n    private final TransMessageRocksDBStore transMessageRocksDBStore;\n    private final MessageStore messageStore;\n    private final BrokerController brokerController;\n\n    private TransStatusCheckService transStatusService;\n    private ExecutorService checkTranStatusTaskExecutor;\n\n    public TransactionalMessageRocksDBService(final MessageStore messageStore, final BrokerController brokerController) {\n        this.messageStore = messageStore;\n        this.transMessageRocksDBStore = messageStore.getTransMessageRocksDBStore();\n        this.messageRocksDBStorage = transMessageRocksDBStore.getMessageRocksDBStorage();\n        this.brokerController = brokerController;\n    }\n\n    public void start() {\n        if (this.state == RUNNING) {\n            return;\n        }\n        initService();\n        this.transStatusService.start();\n        this.state = RUNNING;\n        log.info(\"TransactionalMessageRocksDBService start success\");\n    }\n\n    private void initService() {\n        this.transStatusService = new TransStatusCheckService();\n        this.checkTranStatusTaskExecutor = ThreadUtils.newThreadPoolExecutor(\n            brokerController.getBrokerConfig().getTransactionCheckRocksdbCoreThreads(),\n            brokerController.getBrokerConfig().getTransactionCheckRocksdbMaxThreads(),\n            100,\n            TimeUnit.SECONDS,\n            new ArrayBlockingQueue<>(brokerController.getBrokerConfig().getTransactionCheckRocksdbQueueCapacity()),\n            new ThreadFactoryImpl(\"Transaction-rocksdb-msg-check-thread\", brokerController.getBrokerIdentity()),\n            new CallerRunsPolicy());\n    }\n\n    public void shutdown() {\n        if (this.state != RUNNING || this.state == SHUTDOWN) {\n            return;\n        }\n        if (null != this.transStatusService) {\n            this.transStatusService.shutdown();\n        }\n        if (null != this.checkTranStatusTaskExecutor) {\n            this.checkTranStatusTaskExecutor.shutdown();\n        }\n        this.state = SHUTDOWN;\n        log.info(\"TransactionalMessageRocksDBService shutdown success\");\n    }\n\n    private void checkTransStatus() {\n        long count = 0;\n        byte[] lastKey = null;\n        while (true) {\n            try {\n                List<TransRocksDBRecord> trs = messageRocksDBStorage.scanRecordsForTrans(TRANS_COLUMN_FAMILY, MAX_BATCH_SIZE_FROM_ROCKSDB, lastKey);\n                if (CollectionUtils.isEmpty(trs)) {\n                    log.info(\"TransactionalMessageRocksDBService checkTransStatus trs is empty\");\n                    break;\n                }\n                count += trs.size();\n                checkTransRecordsStatus(trs);\n                lastKey = trs.size() >= MAX_BATCH_SIZE_FROM_ROCKSDB ? trs.get(trs.size() - 1).getKeyBytes() : null;\n                if (null == lastKey) {\n                    break;\n                }\n            } catch (Exception e) {\n                log.error(\"TransactionalMessageRocksDBService checkTransStatus error, error: {}, count: {}\", e.getMessage(), count);\n                break;\n            }\n        }\n        log.info(\"TransactionalMessageRocksDBService checkTransStatus count: {}\", count);\n    }\n\n    private void checkTransRecordsStatus(List<TransRocksDBRecord> trs) {\n        if (CollectionUtils.isEmpty(trs)) {\n            log.error(\"TransactionalMessageRocksDBService checkTransRecordsStatus, trs is empty\");\n            return;\n        }\n        try {\n            List<TransRocksDBRecord> updateList = new ArrayList<>();\n            for (TransRocksDBRecord halfRecord : trs) {\n                if (null == halfRecord) {\n                    log.error(\"TransactionalMessageRocksDBService checkTransRecordsStatus, halfRecord is null\");\n                    continue;\n                }\n                try {\n                    if (halfRecord.getCheckTimes() > brokerController.getBrokerConfig().getTransactionCheckMax()) {\n                        halfRecord.setDelete(true);\n                        updateList.add(halfRecord);\n                        log.info(\"TransactionalMessageRocksDBService checkTransRecordsStatus checkTimes > {}, need delete, checkTimes: {}, msgId: {}\", brokerController.getBrokerConfig().getTransactionCheckMax(), halfRecord.getCheckTimes(), halfRecord.getUniqKey());\n                        continue;\n                    }\n                    MessageExt msgExt = transMessageRocksDBStore.getMessage(halfRecord.getOffsetPy(), halfRecord.getSizePy());\n                    if (null == msgExt) {\n                        log.error(\"TransactionalMessageRocksDBService checkTransRecordsStatus, msgExt is null, offsetPy: {}, sizePy: {}\", halfRecord.getOffsetPy(), halfRecord.getSizePy());\n                        halfRecord.setDelete(true);\n                        updateList.add(halfRecord);\n                        continue;\n                    }\n                    if (!isImmunityTimeExpired(msgExt)) {\n                        continue;\n                    }\n                    resolveHalfMsg(msgExt);\n                    halfRecord.setCheckTimes(halfRecord.getCheckTimes() + 1);\n                    if (halfRecord.getCheckTimes() > brokerController.getBrokerConfig().getTransactionCheckMax()) {\n                        halfRecord.setDelete(true);\n                        log.info(\"TransactionalMessageRocksDBService checkTransRecordsStatus checkTimes > {}, need delete, checkTimes: {}, msgId: {}\", brokerController.getBrokerConfig().getTransactionCheckMax(), halfRecord.getCheckTimes(), halfRecord.getUniqKey());\n                    }\n                    updateList.add(halfRecord);\n                } catch (Exception e) {\n                    log.error(\"TransactionalMessageRocksDBService checkTransRecordsStatus error : {}\", e.getMessage());\n                }\n            }\n            if (!CollectionUtils.isEmpty(updateList)) {\n                messageRocksDBStorage.updateRecordsForTrans(TRANS_COLUMN_FAMILY, updateList);\n            }\n        } catch (Exception e) {\n            log.error(\"TransactionalMessageRocksDBService checkTransRecordsStatus error: {}\", e.getMessage());\n        }\n    }\n\n    private boolean isImmunityTimeExpired(MessageExt msgExt) {\n        String immunityTimeStr = msgExt.getUserProperty(MessageConst.PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS);\n        long immunityTime = brokerController.getBrokerConfig().getTransactionTimeOut();\n        if (!StringUtils.isEmpty(immunityTimeStr)) {\n            try {\n                immunityTime = Long.parseLong(immunityTimeStr);\n                immunityTime *= 1000;\n            } catch (Exception e) {\n                log.error(\"parse immunityTimesStr error: {}, msgId: {}\", e.getMessage(), msgExt.getMsgId());\n            }\n        }\n        if ((System.currentTimeMillis() - msgExt.getBornTimestamp()) < immunityTime) {\n            return false;\n        }\n        return true;\n    }\n\n    private String getServiceThreadName() {\n        String brokerIdentifier = \"\";\n        if (TransactionalMessageRocksDBService.this.messageStore instanceof DefaultMessageStore) {\n            DefaultMessageStore messageStore = (DefaultMessageStore) TransactionalMessageRocksDBService.this.messageStore;\n            if (messageStore.getBrokerConfig().isInBrokerContainer()) {\n                brokerIdentifier = messageStore.getBrokerConfig().getIdentifier();\n            }\n        }\n        return brokerIdentifier;\n    }\n\n    private void resolveHalfMsg(final MessageExt msgExt) {\n        if (checkTranStatusTaskExecutor != null) {\n            checkTranStatusTaskExecutor.execute(new Runnable() {\n                @Override\n                public void run() {\n                    try {\n                        sendCheckMessage(msgExt);\n                    } catch (Exception e) {\n                        log.error(\"TransactionalMessageRocksDBService Send check message error: {}, msgId: {}\", e.getMessage(), msgExt.getMsgId());\n                    }\n                }\n            });\n        } else {\n            log.error(\"TransactionalMessageRocksDBService checkTranStatusTaskExecutor not init, msgId: {}\", msgExt.getMsgId());\n        }\n    }\n\n    private void sendCheckMessage(MessageExt msgExt) {\n        if (null == msgExt) {\n            log.info(\"TransactionalMessageRocksDBService sendCheckMessage msgExt is null\");\n            return;\n        }\n        try {\n            CheckTransactionStateRequestHeader checkTransactionStateRequestHeader = new CheckTransactionStateRequestHeader();\n            checkTransactionStateRequestHeader.setTopic(msgExt.getTopic());\n            checkTransactionStateRequestHeader.setCommitLogOffset(msgExt.getCommitLogOffset());\n            checkTransactionStateRequestHeader.setOffsetMsgId(msgExt.getMsgId());\n            checkTransactionStateRequestHeader.setMsgId(MessageClientIDSetter.getUniqID(msgExt));\n            checkTransactionStateRequestHeader.setTransactionId(checkTransactionStateRequestHeader.getMsgId());\n            checkTransactionStateRequestHeader.setTranStateTableOffset(msgExt.getQueueOffset());\n            checkTransactionStateRequestHeader.setBrokerName(brokerController.getBrokerConfig().getBrokerName());\n            msgExt.setTopic(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_TOPIC));\n            msgExt.setQueueId(Integer.parseInt(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_QUEUE_ID)));\n            msgExt.setStoreSize(0);\n            String groupId = msgExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP);\n            Channel channel = brokerController.getProducerManager().getAvailableChannel(groupId);\n            if (channel != null) {\n                brokerController.getBroker2Client().checkProducerTransactionState(groupId, channel, checkTransactionStateRequestHeader, msgExt);\n            } else {\n                log.warn(\"TransactionalMessageRocksDBService checkProducerTransactionState failed, channel is null. groupId: {}, msgId: {}\", groupId, msgExt.getMsgId());\n            }\n        } catch (Exception e) {\n            log.error(\"TransactionalMessageRocksDBService sendCheckMessage error: {}, msgId: {}\", e.getMessage(), msgExt.getMsgId());\n        }\n    }\n\n    private class TransStatusCheckService extends ServiceThread {\n        private final Logger log = TransactionalMessageRocksDBService.log;\n        @Override\n        public String getServiceName() {\n            return getServiceThreadName() + this.getClass().getSimpleName();\n        }\n\n        @Override\n        public void run() {\n            log.info(this.getServiceName() + \" service start\");\n            while (!this.isStopped()) {\n                try {\n                    long begin = System.currentTimeMillis();\n                    checkTransStatus();\n                    log.info(\"TransactionalMessageRocksDBService ScanTransAndStatusCheckService check trans status, check cost: {}\", System.currentTimeMillis() - begin);\n                    waitForRunning(brokerController.getBrokerConfig().getTransactionCheckInterval());\n                } catch (Exception e) {\n                    log.error(\"TransactionalMessageRocksDBService ScanTransAndStatusCheckService error: {}\", e.getMessage());\n                }\n            }\n            log.info(this.getServiceName() + \" service end\");\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/util/HookUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.util;\n\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Optional;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.schedule.ScheduleMessageService;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.common.utils.QueueTypeUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface;\nimport org.apache.rocketmq.store.timer.TimerMessageStore;\n\npublic class HookUtils {\n\n    protected static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n\n    private static final AtomicLong PRINT_TIMES = new AtomicLong(0);\n\n    /**\n     * On Linux: The maximum length for a file name is 255 bytes.\n     * The maximum combined length of both the file name and path name is 4096 bytes.\n     * This length matches the PATH_MAX that is supported by the operating system.\n     * The Unicode representation of a character can occupy several bytes,\n     * so the maximum number of characters that comprises a path and file name can vary.\n     * The actual limitation is the number of bytes in the path and file components,\n     * which might correspond to an equal number of characters.\n     */\n    private static final Integer MAX_TOPIC_LENGTH = 255;\n\n    public static PutMessageResult checkBeforePutMessage(BrokerController brokerController, final MessageExt msg) {\n        if (brokerController.getMessageStore().isShutdown()) {\n            LOG.warn(\"message store has shutdown, so putMessage is forbidden\");\n            return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null);\n        }\n\n        if (!brokerController.getMessageStoreConfig().isDuplicationEnable() && BrokerRole.SLAVE == brokerController.getMessageStoreConfig().getBrokerRole()) {\n            long value = PRINT_TIMES.getAndIncrement();\n            if ((value % 50000) == 0) {\n                LOG.warn(\"message store is in slave mode, so putMessage is forbidden \");\n            }\n\n            return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null);\n        }\n\n        if (!brokerController.getMessageStore().getRunningFlags().isWriteable()) {\n            long value = PRINT_TIMES.getAndIncrement();\n            if ((value % 50000) == 0) {\n                LOG.warn(\"message store is not writeable, so putMessage is forbidden \" + brokerController.getMessageStore().getRunningFlags().getFlagBits());\n            }\n\n            return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null);\n        } else {\n            PRINT_TIMES.set(0);\n        }\n\n        final byte[] topicData = msg.getTopic().getBytes(MessageDecoder.CHARSET_UTF8);\n        boolean retryTopic = msg.getTopic() != null && msg.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX);\n        if (!retryTopic && topicData.length > Byte.MAX_VALUE) {\n            LOG.warn(\"putMessage message topic[{}] length too long {}, but it is not supported by broker\",\n                msg.getTopic(), topicData.length);\n            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null);\n        }\n\n        if (topicData.length > MAX_TOPIC_LENGTH) {\n            LOG.warn(\"putMessage message topic[{}] length too long {}, but it is not supported by broker\",\n                msg.getTopic(), topicData.length);\n            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null);\n        }\n\n        if (msg.getBody() == null) {\n            LOG.warn(\"putMessage message topic[{}], but message body is null\", msg.getTopic());\n            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null);\n        }\n\n        if (brokerController.getMessageStore().isOSPageCacheBusy()) {\n            return new PutMessageResult(PutMessageStatus.OS_PAGE_CACHE_BUSY, null);\n        }\n        return null;\n    }\n\n    public static PutMessageResult checkInnerBatch(BrokerController brokerController, final MessageExt msg) {\n        if (msg.getProperties().containsKey(MessageConst.PROPERTY_INNER_NUM)\n            && !MessageSysFlag.check(msg.getSysFlag(), MessageSysFlag.INNER_BATCH_FLAG)) {\n            LOG.warn(\"[BUG]The message had property {} but is not an inner batch\", MessageConst.PROPERTY_INNER_NUM);\n            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null);\n        }\n\n        if (MessageSysFlag.check(msg.getSysFlag(), MessageSysFlag.INNER_BATCH_FLAG)) {\n            Optional<TopicConfig> topicConfig = Optional.ofNullable(brokerController.getTopicConfigManager().getTopicConfigTable().get(msg.getTopic()));\n            if (!QueueTypeUtils.isBatchCq(topicConfig)) {\n                LOG.error(\"[BUG]The message is an inner batch but cq type is not batch cq\");\n                return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null);\n            }\n        }\n\n        return null;\n    }\n\n    public static PutMessageResult handleScheduleMessage(BrokerController brokerController,\n        final MessageExtBrokerInner msg) {\n        final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag());\n        if (tranType == MessageSysFlag.TRANSACTION_NOT_TYPE\n            || tranType == MessageSysFlag.TRANSACTION_COMMIT_TYPE) {\n            if (!isRolledTimerMessage(msg)) {\n                if (checkIfTimerMessage(msg)) {\n                    if (!brokerController.getMessageStoreConfig().isTimerWheelEnable()) {\n                        //wheel timer is not enabled, reject the message\n                        return new PutMessageResult(PutMessageStatus.WHEEL_TIMER_NOT_ENABLE, null);\n                    }\n                    PutMessageResult transformRes = transformTimerMessage(brokerController, msg);\n                    if (null != transformRes) {\n                        return transformRes;\n                    }\n                }\n            }\n            // Delay Delivery\n            if (msg.getDelayTimeLevel() > 0) {\n                transformDelayLevelMessage(brokerController, msg);\n            }\n        }\n        return null;\n    }\n\n    public static PutMessageResult handleLmqQuota(BrokerController brokerController, final MessageExtBrokerInner msg) {\n        if (!brokerController.getMessageStoreConfig().isEnableLmqQuota()\n            || !brokerController.getMessageStoreConfig().isEnableLmq()\n            || !brokerController.getMessageStoreConfig().isEnableMultiDispatch()\n            || !msg.needDispatchLMQ()) {\n            return null;\n        }\n\n        ConsumeQueueStoreInterface cqStore = brokerController.getMessageStore().getQueueStore();\n        String[] queueNames =\n            msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH).split(MixAll.LMQ_DISPATCH_SEPARATOR);\n        for (String queueName : queueNames) {\n            if (!MixAll.isLmq(queueName)) {\n                continue;\n            }\n            if (cqStore.getLmqNum() >= brokerController.getMessageStoreConfig().getMaxLmqConsumeQueueNum()) {\n                if (!cqStore.isLmqExist(queueName)) {\n                    return new PutMessageResult(PutMessageStatus.LMQ_CONSUME_QUEUE_NUM_EXCEEDED, null);\n                }\n            }\n        }\n        return null;\n    }\n\n    private static boolean isRolledTimerMessage(MessageExtBrokerInner msg) {\n        return TimerMessageStore.TIMER_TOPIC.equals(msg.getTopic());\n    }\n\n    public static boolean checkIfTimerMessage(MessageExtBrokerInner msg) {\n        if (msg.getDelayTimeLevel() > 0) {\n            if (null != msg.getProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS)) {\n                MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_TIMER_DELIVER_MS);\n            }\n            if (null != msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC)) {\n                MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_TIMER_DELAY_SEC);\n            }\n            if (null != msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_MS)) {\n                MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_TIMER_DELAY_MS);\n            }\n            return false;\n            //return this.defaultMessageStore.getMessageStoreConfig().isTimerInterceptDelayLevel();\n        }\n        //double check\n        if (TimerMessageStore.TIMER_TOPIC.equals(msg.getTopic()) || null != msg.getProperty(MessageConst.PROPERTY_TIMER_OUT_MS)) {\n            return false;\n        }\n        return null != msg.getProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS) || null != msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_MS) || null != msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC);\n    }\n\n    private static PutMessageResult transformTimerMessage(BrokerController brokerController,\n        MessageExtBrokerInner msg) {\n        //do transform\n        int delayLevel = msg.getDelayTimeLevel();\n        long deliverMs;\n        try {\n            if (msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC) != null) {\n                deliverMs = System.currentTimeMillis() + Long.parseLong(msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC)) * 1000;\n            } else if (msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_MS) != null) {\n                deliverMs = System.currentTimeMillis() + Long.parseLong(msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_MS));\n            } else {\n                deliverMs = Long.parseLong(msg.getProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS));\n            }\n        } catch (Exception e) {\n            return new PutMessageResult(PutMessageStatus.WHEEL_TIMER_MSG_ILLEGAL, null);\n        }\n        if (deliverMs > System.currentTimeMillis()) {\n            if (delayLevel <= 0 && deliverMs - System.currentTimeMillis() > brokerController.getMessageStoreConfig().getTimerMaxDelaySec() * 1000L) {\n                return new PutMessageResult(PutMessageStatus.WHEEL_TIMER_MSG_ILLEGAL, null);\n            }\n\n            int timerPrecisionMs = brokerController.getMessageStoreConfig().getTimerPrecisionMs();\n            if (deliverMs % timerPrecisionMs == 0) {\n                deliverMs -= timerPrecisionMs;\n            } else {\n                deliverMs = deliverMs / timerPrecisionMs * timerPrecisionMs;\n            }\n\n            if (brokerController.getTimerMessageStore().isReject(deliverMs)) {\n                return new PutMessageResult(PutMessageStatus.WHEEL_TIMER_FLOW_CONTROL, null);\n            }\n            MessageAccessor.putProperty(msg, MessageConst.PROPERTY_TIMER_OUT_MS, deliverMs + \"\");\n            MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, msg.getTopic());\n            MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_QUEUE_ID, String.valueOf(msg.getQueueId()));\n            msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n            msg.setTopic(TimerMessageStore.TIMER_TOPIC);\n            msg.setQueueId(0);\n        } else if (null != msg.getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY)) {\n            return new PutMessageResult(PutMessageStatus.WHEEL_TIMER_MSG_ILLEGAL, null);\n        }\n        return null;\n    }\n\n    public static void transformDelayLevelMessage(BrokerController brokerController, MessageExtBrokerInner msg) {\n\n        if (msg.getDelayTimeLevel() > brokerController.getScheduleMessageService().getMaxDelayLevel()) {\n            msg.setDelayTimeLevel(brokerController.getScheduleMessageService().getMaxDelayLevel());\n        }\n\n        // Backup real topic, queueId\n        MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, msg.getTopic());\n        MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_QUEUE_ID, String.valueOf(msg.getQueueId()));\n        msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n\n        msg.setTopic(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC);\n        msg.setQueueId(ScheduleMessageService.delayLevel2QueueId(msg.getDelayTimeLevel()));\n    }\n\n    public static boolean sendMessageBack(BrokerController brokerController, List<MessageExt> msgList,\n        String brokerName, String brokerAddr) {\n        try {\n            Iterator<MessageExt> it = msgList.iterator();\n            while (it.hasNext()) {\n                MessageExt msg = it.next();\n                msg.setWaitStoreMsgOK(false);\n                brokerController.getBrokerOuterAPI().sendMessageToSpecificBroker(brokerAddr, brokerName, msg, \"InnerSendMessageBackGroup\", 3000);\n                it.remove();\n            }\n        } catch (Exception e) {\n            LOG.error(\"send message back to broker {} addr {} failed\", brokerName, brokerAddr, e);\n            return false;\n        }\n        return true;\n    }\n}\n"
  },
  {
    "path": "broker/src/main/java/org/apache/rocketmq/broker/util/PositiveAtomicCounter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.util;\n\nimport java.util.concurrent.atomic.AtomicInteger;\n\npublic class PositiveAtomicCounter {\n    private static final int MASK = 0x7FFFFFFF;\n    private final AtomicInteger atom;\n\n\n    public PositiveAtomicCounter() {\n        atom = new AtomicInteger(0);\n    }\n\n\n    public final int incrementAndGet() {\n        final int rt = atom.incrementAndGet();\n        return rt & MASK;\n    }\n\n\n    public int intValue() {\n        return atom.intValue();\n    }\n}\n"
  },
  {
    "path": "broker/src/main/resources/rmq.broker.logback.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n<configuration scan=\"true\" scanPeriod=\"30 seconds\">\n\n    <appender name=\"DefaultSiftingAppender_inner\" class=\"ch.qos.logback.classic.sift.SiftingAppender\">\n        <discriminator>\n            <key>brokerContainerLogDir</key>\n            <defaultValue>${file.separator}</defaultValue>\n        </discriminator>\n        <sift>\n            <appender name=\"DefaultAppender\"\n                      class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n                <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}broker_default.log</file>\n                <append>true</append>\n                <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n                    <fileNamePattern>\n                        ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}broker_default.%i.log.gz\n                    </fileNamePattern>\n                    <minIndex>1</minIndex>\n                    <maxIndex>10</maxIndex>\n                </rollingPolicy>\n                <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n                    <maxFileSize>100MB</maxFileSize>\n                </triggeringPolicy>\n                <encoder>\n                    <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n                    <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n                </encoder>\n            </appender>\n        </sift>\n    </appender>\n    <appender name=\"DefaultSiftingAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"DefaultSiftingAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqBrokerSiftingAppender_inner\" class=\"ch.qos.logback.classic.sift.SiftingAppender\">\n        <discriminator>\n            <key>brokerContainerLogDir</key>\n            <defaultValue>${file.separator}</defaultValue>\n        </discriminator>\n        <sift>\n            <appender name=\"RocketmqBrokerAppender\"\n                      class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n                <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}broker.log</file>\n                <append>true</append>\n                <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n                    <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}broker.%i.log.gz\n                    </fileNamePattern>\n                    <minIndex>1</minIndex>\n                    <maxIndex>20</maxIndex>\n                </rollingPolicy>\n                <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n                    <maxFileSize>128MB</maxFileSize>\n                </triggeringPolicy>\n                <encoder>\n                    <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n                    <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n                </encoder>\n            </appender>\n        </sift>\n    </appender>\n    <appender name=\"RocketmqBrokerSiftingAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqBrokerSiftingAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqProtectionSiftingAppender_inner\" class=\"ch.qos.logback.classic.sift.SiftingAppender\">\n        <discriminator>\n            <key>brokerContainerLogDir</key>\n            <defaultValue>${file.separator}</defaultValue>\n        </discriminator>\n        <sift>\n            <appender name=\"RocketmqProtectionAppender\"\n                      class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n                <file>\n                    ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}protection.log\n                </file>\n                <append>true</append>\n                <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n                    <fileNamePattern>\n                        ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}protection.%i.log.gz\n                    </fileNamePattern>\n                    <minIndex>1</minIndex>\n                    <maxIndex>10</maxIndex>\n                </rollingPolicy>\n                <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n                    <maxFileSize>100MB</maxFileSize>\n                </triggeringPolicy>\n                <encoder>\n                    <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} - %m%n</pattern>\n                    <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n                </encoder>\n            </appender>\n        </sift>\n    </appender>\n    <appender name=\"RocketmqProtectionSiftingAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqProtectionSiftingAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqWaterMarkSiftingAppender_inner\" class=\"ch.qos.logback.classic.sift.SiftingAppender\">\n        <discriminator>\n            <key>brokerContainerLogDir</key>\n            <defaultValue>${file.separator}</defaultValue>\n        </discriminator>\n        <sift>\n            <appender name=\"RocketmqWaterMarkAppender\"\n                      class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n                <file>\n                    ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}watermark.log\n                </file>\n                <append>true</append>\n                <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n                    <fileNamePattern>\n                        ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}watermark.%i.log.gz\n                    </fileNamePattern>\n                    <minIndex>1</minIndex>\n                    <maxIndex>10</maxIndex>\n                </rollingPolicy>\n                <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n                    <maxFileSize>100MB</maxFileSize>\n                </triggeringPolicy>\n                <encoder>\n                    <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} - %m%n</pattern>\n                    <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n                </encoder>\n            </appender>\n        </sift>\n    </appender>\n    <appender name=\"RocketmqWaterMarkSiftingAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqWaterMarkSiftingAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqRocksDBSiftingAppender_inner\" class=\"ch.qos.logback.classic.sift.SiftingAppender\">\n        <discriminator>\n            <key>brokerContainerLogDir</key>\n            <defaultValue>${file.separator}</defaultValue>\n        </discriminator>\n        <sift>\n            <appender name=\"RocketmqStoreAppender\"\n                      class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n                <file>\n                    ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}rocksdb.log\n                </file>\n                <append>true</append>\n                <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n                    <fileNamePattern>\n                        ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}rocksdb.%i.log.gz\n                    </fileNamePattern>\n                    <minIndex>1</minIndex>\n                    <maxIndex>10</maxIndex>\n                </rollingPolicy>\n                <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n                    <maxFileSize>128MB</maxFileSize>\n                </triggeringPolicy>\n                <encoder>\n                    <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n                    <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n                </encoder>\n            </appender>\n        </sift>\n    </appender>\n    <appender name=\"RocketmqRocksDBSiftingAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqRocksDBSiftingAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqStoreSiftingAppender_inner\" class=\"ch.qos.logback.classic.sift.SiftingAppender\">\n        <discriminator>\n            <key>brokerContainerLogDir</key>\n            <defaultValue>${file.separator}</defaultValue>\n        </discriminator>\n        <sift>\n            <appender name=\"RocketmqStoreAppender\"\n                      class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n                <file>\n                    ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}store.log\n                </file>\n                <append>true</append>\n                <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n                    <fileNamePattern>\n                        ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}store.%i.log.gz\n                    </fileNamePattern>\n                    <minIndex>1</minIndex>\n                    <maxIndex>10</maxIndex>\n                </rollingPolicy>\n                <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n                    <maxFileSize>128MB</maxFileSize>\n                </triggeringPolicy>\n                <encoder>\n                    <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n                    <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n                </encoder>\n            </appender>\n        </sift>\n    </appender>\n    <appender name=\"RocketmqStoreSiftingAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqStoreSiftingAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqTieredStoreSiftingAppender_inner\" class=\"ch.qos.logback.classic.sift.SiftingAppender\">\n        <discriminator>\n            <key>brokerContainerLogDir</key>\n            <defaultValue>${file.separator}</defaultValue>\n        </discriminator>\n        <sift>\n            <appender name=\"RocketmqTieredStoreAppender\"\n                      class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n                <file>\n                    ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}tiered_store.log\n                </file>\n                <append>true</append>\n                <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n                    <fileNamePattern>\n                        ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}tiered_store.%i.log.gz\n                    </fileNamePattern>\n                    <minIndex>1</minIndex>\n                    <maxIndex>10</maxIndex>\n                </rollingPolicy>\n                <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n                    <maxFileSize>128MB</maxFileSize>\n                </triggeringPolicy>\n                <encoder>\n                    <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n                    <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n                </encoder>\n            </appender>\n        </sift>\n    </appender>\n    <appender name=\"RocketmqTieredStoreSiftingAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqTieredStoreSiftingAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqTrafficSiftingAppender_inner\" class=\"ch.qos.logback.classic.sift.SiftingAppender\">\n        <discriminator>\n            <key>brokerContainerLogDir</key>\n            <defaultValue>${file.separator}</defaultValue>\n        </discriminator>\n        <sift>\n            <appender name=\"RocketmqTrafficAppender\"\n                      class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n                <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}broker_traffic.log</file>\n                <append>true</append>\n                <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n                    <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}broker_traffic.%i.log.gz</fileNamePattern>\n                    <minIndex>1</minIndex>\n                    <maxIndex>10</maxIndex>\n                </rollingPolicy>\n                <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n                    <maxFileSize>100MB</maxFileSize>\n                </triggeringPolicy>\n                <encoder>\n                    <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n                    <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n                </encoder>\n            </appender>\n        </sift>\n    </appender>\n    <appender name=\"RocketmqTrafficSiftingAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqTrafficSiftingAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqRemotingSiftingAppender_inner\" class=\"ch.qos.logback.classic.sift.SiftingAppender\">\n        <discriminator>\n            <key>brokerContainerLogDir</key>\n            <defaultValue>${file.separator}</defaultValue>\n        </discriminator>\n        <sift>\n            <appender name=\"RocketmqRemotingAppender\"\n                      class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n                <file>\n                    ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}remoting.log\n                </file>\n                <append>true</append>\n                <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n                    <fileNamePattern>\n                        ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}remoting.%i.log.gz\n                    </fileNamePattern>\n                    <minIndex>1</minIndex>\n                    <maxIndex>10</maxIndex>\n                </rollingPolicy>\n                <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n                    <maxFileSize>100MB</maxFileSize>\n                </triggeringPolicy>\n                <encoder>\n                    <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n                    <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n                </encoder>\n            </appender>\n        </sift>\n    </appender>\n    <appender name=\"RocketmqRemotingSiftingAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqRemotingSiftingAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqStoreErrorSiftingAppender_inner\" class=\"ch.qos.logback.classic.sift.SiftingAppender\">\n        <discriminator>\n            <key>brokerContainerLogDir</key>\n            <defaultValue>${file.separator}</defaultValue>\n        </discriminator>\n        <sift>\n            <appender name=\"RocketmqStoreErrorAppender\"\n                      class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n                <file>\n                    ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}storeerror.log\n                </file>\n                <append>true</append>\n                <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n                    <fileNamePattern>\n                        ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}storeerror.%i.log.gz\n                    </fileNamePattern>\n                    <minIndex>1</minIndex>\n                    <maxIndex>10</maxIndex>\n                </rollingPolicy>\n                <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n                    <maxFileSize>100MB</maxFileSize>\n                </triggeringPolicy>\n                <encoder>\n                    <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n                    <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n                </encoder>\n            </appender>\n        </sift>\n    </appender>\n    <appender name=\"RocketmqStoreErrorSiftingAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqStoreErrorSiftingAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqTransactionSiftingAppender_inner\" class=\"ch.qos.logback.classic.sift.SiftingAppender\">\n        <discriminator>\n            <key>brokerContainerLogDir</key>\n            <defaultValue>${file.separator}</defaultValue>\n        </discriminator>\n        <sift>\n            <appender name=\"RocketmqTransactionAppender\"\n                      class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n                <file>\n                    ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}transaction.log\n                </file>\n                <append>true</append>\n                <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n                    <fileNamePattern>\n                        ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}transaction.%i.log.gz\n                    </fileNamePattern>\n                    <minIndex>1</minIndex>\n                    <maxIndex>10</maxIndex>\n                </rollingPolicy>\n                <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n                    <maxFileSize>100MB</maxFileSize>\n                </triggeringPolicy>\n                <encoder>\n                    <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n                    <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n                </encoder>\n            </appender>\n        </sift>\n    </appender>\n    <appender name=\"RocketmqTransactionSiftingAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqTransactionSiftingAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqRebalanceLockSiftingAppender_inner\" class=\"ch.qos.logback.classic.sift.SiftingAppender\">\n        <discriminator>\n            <key>brokerContainerLogDir</key>\n            <defaultValue>${file.separator}</defaultValue>\n        </discriminator>\n        <sift>\n            <appender name=\"RocketmqRebalanceLockAppender\"\n                      class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n                <file>\n                    ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}lock.log\n                </file>\n                <append>true</append>\n                <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n                    <fileNamePattern>\n                        ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}lock.%i.log.gz\n                    </fileNamePattern>\n                    <minIndex>1</minIndex>\n                    <maxIndex>5</maxIndex>\n                </rollingPolicy>\n                <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n                    <maxFileSize>100MB</maxFileSize>\n                </triggeringPolicy>\n                <encoder>\n                    <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n                    <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n                </encoder>\n            </appender>\n        </sift>\n    </appender>\n    <appender name=\"RocketmqRebalanceLockSiftingAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqRebalanceLockSiftingAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqFilterSiftingAppender_inner\" class=\"ch.qos.logback.classic.sift.SiftingAppender\">\n        <discriminator>\n            <key>brokerContainerLogDir</key>\n            <defaultValue>${file.separator}</defaultValue>\n        </discriminator>\n        <sift>\n            <appender name=\"RocketmqFilterAppender\"\n                      class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n                <file>\n                    ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}filter.log\n                </file>\n                <append>true</append>\n                <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n                    <fileNamePattern>\n                        ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}filter.%i.log.gz\n                    </fileNamePattern>\n                    <minIndex>1</minIndex>\n                    <maxIndex>10</maxIndex>\n                </rollingPolicy>\n                <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n                    <maxFileSize>100MB</maxFileSize>\n                </triggeringPolicy>\n                <encoder>\n                    <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n                    <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n                </encoder>\n            </appender>\n        </sift>\n    </appender>\n    <appender name=\"RocketmqRebalanceLockSiftingAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqFilterSiftingAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqStatsSiftingAppender_inner\" class=\"ch.qos.logback.classic.sift.SiftingAppender\">\n        <discriminator>\n            <key>brokerContainerLogDir</key>\n            <defaultValue>${file.separator}</defaultValue>\n        </discriminator>\n        <sift>\n            <appender name=\"RocketmqStatsAppender\"\n                      class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n                <file>\n                    ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}stats.log\n                </file>\n                <append>true</append>\n                <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n                    <fileNamePattern>\n                        ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}stats.%i.log.gz\n                    </fileNamePattern>\n                    <minIndex>1</minIndex>\n                    <maxIndex>5</maxIndex>\n                </rollingPolicy>\n                <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n                    <maxFileSize>100MB</maxFileSize>\n                </triggeringPolicy>\n                <encoder>\n                    <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p - %m%n</pattern>\n                    <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n                </encoder>\n            </appender>\n        </sift>\n    </appender>\n    <appender name=\"RocketmqStatsSiftingAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqStatsSiftingAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqCommercialSiftingAppender_inner\" class=\"ch.qos.logback.classic.sift.SiftingAppender\">\n        <discriminator>\n            <key>brokerContainerLogDir</key>\n            <defaultValue>${file.separator}</defaultValue>\n        </discriminator>\n        <sift>\n            <appender name=\"RocketmqCommercialAppender\"\n                      class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n                <file>\n                    ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}commercial.log\n                </file>\n                <append>true</append>\n                <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n                    <fileNamePattern>\n                        ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}commercial.%i.log.gz\n                    </fileNamePattern>\n                    <minIndex>1</minIndex>\n                    <maxIndex>10</maxIndex>\n                </rollingPolicy>\n                <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n                    <maxFileSize>500MB</maxFileSize>\n                </triggeringPolicy>\n            </appender>\n        </sift>\n    </appender>\n    <appender name=\"RocketmqCommercialSiftingAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqCommercialSiftingAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqPopSiftingAppender_inner\" class=\"ch.qos.logback.classic.sift.SiftingAppender\">\n        <discriminator>\n            <key>brokerContainerLogDir</key>\n            <defaultValue>${file.separator}</defaultValue>\n        </discriminator>\n        <sift>\n            <appender name=\"RocketmqPopAppender\"\n                      class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n                <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}pop.log</file>\n                <append>true</append>\n                <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n                    <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}pop.%i.log\n                    </fileNamePattern>\n                    <minIndex>1</minIndex>\n                    <maxIndex>20</maxIndex>\n                </rollingPolicy>\n                <triggeringPolicy\n                        class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n                    <maxFileSize>128MB</maxFileSize>\n                </triggeringPolicy>\n                <encoder>\n                    <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n                    <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n                </encoder>\n            </appender>\n        </sift>\n    </appender>\n    <appender name=\"RocketmqPopSiftingAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqPopSiftingAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqPopLiteSiftingAppender_inner\" class=\"ch.qos.logback.classic.sift.SiftingAppender\">\n        <discriminator>\n            <key>brokerContainerLogDir</key>\n            <defaultValue>${file.separator}</defaultValue>\n        </discriminator>\n        <sift>\n            <appender name=\"RocketmqPopLiteAppender\"\n                      class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n                <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}lite.log</file>\n                <append>true</append>\n                <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n                    <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}lite.%i.log\n                    </fileNamePattern>\n                    <minIndex>1</minIndex>\n                    <maxIndex>10</maxIndex>\n                </rollingPolicy>\n                <triggeringPolicy\n                        class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n                    <maxFileSize>128MB</maxFileSize>\n                </triggeringPolicy>\n                <encoder>\n                    <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n                    <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n                </encoder>\n            </appender>\n        </sift>\n    </appender>\n    <appender name=\"RocketmqPopLiteSiftingAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqPopLiteSiftingAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqColdCtrSiftingAppender\" class=\"ch.qos.logback.classic.sift.SiftingAppender\">\n        <discriminator>\n            <key>brokerContainerLogDir</key>\n            <defaultValue>${file.separator}</defaultValue>\n        </discriminator>\n        <sift>\n            <appender name=\"RocketmqColdCtrAppender\"\n                      class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n                <file>${user.home}/logs/rocketmqlogs/coldctr.log</file>\n                <append>true</append>\n                <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n                    <fileNamePattern>${user.home}/logs/rocketmqlogs/otherdays/coldctr.%i.log\n                    </fileNamePattern>\n                    <minIndex>1</minIndex>\n                    <maxIndex>10</maxIndex>\n                </rollingPolicy>\n                <triggeringPolicy\n                        class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n                    <maxFileSize>100MB</maxFileSize>\n                </triggeringPolicy>\n                <encoder>\n                    <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n                    <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n                </encoder>\n            </appender>\n        </sift>\n    </appender>\n\n    <appender name=\"RocketmqBrokerMetricsSiftingAppender_inner\" class=\"ch.qos.logback.classic.sift.SiftingAppender\">\n        <discriminator>\n            <key>brokerContainerLogDir</key>\n            <defaultValue>${file.separator}</defaultValue>\n        </discriminator>\n        <sift>\n            <appender name=\"RocketmqBrokerMetricsAppender\"\n                      class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n                <file>\n                    ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}broker_metrics.log\n                </file>\n                <append>true</append>\n                <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n                    <fileNamePattern>\n                        ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}broker_metrics.%i.log.gz\n                    </fileNamePattern>\n                    <minIndex>1</minIndex>\n                    <maxIndex>3</maxIndex>\n                </rollingPolicy>\n                <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n                    <maxFileSize>512MB</maxFileSize>\n                </triggeringPolicy>\n                <encoder>\n                    <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n                    <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n                </encoder>\n            </appender>\n        </sift>\n    </appender>\n    <appender name=\"RocketmqBrokerMetricsSiftingAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqBrokerMetricsSiftingAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqAuthAuditSiftingAppender_inner\" class=\"ch.qos.logback.classic.sift.SiftingAppender\">\n        <discriminator>\n            <key>brokerContainerLogDir</key>\n            <defaultValue>${file.separator}</defaultValue>\n        </discriminator>\n        <sift>\n            <appender name=\"RocketmqAuthAuditAppender\"\n                      class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n                <file>\n                    ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}auth_audit.log\n                </file>\n                <append>true</append>\n                <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n                    <fileNamePattern>\n                        ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}auth_audit.%i.log.gz\n                    </fileNamePattern>\n                    <minIndex>1</minIndex>\n                    <maxIndex>3</maxIndex>\n                </rollingPolicy>\n                <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n                    <maxFileSize>512MB</maxFileSize>\n                </triggeringPolicy>\n                <encoder>\n                    <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n                    <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n                </encoder>\n            </appender>\n        </sift>\n    </appender>\n    <appender name=\"RocketmqAuthAuditSiftingAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqAuthAuditSiftingAppender_inner\"/>\n    </appender>\n\n    <appender name=\"STDOUT\" class=\"ch.qos.logback.core.ConsoleAppender\">\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n\n    <logger name=\"RocketmqBroker\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqBrokerSiftingAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqProtection\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqProtectionSiftingAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqWaterMark\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqWaterMarkSiftingAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqCommon\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqBrokerSiftingAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqRocksDB\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqRocksDBSiftingAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqStore\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqStoreSiftingAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqStoreError\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqStoreErrorSiftingAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqTieredStore\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqTieredStoreSiftingAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqTransaction\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqTransactionSiftingAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqRebalanceLock\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqRebalanceLockSiftingAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqRemoting\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqRemotingSiftingAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqStats\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqStatsSiftingAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqCommercial\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqCommercialSiftingAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqFilter\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqFilterSiftingAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqConsole\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"STDOUT\"/>\n    </logger>\n\n    <logger name=\"RocketmqPop\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqPopSiftingAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqPopLite\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqPopLiteSiftingAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqColdCtr\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqColdCtrSiftingAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqTraffic\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqTrafficSiftingAppender\"/>\n    </logger>\n\n    <!-- Use json formatter to log metrics -->\n    <logger name=\"io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqBrokerMetricsSiftingAppender\"/>\n    </logger>\n\n    <logger name=\"io.opentelemetry.exporter.logging.LoggingMetricExporter\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqBrokerMetricsSiftingAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqAuthAudit\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqAuthAuditSiftingAppender\"/>\n    </logger>\n\n    <root level=\"INFO\">\n        <appender-ref ref=\"DefaultSiftingAppender\"/>\n    </root>\n</configuration>\n"
  },
  {
    "path": "broker/src/main/resources/transaction.sql",
    "content": "--\n-- Licensed to the Apache Software Foundation (ASF) under one\n-- or more contributor license agreements.  See the NOTICE file\n-- distributed with this work for additional information\n-- regarding copyright ownership.  The ASF licenses this file\n-- to you under the Apache License, Version 2.0 (the\n-- \"License\"); you may not use this file except in compliance\n-- with the License.  You may obtain a copy of the License at\n--\n--    http://www.apache.org/licenses/LICENSE-2.0\n--\n-- Unless required by applicable law or agreed to in writing,\n-- software distributed under the License is distributed on an\n-- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n-- KIND, either express or implied.  See the License for the\n-- specific language governing permissions and limitations\n-- under the License.\n--\nCREATE TABLE t_transaction(\n\toffset\t\t\t\tNUMERIC(20) PRIMARY KEY,\n\tproducerGroup\t\tVARCHAR(64)\n)\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker;\n\nimport java.io.File;\nimport java.util.UUID;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.TimeUnit;\n\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.future.FutureTaskExt;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.RemotingServer;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingAbstract;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingServer;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.netty.RequestTask;\nimport org.apache.rocketmq.remoting.pipeline.RequestPipeline;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class BrokerControllerTest {\n\n    private MessageStoreConfig messageStoreConfig;\n\n    private BrokerConfig brokerConfig;\n\n    private NettyServerConfig nettyServerConfig;\n\n\n    @Before\n    public void setUp() {\n        messageStoreConfig = new MessageStoreConfig();\n        String storePathRootDir = System.getProperty(\"java.io.tmpdir\") + File.separator + \"store-\"\n                + UUID.randomUUID().toString();\n        messageStoreConfig.setStorePathRootDir(storePathRootDir);\n\n        brokerConfig = new BrokerConfig();\n\n        nettyServerConfig = new NettyServerConfig();\n        nettyServerConfig.setListenPort(0);\n\n    }\n\n    @Test\n    public void testBrokerRestart() throws Exception {\n        BrokerController brokerController = new BrokerController(brokerConfig, nettyServerConfig, new NettyClientConfig(), messageStoreConfig);\n        assertThat(brokerController.initialize()).isTrue();\n        brokerController.start();\n        brokerController.shutdown();\n    }\n\n    @Test\n    public void testBrokerMetricsManagerInitialization() throws Exception {\n        BrokerController brokerController = new BrokerController(brokerConfig, nettyServerConfig, new NettyClientConfig(), messageStoreConfig);\n        assertThat(brokerController.initialize()).isTrue();\n        // Verify that brokerMetricsManager is properly initialized and not null\n        assertThat(brokerController.getBrokerMetricsManager()).isNotNull();\n        brokerController.shutdown();\n    }\n\n    @After\n    public void destroy() {\n        UtilAll.deleteFile(new File(messageStoreConfig.getStorePathRootDir()));\n    }\n\n    @Test\n    public void testHeadSlowTimeMills() throws Exception {\n        BrokerController brokerController = new BrokerController(brokerConfig, nettyServerConfig, new NettyClientConfig(), messageStoreConfig);\n        brokerController.initialize();\n        BlockingQueue<Runnable> queue = new LinkedBlockingQueue<>();\n\n        //create task is not instance of FutureTaskExt;\n        Runnable runnable = new Runnable() {\n            @Override\n            public void run() {\n\n            }\n        };\n\n        RequestTask requestTask = new RequestTask(runnable, null, null);\n        // the requestTask is not the head of queue;\n        queue.add(new FutureTaskExt<>(requestTask, null));\n\n        long headSlowTimeMills = 100;\n        TimeUnit.MILLISECONDS.sleep(headSlowTimeMills);\n        assertThat(brokerController.headSlowTimeMills(queue)).isGreaterThanOrEqualTo(headSlowTimeMills);\n    }\n\n    @Test\n    public void testCustomRemotingServer() throws CloneNotSupportedException {\n        final RemotingServer mockRemotingServer = new NettyRemotingServer(nettyServerConfig);\n        final String mockRemotingServerName = \"MOCK_REMOTING_SERVER\";\n\n        BrokerController brokerController = new BrokerController(brokerConfig, nettyServerConfig, new NettyClientConfig(), messageStoreConfig);\n        brokerController.setRemotingServerByName(mockRemotingServerName, mockRemotingServer);\n        brokerController.initializeRemotingServer();\n\n        final RPCHook rpcHook = new RPCHook() {\n            @Override\n            public void doBeforeRequest(String remoteAddr, RemotingCommand request) {\n\n            }\n\n            @Override\n            public void doAfterResponse(String remoteAddr, RemotingCommand request, RemotingCommand response) {\n\n            }\n        };\n        brokerController.registerServerRPCHook(rpcHook);\n\n        // setRequestPipelineTest\n        final RequestPipeline requestPipeline = (ctx, request) -> {\n\n        };\n        brokerController.setRequestPipeline(requestPipeline);\n\n        NettyRemotingAbstract tcpRemotingServer = (NettyRemotingAbstract) brokerController.getRemotingServer();\n        Assert.assertTrue(tcpRemotingServer.getRPCHook().contains(rpcHook));\n\n        NettyRemotingAbstract fastRemotingServer = (NettyRemotingAbstract) brokerController.getFastRemotingServer();\n        Assert.assertTrue(fastRemotingServer.getRPCHook().contains(rpcHook));\n\n        NettyRemotingAbstract mockRemotingServer1 = (NettyRemotingAbstract) brokerController.getRemotingServerByName(mockRemotingServerName);\n        Assert.assertTrue(mockRemotingServer1.getRPCHook().contains(rpcHook));\n        Assert.assertSame(mockRemotingServer, mockRemotingServer1);\n    }\n\n    @Test\n    public void testConfigContextMethods() throws Exception {\n        // Test ConfigContext setter and getter methods\n        BrokerController brokerController = new BrokerController(brokerConfig, nettyServerConfig, new NettyClientConfig(), messageStoreConfig);\n        \n        // Initially, ConfigContext should be null\n        assertThat(brokerController.getConfigContext()).isNull();\n        \n        // Create a test ConfigContext\n        ConfigContext configContext = new ConfigContext.Builder()\n            .brokerConfig(brokerConfig)\n            .messageStoreConfig(messageStoreConfig)\n            .nettyServerConfig(nettyServerConfig)\n            .nettyClientConfig(new NettyClientConfig())\n            .authConfig(new AuthConfig())\n            .build();\n        \n        // Set the ConfigContext\n        brokerController.setConfigContext(configContext);\n        \n        // Verify it was set correctly\n        assertThat(brokerController.getConfigContext()).isNotNull();\n        assertThat(brokerController.getConfigContext()).isSameAs(configContext);\n        assertThat(brokerController.getConfigContext().getBrokerConfig()).isSameAs(brokerConfig);\n        assertThat(brokerController.getConfigContext().getMessageStoreConfig()).isSameAs(messageStoreConfig);\n        assertThat(brokerController.getConfigContext().getNettyServerConfig()).isSameAs(nettyServerConfig);\n        \n        // Test setting null ConfigContext\n        brokerController.setConfigContext(null);\n        assertThat(brokerController.getConfigContext()).isNull();\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker;\n\nimport com.google.common.base.Predicate;\nimport com.google.common.collect.Iterables;\nimport com.google.common.collect.Lists;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.lang.reflect.Field;\nimport java.lang.reflect.Method;\nimport java.net.InetSocketAddress;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.Semaphore;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport io.netty.channel.DefaultChannelPromise;\nimport io.netty.util.concurrent.DefaultEventExecutor;\nimport org.apache.commons.lang3.tuple.Triple;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.broker.out.BrokerOuterAPI;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.BrokerIdentity;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingClient;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.netty.ResponseFuture;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.ClusterInfo;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.QueryDataVersionResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.namesrv.RegisterBrokerResult;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Assert;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.ArgumentCaptor;\nimport org.mockito.ArgumentMatchers;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.Mockito;\nimport org.mockito.invocation.InvocationOnMock;\nimport org.mockito.stubbing.Answer;\nimport org.powermock.api.mockito.PowerMockito;\nimport org.powermock.core.classloader.annotations.PrepareForTest;\nimport org.powermock.modules.junit4.PowerMockRunner;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.argThat;\nimport static org.mockito.ArgumentMatchers.isNull;\nimport static org.mockito.AdditionalMatchers.or;\nimport static org.mockito.Mockito.doReturn;\nimport static org.mockito.Mockito.when;\n\n@RunWith(PowerMockRunner.class)\n@PrepareForTest(NettyRemotingClient.class)\npublic class BrokerOuterAPITest {\n    @Mock\n    private ChannelHandlerContext handlerContext;\n    @Spy\n    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());\n    @Mock\n    private MessageStore messageStore;\n    private String clusterName = \"clusterName\";\n    private String brokerName = \"brokerName\";\n    private String brokerAddr = \"brokerAddr\";\n    private long brokerId = 0L;\n    private String nameserver1 = \"127.0.0.1\";\n    private String nameserver2 = \"127.0.0.2\";\n    private String nameserver3 = \"127.0.0.3\";\n    private int timeOut = 3000;\n\n    @Mock\n    private NettyRemotingClient nettyRemotingClient;\n\n    private BrokerOuterAPI brokerOuterAPI;\n\n    public void init() throws Exception {\n        brokerOuterAPI = new BrokerOuterAPI(new NettyClientConfig(), new AuthConfig());\n        Field field = BrokerOuterAPI.class.getDeclaredField(\"remotingClient\");\n        field.setAccessible(true);\n        field.set(brokerOuterAPI, nettyRemotingClient);\n    }\n\n    @Test\n    public void test_needRegister_normal() throws Exception {\n        init();\n        brokerOuterAPI.start();\n        final RemotingCommand response = buildResponse(Boolean.TRUE);\n\n        TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();\n\n        when(nettyRemotingClient.getNameServerAddressList()).thenReturn(Lists.asList(nameserver1, nameserver2, new String[] {nameserver3}));\n        when(nettyRemotingClient.invokeSync(anyString(), any(RemotingCommand.class), anyLong())).thenReturn(response);\n        List<Boolean> booleanList = brokerOuterAPI.needRegister(clusterName, brokerAddr, brokerName, brokerId, topicConfigSerializeWrapper, timeOut, false);\n        assertTrue(booleanList.size() > 0);\n        assertFalse(booleanList.contains(Boolean.FALSE));\n    }\n\n    @Test\n    public void test_needRegister_timeout() throws Exception {\n        if (MixAll.isMac()) {\n            return;\n        }\n        init();\n        brokerOuterAPI.start();\n\n        TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();\n\n        when(nettyRemotingClient.getNameServerAddressList()).thenReturn(Lists.asList(nameserver1, nameserver2, new String[] {nameserver3}));\n\n        when(nettyRemotingClient.invokeSync(anyString(), any(RemotingCommand.class), anyLong())).thenAnswer(new Answer<RemotingCommand>() {\n            @Override\n            public RemotingCommand answer(InvocationOnMock invocation) throws Throwable {\n                if (invocation.getArgument(0) == nameserver1) {\n                    return buildResponse(Boolean.TRUE);\n                } else if (invocation.getArgument(0) == nameserver2) {\n                    return buildResponse(Boolean.FALSE);\n                } else if (invocation.getArgument(0) == nameserver3) {\n                    TimeUnit.MILLISECONDS.sleep(timeOut + 100);  // Increase sleep time to force timeout\n                    return buildResponse(Boolean.TRUE);\n                }\n                return buildResponse(Boolean.TRUE);\n            }\n        });\n        List<Boolean> booleanList = brokerOuterAPI.needRegister(clusterName, brokerAddr, brokerName, brokerId, topicConfigSerializeWrapper, timeOut, false);\n        assertEquals(2, booleanList.size());\n        boolean success = Iterables.any(booleanList,\n            new Predicate<Boolean>() {\n                public boolean apply(Boolean input) {\n                    return input;\n                }\n            });\n\n        assertTrue(success);\n\n    }\n\n    @Test\n    public void test_register_normal() throws Exception {\n        init();\n        brokerOuterAPI.start();\n\n        final RemotingCommand response = RemotingCommand.createResponseCommand(RegisterBrokerResponseHeader.class);\n        final RegisterBrokerResponseHeader responseHeader = (RegisterBrokerResponseHeader) response.readCustomHeader();\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n\n        TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();\n\n        when(nettyRemotingClient.getAvailableNameSrvList()).thenReturn(Lists.asList(nameserver1, nameserver2, new String[] {nameserver3}));\n        when(nettyRemotingClient.invokeSync(anyString(), any(RemotingCommand.class), anyLong())).then(new Answer<RemotingCommand>() {\n            @Override\n            public RemotingCommand answer(InvocationOnMock mock) throws Throwable {\n                RemotingCommand request = mock.getArgument(1);\n                return response;\n            }\n        });\n        List<RegisterBrokerResult> registerBrokerResultList = brokerOuterAPI.registerBrokerAll(clusterName, brokerAddr,\n            brokerName,\n            brokerId,\n            \"hasServerAddr\",\n            topicConfigSerializeWrapper,\n            Lists.newArrayList(),\n            false,\n            timeOut,\n            false,\n            true,\n            new BrokerIdentity());\n\n        assertTrue(registerBrokerResultList.size() > 0);\n    }\n\n    @Test\n    public void test_register_timeout() throws Exception {\n        init();\n        brokerOuterAPI.start();\n\n        final RemotingCommand response = RemotingCommand.createResponseCommand(RegisterBrokerResponseHeader.class);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n\n        TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();\n\n        when(nettyRemotingClient.getAvailableNameSrvList()).thenReturn(Lists.asList(nameserver1, nameserver2, new String[] {nameserver3}));\n        final ArgumentCaptor<Long> timeoutMillisCaptor = ArgumentCaptor.forClass(Long.class);\n        when(nettyRemotingClient.invokeSync(or(ArgumentMatchers.eq(nameserver1), ArgumentMatchers.eq(nameserver2)), any(RemotingCommand.class),\n            timeoutMillisCaptor.capture())).thenReturn(response);\n        when(nettyRemotingClient.invokeSync(ArgumentMatchers.eq(nameserver3), any(RemotingCommand.class), anyLong())).thenThrow(RemotingTimeoutException.class);\n        List<RegisterBrokerResult> registerBrokerResultList = brokerOuterAPI.registerBrokerAll(clusterName, brokerAddr, brokerName, brokerId, \"hasServerAddr\", topicConfigSerializeWrapper, Lists.<String>newArrayList(), false, timeOut, false, true, new BrokerIdentity());\n\n        assertEquals(2, registerBrokerResultList.size());\n    }\n\n    @Test\n    public void testGetBrokerClusterInfo() throws Exception {\n        init();\n        brokerOuterAPI.start();\n\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n\n        ClusterInfo want = new ClusterInfo();\n        want.setBrokerAddrTable(new HashMap<>(Collections.singletonMap(\"key\", new BrokerData(\"cluster\", \"broker\", new HashMap<>(Collections.singletonMap(MixAll.MASTER_ID, \"127.0.0.1:10911\"))))));\n        response.setBody(RemotingSerializable.encode(want));\n\n        when(nettyRemotingClient.invokeSync(isNull(), argThat(argument -> argument.getCode() == RequestCode.GET_BROKER_CLUSTER_INFO), anyLong())).thenReturn(response);\n        ClusterInfo got = brokerOuterAPI.getBrokerClusterInfo();\n\n        assertEquals(want, got);\n    }\n\n    private RemotingCommand buildResponse(Boolean changed) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(QueryDataVersionResponseHeader.class);\n        final QueryDataVersionResponseHeader responseHeader = (QueryDataVersionResponseHeader) response.readCustomHeader();\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        responseHeader.setChanged(changed);\n        return response;\n    }\n\n    @Test\n    public void testLookupAddressByDomain() throws Exception {\n        init();\n        brokerOuterAPI.start();\n        Class<BrokerOuterAPI> clazz = BrokerOuterAPI.class;\n        Method method = clazz.getDeclaredMethod(\"dnsLookupAddressByDomain\", String.class);\n        method.setAccessible(true);\n        List<String> addressList = (List<String>) method.invoke(brokerOuterAPI, \"localhost:6789\");\n        AtomicBoolean result = new AtomicBoolean(false);\n        addressList.forEach(s -> {\n            if (s.contains(\"127.0.0.1:6789\")) {\n                result.set(true);\n            }\n        });\n        Assert.assertTrue(result.get());\n    }\n\n    @Test\n    public void testPullMessageFromSpecificBrokerAsync_createChannel_null() throws Exception {\n        NettyRemotingClient mockClient = PowerMockito.spy(new NettyRemotingClient(new NettyClientConfig()));\n        PowerMockito.when(mockClient, \"getAndCreateChannelAsync\", any()).thenReturn(null);\n        BrokerOuterAPI api = new BrokerOuterAPI(new NettyClientConfig(), new AuthConfig());\n        Field field = BrokerOuterAPI.class.getDeclaredField(\"remotingClient\");\n        field.setAccessible(true);\n        field.set(api, mockClient);\n\n        Triple<PullResult, String, Boolean> rst = api.pullMessageFromSpecificBrokerAsync(\"\", \"\", \"\", \"\", 1, 1, 1, 3000L).join();\n        Assert.assertNull(rst.getLeft());\n        Assert.assertTrue(rst.getMiddle().contains(\"connect\"));\n        Assert.assertTrue(rst.getRight()); // need retry\n    }\n\n    @Test\n    public void testPullMessageFromSpecificBrokerAsync_createChannel_future_notSuccess() throws Exception {\n        NettyRemotingClient mockClient = PowerMockito.spy(new NettyRemotingClient(new NettyClientConfig()));\n        DefaultChannelPromise promise = PowerMockito.spy(new DefaultChannelPromise(PowerMockito.mock(Channel.class), new DefaultEventExecutor()));\n        PowerMockito.when(mockClient, \"getAndCreateChannelAsync\", any()).thenReturn(promise);\n        BrokerOuterAPI api = new BrokerOuterAPI(new NettyClientConfig(), new AuthConfig());\n        Field field = BrokerOuterAPI.class.getDeclaredField(\"remotingClient\");\n        field.setAccessible(true);\n        field.set(api, mockClient);\n\n        promise.tryFailure(new Throwable());\n        Triple<PullResult, String, Boolean> rst\n                = api.pullMessageFromSpecificBrokerAsync(\"\", \"\", \"\", \"\", 1, 1, 1, 3000L).join();\n        Assert.assertNull(rst.getLeft());\n        Assert.assertTrue(rst.getMiddle().contains(\"connect\"));\n        Assert.assertTrue(rst.getRight()); // need retry\n    }\n\n    // skip other future status test\n\n    @Test\n    public void testPullMessageFromSpecificBrokerAsync_timeout() throws Exception {\n        Channel channel = Mockito.mock(Channel.class);\n        when(channel.isActive()).thenReturn(true);\n        NettyRemotingClient mockClient = PowerMockito.spy(new NettyRemotingClient(new NettyClientConfig()));\n        DefaultChannelPromise promise = PowerMockito.spy(new DefaultChannelPromise(PowerMockito.mock(Channel.class), new DefaultEventExecutor()));\n        PowerMockito.when(mockClient, \"getAndCreateChannelAsync\", any()).thenReturn(promise);\n        when(promise.channel()).thenReturn(channel);\n        BrokerOuterAPI api = new BrokerOuterAPI(new NettyClientConfig(), new AuthConfig());\n        Field field = BrokerOuterAPI.class.getDeclaredField(\"remotingClient\");\n        field.setAccessible(true);\n        field.set(api, mockClient);\n\n        CompletableFuture<ResponseFuture> future = new CompletableFuture<>();\n        doReturn(future).when(mockClient).invokeImpl(any(Channel.class), any(RemotingCommand.class), anyLong());\n        promise.trySuccess(null);\n        future.completeExceptionally(new RemotingTimeoutException(\"wait response on the channel timeout\"));\n        Triple<PullResult, String, Boolean> rst = api.pullMessageFromSpecificBrokerAsync(\"\", \"\", \"\", \"\", 1, 1, 1, 3000L).join();\n        Assert.assertNull(rst.getLeft());\n        Assert.assertTrue(rst.getMiddle().contains(\"timeout\"));\n        Assert.assertTrue(rst.getRight()); // need retry\n    }\n\n    @Test\n    public void testPullMessageFromSpecificBrokerAsync_brokerReturn_pullStatusCode() throws Exception {\n        Channel channel = Mockito.mock(Channel.class);\n        when(channel.isActive()).thenReturn(true);\n        NettyRemotingClient mockClient = PowerMockito.spy(new NettyRemotingClient(new NettyClientConfig()));\n        DefaultChannelPromise promise = PowerMockito.spy(new DefaultChannelPromise(PowerMockito.mock(Channel.class), new DefaultEventExecutor()));\n        PowerMockito.when(mockClient, \"getAndCreateChannelAsync\", any()).thenReturn(promise);\n        when(promise.channel()).thenReturn(channel);\n        BrokerOuterAPI api = new BrokerOuterAPI(new NettyClientConfig(), new AuthConfig());\n        Field field = BrokerOuterAPI.class.getDeclaredField(\"remotingClient\");\n        field.setAccessible(true);\n        field.set(api, mockClient);\n\n        int[] respCodes = new int[] {ResponseCode.SUCCESS, ResponseCode.PULL_NOT_FOUND, ResponseCode.PULL_RETRY_IMMEDIATELY, ResponseCode.PULL_OFFSET_MOVED};\n        PullStatus[] respStatus = new PullStatus[] {PullStatus.FOUND, PullStatus.NO_NEW_MSG, PullStatus.NO_MATCHED_MSG, PullStatus.OFFSET_ILLEGAL};\n        for (int i = 0; i < respCodes.length; i++) {\n            CompletableFuture<ResponseFuture> future = new CompletableFuture<>();\n            doReturn(future).when(mockClient).invokeImpl(any(Channel.class), any(RemotingCommand.class), anyLong());\n            RemotingCommand response = mockPullMessageResponse(respCodes[i]);\n            ResponseFuture responseFuture = new ResponseFuture(channel, 0, null, 1000,\n                    resp -> { }, new SemaphoreReleaseOnlyOnce(new Semaphore(1)));\n            responseFuture.setResponseCommand(response);\n            promise.trySuccess(null);\n            future.complete(responseFuture);\n\n            Triple<PullResult, String, Boolean> rst = api.pullMessageFromSpecificBrokerAsync(\"\", \"\", \"\", \"\", 1, 1, 1, 3000L).join();\n            Assert.assertEquals(respStatus[i], rst.getLeft().getPullStatus());\n            if (ResponseCode.SUCCESS == respCodes[i]) {\n                Assert.assertEquals(1, rst.getLeft().getMsgFoundList().size());\n            } else {\n                Assert.assertNull(rst.getLeft().getMsgFoundList());\n            }\n            Assert.assertEquals(respStatus[i].name(), rst.getMiddle());\n            Assert.assertFalse(rst.getRight()); // no retry\n        }\n    }\n\n    @Test\n    public void testPullMessageFromSpecificBrokerAsync_brokerReturn_allOtherResponseCode() throws Exception {\n        Channel channel = Mockito.mock(Channel.class);\n        when(channel.isActive()).thenReturn(true);\n        NettyRemotingClient mockClient = PowerMockito.spy(new NettyRemotingClient(new NettyClientConfig()));\n        DefaultChannelPromise promise = PowerMockito.spy(new DefaultChannelPromise(PowerMockito.mock(Channel.class), new DefaultEventExecutor()));\n        PowerMockito.when(mockClient, \"getAndCreateChannelAsync\", any()).thenReturn(promise);\n        when(promise.channel()).thenReturn(channel);\n        BrokerOuterAPI api = new BrokerOuterAPI(new NettyClientConfig(), new AuthConfig());\n        Field field = BrokerOuterAPI.class.getDeclaredField(\"remotingClient\");\n        field.setAccessible(true);\n        field.set(api, mockClient);\n\n        CompletableFuture<ResponseFuture> future = new CompletableFuture<>();\n        doReturn(future).when(mockClient).invokeImpl(any(Channel.class), any(RemotingCommand.class), anyLong());\n        // test one code here, skip others\n        RemotingCommand response = mockPullMessageResponse(ResponseCode.SUBSCRIPTION_NOT_EXIST);\n        ResponseFuture responseFuture = new ResponseFuture(channel, 0, null, 1000,\n                resp -> { }, new SemaphoreReleaseOnlyOnce(new Semaphore(1)));\n        responseFuture.setResponseCommand(response);\n        promise.trySuccess(null);\n        future.complete(responseFuture);\n\n        Triple<PullResult, String, Boolean> rst = api.pullMessageFromSpecificBrokerAsync(\"\", \"\", \"\", \"\", 1, 1, 1, 3000L).join();\n        Assert.assertNull(rst.getLeft());\n        Assert.assertTrue(rst.getMiddle().contains(ResponseCode.SUBSCRIPTION_NOT_EXIST + \"\"));\n        Assert.assertTrue(rst.getRight()); // need retry\n    }\n\n    private RemotingCommand mockPullMessageResponse(int responseCode) throws Exception {\n        RemotingCommand response = RemotingCommand.createResponseCommand(PullMessageResponseHeader.class);\n        response.setCode(responseCode);\n        if (responseCode == ResponseCode.SUCCESS) {\n            MessageExt msg = new MessageExt();\n            msg.setBody(\"HW\".getBytes());\n            msg.setTopic(\"topic\");\n            msg.setBornHost(new InetSocketAddress(\"127.0.0.1\", 9000));\n            msg.setStoreHost(new InetSocketAddress(\"127.0.0.1\", 9000));\n            byte[] encode = MessageDecoder.encode(msg, false);\n            response.setBody(encode);\n        }\n        PullMessageResponseHeader responseHeader = (PullMessageResponseHeader) response.readCustomHeader();\n        responseHeader.setNextBeginOffset(0L);\n        responseHeader.setMaxOffset(0L);\n        responseHeader.setMinOffset(0L);\n        responseHeader.setOffsetDelta(0L);\n        responseHeader.setTopicSysFlag(0);\n        responseHeader.setGroupSysFlag(0);\n        responseHeader.setSuggestWhichBrokerId(0L);\n        responseHeader.setForbiddenType(0);\n        response.makeCustomHeaderToNet();\n        return response;\n    }\n\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/BrokerPathConfigHelperTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker;\n\nimport java.io.File;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\n\npublic class BrokerPathConfigHelperTest {\n\n    @Test\n    public void testGetPath() {\n        String lmqConsumerOffsetPath = BrokerPathConfigHelper.getLmqConsumerOffsetPath(\"/home/admin/store\".replace(\"/\", File.separator));\n        assertEquals(\"/home/admin/store/config/lmqConsumerOffset.json\".replace(\"/\", File.separator), lmqConsumerOffsetPath);\n\n        String consumerOffsetPath = BrokerPathConfigHelper.getConsumerOffsetPath(\"/home/admin/store\".replace(\"/\", File.separator));\n        assertEquals(\"/home/admin/store/config/consumerOffset.json\".replace(\"/\", File.separator), consumerOffsetPath);\n\n        String topicConfigPath = BrokerPathConfigHelper.getTopicConfigPath(\"/home/admin/store\".replace(\"/\", File.separator));\n        assertEquals(\"/home/admin/store/config/topics.json\".replace(\"/\", File.separator), topicConfigPath);\n\n        String subscriptionGroupPath = BrokerPathConfigHelper.getSubscriptionGroupPath(\"/home/admin/store\".replace(\"/\", File.separator));\n        assertEquals(\"/home/admin/store/config/subscriptionGroup.json\".replace(\"/\", File.separator), subscriptionGroupPath);\n\n        String topicQueueMappingPath = BrokerPathConfigHelper.getTopicQueueMappingPath(\"/home/admin/store\".replace(\"/\", File.separator));\n        assertEquals(\"/home/admin/store/config/topicQueueMapping.json\".replace(\"/\", File.separator), topicQueueMappingPath);\n\n        String consumerOrderInfoPath = BrokerPathConfigHelper.getConsumerOrderInfoPath(\"/home/admin/store\".replace(\"/\", File.separator));\n        assertEquals(\"/home/admin/store/config/consumerOrderInfo.json\".replace(\"/\", File.separator), consumerOrderInfoPath);\n\n        String timercheckPath = BrokerPathConfigHelper.getTimerCheckPath(\"/home/admin/store\".replace(\"/\", File.separator));\n        assertEquals(\"/home/admin/store/config/timercheck\".replace(\"/\", File.separator), timercheckPath);\n\n        String timermetricsPath = BrokerPathConfigHelper.getTimerMetricsPath(\"/home/admin/store\".replace(\"/\", File.separator));\n        assertEquals(\"/home/admin/store/config/timermetrics\".replace(\"/\", File.separator), timermetricsPath);\n\n        String transactionMetricsPath = BrokerPathConfigHelper.getTransactionMetricsPath(\"/home/admin/store\".replace(\"/\", File.separator));\n        assertEquals(\"/home/admin/store/config/transactionMetrics\".replace(\"/\", File.separator), transactionMetricsPath);\n\n        String consumerFilterPath = BrokerPathConfigHelper.getConsumerFilterPath(\"/home/admin/store\".replace(\"/\", File.separator));\n        assertEquals(\"/home/admin/store/config/consumerFilter.json\".replace(\"/\", File.separator), consumerFilterPath);\n\n        String messageRequestModePath = BrokerPathConfigHelper.getMessageRequestModePath(\"/home/admin/store\".replace(\"/\", File.separator));\n        assertEquals(\"/home/admin/store/config/messageRequestMode.json\".replace(\"/\", File.separator), messageRequestModePath);\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/BrokerShutdownTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker;\n\nimport java.io.File;\nimport java.util.UUID;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicBoolean;\n\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class BrokerShutdownTest {\n\n    private MessageStoreConfig messageStoreConfig;\n    private BrokerConfig brokerConfig;\n    private NettyServerConfig nettyServerConfig;\n    private AuthConfig authConfig;\n\n    @Before\n    public void setUp() {\n        messageStoreConfig = new MessageStoreConfig();\n        String storePathRootDir = System.getProperty(\"java.io.tmpdir\") + File.separator + \"store-\"\n                + UUID.randomUUID().toString();\n        messageStoreConfig.setStorePathRootDir(storePathRootDir);\n\n        brokerConfig = new BrokerConfig();\n        nettyServerConfig = new NettyServerConfig();\n        nettyServerConfig.setListenPort(0);\n        authConfig = new AuthConfig();\n    }\n\n    @After\n    public void destroy() {\n        UtilAll.deleteFile(new File(messageStoreConfig.getStorePathRootDir()));\n    }\n\n    @Test\n    public void testBrokerGracefulShutdown() throws Exception {\n        // Test that broker shuts down gracefully with proper resource cleanup\n        BrokerController brokerController = new BrokerController(\n            brokerConfig, nettyServerConfig, new NettyClientConfig(), messageStoreConfig, authConfig);\n        \n        // Initialize and start the broker\n        assertThat(brokerController.initialize()).isTrue();\n        brokerController.start();\n        \n        // Verify broker is running\n        assertThat(brokerController.getBrokerMetricsManager()).isNotNull();\n        \n        // Test graceful shutdown\n        long startTime = System.currentTimeMillis();\n        brokerController.shutdown();\n        long shutdownTime = System.currentTimeMillis() - startTime;\n        \n        // Shutdown should complete within reasonable time (10 seconds)\n        assertThat(shutdownTime).isLessThan(40000);\n    }\n\n    @Test\n    public void testChainedShutdownOrdering() throws Exception {\n        // Test that shutdown components are called in proper order\n        BrokerController brokerController = new BrokerController(\n            brokerConfig, nettyServerConfig, new NettyClientConfig(), messageStoreConfig, authConfig);\n        \n        assertThat(brokerController.initialize()).isTrue();\n        \n        // Track shutdown order using atomic flags\n        AtomicBoolean metricsManagerShutdown = new AtomicBoolean(false);\n        AtomicBoolean brokerStatsShutdown = new AtomicBoolean(false);\n        \n        // Start broker\n        brokerController.start();\n        \n        // Verify services are initialized\n        assertThat(brokerController.getBrokerMetricsManager()).isNotNull();\n        assertThat(brokerController.getBrokerStatsManager()).isNotNull();\n        \n        // Shutdown should not throw exceptions\n        brokerController.shutdown();\n        \n        // After shutdown, services should be properly cleaned up\n        // (We can't easily verify the exact order without modifying the implementation,\n        // but we can verify shutdown completes successfully)\n        assertThat(true).isTrue(); // Placeholder for successful completion\n    }\n\n    @Test\n    public void testShutdownWithConcurrentOperations() throws Exception {\n        // Test shutdown behavior when concurrent operations are running\n        BrokerController brokerController = new BrokerController(\n            brokerConfig, nettyServerConfig, new NettyClientConfig(), messageStoreConfig, authConfig);\n        \n        assertThat(brokerController.initialize()).isTrue();\n        brokerController.start();\n        \n        CountDownLatch shutdownLatch = new CountDownLatch(1);\n        AtomicBoolean shutdownSuccess = new AtomicBoolean(false);\n        \n        // Simulate concurrent shutdown from another thread\n        Thread shutdownThread = new Thread(() -> {\n            try {\n                brokerController.shutdown();\n                shutdownSuccess.set(true);\n            } catch (Exception e) {\n                // Should not happen in graceful shutdown\n            } finally {\n                shutdownLatch.countDown();\n            }\n        });\n        \n        shutdownThread.start();\n        \n        // Wait for shutdown to complete\n        assertThat(shutdownLatch.await(40, TimeUnit.SECONDS)).isTrue();\n        assertThat(shutdownSuccess.get()).isTrue();\n    }\n\n    @Test\n    public void testResourceCleanupDuringShutdown() throws Exception {\n        // Test that resources are properly cleaned up during shutdown\n        BrokerController brokerController = new BrokerController(\n            brokerConfig, nettyServerConfig, new NettyClientConfig(), messageStoreConfig, authConfig);\n        \n        assertThat(brokerController.initialize()).isTrue();\n        \n        // Verify essential components are initialized\n        assertThat(brokerController.getBrokerMetricsManager()).isNotNull();\n        assertThat(brokerController.getBrokerStatsManager()).isNotNull();\n        assertThat(brokerController.getConsumerOffsetManager()).isNotNull();\n        assertThat(brokerController.getTopicConfigManager()).isNotNull();\n        \n        brokerController.start();\n        \n        // Shutdown should clean up all resources\n        brokerController.shutdown();\n        \n        // After shutdown, the broker should be in a clean state\n        // We verify this by ensuring a second shutdown call doesn't cause issues\n        brokerController.shutdown(); // Should be safe to call multiple times\n    }\n}"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/BrokerStartupTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker;\n\nimport java.lang.reflect.InvocationTargetException;\nimport java.lang.reflect.Method;\nimport java.util.Properties;\nimport org.apache.rocketmq.common.MixAll;\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class BrokerStartupTest {\n\n    private String storePathRootDir = \".\";\n\n    @Test\n    public void testProperties2SystemEnv() throws NoSuchMethodException, InvocationTargetException,\n        IllegalAccessException {\n        Properties properties = new Properties();\n        Class<BrokerStartup> clazz = BrokerStartup.class;\n        Method method = clazz.getDeclaredMethod(\"properties2SystemEnv\", Properties.class);\n        method.setAccessible(true);\n        {\n            properties.put(\"rmqAddressServerDomain\", \"value1\");\n            properties.put(\"rmqAddressServerSubGroup\", \"value2\");\n            method.invoke(null, properties);\n            Assert.assertEquals(\"value1\", System.getProperty(\"rocketmq.namesrv.domain\"));\n            Assert.assertEquals(\"value2\", System.getProperty(\"rocketmq.namesrv.domain.subgroup\"));\n        }\n        {\n            properties.put(\"rmqAddressServerDomain\", MixAll.WS_DOMAIN_NAME);\n            properties.put(\"rmqAddressServerSubGroup\", MixAll.WS_DOMAIN_SUBGROUP);\n            method.invoke(null, properties);\n            Assert.assertEquals(MixAll.WS_DOMAIN_NAME, System.getProperty(\"rocketmq.namesrv.domain\"));\n            Assert.assertEquals(MixAll.WS_DOMAIN_SUBGROUP, System.getProperty(\"rocketmq.namesrv.domain.subgroup\"));\n        }\n\n\n    }\n}"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/RocksDBConfigManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker;\n\nimport org.apache.rocketmq.broker.config.v1.RocksDBConfigManager;\nimport org.apache.rocketmq.common.config.ConfigRocksDBStorage;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\nimport static org.powermock.api.mockito.PowerMockito.mock;\n\npublic class RocksDBConfigManagerTest {\n\n    private ConfigRocksDBStorage configRocksDBStorage;\n\n    private RocksDBConfigManager rocksDBConfigManager;\n\n    @Before\n    public void setUp() throws IllegalAccessException {\n        configRocksDBStorage = mock(ConfigRocksDBStorage.class);\n        rocksDBConfigManager = spy(new RocksDBConfigManager(\"testPath\", 1000L, null));\n        rocksDBConfigManager.configRocksDBStorage = configRocksDBStorage;\n    }\n\n    @Test\n    public void testLoadDataVersion() throws Exception {\n        DataVersion expected = new DataVersion();\n        expected.nextVersion();\n\n        when(rocksDBConfigManager.getKvDataVersion()).thenReturn(expected);\n\n        boolean result = rocksDBConfigManager.loadDataVersion();\n\n        assertTrue(result);\n        assertEquals(expected.getCounter().get(), rocksDBConfigManager.getKvDataVersion().getCounter().get());\n        assertEquals(expected.getTimestamp(), rocksDBConfigManager.getKvDataVersion().getTimestamp());\n    }\n\n    @Test\n    public void testUpdateKvDataVersion() throws Exception {\n        rocksDBConfigManager.updateKvDataVersion();\n\n        verify(rocksDBConfigManager, times(1)).updateKvDataVersion();\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/client/ConsumerManagerScannerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.client;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelFuture;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ConsumerManagerScannerTest {\n    private ConsumerManager consumerManager;\n    private String group = \"FooBar\";\n    private String clientId = \"clientId\";\n    private ClientChannelInfo clientInfo;\n    private Map<ConsumerGroupEvent, List<ConsumerIdsChangeListenerData>> groupEventListMap = new HashMap<>();\n\n    @Mock\n    private Channel channel;\n\n    @Before\n    public void init() {\n        clientInfo = new ClientChannelInfo(channel, clientId, LanguageCode.JAVA, 0);\n\n        consumerManager = new ConsumerManager(new ConsumerIdsChangeListener() {\n            @Override\n            public void handle(ConsumerGroupEvent event, String group, Object... args) {\n                groupEventListMap.compute(event, (eventKey, dataListVal) -> {\n                    if (dataListVal == null) {\n                        dataListVal = new ArrayList<>();\n                    }\n                    dataListVal.add(new ConsumerIdsChangeListenerData(event, group, args));\n                    return dataListVal;\n                });\n            }\n\n            @Override\n            public void shutdown() {\n\n            }\n        }, 1000 * 120);\n    }\n\n    private static class ConsumerIdsChangeListenerData {\n        private ConsumerGroupEvent event;\n        private String group;\n        private Object[] args;\n\n        public ConsumerIdsChangeListenerData(ConsumerGroupEvent event, String group, Object[] args) {\n            this.event = event;\n            this.group = group;\n            this.args = args;\n        }\n    }\n\n    @Test\n    public void testClientUnregisterEventInDoChannelCloseEvent() {\n        assertThat(consumerManager.registerConsumer(\n            group,\n            clientInfo,\n            ConsumeType.CONSUME_PASSIVELY,\n            MessageModel.CLUSTERING,\n            ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET,\n            new HashSet<>(),\n            false\n        )).isTrue();\n\n        consumerManager.doChannelCloseEvent(\"remoteAddr\", channel);\n\n        assertThat(groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).size()).isEqualTo(1);\n        assertThat(groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).get(0).args[0]).isInstanceOf(ClientChannelInfo.class);\n        ClientChannelInfo clientChannelInfo = (ClientChannelInfo) groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).get(0).args[0];\n        assertThat(clientChannelInfo).isSameAs(clientInfo);\n    }\n\n    @Test\n    public void testClientUnregisterEventInUnregisterConsumer() {\n        assertThat(consumerManager.registerConsumer(\n            group,\n            clientInfo,\n            ConsumeType.CONSUME_PASSIVELY,\n            MessageModel.CLUSTERING,\n            ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET,\n            new HashSet<>(),\n            false\n        )).isTrue();\n\n        consumerManager.unregisterConsumer(group, clientInfo, false);\n\n        assertThat(groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).size()).isEqualTo(1);\n        assertThat(groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).get(0).args[0]).isInstanceOf(ClientChannelInfo.class);\n        ClientChannelInfo clientChannelInfo = (ClientChannelInfo) groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).get(0).args[0];\n        assertThat(clientChannelInfo).isSameAs(clientInfo);\n    }\n\n    @Test\n    public void testClientUnregisterEventInScanNotActiveChannel() {\n        assertThat(consumerManager.registerConsumer(\n            group,\n            clientInfo,\n            ConsumeType.CONSUME_PASSIVELY,\n            MessageModel.CLUSTERING,\n            ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET,\n            new HashSet<>(),\n            false\n        )).isTrue();\n        clientInfo.setLastUpdateTimestamp(0);\n        when(channel.close()).thenReturn(mock(ChannelFuture.class));\n\n        consumerManager.scanNotActiveChannel();\n        assertThat(groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).size()).isEqualTo(1);\n        assertThat(groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).get(0).args[0]).isInstanceOf(ClientChannelInfo.class);\n        ClientChannelInfo clientChannelInfo = (ClientChannelInfo) groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).get(0).args[0];\n        assertThat(clientChannelInfo).isSameAs(clientInfo);\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/client/ConsumerManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.client;\n\nimport com.google.common.collect.ImmutableSet;\nimport io.netty.channel.Channel;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.client.net.Broker2Client;\nimport org.apache.rocketmq.broker.filter.ConsumerFilterManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.util.HashSet;\nimport java.util.Set;\n\nimport static org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType.CONSUME_PASSIVELY;\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.never;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ConsumerManagerTest {\n\n    private ClientChannelInfo clientChannelInfo;\n\n    @Mock\n    private Channel channel;\n\n    private ConsumerManager consumerManager;\n\n    @Mock\n    private BrokerController brokerController;\n\n    private final BrokerConfig brokerConfig = new BrokerConfig();\n\n    private static final String GROUP = \"DEFAULT_GROUP\";\n\n    private static final String CLIENT_ID = \"1\";\n\n    private static final int VERSION = 1;\n\n    private static final String TOPIC = \"DEFAULT_TOPIC\";\n\n    @Before\n    public void before() {\n        clientChannelInfo = new ClientChannelInfo(channel, CLIENT_ID, LanguageCode.JAVA, VERSION);\n        DefaultConsumerIdsChangeListener defaultConsumerIdsChangeListener = new DefaultConsumerIdsChangeListener(brokerController);\n        BrokerStatsManager brokerStatsManager = new BrokerStatsManager(brokerConfig);\n        consumerManager = spy(new ConsumerManager(defaultConsumerIdsChangeListener, brokerStatsManager, brokerConfig));\n        ConsumerFilterManager consumerFilterManager = mock(ConsumerFilterManager.class);\n        when(brokerController.getConsumerFilterManager()).thenReturn(consumerFilterManager);\n    }\n\n    @Test\n    public void compensateBasicConsumerInfoTest() {\n        ConsumerGroupInfo consumerGroupInfo = consumerManager.getConsumerGroupInfo(GROUP, true);\n        assertThat(consumerGroupInfo).isNull();\n\n        consumerManager.compensateBasicConsumerInfo(GROUP, ConsumeType.CONSUME_ACTIVELY, MessageModel.BROADCASTING);\n        consumerGroupInfo = consumerManager.getConsumerGroupInfo(GROUP, true);\n        assertThat(consumerGroupInfo).isNotNull();\n        assertThat(consumerGroupInfo.getConsumeType()).isEqualTo(ConsumeType.CONSUME_ACTIVELY);\n        assertThat(consumerGroupInfo.getMessageModel()).isEqualTo(MessageModel.BROADCASTING);\n    }\n\n    @Test\n    public void compensateSubscribeDataTest() {\n        ConsumerGroupInfo consumerGroupInfo = consumerManager.getConsumerGroupInfo(GROUP, true);\n        assertThat(consumerGroupInfo).isNull();\n\n        consumerManager.compensateSubscribeData(GROUP, TOPIC, new SubscriptionData(TOPIC, SubscriptionData.SUB_ALL));\n        consumerGroupInfo = consumerManager.getConsumerGroupInfo(GROUP, true);\n        assertThat(consumerGroupInfo).isNotNull();\n        assertThat(consumerGroupInfo.getSubscriptionTable().size()).isEqualTo(1);\n        SubscriptionData subscriptionData = consumerGroupInfo.getSubscriptionTable().get(TOPIC);\n        assertThat(subscriptionData).isNotNull();\n        assertThat(subscriptionData.getTopic()).isEqualTo(TOPIC);\n        assertThat(subscriptionData.getSubString()).isEqualTo(SubscriptionData.SUB_ALL);\n    }\n\n    @Test\n    public void registerConsumerTest() {\n        register();\n        final Set<SubscriptionData> subList = new HashSet<>();\n        SubscriptionData subscriptionData = new SubscriptionData(TOPIC, \"*\");\n        subList.add(subscriptionData);\n        consumerManager.registerConsumer(GROUP, clientChannelInfo, CONSUME_PASSIVELY,\n            MessageModel.BROADCASTING, ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET, subList, true);\n        verify(consumerManager, never()).callConsumerIdsChangeListener(eq(ConsumerGroupEvent.CHANGE), any(), any());\n        assertThat(consumerManager.getConsumerTable().get(GROUP)).isNotNull();\n    }\n\n    @Test\n    public void unregisterConsumerTest() {\n        // register\n        register();\n\n        // unregister\n        consumerManager.unregisterConsumer(GROUP, clientChannelInfo, true);\n        verify(consumerManager, never()).callConsumerIdsChangeListener(eq(ConsumerGroupEvent.CHANGE), any(), any());\n        assertThat(consumerManager.getConsumerTable().get(GROUP)).isNull();\n    }\n\n    @Test\n    public void findChannelTest() {\n        register();\n        final ClientChannelInfo consumerManagerChannel = consumerManager.findChannel(GROUP, CLIENT_ID);\n        assertThat(consumerManagerChannel).isNotNull();\n    }\n\n    @Test\n    public void findSubscriptionDataTest() {\n        register();\n        final SubscriptionData subscriptionData = consumerManager.findSubscriptionData(GROUP, TOPIC);\n        assertThat(subscriptionData).isNotNull();\n    }\n\n    @Test\n    public void findSubscriptionDataCountTest() {\n        register();\n        final int count = consumerManager.findSubscriptionDataCount(GROUP);\n        assertTrue(count > 0);\n    }\n\n    @Test\n    public void findSubscriptionTest() {\n        SubscriptionData subscriptionData = consumerManager.findSubscriptionData(GROUP, TOPIC, true);\n        assertThat(subscriptionData).isNull();\n\n        consumerManager.compensateSubscribeData(GROUP, TOPIC, new SubscriptionData(TOPIC, SubscriptionData.SUB_ALL));\n        subscriptionData = consumerManager.findSubscriptionData(GROUP, TOPIC, true);\n        assertThat(subscriptionData).isNotNull();\n        assertThat(subscriptionData.getTopic()).isEqualTo(TOPIC);\n        assertThat(subscriptionData.getSubString()).isEqualTo(SubscriptionData.SUB_ALL);\n\n        subscriptionData = consumerManager.findSubscriptionData(GROUP, TOPIC, false);\n        assertThat(subscriptionData).isNull();\n    }\n\n    @Test\n    public void scanNotActiveChannelTest() {\n        clientChannelInfo.setLastUpdateTimestamp(System.currentTimeMillis() - brokerConfig.getChannelExpiredTimeout() * 2);\n        consumerManager.scanNotActiveChannel();\n        assertThat(consumerManager.getConsumerTable().size()).isEqualTo(0);\n    }\n\n    @Test\n    public void queryTopicConsumeByWhoTest() {\n        register();\n        final HashSet<String> consumeGroup = consumerManager.queryTopicConsumeByWho(TOPIC);\n        assertFalse(consumeGroup.isEmpty());\n        assertThat(consumerManager.queryTopicConsumeByWho(TOPIC)).isEqualTo(ImmutableSet.of(GROUP));\n    }\n\n    @Test\n    public void doChannelCloseEventTest() {\n        consumerManager.doChannelCloseEvent(\"127.0.0.1\", channel);\n        verify(consumerManager, never()).callConsumerIdsChangeListener(eq(ConsumerGroupEvent.CHANGE), any(), any());\n        assertEquals(0, consumerManager.getConsumerTable().size());\n    }\n\n    private void register() {\n        // register\n        final Set<SubscriptionData> subList = new HashSet<>();\n        SubscriptionData subscriptionData = new SubscriptionData(TOPIC, \"*\");\n        subList.add(subscriptionData);\n        consumerManager.registerConsumer(GROUP, clientChannelInfo, CONSUME_PASSIVELY,\n            MessageModel.BROADCASTING, ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET, subList, true);\n    }\n\n    @Test\n    public void removeExpireConsumerGroupInfo() {\n        SubscriptionData subscriptionData = new SubscriptionData(TOPIC, SubscriptionData.SUB_ALL);\n        subscriptionData.setSubVersion(System.currentTimeMillis() - brokerConfig.getSubscriptionExpiredTimeout() * 2);\n        consumerManager.compensateSubscribeData(GROUP, TOPIC, subscriptionData);\n        consumerManager.compensateSubscribeData(GROUP, TOPIC + \"_1\", new SubscriptionData(TOPIC, SubscriptionData.SUB_ALL));\n        consumerManager.removeExpireConsumerGroupInfo();\n        assertThat(consumerManager.getConsumerGroupInfo(GROUP, true)).isNotNull();\n        assertThat(consumerManager.findSubscriptionData(GROUP, TOPIC)).isNull();\n        assertThat(consumerManager.findSubscriptionData(GROUP, TOPIC + \"_1\")).isNotNull();\n    }\n    \n    @Test\n    public void testRegisterConsumerWithoutSub() {\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        Broker2Client broker2Client = mock(Broker2Client.class);\n        when(brokerController.getBroker2Client()).thenReturn(broker2Client);\n        ConsumerGroupInfo groupInfo = new ConsumerGroupInfo(GROUP, CONSUME_PASSIVELY,\n                MessageModel.CLUSTERING, ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        SubscriptionData subscriptionData = new SubscriptionData(TOPIC, \"*\");\n        groupInfo.getSubscriptionTable().put(TOPIC, subscriptionData);\n        consumerManager.getConsumerTable().put(GROUP, groupInfo);\n        \n        consumerManager.registerConsumerWithoutSub(GROUP,\n                clientChannelInfo,\n                CONSUME_PASSIVELY,\n                MessageModel.CLUSTERING,\n                ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET,\n                true);\n        \n        Set<String> actual = consumerManager.queryTopicConsumeByWho(TOPIC);\n        assertThat(actual).contains(GROUP);\n        assertThat(actual).doesNotContain(TOPIC);\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.client;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelFuture;\nimport java.lang.reflect.Field;\nimport java.util.Map;\n\nimport java.util.concurrent.atomic.AtomicReference;\n\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ProducerManagerTest {\n\n    private BrokerConfig brokerConfig;\n    private ProducerManager producerManager;\n    private String group = \"FooBar\";\n    private ClientChannelInfo clientInfo;\n\n    @Mock\n    private Channel channel;\n\n    @Before\n    public void init() {\n        brokerConfig = new BrokerConfig();\n        producerManager = new ProducerManager(null, brokerConfig);\n        clientInfo = new ClientChannelInfo(channel, \"clientId\", LanguageCode.JAVA, 0);\n    }\n\n    @Test\n    public void scanNotActiveChannel() throws Exception {\n        producerManager.registerProducer(group, clientInfo);\n        AtomicReference<String> groupRef = new AtomicReference<>();\n        AtomicReference<ClientChannelInfo> clientChannelInfoRef = new AtomicReference<>();\n        producerManager.appendProducerChangeListener((event, group, clientChannelInfo) -> {\n            switch (event) {\n                case GROUP_UNREGISTER:\n                    groupRef.set(group);\n                    break;\n                case CLIENT_UNREGISTER:\n                    clientChannelInfoRef.set(clientChannelInfo);\n                    break;\n                default:\n                    break;\n            }\n        });\n        assertThat(producerManager.getGroupChannelTable().get(group).get(channel)).isNotNull();\n        assertThat(producerManager.findChannel(\"clientId\")).isNotNull();\n        Field field = ProducerManager.class.getDeclaredField(\"CHANNEL_EXPIRED_TIMEOUT\");\n        field.setAccessible(true);\n        long channelExpiredTimeout = field.getLong(producerManager);\n        clientInfo.setLastUpdateTimestamp(System.currentTimeMillis() - channelExpiredTimeout - 10);\n        when(channel.close()).thenReturn(mock(ChannelFuture.class));\n        producerManager.scanNotActiveChannel();\n        assertThat(producerManager.getGroupChannelTable().get(group)).isNull();\n        assertThat(groupRef.get()).isEqualTo(group);\n        assertThat(clientChannelInfoRef.get()).isSameAs(clientInfo);\n        assertThat(producerManager.findChannel(\"clientId\")).isNull();\n    }\n\n    @Test\n    public void scanNotActiveChannelWithSameClientId() throws Exception {\n        producerManager.registerProducer(group, clientInfo);\n        Channel channel1 = Mockito.mock(Channel.class);\n        ClientChannelInfo clientInfo1 = new ClientChannelInfo(channel1, clientInfo.getClientId(), LanguageCode.JAVA, 0);\n        producerManager.registerProducer(group, clientInfo1);\n        AtomicReference<String> groupRef = new AtomicReference<>();\n        AtomicReference<ClientChannelInfo> clientChannelInfoRef = new AtomicReference<>();\n        producerManager.appendProducerChangeListener((event, group, clientChannelInfo) -> {\n            switch (event) {\n                case GROUP_UNREGISTER:\n                    groupRef.set(group);\n                    break;\n                case CLIENT_UNREGISTER:\n                    clientChannelInfoRef.set(clientChannelInfo);\n                    break;\n                default:\n                    break;\n            }\n        });\n        assertThat(producerManager.getGroupChannelTable().get(group).get(channel)).isNotNull();\n        assertThat(producerManager.getGroupChannelTable().get(group).get(channel1)).isNotNull();\n        assertThat(producerManager.findChannel(\"clientId\")).isNotNull();\n        Field field = ProducerManager.class.getDeclaredField(\"CHANNEL_EXPIRED_TIMEOUT\");\n        field.setAccessible(true);\n        long channelExpiredTimeout = field.getLong(producerManager);\n        clientInfo.setLastUpdateTimestamp(System.currentTimeMillis() - channelExpiredTimeout - 10);\n        when(channel.close()).thenReturn(mock(ChannelFuture.class));\n        producerManager.scanNotActiveChannel();\n        assertThat(producerManager.getGroupChannelTable().get(group).get(channel1)).isNotNull();\n        assertThat(producerManager.findChannel(\"clientId\")).isNotNull();\n    }\n\n    @Test\n    public void doChannelCloseEvent() throws Exception {\n        producerManager.registerProducer(group, clientInfo);\n        AtomicReference<String> groupRef = new AtomicReference<>();\n        AtomicReference<ClientChannelInfo> clientChannelInfoRef = new AtomicReference<>();\n        producerManager.appendProducerChangeListener((event, group, clientChannelInfo) -> {\n            switch (event) {\n                case GROUP_UNREGISTER:\n                    groupRef.set(group);\n                    break;\n                case CLIENT_UNREGISTER:\n                    clientChannelInfoRef.set(clientChannelInfo);\n                    break;\n                default:\n                    break;\n            }\n        });\n        assertThat(producerManager.getGroupChannelTable().get(group).get(channel)).isNotNull();\n        assertThat(producerManager.findChannel(\"clientId\")).isNotNull();\n        producerManager.doChannelCloseEvent(\"127.0.0.1\", channel);\n        assertThat(producerManager.getGroupChannelTable().get(group)).isNull();\n        assertThat(groupRef.get()).isEqualTo(group);\n        assertThat(clientChannelInfoRef.get()).isSameAs(clientInfo);\n        assertThat(producerManager.findChannel(\"clientId\")).isNull();\n    }\n\n    @Test\n    public void testRegisterProducer() {\n        brokerConfig.setEnableRegisterProducer(false);\n        brokerConfig.setRejectTransactionMessage(true);\n        producerManager.registerProducer(group, clientInfo);\n        Map<Channel, ClientChannelInfo> channelMap = producerManager.getGroupChannelTable().get(group);\n        Channel channel1 = producerManager.findChannel(\"clientId\");\n        assertThat(channelMap).isNull();\n        assertThat(channel1).isNull();\n\n        brokerConfig.setEnableRegisterProducer(true);\n        brokerConfig.setRejectTransactionMessage(false);\n        producerManager.registerProducer(group, clientInfo);\n        channelMap = producerManager.getGroupChannelTable().get(group);\n        channel1 = producerManager.findChannel(\"clientId\");\n        assertThat(channelMap).isNotNull();\n        assertThat(channel1).isNotNull();\n        assertThat(channelMap.get(channel)).isEqualTo(clientInfo);\n        assertThat(channel1).isEqualTo(channel);\n    }\n\n    @Test\n    public void unregisterProducer() throws Exception {\n        producerManager.registerProducer(group, clientInfo);\n        AtomicReference<String> groupRef = new AtomicReference<>();\n        AtomicReference<ClientChannelInfo> clientChannelInfoRef = new AtomicReference<>();\n        producerManager.appendProducerChangeListener((event, group, clientChannelInfo) -> {\n            switch (event) {\n                case GROUP_UNREGISTER:\n                    groupRef.set(group);\n                    break;\n                case CLIENT_UNREGISTER:\n                    clientChannelInfoRef.set(clientChannelInfo);\n                    break;\n                default:\n                    break;\n            }\n        });\n        Map<Channel, ClientChannelInfo> channelMap = producerManager.getGroupChannelTable().get(group);\n        assertThat(channelMap).isNotNull();\n        assertThat(channelMap.get(channel)).isEqualTo(clientInfo);\n        Channel channel1 = producerManager.findChannel(\"clientId\");\n        assertThat(channel1).isNotNull();\n        assertThat(channel1).isEqualTo(channel);\n        producerManager.unregisterProducer(group, clientInfo);\n        channelMap = producerManager.getGroupChannelTable().get(group);\n        channel1 = producerManager.findChannel(\"clientId\");\n        assertThat(groupRef.get()).isEqualTo(group);\n        assertThat(clientChannelInfoRef.get()).isSameAs(clientInfo);\n        assertThat(channelMap).isNull();\n        assertThat(channel1).isNull();\n\n    }\n\n    @Test\n    public void testGetGroupChannelTable() throws Exception {\n        producerManager.registerProducer(group, clientInfo);\n        Map<Channel, ClientChannelInfo> oldMap = producerManager.getGroupChannelTable().get(group);\n        \n        producerManager.unregisterProducer(group, clientInfo);\n        assertThat(oldMap.size()).isEqualTo(0);\n    }\n\n    @Test\n    public void testGetAvailableChannel() {\n        producerManager.registerProducer(group, clientInfo);\n\n        when(channel.isActive()).thenReturn(true);\n        when(channel.isWritable()).thenReturn(true);\n        Channel c = producerManager.getAvailableChannel(group);\n        assertThat(c).isSameAs(channel);\n\n        when(channel.isWritable()).thenReturn(false);\n        c = producerManager.getAvailableChannel(group);\n        assertThat(c).isSameAs(channel);\n\n        when(channel.isActive()).thenReturn(false);\n        c = producerManager.getAvailableChannel(group);\n        assertThat(c).isNull();\n    }\n\n}"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.client.net;\n\nimport io.netty.channel.Channel;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ConsumerGroupInfo;\nimport org.apache.rocketmq.broker.client.ConsumerManager;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.remoting.RemotingServer;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.GetConsumerStatusBody;\nimport org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.nio.charset.StandardCharsets;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.doThrow;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class Broker2ClientTest {\n    \n    @Mock\n    private BrokerController brokerController;\n    \n    @Mock\n    private RemotingServer remotingServer;\n    \n    @Mock\n    private ConsumerManager consumerManager;\n    \n    @Mock\n    private TopicConfigManager topicConfigManager;\n    \n    @Mock\n    private ConsumerOffsetManager consumerOffsetManager;\n    \n    @Mock\n    private Channel channel;\n    \n    @Mock\n    private ConsumerGroupInfo consumerGroupInfo;\n    \n    private Broker2Client broker2Client;\n    \n    private final String defaultTopic = \"defaultTopic\";\n    \n    private final String defaultBroker = \"defaultBroker\";\n    \n    private final String defaultGroup = \"defaultGroup\";\n    \n    private final long timestamp = System.currentTimeMillis();\n    \n    private final boolean isForce = true;\n    \n    @Before\n    public void init() {\n        broker2Client = new Broker2Client(brokerController);\n        when(brokerController.getRemotingServer()).thenReturn(remotingServer);\n        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);\n        when(brokerController.getConsumerManager()).thenReturn(consumerManager);\n        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);\n        when(brokerController.getBrokerConfig()).thenReturn(mock(BrokerConfig.class));\n        when(brokerController.getMessageStore()).thenReturn(mock(MessageStore.class));\n        when(consumerManager.getConsumerGroupInfo(any())).thenReturn(consumerGroupInfo);\n    }\n    \n    @Test\n    public void testCheckProducerTransactionState() throws Exception {\n        CheckTransactionStateRequestHeader requestHeader = new CheckTransactionStateRequestHeader();\n        broker2Client.checkProducerTransactionState(\"group\", channel, requestHeader, createMessageExt());\n        verify(remotingServer).invokeOneway(eq(channel), any(RemotingCommand.class), eq(10L));\n    }\n    \n    @Test\n    public void testCheckProducerTransactionStateException() throws Exception {\n        CheckTransactionStateRequestHeader requestHeader = new CheckTransactionStateRequestHeader();\n        MessageExt messageExt = createMessageExt();\n        doThrow(new RuntimeException(\"Test Exception\"))\n                .when(remotingServer)\n                .invokeOneway(any(Channel.class),\n                        any(RemotingCommand.class),\n                        anyLong());\n        broker2Client.checkProducerTransactionState(\"group\", channel, requestHeader, messageExt);\n        verify(brokerController.getRemotingServer()).invokeOneway(eq(channel), any(RemotingCommand.class), eq(10L));\n    }\n    \n    @Test\n    public void testResetOffsetNoTopicConfig() throws RemotingCommandException {\n        when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(null);\n        RemotingCommand response = broker2Client.resetOffset(defaultTopic, defaultGroup, timestamp, isForce);\n        assertEquals(ResponseCode.TOPIC_NOT_EXIST, response.getCode());\n    }\n    \n    @Test\n    public void testResetOffsetNoConsumerGroupInfo() throws RemotingCommandException {\n        TopicConfig topicConfig = mock(TopicConfig.class);\n        when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(topicConfig);\n        when(topicConfig.getWriteQueueNums()).thenReturn(1);\n        when(consumerOffsetManager.queryOffset(defaultGroup, defaultTopic, 0)).thenReturn(0L);\n        RemotingCommand response = broker2Client.resetOffset(defaultTopic, defaultGroup, timestamp, isForce);\n        assertEquals(ResponseCode.CONSUMER_NOT_ONLINE, response.getCode());\n    }\n    \n    @Test\n    public void testResetOffset() throws RemotingCommandException {\n        TopicConfig topicConfig = mock(TopicConfig.class);\n        when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(topicConfig);\n        when(topicConfig.getWriteQueueNums()).thenReturn(1);\n        when(brokerController.getConsumerOffsetManager().queryOffset(defaultGroup, defaultTopic, 0)).thenReturn(0L);\n        BrokerConfig brokerConfig = mock(BrokerConfig.class);\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        when(brokerConfig.getBrokerName()).thenReturn(defaultBroker);\n        ConsumerGroupInfo consumerGroupInfo = mock(ConsumerGroupInfo.class);\n        when(consumerManager.getConsumerGroupInfo(defaultGroup)).thenReturn(consumerGroupInfo);\n        RemotingCommand response = broker2Client.resetOffset(defaultTopic, defaultGroup, timestamp, isForce);\n        assertEquals(ResponseCode.CONSUMER_NOT_ONLINE, response.getCode());\n    }\n    \n    @Test\n    public void testGetConsumeStatusNoConsumerOnline() {\n        when(consumerGroupInfo.getChannelInfoTable()).thenReturn(new ConcurrentHashMap<>());\n        RemotingCommand response = broker2Client.getConsumeStatus(defaultTopic, defaultGroup, \"\");\n        assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode());\n    }\n    \n    @Test\n    public void testGetConsumeStatusClientDoesNotSupportFeature() {\n        ClientChannelInfo clientChannelInfo = new ClientChannelInfo(channel, \"defaultClientId\", null, MQVersion.Version.V3_0_6.ordinal());\n        ConcurrentMap<Channel, ClientChannelInfo> channelInfoTable = new ConcurrentHashMap<>();\n        channelInfoTable.put(channel, clientChannelInfo);\n        when(consumerGroupInfo.getChannelInfoTable()).thenReturn(channelInfoTable);\n        RemotingCommand response = broker2Client.getConsumeStatus(defaultTopic, defaultGroup, \"\");\n        assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode());\n    }\n    \n    @Test\n    public void testGetConsumeStatus() throws Exception {\n        ConcurrentMap<Channel, ClientChannelInfo> channelInfoTable = new ConcurrentHashMap<>();\n        ClientChannelInfo clientChannelInfo = mock(ClientChannelInfo.class);\n        when(clientChannelInfo.getVersion()).thenReturn(MQVersion.CURRENT_VERSION);\n        channelInfoTable.put(channel, clientChannelInfo);\n        when(consumerGroupInfo.getChannelInfoTable()).thenReturn(channelInfoTable);\n        RemotingCommand responseMock = mock(RemotingCommand.class);\n        when(responseMock.getCode()).thenReturn(ResponseCode.SUCCESS);\n        when(responseMock.getBody()).thenReturn(\"{\\\"consumerTable\\\":{}}\".getBytes(StandardCharsets.UTF_8));\n        when(remotingServer.invokeSync(any(Channel.class), any(RemotingCommand.class), anyLong())).thenReturn(responseMock);\n        RemotingCommand response = broker2Client.getConsumeStatus(defaultTopic, defaultGroup, \"\");\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n        GetConsumerStatusBody body = RemotingSerializable.decode(response.getBody(), GetConsumerStatusBody.class);\n        assertEquals(1, body.getConsumerTable().size());\n    }\n    \n    private MessageExt createMessageExt() {\n        MessageExt result = new MessageExt();\n        result.setBody(\"body\".getBytes(StandardCharsets.UTF_8));\n        result.setTopic(defaultTopic);\n        result.setBrokerName(defaultBroker);\n        result.putUserProperty(\"key\", \"value\");\n        result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, defaultGroup);\n        result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, \"TX1\");\n        result.setKeys(\"keys\");\n        SocketAddress bornHost = new InetSocketAddress(\"127.0.0.1\", 12911);\n        SocketAddress storeHost = new InetSocketAddress(\"127.0.0.1\", 10911);\n        result.setStoreHost(storeHost);\n        result.setBornHost(bornHost);\n        return result;\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.client.rebalance;\n\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.util.HashSet;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class RebalanceLockManagerTest {\n    \n    @Mock\n    private RebalanceLockManager.LockEntry lockEntry;\n    \n    private final RebalanceLockManager rebalanceLockManager = new RebalanceLockManager();\n    \n    private final String defaultTopic = \"defaultTopic\";\n    \n    private final String defaultBroker = \"defaultBroker\";\n    \n    private final String defaultGroup = \"defaultGroup\";\n    \n    private final String defaultClientId = \"defaultClientId\";\n    \n    @Test\n    public void testIsLockAllExpiredGroupNotExist() {\n        assertTrue(rebalanceLockManager.isLockAllExpired(defaultGroup));\n    }\n    \n    @Test\n    public void testIsLockAllExpiredGroupExist() throws IllegalAccessException {\n        FieldUtils.writeDeclaredField(rebalanceLockManager, \"mqLockTable\", createMQLockTable(), true);\n        when(lockEntry.isExpired()).thenReturn(false);\n        assertFalse(rebalanceLockManager.isLockAllExpired(defaultGroup));\n    }\n    \n    @Test\n    public void testIsLockAllExpiredGroupExistSomeExpired() throws IllegalAccessException {\n        FieldUtils.writeDeclaredField(rebalanceLockManager, \"mqLockTable\", createMQLockTable(), true);\n        when(lockEntry.isExpired()).thenReturn(true).thenReturn(false);\n        assertFalse(rebalanceLockManager.isLockAllExpired(defaultGroup));\n    }\n    \n    @Test\n    public void testTryLockNotLocked() {\n        assertTrue(rebalanceLockManager.tryLock(defaultGroup, createDefaultMessageQueue(), defaultClientId));\n    }\n    \n    @Test\n    public void testTryLockSameClient() throws IllegalAccessException {\n        when(lockEntry.isLocked(defaultClientId)).thenReturn(true);\n        FieldUtils.writeDeclaredField(rebalanceLockManager, \"mqLockTable\", createMQLockTable(), true);\n        assertTrue(rebalanceLockManager.tryLock(defaultGroup, createDefaultMessageQueue(), defaultClientId));\n    }\n    \n    @Test\n    public void testTryLockDifferentClient() throws Exception {\n        when(lockEntry.isLocked(defaultClientId)).thenReturn(false);\n        FieldUtils.writeDeclaredField(rebalanceLockManager, \"mqLockTable\", createMQLockTable(), true);\n        assertFalse(rebalanceLockManager.tryLock(defaultGroup, createDefaultMessageQueue(), defaultClientId));\n    }\n    \n    @Test\n    public void testTryLockButExpired() throws IllegalAccessException {\n        when(lockEntry.isExpired()).thenReturn(true);\n        FieldUtils.writeDeclaredField(rebalanceLockManager, \"mqLockTable\", createMQLockTable(), true);\n        assertTrue(rebalanceLockManager.tryLock(defaultGroup, createDefaultMessageQueue(), defaultClientId));\n    }\n    \n    @Test\n    public void testTryLockBatchAllLocked() {\n        Set<MessageQueue> mqs = createMessageQueue(2);\n        Set<MessageQueue> actual = rebalanceLockManager.tryLockBatch(defaultGroup, mqs, defaultClientId);\n        assertEquals(mqs, actual);\n    }\n    \n    @Test\n    public void testTryLockBatchNoneLocked() throws IllegalAccessException {\n        when(lockEntry.isLocked(defaultClientId)).thenReturn(false);\n        FieldUtils.writeDeclaredField(rebalanceLockManager, \"mqLockTable\", createMQLockTable(), true);\n        Set<MessageQueue> actual = rebalanceLockManager.tryLockBatch(defaultGroup, createMessageQueue(2), defaultClientId);\n        assertTrue(actual.isEmpty());\n    }\n    \n    @Test\n    public void testTryLockBatchSomeLocked() throws IllegalAccessException {\n        Set<MessageQueue> mqs = new HashSet<>();\n        MessageQueue mq1 = new MessageQueue(defaultTopic, defaultBroker, 0);\n        MessageQueue mq2 = new MessageQueue(defaultTopic, defaultBroker, 1);\n        mqs.add(mq1);\n        mqs.add(mq2);\n        when(lockEntry.isLocked(defaultClientId)).thenReturn(true).thenReturn(false);\n        FieldUtils.writeDeclaredField(rebalanceLockManager, \"mqLockTable\", createMQLockTable(), true);\n        Set<MessageQueue> actual = rebalanceLockManager.tryLockBatch(defaultGroup, mqs, defaultClientId);\n        Set<MessageQueue> expected = new HashSet<>();\n        expected.add(mq2);\n        assertEquals(expected, actual);\n    }\n    \n    @Test\n    public void testUnlockBatch() throws IllegalAccessException {\n        when(lockEntry.getClientId()).thenReturn(defaultClientId);\n        ConcurrentMap<String, ConcurrentHashMap<MessageQueue, RebalanceLockManager.LockEntry>> mqLockTable = createMQLockTable();\n        FieldUtils.writeDeclaredField(rebalanceLockManager, \"mqLockTable\", mqLockTable, true);\n        rebalanceLockManager.unlockBatch(defaultGroup, createMessageQueue(1), defaultClientId);\n        assertEquals(1, mqLockTable.get(defaultGroup).values().size());\n    }\n    \n    @Test\n    public void testUnlockBatchByOtherClient() throws IllegalAccessException {\n        when(lockEntry.getClientId()).thenReturn(\"otherClientId\");\n        ConcurrentMap<String, ConcurrentHashMap<MessageQueue, RebalanceLockManager.LockEntry>> mqLockTable = createMQLockTable();\n        FieldUtils.writeDeclaredField(rebalanceLockManager, \"mqLockTable\", mqLockTable, true);\n        rebalanceLockManager.unlockBatch(defaultGroup, createMessageQueue(1), defaultClientId);\n        assertEquals(2, mqLockTable.get(defaultGroup).values().size());\n    }\n    \n    private MessageQueue createDefaultMessageQueue() {\n        return createMessageQueue(1).iterator().next();\n    }\n    \n    private Set<MessageQueue> createMessageQueue(final int count) {\n        Set<MessageQueue> result = new HashSet<>();\n        for (int i = 0; i < count; i++) {\n            result.add(new MessageQueue(defaultTopic, defaultBroker, i));\n        }\n        return result;\n    }\n    \n    private ConcurrentMap<String, ConcurrentHashMap<MessageQueue, RebalanceLockManager.LockEntry>> createMQLockTable() {\n        MessageQueue messageQueue1 = new MessageQueue(defaultTopic, defaultBroker, 0);\n        MessageQueue messageQueue2 = new MessageQueue(defaultTopic, defaultBroker, 1);\n        ConcurrentHashMap<MessageQueue, RebalanceLockManager.LockEntry> lockEntryMap = new ConcurrentHashMap<>();\n        lockEntryMap.put(messageQueue1, lockEntry);\n        lockEntryMap.put(messageQueue2, lockEntry);\n        ConcurrentMap<String, ConcurrentHashMap<MessageQueue, RebalanceLockManager.LockEntry>> result = new ConcurrentHashMap<>();\n        result.put(defaultGroup, lockEntryMap);\n        return result;\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/coldctr/ColdDataCgCtrServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.coldctr;\n\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.coldctr.AccAndTimeStamp;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.atomic.AtomicLong;\n\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ColdDataCgCtrServiceTest {\n\n    @Mock\n    private BrokerController brokerController;\n\n    @Mock\n    private BrokerConfig brokerConfig;\n\n    private ColdDataCgCtrService coldDataCgCtrService;\n\n    @Before\n    public void init() throws IllegalAccessException {\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        coldDataCgCtrService = new ColdDataCgCtrService(brokerController);\n        FieldUtils.writeField(coldDataCgCtrService, \"cgColdThresholdMapRuntime\", createCgColdThresholdMapRuntime(), true);\n        FieldUtils.writeField(coldDataCgCtrService, \"cgColdThresholdMapConfig\", createCgColdThresholdMapConfig(), true);\n    }\n\n    @Test\n    public void testGetColdDataFlowCtrInfo() {\n        String actual = coldDataCgCtrService.getColdDataFlowCtrInfo();\n        assertTrue(actual.contains(\"\\\"globalAcc\\\":0\"));\n        assertTrue(actual.contains(\"\\\"cgColdReadThreshold\\\":0\"));\n        assertTrue(actual.contains(\"\\\"globalColdReadThreshold\\\":0\"));\n        assertTrue(actual.contains(\"\\\"configTable\\\":{\\\"consumerGroup2\\\":2048}\"));\n        assertTrue(actual.contains(\"\\\"runtimeTable\\\":{\\\"consumerGroup1\\\":{\\\"coldAcc\\\":1,\\\"createTimeMills\\\":1,\\\"lastColdReadTimeMills\\\":1}}\"));\n    }\n\n    private Map<String, AccAndTimeStamp> createCgColdThresholdMapRuntime() {\n        Map<String, AccAndTimeStamp> result = new ConcurrentHashMap<>();\n        AccAndTimeStamp accAndTimeStamp = new AccAndTimeStamp(new AtomicLong(1L));\n        accAndTimeStamp.setCreateTimeMills(1L);\n        accAndTimeStamp.setLastColdReadTimeMills(1L);\n        result.put(\"consumerGroup1\", accAndTimeStamp);\n        return result;\n    }\n\n    private ConcurrentHashMap<String, Long> createCgColdThresholdMapConfig() {\n        ConcurrentHashMap<String, Long> result = new ConcurrentHashMap<>();\n        result.put(\"consumerGroup2\", 2048L);\n        return result;\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/config/v1/RocksDBConsumerOffsetManagerMigrationTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.config.v1;\n\nimport java.io.File;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.Mockito;\n\npublic class RocksDBConsumerOffsetManagerMigrationTest {\n\n    private static final String TEST_GROUP = \"TestGroup\";\n    private static final String TEST_TOPIC = \"TestTopic\";\n    private static final String TEST_KEY = TEST_TOPIC + \"@\" + TEST_GROUP;\n    \n    private BrokerController brokerController;\n    private String storePath;\n    private String separateRocksDBPath;\n    private String unifiedRocksDBPath;\n\n    @Before\n    public void init() {\n        brokerController = Mockito.mock(BrokerController.class);\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        storePath = System.getProperty(\"java.io.tmpdir\") + File.separator + \"rocketmq-test-\" + System.currentTimeMillis();\n        messageStoreConfig.setStorePathRootDir(storePath);\n        Mockito.when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n\n        BrokerConfig brokerConfig = new BrokerConfig();\n        brokerConfig.setConsumerOffsetUpdateVersionStep(1);\n        Mockito.when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        \n        separateRocksDBPath = storePath + File.separator + \"config\" + File.separator + \"consumerOffsets\" + File.separator;\n        unifiedRocksDBPath = storePath + File.separator + \"config\" + File.separator + \"metadata\" + File.separator;\n        \n        // Create directories\n        UtilAll.ensureDirOK(separateRocksDBPath);\n        UtilAll.ensureDirOK(unifiedRocksDBPath);\n    }\n\n    @After\n    public void destroy() {\n        // Clean up test directories\n        UtilAll.deleteFile(new File(storePath));\n    }\n\n    @Test\n    public void testMigrationFromSeparateToUnifiedRocksDB() {\n        \n        // First, create data in separate RocksDB mode\n        RocksDBConsumerOffsetManager separateManager = new RocksDBConsumerOffsetManager(brokerController, false);\n        separateManager.load();\n        \n        // Add some consumer offsets\n        separateManager.commitOffset(\"client\", TEST_GROUP, TEST_TOPIC, 0, 100L);\n        separateManager.commitOffset(\"client\", TEST_GROUP, TEST_TOPIC, 1, 200L);\n        separateManager.persist();\n        separateManager.stop();\n        \n        // Now create unified RocksDB manager which should migrate data\n        RocksDBConsumerOffsetManager unifiedManager = new RocksDBConsumerOffsetManager(brokerController, true);\n        boolean loaded = unifiedManager.load();\n        Assert.assertTrue(\"Unified manager should load successfully\", loaded);\n        \n        // Verify that data was migrated\n        ConcurrentMap<Integer, Long> migratedOffsetMap = unifiedManager.getOffsetTable().get(TEST_KEY);\n        Assert.assertNotNull(\"Consumer offset should be migrated\", migratedOffsetMap);\n        Assert.assertEquals(\"Offset for queue 0 should match\", Long.valueOf(100L), migratedOffsetMap.get(0));\n        Assert.assertEquals(\"Offset for queue 1 should match\", Long.valueOf(200L), migratedOffsetMap.get(1));\n\n        unifiedManager.commitOffset(\"client\", TEST_GROUP, TEST_TOPIC, 0, 300L);\n        unifiedManager.commitOffset(\"client\", TEST_GROUP, TEST_TOPIC, 1, 400L);\n        unifiedManager.persist();\n        unifiedManager.stop();\n\n        // reload unified RocksDB manager which should not migrate data\n        unifiedManager = new RocksDBConsumerOffsetManager(brokerController, true);\n        unifiedManager.load();\n\n        // Verify that data was new\n        migratedOffsetMap = unifiedManager.getOffsetTable().get(TEST_KEY);\n        Assert.assertEquals(\"Offset for queue 0 should match\", Long.valueOf(300L), migratedOffsetMap.get(0));\n        Assert.assertEquals(\"Offset for queue 1 should match\", Long.valueOf(400L), migratedOffsetMap.get(1));\n        unifiedManager.stop();\n    }\n\n    @Test\n    public void testMigrationWithNoSeparateRocksDB() {\n        \n        // Ensure separate RocksDB doesn't exist\n        UtilAll.deleteFile(new File(separateRocksDBPath));\n        \n        // Create unified RocksDB manager - should not fail even without separate DB\n        RocksDBConsumerOffsetManager unifiedManager = new RocksDBConsumerOffsetManager(brokerController, true);\n        boolean loaded = unifiedManager.load();\n        Assert.assertTrue(\"Unified manager should load successfully even without separate DB\", loaded);\n        \n        unifiedManager.stop();\n    }\n\n    @Test\n    public void testNoMigrationWhenDisabled() {\n        \n        // Create data in separate RocksDB mode\n        RocksDBConsumerOffsetManager separateManager = new RocksDBConsumerOffsetManager(brokerController, false);\n        separateManager.load();\n\n        separateManager.commitOffset(\"client\", TEST_GROUP, TEST_TOPIC, 0, 100L);\n        separateManager.commitOffset(\"client\", TEST_GROUP, TEST_TOPIC, 1, 200L);\n        separateManager.persist();\n        separateManager.stop();\n\n        long version = separateManager.getDataVersion().getCounter().get();\n        Assert.assertEquals(2, version);\n\n        // Create another separate manager - should not trigger migration\n        RocksDBConsumerOffsetManager anotherSeparateManager = new RocksDBConsumerOffsetManager(brokerController, false);\n        boolean loaded = anotherSeparateManager.load();\n        Assert.assertTrue(\"Separate manager should load successfully\", loaded);\n\n        anotherSeparateManager.loadDataVersion();\n        Assert.assertEquals(version, anotherSeparateManager.getDataVersion().getCounter().get());\n        anotherSeparateManager.stop();\n    }\n}"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/config/v1/RocksDBSubscriptionGroupManagerMigrationTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.config.v1;\n\nimport java.io.File;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.Mockito;\n\npublic class RocksDBSubscriptionGroupManagerMigrationTest {\n\n    private static final String TEST_GROUP = \"TestGroup\";\n    \n    private BrokerController brokerController;\n    private String storePath;\n    private String separateRocksDBPath;\n    private String unifiedRocksDBPath;\n\n    @Before\n    public void init() {\n        \n        brokerController = Mockito.mock(BrokerController.class);\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        storePath = System.getProperty(\"java.io.tmpdir\") + File.separator + \"rocketmq-test-\" + System.currentTimeMillis();\n        messageStoreConfig.setStorePathRootDir(storePath);\n        Mockito.when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n        \n        separateRocksDBPath = storePath + File.separator + \"config\" + File.separator + \"subscriptionGroups\" + File.separator;\n        unifiedRocksDBPath = storePath + File.separator + \"config\" + File.separator + \"metadata\" + File.separator;\n        \n        // Create directories\n        UtilAll.ensureDirOK(separateRocksDBPath);\n        UtilAll.ensureDirOK(unifiedRocksDBPath);\n    }\n\n    @After\n    public void destroy() {\n        \n        // Clean up test directories\n        UtilAll.deleteFile(new File(storePath));\n    }\n\n    @Test\n    public void testMigrationFromSeparateToUnifiedRocksDB() {\n        \n        // First, create data in separate RocksDB mode\n        RocksDBSubscriptionGroupManager separateManager = new RocksDBSubscriptionGroupManager(brokerController, false);\n        separateManager.load();\n        \n        // Add some subscription groups\n        SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig();\n        groupConfig.setGroupName(TEST_GROUP);\n        groupConfig.setConsumeEnable(true);\n        groupConfig.setConsumeFromMinEnable(true);\n        groupConfig.setRetryMaxTimes(3);\n        separateManager.updateSubscriptionGroupConfig(groupConfig);\n        separateManager.persist();\n        separateManager.stop();\n\n        {\n            // Now create unified RocksDB manager which should migrate data\n            RocksDBSubscriptionGroupManager unifiedManager = new RocksDBSubscriptionGroupManager(brokerController, true);\n            boolean loaded = unifiedManager.load();\n            Assert.assertTrue(\"Unified manager should load successfully\", loaded);\n\n            // Verify that data was migrated\n            SubscriptionGroupConfig migratedConfig = unifiedManager.findSubscriptionGroupConfig(TEST_GROUP);\n            Assert.assertNotNull(\"Subscription group should be migrated\", migratedConfig);\n            Assert.assertEquals(\"Group name should match\", TEST_GROUP, migratedConfig.getGroupName());\n            Assert.assertEquals(\"Retry max times should match\", 3, migratedConfig.getRetryMaxTimes());\n            Assert.assertTrue(\"Consume enable should match\", migratedConfig.isConsumeEnable());\n            Assert.assertTrue(\"Consume from min enable should match\", migratedConfig.isConsumeFromMinEnable());\n\n            groupConfig.setRetryMaxTimes(4);\n            unifiedManager.updateSubscriptionGroupConfig(groupConfig);\n            unifiedManager.persist();\n            unifiedManager.stop();\n        }\n\n        {\n            // Now create unified RocksDB manager which should migrate data\n            RocksDBSubscriptionGroupManager unifiedManager = new RocksDBSubscriptionGroupManager(brokerController, true);\n            boolean loaded = unifiedManager.load();\n            Assert.assertTrue(\"Unified manager should load successfully\", loaded);\n\n            // Verify that data was migrated\n            SubscriptionGroupConfig migratedConfig = unifiedManager.findSubscriptionGroupConfig(TEST_GROUP);\n            Assert.assertNotNull(\"Subscription group should be migrated\", migratedConfig);\n            Assert.assertEquals(\"Group name should match\", TEST_GROUP, migratedConfig.getGroupName());\n            Assert.assertEquals(\"Retry max times should match\", 4, migratedConfig.getRetryMaxTimes());\n            Assert.assertTrue(\"Consume enable should match\", migratedConfig.isConsumeEnable());\n            Assert.assertTrue(\"Consume from min enable should match\", migratedConfig.isConsumeFromMinEnable());\n\n            unifiedManager.stop();\n        }\n    }\n\n    @Test\n    public void testMigrationWithNoSeparateRocksDB() {\n        \n        // Ensure separate RocksDB doesn't exist\n        UtilAll.deleteFile(new File(separateRocksDBPath));\n        \n        // Create unified RocksDB manager - should not fail even without separate DB\n        RocksDBSubscriptionGroupManager unifiedManager = new RocksDBSubscriptionGroupManager(brokerController, true);\n        boolean loaded = unifiedManager.load();\n        Assert.assertTrue(\"Unified manager should load successfully even without separate DB\", loaded);\n        \n        unifiedManager.stop();\n    }\n\n    @Test\n    public void testNoMigrationWhenDisabled() {\n        \n        // Create data in separate RocksDB mode\n        RocksDBSubscriptionGroupManager separateManager = new RocksDBSubscriptionGroupManager(brokerController, false);\n        separateManager.load();\n        \n        SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig();\n        groupConfig.setGroupName(TEST_GROUP);\n        groupConfig.setConsumeEnable(true);\n        groupConfig.setConsumeFromMinEnable(true);\n        separateManager.putSubscriptionGroupConfig(groupConfig);\n        separateManager.persist();\n        separateManager.stop();\n        \n        // Create another separate manager - should not trigger migration\n        RocksDBSubscriptionGroupManager anotherSeparateManager = new RocksDBSubscriptionGroupManager(brokerController, false);\n        boolean loaded = anotherSeparateManager.load();\n        Assert.assertTrue(\"Separate manager should load successfully\", loaded);\n        \n        anotherSeparateManager.stop();\n    }\n}"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/config/v1/RocksDBSubscriptionGroupManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.config.v1;\n\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.concurrent.ConcurrentMap;\n\nimport static org.junit.Assert.assertTrue;\nimport static org.junit.jupiter.api.Assertions.assertEquals;\nimport static org.junit.jupiter.api.Assertions.assertNull;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class RocksDBSubscriptionGroupManagerTest {\n\n    @Mock\n    private BrokerController brokerController;\n\n    @Mock\n    private RocksDBConfigManager rocksDBConfigManager;\n\n    private RocksDBSubscriptionGroupManager rocksDBSubscriptionGroupManager;\n\n    @Mock\n    private MessageStoreConfig messageStoreConfig;\n\n    @Before\n    public void init() throws IllegalAccessException {\n        when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n        when(messageStoreConfig.getMemTableFlushIntervalMs()).thenReturn(1000L);\n        when(messageStoreConfig.getRocksdbCompressionType()).thenReturn(\"LZ4_COMPRESSION\");\n        when(messageStoreConfig.getStorePathRootDir()).thenReturn(\"/\");\n        BrokerConfig brokerConfig = mock(BrokerConfig.class);\n        when(brokerConfig.isUseSingleRocksDBForAllConfigs()).thenReturn(true);\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        rocksDBSubscriptionGroupManager = new RocksDBSubscriptionGroupManager(brokerController);\n        FieldUtils.writeDeclaredField(rocksDBSubscriptionGroupManager, \"rocksDBConfigManager\", rocksDBConfigManager, true);\n    }\n\n    @Test\n    public void testPutSubscriptionGroupConfig() {\n        SubscriptionGroupConfig newConfig = new SubscriptionGroupConfig();\n        newConfig.setGroupName(\"group\");\n        SubscriptionGroupConfig oldConfig = new SubscriptionGroupConfig();\n        oldConfig.setGroupName(\"group\");\n        rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().put(\"group\", oldConfig);\n\n        assertEquals(oldConfig, rocksDBSubscriptionGroupManager.putSubscriptionGroupConfig(newConfig));\n        assertEquals(newConfig, rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().get(\"group\"));\n    }\n\n    @Test\n    public void testPutSubscriptionGroupConfigIfAbsent() {\n        SubscriptionGroupConfig newConfig = new SubscriptionGroupConfig();\n        newConfig.setGroupName(\"group\");\n        SubscriptionGroupConfig oldConfig = new SubscriptionGroupConfig();\n        oldConfig.setGroupName(\"group\");\n\n        assertNull(rocksDBSubscriptionGroupManager.putSubscriptionGroupConfigIfAbsent(newConfig));\n        assertEquals(newConfig, rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().get(\"group\"));\n    }\n\n    @Test\n    public void testDecodeForbidden() {\n        String forbiddenGroupName = \"group\";\n        String bodyJson = \"{\\\"topic1\\\":1,\\\"topic2\\\":2}\";\n        byte[] key = forbiddenGroupName.getBytes(StandardCharsets.UTF_8);\n        byte[] body = bodyJson.getBytes(StandardCharsets.UTF_8);\n\n        rocksDBSubscriptionGroupManager.decodeForbidden(key, body);\n        ConcurrentMap<String, ConcurrentMap<String, Integer>> forbiddenTable = rocksDBSubscriptionGroupManager.getForbiddenTable();\n        assertTrue(forbiddenTable.containsKey(forbiddenGroupName));\n\n        ConcurrentMap<String, Integer> forbiddenGroup = forbiddenTable.get(forbiddenGroupName);\n        assertEquals(2, forbiddenGroup.size());\n        assertEquals(Integer.valueOf(1), forbiddenGroup.get(\"topic1\"));\n        assertEquals(Integer.valueOf(2), forbiddenGroup.get(\"topic2\"));\n    }\n\n    @Test\n    public void testDecodeSubscriptionGroup() {\n        String groupName = \"group\";\n        String bodyJson = \"{\\\"groupName\\\":\\\"group\\\",\\\"consumeEnable\\\":true}\";\n        byte[] key = groupName.getBytes(StandardCharsets.UTF_8);\n        byte[] body = bodyJson.getBytes(StandardCharsets.UTF_8);\n\n        rocksDBSubscriptionGroupManager.decodeSubscriptionGroup(key, body);\n        ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroupTable = rocksDBSubscriptionGroupManager.getSubscriptionGroupTable();\n        assertEquals(1, subscriptionGroupTable.size());\n        SubscriptionGroupConfig config = subscriptionGroupTable.get(groupName);\n        assertEquals(groupName, config.getGroupName());\n        assertTrue(config.isConsumeEnable());\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/config/v1/RocksDBTopicConfigManagerMigrationTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.config.v1;\n\nimport java.io.File;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.Mockito;\n\npublic class RocksDBTopicConfigManagerMigrationTest {\n\n    private static final String TEST_TOPIC = \"TestTopic\";\n    \n    private BrokerController brokerController;\n    private String storePath;\n    private String separateRocksDBPath;\n    private String unifiedRocksDBPath;\n\n    @Before\n    public void init() {\n        \n        brokerController = Mockito.mock(BrokerController.class);\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        storePath = System.getProperty(\"java.io.tmpdir\") + File.separator + \"rocketmq-test-\" + System.currentTimeMillis();\n        messageStoreConfig.setStorePathRootDir(storePath);\n        Mockito.when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n        Mockito.when(brokerController.getBrokerConfig()).thenReturn(new BrokerConfig());\n        \n        separateRocksDBPath = storePath + File.separator + \"config\" + File.separator + \"topics\" + File.separator;\n        unifiedRocksDBPath = storePath + File.separator + \"config\" + File.separator + \"metadata\" + File.separator;\n        \n        // Create directories\n        UtilAll.ensureDirOK(separateRocksDBPath);\n        UtilAll.ensureDirOK(unifiedRocksDBPath);\n    }\n\n    @After\n    public void destroy() {\n        \n        // Clean up test directories\n        UtilAll.deleteFile(new File(storePath));\n    }\n\n    @Test\n    public void testMigrationFromSeparateToUnifiedRocksDB() {\n        \n        // First, create data in separate RocksDB mode\n        RocksDBTopicConfigManager separateManager = new RocksDBTopicConfigManager(brokerController, false);\n        separateManager.load();\n        \n        // Add some topic configs\n        TopicConfig topicConfig = new TopicConfig(TEST_TOPIC, 4, 4);\n        separateManager.updateTopicConfig(topicConfig);\n        separateManager.persist();\n        separateManager.stop();\n\n        {\n            // Now create unified RocksDB manager which should migrate data\n            RocksDBTopicConfigManager unifiedManager = new RocksDBTopicConfigManager(brokerController, true);\n            boolean loaded = unifiedManager.load();\n            Assert.assertTrue(\"Unified manager should load successfully\", loaded);\n\n            // Verify that data was migrated\n            TopicConfig migratedConfig = unifiedManager.selectTopicConfig(TEST_TOPIC);\n            Assert.assertNotNull(\"Topic config should be migrated\", migratedConfig);\n            Assert.assertEquals(\"Topic name should match\", TEST_TOPIC, migratedConfig.getTopicName());\n            Assert.assertEquals(\"Read queue num should match\", 4, migratedConfig.getReadQueueNums());\n            Assert.assertEquals(\"Write queue num should match\", 4, migratedConfig.getWriteQueueNums());\n\n            topicConfig.setReadQueueNums(8);\n            topicConfig.setWriteQueueNums(8);\n            unifiedManager.updateTopicConfig(topicConfig);\n            unifiedManager.persist();\n            unifiedManager.stop();\n        }\n\n        {\n            // Now create unified RocksDB manager which should migrate data\n            RocksDBTopicConfigManager unifiedManager = new RocksDBTopicConfigManager(brokerController, true);\n            boolean loaded = unifiedManager.load();\n            Assert.assertTrue(\"Unified manager should load successfully\", loaded);\n\n            // Verify that data was migrated\n            TopicConfig migratedConfig = unifiedManager.selectTopicConfig(TEST_TOPIC);\n            Assert.assertNotNull(\"Topic config should be migrated\", migratedConfig);\n            Assert.assertEquals(\"Topic name should match\", TEST_TOPIC, migratedConfig.getTopicName());\n            Assert.assertEquals(\"Read queue num should match\", 8, migratedConfig.getReadQueueNums());\n            Assert.assertEquals(\"Write queue num should match\", 8, migratedConfig.getWriteQueueNums());\n\n            unifiedManager.stop();\n        }\n\n    }\n\n    @Test\n    public void testMigrationWithNoSeparateRocksDB() {\n        \n        // Ensure separate RocksDB doesn't exist\n        UtilAll.deleteFile(new File(separateRocksDBPath));\n        \n        // Create unified RocksDB manager - should not fail even without separate DB\n        RocksDBTopicConfigManager unifiedManager = new RocksDBTopicConfigManager(brokerController, true);\n        boolean loaded = unifiedManager.load();\n        Assert.assertTrue(\"Unified manager should load successfully even without separate DB\", loaded);\n        \n        unifiedManager.stop();\n    }\n\n    @Test\n    public void testNoMigrationWhenDisabled() {\n        // Create data in separate RocksDB mode\n        RocksDBTopicConfigManager separateManager = new RocksDBTopicConfigManager(brokerController, false);\n        separateManager.load();\n        \n        TopicConfig topicConfig = new TopicConfig(TEST_TOPIC, 4, 4);\n        separateManager.putTopicConfig(topicConfig);\n        separateManager.persist();\n        separateManager.stop();\n        \n        // Create another separate manager - should not trigger migration\n        RocksDBTopicConfigManager anotherSeparateManager = new RocksDBTopicConfigManager(brokerController, false);\n        boolean loaded = anotherSeparateManager.load();\n        Assert.assertTrue(\"Separate manager should load successfully\", loaded);\n        \n        anotherSeparateManager.stop();\n    }\n}"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/config/v1/RocksDBTopicConfigManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.config.v1;\n\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.concurrent.ConcurrentMap;\n\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.jupiter.api.Assertions.assertEquals;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class RocksDBTopicConfigManagerTest {\n\n    @Mock\n    private BrokerController brokerController;\n\n    @Mock\n    private MessageStoreConfig messageStoreConfig;\n\n    @Mock\n    private RocksDBConfigManager rocksDBConfigManager;\n\n    private RocksDBTopicConfigManager rocksDBTopicConfigManager;\n\n    @Before\n    public void init() throws Exception {\n        when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n        when(messageStoreConfig.getMemTableFlushIntervalMs()).thenReturn(1000L);\n        when(messageStoreConfig.getRocksdbCompressionType()).thenReturn(\"LZ4_COMPRESSION\");\n        when(messageStoreConfig.getStorePathRootDir()).thenReturn(\"/\");\n        BrokerConfig brokerConfig = mock(BrokerConfig.class);\n        when(brokerConfig.isUseSingleRocksDBForAllConfigs()).thenReturn(true);\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        rocksDBTopicConfigManager = new RocksDBTopicConfigManager(brokerController);\n        FieldUtils.writeDeclaredField(rocksDBTopicConfigManager, \"rocksDBConfigManager\", rocksDBConfigManager, true);\n    }\n\n    @Test\n    public void testDecodeTopicConfig() {\n        String topicName = \"testTopic\";\n        String topicConfigJson = \"{\\\"topicName\\\":\\\"testTopic\\\",\\\"readQueueNums\\\":10,\\\"writeQueueNums\\\":10}\";\n        byte[] key = topicName.getBytes(StandardCharsets.UTF_8);\n        byte[] body = topicConfigJson.getBytes(StandardCharsets.UTF_8);\n\n        rocksDBTopicConfigManager.decodeTopicConfig(key, body);\n\n        ConcurrentMap<String, TopicConfig> topicConfigTable = rocksDBTopicConfigManager.getTopicConfigTable();\n        assertNotNull(topicConfigTable);\n        assertEquals(1, topicConfigTable.size());\n        TopicConfig topicConfig = topicConfigTable.get(topicName);\n        assertNotNull(topicConfig);\n        assertEquals(topicName, topicConfig.getTopicName());\n        assertEquals(10, topicConfig.getReadQueueNums());\n        assertEquals(10, topicConfig.getWriteQueueNums());\n    }\n\n    @Test\n    public void testPutTopicConfig() throws Exception {\n        TopicConfig newTopicConfig = new TopicConfig(\"newTopic\");\n        newTopicConfig.setReadQueueNums(10);\n        newTopicConfig.setWriteQueueNums(10);\n\n        assertNull(rocksDBTopicConfigManager.putTopicConfig(newTopicConfig));\n        verify(rocksDBConfigManager, times(1)).put(any(byte[].class), any(byte[].class));\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2Test.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.config.v2;\n\nimport java.io.File;\nimport java.io.IOException;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Rule;\nimport org.junit.Test;\nimport org.junit.rules.TemporaryFolder;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ConsumerOffsetManagerV2Test {\n\n    private ConfigStorage configStorage;\n\n    private ConsumerOffsetManagerV2 consumerOffsetManagerV2;\n\n    @Mock\n    private BrokerController controller;\n\n    private MessageStoreConfig messageStoreConfig;\n\n    @Rule\n    public TemporaryFolder tf = new TemporaryFolder();\n\n    @After\n    public void cleanUp() {\n        if (null != configStorage) {\n            configStorage.shutdown();\n        }\n    }\n\n    @Before\n    public void setUp() throws IOException {\n        BrokerConfig brokerConfig = new BrokerConfig();\n        Mockito.doReturn(brokerConfig).when(controller).getBrokerConfig();\n\n        File configStoreDir = tf.newFolder();\n        messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setStorePathRootDir(configStoreDir.getAbsolutePath());\n        configStorage = new ConfigStorage(messageStoreConfig);\n        configStorage.start();\n        consumerOffsetManagerV2 = new ConsumerOffsetManagerV2(controller, configStorage);\n    }\n\n    /**\n     * Verify consumer offset can survive restarts\n     */\n    @Test\n    public void testCommitOffset_Standard() {\n        Assert.assertTrue(consumerOffsetManagerV2.load());\n\n        String clientHost = \"localhost\";\n        String topic = \"T1\";\n        String group = \"G0\";\n        int queueId = 1;\n        long queueOffset = 100;\n        consumerOffsetManagerV2.commitOffset(clientHost, group, topic, queueId, queueOffset);\n        Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic, queueId));\n\n        configStorage.shutdown();\n        consumerOffsetManagerV2.getOffsetTable().clear();\n        Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic, queueId));\n\n        configStorage = new ConfigStorage(messageStoreConfig);\n        configStorage.start();\n        consumerOffsetManagerV2 = new ConsumerOffsetManagerV2(controller, configStorage);\n        consumerOffsetManagerV2.load();\n        Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic, queueId));\n    }\n\n    /**\n     * Verify commit offset can survive config store restart\n     */\n    @Test\n    public void testCommitOffset_LMQ() {\n        Assert.assertTrue(consumerOffsetManagerV2.load());\n\n        String clientHost = \"localhost\";\n        String topic = MixAll.LMQ_PREFIX + \"T1\";\n        String group = \"G0\";\n        int queueId = 1;\n        long queueOffset = 100;\n        consumerOffsetManagerV2.commitOffset(clientHost, group, topic, queueId, queueOffset);\n        Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic, queueId));\n\n        configStorage.shutdown();\n\n        configStorage = new ConfigStorage(messageStoreConfig);\n        configStorage.start();\n        consumerOffsetManagerV2 = new ConsumerOffsetManagerV2(controller, configStorage);\n        consumerOffsetManagerV2.load();\n        Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic, queueId));\n    }\n\n\n    /**\n     * Verify commit offset can survive config store restart\n     */\n    @Test\n    public void testCommitPullOffset_LMQ() {\n        Assert.assertTrue(consumerOffsetManagerV2.load());\n\n        String clientHost = \"localhost\";\n        String topic = MixAll.LMQ_PREFIX + \"T1\";\n        String group = \"G0\";\n        int queueId = 1;\n        long queueOffset = 100;\n        consumerOffsetManagerV2.commitPullOffset(clientHost, group, topic, queueId, queueOffset);\n        Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryPullOffset(group, topic, queueId));\n\n        configStorage.shutdown();\n\n        configStorage = new ConfigStorage(messageStoreConfig);\n        configStorage.start();\n        consumerOffsetManagerV2 = new ConsumerOffsetManagerV2(controller, configStorage);\n        consumerOffsetManagerV2.load();\n        Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryPullOffset(group, topic, queueId));\n    }\n\n    /**\n     * Verify commit offset can survive config store restart\n     */\n    @Test\n    public void testRemoveByTopicAtGroup() {\n        Assert.assertTrue(consumerOffsetManagerV2.load());\n\n        String clientHost = \"localhost\";\n        String topic = MixAll.LMQ_PREFIX + \"T1\";\n        String topic2 = MixAll.LMQ_PREFIX + \"T2\";\n        String group = \"G0\";\n        int queueId = 1;\n        long queueOffset = 100;\n        consumerOffsetManagerV2.commitOffset(clientHost, group, topic, queueId, queueOffset);\n        consumerOffsetManagerV2.commitOffset(clientHost, group, topic2, queueId, queueOffset);\n        Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic, queueId));\n        Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic2, queueId));\n\n        consumerOffsetManagerV2.removeConsumerOffset(topic + ConsumerOffsetManager.TOPIC_GROUP_SEPARATOR + group);\n        Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic, queueId));\n        Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic2, queueId));\n\n        configStorage.shutdown();\n\n        configStorage = new ConfigStorage(messageStoreConfig);\n        configStorage.start();\n        consumerOffsetManagerV2 = new ConsumerOffsetManagerV2(controller, configStorage);\n        consumerOffsetManagerV2.load();\n        Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic, queueId));\n        Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic2, queueId));\n    }\n\n    /**\n     * Verify commit offset can survive config store restart\n     */\n    @Test\n    public void testRemoveByGroup() {\n        Assert.assertTrue(consumerOffsetManagerV2.load());\n\n        String clientHost = \"localhost\";\n        String topic = MixAll.LMQ_PREFIX + \"T1\";\n        String topic2 = MixAll.LMQ_PREFIX + \"T2\";\n        String group = \"G0\";\n        int queueId = 1;\n        long queueOffset = 100;\n        consumerOffsetManagerV2.commitOffset(clientHost, group, topic, queueId, queueOffset);\n        consumerOffsetManagerV2.commitOffset(clientHost, group, topic2, queueId, queueOffset);\n        Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic, queueId));\n        consumerOffsetManagerV2.removeOffset(group);\n        Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic, queueId));\n        Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic2, queueId));\n\n        configStorage.shutdown();\n\n        configStorage = new ConfigStorage(messageStoreConfig);\n        configStorage.start();\n        consumerOffsetManagerV2 = new ConsumerOffsetManagerV2(controller, configStorage);\n        consumerOffsetManagerV2.load();\n        Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic, queueId));\n        Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic2, queueId));\n    }\n\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2Test.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.config.v2;\n\nimport java.io.File;\nimport java.io.IOException;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.remoting.protocol.subscription.GroupRetryPolicy;\nimport org.apache.rocketmq.remoting.protocol.subscription.GroupRetryPolicyType;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Rule;\nimport org.junit.Test;\nimport org.junit.rules.TemporaryFolder;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class SubscriptionGroupManagerV2Test {\n\n    private MessageStoreConfig messageStoreConfig;\n\n    private ConfigStorage configStorage;\n\n    private SubscriptionGroupManagerV2 subscriptionGroupManagerV2;\n\n    @Mock\n    private BrokerController controller;\n\n    @Mock\n    private MessageStore messageStore;\n\n    @Rule\n    public TemporaryFolder tf = new TemporaryFolder();\n\n    @After\n    public void cleanUp() {\n        if (null != configStorage) {\n            configStorage.shutdown();\n        }\n    }\n\n    @Before\n    public void setUp() throws IOException {\n        BrokerConfig brokerConfig = new BrokerConfig();\n        brokerConfig.setAutoCreateSubscriptionGroup(false);\n        Mockito.doReturn(brokerConfig).when(controller).getBrokerConfig();\n\n        Mockito.doReturn(messageStore).when(controller).getMessageStore();\n        Mockito.doReturn(1L).when(messageStore).getStateMachineVersion();\n\n        File configStoreDir = tf.newFolder();\n        messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setStorePathRootDir(configStoreDir.getAbsolutePath());\n        configStorage = new ConfigStorage(messageStoreConfig);\n        configStorage.start();\n        subscriptionGroupManagerV2 = new SubscriptionGroupManagerV2(controller, configStorage);\n    }\n\n\n    @Test\n    public void testUpdateSubscriptionGroupConfig() {\n        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n        subscriptionGroupConfig.setGroupName(\"G1\");\n        subscriptionGroupConfig.setConsumeEnable(true);\n        subscriptionGroupConfig.setRetryMaxTimes(16);\n        subscriptionGroupConfig.setGroupSysFlag(1);\n        GroupRetryPolicy retryPolicy = new GroupRetryPolicy();\n        retryPolicy.setType(GroupRetryPolicyType.EXPONENTIAL);\n        subscriptionGroupConfig.setGroupRetryPolicy(retryPolicy);\n        subscriptionGroupConfig.setBrokerId(1);\n        subscriptionGroupConfig.setConsumeBroadcastEnable(true);\n        subscriptionGroupConfig.setConsumeMessageOrderly(true);\n        subscriptionGroupConfig.setConsumeTimeoutMinute(30);\n        subscriptionGroupConfig.setConsumeFromMinEnable(true);\n        subscriptionGroupConfig.setWhichBrokerWhenConsumeSlowly(1);\n        subscriptionGroupConfig.setNotifyConsumerIdsChangedEnable(true);\n        subscriptionGroupManagerV2.updateSubscriptionGroupConfig(subscriptionGroupConfig);\n\n        SubscriptionGroupConfig found = subscriptionGroupManagerV2.findSubscriptionGroupConfig(subscriptionGroupConfig.getGroupName());\n        Assert.assertEquals(subscriptionGroupConfig, found);\n\n        subscriptionGroupManagerV2.getSubscriptionGroupTable().clear();\n        configStorage.shutdown();\n\n        configStorage = new ConfigStorage(messageStoreConfig);\n        configStorage.start();\n        subscriptionGroupManagerV2 = new SubscriptionGroupManagerV2(controller, configStorage);\n        subscriptionGroupManagerV2.load();\n        found = subscriptionGroupManagerV2.findSubscriptionGroupConfig(subscriptionGroupConfig.getGroupName());\n        Assert.assertEquals(subscriptionGroupConfig, found);\n    }\n\n\n    @Test\n    public void testDeleteSubscriptionGroupConfig() {\n        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n        subscriptionGroupConfig.setGroupName(\"G1\");\n        subscriptionGroupConfig.setConsumeEnable(true);\n        subscriptionGroupConfig.setRetryMaxTimes(16);\n        subscriptionGroupConfig.setGroupSysFlag(1);\n        GroupRetryPolicy retryPolicy = new GroupRetryPolicy();\n        retryPolicy.setType(GroupRetryPolicyType.EXPONENTIAL);\n        subscriptionGroupConfig.setGroupRetryPolicy(retryPolicy);\n        subscriptionGroupConfig.setBrokerId(1);\n        subscriptionGroupConfig.setConsumeBroadcastEnable(true);\n        subscriptionGroupConfig.setConsumeMessageOrderly(true);\n        subscriptionGroupConfig.setConsumeTimeoutMinute(30);\n        subscriptionGroupConfig.setConsumeFromMinEnable(true);\n        subscriptionGroupConfig.setWhichBrokerWhenConsumeSlowly(1);\n        subscriptionGroupConfig.setNotifyConsumerIdsChangedEnable(true);\n        subscriptionGroupManagerV2.updateSubscriptionGroupConfig(subscriptionGroupConfig);\n\n        SubscriptionGroupConfig found = subscriptionGroupManagerV2.findSubscriptionGroupConfig(subscriptionGroupConfig.getGroupName());\n        Assert.assertEquals(subscriptionGroupConfig, found);\n        subscriptionGroupManagerV2.removeSubscriptionGroupConfig(subscriptionGroupConfig.getGroupName());\n\n        found = subscriptionGroupManagerV2.findSubscriptionGroupConfig(subscriptionGroupConfig.getGroupName());\n        Assert.assertNull(found);\n\n        configStorage.shutdown();\n\n        configStorage = new ConfigStorage(messageStoreConfig);\n        configStorage.start();\n\n        subscriptionGroupManagerV2 = new SubscriptionGroupManagerV2(controller, configStorage);\n        subscriptionGroupManagerV2.load();\n        found = subscriptionGroupManagerV2.findSubscriptionGroupConfig(subscriptionGroupConfig.getGroupName());\n        Assert.assertNull(found);\n    }\n\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2Test.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.config.v2;\n\nimport java.io.File;\nimport java.io.IOException;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Rule;\nimport org.junit.Test;\nimport org.junit.rules.TemporaryFolder;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\n\n@RunWith(value = MockitoJUnitRunner.class)\npublic class TopicConfigManagerV2Test {\n\n    private MessageStoreConfig messageStoreConfig;\n\n    private ConfigStorage configStorage;\n\n    @Mock\n    private BrokerController controller;\n\n    @Mock\n    private MessageStore messageStore;\n\n    @Rule\n    public TemporaryFolder tf = new TemporaryFolder();\n\n    @After\n    public void cleanUp() {\n        if (null != configStorage) {\n            configStorage.shutdown();\n        }\n    }\n\n    @Before\n    public void setUp() throws IOException {\n        BrokerConfig brokerConfig = new BrokerConfig();\n        Mockito.doReturn(brokerConfig).when(controller).getBrokerConfig();\n\n        messageStoreConfig = new MessageStoreConfig();\n        Mockito.doReturn(messageStoreConfig).when(controller).getMessageStoreConfig();\n        Mockito.doReturn(messageStore).when(controller).getMessageStore();\n\n        File configStoreDir = tf.newFolder();\n        messageStoreConfig.setStorePathRootDir(configStoreDir.getAbsolutePath());\n\n        configStorage = new ConfigStorage(messageStoreConfig);\n        configStorage.start();\n    }\n\n    @Test\n    public void testUpdateTopicConfig() {\n        TopicConfigManagerV2 topicConfigManagerV2 = new TopicConfigManagerV2(controller, configStorage);\n        topicConfigManagerV2.load();\n\n        TopicConfig topicConfig = new TopicConfig();\n        String topicName = \"T1\";\n        topicConfig.setTopicName(topicName);\n        topicConfig.setPerm(6);\n        topicConfig.setReadQueueNums(8);\n        topicConfig.setWriteQueueNums(4);\n        topicConfig.setOrder(true);\n        topicConfig.setTopicSysFlag(4);\n        topicConfigManagerV2.updateTopicConfig(topicConfig);\n\n        Assert.assertTrue(configStorage.shutdown());\n\n        topicConfigManagerV2.getTopicConfigTable().clear();\n\n        configStorage = new ConfigStorage(messageStoreConfig);\n        Assert.assertTrue(configStorage.start());\n        topicConfigManagerV2 = new TopicConfigManagerV2(controller, configStorage);\n        Assert.assertTrue(topicConfigManagerV2.load());\n\n        TopicConfig loaded = topicConfigManagerV2.selectTopicConfig(topicName);\n        Assert.assertNotNull(loaded);\n        Assert.assertEquals(topicName, loaded.getTopicName());\n        Assert.assertEquals(6, loaded.getPerm());\n        Assert.assertEquals(8, loaded.getReadQueueNums());\n        Assert.assertEquals(4, loaded.getWriteQueueNums());\n        Assert.assertTrue(loaded.isOrder());\n        Assert.assertEquals(4, loaded.getTopicSysFlag());\n\n        Assert.assertTrue(topicConfigManagerV2.containsTopic(topicName));\n    }\n\n    @Test\n    public void testRemoveTopicConfig() {\n        TopicConfig topicConfig = new TopicConfig();\n        String topicName = \"T1\";\n        topicConfig.setTopicName(topicName);\n        topicConfig.setPerm(6);\n        topicConfig.setReadQueueNums(8);\n        topicConfig.setWriteQueueNums(4);\n        topicConfig.setOrder(true);\n        topicConfig.setTopicSysFlag(4);\n        TopicConfigManagerV2 topicConfigManagerV2 = new TopicConfigManagerV2(controller, configStorage);\n        topicConfigManagerV2.updateTopicConfig(topicConfig);\n        topicConfigManagerV2.removeTopicConfig(topicName);\n        Assert.assertFalse(topicConfigManagerV2.containsTopic(topicName));\n        Assert.assertTrue(configStorage.shutdown());\n\n        configStorage = new ConfigStorage(messageStoreConfig);\n        Assert.assertTrue(configStorage.start());\n        topicConfigManagerV2 = new TopicConfigManagerV2(controller, configStorage);\n        Assert.assertTrue(topicConfigManagerV2.load());\n        Assert.assertFalse(topicConfigManagerV2.containsTopic(topicName));\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerRegisterTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.controller;\n\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.out.BrokerOuterAPI;\nimport org.apache.rocketmq.broker.slave.SlaveSynchronize;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerResponseHeader;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.RunningFlags;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService;\nimport org.apache.rocketmq.store.ha.autoswitch.BrokerMetadata;\nimport org.apache.rocketmq.store.ha.autoswitch.TempBrokerMetadata;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.io.File;\nimport java.time.Duration;\nimport java.util.Collections;\nimport java.util.HashSet;\nimport java.util.UUID;\n\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertNotEquals;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.Mockito.doThrow;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ReplicasManagerRegisterTest {\n\n    public static final String STORE_BASE_PATH = System.getProperty(\"java.io.tmpdir\") + File.separator + \"ReplicasManagerRegisterTest\";\n\n    public static final String STORE_PATH = STORE_BASE_PATH + File.separator + UUID.randomUUID();\n\n    public static final String BROKER_NAME = \"default-broker\";\n\n    public static final String CLUSTER_NAME = \"default-cluster\";\n\n    public static final String NAME_SRV_ADDR = \"127.0.0.1:9999\";\n\n    public static final String CONTROLLER_ADDR = \"127.0.0.1:8888\";\n\n    public static final BrokerConfig BROKER_CONFIG;\n    \n    private final HashSet<Long> syncStateSet = new HashSet<>(Collections.singletonList(1L));\n    \n    @Mock\n    private BrokerMetadata brokerMetadata;\n    \n    @Mock\n    private TempBrokerMetadata tempBrokerMetadata;\n\n    static {\n        BROKER_CONFIG = new BrokerConfig();\n        BROKER_CONFIG.setListenPort(21030);\n        BROKER_CONFIG.setNamesrvAddr(NAME_SRV_ADDR);\n        BROKER_CONFIG.setControllerAddr(CONTROLLER_ADDR);\n        BROKER_CONFIG.setSyncControllerMetadataPeriod(2 * 1000);\n        BROKER_CONFIG.setEnableControllerMode(true);\n        BROKER_CONFIG.setBrokerName(BROKER_NAME);\n        BROKER_CONFIG.setBrokerClusterName(CLUSTER_NAME);\n    }\n\n    private MessageStoreConfig buildMessageStoreConfig(int id) {\n        MessageStoreConfig config = new MessageStoreConfig();\n        config.setStorePathRootDir(STORE_PATH + File.separator + id);\n        config.setStorePathCommitLog(config.getStorePathRootDir() + File.separator + \"commitLog\");\n        config.setStorePathEpochFile(config.getStorePathRootDir() + File.separator + \"epochFileCache\");\n        config.setStorePathBrokerIdentity(config.getStorePathRootDir() + File.separator + \"brokerIdentity\");\n        return config;\n    }\n\n    private BrokerController mockedBrokerController;\n\n    private DefaultMessageStore mockedMessageStore;\n\n    private BrokerOuterAPI mockedBrokerOuterAPI;\n\n    private AutoSwitchHAService mockedAutoSwitchHAService;\n\n    private RunningFlags runningFlags = new RunningFlags();\n\n    @Before\n    public void setUp() throws Exception {\n        UtilAll.deleteFile(new File(STORE_BASE_PATH));\n        this.mockedBrokerController = Mockito.mock(BrokerController.class);\n        this.mockedMessageStore = Mockito.mock(DefaultMessageStore.class);\n        this.mockedBrokerOuterAPI = Mockito.mock(BrokerOuterAPI.class);\n        this.mockedAutoSwitchHAService = Mockito.mock(AutoSwitchHAService.class);\n        TopicConfigManager mockedTopicConfigManager = new TopicConfigManager();\n        when(mockedBrokerController.getBrokerOuterAPI()).thenReturn(mockedBrokerOuterAPI);\n        when(mockedBrokerController.getMessageStore()).thenReturn(mockedMessageStore);\n        when(mockedBrokerController.getBrokerConfig()).thenReturn(BROKER_CONFIG);\n        when(mockedBrokerController.getTopicConfigManager()).thenReturn(mockedTopicConfigManager);\n        when(mockedMessageStore.getHaService()).thenReturn(mockedAutoSwitchHAService);\n        when(mockedMessageStore.getRunningFlags()).thenReturn(runningFlags);\n        when(mockedBrokerController.getSlaveSynchronize()).thenReturn(new SlaveSynchronize(mockedBrokerController));\n        when(mockedBrokerOuterAPI.getControllerMetaData(any())).thenReturn(\n                new GetMetaDataResponseHeader(\"default-group\", \"dledger-a\", CONTROLLER_ADDR, true, CONTROLLER_ADDR));\n        when(mockedBrokerOuterAPI.checkAddressReachable(any())).thenReturn(true);\n        when(mockedBrokerController.getMessageStoreConfig()).thenReturn(buildMessageStoreConfig(0));\n    }\n\n    @Test\n    public void testBrokerRegisterSuccess() throws Exception {\n\n        when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 1L));\n        when(mockedBrokerOuterAPI.applyBrokerId(any(), any(), anyLong(), any(), any())).thenReturn(new ApplyBrokerIdResponseHeader());\n        when(mockedBrokerOuterAPI.registerBrokerToController(any(), any(), anyLong(), any(), any())).thenReturn(new Pair<>(new RegisterBrokerToControllerResponseHeader(),  syncStateSet));\n        when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong()))\n                .thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, \"127.0.0.1:13131\", 1, 1), syncStateSet));\n\n        ReplicasManager replicasManager0 = new ReplicasManager(mockedBrokerController);\n        replicasManager0.start();\n        await().atMost(Duration.ofMillis(1000)).until(() ->\n            replicasManager0.getState() == ReplicasManager.State.RUNNING\n        );\n        assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManager0.getRegisterState());\n        assertEquals(1L, replicasManager0.getBrokerControllerId().longValue());\n        checkMetadataFile(replicasManager0.getBrokerMetadata(), 1L);\n        assertFalse(replicasManager0.getTempBrokerMetadata().isLoaded());\n        assertFalse(replicasManager0.getTempBrokerMetadata().fileExists());\n        replicasManager0.shutdown();\n    }\n\n    @Test\n    public void testBrokerRegisterSuccessAndRestartWithChangedBrokerConfig() throws Exception {\n        when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 1L));\n        when(mockedBrokerOuterAPI.applyBrokerId(any(), any(), anyLong(), any(), any())).thenReturn(new ApplyBrokerIdResponseHeader());\n        when(mockedBrokerOuterAPI.registerBrokerToController(any(), any(), anyLong(), any(), any())).thenReturn(new Pair<>(new RegisterBrokerToControllerResponseHeader(),  syncStateSet));\n        when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())).thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, \"127.0.0.1:13131\", 1, 1), syncStateSet));\n\n        ReplicasManager replicasManager0 = new ReplicasManager(mockedBrokerController);\n        replicasManager0.start();\n        await().atMost(Duration.ofMillis(1000)).until(() ->\n                replicasManager0.getState() == ReplicasManager.State.RUNNING\n        );\n        assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManager0.getRegisterState());\n        assertEquals(1L, replicasManager0.getBrokerControllerId().longValue());\n        checkMetadataFile(replicasManager0.getBrokerMetadata(), 1L);\n        assertFalse(replicasManager0.getTempBrokerMetadata().isLoaded());\n        assertFalse(replicasManager0.getTempBrokerMetadata().fileExists());\n        replicasManager0.shutdown();\n\n        // change broker name in broker config\n        mockedBrokerController.getBrokerConfig().setBrokerName(BROKER_NAME + \"1\");\n        ReplicasManager replicasManagerRestart = new ReplicasManager(mockedBrokerController);\n        replicasManagerRestart.start();\n        assertEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManagerRestart.getRegisterState());\n        mockedBrokerController.getBrokerConfig().setBrokerName(BROKER_NAME);\n        replicasManagerRestart.shutdown();\n\n        // change cluster name in broker config\n        mockedBrokerController.getBrokerConfig().setBrokerClusterName(CLUSTER_NAME + \"1\");\n        replicasManagerRestart = new ReplicasManager(mockedBrokerController);\n        replicasManagerRestart.start();\n        assertEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManagerRestart.getRegisterState());\n        mockedBrokerController.getBrokerConfig().setBrokerClusterName(CLUSTER_NAME);\n        replicasManagerRestart.shutdown();\n    }\n\n    @Test\n    public void testRegisterFailedAtGetNextBrokerId() throws Exception {\n        ReplicasManager replicasManager = new ReplicasManager(mockedBrokerController);\n        when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenThrow(new RuntimeException());\n\n        replicasManager.start();\n        \n        assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, replicasManager.getState());\n        assertEquals(ReplicasManager.RegisterState.INITIAL, replicasManager.getRegisterState());\n        assertFalse(replicasManager.getTempBrokerMetadata().fileExists());\n        assertFalse(replicasManager.getBrokerMetadata().fileExists());\n        assertNull(replicasManager.getBrokerControllerId());\n        replicasManager.shutdown();\n    }\n\n    @Test\n    public void testRegisterFailedAtCreateTempFile() throws Exception {\n        ReplicasManager spyReplicasManager = new ReplicasManager(mockedBrokerController);\n        when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 1L));\n        FieldUtils.writeDeclaredField(spyReplicasManager, \"tempBrokerMetadata\", tempBrokerMetadata, true);\n        doThrow(new RuntimeException(\"Test exception\")).when(tempBrokerMetadata).updateAndPersist(any(), any(), anyLong(), any());\n\n        spyReplicasManager.start();\n        \n        assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, spyReplicasManager.getState());\n        assertEquals(ReplicasManager.RegisterState.INITIAL, spyReplicasManager.getRegisterState());\n        assertFalse(spyReplicasManager.getTempBrokerMetadata().fileExists());\n        assertFalse(spyReplicasManager.getBrokerMetadata().fileExists());\n        assertNull(spyReplicasManager.getBrokerControllerId());\n        spyReplicasManager.shutdown();\n    }\n\n    @Test\n    public void testRegisterFailedAtApplyBrokerIdFailed() throws Exception {\n        ReplicasManager replicasManager = new ReplicasManager(mockedBrokerController);\n        when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 1L));\n        when(mockedBrokerOuterAPI.applyBrokerId(any(), any(), anyLong(), any(), any())).thenThrow(new RuntimeException());\n\n        replicasManager.start();\n        \n        assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, replicasManager.getState());\n        assertNotEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManager.getRegisterState());\n        assertNotEquals(ReplicasManager.RegisterState.REGISTERED, replicasManager.getRegisterState());\n\n        replicasManager.shutdown();\n        \n        assertFalse(replicasManager.getBrokerMetadata().fileExists());\n        assertNull(replicasManager.getBrokerControllerId());\n    }\n\n    @Test\n    public void testRegisterFailedAtCreateMetadataFileAndDeleteTemp() throws Exception {\n        ReplicasManager spyReplicasManager = new ReplicasManager(mockedBrokerController);\n        when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 1L));\n        when(mockedBrokerOuterAPI.applyBrokerId(any(), any(), anyLong(), any(), any())).thenReturn(new ApplyBrokerIdResponseHeader());\n        when(mockedBrokerOuterAPI.registerBrokerToController(any(), any(), anyLong(), any(), any())).thenReturn(new Pair<>(new RegisterBrokerToControllerResponseHeader(),  syncStateSet));\n        when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())).thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, \"127.0.0.1:13131\", 1, 1), syncStateSet));\n        \n        FieldUtils.writeDeclaredField(spyReplicasManager, \"brokerMetadata\", brokerMetadata, true);\n        doThrow(new RuntimeException(\"Test exception\")).when(brokerMetadata).updateAndPersist(any(), any(), anyLong());\n\n        spyReplicasManager.start();\n        \n        assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, spyReplicasManager.getState());\n        assertEquals(ReplicasManager.RegisterState.CREATE_TEMP_METADATA_FILE_DONE, spyReplicasManager.getRegisterState());\n        TempBrokerMetadata tempBrokerMetadata = spyReplicasManager.getTempBrokerMetadata();\n        assertTrue(tempBrokerMetadata.fileExists());\n        assertTrue(tempBrokerMetadata.isLoaded());\n        assertFalse(spyReplicasManager.getBrokerMetadata().fileExists());\n        assertNull(spyReplicasManager.getBrokerControllerId());\n\n        spyReplicasManager.shutdown();\n\n        // restart, we expect that this replicasManager still keep the tempMetadata and still try to finish its registering\n        ReplicasManager replicasManagerNew = new ReplicasManager(mockedBrokerController);\n\n        replicasManagerNew.start();\n        \n        assertEquals(ReplicasManager.State.RUNNING, replicasManagerNew.getState());\n        assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManagerNew.getRegisterState());\n        // tempMetadata has been cleared\n        assertFalse(replicasManagerNew.getTempBrokerMetadata().fileExists());\n        assertFalse(replicasManagerNew.getTempBrokerMetadata().isLoaded());\n        // metadata has been persisted\n        assertTrue(replicasManagerNew.getBrokerMetadata().fileExists());\n        assertTrue(replicasManagerNew.getBrokerMetadata().isLoaded());\n        assertEquals(1L, replicasManagerNew.getBrokerMetadata().getBrokerId().longValue());\n        assertEquals(1L, replicasManagerNew.getBrokerControllerId().longValue());\n        replicasManagerNew.shutdown();\n    }\n\n    @Test\n    public void testRegisterFailedAtRegisterSuccess() throws Exception {\n        ReplicasManager replicasManager = new ReplicasManager(mockedBrokerController);\n        when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 1L));\n        when(mockedBrokerOuterAPI.applyBrokerId(any(), any(), anyLong(), any(), any())).thenReturn(new ApplyBrokerIdResponseHeader());\n        when(mockedBrokerOuterAPI.registerBrokerToController(any(), any(), anyLong(), any(), any())).thenThrow(new RuntimeException());\n        when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())).thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, \"127.0.0.1:13131\", 1, 1), syncStateSet));\n\n        replicasManager.start();\n        \n        assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, replicasManager.getState());\n        assertEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManager.getRegisterState());\n        TempBrokerMetadata tempBrokerMetadata = replicasManager.getTempBrokerMetadata();\n        // temp metadata has been cleared\n        assertFalse(tempBrokerMetadata.fileExists());\n        assertFalse(tempBrokerMetadata.isLoaded());\n        // metadata has been persisted\n        assertTrue(replicasManager.getBrokerMetadata().fileExists());\n        assertTrue(replicasManager.getBrokerMetadata().isLoaded());\n        assertEquals(1L, replicasManager.getBrokerMetadata().getBrokerId().longValue());\n        assertEquals(1L, replicasManager.getBrokerControllerId().longValue());\n\n        replicasManager.shutdown();\n\n        Mockito.reset(mockedBrokerOuterAPI);\n        when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong()))\n                .thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, \"127.0.0.1:13131\", 1, 1), syncStateSet));\n        when(mockedBrokerOuterAPI.getControllerMetaData(any())).thenReturn(\n                new GetMetaDataResponseHeader(\"default-group\", \"dledger-a\", CONTROLLER_ADDR, true, CONTROLLER_ADDR));\n        when(mockedBrokerOuterAPI.checkAddressReachable(any())).thenReturn(true);\n\n        // restart, we expect that this replicasManager still keep the metadata and still try to finish its registering\n        ReplicasManager replicasManagerNew = new ReplicasManager(mockedBrokerController);\n        when(mockedBrokerOuterAPI.registerBrokerToController(any(), any(), anyLong(), any(), any())).thenReturn(new Pair<>(new RegisterBrokerToControllerResponseHeader(),  syncStateSet));\n        replicasManagerNew.start();\n        \n        assertEquals(ReplicasManager.State.RUNNING, replicasManagerNew.getState());\n        assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManagerNew.getRegisterState());\n        // tempMetadata has been cleared\n        assertFalse(replicasManagerNew.getTempBrokerMetadata().fileExists());\n        assertFalse(replicasManagerNew.getTempBrokerMetadata().isLoaded());\n        // metadata has been persisted\n        assertTrue(replicasManagerNew.getBrokerMetadata().fileExists());\n        assertTrue(replicasManagerNew.getBrokerMetadata().isLoaded());\n        assertEquals(1L, replicasManagerNew.getBrokerMetadata().getBrokerId().longValue());\n        assertEquals(1L, replicasManagerNew.getBrokerControllerId().longValue());\n        replicasManagerNew.shutdown();\n    }\n\n\n    private void checkMetadataFile(BrokerMetadata brokerMetadata0 ,Long brokerId) throws Exception {\n        assertEquals(brokerId, brokerMetadata0.getBrokerId());\n        assertTrue(brokerMetadata0.fileExists());\n        BrokerMetadata brokerMetadata = new BrokerMetadata(brokerMetadata0.getFilePath());\n        brokerMetadata.readFromFile();\n        assertEquals(brokerMetadata0, brokerMetadata);\n    }\n\n    @After\n    public void clear() {\n        UtilAll.deleteFile(new File(STORE_BASE_PATH));\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.controller;\n\nimport com.google.common.collect.Lists;\nimport java.io.File;\nimport java.lang.reflect.Method;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.UUID;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.out.BrokerOuterAPI;\nimport org.apache.rocketmq.broker.slave.SlaveSynchronize;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.remoting.protocol.body.SyncStateSet;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerResponseHeader;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.RunningFlags;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService;\nimport org.assertj.core.api.Assertions;\nimport org.assertj.core.util.Sets;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ReplicasManagerTest {\n\n    public static final String STORE_BASE_PATH = System.getProperty(\"java.io.tmpdir\") + File.separator + \"ReplicasManagerTest\";\n\n    public static final String STORE_PATH = STORE_BASE_PATH + File.separator + UUID.randomUUID();\n\n    @Mock\n    private BrokerController brokerController;\n\n    private ReplicasManager replicasManager;\n\n    @Mock\n    private DefaultMessageStore defaultMessageStore;\n\n    private SlaveSynchronize slaveSynchronize;\n\n    private AutoSwitchHAService autoSwitchHAService;\n\n    private MessageStoreConfig messageStoreConfig;\n\n    private GetMetaDataResponseHeader getMetaDataResponseHeader;\n\n    private BrokerConfig brokerConfig;\n\n    @Mock\n    private BrokerOuterAPI brokerOuterAPI;\n\n    private GetNextBrokerIdResponseHeader getNextBrokerIdResponseHeader;\n\n    private ApplyBrokerIdResponseHeader applyBrokerIdResponseHeader;\n\n    private RegisterBrokerToControllerResponseHeader registerBrokerToControllerResponseHeader;\n\n    private ElectMasterResponseHeader brokerTryElectResponseHeader;\n\n    private Pair<GetReplicaInfoResponseHeader, SyncStateSet> result;\n\n    private GetReplicaInfoResponseHeader getReplicaInfoResponseHeader;\n\n    private SyncStateSet syncStateSet;\n\n    private RunningFlags runningFlags = new RunningFlags();\n\n    private static final String OLD_MASTER_ADDRESS = \"192.168.1.1\";\n\n    private static final String NEW_MASTER_ADDRESS = \"192.168.1.2\";\n\n    private static final long BROKER_ID_1 = 1;\n\n    private static final long BROKER_ID_2 = 2;\n\n    private static final int OLD_MASTER_EPOCH = 2;\n    private static final int NEW_MASTER_EPOCH = 3;\n\n    private static final String GROUP = \"DEFAULT_GROUP\";\n\n    private static final String LEADER_ID = \"leader-1\";\n\n    private static final Boolean IS_LEADER = true;\n\n    private static final String PEERS = \"1.1.1.1\";\n\n    private static final long SCHEDULE_SERVICE_EXEC_PERIOD = 5;\n\n    private static final Long SYNC_STATE = 1L;\n\n    private static final HashSet<Long> SYNC_STATE_SET_1 = new HashSet<Long>(Arrays.asList(BROKER_ID_1));\n\n    private static final HashSet<Long> SYNC_STATE_SET_2 = new HashSet<Long>(Arrays.asList(BROKER_ID_2));\n\n    @Before\n    public void before() throws Exception {\n        UtilAll.deleteFile(new File(STORE_BASE_PATH));\n        autoSwitchHAService = new AutoSwitchHAService();\n        messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setStorePathRootDir(STORE_PATH);\n        brokerConfig = new BrokerConfig();\n        slaveSynchronize = new SlaveSynchronize(brokerController);\n        getMetaDataResponseHeader = new GetMetaDataResponseHeader(GROUP, LEADER_ID, OLD_MASTER_ADDRESS, IS_LEADER, PEERS);\n        getNextBrokerIdResponseHeader = new GetNextBrokerIdResponseHeader();\n        getNextBrokerIdResponseHeader.setNextBrokerId(BROKER_ID_1);\n        applyBrokerIdResponseHeader = new ApplyBrokerIdResponseHeader();\n        registerBrokerToControllerResponseHeader = new RegisterBrokerToControllerResponseHeader();\n        brokerTryElectResponseHeader = new ElectMasterResponseHeader();\n        brokerTryElectResponseHeader.setMasterBrokerId(BROKER_ID_1);\n        brokerTryElectResponseHeader.setMasterAddress(OLD_MASTER_ADDRESS);\n        brokerTryElectResponseHeader.setMasterEpoch(OLD_MASTER_EPOCH);\n        brokerTryElectResponseHeader.setSyncStateSetEpoch(OLD_MASTER_EPOCH);\n        getReplicaInfoResponseHeader = new GetReplicaInfoResponseHeader();\n        getReplicaInfoResponseHeader.setMasterAddress(OLD_MASTER_ADDRESS);\n        getReplicaInfoResponseHeader.setMasterBrokerId(BROKER_ID_1);\n        getReplicaInfoResponseHeader.setMasterEpoch(NEW_MASTER_EPOCH);\n        syncStateSet = new SyncStateSet(Sets.newLinkedHashSet(SYNC_STATE), NEW_MASTER_EPOCH);\n        result = new Pair<>(getReplicaInfoResponseHeader, syncStateSet);\n        TopicConfigManager topicConfigManager = new TopicConfigManager();\n        when(defaultMessageStore.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n        when(brokerController.getMessageStore()).thenReturn(defaultMessageStore);\n        when(brokerController.getMessageStore().getHaService()).thenReturn(autoSwitchHAService);\n        when(brokerController.getMessageStore().getRunningFlags()).thenReturn(runningFlags);\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n        when(brokerController.getSlaveSynchronize()).thenReturn(slaveSynchronize);\n        when(brokerController.getBrokerOuterAPI()).thenReturn(brokerOuterAPI);\n        when(brokerController.getBrokerAddr()).thenReturn(OLD_MASTER_ADDRESS);\n        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);\n        when(brokerOuterAPI.getControllerMetaData(any())).thenReturn(getMetaDataResponseHeader);\n        when(brokerOuterAPI.checkAddressReachable(any())).thenReturn(true);\n        when(brokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(getNextBrokerIdResponseHeader);\n        when(brokerOuterAPI.applyBrokerId(any(), any(), anyLong(), any(), any())).thenReturn(applyBrokerIdResponseHeader);\n        when(brokerOuterAPI.registerBrokerToController(any(), any(), anyLong(), any(), any())).thenReturn(new Pair<>(new RegisterBrokerToControllerResponseHeader(), SYNC_STATE_SET_1));\n        when(brokerOuterAPI.getReplicaInfo(any(), any())).thenReturn(result);\n        when(brokerOuterAPI.brokerElect(any(), any(), any(), any())).thenReturn(new Pair<>(brokerTryElectResponseHeader, SYNC_STATE_SET_1));\n        replicasManager = new ReplicasManager(brokerController);\n        autoSwitchHAService.init(defaultMessageStore);\n        replicasManager.start();\n        // execute schedulingSyncBrokerMetadata()\n        TimeUnit.SECONDS.sleep(SCHEDULE_SERVICE_EXEC_PERIOD);\n    }\n\n    @After\n    public void after() {\n        replicasManager.shutdown();\n        brokerController.shutdown();\n        UtilAll.deleteFile(new File(STORE_BASE_PATH));\n    }\n\n    @Test\n    public void changeBrokerRoleTest() {\n        HashSet<Long> syncStateSetA = new HashSet<>();\n        syncStateSetA.add(BROKER_ID_1);\n        HashSet<Long> syncStateSetB = new HashSet<>();\n        syncStateSetA.add(BROKER_ID_2);\n        // not equal to localAddress\n        Assertions.assertThatCode(() -> replicasManager.changeBrokerRole(BROKER_ID_2, NEW_MASTER_ADDRESS, NEW_MASTER_EPOCH, OLD_MASTER_EPOCH, syncStateSetB))\n            .doesNotThrowAnyException();\n\n        // equal to localAddress\n        Assertions.assertThatCode(() -> replicasManager.changeBrokerRole(BROKER_ID_1, OLD_MASTER_ADDRESS, NEW_MASTER_EPOCH, OLD_MASTER_EPOCH, syncStateSetA))\n            .doesNotThrowAnyException();\n    }\n\n    @Test\n    public void changeToMasterTest() {\n        HashSet<Long> syncStateSet = new HashSet<>();\n        syncStateSet.add(BROKER_ID_1);\n        Assertions.assertThatCode(() -> replicasManager.changeToMaster(NEW_MASTER_EPOCH, OLD_MASTER_EPOCH, syncStateSet)).doesNotThrowAnyException();\n    }\n\n    @Test\n    public void changeToSlaveTest() {\n        Assertions.assertThatCode(() -> replicasManager.changeToSlave(NEW_MASTER_ADDRESS, NEW_MASTER_EPOCH, BROKER_ID_2))\n            .doesNotThrowAnyException();\n    }\n\n    @Test\n    public void testUpdateControllerAddr() throws Exception {\n        final String controllerAddr = \"192.168.1.1\";\n        brokerConfig.setFetchControllerAddrByDnsLookup(true);\n        when(brokerOuterAPI.dnsLookupAddressByDomain(anyString())).thenReturn(Lists.newArrayList(controllerAddr));\n        Method method = ReplicasManager.class.getDeclaredMethod(\"updateControllerAddr\");\n        method.setAccessible(true);\n        method.invoke(replicasManager);\n\n        List<String> addresses = replicasManager.getControllerAddresses();\n        Assertions.assertThat(addresses).contains(controllerAddr);\n\n        // Simulating dns resolution exceptions\n        when(brokerOuterAPI.dnsLookupAddressByDomain(anyString())).thenReturn(new ArrayList<>());\n\n        method.invoke(replicasManager);\n        addresses = replicasManager.getControllerAddresses();\n        Assertions.assertThat(addresses).contains(controllerAddr);\n\n    }\n\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/failover/EscapeBridgeTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.failover;\n\nimport java.net.InetSocketAddress;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.commons.lang3.tuple.Triple;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.out.BrokerOuterAPI;\nimport org.apache.rocketmq.broker.topic.TopicRouteInfoManager;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.client.impl.producer.TopicPublishInfo;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.logfile.DefaultMappedFile;\nimport org.apache.rocketmq.store.logfile.MappedFile;\nimport org.apache.rocketmq.tieredstore.TieredMessageStore;\nimport org.assertj.core.api.Assertions;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyBoolean;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.when;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.eq;\nimport static org.mockito.Mockito.verify;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class EscapeBridgeTest {\n\n    private EscapeBridge escapeBridge;\n\n    @Mock\n    private BrokerController brokerController;\n\n    @Mock\n    private MessageExtBrokerInner messageExtBrokerInner;\n\n    private BrokerConfig brokerConfig;\n\n    @Mock\n    private DefaultMessageStore defaultMessageStore;\n\n    @Mock\n    private TieredMessageStore tieredMessageStore;\n\n    private GetMessageResult getMessageResult;\n\n    @Mock\n    private DefaultMQProducer defaultMQProducer;\n\n    @Mock\n    private TopicRouteInfoManager topicRouteInfoManager;\n\n    @Mock\n    private BrokerOuterAPI brokerOuterAPI;\n\n    private static final String BROKER_NAME = \"broker_a\";\n\n    private static final String TEST_TOPIC = \"TEST_TOPIC\";\n\n    private static final int DEFAULT_QUEUE_ID = 0;\n\n\n    @Before\n    public void before() throws Exception {\n        brokerConfig = new BrokerConfig();\n        getMessageResult = new GetMessageResult();\n        brokerConfig.setBrokerName(BROKER_NAME);\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n\n        escapeBridge = new EscapeBridge(brokerController);\n        messageExtBrokerInner = new MessageExtBrokerInner();\n        when(brokerController.getMessageStore()).thenReturn(defaultMessageStore);\n        when(defaultMessageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(CompletableFuture.completedFuture(getMessageResult));\n\n        when(brokerController.getTopicRouteInfoManager()).thenReturn(topicRouteInfoManager);\n        when(topicRouteInfoManager.findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean())).thenReturn(\"\");\n\n        when(brokerController.getBrokerOuterAPI()).thenReturn(brokerOuterAPI);\n\n        brokerConfig.setEnableSlaveActingMaster(true);\n        brokerConfig.setEnableRemoteEscape(true);\n        escapeBridge.start();\n        defaultMQProducer.start();\n    }\n\n    @After\n    public void after() {\n        escapeBridge.shutdown();\n        brokerController.shutdown();\n        defaultMQProducer.shutdown();\n    }\n\n    @Test\n    public void putMessageTest() {\n        messageExtBrokerInner.setTopic(TEST_TOPIC);\n        messageExtBrokerInner.setQueueId(DEFAULT_QUEUE_ID);\n        messageExtBrokerInner.setBody(\"Hello World\".getBytes(StandardCharsets.UTF_8));\n        // masterBroker is null\n        final PutMessageResult result1 = escapeBridge.putMessage(messageExtBrokerInner);\n        assert result1 != null;\n        assert PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL.equals(result1.getPutMessageStatus());\n\n        // masterBroker is not null\n        messageExtBrokerInner.setBody(\"Hello World2\".getBytes(StandardCharsets.UTF_8));\n        when(brokerController.peekMasterBroker()).thenReturn(brokerController);\n        Assertions.assertThatCode(() -> escapeBridge.putMessage(messageExtBrokerInner)).doesNotThrowAnyException();\n\n        when(brokerController.peekMasterBroker()).thenReturn(null);\n        final PutMessageResult result3 = escapeBridge.putMessage(messageExtBrokerInner);\n        assert result3 != null;\n        assert PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL.equals(result3.getPutMessageStatus());\n    }\n\n    @Test\n    public void asyncPutMessageTest() {\n\n        // masterBroker is null\n        Assertions.assertThatCode(() -> escapeBridge.asyncPutMessage(messageExtBrokerInner)).doesNotThrowAnyException();\n\n        // masterBroker is not null\n        when(brokerController.peekMasterBroker()).thenReturn(brokerController);\n        Assertions.assertThatCode(() -> escapeBridge.asyncPutMessage(messageExtBrokerInner)).doesNotThrowAnyException();\n\n        when(brokerController.peekMasterBroker()).thenReturn(null);\n        Assertions.assertThatCode(() -> escapeBridge.asyncPutMessage(messageExtBrokerInner)).doesNotThrowAnyException();\n    }\n\n    @Test\n    public void putMessageToSpecificQueueTest() {\n        // masterBroker is null\n        final PutMessageResult result1 = escapeBridge.putMessageToSpecificQueue(messageExtBrokerInner);\n        assert result1 != null;\n        assert PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL.equals(result1.getPutMessageStatus());\n\n        // masterBroker is not null\n        when(brokerController.peekMasterBroker()).thenReturn(brokerController);\n        Assertions.assertThatCode(() -> escapeBridge.putMessageToSpecificQueue(messageExtBrokerInner)).doesNotThrowAnyException();\n    }\n\n    @Test\n    public void getMessageTest() {\n        when(brokerController.peekMasterBroker()).thenReturn(brokerController);\n        when(brokerController.getMessageStoreByBrokerName(any())).thenReturn(defaultMessageStore);\n        Assertions.assertThatCode(() -> escapeBridge.putMessage(messageExtBrokerInner)).doesNotThrowAnyException();\n\n        Assertions.assertThatCode(() -> escapeBridge.getMessage(TEST_TOPIC, 0, DEFAULT_QUEUE_ID, BROKER_NAME, false)).doesNotThrowAnyException();\n    }\n\n    @Test\n    public void getMessageAsyncTest() {\n        when(brokerController.peekMasterBroker()).thenReturn(brokerController);\n        when(brokerController.getMessageStoreByBrokerName(any())).thenReturn(defaultMessageStore);\n        Assertions.assertThatCode(() -> escapeBridge.putMessage(messageExtBrokerInner)).doesNotThrowAnyException();\n\n        Assertions.assertThatCode(() -> escapeBridge.getMessageAsync(TEST_TOPIC, 0, DEFAULT_QUEUE_ID, BROKER_NAME, false)).doesNotThrowAnyException();\n    }\n\n    @Test\n    public void getMessageAsyncTest_localStore_getMessageAsync_null() {\n        when(brokerController.getMessageStoreByBrokerName(any())).thenReturn(defaultMessageStore);\n        when(defaultMessageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any()))\n                .thenReturn(CompletableFuture.completedFuture(null));\n        Triple<MessageExt, String, Boolean> rst = escapeBridge.getMessageAsync(TEST_TOPIC, 0, DEFAULT_QUEUE_ID, BROKER_NAME, false).join();\n        Assert.assertNull(rst.getLeft());\n        Assert.assertEquals(\"getMessageResult is null\", rst.getMiddle());\n        Assert.assertFalse(rst.getRight()); // no retry\n    }\n\n    @Test\n    public void getMessageAsyncTest_localStore_decodeNothing_DefaultMessageStore() throws Exception {\n        when(brokerController.getMessageStoreByBrokerName(any())).thenReturn(defaultMessageStore);\n        for (GetMessageStatus status : GetMessageStatus.values()) {\n            GetMessageResult getMessageResult = mockGetMessageResult(0, TEST_TOPIC, null);\n            getMessageResult.setStatus(status);\n            when(defaultMessageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any()))\n                    .thenReturn(CompletableFuture.completedFuture(getMessageResult));\n            Triple<MessageExt, String, Boolean> rst = escapeBridge.getMessageAsync(TEST_TOPIC, 0, DEFAULT_QUEUE_ID, BROKER_NAME, false).join();\n            Assert.assertNull(rst.getLeft());\n            Assert.assertEquals(\"Can not get msg\", rst.getMiddle());\n            Assert.assertFalse(rst.getRight()); // DefaultMessageStore, no retry\n        }\n    }\n\n    @Test\n    public void getMessageAsyncTest_localStore_decodeNothing_TieredMessageStore() throws Exception {\n        when(brokerController.getMessageStoreByBrokerName(any())).thenReturn(tieredMessageStore);\n        for (GetMessageStatus status : GetMessageStatus.values()) {\n            GetMessageResult getMessageResult = new GetMessageResult();\n            getMessageResult.setStatus(status);\n            when(tieredMessageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any()))\n                    .thenReturn(CompletableFuture.completedFuture(getMessageResult));\n            Triple<MessageExt, String, Boolean> rst = escapeBridge.getMessageAsync(TEST_TOPIC, 0, DEFAULT_QUEUE_ID, BROKER_NAME, false).join();\n            Assert.assertNull(rst.getLeft());\n            Assert.assertEquals(\"Can not get msg\", rst.getMiddle());\n            if (GetMessageStatus.OFFSET_FOUND_NULL.equals(status)) {\n                Assert.assertTrue(rst.getRight()); // TieredMessageStore returns OFFSET_FOUND_NULL, need retry\n            } else {\n                Assert.assertFalse(rst.getRight()); // other status, like DefaultMessageStore, no retry\n            }\n        }\n    }\n\n    @Test\n    public void getMessageAsyncTest_localStore_message_found() throws Exception {\n        when(brokerController.getMessageStoreByBrokerName(any())).thenReturn(defaultMessageStore);\n        when(defaultMessageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any()))\n                .thenReturn(CompletableFuture.completedFuture(mockGetMessageResult(2, TEST_TOPIC, \"HW\".getBytes())));\n        Triple<MessageExt, String, Boolean> rst = escapeBridge.getMessageAsync(TEST_TOPIC, 0, DEFAULT_QUEUE_ID, BROKER_NAME, false).join();\n        Assert.assertNotNull(rst.getLeft());\n        Assert.assertEquals(0, rst.getLeft().getQueueOffset());\n        Assert.assertTrue(Arrays.equals(\"HW\".getBytes(), rst.getLeft().getBody()));\n        Assert.assertFalse(rst.getRight());\n    }\n\n    @Test\n    public void getMessageAsyncTest_remoteStore_addressNotFound() throws Exception {\n        when(brokerController.getMessageStoreByBrokerName(any())).thenReturn(null);\n\n        // just test address not found, since we have complete tests of getMessageFromRemoteAsync()\n        when(topicRouteInfoManager.findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean())).thenReturn(null);\n        Triple<MessageExt, String, Boolean> rst = escapeBridge.getMessageAsync(TEST_TOPIC, 0, DEFAULT_QUEUE_ID, BROKER_NAME, false).join();\n        Assert.assertNull(rst.getLeft());\n        Assert.assertEquals(\"brokerAddress not found\", rst.getMiddle());\n        Assert.assertTrue(rst.getRight()); // need retry\n    }\n\n    @Test\n    public void getMessageFromRemoteTest() {\n        Assertions.assertThatCode(() -> escapeBridge.getMessageFromRemote(TEST_TOPIC, 1, DEFAULT_QUEUE_ID, BROKER_NAME)).doesNotThrowAnyException();\n    }\n\n    @Test\n    public void getMessageFromRemoteAsyncTest() {\n        Assertions.assertThatCode(() -> escapeBridge.getMessageFromRemoteAsync(TEST_TOPIC, 1, DEFAULT_QUEUE_ID, BROKER_NAME)).doesNotThrowAnyException();\n    }\n\n    @Test\n    public void getMessageFromRemoteAsyncTest_exception_caught() throws Exception {\n        when(brokerOuterAPI.pullMessageFromSpecificBrokerAsync(anyString(), anyString(), anyString(), anyString(), anyInt(), anyLong(), anyInt(), anyLong()))\n                .thenThrow(new RemotingException(\"mock remoting exception\"));\n        Triple<MessageExt, String, Boolean> rst = escapeBridge.getMessageFromRemoteAsync(TEST_TOPIC, 1, DEFAULT_QUEUE_ID, BROKER_NAME).join();\n        Assert.assertNull(rst.getLeft());\n        Assert.assertEquals(\"Get message from remote failed\", rst.getMiddle());\n        Assert.assertTrue(rst.getRight()); // need retry\n    }\n\n    @Test\n    public void getMessageFromRemoteAsyncTest_brokerAddressNotFound() throws Exception {\n        when(topicRouteInfoManager.findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean())).thenReturn(null);\n        Triple<MessageExt, String, Boolean> rst = escapeBridge.getMessageFromRemoteAsync(TEST_TOPIC, 1, DEFAULT_QUEUE_ID, BROKER_NAME).join();\n        Assert.assertNull(rst.getLeft());\n        Assert.assertEquals(\"brokerAddress not found\", rst.getMiddle());\n        Assert.assertTrue(rst.getRight()); // need retry\n    }\n\n    @Test\n    public void getMessageFromRemoteAsyncTest_message_found() throws Exception {\n        PullResult pullResult = new PullResult(PullStatus.FOUND, 1, 1, 1, Arrays.asList(new MessageExt()));\n        when(brokerOuterAPI.pullMessageFromSpecificBrokerAsync(anyString(), anyString(), anyString(), anyString(), anyInt(), anyLong(), anyInt(), anyLong()))\n                .thenReturn(CompletableFuture.completedFuture(Triple.of(pullResult, \"\", false))); // right value is ignored\n        Triple<MessageExt, String, Boolean> rst = escapeBridge.getMessageFromRemoteAsync(TEST_TOPIC, 1, DEFAULT_QUEUE_ID, BROKER_NAME).join();\n        Assert.assertNotNull(rst.getLeft());\n        Assert.assertTrue(StringUtils.isEmpty(rst.getMiddle()));\n        Assert.assertFalse(rst.getRight()); // no retry\n    }\n\n    @Test\n    public void getMessageFromRemoteAsyncTest_message_notFound() throws Exception {\n        PullResult pullResult = new PullResult(PullStatus.NO_MATCHED_MSG, 1, 1, 1, null);\n        when(brokerOuterAPI.pullMessageFromSpecificBrokerAsync(anyString(), anyString(), anyString(), anyString(), anyInt(), anyLong(), anyInt(), anyLong()))\n                .thenReturn(CompletableFuture.completedFuture(Triple.of(pullResult, \"no msg\", false)));\n        Triple<MessageExt, String, Boolean> rst = escapeBridge.getMessageFromRemoteAsync(TEST_TOPIC, 1, DEFAULT_QUEUE_ID, BROKER_NAME).join();\n        Assert.assertNull(rst.getLeft());\n        Assert.assertEquals(\"no msg\", rst.getMiddle());\n        Assert.assertFalse(rst.getRight()); // no retry\n\n        when(brokerOuterAPI.pullMessageFromSpecificBrokerAsync(anyString(), anyString(), anyString(), anyString(), anyInt(), anyLong(), anyInt(), anyLong()))\n                .thenReturn(CompletableFuture.completedFuture(Triple.of(null, \"other resp code\", true)));\n        rst = escapeBridge.getMessageFromRemoteAsync(TEST_TOPIC, 1, DEFAULT_QUEUE_ID, BROKER_NAME).join();\n        Assert.assertNull(rst.getLeft());\n        Assert.assertEquals(\"other resp code\", rst.getMiddle());\n        Assert.assertTrue(rst.getRight()); // need retry\n    }\n\n    @Test\n    public void decodeMsgListTest() {\n        ByteBuffer byteBuffer = ByteBuffer.allocate(10);\n        MappedFile mappedFile = new DefaultMappedFile();\n        SelectMappedBufferResult result = new SelectMappedBufferResult(0, byteBuffer, 10, mappedFile);\n\n        getMessageResult.addMessage(result);\n        Assertions.assertThatCode(() -> escapeBridge.decodeMsgList(getMessageResult, false)).doesNotThrowAnyException();\n    }\n\n    @Test\n    public void decodeMsgListTest_messageNotNull() throws Exception {\n        MessageExt msg = new MessageExt();\n        msg.setBody(\"HW\".getBytes());\n        msg.setTopic(\"topic\");\n        msg.setBornHost(new InetSocketAddress(\"127.0.0.1\", 9000));\n        msg.setStoreHost(new InetSocketAddress(\"127.0.0.1\", 9000));\n        ByteBuffer byteBuffer = ByteBuffer.wrap(MessageDecoder.encode(msg, false));\n        SelectMappedBufferResult result = new SelectMappedBufferResult(0, byteBuffer, 10, new DefaultMappedFile());\n\n\n        getMessageResult.addMessage(result);\n        getMessageResult.getMessageQueueOffset().add(0L);\n        List<MessageExt> list = escapeBridge.decodeMsgList(getMessageResult, false); // skip deCompressBody test\n        Assert.assertEquals(1, list.size());\n        Assert.assertTrue(Arrays.equals(msg.getBody(), list.get(0).getBody()));\n    }\n\n    @Test\n    public void testPutMessageToRemoteBroker_noSpecificBrokerName_hasRemoteBroker() throws Exception {\n        MessageExtBrokerInner message = new MessageExtBrokerInner();\n        message.setTopic(TEST_TOPIC);\n        String anotherBrokerName = \"broker_b\";\n        TopicPublishInfo publishInfo = mockTopicPublishInfo(BROKER_NAME, anotherBrokerName);\n        when(topicRouteInfoManager.tryToFindTopicPublishInfo(anyString())).thenReturn(publishInfo);\n        when(topicRouteInfoManager.findBrokerAddressInPublish(anotherBrokerName)).thenReturn(\"127.0.0.1\");\n        escapeBridge.putMessageToRemoteBroker(message, null);\n        verify(brokerOuterAPI).sendMessageToSpecificBroker(eq(\"127.0.0.1\"), eq(anotherBrokerName), any(MessageExtBrokerInner.class), anyString(), anyLong());\n    }\n\n    @Test\n    public void testPutMessageToRemoteBroker_noSpecificBrokerName_noRemoteBroker() throws Exception {\n        MessageExtBrokerInner message = new MessageExtBrokerInner();\n        message.setTopic(TEST_TOPIC);\n        TopicPublishInfo publishInfo = mockTopicPublishInfo(BROKER_NAME);\n        when(topicRouteInfoManager.tryToFindTopicPublishInfo(anyString())).thenReturn(publishInfo);\n        escapeBridge.putMessageToRemoteBroker(message, null);\n        verify(topicRouteInfoManager, times(0)).findBrokerAddressInPublish(anyString());\n    }\n\n    @Test\n    public void testPutMessageToRemoteBroker_specificBrokerName_equals() throws Exception {\n        escapeBridge.putMessageToRemoteBroker(new MessageExtBrokerInner(), BROKER_NAME);\n        verify(topicRouteInfoManager, times(0)).tryToFindTopicPublishInfo(anyString());\n    }\n\n    @Test\n    public void testPutMessageToRemoteBroker_specificBrokerName_addressNotFound() throws Exception {\n        MessageExtBrokerInner message = new MessageExtBrokerInner();\n        message.setTopic(TEST_TOPIC);\n        TopicPublishInfo publishInfo = mockTopicPublishInfo(BROKER_NAME);\n        when(topicRouteInfoManager.tryToFindTopicPublishInfo(anyString())).thenReturn(publishInfo);\n        escapeBridge.putMessageToRemoteBroker(message, \"whatever\");\n        verify(topicRouteInfoManager).findBrokerAddressInPublish(eq(\"whatever\"));\n        verify(brokerOuterAPI, times(0)).sendMessageToSpecificBroker(anyString(), anyString(), any(MessageExtBrokerInner.class), anyString(), anyLong());\n    }\n\n    @Test\n    public void testPutMessageToRemoteBroker_specificBrokerName_addressFound() throws Exception {\n        MessageExtBrokerInner message = new MessageExtBrokerInner();\n        message.setTopic(TEST_TOPIC);\n        String anotherBrokerName = \"broker_b\";\n        TopicPublishInfo publishInfo = mockTopicPublishInfo(BROKER_NAME, anotherBrokerName);\n        when(topicRouteInfoManager.tryToFindTopicPublishInfo(anyString())).thenReturn(publishInfo);\n        when(topicRouteInfoManager.findBrokerAddressInPublish(anotherBrokerName)).thenReturn(\"127.0.0.1\");\n        escapeBridge.putMessageToRemoteBroker(message, anotherBrokerName);\n        verify(brokerOuterAPI).sendMessageToSpecificBroker(eq(\"127.0.0.1\"), eq(anotherBrokerName), any(MessageExtBrokerInner.class), anyString(), anyLong());\n    }\n\n    private GetMessageResult mockGetMessageResult(int count, String topic, byte[] body) throws Exception {\n        GetMessageResult result = new GetMessageResult();\n        for (int i = 0; i < count; i++) {\n            MessageExt msg = new MessageExt();\n            msg.setBody(body);\n            msg.setTopic(topic);\n            msg.setBornHost(new InetSocketAddress(\"127.0.0.1\", 9000));\n            msg.setStoreHost(new InetSocketAddress(\"127.0.0.1\", 9000));\n            ByteBuffer byteBuffer = ByteBuffer.wrap(MessageDecoder.encode(msg, false));\n            SelectMappedBufferResult bufferResult = new SelectMappedBufferResult(0, byteBuffer, body.length, new DefaultMappedFile());\n\n            result.addMessage(bufferResult);\n            result.getMessageQueueOffset().add(i + 0L);\n        }\n        return result;\n    }\n\n    private TopicPublishInfo mockTopicPublishInfo(String... brokerNames) {\n        TopicPublishInfo topicPublishInfo = new TopicPublishInfo();\n        for (String brokerName : brokerNames) {\n            topicPublishInfo.getMessageQueueList().add(new MessageQueue(TEST_TOPIC, brokerName, 0));\n        }\n        return topicPublishInfo;\n    }\n\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/filter/CommitLogDispatcherCalcBitMapTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.filter;\n\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.filter.util.BitsArray;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.junit.Test;\n\nimport java.util.Collection;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.UUID;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class CommitLogDispatcherCalcBitMapTest {\n\n    @Test\n    public void testDispatch_filterDataIllegal() {\n        BrokerConfig brokerConfig = new BrokerConfig();\n        brokerConfig.setEnableCalcFilterBitMap(true);\n\n        ConsumerFilterManager filterManager = new ConsumerFilterManager();\n\n        filterManager.register(\"topic0\", \"CID_0\", \"a is not null and a >= 5\",\n            ExpressionType.SQL92, System.currentTimeMillis());\n\n        filterManager.register(\"topic0\", \"CID_1\", \"a is not null and a >= 15\",\n            ExpressionType.SQL92, System.currentTimeMillis());\n\n        ConsumerFilterData nullExpression = filterManager.get(\"topic0\", \"CID_0\");\n        nullExpression.setExpression(null);\n        nullExpression.setCompiledExpression(null);\n        ConsumerFilterData nullBloomData = filterManager.get(\"topic0\", \"CID_1\");\n        nullBloomData.setBloomFilterData(null);\n\n        CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,\n            filterManager);\n\n        for (int i = 0; i < 1; i++) {\n            Map<String, String> properties = new HashMap<>(4);\n            properties.put(\"a\", String.valueOf(i * 10 + 5));\n\n            String topic = \"topic\" + i;\n\n            DispatchRequest dispatchRequest = new DispatchRequest(\n                topic,\n                0,\n                i * 100 + 123,\n                100,\n                (long) (\"tags\" + i).hashCode(),\n                System.currentTimeMillis(),\n                i,\n                null,\n                UUID.randomUUID().toString(),\n                0,\n                0,\n                properties\n            );\n\n            calcBitMap.dispatch(dispatchRequest);\n\n            assertThat(dispatchRequest.getBitMap()).isNotNull();\n\n            BitsArray bitsArray = BitsArray.create(dispatchRequest.getBitMap(),\n                filterManager.getBloomFilter().getM());\n\n            for (int j = 0; j < bitsArray.bitLength(); j++) {\n                assertThat(bitsArray.getBit(j)).isFalse();\n            }\n        }\n    }\n\n    @Test\n    public void testDispatch_blankFilterData() {\n        BrokerConfig brokerConfig = new BrokerConfig();\n        brokerConfig.setEnableCalcFilterBitMap(true);\n\n        ConsumerFilterManager filterManager = new ConsumerFilterManager();\n\n        CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,\n            filterManager);\n\n        for (int i = 0; i < 10; i++) {\n            Map<String, String> properties = new HashMap<>(4);\n            properties.put(\"a\", String.valueOf(i * 10 + 5));\n\n            String topic = \"topic\" + i;\n\n            DispatchRequest dispatchRequest = new DispatchRequest(\n                topic,\n                0,\n                i * 100 + 123,\n                100,\n                (long) (\"tags\" + i).hashCode(),\n                System.currentTimeMillis(),\n                i,\n                null,\n                UUID.randomUUID().toString(),\n                0,\n                0,\n                properties\n            );\n\n            calcBitMap.dispatch(dispatchRequest);\n\n            assertThat(dispatchRequest.getBitMap()).isNull();\n        }\n    }\n\n    @Test\n    public void testDispatch() {\n        BrokerConfig brokerConfig = new BrokerConfig();\n        brokerConfig.setEnableCalcFilterBitMap(true);\n\n        ConsumerFilterManager filterManager = ConsumerFilterManagerTest.gen(10, 10);\n\n        CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,\n            filterManager);\n\n        for (int i = 0; i < 10; i++) {\n            Map<String, String> properties = new HashMap<>(4);\n            properties.put(\"a\", String.valueOf(i * 10 + 5));\n\n            String topic = \"topic\" + i;\n\n            DispatchRequest dispatchRequest = new DispatchRequest(\n                topic,\n                0,\n                i * 100 + 123,\n                100,\n                (long) (\"tags\" + i).hashCode(),\n                System.currentTimeMillis(),\n                i,\n                null,\n                UUID.randomUUID().toString(),\n                0,\n                0,\n                properties\n            );\n\n            calcBitMap.dispatch(dispatchRequest);\n\n            assertThat(dispatchRequest.getBitMap()).isNotNull();\n\n            BitsArray bits = BitsArray.create(dispatchRequest.getBitMap());\n\n            Collection<ConsumerFilterData> filterDatas = filterManager.get(topic);\n\n            for (ConsumerFilterData filterData : filterDatas) {\n\n                if (filterManager.getBloomFilter().isHit(filterData.getBloomFilterData(), bits)) {\n                    try {\n                        assertThat((Boolean) filterData.getCompiledExpression().evaluate(\n                            new MessageEvaluationContext(properties)\n                        )).isTrue();\n                    } catch (Exception e) {\n                        e.printStackTrace();\n                        assertThat(true).isFalse();\n                    }\n                } else {\n                    try {\n                        assertThat((Boolean) filterData.getCompiledExpression().evaluate(\n                            new MessageEvaluationContext(properties)\n                        )).isFalse();\n                    } catch (Exception e) {\n                        e.printStackTrace();\n                        assertThat(true).isFalse();\n                    }\n                }\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/filter/ConsumerFilterManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.filter;\n\nimport java.io.File;\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.Iterator;\nimport java.util.List;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.remoting.protocol.filter.FilterAPI;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class ConsumerFilterManagerTest {\n\n    public static ConsumerFilterManager gen(int topicCount, int consumerCount) {\n        ConsumerFilterManager filterManager = new ConsumerFilterManager();\n\n        for (int i = 0; i < topicCount; i++) {\n            String topic = \"topic\" + i;\n\n            for (int j = 0; j < consumerCount; j++) {\n\n                String consumer = \"CID_\" + j;\n\n                filterManager.register(topic, consumer, expr(j), ExpressionType.SQL92, System.currentTimeMillis());\n            }\n        }\n\n        return filterManager;\n    }\n\n    public static String expr(int i) {\n        return \"a is not null and a > \" + ((i - 1) * 10) + \" and a < \" + ((i + 1) * 10);\n    }\n\n    @Test\n    public void testRegister_newExpressionCompileErrorAndRemoveOld() {\n        ConsumerFilterManager filterManager = gen(10, 10);\n\n        assertThat(filterManager.get(\"topic9\", \"CID_9\")).isNotNull();\n\n        String newExpr = \"a between 10,20\";\n\n        assertThat(filterManager.register(\"topic9\", \"CID_9\", newExpr, ExpressionType.SQL92, System.currentTimeMillis() + 1))\n            .isFalse();\n        assertThat(filterManager.get(\"topic9\", \"CID_9\")).isNull();\n\n        newExpr = \"a between 10 AND 20\";\n\n        assertThat(filterManager.register(\"topic9\", \"CID_9\", newExpr, ExpressionType.SQL92, System.currentTimeMillis() + 1))\n            .isTrue();\n\n        ConsumerFilterData filterData = filterManager.get(\"topic9\", \"CID_9\");\n\n        assertThat(filterData).isNotNull();\n        assertThat(newExpr).isEqualTo(filterData.getExpression());\n    }\n\n    @Test\n    public void testRegister_change() {\n        ConsumerFilterManager filterManager = gen(10, 10);\n\n        ConsumerFilterData filterData;\n\n        String newExpr = \"a > 0 and a < 10\";\n\n        filterManager.register(\"topic9\", \"CID_9\", newExpr, ExpressionType.SQL92, System.currentTimeMillis() + 1);\n\n        filterData = filterManager.get(\"topic9\", \"CID_9\");\n\n        assertThat(newExpr).isEqualTo(filterData.getExpression());\n    }\n\n    @Test\n    public void testRegister() {\n        ConsumerFilterManager filterManager = gen(10, 10);\n\n        ConsumerFilterData filterData = filterManager.get(\"topic9\", \"CID_9\");\n\n        assertThat(filterData).isNotNull();\n        assertThat(filterData.isDead()).isFalse();\n\n        // new version\n        assertThat(filterManager.register(\n            \"topic9\", \"CID_9\", \"a is not null\", ExpressionType.SQL92, System.currentTimeMillis() + 1000\n        )).isTrue();\n\n        ConsumerFilterData newFilter = filterManager.get(\"topic9\", \"CID_9\");\n\n        assertThat(newFilter).isNotEqualTo(filterData);\n\n        // same version\n        assertThat(filterManager.register(\n            \"topic9\", \"CID_9\", \"a is null\", ExpressionType.SQL92, newFilter.getClientVersion()\n        )).isFalse();\n\n        ConsumerFilterData filterData1 = filterManager.get(\"topic9\", \"CID_9\");\n\n        assertThat(newFilter).isEqualTo(filterData1);\n    }\n\n    @Test\n    public void testRegister_reAlive() {\n        ConsumerFilterManager filterManager = gen(10, 10);\n\n        ConsumerFilterData filterData = filterManager.get(\"topic9\", \"CID_9\");\n\n        assertThat(filterData).isNotNull();\n        assertThat(filterData.isDead()).isFalse();\n\n        //make dead\n        filterManager.unRegister(\"CID_9\");\n\n        //reAlive\n        filterManager.register(\n            filterData.getTopic(),\n            filterData.getConsumerGroup(),\n            filterData.getExpression(),\n            filterData.getExpressionType(),\n            System.currentTimeMillis()\n        );\n\n        ConsumerFilterData newFilterData = filterManager.get(\"topic9\", \"CID_9\");\n\n        assertThat(newFilterData).isNotNull();\n        assertThat(newFilterData.isDead()).isFalse();\n    }\n\n    @Test\n    public void testRegister_bySubscriptionData() {\n        ConsumerFilterManager filterManager = new ConsumerFilterManager();\n        List<SubscriptionData> subscriptionDatas = new ArrayList<>();\n        for (int i = 0; i < 10; i++) {\n            try {\n                subscriptionDatas.add(\n                    FilterAPI.build(\n                        \"topic\" + i,\n                        \"a is not null and a > \" + i,\n                        ExpressionType.SQL92\n                    )\n                );\n            } catch (Exception e) {\n                e.printStackTrace();\n                assertThat(true).isFalse();\n            }\n        }\n\n        filterManager.register(\"CID_0\", subscriptionDatas);\n\n        Collection<ConsumerFilterData> filterDatas = filterManager.getByGroup(\"CID_0\");\n\n        assertThat(filterDatas).isNotNull();\n        assertThat(filterDatas.size()).isEqualTo(10);\n\n        Iterator<ConsumerFilterData> iterator = filterDatas.iterator();\n        while (iterator.hasNext()) {\n            ConsumerFilterData filterData = iterator.next();\n\n            assertThat(filterData).isNotNull();\n            assertThat(filterManager.getBloomFilter().isValid(filterData.getBloomFilterData())).isTrue();\n        }\n    }\n\n    @Test\n    public void testRegister_tag() {\n        ConsumerFilterManager filterManager = new ConsumerFilterManager();\n\n        assertThat(filterManager.register(\"topic0\", \"CID_0\", \"*\", null, System.currentTimeMillis())).isFalse();\n\n        Collection<ConsumerFilterData> filterDatas = filterManager.getByGroup(\"CID_0\");\n\n        assertThat(filterDatas).isNullOrEmpty();\n    }\n\n    @Test\n    public void testUnregister() {\n        ConsumerFilterManager filterManager = gen(10, 10);\n\n        ConsumerFilterData filterData = filterManager.get(\"topic9\", \"CID_9\");\n\n        assertThat(filterData).isNotNull();\n        assertThat(filterData.isDead()).isFalse();\n\n        filterManager.unRegister(\"CID_9\");\n\n        assertThat(filterData.isDead()).isTrue();\n    }\n\n    @Test\n    public void testPersist() {\n        ConsumerFilterManager filterManager = gen(10, 10);\n\n        try {\n            filterManager.persist();\n\n            ConsumerFilterData filterData = filterManager.get(\"topic9\", \"CID_9\");\n\n            assertThat(filterData).isNotNull();\n            assertThat(filterData.isDead()).isFalse();\n\n            ConsumerFilterManager loadFilter = new ConsumerFilterManager();\n\n            assertThat(loadFilter.load()).isTrue();\n\n            filterData = loadFilter.get(\"topic9\", \"CID_9\");\n\n            assertThat(filterData).isNotNull();\n            assertThat(filterData.isDead()).isTrue();\n            assertThat(filterData.getCompiledExpression()).isNotNull();\n        } finally {\n            UtilAll.deleteFile(new File(\"./unit_test\"));\n        }\n    }\n\n    @Test\n    public void testPersist_clean() {\n        ConsumerFilterManager filterManager = gen(10, 10);\n\n        String topic = \"topic9\";\n        for (int i = 0; i < 10; i++) {\n            String cid = \"CID_\" + i;\n\n            ConsumerFilterData filterData = filterManager.get(topic, cid);\n\n            assertThat(filterData).isNotNull();\n            assertThat(filterData.isDead()).isFalse();\n\n            //make dead more than 24h\n            filterData.setBornTime(System.currentTimeMillis() - 26 * 60 * 60 * 1000);\n            filterData.setDeadTime(System.currentTimeMillis() - 25 * 60 * 60 * 1000);\n        }\n\n        try {\n            filterManager.persist();\n\n            ConsumerFilterManager loadFilter = new ConsumerFilterManager();\n\n            assertThat(loadFilter.load()).isTrue();\n\n            ConsumerFilterData filterData = loadFilter.get(topic, \"CID_9\");\n\n            assertThat(filterData).isNull();\n\n            Collection<ConsumerFilterData> topicData = loadFilter.get(topic);\n\n            assertThat(topicData).isNullOrEmpty();\n        } finally {\n            UtilAll.deleteFile(new File(\"./unit_test\"));\n        }\n    }\n\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/filter/MessageStoreWithFilterTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.filter;\n\nimport java.io.File;\nimport java.net.InetAddress;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.net.UnknownHostException;\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.store.CommitLogDispatcher;\nimport org.apache.rocketmq.store.ConsumeQueueExt;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.MessageArrivingListener;\nimport org.apache.rocketmq.store.MessageFilter;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.awaitility.core.ThrowingRunnable;\nimport org.junit.After;\nimport org.junit.Assume;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.awaitility.Awaitility.await;\n\npublic class MessageStoreWithFilterTest {\n\n    private static final String MSG = \"Once, there was a chance for me!\";\n    private static final byte[] MSG_BODY = MSG.getBytes();\n\n    private static final String TOPIC = \"topic\";\n    private static final int QUEUE_ID = 0;\n    private static final String STORE_PATH = System.getProperty(\"java.io.tmpdir\") + File.separator + \"unit_test_store\";\n    private static final int COMMIT_LOG_FILE_SIZE = 1024 * 1024 * 256;\n    private static final int CQ_FILE_SIZE = 300000 * 20;\n    private static final int CQ_EXT_FILE_SIZE = 300000 * 128;\n\n    private static SocketAddress bornHost;\n\n    private static SocketAddress storeHost;\n\n    private DefaultMessageStore master;\n\n    private ConsumerFilterManager filterManager;\n\n    private int topicCount = 3;\n\n    private int msgPerTopic = 30;\n\n    static {\n        try {\n            storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123);\n        } catch (UnknownHostException e) {\n        }\n        try {\n            bornHost = new InetSocketAddress(InetAddress.getByName(\"127.0.0.1\"), 0);\n        } catch (UnknownHostException e) {\n        }\n    }\n\n    @Before\n    public void init() throws Exception {\n        filterManager = ConsumerFilterManagerTest.gen(topicCount, msgPerTopic);\n        master = gen(filterManager);\n    }\n\n    @After\n    public void destroy() {\n        if (master != null) {\n            master.shutdown();\n            master.destroy();\n        }\n        UtilAll.deleteFile(new File(STORE_PATH));\n    }\n\n    public MessageExtBrokerInner buildMessage() {\n        MessageExtBrokerInner msg = new MessageExtBrokerInner();\n        msg.setTopic(TOPIC);\n        msg.setTags(System.currentTimeMillis() + \"TAG\");\n        msg.setKeys(\"Hello\");\n        msg.setBody(MSG_BODY);\n        msg.setKeys(String.valueOf(System.currentTimeMillis()));\n        msg.setQueueId(QUEUE_ID);\n        msg.setSysFlag(0);\n        msg.setBornTimestamp(System.currentTimeMillis());\n        msg.setStoreHost(storeHost);\n        msg.setBornHost(bornHost);\n        for (int i = 1; i < 3; i++) {\n            msg.putUserProperty(String.valueOf(i), \"imagoodperson\" + i);\n        }\n        msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n\n        return msg;\n    }\n\n    public MessageStoreConfig buildStoreConfig(int commitLogFileSize, int cqFileSize,\n                                               boolean enableCqExt, int cqExtFileSize) {\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setMappedFileSizeCommitLog(commitLogFileSize);\n        messageStoreConfig.setMappedFileSizeConsumeQueue(cqFileSize);\n        messageStoreConfig.setMappedFileSizeConsumeQueueExt(cqExtFileSize);\n        messageStoreConfig.setMessageIndexEnable(false);\n        messageStoreConfig.setEnableConsumeQueueExt(enableCqExt);\n\n        messageStoreConfig.setStorePathRootDir(STORE_PATH);\n        messageStoreConfig.setStorePathCommitLog(STORE_PATH + File.separator + \"commitlog\");\n\n        return messageStoreConfig;\n    }\n\n    protected DefaultMessageStore gen(ConsumerFilterManager filterManager) throws Exception {\n        MessageStoreConfig messageStoreConfig = buildStoreConfig(\n                COMMIT_LOG_FILE_SIZE, CQ_FILE_SIZE, true, CQ_EXT_FILE_SIZE\n        );\n\n        BrokerConfig brokerConfig = new BrokerConfig();\n        brokerConfig.setEnableCalcFilterBitMap(true);\n        brokerConfig.setMaxErrorRateOfBloomFilter(20);\n        brokerConfig.setExpectConsumerNumUseFilter(64);\n\n        DefaultMessageStore master = new DefaultMessageStore(\n            messageStoreConfig,\n            new BrokerStatsManager(brokerConfig.getBrokerClusterName(), brokerConfig.isEnableDetailStat()),\n            new MessageArrivingListener() {\n                @Override\n                public void arriving(String topic, int queueId, long logicOffset, long tagsCode,\n                                     long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {\n                }\n            }\n            , brokerConfig, new ConcurrentHashMap<>());\n\n        master.getDispatcherList().addFirst(new CommitLogDispatcher() {\n            @Override\n            public void dispatch(DispatchRequest request) {\n                try {\n                } catch (Throwable e) {\n                    e.printStackTrace();\n                }\n            }\n        });\n        master.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(brokerConfig, filterManager));\n\n        if (MixAll.isWindows()) {\n            Assume.assumeTrue(master.load());\n        } else {\n            assertThat(master.load()).isTrue();\n        }\n\n        master.start();\n\n        return master;\n    }\n\n    protected List<MessageExtBrokerInner> putMsg(DefaultMessageStore master, int topicCount,\n                                                 int msgCountPerTopic) throws Exception {\n        List<MessageExtBrokerInner> msgs = new ArrayList<>();\n        for (int i = 0; i < topicCount; i++) {\n            String realTopic = TOPIC + i;\n            for (int j = 0; j < msgCountPerTopic; j++) {\n                MessageExtBrokerInner msg = buildMessage();\n                msg.setTopic(realTopic);\n                msg.putUserProperty(\"a\", String.valueOf(j * 10 + 5));\n                msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n\n                PutMessageResult result = master.putMessage(msg);\n\n                msg.setMsgId(result.getAppendMessageResult().getMsgId());\n\n                msgs.add(msg);\n            }\n        }\n\n        return msgs;\n    }\n\n    protected List<MessageExtBrokerInner> filtered(List<MessageExtBrokerInner> msgs, ConsumerFilterData filterData) {\n        List<MessageExtBrokerInner> filteredMsgs = new ArrayList<>();\n\n        for (MessageExtBrokerInner messageExtBrokerInner : msgs) {\n\n            if (!messageExtBrokerInner.getTopic().equals(filterData.getTopic())) {\n                continue;\n            }\n\n            try {\n                Object evlRet = filterData.getCompiledExpression().evaluate(new MessageEvaluationContext(messageExtBrokerInner.getProperties()));\n\n                if (evlRet == null || !(evlRet instanceof Boolean) || (Boolean) evlRet) {\n                    filteredMsgs.add(messageExtBrokerInner);\n                }\n            } catch (Exception e) {\n                e.printStackTrace();\n                assertThat(true).isFalse();\n            }\n        }\n\n        return filteredMsgs;\n    }\n\n    @Test\n    public void testGetMessage_withFilterBitMapAndConsumerChanged() throws Exception {\n        List<MessageExtBrokerInner> msgs = putMsg(master, topicCount, msgPerTopic);\n\n        Thread.sleep(200);\n\n        // reset consumer;\n        String topic = \"topic\" + 0;\n        String resetGroup = \"CID_\" + 2;\n        String normalGroup = \"CID_\" + 3;\n\n        {\n            // reset CID_2@topic0 to get all messages.\n            SubscriptionData resetSubData = new SubscriptionData();\n            resetSubData.setExpressionType(ExpressionType.SQL92);\n            resetSubData.setTopic(topic);\n            resetSubData.setClassFilterMode(false);\n            resetSubData.setSubString(\"a is not null OR a is null\");\n\n            ConsumerFilterData resetFilterData = ConsumerFilterManager.build(topic,\n                resetGroup, resetSubData.getSubString(), resetSubData.getExpressionType(),\n                System.currentTimeMillis());\n\n            GetMessageResult resetGetResult = master.getMessage(resetGroup, topic, QUEUE_ID, 0, 1000,\n                new ExpressionMessageFilter(resetSubData, resetFilterData, filterManager));\n\n            try {\n                assertThat(resetGetResult).isNotNull();\n\n                List<MessageExtBrokerInner> filteredMsgs = filtered(msgs, resetFilterData);\n\n                assertThat(resetGetResult.getMessageBufferList().size()).isEqualTo(filteredMsgs.size());\n            } finally {\n                resetGetResult.release();\n            }\n        }\n\n        {\n            ConsumerFilterData normalFilterData = filterManager.get(topic, normalGroup);\n            assertThat(normalFilterData).isNotNull();\n            assertThat(normalFilterData.getBornTime()).isLessThan(System.currentTimeMillis());\n\n            SubscriptionData normalSubData = new SubscriptionData();\n            normalSubData.setExpressionType(normalFilterData.getExpressionType());\n            normalSubData.setTopic(topic);\n            normalSubData.setClassFilterMode(false);\n            normalSubData.setSubString(normalFilterData.getExpression());\n\n            List<MessageExtBrokerInner> filteredMsgs = filtered(msgs, normalFilterData);\n\n            GetMessageResult normalGetResult = master.getMessage(normalGroup, topic, QUEUE_ID, 0, 1000,\n                new ExpressionMessageFilter(normalSubData, normalFilterData, filterManager));\n\n            try {\n                assertThat(normalGetResult).isNotNull();\n                assertThat(normalGetResult.getMessageBufferList().size()).isEqualTo(filteredMsgs.size());\n            } finally {\n                normalGetResult.release();\n            }\n        }\n    }\n\n    @Test\n    public void testGetMessage_withFilterBitMap() throws Exception {\n        List<MessageExtBrokerInner> msgs = putMsg(master, topicCount, msgPerTopic);\n\n        Thread.sleep(100);\n\n        for (int i = 0; i < topicCount; i++) {\n            String realTopic = TOPIC + i;\n\n            for (int j = 0; j < msgPerTopic; j++) {\n                String group = \"CID_\" + j;\n\n                ConsumerFilterData filterData = filterManager.get(realTopic, group);\n                assertThat(filterData).isNotNull();\n\n                List<MessageExtBrokerInner> filteredMsgs = filtered(msgs, filterData);\n\n                SubscriptionData subscriptionData = new SubscriptionData();\n                subscriptionData.setExpressionType(filterData.getExpressionType());\n                subscriptionData.setTopic(filterData.getTopic());\n                subscriptionData.setClassFilterMode(false);\n                subscriptionData.setSubString(filterData.getExpression());\n\n                GetMessageResult getMessageResult = master.getMessage(group, realTopic, QUEUE_ID, 0, 10000,\n                    new ExpressionMessageFilter(subscriptionData, filterData, filterManager));\n                String assertMsg = group + \"-\" + realTopic;\n                try {\n                    assertThat(getMessageResult).isNotNull();\n                    assertThat(GetMessageStatus.FOUND).isEqualTo(getMessageResult.getStatus());\n                    assertThat(getMessageResult.getMessageBufferList()).isNotNull().isNotEmpty();\n                    assertThat(getMessageResult.getMessageBufferList().size()).isEqualTo(filteredMsgs.size());\n\n                    for (ByteBuffer buffer : getMessageResult.getMessageBufferList()) {\n                        MessageExt messageExt = MessageDecoder.decode(buffer.slice(), false);\n                        assertThat(messageExt).isNotNull();\n\n                        Object evlRet = null;\n                        try {\n                            evlRet = filterData.getCompiledExpression().evaluate(new MessageEvaluationContext(messageExt.getProperties()));\n                        } catch (Exception e) {\n                            e.printStackTrace();\n                            assertThat(true).isFalse();\n                        }\n\n                        assertThat(evlRet).isNotNull().isEqualTo(Boolean.TRUE);\n\n                        // check\n                        boolean find = false;\n                        for (MessageExtBrokerInner messageExtBrokerInner : filteredMsgs) {\n                            if (messageExtBrokerInner.getMsgId().equals(messageExt.getMsgId())) {\n                                find = true;\n                            }\n                        }\n                        assertThat(find).isTrue();\n                    }\n                } finally {\n                    getMessageResult.release();\n                }\n            }\n        }\n    }\n\n    @Test\n    public void testGetMessage_withFilter_checkTagsCode() throws Exception {\n        putMsg(master, topicCount, msgPerTopic);\n\n        await().atMost(3, TimeUnit.SECONDS).untilAsserted(new ThrowingRunnable() {\n            @Override\n            public void run() throws Throwable {\n                for (int i = 0; i < topicCount; i++) {\n                    final String realTopic = TOPIC + i;\n                    GetMessageResult getMessageResult = master.getMessage(\"test\", realTopic, QUEUE_ID, 0, 10000,\n                        new MessageFilter() {\n                            @Override\n                            public boolean isMatchedByConsumeQueue(Long tagsCode,\n                                ConsumeQueueExt.CqExtUnit cqExtUnit) {\n                                if (tagsCode != null && tagsCode <= ConsumeQueueExt.MAX_ADDR) {\n                                    return false;\n                                }\n                                return true;\n                            }\n\n                            @Override\n                            public boolean isMatchedByCommitLog(ByteBuffer msgBuffer,\n                                Map<String, String> properties) {\n                                return true;\n                            }\n                        });\n                    assertThat(getMessageResult.getMessageCount()).isEqualTo(msgPerTopic);\n                }\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.latency;\n\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.future.FutureTaskExt;\nimport org.apache.rocketmq.remoting.netty.RequestTask;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.Mockito;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class BrokerFastFailureTest {\n\n    private BrokerController brokerController;\n\n    private final BrokerConfig brokerConfig = new BrokerConfig();\n\n    private MessageStore messageStore;\n\n    @Before\n    public void setUp() {\n        brokerController = Mockito.mock(BrokerController.class);\n        messageStore = Mockito.mock(DefaultMessageStore.class);\n        BlockingQueue<Runnable> queue = new LinkedBlockingQueue<>();\n        Mockito.when(brokerController.getSendThreadPoolQueue()).thenReturn(queue);\n        Mockito.when(brokerController.getPullThreadPoolQueue()).thenReturn(queue);\n        Mockito.when(brokerController.getLitePullThreadPoolQueue()).thenReturn(queue);\n        Mockito.when(brokerController.getHeartbeatThreadPoolQueue()).thenReturn(queue);\n        Mockito.when(brokerController.getEndTransactionThreadPoolQueue()).thenReturn(queue);\n        Mockito.when(brokerController.getAdminBrokerThreadPoolQueue()).thenReturn(queue);\n        Mockito.when(brokerController.getAckThreadPoolQueue()).thenReturn(queue);\n        Mockito.when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        Mockito.when(messageStore.isOSPageCacheBusy()).thenReturn(false);\n        Mockito.when(brokerController.getMessageStore()).thenReturn(messageStore);\n    }\n\n    @Test\n    public void testCleanExpiredRequestInQueue() throws Exception {\n        BrokerFastFailure brokerFastFailure = new BrokerFastFailure(brokerController);\n\n        BlockingQueue<Runnable> queue = new LinkedBlockingQueue<>();\n        brokerFastFailure.cleanExpiredRequestInQueue(queue, 1);\n        assertThat(queue.size()).isZero();\n\n        //Normal Runnable\n        Runnable runnable = new Runnable() {\n            @Override\n            public void run() {\n\n            }\n        };\n        queue.add(runnable);\n\n        assertThat(queue.size()).isEqualTo(1);\n        brokerFastFailure.cleanExpiredRequestInQueue(queue, 1);\n        assertThat(queue.size()).isEqualTo(1);\n\n        queue.clear();\n\n        //With expired request\n        RequestTask expiredRequest = new RequestTask(runnable, null, null);\n        queue.add(new FutureTaskExt<>(expiredRequest, null));\n        TimeUnit.MILLISECONDS.sleep(100);\n\n        RequestTask requestTask = new RequestTask(runnable, null, null);\n        queue.add(new FutureTaskExt<>(requestTask, null));\n\n        assertThat(queue.size()).isEqualTo(2);\n        brokerFastFailure.cleanExpiredRequestInQueue(queue, 100);\n        assertThat(queue.size()).isEqualTo(1);\n        assertThat(((FutureTaskExt) queue.peek()).getRunnable()).isEqualTo(requestTask);\n    }\n\n    @Test\n    public void testCleanExpiredCustomRequestInQueue() throws Exception {\n        BrokerFastFailure brokerFastFailure = new BrokerFastFailure(brokerController);\n        brokerFastFailure.start();\n        brokerConfig.setWaitTimeMillsInAckQueue(10);\n        BlockingQueue<Runnable> customThreadPoolQueue = new LinkedBlockingQueue<>();\n        brokerFastFailure.addCleanExpiredRequestQueue(customThreadPoolQueue, () -> brokerConfig.getWaitTimeMillsInAckQueue());\n\n        Runnable runnable = new Runnable() {\n            @Override\n            public void run() {\n\n            }\n        };\n        RequestTask requestTask = new RequestTask(runnable, null, null);\n        customThreadPoolQueue.add(new FutureTaskExt<>(requestTask, null));\n\n        Thread.sleep(2000);\n\n        assertThat(customThreadPoolQueue.size()).isEqualTo(0);\n        assertThat(requestTask.isStopRun()).isEqualTo(true);\n\n        brokerConfig.setWaitTimeMillsInAckQueue(10000);\n\n        RequestTask requestTask2 = new RequestTask(runnable, null, null);\n        customThreadPoolQueue.add(new FutureTaskExt<>(requestTask2, null));\n\n        Thread.sleep(1000);\n\n        assertThat(customThreadPoolQueue.size()).isEqualTo(1);\n        assertThat(((FutureTaskExt) customThreadPoolQueue.peek()).getRunnable()).isEqualTo(requestTask2);\n\n        brokerFastFailure.shutdown();\n\n    }\n\n}"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/lite/AbstractLiteLifecycleManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.lite;\n\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.config.v1.RocksDBConsumerOffsetManager;\nimport org.apache.rocketmq.broker.pop.orderly.ConsumerOrderInfoManager;\nimport org.apache.rocketmq.broker.processor.PopLiteMessageProcessor;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.TopicAttributes;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.apache.rocketmq.broker.offset.ConsumerOffsetManager.TOPIC_GROUP_SEPARATOR;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.when;\nimport static org.mockito.Mockito.anyInt;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.never;\nimport static org.mockito.Mockito.atLeastOnce;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class AbstractLiteLifecycleManagerTest {\n    private static final String PARENT_TOPIC = \"parentTopic\";\n    private static final String EXIST_LMQ_NAME = LiteUtil.toLmqName(PARENT_TOPIC, \"HW\");\n    private static final String GROUP = \"group\";\n\n    @Mock\n    private BrokerController brokerController;\n    @Mock\n    private LiteSharding liteSharding;\n    @Mock\n    private MessageStore messageStore;\n    @Mock\n    private TopicConfigManager topicConfigManager;\n    @Mock\n    private SubscriptionGroupManager subscriptionGroupManager;\n    @Mock\n    private RocksDBConsumerOffsetManager consumerOffsetManager;\n    @Mock\n    private PopLiteMessageProcessor popLiteMessageProcessor;\n    @Mock\n    private ConsumerOrderInfoManager consumerOrderInfoManager;\n    @Mock\n    private LiteSubscriptionRegistry liteSubscriptionRegistry;\n\n    private TestLiteLifecycleManager lifecycleManager;\n    private BrokerConfig brokerConfig;\n\n    private final TopicConfig topicConfig = new TopicConfig(PARENT_TOPIC, 1, 1);\n    private final SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig();\n    private final ConcurrentMap<String, ConcurrentMap<Integer, Long>> offsetTable = new ConcurrentHashMap<>();\n\n    @Before\n    public void setUp() {\n        brokerConfig = new BrokerConfig();\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        when(brokerController.getMessageStore()).thenReturn(messageStore);\n        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);\n        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);\n        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);\n        when(brokerController.getPopLiteMessageProcessor()).thenReturn(popLiteMessageProcessor);\n        when(popLiteMessageProcessor.getConsumerOrderInfoManager()).thenReturn(consumerOrderInfoManager);\n        when(brokerController.getLiteSubscriptionRegistry()).thenReturn(liteSubscriptionRegistry);\n\n        topicConfig.getAttributes().put(\n            TopicAttributes.TOPIC_MESSAGE_TYPE_ATTRIBUTE.getName(), TopicMessageType.LITE.getValue());\n        ConcurrentMap<String, TopicConfig> topicConfigTable = new ConcurrentHashMap<>();\n        topicConfigTable.put(PARENT_TOPIC, topicConfig);\n        when(topicConfigManager.getTopicConfigTable()).thenReturn(topicConfigTable);\n        when(topicConfigManager.selectTopicConfig(PARENT_TOPIC)).thenReturn(topicConfig);\n\n        groupConfig.setGroupName(GROUP);\n        groupConfig.setLiteBindTopic(PARENT_TOPIC);\n        ConcurrentMap<String, SubscriptionGroupConfig> groupTable = new ConcurrentHashMap<>();\n        groupTable.put(GROUP, groupConfig);\n        when(subscriptionGroupManager.getSubscriptionGroupTable()).thenReturn(groupTable);\n\n        when(consumerOffsetManager.getOffsetTable()).thenReturn(offsetTable);\n        when(consumerOffsetManager.getPullOffsetTable()).thenReturn(offsetTable);\n\n        TestLiteLifecycleManager testObject = new TestLiteLifecycleManager(brokerController, liteSharding);\n        lifecycleManager = Mockito.spy(testObject);\n        lifecycleManager.init();\n    }\n\n    @After\n    public void reset() {\n        topicConfig.getAttributes().clear();\n        groupConfig.getAttributes().clear();\n        offsetTable.clear();\n    }\n\n    @Test\n    public void testIsSubscriptionActive() {\n        when(liteSharding.shardingByLmqName(PARENT_TOPIC, EXIST_LMQ_NAME)).thenReturn(brokerConfig.getBrokerName());\n        Assert.assertTrue(lifecycleManager.isSubscriptionActive(PARENT_TOPIC, EXIST_LMQ_NAME));\n        Assert.assertFalse(lifecycleManager.isSubscriptionActive(\"whatever\", \"whatever\"));\n\n        when(liteSharding.shardingByLmqName(anyString(), anyString())).thenReturn(brokerConfig.getBrokerName());\n        Assert.assertTrue(lifecycleManager.isSubscriptionActive(PARENT_TOPIC, EXIST_LMQ_NAME));\n        Assert.assertTrue(lifecycleManager.isSubscriptionActive(\"whatever\", \"whatever\"));\n\n        when(liteSharding.shardingByLmqName(anyString(), anyString())).thenReturn(\"otherBrokerName\");\n        Assert.assertTrue(lifecycleManager.isSubscriptionActive(PARENT_TOPIC, EXIST_LMQ_NAME));\n        Assert.assertFalse(lifecycleManager.isSubscriptionActive(\"whatever\", \"whatever\"));\n    }\n\n    @Test\n    public void testIsLmqExist() {\n        Assert.assertTrue(lifecycleManager.isLmqExist(EXIST_LMQ_NAME));\n        Assert.assertFalse(lifecycleManager.isLmqExist(\"whatever\"));\n    }\n\n    @Test\n    public void testGetLiteTopicCount() {\n        Assert.assertEquals(1, lifecycleManager.getLiteTopicCount(PARENT_TOPIC));\n        verify(lifecycleManager).collectByParentTopic(PARENT_TOPIC);\n\n        Assert.assertEquals(0, lifecycleManager.getLiteTopicCount(\"whatever\"));\n        verify(lifecycleManager, never()).collectByParentTopic(\"whatever\");\n    }\n\n    @Test\n    public void testIsLiteTopicExpired() {\n        // not lite topic queue\n        Assert.assertFalse(lifecycleManager.isLiteTopicExpired(PARENT_TOPIC, \"whatever\", 10L));\n\n        // maxOffset invalid\n        Assert.assertFalse(lifecycleManager.isLiteTopicExpired(PARENT_TOPIC, EXIST_LMQ_NAME, 0L));\n\n        // less than minLiteTTl\n        long mockStoreTime = System.currentTimeMillis();\n        when(messageStore.getMessageStoreTimeStamp(anyString(), anyInt(), anyLong())).thenReturn(mockStoreTime);\n        Assert.assertFalse(lifecycleManager.isLiteTopicExpired(PARENT_TOPIC, EXIST_LMQ_NAME, 100L));\n\n        // topic ttl not found\n        mockStoreTime = System.currentTimeMillis() - brokerConfig.getMinLiteTTl() - 2000;\n        when(messageStore.getMessageStoreTimeStamp(anyString(), anyInt(), anyLong())).thenReturn(mockStoreTime);\n        Assert.assertFalse(lifecycleManager.isLiteTopicExpired(PARENT_TOPIC, EXIST_LMQ_NAME, 100L));\n\n        // topic ttl no expiration\n        topicConfig.getAttributes().put(TopicAttributes.LITE_EXPIRATION_ATTRIBUTE.getName(), \"-1\");\n        lifecycleManager.updateMetadata();\n        mockStoreTime = System.currentTimeMillis() - brokerConfig.getMinLiteTTl() - 2000;\n        when(messageStore.getMessageStoreTimeStamp(anyString(), anyInt(), anyLong())).thenReturn(mockStoreTime);\n        Assert.assertFalse(lifecycleManager.isLiteTopicExpired(PARENT_TOPIC, EXIST_LMQ_NAME, 100L));\n\n        // topic ttl expired\n        topicConfig.getAttributes().put(\n            TopicAttributes.LITE_EXPIRATION_ATTRIBUTE.getName(), \"\" + brokerConfig.getMinLiteTTl() / 1000 / 60);\n        lifecycleManager.updateMetadata();\n        mockStoreTime = System.currentTimeMillis() - brokerConfig.getMinLiteTTl() - 2000;\n        when(messageStore.getMessageStoreTimeStamp(anyString(), anyInt(), anyLong())).thenReturn(mockStoreTime);\n        Assert.assertTrue(lifecycleManager.isLiteTopicExpired(PARENT_TOPIC, EXIST_LMQ_NAME, 100L));\n    }\n\n    @Test\n    public void testDeleteLmq() {\n        lifecycleManager.updateMetadata();\n        String otherKey = \"otherTopic@otherGroup\";\n        String removeKey = EXIST_LMQ_NAME + TOPIC_GROUP_SEPARATOR + GROUP;\n        offsetTable.put(otherKey, new ConcurrentHashMap<>());\n        offsetTable.put(removeKey, new ConcurrentHashMap<>());\n\n        // sharding to this broker\n        when(liteSharding.shardingByLmqName(PARENT_TOPIC, EXIST_LMQ_NAME)).thenReturn(brokerConfig.getBrokerName());\n        lifecycleManager.deleteLmq(PARENT_TOPIC, EXIST_LMQ_NAME);\n\n        Assert.assertTrue(offsetTable.containsKey(otherKey));\n        Assert.assertFalse(offsetTable.containsKey(removeKey));\n        verify(consumerOffsetManager).removeConsumerOffset(removeKey);\n        verify(messageStore).deleteTopics(Collections.singleton(EXIST_LMQ_NAME));\n        verify(liteSubscriptionRegistry).cleanSubscription(EXIST_LMQ_NAME, false);\n        verify(consumerOrderInfoManager, times(1)).remove(EXIST_LMQ_NAME, GROUP);\n\n        // not sharding to this broker\n        when(liteSharding.shardingByLmqName(PARENT_TOPIC, EXIST_LMQ_NAME)).thenReturn(\"otherBrokerName\");\n        lifecycleManager.deleteLmq(PARENT_TOPIC, EXIST_LMQ_NAME);\n\n        Assert.assertTrue(offsetTable.containsKey(otherKey));\n        Assert.assertFalse(offsetTable.containsKey(removeKey));\n        verify(consumerOffsetManager, times(2)).removeConsumerOffset(removeKey);\n        verify(messageStore, times(2)).deleteTopics(Collections.singleton(EXIST_LMQ_NAME));\n        verify(liteSubscriptionRegistry, times(2)).cleanSubscription(EXIST_LMQ_NAME, false);\n    }\n\n    @Test\n    public void testCleanExpiredLiteTopic() {\n        String removeKey = EXIST_LMQ_NAME + TOPIC_GROUP_SEPARATOR + GROUP;\n        when(liteSharding.shardingByLmqName(PARENT_TOPIC, EXIST_LMQ_NAME)).thenReturn(brokerConfig.getBrokerName());\n\n        lifecycleManager.cleanExpiredLiteTopic();\n        verify(consumerOffsetManager).removeConsumerOffset(removeKey);\n        verify(messageStore).deleteTopics(Collections.singleton(EXIST_LMQ_NAME));\n        verify(liteSubscriptionRegistry).cleanSubscription(EXIST_LMQ_NAME, false);\n    }\n\n    @Test\n    public void testCleanByParentTopic() {\n        String removeKey = EXIST_LMQ_NAME + TOPIC_GROUP_SEPARATOR + GROUP;\n        when(liteSharding.shardingByLmqName(PARENT_TOPIC, EXIST_LMQ_NAME)).thenReturn(brokerConfig.getBrokerName());\n\n        lifecycleManager.cleanByParentTopic(PARENT_TOPIC);\n        verify(consumerOffsetManager).removeConsumerOffset(removeKey);\n        verify(messageStore).deleteTopics(Collections.singleton(EXIST_LMQ_NAME));\n        verify(liteSubscriptionRegistry).cleanSubscription(EXIST_LMQ_NAME, false);\n\n        lifecycleManager.cleanByParentTopic(\"whatever\");\n        verify(lifecycleManager, never()).collectByParentTopic(\"whatever\");\n    }\n\n    @Test\n    public void testRun() throws InterruptedException {\n        brokerConfig.setLiteTtlCheckInterval(100L);\n        when(liteSharding.shardingByLmqName(PARENT_TOPIC, EXIST_LMQ_NAME)).thenReturn(brokerConfig.getBrokerName());\n        lifecycleManager.start();\n        Thread.sleep(300);\n        lifecycleManager.shutdown();\n\n        verify(consumerOffsetManager, atLeastOnce()).removeConsumerOffset(anyString());\n        verify(messageStore, atLeastOnce()).deleteTopics(Collections.singleton(EXIST_LMQ_NAME));\n        verify(liteSubscriptionRegistry, atLeastOnce()).cleanSubscription(EXIST_LMQ_NAME, false);\n    }\n\n    private static class TestLiteLifecycleManager extends AbstractLiteLifecycleManager {\n        public TestLiteLifecycleManager(BrokerController brokerController, LiteSharding liteSharding) {\n            super(brokerController, liteSharding);\n        }\n\n        @Override\n        public long getMaxOffsetInQueue(String lmqName) {\n            return EXIST_LMQ_NAME.equals(lmqName) ? 100 : -1;\n        }\n\n        @Override\n        public List<Pair<String, String>> collectExpiredLiteTopic() {\n            return Collections.singletonList(new Pair<>(PARENT_TOPIC, EXIST_LMQ_NAME));\n        }\n\n        @Override\n        public List<String> collectByParentTopic(String parentTopic) {\n            return PARENT_TOPIC.equals(parentTopic) ? Collections.singletonList(EXIST_LMQ_NAME) : Collections.emptyList();\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/lite/LiteEventDispatcherTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.lite;\n\nimport com.google.common.cache.Cache;\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.longpolling.PopLiteLongPollingService;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.broker.pop.orderly.ConsumerOrderInfoManager;\nimport org.apache.rocketmq.broker.processor.PopLiteMessageProcessor;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.entity.ClientGroup;\nimport org.apache.rocketmq.common.lite.LiteSubscription;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.UUID;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.ConcurrentSkipListSet;\n\nimport static org.apache.rocketmq.broker.lite.LiteEventDispatcher.COMPARATOR;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.never;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n\n@RunWith(MockitoJUnitRunner.class)\npublic class LiteEventDispatcherTest {\n\n    @Mock\n    private BrokerController brokerController;\n    @Mock\n    private LiteSubscriptionRegistry liteSubscriptionRegistry;\n    @Mock\n    private AbstractLiteLifecycleManager liteLifecycleManager;\n    @Mock\n    private ConsumerOffsetManager consumerOffsetManager;\n    @Mock\n    private PopLiteMessageProcessor popLiteMessageProcessor;\n    @Mock\n    private PopLiteLongPollingService popLiteLongPollingService;\n    @Mock\n    private ConsumerOrderInfoManager consumerOrderInfoManager;\n    @Mock\n    private SubscriptionGroupManager subscriptionGroupManager;\n\n    private BrokerConfig brokerConfig;\n    private LiteEventDispatcher liteEventDispatcher;\n    private ConcurrentMap<String, LiteEventDispatcher.ClientEventSet> clientEventMap;\n    private Cache<String, Object> blacklist;\n\n    @SuppressWarnings(\"unchecked\")\n    @Before\n    public void setUp() throws IllegalAccessException {\n        brokerConfig = new BrokerConfig();\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);\n        when(brokerController.getPopLiteMessageProcessor()).thenReturn(popLiteMessageProcessor);\n        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);\n        when(popLiteMessageProcessor.getPopLiteLongPollingService()).thenReturn(popLiteLongPollingService);\n        when(popLiteMessageProcessor.getConsumerOrderInfoManager()).thenReturn(consumerOrderInfoManager);\n\n        LiteEventDispatcher testObject = new LiteEventDispatcher(brokerController, liteSubscriptionRegistry, liteLifecycleManager);\n        liteEventDispatcher = Mockito.spy(testObject);\n        liteEventDispatcher.init();\n\n        clientEventMap = (ConcurrentMap<String, LiteEventDispatcher.ClientEventSet>)\n            FieldUtils.readDeclaredField(testObject, \"clientEventMap\", true);\n        blacklist = (Cache<String, Object>) FieldUtils.readDeclaredField(testObject, \"blacklist\", true);\n    }\n\n    @After\n    public void reset() {\n        brokerConfig = new BrokerConfig();\n        clientEventMap.clear();\n        blacklist.invalidateAll();\n    }\n\n    @Test\n    public void testFullDispatchRequestComparator() {\n        LiteEventDispatcher.FullDispatchRequest request1 =\n            new LiteEventDispatcher.FullDispatchRequest(\"client1\", \"whatever\", 1000);\n        LiteEventDispatcher.FullDispatchRequest request2 =\n            new LiteEventDispatcher.FullDispatchRequest(\"client2\", \"whatever\", 2000);\n        LiteEventDispatcher.FullDispatchRequest request3 =\n            new LiteEventDispatcher.FullDispatchRequest(\"client1\", \"whatever\", 1000);\n\n        Assert.assertTrue(COMPARATOR.compare(request1, request2) < 0);\n        Assert.assertTrue(COMPARATOR.compare(request2, request1) > 0);\n        Assert.assertEquals(0, COMPARATOR.compare(request1, request3));\n    }\n\n    @Test\n    public void testFullDispatchSet() {\n        ConcurrentSkipListSet<LiteEventDispatcher.FullDispatchRequest> set =\n            new ConcurrentSkipListSet<>(COMPARATOR);\n\n        LiteEventDispatcher.FullDispatchRequest request1 =\n            new LiteEventDispatcher.FullDispatchRequest(\"client1\", \"whatever\", 1000);\n        LiteEventDispatcher.FullDispatchRequest request2 =\n            new LiteEventDispatcher.FullDispatchRequest(\"client2\", \"whatever\", 2000);\n        LiteEventDispatcher.FullDispatchRequest request3 =\n            new LiteEventDispatcher.FullDispatchRequest(\"client1\", \"whatever\", 1000);\n        LiteEventDispatcher.FullDispatchRequest request4 =\n            new LiteEventDispatcher.FullDispatchRequest(\"client3\", \"whatever\", 500);\n        LiteEventDispatcher.FullDispatchRequest request5 =\n            new LiteEventDispatcher.FullDispatchRequest(\"client4\", \"whatever\", 1000);\n        LiteEventDispatcher.FullDispatchRequest request6 =\n            new LiteEventDispatcher.FullDispatchRequest(null, \"whatever\", 1000);\n\n        set.add(request1);\n        set.add(request3);\n        set.add(request6);\n        Assert.assertEquals(1, set.size());\n        Assert.assertEquals(request1, set.pollFirst());\n\n        set.clear();\n        set.add(request1);\n        set.add(request2);\n        set.add(request3);\n        set.add(request4);\n        set.add(request5);\n        Assert.assertEquals(4, set.size());\n        Assert.assertEquals(request4, set.pollFirst());\n        Assert.assertEquals(request1, set.pollFirst());\n        Assert.assertEquals(request5, set.pollFirst());\n        Assert.assertEquals(request2, set.pollFirst());\n    }\n\n    @Test\n    public void testEventSetIterator() {\n        LiteEventDispatcher.ClientEventSet clientEventSet = liteEventDispatcher.new ClientEventSet(\"group\");\n        clientEventSet.offer(\"event1\");\n        clientEventSet.offer(\"event2\");\n\n        LiteEventDispatcher.EventSetIterator iterator = new LiteEventDispatcher.EventSetIterator(clientEventSet);\n\n        Assert.assertTrue(iterator.hasNext());\n        Assert.assertEquals(\"event1\", iterator.next());\n        Assert.assertTrue(iterator.hasNext());\n        Assert.assertEquals(\"event2\", iterator.next());\n        Assert.assertFalse(iterator.hasNext());\n    }\n\n    @Test\n    public void testLiteSubscriptionIterator() {\n        Iterator<String> topicIterator = Arrays.asList(\"event1\", \"event2\").iterator();\n\n        LiteEventDispatcher.LiteSubscriptionIterator iterator =\n            new LiteEventDispatcher.LiteSubscriptionIterator(\"parentTopic\", topicIterator);\n\n        Assert.assertTrue(iterator.hasNext());\n        Assert.assertEquals(\"event1\", iterator.next());\n        Assert.assertTrue(iterator.hasNext());\n        Assert.assertEquals(\"event2\", iterator.next());\n        Assert.assertFalse(iterator.hasNext());\n    }\n\n    @Test\n    public void testClientEventSet_offerAndPoll() {\n        brokerConfig.setMaxClientEventCount(3);\n        LiteEventDispatcher.ClientEventSet clientEventSet = liteEventDispatcher.new ClientEventSet(\"group\");\n\n        Assert.assertTrue(clientEventSet.offer(\"event1\"));\n        Assert.assertTrue(clientEventSet.offer(\"event2\"));\n        Assert.assertTrue(clientEventSet.offer(\"event1\"));\n        Assert.assertTrue(clientEventSet.offer(\"event3\"));\n        Assert.assertFalse(clientEventSet.offer(\"event4\"));\n\n        Assert.assertEquals(3, clientEventSet.size());\n        Assert.assertEquals(\"event1\", clientEventSet.poll());\n        Assert.assertEquals(\"event2\", clientEventSet.poll());\n        Assert.assertEquals(\"event3\", clientEventSet.poll());\n        Assert.assertEquals(0, clientEventSet.size());\n        Assert.assertNull(clientEventSet.poll());\n    }\n\n    @Test\n    public void testClientEventSet_isLowWaterMark() {\n        brokerConfig.setMaxClientEventCount(10);\n        LiteEventDispatcher.ClientEventSet clientEventSet = liteEventDispatcher.new ClientEventSet(\"group\");\n        Assert.assertTrue(clientEventSet.isLowWaterMark());\n\n        for (int i = 0; i < 4; i++) {\n            clientEventSet.offer(\"event\" + i);\n        }\n        Assert.assertFalse(clientEventSet.isLowWaterMark());\n    }\n\n    @Test\n    public void testClientEventSetMaybeBlock() throws Exception {\n        LiteEventDispatcher.ClientEventSet clientEventSet = liteEventDispatcher.new ClientEventSet(\"group\");\n        Assert.assertFalse(clientEventSet.maybeBlock());\n\n        clientEventSet.offer(\"event\");\n        FieldUtils.writeDeclaredField(clientEventSet, \"lastAccessTime\", 0L, true);\n        Assert.assertTrue(clientEventSet.maybeBlock());\n        clientEventSet.poll();\n        Assert.assertFalse(clientEventSet.maybeBlock());\n    }\n\n    @Test\n    public void testGetAllSubscriber_noSubscribers() {\n        when(liteSubscriptionRegistry.getSubscriber(\"event\")).thenReturn(null);\n        Object result = liteEventDispatcher.getAllSubscriber(\"group\", \"event\");\n        Assert.assertNull(result);\n    }\n\n    @Test\n    @SuppressWarnings(\"unchecked\")\n    public void testGetAllSubscriber_singleSubscriber() {\n        Set<ClientGroup> subscribers = new HashSet<>();\n        subscribers.add(new ClientGroup(\"clientId\", \"group\"));\n        when(liteSubscriptionRegistry.getSubscriber(\"event\")).thenReturn(subscribers);\n\n        Object result = liteEventDispatcher.getAllSubscriber(\"group\", \"event\"); // specified\n        Assert.assertTrue(result instanceof List);\n        Assert.assertEquals(1, ((List<?>) result).size());\n        Assert.assertEquals(\"clientId\", ((List<ClientGroup>) result).get(0).clientId);\n\n        result = liteEventDispatcher.getAllSubscriber(null, \"event\"); // not specified\n        Assert.assertTrue(result instanceof List);\n        Assert.assertEquals(1, ((List<?>) result).size());\n        Assert.assertEquals(\"clientId\", ((List<ClientGroup>) result).get(0).clientId);\n\n        result = liteEventDispatcher.getAllSubscriber(\"otherGroup\", \"event\"); // specified but not match\n        Assert.assertNull(result);\n    }\n\n    @Test\n    @SuppressWarnings(\"unchecked\")\n    public void testGetAllSubscriber_multipleSubscribers() {\n        Set<ClientGroup> subscribers = new HashSet<>();\n        subscribers.add(new ClientGroup(\"clientId1\", \"group1\"));\n        subscribers.add(new ClientGroup(\"clientId2\", \"group1\"));\n        subscribers.add(new ClientGroup(\"clientId3\", \"group2\"));\n        when(liteSubscriptionRegistry.getSubscriber(\"event\")).thenReturn(subscribers);\n\n        Object result = liteEventDispatcher.getAllSubscriber(\"group1\", \"event\"); // specified\n        Assert.assertTrue(result instanceof List);\n        Assert.assertEquals(2, ((List<?>) result).size());\n        Assert.assertEquals(\"clientId1\", ((List<ClientGroup>) result).get(0).clientId);\n\n        result = liteEventDispatcher.getAllSubscriber(\"group2\", \"event\"); // specified\n        Assert.assertTrue(result instanceof List);\n        Assert.assertEquals(1, ((List<?>) result).size());\n        Assert.assertEquals(\"clientId3\", ((List<ClientGroup>) result).get(0).clientId);\n\n        result = liteEventDispatcher.getAllSubscriber(\"otherGroup\", \"event\"); // specified but not match\n        Assert.assertNull(result);\n\n        result = liteEventDispatcher.getAllSubscriber(null, \"event\"); // not specified\n        Assert.assertTrue(result instanceof Map);\n        Assert.assertEquals(2, ((Map<?, ?>) result).size());\n        Assert.assertEquals(2, ((Map<String, List<ClientGroup>>) result).get(\"group1\").size());\n        Assert.assertEquals(1, ((Map<String, List<ClientGroup>>) result).get(\"group2\").size());\n    }\n\n    @Test\n    public void testTryDispatchToClient() {\n        brokerConfig.setMaxClientEventCount(1);\n        String clientId = \"clientId\";\n\n        boolean result = liteEventDispatcher.tryDispatchToClient(\"event1\", clientId, \"group\");\n        Assert.assertTrue(result);\n\n        // not in blacklist\n        result = liteEventDispatcher.tryDispatchToClient(\"event2\", clientId, \"group\");\n        Assert.assertFalse(result);\n        verify(liteEventDispatcher).scheduleFullDispatch(clientId, \"group\", false);\n\n        // in blacklist\n        blacklist.put(clientId, Boolean.TRUE);\n        result = liteEventDispatcher.tryDispatchToClient(\"event3\", clientId, \"group\");\n        Assert.assertFalse(result);\n        verify(liteEventDispatcher).scheduleFullDispatch(clientId, \"group\", true);\n\n        blacklist.invalidate(clientId);\n        result = liteEventDispatcher.tryDispatchToClient(\"event3\", clientId, \"group\");\n        Assert.assertFalse(result);\n        verify(liteEventDispatcher, times(2)).scheduleFullDispatch(clientId, \"group\", false);\n    }\n\n    @Test\n    public void testSelectAndDispatch_empty_or_singleClient() {\n        List<ClientGroup> clients = Collections.singletonList(new ClientGroup(\"client\", \"group\"));\n        // disable event mode\n        brokerConfig.setEnableLiteEventMode(false);\n        liteEventDispatcher.selectAndDispatch(\"event\", clients, null);\n        verify(liteEventDispatcher, never()).tryDispatchToClient(anyString(), anyString(), anyString());\n\n        // empty list\n        liteEventDispatcher.selectAndDispatch(\"event\", Collections.emptyList(), null);\n        verify(liteEventDispatcher, never()).tryDispatchToClient(anyString(), anyString(), anyString());\n\n        // event mode\n        brokerConfig.setMaxClientEventCount(2);\n        brokerConfig.setEnableLiteEventMode(true);\n\n        liteEventDispatcher.selectAndDispatch(\"event1\", clients, null);\n        liteEventDispatcher.selectAndDispatch(\"event2\", clients, \"client\"); // exclude\n        liteEventDispatcher.selectAndDispatch(\"event3\", clients, null);\n        verify(popLiteLongPollingService, times(2)).notifyMessageArriving(\"client\", true, 0, \"group\");\n    }\n\n    @Test\n    public void testSelectAndDispatch_multipleClients() {\n        brokerConfig.setMaxClientEventCount(2);\n        String client1 = UUID.randomUUID().toString();\n        String client2 = UUID.randomUUID().toString();\n        List<ClientGroup> clients = Arrays.asList(\n            new ClientGroup(client1, \"group\"),\n            new ClientGroup(client2, \"group\"));\n\n        // no fallback\n        liteEventDispatcher.selectAndDispatch(\"event1\", clients, client1);\n        verify(popLiteLongPollingService).notifyMessageArriving(client2, true, 0, \"group\");\n\n        // no fallback\n        liteEventDispatcher.selectAndDispatch(\"event2\", clients, client2);\n        verify(popLiteLongPollingService).notifyMessageArriving(client1, true, 0, \"group\");\n\n        // fallback\n        blacklist.put(client1, Boolean.TRUE);\n        liteEventDispatcher.selectAndDispatch(\"event3\", clients, null);\n        verify(popLiteLongPollingService, times(2)).notifyMessageArriving(client2, true, 0, \"group\");\n\n        // fallback\n        blacklist.invalidate(client1);\n        blacklist.put(client2, Boolean.TRUE);\n        liteEventDispatcher.selectAndDispatch(\"event4\", clients, null);\n        verify(popLiteLongPollingService, times(2)).notifyMessageArriving(client1, true, 0, \"group\");\n\n        // queue all full\n        liteEventDispatcher.selectAndDispatch(\"event5\", clients, null);\n        verify(popLiteLongPollingService, times(2)).notifyMessageArriving(client1, true, 0, \"group\");\n        verify(popLiteLongPollingService, times(2)).notifyMessageArriving(client2, true, 0, \"group\");\n    }\n\n    @Test\n    public void testDispatch() {\n        // disable event mode\n        brokerConfig.setEnableLiteEventMode(false);\n        liteEventDispatcher.dispatch(\"group\", \"event\", 0, 0, System.currentTimeMillis());\n        verify(liteEventDispatcher, never()).getAllSubscriber(anyString(), anyString());\n\n        // event mode\n        brokerConfig.setEnableLiteEventMode(true);\n        liteEventDispatcher.dispatch(\"group\", \"event\", 1, 0, System.currentTimeMillis()); // queue id not match\n        liteEventDispatcher.dispatch(\"group\", \"event\", 0, 0, System.currentTimeMillis()); // queue name not match\n        verify(liteEventDispatcher, never()).getAllSubscriber(anyString(), anyString());\n\n        // do dispatch\n        liteEventDispatcher.dispatch(\"group\", LiteUtil.toLmqName(\"p\", \"l\"), 0, 0, System.currentTimeMillis());\n        verify(liteEventDispatcher).getAllSubscriber(anyString(), anyString());\n    }\n\n    @Test\n    public void testDoFullDispatch_disable_or_emptySubscription() {\n        String clientId = \"clientId\";\n        String group = \"group\";\n\n        // disable event mode\n        brokerConfig.setEnableLiteEventMode(false);\n        liteEventDispatcher.doFullDispatch(clientId, group);\n        verify(liteSubscriptionRegistry, never()).getLiteSubscription(clientId);\n\n        // empty subscription\n        brokerConfig.setEnableLiteEventMode(true);\n        when(liteSubscriptionRegistry.getLiteSubscription(\"clientId\")).thenReturn(null);\n        liteEventDispatcher.doFullDispatch(clientId, group);\n        verify(liteLifecycleManager, never()).getMaxOffsetInQueue(anyString());\n    }\n\n    @Test\n    public void testDoFullDispatch_maybeBlock() throws Exception {\n        int num = 10;\n        String clientId = \"clientId\";\n        String group = \"group\";\n        LiteSubscription subscription = new LiteSubscription();\n        subscription.setTopic(\"parentTopic\");\n        for (int i = 0; i < num; i++) {\n            subscription.addLiteTopic(LiteUtil.toLmqName(subscription.getTopic(), \"l\" + i));\n        }\n        when(liteSubscriptionRegistry.getLiteSubscription(clientId)).thenReturn(subscription);\n\n        // maybe block\n        liteEventDispatcher.tryDispatchToClient(\"event\", clientId, group);\n        Assert.assertNotNull(clientEventMap.get(clientId));\n        FieldUtils.writeDeclaredField(clientEventMap.get(clientId), \"lastAccessTime\", 0L, true);\n        liteEventDispatcher.doFullDispatch(clientId, group);\n        verify(liteEventDispatcher).scheduleFullDispatch(clientId, group, true);\n        verify(liteLifecycleManager, never()).getMaxOffsetInQueue(anyString());\n    }\n\n    @Test\n    public void testDoFullDispatch_highWaterMark() throws Exception {\n        int num = 10;\n        String clientId = \"clientId\";\n        String group = \"group\";\n        LiteSubscription subscription = new LiteSubscription();\n        subscription.setTopic(\"parentTopic\");\n        for (int i = 0; i < num; i++) {\n            subscription.addLiteTopic(LiteUtil.toLmqName(subscription.getTopic(), \"l\" + i));\n        }\n        when(liteSubscriptionRegistry.getLiteSubscription(clientId)).thenReturn(subscription);\n\n        brokerConfig.setMaxClientEventCount(1);\n\n        // active consuming\n        liteEventDispatcher.tryDispatchToClient(\"event\", clientId, group);\n        liteEventDispatcher.doFullDispatch(clientId, group);\n\n        verify(liteEventDispatcher).scheduleFullDispatch(clientId, group, false);\n        verify(liteLifecycleManager, never()).getMaxOffsetInQueue(anyString());\n\n        // not active consuming\n        clientEventMap.clear();\n        liteEventDispatcher.tryDispatchToClient(\"event\", clientId, group);\n        FieldUtils.writeDeclaredField(clientEventMap.get(clientId), \"lastAccessTime\", System.currentTimeMillis() - 6000L, true);\n        liteEventDispatcher.doFullDispatch(clientId, group);\n\n        verify(liteEventDispatcher).scheduleFullDispatch(clientId, group, true);\n        verify(liteLifecycleManager, never()).getMaxOffsetInQueue(anyString());\n    }\n\n    @Test\n    public void testDoFullDispatch_multipleTopics() {\n        String clientId = \"clientId\";\n        String group = \"group\";\n\n        String lmqName1 = \"lmqName1\";\n        String lmqName2 = \"lmqName2\";\n        String lmqName3 = \"lmqName2\";\n        LiteSubscription subscription = new LiteSubscription();\n        subscription.setTopic(\"parentTopic\");\n        subscription.addLiteTopic(lmqName1);\n        subscription.addLiteTopic(lmqName2);\n        subscription.addLiteTopic(lmqName3);\n        when(liteSubscriptionRegistry.getLiteSubscription(clientId)).thenReturn(subscription);\n\n\n        when(liteLifecycleManager.getMaxOffsetInQueue(lmqName1)).thenReturn(0L);\n\n        when(liteLifecycleManager.getMaxOffsetInQueue(lmqName2)).thenReturn(10L);\n        when(consumerOffsetManager.queryOffset(group, lmqName2, 0)).thenReturn(10L);\n\n        when(liteLifecycleManager.getMaxOffsetInQueue(lmqName3)).thenReturn(10L);\n        when(consumerOffsetManager.queryOffset(group, lmqName3, 0)).thenReturn(5L);\n\n        liteEventDispatcher.doFullDispatch(clientId, group);\n\n        verify(liteLifecycleManager).getMaxOffsetInQueue(lmqName1);\n        verify(liteLifecycleManager).getMaxOffsetInQueue(lmqName2);\n        verify(liteLifecycleManager).getMaxOffsetInQueue(lmqName3);\n        verify(consumerOffsetManager, never()).queryOffset(group, lmqName1, 0);\n        verify(consumerOffsetManager).queryOffset(group, lmqName2, 0);\n        verify(consumerOffsetManager).queryOffset(group, lmqName3, 0);\n\n        verify(liteEventDispatcher, never()).scheduleFullDispatch(clientId, group, true);\n        verify(popLiteLongPollingService, times(2)).notifyMessageArriving(clientId, true, 0, group);\n    }\n\n    @Test\n    public void testDoFullDispatch_eventQueueFull() throws IllegalAccessException {\n        brokerConfig.setMaxClientEventCount(2);\n        String clientId = \"clientId\";\n        String group = \"group\";\n\n        String lmqName1 = \"lmqName1\";\n        String lmqName2 = \"lmqName2\";\n        String lmqName3 = \"lmqName3\";\n        LiteSubscription subscription = new LiteSubscription();\n        subscription.setTopic(\"parentTopic\");\n        subscription.addLiteTopic(lmqName1);\n        subscription.addLiteTopic(lmqName2);\n        subscription.addLiteTopic(lmqName3);\n        when(liteSubscriptionRegistry.getLiteSubscription(clientId)).thenReturn(subscription);\n\n        when(liteLifecycleManager.getMaxOffsetInQueue(lmqName1)).thenReturn(10L);\n        when(consumerOffsetManager.queryOffset(group, lmqName1, 0)).thenReturn(5L);\n\n        when(liteLifecycleManager.getMaxOffsetInQueue(lmqName2)).thenReturn(10L);\n        when(consumerOffsetManager.queryOffset(group, lmqName2, 0)).thenReturn(5L);\n\n        when(liteLifecycleManager.getMaxOffsetInQueue(lmqName3)).thenReturn(10L);\n        when(consumerOffsetManager.queryOffset(group, lmqName3, 0)).thenReturn(5L);\n\n        // active consuming\n        liteEventDispatcher.doFullDispatch(clientId, group);\n        verify(liteEventDispatcher).scheduleFullDispatch(clientId, group, false);\n        verify(popLiteLongPollingService, times(2)).notifyMessageArriving(clientId, true, 0, group);\n        Assert.assertNotNull(clientEventMap.get(clientId).poll());\n        Assert.assertNotNull(clientEventMap.get(clientId).poll());\n\n        // not active consuming\n        FieldUtils.writeDeclaredField(clientEventMap.get(clientId), \"lastAccessTime\", System.currentTimeMillis() - 6000L, true);\n        liteEventDispatcher.doFullDispatch(clientId, group);\n        verify(liteEventDispatcher).scheduleFullDispatch(clientId, group, true);\n        verify(popLiteLongPollingService, times(4)).notifyMessageArriving(clientId, true, 0, group);\n    }\n\n    @Test\n    public void testDoFullDispatchByGroup() {\n        String group = \"group\";\n        String clientId1 = \"client1\";\n        String clientId2 = \"client2\";\n        List<String> clientIds = Arrays.asList(clientId1, clientId2);\n        Mockito.when(liteSubscriptionRegistry.getAllClientIdByGroup(group)).thenReturn(clientIds);\n\n        liteEventDispatcher.doFullDispatchByGroup(group);\n\n        verify(liteSubscriptionRegistry, times(1)).getAllClientIdByGroup(group);\n        verify(liteEventDispatcher, times(1)).doFullDispatch(clientId1, group);\n        verify(liteEventDispatcher, times(1)).doFullDispatch(clientId2, group);\n    }\n\n    @Test\n    public void testScan() throws Exception {\n        String clientId = \"clientId\";\n        String group = \"group\";\n        String event = \"event\";\n        liteEventDispatcher.tryDispatchToClient(event, clientId, group);\n\n        Assert.assertNotNull(clientEventMap.get(clientId));\n        FieldUtils.writeDeclaredField(clientEventMap.get(clientId), \"lastAccessTime\", 0L, true);\n        liteEventDispatcher.scan();\n        verify(liteEventDispatcher).getAllSubscriber(group, event);\n    }\n\n    @Test\n    public void testFullDispatchDeduplication() throws InterruptedException {\n        String clientId1 = \"clientId1\";\n        String clientId2 = \"clientId2\";\n        String group = \"group\";\n        brokerConfig.setLiteEventFullDispatchDelayTime(10L);\n        liteEventDispatcher.scheduleFullDispatch(clientId1, group, false);\n        liteEventDispatcher.scheduleFullDispatch(clientId1, group, false);\n        liteEventDispatcher.scheduleFullDispatch(clientId1, group, false);\n        liteEventDispatcher.scheduleFullDispatch(clientId1, group, false);\n        liteEventDispatcher.scheduleFullDispatch(clientId2, group, false);\n\n        Thread.sleep(20L);\n        liteEventDispatcher.scan();\n        verify(liteEventDispatcher, times(1)).doFullDispatch(clientId1, group);\n        verify(liteEventDispatcher, times(1)).doFullDispatch(clientId2, group);\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/lite/LiteLifecycleManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.lite;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.TopicAttributes;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.junit.AfterClass;\nimport org.junit.Assert;\nimport org.junit.BeforeClass;\nimport org.junit.Ignore;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.io.File;\nimport java.util.List;\nimport java.util.UUID;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.stream.Collectors;\nimport java.util.stream.IntStream;\n\nimport static java.util.concurrent.TimeUnit.MILLISECONDS;\nimport static java.util.concurrent.TimeUnit.SECONDS;\nimport static org.awaitility.Awaitility.await;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class LiteLifecycleManagerTest {\n\n    private final static BrokerConfig BROKER_CONFIG = new BrokerConfig();\n    private final static ConcurrentMap<String, TopicConfig> TOPIC_CONFIG_TABLE = new ConcurrentHashMap<>();\n    private static String storePathRootDir;\n    private static MessageStore messageStore;\n    private static LiteLifecycleManager liteLifecycleManager;\n    private static TopicConfig mockTopicConfig = new TopicConfig();\n\n    @BeforeClass\n    public static void setUp() throws Exception {\n        storePathRootDir = System.getProperty(\"java.io.tmpdir\") + File.separator + \"store-lifecycleTest\";\n        UtilAll.deleteFile(new File(storePathRootDir));\n\n        messageStore = LiteTestUtil.buildMessageStore(storePathRootDir, BROKER_CONFIG, TOPIC_CONFIG_TABLE, false);\n        messageStore.load();\n        messageStore.start();\n\n        BrokerController brokerController = Mockito.mock(BrokerController.class);\n        LiteSharding liteSharding = Mockito.mock(LiteSharding.class);\n        TopicConfigManager topicConfigManager = Mockito.mock(TopicConfigManager.class);\n        SubscriptionGroupManager subscriptionGroupManager = Mockito.mock(SubscriptionGroupManager.class);\n\n        when(brokerController.getBrokerConfig()).thenReturn(BROKER_CONFIG);\n        when(brokerController.getMessageStore()).thenReturn(messageStore);\n        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);\n        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);\n        when(topicConfigManager.getTopicConfigTable()).thenReturn(TOPIC_CONFIG_TABLE);\n        when(topicConfigManager.selectTopicConfig(anyString())).thenReturn(mockTopicConfig);\n        when(subscriptionGroupManager.getSubscriptionGroupTable()).thenReturn(new ConcurrentHashMap<>());\n\n        LiteLifecycleManager testObject = new LiteLifecycleManager(brokerController, liteSharding);\n        liteLifecycleManager = Mockito.spy(testObject);\n        liteLifecycleManager.init();\n    }\n\n    @AfterClass\n    public static void reset() {\n        messageStore.shutdown();\n        messageStore.destroy();\n        UtilAll.deleteFile(new File(storePathRootDir));\n    }\n\n    @Test\n    public void testGetMaxOffsetInQueue() {\n        int num = 3;\n        String topic = UUID.randomUUID().toString();\n        for (int i = 0; i < num; i++) {\n            messageStore.putMessage(LiteTestUtil.buildMessage(topic, null));\n        }\n        await().atMost(5, SECONDS).pollInterval(200, MILLISECONDS).until(() -> messageStore.dispatchBehindBytes() <= 0);\n        Assert.assertEquals(num, liteLifecycleManager.getMaxOffsetInQueue(topic));\n        Assert.assertEquals(0, liteLifecycleManager.getMaxOffsetInQueue(UUID.randomUUID().toString()));\n    }\n\n    @Test\n    public void testCollectByParentTopic() {\n        int num = 3;\n        String parentTopic = UUID.randomUUID().toString();\n        for (int i = 0; i < num; i++) {\n            messageStore.putMessage(LiteTestUtil.buildMessage(parentTopic, UUID.randomUUID().toString()));\n            messageStore.putMessage(LiteTestUtil.buildMessage(UUID.randomUUID().toString(), UUID.randomUUID().toString()));\n        }\n        await().atMost(5, SECONDS).pollInterval(200, MILLISECONDS).until(() -> messageStore.dispatchBehindBytes() <= 0);\n        List<String> result = liteLifecycleManager.collectByParentTopic(parentTopic);\n        Assert.assertEquals(num, result.size());\n        for (String lmqName : result) {\n            Assert.assertTrue(LiteUtil.belongsTo(lmqName, parentTopic));\n        }\n\n        result = liteLifecycleManager.collectByParentTopic(UUID.randomUUID().toString());\n        Assert.assertEquals(0, result.size());\n    }\n\n    @Test\n    public void testCollectExpiredLiteTopic() {\n        int num = 3;\n        String parentTopic = UUID.randomUUID().toString();\n        for (int i = 0; i < num; i++) {\n            messageStore.putMessage(LiteTestUtil.buildMessage(parentTopic, UUID.randomUUID().toString()));\n            messageStore.putMessage(LiteTestUtil.buildMessage(UUID.randomUUID().toString(), null));\n        }\n        await().atMost(5, SECONDS).pollInterval(200, MILLISECONDS).until(() -> messageStore.dispatchBehindBytes() <= 0);\n\n        when(liteLifecycleManager.isLiteTopicExpired(anyString(), anyString(), anyLong())).thenReturn(false);\n        List<Pair<String, String>> result = liteLifecycleManager.collectExpiredLiteTopic();\n        Assert.assertEquals(0, result.size());\n\n        when(liteLifecycleManager.isLiteTopicExpired(eq(parentTopic), anyString(), anyLong())).thenReturn(true);\n        result = liteLifecycleManager.collectExpiredLiteTopic();\n        Assert.assertEquals(num, result.size());\n        for (Pair<String, String> pair : result) {\n            Assert.assertEquals(parentTopic, pair.getObject1());\n            Assert.assertTrue(LiteUtil.belongsTo(pair.getObject2(), parentTopic));\n        }\n    }\n\n    @Ignore\n    @Test\n    public void testCleanExpiredLiteTopic() {\n        int num = 3;\n        String parentTopic = UUID.randomUUID().toString();\n        List<String> liteTopics =\n            IntStream.range(0, 3).mapToObj(i -> UUID.randomUUID().toString()).collect(Collectors.toList());\n        for (int i = 0; i < num; i++) {\n            messageStore.putMessage(LiteTestUtil.buildMessage(parentTopic, liteTopics.get(i)));\n        }\n        await().atMost(5, SECONDS).pollInterval(200, MILLISECONDS).until(() -> messageStore.dispatchBehindBytes() <= 0);\n\n        for (int i = 0; i < num; i++) {\n            String lmqName = LiteUtil.toLmqName(parentTopic, liteTopics.get(i));\n            Assert.assertTrue(messageStore.getQueueStore().getConsumeQueueTable().containsKey(lmqName));\n        }\n\n        when(liteLifecycleManager.isLiteTopicExpired(eq(parentTopic), anyString(), anyLong())).thenReturn(true);\n        liteLifecycleManager.cleanExpiredLiteTopic();\n\n        for (int i = 0; i < num; i++) {\n            String lmqName = LiteUtil.toLmqName(parentTopic, liteTopics.get(i));\n            Assert.assertFalse(messageStore.getQueueStore().getConsumeQueueTable().containsKey(lmqName));\n        }\n    }\n\n    @Test\n    public void testCleanByParentTopic() {\n        int num = 3;\n        String parentTopic = UUID.randomUUID().toString();\n        mockTopicConfig.getAttributes().put(\n            TopicAttributes.TOPIC_MESSAGE_TYPE_ATTRIBUTE.getName(), TopicMessageType.LITE.getValue());\n\n        List<String> liteTopics =\n            IntStream.range(0, 3).mapToObj(i -> UUID.randomUUID().toString()).collect(Collectors.toList());\n        for (int i = 0; i < num; i++) {\n            messageStore.putMessage(LiteTestUtil.buildMessage(parentTopic, liteTopics.get(i)));\n        }\n        await().atMost(5, SECONDS).pollInterval(200, MILLISECONDS).until(() -> messageStore.dispatchBehindBytes() <= 0);\n\n        for (int i = 0; i < num; i++) {\n            String lmqName = LiteUtil.toLmqName(parentTopic, liteTopics.get(i));\n            Assert.assertTrue(messageStore.getQueueStore().getConsumeQueueTable().containsKey(lmqName));\n        }\n\n        liteLifecycleManager.cleanByParentTopic(parentTopic);\n\n        for (int i = 0; i < num; i++) {\n            String lmqName = LiteUtil.toLmqName(parentTopic, liteTopics.get(i));\n            Assert.assertFalse(messageStore.getQueueStore().getConsumeQueueTable().containsKey(lmqName));\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/lite/LiteShardingImplTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.lite;\n\nimport com.google.common.hash.Hashing;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.topic.TopicRouteInfoManager;\nimport org.apache.rocketmq.client.impl.producer.TopicPublishInfo;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class LiteShardingImplTest {\n\n    @Mock\n    private BrokerController brokerController;\n\n    @Mock\n    private TopicRouteInfoManager topicRouteInfoManager;\n\n    private LiteShardingImpl liteSharding;\n\n    @Before\n    public void setUp() {\n        liteSharding = new LiteShardingImpl(brokerController, topicRouteInfoManager);\n    }\n\n    /**\n     * Test normal case: multiple MessageQueues, verify consistent hash selects correct brokerName\n     */\n    @Test\n    public void testShardingByLmqName_NormalCase() {\n        // Prepare data\n        String parentTopic = \"TestTopic\";\n        String liteTopic = \"lite_topic\";\n        String lmqName = LiteUtil.toLmqName(parentTopic, liteTopic);\n        String brokerName1 = \"BrokerA\";\n        String brokerName2 = \"BrokerB\";\n\n        TopicPublishInfo topicPublishInfo = mock(TopicPublishInfo.class);\n        List<MessageQueue> messageQueues = new ArrayList<>();\n        MessageQueue mq1 = mock(MessageQueue.class);\n        MessageQueue mq2 = mock(MessageQueue.class);\n        when(mq1.getBrokerName()).thenReturn(brokerName1);\n//        when(mq2.getBrokerName()).thenReturn(brokerName2);\n        messageQueues.add(mq1);\n        messageQueues.add(mq2);\n\n        when(topicPublishInfo.getMessageQueueList()).thenReturn(messageQueues);\n        when(topicRouteInfoManager.tryToFindTopicPublishInfo(parentTopic)).thenReturn(topicPublishInfo);\n\n        // Execute method\n        String brokerName = liteSharding.shardingByLmqName(parentTopic, lmqName);\n\n        // Verify consistent hash selected bucket\n        int bucket = Hashing.consistentHash(liteTopic.hashCode(), messageQueues.size());\n        MessageQueue expectedMq = messageQueues.get(bucket);\n        String expectedBrokerName = expectedMq.getBrokerName();\n\n        assertEquals(expectedBrokerName, brokerName);\n    }\n\n    /**\n     * Test edge case: empty MessageQueue list should return current broker name\n     */\n    @Test\n    public void testShardingByLmqName_EmptyQueueList() {\n        String parentTopic = \"TestTopic\";\n        String lmqName = \"LmqName2\";\n        String currentBrokerName = \"CurrentBroker\";\n\n        BrokerConfig brokerConfig = mock(BrokerConfig.class);\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        when(brokerConfig.getBrokerName()).thenReturn(currentBrokerName);\n\n        TopicPublishInfo topicPublishInfo = mock(TopicPublishInfo.class);\n        when(topicPublishInfo.getMessageQueueList()).thenReturn(new ArrayList<>());\n        when(topicRouteInfoManager.tryToFindTopicPublishInfo(parentTopic)).thenReturn(topicPublishInfo);\n\n        String brokerName = liteSharding.shardingByLmqName(parentTopic, lmqName);\n\n        assertEquals(currentBrokerName, brokerName);\n    }\n\n    /**\n     * Test exception case: tryToFindTopicPublishInfo returns null, should return current broker name\n     */\n    @Test\n    public void testShardingByLmqName_NullTopicPublishInfo() {\n        String parentTopic = \"TestTopic\";\n        String lmqName = \"LmqName3\";\n        String currentBrokerName = \"CurrentBroker\";\n\n        BrokerConfig brokerConfig = mock(BrokerConfig.class);\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        when(brokerConfig.getBrokerName()).thenReturn(currentBrokerName);\n\n        when(topicRouteInfoManager.tryToFindTopicPublishInfo(parentTopic)).thenReturn(null);\n\n        String brokerName = liteSharding.shardingByLmqName(parentTopic, lmqName);\n\n        assertEquals(currentBrokerName, brokerName);\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/lite/LiteSubscriptionRegistryImplTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.lite;\n\nimport io.netty.channel.Channel;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.broker.pop.orderly.QueueLevelConsumerManager;\nimport org.apache.rocketmq.broker.processor.PopLiteMessageProcessor;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.attribute.LiteSubModel;\nimport org.apache.rocketmq.common.entity.ClientGroup;\nimport org.apache.rocketmq.common.lite.LiteSubscription;\nimport org.apache.rocketmq.common.lite.OffsetOption;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.apache.rocketmq.common.SubscriptionGroupAttributes.LITE_SUB_MODEL_ATTRIBUTE;\nimport static org.junit.jupiter.api.Assertions.assertEquals;\nimport static org.junit.jupiter.api.Assertions.assertFalse;\nimport static org.junit.jupiter.api.Assertions.assertNotNull;\nimport static org.junit.jupiter.api.Assertions.assertNull;\nimport static org.junit.jupiter.api.Assertions.assertThrows;\nimport static org.junit.jupiter.api.Assertions.assertTrue;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.clearInvocations;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.never;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\npublic class LiteSubscriptionRegistryImplTest {\n\n    private LiteSubscriptionRegistryImpl registry;\n    private LiteCtlListener mockListener;\n    private AbstractLiteLifecycleManager mockLifecycleManager;\n    private BrokerConfig mockBrokerConfig;\n    private SubscriptionGroupManager mockSubscriptionGroupManager;\n    private ConsumerOffsetManager mockConsumerOffsetManager;\n\n    @Before\n    public void setUp() {\n        BrokerController mockBrokerController = mock(BrokerController.class);\n        mockLifecycleManager = mock(AbstractLiteLifecycleManager.class);\n        mockBrokerConfig = mock(BrokerConfig.class);\n        mockSubscriptionGroupManager = mock(SubscriptionGroupManager.class);\n        mockConsumerOffsetManager = mock(ConsumerOffsetManager.class);\n        PopLiteMessageProcessor mockPopLiteMessageProcessor = mock(PopLiteMessageProcessor.class);\n        QueueLevelConsumerManager mockConsumerOrderInfoManager = mock(QueueLevelConsumerManager.class);\n\n        when(mockBrokerController.getBrokerConfig()).thenReturn(mockBrokerConfig);\n        when(mockBrokerController.getSubscriptionGroupManager()).thenReturn(mockSubscriptionGroupManager);\n        when(mockBrokerController.getConsumerOffsetManager()).thenReturn(mockConsumerOffsetManager);\n        when(mockBrokerController.getPopLiteMessageProcessor()).thenReturn(mockPopLiteMessageProcessor);\n        when(mockPopLiteMessageProcessor.getConsumerOrderInfoManager()).thenReturn(mockConsumerOrderInfoManager);\n        when(mockConsumerOrderInfoManager.getTable()).thenReturn(new ConcurrentHashMap<>());\n        when(mockBrokerConfig.getMaxLiteSubscriptionCount()).thenReturn(1000L);\n        when(mockBrokerConfig.getLiteSubscriptionCheckTimeoutMills()).thenReturn(60000L);\n        when(mockBrokerConfig.getLiteSubscriptionCheckInterval()).thenReturn(10000L);\n\n        registry = new LiteSubscriptionRegistryImpl(mockBrokerController, mockLifecycleManager);\n        mockListener = mock(LiteCtlListener.class);\n        registry.addListener(mockListener);\n    }\n\n    // Test addIncremental method\n    @Test\n    public void testAddPartialSubscription_BasicFunctionality() {\n        String clientId = \"client1\";\n        String group = \"group1\";\n        String topic = \"topic1\";\n        Set<String> liteTopicSet = new HashSet<>();\n        liteTopicSet.add(\"lmq1\");\n        liteTopicSet.add(\"lmq2\");\n\n        when(mockLifecycleManager.isSubscriptionActive(anyString(), anyString())).thenReturn(true);\n\n        registry.addPartialSubscription(clientId, group, topic, liteTopicSet, null);\n\n        LiteSubscription subscription = registry.getLiteSubscription(clientId);\n        assertNotNull(subscription);\n        assertEquals(group, subscription.getGroup());\n        assertEquals(topic, subscription.getTopic());\n        assertTrue(subscription.getLiteTopicSet().containsAll(liteTopicSet));\n\n        assertEquals(liteTopicSet.size(), registry.liteTopic2Group.size());\n        Set<ClientGroup> topicGroupSet = registry.liteTopic2Group.get(\"lmq1\");\n        assertEquals(1, topicGroupSet.size());\n        ClientGroup registeredGroup = topicGroupSet.iterator().next();\n        assertEquals(clientId, registeredGroup.clientId);\n        assertEquals(group, registeredGroup.group);\n\n        verify(mockListener, times(2)).onRegister(eq(clientId), eq(group), anyString());\n    }\n\n    @Test\n    public void testAddPartialSubscription_ExclusiveMode() {\n        String existingClientId = \"existingClient\";\n        String newClientId = \"newClient\";\n        String group = \"group\";\n        String topic = \"topic\";\n        String liteTopic = \"lmq1\";\n\n        Set<String> liteTopicSet = new HashSet<>();\n        liteTopicSet.add(liteTopic);\n\n        when(mockLifecycleManager.isSubscriptionActive(anyString(), anyString())).thenReturn(true);\n\n        // Mock subscription group config for reset offset behavior\n        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n        subscriptionGroupConfig.setGroupName(group);\n        subscriptionGroupConfig.getAttributes().put(LITE_SUB_MODEL_ATTRIBUTE.getName(), LiteSubModel.Exclusive.name());\n        when(mockSubscriptionGroupManager.findSubscriptionGroupConfig(group)).thenReturn(subscriptionGroupConfig);\n\n        // Add existing client\n        registry.addPartialSubscription(existingClientId, group, topic, liteTopicSet, null);\n\n        // Verify that the existing client is correctly registered\n        LiteSubscription existingSubscription = registry.getLiteSubscription(existingClientId);\n        assertNotNull(existingSubscription);\n        assertTrue(existingSubscription.getLiteTopicSet().contains(liteTopic));\n\n        // Execute exclusive mode addition\n        Set<String> newLiteTopicSet = new HashSet<>();\n        newLiteTopicSet.add(liteTopic);\n        registry.addPartialSubscription(newClientId, group, topic, newLiteTopicSet, null);\n\n        // Verify that new client subscription has been added.\n        LiteSubscription newSubscription = registry.getLiteSubscription(newClientId);\n        assertNotNull(newSubscription);\n        assertTrue(newSubscription.getLiteTopicSet().contains(liteTopic));\n\n        assertEquals(liteTopicSet.size(), registry.liteTopic2Group.size());\n        Set<ClientGroup> topicGroupSet = registry.liteTopic2Group.get(liteTopic);\n        assertEquals(1, topicGroupSet.size());\n        ClientGroup registeredGroup = topicGroupSet.iterator().next();\n        assertEquals(newClientId, registeredGroup.clientId);\n        assertEquals(group, registeredGroup.group);\n\n        verify(mockListener).onRegister(existingClientId, group, liteTopic);\n        verify(mockListener).onRegister(newClientId, group, liteTopic);\n        verify(mockListener).onUnregister(existingClientId, group, liteTopic);\n    }\n\n    @Test\n    public void testAddPartialSubscription_NonExclusiveMode() {\n        // Add an existing client subscription first\n        String existingClientId = \"existingClient\";\n        String newClientId = \"newClient\";\n        String group = \"group1\";\n        String topic = \"topic1\";\n        String liteTopic = \"lmq1\";\n\n        Set<String> existingLiteTopicSet = new HashSet<>();\n        existingLiteTopicSet.add(liteTopic);\n\n        when(mockLifecycleManager.isSubscriptionActive(anyString(), anyString())).thenReturn(true);\n\n        // Mock subscription group config\n        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n        subscriptionGroupConfig.setGroupName(group);\n        when(mockSubscriptionGroupManager.findSubscriptionGroupConfig(group)).thenReturn(subscriptionGroupConfig);\n\n        // Add existing client\n        registry.addPartialSubscription(existingClientId, group, topic, existingLiteTopicSet, null);\n\n        // Add new client in non-exclusive mode\n        Set<String> newLiteTopicSet = new HashSet<>();\n        newLiteTopicSet.add(liteTopic);\n        registry.addPartialSubscription(newClientId, group, topic, newLiteTopicSet, null);\n\n        // Verify both client subscriptions exist\n        LiteSubscription existingSubscription = registry.getLiteSubscription(existingClientId);\n        LiteSubscription newSubscription = registry.getLiteSubscription(newClientId);\n        assertNotNull(existingSubscription);\n        assertNotNull(newSubscription);\n        assertTrue(existingSubscription.getLiteTopicSet().contains(liteTopic));\n        assertTrue(newSubscription.getLiteTopicSet().contains(liteTopic));\n\n        // Verify listener was only called for registration, not unregistration\n        verify(mockListener, times(2)).onRegister(anyString(), eq(group), eq(liteTopic));\n        verify(mockListener, never()).onUnregister(anyString(), anyString(), anyString());\n    }\n\n    @Test\n    public void testAddPartialSubscription_WithEmptyLiteTopicSet() {\n        String clientId = \"client1\";\n        String group = \"group1\";\n        String topic = \"topic1\";\n        Set<String> liteTopicSet = new HashSet<>();\n\n        registry.addPartialSubscription(clientId, group, topic, liteTopicSet, null);\n\n        LiteSubscription subscription = registry.getLiteSubscription(clientId);\n        assertNotNull(subscription);\n        assertEquals(group, subscription.getGroup());\n        assertEquals(topic, subscription.getTopic());\n        assertTrue(subscription.getLiteTopicSet().isEmpty());\n\n        // Verify listener was not called\n        verify(mockListener, never()).onRegister(anyString(), anyString(), anyString());\n    }\n\n    @Test\n    public void testAddPartialSubscription_InactiveSubscription() {\n        String clientId = \"client1\";\n        String group = \"group1\";\n        String topic = \"topic1\";\n        String inactiveLiteTopic = \"inactive_lmq1\";\n\n        Set<String> liteTopicSet = new HashSet<>();\n        liteTopicSet.add(inactiveLiteTopic);\n\n        // Mock inactive subscription\n        when(mockLifecycleManager.isSubscriptionActive(topic, inactiveLiteTopic)).thenReturn(false);\n\n        // Should not add inactive subscriptions\n        registry.addPartialSubscription(clientId, group, topic, liteTopicSet, null);\n\n        LiteSubscription subscription = registry.getLiteSubscription(clientId);\n        assertNotNull(subscription);\n        assertFalse(subscription.getLiteTopicSet().contains(inactiveLiteTopic));\n        assertEquals(0, registry.getActiveSubscriptionNum());\n    }\n\n    @Test\n    public void testAddPartialSubscription_ExclusiveModeDifferentGroups() {\n        // Add two clients from different groups\n        String client1 = \"client1\";\n        String group1 = \"group1\";\n        String client2 = \"client2\";\n        String group2 = \"group2\";\n        String topic = \"topic1\";\n        String liteTopic = \"lmq1\";\n\n        Set<String> liteTopicSet = new HashSet<>();\n        liteTopicSet.add(liteTopic);\n\n        when(mockLifecycleManager.isSubscriptionActive(anyString(), anyString())).thenReturn(true);\n\n        // Mock subscription group configs\n        SubscriptionGroupConfig subscriptionGroupConfig1 = new SubscriptionGroupConfig();\n        subscriptionGroupConfig1.setGroupName(group1);\n        subscriptionGroupConfig1.getAttributes().put(LITE_SUB_MODEL_ATTRIBUTE.getName(), LiteSubModel.Exclusive.name());\n        when(mockSubscriptionGroupManager.findSubscriptionGroupConfig(group1)).thenReturn(subscriptionGroupConfig1);\n\n        SubscriptionGroupConfig subscriptionGroupConfig2 = new SubscriptionGroupConfig();\n        subscriptionGroupConfig2.setGroupName(group2);\n        subscriptionGroupConfig2.getAttributes().put(LITE_SUB_MODEL_ATTRIBUTE.getName(), LiteSubModel.Exclusive.name());\n        when(mockSubscriptionGroupManager.findSubscriptionGroupConfig(group2)).thenReturn(subscriptionGroupConfig2);\n\n        // Add first client\n        registry.addPartialSubscription(client1, group1, topic, liteTopicSet, null);\n\n        // Add second client\n        registry.addPartialSubscription(client2, group2, topic, liteTopicSet, null);\n\n        // Verify both clients are registered for the same topic\n        Set<ClientGroup> observers = registry.getSubscriber(liteTopic);\n        assertEquals(2, observers.size());\n\n        // Add new client in exclusive mode from the same group as client1\n        String client3 = \"client3\";\n        registry.addPartialSubscription(client3, group1, topic, liteTopicSet, null);\n\n        // Verify only client1 was removed (same group), client2 remains (different group)\n        observers = registry.getSubscriber(liteTopic);\n        assertEquals(2, observers.size()); // client2(group2) and client3(group1)\n\n        boolean hasClient2 = false;\n        boolean hasClient3 = false;\n        for (ClientGroup cg : observers) {\n            if (cg.clientId.equals(client2) && cg.group.equals(group2)) {\n                hasClient2 = true;\n            }\n            if (cg.clientId.equals(client3) && cg.group.equals(group1)) {\n                hasClient3 = true;\n            }\n        }\n\n        assertTrue(hasClient2, \"Client2 (group2) should still be registered\");\n        assertTrue(hasClient3, \"Client3 (group1) should be registered\");\n\n        // Verify listener calls\n        verify(mockListener).onUnregister(client1, group1, liteTopic); // Same group client1 removed\n        verify(mockListener, never()).onUnregister(client2, group2, liteTopic); // Different group client2 retained\n    }\n\n    @Test\n    public void testAddPartialSubscription_QuotaLimit() {\n        // Set quota to 1\n        when(mockBrokerConfig.getMaxLiteSubscriptionCount()).thenReturn(1L);\n\n        when(mockLifecycleManager.isSubscriptionActive(anyString(), anyString())).thenReturn(true);\n\n        // Add first subscription\n        String clientId1 = \"client1\";\n        String group1 = \"group1\";\n        String topic1 = \"topic1\";\n        Set<String> liteTopicSet1 = new HashSet<>();\n        liteTopicSet1.add(\"lmq1\");\n\n        registry.addPartialSubscription(clientId1, group1, topic1, liteTopicSet1, null);\n\n        // Try to add second subscription, should throw exception\n        String clientId2 = \"client2\";\n        String group2 = \"group2\";\n        String topic2 = \"topic2\";\n        Set<String> liteTopicSet2 = new HashSet<>();\n        liteTopicSet2.add(\"lmq2\");\n\n        assertThrows(LiteQuotaException.class, () -> {\n            registry.addPartialSubscription(clientId2, group2, topic2, liteTopicSet2, null);\n        });\n    }\n\n    // Test removeIncremental method\n    @Test\n    public void testRemovePartialSubscription() {\n        String clientId = \"client1\";\n        String group = \"group1\";\n        String topic = \"topic1\";\n        String liteTopic1 = \"lmq1\";\n        String liteTopic2 = \"lmq2\";\n\n        Set<String> liteTopicSet = new HashSet<>();\n        liteTopicSet.add(liteTopic1);\n        liteTopicSet.add(liteTopic2);\n\n        when(mockLifecycleManager.isSubscriptionActive(anyString(), anyString())).thenReturn(true);\n\n        // Add subscriptions first\n        registry.addPartialSubscription(clientId, group, topic, liteTopicSet, null);\n\n        // Verify subscriptions were added\n        LiteSubscription subscription = registry.getLiteSubscription(clientId);\n        assertTrue(subscription.getLiteTopicSet().contains(liteTopic1));\n        assertTrue(subscription.getLiteTopicSet().contains(liteTopic2));\n\n        // Remove some subscriptions\n        Set<String> toRemove = new HashSet<>();\n        toRemove.add(liteTopic1);\n        registry.removePartialSubscription(clientId, group, topic, toRemove);\n\n        // Verify removal was successful\n        subscription = registry.getLiteSubscription(clientId);\n        assertFalse(subscription.getLiteTopicSet().contains(liteTopic1));\n        assertTrue(subscription.getLiteTopicSet().contains(liteTopic2));\n\n        verify(mockListener).onUnregister(clientId, group, liteTopic1);\n        verify(mockListener, never()).onUnregister(clientId, group, liteTopic2);\n    }\n\n    // Test addAll method\n    @Test\n    public void testAddCompleteSubscription() {\n        String clientId = \"client1\";\n        String group = \"group1\";\n        String topic = \"topic1\";\n        String liteTopic1 = \"lmq1\";\n        String liteTopic2 = \"lmq2\";\n        String liteTopic3 = \"lmq3\";\n\n        // Initial subscriptions\n        Set<String> initialSet = new HashSet<>();\n        initialSet.add(liteTopic1);\n        initialSet.add(liteTopic2);\n\n        // New full subscription set\n        Set<String> newFullSet = new HashSet<>();\n        newFullSet.add(liteTopic2);\n        newFullSet.add(liteTopic3);\n\n        when(mockLifecycleManager.isSubscriptionActive(anyString(), anyString())).thenReturn(true);\n\n        // Add initial subscriptions\n        registry.addPartialSubscription(clientId, group, topic, initialSet, null);\n\n        // Reset mock to ignore previous interactions\n        clearInvocations(mockListener);\n\n        // Update with addAll\n        registry.addCompleteSubscription(clientId, group, topic, newFullSet, 1L);\n\n        // Verify update results\n        LiteSubscription subscription = registry.getLiteSubscription(clientId);\n        assertFalse(subscription.getLiteTopicSet().contains(liteTopic1)); // Should be removed\n        assertTrue(subscription.getLiteTopicSet().contains(liteTopic2));  // Should be retained\n        assertTrue(subscription.getLiteTopicSet().contains(liteTopic3));  // Should be added\n\n        // Verify that liteTopic1 was unregistered (no longer in new set)\n        verify(mockListener).onUnregister(clientId, group, liteTopic1);\n\n        // Verify that liteTopic3 was registered (new in the set)\n        verify(mockListener).onRegister(clientId, group, liteTopic3);\n\n        // Verify that liteTopic2 was neither unregistered nor registered again\n        // (it was already registered and remains in the new set)\n        verify(mockListener, never()).onUnregister(clientId, group, liteTopic2);\n    }\n\n    // Test removeAll method\n    @Test\n    public void testRemoveCompleteSubscription() {\n        String clientId = \"client1\";\n        String group = \"group1\";\n        String topic = \"topic1\";\n        String liteTopic1 = \"lmq1\";\n        String liteTopic2 = \"lmq2\";\n\n        Set<String> liteTopicSet = new HashSet<>();\n        liteTopicSet.add(liteTopic1);\n        liteTopicSet.add(liteTopic2);\n\n        when(mockLifecycleManager.isSubscriptionActive(anyString(), anyString())).thenReturn(true);\n\n        // Add subscriptions\n        registry.addPartialSubscription(clientId, group, topic, liteTopicSet, null);\n\n        // Verify subscriptions were added\n        assertNotNull(registry.getLiteSubscription(clientId));\n        assertEquals(2, registry.getActiveSubscriptionNum());\n\n        // Remove all subscriptions\n        registry.removeCompleteSubscription(clientId);\n\n        // Verify all subscriptions were removed\n        assertNull(registry.getLiteSubscription(clientId));\n        assertEquals(0, registry.getActiveSubscriptionNum());\n\n        verify(mockListener).onRemoveAll(clientId, group);\n    }\n\n    @Test\n    public void testRemoveCompleteSubscription_NonExistentClient() {\n        String nonExistentClientId = \"nonexistent\";\n\n        // Should not throw exception\n        registry.removeCompleteSubscription(nonExistentClientId);\n\n        // Verify no changes to registry state\n        assertEquals(0, registry.getActiveSubscriptionNum());\n        assertNull(registry.getLiteSubscription(nonExistentClientId));\n    }\n\n    // Test cleanSubscription method\n    @Test\n    public void testCleanSubscription() {\n        String clientId = \"client1\";\n        String group = \"group1\";\n        String topic = \"topic1\";\n        String liteTopic1 = \"lmq1\";\n        String liteTopic2 = \"lmq2\";\n\n        Set<String> liteTopicSet = new HashSet<>();\n        liteTopicSet.add(liteTopic1);\n        liteTopicSet.add(liteTopic2);\n\n        when(mockLifecycleManager.isSubscriptionActive(anyString(), anyString())).thenReturn(true);\n\n        // Add subscription\n        registry.addPartialSubscription(clientId, group, topic, liteTopicSet, null);\n        assertEquals(2, registry.getActiveSubscriptionNum());\n\n        // Verify subscription was added\n        LiteSubscription subscription = registry.getLiteSubscription(clientId);\n        assertTrue(subscription.getLiteTopicSet().contains(liteTopic1));\n        assertTrue(subscription.getLiteTopicSet().contains(liteTopic2));\n\n        // Clean subscription\n        registry.cleanSubscription(liteTopic1, true);\n        registry.cleanSubscription(liteTopic2, false);\n\n        // Verify subscription was cleaned\n        subscription = registry.getLiteSubscription(clientId);\n        assertFalse(subscription.getLiteTopicSet().contains(liteTopic1));\n        assertFalse(subscription.getLiteTopicSet().contains(liteTopic2));\n        assertNull(registry.getSubscriber(liteTopic1));\n        assertNull(registry.getSubscriber(liteTopic2));\n        assertEquals(0, registry.getActiveSubscriptionNum());\n    }\n\n    // Test getSubscriber method\n    @Test\n    public void testGetSubscriber() {\n        String clientId = \"client1\";\n        String group = \"group1\";\n        String topic = \"topic1\";\n        String liteTopic = \"lmq1\";\n\n        Set<String> liteTopicSet = new HashSet<>();\n        liteTopicSet.add(liteTopic);\n\n        when(mockLifecycleManager.isSubscriptionActive(anyString(), anyString())).thenReturn(true);\n\n        registry.addPartialSubscription(clientId, group, topic, liteTopicSet, null);\n\n        Set<ClientGroup> observers = registry.getSubscriber(liteTopic);\n        assertNotNull(observers);\n        assertEquals(1, observers.size());\n        ClientGroup clientGroup = observers.iterator().next();\n        assertEquals(clientId, clientGroup.clientId);\n        assertEquals(group, clientGroup.group);\n    }\n\n    @Test\n    public void testGetSubscriber_NonExistentTopic() {\n        String nonExistentTopic = \"nonexistent_lmq\";\n\n        Set<ClientGroup> result = registry.getSubscriber(nonExistentTopic);\n\n        // Should return null for non-existent topic\n        assertNull(result);\n    }\n\n    // Test updateClientChannel method\n    @Test\n    public void testUpdateClientChannel() {\n        String clientId = \"client1\";\n        Channel mockChannel = mock(Channel.class);\n\n        registry.updateClientChannel(clientId, mockChannel);\n\n        // Verify channel was updated\n        assertEquals(mockChannel, registry.clientChannels.get(clientId));\n    }\n\n    // Test getActiveSubscriptionNum method\n    @Test\n    public void testGetActiveSubscriptionNum() {\n        String clientId1 = \"client1\";\n        String clientId2 = \"client2\";\n        String group = \"group1\";\n        String topic = \"topic1\";\n        String liteTopic1 = \"lmq1\";\n        String liteTopic2 = \"lmq2\";\n\n        Set<String> liteTopicSet1 = new HashSet<>();\n        liteTopicSet1.add(liteTopic1);\n\n        Set<String> liteTopicSet2 = new HashSet<>();\n        liteTopicSet2.add(liteTopic1); // Same topic\n        liteTopicSet2.add(liteTopic2); // New topic\n\n        when(mockLifecycleManager.isSubscriptionActive(anyString(), anyString())).thenReturn(true);\n\n        // Initial state\n        assertEquals(0, registry.getActiveSubscriptionNum());\n\n        // Add first client\n        registry.addPartialSubscription(clientId1, group, topic, liteTopicSet1, null);\n        assertEquals(1, registry.getActiveSubscriptionNum());\n\n        // Add second client\n        registry.addPartialSubscription(clientId2, group, topic, liteTopicSet2, null);\n        assertEquals(3, registry.getActiveSubscriptionNum()); // 3 references: client1->topic1, client2->topic1, client2->topic2\n    }\n\n    // Test cleanupExpiredSubscriptions method\n    @Test\n    public void testCleanupExpiredSubscriptions_NoExpiredClients() {\n        String clientId = \"client1\";\n        String group = \"group1\";\n        String topic = \"topic1\";\n        Set<String> liteTopics = new HashSet<>();\n        liteTopics.add(\"lmq1\");\n        liteTopics.add(\"lmq2\");\n\n        LiteSubscription subscription = new LiteSubscription();\n        subscription.setGroup(group);\n        subscription.setTopic(topic);\n        subscription.addLiteTopic(liteTopics);\n        subscription.setUpdateTime(System.currentTimeMillis()); // Not expired\n\n        Channel channel = mock(Channel.class);\n\n        registry.client2Subscription.put(clientId, subscription);\n        registry.clientChannels.put(clientId, channel);\n\n        // Initialize liteTopic2Group\n        for (String lmq : liteTopics) {\n            registry.liteTopic2Group.computeIfAbsent(lmq, k -> ConcurrentHashMap.newKeySet())\n                .add(new ClientGroup(clientId, group));\n        }\n\n        registry.activeNum.set(liteTopics.size());\n\n        // Perform cleanup with a timeout of 10 seconds\n        registry.cleanupExpiredSubscriptions(10000);\n\n        // Verify that the client has not been cleaned up\n        assertNotNull(registry.client2Subscription.get(clientId));\n        assertNotNull(registry.clientChannels.get(clientId));\n        assertEquals(liteTopics.size(), registry.activeNum.get());\n    }\n\n    @Test\n    public void testCleanupExpiredSubscriptions_WithExpiredClients() {\n        String clientId = \"client1\";\n        String group = \"group1\";\n        String topic = \"topic1\";\n        Set<String> liteTopics = new HashSet<>();\n        liteTopics.add(\"lmq1\");\n        liteTopics.add(\"lmq2\");\n\n        LiteSubscription subscription = new LiteSubscription();\n        subscription.setGroup(group);\n        subscription.setTopic(topic);\n        subscription.addLiteTopic(liteTopics);\n        subscription.setUpdateTime(System.currentTimeMillis() - 20000);\n\n        Channel channel = mock(Channel.class);\n\n        registry.client2Subscription.put(clientId, subscription);\n        registry.clientChannels.put(clientId, channel);\n\n        // Initialize liteTopic2Group\n        for (String lmq : liteTopics) {\n            registry.liteTopic2Group.computeIfAbsent(lmq, k -> ConcurrentHashMap.newKeySet())\n                .add(new ClientGroup(clientId, group));\n        }\n\n        registry.activeNum.set(liteTopics.size());\n\n        LiteCtlListener mockListener = mock(LiteCtlListener.class);\n        registry.addListener(mockListener);\n\n        // Perform cleanup with a timeout of 10 seconds\n        registry.cleanupExpiredSubscriptions(10000);\n\n        // Verify that the client has been cleaned up\n        assertNull(registry.client2Subscription.get(clientId));\n        assertNull(registry.clientChannels.get(clientId));\n        assertEquals(0, registry.activeNum.get());\n\n        // Verify that the listener was called\n        verify(mockListener, times(1)).onUnregister(eq(clientId), eq(group), eq(\"lmq1\"));\n        verify(mockListener, times(1)).onUnregister(eq(clientId), eq(group), eq(\"lmq2\"));\n        verify(mockListener, times(1)).onRemoveAll(eq(clientId), eq(group));\n\n        // Verify that topics in liteTopic2Group have been removed\n        assertNull(registry.liteTopic2Group.get(\"lmq1\"));\n        assertNull(registry.liteTopic2Group.get(\"lmq2\"));\n    }\n\n    @Test\n    public void testCleanupExpiredSubscriptions_ExpiredClientWithNoSubscriptions() {\n        String clientId = \"client1\";\n        String group = \"group1\";\n        String topic = \"topic1\";\n        Set<String> liteTopics = new HashSet<>();\n\n        LiteSubscription subscription = new LiteSubscription();\n        subscription.setGroup(group);\n        subscription.setTopic(topic);\n        subscription.addLiteTopic(liteTopics);\n        subscription.setUpdateTime(System.currentTimeMillis() - 20000); // Expired\n\n        Channel channel = mock(Channel.class);\n\n        registry.client2Subscription.put(clientId, subscription);\n        registry.clientChannels.put(clientId, channel);\n\n        registry.activeNum.set(0);\n\n        LiteCtlListener mockListener = mock(LiteCtlListener.class);\n        registry.addListener(mockListener);\n\n        // Perform cleanup with 10 second timeout\n        registry.cleanupExpiredSubscriptions(10000);\n\n        // Verify that the client has been cleaned up\n        assertNull(registry.client2Subscription.get(clientId));\n        assertNull(registry.clientChannels.get(clientId));\n        assertEquals(0, registry.activeNum.get());\n\n        // Verify that the listener was not called\n        verify(mockListener, never()).onUnregister(anyString(), anyString(), anyString());\n    }\n\n    // Test removeTopicGroup method\n    @Test\n    public void testRemoveTopicGroup_EmptyTopicGroupSet() {\n        String clientId = \"client1\";\n        String group = \"group1\";\n        String liteTopic = \"lmq1\";\n\n        ClientGroup clientGroup = new ClientGroup(clientId, group);\n\n        // Initialize with a single client\n        Set<ClientGroup> topicGroupSet = ConcurrentHashMap.newKeySet();\n        topicGroupSet.add(clientGroup);\n        registry.liteTopic2Group.put(liteTopic, topicGroupSet);\n        registry.activeNum.set(1);\n\n        // Remove the only client\n        registry.removeTopicGroup(clientGroup, liteTopic, false);\n\n        // Verify that the topic is completely removed from liteTopic2Group\n        assertNull(registry.liteTopic2Group.get(liteTopic));\n        assertEquals(0, registry.getActiveSubscriptionNum());\n    }\n\n    // Test excludeClientByLmqName method\n    @Test\n    public void testExcludeClientByLmqName_EmptyClientSet() {\n        String newClientId = \"newClient\";\n        String group = \"group1\";\n        String lmqName = \"lmq1\";\n\n        // Ensure the liteTopic2Group map exists but is empty\n        registry.liteTopic2Group.put(lmqName, ConcurrentHashMap.newKeySet());\n\n        // Should not throw any exception\n        registry.excludeClientByLmqName(newClientId, group, lmqName);\n\n        // Verify no changes\n        assertTrue(registry.liteTopic2Group.get(lmqName).isEmpty());\n    }\n\n    @Test\n    public void testGetAllClientIdByGroup() {\n        String group1 = \"group1\";\n        String group2 = \"group2\";\n        String clientId1 = \"client1\";\n        String clientId2 = \"client2\";\n        String clientId3 = \"client3\";\n        String topic = \"parentTopic\";\n\n        LiteSubscription sub1 = new LiteSubscription();\n        sub1.setGroup(group1);\n        sub1.setTopic(topic);\n\n        LiteSubscription sub2 = new LiteSubscription();\n        sub2.setGroup(group1);\n        sub2.setTopic(topic);\n\n        LiteSubscription sub3 = new LiteSubscription();\n        sub3.setGroup(group2);\n        sub3.setTopic(topic);\n\n        registry.client2Subscription.put(clientId1, sub1);\n        registry.client2Subscription.put(clientId2, sub2);\n        registry.client2Subscription.put(clientId3, sub3);\n\n        List<String> result;\n\n        // group1\n        result = registry.getAllClientIdByGroup(group1);\n        assertEquals(2, result.size());\n        assertTrue(result.contains(clientId1));\n        assertTrue(result.contains(clientId2));\n\n        // group2\n        result = registry.getAllClientIdByGroup(group2);\n        assertEquals(1, result.size());\n        assertTrue(result.contains(clientId3));\n\n        // not exist\n        result = registry.getAllClientIdByGroup(\"notExistGroup\");\n        assertTrue(result.isEmpty());\n\n        // null\n        result = registry.getAllClientIdByGroup(null);\n        assertTrue(result.isEmpty());\n    }\n\n    @Test\n    public void testResetOffset_minOffset() {\n        String lmqName = \"lmq1\";\n        String group = \"group1\";\n        String clientId = \"client1\";\n\n        when(mockConsumerOffsetManager.queryOffset(group, lmqName, 0)).thenReturn(100L);\n\n        OffsetOption offsetOption = new OffsetOption(OffsetOption.Type.POLICY, OffsetOption.POLICY_MIN_VALUE);\n        registry.resetOffset(lmqName, group, clientId, offsetOption);\n\n        verify(mockConsumerOffsetManager).assignResetOffset(lmqName, group, 0, 0L);\n    }\n\n    @Test\n    public void testResetOffset_maxOffset() {\n        String lmqName = \"lmq1\";\n        String group = \"group1\";\n        String clientId = \"client1\";\n        long maxOffset = 500L;\n\n        when(mockConsumerOffsetManager.queryOffset(group, lmqName, 0)).thenReturn(100L);\n        when(mockLifecycleManager.getMaxOffsetInQueue(lmqName)).thenReturn(maxOffset);\n\n        OffsetOption offsetOption = new OffsetOption(OffsetOption.Type.POLICY, OffsetOption.POLICY_MAX_VALUE);\n        registry.resetOffset(lmqName, group, clientId, offsetOption);\n\n        verify(mockConsumerOffsetManager).assignResetOffset(lmqName, group, 0, maxOffset);\n    }\n\n    @Test\n    public void testResetOffset_absolute() {\n        String lmqName = \"lmq1\";\n        String group = \"group1\";\n        String clientId = \"client1\";\n        long specifiedOffset = 250L;\n\n        when(mockConsumerOffsetManager.queryOffset(group, lmqName, 0)).thenReturn(100L);\n\n        OffsetOption offsetOption = new OffsetOption(OffsetOption.Type.OFFSET, specifiedOffset);\n        registry.resetOffset(lmqName, group, clientId, offsetOption);\n\n        verify(mockConsumerOffsetManager).assignResetOffset(lmqName, group, 0, specifiedOffset);\n    }\n\n    @Test\n    public void testResetOffset_LastN() {\n        String lmqName = \"lmq1\";\n        String group1 = \"group1\";\n        String group2 = \"group2\";\n        String clientId = \"client1\";\n        long currentOffset = 100L;\n        long lastN = 20L;\n        long expectedTargetOffset = 80L;\n\n        when(mockConsumerOffsetManager.queryOffset(group1, lmqName, 0)).thenReturn(currentOffset);\n        when(mockConsumerOffsetManager.queryOffset(group2, lmqName, 0)).thenReturn(-1L);\n\n        OffsetOption offsetOption = new OffsetOption(OffsetOption.Type.TAIL_N, lastN);\n\n        registry.resetOffset(lmqName, group1, clientId, offsetOption);\n        registry.resetOffset(lmqName, group2, clientId, offsetOption);\n\n        verify(mockConsumerOffsetManager).assignResetOffset(lmqName, group1, 0, expectedTargetOffset);\n        verify(mockConsumerOffsetManager, never()).assignResetOffset(lmqName, group2, 0, expectedTargetOffset);\n    }\n\n    @Test\n    public void testResetOffset_timestamp_not_supported() {\n        String lmqName = \"lmq1\";\n        String group = \"group1\";\n        String clientId = \"client1\";\n        long timestamp = System.currentTimeMillis();\n\n        when(mockConsumerOffsetManager.queryOffset(group, lmqName, 0)).thenReturn(100L);\n\n        OffsetOption offsetOption = new OffsetOption(OffsetOption.Type.TIMESTAMP, timestamp);\n        registry.resetOffset(lmqName, group, clientId, offsetOption);\n\n        verify(mockConsumerOffsetManager, never()).assignResetOffset(anyString(), anyString(), anyInt(), anyLong());\n    }\n}"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/lite/LiteTestUtil.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.lite;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.RocksDBMessageStore;\nimport org.apache.rocketmq.store.config.FlushDiskType;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\n\nimport java.net.InetSocketAddress;\nimport java.util.concurrent.ConcurrentMap;\n\npublic class LiteTestUtil {\n\n    public static MessageStore buildMessageStore(String storePathRootDir, final BrokerConfig brokerConfig,\n        final ConcurrentMap<String, TopicConfig> topicConfigTable, boolean isRocksDBStore) throws Exception {\n        MessageStoreConfig storeConfig = new MessageStoreConfig();\n        storeConfig.setMappedFileSizeCommitLog(1024 * 1024 * 10);\n        storeConfig.setMappedFileSizeConsumeQueue(1024 * 1024 * 10);\n        storeConfig.setMaxHashSlotNum(10000);\n        storeConfig.setMaxIndexNum(100 * 100);\n        storeConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);\n        storeConfig.setFlushIntervalConsumeQueue(1);\n        storeConfig.setHaListenPort(0);\n        storeConfig.setEnableLmq(true);\n        storeConfig.setEnableMultiDispatch(true);\n        storeConfig.setStorePathRootDir(storePathRootDir);\n\n        BrokerStatsManager brokerStatsManager = new BrokerStatsManager(brokerConfig);\n        MessageStore messageStore;\n        if (isRocksDBStore) {\n            messageStore = new RocksDBMessageStore(storeConfig, brokerStatsManager, null, brokerConfig, topicConfigTable);\n        } else {\n            messageStore = new DefaultMessageStore(storeConfig, brokerStatsManager, null, brokerConfig, topicConfigTable);\n        }\n        return messageStore;\n    }\n\n    public static MessageExtBrokerInner buildMessage(String parentTopic, String liteTopic) {\n        MessageExtBrokerInner msg = new MessageExtBrokerInner();\n        msg.setTopic(parentTopic);\n        msg.setTags(\"TAG1\");\n        msg.setKeys(\"Hello\");\n        msg.setBody(\"HW\".getBytes());\n        msg.setQueueId(0);\n        msg.setSysFlag(0);\n        msg.setBornTimestamp(System.currentTimeMillis());\n        msg.setStoreHost(new InetSocketAddress(\"localhost\", 10911));\n        msg.setBornHost(new InetSocketAddress(\"localhost\", 0));\n\n        if (StringUtils.isNotEmpty(liteTopic)) {\n            String lmqName = LiteUtil.toLmqName(parentTopic, liteTopic);\n            MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_MULTI_DISPATCH, lmqName);\n        }\n        msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n        return msg;\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/lite/RocksDBLiteLifecycleManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.lite;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.TopicAttributes;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.plugin.AbstractPluginMessageStore;\nimport org.apache.rocketmq.store.plugin.MessageStorePluginContext;\nimport org.apache.rocketmq.tieredstore.TieredMessageStore;\nimport org.junit.AfterClass;\nimport org.junit.Assert;\nimport org.junit.BeforeClass;\nimport org.junit.Ignore;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.io.File;\nimport java.util.List;\nimport java.util.UUID;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.stream.Collectors;\nimport java.util.stream.IntStream;\n\nimport static java.util.concurrent.TimeUnit.MILLISECONDS;\nimport static java.util.concurrent.TimeUnit.SECONDS;\nimport static org.awaitility.Awaitility.await;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class RocksDBLiteLifecycleManagerTest {\n\n    private final static BrokerConfig BROKER_CONFIG = new BrokerConfig();\n    private final static ConcurrentMap<String, TopicConfig> TOPIC_CONFIG_TABLE = new ConcurrentHashMap<>();\n    private static String storePathRootDir;\n    private static MessageStore messageStore;\n    private static RocksDBLiteLifecycleManager liteLifecycleManager;\n    private static TopicConfig mockTopicConfig = new TopicConfig();\n\n    @BeforeClass\n    public static void setUp() throws Exception {\n        storePathRootDir = System.getProperty(\"java.io.tmpdir\") + File.separator + \"store-rocksDBLifecycleTest\";\n        UtilAll.deleteFile(new File(storePathRootDir));\n\n        messageStore = LiteTestUtil.buildMessageStore(storePathRootDir, BROKER_CONFIG, TOPIC_CONFIG_TABLE, true);\n        messageStore.load();\n        messageStore.start();\n\n        BrokerController brokerController = Mockito.mock(BrokerController.class);\n        LiteSharding liteSharding = Mockito.mock(LiteSharding.class);\n        TopicConfigManager topicConfigManager = Mockito.mock(TopicConfigManager.class);\n        SubscriptionGroupManager subscriptionGroupManager = Mockito.mock(SubscriptionGroupManager.class);\n\n        when(brokerController.getBrokerConfig()).thenReturn(BROKER_CONFIG);\n        when(brokerController.getMessageStore()).thenReturn(messageStore);\n        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);\n        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);\n        when(topicConfigManager.getTopicConfigTable()).thenReturn(TOPIC_CONFIG_TABLE);\n        when(topicConfigManager.selectTopicConfig(anyString())).thenReturn(mockTopicConfig);\n        when(subscriptionGroupManager.getSubscriptionGroupTable()).thenReturn(new ConcurrentHashMap<>());\n\n        RocksDBLiteLifecycleManager testObject = new RocksDBLiteLifecycleManager(brokerController, liteSharding);\n        liteLifecycleManager = Mockito.spy(testObject);\n        liteLifecycleManager.init();\n    }\n\n    @AfterClass\n    public static void reset() {\n        messageStore.shutdown();\n        messageStore.destroy();\n        UtilAll.deleteFile(new File(storePathRootDir));\n        mockTopicConfig = new TopicConfig();\n    }\n\n    @Ignore\n    @Test\n    public void testInit_tieredStore() {\n        BrokerController brokerController = Mockito.mock(BrokerController.class);\n        LiteSharding liteSharding = Mockito.mock(LiteSharding.class);\n        MessageStorePluginContext context = Mockito.mock(MessageStorePluginContext.class);\n\n        TieredMessageStore tieredMessageStore = new TieredMessageStore(context, messageStore);\n        when(brokerController.getBrokerConfig()).thenReturn(BROKER_CONFIG);\n        when(brokerController.getMessageStore()).thenReturn(tieredMessageStore);\n\n        RocksDBLiteLifecycleManager manager = new RocksDBLiteLifecycleManager(brokerController, liteSharding);\n        manager.init();\n        Assert.assertEquals(0, manager.getMaxOffsetInQueue(UUID.randomUUID().toString()));\n    }\n\n    @Test\n    public void testInit_otherStore() {\n        BrokerController brokerController = Mockito.mock(BrokerController.class);\n        LiteSharding liteSharding = Mockito.mock(LiteSharding.class);\n        AbstractPluginMessageStore pluginMessageStore = Mockito.mock(AbstractPluginMessageStore.class);\n\n        when(brokerController.getBrokerConfig()).thenReturn(BROKER_CONFIG);\n        when(brokerController.getMessageStore()).thenReturn(pluginMessageStore);\n\n        RocksDBLiteLifecycleManager manager = new RocksDBLiteLifecycleManager(brokerController, liteSharding);\n        manager.init();\n        Assert.assertThrows(NullPointerException.class, () -> manager.getMaxOffsetInQueue(\"HW\"));\n    }\n\n    @Test\n    public void testGetMaxOffsetInQueue() {\n        int num = 3;\n        String topic = UUID.randomUUID().toString();\n        for (int i = 0; i < num; i++) {\n            messageStore.putMessage(LiteTestUtil.buildMessage(topic, null));\n        }\n        await().atMost(5, SECONDS).pollInterval(200, MILLISECONDS).until(() -> messageStore.dispatchBehindBytes() <= 0);\n        Assert.assertEquals(num, liteLifecycleManager.getMaxOffsetInQueue(topic));\n        Assert.assertEquals(0, liteLifecycleManager.getMaxOffsetInQueue(UUID.randomUUID().toString()));\n    }\n\n    @Test\n    public void testCollectByParentTopic() {\n        int num = 3;\n        String parentTopic = UUID.randomUUID().toString();\n        for (int i = 0; i < num; i++) {\n            messageStore.putMessage(LiteTestUtil.buildMessage(parentTopic, UUID.randomUUID().toString()));\n            messageStore.putMessage(LiteTestUtil.buildMessage(UUID.randomUUID().toString(), UUID.randomUUID().toString()));\n        }\n        await().atMost(5, SECONDS).pollInterval(200, MILLISECONDS).until(() -> messageStore.dispatchBehindBytes() <= 0);\n        List<String> result = liteLifecycleManager.collectByParentTopic(parentTopic);\n        Assert.assertEquals(num, result.size());\n        for (String lmqName : result) {\n            Assert.assertTrue(LiteUtil.belongsTo(lmqName, parentTopic));\n        }\n\n        result = liteLifecycleManager.collectByParentTopic(UUID.randomUUID().toString());\n        Assert.assertEquals(0, result.size());\n    }\n\n    @Test\n    public void testCollectExpiredLiteTopic() {\n        int num = 3;\n        String parentTopic = UUID.randomUUID().toString();\n        for (int i = 0; i < num; i++) {\n            messageStore.putMessage(LiteTestUtil.buildMessage(parentTopic, UUID.randomUUID().toString()));\n            messageStore.putMessage(LiteTestUtil.buildMessage(UUID.randomUUID().toString(), null));\n        }\n        await().atMost(5, SECONDS).pollInterval(200, MILLISECONDS).until(() -> messageStore.dispatchBehindBytes() <= 0);\n\n        when(liteLifecycleManager.isLiteTopicExpired(anyString(), anyString(), anyLong())).thenReturn(false);\n        List<Pair<String, String>> result = liteLifecycleManager.collectExpiredLiteTopic();\n        Assert.assertEquals(0, result.size());\n\n        when(liteLifecycleManager.isLiteTopicExpired(eq(parentTopic), anyString(), anyLong())).thenReturn(true);\n        result = liteLifecycleManager.collectExpiredLiteTopic();\n        Assert.assertEquals(num, result.size());\n        for (Pair<String, String> pair : result) {\n            Assert.assertEquals(parentTopic, pair.getObject1());\n            Assert.assertTrue(LiteUtil.belongsTo(pair.getObject2(), parentTopic));\n        }\n    }\n\n    @Test\n    public void testCleanExpiredLiteTopic() throws Exception {\n        int num = 3;\n        String parentTopic = UUID.randomUUID().toString();\n        List<String> liteTopics =\n            IntStream.range(0, 3).mapToObj(i -> UUID.randomUUID().toString()).collect(Collectors.toList());\n        for (int i = 0; i < num; i++) {\n            messageStore.putMessage(LiteTestUtil.buildMessage(parentTopic, liteTopics.get(i)));\n        }\n        await().atMost(5, SECONDS).pollInterval(200, MILLISECONDS).until(() -> messageStore.dispatchBehindBytes() <= 0);\n\n        for (int i = 0; i < num; i++) {\n            String lmqName = LiteUtil.toLmqName(parentTopic, liteTopics.get(i));\n            Assert.assertEquals(1, (long) messageStore.getQueueStore().getMaxOffset(lmqName, 0));\n            Assert.assertEquals(1, liteLifecycleManager.getMaxOffsetInQueue(lmqName));\n        }\n\n        when(liteLifecycleManager.isLiteTopicExpired(eq(parentTopic), anyString(), anyLong())).thenReturn(true);\n        liteLifecycleManager.cleanExpiredLiteTopic();\n\n        for (int i = 0; i < num; i++) {\n            String lmqName = LiteUtil.toLmqName(parentTopic, liteTopics.get(i));\n            Assert.assertEquals(0, (long) messageStore.getQueueStore().getMaxOffset(lmqName, 0));\n            Assert.assertEquals(0, liteLifecycleManager.getMaxOffsetInQueue(lmqName));\n        }\n    }\n\n    @Test\n    public void testCleanByParentTopic() throws Exception {\n        int num = 3;\n        String parentTopic = UUID.randomUUID().toString();\n        mockTopicConfig.getAttributes().put(\n            TopicAttributes.TOPIC_MESSAGE_TYPE_ATTRIBUTE.getName(), TopicMessageType.LITE.getValue());\n        List<String> liteTopics =\n            IntStream.range(0, 3).mapToObj(i -> UUID.randomUUID().toString()).collect(Collectors.toList());\n        for (int i = 0; i < num; i++) {\n            messageStore.putMessage(LiteTestUtil.buildMessage(parentTopic, liteTopics.get(i)));\n        }\n        await().atMost(5, SECONDS).pollInterval(200, MILLISECONDS).until(() -> messageStore.dispatchBehindBytes() <= 0);\n\n        for (int i = 0; i < num; i++) {\n            String lmqName = LiteUtil.toLmqName(parentTopic, liteTopics.get(i));\n            Assert.assertEquals(1, (long) messageStore.getQueueStore().getMaxOffset(lmqName, 0));\n            Assert.assertEquals(1, liteLifecycleManager.getMaxOffsetInQueue(lmqName));\n        }\n\n        liteLifecycleManager.cleanByParentTopic(parentTopic);\n\n        for (int i = 0; i < num; i++) {\n            String lmqName = LiteUtil.toLmqName(parentTopic, liteTopics.get(i));\n            Assert.assertEquals(0, (long) messageStore.getQueueStore().getMaxOffset(lmqName, 0));\n            Assert.assertEquals(0, liteLifecycleManager.getMaxOffsetInQueue(lmqName));\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/longpolling/PopLiteLongPollingServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.longpolling;\n\nimport com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.util.concurrent.ConcurrentSkipListSet;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.atomic.AtomicLong;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.never;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class PopLiteLongPollingServiceTest {\n    \n    @Mock\n    private BrokerController brokerController;\n    @Mock\n    private NettyRequestProcessor processor;\n    @Mock\n    private ChannelHandlerContext ctx;\n    @Mock\n    private ExecutorService pullMessageExecutor;\n\n    private BrokerConfig brokerConfig;\n    private PopLiteLongPollingService popLiteLongPollingService;\n    private ConcurrentLinkedHashMap<String, ConcurrentSkipListSet<PopRequest>> pollingMap;\n    private AtomicLong totalPollingNum;\n\n    @SuppressWarnings(\"unchecked\")\n    @Before\n    public void init() throws IllegalAccessException {\n        brokerConfig = new BrokerConfig();\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        when(brokerController.getPullMessageExecutor()).thenReturn(pullMessageExecutor);\n        popLiteLongPollingService = new PopLiteLongPollingService(brokerController, processor, true);\n        pollingMap = (ConcurrentLinkedHashMap<String, ConcurrentSkipListSet<PopRequest>>)\n            FieldUtils.readDeclaredField(popLiteLongPollingService, \"pollingMap\", true);\n        totalPollingNum = (AtomicLong) FieldUtils.readDeclaredField(popLiteLongPollingService, \"totalPollingNum\", true);\n    }\n\n    @Test\n    public void testNotifyMessageArriving_noRequest() {\n        assertFalse(popLiteLongPollingService.notifyMessageArriving(\"clientId\", true, 0, \"group\"));\n    }\n\n    @Test\n    public void testNotifyMessageArriving_inactiveChannel() throws Exception {\n        String clientId = \"clientId\";\n        String group = \"group\";\n\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        RemotingCommand remotingCommand = mock(RemotingCommand.class);\n        Channel channel = mock(Channel.class);\n        when(channel.isActive()).thenReturn(false);\n        when(ctx.channel()).thenReturn(channel);\n\n        PollingResult result = popLiteLongPollingService.polling(\n            ctx, remotingCommand, System.currentTimeMillis(), 10000, clientId, group);\n        assertEquals(PollingResult.POLLING_SUC, result);\n        assertEquals(1, totalPollingNum.get());\n\n        assertFalse(popLiteLongPollingService.notifyMessageArriving(clientId, true, 0, group));\n        assertEquals(0, totalPollingNum.get());\n    }\n\n    @Test\n    public void testNotifyMessageArriving_success() throws Exception {\n        String clientId = \"clientId\";\n        String group = \"group\";\n\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        RemotingCommand remotingCommand1 = mock(RemotingCommand.class);\n        RemotingCommand remotingCommand2 = mock(RemotingCommand.class);\n        Channel channel = mock(Channel.class);\n        when(channel.isActive()).thenReturn(true);\n        when(ctx.channel()).thenReturn(channel);\n\n        PollingResult result1 = popLiteLongPollingService.polling(\n            ctx, remotingCommand1, System.currentTimeMillis(), 10000, clientId, group);\n        PollingResult result2 = popLiteLongPollingService.polling(\n            ctx, remotingCommand2, System.currentTimeMillis(), 15000, clientId, group);\n\n        assertEquals(PollingResult.POLLING_SUC, result1);\n        assertEquals(PollingResult.POLLING_SUC, result2);\n        assertEquals(2, totalPollingNum.get());\n\n        assertTrue(popLiteLongPollingService.notifyMessageArriving(clientId, true, 0, group));\n        assertEquals(1, totalPollingNum.get());\n        assertEquals(remotingCommand1, pollingMap.get(clientId).pollFirst().getRemotingCommand()); // notify last\n    }\n\n    @Test\n    public void testWakeUp_nullRequest() {\n        assertFalse(popLiteLongPollingService.wakeUp(null));\n    }\n\n    @Test\n    public void testWakeUp_completeRequest() {\n        PopRequest request = mock(PopRequest.class);\n        when(request.complete()).thenReturn(false);\n\n        assertFalse(popLiteLongPollingService.wakeUp(request));\n    }\n\n    @Test\n    public void testWakeUp_inactiveChannel() {\n        PopRequest request = mock(PopRequest.class);\n        when(request.complete()).thenReturn(true);\n        when(request.getCtx()).thenReturn(ctx);\n        Channel channel = mock(Channel.class);\n        when(ctx.channel()).thenReturn(channel);\n        when(channel.isActive()).thenReturn(false);\n\n        assertFalse(popLiteLongPollingService.wakeUp(request));\n        verify(pullMessageExecutor, never()).submit(any(Runnable.class));\n    }\n\n    @Test\n    public void testWakeUp_success() {\n        PopRequest request = mock(PopRequest.class);\n        when(request.complete()).thenReturn(true);\n        when(request.getCtx()).thenReturn(ctx);\n        Channel channel = mock(Channel.class);\n        when(ctx.channel()).thenReturn(channel);\n        when(channel.isActive()).thenReturn(true);\n\n        assertTrue(popLiteLongPollingService.wakeUp(request));\n        verify(pullMessageExecutor).submit(any(Runnable.class));\n    }\n\n    @Test\n    public void testPolling_notPolling() {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        RemotingCommand remotingCommand = mock(RemotingCommand.class);\n\n        PollingResult result = popLiteLongPollingService.polling(ctx, remotingCommand, 0, 0, \"clientId\", \"group\");\n        assertEquals(PollingResult.NOT_POLLING, result);\n    }\n\n    @Test\n    public void testPolling_timeout() {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        RemotingCommand remotingCommand = mock(RemotingCommand.class);\n\n        PollingResult result =\n            popLiteLongPollingService.polling(ctx, remotingCommand, System.currentTimeMillis(), 40, \"clientId\", \"group\");\n        assertEquals(PollingResult.POLLING_TIMEOUT, result);\n    }\n\n    @Test\n    public void testPolling_success() {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        RemotingCommand remotingCommand = mock(RemotingCommand.class);\n\n        PollingResult result = popLiteLongPollingService.polling(\n            ctx, remotingCommand, System.currentTimeMillis(), 10000, \"clientId\", \"group\");\n        assertEquals(PollingResult.POLLING_SUC, result);\n    }\n\n    @Test\n    public void testPolling_totalPollingFull() {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        RemotingCommand remotingCommand = mock(RemotingCommand.class);\n        totalPollingNum.set(brokerConfig.getMaxPopPollingSize() + 1);\n\n        PollingResult result = popLiteLongPollingService.polling(\n            ctx, remotingCommand, System.currentTimeMillis(), 10000, \"clientId\", \"group\");\n        assertEquals(PollingResult.POLLING_FULL, result);\n    }\n\n    @Test\n    public void testPolling_singlePollingFull() {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        RemotingCommand remotingCommand = mock(RemotingCommand.class);\n        brokerConfig.setPopPollingSize(-1);\n\n        PollingResult result = popLiteLongPollingService.polling(\n            ctx, remotingCommand, System.currentTimeMillis(), 10000, \"clientId\", \"group\");\n        assertEquals(PollingResult.POLLING_SUC, result);\n\n        result = popLiteLongPollingService.polling(\n            ctx, remotingCommand, System.currentTimeMillis(), 10000, \"clientId\", \"group\");\n        assertEquals(PollingResult.POLLING_FULL, result);\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/longpolling/PopLongPollingServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.longpolling;\n\nimport com.github.benmanes.caffeine.cache.Cache;\nimport com.github.benmanes.caffeine.cache.Caffeine;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentSkipListSet;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.store.MessageFilter;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.ArgumentCaptor;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.Mockito.doNothing;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class PopLongPollingServiceTest {\n    \n    @Mock\n    private BrokerController brokerController;\n\n    @Mock\n    private NettyRequestProcessor processor;\n\n    @Mock\n    private ChannelHandlerContext ctx;\n\n    @Mock\n    private ExecutorService pullMessageExecutor;\n\n    private PopLongPollingService popLongPollingService;\n\n    private final String defaultTopic = \"defaultTopic\";\n\n    @Before\n    public void init() {\n        BrokerConfig brokerConfig = new BrokerConfig();\n        brokerConfig.setPopPollingMapSize(100);\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        popLongPollingService = spy(new PopLongPollingService(brokerController, processor, true));\n    }\n\n    @Test\n    public void testNotifyMessageArrivingWithRetryTopic() {\n        int queueId = 0;\n        doNothing().when(popLongPollingService).notifyMessageArrivingWithRetryTopic(defaultTopic, queueId, -1L, null, 0L, null, null);\n        popLongPollingService.notifyMessageArrivingWithRetryTopic(defaultTopic, queueId);\n        verify(popLongPollingService, times(1)).notifyMessageArrivingWithRetryTopic(defaultTopic, queueId, -1L, null, 0L, null, null);\n    }\n\n    @Test\n    public void testNotifyMessageArrivingFromRetry() {\n        int queueId = -1;\n        String group = \"group\";\n        String pullRetryTopic = MixAll.getRetryTopic(group);\n        String popRetryTopicV1 = KeyBuilder.buildPopRetryTopic(defaultTopic, group, false);\n        String popRetryTopicV2 = KeyBuilder.buildPopRetryTopic(defaultTopic, group, true);\n\n        Map<String, String> properties = new HashMap<>();\n        properties.putIfAbsent(MessageConst.PROPERTY_ORIGIN_GROUP, group);\n        // pull retry\n        popLongPollingService.notifyMessageArrivingWithRetryTopic(pullRetryTopic, queueId, queueId, -1L, 0L, null, properties);\n        verify(popLongPollingService, times(0)).notifyMessageArriving(defaultTopic, queueId, group, true, -1L, 0L, null, properties, null);\n        // pop retry v1\n        popLongPollingService.notifyMessageArrivingWithRetryTopic(popRetryTopicV1, queueId, queueId, -1L, 0L, null, properties);\n        verify(popLongPollingService, times(1)).notifyMessageArriving(defaultTopic, queueId, group, true, -1L, 0L, null, properties, null);\n        // pop retry v2\n        popLongPollingService.notifyMessageArrivingWithRetryTopic(popRetryTopicV2, queueId, queueId, -1L, 0L, null, properties);\n        verify(popLongPollingService, times(2)).notifyMessageArriving(defaultTopic, queueId, group, true, -1L, 0L, null, properties, null);\n    }\n\n    @Test\n    public void testNotifyMessageArriving() {\n        int queueId = 0;\n        Long tagsCode = 123L;\n        long offset = 123L;\n        long msgStoreTime = System.currentTimeMillis();\n        byte[] filterBitMap = new byte[] {0x01};\n        Map<String, String> properties = new ConcurrentHashMap<>();\n        doNothing().when(popLongPollingService).notifyMessageArriving(defaultTopic, queueId, offset, tagsCode, msgStoreTime, filterBitMap, properties);\n        popLongPollingService.notifyMessageArrivingWithRetryTopic(defaultTopic, queueId, offset, tagsCode, msgStoreTime, filterBitMap, properties);\n        verify(popLongPollingService).notifyMessageArriving(defaultTopic, queueId, offset, tagsCode, msgStoreTime, filterBitMap, properties);\n    }\n\n    @Test\n    public void testNotifyMessageArrivingValidRequest() throws Exception {\n        String cid = \"CID_1\";\n        int queueId = 0;\n        Cache<String, ConcurrentHashMap<String, Byte>> topicCidMap = Caffeine.newBuilder()\n            .maximumSize(10)\n            .expireAfterAccess(300, TimeUnit.SECONDS)\n            .build();\n        ConcurrentHashMap<String, Byte> cids = new ConcurrentHashMap<>();\n        cids.put(cid, (byte) 1);\n        topicCidMap.put(defaultTopic, cids);\n        popLongPollingService = new PopLongPollingService(brokerController, processor, true);\n        Cache<String, ConcurrentSkipListSet<PopRequest>> pollingMap = Caffeine.newBuilder()\n            .maximumSize(10)\n            .expireAfterAccess(300, TimeUnit.SECONDS)\n            .build();\n        Channel channel = mock(Channel.class);\n        when(channel.isActive()).thenReturn(true);\n        PopRequest popRequest = mock(PopRequest.class);\n        MessageFilter messageFilter = mock(MessageFilter.class);\n        SubscriptionData subscriptionData = mock(SubscriptionData.class);\n        when(popRequest.getMessageFilter()).thenReturn(messageFilter);\n        when(popRequest.getSubscriptionData()).thenReturn(subscriptionData);\n        when(popRequest.getChannel()).thenReturn(channel);\n        String pollingKey = KeyBuilder.buildPollingKey(defaultTopic, cid, queueId);\n        ConcurrentSkipListSet popRequests = mock(ConcurrentSkipListSet.class);\n        when(popRequests.pollLast()).thenReturn(popRequest);\n        pollingMap.put(pollingKey, popRequests);\n        FieldUtils.writeDeclaredField(popLongPollingService, \"topicCidMap\", topicCidMap, true);\n        FieldUtils.writeDeclaredField(popLongPollingService, \"pollingMap\", pollingMap, true);\n        boolean actual = popLongPollingService.notifyMessageArriving(defaultTopic, queueId, cid, null, 0, null, null);\n        assertFalse(actual);\n    }\n\n    @Test\n    public void testWakeUpNullRequest() {\n        assertFalse(popLongPollingService.wakeUp(null));\n    }\n\n    @Test\n    public void testWakeUpIncompleteRequest() {\n        PopRequest request = mock(PopRequest.class);\n        when(request.complete()).thenReturn(false);\n        assertFalse(popLongPollingService.wakeUp(request));\n    }\n\n    @Test\n    public void testWakeUpInactiveChannel() {\n        PopRequest request = mock(PopRequest.class);\n        when(request.complete()).thenReturn(true);\n        when(request.getCtx()).thenReturn(ctx);\n        Channel channel = mock(Channel.class);\n        when(ctx.channel()).thenReturn(channel);\n        when(channel.isActive()).thenReturn(true);\n        when(brokerController.getPullMessageExecutor()).thenReturn(pullMessageExecutor);\n        assertTrue(popLongPollingService.wakeUp(request));\n    }\n\n    @Test\n    public void testWakeUpValidRequestWithException() throws Exception {\n        PopRequest request = mock(PopRequest.class);\n        when(request.complete()).thenReturn(true);\n        when(request.getCtx()).thenReturn(ctx);\n        Channel channel = mock(Channel.class);\n        when(ctx.channel()).thenReturn(channel);\n        when(request.getChannel()).thenReturn(channel);\n        when(channel.isActive()).thenReturn(true);\n        when(brokerController.getPullMessageExecutor()).thenReturn(pullMessageExecutor);\n        when(processor.processRequest(any(), any())).thenThrow(new RuntimeException(\"Test Exception\"));\n        assertTrue(popLongPollingService.wakeUp(request));\n        ArgumentCaptor<Runnable> captor = ArgumentCaptor.forClass(Runnable.class);\n        verify(pullMessageExecutor).submit(captor.capture());\n        captor.getValue().run();\n        verify(processor).processRequest(any(), any());\n    }\n\n    @Test\n    public void testPollingNotPolling() {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        RemotingCommand remotingCommand = mock(RemotingCommand.class);\n        PollingHeader requestHeader = mock(PollingHeader.class);\n        SubscriptionData subscriptionData = mock(SubscriptionData.class);\n        MessageFilter messageFilter = mock(MessageFilter.class);\n        when(requestHeader.getPollTime()).thenReturn(0L);\n        PollingResult result = popLongPollingService.polling(ctx, remotingCommand, requestHeader, subscriptionData, messageFilter);\n        assertEquals(PollingResult.NOT_POLLING, result);\n    }\n\n    @Test\n    public void testPollingServicePollingTimeout() throws IllegalAccessException {\n        String cid = \"CID_1\";\n        popLongPollingService = new PopLongPollingService(brokerController, processor, true);\n        popLongPollingService.shutdown();\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        RemotingCommand remotingCommand = mock(RemotingCommand.class);\n        PollingHeader requestHeader = mock(PollingHeader.class);\n        SubscriptionData subscriptionData = mock(SubscriptionData.class);\n        MessageFilter messageFilter = mock(MessageFilter.class);\n        when(requestHeader.getPollTime()).thenReturn(1000L);\n        when(requestHeader.getTopic()).thenReturn(defaultTopic);\n        when(requestHeader.getConsumerGroup()).thenReturn(\"defaultGroup\");\n        Cache<String, ConcurrentHashMap<String, Byte>> topicCidMap = Caffeine.newBuilder()\n            .maximumSize(10)\n            .expireAfterAccess(300, TimeUnit.SECONDS)\n            .build();\n        ConcurrentHashMap<String, Byte> cids = new ConcurrentHashMap<>();\n        cids.put(cid, (byte) 1);\n        topicCidMap.put(defaultTopic, cids);\n        FieldUtils.writeDeclaredField(popLongPollingService, \"topicCidMap\", topicCidMap, true);\n        PollingResult result = popLongPollingService.polling(ctx, remotingCommand, requestHeader, subscriptionData, messageFilter);\n        assertEquals(PollingResult.POLLING_TIMEOUT, result);\n    }\n\n    @Test\n    public void testPollingPollingSuc() {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        RemotingCommand remotingCommand = mock(RemotingCommand.class);\n        PollingHeader requestHeader = mock(PollingHeader.class);\n        SubscriptionData subscriptionData = mock(SubscriptionData.class);\n        MessageFilter messageFilter = mock(MessageFilter.class);\n        when(requestHeader.getPollTime()).thenReturn(1000L);\n        when(requestHeader.getBornTime()).thenReturn(System.currentTimeMillis());\n        when(requestHeader.getTopic()).thenReturn(\"topic\");\n        when(requestHeader.getConsumerGroup()).thenReturn(\"cid\");\n        when(requestHeader.getQueueId()).thenReturn(0);\n        PollingResult result = popLongPollingService.polling(ctx, remotingCommand, requestHeader, subscriptionData, messageFilter);\n        assertEquals(PollingResult.POLLING_SUC, result);\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.longpolling;\n\nimport io.netty.channel.Channel;\nimport java.util.HashMap;\nimport java.util.concurrent.Executors;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.processor.PullMessageProcessor;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.store.DefaultMessageFilter;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.assertj.core.api.Assertions;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class PullRequestHoldServiceTest {\n\n    @Mock\n    private BrokerController brokerController;\n\n    private PullRequestHoldService pullRequestHoldService;\n\n    @Mock\n    private PullRequest pullRequest;\n\n    private BrokerConfig brokerConfig = new BrokerConfig();\n\n    @Mock\n    private DefaultMessageStore defaultMessageStore;\n\n    @Mock\n    private DefaultMessageFilter defaultMessageFilter;\n\n    @Mock\n    private RemotingCommand remotingCommand;\n\n    @Mock\n    private Channel channel;\n\n    private SubscriptionData subscriptionData;\n\n    private static final String TEST_TOPIC = \"TEST_TOPIC\";\n\n    private static final int DEFAULT_QUEUE_ID = 0;\n\n    private static final long MAX_OFFSET = 100L;\n\n    @Before\n    public void before() {\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        when(brokerController.getPullMessageProcessor()).thenReturn(new PullMessageProcessor(brokerController));\n        when(brokerController.getPullMessageExecutor()).thenReturn(Executors.newCachedThreadPool());\n        pullRequestHoldService = new PullRequestHoldService(brokerController);\n        subscriptionData = new SubscriptionData(TEST_TOPIC, \"*\");\n        pullRequest = new PullRequest(remotingCommand, channel, 3000, 3000, 0L, subscriptionData, defaultMessageFilter);\n        pullRequestHoldService.start();\n    }\n\n    @After\n    public void after() {\n        pullRequestHoldService.shutdown();\n    }\n\n    @Test\n    public void suspendPullRequestTest() {\n        Assertions.assertThatCode(() -> pullRequestHoldService.suspendPullRequest(TEST_TOPIC, DEFAULT_QUEUE_ID, pullRequest)).doesNotThrowAnyException();\n    }\n\n    @Test\n    public void getServiceNameTest() {\n        final String name = pullRequestHoldService.getServiceName();\n        assert StringUtils.isNotEmpty(name);\n    }\n\n    @Test\n    public void checkHoldRequestTest() {\n        Assertions.assertThatCode(() -> pullRequestHoldService.checkHoldRequest()).doesNotThrowAnyException();\n    }\n\n    @Test\n    public void notifyMessageArrivingTest() {\n        Assertions.assertThatCode(() -> pullRequestHoldService.notifyMessageArriving(TEST_TOPIC, DEFAULT_QUEUE_ID, MAX_OFFSET)).doesNotThrowAnyException();\n\n        Assertions.assertThatCode(() -> pullRequestHoldService.suspendPullRequest(TEST_TOPIC, DEFAULT_QUEUE_ID, pullRequest)).doesNotThrowAnyException();\n\n        Assertions.assertThatCode(() -> pullRequestHoldService.notifyMessageArriving(TEST_TOPIC, DEFAULT_QUEUE_ID, MAX_OFFSET,\n            1L, System.currentTimeMillis(), new byte[10], new HashMap<>())).doesNotThrowAnyException();\n    }\n\n    @Test\n    public void notifyMasterOnlineTest() {\n        Assertions.assertThatCode(() -> pullRequestHoldService.suspendPullRequest(TEST_TOPIC, DEFAULT_QUEUE_ID, pullRequest)).doesNotThrowAnyException();\n\n        Assertions.assertThatCode(() -> pullRequestHoldService.notifyMasterOnline()).doesNotThrowAnyException();\n    }\n\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.metrics;\n\nimport io.opentelemetry.api.common.AttributeKey;\nimport io.opentelemetry.api.common.Attributes;\nimport io.opentelemetry.api.common.AttributesBuilder;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.metrics.MetricsExporterType;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Test;\n\nimport java.io.File;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.UUID;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class BrokerMetricsManagerTest {\n\n    private BrokerMetricsManager createTestBrokerMetricsManager() {\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        String storePathRootDir = System.getProperty(\"java.io.tmpdir\") + File.separator + \"store-\"\n                + UUID.randomUUID();\n        messageStoreConfig.setStorePathRootDir(storePathRootDir);\n        BrokerConfig brokerConfig = new BrokerConfig();\n\n        NettyServerConfig nettyServerConfig = new NettyServerConfig();\n        nettyServerConfig.setListenPort(0);\n\n        BrokerController brokerController = new BrokerController(brokerConfig, nettyServerConfig,\n                new NettyClientConfig(), messageStoreConfig);\n\n        return new BrokerMetricsManager(brokerController);\n    }\n\n    @Test\n    public void testNewAttributesBuilder() {\n        BrokerMetricsManager metricsManager = createTestBrokerMetricsManager();\n        Attributes attributes = metricsManager.newAttributesBuilder().put(\"a\", \"b\")\n                .build();\n        assertThat(attributes.get(AttributeKey.stringKey(\"a\"))).isEqualTo(\"b\");\n    }\n\n    @Test\n    public void testCustomizedAttributesBuilder() {\n        BrokerMetricsManager metricsManager = createTestBrokerMetricsManager();\n        \n        // Create a custom attributes builder supplier for testing\n        metricsManager.setAttributesBuilderSupplier(() -> new AttributesBuilder() {\n            private AttributesBuilder attributesBuilder = Attributes.builder();\n\n            @Override\n            public Attributes build() {\n                return attributesBuilder.put(\"customized\", \"value\").build();\n            }\n\n            @Override\n            public <T> AttributesBuilder put(AttributeKey<Long> key, int value) {\n                attributesBuilder.put(key, value);\n                return this;\n            }\n\n            @Override\n            public <T> AttributesBuilder put(AttributeKey<T> key, T value) {\n                attributesBuilder.put(key, value);\n                return this;\n            }\n\n            @Override\n            public AttributesBuilder putAll(Attributes attributes) {\n                attributesBuilder.putAll(attributes);\n                return this;\n            }\n        });\n        \n        Attributes attributes = metricsManager.newAttributesBuilder().put(\"a\", \"b\")\n                .build();\n        assertThat(attributes.get(AttributeKey.stringKey(\"a\"))).isEqualTo(\"b\");\n        assertThat(attributes.get(AttributeKey.stringKey(\"customized\"))).isEqualTo(\"value\");\n    }\n\n\n    @Test\n    public void testIsRetryOrDlqTopicWithRetryTopic() {\n        String topic = MixAll.RETRY_GROUP_TOPIC_PREFIX + \"TestTopic\";\n        boolean result = BrokerMetricsManager.isRetryOrDlqTopic(topic);\n        assertThat(result).isTrue();\n    }\n\n    @Test\n    public void testIsRetryOrDlqTopicWithDlqTopic() {\n        String topic = MixAll.DLQ_GROUP_TOPIC_PREFIX + \"TestTopic\";\n        boolean result = BrokerMetricsManager.isRetryOrDlqTopic(topic);\n        assertThat(result).isTrue();\n    }\n\n    @Test\n    public void testIsRetryOrDlqTopicWithNonRetryOrDlqTopic() {\n        String topic = \"NormalTopic\";\n        boolean result = BrokerMetricsManager.isRetryOrDlqTopic(topic);\n        assertThat(result).isFalse();\n    }\n\n    @Test\n    public void testIsRetryOrDlqTopicWithEmptyTopic() {\n        String topic = \"\";\n        boolean result = BrokerMetricsManager.isRetryOrDlqTopic(topic);\n        assertThat(result).isFalse();\n    }\n\n    @Test\n    public void testIsRetryOrDlqTopicWithNullTopic() {\n        String topic = null;\n        boolean result = BrokerMetricsManager.isRetryOrDlqTopic(topic);\n        assertThat(result).isFalse();\n    }\n\n    @Test\n    public void testIsSystemGroup_SystemGroup_ReturnsTrue() {\n        String group = \"FooGroup\";\n        String systemGroup = MixAll.CID_RMQ_SYS_PREFIX + group;\n        boolean result = BrokerMetricsManager.isSystemGroup(systemGroup);\n        assertThat(result).isTrue();\n    }\n\n    @Test\n    public void testIsSystemGroup_NonSystemGroup_ReturnsFalse() {\n        String group = \"FooGroup\";\n        boolean result = BrokerMetricsManager.isSystemGroup(group);\n        assertThat(result).isFalse();\n    }\n\n    @Test\n    public void testIsSystemGroup_EmptyGroup_ReturnsFalse() {\n        String group = \"\";\n        boolean result = BrokerMetricsManager.isSystemGroup(group);\n        assertThat(result).isFalse();\n    }\n\n    @Test\n    public void testIsSystemGroup_NullGroup_ReturnsFalse() {\n        String group = null;\n        boolean result = BrokerMetricsManager.isSystemGroup(group);\n        assertThat(result).isFalse();\n    }\n\n    @Test\n    public void testIsSystem_SystemTopicOrSystemGroup_ReturnsTrue() {\n        String topic = \"FooTopic\";\n        String group = \"FooGroup\";\n        String systemTopic = TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC;\n        String systemGroup = MixAll.CID_RMQ_SYS_PREFIX + group;\n\n        boolean resultTopic = BrokerMetricsManager.isSystem(systemTopic, group);\n        assertThat(resultTopic).isTrue();\n\n        boolean resultGroup = BrokerMetricsManager.isSystem(topic, systemGroup);\n        assertThat(resultGroup).isTrue();\n    }\n\n    @Test\n    public void testIsSystem_NonSystemTopicAndGroup_ReturnsFalse() {\n        String topic = \"FooTopic\";\n        String group = \"FooGroup\";\n        boolean result = BrokerMetricsManager.isSystem(topic, group);\n        assertThat(result).isFalse();\n    }\n\n    @Test\n    public void testIsSystem_EmptyTopicAndGroup_ReturnsFalse() {\n        String topic = \"\";\n        String group = \"\";\n        boolean result = BrokerMetricsManager.isSystem(topic, group);\n        assertThat(result).isFalse();\n    }\n\n    @Test\n    public void testGetMessageTypeAsNormal() {\n        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();\n        requestHeader.setProperties(\"\");\n\n        TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader);\n        assertThat(TopicMessageType.NORMAL).isEqualTo(result);\n    }\n\n    @Test\n    public void testGetMessageTypeAsTransaction() {\n        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();\n\n        Map<String, String> map = new HashMap<>();\n        map.put(MessageConst.PROPERTY_TRANSACTION_PREPARED, \"true\");\n        requestHeader.setProperties(MessageDecoder.messageProperties2String(map));\n\n        TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader);\n        assertThat(TopicMessageType.TRANSACTION).isEqualTo(result);\n    }\n\n    @Test\n    public void testGetMessageTypeAsFifo() {\n        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();\n        Map<String, String> map = new HashMap<>();\n        map.put(MessageConst.PROPERTY_SHARDING_KEY, \"shardingKey\");\n        requestHeader.setProperties(MessageDecoder.messageProperties2String(map));\n\n        TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader);\n        assertThat(TopicMessageType.FIFO).isEqualTo(result);\n    }\n\n    @Test\n    public void testGetMessageTypeAsDelayLevel() {\n        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();\n        Map<String, String> map = new HashMap<>();\n        map.put(MessageConst.PROPERTY_DELAY_TIME_LEVEL, \"1\");\n        requestHeader.setProperties(MessageDecoder.messageProperties2String(map));\n\n        TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader);\n        assertThat(TopicMessageType.DELAY).isEqualTo(result);\n    }\n\n    @Test\n    public void testGetMessageTypeAsDeliverMS() {\n        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();\n        Map<String, String> map = new HashMap<>();\n        map.put(MessageConst.PROPERTY_TIMER_DELIVER_MS, \"10\");\n        requestHeader.setProperties(MessageDecoder.messageProperties2String(map));\n\n        TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader);\n        assertThat(TopicMessageType.DELAY).isEqualTo(result);\n    }\n\n    @Test\n    public void testGetMessageTypeAsDelaySEC() {\n        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();\n        Map<String, String> map = new HashMap<>();\n        map.put(MessageConst.PROPERTY_TIMER_DELAY_SEC, \"1\");\n        requestHeader.setProperties(MessageDecoder.messageProperties2String(map));\n\n        TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader);\n        assertThat(TopicMessageType.DELAY).isEqualTo(result);\n    }\n\n    @Test\n    public void testGetMessageTypeAsDelayMS() {\n        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();\n        Map<String, String> map = new HashMap<>();\n        map.put(MessageConst.PROPERTY_TIMER_DELAY_MS, \"10\");\n        requestHeader.setProperties(MessageDecoder.messageProperties2String(map));\n\n        TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader);\n        assertThat(TopicMessageType.DELAY).isEqualTo(result);\n    }\n\n    @Test\n    public void testGetMessageTypeWithUnknownProperty() {\n        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();\n        Map<String, String> map = new HashMap<>();\n        map.put(\"unknownProperty\", \"unknownValue\");\n        requestHeader.setProperties(MessageDecoder.messageProperties2String(map));\n\n        TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader);\n        assertThat(TopicMessageType.NORMAL).isEqualTo(result);\n    }\n\n    @Test\n    public void testGetMessageTypeWithMultipleProperties() {\n        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();\n        Map<String, String> map = new HashMap<>();\n        map.put(MessageConst.PROPERTY_DELAY_TIME_LEVEL, \"1\");\n        map.put(MessageConst.PROPERTY_SHARDING_KEY, \"shardingKey\");\n        requestHeader.setProperties(MessageDecoder.messageProperties2String(map));\n\n        TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader);\n        assertThat(TopicMessageType.FIFO).isEqualTo(result);\n    }\n\n    @Test\n    public void testGetMessageTypeWithTransactionFlagButOtherPropertiesPresent() {\n        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();\n        Map<String, String> map = new HashMap<>();\n        map.put(MessageConst.PROPERTY_TRANSACTION_PREPARED, \"true\");\n        map.put(MessageConst.PROPERTY_SHARDING_KEY, \"shardingKey\");\n        requestHeader.setProperties(MessageDecoder.messageProperties2String(map));\n\n        TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader);\n        assertThat(TopicMessageType.TRANSACTION).isEqualTo(result);\n    }\n\n    @Test\n    public void testGetMessageTypeWithEmptyProperties() {\n        TopicMessageType result = BrokerMetricsManager.getMessageType(new SendMessageRequestHeader());\n        assertThat(TopicMessageType.NORMAL).isEqualTo(result);\n    }\n\n    @Test\n    public void testCreateMetricsManager() {\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        String storePathRootDir = System.getProperty(\"java.io.tmpdir\") + File.separator + \"store-\"\n                + UUID.randomUUID();\n        messageStoreConfig.setStorePathRootDir(storePathRootDir);\n        BrokerConfig brokerConfig = new BrokerConfig();\n\n        NettyServerConfig nettyServerConfig = new NettyServerConfig();\n        nettyServerConfig.setListenPort(0);\n\n        BrokerController brokerController = new BrokerController(brokerConfig, nettyServerConfig,\n                new NettyClientConfig(), messageStoreConfig);\n\n        BrokerMetricsManager metricsManager = new BrokerMetricsManager(brokerController);\n\n        assertThat(metricsManager.getBrokerMeter()).isNull();\n    }\n\n    @Test\n    public void testCreateMetricsManagerLogType() throws CloneNotSupportedException {\n        BrokerConfig brokerConfig = new BrokerConfig();\n        brokerConfig.setMetricsExporterType(MetricsExporterType.LOG);\n        brokerConfig.setMetricsLabel(\"label1:value1;label2:value2\");\n        brokerConfig.setMetricsOtelCardinalityLimit(1);\n\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        String storePathRootDir = System.getProperty(\"java.io.tmpdir\") + File.separator + \"store-\"\n                + UUID.randomUUID();\n        messageStoreConfig.setStorePathRootDir(storePathRootDir);\n\n        NettyServerConfig nettyServerConfig = new NettyServerConfig();\n        nettyServerConfig.setListenPort(0);\n\n        BrokerController brokerController = new BrokerController(brokerConfig, nettyServerConfig,\n                new NettyClientConfig(), messageStoreConfig);\n        brokerController.initialize();\n\n        BrokerMetricsManager metricsManager = new BrokerMetricsManager(brokerController);\n\n        assertThat(metricsManager.getBrokerMeter()).isNotNull();\n    }\n}"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/metrics/LiteConsumerLagCalculatorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.metrics;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.PriorityBlockingQueue;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.entity.TopicGroup;\nimport org.apache.rocketmq.common.lite.LiteLagInfo;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.Mockito.doReturn;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class LiteConsumerLagCalculatorTest {\n\n    private LiteConsumerLagCalculator liteConsumerLagCalculator;\n\n    @Mock\n    private BrokerController brokerController;\n\n    @Mock\n    private ConsumerOffsetManager consumerOffsetManager;\n\n    private final BrokerConfig brokerConfig = new BrokerConfig();\n\n    @Before\n    public void setUp() {\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);\n\n        liteConsumerLagCalculator = new LiteConsumerLagCalculator(brokerController);\n    }\n\n    @Test\n    public void testUpdateLagInfo() {\n        String group = \"testGroup\";\n        String topic = \"testTopic\";\n        String lmqName = LiteUtil.toLmqName(topic, \"lmq1\");\n        long storeTimestamp = System.currentTimeMillis();\n\n        liteConsumerLagCalculator.updateLagInfo(group, topic, lmqName, storeTimestamp);\n\n        TopicGroup topicGroup = new TopicGroup(topic, group);\n        PriorityBlockingQueue<LiteConsumerLagCalculator.LagTimeInfo> lagHeap =\n            liteConsumerLagCalculator.topicGroupLagTimeMap.get(topicGroup);\n        assertNotNull(lagHeap);\n        assertEquals(1, lagHeap.size());\n        LiteConsumerLagCalculator.LagTimeInfo lagInfo = lagHeap.peek();\n        assertNotNull(lagInfo);\n        assertEquals(lmqName, lagInfo.getLmqName());\n        assertEquals(storeTimestamp, lagInfo.getLagTimestamp());\n    }\n\n    @Test\n    public void testUpdateLagInfo_KeepSmallestWhenExceedsCapacity() {\n        String group = \"testGroup\";\n        String topic = \"testTopic\";\n\n        // Set topK to 3, so the heap will retain at most 3 elements\n        brokerConfig.setLiteLagLatencyTopK(3);\n\n        // Add 5 elements with timestamps 1000, 2000, 3000, 4000, 5000\n        // Expected result is to retain the smallest 3: 1000, 2000, 3000\n        liteConsumerLagCalculator.updateLagInfo(group, topic,\n            LiteUtil.toLmqName(topic, \"lmq1\"), 3000L);\n        liteConsumerLagCalculator.updateLagInfo(group, topic,\n            LiteUtil.toLmqName(topic, \"lmq2\"), 1000L);\n        liteConsumerLagCalculator.updateLagInfo(group, topic,\n            LiteUtil.toLmqName(topic, \"lmq3\"), 5000L);\n        liteConsumerLagCalculator.updateLagInfo(group, topic,\n            LiteUtil.toLmqName(topic, \"lmq4\"), 2000L);\n        liteConsumerLagCalculator.updateLagInfo(group, topic,\n            LiteUtil.toLmqName(topic, \"lmq5\"), 4000L);\n\n        // Verify that the heap contains only 3 elements\n        TopicGroup topicGroup = new TopicGroup(topic, group);\n        PriorityBlockingQueue<LiteConsumerLagCalculator.LagTimeInfo> lagHeap =\n            liteConsumerLagCalculator.topicGroupLagTimeMap.get(topicGroup);\n        assertNotNull(lagHeap);\n        assertEquals(3, lagHeap.size());\n\n        // Verify that the retained elements have the smallest timestamps: 1000, 2000, 3000\n        List<Long> timestamps = new ArrayList<>();\n        for (LiteConsumerLagCalculator.LagTimeInfo info : lagHeap) {\n            timestamps.add(info.getLagTimestamp());\n        }\n        Collections.sort(timestamps);\n        assertEquals(3, timestamps.size());\n        assertEquals(1000L, timestamps.get(0).longValue());\n        assertEquals(2000L, timestamps.get(1).longValue());\n        assertEquals(3000L, timestamps.get(2).longValue());\n    }\n\n    @Test\n    public void testRemoveLagInfo() {\n        String group = \"testGroup\";\n        String topic = \"testTopic\";\n        String lmqName = LiteUtil.toLmqName(topic, \"lmq1\");\n        long storeTimestamp = System.currentTimeMillis();\n\n        liteConsumerLagCalculator.updateLagInfo(group, topic, lmqName, storeTimestamp);\n        liteConsumerLagCalculator.removeLagInfo(group, topic, lmqName);\n\n        TopicGroup topicGroup = new TopicGroup(topic, group);\n        PriorityBlockingQueue<LiteConsumerLagCalculator.LagTimeInfo> lagHeap =\n            liteConsumerLagCalculator.topicGroupLagTimeMap.get(topicGroup);\n        assertTrue(lagHeap.isEmpty());\n    }\n\n    @Test\n    public void testOffsetTableForEachByGroup() {\n        String testTopic = \"testTopic\";\n        String liteTopic = \"lmq1\";\n        String testGroup = \"testGroup\";\n        String otherGroup = \"otherGroup\";\n        String lmqName = LiteUtil.toLmqName(testTopic, liteTopic);\n        String key = lmqName + \"@\" + testGroup;\n\n        // Prepare test data without thread-safe classes\n        ConcurrentMap<String, ConcurrentMap<Integer, Long>> offsetTable = new ConcurrentHashMap<>();\n        ConcurrentMap<Integer, Long> queueOffsetMap = new ConcurrentHashMap<>();\n        queueOffsetMap.put(0, 100L);\n        offsetTable.put(key, queueOffsetMap);\n\n        when(consumerOffsetManager.getOffsetTable()).thenReturn(offsetTable);\n\n        // Test processing all groups\n        final boolean[] processed = {false};\n        liteConsumerLagCalculator.offsetTableForEachByGroup(null, (topicGroup, offset) -> {\n            processed[0] = true;\n            assertEquals(lmqName, topicGroup.topic);\n            assertEquals(testGroup, topicGroup.group);\n            assertEquals(Long.valueOf(100L), offset);\n        });\n        assertTrue(processed[0]);\n\n        // Test processing specific group\n        processed[0] = false;\n        liteConsumerLagCalculator.offsetTableForEachByGroup(testGroup, (topicGroup, offset) -> {\n            processed[0] = true;\n            assertEquals(lmqName, topicGroup.topic);\n            assertEquals(testGroup, topicGroup.group);\n            assertEquals(Long.valueOf(100L), offset);\n        });\n        assertTrue(processed[0]);\n\n        // Test processing non-matching group\n        processed[0] = false;\n        liteConsumerLagCalculator.offsetTableForEachByGroup(otherGroup,\n            (topicGroup, offset) -> processed[0] = true);\n        assertFalse(processed[0]);\n    }\n\n    @Test\n    public void testGetLagTimestampTopK_NormalCase() {\n        // Prepare test data\n        String group = \"testGroup\";\n        String parentTopic = \"testParentTopic\";\n        String lmq1 = LiteUtil.toLmqName(parentTopic, \"lmq1\");\n        String lmq2 = LiteUtil.toLmqName(parentTopic, \"lmq2\");\n        String lmq3 = LiteUtil.toLmqName(parentTopic, \"lmq3\");\n\n        long timestamp1 = 1000L;\n        long timestamp2 = 2000L;\n        long timestamp3 = 1500L;\n\n        // Consumer offsets\n        long consumerOffset1 = 50L;\n//        long consumerOffset2 = 30L;\n        long consumerOffset3 = 40L;\n\n        // Max offsets\n        long maxOffset1 = 100L;\n//        long maxOffset2 = 80L;\n        long maxOffset3 = 90L;\n\n        // Create a spy of the calculator to allow partial mocking\n        LiteConsumerLagCalculator spyCalculator = spy(liteConsumerLagCalculator);\n\n        // Add lag info to the spy calculator\n        spyCalculator.updateLagInfo(group, parentTopic, lmq1, timestamp1);\n        spyCalculator.updateLagInfo(group, parentTopic, lmq2, timestamp2);\n        spyCalculator.updateLagInfo(group, parentTopic, lmq3, timestamp3);\n\n        // Mock getOffset and getMaxOffset methods on the spy\n        doReturn(consumerOffset1).when(spyCalculator).getOffset(group, lmq1);\n//        doReturn(consumerOffset2).when(spyCalculator).getOffset(group, lmq2);\n        doReturn(consumerOffset3).when(spyCalculator).getOffset(group, lmq3);\n\n        doReturn(maxOffset1).when(spyCalculator).getMaxOffset(lmq1);\n//        doReturn(maxOffset2).when(spyCalculator).getMaxOffset(lmq2);\n        doReturn(maxOffset3).when(spyCalculator).getMaxOffset(lmq3);\n\n        // Test with topK = 2\n        Pair<List<LiteLagInfo>, Long> result = spyCalculator.getLagTimestampTopK(group, parentTopic, 2);\n\n        // Verify results\n        assertNotNull(result);\n        assertEquals(2, result.getObject1().size());\n\n        // Should be sorted by timestamp in ascending order\n        assertEquals(timestamp1, result.getObject1().get(0).getEarliestUnconsumedTimestamp());\n        assertEquals(timestamp3, result.getObject1().get(1).getEarliestUnconsumedTimestamp());\n\n        // Verify lag counts (maxOffset - consumerOffset)\n        assertEquals(maxOffset1 - consumerOffset1, result.getObject1().get(0).getLagCount());\n        assertEquals(maxOffset3 - consumerOffset3, result.getObject1().get(1).getLagCount());\n\n        // Verify lite topics\n        assertEquals(\"lmq1\", result.getObject1().get(0).getLiteTopic());\n        assertEquals(\"lmq3\", result.getObject1().get(1).getLiteTopic());\n\n        // Verify earliest timestamp\n        assertEquals(timestamp1, result.getObject2().longValue());\n    }\n\n    @Test\n    public void testGetLagCountTopK_NormalCase() {\n        String group = \"testGroup\";\n        String topic = \"testTopic\";\n        String lmqName1 = LiteUtil.toLmqName(topic, \"lmq1\");\n        String lmqName2 = LiteUtil.toLmqName(topic, \"lmq2\");\n        String lmqName3 = LiteUtil.toLmqName(topic, \"lmq3\");\n\n        // Prepare offset table data\n        ConcurrentMap<String, ConcurrentMap<Integer, Long>> offsetTable = new ConcurrentHashMap<>();\n        ConcurrentMap<Integer, Long> queueOffsetMap1 = new ConcurrentHashMap<>();\n        ConcurrentMap<Integer, Long> queueOffsetMap2 = new ConcurrentHashMap<>();\n        ConcurrentMap<Integer, Long> queueOffsetMap3 = new ConcurrentHashMap<>();\n\n        long consumerOffset1 = 50L;\n        long consumerOffset2 = 30L;\n        long consumerOffset3 = 70L;\n\n        queueOffsetMap1.put(0, consumerOffset1);\n        queueOffsetMap2.put(0, consumerOffset2);\n        queueOffsetMap3.put(0, consumerOffset3);\n\n        offsetTable.put(lmqName1 + \"@\" + group, queueOffsetMap1);\n        offsetTable.put(lmqName2 + \"@\" + group, queueOffsetMap2);\n        offsetTable.put(lmqName3 + \"@\" + group, queueOffsetMap3);\n\n        when(consumerOffsetManager.getOffsetTable()).thenReturn(offsetTable);\n\n        // Mock store timestamps\n        long timestamp1 = 1000L;\n        long timestamp2 = 2000L;\n        long timestamp3 = 1500L;\n\n        // Create a spy of the calculator to allow partial mocking\n        LiteConsumerLagCalculator spyCalculator = spy(liteConsumerLagCalculator);\n\n        // Mock getStoreTimestamp method on the spy\n        doReturn(timestamp1).when(spyCalculator).getStoreTimestamp(lmqName1, consumerOffset1);\n        doReturn(timestamp2).when(spyCalculator).getStoreTimestamp(lmqName2, consumerOffset2);\n        doReturn(timestamp3).when(spyCalculator).getStoreTimestamp(lmqName3, consumerOffset3);\n\n        // Mock getMaxOffset method on the spy\n        doReturn(100L).when(spyCalculator).getMaxOffset(lmqName1);\n        doReturn(80L).when(spyCalculator).getMaxOffset(lmqName2);\n        doReturn(90L).when(spyCalculator).getMaxOffset(lmqName3);\n\n        // Test with topK = 2\n        Pair<List<LiteLagInfo>, Long> result = spyCalculator.getLagCountTopK(group, 2);\n\n        // Verify results\n        assertNotNull(result);\n        assertNotNull(result.getObject1());\n        assertEquals(2, result.getObject1().size());\n\n        // Should be sorted by lag count in descending order\n        // lmq1: 100-50=50, lmq2: 80-30=50, lmq3: 90-70=20\n        // So order should be lmq1(50), lmq2(50) or lmq2(50), lmq1(50) (both have same lag count)\n        LiteLagInfo first = result.getObject1().get(0);\n        LiteLagInfo second = result.getObject1().get(1);\n\n        // Verify lag counts\n        assertEquals(50L, first.getLagCount());\n        assertEquals(50L, second.getLagCount());\n\n        // Verify lite topics\n        assertTrue(first.getLiteTopic().equals(\"lmq1\") || first.getLiteTopic().equals(\"lmq2\"));\n        assertTrue(second.getLiteTopic().equals(\"lmq1\") || second.getLiteTopic().equals(\"lmq2\"));\n\n        // Verify timestamps\n        assertTrue(first.getEarliestUnconsumedTimestamp() == timestamp1 || first.getEarliestUnconsumedTimestamp() == timestamp2);\n        assertTrue(second.getEarliestUnconsumedTimestamp() == timestamp1 || second.getEarliestUnconsumedTimestamp() == timestamp2);\n\n        // Verify total lag count\n        assertEquals(120L, result.getObject2().longValue()); // 50 + 50 + 20\n    }\n\n    @Test\n    public void testCalculateLiteLagCount() {\n        brokerConfig.setLiteLagCountMetricsEnable(true);\n\n        String group = \"testGroup\";\n        String parentTopic = \"testParentTopic\";\n        String lmqName = LiteUtil.toLmqName(parentTopic, \"lmq1\");\n\n        ConcurrentMap<String, ConcurrentMap<Integer, Long>> offsetTable = new ConcurrentHashMap<>();\n        ConcurrentMap<Integer, Long> queueOffsetMap = new ConcurrentHashMap<>();\n        queueOffsetMap.put(0, 50L);\n        offsetTable.put(lmqName + \"@\" + group, queueOffsetMap);\n\n        when(consumerOffsetManager.getOffsetTable()).thenReturn(offsetTable);\n\n        LiteConsumerLagCalculator spyCalculator = spy(liteConsumerLagCalculator);\n        doReturn(100L).when(spyCalculator).getMaxOffset(lmqName);\n\n        final ConsumerLagCalculator.CalculateLagResult[] result = {null};\n        spyCalculator.calculateLiteLagCount(lagResult -> result[0] = lagResult);\n\n        assertNotNull(result[0]);\n        assertEquals(group, result[0].group);\n        // The metrics of liteTopic are aggregated under its parent topic\n        assertEquals(parentTopic, result[0].topic);\n        assertEquals(50L, result[0].lag);\n    }\n\n    @Test\n    public void testCalculateLiteLagLatency() {\n        brokerConfig.setLiteLagLatencyMetricsEnable(true);\n\n        String group = \"testGroup\";\n        String parentTopic = \"testParentTopic\";\n        String lmqName = LiteUtil.toLmqName(parentTopic, \"lmq1\");\n        long storeTimestamp = System.currentTimeMillis();\n\n        liteConsumerLagCalculator.updateLagInfo(group, parentTopic, lmqName, storeTimestamp);\n\n        final ConsumerLagCalculator.CalculateLagResult[] result = {null};\n        liteConsumerLagCalculator.calculateLiteLagLatency(lagResult -> result[0] = lagResult);\n\n        assertNotNull(result[0]);\n        assertEquals(group, result[0].group);\n        // The metrics of liteTopic are aggregated under its parent topic\n        assertEquals(parentTopic, result[0].topic);\n        assertEquals(storeTimestamp, result[0].earliestUnconsumedTimestamp);\n    }\n\n    @Test\n    public void testUpdateLagInfoWithDuplicateElements() {\n        String group = \"testGroup\";\n        String parentTopic = \"testParentTopic\";\n        String lmqName1 = \"lmq1\";\n        String lmqName2 = \"lmq2\";\n        String lmqName3 = \"lmq3\";\n        long storeTimestamp1 = 1000L;\n        long storeTimestamp2 = 2000L;\n        long storeTimestamp3 = 3000L;\n\n        // Add three LMQs with different timestamps, each added three times\n        for (int i = 0; i < 3; i++) {\n            liteConsumerLagCalculator.updateLagInfo(group, parentTopic, lmqName1, storeTimestamp1 + i * 100);\n            liteConsumerLagCalculator.updateLagInfo(group, parentTopic, lmqName2, storeTimestamp2 + i * 100);\n            liteConsumerLagCalculator.updateLagInfo(group, parentTopic, lmqName3, storeTimestamp3 + i * 100);\n        }\n\n        // Verify that the heap contains exactly 3 elements\n        PriorityBlockingQueue<LiteConsumerLagCalculator.LagTimeInfo> lagHeap = liteConsumerLagCalculator.topicGroupLagTimeMap\n                .get(new TopicGroup(parentTopic, group));\n        assertNotNull(lagHeap);\n        assertEquals(3, lagHeap.size());\n\n        // Verify that each LMQ is present with its latest timestamp\n        assertTrue(lagHeap.contains(new LiteConsumerLagCalculator.LagTimeInfo(lmqName1, storeTimestamp1 + 200)));\n        assertTrue(lagHeap.contains(new LiteConsumerLagCalculator.LagTimeInfo(lmqName2, storeTimestamp2 + 200)));\n        assertTrue(lagHeap.contains(new LiteConsumerLagCalculator.LagTimeInfo(lmqName3, storeTimestamp3 + 200)));\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.offset;\n\nimport java.time.Duration;\nimport java.util.HashSet;\nimport java.util.Set;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ConsumerManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.stubbing.Answer;\n\nimport static org.awaitility.Awaitility.await;\nimport static org.mockito.ArgumentMatchers.anyBoolean;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.doAnswer;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\npublic class BroadcastOffsetManagerTest {\n\n    private final AtomicLong maxOffset = new AtomicLong(10L);\n    private final AtomicLong commitOffset = new AtomicLong(-1);\n\n    private final ConsumerOffsetManager consumerOffsetManager = mock(ConsumerOffsetManager.class);\n    private final ConsumerManager consumerManager = mock(ConsumerManager.class);\n    private final BrokerConfig brokerConfig = new BrokerConfig();\n    private final Set<String> onlineClientIdSet = new HashSet<>();\n    private BroadcastOffsetManager broadcastOffsetManager;\n\n    @Before\n    public void before() throws ConsumeQueueException {\n        brokerConfig.setEnableBroadcastOffsetStore(true);\n        brokerConfig.setBroadcastOffsetExpireSecond(1);\n        brokerConfig.setBroadcastOffsetExpireMaxSecond(5);\n        BrokerController brokerController = mock(BrokerController.class);\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n\n        when(brokerController.getConsumerManager()).thenReturn(consumerManager);\n        doAnswer((Answer<ClientChannelInfo>) mock -> {\n            String clientId = mock.getArgument(1);\n            if (onlineClientIdSet.contains(clientId)) {\n                return new ClientChannelInfo(null);\n            }\n            return null;\n        }).when(consumerManager).findChannel(anyString(), anyString());\n\n        doAnswer((Answer<Long>) mock -> commitOffset.get())\n            .when(consumerOffsetManager).queryOffset(anyString(), anyString(), anyInt());\n        doAnswer((Answer<Void>) mock -> {\n            commitOffset.set(mock.getArgument(4));\n            return null;\n        }).when(consumerOffsetManager).commitOffset(anyString(), anyString(), anyString(), anyInt(), anyLong());\n        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);\n\n        MessageStore messageStore = mock(MessageStore.class);\n        doAnswer((Answer<Long>) mock -> maxOffset.get())\n            .when(messageStore).getMaxOffsetInQueue(anyString(), anyInt(), anyBoolean());\n        when(brokerController.getMessageStore()).thenReturn(messageStore);\n\n        broadcastOffsetManager = new BroadcastOffsetManager(brokerController);\n    }\n\n    @Test\n    public void testBroadcastOffsetSwitch() throws ConsumeQueueException {\n        // client1 connect to broker\n        onlineClientIdSet.add(\"client1\");\n        long offset = broadcastOffsetManager.queryInitOffset(\"group\", \"topic\", 0, \"client1\", 0, false);\n        Assert.assertEquals(-1, offset);\n        broadcastOffsetManager.updateOffset(\"group\", \"topic\", 0, 10, \"client1\", false);\n        offset = broadcastOffsetManager.queryInitOffset(\"group\", \"topic\", 0, \"client1\", 11, false);\n        Assert.assertEquals(-1, offset);\n        broadcastOffsetManager.updateOffset(\"group\", \"topic\", 0, 11, \"client1\", false);\n\n        // client1 connect to proxy\n        offset = broadcastOffsetManager.queryInitOffset(\"group\", \"topic\", 0, \"client1\", -1, true);\n        Assert.assertEquals(11, offset);\n        broadcastOffsetManager.updateOffset(\"group\", \"topic\", 0, 11, \"client1\", true);\n        offset = broadcastOffsetManager.queryInitOffset(\"group\", \"topic\", 0, \"client1\", 11, true);\n        Assert.assertEquals(-1, offset);\n        broadcastOffsetManager.updateOffset(\"group\", \"topic\", 0, 12, \"client1\", true);\n\n        broadcastOffsetManager.scanOffsetData();\n        Assert.assertEquals(12L, commitOffset.get());\n\n        // client2 connect to proxy\n        onlineClientIdSet.add(\"client2\");\n        offset = broadcastOffsetManager.queryInitOffset(\"group\", \"topic\", 0, \"client2\", -1, true);\n        Assert.assertEquals(12, offset);\n        broadcastOffsetManager.updateOffset(\"group\", \"topic\", 0, 12, \"client2\", true);\n        offset = broadcastOffsetManager.queryInitOffset(\"group\", \"topic\", 0, \"client2\", 11, true);\n        Assert.assertEquals(-1, offset);\n        broadcastOffsetManager.updateOffset(\"group\", \"topic\", 0, 13, \"client2\", true);\n\n        broadcastOffsetManager.scanOffsetData();\n        Assert.assertEquals(12L, commitOffset.get());\n\n        // client1 connect to broker\n        offset = broadcastOffsetManager.queryInitOffset(\"group\", \"topic\", 0, \"client1\", 20, false);\n        Assert.assertEquals(12, offset);\n        broadcastOffsetManager.updateOffset(\"group\", \"topic\", 0, 12, \"client1\", false);\n        offset = broadcastOffsetManager.queryInitOffset(\"group\", \"topic\", 0, \"client1\", 12, false);\n        Assert.assertEquals(-1, offset);\n\n        onlineClientIdSet.clear();\n\n        maxOffset.set(30L);\n\n        // client3 connect to broker\n        onlineClientIdSet.add(\"client3\");\n        offset = broadcastOffsetManager.queryInitOffset(\"group\", \"topic\", 0, \"client3\", 30, false);\n        Assert.assertEquals(-1, offset);\n        broadcastOffsetManager.updateOffset(\"group\", \"topic\", 0, 30, \"client3\", false);\n\n        await().atMost(Duration.ofSeconds(brokerConfig.getBroadcastOffsetExpireSecond() + 1)).until(() -> {\n            broadcastOffsetManager.scanOffsetData();\n            return commitOffset.get() == 30L;\n        });\n    }\n\n    @Test\n    public void testBroadcastOffsetExpire() {\n        onlineClientIdSet.add(\"client1\");\n        broadcastOffsetManager.updateOffset(\n            \"group\", \"topic\", 0, 10, \"client1\", false);\n        onlineClientIdSet.clear();\n\n        await().atMost(Duration.ofSeconds(brokerConfig.getBroadcastOffsetExpireSecond() + 1)).until(() -> {\n            broadcastOffsetManager.scanOffsetData();\n            return broadcastOffsetManager.offsetStoreMap.isEmpty();\n        });\n\n        onlineClientIdSet.add(\"client1\");\n        broadcastOffsetManager.updateOffset(\n            \"group\", \"topic\", 0, 10, \"client1\", false);\n        await().atMost(Duration.ofSeconds(brokerConfig.getBroadcastOffsetExpireMaxSecond() + 1)).until(() -> {\n            broadcastOffsetManager.scanOffsetData();\n            return broadcastOffsetManager.offsetStoreMap.isEmpty();\n        });\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/offset/BroadcastOffsetStoreTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.offset;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class BroadcastOffsetStoreTest {\n\n    @Test\n    public void testBasicOffsetStore() {\n        BroadcastOffsetStore offsetStore = new BroadcastOffsetStore();\n        offsetStore.updateOffset(0, 100L, false);\n        offsetStore.updateOffset(1, 200L, false);\n        Assert.assertEquals(100L, offsetStore.readOffset(0));\n    }\n}"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.offset;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.mockito.Mockito;\n\nimport static org.apache.rocketmq.broker.offset.ConsumerOffsetManager.TOPIC_GROUP_SEPARATOR;\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class ConsumerOffsetManagerTest {\n\n    private static final String KEY = \"FooBar@FooBarGroup\";\n\n    private BrokerController brokerController;\n\n    private ConsumerOffsetManager consumerOffsetManager;\n\n    @Before\n    @SuppressWarnings(\"DoubleBraceInitialization\")\n    public void init() {\n        brokerController = Mockito.mock(BrokerController.class);\n        consumerOffsetManager = new ConsumerOffsetManager(brokerController);\n\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        Mockito.when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n\n        ConcurrentHashMap<String, ConcurrentMap<Integer, Long>> offsetTable = new ConcurrentHashMap<>(512);\n        offsetTable.put(KEY,new ConcurrentHashMap<Integer, Long>() {{\n                put(1,2L);\n                put(2,3L);\n            }});\n        consumerOffsetManager.setOffsetTable(offsetTable);\n    }\n\n    @Test\n    public void cleanOffsetByTopic_NotExist() {\n        consumerOffsetManager.cleanOffsetByTopic(\"InvalidTopic\");\n        assertThat(consumerOffsetManager.getOffsetTable().containsKey(KEY)).isTrue();\n    }\n\n    @Test\n    public void cleanOffsetByTopic_Exist() {\n        consumerOffsetManager.cleanOffsetByTopic(\"FooBar\");\n        assertThat(!consumerOffsetManager.getOffsetTable().containsKey(KEY)).isTrue();\n    }\n\n    @Test\n    public void removeOffsetByGroupTest() {\n        String topic = \"TopicName\";\n        String group = \"GroupName\";\n        Mockito.when(brokerController.getBrokerConfig()).thenReturn(new BrokerConfig());\n        consumerOffsetManager.commitOffset(\"Commit\", group, topic, 0, 100);\n        consumerOffsetManager.assignResetOffset(topic, group, 0, 100);\n        consumerOffsetManager.commitPullOffset(\"Pull\", group, topic, 0, 100);\n        consumerOffsetManager.removeOffset(group);\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(topic + TOPIC_GROUP_SEPARATOR + group));\n\n        consumerOffsetManager.commitPullOffset(\"Pull\", group, topic, 0, 100);\n        consumerOffsetManager.clearPullOffset(group, topic);\n        Assert.assertEquals(-1L, consumerOffsetManager.queryPullOffset(group, topic, 0));\n    }\n\n    @Test\n    public void testOffsetPersistInMemory() {\n        ConcurrentMap<String, ConcurrentMap<Integer, Long>> offsetTable = consumerOffsetManager.getOffsetTable();\n        ConcurrentMap<Integer, Long> table = new ConcurrentHashMap<>();\n        table.put(0, 1L);\n        table.put(1, 3L);\n        String group = \"G1\";\n        offsetTable.put(group, table);\n\n        consumerOffsetManager.persist();\n        ConsumerOffsetManager manager = new ConsumerOffsetManager(brokerController);\n        manager.load();\n\n        ConcurrentMap<Integer, Long> offsetTableLoaded = manager.getOffsetTable().get(group);\n        Assert.assertEquals(table, offsetTableLoaded);\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/offset/LmqConsumerOffsetManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.offset;\n\nimport java.io.File;\nimport java.util.Map;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.subscription.LmqSubscriptionGroupManager;\nimport org.apache.rocketmq.broker.topic.LmqTopicConfigManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.After;\nimport org.junit.Test;\nimport org.mockito.Spy;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class LmqConsumerOffsetManagerTest {\n\n    @Spy\n    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(),\n        new NettyClientConfig(), new MessageStoreConfig());\n\n    @Test\n    public void testOffsetManage() {\n        LmqConsumerOffsetManager lmqConsumerOffsetManager = new LmqConsumerOffsetManager(brokerController);\n        LmqTopicConfigManager lmqTopicConfigManager = new LmqTopicConfigManager(brokerController);\n        LmqSubscriptionGroupManager lmqSubscriptionGroupManager = new LmqSubscriptionGroupManager(brokerController);\n\n        String lmqTopicName = \"%LMQ%1111\";\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(lmqTopicName);\n        lmqTopicConfigManager.updateTopicConfig(topicConfig);\n        TopicConfig topicConfig1 = lmqTopicConfigManager.selectTopicConfig(lmqTopicName);\n        assertThat(topicConfig1.getTopicName()).isEqualTo(topicConfig.getTopicName());\n\n        String lmqGroupName = \"%LMQ%GID_test\";\n        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n        subscriptionGroupConfig.setGroupName(lmqGroupName);\n        lmqSubscriptionGroupManager.updateSubscriptionGroupConfig(subscriptionGroupConfig);\n        SubscriptionGroupConfig subscriptionGroupConfig1 = lmqSubscriptionGroupManager.findSubscriptionGroupConfig(\n            lmqGroupName);\n        assertThat(subscriptionGroupConfig1.getGroupName()).isEqualTo(subscriptionGroupConfig.getGroupName());\n\n        lmqConsumerOffsetManager.commitOffset(\"127.0.0.1\", lmqGroupName, lmqTopicName, 0, 10L);\n        Map<Integer, Long> integerLongMap = lmqConsumerOffsetManager.queryOffset(lmqGroupName, lmqTopicName);\n        assertThat(integerLongMap.get(0)).isEqualTo(10L);\n        long offset = lmqConsumerOffsetManager.queryOffset(lmqGroupName, lmqTopicName, 0);\n        assertThat(offset).isEqualTo(10L);\n\n        long offset1 = lmqConsumerOffsetManager.queryOffset(lmqGroupName, lmqTopicName + \"test\", 0);\n        assertThat(offset1).isEqualTo(-1L);\n    }\n\n    @Test\n    public void testOffsetManage1() {\n        LmqConsumerOffsetManager lmqConsumerOffsetManager = new LmqConsumerOffsetManager(brokerController);\n\n        String lmqTopicName = \"%LMQ%1111\";\n\n        String lmqGroupName = \"%LMQ%GID_test\";\n\n        lmqConsumerOffsetManager.commitOffset(\"127.0.0.1\", lmqGroupName, lmqTopicName, 0, 10L);\n\n        lmqTopicName = \"%LMQ%1222\";\n\n        lmqGroupName = \"%LMQ%GID_test222\";\n\n        lmqConsumerOffsetManager.commitOffset(\"127.0.0.1\", lmqGroupName, lmqTopicName, 0, 10L);\n        lmqConsumerOffsetManager.commitOffset(\"127.0.0.1\",\"GID_test1\", \"MqttTest\",0, 10L);\n\n        String json = lmqConsumerOffsetManager.encode(true);\n\n        LmqConsumerOffsetManager lmqConsumerOffsetManager1 = new LmqConsumerOffsetManager(brokerController);\n\n        lmqConsumerOffsetManager1.decode(json);\n\n        assertThat(lmqConsumerOffsetManager1.getOffsetTable().size()).isEqualTo(1);\n        assertThat(lmqConsumerOffsetManager1.getLmqOffsetTable().size()).isEqualTo(2);\n    }\n\n    @After\n    public void destroy() {\n        UtilAll.deleteFile(new File(new MessageStoreConfig().getStorePathRootDir()));\n    }\n\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.offset;\n\nimport java.io.File;\nimport java.util.UUID;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.config.v1.RocksDBConsumerOffsetManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Assume;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.Mockito;\n\nimport static org.apache.rocketmq.broker.offset.ConsumerOffsetManager.TOPIC_GROUP_SEPARATOR;\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class RocksDBConsumerOffsetManagerTest {\n\n    private static final String SKIP_MAC_KEY = \"skipMac\";\n\n    private static final String KEY = \"FooBar@FooBarGroup\";\n\n    private BrokerController brokerController;\n\n    private ConsumerOffsetManager consumerOffsetManager;\n\n    private BrokerConfig brokerConfig;\n\n    @Before\n    public void init() {\n//        System.setProperty(SKIP_MAC_KEY, \"false\");\n        skipMacIfNecessary();\n        brokerController = Mockito.mock(BrokerController.class);\n        brokerConfig = new BrokerConfig();\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        Mockito.when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n        Mockito.when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n\n        consumerOffsetManager = new RocksDBConsumerOffsetManager(brokerController);\n        consumerOffsetManager.load();\n\n        ConcurrentHashMap<String, ConcurrentMap<Integer, Long>> offsetTable = new ConcurrentHashMap<>(512);\n        ConcurrentHashMap<Integer, Long> innerMap = new ConcurrentHashMap<>();\n        innerMap.put(1, 2L);\n        innerMap.put(2, 3L);\n        offsetTable.put(KEY, innerMap);\n        consumerOffsetManager.setOffsetTable(offsetTable);\n    }\n\n    @After\n    public void destroy() {\n        if (consumerOffsetManager != null) {\n            consumerOffsetManager.stop();\n            File file = new File(((RocksDBConsumerOffsetManager) consumerOffsetManager).rocksdbConfigFilePath(null, false));\n            UtilAll.deleteFile(file);\n        }\n    }\n\n    @Test\n    public void cleanOffsetByTopic_NotExist() {\n        consumerOffsetManager.cleanOffsetByTopic(\"InvalidTopic\");\n        assertThat(consumerOffsetManager.getOffsetTable().containsKey(KEY)).isTrue();\n    }\n\n    @Test\n    public void cleanOffsetByTopic_Exist() {\n        consumerOffsetManager.cleanOffsetByTopic(\"FooBar\");\n        assertThat(!consumerOffsetManager.getOffsetTable().containsKey(KEY)).isTrue();\n    }\n\n    @Test\n    public void testOffsetPersistInMemory() {\n        ConcurrentMap<String, ConcurrentMap<Integer, Long>> offsetTable = consumerOffsetManager.getOffsetTable();\n        ConcurrentMap<Integer, Long> table = new ConcurrentHashMap<>();\n        table.put(0, 1L);\n        table.put(1, 3L);\n        String group = \"G1\";\n        offsetTable.put(group, table);\n\n        consumerOffsetManager.persist();\n        consumerOffsetManager.stop();\n        consumerOffsetManager.load();\n\n        ConcurrentMap<Integer, Long> offsetTableLoaded = consumerOffsetManager.getOffsetTable().get(group);\n        Assert.assertEquals(table, offsetTableLoaded);\n    }\n\n    @Test\n    public void testCommitOffset_persist_periodically() {\n        brokerConfig.setPersistConsumerOffsetIncrementally(false);\n        String group = UUID.randomUUID().toString();\n        String topic = UUID.randomUUID().toString();\n        String key = topic + TOPIC_GROUP_SEPARATOR + group;\n\n        // 1. commit but not persist\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key));\n        consumerOffsetManager.commitOffset(\"ClientID\", group, topic, 0, 1);\n        Assert.assertTrue(consumerOffsetManager.getOffsetTable().containsKey(key));\n\n        consumerOffsetManager.stop();\n        consumerOffsetManager.getOffsetTable().clear();\n        consumerOffsetManager.load();\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key)); // not in kv\n\n        // 2. commit and persist\n        consumerOffsetManager.commitOffset(\"ClientID\", group, topic, 0, 1);\n        Assert.assertTrue(consumerOffsetManager.getOffsetTable().containsKey(key));\n        consumerOffsetManager.persist();\n        consumerOffsetManager.stop();\n        consumerOffsetManager.getOffsetTable().clear();\n        consumerOffsetManager.load();\n        Assert.assertTrue(consumerOffsetManager.getOffsetTable().containsKey(key)); // load from kv\n    }\n\n    @Test\n    public void testCommitOffset_persist_incrementally() {\n        brokerConfig.setPersistConsumerOffsetIncrementally(true);\n        String group = UUID.randomUUID().toString();\n        String topic = UUID.randomUUID().toString();\n        String key = topic + TOPIC_GROUP_SEPARATOR + group;\n\n        // commit but not persist\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key));\n        consumerOffsetManager.commitOffset(\"ClientID\", group, topic, 0, 1);\n        Assert.assertTrue(consumerOffsetManager.getOffsetTable().containsKey(key));\n\n        consumerOffsetManager.stop();\n        consumerOffsetManager.getOffsetTable().clear();\n        consumerOffsetManager.load();\n        Assert.assertTrue(consumerOffsetManager.getOffsetTable().containsKey(key)); // reload from kv\n    }\n\n    @Test\n    public void testRemoveConsumerOffset() {\n        String group = UUID.randomUUID().toString();\n        String topic = UUID.randomUUID().toString();\n        String key = topic + TOPIC_GROUP_SEPARATOR + group;\n\n        // commit and persist\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key));\n        consumerOffsetManager.commitOffset(\"ClientID\", group, topic, 0, 1);\n        Assert.assertTrue(consumerOffsetManager.getOffsetTable().containsKey(key));\n        consumerOffsetManager.persist();\n\n        consumerOffsetManager.removeConsumerOffset(key);\n        consumerOffsetManager.stop();\n        consumerOffsetManager.getOffsetTable().clear();\n        consumerOffsetManager.load();\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key)); // removed from kv\n    }\n\n    @Test\n    public void testRemoveOffset() {\n        String group = UUID.randomUUID().toString();\n        String topic1 = UUID.randomUUID().toString();\n        String topic2 = UUID.randomUUID().toString();\n        String key1 = topic1 + TOPIC_GROUP_SEPARATOR + group;\n        String key2 = topic2 + TOPIC_GROUP_SEPARATOR + group;\n\n        // commit and persist\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key1));\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key2));\n        consumerOffsetManager.commitOffset(\"ClientID\", group, topic1, 0, 1);\n        consumerOffsetManager.commitOffset(\"ClientID\", group, topic2, 0, 1);\n        Assert.assertTrue(consumerOffsetManager.getOffsetTable().containsKey(key1));\n        Assert.assertTrue(consumerOffsetManager.getOffsetTable().containsKey(key2));\n        consumerOffsetManager.persist();\n\n        // remove all offsets by group\n        consumerOffsetManager.removeOffset(group);\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key1));\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key2));\n        consumerOffsetManager.stop();\n        consumerOffsetManager.getOffsetTable().clear();\n        consumerOffsetManager.load();\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key1)); // removed from kv\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key2)); // removed from kv\n    }\n\n    @Test\n    // similar to testRemoveOffset()\n    public void testCleanOffset() {\n        String group = UUID.randomUUID().toString();\n        String topic1 = UUID.randomUUID().toString();\n        String topic2 = UUID.randomUUID().toString();\n        String key1 = topic1 + TOPIC_GROUP_SEPARATOR + group;\n        String key2 = topic2 + TOPIC_GROUP_SEPARATOR + group;\n\n        // commit and persist\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key1));\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key2));\n        consumerOffsetManager.commitOffset(\"ClientID\", group, topic1, 0, 1);\n        consumerOffsetManager.commitOffset(\"ClientID\", group, topic2, 0, 1);\n        Assert.assertTrue(consumerOffsetManager.getOffsetTable().containsKey(key1));\n        Assert.assertTrue(consumerOffsetManager.getOffsetTable().containsKey(key2));\n        consumerOffsetManager.persist();\n\n        // remove all offsets by group\n        consumerOffsetManager.cleanOffset(group);\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key1));\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key2));\n        consumerOffsetManager.stop();\n        consumerOffsetManager.getOffsetTable().clear();\n        consumerOffsetManager.load();\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key1)); // removed from kv\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key2)); // removed from kv\n    }\n\n    @Test\n    public void testCleanOffsetByTopic() {\n        String group1 = UUID.randomUUID().toString();\n        String group2 = UUID.randomUUID().toString();\n        String topic = UUID.randomUUID().toString();\n        String key1 = topic + TOPIC_GROUP_SEPARATOR + group1;\n        String key2 = topic + TOPIC_GROUP_SEPARATOR + group2;\n\n        // commit and persist\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key1));\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key2));\n        consumerOffsetManager.commitOffset(\"ClientID\", group1, topic, 0, 1);\n        consumerOffsetManager.commitOffset(\"ClientID\", group2, topic, 0, 1);\n        Assert.assertTrue(consumerOffsetManager.getOffsetTable().containsKey(key1));\n        Assert.assertTrue(consumerOffsetManager.getOffsetTable().containsKey(key2));\n        consumerOffsetManager.persist();\n\n        // remove all offsets by group\n        consumerOffsetManager.cleanOffsetByTopic(topic);\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key1));\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key2));\n        consumerOffsetManager.stop();\n        consumerOffsetManager.getOffsetTable().clear();\n        consumerOffsetManager.load();\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key1)); // removed from kv\n        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(key2)); // removed from kv\n    }\n\n    @Test\n    public void testUpdateDataVersion() {\n        Assert.assertEquals(0, consumerOffsetManager.getDataVersion().getCounter().get());\n        for (int i = 0; i < 10; i++) {\n            ((RocksDBConsumerOffsetManager) consumerOffsetManager).updateDataVersion();\n        }\n        Assert.assertEquals(10, consumerOffsetManager.getDataVersion().getCounter().get());\n    }\n\n    @Test\n    public void testLoadDataVersion() {\n        for (int i = 0; i < 10; i++) {\n            ((RocksDBConsumerOffsetManager) consumerOffsetManager).updateDataVersion();\n        }\n        consumerOffsetManager.stop();\n        consumerOffsetManager.load();\n        Assert.assertEquals(10, consumerOffsetManager.getDataVersion().getCounter().get());\n    }\n\n    private static void skipMacIfNecessary() {\n        boolean skipMac = Boolean.parseBoolean(System.getProperty(SKIP_MAC_KEY, \"true\"));\n        Assume.assumeFalse(MixAll.isMac() && skipMac);\n    }\n}"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.offset;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.config.v1.RocksDBConsumerOffsetManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.Mockito;\n\nimport java.util.HashMap;\nimport java.util.Map;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\nimport static org.mockito.Mockito.when;\n\npublic class RocksDBLmqConsumerOffsetManagerTest {\n    private static final String LMQ_GROUP = MixAll.LMQ_PREFIX + \"FooBarGroup\";\n    private static final String NON_LMQ_GROUP = \"nonLmqGroup\";\n\n    private static final String LMQ_TOPIC = MixAll.LMQ_PREFIX + \"FooBarTopic\";\n    private static final String NON_LMQ_TOPIC = \"FooBarTopic\";\n    private static final int QUEUE_ID = 0;\n    private static final long OFFSET = 12345;\n\n    private BrokerController brokerController;\n\n    private RocksDBConsumerOffsetManager offsetManager;\n\n    @Before\n    public void setUp() {\n        brokerController = Mockito.mock(BrokerController.class);\n        when(brokerController.getMessageStoreConfig()).thenReturn(new MessageStoreConfig());\n        when(brokerController.getBrokerConfig()).thenReturn(new BrokerConfig());\n        offsetManager = new RocksDBConsumerOffsetManager(brokerController);\n    }\n\n\n    @Test\n    public void testQueryOffsetForNonLmq() {\n        long actualOffset = offsetManager.queryOffset(NON_LMQ_GROUP, NON_LMQ_TOPIC, QUEUE_ID);\n        // Verify\n        assertEquals(\"Offset should not be null.\", -1, actualOffset);\n    }\n\n\n    @Test\n    public void testQueryOffsetForLmqGroupWithExistingOffset() {\n        offsetManager.commitOffset(\"127.0.0.1\",LMQ_GROUP, LMQ_TOPIC, QUEUE_ID, OFFSET);\n\n        // Act\n        Map<Integer, Long> actualOffsets = offsetManager.queryOffset(LMQ_GROUP, LMQ_TOPIC);\n\n        // Assert\n        assertNotNull(actualOffsets);\n        assertEquals(1, actualOffsets.size());\n        assertEquals(OFFSET, (long) actualOffsets.get(0));\n    }\n\n    @Test\n    public void testQueryOffsetForLmqGroupWithoutExistingOffset() {\n        // Act\n        Map<Integer, Long> actualOffsets = offsetManager.queryOffset(LMQ_GROUP, \"nonExistingTopic\");\n        // Assert\n        assertNull(actualOffsets);\n    }\n\n    @Test\n    public void testQueryOffsetForNonLmqGroup() {\n        // Arrange\n        Map<Integer, Long> mockOffsets = new HashMap<>();\n        mockOffsets.put(QUEUE_ID, OFFSET);\n\n        offsetManager.commitOffset(\"clientHost\", NON_LMQ_GROUP, NON_LMQ_TOPIC, QUEUE_ID, OFFSET);\n\n        // Act\n        Map<Integer, Long> actualOffsets = offsetManager.queryOffset(NON_LMQ_GROUP, NON_LMQ_TOPIC);\n\n        // Assert\n        assertNotNull(actualOffsets);\n        assertEquals(\"Offsets should match the mocked return value for non-LMQ groups\", mockOffsets, actualOffsets);\n    }\n\n    @Test\n    public void testCommitOffsetForLmq() {\n        // Execute\n        offsetManager.commitOffset(\"clientHost\", LMQ_GROUP, LMQ_TOPIC, QUEUE_ID, OFFSET);\n        // Verify\n        Long expectedOffset = offsetManager.getOffsetTable().get(getLMQKey()).get(QUEUE_ID);\n        assertEquals(\"Offset should be updated correctly.\", OFFSET, expectedOffset.longValue());\n    }\n\n    private String getLMQKey() {\n        return LMQ_TOPIC + \"@\" + LMQ_GROUP;\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBOffsetSerializeWrapperTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.offset;\n\n\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\n\nimport org.apache.rocketmq.broker.config.v1.RocksDBOffsetSerializeWrapper;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\n\npublic class RocksDBOffsetSerializeWrapperTest {\n\n    private RocksDBOffsetSerializeWrapper wrapper;\n\n    @Before\n    public void setUp() {\n        wrapper = new RocksDBOffsetSerializeWrapper();\n    }\n\n    @Test\n    public void testSetOffsetTable_ShouldSetTheOffsetTableCorrectly() {\n        ConcurrentMap<Integer, Long> newOffsetTable = new ConcurrentHashMap<>();\n        wrapper.setOffsetTable(newOffsetTable);\n        ConcurrentMap<Integer, Long> offsetTable = wrapper.getOffsetTable();\n        assertEquals(\"The offsetTable should be the same as the one set\", newOffsetTable, offsetTable);\n    }\n\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.offset;\n\nimport java.io.IOException;\nimport java.nio.file.Paths;\nimport java.util.HashMap;\nimport java.util.UUID;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.collections.MapUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.config.v1.RocksDBConsumerOffsetManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.CheckRocksdbCqWriteResult;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.StoreType;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.queue.CombineConsumeQueueStore;\nimport org.apache.rocketmq.store.queue.ConsumeQueueInterface;\nimport org.apache.rocketmq.store.queue.ConsumeQueueStore;\nimport org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface;\nimport org.apache.rocketmq.store.queue.CqUnit;\nimport org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.awaitility.Awaitility;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\nimport org.rocksdb.RocksDBException;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class RocksdbTransferOffsetAndCqTest {\n\n    private final String basePath = Paths.get(System.getProperty(\"user.home\"),\n        \"unit-test-store\", UUID.randomUUID().toString().substring(0, 16).toUpperCase()).toString();\n\n    private final String topic = \"topic\";\n    private final String group = \"group\";\n    private final String clientHost = \"clientHost\";\n    private final int queueId = 1;\n\n    private RocksDBConsumerOffsetManager rocksdbConsumerOffsetManager;\n\n    private ConsumerOffsetManager consumerOffsetManager;\n\n    private DefaultMessageStore defaultMessageStore;\n\n    @Mock\n    private BrokerController brokerController;\n\n    @Before\n    public void init() throws IOException {\n        if (notToBeExecuted()) {\n            return;\n        }\n        BrokerConfig brokerConfig = new BrokerConfig();\n        brokerConfig.setConsumerOffsetUpdateVersionStep(10);\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setStorePathRootDir(basePath);\n        messageStoreConfig.setRocksdbCQDoubleWriteEnable(true);\n        Mockito.lenient().when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        Mockito.lenient().when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n\n        defaultMessageStore = new DefaultMessageStore(messageStoreConfig, new BrokerStatsManager(\"for-test\", true), null,\n            brokerConfig, new ConcurrentHashMap<>());\n        defaultMessageStore.loadCheckPoint();\n\n        consumerOffsetManager = new ConsumerOffsetManager(brokerController);\n        consumerOffsetManager.load();\n\n        rocksdbConsumerOffsetManager = new RocksDBConsumerOffsetManager(brokerController);\n    }\n\n    @Test\n    public void testTransferOffset() {\n        if (notToBeExecuted()) {\n            return;\n        }\n\n        for (int i = 0; i < 200; i++) {\n            consumerOffsetManager.commitOffset(clientHost, group, topic, queueId, i);\n        }\n\n        ConcurrentMap<String, ConcurrentMap<Integer, Long>> offsetTable = consumerOffsetManager.getOffsetTable();\n        ConcurrentMap<Integer, Long> map = offsetTable.get(topic + \"@\" + group);\n        Assert.assertTrue(MapUtils.isNotEmpty(map));\n\n        Long offset = map.get(queueId);\n        Assert.assertEquals(199L, (long) offset);\n\n        long offsetDataVersion = consumerOffsetManager.getDataVersion().getCounter().get();\n        Assert.assertEquals(20L, offsetDataVersion);\n\n        consumerOffsetManager.persist();\n\n        boolean loadResult = rocksdbConsumerOffsetManager.load();\n        Assert.assertTrue(loadResult);\n\n        ConcurrentMap<String, ConcurrentMap<Integer, Long>> rocksdbOffsetTable = rocksdbConsumerOffsetManager.getOffsetTable();\n\n        ConcurrentMap<Integer, Long> rocksdbMap = rocksdbOffsetTable.get(topic + \"@\" + group);\n        Assert.assertTrue(MapUtils.isNotEmpty(rocksdbMap));\n\n        Long aLong1 = rocksdbMap.get(queueId);\n        Assert.assertEquals(199L, (long) aLong1);\n\n        long rocksdbOffset = rocksdbConsumerOffsetManager.getDataVersion().getCounter().get();\n        Assert.assertEquals(21L, rocksdbOffset);\n    }\n\n    @Test\n    public void testRocksdbCqWrite() throws RocksDBException {\n        if (notToBeExecuted()) {\n            return;\n        }\n        long startTimestamp = System.currentTimeMillis();\n\n        ConsumeQueueStoreInterface combineConsumeQueueStore = defaultMessageStore.getQueueStore();\n        Assert.assertTrue(combineConsumeQueueStore instanceof CombineConsumeQueueStore);\n        combineConsumeQueueStore.load();\n        combineConsumeQueueStore.recover(false);\n        combineConsumeQueueStore.start();\n\n        RocksDBConsumeQueueStore rocksDBConsumeQueueStore = ((CombineConsumeQueueStore) combineConsumeQueueStore).getRocksDBConsumeQueueStore();\n        ConsumeQueueStore consumeQueueStore = ((CombineConsumeQueueStore) combineConsumeQueueStore).getConsumeQueueStore();\n\n        for (int i = 0; i < 200; i++) {\n            DispatchRequest request = new DispatchRequest(topic, queueId, i, 200, 0, System.currentTimeMillis(), i, \"\", \"\", 0, 0, new HashMap<>());\n            combineConsumeQueueStore.putMessagePositionInfoWrapper(request);\n        }\n\n        ConsumeQueueInterface rocksdbCq = rocksDBConsumeQueueStore.findOrCreateConsumeQueue(topic, queueId);\n        ConsumeQueueInterface fileCq = consumeQueueStore.findOrCreateConsumeQueue(topic, queueId);\n\n        Awaitility.await()\n            .pollInterval(100, TimeUnit.MILLISECONDS)\n            .atMost(3, TimeUnit.SECONDS)\n            .until(() -> rocksdbCq.getMaxOffsetInQueue() == 200);\n        Pair<CqUnit, Long> unit = rocksdbCq.getCqUnitAndStoreTime(100);\n        Pair<CqUnit, Long> unit1 = fileCq.getCqUnitAndStoreTime(100);\n        Assert.assertEquals(unit.getObject1().getPos(), unit1.getObject1().getPos());\n\n        CheckRocksdbCqWriteResult result = ((CombineConsumeQueueStore) combineConsumeQueueStore).doCheckCqWriteProgress(topic, startTimestamp, StoreType.DEFAULT, StoreType.DEFAULT_ROCKSDB);\n        Assert.assertEquals(CheckRocksdbCqWriteResult.CheckStatus.CHECK_OK.getValue(), result.getCheckStatus());\n    }\n\n//    /**\n//     * No need to skip macOS platform.\n//     * @return true if some platform is NOT a good fit for this test case.\n//     */\n    private boolean notToBeExecuted() {\n        return MixAll.isMac();\n    }\n\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/pagecache/ManyMessageTransferTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.pagecache;\n\nimport java.nio.ByteBuffer;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class ManyMessageTransferTest {\n\n    @Test\n    public void ManyMessageTransferBuilderTest() {\n        ByteBuffer byteBuffer = ByteBuffer.allocate(20);\n        byteBuffer.putInt(20);\n        GetMessageResult getMessageResult = new GetMessageResult();\n        ManyMessageTransfer manyMessageTransfer = new ManyMessageTransfer(byteBuffer,getMessageResult);\n    }\n\n    @Test\n    public void ManyMessageTransferPosTest() {\n        ByteBuffer byteBuffer = ByteBuffer.allocate(20);\n        byteBuffer.putInt(20);\n        GetMessageResult getMessageResult = new GetMessageResult();\n        ManyMessageTransfer manyMessageTransfer = new ManyMessageTransfer(byteBuffer,getMessageResult);\n        Assert.assertEquals(manyMessageTransfer.position(),4);\n    }\n\n    @Test\n    public void ManyMessageTransferCountTest() {\n        ByteBuffer byteBuffer = ByteBuffer.allocate(20);\n        byteBuffer.putInt(20);\n        GetMessageResult getMessageResult = new GetMessageResult();\n        ManyMessageTransfer manyMessageTransfer = new ManyMessageTransfer(byteBuffer,getMessageResult);\n\n        Assert.assertEquals(manyMessageTransfer.count(),20);\n\n    }\n\n    @Test\n    public void ManyMessageTransferCloseTest() {\n        ByteBuffer byteBuffer = ByteBuffer.allocate(20);\n        byteBuffer.putInt(20);\n        GetMessageResult getMessageResult = new GetMessageResult();\n        ManyMessageTransfer manyMessageTransfer = new ManyMessageTransfer(byteBuffer,getMessageResult);\n        manyMessageTransfer.close();\n        manyMessageTransfer.deallocate();\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/pagecache/OneMessageTransferTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.pagecache;\n\nimport java.nio.ByteBuffer;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.logfile.DefaultMappedFile;\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class OneMessageTransferTest {\n\n    @Test\n    public void OneMessageTransferTest() {\n        ByteBuffer byteBuffer = ByteBuffer.allocate(20);\n        byteBuffer.putInt(20);\n        SelectMappedBufferResult selectMappedBufferResult = new SelectMappedBufferResult(0,byteBuffer,20,new DefaultMappedFile());\n        OneMessageTransfer manyMessageTransfer = new OneMessageTransfer(byteBuffer,selectMappedBufferResult);\n    }\n\n    @Test\n    public void OneMessageTransferCountTest() {\n        ByteBuffer byteBuffer = ByteBuffer.allocate(20);\n        byteBuffer.putInt(20);\n        SelectMappedBufferResult selectMappedBufferResult = new SelectMappedBufferResult(0,byteBuffer,20,new DefaultMappedFile());\n        OneMessageTransfer manyMessageTransfer = new OneMessageTransfer(byteBuffer,selectMappedBufferResult);\n        Assert.assertEquals(manyMessageTransfer.count(),40);\n    }\n\n    @Test\n    public void OneMessageTransferPosTest() {\n        ByteBuffer byteBuffer = ByteBuffer.allocate(20);\n        byteBuffer.putInt(20);\n        SelectMappedBufferResult selectMappedBufferResult = new SelectMappedBufferResult(0,byteBuffer,20,new DefaultMappedFile());\n        OneMessageTransfer manyMessageTransfer = new OneMessageTransfer(byteBuffer,selectMappedBufferResult);\n        Assert.assertEquals(manyMessageTransfer.position(),8);\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/pagecache/QueryMessageTransferTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.pagecache;\n\nimport org.apache.rocketmq.store.QueryMessageResult;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.nio.ByteBuffer;\nimport java.nio.channels.WritableByteChannel;\nimport java.util.ArrayList;\nimport java.util.Arrays;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.mockito.Mockito.never;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class QueryMessageTransferTest {\n\n    @Mock\n    private WritableByteChannel writableByteChannel;\n\n    @Mock\n    private QueryMessageResult queryMessageResult;\n\n    private QueryMessageTransfer queryMessageTransfer;\n\n    private ByteBuffer byteBufferHeader;\n\n    private ByteBuffer bb1;\n\n    private ByteBuffer bb2;\n\n    @Before\n    public void init() {\n        byteBufferHeader = ByteBuffer.allocate(4);\n        byteBufferHeader.putInt(1);\n        byteBufferHeader.flip();\n\n        bb1 = ByteBuffer.allocate(4);\n        bb1.putInt(2);\n        bb1.flip();\n\n        bb2 = ByteBuffer.allocate(4);\n        bb2.putInt(3);\n        bb2.flip();\n\n        when(queryMessageResult.getMessageBufferList()).thenReturn(Arrays.asList(bb1, bb2));\n\n        queryMessageTransfer = new QueryMessageTransfer(byteBufferHeader, queryMessageResult);\n    }\n\n    @Test\n    public void testPosition_WithHeaderAndMessageBuffers() {\n        byteBufferHeader.position(2);\n        bb1.position(1);\n        bb2.position(3);\n\n        long actual = queryMessageTransfer.position();\n\n        long expected = byteBufferHeader.position() + bb1.position() + bb2.position();\n        assertEquals(expected, actual);\n    }\n\n    @Test\n    public void testPosition_WithHeaderOnly() {\n        byteBufferHeader.position(2);\n\n        when(queryMessageResult.getMessageBufferList()).thenReturn(new ArrayList<>());\n\n        long actual = queryMessageTransfer.position();\n\n        long expected = byteBufferHeader.position();\n        assertEquals(expected, actual);\n    }\n\n    @Test\n    public void testPosition_WithMessageBuffersOnly() {\n        byteBufferHeader.clear();\n        byteBufferHeader.flip();\n\n        bb1.position(1);\n        bb2.position(3);\n\n        long actual = queryMessageTransfer.position();\n\n        long expected = bb1.position() + bb2.position();\n        assertEquals(expected, actual);\n    }\n\n    @Test\n    public void testTransferTo_OnlyHeaderData() throws Exception {\n        bb1.clear();\n        bb2.clear();\n\n        when(writableByteChannel.write(byteBufferHeader)).thenReturn(4);\n\n        long actual = queryMessageTransfer.transferTo(writableByteChannel, 0);\n\n        assertEquals(4, actual);\n        verify(writableByteChannel, times(1)).write(byteBufferHeader);\n        verify(writableByteChannel, never()).write(bb1);\n        verify(writableByteChannel, never()).write(bb2);\n    }\n\n    @Test\n    public void testTransferTo_OnlyMessageBuffersData() throws Exception {\n        byteBufferHeader.clear();\n        byteBufferHeader.flip();\n\n        when(writableByteChannel.write(bb1)).thenReturn(4);\n\n        long actual = queryMessageTransfer.transferTo(writableByteChannel, 0);\n\n        assertEquals(4, actual);\n        verify(writableByteChannel, never()).write(byteBufferHeader);\n        verify(writableByteChannel, times(1)).write(bb1);\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerCacheTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.pop;\n\nimport java.util.Collections;\nimport java.util.Queue;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.awaitility.Awaitility;\nimport org.junit.Assert;\nimport org.junit.Test;\nimport org.mockito.Mockito;\n\nimport static org.mockito.ArgumentMatchers.any;\n\npublic class PopConsumerCacheTest {\n\n    private final String attemptId = \"attemptId\";\n    private final String topicId = \"TopicTest\";\n    private final String groupId = \"GroupTest\";\n    private final int queueId = 2;\n\n    @Test\n    public void consumerRecordsTest() {\n        BrokerConfig brokerConfig = new BrokerConfig();\n        brokerConfig.setPopConsumerKVServiceLog(true);\n        PopConsumerCache.ConsumerRecords consumerRecords =\n            new PopConsumerCache.ConsumerRecords(brokerConfig, groupId, topicId, queueId);\n        Assert.assertNotNull(consumerRecords.toString());\n\n        for (int i = 0; i < 5; i++) {\n            consumerRecords.write(new PopConsumerRecord(i, groupId, topicId, queueId, 0,\n                20000, 100 + i, attemptId));\n        }\n        Assert.assertEquals(100, consumerRecords.getMinOffsetInBuffer());\n        Assert.assertEquals(5, consumerRecords.getInFlightRecordCount());\n\n        for (int i = 0; i < 2; i++) {\n            consumerRecords.delete(new PopConsumerRecord(i, groupId, topicId, queueId, 0,\n                20000, 100 + i, attemptId));\n        }\n        Assert.assertEquals(102, consumerRecords.getMinOffsetInBuffer());\n        Assert.assertEquals(3, consumerRecords.getInFlightRecordCount());\n\n        long bufferTimeout = brokerConfig.getPopCkStayBufferTime();\n        consumerRecords.stageExpiredRecords(bufferTimeout + 2);\n        Assert.assertEquals(1, consumerRecords.getRemoveTreeMap().size());\n        consumerRecords.clearStagedRecords();\n        consumerRecords.stageExpiredRecords(bufferTimeout + 4);\n        Assert.assertEquals(2, consumerRecords.getRemoveTreeMap().size());\n        consumerRecords.clearStagedRecords();\n    }\n\n    @Test\n    public void consumerOffsetTest() throws IllegalAccessException {\n        BrokerController brokerController = Mockito.mock(BrokerController.class);\n        PopConsumerKVStore consumerKVStore = Mockito.mock(PopConsumerRocksdbStore.class);\n        PopConsumerLockService consumerLockService = Mockito.mock(PopConsumerLockService.class);\n        ConsumerOffsetManager consumerOffsetManager = Mockito.mock(ConsumerOffsetManager.class);\n        Mockito.when(brokerController.getBrokerConfig()).thenReturn(new BrokerConfig());\n        Mockito.when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);\n        Mockito.when(consumerLockService.tryLock(groupId, topicId)).thenReturn(true);\n\n        PopConsumerCache consumerCache =\n            new PopConsumerCache(brokerController, consumerKVStore, consumerLockService, null);\n        consumerCache.commitOffset(\"CommitOffsetTest\", groupId, topicId, queueId, 100L);\n        consumerCache.removeRecords(groupId, topicId, queueId);\n\n        AtomicInteger estimateCacheSize = (AtomicInteger) FieldUtils.readField(\n            consumerCache, \"estimateCacheSize\", true);\n        estimateCacheSize.set(2);\n        consumerCache.start();\n        Awaitility.await().until(() -> estimateCacheSize.get() == 0);\n        consumerCache.shutdown();\n    }\n\n    @Test\n    public void consumerCacheTest() {\n        BrokerController brokerController = Mockito.mock(BrokerController.class);\n        PopConsumerKVStore consumerKVStore = Mockito.mock(PopConsumerRocksdbStore.class);\n        PopConsumerLockService consumerLockService = Mockito.mock(PopConsumerLockService.class);\n        Mockito.when(brokerController.getBrokerConfig()).thenReturn(new BrokerConfig());\n\n        PopConsumerCache consumerCache =\n            new PopConsumerCache(brokerController, consumerKVStore, consumerLockService, null);\n        Assert.assertEquals(-1L, consumerCache.getMinOffsetInCache(groupId, topicId, queueId));\n        Assert.assertEquals(0, consumerCache.getPopInFlightMessageCount(groupId, topicId, queueId));\n        Assert.assertEquals(0, consumerCache.getCacheKeySize());\n\n        // write\n        for (int i = 0; i < 3; i++) {\n            PopConsumerRecord record = new PopConsumerRecord(2L, groupId, topicId, queueId,\n                0, 20000, 100 + i, attemptId);\n            Assert.assertEquals(consumerCache.getKey(record), consumerCache.getKey(groupId, topicId, queueId));\n            consumerCache.writeRecords(Collections.singletonList(record));\n        }\n        Assert.assertEquals(100, consumerCache.getMinOffsetInCache(groupId, topicId, queueId));\n        Assert.assertEquals(3, consumerCache.getPopInFlightMessageCount(groupId, topicId, queueId));\n        Assert.assertEquals(1, consumerCache.getCacheKeySize());\n        Assert.assertEquals(3, consumerCache.getCacheSize());\n        Assert.assertFalse(consumerCache.isCacheFull());\n\n        // delete\n        PopConsumerRecord record = new PopConsumerRecord(2L, groupId, topicId, queueId,\n            0, 20000, 100, attemptId);\n        Assert.assertEquals(0, consumerCache.deleteRecords(Collections.singletonList(record)).size());\n        Assert.assertEquals(101, consumerCache.getMinOffsetInCache(groupId, topicId, queueId));\n        Assert.assertEquals(2, consumerCache.getPopInFlightMessageCount(groupId, topicId, queueId));\n        Assert.assertEquals(2, consumerCache.getCacheSize());\n\n        record = new PopConsumerRecord(2L, groupId, topicId, queueId,\n            0, 20000, 104, attemptId);\n        Assert.assertEquals(1, consumerCache.deleteRecords(Collections.singletonList(record)).size());\n        Assert.assertEquals(101, consumerCache.getMinOffsetInCache(groupId, topicId, queueId));\n        Assert.assertEquals(2, consumerCache.getPopInFlightMessageCount(groupId, topicId, queueId));\n\n        // clean expired records\n        Queue<PopConsumerRecord> consumerRecordList = new LinkedBlockingQueue<>();\n        consumerCache.cleanupRecords(consumerRecordList::add);\n        Assert.assertEquals(2, consumerRecordList.size());\n\n        // clean all\n        Mockito.when(consumerLockService.isLockTimeout(any(), any())).thenReturn(true);\n        consumerRecordList.clear();\n        consumerCache.cleanupRecords(consumerRecordList::add);\n        Assert.assertEquals(0, consumerRecordList.size());\n    }\n}"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerContextTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.pop;\n\nimport org.apache.rocketmq.common.constant.ConsumeInitMode;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.junit.Assert;\nimport org.junit.Test;\nimport org.mockito.Mockito;\n\npublic class PopConsumerContextTest {\n\n    @Test\n    public void consumerContextTest() {\n        long popTime = System.currentTimeMillis();\n        PopConsumerContext context = new PopConsumerContext(\"127.0.0.1:6789\",\n            popTime, 20_000, \"GroupId\", true, ConsumeInitMode.MIN, \"attemptId\");\n\n        Assert.assertFalse(context.isFound());\n        Assert.assertEquals(\"127.0.0.1:6789\", context.getClientHost());\n        Assert.assertEquals(popTime, context.getPopTime());\n        Assert.assertEquals(20_000, context.getInvisibleTime());\n        Assert.assertEquals(\"GroupId\", context.getGroupId());\n        Assert.assertTrue(context.isFifo());\n        Assert.assertEquals(\"attemptId\", context.getAttemptId());\n        Assert.assertEquals(0, context.getRestCount());\n\n        GetMessageResult getMessageResult = new GetMessageResult();\n        getMessageResult.setStatus(GetMessageStatus.FOUND);\n        getMessageResult.setMinOffset(10L);\n        getMessageResult.setMaxOffset(20L);\n        getMessageResult.setNextBeginOffset(15L);\n        getMessageResult.addMessage(Mockito.mock(SelectMappedBufferResult.class), 10);\n        getMessageResult.addMessage(Mockito.mock(SelectMappedBufferResult.class), 12);\n        getMessageResult.addMessage(Mockito.mock(SelectMappedBufferResult.class), 13);\n\n        context.addGetMessageResult(getMessageResult,\n            \"TopicId\", 3, PopConsumerRecord.RetryType.NORMAL_TOPIC, 1);\n\n        Assert.assertEquals(3, context.getMessageCount());\n        Assert.assertEquals(\n            getMessageResult.getMaxOffset() - getMessageResult.getNextBeginOffset(), context.getRestCount());\n\n        // check header\n        Assert.assertNotNull(context.toString());\n        Assert.assertEquals(\"0 3 1\", context.getStartOffsetInfo());\n        Assert.assertEquals(\"0 3 10,12,13\", context.getMsgOffsetInfo());\n        Assert.assertNotNull(context.getOrderCountInfoBuilder());\n        Assert.assertEquals(\"\", context.getOrderCountInfo());\n\n        Assert.assertEquals(1, context.getGetMessageResultList().size());\n        Assert.assertEquals(3, context.getPopConsumerRecordList().size());\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerLockServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.pop;\n\nimport java.lang.reflect.Field;\nimport java.util.Map;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.common.PopAckConstants;\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class PopConsumerLockServiceTest {\n\n    @Test\n    @SuppressWarnings(\"unchecked\")\n    public void consumerLockTest() throws NoSuchFieldException, IllegalAccessException {\n        String groupId = \"groupId\";\n        String topicId = \"topicId\";\n\n        PopConsumerLockService lockService =\n            new PopConsumerLockService(TimeUnit.MINUTES.toMillis(2));\n\n        Assert.assertTrue(lockService.tryLock(groupId, topicId));\n        Assert.assertFalse(lockService.tryLock(groupId, topicId));\n        lockService.unlock(groupId, topicId);\n\n        Assert.assertTrue(lockService.tryLock(groupId, topicId));\n        Assert.assertFalse(lockService.tryLock(groupId, topicId));\n        Assert.assertFalse(lockService.isLockTimeout(groupId, topicId));\n        lockService.removeTimeout();\n\n        // set expired\n        Field field = PopConsumerLockService.class.getDeclaredField(\"lockTable\");\n        field.setAccessible(true);\n        Map<String, PopConsumerLockService.TimedLock> table =\n            (Map<String, PopConsumerLockService.TimedLock>) field.get(lockService);\n\n        Field lockTime = PopConsumerLockService.TimedLock.class.getDeclaredField(\"lockTime\");\n        lockTime.setAccessible(true);\n        lockTime.set(table.get(groupId + PopAckConstants.SPLIT + topicId),\n            System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(3));\n        lockService.removeTimeout();\n\n        Assert.assertEquals(0, table.size());\n    }\n}"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerRecordTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.pop;\n\nimport java.util.UUID;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class PopConsumerRecordTest {\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);\n\n    @Test\n    public void retryCodeTest() {\n        Assert.assertEquals(\"NORMAL_TOPIC code should be 0\",\n            0, PopConsumerRecord.RetryType.NORMAL_TOPIC.getCode());\n        Assert.assertEquals(\"RETRY_TOPIC code should be 1\",\n            1, PopConsumerRecord.RetryType.RETRY_TOPIC_V1.getCode());\n        Assert.assertEquals(\"RETRY_TOPIC_V2 code should be 2\",\n            2, PopConsumerRecord.RetryType.RETRY_TOPIC_V2.getCode());\n    }\n\n    @Test\n    public void deliveryRecordSerializeTest() {\n        PopConsumerRecord consumerRecord = new PopConsumerRecord();\n        consumerRecord.setPopTime(System.currentTimeMillis());\n        consumerRecord.setGroupId(\"GroupId\");\n        consumerRecord.setTopicId(\"TopicId\");\n        consumerRecord.setQueueId(3);\n        consumerRecord.setRetryFlag(PopConsumerRecord.RetryType.RETRY_TOPIC_V1.getCode());\n        consumerRecord.setInvisibleTime(20);\n        consumerRecord.setOffset(100);\n        consumerRecord.setAttemptTimes(2);\n        consumerRecord.setAttemptId(UUID.randomUUID().toString().toUpperCase());\n\n        Assert.assertTrue(consumerRecord.isRetry());\n        Assert.assertEquals(consumerRecord.getPopTime() + consumerRecord.getInvisibleTime(),\n            consumerRecord.getVisibilityTimeout());\n        Assert.assertEquals(8 + \"GroupId\".length() + 1 + \"TopicId\".length() + 1 + 4 + 1 + 8,\n            consumerRecord.getKeyBytes().length);\n        log.info(\"ConsumerRecord={}\", consumerRecord.toString());\n\n        PopConsumerRecord decodeRecord = PopConsumerRecord.decode(consumerRecord.getValueBytes());\n        PopConsumerRecord consumerRecord2 = new PopConsumerRecord(consumerRecord.getPopTime(),\n            consumerRecord.getGroupId(), consumerRecord.getTopicId(), consumerRecord.getQueueId(),\n            consumerRecord.getRetryFlag(), consumerRecord.getInvisibleTime(),\n            consumerRecord.getOffset(), consumerRecord.getAttemptId());\n        Assert.assertEquals(decodeRecord.getPopTime(), consumerRecord2.getPopTime());\n        Assert.assertEquals(decodeRecord.getGroupId(), consumerRecord2.getGroupId());\n        Assert.assertEquals(decodeRecord.getTopicId(), consumerRecord2.getTopicId());\n        Assert.assertEquals(decodeRecord.getQueueId(), consumerRecord2.getQueueId());\n        Assert.assertEquals(decodeRecord.getRetryFlag(), consumerRecord2.getRetryFlag());\n        Assert.assertEquals(decodeRecord.getInvisibleTime(), consumerRecord2.getInvisibleTime());\n        Assert.assertEquals(decodeRecord.getOffset(), consumerRecord2.getOffset());\n        Assert.assertEquals(0, consumerRecord2.getAttemptTimes());\n        Assert.assertEquals(decodeRecord.getAttemptId(), consumerRecord2.getAttemptId());\n    }\n\n    @Test\n    public void testSuspendFlagInitialization() {\n        // Test constructor without suspend flag (should default to false)\n        PopConsumerRecord record1 = new PopConsumerRecord(\n            System.currentTimeMillis(), \"test-group\", \"test-topic\", 0, 0, 30000L, 100L, \"attempt-id\");\n        Assert.assertFalse(\"Suspend flag should default to false\", record1.isSuspend());\n\n        // Test constructor with suspend flag set to true\n        PopConsumerRecord record2 = new PopConsumerRecord(\n            System.currentTimeMillis(), \"test-group\", \"test-topic\", 0, 0, 30000L, 100L, \"attempt-id\", true);\n        Assert.assertTrue(\"Suspend flag should be true\", record2.isSuspend());\n\n        // Test constructor with suspend flag set to false\n        PopConsumerRecord record3 = new PopConsumerRecord(\n            System.currentTimeMillis(), \"test-group\", \"test-topic\", 0, 0, 30000L, 100L, \"attempt-id\", false);\n        Assert.assertFalse(\"Suspend flag should be false\", record3.isSuspend());\n    }\n\n    @Test\n    public void testSuspendFlagSerialization() {\n        // Test serialization/deserialization with suspend flag\n        PopConsumerRecord originalRecord = new PopConsumerRecord(\n            1234567890L, \"test-group\", \"test-topic\", 0, 0, 30000L, 100L, \"attempt-id\", true);\n\n        byte[] serialized = originalRecord.getValueBytes();\n        PopConsumerRecord deserialized = PopConsumerRecord.decode(serialized);\n\n        Assert.assertTrue(\"Deserialized record should have suspend flag true\", deserialized.isSuspend());\n        Assert.assertEquals(\"Other fields should match\", originalRecord.getGroupId(), deserialized.getGroupId());\n        Assert.assertEquals(\"Other fields should match\", originalRecord.getTopicId(), deserialized.getTopicId());\n        Assert.assertEquals(\"Other fields should match\", originalRecord.getOffset(), deserialized.getOffset());\n    }\n\n    @Test\n    public void testSuspendFlagGetterSetter() {\n        PopConsumerRecord record = new PopConsumerRecord();\n\n        // Test initial value\n        Assert.assertFalse(\"Initial suspend value should be false\", record.isSuspend());\n\n        // Test setter\n        record.setSuspend(true);\n        Assert.assertTrue(\"After setting to true, should be true\", record.isSuspend());\n\n        record.setSuspend(false);\n        Assert.assertFalse(\"After setting to false, should be false\", record.isSuspend());\n    }\n\n    @Test\n    public void testSuspendInToString() {\n        PopConsumerRecord record = new PopConsumerRecord(\n            1234567890L, \"test-group\", \"test-topic\", 0, 0, 30000L, 100L, \"attempt-id\", true);\n\n        String toString = record.toString();\n        Assert.assertTrue(\"toString should include suspend information\", toString.contains(\"suspend=true\"));\n\n        PopConsumerRecord record2 = new PopConsumerRecord(\n            1234567890L, \"test-group\", \"test-topic\", 0, 0, 30000L, 100L, \"attempt-id\", false);\n\n        String toString2 = record2.toString();\n        Assert.assertTrue(\"toString should include suspend information\", toString2.contains(\"suspend=false\"));\n    }\n\n    @Test\n    public void testSuspendFlagSerializationWithFalse() {\n        // Test serialization/deserialization with suspend flag set to false\n        PopConsumerRecord originalRecord = new PopConsumerRecord(\n            1234567890L, \"test-group\", \"test-topic\", 0, 0, 30000L, 100L, \"attempt-id\", false);\n\n        byte[] serialized = originalRecord.getValueBytes();\n        PopConsumerRecord deserialized = PopConsumerRecord.decode(serialized);\n\n        Assert.assertFalse(\"Deserialized record should have suspend flag false\", deserialized.isSuspend());\n        Assert.assertEquals(\"GroupId should match\", originalRecord.getGroupId(), deserialized.getGroupId());\n        Assert.assertEquals(\"TopicId should match\", originalRecord.getTopicId(), deserialized.getTopicId());\n        Assert.assertEquals(\"Offset should match\", originalRecord.getOffset(), deserialized.getOffset());\n        Assert.assertEquals(\"PopTime should match\", originalRecord.getPopTime(), deserialized.getPopTime());\n        Assert.assertEquals(\"QueueId should match\", originalRecord.getQueueId(), deserialized.getQueueId());\n        Assert.assertEquals(\"InvisibleTime should match\", originalRecord.getInvisibleTime(), deserialized.getInvisibleTime());\n        Assert.assertEquals(\"RetryFlag should match\", originalRecord.getRetryFlag(), deserialized.getRetryFlag());\n        Assert.assertEquals(\"AttemptId should match\", originalRecord.getAttemptId(), deserialized.getAttemptId());\n    }\n\n    @Test\n    public void testSuspendFlagJSONSerializationCompleteness() {\n        // Test complete serialization/deserialization with all fields including suspend\n        long popTime = System.currentTimeMillis();\n        String groupId = \"test-group\";\n        String topicId = \"test-topic\";\n        int queueId = 1;\n        int retryFlag = PopConsumerRecord.RetryType.RETRY_TOPIC_V2.getCode();\n        long invisibleTime = 30000L;\n        long offset = 100L;\n        String attemptId = UUID.randomUUID().toString().toUpperCase();\n\n        // Test with suspend = true\n        PopConsumerRecord recordWithSuspend = new PopConsumerRecord(\n            popTime, groupId, topicId, queueId, retryFlag, invisibleTime, offset, attemptId, true);\n        recordWithSuspend.setAttemptTimes(3);\n\n        byte[] serialized = recordWithSuspend.getValueBytes();\n        PopConsumerRecord deserialized = PopConsumerRecord.decode(serialized);\n\n        Assert.assertTrue(\"Suspend flag should be true\", deserialized.isSuspend());\n        Assert.assertEquals(\"PopTime should match\", popTime, deserialized.getPopTime());\n        Assert.assertEquals(\"GroupId should match\", groupId, deserialized.getGroupId());\n        Assert.assertEquals(\"TopicId should match\", topicId, deserialized.getTopicId());\n        Assert.assertEquals(\"QueueId should match\", queueId, deserialized.getQueueId());\n        Assert.assertEquals(\"RetryFlag should match\", retryFlag, deserialized.getRetryFlag());\n        Assert.assertEquals(\"InvisibleTime should match\", invisibleTime, deserialized.getInvisibleTime());\n        Assert.assertEquals(\"Offset should match\", offset, deserialized.getOffset());\n        Assert.assertEquals(\"AttemptTimes should match\", 3, deserialized.getAttemptTimes());\n        Assert.assertEquals(\"AttemptId should match\", attemptId, deserialized.getAttemptId());\n\n        // Test with suspend = false\n        PopConsumerRecord recordWithoutSuspend = new PopConsumerRecord(\n            popTime, groupId, topicId, queueId, retryFlag, invisibleTime, offset, attemptId, false);\n        recordWithoutSuspend.setAttemptTimes(3);\n\n        serialized = recordWithoutSuspend.getValueBytes();\n        deserialized = PopConsumerRecord.decode(serialized);\n\n        Assert.assertFalse(\"Suspend flag should be false\", deserialized.isSuspend());\n        Assert.assertEquals(\"PopTime should match\", popTime, deserialized.getPopTime());\n        Assert.assertEquals(\"GroupId should match\", groupId, deserialized.getGroupId());\n        Assert.assertEquals(\"TopicId should match\", topicId, deserialized.getTopicId());\n        Assert.assertEquals(\"QueueId should match\", queueId, deserialized.getQueueId());\n        Assert.assertEquals(\"RetryFlag should match\", retryFlag, deserialized.getRetryFlag());\n        Assert.assertEquals(\"InvisibleTime should match\", invisibleTime, deserialized.getInvisibleTime());\n        Assert.assertEquals(\"Offset should match\", offset, deserialized.getOffset());\n        Assert.assertEquals(\"AttemptTimes should match\", 3, deserialized.getAttemptTimes());\n        Assert.assertEquals(\"AttemptId should match\", attemptId, deserialized.getAttemptId());\n    }\n\n    @Test\n    public void testSuspendFlagDefaultValueInNoArgConstructor() {\n        // Test that no-arg constructor defaults suspend to false\n        PopConsumerRecord record = new PopConsumerRecord();\n        Assert.assertFalse(\"No-arg constructor should default suspend to false\", record.isSuspend());\n\n        // Set all fields manually\n        record.setPopTime(System.currentTimeMillis());\n        record.setGroupId(\"test-group\");\n        record.setTopicId(\"test-topic\");\n        record.setQueueId(0);\n        record.setRetryFlag(0);\n        record.setInvisibleTime(30000L);\n        record.setOffset(100L);\n        record.setAttemptId(\"attempt-id\");\n        record.setSuspend(true);\n\n        Assert.assertTrue(\"After setting suspend to true, should be true\", record.isSuspend());\n\n        // Serialize and deserialize to verify\n        byte[] serialized = record.getValueBytes();\n        PopConsumerRecord deserialized = PopConsumerRecord.decode(serialized);\n        Assert.assertTrue(\"Deserialized record should preserve suspend=true\", deserialized.isSuspend());\n    }\n\n    @Test\n    public void testSuspendFlagInDeliveryRecordSerializeTest() {\n        // Enhance existing deliveryRecordSerializeTest to include suspend flag\n        PopConsumerRecord consumerRecord = new PopConsumerRecord();\n        consumerRecord.setPopTime(System.currentTimeMillis());\n        consumerRecord.setGroupId(\"GroupId\");\n        consumerRecord.setTopicId(\"TopicId\");\n        consumerRecord.setQueueId(3);\n        consumerRecord.setRetryFlag(PopConsumerRecord.RetryType.RETRY_TOPIC_V1.getCode());\n        consumerRecord.setInvisibleTime(20);\n        consumerRecord.setOffset(100);\n        consumerRecord.setAttemptTimes(2);\n        consumerRecord.setAttemptId(UUID.randomUUID().toString().toUpperCase());\n        consumerRecord.setSuspend(true);\n\n        PopConsumerRecord decodeRecord = PopConsumerRecord.decode(consumerRecord.getValueBytes());\n        Assert.assertTrue(\"Decoded record should preserve suspend flag\", decodeRecord.isSuspend());\n        Assert.assertEquals(\"Suspend flag should match\", consumerRecord.isSuspend(), decodeRecord.isSuspend());\n\n        // Test with suspend = false\n        consumerRecord.setSuspend(false);\n        decodeRecord = PopConsumerRecord.decode(consumerRecord.getValueBytes());\n        Assert.assertFalse(\"Decoded record should preserve suspend=false\", decodeRecord.isSuspend());\n        Assert.assertEquals(\"Suspend flag should match\", consumerRecord.isSuspend(), decodeRecord.isSuspend());\n    }\n}"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerRocksdbStoreTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.pop;\n\nimport com.google.common.base.Stopwatch;\nimport java.io.File;\nimport java.io.IOException;\nimport java.lang.reflect.Field;\nimport java.nio.file.Paths;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.UUID;\nimport java.util.concurrent.TimeUnit;\nimport java.util.stream.Collectors;\nimport java.util.stream.IntStream;\nimport org.apache.commons.io.FileUtils;\nimport org.apache.rocketmq.common.config.AbstractRocksDBStorage;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.tieredstore.util.MessageStoreUtil;\nimport org.junit.Assert;\nimport org.junit.Ignore;\nimport org.junit.Test;\nimport org.rocksdb.RocksDB;\nimport org.rocksdb.RocksIterator;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\npublic class PopConsumerRocksdbStoreTest {\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);\n    private static final String CONSUMER_STORE_PATH = \"consumer_rocksdb\";\n\n    public static String getRandomStorePath() {\n        return Paths.get(System.getProperty(\"user.home\"), \"store_test\", CONSUMER_STORE_PATH,\n            UUID.randomUUID().toString().replace(\"-\", \"\").toUpperCase().substring(0, 16)).toString();\n    }\n\n    public static void deleteStoreDirectory(String storePath) {\n        try {\n            FileUtils.deleteDirectory(new File(storePath));\n        } catch (IOException e) {\n            log.error(\"Delete store directory failed, filePath: {}\", storePath, e);\n        }\n    }\n\n    public static PopConsumerRecord getConsumerRecord() {\n        return new PopConsumerRecord(1L, \"GroupTest\", \"TopicTest\", 2,\n            PopConsumerRecord.RetryType.NORMAL_TOPIC.getCode(), TimeUnit.SECONDS.toMillis(20), 100L, \"AttemptId\");\n    }\n\n    @Test\n    public void rocksdbStoreWriteDeleteTest() {\n        String filePath = getRandomStorePath();\n        PopConsumerKVStore consumerStore = new PopConsumerRocksdbStore(filePath);\n        Assert.assertEquals(filePath, consumerStore.getFilePath());\n\n        consumerStore.start();\n        consumerStore.writeRecords(IntStream.range(0, 3).boxed()\n            .flatMap(i ->\n                IntStream.range(0, 5).mapToObj(j -> {\n                    PopConsumerRecord consumerRecord = getConsumerRecord();\n                    consumerRecord.setPopTime(j);\n                    consumerRecord.setQueueId(i);\n                    consumerRecord.setOffset(100L + j);\n                    return consumerRecord;\n                })\n            )\n            .collect(Collectors.toList()));\n        consumerStore.deleteRecords(IntStream.range(0, 2).boxed()\n            .flatMap(i ->\n                IntStream.range(0, 5).mapToObj(j -> {\n                    PopConsumerRecord consumerRecord = getConsumerRecord();\n                    consumerRecord.setPopTime(j);\n                    consumerRecord.setQueueId(i);\n                    consumerRecord.setOffset(100L + j);\n                    return consumerRecord;\n                })\n            )\n            .collect(Collectors.toList()));\n\n        List<PopConsumerRecord> consumerRecords =\n            consumerStore.scanExpiredRecords(0, 20002, 2);\n        Assert.assertEquals(2, consumerRecords.size());\n        consumerStore.deleteRecords(consumerRecords);\n\n        consumerRecords = consumerStore.scanExpiredRecords(0, 20003, 2);\n        Assert.assertEquals(1, consumerRecords.size());\n        consumerStore.deleteRecords(consumerRecords);\n\n        consumerRecords = consumerStore.scanExpiredRecords(0, 20005, 3);\n        Assert.assertEquals(2, consumerRecords.size());\n\n        consumerStore.shutdown();\n        deleteStoreDirectory(filePath);\n    }\n\n    private long getDirectorySizeRecursive(File directory) {\n        long size = 0;\n        File[] files = directory.listFiles();\n        if (files != null) {\n            for (File file : files) {\n                if (file.isFile()) {\n                    size += file.length();\n                } else if (file.isDirectory()) {\n                    size += getDirectorySizeRecursive(file);\n                }\n            }\n        }\n        return size;\n    }\n\n    @Test\n    @Ignore\n    @SuppressWarnings(\"ConstantValue\")\n    public void tombstoneDeletionTest() throws IllegalAccessException, NoSuchFieldException {\n        PopConsumerRocksdbStore rocksdbStore = new PopConsumerRocksdbStore(getRandomStorePath());\n        rocksdbStore.start();\n\n        int iterCount = 1000 * 1000;\n        boolean useSeekFirstDelete = false;\n        Field dbField = AbstractRocksDBStorage.class.getDeclaredField(\"db\");\n        dbField.setAccessible(true);\n        RocksDB rocksDB = (RocksDB) dbField.get(rocksdbStore);\n\n        long currentTime = 0L;\n        Stopwatch stopwatch = Stopwatch.createStarted();\n        for (int i = 0; i < iterCount; i++) {\n            List<PopConsumerRecord> records = new ArrayList<>();\n            for (int j = 0; j < 1000; j++) {\n                PopConsumerRecord record = getConsumerRecord();\n                record.setPopTime((long) i * iterCount + j);\n                record.setGroupId(\"GroupTest\");\n                record.setTopicId(\"TopicTest\");\n                record.setQueueId(i % 10);\n                record.setRetryFlag(0);\n                record.setInvisibleTime(TimeUnit.SECONDS.toMillis(30));\n                record.setOffset(i);\n                records.add(record);\n            }\n            rocksdbStore.writeRecords(records);\n\n            long start = stopwatch.elapsed(TimeUnit.MILLISECONDS);\n            List<PopConsumerRecord> deleteList = new ArrayList<>();\n            if (useSeekFirstDelete) {\n                try (RocksIterator iterator = rocksDB.newIterator(rocksdbStore.columnFamilyHandle)) {\n                    iterator.seekToFirst();\n                    if (i % 10 == 0) {\n                        long fileSize = getDirectorySizeRecursive(new File(rocksdbStore.getFilePath()));\n                        log.info(\"DirectorySize={}, Cost={}ms\",\n                            MessageStoreUtil.toHumanReadable(fileSize), stopwatch.elapsed(TimeUnit.MILLISECONDS) - start);\n                    }\n                    while (iterator.isValid() && deleteList.size() < 1024) {\n                        deleteList.add(PopConsumerRecord.decode(iterator.value()));\n                        iterator.next();\n                    }\n                }\n            } else {\n                long upper = System.currentTimeMillis();\n                deleteList = rocksdbStore.scanExpiredRecords(currentTime, upper, 800);\n                if (!deleteList.isEmpty()) {\n                    currentTime = deleteList.get(deleteList.size() - 1).getVisibilityTimeout();\n                }\n                long scanCost = stopwatch.elapsed(TimeUnit.MILLISECONDS) - start;\n                if (i % 100 == 0) {\n                    long fileSize = getDirectorySizeRecursive(new File(rocksdbStore.getFilePath()));\n                    long seekTime = stopwatch.elapsed(TimeUnit.MILLISECONDS);\n                    try (RocksIterator iterator = rocksDB.newIterator(rocksdbStore.columnFamilyHandle)) {\n                        iterator.seekToFirst();\n                    }\n                    log.info(\"DirectorySize={}, Cost={}ms, SeekFirstCost={}ms\", MessageStoreUtil.toHumanReadable(fileSize),\n                        scanCost, stopwatch.elapsed(TimeUnit.MILLISECONDS) - seekTime);\n                }\n            }\n            rocksdbStore.deleteRecords(deleteList);\n        }\n        rocksdbStore.shutdown();\n        deleteStoreDirectory(rocksdbStore.getFilePath());\n    }\n}"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.pop;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.net.InetSocketAddress;\nimport java.nio.ByteBuffer;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.UUID;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.stream.Collectors;\nimport java.util.stream.IntStream;\nimport org.apache.commons.io.FileUtils;\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.commons.lang3.tuple.Triple;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.failover.EscapeBridge;\nimport org.apache.rocketmq.broker.longpolling.PopLongPollingService;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.broker.pop.orderly.ConsumerOrderInfoManager;\nimport org.apache.rocketmq.broker.processor.PopMessageProcessor;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.ConsumeInitMode;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.AppendMessageResult;\nimport org.apache.rocketmq.store.AppendMessageStatus;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.ArgumentCaptor;\nimport org.mockito.Mockito;\n\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\n\npublic class PopConsumerServiceTest {\n\n    private final String clientHost = \"127.0.0.1:8888\";\n    private final String groupId = \"groupId\";\n    private final String topicId = \"topicId\";\n    private final int queueId = 2;\n    private final String attemptId = UUID.randomUUID().toString().toUpperCase();\n    private final String filePath = PopConsumerRocksdbStoreTest.getRandomStorePath();\n\n    private BrokerController brokerController;\n    private PopConsumerService consumerService;\n\n    @Before\n    public void init() throws IOException {\n        BrokerConfig brokerConfig = new BrokerConfig();\n        brokerConfig.setEnablePopLog(true);\n        brokerConfig.setEnablePopBufferMerge(true);\n        brokerConfig.setEnablePopMessageThreshold(true);\n        brokerConfig.setPopInflightMessageThreshold(100);\n        brokerConfig.setPopConsumerKVServiceLog(true);\n        brokerConfig.setEnableRetryTopicV2(true);\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setStorePathRootDir(filePath);\n\n        TopicConfigManager topicConfigManager = Mockito.mock(TopicConfigManager.class);\n        SubscriptionGroupManager subscriptionGroupManager = Mockito.mock(SubscriptionGroupManager.class);\n        ConsumerOffsetManager consumerOffsetManager = Mockito.mock(ConsumerOffsetManager.class);\n        PopMessageProcessor popMessageProcessor = Mockito.mock(PopMessageProcessor.class);\n        PopLongPollingService popLongPollingService = Mockito.mock(PopLongPollingService.class);\n        ConsumerOrderInfoManager consumerOrderInfoManager = Mockito.mock(ConsumerOrderInfoManager.class);\n\n        brokerController = Mockito.mock(BrokerController.class);\n        Mockito.when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        Mockito.when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);\n        Mockito.when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);\n        Mockito.when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n        Mockito.when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);\n        Mockito.when(brokerController.getPopMessageProcessor()).thenReturn(popMessageProcessor);\n        Mockito.when(popMessageProcessor.getPopLongPollingService()).thenReturn(popLongPollingService);\n        Mockito.when(brokerController.getConsumerOrderInfoManager()).thenReturn(consumerOrderInfoManager);\n\n        consumerService = new PopConsumerService(brokerController);\n    }\n\n    @After\n    public void shutdown() throws IOException {\n        FileUtils.deleteDirectory(new File(filePath));\n    }\n\n    public PopConsumerRecord getConsumerTestRecord() {\n        PopConsumerRecord popConsumerRecord = new PopConsumerRecord();\n        popConsumerRecord.setPopTime(System.currentTimeMillis());\n        popConsumerRecord.setGroupId(groupId);\n        popConsumerRecord.setTopicId(topicId);\n        popConsumerRecord.setQueueId(queueId);\n        popConsumerRecord.setRetryFlag(PopConsumerRecord.RetryType.NORMAL_TOPIC.getCode());\n        popConsumerRecord.setAttemptTimes(0);\n        popConsumerRecord.setInvisibleTime(TimeUnit.SECONDS.toMillis(20));\n        popConsumerRecord.setAttemptId(UUID.randomUUID().toString().toUpperCase());\n        return popConsumerRecord;\n    }\n\n    @Test\n    public void isPopShouldStopTest() throws IllegalAccessException {\n        Assert.assertFalse(consumerService.isPopShouldStop(groupId, topicId, queueId));\n        PopConsumerCache consumerCache = (PopConsumerCache) FieldUtils.readField(\n            consumerService, \"popConsumerCache\", true);\n        for (int i = 0; i < 100; i++) {\n            PopConsumerRecord record = getConsumerTestRecord();\n            record.setOffset(i);\n            consumerCache.writeRecords(Collections.singletonList(record));\n        }\n        Assert.assertTrue(consumerService.isPopShouldStop(groupId, topicId, queueId));\n    }\n\n    @Test\n    public void pendingFilterCountTest() throws ConsumeQueueException {\n        MessageStore messageStore = Mockito.mock(MessageStore.class);\n        Mockito.when(messageStore.getMaxOffsetInQueue(topicId, queueId)).thenReturn(100L);\n        Mockito.when(brokerController.getMessageStore()).thenReturn(messageStore);\n        ConsumerOffsetManager consumerOffsetManager = brokerController.getConsumerOffsetManager();\n        Mockito.when(consumerOffsetManager.queryOffset(groupId, topicId, queueId)).thenReturn(20L);\n        Assert.assertEquals(consumerService.getPendingFilterCount(groupId, topicId, queueId), 80L);\n    }\n\n    private MessageExt getMessageExt() {\n        MessageExt messageExt = new MessageExt();\n        messageExt.setTopic(topicId);\n        messageExt.setQueueId(queueId);\n        messageExt.setBody(new byte[128]);\n        messageExt.setBornHost(new InetSocketAddress(\"127.0.0.1\", 8080));\n        messageExt.setStoreHost(new InetSocketAddress(\"127.0.0.1\", 8080));\n        messageExt.putUserProperty(\"Key\", \"Value\");\n        return messageExt;\n    }\n\n    @Test\n    public void recodeRetryMessageTest() throws Exception {\n        GetMessageResult getMessageResult = new GetMessageResult();\n        getMessageResult.setStatus(GetMessageStatus.FOUND);\n\n        // result is empty\n        SelectMappedBufferResult bufferResult = new SelectMappedBufferResult(\n            0, ByteBuffer.allocate(10), 10, null);\n        getMessageResult.addMessage(bufferResult);\n        getMessageResult.getMessageMapedList().clear();\n        GetMessageResult result = consumerService.recodeRetryMessage(\n            getMessageResult, topicId, 0, 100, 200);\n        Assert.assertEquals(0, result.getMessageMapedList().size());\n\n        ByteBuffer buffer = ByteBuffer.wrap(\n            MessageDecoder.encode(getMessageExt(), false));\n        getMessageResult = new GetMessageResult();\n        getMessageResult.setStatus(GetMessageStatus.FOUND);\n        getMessageResult.addMessage(new SelectMappedBufferResult(\n            0, buffer, buffer.remaining(), null));\n        result = consumerService.recodeRetryMessage(\n            getMessageResult, topicId, 0, 100, 200);\n        Assert.assertNotNull(result);\n        Assert.assertEquals(1, result.getMessageMapedList().size());\n    }\n\n    @Test\n    public void addGetMessageResultTest() {\n        PopConsumerContext context = new PopConsumerContext(\n            clientHost, System.currentTimeMillis(), 20000, groupId, false, ConsumeInitMode.MIN, attemptId);\n        GetMessageResult result = new GetMessageResult();\n        result.setStatus(GetMessageStatus.FOUND);\n        result.getMessageQueueOffset().add(100L);\n        consumerService.handleGetMessageResult(\n            context, result, topicId, queueId, PopConsumerRecord.RetryType.NORMAL_TOPIC, 100);\n        Assert.assertEquals(1, context.getGetMessageResultList().size());\n    }\n\n    @Test\n    public void getMessageAsyncTest() throws Exception {\n        MessageStore messageStore = Mockito.mock(MessageStore.class);\n        Mockito.when(brokerController.getMessageStore()).thenReturn(messageStore);\n        Mockito.when(messageStore.getMessageAsync(groupId, topicId, queueId, 0, 10, null))\n            .thenReturn(CompletableFuture.completedFuture(null));\n        GetMessageResult getMessageResult = consumerService.getMessageAsync(\n            \"127.0.0.1:8888\", groupId, topicId, queueId, 0, 10, null).join();\n        Assert.assertNull(getMessageResult);\n\n        // success when first get message\n        GetMessageResult firstGetMessageResult = new GetMessageResult();\n        firstGetMessageResult.setStatus(GetMessageStatus.FOUND);\n        Mockito.when(messageStore.getMessageAsync(groupId, topicId, queueId, 0, 10, null))\n            .thenReturn(CompletableFuture.completedFuture(firstGetMessageResult));\n        getMessageResult = consumerService.getMessageAsync(\n            \"127.0.0.1:8888\", groupId, topicId, queueId, 0, 10, null).join();\n        Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());\n\n        // reset offset from server\n        firstGetMessageResult.setStatus(GetMessageStatus.OFFSET_FOUND_NULL);\n        firstGetMessageResult.setNextBeginOffset(25);\n        GetMessageResult resetGetMessageResult = new GetMessageResult();\n        resetGetMessageResult.setStatus(GetMessageStatus.FOUND);\n        Mockito.when(messageStore.getMessageAsync(groupId, topicId, queueId, 25, 10, null))\n            .thenReturn(CompletableFuture.completedFuture(resetGetMessageResult));\n        getMessageResult = consumerService.getMessageAsync(\n            \"127.0.0.1:8888\", groupId, topicId, queueId, 0, 10, null).join();\n        Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());\n\n        // fifo block\n        PopConsumerContext context = new PopConsumerContext(\n            clientHost, System.currentTimeMillis(), 20000, groupId, false, ConsumeInitMode.MIN, attemptId);\n        consumerService.setFifoBlocked(context, groupId, topicId, queueId, Collections.singletonList(100L), resetGetMessageResult);\n        Mockito.when(brokerController.getConsumerOrderInfoManager()\n            .checkBlock(anyString(), anyString(), anyString(), anyInt(), anyLong())).thenReturn(true);\n        Assert.assertTrue(consumerService.isFifoBlocked(context, groupId, topicId, queueId));\n\n        // get message async normal\n        CompletableFuture<PopConsumerContext> future = CompletableFuture.completedFuture(context);\n        Assert.assertEquals(0L, consumerService.getMessageAsync(future, clientHost, groupId, topicId, queueId,\n            10, null, PopConsumerRecord.RetryType.NORMAL_TOPIC).join().getRestCount());\n\n        // get message result full, no need get again\n        for (int i = 0; i < 10; i++) {\n            ByteBuffer buffer = ByteBuffer.wrap(MessageDecoder.encode(getMessageExt(), false));\n            getMessageResult.addMessage(new SelectMappedBufferResult(\n                0, buffer, buffer.remaining(), null), i);\n        }\n        context.addGetMessageResult(getMessageResult, topicId, queueId, PopConsumerRecord.RetryType.NORMAL_TOPIC, 0);\n\n        Mockito.when(brokerController.getMessageStore().getMaxOffsetInQueue(topicId, queueId)).thenReturn(100L);\n        Mockito.when(brokerController.getConsumerOffsetManager().queryOffset(groupId, topicId, queueId)).thenReturn(0L);\n        Assert.assertEquals(100L, consumerService.getMessageAsync(future, clientHost, groupId, topicId, queueId,\n            10, null, PopConsumerRecord.RetryType.NORMAL_TOPIC).join().getRestCount());\n\n        // fifo block test\n        context = new PopConsumerContext(\n            clientHost, System.currentTimeMillis(), 20000, groupId, true, ConsumeInitMode.MIN, attemptId);\n        future = CompletableFuture.completedFuture(context);\n        Assert.assertEquals(0L, consumerService.getMessageAsync(future, clientHost, groupId, topicId, queueId,\n            10, null, PopConsumerRecord.RetryType.NORMAL_TOPIC).join().getRestCount());\n    }\n\n    @Test\n    public void popAsyncTest() {\n        PopConsumerService consumerServiceSpy = Mockito.spy(consumerService);\n        TopicConfigManager topicConfigManager = Mockito.mock(TopicConfigManager.class);\n        Mockito.when(topicConfigManager.selectTopicConfig(topicId)).thenReturn(new TopicConfig(\n            topicId, 2, 2, PermName.PERM_READ | PermName.PERM_WRITE, 0));\n        Mockito.when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);\n\n        String[] retryTopic = new String[] {\n            KeyBuilder.buildPopRetryTopicV1(topicId, groupId),\n            KeyBuilder.buildPopRetryTopicV2(topicId, groupId)\n        };\n\n        for (String retry : retryTopic) {\n            GetMessageResult getMessageResult = new GetMessageResult();\n            getMessageResult.setStatus(GetMessageStatus.NO_MATCHED_MESSAGE);\n            getMessageResult.setMinOffset(0L);\n            getMessageResult.setMaxOffset(1L);\n            getMessageResult.setNextBeginOffset(1L);\n            Mockito.doReturn(CompletableFuture.completedFuture(getMessageResult))\n                .when(consumerServiceSpy).getMessageAsync(clientHost, groupId, retry, 0, 0, 10, null);\n            Mockito.doReturn(CompletableFuture.completedFuture(getMessageResult))\n                .when(consumerServiceSpy).getMessageAsync(clientHost, groupId, retry, 0, 0, 8, null);\n        }\n\n        for (int i = -1; i < 2; i++) {\n            GetMessageResult getMessageResult = new GetMessageResult();\n            getMessageResult.setStatus(GetMessageStatus.FOUND);\n            getMessageResult.setMinOffset(0L);\n            getMessageResult.setMaxOffset(1L);\n            getMessageResult.setNextBeginOffset(1L);\n            getMessageResult.addMessage(Mockito.mock(SelectMappedBufferResult.class), 1L);\n\n            Mockito.doReturn(CompletableFuture.completedFuture(getMessageResult))\n                .when(consumerServiceSpy).getMessageAsync(clientHost, groupId, topicId, i, 0, 8, null);\n            Mockito.doReturn(CompletableFuture.completedFuture(getMessageResult))\n                .when(consumerServiceSpy).getMessageAsync(clientHost, groupId, topicId, i, 0, 9, null);\n            Mockito.doReturn(CompletableFuture.completedFuture(getMessageResult))\n                .when(consumerServiceSpy).getMessageAsync(clientHost, groupId, topicId, i, 0, 10, null);\n        }\n\n        // pop broker\n        consumerServiceSpy.popAsync(clientHost, System.currentTimeMillis(),\n            20000, groupId, topicId, -1, 10, false, attemptId, ConsumeInitMode.MIN, null).join();\n    }\n\n    @Test\n    public void ackAsyncTest() {\n        long current = System.currentTimeMillis();\n        consumerService.getPopConsumerStore().start();\n        consumerService.ackAsync(\n            current, 10, groupId, topicId, queueId, 100).join();\n        consumerService.changeInvisibilityDuration(current, 10,\n            current + 100, 10, groupId, topicId, queueId, 100, false);\n        consumerService.shutdown();\n    }\n\n    @Test\n    public void reviveSkipIfGroupAbsent() {\n        String groupName = \"PopGroupAbsent\";\n        brokerController.getBrokerConfig().setPopReviveSkipIfGroupAbsent(true);\n        PopConsumerRecord record = Mockito.mock(PopConsumerRecord.class);\n        Mockito.when(record.getGroupId()).thenReturn(groupName);\n        Mockito.when(brokerController.getSubscriptionGroupManager()\n            .containsSubscriptionGroup(groupName)).thenReturn(false);\n        CompletableFuture<Boolean> result = consumerService.revive(record);\n        Assert.assertTrue(result.join());\n    }\n\n    @Test\n    public void reviveRetryTest() {\n        Mockito.when(brokerController.getTopicConfigManager().selectTopicConfig(topicId)).thenReturn(null);\n        Mockito.when(brokerController.getConsumerOffsetManager().queryOffset(groupId, topicId, 0)).thenReturn(-1L);\n\n        consumerService.createRetryTopicIfNeeded(groupId, topicId);\n        consumerService.clearCache(groupId, topicId, queueId);\n        MessageExt messageExt = new MessageExt();\n        messageExt.setBody(\"body\".getBytes());\n        messageExt.setBornTimestamp(System.currentTimeMillis());\n        messageExt.setFlag(0);\n        messageExt.setSysFlag(0);\n        messageExt.setReconsumeTimes(1);\n        messageExt.putUserProperty(\"key\", \"value\");\n\n        PopConsumerRecord record = new PopConsumerRecord();\n        record.setTopicId(\"topic\");\n        record.setGroupId(\"group\");\n        Mockito.when(brokerController.getBrokerStatsManager()).thenReturn(Mockito.mock(BrokerStatsManager.class));\n        Mockito.when(brokerController.getEscapeBridge()).thenReturn(Mockito.mock(EscapeBridge.class));\n        Mockito.when(brokerController.getEscapeBridge().putMessageToSpecificQueue(any(MessageExtBrokerInner.class)))\n            .thenReturn(new PutMessageResult(\n                PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));\n\n        PopConsumerService consumerServiceSpy = Mockito.spy(consumerService);\n        Mockito.doNothing().when(consumerServiceSpy).createRetryTopicIfNeeded(any(), any());\n        Assert.assertTrue(consumerServiceSpy.reviveRetry(record, messageExt));\n\n        // write message error\n        Mockito.when(brokerController.getEscapeBridge().putMessageToSpecificQueue(any(MessageExtBrokerInner.class)))\n            .thenReturn(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR,\n                new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR)));\n        Assert.assertFalse(consumerServiceSpy.reviveRetry(record, messageExt));\n\n        // revive backoff\n        consumerService.getPopConsumerStore().start();\n        List<PopConsumerRecord> consumerRecordList = IntStream.range(0, 3)\n            .mapToObj(i -> {\n                PopConsumerRecord temp = new PopConsumerRecord();\n                temp.setPopTime(0);\n                temp.setInvisibleTime(20 * 1000);\n                temp.setTopicId(\"topic\");\n                temp.setGroupId(\"group\");\n                temp.setQueueId(2);\n                temp.setOffset(i);\n                return temp;\n            })\n            .collect(Collectors.toList());\n        consumerService.getPopConsumerStore().writeRecords(consumerRecordList);\n\n        Mockito.doReturn(CompletableFuture.completedFuture(null))\n            .when(consumerServiceSpy).getMessageAsync(any(PopConsumerRecord.class));\n        consumerServiceSpy.revive(new AtomicLong(20 * 1000), 1);\n\n        Mockito.doReturn(CompletableFuture.completedFuture(\n                Triple.of(null, \"GetMessageResult is null\", false)))\n            .when(consumerServiceSpy).getMessageAsync(any(PopConsumerRecord.class));\n        consumerServiceSpy.revive(new AtomicLong(20 * 1000), 1);\n\n        Mockito.doReturn(CompletableFuture.completedFuture(\n                Triple.of(Mockito.mock(MessageExt.class), null, false)))\n            .when(consumerServiceSpy).getMessageAsync(any(PopConsumerRecord.class));\n        consumerServiceSpy.revive(new AtomicLong(20 * 1000), 1);\n        consumerService.shutdown();\n    }\n\n    @Test\n    public void reviveBackoffRetryTest() {\n        Mockito.when(brokerController.getEscapeBridge()).thenReturn(Mockito.mock(EscapeBridge.class));\n        Mockito.when(brokerController.getSubscriptionGroupManager()\n            .containsSubscriptionGroup(anyString())).thenReturn(true);\n        PopConsumerService consumerServiceSpy = Mockito.spy(consumerService);\n\n        consumerService.getPopConsumerStore().start();\n\n        long popTime = 1000000000L;\n        long invisibleTime = 60 * 1000L;\n        PopConsumerRecord record = new PopConsumerRecord();\n        record.setPopTime(popTime);\n        record.setInvisibleTime(invisibleTime);\n        record.setTopicId(\"topic\");\n        record.setGroupId(\"group\");\n        record.setQueueId(0);\n        record.setOffset(0);\n        consumerService.getPopConsumerStore().writeRecords(Collections.singletonList(record));\n\n        Mockito.doReturn(CompletableFuture.completedFuture(Triple.of(Mockito.mock(MessageExt.class), \"\", false)))\n            .when(consumerServiceSpy).getMessageAsync(any(PopConsumerRecord.class));\n        Mockito.when(brokerController.getEscapeBridge().putMessageToSpecificQueue(any(MessageExtBrokerInner.class))).thenReturn(\n            new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))\n        );\n\n        long visibleTimestamp = popTime + invisibleTime;\n\n        // revive fails\n        Assert.assertEquals(1, consumerServiceSpy.revive(new AtomicLong(visibleTimestamp), 1));\n        // should be invisible now\n        Assert.assertEquals(0, consumerService.getPopConsumerStore().scanExpiredRecords(0, visibleTimestamp, 1).size());\n        // will be visible again in 10 seconds\n        Assert.assertEquals(1, consumerService.getPopConsumerStore().scanExpiredRecords(visibleTimestamp, System.currentTimeMillis() + visibleTimestamp + 10 * 1000, 1).size());\n\n        consumerService.shutdown();\n    }\n\n    @Test\n    public void transferToFsStoreTest() {\n        Assert.assertNotNull(consumerService.getServiceName());\n        List<PopConsumerRecord> consumerRecordList = IntStream.range(0, 3)\n            .mapToObj(i -> {\n                PopConsumerRecord temp = new PopConsumerRecord();\n                temp.setPopTime(0);\n                temp.setInvisibleTime(20 * 1000);\n                temp.setTopicId(\"topic\");\n                temp.setGroupId(\"group\");\n                temp.setQueueId(2);\n                temp.setOffset(i);\n                return temp;\n            })\n            .collect(Collectors.toList());\n\n        Mockito.when(brokerController.getPopMessageProcessor().buildCkMsg(any(), anyInt()))\n            .thenReturn(new MessageExtBrokerInner());\n        Mockito.when(brokerController.getMessageStore()).thenReturn(Mockito.mock(MessageStore.class));\n        Mockito.when(brokerController.getMessageStore().asyncPutMessage(any()))\n            .thenReturn(CompletableFuture.completedFuture(\n                new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))));\n\n        consumerService.start();\n        consumerService.getPopConsumerStore().writeRecords(consumerRecordList);\n        consumerService.transferToFsStore();\n        consumerService.shutdown();\n    }\n\n    @Test\n    public void testChangeInvisibilityDurationWithSuspendTrue() {\n        long current = System.currentTimeMillis();\n        long popTime = current - 1000;\n        long invisibleTime = 10000;\n        long changedPopTime = current;\n        long changedInvisibleTime = 20000;\n        long offset = 100L;\n\n        consumerService.getPopConsumerStore().start();\n        Mockito.when(brokerController.getSubscriptionGroupManager().containsSubscriptionGroup(groupId)).thenReturn(true);\n\n        // Test with suspend = true\n        consumerService.changeInvisibilityDuration(popTime, invisibleTime, changedPopTime,\n            changedInvisibleTime, groupId, topicId, queueId, offset, true);\n\n        // Verify that the record was written with suspend = true\n        List<PopConsumerRecord> records = consumerService.getPopConsumerStore()\n            .scanExpiredRecords(0, changedPopTime + changedInvisibleTime + 1000, 10);\n        Assert.assertFalse(\"Should have at least one record\", records.isEmpty());\n        PopConsumerRecord ckRecord = records.stream()\n            .filter(r -> r.getOffset() == offset && r.getPopTime() == changedPopTime)\n            .findFirst()\n            .orElse(null);\n        Assert.assertNotNull(\"Should find the checkpoint record\", ckRecord);\n        Assert.assertTrue(\"Suspend flag should be true\", ckRecord.isSuspend());\n        Assert.assertEquals(\"GroupId should match\", groupId, ckRecord.getGroupId());\n        Assert.assertEquals(\"TopicId should match\", topicId, ckRecord.getTopicId());\n        Assert.assertEquals(\"QueueId should match\", queueId, ckRecord.getQueueId());\n        Assert.assertEquals(\"Offset should match\", offset, ckRecord.getOffset());\n\n        consumerService.shutdown();\n    }\n\n    @Test\n    public void testChangeInvisibilityDurationWithSuspendFalse() {\n        long current = System.currentTimeMillis();\n        long popTime = current - 1000;\n        long invisibleTime = 10000;\n        long changedPopTime = current;\n        long changedInvisibleTime = 20000;\n        long offset = 200L;\n\n        consumerService.getPopConsumerStore().start();\n        Mockito.when(brokerController.getSubscriptionGroupManager().containsSubscriptionGroup(groupId)).thenReturn(true);\n\n        // Test with suspend = false\n        consumerService.changeInvisibilityDuration(popTime, invisibleTime, changedPopTime,\n            changedInvisibleTime, groupId, topicId, queueId, offset, false);\n\n        // Verify that the record was written with suspend = false\n        List<PopConsumerRecord> records = consumerService.getPopConsumerStore()\n            .scanExpiredRecords(0, changedPopTime + changedInvisibleTime + 1000, 10);\n        Assert.assertFalse(\"Should have at least one record\", records.isEmpty());\n        PopConsumerRecord ckRecord = records.stream()\n            .filter(r -> r.getOffset() == offset && r.getPopTime() == changedPopTime)\n            .findFirst()\n            .orElse(null);\n        Assert.assertNotNull(\"Should find the checkpoint record\", ckRecord);\n        Assert.assertFalse(\"Suspend flag should be false\", ckRecord.isSuspend());\n        Assert.assertEquals(\"GroupId should match\", groupId, ckRecord.getGroupId());\n        Assert.assertEquals(\"TopicId should match\", topicId, ckRecord.getTopicId());\n        Assert.assertEquals(\"QueueId should match\", queueId, ckRecord.getQueueId());\n        Assert.assertEquals(\"Offset should match\", offset, ckRecord.getOffset());\n\n        consumerService.shutdown();\n    }\n\n    @Test\n    public void testReviveRetryWithSuspendTrue() {\n        Mockito.when(brokerController.getTopicConfigManager().selectTopicConfig(topicId)).thenReturn(null);\n        Mockito.when(brokerController.getConsumerOffsetManager().queryOffset(groupId, topicId, 0)).thenReturn(-1L);\n\n        consumerService.createRetryTopicIfNeeded(groupId, topicId);\n        consumerService.clearCache(groupId, topicId, queueId);\n\n        // Create message with reconsumeTimes = 2\n        MessageExt messageExt = new MessageExt();\n        messageExt.setBody(\"body\".getBytes());\n        messageExt.setBornTimestamp(System.currentTimeMillis());\n        messageExt.setFlag(0);\n        messageExt.setSysFlag(0);\n        messageExt.setReconsumeTimes(2);\n        messageExt.putUserProperty(\"key\", \"value\");\n\n        // Create record with suspend = true\n        PopConsumerRecord record = new PopConsumerRecord();\n        record.setTopicId(topicId);\n        record.setGroupId(groupId);\n        record.setQueueId(queueId);\n        record.setPopTime(System.currentTimeMillis());\n        record.setInvisibleTime(30000);\n        record.setOffset(100L);\n        record.setSuspend(true);\n\n        Mockito.when(brokerController.getBrokerStatsManager()).thenReturn(Mockito.mock(BrokerStatsManager.class));\n        EscapeBridge escapeBridge = Mockito.mock(EscapeBridge.class);\n        Mockito.when(brokerController.getEscapeBridge()).thenReturn(escapeBridge);\n\n        // Capture the MessageExtBrokerInner to verify reconsumeTimes\n        ArgumentCaptor<MessageExtBrokerInner> messageCaptor =\n            ArgumentCaptor.forClass(MessageExtBrokerInner.class);\n        Mockito.when(escapeBridge.putMessageToSpecificQueue(messageCaptor.capture()))\n            .thenReturn(new PutMessageResult(\n                PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));\n\n        PopConsumerService consumerServiceSpy = Mockito.spy(consumerService);\n        Mockito.doNothing().when(consumerServiceSpy).createRetryTopicIfNeeded(any(), any());\n        Assert.assertTrue(\"Revive should succeed\", consumerServiceSpy.reviveRetry(record, messageExt));\n\n        // Verify that reconsumeTimes was NOT incremented (should remain 2)\n        MessageExtBrokerInner capturedMessage = messageCaptor.getValue();\n        Assert.assertNotNull(\"Message should be captured\", capturedMessage);\n        Assert.assertEquals(\"ReconsumeTimes should remain 2 when suspend=true\", 2, capturedMessage.getReconsumeTimes());\n    }\n\n    @Test\n    public void testReviveRetryWithSuspendFalse() {\n        Mockito.when(brokerController.getTopicConfigManager().selectTopicConfig(topicId)).thenReturn(null);\n        Mockito.when(brokerController.getConsumerOffsetManager().queryOffset(groupId, topicId, 0)).thenReturn(-1L);\n\n        consumerService.createRetryTopicIfNeeded(groupId, topicId);\n        consumerService.clearCache(groupId, topicId, queueId);\n\n        // Create message with reconsumeTimes = 2\n        MessageExt messageExt = new MessageExt();\n        messageExt.setBody(\"body\".getBytes());\n        messageExt.setBornTimestamp(System.currentTimeMillis());\n        messageExt.setFlag(0);\n        messageExt.setSysFlag(0);\n        messageExt.setReconsumeTimes(2);\n        messageExt.putUserProperty(\"key\", \"value\");\n\n        // Create record with suspend = false\n        PopConsumerRecord record = new PopConsumerRecord();\n        record.setTopicId(topicId);\n        record.setGroupId(groupId);\n        record.setQueueId(queueId);\n        record.setPopTime(System.currentTimeMillis());\n        record.setInvisibleTime(30000);\n        record.setOffset(200L);\n        record.setSuspend(false);\n\n        Mockito.when(brokerController.getBrokerStatsManager()).thenReturn(Mockito.mock(BrokerStatsManager.class));\n        EscapeBridge escapeBridge = Mockito.mock(EscapeBridge.class);\n        Mockito.when(brokerController.getEscapeBridge()).thenReturn(escapeBridge);\n\n        // Capture the MessageExtBrokerInner to verify reconsumeTimes\n        ArgumentCaptor<MessageExtBrokerInner> messageCaptor =\n            ArgumentCaptor.forClass(MessageExtBrokerInner.class);\n        Mockito.when(escapeBridge.putMessageToSpecificQueue(messageCaptor.capture()))\n            .thenReturn(new PutMessageResult(\n                PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));\n\n        PopConsumerService consumerServiceSpy = Mockito.spy(consumerService);\n        Mockito.doNothing().when(consumerServiceSpy).createRetryTopicIfNeeded(any(), any());\n        Assert.assertTrue(\"Revive should succeed\", consumerServiceSpy.reviveRetry(record, messageExt));\n\n        // Verify that reconsumeTimes was incremented (should be 3)\n        MessageExtBrokerInner capturedMessage = messageCaptor.getValue();\n        Assert.assertNotNull(\"Message should be captured\", capturedMessage);\n        Assert.assertEquals(\"ReconsumeTimes should be incremented to 3 when suspend=false\", 3, capturedMessage.getReconsumeTimes());\n    }\n\n    @Test\n    public void testReviveRetryWithSuspendTrueMultipleTimes() {\n        Mockito.when(brokerController.getTopicConfigManager().selectTopicConfig(topicId)).thenReturn(null);\n        Mockito.when(brokerController.getConsumerOffsetManager().queryOffset(groupId, topicId, 0)).thenReturn(-1L);\n\n        consumerService.createRetryTopicIfNeeded(groupId, topicId);\n        consumerService.clearCache(groupId, topicId, queueId);\n\n        // Create message with reconsumeTimes = 0\n        MessageExt messageExt = new MessageExt();\n        messageExt.setBody(\"body\".getBytes());\n        messageExt.setBornTimestamp(System.currentTimeMillis());\n        messageExt.setFlag(0);\n        messageExt.setSysFlag(0);\n        messageExt.setReconsumeTimes(0);\n        messageExt.putUserProperty(\"key\", \"value\");\n\n        Mockito.when(brokerController.getBrokerStatsManager()).thenReturn(Mockito.mock(BrokerStatsManager.class));\n        EscapeBridge escapeBridge = Mockito.mock(EscapeBridge.class);\n        Mockito.when(brokerController.getEscapeBridge()).thenReturn(escapeBridge);\n\n        PopConsumerService consumerServiceSpy = Mockito.spy(consumerService);\n        Mockito.doNothing().when(consumerServiceSpy).createRetryTopicIfNeeded(any(), any());\n\n        // Simulate multiple nacks with suspend = true\n        for (int i = 0; i < 3; i++) {\n            PopConsumerRecord record = new PopConsumerRecord();\n            record.setTopicId(topicId);\n            record.setGroupId(groupId);\n            record.setQueueId(queueId);\n            record.setPopTime(System.currentTimeMillis());\n            record.setInvisibleTime(30000);\n            record.setOffset(300L + i);\n            record.setSuspend(true);\n\n            // Capture the MessageExtBrokerInner to verify reconsumeTimes\n            org.mockito.ArgumentCaptor<MessageExtBrokerInner> messageCaptor =\n                org.mockito.ArgumentCaptor.forClass(MessageExtBrokerInner.class);\n            Mockito.when(escapeBridge.putMessageToSpecificQueue(messageCaptor.capture()))\n                .thenReturn(new PutMessageResult(\n                    PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));\n\n            Assert.assertTrue(\"Revive should succeed\", consumerServiceSpy.reviveRetry(record, messageExt));\n\n            // Verify that reconsumeTimes remains 0 (not incremented)\n            MessageExtBrokerInner capturedMessage = messageCaptor.getValue();\n            Assert.assertNotNull(\"Message should be captured\", capturedMessage);\n            Assert.assertEquals(\"ReconsumeTimes should remain 0 after \" + (i + 1) + \" nacks with suspend=true\",\n                0, capturedMessage.getReconsumeTimes());\n\n            // Update messageExt for next iteration (simulate the message being re-consumed)\n            messageExt.setReconsumeTimes(capturedMessage.getReconsumeTimes());\n        }\n    }\n\n    @Test\n    public void testReviveRetryWithSuspendFalseMultipleTimes() {\n        Mockito.when(brokerController.getTopicConfigManager().selectTopicConfig(topicId)).thenReturn(null);\n        Mockito.when(brokerController.getConsumerOffsetManager().queryOffset(groupId, topicId, 0)).thenReturn(-1L);\n\n        consumerService.createRetryTopicIfNeeded(groupId, topicId);\n        consumerService.clearCache(groupId, topicId, queueId);\n\n        // Create message with reconsumeTimes = 0\n        MessageExt messageExt = new MessageExt();\n        messageExt.setBody(\"body\".getBytes());\n        messageExt.setBornTimestamp(System.currentTimeMillis());\n        messageExt.setFlag(0);\n        messageExt.setSysFlag(0);\n        messageExt.setReconsumeTimes(0);\n        messageExt.putUserProperty(\"key\", \"value\");\n\n        Mockito.when(brokerController.getBrokerStatsManager()).thenReturn(Mockito.mock(BrokerStatsManager.class));\n        EscapeBridge escapeBridge = Mockito.mock(EscapeBridge.class);\n        Mockito.when(brokerController.getEscapeBridge()).thenReturn(escapeBridge);\n\n        PopConsumerService consumerServiceSpy = Mockito.spy(consumerService);\n        Mockito.doNothing().when(consumerServiceSpy).createRetryTopicIfNeeded(any(), any());\n\n        // Simulate multiple nacks with suspend = false\n        for (int i = 0; i < 3; i++) {\n            PopConsumerRecord record = new PopConsumerRecord();\n            record.setTopicId(topicId);\n            record.setGroupId(groupId);\n            record.setQueueId(queueId);\n            record.setPopTime(System.currentTimeMillis());\n            record.setInvisibleTime(30000);\n            record.setOffset(400L + i);\n            record.setSuspend(false);\n\n            // Capture the MessageExtBrokerInner to verify reconsumeTimes\n            org.mockito.ArgumentCaptor<MessageExtBrokerInner> messageCaptor =\n                org.mockito.ArgumentCaptor.forClass(MessageExtBrokerInner.class);\n            Mockito.when(escapeBridge.putMessageToSpecificQueue(messageCaptor.capture()))\n                .thenReturn(new PutMessageResult(\n                    PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));\n\n            Assert.assertTrue(\"Revive should succeed\", consumerServiceSpy.reviveRetry(record, messageExt));\n\n            // Verify that reconsumeTimes is incremented each time\n            MessageExtBrokerInner capturedMessage = messageCaptor.getValue();\n            Assert.assertNotNull(\"Message should be captured\", capturedMessage);\n            Assert.assertEquals(\"ReconsumeTimes should be \" + (i + 1) + \" after \" + (i + 1) + \" nacks with suspend=false\",\n                i + 1, capturedMessage.getReconsumeTimes());\n\n            // Update messageExt for next iteration (simulate the message being re-consumed)\n            messageExt.setReconsumeTimes(capturedMessage.getReconsumeTimes());\n        }\n    }\n}"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/pop/orderly/ConsumerOrderInfoManagerLockFreeNotifyTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.pop.orderly;\n\nimport java.time.Duration;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.processor.PopMessageProcessor;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.assertj.core.util.Lists;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.stubbing.Answer;\n\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.doAnswer;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\npublic class ConsumerOrderInfoManagerLockFreeNotifyTest {\n\n    private static final String TOPIC = \"topic\";\n    private static final String GROUP = \"group\";\n    private static final int QUEUE_ID_0 = 0;\n\n    private long popTime;\n    private QueueLevelConsumerManager consumerOrderInfoManager;\n    private AtomicBoolean notified;\n\n    private final BrokerConfig brokerConfig = new BrokerConfig();\n    private final PopMessageProcessor popMessageProcessor = mock(PopMessageProcessor.class);\n    private final BrokerController brokerController = mock(BrokerController.class);\n\n    @Before\n    public void before() throws ConsumeQueueException {\n        notified = new AtomicBoolean(false);\n        brokerConfig.setEnableNotifyAfterPopOrderLockRelease(true);\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        when(brokerController.getPopMessageProcessor()).thenReturn(popMessageProcessor);\n        doAnswer((Answer<Void>) mock -> {\n            notified.set(true);\n            return null;\n        }).when(popMessageProcessor).notifyLongPollingRequestIfNeed(anyString(), anyString(), anyInt());\n\n        consumerOrderInfoManager = new QueueLevelConsumerManager(brokerController);\n        popTime = System.currentTimeMillis();\n    }\n\n    @Test\n    public void testConsumeMessageThenNoAck() {\n        consumerOrderInfoManager.update(\n            null,\n            false,\n            TOPIC,\n            GROUP,\n            QUEUE_ID_0,\n            popTime,\n            3000,\n            Lists.newArrayList(1L),\n            new StringBuilder()\n        );\n        await().atLeast(Duration.ofSeconds(2)).atMost(Duration.ofSeconds(4)).until(notified::get);\n        assertTrue(consumerOrderInfoManager.getConsumerOrderInfoLockManager().getTimeoutMap().isEmpty());\n    }\n\n    @Test\n    public void testConsumeMessageThenAck() {\n        consumerOrderInfoManager.update(\n            null,\n            false,\n            TOPIC,\n            GROUP,\n            QUEUE_ID_0,\n            popTime,\n            3000,\n            Lists.newArrayList(1L),\n            new StringBuilder()\n        );\n        consumerOrderInfoManager.commitAndNext(\n            TOPIC,\n            GROUP,\n            QUEUE_ID_0,\n            1,\n            popTime\n        );\n        await().atMost(Duration.ofSeconds(1)).until(notified::get);\n        assertTrue(consumerOrderInfoManager.getConsumerOrderInfoLockManager().getTimeoutMap().isEmpty());\n    }\n\n    @Test\n    public void testConsumeTheChangeInvisibleLonger() {\n        consumerOrderInfoManager.update(\n            null,\n            false,\n            TOPIC,\n            GROUP,\n            QUEUE_ID_0,\n            popTime,\n            3000,\n            Lists.newArrayList(1L),\n            new StringBuilder()\n        );\n        consumerOrderInfoManager.updateNextVisibleTime(\n            TOPIC,\n            GROUP,\n            QUEUE_ID_0,\n            1,\n            popTime,\n            popTime + 5000\n        );\n        await().atLeast(Duration.ofSeconds(4)).atMost(Duration.ofSeconds(6)).until(notified::get);\n        assertTrue(consumerOrderInfoManager.getConsumerOrderInfoLockManager().getTimeoutMap().isEmpty());\n    }\n\n    @Test\n    public void testConsumeTheChangeInvisibleShorter() {\n        consumerOrderInfoManager.update(\n            null,\n            false,\n            TOPIC,\n            GROUP,\n            QUEUE_ID_0,\n            popTime,\n            3000,\n            Lists.newArrayList(1L),\n            new StringBuilder()\n        );\n        consumerOrderInfoManager.updateNextVisibleTime(\n            TOPIC,\n            GROUP,\n            QUEUE_ID_0,\n            1,\n            popTime,\n            popTime + 1000\n        );\n        await().atLeast(Duration.ofMillis(500)).atMost(Duration.ofSeconds(2)).until(notified::get);\n        assertTrue(consumerOrderInfoManager.getConsumerOrderInfoLockManager().getTimeoutMap().isEmpty());\n    }\n\n    @Test\n    public void testRecover() {\n        QueueLevelConsumerManager savedConsumerOrderInfoManager = new QueueLevelConsumerManager();\n        savedConsumerOrderInfoManager.update(\n            null,\n            false,\n            TOPIC,\n            GROUP,\n            QUEUE_ID_0,\n            popTime,\n            3000,\n            Lists.newArrayList(1L),\n            new StringBuilder()\n        );\n        String encodedData = savedConsumerOrderInfoManager.encode();\n\n        consumerOrderInfoManager.decode(encodedData);\n        await().atLeast(Duration.ofSeconds(2)).atMost(Duration.ofSeconds(4)).until(notified::get);\n        assertTrue(consumerOrderInfoManager.getConsumerOrderInfoLockManager().getTimeoutMap().isEmpty());\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/pop/orderly/ConsumerOrderInfoManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.pop.orderly;\n\nimport java.time.Duration;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.UUID;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;\nimport org.assertj.core.util.Lists;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNotSame;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\npublic class ConsumerOrderInfoManagerTest {\n\n    private static final String TOPIC = \"topic\";\n    private static final String GROUP = \"group\";\n    private static final int QUEUE_ID_0 = 0;\n    private static final int QUEUE_ID_1 = 1;\n\n    private long popTime;\n    private QueueLevelConsumerManager consumerOrderInfoManager;\n\n    @Before\n    public void before() {\n        consumerOrderInfoManager = new QueueLevelConsumerManager();\n        popTime = System.currentTimeMillis();\n    }\n\n    @Test\n    public void testCommitAndNext() {\n        consumerOrderInfoManager.update(\n            null,\n            false,\n            TOPIC,\n            GROUP,\n            QUEUE_ID_0,\n            popTime,\n            3000,\n            Lists.newArrayList(1L),\n            new StringBuilder()\n        );\n        assertEncodeAndDecode();\n        assertEquals(-2, consumerOrderInfoManager.commitAndNext(\n            TOPIC,\n            GROUP,\n            QUEUE_ID_0,\n            1L,\n            popTime - 10\n        ));\n        assertEncodeAndDecode();\n        assertTrue(consumerOrderInfoManager.checkBlock(\n            null,\n            TOPIC,\n            GROUP,\n            QUEUE_ID_0,\n            TimeUnit.SECONDS.toMillis(3)\n        ));\n\n        assertEquals(2, consumerOrderInfoManager.commitAndNext(\n            TOPIC,\n            GROUP,\n            QUEUE_ID_0,\n            1L,\n            popTime\n        ));\n        assertEncodeAndDecode();\n        assertFalse(consumerOrderInfoManager.checkBlock(\n            null,\n            TOPIC,\n            GROUP,\n            QUEUE_ID_0,\n            TimeUnit.SECONDS.toMillis(3)\n        ));\n    }\n\n    @Test\n    public void testConsumedCount() {\n        {\n            // consume three new messages\n            StringBuilder orderInfoBuilder = new StringBuilder();\n            consumerOrderInfoManager.update(\n                null,\n                false,\n                TOPIC,\n                GROUP,\n                QUEUE_ID_0,\n                popTime,\n                3000,\n                Lists.newArrayList(1L, 2L, 3L),\n                orderInfoBuilder\n            );\n            assertEncodeAndDecode();\n            Map<String, Integer> orderInfoMap = ExtraInfoUtil.parseOrderCountInfo(orderInfoBuilder.toString());\n            assertEquals(1, orderInfoMap.size());\n            assertEquals(0, orderInfoMap.get(ExtraInfoUtil.getStartOffsetInfoMapKey(TOPIC, QUEUE_ID_0)).intValue());\n        }\n\n        {\n            // reconsume same messages\n            StringBuilder orderInfoBuilder = new StringBuilder();\n            consumerOrderInfoManager.update(\n                null,\n                false,\n                TOPIC,\n                GROUP,\n                QUEUE_ID_0,\n                popTime,\n                3000,\n                Lists.newArrayList(1L, 2L, 3L),\n                orderInfoBuilder\n            );\n            assertEncodeAndDecode();\n            Map<String, Integer> orderInfoMap = ExtraInfoUtil.parseOrderCountInfo(orderInfoBuilder.toString());\n            assertEquals(4, orderInfoMap.size());\n            assertEquals(1, orderInfoMap.get(ExtraInfoUtil.getStartOffsetInfoMapKey(TOPIC, QUEUE_ID_0)).intValue());\n            for (int i = 1; i <= 3; i++) {\n                assertEquals(1, orderInfoMap.get(ExtraInfoUtil.getQueueOffsetMapKey(TOPIC, QUEUE_ID_0, i)).intValue());\n            }\n        }\n\n        {\n            // reconsume last two message\n            StringBuilder orderInfoBuilder = new StringBuilder();\n            consumerOrderInfoManager.update(\n                null,\n                false,\n                TOPIC,\n                GROUP,\n                QUEUE_ID_0,\n                popTime,\n                3000,\n                Lists.newArrayList(2L, 3L),\n                orderInfoBuilder\n            );\n            assertEncodeAndDecode();\n            Map<String, Integer> orderInfoMap = ExtraInfoUtil.parseOrderCountInfo(orderInfoBuilder.toString());\n            assertEquals(3, orderInfoMap.size());\n            assertEquals(2, orderInfoMap.get(ExtraInfoUtil.getStartOffsetInfoMapKey(TOPIC, QUEUE_ID_0)).intValue());\n            for (int i = 2; i <= 3; i++) {\n                assertEquals(2, orderInfoMap.get(ExtraInfoUtil.getQueueOffsetMapKey(TOPIC, QUEUE_ID_0, i)).intValue());\n            }\n        }\n\n        {\n            // consume a new message and reconsume last message\n            StringBuilder orderInfoBuilder = new StringBuilder();\n            consumerOrderInfoManager.update(\n                null,\n                false,\n                TOPIC,\n                GROUP,\n                QUEUE_ID_0,\n                popTime,\n                3000,\n                Lists.newArrayList(3L, 4L),\n                orderInfoBuilder\n            );\n            assertEncodeAndDecode();\n            Map<String, Integer> orderInfoMap = ExtraInfoUtil.parseOrderCountInfo(orderInfoBuilder.toString());\n            assertEquals(2, orderInfoMap.size());\n            assertEquals(0, orderInfoMap.get(ExtraInfoUtil.getStartOffsetInfoMapKey(TOPIC, QUEUE_ID_0)).intValue());\n            assertEquals(3, orderInfoMap.get(ExtraInfoUtil.getQueueOffsetMapKey(TOPIC, QUEUE_ID_0, 3)).intValue());\n        }\n\n        {\n            // consume two new messages\n            StringBuilder orderInfoBuilder = new StringBuilder();\n            consumerOrderInfoManager.update(\n                null,\n                false,\n                TOPIC,\n                GROUP,\n                QUEUE_ID_0,\n                popTime,\n                3000,\n                Lists.newArrayList(5L, 6L),\n                orderInfoBuilder\n            );\n            assertEncodeAndDecode();\n            Map<String, Integer> orderInfoMap = ExtraInfoUtil.parseOrderCountInfo(orderInfoBuilder.toString());\n            assertEquals(1, orderInfoMap.size());\n            assertEquals(0, orderInfoMap.get(ExtraInfoUtil.getStartOffsetInfoMapKey(TOPIC, QUEUE_ID_0)).intValue());\n        }\n    }\n\n    @Test\n    public void testConsumedCountForMultiQueue() {\n        {\n            // consume two new messages\n            StringBuilder orderInfoBuilder = new StringBuilder();\n            consumerOrderInfoManager.update(\n                null,\n                false,\n                TOPIC,\n                GROUP,\n                QUEUE_ID_0,\n                popTime,\n                3000,\n                Lists.newArrayList(0L),\n                orderInfoBuilder\n            );\n            consumerOrderInfoManager.update(\n                null,\n                false,\n                TOPIC,\n                GROUP,\n                QUEUE_ID_1,\n                popTime,\n                3000,\n                Lists.newArrayList(0L),\n                orderInfoBuilder\n            );\n            assertEncodeAndDecode();\n            Map<String, Integer> orderInfoMap = ExtraInfoUtil.parseOrderCountInfo(orderInfoBuilder.toString());\n            assertEquals(2, orderInfoMap.size());\n            assertEquals(0, orderInfoMap.get(ExtraInfoUtil.getStartOffsetInfoMapKey(TOPIC, QUEUE_ID_0)).intValue());\n            assertEquals(0, orderInfoMap.get(ExtraInfoUtil.getStartOffsetInfoMapKey(TOPIC, QUEUE_ID_1)).intValue());\n        }\n        {\n            // reconsume two message\n            StringBuilder orderInfoBuilder = new StringBuilder();\n            consumerOrderInfoManager.update(\n                null,\n                false,\n                TOPIC,\n                GROUP,\n                QUEUE_ID_0,\n                popTime,\n                3000,\n                Lists.newArrayList(0L),\n                orderInfoBuilder\n            );\n            consumerOrderInfoManager.update(\n                null,\n                false,\n                TOPIC,\n                GROUP,\n                QUEUE_ID_1,\n                popTime,\n                3000,\n                Lists.newArrayList(0L),\n                orderInfoBuilder\n            );\n            assertEncodeAndDecode();\n            Map<String, Integer> orderInfoMap = ExtraInfoUtil.parseOrderCountInfo(orderInfoBuilder.toString());\n            assertEquals(4, orderInfoMap.size());\n            assertEquals(1, orderInfoMap.get(ExtraInfoUtil.getStartOffsetInfoMapKey(TOPIC, QUEUE_ID_0)).intValue());\n            assertEquals(1, orderInfoMap.get(ExtraInfoUtil.getStartOffsetInfoMapKey(TOPIC, QUEUE_ID_1)).intValue());\n            assertEquals(1, orderInfoMap.get(ExtraInfoUtil.getQueueOffsetMapKey(TOPIC, QUEUE_ID_0, 0L)).intValue());\n            assertEquals(1, orderInfoMap.get(ExtraInfoUtil.getQueueOffsetMapKey(TOPIC, QUEUE_ID_1, 0L)).intValue());\n        }\n        {\n            // reconsume with a new message\n            StringBuilder orderInfoBuilder = new StringBuilder();\n            consumerOrderInfoManager.update(\n                null,\n                false,\n                TOPIC,\n                GROUP,\n                QUEUE_ID_0,\n                popTime,\n                3000,\n                Lists.newArrayList(0L, 1L),\n                orderInfoBuilder\n            );\n            consumerOrderInfoManager.update(\n                null,\n                false,\n                TOPIC,\n                GROUP,\n                QUEUE_ID_1,\n                popTime,\n                3000,\n                Lists.newArrayList(0L),\n                orderInfoBuilder\n            );\n            assertEncodeAndDecode();\n            Map<String, Integer> orderInfoMap = ExtraInfoUtil.parseOrderCountInfo(orderInfoBuilder.toString());\n            assertEquals(4, orderInfoMap.size());\n            assertEquals(0, orderInfoMap.get(ExtraInfoUtil.getStartOffsetInfoMapKey(TOPIC, QUEUE_ID_0)).intValue());\n            assertEquals(2, orderInfoMap.get(ExtraInfoUtil.getStartOffsetInfoMapKey(TOPIC, QUEUE_ID_1)).intValue());\n            assertEquals(2, orderInfoMap.get(ExtraInfoUtil.getQueueOffsetMapKey(TOPIC, QUEUE_ID_0, 0L)).intValue());\n            assertNull(orderInfoMap.get(ExtraInfoUtil.getQueueOffsetMapKey(TOPIC, QUEUE_ID_0, 1L)));\n            assertEquals(2, orderInfoMap.get(ExtraInfoUtil.getQueueOffsetMapKey(TOPIC, QUEUE_ID_1, 0L)).intValue());\n        }\n    }\n\n    @Test\n    public void testUpdateNextVisibleTime() {\n        long invisibleTime = 3000;\n\n        StringBuilder orderInfoBuilder = new StringBuilder();\n        consumerOrderInfoManager.update(\n            null,\n            false,\n            TOPIC,\n            GROUP,\n            QUEUE_ID_0,\n            popTime,\n            1,\n            Lists.newArrayList(1L, 2L, 3L),\n            orderInfoBuilder\n        );\n\n        consumerOrderInfoManager.updateNextVisibleTime(TOPIC, GROUP, QUEUE_ID_0, 2L, popTime, System.currentTimeMillis() + invisibleTime);\n        assertEncodeAndDecode();\n\n        assertEquals(2, consumerOrderInfoManager.commitAndNext(TOPIC, GROUP, QUEUE_ID_0, 1L, popTime));\n        assertEncodeAndDecode();\n        assertEquals(2, consumerOrderInfoManager.commitAndNext(TOPIC, GROUP, QUEUE_ID_0, 3L, popTime));\n        assertEncodeAndDecode();\n\n        await().atMost(Duration.ofSeconds(invisibleTime + 1)).until(() -> !consumerOrderInfoManager.checkBlock(null, TOPIC, GROUP, QUEUE_ID_0, invisibleTime));\n\n        orderInfoBuilder = new StringBuilder();\n        consumerOrderInfoManager.update(\n            null,\n            false,\n            TOPIC,\n            GROUP,\n            QUEUE_ID_0,\n            popTime,\n            1,\n            Lists.newArrayList(2L, 3L, 4L),\n            orderInfoBuilder\n        );\n\n        consumerOrderInfoManager.updateNextVisibleTime(TOPIC, GROUP, QUEUE_ID_0, 2L, popTime, System.currentTimeMillis() + invisibleTime);\n        assertEncodeAndDecode();\n\n        assertEquals(2, consumerOrderInfoManager.commitAndNext(TOPIC, GROUP, QUEUE_ID_0, 3L, popTime));\n        assertEncodeAndDecode();\n        assertEquals(2, consumerOrderInfoManager.commitAndNext(TOPIC, GROUP, QUEUE_ID_0, 4L, popTime));\n        assertEncodeAndDecode();\n        assertTrue(consumerOrderInfoManager.checkBlock(null, TOPIC, GROUP, QUEUE_ID_0, invisibleTime));\n\n        assertEquals(5L, consumerOrderInfoManager.commitAndNext(TOPIC, GROUP, QUEUE_ID_0, 2L, popTime));\n        assertEncodeAndDecode();\n        assertFalse(consumerOrderInfoManager.checkBlock(null, TOPIC, GROUP, QUEUE_ID_0, invisibleTime));\n    }\n\n    @Test\n    public void testAutoCleanAndEncode() {\n        BrokerConfig brokerConfig = new BrokerConfig();\n        BrokerController brokerController = mock(BrokerController.class);\n        TopicConfigManager topicConfigManager = mock(TopicConfigManager.class);\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);\n\n        SubscriptionGroupManager subscriptionGroupManager = mock(SubscriptionGroupManager.class);\n        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);\n        when(subscriptionGroupManager.containsSubscriptionGroup(GROUP)).thenReturn(true);\n\n        TopicConfig topicConfig = new TopicConfig(TOPIC);\n        when(topicConfigManager.selectTopicConfig(eq(TOPIC))).thenReturn(topicConfig);\n\n        QueueLevelConsumerManager consumerOrderInfoManager = new QueueLevelConsumerManager(brokerController);\n\n        {\n            consumerOrderInfoManager.update(null, false,\n                \"errTopic\",\n                \"errGroup\",\n                QUEUE_ID_0,\n                popTime,\n                1,\n                Lists.newArrayList(2L, 3L, 4L),\n                new StringBuilder());\n\n            consumerOrderInfoManager.autoClean();\n            assertEquals(0, consumerOrderInfoManager.getTable().size());\n        }\n        {\n            consumerOrderInfoManager.update(null, false,\n                TOPIC,\n                \"errGroup\",\n                QUEUE_ID_0,\n                popTime,\n                1,\n                Lists.newArrayList(2L, 3L, 4L),\n                new StringBuilder());\n\n            consumerOrderInfoManager.autoClean();\n            assertEquals(0, consumerOrderInfoManager.getTable().size());\n        }\n        {\n            topicConfig.setReadQueueNums(0);\n            consumerOrderInfoManager.update(null, false,\n                TOPIC,\n                GROUP,\n                QUEUE_ID_0,\n                popTime,\n                1,\n                Lists.newArrayList(2L, 3L, 4L),\n                new StringBuilder());\n\n            await().atMost(Duration.ofSeconds(1)).until(() -> {\n                consumerOrderInfoManager.autoClean();\n                return consumerOrderInfoManager.getTable().size() == 0;\n            });\n        }\n        {\n            topicConfig.setReadQueueNums(8);\n            consumerOrderInfoManager.update(null, false,\n                TOPIC,\n                GROUP,\n                QUEUE_ID_0,\n                popTime,\n                1,\n                Lists.newArrayList(2L, 3L, 4L),\n                new StringBuilder());\n\n            consumerOrderInfoManager.autoClean();\n            assertEquals(1, consumerOrderInfoManager.getTable().size());\n            for (ConcurrentHashMap<Integer, QueueLevelConsumerManager.OrderInfo> orderInfoMap : consumerOrderInfoManager.getTable().values()) {\n                assertEquals(1, orderInfoMap.size());\n                assertNotNull(orderInfoMap.get(QUEUE_ID_0));\n                break;\n            }\n        }\n    }\n\n    private void assertEncodeAndDecode() {\n        QueueLevelConsumerManager.OrderInfo prevOrderInfo = consumerOrderInfoManager.getTable().values().stream().findFirst()\n            .get().get(QUEUE_ID_0);\n\n        String dataEncoded = consumerOrderInfoManager.encode();\n\n        consumerOrderInfoManager.decode(dataEncoded);\n        QueueLevelConsumerManager.OrderInfo newOrderInfo = consumerOrderInfoManager.getTable().values().stream().findFirst()\n            .get().get(QUEUE_ID_0);\n\n        assertNotSame(prevOrderInfo, newOrderInfo);\n        assertEquals(prevOrderInfo.getPopTime(), newOrderInfo.getPopTime());\n        assertEquals(prevOrderInfo.getInvisibleTime(), newOrderInfo.getInvisibleTime());\n        assertEquals(prevOrderInfo.getOffsetList(), newOrderInfo.getOffsetList());\n        assertEquals(prevOrderInfo.getOffsetConsumedCount(), newOrderInfo.getOffsetConsumedCount());\n        assertEquals(prevOrderInfo.getOffsetNextVisibleTime(), newOrderInfo.getOffsetNextVisibleTime());\n        assertEquals(prevOrderInfo.getLastConsumeTimestamp(), newOrderInfo.getLastConsumeTimestamp());\n        assertEquals(prevOrderInfo.getCommitOffsetBit(), newOrderInfo.getCommitOffsetBit());\n    }\n\n    @Test\n    public void testLoadFromOldVersionOrderInfoData() {\n        consumerOrderInfoManager.update(null, false,\n            TOPIC,\n            GROUP,\n            QUEUE_ID_0,\n            popTime,\n            1,\n            Lists.newArrayList(2L, 3L, 4L),\n            new StringBuilder());\n        QueueLevelConsumerManager.OrderInfo orderInfo = consumerOrderInfoManager.getTable().values().stream().findFirst()\n            .get().get(QUEUE_ID_0);\n\n        orderInfo.setInvisibleTime(null);\n        orderInfo.setOffsetConsumedCount(null);\n        orderInfo.setOffsetNextVisibleTime(null);\n\n        String dataEncoded = consumerOrderInfoManager.encode();\n\n        consumerOrderInfoManager.decode(dataEncoded);\n        assertTrue(consumerOrderInfoManager.checkBlock(null, TOPIC, GROUP, QUEUE_ID_0, 3000));\n\n        StringBuilder orderInfoBuilder = new StringBuilder();\n        consumerOrderInfoManager.update(null, false,\n            TOPIC,\n            GROUP,\n            QUEUE_ID_0,\n            popTime,\n            1,\n            Lists.newArrayList(3L, 4L, 5L),\n            orderInfoBuilder);\n        assertEncodeAndDecode();\n        Map<String, Integer> orderInfoMap = ExtraInfoUtil.parseOrderCountInfo(orderInfoBuilder.toString());\n        assertEquals(3, orderInfoMap.size());\n        assertEquals(0, orderInfoMap.get(ExtraInfoUtil.getStartOffsetInfoMapKey(TOPIC, QUEUE_ID_0)).intValue());\n        assertEquals(1, orderInfoMap.get(ExtraInfoUtil.getQueueOffsetMapKey(TOPIC, QUEUE_ID_0, 3)).intValue());\n        assertEquals(1, orderInfoMap.get(ExtraInfoUtil.getQueueOffsetMapKey(TOPIC, QUEUE_ID_0, 4)).intValue());\n    }\n\n    @Test\n    public void testReentrant() {\n        StringBuilder orderInfoBuilder = new StringBuilder();\n        String attemptId = UUID.randomUUID().toString();\n        consumerOrderInfoManager.update(\n            attemptId,\n            false,\n            TOPIC,\n            GROUP,\n            QUEUE_ID_0,\n            popTime,\n            3000,\n            Lists.newArrayList(1L, 2L, 3L),\n            orderInfoBuilder\n        );\n\n        assertTrue(consumerOrderInfoManager.checkBlock(null, TOPIC, GROUP, QUEUE_ID_0, 3000));\n        assertFalse(consumerOrderInfoManager.checkBlock(attemptId, TOPIC, GROUP, QUEUE_ID_0, 3000));\n    }\n\n    @Test\n    public void testGetMaxLockFreeTimestamp() {\n        QueueLevelConsumerManager.OrderInfo orderInfo = new QueueLevelConsumerManager.OrderInfo();\n        orderInfo.setOffsetList(new ArrayList<>());\n        assertNull(orderInfo.getMaxLockFreeTimestamp());\n\n        QueueLevelConsumerManager.OrderInfo nullOrderInfo = new QueueLevelConsumerManager.OrderInfo();\n        nullOrderInfo.setOffsetList(null);\n        assertNull(nullOrderInfo.getMaxLockFreeTimestamp());\n\n        List<Long> offsetList = Arrays.asList(100L, 1L, 2L);\n\n        QueueLevelConsumerManager.OrderInfo allAckOrderInfo = new QueueLevelConsumerManager.OrderInfo();\n        allAckOrderInfo.setOffsetList(offsetList);\n        allAckOrderInfo.setCommitOffsetBit(7);\n        allAckOrderInfo.setPopTime(System.currentTimeMillis());\n        allAckOrderInfo.setInvisibleTime(30000L);\n        assertEquals(System.currentTimeMillis(), allAckOrderInfo.getMaxLockFreeTimestamp(), 1000L);\n\n        QueueLevelConsumerManager.OrderInfo unackOrderInfo = new QueueLevelConsumerManager.OrderInfo();\n        unackOrderInfo.setOffsetList(offsetList);\n        unackOrderInfo.setCommitOffsetBit(0);\n        long popTime = System.currentTimeMillis();\n        unackOrderInfo.setPopTime(popTime);\n        unackOrderInfo.setInvisibleTime(30000L);\n        Long expectedTime = popTime + 30000L;\n        assertEquals(expectedTime, unackOrderInfo.getMaxLockFreeTimestamp());\n\n        QueueLevelConsumerManager.OrderInfo hasVisibleButAckedOrderInfo = new QueueLevelConsumerManager.OrderInfo();\n        hasVisibleButAckedOrderInfo.setOffsetList(offsetList);\n        hasVisibleButAckedOrderInfo.setCommitOffsetBit(1);\n        hasVisibleButAckedOrderInfo.setPopTime(popTime);\n        hasVisibleButAckedOrderInfo.setInvisibleTime(30000L);\n        Map<Long, Long> offsetNextVisibleTime = new HashMap<>();\n        offsetNextVisibleTime.put(100L, popTime + 60000L);\n        hasVisibleButAckedOrderInfo.setOffsetNextVisibleTime(offsetNextVisibleTime);\n        assertEquals(Long.valueOf(popTime + 30000L), hasVisibleButAckedOrderInfo.getMaxLockFreeTimestamp());\n\n        QueueLevelConsumerManager.OrderInfo multiUnackOrderInfo = new QueueLevelConsumerManager.OrderInfo();\n        multiUnackOrderInfo.setOffsetList(offsetList);\n        multiUnackOrderInfo.setCommitOffsetBit(0);\n        multiUnackOrderInfo.setPopTime(popTime);\n        multiUnackOrderInfo.setInvisibleTime(30000L);\n        Map<Long, Long> multiOffsetNextVisibleTime = new HashMap<>();\n        multiOffsetNextVisibleTime.put(100L, popTime + 20000L);\n        multiOffsetNextVisibleTime.put(101L, popTime + 40000L);\n        multiOffsetNextVisibleTime.put(102L, popTime + 60000L);\n        multiUnackOrderInfo.setOffsetNextVisibleTime(multiOffsetNextVisibleTime);\n        assertEquals(Long.valueOf(popTime + 60000L), multiUnackOrderInfo.getMaxLockFreeTimestamp());\n    }\n\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/processor/AckMessageProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.net.Broker2Client;\nimport org.apache.rocketmq.broker.failover.EscapeBridge;\nimport org.apache.rocketmq.broker.metrics.BrokerMetricsManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.BatchAck;\nimport org.apache.rocketmq.remoting.protocol.body.BatchAckMessageRequestBody;\nimport org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumerData;\nimport org.apache.rocketmq.store.AppendMessageResult;\nimport org.apache.rocketmq.store.AppendMessageStatus;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.lang.reflect.Field;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.BitSet;\nimport java.util.Collections;\n\nimport static org.apache.rocketmq.broker.processor.PullMessageProcessorTest.createConsumerData;\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class AckMessageProcessorTest {\n    private AckMessageProcessor ackMessageProcessor;\n    @Mock\n    private PopMessageProcessor popMessageProcessor;\n    @Spy\n    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());\n    @Mock\n    private ChannelHandlerContext handlerContext;\n    @Mock\n    private DefaultMessageStore messageStore;\n    @Mock\n    private Channel channel;\n\n    private String topic = \"FooBar\";\n    private String group = \"FooBarGroup\";\n    private ClientChannelInfo clientInfo;\n    @Mock\n    private Broker2Client broker2Client;\n\n    private static final long MIN_OFFSET_IN_QUEUE = 100;\n    private static final long MAX_OFFSET_IN_QUEUE = 999;\n\n    @Before\n    public void init() throws IllegalAccessException, NoSuchFieldException, ConsumeQueueException {\n        clientInfo = new ClientChannelInfo(channel, \"127.0.0.1\", LanguageCode.JAVA, 0);\n        brokerController.setMessageStore(messageStore);\n        Field field = BrokerController.class.getDeclaredField(\"broker2Client\");\n        field.setAccessible(true);\n        field.set(brokerController, broker2Client);\n        EscapeBridge escapeBridge = new EscapeBridge(brokerController);\n        Mockito.when(brokerController.getEscapeBridge()).thenReturn(escapeBridge);\n        \n        // Initialize BrokerMetricsManager for tests\n        Field bmmField = BrokerController.class.getDeclaredField(\"brokerMetricsManager\");\n        bmmField.setAccessible(true);\n        bmmField.set(brokerController, new BrokerMetricsManager(brokerController));\n        \n        Channel mockChannel = mock(Channel.class);\n        when(handlerContext.channel()).thenReturn(mockChannel);\n        brokerController.getTopicConfigManager().getTopicConfigTable().put(topic, new TopicConfig());\n        ConsumerData consumerData = createConsumerData(group, topic);\n        brokerController.getConsumerManager().registerConsumer(\n                consumerData.getGroupName(),\n                clientInfo,\n                consumerData.getConsumeType(),\n                consumerData.getMessageModel(),\n                consumerData.getConsumeFromWhere(),\n                consumerData.getSubscriptionDataSet(),\n                false);\n        ackMessageProcessor = new AckMessageProcessor(brokerController);\n\n        when(messageStore.getMinOffsetInQueue(anyString(), anyInt())).thenReturn(MIN_OFFSET_IN_QUEUE);\n        when(messageStore.getMaxOffsetInQueue(anyString(), anyInt())).thenReturn(MAX_OFFSET_IN_QUEUE);\n\n        when(brokerController.getPopMessageProcessor()).thenReturn(popMessageProcessor);\n    }\n\n    @Test\n    public void testProcessRequest_Success() throws RemotingCommandException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException {\n        when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));\n        PopBufferMergeService popBufferMergeService = mock(PopBufferMergeService.class);\n        when(popBufferMergeService.addAk(anyInt(), any())).thenReturn(false);\n        when(popMessageProcessor.getPopBufferMergeService()).thenReturn(popBufferMergeService);\n\n        int queueId = 0;\n        long queueOffset = 0;\n        long popTime = System.currentTimeMillis() - 1_000;\n        long invisibleTime = 30_000;\n        int reviveQid = 0;\n        String brokerName = \"test_broker\";\n        String extraInfo = ExtraInfoUtil.buildExtraInfo(queueOffset, popTime, invisibleTime, reviveQid,\n                topic, brokerName, queueId) + MessageConst.KEY_SEPARATOR + queueOffset;\n        AckMessageRequestHeader requestHeader = new AckMessageRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setQueueId(0);\n        requestHeader.setOffset(MIN_OFFSET_IN_QUEUE + 1);\n        requestHeader.setConsumerGroup(group);\n        requestHeader.setExtraInfo(extraInfo);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader);\n        request.makeCustomHeaderToNet();\n        RemotingCommand responseToReturn = ackMessageProcessor.processRequest(handlerContext, request);\n        assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(responseToReturn.getOpaque()).isEqualTo(request.getOpaque());\n    }\n\n    @Test\n    public void testProcessRequest_WrongRequestCode() throws Exception {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, null);\n        RemotingCommand response = ackMessageProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.MESSAGE_ILLEGAL);\n        assertThat(response.getRemark()).isEqualTo(\"AckMessageProcessor failed to process RequestCode: \" + RequestCode.SEND_MESSAGE);\n    }\n\n    @Test\n    public void testSingleAck_TopicCheck() throws RemotingCommandException {\n        AckMessageRequestHeader requestHeader = new AckMessageRequestHeader();\n        requestHeader.setTopic(\"wrongTopic\");\n        requestHeader.setQueueId(0);\n        requestHeader.setOffset(0L);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader);\n        request.makeCustomHeaderToNet();\n        RemotingCommand response = ackMessageProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.TOPIC_NOT_EXIST);\n        assertThat(response.getRemark()).contains(\"not exist, apply first\");\n    }\n\n    @Test\n    public void testSingleAck_QueueCheck() throws RemotingCommandException {\n        {\n            int qId = -1;\n            AckMessageRequestHeader requestHeader = new AckMessageRequestHeader();\n            requestHeader.setTopic(topic);\n            requestHeader.setQueueId(qId);\n            requestHeader.setOffset(0L);\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader);\n            request.makeCustomHeaderToNet();\n            RemotingCommand response = ackMessageProcessor.processRequest(handlerContext, request);\n            assertThat(response.getCode()).isEqualTo(ResponseCode.MESSAGE_ILLEGAL);\n            assertThat(response.getRemark()).contains(\"queueId[\" + qId + \"] is illegal\");\n        }\n\n        {\n            int qId = 17;\n            AckMessageRequestHeader requestHeader = new AckMessageRequestHeader();\n            requestHeader.setTopic(topic);\n            requestHeader.setQueueId(qId);\n            requestHeader.setOffset(0L);\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader);\n            request.makeCustomHeaderToNet();\n            RemotingCommand response = ackMessageProcessor.processRequest(handlerContext, request);\n            assertThat(response.getCode()).isEqualTo(ResponseCode.MESSAGE_ILLEGAL);\n            assertThat(response.getRemark()).contains(\"queueId[\" + qId + \"] is illegal\");\n        }\n    }\n\n    @Test\n    public void testSingleAck_OffsetCheck() throws RemotingCommandException {\n        {\n            AckMessageRequestHeader requestHeader = new AckMessageRequestHeader();\n            requestHeader.setTopic(topic);\n            requestHeader.setQueueId(0);\n            requestHeader.setOffset(MIN_OFFSET_IN_QUEUE - 1);\n            //requestHeader.setOffset(maxOffsetInQueue + 1);\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader);\n            request.makeCustomHeaderToNet();\n            RemotingCommand response = ackMessageProcessor.processRequest(handlerContext, request);\n            assertThat(response.getCode()).isEqualTo(ResponseCode.NO_MESSAGE);\n            assertThat(response.getRemark()).contains(\"offset is illegal\");\n        }\n\n        {\n            AckMessageRequestHeader requestHeader = new AckMessageRequestHeader();\n            requestHeader.setTopic(topic);\n            requestHeader.setQueueId(0);\n            //requestHeader.setOffset(minOffsetInQueue - 1);\n            requestHeader.setOffset(MAX_OFFSET_IN_QUEUE + 1);\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader);\n            request.makeCustomHeaderToNet();\n            RemotingCommand response = ackMessageProcessor.processRequest(handlerContext, request);\n            assertThat(response.getCode()).isEqualTo(ResponseCode.NO_MESSAGE);\n            assertThat(response.getRemark()).contains(\"offset is illegal\");\n        }\n    }\n\n    @Test\n    public void testBatchAck_NoMessage() throws RemotingCommandException {\n        {\n            //reqBody == null\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.BATCH_ACK_MESSAGE, null);\n            RemotingCommand response = ackMessageProcessor.processRequest(handlerContext, request);\n            assertThat(response.getCode()).isEqualTo(ResponseCode.NO_MESSAGE);\n        }\n\n        {\n            //reqBody.getAcks() == null\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.BATCH_ACK_MESSAGE, null);\n            BatchAckMessageRequestBody reqBody = new BatchAckMessageRequestBody();\n            request.setBody(reqBody.encode());\n            RemotingCommand response = ackMessageProcessor.processRequest(handlerContext, request);\n            assertThat(response.getCode()).isEqualTo(ResponseCode.NO_MESSAGE);\n        }\n\n        {\n            //reqBody.getAcks().isEmpty()\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.BATCH_ACK_MESSAGE, null);\n            BatchAckMessageRequestBody reqBody = new BatchAckMessageRequestBody();\n            reqBody.setAcks(new ArrayList<>());\n            request.setBody(reqBody.encode());\n            RemotingCommand response = ackMessageProcessor.processRequest(handlerContext, request);\n            assertThat(response.getCode()).isEqualTo(ResponseCode.NO_MESSAGE);\n        }\n    }\n\n    @Test\n    public void testSingleAck_appendAck() throws RemotingCommandException {\n        {\n            // buffer addAk OK\n            PopBufferMergeService popBufferMergeService = mock(PopBufferMergeService.class);\n            when(popBufferMergeService.addAk(anyInt(), any())).thenReturn(true);\n            when(popMessageProcessor.getPopBufferMergeService()).thenReturn(popBufferMergeService);\n\n            AckMessageRequestHeader requestHeader = new AckMessageRequestHeader();\n            long ackOffset = MIN_OFFSET_IN_QUEUE + 10;\n            requestHeader.setTopic(topic);\n            requestHeader.setQueueId(0);\n            requestHeader.setOffset(ackOffset);\n            requestHeader.setConsumerGroup(MixAll.DEFAULT_CONSUMER_GROUP);\n            requestHeader.setExtraInfo(\"64 1666860736757 60000 4 0 broker-a 0 \" + ackOffset);\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader);\n            request.makeCustomHeaderToNet();\n            RemotingCommand response = ackMessageProcessor.processRequest(handlerContext, request);\n\n            assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        }\n\n        {\n            // buffer addAk fail\n            PopBufferMergeService popBufferMergeService = mock(PopBufferMergeService.class);\n            when(popBufferMergeService.addAk(anyInt(), any())).thenReturn(false);\n            when(popMessageProcessor.getPopBufferMergeService()).thenReturn(popBufferMergeService);\n            // store putMessage OK\n            PutMessageResult putMessageResult = new PutMessageResult(PutMessageStatus.PUT_OK, null);\n            when(messageStore.putMessage(any())).thenReturn(putMessageResult);\n\n            AckMessageRequestHeader requestHeader = new AckMessageRequestHeader();\n            long ackOffset = MIN_OFFSET_IN_QUEUE + 10;\n            requestHeader.setTopic(topic);\n            requestHeader.setQueueId(0);\n            requestHeader.setOffset(ackOffset);\n            requestHeader.setConsumerGroup(MixAll.DEFAULT_CONSUMER_GROUP);\n            requestHeader.setExtraInfo(\"64 1666860736757 60000 4 0 broker-a 0 \" + ackOffset);\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader);\n            request.makeCustomHeaderToNet();\n            RemotingCommand response = ackMessageProcessor.processRequest(handlerContext, request);\n\n            assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        }\n    }\n\n    @Test\n    public void testBatchAck_appendAck() throws RemotingCommandException {\n        {\n            // buffer addAk OK\n            PopBufferMergeService popBufferMergeService = mock(PopBufferMergeService.class);\n            when(popBufferMergeService.addAk(anyInt(), any())).thenReturn(true);\n            when(popMessageProcessor.getPopBufferMergeService()).thenReturn(popBufferMergeService);\n\n            BatchAck bAck1 = new BatchAck();\n            bAck1.setConsumerGroup(MixAll.DEFAULT_CONSUMER_GROUP);\n            bAck1.setTopic(topic);\n            bAck1.setStartOffset(MIN_OFFSET_IN_QUEUE);\n            bAck1.setBitSet(new BitSet());\n            bAck1.getBitSet().set(1);\n            bAck1.setRetry(\"0\");\n\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.BATCH_ACK_MESSAGE, null);\n            BatchAckMessageRequestBody reqBody = new BatchAckMessageRequestBody();\n            reqBody.setAcks(Collections.singletonList(bAck1));\n            request.setBody(reqBody.encode());\n            request.makeCustomHeaderToNet();\n            RemotingCommand response = ackMessageProcessor.processRequest(handlerContext, request);\n\n            assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        }\n\n        {\n            // buffer addAk fail\n            PopBufferMergeService popBufferMergeService = mock(PopBufferMergeService.class);\n            when(popBufferMergeService.addAk(anyInt(), any())).thenReturn(false);\n            when(popMessageProcessor.getPopBufferMergeService()).thenReturn(popBufferMergeService);\n            // store putMessage OK\n            PutMessageResult putMessageResult = new PutMessageResult(PutMessageStatus.PUT_OK, null);\n            when(messageStore.putMessage(any())).thenReturn(putMessageResult);\n\n            BatchAck bAck1 = new BatchAck();\n            bAck1.setConsumerGroup(MixAll.DEFAULT_CONSUMER_GROUP);\n            bAck1.setTopic(topic);\n            bAck1.setStartOffset(MIN_OFFSET_IN_QUEUE);\n            bAck1.setBitSet(new BitSet());\n            bAck1.getBitSet().set(1);\n            bAck1.setRetry(\"0\");\n\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.BATCH_ACK_MESSAGE, null);\n            BatchAckMessageRequestBody reqBody = new BatchAckMessageRequestBody();\n            reqBody.setAcks(Arrays.asList(bAck1));\n            request.setBody(reqBody.encode());\n            request.makeCustomHeaderToNet();\n            RemotingCommand response = ackMessageProcessor.processRequest(handlerContext, request);\n\n            assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        }\n    }\n\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.alibaba.fastjson2.JSONObject;\nimport com.google.common.collect.ImmutableMap;\nimport com.google.common.collect.Maps;\nimport com.google.common.collect.Sets;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.auth.authentication.enums.UserType;\nimport org.apache.rocketmq.auth.authentication.manager.AuthenticationMetadataManager;\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.auth.authorization.enums.Decision;\nimport org.apache.rocketmq.auth.authorization.manager.AuthorizationMetadataManager;\nimport org.apache.rocketmq.auth.authorization.model.Acl;\nimport org.apache.rocketmq.auth.authorization.model.Environment;\nimport org.apache.rocketmq.auth.authorization.model.Resource;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ConsumerGroupInfo;\nimport org.apache.rocketmq.broker.client.ConsumerManager;\nimport org.apache.rocketmq.broker.client.net.Broker2Client;\nimport org.apache.rocketmq.broker.config.v1.RocksDBSubscriptionGroupManager;\nimport org.apache.rocketmq.broker.config.v1.RocksDBTopicConfigManager;\nimport org.apache.rocketmq.broker.metrics.BrokerMetricsManager;\nimport org.apache.rocketmq.broker.lite.LiteLifecycleManager;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.broker.schedule.ScheduleMessageService;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.TopicFilterType;\nimport org.apache.rocketmq.common.TopicQueueId;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.attribute.AttributeParser;\nimport org.apache.rocketmq.common.constant.FIleReadaheadMode;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.AclInfo;\nimport org.apache.rocketmq.remoting.protocol.body.CreateTopicListRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.GroupList;\nimport org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;\nimport org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.QueryCorrectionOffsetBody;\nimport org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.UserInfo;\nimport org.apache.rocketmq.remoting.protocol.header.CheckRocksdbCqWriteProgressRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.CreateAclRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.CreateTopicRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.CreateUserRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.DeleteAclRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.DeleteTopicRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.DeleteUserRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ExchangeHAInfoResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetAclRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetAllSubscriptionGroupRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetAllSubscriptionGroupResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetAllTopicConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetAllTopicConfigResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerStatusRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetEarliestMsgStoretimeRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMinOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetSubscriptionGroupConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetTopicConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetUserRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ListAclsRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ListUsersRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.NotifyMinBrokerIdChangeRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumeQueueRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryCorrectionOffsetHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QuerySubscriptionByConsumerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryTopicConsumeByWhoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryTopicsByConsumerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ResetMasterFlushOffsetHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ResumeCheckHalfMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SearchOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SearchOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateAclRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateUserRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicConfigAndQueueMapping;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.CommitLog;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.logfile.DefaultMappedFile;\nimport org.apache.rocketmq.store.queue.ConsumeQueueInterface;\nimport org.apache.rocketmq.store.stats.BrokerStats;\nimport org.apache.rocketmq.store.timer.TimerCheckpoint;\nimport org.apache.rocketmq.store.timer.TimerMessageStore;\nimport org.apache.rocketmq.store.timer.TimerMetrics;\nimport org.apache.rocketmq.store.util.LibC;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.io.IOException;\nimport java.lang.reflect.Field;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.net.UnknownHostException;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Optional;\nimport java.util.Properties;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.ThreadLocalRandom;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.concurrent.atomic.LongAdder;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anySet;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class AdminBrokerProcessorTest {\n\n    private AdminBrokerProcessor adminBrokerProcessor;\n\n    @Mock\n    private ChannelHandlerContext handlerContext;\n\n    @Mock\n    private Channel channel;\n\n    @Spy\n    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(),\n        new MessageStoreConfig(), null);\n\n    @Mock\n    private MessageStore messageStore;\n\n    @Mock\n    private SendMessageProcessor sendMessageProcessor;\n\n    @Mock\n    private ConcurrentMap<TopicQueueId, LongAdder> inFlyWritingCounterMap;\n\n    private Set<String> systemTopicSet;\n    private String topic;\n\n    @Mock\n    private SocketAddress socketAddress;\n    @Mock\n    private BrokerStats brokerStats;\n    @Mock\n    private TopicConfigManager topicConfigManager;\n    @Mock\n    private ConsumerManager consumerManager;\n    @Mock\n    private ConsumerOffsetManager consumerOffsetManager;\n    @Mock\n    private DefaultMessageStore defaultMessageStore;\n    @Mock\n    private ScheduleMessageService scheduleMessageService;\n    @Mock\n    private AuthenticationMetadataManager authenticationMetadataManager;\n    @Mock\n    private AuthorizationMetadataManager authorizationMetadataManager;\n\n    @Mock\n    private TimerMessageStore timerMessageStore;\n\n    @Mock\n    private TimerMetrics timerMetrics;\n\n    @Mock\n    private MessageStoreConfig messageStoreConfig;\n\n    @Mock\n    private CommitLog commitLog;\n\n    @Mock\n    private Broker2Client broker2Client;\n\n    @Mock\n    private ClientChannelInfo clientChannelInfo;\n\n    @Before\n    public void init() throws Exception {\n        brokerController.setMessageStore(messageStore);\n        brokerController.setAuthenticationMetadataManager(authenticationMetadataManager);\n        brokerController.setAuthorizationMetadataManager(authorizationMetadataManager);\n        // Initialize BrokerMetricsManager to prevent NPE in tests\n        brokerController.setBrokerMetricsManager(new BrokerMetricsManager(brokerController));\n        Field field = BrokerController.class.getDeclaredField(\"broker2Client\");\n        field.setAccessible(true);\n        field.set(brokerController, broker2Client);\n\n        //doReturn(sendMessageProcessor).when(brokerController).getSendMessageProcessor();\n\n        adminBrokerProcessor = new AdminBrokerProcessor(brokerController);\n\n        systemTopicSet = Sets.newHashSet(\n            TopicValidator.RMQ_SYS_SELF_TEST_TOPIC,\n            TopicValidator.RMQ_SYS_BENCHMARK_TOPIC,\n            TopicValidator.RMQ_SYS_SCHEDULE_TOPIC,\n            TopicValidator.RMQ_SYS_OFFSET_MOVED_EVENT,\n            TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC,\n            this.brokerController.getBrokerConfig().getBrokerClusterName(),\n            this.brokerController.getBrokerConfig().getBrokerClusterName() + \"_\" + MixAll.REPLY_TOPIC_POSTFIX);\n        if (this.brokerController.getBrokerConfig().isTraceTopicEnable()) {\n            systemTopicSet.add(this.brokerController.getBrokerConfig().getMsgTraceTopicName());\n        }\n        when(handlerContext.channel()).thenReturn(channel);\n        when(channel.remoteAddress()).thenReturn(new InetSocketAddress(\"127.0.0.1\", 12345));\n\n        topic = \"FooBar\" + System.nanoTime();\n\n        brokerController.getTopicConfigManager().getTopicConfigTable().put(topic, new TopicConfig(topic));\n        brokerController.getMessageStoreConfig().setTimerWheelEnable(false);\n        when(this.brokerController.getMessageStore().getTimerMessageStore()).thenReturn(timerMessageStore);\n        when(this.timerMessageStore.getTimerMetrics()).thenReturn(timerMetrics);\n    }\n\n    @After\n    public void destroy() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        if (brokerController.getSubscriptionGroupManager() != null) {\n            brokerController.getSubscriptionGroupManager().stop();\n        }\n        if (brokerController.getTopicConfigManager() != null) {\n            brokerController.getTopicConfigManager().stop();\n        }\n        if (brokerController.getConsumerOffsetManager() != null) {\n            brokerController.getConsumerOffsetManager().stop();\n        }\n    }\n\n    private void initRocksdbTopicManager() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        RocksDBTopicConfigManager rocksDBTopicConfigManager = new RocksDBTopicConfigManager(brokerController);\n        brokerController.setTopicConfigManager(rocksDBTopicConfigManager);\n        rocksDBTopicConfigManager.load();\n    }\n\n    private void initRocksdbSubscriptionManager() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        RocksDBSubscriptionGroupManager rocksDBSubscriptionGroupManager = new RocksDBSubscriptionGroupManager(brokerController);\n        brokerController.setSubscriptionGroupManager(rocksDBSubscriptionGroupManager);\n        rocksDBSubscriptionGroupManager.load();\n    }\n\n    @Test\n    public void testProcessRequest_success() throws RemotingCommandException, UnknownHostException {\n        RemotingCommand request = createUpdateBrokerConfigCommand();\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testProcessRequest_fail() throws RemotingCommandException, UnknownHostException {\n        RemotingCommand request = createResumeCheckHalfMessageCommand();\n        when(messageStore.selectOneMessageByOffset(any(Long.class))).thenReturn(createSelectMappedBufferResult());\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);\n    }\n\n    @Test\n    public void testUpdateAndCreateTopicInRocksdb() throws Exception {\n        if (notToBeExecuted()) {\n            return;\n        }\n        initRocksdbTopicManager();\n        testUpdateAndCreateTopic();\n    }\n\n    @Test\n    public void testUpdateAndCreateTopic() throws Exception {\n        //test system topic\n        for (String topic : systemTopicSet) {\n            RemotingCommand request = buildCreateTopicRequest(topic);\n            RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n            assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER);\n            assertThat(response.getRemark()).isEqualTo(\"The topic[\" + topic + \"] is conflict with system topic.\");\n        }\n\n        //test validate error topic\n        String topic = \"\";\n        RemotingCommand request = buildCreateTopicRequest(topic);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER);\n\n        topic = \"TEST_CREATE_TOPIC\";\n        request = buildCreateTopicRequest(topic);\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        // test deny MIXED topic type\n        brokerController.getBrokerConfig().setEnableMixedMessageType(false);\n        topic = \"TEST_MIXED_TYPE\";\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"+message.type\", \"MIXED\");\n        request = buildCreateTopicRequest(topic, attributes);\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER);\n        // test allow MIXED topic type\n        brokerController.getBrokerConfig().setEnableMixedMessageType(true);\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testUpdateAndCreateTopicList() throws RemotingCommandException {\n        List<String> systemTopicList = new ArrayList<>(systemTopicSet);\n        RemotingCommand request = buildCreateTopicListRequest(systemTopicList);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER);\n        assertThat(response.getRemark()).isEqualTo(\"The topic[\" + systemTopicList.get(0) + \"] is conflict with system topic.\");\n\n        List<String> inValidTopicList = new ArrayList<>();\n        inValidTopicList.add(\"\");\n        request = buildCreateTopicListRequest(inValidTopicList);\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER);\n\n        List<String> topicList = new ArrayList<>();\n        topicList.add(\"TEST_CREATE_TOPIC\");\n        topicList.add(\"TEST_CREATE_TOPIC1\");\n        request = buildCreateTopicListRequest(topicList);\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        //test no changes\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        // test deny MIXED topic type\n        brokerController.getBrokerConfig().setEnableMixedMessageType(false);\n        topicList.add(\"TEST_MIXED_TYPE\");\n        topicList.add(\"TEST_MIXED_TYPE1\");\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"+message.type\", \"MIXED\");\n        request = buildCreateTopicListRequest(topicList, attributes);\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER);\n        // test allow MIXED topic type\n        brokerController.getBrokerConfig().setEnableMixedMessageType(true);\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testDeleteTopicInRocksdb() throws Exception {\n        if (notToBeExecuted()) {\n            return;\n        }\n        initRocksdbTopicManager();\n        testDeleteTopic();\n    }\n\n    @Test\n    public void testDeleteTopic() throws Exception {\n        //test system topic\n        for (String topic : systemTopicSet) {\n            RemotingCommand request = buildDeleteTopicRequest(topic);\n            RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n            assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER);\n            assertThat(response.getRemark()).isEqualTo(\"The topic[\" + topic + \"] is conflict with system topic.\");\n        }\n\n        String topic = \"TEST_DELETE_TOPIC\";\n        RemotingCommand request = buildDeleteTopicRequest(topic);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testDeleteWithPopRetryTopic() throws Exception {\n        String topic = \"topicA\";\n        String anotherTopic = \"another_topicA\";\n        BrokerConfig brokerConfig = new BrokerConfig();\n\n        topicConfigManager = mock(TopicConfigManager.class);\n        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);\n        final ConcurrentHashMap<String, TopicConfig> topicConfigTable = new ConcurrentHashMap<>();\n        topicConfigTable.put(topic, new TopicConfig());\n        topicConfigTable.put(KeyBuilder.buildPopRetryTopic(topic, \"cid1\", brokerConfig.isEnableRetryTopicV2()), new TopicConfig());\n\n        topicConfigTable.put(anotherTopic, new TopicConfig());\n        topicConfigTable.put(KeyBuilder.buildPopRetryTopic(anotherTopic, \"cid2\", brokerConfig.isEnableRetryTopicV2()), new TopicConfig());\n        when(topicConfigManager.getTopicConfigTable()).thenReturn(topicConfigTable);\n        when(topicConfigManager.selectTopicConfig(anyString())).thenAnswer(invocation -> {\n            final String selectTopic = invocation.getArgument(0);\n            return topicConfigManager.getTopicConfigTable().get(selectTopic);\n        });\n\n        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);\n        when(consumerOffsetManager.whichGroupByTopic(topic)).thenReturn(Sets.newHashSet(\"cid1\"));\n\n        RemotingCommand request = buildDeleteTopicRequest(topic);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        verify(topicConfigManager).deleteTopicConfig(topic);\n        verify(topicConfigManager).deleteTopicConfig(KeyBuilder.buildPopRetryTopic(topic, \"cid1\", brokerConfig.isEnableRetryTopicV2()));\n        verify(messageStore, times(2)).deleteTopics(anySet());\n    }\n\n    @Test\n    public void testGetAllTopicConfigInRocksdb() throws Exception {\n        if (notToBeExecuted()) {\n            return;\n        }\n        initRocksdbTopicManager();\n        testGetAllTopicConfig();\n    }\n\n    @Test\n    public void testGetAllTopicConfig() throws Exception {\n        GetAllTopicConfigRequestHeader getAllTopicConfigResponseHeader = new GetAllTopicConfigRequestHeader();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_TOPIC_CONFIG, getAllTopicConfigResponseHeader);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n\n    private void getAllTopicConfig(boolean enableSplitMetadata) throws RemotingCommandException {\n        brokerController.getBrokerConfig().setEnableSplitMetadata(enableSplitMetadata);\n\n        // old client, request null\n        RemotingCommand requestOldClient = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_TOPIC_CONFIG, null);\n        RemotingCommand responseOldClient = adminBrokerProcessor.processRequest(handlerContext, requestOldClient);\n        assertThat(responseOldClient.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        TopicConfigSerializeWrapper topicConfigSerializeWrapperOldClient =\n            TopicConfigSerializeWrapper.decode(responseOldClient.getBody(), TopicConfigSerializeWrapper.class);\n        assertThat(Maps.difference(topicConfigSerializeWrapperOldClient.getTopicConfigTable(),\n            brokerController.getTopicConfigManager().getTopicConfigTable()).areEqual()).isTrue();\n\n        // new client, request seq from 0\n        AtomicBoolean dataVersionChanged = new AtomicBoolean(false);\n        int topicSeq = 0;\n        DataVersion dataVersion = null;\n        int pageSize = ThreadLocalRandom.current().nextInt(500, brokerController.getBrokerConfig().getSplitMetadataSize());\n        ConcurrentMap<String, TopicConfig> topicConfigTable = new ConcurrentHashMap<>();\n        while (true) {\n            GetAllTopicConfigRequestHeader requestHeader = new GetAllTopicConfigRequestHeader();\n            requestHeader.setTopicSeq(topicSeq);\n            requestHeader.setMaxTopicNum(pageSize);\n            requestHeader.setDataVersion(Optional.ofNullable(dataVersion).map(DataVersion::toJson).orElse(StringUtils.EMPTY));\n            RemotingCommand requestNewClient = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_TOPIC_CONFIG, requestHeader);\n            requestNewClient.makeCustomHeaderToNet();\n\n            RemotingCommand responseNewClient = adminBrokerProcessor.processRequest(handlerContext, requestNewClient);\n            GetAllTopicConfigResponseHeader responseHeader = (GetAllTopicConfigResponseHeader) responseNewClient.readCustomHeader();\n            assertThat(responseNewClient.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n            TopicConfigSerializeWrapper topicConfigSerializeWrapper =\n                TopicConfigSerializeWrapper.decode(responseNewClient.getBody(), TopicConfigSerializeWrapper.class);\n            topicSeq += topicConfigSerializeWrapper.getTopicConfigTable().size();\n\n            assertThat(responseHeader.getTotalTopicNum())\n                .isEqualTo(brokerController.getTopicConfigManager().getTopicConfigTable().size());\n            assertThat(topicConfigSerializeWrapper.getDataVersion())\n                .isEqualTo(brokerController.getTopicConfigManager().getDataVersion());\n\n            DataVersion newDataVersion = topicConfigSerializeWrapper.getDataVersion();\n            if (dataVersion == null) {\n                dataVersion = newDataVersion;\n            }\n\n            // mock server side data version changed\n            if (topicSeq > responseHeader.getTotalTopicNum() / 2 && dataVersionChanged.compareAndSet(false, true)) {\n                brokerController.getTopicConfigManager().getDataVersion().nextVersion();\n            }\n\n            if (!Objects.equals(dataVersion, newDataVersion)) {\n                dataVersion = newDataVersion;\n                topicSeq = 0;   // data version diff, from 0\n                topicConfigTable.clear();\n                continue;\n            }\n\n\n            topicConfigTable.putAll(topicConfigSerializeWrapper.getTopicConfigTable());\n            if (topicSeq >= responseHeader.getTotalTopicNum() - 1) {\n                break;\n            } else {\n                assertThat(topicConfigSerializeWrapper.getTopicConfigTable().size()).isEqualTo(pageSize);\n            }\n        }\n        assertThat(Maps.difference(topicConfigTable, brokerController.getTopicConfigManager().getTopicConfigTable()).areEqual()).isTrue();\n    }\n\n    @Test\n    public void testGetAllTopicConfigWithRequestHeader() throws RemotingCommandException {\n        // from [0, 50000)\n        fillTopicConfigTable(50000);\n\n        getAllTopicConfig(true);\n        getAllTopicConfig(false);   // broker side disable split , will return all topic config\n    }\n\n\n    private void getAllSubscriptionGroup(boolean enableSplitMetadata) throws RemotingCommandException {\n        brokerController.getBrokerConfig().setEnableSplitMetadata(enableSplitMetadata);\n\n        // old client, request null\n        RemotingCommand requestOldClient = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_SUBSCRIPTIONGROUP_CONFIG, null);\n        RemotingCommand responseOldClient = adminBrokerProcessor.processRequest(handlerContext, requestOldClient);\n        assertThat(responseOldClient.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        // new client, request from 0\n        AtomicBoolean dataVersionChanged = new AtomicBoolean(false);\n        int groupSeq = 0;\n        int pageSize = ThreadLocalRandom.current().nextInt(500, brokerController.getBrokerConfig().getSplitMetadataSize());\n        DataVersion dataVersion = null;\n        ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroupTable = new ConcurrentHashMap<>();\n        ConcurrentMap<String, ConcurrentMap<String, Integer>> forbiddenTable = new ConcurrentHashMap<>();\n        while (true) {\n            GetAllSubscriptionGroupRequestHeader requestHeader = new GetAllSubscriptionGroupRequestHeader();\n            requestHeader.setGroupSeq(groupSeq);\n            requestHeader.setMaxGroupNum(pageSize);\n            requestHeader.setDataVersion(Optional.ofNullable(dataVersion).map(DataVersion::toJson).orElse(StringUtils.EMPTY));\n            RemotingCommand requestNewClient = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_SUBSCRIPTIONGROUP_CONFIG, requestHeader);\n            requestNewClient.makeCustomHeaderToNet();\n            RemotingCommand responseNewClient = adminBrokerProcessor.processRequest(handlerContext, requestNewClient);\n            GetAllSubscriptionGroupResponseHeader responseHeader = (GetAllSubscriptionGroupResponseHeader) responseNewClient.readCustomHeader();\n            assertThat(responseNewClient.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n            SubscriptionGroupWrapper subscriptionGroupWrapper =\n                SubscriptionGroupWrapper.decode(responseNewClient.getBody(), SubscriptionGroupWrapper.class);\n\n            groupSeq += subscriptionGroupWrapper.getSubscriptionGroupTable().size();\n            DataVersion newDataVersion = subscriptionGroupWrapper.getDataVersion();\n\n            assertThat(responseHeader.getTotalGroupNum()).isEqualTo(\n                brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().size());\n            assertThat(newDataVersion).isEqualTo(brokerController.getSubscriptionGroupManager().getDataVersion());\n\n            if (dataVersion == null) {\n                dataVersion = newDataVersion;\n            }\n\n\n            // mock server side data version changed\n            if (groupSeq > responseHeader.getTotalGroupNum() / 2 && dataVersionChanged.compareAndSet(false, true)) {\n                brokerController.getSubscriptionGroupManager().getDataVersion().nextVersion();\n            }\n\n            if (!Objects.equals(dataVersion, newDataVersion)) {\n                dataVersion = newDataVersion;\n                groupSeq = 0;   // data version diff, from 0\n                subscriptionGroupTable.clear();\n                forbiddenTable.clear();\n                continue;\n            }\n\n            subscriptionGroupTable.putAll(subscriptionGroupWrapper.getSubscriptionGroupTable());\n            forbiddenTable.putAll(subscriptionGroupWrapper.getForbiddenTable());\n            if (groupSeq >= responseHeader.getTotalGroupNum() - 1) {\n                break;\n            } else {\n                assertThat(subscriptionGroupWrapper.getSubscriptionGroupTable().size()).isEqualTo(pageSize);\n            }\n        }\n        assertThat(Maps.difference(subscriptionGroupTable, brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable()).areEqual()).isTrue();\n        assertThat(Maps.difference(forbiddenTable, brokerController.getSubscriptionGroupManager().getForbiddenTable()).areEqual()).isTrue();\n    }\n\n    @Test\n    public void testGetAllSubscriptionGroupWithRequestHeader() throws RemotingCommandException {\n        fillSubscriptionGroupManager(50000);\n\n        getAllSubscriptionGroup(true);\n        getAllSubscriptionGroup(false);\n\n    }\n\n    @Test\n    public void testUpdateBrokerConfig() throws Exception {\n        handlerContext = mock(ChannelHandlerContext.class);\n        channel = mock(Channel.class);\n        when(handlerContext.channel()).thenReturn(channel);\n        socketAddress = mock(SocketAddress.class);\n        when(channel.remoteAddress()).thenReturn(socketAddress);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_BROKER_CONFIG, null);\n        Map<String, String> bodyMap = new HashMap<>();\n        bodyMap.put(\"key\", \"value\");\n        request.setBody(bodyMap.toString().getBytes());\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetBrokerConfig() throws Exception {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_CONFIG, null);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testProcessRequest_UpdateConfigPath() throws RemotingCommandException {\n        final RemotingCommand updateConfigRequest = RemotingCommand.createRequestCommand(RequestCode.UPDATE_BROKER_CONFIG, null);\n        Properties properties = new Properties();\n\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        when(ctx.channel()).thenReturn(null);\n\n        // Update allowed value\n        properties.setProperty(\"allAckInSyncStateSet\", \"true\");\n        updateConfigRequest.setBody(MixAll.properties2String(properties).getBytes(StandardCharsets.UTF_8));\n\n        RemotingCommand response = adminBrokerProcessor.processRequest(ctx, updateConfigRequest);\n\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        //update disallowed value\n        properties.clear();\n        properties.setProperty(\"brokerConfigPath\", \"test/path\");\n        updateConfigRequest.setBody(MixAll.properties2String(properties).getBytes(StandardCharsets.UTF_8));\n\n        response = adminBrokerProcessor.processRequest(ctx, updateConfigRequest);\n\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);\n        assertThat(response.getRemark()).contains(\"Can not update config in black list.\");\n\n        //update disallowed value\n        properties.clear();\n        properties.setProperty(\"configBlackList\", \"test;path\");\n        updateConfigRequest.setBody(MixAll.properties2String(properties).getBytes(StandardCharsets.UTF_8));\n\n        response = adminBrokerProcessor.processRequest(ctx, updateConfigRequest);\n\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);\n        assertThat(response.getRemark()).contains(\"Can not update config in black list.\");\n    }\n\n    @Test\n    public void testSearchOffsetByTimestamp() throws Exception {\n        messageStore = mock(MessageStore.class);\n        when(messageStore.getOffsetInQueueByTime(anyString(), anyInt(), anyLong(), any(BoundaryType.class))).thenReturn(Long.MIN_VALUE);\n        when(brokerController.getMessageStore()).thenReturn(messageStore);\n        SearchOffsetRequestHeader searchOffsetRequestHeader = new SearchOffsetRequestHeader();\n        searchOffsetRequestHeader.setTopic(\"topic\");\n        searchOffsetRequestHeader.setQueueId(0);\n        searchOffsetRequestHeader.setTimestamp(System.currentTimeMillis());\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, searchOffsetRequestHeader);\n        request.makeCustomHeaderToNet();\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testSearchOffsetByTimestampWithLiteTopic() throws Exception {\n        // Prepare test data\n        String topic = \"testTopic\";\n        String liteTopic = \"liteTestTopic\";\n        long timestamp = System.currentTimeMillis();\n        long mockOffset = 100L;\n        long mockMaxOffset = 500L;\n\n        MessageStore messageStore = mock(MessageStore.class);\n        LiteLifecycleManager liteLifecycleManager = mock(LiteLifecycleManager.class);\n        when(brokerController.getMessageStore()).thenReturn(messageStore);\n        when(brokerController.getLiteLifecycleManager()).thenReturn(liteLifecycleManager);\n\n        when(liteLifecycleManager.getMaxOffsetInQueue(anyString())).thenReturn(mockMaxOffset);\n        when(messageStore.getOffsetInQueueByTime(anyString(), anyInt(), anyLong(), any(BoundaryType.class)))\n            .thenReturn(mockOffset);\n\n        SearchOffsetRequestHeader requestHeader = new SearchOffsetRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setQueueId(0);\n        requestHeader.setTimestamp(timestamp);\n        requestHeader.setLiteTopic(liteTopic);\n        requestHeader.setBoundaryType(BoundaryType.LOWER);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, requestHeader);\n        request.makeCustomHeaderToNet();\n\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(response.readCustomHeader()).isInstanceOf(SearchOffsetResponseHeader.class);\n\n        SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader();\n        assertThat(responseHeader.getOffset()).isEqualTo(mockOffset);\n\n        // Verify that the LMQ conversion logic is correctly invoked\n        // When maxOffset > 0, the offset query operation should be executed\n        String expectedLmqTopic = LiteUtil.toLmqName(topic, liteTopic);\n        verify(liteLifecycleManager).getMaxOffsetInQueue(expectedLmqTopic);\n        verify(messageStore).getOffsetInQueueByTime(eq(expectedLmqTopic), eq(0), anyLong(), any(BoundaryType.class));\n        // Verify that queueId is correctly set to 0 (LMQ characteristic)\n        verify(messageStore).getOffsetInQueueByTime(anyString(), eq(0), anyLong(), any(BoundaryType.class));\n    }\n\n    @Test\n    public void testGetMaxOffset() throws Exception {\n        messageStore = mock(MessageStore.class);\n        when(messageStore.getMaxOffsetInQueue(anyString(), anyInt())).thenReturn(Long.MIN_VALUE);\n        when(brokerController.getMessageStore()).thenReturn(messageStore);\n        GetMaxOffsetRequestHeader getMaxOffsetRequestHeader = new GetMaxOffsetRequestHeader();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_MAX_OFFSET, getMaxOffsetRequestHeader);\n        request.addExtField(\"topic\", \"topic\");\n        request.addExtField(\"queueId\", \"0\");\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetMinOffset() throws Exception {\n        messageStore = mock(MessageStore.class);\n        when(messageStore.getMinOffsetInQueue(anyString(), anyInt())).thenReturn(Long.MIN_VALUE);\n        when(brokerController.getMessageStore()).thenReturn(messageStore);\n        GetMinOffsetRequestHeader getMinOffsetRequestHeader = new GetMinOffsetRequestHeader();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_MIN_OFFSET, getMinOffsetRequestHeader);\n        request.addExtField(\"topic\", \"topic\");\n        request.addExtField(\"queueId\", \"0\");\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetEarliestMsgStoretime() throws Exception {\n        messageStore = mock(MessageStore.class);\n        when(brokerController.getMessageStore()).thenReturn(messageStore);\n        GetEarliestMsgStoretimeRequestHeader getEarliestMsgStoretimeRequestHeader = new GetEarliestMsgStoretimeRequestHeader();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_EARLIEST_MSG_STORETIME, getEarliestMsgStoretimeRequestHeader);\n        request.addExtField(\"topic\", \"topic\");\n        request.addExtField(\"queueId\", \"0\");\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetBrokerRuntimeInfo() throws Exception {\n        brokerStats = mock(BrokerStats.class);\n        when(brokerController.getBrokerStats()).thenReturn(brokerStats);\n        when(brokerStats.getMsgPutTotalYesterdayMorning()).thenReturn(Long.MIN_VALUE);\n        when(brokerStats.getMsgPutTotalTodayMorning()).thenReturn(Long.MIN_VALUE);\n        when(brokerStats.getMsgPutTotalTodayNow()).thenReturn(Long.MIN_VALUE);\n        when(brokerStats.getMsgGetTotalTodayMorning()).thenReturn(Long.MIN_VALUE);\n        when(brokerStats.getMsgGetTotalTodayNow()).thenReturn(Long.MIN_VALUE);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_RUNTIME_INFO, null);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testLockBatchMQ() throws Exception {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, null);\n        LockBatchRequestBody lockBatchRequestBody = new LockBatchRequestBody();\n        lockBatchRequestBody.setClientId(\"1111\");\n        lockBatchRequestBody.setConsumerGroup(\"group\");\n        request.setBody(JSON.toJSON(lockBatchRequestBody).toString().getBytes());\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testUnlockBatchMQ() throws Exception {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UNLOCK_BATCH_MQ, null);\n        UnlockBatchRequestBody unlockBatchRequestBody = new UnlockBatchRequestBody();\n        unlockBatchRequestBody.setClientId(\"11111\");\n        unlockBatchRequestBody.setConsumerGroup(\"group\");\n        request.setBody(JSON.toJSON(unlockBatchRequestBody).toString().getBytes());\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testUpdateAndCreateSubscriptionGroupInRocksdb() throws Exception {\n        initRocksdbSubscriptionManager();\n        testUpdateAndCreateSubscriptionGroup();\n    }\n\n    @Test\n    public void testUpdateAndCreateSubscriptionGroup() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP, null);\n        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n        subscriptionGroupConfig.setBrokerId(1);\n        subscriptionGroupConfig.setGroupName(\"groupId\");\n        subscriptionGroupConfig.setConsumeEnable(Boolean.TRUE);\n        subscriptionGroupConfig.setConsumeBroadcastEnable(Boolean.TRUE);\n        subscriptionGroupConfig.setRetryMaxTimes(111);\n        subscriptionGroupConfig.setConsumeFromMinEnable(Boolean.TRUE);\n        request.setBody(JSON.toJSON(subscriptionGroupConfig).toString().getBytes());\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetAllSubscriptionGroupInRocksdb() throws Exception {\n        initRocksdbSubscriptionManager();\n        testGetAllSubscriptionGroup();\n    }\n\n    @Test\n    public void testGetAllSubscriptionGroup() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_SUBSCRIPTIONGROUP_CONFIG, null);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testDeleteSubscriptionGroupInRocksdb() throws Exception {\n        initRocksdbSubscriptionManager();\n        testDeleteSubscriptionGroup();\n    }\n\n    @Test\n    public void testDeleteSubscriptionGroup() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_SUBSCRIPTIONGROUP, null);\n        request.addExtField(\"groupName\", \"GID-Group-Name\");\n        request.addExtField(\"removeOffset\", \"true\");\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetTopicStatsInfo() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_STATS_INFO, null);\n        request.addExtField(\"topic\", \"topicTest\");\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.TOPIC_NOT_EXIST);\n        topicConfigManager = mock(TopicConfigManager.class);\n        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(\"topicTest\");\n        when(topicConfigManager.selectTopicConfig(anyString())).thenReturn(topicConfig);\n        RemotingCommand responseSuccess = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(responseSuccess.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetConsumerConnectionList() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_CONNECTION_LIST, null);\n        request.addExtField(\"consumerGroup\", \"GID-group-test\");\n        consumerManager = mock(ConsumerManager.class);\n        when(brokerController.getConsumerManager()).thenReturn(consumerManager);\n        ConsumerGroupInfo consumerGroupInfo = new ConsumerGroupInfo(\"GID-group-test\", ConsumeType.CONSUME_ACTIVELY, MessageModel.CLUSTERING, ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        when(consumerManager.getConsumerGroupInfo(anyString())).thenReturn(consumerGroupInfo);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetProducerConnectionList() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_PRODUCER_CONNECTION_LIST, null);\n        request.addExtField(\"producerGroup\", \"ProducerGroupId\");\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);\n    }\n\n    @Test\n    public void testGetAllProducerInfo() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_PRODUCER_INFO, null);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetConsumeStats() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUME_STATS, null);\n        request.addExtField(\"topic\", \"topicTest\");\n        request.addExtField(\"consumerGroup\", \"GID-test\");\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetAllConsumerOffset() throws RemotingCommandException {\n        consumerOffsetManager = mock(ConsumerOffsetManager.class);\n        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);\n        ConsumerOffsetManager consumerOffset = new ConsumerOffsetManager();\n        when(consumerOffsetManager.encode()).thenReturn(JSON.toJSONString(consumerOffset));\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_CONSUMER_OFFSET, null);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetAllDelayOffset() throws Exception {\n        defaultMessageStore = mock(DefaultMessageStore.class);\n        scheduleMessageService = mock(ScheduleMessageService.class);\n//        when(brokerController.getMessageStore()).thenReturn(defaultMessageStore);\n        when(brokerController.getScheduleMessageService()).thenReturn(scheduleMessageService);\n        when(scheduleMessageService.encode()).thenReturn(\"content\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_DELAY_OFFSET, null);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetTopicConfigInRocksdb() throws Exception {\n        if (notToBeExecuted()) {\n            return;\n        }\n        initRocksdbTopicManager();\n        testGetTopicConfig();\n    }\n\n    @Test\n    public void testGetTopicConfig() throws Exception {\n        String topic = \"foobar\";\n\n        brokerController.getTopicConfigManager().getTopicConfigTable().put(topic, new TopicConfig(topic));\n\n        {\n            GetTopicConfigRequestHeader requestHeader = new GetTopicConfigRequestHeader();\n            requestHeader.setTopic(topic);\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_CONFIG, requestHeader);\n            request.makeCustomHeaderToNet();\n            RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n            assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n            assertThat(response.getBody()).isNotEmpty();\n        }\n        {\n            GetTopicConfigRequestHeader requestHeader = new GetTopicConfigRequestHeader();\n            requestHeader.setTopic(\"aaaaaaa\");\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_CONFIG, requestHeader);\n            request.makeCustomHeaderToNet();\n            RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n            assertThat(response.getCode()).isEqualTo(ResponseCode.TOPIC_NOT_EXIST);\n            assertThat(response.getRemark()).contains(\"No topic in this broker.\");\n        }\n    }\n\n    @Test\n    public void testCreateUser() throws RemotingCommandException {\n        when(authenticationMetadataManager.createUser(any(User.class)))\n            .thenReturn(CompletableFuture.completedFuture(null));\n\n        CreateUserRequestHeader createUserRequestHeader = new CreateUserRequestHeader();\n        createUserRequestHeader.setUsername(\"abc\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_CREATE_USER, createUserRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        UserInfo userInfo = UserInfo.of(\"abc\", \"123\", UserType.NORMAL.getName());\n        request.setBody(JSON.toJSONBytes(userInfo));\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        when(authenticationMetadataManager.isSuperUser(eq(\"rocketmq\"))).thenReturn(CompletableFuture.completedFuture(true));\n        createUserRequestHeader = new CreateUserRequestHeader();\n        createUserRequestHeader.setUsername(\"super\");\n        request = RemotingCommand.createRequestCommand(RequestCode.AUTH_CREATE_USER, createUserRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        userInfo = UserInfo.of(\"super\", \"123\", UserType.SUPER.getName());\n        request.setBody(JSON.toJSONBytes(userInfo));\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        when(authenticationMetadataManager.isSuperUser(eq(\"rocketmq\"))).thenReturn(CompletableFuture.completedFuture(false));\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);\n    }\n\n    @Test\n    public void testUpdateUser() throws RemotingCommandException {\n        when(authenticationMetadataManager.updateUser(any(User.class)))\n            .thenReturn(CompletableFuture.completedFuture(null));\n        when(authenticationMetadataManager.getUser(eq(\"abc\"))).thenReturn(CompletableFuture.completedFuture(User.of(\"abc\", \"123\", UserType.NORMAL)));\n        when(authenticationMetadataManager.getUser(eq(\"super\"))).thenReturn(CompletableFuture.completedFuture(User.of(\"super\", \"123\", UserType.SUPER)));\n\n        UpdateUserRequestHeader updateUserRequestHeader = new UpdateUserRequestHeader();\n        updateUserRequestHeader.setUsername(\"abc\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_UPDATE_USER, updateUserRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        UserInfo userInfo = UserInfo.of(\"abc\", \"123\", UserType.NORMAL.getName());\n        request.setBody(JSON.toJSONBytes(userInfo));\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        when(authenticationMetadataManager.isSuperUser(eq(\"rocketmq\"))).thenReturn(CompletableFuture.completedFuture(true));\n        updateUserRequestHeader = new UpdateUserRequestHeader();\n        updateUserRequestHeader.setUsername(\"super\");\n        request = RemotingCommand.createRequestCommand(RequestCode.AUTH_UPDATE_USER, updateUserRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        userInfo = UserInfo.of(\"super\", \"123\", UserType.SUPER.getName());\n        request.setBody(JSON.toJSONBytes(userInfo));\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        when(authenticationMetadataManager.isSuperUser(eq(\"rocketmq\"))).thenReturn(CompletableFuture.completedFuture(false));\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);\n    }\n\n    @Test\n    public void testDeleteUser() throws RemotingCommandException {\n        when(authenticationMetadataManager.deleteUser(any(String.class)))\n            .thenReturn(CompletableFuture.completedFuture(null));\n        when(authenticationMetadataManager.getUser(eq(\"abc\"))).thenReturn(CompletableFuture.completedFuture(User.of(\"abc\", \"123\", UserType.NORMAL)));\n        when(authenticationMetadataManager.getUser(eq(\"super\"))).thenReturn(CompletableFuture.completedFuture(User.of(\"super\", \"123\", UserType.SUPER)));\n\n        DeleteUserRequestHeader deleteUserRequestHeader = new DeleteUserRequestHeader();\n        deleteUserRequestHeader.setUsername(\"abc\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_DELETE_USER, deleteUserRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        when(authenticationMetadataManager.isSuperUser(eq(\"rocketmq\"))).thenReturn(CompletableFuture.completedFuture(true));\n        deleteUserRequestHeader = new DeleteUserRequestHeader();\n        deleteUserRequestHeader.setUsername(\"super\");\n        request = RemotingCommand.createRequestCommand(RequestCode.AUTH_DELETE_USER, deleteUserRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        when(authenticationMetadataManager.isSuperUser(eq(\"rocketmq\"))).thenReturn(CompletableFuture.completedFuture(false));\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);\n    }\n\n    @Test\n    public void testGetUser() throws RemotingCommandException {\n        when(authenticationMetadataManager.getUser(eq(\"abc\"))).thenReturn(CompletableFuture.completedFuture(User.of(\"abc\", \"123\", UserType.NORMAL)));\n\n        GetUserRequestHeader getUserRequestHeader = new GetUserRequestHeader();\n        getUserRequestHeader.setUsername(\"abc\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_GET_USER, getUserRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        UserInfo userInfo = JSON.parseObject(new String(response.getBody()), UserInfo.class);\n        assertThat(userInfo.getUsername()).isEqualTo(\"abc\");\n        assertThat(userInfo.getPassword()).isEqualTo(\"123\");\n        assertThat(userInfo.getUserType()).isEqualTo(\"Normal\");\n    }\n\n    @Test\n    public void testListUser() throws RemotingCommandException {\n        when(authenticationMetadataManager.listUser(eq(\"abc\"))).thenReturn(CompletableFuture.completedFuture(Arrays.asList(User.of(\"abc\", \"123\", UserType.NORMAL))));\n\n        ListUsersRequestHeader listUserRequestHeader = new ListUsersRequestHeader();\n        listUserRequestHeader.setFilter(\"abc\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_LIST_USER, listUserRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        List<UserInfo> userInfo = JSON.parseArray(new String(response.getBody()), UserInfo.class);\n        assertThat(userInfo.get(0).getUsername()).isEqualTo(\"abc\");\n        assertThat(userInfo.get(0).getPassword()).isEqualTo(\"123\");\n        assertThat(userInfo.get(0).getUserType()).isEqualTo(\"Normal\");\n    }\n\n    @Test\n    public void testCreateAcl() throws RemotingCommandException {\n        when(authorizationMetadataManager.createAcl(any(Acl.class)))\n            .thenReturn(CompletableFuture.completedFuture(null));\n\n        CreateAclRequestHeader createAclRequestHeader = new CreateAclRequestHeader();\n        createAclRequestHeader.setSubject(\"User:abc\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_CREATE_ACL, createAclRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        AclInfo aclInfo = AclInfo.of(\"User:abc\", Arrays.asList(\"Topic:*\"), Arrays.asList(\"PUB\"), Arrays.asList(\"192.168.0.1\"), \"Grant\");\n        request.setBody(JSON.toJSONBytes(aclInfo));\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testUpdateAcl() throws RemotingCommandException {\n        when(authorizationMetadataManager.updateAcl(any(Acl.class)))\n            .thenReturn(CompletableFuture.completedFuture(null));\n\n        UpdateAclRequestHeader updateAclRequestHeader = new UpdateAclRequestHeader();\n        updateAclRequestHeader.setSubject(\"User:abc\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_UPDATE_ACL, updateAclRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        AclInfo aclInfo = AclInfo.of(\"User:abc\", Arrays.asList(\"Topic:*\"), Arrays.asList(\"PUB\"), Arrays.asList(\"192.168.0.1\"), \"Grant\");\n        request.setBody(JSON.toJSONBytes(aclInfo));\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testDeleteAcl() throws RemotingCommandException {\n        when(authorizationMetadataManager.deleteAcl(any(), any(), any()))\n            .thenReturn(CompletableFuture.completedFuture(null));\n\n        DeleteAclRequestHeader deleteAclRequestHeader = new DeleteAclRequestHeader();\n        deleteAclRequestHeader.setSubject(\"User:abc\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_DELETE_ACL, deleteAclRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetAcl() throws RemotingCommandException {\n        Acl aclInfo = Acl.of(User.of(\"abc\"), Arrays.asList(Resource.of(\"Topic:*\")), Arrays.asList(Action.PUB), Environment.of(\"192.168.0.1\"), Decision.ALLOW);\n        when(authorizationMetadataManager.getAcl(any(Subject.class))).thenReturn(CompletableFuture.completedFuture(aclInfo));\n\n        GetAclRequestHeader getAclRequestHeader = new GetAclRequestHeader();\n        getAclRequestHeader.setSubject(\"User:abc\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_GET_ACL, getAclRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        AclInfo aclInfoData = JSON.parseObject(new String(response.getBody()), AclInfo.class);\n        assertThat(aclInfoData.getSubject()).isEqualTo(\"User:abc\");\n        assertThat(aclInfoData.getPolicies().get(0).getEntries().get(0).getResource()).isEqualTo(\"Topic:*\");\n        assertThat(aclInfoData.getPolicies().get(0).getEntries().get(0).getActions()).containsAll(Arrays.asList(Action.PUB.getName()));\n        assertThat(aclInfoData.getPolicies().get(0).getEntries().get(0).getSourceIps()).containsAll(Arrays.asList(\"192.168.0.1\"));\n        assertThat(aclInfoData.getPolicies().get(0).getEntries().get(0).getDecision()).isEqualTo(\"Allow\");\n    }\n\n    @Test\n    public void testListAcl() throws RemotingCommandException {\n        Acl aclInfo = Acl.of(User.of(\"abc\"), Arrays.asList(Resource.of(\"Topic:*\")), Arrays.asList(Action.PUB), Environment.of(\"192.168.0.1\"), Decision.ALLOW);\n        when(authorizationMetadataManager.listAcl(any(), any())).thenReturn(CompletableFuture.completedFuture(Arrays.asList(aclInfo)));\n\n        ListAclsRequestHeader listAclRequestHeader = new ListAclsRequestHeader();\n        listAclRequestHeader.setSubjectFilter(\"User:abc\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_LIST_ACL, listAclRequestHeader);\n        request.setVersion(441);\n        request.addExtField(\"AccessKey\", \"rocketmq\");\n        request.makeCustomHeaderToNet();\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        List<AclInfo> aclInfoData = JSON.parseArray(new String(response.getBody()), AclInfo.class);\n        assertThat(aclInfoData.get(0).getSubject()).isEqualTo(\"User:abc\");\n        assertThat(aclInfoData.get(0).getPolicies().get(0).getEntries().get(0).getResource()).isEqualTo(\"Topic:*\");\n        assertThat(aclInfoData.get(0).getPolicies().get(0).getEntries().get(0).getActions()).containsAll(Arrays.asList(Action.PUB.getName()));\n        assertThat(aclInfoData.get(0).getPolicies().get(0).getEntries().get(0).getSourceIps()).containsAll(Arrays.asList(\"192.168.0.1\"));\n        assertThat(aclInfoData.get(0).getPolicies().get(0).getEntries().get(0).getDecision()).isEqualTo(\"Allow\");\n    }\n\n    @Test\n    public void testGetTimeCheckPoint() throws RemotingCommandException {\n        when(this.brokerController.getTimerCheckpoint()).thenReturn(null);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TIMER_CHECK_POINT, null);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);\n        assertThat(response.getRemark()).isEqualTo(\"The checkpoint is null\");\n\n        when(this.brokerController.getTimerCheckpoint()).thenReturn(new TimerCheckpoint());\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n    @Test\n    public void testGetTimeMetrics() throws RemotingCommandException, IOException {\n        when(this.brokerController.getMessageStore().getTimerMessageStore()).thenReturn(null);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TIMER_METRICS, null);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);\n\n        when(this.brokerController.getMessageStore().getTimerMessageStore()).thenReturn(timerMessageStore);\n        when(this.timerMessageStore.getTimerMetrics()).thenReturn(timerMetrics);\n        when(this.timerMetrics.encode()).thenReturn(new TimerMetrics.TimerMetricsSerializeWrapper().toJson(false));\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testUpdateColdDataFlowCtrGroupConfig() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_COLD_DATA_FLOW_CTR_CONFIG, null);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        request.setBody(\"consumerGroup1=1\".getBytes());\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        request.setBody(\"\".getBytes());\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testRemoveColdDataFlowCtrGroupConfig() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.REMOVE_COLD_DATA_FLOW_CTR_CONFIG, null);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        request.setBody(\"consumerGroup1\".getBytes());\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetColdDataFlowCtrInfo() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_COLD_DATA_FLOW_CTR_INFO, null);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testSetCommitLogReadAheadMode() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SET_COMMITLOG_READ_MODE, null);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);\n\n        HashMap<String, String> extfields = new HashMap<>();\n        extfields.put(FIleReadaheadMode.READ_AHEAD_MODE, String.valueOf(LibC.MADV_DONTNEED));\n        request.setExtFields(extfields);\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER);\n\n        extfields.clear();\n        extfields.put(FIleReadaheadMode.READ_AHEAD_MODE, String.valueOf(LibC.MADV_NORMAL));\n        request.setExtFields(extfields);\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        this.brokerController.setMessageStore(defaultMessageStore);\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);\n\n        when(this.defaultMessageStore.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n        when(this.defaultMessageStore.getCommitLog()).thenReturn(commitLog);\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetUnknownCmdResponse() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(10000, null);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.REQUEST_CODE_NOT_SUPPORTED);\n    }\n\n    @Test\n    public void testGetAllMessageRequestMode() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_MESSAGE_REQUEST_MODE, null);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testResetOffset() throws RemotingCommandException {\n        ResetOffsetRequestHeader requestHeader =\n                createRequestHeader(\"topic\",\"group\",-1,false,-1,-1);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.INVOKE_BROKER_TO_RESET_OFFSET, requestHeader);\n        request.makeCustomHeaderToNet();\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.TOPIC_NOT_EXIST);\n\n        this.brokerController.getTopicConfigManager().getTopicConfigTable().put(\"topic\", new TopicConfig(\"topic\"));\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);\n\n        this.brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().put(\"group\", new SubscriptionGroupConfig());\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        requestHeader.setQueueId(0);\n        request = RemotingCommand.createRequestCommand(RequestCode.INVOKE_BROKER_TO_RESET_OFFSET, requestHeader);\n        request.makeCustomHeaderToNet();\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        requestHeader.setOffset(2L);\n        request = RemotingCommand.createRequestCommand(RequestCode.INVOKE_BROKER_TO_RESET_OFFSET, requestHeader);\n        request.makeCustomHeaderToNet();\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);\n    }\n\n    @Test\n    public void testGetConsumerStatus() throws RemotingCommandException {\n        GetConsumerStatusRequestHeader requestHeader = new GetConsumerStatusRequestHeader();\n        requestHeader.setGroup(\"group\");\n        requestHeader.setTopic(\"topic\");\n        requestHeader.setClientAddr(\"\");\n        RemotingCommand request = RemotingCommand\n                .createRequestCommand(RequestCode.INVOKE_BROKER_TO_GET_CONSUMER_STATUS, requestHeader);\n        RemotingCommand responseCommand = RemotingCommand.createResponseCommand(null);\n        responseCommand.setCode(ResponseCode.SUCCESS);\n        when(broker2Client.getConsumeStatus(anyString(),anyString(),anyString())).thenReturn(responseCommand);\n        request.makeCustomHeaderToNet();\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testQueryTopicConsumeByWho() throws RemotingCommandException {\n        QueryTopicConsumeByWhoRequestHeader requestHeader = new QueryTopicConsumeByWhoRequestHeader();\n        requestHeader.setTopic(\"topic\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_TOPIC_CONSUME_BY_WHO, requestHeader);\n        request.makeCustomHeaderToNet();\n        HashSet<String> groups = new HashSet<>();\n        groups.add(\"group\");\n        when(brokerController.getConsumerManager()).thenReturn(consumerManager);\n        when(consumerManager.queryTopicConsumeByWho(anyString())).thenReturn(groups);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(RemotingSerializable.decode(response.getBody(), GroupList.class)\n                .getGroupList().contains(\"group\"))\n                .isEqualTo(groups.contains(\"group\"));\n    }\n\n    @Test\n    public void testQueryTopicByConsumer() throws RemotingCommandException {\n        QueryTopicsByConsumerRequestHeader requestHeader = new QueryTopicsByConsumerRequestHeader();\n        requestHeader.setGroup(\"group\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_TOPICS_BY_CONSUMER, requestHeader);\n        request.makeCustomHeaderToNet();\n        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testQuerySubscriptionByConsumer() throws RemotingCommandException {\n        QuerySubscriptionByConsumerRequestHeader requestHeader = new QuerySubscriptionByConsumerRequestHeader();\n        requestHeader.setGroup(\"group\");\n        requestHeader.setTopic(\"topic\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_SUBSCRIPTION_BY_CONSUMER, requestHeader);\n        request.makeCustomHeaderToNet();\n        when(brokerController.getConsumerManager()).thenReturn(consumerManager);\n        when(consumerManager.findSubscriptionData(anyString(),anyString())).thenReturn(null);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetSystemTopicListFromBroker() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_SYSTEM_TOPIC_LIST_FROM_BROKER, null);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testCleanExpiredConsumeQueue() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CLEAN_EXPIRED_CONSUMEQUEUE, null);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testDeleteExpiredCommitLog() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_EXPIRED_COMMITLOG, null);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testCleanUnusedTopic() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CLEAN_UNUSED_TOPIC, null);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetConsumerRunningInfo() throws RemotingCommandException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException {\n        when(brokerController.getConsumerManager()).thenReturn(consumerManager);\n        when(consumerManager.findChannel(anyString(),anyString())).thenReturn(null);\n        GetConsumerRunningInfoRequestHeader requestHeader = new GetConsumerRunningInfoRequestHeader();\n        requestHeader.setClientId(\"client\");\n        requestHeader.setConsumerGroup(\"group\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_RUNNING_INFO, requestHeader);\n        request.makeCustomHeaderToNet();\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);\n\n        when(consumerManager.findChannel(anyString(),anyString())).thenReturn(clientChannelInfo);\n        when(clientChannelInfo.getVersion()).thenReturn(MQVersion.Version.V3_0_0_SNAPSHOT.ordinal());\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);\n\n        when(clientChannelInfo.getVersion()).thenReturn(MQVersion.Version.V5_2_3.ordinal());\n        when(brokerController.getBroker2Client()).thenReturn(broker2Client);\n        when(clientChannelInfo.getChannel()).thenReturn(channel);\n        RemotingCommand responseCommand = RemotingCommand.createResponseCommand(null);\n        responseCommand.setCode(ResponseCode.SUCCESS);\n        when(broker2Client.callClient(any(Channel.class),any(RemotingCommand.class))).thenReturn(responseCommand);\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        when(broker2Client.callClient(any(Channel.class),any(RemotingCommand.class))).thenThrow(new RemotingTimeoutException(\"timeout\"));\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.CONSUME_MSG_TIMEOUT);\n    }\n\n    @Test\n    public void testQueryCorrectionOffset() throws RemotingCommandException {\n        Map<Integer, Long> correctionOffsetMap = new HashMap<>();\n        correctionOffsetMap.put(0, 100L);\n        correctionOffsetMap.put(1, 200L);\n        Map<Integer, Long> compareOffsetMap = new HashMap<>();\n        compareOffsetMap.put(0, 80L);\n        compareOffsetMap.put(1, 300L);\n        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);\n        when(consumerOffsetManager.queryMinOffsetInAllGroup(anyString(),anyString())).thenReturn(correctionOffsetMap);\n        when(consumerOffsetManager.queryOffset(anyString(),anyString())).thenReturn(compareOffsetMap);\n        QueryCorrectionOffsetHeader queryCorrectionOffsetHeader = new QueryCorrectionOffsetHeader();\n        queryCorrectionOffsetHeader.setTopic(\"topic\");\n        queryCorrectionOffsetHeader.setCompareGroup(\"group\");\n        queryCorrectionOffsetHeader.setFilterGroups(\"\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CORRECTION_OFFSET, queryCorrectionOffsetHeader);\n        request.makeCustomHeaderToNet();\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        QueryCorrectionOffsetBody body = RemotingSerializable.decode(response.getBody(), QueryCorrectionOffsetBody.class);\n        Map<Integer, Long> correctionOffsets = body.getCorrectionOffsets();\n        assertThat(correctionOffsets.get(0)).isEqualTo(Long.MAX_VALUE);\n        assertThat(correctionOffsets.get(1)).isEqualTo(200L);\n    }\n\n    @Test\n    public void testNotifyMinBrokerIdChange() throws RemotingCommandException {\n        NotifyMinBrokerIdChangeRequestHeader requestHeader = new NotifyMinBrokerIdChangeRequestHeader();\n        requestHeader.setMinBrokerId(1L);\n        requestHeader.setMinBrokerAddr(\"127.0.0.1:10912\");\n        requestHeader.setOfflineBrokerAddr(\"127.0.0.1:10911\");\n        requestHeader.setHaBrokerAddr(\"\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.NOTIFY_MIN_BROKER_ID_CHANGE, requestHeader);\n        request.makeCustomHeaderToNet();\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testUpdateBrokerHaInfo() throws RemotingCommandException {\n        ExchangeHAInfoResponseHeader requestHeader = new ExchangeHAInfoResponseHeader();\n        requestHeader.setMasterAddress(\"127.0.0.1:10911\");\n        requestHeader.setMasterFlushOffset(0L);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.EXCHANGE_BROKER_HA_INFO, requestHeader);\n        request.makeCustomHeaderToNet();\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        when(brokerController.getMessageStore()).thenReturn(messageStore);\n        requestHeader.setMasterHaAddress(\"127.0.0.1:10912\");\n        request = RemotingCommand.createRequestCommand(RequestCode.EXCHANGE_BROKER_HA_INFO, requestHeader);\n        request.makeCustomHeaderToNet();\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        when(messageStore.getMasterFlushedOffset()).thenReturn(0L);\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetBrokerHaStatus() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_HA_STATUS,null);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);\n\n        when(brokerController.getMessageStore()).thenReturn(messageStore);\n        when(messageStore.getHARuntimeInfo()).thenReturn(new HARuntimeInfo());\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testResetMasterFlushOffset() throws RemotingCommandException {\n        ResetMasterFlushOffsetHeader requestHeader = new ResetMasterFlushOffsetHeader();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RESET_MASTER_FLUSH_OFFSET,requestHeader);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        requestHeader.setMasterFlushOffset(0L);\n        request.makeCustomHeaderToNet();\n        response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetSubscriptionGroup() throws RemotingCommandException {\n        brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().put(\"group\", new SubscriptionGroupConfig());\n        GetSubscriptionGroupConfigRequestHeader requestHeader = new GetSubscriptionGroupConfigRequestHeader();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_SUBSCRIPTIONGROUP_CONFIG, requestHeader);\n        requestHeader.setGroup(\"group\");\n        request.makeCustomHeaderToNet();\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n    }\n\n    @Test\n    public void testCheckRocksdbCqWriteProgress() throws RemotingCommandException {\n        CheckRocksdbCqWriteProgressRequestHeader requestHeader = new CheckRocksdbCqWriteProgressRequestHeader();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHECK_ROCKSDB_CQ_WRITE_PROGRESS, requestHeader);\n        requestHeader.setTopic(\"topic\");\n        request.makeCustomHeaderToNet();\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n    }\n\n    @Test\n    public void testQueryConsumeQueue() throws RemotingCommandException {\n        messageStore = mock(MessageStore.class);\n        ConsumeQueueInterface consumeQueue = mock(ConsumeQueueInterface.class);\n        when(consumeQueue.getMinOffsetInQueue()).thenReturn(0L);\n        when(consumeQueue.getMaxOffsetInQueue()).thenReturn(1L);\n        when(messageStore.getConsumeQueue(anyString(), anyInt())).thenReturn(consumeQueue);\n        when(brokerController.getMessageStore()).thenReturn(messageStore);\n        QueryConsumeQueueRequestHeader requestHeader = new QueryConsumeQueueRequestHeader();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUME_QUEUE, requestHeader);\n        requestHeader.setTopic(\"topic\");\n        requestHeader.setQueueId(0);\n        requestHeader.setConsumerGroup(\"testGroup\");\n        request.makeCustomHeaderToNet();\n        SubscriptionData subscriptionData = mock(SubscriptionData.class);\n        when(brokerController.getConsumerManager()).thenReturn(consumerManager);\n        when(consumerManager.findSubscriptionData(any(), any())).thenReturn(subscriptionData);\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n    }\n\n    @Test\n    public void testProcessRequest_GetTopicConfig() throws Exception {\n        GetTopicConfigRequestHeader requestHeader = new GetTopicConfigRequestHeader();\n        requestHeader.setTopic(\"testTopic\");\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_CONFIG, requestHeader);\n        request.makeCustomHeaderToNet();\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(\"testTopic\");\n        TopicConfigManager topicConfigManager = mock(TopicConfigManager.class);\n        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);\n        when(topicConfigManager.selectTopicConfig(\"testTopic\"))\n                .thenReturn(topicConfig);\n\n        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);\n\n        assertNotNull(response);\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n\n        String responseBody = new String(response.getBody(), StandardCharsets.UTF_8);\n        TopicConfigAndQueueMapping result = JSONObject.parseObject(responseBody, TopicConfigAndQueueMapping.class);\n        assertEquals(\"testTopic\", result.getTopicName());\n    }\n\n    private ResetOffsetRequestHeader createRequestHeader(String topic,String group,long timestamp,boolean force,long offset,int queueId) {\n        ResetOffsetRequestHeader requestHeader = new ResetOffsetRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setGroup(group);\n        requestHeader.setTimestamp(timestamp);\n        requestHeader.setForce(force);\n        requestHeader.setOffset(offset);\n        requestHeader.setQueueId(queueId);\n        return requestHeader;\n    }\n\n    private RemotingCommand buildCreateTopicRequest(String topic) {\n        return buildCreateTopicRequest(topic, null);\n    }\n\n    private RemotingCommand buildCreateTopicRequest(String topic, Map<String, String> attributes) {\n        CreateTopicRequestHeader requestHeader = new CreateTopicRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setTopicFilterType(TopicFilterType.SINGLE_TAG.name());\n        requestHeader.setReadQueueNums(8);\n        requestHeader.setWriteQueueNums(8);\n        requestHeader.setPerm(PermName.PERM_READ | PermName.PERM_WRITE);\n        if (attributes != null) {\n            requestHeader.setAttributes(AttributeParser.parseToString(attributes));\n        }\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_TOPIC, requestHeader);\n        request.makeCustomHeaderToNet();\n        return request;\n    }\n\n    private RemotingCommand buildCreateTopicListRequest(List<String> topicList) {\n        return buildCreateTopicListRequest(topicList, null);\n    }\n\n    private RemotingCommand buildCreateTopicListRequest(List<String> topicList, Map<String, String> attributes) {\n        List<TopicConfig> topicConfigList = new ArrayList<>();\n        for (String topic:topicList) {\n            TopicConfig topicConfig = new TopicConfig(topic);\n            topicConfig.setReadQueueNums(8);\n            topicConfig.setWriteQueueNums(8);\n            topicConfig.setTopicFilterType(TopicFilterType.SINGLE_TAG);\n            topicConfig.setPerm(PermName.PERM_READ | PermName.PERM_WRITE);\n            topicConfig.setTopicSysFlag(0);\n            topicConfig.setOrder(false);\n            if (attributes != null) {\n                topicConfig.setAttributes(new HashMap<>(attributes));\n            }\n            topicConfigList.add(topicConfig);\n        }\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_TOPIC_LIST, null);\n        CreateTopicListRequestBody createTopicListRequestBody = new CreateTopicListRequestBody(topicConfigList);\n        request.setBody(createTopicListRequestBody.encode());\n        return request;\n    }\n\n    private RemotingCommand buildDeleteTopicRequest(String topic) {\n        DeleteTopicRequestHeader requestHeader = new DeleteTopicRequestHeader();\n        requestHeader.setTopic(topic);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_TOPIC_IN_BROKER, requestHeader);\n        request.makeCustomHeaderToNet();\n        return request;\n    }\n\n    private MessageExt createDefaultMessageExt() {\n        MessageExt messageExt = new MessageExt();\n        messageExt.setMsgId(\"12345678\");\n        messageExt.setQueueId(0);\n        messageExt.setCommitLogOffset(123456789L);\n        messageExt.setQueueOffset(1234);\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_REAL_QUEUE_ID, \"0\");\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_REAL_TOPIC, \"testTopic\");\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES, \"15\");\n        return messageExt;\n    }\n\n    private SelectMappedBufferResult createSelectMappedBufferResult() {\n        SelectMappedBufferResult result = new SelectMappedBufferResult(0, ByteBuffer.allocate(1024), 0, new DefaultMappedFile());\n        return result;\n    }\n\n    private ResumeCheckHalfMessageRequestHeader createResumeCheckHalfMessageRequestHeader() {\n        ResumeCheckHalfMessageRequestHeader header = new ResumeCheckHalfMessageRequestHeader();\n        header.setTopic(\"topic\");\n        header.setMsgId(\"C0A803CA00002A9F0000000000031367\");\n        return header;\n    }\n\n    private RemotingCommand createResumeCheckHalfMessageCommand() {\n        ResumeCheckHalfMessageRequestHeader header = createResumeCheckHalfMessageRequestHeader();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RESUME_CHECK_HALF_MESSAGE, header);\n        request.makeCustomHeaderToNet();\n        return request;\n    }\n\n    private RemotingCommand createUpdateBrokerConfigCommand() {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_BROKER_CONFIG, null);\n        request.makeCustomHeaderToNet();\n        return request;\n    }\n\n    private boolean notToBeExecuted() {\n        return MixAll.isMac();\n    }\n\n    private void fillTopicConfigTable(int num) {\n        for (int i = num - 1; i >= 0; i--) {\n            String topicName = String.format(\"topic%05d\", i);\n            TopicConfig topicConfig = new TopicConfig(topicName, 1, 1,\n                PermName.PERM_READ | PermName.PERM_WRITE, 0);\n            brokerController.getTopicConfigManager().getTopicConfigTable().put(topicName, topicConfig);\n        }\n    }\n\n    private void fillSubscriptionGroupManager(int num) {\n        for (int i = num - 1; i >= 0; i--) {\n            SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n            String groupName = String.format(\"group-%05d\", i);\n            subscriptionGroupConfig.setGroupName(groupName);\n            Map<String, String> attr = ImmutableMap.of(\"+test\", \"true\");\n            subscriptionGroupConfig.setAttributes(attr);\n            brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().put(groupName, subscriptionGroupConfig);\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.net.Broker2Client;\nimport org.apache.rocketmq.broker.failover.EscapeBridge;\nimport org.apache.rocketmq.broker.metrics.BrokerMetricsManager;\nimport org.apache.rocketmq.broker.metrics.PopMetricsManager;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport com.alibaba.fastjson2.JSON;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.PopAckConstants;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumerData;\nimport org.apache.rocketmq.store.AppendMessageResult;\nimport org.apache.rocketmq.store.AppendMessageStatus;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.pop.PopCheckPoint;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.ArgumentCaptor;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.lang.reflect.Field;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentHashMap;\n\nimport static org.apache.rocketmq.broker.processor.PullMessageProcessorTest.createConsumerData;\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.doNothing;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ChangeInvisibleTimeProcessorTest {\n    private ChangeInvisibleTimeProcessor changeInvisibleTimeProcessor;\n    @Spy\n    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());\n    @Mock\n    private ChannelHandlerContext handlerContext;\n    @Mock\n    private DefaultMessageStore messageStore;\n    @Mock\n    private Channel channel;\n\n    private String topic = \"FooBar\";\n    private String group = \"FooBarGroup\";\n    private ClientChannelInfo clientInfo;\n    @Mock\n    private Broker2Client broker2Client;\n\n    @Mock\n    private EscapeBridge escapeBridge;\n\n    @Before\n    public void init() throws IllegalAccessException, NoSuchFieldException {\n        // Inject BrokerMetricsManager if missing\n        Field brokerMetricsManagerField = BrokerController.class.getDeclaredField(\"brokerMetricsManager\");\n        brokerMetricsManagerField.setAccessible(true);\n        if (brokerMetricsManagerField.get(brokerController) == null) {\n            BrokerMetricsManager brokerMetricsManager = new BrokerMetricsManager(brokerController);\n            brokerMetricsManagerField.set(brokerController, brokerMetricsManager);\n        }\n        \n        // Mock necessary dependencies\n        when(brokerController.getMessageStore()).thenReturn(messageStore);\n        when(brokerController.getEscapeBridge()).thenReturn(this.escapeBridge);\n        \n        Channel mockChannel = mock(Channel.class);\n        when(handlerContext.channel()).thenReturn(mockChannel);\n        \n        // Mock TopicConfigManager\n        TopicConfigManager topicConfigManager = mock(TopicConfigManager.class);\n        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);\n        ConcurrentHashMap<String, TopicConfig> topicConfigTable = new ConcurrentHashMap<>();\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(topic);\n        topicConfigTable.put(topic, topicConfig);\n        when(topicConfigManager.selectTopicConfig(topic)).thenReturn(topicConfig);\n        \n        // Mock BrokerStatsManager\n        BrokerStatsManager brokerStatsManager = mock(BrokerStatsManager.class);\n        when(brokerController.getBrokerStatsManager()).thenReturn(brokerStatsManager);\n        \n        // Mock PopMessageProcessor and PopBufferMergeService\n        PopMessageProcessor popMessageProcessor = mock(PopMessageProcessor.class);\n        PopBufferMergeService popBufferMergeService = mock(PopBufferMergeService.class);\n        when(brokerController.getPopMessageProcessor()).thenReturn(popMessageProcessor);\n        when(popMessageProcessor.getPopBufferMergeService()).thenReturn(popBufferMergeService);\n\n        ConsumerData consumerData = createConsumerData(group, topic);\n        clientInfo = new ClientChannelInfo(channel, \"127.0.0.1\", LanguageCode.JAVA, 0);\n        brokerController.getConsumerManager().registerConsumer(\n                consumerData.getGroupName(),\n                clientInfo,\n                consumerData.getConsumeType(),\n                consumerData.getMessageModel(),\n                consumerData.getConsumeFromWhere(),\n                consumerData.getSubscriptionDataSet(),\n                false);\n        \n        clientInfo = new ClientChannelInfo(channel, \"127.0.0.1\", LanguageCode.JAVA, 0);\n        changeInvisibleTimeProcessor = new ChangeInvisibleTimeProcessor(brokerController);\n    }\n\n    @Test\n    public void testProcessRequest_Success() throws RemotingCommandException, ConsumeQueueException {\n        when(messageStore.getMaxOffsetInQueue(anyString(), anyInt())).thenReturn(2L);\n        when(escapeBridge.asyncPutMessageToSpecificQueue(any(MessageExtBrokerInner.class))).thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))));\n        int queueId = 0;\n        long queueOffset = 0;\n        long popTime = System.currentTimeMillis() - 1_000;\n        long invisibleTime = 30_000;\n        int reviveQid = 0;\n        String brokerName = \"test_broker\";\n        String extraInfo = ExtraInfoUtil.buildExtraInfo(queueOffset, popTime, invisibleTime, reviveQid,\n            topic, brokerName, queueId) + MessageConst.KEY_SEPARATOR + queueOffset;\n\n        ChangeInvisibleTimeRequestHeader requestHeader = new ChangeInvisibleTimeRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setQueueId(queueId);\n        requestHeader.setOffset(queueOffset);\n        requestHeader.setConsumerGroup(group);\n        requestHeader.setExtraInfo(extraInfo);\n        requestHeader.setInvisibleTime(invisibleTime);\n\n        final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader);\n        request.makeCustomHeaderToNet();\n        RemotingCommand responseToReturn = changeInvisibleTimeProcessor.processRequest(handlerContext, request);\n        assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(responseToReturn.getOpaque()).isEqualTo(request.getOpaque());\n    }\n\n    @Test\n    public void testProcessRequest_NoMessage() throws RemotingCommandException, ConsumeQueueException {\n        when(messageStore.getMaxOffsetInQueue(anyString(), anyInt())).thenReturn(2L);\n        int queueId = 0;\n        long queueOffset = 2;\n        long popTime = System.currentTimeMillis() - 1_000;\n        long invisibleTime = 30_000;\n        int reviveQid = 0;\n        String brokerName = \"test_broker\";\n        String extraInfo = ExtraInfoUtil.buildExtraInfo(queueOffset, popTime, invisibleTime, reviveQid,\n            topic, brokerName, queueId) + MessageConst.KEY_SEPARATOR + queueOffset;\n\n        ChangeInvisibleTimeRequestHeader requestHeader = new ChangeInvisibleTimeRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setQueueId(queueId);\n        requestHeader.setOffset(queueOffset);\n        requestHeader.setConsumerGroup(group);\n        requestHeader.setExtraInfo(extraInfo);\n        requestHeader.setInvisibleTime(invisibleTime);\n\n        final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader);\n        request.makeCustomHeaderToNet();\n        RemotingCommand responseToReturn = changeInvisibleTimeProcessor.processRequest(handlerContext, request);\n        assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.NO_MESSAGE);\n        assertThat(responseToReturn.getOpaque()).isEqualTo(request.getOpaque());\n    }\n\n    @Test\n    public void testProcessRequestAsync_JsonParsing() throws Exception {\n        Channel mockChannel = mock(Channel.class);\n        RemotingCommand mockRequest = mock(RemotingCommand.class);\n        BrokerController mockBrokerController = mock(BrokerController.class);\n        TopicConfigManager mockTopicConfigManager = mock(TopicConfigManager.class);\n        MessageStore mockMessageStore = mock(MessageStore.class);\n        BrokerConfig mockBrokerConfig = mock(BrokerConfig.class);\n        BrokerStatsManager mockBrokerStatsManager = mock(BrokerStatsManager.class);\n        PopMessageProcessor mockPopMessageProcessor = mock(PopMessageProcessor.class);\n        PopBufferMergeService mockPopBufferMergeService = mock(PopBufferMergeService.class);\n        BrokerMetricsManager brokerMetricsManager = mock(BrokerMetricsManager.class);\n        PopMetricsManager popMetricsManager = mock(PopMetricsManager.class);\n\n        when(brokerMetricsManager.getPopMetricsManager()).thenReturn(popMetricsManager);\n        when(mockBrokerController.getBrokerMetricsManager()).thenReturn(brokerMetricsManager);\n        doNothing().when(popMetricsManager).incPopReviveCkPutCount(any(), any());\n        when(brokerMetricsManager.getPopMetricsManager()).thenReturn(popMetricsManager);\n        when(mockBrokerController.getTopicConfigManager()).thenReturn(mockTopicConfigManager);\n        when(mockBrokerController.getMessageStore()).thenReturn(mockMessageStore);\n        when(mockBrokerController.getBrokerConfig()).thenReturn(mockBrokerConfig);\n        when(mockBrokerController.getBrokerStatsManager()).thenReturn(mockBrokerStatsManager);\n        when(mockBrokerController.getPopMessageProcessor()).thenReturn(mockPopMessageProcessor);\n        when(mockPopMessageProcessor.getPopBufferMergeService()).thenReturn(mockPopBufferMergeService);\n        when(mockPopBufferMergeService.addAk(anyInt(), any())).thenReturn(false);\n        when(mockBrokerController.getEscapeBridge()).thenReturn(escapeBridge);\n        PutMessageResult mockPutMessageResult = new PutMessageResult(PutMessageStatus.PUT_OK, null, true);\n        when(mockBrokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(any()))\n                .thenReturn(CompletableFuture.completedFuture(mockPutMessageResult));\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setReadQueueNums(4);\n        when(mockTopicConfigManager.selectTopicConfig(anyString())).thenReturn(topicConfig);\n        when(mockMessageStore.getMinOffsetInQueue(anyString(), anyInt())).thenReturn(0L);\n        when(mockMessageStore.getMaxOffsetInQueue(anyString(), anyInt())).thenReturn(10L);\n        when(mockBrokerConfig.isPopConsumerKVServiceEnable()).thenReturn(false);\n\n        ChangeInvisibleTimeRequestHeader requestHeader = new ChangeInvisibleTimeRequestHeader();\n        requestHeader.setTopic(\"TestTopic\");\n        requestHeader.setQueueId(1);\n        requestHeader.setOffset(5L);\n        requestHeader.setConsumerGroup(\"TestGroup\");\n        requestHeader.setExtraInfo(\"0 10000 10000 0 TestBroker 1\");\n        requestHeader.setInvisibleTime(60000L);\n        when(mockRequest.decodeCommandCustomHeader(ChangeInvisibleTimeRequestHeader.class)).thenReturn(requestHeader);\n\n        ChangeInvisibleTimeProcessor processor = new ChangeInvisibleTimeProcessor(mockBrokerController);\n        CompletableFuture<RemotingCommand> futureResponse = processor.processRequestAsync(mockChannel, mockRequest, true);\n\n        RemotingCommand response = futureResponse.get();\n        assertNotNull(response);\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n    }\n\n    @Test\n    public void testProcessRequestAsyncWithSuspendTrue() throws Exception {\n        // Setup mocks\n        Channel mockChannel = mock(Channel.class);\n        RemotingCommand mockRequest = mock(RemotingCommand.class);\n        BrokerController mockBrokerController = mock(BrokerController.class);\n        TopicConfigManager mockTopicConfigManager = mock(TopicConfigManager.class);\n        MessageStore mockMessageStore = mock(MessageStore.class);\n        BrokerConfig mockBrokerConfig = mock(BrokerConfig.class);\n        BrokerStatsManager mockBrokerStatsManager = mock(BrokerStatsManager.class);\n        PopMessageProcessor mockPopMessageProcessor = mock(PopMessageProcessor.class);\n        PopBufferMergeService mockPopBufferMergeService = mock(PopBufferMergeService.class);\n        BrokerMetricsManager brokerMetricsManager = mock(BrokerMetricsManager.class);\n        PopMetricsManager popMetricsManager = mock(PopMetricsManager.class);\n        EscapeBridge mockEscapeBridge = mock(EscapeBridge.class);\n\n        when(brokerMetricsManager.getPopMetricsManager()).thenReturn(popMetricsManager);\n        when(mockBrokerController.getBrokerMetricsManager()).thenReturn(brokerMetricsManager);\n        doNothing().when(popMetricsManager).incPopReviveCkPutCount(any(), any());\n        when(mockBrokerController.getTopicConfigManager()).thenReturn(mockTopicConfigManager);\n        when(mockBrokerController.getMessageStore()).thenReturn(mockMessageStore);\n        when(mockBrokerController.getBrokerConfig()).thenReturn(mockBrokerConfig);\n        when(mockBrokerController.getBrokerStatsManager()).thenReturn(mockBrokerStatsManager);\n        when(mockBrokerController.getPopMessageProcessor()).thenReturn(mockPopMessageProcessor);\n        when(mockPopMessageProcessor.getPopBufferMergeService()).thenReturn(mockPopBufferMergeService);\n        when(mockPopBufferMergeService.addAk(anyInt(), any())).thenReturn(false);\n        when(mockBrokerController.getEscapeBridge()).thenReturn(mockEscapeBridge);\n\n        PutMessageResult mockPutMessageResult = new PutMessageResult(PutMessageStatus.PUT_OK, null, true);\n        when(mockEscapeBridge.asyncPutMessageToSpecificQueue(any()))\n                .thenReturn(CompletableFuture.completedFuture(mockPutMessageResult));\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setReadQueueNums(4);\n        when(mockTopicConfigManager.selectTopicConfig(anyString())).thenReturn(topicConfig);\n        when(mockMessageStore.getMinOffsetInQueue(anyString(), anyInt())).thenReturn(0L);\n        when(mockMessageStore.getMaxOffsetInQueue(anyString(), anyInt())).thenReturn(10L);\n        when(mockBrokerConfig.isPopConsumerKVServiceEnable()).thenReturn(false);\n\n        ChangeInvisibleTimeRequestHeader requestHeader = new ChangeInvisibleTimeRequestHeader();\n        requestHeader.setTopic(\"TestTopic\");\n        requestHeader.setQueueId(1);\n        requestHeader.setOffset(5L);\n        requestHeader.setConsumerGroup(\"TestGroup\");\n        requestHeader.setExtraInfo(\"0 10000 10000 0 TestBroker 1\");\n        requestHeader.setInvisibleTime(60000L);\n        requestHeader.setSuspend(true); // Test with suspend=true\n        when(mockRequest.decodeCommandCustomHeader(ChangeInvisibleTimeRequestHeader.class)).thenReturn(requestHeader);\n\n        ChangeInvisibleTimeProcessor processor = new ChangeInvisibleTimeProcessor(mockBrokerController);\n        CompletableFuture<RemotingCommand> futureResponse = processor.processRequestAsync(mockChannel, mockRequest, true);\n\n        RemotingCommand response = futureResponse.get();\n        assertNotNull(response);\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n    }\n\n    @Test\n    public void testProcessRequestAsyncWithSuspendFalse() throws Exception {\n        // Setup mocks\n        Channel mockChannel = mock(Channel.class);\n        RemotingCommand mockRequest = mock(RemotingCommand.class);\n        BrokerController mockBrokerController = mock(BrokerController.class);\n        TopicConfigManager mockTopicConfigManager = mock(TopicConfigManager.class);\n        MessageStore mockMessageStore = mock(MessageStore.class);\n        BrokerConfig mockBrokerConfig = mock(BrokerConfig.class);\n        BrokerStatsManager mockBrokerStatsManager = mock(BrokerStatsManager.class);\n        PopMessageProcessor mockPopMessageProcessor = mock(PopMessageProcessor.class);\n        PopBufferMergeService mockPopBufferMergeService = mock(PopBufferMergeService.class);\n        BrokerMetricsManager brokerMetricsManager = mock(BrokerMetricsManager.class);\n        PopMetricsManager popMetricsManager = mock(PopMetricsManager.class);\n        EscapeBridge mockEscapeBridge = mock(EscapeBridge.class);\n\n        when(brokerMetricsManager.getPopMetricsManager()).thenReturn(popMetricsManager);\n        when(mockBrokerController.getBrokerMetricsManager()).thenReturn(brokerMetricsManager);\n        doNothing().when(popMetricsManager).incPopReviveCkPutCount(any(), any());\n        when(mockBrokerController.getTopicConfigManager()).thenReturn(mockTopicConfigManager);\n        when(mockBrokerController.getMessageStore()).thenReturn(mockMessageStore);\n        when(mockBrokerController.getBrokerConfig()).thenReturn(mockBrokerConfig);\n        when(mockBrokerController.getBrokerStatsManager()).thenReturn(mockBrokerStatsManager);\n        when(mockBrokerController.getPopMessageProcessor()).thenReturn(mockPopMessageProcessor);\n        when(mockPopMessageProcessor.getPopBufferMergeService()).thenReturn(mockPopBufferMergeService);\n        when(mockPopBufferMergeService.addAk(anyInt(), any())).thenReturn(false);\n        when(mockBrokerController.getEscapeBridge()).thenReturn(mockEscapeBridge);\n\n        PutMessageResult mockPutMessageResult = new PutMessageResult(PutMessageStatus.PUT_OK, null, true);\n        when(mockEscapeBridge.asyncPutMessageToSpecificQueue(any()))\n                .thenReturn(CompletableFuture.completedFuture(mockPutMessageResult));\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setReadQueueNums(4);\n        when(mockTopicConfigManager.selectTopicConfig(anyString())).thenReturn(topicConfig);\n        when(mockMessageStore.getMinOffsetInQueue(anyString(), anyInt())).thenReturn(0L);\n        when(mockMessageStore.getMaxOffsetInQueue(anyString(), anyInt())).thenReturn(10L);\n        when(mockBrokerConfig.isPopConsumerKVServiceEnable()).thenReturn(false);\n\n        ChangeInvisibleTimeRequestHeader requestHeader = new ChangeInvisibleTimeRequestHeader();\n        requestHeader.setTopic(\"TestTopic\");\n        requestHeader.setQueueId(1);\n        requestHeader.setOffset(5L);\n        requestHeader.setConsumerGroup(\"TestGroup\");\n        requestHeader.setExtraInfo(\"0 10000 10000 0 TestBroker 1\");\n        requestHeader.setInvisibleTime(60000L);\n        requestHeader.setSuspend(false); // Test with suspend=false\n        when(mockRequest.decodeCommandCustomHeader(ChangeInvisibleTimeRequestHeader.class)).thenReturn(requestHeader);\n\n        ChangeInvisibleTimeProcessor processor = new ChangeInvisibleTimeProcessor(mockBrokerController);\n        CompletableFuture<RemotingCommand> futureResponse = processor.processRequestAsync(mockChannel, mockRequest, true);\n\n        RemotingCommand response = futureResponse.get();\n        assertNotNull(response);\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n    }\n\n    @Test\n    public void testProcessRequestWithSuspendTrue() throws RemotingCommandException, ConsumeQueueException {\n        when(messageStore.getMaxOffsetInQueue(anyString(), anyInt())).thenReturn(2L);\n        when(escapeBridge.asyncPutMessageToSpecificQueue(any(MessageExtBrokerInner.class))).thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))));\n        int queueId = 0;\n        long queueOffset = 0;\n        long popTime = System.currentTimeMillis() - 1_000;\n        long invisibleTime = 30_000;\n        int reviveQid = 0;\n        String brokerName = \"test_broker\";\n        String extraInfo = ExtraInfoUtil.buildExtraInfo(queueOffset, popTime, invisibleTime, reviveQid,\n            topic, brokerName, queueId) + MessageConst.KEY_SEPARATOR + queueOffset;\n\n        ChangeInvisibleTimeRequestHeader requestHeader = new ChangeInvisibleTimeRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setQueueId(queueId);\n        requestHeader.setOffset(queueOffset);\n        requestHeader.setConsumerGroup(group);\n        requestHeader.setExtraInfo(extraInfo);\n        requestHeader.setInvisibleTime(invisibleTime);\n        requestHeader.setSuspend(true); // Set suspend to true\n\n        final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader);\n        request.makeCustomHeaderToNet();\n        RemotingCommand responseToReturn = changeInvisibleTimeProcessor.processRequest(handlerContext, request);\n        assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(responseToReturn.getOpaque()).isEqualTo(request.getOpaque());\n    }\n\n    @Test\n    public void testProcessRequestWithSuspendFalse() throws RemotingCommandException, ConsumeQueueException {\n        when(messageStore.getMaxOffsetInQueue(anyString(), anyInt())).thenReturn(2L);\n        when(escapeBridge.asyncPutMessageToSpecificQueue(any(MessageExtBrokerInner.class))).thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))));\n        int queueId = 0;\n        long queueOffset = 0;\n        long popTime = System.currentTimeMillis() - 1_000;\n        long invisibleTime = 30_000;\n        int reviveQid = 0;\n        String brokerName = \"test_broker\";\n        String extraInfo = ExtraInfoUtil.buildExtraInfo(queueOffset, popTime, invisibleTime, reviveQid,\n            topic, brokerName, queueId) + MessageConst.KEY_SEPARATOR + queueOffset;\n\n        ChangeInvisibleTimeRequestHeader requestHeader = new ChangeInvisibleTimeRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setQueueId(queueId);\n        requestHeader.setOffset(queueOffset);\n        requestHeader.setConsumerGroup(group);\n        requestHeader.setExtraInfo(extraInfo);\n        requestHeader.setInvisibleTime(invisibleTime);\n        requestHeader.setSuspend(false); // Set suspend to false\n\n        final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader);\n        request.makeCustomHeaderToNet();\n        RemotingCommand responseToReturn = changeInvisibleTimeProcessor.processRequest(handlerContext, request);\n        assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(responseToReturn.getOpaque()).isEqualTo(request.getOpaque());\n    }\n\n    @Test\n    public void testAppendCheckPointThenAckOriginWritesSuspendTrueInCheckpoint() throws Exception {\n        when(messageStore.getMaxOffsetInQueue(anyString(), anyInt())).thenReturn(2L);\n        ArgumentCaptor<MessageExtBrokerInner> msgCaptor = ArgumentCaptor.forClass(MessageExtBrokerInner.class);\n        when(escapeBridge.asyncPutMessageToSpecificQueue(msgCaptor.capture()))\n            .thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))));\n\n        int queueId = 0;\n        long queueOffset = 0;\n        long popTime = System.currentTimeMillis() - 1_000;\n        long invisibleTime = 30_000;\n        int reviveQid = 0;\n        String brokerName = \"test_broker\";\n        String extraInfo = ExtraInfoUtil.buildExtraInfo(queueOffset, popTime, invisibleTime, reviveQid,\n            topic, brokerName, queueId) + MessageConst.KEY_SEPARATOR + queueOffset;\n\n        ChangeInvisibleTimeRequestHeader requestHeader = new ChangeInvisibleTimeRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setQueueId(queueId);\n        requestHeader.setOffset(queueOffset);\n        requestHeader.setConsumerGroup(group);\n        requestHeader.setExtraInfo(extraInfo);\n        requestHeader.setInvisibleTime(invisibleTime);\n        requestHeader.setSuspend(true);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader);\n        request.makeCustomHeaderToNet();\n        changeInvisibleTimeProcessor.processRequest(handlerContext, request);\n\n        List<MessageExtBrokerInner> allValues = msgCaptor.getAllValues();\n        MessageExtBrokerInner ckMessage = allValues.stream()\n            .filter(m -> PopAckConstants.CK_TAG.equals(m.getTags()))\n            .findFirst()\n            .orElseThrow(() -> new AssertionError(\"No CK message captured\"));\n        PopCheckPoint ck = JSON.parseObject(new String(ckMessage.getBody(), java.nio.charset.StandardCharsets.UTF_8), PopCheckPoint.class);\n        assertThat(ck.isSuspend()).isTrue();\n    }\n\n    @Test\n    public void testAppendCheckPointThenAckOriginWritesSuspendFalseInCheckpoint() throws Exception {\n        when(messageStore.getMaxOffsetInQueue(anyString(), anyInt())).thenReturn(2L);\n        ArgumentCaptor<MessageExtBrokerInner> msgCaptor = ArgumentCaptor.forClass(MessageExtBrokerInner.class);\n        when(escapeBridge.asyncPutMessageToSpecificQueue(msgCaptor.capture()))\n            .thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))));\n\n        int queueId = 0;\n        long queueOffset = 0;\n        long popTime = System.currentTimeMillis() - 1_000;\n        long invisibleTime = 30_000;\n        int reviveQid = 0;\n        String brokerName = \"test_broker\";\n        String extraInfo = ExtraInfoUtil.buildExtraInfo(queueOffset, popTime, invisibleTime, reviveQid,\n            topic, brokerName, queueId) + MessageConst.KEY_SEPARATOR + queueOffset;\n\n        ChangeInvisibleTimeRequestHeader requestHeader = new ChangeInvisibleTimeRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setQueueId(queueId);\n        requestHeader.setOffset(queueOffset);\n        requestHeader.setConsumerGroup(group);\n        requestHeader.setExtraInfo(extraInfo);\n        requestHeader.setInvisibleTime(invisibleTime);\n        requestHeader.setSuspend(false);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader);\n        request.makeCustomHeaderToNet();\n        changeInvisibleTimeProcessor.processRequest(handlerContext, request);\n\n        List<MessageExtBrokerInner> allValues = msgCaptor.getAllValues();\n        MessageExtBrokerInner ckMessage = allValues.stream()\n            .filter(m -> PopAckConstants.CK_TAG.equals(m.getTags()))\n            .findFirst()\n            .orElseThrow(() -> new AssertionError(\"No CK message captured\"));\n        PopCheckPoint ck = JSON.parseObject(new String(ckMessage.getBody(), java.nio.charset.StandardCharsets.UTF_8), PopCheckPoint.class);\n        assertThat(ck.isSuspend()).isFalse();\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/processor/ClientManageProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.HashSet;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.UUID;\nimport java.util.List;\nimport java.util.ArrayList;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ConsumerGroupInfo;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.UnregisterClientRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumerData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ClientManageProcessorTest {\n    private ClientManageProcessor clientManageProcessor;\n    @Spy\n    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());\n    @Mock\n    private ChannelHandlerContext handlerContext;\n    @Mock\n    private Channel channel;\n\n    private ClientChannelInfo clientChannelInfo;\n    private String clientId = UUID.randomUUID().toString();\n    private String group = \"FooBarGroup\";\n    private String topic = \"FooBar\";\n\n    @Before\n    public void init() {\n        when(handlerContext.channel()).thenReturn(channel);\n        clientManageProcessor = new ClientManageProcessor(brokerController);\n        clientChannelInfo = new ClientChannelInfo(channel, clientId, LanguageCode.JAVA, 100);\n        brokerController.getProducerManager().registerProducer(group, clientChannelInfo);\n\n        ConsumerData consumerData = PullMessageProcessorTest.createConsumerData(group, topic);\n        brokerController.getConsumerManager().registerConsumer(\n            consumerData.getGroupName(),\n            clientChannelInfo,\n            consumerData.getConsumeType(),\n            consumerData.getMessageModel(),\n            consumerData.getConsumeFromWhere(),\n            consumerData.getSubscriptionDataSet(),\n            false);\n    }\n\n    @Test\n    public void processRequest_UnRegisterProducer() throws Exception {\n        brokerController.getProducerManager().registerProducer(group, clientChannelInfo);\n        Map<Channel, ClientChannelInfo> channelMap = brokerController.getProducerManager().getGroupChannelTable().get(group);\n        assertThat(channelMap).isNotNull();\n        assertThat(channelMap.get(channel)).isEqualTo(clientChannelInfo);\n\n        RemotingCommand request = createUnRegisterProducerCommand();\n        RemotingCommand response = clientManageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        channelMap = brokerController.getProducerManager().getGroupChannelTable().get(group);\n        assertThat(channelMap).isNull();\n    }\n\n    @Test\n    public void processRequest_UnRegisterConsumer() throws RemotingCommandException {\n        ConsumerGroupInfo consumerGroupInfo = brokerController.getConsumerManager().getConsumerGroupInfo(group);\n        assertThat(consumerGroupInfo).isNotNull();\n\n        RemotingCommand request = createUnRegisterConsumerCommand();\n        RemotingCommand response = clientManageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        consumerGroupInfo = brokerController.getConsumerManager().getConsumerGroupInfo(group);\n        assertThat(consumerGroupInfo).isNull();\n    }\n\n    @Test\n    public void processRequest_heartbeat() throws RemotingCommandException {\n        RemotingCommand request = createHeartbeatCommand(false, \"topicA\");\n        RemotingCommand response = clientManageProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(Boolean.parseBoolean(response.getExtFields().get(MixAll.IS_SUB_CHANGE))).isFalse();\n        ConsumerGroupInfo consumerGroupInfo = brokerController.getConsumerManager().getConsumerGroupInfo(group);\n\n        RemotingCommand requestSimple = createHeartbeatCommand(true, \"topicA\");\n        RemotingCommand responseSimple = clientManageProcessor.processRequest(handlerContext, requestSimple);\n        assertThat(responseSimple.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(Boolean.parseBoolean(responseSimple.getExtFields().get(MixAll.IS_SUB_CHANGE))).isFalse();\n        ConsumerGroupInfo consumerGroupInfoSimple = brokerController.getConsumerManager().getConsumerGroupInfo(group);\n        assertThat(consumerGroupInfoSimple).isEqualTo(consumerGroupInfo);\n\n        request = createHeartbeatCommand(false, \"topicB\");\n        response = clientManageProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(Boolean.parseBoolean(response.getExtFields().get(MixAll.IS_SUB_CHANGE))).isTrue();\n        consumerGroupInfo = brokerController.getConsumerManager().getConsumerGroupInfo(group);\n\n        requestSimple = createHeartbeatCommand(true, \"topicB\");\n        responseSimple = clientManageProcessor.processRequest(handlerContext, requestSimple);\n        assertThat(responseSimple.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(Boolean.parseBoolean(responseSimple.getExtFields().get(MixAll.IS_SUB_CHANGE))).isFalse();\n        consumerGroupInfoSimple = brokerController.getConsumerManager().getConsumerGroupInfo(group);\n        assertThat(consumerGroupInfoSimple).isEqualTo(consumerGroupInfo);\n    }\n\n    @Test\n    public void test_heartbeat_costTime() {\n        String topic = \"TOPIC_TEST\";\n        List<String> topicList = new ArrayList<>();\n        for (int i = 0; i < 500; i ++) {\n            topicList.add(topic + i);\n        }\n        HeartbeatData heartbeatData = prepareHeartbeatData(false, topicList);\n        long time = System.currentTimeMillis();\n        heartbeatData.computeHeartbeatFingerprint();\n        System.out.print(\"computeHeartbeatFingerprint cost time : \" + (System.currentTimeMillis() - time) + \" ms \\n\");\n    }\n\n    private RemotingCommand createUnRegisterProducerCommand() {\n        UnregisterClientRequestHeader requestHeader = new UnregisterClientRequestHeader();\n        requestHeader.setClientID(clientId);\n        requestHeader.setProducerGroup(group);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UNREGISTER_CLIENT, requestHeader);\n        request.setLanguage(LanguageCode.JAVA);\n        request.setVersion(100);\n        request.makeCustomHeaderToNet();\n        return request;\n    }\n\n    private RemotingCommand createUnRegisterConsumerCommand() {\n        UnregisterClientRequestHeader requestHeader = new UnregisterClientRequestHeader();\n        requestHeader.setClientID(clientId);\n        requestHeader.setConsumerGroup(group);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UNREGISTER_CLIENT, requestHeader);\n        request.setLanguage(LanguageCode.JAVA);\n        request.setVersion(100);\n        request.makeCustomHeaderToNet();\n        return request;\n    }\n\n    private RemotingCommand createHeartbeatCommand(boolean isWithoutSub, String topic) {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.HEART_BEAT, null);\n        request.setLanguage(LanguageCode.JAVA);\n        HeartbeatData heartbeatDataWithSub = prepareHeartbeatData(false, topic);\n        int heartbeatFingerprint = heartbeatDataWithSub.computeHeartbeatFingerprint();\n        HeartbeatData heartbeatData = prepareHeartbeatData(isWithoutSub, topic);\n        heartbeatData.setHeartbeatFingerprint(heartbeatFingerprint);\n        request.setBody(heartbeatData.encode());\n        return request;\n    }\n\n    private HeartbeatData prepareHeartbeatData(boolean isWithoutSub, String topic) {\n        List<String> list = new ArrayList<>();\n        list.add(topic);\n        return prepareHeartbeatData(isWithoutSub, list);\n    }\n\n    private HeartbeatData prepareHeartbeatData(boolean isWithoutSub, List<String> topicList) {\n        HeartbeatData heartbeatData = new HeartbeatData();\n        heartbeatData.setClientID(this.clientId);\n        ConsumerData consumerData = createConsumerData(group);\n        if (!isWithoutSub) {\n            Set<SubscriptionData> subscriptionDataSet = new HashSet<>();\n            for (String topic : topicList) {\n                SubscriptionData subscriptionData = new SubscriptionData();\n                subscriptionData.setTopic(topic);\n                subscriptionData.setSubString(\"*\");\n                subscriptionData.setSubVersion(100L);\n                subscriptionDataSet.add(subscriptionData);\n            }\n            consumerData.getSubscriptionDataSet().addAll(subscriptionDataSet);\n        }\n        heartbeatData.getConsumerDataSet().add(consumerData);\n        heartbeatData.setWithoutSub(isWithoutSub);\n        return heartbeatData;\n    }\n\n    static ConsumerData createConsumerData(String group) {\n        ConsumerData consumerData = new ConsumerData();\n        consumerData.setGroupName(group);\n        consumerData.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        consumerData.setConsumeType(ConsumeType.CONSUME_PASSIVELY);\n        consumerData.setMessageModel(MessageModel.CLUSTERING);\n        return consumerData;\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ConsumerGroupInfo;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.broker.out.BrokerOuterAPI;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.broker.topic.TopicQueueMappingManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.statictopic.LogicQueueMappingItem;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingContext;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingDetail;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.remoting.rpc.RpcClient;\nimport org.apache.rocketmq.remoting.rpc.RpcException;\nimport org.apache.rocketmq.remoting.rpc.RpcResponse;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.Future;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ConsumerManageProcessorTest {\n    private ConsumerManageProcessor consumerManageProcessor;\n    @Mock\n    private ChannelHandlerContext handlerContext;\n    @Spy\n    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());\n    @Mock\n    private MessageStore messageStore;\n    @Mock\n    private Channel channel;\n    @Mock\n    private ConsumerOffsetManager consumerOffsetManager;\n    @Mock\n    private BrokerOuterAPI brokerOuterAPI;\n    @Mock\n    private RpcClient rpcClient;\n    @Mock\n    private Future<RpcResponse> responseFuture;\n    @Mock\n    private TopicQueueMappingContext mappingContext;\n\n    private String topic = \"FooBar\";\n    private String group = \"FooBarGroup\";\n\n    @Before\n    public void init() throws RpcException {\n        brokerController.setMessageStore(messageStore);\n        TopicConfigManager topicConfigManager = new TopicConfigManager(brokerController);\n        topicConfigManager.getTopicConfigTable().put(topic, new TopicConfig(topic));\n        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);\n        SubscriptionGroupManager subscriptionGroupManager = new SubscriptionGroupManager(brokerController);\n        subscriptionGroupManager.getSubscriptionGroupTable().put(group, new SubscriptionGroupConfig());\n        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);\n        consumerManageProcessor = new ConsumerManageProcessor(brokerController);\n        when(brokerController.getBrokerOuterAPI()).thenReturn(brokerOuterAPI);\n        when(brokerOuterAPI.getRpcClient()).thenReturn(rpcClient);\n        when(rpcClient.invoke(any(),anyLong())).thenReturn(responseFuture);\n        TopicQueueMappingDetail topicQueueMappingDetail = new TopicQueueMappingDetail();\n        topicQueueMappingDetail.setBname(\"BrokerA\");\n        when(mappingContext.getMappingDetail()).thenReturn(topicQueueMappingDetail);\n    }\n\n    @Test\n    public void testUpdateConsumerOffset_InvalidTopic() throws Exception {\n        RemotingCommand request = buildUpdateConsumerOffsetRequest(group, \"InvalidTopic\", 0, 0);\n        RemotingCommand response = consumerManageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.TOPIC_NOT_EXIST);\n    }\n\n    @Test\n    public void testUpdateConsumerOffset_GroupNotExist() throws Exception {\n        RemotingCommand request = buildUpdateConsumerOffsetRequest(\"NotExistGroup\", topic, 0, 0);\n        RemotingCommand response = consumerManageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);\n    }\n\n    @Test\n    public void testUpdateConsumerOffset() throws RemotingCommandException {\n        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);\n        when(consumerOffsetManager.hasOffsetReset(anyString(),anyString(),anyInt())).thenReturn(true);\n        RemotingCommand request = buildUpdateConsumerOffsetRequest(group, topic, 0, 0);\n        RemotingCommand response = consumerManageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        when(consumerOffsetManager.hasOffsetReset(anyString(),anyString(),anyInt())).thenReturn(false);\n        response = consumerManageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetConsumerListByGroup() throws RemotingCommandException {\n        GetConsumerListByGroupRequestHeader requestHeader = new GetConsumerListByGroupRequestHeader();\n        requestHeader.setConsumerGroup(group);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_LIST_BY_GROUP, requestHeader);\n        request.makeCustomHeaderToNet();\n        RemotingCommand response = consumerManageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);\n\n        brokerController.getConsumerManager().getConsumerTable().put(group,new ConsumerGroupInfo(group));\n        response = consumerManageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);\n\n        ConsumerGroupInfo consumerGroupInfo =\n                this.brokerController.getConsumerManager().getConsumerGroupInfo(\n                        requestHeader.getConsumerGroup());\n        consumerGroupInfo.getChannelInfoTable().put(channel,new ClientChannelInfo(channel));\n        response = consumerManageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testQueryConsumerOffset() throws RemotingCommandException, ExecutionException, InterruptedException {\n        RemotingCommand request = buildQueryConsumerOffsetRequest(group, topic, 0, true);\n        RemotingCommand response = consumerManageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.QUERY_NOT_FOUND);\n\n        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);\n        when(consumerOffsetManager.queryOffset(anyString(),anyString(),anyInt())).thenReturn(0L);\n        response = consumerManageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        when(consumerOffsetManager.queryOffset(anyString(),anyString(),anyInt())).thenReturn(-1L);\n        when(messageStore.getMinOffsetInQueue(anyString(),anyInt())).thenReturn(-1L);\n        when(messageStore.checkInMemByConsumeOffset(anyString(),anyInt(),anyLong(),anyInt())).thenReturn(true);\n        response = consumerManageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        TopicQueueMappingManager topicQueueMappingManager = mock(TopicQueueMappingManager.class);\n        when(brokerController.getTopicQueueMappingManager()).thenReturn(topicQueueMappingManager);\n        when(topicQueueMappingManager.buildTopicQueueMappingContext(any(QueryConsumerOffsetRequestHeader.class))).thenReturn(mappingContext);\n        response = consumerManageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.NOT_LEADER_FOR_QUEUE);\n\n        List<LogicQueueMappingItem> items = new ArrayList<>();\n        LogicQueueMappingItem item1 = createLogicQueueMappingItem(\"BrokerC\", 0, 0L, 0L);\n        items.add(item1);\n        when(mappingContext.getMappingItemList()).thenReturn(items);\n        when(mappingContext.getLeaderItem()).thenReturn(item1);\n        when(mappingContext.getCurrentItem()).thenReturn(item1);\n        when(mappingContext.isLeader()).thenReturn(true);\n        response = consumerManageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        LogicQueueMappingItem item2 = createLogicQueueMappingItem(\"BrokerA\", 0, 0L, 0L);\n        items.add(item2);\n        QueryConsumerOffsetResponseHeader queryConsumerOffsetResponseHeader = new QueryConsumerOffsetResponseHeader();\n        queryConsumerOffsetResponseHeader.setOffset(0L);\n        RpcResponse rpcResponse = new RpcResponse(ResponseCode.SUCCESS,queryConsumerOffsetResponseHeader,null);\n        when(responseFuture.get()).thenReturn(rpcResponse);\n        response = consumerManageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        queryConsumerOffsetResponseHeader.setOffset(-1L);\n        rpcResponse = new RpcResponse(ResponseCode.SUCCESS,queryConsumerOffsetResponseHeader,null);\n        when(responseFuture.get()).thenReturn(rpcResponse);\n        response = consumerManageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.QUERY_NOT_FOUND);\n    }\n\n    @Test\n    public void testRewriteRequestForStaticTopic() throws RpcException, ExecutionException, InterruptedException {\n        UpdateConsumerOffsetRequestHeader requestHeader = new UpdateConsumerOffsetRequestHeader();\n        requestHeader.setConsumerGroup(group);\n        requestHeader.setTopic(topic);\n        requestHeader.setQueueId(0);\n        requestHeader.setCommitOffset(0L);\n\n        RemotingCommand response = consumerManageProcessor.rewriteRequestForStaticTopic(requestHeader, mappingContext);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.NOT_LEADER_FOR_QUEUE);\n\n        List<LogicQueueMappingItem> items = new ArrayList<>();\n        LogicQueueMappingItem item = createLogicQueueMappingItem(\"BrokerC\", 0, 0L, 0L);\n        items.add(item);\n        when(mappingContext.getMappingItemList()).thenReturn(items);\n        when(mappingContext.isLeader()).thenReturn(true);\n        RpcResponse rpcResponse = new RpcResponse(ResponseCode.SUCCESS,new UpdateConsumerOffsetResponseHeader(),null);\n        when(responseFuture.get()).thenReturn(rpcResponse);\n        response = consumerManageProcessor.rewriteRequestForStaticTopic(requestHeader, mappingContext);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    public RemotingCommand buildQueryConsumerOffsetRequest(String group, String topic, int queueId,boolean setZeroIfNotFound) {\n        QueryConsumerOffsetRequestHeader requestHeader = new QueryConsumerOffsetRequestHeader();\n        requestHeader.setConsumerGroup(group);\n        requestHeader.setTopic(topic);\n        requestHeader.setQueueId(queueId);\n        requestHeader.setSetZeroIfNotFound(setZeroIfNotFound);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader);\n        request.makeCustomHeaderToNet();\n        return request;\n    }\n\n    public LogicQueueMappingItem createLogicQueueMappingItem(String brokerName, int queueId, long startOffset, long logicOffset) {\n        LogicQueueMappingItem item = new LogicQueueMappingItem();\n        item.setBname(brokerName);\n        item.setQueueId(queueId);\n        item.setStartOffset(startOffset);\n        item.setLogicOffset(logicOffset);\n        return item;\n    }\n\n    private RemotingCommand buildUpdateConsumerOffsetRequest(String group, String topic, int queueId, long offset) {\n        UpdateConsumerOffsetRequestHeader requestHeader = new UpdateConsumerOffsetRequestHeader();\n        requestHeader.setConsumerGroup(group);\n        requestHeader.setTopic(topic);\n        requestHeader.setQueueId(queueId);\n        requestHeader.setCommitOffset(offset);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_CONSUMER_OFFSET, requestHeader);\n        request.makeCustomHeaderToNet();\n        return request;\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/processor/EndTransactionProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.metrics.BrokerMetricsManager;\nimport org.apache.rocketmq.broker.transaction.OperationResult;\nimport org.apache.rocketmq.broker.transaction.TransactionMetrics;\nimport org.apache.rocketmq.broker.transaction.TransactionalMessageService;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.stats.Stats;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;\nimport org.apache.rocketmq.store.AppendMessageResult;\nimport org.apache.rocketmq.store.AppendMessageStatus;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.nio.charset.StandardCharsets;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class EndTransactionProcessorTest {\n\n    private static final String TOPIC = \"trans_topic_test\";\n\n    private EndTransactionProcessor endTransactionProcessor;\n\n    @Mock\n    private ChannelHandlerContext handlerContext;\n\n    @Spy\n    private BrokerController\n        brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(),\n            new MessageStoreConfig(), null);\n\n    @Mock\n    private MessageStore messageStore;\n\n    @Mock\n    private TransactionalMessageService transactionMsgService;\n\n    @Mock\n    private TransactionMetrics transactionMetrics;\n\n    @Before\n    public void init() {\n        when(transactionMsgService.getTransactionMetrics()).thenReturn(transactionMetrics);\n        brokerController.setMessageStore(messageStore);\n        brokerController.setTransactionalMessageService(transactionMsgService);\n        // Initialize BrokerMetricsManager to prevent NPE in tests\n        brokerController.setBrokerMetricsManager(new BrokerMetricsManager(brokerController));\n        endTransactionProcessor = new EndTransactionProcessor(brokerController);\n    }\n\n    private OperationResult createResponse(int status) {\n        OperationResult response = new OperationResult();\n        response.setPrepareMessage(createDefaultMessageExt());\n        response.setResponseCode(status);\n        response.setResponseRemark(null);\n        return response;\n    }\n\n    @Test\n    public void testProcessRequest() throws RemotingCommandException {\n        when(transactionMsgService.commitMessage(any(EndTransactionRequestHeader.class))).thenReturn(createResponse(ResponseCode.SUCCESS));\n        when(messageStore.putMessage(any(MessageExtBrokerInner.class)))\n                .thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, createAppendMessageResult(AppendMessageStatus.PUT_OK)));\n        RemotingCommand request = createEndTransactionMsgCommand(MessageSysFlag.TRANSACTION_COMMIT_TYPE, false);\n        RemotingCommand response = endTransactionProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(brokerController.getBrokerStatsManager().getStatsItem(Stats.BROKER_PUT_NUMS, brokerController.getBrokerConfig().getBrokerClusterName()).getValue().sum()).isEqualTo(1);\n        assertThat(brokerController.getBrokerStatsManager().getStatsItem(Stats.TOPIC_PUT_NUMS, TOPIC).getValue().sum()).isEqualTo(1L);\n        assertThat(brokerController.getBrokerStatsManager().getStatsItem(Stats.TOPIC_PUT_SIZE, TOPIC).getValue().sum()).isEqualTo(1L);\n    }\n\n    @Test\n    public void testProcessRequest_CheckMessage() throws RemotingCommandException {\n        when(transactionMsgService.commitMessage(any(EndTransactionRequestHeader.class))).thenReturn(createResponse(ResponseCode.SUCCESS));\n        when(messageStore.putMessage(any(MessageExtBrokerInner.class)))\n                .thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, createAppendMessageResult(AppendMessageStatus.PUT_OK)));\n        RemotingCommand request = createEndTransactionMsgCommand(MessageSysFlag.TRANSACTION_COMMIT_TYPE, true);\n        RemotingCommand response = endTransactionProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(brokerController.getBrokerStatsManager().getStatsItem(Stats.BROKER_PUT_NUMS, brokerController.getBrokerConfig().getBrokerClusterName()).getValue().sum()).isEqualTo(1);\n        assertThat(brokerController.getBrokerStatsManager().getStatsItem(Stats.TOPIC_PUT_NUMS, TOPIC).getValue().sum()).isEqualTo(1L);\n        assertThat(brokerController.getBrokerStatsManager().getStatsItem(Stats.TOPIC_PUT_SIZE, TOPIC).getValue().sum()).isEqualTo(1L);\n    }\n\n    @Test\n    public void testProcessRequest_NotType() throws RemotingCommandException {\n        RemotingCommand request = createEndTransactionMsgCommand(MessageSysFlag.TRANSACTION_NOT_TYPE, true);\n        RemotingCommand response = endTransactionProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNull();\n    }\n\n    @Test\n    public void testProcessRequest_RollBack() throws RemotingCommandException {\n        when(transactionMsgService.rollbackMessage(any(EndTransactionRequestHeader.class))).thenReturn(createResponse(ResponseCode.SUCCESS));\n        RemotingCommand request = createEndTransactionMsgCommand(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE, true);\n        RemotingCommand response = endTransactionProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testProcessRequest_RejectCommitMessage() throws RemotingCommandException {\n        when(transactionMsgService.commitMessage(any(EndTransactionRequestHeader.class))).thenReturn(createRejectResponse());\n        RemotingCommand request = createEndTransactionMsgCommand(MessageSysFlag.TRANSACTION_COMMIT_TYPE, false);\n        RemotingCommand response = endTransactionProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.ILLEGAL_OPERATION);\n    }\n\n    @Test\n    public void testProcessRequest_RejectRollBackMessage() throws RemotingCommandException {\n        when(transactionMsgService.rollbackMessage(any(EndTransactionRequestHeader.class))).thenReturn(createRejectResponse());\n        RemotingCommand request = createEndTransactionMsgCommand(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE, false);\n        RemotingCommand response = endTransactionProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.ILLEGAL_OPERATION);\n    }\n\n    private MessageExt createDefaultMessageExt() {\n        MessageExt messageExt = new MessageExt();\n        messageExt.setMsgId(\"12345678\");\n        messageExt.setQueueId(0);\n        messageExt.setCommitLogOffset(123456789L);\n        messageExt.setQueueOffset(1234);\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_REAL_TOPIC, TOPIC);\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_REAL_QUEUE_ID, \"0\");\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_TRANSACTION_PREPARED, \"true\");\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_PRODUCER_GROUP, \"testTransactionGroup\");\n        return messageExt;\n    }\n\n    private EndTransactionRequestHeader createEndTransactionRequestHeader(int status, boolean isCheckMsg) {\n        EndTransactionRequestHeader header = new EndTransactionRequestHeader();\n        header.setTopic(\"topic\");\n        header.setCommitLogOffset(123456789L);\n        header.setFromTransactionCheck(isCheckMsg);\n        header.setCommitOrRollback(status);\n        header.setMsgId(\"12345678\");\n        header.setTransactionId(\"123\");\n        header.setProducerGroup(\"testTransactionGroup\");\n        header.setTranStateTableOffset(1234L);\n        return header;\n    }\n\n    private RemotingCommand createEndTransactionMsgCommand(int status, boolean isCheckMsg) {\n        EndTransactionRequestHeader header = createEndTransactionRequestHeader(status, isCheckMsg);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.END_TRANSACTION, header);\n        request.makeCustomHeaderToNet();\n        return request;\n    }\n\n    private OperationResult createRejectResponse() {\n        OperationResult response = new OperationResult();\n        response.setPrepareMessage(createRejectMessageExt());\n        response.setResponseCode(ResponseCode.SUCCESS);\n        response.setResponseRemark(null);\n        return response;\n    }\n    private MessageExt createRejectMessageExt() {\n        MessageExt messageExt = new MessageExt();\n        messageExt.setMsgId(\"12345678\");\n        messageExt.setQueueId(0);\n        messageExt.setCommitLogOffset(123456789L);\n        messageExt.setQueueOffset(1234);\n        messageExt.setBody(\"body\".getBytes(StandardCharsets.UTF_8));\n        messageExt.setBornTimestamp(System.currentTimeMillis() - 65 * 1000);\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_REAL_QUEUE_ID, \"0\");\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_TRANSACTION_PREPARED, \"true\");\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_PRODUCER_GROUP, \"testTransactionGroup\");\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_REAL_TOPIC, \"TEST\");\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS, \"60\");\n        return messageExt;\n    }\n\n    private AppendMessageResult createAppendMessageResult(AppendMessageStatus status) {\n        AppendMessageResult result = new AppendMessageResult(status);\n        result.setMsgId(\"12345678\");\n        result.setMsgNum(1);\n        result.setWroteBytes(1);\n        return result;\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/processor/LiteManagerProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.lite.AbstractLiteLifecycleManager;\nimport org.apache.rocketmq.broker.lite.LiteEventDispatcher;\nimport org.apache.rocketmq.broker.lite.LiteSharding;\nimport org.apache.rocketmq.broker.lite.LiteSubscriptionRegistry;\nimport org.apache.rocketmq.broker.metrics.BrokerMetricsManager;\nimport org.apache.rocketmq.broker.metrics.LiteConsumerLagCalculator;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.broker.offset.MemoryConsumerOrderInfoManager;\nimport org.apache.rocketmq.broker.pop.orderly.ConsumerOrderInfoManager;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.entity.ClientGroup;\nimport org.apache.rocketmq.common.lite.LiteLagInfo;\nimport org.apache.rocketmq.common.lite.LiteSubscription;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.admin.TopicOffset;\nimport org.apache.rocketmq.remoting.protocol.body.GetBrokerLiteInfoResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.GetLiteClientInfoResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.GetLiteGroupInfoResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.GetLiteTopicInfoResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.GetParentTopicInfoResponseBody;\nimport org.apache.rocketmq.remoting.protocol.header.GetLiteClientInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetLiteGroupInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetLiteTopicInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetParentTopicInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.TriggerLiteDispatchRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.never;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class LiteManagerProcessorTest {\n\n    @Mock\n    private BrokerController brokerController;\n\n    @Mock\n    private AbstractLiteLifecycleManager liteLifecycleManager;\n\n    @Mock\n    private LiteSharding liteSharding;\n\n    @Mock\n    private ChannelHandlerContext ctx;\n\n    @Mock\n    private MessageStoreConfig messageStoreConfig;\n\n    @Mock\n    private MessageStore messageStore;\n\n    @Mock\n    private ConsumeQueueStoreInterface consumeQueueStore;\n\n    @Mock\n    private TopicConfigManager topicConfigManager;\n\n    @Mock\n    private SubscriptionGroupManager subscriptionGroupManager;\n\n    @Mock\n    private LiteSubscriptionRegistry liteSubscriptionRegistry;\n\n    @Mock\n    private ConsumerOffsetManager consumerOffsetManager;\n\n    @Mock\n    private BrokerMetricsManager brokerMetricsManager;\n\n    @Mock\n    private LiteConsumerLagCalculator liteConsumerLagCalculator;\n\n    @Mock\n    private LiteEventDispatcher liteEventDispatcher;\n\n    @Mock\n    private PopLiteMessageProcessor popLiteMessageProcessor;\n\n    private LiteManagerProcessor processor;\n\n    @Before\n    public void setUp() {\n        processor = new LiteManagerProcessor(brokerController, liteLifecycleManager, liteSharding);\n\n        when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n        when(brokerController.getMessageStore()).thenReturn(messageStore);\n        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);\n        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);\n        when(brokerController.getLiteSubscriptionRegistry()).thenReturn(liteSubscriptionRegistry);\n        when(brokerController.getBrokerMetricsManager()).thenReturn(brokerMetricsManager);\n        when(brokerController.getLiteEventDispatcher()).thenReturn(liteEventDispatcher);\n        when(brokerController.getPopLiteMessageProcessor()).thenReturn(popLiteMessageProcessor);\n        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);\n\n        ConsumerOrderInfoManager consumerOrderInfoManager = new MemoryConsumerOrderInfoManager(brokerController);\n        when(popLiteMessageProcessor.getConsumerOrderInfoManager()).thenReturn(consumerOrderInfoManager);\n\n        when(messageStore.getQueueStore()).thenReturn(consumeQueueStore);\n        when(consumeQueueStore.getConsumeQueueTable()).thenReturn(new ConcurrentHashMap<>());\n        when(brokerMetricsManager.getLiteConsumerLagCalculator()).thenReturn(liteConsumerLagCalculator);\n\n        when(consumerOffsetManager.getOffsetTable()).thenReturn(new ConcurrentHashMap<>());\n    }\n\n    @Test\n    public void testProcessRequest_GetBrokerLiteInfo() throws Exception {\n        RemotingCommand request = mock(RemotingCommand.class);\n        when(request.getCode()).thenReturn(RequestCode.GET_BROKER_LITE_INFO);\n\n        ConcurrentMap<String, TopicConfig> topicConfigTable = new ConcurrentHashMap<>();\n        when(topicConfigManager.getTopicConfigTable()).thenReturn(topicConfigTable);\n\n        ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroupTable = new ConcurrentHashMap<>();\n        when(subscriptionGroupManager.getSubscriptionGroupTable()).thenReturn(subscriptionGroupTable);\n\n        RemotingCommand response = processor.processRequest(ctx, request);\n\n        assertNotNull(response);\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n    }\n\n    @Test\n    public void testProcessRequest_UnsupportedRequestCode() throws Exception {\n        RemotingCommand request = mock(RemotingCommand.class);\n        when(request.getCode()).thenReturn(99999);\n\n        assertNull(processor.processRequest(ctx, request));\n    }\n\n    @Test\n    public void testGetBrokerLiteInfo() throws RemotingCommandException {\n        when(messageStoreConfig.getStoreType()).thenReturn(\"RocksDB\");\n        when(messageStoreConfig.getMaxLmqConsumeQueueNum()).thenReturn(10000);\n        when(consumeQueueStore.getLmqNum()).thenReturn(100);\n        when(liteSubscriptionRegistry.getActiveSubscriptionNum()).thenReturn(50);\n\n        ConcurrentHashMap<String, TopicConfig> topicConfigMap = new ConcurrentHashMap<>();\n        topicConfigMap.put(\"SYSTEM_TOPIC\", new TopicConfig(\"SYSTEM_TOPIC\"));\n        when(topicConfigManager.getTopicConfigTable()).thenReturn(topicConfigMap);\n\n        ConcurrentHashMap<String, SubscriptionGroupConfig> subscriptionGroupMap = new ConcurrentHashMap<>();\n        SubscriptionGroupConfig config = new SubscriptionGroupConfig();\n        config.setGroupName(\"test_group\");\n        config.setLiteBindTopic(\"test_topic\");\n        subscriptionGroupMap.put(\"test_group\", config);\n        when(subscriptionGroupManager.getSubscriptionGroupTable()).thenReturn(subscriptionGroupMap);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_LITE_INFO, null);\n\n        RemotingCommand response = processor.getBrokerLiteInfo(ctx, request);\n\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n        assertNotNull(response.getBody());\n\n        GetBrokerLiteInfoResponseBody body = GetBrokerLiteInfoResponseBody.decode(response.getBody(), GetBrokerLiteInfoResponseBody.class);\n        assertEquals(\"RocksDB\", body.getStoreType());\n        assertEquals(10000, body.getMaxLmqNum());\n        assertEquals(100, body.getCurrentLmqNum());\n        assertEquals(50, body.getLiteSubscriptionCount());\n        assertNotNull(body.getTopicMeta());\n        assertNotNull(body.getGroupMeta());\n    }\n\n    @Test\n    public void testGetParentTopicInfo_TopicNotExist() throws RemotingCommandException {\n        GetParentTopicInfoRequestHeader requestHeader = new GetParentTopicInfoRequestHeader();\n        requestHeader.setTopic(\"nonexistent_topic\");\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_PARENT_TOPIC_INFO, requestHeader);\n        request.makeCustomHeaderToNet();\n\n        when(topicConfigManager.selectTopicConfig(\"nonexistent_topic\")).thenReturn(null);\n\n        RemotingCommand response = processor.getParentTopicInfo(ctx, request);\n\n        assertEquals(ResponseCode.TOPIC_NOT_EXIST, response.getCode());\n        assertTrue(response.getRemark().contains(\"nonexistent_topic\"));\n    }\n\n    @Test\n    public void testGetParentTopicInfo_InvalidTopicType() throws RemotingCommandException {\n        GetParentTopicInfoRequestHeader requestHeader = new GetParentTopicInfoRequestHeader();\n        requestHeader.setTopic(\"invalid_topic\");\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_PARENT_TOPIC_INFO, requestHeader);\n        request.makeCustomHeaderToNet();\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(\"invalid_topic\");\n        topicConfig.setTopicMessageType(TopicMessageType.NORMAL);\n\n        when(topicConfigManager.selectTopicConfig(\"invalid_topic\")).thenReturn(topicConfig);\n\n        RemotingCommand response = processor.getParentTopicInfo(ctx, request);\n\n        assertEquals(ResponseCode.INVALID_PARAMETER, response.getCode());\n        assertTrue(response.getRemark().contains(\"invalid_topic\"));\n    }\n\n    @Test\n    public void testGetParentTopicInfo_Success() throws RemotingCommandException {\n        GetParentTopicInfoRequestHeader requestHeader = new GetParentTopicInfoRequestHeader();\n        requestHeader.setTopic(\"parent_topic\");\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_PARENT_TOPIC_INFO, requestHeader);\n        request.makeCustomHeaderToNet();\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(\"parent_topic\");\n        topicConfig.setTopicMessageType(TopicMessageType.LITE);\n        topicConfig.setLiteTopicExpiration(3600);\n\n        when(topicConfigManager.selectTopicConfig(\"parent_topic\")).thenReturn(topicConfig);\n        when(consumeQueueStore.getLmqNum()).thenReturn(200);\n        when(liteLifecycleManager.getLiteTopicCount(\"parent_topic\")).thenReturn(10);\n\n        ConcurrentHashMap<String, SubscriptionGroupConfig> subscriptionGroupMap = new ConcurrentHashMap<>();\n        SubscriptionGroupConfig config = new SubscriptionGroupConfig();\n        config.setGroupName(\"test_group\");\n        config.setLiteBindTopic(\"parent_topic\");\n        subscriptionGroupMap.put(\"test_group\", config);\n        when(subscriptionGroupManager.getSubscriptionGroupTable()).thenReturn(subscriptionGroupMap);\n\n        RemotingCommand response = processor.getParentTopicInfo(ctx, request);\n\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n        assertNotNull(response.getBody());\n\n        GetParentTopicInfoResponseBody body = GetParentTopicInfoResponseBody.decode(response.getBody(), GetParentTopicInfoResponseBody.class);\n        assertEquals(\"parent_topic\", body.getTopic());\n        assertEquals(3600, body.getTtl());\n        assertEquals(200, body.getLmqNum());\n        assertEquals(10, body.getLiteTopicCount());\n        assertTrue(body.getGroups().contains(\"test_group\"));\n    }\n\n    @Test\n    public void testGetLiteTopicInfo_ParentTopicNotExist() throws RemotingCommandException {\n        GetLiteTopicInfoRequestHeader requestHeader = new GetLiteTopicInfoRequestHeader();\n        requestHeader.setParentTopic(\"nonexistent_parent\");\n        requestHeader.setLiteTopic(\"lite_topic\");\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_LITE_TOPIC_INFO, requestHeader);\n        request.makeCustomHeaderToNet();\n\n        when(topicConfigManager.selectTopicConfig(\"nonexistent_parent\")).thenReturn(null);\n\n        RemotingCommand response = processor.getLiteTopicInfo(ctx, request);\n\n        assertEquals(ResponseCode.TOPIC_NOT_EXIST, response.getCode());\n        assertTrue(response.getRemark().contains(\"nonexistent_parent\"));\n    }\n\n    @Test\n    public void testGetLiteTopicInfo_InvalidParentTopicType() throws RemotingCommandException {\n        GetLiteTopicInfoRequestHeader requestHeader = new GetLiteTopicInfoRequestHeader();\n        requestHeader.setParentTopic(\"invalid_parent\");\n        requestHeader.setLiteTopic(\"lite_topic\");\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_LITE_TOPIC_INFO, requestHeader);\n        request.makeCustomHeaderToNet();\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(\"invalid_parent\");\n        topicConfig.setTopicMessageType(TopicMessageType.NORMAL);\n\n        when(topicConfigManager.selectTopicConfig(\"invalid_parent\")).thenReturn(topicConfig);\n\n        RemotingCommand response = processor.getLiteTopicInfo(ctx, request);\n\n        assertEquals(ResponseCode.INVALID_PARAMETER, response.getCode());\n        assertTrue(response.getRemark().contains(\"invalid_parent\"));\n    }\n\n    @Test\n    public void testGetLiteTopicInfo_Success() throws RemotingCommandException {\n        GetLiteTopicInfoRequestHeader requestHeader = new GetLiteTopicInfoRequestHeader();\n        requestHeader.setParentTopic(\"parent_topic\");\n        requestHeader.setLiteTopic(\"lite_topic\");\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_LITE_TOPIC_INFO, requestHeader);\n        request.makeCustomHeaderToNet();\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(\"parent_topic\");\n        topicConfig.setTopicMessageType(TopicMessageType.LITE);\n\n        String lmqName = LiteUtil.toLmqName(\"parent_topic\", \"lite_topic\");\n        long maxOffset = 100L;\n        long minOffset = 10L;\n        long lastUpdateTimestamp = System.currentTimeMillis();\n\n        when(topicConfigManager.selectTopicConfig(\"parent_topic\")).thenReturn(topicConfig);\n        when(liteLifecycleManager.getMaxOffsetInQueue(lmqName)).thenReturn(maxOffset);\n        when(messageStore.getMinOffsetInQueue(lmqName, 0)).thenReturn(minOffset);\n        when(messageStore.getMessageStoreTimeStamp(lmqName, 0, maxOffset - 1)).thenReturn(lastUpdateTimestamp);\n        Set<ClientGroup> subscribers = new HashSet<>();\n        subscribers.add(new ClientGroup(\"clientId1\", \"group1\"));\n        when(liteSubscriptionRegistry.getSubscriber(lmqName)).thenReturn(subscribers);\n        when(brokerController.getBrokerConfig()).thenReturn(mock(BrokerConfig.class));\n        when(brokerController.getBrokerConfig().getBrokerName()).thenReturn(\"broker1\");\n        when(liteSharding.shardingByLmqName(\"parent_topic\", lmqName)).thenReturn(\"broker1\");\n\n        RemotingCommand response = processor.getLiteTopicInfo(ctx, request);\n\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n        assertNotNull(response.getBody());\n\n        GetLiteTopicInfoResponseBody body = GetLiteTopicInfoResponseBody.decode(response.getBody(), GetLiteTopicInfoResponseBody.class);\n        assertEquals(\"parent_topic\", body.getParentTopic());\n        assertEquals(\"lite_topic\", body.getLiteTopic());\n        assertEquals(subscribers, body.getSubscriber());\n\n        TopicOffset topicOffset = body.getTopicOffset();\n        assertEquals(minOffset, topicOffset.getMinOffset());\n        assertEquals(maxOffset, topicOffset.getMaxOffset());\n        assertEquals(lastUpdateTimestamp, topicOffset.getLastUpdateTimestamp());\n        assertTrue(body.isShardingToBroker());\n    }\n\n    @Test\n    public void testGetLiteClientInfo_ParentTopicNotExist() throws RemotingCommandException {\n        GetLiteClientInfoRequestHeader requestHeader = new GetLiteClientInfoRequestHeader();\n        requestHeader.setParentTopic(\"nonexistent_parent\");\n        requestHeader.setGroup(\"group1\");\n        requestHeader.setClientId(\"client1\");\n        requestHeader.setMaxCount(100);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_LITE_CLIENT_INFO, requestHeader);\n        request.makeCustomHeaderToNet();\n\n        when(topicConfigManager.selectTopicConfig(\"nonexistent_parent\")).thenReturn(null);\n\n        RemotingCommand response = processor.getLiteClientInfo(ctx, request);\n\n        assertEquals(ResponseCode.TOPIC_NOT_EXIST, response.getCode());\n        assertTrue(response.getRemark().contains(\"nonexistent_parent\"));\n    }\n\n    @Test\n    public void testGetLiteClientInfo_GroupNotExist() throws RemotingCommandException {\n        GetLiteClientInfoRequestHeader requestHeader = new GetLiteClientInfoRequestHeader();\n        requestHeader.setParentTopic(\"parent_topic\");\n        requestHeader.setGroup(\"nonexistent_group\");\n        requestHeader.setClientId(\"client1\");\n        requestHeader.setMaxCount(100);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_LITE_CLIENT_INFO, requestHeader);\n        request.makeCustomHeaderToNet();\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(\"parent_topic\");\n        topicConfig.setTopicMessageType(TopicMessageType.LITE);\n\n        when(topicConfigManager.selectTopicConfig(\"parent_topic\")).thenReturn(topicConfig);\n        when(subscriptionGroupManager.findSubscriptionGroupConfig(\"nonexistent_group\")).thenReturn(null);\n\n        RemotingCommand response = processor.getLiteClientInfo(ctx, request);\n\n        assertEquals(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST, response.getCode());\n        assertTrue(response.getRemark().contains(\"nonexistent_group\"));\n    }\n\n    @Test\n    public void testGetLiteClientInfo_NoSubscription() throws RemotingCommandException {\n        GetLiteClientInfoRequestHeader requestHeader = new GetLiteClientInfoRequestHeader();\n        requestHeader.setParentTopic(\"parent_topic\");\n        requestHeader.setGroup(\"group1\");\n        requestHeader.setClientId(\"client1\");\n        requestHeader.setMaxCount(100);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_LITE_CLIENT_INFO, requestHeader);\n        request.makeCustomHeaderToNet();\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(\"parent_topic\");\n        topicConfig.setTopicMessageType(TopicMessageType.LITE);\n\n        SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig();\n        groupConfig.setGroupName(\"group1\");\n        groupConfig.setLiteBindTopic(\"parent_topic\");\n\n        when(topicConfigManager.selectTopicConfig(\"parent_topic\")).thenReturn(topicConfig);\n        when(subscriptionGroupManager.findSubscriptionGroupConfig(\"group1\")).thenReturn(groupConfig);\n        when(liteSubscriptionRegistry.getLiteSubscription(\"client1\")).thenReturn(null);\n\n        RemotingCommand response = processor.getLiteClientInfo(ctx, request);\n\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n        assertNotNull(response.getBody());\n\n        GetLiteClientInfoResponseBody body = GetLiteClientInfoResponseBody.decode(response.getBody(), GetLiteClientInfoResponseBody.class);\n        assertEquals(\"parent_topic\", body.getParentTopic());\n        assertEquals(\"group1\", body.getGroup());\n        assertEquals(\"client1\", body.getClientId());\n        assertEquals(-1, body.getLiteTopicCount());\n        assertNull(body.getLiteTopicSet());\n    }\n\n    @Test\n    public void testGetLiteClientInfo_WithSubscription() throws RemotingCommandException {\n        GetLiteClientInfoRequestHeader requestHeader = new GetLiteClientInfoRequestHeader();\n        requestHeader.setParentTopic(\"parent_topic\");\n        requestHeader.setGroup(\"group1\");\n        requestHeader.setClientId(\"client1\");\n        requestHeader.setMaxCount(100);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_LITE_CLIENT_INFO, requestHeader);\n        request.makeCustomHeaderToNet();\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(\"parent_topic\");\n        topicConfig.setTopicMessageType(TopicMessageType.LITE);\n\n        SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig();\n        groupConfig.setGroupName(\"group1\");\n        groupConfig.setLiteBindTopic(\"parent_topic\");\n\n        Set<String> liteTopicSet = new HashSet<>();\n        liteTopicSet.add(\"lite_topic1\");\n        liteTopicSet.add(\"lite_topic2\");\n\n        LiteSubscription liteSubscription = new LiteSubscription();\n        liteSubscription.setLiteTopicSet(liteTopicSet);\n\n        when(topicConfigManager.selectTopicConfig(\"parent_topic\")).thenReturn(topicConfig);\n        when(subscriptionGroupManager.findSubscriptionGroupConfig(\"group1\")).thenReturn(groupConfig);\n        when(liteSubscriptionRegistry.getLiteSubscription(\"client1\")).thenReturn(liteSubscription);\n\n        RemotingCommand response = processor.getLiteClientInfo(ctx, request);\n\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n        assertNotNull(response.getBody());\n\n        GetLiteClientInfoResponseBody body = GetLiteClientInfoResponseBody.decode(response.getBody(), GetLiteClientInfoResponseBody.class);\n        assertEquals(\"parent_topic\", body.getParentTopic());\n        assertEquals(\"group1\", body.getGroup());\n        assertEquals(\"client1\", body.getClientId());\n        assertEquals(2, body.getLiteTopicCount());\n        assertEquals(liteTopicSet, body.getLiteTopicSet());\n    }\n\n    @Test\n    public void testGetLiteGroupInfo_GroupNotExist() throws RemotingCommandException {\n        GetLiteGroupInfoRequestHeader requestHeader = new GetLiteGroupInfoRequestHeader();\n        requestHeader.setGroup(\"nonexistent_group\");\n        requestHeader.setLiteTopic(\"\");\n        requestHeader.setTopK(10);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_LITE_GROUP_INFO, requestHeader);\n        request.makeCustomHeaderToNet();\n\n        when(subscriptionGroupManager.findSubscriptionGroupConfig(\"nonexistent_group\")).thenReturn(null);\n\n        RemotingCommand response = processor.getLiteGroupInfo(ctx, request);\n\n        assertEquals(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST, response.getCode());\n        assertTrue(response.getRemark().contains(\"nonexistent_group\"));\n    }\n\n    @Test\n    public void testGetLiteGroupInfo_NotLiteGroup() throws RemotingCommandException {\n        GetLiteGroupInfoRequestHeader requestHeader = new GetLiteGroupInfoRequestHeader();\n        requestHeader.setGroup(\"normal_group\");\n        requestHeader.setLiteTopic(\"\");\n        requestHeader.setTopK(10);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_LITE_GROUP_INFO, requestHeader);\n        request.makeCustomHeaderToNet();\n\n        SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig();\n        groupConfig.setGroupName(\"normal_group\");\n        groupConfig.setLiteBindTopic(\"\");\n\n        when(subscriptionGroupManager.findSubscriptionGroupConfig(\"normal_group\")).thenReturn(groupConfig);\n\n        RemotingCommand response = processor.getLiteGroupInfo(ctx, request);\n\n        assertEquals(ResponseCode.INVALID_PARAMETER, response.getCode());\n        assertTrue(response.getRemark().contains(\"normal_group\"));\n        assertTrue(response.getRemark().contains(\"not a LITE group\"));\n    }\n\n    @Test\n    public void testGetLiteGroupInfo_GetTopKInfo() throws RemotingCommandException {\n        GetLiteGroupInfoRequestHeader requestHeader = new GetLiteGroupInfoRequestHeader();\n        requestHeader.setGroup(\"lite_group\");\n        requestHeader.setLiteTopic(\"\");\n        requestHeader.setTopK(10);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_LITE_GROUP_INFO, requestHeader);\n        request.makeCustomHeaderToNet();\n\n        SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig();\n        groupConfig.setGroupName(\"lite_group\");\n        groupConfig.setLiteBindTopic(\"parent_topic\");\n\n        List<LiteLagInfo> lagCountList = new ArrayList<>();\n        LiteLagInfo lagCountInfo = new LiteLagInfo();\n        lagCountInfo.setLiteTopic(\"topic1\");\n        lagCountInfo.setLagCount(100L);\n        lagCountList.add(lagCountInfo);\n        Pair<List<LiteLagInfo>, Long> lagCountPair = new Pair<>(lagCountList, 100L);\n\n        List<LiteLagInfo> lagTimeList = new ArrayList<>();\n        LiteLagInfo lagTimeInfo = new LiteLagInfo();\n        lagTimeInfo.setLiteTopic(\"topic1\");\n        lagTimeInfo.setEarliestUnconsumedTimestamp(System.currentTimeMillis());\n        lagTimeList.add(lagTimeInfo);\n        Pair<List<LiteLagInfo>, Long> lagTimePair = new Pair<>(lagTimeList, System.currentTimeMillis());\n\n        when(subscriptionGroupManager.findSubscriptionGroupConfig(\"lite_group\")).thenReturn(groupConfig);\n        when(liteConsumerLagCalculator.getLagCountTopK(\"lite_group\", 10)).thenReturn(lagCountPair);\n        when(liteConsumerLagCalculator.getLagTimestampTopK(\"lite_group\", \"parent_topic\", 10)).thenReturn(lagTimePair);\n\n        RemotingCommand response = processor.getLiteGroupInfo(ctx, request);\n\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n        assertNotNull(response.getBody());\n\n        GetLiteGroupInfoResponseBody body = GetLiteGroupInfoResponseBody.decode(response.getBody(), GetLiteGroupInfoResponseBody.class);\n        assertEquals(\"lite_group\", body.getGroup());\n        assertEquals(\"parent_topic\", body.getParentTopic());\n        assertTrue(StringUtils.isEmpty(body.getLiteTopic()));\n        List<LiteLagInfo> actualLagCountList = body.getLagCountTopK();\n        assertEquals(lagCountList.size(), actualLagCountList.size());\n        for (int i = 0; i < lagCountList.size(); i++) {\n            LiteLagInfo expected = lagCountList.get(i);\n            LiteLagInfo actual = actualLagCountList.get(i);\n            assertEquals(expected.getLiteTopic(), actual.getLiteTopic());\n            assertEquals(expected.getLagCount(), actual.getLagCount());\n        }\n        assertEquals(Long.valueOf(100L), Long.valueOf(body.getTotalLagCount()));\n        List<LiteLagInfo> actualLagTimeList = body.getLagTimestampTopK();\n        assertEquals(lagTimeList.size(), actualLagTimeList.size());\n        for (int i = 0; i < lagTimeList.size(); i++) {\n            LiteLagInfo expected = lagTimeList.get(i);\n            LiteLagInfo actual = actualLagTimeList.get(i);\n            assertEquals(expected.getLiteTopic(), actual.getLiteTopic());\n            assertEquals(expected.getEarliestUnconsumedTimestamp(), actual.getEarliestUnconsumedTimestamp());\n        }\n    }\n\n    @Test\n    public void testGetLiteGroupInfo_SpecificLiteTopic_WithMessages() throws RemotingCommandException {\n        GetLiteGroupInfoRequestHeader requestHeader = new GetLiteGroupInfoRequestHeader();\n        requestHeader.setGroup(\"lite_group\");\n        requestHeader.setLiteTopic(\"specific_lite_topic\");\n        requestHeader.setTopK(10);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_LITE_GROUP_INFO, requestHeader);\n        request.makeCustomHeaderToNet();\n\n        SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig();\n        groupConfig.setGroupName(\"lite_group\");\n        groupConfig.setLiteBindTopic(\"parent_topic\");\n\n        String lmqName = LiteUtil.toLmqName(\"parent_topic\", \"specific_lite_topic\");\n        long maxOffset = 100L;\n        long commitOffset = 50L;\n        long messageTimestamp = System.currentTimeMillis() - 10000;\n\n        when(subscriptionGroupManager.findSubscriptionGroupConfig(\"lite_group\")).thenReturn(groupConfig);\n        when(liteLifecycleManager.getMaxOffsetInQueue(lmqName)).thenReturn(maxOffset);\n        when(consumerOffsetManager.queryOffset(\"lite_group\", lmqName, 0)).thenReturn(commitOffset);\n        when(messageStore.getMessageStoreTimeStamp(lmqName, 0, commitOffset)).thenReturn(messageTimestamp);\n        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);\n\n        RemotingCommand response = processor.getLiteGroupInfo(ctx, request);\n\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n        assertNotNull(response.getBody());\n\n        GetLiteGroupInfoResponseBody body = GetLiteGroupInfoResponseBody.decode(response.getBody(), GetLiteGroupInfoResponseBody.class);\n        assertEquals(\"lite_group\", body.getGroup());\n        assertEquals(\"parent_topic\", body.getParentTopic());\n        assertEquals(\"specific_lite_topic\", body.getLiteTopic());\n        assertEquals(maxOffset - commitOffset, body.getTotalLagCount());\n        assertEquals(messageTimestamp, body.getEarliestUnconsumedTimestamp());\n    }\n\n    @Test\n    public void testGetLiteGroupInfo_SpecificLiteTopic_WithoutMessages() throws RemotingCommandException {\n        GetLiteGroupInfoRequestHeader requestHeader = new GetLiteGroupInfoRequestHeader();\n        requestHeader.setGroup(\"lite_group\");\n        requestHeader.setLiteTopic(\"specific_lite_topic\");\n        requestHeader.setTopK(10);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_LITE_GROUP_INFO, requestHeader);\n        request.makeCustomHeaderToNet();\n\n        SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig();\n        groupConfig.setGroupName(\"lite_group\");\n        groupConfig.setLiteBindTopic(\"parent_topic\");\n\n        String lmqName = LiteUtil.toLmqName(\"parent_topic\", \"specific_lite_topic\");\n        long maxOffset = 0L;\n\n        when(subscriptionGroupManager.findSubscriptionGroupConfig(\"lite_group\")).thenReturn(groupConfig);\n        when(liteLifecycleManager.getMaxOffsetInQueue(lmqName)).thenReturn(maxOffset);\n\n        RemotingCommand response = processor.getLiteGroupInfo(ctx, request);\n\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n        assertNotNull(response.getBody());\n\n        GetLiteGroupInfoResponseBody body = GetLiteGroupInfoResponseBody.decode(response.getBody(), GetLiteGroupInfoResponseBody.class);\n        assertEquals(\"lite_group\", body.getGroup());\n        assertEquals(\"parent_topic\", body.getParentTopic());\n        assertEquals(\"specific_lite_topic\", body.getLiteTopic());\n        assertEquals(-1, body.getTotalLagCount());\n        assertEquals(-1L, body.getEarliestUnconsumedTimestamp());\n    }\n\n    @Test\n    public void testGetLiteGroupInfo_SpecificLiteTopic_ZeroCommitOffset() throws RemotingCommandException {\n        GetLiteGroupInfoRequestHeader requestHeader = new GetLiteGroupInfoRequestHeader();\n        requestHeader.setGroup(\"lite_group\");\n        requestHeader.setLiteTopic(\"specific_lite_topic\");\n        requestHeader.setTopK(10);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_LITE_GROUP_INFO, requestHeader);\n        request.makeCustomHeaderToNet();\n\n        SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig();\n        groupConfig.setGroupName(\"lite_group\");\n        groupConfig.setLiteBindTopic(\"parent_topic\");\n\n        String lmqName = LiteUtil.toLmqName(\"parent_topic\", \"specific_lite_topic\");\n        long maxOffset = 100L;\n        long commitOffset = 0L;\n\n        when(subscriptionGroupManager.findSubscriptionGroupConfig(\"lite_group\")).thenReturn(groupConfig);\n        when(liteLifecycleManager.getMaxOffsetInQueue(lmqName)).thenReturn(maxOffset);\n        when(consumerOffsetManager.queryOffset(\"lite_group\", lmqName, 0)).thenReturn(commitOffset);\n        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);\n\n        RemotingCommand response = processor.getLiteGroupInfo(ctx, request);\n\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n        assertNotNull(response.getBody());\n\n        GetLiteGroupInfoResponseBody body = GetLiteGroupInfoResponseBody.decode(response.getBody(), GetLiteGroupInfoResponseBody.class);\n        assertEquals(\"lite_group\", body.getGroup());\n        assertEquals(\"parent_topic\", body.getParentTopic());\n        assertEquals(\"specific_lite_topic\", body.getLiteTopic());\n        assertEquals(maxOffset - commitOffset, body.getTotalLagCount());\n        assertEquals(0, body.getEarliestUnconsumedTimestamp());\n    }\n\n    @Test\n    public void testTriggerLiteDispatch() throws Exception {\n        String group = \"group\";\n        String clientId = \"clientId\";\n        TriggerLiteDispatchRequestHeader requestHeader;\n\n        // with clientId\n        requestHeader = new TriggerLiteDispatchRequestHeader();\n        requestHeader.setGroup(group);\n        requestHeader.setClientId(clientId);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.TRIGGER_LITE_DISPATCH, requestHeader);\n        request.makeCustomHeaderToNet();\n\n        SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig();\n        groupConfig.setGroupName(group);\n        groupConfig.setLiteBindTopic(\"parent_topic\");\n        when(subscriptionGroupManager.findSubscriptionGroupConfig(group)).thenReturn(groupConfig);\n\n        RemotingCommand response = processor.triggerLiteDispatch(ctx, request);\n\n        assertNotNull(response);\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n        verify(liteEventDispatcher, times(1)).doFullDispatch(clientId, group);\n        verify(liteEventDispatcher, never()).doFullDispatchByGroup(group);\n\n        // without clientId\n        requestHeader = new TriggerLiteDispatchRequestHeader();\n        requestHeader.setGroup(group);\n        request = RemotingCommand.createRequestCommand(RequestCode.TRIGGER_LITE_DISPATCH, requestHeader);\n        request.makeCustomHeaderToNet();\n\n        response = processor.triggerLiteDispatch(ctx, request);\n\n        assertNotNull(response);\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n        verify(liteEventDispatcher, times(1)).doFullDispatch(clientId, group);\n        verify(liteEventDispatcher, times(1)).doFullDispatchByGroup(group);\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/processor/LiteSubscriptionCtlProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.Collections;\nimport java.util.HashSet;\nimport java.util.Set;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.lite.LiteSubscriptionRegistry;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.common.lite.LiteSubscriptionAction;\nimport org.apache.rocketmq.common.lite.LiteSubscriptionDTO;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.LiteSubscriptionCtlRequestBody;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.InjectMocks;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\nimport static org.junit.jupiter.api.Assertions.assertTrue;\nimport static org.mockito.Mockito.any;\nimport static org.mockito.Mockito.anySet;\nimport static org.mockito.Mockito.eq;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class LiteSubscriptionCtlProcessorTest {\n\n    @Mock\n    private BrokerController brokerController;\n\n    @Mock\n    private SubscriptionGroupManager subscriptionGroupManager;\n\n    @Mock\n    private LiteSubscriptionRegistry liteSubscriptionRegistry;\n\n    @Mock\n    private ChannelHandlerContext ctx;\n\n    @Mock\n    private Channel channel;\n\n    @InjectMocks\n    private LiteSubscriptionCtlProcessor processor;\n\n    @Test\n    public void testProcessRequest_BodyIsNull() throws Exception {\n        RemotingCommand request = RemotingCommand.createRequestCommand(0, null);\n        RemotingCommand response = processor.processRequest(ctx, request);\n        assertEquals(ResponseCode.ILLEGAL_OPERATION, response.getCode());\n    }\n\n    @Test\n    public void testProcessRequest_SubscriptionSetIsEmpty() throws Exception {\n        LiteSubscriptionCtlRequestBody requestBody = new LiteSubscriptionCtlRequestBody();\n        requestBody.setSubscriptionSet(Collections.emptySet());\n        RemotingCommand request = RemotingCommand.createRequestCommand(0, null);\n        request.setBody(requestBody.encode());\n        RemotingCommand response = processor.processRequest(ctx, request);\n        assertEquals(ResponseCode.ILLEGAL_OPERATION, response.getCode());\n    }\n\n    @Test\n    public void testProcessRequest_ActionIsIncrementalAdd() throws Exception {\n        String clientId = \"clientId\";\n        String group = \"group\";\n        String topic = \"topic\";\n        String liteTopic = \"liteTopic\";\n        Set<String> liteTopicSet = new HashSet<>();\n        liteTopicSet.add(liteTopic);\n\n        LiteSubscriptionDTO dto = new LiteSubscriptionDTO();\n        dto.setClientId(clientId);\n        dto.setGroup(group);\n        dto.setTopic(topic);\n        dto.setLiteTopicSet(liteTopicSet);\n        dto.setAction(LiteSubscriptionAction.PARTIAL_ADD);\n        dto.setVersion(1L);\n\n        Set<LiteSubscriptionDTO> subscriptionSet = new HashSet<>();\n        subscriptionSet.add(dto);\n\n        LiteSubscriptionCtlRequestBody requestBody = new LiteSubscriptionCtlRequestBody();\n        requestBody.setSubscriptionSet(subscriptionSet);\n        RemotingCommand request = RemotingCommand.createRequestCommand(0, null);\n        request.setBody(requestBody.encode());\n\n        when(ctx.channel()).thenReturn(channel);\n        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);\n        SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig();\n        groupConfig.setConsumeEnable(true);\n        when(subscriptionGroupManager.findSubscriptionGroupConfig(group)).thenReturn(groupConfig);\n\n        RemotingCommand response = processor.processRequest(ctx, request);\n\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n        verify(liteSubscriptionRegistry).updateClientChannel(eq(clientId), eq(channel));\n        verify(liteSubscriptionRegistry).addPartialSubscription(eq(clientId), eq(group), eq(topic), anySet(), any());\n    }\n\n    @Test\n    public void testProcessRequest_ActionIsAllAdd() throws Exception {\n        String clientId = \"clientId\";\n        String group = \"group\";\n        String topic = \"topic\";\n        String liteTopic = \"liteTopic\";\n        Set<String> liteTopicSet = new HashSet<>();\n        liteTopicSet.add(liteTopic);\n\n        LiteSubscriptionDTO dto = new LiteSubscriptionDTO();\n        dto.setClientId(clientId);\n        dto.setGroup(group);\n        dto.setTopic(topic);\n        dto.setLiteTopicSet(liteTopicSet);\n        dto.setAction(LiteSubscriptionAction.COMPLETE_ADD);\n        dto.setVersion(1L);\n\n        Set<LiteSubscriptionDTO> subscriptionSet = new HashSet<>();\n        subscriptionSet.add(dto);\n\n        LiteSubscriptionCtlRequestBody requestBody = new LiteSubscriptionCtlRequestBody();\n        requestBody.setSubscriptionSet(subscriptionSet);\n        RemotingCommand request = RemotingCommand.createRequestCommand(0, null);\n        request.setBody(requestBody.encode());\n\n        when(ctx.channel()).thenReturn(channel);\n        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);\n        SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig();\n        groupConfig.setConsumeEnable(true);\n        when(subscriptionGroupManager.findSubscriptionGroupConfig(group)).thenReturn(groupConfig);\n\n        RemotingCommand response = processor.processRequest(ctx, request);\n\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n        verify(liteSubscriptionRegistry).updateClientChannel(eq(clientId), eq(channel));\n        verify(liteSubscriptionRegistry).addCompleteSubscription(eq(clientId), eq(group), eq(topic), anySet(), eq(1L));\n    }\n\n    @Test\n    public void testProcessRequest_ActionIsIncrementalRemove() throws Exception {\n        String clientId = \"clientId\";\n        String group = \"group\";\n        String topic = \"topic\";\n        String liteTopic = \"liteTopic\";\n        Set<String> liteTopicSet = new HashSet<>();\n        liteTopicSet.add(liteTopic);\n\n        LiteSubscriptionDTO dto = new LiteSubscriptionDTO();\n        dto.setClientId(clientId);\n        dto.setGroup(group);\n        dto.setTopic(topic);\n        dto.setLiteTopicSet(liteTopicSet);\n        dto.setAction(LiteSubscriptionAction.PARTIAL_REMOVE);\n\n        Set<LiteSubscriptionDTO> subscriptionSet = new HashSet<>();\n        subscriptionSet.add(dto);\n\n        LiteSubscriptionCtlRequestBody requestBody = new LiteSubscriptionCtlRequestBody();\n        requestBody.setSubscriptionSet(subscriptionSet);\n        RemotingCommand request = RemotingCommand.createRequestCommand(0, null);\n        request.setBody(requestBody.encode());\n\n        RemotingCommand response = processor.processRequest(ctx, request);\n\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n        verify(liteSubscriptionRegistry).removePartialSubscription(eq(clientId), eq(group), eq(topic), anySet());\n    }\n\n    @Test\n    public void testProcessRequest_ActionIsAllRemove() throws Exception {\n        String clientId = \"clientId\";\n\n        LiteSubscriptionDTO dto = new LiteSubscriptionDTO();\n        String group = \"group\";\n        String topic = \"topic\";\n        dto.setClientId(clientId);\n        dto.setTopic(topic);\n        dto.setGroup(group);\n        dto.setAction(LiteSubscriptionAction.COMPLETE_REMOVE);\n\n        Set<LiteSubscriptionDTO> subscriptionSet = new HashSet<>();\n        subscriptionSet.add(dto);\n\n        LiteSubscriptionCtlRequestBody requestBody = new LiteSubscriptionCtlRequestBody();\n        requestBody.setSubscriptionSet(subscriptionSet);\n        RemotingCommand request = RemotingCommand.createRequestCommand(0, null);\n        request.setBody(requestBody.encode());\n\n        RemotingCommand response = processor.processRequest(ctx, request);\n\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n        verify(liteSubscriptionRegistry).removeCompleteSubscription(eq(clientId));\n    }\n\n    @Test\n    public void testProcessRequest_CheckConsumeEnableThrowsException() throws Exception {\n        String clientId = \"clientId\";\n        String group = \"group\";\n        String topic = \"topic\";\n        String liteTopic = \"liteTopic\";\n        Set<String> liteTopicSet = new HashSet<>();\n        liteTopicSet.add(liteTopic);\n\n        LiteSubscriptionDTO dto = new LiteSubscriptionDTO();\n        dto.setClientId(clientId);\n        dto.setGroup(group);\n        dto.setTopic(topic);\n        dto.setLiteTopicSet(liteTopicSet);\n        dto.setAction(LiteSubscriptionAction.PARTIAL_ADD);\n\n        Set<LiteSubscriptionDTO> subscriptionSet = new HashSet<>();\n        subscriptionSet.add(dto);\n\n        LiteSubscriptionCtlRequestBody requestBody = new LiteSubscriptionCtlRequestBody();\n        requestBody.setSubscriptionSet(subscriptionSet);\n        RemotingCommand request = RemotingCommand.createRequestCommand(0, null);\n        request.setBody(requestBody.encode());\n\n        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);\n        SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig();\n        groupConfig.setConsumeEnable(false);\n        when(subscriptionGroupManager.findSubscriptionGroupConfig(group)).thenReturn(groupConfig);\n\n        RemotingCommand response = processor.processRequest(ctx, request);\n\n        assertEquals(ResponseCode.ILLEGAL_OPERATION, response.getCode());\n        assertTrue(response.getRemark().contains(\"Consumer group is not allowed to consume.\"));\n    }\n\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/processor/PeekMessageProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.metrics.BrokerMetricsManager;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.PeekMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.net.InetSocketAddress;\nimport java.nio.ByteBuffer;\n\nimport static org.assertj.core.api.AssertionsForClassTypes.assertThat;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class PeekMessageProcessorTest {\n\n    private PeekMessageProcessor peekMessageProcessor;\n\n    @Spy\n    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());\n\n    @Mock\n    private ChannelHandlerContext handlerContext;\n\n    @Mock\n    private MessageStore messageStore;\n\n    @Mock\n    private SubscriptionGroupManager subscriptionGroupManager;\n\n    @Mock\n    private ConsumerOffsetManager consumerOffsetManager;\n\n    @Mock\n    private SubscriptionGroupConfig subscriptionGroupConfig;\n\n    @Mock\n    private Channel channel;\n\n    private TopicConfigManager topicConfigManager;\n\n    @Before\n    public void init() {\n        // Initialize BrokerMetricsManager to prevent NPE in tests\n        brokerController.setBrokerMetricsManager(new BrokerMetricsManager(brokerController));\n        peekMessageProcessor = new PeekMessageProcessor(brokerController);\n        when(brokerController.getMessageStore()).thenReturn(messageStore);\n        topicConfigManager = new TopicConfigManager(brokerController);\n        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);\n        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);\n        when(subscriptionGroupManager.findSubscriptionGroupConfig(anyString())).thenReturn(subscriptionGroupConfig);\n        when(subscriptionGroupConfig.isConsumeEnable()).thenReturn(true);\n        topicConfigManager.getTopicConfigTable().put(\"topic\", new TopicConfig(\"topic\"));\n        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);\n        when(consumerOffsetManager.queryOffset(anyString(), anyString(), anyInt())).thenReturn(-1L);\n        when(messageStore.getMinOffsetInQueue(anyString(),anyInt())).thenReturn(0L);\n        when(handlerContext.channel()).thenReturn(channel);\n        when(channel.remoteAddress()).thenReturn(new InetSocketAddress(\"127.0.0.1\", 12345));\n    }\n\n    @Test\n    public void testProcessRequest() throws RemotingCommandException {\n        RemotingCommand request = createPeekMessageRequest(\"group\",\"topic\",0);\n        GetMessageResult getMessageResult = new GetMessageResult();\n        getMessageResult.setStatus(GetMessageStatus.FOUND);\n        ByteBuffer bb = ByteBuffer.allocate(64);\n        bb.putLong(MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION, System.currentTimeMillis());\n        SelectMappedBufferResult mappedBufferResult1 = new SelectMappedBufferResult(0, bb, 64, null);\n        for (int i = 0; i < 10;i++) {\n            getMessageResult.addMessage(mappedBufferResult1);\n        }\n        when(messageStore.getMessage(anyString(),anyString(),anyInt(),anyLong(),anyInt(),any())).thenReturn(getMessageResult);\n        RemotingCommand response = peekMessageProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testProcessRequest_NoPermission() throws RemotingCommandException {\n        this.brokerController.getBrokerConfig().setBrokerPermission(PermName.PERM_WRITE);\n        RemotingCommand request = createPeekMessageRequest(\"group\",\"topic\",0);\n        RemotingCommand response = peekMessageProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);\n        this.brokerController.getBrokerConfig().setBrokerPermission(PermName.PERM_WRITE | PermName.PERM_READ);\n\n        topicConfigManager.getTopicConfigTable().get(\"topic\").setPerm(PermName.PERM_WRITE);\n        response = peekMessageProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);\n        topicConfigManager.getTopicConfigTable().get(\"topic\").setPerm(PermName.PERM_WRITE | PermName.PERM_READ);\n\n        when(subscriptionGroupConfig.isConsumeEnable()).thenReturn(false);\n        response = peekMessageProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);\n    }\n\n    @Test\n    public void testProcessRequest_TopicNotExist() throws RemotingCommandException {\n        RemotingCommand request = createPeekMessageRequest(\"group1\",\"topic1\",0);\n        RemotingCommand response = peekMessageProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.TOPIC_NOT_EXIST);\n    }\n\n    @Test\n    public void testProcessRequest_SubscriptionGroupNotExist() throws RemotingCommandException {\n        when(subscriptionGroupManager.findSubscriptionGroupConfig(anyString())).thenReturn(null);\n        RemotingCommand request = createPeekMessageRequest(\"group\",\"topic\",0);\n        RemotingCommand response = peekMessageProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);\n    }\n\n    @Test\n    public void testProcessRequest_QueueIdError() throws RemotingCommandException {\n        RemotingCommand request = createPeekMessageRequest(\"group\",\"topic\",17);\n        RemotingCommand response = peekMessageProcessor.processRequest(handlerContext, request);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER);\n    }\n\n    private RemotingCommand createPeekMessageRequest(String group,String topic,int queueId) {\n        PeekMessageRequestHeader peekMessageRequestHeader = new PeekMessageRequestHeader();\n        peekMessageRequestHeader.setConsumerGroup(group);\n        peekMessageRequestHeader.setTopic(topic);\n        peekMessageRequestHeader.setMaxMsgNums(10);\n        peekMessageRequestHeader.setQueueId(queueId);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PEEK_MESSAGE, peekMessageRequestHeader);\n        request.makeCustomHeaderToNet();\n        return request;\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/processor/PopBufferMergeServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport com.alibaba.fastjson2.JSON;\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.client.ConsumerManager;\nimport org.apache.rocketmq.broker.failover.EscapeBridge;\nimport org.apache.rocketmq.broker.metrics.BrokerMetricsManager;\nimport org.apache.rocketmq.broker.metrics.PopMetricsManager;\nimport org.apache.rocketmq.broker.schedule.ScheduleMessageService;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.pop.AckMsg;\nimport org.apache.rocketmq.store.pop.PopCheckPoint;\nimport org.junit.Assume;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.lang.reflect.Method;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.atomic.AtomicInteger;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.Mockito.doNothing;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.Silent.class)\npublic class PopBufferMergeServiceTest {\n\n    @Mock\n    private BrokerController brokerController;\n\n    private PopMessageProcessor popMessageProcessor;\n\n    @Mock\n    private ScheduleMessageService scheduleMessageService;\n\n    @Mock\n    private TopicConfigManager topicConfigManager;\n\n    @Mock\n    private ConsumerManager consumerManager;\n\n    @Mock\n    private DefaultMessageStore messageStore;\n\n    @Mock\n    private MessageStoreConfig messageStoreConfig;\n\n    private String defaultGroup = \"defaultGroup\";\n\n    private String defaultTopic = \"defaultTopic\";\n\n    private PopBufferMergeService popBufferMergeService;\n\n    @Mock\n    private BrokerConfig brokerConfig;\n\n    @Mock\n    private EscapeBridge escapeBridge;\n\n    @Before\n    public void init() throws Exception {\n        when(brokerConfig.getBrokerIP1()).thenReturn(\"127.0.0.1\");\n        when(brokerConfig.isEnablePopBufferMerge()).thenReturn(true);\n        when(brokerConfig.getPopCkStayBufferTime()).thenReturn(10 * 1000);\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        when(brokerController.getEscapeBridge()).thenReturn(escapeBridge);\n        when(brokerController.getMessageStore()).thenReturn(messageStore);\n        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);\n        when(brokerController.getScheduleMessageService()).thenReturn(scheduleMessageService);\n        when(brokerController.getConsumerManager()).thenReturn(consumerManager);\n        when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n        popMessageProcessor = new PopMessageProcessor(brokerController);\n        popBufferMergeService = new PopBufferMergeService(brokerController, popMessageProcessor);\n        FieldUtils.writeDeclaredField(popBufferMergeService, \"brokerController\", brokerController, true);\n        ConcurrentMap<String, TopicConfig> topicConfigTable = new ConcurrentHashMap<>();\n        topicConfigTable.put(defaultTopic, new TopicConfig());\n        when(topicConfigManager.getTopicConfigTable()).thenReturn(topicConfigTable);\n    }\n\n    @Test(timeout = 15_000)\n    public void testBasic() throws Exception {\n        // This test case fails on Windows in CI pipeline\n        // Disable it for later fix\n        Assume.assumeFalse(MixAll.isWindows());\n        PopCheckPoint ck = new PopCheckPoint();\n        ck.setBitMap(0);\n        int msgCnt = 1;\n        ck.setNum((byte) msgCnt);\n        long popTime = System.currentTimeMillis() - 1000;\n        ck.setPopTime(popTime);\n        int invisibleTime = 30_000;\n        ck.setInvisibleTime(invisibleTime);\n        int offset = 100;\n        ck.setStartOffset(offset);\n        ck.setCId(defaultGroup);\n        ck.setTopic(defaultTopic);\n        int queueId = 0;\n        ck.setQueueId(queueId);\n\n        int reviveQid = 0;\n        long nextBeginOffset = 101L;\n        long ackOffset = offset;\n        AckMsg ackMsg = new AckMsg();\n        ackMsg.setAckOffset(ackOffset);\n        ackMsg.setStartOffset(offset);\n        ackMsg.setConsumerGroup(defaultGroup);\n        ackMsg.setTopic(defaultTopic);\n        ackMsg.setQueueId(queueId);\n        ackMsg.setPopTime(popTime);\n        try {\n            assertThat(popBufferMergeService.addCk(ck, reviveQid, ackOffset, nextBeginOffset)).isTrue();\n            assertThat(popBufferMergeService.getLatestOffset(defaultTopic, defaultGroup, queueId)).isEqualTo(nextBeginOffset);\n            Thread.sleep(1000); // wait background threads of PopBufferMergeService run for some time\n            assertThat(popBufferMergeService.addAk(reviveQid, ackMsg)).isTrue();\n            assertThat(popBufferMergeService.getLatestOffset(defaultTopic, defaultGroup, queueId)).isEqualTo(nextBeginOffset);\n        } finally {\n            popBufferMergeService.shutdown(true);\n        }\n    }\n\n    @Test\n    public void testAddCkJustOffset_MergeKeyConflict() {\n        PopCheckPoint point = mock(PopCheckPoint.class);\n        String mergeKey = \"testMergeKey\";\n        when(point.getTopic()).thenReturn(mergeKey);\n        when(point.getCId()).thenReturn(\"\");\n        when(point.getQueueId()).thenReturn(0);\n        when(point.getStartOffset()).thenReturn(0L);\n        when(point.getPopTime()).thenReturn(0L);\n        when(point.getBrokerName()).thenReturn(\"\");\n        popBufferMergeService.buffer.put(mergeKey + \"000\", mock(PopBufferMergeService.PopCheckPointWrapper.class));\n\n        assertFalse(popBufferMergeService.addCkJustOffset(point, 0, 0, 0));\n    }\n\n    @Test\n    public void testAddCkMock() {\n        int queueId = 0;\n        long startOffset = 100L;\n        long invisibleTime = 30_000L;\n        long popTime = System.currentTimeMillis();\n        int reviveQueueId = 0;\n        long nextBeginOffset = 101L;\n        String brokerName = \"brokerName\";\n        popBufferMergeService.addCkMock(defaultGroup, defaultTopic, queueId, startOffset, invisibleTime, popTime, reviveQueueId, nextBeginOffset, brokerName);\n        verify(brokerConfig, times(1)).isEnablePopLog();\n    }\n\n    @Test\n    public void testPutAckToStore() throws Exception {\n        PopCheckPoint point = new PopCheckPoint();\n        point.setStartOffset(100L);\n        point.setCId(\"testGroup\");\n        point.setTopic(\"testTopic\");\n        point.setQueueId(1);\n        point.setPopTime(System.currentTimeMillis());\n        point.setBrokerName(\"testBroker\");\n\n        PopBufferMergeService.PopCheckPointWrapper pointWrapper = mock(PopBufferMergeService.PopCheckPointWrapper.class);\n        when(pointWrapper.getCk()).thenReturn(point);\n        when(pointWrapper.getReviveQueueId()).thenReturn(0);\n\n        AtomicInteger toStoreBits = new AtomicInteger(0);\n        when(pointWrapper.getToStoreBits()).thenReturn(toStoreBits);\n\n        byte msgIndex = 0;\n        AtomicInteger count = new AtomicInteger(0);\n\n        EscapeBridge escapeBridge = mock(EscapeBridge.class);\n        when(brokerController.getEscapeBridge()).thenReturn(escapeBridge);\n        when(brokerController.getBrokerConfig().isAppendAckAsync()).thenReturn(false);\n        BrokerMetricsManager brokerMetricsManager = mock(BrokerMetricsManager.class);\n        PopMetricsManager popMetricsManager = mock(PopMetricsManager.class);\n\n        when(brokerMetricsManager.getPopMetricsManager()).thenReturn(popMetricsManager);\n        when(brokerController.getBrokerMetricsManager()).thenReturn(brokerMetricsManager);\n        doNothing().when(popMetricsManager).incPopReviveCkPutCount(any(), any());\n        when(brokerMetricsManager.getPopMetricsManager()).thenReturn(popMetricsManager);\n\n        when(escapeBridge.putMessageToSpecificQueue(any())).thenAnswer(invocation -> {\n            MessageExtBrokerInner capturedMessage = invocation.getArgument(0);\n            AckMsg ackMsg = JSON.parseObject(capturedMessage.getBody(), AckMsg.class);\n\n            assertEquals(point.ackOffsetByIndex(msgIndex), ackMsg.getAckOffset());\n            assertEquals(point.getStartOffset(), ackMsg.getStartOffset());\n            assertEquals(point.getCId(), ackMsg.getConsumerGroup());\n            assertEquals(point.getTopic(), ackMsg.getTopic());\n            assertEquals(point.getQueueId(), ackMsg.getQueueId());\n            assertEquals(point.getPopTime(), ackMsg.getPopTime());\n            assertEquals(point.getBrokerName(), ackMsg.getBrokerName());\n\n            PutMessageResult result = mock(PutMessageResult.class);\n            when(result.getPutMessageStatus()).thenReturn(PutMessageStatus.PUT_OK);\n            return result;\n        });\n\n        Method method = PopBufferMergeService.class.getDeclaredMethod(\"putAckToStore\", PopBufferMergeService.PopCheckPointWrapper.class, byte.class, AtomicInteger.class);\n        method.setAccessible(true);\n        method.invoke(popBufferMergeService, pointWrapper, msgIndex, count);\n        verify(escapeBridge, times(1)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class));\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/processor/PopInflightMessageCounterTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.store.pop.PopCheckPoint;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\npublic class PopInflightMessageCounterTest {\n\n    @Test\n    public void testNum() {\n        BrokerController brokerController = mock(BrokerController.class);\n        long brokerStartTime = System.currentTimeMillis();\n        when(brokerController.getShouldStartTime()).thenReturn(brokerStartTime);\n        PopInflightMessageCounter counter = new PopInflightMessageCounter(brokerController);\n\n        final String topic = \"topic\";\n        final String group = \"group\";\n\n        assertEquals(0, counter.getGroupPopInFlightMessageNum(topic, group, 0));\n\n        counter.incrementInFlightMessageNum(topic, group, 0, 3);\n        assertEquals(3, counter.getGroupPopInFlightMessageNum(topic, group, 0));\n\n        counter.decrementInFlightMessageNum(topic, group, System.currentTimeMillis(), 0, 1);\n        assertEquals(2, counter.getGroupPopInFlightMessageNum(topic, group, 0));\n\n        counter.decrementInFlightMessageNum(topic, group, System.currentTimeMillis() - 1000, 0, 1);\n        assertEquals(2, counter.getGroupPopInFlightMessageNum(topic, group, 0));\n\n        PopCheckPoint popCheckPoint = new PopCheckPoint();\n        popCheckPoint.setTopic(topic);\n        popCheckPoint.setCId(group);\n        popCheckPoint.setQueueId(0);\n        popCheckPoint.setPopTime(System.currentTimeMillis());\n\n        counter.decrementInFlightMessageNum(popCheckPoint);\n        assertEquals(1, counter.getGroupPopInFlightMessageNum(topic, group, 0));\n\n        counter.decrementInFlightMessageNum(topic, group, System.currentTimeMillis(), 0 ,1);\n        assertEquals(0, counter.getGroupPopInFlightMessageNum(topic, group, 0));\n\n        counter.decrementInFlightMessageNum(topic, group, System.currentTimeMillis(), 0, 1);\n        assertEquals(0, counter.getGroupPopInFlightMessageNum(topic, group, 0));\n    }\n\n    @Test\n    public void testClearInFlightMessageNum() {\n        BrokerController brokerController = mock(BrokerController.class);\n        long brokerStartTime = System.currentTimeMillis();\n        when(brokerController.getShouldStartTime()).thenReturn(brokerStartTime);\n        PopInflightMessageCounter counter = new PopInflightMessageCounter(brokerController);\n\n        final String topic = \"topic\";\n        final String group = \"group\";\n\n        assertEquals(0, counter.getGroupPopInFlightMessageNum(topic, group, 0));\n\n        counter.incrementInFlightMessageNum(topic, group, 0, 3);\n        assertEquals(3, counter.getGroupPopInFlightMessageNum(topic, group, 0));\n\n        counter.clearInFlightMessageNumByTopicName(\"errorTopic\");\n        assertEquals(3, counter.getGroupPopInFlightMessageNum(topic, group, 0));\n\n        counter.clearInFlightMessageNumByTopicName(topic);\n        assertEquals(0, counter.getGroupPopInFlightMessageNum(topic, group, 0));\n\n        counter.incrementInFlightMessageNum(topic, group, 0, 3);\n        assertEquals(3, counter.getGroupPopInFlightMessageNum(topic, group, 0));\n\n        counter.clearInFlightMessageNumByGroupName(\"errorGroup\");\n        assertEquals(3, counter.getGroupPopInFlightMessageNum(topic, group, 0));\n\n        counter.clearInFlightMessageNumByGroupName(group);\n        assertEquals(0, counter.getGroupPopInFlightMessageNum(topic, group, 0));\n    }\n}"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/processor/PopLiteMessageProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.lite.AbstractLiteLifecycleManager;\nimport org.apache.rocketmq.broker.lite.LiteEventDispatcher;\nimport org.apache.rocketmq.broker.longpolling.PopLiteLongPollingService;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.broker.pop.PopConsumerLockService;\nimport org.apache.rocketmq.broker.pop.orderly.ConsumerOrderInfoManager;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.PopLiteMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopLiteMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.util.Iterator;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.doNothing;\nimport static org.mockito.Mockito.when;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.never;\nimport static org.mockito.Mockito.doReturn;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class PopLiteMessageProcessorTest {\n\n    @Mock\n    private BrokerController brokerController;\n    @Mock\n    private MessageStore messageStore;\n    @Mock\n    private LiteEventDispatcher liteEventDispatcher;\n    @Mock\n    private PopLiteLongPollingService popLiteLongPollingService;\n    @Mock\n    private PopConsumerLockService lockService;\n    @Mock\n    private ConsumerOrderInfoManager consumerOrderInfoManager;\n    @Mock\n    private ConsumerOffsetManager consumerOffsetManager;\n    @Mock\n    private PopMessageProcessor popMessageProcessor;\n    @Mock\n    private TopicConfigManager topicConfigManager;\n    @Mock\n    private SubscriptionGroupManager subscriptionGroupManager;\n    @Mock\n    private AbstractLiteLifecycleManager liteLifecycleManager;\n\n    private BrokerConfig brokerConfig;\n    private PopLiteMessageProcessor popLiteMessageProcessor;\n\n    @Before\n    public void setUp() throws Exception {\n        brokerConfig = new BrokerConfig();\n        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);\n        when(brokerController.getPopMessageProcessor()).thenReturn(popMessageProcessor);\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        when(brokerController.getMessageStore()).thenReturn(messageStore);\n        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);\n        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);\n        when(brokerController.getLiteLifecycleManager()).thenReturn(liteLifecycleManager);\n\n        PopLiteMessageProcessor testObject = new PopLiteMessageProcessor(brokerController, liteEventDispatcher);\n        FieldUtils.writeDeclaredField(testObject, \"popLiteLongPollingService\", popLiteLongPollingService, true);\n        FieldUtils.writeDeclaredField(testObject, \"lockService\", lockService, true);\n        FieldUtils.writeDeclaredField(testObject, \"consumerOrderInfoManager\", consumerOrderInfoManager, true);\n        popLiteMessageProcessor = Mockito.spy(testObject);\n    }\n\n    @Test\n    public void testRejectRequest() {\n        assertFalse(popLiteMessageProcessor.rejectRequest());\n    }\n\n    @Test\n    public void testTransformOrderCountInfo_empty() {\n        StringBuilder result = popLiteMessageProcessor.transformOrderCountInfo(new StringBuilder(), 3);\n        assertEquals(\"0;0;0\", result.toString());\n    }\n\n    @Test\n    public void testTransformOrderCountInfo_onlyQueueIdInfo() {\n        StringBuilder input = new StringBuilder(\"0\" + MessageConst.KEY_SEPARATOR + \"0\" + MessageConst.KEY_SEPARATOR + \"2\");\n        StringBuilder result = popLiteMessageProcessor.transformOrderCountInfo(input, 3);\n        assertEquals(\"2;2;2\", result.toString());\n    }\n\n    @Test\n    public void testTransformOrderCountInfo_consumeCountAndQueueIdInfo() {\n        StringBuilder input = new StringBuilder(\"0 qo0%0 0;0 qo0%1 1;0 0 1\");\n        StringBuilder result = popLiteMessageProcessor.transformOrderCountInfo(input, 2);\n        assertEquals(\"0 qo0%0 0;0 qo0%1 1\", result.toString());\n    }\n\n    @Test\n    public void testIsFifoBlocked() {\n        when(consumerOrderInfoManager.checkBlock(anyString(), anyString(), anyString(), anyInt(), anyLong()))\n            .thenReturn(true);\n        assertTrue(popLiteMessageProcessor.isFifoBlocked(\"attemptId\", \"group\", \"lmqName\", 1000L));\n        verify(consumerOrderInfoManager).checkBlock(\"attemptId\", \"lmqName\", \"group\", 0, 1000L);\n    }\n\n    @Test\n    public void testGetPopOffset_normal() throws ConsumeQueueException {\n        String group = \"group\";\n        String lmqName = \"lmqName\";\n        long consumerOffset = 100L;\n\n        // exist\n        when(consumerOffsetManager.queryOffset(group, lmqName, 0)).thenReturn(consumerOffset);\n        when(consumerOffsetManager.queryThenEraseResetOffset(lmqName, group, 0)).thenReturn(null);\n        assertEquals(consumerOffset, popLiteMessageProcessor.getPopOffset(group, lmqName));\n\n        // not exist, init mode\n        long initOffset = 10L;\n        when(consumerOffsetManager.queryOffset(group, lmqName, 0)).thenReturn(-1L);\n        when(popMessageProcessor.getInitOffset(lmqName, group, 0, 1, true)).thenReturn(initOffset);\n\n        assertEquals(initOffset, popLiteMessageProcessor.getPopOffset(group, lmqName));\n\n        verify(consumerOffsetManager, times(2)).queryThenEraseResetOffset(lmqName, group, 0);\n        verify(consumerOrderInfoManager, never()).clearBlock(anyString(), anyString(), anyInt());\n        verify(consumerOffsetManager, never()).commitOffset(anyString(), anyString(), anyString(), anyInt(), anyLong());\n    }\n\n\n    @Test\n    public void testGetPopOffset_resetOffset() {\n        String group = \"group\";\n        String lmqName = \"lmq\";\n        long consumerOffset = 100L;\n        long resetOffset = 50L;\n\n        when(consumerOffsetManager.queryOffset(group, lmqName, 0)).thenReturn(consumerOffset);\n        when(consumerOffsetManager.queryThenEraseResetOffset(lmqName, group, 0)).thenReturn(resetOffset);\n\n        assertEquals(resetOffset, popLiteMessageProcessor.getPopOffset(group, lmqName));\n\n        verify(consumerOffsetManager).queryOffset(group, lmqName, 0);\n        verify(consumerOffsetManager).queryThenEraseResetOffset(lmqName, group, 0);\n        verify(consumerOrderInfoManager).clearBlock(lmqName, group, 0);\n        verify(consumerOffsetManager).commitOffset(\"ResetOffset\", group, lmqName, 0, resetOffset);\n    }\n\n    @SuppressWarnings(\"unchecked\")\n    @Test\n    public void testPopByClientId_noEvent() {\n        Iterator<String> mockIterator = mock(Iterator.class);\n        when(mockIterator.hasNext()).thenReturn(false);\n        when(liteEventDispatcher.getEventIterator(\"clientId\")).thenReturn(mockIterator);\n\n        Pair<StringBuilder, GetMessageResult> result = popLiteMessageProcessor.popByClientId(\n            \"clientHost\", \"parentTopic\", \"group\", \"clientId\", System.currentTimeMillis(), 6000L, 32, \"attemptId\");\n\n        assertEquals(0, result.getObject1().length());\n        assertEquals(0, result.getObject2().getMessageCount());\n        verify(liteEventDispatcher).getEventIterator(\"clientId\");\n    }\n\n    @SuppressWarnings(\"unchecked\")\n    @Test\n    public void testPopByClientId_oneEvent() {\n        String event = \"lmqName\";\n        int msgCount = 1;\n        GetMessageResult mockResult = mockGetMessageResult(GetMessageStatus.FOUND, msgCount, 100L);\n        long pollTime = System.currentTimeMillis();\n\n        Iterator<String> mockIterator = mock(Iterator.class);\n        when(mockIterator.hasNext()).thenReturn(true, false);\n        when(mockIterator.next()).thenReturn(event);\n        when(liteEventDispatcher.getEventIterator(\"clientId\")).thenReturn(mockIterator);\n        doReturn(new Pair<>(new StringBuilder(\"0\"), mockResult))\n            .when(popLiteMessageProcessor)\n            .popLiteTopic(anyString(), anyString(), anyString(), anyString(), anyLong(), anyLong(), anyLong(), anyString());\n\n        Pair<StringBuilder, GetMessageResult> result = popLiteMessageProcessor.popByClientId(\n            \"clientHost\", \"parentTopic\", \"group\", \"clientId\", pollTime, 6000L, 32, \"attemptId\");\n\n        assertEquals(msgCount, result.getObject2().getMessageCount());\n        verify(mockIterator, times(2)).hasNext();\n        verify(popLiteMessageProcessor).popLiteTopic(\"parentTopic\" ,\"clientHost\", \"group\", event, 32L, pollTime, 6000L, \"attemptId\");\n    }\n\n    @SuppressWarnings(\"unchecked\")\n    @Test\n    public void testPopByClientId_resultFull() {\n        String event1 = \"lmqName1\";\n        String event2 = \"lmqName2\";\n        int msgCount = 1;\n        GetMessageResult mockResult = mockGetMessageResult(GetMessageStatus.FOUND, msgCount, 100L);\n        long pollTime = System.currentTimeMillis();\n\n        Iterator<String> mockIterator = mock(Iterator.class);\n        when(mockIterator.hasNext()).thenReturn(true, true, true, true, false);\n        when(mockIterator.next()).thenReturn(event1, event2, \"event3\", \"event4\");\n        when(liteEventDispatcher.getEventIterator(\"clientId\")).thenReturn(mockIterator);\n        doReturn(new Pair<>(new StringBuilder(\"0\"), mockResult))\n            .when(popLiteMessageProcessor)\n            .popLiteTopic(anyString(), anyString(), anyString(), anyString(), anyLong(), anyLong(), anyLong(), anyString());\n\n        Pair<StringBuilder, GetMessageResult> result = popLiteMessageProcessor.popByClientId(\n            \"clientHost\", \"parentTopic\", \"group\", \"clientId\", pollTime, 6000L, 2, \"attemptId\");\n\n        assertEquals(2, result.getObject2().getMessageCount());\n        assertEquals(\"0;0\", result.getObject1().toString());\n        verify(mockIterator, times(2)).hasNext();\n        verify(popLiteMessageProcessor).popLiteTopic(\"parentTopic\", \"clientHost\", \"group\", event1, 2L, pollTime, 6000L, \"attemptId\");\n        verify(popLiteMessageProcessor).popLiteTopic(\"parentTopic\", \"clientHost\", \"group\", event2, 1L, pollTime, 6000L, \"attemptId\");\n    }\n\n    @SuppressWarnings(\"unchecked\")\n    @Test\n    public void testPopByClientId_duplicateEvent() {\n        String event1 = \"lmqName1\";\n        String event2 = \"lmqName2\";\n        String event3 = \"lmqName1\";\n        int msgCount = 1;\n        GetMessageResult mockResult = mockGetMessageResult(GetMessageStatus.FOUND, msgCount, 100L);\n        long pollTime = System.currentTimeMillis();\n\n        Iterator<String> mockIterator = mock(Iterator.class);\n        when(mockIterator.hasNext()).thenReturn(true, true, true, false);\n        when(mockIterator.next()).thenReturn(event1, event2, event3);\n        when(liteEventDispatcher.getEventIterator(\"clientId\")).thenReturn(mockIterator);\n        doReturn(new Pair<>(new StringBuilder(\"0\"), mockResult))\n            .when(popLiteMessageProcessor)\n            .popLiteTopic(anyString(), anyString(), anyString(), anyString(), anyLong(), anyLong(), anyLong(), anyString());\n\n        Pair<StringBuilder, GetMessageResult> result = popLiteMessageProcessor.popByClientId(\n            \"clientHost\", \"parentTopic\", \"group\", \"clientId\", pollTime, 6000L, 32, \"attemptId\");\n\n        assertEquals(2, result.getObject2().getMessageCount());\n        assertEquals(\"0;0\", result.getObject1().toString());\n        verify(mockIterator, times(4)).hasNext();\n        verify(popLiteMessageProcessor).popLiteTopic(\"parentTopic\", \"clientHost\", \"group\", event1, 32L, pollTime, 6000L, \"attemptId\");\n        verify(popLiteMessageProcessor).popLiteTopic(\"parentTopic\", \"clientHost\", \"group\", event2, 31L, pollTime, 6000L, \"attemptId\");\n    }\n\n    @Test\n    public void testGetMessage_found() {\n        String group = \"group\";\n        String lmqName = \"lmqName\";\n        String clientHost = \"clientHost\";\n        long offset = 50L;\n        int batchSize = 16;\n        GetMessageResult mockResult = mockGetMessageResult(GetMessageStatus.FOUND, 1, 100L);\n        when(messageStore.getMessage(group, lmqName, 0, offset, batchSize, null)).thenReturn(mockResult);\n\n        GetMessageResult getMessageResult =\n            popLiteMessageProcessor.getMessage(clientHost, group, lmqName, offset, batchSize);\n        assertEquals(mockResult, getMessageResult);\n        verify(consumerOffsetManager, never()).commitOffset(clientHost, group, lmqName, 0, 100L);\n    }\n\n    @Test\n    public void testGetMessage_notFound() {\n        String group = \"group\";\n        String lmqName = \"lmqName\";\n        String clientHost = \"clientHost\";\n        long offset = 50L;\n        long nextBeginOffset = 100L;\n        int batchSize = 16;\n\n        GetMessageResult firstResult = mockGetMessageResult(GetMessageStatus.MESSAGE_WAS_REMOVING, 0, nextBeginOffset);\n        when(messageStore.getMessage(group, lmqName, 0, offset, batchSize, null)).thenReturn(firstResult);\n        GetMessageResult secondResult = mockGetMessageResult(GetMessageStatus.FOUND, batchSize, nextBeginOffset + batchSize);\n        when(messageStore.getMessage(group, lmqName, 0, nextBeginOffset, batchSize, null)).thenReturn(secondResult);\n\n        GetMessageResult getMessageResult =\n            popLiteMessageProcessor.getMessage(clientHost, group, lmqName, offset, batchSize);\n        assertEquals(secondResult, getMessageResult);\n        assertEquals(116, secondResult.getNextBeginOffset());\n        verify(consumerOffsetManager).commitOffset(\"CorrectOffset\", group, lmqName, 0, nextBeginOffset);\n    }\n\n    @Test\n    public void testHandleGetMessageResult_nullResult() {\n        Pair<StringBuilder, GetMessageResult> result = popLiteMessageProcessor.handleGetMessageResult(\n            null, \"parentTopic\", \"group\", \"lmqName\", System.currentTimeMillis(), 6000L, \"attemptId\");\n        assertNull(result);\n    }\n\n    @Test\n    public void testHandleGetMessageResult_found() {\n        int msgCount = 2;\n        GetMessageResult getResult = mockGetMessageResult(GetMessageStatus.FOUND, msgCount, 100L);\n        getResult.getMessageQueueOffset().add(0L);\n        getResult.getMessageQueueOffset().add(1L);\n\n        doNothing().when(popLiteMessageProcessor).recordPopLiteMetrics(any(), anyString(), anyString());\n\n        Pair<StringBuilder, GetMessageResult> result = popLiteMessageProcessor.handleGetMessageResult(\n            getResult, \"parentTopic\", \"group\", \"lmqName\", System.currentTimeMillis(), 6000L, \"attemptId\");\n\n        assertNotNull(result);\n        assertEquals(getResult, result.getObject2());\n        assertEquals(\"0;0\", result.getObject1().toString());\n    }\n\n    @Test\n    public void testPopLiteTopic_lockFailed() {\n        when(lockService.tryLock(anyString())).thenReturn(false);\n\n        Pair<StringBuilder, GetMessageResult> result = popLiteMessageProcessor.popLiteTopic(\"parentTopic\",\n            \"clientHost\", \"group\", \"lmqName\", 32L, System.currentTimeMillis(), 6000L, \"attemptId\");\n\n        assertNull(result);\n        verify(lockService).tryLock(anyString());\n        verify(lockService, never()).unlock(anyString());\n    }\n\n    @Test\n    public void testPopLiteTopic_fifoBlocked() {\n        when(lockService.tryLock(anyString())).thenReturn(true);\n        when(consumerOrderInfoManager.checkBlock(anyString(), anyString(), anyString(), anyInt(), anyLong()))\n            .thenReturn(true);\n\n        Pair<StringBuilder, GetMessageResult> result = popLiteMessageProcessor.popLiteTopic(\"parentTopic\",\n            \"clientHost\", \"group\", \"lmqName\", 32L, System.currentTimeMillis(), 6000L, \"attemptId\");\n\n        assertThat(result).isNull();\n        verify(lockService).tryLock(anyString());\n        verify(lockService).unlock(anyString());\n    }\n\n    @Test\n    public void testPopLiteTopic_lmqNotExist() {\n        when(liteLifecycleManager.isLmqExist(\"lmqName\")).thenReturn(false);\n        brokerConfig.setEnableLiteEventMode(false);\n\n        Pair<StringBuilder, GetMessageResult> result = popLiteMessageProcessor.popLiteTopic(\"parentTopic\",\n            \"clientHost\", \"group\", \"lmqName\", 32L, System.currentTimeMillis(), 6000L, \"attemptId\");\n\n        assertThat(result).isNull();\n        verify(lockService, never()).tryLock(anyString());\n    }\n\n    @Test\n    public void testPopLiteTopic_found() {\n        when(lockService.tryLock(anyString())).thenReturn(true);\n        when(consumerOrderInfoManager.checkBlock(anyString(), anyString(), anyString(), anyInt(), anyLong()))\n            .thenReturn(false);\n        GetMessageResult mockResult = mockGetMessageResult(GetMessageStatus.FOUND, 1, 100L);\n        when(messageStore.getMessage(\"group\", \"lmqName\", 0, 0, 32, null)).thenReturn(mockResult);\n\n        Pair<StringBuilder, GetMessageResult> result = popLiteMessageProcessor.popLiteTopic(\"parentTopic\",\n            \"clientHost\", \"group\", \"lmqName\", 32L, System.currentTimeMillis(), 6000L, \"attemptId\");\n\n        assertEquals(mockResult, result.getObject2());\n        verify(lockService).tryLock(anyString());\n        verify(lockService).unlock(anyString());\n    }\n\n    @Test\n    public void testPreCheck() {\n        final String parentTopic = \"parentTopic\";\n        final String group = \"group\";\n        final TopicConfig topicConfig = new TopicConfig();\n        final SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig();\n        final ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n\n        RemotingCommand response = RemotingCommand.createResponseCommand(PopLiteMessageResponseHeader.class);\n        PopLiteMessageRequestHeader requestHeader = new PopLiteMessageRequestHeader();\n        when(topicConfigManager.selectTopicConfig(parentTopic)).thenReturn(topicConfig);\n        when(subscriptionGroupManager.findSubscriptionGroupConfig(group)).thenReturn(groupConfig);\n\n        // timeout too much\n        requestHeader.setBornTime(System.currentTimeMillis() - 60000);\n        requestHeader.setPollTime(30000);\n\n        RemotingCommand result = popLiteMessageProcessor.preCheck(ctx, requestHeader, response);\n        assertEquals(ResponseCode.POLLING_TIMEOUT, result.getCode());\n\n        // not readable\n        brokerConfig.setBrokerPermission(PermName.PERM_WRITE);\n        requestHeader.setBornTime(System.currentTimeMillis());\n        requestHeader.setPollTime(30000);\n\n        result = popLiteMessageProcessor.preCheck(ctx, requestHeader, response);\n        assertEquals(ResponseCode.NO_PERMISSION, result.getCode());\n        brokerConfig.setBrokerPermission(PermName.PERM_READ | PermName.PERM_WRITE);\n\n        // topic not exist\n        requestHeader.setTopic(\"whatever\");\n\n        result = popLiteMessageProcessor.preCheck(ctx, requestHeader, response);\n        assertEquals(ResponseCode.TOPIC_NOT_EXIST, result.getCode());\n\n        // not lite topic type\n        requestHeader.setTopic(parentTopic);\n\n        result = popLiteMessageProcessor.preCheck(ctx, requestHeader, response);\n        assertEquals(ResponseCode.INVALID_PARAMETER, result.getCode());\n\n        // group not exist\n        topicConfig.setPerm(PermName.PERM_READ | PermName.PERM_WRITE);\n        topicConfig.setTopicMessageType(TopicMessageType.LITE);\n        requestHeader.setConsumerGroup(\"whatever\");\n\n        result = popLiteMessageProcessor.preCheck(ctx, requestHeader, response);\n        assertEquals(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST, result.getCode());\n\n        // group disable\n        groupConfig.setConsumeEnable(false);\n        requestHeader.setConsumerGroup(group);\n\n        result = popLiteMessageProcessor.preCheck(ctx, requestHeader, response);\n        assertEquals(ResponseCode.NO_PERMISSION, result.getCode());\n        groupConfig.setConsumeEnable(true);\n\n        // bind topic not match\n        groupConfig.setLiteBindTopic(\"otherTopic\");\n        requestHeader.setMaxMsgNum(32);\n\n        result = popLiteMessageProcessor.preCheck(ctx, requestHeader, response);\n        assertEquals(ResponseCode.INVALID_PARAMETER, result.getCode());\n\n        // normal\n        groupConfig.setLiteBindTopic(parentTopic);\n        result = popLiteMessageProcessor.preCheck(ctx, requestHeader, response);\n        assertNull(result);\n    }\n\n\n    private GetMessageResult mockGetMessageResult(GetMessageStatus status, int messageCount, long nextBeginOffset) {\n        GetMessageResult getMessageResult = new GetMessageResult();\n        getMessageResult.setStatus(status);\n        getMessageResult.setMinOffset(0);\n        getMessageResult.setMaxOffset(1024);\n        getMessageResult.setNextBeginOffset(nextBeginOffset);\n\n        if (GetMessageStatus.FOUND.equals(status)) {\n            for (int i = 0; i < messageCount; i++) {\n                getMessageResult.addMessage(Mockito.mock(SelectMappedBufferResult.class));\n            }\n        }\n        return getMessageResult;\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport com.alibaba.fastjson2.JSON;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.channel.embedded.EmbeddedChannel;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.metrics.BrokerMetricsManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.ConsumeInitMode;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumerData;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.logfile.DefaultMappedFile;\nimport org.apache.rocketmq.store.pop.PopCheckPoint;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.util.concurrent.CompletableFuture;\n\nimport static org.apache.rocketmq.broker.processor.PullMessageProcessorTest.createConsumerData;\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.junit.Assert.assertEquals;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class PopMessageProcessorTest {\n    private PopMessageProcessor popMessageProcessor;\n\n    @Spy\n    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());\n    @Mock\n    private ChannelHandlerContext handlerContext;\n    private final EmbeddedChannel embeddedChannel = new EmbeddedChannel();\n    @Mock\n    private DefaultMessageStore messageStore;\n    private ClientChannelInfo clientChannelInfo;\n    private String group = \"FooBarGroup\";\n    private String topic = \"FooBar\";\n\n    @Before\n    public void init() {\n        brokerController.setMessageStore(messageStore);\n        brokerController.getBrokerConfig().setEnablePopBufferMerge(true);\n        // Initialize BrokerMetricsManager to prevent NPE in tests\n        brokerController.setBrokerMetricsManager(new BrokerMetricsManager(brokerController));\n        popMessageProcessor = new PopMessageProcessor(brokerController);\n        when(handlerContext.channel()).thenReturn(embeddedChannel);\n        brokerController.getTopicConfigManager().getTopicConfigTable().put(topic, new TopicConfig(topic));\n        clientChannelInfo = new ClientChannelInfo(embeddedChannel);\n        ConsumerData consumerData = createConsumerData(group, topic);\n        brokerController.getConsumerManager().registerConsumer(\n            consumerData.getGroupName(),\n            clientChannelInfo,\n            consumerData.getConsumeType(),\n            consumerData.getMessageModel(),\n            consumerData.getConsumeFromWhere(),\n            consumerData.getSubscriptionDataSet(),\n            false);\n    }\n\n    @Test\n    public void testProcessRequest_TopicNotExist() throws RemotingCommandException {\n        when(messageStore.getMessageStoreConfig()).thenReturn(new MessageStoreConfig());\n        brokerController.getTopicConfigManager().getTopicConfigTable().remove(topic);\n        final RemotingCommand request = createPopMsgCommand();\n        RemotingCommand response = popMessageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.TOPIC_NOT_EXIST);\n        assertThat(response.getRemark()).contains(\"topic[\" + topic + \"] not exist\");\n    }\n\n    @Test\n    public void testProcessRequest_Found() throws RemotingCommandException, InterruptedException {\n        GetMessageResult getMessageResult = createGetMessageResult(1);\n        when(messageStore.getMessageStoreConfig()).thenReturn(new MessageStoreConfig());\n        when(messageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(CompletableFuture.completedFuture(getMessageResult));\n\n        final RemotingCommand request = createPopMsgCommand();\n        popMessageProcessor.processRequest(handlerContext, request);\n        RemotingCommand response = embeddedChannel.readOutbound();\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testProcessRequest_MsgWasRemoving() throws RemotingCommandException {\n        GetMessageResult getMessageResult = createGetMessageResult(1);\n        getMessageResult.setStatus(GetMessageStatus.MESSAGE_WAS_REMOVING);\n        when(messageStore.getMessageStoreConfig()).thenReturn(new MessageStoreConfig());\n        when(messageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(CompletableFuture.completedFuture(getMessageResult));\n\n        final RemotingCommand request = createPopMsgCommand();\n        popMessageProcessor.processRequest(handlerContext, request);\n        RemotingCommand response = embeddedChannel.readOutbound();\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testProcessRequest_NoMsgInQueue() throws RemotingCommandException {\n        GetMessageResult getMessageResult = createGetMessageResult(0);\n        getMessageResult.setStatus(GetMessageStatus.NO_MESSAGE_IN_QUEUE);\n        when(messageStore.getMessageStoreConfig()).thenReturn(new MessageStoreConfig());\n        when(messageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(CompletableFuture.completedFuture(getMessageResult));\n\n        final RemotingCommand request = createPopMsgCommand();\n        RemotingCommand response = popMessageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNull();\n    }\n\n    @Test\n    public void testProcessRequest_whenTimerWheelIsFalse() throws RemotingCommandException {\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setTimerWheelEnable(false);\n        when(messageStore.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n        final RemotingCommand request = createPopMsgCommand();\n        RemotingCommand response = popMessageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);\n        assertThat(response.getRemark()).contains(\"pop message is forbidden because timerWheelEnable is false\");\n    }\n\n    @Test\n    public void testGetInitOffset_retryTopic() throws RemotingCommandException {\n        when(messageStore.getMessageStoreConfig()).thenReturn(new MessageStoreConfig());\n        String newGroup = group + \"-\" + System.currentTimeMillis();\n        String retryTopic = KeyBuilder.buildPopRetryTopic(topic, newGroup);\n        long minOffset = 100L;\n        when(messageStore.getMinOffsetInQueue(retryTopic, 0)).thenReturn(minOffset);\n        brokerController.getTopicConfigManager().getTopicConfigTable().put(retryTopic, new TopicConfig(retryTopic, 1, 1));\n        GetMessageResult getMessageResult = createGetMessageResult(0);\n        when(messageStore.getMessageAsync(eq(newGroup), anyString(), anyInt(), anyLong(), anyInt(), any()))\n                .thenReturn(CompletableFuture.completedFuture(getMessageResult));\n\n        long offset = brokerController.getConsumerOffsetManager().queryOffset(newGroup, retryTopic, 0);\n        assertEquals(-1, offset);\n\n        RemotingCommand request = createPopMsgCommand(newGroup, topic, 0, ConsumeInitMode.MAX);\n        popMessageProcessor.processRequest(handlerContext, request);\n        offset = brokerController.getConsumerOffsetManager().queryOffset(newGroup, retryTopic, 0);\n        assertEquals(minOffset, offset);\n\n        when(messageStore.getMinOffsetInQueue(retryTopic, 0)).thenReturn(minOffset * 2);\n        popMessageProcessor.processRequest(handlerContext, request);\n        offset = brokerController.getConsumerOffsetManager().queryOffset(newGroup, retryTopic, 0);\n        assertEquals(minOffset, offset); // will not entry getInitOffset() again\n        messageStore.getMinOffsetInQueue(retryTopic, 0); // prevent UnnecessaryStubbingException\n    }\n\n    @Test\n    public void testGetInitOffset_normalTopic() throws RemotingCommandException, ConsumeQueueException {\n        long maxOffset = 999L;\n        when(messageStore.getMessageStoreConfig()).thenReturn(new MessageStoreConfig());\n        when(messageStore.getMaxOffsetInQueue(topic, 0)).thenReturn(maxOffset);\n        String newGroup = group + \"-\" + System.currentTimeMillis();\n        GetMessageResult getMessageResult = createGetMessageResult(0);\n        when(messageStore.getMessageAsync(eq(newGroup), anyString(), anyInt(), anyLong(), anyInt(), any()))\n                .thenReturn(CompletableFuture.completedFuture(getMessageResult));\n\n        long offset = brokerController.getConsumerOffsetManager().queryOffset(newGroup, topic, 0);\n        assertEquals(-1, offset);\n\n        RemotingCommand request = createPopMsgCommand(newGroup, topic, 0, ConsumeInitMode.MAX);\n        popMessageProcessor.processRequest(handlerContext, request);\n        offset = brokerController.getConsumerOffsetManager().queryOffset(newGroup, topic, 0);\n        assertEquals(maxOffset - 1, offset); // checkInMem return false\n\n        when(messageStore.getMaxOffsetInQueue(topic, 0)).thenReturn(maxOffset * 2);\n        popMessageProcessor.processRequest(handlerContext, request);\n        offset = brokerController.getConsumerOffsetManager().queryOffset(newGroup, topic, 0);\n        assertEquals(maxOffset - 1, offset); // will not entry getInitOffset() again\n        messageStore.getMaxOffsetInQueue(topic, 0); // prevent UnnecessaryStubbingException\n    }\n\n    @Test\n    public void testBuildCkMsgJsonParsing() {\n        PopCheckPoint ck = new PopCheckPoint();\n        ck.setTopic(\"TestTopic\");\n        ck.setQueueId(1);\n        ck.setStartOffset(100L);\n        ck.setCId(\"TestConsumer\");\n        ck.setPopTime(System.currentTimeMillis());\n        ck.setBrokerName(\"TestBroker\");\n\n        int reviveQid = 0;\n        PopMessageProcessor processor = new PopMessageProcessor(brokerController);\n\n        MessageExtBrokerInner result = processor.buildCkMsg(ck, reviveQid);\n\n        String jsonBody = new String(result.getBody(), StandardCharsets.UTF_8);\n        PopCheckPoint actual = JSON.parseObject(jsonBody, PopCheckPoint.class);\n\n        assertEquals(ck.getTopic(), actual.getTopic());\n        assertEquals(ck.getQueueId(), actual.getQueueId());\n        assertEquals(ck.getStartOffset(), actual.getStartOffset());\n        assertEquals(ck.getCId(), actual.getCId());\n        assertEquals(ck.getPopTime(), actual.getPopTime());\n        assertEquals(ck.getBrokerName(), actual.getBrokerName());\n        assertEquals(ck.getReviveTime(), actual.getReviveTime());\n    }\n\n    private RemotingCommand createPopMsgCommand() {\n        return createPopMsgCommand(group, topic, -1, ConsumeInitMode.MAX);\n    }\n\n    private RemotingCommand createPopMsgCommand(String group, String topic, int queueId, int initMode) {\n        PopMessageRequestHeader requestHeader = new PopMessageRequestHeader();\n        requestHeader.setConsumerGroup(group);\n        requestHeader.setMaxMsgNums(30);\n        requestHeader.setQueueId(queueId);\n        requestHeader.setTopic(topic);\n        requestHeader.setInvisibleTime(10_000);\n        requestHeader.setInitMode(initMode);\n        requestHeader.setOrder(false);\n        requestHeader.setPollTime(15_000);\n        requestHeader.setBornTime(System.currentTimeMillis());\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.POP_MESSAGE, requestHeader);\n        request.makeCustomHeaderToNet();\n        return request;\n    }\n\n    private GetMessageResult createGetMessageResult(int msgCnt) {\n        GetMessageResult getMessageResult = new GetMessageResult();\n        getMessageResult.setStatus(GetMessageStatus.FOUND);\n        getMessageResult.setMinOffset(100);\n        getMessageResult.setMaxOffset(1024);\n        getMessageResult.setNextBeginOffset(516);\n        for (int i = 0; i < msgCnt; i++) {\n            ByteBuffer bb = ByteBuffer.allocate(64);\n            bb.putLong(MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION, System.currentTimeMillis());\n            getMessageResult.addMessage(new SelectMappedBufferResult(200, bb, 64, new DefaultMappedFile()));\n        }\n        return getMessageResult;\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport com.alibaba.fastjson2.JSON;\nimport org.apache.commons.lang3.tuple.Triple;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.failover.EscapeBridge;\nimport org.apache.rocketmq.broker.metrics.BrokerMetricsManager;\nimport org.apache.rocketmq.broker.metrics.PopMetricsManager;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.PopAckConstants;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.utils.DataConverter;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.AppendMessageResult;\nimport org.apache.rocketmq.store.AppendMessageStatus;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.pop.AckMsg;\nimport org.apache.rocketmq.store.pop.BatchAckMsg;\nimport org.apache.rocketmq.store.pop.PopCheckPoint;\nimport org.apache.rocketmq.store.timer.TimerMessageStore;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.ArgumentCaptor;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.net.SocketAddress;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collection;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.mockito.ArgumentMatchers.anyBoolean;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.any;\nimport static org.mockito.Mockito.doNothing;\nimport static org.mockito.Mockito.doReturn;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.Silent.class)\npublic class PopReviveServiceTest {\n\n    private static final String CLUSTER_NAME = \"test\";\n    private static final String REVIVE_TOPIC = PopAckConstants.buildClusterReviveTopic(CLUSTER_NAME);\n    private static final int REVIVE_QUEUE_ID = 0;\n    private static final String GROUP = \"group\";\n    private static final String TOPIC = \"topic\";\n    private static final SocketAddress STORE_HOST = NetworkUtil.string2SocketAddress(\"127.0.0.1:8080\");\n    private static final Long INVISIBLE_TIME = 1000L;\n\n    @Mock\n    private MessageStore messageStore;\n    @Mock\n    private ConsumerOffsetManager consumerOffsetManager;\n    @Mock\n    private TopicConfigManager topicConfigManager;\n    @Mock\n    private TimerMessageStore timerMessageStore;\n    @Mock\n    private SubscriptionGroupManager subscriptionGroupManager;\n    @Mock\n    private BrokerController brokerController;\n    @Mock\n    private EscapeBridge escapeBridge;\n    @Mock\n    private BrokerMetricsManager brokerMetricsManager;\n    @Mock\n    private PopMetricsManager popMetricsManager;\n    private PopMessageProcessor popMessageProcessor;\n\n    private BrokerConfig brokerConfig;\n    private PopReviveService popReviveService;\n\n    @Before\n    public void before() {\n        brokerConfig = new BrokerConfig();\n        brokerConfig.setBrokerClusterName(CLUSTER_NAME);\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);\n        when(brokerController.getMessageStore()).thenReturn(messageStore);\n        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);\n        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);\n        when(brokerController.getEscapeBridge()).thenReturn(escapeBridge);\n        when(messageStore.getTimerMessageStore()).thenReturn(timerMessageStore);\n        when(timerMessageStore.getDequeueBehind()).thenReturn(0L);\n        when(timerMessageStore.getEnqueueBehind()).thenReturn(0L);\n\n        when(topicConfigManager.selectTopicConfig(anyString())).thenReturn(new TopicConfig());\n        when(subscriptionGroupManager.findSubscriptionGroupConfig(anyString())).thenReturn(new SubscriptionGroupConfig());\n\n        // Initialize BrokerMetricsManager for tests\n        when(brokerController.getBrokerMetricsManager()).thenReturn(brokerMetricsManager);\n        when(brokerMetricsManager.getPopMetricsManager()).thenReturn(popMetricsManager);\n\n        popMessageProcessor = new PopMessageProcessor(brokerController); // a real one, not mock\n        when(brokerController.getPopMessageProcessor()).thenReturn(popMessageProcessor);\n\n        popReviveService = spy(new PopReviveService(brokerController, REVIVE_TOPIC, REVIVE_QUEUE_ID));\n        popReviveService.setShouldRunPopRevive(true);\n    }\n\n    @Test\n    public void testWhenAckMoreThanCk() throws Throwable {\n        brokerConfig.setEnableSkipLongAwaitingAck(true);\n        long maxReviveOffset = 4;\n\n        when(consumerOffsetManager.queryOffset(PopAckConstants.REVIVE_GROUP, REVIVE_TOPIC, REVIVE_QUEUE_ID))\n                .thenReturn(0L);\n        List<MessageExt> reviveMessageExtList = new ArrayList<>();\n        long basePopTime = System.currentTimeMillis();\n        {\n            // put a pair of ck and ack\n            PopCheckPoint ck = buildPopCheckPoint(1, basePopTime, 1);\n            reviveMessageExtList.add(buildCkMsg(ck));\n            reviveMessageExtList.add(buildAckMsg(buildAckMsg(1, basePopTime), ck.getReviveTime(), 1, basePopTime));\n        }\n        {\n            for (int i = 2; i <= maxReviveOffset; i++) {\n                long popTime = basePopTime + i;\n                PopCheckPoint ck = buildPopCheckPoint(i, popTime, i);\n                reviveMessageExtList.add(buildAckMsg(buildAckMsg(i, popTime), ck.getReviveTime(), i, popTime));\n            }\n        }\n        doReturn(reviveMessageExtList, new ArrayList<>()).when(popReviveService).getReviveMessage(anyLong(), anyInt());\n\n        PopReviveService.ConsumeReviveObj consumeReviveObj = new PopReviveService.ConsumeReviveObj();\n        popReviveService.consumeReviveMessage(consumeReviveObj);\n\n        assertEquals(1, consumeReviveObj.map.size());\n\n        ArgumentCaptor<Long> commitOffsetCaptor = ArgumentCaptor.forClass(Long.class);\n        doNothing().when(consumerOffsetManager).commitOffset(anyString(), anyString(), anyString(), anyInt(), commitOffsetCaptor.capture());\n        popReviveService.mergeAndRevive(consumeReviveObj);\n\n        assertEquals(1, commitOffsetCaptor.getValue().longValue());\n    }\n\n    @Test\n    public void testSkipLongWaiteAck() throws Throwable {\n        brokerConfig.setEnableSkipLongAwaitingAck(true);\n        brokerConfig.setReviveAckWaitMs(TimeUnit.SECONDS.toMillis(2));\n        long maxReviveOffset = 4;\n\n        when(consumerOffsetManager.queryOffset(PopAckConstants.REVIVE_GROUP, REVIVE_TOPIC, REVIVE_QUEUE_ID))\n                .thenReturn(0L);\n        List<MessageExt> reviveMessageExtList = new ArrayList<>();\n        long basePopTime = System.currentTimeMillis() - brokerConfig.getReviveAckWaitMs() * 2;\n        {\n            // put a pair of ck and ack\n            PopCheckPoint ck = buildPopCheckPoint(1, basePopTime, 1);\n            reviveMessageExtList.add(buildCkMsg(ck));\n            reviveMessageExtList.add(buildAckMsg(buildAckMsg(1, basePopTime), ck.getReviveTime(), 1, basePopTime));\n        }\n        {\n            for (int i = 2; i <= maxReviveOffset; i++) {\n                long popTime = basePopTime + i;\n                PopCheckPoint ck = buildPopCheckPoint(i, popTime, i);\n                reviveMessageExtList.add(buildAckMsg(buildAckMsg(i, popTime), ck.getReviveTime(), i, popTime));\n            }\n        }\n        doReturn(reviveMessageExtList, new ArrayList<>()).when(popReviveService).getReviveMessage(anyLong(), anyInt());\n\n        PopReviveService.ConsumeReviveObj consumeReviveObj = new PopReviveService.ConsumeReviveObj();\n        popReviveService.consumeReviveMessage(consumeReviveObj);\n\n        assertEquals(4, consumeReviveObj.map.size());\n\n        ArgumentCaptor<Long> commitOffsetCaptor = ArgumentCaptor.forClass(Long.class);\n        doNothing().when(consumerOffsetManager).commitOffset(anyString(), anyString(), anyString(), anyInt(), commitOffsetCaptor.capture());\n        popReviveService.mergeAndRevive(consumeReviveObj);\n\n        assertEquals(maxReviveOffset, commitOffsetCaptor.getValue().longValue());\n    }\n\n    @Test\n    public void testSkipLongWaiteAckWithSameAck() throws Throwable {\n        brokerConfig.setEnableSkipLongAwaitingAck(true);\n        brokerConfig.setReviveAckWaitMs(TimeUnit.SECONDS.toMillis(2));\n        long maxReviveOffset = 4;\n\n        when(consumerOffsetManager.queryOffset(PopAckConstants.REVIVE_GROUP, REVIVE_TOPIC, REVIVE_QUEUE_ID))\n                .thenReturn(0L);\n        List<MessageExt> reviveMessageExtList = new ArrayList<>();\n        long basePopTime = System.currentTimeMillis() - brokerConfig.getReviveAckWaitMs() * 2;\n        {\n            for (int i = 2; i <= maxReviveOffset; i++) {\n                long popTime = basePopTime + i;\n                PopCheckPoint ck = buildPopCheckPoint(0, basePopTime, i);\n                reviveMessageExtList.add(buildAckMsg(buildAckMsg(0, basePopTime), ck.getReviveTime(), i, popTime));\n            }\n        }\n        doReturn(reviveMessageExtList, new ArrayList<>()).when(popReviveService).getReviveMessage(anyLong(), anyInt());\n\n        PopReviveService.ConsumeReviveObj consumeReviveObj = new PopReviveService.ConsumeReviveObj();\n        popReviveService.consumeReviveMessage(consumeReviveObj);\n\n        assertEquals(1, consumeReviveObj.map.size());\n\n        ArgumentCaptor<Long> commitOffsetCaptor = ArgumentCaptor.forClass(Long.class);\n        doNothing().when(consumerOffsetManager).commitOffset(anyString(), anyString(), anyString(), anyInt(), commitOffsetCaptor.capture());\n        popReviveService.mergeAndRevive(consumeReviveObj);\n\n        assertEquals(maxReviveOffset, commitOffsetCaptor.getValue().longValue());\n    }\n\n    @Test\n    public void testReviveMsgFromCk_messageFound_writeRetryOK() throws Throwable {\n        PopCheckPoint ck = buildPopCheckPoint(0, 0, 0);\n        PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj();\n        reviveObj.map.put(\"\", ck);\n        reviveObj.endTime = System.currentTimeMillis();\n        StringBuilder actualRetryTopic = new StringBuilder();\n\n        when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean()))\n                .thenReturn(CompletableFuture.completedFuture(Triple.of(new MessageExt(), \"\", false)));\n        when(escapeBridge.putMessageToSpecificQueue(any(MessageExtBrokerInner.class))).thenAnswer(invocation -> {\n            MessageExtBrokerInner msg = invocation.getArgument(0);\n            actualRetryTopic.append(msg.getTopic());\n            return new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK));\n        });\n\n        popReviveService.mergeAndRevive(reviveObj);\n        Assert.assertEquals(KeyBuilder.buildPopRetryTopic(TOPIC, GROUP, false), actualRetryTopic.toString());\n        verify(escapeBridge, times(1)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry\n        verify(messageStore, times(0)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK\n    }\n\n    @Test\n    public void testReviveMsgFromCk_messageFound_writeRetryFailed_rewriteCK() throws Throwable {\n        PopCheckPoint ck = buildPopCheckPoint(0, 0, 0);\n        PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj();\n        reviveObj.map.put(\"\", ck);\n        reviveObj.endTime = System.currentTimeMillis();\n        StringBuilder actualRetryTopic = new StringBuilder();\n        StringBuilder actualReviveTopic = new StringBuilder();\n        AtomicLong actualInvisibleTime = new AtomicLong(0L);\n\n        when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean()))\n            .thenReturn(CompletableFuture.completedFuture(Triple.of(new MessageExt(), \"\", false)));\n        when(escapeBridge.putMessageToSpecificQueue(any(MessageExtBrokerInner.class))).thenAnswer(invocation -> {\n            MessageExtBrokerInner msg = invocation.getArgument(0);\n            actualRetryTopic.append(msg.getTopic());\n            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(AppendMessageStatus.MESSAGE_SIZE_EXCEEDED));\n        });\n        when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenAnswer(invocation -> {\n            MessageExtBrokerInner msg = invocation.getArgument(0);\n            actualReviveTopic.append(msg.getTopic());\n            PopCheckPoint rewriteCK = JSON.parseObject(msg.getBody(), PopCheckPoint.class);\n            actualInvisibleTime.set(rewriteCK.getReviveTime());\n            return new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK));\n        });\n\n        popReviveService.mergeAndRevive(reviveObj);\n        \n        // Wait for async operations to complete\n        Thread.sleep(1000);\n        \n        Assert.assertEquals(KeyBuilder.buildPopRetryTopic(TOPIC, GROUP, false), actualRetryTopic.toString());\n        Assert.assertEquals(REVIVE_TOPIC, actualReviveTopic.toString());\n        Assert.assertEquals(INVISIBLE_TIME + 10 * 1000L, actualInvisibleTime.get()); // first interval is 10s\n        verify(escapeBridge, times(1)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry\n        verify(messageStore, times(1)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK\n    }\n\n    @Test\n    public void testReviveMsgFromCk_messageFound_writeRetryFailed_rewriteCK_end() throws Throwable {\n        brokerConfig.setSkipWhenCKRePutReachMaxTimes(true);\n        PopCheckPoint ck = buildPopCheckPoint(0, 0, 0);\n        ck.setRePutTimes(\"17\");\n        PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj();\n        reviveObj.map.put(\"\", ck);\n        reviveObj.endTime = System.currentTimeMillis();\n        StringBuilder actualRetryTopic = new StringBuilder();\n\n        when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean()))\n                .thenReturn(CompletableFuture.completedFuture(Triple.of(new MessageExt(), \"\", false)));\n        when(escapeBridge.putMessageToSpecificQueue(any(MessageExtBrokerInner.class))).thenAnswer(invocation -> {\n            MessageExtBrokerInner msg = invocation.getArgument(0);\n            actualRetryTopic.append(msg.getTopic());\n            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(AppendMessageStatus.MESSAGE_SIZE_EXCEEDED));\n        });\n\n        popReviveService.mergeAndRevive(reviveObj);\n        Assert.assertEquals(KeyBuilder.buildPopRetryTopic(TOPIC, GROUP, false), actualRetryTopic.toString());\n        verify(escapeBridge, times(1)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry\n        verify(messageStore, times(0)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK\n    }\n\n    @Test\n    public void testReviveMsgFromCk_messageFound_writeRetryFailed_rewriteCK_noEnd() throws Throwable {\n        brokerConfig.setSkipWhenCKRePutReachMaxTimes(false);\n        PopCheckPoint ck = buildPopCheckPoint(0, 0, 0);\n        ck.setRePutTimes(Byte.MAX_VALUE + \"\");\n        PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj();\n        reviveObj.map.put(\"\", ck);\n        reviveObj.endTime = System.currentTimeMillis();\n        StringBuilder actualRetryTopic = new StringBuilder();\n\n        when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean()))\n                .thenReturn(CompletableFuture.completedFuture(Triple.of(new MessageExt(), \"\", false)));\n        when(escapeBridge.putMessageToSpecificQueue(any(MessageExtBrokerInner.class))).thenAnswer(invocation -> {\n            MessageExtBrokerInner msg = invocation.getArgument(0);\n            actualRetryTopic.append(msg.getTopic());\n            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(AppendMessageStatus.MESSAGE_SIZE_EXCEEDED));\n        });\n\n        popReviveService.mergeAndRevive(reviveObj);\n        Assert.assertEquals(KeyBuilder.buildPopRetryTopic(TOPIC, GROUP, false), actualRetryTopic.toString());\n        verify(escapeBridge, times(1)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry\n        verify(messageStore, times(1)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK\n    }\n\n    @Test\n    public void testReviveMsgFromCk_messageNotFound_noRetry() throws Throwable {\n        PopCheckPoint ck = buildPopCheckPoint(0, 0, 0);\n        PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj();\n        reviveObj.map.put(\"\", ck);\n        reviveObj.endTime = System.currentTimeMillis();\n\n        when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean()))\n                .thenReturn(CompletableFuture.completedFuture(Triple.of(null, \"\", false)));\n\n        popReviveService.mergeAndRevive(reviveObj);\n        verify(escapeBridge, times(0)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry\n        verify(messageStore, times(0)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK\n    }\n\n    @Test\n    public void testReviveMsgFromCk_messageNotFound_needRetry() throws Throwable {\n        PopCheckPoint ck = buildPopCheckPoint(0, 0, 0);\n        PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj();\n        reviveObj.map.put(\"\", ck);\n        reviveObj.endTime = System.currentTimeMillis();\n        StringBuilder actualReviveTopic = new StringBuilder();\n        AtomicLong actualInvisibleTime = new AtomicLong(0L);\n\n        when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean()))\n                .thenReturn(CompletableFuture.completedFuture(Triple.of(null, \"\", true)));\n        when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenAnswer(invocation -> {\n            MessageExtBrokerInner msg = invocation.getArgument(0);\n            actualReviveTopic.append(msg.getTopic());\n            PopCheckPoint rewriteCK = JSON.parseObject(msg.getBody(), PopCheckPoint.class);\n            actualInvisibleTime.set(rewriteCK.getReviveTime());\n            return new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK));\n        });\n\n        popReviveService.mergeAndRevive(reviveObj);\n        Assert.assertEquals(REVIVE_TOPIC, actualReviveTopic.toString());\n        Assert.assertEquals(INVISIBLE_TIME + 10 * 1000L, actualInvisibleTime.get()); // first interval is 10s\n        verify(escapeBridge, times(0)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry\n        verify(messageStore, times(1)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK\n    }\n\n    @Test\n    public void testReviveMsgFromCk_messageNotFound_needRetry_end() throws Throwable {\n        brokerConfig.setSkipWhenCKRePutReachMaxTimes(true);\n        PopCheckPoint ck = buildPopCheckPoint(0, 0, 0);\n        ck.setRePutTimes(\"17\");\n        PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj();\n        reviveObj.map.put(\"\", ck);\n        reviveObj.endTime = System.currentTimeMillis();\n\n        when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean()))\n                .thenReturn(CompletableFuture.completedFuture(Triple.of(null, \"\", true)));\n\n        popReviveService.mergeAndRevive(reviveObj);\n        verify(escapeBridge, times(0)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry\n        verify(messageStore, times(0)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK\n    }\n\n    @Test\n    public void testReviveMsgFromCk_messageNotFound_needRetry_noEnd() throws Throwable {\n        brokerConfig.setSkipWhenCKRePutReachMaxTimes(false);\n        PopCheckPoint ck = buildPopCheckPoint(0, 0, 0);\n        ck.setRePutTimes(Byte.MAX_VALUE + \"\");\n        PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj();\n        reviveObj.map.put(\"\", ck);\n        reviveObj.endTime = System.currentTimeMillis();\n\n        when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean()))\n                .thenReturn(CompletableFuture.completedFuture(Triple.of(null, \"\", true)));\n\n        popReviveService.mergeAndRevive(reviveObj);\n        verify(escapeBridge, times(0)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry\n        verify(messageStore, times(1)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK\n    }\n\n    @Test\n    public void testReviveMsgFromBatchAck() throws Throwable {\n        brokerConfig.setEnableSkipLongAwaitingAck(true);\n        when(consumerOffsetManager.queryOffset(PopAckConstants.REVIVE_GROUP, REVIVE_TOPIC, REVIVE_QUEUE_ID)).thenReturn(0L);\n        List<MessageExt> reviveMessageExtList = new ArrayList<>();\n        long basePopTime = System.currentTimeMillis();\n        reviveMessageExtList.add(buildBatchAckMsg(buildBatchAckMsg(Arrays.asList(1L, 2L, 3L), basePopTime), 1, 1, basePopTime));\n        doReturn(reviveMessageExtList, new ArrayList<>()).when(popReviveService).getReviveMessage(anyLong(), anyInt());\n\n        PopReviveService.ConsumeReviveObj consumeReviveObj = new PopReviveService.ConsumeReviveObj();\n        popReviveService.consumeReviveMessage(consumeReviveObj);\n        assertEquals(1, consumeReviveObj.map.size());\n\n        ArgumentCaptor<Long> commitOffsetCaptor = ArgumentCaptor.forClass(Long.class);\n        doNothing().when(consumerOffsetManager).commitOffset(anyString(), anyString(), anyString(), anyInt(), commitOffsetCaptor.capture());\n        popReviveService.mergeAndRevive(consumeReviveObj);\n        assertEquals(1, commitOffsetCaptor.getValue().longValue());\n    }\n\n    public static MessageExtBrokerInner buildBatchAckMsg(BatchAckMsg batchAckMsg, long deliverMs, long reviveOffset, long deliverTime) {\n        MessageExtBrokerInner result = buildBatchAckInnerMessage(REVIVE_TOPIC, batchAckMsg, REVIVE_QUEUE_ID, STORE_HOST, deliverMs, PopMessageProcessor.genAckUniqueId(batchAckMsg));\n        result.setQueueOffset(reviveOffset);\n        result.setDeliverTimeMs(deliverMs);\n        result.setStoreTimestamp(deliverTime);\n        return result;\n    }\n\n    public static BatchAckMsg buildBatchAckMsg(Collection<Long> offsets, long popTime) {\n        BatchAckMsg result = new BatchAckMsg();\n        result.setConsumerGroup(GROUP);\n        result.setTopic(TOPIC);\n        result.setQueueId(0);\n        result.setPopTime(popTime);\n        result.setBrokerName(\"broker-a\");\n        result.getAckOffsetList().addAll(offsets);\n        return result;\n    }\n\n    public static MessageExtBrokerInner buildBatchAckInnerMessage(String reviveTopic, AckMsg ackMsg, int reviveQid, SocketAddress host, long deliverMs, String ackUniqueId) {\n        MessageExtBrokerInner result = new MessageExtBrokerInner();\n        result.setTopic(reviveTopic);\n        result.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8));\n        result.setQueueId(reviveQid);\n        result.setTags(PopAckConstants.BATCH_ACK_TAG);\n        result.setBornTimestamp(System.currentTimeMillis());\n        result.setBornHost(host);\n        result.setStoreHost(host);\n        result.setDeliverTimeMs(deliverMs);\n        result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, ackUniqueId);\n        result.setPropertiesString(MessageDecoder.messageProperties2String(result.getProperties()));\n        return result;\n    }\n\n    public static PopCheckPoint buildPopCheckPoint(long startOffset, long popTime, long reviveOffset) {\n        PopCheckPoint ck = new PopCheckPoint();\n        ck.setStartOffset(startOffset);\n        ck.setPopTime(popTime);\n        ck.setQueueId(0);\n        ck.setCId(GROUP);\n        ck.setTopic(TOPIC);\n        ck.setNum((byte) 1);\n        ck.setBitMap(0);\n        ck.setReviveOffset(reviveOffset);\n        ck.setInvisibleTime(INVISIBLE_TIME);\n        ck.setBrokerName(\"broker-a\");\n        return ck;\n    }\n\n    public static AckMsg buildAckMsg(long offset, long popTime) {\n        AckMsg ackMsg = new AckMsg();\n        ackMsg.setAckOffset(offset);\n        ackMsg.setStartOffset(offset);\n        ackMsg.setConsumerGroup(GROUP);\n        ackMsg.setTopic(TOPIC);\n        ackMsg.setQueueId(0);\n        ackMsg.setPopTime(popTime);\n        ackMsg.setBrokerName(\"broker-a\");\n\n        return ackMsg;\n    }\n\n    public static MessageExtBrokerInner buildCkMsg(PopCheckPoint ck) {\n        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n\n        msgInner.setTopic(REVIVE_TOPIC);\n        msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8));\n        msgInner.setQueueId(REVIVE_QUEUE_ID);\n        msgInner.setTags(PopAckConstants.CK_TAG);\n        msgInner.setBornTimestamp(System.currentTimeMillis());\n        msgInner.setBornHost(STORE_HOST);\n        msgInner.setStoreHost(STORE_HOST);\n        msgInner.setDeliverTimeMs(ck.getReviveTime() - PopAckConstants.ackTimeInterval);\n        msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genCkUniqueId(ck));\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));\n\n        msgInner.setQueueOffset(ck.getReviveOffset());\n\n        return msgInner;\n    }\n\n    public static MessageExtBrokerInner buildAckMsg(AckMsg ackMsg, long deliverMs, long reviveOffset,\n                                                    long deliverTime) {\n        MessageExtBrokerInner messageExtBrokerInner = buildAckInnerMessage(\n                REVIVE_TOPIC,\n                ackMsg,\n                REVIVE_QUEUE_ID,\n                STORE_HOST,\n                deliverMs,\n                PopMessageProcessor.genAckUniqueId(ackMsg)\n        );\n        messageExtBrokerInner.setQueueOffset(reviveOffset);\n        messageExtBrokerInner.setDeliverTimeMs(deliverMs);\n        messageExtBrokerInner.setStoreTimestamp(deliverTime);\n        return messageExtBrokerInner;\n    }\n\n    public static MessageExtBrokerInner buildAckInnerMessage(String reviveTopic, AckMsg ackMsg, int reviveQid,\n                                                             SocketAddress host, long deliverMs, String ackUniqueId) {\n        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n        msgInner.setTopic(reviveTopic);\n        msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8));\n        msgInner.setQueueId(reviveQid);\n        msgInner.setTags(PopAckConstants.ACK_TAG);\n        msgInner.setBornTimestamp(System.currentTimeMillis());\n        msgInner.setBornHost(host);\n        msgInner.setStoreHost(host);\n        msgInner.setDeliverTimeMs(deliverMs);\n        msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, ackUniqueId);\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));\n\n        return msgInner;\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/processor/PullMessageProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.channel.embedded.EmbeddedChannel;\nimport java.lang.reflect.Method;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ConsumerGroupInfo;\nimport org.apache.rocketmq.broker.filter.ExpressionMessageFilter;\nimport org.apache.rocketmq.broker.metrics.BrokerMetricsManager;\nimport org.apache.rocketmq.broker.mqtrace.ConsumeMessageContext;\nimport org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumerData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class PullMessageProcessorTest {\n    private PullMessageProcessor pullMessageProcessor;\n    @Spy\n    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(),\n        new NettyClientConfig(), new MessageStoreConfig());\n    @Mock\n    private ChannelHandlerContext handlerContext;\n    private final EmbeddedChannel embeddedChannel = new EmbeddedChannel();\n    @Mock\n    private MessageStore messageStore;\n    private ClientChannelInfo clientChannelInfo;\n    private String group = \"FooBarGroup\";\n    private String topic = \"FooBar\";\n\n    @Before\n    public void init() {\n        brokerController.setMessageStore(messageStore);\n        // Initialize BrokerMetricsManager to prevent NPE in tests\n        brokerController.setBrokerMetricsManager(new BrokerMetricsManager(brokerController));\n        SubscriptionGroupManager subscriptionGroupManager = new SubscriptionGroupManager(brokerController);\n        pullMessageProcessor = new PullMessageProcessor(brokerController);\n        when(brokerController.getPullMessageProcessor()).thenReturn(pullMessageProcessor);\n        when(handlerContext.channel()).thenReturn(embeddedChannel);\n        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);\n        brokerController.getTopicConfigManager().getTopicConfigTable().put(topic, new TopicConfig());\n        clientChannelInfo = new ClientChannelInfo(embeddedChannel);\n        ConsumerData consumerData = createConsumerData(group, topic);\n        brokerController.getConsumerManager().registerConsumer(\n            consumerData.getGroupName(),\n            clientChannelInfo,\n            consumerData.getConsumeType(),\n            consumerData.getMessageModel(),\n            consumerData.getConsumeFromWhere(),\n            consumerData.getSubscriptionDataSet(),\n            false);\n    }\n\n    @Test\n    public void testProcessRequest_TopicNotExist() throws RemotingCommandException {\n        brokerController.getTopicConfigManager().getTopicConfigTable().remove(topic);\n        final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE);\n        RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.TOPIC_NOT_EXIST);\n        assertThat(response.getRemark()).contains(\"topic[\" + topic + \"] not exist\");\n    }\n\n    @Test\n    public void testProcessRequest_SubNotExist() throws RemotingCommandException {\n        brokerController.getConsumerManager().unregisterConsumer(group, clientChannelInfo, false);\n        final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE);\n        RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUBSCRIPTION_NOT_EXIST);\n        assertThat(response.getRemark()).contains(\"consumer's group info not exist\");\n    }\n\n    @Test\n    public void testProcessRequest_SubNotLatest() throws RemotingCommandException {\n        final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE);\n        request.addExtField(\"subVersion\", String.valueOf(101));\n        RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUBSCRIPTION_NOT_LATEST);\n        assertThat(response.getRemark()).contains(\"subscription not latest\");\n    }\n\n    @Test\n    public void testProcessRequest_Found() throws RemotingCommandException {\n        GetMessageResult getMessageResult = createGetMessageResult();\n        when(messageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any(ExpressionMessageFilter.class))).thenReturn(CompletableFuture.completedFuture(getMessageResult));\n\n        final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE);\n        pullMessageProcessor.processRequest(handlerContext, request);\n        RemotingCommand response = embeddedChannel.readOutbound();\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testProcessRequest_FoundWithHook() throws RemotingCommandException {\n        GetMessageResult getMessageResult = createGetMessageResult();\n        when(messageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any(ExpressionMessageFilter.class))).thenReturn(CompletableFuture.completedFuture(getMessageResult));\n        List<ConsumeMessageHook> consumeMessageHookList = new ArrayList<>();\n        final ConsumeMessageContext[] messageContext = new ConsumeMessageContext[1];\n        ConsumeMessageHook consumeMessageHook = new ConsumeMessageHook() {\n            @Override\n            public String hookName() {\n                return \"TestHook\";\n            }\n\n            @Override\n            public void consumeMessageBefore(ConsumeMessageContext context) {\n                messageContext[0] = context;\n            }\n\n            @Override\n            public void consumeMessageAfter(ConsumeMessageContext context) {\n            }\n        };\n        consumeMessageHookList.add(consumeMessageHook);\n        pullMessageProcessor.registerConsumeMessageHook(consumeMessageHookList);\n        final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE);\n        pullMessageProcessor.processRequest(handlerContext, request);\n        RemotingCommand response = embeddedChannel.readOutbound();\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(messageContext[0]).isNotNull();\n        assertThat(messageContext[0].getConsumerGroup()).isEqualTo(group);\n        assertThat(messageContext[0].getTopic()).isEqualTo(topic);\n        assertThat(messageContext[0].getQueueId()).isEqualTo(1);\n    }\n\n    @Test\n    public void testProcessRequest_MsgWasRemoving() throws RemotingCommandException {\n        GetMessageResult getMessageResult = createGetMessageResult();\n        getMessageResult.setStatus(GetMessageStatus.MESSAGE_WAS_REMOVING);\n        when(messageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any(ExpressionMessageFilter.class))).thenReturn(CompletableFuture.completedFuture(getMessageResult));\n\n        final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE);\n        pullMessageProcessor.processRequest(handlerContext, request);\n        RemotingCommand response = embeddedChannel.readOutbound();\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.PULL_RETRY_IMMEDIATELY);\n    }\n\n    @Test\n    public void testProcessRequest_NoMsgInQueue() throws RemotingCommandException {\n        GetMessageResult getMessageResult = createGetMessageResult();\n        getMessageResult.setStatus(GetMessageStatus.NO_MESSAGE_IN_QUEUE);\n        when(messageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any(ExpressionMessageFilter.class))).thenReturn(CompletableFuture.completedFuture(getMessageResult));\n\n        final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE);\n        pullMessageProcessor.processRequest(handlerContext, request);\n        RemotingCommand response = embeddedChannel.readOutbound();\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.PULL_OFFSET_MOVED);\n    }\n\n    @Test\n    public void test_LitePullRequestForbidden() throws Exception {\n        brokerController.getBrokerConfig().setLitePullMessageEnable(false);\n        RemotingCommand remotingCommand = createPullMsgCommand(RequestCode.LITE_PULL_MESSAGE);\n        RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, remotingCommand);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);\n    }\n\n    @Test\n    public void testIfBroadcast() throws Exception {\n        Class<? extends PullMessageProcessor> clazz = pullMessageProcessor.getClass();\n        Method method = clazz.getDeclaredMethod(\"isBroadcast\", boolean.class, ConsumerGroupInfo.class);\n        method.setAccessible(true);\n\n        ConsumerGroupInfo consumerGroupInfo = new ConsumerGroupInfo(\"GID-1\",\n            ConsumeType.CONSUME_PASSIVELY, MessageModel.CLUSTERING, ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        Assert.assertTrue((Boolean) method.invoke(pullMessageProcessor, true, consumerGroupInfo));\n\n        ConsumerGroupInfo consumerGroupInfo2 = new ConsumerGroupInfo(\"GID-2\",\n            ConsumeType.CONSUME_ACTIVELY, MessageModel.BROADCASTING, ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        Assert.assertFalse((Boolean) method.invoke(pullMessageProcessor, false, consumerGroupInfo2));\n\n        ConsumerGroupInfo consumerGroupInfo3 = new ConsumerGroupInfo(\"GID-3\",\n            ConsumeType.CONSUME_PASSIVELY, MessageModel.BROADCASTING, ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        Assert.assertTrue((Boolean) method.invoke(pullMessageProcessor, false, consumerGroupInfo3));\n    }\n\n    @Test\n    public void testCommitPullOffset() throws RemotingCommandException {\n        GetMessageResult getMessageResult = createGetMessageResult();\n        when(messageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any(ExpressionMessageFilter.class))).thenReturn(CompletableFuture.completedFuture(getMessageResult));\n\n        final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE);\n        pullMessageProcessor.processRequest(handlerContext, request);\n        RemotingCommand response = embeddedChannel.readOutbound();\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(this.brokerController.getConsumerOffsetManager().queryPullOffset(group, topic, 1))\n            .isEqualTo(getMessageResult.getNextBeginOffset());\n    }\n\n    private RemotingCommand createPullMsgCommand(int requestCode) {\n        PullMessageRequestHeader requestHeader = new PullMessageRequestHeader();\n        requestHeader.setCommitOffset(123L);\n        requestHeader.setConsumerGroup(group);\n        requestHeader.setMaxMsgNums(100);\n        requestHeader.setQueueId(1);\n        requestHeader.setQueueOffset(456L);\n        requestHeader.setSubscription(\"*\");\n        requestHeader.setTopic(topic);\n        requestHeader.setSysFlag(0);\n        requestHeader.setSubVersion(100L);\n        RemotingCommand request = RemotingCommand.createRequestCommand(requestCode, requestHeader);\n        request.makeCustomHeaderToNet();\n        return request;\n    }\n\n    static ConsumerData createConsumerData(String group, String topic) {\n        ConsumerData consumerData = new ConsumerData();\n        consumerData.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        consumerData.setConsumeType(ConsumeType.CONSUME_PASSIVELY);\n        consumerData.setGroupName(group);\n        consumerData.setMessageModel(MessageModel.CLUSTERING);\n        Set<SubscriptionData> subscriptionDataSet = new HashSet<>();\n        SubscriptionData subscriptionData = new SubscriptionData();\n        subscriptionData.setTopic(topic);\n        subscriptionData.setSubString(\"*\");\n        subscriptionData.setSubVersion(100L);\n        subscriptionDataSet.add(subscriptionData);\n        consumerData.setSubscriptionDataSet(subscriptionDataSet);\n        return consumerData;\n    }\n\n    private GetMessageResult createGetMessageResult() {\n        GetMessageResult getMessageResult = new GetMessageResult();\n        getMessageResult.setStatus(GetMessageStatus.FOUND);\n        getMessageResult.setMinOffset(100);\n        getMessageResult.setMaxOffset(1024);\n        getMessageResult.setNextBeginOffset(516);\n        return getMessageResult;\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport com.google.common.collect.ImmutableSet;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.lang.reflect.Method;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Set;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.topic.TopicRouteInfoManager;\nimport org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;\nimport org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely;\nimport org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragelyByCircle;\nimport org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueConsistentHash;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.message.MessageRequestMode;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.QueryAssignmentRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.QueryAssignmentResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.SetMessageRequestModeRequestBody;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumerData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.apache.rocketmq.broker.processor.PullMessageProcessorTest.createConsumerData;\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.Mockito.doReturn;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class QueryAssignmentProcessorTest {\n    private QueryAssignmentProcessor queryAssignmentProcessor;\n    @Spy\n    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());\n\n    @Mock\n    private TopicRouteInfoManager topicRouteInfoManager;\n    @Mock\n    private ChannelHandlerContext handlerContext;\n    @Mock\n    private MessageStore messageStore;\n    @Mock\n    private Channel channel;\n\n    private String broker = \"defaultBroker\";\n    private String topic = \"FooBar\";\n    private String group = \"FooBarGroup\";\n    private String clientId = \"127.0.0.1\";\n    private ClientChannelInfo clientInfo;\n\n    @Before\n    public void init() throws IllegalAccessException, NoSuchFieldException {\n        clientInfo = new ClientChannelInfo(channel, \"127.0.0.1\", LanguageCode.JAVA, 0);\n        brokerController.setMessageStore(messageStore);\n        doReturn(topicRouteInfoManager).when(brokerController).getTopicRouteInfoManager();\n        when(topicRouteInfoManager.getTopicSubscribeInfo(topic)).thenReturn(ImmutableSet.of(new MessageQueue(topic, \"broker-1\", 0), new MessageQueue(topic, \"broker-2\", 1)));\n        queryAssignmentProcessor = new QueryAssignmentProcessor(brokerController);\n        brokerController.getTopicConfigManager().getTopicConfigTable().put(topic, new TopicConfig());\n        ConsumerData consumerData = createConsumerData(group, topic);\n        brokerController.getConsumerManager().registerConsumer(\n            consumerData.getGroupName(),\n            clientInfo,\n            consumerData.getConsumeType(),\n            consumerData.getMessageModel(),\n            consumerData.getConsumeFromWhere(),\n            consumerData.getSubscriptionDataSet(),\n            false);\n    }\n\n    @Test\n    public void testQueryAssignment() throws Exception {\n        brokerController.getProducerManager().registerProducer(group, clientInfo);\n        final RemotingCommand request = createQueryAssignmentRequest();\n        RemotingCommand responseToReturn = queryAssignmentProcessor.processRequest(handlerContext, request);\n        assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(responseToReturn.getBody()).isNotNull();\n        QueryAssignmentResponseBody responseBody = QueryAssignmentResponseBody.decode(responseToReturn.getBody(), QueryAssignmentResponseBody.class);\n        assertThat(responseBody.getMessageQueueAssignments()).size().isEqualTo(2);\n    }\n\n    @Test\n    public void testSetMessageRequestMode_Success() throws Exception {\n        brokerController.getProducerManager().registerProducer(group, clientInfo);\n        final RemotingCommand request = createSetMessageRequestModeRequest(topic);\n        RemotingCommand responseToReturn = queryAssignmentProcessor.processRequest(handlerContext, request);\n        assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testSetMessageRequestMode_RetryTopic() throws Exception {\n        brokerController.getProducerManager().registerProducer(group, clientInfo);\n        final RemotingCommand request = createSetMessageRequestModeRequest(MixAll.RETRY_GROUP_TOPIC_PREFIX + topic);\n        RemotingCommand responseToReturn = queryAssignmentProcessor.processRequest(handlerContext, request);\n        assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);\n    }\n\n    @Test\n    public void testDoLoadBalance() throws Exception {\n        Method method = queryAssignmentProcessor.getClass()\n            .getDeclaredMethod(\"doLoadBalance\", String.class, String.class, String.class, MessageModel.class,\n                String.class, SetMessageRequestModeRequestBody.class, ChannelHandlerContext.class);\n        method.setAccessible(true);\n\n        Set<MessageQueue> mqs1 = (Set<MessageQueue>) method.invoke(\n            queryAssignmentProcessor, MixAll.LMQ_PREFIX + topic, group, \"127.0.0.1\", MessageModel.CLUSTERING,\n            new AllocateMessageQueueAveragely().getName(), new SetMessageRequestModeRequestBody(), handlerContext);\n        Set<MessageQueue> mqs2 = (Set<MessageQueue>) method.invoke(\n            queryAssignmentProcessor, MixAll.LMQ_PREFIX + topic, group, \"127.0.0.2\", MessageModel.CLUSTERING,\n            new AllocateMessageQueueAveragely().getName(), new SetMessageRequestModeRequestBody(), handlerContext);\n\n        assertThat(mqs1).hasSize(1);\n        assertThat(mqs2).isEmpty();\n    }\n\n    @Test\n    public void testAllocate4Pop() {\n        testAllocate4Pop(new AllocateMessageQueueAveragely());\n        testAllocate4Pop(new AllocateMessageQueueAveragelyByCircle());\n        testAllocate4Pop(new AllocateMessageQueueConsistentHash());\n    }\n\n    private void testAllocate4Pop(AllocateMessageQueueStrategy strategy) {\n        int testNum = 16;\n        List<MessageQueue> mqAll = new ArrayList<>();\n        for (int mqSize = 0; mqSize < testNum; mqSize++) {\n            mqAll.add(new MessageQueue(topic, broker, mqSize));\n\n            List<String> cidAll = new ArrayList<>();\n            for (int cidSize = 0; cidSize < testNum; cidSize++) {\n                String clientId = String.valueOf(cidSize);\n                cidAll.add(clientId);\n\n                for (int popShareQueueNum = 0; popShareQueueNum < testNum; popShareQueueNum++) {\n                    List<MessageQueue> allocateResult =\n                        queryAssignmentProcessor.allocate4Pop(strategy, group, clientId, mqAll, cidAll, popShareQueueNum);\n                    Assert.assertTrue(checkAllocateResult(popShareQueueNum, mqAll.size(), cidAll.size(), allocateResult.size(), strategy));\n                }\n            }\n        }\n    }\n\n    private boolean checkAllocateResult(int popShareQueueNum, int mqSize, int cidSize, int allocateSize,\n        AllocateMessageQueueStrategy strategy) {\n\n        //The maximum size of allocations will not exceed mqSize.\n        if (allocateSize > mqSize) {\n            return false;\n        }\n\n        //It is not allowed that the client is not assigned to the consumeQueue.\n        if (allocateSize <= 0) {\n            return false;\n        }\n\n        if (popShareQueueNum <= 0 || popShareQueueNum >= cidSize - 1) {\n            return allocateSize == mqSize;\n        } else if (mqSize < cidSize) {\n            return allocateSize == 1;\n        }\n\n        if (strategy instanceof AllocateMessageQueueAveragely\n            || strategy instanceof AllocateMessageQueueAveragelyByCircle) {\n\n            if (mqSize % cidSize == 0) {\n                return allocateSize == (mqSize / cidSize) * (popShareQueueNum + 1);\n            } else {\n                int avgSize = mqSize / cidSize;\n                return allocateSize >= avgSize * (popShareQueueNum + 1)\n                    && allocateSize <= (avgSize + 1) * (popShareQueueNum + 1);\n            }\n        }\n\n        if (strategy instanceof AllocateMessageQueueConsistentHash) {\n            //Just skip\n            return true;\n        }\n\n        return false;\n    }\n\n    private RemotingCommand createQueryAssignmentRequest() {\n        QueryAssignmentRequestBody requestBody = new QueryAssignmentRequestBody();\n        requestBody.setTopic(topic);\n        requestBody.setConsumerGroup(group);\n        requestBody.setClientId(clientId);\n        requestBody.setMessageModel(MessageModel.CLUSTERING);\n        requestBody.setStrategyName(\"AVG\");\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_ASSIGNMENT, null);\n        request.setBody(requestBody.encode());\n        return request;\n    }\n\n    private RemotingCommand createSetMessageRequestModeRequest(String topic) {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SET_MESSAGE_REQUEST_MODE, null);\n\n        SetMessageRequestModeRequestBody requestBody = new SetMessageRequestModeRequestBody();\n        requestBody.setTopic(topic);\n        requestBody.setConsumerGroup(group);\n        requestBody.setMode(MessageRequestMode.POP);\n        requestBody.setPopShareQueueNum(0);\n        request.setBody(requestBody.encode());\n\n        return request;\n    }\n\n    private RemotingCommand createResponse(int code, RemotingCommand request) {\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(code);\n        response.setOpaque(request.getOpaque());\n        return response;\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/processor/QueryMessageProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelFuture;\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ViewMessageRequestHeader;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.QueryMessageResult;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.util.HashMap;\n\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class QueryMessageProcessorTest {\n    private QueryMessageProcessor queryMessageProcessor;\n    @Spy\n    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());\n\n    @Mock\n    private MessageStore messageStore;\n\n    @Mock\n    private ChannelHandlerContext handlerContext;\n\n    @Mock\n    private Channel channel;\n\n    @Mock\n    private ChannelFuture channelFuture;\n\n    @Before\n    public void init() {\n        when(handlerContext.channel()).thenReturn(channel);\n        queryMessageProcessor = new QueryMessageProcessor(brokerController);\n        when(brokerController.getMessageStore()).thenReturn(messageStore);\n        when(channel.writeAndFlush(any())).thenReturn(channelFuture);\n    }\n\n    @Test\n    public void testQueryMessage() throws RemotingCommandException {\n        QueryMessageResult result = new QueryMessageResult();\n        result.setIndexLastUpdateTimestamp(100);\n        result.setIndexLastUpdatePhyoffset(0);\n        result.addMessage(new SelectMappedBufferResult(0, null, 0, null));\n\n        when(messageStore.queryMessage(anyString(),anyString(),anyInt(),anyLong(),anyLong(),any(),any())).thenReturn(result);\n        RemotingCommand request = createQueryMessageRequest(\"topic\", \"msgKey\", 1, 100, 200,\"false\");\n        request.makeCustomHeaderToNet();\n        RemotingCommand response = queryMessageProcessor.processRequest(handlerContext, request);\n        Assert.assertEquals(response.getCode(), ResponseCode.QUERY_NOT_FOUND);\n\n        result.addMessage(new SelectMappedBufferResult(0, null, 1, null));\n        response = queryMessageProcessor.processRequest(handlerContext, request);\n        Assert.assertNull(response);\n    }\n\n    @Test\n    public void testViewMessageById() throws RemotingCommandException {\n        ViewMessageRequestHeader viewMessageRequestHeader = new ViewMessageRequestHeader();\n        viewMessageRequestHeader.setTopic(\"topic\");\n        viewMessageRequestHeader.setOffset(0L);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.VIEW_MESSAGE_BY_ID, viewMessageRequestHeader);\n        request.makeCustomHeaderToNet();\n        request.setCode(RequestCode.VIEW_MESSAGE_BY_ID);\n\n        when(messageStore.selectOneMessageByOffset(anyLong())).thenReturn(null);\n        RemotingCommand response = queryMessageProcessor.processRequest(handlerContext, request);\n        Assert.assertEquals(response.getCode(), ResponseCode.SYSTEM_ERROR);\n\n        when(messageStore.selectOneMessageByOffset(anyLong())).thenReturn(new SelectMappedBufferResult(0, null, 0, null));\n        response = queryMessageProcessor.processRequest(handlerContext, request);\n        Assert.assertNull(response);\n    }\n\n    private RemotingCommand createQueryMessageRequest(String topic, String key, int maxNum, long beginTimestamp, long endTimestamp,String flag) {\n        QueryMessageRequestHeader requestHeader = new QueryMessageRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setKey(key);\n        requestHeader.setMaxNum(maxNum);\n        requestHeader.setBeginTimestamp(beginTimestamp);\n        requestHeader.setEndTimestamp(endTimestamp);\n\n        HashMap<String, String> extFields = new HashMap<>();\n        extFields.put(MixAll.UNIQUE_MSG_QUERY_FLAG, flag);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_MESSAGE, requestHeader);\n        request.setExtFields(extFields);\n        return request;\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/processor/RecallMessageProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.producer.RecallMessageHandle;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader;\nimport org.apache.rocketmq.store.AppendMessageResult;\nimport org.apache.rocketmq.store.AppendMessageStatus;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.Map;\n\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.when;\nimport static org.mockito.Mockito.verify;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class RecallMessageProcessorTest {\n    private static final String TOPIC = \"topic\";\n    private static final String BROKER_NAME = \"brokerName\";\n\n    private RecallMessageProcessor recallMessageProcessor;\n    @Mock\n    private BrokerConfig brokerConfig;\n    @Mock\n    private BrokerController brokerController;\n    @Mock\n    private ChannelHandlerContext handlerContext;\n    @Mock\n    private MessageStoreConfig messageStoreConfig;\n    @Mock\n    private TopicConfigManager topicConfigManager;\n    @Mock\n    private MessageStore messageStore;\n    @Mock\n    private BrokerStatsManager brokerStatsManager;\n    @Mock\n    private Channel channel;\n\n    @Before\n    public void init() throws IllegalAccessException, NoSuchFieldException {\n        when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);\n        when(brokerController.getMessageStore()).thenReturn(messageStore);\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        when(brokerConfig.getBrokerName()).thenReturn(BROKER_NAME);\n        when(brokerConfig.isRecallMessageEnable()).thenReturn(true);\n        when(brokerController.getBrokerStatsManager()).thenReturn(brokerStatsManager);\n        when(handlerContext.channel()).thenReturn(channel);\n        recallMessageProcessor = new RecallMessageProcessor(brokerController);\n    }\n\n    @Test\n    public void testBuildMessage_withNamespace() {\n        when(messageStoreConfig.isAppendTopicForTimerDeleteKey()).thenReturn(true);\n        String timestampStr = String.valueOf(System.currentTimeMillis());\n        String id = \"id\";\n        RecallMessageHandle.HandleV1 handle = new RecallMessageHandle.HandleV1(TOPIC, \"brokerName\", timestampStr, id);\n        MessageExtBrokerInner msg =\n            recallMessageProcessor.buildMessage(handlerContext, new RecallMessageRequestHeader(), handle);\n\n        Assert.assertEquals(TOPIC, msg.getTopic());\n        Map<String, String> properties = MessageDecoder.string2messageProperties(msg.getPropertiesString());\n        Assert.assertEquals(timestampStr, properties.get(MessageConst.PROPERTY_TIMER_DELIVER_MS));\n        Assert.assertEquals(id, properties.get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));\n        Assert.assertEquals(TOPIC + \"+\" + id, properties.get(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY));\n    }\n\n    @Test\n    public void testBuildMessage_withoutNamespace() {\n        when(messageStoreConfig.isAppendTopicForTimerDeleteKey()).thenReturn(false);\n        String timestampStr = String.valueOf(System.currentTimeMillis());\n        String id = \"id\";\n        RecallMessageHandle.HandleV1 handle = new RecallMessageHandle.HandleV1(TOPIC, \"brokerName\", timestampStr, id);\n        MessageExtBrokerInner msg =\n            recallMessageProcessor.buildMessage(handlerContext, new RecallMessageRequestHeader(), handle);\n\n        Assert.assertEquals(TOPIC, msg.getTopic());\n        Map<String, String> properties = MessageDecoder.string2messageProperties(msg.getPropertiesString());\n        Assert.assertEquals(timestampStr, properties.get(MessageConst.PROPERTY_TIMER_DELIVER_MS));\n        Assert.assertEquals(id, properties.get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));\n        Assert.assertEquals(id, properties.get(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY));\n    }\n\n    @Test\n    public void testHandlePutMessageResult() {\n        MessageExt message = new MessageExt();\n        MessageAccessor.putProperty(message, MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, \"id\");\n        RemotingCommand response = RemotingCommand.createResponseCommand(RecallMessageResponseHeader.class);\n        recallMessageProcessor.handlePutMessageResult(null, null, response, message, handlerContext, 0L);\n        Assert.assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode());\n\n        List<PutMessageStatus> okStatus = Arrays.asList(PutMessageStatus.PUT_OK, PutMessageStatus.FLUSH_DISK_TIMEOUT,\n            PutMessageStatus.FLUSH_SLAVE_TIMEOUT, PutMessageStatus.SLAVE_NOT_AVAILABLE);\n\n        for (PutMessageStatus status : PutMessageStatus.values()) {\n            PutMessageResult putMessageResult =\n                new PutMessageResult(status, new AppendMessageResult(AppendMessageStatus.PUT_OK));\n            recallMessageProcessor.handlePutMessageResult(putMessageResult, null, response, message, handlerContext, 0L);\n            if (okStatus.contains(status)) {\n                Assert.assertEquals(ResponseCode.SUCCESS, response.getCode());\n                RecallMessageResponseHeader responseHeader = (RecallMessageResponseHeader) response.readCustomHeader();\n                Assert.assertEquals(\"id\", responseHeader.getMsgId());\n            } else {\n                Assert.assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode());\n            }\n        }\n    }\n\n    @Test\n    public void testProcessRequest_notEnable() throws RemotingCommandException {\n        when(brokerConfig.isRecallMessageEnable()).thenReturn(false);\n        RemotingCommand request = mockRequest(0, TOPIC, TOPIC, \"id\", BROKER_NAME);\n        RemotingCommand response = recallMessageProcessor.processRequest(handlerContext, request);\n        Assert.assertEquals(ResponseCode.NO_PERMISSION, response.getCode());\n    }\n\n    @Test\n    public void testProcessRequest_invalidStatus() throws RemotingCommandException {\n        RemotingCommand request = mockRequest(0, TOPIC, TOPIC, \"id\", BROKER_NAME);\n        RemotingCommand response;\n\n        // role slave\n        when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE);\n        response = recallMessageProcessor.processRequest(handlerContext, request);\n        Assert.assertEquals(ResponseCode.SLAVE_NOT_AVAILABLE, response.getCode());\n\n        // not reach startTimestamp\n        when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SYNC_MASTER);\n        when(messageStore.now()).thenReturn(0L);\n        when(brokerConfig.getStartAcceptSendRequestTimeStamp()).thenReturn(System.currentTimeMillis());\n        response = recallMessageProcessor.processRequest(handlerContext, request);\n        Assert.assertEquals(ResponseCode.SERVICE_NOT_AVAILABLE, response.getCode());\n    }\n\n    @Test\n    public void testProcessRequest_notWriteable() throws RemotingCommandException {\n        when(brokerConfig.getBrokerPermission()).thenReturn(4);\n        when(brokerConfig.isAllowRecallWhenBrokerNotWriteable()).thenReturn(false);\n        RemotingCommand request = mockRequest(0, TOPIC, TOPIC, \"id\", BROKER_NAME);\n        RemotingCommand response = recallMessageProcessor.processRequest(handlerContext, request);\n        Assert.assertEquals(ResponseCode.SERVICE_NOT_AVAILABLE, response.getCode());\n    }\n\n    @Test\n    public void testProcessRequest_topicNotFound_or_notMatch() throws RemotingCommandException {\n        when(brokerConfig.getBrokerPermission()).thenReturn(6);\n        RemotingCommand request;\n        RemotingCommand response;\n\n        // not found\n        request = mockRequest(0, TOPIC, TOPIC, \"id\", BROKER_NAME);\n        response = recallMessageProcessor.processRequest(handlerContext, request);\n        Assert.assertEquals(ResponseCode.TOPIC_NOT_EXIST, response.getCode());\n\n        // not match\n        when(topicConfigManager.selectTopicConfig(TOPIC)).thenReturn(new TopicConfig(TOPIC));\n        request = mockRequest(0, TOPIC, \"anotherTopic\", \"id\", BROKER_NAME);\n        response = recallMessageProcessor.processRequest(handlerContext, request);\n        Assert.assertEquals(ResponseCode.ILLEGAL_OPERATION, response.getCode());\n    }\n\n    @Test\n    public void testProcessRequest_brokerNameNotMatch() throws RemotingCommandException {\n        when(brokerConfig.getBrokerPermission()).thenReturn(6);\n        when(topicConfigManager.selectTopicConfig(TOPIC)).thenReturn(new TopicConfig(TOPIC));\n\n        RemotingCommand request = mockRequest(0, TOPIC, \"anotherTopic\", \"id\", BROKER_NAME + \"_other\");\n        RemotingCommand response = recallMessageProcessor.processRequest(handlerContext, request);\n        Assert.assertEquals(ResponseCode.ILLEGAL_OPERATION, response.getCode());\n    }\n\n    @Test\n    public void testProcessRequest_timestampInvalid() throws RemotingCommandException {\n        when(brokerConfig.getBrokerPermission()).thenReturn(6);\n        when(topicConfigManager.selectTopicConfig(TOPIC)).thenReturn(new TopicConfig(TOPIC));\n        RemotingCommand request;\n        RemotingCommand response;\n\n        // past timestamp\n        request = mockRequest(0, TOPIC, TOPIC, \"id\", BROKER_NAME);\n        response = recallMessageProcessor.processRequest(handlerContext, request);\n        Assert.assertEquals(ResponseCode.ILLEGAL_OPERATION, response.getCode());\n\n        // timestamp overflow\n        when(messageStoreConfig.getTimerMaxDelaySec()).thenReturn(86400);\n        request = mockRequest(System.currentTimeMillis() + 86400 * 2 * 1000, TOPIC, TOPIC, \"id\", BROKER_NAME);\n        response = recallMessageProcessor.processRequest(handlerContext, request);\n        Assert.assertEquals(ResponseCode.ILLEGAL_OPERATION, response.getCode());\n    }\n\n    @Test\n    public void testProcessRequest_success() throws RemotingCommandException {\n        when(brokerConfig.getBrokerPermission()).thenReturn(6);\n        when(topicConfigManager.selectTopicConfig(TOPIC)).thenReturn(new TopicConfig(TOPIC));\n        when(messageStoreConfig.getTimerMaxDelaySec()).thenReturn(86400);\n        when(messageStore.putMessage(any())).thenReturn(\n            new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));\n\n        String msgId = \"msgId\";\n        RemotingCommand request = mockRequest(System.currentTimeMillis() + 90 * 1000, TOPIC, TOPIC, msgId, BROKER_NAME);\n        RemotingCommand response = recallMessageProcessor.processRequest(handlerContext, request);\n        RecallMessageResponseHeader responseHeader = (RecallMessageResponseHeader) response.readCustomHeader();\n        Assert.assertEquals(ResponseCode.SUCCESS, response.getCode());\n        Assert.assertEquals(msgId, responseHeader.getMsgId());\n        verify(messageStore, times(1)).putMessage(any());\n    }\n\n    private RemotingCommand mockRequest(long timestamp, String requestTopic, String handleTopic,\n        String msgId, String brokerName) {\n        String handle =\n            RecallMessageHandle.HandleV1.buildHandle(handleTopic, brokerName, String.valueOf(timestamp), msgId);\n        RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader();\n        requestHeader.setProducerGroup(\"group\");\n        requestHeader.setTopic(requestTopic);\n        requestHeader.setRecallHandle(handle);\n        requestHeader.setBrokerName(brokerName);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RECALL_MESSAGE, requestHeader);\n        request.makeCustomHeaderToNet();\n        return request;\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.lang.reflect.Field;\nimport java.net.InetSocketAddress;\nimport java.util.HashMap;\nimport java.util.Map;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.net.Broker2Client;\nimport org.apache.rocketmq.broker.metrics.BrokerMetricsManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader;\nimport org.apache.rocketmq.store.AppendMessageResult;\nimport org.apache.rocketmq.store.AppendMessageStatus;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ReplyMessageProcessorTest {\n    private ReplyMessageProcessor replyMessageProcessor;\n    @Spy\n    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());\n    @Mock\n    private ChannelHandlerContext handlerContext;\n    @Mock\n    private MessageStore messageStore;\n    @Mock\n    private Channel channel;\n\n    private String topic = \"FooBar\";\n    private String group = \"FooBarGroup\";\n    private ClientChannelInfo clientInfo;\n    @Mock\n    private Broker2Client broker2Client;\n\n    @Before\n    public void init() throws IllegalAccessException, NoSuchFieldException {\n        clientInfo = new ClientChannelInfo(channel, \"127.0.0.1\", LanguageCode.JAVA, 0);\n        brokerController.setMessageStore(messageStore);\n        // Initialize BrokerMetricsManager to prevent NPE in tests\n        brokerController.setBrokerMetricsManager(new BrokerMetricsManager(brokerController));\n        Field field = BrokerController.class.getDeclaredField(\"broker2Client\");\n        field.setAccessible(true);\n        field.set(brokerController, broker2Client);\n        when(messageStore.now()).thenReturn(System.currentTimeMillis());\n        Channel mockChannel = mock(Channel.class);\n        when(mockChannel.remoteAddress()).thenReturn(new InetSocketAddress(1024));\n        when(handlerContext.channel()).thenReturn(mockChannel);\n        replyMessageProcessor = new ReplyMessageProcessor(brokerController);\n    }\n\n    @Test\n    public void testProcessRequest_Success() throws RemotingCommandException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException {\n        when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));\n        brokerController.getProducerManager().registerProducer(group, clientInfo);\n        final RemotingCommand request = createSendMessageRequestHeaderCommand(RequestCode.SEND_REPLY_MESSAGE);\n        when(brokerController.getBroker2Client().callClient(any(), any(RemotingCommand.class))).thenReturn(createResponse(ResponseCode.SUCCESS, request));\n        RemotingCommand responseToReturn = replyMessageProcessor.processRequest(handlerContext, request);\n        assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(responseToReturn.getOpaque()).isEqualTo(request.getOpaque());\n    }\n\n    private RemotingCommand createSendMessageRequestHeaderCommand(int requestCode) {\n        SendMessageRequestHeader requestHeader = createSendMessageRequestHeader();\n        RemotingCommand request = RemotingCommand.createRequestCommand(requestCode, requestHeader);\n        request.setBody(new byte[] {'a'});\n        request.makeCustomHeaderToNet();\n        return request;\n    }\n\n    private SendMessageRequestHeader createSendMessageRequestHeader() {\n        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();\n        requestHeader.setProducerGroup(group);\n        requestHeader.setTopic(topic);\n        requestHeader.setDefaultTopic(TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC);\n        requestHeader.setDefaultTopicQueueNums(3);\n        requestHeader.setQueueId(1);\n        requestHeader.setSysFlag(0);\n        requestHeader.setBornTimestamp(System.currentTimeMillis());\n        requestHeader.setFlag(124);\n        requestHeader.setReconsumeTimes(0);\n        Map<String, String> map = new HashMap<>();\n        map.put(MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT, \"127.0.0.1\");\n        requestHeader.setProperties(MessageDecoder.messageProperties2String(map));\n        return requestHeader;\n    }\n\n    private RemotingCommand createResponse(int code, RemotingCommand request) {\n        RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);\n        response.setCode(code);\n        response.setOpaque(request.getOpaque());\n        return response;\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.processor;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.net.InetSocketAddress;\nimport java.time.Duration;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.Executors;\nimport org.apache.commons.codec.DecoderException;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.metrics.BrokerMetricsManager;\nimport org.apache.rocketmq.common.AbortProcessException;\nimport org.apache.rocketmq.broker.mqtrace.ConsumeMessageContext;\nimport org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook;\nimport org.apache.rocketmq.broker.mqtrace.SendMessageContext;\nimport org.apache.rocketmq.broker.mqtrace.SendMessageHook;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.broker.transaction.TransactionalMessageService;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.producer.RecallMessageHandle;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader;\nimport org.apache.rocketmq.store.AppendMessageResult;\nimport org.apache.rocketmq.store.AppendMessageStatus;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.awaitility.Awaitility.await;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.Mockito.doAnswer;\nimport static org.mockito.Mockito.when;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.times;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class SendMessageProcessorTest {\n    private SendMessageProcessor sendMessageProcessor;\n    @Mock\n    private ChannelHandlerContext handlerContext;\n    @Mock\n    private Channel channel;\n    @Spy\n    private BrokerConfig brokerConfig;\n    @Spy\n    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(),\n        new NettyClientConfig(), new MessageStoreConfig());\n    @Mock\n    private MessageStore messageStore;\n\n    @Mock\n    private TransactionalMessageService transactionMsgService;\n\n    private String topic = \"FooBar\";\n    private String group = \"FooBarGroup\";\n\n    @Before\n    public void init() {\n        brokerController.setMessageStore(messageStore);\n        // Initialize BrokerMetricsManager to prevent NPE in tests\n        brokerController.setBrokerMetricsManager(new BrokerMetricsManager(brokerController));\n        TopicConfigManager topicConfigManager = new TopicConfigManager(brokerController);\n        topicConfigManager.getTopicConfigTable().put(topic, new TopicConfig(topic));\n        SubscriptionGroupManager subscriptionGroupManager = new SubscriptionGroupManager(brokerController);\n        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);\n        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);\n        when(brokerController.getPutMessageFutureExecutor()).thenReturn(Executors.newSingleThreadExecutor());\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        when(messageStore.now()).thenReturn(System.currentTimeMillis());\n        when(channel.remoteAddress()).thenReturn(new InetSocketAddress(1024));\n        when(handlerContext.channel()).thenReturn(channel);\n        when(messageStore.lookMessageByOffset(anyLong())).thenReturn(new MessageExt());\n        sendMessageProcessor = new SendMessageProcessor(brokerController);\n    }\n\n    @Test\n    public void testProcessRequest() throws Exception {\n        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))).\n            thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))));\n        assertPutResult(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testProcessRequest_WithHook() throws Exception {\n        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))).\n            thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))));\n        List<SendMessageHook> sendMessageHookList = new ArrayList<>();\n        final SendMessageContext[] sendMessageContext = new SendMessageContext[1];\n        SendMessageHook sendMessageHook = new SendMessageHook() {\n            @Override\n            public String hookName() {\n                return null;\n            }\n\n            @Override\n            public void sendMessageBefore(SendMessageContext context) {\n                sendMessageContext[0] = context;\n            }\n\n            @Override\n            public void sendMessageAfter(SendMessageContext context) {\n\n            }\n        };\n        sendMessageHookList.add(sendMessageHook);\n        sendMessageProcessor.registerSendMessageHook(sendMessageHookList);\n        assertPutResult(ResponseCode.SUCCESS);\n        assertThat(sendMessageContext[0]).isNotNull();\n        assertThat(sendMessageContext[0].getTopic()).isEqualTo(topic);\n        assertThat(sendMessageContext[0].getProducerGroup()).isEqualTo(group);\n    }\n\n    @Test\n    public void testProcessRequest_FlushTimeOut() throws Exception {\n        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))).\n            thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.FLUSH_DISK_TIMEOUT, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));\n        assertPutResult(ResponseCode.FLUSH_DISK_TIMEOUT);\n    }\n\n    @Test\n    public void testProcessRequest_MessageIllegal() throws Exception {\n        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))).\n            thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));\n        assertPutResult(ResponseCode.MESSAGE_ILLEGAL);\n    }\n\n    @Test\n    public void testProcessRequest_CreateMappedFileFailed() throws Exception {\n        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))).\n            thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));\n        assertPutResult(ResponseCode.SYSTEM_ERROR);\n    }\n\n    @Test\n    public void testProcessRequest_FlushSlaveTimeout() throws Exception {\n        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))).\n            thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.FLUSH_SLAVE_TIMEOUT, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));\n        assertPutResult(ResponseCode.FLUSH_SLAVE_TIMEOUT);\n    }\n\n    @Test\n    public void testProcessRequest_PageCacheBusy() throws Exception {\n        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))).\n            thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.OS_PAGE_CACHE_BUSY, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));\n        assertPutResult(ResponseCode.SYSTEM_BUSY);\n    }\n\n    @Test\n    public void testProcessRequest_PropertiesTooLong() throws Exception {\n        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))).\n            thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PROPERTIES_SIZE_EXCEEDED, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));\n        assertPutResult(ResponseCode.MESSAGE_ILLEGAL);\n    }\n\n    @Test\n    public void testProcessRequest_ServiceNotAvailable() throws Exception {\n        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))).\n            thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));\n        assertPutResult(ResponseCode.SERVICE_NOT_AVAILABLE);\n    }\n\n    @Test\n    public void testProcessRequest_SlaveNotAvailable() throws Exception {\n        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))).\n            thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.SLAVE_NOT_AVAILABLE, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));\n        assertPutResult(ResponseCode.SLAVE_NOT_AVAILABLE);\n    }\n\n    @Test\n    public void testProcessRequest_WithMsgBack() throws Exception {\n        when(messageStore.putMessage(any(MessageExtBrokerInner.class))).\n            thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));\n        final RemotingCommand request = createSendMsgBackCommand(RequestCode.CONSUMER_SEND_MSG_BACK);\n\n        sendMessageProcessor = new SendMessageProcessor(brokerController);\n        final RemotingCommand response = sendMessageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testProcessRequest_Transaction() throws RemotingCommandException {\n        brokerController.setTransactionalMessageService(transactionMsgService);\n        when(brokerController.getTransactionalMessageService().asyncPrepareMessage(any(MessageExtBrokerInner.class)))\n            .thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))));\n        RemotingCommand request = createSendTransactionMsgCommand(RequestCode.SEND_MESSAGE);\n        final RemotingCommand[] response = new RemotingCommand[1];\n        doAnswer(invocation -> {\n            response[0] = invocation.getArgument(0);\n            return null;\n        }).when(channel).writeAndFlush(any(Object.class));\n        await().atMost(Duration.ofSeconds(10)).until(() -> {\n            RemotingCommand responseToReturn = sendMessageProcessor.processRequest(handlerContext, request);\n            if (responseToReturn != null) {\n                assertThat(response[0]).isNull();\n                response[0] = responseToReturn;\n            }\n\n            if (response[0] == null) {\n                return false;\n            }\n            assertThat(response[0].getCode()).isEqualTo(ResponseCode.SUCCESS);\n            assertThat(response[0].getOpaque()).isEqualTo(request.getOpaque());\n            return true;\n        });\n    }\n\n    @Test\n    public void testProcessRequest_WithAbortProcessSendMessageBeforeHook() throws Exception {\n        List<SendMessageHook> sendMessageHookList = new ArrayList<>();\n        final SendMessageContext[] sendMessageContext = new SendMessageContext[1];\n        SendMessageHook sendMessageHook = new SendMessageHook() {\n            @Override\n            public String hookName() {\n                return null;\n            }\n\n            @Override\n            public void sendMessageBefore(SendMessageContext context) {\n                sendMessageContext[0] = context;\n                throw new AbortProcessException(ResponseCode.FLOW_CONTROL, \"flow control test\");\n            }\n\n            @Override\n            public void sendMessageAfter(SendMessageContext context) {\n\n            }\n        };\n        sendMessageHookList.add(sendMessageHook);\n        sendMessageProcessor.registerSendMessageHook(sendMessageHookList);\n        assertPutResult(ResponseCode.FLOW_CONTROL);\n        assertThat(sendMessageContext[0]).isNotNull();\n        assertThat(sendMessageContext[0].getTopic()).isEqualTo(topic);\n        assertThat(sendMessageContext[0].getProducerGroup()).isEqualTo(group);\n    }\n\n    @Test\n    public void testProcessRequest_WithMsgBackWithConsumeMessageAfterHook() throws Exception {\n        when(messageStore.putMessage(any(MessageExtBrokerInner.class))).\n            thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));\n        final RemotingCommand request = createSendMsgBackCommand(RequestCode.CONSUMER_SEND_MSG_BACK);\n\n        sendMessageProcessor = new SendMessageProcessor(brokerController);\n        List<ConsumeMessageHook> consumeMessageHookList = new ArrayList<>();\n        final ConsumeMessageContext[] messageContext = new ConsumeMessageContext[1];\n        ConsumeMessageHook consumeMessageHook = new ConsumeMessageHook() {\n            @Override\n            public String hookName() {\n                return \"TestHook\";\n            }\n\n            @Override\n            public void consumeMessageBefore(ConsumeMessageContext context) {\n\n            }\n\n            @Override\n            public void consumeMessageAfter(ConsumeMessageContext context) {\n                messageContext[0] = context;\n                throw new AbortProcessException(ResponseCode.FLOW_CONTROL, \"flow control test\");\n            }\n        };\n        consumeMessageHookList.add(consumeMessageHook);\n        sendMessageProcessor.registerConsumeMessageHook(consumeMessageHookList);\n        final RemotingCommand response = sendMessageProcessor.processRequest(handlerContext, request);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testAttachRecallHandle_skip() {\n        MessageExt message = new MessageExt();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_BATCH_MESSAGE, null);\n        sendMessageProcessor.attachRecallHandle(request, message, new SendMessageResponseHeader());\n\n        request = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, null);\n        sendMessageProcessor.attachRecallHandle(request, message, new SendMessageResponseHeader());\n\n        request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, null);\n        sendMessageProcessor.attachRecallHandle(request, message, new SendMessageResponseHeader());\n\n        request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, null);\n        sendMessageProcessor.attachRecallHandle(request, message, new SendMessageResponseHeader());\n\n        verify(brokerConfig, times(0)).getBrokerName();\n    }\n\n    @Test\n    public void testAttachRecallHandle_doAttach() throws DecoderException {\n        int[] precisionSet = {100, 200, 500, 1000};\n        SendMessageResponseHeader responseHeader = new SendMessageResponseHeader();\n        String id = MessageClientIDSetter.createUniqID();\n        long timestamp = System.currentTimeMillis();\n\n        for (int precisionMs : precisionSet) {\n            long deliverMs = floor(timestamp, precisionMs);\n            MessageExt message = new MessageExt();\n            MessageAccessor.putProperty(message, MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, id);\n            MessageAccessor.putProperty(message, MessageConst.PROPERTY_TIMER_OUT_MS, String.valueOf(deliverMs));\n            MessageAccessor.putProperty(message, MessageConst.PROPERTY_REAL_TOPIC, topic);\n\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, null);\n            sendMessageProcessor.attachRecallHandle(request, message, responseHeader);\n            Assert.assertNotNull(responseHeader.getRecallHandle());\n            RecallMessageHandle.HandleV1 v1 =\n                (RecallMessageHandle.HandleV1) RecallMessageHandle.decodeHandle(responseHeader.getRecallHandle());\n            Assert.assertEquals(id, v1.getMessageId());\n            Assert.assertEquals(topic, v1.getTopic());\n            Assert.assertEquals(deliverMs + 1, Long.parseLong(v1.getTimestampStr()));\n            Assert.assertEquals(deliverMs, floor(Long.valueOf(v1.getTimestampStr()), precisionMs));\n        }\n    }\n\n    private long floor(long deliverMs, int precisionMs) {\n        assert precisionMs > 0;\n        if (deliverMs % precisionMs == 0) {\n            deliverMs -= precisionMs;\n        } else {\n            deliverMs = deliverMs / precisionMs * precisionMs;\n        }\n        return deliverMs;\n    }\n\n    private RemotingCommand createSendTransactionMsgCommand(int requestCode) {\n        SendMessageRequestHeader header = createSendMsgRequestHeader();\n        int sysFlag = header.getSysFlag();\n        Map<String, String> oriProps = MessageDecoder.string2messageProperties(header.getProperties());\n        oriProps.put(MessageConst.PROPERTY_TRANSACTION_PREPARED, \"true\");\n        header.setProperties(MessageDecoder.messageProperties2String(oriProps));\n        sysFlag |= MessageSysFlag.TRANSACTION_PREPARED_TYPE;\n        header.setSysFlag(sysFlag);\n        RemotingCommand request = RemotingCommand.createRequestCommand(requestCode, header);\n        request.setBody(new byte[] {'a'});\n        request.makeCustomHeaderToNet();\n        return request;\n    }\n\n    private SendMessageRequestHeader createSendMsgRequestHeader() {\n        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();\n        requestHeader.setProducerGroup(group);\n        requestHeader.setTopic(topic);\n        requestHeader.setDefaultTopic(TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC);\n        requestHeader.setDefaultTopicQueueNums(3);\n        requestHeader.setQueueId(1);\n        requestHeader.setSysFlag(0);\n        requestHeader.setBornTimestamp(System.currentTimeMillis());\n        requestHeader.setFlag(124);\n        requestHeader.setReconsumeTimes(0);\n        return requestHeader;\n    }\n\n    private RemotingCommand createSendMsgCommand(int requestCode) {\n        SendMessageRequestHeader requestHeader = createSendMsgRequestHeader();\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(requestCode, requestHeader);\n        request.setBody(new byte[] {'a'});\n        request.makeCustomHeaderToNet();\n        return request;\n    }\n\n    private RemotingCommand createSendMsgBackCommand(int requestCode) {\n        ConsumerSendMsgBackRequestHeader requestHeader = new ConsumerSendMsgBackRequestHeader();\n\n        requestHeader.setMaxReconsumeTimes(3);\n        requestHeader.setDelayLevel(4);\n        requestHeader.setGroup(group);\n        requestHeader.setOffset(123L);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(requestCode, requestHeader);\n        request.makeCustomHeaderToNet();\n        return request;\n    }\n\n    /**\n     * We will explain the logic of this method so you can get a better feeling of how to use it: This method assumes\n     * that if responseToReturn is not null, then there would be an error, which means the writeAndFlush are never\n     * reached. If responseToReturn is null, means everything ok, so writeAndFlush should record the actual response.\n     *\n     * @param responseCode\n     * @throws RemotingCommandException\n     */\n    private void assertPutResult(int responseCode) throws RemotingCommandException {\n        final RemotingCommand request = createSendMsgCommand(RequestCode.SEND_MESSAGE);\n        final RemotingCommand[] response = new RemotingCommand[1];\n        doAnswer(invocation -> {\n            response[0] = invocation.getArgument(0);\n            return null;\n        }).when(channel).writeAndFlush(any(Object.class));\n        await().atMost(Duration.ofSeconds(10)).until(() -> {\n            RemotingCommand responseToReturn = sendMessageProcessor.processRequest(handlerContext, request);\n            if (responseToReturn != null) {\n                assertThat(response[0]).isNull();\n                response[0] = responseToReturn;\n            }\n\n            if (response[0] == null) {\n                return false;\n            }\n            assertThat(response[0].getCode()).isEqualTo(responseCode);\n            assertThat(response[0].getOpaque()).isEqualTo(request.getOpaque());\n            return true;\n        });\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/schedule/ScheduleMessageServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.schedule;\n\nimport java.io.File;\nimport java.lang.reflect.Field;\nimport java.net.InetAddress;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.net.UnknownHostException;\nimport java.nio.ByteBuffer;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Random;\nimport java.util.UUID;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.failover.EscapeBridge;\nimport org.apache.rocketmq.broker.metrics.BrokerMetricsManager;\nimport org.apache.rocketmq.broker.util.HookUtils;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.ConsumeQueueExt;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.MessageArrivingListener;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.apache.rocketmq.common.metrics.NopLongCounter;\nimport org.apache.rocketmq.common.metrics.NopLongHistogram;\nimport io.opentelemetry.api.common.Attributes;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.Mockito;\n\nimport static org.apache.rocketmq.common.stats.Stats.BROKER_PUT_NUMS;\nimport static org.apache.rocketmq.common.stats.Stats.TOPIC_PUT_NUMS;\nimport static org.apache.rocketmq.common.stats.Stats.TOPIC_PUT_SIZE;\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.junit.Assert.assertTrue;\nimport static org.junit.Assert.assertEquals;\n\npublic class ScheduleMessageServiceTest {\n\n    private BrokerController brokerController;\n    private ScheduleMessageService scheduleMessageService;\n\n    /**\n     * t defaultMessageDelayLevel = \"1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h\"\n     */\n    String testMessageDelayLevel = \"5s 8s\";\n    /**\n     * choose delay level\n     */\n    int delayLevel = 3;\n\n    private static final String STORE_PATH = System.getProperty(\"java.io.tmpdir\") + File.separator + \"schedule_test#\" + UUID.randomUUID();\n    private static final int COMMIT_LOG_FILE_SIZE = 1024;\n    private static final int CQ_FILE_SIZE = 10;\n    private static final int CQ_EXT_FILE_SIZE = 10 * (ConsumeQueueExt.CqExtUnit.MIN_EXT_UNIT_SIZE + 64);\n\n    private static SocketAddress bornHost;\n    private static SocketAddress storeHost;\n    private DefaultMessageStore messageStore;\n    private MessageStoreConfig messageStoreConfig;\n    private BrokerConfig brokerConfig;\n\n    static String sendMessage = \" ------- schedule message test -------\";\n    static String topic = \"schedule_topic_test\";\n    static String messageGroup = \"delayGroupTest\";\n    private Random random = new Random();\n\n    static {\n        try {\n            bornHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123);\n        } catch (UnknownHostException e) {\n            e.printStackTrace();\n        }\n        try {\n            storeHost = new InetSocketAddress(InetAddress.getByName(\"127.0.0.1\"), 0);\n        } catch (UnknownHostException e) {\n            e.printStackTrace();\n        }\n    }\n\n    @Before\n    public void setUp() throws Exception {\n        messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setMessageDelayLevel(testMessageDelayLevel);\n        messageStoreConfig.setMappedFileSizeCommitLog(COMMIT_LOG_FILE_SIZE);\n        messageStoreConfig.setMappedFileSizeConsumeQueue(CQ_FILE_SIZE);\n        messageStoreConfig.setMappedFileSizeConsumeQueueExt(CQ_EXT_FILE_SIZE);\n        messageStoreConfig.setMessageIndexEnable(false);\n        messageStoreConfig.setEnableConsumeQueueExt(true);\n        messageStoreConfig.setStorePathRootDir(STORE_PATH);\n        messageStoreConfig.setStorePathCommitLog(STORE_PATH + File.separator + \"commitlog\");\n        // Let OS pick an available port\n        messageStoreConfig.setHaListenPort(0);\n\n        brokerConfig = new BrokerConfig();\n        BrokerStatsManager manager = new BrokerStatsManager(brokerConfig.getBrokerClusterName(), brokerConfig.isEnableDetailStat());\n        messageStore = new DefaultMessageStore(messageStoreConfig, manager, new MyMessageArrivingListener(), new BrokerConfig(), new ConcurrentHashMap<>());\n\n        assertThat(messageStore.load()).isTrue();\n\n        messageStore.start();\n        brokerController = Mockito.mock(BrokerController.class);\n        Mockito.when(brokerController.getMessageStore()).thenReturn(messageStore);\n        Mockito.when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n        Mockito.when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        Mockito.when(brokerController.peekMasterBroker()).thenReturn(brokerController);\n        Mockito.when(brokerController.getBrokerStatsManager()).thenReturn(manager);\n        EscapeBridge escapeBridge = new EscapeBridge(brokerController);\n        Mockito.when(brokerController.getEscapeBridge()).thenReturn(escapeBridge);\n        // Initialize BrokerMetricsManager to prevent NPE in tests\n        BrokerMetricsManager brokerMetricsManager = Mockito.mock(BrokerMetricsManager.class);\n        // Mock newAttributesBuilder to return a valid AttributesBuilder instead of null\n        Mockito.when(brokerMetricsManager.newAttributesBuilder()).thenReturn(Attributes.builder());\n        // Mock metrics getter methods to return Nop implementations to prevent NPE\n        Mockito.when(brokerMetricsManager.getMessagesInTotal()).thenReturn(new NopLongCounter());\n        Mockito.when(brokerMetricsManager.getMessagesOutTotal()).thenReturn(new NopLongCounter());\n        Mockito.when(brokerMetricsManager.getThroughputInTotal()).thenReturn(new NopLongCounter());\n        Mockito.when(brokerMetricsManager.getThroughputOutTotal()).thenReturn(new NopLongCounter());\n        Mockito.when(brokerMetricsManager.getMessageSize()).thenReturn(new NopLongHistogram());\n        Mockito.when(brokerController.getBrokerMetricsManager()).thenReturn(brokerMetricsManager);\n        scheduleMessageService = Mockito.spy(new ScheduleMessageService(brokerController));\n        // Mock ScheduleMessageService before it's used in HookUtils\n        Mockito.when(brokerController.getScheduleMessageService()).thenReturn(scheduleMessageService);\n        scheduleMessageService.load();\n        scheduleMessageService.start();\n    }\n\n    @Test\n    public void testLoad() {\n        ConcurrentMap<Integer, Long> offsetTable = scheduleMessageService.getOffsetTable();\n        //offsetTable.put(0, 1L);\n        offsetTable.put(1, 3L);\n        offsetTable.put(2, 5L);\n        scheduleMessageService.persist();\n\n        ScheduleMessageService controlInstance = new ScheduleMessageService(brokerController);\n        assertTrue(controlInstance.load());\n\n        ConcurrentMap<Integer, Long> loaded = controlInstance.getOffsetTable();\n        for (long offset : loaded.values()) {\n            assertEquals(0, offset);\n        }\n    }\n\n    @Test\n    public void testCorrectDelayOffset_whenInit() throws Exception {\n\n        ConcurrentMap<Integer /* level */, Long/* offset */> offsetTable = null;\n\n        scheduleMessageService = new ScheduleMessageService(brokerController);\n        scheduleMessageService.parseDelayLevel();\n\n        ConcurrentMap<Integer /* level */, Long/* offset */> offsetTable1 = new ConcurrentHashMap<>();\n        for (int i = 1; i <= 2; i++) {\n            offsetTable1.put(i, random.nextLong());\n        }\n\n        Field field = scheduleMessageService.getClass().getDeclaredField(\"offsetTable\");\n        field.setAccessible(true);\n        field.set(scheduleMessageService, offsetTable1);\n\n        String jsonStr = scheduleMessageService.encode();\n        scheduleMessageService.decode(jsonStr);\n\n        offsetTable = (ConcurrentMap<Integer, Long>) field.get(scheduleMessageService);\n\n        for (Map.Entry<Integer, Long> entry : offsetTable.entrySet()) {\n            assertEquals(entry.getValue(), offsetTable1.get(entry.getKey()));\n        }\n\n        boolean success = scheduleMessageService.correctDelayOffset();\n\n        System.out.printf(\"correctDelayOffset %s\", success);\n\n        offsetTable = (ConcurrentMap<Integer, Long>) field.get(scheduleMessageService);\n\n        for (long offset : offsetTable.values()) {\n            assertEquals(0, offset);\n        }\n    }\n\n    @Test\n    public void testDeliverDelayedMessageTimerTask() throws Exception {\n        assertThat(messageStore.getMessageStoreConfig().isEnableScheduleMessageStats()).isTrue();\n\n        assertThat(messageStore.getBrokerStatsManager().getStatsItem(TOPIC_PUT_NUMS, topic)).isNull();\n\n        MessageExtBrokerInner msg = buildMessage();\n        int realQueueId = msg.getQueueId();\n        // set delayLevel,and send delay message\n        msg.setDelayTimeLevel(delayLevel);\n        HookUtils.handleScheduleMessage(brokerController, msg);\n        PutMessageResult result = messageStore.putMessage(msg);\n        assertThat(result.isOk()).isTrue();\n\n        // consumer message\n        int delayQueueId = ScheduleMessageService.delayLevel2QueueId(delayLevel);\n        assertThat(delayQueueId).isEqualTo(delayLevel - 1);\n\n        Long offset = result.getAppendMessageResult().getLogicsOffset();\n\n        // now, no message in queue,must wait > delayTime\n        GetMessageResult messageResult = getMessage(realQueueId, offset);\n        assertThat(messageResult.getStatus()).isEqualTo(GetMessageStatus.NO_MESSAGE_IN_QUEUE);\n\n        // timer run maybe delay, then consumer message again\n        // and wait offsetTable\n        TimeUnit.SECONDS.sleep(15);\n        scheduleMessageService.buildRunningStats(new HashMap<>());\n\n        messageResult = getMessage(realQueueId, offset);\n        // now,found the message\n        assertThat(messageResult.getStatus()).isEqualTo(GetMessageStatus.FOUND);\n\n        // get the stats change\n        assertThat(messageStore.getBrokerStatsManager().getStatsItem(BROKER_PUT_NUMS, brokerConfig.getBrokerClusterName()).getValue().sum()).isEqualTo(1);\n        assertThat(messageStore.getBrokerStatsManager().getStatsItem(TOPIC_PUT_NUMS, topic).getValue().sum()).isEqualTo(1L);\n        assertThat(messageStore.getBrokerStatsManager().getStatsItem(TOPIC_PUT_SIZE, topic).getValue().sum()).isEqualTo(messageResult.getBufferTotalSize());\n\n        // get the message body\n        ByteBuffer byteBuffer = ByteBuffer.allocate(messageResult.getBufferTotalSize());\n        List<ByteBuffer> byteBufferList = messageResult.getMessageBufferList();\n        for (ByteBuffer bb : byteBufferList) {\n            byteBuffer.put(bb);\n        }\n\n        // warp and decode the message\n        byteBuffer = ByteBuffer.wrap(byteBuffer.array());\n        List<MessageExt> msgList = MessageDecoder.decodes(byteBuffer);\n        String retryMsg = new String(msgList.get(0).getBody());\n        assertThat(sendMessage).isEqualTo(retryMsg);\n\n        //  method will wait 10s,so I run it by myself\n        scheduleMessageService.persist();\n\n        // add mapFile release\n        messageResult.release();\n\n    }\n\n    /**\n     * add some [error/no use] code test\n     */\n    @Test\n    public void otherTest() {\n        // the method no use ,why need ?\n        int queueId = ScheduleMessageService.queueId2DelayLevel(delayLevel);\n        assertThat(queueId).isEqualTo(delayLevel + 1);\n\n        // error delayLevelTest\n        Long time = scheduleMessageService.computeDeliverTimestamp(999, 0);\n        assertThat(time).isEqualTo(1000);\n\n        // just decode\n        scheduleMessageService.decode(new DelayOffsetSerializeWrapper().toJson());\n    }\n\n    private GetMessageResult getMessage(int queueId, Long offset) {\n        return messageStore.getMessage(messageGroup, topic,\n            queueId, offset, 1, null);\n\n    }\n\n    @After\n    public void shutdown() throws InterruptedException {\n        scheduleMessageService.shutdown();\n        messageStore.shutdown();\n        messageStore.destroy();\n        File file = new File(messageStoreConfig.getStorePathRootDir());\n        UtilAll.deleteFile(file);\n    }\n\n    public MessageExtBrokerInner buildMessage() {\n\n        byte[] msgBody = sendMessage.getBytes();\n        MessageExtBrokerInner msg = new MessageExtBrokerInner();\n        msg.setTopic(topic);\n        msg.setTags(\"schedule_tag\");\n        msg.setKeys(\"schedule_key\");\n        msg.setBody(msgBody);\n        msg.setSysFlag(0);\n        msg.setBornTimestamp(System.currentTimeMillis());\n        msg.setStoreHost(storeHost);\n        msg.setBornHost(bornHost);\n        return msg;\n    }\n\n    private class MyMessageArrivingListener implements MessageArrivingListener {\n        @Override\n        public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime,\n            byte[] filterBitMap, Map<String, String> properties) {\n        }\n    }\n}"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/slave/SlaveSynchronizeAtomicTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.slave;\n\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.when;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.ThreadLocalRandom;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.loadbalance.MessageRequestModeManager;\nimport org.apache.rocketmq.broker.out.BrokerOuterAPI;\nimport org.apache.rocketmq.broker.processor.QueryAssignmentProcessor;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.exception.RemotingConnectException;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.body.MessageRequestModeSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class SlaveSynchronizeAtomicTest {\n    @Spy\n    private BrokerController brokerController =\n            new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(),\n                    new MessageStoreConfig());\n\n    private SlaveSynchronize slaveSynchronize;\n\n    @Mock\n    private BrokerOuterAPI brokerOuterAPI;\n\n    @Mock\n    private TopicConfigManager topicConfigManager;\n\n\n    @Mock\n    private SubscriptionGroupManager subscriptionGroupManager;\n\n    @Mock\n    private QueryAssignmentProcessor queryAssignmentProcessor;\n\n    @Mock\n    private MessageRequestModeManager messageRequestModeManager;\n\n\n    private static final String BROKER_ADDR = \"127.0.0.1:10911\";\n    private final SubscriptionGroupWrapper subscriptionGroupWrapper = createSubscriptionGroupWrapper();\n    private final MessageRequestModeSerializeWrapper requestModeSerializeWrapper = createMessageRequestModeWrapper();\n    private final DataVersion dataVersion = new DataVersion();\n\n    @Before\n    public void init() {\n        for (int i = 0; i < 100000; i++) {\n            subscriptionGroupWrapper.getSubscriptionGroupTable().put(\"group\" + i, new SubscriptionGroupConfig());\n        }\n        for (int i = 0; i < 100000; i++) {\n            requestModeSerializeWrapper.getMessageRequestModeMap().put(\"topic\" + i, new ConcurrentHashMap<>());\n        }\n        when(brokerController.getBrokerOuterAPI()).thenReturn(brokerOuterAPI);\n        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);\n        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);\n        when(subscriptionGroupManager.getDataVersion()).thenReturn(dataVersion);\n        when(subscriptionGroupManager.getSubscriptionGroupTable()).thenReturn(\n                subscriptionGroupWrapper.getSubscriptionGroupTable());\n        slaveSynchronize = new SlaveSynchronize(brokerController);\n        slaveSynchronize.setMasterAddr(BROKER_ADDR);\n    }\n\n    private SubscriptionGroupWrapper createSubscriptionGroupWrapper() {\n        SubscriptionGroupWrapper wrapper = new SubscriptionGroupWrapper();\n        wrapper.setSubscriptionGroupTable(new ConcurrentHashMap<>());\n        DataVersion dataVersion = new DataVersion();\n        dataVersion.setStateVersion(1L);\n        wrapper.setDataVersion(dataVersion);\n        return wrapper;\n    }\n\n    private MessageRequestModeSerializeWrapper createMessageRequestModeWrapper() {\n        MessageRequestModeSerializeWrapper wrapper = new MessageRequestModeSerializeWrapper();\n        wrapper.setMessageRequestModeMap(new ConcurrentHashMap<>());\n        return wrapper;\n    }\n\n    @Test\n    public void testSyncAtomically()\n        throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException,\n        InterruptedException, RemotingCommandException {\n        when(brokerOuterAPI.getAllSubscriptionGroupConfig(anyString())).thenReturn(subscriptionGroupWrapper);\n        when(brokerOuterAPI.getAllMessageRequestMode(anyString())).thenReturn(requestModeSerializeWrapper);\n\n        CountDownLatch countDownLatch = new CountDownLatch(1);\n        new Thread(() -> {\n            while (countDownLatch.getCount() > 0) {\n                dataVersion.nextVersion();\n                try {\n                    slaveSynchronize.syncAll();\n                } catch (Exception e) {\n                    e.printStackTrace();\n                }\n            }\n        }).start();\n\n        for (int i = 0; i < 10000000; i++) {\n            Assert.assertTrue(subscriptionGroupWrapper.getSubscriptionGroupTable()\n                    .containsKey(\"group\" + ThreadLocalRandom.current().nextInt(0, 100000)));\n            Assert.assertTrue(requestModeSerializeWrapper.getMessageRequestModeMap()\n                    .containsKey(\"topic\" + ThreadLocalRandom.current().nextInt(0, 100000)));\n        }\n        countDownLatch.countDown();\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/slave/SlaveSynchronizeTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.slave;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.loadbalance.MessageRequestModeManager;\nimport org.apache.rocketmq.broker.offset.ConsumerOffsetManager;\nimport org.apache.rocketmq.broker.out.BrokerOuterAPI;\nimport org.apache.rocketmq.broker.processor.QueryAssignmentProcessor;\nimport org.apache.rocketmq.broker.schedule.ScheduleMessageService;\nimport org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.exception.RemotingConnectException;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerOffsetSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.MessageRequestModeSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.timer.TimerCheckpoint;\nimport org.apache.rocketmq.store.timer.TimerMessageStore;\nimport org.apache.rocketmq.store.timer.TimerMetrics;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.io.UnsupportedEncodingException;\nimport java.util.concurrent.ConcurrentHashMap;\n\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class SlaveSynchronizeTest {\n    @Spy\n    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());\n\n    private SlaveSynchronize slaveSynchronize;\n\n    @Mock\n    private BrokerOuterAPI brokerOuterAPI;\n\n    @Mock\n    private TopicConfigManager topicConfigManager;\n\n    @Mock\n    private ConsumerOffsetManager consumerOffsetManager;\n\n    @Mock\n    private MessageStoreConfig messageStoreConfig;\n\n    @Mock\n    private MessageStore messageStore;\n\n    @Mock\n    private ScheduleMessageService scheduleMessageService;\n\n    @Mock\n    private SubscriptionGroupManager subscriptionGroupManager;\n\n    @Mock\n    private QueryAssignmentProcessor queryAssignmentProcessor;\n\n    @Mock\n    private MessageRequestModeManager messageRequestModeManager;\n\n    @Mock\n    private TimerMessageStore timerMessageStore;\n\n    @Mock\n    private TimerMetrics timerMetrics;\n\n    @Mock\n    private TimerCheckpoint timerCheckpoint;\n\n    private static final String BROKER_ADDR = \"127.0.0.1:10911\";\n\n    @Before\n    public void init() {\n        when(brokerController.getBrokerOuterAPI()).thenReturn(brokerOuterAPI);\n        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);\n        when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n        when(brokerController.getScheduleMessageService()).thenReturn(scheduleMessageService);\n        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);\n        when(brokerController.getQueryAssignmentProcessor()).thenReturn(queryAssignmentProcessor);\n        when(brokerController.getMessageStore()).thenReturn(messageStore);\n        when(brokerController.getTimerMessageStore()).thenReturn(timerMessageStore);\n        when(brokerController.getTimerCheckpoint()).thenReturn(timerCheckpoint);\n        when(topicConfigManager.getDataVersion()).thenReturn(new DataVersion());\n        when(topicConfigManager.getTopicConfigTable()).thenReturn(new ConcurrentHashMap<>());\n        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);\n        when(consumerOffsetManager.getOffsetTable()).thenReturn(new ConcurrentHashMap<>());\n        when(consumerOffsetManager.getDataVersion()).thenReturn(new DataVersion());\n        when(subscriptionGroupManager.getDataVersion()).thenReturn(new DataVersion());\n        when(subscriptionGroupManager.getSubscriptionGroupTable()).thenReturn(new ConcurrentHashMap<>());\n        when(queryAssignmentProcessor.getMessageRequestModeManager()).thenReturn(messageRequestModeManager);\n        when(messageRequestModeManager.getMessageRequestModeMap()).thenReturn(new ConcurrentHashMap<>());\n        when(messageStoreConfig.isTimerWheelEnable()).thenReturn(true);\n        when(messageStore.getTimerMessageStore()).thenReturn(timerMessageStore);\n        when(timerMessageStore.isShouldRunningDequeue()).thenReturn(false);\n        when(timerMessageStore.getTimerMetrics()).thenReturn(timerMetrics);\n        when(timerMetrics.getDataVersion()).thenReturn(new DataVersion());\n        when(timerCheckpoint.getDataVersion()).thenReturn(new DataVersion());\n        slaveSynchronize = new SlaveSynchronize(brokerController);\n        slaveSynchronize.setMasterAddr(BROKER_ADDR);\n    }\n\n    @Test\n    public void testSyncAll() throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException,\n        MQBrokerException, InterruptedException, UnsupportedEncodingException, RemotingCommandException {\n        TopicConfig newTopicConfig = new TopicConfig(\"NewTopic\");\n        when(brokerOuterAPI.getAllTopicConfig(anyString())).thenReturn(createTopicConfigWrapper(newTopicConfig));\n        when(brokerOuterAPI.getAllConsumerOffset(anyString())).thenReturn(createConsumerOffsetWrapper());\n        when(brokerOuterAPI.getAllDelayOffset(anyString())).thenReturn(\"\");\n        when(brokerOuterAPI.getAllSubscriptionGroupConfig(anyString())).thenReturn(createSubscriptionGroupWrapper());\n        when(brokerOuterAPI.getAllMessageRequestMode(anyString())).thenReturn(createMessageRequestModeWrapper());\n        when(brokerOuterAPI.getTimerMetrics(anyString())).thenReturn(createTimerMetricsWrapper());\n        slaveSynchronize.syncAll();\n        Assert.assertEquals(1, this.brokerController.getTopicConfigManager().getDataVersion().getStateVersion());\n        Assert.assertEquals(1, this.brokerController.getTopicQueueMappingManager().getDataVersion().getStateVersion());\n        Assert.assertEquals(1, consumerOffsetManager.getDataVersion().getStateVersion());\n        Assert.assertEquals(1, subscriptionGroupManager.getDataVersion().getStateVersion());\n        Assert.assertEquals(1, timerMetrics.getDataVersion().getStateVersion());\n    }\n\n    @Test\n    public void testGetMasterAddr() {\n        Assert.assertEquals(BROKER_ADDR, slaveSynchronize.getMasterAddr());\n    }\n\n    @Test\n    public void testSyncTimerCheckPoint() throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {\n        when(brokerOuterAPI.getTimerCheckPoint(anyString())).thenReturn(timerCheckpoint);\n        slaveSynchronize.syncTimerCheckPoint();\n        Assert.assertEquals(0, timerCheckpoint.getDataVersion().getStateVersion());\n    }\n\n    private TopicConfigAndMappingSerializeWrapper createTopicConfigWrapper(TopicConfig topicConfig) {\n        TopicConfigAndMappingSerializeWrapper wrapper = new TopicConfigAndMappingSerializeWrapper();\n        wrapper.setTopicConfigTable(new ConcurrentHashMap<>());\n        wrapper.getTopicConfigTable().put(topicConfig.getTopicName(), topicConfig);\n        DataVersion dataVersion = new DataVersion();\n        dataVersion.setStateVersion(1L);\n        wrapper.setDataVersion(dataVersion);\n        wrapper.setMappingDataVersion(dataVersion);\n        return wrapper;\n    }\n\n    private ConsumerOffsetSerializeWrapper createConsumerOffsetWrapper() {\n        ConsumerOffsetSerializeWrapper wrapper = new ConsumerOffsetSerializeWrapper();\n        wrapper.setOffsetTable(new ConcurrentHashMap<>());\n        DataVersion dataVersion = new DataVersion();\n        dataVersion.setStateVersion(1L);\n        wrapper.setDataVersion(dataVersion);\n        return wrapper;\n    }\n\n    private SubscriptionGroupWrapper createSubscriptionGroupWrapper() {\n        SubscriptionGroupWrapper wrapper = new SubscriptionGroupWrapper();\n        wrapper.setSubscriptionGroupTable(new ConcurrentHashMap<>());\n        DataVersion dataVersion = new DataVersion();\n        dataVersion.setStateVersion(1L);\n        wrapper.setDataVersion(dataVersion);\n        return wrapper;\n    }\n\n    private MessageRequestModeSerializeWrapper createMessageRequestModeWrapper() {\n        MessageRequestModeSerializeWrapper wrapper = new MessageRequestModeSerializeWrapper();\n        wrapper.setMessageRequestModeMap(new ConcurrentHashMap<>());\n        return wrapper;\n    }\n\n    private TimerMetrics.TimerMetricsSerializeWrapper createTimerMetricsWrapper() {\n        TimerMetrics.TimerMetricsSerializeWrapper wrapper = new TimerMetrics.TimerMetricsSerializeWrapper();\n        wrapper.setTimingCount(new ConcurrentHashMap<>());\n        DataVersion dataVersion = new DataVersion();\n        dataVersion.setStateVersion(1L);\n        wrapper.setDataVersion(dataVersion);\n        return wrapper;\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/subscription/ForbiddenTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.subscription;\n\nimport static org.junit.Assert.assertEquals;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Test;\n\npublic class ForbiddenTest {\n    @Test\n    public void testBrokerRestart() throws Exception {\n        SubscriptionGroupManager s = new SubscriptionGroupManager(\n            new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig()));\n        s.updateForbidden(\"g\", \"t\", 0, true);\n        assertEquals(1, s.getForbidden(\"g\", \"t\"));\n        assertEquals(true, s.getForbidden(\"g\", \"t\", 0));\n\n        s.updateForbidden(\"g\", \"t\", 1, true);\n        assertEquals(3, s.getForbidden(\"g\", \"t\"));\n        assertEquals(true, s.getForbidden(\"g\", \"t\", 1));\n\n        s.updateForbidden(\"g\", \"t\", 2, true);\n        assertEquals(7, s.getForbidden(\"g\", \"t\"));\n        assertEquals(true, s.getForbidden(\"g\", \"t\", 2));\n\n        s.updateForbidden(\"g\", \"t\", 1, false);\n        assertEquals(5, s.getForbidden(\"g\", \"t\"));\n        assertEquals(false, s.getForbidden(\"g\", \"t\", 1));\n\n        s.updateForbidden(\"g\", \"t\", 1, false);\n        assertEquals(5, s.getForbidden(\"g\", \"t\"));\n        assertEquals(false, s.getForbidden(\"g\", \"t\", 1));\n\n        s.updateForbidden(\"g\", \"t\", 0, false);\n        assertEquals(4, s.getForbidden(\"g\", \"t\"));\n        assertEquals(false, s.getForbidden(\"g\", \"t\", 0));\n\n        s.updateForbidden(\"g\", \"t\", 2, false);\n        assertEquals(0, s.getForbidden(\"g\", \"t\"));\n        assertEquals(false, s.getForbidden(\"g\", \"t\", 2));\n    }\n\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/subscription/RocksdbGroupConfigTransferTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.subscription;\n\nimport java.io.IOException;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.util.Comparator;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.UUID;\nimport java.util.stream.Stream;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.config.v1.RocksDBSubscriptionGroupManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class RocksdbGroupConfigTransferTest {\n    private final String basePath = Paths.get(System.getProperty(\"user.home\"),\n            \"unit-test-store\", UUID.randomUUID().toString().substring(0, 16).toUpperCase()).toString();\n\n    private RocksDBSubscriptionGroupManager rocksDBSubscriptionGroupManager;\n\n    private SubscriptionGroupManager jsonSubscriptionGroupManager;\n    @Mock\n    private BrokerController brokerController;\n\n    @Mock\n    private DefaultMessageStore defaultMessageStore;\n\n    @Before\n    public void init() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        BrokerConfig brokerConfig = new BrokerConfig();\n        Mockito.lenient().when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setStorePathRootDir(basePath);\n        Mockito.lenient().when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n        Mockito.lenient().when(brokerController.getMessageStore()).thenReturn(defaultMessageStore);\n        when(defaultMessageStore.getStateMachineVersion()).thenReturn(0L);\n    }\n\n    @After\n    public void destroy() {\n        if (notToBeExecuted()) {\n            return;\n        }\n\n        if (rocksDBSubscriptionGroupManager != null) {\n            rocksDBSubscriptionGroupManager.stop();\n        }\n\n        Path root = Paths.get(basePath);\n        if (Files.notExists(root)) {\n            return;\n        }\n\n        try (Stream<Path> walk = Files.walk(root)) {\n            walk.sorted(Comparator.reverseOrder())\n                .forEach(p -> {\n                    try {\n                        Files.deleteIfExists(p);\n                    } catch (IOException e) {\n                        // ignore\n                    }\n                });\n        } catch (IOException e) {\n                // ignore\n        }\n    }\n\n\n    public void initRocksDBSubscriptionGroupManager() {\n        if (rocksDBSubscriptionGroupManager == null) {\n            rocksDBSubscriptionGroupManager = new RocksDBSubscriptionGroupManager(brokerController);\n            rocksDBSubscriptionGroupManager.load();\n        }\n    }\n\n    public void initJsonSubscriptionGroupManager() {\n        if (jsonSubscriptionGroupManager == null) {\n            jsonSubscriptionGroupManager = new SubscriptionGroupManager(brokerController);\n            jsonSubscriptionGroupManager.load();\n        }\n    }\n\n    @Test\n    public void theFirstTimeLoadJsonSubscriptionGroupManager() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        initJsonSubscriptionGroupManager();\n        DataVersion dataVersion = jsonSubscriptionGroupManager.getDataVersion();\n        Assert.assertNotNull(dataVersion);\n        Assert.assertEquals(0L, dataVersion.getCounter().get());\n        Assert.assertEquals(0L, dataVersion.getStateVersion());\n        Assert.assertNotEquals(0, jsonSubscriptionGroupManager.getSubscriptionGroupTable().size());\n    }\n\n    @Test\n    public void theFirstTimeLoadRocksDBSubscriptionGroupManager() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        initRocksDBSubscriptionGroupManager();\n        DataVersion dataVersion = rocksDBSubscriptionGroupManager.getDataVersion();\n        Assert.assertNotNull(dataVersion);\n        Assert.assertEquals(0L, dataVersion.getCounter().get());\n        Assert.assertEquals(0L, dataVersion.getStateVersion());\n        Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size());\n    }\n\n\n    @Test\n    public void addGroupLoadJsonSubscriptionGroupManager() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        initJsonSubscriptionGroupManager();\n        int beforeSize = jsonSubscriptionGroupManager.getSubscriptionGroupTable().size();\n        String groupName = \"testAddGroupConfig-\" + System.currentTimeMillis();\n\n        Map<String, String> attributes = new HashMap<>();\n\n        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n        subscriptionGroupConfig.setGroupName(groupName);\n        subscriptionGroupConfig.setAttributes(attributes);\n        DataVersion beforeDataVersion = jsonSubscriptionGroupManager.getDataVersion();\n        long beforeDataVersionCounter = beforeDataVersion.getCounter().get();\n        long beforeTimestamp = beforeDataVersion.getTimestamp();\n\n        jsonSubscriptionGroupManager.updateSubscriptionGroupConfig(subscriptionGroupConfig);\n\n        int afterSize = jsonSubscriptionGroupManager.getSubscriptionGroupTable().size();\n        DataVersion afterDataVersion = jsonSubscriptionGroupManager.getDataVersion();\n        long afterDataVersionCounter = afterDataVersion.getCounter().get();\n        long afterTimestamp = afterDataVersion.getTimestamp();\n\n        Assert.assertEquals(0, beforeDataVersionCounter);\n        Assert.assertEquals(1, afterDataVersionCounter);\n        Assert.assertEquals(1, afterSize - beforeSize);\n        Assert.assertTrue(afterTimestamp >= beforeTimestamp);\n    }\n\n    @Test\n    public void addForbiddenGroupLoadJsonSubscriptionGroupManager() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        initJsonSubscriptionGroupManager();\n        int beforeSize = jsonSubscriptionGroupManager.getForbiddenTable().size();\n        String groupName = \"testAddGroupConfig-\" + System.currentTimeMillis();\n\n        Map<String, String> attributes = new HashMap<>();\n\n        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n        subscriptionGroupConfig.setGroupName(groupName);\n        subscriptionGroupConfig.setAttributes(attributes);\n        DataVersion beforeDataVersion = jsonSubscriptionGroupManager.getDataVersion();\n        long beforeDataVersionCounter = beforeDataVersion.getCounter().get();\n        long beforeTimestamp = beforeDataVersion.getTimestamp();\n\n        jsonSubscriptionGroupManager.setForbidden(groupName, \"topic\", 0);\n        int afterSize = jsonSubscriptionGroupManager.getForbiddenTable().size();\n        DataVersion afterDataVersion = jsonSubscriptionGroupManager.getDataVersion();\n        long afterDataVersionCounter = afterDataVersion.getCounter().get();\n        long afterTimestamp = afterDataVersion.getTimestamp();\n\n        Assert.assertEquals(1, afterDataVersionCounter - beforeDataVersionCounter);\n        Assert.assertEquals(1, afterSize - beforeSize);\n        Assert.assertTrue(afterTimestamp >= beforeTimestamp);\n    }\n\n    @Test\n    public void addGroupLoadRocksdbSubscriptionGroupManager() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        initRocksDBSubscriptionGroupManager();\n        int beforeSize = rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size();\n        String groupName = \"testAddGroupConfig-\" + System.currentTimeMillis();\n\n        Map<String, String> attributes = new HashMap<>();\n\n        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n        subscriptionGroupConfig.setGroupName(groupName);\n        subscriptionGroupConfig.setAttributes(attributes);\n        DataVersion beforeDataVersion = rocksDBSubscriptionGroupManager.getDataVersion();\n        long beforeDataVersionCounter = beforeDataVersion.getCounter().get();\n        long beforeTimestamp = beforeDataVersion.getTimestamp();\n\n        rocksDBSubscriptionGroupManager.updateSubscriptionGroupConfig(subscriptionGroupConfig);\n        int afterSize = rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size();\n        DataVersion afterDataVersion = rocksDBSubscriptionGroupManager.getDataVersion();\n        long afterDataVersionCounter = afterDataVersion.getCounter().get();\n        long afterTimestamp = afterDataVersion.getTimestamp();\n        Assert.assertEquals(1, afterDataVersionCounter);\n        Assert.assertEquals(0, beforeDataVersionCounter);\n        Assert.assertEquals(1, afterSize - beforeSize);\n        Assert.assertTrue(afterTimestamp >= beforeTimestamp);\n    }\n\n    @Test\n    public void addForbiddenLoadRocksdbSubscriptionGroupManager() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        initRocksDBSubscriptionGroupManager();\n        int beforeSize = rocksDBSubscriptionGroupManager.getForbiddenTable().size();\n        String groupName = \"testAddGroupConfig-\" + System.currentTimeMillis();\n\n        Map<String, String> attributes = new HashMap<>();\n\n        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n        subscriptionGroupConfig.setGroupName(groupName);\n        subscriptionGroupConfig.setAttributes(attributes);\n        DataVersion beforeDataVersion = rocksDBSubscriptionGroupManager.getDataVersion();\n        long beforeDataVersionCounter = beforeDataVersion.getCounter().get();\n        long beforeTimestamp = beforeDataVersion.getTimestamp();\n\n        rocksDBSubscriptionGroupManager.updateForbidden(groupName, \"topic\", 0, true);\n\n        int afterSize = rocksDBSubscriptionGroupManager.getForbiddenTable().size();\n        DataVersion afterDataVersion = rocksDBSubscriptionGroupManager.getDataVersion();\n        long afterDataVersionCounter = afterDataVersion.getCounter().get();\n        long afterTimestamp = afterDataVersion.getTimestamp();\n        Assert.assertEquals(1, afterDataVersionCounter - beforeDataVersionCounter);\n        Assert.assertEquals(1, afterSize - beforeSize);\n        Assert.assertTrue(afterTimestamp >= beforeTimestamp);\n        Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size());\n    }\n\n    @Test\n    public void theSecondTimeLoadJsonSubscriptionGroupManager() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        addGroupLoadJsonSubscriptionGroupManager();\n        jsonSubscriptionGroupManager.stop();\n        rocksDBSubscriptionGroupManager = null;\n        addForbiddenGroupLoadJsonSubscriptionGroupManager();\n        jsonSubscriptionGroupManager.stop();\n        rocksDBSubscriptionGroupManager = null;\n        jsonSubscriptionGroupManager = new SubscriptionGroupManager(brokerController);\n        jsonSubscriptionGroupManager.load();\n        DataVersion dataVersion = jsonSubscriptionGroupManager.getDataVersion();\n        Assert.assertNotNull(dataVersion);\n        Assert.assertEquals(2L, dataVersion.getCounter().get());\n        Assert.assertEquals(0L, dataVersion.getStateVersion());\n        Assert.assertNotEquals(0, jsonSubscriptionGroupManager.getSubscriptionGroupTable().size());\n        Assert.assertNotEquals(0, jsonSubscriptionGroupManager.getForbiddenTable().size());\n        Assert.assertNotEquals(0, jsonSubscriptionGroupManager.getSubscriptionGroupTable().size());\n    }\n\n    @Test\n    public void theSecondTimeLoadRocksdbTopicConfigManager() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        addGroupLoadRocksdbSubscriptionGroupManager();\n        rocksDBSubscriptionGroupManager.stop();\n        rocksDBSubscriptionGroupManager = null;\n        addForbiddenLoadRocksdbSubscriptionGroupManager();\n        rocksDBSubscriptionGroupManager.stop();\n        rocksDBSubscriptionGroupManager = null;\n        rocksDBSubscriptionGroupManager = new RocksDBSubscriptionGroupManager(brokerController);\n        rocksDBSubscriptionGroupManager.load();\n        DataVersion dataVersion = rocksDBSubscriptionGroupManager.getDataVersion();\n        Assert.assertNotNull(dataVersion);\n        Assert.assertEquals(2L, dataVersion.getCounter().get());\n        Assert.assertEquals(0L, dataVersion.getStateVersion());\n        Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size());\n        Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getForbiddenTable().size());\n        Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size());\n    }\n\n\n    @Test\n    public void jsonUpgradeToRocksdb() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        addGroupLoadJsonSubscriptionGroupManager();\n        addForbiddenGroupLoadJsonSubscriptionGroupManager();\n        initRocksDBSubscriptionGroupManager();\n        DataVersion dataVersion = rocksDBSubscriptionGroupManager.getDataVersion();\n        Assert.assertNotNull(dataVersion);\n        Assert.assertEquals(3L, dataVersion.getCounter().get());\n        Assert.assertEquals(0L, dataVersion.getStateVersion());\n        Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getForbiddenTable().size());\n        Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size());\n        Assert.assertEquals(rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size(), jsonSubscriptionGroupManager.getSubscriptionGroupTable().size());\n        Assert.assertEquals(rocksDBSubscriptionGroupManager.getForbiddenTable().size(), jsonSubscriptionGroupManager.getForbiddenTable().size());\n\n        rocksDBSubscriptionGroupManager.stop();\n        rocksDBSubscriptionGroupManager = new RocksDBSubscriptionGroupManager(brokerController);\n        rocksDBSubscriptionGroupManager.load();\n        dataVersion = rocksDBSubscriptionGroupManager.getDataVersion();\n        Assert.assertEquals(3L, dataVersion.getCounter().get());\n        Assert.assertEquals(0L, dataVersion.getStateVersion());\n        Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getForbiddenTable().size());\n        Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size());\n        Assert.assertEquals(rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size(), jsonSubscriptionGroupManager.getSubscriptionGroupTable().size());\n        Assert.assertEquals(rocksDBSubscriptionGroupManager.getForbiddenTable().size(), jsonSubscriptionGroupManager.getForbiddenTable().size());\n    }\n\n    private boolean notToBeExecuted() {\n        return MixAll.isMac() || MixAll.isWindows();\n    }\n\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.subscription;\n\nimport com.google.common.collect.ImmutableMap;\nimport java.nio.file.Paths;\nimport java.util.Collections;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.UUID;\nimport java.util.concurrent.ThreadLocalRandom;\nimport java.util.concurrent.atomic.AtomicLong;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.SubscriptionGroupAttributes;\nimport org.apache.rocketmq.common.attribute.BooleanAttribute;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.assertj.core.api.Assertions.assertThatThrownBy;\nimport static org.mockito.Mockito.never;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.verify;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class SubscriptionGroupManagerTest {\n    private String group = \"group\";\n\n    private final String basePath = Paths.get(System.getProperty(\"user.home\"),\n            \"unit-test-store\", UUID.randomUUID().toString().substring(0, 16).toUpperCase()).toString();\n    @Mock\n    private BrokerController brokerControllerMock;\n    private SubscriptionGroupManager subscriptionGroupManager;\n\n    @Before\n    public void before() {\n        SubscriptionGroupAttributes.ALL.put(\"test\", new BooleanAttribute(\n            \"test\",\n            false,\n            false\n        ));\n        subscriptionGroupManager = spy(new SubscriptionGroupManager(brokerControllerMock));\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setStorePathRootDir(basePath);\n        Mockito.lenient().when(brokerControllerMock.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n    }\n\n    @After\n    public void destroy() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        if (subscriptionGroupManager != null) {\n            subscriptionGroupManager.stop();\n        }\n    }\n\n    @Test\n    public void testUpdateAndCreateSubscriptionGroupInRocksdb() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        group += System.currentTimeMillis();\n        updateSubscriptionGroupConfig();\n    }\n\n    @Test\n    public void updateSubscriptionGroupConfig() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n        subscriptionGroupConfig.setGroupName(group);\n        Map<String, String> attr = ImmutableMap.of(\"+test\", \"true\");\n        subscriptionGroupConfig.setAttributes(attr);\n        SubscriptionGroupManager subscriptionGroupManager = new SubscriptionGroupManager(brokerControllerMock);\n        subscriptionGroupManager.updateSubscriptionGroupConfig(subscriptionGroupConfig);\n        SubscriptionGroupConfig result = subscriptionGroupManager.getSubscriptionGroupTable().get(group);\n        assertThat(result).isNotNull();\n        assertThat(result.getGroupName()).isEqualTo(group);\n        assertThat(result.getAttributes().get(\"test\")).isEqualTo(\"true\");\n\n        SubscriptionGroupConfig subscriptionGroupConfig1 = new SubscriptionGroupConfig();\n        subscriptionGroupConfig1.setGroupName(group);\n        Map<String, String> attrRemove = ImmutableMap.of(\"-test\", \"\");\n        subscriptionGroupConfig1.setAttributes(attrRemove);\n        assertThatThrownBy(() -> subscriptionGroupManager.updateSubscriptionGroupConfig(subscriptionGroupConfig1))\n            .isInstanceOf(RuntimeException.class).hasMessage(\"attempt to update an unchangeable attribute. key: test\");\n    }\n\n    private boolean notToBeExecuted() {\n        return MixAll.isMac();\n    }\n    @Test\n    public void testUpdateSubscriptionGroupConfigList_NullConfigList() {\n        if (notToBeExecuted()) {\n            return;\n        }\n\n        subscriptionGroupManager.updateSubscriptionGroupConfigList(null);\n        // Verifying that persist() is not called\n        verify(subscriptionGroupManager, never()).persist();\n    }\n\n    @Test\n    public void testUpdateSubscriptionGroupConfigList_EmptyConfigList() {\n        if (notToBeExecuted()) {\n            return;\n        }\n\n        subscriptionGroupManager.updateSubscriptionGroupConfigList(Collections.emptyList());\n        // Verifying that persist() is not called\n        verify(subscriptionGroupManager, never()).persist();\n    }\n\n    @Test\n    public void testUpdateSubscriptionGroupConfigList_ValidConfigList() {\n        if (notToBeExecuted()) {\n            return;\n        }\n\n        final List<SubscriptionGroupConfig> configList = new LinkedList<>();\n        final List<String> groupNames = new LinkedList<>();\n        for (int i = 0; i < 10; i++) {\n            SubscriptionGroupConfig config = new SubscriptionGroupConfig();\n            String groupName = String.format(\"group-%d\", i);\n            config.setGroupName(groupName);\n            configList.add(config);\n            groupNames.add(groupName);\n        }\n\n        SubscriptionGroupManager subscriptionGroupManager = new SubscriptionGroupManager(brokerControllerMock);\n        subscriptionGroupManager.updateSubscriptionGroupConfigList(configList);\n\n        groupNames.forEach(groupName ->\n            assertThat(subscriptionGroupManager.getSubscriptionGroupTable().get(groupName)).isNotNull());\n    }\n\n    @Test\n    public void testSubGroupTable() {\n        // Empty SubscriptionGroupManager\n        subscriptionGroupManager.getSubscriptionGroupTable().clear();\n        Map<String, SubscriptionGroupConfig> result =\n            subscriptionGroupManager.subGroupTable(subscriptionGroupManager.getDataVersion().toJson(), 0, 200);\n        assertThat(result).isEmpty();\n\n        // fill SubscriptionGroupManager\n        int totalGroupNum = 50000;\n        fillSubscriptionGroupManager(totalGroupNum);\n\n        // Null DataVersion\n        int beginIndex = 0, maxNum = 200;\n        int endIndex = beginIndex + maxNum - 1;\n        result = subscriptionGroupManager.subGroupTable(null, beginIndex, maxNum);\n\n        Assert.assertEquals(maxNum, result.size());\n        Assert.assertTrue(result.containsKey(String.format(\"group-%05d\", ThreadLocalRandom.current().nextInt(beginIndex, endIndex))));\n        Assert.assertFalse(result.containsKey(String.format(\"group-%05d\", beginIndex - 1)));\n        Assert.assertFalse(result.containsKey(String.format(\"group-%05d\", endIndex + 1)));\n\n        // Different DataVersion\n        DataVersion differentVersion = new DataVersion();\n        differentVersion.setCounter(new AtomicLong(1000L));  // different counter\n        differentVersion.setTimestamp(System.currentTimeMillis());\n        result = subscriptionGroupManager.subGroupTable(differentVersion.toJson(), 300, maxNum);\n\n        Assert.assertEquals(maxNum, result.size());\n        Assert.assertTrue(result.containsKey(String.format(\"group-%05d\", ThreadLocalRandom.current().nextInt(beginIndex, endIndex))));\n        Assert.assertFalse(result.containsKey(String.format(\"group-%05d\", beginIndex - 1)));\n        Assert.assertFalse(result.containsKey(String.format(\"group-%05d\", endIndex + 1)));\n\n        // BeginIndexOutOfRange\n        result = subscriptionGroupManager.subGroupTable(subscriptionGroupManager.getDataVersion().toJson(), totalGroupNum, 200);\n\n        Assert.assertTrue(result.isEmpty());\n\n        // Normal Case\n        beginIndex = 300;\n        endIndex = beginIndex + maxNum - 1;\n        result = subscriptionGroupManager.subGroupTable(subscriptionGroupManager.getDataVersion().toJson(), beginIndex, maxNum);\n\n        Assert.assertEquals(maxNum, result.size());\n        Assert.assertTrue(result.containsKey(String.format(\"group-%05d\", ThreadLocalRandom.current().nextInt(beginIndex, endIndex))));\n        Assert.assertFalse(result.containsKey(String.format(\"group-%05d\", beginIndex - 1)));\n        Assert.assertFalse(result.containsKey(String.format(\"group-%05d\", endIndex + 1)));\n\n        // NotFullTopicConfigTable\n        beginIndex = 49950;\n        endIndex = Math.min(subscriptionGroupManager.getSubscriptionGroupTable().size() - 1, beginIndex + maxNum - 1);\n        result = subscriptionGroupManager.subGroupTable(subscriptionGroupManager.getDataVersion().toJson(), beginIndex, maxNum);\n\n        Assert.assertEquals(totalGroupNum - beginIndex, result.size());\n        Assert.assertTrue(result.containsKey(String.format(\"group-%05d\", ThreadLocalRandom.current().nextInt(beginIndex, endIndex))));\n        Assert.assertFalse(result.containsKey(String.format(\"group-%05d\", beginIndex - 1)));\n        Assert.assertFalse(result.containsKey(String.format(\"group-%05d\", endIndex + 1)));\n\n    }\n\n    private void fillSubscriptionGroupManager(int num) {\n        for (int i = num - 1; i >= 0; i--) {\n            SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n            String groupName = String.format(\"group-%05d\", i);\n            subscriptionGroupConfig.setGroupName(groupName);\n            Map<String, String> attr = ImmutableMap.of(\"+test\", \"true\");\n            subscriptionGroupConfig.setAttributes(attr);\n            subscriptionGroupManager.getSubscriptionGroupTable().put(groupName, subscriptionGroupConfig);\n        }\n    }\n\n}"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.topic;\n\nimport java.nio.file.Paths;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Optional;\nimport java.util.UUID;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.config.v1.RocksDBTopicConfigManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicAttributes;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.attribute.Attribute;\nimport org.apache.rocketmq.common.attribute.BooleanAttribute;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.attribute.EnumAttribute;\nimport org.apache.rocketmq.common.attribute.LongRangeAttribute;\nimport org.apache.rocketmq.common.utils.QueueTypeUtils;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static com.google.common.collect.Sets.newHashSet;\nimport static java.util.Arrays.asList;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class RocksdbTopicConfigManagerTest {\n\n    private final String basePath = Paths.get(System.getProperty(\"user.home\"),\n            \"unit-test-store\", UUID.randomUUID().toString().substring(0, 16).toUpperCase()).toString();\n\n    private RocksDBTopicConfigManager topicConfigManager;\n    @Mock\n    private BrokerController brokerController;\n\n    @Mock\n    private DefaultMessageStore defaultMessageStore;\n\n    @Before\n    public void init() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        BrokerConfig brokerConfig = new BrokerConfig();\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setStorePathRootDir(basePath);\n        when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n        Mockito.lenient().when(brokerController.getMessageStore()).thenReturn(defaultMessageStore);\n        Mockito.lenient().when(defaultMessageStore.getStateMachineVersion()).thenReturn(0L);\n        topicConfigManager = new RocksDBTopicConfigManager(brokerController);\n        topicConfigManager.load();\n    }\n\n    @After\n    public void destroy() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        if (topicConfigManager != null) {\n            topicConfigManager.stop();\n        }\n    }\n\n    @Test\n    public void testAddUnsupportedKeyOnCreating() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        String unsupportedKey = \"key4\";\n        String topicName = \"testAddUnsupportedKeyOnCreating-\" + System.currentTimeMillis();\n\n        supportAttributes(asList(\n            new EnumAttribute(\"enum.key\", true, newHashSet(\"enum-1\", \"enum-2\", \"enum-3\"), \"enum-1\"),\n            new BooleanAttribute(\"bool.key\", false, false),\n            new LongRangeAttribute(\"long.range.key\", true, 10, 20, 15)\n        ));\n\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"+enum.key\", \"enum-2\");\n        attributes.put(\"+\" + unsupportedKey, \"value1\");\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(topicName);\n        topicConfig.setAttributes(attributes);\n\n        RuntimeException runtimeException = Assert.assertThrows(RuntimeException.class, () -> topicConfigManager.updateTopicConfig(topicConfig));\n        Assert.assertEquals(\"unsupported key: \" + unsupportedKey, runtimeException.getMessage());\n    }\n\n    @Test\n    public void testAddWrongFormatKeyOnCreating() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        String topicName = \"testAddWrongFormatKeyOnCreating-\" + System.currentTimeMillis();\n\n        supportAttributes(asList(\n            new EnumAttribute(\"enum.key\", true, newHashSet(\"enum-1\", \"enum-2\", \"enum-3\"), \"enum-1\"),\n            new BooleanAttribute(\"bool.key\", false, false),\n            new LongRangeAttribute(\"long.range.key\", true, 10, 20, 15)\n        ));\n\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"++enum.key\", \"value1\");\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(topicName);\n        topicConfig.setAttributes(attributes);\n\n        RuntimeException runtimeException = Assert.assertThrows(RuntimeException.class, () -> topicConfigManager.updateTopicConfig(topicConfig));\n        Assert.assertEquals(\"kv string format wrong.\", runtimeException.getMessage());\n    }\n\n    @Test\n    public void testDeleteKeyOnCreating() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        String topicName = \"testDeleteKeyOnCreating-\" + System.currentTimeMillis();\n\n        String key = \"enum.key\";\n        supportAttributes(asList(\n            new EnumAttribute(\"enum.key\", true, newHashSet(\"enum-1\", \"enum-2\", \"enum-3\"), \"enum-1\"),\n            new BooleanAttribute(\"bool.key\", false, false),\n            new LongRangeAttribute(\"long.range.key\", true, 10, 20, 15)\n        ));\n\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"-\" + key, \"\");\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(topicName);\n        topicConfig.setAttributes(attributes);\n\n        RuntimeException runtimeException = Assert.assertThrows(RuntimeException.class, () -> topicConfigManager.updateTopicConfig(topicConfig));\n        Assert.assertEquals(\"only add attribute is supported while creating topic. key: \" + key, runtimeException.getMessage());\n    }\n\n    @Test\n    public void testAddWrongValueOnCreating() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        String topicName = \"testAddWrongValueOnCreating-\" + System.currentTimeMillis();\n\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"+\" + TopicAttributes.QUEUE_TYPE_ATTRIBUTE.getName(), \"wrong-value\");\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(topicName);\n        topicConfig.setAttributes(attributes);\n\n        RuntimeException runtimeException = Assert.assertThrows(RuntimeException.class, () -> topicConfigManager.updateTopicConfig(topicConfig));\n        Assert.assertEquals(\"value is not in set: [SimpleCQ, BatchCQ]\", runtimeException.getMessage());\n    }\n\n    @Test\n    public void testNormalAddKeyOnCreating() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        String topic = \"testNormalAddKeyOnCreating-\" + System.currentTimeMillis();\n\n        supportAttributes(asList(\n            new EnumAttribute(\"enum.key\", true, newHashSet(\"enum-1\", \"enum-2\", \"enum-3\"), \"enum-1\"),\n            new BooleanAttribute(\"bool.key\", false, false),\n            new LongRangeAttribute(\"long.range.key\", true, 10, 20, 15)\n        ));\n\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"+enum.key\", \"enum-2\");\n        attributes.put(\"+long.range.key\", \"16\");\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(topic);\n        topicConfig.setAttributes(attributes);\n        topicConfigManager.updateTopicConfig(topicConfig);\n\n        TopicConfig existingTopicConfig = topicConfigManager.getTopicConfigTable().get(topic);\n        Assert.assertEquals(\"enum-2\", existingTopicConfig.getAttributes().get(\"enum.key\"));\n        Assert.assertEquals(\"16\", existingTopicConfig.getAttributes().get(\"long.range.key\"));\n    }\n\n    @Test\n    public void testAddDuplicatedKeyOnUpdating() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        String duplicatedKey = \"long.range.key\";\n        String topicName = \"testAddDuplicatedKeyOnUpdating-\" + System.currentTimeMillis();\n\n        supportAttributes(asList(\n            new EnumAttribute(\"enum.key\", true, newHashSet(\"enum-1\", \"enum-2\", \"enum-3\"), \"enum-1\"),\n            new BooleanAttribute(\"bool.key\", false, false),\n            new LongRangeAttribute(\"long.range.key\", true, 10, 20, 15)\n        ));\n\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"+enum.key\", \"enum-3\");\n        attributes.put(\"+bool.key\", \"true\");\n        attributes.put(\"+long.range.key\", \"12\");\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(topicName);\n        topicConfig.setAttributes(attributes);\n        topicConfigManager.updateTopicConfig(topicConfig);\n\n\n\n        attributes = new HashMap<>();\n        attributes.put(\"+\" + duplicatedKey, \"11\");\n        attributes.put(\"-\" + duplicatedKey, \"\");\n        TopicConfig duplicateTopicConfig = new TopicConfig();\n        duplicateTopicConfig.setTopicName(topicName);\n        duplicateTopicConfig.setAttributes(attributes);\n\n        RuntimeException runtimeException = Assert.assertThrows(RuntimeException.class, () -> topicConfigManager.updateTopicConfig(duplicateTopicConfig));\n        Assert.assertEquals(\"alter duplication key. key: \" + duplicatedKey, runtimeException.getMessage());\n    }\n\n    @Test\n    public void testDeleteNonexistentKeyOnUpdating() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        String key = \"nonexisting.key\";\n        String topicName = \"testDeleteNonexistentKeyOnUpdating-\" + System.currentTimeMillis();\n\n        supportAttributes(asList(\n            new EnumAttribute(\"enum.key\", true, newHashSet(\"enum-1\", \"enum-2\", \"enum-3\"), \"enum-1\"),\n            new BooleanAttribute(\"bool.key\", false, false),\n            new LongRangeAttribute(\"long.range.key\", true, 10, 20, 15)\n        ));\n\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"+enum.key\", \"enum-2\");\n        attributes.put(\"+bool.key\", \"true\");\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(topicName);\n        topicConfig.setAttributes(attributes);\n\n        topicConfigManager.updateTopicConfig(topicConfig);\n\n        attributes = new HashMap<>();\n        attributes.clear();\n        attributes.put(\"-\" + key, \"\");\n        topicConfig.setAttributes(attributes);\n        RuntimeException runtimeException = Assert.assertThrows(RuntimeException.class, () -> topicConfigManager.updateTopicConfig(topicConfig));\n        Assert.assertEquals(\"attempt to delete a nonexistent key: \" + key, runtimeException.getMessage());\n    }\n\n    @Test\n    public void testAlterTopicWithoutChangingAttributes() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        String topic = \"testAlterTopicWithoutChangingAttributes-\" + System.currentTimeMillis();\n\n        supportAttributes(asList(\n            new EnumAttribute(\"enum.key\", true, newHashSet(\"enum-1\", \"enum-2\", \"enum-3\"), \"enum-1\"),\n            new BooleanAttribute(\"bool.key\", false, false),\n            new LongRangeAttribute(\"long.range.key\", true, 10, 20, 15)\n        ));\n\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"+enum.key\", \"enum-2\");\n        attributes.put(\"+bool.key\", \"true\");\n\n        TopicConfig topicConfigInit = new TopicConfig();\n        topicConfigInit.setTopicName(topic);\n        topicConfigInit.setAttributes(attributes);\n\n        topicConfigManager.updateTopicConfig(topicConfigInit);\n        Assert.assertEquals(\"enum-2\", topicConfigManager.getTopicConfigTable().get(topic).getAttributes().get(\"enum.key\"));\n        Assert.assertEquals(\"true\", topicConfigManager.getTopicConfigTable().get(topic).getAttributes().get(\"bool.key\"));\n\n        TopicConfig topicConfigAlter = new TopicConfig();\n        topicConfigAlter.setTopicName(topic);\n        topicConfigAlter.setReadQueueNums(10);\n        topicConfigAlter.setWriteQueueNums(10);\n        topicConfigManager.updateTopicConfig(topicConfigAlter);\n        Assert.assertEquals(\"enum-2\", topicConfigManager.getTopicConfigTable().get(topic).getAttributes().get(\"enum.key\"));\n        Assert.assertEquals(\"true\", topicConfigManager.getTopicConfigTable().get(topic).getAttributes().get(\"bool.key\"));\n    }\n\n    @Test\n    public void testNormalUpdateUnchangeableKeyOnUpdating() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        String topic = \"testNormalUpdateUnchangeableKeyOnUpdating-\" + System.currentTimeMillis();\n\n        supportAttributes(asList(\n            new EnumAttribute(\"enum.key\", true, newHashSet(\"enum-1\", \"enum-2\", \"enum-3\"), \"enum-1\"),\n            new BooleanAttribute(\"bool.key\", true, false),\n            new LongRangeAttribute(\"long.range.key\", false, 10, 20, 15)\n        ));\n\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"+long.range.key\", \"14\");\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(topic);\n        topicConfig.setAttributes(attributes);\n\n        topicConfigManager.updateTopicConfig(topicConfig);\n\n        attributes.put(\"+long.range.key\", \"16\");\n        topicConfig.setAttributes(attributes);\n        RuntimeException runtimeException = Assert.assertThrows(RuntimeException.class, () -> topicConfigManager.updateTopicConfig(topicConfig));\n        Assert.assertEquals(\"attempt to update an unchangeable attribute. key: long.range.key\", runtimeException.getMessage());\n    }\n\n    @Test\n    public void testNormalQueryKeyOnGetting() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        String topic = \"testNormalQueryKeyOnGetting-\" + System.currentTimeMillis();\n        String unchangeable = \"bool.key\";\n\n        supportAttributes(asList(\n            new EnumAttribute(\"enum.key\", true, newHashSet(\"enum-1\", \"enum-2\", \"enum-3\"), \"enum-1\"),\n            new BooleanAttribute(\"bool.key\", false, false),\n            new LongRangeAttribute(\"long.range.key\", true, 10, 20, 15)\n        ));\n\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"+\" + unchangeable, \"true\");\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(topic);\n        topicConfig.setAttributes(attributes);\n\n        topicConfigManager.updateTopicConfig(topicConfig);\n\n        TopicConfig topicConfigUpdated = topicConfigManager.getTopicConfigTable().get(topic);\n        Assert.assertEquals(CQType.SimpleCQ, QueueTypeUtils.getCQType(Optional.of(topicConfigUpdated)));\n\n        Assert.assertEquals(\"true\", topicConfigUpdated.getAttributes().get(unchangeable));\n    }\n\n    private void supportAttributes(List<Attribute> supportAttributes) {\n        Map<String, Attribute> supportedAttributes = new HashMap<>();\n\n        for (Attribute supportAttribute : supportAttributes) {\n            supportedAttributes.put(supportAttribute.getName(), supportAttribute);\n        }\n\n        TopicAttributes.ALL.putAll(supportedAttributes);\n    }\n\n    private boolean notToBeExecuted() {\n        return MixAll.isMac();\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigTransferTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.topic;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.config.v1.RocksDBTopicConfigManager;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.io.IOException;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.util.Comparator;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.UUID;\n\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class RocksdbTopicConfigTransferTest {\n\n    private final String basePath = Paths.get(System.getProperty(\"user.home\"),\n            \"unit-test-store\", UUID.randomUUID().toString().substring(0, 16).toUpperCase()).toString();\n\n    private RocksDBTopicConfigManager rocksdbTopicConfigManager;\n\n    private TopicConfigManager jsonTopicConfigManager;\n    @Mock\n    private BrokerController brokerController;\n\n    @Mock\n    private DefaultMessageStore defaultMessageStore;\n\n    @Before\n    public void init() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        BrokerConfig brokerConfig = new BrokerConfig();\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setStorePathRootDir(basePath);\n        Mockito.lenient().when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n        when(brokerController.getMessageStore()).thenReturn(defaultMessageStore);\n        when(defaultMessageStore.getStateMachineVersion()).thenReturn(0L);\n    }\n\n    @After\n    public void destroy() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        Path pathToBeDeleted = Paths.get(basePath);\n        try {\n            Files.walk(pathToBeDeleted)\n                    .sorted(Comparator.reverseOrder())\n                    .forEach(path -> {\n                        try {\n                            Files.delete(path);\n                        } catch (IOException e) {\n                            // ignore\n                        }\n                    });\n        } catch (IOException e) {\n            // ignore\n        }\n        if (rocksdbTopicConfigManager != null) {\n            rocksdbTopicConfigManager.stop();\n        }\n    }\n\n    public void initRocksdbTopicConfigManager() {\n        if (rocksdbTopicConfigManager == null) {\n            rocksdbTopicConfigManager = new RocksDBTopicConfigManager(brokerController);\n            rocksdbTopicConfigManager.load();\n        }\n    }\n\n    public void initJsonTopicConfigManager() {\n        if (jsonTopicConfigManager == null) {\n            jsonTopicConfigManager = new TopicConfigManager(brokerController);\n            jsonTopicConfigManager.load();\n        }\n    }\n\n    @Test\n    public void theFirstTimeLoadJsonTopicConfigManager() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        initJsonTopicConfigManager();\n        DataVersion dataVersion = jsonTopicConfigManager.getDataVersion();\n        Assert.assertNotNull(dataVersion);\n        Assert.assertEquals(0L, dataVersion.getCounter().get());\n        Assert.assertEquals(0L, dataVersion.getStateVersion());\n        Assert.assertNotEquals(0, jsonTopicConfigManager.getTopicConfigTable().size());\n    }\n\n    @Test\n    public void theFirstTimeLoadRocksdbTopicConfigManager() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        initRocksdbTopicConfigManager();\n        DataVersion dataVersion = rocksdbTopicConfigManager.getDataVersion();\n        Assert.assertNotNull(dataVersion);\n        Assert.assertEquals(0L, dataVersion.getCounter().get());\n        Assert.assertEquals(0L, dataVersion.getStateVersion());\n        Assert.assertNotEquals(0, rocksdbTopicConfigManager.getTopicConfigTable().size());\n    }\n\n\n    @Test\n    public void addTopicLoadJsonTopicConfigManager() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        initJsonTopicConfigManager();\n        String topicName = \"testAddTopicConfig-\" + System.currentTimeMillis();\n\n        Map<String, String> attributes = new HashMap<>();\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(topicName);\n        topicConfig.setAttributes(attributes);\n        DataVersion beforeDataVersion = jsonTopicConfigManager.getDataVersion();\n        long beforeDataVersionCounter = beforeDataVersion.getCounter().get();\n        long beforeTimestamp = beforeDataVersion.getTimestamp();\n\n        jsonTopicConfigManager.updateTopicConfig(topicConfig);\n\n        DataVersion afterDataVersion = jsonTopicConfigManager.getDataVersion();\n        long afterDataVersionCounter = afterDataVersion.getCounter().get();\n        long afterTimestamp = afterDataVersion.getTimestamp();\n\n        Assert.assertEquals(0, beforeDataVersionCounter);\n        Assert.assertEquals(1, afterDataVersionCounter);\n        Assert.assertTrue(afterTimestamp >= beforeTimestamp);\n    }\n\n    @Test\n    public void addTopicLoadRocksdbTopicConfigManager() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        initRocksdbTopicConfigManager();\n        String topicName = \"testAddTopicConfig-\" + System.currentTimeMillis();\n\n        Map<String, String> attributes = new HashMap<>();\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(topicName);\n        topicConfig.setAttributes(attributes);\n        DataVersion beforeDataVersion = rocksdbTopicConfigManager.getDataVersion();\n        long beforeDataVersionCounter = beforeDataVersion.getCounter().get();\n        long beforeTimestamp = beforeDataVersion.getTimestamp();\n\n        rocksdbTopicConfigManager.updateTopicConfig(topicConfig);\n\n        DataVersion afterDataVersion = rocksdbTopicConfigManager.getDataVersion();\n        long afterDataVersionCounter = afterDataVersion.getCounter().get();\n        long afterTimestamp = afterDataVersion.getTimestamp();\n        Assert.assertEquals(0, beforeDataVersionCounter);\n        Assert.assertEquals(1, afterDataVersionCounter);\n        Assert.assertTrue(afterTimestamp >= beforeTimestamp);\n    }\n\n    @Test\n    public void theSecondTimeLoadJsonTopicConfigManager() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        addTopicLoadJsonTopicConfigManager();\n        jsonTopicConfigManager.stop();\n        jsonTopicConfigManager = new TopicConfigManager(brokerController);\n        jsonTopicConfigManager.load();\n        DataVersion dataVersion = jsonTopicConfigManager.getDataVersion();\n        Assert.assertNotNull(dataVersion);\n        Assert.assertEquals(1L, dataVersion.getCounter().get());\n        Assert.assertEquals(0L, dataVersion.getStateVersion());\n        Assert.assertNotEquals(0, jsonTopicConfigManager.getTopicConfigTable().size());\n    }\n\n    @Test\n    public void theSecondTimeLoadRocksdbTopicConfigManager() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        addTopicLoadRocksdbTopicConfigManager();\n        rocksdbTopicConfigManager.stop();\n        rocksdbTopicConfigManager = null;\n        rocksdbTopicConfigManager = new RocksDBTopicConfigManager(brokerController);\n        rocksdbTopicConfigManager.load();\n        DataVersion dataVersion = rocksdbTopicConfigManager.getDataVersion();\n        Assert.assertNotNull(dataVersion);\n        Assert.assertEquals(1L, dataVersion.getCounter().get());\n        Assert.assertEquals(0L, dataVersion.getStateVersion());\n        Assert.assertNotEquals(0, rocksdbTopicConfigManager.getTopicConfigTable().size());\n    }\n\n    @Test\n    public void jsonUpgradeToRocksdb() {\n        if (notToBeExecuted()) {\n            return;\n        }\n        addTopicLoadJsonTopicConfigManager();\n        initRocksdbTopicConfigManager();\n        DataVersion dataVersion = rocksdbTopicConfigManager.getDataVersion();\n        Assert.assertNotNull(dataVersion);\n        Assert.assertEquals(2L, dataVersion.getCounter().get());\n        Assert.assertEquals(0L, dataVersion.getStateVersion());\n        Assert.assertNotEquals(0, rocksdbTopicConfigManager.getTopicConfigTable().size());\n        Assert.assertEquals(rocksdbTopicConfigManager.getTopicConfigTable().size(), jsonTopicConfigManager.getTopicConfigTable().size());\n\n        rocksdbTopicConfigManager.stop();\n        rocksdbTopicConfigManager = new RocksDBTopicConfigManager(brokerController);\n        rocksdbTopicConfigManager.load();\n        dataVersion = rocksdbTopicConfigManager.getDataVersion();\n        Assert.assertEquals(2L, dataVersion.getCounter().get());\n        Assert.assertEquals(0L, dataVersion.getStateVersion());\n        Assert.assertNotEquals(0, rocksdbTopicConfigManager.getTopicConfigTable().size());\n        Assert.assertEquals(rocksdbTopicConfigManager.getTopicConfigTable().size(), rocksdbTopicConfigManager.getTopicConfigTable().size());\n    }\n\n\n    private boolean notToBeExecuted() {\n        return MixAll.isMac();\n    }\n\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/topic/TopicConfigManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.topic;\n\nimport java.nio.file.Paths;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Optional;\nimport java.util.UUID;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ThreadLocalRandom;\nimport java.util.concurrent.atomic.AtomicLong;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.TopicAttributes;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.attribute.Attribute;\nimport org.apache.rocketmq.common.attribute.BooleanAttribute;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.attribute.EnumAttribute;\nimport org.apache.rocketmq.common.attribute.LongRangeAttribute;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.utils.QueueTypeUtils;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static com.google.common.collect.Sets.newHashSet;\nimport static java.util.Arrays.asList;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class TopicConfigManagerTest {\n\n    private final String basePath = Paths.get(System.getProperty(\"user.home\"),\n            \"unit-test-store\", UUID.randomUUID().toString().substring(0, 16).toUpperCase()).toString();\n    private TopicConfigManager topicConfigManager;\n    @Mock\n    private BrokerController brokerController;\n\n    @Mock\n    private DefaultMessageStore defaultMessageStore;\n\n    @Before\n    public void init() {\n        BrokerConfig brokerConfig = new BrokerConfig();\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setStorePathRootDir(basePath);\n        when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n        Mockito.lenient().when(brokerController.getMessageStore()).thenReturn(defaultMessageStore);\n        when(defaultMessageStore.getStateMachineVersion()).thenReturn(0L);\n        topicConfigManager = new TopicConfigManager(brokerController);\n    }\n\n    @Test\n    public void testAddUnsupportedKeyOnCreating() {\n        String unsupportedKey = \"key4\";\n\n        supportAttributes(asList(\n                new EnumAttribute(\"enum.key\", true, newHashSet(\"enum-1\", \"enum-2\", \"enum-3\"), \"enum-1\"),\n                new BooleanAttribute(\"bool.key\", false, false),\n                new LongRangeAttribute(\"long.range.key\", true, 10, 20, 15)\n        ));\n\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"+enum.key\", \"enum-2\");\n        attributes.put(\"+\" + unsupportedKey, \"value1\");\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(\"new-topic\");\n        topicConfig.setAttributes(attributes);\n\n        RuntimeException runtimeException = Assert.assertThrows(RuntimeException.class, () -> topicConfigManager.updateTopicConfig(topicConfig));\n        Assert.assertEquals(\"unsupported key: \" + unsupportedKey, runtimeException.getMessage());\n    }\n\n    @Test\n    public void testAddWrongFormatKeyOnCreating() {\n        supportAttributes(asList(\n                new EnumAttribute(\"enum.key\", true, newHashSet(\"enum-1\", \"enum-2\", \"enum-3\"), \"enum-1\"),\n                new BooleanAttribute(\"bool.key\", false, false),\n                new LongRangeAttribute(\"long.range.key\", true, 10, 20, 15)\n        ));\n\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"++enum.key\", \"value1\");\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(\"new-topic\");\n        topicConfig.setAttributes(attributes);\n\n        RuntimeException runtimeException = Assert.assertThrows(RuntimeException.class, () -> topicConfigManager.updateTopicConfig(topicConfig));\n        Assert.assertEquals(\"kv string format wrong.\", runtimeException.getMessage());\n    }\n\n    @Test\n    public void testDeleteKeyOnCreating() {\n        String key = \"enum.key\";\n        supportAttributes(asList(\n                new EnumAttribute(\"enum.key\", true, newHashSet(\"enum-1\", \"enum-2\", \"enum-3\"), \"enum-1\"),\n                new BooleanAttribute(\"bool.key\", false, false),\n                new LongRangeAttribute(\"long.range.key\", true, 10, 20, 15)\n        ));\n\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"-\" + key, \"\");\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(\"new-topic\");\n        topicConfig.setAttributes(attributes);\n\n        RuntimeException runtimeException = Assert.assertThrows(RuntimeException.class, () -> topicConfigManager.updateTopicConfig(topicConfig));\n        Assert.assertEquals(\"only add attribute is supported while creating topic. key: \" + key, runtimeException.getMessage());\n    }\n\n    @Test\n    public void testAddWrongValueOnCreating() {\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"+\" + TopicAttributes.QUEUE_TYPE_ATTRIBUTE.getName(), \"wrong-value\");\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(\"new-topic\");\n        topicConfig.setAttributes(attributes);\n\n        RuntimeException runtimeException = Assert.assertThrows(RuntimeException.class, () -> topicConfigManager.updateTopicConfig(topicConfig));\n        Assert.assertEquals(\"value is not in set: [SimpleCQ, BatchCQ]\", runtimeException.getMessage());\n    }\n\n    @Test\n    public void testNormalAddKeyOnCreating() {\n        String topic = \"new-topic\";\n\n        supportAttributes(asList(\n                new EnumAttribute(\"enum.key\", true, newHashSet(\"enum-1\", \"enum-2\", \"enum-3\"), \"enum-1\"),\n                new BooleanAttribute(\"bool.key\", false, false),\n                new LongRangeAttribute(\"long.range.key\", true, 10, 20, 15)\n        ));\n\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"+enum.key\", \"enum-2\");\n        attributes.put(\"+long.range.key\", \"16\");\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(topic);\n        topicConfig.setAttributes(attributes);\n        topicConfigManager.updateTopicConfig(topicConfig);\n\n        TopicConfig existingTopicConfig = topicConfigManager.getTopicConfigTable().get(topic);\n        Assert.assertEquals(\"enum-2\", existingTopicConfig.getAttributes().get(\"enum.key\"));\n        Assert.assertEquals(\"16\", existingTopicConfig.getAttributes().get(\"long.range.key\"));\n//        assert file\n    }\n\n    @Test\n    public void testAddDuplicatedKeyOnUpdating() {\n        String duplicatedKey = \"long.range.key\";\n\n        supportAttributes(asList(\n                new EnumAttribute(\"enum.key\", true, newHashSet(\"enum-1\", \"enum-2\", \"enum-3\"), \"enum-1\"),\n                new BooleanAttribute(\"bool.key\", false, false),\n                new LongRangeAttribute(\"long.range.key\", true, 10, 20, 15)\n        ));\n\n        createTopic();\n\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"+\" + duplicatedKey, \"11\");\n        attributes.put(\"-\" + duplicatedKey, \"\");\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(\"new-topic\");\n        topicConfig.setAttributes(attributes);\n\n        RuntimeException runtimeException = Assert.assertThrows(RuntimeException.class, () -> topicConfigManager.updateTopicConfig(topicConfig));\n        Assert.assertEquals(\"alter duplication key. key: \" + duplicatedKey, runtimeException.getMessage());\n    }\n\n    private void createTopic() {\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"+enum.key\", \"enum-3\");\n        attributes.put(\"+bool.key\", \"true\");\n        attributes.put(\"+long.range.key\", \"12\");\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(\"new-topic\");\n        topicConfig.setAttributes(attributes);\n\n        topicConfigManager.updateTopicConfig(topicConfig);\n    }\n\n    @Test\n    public void testDeleteNonexistentKeyOnUpdating() {\n        String key = \"nonexisting.key\";\n\n        supportAttributes(asList(\n                new EnumAttribute(\"enum.key\", true, newHashSet(\"enum-1\", \"enum-2\", \"enum-3\"), \"enum-1\"),\n                new BooleanAttribute(\"bool.key\", false, false),\n                new LongRangeAttribute(\"long.range.key\", true, 10, 20, 15)\n        ));\n\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"+enum.key\", \"enum-2\");\n        attributes.put(\"+bool.key\", \"true\");\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(\"new-topic\");\n        topicConfig.setAttributes(attributes);\n\n        topicConfigManager.updateTopicConfig(topicConfig);\n\n        attributes = new HashMap<>();\n        attributes.clear();\n        attributes.put(\"-\" + key, \"\");\n        topicConfig.setAttributes(attributes);\n        RuntimeException runtimeException = Assert.assertThrows(RuntimeException.class, () -> topicConfigManager.updateTopicConfig(topicConfig));\n        Assert.assertEquals(\"attempt to delete a nonexistent key: \" + key, runtimeException.getMessage());\n    }\n\n    @Test\n    public void testAlterTopicWithoutChangingAttributes() {\n        String topic = \"new-topic\";\n\n        supportAttributes(asList(\n                new EnumAttribute(\"enum.key\", true, newHashSet(\"enum-1\", \"enum-2\", \"enum-3\"), \"enum-1\"),\n                new BooleanAttribute(\"bool.key\", false, false),\n                new LongRangeAttribute(\"long.range.key\", true, 10, 20, 15)\n        ));\n\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"+enum.key\", \"enum-2\");\n        attributes.put(\"+bool.key\", \"true\");\n\n        TopicConfig topicConfigInit = new TopicConfig();\n        topicConfigInit.setTopicName(topic);\n        topicConfigInit.setAttributes(attributes);\n\n        topicConfigManager.updateTopicConfig(topicConfigInit);\n        Assert.assertEquals(\"enum-2\", topicConfigManager.getTopicConfigTable().get(topic).getAttributes().get(\"enum.key\"));\n        Assert.assertEquals(\"true\", topicConfigManager.getTopicConfigTable().get(topic).getAttributes().get(\"bool.key\"));\n\n        TopicConfig topicConfigAlter = new TopicConfig();\n        topicConfigAlter.setTopicName(topic);\n        topicConfigAlter.setReadQueueNums(10);\n        topicConfigAlter.setWriteQueueNums(10);\n        topicConfigManager.updateTopicConfig(topicConfigAlter);\n        Assert.assertEquals(\"enum-2\", topicConfigManager.getTopicConfigTable().get(topic).getAttributes().get(\"enum.key\"));\n        Assert.assertEquals(\"true\", topicConfigManager.getTopicConfigTable().get(topic).getAttributes().get(\"bool.key\"));\n    }\n\n    @Test\n    public void testNormalUpdateUnchangeableKeyOnUpdating() {\n        String topic = \"exist-topic\";\n\n        supportAttributes(asList(\n                new EnumAttribute(\"enum.key\", true, newHashSet(\"enum-1\", \"enum-2\", \"enum-3\"), \"enum-1\"),\n                new BooleanAttribute(\"bool.key\", true, false),\n                new LongRangeAttribute(\"long.range.key\", false, 10, 20, 15)\n        ));\n\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"+long.range.key\", \"14\");\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(topic);\n        topicConfig.setAttributes(attributes);\n\n        topicConfigManager.updateTopicConfig(topicConfig);\n\n        attributes.put(\"+long.range.key\", \"16\");\n        topicConfig.setAttributes(attributes);\n        RuntimeException runtimeException = Assert.assertThrows(RuntimeException.class, () -> topicConfigManager.updateTopicConfig(topicConfig));\n        Assert.assertEquals(\"attempt to update an unchangeable attribute. key: long.range.key\", runtimeException.getMessage());\n    }\n\n    @Test\n    public void testNormalQueryKeyOnGetting() {\n        String topic = \"exist-topic\";\n        String unchangeable = \"bool.key\";\n\n        supportAttributes(asList(\n                new EnumAttribute(\"enum.key\", true, newHashSet(\"enum-1\", \"enum-2\", \"enum-3\"), \"enum-1\"),\n                new BooleanAttribute(\"bool.key\", false, false),\n                new LongRangeAttribute(\"long.range.key\", true, 10, 20, 15)\n        ));\n\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"+\" + unchangeable, \"true\");\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(topic);\n        topicConfig.setAttributes(attributes);\n\n        topicConfigManager.updateTopicConfig(topicConfig);\n\n        TopicConfig topicConfigUpdated = topicConfigManager.getTopicConfigTable().get(topic);\n        Assert.assertEquals(CQType.SimpleCQ, QueueTypeUtils.getCQType(Optional.of(topicConfigUpdated)));\n\n        Assert.assertEquals(\"true\", topicConfigUpdated.getAttributes().get(unchangeable));\n    }\n\n    private void supportAttributes(List<Attribute> supportAttributes) {\n        Map<String, Attribute> supportedAttributes = new HashMap<>();\n\n        for (Attribute supportAttribute : supportAttributes) {\n            supportedAttributes.put(supportAttribute.getName(), supportAttribute);\n        }\n\n        TopicAttributes.ALL.putAll(supportedAttributes);\n    }\n\n    private void fillTopicConfigTable(int num) {\n        ConcurrentHashMap<String, TopicConfig> topicConfigTable = new ConcurrentHashMap<>();\n        for (int i = num - 1; i >= 0; i--) {\n            String topicName = String.format(\"topic%05d\", i);\n            TopicConfig topicConfig = new TopicConfig(topicName, 1, 1,\n                PermName.PERM_READ | PermName.PERM_WRITE, 0);\n            topicConfigTable.put(topicName, topicConfig);\n        }\n        topicConfigManager.setTopicConfigTable(topicConfigTable);\n    }\n\n    @Test\n    public void testSubTopicConfigTable() {\n        // Empty TopicConfigTable\n        topicConfigManager.getTopicConfigTable().clear();\n        Map<String, TopicConfig> result = topicConfigManager.subTopicConfigTable(topicConfigManager.getDataVersion().toJson(), 0, 200);\n        Assert.assertTrue(result.isEmpty());\n\n        // init table, topic range [0, N)\n        final int totalTopicNum = 50000;\n        fillTopicConfigTable(totalTopicNum);\n\n        // Null DataVersion\n        int beginIndex = 0, maxNum = 200;\n        int endIndex = beginIndex + maxNum - 1;\n        result = topicConfigManager.subTopicConfigTable(null, beginIndex, maxNum);\n\n        Assert.assertEquals(maxNum, result.size());\n        Assert.assertTrue(result.containsKey(String.format(\"topic%05d\", ThreadLocalRandom.current().nextInt(beginIndex, endIndex))));\n        Assert.assertFalse(result.containsKey(String.format(\"topic%05d\", beginIndex - 1)));\n        Assert.assertFalse(result.containsKey(String.format(\"topic%05d\", endIndex + 1)));\n\n        // Different DataVersion\n        DataVersion differentVersion = new DataVersion();\n        differentVersion.setCounter(new AtomicLong(1000L));  // different counter\n        differentVersion.setTimestamp(System.currentTimeMillis());\n        result = topicConfigManager.subTopicConfigTable(differentVersion.toJson(), 300, maxNum);\n\n        Assert.assertEquals(maxNum, result.size());\n        Assert.assertTrue(result.containsKey(String.format(\"topic%05d\", ThreadLocalRandom.current().nextInt(beginIndex, endIndex))));\n        Assert.assertFalse(result.containsKey(String.format(\"topic%05d\", beginIndex - 1)));\n        Assert.assertFalse(result.containsKey(String.format(\"topic%05d\", endIndex + 1)));\n\n        // BeginIndexOutOfRange\n        result = topicConfigManager.subTopicConfigTable(topicConfigManager.getDataVersion().toJson(), totalTopicNum, 200);\n\n        Assert.assertTrue(result.isEmpty());\n\n        // Normal Case\n        beginIndex = 300;\n        endIndex = beginIndex + maxNum - 1;\n        result = topicConfigManager.subTopicConfigTable(topicConfigManager.getDataVersion().toJson(), beginIndex, maxNum);\n\n        Assert.assertEquals(maxNum, result.size());\n        Assert.assertTrue(result.containsKey(String.format(\"topic%05d\", ThreadLocalRandom.current().nextInt(beginIndex, endIndex))));\n        Assert.assertFalse(result.containsKey(String.format(\"topic%05d\", beginIndex - 1)));\n        Assert.assertFalse(result.containsKey(String.format(\"topic%05d\", endIndex + 1)));\n\n        // NotFullTopicConfigTable\n        beginIndex = 49950;\n        endIndex = Math.min(topicConfigManager.getTopicConfigTable().size() - 1, beginIndex + maxNum - 1);\n        result = topicConfigManager.subTopicConfigTable(topicConfigManager.getDataVersion().toJson(), beginIndex, maxNum);\n\n        Assert.assertEquals(totalTopicNum - beginIndex, result.size());\n        Assert.assertTrue(result.containsKey(String.format(\"topic%05d\", ThreadLocalRandom.current().nextInt(beginIndex, endIndex))));\n        Assert.assertFalse(result.containsKey(String.format(\"topic%05d\", beginIndex - 1)));\n        Assert.assertFalse(result.containsKey(String.format(\"topic%05d\", endIndex + 1)));\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/topic/TopicQueueMappingCleanServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.topic;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.out.BrokerOuterAPI;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.admin.TopicOffset;\nimport org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.apache.rocketmq.remoting.protocol.statictopic.LogicQueueMappingItem;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingDetail;\nimport org.apache.rocketmq.remoting.rpc.RpcClient;\nimport org.apache.rocketmq.remoting.rpc.RpcRequest;\nimport org.apache.rocketmq.remoting.rpc.RpcResponse;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.Map;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\n\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyBoolean;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.never;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class TopicQueueMappingCleanServiceTest {\n    \n    @Mock\n    private BrokerController brokerController;\n    \n    @Mock\n    private TopicQueueMappingManager topicQueueMappingManager;\n    \n    @Mock\n    private RpcClient rpcClient;\n    \n    @Mock\n    private MessageStoreConfig messageStoreConfig;\n    \n    @Mock\n    private BrokerConfig brokerConfig;\n    \n    @Mock\n    private BrokerOuterAPI brokerOuterAPI;\n    \n    private TopicQueueMappingCleanService topicQueueMappingCleanService;\n    \n    private final String defaultTopic = \"defaultTopic\";\n    \n    private final String defaultBroker = \"defaultBroker\";\n    \n    private final String deleteWhen = \"00;01;02;03;04;05;06;07;08;09;10;11;12;13;14;15;16;17;18;19;20;21;22;23\";\n    \n    @Before\n    public void init() {\n        when(brokerOuterAPI.getRpcClient()).thenReturn(rpcClient);\n        when(brokerController.getBrokerOuterAPI()).thenReturn(brokerOuterAPI);\n        when(brokerController.getTopicQueueMappingManager()).thenReturn(topicQueueMappingManager);\n        when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n        topicQueueMappingCleanService = new TopicQueueMappingCleanService(brokerController);\n    }\n    \n    @Test\n    public void testCleanItemExpiredNoChange() throws Exception {\n        when(messageStoreConfig.getDeleteWhen()).thenReturn(\"04\");\n        topicQueueMappingCleanService.cleanItemExpired();\n        verify(topicQueueMappingManager, never()).updateTopicQueueMapping(any(), anyBoolean(), anyBoolean(), anyBoolean());\n    }\n    \n    @Test\n    public void testCleanItemExpiredWithChange() throws Exception {\n        when(messageStoreConfig.getDeleteWhen()).thenReturn(deleteWhen);\n        TopicQueueMappingDetail mappingDetail = new TopicQueueMappingDetail(defaultTopic, 2, defaultBroker, 1);\n        mappingDetail.getHostedQueues().put(0,\n                Arrays.asList(new LogicQueueMappingItem(0, 0, defaultBroker, 0, 0, 100, 0, 0),\n                        new LogicQueueMappingItem(0, 1, defaultBroker, 1, 100, 200, 0, 0)));\n        when(topicQueueMappingManager.getTopicQueueMappingTable()).thenReturn(new ConcurrentHashMap<>(Collections.singletonMap(defaultTopic, mappingDetail)));\n        when(brokerConfig.getBrokerName()).thenReturn(defaultBroker);\n        TopicStatsTable topicStatsTable = mock(TopicStatsTable.class);\n        Map<MessageQueue, TopicOffset> offsetTable = new ConcurrentHashMap<>();\n        TopicOffset topicOffset = new TopicOffset();\n        topicOffset.setMinOffset(0);\n        topicOffset.setMaxOffset(0);\n        MessageQueue messageQueue = new MessageQueue(defaultTopic, defaultBroker, 0);\n        offsetTable.put(messageQueue, topicOffset);\n        when(topicStatsTable.getOffsetTable()).thenReturn(offsetTable);\n        when(rpcClient.invoke(any(RpcRequest.class), anyLong())).thenReturn(CompletableFuture.completedFuture(new RpcResponse(0, null, topicStatsTable)));\n        DataVersion dataVersion = mock(DataVersion.class);\n        when(topicQueueMappingManager.getDataVersion()).thenReturn(dataVersion);\n        topicQueueMappingCleanService.cleanItemExpired();\n        verify(topicQueueMappingManager, times(1)).updateTopicQueueMapping(any(), anyBoolean(), anyBoolean(), anyBoolean());\n    }\n    \n    @Test\n    public void testCleanItemListMoreThanSecondGen() throws Exception {\n        when(brokerConfig.getBrokerName()).thenReturn(defaultBroker);\n        when(messageStoreConfig.getDeleteWhen()).thenReturn(deleteWhen);\n        TopicQueueMappingDetail mappingDetail = new TopicQueueMappingDetail(defaultTopic, 1, defaultBroker, 1);\n        mappingDetail.setHostedQueues(new ConcurrentHashMap<>());\n        LogicQueueMappingItem logicQueueMappingItem = mock(LogicQueueMappingItem.class);\n        when(logicQueueMappingItem.getBname()).thenReturn(\"broker\");\n        mappingDetail.getHostedQueues().put(0, Collections.singletonList(logicQueueMappingItem));\n        ConcurrentMap<String, TopicQueueMappingDetail> topicQueueMappingTable = new ConcurrentHashMap<>();\n        topicQueueMappingTable.put(defaultBroker, mappingDetail);\n        when(topicQueueMappingManager.getTopicQueueMappingTable()).thenReturn(topicQueueMappingTable);\n        TopicRouteData topicRouteData = new TopicRouteData();\n        when(brokerOuterAPI.getTopicRouteInfoFromNameServer(any(), anyLong())).thenReturn(topicRouteData);\n        topicQueueMappingCleanService.cleanItemListMoreThanSecondGen();\n        verify(brokerOuterAPI, times(1)).getTopicRouteInfoFromNameServer(any(), anyLong());\n    }\n    \n    @Test\n    public void testCleanItemListMoreThanSecondGenNoChange() throws Exception {\n        when(messageStoreConfig.getDeleteWhen()).thenReturn(\"04\");\n        topicQueueMappingCleanService.cleanItemListMoreThanSecondGen();\n        verify(brokerOuterAPI, never()).getTopicRouteInfoFromNameServer(anyString(), anyLong());\n        verify(rpcClient, never()).invoke(any(RpcRequest.class), anyLong());\n    }\n    \n    @Test\n    public void testCleanItemListMoreThanSecondGenException() throws Exception {\n        when(brokerConfig.getBrokerName()).thenReturn(defaultBroker);\n        when(messageStoreConfig.getDeleteWhen()).thenReturn(deleteWhen);\n        TopicQueueMappingDetail mappingDetail = new TopicQueueMappingDetail(defaultTopic, 1, defaultBroker, 1);\n        mappingDetail.setHostedQueues(new ConcurrentHashMap<>());\n        LogicQueueMappingItem logicQueueMappingItem = mock(LogicQueueMappingItem.class);\n        when(logicQueueMappingItem.getBname()).thenReturn(\"broker\");\n        mappingDetail.getHostedQueues().put(0, Collections.singletonList(logicQueueMappingItem));\n        ConcurrentMap<String, TopicQueueMappingDetail> topicQueueMappingTable = new ConcurrentHashMap<>();\n        topicQueueMappingTable.put(defaultBroker, mappingDetail);\n        when(topicQueueMappingManager.getTopicQueueMappingTable()).thenReturn(topicQueueMappingTable);\n        when(brokerOuterAPI.getTopicRouteInfoFromNameServer(any(), anyLong())).thenThrow(new RemotingException(\"Test exception\"));\n        topicQueueMappingCleanService.cleanItemListMoreThanSecondGen();\n        verify(brokerOuterAPI, times(1)).getTopicRouteInfoFromNameServer(any(), anyLong());\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/topic/TopicQueueMappingManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.topic;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.alibaba.fastjson2.JSONWriter;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.remoting.protocol.body.TopicQueueMappingSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingDetail;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingUtils;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicRemappingDetailWrapper;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.nio.file.Files;\nimport java.nio.file.Paths;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.UUID;\nimport java.util.concurrent.ConcurrentHashMap;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class TopicQueueMappingManagerTest {\n    @Mock\n    private BrokerController brokerController;\n    private static final String BROKER1_NAME = \"broker1\";\n\n    @Before\n    public void before() {\n        BrokerConfig brokerConfig = new BrokerConfig();\n        brokerConfig.setBrokerName(BROKER1_NAME);\n        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);\n\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setStorePathRootDir(System.getProperty(\"java.io.tmpdir\"));\n        messageStoreConfig.setDeleteWhen(\"01;02;03;04;05;06;07;08;09;10;11;12;13;14;15;16;17;18;19;20;21;22;23;00\");\n        when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n    }\n\n\n    private void delete(TopicQueueMappingManager topicQueueMappingManager) throws Exception {\n        if (topicQueueMappingManager == null) {\n            return;\n        }\n        Files.deleteIfExists(Paths.get(topicQueueMappingManager.configFilePath()));\n        Files.deleteIfExists(Paths.get(topicQueueMappingManager.configFilePath() + \".bak\"));\n\n\n    }\n\n    @Test\n    public void testEncodeDecode() throws Exception {\n        Map<String, TopicQueueMappingDetail> mappingDetailMap = new HashMap<>();\n        TopicQueueMappingManager topicQueueMappingManager = null;\n        Set<String> brokers = new HashSet<>();\n        brokers.add(BROKER1_NAME);\n        {\n            for (int i = 0; i < 10; i++) {\n                String topic = UUID.randomUUID().toString();\n                int queueNum = 10;\n                TopicRemappingDetailWrapper topicRemappingDetailWrapper  = TopicQueueMappingUtils.createTopicConfigMapping(topic, queueNum, brokers, new HashMap<>());\n                assertEquals(1, topicRemappingDetailWrapper.getBrokerConfigMap().size());\n                TopicQueueMappingDetail topicQueueMappingDetail  = topicRemappingDetailWrapper.getBrokerConfigMap().values().iterator().next().getMappingDetail();\n                assertEquals(queueNum, topicQueueMappingDetail.getHostedQueues().size());\n                mappingDetailMap.put(topic, topicQueueMappingDetail);\n            }\n        }\n\n        {\n            topicQueueMappingManager = new TopicQueueMappingManager(brokerController);\n            Assert.assertTrue(topicQueueMappingManager.load());\n            assertEquals(0, topicQueueMappingManager.getTopicQueueMappingTable().size());\n            for (TopicQueueMappingDetail mappingDetail : mappingDetailMap.values()) {\n                for (int i = 0; i < 10; i++) {\n                    topicQueueMappingManager.updateTopicQueueMapping(mappingDetail, false, false, true);\n                }\n            }\n            topicQueueMappingManager.persist();\n        }\n\n        {\n            topicQueueMappingManager = new TopicQueueMappingManager(brokerController);\n            Assert.assertTrue(topicQueueMappingManager.load());\n            assertEquals(mappingDetailMap.size(), topicQueueMappingManager.getTopicQueueMappingTable().size());\n            for (TopicQueueMappingDetail topicQueueMappingDetail: topicQueueMappingManager.getTopicQueueMappingTable().values()) {\n                assertEquals(topicQueueMappingDetail, mappingDetailMap.get(topicQueueMappingDetail.getTopic()));\n            }\n        }\n        delete(topicQueueMappingManager);\n    }\n\n    @Test\n    public void testEncodePretty() {\n        TopicQueueMappingManager topicQueueMappingManager = new TopicQueueMappingManager(null);\n        TopicQueueMappingDetail detail = new TopicQueueMappingDetail();\n        detail.setTopic(\"testTopic\");\n        detail.setBname(\"testBroker\");\n\n        topicQueueMappingManager.getTopicQueueMappingTable().put(\"testTopic\", detail);\n        topicQueueMappingManager.getDataVersion().nextVersion();\n\n        String actual = topicQueueMappingManager.encode(true);\n        TopicQueueMappingSerializeWrapper expectedWrapper = new TopicQueueMappingSerializeWrapper();\n        expectedWrapper.setTopicQueueMappingInfoMap(new ConcurrentHashMap<>(topicQueueMappingManager.getTopicQueueMappingTable()));\n        expectedWrapper.setDataVersion(topicQueueMappingManager.getDataVersion());\n        String expected = JSON.toJSONString(expectedWrapper, JSONWriter.Feature.PrettyFormat);\n\n        assertEquals(expected, actual);\n    }\n\n    @Test\n    public void testEncodeNonPretty() {\n        TopicQueueMappingManager topicQueueMappingManager = new TopicQueueMappingManager(null);\n        TopicQueueMappingDetail detail = new TopicQueueMappingDetail();\n        detail.setTopic(\"testTopic\");\n        detail.setBname(\"testBroker\");\n\n        topicQueueMappingManager.getTopicQueueMappingTable().put(\"testTopic\", detail);\n        topicQueueMappingManager.getDataVersion().nextVersion();\n\n        String actual = topicQueueMappingManager.encode(false);\n        TopicQueueMappingSerializeWrapper expectedWrapper = new TopicQueueMappingSerializeWrapper();\n        expectedWrapper.setTopicQueueMappingInfoMap(new ConcurrentHashMap<>(topicQueueMappingManager.getTopicQueueMappingTable()));\n        expectedWrapper.setDataVersion(topicQueueMappingManager.getDataVersion());\n        String expected = JSON.toJSONString(expectedWrapper);\n\n        assertEquals(expected, actual);\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/DefaultTransactionalMessageCheckListenerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.transaction.queue;\n\nimport java.net.InetSocketAddress;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class DefaultTransactionalMessageCheckListenerTest {\n\n    private DefaultTransactionalMessageCheckListener listener;\n    @Mock\n    private MessageStore messageStore;\n\n    @Spy\n    private BrokerController brokerController = new BrokerController(new BrokerConfig(),\n        new NettyServerConfig(),\n        new NettyClientConfig(), new MessageStoreConfig());\n\n    @Before\n    public void init() throws Exception {\n        listener = new DefaultTransactionalMessageCheckListener();\n        listener.setBrokerController(brokerController);\n        brokerController.setMessageStore(messageStore);\n\n    }\n\n    @After\n    public void destroy() {\n//        brokerController.shutdown();\n    }\n\n    @Test\n    public void testResolveHalfMsg() {\n        listener.resolveHalfMsg(createMessageExt());\n    }\n\n    @Test\n    public void testSendCheckMessage() throws Exception {\n        MessageExt messageExt = createMessageExt();\n        listener.sendCheckMessage(messageExt);\n    }\n\n    @Test\n    public void sendCheckMessage() {\n        listener.resolveDiscardMsg(createMessageExt());\n    }\n\n    private MessageExtBrokerInner createMessageExt() {\n        MessageExtBrokerInner inner = new MessageExtBrokerInner();\n        MessageAccessor.putProperty(inner, MessageConst.PROPERTY_REAL_QUEUE_ID, \"1\");\n        MessageAccessor.putProperty(inner, MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, \"1234255\");\n        MessageAccessor.putProperty(inner, MessageConst.PROPERTY_REAL_TOPIC, \"realTopic\");\n        inner.setTransactionId(inner.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));\n        inner.setBody(\"check\".getBytes());\n        inner.setMsgId(\"12344567890\");\n        inner.setQueueId(0);\n        return inner;\n    }\n\n    @Test\n    public void testResolveDiscardMsg() {\n        MessageExt messageExt = new MessageExt();\n        messageExt.setTopic(TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC);\n        messageExt.setQueueId(0);\n        messageExt.setBody(\"test resolve discard msg\".getBytes());\n        messageExt.setStoreHost(new InetSocketAddress(\"127.0.0.1\", 10911));\n        messageExt.setBornHost(new InetSocketAddress(\"127.0.0.1\", 54270));\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_REAL_TOPIC, \"test_topic\");\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_PRODUCER_GROUP, \"PID_TEST_DISCARD_MSG\");\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_TRANSACTION_PREPARED, \"true\");\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES, \"15\");\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_REAL_QUEUE_ID, \"2\");\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_TAGS, \"test_discard_msg\");\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, \"AC14157E4F1C18B4AAC27EB1A0F30000\");\n        listener.resolveDiscardMsg(messageExt);\n    }\n\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionMetricsTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.transaction.queue;\n\nimport org.apache.rocketmq.broker.transaction.TransactionMetrics;\nimport org.apache.rocketmq.broker.transaction.TransactionMetrics.Metric;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.io.File;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.util.Collections;\nimport java.util.UUID;\n\nimport static org.junit.jupiter.api.Assertions.assertFalse;\nimport static org.junit.jupiter.api.Assertions.assertTrue;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.verify;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class TransactionMetricsTest {\n    private TransactionMetrics transactionMetrics;\n    private String configPath;\n    private Path path;\n\n    @Before\n    public void before() throws Exception {\n        configPath = createBaseDir();\n        path = Paths.get(configPath);\n        transactionMetrics = spy(new TransactionMetrics(configPath));\n    }\n\n    @After\n    public void after() throws Exception {\n        deleteFile(configPath);\n        assertFalse(path.toFile().exists());\n    }\n\n    /**\n     * test addAndGet method\n     */\n    @Test\n    public void testAddAndGet() {\n        String topic = \"testAddAndGet\";\n        int value = 10;\n        long result = transactionMetrics.addAndGet(topic, value);\n\n        assert result == value;\n    }\n\n    @Test\n    public void testGetTopicPair() {\n        String topic = \"getTopicPair\";\n        Metric result = transactionMetrics.getTopicPair(topic);\n        assert result != null;\n    }\n\n    @Test\n    public void testGetTransactionCount() {\n        String topicExist = \"topicExist\";\n        String topicNotExist = \"topicNotExist\";\n\n        transactionMetrics.addAndGet(topicExist, 10);\n\n        assert transactionMetrics.getTransactionCount(topicExist) == 10;\n        assert transactionMetrics.getTransactionCount(topicNotExist) == 0;\n    }\n\n\n    /**\n     * test clean metrics\n     */\n    @Test\n    public void testCleanMetrics() {\n        String topic = \"testCleanMetrics\";\n        int value = 10;\n        assert transactionMetrics.addAndGet(topic, value) == value;\n        transactionMetrics.cleanMetrics(Collections.singleton(topic));\n        assert transactionMetrics.getTransactionCount(topic) == 0;\n    }\n\n    @Test\n    public void testPersist() {\n        assertFalse(path.toFile().exists());\n        transactionMetrics.persist();\n        assertTrue(path.toFile().exists());\n        verify(transactionMetrics).persist();\n    }\n\n    private String createBaseDir() {\n        String baseDir = System.getProperty(\"java.io.tmpdir\") + File.separator + \"unitteststore-\" + UUID.randomUUID();\n        final File file = new File(baseDir);\n        if (file.exists()) {\n            System.exit(1);\n        }\n        return baseDir;\n    }\n\n    private void deleteFile(String fileName) {\n        deleteFile(new File(fileName));\n    }\n\n    private void deleteFile(File file) {\n        if (!file.exists()) {\n            return;\n        }\n        if (file.isFile()) {\n            file.delete();\n        } else if (file.isDirectory()) {\n            File[] files = file.listFiles();\n            for (File file1 : files) {\n                deleteFile(file1);\n            }\n            file.delete();\n        }\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.transaction.queue;\n\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.store.AppendMessageResult;\nimport org.apache.rocketmq.store.AppendMessageStatus;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.MessageFilter;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.ArgumentMatchers;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class TransactionalMessageBridgeTest {\n\n    private TransactionalMessageBridge transactionBridge;\n\n    @Spy\n    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(),\n        new NettyClientConfig(), new MessageStoreConfig());\n\n    @Mock\n    private MessageStore messageStore;\n\n    @Before\n    public void init() {\n        brokerController.setMessageStore(messageStore);\n        transactionBridge = new TransactionalMessageBridge(brokerController, messageStore);\n    }\n\n    @Test\n    public void testPutOpMessage() {\n        when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(\n                new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));\n        boolean isSuccess = transactionBridge.writeOp(0, createMessageBrokerInner());\n        assertThat(isSuccess).isTrue();\n    }\n\n    @Test\n    public void testPutHalfMessage() {\n        when(messageStore.putMessage(any(MessageExtBrokerInner.class)))\n                .thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));\n        PutMessageResult result = transactionBridge.putHalfMessage(createMessageBrokerInner());\n        assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK);\n    }\n\n    @Test\n    public void testAsyncPutHalfMessage() throws Exception {\n        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class)))\n                .thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))));\n        CompletableFuture<PutMessageResult> result = transactionBridge.asyncPutHalfMessage(createMessageBrokerInner());\n        assertThat(result.get().getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK);\n    }\n\n    @Test\n    public void testFetchMessageQueues() {\n        Set<MessageQueue> messageQueues = transactionBridge.fetchMessageQueues(TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC);\n        assertThat(messageQueues.size()).isEqualTo(1);\n    }\n\n    @Test\n    public void testFetchConsumeOffset() {\n        MessageQueue mq = new MessageQueue(TransactionalMessageUtil.buildOpTopic(), this.brokerController.getBrokerConfig().getBrokerName(),\n            0);\n        long offset = transactionBridge.fetchConsumeOffset(mq);\n        assertThat(offset).isGreaterThan(-1);\n    }\n\n    @Test\n    public void updateConsumeOffset() {\n        MessageQueue mq = new MessageQueue(TransactionalMessageUtil.buildOpTopic(), this.brokerController.getBrokerConfig().getBrokerName(),\n            0);\n        transactionBridge.updateConsumeOffset(mq, 0);\n    }\n\n    @Test\n    public void testGetHalfMessage() {\n        when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(),  ArgumentMatchers.nullable(MessageFilter.class))).thenReturn(createGetMessageResult(GetMessageStatus.NO_MESSAGE_IN_QUEUE));\n        PullResult result = transactionBridge.getHalfMessage(0, 0, 1);\n        assertThat(result.getPullStatus()).isEqualTo(PullStatus.NO_NEW_MSG);\n    }\n\n    @Test\n    public void testGetOpMessage() {\n        when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(),  ArgumentMatchers.nullable(MessageFilter.class))).thenReturn(createGetMessageResult(GetMessageStatus.NO_MESSAGE_IN_QUEUE));\n        PullResult result = transactionBridge.getOpMessage(0, 0, 1);\n        assertThat(result.getPullStatus()).isEqualTo(PullStatus.NO_NEW_MSG);\n    }\n\n    @Test\n    public void testPutMessageReturnResult() {\n        when(messageStore.putMessage(any(MessageExtBrokerInner.class)))\n                .thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));\n        PutMessageResult result = transactionBridge.putMessageReturnResult(createMessageBrokerInner());\n        assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK);\n    }\n\n    @Test\n    public void testPutMessage() {\n        when(messageStore.putMessage(any(MessageExtBrokerInner.class)))\n                .thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));\n        Boolean success = transactionBridge.putMessage(createMessageBrokerInner());\n        assertThat(success).isEqualTo(true);\n    }\n\n    @Test\n    public void testRenewImmunityHalfMessageInner() {\n        MessageExt messageExt = createMessageBrokerInner();\n        final String offset = \"123456789\";\n        MessageExtBrokerInner msgInner = transactionBridge.renewImmunityHalfMessageInner(messageExt);\n        MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET,offset);\n        assertThat(msgInner).isNotNull();\n        Map<String,String> properties = msgInner.getProperties();\n        assertThat(properties).isNotNull();\n        String resOffset = properties.get(MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET);\n        assertThat(resOffset).isEqualTo(offset);\n    }\n\n\n    @Test\n    public void testRenewHalfMessageInner() {\n        MessageExt messageExt = new MessageExt();\n        long bornTimeStamp = messageExt.getBornTimestamp();\n        MessageExt messageExtRes = transactionBridge.renewHalfMessageInner(messageExt);\n        assertThat(messageExtRes.getBornTimestamp()).isEqualTo(bornTimeStamp);\n    }\n\n    @Test\n    public void testLookMessageByOffset() {\n        when(messageStore.lookMessageByOffset(anyLong())).thenReturn(new MessageExt());\n        MessageExt messageExt = transactionBridge.lookMessageByOffset(123);\n        assertThat(messageExt).isNotNull();\n    }\n\n    @Test\n    public void testGetHalfMessageStatusFound() {\n        when(messageStore\n                .getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class)))\n                .thenReturn(createGetMessageResult(GetMessageStatus.FOUND));\n        PullResult result = transactionBridge.getHalfMessage(0, 0, 1);\n        assertThat(result.getPullStatus()).isEqualTo(PullStatus.FOUND);\n    }\n\n    @Test\n    public void testGetHalfMessageNull() {\n        when(messageStore\n                .getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class)))\n                .thenReturn(null);\n        PullResult result = transactionBridge.getHalfMessage(0, 0, 1);\n        assertThat(result).isNull();\n    }\n\n    private GetMessageResult createGetMessageResult(GetMessageStatus status) {\n        GetMessageResult getMessageResult = new GetMessageResult();\n        getMessageResult.setStatus(status);\n        getMessageResult.setMinOffset(100);\n        getMessageResult.setMaxOffset(1024);\n        getMessageResult.setNextBeginOffset(516);\n        return getMessageResult;\n    }\n\n    private MessageExtBrokerInner createMessageBrokerInner() {\n        MessageExtBrokerInner inner = new MessageExtBrokerInner();\n        inner.setTransactionId(\"12342123444\");\n        inner.setBornTimestamp(System.currentTimeMillis());\n        inner.setBody(\"prepare\".getBytes());\n        inner.setMsgId(\"123456-123\");\n        inner.setQueueId(0);\n        inner.setTopic(\"hello\");\n        return inner;\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImplTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.transaction.queue;\n\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener;\nimport org.apache.rocketmq.broker.transaction.OperationResult;\nimport org.apache.rocketmq.broker.transaction.TransactionalMessageService;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;\nimport org.apache.rocketmq.store.AppendMessageResult;\nimport org.apache.rocketmq.store.AppendMessageStatus;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.invocation.InvocationOnMock;\nimport org.mockito.junit.MockitoJUnitRunner;\nimport org.mockito.stubbing.Answer;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.Mockito.doAnswer;\nimport static org.mockito.Mockito.timeout;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class TransactionalMessageServiceImplTest {\n\n    private TransactionalMessageService queueTransactionMsgService;\n\n    @Mock\n    private TransactionalMessageBridge bridge;\n\n    @Spy\n    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(),\n        new NettyClientConfig(), new MessageStoreConfig(), null);\n\n    @Mock\n    private AbstractTransactionalMessageCheckListener listener;\n\n    @Before\n    public void init() {\n        when(bridge.getBrokerController()).thenReturn(brokerController);\n        listener.setBrokerController(brokerController);\n        queueTransactionMsgService = new TransactionalMessageServiceImpl(bridge);\n    }\n\n    @Test\n    public void testPrepareMessage() {\n        MessageExtBrokerInner inner = createMessageBrokerInner();\n        when(bridge.putHalfMessage(any(MessageExtBrokerInner.class)))\n                .thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));\n        PutMessageResult result = queueTransactionMsgService.prepareMessage(inner);\n        assert result.isOk();\n    }\n\n    @Test\n    public void testCommitMessage() {\n        when(bridge.lookMessageByOffset(anyLong())).thenReturn(createMessageBrokerInner());\n        OperationResult result = queueTransactionMsgService.commitMessage(createEndTransactionRequestHeader(MessageSysFlag.TRANSACTION_COMMIT_TYPE));\n        assertThat(result.getResponseCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testRollbackMessage() {\n        when(bridge.lookMessageByOffset(anyLong())).thenReturn(createMessageBrokerInner());\n        OperationResult result = queueTransactionMsgService.commitMessage(createEndTransactionRequestHeader(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE));\n        assertThat(result.getResponseCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testCheck_withDiscard() {\n        when(bridge.fetchMessageQueues(TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC)).thenReturn(createMessageQueueSet(TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC));\n        when(bridge.getHalfMessage(0, 0, 1)).thenReturn(createDiscardPullResult(TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC, 5, \"hellp\", 1));\n        when(bridge.getHalfMessage(0, 1, 1)).thenReturn(createPullResult(TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC, 6, \"hellp\", 0));\n        when(bridge.getOpMessage(anyInt(), anyLong(), anyInt())).thenReturn(createOpPulResult(TopicValidator.RMQ_SYS_TRANS_OP_HALF_TOPIC, 1, \"10\", 1));\n        when(bridge.getBrokerController()).thenReturn(this.brokerController);\n        long timeOut = this.brokerController.getBrokerConfig().getTransactionTimeOut();\n        int checkMax = this.brokerController.getBrokerConfig().getTransactionCheckMax();\n        final AtomicInteger checkMessage = new AtomicInteger(0);\n        doAnswer(new Answer() {\n            @Override\n            public Object answer(InvocationOnMock invocation) {\n                checkMessage.addAndGet(1);\n                return null;\n            }\n        }).when(listener).resolveDiscardMsg(any(MessageExt.class));\n        queueTransactionMsgService.check(timeOut, checkMax, listener);\n        assertThat(checkMessage.get()).isEqualTo(1);\n    }\n\n    @Test\n    public void testCheck_withCheck() {\n        when(bridge.fetchMessageQueues(TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC)).thenReturn(createMessageQueueSet(TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC));\n        when(bridge.getHalfMessage(0, 0, 1)).thenReturn(createPullResult(TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC, 5, \"hello\", 1));\n        when(bridge.getHalfMessage(0, 1, 1)).thenReturn(createPullResult(TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC, 6, \"hellp\", 0));\n        when(bridge.getOpMessage(anyInt(), anyLong(), anyInt())).thenReturn(createPullResult(TopicValidator.RMQ_SYS_TRANS_OP_HALF_TOPIC, 1, \"5\", 0));\n        when(bridge.getBrokerController()).thenReturn(this.brokerController);\n        when(bridge.renewHalfMessageInner(any(MessageExtBrokerInner.class))).thenReturn(createMessageBrokerInner());\n        when(bridge.putMessageReturnResult(any(MessageExtBrokerInner.class)))\n                .thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));\n        long timeOut = this.brokerController.getBrokerConfig().getTransactionTimeOut();\n        final int checkMax = this.brokerController.getBrokerConfig().getTransactionCheckMax();\n        final AtomicInteger checkMessage = new AtomicInteger(0);\n        doAnswer(new Answer() {\n            @Override\n            public Object answer(InvocationOnMock invocation) {\n                checkMessage.addAndGet(1);\n                return checkMessage;\n            }\n        }).when(listener).resolveHalfMsg(any(MessageExt.class));\n        queueTransactionMsgService.check(timeOut, checkMax, listener);\n        assertThat(checkMessage.get()).isEqualTo(1);\n    }\n\n    @Test\n    public void testDeletePrepareMessage_queueFull() throws InterruptedException {\n        ((TransactionalMessageServiceImpl)queueTransactionMsgService).getDeleteContext().put(0, new MessageQueueOpContext(0, 1));\n        boolean res = queueTransactionMsgService.deletePrepareMessage(createMessageBrokerInner());\n        assertThat(res).isTrue();\n        when(bridge.writeOp(any(Integer.class), any(Message.class))).thenReturn(false);\n        res = queueTransactionMsgService.deletePrepareMessage(createMessageBrokerInner());\n        assertThat(res).isFalse();\n    }\n\n    @Test\n    public void testDeletePrepareMessage_maxSize() throws InterruptedException {\n        brokerController.getBrokerConfig().setTransactionOpMsgMaxSize(1);\n        brokerController.getBrokerConfig().setTransactionOpBatchInterval(3000);\n        queueTransactionMsgService.open();\n        boolean res = queueTransactionMsgService.deletePrepareMessage(createMessageBrokerInner(1000, \"test\", \"testHello\"));\n        assertThat(res).isTrue();\n        verify(bridge, timeout(50)).writeOp(any(Integer.class), any(Message.class));\n        queueTransactionMsgService.close();\n    }\n\n    @Test\n    public void testOpen() {\n        boolean isOpen = queueTransactionMsgService.open();\n        assertThat(isOpen).isTrue();\n    }\n\n    private PullResult createDiscardPullResult(String topic, long queueOffset, String body, int size) {\n        PullResult result = createPullResult(topic, queueOffset, body, size);\n        List<MessageExt> msgs = result.getMsgFoundList();\n        for (MessageExt msg : msgs) {\n            msg.putUserProperty(MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES, \"100000\");\n        }\n        return result;\n    }\n\n    private PullResult createPullResult(String topic, long queueOffset, String body, int size) {\n        PullResult result = null;\n        if (0 == size) {\n            result = new PullResult(PullStatus.NO_NEW_MSG, 1, 0, 1,\n                null);\n        } else {\n            result = new PullResult(PullStatus.FOUND, 1, 0, 1,\n                getMessageList(queueOffset, topic, body, 1));\n            return result;\n        }\n        return result;\n    }\n\n    private PullResult createOpPulResult(String topic, long queueOffset, String body, int size) {\n        PullResult result = createPullResult(topic, queueOffset, body, size);\n        List<MessageExt> msgs = result.getMsgFoundList();\n        for (MessageExt msg : msgs) {\n            msg.setTags(TransactionalMessageUtil.REMOVE_TAG);\n        }\n        return result;\n    }\n\n    private PullResult createImmunityPulResult(String topic, long queueOffset, String body, int size) {\n        PullResult result = createPullResult(topic, queueOffset, body, size);\n        List<MessageExt> msgs = result.getMsgFoundList();\n        for (MessageExt msg : msgs) {\n            msg.putUserProperty(MessageConst.PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS, \"0\");\n        }\n        return result;\n    }\n\n    private List<MessageExt> getMessageList(long queueOffset, String topic, String body, int size) {\n        List<MessageExt> msgs = new ArrayList<>();\n        for (int i = 0; i < size; i++) {\n            MessageExt messageExt = createMessageBrokerInner(queueOffset, topic, body);\n            msgs.add(messageExt);\n        }\n        return msgs;\n    }\n\n    private Set<MessageQueue> createMessageQueueSet(String topic) {\n        Set<MessageQueue> messageQueues = new HashSet<>();\n        MessageQueue messageQueue = new MessageQueue(topic, \"DefaultCluster\", 0);\n        messageQueues.add(messageQueue);\n        return messageQueues;\n    }\n\n    private EndTransactionRequestHeader createEndTransactionRequestHeader(int status) {\n        EndTransactionRequestHeader header = new EndTransactionRequestHeader();\n        header.setTopic(\"topic\");\n        header.setCommitLogOffset(123456789L);\n        header.setCommitOrRollback(status);\n        header.setMsgId(\"12345678\");\n        header.setTransactionId(\"123\");\n        header.setProducerGroup(\"testTransactionGroup\");\n        header.setTranStateTableOffset(1234L);\n        return header;\n    }\n\n    private MessageExtBrokerInner createMessageBrokerInner(long queueOffset, String topic, String body) {\n        MessageExtBrokerInner inner = new MessageExtBrokerInner();\n        inner.setBornTimestamp(System.currentTimeMillis() - 80000);\n        inner.setTransactionId(\"123456123\");\n        inner.setTopic(topic);\n        inner.setQueueOffset(queueOffset);\n        inner.setBody(body.getBytes());\n        inner.setMsgId(\"123456123\");\n        inner.setQueueId(0);\n        inner.setTopic(\"hello\");\n        return inner;\n    }\n\n    private MessageExtBrokerInner createMessageBrokerInner() {\n        return createMessageBrokerInner(1, \"testTopic\", \"hello world\");\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageUtilTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.transaction.queue;\n\n\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\n\npublic class TransactionalMessageUtilTest {\n\n    @Test\n    public void testBuildTransactionalMessageFromHalfMessage() {\n        MessageExt halfMessage = new MessageExt();\n        halfMessage.setTopic(TransactionalMessageUtil.buildHalfTopic());\n        MessageAccessor.putProperty(halfMessage, MessageConst.PROPERTY_REAL_TOPIC, \"real-topic\");\n        halfMessage.setMsgId(\"msgId\");\n        halfMessage.setTransactionId(\"tranId\");\n        MessageAccessor.putProperty(halfMessage, MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, \"tranId\");\n        MessageAccessor.putProperty(halfMessage, MessageConst.PROPERTY_PRODUCER_GROUP, \"trans-producer-grp\");\n\n        MessageExtBrokerInner msgExtInner = TransactionalMessageUtil.buildTransactionalMessageFromHalfMessage(halfMessage);\n\n\n        assertEquals(\"real-topic\", msgExtInner.getTopic());\n        assertEquals(\"true\", msgExtInner.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED));\n        assertEquals(msgExtInner.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX),\n            halfMessage.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));\n        assertEquals(msgExtInner.getMsgId(), halfMessage.getMsgId());\n        assertTrue(MessageSysFlag.check(msgExtInner.getSysFlag(), MessageSysFlag.TRANSACTION_PREPARED_TYPE));\n        assertEquals(msgExtInner.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP), halfMessage.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP));\n    }\n\n    @Test\n    public void testGetImmunityTime() {\n        long transactionTimeout = 6 * 1000;\n\n        String checkImmunityTimeStr = \"1\";\n        long immunityTime = TransactionalMessageUtil.getImmunityTime(checkImmunityTimeStr, transactionTimeout);\n        Assert.assertEquals(6 * 1000, immunityTime);\n\n        checkImmunityTimeStr = \"5\";\n        immunityTime = TransactionalMessageUtil.getImmunityTime(checkImmunityTimeStr, transactionTimeout);\n        Assert.assertEquals(6 * 1000, immunityTime);\n\n        checkImmunityTimeStr = \"7\";\n        immunityTime = TransactionalMessageUtil.getImmunityTime(checkImmunityTimeStr, transactionTimeout);\n        Assert.assertEquals(7 * 1000, immunityTime);\n\n\n        checkImmunityTimeStr = null;\n        immunityTime = TransactionalMessageUtil.getImmunityTime(checkImmunityTimeStr, transactionTimeout);\n        Assert.assertEquals(6 * 1000, immunityTime);\n\n        checkImmunityTimeStr = \"-1\";\n        immunityTime = TransactionalMessageUtil.getImmunityTime(checkImmunityTimeStr, transactionTimeout);\n        Assert.assertEquals(6 * 1000, immunityTime);\n\n        checkImmunityTimeStr = \"60\";\n        immunityTime = TransactionalMessageUtil.getImmunityTime(checkImmunityTimeStr, transactionTimeout);\n        Assert.assertEquals(60 * 1000, immunityTime);\n\n        checkImmunityTimeStr = \"100\";\n        immunityTime = TransactionalMessageUtil.getImmunityTime(checkImmunityTimeStr, transactionTimeout);\n        Assert.assertEquals(100 * 1000, immunityTime);\n\n\n        checkImmunityTimeStr = \"100.5\";\n        immunityTime = TransactionalMessageUtil.getImmunityTime(checkImmunityTimeStr, transactionTimeout);\n        Assert.assertEquals(6 * 1000, immunityTime);\n    }\n}"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/util/HookUtilsTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.util;\n\nimport java.util.Objects;\nimport org.apache.commons.lang3.RandomStringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.RunningFlags;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Assert;\nimport org.junit.Test;\nimport org.mockito.Mockito;\n\npublic class HookUtilsTest {\n\n    @Test\n    public void testCheckBeforePutMessage() {\n        BrokerController brokerController = Mockito.mock(BrokerController.class);\n        MessageStore messageStore = Mockito.mock(MessageStore.class);\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        RunningFlags runningFlags = Mockito.mock(RunningFlags.class);\n\n        Mockito.when(brokerController.getMessageStore()).thenReturn(messageStore);\n        Mockito.when(brokerController.getMessageStore().isShutdown()).thenReturn(false);\n        Mockito.when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);\n        Mockito.when(messageStore.getRunningFlags()).thenReturn(runningFlags);\n        Mockito.when(messageStore.getRunningFlags().isWriteable()).thenReturn(true);\n\n        MessageExt messageExt = new MessageExt();\n        messageExt.setTopic(RandomStringUtils.randomAlphabetic(Byte.MAX_VALUE).toUpperCase());\n        messageExt.setBody(RandomStringUtils.randomAlphabetic(Byte.MAX_VALUE).toUpperCase().getBytes());\n        Assert.assertNull(HookUtils.checkBeforePutMessage(brokerController, messageExt));\n\n        messageExt.setTopic(RandomStringUtils.randomAlphabetic(Byte.MAX_VALUE + 1).toUpperCase());\n        Assert.assertEquals(PutMessageStatus.MESSAGE_ILLEGAL, Objects.requireNonNull(\n            HookUtils.checkBeforePutMessage(brokerController, messageExt)).getPutMessageStatus());\n\n        messageExt.setTopic(MixAll.RETRY_GROUP_TOPIC_PREFIX +\n            RandomStringUtils.randomAlphabetic(Byte.MAX_VALUE + 1).toUpperCase());\n        Assert.assertNull(HookUtils.checkBeforePutMessage(brokerController, messageExt));\n\n        messageExt.setTopic(MixAll.RETRY_GROUP_TOPIC_PREFIX +\n            RandomStringUtils.randomAlphabetic(255 - MixAll.RETRY_GROUP_TOPIC_PREFIX.length()).toUpperCase());\n        Assert.assertNull(HookUtils.checkBeforePutMessage(brokerController, messageExt));\n\n        messageExt.setTopic(MixAll.RETRY_GROUP_TOPIC_PREFIX +\n            RandomStringUtils.randomAlphabetic(256 - MixAll.RETRY_GROUP_TOPIC_PREFIX.length()).toUpperCase());\n        Assert.assertEquals(PutMessageStatus.MESSAGE_ILLEGAL, Objects.requireNonNull(\n            HookUtils.checkBeforePutMessage(brokerController, messageExt)).getPutMessageStatus());\n    }\n}"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/util/LogTransactionalMessageCheckListener.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.util;\n\nimport org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener;\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic class LogTransactionalMessageCheckListener extends AbstractTransactionalMessageCheckListener {\n\n    @Override\n    public void resolveDiscardMsg(MessageExt msgExt) {\n\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/util/ServiceProviderTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.broker.util;\n\nimport org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener;\nimport org.apache.rocketmq.broker.transaction.TransactionalMessageService;\nimport org.apache.rocketmq.common.utils.ServiceProvider;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class ServiceProviderTest {\n\n    @Test\n    public void loadTransactionMsgServiceTest() {\n        TransactionalMessageService transactionService = ServiceProvider.loadClass(TransactionalMessageService.class);\n        assertThat(transactionService).isNotNull();\n    }\n\n    @Test\n    public void loadAbstractTransactionListenerTest() {\n        AbstractTransactionalMessageCheckListener listener = ServiceProvider.loadClass(\n                AbstractTransactionalMessageCheckListener.class);\n        assertThat(listener).isNotNull();\n    }\n}\n"
  },
  {
    "path": "broker/src/test/java/org/apache/rocketmq/broker/util/TransactionalMessageServiceImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.broker.util;\n\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener;\nimport org.apache.rocketmq.broker.transaction.OperationResult;\nimport org.apache.rocketmq.broker.transaction.TransactionMetrics;\nimport org.apache.rocketmq.broker.transaction.TransactionalMessageService;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;\nimport org.apache.rocketmq.store.PutMessageResult;\n\npublic class TransactionalMessageServiceImpl implements TransactionalMessageService {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME);\n\n    @Override\n    public PutMessageResult prepareMessage(MessageExtBrokerInner messageInner) {\n        return null;\n    }\n\n    @Override\n    public CompletableFuture<PutMessageResult> asyncPrepareMessage(MessageExtBrokerInner messageInner) {\n        return null;\n    }\n\n    @Override\n    public boolean deletePrepareMessage(MessageExt messageExt) {\n        return false;\n    }\n\n    @Override\n    public OperationResult commitMessage(EndTransactionRequestHeader requestHeader) {\n        return null;\n    }\n\n    @Override\n    public OperationResult rollbackMessage(EndTransactionRequestHeader requestHeader) {\n        return null;\n    }\n\n    @Override\n    public void check(long transactionTimeout, int transactionCheckMax, AbstractTransactionalMessageCheckListener listener) {\n        log.warn(\"check check!\");\n    }\n\n    @Override\n    public boolean open() {\n        return true;\n    }\n\n    @Override\n    public void close() {\n\n    }\n\n    @Override\n    public TransactionMetrics getTransactionMetrics() {\n        return null;\n    }\n\n    @Override\n    public void setTransactionMetrics(TransactionMetrics transactionMetrics) {\n\n    }\n}\n"
  },
  {
    "path": "broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener",
    "content": "org.apache.rocketmq.broker.util.LogTransactionalMessageCheckListener"
  },
  {
    "path": "broker/src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.TransactionalMessageService",
    "content": "org.apache.rocketmq.broker.util.TransactionalMessageServiceImpl"
  },
  {
    "path": "broker/src/test/resources/rmq.logback-test.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n<configuration>\n\n    <appender name=\"CONSOLE\" class=\"ch.qos.logback.core.ConsoleAppender\">\n        <layout class=\"ch.qos.logback.classic.PatternLayout\">\n            <pattern>${CONSOLE_LOG_PATTERN}</pattern>\n        </layout>\n    </appender>\n\n    <logger name=\"org.apache.rocketmq\" level=\"error\" additivity=\"false\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </logger>\n\n    <property name=\"CONSOLE_LOG_PATTERN\"\n              value=\"%d{yyyy-MM-dd HH:mm:ss.SSS,GMT+8} ${LOG_LEVEL_PATTERN:-%5p} [%20.20logger{39}] [%30.30t] %m%n\"/>\n\n    <root level=\"info\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </root>\n\n</configuration>"
  },
  {
    "path": "client/BUILD.bazel",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\nload(\"//bazel:GenTestRules.bzl\", \"GenTestRules\")\n\njava_library(\n    name = \"client\",\n    srcs = glob([\"src/main/java/**/*.java\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \"//common\",\n        \"//remoting\",\n        \"@maven//:org_apache_commons_commons_lang3\",\n        \"@maven//:commons_validator_commons_validator\",\n        \"@maven//:com_github_luben_zstd_jni\",\n        \"@maven//:org_lz4_lz4_java\",\n        \"@maven//:com_alibaba_fastjson2_fastjson2\",\n        \"@maven//:io_netty_netty_all\",\n        \"@maven//:io_opentracing_opentracing_api\",\n        \"@maven//:commons_collections_commons_collections\",\n        \"@maven//:io_github_aliyunmq_rocketmq_slf4j_api\",\n        \"@maven//:io_github_aliyunmq_rocketmq_logback_classic\",\n        \"@maven//:com_google_guava_guava\",\n        \"@maven//:commons_codec_commons_codec\",\n        \"@maven//:org_yaml_snakeyaml\",\n    ],\n)\n\njava_library(\n    name = \"tests\",\n    srcs = glob([\"src/test/java/**/*.java\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \":client\",\n        \"//remoting\",\n        \"//common\",\n        \"//:test_deps\",\n        \"@maven//:org_apache_commons_commons_lang3\", \n        \"@maven//:io_netty_netty_all\",   \n        \"@maven//:io_opentracing_opentracing_api\",        \n        \"@maven//:io_opentracing_opentracing_mock\",   \n        \"@maven//:org_awaitility_awaitility\",\n        \"@maven//:org_mockito_mockito_junit_jupiter\",\n        \"@maven//:com_alibaba_fastjson2_fastjson2\",\n    ],\n    resources = glob([\"src/test/resources/certs/*.pem\"]) + glob([\"src/test/resources/certs/*.key\"]) + glob([\"src/test/resources/**/*.yml\"])\n)\n\nGenTestRules(\n    name = \"GeneratedTestRules\",\n    test_files = glob([\"src/test/java/**/*Test.java\"]),\n    deps = [\n        \":tests\",\n    ],\n    exclude_tests = [\n        \"src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest\",\n        \"src/test/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPITest\",\n    ],\n)\n"
  },
  {
    "path": "client/pom.xml",
    "content": "<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n    <parent>\n        <groupId>org.apache.rocketmq</groupId>\n        <artifactId>rocketmq-all</artifactId>\n        <version>${revision}</version>\n    </parent>\n\n    <modelVersion>4.0.0</modelVersion>\n    <packaging>jar</packaging>\n    <artifactId>rocketmq-client</artifactId>\n    <name>rocketmq-client ${project.version}</name>\n\n    <properties>\n        <project.root>${basedir}/..</project.root>\n    </properties>\n\n    <dependencies>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-remoting</artifactId>\n            <exclusions>\n                <exclusion>\n                    <groupId>io.netty</groupId>\n                    <artifactId>netty-tcnative</artifactId>\n                </exclusion>\n            </exclusions>\n        </dependency>\n        <dependency>\n            <groupId>org.apache.commons</groupId>\n            <artifactId>commons-lang3</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.opentracing</groupId>\n            <artifactId>opentracing-api</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.opentracing</groupId>\n            <artifactId>opentracing-mock</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.github.aliyunmq</groupId>\n            <artifactId>rocketmq-slf4j-api</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.github.aliyunmq</groupId>\n            <artifactId>rocketmq-logback-classic</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.yaml</groupId>\n            <artifactId>snakeyaml</artifactId>\n        </dependency>\n    </dependencies>\n</project>\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/acl/common/AclClientRPCHook.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.acl.common;\n\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\nimport java.util.Map;\nimport java.util.SortedMap;\nimport java.util.TreeMap;\n\npublic class AclClientRPCHook implements RPCHook {\n    private final SessionCredentials sessionCredentials;\n\n    public AclClientRPCHook(SessionCredentials sessionCredentials) {\n        this.sessionCredentials = sessionCredentials;\n    }\n\n    @Override\n    public void doBeforeRequest(String remoteAddr, RemotingCommand request) {\n        // Add AccessKey and SecurityToken into signature calculating.\n        request.addExtField(SessionCredentials.ACCESS_KEY, sessionCredentials.getAccessKey());\n        // The SecurityToken value is unnecessary,user can choose this one.\n        if (sessionCredentials.getSecurityToken() != null) {\n            request.addExtField(SessionCredentials.SECURITY_TOKEN, sessionCredentials.getSecurityToken());\n        }\n        byte[] total = AclUtils.combineRequestContent(request, parseRequestContent(request));\n        String signature = AclUtils.calSignature(total, sessionCredentials.getSecretKey());\n        request.addExtField(SessionCredentials.SIGNATURE, signature);\n    }\n\n    @Override\n    public void doAfterResponse(String remoteAddr, RemotingCommand request, RemotingCommand response) {\n\n    }\n\n    protected SortedMap<String, String> parseRequestContent(RemotingCommand request) {\n        request.makeCustomHeaderToNet();\n        Map<String, String> extFields = request.getExtFields();\n        // Sort property\n        return new TreeMap<>(extFields);\n    }\n\n    public SessionCredentials getSessionCredentials() {\n        return sessionCredentials;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/acl/common/AclConstants.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.acl.common;\n\npublic class AclConstants {\n\n    public static final String CONFIG_ACCESS_KEY = \"accessKey\";\n\n    public static final String CONFIG_SECRET_KEY = \"secretKey\";\n\n    public static final String PUB = \"PUB\";\n\n    public static final String SUB = \"SUB\";\n\n    public static final String DENY = \"DENY\";\n\n    public static final String PUB_SUB = \"PUB|SUB\";\n\n    public static final String SUB_PUB = \"SUB|PUB\";\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/acl/common/AclException.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.acl.common;\n\npublic class AclException extends RuntimeException {\n    private static final long serialVersionUID = -7256002576788700354L;\n\n    private String status;\n    private int code;\n\n    public AclException(String status, int code) {\n        super();\n        this.status = status;\n        this.code = code;\n    }\n\n    public AclException(String status, int code, String message) {\n        super(message);\n        this.status = status;\n        this.code = code;\n    }\n\n    public AclException(String message) {\n        super(message);\n    }\n\n    public AclException(String message, Throwable throwable) {\n        super(message, throwable);\n    }\n\n    public AclException(String status, int code, String message, Throwable throwable) {\n        super(message, throwable);\n        this.status = status;\n        this.code = code;\n    }\n\n    public String getStatus() {\n        return status;\n    }\n\n    public void setStatus(String status) {\n        this.status = status;\n    }\n\n    public int getCode() {\n        return code;\n    }\n\n    public void setCode(int code) {\n        this.code = code;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/acl/common/AclSigner.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.acl.common;\n\nimport org.apache.commons.codec.binary.Base64;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\nimport javax.crypto.Mac;\nimport javax.crypto.spec.SecretKeySpec;\nimport java.nio.charset.Charset;\nimport java.nio.charset.StandardCharsets;\n\npublic class AclSigner {\n    public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;\n    public static final SigningAlgorithm DEFAULT_ALGORITHM = SigningAlgorithm.HmacSHA1;\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_AUTHORIZE_LOGGER_NAME);\n    private static final int CAL_SIGNATURE_FAILED = 10015;\n    private static final String CAL_SIGNATURE_FAILED_MSG = \"[%s:signature-failed] unable to calculate a request signature. error=%s\";\n\n    public static String calSignature(String data, String key) throws AclException {\n        return calSignature(data, key, DEFAULT_ALGORITHM, DEFAULT_CHARSET);\n    }\n\n    public static String calSignature(String data, String key, SigningAlgorithm algorithm,\n        Charset charset) throws AclException {\n        return signAndBase64Encode(data, key, algorithm, charset);\n    }\n\n    private static String signAndBase64Encode(String data, String key, SigningAlgorithm algorithm, Charset charset)\n        throws AclException {\n        try {\n            byte[] signature = sign(data.getBytes(charset), key.getBytes(charset), algorithm);\n            return new String(Base64.encodeBase64(signature), DEFAULT_CHARSET);\n        } catch (Exception e) {\n            String message = String.format(CAL_SIGNATURE_FAILED_MSG, CAL_SIGNATURE_FAILED, e.getMessage());\n            log.error(message, e);\n            throw new AclException(\"CAL_SIGNATURE_FAILED\", CAL_SIGNATURE_FAILED, message, e);\n        }\n    }\n\n    private static byte[] sign(byte[] data, byte[] key, SigningAlgorithm algorithm) throws AclException {\n        try {\n            Mac mac = Mac.getInstance(algorithm.toString());\n            mac.init(new SecretKeySpec(key, algorithm.toString()));\n            return mac.doFinal(data);\n        } catch (Exception e) {\n            String message = String.format(CAL_SIGNATURE_FAILED_MSG, CAL_SIGNATURE_FAILED, e.getMessage());\n            log.error(message, e);\n            throw new AclException(\"CAL_SIGNATURE_FAILED\", CAL_SIGNATURE_FAILED, message, e);\n        }\n    }\n\n    public static String calSignature(byte[] data, String key) throws AclException {\n        return calSignature(data, key, DEFAULT_ALGORITHM, DEFAULT_CHARSET);\n    }\n\n    public static String calSignature(byte[] data, String key, SigningAlgorithm algorithm,\n        Charset charset) throws AclException {\n        return signAndBase64Encode(data, key, algorithm, charset);\n    }\n\n    private static String signAndBase64Encode(byte[] data, String key, SigningAlgorithm algorithm, Charset charset)\n        throws AclException {\n        try {\n            byte[] signature = sign(data, key.getBytes(charset), algorithm);\n            return new String(Base64.encodeBase64(signature), DEFAULT_CHARSET);\n        } catch (Exception e) {\n            String message = String.format(CAL_SIGNATURE_FAILED_MSG, CAL_SIGNATURE_FAILED, e.getMessage());\n            log.error(message, e);\n            throw new AclException(\"CAL_SIGNATURE_FAILED\", CAL_SIGNATURE_FAILED, message, e);\n        }\n    }\n\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.acl.common;\n\nimport com.alibaba.fastjson2.JSONObject;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.yaml.snakeyaml.Yaml;\n\nimport java.io.FileInputStream;\nimport java.io.FileNotFoundException;\nimport java.io.InputStream;\nimport java.util.Map;\nimport java.util.SortedMap;\n\nimport static org.apache.rocketmq.acl.common.SessionCredentials.CHARSET;\n\npublic class AclUtils {\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);\n\n    public static byte[] combineRequestContent(RemotingCommand request, SortedMap<String, String> fieldsMap) {\n        try {\n            StringBuilder sb = new StringBuilder();\n            for (Map.Entry<String, String> entry : fieldsMap.entrySet()) {\n                if (!SessionCredentials.SIGNATURE.equals(entry.getKey())) {\n                    sb.append(entry.getValue());\n                }\n            }\n\n            return AclUtils.combineBytes(sb.toString().getBytes(CHARSET), request.getBody());\n        } catch (Exception e) {\n            throw new RuntimeException(\"Incompatible exception.\", e);\n        }\n    }\n\n    public static byte[] combineBytes(byte[] b1, byte[] b2) {\n        if (b1 == null || b1.length == 0) return b2;\n        if (b2 == null || b2.length == 0) return b1;\n        byte[] total = new byte[b1.length + b2.length];\n        System.arraycopy(b1, 0, total, 0, b1.length);\n        System.arraycopy(b2, 0, total, b1.length, b2.length);\n        return total;\n    }\n\n    public static String calSignature(byte[] data, String secretKey) {\n        return AclSigner.calSignature(data, secretKey);\n    }\n\n    public static void IPv6AddressCheck(String netAddress) {\n        if (isAsterisk(netAddress) || isMinus(netAddress)) {\n            int asterisk = netAddress.indexOf(\"*\");\n            int minus = netAddress.indexOf(\"-\");\n            // '*' must be the end of netAddress if it exists\n            if (asterisk > -1 && asterisk != netAddress.length() - 1) {\n                throw new AclException(String.format(\"NetAddress examine scope Exception netAddress is %s\", netAddress));\n            }\n\n            // format like \"2::ac5:78:1-200:*\" or \"2::ac5:78:1-200\" is legal\n            if (minus > -1) {\n                if (asterisk == -1) {\n                    if (minus <= netAddress.lastIndexOf(\":\")) {\n                        throw new AclException(String.format(\"NetAddress examine scope Exception netAddress is %s\", netAddress));\n                    }\n                } else {\n                    if (minus <= netAddress.lastIndexOf(\":\", netAddress.lastIndexOf(\":\") - 1)) {\n                        throw new AclException(String.format(\"NetAddress examine scope Exception netAddress is %s\", netAddress));\n                    }\n                }\n            }\n        }\n    }\n\n    public static String v6ipProcess(String netAddress) {\n        int part;\n        String subAddress;\n        boolean isAsterisk = isAsterisk(netAddress);\n        boolean isMinus = isMinus(netAddress);\n        if (isAsterisk && isMinus) {\n            part = 6;\n            int lastColon = netAddress.lastIndexOf(':');\n            int secondLastColon = netAddress.substring(0, lastColon).lastIndexOf(':');\n            subAddress = netAddress.substring(0, secondLastColon);\n        } else if (!isAsterisk && !isMinus) {\n            part = 8;\n            subAddress = netAddress;\n        } else {\n            part = 7;\n            subAddress = netAddress.substring(0, netAddress.lastIndexOf(':'));\n        }\n        return expandIP(subAddress, part);\n    }\n\n    public static void verify(String netAddress, int index) {\n        if (!AclUtils.isScope(netAddress, index)) {\n            throw new AclException(String.format(\"NetAddress examine scope Exception netAddress is %s\", netAddress));\n        }\n    }\n\n    public static String[] getAddresses(String netAddress, String partialAddress) {\n        String[] parAddStrArray = StringUtils.split(partialAddress.substring(1, partialAddress.length() - 1), \",\");\n        String address = netAddress.substring(0, netAddress.indexOf(\"{\"));\n        String[] addressStrArray = new String[parAddStrArray.length];\n        for (int i = 0; i < parAddStrArray.length; i++) {\n            addressStrArray[i] = address + parAddStrArray[i];\n        }\n        return addressStrArray;\n    }\n\n    public static boolean isScope(String netAddress, int index) {\n        // IPv6 Address\n        if (isColon(netAddress)) {\n            netAddress = expandIP(netAddress, 8);\n            String[] strArray = StringUtils.split(netAddress, \":\");\n            return isIPv6Scope(strArray, index);\n        }\n\n        String[] strArray = StringUtils.split(netAddress, \".\");\n        if (strArray.length != 4) {\n            return false;\n        }\n        return isScope(strArray, index);\n\n    }\n\n    public static boolean isScope(String[] num, int index) {\n        for (int i = 0; i < index; i++) {\n            if (!isScope(num[i])) {\n                return false;\n            }\n        }\n        return true;\n    }\n\n    public static boolean isColon(String netAddress) {\n        return netAddress.indexOf(':') > -1;\n    }\n\n    public static boolean isScope(String num) {\n        return isScope(Integer.parseInt(num.trim()));\n    }\n\n    public static boolean isScope(int num) {\n        return num >= 0 && num <= 255;\n    }\n\n    public static boolean isAsterisk(String asterisk) {\n        return asterisk.indexOf('*') > -1;\n    }\n\n    public static boolean isComma(String colon) {\n        return colon.indexOf(',') > -1;\n    }\n\n    public static boolean isMinus(String minus) {\n        return minus.indexOf('-') > -1;\n\n    }\n\n    public static boolean isIPv6Scope(String[] num, int index) {\n        for (int i = 0; i < index; i++) {\n            int value;\n            try {\n                value = Integer.parseInt(num[i], 16);\n            } catch (NumberFormatException e) {\n                return false;\n            }\n            if (!isIPv6Scope(value)) {\n                return false;\n            }\n        }\n        return true;\n    }\n\n    public static boolean isIPv6Scope(int num) {\n        int min = Integer.parseInt(\"0\", 16);\n        int max = Integer.parseInt(\"ffff\", 16);\n        return num >= min && num <= max;\n    }\n\n    public static String expandIP(String netAddress, int part) {\n        netAddress = netAddress.toUpperCase();\n        // expand netAddress\n        int separatorCount = StringUtils.countMatches(netAddress, \":\");\n        int padCount = part - separatorCount;\n        if (padCount > 0) {\n            StringBuilder padStr = new StringBuilder(\":\");\n            for (int i = 0; i < padCount; i++) {\n                padStr.append(\":\");\n            }\n            netAddress = StringUtils.replace(netAddress, \"::\", padStr.toString());\n        }\n\n        // pad netAddress\n        String[] strArray = StringUtils.splitPreserveAllTokens(netAddress, \":\");\n        for (int i = 0; i < strArray.length; i++) {\n            if (strArray[i].length() < 4) {\n                strArray[i] = StringUtils.leftPad(strArray[i], 4, '0');\n            }\n        }\n\n        // output\n        StringBuilder sb = new StringBuilder();\n        for (int i = 0; i < strArray.length; i++) {\n            sb.append(strArray[i]);\n            if (i != strArray.length - 1) {\n                sb.append(\":\");\n            }\n        }\n        return sb.toString();\n    }\n\n    public static <T> T getYamlDataObject(String path, Class<T> clazz) {\n        try (FileInputStream fis = new FileInputStream(path)) {\n            return getYamlDataObject(fis, clazz);\n        } catch (FileNotFoundException ignore) {\n            return null;\n        } catch (Exception e) {\n            throw new AclException(e.getMessage(), e);\n        }\n    }\n\n    public static <T> T getYamlDataObject(InputStream fis, Class<T> clazz) {\n        Yaml yaml = new Yaml();\n        try {\n            return yaml.loadAs(fis, clazz);\n        } catch (Exception e) {\n            throw new AclException(e.getMessage(), e);\n        }\n    }\n\n    public static RPCHook getAclRPCHook(String fileName) {\n        JSONObject yamlDataObject;\n        try {\n            yamlDataObject = AclUtils.getYamlDataObject(fileName,\n                JSONObject.class);\n        } catch (Exception e) {\n            log.error(\"Convert yaml file to data object error, \", e);\n            return null;\n        }\n        return buildRpcHook(yamlDataObject);\n    }\n\n    public static RPCHook getAclRPCHook(InputStream inputStream) {\n        JSONObject yamlDataObject = null;\n        try {\n            yamlDataObject = AclUtils.getYamlDataObject(inputStream, JSONObject.class);\n        } catch (Exception e) {\n            log.error(\"Convert yaml file to data object error, \", e);\n            return null;\n        }\n        return buildRpcHook(yamlDataObject);\n    }\n\n    private static RPCHook buildRpcHook(JSONObject yamlDataObject) {\n        if (yamlDataObject == null || yamlDataObject.isEmpty()) {\n            log.warn(\"Failed to parse configuration to enable ACL.\");\n            return null;\n        }\n\n        String accessKey = yamlDataObject.getString(AclConstants.CONFIG_ACCESS_KEY);\n        String secretKey = yamlDataObject.getString(AclConstants.CONFIG_SECRET_KEY);\n\n        if (StringUtils.isBlank(accessKey) || StringUtils.isBlank(secretKey)) {\n            log.warn(\"Failed to enable ACL. Either AccessKey or secretKey is blank\");\n            return null;\n        }\n        return new AclClientRPCHook(new SessionCredentials(accessKey, secretKey));\n    }\n\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/acl/common/Permission.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.acl.common;\n\npublic class Permission {\n\n    public static final byte DENY = 1;\n    public static final byte ANY = 1 << 1;\n    public static final byte PUB = 1 << 2;\n    public static final byte SUB = 1 << 3;\n\n    public static byte parsePermFromString(String permString) {\n        if (permString == null) {\n            return Permission.DENY;\n        }\n        switch (permString.trim()) {\n            case AclConstants.PUB:\n                return Permission.PUB;\n            case AclConstants.SUB:\n                return Permission.SUB;\n            case AclConstants.PUB_SUB:\n            case AclConstants.SUB_PUB:\n                return Permission.PUB | Permission.SUB;\n            case AclConstants.DENY:\n                return Permission.DENY;\n            default:\n                return Permission.DENY;\n        }\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/acl/common/SessionCredentials.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.acl.common;\n\nimport org.apache.rocketmq.common.MixAll;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.nio.charset.Charset;\nimport java.nio.charset.StandardCharsets;\nimport java.util.Properties;\n\npublic class SessionCredentials {\n    public static final Charset CHARSET = StandardCharsets.UTF_8;\n    public static final String ACCESS_KEY = \"AccessKey\";\n    public static final String SECRET_KEY = \"SecretKey\";\n    public static final String SIGNATURE = \"Signature\";\n    public static final String SECURITY_TOKEN = \"SecurityToken\";\n\n    public static final String KEY_FILE = System.getProperty(\"rocketmq.client.keyFile\",\n        System.getProperty(\"user.home\") + File.separator + \"key\");\n\n    private String accessKey;\n    private String secretKey;\n    private String securityToken;\n    private String signature;\n\n    public SessionCredentials() {\n        String keyContent = null;\n        try {\n            keyContent = MixAll.file2String(KEY_FILE);\n        } catch (IOException ignore) {\n        }\n        if (keyContent != null) {\n            Properties prop = MixAll.string2Properties(keyContent);\n            if (prop != null) {\n                this.updateContent(prop);\n            }\n        }\n    }\n\n    public SessionCredentials(String accessKey, String secretKey) {\n        this.accessKey = accessKey;\n        this.secretKey = secretKey;\n    }\n\n    public SessionCredentials(String accessKey, String secretKey, String securityToken) {\n        this(accessKey, secretKey);\n        this.securityToken = securityToken;\n    }\n\n    public void updateContent(Properties prop) {\n        {\n            String value = prop.getProperty(ACCESS_KEY);\n            if (value != null) {\n                this.accessKey = value.trim();\n            }\n        }\n        {\n            String value = prop.getProperty(SECRET_KEY);\n            if (value != null) {\n                this.secretKey = value.trim();\n            }\n        }\n        {\n            String value = prop.getProperty(SECURITY_TOKEN);\n            if (value != null) {\n                this.securityToken = value.trim();\n            }\n        }\n    }\n\n    public String getAccessKey() {\n        return accessKey;\n    }\n\n    public void setAccessKey(String accessKey) {\n        this.accessKey = accessKey;\n    }\n\n    public String getSecretKey() {\n        return secretKey;\n    }\n\n    public void setSecretKey(String secretKey) {\n        this.secretKey = secretKey;\n    }\n\n    public String getSignature() {\n        return signature;\n    }\n\n    public void setSignature(String signature) {\n        this.signature = signature;\n    }\n\n    public String getSecurityToken() {\n        return securityToken;\n    }\n\n    public void setSecurityToken(final String securityToken) {\n        this.securityToken = securityToken;\n    }\n\n    @Override\n    public int hashCode() {\n        final int prime = 31;\n        int result = 1;\n        result = prime * result + ((accessKey == null) ? 0 : accessKey.hashCode());\n        result = prime * result + ((secretKey == null) ? 0 : secretKey.hashCode());\n        result = prime * result + ((signature == null) ? 0 : signature.hashCode());\n        return result;\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj)\n            return true;\n        if (obj == null)\n            return false;\n        if (getClass() != obj.getClass())\n            return false;\n\n        SessionCredentials other = (SessionCredentials) obj;\n        if (accessKey == null) {\n            if (other.accessKey != null)\n                return false;\n        } else if (!accessKey.equals(other.accessKey))\n            return false;\n\n        if (secretKey == null) {\n            if (other.secretKey != null)\n                return false;\n        } else if (!secretKey.equals(other.secretKey))\n            return false;\n\n        if (signature == null) {\n            return other.signature == null;\n        } else return signature.equals(other.signature);\n    }\n\n    @Override\n    public String toString() {\n        return \"SessionCredentials [accessKey=\" + accessKey + \", secretKey=\" + secretKey + \", signature=\"\n            + signature + \", SecurityToken=\" + securityToken + \"]\";\n    }\n}"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/acl/common/SigningAlgorithm.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.acl.common;\n\npublic enum SigningAlgorithm {\n    HmacSHA1,\n    HmacSHA256,\n    HmacMD5;\n\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/AccessChannel.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client;\n\n/**\n * Used for set access channel, if need migrate the rocketmq service to cloud, it is We recommend set the value with\n * \"CLOUD\". otherwise set with \"LOCAL\", especially used the message trace feature.\n */\npublic enum AccessChannel {\n    /**\n     * Means connect to private IDC cluster.\n     */\n    LOCAL,\n\n    /**\n     * Means connect to Cloud service.\n     */\n    CLOUD,\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/ClientConfig.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client;\n\nimport java.util.Collection;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.Set;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.utils.NameServerAddressUtils;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.remoting.netty.TlsSystemConfig;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.RequestType;\n\n/**\n * Client Common configuration\n */\npublic class ClientConfig {\n    public static final String SEND_MESSAGE_WITH_VIP_CHANNEL_PROPERTY = \"com.rocketmq.sendMessageWithVIPChannel\";\n    public static final String SOCKS_PROXY_CONFIG = \"com.rocketmq.socks.proxy.config\";\n    public static final String DECODE_READ_BODY = \"com.rocketmq.read.body\";\n    public static final String DECODE_DECOMPRESS_BODY = \"com.rocketmq.decompress.body\";\n    public static final String SEND_LATENCY_ENABLE = \"com.rocketmq.sendLatencyEnable\";\n    public static final String START_DETECTOR_ENABLE = \"com.rocketmq.startDetectorEnable\";\n    public static final String HEART_BEAT_V2 = \"com.rocketmq.heartbeat.v2\";\n    private String namesrvAddr = NameServerAddressUtils.getNameServerAddresses();\n    private String clientIP = NetworkUtil.getLocalAddress();\n    private String instanceName = System.getProperty(\"rocketmq.client.name\", \"DEFAULT\");\n    private int clientCallbackExecutorThreads = Runtime.getRuntime().availableProcessors();\n    @Deprecated\n    protected String namespace;\n    private boolean namespaceInitialized = false;\n    protected String namespaceV2;\n    protected AccessChannel accessChannel = AccessChannel.LOCAL;\n\n    /**\n     * Pulling topic information interval from the named server\n     */\n    private int pollNameServerInterval = 1000 * 30;\n    /**\n     * Heartbeat interval in microseconds with message broker\n     */\n    private int heartbeatBrokerInterval = 1000 * 30;\n    /**\n     * Offset persistent interval for consumer\n     */\n    private int persistConsumerOffsetInterval = 1000 * 5;\n    private long pullTimeDelayMillsWhenException = 1000;\n\n    private int traceMsgBatchNum = 10;\n    private boolean unitMode = false;\n    private String unitName;\n    private boolean decodeReadBody = Boolean.parseBoolean(System.getProperty(DECODE_READ_BODY, \"true\"));\n    private boolean decodeDecompressBody = Boolean.parseBoolean(System.getProperty(DECODE_DECOMPRESS_BODY, \"true\"));\n    private boolean vipChannelEnabled = Boolean.parseBoolean(System.getProperty(SEND_MESSAGE_WITH_VIP_CHANNEL_PROPERTY, \"false\"));\n    private boolean useHeartbeatV2 = Boolean.parseBoolean(System.getProperty(HEART_BEAT_V2, \"false\"));\n\n    private boolean useTLS = TlsSystemConfig.tlsEnable;\n\n    private String socksProxyConfig = System.getProperty(SOCKS_PROXY_CONFIG, \"{}\");\n\n    private int mqClientApiTimeout = 3 * 1000;\n    private int detectTimeout = 200;\n    private int detectInterval = 2 * 1000;\n\n    private LanguageCode language = LanguageCode.JAVA;\n\n    /**\n     * Enable stream request type will inject a RPCHook to add corresponding request type to remoting layer.\n     * And it will also generate a different client id to prevent unexpected reuses of MQClientInstance.\n     */\n    protected boolean enableStreamRequestType = false;\n\n    /**\n     * Enable the fault tolerance mechanism of the client sending process.\n     * DO NOT OPEN when ORDER messages are required.\n     * Turning on will interfere with the queue selection functionality,\n     * possibly conflicting with the order message.\n     */\n    private boolean sendLatencyEnable = Boolean.parseBoolean(System.getProperty(SEND_LATENCY_ENABLE, \"false\"));\n    private boolean startDetectorEnable = Boolean.parseBoolean(System.getProperty(START_DETECTOR_ENABLE, \"false\"));\n\n    private boolean enableHeartbeatChannelEventListener = true;\n\n    private boolean enableConcurrentHeartbeat = false;\n\n    private int concurrentHeartbeatThreadPoolSize = Runtime.getRuntime().availableProcessors();\n\n    /**\n     * The switch for message trace\n     */\n    protected boolean enableTrace = false;\n\n    /**\n     * The name value of message trace topic. If not set, the default trace topic name will be used.\n     */\n    protected String traceTopic;\n\n    protected int maxPageSizeInGetMetadata = 2000;\n\n    public String buildMQClientId() {\n        StringBuilder sb = new StringBuilder();\n        sb.append(this.getClientIP());\n\n        sb.append(\"@\");\n        sb.append(this.getInstanceName());\n        if (!UtilAll.isBlank(this.unitName)) {\n            sb.append(\"@\");\n            sb.append(this.unitName);\n        }\n\n        if (enableStreamRequestType) {\n            sb.append(\"@\");\n            sb.append(RequestType.STREAM);\n        }\n\n        return sb.toString();\n    }\n\n    public int getTraceMsgBatchNum() {\n        return traceMsgBatchNum;\n    }\n\n    public void setTraceMsgBatchNum(int traceMsgBatchNum) {\n        this.traceMsgBatchNum = traceMsgBatchNum;\n    }\n\n    public String getClientIP() {\n        return clientIP;\n    }\n\n    public void setClientIP(String clientIP) {\n        this.clientIP = clientIP;\n    }\n\n    public String getInstanceName() {\n        return instanceName;\n    }\n\n    public void setInstanceName(String instanceName) {\n        this.instanceName = instanceName;\n    }\n\n    public void changeInstanceNameToPID() {\n        if (this.instanceName.equals(\"DEFAULT\")) {\n            this.instanceName = UtilAll.getPid() + \"#\" + System.nanoTime();\n        }\n    }\n\n    @Deprecated\n    public String withNamespace(String resource) {\n        return NamespaceUtil.wrapNamespace(this.getNamespace(), resource);\n    }\n\n    @Deprecated\n    public Set<String> withNamespace(Set<String> resourceSet) {\n        Set<String> resourceWithNamespace = new HashSet<>();\n        for (String resource : resourceSet) {\n            resourceWithNamespace.add(withNamespace(resource));\n        }\n        return resourceWithNamespace;\n    }\n\n    @Deprecated\n    public String withoutNamespace(String resource) {\n        return NamespaceUtil.withoutNamespace(resource, this.getNamespace());\n    }\n\n    @Deprecated\n    public Set<String> withoutNamespace(Set<String> resourceSet) {\n        Set<String> resourceWithoutNamespace = new HashSet<>();\n        for (String resource : resourceSet) {\n            resourceWithoutNamespace.add(withoutNamespace(resource));\n        }\n        return resourceWithoutNamespace;\n    }\n\n    @Deprecated\n    public MessageQueue queueWithNamespace(MessageQueue queue) {\n        if (StringUtils.isEmpty(this.getNamespace())) {\n            return queue;\n        }\n        return new MessageQueue(withNamespace(queue.getTopic()), queue.getBrokerName(), queue.getQueueId());\n    }\n\n    @Deprecated\n    public Collection<MessageQueue> queuesWithNamespace(Collection<MessageQueue> queues) {\n        if (StringUtils.isEmpty(this.getNamespace())) {\n            return queues;\n        }\n        Iterator<MessageQueue> iter = queues.iterator();\n        while (iter.hasNext()) {\n            MessageQueue queue = iter.next();\n            queue.setTopic(withNamespace(queue.getTopic()));\n        }\n        return queues;\n    }\n\n    public void resetClientConfig(final ClientConfig cc) {\n        this.namesrvAddr = cc.namesrvAddr;\n        this.clientIP = cc.clientIP;\n        this.instanceName = cc.instanceName;\n        this.clientCallbackExecutorThreads = cc.clientCallbackExecutorThreads;\n        this.pollNameServerInterval = cc.pollNameServerInterval;\n        this.heartbeatBrokerInterval = cc.heartbeatBrokerInterval;\n        this.persistConsumerOffsetInterval = cc.persistConsumerOffsetInterval;\n        this.pullTimeDelayMillsWhenException = cc.pullTimeDelayMillsWhenException;\n        this.unitMode = cc.unitMode;\n        this.unitName = cc.unitName;\n        this.vipChannelEnabled = cc.vipChannelEnabled;\n        this.useTLS = cc.useTLS;\n        this.socksProxyConfig = cc.socksProxyConfig;\n        this.namespace = cc.namespace;\n        this.language = cc.language;\n        this.mqClientApiTimeout = cc.mqClientApiTimeout;\n        this.decodeReadBody = cc.decodeReadBody;\n        this.decodeDecompressBody = cc.decodeDecompressBody;\n        this.enableStreamRequestType = cc.enableStreamRequestType;\n        this.useHeartbeatV2 = cc.useHeartbeatV2;\n        this.startDetectorEnable = cc.startDetectorEnable;\n        this.sendLatencyEnable = cc.sendLatencyEnable;\n        this.enableHeartbeatChannelEventListener = cc.enableHeartbeatChannelEventListener;\n        this.detectInterval = cc.detectInterval;\n        this.detectTimeout = cc.detectTimeout;\n        this.namespaceV2 = cc.namespaceV2;\n        this.enableTrace = cc.enableTrace;\n        this.traceTopic = cc.traceTopic;\n        this.enableConcurrentHeartbeat = cc.enableConcurrentHeartbeat;\n        this.concurrentHeartbeatThreadPoolSize = cc.concurrentHeartbeatThreadPoolSize;\n    }\n\n    public ClientConfig cloneClientConfig() {\n        ClientConfig cc = new ClientConfig();\n        cc.namesrvAddr = namesrvAddr;\n        cc.clientIP = clientIP;\n        cc.instanceName = instanceName;\n        cc.clientCallbackExecutorThreads = clientCallbackExecutorThreads;\n        cc.pollNameServerInterval = pollNameServerInterval;\n        cc.heartbeatBrokerInterval = heartbeatBrokerInterval;\n        cc.persistConsumerOffsetInterval = persistConsumerOffsetInterval;\n        cc.pullTimeDelayMillsWhenException = pullTimeDelayMillsWhenException;\n        cc.unitMode = unitMode;\n        cc.unitName = unitName;\n        cc.vipChannelEnabled = vipChannelEnabled;\n        cc.useTLS = useTLS;\n        cc.socksProxyConfig = socksProxyConfig;\n        cc.namespace = namespace;\n        cc.language = language;\n        cc.mqClientApiTimeout = mqClientApiTimeout;\n        cc.decodeReadBody = decodeReadBody;\n        cc.decodeDecompressBody = decodeDecompressBody;\n        cc.enableStreamRequestType = enableStreamRequestType;\n        cc.useHeartbeatV2 = useHeartbeatV2;\n        cc.startDetectorEnable = startDetectorEnable;\n        cc.enableHeartbeatChannelEventListener = enableHeartbeatChannelEventListener;\n        cc.sendLatencyEnable = sendLatencyEnable;\n        cc.detectInterval = detectInterval;\n        cc.detectTimeout = detectTimeout;\n        cc.namespaceV2 = namespaceV2;\n        cc.enableTrace = enableTrace;\n        cc.traceTopic = traceTopic;\n        cc.enableConcurrentHeartbeat = enableConcurrentHeartbeat;\n        cc.concurrentHeartbeatThreadPoolSize = concurrentHeartbeatThreadPoolSize;\n        return cc;\n    }\n\n    public String getNamesrvAddr() {\n        if (StringUtils.isNotEmpty(namesrvAddr) && NameServerAddressUtils.NAMESRV_ENDPOINT_PATTERN.matcher(namesrvAddr.trim()).matches()) {\n            return NameServerAddressUtils.getNameSrvAddrFromNamesrvEndpoint(namesrvAddr);\n        }\n        return namesrvAddr;\n    }\n\n    /**\n     * Domain name mode access way does not support the delimiter(;), and only one domain name can be set.\n     *\n     * @param namesrvAddr name server address\n     */\n    public void setNamesrvAddr(String namesrvAddr) {\n        this.namesrvAddr = namesrvAddr;\n        this.namespaceInitialized = false;\n    }\n\n    public int getClientCallbackExecutorThreads() {\n        return clientCallbackExecutorThreads;\n    }\n\n    public void setClientCallbackExecutorThreads(int clientCallbackExecutorThreads) {\n        this.clientCallbackExecutorThreads = clientCallbackExecutorThreads;\n    }\n\n    public int getPollNameServerInterval() {\n        return pollNameServerInterval;\n    }\n\n    public void setPollNameServerInterval(int pollNameServerInterval) {\n        this.pollNameServerInterval = pollNameServerInterval;\n    }\n\n    public int getHeartbeatBrokerInterval() {\n        return heartbeatBrokerInterval;\n    }\n\n    public void setHeartbeatBrokerInterval(int heartbeatBrokerInterval) {\n        this.heartbeatBrokerInterval = heartbeatBrokerInterval;\n    }\n\n    public int getPersistConsumerOffsetInterval() {\n        return persistConsumerOffsetInterval;\n    }\n\n    public void setPersistConsumerOffsetInterval(int persistConsumerOffsetInterval) {\n        this.persistConsumerOffsetInterval = persistConsumerOffsetInterval;\n    }\n\n    public long getPullTimeDelayMillsWhenException() {\n        return pullTimeDelayMillsWhenException;\n    }\n\n    public void setPullTimeDelayMillsWhenException(long pullTimeDelayMillsWhenException) {\n        this.pullTimeDelayMillsWhenException = pullTimeDelayMillsWhenException;\n    }\n\n    public String getUnitName() {\n        return unitName;\n    }\n\n    public void setUnitName(String unitName) {\n        this.unitName = unitName;\n    }\n\n    public boolean isUnitMode() {\n        return unitMode;\n    }\n\n    public void setUnitMode(boolean unitMode) {\n        this.unitMode = unitMode;\n    }\n\n    public boolean isVipChannelEnabled() {\n        return vipChannelEnabled;\n    }\n\n    public void setVipChannelEnabled(final boolean vipChannelEnabled) {\n        this.vipChannelEnabled = vipChannelEnabled;\n    }\n\n    public boolean isUseTLS() {\n        return useTLS;\n    }\n\n    public void setUseTLS(boolean useTLS) {\n        this.useTLS = useTLS;\n    }\n\n    public String getSocksProxyConfig() {\n        return socksProxyConfig;\n    }\n\n    public void setSocksProxyConfig(String socksProxyConfig) {\n        this.socksProxyConfig = socksProxyConfig;\n    }\n\n    public LanguageCode getLanguage() {\n        return language;\n    }\n\n    public void setLanguage(LanguageCode language) {\n        this.language = language;\n    }\n\n    public boolean isDecodeReadBody() {\n        return decodeReadBody;\n    }\n\n    public void setDecodeReadBody(boolean decodeReadBody) {\n        this.decodeReadBody = decodeReadBody;\n    }\n\n    public boolean isDecodeDecompressBody() {\n        return decodeDecompressBody;\n    }\n\n    public void setDecodeDecompressBody(boolean decodeDecompressBody) {\n        this.decodeDecompressBody = decodeDecompressBody;\n    }\n\n    @Deprecated\n    public String getNamespace() {\n        if (namespaceInitialized) {\n            return namespace;\n        }\n\n        if (StringUtils.isNotEmpty(namespace)) {\n            return namespace;\n        }\n\n        if (StringUtils.isNotEmpty(this.namesrvAddr)) {\n            if (NameServerAddressUtils.validateInstanceEndpoint(namesrvAddr)) {\n                namespace = NameServerAddressUtils.parseInstanceIdFromEndpoint(namesrvAddr);\n            }\n        }\n        namespaceInitialized = true;\n        return namespace;\n    }\n\n    @Deprecated\n    public void setNamespace(String namespace) {\n        this.namespace = namespace;\n        this.namespaceInitialized = true;\n    }\n\n    public String getNamespaceV2() {\n        return namespaceV2;\n    }\n\n    public void setNamespaceV2(String namespaceV2) {\n        this.namespaceV2 = namespaceV2;\n    }\n\n    public AccessChannel getAccessChannel() {\n        return this.accessChannel;\n    }\n\n    public void setAccessChannel(AccessChannel accessChannel) {\n        this.accessChannel = accessChannel;\n    }\n\n    public int getMqClientApiTimeout() {\n        return mqClientApiTimeout;\n    }\n\n    public void setMqClientApiTimeout(int mqClientApiTimeout) {\n        this.mqClientApiTimeout = mqClientApiTimeout;\n    }\n\n    public boolean isEnableStreamRequestType() {\n        return enableStreamRequestType;\n    }\n\n    public void setEnableStreamRequestType(boolean enableStreamRequestType) {\n        this.enableStreamRequestType = enableStreamRequestType;\n    }\n\n    public boolean isSendLatencyEnable() {\n        return sendLatencyEnable;\n    }\n\n    public void setSendLatencyEnable(boolean sendLatencyEnable) {\n        this.sendLatencyEnable = sendLatencyEnable;\n    }\n\n    public boolean isStartDetectorEnable() {\n        return startDetectorEnable;\n    }\n\n    public void setStartDetectorEnable(boolean startDetectorEnable) {\n        this.startDetectorEnable = startDetectorEnable;\n    }\n\n    public boolean isEnableHeartbeatChannelEventListener() {\n        return enableHeartbeatChannelEventListener;\n    }\n\n    public void setEnableHeartbeatChannelEventListener(boolean enableHeartbeatChannelEventListener) {\n        this.enableHeartbeatChannelEventListener = enableHeartbeatChannelEventListener;\n    }\n\n    public int getDetectTimeout() {\n        return this.detectTimeout;\n    }\n\n    public void setDetectTimeout(int detectTimeout) {\n        this.detectTimeout = detectTimeout;\n    }\n\n    public int getDetectInterval() {\n        return this.detectInterval;\n    }\n\n    public void setDetectInterval(int detectInterval) {\n        this.detectInterval = detectInterval;\n    }\n\n    public boolean isUseHeartbeatV2() {\n        return useHeartbeatV2;\n    }\n\n    public void setUseHeartbeatV2(boolean useHeartbeatV2) {\n        this.useHeartbeatV2 = useHeartbeatV2;\n    }\n\n    public boolean isEnableTrace() {\n        return enableTrace;\n    }\n\n    public void setEnableTrace(boolean enableTrace) {\n        this.enableTrace = enableTrace;\n    }\n\n    public String getTraceTopic() {\n        return traceTopic;\n    }\n\n    public void setTraceTopic(String traceTopic) {\n        this.traceTopic = traceTopic;\n    }\n\n    public int getMaxPageSizeInGetMetadata() {\n        return maxPageSizeInGetMetadata;\n    }\n\n    public void setMaxPageSizeInGetMetadata(int maxPageSizeInGetMetadata) {\n        this.maxPageSizeInGetMetadata = maxPageSizeInGetMetadata;\n    }\n\n    public boolean isEnableConcurrentHeartbeat() {\n        return this.enableConcurrentHeartbeat;\n    }\n\n    public void setEnableConcurrentHeartbeat(boolean enableConcurrentHeartbeat) {\n        this.enableConcurrentHeartbeat = enableConcurrentHeartbeat;\n    }\n\n    public int getConcurrentHeartbeatThreadPoolSize() {\n        return concurrentHeartbeatThreadPoolSize;\n    }\n\n    public void setConcurrentHeartbeatThreadPoolSize(int concurrentHeartbeatThreadPoolSize) {\n        this.concurrentHeartbeatThreadPoolSize = concurrentHeartbeatThreadPoolSize;\n    }\n\n    @Override\n    public String toString() {\n        return \"ClientConfig{\" +\n            \"namesrvAddr='\" + namesrvAddr + '\\'' +\n            \", clientIP='\" + clientIP + '\\'' +\n            \", instanceName='\" + instanceName + '\\'' +\n            \", clientCallbackExecutorThreads=\" + clientCallbackExecutorThreads +\n            \", namespace='\" + namespace + '\\'' +\n            \", namespaceInitialized=\" + namespaceInitialized +\n            \", namespaceV2='\" + namespaceV2 + '\\'' +\n            \", accessChannel=\" + accessChannel +\n            \", pollNameServerInterval=\" + pollNameServerInterval +\n            \", heartbeatBrokerInterval=\" + heartbeatBrokerInterval +\n            \", persistConsumerOffsetInterval=\" + persistConsumerOffsetInterval +\n            \", pullTimeDelayMillsWhenException=\" + pullTimeDelayMillsWhenException +\n            \", unitMode=\" + unitMode +\n            \", unitName='\" + unitName + '\\'' +\n            \", decodeReadBody=\" + decodeReadBody +\n            \", decodeDecompressBody=\" + decodeDecompressBody +\n            \", vipChannelEnabled=\" + vipChannelEnabled +\n            \", useHeartbeatV2=\" + useHeartbeatV2 +\n            \", useTLS=\" + useTLS +\n            \", socksProxyConfig='\" + socksProxyConfig + '\\'' +\n            \", mqClientApiTimeout=\" + mqClientApiTimeout +\n            \", detectTimeout=\" + detectTimeout +\n            \", detectInterval=\" + detectInterval +\n            \", language=\" + language +\n            \", enableStreamRequestType=\" + enableStreamRequestType +\n            \", sendLatencyEnable=\" + sendLatencyEnable +\n            \", startDetectorEnable=\" + startDetectorEnable +\n            \", enableHeartbeatChannelEventListener=\" + enableHeartbeatChannelEventListener +\n            \", enableTrace=\" + enableTrace +\n            \", traceTopic='\" + traceTopic + '\\'' +\n            \", enableConcurrentHeartbeat=\" + enableConcurrentHeartbeat +\n            \", concurrentHeartbeatThreadPoolSize=\" + concurrentHeartbeatThreadPoolSize +\n            '}';\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/MQAdmin.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client;\n\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\n\nimport java.util.Map;\n\n/**\n * Base interface for MQ management\n */\npublic interface MQAdmin {\n    /**\n     * Creates a topic\n     *  @param key accessKey\n     * @param newTopic topic name\n     * @param queueNum topic's queue number\n     * @param attributes\n     */\n    void createTopic(final String key, final String newTopic, final int queueNum, Map<String, String> attributes)\n        throws MQClientException;\n\n    /**\n     * Creates a topic\n     *  @param key accessKey\n     * @param newTopic topic name\n     * @param queueNum topic's queue number\n     * @param topicSysFlag topic system flag\n     * @param attributes\n     */\n    void createTopic(String key, String newTopic, int queueNum, int topicSysFlag, Map<String, String> attributes)\n        throws MQClientException;\n\n    /**\n     * Gets the message queue offset according to some time in milliseconds<br>\n     * be cautious to call because of more IO overhead\n     *\n     * @param mq Instance of MessageQueue\n     * @param timestamp from when in milliseconds.\n     * @return offset\n     */\n    long searchOffset(final MessageQueue mq, final long timestamp) throws MQClientException;\n\n    /**\n     * Gets the max offset\n     *\n     * @param mq Instance of MessageQueue\n     * @return the max offset\n     */\n    long maxOffset(final MessageQueue mq) throws MQClientException;\n\n    /**\n     * Gets the minimum offset\n     *\n     * @param mq Instance of MessageQueue\n     * @return the minimum offset\n     */\n    long minOffset(final MessageQueue mq) throws MQClientException;\n\n    /**\n     * Gets the earliest stored message time\n     *\n     * @param mq Instance of MessageQueue\n     * @return the time in microseconds\n     */\n    long earliestMsgStoreTime(final MessageQueue mq) throws MQClientException;\n\n    /**\n     * Query messages\n     *\n     * @param topic message topic\n     * @param key message key index word\n     * @param maxNum max message number\n     * @param begin from when\n     * @param end to when\n     * @return Instance of QueryResult\n     */\n    QueryResult queryMessage(final String topic, final String key, final int maxNum, final long begin,\n        final long end) throws MQClientException, InterruptedException;\n\n    /**\n     * @return The {@code MessageExt} of given msgId\n     */\n    MessageExt viewMessage(String topic,\n        String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException;\n\n}"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/MQHelper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client;\n\nimport java.util.Set;\nimport java.util.TreeSet;\nimport org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class MQHelper {\n    private static final Logger log = LoggerFactory.getLogger(MQHelper.class);\n\n    @Deprecated\n    public static void resetOffsetByTimestamp(\n        final MessageModel messageModel,\n        final String consumerGroup,\n        final String topic,\n        final long timestamp) throws Exception {\n        resetOffsetByTimestamp(messageModel, \"DEFAULT\", consumerGroup, topic, timestamp);\n    }\n\n    /**\n     * Reset consumer topic offset according to time\n     *\n     * @param messageModel  which model\n     * @param instanceName  which instance\n     * @param consumerGroup consumer group\n     * @param topic         topic\n     * @param timestamp     time\n     */\n    public static void resetOffsetByTimestamp(\n        final MessageModel messageModel,\n        final String instanceName,\n        final String consumerGroup,\n        final String topic,\n        final long timestamp) throws Exception {\n\n        DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(consumerGroup);\n        consumer.setInstanceName(instanceName);\n        consumer.setMessageModel(messageModel);\n        consumer.start();\n\n        Set<MessageQueue> mqs = null;\n        try {\n            mqs = consumer.fetchSubscribeMessageQueues(topic);\n            if (mqs != null && !mqs.isEmpty()) {\n                TreeSet<MessageQueue> mqsNew = new TreeSet<>(mqs);\n                for (MessageQueue mq : mqsNew) {\n                    long offset = consumer.searchOffset(mq, timestamp);\n                    if (offset >= 0) {\n                        consumer.updateConsumeOffset(mq, offset);\n                        log.info(\"resetOffsetByTimestamp updateConsumeOffset success, {} {} {}\",\n                            consumerGroup, offset, mq);\n                    }\n                }\n            }\n        } catch (Exception e) {\n            log.warn(\"resetOffsetByTimestamp Exception\", e);\n            throw e;\n        } finally {\n            if (mqs != null) {\n                consumer.getDefaultMQPullConsumerImpl().getOffsetStore().persistAll(mqs);\n            }\n            consumer.shutdown();\n        }\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/MqClientAdmin.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client;\n\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.admin.ConsumeStats;\nimport org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;\nimport org.apache.rocketmq.remoting.protocol.body.ClusterInfo;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerConnection;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.body.GroupList;\nimport org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan;\nimport org.apache.rocketmq.remoting.protocol.body.TopicList;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.CreateTopicRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.DeleteSubscriptionGroupRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.DeleteTopicRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumeStatsRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerConnectionListRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetTopicStatsInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumeTimeSpanRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QuerySubscriptionByConsumerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryTopicConsumeByWhoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryTopicsByConsumerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ViewMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.DeleteKVConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.DeleteTopicFromNamesrvRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\n\npublic interface MqClientAdmin {\n    CompletableFuture<List<MessageExt>> queryMessage(String address, boolean uniqueKeyFlag, boolean decompressBody,\n        QueryMessageRequestHeader requestHeader, long timeoutMillis);\n\n    CompletableFuture<TopicStatsTable> getTopicStatsInfo(String address,\n        GetTopicStatsInfoRequestHeader requestHeader, long timeoutMillis);\n\n    CompletableFuture<List<QueueTimeSpan>> queryConsumeTimeSpan(String address,\n        QueryConsumeTimeSpanRequestHeader requestHeader, long timeoutMillis);\n\n    CompletableFuture<Void> updateOrCreateTopic(String address, CreateTopicRequestHeader requestHeader,\n        long timeoutMillis);\n\n    CompletableFuture<Void> updateOrCreateSubscriptionGroup(String address, SubscriptionGroupConfig config,\n        long timeoutMillis);\n\n    CompletableFuture<Void> deleteTopicInBroker(String address, DeleteTopicRequestHeader requestHeader,\n        long timeoutMillis);\n\n    CompletableFuture<Void> deleteTopicInNameserver(String address, DeleteTopicFromNamesrvRequestHeader requestHeader,\n        long timeoutMillis);\n\n    CompletableFuture<Void> deleteKvConfig(String address, DeleteKVConfigRequestHeader requestHeader,\n        long timeoutMillis);\n\n    CompletableFuture<Void> deleteSubscriptionGroup(String address, DeleteSubscriptionGroupRequestHeader requestHeader,\n        long timeoutMillis);\n\n    CompletableFuture<Map<MessageQueue, Long>> invokeBrokerToResetOffset(String address,\n        ResetOffsetRequestHeader requestHeader, long timeoutMillis);\n\n    CompletableFuture<MessageExt> viewMessage(String address, ViewMessageRequestHeader requestHeader,\n        long timeoutMillis);\n\n    CompletableFuture<ClusterInfo> getBrokerClusterInfo(String address, long timeoutMillis);\n\n    CompletableFuture<ConsumerConnection> getConsumerConnectionList(String address,\n        GetConsumerConnectionListRequestHeader requestHeader, long timeoutMillis);\n\n    CompletableFuture<TopicList> queryTopicsByConsumer(String address,\n        QueryTopicsByConsumerRequestHeader requestHeader, long timeoutMillis);\n\n    CompletableFuture<SubscriptionData> querySubscriptionByConsumer(String address,\n        QuerySubscriptionByConsumerRequestHeader requestHeader, long timeoutMillis);\n\n    CompletableFuture<ConsumeStats> getConsumeStats(String address, GetConsumeStatsRequestHeader requestHeader,\n        long timeoutMillis);\n\n    CompletableFuture<GroupList> queryTopicConsumeByWho(String address,\n        QueryTopicConsumeByWhoRequestHeader requestHeader, long timeoutMillis);\n\n    CompletableFuture<ConsumerRunningInfo> getConsumerRunningInfo(String address,\n        GetConsumerRunningInfoRequestHeader requestHeader, long timeoutMillis);\n\n    CompletableFuture<ConsumeMessageDirectlyResult> consumeMessageDirectly(String address,\n        ConsumeMessageDirectlyResultRequestHeader requestHeader, long timeoutMillis);\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/QueryResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client;\n\nimport java.util.List;\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic class QueryResult {\n    private final long indexLastUpdateTimestamp;\n    private final List<MessageExt> messageList;\n\n    public QueryResult(long indexLastUpdateTimestamp, List<MessageExt> messageList) {\n        this.indexLastUpdateTimestamp = indexLastUpdateTimestamp;\n        this.messageList = messageList;\n    }\n\n    public long getIndexLastUpdateTimestamp() {\n        return indexLastUpdateTimestamp;\n    }\n\n    public List<MessageExt> getMessageList() {\n        return messageList;\n    }\n\n    @Override\n    public String toString() {\n        return \"QueryResult [indexLastUpdateTimestamp=\" + indexLastUpdateTimestamp + \", messageList=\"\n            + messageList + \"]\";\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/Validators.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client;\n\nimport java.io.File;\nimport java.util.Properties;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\n\nimport static org.apache.rocketmq.common.topic.TopicValidator.isTopicOrGroupIllegal;\n\n/**\n * Common Validator\n */\npublic class Validators {\n    public static final int CHARACTER_MAX_LENGTH = 255;\n    public static final int TOPIC_MAX_LENGTH = 127;\n    /*\n     * Group name max length is 120, for it will be used to make up retry and DLQ topic,\n     * like pull retry: %RETRY%group_topic and pop retry: %RETRY%group_topic.\n     */\n    public static final int GROUP_MAX_LENGTH = 120;\n\n    /**\n     * Validate group\n     */\n    public static void checkGroup(String group) throws MQClientException {\n        if (UtilAll.isBlank(group)) {\n            throw new MQClientException(\"the specified group is blank\", null);\n        }\n\n        if (group.length() > GROUP_MAX_LENGTH) {\n            throw new MQClientException(String.format(\"the specified group[%s] is longer than group max length: %s.\", group, GROUP_MAX_LENGTH), null);\n        }\n\n        if (isTopicOrGroupIllegal(group)) {\n            throw new MQClientException(String.format(\n                    \"the specified group[%s] contains illegal characters, allowing only %s\", group,\n                    \"^[%|a-zA-Z0-9_-]+$\"), null);\n        }\n    }\n\n    public static void checkMessage(Message msg, DefaultMQProducer defaultMQProducer) throws MQClientException {\n        if (null == msg) {\n            throw new MQClientException(ResponseCode.MESSAGE_ILLEGAL, \"the message is null\");\n        }\n        // topic\n        Validators.checkTopic(msg.getTopic());\n        Validators.isNotAllowedSendTopic(msg.getTopic());\n\n        // body\n        if (null == msg.getBody()) {\n            throw new MQClientException(ResponseCode.MESSAGE_ILLEGAL, \"the message body is null\");\n        }\n\n        if (0 == msg.getBody().length) {\n            throw new MQClientException(ResponseCode.MESSAGE_ILLEGAL, \"the message body length is zero\");\n        }\n\n        if (msg.getBody().length > defaultMQProducer.getMaxMessageSize()) {\n            throw new MQClientException(ResponseCode.MESSAGE_ILLEGAL,\n                \"the message body size over max value, MAX: \" + defaultMQProducer.getMaxMessageSize());\n        }\n\n        String lmqPath = msg.getUserProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);\n        if (StringUtils.contains(lmqPath, File.separator)) {\n            throw new MQClientException(ResponseCode.MESSAGE_ILLEGAL,\n                \"INNER_MULTI_DISPATCH \" + lmqPath + \" can not contains \" + File.separator + \" character\");\n        }\n    }\n\n    public static void checkTopic(String topic) throws MQClientException {\n        if (UtilAll.isBlank(topic)) {\n            throw new MQClientException(\"The specified topic is blank\", null);\n        }\n\n        if (topic.length() > TOPIC_MAX_LENGTH) {\n            throw new MQClientException(\n                String.format(\"The specified topic is longer than topic max length %d.\", TOPIC_MAX_LENGTH), null);\n        }\n\n        if (isTopicOrGroupIllegal(topic)) {\n            throw new MQClientException(String.format(\n                    \"The specified topic[%s] contains illegal characters, allowing only %s\", topic,\n                    \"^[%|a-zA-Z0-9_-]+$\"), null);\n        }\n    }\n\n    public static void isSystemTopic(String topic) throws MQClientException {\n        if (TopicValidator.isSystemTopic(topic)) {\n            throw new MQClientException(\n                    String.format(\"The topic[%s] is conflict with system topic.\", topic), null);\n        }\n    }\n\n    public static void isNotAllowedSendTopic(String topic) throws MQClientException {\n        if (TopicValidator.isNotAllowedSendTopic(topic)) {\n            throw new MQClientException(\n                    String.format(\"Sending message to topic[%s] is forbidden.\", topic), null);\n        }\n    }\n\n    public static void checkTopicConfig(final TopicConfig topicConfig) throws MQClientException {\n        if (!PermName.isValid(topicConfig.getPerm())) {\n            throw new MQClientException(ResponseCode.NO_PERMISSION,\n                String.format(\"topicPermission value: %s is invalid.\", topicConfig.getPerm()));\n        }\n    }\n\n    public static void checkBrokerConfig(final Properties brokerConfig) throws MQClientException {\n        String brokerPermission = brokerConfig.getProperty(\"brokerPermission\");\n        if (brokerPermission != null && !PermName.isValid(brokerPermission)) {\n            throw new MQClientException(ResponseCode.NO_PERMISSION,\n                    String.format(\"brokerPermission value: %s is invalid.\", brokerPermission));\n        }\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/admin/MQAdminExtInner.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.admin;\n\npublic interface MQAdminExtInner {\n\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/common/ClientErrorCode.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.common;\n\npublic class ClientErrorCode {\n    public static final int CONNECT_BROKER_EXCEPTION = 10001;\n    public static final int ACCESS_BROKER_TIMEOUT = 10002;\n    public static final int BROKER_NOT_EXIST_EXCEPTION = 10003;\n    public static final int NO_NAME_SERVER_EXCEPTION = 10004;\n    public static final int NOT_FOUND_TOPIC_EXCEPTION = 10005;\n    public static final int REQUEST_TIMEOUT_EXCEPTION = 10006;\n    public static final int CREATE_REPLY_MESSAGE_EXCEPTION = 10007;\n}"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/common/NameserverAccessConfig.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.common;\n\npublic class NameserverAccessConfig {\n    private String namesrvAddr;\n    private String namesrvDomain;\n    private String namesrvDomainSubgroup;\n\n    public NameserverAccessConfig(String namesrvAddr, String namesrvDomain, String namesrvDomainSubgroup) {\n        this.namesrvAddr = namesrvAddr;\n        this.namesrvDomain = namesrvDomain;\n        this.namesrvDomainSubgroup = namesrvDomainSubgroup;\n    }\n\n    public String getNamesrvAddr() {\n        return namesrvAddr;\n    }\n\n    public String getNamesrvDomain() {\n        return namesrvDomain;\n    }\n\n    public String getNamesrvDomainSubgroup() {\n        return namesrvDomainSubgroup;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.common;\n\nimport java.util.Random;\n\npublic class ThreadLocalIndex {\n    private final ThreadLocal<Integer> threadLocalIndex = new ThreadLocal<>();\n    private final Random random = new Random();\n    private final static int POSITIVE_MASK = 0x7FFFFFFF;\n\n    public int incrementAndGet() {\n        Integer index = this.threadLocalIndex.get();\n        if (null == index) {\n            index = random.nextInt();\n        }\n        this.threadLocalIndex.set(++index);\n        return index & POSITIVE_MASK;\n    }\n\n    public void reset() {\n        int index = Math.abs(random.nextInt(Integer.MAX_VALUE));\n        this.threadLocalIndex.set(index);\n    }\n\n    @Override\n    public String toString() {\n        return \"ThreadLocalIndex{\" +\n            \"threadLocalIndex=\" + threadLocalIndex.get() +\n            '}';\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/AckCallback.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\npublic interface AckCallback {\n    void onSuccess(final AckResult ackResult);\n\n    void onException(final Throwable e);\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/AckResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\n\npublic class AckResult {\n    private AckStatus status;\n    private String extraInfo;\n    private long popTime;\n\n    public void setPopTime(long popTime) {\n        this.popTime = popTime;\n    }\n\n    public long getPopTime() {\n        return popTime;\n    }\n\n    public AckStatus getStatus() {\n        return status;\n    }\n\n    public void setStatus(AckStatus status) {\n        this.status = status;\n    }\n\n    public void setExtraInfo(String extraInfo) {\n        this.extraInfo = extraInfo;\n    }\n\n    public String getExtraInfo() {\n        return extraInfo;\n    }\n\n    @Override\n    public String toString() {\n        return \"AckResult [AckStatus=\" + status + \",extraInfo=\" + extraInfo + \"]\";\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/AckStatus.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\npublic enum AckStatus {\n    /**\n     * ack success\n     */\n    OK,\n    /**\n     * msg not exist\n     */\n    NO_EXIST,\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/AllocateMessageQueueStrategy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\nimport java.util.List;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\n/**\n * Strategy Algorithm for message allocating between consumers\n */\npublic interface AllocateMessageQueueStrategy {\n\n    /**\n     * Allocating by consumer id\n     *\n     * @param consumerGroup current consumer group\n     * @param currentCID current consumer id\n     * @param mqAll message queue set in current topic\n     * @param cidAll consumer set in current consumer group\n     * @return The allocate result of given strategy\n     */\n    List<MessageQueue> allocate(\n        final String consumerGroup,\n        final String currentCID,\n        final List<MessageQueue> mqAll,\n        final List<String> cidAll\n    );\n\n    /**\n     * Algorithm name\n     *\n     * @return The strategy name\n     */\n    String getName();\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\nimport java.util.Collection;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely;\nimport org.apache.rocketmq.client.consumer.store.OffsetStore;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.consumer.DefaultLitePullConsumerImpl;\nimport org.apache.rocketmq.client.trace.AsyncTraceDispatcher;\nimport org.apache.rocketmq.client.trace.TraceDispatcher;\nimport org.apache.rocketmq.client.trace.hook.ConsumeMessageTraceHookImpl;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.filter.FilterAPI;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\nimport static org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData.SUB_ALL;\n\npublic class DefaultLitePullConsumer extends ClientConfig implements LitePullConsumer {\n\n    private static final Logger log = LoggerFactory.getLogger(DefaultLitePullConsumer.class);\n\n    private final DefaultLitePullConsumerImpl defaultLitePullConsumerImpl;\n\n    /**\n     * Consumers belonging to the same consumer group share a group id. The consumers in a group then divides the topic\n     * as fairly amongst themselves as possible by establishing that each queue is only consumed by a single consumer\n     * from the group. If all consumers are from the same group, it functions as a traditional message queue. Each\n     * message would be consumed by one consumer of the group only. When multiple consumer groups exist, the flow of the\n     * data consumption model aligns with the traditional publish-subscribe model. The messages are broadcast to all\n     * consumer groups.\n     */\n    private String consumerGroup;\n\n    /**\n     * Long polling mode, the Consumer connection max suspend time, it is not recommended to modify\n     */\n    private long brokerSuspendMaxTimeMillis = 1000 * 20;\n\n    /**\n     * Long polling mode, the Consumer connection timeout(must greater than brokerSuspendMaxTimeMillis), it is not\n     * recommended to modify\n     */\n    private long consumerTimeoutMillisWhenSuspend = 1000 * 30;\n\n    /**\n     * The socket timeout in milliseconds\n     */\n    private long consumerPullTimeoutMillis = 1000 * 10;\n\n    /**\n     * Consumption pattern,default is clustering\n     */\n    private MessageModel messageModel = MessageModel.CLUSTERING;\n    /**\n     * Message queue listener\n     */\n    private MessageQueueListener messageQueueListener;\n    /**\n     * Offset Storage\n     */\n    private OffsetStore offsetStore;\n\n    /**\n     * Queue allocation algorithm\n     */\n    private AllocateMessageQueueStrategy allocateMessageQueueStrategy = new AllocateMessageQueueAveragely();\n    /**\n     * Whether the unit of subscription group\n     */\n    private boolean unitMode = false;\n\n    /**\n     * The flag for auto commit offset\n     */\n    private boolean autoCommit = true;\n\n    /**\n     * Pull thread number\n     */\n    private int pullThreadNums = 20;\n\n    /**\n     * Minimum commit offset interval time in milliseconds.\n     */\n    private static final long MIN_AUTOCOMMIT_INTERVAL_MILLIS = 1000;\n\n    /**\n     * Maximum commit offset interval time in milliseconds.\n     */\n    private long autoCommitIntervalMillis = 5 * 1000;\n\n    /**\n     * Maximum number of messages pulled each time.\n     */\n    private int pullBatchSize = 10;\n\n    /**\n     * Flow control threshold for consume request, each consumer will cache at most 10000 consume requests by default.\n     * Consider the {@code pullBatchSize}, the instantaneous value may exceed the limit\n     */\n    private long pullThresholdForAll = 10000;\n\n    /**\n     * Consume max span offset.\n     */\n    private int consumeMaxSpan = 2000;\n\n    /**\n     * Flow control threshold on queue level, each message queue will cache at most 1000 messages by default, Consider\n     * the {@code pullBatchSize}, the instantaneous value may exceed the limit\n     */\n    private int pullThresholdForQueue = 1000;\n\n    /**\n     * Limit the cached message size on queue level, each message queue will cache at most 100 MiB messages by default,\n     * Consider the {@code pullBatchSize}, the instantaneous value may exceed the limit\n     *\n     * <p>\n     * The size of a message only measured by message body, so it's not accurate\n     */\n    private int pullThresholdSizeForQueue = 100;\n\n    /**\n     * The poll timeout in milliseconds\n     */\n    private long pollTimeoutMillis = 1000 * 5;\n\n    /**\n     * Interval time in in milliseconds for checking changes in topic metadata.\n     */\n    private long topicMetadataCheckIntervalMillis = 30 * 1000;\n\n    private ConsumeFromWhere consumeFromWhere = ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET;\n\n    /**\n     * Backtracking consumption time with second precision. Time format is 20131223171201<br> Implying Seventeen twelve\n     * and 01 seconds on December 23, 2013 year<br> Default backtracking consumption time Half an hour ago.\n     */\n    private String consumeTimestamp = UtilAll.timeMillisToHumanString3(System.currentTimeMillis() - (1000 * 60 * 30));\n\n    /**\n     * Interface of asynchronous transfer data\n     */\n    private TraceDispatcher traceDispatcher = null;\n\n    private RPCHook rpcHook;\n\n    private final Set<SubscriptionData> subscriptionsForHeartbeat = new HashSet<>();\n\n    /**\n     * Default constructor.\n     */\n    public DefaultLitePullConsumer() {\n        this(MixAll.DEFAULT_CONSUMER_GROUP, null);\n    }\n\n    /**\n     * Constructor specifying consumer group.\n     *\n     * @param consumerGroup Consumer group.\n     */\n    public DefaultLitePullConsumer(final String consumerGroup) {\n        this(consumerGroup, null);\n    }\n\n    /**\n     * Constructor specifying RPC hook.\n     *\n     * @param rpcHook RPC hook to execute before each remoting command.\n     */\n    public DefaultLitePullConsumer(RPCHook rpcHook) {\n        this(MixAll.DEFAULT_CONSUMER_GROUP, rpcHook);\n    }\n\n    /**\n     * Constructor specifying consumer group, RPC hook\n     *\n     * @param consumerGroup Consumer group.\n     * @param rpcHook       RPC hook to execute before each remoting command.\n     */\n    public DefaultLitePullConsumer(final String consumerGroup, RPCHook rpcHook) {\n        this.consumerGroup = consumerGroup;\n        this.rpcHook = rpcHook;\n        this.enableStreamRequestType = true;\n        defaultLitePullConsumerImpl = new DefaultLitePullConsumerImpl(this, rpcHook);\n    }\n\n    /**\n     * Constructor specifying namespace, consumer group and RPC hook.\n     *\n     * @param consumerGroup Consumer group.\n     * @param rpcHook       RPC hook to execute before each remoting command.\n     */\n    @Deprecated\n    public DefaultLitePullConsumer(final String namespace, final String consumerGroup, RPCHook rpcHook) {\n        this.namespace = namespace;\n        this.consumerGroup = consumerGroup;\n        this.rpcHook = rpcHook;\n        this.enableStreamRequestType = true;\n        defaultLitePullConsumerImpl = new DefaultLitePullConsumerImpl(this, rpcHook);\n    }\n\n    @Override\n    public void start() throws MQClientException {\n        setTraceDispatcher();\n        setConsumerGroup(NamespaceUtil.wrapNamespace(this.getNamespace(), this.consumerGroup));\n        this.defaultLitePullConsumerImpl.start();\n        if (null != traceDispatcher) {\n            try {\n                traceDispatcher.start(this.getNamesrvAddr(), this.getAccessChannel());\n            } catch (MQClientException e) {\n                log.warn(\"trace dispatcher start failed \", e);\n            }\n        }\n    }\n\n    @Override\n    public void shutdown() {\n        this.defaultLitePullConsumerImpl.shutdown();\n        if (null != traceDispatcher) {\n            traceDispatcher.shutdown();\n        }\n    }\n\n    @Override\n    public boolean isRunning() {\n        return this.defaultLitePullConsumerImpl.isRunning();\n    }\n\n    @Override\n    public void subscribe(String topic) throws MQClientException {\n        this.subscribe(topic, SUB_ALL);\n    }\n\n    @Override\n    public void subscribe(String topic, String subExpression) throws MQClientException {\n        this.defaultLitePullConsumerImpl.subscribe(withNamespace(topic), subExpression);\n    }\n\n    @Override\n    public void subscribe(String topic, MessageSelector messageSelector) throws MQClientException {\n        this.defaultLitePullConsumerImpl.subscribe(withNamespace(topic), messageSelector);\n    }\n\n    @Override\n    public void unsubscribe(String topic) {\n        this.defaultLitePullConsumerImpl.unsubscribe(withNamespace(topic));\n    }\n\n    @Override\n    public void assign(Collection<MessageQueue> messageQueues) {\n        defaultLitePullConsumerImpl.assign(queuesWithNamespace(messageQueues));\n    }\n\n    @Override\n    public void setSubExpressionForAssign(final String topic, final String subExpresion) {\n        defaultLitePullConsumerImpl.setSubExpressionForAssign(withNamespace(topic), subExpresion);\n    }\n\n    @Override\n    public List<MessageExt> poll() {\n        return defaultLitePullConsumerImpl.poll(this.getPollTimeoutMillis());\n    }\n\n    @Override\n    public List<MessageExt> poll(long timeout) {\n        return defaultLitePullConsumerImpl.poll(timeout);\n    }\n\n    @Override\n    public void seek(MessageQueue messageQueue, long offset) throws MQClientException {\n        this.defaultLitePullConsumerImpl.seek(queueWithNamespace(messageQueue), offset);\n    }\n\n    @Override\n    public void pause(Collection<MessageQueue> messageQueues) {\n        this.defaultLitePullConsumerImpl.pause(queuesWithNamespace(messageQueues));\n    }\n\n    @Override\n    public void resume(Collection<MessageQueue> messageQueues) {\n        this.defaultLitePullConsumerImpl.resume(queuesWithNamespace(messageQueues));\n    }\n\n    @Override\n    public Collection<MessageQueue> fetchMessageQueues(String topic) throws MQClientException {\n        return this.defaultLitePullConsumerImpl.fetchMessageQueues(withNamespace(topic));\n    }\n\n    @Override\n    public Long offsetForTimestamp(MessageQueue messageQueue, Long timestamp) throws MQClientException {\n        return this.defaultLitePullConsumerImpl.searchOffset(queueWithNamespace(messageQueue), timestamp);\n    }\n\n    @Override\n    public void registerTopicMessageQueueChangeListener(String topic,\n        TopicMessageQueueChangeListener topicMessageQueueChangeListener) throws MQClientException {\n        this.defaultLitePullConsumerImpl.registerTopicMessageQueueChangeListener(withNamespace(topic), topicMessageQueueChangeListener);\n    }\n\n    @Deprecated\n    @Override\n    public void commitSync() {\n        this.defaultLitePullConsumerImpl.commitAll();\n    }\n\n    @Deprecated\n    @Override\n    public void commitSync(Map<MessageQueue, Long> offsetMap, boolean persist) {\n        this.defaultLitePullConsumerImpl.commit(offsetMap, persist);\n    }\n\n    @Override\n    public void commit() {\n        this.defaultLitePullConsumerImpl.commitAll();\n    }\n\n    @Override\n    public void commit(Map<MessageQueue, Long> offsetMap, boolean persist) {\n        this.defaultLitePullConsumerImpl.commit(offsetMap, persist);\n    }\n\n    /**\n     * Get the MessageQueue assigned in subscribe mode\n     *\n     * @return\n     * @throws MQClientException\n     */\n    @Override\n    public Set<MessageQueue> assignment() throws MQClientException {\n        return this.defaultLitePullConsumerImpl.assignment();\n    }\n\n    /**\n     * Subscribe some topic with subExpression and messageQueueListener\n     *\n     * @param topic\n     * @param subExpression\n     * @param messageQueueListener\n     */\n    @Override\n    public void subscribe(String topic, String subExpression,\n        MessageQueueListener messageQueueListener) throws MQClientException {\n        this.defaultLitePullConsumerImpl.subscribe(withNamespace(topic), subExpression, messageQueueListener);\n    }\n\n    @Override\n    public void commit(final Set<MessageQueue> messageQueues, boolean persist) {\n        this.defaultLitePullConsumerImpl.commit(messageQueues, persist);\n    }\n\n    @Override\n    public Long committed(MessageQueue messageQueue) throws MQClientException {\n        return this.defaultLitePullConsumerImpl.committed(queueWithNamespace(messageQueue));\n    }\n\n    @Override\n    public void updateNameServerAddress(String nameServerAddress) {\n        this.defaultLitePullConsumerImpl.updateNameServerAddr(nameServerAddress);\n    }\n\n    @Override\n    public void seekToBegin(MessageQueue messageQueue) throws MQClientException {\n        this.defaultLitePullConsumerImpl.seekToBegin(queueWithNamespace(messageQueue));\n    }\n\n    @Override\n    public void seekToEnd(MessageQueue messageQueue) throws MQClientException {\n        this.defaultLitePullConsumerImpl.seekToEnd(queueWithNamespace(messageQueue));\n    }\n\n    @Override\n    public boolean isAutoCommit() {\n        return autoCommit;\n    }\n\n    @Override\n    public void setAutoCommit(boolean autoCommit) {\n        this.autoCommit = autoCommit;\n    }\n\n    public boolean isConnectBrokerByUser() {\n        return this.defaultLitePullConsumerImpl.getPullAPIWrapper().isConnectBrokerByUser();\n    }\n\n    public void setConnectBrokerByUser(boolean connectBrokerByUser) {\n        this.defaultLitePullConsumerImpl.getPullAPIWrapper().setConnectBrokerByUser(connectBrokerByUser);\n    }\n\n    public long getDefaultBrokerId() {\n        return this.defaultLitePullConsumerImpl.getPullAPIWrapper().getDefaultBrokerId();\n    }\n\n    public void setDefaultBrokerId(long defaultBrokerId) {\n        this.defaultLitePullConsumerImpl.getPullAPIWrapper().setDefaultBrokerId(defaultBrokerId);\n    }\n\n    public int getPullThreadNums() {\n        return pullThreadNums;\n    }\n\n    public void setPullThreadNums(int pullThreadNums) {\n        this.pullThreadNums = pullThreadNums;\n    }\n\n    public long getAutoCommitIntervalMillis() {\n        return autoCommitIntervalMillis;\n    }\n\n    public void setAutoCommitIntervalMillis(long autoCommitIntervalMillis) {\n        if (autoCommitIntervalMillis >= MIN_AUTOCOMMIT_INTERVAL_MILLIS) {\n            this.autoCommitIntervalMillis = autoCommitIntervalMillis;\n        }\n    }\n\n    public int getPullBatchSize() {\n        return pullBatchSize;\n    }\n\n    public void setPullBatchSize(int pullBatchSize) {\n        this.pullBatchSize = pullBatchSize;\n    }\n\n    public long getPullThresholdForAll() {\n        return pullThresholdForAll;\n    }\n\n    public void setPullThresholdForAll(long pullThresholdForAll) {\n        this.pullThresholdForAll = pullThresholdForAll;\n    }\n\n    public int getConsumeMaxSpan() {\n        return consumeMaxSpan;\n    }\n\n    public void setConsumeMaxSpan(int consumeMaxSpan) {\n        this.consumeMaxSpan = consumeMaxSpan;\n    }\n\n    public int getPullThresholdForQueue() {\n        return pullThresholdForQueue;\n    }\n\n    public void setPullThresholdForQueue(int pullThresholdForQueue) {\n        this.pullThresholdForQueue = pullThresholdForQueue;\n    }\n\n    public int getPullThresholdSizeForQueue() {\n        return pullThresholdSizeForQueue;\n    }\n\n    public void setPullThresholdSizeForQueue(int pullThresholdSizeForQueue) {\n        this.pullThresholdSizeForQueue = pullThresholdSizeForQueue;\n    }\n\n    public AllocateMessageQueueStrategy getAllocateMessageQueueStrategy() {\n        return allocateMessageQueueStrategy;\n    }\n\n    public void setAllocateMessageQueueStrategy(AllocateMessageQueueStrategy allocateMessageQueueStrategy) {\n        this.allocateMessageQueueStrategy = allocateMessageQueueStrategy;\n    }\n\n    public long getBrokerSuspendMaxTimeMillis() {\n        return brokerSuspendMaxTimeMillis;\n    }\n\n    public long getPollTimeoutMillis() {\n        return pollTimeoutMillis;\n    }\n\n    public void setPollTimeoutMillis(long pollTimeoutMillis) {\n        this.pollTimeoutMillis = pollTimeoutMillis;\n    }\n\n    public OffsetStore getOffsetStore() {\n        return offsetStore;\n    }\n\n    public void setOffsetStore(OffsetStore offsetStore) {\n        this.offsetStore = offsetStore;\n    }\n\n    @Override\n    public boolean isUnitMode() {\n        return unitMode;\n    }\n\n    @Override\n    public void setUnitMode(boolean isUnitMode) {\n        this.unitMode = isUnitMode;\n    }\n\n    public MessageModel getMessageModel() {\n        return messageModel;\n    }\n\n    public void setMessageModel(MessageModel messageModel) {\n        this.messageModel = messageModel;\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public MessageQueueListener getMessageQueueListener() {\n        return messageQueueListener;\n    }\n\n    public void setMessageQueueListener(MessageQueueListener messageQueueListener) {\n        this.messageQueueListener = messageQueueListener;\n    }\n\n    public long getConsumerPullTimeoutMillis() {\n        return consumerPullTimeoutMillis;\n    }\n\n    public void setConsumerPullTimeoutMillis(long consumerPullTimeoutMillis) {\n        this.consumerPullTimeoutMillis = consumerPullTimeoutMillis;\n    }\n\n    public long getConsumerTimeoutMillisWhenSuspend() {\n        return consumerTimeoutMillisWhenSuspend;\n    }\n\n    public void setConsumerTimeoutMillisWhenSuspend(long consumerTimeoutMillisWhenSuspend) {\n        this.consumerTimeoutMillisWhenSuspend = consumerTimeoutMillisWhenSuspend;\n    }\n\n    public long getTopicMetadataCheckIntervalMillis() {\n        return topicMetadataCheckIntervalMillis;\n    }\n\n    public void setTopicMetadataCheckIntervalMillis(long topicMetadataCheckIntervalMillis) {\n        this.topicMetadataCheckIntervalMillis = topicMetadataCheckIntervalMillis;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public ConsumeFromWhere getConsumeFromWhere() {\n        return consumeFromWhere;\n    }\n\n    public void setConsumeFromWhere(ConsumeFromWhere consumeFromWhere) {\n        if (consumeFromWhere != ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET\n            && consumeFromWhere != ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET\n            && consumeFromWhere != ConsumeFromWhere.CONSUME_FROM_TIMESTAMP) {\n            throw new RuntimeException(\"Invalid ConsumeFromWhere Value\", null);\n        }\n        this.consumeFromWhere = consumeFromWhere;\n    }\n\n    public String getConsumeTimestamp() {\n        return consumeTimestamp;\n    }\n\n    public void setConsumeTimestamp(String consumeTimestamp) {\n        this.consumeTimestamp = consumeTimestamp;\n    }\n\n    public TraceDispatcher getTraceDispatcher() {\n        return traceDispatcher;\n    }\n\n    private void setTraceDispatcher() {\n        if (enableTrace) {\n            try {\n                AsyncTraceDispatcher traceDispatcher = new AsyncTraceDispatcher(consumerGroup, TraceDispatcher.Type.CONSUME, getTraceMsgBatchNum(), traceTopic, rpcHook);\n                traceDispatcher.getTraceProducer().setUseTLS(this.isUseTLS());\n                traceDispatcher.setNamespaceV2(namespaceV2);\n                this.traceDispatcher = traceDispatcher;\n                this.defaultLitePullConsumerImpl.registerConsumeMessageHook(\n                    new ConsumeMessageTraceHookImpl(traceDispatcher));\n            } catch (Throwable e) {\n                log.error(\"system mqtrace hook init failed ,maybe can't send msg trace data\");\n            }\n        }\n    }\n\n    public String getCustomizedTraceTopic() {\n        return traceTopic;\n    }\n\n    public void setCustomizedTraceTopic(String customizedTraceTopic) {\n        this.traceTopic = customizedTraceTopic;\n    }\n\n    public boolean isEnableMsgTrace() {\n        return enableTrace;\n    }\n\n    public void setEnableMsgTrace(boolean enableMsgTrace) {\n        this.enableTrace = enableMsgTrace;\n    }\n\n    public Set<SubscriptionData> getSubscriptionsForHeartbeat() {\n        return this.subscriptionsForHeartbeat;\n    }\n\n    public synchronized void buildSubscriptionsForHeartbeat(Map<String, MessageSelector> messageSelectorMap) throws Exception {\n        this.subscriptionsForHeartbeat.clear();\n        for (Map.Entry<String, MessageSelector> entry : messageSelectorMap.entrySet()) {\n            SubscriptionData subscriptionData = FilterAPI.build(entry.getKey(),\n                entry.getValue().getExpression(), entry.getValue().getExpressionType());\n            this.subscriptionsForHeartbeat.add(subscriptionData);\n        }\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\nimport java.util.Collections;\nimport java.util.HashSet;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.QueryResult;\nimport org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely;\nimport org.apache.rocketmq.client.consumer.store.OffsetStore;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.consumer.DefaultMQPullConsumerImpl;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.filter.FilterAPI;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\n/**\n * @deprecated Default pulling consumer. This class will be removed in 2022, and a better implementation\n * {@link DefaultLitePullConsumer} is recommend to use in the scenario of actively pulling messages.\n */\n@Deprecated\npublic class DefaultMQPullConsumer extends ClientConfig implements MQPullConsumer {\n\n    protected final transient DefaultMQPullConsumerImpl defaultMQPullConsumerImpl;\n\n    /**\n     * Do the same thing for the same Group, the application must be set,and guarantee Globally unique\n     */\n    private String consumerGroup;\n    /**\n     * Long polling mode, the Consumer connection max suspend time, it is not recommended to modify\n     */\n    private long brokerSuspendMaxTimeMillis = 1000 * 20;\n    /**\n     * Long polling mode, the Consumer connection timeout(must greater than brokerSuspendMaxTimeMillis), it is not\n     * recommended to modify\n     */\n    private long consumerTimeoutMillisWhenSuspend = 1000 * 30;\n    /**\n     * The socket timeout in milliseconds\n     */\n    private long consumerPullTimeoutMillis = 1000 * 10;\n    /**\n     * Consumption pattern,default is clustering\n     */\n    private MessageModel messageModel = MessageModel.CLUSTERING;\n    /**\n     * Message queue listener\n     */\n    private MessageQueueListener messageQueueListener;\n    /**\n     * Offset Storage\n     */\n    private OffsetStore offsetStore;\n    /**\n     * Topic set you want to register\n     */\n    private Set<String> registerTopics = new HashSet<>();\n\n    private final Set<SubscriptionData> registerSubscriptions = Collections.newSetFromMap(new ConcurrentHashMap<>());\n    /**\n     * Queue allocation algorithm\n     */\n    private AllocateMessageQueueStrategy allocateMessageQueueStrategy = new AllocateMessageQueueAveragely();\n    /**\n     * Whether the unit of subscription group\n     */\n    private boolean unitMode = false;\n\n    private int maxReconsumeTimes = 16;\n\n    private boolean enableRebalance = true;\n\n    public DefaultMQPullConsumer() {\n        this(MixAll.DEFAULT_CONSUMER_GROUP, null);\n    }\n\n    public DefaultMQPullConsumer(final String consumerGroup) {\n        this(consumerGroup, null);\n    }\n\n    public DefaultMQPullConsumer(RPCHook rpcHook) {\n        this(MixAll.DEFAULT_CONSUMER_GROUP, rpcHook);\n    }\n\n    public DefaultMQPullConsumer(final String consumerGroup, RPCHook rpcHook) {\n        this.consumerGroup = consumerGroup;\n        this.enableStreamRequestType = true;\n        defaultMQPullConsumerImpl = new DefaultMQPullConsumerImpl(this, rpcHook);\n    }\n\n    /**\n     * Constructor specifying namespace, consumer group and RPC hook.\n     *\n     * @param consumerGroup Consumer group.\n     * @param rpcHook RPC hook to execute before each remoting command.\n     */\n    public DefaultMQPullConsumer(final String namespace, final String consumerGroup, RPCHook rpcHook) {\n        this.namespace = namespace;\n        this.consumerGroup = consumerGroup;\n        this.enableStreamRequestType = true;\n        defaultMQPullConsumerImpl = new DefaultMQPullConsumerImpl(this, rpcHook);\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    @Override\n    public void createTopic(String key, String newTopic, int queueNum,\n        Map<String, String> attributes) throws MQClientException {\n        createTopic(key, withNamespace(newTopic), queueNum, 0, null);\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    @Override\n    public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag,\n        Map<String, String> attributes) throws MQClientException {\n        this.defaultMQPullConsumerImpl.createTopic(key, withNamespace(newTopic), queueNum, topicSysFlag);\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    @Override\n    public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException {\n        return this.defaultMQPullConsumerImpl.searchOffset(queueWithNamespace(mq), timestamp);\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    @Override\n    public long maxOffset(MessageQueue mq) throws MQClientException {\n        return this.defaultMQPullConsumerImpl.maxOffset(queueWithNamespace(mq));\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    @Override\n    public long minOffset(MessageQueue mq) throws MQClientException {\n        return this.defaultMQPullConsumerImpl.minOffset(queueWithNamespace(mq));\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    @Override\n    public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException {\n        return this.defaultMQPullConsumerImpl.earliestMsgStoreTime(queueWithNamespace(mq));\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    @Override\n    public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end)\n        throws MQClientException, InterruptedException {\n        return this.defaultMQPullConsumerImpl.queryMessage(withNamespace(topic), key, maxNum, begin, end);\n    }\n\n    public AllocateMessageQueueStrategy getAllocateMessageQueueStrategy() {\n        return allocateMessageQueueStrategy;\n    }\n\n    public void setAllocateMessageQueueStrategy(AllocateMessageQueueStrategy allocateMessageQueueStrategy) {\n        this.allocateMessageQueueStrategy = allocateMessageQueueStrategy;\n    }\n\n    public long getBrokerSuspendMaxTimeMillis() {\n        return brokerSuspendMaxTimeMillis;\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    public void setBrokerSuspendMaxTimeMillis(long brokerSuspendMaxTimeMillis) {\n        this.brokerSuspendMaxTimeMillis = brokerSuspendMaxTimeMillis;\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public long getConsumerPullTimeoutMillis() {\n        return consumerPullTimeoutMillis;\n    }\n\n    public void setConsumerPullTimeoutMillis(long consumerPullTimeoutMillis) {\n        this.consumerPullTimeoutMillis = consumerPullTimeoutMillis;\n    }\n\n    public long getConsumerTimeoutMillisWhenSuspend() {\n        return consumerTimeoutMillisWhenSuspend;\n    }\n\n    public void setConsumerTimeoutMillisWhenSuspend(long consumerTimeoutMillisWhenSuspend) {\n        this.consumerTimeoutMillisWhenSuspend = consumerTimeoutMillisWhenSuspend;\n    }\n\n    public MessageModel getMessageModel() {\n        return messageModel;\n    }\n\n    public void setMessageModel(MessageModel messageModel) {\n        this.messageModel = messageModel;\n    }\n\n    public MessageQueueListener getMessageQueueListener() {\n        return messageQueueListener;\n    }\n\n    public void setMessageQueueListener(MessageQueueListener messageQueueListener) {\n        this.messageQueueListener = messageQueueListener;\n    }\n\n    public Set<String> getRegisterTopics() {\n        return registerTopics;\n    }\n\n    public void setRegisterTopics(Set<String> registerTopics) {\n        this.registerTopics = withNamespace(registerTopics);\n    }\n\n    public Set<SubscriptionData> getRegisterSubscriptions() {\n        return registerSubscriptions;\n    }\n\n    public void addRegisterSubscriptions(String topic, MessageSelector messageSelector) throws MQClientException {\n        try {\n            if (messageSelector == null) {\n                messageSelector = MessageSelector.byTag(SubscriptionData.SUB_ALL);\n            }\n\n            SubscriptionData subscriptionData = FilterAPI.build(withNamespace(topic),\n                messageSelector.getExpression(), messageSelector.getExpressionType());\n\n            this.registerSubscriptions.add(subscriptionData);\n        } catch (Exception e) {\n            throw new MQClientException(\"add subscription exception\", e);\n        }\n    }\n\n    public void clearRegisterSubscriptions() {\n        this.registerSubscriptions.clear();\n    }\n\n    /**\n     * This method will be removed or it's visibility will be changed in a certain version after April 5, 2020, so\n     * please do not use this method.\n     */\n    @Deprecated\n    @Override\n    public void sendMessageBack(MessageExt msg, int delayLevel)\n        throws RemotingException, MQBrokerException, InterruptedException, MQClientException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        this.defaultMQPullConsumerImpl.sendMessageBack(msg, delayLevel, msg.getBrokerName());\n    }\n\n    /**\n     * This method will be removed or it's visibility will be changed in a certain version after April 5, 2020, so\n     * please do not use this method.\n     */\n    @Deprecated\n    @Override\n    public void sendMessageBack(MessageExt msg, int delayLevel, String brokerName)\n        throws RemotingException, MQBrokerException, InterruptedException, MQClientException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        this.defaultMQPullConsumerImpl.sendMessageBack(msg, delayLevel, brokerName);\n    }\n\n    @Override\n    public Set<MessageQueue> fetchSubscribeMessageQueues(String topic) throws MQClientException {\n        return this.defaultMQPullConsumerImpl.fetchSubscribeMessageQueues(withNamespace(topic));\n    }\n\n    @Override\n    public void start() throws MQClientException {\n        this.setConsumerGroup(NamespaceUtil.wrapNamespace(this.getNamespace(), this.consumerGroup));\n        this.defaultMQPullConsumerImpl.start();\n    }\n\n    @Override\n    public void shutdown() {\n        this.defaultMQPullConsumerImpl.shutdown();\n    }\n\n    @Override\n    public void registerMessageQueueListener(String topic, MessageQueueListener listener) {\n        synchronized (this.registerTopics) {\n            this.registerTopics.add(withNamespace(topic));\n            if (listener != null) {\n                this.messageQueueListener = listener;\n            }\n        }\n    }\n\n    @Override\n    public PullResult pull(MessageQueue mq, String subExpression, long offset, int maxNums)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        return this.defaultMQPullConsumerImpl.pull(queueWithNamespace(mq), subExpression, offset, maxNums);\n    }\n\n    @Override\n    public PullResult pull(MessageQueue mq, String subExpression, long offset, int maxNums, long timeout)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        return this.defaultMQPullConsumerImpl.pull(queueWithNamespace(mq), subExpression, offset, maxNums, timeout);\n    }\n\n    @Override\n    public PullResult pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        return this.defaultMQPullConsumerImpl.pull(queueWithNamespace(mq), messageSelector, offset, maxNums);\n    }\n\n    @Override\n    public PullResult pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums, long timeout)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        return this.defaultMQPullConsumerImpl.pull(queueWithNamespace(mq), messageSelector, offset, maxNums, timeout);\n    }\n\n    @Override\n    public void pull(MessageQueue mq, String subExpression, long offset, int maxNums, PullCallback pullCallback)\n        throws MQClientException, RemotingException, InterruptedException {\n        this.defaultMQPullConsumerImpl.pull(queueWithNamespace(mq), subExpression, offset, maxNums, pullCallback);\n    }\n\n    @Override\n    public void pull(MessageQueue mq, String subExpression, long offset, int maxNums, PullCallback pullCallback,\n        long timeout)\n        throws MQClientException, RemotingException, InterruptedException {\n        this.defaultMQPullConsumerImpl.pull(queueWithNamespace(mq), subExpression, offset, maxNums, pullCallback, timeout);\n    }\n\n    @Override\n    public void pull(MessageQueue mq, String subExpression, long offset, int maxNums, int maxSize,\n        PullCallback pullCallback,\n        long timeout)\n        throws MQClientException, RemotingException, InterruptedException {\n        this.defaultMQPullConsumerImpl.pull(mq, subExpression, offset, maxNums, maxSize, pullCallback, timeout);\n    }\n\n    @Override\n    public void pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums,\n        PullCallback pullCallback)\n        throws MQClientException, RemotingException, InterruptedException {\n        this.defaultMQPullConsumerImpl.pull(queueWithNamespace(mq), messageSelector, offset, maxNums, pullCallback);\n    }\n\n    @Override\n    public void pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums,\n        PullCallback pullCallback, long timeout)\n        throws MQClientException, RemotingException, InterruptedException {\n        this.defaultMQPullConsumerImpl.pull(queueWithNamespace(mq), messageSelector, offset, maxNums, pullCallback, timeout);\n    }\n\n    @Override\n    public PullResult pullBlockIfNotFound(MessageQueue mq, String subExpression, long offset, int maxNums)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        return this.defaultMQPullConsumerImpl.pullBlockIfNotFound(queueWithNamespace(mq), subExpression, offset, maxNums);\n    }\n\n    @Override\n    public void pullBlockIfNotFound(MessageQueue mq, String subExpression, long offset, int maxNums,\n        PullCallback pullCallback)\n        throws MQClientException, RemotingException, InterruptedException {\n        this.defaultMQPullConsumerImpl.pullBlockIfNotFound(queueWithNamespace(mq), subExpression, offset, maxNums, pullCallback);\n    }\n\n    @Override\n    public void pullBlockIfNotFoundWithMessageSelector(MessageQueue mq, MessageSelector selector,\n        long offset, int maxNums,\n        PullCallback pullCallback) throws MQClientException, RemotingException, InterruptedException {\n        this.defaultMQPullConsumerImpl.pullBlockIfNotFoundWithMessageSelector(mq, selector, offset, maxNums, pullCallback);\n    }\n\n    @Override\n    public PullResult pullBlockIfNotFoundWithMessageSelector(MessageQueue mq, MessageSelector selector,\n        long offset,\n        int maxNums) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        return this.defaultMQPullConsumerImpl.pullBlockIfNotFoundWithMessageSelector(mq, selector, offset, maxNums);\n    }\n\n    @Override\n    public void updateConsumeOffset(MessageQueue mq, long offset) throws MQClientException {\n        this.defaultMQPullConsumerImpl.updateConsumeOffset(queueWithNamespace(mq), offset);\n    }\n\n    @Override\n    public long fetchConsumeOffset(MessageQueue mq, boolean fromStore) throws MQClientException {\n        return this.defaultMQPullConsumerImpl.fetchConsumeOffset(queueWithNamespace(mq), fromStore);\n    }\n\n    @Override\n    public Set<MessageQueue> fetchMessageQueuesInBalance(String topic) throws MQClientException {\n        return this.defaultMQPullConsumerImpl.fetchMessageQueuesInBalance(withNamespace(topic));\n    }\n\n    @Override\n    public MessageExt viewMessage(String topic,\n        String uniqKey) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {\n        try {\n            MessageDecoder.decodeMessageId(uniqKey);\n            return this.defaultMQPullConsumerImpl.viewMessage(topic, uniqKey);\n        } catch (Exception e) {\n            // Ignore\n        }\n        return this.defaultMQPullConsumerImpl.queryMessageByUniqKey(withNamespace(topic), uniqKey);\n    }\n\n    @Override\n    public void sendMessageBack(MessageExt msg, int delayLevel, String brokerName, String consumerGroup)\n        throws RemotingException, MQBrokerException, InterruptedException, MQClientException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        this.defaultMQPullConsumerImpl.sendMessageBack(msg, delayLevel, brokerName, consumerGroup);\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    public OffsetStore getOffsetStore() {\n        return offsetStore;\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    public void setOffsetStore(OffsetStore offsetStore) {\n        this.offsetStore = offsetStore;\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    public DefaultMQPullConsumerImpl getDefaultMQPullConsumerImpl() {\n        return defaultMQPullConsumerImpl;\n    }\n\n    @Override\n    public boolean isUnitMode() {\n        return unitMode;\n    }\n\n    @Override\n    public void setUnitMode(boolean isUnitMode) {\n        this.unitMode = isUnitMode;\n    }\n\n    public int getMaxReconsumeTimes() {\n        return maxReconsumeTimes;\n    }\n\n    public void setMaxReconsumeTimes(final int maxReconsumeTimes) {\n        this.maxReconsumeTimes = maxReconsumeTimes;\n    }\n\n    public void persist(MessageQueue mq) {\n        this.getOffsetStore().persist(queueWithNamespace(mq));\n    }\n\n    public boolean isEnableRebalance() {\n        return enableRebalance;\n    }\n\n    public void setEnableRebalance(boolean enableRebalance) {\n        this.enableRebalance = enableRebalance;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.QueryResult;\nimport org.apache.rocketmq.client.consumer.listener.MessageListener;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;\nimport org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely;\nimport org.apache.rocketmq.client.consumer.store.OffsetStore;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.hook.ConsumeMessageHook;\nimport org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl;\nimport org.apache.rocketmq.client.trace.AsyncTraceDispatcher;\nimport org.apache.rocketmq.client.trace.TraceDispatcher;\nimport org.apache.rocketmq.client.trace.hook.ConsumeMessageTraceHookImpl;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\n\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.Map.Entry;\nimport java.util.Set;\n\n/**\n * In most scenarios, this is the mostly recommended class to consume messages.\n * </p>\n * Technically speaking, this push client is virtually a wrapper of the underlying pull service. Specifically, on\n * arrival of messages pulled from brokers, it roughly invokes the registered callback handler to feed the messages.\n * </p>\n * See quickstart/Consumer in the example module for a typical usage.\n * </p>\n *\n * <p>\n * <strong>Thread Safety:</strong> After initialization, the instance can be regarded as thread-safe.\n * </p>\n */\npublic class DefaultMQPushConsumer extends ClientConfig implements MQPushConsumer {\n\n    private final Logger log = LoggerFactory.getLogger(DefaultMQPushConsumer.class);\n\n    /**\n     * Internal implementation. Most of the functions herein are delegated to it.\n     */\n    protected final transient DefaultMQPushConsumerImpl defaultMQPushConsumerImpl;\n\n    /**\n     * Consumers of the same role is required to have exactly same subscriptions and consumerGroup to correctly achieve\n     * load balance. It's required and needs to be globally unique.\n     * </p>\n     * See <a href=\"https://rocketmq.apache.org/docs/introduction/02concepts\">here</a> for further discussion.\n     */\n    private String consumerGroup;\n\n    /**\n     * Message model defines the way how messages are delivered to each consumer clients.\n     * </p>\n     * RocketMQ supports two message models: clustering and broadcasting. If clustering is set, consumer clients with\n     * the same {@link #consumerGroup} would only consume shards of the messages subscribed, which achieves load\n     * balances; Conversely, if the broadcasting is set, each consumer client will consume all subscribed messages\n     * separately.\n     * </p>\n     * This field defaults to clustering.\n     */\n    private MessageModel messageModel = MessageModel.CLUSTERING;\n\n    /**\n     * Consuming point on consumer booting.\n     * </p>\n     * There are three consuming points:\n     * <ul>\n     * <li>\n     * <code>CONSUME_FROM_LAST_OFFSET</code>: consumer clients pick up where it stopped previously.\n     * If it were a newly booting up consumer client, according aging of the consumer group, there are two\n     * cases:\n     * <ol>\n     * <li>\n     * if the consumer group is created so recently that the earliest message being subscribed has yet\n     * expired, which means the consumer group represents a lately launched business, consuming will\n     * start from the very beginning;\n     * </li>\n     * <li>\n     * if the earliest message being subscribed has expired, consuming will start from the latest\n     * messages, meaning messages born prior to the booting timestamp would be ignored.\n     * </li>\n     * </ol>\n     * </li>\n     * <li>\n     * <code>CONSUME_FROM_FIRST_OFFSET</code>: Consumer client will start from earliest messages available.\n     * </li>\n     * <li>\n     * <code>CONSUME_FROM_TIMESTAMP</code>: Consumer client will start from specified timestamp, which means\n     * messages born prior to {@link #consumeTimestamp} will be ignored\n     * </li>\n     * </ul>\n     */\n    private ConsumeFromWhere consumeFromWhere = ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET;\n\n    /**\n     * Backtracking consumption time with second precision. Time format is\n     * 20131223171201<br>\n     * Implying Seventeen twelve and 01 seconds on December 23, 2013 year<br>\n     * Default backtracking consumption time Half an hour ago.\n     */\n    private String consumeTimestamp = UtilAll.timeMillisToHumanString3(System.currentTimeMillis() - (1000 * 60 * 30));\n\n    /**\n     * Queue allocation algorithm specifying how message queues are allocated to each consumer clients.\n     */\n    private AllocateMessageQueueStrategy allocateMessageQueueStrategy;\n\n    /**\n     * Subscription relationship\n     */\n    private Map<String /* topic */, String /* sub expression */> subscription = new HashMap<>();\n\n    /**\n     * Message listener\n     */\n    private MessageListener messageListener;\n\n    /**\n     * Listener to call if message queue assignment is changed.\n     */\n    private MessageQueueListener messageQueueListener;\n\n    /**\n     * Offset Storage\n     */\n    private OffsetStore offsetStore;\n\n    /**\n     * Minimum consumer thread number\n     */\n    private int consumeThreadMin = 20;\n\n    /**\n     * Max consumer thread number\n     */\n    private int consumeThreadMax = 20;\n\n    /**\n     * Threshold for dynamic adjustment of the number of thread pool\n     */\n    private long adjustThreadPoolNumsThreshold = 100000;\n\n    /**\n     * Concurrently max span offset.it has no effect on sequential consumption\n     */\n    private int consumeConcurrentlyMaxSpan = 2000;\n\n    /**\n     * Flow control threshold on queue level, each message queue will cache at most 1000 messages by default,\n     * Consider the {@code pullBatchSize}, the instantaneous value may exceed the limit\n     */\n    private int pullThresholdForQueue = 1000;\n\n    /**\n     * Flow control threshold on queue level, means max num of messages waiting to ack.\n     * in contrast with pull threshold, once a message is popped, it's considered the beginning of consumption.\n     */\n    private int popThresholdForQueue = 96;\n\n    /**\n     * Limit the cached message size on queue level, each message queue will cache at most 100 MiB messages by default,\n     * Consider the {@code pullBatchSize}, the instantaneous value may exceed the limit\n     *\n     * <p>\n     * The size(MB) of a message only measured by message body, so it's not accurate\n     */\n    private int pullThresholdSizeForQueue = 100;\n\n    /**\n     * Flow control threshold on topic level, default value is -1(Unlimited)\n     * <p>\n     * The value of {@code pullThresholdForQueue} will be overwritten and calculated based on\n     * {@code pullThresholdForTopic} if it isn't unlimited\n     * <p>\n     * For example, if the value of pullThresholdForTopic is 1000 and 10 message queues are assigned to this consumer,\n     * then pullThresholdForQueue will be set to 100\n     */\n    private int pullThresholdForTopic = -1;\n\n    /**\n     * Limit the cached message size on topic level, default value is -1 MiB(Unlimited)\n     * <p>\n     * The value of {@code pullThresholdSizeForQueue} will be overwritten and calculated based on\n     * {@code pullThresholdSizeForTopic} if it isn't unlimited\n     * <p>\n     * For example, if the value of pullThresholdSizeForTopic is 1000 MiB and 10 message queues are\n     * assigned to this consumer, then pullThresholdSizeForQueue will be set to 100 MiB\n     */\n    private int pullThresholdSizeForTopic = -1;\n\n    /**\n     * Message pull Interval\n     */\n    private long pullInterval = 0;\n\n    /**\n     * Batch consumption size\n     */\n    private int consumeMessageBatchMaxSize = 1;\n\n    /**\n     * Batch pull size\n     */\n    private int pullBatchSize = 32;\n\n    private int pullBatchSizeInBytes = 256 * 1024;\n\n    /**\n     * Whether update subscription relationship when every pull\n     */\n    private boolean postSubscriptionWhenPull = false;\n\n    /**\n     * Whether the unit of subscription group\n     */\n    private boolean unitMode = false;\n\n    /**\n     * Max re-consume times.\n     * In concurrently mode, -1 means 16;\n     * In orderly mode, -1 means Integer.MAX_VALUE.\n     * If messages are re-consumed more than {@link #maxReconsumeTimes} before success.\n     */\n    private int maxReconsumeTimes = -1;\n\n    /**\n     * Suspending pulling time for cases requiring slow pulling like flow-control scenario.\n     */\n    private long suspendCurrentQueueTimeMillis = 1000;\n\n    /**\n     * Maximum amount of time in minutes a message may block the consuming thread.\n     */\n    private long consumeTimeout = 15;\n\n    /**\n     * Maximum amount of invisible time in millisecond of a message, rang is [5000, 300000]\n     */\n    private long popInvisibleTime = 60000;\n\n    /**\n     * Batch pop size. range is [1, 32]\n     */\n    private int popBatchNums = 32;\n\n    /**\n     * Maximum time to await message consuming when shutdown consumer, 0 indicates no await.\n     */\n    private long awaitTerminationMillisWhenShutdown = 0;\n\n    /**\n     * Interface of asynchronous transfer data\n     */\n    private TraceDispatcher traceDispatcher = null;\n\n    // force to use client rebalance\n    private boolean clientRebalance = true;\n\n    private RPCHook rpcHook = null;\n\n    /**\n     * Default constructor.\n     */\n    public DefaultMQPushConsumer() {\n        this(MixAll.DEFAULT_CONSUMER_GROUP, null, new AllocateMessageQueueAveragely());\n    }\n\n    /**\n     * Constructor specifying consumer group.\n     *\n     * @param consumerGroup Consumer group.\n     */\n    public DefaultMQPushConsumer(final String consumerGroup) {\n        this(consumerGroup, null, new AllocateMessageQueueAveragely());\n    }\n\n    /**\n     * Constructor specifying RPC hook.\n     *\n     * @param rpcHook RPC hook to execute before each remoting command.\n     */\n    public DefaultMQPushConsumer(RPCHook rpcHook) {\n        this(MixAll.DEFAULT_CONSUMER_GROUP, rpcHook, new AllocateMessageQueueAveragely());\n    }\n\n    /**\n     * Constructor specifying consumer group, RPC hook.\n     *\n     * @param consumerGroup Consumer group.\n     * @param rpcHook       RPC hook to execute before each remoting command.\n     */\n    public DefaultMQPushConsumer(final String consumerGroup, RPCHook rpcHook) {\n        this(consumerGroup, rpcHook, new AllocateMessageQueueAveragely());\n    }\n\n    /**\n     * Constructor specifying consumer group, enabled msg trace flag and customized trace topic name.\n     *\n     * @param consumerGroup        Consumer group.\n     * @param enableMsgTrace       Switch flag instance for message trace.\n     * @param customizedTraceTopic The name value of message trace topic.If you don't config,you can use the default trace topic name.\n     */\n    public DefaultMQPushConsumer(final String consumerGroup, boolean enableMsgTrace,\n        final String customizedTraceTopic) {\n        this(consumerGroup, null, new AllocateMessageQueueAveragely(), enableMsgTrace, customizedTraceTopic);\n    }\n\n    /**\n     * Constructor specifying consumer group, RPC hook and message queue allocating algorithm.\n     *\n     * @param consumerGroup                Consumer group.\n     * @param rpcHook                      RPC hook to execute before each remoting command.\n     * @param allocateMessageQueueStrategy Message queue allocating algorithm.\n     */\n    public DefaultMQPushConsumer(final String consumerGroup, RPCHook rpcHook,\n        AllocateMessageQueueStrategy allocateMessageQueueStrategy) {\n        this(consumerGroup, rpcHook, allocateMessageQueueStrategy, false, null);\n    }\n\n    /**\n     * Constructor specifying consumer group, RPC hook, message queue allocating algorithm, enabled msg trace flag and customized trace topic name.\n     *\n     * @param consumerGroup                Consumer group.\n     * @param rpcHook                      RPC hook to execute before each remoting command.\n     * @param allocateMessageQueueStrategy message queue allocating algorithm.\n     * @param enableMsgTrace               Switch flag instance for message trace.\n     * @param customizedTraceTopic         The name value of message trace topic.If you don't config,you can use the default trace topic name.\n     */\n    public DefaultMQPushConsumer(final String consumerGroup, RPCHook rpcHook,\n        AllocateMessageQueueStrategy allocateMessageQueueStrategy, boolean enableMsgTrace,\n        final String customizedTraceTopic) {\n        this.consumerGroup = consumerGroup;\n        this.rpcHook = rpcHook;\n        this.allocateMessageQueueStrategy = allocateMessageQueueStrategy;\n        defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(this, rpcHook);\n        this.enableTrace = enableMsgTrace;\n        this.traceTopic = customizedTraceTopic;\n    }\n\n    /**\n     * Constructor specifying namespace and consumer group.\n     *\n     * @param namespace     Namespace for this MQ Producer instance.\n     * @param consumerGroup Consumer group.\n     */\n    @Deprecated\n    public DefaultMQPushConsumer(final String namespace, final String consumerGroup) {\n        this(namespace, consumerGroup, null, new AllocateMessageQueueAveragely());\n    }\n\n    /**\n     * Constructor specifying namespace, consumer group and RPC hook .\n     *\n     * @param namespace     Namespace for this MQ Producer instance.\n     * @param consumerGroup Consumer group.\n     * @param rpcHook       RPC hook to execute before each remoting command.\n     */\n    @Deprecated\n    public DefaultMQPushConsumer(final String namespace, final String consumerGroup, RPCHook rpcHook) {\n        this(namespace, consumerGroup, rpcHook, new AllocateMessageQueueAveragely());\n    }\n\n    /**\n     * Constructor specifying namespace, consumer group, RPC hook and message queue allocating algorithm.\n     *\n     * @param namespace                    Namespace for this MQ Producer instance.\n     * @param consumerGroup                Consumer group.\n     * @param rpcHook                      RPC hook to execute before each remoting command.\n     * @param allocateMessageQueueStrategy Message queue allocating algorithm.\n     */\n    @Deprecated\n    public DefaultMQPushConsumer(final String namespace, final String consumerGroup, RPCHook rpcHook,\n        AllocateMessageQueueStrategy allocateMessageQueueStrategy) {\n        this.consumerGroup = consumerGroup;\n        this.namespace = namespace;\n        this.rpcHook = rpcHook;\n        this.allocateMessageQueueStrategy = allocateMessageQueueStrategy;\n        defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(this, rpcHook);\n    }\n\n    /**\n     * Constructor specifying namespace, consumer group, RPC hook, message queue allocating algorithm, enabled msg trace flag and customized trace topic name.\n     *\n     * @param namespace                    Namespace for this MQ Producer instance.\n     * @param consumerGroup                Consumer group.\n     * @param rpcHook                      RPC hook to execute before each remoting command.\n     * @param allocateMessageQueueStrategy message queue allocating algorithm.\n     * @param enableMsgTrace               Switch flag instance for message trace.\n     * @param customizedTraceTopic         The name value of message trace topic.If you don't config,you can use the default trace topic name.\n     */\n    @Deprecated\n    public DefaultMQPushConsumer(final String namespace, final String consumerGroup, RPCHook rpcHook,\n        AllocateMessageQueueStrategy allocateMessageQueueStrategy, boolean enableMsgTrace,\n        final String customizedTraceTopic) {\n        this.consumerGroup = consumerGroup;\n        this.namespace = namespace;\n        this.rpcHook = rpcHook;\n        this.allocateMessageQueueStrategy = allocateMessageQueueStrategy;\n        defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(this, rpcHook);\n        this.enableTrace = enableMsgTrace;\n        this.traceTopic = customizedTraceTopic;\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    @Override\n    public void createTopic(String key, String newTopic, int queueNum,\n        Map<String, String> attributes) throws MQClientException {\n        createTopic(key, withNamespace(newTopic), queueNum, 0, null);\n    }\n\n    @Override\n    public void setUseTLS(boolean useTLS) {\n        super.setUseTLS(useTLS);\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    @Override\n    public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag,\n        Map<String, String> attributes) throws MQClientException {\n        this.defaultMQPushConsumerImpl.createTopic(key, withNamespace(newTopic), queueNum, topicSysFlag);\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    @Override\n    public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException {\n        return this.defaultMQPushConsumerImpl.searchOffset(queueWithNamespace(mq), timestamp);\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    @Override\n    public long maxOffset(MessageQueue mq) throws MQClientException {\n        return this.defaultMQPushConsumerImpl.maxOffset(queueWithNamespace(mq));\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    @Override\n    public long minOffset(MessageQueue mq) throws MQClientException {\n        return this.defaultMQPushConsumerImpl.minOffset(queueWithNamespace(mq));\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    @Override\n    public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException {\n        return this.defaultMQPushConsumerImpl.earliestMsgStoreTime(queueWithNamespace(mq));\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    @Override\n    public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end)\n        throws MQClientException, InterruptedException {\n        return this.defaultMQPushConsumerImpl.queryMessage(withNamespace(topic), key, maxNum, begin, end);\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    @Override\n    public MessageExt viewMessage(String topic,\n        String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {\n        try {\n            MessageDecoder.decodeMessageId(msgId);\n            return this.defaultMQPushConsumerImpl.viewMessage(withNamespace(topic), msgId);\n        } catch (Exception e) {\n            // Ignore\n        }\n        return this.defaultMQPushConsumerImpl.queryMessageByUniqKey(withNamespace(topic), msgId);\n    }\n\n    public AllocateMessageQueueStrategy getAllocateMessageQueueStrategy() {\n        return allocateMessageQueueStrategy;\n    }\n\n    public void setAllocateMessageQueueStrategy(AllocateMessageQueueStrategy allocateMessageQueueStrategy) {\n        this.allocateMessageQueueStrategy = allocateMessageQueueStrategy;\n    }\n\n    public int getConsumeConcurrentlyMaxSpan() {\n        return consumeConcurrentlyMaxSpan;\n    }\n\n    public void setConsumeConcurrentlyMaxSpan(int consumeConcurrentlyMaxSpan) {\n        this.consumeConcurrentlyMaxSpan = consumeConcurrentlyMaxSpan;\n    }\n\n    public ConsumeFromWhere getConsumeFromWhere() {\n        return consumeFromWhere;\n    }\n\n    public void setConsumeFromWhere(ConsumeFromWhere consumeFromWhere) {\n        this.consumeFromWhere = consumeFromWhere;\n    }\n\n    public int getConsumeMessageBatchMaxSize() {\n        return consumeMessageBatchMaxSize;\n    }\n\n    public void setConsumeMessageBatchMaxSize(int consumeMessageBatchMaxSize) {\n        this.consumeMessageBatchMaxSize = consumeMessageBatchMaxSize;\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public int getConsumeThreadMax() {\n        return consumeThreadMax;\n    }\n\n    public void setConsumeThreadMax(int consumeThreadMax) {\n        this.consumeThreadMax = consumeThreadMax;\n    }\n\n    public int getConsumeThreadMin() {\n        return consumeThreadMin;\n    }\n\n    public void setConsumeThreadMin(int consumeThreadMin) {\n        this.consumeThreadMin = consumeThreadMin;\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    public DefaultMQPushConsumerImpl getDefaultMQPushConsumerImpl() {\n        return defaultMQPushConsumerImpl;\n    }\n\n    public MessageListener getMessageListener() {\n        return messageListener;\n    }\n\n    public void setMessageListener(MessageListener messageListener) {\n        this.messageListener = messageListener;\n    }\n\n    public MessageModel getMessageModel() {\n        return messageModel;\n    }\n\n    public void setMessageModel(MessageModel messageModel) {\n        this.messageModel = messageModel;\n    }\n\n    public int getPullBatchSize() {\n        return pullBatchSize;\n    }\n\n    public void setPullBatchSize(int pullBatchSize) {\n        this.pullBatchSize = pullBatchSize;\n    }\n\n    public long getPullInterval() {\n        return pullInterval;\n    }\n\n    public void setPullInterval(long pullInterval) {\n        this.pullInterval = pullInterval;\n    }\n\n    public int getPullThresholdForQueue() {\n        return pullThresholdForQueue;\n    }\n\n    public void setPullThresholdForQueue(int pullThresholdForQueue) {\n        this.pullThresholdForQueue = pullThresholdForQueue;\n    }\n\n    public int getPopThresholdForQueue() {\n        return popThresholdForQueue;\n    }\n\n    public void setPopThresholdForQueue(int popThresholdForQueue) {\n        this.popThresholdForQueue = popThresholdForQueue;\n    }\n\n    public int getPullThresholdForTopic() {\n        return pullThresholdForTopic;\n    }\n\n    public void setPullThresholdForTopic(final int pullThresholdForTopic) {\n        this.pullThresholdForTopic = pullThresholdForTopic;\n    }\n\n    public int getPullThresholdSizeForQueue() {\n        return pullThresholdSizeForQueue;\n    }\n\n    public void setPullThresholdSizeForQueue(final int pullThresholdSizeForQueue) {\n        this.pullThresholdSizeForQueue = pullThresholdSizeForQueue;\n    }\n\n    public int getPullThresholdSizeForTopic() {\n        return pullThresholdSizeForTopic;\n    }\n\n    public void setPullThresholdSizeForTopic(final int pullThresholdSizeForTopic) {\n        this.pullThresholdSizeForTopic = pullThresholdSizeForTopic;\n    }\n\n    public Map<String, String> getSubscription() {\n        return subscription;\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    public void setSubscription(Map<String, String> subscription) {\n        Map<String, String> subscriptionWithNamespace = new HashMap<>(subscription.size(), 1);\n        for (Entry<String, String> topicEntry : subscription.entrySet()) {\n            subscriptionWithNamespace.put(withNamespace(topicEntry.getKey()), topicEntry.getValue());\n        }\n        this.subscription = subscriptionWithNamespace;\n    }\n\n    /**\n     * Send message back to broker which will be re-delivered in future.\n     * <p>\n     * This method will be removed or it's visibility will be changed in a certain version after April 5, 2020, so\n     * please do not use this method.\n     *\n     * @param msg        Message to send back.\n     * @param delayLevel delay level.\n     * @throws RemotingException    if there is any network-tier error.\n     * @throws MQBrokerException    if there is any broker error.\n     * @throws InterruptedException if the thread is interrupted.\n     * @throws MQClientException    if there is any client error.\n     */\n    @Deprecated\n    @Override\n    public void sendMessageBack(MessageExt msg, int delayLevel)\n        throws RemotingException, MQBrokerException, InterruptedException, MQClientException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, msg.getBrokerName());\n    }\n\n    /**\n     * Send message back to the broker whose name is <code>brokerName</code> and the message will be re-delivered in\n     * future.\n     * <p>\n     * This method will be removed or it's visibility will be changed in a certain version after April 5, 2020, so\n     * please do not use this method.\n     *\n     * @param msg        Message to send back.\n     * @param delayLevel delay level.\n     * @param brokerName broker name.\n     * @throws RemotingException    if there is any network-tier error.\n     * @throws MQBrokerException    if there is any broker error.\n     * @throws InterruptedException if the thread is interrupted.\n     * @throws MQClientException    if there is any client error.\n     */\n    @Deprecated\n    @Override\n    public void sendMessageBack(MessageExt msg, int delayLevel, String brokerName)\n        throws RemotingException, MQBrokerException, InterruptedException, MQClientException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, brokerName);\n    }\n\n    @Override\n    public Set<MessageQueue> fetchSubscribeMessageQueues(String topic) throws MQClientException {\n        return this.defaultMQPushConsumerImpl.fetchSubscribeMessageQueues(withNamespace(topic));\n    }\n\n    /**\n     * This method gets internal infrastructure readily to serve. Instances must call this method after configuration.\n     *\n     * @throws MQClientException if there is any client error.\n     */\n    @Override\n    public void start() throws MQClientException {\n        setConsumerGroup(NamespaceUtil.wrapNamespace(this.getNamespace(), this.consumerGroup));\n        this.defaultMQPushConsumerImpl.start();\n        if (enableTrace) {\n            try {\n                AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(consumerGroup, TraceDispatcher.Type.CONSUME, getTraceMsgBatchNum(), traceTopic, rpcHook);\n                dispatcher.setHostConsumer(this.defaultMQPushConsumerImpl);\n                dispatcher.setNamespaceV2(namespaceV2);\n                traceDispatcher = dispatcher;\n                this.defaultMQPushConsumerImpl.registerConsumeMessageHook(new ConsumeMessageTraceHookImpl(traceDispatcher));\n            } catch (Throwable e) {\n                log.error(\"system mqtrace hook init failed ,maybe can't send msg trace data\");\n            }\n        }\n        if (null != traceDispatcher) {\n            if (traceDispatcher instanceof AsyncTraceDispatcher) {\n                ((AsyncTraceDispatcher) traceDispatcher).getTraceProducer().setUseTLS(isUseTLS());\n            }\n            try {\n                traceDispatcher.start(this.getNamesrvAddr(), this.getAccessChannel());\n            } catch (MQClientException e) {\n                log.warn(\"trace dispatcher start failed \", e);\n            }\n        }\n    }\n\n    /**\n     * Shut down this client and releasing underlying resources.\n     */\n    @Override\n    public void shutdown() {\n        this.defaultMQPushConsumerImpl.shutdown(awaitTerminationMillisWhenShutdown);\n        if (null != traceDispatcher) {\n            traceDispatcher.shutdown();\n        }\n    }\n\n    @Override\n    @Deprecated\n    public void registerMessageListener(MessageListener messageListener) {\n        this.messageListener = messageListener;\n        this.defaultMQPushConsumerImpl.registerMessageListener(messageListener);\n    }\n\n    /**\n     * Register a callback to execute on message arrival for concurrent consuming.\n     *\n     * @param messageListener message handling callback.\n     */\n    @Override\n    public void registerMessageListener(MessageListenerConcurrently messageListener) {\n        this.messageListener = messageListener;\n        this.defaultMQPushConsumerImpl.registerMessageListener(messageListener);\n    }\n\n    /**\n     * Register a callback to execute on message arrival for orderly consuming.\n     *\n     * @param messageListener message handling callback.\n     */\n    @Override\n    public void registerMessageListener(MessageListenerOrderly messageListener) {\n        this.messageListener = messageListener;\n        this.defaultMQPushConsumerImpl.registerMessageListener(messageListener);\n    }\n\n    /**\n     * Subscribe a topic to consuming subscription.\n     *\n     * @param topic         topic to subscribe.\n     * @param subExpression subscription expression.it only support or operation such as \"tag1 || tag2 || tag3\" <br>\n     *                      if null or * expression,meaning subscribe all\n     * @throws MQClientException if there is any client error.\n     */\n    @Override\n    public void subscribe(String topic, String subExpression) throws MQClientException {\n        this.defaultMQPushConsumerImpl.subscribe(withNamespace(topic), subExpression);\n    }\n\n    /**\n     * Subscribe a topic to consuming subscription.\n     *\n     * @param topic             topic to consume.\n     * @param fullClassName     full class name,must extend org.apache.rocketmq.common.filter. MessageFilter\n     * @param filterClassSource class source code,used UTF-8 file encoding,must be responsible for your code safety\n     */\n    @Override\n    public void subscribe(String topic, String fullClassName, String filterClassSource) throws MQClientException {\n        this.defaultMQPushConsumerImpl.subscribe(withNamespace(topic), fullClassName, filterClassSource);\n    }\n\n    /**\n     * Subscribe a topic by message selector.\n     *\n     * @param topic           topic to consume.\n     * @param messageSelector {@link org.apache.rocketmq.client.consumer.MessageSelector}\n     * @see org.apache.rocketmq.client.consumer.MessageSelector#bySql\n     * @see org.apache.rocketmq.client.consumer.MessageSelector#byTag\n     */\n    @Override\n    public void subscribe(final String topic, final MessageSelector messageSelector) throws MQClientException {\n        this.defaultMQPushConsumerImpl.subscribe(withNamespace(topic), messageSelector);\n    }\n\n    /**\n     * Un-subscribe the specified topic from subscription.\n     *\n     * @param topic message topic\n     */\n    @Override\n    public void unsubscribe(String topic) {\n        this.defaultMQPushConsumerImpl.unsubscribe(topic);\n    }\n\n    /**\n     * Update the message consuming thread core pool size.\n     *\n     * @param corePoolSize new core pool size.\n     */\n    @Override\n    public void updateCorePoolSize(int corePoolSize) {\n        this.defaultMQPushConsumerImpl.updateCorePoolSize(corePoolSize);\n    }\n\n    /**\n     * Suspend pulling new messages.\n     */\n    @Override\n    public void suspend() {\n        this.defaultMQPushConsumerImpl.suspend();\n    }\n\n    /**\n     * Resume pulling.\n     */\n    @Override\n    public void resume() {\n        this.defaultMQPushConsumerImpl.resume();\n    }\n\n    public boolean isPause() {\n        return this.defaultMQPushConsumerImpl.isPause();\n    }\n\n    public boolean isConsumeOrderly() {\n        return this.defaultMQPushConsumerImpl.isConsumeOrderly();\n    }\n\n    public void registerConsumeMessageHook(final ConsumeMessageHook hook) {\n        this.defaultMQPushConsumerImpl.registerConsumeMessageHook(hook);\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    public OffsetStore getOffsetStore() {\n        return offsetStore;\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     */\n    @Deprecated\n    public void setOffsetStore(OffsetStore offsetStore) {\n        this.offsetStore = offsetStore;\n    }\n\n    public String getConsumeTimestamp() {\n        return consumeTimestamp;\n    }\n\n    public void setConsumeTimestamp(String consumeTimestamp) {\n        this.consumeTimestamp = consumeTimestamp;\n    }\n\n    public boolean isPostSubscriptionWhenPull() {\n        return postSubscriptionWhenPull;\n    }\n\n    public void setPostSubscriptionWhenPull(boolean postSubscriptionWhenPull) {\n        this.postSubscriptionWhenPull = postSubscriptionWhenPull;\n    }\n\n    @Override\n    public boolean isUnitMode() {\n        return unitMode;\n    }\n\n    @Override\n    public void setUnitMode(boolean isUnitMode) {\n        this.unitMode = isUnitMode;\n    }\n\n    public long getAdjustThreadPoolNumsThreshold() {\n        return adjustThreadPoolNumsThreshold;\n    }\n\n    public void setAdjustThreadPoolNumsThreshold(long adjustThreadPoolNumsThreshold) {\n        this.adjustThreadPoolNumsThreshold = adjustThreadPoolNumsThreshold;\n    }\n\n    public int getMaxReconsumeTimes() {\n        return maxReconsumeTimes;\n    }\n\n    public void setMaxReconsumeTimes(final int maxReconsumeTimes) {\n        this.maxReconsumeTimes = maxReconsumeTimes;\n    }\n\n    public long getSuspendCurrentQueueTimeMillis() {\n        return suspendCurrentQueueTimeMillis;\n    }\n\n    public void setSuspendCurrentQueueTimeMillis(final long suspendCurrentQueueTimeMillis) {\n        this.suspendCurrentQueueTimeMillis = suspendCurrentQueueTimeMillis;\n    }\n\n    public long getConsumeTimeout() {\n        return consumeTimeout;\n    }\n\n    public void setConsumeTimeout(final long consumeTimeout) {\n        this.consumeTimeout = consumeTimeout;\n    }\n\n    public long getPopInvisibleTime() {\n        return popInvisibleTime;\n    }\n\n    public void setPopInvisibleTime(long popInvisibleTime) {\n        this.popInvisibleTime = popInvisibleTime;\n    }\n\n    public long getAwaitTerminationMillisWhenShutdown() {\n        return awaitTerminationMillisWhenShutdown;\n    }\n\n    public void setAwaitTerminationMillisWhenShutdown(long awaitTerminationMillisWhenShutdown) {\n        this.awaitTerminationMillisWhenShutdown = awaitTerminationMillisWhenShutdown;\n    }\n\n    public int getPullBatchSizeInBytes() {\n        return pullBatchSizeInBytes;\n    }\n\n    public void setPullBatchSizeInBytes(int pullBatchSizeInBytes) {\n        this.pullBatchSizeInBytes = pullBatchSizeInBytes;\n    }\n\n    public TraceDispatcher getTraceDispatcher() {\n        return traceDispatcher;\n    }\n\n    public int getPopBatchNums() {\n        return popBatchNums;\n    }\n\n    public void setPopBatchNums(int popBatchNums) {\n        this.popBatchNums = popBatchNums;\n    }\n\n    public boolean isClientRebalance() {\n        return clientRebalance;\n    }\n\n    public void setClientRebalance(boolean clientRebalance) {\n        this.clientRebalance = clientRebalance;\n    }\n\n    public MessageQueueListener getMessageQueueListener() {\n        return messageQueueListener;\n    }\n\n    public void setMessageQueueListener(MessageQueueListener messageQueueListener) {\n        this.messageQueueListener = messageQueueListener;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\nimport java.util.Collection;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\npublic interface LitePullConsumer {\n\n    /**\n     * Start the consumer\n     */\n    void start() throws MQClientException;\n\n    /**\n     * Shutdown the consumer\n     */\n    void shutdown();\n\n    /**\n     * This consumer is still running\n     *\n     * @return true if consumer is still running\n     */\n    boolean isRunning();\n\n    /**\n     * Subscribe some topic with all tags\n     * @throws MQClientException if there is any client error.\n     */\n    void subscribe(final String topic) throws MQClientException;\n\n    /**\n     * Subscribe some topic with subExpression\n     *\n     * @param subExpression subscription expression.it only support or operation such as \"tag1 || tag2 || tag3\" <br> if\n     * null or * expression,meaning subscribe all\n     * @throws MQClientException if there is any client error.\n     */\n    void subscribe(final String topic, final String subExpression) throws MQClientException;\n\n    /**\n     * Subscribe some topic with subExpression and messageQueueListener\n     * @param topic\n     * @param subExpression\n     * @param messageQueueListener\n     */\n    void subscribe(final String topic, final String subExpression, final MessageQueueListener messageQueueListener) throws MQClientException;\n\n    /**\n     * Subscribe some topic with selector.\n     *\n     * @param selector message selector({@link MessageSelector}), can be null.\n     * @throws MQClientException if there is any client error.\n     */\n    void subscribe(final String topic, final MessageSelector selector) throws MQClientException;\n\n    /**\n     * Unsubscribe consumption some topic\n     *\n     * @param topic Message topic that needs to be unsubscribe.\n     */\n    void unsubscribe(final String topic);\n\n\n    /**\n     * subscribe mode, get assigned MessageQueue\n     * @return\n     * @throws MQClientException\n     */\n    Set<MessageQueue> assignment() throws MQClientException;\n\n    /**\n     * Manually assign a list of message queues to this consumer. This interface does not allow for incremental\n     * assignment and will replace the previous assignment (if there is one).\n     *\n     * @param messageQueues Message queues that needs to be assigned.\n     */\n    void assign(Collection<MessageQueue> messageQueues);\n\n    /**\n     * Set topic subExpression for assign mode. This interface does not allow be call after start(). Default value is * if not set.\n     * assignment and will replace the previous assignment (if there is one).\n     *\n     * @param subExpression subscription expression.it only support or operation such as \"tag1 || tag2 || tag3\" <br> if\n     *      * null or * expression,meaning subscribe all\n     */\n    void setSubExpressionForAssign(final String topic, final String subExpression);\n\n    void buildSubscriptionsForHeartbeat(Map<String, MessageSelector> subExpressionMap) throws Exception;\n\n    /**\n     * Fetch data for the topics or partitions specified using assign API\n     *\n     * @return list of message, can be null.\n     */\n    List<MessageExt> poll();\n\n    /**\n     * Fetch data for the topics or partitions specified using assign API\n     *\n     * @param timeout The amount time, in milliseconds, spent waiting in poll if data is not available. Must not be\n     * negative\n     * @return list of message, can be null.\n     */\n    List<MessageExt> poll(long timeout);\n\n    /**\n     * Overrides the fetch offsets that the consumer will use on the next poll. If this API is invoked for the same\n     * message queue more than once, the latest offset will be used on the next poll(). Note that you may lose data if\n     * this API is arbitrarily used in the middle of consumption.\n     *\n     * @param messageQueue\n     * @param offset\n     */\n    void seek(MessageQueue messageQueue, long offset) throws MQClientException;\n\n    /**\n     * Suspend pulling from the requested message queues.\n     *\n     * Because of the implementation of pre-pull, fetch data in {@link #poll()} will not stop immediately until the\n     * messages of the requested message queues drain.\n     *\n     * Note that this method does not affect message queue subscription. In particular, it does not cause a group\n     * rebalance.\n     *\n     * @param messageQueues Message queues that needs to be paused.\n     */\n    void pause(Collection<MessageQueue> messageQueues);\n\n    /**\n     * Resume specified message queues which have been paused with {@link #pause(Collection)}.\n     *\n     * @param messageQueues Message queues that needs to be resumed.\n     */\n    void resume(Collection<MessageQueue> messageQueues);\n\n    /**\n     * Whether to enable auto-commit consume offset.\n     *\n     * @return true if enable auto-commit, false if disable auto-commit.\n     */\n    boolean isAutoCommit();\n\n    /**\n     * Set whether to enable auto-commit consume offset.\n     *\n     * @param autoCommit Whether to enable auto-commit.\n     */\n    void setAutoCommit(boolean autoCommit);\n\n    /**\n     * Get metadata about the message queues for a given topic.\n     *\n     * @param topic The topic that need to get metadata.\n     * @return collection of message queues\n     * @throws MQClientException if there is any client error.\n     */\n    Collection<MessageQueue> fetchMessageQueues(String topic) throws MQClientException;\n\n    /**\n     * Look up the offsets for the given message queue by timestamp. The returned offset for each message queue is the\n     * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding message\n     * queue.\n     *\n     * @param messageQueue Message queues that needs to get offset by timestamp.\n     * @param timestamp\n     * @return offset\n     * @throws MQClientException if there is any client error.\n     */\n    Long offsetForTimestamp(MessageQueue messageQueue, Long timestamp) throws MQClientException;\n\n    @Deprecated\n    /**\n     * The method is deprecated because its name is ambiguous, this method relies on the background thread commit consumerOffset rather than the synchronous commit offset.\n     * The method is expected to be removed after version 5.1.0. It is recommended to use the {@link #commit()} method.\n     *\n     * Manually commit consume offset saved by the system.\n     */\n    void commitSync();\n\n    @Deprecated\n    /**\n     * The method is deprecated because its name is ambiguous, this method relies on the background thread commit consumerOffset rather than the synchronous commit offset.\n     * The method is expected to be removed after version 5.1.0. It is recommended to use the {@link #commit(java.util.Map, boolean)} method.\n     *\n     * @param offsetMap Offset specified by batch commit\n     */\n    void commitSync(Map<MessageQueue, Long> offsetMap, boolean persist);\n\n    /**\n     * Manually commit consume offset saved by the system. This is a non-blocking method.\n     */\n    void commit();\n\n    /**\n     * Offset specified by batch commit\n     *\n     * @param offsetMap Offset specified by batch commit\n     * @param persist Whether to persist to the broker\n     */\n    void commit(Map<MessageQueue, Long> offsetMap, boolean persist);\n\n    /**\n     * Manually commit consume offset saved by the system.\n     *\n     * @param messageQueues Message queues that need to submit consumer offset\n     * @param persist hether to persist to the broker\n     */\n    void commit(final Set<MessageQueue> messageQueues, boolean persist);\n\n    /**\n     * Get the last committed offset for the given message queue.\n     *\n     * @param messageQueue\n     * @return offset, if offset equals -1 means no offset in broker.\n     * @throws MQClientException if there is any client error.\n     */\n    Long committed(MessageQueue messageQueue) throws MQClientException;\n\n    /**\n     * Register a callback for sensing topic metadata changes.\n     *\n     * @param topic The topic that need to monitor.\n     * @param topicMessageQueueChangeListener Callback when topic metadata changes, refer {@link\n     * TopicMessageQueueChangeListener}\n     * @throws MQClientException if there is any client error.\n     */\n    void registerTopicMessageQueueChangeListener(String topic,\n        TopicMessageQueueChangeListener topicMessageQueueChangeListener) throws MQClientException;\n\n    /**\n     * Update name server addresses.\n     */\n    void updateNameServerAddress(String nameServerAddress);\n\n    /**\n     * Overrides the fetch offsets with the begin offset that the consumer will use on the next poll. If this API is\n     * invoked for the same message queue more than once, the latest offset will be used on the next poll(). Note that\n     * you may lose data if this API is arbitrarily used in the middle of consumption.\n     *\n     * @param messageQueue\n     */\n    void seekToBegin(MessageQueue messageQueue)throws MQClientException;\n\n    /**\n     * Overrides the fetch offsets with the end offset that the consumer will use on the next poll. If this API is\n     * invoked for the same message queue more than once, the latest offset will be used on the next poll(). Note that\n     * you may lose data if this API is arbitrarily used in the middle of consumption.\n     *\n     * @param messageQueue\n     */\n    void seekToEnd(MessageQueue messageQueue)throws MQClientException;\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/MQConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\nimport java.util.Set;\nimport org.apache.rocketmq.client.MQAdmin;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\n\n/**\n * Message queue consumer interface\n */\npublic interface MQConsumer extends MQAdmin {\n    /**\n     * If consuming of messages failed, they will be sent back to the brokers for another delivery attempt after\n     * interval specified in delay level.\n     */\n    @Deprecated\n    void sendMessageBack(final MessageExt msg, final int delayLevel) throws RemotingException,\n        MQBrokerException, InterruptedException, MQClientException;\n\n    /**\n     * If consuming of messages failed, they will be sent back to the brokers for another delivery attempt after\n     * interval specified in delay level.\n     */\n    void sendMessageBack(final MessageExt msg, final int delayLevel, final String brokerName)\n        throws RemotingException, MQBrokerException, InterruptedException, MQClientException;\n\n    /**\n     * Fetch message queues from consumer cache pertaining to the given topic.\n     *\n     * @param topic message topic\n     * @return queue set\n     */\n    Set<MessageQueue> fetchSubscribeMessageQueues(final String topic) throws MQClientException;\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\nimport java.util.Set;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\n\n/**\n * Pulling consumer interface\n */\npublic interface MQPullConsumer extends MQConsumer {\n    /**\n     * Start the consumer\n     */\n    void start() throws MQClientException;\n\n    /**\n     * Shutdown the consumer\n     */\n    void shutdown();\n\n    /**\n     * Register the message queue listener\n     */\n    void registerMessageQueueListener(final String topic, final MessageQueueListener listener);\n\n    /**\n     * Pulling the messages,not blocking\n     *\n     * @param mq from which message queue\n     * @param subExpression subscription expression.it only support or operation such as \"tag1 || tag2 || tag3\" <br> if\n     * null or * expression,meaning subscribe all\n     * @param offset from where to pull\n     * @param maxNums max pulling numbers\n     * @return The resulting {@code PullRequest}\n     */\n    PullResult pull(final MessageQueue mq, final String subExpression, final long offset,\n        final int maxNums) throws MQClientException, RemotingException, MQBrokerException,\n        InterruptedException;\n\n    /**\n     * Pulling the messages in the specified timeout\n     *\n     * @return The resulting {@code PullRequest}\n     */\n    PullResult pull(final MessageQueue mq, final String subExpression, final long offset,\n        final int maxNums, final long timeout) throws MQClientException, RemotingException,\n        MQBrokerException, InterruptedException;\n\n    /**\n     * Pulling the messages, not blocking\n     * <p>\n     * support other message selection, such as {@link org.apache.rocketmq.common.filter.ExpressionType#SQL92}\n     * </p>\n     *\n     * @param mq from which message queue\n     * @param selector message selector({@link MessageSelector}), can be null.\n     * @param offset from where to pull\n     * @param maxNums max pulling numbers\n     * @return The resulting {@code PullRequest}\n     */\n    PullResult pull(final MessageQueue mq, final MessageSelector selector, final long offset,\n        final int maxNums) throws MQClientException, RemotingException, MQBrokerException,\n        InterruptedException;\n\n    /**\n     * Pulling the messages in the specified timeout\n     * <p>\n     * support other message selection, such as {@link org.apache.rocketmq.common.filter.ExpressionType#SQL92}\n     * </p>\n     *\n     * @param mq from which message queue\n     * @param selector message selector({@link MessageSelector}), can be null.\n     * @param offset from where to pull\n     * @param maxNums max pulling numbers\n     * @param timeout Pulling the messages in the specified timeout\n     * @return The resulting {@code PullRequest}\n     */\n    PullResult pull(final MessageQueue mq, final MessageSelector selector, final long offset,\n        final int maxNums, final long timeout) throws MQClientException, RemotingException, MQBrokerException,\n        InterruptedException;\n\n    /**\n     * Pulling the messages in a async. way\n     */\n    void pull(final MessageQueue mq, final String subExpression, final long offset, final int maxNums,\n        final PullCallback pullCallback) throws MQClientException, RemotingException,\n        InterruptedException;\n\n    /**\n     * Pulling the messages in a async. way\n     */\n    void pull(final MessageQueue mq, final String subExpression, final long offset, final int maxNums,\n        final PullCallback pullCallback, long timeout) throws MQClientException, RemotingException,\n        InterruptedException;\n\n    /**\n     * Pulling the messages in a async. way\n     */\n    void pull(final MessageQueue mq, final String subExpression, final long offset, final int maxNums, final int maxSize,\n        final PullCallback pullCallback, long timeout) throws MQClientException, RemotingException,\n        InterruptedException;\n\n    /**\n     * Pulling the messages in a async way. Support message selection\n     */\n    void pull(final MessageQueue mq, final MessageSelector selector, final long offset, final int maxNums,\n        final PullCallback pullCallback) throws MQClientException, RemotingException,\n        InterruptedException;\n\n    /**\n     * Pulling the messages in a async. way. Support message selection\n     */\n    void pull(final MessageQueue mq, final MessageSelector selector, final long offset, final int maxNums,\n        final PullCallback pullCallback, long timeout) throws MQClientException, RemotingException,\n        InterruptedException;\n\n    /**\n     * Pulling the messages,if no message arrival,blocking some time\n     *\n     * @return The resulting {@code PullRequest}\n     */\n    PullResult pullBlockIfNotFound(final MessageQueue mq, final String subExpression,\n        final long offset, final int maxNums) throws MQClientException, RemotingException,\n        MQBrokerException, InterruptedException;\n\n    /**\n     * Pulling the messages through callback function,if no message arrival,blocking.\n     */\n    void pullBlockIfNotFound(final MessageQueue mq, final String subExpression, final long offset,\n        final int maxNums, final PullCallback pullCallback) throws MQClientException, RemotingException,\n        InterruptedException;\n\n    /**\n     * Pulling the messages through callback function,if no message arrival,blocking. Support message selection\n     */\n    void pullBlockIfNotFoundWithMessageSelector(final MessageQueue mq, final MessageSelector selector,\n        final long offset, final int maxNums,\n        final PullCallback pullCallback) throws MQClientException, RemotingException,\n        InterruptedException;\n\n    /**\n     * Pulling the messages,if no message arrival,blocking some time. Support message selection\n     *\n     * @return The resulting {@code PullRequest}\n     */\n    PullResult pullBlockIfNotFoundWithMessageSelector(final MessageQueue mq, final MessageSelector selector,\n        final long offset, final int maxNums) throws MQClientException, RemotingException,\n        MQBrokerException, InterruptedException;\n\n    /**\n     * Update the offset\n     */\n    void updateConsumeOffset(final MessageQueue mq, final long offset) throws MQClientException;\n\n    /**\n     * Fetch the offset\n     *\n     * @return The fetched offset of given queue\n     */\n    long fetchConsumeOffset(final MessageQueue mq, final boolean fromStore) throws MQClientException;\n\n    /**\n     * Fetch the message queues according to the topic\n     *\n     * @param topic message topic\n     * @return message queue set\n     */\n    Set<MessageQueue> fetchMessageQueuesInBalance(final String topic) throws MQClientException;\n\n    /**\n     * If consuming failure,message will be send back to the broker,and delay consuming in some time later.<br>\n     * Mind! message can only be consumed in the same group.\n     */\n    void sendMessageBack(MessageExt msg, int delayLevel, String brokerName, String consumerGroup)\n        throws RemotingException, MQBrokerException, InterruptedException, MQClientException;\n\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumerScheduleService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\nimport java.util.Iterator;\nimport java.util.Map.Entry;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.ScheduledThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\n/**\n * Schedule service for pull consumer.\n * This Consumer will be removed in 2022, and a better implementation {@link\n * DefaultLitePullConsumer} is recommend to use in the scenario of actively pulling messages.\n */\npublic class MQPullConsumerScheduleService {\n    private final Logger log = LoggerFactory.getLogger(MQPullConsumerScheduleService.class);\n    private final MessageQueueListener messageQueueListener = new MessageQueueListenerImpl();\n    private final ConcurrentMap<MessageQueue, PullTaskImpl> taskTable =\n        new ConcurrentHashMap<>();\n    private DefaultMQPullConsumer defaultMQPullConsumer;\n    private int pullThreadNums = 20;\n    private ConcurrentMap<String /* topic */, PullTaskCallback> callbackTable =\n        new ConcurrentHashMap<>();\n    private ScheduledThreadPoolExecutor scheduledThreadPoolExecutor;\n\n    public MQPullConsumerScheduleService(final String consumerGroup) {\n        this.defaultMQPullConsumer = new DefaultMQPullConsumer(consumerGroup);\n        this.defaultMQPullConsumer.setMessageModel(MessageModel.CLUSTERING);\n    }\n\n    public MQPullConsumerScheduleService(final String consumerGroup, final RPCHook rpcHook) {\n        this.defaultMQPullConsumer = new DefaultMQPullConsumer(consumerGroup, rpcHook);\n        this.defaultMQPullConsumer.setMessageModel(MessageModel.CLUSTERING);\n    }\n\n    public void putTask(String topic, Set<MessageQueue> mqNewSet) {\n        Iterator<Entry<MessageQueue, PullTaskImpl>> it = this.taskTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<MessageQueue, PullTaskImpl> next = it.next();\n            if (next.getKey().getTopic().equals(topic)) {\n                if (!mqNewSet.contains(next.getKey())) {\n                    next.getValue().setCancelled(true);\n                    it.remove();\n                }\n            }\n        }\n\n        for (MessageQueue mq : mqNewSet) {\n            if (!this.taskTable.containsKey(mq)) {\n                PullTaskImpl command = new PullTaskImpl(mq);\n                this.taskTable.put(mq, command);\n                this.scheduledThreadPoolExecutor.schedule(command, 0, TimeUnit.MILLISECONDS);\n\n            }\n        }\n    }\n\n    public void start() throws MQClientException {\n        final String group = this.defaultMQPullConsumer.getConsumerGroup();\n        this.scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(\n            this.pullThreadNums,\n            new ThreadFactoryImpl(\"PullMsgThread-\" + group)\n        );\n\n        this.defaultMQPullConsumer.setMessageQueueListener(this.messageQueueListener);\n\n        this.defaultMQPullConsumer.start();\n\n        log.info(\"MQPullConsumerScheduleService start OK, {} {}\",\n            this.defaultMQPullConsumer.getConsumerGroup(), this.callbackTable);\n    }\n\n    public void registerPullTaskCallback(final String topic, final PullTaskCallback callback) {\n        this.callbackTable.put(NamespaceUtil.wrapNamespace(this.defaultMQPullConsumer.getNamespace(), topic), callback);\n        this.defaultMQPullConsumer.registerMessageQueueListener(topic, null);\n    }\n\n    public void shutdown() {\n        if (this.scheduledThreadPoolExecutor != null) {\n            this.scheduledThreadPoolExecutor.shutdown();\n        }\n\n        if (this.defaultMQPullConsumer != null) {\n            this.defaultMQPullConsumer.shutdown();\n        }\n    }\n\n    public ConcurrentMap<String, PullTaskCallback> getCallbackTable() {\n        return callbackTable;\n    }\n\n    public void setCallbackTable(ConcurrentHashMap<String, PullTaskCallback> callbackTable) {\n        this.callbackTable = callbackTable;\n    }\n\n    public int getPullThreadNums() {\n        return pullThreadNums;\n    }\n\n    public void setPullThreadNums(int pullThreadNums) {\n        this.pullThreadNums = pullThreadNums;\n    }\n\n    public DefaultMQPullConsumer getDefaultMQPullConsumer() {\n        return defaultMQPullConsumer;\n    }\n\n    public void setDefaultMQPullConsumer(DefaultMQPullConsumer defaultMQPullConsumer) {\n        this.defaultMQPullConsumer = defaultMQPullConsumer;\n    }\n\n    public MessageModel getMessageModel() {\n        return this.defaultMQPullConsumer.getMessageModel();\n    }\n\n    public void setMessageModel(MessageModel messageModel) {\n        this.defaultMQPullConsumer.setMessageModel(messageModel);\n    }\n\n    class MessageQueueListenerImpl implements MessageQueueListener {\n        @Override\n        public void messageQueueChanged(String topic, Set<MessageQueue> mqAll, Set<MessageQueue> mqDivided) {\n            MessageModel messageModel =\n                MQPullConsumerScheduleService.this.defaultMQPullConsumer.getMessageModel();\n            switch (messageModel) {\n                case BROADCASTING:\n                    MQPullConsumerScheduleService.this.putTask(topic, mqAll);\n                    break;\n                case CLUSTERING:\n                    MQPullConsumerScheduleService.this.putTask(topic, mqDivided);\n                    break;\n                default:\n                    break;\n            }\n        }\n    }\n\n    public class PullTaskImpl implements Runnable {\n        private final MessageQueue messageQueue;\n        private volatile boolean cancelled = false;\n\n        public PullTaskImpl(final MessageQueue messageQueue) {\n            this.messageQueue = messageQueue;\n        }\n\n        @Override\n        public void run() {\n            String topic = this.messageQueue.getTopic();\n            if (!this.isCancelled()) {\n                PullTaskCallback pullTaskCallback =\n                    MQPullConsumerScheduleService.this.callbackTable.get(topic);\n                if (pullTaskCallback != null) {\n                    final PullTaskContext context = new PullTaskContext();\n                    context.setPullConsumer(MQPullConsumerScheduleService.this.defaultMQPullConsumer);\n                    try {\n                        pullTaskCallback.doPullTask(this.messageQueue, context);\n                    } catch (Throwable e) {\n                        context.setPullNextDelayTimeMillis(1000);\n                        log.error(\"doPullTask Exception\", e);\n                    }\n\n                    if (!this.isCancelled()) {\n                        MQPullConsumerScheduleService.this.scheduledThreadPoolExecutor.schedule(this,\n                            context.getPullNextDelayTimeMillis(), TimeUnit.MILLISECONDS);\n                    } else {\n                        log.warn(\"The Pull Task is cancelled after doPullTask, {}\", messageQueue);\n                    }\n                } else {\n                    log.warn(\"Pull Task Callback not exist , {}\", topic);\n                }\n            } else {\n                log.warn(\"The Pull Task is cancelled, {}\", messageQueue);\n            }\n        }\n\n        public boolean isCancelled() {\n            return cancelled;\n        }\n\n        public void setCancelled(boolean cancelled) {\n            this.cancelled = cancelled;\n        }\n\n        public MessageQueue getMessageQueue() {\n            return messageQueue;\n        }\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/MQPushConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\nimport org.apache.rocketmq.client.consumer.listener.MessageListener;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;\nimport org.apache.rocketmq.client.exception.MQClientException;\n\n/**\n * Push consumer\n */\npublic interface MQPushConsumer extends MQConsumer {\n    /**\n     * Start the consumer\n     */\n    void start() throws MQClientException;\n\n    /**\n     * Shutdown the consumer\n     */\n    void shutdown();\n\n    /**\n     * Register the message listener\n     */\n    @Deprecated\n    void registerMessageListener(MessageListener messageListener);\n\n    void registerMessageListener(final MessageListenerConcurrently messageListener);\n\n    void registerMessageListener(final MessageListenerOrderly messageListener);\n\n    /**\n     * Subscribe some topic\n     *\n     * @param subExpression subscription expression.it only support or operation such as \"tag1 || tag2 || tag3\" <br> if\n     * null or * expression,meaning subscribe\n     * all\n     */\n    void subscribe(final String topic, final String subExpression) throws MQClientException;\n\n    /**\n     * This method will be removed in the version 5.0.0,because filterServer was removed,and method <code>subscribe(final String topic, final MessageSelector messageSelector)</code>\n     * is recommended.\n     *\n     * Subscribe some topic\n     *\n     * @param fullClassName full class name,must extend org.apache.rocketmq.common.filter. MessageFilter\n     * @param filterClassSource class source code,used UTF-8 file encoding,must be responsible for your code safety\n     */\n    @Deprecated\n    void subscribe(final String topic, final String fullClassName,\n        final String filterClassSource) throws MQClientException;\n\n    /**\n     * Subscribe some topic with selector.\n     * <p>\n     * This interface also has the ability of {@link #subscribe(String, String)},\n     * and, support other message selection, such as {@link org.apache.rocketmq.common.filter.ExpressionType#SQL92}.\n     * </p>\n     * <p/>\n     * <p>\n     * Choose Tag: {@link MessageSelector#byTag(java.lang.String)}\n     * </p>\n     * <p/>\n     * <p>\n     * Choose SQL92: {@link MessageSelector#bySql(java.lang.String)}\n     * </p>\n     *\n     * @param selector message selector({@link MessageSelector}), can be null.\n     */\n    void subscribe(final String topic, final MessageSelector selector) throws MQClientException;\n\n    /**\n     * Unsubscribe consumption some topic\n     *\n     * @param topic message topic\n     */\n    void unsubscribe(final String topic);\n\n    /**\n     * Update the consumer thread pool size Dynamically\n     */\n    void updateCorePoolSize(int corePoolSize);\n\n    /**\n     * Suspend the consumption\n     */\n    void suspend();\n\n    /**\n     * Resume the consumption\n     */\n    void resume();\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/MessageQueueListener.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\nimport java.util.Set;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\n/**\n * A MessageQueueListener is implemented by the application and may be specified when a message queue changed\n */\npublic interface MessageQueueListener {\n    /**\n     * @param topic message topic\n     * @param mqAll all queues in this message topic\n     * @param mqAssigned collection of queues, assigned to the current consumer\n     */\n    void messageQueueChanged(final String topic, final Set<MessageQueue> mqAll, final Set<MessageQueue> mqAssigned);\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/MessageSelector.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.consumer;\n\nimport org.apache.rocketmq.common.filter.ExpressionType;\n\n/**\n * Message selector: select message at server.\n * <p>\n * Now, support:\n * <li>Tag: {@link org.apache.rocketmq.common.filter.ExpressionType#TAG}\n * </li>\n * <li>SQL92: {@link org.apache.rocketmq.common.filter.ExpressionType#SQL92}\n * </li>\n * </p>\n */\npublic class MessageSelector {\n\n    /**\n     * @see org.apache.rocketmq.common.filter.ExpressionType\n     */\n    private String type;\n\n    /**\n     * expression content.\n     */\n    private String expression;\n\n    private MessageSelector(String type, String expression) {\n        this.type = type;\n        this.expression = expression;\n    }\n\n    /**\n     * Use SQL92 to select message.\n     *\n     * @param sql if null or empty, will be treated as select all message.\n     */\n    public static MessageSelector bySql(String sql) {\n        return new MessageSelector(ExpressionType.SQL92, sql);\n    }\n\n    /**\n     * Use tag to select message.\n     *\n     * @param tag if null or empty or \"*\", will be treated as select all message.\n     */\n    public static MessageSelector byTag(String tag) {\n        return new MessageSelector(ExpressionType.TAG, tag);\n    }\n\n    public String getExpressionType() {\n        return type;\n    }\n\n    public String getExpression() {\n        return expression;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/NotifyResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\npublic class NotifyResult {\n    private boolean hasMsg;\n    private boolean pollingFull;\n\n    public boolean isHasMsg() {\n        return hasMsg;\n    }\n\n    public boolean isPollingFull() {\n        return pollingFull;\n    }\n\n    public void setHasMsg(boolean hasMsg) {\n        this.hasMsg = hasMsg;\n    }\n\n    public void setPollingFull(boolean pollingFull) {\n        this.pollingFull = pollingFull;\n    }\n\n    @Override public String toString() {\n        return \"NotifyResult{\" +\n            \"hasMsg=\" + hasMsg +\n            \", pollingFull=\" + pollingFull +\n            '}';\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/PopCallback.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\n/**\n * Async message pop interface\n */\npublic interface PopCallback {\n    void onSuccess(final PopResult popResult);\n\n    void onException(final Throwable e);\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/PopResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\nimport java.util.List;\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic class PopResult {\n    private List<MessageExt> msgFoundList;\n    private PopStatus popStatus;\n    private long popTime;\n    private long invisibleTime;\n    private long restNum;\n\n    public PopResult(PopStatus popStatus, List<MessageExt> msgFoundList) {\n        this.popStatus = popStatus;\n        this.msgFoundList = msgFoundList;\n    }\n\n    public long getPopTime() {\n        return popTime;\n    }\n\n\n    public void setPopTime(long popTime) {\n        this.popTime = popTime;\n    }\n\n    public long getRestNum() {\n        return restNum;\n    }\n\n    public void setRestNum(long restNum) {\n        this.restNum = restNum;\n    }\n\n    public long getInvisibleTime() {\n        return invisibleTime;\n    }\n\n\n    public void setInvisibleTime(long invisibleTime) {\n        this.invisibleTime = invisibleTime;\n    }\n\n\n    public void setPopStatus(PopStatus popStatus) {\n        this.popStatus = popStatus;\n    }\n\n    public PopStatus getPopStatus() {\n        return popStatus;\n    }\n\n    public List<MessageExt> getMsgFoundList() {\n        return msgFoundList;\n    }\n\n    public void setMsgFoundList(List<MessageExt> msgFoundList) {\n        this.msgFoundList = msgFoundList;\n    }\n\n    @Override\n    public String toString() {\n        return \"PopResult [popStatus=\" + popStatus + \",msgFoundList=\"\n            + (msgFoundList == null ? 0 : msgFoundList.size()) + \",restNum=\" + restNum + \"]\";\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/PopStatus.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\npublic enum PopStatus {\n    /**\n     * Founded\n     */\n    FOUND,\n    /**\n     * No new message can be pull after polling time out\n     * delete after next release\n     */\n    NO_NEW_MSG,\n    /**\n     * polling pool is full, do not try again immediately.\n     */\n    POLLING_FULL,\n    /**\n     * polling time out but no message find\n     */\n    POLLING_NOT_FOUND\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/PullCallback.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\n/**\n * Async message pulling interface\n */\npublic interface PullCallback {\n    void onSuccess(final PullResult pullResult);\n\n    void onException(final Throwable e);\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/PullResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\nimport java.util.List;\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic class PullResult {\n    private final PullStatus pullStatus;\n    private final long nextBeginOffset;\n    private final long minOffset;\n    private final long maxOffset;\n    private List<MessageExt> msgFoundList;\n\n\n    public PullResult(PullStatus pullStatus, long nextBeginOffset, long minOffset, long maxOffset,\n        List<MessageExt> msgFoundList) {\n        super();\n        this.pullStatus = pullStatus;\n        this.nextBeginOffset = nextBeginOffset;\n        this.minOffset = minOffset;\n        this.maxOffset = maxOffset;\n        this.msgFoundList = msgFoundList;\n    }\n\n    public PullStatus getPullStatus() {\n        return pullStatus;\n    }\n\n    public long getNextBeginOffset() {\n        return nextBeginOffset;\n    }\n\n    public long getMinOffset() {\n        return minOffset;\n    }\n\n    public long getMaxOffset() {\n        return maxOffset;\n    }\n\n    public List<MessageExt> getMsgFoundList() {\n        return msgFoundList;\n    }\n\n    public void setMsgFoundList(List<MessageExt> msgFoundList) {\n        this.msgFoundList = msgFoundList;\n    }\n\n    @Override\n    public String toString() {\n        return \"PullResult [pullStatus=\" + pullStatus + \", nextBeginOffset=\" + nextBeginOffset\n            + \", minOffset=\" + minOffset + \", maxOffset=\" + maxOffset + \", msgFoundList=\"\n            + (msgFoundList == null ? 0 : msgFoundList.size()) + \"]\";\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/PullStatus.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\npublic enum PullStatus {\n    /**\n     * Founded\n     */\n    FOUND,\n    /**\n     * No new message can be pull\n     */\n    NO_NEW_MSG,\n    /**\n     * Filtering results can not match\n     */\n    NO_MATCHED_MSG,\n    /**\n     * Illegal offset,may be too big or too small\n     */\n    OFFSET_ILLEGAL\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/PullTaskCallback.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\nimport org.apache.rocketmq.common.message.MessageQueue;\n\npublic interface PullTaskCallback {\n    void doPullTask(final MessageQueue mq, final PullTaskContext context);\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/PullTaskContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\npublic class PullTaskContext {\n\n    private int pullNextDelayTimeMillis = 200;\n\n    private MQPullConsumer pullConsumer;\n\n    public int getPullNextDelayTimeMillis() {\n        return pullNextDelayTimeMillis;\n    }\n\n    public void setPullNextDelayTimeMillis(int pullNextDelayTimeMillis) {\n        this.pullNextDelayTimeMillis = pullNextDelayTimeMillis;\n    }\n\n    public MQPullConsumer getPullConsumer() {\n        return pullConsumer;\n    }\n\n    public void setPullConsumer(MQPullConsumer pullConsumer) {\n        this.pullConsumer = pullConsumer;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/TopicMessageQueueChangeListener.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\nimport java.util.Set;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\npublic interface TopicMessageQueueChangeListener {\n    /**\n     * This method will be invoked in the condition of queue numbers changed, These scenarios occur when the topic is\n     * expanded or shrunk.\n     *\n     * @param messageQueues\n     */\n    void onChanged(String topic, Set<MessageQueue> messageQueues);\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/listener/ConsumeConcurrentlyContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.listener;\n\nimport org.apache.rocketmq.common.message.MessageQueue;\n\n/**\n * Consumer concurrent consumption context\n */\npublic class ConsumeConcurrentlyContext {\n    private final MessageQueue messageQueue;\n    /**\n     * Message consume retry strategy<br>\n     * -1,no retry,put into DLQ directly<br>\n     * 0,broker control retry frequency<br>\n     * >0,client control retry frequency\n     */\n    private int delayLevelWhenNextConsume = 0;\n    private int ackIndex = Integer.MAX_VALUE;\n\n    public ConsumeConcurrentlyContext(MessageQueue messageQueue) {\n        this.messageQueue = messageQueue;\n    }\n\n    public int getDelayLevelWhenNextConsume() {\n        return delayLevelWhenNextConsume;\n    }\n\n    public void setDelayLevelWhenNextConsume(int delayLevelWhenNextConsume) {\n        this.delayLevelWhenNextConsume = delayLevelWhenNextConsume;\n    }\n\n    public MessageQueue getMessageQueue() {\n        return messageQueue;\n    }\n\n    public int getAckIndex() {\n        return ackIndex;\n    }\n\n    public void setAckIndex(int ackIndex) {\n        this.ackIndex = ackIndex;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/listener/ConsumeConcurrentlyStatus.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.listener;\n\npublic enum ConsumeConcurrentlyStatus {\n    /**\n     * Success consumption\n     */\n    CONSUME_SUCCESS,\n    /**\n     * Failure consumption,later try to consume\n     */\n    RECONSUME_LATER;\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/listener/ConsumeOrderlyContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.listener;\n\nimport org.apache.rocketmq.common.message.MessageQueue;\n\n/**\n * Consumer Orderly consumption context\n */\npublic class ConsumeOrderlyContext {\n    private final MessageQueue messageQueue;\n    private boolean autoCommit = true;\n    private long suspendCurrentQueueTimeMillis = -1;\n\n    public ConsumeOrderlyContext(MessageQueue messageQueue) {\n        this.messageQueue = messageQueue;\n    }\n\n    public boolean isAutoCommit() {\n        return autoCommit;\n    }\n\n    public void setAutoCommit(boolean autoCommit) {\n        this.autoCommit = autoCommit;\n    }\n\n    public MessageQueue getMessageQueue() {\n        return messageQueue;\n    }\n\n    public long getSuspendCurrentQueueTimeMillis() {\n        return suspendCurrentQueueTimeMillis;\n    }\n\n    public void setSuspendCurrentQueueTimeMillis(long suspendCurrentQueueTimeMillis) {\n        this.suspendCurrentQueueTimeMillis = suspendCurrentQueueTimeMillis;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/listener/ConsumeOrderlyStatus.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.listener;\n\npublic enum ConsumeOrderlyStatus {\n    /**\n     * Success consumption\n     */\n    SUCCESS,\n    /**\n     * Rollback consumption(only for binlog consumption)\n     */\n    @Deprecated\n    ROLLBACK,\n    /**\n     * Commit offset(only for binlog consumption)\n     */\n    @Deprecated\n    COMMIT,\n    /**\n     * Suspend current queue a moment\n     */\n    SUSPEND_CURRENT_QUEUE_A_MOMENT;\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/listener/ConsumeReturnType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.consumer.listener;\n\npublic enum ConsumeReturnType {\n    /**\n     * consume return success\n     */\n    SUCCESS,\n    /**\n     * consume timeout ,even if success\n     */\n    TIME_OUT,\n    /**\n     * consume throw exception\n     */\n    EXCEPTION,\n    /**\n     * consume return null\n     */\n    RETURNNULL,\n    /**\n     * consume return failed\n     */\n    FAILED\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/listener/MessageListener.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.listener;\n\n/**\n * A MessageListener object is used to receive asynchronously delivered messages.\n */\npublic interface MessageListener {\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/listener/MessageListenerConcurrently.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.listener;\n\nimport java.util.List;\nimport org.apache.rocketmq.common.message.MessageExt;\n\n/**\n * A MessageListenerConcurrently object is used to receive asynchronously delivered messages concurrently\n */\npublic interface MessageListenerConcurrently extends MessageListener {\n    /**\n     * It is not recommend to throw exception,rather than returning ConsumeConcurrentlyStatus.RECONSUME_LATER if\n     * consumption failure\n     *\n     * @param msgs msgs.size() >= 1<br> DefaultMQPushConsumer.consumeMessageBatchMaxSize=1,you can modify here\n     * @return The consume status\n     */\n    ConsumeConcurrentlyStatus consumeMessage(final List<MessageExt> msgs,\n        final ConsumeConcurrentlyContext context);\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/listener/MessageListenerOrderly.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.listener;\n\nimport java.util.List;\nimport org.apache.rocketmq.common.message.MessageExt;\n\n/**\n * A MessageListenerOrderly object is used to receive messages orderly. One queue by one thread\n */\npublic interface MessageListenerOrderly extends MessageListener {\n    /**\n     * It is not recommend to throw exception,rather than returning ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT\n     * if consumption failure\n     *\n     * @param msgs msgs.size() >= 1<br> DefaultMQPushConsumer.consumeMessageBatchMaxSize=1,you can modify here\n     * @return The consume status\n     */\n    ConsumeOrderlyStatus consumeMessage(final List<MessageExt> msgs,\n        final ConsumeOrderlyContext context);\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AbstractAllocateMessageQueueStrategy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.consumer.rebalance;\n\nimport java.util.List;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic abstract class AbstractAllocateMessageQueueStrategy implements AllocateMessageQueueStrategy {\n\n    private static final Logger log = LoggerFactory.getLogger(AbstractAllocateMessageQueueStrategy.class);\n\n    public boolean check(String consumerGroup, String currentCID, List<MessageQueue> mqAll,\n        List<String> cidAll) {\n        if (StringUtils.isEmpty(currentCID)) {\n            throw new IllegalArgumentException(\"currentCID is empty\");\n        }\n        if (CollectionUtils.isEmpty(mqAll)) {\n            throw new IllegalArgumentException(\"mqAll is null or mqAll empty\");\n        }\n        if (CollectionUtils.isEmpty(cidAll)) {\n            throw new IllegalArgumentException(\"cidAll is null or cidAll empty\");\n        }\n\n        if (!cidAll.contains(currentCID)) {\n            log.info(\"[BUG] ConsumerGroup: {} The consumerId: {} not in cidAll: {}\",\n                consumerGroup,\n                currentCID,\n                cidAll);\n            return false;\n        }\n\n        return true;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMachineRoomNearby.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.rebalance;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Map.Entry;\nimport java.util.TreeMap;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\n/**\n * An allocate strategy proxy for based on machine room nearside priority. An actual allocate strategy can be\n * specified.\n *\n * If any consumer is alive in a machine room, the message queue of the broker which is deployed in the same machine\n * should only be allocated to those. Otherwise, those message queues can be shared along all consumers since there are\n * no alive consumer to monopolize them.\n */\npublic class AllocateMachineRoomNearby extends AbstractAllocateMessageQueueStrategy {\n\n    private final AllocateMessageQueueStrategy allocateMessageQueueStrategy;//actual allocate strategy\n    private final MachineRoomResolver machineRoomResolver;\n\n    public AllocateMachineRoomNearby(AllocateMessageQueueStrategy allocateMessageQueueStrategy,\n        MachineRoomResolver machineRoomResolver) throws NullPointerException {\n        if (allocateMessageQueueStrategy == null) {\n            throw new NullPointerException(\"allocateMessageQueueStrategy is null\");\n        }\n\n        if (machineRoomResolver == null) {\n            throw new NullPointerException(\"machineRoomResolver is null\");\n        }\n\n        this.allocateMessageQueueStrategy = allocateMessageQueueStrategy;\n        this.machineRoomResolver = machineRoomResolver;\n    }\n\n    @Override\n    public List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll,\n        List<String> cidAll) {\n\n        List<MessageQueue> result = new ArrayList<>();\n        if (!check(consumerGroup, currentCID, mqAll, cidAll)) {\n            return result;\n        }\n\n        //group mq by machine room\n        Map<String/*machine room */, List<MessageQueue>> mr2Mq = new TreeMap<>();\n        for (MessageQueue mq : mqAll) {\n            String brokerMachineRoom = machineRoomResolver.brokerDeployIn(mq);\n            if (StringUtils.isNoneEmpty(brokerMachineRoom)) {\n                if (mr2Mq.get(brokerMachineRoom) == null) {\n                    mr2Mq.put(brokerMachineRoom, new ArrayList<>());\n                }\n                mr2Mq.get(brokerMachineRoom).add(mq);\n            } else {\n                throw new IllegalArgumentException(\"Machine room is null for mq \" + mq);\n            }\n        }\n\n        //group consumer by machine room\n        Map<String/*machine room */, List<String/*clientId*/>> mr2c = new TreeMap<>();\n        for (String cid : cidAll) {\n            String consumerMachineRoom = machineRoomResolver.consumerDeployIn(cid);\n            if (StringUtils.isNoneEmpty(consumerMachineRoom)) {\n                if (mr2c.get(consumerMachineRoom) == null) {\n                    mr2c.put(consumerMachineRoom, new ArrayList<>());\n                }\n                mr2c.get(consumerMachineRoom).add(cid);\n            } else {\n                throw new IllegalArgumentException(\"Machine room is null for consumer id \" + cid);\n            }\n        }\n\n        List<MessageQueue> allocateResults = new ArrayList<>();\n\n        //1.allocate the mq that deploy in the same machine room with the current consumer\n        String currentMachineRoom = machineRoomResolver.consumerDeployIn(currentCID);\n        List<MessageQueue> mqInThisMachineRoom = mr2Mq.remove(currentMachineRoom);\n        List<String> consumerInThisMachineRoom = mr2c.get(currentMachineRoom);\n        if (mqInThisMachineRoom != null && !mqInThisMachineRoom.isEmpty()) {\n            allocateResults.addAll(allocateMessageQueueStrategy.allocate(consumerGroup, currentCID, mqInThisMachineRoom, consumerInThisMachineRoom));\n        }\n\n        //2.allocate the rest mq to each machine room if there are no consumer alive in that machine room\n        for (Entry<String, List<MessageQueue>> machineRoomEntry : mr2Mq.entrySet()) {\n            if (!mr2c.containsKey(machineRoomEntry.getKey())) { // no alive consumer in the corresponding machine room, so all consumers share these queues\n                allocateResults.addAll(allocateMessageQueueStrategy.allocate(consumerGroup, currentCID, machineRoomEntry.getValue(), cidAll));\n            }\n        }\n\n        return allocateResults;\n    }\n\n    @Override\n    public String getName() {\n        return \"MACHINE_ROOM_NEARBY\" + \"-\" + allocateMessageQueueStrategy.getName();\n    }\n\n    /**\n     * A resolver object to determine which machine room do the message queues or clients are deployed in.\n     *\n     * AllocateMachineRoomNearby will use the results to group the message queues and clients by machine room.\n     *\n     * The result returned from the implemented method CANNOT be null.\n     */\n    public interface MachineRoomResolver {\n        String brokerDeployIn(MessageQueue messageQueue);\n\n        String consumerDeployIn(String clientID);\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.rebalance;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\n/**\n * Average Hashing queue algorithm\n */\npublic class AllocateMessageQueueAveragely extends AbstractAllocateMessageQueueStrategy {\n\n    @Override\n    public List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll,\n        List<String> cidAll) {\n\n        List<MessageQueue> result = new ArrayList<>();\n        if (!check(consumerGroup, currentCID, mqAll, cidAll)) {\n            return result;\n        }\n\n        int index = cidAll.indexOf(currentCID);\n        int mod = mqAll.size() % cidAll.size();\n        int averageSize =\n            mqAll.size() <= cidAll.size() ? 1 : (mod > 0 && index < mod ? mqAll.size() / cidAll.size()\n                + 1 : mqAll.size() / cidAll.size());\n        int startIndex = (mod > 0 && index < mod) ? index * averageSize : index * averageSize + mod;\n        int range = Math.min(averageSize, mqAll.size() - startIndex);\n        for (int i = 0; i < range; i++) {\n            result.add(mqAll.get(startIndex + i));\n        }\n        return result;\n    }\n\n    @Override\n    public String getName() {\n        return \"AVG\";\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.rebalance;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\n/**\n * Cycle average Hashing queue algorithm\n */\npublic class AllocateMessageQueueAveragelyByCircle extends AbstractAllocateMessageQueueStrategy {\n\n    @Override\n    public List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll,\n        List<String> cidAll) {\n\n        List<MessageQueue> result = new ArrayList<>();\n        if (!check(consumerGroup, currentCID, mqAll, cidAll)) {\n            return result;\n        }\n\n        int index = cidAll.indexOf(currentCID);\n        for (int i = index; i < mqAll.size(); i++) {\n            if (i % cidAll.size() == index) {\n                result.add(mqAll.get(i));\n            }\n        }\n        return result;\n    }\n\n    @Override\n    public String getName() {\n        return \"AVG_BY_CIRCLE\";\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueByConfig.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.rebalance;\n\nimport java.util.List;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\npublic class AllocateMessageQueueByConfig extends AbstractAllocateMessageQueueStrategy {\n    private List<MessageQueue> messageQueueList;\n\n    @Override\n    public List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll,\n        List<String> cidAll) {\n        return this.messageQueueList;\n    }\n\n    @Override\n    public String getName() {\n        return \"CONFIG\";\n    }\n\n    public List<MessageQueue> getMessageQueueList() {\n        return messageQueueList;\n    }\n\n    public void setMessageQueueList(List<MessageQueue> messageQueueList) {\n        this.messageQueueList = messageQueueList;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueByMachineRoom.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.rebalance;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Set;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\n/**\n * Computer room Hashing queue algorithm, such as Alipay logic room\n */\npublic class AllocateMessageQueueByMachineRoom extends AbstractAllocateMessageQueueStrategy {\n    private Set<String> consumeridcs;\n\n    @Override\n    public List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll,\n        List<String> cidAll) {\n\n        List<MessageQueue> result = new ArrayList<>();\n        if (!check(consumerGroup, currentCID, mqAll, cidAll)) {\n            return result;\n        }\n        int currentIndex = cidAll.indexOf(currentCID);\n        if (currentIndex < 0) {\n            return result;\n        }\n        List<MessageQueue> premqAll = new ArrayList<>();\n        for (MessageQueue mq : mqAll) {\n            String[] temp = mq.getBrokerName().split(\"@\");\n            if (temp.length == 2 && consumeridcs.contains(temp[0])) {\n                premqAll.add(mq);\n            }\n        }\n\n        int mod = premqAll.size() / cidAll.size();\n        int rem = premqAll.size() % cidAll.size();\n        int startIndex = mod * currentIndex;\n        int endIndex = startIndex + mod;\n        for (int i = startIndex; i < endIndex; i++) {\n            result.add(premqAll.get(i));\n        }\n        if (rem > currentIndex) {\n            result.add(premqAll.get(currentIndex + mod * cidAll.size()));\n        }\n        return result;\n    }\n\n    @Override\n    public String getName() {\n        return \"MACHINE_ROOM\";\n    }\n\n    public Set<String> getConsumeridcs() {\n        return consumeridcs;\n    }\n\n    public void setConsumeridcs(Set<String> consumeridcs) {\n        this.consumeridcs = consumeridcs;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsistentHash.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.rebalance;\n\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.List;\nimport org.apache.rocketmq.common.consistenthash.ConsistentHashRouter;\nimport org.apache.rocketmq.common.consistenthash.HashFunction;\nimport org.apache.rocketmq.common.consistenthash.Node;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\n/**\n * Consistent Hashing queue algorithm\n */\npublic class AllocateMessageQueueConsistentHash extends AbstractAllocateMessageQueueStrategy {\n\n    private final int virtualNodeCnt;\n    private final HashFunction customHashFunction;\n\n    public AllocateMessageQueueConsistentHash() {\n        this(10);\n    }\n\n    public AllocateMessageQueueConsistentHash(int virtualNodeCnt) {\n        this(virtualNodeCnt, null);\n    }\n\n    public AllocateMessageQueueConsistentHash(int virtualNodeCnt, HashFunction customHashFunction) {\n        if (virtualNodeCnt < 0) {\n            throw new IllegalArgumentException(\"illegal virtualNodeCnt :\" + virtualNodeCnt);\n        }\n        this.virtualNodeCnt = virtualNodeCnt;\n        this.customHashFunction = customHashFunction;\n    }\n\n    @Override\n    public List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll,\n        List<String> cidAll) {\n\n        List<MessageQueue> result = new ArrayList<>();\n        if (!check(consumerGroup, currentCID, mqAll, cidAll)) {\n            return result;\n        }\n\n        Collection<ClientNode> cidNodes = new ArrayList<>();\n        for (String cid : cidAll) {\n            cidNodes.add(new ClientNode(cid));\n        }\n\n        final ConsistentHashRouter<ClientNode> router; //for building hash ring\n        if (customHashFunction != null) {\n            router = new ConsistentHashRouter<>(cidNodes, virtualNodeCnt, customHashFunction);\n        } else {\n            router = new ConsistentHashRouter<>(cidNodes, virtualNodeCnt);\n        }\n\n        List<MessageQueue> results = new ArrayList<>();\n        for (MessageQueue mq : mqAll) {\n            ClientNode clientNode = router.routeNode(mq.toString());\n            if (clientNode != null && currentCID.equals(clientNode.getKey())) {\n                results.add(mq);\n            }\n        }\n\n        return results;\n\n    }\n\n    @Override\n    public String getName() {\n        return \"CONSISTENT_HASH\";\n    }\n\n    private static class ClientNode implements Node {\n        private final String clientID;\n\n        public ClientNode(String clientID) {\n            this.clientID = clientID;\n        }\n\n        @Override\n        public String getKey() {\n            return clientID;\n        }\n    }\n\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/store/ControllableOffset.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.consumer.store;\n\nimport java.util.concurrent.atomic.AtomicLong;\n\n/**\n * The ControllableOffset class encapsulates a thread-safe offset value that can be\n * updated atomically. Additionally, this class allows for the offset to be \"frozen,\"\n * which prevents further updates after the freeze operation has been performed.\n * <p>\n * Concurrency Scenarios:\n * If {@code updateAndFreeze} is called before any {@code update} operations, it sets\n * {@code allowToUpdate} to false and updates the offset to the target value specified.\n * After this operation, further invocations of {@code update} will not affect the offset,\n * as it is considered frozen.\n * <p>\n * If {@code update} is in progress while {@code updateAndFreeze} is invoked concurrently,\n * the final outcome depends on the sequence of operations:\n * 1. If {@code update}'s atomic update operation completes before {@code updateAndFreeze},\n * the latter will overwrite the offset and set {@code allowToUpdate} to false,\n * preventing any further updates.\n * 2. If {@code updateAndFreeze} executes before the {@code update} finalizes its operation,\n * the ongoing {@code update} will not proceed with its changes. The {@link AtomicLong#getAndUpdate}\n * method used in both operations ensures atomicity and respects the final state imposed by\n * {@code updateAndFreeze}, even if the {@code update} function has already begun.\n * <p>\n * In essence, once the {@code updateAndFreeze} operation is executed, the offset value remains\n * immutable to any subsequent {@code update} calls due to the immediate visibility of the\n * {@code allowToUpdate} state change, courtesy of its volatile nature.\n * <p>\n * The combination of an AtomicLong for the offset value and a volatile boolean flag for update\n * control provides a reliable mechanism for managing offset values in concurrent environments.\n */\npublic class ControllableOffset {\n    // Holds the current offset value in an atomic way.\n    private final AtomicLong value;\n    // Controls whether updates to the offset are allowed.\n    private volatile boolean allowToUpdate;\n\n    public ControllableOffset(long value) {\n        this.value = new AtomicLong(value);\n        this.allowToUpdate = true;\n    }\n\n    /**\n     * Attempts to update the offset to the target value. If increaseOnly is true,\n     * the offset will not be decreased. The update operation is atomic and thread-safe.\n     * The operation will respect the current allowToUpdate state, and if the offset\n     * has been frozen by a previous call to {@link #updateAndFreeze(long)},\n     * this method will not update the offset.\n     *\n     * @param target       the new target offset value.\n     * @param increaseOnly if true, the offset will only be updated if the target value\n     *                     is greater than the current value.\n     */\n    public void update(long target, boolean increaseOnly) {\n        if (allowToUpdate) {\n            value.getAndUpdate(val -> {\n                if (allowToUpdate) {\n                    if (increaseOnly) {\n                        return Math.max(target, val);\n                    } else {\n                        return target;\n                    }\n                } else {\n                    return val;\n                }\n            });\n        }\n    }\n\n    /**\n     * Overloaded method for updating the offset value unconditionally.\n     *\n     * @param target The new target value for the offset.\n     */\n    public void update(long target) {\n        update(target, false);\n    }\n\n    /**\n     * Freezes the offset at the target value provided. Once frozen, the offset\n     * cannot be updated by subsequent calls to {@link #update(long, boolean)}.\n     * This method will set allowToUpdate to false and then update the offset,\n     * ensuring the new value is the final state of the offset.\n     *\n     * @param target the new target offset value to freeze at.\n     */\n    public void updateAndFreeze(long target) {\n        value.getAndUpdate(val -> {\n            allowToUpdate = false;\n            return target;\n        });\n    }\n\n    public long getOffset() {\n        return value.get();\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/store/LocalFileOffsetStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.store;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.Map.Entry;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.help.FAQUrl;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\n/**\n * Local storage implementation\n */\npublic class LocalFileOffsetStore implements OffsetStore {\n    public final static String LOCAL_OFFSET_STORE_DIR = System.getProperty(\n        \"rocketmq.client.localOffsetStoreDir\",\n        System.getProperty(\"user.home\") + File.separator + \".rocketmq_offsets\");\n    private final static Logger log = LoggerFactory.getLogger(LocalFileOffsetStore.class);\n    private final MQClientInstance mQClientFactory;\n    private final String groupName;\n    private final String storePath;\n    private ConcurrentMap<MessageQueue, ControllableOffset> offsetTable =\n        new ConcurrentHashMap<>();\n\n    public LocalFileOffsetStore(MQClientInstance mQClientFactory, String groupName) {\n        this.mQClientFactory = mQClientFactory;\n        this.groupName = groupName;\n        this.storePath = LOCAL_OFFSET_STORE_DIR + File.separator +\n            this.mQClientFactory.getClientId() + File.separator +\n            this.groupName + File.separator +\n            \"offsets.json\";\n    }\n\n    @Override\n    public void load() throws MQClientException {\n        OffsetSerializeWrapper offsetSerializeWrapper = this.readLocalOffset();\n        if (offsetSerializeWrapper != null && offsetSerializeWrapper.getOffsetTable() != null) {\n            for (Entry<MessageQueue, AtomicLong> mqEntry : offsetSerializeWrapper.getOffsetTable().entrySet()) {\n                AtomicLong offset = mqEntry.getValue();\n                offsetTable.put(mqEntry.getKey(), new ControllableOffset(offset.get()));\n                log.info(\"load consumer's offset, {} {} {}\",\n                        this.groupName,\n                        mqEntry.getKey(),\n                        offset.get());\n            }\n        }\n    }\n\n    @Override\n    public void updateOffset(MessageQueue mq, long offset, boolean increaseOnly) {\n        if (mq != null) {\n            ControllableOffset offsetOld = this.offsetTable.get(mq);\n            if (null == offsetOld) {\n                offsetOld = this.offsetTable.putIfAbsent(mq, new ControllableOffset(offset));\n            }\n\n            if (null != offsetOld) {\n                if (increaseOnly) {\n                    offsetOld.update(offset, true);\n                } else {\n                    offsetOld.update(offset);\n                }\n            }\n        }\n    }\n\n    @Override\n    public void updateAndFreezeOffset(MessageQueue mq, long offset) {\n        if (mq != null) {\n            this.offsetTable.computeIfAbsent(mq, k -> new ControllableOffset(offset))\n                .updateAndFreeze(offset);\n        }\n    }\n\n    @Override\n    public long readOffset(final MessageQueue mq, final ReadOffsetType type) {\n        if (mq != null) {\n            switch (type) {\n                case MEMORY_FIRST_THEN_STORE:\n                case READ_FROM_MEMORY: {\n                    ControllableOffset offset = this.offsetTable.get(mq);\n                    if (offset != null) {\n                        return offset.getOffset();\n                    } else if (ReadOffsetType.READ_FROM_MEMORY == type) {\n                        return -1;\n                    }\n                }\n                case READ_FROM_STORE: {\n                    OffsetSerializeWrapper offsetSerializeWrapper;\n                    try {\n                        offsetSerializeWrapper = this.readLocalOffset();\n                    } catch (MQClientException e) {\n                        return -1;\n                    }\n                    if (offsetSerializeWrapper != null && offsetSerializeWrapper.getOffsetTable() != null) {\n                        AtomicLong offset = offsetSerializeWrapper.getOffsetTable().get(mq);\n                        if (offset != null) {\n                            this.updateOffset(mq, offset.get(), false);\n                            return offset.get();\n                        }\n                    }\n                }\n                default:\n                    break;\n            }\n        }\n\n        return -1;\n    }\n\n    @Override\n    public void persistAll(Set<MessageQueue> mqs) {\n        if (null == mqs || mqs.isEmpty()) {\n            return;\n        }\n        OffsetSerializeWrapper offsetSerializeWrapper = null;\n        try {\n            offsetSerializeWrapper = readLocalOffset();\n        } catch (MQClientException e) {\n            log.error(\"readLocalOffset exception\", e);\n            return;\n        }\n\n        if (offsetSerializeWrapper == null) {\n            offsetSerializeWrapper = new OffsetSerializeWrapper();\n        }\n        for (Map.Entry<MessageQueue, ControllableOffset> entry : this.offsetTable.entrySet()) {\n            if (mqs.contains(entry.getKey())) {\n                AtomicLong offset = new AtomicLong(entry.getValue().getOffset());\n                offsetSerializeWrapper.getOffsetTable().put(entry.getKey(), offset);\n            }\n        }\n\n        String jsonString = offsetSerializeWrapper.toJson(true);\n        if (jsonString != null) {\n            try {\n                MixAll.string2File(jsonString, this.storePath);\n            } catch (IOException e) {\n                log.error(\"persistAll consumer offset Exception, \" + this.storePath, e);\n            }\n        }\n    }\n\n    @Override\n    public void persist(MessageQueue mq) {\n        if (mq == null) {\n            return;\n        }\n        ControllableOffset offset = this.offsetTable.get(mq);\n        if (offset != null) {\n            OffsetSerializeWrapper offsetSerializeWrapper = null;\n            try {\n                offsetSerializeWrapper = readLocalOffset();\n            } catch (MQClientException e) {\n                log.error(\"readLocalOffset exception\", e);\n                return;\n            }\n            if (offsetSerializeWrapper == null) {\n                offsetSerializeWrapper = new OffsetSerializeWrapper();\n            }\n            offsetSerializeWrapper.getOffsetTable().put(mq, new AtomicLong(offset.getOffset()));\n            String jsonString = offsetSerializeWrapper.toJson(true);\n            if (jsonString != null) {\n                try {\n                    MixAll.string2File(jsonString, this.storePath);\n                } catch (IOException e) {\n                    log.error(\"persist consumer offset exception, \" + this.storePath, e);\n                }\n            }\n        }\n    }\n\n    @Override\n    public void removeOffset(MessageQueue mq) {\n        if (mq != null) {\n            this.offsetTable.remove(mq);\n            log.info(\"remove unnecessary messageQueue offset. group={}, mq={}, offsetTableSize={}\", this.groupName, mq,\n                offsetTable.size());\n        }\n    }\n\n    @Override\n    public void updateConsumeOffsetToBroker(final MessageQueue mq, final long offset, final boolean isOneway)\n        throws RemotingException, MQBrokerException, InterruptedException, MQClientException {\n\n    }\n\n    @Override\n    public Map<MessageQueue, Long> cloneOffsetTable(String topic) {\n        Map<MessageQueue, Long> cloneOffsetTable = new HashMap<>(this.offsetTable.size(), 1);\n        for (Map.Entry<MessageQueue, ControllableOffset> entry : this.offsetTable.entrySet()) {\n            MessageQueue mq = entry.getKey();\n            if (!UtilAll.isBlank(topic) && !topic.equals(mq.getTopic())) {\n                continue;\n            }\n            cloneOffsetTable.put(mq, entry.getValue().getOffset());\n\n        }\n        return cloneOffsetTable;\n    }\n\n    private OffsetSerializeWrapper readLocalOffset() throws MQClientException {\n        String content = null;\n        try {\n            content = MixAll.file2String(this.storePath);\n        } catch (IOException e) {\n            log.warn(\"Load local offset store file exception\", e);\n        }\n        if (null == content || content.length() == 0) {\n            return this.readLocalOffsetBak();\n        } else {\n            OffsetSerializeWrapper offsetSerializeWrapper = null;\n            try {\n                offsetSerializeWrapper =\n                    OffsetSerializeWrapper.fromJson(content, OffsetSerializeWrapper.class);\n            } catch (Exception e) {\n                log.warn(\"readLocalOffset Exception, and try to correct\", e);\n                return this.readLocalOffsetBak();\n            }\n\n            return offsetSerializeWrapper;\n        }\n    }\n\n    private OffsetSerializeWrapper readLocalOffsetBak() throws MQClientException {\n        String content = null;\n        try {\n            content = MixAll.file2String(this.storePath + \".bak\");\n        } catch (IOException e) {\n            log.warn(\"Load local offset store bak file exception\", e);\n        }\n        if (content != null && content.length() > 0) {\n            OffsetSerializeWrapper offsetSerializeWrapper = null;\n            try {\n                offsetSerializeWrapper =\n                    OffsetSerializeWrapper.fromJson(content, OffsetSerializeWrapper.class);\n            } catch (Exception e) {\n                log.warn(\"readLocalOffset Exception\", e);\n                throw new MQClientException(\"readLocalOffset Exception, maybe fastjson version too low\"\n                    + FAQUrl.suggestTodo(FAQUrl.LOAD_JSON_EXCEPTION),\n                    e);\n            }\n            return offsetSerializeWrapper;\n        }\n\n        return null;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/store/OffsetSerializeWrapper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.store;\n\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\n/**\n * Wrapper class for offset serialization\n */\npublic class OffsetSerializeWrapper extends RemotingSerializable {\n    private ConcurrentMap<MessageQueue, AtomicLong> offsetTable =\n        new ConcurrentHashMap<>();\n\n    public ConcurrentMap<MessageQueue, AtomicLong> getOffsetTable() {\n        return offsetTable;\n    }\n\n    public void setOffsetTable(ConcurrentMap<MessageQueue, AtomicLong> offsetTable) {\n        this.offsetTable = offsetTable;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/store/OffsetStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.store;\n\nimport java.util.Map;\nimport java.util.Set;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\n\n/**\n * Offset store interface\n */\npublic interface OffsetStore {\n    /**\n     * Load\n     */\n    void load() throws MQClientException;\n\n    /**\n     * Update the offset,store it in memory\n     */\n    void updateOffset(final MessageQueue mq, final long offset, final boolean increaseOnly);\n\n    /**\n     * Update and freeze the message queue to prevent concurrent update action\n     *\n     * @param mq target message queue\n     * @param offset expect update offset\n     */\n    void updateAndFreezeOffset(final MessageQueue mq, final long offset);\n\n    /**\n     * Get offset from local storage\n     *\n     * @return The fetched offset\n     */\n    long readOffset(final MessageQueue mq, final ReadOffsetType type);\n\n    /**\n     * Persist all offsets,may be in local storage or remote name server\n     */\n    void persistAll(final Set<MessageQueue> mqs);\n\n    /**\n     * Persist the offset,may be in local storage or remote name server\n     */\n    void persist(final MessageQueue mq);\n\n    /**\n     * Remove offset\n     */\n    void removeOffset(MessageQueue mq);\n\n    /**\n     * @return The cloned offset table of given topic\n     */\n    Map<MessageQueue, Long> cloneOffsetTable(String topic);\n\n    /**\n     * @param mq\n     * @param offset\n     * @param isOneway\n     */\n    void updateConsumeOffsetToBroker(MessageQueue mq, long offset, boolean isOneway) throws RemotingException,\n        MQBrokerException, InterruptedException, MQClientException;\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/store/ReadOffsetType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.store;\n\npublic enum ReadOffsetType {\n    /**\n     * From memory\n     */\n    READ_FROM_MEMORY,\n    /**\n     * From storage\n     */\n    READ_FROM_STORE,\n    /**\n     * From memory,then from storage\n     */\n    MEMORY_FIRST_THEN_STORE;\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.store;\n\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.exception.OffsetNotFoundException;\nimport org.apache.rocketmq.client.impl.FindBrokerResult;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;\n\n/**\n * Remote storage implementation\n */\npublic class RemoteBrokerOffsetStore implements OffsetStore {\n    private final static Logger log = LoggerFactory.getLogger(RemoteBrokerOffsetStore.class);\n    private final MQClientInstance mQClientFactory;\n    private final String groupName;\n    private ConcurrentMap<MessageQueue, ControllableOffset> offsetTable =\n        new ConcurrentHashMap<>();\n\n    public RemoteBrokerOffsetStore(MQClientInstance mQClientFactory, String groupName) {\n        this.mQClientFactory = mQClientFactory;\n        this.groupName = groupName;\n    }\n\n    @Override\n    public void load() {\n    }\n\n    @Override\n    public void updateOffset(MessageQueue mq, long offset, boolean increaseOnly) {\n        if (mq != null) {\n            ControllableOffset offsetOld = this.offsetTable.get(mq);\n            if (null == offsetOld) {\n                offsetOld = this.offsetTable.putIfAbsent(mq, new ControllableOffset(offset));\n            }\n\n            if (null != offsetOld) {\n                if (increaseOnly) {\n                    offsetOld.update(offset, true);\n                } else {\n                    offsetOld.update(offset);\n                }\n            }\n        }\n    }\n\n    @Override\n    public void updateAndFreezeOffset(MessageQueue mq, long offset) {\n        if (mq != null) {\n            this.offsetTable.computeIfAbsent(mq, k -> new ControllableOffset(offset))\n                .updateAndFreeze(offset);\n        }\n    }\n\n    @Override\n    public long readOffset(final MessageQueue mq, final ReadOffsetType type) {\n        if (mq != null) {\n            switch (type) {\n                case MEMORY_FIRST_THEN_STORE:\n                case READ_FROM_MEMORY: {\n                    ControllableOffset offset = this.offsetTable.get(mq);\n                    if (offset != null) {\n                        return offset.getOffset();\n                    } else if (ReadOffsetType.READ_FROM_MEMORY == type) {\n                        return -1;\n                    }\n                }\n                case READ_FROM_STORE: {\n                    try {\n                        long brokerOffset = this.fetchConsumeOffsetFromBroker(mq);\n                        this.updateOffset(mq, brokerOffset, false);\n                        return brokerOffset;\n                    }\n                    // No offset in broker\n                    catch (OffsetNotFoundException e) {\n                        return -1;\n                    }\n                    //Other exceptions\n                    catch (Exception e) {\n                        log.warn(\"fetchConsumeOffsetFromBroker exception, \" + mq, e);\n                        return -2;\n                    }\n                }\n                default:\n                    break;\n            }\n        }\n\n        return -3;\n    }\n\n    @Override\n    public void persistAll(Set<MessageQueue> mqs) {\n        if (null == mqs || mqs.isEmpty())\n            return;\n\n        final HashSet<MessageQueue> unusedMQ = new HashSet<>();\n\n        for (Map.Entry<MessageQueue, ControllableOffset> entry : this.offsetTable.entrySet()) {\n            MessageQueue mq = entry.getKey();\n            ControllableOffset offset = entry.getValue();\n            if (offset != null) {\n                if (mqs.contains(mq)) {\n                    try {\n                        this.updateConsumeOffsetToBroker(mq, offset.getOffset());\n                        log.info(\"[persistAll] Group: {} ClientId: {} updateConsumeOffsetToBroker {} {}\",\n                            this.groupName,\n                            this.mQClientFactory.getClientId(),\n                            mq,\n                            offset.getOffset());\n                    } catch (Exception e) {\n                        log.error(\"updateConsumeOffsetToBroker exception, \" + mq.toString(), e);\n                    }\n                } else {\n                    unusedMQ.add(mq);\n                }\n            }\n        }\n\n        if (!unusedMQ.isEmpty()) {\n            for (MessageQueue mq : unusedMQ) {\n                this.offsetTable.remove(mq);\n                log.info(\"remove unused mq, {}, {}\", mq, this.groupName);\n            }\n        }\n    }\n\n    @Override\n    public void persist(MessageQueue mq) {\n        ControllableOffset offset = this.offsetTable.get(mq);\n        if (offset != null) {\n            try {\n                this.updateConsumeOffsetToBroker(mq, offset.getOffset());\n                log.info(\"[persist] Group: {} ClientId: {} updateConsumeOffsetToBroker {} {}\",\n                    this.groupName,\n                    this.mQClientFactory.getClientId(),\n                    mq,\n                    offset.getOffset());\n            } catch (Exception e) {\n                log.error(\"updateConsumeOffsetToBroker exception, \" + mq.toString(), e);\n            }\n        }\n    }\n\n    public void removeOffset(MessageQueue mq) {\n        if (mq != null) {\n            this.offsetTable.remove(mq);\n            log.info(\"remove unnecessary messageQueue offset. group={}, mq={}, offsetTableSize={}\", this.groupName, mq,\n                offsetTable.size());\n        }\n    }\n\n    @Override\n    public Map<MessageQueue, Long> cloneOffsetTable(String topic) {\n        Map<MessageQueue, Long> cloneOffsetTable = new HashMap<>(this.offsetTable.size(), 1);\n        for (Map.Entry<MessageQueue, ControllableOffset> entry : this.offsetTable.entrySet()) {\n            MessageQueue mq = entry.getKey();\n            if (!UtilAll.isBlank(topic) && !topic.equals(mq.getTopic())) {\n                continue;\n            }\n            cloneOffsetTable.put(mq, entry.getValue().getOffset());\n        }\n        return cloneOffsetTable;\n    }\n\n    /**\n     * Update the Consumer Offset in one way, once the Master is off, updated to Slave, here need to be optimized.\n     */\n    private void updateConsumeOffsetToBroker(MessageQueue mq, long offset) throws RemotingException,\n        MQBrokerException, InterruptedException, MQClientException {\n        updateConsumeOffsetToBroker(mq, offset, true);\n    }\n\n    /**\n     * Update the Consumer Offset synchronously, once the Master is off, updated to Slave, here need to be optimized.\n     */\n    @Override\n    public void updateConsumeOffsetToBroker(MessageQueue mq, long offset, boolean isOneway) throws RemotingException,\n        MQBrokerException, InterruptedException, MQClientException {\n        FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(this.mQClientFactory.getBrokerNameFromMessageQueue(mq), MixAll.MASTER_ID, false);\n        if (null == findBrokerResult) {\n            this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());\n            findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(this.mQClientFactory.getBrokerNameFromMessageQueue(mq), MixAll.MASTER_ID, false);\n        }\n\n        if (findBrokerResult != null) {\n            UpdateConsumerOffsetRequestHeader requestHeader = new UpdateConsumerOffsetRequestHeader();\n            requestHeader.setTopic(mq.getTopic());\n            requestHeader.setConsumerGroup(this.groupName);\n            requestHeader.setQueueId(mq.getQueueId());\n            requestHeader.setCommitOffset(offset);\n            requestHeader.setBrokerName(mq.getBrokerName());\n\n            if (isOneway) {\n                this.mQClientFactory.getMQClientAPIImpl().updateConsumerOffsetOneway(\n                    findBrokerResult.getBrokerAddr(), requestHeader, 1000 * 5);\n            } else {\n                this.mQClientFactory.getMQClientAPIImpl().updateConsumerOffset(\n                    findBrokerResult.getBrokerAddr(), requestHeader, 1000 * 5);\n            }\n        } else {\n            throw new MQClientException(\"The broker[\" + mq.getBrokerName() + \"] not exist\", null);\n        }\n    }\n\n    private long fetchConsumeOffsetFromBroker(MessageQueue mq) throws RemotingException, MQBrokerException,\n        InterruptedException, MQClientException {\n        FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(this.mQClientFactory.getBrokerNameFromMessageQueue(mq), MixAll.MASTER_ID, true);\n        if (null == findBrokerResult) {\n            this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());\n            findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(this.mQClientFactory.getBrokerNameFromMessageQueue(mq), MixAll.MASTER_ID, false);\n        }\n\n        if (findBrokerResult != null) {\n            QueryConsumerOffsetRequestHeader requestHeader = new QueryConsumerOffsetRequestHeader();\n            requestHeader.setTopic(mq.getTopic());\n            requestHeader.setConsumerGroup(this.groupName);\n            requestHeader.setQueueId(mq.getQueueId());\n            requestHeader.setBrokerName(mq.getBrokerName());\n\n            return this.mQClientFactory.getMQClientAPIImpl().queryConsumerOffset(\n                findBrokerResult.getBrokerAddr(), requestHeader, 1000 * 5);\n        } else {\n            throw new MQClientException(\"The broker[\" + mq.getBrokerName() + \"] not exist\", null);\n        }\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/exception/MQBrokerException.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.exception;\n\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.help.FAQUrl;\n\npublic class MQBrokerException extends Exception {\n    private static final long serialVersionUID = 5975020272601250368L;\n    private final int responseCode;\n    private final String errorMessage;\n    private final String brokerAddr;\n\n    MQBrokerException() {\n        this.responseCode = 0;\n        this.errorMessage = null;\n        this.brokerAddr = null;\n    }\n\n    public MQBrokerException(int responseCode, String errorMessage) {\n        super(FAQUrl.attachDefaultURL(\"CODE: \" + UtilAll.responseCode2String(responseCode) + \"  DESC: \"\n                + errorMessage));\n        this.responseCode = responseCode;\n        this.errorMessage = errorMessage;\n        this.brokerAddr = null;\n    }\n\n    public MQBrokerException(int responseCode, String errorMessage, String brokerAddr) {\n        super(FAQUrl.attachDefaultURL(\"CODE: \" + UtilAll.responseCode2String(responseCode) + \"  DESC: \"\n            + errorMessage + (brokerAddr != null ? \" BROKER: \" + brokerAddr : \"\")));\n        this.responseCode = responseCode;\n        this.errorMessage = errorMessage;\n        this.brokerAddr = brokerAddr;\n    }\n\n    public int getResponseCode() {\n        return responseCode;\n    }\n\n    public String getErrorMessage() {\n        return errorMessage;\n    }\n\n    public String getBrokerAddr() {\n        return brokerAddr;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/exception/MQClientException.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.exception;\n\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.help.FAQUrl;\n\npublic class MQClientException extends Exception {\n    private static final long serialVersionUID = -5758410930844185841L;\n    private int responseCode;\n    private String errorMessage;\n\n    public MQClientException(String errorMessage, Throwable cause) {\n        super(FAQUrl.attachDefaultURL(errorMessage), cause);\n        this.responseCode = -1;\n        this.errorMessage = errorMessage;\n    }\n\n    public MQClientException(int responseCode, String errorMessage) {\n        super(FAQUrl.attachDefaultURL(\"CODE: \" + UtilAll.responseCode2String(responseCode) + \"  DESC: \"\n            + errorMessage));\n        this.responseCode = responseCode;\n        this.errorMessage = errorMessage;\n    }\n\n    public MQClientException(int responseCode, String errorMessage, Throwable cause) {\n        super(FAQUrl.attachDefaultURL(\"CODE: \" + UtilAll.responseCode2String(responseCode) + \"  DESC: \"\n            + errorMessage), cause);\n        this.responseCode = responseCode;\n        this.errorMessage = errorMessage;\n    }\n\n    public int getResponseCode() {\n        return responseCode;\n    }\n\n    public MQClientException setResponseCode(final int responseCode) {\n        this.responseCode = responseCode;\n        return this;\n    }\n\n    public String getErrorMessage() {\n        return errorMessage;\n    }\n\n    public void setErrorMessage(final String errorMessage) {\n        this.errorMessage = errorMessage;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/exception/OffsetNotFoundException.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.exception;\n\npublic class OffsetNotFoundException extends MQBrokerException {\n\n    public OffsetNotFoundException() {\n    }\n\n    public OffsetNotFoundException(int responseCode, String errorMessage) {\n        super(responseCode, errorMessage);\n    }\n\n    public OffsetNotFoundException(int responseCode, String errorMessage, String brokerAddr) {\n        super(responseCode, errorMessage, brokerAddr);\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/exception/RequestTimeoutException.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.exception;\n\nimport org.apache.rocketmq.common.UtilAll;\n\npublic class RequestTimeoutException extends Exception {\n    private static final long serialVersionUID = -5758410930844185841L;\n    private int responseCode;\n    private String errorMessage;\n\n    public RequestTimeoutException(String errorMessage, Throwable cause) {\n        super(errorMessage, cause);\n        this.responseCode = -1;\n        this.errorMessage = errorMessage;\n    }\n\n    public RequestTimeoutException(int responseCode, String errorMessage) {\n        super(\"CODE: \" + UtilAll.responseCode2String(responseCode) + \"  DESC: \"\n            + errorMessage);\n        this.responseCode = responseCode;\n        this.errorMessage = errorMessage;\n    }\n\n    public int getResponseCode() {\n        return responseCode;\n    }\n\n    public RequestTimeoutException setResponseCode(final int responseCode) {\n        this.responseCode = responseCode;\n        return this;\n    }\n\n    public String getErrorMessage() {\n        return errorMessage;\n    }\n\n    public void setErrorMessage(final String errorMessage) {\n        this.errorMessage = errorMessage;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/hook/CheckForbiddenContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.hook;\n\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\npublic class CheckForbiddenContext {\n    private String nameSrvAddr;\n    private String group;\n    private Message message;\n    private MessageQueue mq;\n    private String brokerAddr;\n    private CommunicationMode communicationMode;\n    private SendResult sendResult;\n    private Exception exception;\n    private Object arg;\n    private boolean unitMode = false;\n\n    public String getGroup() {\n        return group;\n    }\n\n    public void setGroup(String group) {\n        this.group = group;\n    }\n\n    public Message getMessage() {\n        return message;\n    }\n\n    public void setMessage(Message message) {\n        this.message = message;\n    }\n\n    public MessageQueue getMq() {\n        return mq;\n    }\n\n    public void setMq(MessageQueue mq) {\n        this.mq = mq;\n    }\n\n    public String getBrokerAddr() {\n        return brokerAddr;\n    }\n\n    public void setBrokerAddr(String brokerAddr) {\n        this.brokerAddr = brokerAddr;\n    }\n\n    public CommunicationMode getCommunicationMode() {\n        return communicationMode;\n    }\n\n    public void setCommunicationMode(CommunicationMode communicationMode) {\n        this.communicationMode = communicationMode;\n    }\n\n    public SendResult getSendResult() {\n        return sendResult;\n    }\n\n    public void setSendResult(SendResult sendResult) {\n        this.sendResult = sendResult;\n    }\n\n    public Exception getException() {\n        return exception;\n    }\n\n    public void setException(Exception exception) {\n        this.exception = exception;\n    }\n\n    public Object getArg() {\n        return arg;\n    }\n\n    public void setArg(Object arg) {\n        this.arg = arg;\n    }\n\n    public boolean isUnitMode() {\n        return unitMode;\n    }\n\n    public void setUnitMode(boolean isUnitMode) {\n        this.unitMode = isUnitMode;\n    }\n\n    public String getNameSrvAddr() {\n        return nameSrvAddr;\n    }\n\n    public void setNameSrvAddr(String nameSrvAddr) {\n        this.nameSrvAddr = nameSrvAddr;\n    }\n\n    @Override\n    public String toString() {\n        return \"SendMessageContext [nameSrvAddr=\" + nameSrvAddr + \", group=\" + group + \", message=\" + message\n            + \", mq=\" + mq + \", brokerAddr=\" + brokerAddr + \", communicationMode=\" + communicationMode\n            + \", sendResult=\" + sendResult + \", exception=\" + exception + \", unitMode=\" + unitMode\n            + \", arg=\" + arg + \"]\";\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/hook/CheckForbiddenHook.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.hook;\n\nimport org.apache.rocketmq.client.exception.MQClientException;\n\npublic interface CheckForbiddenHook {\n    String hookName();\n\n    void checkForbidden(final CheckForbiddenContext context) throws MQClientException;\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/hook/ConsumeMessageContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.hook;\n\nimport java.util.List;\nimport java.util.Map;\n\nimport org.apache.rocketmq.client.AccessChannel;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\npublic class ConsumeMessageContext {\n    private String consumerGroup;\n    private List<MessageExt> msgList;\n    private MessageQueue mq;\n    private boolean success;\n    private String status;\n    private Object mqTraceContext;\n    private Map<String, String> props;\n    private String namespace;\n    private AccessChannel accessChannel;\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public List<MessageExt> getMsgList() {\n        return msgList;\n    }\n\n    public void setMsgList(List<MessageExt> msgList) {\n        this.msgList = msgList;\n    }\n\n    public MessageQueue getMq() {\n        return mq;\n    }\n\n    public void setMq(MessageQueue mq) {\n        this.mq = mq;\n    }\n\n    public boolean isSuccess() {\n        return success;\n    }\n\n    public void setSuccess(boolean success) {\n        this.success = success;\n    }\n\n    public Object getMqTraceContext() {\n        return mqTraceContext;\n    }\n\n    public void setMqTraceContext(Object mqTraceContext) {\n        this.mqTraceContext = mqTraceContext;\n    }\n\n    public Map<String, String> getProps() {\n        return props;\n    }\n\n    public void setProps(Map<String, String> props) {\n        this.props = props;\n    }\n\n    public String getStatus() {\n        return status;\n    }\n\n    public void setStatus(String status) {\n        this.status = status;\n    }\n\n    public String getNamespace() {\n        return namespace;\n    }\n\n    public void setNamespace(String namespace) {\n        this.namespace = namespace;\n    }\n\n    public AccessChannel getAccessChannel() {\n        return accessChannel;\n    }\n\n    public void setAccessChannel(AccessChannel accessChannel) {\n        this.accessChannel = accessChannel;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/hook/ConsumeMessageHook.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.hook;\n\npublic interface ConsumeMessageHook {\n    String hookName();\n\n    void consumeMessageBefore(final ConsumeMessageContext context);\n\n    void consumeMessageAfter(final ConsumeMessageContext context);\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/hook/EndTransactionContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.hook;\n\nimport org.apache.rocketmq.client.producer.LocalTransactionState;\nimport org.apache.rocketmq.common.message.Message;\n\npublic class EndTransactionContext {\n    private String producerGroup;\n    private Message message;\n    private String brokerAddr;\n    private String msgId;\n    private String transactionId;\n    private LocalTransactionState transactionState;\n    private boolean fromTransactionCheck;\n\n    public String getProducerGroup() {\n        return producerGroup;\n    }\n\n    public void setProducerGroup(String producerGroup) {\n        this.producerGroup = producerGroup;\n    }\n\n    public Message getMessage() {\n        return message;\n    }\n\n    public void setMessage(Message message) {\n        this.message = message;\n    }\n\n    public String getBrokerAddr() {\n        return brokerAddr;\n    }\n\n    public void setBrokerAddr(String brokerAddr) {\n        this.brokerAddr = brokerAddr;\n    }\n\n    public String getMsgId() {\n        return msgId;\n    }\n\n    public void setMsgId(String msgId) {\n        this.msgId = msgId;\n    }\n\n    public String getTransactionId() {\n        return transactionId;\n    }\n\n    public void setTransactionId(String transactionId) {\n        this.transactionId = transactionId;\n    }\n\n    public LocalTransactionState getTransactionState() {\n        return transactionState;\n    }\n\n    public void setTransactionState(LocalTransactionState transactionState) {\n        this.transactionState = transactionState;\n    }\n\n    public boolean isFromTransactionCheck() {\n        return fromTransactionCheck;\n    }\n\n    public void setFromTransactionCheck(boolean fromTransactionCheck) {\n        this.fromTransactionCheck = fromTransactionCheck;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/hook/EndTransactionHook.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.hook;\n\npublic interface EndTransactionHook {\n    String hookName();\n\n    void endTransaction(final EndTransactionContext context);\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/hook/FilterMessageContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.hook;\n\nimport java.util.List;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\npublic class FilterMessageContext {\n    private String consumerGroup;\n    private List<MessageExt> msgList;\n    private MessageQueue mq;\n    private Object arg;\n    private boolean unitMode;\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public List<MessageExt> getMsgList() {\n        return msgList;\n    }\n\n    public void setMsgList(List<MessageExt> msgList) {\n        this.msgList = msgList;\n    }\n\n    public MessageQueue getMq() {\n        return mq;\n    }\n\n    public void setMq(MessageQueue mq) {\n        this.mq = mq;\n    }\n\n    public Object getArg() {\n        return arg;\n    }\n\n    public void setArg(Object arg) {\n        this.arg = arg;\n    }\n\n    public boolean isUnitMode() {\n        return unitMode;\n    }\n\n    public void setUnitMode(boolean isUnitMode) {\n        this.unitMode = isUnitMode;\n    }\n\n    @Override\n    public String toString() {\n        return \"ConsumeMessageContext [consumerGroup=\" + consumerGroup + \", msgList=\" + msgList + \", mq=\"\n            + mq + \", arg=\" + arg + \"]\";\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/hook/FilterMessageHook.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.hook;\n\npublic interface FilterMessageHook {\n    String hookName();\n\n    void filterMessage(final FilterMessageContext context);\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/hook/SendMessageContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.hook;\n\nimport java.util.Map;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.message.MessageType;\n\npublic class SendMessageContext {\n    private String producerGroup;\n    private Message message;\n    private MessageQueue mq;\n    private String brokerAddr;\n    private String bornHost;\n    private CommunicationMode communicationMode;\n    private SendResult sendResult;\n    private Exception exception;\n    private Object mqTraceContext;\n    private Map<String, String> props;\n    private DefaultMQProducerImpl producer;\n    private MessageType msgType = MessageType.Normal_Msg;\n    private String namespace;\n\n    public MessageType getMsgType() {\n        return msgType;\n    }\n\n    public void setMsgType(final MessageType msgType) {\n        this.msgType = msgType;\n    }\n\n    public DefaultMQProducerImpl getProducer() {\n        return producer;\n    }\n\n    public void setProducer(final DefaultMQProducerImpl producer) {\n        this.producer = producer;\n    }\n\n    public String getProducerGroup() {\n        return producerGroup;\n    }\n\n    public void setProducerGroup(String producerGroup) {\n        this.producerGroup = producerGroup;\n    }\n\n    public Message getMessage() {\n        return message;\n    }\n\n    public void setMessage(Message message) {\n        this.message = message;\n    }\n\n    public MessageQueue getMq() {\n        return mq;\n    }\n\n    public void setMq(MessageQueue mq) {\n        this.mq = mq;\n    }\n\n    public String getBrokerAddr() {\n        return brokerAddr;\n    }\n\n    public void setBrokerAddr(String brokerAddr) {\n        this.brokerAddr = brokerAddr;\n    }\n\n    public CommunicationMode getCommunicationMode() {\n        return communicationMode;\n    }\n\n    public void setCommunicationMode(CommunicationMode communicationMode) {\n        this.communicationMode = communicationMode;\n    }\n\n    public SendResult getSendResult() {\n        return sendResult;\n    }\n\n    public void setSendResult(SendResult sendResult) {\n        this.sendResult = sendResult;\n    }\n\n    public Exception getException() {\n        return exception;\n    }\n\n    public void setException(Exception exception) {\n        this.exception = exception;\n    }\n\n    public Object getMqTraceContext() {\n        return mqTraceContext;\n    }\n\n    public void setMqTraceContext(Object mqTraceContext) {\n        this.mqTraceContext = mqTraceContext;\n    }\n\n    public Map<String, String> getProps() {\n        return props;\n    }\n\n    public void setProps(Map<String, String> props) {\n        this.props = props;\n    }\n\n    public String getBornHost() {\n        return bornHost;\n    }\n\n    public void setBornHost(String bornHost) {\n        this.bornHost = bornHost;\n    }\n\n    public String getNamespace() {\n        return namespace;\n    }\n\n    public void setNamespace(String namespace) {\n        this.namespace = namespace;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/hook/SendMessageHook.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.hook;\n\npublic interface SendMessageHook {\n    String hookName();\n\n    void sendMessageBefore(final SendMessageContext context);\n\n    void sendMessageAfter(final SendMessageContext context);\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport java.util.HashMap;\nimport java.util.Map;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.client.impl.producer.MQProducerInner;\nimport org.apache.rocketmq.client.producer.RequestFutureHolder;\nimport org.apache.rocketmq.client.producer.RequestResponseFuture;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.compression.Compressor;\nimport org.apache.rocketmq.common.compression.CompressorFactory;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.body.GetConsumerStatusBody;\nimport org.apache.rocketmq.remoting.protocol.body.ResetOffsetBody;\nimport org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerStatusRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.NotifyConsumerIdsChangedRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ReplyMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class ClientRemotingProcessor implements NettyRequestProcessor {\n    private final Logger logger = LoggerFactory.getLogger(ClientRemotingProcessor.class);\n    private final MQClientInstance mqClientFactory;\n\n    public ClientRemotingProcessor(final MQClientInstance mqClientFactory) {\n        this.mqClientFactory = mqClientFactory;\n    }\n\n    @Override\n    public RemotingCommand processRequest(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        switch (request.getCode()) {\n            case RequestCode.CHECK_TRANSACTION_STATE:\n                return this.checkTransactionState(ctx, request);\n            case RequestCode.NOTIFY_CONSUMER_IDS_CHANGED:\n                return this.notifyConsumerIdsChanged(ctx, request);\n            case RequestCode.RESET_CONSUMER_CLIENT_OFFSET:\n                return this.resetOffset(ctx, request);\n            case RequestCode.GET_CONSUMER_STATUS_FROM_CLIENT:\n                return this.getConsumeStatus(ctx, request);\n\n            case RequestCode.GET_CONSUMER_RUNNING_INFO:\n                return this.getConsumerRunningInfo(ctx, request);\n\n            case RequestCode.CONSUME_MESSAGE_DIRECTLY:\n                return this.consumeMessageDirectly(ctx, request);\n\n            case RequestCode.PUSH_REPLY_MESSAGE_TO_CLIENT:\n                return this.receiveReplyMessage(ctx, request);\n            default:\n                break;\n        }\n        return null;\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n\n    public RemotingCommand checkTransactionState(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final CheckTransactionStateRequestHeader requestHeader =\n            (CheckTransactionStateRequestHeader) request.decodeCommandCustomHeader(CheckTransactionStateRequestHeader.class);\n        final ByteBuffer byteBuffer = ByteBuffer.wrap(request.getBody());\n        final MessageExt messageExt = MessageDecoder.decode(byteBuffer);\n        if (messageExt != null) {\n            if (StringUtils.isNotEmpty(this.mqClientFactory.getClientConfig().getNamespace())) {\n                messageExt.setTopic(NamespaceUtil\n                    .withoutNamespace(messageExt.getTopic(), this.mqClientFactory.getClientConfig().getNamespace()));\n            }\n            String transactionId = messageExt.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX);\n            if (null != transactionId && !\"\".equals(transactionId)) {\n                messageExt.setTransactionId(transactionId);\n            }\n            final String group = messageExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP);\n            if (group != null) {\n                MQProducerInner producer = this.mqClientFactory.selectProducer(group);\n                if (producer != null) {\n                    final String addr = RemotingHelper.parseChannelRemoteAddr(ctx.channel());\n                    producer.checkTransactionState(addr, messageExt, requestHeader);\n                } else {\n                    logger.debug(\"checkTransactionState, pick producer by group[{}] failed\", group);\n                }\n            } else {\n                logger.warn(\"checkTransactionState, pick producer group failed\");\n            }\n        } else {\n            logger.warn(\"checkTransactionState, decode message failed\");\n        }\n\n        return null;\n    }\n\n    public RemotingCommand notifyConsumerIdsChanged(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        try {\n            final NotifyConsumerIdsChangedRequestHeader requestHeader =\n                (NotifyConsumerIdsChangedRequestHeader) request.decodeCommandCustomHeader(NotifyConsumerIdsChangedRequestHeader.class);\n            logger.info(\"receive broker's notification[{}], the consumer group: {} changed, rebalance immediately\",\n                RemotingHelper.parseChannelRemoteAddr(ctx.channel()),\n                requestHeader.getConsumerGroup());\n            this.mqClientFactory.rebalanceImmediately();\n        } catch (Exception e) {\n            logger.error(\"notifyConsumerIdsChanged exception\", UtilAll.exceptionSimpleDesc(e));\n        }\n        return null;\n    }\n\n    public RemotingCommand resetOffset(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final ResetOffsetRequestHeader requestHeader =\n            (ResetOffsetRequestHeader) request.decodeCommandCustomHeader(ResetOffsetRequestHeader.class);\n        logger.info(\"invoke reset offset operation from broker. brokerAddr={}, topic={}, group={}, timestamp={}\",\n            RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.getTopic(), requestHeader.getGroup(),\n            requestHeader.getTimestamp());\n        Map<MessageQueue, Long> offsetTable = new HashMap<>();\n        if (request.getBody() != null) {\n            ResetOffsetBody body = ResetOffsetBody.decode(request.getBody(), ResetOffsetBody.class);\n            offsetTable = body.getOffsetTable();\n        }\n        this.mqClientFactory.resetOffset(requestHeader.getTopic(), requestHeader.getGroup(), offsetTable);\n        return null;\n    }\n\n    @Deprecated\n    public RemotingCommand getConsumeStatus(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final GetConsumerStatusRequestHeader requestHeader =\n            (GetConsumerStatusRequestHeader) request.decodeCommandCustomHeader(GetConsumerStatusRequestHeader.class);\n\n        Map<MessageQueue, Long> offsetTable = this.mqClientFactory.getConsumerStatus(requestHeader.getTopic(), requestHeader.getGroup());\n        GetConsumerStatusBody body = new GetConsumerStatusBody();\n        body.setMessageQueueTable(offsetTable);\n        response.setBody(body.encode());\n        response.setCode(ResponseCode.SUCCESS);\n        return response;\n    }\n\n    private RemotingCommand getConsumerRunningInfo(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final GetConsumerRunningInfoRequestHeader requestHeader =\n            (GetConsumerRunningInfoRequestHeader) request.decodeCommandCustomHeader(GetConsumerRunningInfoRequestHeader.class);\n\n        ConsumerRunningInfo consumerRunningInfo = this.mqClientFactory.consumerRunningInfo(requestHeader.getConsumerGroup());\n        if (null != consumerRunningInfo) {\n            if (requestHeader.isJstackEnable()) {\n                Map<Thread, StackTraceElement[]> map = Thread.getAllStackTraces();\n                String jstack = UtilAll.jstack(map);\n                consumerRunningInfo.setJstack(jstack);\n            }\n\n            response.setCode(ResponseCode.SUCCESS);\n            response.setBody(consumerRunningInfo.encode());\n        } else {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(String.format(\"The Consumer Group <%s> not exist in this consumer\", requestHeader.getConsumerGroup()));\n        }\n\n        return response;\n    }\n\n    private RemotingCommand consumeMessageDirectly(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final ConsumeMessageDirectlyResultRequestHeader requestHeader =\n            (ConsumeMessageDirectlyResultRequestHeader) request\n                .decodeCommandCustomHeader(ConsumeMessageDirectlyResultRequestHeader.class);\n\n        final MessageExt msg = MessageDecoder.clientDecode(ByteBuffer.wrap(request.getBody()), true);\n\n        ConsumeMessageDirectlyResult result =\n            this.mqClientFactory.consumeMessageDirectly(msg, requestHeader.getConsumerGroup(), requestHeader.getBrokerName());\n\n        if (null != result) {\n            response.setCode(ResponseCode.SUCCESS);\n            response.setBody(result.encode());\n        } else {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(String.format(\"The Consumer Group <%s> not exist in this consumer\", requestHeader.getConsumerGroup()));\n        }\n\n        return response;\n    }\n\n    private RemotingCommand receiveReplyMessage(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        long receiveTime = System.currentTimeMillis();\n        ReplyMessageRequestHeader requestHeader = (ReplyMessageRequestHeader) request.decodeCommandCustomHeader(ReplyMessageRequestHeader.class);\n\n        try {\n            MessageExt msg = new MessageExt();\n            msg.setTopic(requestHeader.getTopic());\n            msg.setQueueId(requestHeader.getQueueId());\n            msg.setStoreTimestamp(requestHeader.getStoreTimestamp());\n\n            if (requestHeader.getBornHost() != null) {\n                msg.setBornHost(NetworkUtil.string2SocketAddress(requestHeader.getBornHost()));\n            }\n\n            if (requestHeader.getStoreHost() != null) {\n                msg.setStoreHost(NetworkUtil.string2SocketAddress(requestHeader.getStoreHost()));\n            }\n\n            byte[] body = request.getBody();\n            int sysFlag = requestHeader.getSysFlag();\n            if ((sysFlag & MessageSysFlag.COMPRESSED_FLAG) == MessageSysFlag.COMPRESSED_FLAG) {\n                try {\n                    Compressor compressor = CompressorFactory.getCompressor(MessageSysFlag.getCompressionType(sysFlag));\n                    body = compressor.decompress(body);\n                } catch (IOException e) {\n                    logger.warn(\"err when uncompress constant\", e);\n                }\n            }\n            msg.setBody(body);\n            msg.setFlag(requestHeader.getFlag());\n            MessageAccessor.setProperties(msg, MessageDecoder.string2messageProperties(requestHeader.getProperties()));\n            MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REPLY_MESSAGE_ARRIVE_TIME, String.valueOf(receiveTime));\n            msg.setBornTimestamp(requestHeader.getBornTimestamp());\n            msg.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes());\n            logger.debug(\"receive reply message :{}\", msg);\n\n            processReplyMessage(msg);\n\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n        } catch (Exception e) {\n            logger.warn(\"unknown err when receiveReplyMsg\", e);\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"process reply message fail\");\n        }\n        return response;\n    }\n\n    private void processReplyMessage(MessageExt replyMsg) {\n        final String correlationId = replyMsg.getUserProperty(MessageConst.PROPERTY_CORRELATION_ID);\n        final RequestResponseFuture requestResponseFuture = RequestFutureHolder.getInstance().getRequestFutureTable().get(correlationId);\n        if (requestResponseFuture != null) {\n            requestResponseFuture.putResponseMessage(replyMsg);\n\n            RequestFutureHolder.getInstance().getRequestFutureTable().remove(correlationId);\n\n            if (requestResponseFuture.getRequestCallback() != null) {\n                requestResponseFuture.getRequestCallback().onSuccess(replyMsg);\n            }\n        } else {\n            String bornHost = replyMsg.getBornHostString();\n            logger.warn(\"receive reply message, but not matched any request, CorrelationId: {} , reply from host: {}\",\n                correlationId, bornHost);\n        }\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/CommunicationMode.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl;\n\npublic enum CommunicationMode {\n    SYNC,\n    ASYNC,\n    ONEWAY,\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/FindBrokerResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl;\n\npublic class FindBrokerResult {\n    private final String brokerAddr;\n    private final boolean slave;\n    private final int brokerVersion;\n\n    public FindBrokerResult(String brokerAddr, boolean slave) {\n        this.brokerAddr = brokerAddr;\n        this.slave = slave;\n        this.brokerVersion = 0;\n    }\n\n    public FindBrokerResult(String brokerAddr, boolean slave, int brokerVersion) {\n        this.brokerAddr = brokerAddr;\n        this.slave = slave;\n        this.brokerVersion = brokerVersion;\n    }\n\n    public String getBrokerAddr() {\n        return brokerAddr;\n    }\n\n    public boolean isSlave() {\n        return slave;\n    }\n\n    public int getBrokerVersion() {\n        return brokerVersion;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl;\n\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Set;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.locks.ReadWriteLock;\nimport java.util.concurrent.locks.ReentrantReadWriteLock;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.QueryResult;\nimport org.apache.rocketmq.client.Validators;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.client.impl.producer.TopicPublishInfo;\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.help.FAQUrl;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageId;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.InvokeCallback;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.netty.ResponseFuture;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\n\npublic class MQAdminImpl {\n\n    private static final Logger log = LoggerFactory.getLogger(MQAdminImpl.class);\n    private final MQClientInstance mQClientFactory;\n    private long timeoutMillis = 6000;\n\n    public MQAdminImpl(MQClientInstance mQClientFactory) {\n        this.mQClientFactory = mQClientFactory;\n    }\n\n    public long getTimeoutMillis() {\n        return timeoutMillis;\n    }\n\n    public void setTimeoutMillis(long timeoutMillis) {\n        this.timeoutMillis = timeoutMillis;\n    }\n\n    public void createTopic(String key, String newTopic, int queueNum) throws MQClientException {\n        createTopic(key, newTopic, queueNum, 0, null);\n    }\n\n    public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag,\n        Map<String, String> attributes) throws MQClientException {\n        try {\n            Validators.checkTopic(newTopic);\n            Validators.isSystemTopic(newTopic);\n            TopicRouteData topicRouteData = this.mQClientFactory.getMQClientAPIImpl().getTopicRouteInfoFromNameServer(key, timeoutMillis);\n            List<BrokerData> brokerDataList = topicRouteData.getBrokerDatas();\n            if (brokerDataList != null && !brokerDataList.isEmpty()) {\n                Collections.sort(brokerDataList);\n\n                boolean createOKAtLeastOnce = false;\n                MQClientException exception = null;\n\n                StringBuilder orderTopicString = new StringBuilder();\n\n                for (BrokerData brokerData : brokerDataList) {\n                    String addr = brokerData.getBrokerAddrs().get(MixAll.MASTER_ID);\n                    if (addr != null) {\n                        TopicConfig topicConfig = new TopicConfig(newTopic);\n                        topicConfig.setReadQueueNums(queueNum);\n                        topicConfig.setWriteQueueNums(queueNum);\n                        topicConfig.setTopicSysFlag(topicSysFlag);\n                        topicConfig.setAttributes(attributes);\n\n                        boolean createOK = false;\n                        for (int i = 0; i < 5; i++) {\n                            try {\n                                this.mQClientFactory.getMQClientAPIImpl().createTopic(addr, key, topicConfig, timeoutMillis);\n                                createOK = true;\n                                createOKAtLeastOnce = true;\n                                break;\n                            } catch (Exception e) {\n                                if (4 == i) {\n                                    exception = new MQClientException(\"create topic to broker exception\", e);\n                                }\n                            }\n                        }\n\n                        if (createOK) {\n                            orderTopicString.append(brokerData.getBrokerName());\n                            orderTopicString.append(\":\");\n                            orderTopicString.append(queueNum);\n                            orderTopicString.append(\";\");\n                        }\n                    }\n                }\n\n                if (exception != null && !createOKAtLeastOnce) {\n                    throw exception;\n                }\n            } else {\n                throw new MQClientException(\"Not found broker, maybe key is wrong\", null);\n            }\n        } catch (Exception e) {\n            throw new MQClientException(\"create new topic failed\", e);\n        }\n    }\n\n    public List<MessageQueue> fetchPublishMessageQueues(String topic) throws MQClientException {\n        try {\n            TopicRouteData topicRouteData = this.mQClientFactory.getMQClientAPIImpl().getTopicRouteInfoFromNameServer(topic, timeoutMillis);\n            if (topicRouteData != null) {\n                TopicPublishInfo topicPublishInfo = MQClientInstance.topicRouteData2TopicPublishInfo(topic, topicRouteData);\n                if (topicPublishInfo != null && topicPublishInfo.ok()) {\n                    return parsePublishMessageQueues(topicPublishInfo.getMessageQueueList());\n                }\n            }\n        } catch (Exception e) {\n            throw new MQClientException(\"Can not find Message Queue for this topic, \" + topic, e);\n        }\n\n        throw new MQClientException(\"Unknown why, Can not find Message Queue for this topic, \" + topic, null);\n    }\n\n    public List<MessageQueue> parsePublishMessageQueues(List<MessageQueue> messageQueueList) {\n        List<MessageQueue> resultQueues = new ArrayList<>();\n        for (MessageQueue queue : messageQueueList) {\n            String userTopic = NamespaceUtil.withoutNamespace(queue.getTopic(), this.mQClientFactory.getClientConfig().getNamespace());\n            resultQueues.add(new MessageQueue(userTopic, queue.getBrokerName(), queue.getQueueId()));\n        }\n\n        return resultQueues;\n    }\n\n    public Set<MessageQueue> fetchSubscribeMessageQueues(String topic) throws MQClientException {\n        try {\n            TopicRouteData topicRouteData = this.mQClientFactory.getMQClientAPIImpl().getTopicRouteInfoFromNameServer(topic, timeoutMillis);\n            if (topicRouteData != null) {\n                Set<MessageQueue> mqList = MQClientInstance.topicRouteData2TopicSubscribeInfo(topic, topicRouteData);\n                if (!mqList.isEmpty()) {\n                    return mqList;\n                } else {\n                    throw new MQClientException(\"Can not find Message Queue for this topic, \" + topic + \" Namesrv return empty\", null);\n                }\n            }\n        } catch (Exception e) {\n            throw new MQClientException(\n                \"Can not find Message Queue for this topic, \" + topic + FAQUrl.suggestTodo(FAQUrl.MQLIST_NOT_EXIST),\n                e);\n        }\n\n        throw new MQClientException(\"Unknown why, Can not find Message Queue for this topic, \" + topic, null);\n    }\n\n    public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException {\n        // default return lower boundary offset when there are more than one offsets.\n        return searchOffset(mq, timestamp, BoundaryType.LOWER);\n    }\n\n    public long searchOffset(MessageQueue mq, long timestamp, BoundaryType boundaryType) throws MQClientException {\n        String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq));\n        if (null == brokerAddr) {\n            this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());\n            brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq));\n        }\n\n        if (brokerAddr != null) {\n            try {\n                return this.mQClientFactory.getMQClientAPIImpl().searchOffset(brokerAddr, mq, timestamp,\n                    boundaryType, timeoutMillis);\n            } catch (Exception e) {\n                throw new MQClientException(\"Invoke Broker[\" + brokerAddr + \"] exception\", e);\n            }\n        }\n\n        throw new MQClientException(\"The broker[\" + mq.getBrokerName() + \"] not exist\", null);\n    }\n\n    public long maxOffset(MessageQueue mq) throws MQClientException {\n        String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq));\n        if (null == brokerAddr) {\n            this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());\n            brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq));\n        }\n\n        if (brokerAddr != null) {\n            try {\n                return this.mQClientFactory.getMQClientAPIImpl().getMaxOffset(brokerAddr, mq, timeoutMillis);\n            } catch (Exception e) {\n                throw new MQClientException(\"Invoke Broker[\" + brokerAddr + \"] exception\", e);\n            }\n        }\n\n        throw new MQClientException(\"The broker[\" + mq.getBrokerName() + \"] not exist\", null);\n    }\n\n    public long minOffset(MessageQueue mq) throws MQClientException {\n        String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq));\n        if (null == brokerAddr) {\n            this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());\n            brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq));\n        }\n\n        if (brokerAddr != null) {\n            try {\n                return this.mQClientFactory.getMQClientAPIImpl().getMinOffset(brokerAddr, mq, timeoutMillis);\n            } catch (Exception e) {\n                throw new MQClientException(\"Invoke Broker[\" + brokerAddr + \"] exception\", e);\n            }\n        }\n\n        throw new MQClientException(\"The broker[\" + mq.getBrokerName() + \"] not exist\", null);\n    }\n\n    public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException {\n        String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq));\n        if (null == brokerAddr) {\n            this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());\n            brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq));\n        }\n\n        if (brokerAddr != null) {\n            try {\n                return this.mQClientFactory.getMQClientAPIImpl().getEarliestMsgStoretime(brokerAddr, mq, timeoutMillis);\n            } catch (Exception e) {\n                throw new MQClientException(\"Invoke Broker[\" + brokerAddr + \"] exception\", e);\n            }\n        }\n\n        throw new MQClientException(\"The broker[\" + mq.getBrokerName() + \"] not exist\", null);\n    }\n\n    public MessageExt viewMessage(String topic, String msgId)\n        throws RemotingException, MQBrokerException, InterruptedException, MQClientException {\n        MessageId messageId;\n        try {\n            messageId = MessageDecoder.decodeMessageId(msgId);\n        } catch (Exception e) {\n            throw new MQClientException(ResponseCode.NO_MESSAGE, \"query message by id finished, but no message.\");\n        }\n        return this.mQClientFactory.getMQClientAPIImpl().viewMessage(NetworkUtil.socketAddress2String(messageId.getAddress()),\n            topic, messageId.getOffset(), timeoutMillis);\n    }\n\n    public QueryResult queryMessage(String topic, String key, int maxNum, long begin,\n        long end) throws MQClientException,\n        InterruptedException {\n        return queryMessage(null, topic, key, maxNum, begin, end, false, MessageConst.INDEX_KEY_TYPE, null);\n    }\n\n    public QueryResult queryMessageByUniqKey(String topic, String uniqKey, int maxNum, long begin, long end)\n        throws MQClientException, InterruptedException {\n        return queryMessage(null, topic, uniqKey, maxNum, begin, end, true, MessageConst.INDEX_UNIQUE_TYPE, null);\n    }\n\n    public QueryResult queryMessageByUniqKey(String clusterName, String topic, String uniqKey, int maxNum, long begin,\n        long end)\n        throws MQClientException, InterruptedException {\n        return queryMessage(clusterName, topic, uniqKey, maxNum, begin, end, true, MessageConst.INDEX_UNIQUE_TYPE, null);\n    }\n\n    public MessageExt queryMessageByUniqKey(String topic,\n        String uniqKey) throws InterruptedException, MQClientException {\n        return queryMessageByUniqKey(topic, uniqKey, System.currentTimeMillis() - 3L * 24 * 60L * 60L * 1000L, Long.MAX_VALUE);\n    }\n\n    public MessageExt queryMessageByUniqKey(String clusterName, String topic,\n        String uniqKey) throws InterruptedException, MQClientException {\n        return queryMessageByUniqKey(clusterName, topic, uniqKey, System.currentTimeMillis() - 3L * 24 * 60L * 60L * 1000L, Long.MAX_VALUE);\n    }\n\n    public MessageExt queryMessageByUniqKey(String topic,\n        String uniqKey, long begin, long end) throws InterruptedException, MQClientException {\n        return queryMessageByUniqKey(null, topic, uniqKey, begin, end);\n    }\n\n    public MessageExt queryMessageByUniqKey(String clusterName, String topic,\n        String uniqKey, long begin, long end) throws InterruptedException, MQClientException {\n        QueryResult qr = this.queryMessage(clusterName, topic, uniqKey, 32, begin, end, true, MessageConst.INDEX_UNIQUE_TYPE, null);\n        if (qr != null && qr.getMessageList() != null && qr.getMessageList().size() > 0) {\n            return qr.getMessageList().get(0);\n        } else {\n            return null;\n        }\n    }\n\n    public QueryResult queryMessage(String clusterName, String topic, String key, int maxNum, long begin, long end, boolean isUniqKey) throws MQClientException,\n        InterruptedException {\n        return queryMessage(clusterName, topic, key, maxNum, begin, end, isUniqKey, null, null);\n    }\n\n    public QueryResult queryMessage(String clusterName, String topic, String key, int maxNum, long begin, long end, boolean isUniqKey, String indexType, String lastKey) throws MQClientException,\n        InterruptedException {\n        boolean isLmq = MixAll.isLmq(topic);\n\n        String routeTopic = topic;\n        // if topic is lmq ,then use clusterName as lmq parent topic\n        // Use clusterName or lmq parent topic to get topic route for lmq or rmq_sys_wheel_timer\n        if (!StringUtils.isEmpty(topic) && (isLmq || topic.equals(TopicValidator.SYSTEM_TOPIC_PREFIX + \"wheel_timer\"))\n            && !StringUtils.isEmpty(clusterName)) {\n            routeTopic = clusterName;\n        }\n\n        TopicRouteData topicRouteData = this.mQClientFactory.getAnExistTopicRouteData(routeTopic);\n        if (null == topicRouteData) {\n            this.mQClientFactory.updateTopicRouteInfoFromNameServer(routeTopic);\n            topicRouteData = this.mQClientFactory.getAnExistTopicRouteData(routeTopic);\n        }\n\n        if (topicRouteData != null) {\n            List<String> brokerAddrs = new LinkedList<>();\n            for (BrokerData brokerData : topicRouteData.getBrokerDatas()) {\n                if (!isLmq && clusterName != null && !clusterName.isEmpty()\n                    && !clusterName.equals(brokerData.getCluster())) {\n                    continue;\n                }\n                String addr = brokerData.selectBrokerAddr();\n                if (addr != null) {\n                    brokerAddrs.add(addr);\n                }\n            }\n\n            if (!brokerAddrs.isEmpty()) {\n                final CountDownLatch countDownLatch = new CountDownLatch(brokerAddrs.size());\n                final List<QueryResult> queryResultList = new LinkedList<>();\n                final ReadWriteLock lock = new ReentrantReadWriteLock(false);\n\n                for (String addr : brokerAddrs) {\n                    try {\n                        QueryMessageRequestHeader requestHeader = new QueryMessageRequestHeader();\n                        if (isLmq) {\n                            requestHeader.setTopic(clusterName);\n                        } else {\n                            requestHeader.setTopic(topic);\n                        }\n                        requestHeader.setKey(key);\n                        requestHeader.setMaxNum(maxNum);\n                        requestHeader.setBeginTimestamp(begin);\n                        requestHeader.setEndTimestamp(end);\n                        requestHeader.setIndexType(indexType);\n                        requestHeader.setLastKey(lastKey);\n\n                        this.mQClientFactory.getMQClientAPIImpl().queryMessage(addr, requestHeader, timeoutMillis * 3,\n                            new InvokeCallback() {\n                                @Override\n                                public void operationComplete(ResponseFuture responseFuture) {\n\n                                }\n\n                                @Override\n                                public void operationSucceed(RemotingCommand response) {\n                                    try {\n                                        switch (response.getCode()) {\n                                            case ResponseCode.SUCCESS: {\n                                                QueryMessageResponseHeader responseHeader = null;\n                                                try {\n                                                    responseHeader =\n                                                        (QueryMessageResponseHeader) response\n                                                            .decodeCommandCustomHeader(QueryMessageResponseHeader.class);\n                                                } catch (RemotingCommandException e) {\n                                                    log.error(\"decodeCommandCustomHeader exception\", e);\n                                                    return;\n                                                }\n\n                                                List<MessageExt> wrappers =\n                                                    MessageDecoder.decodes(ByteBuffer.wrap(response.getBody()), true);\n\n                                                QueryResult qr = new QueryResult(responseHeader.getIndexLastUpdateTimestamp(), wrappers);\n                                                try {\n                                                    lock.writeLock().lock();\n                                                    queryResultList.add(qr);\n                                                } finally {\n                                                    lock.writeLock().unlock();\n                                                }\n                                                break;\n                                            }\n                                            default:\n                                                log.warn(\"getResponseCommand failed, {} {}\", response.getCode(), response.getRemark());\n                                                break;\n                                        }\n\n                                    } finally {\n                                        countDownLatch.countDown();\n                                    }\n                                }\n\n                                @Override\n                                public void operationFail(Throwable throwable) {\n                                    log.error(\"queryMessage error, requestHeader={}\", requestHeader);\n                                    countDownLatch.countDown();\n                                }\n                            }, isUniqKey);\n                    } catch (Exception e) {\n                        log.warn(\"queryMessage exception\", e);\n                    }\n\n                }\n\n                boolean ok = countDownLatch.await(timeoutMillis * 4, TimeUnit.MILLISECONDS);\n                if (!ok) {\n                    log.warn(\"queryMessage, maybe some broker failed\");\n                }\n\n                long indexLastUpdateTimestamp = 0;\n                List<MessageExt> messageList = new LinkedList<>();\n                for (QueryResult qr : queryResultList) {\n                    if (qr.getIndexLastUpdateTimestamp() > indexLastUpdateTimestamp) {\n                        indexLastUpdateTimestamp = qr.getIndexLastUpdateTimestamp();\n                    }\n\n                    for (MessageExt msgExt : qr.getMessageList()) {\n                        if (isUniqKey) {\n                            if (msgExt.getMsgId().equals(key)) {\n                                messageList.add(msgExt);\n                            } else {\n                                log.warn(\"queryMessage by uniqKey, find message key not matched, maybe hash duplicate {}\", msgExt.toString());\n                            }\n                        }  else if (!StringUtils.isEmpty(indexType) && MessageConst.INDEX_KEY_TYPE.equals(indexType))  {\n                            String keys = msgExt.getKeys();\n                            String msgTopic = msgExt.getTopic();\n                            if (keys != null) {\n                                boolean matched = false;\n                                String[] keyArray = keys.split(MessageConst.KEY_SEPARATOR);\n                                for (String k : keyArray) {\n                                    // both topic and key must be equal at the same time\n                                    if (Objects.equals(key, k) && (isLmq || Objects.equals(topic, msgTopic))) {\n                                        matched = true;\n                                        break;\n                                    }\n                                }\n\n                                if (matched) {\n                                    messageList.add(msgExt);\n                                } else {\n                                    log.warn(\"queryMessage, find message key not matched, maybe hash duplicate {}\", msgExt.toString());\n                                }\n                            }\n                        } else if (!StringUtils.isEmpty(indexType) && MessageConst.INDEX_TAG_TYPE.equals(indexType)) {\n                            String tags = msgExt.getTags();\n                            String msgTopic = msgExt.getTopic();\n                            boolean matched = false;\n                            if (tags != null) {\n                                if (Objects.equals(key, tags) && (isLmq || Objects.equals(topic, msgTopic))) {\n                                    matched = true;\n                                }\n                            }\n                            if (matched) {\n                                messageList.add(msgExt);\n                            } else {\n                                log.warn(\"queryMessage, find message key not matched, maybe hash duplicate {}\", msgExt.toString());\n                            }\n                        }\n                    }\n                }\n\n                //If namespace not null , reset Topic without namespace.\n                if (null != this.mQClientFactory.getClientConfig().getNamespace()) {\n                    for (MessageExt messageExt : messageList) {\n                        messageExt.setTopic(NamespaceUtil.withoutNamespace(messageExt.getTopic(), this.mQClientFactory.getClientConfig().getNamespace()));\n                    }\n                }\n\n                if (!messageList.isEmpty()) {\n                    return new QueryResult(indexLastUpdateTimestamp, messageList);\n                } else {\n                    throw new MQClientException(ResponseCode.NO_MESSAGE, \"query message by key finished, but no message.\");\n                }\n            }\n        }\n\n        throw new MQClientException(ResponseCode.TOPIC_NOT_EXIST, \"The topic[\" + topic + \"] not matched route info\");\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl;\n\nimport com.alibaba.fastjson2.JSON;\nimport java.io.UnsupportedEncodingException;\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.BitSet;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Optional;\nimport java.util.Properties;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.commons.lang3.ArrayUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.Validators;\nimport org.apache.rocketmq.client.consumer.AckCallback;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.AckStatus;\nimport org.apache.rocketmq.client.consumer.PopCallback;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PopStatus;\nimport org.apache.rocketmq.client.consumer.PullCallback;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.exception.OffsetNotFoundException;\nimport org.apache.rocketmq.client.hook.SendMessageContext;\nimport org.apache.rocketmq.client.impl.consumer.PullResultExt;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;\nimport org.apache.rocketmq.client.impl.producer.TopicPublishInfo;\nimport org.apache.rocketmq.client.producer.SendCallback;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.client.rpchook.NamespaceRpcHook;\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.CheckRocksdbCqWriteResult;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ObjectCreator;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.attribute.AttributeParser;\nimport org.apache.rocketmq.common.constant.FIleReadaheadMode;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageBatch;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.message.MessageQueueAssignment;\nimport org.apache.rocketmq.common.message.MessageRequestMode;\nimport org.apache.rocketmq.common.namesrv.DefaultTopAddressing;\nimport org.apache.rocketmq.common.namesrv.NameServerUpdateCallback;\nimport org.apache.rocketmq.common.namesrv.TopAddressing;\nimport org.apache.rocketmq.common.sysflag.PullSysFlag;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.common.utils.StartAndShutdown;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.ChannelEventListener;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.InvokeCallback;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.RemotingClient;\nimport org.apache.rocketmq.remoting.common.HeartbeatV2Result;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.exception.RemotingConnectException;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingClient;\nimport org.apache.rocketmq.remoting.netty.ResponseFuture;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.admin.ConsumeStats;\nimport org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;\nimport org.apache.rocketmq.remoting.protocol.body.AclInfo;\nimport org.apache.rocketmq.remoting.protocol.body.BatchAck;\nimport org.apache.rocketmq.remoting.protocol.body.BatchAckMessageRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;\nimport org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo;\nimport org.apache.rocketmq.remoting.protocol.body.BrokerStatsData;\nimport org.apache.rocketmq.remoting.protocol.body.CheckClientRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.ClusterInfo;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerConnection;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.body.CreateTopicListRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.EpochEntryCache;\nimport org.apache.rocketmq.remoting.protocol.body.GetBrokerLiteInfoResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.GetConsumerStatusBody;\nimport org.apache.rocketmq.remoting.protocol.body.GetLiteClientInfoResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.GetLiteGroupInfoResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.GetLiteTopicInfoResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.GetParentTopicInfoResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.GroupList;\nimport org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;\nimport org.apache.rocketmq.remoting.protocol.body.KVTable;\nimport org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.LockBatchResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.ProducerConnection;\nimport org.apache.rocketmq.remoting.protocol.body.ProducerTableInfo;\nimport org.apache.rocketmq.remoting.protocol.body.QueryAssignmentRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.QueryAssignmentResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.QueryConsumeQueueResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.QueryConsumeTimeSpanBody;\nimport org.apache.rocketmq.remoting.protocol.body.QueryCorrectionOffsetBody;\nimport org.apache.rocketmq.remoting.protocol.body.QuerySubscriptionResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan;\nimport org.apache.rocketmq.remoting.protocol.body.ResetOffsetBody;\nimport org.apache.rocketmq.remoting.protocol.body.SetMessageRequestModeRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupList;\nimport org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.TopicList;\nimport org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.UserInfo;\nimport org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.AddBrokerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.CheckRocksdbCqWriteProgressRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.CloneGroupOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.CreateAclRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.CreateTopicListRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.CreateTopicRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.CreateUserRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.DeleteAclRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.DeleteSubscriptionGroupRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.DeleteTopicRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.DeleteUserRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ExportRocksDBConfigToJsonRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;\nimport org.apache.rocketmq.remoting.protocol.header.GetAclRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetAllProducerInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetAllSubscriptionGroupRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetAllSubscriptionGroupResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetAllTopicConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetAllTopicConfigResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumeStatsInBrokerHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumeStatsRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerConnectionListRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupResponseBody;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerStatusRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetEarliestMsgStoretimeRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetEarliestMsgStoretimeResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetLiteClientInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetLiteGroupInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetLiteTopicInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMinOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMinOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetParentTopicInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetProducerConnectionListRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetSubscriptionGroupConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetTopicConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetTopicStatsInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetTopicsByClusterRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetUserRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.HeartbeatRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ListAclsRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ListUsersRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.LockBatchMqRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopLiteMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopLiteMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumeQueueRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumeTimeSpanRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryCorrectionOffsetHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QuerySubscriptionByConsumerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryTopicConsumeByWhoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryTopicsByConsumerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.RemoveBrokerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ResetMasterFlushOffsetHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ResumeCheckHalfMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SearchOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SearchOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.TriggerLiteDispatchRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UnlockBatchMqRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UnregisterClientRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateAclRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateGroupForbiddenRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateUserRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ViewBrokerStatsDataRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ViewMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.AddWritePermOfBrokerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.AddWritePermOfBrokerResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.DeleteKVConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.DeleteTopicFromNamesrvRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.GetKVConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.GetKVConfigResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.GetKVListByNamespaceRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.GetRouteInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.PutKVConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.WipeWritePermOfBrokerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.WipeWritePermOfBrokerResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicConfigAndQueueMapping;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingDetail;\nimport org.apache.rocketmq.remoting.protocol.subscription.GroupForbidden;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.remoting.rpchook.DynamicalExtFieldRPCHook;\nimport org.apache.rocketmq.remoting.rpchook.StreamTypeRPCHook;\n\nimport static org.apache.rocketmq.common.message.MessageConst.TIMER_ENGINE_TYPE;\nimport static org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode.SUCCESS;\n\npublic class MQClientAPIImpl implements NameServerUpdateCallback, StartAndShutdown {\n    private final static Logger log = LoggerFactory.getLogger(MQClientAPIImpl.class);\n    private static boolean sendSmartMsg =\n        Boolean.parseBoolean(System.getProperty(\"org.apache.rocketmq.client.sendSmartMsg\", \"true\"));\n\n    static {\n        System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION));\n    }\n\n    private final RemotingClient remotingClient;\n    private final TopAddressing topAddressing;\n    private final ClientRemotingProcessor clientRemotingProcessor;\n    private String nameSrvAddr = null;\n    private ClientConfig clientConfig;\n\n    public MQClientAPIImpl(\n        final NettyClientConfig nettyClientConfig,\n        final ClientRemotingProcessor clientRemotingProcessor,\n        final RPCHook rpcHook,\n        final ClientConfig clientConfig\n    ) {\n        this(nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig, null);\n    }\n\n    public MQClientAPIImpl(\n        final NettyClientConfig nettyClientConfig,\n        final ClientRemotingProcessor clientRemotingProcessor,\n        final RPCHook rpcHook,\n        final ClientConfig clientConfig,\n        final ChannelEventListener channelEventListener\n    ) {\n        this(\n            nettyClientConfig,\n            clientRemotingProcessor,\n            rpcHook,\n            clientConfig,\n            channelEventListener,\n            null\n        );\n    }\n\n    public MQClientAPIImpl(final NettyClientConfig nettyClientConfig,\n                           final ClientRemotingProcessor clientRemotingProcessor,\n                           RPCHook rpcHook, final ClientConfig clientConfig,\n                           final ChannelEventListener channelEventListener,\n                           final ObjectCreator<RemotingClient> remotingClientCreator) {\n        this.clientConfig = clientConfig;\n        topAddressing = new DefaultTopAddressing(MixAll.getWSAddr(), clientConfig.getUnitName());\n        topAddressing.registerChangeCallBack(this);\n        this.remotingClient = remotingClientCreator != null\n            ? remotingClientCreator.create(nettyClientConfig, channelEventListener)\n            : new NettyRemotingClient(nettyClientConfig, channelEventListener);\n        this.clientRemotingProcessor = clientRemotingProcessor;\n\n        this.remotingClient.registerRPCHook(new NamespaceRpcHook(clientConfig));\n        // Inject stream rpc hook first to make reserve field signature\n        if (clientConfig.isEnableStreamRequestType()) {\n            this.remotingClient.registerRPCHook(new StreamTypeRPCHook());\n        }\n        this.remotingClient.registerRPCHook(rpcHook);\n        this.remotingClient.registerRPCHook(new DynamicalExtFieldRPCHook());\n        this.remotingClient.registerProcessor(RequestCode.CHECK_TRANSACTION_STATE, this.clientRemotingProcessor, null);\n\n        this.remotingClient.registerProcessor(RequestCode.NOTIFY_UNSUBSCRIBE_LITE, this.clientRemotingProcessor, null);\n\n        this.remotingClient.registerProcessor(RequestCode.NOTIFY_CONSUMER_IDS_CHANGED, this.clientRemotingProcessor, null);\n\n        this.remotingClient.registerProcessor(RequestCode.RESET_CONSUMER_CLIENT_OFFSET, this.clientRemotingProcessor, null);\n\n        this.remotingClient.registerProcessor(RequestCode.GET_CONSUMER_STATUS_FROM_CLIENT, this.clientRemotingProcessor, null);\n\n        this.remotingClient.registerProcessor(RequestCode.GET_CONSUMER_RUNNING_INFO, this.clientRemotingProcessor, null);\n\n        this.remotingClient.registerProcessor(RequestCode.CONSUME_MESSAGE_DIRECTLY, this.clientRemotingProcessor, null);\n\n        this.remotingClient.registerProcessor(RequestCode.PUSH_REPLY_MESSAGE_TO_CLIENT, this.clientRemotingProcessor, null);\n    }\n\n    public List<String> getNameServerAddressList() {\n        return this.remotingClient.getNameServerAddressList();\n    }\n\n    public RemotingClient getRemotingClient() {\n        return remotingClient;\n    }\n\n    public String fetchNameServerAddr() {\n        try {\n            String addrs = this.topAddressing.fetchNSAddr();\n            if (!UtilAll.isBlank(addrs)) {\n                if (!addrs.equals(this.nameSrvAddr)) {\n                    log.info(\"name server address changed, old=\" + this.nameSrvAddr + \", new=\" + addrs);\n                    this.updateNameServerAddressList(addrs);\n                    this.nameSrvAddr = addrs;\n                    return nameSrvAddr;\n                }\n            }\n        } catch (Exception e) {\n            log.error(\"fetchNameServerAddr Exception\", e);\n        }\n        return nameSrvAddr;\n    }\n\n    @Override\n    public String onNameServerAddressChange(String namesrvAddress) {\n        if (namesrvAddress != null) {\n            if (!namesrvAddress.equals(this.nameSrvAddr)) {\n                log.info(\"name server address changed, old=\" + this.nameSrvAddr + \", new=\" + namesrvAddress);\n                this.updateNameServerAddressList(namesrvAddress);\n                this.nameSrvAddr = namesrvAddress;\n                return nameSrvAddr;\n            }\n        }\n        return nameSrvAddr;\n    }\n\n    public void updateNameServerAddressList(final String addrs) {\n        String[] addrArray = addrs.split(\";\");\n        List<String> list = Arrays.asList(addrArray);\n        this.remotingClient.updateNameServerAddressList(list);\n    }\n\n    public void start() {\n        this.remotingClient.start();\n    }\n\n    public void shutdown() {\n        this.remotingClient.shutdown();\n    }\n\n    public Set<MessageQueueAssignment> queryAssignment(final String addr, final String topic,\n        final String consumerGroup, final String clientId, final String strategyName,\n        final MessageModel messageModel, final long timeoutMillis)\n        throws RemotingException, MQBrokerException, InterruptedException {\n        QueryAssignmentRequestBody requestBody = new QueryAssignmentRequestBody();\n        requestBody.setTopic(topic);\n        requestBody.setConsumerGroup(consumerGroup);\n        requestBody.setClientId(clientId);\n        requestBody.setMessageModel(messageModel);\n        requestBody.setStrategyName(strategyName);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_ASSIGNMENT, null);\n        request.setBody(requestBody.encode());\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                QueryAssignmentResponseBody queryAssignmentResponseBody = QueryAssignmentResponseBody.decode(response.getBody(), QueryAssignmentResponseBody.class);\n                return queryAssignmentResponseBody.getMessageQueueAssignments();\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public void createSubscriptionGroup(final String addr, final SubscriptionGroupConfig config,\n        final long timeoutMillis) throws RemotingException, InterruptedException, MQClientException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP, null);\n\n        byte[] body = RemotingSerializable.encode(config);\n        request.setBody(body);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return;\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n\n    }\n\n    public void createSubscriptionGroupList(final String address, final List<SubscriptionGroupConfig> configs,\n        final long timeoutMillis) throws RemotingException, InterruptedException, MQClientException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP_LIST, null);\n        SubscriptionGroupList requestBody = new SubscriptionGroupList(configs);\n        request.setBody(requestBody.encode());\n\n        RemotingCommand response = this.remotingClient.invokeSync(\n            MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), address), request, timeoutMillis);\n        assert response != null;\n        if (response.getCode() == ResponseCode.SUCCESS) {\n            return;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public void createTopic(final String addr, final String defaultTopic, final TopicConfig topicConfig,\n        final long timeoutMillis)\n        throws RemotingException, MQBrokerException, InterruptedException, MQClientException {\n        Validators.checkTopicConfig(topicConfig);\n\n        CreateTopicRequestHeader requestHeader = new CreateTopicRequestHeader();\n        requestHeader.setTopic(topicConfig.getTopicName());\n        requestHeader.setDefaultTopic(defaultTopic);\n        requestHeader.setReadQueueNums(topicConfig.getReadQueueNums());\n        requestHeader.setWriteQueueNums(topicConfig.getWriteQueueNums());\n        requestHeader.setPerm(topicConfig.getPerm());\n        requestHeader.setTopicFilterType(topicConfig.getTopicFilterType().name());\n        requestHeader.setTopicSysFlag(topicConfig.getTopicSysFlag());\n        requestHeader.setOrder(topicConfig.isOrder());\n        requestHeader.setAttributes(AttributeParser.parseToString(topicConfig.getAttributes()));\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_TOPIC, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return;\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public void createTopicList(final String address, final List<TopicConfig> topicConfigList, final long timeoutMillis)\n        throws InterruptedException, RemotingException, MQClientException {\n        CreateTopicListRequestHeader requestHeader = new CreateTopicListRequestHeader();\n        CreateTopicListRequestBody requestBody = new CreateTopicListRequestBody(topicConfigList);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_TOPIC_LIST, requestHeader);\n        request.setBody(requestBody.encode());\n\n        RemotingCommand response = this.remotingClient.invokeSync(\n            MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), address), request, timeoutMillis);\n        assert response != null;\n        if (response.getCode() == ResponseCode.SUCCESS) {\n            return;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public SendResult sendMessage(\n        final String addr,\n        final String brokerName,\n        final Message msg,\n        final SendMessageRequestHeader requestHeader,\n        final long timeoutMillis,\n        final CommunicationMode communicationMode,\n        final SendMessageContext context,\n        final DefaultMQProducerImpl producer\n    ) throws RemotingException, MQBrokerException, InterruptedException {\n        return sendMessage(addr, brokerName, msg, requestHeader, timeoutMillis, communicationMode, null, null, null, 0, context, producer);\n    }\n\n    public SendResult sendMessage(\n        final String addr,\n        final String brokerName,\n        final Message msg,\n        final SendMessageRequestHeader requestHeader,\n        final long timeoutMillis,\n        final CommunicationMode communicationMode,\n        final SendCallback sendCallback,\n        final TopicPublishInfo topicPublishInfo,\n        final MQClientInstance instance,\n        final int retryTimesWhenSendFailed,\n        final SendMessageContext context,\n        final DefaultMQProducerImpl producer\n    ) throws RemotingException, MQBrokerException, InterruptedException {\n        long beginStartTime = System.currentTimeMillis();\n        RemotingCommand request = null;\n        String msgType = msg.getProperty(MessageConst.PROPERTY_MESSAGE_TYPE);\n        boolean isReply = msgType != null && msgType.equals(MixAll.REPLY_MESSAGE_FLAG);\n        if (isReply) {\n            if (sendSmartMsg) {\n                SendMessageRequestHeaderV2 requestHeaderV2 = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(requestHeader);\n                request = RemotingCommand.createRequestCommand(RequestCode.SEND_REPLY_MESSAGE_V2, requestHeaderV2);\n            } else {\n                request = RemotingCommand.createRequestCommand(RequestCode.SEND_REPLY_MESSAGE, requestHeader);\n            }\n        } else {\n            if (sendSmartMsg || msg instanceof MessageBatch) {\n                SendMessageRequestHeaderV2 requestHeaderV2 = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(requestHeader);\n                request = RemotingCommand.createRequestCommand(msg instanceof MessageBatch ? RequestCode.SEND_BATCH_MESSAGE : RequestCode.SEND_MESSAGE_V2, requestHeaderV2);\n            } else {\n                request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, requestHeader);\n            }\n        }\n        request.setBody(msg.getBody());\n\n        switch (communicationMode) {\n            case ONEWAY:\n                this.remotingClient.invokeOneway(addr, request, timeoutMillis);\n                return null;\n            case ASYNC:\n                final AtomicInteger times = new AtomicInteger();\n                long costTimeAsync = System.currentTimeMillis() - beginStartTime;\n                if (timeoutMillis < costTimeAsync) {\n                    throw new RemotingTooMuchRequestException(\"sendMessage call timeout\");\n                }\n                this.sendMessageAsync(addr, brokerName, msg, timeoutMillis - costTimeAsync, request, sendCallback, topicPublishInfo, instance,\n                    retryTimesWhenSendFailed, times, context, producer);\n                return null;\n            case SYNC:\n                long costTimeSync = System.currentTimeMillis() - beginStartTime;\n                if (timeoutMillis < costTimeSync) {\n                    throw new RemotingTooMuchRequestException(\"sendMessage call timeout\");\n                }\n                return this.sendMessageSync(addr, brokerName, msg, timeoutMillis - costTimeSync, request);\n            default:\n                assert false;\n                break;\n        }\n\n        return null;\n    }\n\n    private SendResult sendMessageSync(\n        final String addr,\n        final String brokerName,\n        final Message msg,\n        final long timeoutMillis,\n        final RemotingCommand request\n    ) throws RemotingException, MQBrokerException, InterruptedException {\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis);\n        assert response != null;\n        return this.processSendResponse(brokerName, msg, response, addr);\n    }\n\n    void execRpcHooksAfterRequest(ResponseFuture responseFuture) {\n        if (this.remotingClient instanceof NettyRemotingClient) {\n            NettyRemotingClient remotingClient = (NettyRemotingClient) this.remotingClient;\n            RemotingCommand response = responseFuture.getResponseCommand();\n            remotingClient.doAfterRpcHooks(RemotingHelper.parseChannelRemoteAddr(responseFuture.getChannel()), responseFuture.getRequestCommand(), response);\n        }\n    }\n\n    private void sendMessageAsync(\n        final String addr,\n        final String brokerName,\n        final Message msg,\n        final long timeoutMillis,\n        final RemotingCommand request,\n        final SendCallback sendCallback,\n        final TopicPublishInfo topicPublishInfo,\n        final MQClientInstance instance,\n        final int retryTimesWhenSendFailed,\n        final AtomicInteger times,\n        final SendMessageContext context,\n        final DefaultMQProducerImpl producer\n    ) {\n        final long beginStartTime = System.currentTimeMillis();\n        try {\n            this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {\n                @Override\n                public void operationComplete(ResponseFuture responseFuture) {\n\n                }\n\n                @Override\n                public void operationSucceed(RemotingCommand response) {\n                    long cost = System.currentTimeMillis() - beginStartTime;\n                    if (null == sendCallback) {\n                        try {\n                            SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr);\n                            if (context != null && sendResult != null) {\n                                context.setSendResult(sendResult);\n                                context.getProducer().executeSendMessageHookAfter(context);\n                            }\n                        } catch (Throwable e) {\n                        }\n\n                        producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, false, true);\n                        return;\n                    }\n\n                    try {\n                        SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr);\n                        assert sendResult != null;\n                        if (context != null) {\n                            context.setSendResult(sendResult);\n                            context.getProducer().executeSendMessageHookAfter(context);\n                        }\n\n                        try {\n                            sendCallback.onSuccess(sendResult);\n                        } catch (Throwable e) {\n                        }\n\n                        producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, false, true);\n                    } catch (Exception e) {\n                        producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, true, true);\n                        onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,\n                            retryTimesWhenSendFailed, times, e, context, false, producer);\n                    }\n                }\n\n                @Override\n                public void operationFail(Throwable throwable) {\n                    producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, true, true);\n                    long cost = System.currentTimeMillis() - beginStartTime;\n                    if (throwable instanceof RemotingSendRequestException) {\n                        MQClientException ex = new MQClientException(\"send request failed\", throwable);\n                        onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,\n                            retryTimesWhenSendFailed, times, ex, context, true, producer);\n                    } else if (throwable instanceof RemotingTimeoutException) {\n                        MQClientException ex = new MQClientException(\"wait response timeout, cost=\" + cost, throwable);\n                        onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,\n                            retryTimesWhenSendFailed, times, ex, context, true, producer);\n                    } else {\n                        MQClientException ex = new MQClientException(\"unknown reason\", throwable);\n                        boolean needRetry = !(throwable instanceof RemotingTooMuchRequestException);\n                        onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,\n                            retryTimesWhenSendFailed, times, ex, context, needRetry, producer);\n                    }\n                }\n            });\n        } catch (Exception ex) {\n            long cost = System.currentTimeMillis() - beginStartTime;\n            producer.updateFaultItem(brokerName, cost, true, false);\n            onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,\n                retryTimesWhenSendFailed, times, ex, context, true, producer);\n        }\n    }\n\n    private void onExceptionImpl(final String brokerName,\n        final Message msg,\n        final long timeoutMillis,\n        final RemotingCommand request,\n        final SendCallback sendCallback,\n        final TopicPublishInfo topicPublishInfo,\n        final MQClientInstance instance,\n        final int timesTotal,\n        final AtomicInteger curTimes,\n        final Exception e,\n        final SendMessageContext context,\n        final boolean needRetry,\n        final DefaultMQProducerImpl producer\n    ) {\n        int tmp = curTimes.incrementAndGet();\n        if (needRetry && tmp <= timesTotal && timeoutMillis > 0) {\n            String retryBrokerName = brokerName;//by default, it will send to the same broker\n            if (topicPublishInfo != null) { //select one message queue accordingly, in order to determine which broker to send\n                MessageQueue mqChosen = producer.selectOneMessageQueue(topicPublishInfo, brokerName, false);\n                retryBrokerName = instance.getBrokerNameFromMessageQueue(mqChosen);\n            }\n            String addr = instance.findBrokerAddressInPublish(retryBrokerName);\n            log.warn(\"async send msg by retry {} times. topic={}, brokerAddr={}, brokerName={}\", tmp, msg.getTopic(), addr,\n                retryBrokerName, e);\n            request.setOpaque(RemotingCommand.createNewRequestId());\n            sendMessageAsync(addr, retryBrokerName, msg, timeoutMillis, request, sendCallback, topicPublishInfo, instance,\n                timesTotal, curTimes, context, producer);\n        } else {\n\n            if (context != null) {\n                context.setException(e);\n                context.getProducer().executeSendMessageHookAfter(context);\n            }\n\n            try {\n                sendCallback.onException(e);\n            } catch (Exception ignored) {\n            }\n        }\n    }\n\n    protected SendResult processSendResponse(\n        final String brokerName,\n        final Message msg,\n        final RemotingCommand response,\n        final String addr\n    ) throws MQBrokerException, RemotingCommandException {\n        SendStatus sendStatus;\n        switch (response.getCode()) {\n            case ResponseCode.FLUSH_DISK_TIMEOUT: {\n                sendStatus = SendStatus.FLUSH_DISK_TIMEOUT;\n                break;\n            }\n            case ResponseCode.FLUSH_SLAVE_TIMEOUT: {\n                sendStatus = SendStatus.FLUSH_SLAVE_TIMEOUT;\n                break;\n            }\n            case ResponseCode.SLAVE_NOT_AVAILABLE: {\n                sendStatus = SendStatus.SLAVE_NOT_AVAILABLE;\n                break;\n            }\n            case ResponseCode.SUCCESS: {\n                sendStatus = SendStatus.SEND_OK;\n                break;\n            }\n            default: {\n                throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n            }\n        }\n\n        SendMessageResponseHeader responseHeader =\n            (SendMessageResponseHeader) response.decodeCommandCustomHeader(SendMessageResponseHeader.class);\n\n        //If namespace not null , reset Topic without namespace.\n        String topic = msg.getTopic();\n        if (StringUtils.isNotEmpty(this.clientConfig.getNamespace())) {\n            topic = NamespaceUtil.withoutNamespace(topic, this.clientConfig.getNamespace());\n        }\n\n        MessageQueue messageQueue = new MessageQueue(topic, brokerName, responseHeader.getQueueId());\n\n        String uniqMsgId = MessageClientIDSetter.getUniqID(msg);\n        if (msg instanceof MessageBatch && responseHeader.getBatchUniqId() == null) {\n            // This means it is not an inner batch\n            StringBuilder sb = new StringBuilder();\n            for (Message message : (MessageBatch) msg) {\n                sb.append(sb.length() == 0 ? \"\" : \",\").append(MessageClientIDSetter.getUniqID(message));\n            }\n            uniqMsgId = sb.toString();\n        }\n        SendResult sendResult = new SendResult(sendStatus,\n            uniqMsgId,\n            responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset());\n        sendResult.setTransactionId(responseHeader.getTransactionId());\n        sendResult.setRecallHandle(responseHeader.getRecallHandle());\n        String regionId = response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION);\n        if (regionId == null || regionId.isEmpty()) {\n            regionId = MixAll.DEFAULT_TRACE_REGION_ID;\n        }\n        sendResult.setRegionId(regionId);\n        String traceOn = response.getExtFields().get(MessageConst.PROPERTY_TRACE_SWITCH);\n        sendResult.setTraceOn(!Boolean.FALSE.toString().equals(traceOn));\n        return sendResult;\n    }\n\n    public PullResult pullMessage(\n        final String addr,\n        final PullMessageRequestHeader requestHeader,\n        final long timeoutMillis,\n        final CommunicationMode communicationMode,\n        final PullCallback pullCallback\n    ) throws RemotingException, MQBrokerException, InterruptedException {\n        RemotingCommand request;\n        if (PullSysFlag.hasLitePullFlag(requestHeader.getSysFlag())) {\n            request = RemotingCommand.createRequestCommand(RequestCode.LITE_PULL_MESSAGE, requestHeader);\n        } else {\n            request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, requestHeader);\n        }\n\n        switch (communicationMode) {\n            case ONEWAY:\n                assert false;\n                return null;\n            case ASYNC:\n                this.pullMessageAsync(addr, request, timeoutMillis, pullCallback);\n                return null;\n            case SYNC:\n                return this.pullMessageSync(addr, request, timeoutMillis);\n            default:\n                assert false;\n                break;\n        }\n\n        return null;\n    }\n\n    public void popMessageAsync(\n        final String brokerName, final String addr, final PopMessageRequestHeader requestHeader,\n        final long timeoutMillis, final PopCallback popCallback\n    ) throws RemotingException, InterruptedException {\n        final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.POP_MESSAGE, requestHeader);\n        this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {\n            @Override\n            public void operationComplete(ResponseFuture responseFuture) {\n\n            }\n\n            @Override\n            public void operationSucceed(RemotingCommand response) {\n                try {\n                    PopResult popResult = MQClientAPIImpl.this.processPopResponse(brokerName, response, requestHeader.getTopic(), requestHeader);\n                    popCallback.onSuccess(popResult);\n                } catch (Exception e) {\n                    popCallback.onException(e);\n                }\n            }\n            @Override\n            public void operationFail(Throwable throwable) {\n                popCallback.onException(throwable);\n            }\n        });\n    }\n\n    public void popLiteMessageAsync(\n        final String brokerName, final String addr, final PopLiteMessageRequestHeader requestHeader,\n        final long timeoutMillis, final PopCallback popCallback\n    ) throws RemotingException, InterruptedException {\n        final String bindTopic = requestHeader.getTopic();\n        final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.POP_LITE_MESSAGE, requestHeader);\n        this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {\n            @Override\n            public void operationComplete(ResponseFuture responseFuture) {\n            }\n\n            @Override\n            public void operationSucceed(RemotingCommand response) {\n                try {\n                    PopResult popResult = MQClientAPIImpl.this.processPopLiteResponse(brokerName, response, bindTopic, requestHeader);\n                    popCallback.onSuccess(popResult);\n                } catch (Exception e) {\n                    popCallback.onException(e);\n                }\n            }\n\n            @Override\n            public void operationFail(Throwable throwable) {\n                popCallback.onException(throwable);\n            }\n        });\n    }\n\n    public void ackMessageAsync(\n        final String addr,\n        final long timeOut,\n        final AckCallback ackCallback,\n        final AckMessageRequestHeader requestHeader\n    ) throws RemotingException, MQBrokerException, InterruptedException {\n        ackMessageAsync(addr, timeOut, ackCallback, requestHeader, null);\n    }\n\n    public void ackLiteMessageAsync(\n        final String addr,\n        final long timeout,\n        final AckCallback ackCallback,\n        final AckMessageRequestHeader requestHeader\n    ) throws RemotingException, MQBrokerException, InterruptedException {\n        ackMessageAsync(addr, timeout, ackCallback, requestHeader, null);\n    }\n\n    public void batchAckMessageAsync(\n        final String addr,\n        final long timeOut,\n        final AckCallback ackCallback,\n        final String topic,\n        final String consumerGroup,\n        final List<String> extraInfoList\n    ) throws RemotingException, MQBrokerException, InterruptedException {\n        String brokerName = null;\n        Map<String, BatchAck> batchAckMap = new HashMap<>();\n        for (String extraInfo : extraInfoList) {\n            String[] extraInfoData = ExtraInfoUtil.split(extraInfo);\n            if (brokerName == null) {\n                brokerName = ExtraInfoUtil.getBrokerName(extraInfoData);\n            }\n            String mergeKey = ExtraInfoUtil.getRetry(extraInfoData) + \"@\" +\n                ExtraInfoUtil.getQueueId(extraInfoData) + \"@\" +\n                ExtraInfoUtil.getCkQueueOffset(extraInfoData) + \"@\" +\n                ExtraInfoUtil.getPopTime(extraInfoData);\n            BatchAck bAck = batchAckMap.computeIfAbsent(mergeKey, k -> {\n                BatchAck newBatchAck = new BatchAck();\n                newBatchAck.setConsumerGroup(consumerGroup);\n                newBatchAck.setTopic(topic);\n                newBatchAck.setRetry(ExtraInfoUtil.getRetry(extraInfoData));\n                newBatchAck.setStartOffset(ExtraInfoUtil.getCkQueueOffset(extraInfoData));\n                newBatchAck.setQueueId(ExtraInfoUtil.getQueueId(extraInfoData));\n                newBatchAck.setReviveQueueId(ExtraInfoUtil.getReviveQid(extraInfoData));\n                newBatchAck.setPopTime(ExtraInfoUtil.getPopTime(extraInfoData));\n                newBatchAck.setInvisibleTime(ExtraInfoUtil.getInvisibleTime(extraInfoData));\n                newBatchAck.setBitSet(new BitSet());\n                return newBatchAck;\n            });\n            bAck.getBitSet().set((int) (ExtraInfoUtil.getQueueOffset(extraInfoData) - ExtraInfoUtil.getCkQueueOffset(extraInfoData)));\n        }\n\n        BatchAckMessageRequestBody requestBody = new BatchAckMessageRequestBody();\n        requestBody.setBrokerName(brokerName);\n        requestBody.setAcks(new ArrayList<>(batchAckMap.values()));\n        batchAckMessageAsync(addr, timeOut, ackCallback, requestBody);\n    }\n\n    public void batchAckMessageAsync(\n        final String addr,\n        final long timeOut,\n        final AckCallback ackCallback,\n        final BatchAckMessageRequestBody requestBody\n    ) throws RemotingException, MQBrokerException, InterruptedException {\n        ackMessageAsync(addr, timeOut, ackCallback, null, requestBody);\n    }\n\n    protected void ackMessageAsync(\n        final String addr,\n        final long timeOut,\n        final AckCallback ackCallback,\n        final AckMessageRequestHeader requestHeader,\n        final BatchAckMessageRequestBody requestBody\n    ) throws RemotingException, MQBrokerException, InterruptedException {\n        RemotingCommand request;\n        if (requestHeader != null) {\n            request = RemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader);\n        } else {\n            request = RemotingCommand.createRequestCommand(RequestCode.BATCH_ACK_MESSAGE, null);\n            if (requestBody != null) {\n                request.setBody(requestBody.encode());\n            }\n        }\n        this.remotingClient.invokeAsync(addr, request, timeOut, new InvokeCallback() {\n            @Override\n            public void operationComplete(ResponseFuture responseFuture) {\n\n            }\n\n            @Override\n            public void operationSucceed(RemotingCommand response) {\n                AckResult ackResult = new AckResult();\n                if (ResponseCode.SUCCESS == response.getCode()) {\n                    ackResult.setStatus(AckStatus.OK);\n                } else {\n                    ackResult.setStatus(AckStatus.NO_EXIST);\n                }\n                ackCallback.onSuccess(ackResult);\n            }\n\n            @Override\n            public void operationFail(Throwable throwable) {\n                ackCallback.onException(throwable);\n            }\n        });\n    }\n\n    public void changeInvisibleTimeAsync(//\n        final String brokerName,\n        final String addr, //\n        final ChangeInvisibleTimeRequestHeader requestHeader,//\n        final long timeoutMillis,\n        final AckCallback ackCallback\n    ) throws RemotingException, MQBrokerException, InterruptedException {\n        final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader);\n        this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {\n            @Override\n            public void operationComplete(ResponseFuture responseFuture) {\n\n            }\n\n            @Override\n            public void operationSucceed(RemotingCommand response) {\n                try {\n                    ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.decodeCommandCustomHeader(ChangeInvisibleTimeResponseHeader.class);\n                    AckResult ackResult = new AckResult();\n                    if (ResponseCode.SUCCESS == response.getCode()) {\n                        ackResult.setStatus(AckStatus.OK);\n                        ackResult.setPopTime(responseHeader.getPopTime());\n                        ackResult.setExtraInfo(ExtraInfoUtil\n                            .buildExtraInfo(requestHeader.getOffset(), responseHeader.getPopTime(), responseHeader.getInvisibleTime(),\n                                responseHeader.getReviveQid(), requestHeader.getTopic(), brokerName, requestHeader.getQueueId()) + MessageConst.KEY_SEPARATOR\n                            + requestHeader.getOffset());\n                    } else {\n                        ackResult.setStatus(AckStatus.NO_EXIST);\n                    }\n                    ackCallback.onSuccess(ackResult);\n                } catch (Exception e) {\n                    ackCallback.onException(e);\n                }\n            }\n\n            @Override\n            public void operationFail(Throwable throwable) {\n                ackCallback.onException(throwable);\n            }\n        });\n    }\n\n    private void pullMessageAsync(\n        final String addr,\n        final RemotingCommand request,\n        final long timeoutMillis,\n        final PullCallback pullCallback\n    ) throws RemotingException, InterruptedException {\n        this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {\n            @Override\n            public void operationComplete(ResponseFuture responseFuture) {\n\n            }\n\n            @Override\n            public void operationSucceed(RemotingCommand response) {\n                try {\n                    PullResult pullResult = MQClientAPIImpl.this.processPullResponse(response, addr);\n                    pullCallback.onSuccess(pullResult);\n                } catch (Exception e) {\n                    pullCallback.onException(e);\n                }\n            }\n\n            @Override\n            public void operationFail(Throwable throwable) {\n                pullCallback.onException(throwable);\n            }\n        });\n    }\n\n    private PullResult pullMessageSync(\n        final String addr,\n        final RemotingCommand request,\n        final long timeoutMillis\n    ) throws RemotingException, InterruptedException, MQBrokerException {\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis);\n        assert response != null;\n        return this.processPullResponse(response, addr);\n    }\n\n    private PullResult processPullResponse(\n        final RemotingCommand response,\n        final String addr) throws MQBrokerException, RemotingCommandException {\n        PullStatus pullStatus = PullStatus.NO_NEW_MSG;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS:\n                pullStatus = PullStatus.FOUND;\n                break;\n            case ResponseCode.PULL_NOT_FOUND:\n                pullStatus = PullStatus.NO_NEW_MSG;\n                break;\n            case ResponseCode.PULL_RETRY_IMMEDIATELY:\n                pullStatus = PullStatus.NO_MATCHED_MSG;\n                break;\n            case ResponseCode.PULL_OFFSET_MOVED:\n                pullStatus = PullStatus.OFFSET_ILLEGAL;\n                break;\n\n            default:\n                throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n        }\n\n        PullMessageResponseHeader responseHeader =\n            (PullMessageResponseHeader) response.decodeCommandCustomHeader(PullMessageResponseHeader.class);\n\n        return new PullResultExt(pullStatus, responseHeader.getNextBeginOffset(), responseHeader.getMinOffset(),\n            responseHeader.getMaxOffset(), null, responseHeader.getSuggestWhichBrokerId(), response.getBody(), responseHeader.getOffsetDelta());\n    }\n\n    private PopResult processPopResponse(final String brokerName, final RemotingCommand response, String topic,\n        CommandCustomHeader requestHeader) throws MQBrokerException, RemotingCommandException {\n        PopStatus popStatus = PopStatus.NO_NEW_MSG;\n        List<MessageExt> msgFoundList = null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS:\n                popStatus = PopStatus.FOUND;\n                ByteBuffer byteBuffer = ByteBuffer.wrap(response.getBody());\n                msgFoundList = MessageDecoder.decodesBatch(\n                    byteBuffer,\n                    clientConfig.isDecodeReadBody(),\n                    clientConfig.isDecodeDecompressBody(),\n                    true);\n                break;\n            case ResponseCode.POLLING_FULL:\n                popStatus = PopStatus.POLLING_FULL;\n                break;\n            case ResponseCode.POLLING_TIMEOUT:\n                popStatus = PopStatus.POLLING_NOT_FOUND;\n                break;\n            case ResponseCode.PULL_NOT_FOUND:\n                popStatus = PopStatus.POLLING_NOT_FOUND;\n                break;\n            default:\n                throw new MQBrokerException(response.getCode(), response.getRemark());\n        }\n\n        PopResult popResult = new PopResult(popStatus, msgFoundList);\n        PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.decodeCommandCustomHeader(PopMessageResponseHeader.class);\n        popResult.setRestNum(responseHeader.getRestNum());\n        if (popStatus != PopStatus.FOUND) {\n            return popResult;\n        }\n        // it is a pop command if pop time greater than 0, we should set the check point info to extraInfo field\n        Map<String, Long> startOffsetInfo = null;\n        Map<String, List<Long>> msgOffsetInfo = null;\n        Map<String, Integer> orderCountInfo = null;\n        if (requestHeader instanceof PopMessageRequestHeader) {\n            popResult.setInvisibleTime(responseHeader.getInvisibleTime());\n            popResult.setPopTime(responseHeader.getPopTime());\n            startOffsetInfo = ExtraInfoUtil.parseStartOffsetInfo(responseHeader.getStartOffsetInfo());\n            msgOffsetInfo = ExtraInfoUtil.parseMsgOffsetInfo(responseHeader.getMsgOffsetInfo());\n            orderCountInfo = ExtraInfoUtil.parseOrderCountInfo(responseHeader.getOrderCountInfo());\n        }\n        Map<String/*topicMark@queueId*/, List<Long>/*msg queueOffset*/> sortMap\n            = buildQueueOffsetSortedMap(topic, msgFoundList);\n        Map<String, String> map = new HashMap<>(5);\n        for (MessageExt messageExt : msgFoundList) {\n            if (requestHeader instanceof PopMessageRequestHeader) {\n                if (startOffsetInfo == null) {\n                    // we should set the check point info to extraInfo field , if the command is popMsg\n                    // find pop ck offset\n                    String key = messageExt.getTopic() + messageExt.getQueueId();\n                    if (!map.containsKey(messageExt.getTopic() + messageExt.getQueueId())) {\n                        map.put(key, ExtraInfoUtil.buildExtraInfo(messageExt.getQueueOffset(), responseHeader.getPopTime(), responseHeader.getInvisibleTime(), responseHeader.getReviveQid(),\n                            messageExt.getTopic(), brokerName, messageExt.getQueueId()));\n\n                    }\n                    messageExt.getProperties().put(MessageConst.PROPERTY_POP_CK, map.get(key) + MessageConst.KEY_SEPARATOR + messageExt.getQueueOffset());\n                } else {\n                    if (messageExt.getProperty(MessageConst.PROPERTY_POP_CK) == null) {\n                        final String queueIdKey;\n                        final String queueOffsetKey;\n                        final int index;\n                        final Long msgQueueOffset;\n                        if (MixAll.isLmq(topic) && messageExt.getReconsumeTimes() == 0 && StringUtils.isNotEmpty(\n                            messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH))) {\n                            // process LMQ\n                            String[] queues = messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH)\n                                .split(MixAll.LMQ_DISPATCH_SEPARATOR);\n                            String[] queueOffsets = messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET)\n                                .split(MixAll.LMQ_DISPATCH_SEPARATOR);\n                            long offset = Long.parseLong(queueOffsets[ArrayUtils.indexOf(queues, topic)]);\n                            // LMQ topic has only 1 queue, which queue id is 0\n                            queueIdKey = ExtraInfoUtil.getStartOffsetInfoMapKey(topic, MixAll.LMQ_QUEUE_ID);\n                            queueOffsetKey = ExtraInfoUtil.getQueueOffsetMapKey(topic, MixAll.LMQ_QUEUE_ID, offset);\n                            index = sortMap.get(queueIdKey).indexOf(offset);\n                            msgQueueOffset = msgOffsetInfo.get(queueIdKey).get(index);\n                            if (msgQueueOffset != offset) {\n                                log.warn(\"Queue offset[{}] of msg is strange, not equal to the stored in msg, {}\",\n                                    msgQueueOffset, messageExt);\n                            }\n                            messageExt.getProperties().put(MessageConst.PROPERTY_POP_CK,\n                                ExtraInfoUtil.buildExtraInfo(startOffsetInfo.get(queueIdKey), responseHeader.getPopTime(), responseHeader.getInvisibleTime(),\n                                    responseHeader.getReviveQid(), topic, brokerName, 0, msgQueueOffset)\n                            );\n                        } else {\n                            queueIdKey = ExtraInfoUtil.getStartOffsetInfoMapKey(messageExt.getTopic(), messageExt.getQueueId());\n                            queueOffsetKey = ExtraInfoUtil.getQueueOffsetMapKey(messageExt.getTopic(), messageExt.getQueueId(), messageExt.getQueueOffset());\n                            index = sortMap.get(queueIdKey).indexOf(messageExt.getQueueOffset());\n                            msgQueueOffset = msgOffsetInfo.get(queueIdKey).get(index);\n                            if (msgQueueOffset != messageExt.getQueueOffset()) {\n                                log.warn(\"Queue offset[{}] of msg is strange, not equal to the stored in msg, {}\", msgQueueOffset, messageExt);\n                            }\n                            messageExt.getProperties().put(MessageConst.PROPERTY_POP_CK,\n                                ExtraInfoUtil.buildExtraInfo(startOffsetInfo.get(queueIdKey), responseHeader.getPopTime(), responseHeader.getInvisibleTime(),\n                                    responseHeader.getReviveQid(), messageExt.getTopic(), brokerName, messageExt.getQueueId(), msgQueueOffset)\n                            );\n                        }\n                        if (((PopMessageRequestHeader) requestHeader).isOrder() && orderCountInfo != null) {\n                            Integer count = orderCountInfo.get(queueOffsetKey);\n                            if (count == null) {\n                                count = orderCountInfo.get(queueIdKey);\n                            }\n                            if (count != null && count > 0) {\n                                messageExt.setReconsumeTimes(count);\n                            }\n                        }\n                    }\n                }\n                messageExt.getProperties().computeIfAbsent(\n                    MessageConst.PROPERTY_FIRST_POP_TIME, k -> String.valueOf(responseHeader.getPopTime()));\n            }\n            messageExt.setBrokerName(brokerName);\n            messageExt.setTopic(NamespaceUtil.withoutNamespace(topic, this.clientConfig.getNamespace()));\n        }\n        return popResult;\n    }\n\n    private PopResult processPopLiteResponse(final String brokerName, final RemotingCommand response, String topic,\n        CommandCustomHeader requestHeader) throws MQBrokerException, RemotingCommandException {\n        PopStatus popStatus;\n        List<MessageExt> msgFoundList = null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS:\n                popStatus = PopStatus.FOUND;\n                ByteBuffer byteBuffer = ByteBuffer.wrap(response.getBody());\n                msgFoundList = MessageDecoder.decodesBatch(\n                    byteBuffer,\n                    clientConfig.isDecodeReadBody(),\n                    clientConfig.isDecodeDecompressBody(),\n                    true);\n                break;\n            case ResponseCode.POLLING_FULL:\n                popStatus = PopStatus.POLLING_FULL;\n                break;\n            case ResponseCode.POLLING_TIMEOUT:\n                popStatus = PopStatus.POLLING_NOT_FOUND;\n                break;\n            case ResponseCode.PULL_NOT_FOUND:\n                popStatus = PopStatus.POLLING_NOT_FOUND;\n                break;\n            default:\n                throw new MQBrokerException(response.getCode(), response.getRemark());\n        }\n\n        PopResult popResult = new PopResult(popStatus, msgFoundList);\n        PopLiteMessageResponseHeader responseHeader = response.decodeCommandCustomHeader(PopLiteMessageResponseHeader.class);\n        if (popStatus != PopStatus.FOUND) {\n            return popResult;\n        }\n\n        List<Integer> orderCountList = ExtraInfoUtil.parseLiteOrderCountInfo(responseHeader.getOrderCountInfo(), msgFoundList.size());\n        for (int i = 0; i < msgFoundList.size(); i++) {\n            MessageExt messageExt = msgFoundList.get(i);\n            String[] queues = StringUtils.split(\n                messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH), MixAll.LMQ_DISPATCH_SEPARATOR);\n            String[] queueOffsets = StringUtils.split(\n                messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET), MixAll.LMQ_DISPATCH_SEPARATOR);\n\n            if (null == queues || null == queueOffsets || queues.length != 1 || queues.length != queueOffsets.length) {\n                continue;\n            }\n            messageExt.getProperties().put(MessageConst.PROPERTY_POP_CK,\n                ExtraInfoUtil.buildExtraInfo(0, responseHeader.getPopTime(), responseHeader.getInvisibleTime(),\n                    responseHeader.getReviveQid(), topic, brokerName, 0, Long.parseLong(queueOffsets[0])));\n            messageExt.getProperties().computeIfAbsent(\n                MessageConst.PROPERTY_FIRST_POP_TIME, k -> String.valueOf(responseHeader.getPopTime()));\n            messageExt.setBrokerName(brokerName);\n            messageExt.setReconsumeTimes(orderCountList != null ? orderCountList.get(i) : 0);\n            messageExt.setQueueOffset(Long.parseLong(queueOffsets[0]));\n        }\n        return popResult;\n    }\n\n    /**\n     * Build queue offset sorted map\n     *\n     * @param topic pop consumer topic\n     * @param msgFoundList popped message list\n     * @return sorted map, key is topicMark@queueId, value is sorted msg queueOffset list\n     */\n    private static Map<String, List<Long>> buildQueueOffsetSortedMap(String topic, List<MessageExt> msgFoundList) {\n        Map<String/*topicMark@queueId*/, List<Long>/*msg queueOffset*/> sortMap = new HashMap<>(16);\n        for (MessageExt messageExt : msgFoundList) {\n            final String key;\n            if (MixAll.isLmq(topic) && messageExt.getReconsumeTimes() == 0\n                && StringUtils.isNotEmpty(messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH))) {\n                // process LMQ\n                String[] queues = messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH)\n                    .split(MixAll.LMQ_DISPATCH_SEPARATOR);\n                String[] queueOffsets = messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET)\n                    .split(MixAll.LMQ_DISPATCH_SEPARATOR);\n                // LMQ topic has only 1 queue, which queue id is 0\n                key = ExtraInfoUtil.getStartOffsetInfoMapKey(topic, MixAll.LMQ_QUEUE_ID);\n                sortMap.putIfAbsent(key, new ArrayList<>(4));\n                sortMap.get(key).add(Long.parseLong(queueOffsets[ArrayUtils.indexOf(queues, topic)]));\n                continue;\n            }\n            // Value of POP_CK is used to determine whether it is a pop retry,\n            // cause topic could be rewritten by broker.\n            key = ExtraInfoUtil.getStartOffsetInfoMapKey(messageExt.getTopic(),\n                messageExt.getProperty(MessageConst.PROPERTY_POP_CK), messageExt.getQueueId());\n            if (!sortMap.containsKey(key)) {\n                sortMap.put(key, new ArrayList<>(4));\n            }\n            sortMap.get(key).add(messageExt.getQueueOffset());\n        }\n        return sortMap;\n    }\n\n    public MessageExt viewMessage(final String addr, final String topic, final long phyoffset, final long timeoutMillis)\n        throws RemotingException, MQBrokerException, InterruptedException {\n        ViewMessageRequestHeader requestHeader = new ViewMessageRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setOffset(phyoffset);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.VIEW_MESSAGE_BY_ID, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                ByteBuffer byteBuffer = ByteBuffer.wrap(response.getBody());\n                MessageExt messageExt = MessageDecoder.clientDecode(byteBuffer, true);\n                //If namespace not null , reset Topic without namespace.\n                if (StringUtils.isNotEmpty(this.clientConfig.getNamespace())) {\n                    messageExt.setTopic(NamespaceUtil.withoutNamespace(messageExt.getTopic(), this.clientConfig.getNamespace()));\n                }\n                return messageExt;\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    @Deprecated\n    public long searchOffset(final String addr, final String topic, final int queueId, final long timestamp,\n        final long timeoutMillis)\n        throws RemotingException, MQBrokerException, InterruptedException {\n        SearchOffsetRequestHeader requestHeader = new SearchOffsetRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setQueueId(queueId);\n        requestHeader.setTimestamp(timestamp);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                SearchOffsetResponseHeader responseHeader =\n                    (SearchOffsetResponseHeader) response.decodeCommandCustomHeader(SearchOffsetResponseHeader.class);\n                return responseHeader.getOffset();\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public long searchOffset(final String addr, final MessageQueue messageQueue, final long timestamp,\n        final long timeoutMillis)\n        throws RemotingException, MQBrokerException, InterruptedException {\n        // default return lower boundary offset when there are more than one offsets.\n        return searchOffset(addr, messageQueue, timestamp, BoundaryType.LOWER, timeoutMillis);\n    }\n\n    public long searchOffset(final String addr, final MessageQueue messageQueue, final long timestamp,\n        final BoundaryType boundaryType, final long timeoutMillis)\n        throws RemotingException, MQBrokerException, InterruptedException {\n        SearchOffsetRequestHeader requestHeader = new SearchOffsetRequestHeader();\n        requestHeader.setTopic(messageQueue.getTopic());\n        requestHeader.setQueueId(messageQueue.getQueueId());\n        requestHeader.setBrokerName(messageQueue.getBrokerName());\n        requestHeader.setTimestamp(timestamp);\n        requestHeader.setBoundaryType(boundaryType);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                SearchOffsetResponseHeader responseHeader =\n                    (SearchOffsetResponseHeader) response.decodeCommandCustomHeader(SearchOffsetResponseHeader.class);\n                return responseHeader.getOffset();\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public long getMaxOffset(final String addr, final MessageQueue messageQueue, final long timeoutMillis)\n        throws RemotingException, MQBrokerException, InterruptedException {\n        GetMaxOffsetRequestHeader requestHeader = new GetMaxOffsetRequestHeader();\n        requestHeader.setTopic(messageQueue.getTopic());\n        requestHeader.setQueueId(messageQueue.getQueueId());\n        requestHeader.setBrokerName(messageQueue.getBrokerName());\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_MAX_OFFSET, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                GetMaxOffsetResponseHeader responseHeader =\n                    (GetMaxOffsetResponseHeader) response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class);\n\n                return responseHeader.getOffset();\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public List<String> getConsumerIdListByGroup(\n        final String addr,\n        final String consumerGroup,\n        final long timeoutMillis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException,\n        MQBrokerException, InterruptedException {\n        GetConsumerListByGroupRequestHeader requestHeader = new GetConsumerListByGroupRequestHeader();\n        requestHeader.setConsumerGroup(consumerGroup);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_LIST_BY_GROUP, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                if (response.getBody() != null) {\n                    GetConsumerListByGroupResponseBody body =\n                        GetConsumerListByGroupResponseBody.decode(response.getBody(), GetConsumerListByGroupResponseBody.class);\n                    return body.getConsumerIdList();\n                }\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public long getMinOffset(final String addr, final MessageQueue messageQueue, final long timeoutMillis)\n        throws RemotingException, MQBrokerException, InterruptedException {\n        GetMinOffsetRequestHeader requestHeader = new GetMinOffsetRequestHeader();\n        requestHeader.setTopic(messageQueue.getTopic());\n        requestHeader.setQueueId(messageQueue.getQueueId());\n        requestHeader.setBrokerName(messageQueue.getBrokerName());\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_MIN_OFFSET, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                GetMinOffsetResponseHeader responseHeader =\n                    (GetMinOffsetResponseHeader) response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class);\n\n                return responseHeader.getOffset();\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public long getEarliestMsgStoretime(final String addr, final MessageQueue mq, final long timeoutMillis)\n        throws RemotingException, MQBrokerException, InterruptedException {\n        GetEarliestMsgStoretimeRequestHeader requestHeader = new GetEarliestMsgStoretimeRequestHeader();\n        requestHeader.setTopic(mq.getTopic());\n        requestHeader.setQueueId(mq.getQueueId());\n        requestHeader.setBrokerName(mq.getBrokerName());\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_EARLIEST_MSG_STORETIME, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                GetEarliestMsgStoretimeResponseHeader responseHeader =\n                    (GetEarliestMsgStoretimeResponseHeader) response.decodeCommandCustomHeader(GetEarliestMsgStoretimeResponseHeader.class);\n\n                return responseHeader.getTimestamp();\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public long queryConsumerOffset(\n        final String addr,\n        final QueryConsumerOffsetRequestHeader requestHeader,\n        final long timeoutMillis\n    ) throws RemotingException, MQBrokerException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                QueryConsumerOffsetResponseHeader responseHeader =\n                    (QueryConsumerOffsetResponseHeader) response.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class);\n                return responseHeader.getOffset();\n            }\n            case ResponseCode.QUERY_NOT_FOUND: {\n                throw new OffsetNotFoundException(response.getCode(), response.getRemark(), addr);\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public void updateConsumerOffset(\n        final String addr,\n        final UpdateConsumerOffsetRequestHeader requestHeader,\n        final long timeoutMillis\n    ) throws RemotingException, MQBrokerException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_CONSUMER_OFFSET, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return;\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public void updateConsumerOffsetOneway(\n        final String addr,\n        final UpdateConsumerOffsetRequestHeader requestHeader,\n        final long timeoutMillis\n    ) throws RemotingConnectException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException,\n        InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_CONSUMER_OFFSET, requestHeader);\n\n        this.remotingClient.invokeOneway(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), request, timeoutMillis);\n    }\n\n    public int sendHeartbeat(\n        final String addr,\n        final HeartbeatData heartbeatData,\n        final long timeoutMillis\n    ) throws RemotingException, MQBrokerException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.HEART_BEAT, new HeartbeatRequestHeader());\n        request.setLanguage(clientConfig.getLanguage());\n        request.setBody(heartbeatData.encode());\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return response.getVersion();\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public HeartbeatV2Result sendHeartbeatV2(\n        final String addr,\n        final HeartbeatData heartbeatData,\n        final long timeoutMillis\n    ) throws RemotingException, MQBrokerException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.HEART_BEAT, new HeartbeatRequestHeader());\n        request.setLanguage(clientConfig.getLanguage());\n        request.setBody(heartbeatData.encode());\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                if (response.getExtFields() != null) {\n                    return new HeartbeatV2Result(response.getVersion(), Boolean.parseBoolean(response.getExtFields().get(MixAll.IS_SUB_CHANGE)), Boolean.parseBoolean(response.getExtFields().get(MixAll.IS_SUPPORT_HEART_BEAT_V2)));\n                }\n                return new HeartbeatV2Result(response.getVersion(), false, false);\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public void unregisterClient(\n        final String addr,\n        final String clientID,\n        final String producerGroup,\n        final String consumerGroup,\n        final long timeoutMillis\n    ) throws RemotingException, MQBrokerException, InterruptedException {\n        final UnregisterClientRequestHeader requestHeader = new UnregisterClientRequestHeader();\n        requestHeader.setClientID(clientID);\n        requestHeader.setProducerGroup(producerGroup);\n        requestHeader.setConsumerGroup(consumerGroup);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UNREGISTER_CLIENT, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return;\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public void endTransactionOneway(\n        final String addr,\n        final EndTransactionRequestHeader requestHeader,\n        final String remark,\n        final long timeoutMillis\n    ) throws RemotingException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.END_TRANSACTION, requestHeader);\n\n        request.setRemark(remark);\n        this.remotingClient.invokeOneway(addr, request, timeoutMillis);\n    }\n\n    public void queryMessage(\n        final String addr,\n        final QueryMessageRequestHeader requestHeader,\n        final long timeoutMillis,\n        final InvokeCallback invokeCallback,\n        final Boolean isUniqueKey\n    ) throws RemotingException, MQBrokerException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_MESSAGE, requestHeader);\n        request.addExtField(MixAll.UNIQUE_MSG_QUERY_FLAG, isUniqueKey.toString());\n        this.remotingClient.invokeAsync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), request, timeoutMillis,\n            invokeCallback);\n    }\n\n    public boolean registerClient(final String addr, final HeartbeatData heartbeat, final long timeoutMillis)\n        throws RemotingException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.HEART_BEAT, new HeartbeatRequestHeader());\n\n        request.setBody(heartbeat.encode());\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis);\n        return response.getCode() == ResponseCode.SUCCESS;\n    }\n\n    public void consumerSendMessageBack(\n        final String addr,\n        final String brokerName,\n        final MessageExt msg,\n        final String consumerGroup,\n        final int delayLevel,\n        final long timeoutMillis,\n        final int maxConsumeRetryTimes\n    ) throws RemotingException, MQBrokerException, InterruptedException {\n        ConsumerSendMsgBackRequestHeader requestHeader = new ConsumerSendMsgBackRequestHeader();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, requestHeader);\n\n        requestHeader.setGroup(consumerGroup);\n        requestHeader.setOriginTopic(msg.getTopic());\n        requestHeader.setOffset(msg.getCommitLogOffset());\n        requestHeader.setDelayLevel(delayLevel);\n        requestHeader.setOriginMsgId(msg.getMsgId());\n        requestHeader.setMaxReconsumeTimes(maxConsumeRetryTimes);\n        requestHeader.setBrokerName(brokerName);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return;\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public Set<MessageQueue> lockBatchMQ(\n        final String addr,\n        final LockBatchRequestBody requestBody,\n        final long timeoutMillis) throws RemotingException, MQBrokerException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, new LockBatchMqRequestHeader());\n\n        request.setBody(requestBody.encode());\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), LockBatchResponseBody.class);\n                Set<MessageQueue> messageQueues = responseBody.getLockOKMQSet();\n                return messageQueues;\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public void unlockBatchMQ(\n        final String addr,\n        final UnlockBatchRequestBody requestBody,\n        final long timeoutMillis,\n        final boolean oneway\n    ) throws RemotingException, MQBrokerException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UNLOCK_BATCH_MQ, new UnlockBatchMqRequestHeader());\n\n        request.setBody(requestBody.encode());\n\n        if (oneway) {\n            this.remotingClient.invokeOneway(addr, request, timeoutMillis);\n        } else {\n            RemotingCommand response = this.remotingClient\n                .invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), request, timeoutMillis);\n            switch (response.getCode()) {\n                case ResponseCode.SUCCESS: {\n                    return;\n                }\n                default:\n                    break;\n            }\n\n            throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n        }\n    }\n\n    public TopicStatsTable getTopicStatsInfo(final String addr, final String topic,\n        final long timeoutMillis) throws InterruptedException,\n        RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {\n        GetTopicStatsInfoRequestHeader requestHeader = new GetTopicStatsInfoRequestHeader();\n        requestHeader.setTopic(topic);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_STATS_INFO, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                TopicStatsTable topicStatsTable = TopicStatsTable.decode(response.getBody(), TopicStatsTable.class);\n                return topicStatsTable;\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public ConsumeStats getConsumeStats(final String addr, final String consumerGroup, final long timeoutMillis)\n        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException,\n        MQBrokerException {\n        return getConsumeStats(addr, consumerGroup, null, null, timeoutMillis);\n    }\n\n    public ConsumeStats getConsumeStats(final String addr, final String consumerGroup, final List<String> topicList,\n        final long timeoutMillis) throws RemotingSendRequestException, RemotingConnectException, RemotingTimeoutException, MQBrokerException, InterruptedException {\n        return getConsumeStats(addr, consumerGroup, null, topicList, timeoutMillis);\n    }\n\n    public ConsumeStats getConsumeStats(final String addr, final String consumerGroup, final String topic,\n        final long timeoutMillis) throws RemotingSendRequestException, RemotingConnectException, RemotingTimeoutException, MQBrokerException, InterruptedException {\n        return getConsumeStats(addr, consumerGroup, topic, null, timeoutMillis);\n    }\n\n    public ConsumeStats getConsumeStats(final String addr, final String consumerGroup, final String topic,\n        final List<String> topicList, final long timeoutMillis)\n        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException,\n        MQBrokerException {\n        GetConsumeStatsRequestHeader requestHeader = new GetConsumeStatsRequestHeader();\n        requestHeader.setConsumerGroup(consumerGroup);\n        requestHeader.setTopic(topic);\n        requestHeader.updateTopicList(topicList);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUME_STATS, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                ConsumeStats consumeStats = ConsumeStats.decode(response.getBody(), ConsumeStats.class);\n                return consumeStats;\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public ProducerConnection getProducerConnectionList(final String addr, final String producerGroup,\n        final long timeoutMillis)\n        throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException,\n        MQBrokerException {\n        GetProducerConnectionListRequestHeader requestHeader = new GetProducerConnectionListRequestHeader();\n        requestHeader.setProducerGroup(producerGroup);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_PRODUCER_CONNECTION_LIST, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return ProducerConnection.decode(response.getBody(), ProducerConnection.class);\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public ProducerTableInfo getAllProducerInfo(final String addr, final long timeoutMillis)\n        throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException,\n        MQBrokerException {\n        GetAllProducerInfoRequestHeader requestHeader = new GetAllProducerInfoRequestHeader();\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_PRODUCER_INFO, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return ProducerTableInfo.decode(response.getBody(), ProducerTableInfo.class);\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public ConsumerConnection getConsumerConnectionList(final String addr, final String consumerGroup,\n        final long timeoutMillis)\n        throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException,\n        MQBrokerException {\n        GetConsumerConnectionListRequestHeader requestHeader = new GetConsumerConnectionListRequestHeader();\n        requestHeader.setConsumerGroup(consumerGroup);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_CONNECTION_LIST, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return ConsumerConnection.decode(response.getBody(), ConsumerConnection.class);\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public KVTable getBrokerRuntimeInfo(final String addr, final long timeoutMillis) throws RemotingConnectException,\n        RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_RUNTIME_INFO, null);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return KVTable.decode(response.getBody(), KVTable.class);\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public void addBroker(final String addr, final String brokerConfigPath, final long timeoutMillis)\n        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {\n        AddBrokerRequestHeader requestHeader = new AddBrokerRequestHeader();\n        requestHeader.setConfigPath(brokerConfigPath);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.ADD_BROKER, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis);\n        assert response != null;\n\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS:\n                return;\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public void removeBroker(final String addr, String clusterName, String brokerName, long brokerId,\n        final long timeoutMillis)\n        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {\n        RemoveBrokerRequestHeader requestHeader = new RemoveBrokerRequestHeader();\n        requestHeader.setBrokerClusterName(clusterName);\n        requestHeader.setBrokerName(brokerName);\n        requestHeader.setBrokerId(brokerId);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.REMOVE_BROKER, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis);\n        assert response != null;\n\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS:\n                return;\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public void updateBrokerConfig(final String addr, final Properties properties, final long timeoutMillis)\n        throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException,\n        MQBrokerException, MQClientException, UnsupportedEncodingException {\n        Validators.checkBrokerConfig(properties);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_BROKER_CONFIG, null);\n\n        String str = MixAll.properties2String(properties);\n        if (str != null && str.length() > 0) {\n            request.setBody(str.getBytes(MixAll.DEFAULT_CHARSET));\n            RemotingCommand response = this.remotingClient\n                .invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), request, timeoutMillis);\n            switch (response.getCode()) {\n                case ResponseCode.SUCCESS: {\n                    return;\n                }\n                default:\n                    break;\n            }\n\n            throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n        }\n    }\n\n    public Properties getBrokerConfig(final String addr, final long timeoutMillis)\n        throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException,\n        MQBrokerException, UnsupportedEncodingException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_CONFIG, null);\n\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return MixAll.string2Properties(new String(response.getBody(), MixAll.DEFAULT_CHARSET));\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public void updateColdDataFlowCtrGroupConfig(final String addr, final Properties properties, final long timeoutMillis)\n        throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException, UnsupportedEncodingException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_COLD_DATA_FLOW_CTR_CONFIG, null);\n        String str = MixAll.properties2String(properties);\n        if (str != null && str.length() > 0) {\n            request.setBody(str.getBytes(MixAll.DEFAULT_CHARSET));\n            RemotingCommand response = this.remotingClient.invokeSync(\n                MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), request, timeoutMillis);\n            switch (response.getCode()) {\n                case ResponseCode.SUCCESS: {\n                    return;\n                }\n                default:\n                    break;\n            }\n            throw new MQBrokerException(response.getCode(), response.getRemark());\n        }\n    }\n\n    public void removeColdDataFlowCtrGroupConfig(final String addr, final String consumerGroup, final long timeoutMillis)\n        throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException, UnsupportedEncodingException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.REMOVE_COLD_DATA_FLOW_CTR_CONFIG, null);\n        if (consumerGroup != null && consumerGroup.length() > 0) {\n            request.setBody(consumerGroup.getBytes(MixAll.DEFAULT_CHARSET));\n            RemotingCommand response = this.remotingClient.invokeSync(\n                MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), request, timeoutMillis);\n            switch (response.getCode()) {\n                case ResponseCode.SUCCESS: {\n                    return;\n                }\n                default:\n                    break;\n            }\n            throw new MQBrokerException(response.getCode(), response.getRemark());\n        }\n    }\n\n    public String getColdDataFlowCtrInfo(final String addr, final long timeoutMillis)\n        throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException, UnsupportedEncodingException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_COLD_DATA_FLOW_CTR_INFO, null);\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                if (null != response.getBody() && response.getBody().length > 0) {\n                    return new String(response.getBody(), MixAll.DEFAULT_CHARSET);\n                }\n                return null;\n            }\n            default:\n                break;\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public String setCommitLogReadAheadMode(final String addr, final String mode, final long timeoutMillis)\n        throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SET_COMMITLOG_READ_MODE, null);\n        HashMap<String, String> extFields = new HashMap<>();\n        extFields.put(FIleReadaheadMode.READ_AHEAD_MODE, mode);\n        request.setExtFields(extFields);\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                if (null != response.getRemark() && response.getRemark().length() > 0) {\n                    return response.getRemark();\n                }\n                return null;\n            }\n            default:\n                break;\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public ClusterInfo getBrokerClusterInfo(\n        final long timeoutMillis) throws InterruptedException, RemotingTimeoutException,\n        RemotingSendRequestException, RemotingConnectException, MQBrokerException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_CLUSTER_INFO, null);\n\n        RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return ClusterInfo.decode(response.getBody(), ClusterInfo.class);\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public TopicRouteData getDefaultTopicRouteInfoFromNameServer(final long timeoutMillis)\n        throws RemotingException, MQClientException, InterruptedException {\n\n        return getTopicRouteInfoFromNameServer(TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC, timeoutMillis, false);\n    }\n\n    public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final long timeoutMillis)\n        throws RemotingException, MQClientException, InterruptedException {\n        return getTopicRouteInfoFromNameServer(topic, timeoutMillis, true);\n    }\n\n    public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final long timeoutMillis,\n        boolean allowTopicNotExist) throws MQClientException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {\n        GetRouteInfoRequestHeader requestHeader = new GetRouteInfoRequestHeader();\n        requestHeader.setTopic(topic);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.TOPIC_NOT_EXIST: {\n                if (allowTopicNotExist) {\n                    log.warn(\"get Topic [{}] RouteInfoFromNameServer is not exist value\", topic);\n                }\n\n                break;\n            }\n            case ResponseCode.SUCCESS: {\n                byte[] body = response.getBody();\n                if (body != null) {\n                    return TopicRouteData.decode(body, TopicRouteData.class);\n                }\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public TopicList getTopicListFromNameServer(final long timeoutMillis)\n        throws RemotingException, MQClientException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_TOPIC_LIST_FROM_NAMESERVER, null);\n        RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                byte[] body = response.getBody();\n                if (body != null) {\n                    return TopicList.decode(body, TopicList.class);\n                }\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public int wipeWritePermOfBroker(final String namesrvAddr, String brokerName,\n        final long timeoutMillis) throws RemotingCommandException,\n        RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQClientException {\n        WipeWritePermOfBrokerRequestHeader requestHeader = new WipeWritePermOfBrokerRequestHeader();\n        requestHeader.setBrokerName(brokerName);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.WIPE_WRITE_PERM_OF_BROKER, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(namesrvAddr, request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                WipeWritePermOfBrokerResponseHeader responseHeader =\n                    (WipeWritePermOfBrokerResponseHeader) response.decodeCommandCustomHeader(WipeWritePermOfBrokerResponseHeader.class);\n                return responseHeader.getWipeTopicCount();\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public int addWritePermOfBroker(final String nameSrvAddr, String brokerName, final long timeoutMillis)\n        throws RemotingCommandException,\n        RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQClientException {\n        AddWritePermOfBrokerRequestHeader requestHeader = new AddWritePermOfBrokerRequestHeader();\n        requestHeader.setBrokerName(brokerName);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.ADD_WRITE_PERM_OF_BROKER, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(nameSrvAddr, request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                AddWritePermOfBrokerResponseHeader responseHeader =\n                    (AddWritePermOfBrokerResponseHeader) response.decodeCommandCustomHeader(AddWritePermOfBrokerResponseHeader.class);\n                return responseHeader.getAddTopicCount();\n            }\n            default:\n                break;\n        }\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public void deleteTopicInBroker(final String addr, final String topic, final long timeoutMillis)\n        throws RemotingException, InterruptedException, MQClientException {\n        DeleteTopicRequestHeader requestHeader = new DeleteTopicRequestHeader();\n        requestHeader.setTopic(topic);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_TOPIC_IN_BROKER, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return;\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public void deleteTopicInNameServer(final String addr, final String topic, final long timeoutMillis)\n        throws RemotingException, InterruptedException, MQClientException {\n        DeleteTopicFromNamesrvRequestHeader requestHeader = new DeleteTopicFromNamesrvRequestHeader();\n        requestHeader.setTopic(topic);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_TOPIC_IN_NAMESRV, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return;\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public void deleteTopicInNameServer(final String addr, final String clusterName, final String topic,\n        final long timeoutMillis)\n        throws RemotingException, MQBrokerException, InterruptedException, MQClientException {\n        DeleteTopicFromNamesrvRequestHeader requestHeader = new DeleteTopicFromNamesrvRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setClusterName(clusterName);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_TOPIC_IN_NAMESRV, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return;\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public void deleteSubscriptionGroup(final String addr, final String groupName, final boolean removeOffset,\n        final long timeoutMillis)\n        throws RemotingException, InterruptedException, MQClientException {\n        DeleteSubscriptionGroupRequestHeader requestHeader = new DeleteSubscriptionGroupRequestHeader();\n        requestHeader.setGroupName(groupName);\n        requestHeader.setCleanOffset(removeOffset);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_SUBSCRIPTIONGROUP, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return;\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public String getKVConfigValue(final String namespace, final String key, final long timeoutMillis)\n        throws RemotingException, MQClientException, InterruptedException {\n        GetKVConfigRequestHeader requestHeader = new GetKVConfigRequestHeader();\n        requestHeader.setNamespace(namespace);\n        requestHeader.setKey(key);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_KV_CONFIG, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                GetKVConfigResponseHeader responseHeader =\n                    (GetKVConfigResponseHeader) response.decodeCommandCustomHeader(GetKVConfigResponseHeader.class);\n                return responseHeader.getValue();\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public void putKVConfigValue(final String namespace, final String key, final String value, final long timeoutMillis)\n        throws RemotingException, MQClientException, InterruptedException {\n        PutKVConfigRequestHeader requestHeader = new PutKVConfigRequestHeader();\n        requestHeader.setNamespace(namespace);\n        requestHeader.setKey(key);\n        requestHeader.setValue(value);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PUT_KV_CONFIG, requestHeader);\n\n        List<String> nameServerAddressList = this.remotingClient.getNameServerAddressList();\n        if (nameServerAddressList != null) {\n            RemotingCommand errResponse = null;\n            for (String namesrvAddr : nameServerAddressList) {\n                RemotingCommand response = this.remotingClient.invokeSync(namesrvAddr, request, timeoutMillis);\n                assert response != null;\n                switch (response.getCode()) {\n                    case ResponseCode.SUCCESS: {\n                        break;\n                    }\n                    default:\n                        errResponse = response;\n                }\n            }\n\n            if (errResponse != null) {\n                throw new MQClientException(errResponse.getCode(), errResponse.getRemark());\n            }\n        }\n    }\n\n    public void deleteKVConfigValue(final String namespace, final String key, final long timeoutMillis)\n        throws RemotingException, MQClientException, InterruptedException {\n        DeleteKVConfigRequestHeader requestHeader = new DeleteKVConfigRequestHeader();\n        requestHeader.setNamespace(namespace);\n        requestHeader.setKey(key);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_KV_CONFIG, requestHeader);\n\n        List<String> nameServerAddressList = this.remotingClient.getNameServerAddressList();\n        if (nameServerAddressList != null) {\n            RemotingCommand errResponse = null;\n            for (String namesrvAddr : nameServerAddressList) {\n                RemotingCommand response = this.remotingClient.invokeSync(namesrvAddr, request, timeoutMillis);\n                assert response != null;\n                switch (response.getCode()) {\n                    case ResponseCode.SUCCESS: {\n                        break;\n                    }\n                    default:\n                        errResponse = response;\n                }\n            }\n            if (errResponse != null) {\n                throw new MQClientException(errResponse.getCode(), errResponse.getRemark());\n            }\n        }\n    }\n\n    public KVTable getKVListByNamespace(final String namespace, final long timeoutMillis)\n        throws RemotingException, MQClientException, InterruptedException {\n        GetKVListByNamespaceRequestHeader requestHeader = new GetKVListByNamespaceRequestHeader();\n        requestHeader.setNamespace(namespace);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_KVLIST_BY_NAMESPACE, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return KVTable.decode(response.getBody(), KVTable.class);\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public Map<MessageQueue, Long> invokeBrokerToResetOffset(final String addr, final String topic, final String group,\n        final long timestamp, final boolean isForce, final long timeoutMillis)\n        throws RemotingException, MQClientException, InterruptedException {\n        return invokeBrokerToResetOffset(addr, topic, group, timestamp, isForce, timeoutMillis, false);\n    }\n\n    public Map<MessageQueue, Long> invokeBrokerToResetOffset(final String addr, final String topic, final String group,\n        final long timestamp, int queueId, Long offset, final long timeoutMillis)\n        throws RemotingException, MQClientException, InterruptedException {\n\n        ResetOffsetRequestHeader requestHeader = new ResetOffsetRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setGroup(group);\n        requestHeader.setQueueId(queueId);\n        requestHeader.setTimestamp(timestamp);\n        requestHeader.setOffset(offset);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.INVOKE_BROKER_TO_RESET_OFFSET,\n            requestHeader);\n\n        RemotingCommand response = remotingClient.invokeSync(\n            MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), request, timeoutMillis);\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                if (null != response.getBody()) {\n                    return ResetOffsetBody.decode(response.getBody(), ResetOffsetBody.class).getOffsetTable();\n                }\n                break;\n            }\n            case ResponseCode.TOPIC_NOT_EXIST:\n            case ResponseCode.SUBSCRIPTION_NOT_EXIST:\n            case ResponseCode.SYSTEM_ERROR:\n                log.warn(\"Invoke broker to reset offset error code={}, remark={}\",\n                    response.getCode(), response.getRemark());\n                break;\n            default:\n                break;\n        }\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public Map<MessageQueue, Long> invokeBrokerToResetOffset(final String addr, final String topic, final String group,\n        final long timestamp, final boolean isForce, final long timeoutMillis, boolean isC)\n        throws RemotingException, MQClientException, InterruptedException {\n        ResetOffsetRequestHeader requestHeader = new ResetOffsetRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setGroup(group);\n        requestHeader.setTimestamp(timestamp);\n        requestHeader.setForce(isForce);\n        // offset is -1 means offset is null\n        requestHeader.setOffset(-1L);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.INVOKE_BROKER_TO_RESET_OFFSET, requestHeader);\n        if (isC) {\n            request.setLanguage(LanguageCode.CPP);\n        }\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                if (response.getBody() != null) {\n                    ResetOffsetBody body = ResetOffsetBody.decode(response.getBody(), ResetOffsetBody.class);\n                    return body.getOffsetTable();\n                }\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public Map<String, Map<MessageQueue, Long>> invokeBrokerToGetConsumerStatus(final String addr, final String topic,\n        final String group,\n        final String clientAddr,\n        final long timeoutMillis) throws RemotingException, MQClientException, InterruptedException {\n        GetConsumerStatusRequestHeader requestHeader = new GetConsumerStatusRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setGroup(group);\n        requestHeader.setClientAddr(clientAddr);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.INVOKE_BROKER_TO_GET_CONSUMER_STATUS, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                if (response.getBody() != null) {\n                    GetConsumerStatusBody body = GetConsumerStatusBody.decode(response.getBody(), GetConsumerStatusBody.class);\n                    return body.getConsumerTable();\n                }\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public GroupList queryTopicConsumeByWho(final String addr, final String topic, final long timeoutMillis)\n        throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException,\n        MQBrokerException {\n        QueryTopicConsumeByWhoRequestHeader requestHeader = new QueryTopicConsumeByWhoRequestHeader();\n        requestHeader.setTopic(topic);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_TOPIC_CONSUME_BY_WHO, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                GroupList groupList = GroupList.decode(response.getBody(), GroupList.class);\n                return groupList;\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public TopicList queryTopicsByConsumer(final String addr, final String group, final long timeoutMillis)\n        throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException,\n        MQBrokerException {\n        QueryTopicsByConsumerRequestHeader requestHeader = new QueryTopicsByConsumerRequestHeader();\n        requestHeader.setGroup(group);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_TOPICS_BY_CONSUMER, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                TopicList topicList = TopicList.decode(response.getBody(), TopicList.class);\n                return topicList;\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public SubscriptionData querySubscriptionByConsumer(final String addr, final String group, final String topic,\n        final long timeoutMillis)\n        throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException,\n        MQBrokerException {\n        QuerySubscriptionByConsumerRequestHeader requestHeader = new QuerySubscriptionByConsumerRequestHeader();\n        requestHeader.setGroup(group);\n        requestHeader.setTopic(topic);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_SUBSCRIPTION_BY_CONSUMER, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                QuerySubscriptionResponseBody subscriptionResponseBody =\n                    QuerySubscriptionResponseBody.decode(response.getBody(), QuerySubscriptionResponseBody.class);\n                return subscriptionResponseBody.getSubscriptionData();\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public List<QueueTimeSpan> queryConsumeTimeSpan(final String addr, final String topic, final String group,\n        final long timeoutMillis)\n        throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException,\n        MQBrokerException {\n        QueryConsumeTimeSpanRequestHeader requestHeader = new QueryConsumeTimeSpanRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setGroup(group);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUME_TIME_SPAN, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                QueryConsumeTimeSpanBody consumeTimeSpanBody = GroupList.decode(response.getBody(), QueryConsumeTimeSpanBody.class);\n                return consumeTimeSpanBody.getConsumeTimeSpanSet();\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public TopicList getTopicsByCluster(final String cluster, final long timeoutMillis)\n        throws RemotingException, MQClientException, InterruptedException {\n        GetTopicsByClusterRequestHeader requestHeader = new GetTopicsByClusterRequestHeader();\n        requestHeader.setCluster(cluster);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPICS_BY_CLUSTER, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                byte[] body = response.getBody();\n                if (body != null) {\n                    TopicList topicList = TopicList.decode(body, TopicList.class);\n                    return topicList;\n                }\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public TopicList getSystemTopicList(\n        final long timeoutMillis) throws RemotingException, MQClientException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_SYSTEM_TOPIC_LIST_FROM_NS, null);\n        RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                byte[] body = response.getBody();\n                if (body != null) {\n                    TopicList topicList = TopicList.decode(response.getBody(), TopicList.class);\n                    if (topicList.getTopicList() != null && !topicList.getTopicList().isEmpty()\n                        && !UtilAll.isBlank(topicList.getBrokerAddr())) {\n                        TopicList tmp = getSystemTopicListFromBroker(topicList.getBrokerAddr(), timeoutMillis);\n                        if (tmp.getTopicList() != null && !tmp.getTopicList().isEmpty()) {\n                            topicList.getTopicList().addAll(tmp.getTopicList());\n                        }\n                    }\n                    return topicList;\n                }\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public TopicList getSystemTopicListFromBroker(final String addr, final long timeoutMillis)\n        throws RemotingException, MQClientException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_SYSTEM_TOPIC_LIST_FROM_BROKER, null);\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                byte[] body = response.getBody();\n                if (body != null) {\n                    TopicList topicList = TopicList.decode(body, TopicList.class);\n                    return topicList;\n                }\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public boolean cleanExpiredConsumeQueue(final String addr,\n        long timeoutMillis) throws MQClientException, RemotingConnectException,\n        RemotingSendRequestException, RemotingTimeoutException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CLEAN_EXPIRED_CONSUMEQUEUE, null);\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return true;\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public boolean deleteExpiredCommitLog(final String addr, long timeoutMillis) throws MQClientException,\n        RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_EXPIRED_COMMITLOG, null);\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return true;\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public boolean cleanUnusedTopicByAddr(final String addr,\n        long timeoutMillis) throws MQClientException, RemotingConnectException,\n        RemotingSendRequestException, RemotingTimeoutException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CLEAN_UNUSED_TOPIC, null);\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return true;\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public ConsumerRunningInfo getConsumerRunningInfo(final String addr, String consumerGroup, String clientId,\n        boolean jstack,\n        final long timeoutMillis) throws RemotingException, MQClientException, InterruptedException {\n        GetConsumerRunningInfoRequestHeader requestHeader = new GetConsumerRunningInfoRequestHeader();\n        requestHeader.setConsumerGroup(consumerGroup);\n        requestHeader.setClientId(clientId);\n        requestHeader.setJstackEnable(jstack);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_RUNNING_INFO, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                byte[] body = response.getBody();\n                if (body != null) {\n                    ConsumerRunningInfo info = ConsumerRunningInfo.decode(body, ConsumerRunningInfo.class);\n                    return info;\n                }\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public ConsumeMessageDirectlyResult consumeMessageDirectly(final String addr,\n        String consumerGroup,\n        String clientId,\n        String topic,\n        String msgId,\n        final long timeoutMillis) throws RemotingException, MQClientException, InterruptedException {\n        ConsumeMessageDirectlyResultRequestHeader requestHeader = new ConsumeMessageDirectlyResultRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setConsumerGroup(consumerGroup);\n        requestHeader.setClientId(clientId);\n        requestHeader.setMsgId(msgId);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONSUME_MESSAGE_DIRECTLY, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                byte[] body = response.getBody();\n                if (body != null) {\n                    ConsumeMessageDirectlyResult info = ConsumeMessageDirectlyResult.decode(body, ConsumeMessageDirectlyResult.class);\n                    return info;\n                }\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public Map<Integer, Long> queryCorrectionOffset(final String addr, final String topic, final String group,\n        Set<String> filterGroup,\n        long timeoutMillis) throws MQClientException, RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException,\n        InterruptedException {\n        QueryCorrectionOffsetHeader requestHeader = new QueryCorrectionOffsetHeader();\n        requestHeader.setCompareGroup(group);\n        requestHeader.setTopic(topic);\n        if (filterGroup != null) {\n            StringBuilder sb = new StringBuilder();\n            String splitor = \"\";\n            for (String s : filterGroup) {\n                sb.append(splitor).append(s);\n                splitor = \",\";\n            }\n            requestHeader.setFilterGroups(sb.toString());\n        }\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CORRECTION_OFFSET, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                if (response.getBody() != null) {\n                    QueryCorrectionOffsetBody body = QueryCorrectionOffsetBody.decode(response.getBody(), QueryCorrectionOffsetBody.class);\n                    return body.getCorrectionOffsets();\n                }\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public TopicList getUnitTopicList(final boolean containRetry, final long timeoutMillis)\n        throws RemotingException, MQClientException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_UNIT_TOPIC_LIST, null);\n        RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                byte[] body = response.getBody();\n                if (body != null) {\n                    TopicList topicList = TopicList.decode(response.getBody(), TopicList.class);\n                    if (!containRetry) {\n                        Iterator<String> it = topicList.getTopicList().iterator();\n                        while (it.hasNext()) {\n                            String topic = it.next();\n                            if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n                                it.remove();\n                            }\n                        }\n                    }\n\n                    return topicList;\n                }\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public TopicList getHasUnitSubTopicList(final boolean containRetry, final long timeoutMillis)\n        throws RemotingException, MQClientException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_HAS_UNIT_SUB_TOPIC_LIST, null);\n        RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                byte[] body = response.getBody();\n                if (body != null) {\n                    TopicList topicList = TopicList.decode(response.getBody(), TopicList.class);\n                    if (!containRetry) {\n                        Iterator<String> it = topicList.getTopicList().iterator();\n                        while (it.hasNext()) {\n                            String topic = it.next();\n                            if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n                                it.remove();\n                            }\n                        }\n                    }\n                    return topicList;\n                }\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public TopicList getHasUnitSubUnUnitTopicList(final boolean containRetry, final long timeoutMillis)\n        throws RemotingException, MQClientException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_HAS_UNIT_SUB_UNUNIT_TOPIC_LIST, null);\n        RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                byte[] body = response.getBody();\n                if (body != null) {\n                    TopicList topicList = TopicList.decode(response.getBody(), TopicList.class);\n                    if (!containRetry) {\n                        Iterator<String> it = topicList.getTopicList().iterator();\n                        while (it.hasNext()) {\n                            String topic = it.next();\n                            if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n                                it.remove();\n                            }\n                        }\n                    }\n                    return topicList;\n                }\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public void cloneGroupOffset(final String addr, final String srcGroup, final String destGroup, final String topic,\n        final boolean isOffline,\n        final long timeoutMillis) throws RemotingException, MQClientException, InterruptedException {\n        CloneGroupOffsetRequestHeader requestHeader = new CloneGroupOffsetRequestHeader();\n        requestHeader.setSrcGroup(srcGroup);\n        requestHeader.setDestGroup(destGroup);\n        requestHeader.setTopic(topic);\n        requestHeader.setOffline(isOffline);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CLONE_GROUP_OFFSET, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return;\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public BrokerStatsData viewBrokerStatsData(String brokerAddr, String statsName, String statsKey, long timeoutMillis)\n        throws MQClientException, RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException,\n        InterruptedException {\n        ViewBrokerStatsDataRequestHeader requestHeader = new ViewBrokerStatsDataRequestHeader();\n        requestHeader.setStatsName(statsName);\n        requestHeader.setStatsKey(statsKey);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.VIEW_BROKER_STATS_DATA, requestHeader);\n        RemotingCommand response = this.remotingClient\n            .invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), brokerAddr), request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                byte[] body = response.getBody();\n                if (body != null) {\n                    return BrokerStatsData.decode(body, BrokerStatsData.class);\n                }\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public Set<String> getClusterList(String topic,\n        long timeoutMillis) {\n        return Collections.EMPTY_SET;\n    }\n\n    public ConsumeStatsList fetchConsumeStatsInBroker(String brokerAddr, boolean isOrder,\n        long timeoutMillis) throws MQClientException,\n        RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException {\n        GetConsumeStatsInBrokerHeader requestHeader = new GetConsumeStatsInBrokerHeader();\n        requestHeader.setIsOrder(isOrder);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_CONSUME_STATS, requestHeader);\n        RemotingCommand response = this.remotingClient\n            .invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), brokerAddr), request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                byte[] body = response.getBody();\n                if (body != null) {\n                    return ConsumeStatsList.decode(body, ConsumeStatsList.class);\n                }\n            }\n            default:\n                break;\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public SubscriptionGroupWrapper getAllSubscriptionGroup(final String brokerAddr, long timeoutMillis)\n        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException,\n        RemotingConnectException, MQBrokerException, RemotingCommandException {\n\n        DataVersion currentDataVersion = null;\n        int groupSeq = 0;\n        ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroupTable = new ConcurrentHashMap<>();\n        ConcurrentMap<String, ConcurrentMap<String, Integer>> forbiddenTable = new ConcurrentHashMap<>();\n        long beginTime = System.nanoTime();\n        while (true) {\n            long leftTime = timeoutMillis - TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - beginTime);\n            if (leftTime < 0) {\n                throw new RemotingTimeoutException(\"invokeSync call timeout\");\n            }\n\n            GetAllSubscriptionGroupRequestHeader requestHeader = new GetAllSubscriptionGroupRequestHeader();\n            requestHeader.setGroupSeq(groupSeq);\n            requestHeader.setMaxGroupNum(clientConfig.getMaxPageSizeInGetMetadata());\n            requestHeader.setDataVersion(Optional.ofNullable(currentDataVersion)\n                .map(DataVersion::toJson).orElse(StringUtils.EMPTY));\n            log.info(\"getAllSubscriptionGroup from seq {}, max {}, dataVersion {}\",\n                    groupSeq, requestHeader.getMaxGroupNum(), requestHeader.getDataVersion());\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_SUBSCRIPTIONGROUP_CONFIG, requestHeader);\n            RemotingCommand response = this.remotingClient.invokeSync(\n                MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), brokerAddr), request, leftTime);\n\n            assert response != null;\n            if (response.getCode() == SUCCESS) {\n                SubscriptionGroupWrapper subscriptionGroupWrapper =\n                    SubscriptionGroupWrapper.decode(response.getBody(), SubscriptionGroupWrapper.class);\n                subscriptionGroupTable.putAll(subscriptionGroupWrapper.getSubscriptionGroupTable());\n                forbiddenTable.putAll(subscriptionGroupWrapper.getForbiddenTable());\n\n                DataVersion newDataVersion = subscriptionGroupWrapper.getDataVersion();\n                if (currentDataVersion == null) {\n                    // fill dataVersion before break the loop to compatible with old version server\n                    currentDataVersion = newDataVersion;\n                }\n\n                groupSeq += subscriptionGroupWrapper.getSubscriptionGroupTable().size();\n\n                GetAllSubscriptionGroupResponseHeader responseHeader =\n                    response.decodeCommandCustomHeader(GetAllSubscriptionGroupResponseHeader.class);\n                Integer totalGroupNum = Optional.ofNullable(responseHeader)\n                    .map(GetAllSubscriptionGroupResponseHeader::getTotalGroupNum).orElse(null);\n\n                if (Objects.isNull(totalGroupNum)) {\n                    // the server side don't support totalGroupNum, all data is returned\n                    break;\n                }\n\n                if (!Objects.equals(currentDataVersion, newDataVersion)) {\n                    log.error(\"dataVersion changed, currentDataVersion: {}, newDataVersion: {}\",\n                        currentDataVersion, newDataVersion);\n                    currentDataVersion = newDataVersion;\n                    groupSeq = 0;\n                    subscriptionGroupTable.clear();\n                    forbiddenTable.clear();\n                    continue;\n                }\n\n                if (groupSeq >= totalGroupNum - 1) {\n                    log.info(\"get all subscription group config, totalGroupNum: {}\", totalGroupNum);\n                    break;\n                }\n            } else {\n                throw new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr);\n            }\n        }\n\n        SubscriptionGroupWrapper allSubscriptionGroup = new SubscriptionGroupWrapper();\n        allSubscriptionGroup.setSubscriptionGroupTable(subscriptionGroupTable);\n        allSubscriptionGroup.setForbiddenTable(forbiddenTable);\n        allSubscriptionGroup.setDataVersion(currentDataVersion);\n        return allSubscriptionGroup;\n    }\n\n    public SubscriptionGroupConfig getSubscriptionGroupConfig(final String brokerAddr, String group,\n        long timeoutMillis) throws InterruptedException,\n        RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {\n        GetSubscriptionGroupConfigRequestHeader header = new GetSubscriptionGroupConfigRequestHeader();\n        header.setGroup(group);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_SUBSCRIPTIONGROUP_CONFIG, header);\n        RemotingCommand response = this.remotingClient\n            .invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), brokerAddr), request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return RemotingSerializable.decode(response.getBody(), SubscriptionGroupConfig.class);\n            }\n            default:\n                break;\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr);\n    }\n\n    public TopicConfigSerializeWrapper getAllTopicConfig(final String addr, long timeoutMillis)\n        throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException,\n        InterruptedException, MQBrokerException, RemotingCommandException {\n\n        DataVersion currentDataVersion = null;\n        int topicSeq = 0;\n        ConcurrentMap<String, TopicConfig> topicConfigTable = new ConcurrentHashMap<>();\n        long beginTime = System.nanoTime();\n        while (true) {\n            long leftTime = timeoutMillis - TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - beginTime);\n            if (leftTime <= 0) {\n                throw new RemotingTimeoutException(\"invokeSync call timeout\");\n            }\n\n            GetAllTopicConfigRequestHeader requestHeader = new GetAllTopicConfigRequestHeader();\n            requestHeader.setTopicSeq(topicSeq);\n            requestHeader.setMaxTopicNum(clientConfig.getMaxPageSizeInGetMetadata());\n            requestHeader.setDataVersion(Optional.ofNullable(currentDataVersion).\n                map(DataVersion::toJson).orElse(StringUtils.EMPTY));\n            log.info(\"getAllTopicConfig from seq {}, max {}, dataVersion {}\",\n                    topicSeq, requestHeader.getMaxTopicNum(), requestHeader.getDataVersion());\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_TOPIC_CONFIG, requestHeader);\n\n            RemotingCommand response = this.remotingClient.invokeSync(\n                MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), request, leftTime);\n\n            assert response != null;\n            if (response.getCode() == SUCCESS) {\n                TopicConfigSerializeWrapper topicConfigSerializeWrapper =\n                    TopicConfigSerializeWrapper.decode(response.getBody(), TopicConfigSerializeWrapper.class);\n                topicConfigTable.putAll(topicConfigSerializeWrapper.getTopicConfigTable());\n                topicSeq += topicConfigSerializeWrapper.getTopicConfigTable().size();\n\n                DataVersion newDataVersion = topicConfigSerializeWrapper.getDataVersion();\n                if (currentDataVersion == null) {\n                    // fill dataVersion before break the loop to compatible with old version server\n                    currentDataVersion = newDataVersion;\n                }\n\n                GetAllTopicConfigResponseHeader responseHeader =\n                    response.decodeCommandCustomHeader(GetAllTopicConfigResponseHeader.class);\n                Integer totalTopicNum = Optional.ofNullable(responseHeader)\n                    .map(GetAllTopicConfigResponseHeader::getTotalTopicNum).orElse(null);\n\n                if (Objects.isNull(totalTopicNum)) {       // compatible with old version server\n                    // the server side don't support totalTopicNum, all data is returned\n                    break;\n                }\n\n                if (!Objects.equals(currentDataVersion, newDataVersion)) {\n                    log.error(\"dataVersion changed, currentDataVersion: {}, newDataVersion: {}\", currentDataVersion, newDataVersion);\n                    currentDataVersion = newDataVersion;\n                    topicSeq = 0;\n                    topicConfigTable.clear();\n                    continue;\n                }\n\n                if (topicSeq >= totalTopicNum - 1) {\n                    log.info(\"get all topic config, totalTopicNum: {}\", totalTopicNum);\n                    break;\n                }\n            } else {\n                throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n            }\n\n        }\n\n        TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();\n        topicConfigSerializeWrapper.setDataVersion(currentDataVersion);\n        topicConfigSerializeWrapper.setTopicConfigTable(topicConfigTable);\n        return topicConfigSerializeWrapper;\n    }\n\n    public void updateNameServerConfig(final Properties properties, final List<String> nameServers, long timeoutMillis)\n        throws UnsupportedEncodingException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException,\n        RemotingConnectException, MQClientException {\n        String str = MixAll.properties2String(properties);\n        if (str == null || str.length() < 1) {\n            return;\n        }\n        List<String> invokeNameServers = (nameServers == null || nameServers.isEmpty()) ?\n            this.remotingClient.getNameServerAddressList() : nameServers;\n        if (invokeNameServers == null || invokeNameServers.isEmpty()) {\n            return;\n        }\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_NAMESRV_CONFIG, null);\n        request.setBody(str.getBytes(MixAll.DEFAULT_CHARSET));\n\n        RemotingCommand errResponse = null;\n        for (String nameServer : invokeNameServers) {\n            RemotingCommand response = this.remotingClient.invokeSync(nameServer, request, timeoutMillis);\n            assert response != null;\n            switch (response.getCode()) {\n                case ResponseCode.SUCCESS: {\n                    break;\n                }\n                default:\n                    errResponse = response;\n            }\n        }\n\n        if (errResponse != null) {\n            throw new MQClientException(errResponse.getCode(), errResponse.getRemark());\n        }\n    }\n\n    public Map<String, Properties> getNameServerConfig(final List<String> nameServers, long timeoutMillis)\n        throws InterruptedException,\n        RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException,\n        MQClientException, UnsupportedEncodingException {\n        List<String> invokeNameServers = (nameServers == null || nameServers.isEmpty()) ?\n            this.remotingClient.getNameServerAddressList() : nameServers;\n        if (invokeNameServers == null || invokeNameServers.isEmpty()) {\n            return null;\n        }\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_NAMESRV_CONFIG, null);\n\n        Map<String, Properties> configMap = new HashMap<>(4);\n        for (String nameServer : invokeNameServers) {\n            RemotingCommand response = this.remotingClient.invokeSync(nameServer, request, timeoutMillis);\n\n            assert response != null;\n\n            if (ResponseCode.SUCCESS == response.getCode()) {\n                configMap.put(nameServer, MixAll.string2Properties(new String(response.getBody(), MixAll.DEFAULT_CHARSET)));\n            } else {\n                throw new MQClientException(response.getCode(), response.getRemark());\n            }\n        }\n        return configMap;\n    }\n\n    public QueryConsumeQueueResponseBody queryConsumeQueue(final String brokerAddr, final String topic,\n        final int queueId,\n        final long index, final int count, final String consumerGroup,\n        final long timeoutMillis) throws InterruptedException,\n        RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException {\n\n        QueryConsumeQueueRequestHeader requestHeader = new QueryConsumeQueueRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setQueueId(queueId);\n        requestHeader.setIndex(index);\n        requestHeader.setCount(count);\n        requestHeader.setConsumerGroup(consumerGroup);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUME_QUEUE, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), brokerAddr), request, timeoutMillis);\n\n        assert response != null;\n\n        if (ResponseCode.SUCCESS == response.getCode()) {\n            return QueryConsumeQueueResponseBody.decode(response.getBody(), QueryConsumeQueueResponseBody.class);\n        }\n\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public CheckRocksdbCqWriteResult checkRocksdbCqWriteProgress(final String brokerAddr, final String topic, final long checkStoreTime, final long timeoutMillis) throws InterruptedException,\n        RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException {\n        CheckRocksdbCqWriteProgressRequestHeader header = new CheckRocksdbCqWriteProgressRequestHeader();\n        header.setTopic(topic);\n        header.setCheckStoreTime(checkStoreTime);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHECK_ROCKSDB_CQ_WRITE_PROGRESS, header);\n        RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis);\n        assert response != null;\n        if (ResponseCode.SUCCESS == response.getCode()) {\n            return JSON.parseObject(response.getBody(), CheckRocksdbCqWriteResult.class);\n        }\n        throw new MQClientException(response.getCode(), response.getRemark());\n    }\n\n    public void exportRocksDBConfigToJson(final String brokerAddr,\n        final List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> configType,\n        final long timeoutMillis) throws InterruptedException,\n        RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException {\n        ExportRocksDBConfigToJsonRequestHeader header = new ExportRocksDBConfigToJsonRequestHeader();\n        header.updateConfigType(configType);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.EXPORT_ROCKSDB_CONFIG_TO_JSON, header);\n        RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis);\n        assert response != null;\n\n        if (ResponseCode.SUCCESS != response.getCode()) {\n            throw new MQClientException(response.getCode(), response.getRemark());\n        }\n    }\n\n    public void checkClientInBroker(final String brokerAddr, final String consumerGroup,\n        final String clientId, final SubscriptionData subscriptionData,\n        final long timeoutMillis)\n        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException,\n        RemotingConnectException, MQClientException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHECK_CLIENT_CONFIG, null);\n\n        CheckClientRequestBody requestBody = new CheckClientRequestBody();\n        requestBody.setClientId(clientId);\n        requestBody.setGroup(consumerGroup);\n        requestBody.setSubscriptionData(subscriptionData);\n\n        request.setBody(requestBody.encode());\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), brokerAddr), request, timeoutMillis);\n\n        assert response != null;\n\n        if (ResponseCode.SUCCESS != response.getCode()) {\n            throw new MQClientException(response.getCode(), response.getRemark());\n        }\n    }\n\n    public boolean resumeCheckHalfMessage(final String addr, String topic, String msgId,\n        final long timeoutMillis) throws RemotingException, InterruptedException {\n        ResumeCheckHalfMessageRequestHeader requestHeader = new ResumeCheckHalfMessageRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setMsgId(msgId);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RESUME_CHECK_HALF_MESSAGE, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return true;\n            }\n            default:\n                log.error(\"Failed to resume half message check logic. Remark={}\", response.getRemark());\n                return false;\n        }\n    }\n\n    public void setMessageRequestMode(final String brokerAddr, final String topic, final String consumerGroup,\n        final MessageRequestMode mode, final int popShareQueueNum, final long timeoutMillis)\n        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException,\n        RemotingConnectException, MQClientException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SET_MESSAGE_REQUEST_MODE, null);\n\n        SetMessageRequestModeRequestBody requestBody = new SetMessageRequestModeRequestBody();\n        requestBody.setTopic(topic);\n        requestBody.setConsumerGroup(consumerGroup);\n        requestBody.setMode(mode);\n        requestBody.setPopShareQueueNum(popShareQueueNum);\n        request.setBody(requestBody.encode());\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), brokerAddr), request, timeoutMillis);\n        assert response != null;\n        if (ResponseCode.SUCCESS != response.getCode()) {\n            throw new MQClientException(response.getCode(), response.getRemark());\n        }\n    }\n\n    public TopicConfigAndQueueMapping getTopicConfig(final String brokerAddr, String topic,\n        long timeoutMillis) throws InterruptedException,\n        RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {\n        GetTopicConfigRequestHeader header = new GetTopicConfigRequestHeader();\n        header.setTopic(topic);\n        header.setLo(true);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_CONFIG, header);\n        RemotingCommand response = this.remotingClient\n            .invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), brokerAddr), request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return RemotingSerializable.decode(response.getBody(), TopicConfigAndQueueMapping.class);\n            }\n            //should check the exist\n            case ResponseCode.TOPIC_NOT_EXIST: {\n                //should return null?\n                break;\n            }\n            default:\n                break;\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public void createStaticTopic(final String addr, final String defaultTopic, final TopicConfig topicConfig,\n        final TopicQueueMappingDetail topicQueueMappingDetail, boolean force,\n        final long timeoutMillis) throws RemotingException, InterruptedException, MQBrokerException {\n        CreateTopicRequestHeader requestHeader = new CreateTopicRequestHeader();\n        requestHeader.setTopic(topicConfig.getTopicName());\n        requestHeader.setDefaultTopic(defaultTopic);\n        requestHeader.setReadQueueNums(topicConfig.getReadQueueNums());\n        requestHeader.setWriteQueueNums(topicConfig.getWriteQueueNums());\n        requestHeader.setPerm(topicConfig.getPerm());\n        requestHeader.setTopicFilterType(topicConfig.getTopicFilterType().name());\n        requestHeader.setTopicSysFlag(topicConfig.getTopicSysFlag());\n        requestHeader.setOrder(topicConfig.isOrder());\n        requestHeader.setForce(force);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_STATIC_TOPIC, requestHeader);\n        request.setBody(topicQueueMappingDetail.encode());\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return;\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    /**\n     * @param addr\n     * @param requestHeader\n     * @param timeoutMillis\n     * @throws InterruptedException\n     * @throws RemotingTimeoutException\n     * @throws RemotingSendRequestException\n     * @throws RemotingConnectException\n     * @throws MQBrokerException\n     */\n    public GroupForbidden updateAndGetGroupForbidden(String addr, UpdateGroupForbiddenRequestHeader requestHeader,\n        long timeoutMillis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_GET_GROUP_FORBIDDEN, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request, timeoutMillis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return RemotingSerializable.decode(response.getBody(), GroupForbidden.class);\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public void resetMasterFlushOffset(final String brokerAddr, final long masterFlushOffset)\n        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {\n        ResetMasterFlushOffsetHeader requestHeader = new ResetMasterFlushOffsetHeader();\n        requestHeader.setMasterFlushOffset(masterFlushOffset);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RESET_MASTER_FLUSH_OFFSET, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, 3000);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return;\n            }\n            default:\n                break;\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr);\n    }\n\n    public HARuntimeInfo getBrokerHAStatus(final String brokerAddr, final long timeoutMillis)\n        throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException,\n        InterruptedException, MQBrokerException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_HA_STATUS, null);\n        RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis);\n        assert response != null;\n\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return HARuntimeInfo.decode(response.getBody(), HARuntimeInfo.class);\n            }\n            default:\n                break;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public GetMetaDataResponseHeader getControllerMetaData(\n        final String controllerAddress) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, RemotingCommandException, MQBrokerException {\n        final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_GET_METADATA_INFO, null);\n        final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000);\n        assert response != null;\n        if (response.getCode() == SUCCESS) {\n            return (GetMetaDataResponseHeader) response.decodeCommandCustomHeader(GetMetaDataResponseHeader.class);\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public BrokerReplicasInfo getInSyncStateData(final String controllerAddress,\n        final List<String> brokers) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException, RemotingCommandException {\n        // Get controller leader address.\n        final GetMetaDataResponseHeader controllerMetaData = getControllerMetaData(controllerAddress);\n        assert controllerMetaData != null;\n        assert controllerMetaData.getControllerLeaderAddress() != null;\n        final String leaderAddress = controllerMetaData.getControllerLeaderAddress();\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_GET_SYNC_STATE_DATA, null);\n        final byte[] body = RemotingSerializable.encode(brokers);\n        request.setBody(body);\n        RemotingCommand response = this.remotingClient.invokeSync(leaderAddress, request, 3000);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return RemotingSerializable.decode(response.getBody(), BrokerReplicasInfo.class);\n            }\n            default:\n                break;\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public EpochEntryCache getBrokerEpochCache(\n        String brokerAddr) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_EPOCH_CACHE, null);\n        final RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, 3000);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return RemotingSerializable.decode(response.getBody(), EpochEntryCache.class);\n            }\n            default:\n                break;\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public Map<String, Properties> getControllerConfig(final List<String> controllerServers,\n        final long timeoutMillis) throws InterruptedException, RemotingTimeoutException,\n        RemotingSendRequestException, RemotingConnectException, MQClientException, UnsupportedEncodingException {\n        List<String> invokeControllerServers = (controllerServers == null || controllerServers.isEmpty()) ?\n            this.remotingClient.getNameServerAddressList() : controllerServers;\n        if (invokeControllerServers == null || invokeControllerServers.isEmpty()) {\n            return null;\n        }\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONTROLLER_CONFIG, null);\n\n        Map<String, Properties> configMap = new HashMap<>(4);\n        for (String controller : invokeControllerServers) {\n            RemotingCommand response = this.remotingClient.invokeSync(controller, request, timeoutMillis);\n\n            assert response != null;\n\n            if (ResponseCode.SUCCESS == response.getCode()) {\n                configMap.put(controller, MixAll.string2Properties(new String(response.getBody(), MixAll.DEFAULT_CHARSET)));\n            } else {\n                throw new MQClientException(response.getCode(), response.getRemark());\n            }\n        }\n        return configMap;\n    }\n\n    public void updateControllerConfig(final Properties properties, final List<String> controllers,\n        final long timeoutMillis) throws InterruptedException, RemotingConnectException, UnsupportedEncodingException,\n        RemotingSendRequestException, RemotingTimeoutException, MQClientException {\n        String str = MixAll.properties2String(properties);\n        if (str.length() < 1 || controllers == null || controllers.isEmpty()) {\n            return;\n        }\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_CONTROLLER_CONFIG, null);\n        request.setBody(str.getBytes(MixAll.DEFAULT_CHARSET));\n\n        RemotingCommand errResponse = null;\n        for (String controller : controllers) {\n            RemotingCommand response = this.remotingClient.invokeSync(controller, request, timeoutMillis);\n            assert response != null;\n            switch (response.getCode()) {\n                case ResponseCode.SUCCESS: {\n                    break;\n                }\n                default:\n                    errResponse = response;\n            }\n        }\n\n        if (errResponse != null) {\n            throw new MQClientException(errResponse.getCode(), errResponse.getRemark());\n        }\n    }\n\n    public Pair<ElectMasterResponseHeader, BrokerMemberGroup> electMaster(String controllerAddr, String clusterName,\n        String brokerName,\n        Long brokerId) throws MQBrokerException, RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, RemotingCommandException {\n\n        //get controller leader address\n        final GetMetaDataResponseHeader controllerMetaData = this.getControllerMetaData(controllerAddr);\n        assert controllerMetaData != null;\n        assert controllerMetaData.getControllerLeaderAddress() != null;\n        final String leaderAddress = controllerMetaData.getControllerLeaderAddress();\n        ElectMasterRequestHeader electRequestHeader = ElectMasterRequestHeader.ofAdminTrigger(clusterName, brokerName, brokerId);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_ELECT_MASTER, electRequestHeader);\n        final RemotingCommand response = this.remotingClient.invokeSync(leaderAddress, request, 3000);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                BrokerMemberGroup brokerMemberGroup = RemotingSerializable.decode(response.getBody(), BrokerMemberGroup.class);\n                ElectMasterResponseHeader responseHeader = (ElectMasterResponseHeader) response.decodeCommandCustomHeader(ElectMasterResponseHeader.class);\n                return new Pair<>(responseHeader, brokerMemberGroup);\n            }\n            default:\n                break;\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public void cleanControllerBrokerData(String controllerAddr, String clusterName,\n        String brokerName, String brokerControllerIdsToClean, boolean isCleanLivingBroker)\n        throws RemotingException, InterruptedException, MQBrokerException {\n\n        //get controller leader address\n        final GetMetaDataResponseHeader controllerMetaData = this.getControllerMetaData(controllerAddr);\n        assert controllerMetaData != null;\n        assert controllerMetaData.getControllerLeaderAddress() != null;\n        final String leaderAddress = controllerMetaData.getControllerLeaderAddress();\n\n        CleanControllerBrokerDataRequestHeader cleanHeader = new CleanControllerBrokerDataRequestHeader(clusterName, brokerName, brokerControllerIdsToClean, isCleanLivingBroker);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CLEAN_BROKER_DATA, cleanHeader);\n\n        final RemotingCommand response = this.remotingClient.invokeSync(leaderAddress, request, 3000);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return;\n            }\n            default:\n                break;\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public void createUser(String addr, UserInfo userInfo, long millis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {\n        CreateUserRequestHeader requestHeader = new CreateUserRequestHeader(userInfo.getUsername());\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_CREATE_USER, requestHeader);\n        request.setBody(RemotingSerializable.encode(userInfo));\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, millis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return;\n            }\n            default:\n                break;\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public void updateUser(String addr, UserInfo userInfo, long millis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {\n        UpdateUserRequestHeader requestHeader = new UpdateUserRequestHeader(userInfo.getUsername());\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_UPDATE_USER, requestHeader);\n        request.setBody(RemotingSerializable.encode(userInfo));\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, millis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return;\n            }\n            default:\n                break;\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public void deleteUser(String addr, String username, long millis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {\n        DeleteUserRequestHeader requestHeader = new DeleteUserRequestHeader(username);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_DELETE_USER, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, millis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return;\n            }\n            default:\n                break;\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public UserInfo getUser(String addr, String username, long millis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {\n        GetUserRequestHeader requestHeader = new GetUserRequestHeader(username);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_GET_USER, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, millis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return RemotingSerializable.decode(response.getBody(), UserInfo.class);\n            }\n            default:\n                break;\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public List<UserInfo> listUser(String addr, String filter, long millis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {\n        ListUsersRequestHeader requestHeader = new ListUsersRequestHeader(filter);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_LIST_USER, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, millis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return RemotingSerializable.decodeList(response.getBody(), UserInfo.class);\n            }\n            default:\n                break;\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public void createAcl(String addr, AclInfo aclInfo, long millis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {\n        CreateAclRequestHeader requestHeader = new CreateAclRequestHeader(aclInfo.getSubject());\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_CREATE_ACL, requestHeader);\n        request.setBody(RemotingSerializable.encode(aclInfo));\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, millis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return;\n            }\n            default:\n                break;\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public void updateAcl(String addr, AclInfo aclInfo, long millis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {\n        UpdateAclRequestHeader requestHeader = new UpdateAclRequestHeader(aclInfo.getSubject());\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_UPDATE_ACL, requestHeader);\n        request.setBody(RemotingSerializable.encode(aclInfo));\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, millis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return;\n            }\n            default:\n                break;\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public void deleteAcl(String addr, String subject, String resource, long millis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {\n        DeleteAclRequestHeader requestHeader = new DeleteAclRequestHeader(subject, resource);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_DELETE_ACL, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, millis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return;\n            }\n            default:\n                break;\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public AclInfo getAcl(String addr, String subject, long millis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {\n        GetAclRequestHeader requestHeader = new GetAclRequestHeader(subject);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_GET_ACL, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, millis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return RemotingSerializable.decode(response.getBody(), AclInfo.class);\n            }\n            default:\n                break;\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public List<AclInfo> listAcl(String addr, String subjectFilter, String resourceFilter, long millis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {\n        ListAclsRequestHeader requestHeader = new ListAclsRequestHeader(subjectFilter, resourceFilter);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_LIST_ACL, requestHeader);\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, millis);\n        assert response != null;\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                return RemotingSerializable.decodeList(response.getBody(), AclInfo.class);\n            }\n            default:\n                break;\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public String recallMessage(\n        final String addr,\n        RecallMessageRequestHeader requestHeader,\n        final long timeoutMillis\n    ) throws RemotingException, MQBrokerException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RECALL_MESSAGE, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis);\n        switch (response.getCode()) {\n            case ResponseCode.SUCCESS: {\n                RecallMessageResponseHeader responseHeader =\n                    response.decodeCommandCustomHeader(RecallMessageResponseHeader.class);\n                return responseHeader.getMsgId();\n            }\n            default:\n                break;\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n    public void recallMessageAsync(\n        final String addr,\n        final RecallMessageRequestHeader requestHeader,\n        final long timeoutMillis,\n        final InvokeCallback invokeCallback\n    ) throws RemotingException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RECALL_MESSAGE, requestHeader);\n\n        this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {\n            @Override\n            public void operationComplete(ResponseFuture responseFuture) {\n            }\n\n            @Override\n            public void operationSucceed(RemotingCommand response) {\n                invokeCallback.operationSucceed(response);\n            }\n\n            @Override\n            public void operationFail(Throwable throwable) {\n                invokeCallback.operationFail(throwable);\n            }\n        });\n    }\n\n    public void exportPopRecord(String brokerAddr, long timeout) throws RemotingConnectException,\n        RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(\n            RequestCode.POP_ROLLBACK, null);\n        RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeout);\n        assert response != null;\n        if (response.getCode() == SUCCESS) {\n            return;\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n    public void switchTimerEngine(String brokerAddr, String engineType, long timeoutMillis) throws RemotingConnectException,\n        RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SWITCH_TIMER_ENGINE, null);\n        request.addExtField(TIMER_ENGINE_TYPE, engineType);\n        RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis);\n        assert response != null;\n        if (response.getCode() == SUCCESS) {\n            return;\n        }\n        throw new MQBrokerException(response.getCode(), response.getRemark());\n    }\n\n\n    public GetBrokerLiteInfoResponseBody getBrokerLiteInfo(String addr, long timeoutMillis)\n        throws RemotingException, MQBrokerException, InterruptedException {\n        return invokeBrokerMethod(addr, RequestCode.GET_BROKER_LITE_INFO, null,\n            GetBrokerLiteInfoResponseBody.class, timeoutMillis);\n    }\n\n    public GetParentTopicInfoResponseBody getParentTopicInfo(String addr, String topic, long timeoutMillis)\n        throws RemotingException, MQBrokerException, InterruptedException {\n        GetParentTopicInfoRequestHeader requestHeader = new GetParentTopicInfoRequestHeader();\n        requestHeader.setTopic(topic);\n        return invokeBrokerMethod(addr, RequestCode.GET_PARENT_TOPIC_INFO, requestHeader,\n            GetParentTopicInfoResponseBody.class, timeoutMillis);\n    }\n\n    public GetLiteTopicInfoResponseBody getLiteTopicInfo(String addr, String parentTopic, String liteTopic,\n        long timeoutMillis)\n        throws RemotingException, MQBrokerException, InterruptedException {\n        GetLiteTopicInfoRequestHeader requestHeader = new GetLiteTopicInfoRequestHeader();\n        requestHeader.setParentTopic(parentTopic);\n        requestHeader.setLiteTopic(liteTopic);\n        return invokeBrokerMethod(addr, RequestCode.GET_LITE_TOPIC_INFO, requestHeader,\n            GetLiteTopicInfoResponseBody.class, timeoutMillis);\n    }\n\n    public GetLiteClientInfoResponseBody getLiteClientInfo(String addr, String parentTopic, String group,\n        String clientId, long timeoutMillis)\n        throws RemotingException, MQBrokerException, InterruptedException {\n        GetLiteClientInfoRequestHeader requestHeader = new GetLiteClientInfoRequestHeader();\n        requestHeader.setParentTopic(parentTopic);\n        requestHeader.setGroup(group);\n        requestHeader.setClientId(clientId);\n        return invokeBrokerMethod(addr, RequestCode.GET_LITE_CLIENT_INFO, requestHeader,\n            GetLiteClientInfoResponseBody.class, timeoutMillis);\n    }\n\n    public GetLiteGroupInfoResponseBody getLiteGroupInfo(String addr, String group,\n        String liteTopic, int topK, long timeoutMillis)\n        throws RemotingException, MQBrokerException, InterruptedException {\n        GetLiteGroupInfoRequestHeader requestHeader = new GetLiteGroupInfoRequestHeader();\n        requestHeader.setGroup(group);\n        requestHeader.setTopK(topK);\n        requestHeader.setLiteTopic(liteTopic);\n        return invokeBrokerMethod(addr, RequestCode.GET_LITE_GROUP_INFO, requestHeader,\n            GetLiteGroupInfoResponseBody.class, timeoutMillis);\n    }\n\n    public void triggerLiteDispatch(String addr, String group, String clientId, long timeoutMillis)\n        throws RemotingException, MQBrokerException, InterruptedException {\n        TriggerLiteDispatchRequestHeader requestHeader = new TriggerLiteDispatchRequestHeader();\n        requestHeader.setGroup(group);\n        requestHeader.setClientId(clientId);\n        invokeBrokerMethod(addr, RequestCode.TRIGGER_LITE_DISPATCH, requestHeader, null, timeoutMillis);\n    }\n\n    private <T extends CommandCustomHeader, R extends RemotingSerializable> R invokeBrokerMethod(\n        final String addr,\n        final int requestCode,\n        final T requestHeader,\n        final Class<R> responseClass,\n        final long timeoutMillis\n    ) throws RemotingException, MQBrokerException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(requestCode, requestHeader);\n\n        RemotingCommand response = this.remotingClient.invokeSync(\n            MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),\n            request,\n            timeoutMillis\n        );\n\n        if (response.getCode() == SUCCESS) {\n            if (response.getBody() != null) {\n                return RemotingSerializable.decode(response.getBody(), responseClass);\n            }\n            return null;\n        }\n\n        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);\n    }\n\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/MQClientManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl;\n\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.client.producer.ProduceAccumulator;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class MQClientManager {\n    private final static Logger log = LoggerFactory.getLogger(MQClientManager.class);\n    private static MQClientManager instance = new MQClientManager();\n    private AtomicInteger factoryIndexGenerator = new AtomicInteger();\n    private ConcurrentMap<String/* clientId */, MQClientInstance> factoryTable =\n        new ConcurrentHashMap<>();\n    private ConcurrentMap<String/* clientId */, ProduceAccumulator> accumulatorTable =\n        new ConcurrentHashMap<String, ProduceAccumulator>();\n\n\n    private MQClientManager() {\n\n    }\n\n    public static MQClientManager getInstance() {\n        return instance;\n    }\n\n    public MQClientInstance getOrCreateMQClientInstance(final ClientConfig clientConfig) {\n        return getOrCreateMQClientInstance(clientConfig, null);\n    }\n    public MQClientInstance getOrCreateMQClientInstance(final ClientConfig clientConfig, RPCHook rpcHook) {\n        String clientId = clientConfig.buildMQClientId();\n        MQClientInstance instance = this.factoryTable.get(clientId);\n        if (null == instance) {\n            instance =\n                new MQClientInstance(clientConfig.cloneClientConfig(),\n                    this.factoryIndexGenerator.getAndIncrement(), clientId, rpcHook);\n            MQClientInstance prev = this.factoryTable.putIfAbsent(clientId, instance);\n            if (prev != null) {\n                instance = prev;\n                log.warn(\"Returned Previous MQClientInstance for clientId:[{}]\", clientId);\n            } else {\n                log.info(\"Created new MQClientInstance for clientId:[{}]\", clientId);\n            }\n        }\n\n        return instance;\n    }\n    public ProduceAccumulator getOrCreateProduceAccumulator(final ClientConfig clientConfig) {\n        String clientId = clientConfig.buildMQClientId();\n        ProduceAccumulator accumulator = this.accumulatorTable.get(clientId);\n        if (null == accumulator) {\n            accumulator = new ProduceAccumulator(clientId);\n            ProduceAccumulator prev = this.accumulatorTable.putIfAbsent(clientId, accumulator);\n            if (prev != null) {\n                accumulator = prev;\n                log.warn(\"Returned Previous ProduceAccumulator for clientId:[{}]\", clientId);\n            } else {\n                log.info(\"Created new ProduceAccumulator for clientId:[{}]\", clientId);\n            }\n        }\n\n        return accumulator;\n    }\n\n    public void removeClientFactory(final String clientId) {\n        this.factoryTable.remove(clientId);\n    }\n\n    public ConcurrentMap<String, MQClientInstance> getFactoryTable() {\n        return factoryTable;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/admin/MqClientAdminImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.impl.admin;\n\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.stream.Collectors;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.MqClientAdmin;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.RemotingClient;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.admin.ConsumeStats;\nimport org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;\nimport org.apache.rocketmq.remoting.protocol.body.ClusterInfo;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerConnection;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.body.GroupList;\nimport org.apache.rocketmq.remoting.protocol.body.QueryConsumeTimeSpanBody;\nimport org.apache.rocketmq.remoting.protocol.body.QuerySubscriptionResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan;\nimport org.apache.rocketmq.remoting.protocol.body.ResetOffsetBody;\nimport org.apache.rocketmq.remoting.protocol.body.TopicList;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.CreateTopicRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.DeleteSubscriptionGroupRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.DeleteTopicRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumeStatsRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerConnectionListRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetTopicStatsInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumeTimeSpanRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QuerySubscriptionByConsumerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryTopicConsumeByWhoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryTopicsByConsumerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ViewMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.DeleteKVConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.DeleteTopicFromNamesrvRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\n\npublic class MqClientAdminImpl implements MqClientAdmin {\n    private final static Logger log = LoggerFactory.getLogger(MqClientAdminImpl.class);\n    private final RemotingClient remotingClient;\n\n    public MqClientAdminImpl(RemotingClient remotingClient) {\n        this.remotingClient = remotingClient;\n    }\n\n    @Override\n    public CompletableFuture<List<MessageExt>> queryMessage(String address, boolean uniqueKeyFlag, boolean decompressBody,\n        QueryMessageRequestHeader requestHeader, long timeoutMillis) {\n        CompletableFuture<List<MessageExt>> future = new CompletableFuture<>();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_MESSAGE, requestHeader);\n        request.addExtField(MixAll.UNIQUE_MSG_QUERY_FLAG, String.valueOf(uniqueKeyFlag));\n        remotingClient.invoke(address, request, timeoutMillis).thenAccept(response -> {\n            if (response.getCode() == ResponseCode.SUCCESS) {\n                List<MessageExt> wrappers = MessageDecoder.decodesBatch(ByteBuffer.wrap(response.getBody()), true, decompressBody, true);\n                future.complete(filterMessages(wrappers, requestHeader.getTopic(), requestHeader.getKey(), uniqueKeyFlag));\n            } else if (response.getCode() == ResponseCode.QUERY_NOT_FOUND)  {\n                List<MessageExt> wrappers = new ArrayList<>();\n                future.complete(wrappers);\n            } else {\n                log.warn(\"queryMessage getResponseCommand failed, {} {}, header={}\", response.getCode(), response.getRemark(), requestHeader);\n                future.completeExceptionally(new MQClientException(response.getCode(), response.getRemark()));\n            }\n        });\n\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<TopicStatsTable> getTopicStatsInfo(String address,\n        GetTopicStatsInfoRequestHeader requestHeader, long timeoutMillis) {\n        CompletableFuture<TopicStatsTable> future = new CompletableFuture<>();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_STATS_INFO, requestHeader);\n        remotingClient.invoke(address, request, timeoutMillis).thenAccept(response -> {\n            if (response.getCode() == ResponseCode.SUCCESS) {\n                TopicStatsTable topicStatsTable = TopicStatsTable.decode(response.getBody(), TopicStatsTable.class);\n                future.complete(topicStatsTable);\n            } else {\n                log.warn(\"getTopicStatsInfo getResponseCommand failed, {} {}, header={}\", response.getCode(), response.getRemark(), requestHeader);\n                future.completeExceptionally(new MQClientException(response.getCode(), response.getRemark()));\n            }\n        });\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<List<QueueTimeSpan>> queryConsumeTimeSpan(String address,\n        QueryConsumeTimeSpanRequestHeader requestHeader, long timeoutMillis) {\n        CompletableFuture<List<QueueTimeSpan>> future = new CompletableFuture<>();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUME_TIME_SPAN, requestHeader);\n        remotingClient.invoke(address, request, timeoutMillis).thenAccept(response -> {\n            if (response.getCode() == ResponseCode.SUCCESS) {\n                QueryConsumeTimeSpanBody consumeTimeSpanBody = GroupList.decode(response.getBody(), QueryConsumeTimeSpanBody.class);\n                future.complete(consumeTimeSpanBody.getConsumeTimeSpanSet());\n            } else {\n                log.warn(\"queryConsumerTimeSpan getResponseCommand failed, {} {}, header={}\", response.getCode(), response.getRemark(), requestHeader);\n                future.completeExceptionally(new MQClientException(response.getCode(), response.getRemark()));\n            }\n        });\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<Void> updateOrCreateTopic(String address, CreateTopicRequestHeader requestHeader,\n        long timeoutMillis) {\n        CompletableFuture<Void> future = new CompletableFuture<>();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_TOPIC, requestHeader);\n        remotingClient.invoke(address, request, timeoutMillis).thenAccept(response -> {\n            if (response.getCode() == ResponseCode.SUCCESS) {\n                future.complete(null);\n            } else {\n                log.warn(\"updateOrCreateTopic getResponseCommand failed, {} {}, header={}\", response.getCode(), response.getRemark(), requestHeader);\n                future.completeExceptionally(new MQClientException(response.getCode(), response.getRemark()));\n            }\n        });\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<Void> updateOrCreateSubscriptionGroup(String address, SubscriptionGroupConfig config,\n        long timeoutMillis) {\n        CompletableFuture<Void> future = new CompletableFuture<>();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP, null);\n        byte[] body = RemotingSerializable.encode(config);\n        request.setBody(body);\n        remotingClient.invoke(address, request, timeoutMillis).thenAccept(response -> {\n            if (response.getCode() == ResponseCode.SUCCESS) {\n                future.complete(null);\n            } else {\n                log.warn(\"updateOrCreateSubscriptionGroup getResponseCommand failed, {} {}, header={}\", response.getCode(), response.getRemark(), config);\n                future.completeExceptionally(new MQClientException(response.getCode(), response.getRemark()));\n            }\n        });\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<Void> deleteTopicInBroker(String address, DeleteTopicRequestHeader requestHeader,\n        long timeoutMillis) {\n        CompletableFuture<Void> future = new CompletableFuture<>();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_TOPIC_IN_BROKER, requestHeader);\n        remotingClient.invoke(address, request, timeoutMillis).thenAccept(response -> {\n            if (response.getCode() == ResponseCode.SUCCESS) {\n                future.complete(null);\n            } else {\n                log.warn(\"deleteTopicInBroker getResponseCommand failed, {} {}, header={}\", response.getCode(), response.getRemark(), requestHeader);\n                future.completeExceptionally(new MQClientException(response.getCode(), response.getRemark()));\n            }\n        });\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<Void> deleteTopicInNameserver(String address, DeleteTopicFromNamesrvRequestHeader requestHeader,\n        long timeoutMillis) {\n        CompletableFuture<Void> future = new CompletableFuture<>();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_TOPIC_IN_NAMESRV, requestHeader);\n        remotingClient.invoke(address, request, timeoutMillis).thenAccept(response -> {\n            if (response.getCode() == ResponseCode.SUCCESS) {\n                future.complete(null);\n            } else {\n                log.warn(\"deleteTopicInNameserver getResponseCommand failed, {} {}, header={}\", response.getCode(), response.getRemark(), requestHeader);\n                future.completeExceptionally(new MQClientException(response.getCode(), response.getRemark()));\n            }\n        });\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<Void> deleteKvConfig(String address, DeleteKVConfigRequestHeader requestHeader,\n        long timeoutMillis) {\n        CompletableFuture<Void> future = new CompletableFuture<>();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_KV_CONFIG, requestHeader);\n        remotingClient.invoke(address, request, timeoutMillis).thenAccept(response -> {\n            if (response.getCode() == ResponseCode.SUCCESS) {\n                future.complete(null);\n            } else {\n                log.warn(\"deleteKvConfig getResponseCommand failed, {} {}, header={}\", response.getCode(), response.getRemark(), requestHeader);\n                future.completeExceptionally(new MQClientException(response.getCode(), response.getRemark()));\n            }\n        });\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<Void> deleteSubscriptionGroup(String address, DeleteSubscriptionGroupRequestHeader requestHeader,\n        long timeoutMillis) {\n        CompletableFuture<Void> future = new CompletableFuture<>();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_SUBSCRIPTIONGROUP, requestHeader);\n        remotingClient.invoke(address, request, timeoutMillis).thenAccept(response -> {\n            if (response.getCode() == ResponseCode.SUCCESS) {\n                future.complete(null);\n            } else {\n                log.warn(\"deleteSubscriptionGroup getResponseCommand failed, {} {}, header={}\", response.getCode(), response.getRemark(), requestHeader);\n                future.completeExceptionally(new MQClientException(response.getCode(), response.getRemark()));\n            }\n        });\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<Map<MessageQueue, Long>> invokeBrokerToResetOffset(String address,\n        ResetOffsetRequestHeader requestHeader, long timeoutMillis) {\n        CompletableFuture<Map<MessageQueue, Long>> future = new CompletableFuture<>();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.INVOKE_BROKER_TO_RESET_OFFSET, requestHeader);\n        remotingClient.invoke(address, request, timeoutMillis).thenAccept(response -> {\n            if (response.getCode() == ResponseCode.SUCCESS && null != response.getBody()) {\n                Map<MessageQueue, Long> offsetTable = ResetOffsetBody.decode(response.getBody(), ResetOffsetBody.class).getOffsetTable();\n                future.complete(offsetTable);\n                log.info(\"Invoke broker to reset offset success. address:{}, header:{}, offsetTable:{}\",\n                    address, requestHeader, offsetTable);\n            } else {\n                log.warn(\"invokeBrokerToResetOffset getResponseCommand failed, {} {}, header={}\", response.getCode(), response.getRemark(), requestHeader);\n                future.completeExceptionally(new MQClientException(response.getCode(), response.getRemark()));\n            }\n        });\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<MessageExt> viewMessage(String address, ViewMessageRequestHeader requestHeader,\n        long timeoutMillis) {\n        CompletableFuture<MessageExt> future = new CompletableFuture<>();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.VIEW_MESSAGE_BY_ID, requestHeader);\n        remotingClient.invoke(address, request, timeoutMillis).thenAccept(response -> {\n            if (response.getCode() == ResponseCode.SUCCESS) {\n                ByteBuffer byteBuffer = ByteBuffer.wrap(response.getBody());\n                MessageExt messageExt = MessageDecoder.clientDecode(byteBuffer, true);\n                future.complete(messageExt);\n            } else {\n                log.warn(\"viewMessage getResponseCommand failed, {} {}, header={}\", response.getCode(), response.getRemark(), requestHeader);\n                future.completeExceptionally(new MQClientException(response.getCode(), response.getRemark()));\n            }\n        });\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<ClusterInfo> getBrokerClusterInfo(String address, long timeoutMillis) {\n        CompletableFuture<ClusterInfo> future = new CompletableFuture<>();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_CLUSTER_INFO, null);\n        remotingClient.invoke(address, request, timeoutMillis).thenAccept(response -> {\n            if (response.getCode() == ResponseCode.SUCCESS) {\n                ClusterInfo clusterInfo = ClusterInfo.decode(response.getBody(), ClusterInfo.class);\n                future.complete(clusterInfo);\n            } else {\n                log.warn(\"getBrokerClusterInfo getResponseCommand failed, {} {}\", response.getCode(), response.getRemark());\n                future.completeExceptionally(new MQClientException(response.getCode(), response.getRemark()));\n            }\n        });\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<ConsumerConnection> getConsumerConnectionList(String address,\n        GetConsumerConnectionListRequestHeader requestHeader, long timeoutMillis) {\n        CompletableFuture<ConsumerConnection> future = new CompletableFuture<>();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_CONNECTION_LIST, requestHeader);\n        remotingClient.invoke(address, request, timeoutMillis).thenAccept(response -> {\n            if (response.getCode() == ResponseCode.SUCCESS) {\n                ConsumerConnection consumerConnection = ConsumerConnection.decode(response.getBody(), ConsumerConnection.class);\n                future.complete(consumerConnection);\n            } else {\n                log.warn(\"getConsumerConnectionList getResponseCommand failed, {} {}\", response.getCode(), response.getRemark());\n                future.completeExceptionally(new MQClientException(response.getCode(), response.getRemark()));\n            }\n        });\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<TopicList> queryTopicsByConsumer(String address,\n        QueryTopicsByConsumerRequestHeader requestHeader, long timeoutMillis) {\n        CompletableFuture<TopicList> future = new CompletableFuture<>();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_TOPICS_BY_CONSUMER, requestHeader);\n        remotingClient.invoke(address, request, timeoutMillis).thenAccept(response -> {\n            if (response.getCode() == ResponseCode.SUCCESS) {\n                TopicList topicList = TopicList.decode(response.getBody(), TopicList.class);\n                future.complete(topicList);\n            } else {\n                log.warn(\"queryTopicsByConsumer getResponseCommand failed, {} {}\", response.getCode(), response.getRemark());\n                future.completeExceptionally(new MQClientException(response.getCode(), response.getRemark()));\n            }\n        });\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<SubscriptionData> querySubscriptionByConsumer(String address,\n        QuerySubscriptionByConsumerRequestHeader requestHeader, long timeoutMillis) {\n        CompletableFuture<SubscriptionData> future = new CompletableFuture<>();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_SUBSCRIPTION_BY_CONSUMER, requestHeader);\n        remotingClient.invoke(address, request, timeoutMillis).thenAccept(response -> {\n            if (response.getCode() == ResponseCode.SUCCESS) {\n                QuerySubscriptionResponseBody subscriptionResponseBody =\n                    QuerySubscriptionResponseBody.decode(response.getBody(), QuerySubscriptionResponseBody.class);\n                future.complete(subscriptionResponseBody.getSubscriptionData());\n            } else {\n                log.warn(\"querySubscriptionByConsumer getResponseCommand failed, {} {}\", response.getCode(), response.getRemark());\n                future.completeExceptionally(new MQClientException(response.getCode(), response.getRemark()));\n            }\n        });\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<ConsumeStats> getConsumeStats(String address, GetConsumeStatsRequestHeader requestHeader,\n        long timeoutMillis) {\n        CompletableFuture<ConsumeStats> future = new CompletableFuture<>();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUME_STATS, requestHeader);\n        remotingClient.invoke(address, request, timeoutMillis).thenAccept(response -> {\n            if (response.getCode() == ResponseCode.SUCCESS) {\n                ConsumeStats consumeStats = ConsumeStats.decode(response.getBody(), ConsumeStats.class);\n                future.complete(consumeStats);\n            } else {\n                log.warn(\"getConsumeStats getResponseCommand failed, {} {}\", response.getCode(), response.getRemark());\n                future.completeExceptionally(new MQClientException(response.getCode(), response.getRemark()));\n            }\n        });\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<GroupList> queryTopicConsumeByWho(String address,\n        QueryTopicConsumeByWhoRequestHeader requestHeader, long timeoutMillis) {\n        CompletableFuture<GroupList> future = new CompletableFuture<>();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_TOPIC_CONSUME_BY_WHO, requestHeader);\n        remotingClient.invoke(address, request, timeoutMillis).thenAccept(response -> {\n            if (response.getCode() == ResponseCode.SUCCESS) {\n                GroupList groupList = GroupList.decode(response.getBody(), GroupList.class);\n                future.complete(groupList);\n            } else {\n                log.warn(\"queryTopicConsumeByWho getResponseCommand failed, {} {}\", response.getCode(), response.getRemark());\n                future.completeExceptionally(new MQClientException(response.getCode(), response.getRemark()));\n            }\n        });\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<ConsumerRunningInfo> getConsumerRunningInfo(String address,\n        GetConsumerRunningInfoRequestHeader requestHeader, long timeoutMillis) {\n        CompletableFuture<ConsumerRunningInfo> future = new CompletableFuture<>();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_RUNNING_INFO, requestHeader);\n        remotingClient.invoke(address, request, timeoutMillis).thenAccept(response -> {\n            if (response.getCode() == ResponseCode.SUCCESS) {\n                ConsumerRunningInfo info = ConsumerRunningInfo.decode(response.getBody(), ConsumerRunningInfo.class);\n                future.complete(info);\n            } else {\n                log.warn(\"getConsumerRunningInfo getResponseCommand failed, {} {}\", response.getCode(), response.getRemark());\n                future.completeExceptionally(new MQClientException(response.getCode(), response.getRemark()));\n            }\n        });\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<ConsumeMessageDirectlyResult> consumeMessageDirectly(String address,\n        ConsumeMessageDirectlyResultRequestHeader requestHeader, long timeoutMillis) {\n        CompletableFuture<ConsumeMessageDirectlyResult> future = new CompletableFuture<>();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONSUME_MESSAGE_DIRECTLY, requestHeader);\n        remotingClient.invoke(address, request, timeoutMillis).thenAccept(response -> {\n            if (response.getCode() == ResponseCode.SUCCESS) {\n                ConsumeMessageDirectlyResult info = ConsumeMessageDirectlyResult.decode(response.getBody(), ConsumeMessageDirectlyResult.class);\n                future.complete(info);\n            } else {\n                log.warn(\"consumeMessageDirectly getResponseCommand failed, {} {}\", response.getCode(), response.getRemark());\n                future.completeExceptionally(new MQClientException(response.getCode(), response.getRemark()));\n            }\n        });\n        return future;\n    }\n\n    private List<MessageExt> filterMessages(List<MessageExt> messageFoundList, String topic, String key,\n        boolean uniqueKeyFlag) {\n        List<MessageExt> matchedMessages = new ArrayList<>();\n        if (uniqueKeyFlag) {\n            matchedMessages.addAll(messageFoundList.stream()\n                .filter(msg -> topic.equals(msg.getTopic()))\n                .filter(msg -> key.equals(msg.getMsgId()))\n                .collect(Collectors.toList())\n            );\n        } else {\n            matchedMessages.addAll(messageFoundList.stream()\n                .filter(msg -> topic.equals(msg.getTopic()))\n                .filter(msg -> {\n                    boolean matched = false;\n                    if (StringUtils.isNotBlank(msg.getKeys())) {\n                        String[] keyArray = msg.getKeys().split(MessageConst.KEY_SEPARATOR);\n                        for (String s : keyArray) {\n                            if (key.equals(s)) {\n                                matched = true;\n                                break;\n                            }\n                        }\n                    }\n\n                    return matched;\n                }).collect(Collectors.toList()));\n        }\n\n        return matchedMessages;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport java.util.Collection;\nimport java.util.Iterator;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\npublic class AssignedMessageQueue {\n\n    private final ConcurrentHashMap<MessageQueue, MessageQueueState> assignedMessageQueueState;\n\n    private RebalanceImpl rebalanceImpl;\n\n    public AssignedMessageQueue() {\n        assignedMessageQueueState = new ConcurrentHashMap<>();\n    }\n\n    public void setRebalanceImpl(RebalanceImpl rebalanceImpl) {\n        this.rebalanceImpl = rebalanceImpl;\n    }\n\n    public boolean isPaused(MessageQueue messageQueue) {\n        MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue);\n        if (messageQueueState != null) {\n            return messageQueueState.isPaused();\n        }\n        return true;\n    }\n\n    public void pause(Collection<MessageQueue> messageQueues) {\n        for (MessageQueue messageQueue : messageQueues) {\n            MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue);\n            if (assignedMessageQueueState.get(messageQueue) != null) {\n                messageQueueState.setPaused(true);\n            }\n        }\n    }\n\n    public void resume(Collection<MessageQueue> messageQueueCollection) {\n        for (MessageQueue messageQueue : messageQueueCollection) {\n            MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue);\n            if (assignedMessageQueueState.get(messageQueue) != null) {\n                messageQueueState.setPaused(false);\n            }\n        }\n    }\n\n    public ProcessQueue getProcessQueue(MessageQueue messageQueue) {\n        MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue);\n        if (messageQueueState != null) {\n            return messageQueueState.getProcessQueue();\n        }\n        return null;\n    }\n\n    public long getPullOffset(MessageQueue messageQueue) {\n        MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue);\n        if (messageQueueState != null) {\n            return messageQueueState.getPullOffset();\n        }\n        return -1;\n    }\n\n    public void updatePullOffset(MessageQueue messageQueue, long offset, ProcessQueue processQueue) {\n        MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue);\n        if (messageQueueState != null) {\n            if (messageQueueState.getProcessQueue() != processQueue) {\n                return;\n            }\n            messageQueueState.setPullOffset(offset);\n        }\n    }\n\n    public long getConsumerOffset(MessageQueue messageQueue) {\n        MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue);\n        if (messageQueueState != null) {\n            return messageQueueState.getConsumeOffset();\n        }\n        return -1;\n    }\n\n    public void updateConsumeOffset(MessageQueue messageQueue, long offset) {\n        MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue);\n        if (messageQueueState != null) {\n            messageQueueState.setConsumeOffset(offset);\n        }\n    }\n\n    public void setSeekOffset(MessageQueue messageQueue, long offset) {\n        MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue);\n        if (messageQueueState != null) {\n            messageQueueState.setSeekOffset(offset);\n        }\n    }\n\n    public long getSeekOffset(MessageQueue messageQueue) {\n        MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue);\n        if (messageQueueState != null) {\n            return messageQueueState.getSeekOffset();\n        }\n        return -1;\n    }\n\n    public void updateAssignedMessageQueue(String topic, Collection<MessageQueue> assigned) {\n        synchronized (this.assignedMessageQueueState) {\n            Iterator<Map.Entry<MessageQueue, MessageQueueState>> it = this.assignedMessageQueueState.entrySet().iterator();\n            while (it.hasNext()) {\n                Map.Entry<MessageQueue, MessageQueueState> next = it.next();\n                if (next.getKey().getTopic().equals(topic)) {\n                    if (!assigned.contains(next.getKey())) {\n                        next.getValue().getProcessQueue().setDropped(true);\n                        it.remove();\n                    }\n                }\n            }\n            addAssignedMessageQueue(assigned);\n        }\n    }\n\n    public void updateAssignedMessageQueue(Collection<MessageQueue> assigned) {\n        synchronized (this.assignedMessageQueueState) {\n            Iterator<Map.Entry<MessageQueue, MessageQueueState>> it = this.assignedMessageQueueState.entrySet().iterator();\n            while (it.hasNext()) {\n                Map.Entry<MessageQueue, MessageQueueState> next = it.next();\n                if (!assigned.contains(next.getKey())) {\n                    next.getValue().getProcessQueue().setDropped(true);\n                    it.remove();\n                }\n            }\n            addAssignedMessageQueue(assigned);\n        }\n    }\n\n    private void addAssignedMessageQueue(Collection<MessageQueue> assigned) {\n        for (MessageQueue messageQueue : assigned) {\n            if (!this.assignedMessageQueueState.containsKey(messageQueue)) {\n                MessageQueueState messageQueueState;\n                if (rebalanceImpl != null && rebalanceImpl.getProcessQueueTable().get(messageQueue) != null) {\n                    messageQueueState = new MessageQueueState(messageQueue, rebalanceImpl.getProcessQueueTable().get(messageQueue));\n                } else {\n                    ProcessQueue processQueue = new ProcessQueue();\n                    messageQueueState = new MessageQueueState(messageQueue, processQueue);\n                }\n                this.assignedMessageQueueState.put(messageQueue, messageQueueState);\n            }\n        }\n    }\n\n    public void removeAssignedMessageQueue(String topic) {\n        synchronized (this.assignedMessageQueueState) {\n            Iterator<Map.Entry<MessageQueue, MessageQueueState>> it = this.assignedMessageQueueState.entrySet().iterator();\n            while (it.hasNext()) {\n                Map.Entry<MessageQueue, MessageQueueState> next = it.next();\n                if (next.getKey().getTopic().equals(topic)) {\n                    it.remove();\n                }\n            }\n        }\n    }\n\n    public Set<MessageQueue> getAssignedMessageQueues() {\n        return this.assignedMessageQueueState.keySet();\n    }\n\n    private class MessageQueueState {\n        private MessageQueue messageQueue;\n        private ProcessQueue processQueue;\n        private volatile boolean paused = false;\n        private volatile long pullOffset = -1;\n        private volatile long consumeOffset = -1;\n        private volatile long seekOffset = -1;\n\n        private MessageQueueState(MessageQueue messageQueue, ProcessQueue processQueue) {\n            this.messageQueue = messageQueue;\n            this.processQueue = processQueue;\n        }\n\n        public MessageQueue getMessageQueue() {\n            return messageQueue;\n        }\n\n        public void setMessageQueue(MessageQueue messageQueue) {\n            this.messageQueue = messageQueue;\n        }\n\n        public boolean isPaused() {\n            return paused;\n        }\n\n        public void setPaused(boolean paused) {\n            this.paused = paused;\n        }\n\n        public long getPullOffset() {\n            return pullOffset;\n        }\n\n        public void setPullOffset(long pullOffset) {\n            this.pullOffset = pullOffset;\n        }\n\n        public ProcessQueue getProcessQueue() {\n            return processQueue;\n        }\n\n        public void setProcessQueue(ProcessQueue processQueue) {\n            this.processQueue = processQueue;\n        }\n\n        public long getConsumeOffset() {\n            return consumeOffset;\n        }\n\n        public void setConsumeOffset(long consumeOffset) {\n            this.consumeOffset = consumeOffset;\n        }\n\n        public long getSeekOffset() {\n            return seekOffset;\n        }\n\n        public void setSeekOffset(long seekOffset) {\n            this.seekOffset = seekOffset;\n        }\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.RejectedExecutionException;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeReturnType;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.hook.ConsumeMessageContext;\nimport org.apache.rocketmq.client.stat.ConsumerStatsManager;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.remoting.protocol.body.CMResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class ConsumeMessageConcurrentlyService implements ConsumeMessageService {\n    private static final Logger log = LoggerFactory.getLogger(ConsumeMessageConcurrentlyService.class);\n    private final DefaultMQPushConsumerImpl defaultMQPushConsumerImpl;\n    private final DefaultMQPushConsumer defaultMQPushConsumer;\n    private final MessageListenerConcurrently messageListener;\n    private final BlockingQueue<Runnable> consumeRequestQueue;\n    private final ThreadPoolExecutor consumeExecutor;\n    private final String consumerGroup;\n\n    private final ScheduledExecutorService scheduledExecutorService;\n    private final ScheduledExecutorService cleanExpireMsgExecutors;\n\n    public ConsumeMessageConcurrentlyService(DefaultMQPushConsumerImpl defaultMQPushConsumerImpl,\n        MessageListenerConcurrently messageListener) {\n        this.defaultMQPushConsumerImpl = defaultMQPushConsumerImpl;\n        this.messageListener = messageListener;\n\n        this.defaultMQPushConsumer = this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer();\n        this.consumerGroup = this.defaultMQPushConsumer.getConsumerGroup();\n        this.consumeRequestQueue = new LinkedBlockingQueue<>();\n\n        String consumerGroupTag = (consumerGroup.length() > 100 ? consumerGroup.substring(0, 100) : consumerGroup) + \"_\";\n        this.consumeExecutor = new ThreadPoolExecutor(\n            this.defaultMQPushConsumer.getConsumeThreadMin(),\n            this.defaultMQPushConsumer.getConsumeThreadMax(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            this.consumeRequestQueue,\n            new ThreadFactoryImpl(\"ConsumeMessageThread_\" + consumerGroupTag));\n\n        this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(\"ConsumeMessageScheduledThread_\" + consumerGroupTag));\n        this.cleanExpireMsgExecutors = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(\"CleanExpireMsgScheduledThread_\" + consumerGroupTag));\n    }\n\n    public void start() {\n        this.cleanExpireMsgExecutors.scheduleAtFixedRate(new Runnable() {\n\n            @Override\n            public void run() {\n                try {\n                    cleanExpireMsg();\n                } catch (Throwable e) {\n                    log.error(\"scheduleAtFixedRate cleanExpireMsg exception\", e);\n                }\n            }\n\n        }, this.defaultMQPushConsumer.getConsumeTimeout(), this.defaultMQPushConsumer.getConsumeTimeout(), TimeUnit.MINUTES);\n    }\n\n    public void shutdown(long awaitTerminateMillis) {\n        this.scheduledExecutorService.shutdown();\n        ThreadUtils.shutdownGracefully(this.consumeExecutor, awaitTerminateMillis, TimeUnit.MILLISECONDS);\n        this.cleanExpireMsgExecutors.shutdown();\n    }\n\n    @Override\n    public void updateCorePoolSize(int corePoolSize) {\n        if (corePoolSize > 0\n            && corePoolSize <= Short.MAX_VALUE\n            && corePoolSize < this.defaultMQPushConsumer.getConsumeThreadMax()) {\n            this.consumeExecutor.setCorePoolSize(corePoolSize);\n        }\n    }\n\n    @Override\n    public void incCorePoolSize() {\n\n    }\n\n    @Override\n    public void decCorePoolSize() {\n\n    }\n\n    @Override\n    public int getCorePoolSize() {\n        return this.consumeExecutor.getCorePoolSize();\n    }\n\n    @Override\n    public ConsumeMessageDirectlyResult consumeMessageDirectly(MessageExt msg, String brokerName) {\n        ConsumeMessageDirectlyResult result = new ConsumeMessageDirectlyResult();\n        result.setOrder(false);\n        result.setAutoCommit(true);\n\n        msg.setBrokerName(brokerName);\n        List<MessageExt> msgs = new ArrayList<>();\n        msgs.add(msg);\n        MessageQueue mq = new MessageQueue();\n        mq.setBrokerName(brokerName);\n        mq.setTopic(msg.getTopic());\n        mq.setQueueId(msg.getQueueId());\n\n        ConsumeConcurrentlyContext context = new ConsumeConcurrentlyContext(mq);\n\n        this.defaultMQPushConsumerImpl.resetRetryAndNamespace(msgs, this.consumerGroup);\n\n        final long beginTime = System.currentTimeMillis();\n\n        log.info(\"consumeMessageDirectly receive new message: {}\", msg);\n\n        try {\n            ConsumeConcurrentlyStatus status = this.messageListener.consumeMessage(msgs, context);\n            if (status != null) {\n                switch (status) {\n                    case CONSUME_SUCCESS:\n                        result.setConsumeResult(CMResult.CR_SUCCESS);\n                        break;\n                    case RECONSUME_LATER:\n                        result.setConsumeResult(CMResult.CR_LATER);\n                        break;\n                    default:\n                        break;\n                }\n            } else {\n                result.setConsumeResult(CMResult.CR_RETURN_NULL);\n            }\n        } catch (Throwable e) {\n            result.setConsumeResult(CMResult.CR_THROW_EXCEPTION);\n            result.setRemark(UtilAll.exceptionSimpleDesc(e));\n\n            log.warn(\"consumeMessageDirectly exception: {} Group: {} Msgs: {} MQ: {}\",\n                UtilAll.exceptionSimpleDesc(e),\n                ConsumeMessageConcurrentlyService.this.consumerGroup,\n                msgs,\n                mq, e);\n        }\n\n        result.setSpentTimeMills(System.currentTimeMillis() - beginTime);\n\n        log.info(\"consumeMessageDirectly Result: {}\", result);\n\n        return result;\n    }\n\n    @Override\n    public void submitConsumeRequest(\n        final List<MessageExt> msgs,\n        final ProcessQueue processQueue,\n        final MessageQueue messageQueue,\n        final boolean dispatchToConsume) {\n        final int consumeBatchSize = this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize();\n        if (msgs.size() <= consumeBatchSize) {\n            ConsumeRequest consumeRequest = new ConsumeRequest(msgs, processQueue, messageQueue);\n            try {\n                this.consumeExecutor.submit(consumeRequest);\n            } catch (RejectedExecutionException e) {\n                this.submitConsumeRequestLater(consumeRequest);\n            }\n        } else {\n            for (int total = 0; total < msgs.size(); ) {\n                List<MessageExt> msgThis = new ArrayList<>(consumeBatchSize);\n                for (int i = 0; i < consumeBatchSize; i++, total++) {\n                    if (total < msgs.size()) {\n                        msgThis.add(msgs.get(total));\n                    } else {\n                        break;\n                    }\n                }\n\n                ConsumeRequest consumeRequest = new ConsumeRequest(msgThis, processQueue, messageQueue);\n                try {\n                    this.consumeExecutor.submit(consumeRequest);\n                } catch (RejectedExecutionException e) {\n                    for (; total < msgs.size(); total++) {\n                        msgThis.add(msgs.get(total));\n                    }\n\n                    this.submitConsumeRequestLater(consumeRequest);\n                }\n            }\n        }\n    }\n\n    @Override\n    public void submitPopConsumeRequest(final List<MessageExt> msgs,\n        final PopProcessQueue processQueue,\n        final MessageQueue messageQueue) {\n        throw new UnsupportedOperationException();\n    }\n\n    private void cleanExpireMsg() {\n        Iterator<Map.Entry<MessageQueue, ProcessQueue>> it =\n            this.defaultMQPushConsumerImpl.getRebalanceImpl().getProcessQueueTable().entrySet().iterator();\n        while (it.hasNext()) {\n            Map.Entry<MessageQueue, ProcessQueue> next = it.next();\n            ProcessQueue pq = next.getValue();\n            pq.cleanExpiredMsg(this.defaultMQPushConsumer);\n        }\n    }\n\n    public void processConsumeResult(\n        final ConsumeConcurrentlyStatus status,\n        final ConsumeConcurrentlyContext context,\n        final ConsumeRequest consumeRequest\n    ) {\n        int ackIndex = context.getAckIndex();\n\n        if (consumeRequest.getMsgs().isEmpty())\n            return;\n\n        switch (status) {\n            case CONSUME_SUCCESS:\n                if (ackIndex >= consumeRequest.getMsgs().size()) {\n                    ackIndex = consumeRequest.getMsgs().size() - 1;\n                }\n                int ok = ackIndex + 1;\n                int failed = consumeRequest.getMsgs().size() - ok;\n                this.getConsumerStatsManager().incConsumeOKTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), ok);\n                this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), failed);\n                break;\n            case RECONSUME_LATER:\n                ackIndex = -1;\n                this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(),\n                    consumeRequest.getMsgs().size());\n                break;\n            default:\n                break;\n        }\n\n        switch (this.defaultMQPushConsumer.getMessageModel()) {\n            case BROADCASTING:\n                for (int i = ackIndex + 1; i < consumeRequest.getMsgs().size(); i++) {\n                    MessageExt msg = consumeRequest.getMsgs().get(i);\n                    log.warn(\"BROADCASTING, the message consume failed, drop it, {}\", msg.toString());\n                }\n                break;\n            case CLUSTERING:\n                List<MessageExt> msgBackFailed = new ArrayList<>(consumeRequest.getMsgs().size());\n                for (int i = ackIndex + 1; i < consumeRequest.getMsgs().size(); i++) {\n                    MessageExt msg = consumeRequest.getMsgs().get(i);\n                    // Maybe message is expired and cleaned, just ignore it.\n                    if (!consumeRequest.getProcessQueue().containsMessage(msg)) {\n                        log.info(\"Message is not found in its process queue; skip send-back-procedure, topic={}, \"\n                                + \"brokerName={}, queueId={}, queueOffset={}\", msg.getTopic(), msg.getBrokerName(),\n                            msg.getQueueId(), msg.getQueueOffset());\n                        continue;\n                    }\n                    boolean result = this.sendMessageBack(msg, context);\n                    if (!result) {\n                        msg.setReconsumeTimes(msg.getReconsumeTimes() + 1);\n                        msgBackFailed.add(msg);\n                    }\n                }\n\n                if (!msgBackFailed.isEmpty()) {\n                    consumeRequest.getMsgs().removeAll(msgBackFailed);\n\n                    this.submitConsumeRequestLater(msgBackFailed, consumeRequest.getProcessQueue(), consumeRequest.getMessageQueue());\n                }\n                break;\n            default:\n                break;\n        }\n\n        long offset = consumeRequest.getProcessQueue().removeMessage(consumeRequest.getMsgs());\n        if (offset >= 0 && !consumeRequest.getProcessQueue().isDropped()) {\n            this.defaultMQPushConsumerImpl.getOffsetStore().updateOffset(consumeRequest.getMessageQueue(), offset, true);\n        }\n    }\n\n    public ConsumerStatsManager getConsumerStatsManager() {\n        return this.defaultMQPushConsumerImpl.getConsumerStatsManager();\n    }\n\n    public boolean sendMessageBack(final MessageExt msg, final ConsumeConcurrentlyContext context) {\n        int delayLevel = context.getDelayLevelWhenNextConsume();\n\n        // Wrap topic with namespace before sending back message.\n        msg.setTopic(this.defaultMQPushConsumer.withNamespace(msg.getTopic()));\n        try {\n            this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, this.defaultMQPushConsumer.queueWithNamespace(context.getMessageQueue()));\n            return true;\n        } catch (Exception e) {\n            log.error(\"sendMessageBack exception, group: \" + this.consumerGroup + \" msg: \" + msg, e);\n        }\n\n        return false;\n    }\n\n    private void submitConsumeRequestLater(\n        final List<MessageExt> msgs,\n        final ProcessQueue processQueue,\n        final MessageQueue messageQueue\n    ) {\n\n        this.scheduledExecutorService.schedule(new Runnable() {\n\n            @Override\n            public void run() {\n                ConsumeMessageConcurrentlyService.this.submitConsumeRequest(msgs, processQueue, messageQueue, true);\n            }\n        }, 5000, TimeUnit.MILLISECONDS);\n    }\n\n    private void submitConsumeRequestLater(final ConsumeRequest consumeRequest\n    ) {\n\n        this.scheduledExecutorService.schedule(new Runnable() {\n\n            @Override\n            public void run() {\n                ConsumeMessageConcurrentlyService.this.consumeExecutor.submit(consumeRequest);\n            }\n        }, 5000, TimeUnit.MILLISECONDS);\n    }\n\n    class ConsumeRequest implements Runnable {\n        private final List<MessageExt> msgs;\n        private final ProcessQueue processQueue;\n        private final MessageQueue messageQueue;\n\n        public ConsumeRequest(List<MessageExt> msgs, ProcessQueue processQueue, MessageQueue messageQueue) {\n            this.msgs = msgs;\n            this.processQueue = processQueue;\n            this.messageQueue = messageQueue;\n        }\n\n        public List<MessageExt> getMsgs() {\n            return msgs;\n        }\n\n        public ProcessQueue getProcessQueue() {\n            return processQueue;\n        }\n\n        @Override\n        public void run() {\n            if (this.processQueue.isDropped()) {\n                log.info(\"the message queue not be able to consume, because it's dropped. group={} {}\", ConsumeMessageConcurrentlyService.this.consumerGroup, this.messageQueue);\n                return;\n            }\n\n            MessageListenerConcurrently listener = ConsumeMessageConcurrentlyService.this.messageListener;\n            ConsumeConcurrentlyContext context = new ConsumeConcurrentlyContext(messageQueue);\n            ConsumeConcurrentlyStatus status = null;\n            defaultMQPushConsumerImpl.tryResetPopRetryTopic(msgs, consumerGroup);\n            defaultMQPushConsumerImpl.resetRetryAndNamespace(msgs, defaultMQPushConsumer.getConsumerGroup());\n\n            ConsumeMessageContext consumeMessageContext = null;\n            if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) {\n                consumeMessageContext = new ConsumeMessageContext();\n                consumeMessageContext.setNamespace(defaultMQPushConsumer.getNamespace());\n                consumeMessageContext.setConsumerGroup(defaultMQPushConsumer.getConsumerGroup());\n                consumeMessageContext.setProps(new HashMap<>());\n                consumeMessageContext.setMq(messageQueue);\n                consumeMessageContext.setMsgList(msgs);\n                consumeMessageContext.setSuccess(false);\n                ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.executeHookBefore(consumeMessageContext);\n            }\n\n            long beginTimestamp = System.currentTimeMillis();\n            boolean hasException = false;\n            ConsumeReturnType returnType = ConsumeReturnType.SUCCESS;\n            try {\n                if (msgs != null && !msgs.isEmpty()) {\n                    for (MessageExt msg : msgs) {\n                        MessageAccessor.setConsumeStartTimeStamp(msg, String.valueOf(System.currentTimeMillis()));\n                    }\n                }\n                status = listener.consumeMessage(Collections.unmodifiableList(msgs), context);\n            } catch (Throwable e) {\n                log.warn(\"consumeMessage exception: {} Group: {} Msgs: {} MQ: {}\",\n                    UtilAll.exceptionSimpleDesc(e),\n                    ConsumeMessageConcurrentlyService.this.consumerGroup,\n                    msgs,\n                    messageQueue, e);\n                hasException = true;\n            }\n            long consumeRT = System.currentTimeMillis() - beginTimestamp;\n            if (null == status) {\n                if (hasException) {\n                    returnType = ConsumeReturnType.EXCEPTION;\n                } else {\n                    returnType = ConsumeReturnType.RETURNNULL;\n                }\n            } else if (consumeRT >= defaultMQPushConsumer.getConsumeTimeout() * 60 * 1000) {\n                returnType = ConsumeReturnType.TIME_OUT;\n            } else if (ConsumeConcurrentlyStatus.RECONSUME_LATER == status) {\n                returnType = ConsumeReturnType.FAILED;\n            } else if (ConsumeConcurrentlyStatus.CONSUME_SUCCESS == status) {\n                returnType = ConsumeReturnType.SUCCESS;\n            }\n\n            if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) {\n                consumeMessageContext.getProps().put(MixAll.CONSUME_CONTEXT_TYPE, returnType.name());\n            }\n\n            if (null == status) {\n                log.warn(\"consumeMessage return null, Group: {} Msgs: {} MQ: {}\",\n                    ConsumeMessageConcurrentlyService.this.consumerGroup,\n                    msgs,\n                    messageQueue);\n                status = ConsumeConcurrentlyStatus.RECONSUME_LATER;\n            }\n\n            if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) {\n                consumeMessageContext.setStatus(status.toString());\n                consumeMessageContext.setSuccess(ConsumeConcurrentlyStatus.CONSUME_SUCCESS == status);\n                consumeMessageContext.setAccessChannel(defaultMQPushConsumer.getAccessChannel());\n                ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.executeHookAfter(consumeMessageContext);\n            }\n\n            ConsumeMessageConcurrentlyService.this.getConsumerStatsManager()\n                .incConsumeRT(ConsumeMessageConcurrentlyService.this.consumerGroup, messageQueue.getTopic(), consumeRT);\n\n            if (!processQueue.isDropped()) {\n                ConsumeMessageConcurrentlyService.this.processConsumeResult(status, context, this);\n            } else {\n                log.warn(\"processQueue is dropped without process consume result. messageQueue={}, msgs={}\", messageQueue, msgs);\n            }\n        }\n\n        public MessageQueue getMessageQueue() {\n            return messageQueue;\n        }\n\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeReturnType;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;\nimport org.apache.rocketmq.client.hook.ConsumeMessageContext;\nimport org.apache.rocketmq.client.stat.ConsumerStatsManager;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.body.CMResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class ConsumeMessageOrderlyService implements ConsumeMessageService {\n    private static final Logger log = LoggerFactory.getLogger(ConsumeMessageOrderlyService.class);\n    private final static long MAX_TIME_CONSUME_CONTINUOUSLY =\n        Long.parseLong(System.getProperty(\"rocketmq.client.maxTimeConsumeContinuously\", \"60000\"));\n    private final DefaultMQPushConsumerImpl defaultMQPushConsumerImpl;\n    private final DefaultMQPushConsumer defaultMQPushConsumer;\n    private final MessageListenerOrderly messageListener;\n    private final BlockingQueue<Runnable> consumeRequestQueue;\n    private final ThreadPoolExecutor consumeExecutor;\n    private final String consumerGroup;\n    private final MessageQueueLock messageQueueLock = new MessageQueueLock();\n    private final ScheduledExecutorService scheduledExecutorService;\n    private volatile boolean stopped = false;\n\n    public ConsumeMessageOrderlyService(DefaultMQPushConsumerImpl defaultMQPushConsumerImpl,\n        MessageListenerOrderly messageListener) {\n        this.defaultMQPushConsumerImpl = defaultMQPushConsumerImpl;\n        this.messageListener = messageListener;\n\n        this.defaultMQPushConsumer = this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer();\n        this.consumerGroup = this.defaultMQPushConsumer.getConsumerGroup();\n        this.consumeRequestQueue = new LinkedBlockingQueue<>();\n\n        String consumerGroupTag = (consumerGroup.length() > 100 ? consumerGroup.substring(0, 100) : consumerGroup) + \"_\";\n        this.consumeExecutor = new ThreadPoolExecutor(\n            this.defaultMQPushConsumer.getConsumeThreadMin(),\n            this.defaultMQPushConsumer.getConsumeThreadMax(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            this.consumeRequestQueue,\n            new ThreadFactoryImpl(\"ConsumeMessageThread_\" + consumerGroupTag));\n\n        this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(\"ConsumeMessageScheduledThread_\" + consumerGroupTag));\n    }\n\n    @Override\n    public void start() {\n        if (MessageModel.CLUSTERING.equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.messageModel())) {\n            this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n                @Override\n                public void run() {\n                    try {\n                        ConsumeMessageOrderlyService.this.lockMQPeriodically();\n                    } catch (Throwable e) {\n                        log.error(\"scheduleAtFixedRate lockMQPeriodically exception\", e);\n                    }\n                }\n            }, 1000, ProcessQueue.REBALANCE_LOCK_INTERVAL, TimeUnit.MILLISECONDS);\n        }\n    }\n\n    @Override\n    public void shutdown(long awaitTerminateMillis) {\n        this.stopped = true;\n        this.scheduledExecutorService.shutdown();\n        ThreadUtils.shutdownGracefully(this.consumeExecutor, awaitTerminateMillis, TimeUnit.MILLISECONDS);\n        if (MessageModel.CLUSTERING.equals(this.defaultMQPushConsumerImpl.messageModel())) {\n            this.unlockAllMQ();\n        }\n    }\n\n    public synchronized void unlockAllMQ() {\n        this.defaultMQPushConsumerImpl.getRebalanceImpl().unlockAll(false);\n    }\n\n    @Override\n    public void updateCorePoolSize(int corePoolSize) {\n        if (corePoolSize > 0\n            && corePoolSize <= Short.MAX_VALUE\n            && corePoolSize < this.defaultMQPushConsumer.getConsumeThreadMax()) {\n            this.consumeExecutor.setCorePoolSize(corePoolSize);\n        }\n    }\n\n    @Override\n    public void incCorePoolSize() {\n    }\n\n    @Override\n    public void decCorePoolSize() {\n    }\n\n    @Override\n    public int getCorePoolSize() {\n        return this.consumeExecutor.getCorePoolSize();\n    }\n\n    @Override\n    public ConsumeMessageDirectlyResult consumeMessageDirectly(MessageExt msg, String brokerName) {\n        ConsumeMessageDirectlyResult result = new ConsumeMessageDirectlyResult();\n        result.setOrder(true);\n\n        List<MessageExt> msgs = new ArrayList<>();\n        msgs.add(msg);\n        MessageQueue mq = new MessageQueue();\n        mq.setBrokerName(brokerName);\n        mq.setTopic(msg.getTopic());\n        mq.setQueueId(msg.getQueueId());\n\n        ConsumeOrderlyContext context = new ConsumeOrderlyContext(mq);\n\n        this.defaultMQPushConsumerImpl.resetRetryAndNamespace(msgs, this.consumerGroup);\n\n        final long beginTime = System.currentTimeMillis();\n\n        log.info(\"consumeMessageDirectly receive new message: {}\", msg);\n\n        try {\n            ConsumeOrderlyStatus status = this.messageListener.consumeMessage(msgs, context);\n            if (status != null) {\n                switch (status) {\n                    case COMMIT:\n                        result.setConsumeResult(CMResult.CR_COMMIT);\n                        break;\n                    case ROLLBACK:\n                        result.setConsumeResult(CMResult.CR_ROLLBACK);\n                        break;\n                    case SUCCESS:\n                        result.setConsumeResult(CMResult.CR_SUCCESS);\n                        break;\n                    case SUSPEND_CURRENT_QUEUE_A_MOMENT:\n                        result.setConsumeResult(CMResult.CR_LATER);\n                        break;\n                    default:\n                        break;\n                }\n            } else {\n                result.setConsumeResult(CMResult.CR_RETURN_NULL);\n            }\n        } catch (Throwable e) {\n            result.setConsumeResult(CMResult.CR_THROW_EXCEPTION);\n            result.setRemark(UtilAll.exceptionSimpleDesc(e));\n\n            log.warn(\"consumeMessageDirectly exception: {} Group: {} Msgs: {} MQ: {}\",\n                UtilAll.exceptionSimpleDesc(e),\n                ConsumeMessageOrderlyService.this.consumerGroup,\n                msgs,\n                mq, e);\n        }\n\n        result.setAutoCommit(context.isAutoCommit());\n        result.setSpentTimeMills(System.currentTimeMillis() - beginTime);\n\n        log.info(\"consumeMessageDirectly Result: {}\", result);\n\n        return result;\n    }\n\n    @Override\n    public void submitConsumeRequest(\n        final List<MessageExt> msgs,\n        final ProcessQueue processQueue,\n        final MessageQueue messageQueue,\n        final boolean dispatchToConsume) {\n        if (dispatchToConsume) {\n            ConsumeRequest consumeRequest = new ConsumeRequest(processQueue, messageQueue);\n            this.consumeExecutor.submit(consumeRequest);\n        }\n    }\n\n    @Override\n    public void submitPopConsumeRequest(final List<MessageExt> msgs,\n                                        final PopProcessQueue processQueue,\n                                        final MessageQueue messageQueue) {\n        throw new UnsupportedOperationException();\n    }\n\n    public synchronized void lockMQPeriodically() {\n        if (!this.stopped) {\n            this.defaultMQPushConsumerImpl.getRebalanceImpl().lockAll();\n        }\n    }\n\n    public void tryLockLaterAndReconsume(final MessageQueue mq, final ProcessQueue processQueue,\n        final long delayMills) {\n        this.scheduledExecutorService.schedule(new Runnable() {\n            @Override\n            public void run() {\n                boolean lockOK = ConsumeMessageOrderlyService.this.lockOneMQ(mq);\n                if (lockOK) {\n                    ConsumeMessageOrderlyService.this.submitConsumeRequestLater(processQueue, mq, 10);\n                } else {\n                    ConsumeMessageOrderlyService.this.submitConsumeRequestLater(processQueue, mq, 3000);\n                }\n            }\n        }, delayMills, TimeUnit.MILLISECONDS);\n    }\n\n    public synchronized boolean lockOneMQ(final MessageQueue mq) {\n        if (!this.stopped) {\n            return this.defaultMQPushConsumerImpl.getRebalanceImpl().lock(mq);\n        }\n\n        return false;\n    }\n\n    private void submitConsumeRequestLater(\n        final ProcessQueue processQueue,\n        final MessageQueue messageQueue,\n        final long suspendTimeMillis\n    ) {\n        long timeMillis = suspendTimeMillis;\n        if (timeMillis == -1) {\n            timeMillis = this.defaultMQPushConsumer.getSuspendCurrentQueueTimeMillis();\n        }\n\n        if (timeMillis < 10) {\n            timeMillis = 10;\n        } else if (timeMillis > 30000) {\n            timeMillis = 30000;\n        }\n\n        this.scheduledExecutorService.schedule(new Runnable() {\n\n            @Override\n            public void run() {\n                ConsumeMessageOrderlyService.this.submitConsumeRequest(null, processQueue, messageQueue, true);\n            }\n        }, timeMillis, TimeUnit.MILLISECONDS);\n    }\n\n    public boolean processConsumeResult(\n        final List<MessageExt> msgs,\n        final ConsumeOrderlyStatus status,\n        final ConsumeOrderlyContext context,\n        final ConsumeRequest consumeRequest\n    ) {\n        boolean continueConsume = true;\n        long commitOffset = -1L;\n        if (context.isAutoCommit()) {\n            switch (status) {\n                case COMMIT:\n                case ROLLBACK:\n                    log.warn(\"the message queue consume result is illegal, we think you want to ack these message {}\",\n                        consumeRequest.getMessageQueue());\n                case SUCCESS:\n                    commitOffset = consumeRequest.getProcessQueue().commit();\n                    this.getConsumerStatsManager().incConsumeOKTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), msgs.size());\n                    break;\n                case SUSPEND_CURRENT_QUEUE_A_MOMENT:\n                    this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), msgs.size());\n                    if (checkReconsumeTimes(msgs)) {\n                        consumeRequest.getProcessQueue().makeMessageToConsumeAgain(msgs);\n                        this.submitConsumeRequestLater(\n                            consumeRequest.getProcessQueue(),\n                            consumeRequest.getMessageQueue(),\n                            context.getSuspendCurrentQueueTimeMillis());\n                        continueConsume = false;\n                    } else {\n                        commitOffset = consumeRequest.getProcessQueue().commit();\n                    }\n                    break;\n                default:\n                    break;\n            }\n        } else {\n            switch (status) {\n                case SUCCESS:\n                    this.getConsumerStatsManager().incConsumeOKTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), msgs.size());\n                    break;\n                case COMMIT:\n                    commitOffset = consumeRequest.getProcessQueue().commit();\n                    break;\n                case ROLLBACK:\n                    consumeRequest.getProcessQueue().rollback();\n                    this.submitConsumeRequestLater(\n                        consumeRequest.getProcessQueue(),\n                        consumeRequest.getMessageQueue(),\n                        context.getSuspendCurrentQueueTimeMillis());\n                    continueConsume = false;\n                    break;\n                case SUSPEND_CURRENT_QUEUE_A_MOMENT:\n                    this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), msgs.size());\n                    if (checkReconsumeTimes(msgs)) {\n                        consumeRequest.getProcessQueue().makeMessageToConsumeAgain(msgs);\n                        this.submitConsumeRequestLater(\n                            consumeRequest.getProcessQueue(),\n                            consumeRequest.getMessageQueue(),\n                            context.getSuspendCurrentQueueTimeMillis());\n                        continueConsume = false;\n                    }\n                    break;\n                default:\n                    break;\n            }\n        }\n\n        if (commitOffset >= 0 && !consumeRequest.getProcessQueue().isDropped()) {\n            this.defaultMQPushConsumerImpl.getOffsetStore().updateOffset(consumeRequest.getMessageQueue(), commitOffset, false);\n        }\n\n        return continueConsume;\n    }\n\n    public ConsumerStatsManager getConsumerStatsManager() {\n        return this.defaultMQPushConsumerImpl.getConsumerStatsManager();\n    }\n\n    private int getMaxReconsumeTimes() {\n        // default reconsume times: Integer.MAX_VALUE\n        if (this.defaultMQPushConsumer.getMaxReconsumeTimes() == -1) {\n            return Integer.MAX_VALUE;\n        } else {\n            return this.defaultMQPushConsumer.getMaxReconsumeTimes();\n        }\n    }\n\n    private boolean checkReconsumeTimes(List<MessageExt> msgs) {\n        boolean suspend = false;\n        if (msgs != null && !msgs.isEmpty()) {\n            for (MessageExt msg : msgs) {\n                if (msg.getReconsumeTimes() >= getMaxReconsumeTimes()) {\n                    MessageAccessor.setReconsumeTime(msg, String.valueOf(msg.getReconsumeTimes()));\n                    if (!sendMessageBack(msg)) {\n                        suspend = true;\n                        msg.setReconsumeTimes(msg.getReconsumeTimes() + 1);\n                    }\n                } else {\n                    suspend = true;\n                    msg.setReconsumeTimes(msg.getReconsumeTimes() + 1);\n                }\n            }\n        }\n        return suspend;\n    }\n\n    public boolean sendMessageBack(final MessageExt msg) {\n        try {\n            // max reconsume times exceeded then send to dead letter queue.\n            Message newMsg = new Message(MixAll.getRetryTopic(this.defaultMQPushConsumer.getConsumerGroup()), msg.getBody());\n            MessageAccessor.setProperties(newMsg, msg.getProperties());\n            String originMsgId = MessageAccessor.getOriginMessageId(msg);\n            MessageAccessor.setOriginMessageId(newMsg, UtilAll.isBlank(originMsgId) ? msg.getMsgId() : originMsgId);\n            newMsg.setFlag(msg.getFlag());\n            MessageAccessor.putProperty(newMsg, MessageConst.PROPERTY_RETRY_TOPIC, msg.getTopic());\n            MessageAccessor.setReconsumeTime(newMsg, String.valueOf(msg.getReconsumeTimes() + 1));\n            MessageAccessor.setMaxReconsumeTimes(newMsg, String.valueOf(getMaxReconsumeTimes()));\n            MessageAccessor.clearProperty(newMsg, MessageConst.PROPERTY_TRANSACTION_PREPARED);\n            newMsg.setDelayTimeLevel(3 + msg.getReconsumeTimes());\n\n            this.defaultMQPushConsumerImpl.getmQClientFactory().getDefaultMQProducer().send(newMsg);\n            return true;\n        } catch (Exception e) {\n            log.error(\"sendMessageBack exception, group: \" + this.consumerGroup + \" msg: \" + msg.toString(), e);\n        }\n\n        return false;\n    }\n\n    public void resetNamespace(final List<MessageExt> msgs) {\n        for (MessageExt msg : msgs) {\n            if (StringUtils.isNotEmpty(this.defaultMQPushConsumer.getNamespace())) {\n                msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQPushConsumer.getNamespace()));\n            }\n        }\n    }\n\n    class ConsumeRequest implements Runnable {\n        private final ProcessQueue processQueue;\n        private final MessageQueue messageQueue;\n\n        public ConsumeRequest(ProcessQueue processQueue, MessageQueue messageQueue) {\n            this.processQueue = processQueue;\n            this.messageQueue = messageQueue;\n        }\n\n        public ProcessQueue getProcessQueue() {\n            return processQueue;\n        }\n\n        public MessageQueue getMessageQueue() {\n            return messageQueue;\n        }\n\n        @Override\n        public void run() {\n            if (this.processQueue.isDropped()) {\n                log.warn(\"run, the message queue not be able to consume, because it's dropped. {}\", this.messageQueue);\n                return;\n            }\n\n            final Object objLock = messageQueueLock.fetchLockObject(this.messageQueue);\n            synchronized (objLock) {\n                if (MessageModel.BROADCASTING.equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.messageModel())\n                    || this.processQueue.isLocked() && !this.processQueue.isLockExpired()) {\n                    final long beginTime = System.currentTimeMillis();\n                    for (boolean continueConsume = true; continueConsume; ) {\n                        if (this.processQueue.isDropped()) {\n                            log.warn(\"the message queue not be able to consume, because it's dropped. {}\", this.messageQueue);\n                            break;\n                        }\n\n                        if (MessageModel.CLUSTERING.equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.messageModel())\n                            && !this.processQueue.isLocked()) {\n                            log.warn(\"the message queue not locked, so consume later, {}\", this.messageQueue);\n                            ConsumeMessageOrderlyService.this.tryLockLaterAndReconsume(this.messageQueue, this.processQueue, 10);\n                            break;\n                        }\n\n                        if (MessageModel.CLUSTERING.equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.messageModel())\n                            && this.processQueue.isLockExpired()) {\n                            log.warn(\"the message queue lock expired, so consume later, {}\", this.messageQueue);\n                            ConsumeMessageOrderlyService.this.tryLockLaterAndReconsume(this.messageQueue, this.processQueue, 10);\n                            break;\n                        }\n\n                        long interval = System.currentTimeMillis() - beginTime;\n                        if (interval > MAX_TIME_CONSUME_CONTINUOUSLY) {\n                            ConsumeMessageOrderlyService.this.submitConsumeRequestLater(processQueue, messageQueue, 10);\n                            break;\n                        }\n\n                        final int consumeBatchSize =\n                            ConsumeMessageOrderlyService.this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize();\n\n                        List<MessageExt> msgs = this.processQueue.takeMessages(consumeBatchSize);\n                        defaultMQPushConsumerImpl.resetRetryAndNamespace(msgs, defaultMQPushConsumer.getConsumerGroup());\n                        if (!msgs.isEmpty()) {\n                            final ConsumeOrderlyContext context = new ConsumeOrderlyContext(this.messageQueue);\n\n                            ConsumeOrderlyStatus status = null;\n\n                            ConsumeMessageContext consumeMessageContext = null;\n                            if (ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.hasHook()) {\n                                consumeMessageContext = new ConsumeMessageContext();\n                                consumeMessageContext\n                                    .setConsumerGroup(ConsumeMessageOrderlyService.this.defaultMQPushConsumer.getConsumerGroup());\n                                consumeMessageContext.setNamespace(defaultMQPushConsumer.getNamespace());\n                                consumeMessageContext.setMq(messageQueue);\n                                consumeMessageContext.setMsgList(msgs);\n                                consumeMessageContext.setSuccess(false);\n                                // init the consume context type\n                                consumeMessageContext.setProps(new HashMap<>());\n                                ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.executeHookBefore(consumeMessageContext);\n                            }\n\n                            long beginTimestamp = System.currentTimeMillis();\n                            ConsumeReturnType returnType = ConsumeReturnType.SUCCESS;\n                            boolean hasException = false;\n                            try {\n                                this.processQueue.getConsumeLock().readLock().lock();\n                                if (this.processQueue.isDropped()) {\n                                    log.warn(\"consumeMessage, the message queue not be able to consume, because it's dropped. {}\",\n                                        this.messageQueue);\n                                    break;\n                                }\n\n                                status = messageListener.consumeMessage(Collections.unmodifiableList(msgs), context);\n                            } catch (Throwable e) {\n                                log.warn(\"consumeMessage exception: {} Group: {} Msgs: {} MQ: {}\",\n                                    UtilAll.exceptionSimpleDesc(e),\n                                    ConsumeMessageOrderlyService.this.consumerGroup,\n                                    msgs,\n                                    messageQueue, e);\n                                hasException = true;\n                            } finally {\n                                this.processQueue.getConsumeLock().readLock().unlock();\n                            }\n\n                            if (null == status\n                                || ConsumeOrderlyStatus.ROLLBACK == status\n                                || ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT == status) {\n                                log.warn(\"consumeMessage Orderly return not OK, Group: {} Msgs: {} MQ: {}\",\n                                    ConsumeMessageOrderlyService.this.consumerGroup,\n                                    msgs,\n                                    messageQueue);\n                            }\n\n                            long consumeRT = System.currentTimeMillis() - beginTimestamp;\n                            if (null == status) {\n                                if (hasException) {\n                                    returnType = ConsumeReturnType.EXCEPTION;\n                                } else {\n                                    returnType = ConsumeReturnType.RETURNNULL;\n                                }\n                            } else if (consumeRT >= defaultMQPushConsumer.getConsumeTimeout() * 60 * 1000) {\n                                returnType = ConsumeReturnType.TIME_OUT;\n                            } else if (ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT == status) {\n                                returnType = ConsumeReturnType.FAILED;\n                            } else if (ConsumeOrderlyStatus.SUCCESS == status) {\n                                returnType = ConsumeReturnType.SUCCESS;\n                            }\n\n                            if (ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.hasHook()) {\n                                consumeMessageContext.getProps().put(MixAll.CONSUME_CONTEXT_TYPE, returnType.name());\n                            }\n\n                            if (null == status) {\n                                status = ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT;\n                            }\n\n                            if (ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.hasHook()) {\n                                consumeMessageContext.setStatus(status.toString());\n                                consumeMessageContext\n                                    .setSuccess(ConsumeOrderlyStatus.SUCCESS == status || ConsumeOrderlyStatus.COMMIT == status);\n                                consumeMessageContext.setAccessChannel(defaultMQPushConsumer.getAccessChannel());\n                                ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.executeHookAfter(consumeMessageContext);\n                            }\n\n                            ConsumeMessageOrderlyService.this.getConsumerStatsManager()\n                                .incConsumeRT(ConsumeMessageOrderlyService.this.consumerGroup, messageQueue.getTopic(), consumeRT);\n\n                            continueConsume = ConsumeMessageOrderlyService.this.processConsumeResult(msgs, status, context, this);\n                        } else {\n                            continueConsume = false;\n                        }\n                    }\n                } else {\n                    if (this.processQueue.isDropped()) {\n                        log.warn(\"the message queue not be able to consume, because it's dropped. {}\", this.messageQueue);\n                        return;\n                    }\n\n                    ConsumeMessageOrderlyService.this.tryLockLaterAndReconsume(this.messageQueue, this.processQueue, 100);\n                }\n            }\n        }\n\n    }\n\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopConcurrentlyService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.RejectedExecutionException;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.client.consumer.AckCallback;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeReturnType;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.hook.ConsumeMessageContext;\nimport org.apache.rocketmq.client.stat.ConsumerStatsManager;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.remoting.protocol.body.CMResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class ConsumeMessagePopConcurrentlyService implements ConsumeMessageService {\n    private static final Logger log = LoggerFactory.getLogger(ConsumeMessagePopConcurrentlyService.class);\n    private final DefaultMQPushConsumerImpl defaultMQPushConsumerImpl;\n    private final DefaultMQPushConsumer defaultMQPushConsumer;\n    private final MessageListenerConcurrently messageListener;\n    private final BlockingQueue<Runnable> consumeRequestQueue;\n    private final ThreadPoolExecutor consumeExecutor;\n    private final String consumerGroup;\n\n    private final ScheduledExecutorService scheduledExecutorService;\n\n    public ConsumeMessagePopConcurrentlyService(DefaultMQPushConsumerImpl defaultMQPushConsumerImpl,\n        MessageListenerConcurrently messageListener) {\n        this.defaultMQPushConsumerImpl = defaultMQPushConsumerImpl;\n        this.messageListener = messageListener;\n\n        this.defaultMQPushConsumer = this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer();\n        this.consumerGroup = this.defaultMQPushConsumer.getConsumerGroup();\n        this.consumeRequestQueue = new LinkedBlockingQueue<>();\n\n        this.consumeExecutor = new ThreadPoolExecutor(\n            this.defaultMQPushConsumer.getConsumeThreadMin(),\n            this.defaultMQPushConsumer.getConsumeThreadMax(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            this.consumeRequestQueue,\n            new ThreadFactoryImpl(\"ConsumeMessageThread_\"));\n\n        this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(\"ConsumeMessageScheduledThread_\"));\n    }\n\n    public void start() {\n    }\n\n    public void shutdown(long awaitTerminateMillis) {\n        this.scheduledExecutorService.shutdown();\n        ThreadUtils.shutdownGracefully(this.consumeExecutor, awaitTerminateMillis, TimeUnit.MILLISECONDS);\n    }\n\n    @Override\n    public void updateCorePoolSize(int corePoolSize) {\n        if (corePoolSize > 0\n            && corePoolSize <= Short.MAX_VALUE\n            && corePoolSize < this.defaultMQPushConsumer.getConsumeThreadMax()) {\n            this.consumeExecutor.setCorePoolSize(corePoolSize);\n        }\n    }\n\n    @Override\n    public void incCorePoolSize() {\n    }\n\n    @Override\n    public void decCorePoolSize() {\n    }\n\n    @Override\n    public int getCorePoolSize() {\n        return this.consumeExecutor.getCorePoolSize();\n    }\n\n\n    @Override\n    public ConsumeMessageDirectlyResult consumeMessageDirectly(MessageExt msg, String brokerName) {\n        ConsumeMessageDirectlyResult result = new ConsumeMessageDirectlyResult();\n        result.setOrder(false);\n        result.setAutoCommit(true);\n\n        List<MessageExt> msgs = new ArrayList<>();\n        msgs.add(msg);\n        MessageQueue mq = new MessageQueue();\n        mq.setBrokerName(brokerName);\n        mq.setTopic(msg.getTopic());\n        mq.setQueueId(msg.getQueueId());\n\n        ConsumeConcurrentlyContext context = new ConsumeConcurrentlyContext(mq);\n\n        this.defaultMQPushConsumerImpl.resetRetryAndNamespace(msgs, this.consumerGroup);\n\n        final long beginTime = System.currentTimeMillis();\n\n        log.info(\"consumeMessageDirectly receive new message: {}\", msg);\n\n        try {\n            ConsumeConcurrentlyStatus status = this.messageListener.consumeMessage(msgs, context);\n            if (status != null) {\n                switch (status) {\n                    case CONSUME_SUCCESS:\n                        result.setConsumeResult(CMResult.CR_SUCCESS);\n                        break;\n                    case RECONSUME_LATER:\n                        result.setConsumeResult(CMResult.CR_LATER);\n                        break;\n                    default:\n                        break;\n                }\n            } else {\n                result.setConsumeResult(CMResult.CR_RETURN_NULL);\n            }\n        } catch (Throwable e) {\n            result.setConsumeResult(CMResult.CR_THROW_EXCEPTION);\n            result.setRemark(UtilAll.exceptionSimpleDesc(e));\n\n            log.warn(\"consumeMessageDirectly exception: {} Group: {} Msgs: {} MQ: {}\",\n                UtilAll.exceptionSimpleDesc(e),\n                ConsumeMessagePopConcurrentlyService.this.consumerGroup,\n                msgs,\n                mq, e);\n        }\n\n        result.setSpentTimeMills(System.currentTimeMillis() - beginTime);\n\n        log.info(\"consumeMessageDirectly Result: {}\", result);\n\n        return result;\n    }\n\n    @Override\n    public void submitConsumeRequest(List<MessageExt> msgs, ProcessQueue processQueue,\n                                     MessageQueue messageQueue, boolean dispathToConsume) {\n        throw new UnsupportedOperationException();\n    }\n\n    @Override\n    public void submitPopConsumeRequest(\n        final List<MessageExt> msgs,\n        final PopProcessQueue processQueue,\n        final MessageQueue messageQueue) {\n        final int consumeBatchSize = this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize();\n        if (msgs.size() <= consumeBatchSize) {\n            ConsumeRequest consumeRequest = new ConsumeRequest(msgs, processQueue, messageQueue);\n            try {\n                this.consumeExecutor.submit(consumeRequest);\n            } catch (RejectedExecutionException e) {\n                this.submitConsumeRequestLater(consumeRequest);\n            }\n        } else {\n            for (int total = 0; total < msgs.size(); ) {\n                List<MessageExt> msgThis = new ArrayList<>(consumeBatchSize);\n                for (int i = 0; i < consumeBatchSize; i++, total++) {\n                    if (total < msgs.size()) {\n                        msgThis.add(msgs.get(total));\n                    } else {\n                        break;\n                    }\n                }\n\n                ConsumeRequest consumeRequest = new ConsumeRequest(msgThis, processQueue, messageQueue);\n                try {\n                    this.consumeExecutor.submit(consumeRequest);\n                } catch (RejectedExecutionException e) {\n                    for (; total < msgs.size(); total++) {\n                        msgThis.add(msgs.get(total));\n                    }\n\n                    this.submitConsumeRequestLater(consumeRequest);\n                }\n            }\n        }\n    }\n\n    public void processConsumeResult(\n        final ConsumeConcurrentlyStatus status,\n        final ConsumeConcurrentlyContext context,\n        final ConsumeRequest consumeRequest) {\n\n        if (consumeRequest.getMsgs().isEmpty()) {\n            return;\n        }\n\n        int ackIndex = context.getAckIndex();\n        String topic = consumeRequest.getMessageQueue().getTopic();\n\n        switch (status) {\n            case CONSUME_SUCCESS:\n                if (ackIndex >= consumeRequest.getMsgs().size()) {\n                    ackIndex = consumeRequest.getMsgs().size() - 1;\n                }\n                int ok = ackIndex + 1;\n                int failed = consumeRequest.getMsgs().size() - ok;\n                this.getConsumerStatsManager().incConsumeOKTPS(consumerGroup, topic, ok);\n                this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, topic, failed);\n                break;\n            case RECONSUME_LATER:\n                ackIndex = -1;\n                this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, topic,\n                        consumeRequest.getMsgs().size());\n                break;\n            default:\n                break;\n        }\n\n        //ack if consume success\n        for (int i = 0; i <= ackIndex; i++) {\n            this.defaultMQPushConsumerImpl.ackAsync(consumeRequest.getMsgs().get(i), consumerGroup);\n            consumeRequest.getPopProcessQueue().ack();\n        }\n\n        //consume later if consume fail\n        for (int i = ackIndex + 1; i < consumeRequest.getMsgs().size(); i++) {\n            MessageExt msgExt = consumeRequest.getMsgs().get(i);\n            consumeRequest.getPopProcessQueue().ack();\n            if (msgExt.getReconsumeTimes() >= this.defaultMQPushConsumerImpl.getMaxReconsumeTimes()) {\n                checkNeedAckOrDelay(msgExt);\n                continue;\n            }\n\n            int delayLevel = context.getDelayLevelWhenNextConsume();\n            changePopInvisibleTime(consumeRequest.getMsgs().get(i), consumerGroup, delayLevel);\n        }\n    }\n\n    private void checkNeedAckOrDelay(MessageExt msgExt) {\n        int[] delayLevelTable = this.defaultMQPushConsumerImpl.getPopDelayLevel();\n\n        long msgDelaytime = System.currentTimeMillis() - msgExt.getBornTimestamp();\n        if (msgDelaytime > delayLevelTable[delayLevelTable.length - 1] * 1000 * 2) {\n            log.warn(\"Consume too many times, ack message async. message {}\", msgExt.toString());\n            this.defaultMQPushConsumerImpl.ackAsync(msgExt, consumerGroup);\n        } else {\n            int delayLevel = delayLevelTable.length - 1;\n            for (; delayLevel >= 0; delayLevel--) {\n                if (msgDelaytime >= delayLevelTable[delayLevel] * 1000) {\n                    delayLevel++;\n                    break;\n                }\n            }\n\n            changePopInvisibleTime(msgExt, consumerGroup, delayLevel);\n            log.warn(\"Consume too many times, but delay time {} not enough. changePopInvisibleTime to delayLevel {} . message key:{}\",\n                msgDelaytime, delayLevel, msgExt.getKeys());\n        }\n    }\n\n    private void changePopInvisibleTime(final MessageExt msg, String consumerGroup, int delayLevel) {\n        if (0 == delayLevel) {\n            delayLevel = msg.getReconsumeTimes();\n        }\n\n        int[] delayLevelTable = this.defaultMQPushConsumerImpl.getPopDelayLevel();\n        int delaySecond = delayLevel >= delayLevelTable.length ? delayLevelTable[delayLevelTable.length - 1] : delayLevelTable[delayLevel];\n        String extraInfo = msg.getProperty(MessageConst.PROPERTY_POP_CK);\n\n        try {\n            this.defaultMQPushConsumerImpl.changePopInvisibleTimeAsync(msg.getTopic(), consumerGroup, extraInfo,\n                    delaySecond * 1000L, new AckCallback() {\n                        @Override\n                        public void onSuccess(AckResult ackResult) {\n                        }\n\n\n                        @Override\n                        public void onException(Throwable e) {\n                            log.error(\"changePopInvisibleTimeAsync fail. msg:{} error info: {}\", msg.toString(), e.toString());\n                        }\n                    });\n        } catch (Throwable t) {\n            log.error(\"changePopInvisibleTimeAsync fail, group:{} msg:{} errorInfo:{}\", consumerGroup, msg.toString(), t.toString());\n        }\n    }\n\n    public ConsumerStatsManager getConsumerStatsManager() {\n        return this.defaultMQPushConsumerImpl.getConsumerStatsManager();\n    }\n\n    private void submitConsumeRequestLater(\n        final List<MessageExt> msgs,\n        final PopProcessQueue processQueue,\n        final MessageQueue messageQueue\n    ) {\n\n        this.scheduledExecutorService.schedule(new Runnable() {\n\n            @Override\n            public void run() {\n                ConsumeMessagePopConcurrentlyService.this.submitPopConsumeRequest(msgs, processQueue, messageQueue);\n            }\n        }, 5000, TimeUnit.MILLISECONDS);\n    }\n\n    private void submitConsumeRequestLater(final ConsumeRequest consumeRequest\n    ) {\n\n        this.scheduledExecutorService.schedule(new Runnable() {\n\n            @Override\n            public void run() {\n                ConsumeMessagePopConcurrentlyService.this.consumeExecutor.submit(consumeRequest);\n            }\n        }, 5000, TimeUnit.MILLISECONDS);\n    }\n\n    class ConsumeRequest implements Runnable {\n        private final List<MessageExt> msgs;\n        private final PopProcessQueue processQueue;\n        private final MessageQueue messageQueue;\n        private long popTime = 0;\n        private long invisibleTime = 0;\n\n        public ConsumeRequest(List<MessageExt> msgs, PopProcessQueue processQueue, MessageQueue messageQueue) {\n            this.msgs = msgs;\n            this.processQueue = processQueue;\n            this.messageQueue = messageQueue;\n\n            try {\n                String extraInfo = msgs.get(0).getProperty(MessageConst.PROPERTY_POP_CK);\n                String[] extraInfoStrs = ExtraInfoUtil.split(extraInfo);\n                popTime = ExtraInfoUtil.getPopTime(extraInfoStrs);\n                invisibleTime = ExtraInfoUtil.getInvisibleTime(extraInfoStrs);\n            } catch (Throwable t) {\n                log.error(\"parse extra info error. msg:\" + msgs.get(0), t);\n            }\n        }\n\n        public boolean isPopTimeout() {\n            if (msgs.size() == 0 || popTime <= 0 || invisibleTime <= 0) {\n                return true;\n            }\n\n            long current = System.currentTimeMillis();\n            return current - popTime >= invisibleTime;\n        }\n\n        public List<MessageExt> getMsgs() {\n            return msgs;\n        }\n\n        public PopProcessQueue getPopProcessQueue() {\n            return processQueue;\n        }\n\n        @Override\n        public void run() {\n            if (this.processQueue.isDropped()) {\n                log.info(\"the message queue not be able to consume, because it's dropped(pop). group={} {}\", ConsumeMessagePopConcurrentlyService.this.consumerGroup, this.messageQueue);\n                return;\n            }\n\n            if (isPopTimeout()) {\n                log.info(\"the pop message time out so abort consume. popTime={} invisibleTime={}, group={} {}\",\n                        popTime, invisibleTime, ConsumeMessagePopConcurrentlyService.this.consumerGroup, this.messageQueue);\n                processQueue.decFoundMsg(-msgs.size());\n                return;\n            }\n\n            MessageListenerConcurrently listener = ConsumeMessagePopConcurrentlyService.this.messageListener;\n            ConsumeConcurrentlyContext context = new ConsumeConcurrentlyContext(messageQueue);\n            ConsumeConcurrentlyStatus status = null;\n            defaultMQPushConsumerImpl.resetRetryAndNamespace(msgs, defaultMQPushConsumer.getConsumerGroup());\n\n            ConsumeMessageContext consumeMessageContext = null;\n            if (ConsumeMessagePopConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) {\n                consumeMessageContext = new ConsumeMessageContext();\n                consumeMessageContext.setNamespace(defaultMQPushConsumer.getNamespace());\n                consumeMessageContext.setConsumerGroup(defaultMQPushConsumer.getConsumerGroup());\n                consumeMessageContext.setProps(new HashMap<>());\n                consumeMessageContext.setMq(messageQueue);\n                consumeMessageContext.setMsgList(msgs);\n                consumeMessageContext.setSuccess(false);\n                ConsumeMessagePopConcurrentlyService.this.defaultMQPushConsumerImpl.executeHookBefore(consumeMessageContext);\n            }\n\n            long beginTimestamp = System.currentTimeMillis();\n            boolean hasException = false;\n            ConsumeReturnType returnType = ConsumeReturnType.SUCCESS;\n            try {\n                if (msgs != null && !msgs.isEmpty()) {\n                    for (MessageExt msg : msgs) {\n                        MessageAccessor.setConsumeStartTimeStamp(msg, String.valueOf(System.currentTimeMillis()));\n                    }\n                }\n                status = listener.consumeMessage(Collections.unmodifiableList(msgs), context);\n            } catch (Throwable e) {\n                log.warn(\"consumeMessage exception: {} Group: {} Msgs: {} MQ: {}\",\n                    UtilAll.exceptionSimpleDesc(e),\n                    ConsumeMessagePopConcurrentlyService.this.consumerGroup,\n                    msgs,\n                    messageQueue);\n                hasException = true;\n            }\n            long consumeRT = System.currentTimeMillis() - beginTimestamp;\n            if (null == status) {\n                if (hasException) {\n                    returnType = ConsumeReturnType.EXCEPTION;\n                } else {\n                    returnType = ConsumeReturnType.RETURNNULL;\n                }\n            } else if (consumeRT >= invisibleTime * 1000) {\n                returnType = ConsumeReturnType.TIME_OUT;\n            } else if (ConsumeConcurrentlyStatus.RECONSUME_LATER == status) {\n                returnType = ConsumeReturnType.FAILED;\n            } else if (ConsumeConcurrentlyStatus.CONSUME_SUCCESS == status) {\n                returnType = ConsumeReturnType.SUCCESS;\n            }\n\n            if (null == status) {\n                log.warn(\"consumeMessage return null, Group: {} Msgs: {} MQ: {}\",\n                    ConsumeMessagePopConcurrentlyService.this.consumerGroup,\n                    msgs,\n                    messageQueue);\n                status = ConsumeConcurrentlyStatus.RECONSUME_LATER;\n            }\n\n            if (ConsumeMessagePopConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) {\n                consumeMessageContext.getProps().put(MixAll.CONSUME_CONTEXT_TYPE, returnType.name());\n                consumeMessageContext.setStatus(status.toString());\n                consumeMessageContext.setSuccess(ConsumeConcurrentlyStatus.CONSUME_SUCCESS == status);\n                consumeMessageContext.setAccessChannel(defaultMQPushConsumer.getAccessChannel());\n                ConsumeMessagePopConcurrentlyService.this.defaultMQPushConsumerImpl.executeHookAfter(consumeMessageContext);\n            }\n\n            ConsumeMessagePopConcurrentlyService.this.getConsumerStatsManager()\n                .incConsumeRT(ConsumeMessagePopConcurrentlyService.this.consumerGroup, messageQueue.getTopic(), consumeRT);\n\n            if (!processQueue.isDropped() && !isPopTimeout()) {\n                ConsumeMessagePopConcurrentlyService.this.processConsumeResult(status, context, this);\n            } else {\n                if (msgs != null) {\n                    processQueue.decFoundMsg(-msgs.size());\n                }\n\n                log.warn(\"processQueue invalid or popTimeout. isDropped={}, isPopTimeout={}, messageQueue={}, msgs={}\",\n                        processQueue.isDropped(), isPopTimeout(), messageQueue, msgs);\n            }\n        }\n\n        public MessageQueue getMessageQueue() {\n            return messageQueue;\n        }\n\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopOrderlyService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport io.netty.util.internal.ConcurrentSet;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;\nimport org.apache.rocketmq.client.stat.ConsumerStatsManager;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.body.CMResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class ConsumeMessagePopOrderlyService implements ConsumeMessageService {\n    private static final Logger log = LoggerFactory.getLogger(ConsumeMessagePopOrderlyService.class);\n    private final DefaultMQPushConsumerImpl defaultMQPushConsumerImpl;\n    private final DefaultMQPushConsumer defaultMQPushConsumer;\n    private final MessageListenerOrderly messageListener;\n    private final BlockingQueue<Runnable> consumeRequestQueue;\n    private final ConcurrentSet<ConsumeRequest> consumeRequestSet = new ConcurrentSet<>();\n    private final ThreadPoolExecutor consumeExecutor;\n    private final String consumerGroup;\n    private final MessageQueueLock messageQueueLock = new MessageQueueLock();\n    private final MessageQueueLock consumeRequestLock = new MessageQueueLock();\n    private final ScheduledExecutorService scheduledExecutorService;\n    private volatile boolean stopped = false;\n\n    public ConsumeMessagePopOrderlyService(DefaultMQPushConsumerImpl defaultMQPushConsumerImpl,\n        MessageListenerOrderly messageListener) {\n        this.defaultMQPushConsumerImpl = defaultMQPushConsumerImpl;\n        this.messageListener = messageListener;\n\n        this.defaultMQPushConsumer = this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer();\n        this.consumerGroup = this.defaultMQPushConsumer.getConsumerGroup();\n        this.consumeRequestQueue = new LinkedBlockingQueue<>();\n\n        this.consumeExecutor = new ThreadPoolExecutor(\n            this.defaultMQPushConsumer.getConsumeThreadMin(),\n            this.defaultMQPushConsumer.getConsumeThreadMax(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            this.consumeRequestQueue,\n            new ThreadFactoryImpl(\"ConsumeMessageThread_\"));\n\n        this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(\"ConsumeMessageScheduledThread_\"));\n    }\n\n    @Override\n    public void start() {\n        if (MessageModel.CLUSTERING.equals(ConsumeMessagePopOrderlyService.this.defaultMQPushConsumerImpl.messageModel())) {\n            this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n                @Override\n                public void run() {\n                    ConsumeMessagePopOrderlyService.this.lockMQPeriodically();\n                }\n            }, 1000 * 1, ProcessQueue.REBALANCE_LOCK_INTERVAL, TimeUnit.MILLISECONDS);\n        }\n    }\n\n    @Override\n    public void shutdown(long awaitTerminateMillis) {\n        this.stopped = true;\n        this.scheduledExecutorService.shutdown();\n        ThreadUtils.shutdownGracefully(this.consumeExecutor, awaitTerminateMillis, TimeUnit.MILLISECONDS);\n        if (MessageModel.CLUSTERING.equals(this.defaultMQPushConsumerImpl.messageModel())) {\n            this.unlockAllMessageQueues();\n        }\n    }\n\n    public synchronized void unlockAllMessageQueues() {\n        this.defaultMQPushConsumerImpl.getRebalanceImpl().unlockAll(false);\n    }\n\n    @Override\n    public void updateCorePoolSize(int corePoolSize) {\n        if (corePoolSize > 0\n            && corePoolSize <= Short.MAX_VALUE\n            && corePoolSize < this.defaultMQPushConsumer.getConsumeThreadMax()) {\n            this.consumeExecutor.setCorePoolSize(corePoolSize);\n        }\n    }\n\n    @Override\n    public void incCorePoolSize() {\n    }\n\n    @Override\n    public void decCorePoolSize() {\n    }\n\n    @Override\n    public int getCorePoolSize() {\n        return this.consumeExecutor.getCorePoolSize();\n    }\n\n    @Override\n    public ConsumeMessageDirectlyResult consumeMessageDirectly(MessageExt msg, String brokerName) {\n        ConsumeMessageDirectlyResult result = new ConsumeMessageDirectlyResult();\n        result.setOrder(true);\n\n        List<MessageExt> msgs = new ArrayList<>();\n        msgs.add(msg);\n        MessageQueue mq = new MessageQueue();\n        mq.setBrokerName(brokerName);\n        mq.setTopic(msg.getTopic());\n        mq.setQueueId(msg.getQueueId());\n\n        ConsumeOrderlyContext context = new ConsumeOrderlyContext(mq);\n\n        this.defaultMQPushConsumerImpl.resetRetryAndNamespace(msgs, this.consumerGroup);\n\n        final long beginTime = System.currentTimeMillis();\n\n        log.info(\"consumeMessageDirectly receive new message: {}\", msg);\n\n        try {\n            ConsumeOrderlyStatus status = this.messageListener.consumeMessage(msgs, context);\n            if (status != null) {\n                switch (status) {\n                    case COMMIT:\n                        result.setConsumeResult(CMResult.CR_COMMIT);\n                        break;\n                    case ROLLBACK:\n                        result.setConsumeResult(CMResult.CR_ROLLBACK);\n                        break;\n                    case SUCCESS:\n                        result.setConsumeResult(CMResult.CR_SUCCESS);\n                        break;\n                    case SUSPEND_CURRENT_QUEUE_A_MOMENT:\n                        result.setConsumeResult(CMResult.CR_LATER);\n                        break;\n                    default:\n                        break;\n                }\n            } else {\n                result.setConsumeResult(CMResult.CR_RETURN_NULL);\n            }\n        } catch (Throwable e) {\n            result.setConsumeResult(CMResult.CR_THROW_EXCEPTION);\n            result.setRemark(UtilAll.exceptionSimpleDesc(e));\n\n            log.warn(\"consumeMessageDirectly exception: {} Group: {} Msgs: {} MQ: {}\",\n                UtilAll.exceptionSimpleDesc(e),\n                ConsumeMessagePopOrderlyService.this.consumerGroup,\n                msgs,\n                mq, e);\n        }\n\n        result.setAutoCommit(context.isAutoCommit());\n        result.setSpentTimeMills(System.currentTimeMillis() - beginTime);\n\n        log.info(\"consumeMessageDirectly Result: {}\", result);\n\n        return result;\n    }\n\n    @Override\n    public void submitConsumeRequest(List<MessageExt> msgs, ProcessQueue processQueue,\n                                     MessageQueue messageQueue, boolean dispathToConsume) {\n        throw new UnsupportedOperationException();\n    }\n\n    @Override\n    public void submitPopConsumeRequest(final List<MessageExt> msgs,\n                                     final PopProcessQueue processQueue,\n                                     final MessageQueue messageQueue) {\n        ConsumeRequest req = new ConsumeRequest(processQueue, messageQueue);\n        submitConsumeRequest(req, false);\n    }\n\n    public synchronized void lockMQPeriodically() {\n        if (!this.stopped) {\n            this.defaultMQPushConsumerImpl.getRebalanceImpl().lockAll();\n        }\n    }\n\n    private void removeConsumeRequest(final ConsumeRequest consumeRequest) {\n        consumeRequestSet.remove(consumeRequest);\n    }\n\n    private void submitConsumeRequest(final ConsumeRequest consumeRequest, boolean force) {\n        Object lock = consumeRequestLock.fetchLockObject(consumeRequest.getMessageQueue(), consumeRequest.shardingKeyIndex);\n        synchronized (lock) {\n            boolean isNewReq = consumeRequestSet.add(consumeRequest);\n            if (force || isNewReq) {\n                try {\n                    consumeExecutor.submit(consumeRequest);\n                } catch (Exception e) {\n                    log.error(\"error submit consume request: {}, mq: {}, shardingKeyIndex: {}\",\n                        e.toString(), consumeRequest.getMessageQueue(), consumeRequest.getShardingKeyIndex());\n                }\n            }\n        }\n    }\n\n    private void submitConsumeRequestLater(final ConsumeRequest consumeRequest, final long suspendTimeMillis) {\n        long timeMillis = suspendTimeMillis;\n        if (timeMillis == -1) {\n            timeMillis = this.defaultMQPushConsumer.getSuspendCurrentQueueTimeMillis();\n        }\n\n        if (timeMillis < 10) {\n            timeMillis = 10;\n        } else if (timeMillis > 30000) {\n            timeMillis = 30000;\n        }\n\n        this.scheduledExecutorService.schedule(new Runnable() {\n\n            @Override\n            public void run() {\n                submitConsumeRequest(consumeRequest, true);\n            }\n        }, timeMillis, TimeUnit.MILLISECONDS);\n    }\n\n    public boolean processConsumeResult(\n        final List<MessageExt> msgs,\n        final ConsumeOrderlyStatus status,\n        final ConsumeOrderlyContext context,\n        final ConsumeRequest consumeRequest\n    ) {\n        return true;\n    }\n\n    public ConsumerStatsManager getConsumerStatsManager() {\n        return this.defaultMQPushConsumerImpl.getConsumerStatsManager();\n    }\n\n    private int getMaxReconsumeTimes() {\n        // default reconsume times: Integer.MAX_VALUE\n        if (this.defaultMQPushConsumer.getMaxReconsumeTimes() == -1) {\n            return Integer.MAX_VALUE;\n        } else {\n            return this.defaultMQPushConsumer.getMaxReconsumeTimes();\n        }\n    }\n\n    private boolean checkReconsumeTimes(List<MessageExt> msgs) {\n        boolean suspend = false;\n        if (msgs != null && !msgs.isEmpty()) {\n            for (MessageExt msg : msgs) {\n                if (msg.getReconsumeTimes() >= getMaxReconsumeTimes()) {\n                    MessageAccessor.setReconsumeTime(msg, String.valueOf(msg.getReconsumeTimes()));\n                    if (!sendMessageBack(msg)) {\n                        suspend = true;\n                        msg.setReconsumeTimes(msg.getReconsumeTimes() + 1);\n                    }\n                } else {\n                    suspend = true;\n                    msg.setReconsumeTimes(msg.getReconsumeTimes() + 1);\n                }\n            }\n        }\n        return suspend;\n    }\n\n    public boolean sendMessageBack(final MessageExt msg) {\n        try {\n            // max reconsume times exceeded then send to dead letter queue.\n            Message newMsg = new Message(MixAll.getRetryTopic(this.defaultMQPushConsumer.getConsumerGroup()), msg.getBody());\n            MessageAccessor.setProperties(newMsg, msg.getProperties());\n            String originMsgId = MessageAccessor.getOriginMessageId(msg);\n            MessageAccessor.setOriginMessageId(newMsg, UtilAll.isBlank(originMsgId) ? msg.getMsgId() : originMsgId);\n            newMsg.setFlag(msg.getFlag());\n            MessageAccessor.putProperty(newMsg, MessageConst.PROPERTY_RETRY_TOPIC, msg.getTopic());\n            MessageAccessor.setReconsumeTime(newMsg, String.valueOf(msg.getReconsumeTimes()));\n            MessageAccessor.setMaxReconsumeTimes(newMsg, String.valueOf(getMaxReconsumeTimes()));\n            newMsg.setDelayTimeLevel(3 + msg.getReconsumeTimes());\n\n            this.defaultMQPushConsumerImpl.getmQClientFactory().getDefaultMQProducer().send(newMsg);\n            return true;\n        } catch (Exception e) {\n            log.error(\"sendMessageBack exception, group: \" + this.consumerGroup + \" msg: \" + msg.toString(), e);\n        }\n\n        return false;\n    }\n\n    public void resetNamespace(final List<MessageExt> msgs) {\n        for (MessageExt msg : msgs) {\n            if (StringUtils.isNotEmpty(this.defaultMQPushConsumer.getNamespace())) {\n                msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQPushConsumer.getNamespace()));\n            }\n        }\n    }\n\n    class ConsumeRequest implements Runnable {\n        private final PopProcessQueue processQueue;\n        private final MessageQueue messageQueue;\n        private int shardingKeyIndex = 0;\n\n        public ConsumeRequest(PopProcessQueue processQueue, MessageQueue messageQueue) {\n            this.processQueue = processQueue;\n            this.messageQueue = messageQueue;\n            this.shardingKeyIndex = 0;\n        }\n\n        public ConsumeRequest(PopProcessQueue processQueue, MessageQueue messageQueue, int shardingKeyIndex) {\n            this.processQueue = processQueue;\n            this.messageQueue = messageQueue;\n            this.shardingKeyIndex = shardingKeyIndex;\n        }\n\n        public PopProcessQueue getProcessQueue() {\n            return processQueue;\n        }\n\n        public MessageQueue getMessageQueue() {\n            return messageQueue;\n        }\n\n        public int getShardingKeyIndex() {\n            return shardingKeyIndex;\n        }\n\n        @Override\n        public void run() {\n            if (this.processQueue.isDropped()) {\n                log.warn(\"run, message queue not be able to consume, because it's dropped. {}\", this.messageQueue);\n                ConsumeMessagePopOrderlyService.this.removeConsumeRequest(this);\n                return;\n            }\n\n            // lock on sharding key index\n            final Object objLock = messageQueueLock.fetchLockObject(this.messageQueue, shardingKeyIndex);\n        }\n\n        @Override\n        public int hashCode() {\n            int hash = shardingKeyIndex;\n            if (processQueue != null) {\n                hash += processQueue.hashCode() * 31;\n            }\n            if (messageQueue != null) {\n                hash += messageQueue.hashCode() * 31;\n            }\n            return hash;\n        }\n\n        @Override\n        public boolean equals(Object obj) {\n            if (this == obj) {\n                return true;\n            }\n            if (obj == null) {\n                return false;\n            }\n            if (getClass() != obj.getClass()) {\n                return false;\n            }\n\n            ConsumeRequest other = (ConsumeRequest) obj;\n            if (shardingKeyIndex != other.shardingKeyIndex) {\n                return false;\n            }\n\n            if (processQueue != other.processQueue) {\n                return false;\n            }\n\n            if (messageQueue == other.messageQueue) {\n                return true;\n            }\n            if (messageQueue != null && messageQueue.equals(other.messageQueue)) {\n                return true;\n            }\n            return false;\n        }\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport java.util.List;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\n\npublic interface ConsumeMessageService {\n    void start();\n\n    void shutdown(long awaitTerminateMillis);\n\n    void updateCorePoolSize(int corePoolSize);\n\n    void incCorePoolSize();\n\n    void decCorePoolSize();\n\n    int getCorePoolSize();\n\n    ConsumeMessageDirectlyResult consumeMessageDirectly(final MessageExt msg, final String brokerName);\n\n    void submitConsumeRequest(\n        final List<MessageExt> msgs,\n        final ProcessQueue processQueue,\n        final MessageQueue messageQueue,\n        final boolean dispathToConsume);\n\n    void submitPopConsumeRequest(\n        final List<MessageExt> msgs,\n        final PopProcessQueue processQueue,\n        final MessageQueue messageQueue);\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Properties;\nimport java.util.Set;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ScheduledThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.Validators;\nimport org.apache.rocketmq.client.consumer.DefaultLitePullConsumer;\nimport org.apache.rocketmq.client.consumer.MessageQueueListener;\nimport org.apache.rocketmq.client.consumer.MessageSelector;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.TopicMessageQueueChangeListener;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.store.LocalFileOffsetStore;\nimport org.apache.rocketmq.client.consumer.store.OffsetStore;\nimport org.apache.rocketmq.client.consumer.store.ReadOffsetType;\nimport org.apache.rocketmq.client.consumer.store.RemoteBrokerOffsetStore;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.hook.ConsumeMessageContext;\nimport org.apache.rocketmq.client.hook.ConsumeMessageHook;\nimport org.apache.rocketmq.client.hook.FilterMessageHook;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.MQClientManager;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ServiceState;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.common.help.FAQUrl;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.sysflag.PullSysFlag;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.body.ProcessQueueInfo;\nimport org.apache.rocketmq.remoting.protocol.filter.FilterAPI;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\npublic class DefaultLitePullConsumerImpl implements MQConsumerInner {\n\n    private static final Logger log = LoggerFactory.getLogger(DefaultLitePullConsumerImpl.class);\n\n    private final long consumerStartTimestamp = System.currentTimeMillis();\n\n    private final RPCHook rpcHook;\n\n    private final ArrayList<FilterMessageHook> filterMessageHookList = new ArrayList<>();\n\n    private volatile ServiceState serviceState = ServiceState.CREATE_JUST;\n\n    protected MQClientInstance mQClientFactory;\n\n    private PullAPIWrapper pullAPIWrapper;\n\n    private OffsetStore offsetStore;\n\n    private RebalanceImpl rebalanceImpl = new RebalanceLitePullImpl(this);\n\n    private enum SubscriptionType {\n        NONE, SUBSCRIBE, ASSIGN\n    }\n\n    private static final String NOT_RUNNING_EXCEPTION_MESSAGE = \"The consumer not running, please start it first.\";\n\n    private static final String SUBSCRIPTION_CONFLICT_EXCEPTION_MESSAGE = \"Subscribe and assign are mutually exclusive.\";\n    /**\n     * the type of subscription\n     */\n    private SubscriptionType subscriptionType = SubscriptionType.NONE;\n    /**\n     * Delay some time when exception occur\n     */\n    private long pullTimeDelayMillsWhenException = 1000;\n    /**\n     * Flow control interval when message cache is full\n     */\n    private static final long PULL_TIME_DELAY_MILLS_WHEN_CACHE_FLOW_CONTROL = 50;\n    /**\n     * Flow control interval when broker return flow control\n     */\n    private static final long PULL_TIME_DELAY_MILLS_WHEN_BROKER_FLOW_CONTROL = 20;\n    /**\n     * Delay some time when suspend pull service\n     */\n    private static final long PULL_TIME_DELAY_MILLS_WHEN_PAUSE = 1000;\n\n    private static final long PULL_TIME_DELAY_MILLS_ON_EXCEPTION = 3 * 1000;\n\n    private ConcurrentHashMap<String/* topic */, String/* subExpression */> topicToSubExpression = new ConcurrentHashMap<>();\n\n    private DefaultLitePullConsumer defaultLitePullConsumer;\n\n    private final ConcurrentMap<MessageQueue, PullTaskImpl> taskTable =\n        new ConcurrentHashMap<>();\n\n    private AssignedMessageQueue assignedMessageQueue = new AssignedMessageQueue();\n\n    private final BlockingQueue<ConsumeRequest> consumeRequestCache = new LinkedBlockingQueue<>();\n\n    private ScheduledThreadPoolExecutor scheduledThreadPoolExecutor;\n\n    private final ScheduledExecutorService scheduledExecutorService;\n\n    private Map<String, TopicMessageQueueChangeListener> topicMessageQueueChangeListenerMap = new HashMap<>();\n\n    private Map<String, Set<MessageQueue>> messageQueuesForTopic = new HashMap<>();\n\n    private long consumeRequestFlowControlTimes = 0L;\n\n    private long queueFlowControlTimes = 0L;\n\n    private long queueMaxSpanFlowControlTimes = 0L;\n\n    private long nextAutoCommitDeadline = -1L;\n\n    private final MessageQueueLock messageQueueLock = new MessageQueueLock();\n\n    private final ArrayList<ConsumeMessageHook> consumeMessageHookList = new ArrayList<>();\n\n    // only for test purpose, will be modified by reflection in unit test.\n    @SuppressWarnings(\"FieldMayBeFinal\")\n    private static boolean doNotUpdateTopicSubscribeInfoWhenSubscriptionChanged = false;\n\n    public DefaultLitePullConsumerImpl(final DefaultLitePullConsumer defaultLitePullConsumer, final RPCHook rpcHook) {\n        this.defaultLitePullConsumer = defaultLitePullConsumer;\n        this.rpcHook = rpcHook;\n        this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(\"MonitorMessageQueueChangeThread\"));\n        this.pullTimeDelayMillsWhenException = defaultLitePullConsumer.getPullTimeDelayMillsWhenException();\n    }\n\n    public void registerConsumeMessageHook(final ConsumeMessageHook hook) {\n        this.consumeMessageHookList.add(hook);\n        log.info(\"register consumeMessageHook Hook, {}\", hook.hookName());\n    }\n\n    public void executeHookBefore(final ConsumeMessageContext context) {\n        if (!this.consumeMessageHookList.isEmpty()) {\n            for (ConsumeMessageHook hook : this.consumeMessageHookList) {\n                try {\n                    hook.consumeMessageBefore(context);\n                } catch (Throwable e) {\n                    log.error(\"consumeMessageHook {} executeHookBefore exception\", hook.hookName(), e);\n                }\n            }\n        }\n    }\n\n    public void executeHookAfter(final ConsumeMessageContext context) {\n        if (!this.consumeMessageHookList.isEmpty()) {\n            for (ConsumeMessageHook hook : this.consumeMessageHookList) {\n                try {\n                    hook.consumeMessageAfter(context);\n                } catch (Throwable e) {\n                    log.error(\"consumeMessageHook {} executeHookAfter exception\", hook.hookName(), e);\n                }\n            }\n        }\n    }\n\n    private void checkServiceState() {\n        if (this.serviceState != ServiceState.RUNNING) {\n            throw new IllegalStateException(NOT_RUNNING_EXCEPTION_MESSAGE);\n        }\n    }\n\n    public void updateNameServerAddr(String newAddresses) {\n        this.mQClientFactory.getMQClientAPIImpl().updateNameServerAddressList(newAddresses);\n    }\n\n    private synchronized void setSubscriptionType(SubscriptionType type) {\n        if (this.subscriptionType == SubscriptionType.NONE) {\n            this.subscriptionType = type;\n        } else if (this.subscriptionType != type) {\n            throw new IllegalStateException(SUBSCRIPTION_CONFLICT_EXCEPTION_MESSAGE);\n        }\n    }\n\n    private void updateAssignedMessageQueue(String topic, Set<MessageQueue> assignedMessageQueue) {\n        this.assignedMessageQueue.updateAssignedMessageQueue(topic, assignedMessageQueue);\n    }\n\n    private void updatePullTask(String topic, Set<MessageQueue> mqNewSet) {\n        Iterator<Map.Entry<MessageQueue, PullTaskImpl>> it = this.taskTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Map.Entry<MessageQueue, PullTaskImpl> next = it.next();\n            if (next.getKey().getTopic().equals(topic)) {\n                if (!mqNewSet.contains(next.getKey())) {\n                    next.getValue().setCancelled(true);\n                    it.remove();\n                }\n            }\n        }\n        startPullTask(mqNewSet);\n    }\n\n    class MessageQueueListenerImpl implements MessageQueueListener {\n        @Override\n        public void messageQueueChanged(String topic, Set<MessageQueue> mqAll, Set<MessageQueue> mqDivided) {\n            updateAssignQueueAndStartPullTask(topic, mqAll, mqDivided);\n        }\n    }\n\n    public void updateAssignQueueAndStartPullTask(String topic, Set<MessageQueue> mqAll, Set<MessageQueue> mqDivided) {\n        MessageModel messageModel = defaultLitePullConsumer.getMessageModel();\n        switch (messageModel) {\n            case BROADCASTING:\n                updateAssignedMessageQueue(topic, mqAll);\n                updatePullTask(topic, mqAll);\n                break;\n            case CLUSTERING:\n                updateAssignedMessageQueue(topic, mqDivided);\n                updatePullTask(topic, mqDivided);\n                break;\n            default:\n                break;\n        }\n    }\n\n    public synchronized void shutdown() {\n        switch (this.serviceState) {\n            case CREATE_JUST:\n                break;\n            case RUNNING:\n                persistConsumerOffset();\n                this.mQClientFactory.unregisterConsumer(this.defaultLitePullConsumer.getConsumerGroup());\n                scheduledThreadPoolExecutor.shutdown();\n                scheduledExecutorService.shutdown();\n                this.mQClientFactory.shutdown();\n                this.serviceState = ServiceState.SHUTDOWN_ALREADY;\n                log.info(\"the consumer [{}] shutdown OK\", this.defaultLitePullConsumer.getConsumerGroup());\n                break;\n            default:\n                break;\n        }\n    }\n\n    public synchronized boolean isRunning() {\n        return this.serviceState == ServiceState.RUNNING;\n    }\n\n    public synchronized void start() throws MQClientException {\n        switch (this.serviceState) {\n            case CREATE_JUST:\n                this.serviceState = ServiceState.START_FAILED;\n\n                this.checkConfig();\n\n                if (this.defaultLitePullConsumer.getMessageModel() == MessageModel.CLUSTERING) {\n                    this.defaultLitePullConsumer.changeInstanceNameToPID();\n                }\n\n                initScheduledThreadPoolExecutor();\n\n                initMQClientFactory();\n\n                initRebalanceImpl();\n\n                initPullAPIWrapper();\n\n                initOffsetStore();\n\n                mQClientFactory.start();\n\n                startScheduleTask();\n\n                this.serviceState = ServiceState.RUNNING;\n\n                log.info(\"the consumer [{}] start OK\", this.defaultLitePullConsumer.getConsumerGroup());\n\n                try {\n                    operateAfterRunning();\n                } catch (Exception e) {\n                    shutdown();\n                    throw e;\n                }\n\n                break;\n            case RUNNING:\n            case START_FAILED:\n            case SHUTDOWN_ALREADY:\n                throw new MQClientException(\"The PullConsumer service state not OK, maybe started once, \"\n                    + this.serviceState\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK),\n                    null);\n            default:\n                break;\n        }\n    }\n\n    private void initScheduledThreadPoolExecutor() {\n        this.scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(\n                this.defaultLitePullConsumer.getPullThreadNums(),\n                new ThreadFactoryImpl(\"PullMsgThread-\" + this.defaultLitePullConsumer.getConsumerGroup())\n        );\n    }\n\n    private void initMQClientFactory() throws MQClientException {\n        this.mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultLitePullConsumer, this.rpcHook);\n        boolean registerOK = mQClientFactory.registerConsumer(this.defaultLitePullConsumer.getConsumerGroup(), this);\n        if (!registerOK) {\n            this.serviceState = ServiceState.CREATE_JUST;\n\n            throw new MQClientException(\"The consumer group[\" + this.defaultLitePullConsumer.getConsumerGroup()\n                + \"] has been created before, specify another name please.\" + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL),\n                null);\n        }\n    }\n\n    private void initRebalanceImpl() {\n        this.rebalanceImpl.setConsumerGroup(this.defaultLitePullConsumer.getConsumerGroup());\n        this.rebalanceImpl.setMessageModel(this.defaultLitePullConsumer.getMessageModel());\n        this.rebalanceImpl.setAllocateMessageQueueStrategy(this.defaultLitePullConsumer.getAllocateMessageQueueStrategy());\n        this.rebalanceImpl.setmQClientFactory(this.mQClientFactory);\n    }\n\n    private void initPullAPIWrapper() {\n        this.pullAPIWrapper = new PullAPIWrapper(\n            mQClientFactory,\n            this.defaultLitePullConsumer.getConsumerGroup(), isUnitMode());\n        this.pullAPIWrapper.registerFilterMessageHook(filterMessageHookList);\n    }\n\n    private void initOffsetStore() throws MQClientException {\n        if (this.defaultLitePullConsumer.getOffsetStore() != null) {\n            this.offsetStore = this.defaultLitePullConsumer.getOffsetStore();\n        } else {\n            switch (this.defaultLitePullConsumer.getMessageModel()) {\n                case BROADCASTING:\n                    this.offsetStore = new LocalFileOffsetStore(this.mQClientFactory, this.defaultLitePullConsumer.getConsumerGroup());\n                    break;\n                case CLUSTERING:\n                    this.offsetStore = new RemoteBrokerOffsetStore(this.mQClientFactory, this.defaultLitePullConsumer.getConsumerGroup());\n                    break;\n                default:\n                    break;\n            }\n            this.defaultLitePullConsumer.setOffsetStore(this.offsetStore);\n        }\n        this.offsetStore.load();\n    }\n\n    private void startScheduleTask() {\n        scheduledExecutorService.scheduleAtFixedRate(\n            new Runnable() {\n                @Override\n                public void run() {\n                    try {\n                        fetchTopicMessageQueuesAndCompare();\n                    } catch (Exception e) {\n                        log.error(\"ScheduledTask fetchMessageQueuesAndCompare exception\", e);\n                    }\n                }\n            }, 1000 * 10, this.getDefaultLitePullConsumer().getTopicMetadataCheckIntervalMillis(), TimeUnit.MILLISECONDS);\n    }\n\n    private void operateAfterRunning() throws MQClientException {\n        // If subscribe function invoke before start function, then update topic subscribe info after initialization.\n        if (subscriptionType == SubscriptionType.SUBSCRIBE) {\n            updateTopicSubscribeInfoWhenSubscriptionChanged();\n        }\n        // If assign function invoke before start function, then update pull task after initialization.\n        if (subscriptionType == SubscriptionType.ASSIGN) {\n            updateAssignPullTask(assignedMessageQueue.getAssignedMessageQueues());\n        }\n\n        for (String topic : topicMessageQueueChangeListenerMap.keySet()) {\n            Set<MessageQueue> messageQueues = fetchMessageQueues(topic);\n            messageQueuesForTopic.put(topic, messageQueues);\n        }\n        this.mQClientFactory.checkClientInBroker();\n    }\n\n    private void checkConfig() throws MQClientException {\n        // Check consumerGroup\n        Validators.checkGroup(this.defaultLitePullConsumer.getConsumerGroup());\n\n        // Check consumerGroup name is not equal default consumer group name.\n        if (this.defaultLitePullConsumer.getConsumerGroup().equals(MixAll.DEFAULT_CONSUMER_GROUP)) {\n            throw new MQClientException(\n                \"consumerGroup can not equal \"\n                    + MixAll.DEFAULT_CONSUMER_GROUP\n                    + \", please specify another one.\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        // Check messageModel is not null.\n        if (null == this.defaultLitePullConsumer.getMessageModel()) {\n            throw new MQClientException(\n                \"messageModel is null\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        // Check allocateMessageQueueStrategy is not null\n        if (null == this.defaultLitePullConsumer.getAllocateMessageQueueStrategy()) {\n            throw new MQClientException(\n                \"allocateMessageQueueStrategy is null\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        if (this.defaultLitePullConsumer.getConsumerTimeoutMillisWhenSuspend() < this.defaultLitePullConsumer.getBrokerSuspendMaxTimeMillis()) {\n            throw new MQClientException(\n                \"Long polling mode, the consumer consumerTimeoutMillisWhenSuspend must greater than brokerSuspendMaxTimeMillis\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n    }\n\n    public PullAPIWrapper getPullAPIWrapper() {\n        return pullAPIWrapper;\n    }\n\n    private void startPullTask(Collection<MessageQueue> mqSet) {\n        for (MessageQueue messageQueue : mqSet) {\n            if (!this.taskTable.containsKey(messageQueue)) {\n                PullTaskImpl pullTask = new PullTaskImpl(messageQueue);\n                this.taskTable.put(messageQueue, pullTask);\n                this.scheduledThreadPoolExecutor.schedule(pullTask, 0, TimeUnit.MILLISECONDS);\n            }\n        }\n    }\n\n    private void updateAssignPullTask(Collection<MessageQueue> mqNewSet) {\n        Iterator<Map.Entry<MessageQueue, PullTaskImpl>> it = this.taskTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Map.Entry<MessageQueue, PullTaskImpl> next = it.next();\n            if (!mqNewSet.contains(next.getKey())) {\n                next.getValue().setCancelled(true);\n                it.remove();\n            }\n        }\n\n        startPullTask(mqNewSet);\n    }\n\n    private void updateTopicSubscribeInfoWhenSubscriptionChanged() {\n        if (doNotUpdateTopicSubscribeInfoWhenSubscriptionChanged) {\n            return;\n        }\n        Map<String, SubscriptionData> subTable = rebalanceImpl.getSubscriptionInner();\n        if (subTable != null) {\n            for (final Map.Entry<String, SubscriptionData> entry : subTable.entrySet()) {\n                final String topic = entry.getKey();\n                this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic);\n            }\n        }\n    }\n\n    /**\n     * subscribe data by customizing messageQueueListener\n     *\n     * @param topic\n     * @param subExpression\n     * @param messageQueueListener\n     * @throws MQClientException\n     */\n    public synchronized void subscribe(String topic, String subExpression,\n        MessageQueueListener messageQueueListener) throws MQClientException {\n        try {\n            if (StringUtils.isEmpty(topic)) {\n                throw new IllegalArgumentException(\"Topic can not be null or empty.\");\n            }\n            setSubscriptionType(SubscriptionType.SUBSCRIBE);\n            SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(topic, subExpression);\n            this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData);\n            this.defaultLitePullConsumer.setMessageQueueListener(new MessageQueueListener() {\n                @Override\n                public void messageQueueChanged(String topic, Set<MessageQueue> mqAll, Set<MessageQueue> mqDivided) {\n                    // First, update the assign queue\n                    updateAssignQueueAndStartPullTask(topic, mqAll, mqDivided);\n                    // run custom listener\n                    messageQueueListener.messageQueueChanged(topic, mqAll, mqDivided);\n                }\n            });\n            assignedMessageQueue.setRebalanceImpl(this.rebalanceImpl);\n            if (serviceState == ServiceState.RUNNING) {\n                this.mQClientFactory.sendHeartbeatToAllBrokerWithLock();\n                updateTopicSubscribeInfoWhenSubscriptionChanged();\n            }\n        } catch (Exception e) {\n            throw new MQClientException(\"subscribe exception\", e);\n        }\n    }\n\n    public synchronized void subscribe(String topic, String subExpression) throws MQClientException {\n        try {\n            if (topic == null || \"\".equals(topic)) {\n                throw new IllegalArgumentException(\"Topic can not be null or empty.\");\n            }\n            setSubscriptionType(SubscriptionType.SUBSCRIBE);\n            SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(topic, subExpression);\n            this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData);\n            this.defaultLitePullConsumer.setMessageQueueListener(new MessageQueueListenerImpl());\n            assignedMessageQueue.setRebalanceImpl(this.rebalanceImpl);\n            if (serviceState == ServiceState.RUNNING) {\n                this.mQClientFactory.sendHeartbeatToAllBrokerWithLock();\n                updateTopicSubscribeInfoWhenSubscriptionChanged();\n            }\n        } catch (Exception e) {\n            throw new MQClientException(\"subscribe exception\", e);\n        }\n    }\n\n    public synchronized void subscribe(String topic, MessageSelector messageSelector) throws MQClientException {\n        try {\n            if (topic == null || \"\".equals(topic)) {\n                throw new IllegalArgumentException(\"Topic can not be null or empty.\");\n            }\n            setSubscriptionType(SubscriptionType.SUBSCRIBE);\n            if (messageSelector == null) {\n                subscribe(topic, SubscriptionData.SUB_ALL);\n                return;\n            }\n            SubscriptionData subscriptionData = FilterAPI.build(topic,\n                messageSelector.getExpression(), messageSelector.getExpressionType());\n            this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData);\n            this.defaultLitePullConsumer.setMessageQueueListener(new MessageQueueListenerImpl());\n            assignedMessageQueue.setRebalanceImpl(this.rebalanceImpl);\n            if (serviceState == ServiceState.RUNNING) {\n                this.mQClientFactory.sendHeartbeatToAllBrokerWithLock();\n                updateTopicSubscribeInfoWhenSubscriptionChanged();\n            }\n        } catch (Exception e) {\n            throw new MQClientException(\"subscribe exception\", e);\n        }\n    }\n\n    public synchronized void unsubscribe(final String topic) {\n        this.rebalanceImpl.getSubscriptionInner().remove(topic);\n        removePullTaskCallback(topic);\n        assignedMessageQueue.removeAssignedMessageQueue(topic);\n    }\n\n    public synchronized void assign(Collection<MessageQueue> messageQueues) {\n        if (messageQueues == null || messageQueues.isEmpty()) {\n            throw new IllegalArgumentException(\"Message queues can not be null or empty.\");\n        }\n        setSubscriptionType(SubscriptionType.ASSIGN);\n        assignedMessageQueue.updateAssignedMessageQueue(messageQueues);\n        if (serviceState == ServiceState.RUNNING) {\n            updateAssignPullTask(messageQueues);\n        }\n    }\n\n    public synchronized void setSubExpressionForAssign(final String topic, final String subExpression) {\n        if (StringUtils.isBlank(subExpression)) {\n            throw new IllegalArgumentException(\"subExpression can not be null or empty.\");\n        }\n        if (serviceState != ServiceState.CREATE_JUST) {\n            throw new IllegalStateException(\"setAssignTag only can be called before start.\");\n        }\n        setSubscriptionType(SubscriptionType.ASSIGN);\n        topicToSubExpression.put(topic, subExpression);\n    }\n\n    private void maybeAutoCommit() {\n        long now = System.currentTimeMillis();\n        if (now >= nextAutoCommitDeadline) {\n            commitAll();\n            nextAutoCommitDeadline = now + defaultLitePullConsumer.getAutoCommitIntervalMillis();\n        }\n    }\n\n    public synchronized List<MessageExt> poll(long timeout) {\n        try {\n            checkServiceState();\n            if (timeout < 0) {\n                throw new IllegalArgumentException(\"Timeout must not be negative\");\n            }\n\n            if (defaultLitePullConsumer.isAutoCommit()) {\n                maybeAutoCommit();\n            }\n            long endTime = System.currentTimeMillis() + timeout;\n\n            ConsumeRequest consumeRequest = consumeRequestCache.poll(endTime - System.currentTimeMillis(), TimeUnit.MILLISECONDS);\n\n            if (endTime - System.currentTimeMillis() > 0) {\n                while (consumeRequest != null && consumeRequest.getProcessQueue().isDropped()) {\n                    consumeRequest = consumeRequestCache.poll(endTime - System.currentTimeMillis(), TimeUnit.MILLISECONDS);\n                    if (endTime - System.currentTimeMillis() <= 0) {\n                        break;\n                    }\n                }\n            }\n\n            if (consumeRequest != null && !consumeRequest.getProcessQueue().isDropped()) {\n                List<MessageExt> messages = consumeRequest.getMessageExts();\n                long offset = consumeRequest.getProcessQueue().removeMessage(messages);\n                assignedMessageQueue.updateConsumeOffset(consumeRequest.getMessageQueue(), offset);\n                //If namespace not null , reset Topic without namespace.\n                this.resetTopic(messages);\n                if (!this.consumeMessageHookList.isEmpty()) {\n                    ConsumeMessageContext consumeMessageContext = new ConsumeMessageContext();\n                    consumeMessageContext.setNamespace(defaultLitePullConsumer.getNamespace());\n                    consumeMessageContext.setConsumerGroup(this.groupName());\n                    consumeMessageContext.setMq(consumeRequest.getMessageQueue());\n                    consumeMessageContext.setMsgList(messages);\n                    consumeMessageContext.setSuccess(false);\n                    this.executeHookBefore(consumeMessageContext);\n                    consumeMessageContext.setStatus(ConsumeConcurrentlyStatus.CONSUME_SUCCESS.toString());\n                    consumeMessageContext.setSuccess(true);\n                    consumeMessageContext.setAccessChannel(defaultLitePullConsumer.getAccessChannel());\n                    this.executeHookAfter(consumeMessageContext);\n                }\n                consumeRequest.getProcessQueue().setLastConsumeTimestamp(System.currentTimeMillis());\n                return messages;\n            }\n        } catch (InterruptedException ignore) {\n\n        }\n\n        return Collections.emptyList();\n    }\n\n    public void pause(Collection<MessageQueue> messageQueues) {\n        assignedMessageQueue.pause(messageQueues);\n    }\n\n    public void resume(Collection<MessageQueue> messageQueues) {\n        assignedMessageQueue.resume(messageQueues);\n    }\n\n    public synchronized void seek(MessageQueue messageQueue, long offset) throws MQClientException {\n        if (!assignedMessageQueue.getAssignedMessageQueues().contains(messageQueue)) {\n            if (subscriptionType == SubscriptionType.SUBSCRIBE) {\n                throw new MQClientException(\"The message queue is not in assigned list, may be rebalancing, message queue: \" + messageQueue, null);\n            } else {\n                throw new MQClientException(\"The message queue is not in assigned list, message queue: \" + messageQueue, null);\n            }\n        }\n        long minOffset = minOffset(messageQueue);\n        long maxOffset = maxOffset(messageQueue);\n        if (offset < minOffset || offset > maxOffset) {\n            throw new MQClientException(\"Seek offset illegal, seek offset = \" + offset + \", min offset = \" + minOffset + \", max offset = \" + maxOffset, null);\n        }\n        final Object objLock = messageQueueLock.fetchLockObject(messageQueue);\n        synchronized (objLock) {\n            clearMessageQueueInCache(messageQueue);\n\n            PullTaskImpl oldPullTaskImpl = this.taskTable.get(messageQueue);\n            if (oldPullTaskImpl != null) {\n                oldPullTaskImpl.tryInterrupt();\n                this.taskTable.remove(messageQueue);\n            }\n            assignedMessageQueue.setSeekOffset(messageQueue, offset);\n            if (!this.taskTable.containsKey(messageQueue)) {\n                PullTaskImpl pullTask = new PullTaskImpl(messageQueue);\n                this.taskTable.put(messageQueue, pullTask);\n                this.scheduledThreadPoolExecutor.schedule(pullTask, 0, TimeUnit.MILLISECONDS);\n            }\n        }\n    }\n\n    public void seekToBegin(MessageQueue messageQueue) throws MQClientException {\n        long begin = minOffset(messageQueue);\n        this.seek(messageQueue, begin);\n    }\n\n    public void seekToEnd(MessageQueue messageQueue) throws MQClientException {\n        long end = maxOffset(messageQueue);\n        this.seek(messageQueue, end);\n    }\n\n    private long maxOffset(MessageQueue messageQueue) throws MQClientException {\n        checkServiceState();\n        return this.mQClientFactory.getMQAdminImpl().maxOffset(messageQueue);\n    }\n\n    private long minOffset(MessageQueue messageQueue) throws MQClientException {\n        checkServiceState();\n        return this.mQClientFactory.getMQAdminImpl().minOffset(messageQueue);\n    }\n\n    private void removePullTaskCallback(final String topic) {\n        removePullTask(topic);\n    }\n\n    private void removePullTask(final String topic) {\n        Iterator<Map.Entry<MessageQueue, PullTaskImpl>> it = this.taskTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Map.Entry<MessageQueue, PullTaskImpl> next = it.next();\n            if (next.getKey().getTopic().equals(topic)) {\n                next.getValue().setCancelled(true);\n                it.remove();\n            }\n        }\n    }\n\n    public synchronized void commitAll() {\n        for (MessageQueue messageQueue : assignedMessageQueue.getAssignedMessageQueues()) {\n            try {\n                commit(messageQueue);\n            } catch (Exception e) {\n                log.error(\"An error occurred when update consume offset Automatically.\");\n            }\n        }\n    }\n\n    /**\n     * Specify offset commit\n     *\n     * @param messageQueues\n     * @param persist\n     */\n    public synchronized void commit(final Map<MessageQueue, Long> messageQueues, boolean persist) {\n        if (messageQueues == null || messageQueues.size() == 0) {\n            log.warn(\"MessageQueues is empty, Ignore this commit \");\n            return;\n        }\n        for (Map.Entry<MessageQueue, Long> messageQueueEntry : messageQueues.entrySet()) {\n            MessageQueue messageQueue = messageQueueEntry.getKey();\n            long offset = messageQueueEntry.getValue();\n            if (offset != -1) {\n                ProcessQueue processQueue = assignedMessageQueue.getProcessQueue(messageQueue);\n                if (processQueue != null && !processQueue.isDropped()) {\n                    updateConsumeOffset(messageQueue, offset);\n                }\n            } else {\n                log.error(\"consumerOffset is -1 in messageQueue [\" + messageQueue + \"].\");\n            }\n        }\n\n        if (persist) {\n            this.offsetStore.persistAll(messageQueues.keySet());\n        }\n    }\n\n    /**\n     * Get the queue assigned in subscribe mode\n     *\n     * @return\n     */\n    public synchronized Set<MessageQueue> assignment() {\n        return assignedMessageQueue.getAssignedMessageQueues();\n    }\n\n    public synchronized void commit(final Set<MessageQueue> messageQueues, boolean persist) {\n        if (messageQueues == null || messageQueues.size() == 0) {\n            return;\n        }\n\n        for (MessageQueue messageQueue : messageQueues) {\n            commit(messageQueue);\n        }\n\n        if (persist) {\n            this.offsetStore.persistAll(messageQueues);\n        }\n    }\n\n    private synchronized void commit(MessageQueue messageQueue) {\n        long consumerOffset = assignedMessageQueue.getConsumerOffset(messageQueue);\n\n        if (consumerOffset != -1) {\n            ProcessQueue processQueue = assignedMessageQueue.getProcessQueue(messageQueue);\n            if (processQueue != null && !processQueue.isDropped()) {\n                updateConsumeOffset(messageQueue, consumerOffset);\n            }\n        } else {\n            log.error(\"consumerOffset is -1 in messageQueue [\" + messageQueue + \"].\");\n        }\n    }\n\n    private void updatePullOffset(MessageQueue messageQueue, long nextPullOffset, ProcessQueue processQueue) {\n        if (assignedMessageQueue.getSeekOffset(messageQueue) == -1) {\n            assignedMessageQueue.updatePullOffset(messageQueue, nextPullOffset, processQueue);\n        }\n    }\n\n    private void submitConsumeRequest(ConsumeRequest consumeRequest) {\n        try {\n            consumeRequestCache.put(consumeRequest);\n        } catch (InterruptedException e) {\n            log.error(\"Submit consumeRequest error\", e);\n        }\n    }\n\n    private long fetchConsumeOffset(MessageQueue messageQueue) throws MQClientException {\n        checkServiceState();\n        long offset = this.rebalanceImpl.computePullFromWhereWithException(messageQueue);\n        return offset;\n    }\n\n    public long committed(MessageQueue messageQueue) throws MQClientException {\n        checkServiceState();\n        long offset = this.offsetStore.readOffset(messageQueue, ReadOffsetType.MEMORY_FIRST_THEN_STORE);\n        if (offset == -2) {\n            throw new MQClientException(\"Fetch consume offset from broker exception\", null);\n        }\n        return offset;\n    }\n\n    private void clearMessageQueueInCache(MessageQueue messageQueue) {\n        ProcessQueue processQueue = assignedMessageQueue.getProcessQueue(messageQueue);\n        if (processQueue != null) {\n            processQueue.clear();\n        }\n        Iterator<ConsumeRequest> iter = consumeRequestCache.iterator();\n        while (iter.hasNext()) {\n            if (iter.next().getMessageQueue().equals(messageQueue)) {\n                iter.remove();\n            }\n        }\n    }\n\n    private long nextPullOffset(MessageQueue messageQueue) throws MQClientException {\n        long offset = -1;\n        long seekOffset = assignedMessageQueue.getSeekOffset(messageQueue);\n        if (seekOffset != -1) {\n            offset = seekOffset;\n            assignedMessageQueue.updateConsumeOffset(messageQueue, offset);\n            assignedMessageQueue.setSeekOffset(messageQueue, -1);\n        } else {\n            offset = assignedMessageQueue.getPullOffset(messageQueue);\n            if (offset == -1) {\n                offset = fetchConsumeOffset(messageQueue);\n            }\n        }\n        return offset;\n    }\n\n    public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException {\n        checkServiceState();\n        return this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp);\n    }\n\n    public class PullTaskImpl implements Runnable {\n        private final MessageQueue messageQueue;\n        private volatile boolean cancelled = false;\n        private Thread currentThread;\n\n        public PullTaskImpl(final MessageQueue messageQueue) {\n            this.messageQueue = messageQueue;\n        }\n\n        public void tryInterrupt() {\n            setCancelled(true);\n            if (currentThread == null) {\n                return;\n            }\n            if (!currentThread.isInterrupted()) {\n                currentThread.interrupt();\n            }\n        }\n\n        @Override\n        public void run() {\n\n            if (!this.isCancelled()) {\n\n                this.currentThread = Thread.currentThread();\n\n                if (assignedMessageQueue.isPaused(messageQueue)) {\n                    scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_PAUSE, TimeUnit.MILLISECONDS);\n                    log.debug(\"Message Queue: {} has been paused!\", messageQueue);\n                    return;\n                }\n\n                ProcessQueue processQueue = assignedMessageQueue.getProcessQueue(messageQueue);\n\n                if (null == processQueue || processQueue.isDropped()) {\n                    log.info(\"The message queue not be able to poll, because it's dropped. group={}, messageQueue={}\", defaultLitePullConsumer.getConsumerGroup(), this.messageQueue);\n                    return;\n                }\n\n                processQueue.setLastPullTimestamp(System.currentTimeMillis());\n\n                if ((long) consumeRequestCache.size() * defaultLitePullConsumer.getPullBatchSize() > defaultLitePullConsumer.getPullThresholdForAll()) {\n                    scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_CACHE_FLOW_CONTROL, TimeUnit.MILLISECONDS);\n                    if ((consumeRequestFlowControlTimes++ % 1000) == 0) {\n                        log.warn(\"The consume request count exceeds threshold {}, so do flow control, consume request count={}, flowControlTimes={}\",\n                                (int)Math.ceil((double)defaultLitePullConsumer.getPullThresholdForAll() / defaultLitePullConsumer.getPullBatchSize()),\n                                consumeRequestCache.size(), consumeRequestFlowControlTimes);\n                    }\n                    return;\n                }\n\n                long cachedMessageCount = processQueue.getMsgCount().get();\n                long cachedMessageSizeInMiB = processQueue.getMsgSize().get() / (1024 * 1024);\n\n                if (cachedMessageCount > defaultLitePullConsumer.getPullThresholdForQueue()) {\n                    scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_CACHE_FLOW_CONTROL, TimeUnit.MILLISECONDS);\n                    if ((queueFlowControlTimes++ % 1000) == 0) {\n                        log.warn(\n                            \"The cached message count exceeds the threshold {}, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, flowControlTimes={}\",\n                            defaultLitePullConsumer.getPullThresholdForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, queueFlowControlTimes);\n                    }\n                    return;\n                }\n\n                if (cachedMessageSizeInMiB > defaultLitePullConsumer.getPullThresholdSizeForQueue()) {\n                    scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_CACHE_FLOW_CONTROL, TimeUnit.MILLISECONDS);\n                    if ((queueFlowControlTimes++ % 1000) == 0) {\n                        log.warn(\n                            \"The cached message size exceeds the threshold {} MiB, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, flowControlTimes={}\",\n                            defaultLitePullConsumer.getPullThresholdSizeForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, queueFlowControlTimes);\n                    }\n                    return;\n                }\n\n                if (processQueue.getMaxSpan() > defaultLitePullConsumer.getConsumeMaxSpan()) {\n                    scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_CACHE_FLOW_CONTROL, TimeUnit.MILLISECONDS);\n                    if ((queueMaxSpanFlowControlTimes++ % 1000) == 0) {\n                        log.warn(\n                            \"The queue's messages, span too long, so do flow control, minOffset={}, maxOffset={}, maxSpan={}, flowControlTimes={}\",\n                            processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), processQueue.getMaxSpan(), queueMaxSpanFlowControlTimes);\n                    }\n                    return;\n                }\n\n                long offset = 0L;\n                try {\n                    offset = nextPullOffset(messageQueue);\n                } catch (Exception e) {\n                    log.error(\"Failed to get next pull offset\", e);\n                    scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_ON_EXCEPTION, TimeUnit.MILLISECONDS);\n                    return;\n                }\n\n                if (this.isCancelled() || processQueue.isDropped()) {\n                    return;\n                }\n                long pullDelayTimeMills = 0;\n                try {\n                    SubscriptionData subscriptionData;\n                    String topic = this.messageQueue.getTopic();\n                    if (subscriptionType == SubscriptionType.SUBSCRIBE) {\n                        subscriptionData = rebalanceImpl.getSubscriptionInner().get(topic);\n                    } else {\n                        String subExpression4Assign = topicToSubExpression.get(topic);\n                        subExpression4Assign = subExpression4Assign == null ? SubscriptionData.SUB_ALL : subExpression4Assign;\n                        subscriptionData = FilterAPI.buildSubscriptionData(topic, subExpression4Assign);\n                    }\n\n                    PullResult pullResult = pull(messageQueue, subscriptionData, offset, defaultLitePullConsumer.getPullBatchSize());\n                    if (this.isCancelled() || processQueue.isDropped()) {\n                        return;\n                    }\n                    switch (pullResult.getPullStatus()) {\n                        case FOUND:\n                            final Object objLock = messageQueueLock.fetchLockObject(messageQueue);\n                            synchronized (objLock) {\n                                if (pullResult.getMsgFoundList() != null && !pullResult.getMsgFoundList().isEmpty() && assignedMessageQueue.getSeekOffset(messageQueue) == -1) {\n                                    processQueue.putMessage(pullResult.getMsgFoundList());\n                                    submitConsumeRequest(new ConsumeRequest(pullResult.getMsgFoundList(), messageQueue, processQueue));\n                                }\n                            }\n                            break;\n                        case OFFSET_ILLEGAL:\n                            log.warn(\"The pull request offset illegal, {}\", pullResult.toString());\n                            break;\n                        default:\n                            break;\n                    }\n                    updatePullOffset(messageQueue, pullResult.getNextBeginOffset(), processQueue);\n                } catch (InterruptedException interruptedException) {\n                    log.warn(\"Polling thread was interrupted.\", interruptedException);\n                } catch (Throwable e) {\n                    if (e instanceof MQBrokerException && ((MQBrokerException) e).getResponseCode() == ResponseCode.FLOW_CONTROL) {\n                        pullDelayTimeMills = PULL_TIME_DELAY_MILLS_WHEN_BROKER_FLOW_CONTROL;\n                    } else {\n                        pullDelayTimeMills = pullTimeDelayMillsWhenException;\n                    }\n                    log.error(\"An error occurred in pull message process.\", e);\n                }\n\n                if (!this.isCancelled()) {\n                    scheduledThreadPoolExecutor.schedule(this, pullDelayTimeMills, TimeUnit.MILLISECONDS);\n                } else {\n                    log.warn(\"The Pull Task is cancelled after doPullTask, {}\", messageQueue);\n                }\n            }\n        }\n\n        public boolean isCancelled() {\n            return cancelled;\n        }\n\n        public void setCancelled(boolean cancelled) {\n            this.cancelled = cancelled;\n        }\n\n        public MessageQueue getMessageQueue() {\n            return messageQueue;\n        }\n    }\n\n    private PullResult pull(MessageQueue mq, SubscriptionData subscriptionData, long offset, int maxNums)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        return pull(mq, subscriptionData, offset, maxNums, this.defaultLitePullConsumer.getConsumerPullTimeoutMillis());\n    }\n\n    private PullResult pull(MessageQueue mq, SubscriptionData subscriptionData, long offset, int maxNums, long timeout)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        return this.pullSyncImpl(mq, subscriptionData, offset, maxNums, true, timeout);\n    }\n\n    private PullResult pullSyncImpl(MessageQueue mq, SubscriptionData subscriptionData, long offset, int maxNums,\n        boolean block,\n        long timeout)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n\n        if (null == mq) {\n            throw new MQClientException(\"mq is null\", null);\n        }\n\n        if (offset < 0) {\n            throw new MQClientException(\"offset < 0\", null);\n        }\n\n        if (maxNums <= 0) {\n            throw new MQClientException(\"maxNums <= 0\", null);\n        }\n\n        int sysFlag = PullSysFlag.buildSysFlag(false, block, true, false, true);\n\n        long timeoutMillis = block ? this.defaultLitePullConsumer.getConsumerTimeoutMillisWhenSuspend() : timeout;\n\n        boolean isTagType = ExpressionType.isTagType(subscriptionData.getExpressionType());\n        PullResult pullResult = this.pullAPIWrapper.pullKernelImpl(\n            mq,\n            subscriptionData.getSubString(),\n            subscriptionData.getExpressionType(),\n            isTagType ? 0L : subscriptionData.getSubVersion(),\n            offset,\n            maxNums,\n            sysFlag,\n            0,\n            this.defaultLitePullConsumer.getBrokerSuspendMaxTimeMillis(),\n            timeoutMillis,\n            CommunicationMode.SYNC,\n            null\n        );\n        this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData);\n        return pullResult;\n    }\n\n    private void resetTopic(List<MessageExt> msgList) {\n        if (null == msgList || msgList.size() == 0) {\n            return;\n        }\n\n        //If namespace not null , reset Topic without namespace.\n        String namespace = this.defaultLitePullConsumer.getNamespace();\n        if (namespace != null) {\n            for (MessageExt messageExt : msgList) {\n                messageExt.setTopic(NamespaceUtil.withoutNamespace(messageExt.getTopic(), namespace));\n            }\n        }\n    }\n\n    public void updateConsumeOffset(MessageQueue mq, long offset) {\n        checkServiceState();\n        this.offsetStore.updateOffset(mq, offset, false);\n    }\n\n    @Override\n    public String groupName() {\n        return this.defaultLitePullConsumer.getConsumerGroup();\n    }\n\n    @Override\n    public MessageModel messageModel() {\n        return this.defaultLitePullConsumer.getMessageModel();\n    }\n\n    @Override\n    public ConsumeType consumeType() {\n        return ConsumeType.CONSUME_ACTIVELY;\n    }\n\n    @Override\n    public ConsumeFromWhere consumeFromWhere() {\n        return ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET;\n    }\n\n    @Override\n    public Set<SubscriptionData> subscriptions() {\n        Set<SubscriptionData> subSet = new HashSet<>();\n\n        subSet.addAll(this.rebalanceImpl.getSubscriptionInner().values());\n        subSet.addAll(this.defaultLitePullConsumer.getSubscriptionsForHeartbeat());\n        \n        return subSet;\n    }\n\n    @Override\n    public void doRebalance() {\n        if (this.rebalanceImpl != null) {\n            this.rebalanceImpl.doRebalance(false);\n        }\n    }\n\n    @Override\n    public boolean tryRebalance() {\n        if (this.rebalanceImpl != null) {\n            return this.rebalanceImpl.doRebalance(false);\n        }\n        return false;\n    }\n\n    @Override\n    public void persistConsumerOffset() {\n        try {\n            checkServiceState();\n            Set<MessageQueue> mqs = new HashSet<>();\n            if (this.subscriptionType == SubscriptionType.SUBSCRIBE) {\n                Set<MessageQueue> allocateMq = this.rebalanceImpl.getProcessQueueTable().keySet();\n                mqs.addAll(allocateMq);\n            } else if (this.subscriptionType == SubscriptionType.ASSIGN) {\n                Set<MessageQueue> assignedMessageQueue = this.assignedMessageQueue.getAssignedMessageQueues();\n                mqs.addAll(assignedMessageQueue);\n            }\n            this.offsetStore.persistAll(mqs);\n        } catch (Exception e) {\n            log.error(\"Persist consumer offset error for group: {} \", this.defaultLitePullConsumer.getConsumerGroup(), e);\n        }\n    }\n\n    @Override\n    public void updateTopicSubscribeInfo(String topic, Set<MessageQueue> info) {\n        Map<String, SubscriptionData> subTable = this.rebalanceImpl.getSubscriptionInner();\n        if (subTable != null) {\n            if (subTable.containsKey(topic)) {\n                this.rebalanceImpl.getTopicSubscribeInfoTable().put(topic, info);\n            }\n        }\n    }\n\n    @Override\n    public boolean isSubscribeTopicNeedUpdate(String topic) {\n        Map<String, SubscriptionData> subTable = this.rebalanceImpl.getSubscriptionInner();\n        if (subTable != null) {\n            if (subTable.containsKey(topic)) {\n                return !this.rebalanceImpl.topicSubscribeInfoTable.containsKey(topic);\n            }\n        }\n\n        return false;\n    }\n\n    @Override\n    public boolean isUnitMode() {\n        return this.defaultLitePullConsumer.isUnitMode();\n    }\n\n    @Override\n    public ConsumerRunningInfo consumerRunningInfo() {\n        ConsumerRunningInfo info = new ConsumerRunningInfo();\n\n        Properties prop = MixAll.object2Properties(this.defaultLitePullConsumer);\n        prop.put(ConsumerRunningInfo.PROP_CONSUMER_START_TIMESTAMP, String.valueOf(this.consumerStartTimestamp));\n        info.setProperties(prop);\n\n        info.getSubscriptionSet().addAll(this.subscriptions());\n\n        for (MessageQueue mq : this.assignedMessageQueue.getAssignedMessageQueues()) {\n            ProcessQueue pq = this.assignedMessageQueue.getProcessQueue(mq);\n            ProcessQueueInfo pqInfo = new ProcessQueueInfo();\n            pqInfo.setCommitOffset(this.offsetStore.readOffset(mq, ReadOffsetType.MEMORY_FIRST_THEN_STORE));\n            pq.fillProcessQueueInfo(pqInfo);\n            info.getMqTable().put(mq, pqInfo);\n        }\n\n        return info;\n    }\n\n    private void updateConsumeOffsetToBroker(MessageQueue mq, long offset, boolean isOneway) throws RemotingException,\n        MQBrokerException, InterruptedException, MQClientException {\n        this.offsetStore.updateConsumeOffsetToBroker(mq, offset, isOneway);\n    }\n\n    public OffsetStore getOffsetStore() {\n        return offsetStore;\n    }\n\n    public DefaultLitePullConsumer getDefaultLitePullConsumer() {\n        return defaultLitePullConsumer;\n    }\n\n    public Set<MessageQueue> fetchMessageQueues(String topic) throws MQClientException {\n        checkServiceState();\n        Set<MessageQueue> result = this.mQClientFactory.getMQAdminImpl().fetchSubscribeMessageQueues(topic);\n        return parseMessageQueues(result);\n    }\n\n    private synchronized void fetchTopicMessageQueuesAndCompare() throws MQClientException {\n        for (Map.Entry<String, TopicMessageQueueChangeListener> entry : topicMessageQueueChangeListenerMap.entrySet()) {\n            String topic = entry.getKey();\n            TopicMessageQueueChangeListener topicMessageQueueChangeListener = entry.getValue();\n            Set<MessageQueue> oldMessageQueues = messageQueuesForTopic.get(topic);\n            Set<MessageQueue> newMessageQueues = fetchMessageQueues(topic);\n            boolean isChanged = !isSetEqual(newMessageQueues, oldMessageQueues);\n            if (isChanged) {\n                messageQueuesForTopic.put(topic, newMessageQueues);\n                if (topicMessageQueueChangeListener != null) {\n                    topicMessageQueueChangeListener.onChanged(topic, newMessageQueues);\n                }\n            }\n        }\n    }\n\n    private boolean isSetEqual(Set<MessageQueue> set1, Set<MessageQueue> set2) {\n        if (set1 == null && set2 == null) {\n            return true;\n        }\n\n        if (set1 == null || set2 == null || set1.size() != set2.size()) {\n            return false;\n        }\n\n        for (MessageQueue messageQueue : set2) {\n            if (!set1.contains(messageQueue)) {\n                return false;\n            }\n        }\n        return true;\n    }\n\n    public AssignedMessageQueue getAssignedMessageQueue() {\n        return assignedMessageQueue;\n    }\n\n    public synchronized void registerTopicMessageQueueChangeListener(String topic,\n        TopicMessageQueueChangeListener listener) throws MQClientException {\n        if (topic == null || listener == null) {\n            throw new MQClientException(\"Topic or listener is null\", null);\n        }\n        if (topicMessageQueueChangeListenerMap.containsKey(topic)) {\n            log.warn(\"Topic {} had been registered, new listener will overwrite the old one\", topic);\n        }\n        topicMessageQueueChangeListenerMap.put(topic, listener);\n        if (this.serviceState == ServiceState.RUNNING) {\n            Set<MessageQueue> messageQueues = fetchMessageQueues(topic);\n            messageQueuesForTopic.put(topic, messageQueues);\n        }\n    }\n\n    private Set<MessageQueue> parseMessageQueues(Set<MessageQueue> queueSet) {\n        Set<MessageQueue> resultQueues = new HashSet<>();\n        for (MessageQueue messageQueue : queueSet) {\n            String userTopic = NamespaceUtil.withoutNamespace(messageQueue.getTopic(),\n                this.defaultLitePullConsumer.getNamespace());\n            resultQueues.add(new MessageQueue(userTopic, messageQueue.getBrokerName(), messageQueue.getQueueId()));\n        }\n        return resultQueues;\n    }\n\n    public class ConsumeRequest {\n        private final List<MessageExt> messageExts;\n        private final MessageQueue messageQueue;\n        private final ProcessQueue processQueue;\n\n        public ConsumeRequest(final List<MessageExt> messageExts, final MessageQueue messageQueue,\n            final ProcessQueue processQueue) {\n            this.messageExts = messageExts;\n            this.messageQueue = messageQueue;\n            this.processQueue = processQueue;\n        }\n\n        public List<MessageExt> getMessageExts() {\n            return messageExts;\n        }\n\n        public MessageQueue getMessageQueue() {\n            return messageQueue;\n        }\n\n        public ProcessQueue getProcessQueue() {\n            return processQueue;\n        }\n\n    }\n\n    public void setPullTimeDelayMillsWhenException(long pullTimeDelayMillsWhenException) {\n        this.pullTimeDelayMillsWhenException = pullTimeDelayMillsWhenException;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Properties;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.client.QueryResult;\nimport org.apache.rocketmq.client.Validators;\nimport org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;\nimport org.apache.rocketmq.client.consumer.MessageSelector;\nimport org.apache.rocketmq.client.consumer.PullCallback;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.store.LocalFileOffsetStore;\nimport org.apache.rocketmq.client.consumer.store.OffsetStore;\nimport org.apache.rocketmq.client.consumer.store.ReadOffsetType;\nimport org.apache.rocketmq.client.consumer.store.RemoteBrokerOffsetStore;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.hook.ConsumeMessageContext;\nimport org.apache.rocketmq.client.hook.ConsumeMessageHook;\nimport org.apache.rocketmq.client.hook.FilterMessageHook;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.MQClientManager;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ServiceState;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.common.help.FAQUrl;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.sysflag.PullSysFlag;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.filter.FilterAPI;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\n/**\n * This class will be removed in 2022, and a better implementation {@link DefaultLitePullConsumerImpl} is recommend to use\n * in the scenario of actively pulling messages.\n */\n@Deprecated\npublic class DefaultMQPullConsumerImpl implements MQConsumerInner {\n    private static final Logger log = LoggerFactory.getLogger(DefaultMQPullConsumerImpl.class);\n    private final DefaultMQPullConsumer defaultMQPullConsumer;\n    private final long consumerStartTimestamp = System.currentTimeMillis();\n    private final RPCHook rpcHook;\n    private final ArrayList<ConsumeMessageHook> consumeMessageHookList = new ArrayList<>();\n    private final ArrayList<FilterMessageHook> filterMessageHookList = new ArrayList<>();\n    private volatile ServiceState serviceState = ServiceState.CREATE_JUST;\n    protected MQClientInstance mQClientFactory;\n    private PullAPIWrapper pullAPIWrapper;\n    private OffsetStore offsetStore;\n    private RebalanceImpl rebalanceImpl = new RebalancePullImpl(this);\n\n    public DefaultMQPullConsumerImpl(final DefaultMQPullConsumer defaultMQPullConsumer, final RPCHook rpcHook) {\n        this.defaultMQPullConsumer = defaultMQPullConsumer;\n        this.rpcHook = rpcHook;\n    }\n\n    public void registerConsumeMessageHook(final ConsumeMessageHook hook) {\n        this.consumeMessageHookList.add(hook);\n        log.info(\"register consumeMessageHook Hook, {}\", hook.hookName());\n    }\n\n    public void createTopic(String key, String newTopic, int queueNum) throws MQClientException {\n        createTopic(key, newTopic, queueNum, 0);\n    }\n\n    public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) throws MQClientException {\n        this.isRunning();\n        this.mQClientFactory.getMQAdminImpl().createTopic(key, newTopic, queueNum, topicSysFlag, null);\n    }\n\n    private void isRunning() throws MQClientException {\n        if (this.serviceState != ServiceState.RUNNING) {\n            throw new MQClientException(\"The consumer is not in running status, \"\n                + this.serviceState\n                + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK),\n                null);\n        }\n    }\n\n    public long fetchConsumeOffset(MessageQueue mq, boolean fromStore) throws MQClientException {\n        this.isRunning();\n        return this.offsetStore.readOffset(mq, fromStore ? ReadOffsetType.READ_FROM_STORE : ReadOffsetType.MEMORY_FIRST_THEN_STORE);\n    }\n\n    public Set<MessageQueue> fetchMessageQueuesInBalance(String topic) throws MQClientException {\n        this.isRunning();\n        if (null == topic) {\n            throw new IllegalArgumentException(\"topic is null\");\n        }\n\n        ConcurrentMap<MessageQueue, ProcessQueue> mqTable = this.rebalanceImpl.getProcessQueueTable();\n        Set<MessageQueue> mqResult = new HashSet<>();\n        for (MessageQueue mq : mqTable.keySet()) {\n            if (mq.getTopic().equals(topic)) {\n                mqResult.add(mq);\n            }\n        }\n\n        return parseSubscribeMessageQueues(mqResult);\n    }\n\n    public List<MessageQueue> fetchPublishMessageQueues(String topic) throws MQClientException {\n        this.isRunning();\n        return this.mQClientFactory.getMQAdminImpl().fetchPublishMessageQueues(topic);\n    }\n\n    public Set<MessageQueue> fetchSubscribeMessageQueues(String topic) throws MQClientException {\n        this.isRunning();\n        // check if has info in memory, otherwise invoke api.\n        Set<MessageQueue> result = this.rebalanceImpl.getTopicSubscribeInfoTable().get(topic);\n        if (null == result) {\n            result = this.mQClientFactory.getMQAdminImpl().fetchSubscribeMessageQueues(topic);\n        }\n\n        return parseSubscribeMessageQueues(result);\n    }\n\n    public Set<MessageQueue> parseSubscribeMessageQueues(Set<MessageQueue> queueSet) {\n        Set<MessageQueue> resultQueues = new HashSet<>();\n        for (MessageQueue messageQueue : queueSet) {\n            String userTopic = NamespaceUtil.withoutNamespace(messageQueue.getTopic(),\n                this.defaultMQPullConsumer.getNamespace());\n            resultQueues.add(new MessageQueue(userTopic, messageQueue.getBrokerName(), messageQueue.getQueueId()));\n        }\n        return resultQueues;\n    }\n\n    public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException {\n        this.isRunning();\n        return this.mQClientFactory.getMQAdminImpl().earliestMsgStoreTime(mq);\n    }\n\n    public long maxOffset(MessageQueue mq) throws MQClientException {\n        this.isRunning();\n        return this.mQClientFactory.getMQAdminImpl().maxOffset(mq);\n    }\n\n    public long minOffset(MessageQueue mq) throws MQClientException {\n        this.isRunning();\n        return this.mQClientFactory.getMQAdminImpl().minOffset(mq);\n    }\n\n    public PullResult pull(MessageQueue mq, String subExpression, long offset, int maxNums)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        return pull(mq, subExpression, offset, maxNums, this.defaultMQPullConsumer.getConsumerPullTimeoutMillis());\n    }\n\n    public PullResult pull(MessageQueue mq, String subExpression, long offset, int maxNums, long timeout)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        SubscriptionData subscriptionData = getSubscriptionData(mq, subExpression);\n        return this.pullSyncImpl(mq, subscriptionData, offset, maxNums, false, timeout);\n    }\n\n    public PullResult pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        return pull(mq, messageSelector, offset, maxNums, this.defaultMQPullConsumer.getConsumerPullTimeoutMillis());\n    }\n\n    public PullResult pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums, long timeout)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        SubscriptionData subscriptionData = getSubscriptionData(mq, messageSelector);\n        return this.pullSyncImpl(mq, subscriptionData, offset, maxNums, false, timeout);\n    }\n\n    private SubscriptionData getSubscriptionData(MessageQueue mq, String subExpression)\n        throws MQClientException {\n\n        if (null == mq) {\n            throw new MQClientException(\"mq is null\", null);\n        }\n\n        try {\n            return FilterAPI.buildSubscriptionData(mq.getTopic(), subExpression);\n        } catch (Exception e) {\n            throw new MQClientException(\"parse subscription error\", e);\n        }\n    }\n\n    private SubscriptionData getSubscriptionData(MessageQueue mq, MessageSelector messageSelector)\n        throws MQClientException {\n\n        if (null == mq) {\n            throw new MQClientException(\"mq is null\", null);\n        }\n\n        try {\n            return FilterAPI.build(mq.getTopic(),\n                messageSelector.getExpression(), messageSelector.getExpressionType());\n        } catch (Exception e) {\n            throw new MQClientException(\"parse subscription error\", e);\n        }\n    }\n\n    private PullResult pullSyncImpl(MessageQueue mq, SubscriptionData subscriptionData, long offset, int maxNums, boolean block,\n        long timeout)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        this.isRunning();\n\n        if (null == mq) {\n            throw new MQClientException(\"mq is null\", null);\n        }\n\n        if (offset < 0) {\n            throw new MQClientException(\"offset < 0\", null);\n        }\n\n        if (maxNums <= 0) {\n            throw new MQClientException(\"maxNums <= 0\", null);\n        }\n\n        this.subscriptionAutomatically(mq.getTopic());\n\n        int sysFlag = PullSysFlag.buildSysFlag(false, block, true, false);\n\n        long timeoutMillis = block ? this.defaultMQPullConsumer.getConsumerTimeoutMillisWhenSuspend() : timeout;\n\n        boolean isTagType = ExpressionType.isTagType(subscriptionData.getExpressionType());\n        PullResult pullResult = this.pullAPIWrapper.pullKernelImpl(\n            mq,\n            subscriptionData.getSubString(),\n            subscriptionData.getExpressionType(),\n            isTagType ? 0L : subscriptionData.getSubVersion(),\n            offset,\n            maxNums,\n            sysFlag,\n            0,\n            this.defaultMQPullConsumer.getBrokerSuspendMaxTimeMillis(),\n            timeoutMillis,\n            CommunicationMode.SYNC,\n            null\n        );\n        this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData);\n        //If namespace is not null , reset Topic without namespace.\n        this.resetTopic(pullResult.getMsgFoundList());\n        if (!this.consumeMessageHookList.isEmpty()) {\n            ConsumeMessageContext consumeMessageContext = null;\n            consumeMessageContext = new ConsumeMessageContext();\n            consumeMessageContext.setNamespace(defaultMQPullConsumer.getNamespace());\n            consumeMessageContext.setConsumerGroup(this.groupName());\n            consumeMessageContext.setMq(mq);\n            consumeMessageContext.setMsgList(pullResult.getMsgFoundList());\n            consumeMessageContext.setSuccess(false);\n            this.executeHookBefore(consumeMessageContext);\n            consumeMessageContext.setStatus(ConsumeConcurrentlyStatus.CONSUME_SUCCESS.toString());\n            consumeMessageContext.setSuccess(true);\n            consumeMessageContext.setAccessChannel(defaultMQPullConsumer.getAccessChannel());\n            this.executeHookAfter(consumeMessageContext);\n        }\n        return pullResult;\n    }\n\n    public void resetTopic(List<MessageExt> msgList) {\n        if (null == msgList || msgList.size() == 0) {\n            return;\n        }\n\n        //If namespace not null , reset Topic without namespace.\n        String namespace = this.getDefaultMQPullConsumer().getNamespace();\n        if (namespace != null) {\n            for (MessageExt messageExt : msgList) {\n                messageExt.setTopic(NamespaceUtil.withoutNamespace(messageExt.getTopic(), namespace));\n            }\n        }\n    }\n\n    public void subscriptionAutomatically(final String topic) {\n        if (!this.rebalanceImpl.getSubscriptionInner().containsKey(topic)) {\n            try {\n                SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(topic, SubscriptionData.SUB_ALL);\n                this.rebalanceImpl.subscriptionInner.putIfAbsent(topic, subscriptionData);\n            } catch (Exception ignore) {\n            }\n        }\n    }\n\n    public void unsubscribe(String topic) {\n        this.rebalanceImpl.getSubscriptionInner().remove(topic);\n    }\n\n    @Override\n    public String groupName() {\n        return this.defaultMQPullConsumer.getConsumerGroup();\n    }\n\n    public void executeHookBefore(final ConsumeMessageContext context) {\n        if (!this.consumeMessageHookList.isEmpty()) {\n            for (ConsumeMessageHook hook : this.consumeMessageHookList) {\n                try {\n                    hook.consumeMessageBefore(context);\n                } catch (Throwable ignored) {\n                }\n            }\n        }\n    }\n\n    public void executeHookAfter(final ConsumeMessageContext context) {\n        if (!this.consumeMessageHookList.isEmpty()) {\n            for (ConsumeMessageHook hook : this.consumeMessageHookList) {\n                try {\n                    hook.consumeMessageAfter(context);\n                } catch (Throwable ignored) {\n                }\n            }\n        }\n    }\n\n    @Override\n    public MessageModel messageModel() {\n        return this.defaultMQPullConsumer.getMessageModel();\n    }\n\n    @Override\n    public ConsumeType consumeType() {\n        return ConsumeType.CONSUME_ACTIVELY;\n    }\n\n    @Override\n    public ConsumeFromWhere consumeFromWhere() {\n        return ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET;\n    }\n\n    @Override\n    public Set<SubscriptionData> subscriptions() {\n        Set<SubscriptionData> registerSubscriptions = defaultMQPullConsumer.getRegisterSubscriptions();\n        if (registerSubscriptions != null && !registerSubscriptions.isEmpty()) {\n            return registerSubscriptions;\n        }\n\n        Set<SubscriptionData> result = new HashSet<>();\n\n        Set<String> topics = this.defaultMQPullConsumer.getRegisterTopics();\n        if (topics != null) {\n            synchronized (topics) {\n                for (String t : topics) {\n                    SubscriptionData ms = null;\n                    try {\n                        ms = FilterAPI.buildSubscriptionData(t, SubscriptionData.SUB_ALL);\n                    } catch (Exception e) {\n                        log.error(\"parse subscription error\", e);\n                    }\n                    if (ms != null) {\n                        ms.setSubVersion(0L);\n                        result.add(ms);\n                    }\n                }\n            }\n        }\n\n        return result;\n    }\n\n    @Override\n    public void doRebalance() {\n        if (!defaultMQPullConsumer.isEnableRebalance()) {\n            return;\n        }\n        if (this.rebalanceImpl != null) {\n            this.rebalanceImpl.doRebalance(false);\n        }\n    }\n\n    @Override\n    public boolean tryRebalance() {\n        if (!defaultMQPullConsumer.isEnableRebalance()) {\n            return true;\n        }\n\n        if (this.rebalanceImpl != null) {\n            return this.rebalanceImpl.doRebalance(false);\n        }\n        return false;\n    }\n\n    @Override\n    public void persistConsumerOffset() {\n        try {\n            this.isRunning();\n            Set<MessageQueue> mqs = new HashSet<>();\n            Set<MessageQueue> allocateMq = this.rebalanceImpl.getProcessQueueTable().keySet();\n            mqs.addAll(allocateMq);\n            this.offsetStore.persistAll(mqs);\n        } catch (Exception e) {\n            log.error(\"group: \" + this.defaultMQPullConsumer.getConsumerGroup() + \" persistConsumerOffset exception\", e);\n        }\n    }\n\n    @Override\n    public void updateTopicSubscribeInfo(String topic, Set<MessageQueue> info) {\n        Map<String, SubscriptionData> subTable = this.rebalanceImpl.getSubscriptionInner();\n        if (subTable != null) {\n            if (subTable.containsKey(topic)) {\n                this.rebalanceImpl.getTopicSubscribeInfoTable().put(topic, info);\n            }\n        }\n    }\n\n    @Override\n    public boolean isSubscribeTopicNeedUpdate(String topic) {\n        Map<String, SubscriptionData> subTable = this.rebalanceImpl.getSubscriptionInner();\n        if (subTable != null) {\n            if (subTable.containsKey(topic)) {\n                return !this.rebalanceImpl.topicSubscribeInfoTable.containsKey(topic);\n            }\n        }\n\n        return false;\n    }\n\n    @Override\n    public boolean isUnitMode() {\n        return this.defaultMQPullConsumer.isUnitMode();\n    }\n\n    @Override\n    public ConsumerRunningInfo consumerRunningInfo() {\n        ConsumerRunningInfo info = new ConsumerRunningInfo();\n\n        Properties prop = MixAll.object2Properties(this.defaultMQPullConsumer);\n        prop.put(ConsumerRunningInfo.PROP_CONSUMER_START_TIMESTAMP, String.valueOf(this.consumerStartTimestamp));\n        info.setProperties(prop);\n\n        info.getSubscriptionSet().addAll(this.subscriptions());\n        return info;\n    }\n\n    public void pull(MessageQueue mq, String subExpression, long offset, int maxNums, PullCallback pullCallback)\n        throws MQClientException, RemotingException, InterruptedException {\n        pull(mq, subExpression, offset, maxNums, pullCallback, this.defaultMQPullConsumer.getConsumerPullTimeoutMillis());\n    }\n\n    public void pull(MessageQueue mq, String subExpression, long offset, int maxNums, PullCallback pullCallback,\n        long timeout)\n        throws MQClientException, RemotingException, InterruptedException {\n        SubscriptionData subscriptionData = getSubscriptionData(mq, subExpression);\n        this.pullAsyncImpl(mq, subscriptionData, offset, maxNums, pullCallback, false, timeout);\n    }\n\n    public void pull(MessageQueue mq, String subExpression, long offset, int maxNums, int maxSize, PullCallback pullCallback,\n        long timeout)\n        throws MQClientException, RemotingException, InterruptedException {\n        SubscriptionData subscriptionData = getSubscriptionData(mq, subExpression);\n        this.pullAsyncImpl(mq, subscriptionData, offset, maxNums, maxSize, pullCallback, false, timeout);\n    }\n\n    public void pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums,\n        PullCallback pullCallback)\n        throws MQClientException, RemotingException, InterruptedException {\n        pull(mq, messageSelector, offset, maxNums, pullCallback, this.defaultMQPullConsumer.getConsumerPullTimeoutMillis());\n    }\n\n    public void pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums,\n        PullCallback pullCallback,\n        long timeout)\n        throws MQClientException, RemotingException, InterruptedException {\n        SubscriptionData subscriptionData = getSubscriptionData(mq, messageSelector);\n        this.pullAsyncImpl(mq, subscriptionData, offset, maxNums, pullCallback, false, timeout);\n    }\n\n    private void pullAsyncImpl(\n        final MessageQueue mq,\n        final SubscriptionData subscriptionData,\n        final long offset,\n        final int maxNums,\n        final int maxSizeInBytes,\n        final PullCallback pullCallback,\n        final boolean block,\n        final long timeout) throws MQClientException, RemotingException, InterruptedException {\n        this.isRunning();\n\n        if (null == mq) {\n            throw new MQClientException(\"mq is null\", null);\n        }\n\n        if (offset < 0) {\n            throw new MQClientException(\"offset < 0\", null);\n        }\n\n        if (maxNums <= 0) {\n            throw new MQClientException(\"maxNums <= 0\", null);\n        }\n\n        if (maxSizeInBytes <= 0) {\n            throw new MQClientException(\"maxSizeInBytes <= 0\", null);\n        }\n\n\n        if (null == pullCallback) {\n            throw new MQClientException(\"pullCallback is null\", null);\n        }\n\n        this.subscriptionAutomatically(mq.getTopic());\n\n        try {\n            int sysFlag = PullSysFlag.buildSysFlag(false, block, true, false);\n\n            long timeoutMillis = block ? this.defaultMQPullConsumer.getConsumerTimeoutMillisWhenSuspend() : timeout;\n\n            boolean isTagType = ExpressionType.isTagType(subscriptionData.getExpressionType());\n            this.pullAPIWrapper.pullKernelImpl(\n                mq,\n                subscriptionData.getSubString(),\n                subscriptionData.getExpressionType(),\n                isTagType ? 0L : subscriptionData.getSubVersion(),\n                offset,\n                maxNums,\n                maxSizeInBytes,\n                sysFlag,\n                0,\n                this.defaultMQPullConsumer.getBrokerSuspendMaxTimeMillis(),\n                timeoutMillis,\n                CommunicationMode.ASYNC,\n                new PullCallback() {\n\n                    @Override\n                    public void onSuccess(PullResult pullResult) {\n                        PullResult userPullResult = DefaultMQPullConsumerImpl.this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData);\n                        resetTopic(userPullResult.getMsgFoundList());\n                        pullCallback.onSuccess(userPullResult);\n                    }\n\n                    @Override\n                    public void onException(Throwable e) {\n                        pullCallback.onException(e);\n                    }\n                });\n        } catch (MQBrokerException e) {\n            throw new MQClientException(\"pullAsync unknown exception\", e);\n        }\n    }\n\n    private void pullAsyncImpl(\n            final MessageQueue mq,\n            final SubscriptionData subscriptionData,\n            final long offset,\n            final int maxNums,\n            final PullCallback pullCallback,\n            final boolean block,\n            final long timeout) throws MQClientException, RemotingException, InterruptedException {\n        pullAsyncImpl(\n                mq,\n                subscriptionData,\n                offset,\n                maxNums,\n                Integer.MAX_VALUE,\n                pullCallback,\n                block,\n                timeout\n        );\n    }\n\n    public PullResult pullBlockIfNotFound(MessageQueue mq, String subExpression, long offset, int maxNums)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        SubscriptionData subscriptionData = getSubscriptionData(mq, subExpression);\n        return this.pullSyncImpl(mq, subscriptionData, offset, maxNums, true, this.getDefaultMQPullConsumer().getConsumerPullTimeoutMillis());\n    }\n\n    public DefaultMQPullConsumer getDefaultMQPullConsumer() {\n        return defaultMQPullConsumer;\n    }\n\n    public void pullBlockIfNotFound(MessageQueue mq, String subExpression, long offset, int maxNums,\n        PullCallback pullCallback)\n        throws MQClientException, RemotingException, InterruptedException {\n        SubscriptionData subscriptionData = getSubscriptionData(mq, subExpression);\n        this.pullAsyncImpl(mq, subscriptionData, offset, maxNums, pullCallback, true,\n            this.getDefaultMQPullConsumer().getConsumerPullTimeoutMillis());\n    }\n\n    public void pullBlockIfNotFoundWithMessageSelector(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums,\n        PullCallback pullCallback)\n        throws MQClientException, RemotingException, InterruptedException {\n        SubscriptionData subscriptionData = getSubscriptionData(mq, messageSelector);\n        this.pullAsyncImpl(mq, subscriptionData, offset, maxNums, pullCallback, true,\n            this.getDefaultMQPullConsumer().getConsumerPullTimeoutMillis());\n    }\n\n    public PullResult pullBlockIfNotFoundWithMessageSelector(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums)\n        throws MQClientException, RemotingException, InterruptedException, MQBrokerException {\n        SubscriptionData subscriptionData = getSubscriptionData(mq, messageSelector);\n        return this.pullSyncImpl(mq, subscriptionData, offset, maxNums, true, this.getDefaultMQPullConsumer().getConsumerPullTimeoutMillis());\n    }\n\n\n    public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end)\n        throws MQClientException, InterruptedException {\n        this.isRunning();\n        return this.mQClientFactory.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end);\n    }\n\n    public MessageExt queryMessageByUniqKey(String topic, String uniqKey)\n        throws MQClientException, InterruptedException {\n        this.isRunning();\n        return this.mQClientFactory.getMQAdminImpl().queryMessageByUniqKey(topic, uniqKey);\n    }\n\n    public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException {\n        this.isRunning();\n        return this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp);\n    }\n\n    public void sendMessageBack(MessageExt msg, int delayLevel, final String brokerName)\n        throws RemotingException, MQBrokerException, InterruptedException, MQClientException {\n        sendMessageBack(msg, delayLevel, brokerName, this.defaultMQPullConsumer.getConsumerGroup());\n    }\n\n    public void updateConsumeOffsetToBroker(MessageQueue mq, long offset, boolean isOneway) throws RemotingException,\n        MQBrokerException, InterruptedException, MQClientException {\n        this.offsetStore.updateConsumeOffsetToBroker(mq, offset, isOneway);\n    }\n\n    @Deprecated\n    public void sendMessageBack(MessageExt msg, int delayLevel, final String brokerName, String consumerGroup)\n        throws RemotingException, MQBrokerException, InterruptedException, MQClientException {\n        try {\n            String destBrokerName = brokerName;\n            if (destBrokerName != null && destBrokerName.startsWith(MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX)) {\n                destBrokerName = this.mQClientFactory.getBrokerNameFromMessageQueue(this.defaultMQPullConsumer.queueWithNamespace(new MessageQueue(msg.getTopic(), msg.getBrokerName(), msg.getQueueId())));\n            }\n            String brokerAddr = (null != destBrokerName) ? this.mQClientFactory.findBrokerAddressInPublish(destBrokerName)\n                : RemotingHelper.parseSocketAddressAddr(msg.getStoreHost());\n\n            if (UtilAll.isBlank(brokerAddr)) {\n                throw new MQClientException(\"Broker[\" + destBrokerName + \"] master node does not exist\", null);\n            }\n\n            if (UtilAll.isBlank(consumerGroup)) {\n                consumerGroup = this.defaultMQPullConsumer.getConsumerGroup();\n            }\n\n            this.mQClientFactory.getMQClientAPIImpl().consumerSendMessageBack(brokerAddr, brokerName, msg, consumerGroup,\n                delayLevel, 3000, this.defaultMQPullConsumer.getMaxReconsumeTimes());\n        } catch (Exception e) {\n            log.error(\"sendMessageBack Exception, \" + this.defaultMQPullConsumer.getConsumerGroup(), e);\n\n            Message newMsg = new Message(MixAll.getRetryTopic(this.defaultMQPullConsumer.getConsumerGroup()), msg.getBody());\n            String originMsgId = MessageAccessor.getOriginMessageId(msg);\n            MessageAccessor.setOriginMessageId(newMsg, UtilAll.isBlank(originMsgId) ? msg.getMsgId() : originMsgId);\n            newMsg.setFlag(msg.getFlag());\n            MessageAccessor.setProperties(newMsg, msg.getProperties());\n            MessageAccessor.putProperty(newMsg, MessageConst.PROPERTY_RETRY_TOPIC, msg.getTopic());\n            MessageAccessor.setReconsumeTime(newMsg, String.valueOf(msg.getReconsumeTimes() + 1));\n            MessageAccessor.setMaxReconsumeTimes(newMsg, String.valueOf(this.defaultMQPullConsumer.getMaxReconsumeTimes()));\n            newMsg.setDelayTimeLevel(3 + msg.getReconsumeTimes());\n            this.mQClientFactory.getDefaultMQProducer().send(newMsg);\n        } finally {\n            msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQPullConsumer.getNamespace()));\n        }\n    }\n\n    public synchronized void shutdown() {\n        switch (this.serviceState) {\n            case CREATE_JUST:\n                break;\n            case RUNNING:\n                this.persistConsumerOffset();\n                this.mQClientFactory.unregisterConsumer(this.defaultMQPullConsumer.getConsumerGroup());\n                this.mQClientFactory.shutdown();\n                log.info(\"the consumer [{}] shutdown OK\", this.defaultMQPullConsumer.getConsumerGroup());\n                this.serviceState = ServiceState.SHUTDOWN_ALREADY;\n                break;\n            case SHUTDOWN_ALREADY:\n                break;\n            default:\n                break;\n        }\n    }\n\n    public synchronized void start() throws MQClientException {\n        switch (this.serviceState) {\n            case CREATE_JUST:\n                this.serviceState = ServiceState.START_FAILED;\n\n                this.checkConfig();\n\n                this.copySubscription();\n\n                if (this.defaultMQPullConsumer.getMessageModel() == MessageModel.CLUSTERING) {\n                    this.defaultMQPullConsumer.changeInstanceNameToPID();\n                }\n\n                this.mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultMQPullConsumer, this.rpcHook);\n\n                this.rebalanceImpl.setConsumerGroup(this.defaultMQPullConsumer.getConsumerGroup());\n                this.rebalanceImpl.setMessageModel(this.defaultMQPullConsumer.getMessageModel());\n                this.rebalanceImpl.setAllocateMessageQueueStrategy(this.defaultMQPullConsumer.getAllocateMessageQueueStrategy());\n                this.rebalanceImpl.setmQClientFactory(this.mQClientFactory);\n\n                this.pullAPIWrapper = new PullAPIWrapper(\n                    mQClientFactory,\n                    this.defaultMQPullConsumer.getConsumerGroup(), isUnitMode());\n                this.pullAPIWrapper.registerFilterMessageHook(filterMessageHookList);\n\n                if (this.defaultMQPullConsumer.getOffsetStore() != null) {\n                    this.offsetStore = this.defaultMQPullConsumer.getOffsetStore();\n                } else {\n                    switch (this.defaultMQPullConsumer.getMessageModel()) {\n                        case BROADCASTING:\n                            this.offsetStore = new LocalFileOffsetStore(this.mQClientFactory, this.defaultMQPullConsumer.getConsumerGroup());\n                            break;\n                        case CLUSTERING:\n                            this.offsetStore = new RemoteBrokerOffsetStore(this.mQClientFactory, this.defaultMQPullConsumer.getConsumerGroup());\n                            break;\n                        default:\n                            break;\n                    }\n                    this.defaultMQPullConsumer.setOffsetStore(this.offsetStore);\n                }\n\n                this.offsetStore.load();\n\n                boolean registerOK = mQClientFactory.registerConsumer(this.defaultMQPullConsumer.getConsumerGroup(), this);\n                if (!registerOK) {\n                    this.serviceState = ServiceState.CREATE_JUST;\n\n                    throw new MQClientException(\"The consumer group[\" + this.defaultMQPullConsumer.getConsumerGroup()\n                        + \"] has been created before, specify another name please.\" + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL),\n                        null);\n                }\n\n                mQClientFactory.start();\n                log.info(\"the consumer [{}] start OK\", this.defaultMQPullConsumer.getConsumerGroup());\n                this.serviceState = ServiceState.RUNNING;\n                break;\n            case RUNNING:\n            case START_FAILED:\n            case SHUTDOWN_ALREADY:\n                throw new MQClientException(\"The PullConsumer service state not OK, maybe started once, \"\n                    + this.serviceState\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK),\n                    null);\n            default:\n                break;\n        }\n\n    }\n\n    private void checkConfig() throws MQClientException {\n        // check consumerGroup\n        Validators.checkGroup(this.defaultMQPullConsumer.getConsumerGroup());\n\n        // consumerGroup\n        if (null == this.defaultMQPullConsumer.getConsumerGroup()) {\n            throw new MQClientException(\n                \"consumerGroup is null\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        // consumerGroup\n        if (this.defaultMQPullConsumer.getConsumerGroup().equals(MixAll.DEFAULT_CONSUMER_GROUP)) {\n            throw new MQClientException(\n                \"consumerGroup can not equal \"\n                    + MixAll.DEFAULT_CONSUMER_GROUP\n                    + \", please specify another one.\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        // messageModel\n        if (null == this.defaultMQPullConsumer.getMessageModel()) {\n            throw new MQClientException(\n                \"messageModel is null\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        // allocateMessageQueueStrategy\n        if (null == this.defaultMQPullConsumer.getAllocateMessageQueueStrategy()) {\n            throw new MQClientException(\n                \"allocateMessageQueueStrategy is null\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        // allocateMessageQueueStrategy\n        if (this.defaultMQPullConsumer.getConsumerTimeoutMillisWhenSuspend() < this.defaultMQPullConsumer.getBrokerSuspendMaxTimeMillis()) {\n            throw new MQClientException(\n                \"Long polling mode, the consumer consumerTimeoutMillisWhenSuspend must greater than brokerSuspendMaxTimeMillis\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n    }\n\n    private void copySubscription() throws MQClientException {\n        try {\n            Set<String> registerTopics = this.defaultMQPullConsumer.getRegisterTopics();\n            if (registerTopics != null) {\n                for (final String topic : registerTopics) {\n                    SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(topic, SubscriptionData.SUB_ALL);\n                    this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData);\n                }\n            }\n        } catch (Exception e) {\n            throw new MQClientException(\"subscription exception\", e);\n        }\n    }\n\n    public void updateConsumeOffset(MessageQueue mq, long offset) throws MQClientException {\n        this.isRunning();\n        this.offsetStore.updateOffset(mq, offset, false);\n    }\n\n    public MessageExt viewMessage(String topic, String msgId)\n        throws RemotingException, MQBrokerException, InterruptedException, MQClientException {\n        this.isRunning();\n        return this.mQClientFactory.getMQAdminImpl().viewMessage(topic, msgId);\n    }\n\n    public void registerFilterMessageHook(final FilterMessageHook hook) {\n        this.filterMessageHookList.add(hook);\n        log.info(\"register FilterMessageHook Hook, {}\", hook.hookName());\n    }\n\n    public OffsetStore getOffsetStore() {\n        return offsetStore;\n    }\n\n    public void setOffsetStore(OffsetStore offsetStore) {\n        this.offsetStore = offsetStore;\n    }\n\n    public PullAPIWrapper getPullAPIWrapper() {\n        return pullAPIWrapper;\n    }\n\n    public void setPullAPIWrapper(PullAPIWrapper pullAPIWrapper) {\n        this.pullAPIWrapper = pullAPIWrapper;\n    }\n\n    public ServiceState getServiceState() {\n        return serviceState;\n    }\n\n    //Don't use this deprecated setter, which will be removed soon.\n    @Deprecated\n    public void setServiceState(ServiceState serviceState) {\n        this.serviceState = serviceState;\n    }\n\n    public long getConsumerStartTimestamp() {\n        return consumerStartTimestamp;\n    }\n\n    public RebalanceImpl getRebalanceImpl() {\n        return rebalanceImpl;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport java.util.ArrayList;\nimport java.util.Date;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Map.Entry;\nimport java.util.Properties;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.QueryResult;\nimport org.apache.rocketmq.client.Validators;\nimport org.apache.rocketmq.client.consumer.AckCallback;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.AckStatus;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.MessageQueueListener;\nimport org.apache.rocketmq.client.consumer.MessageSelector;\nimport org.apache.rocketmq.client.consumer.PopCallback;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PopStatus;\nimport org.apache.rocketmq.client.consumer.PullCallback;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.listener.MessageListener;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;\nimport org.apache.rocketmq.client.consumer.store.LocalFileOffsetStore;\nimport org.apache.rocketmq.client.consumer.store.OffsetStore;\nimport org.apache.rocketmq.client.consumer.store.ReadOffsetType;\nimport org.apache.rocketmq.client.consumer.store.RemoteBrokerOffsetStore;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.hook.ConsumeMessageContext;\nimport org.apache.rocketmq.client.hook.ConsumeMessageHook;\nimport org.apache.rocketmq.client.hook.FilterMessageContext;\nimport org.apache.rocketmq.client.hook.FilterMessageHook;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.FindBrokerResult;\nimport org.apache.rocketmq.client.impl.MQClientManager;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.client.stat.ConsumerStatsManager;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ServiceState;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.help.FAQUrl;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.sysflag.PullSysFlag;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeStatus;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.body.PopProcessQueueInfo;\nimport org.apache.rocketmq.remoting.protocol.body.ProcessQueueInfo;\nimport org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan;\nimport org.apache.rocketmq.remoting.protocol.filter.FilterAPI;\nimport org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class DefaultMQPushConsumerImpl implements MQConsumerInner {\n    /**\n     * Delay some time when exception occur\n     */\n    private long pullTimeDelayMillsWhenException = 3000;\n    /**\n     * Flow control interval when message cache is full\n     */\n    private static final long PULL_TIME_DELAY_MILLS_WHEN_CACHE_FLOW_CONTROL = 50;\n    /**\n     * Flow control interval when broker return flow control\n     */\n    private static final long PULL_TIME_DELAY_MILLS_WHEN_BROKER_FLOW_CONTROL = 20;\n    /**\n     * Delay some time when suspend pull service\n     */\n    private static final long PULL_TIME_DELAY_MILLS_WHEN_SUSPEND = 1000;\n    private static final long BROKER_SUSPEND_MAX_TIME_MILLIS = 1000 * 15;\n    private static final long CONSUMER_TIMEOUT_MILLIS_WHEN_SUSPEND = 1000 * 30;\n    private static final Logger log = LoggerFactory.getLogger(DefaultMQPushConsumerImpl.class);\n    private final DefaultMQPushConsumer defaultMQPushConsumer;\n    private final RebalanceImpl rebalanceImpl = new RebalancePushImpl(this);\n    private final ArrayList<FilterMessageHook> filterMessageHookList = new ArrayList<>();\n    private final long consumerStartTimestamp = System.currentTimeMillis();\n    private final ArrayList<ConsumeMessageHook> consumeMessageHookList = new ArrayList<>();\n    private final RPCHook rpcHook;\n    private volatile ServiceState serviceState = ServiceState.CREATE_JUST;\n    private MQClientInstance mQClientFactory;\n    private PullAPIWrapper pullAPIWrapper;\n    private volatile boolean pause = false;\n    private boolean consumeOrderly = false;\n    private MessageListener messageListenerInner;\n    private OffsetStore offsetStore;\n    private ConsumeMessageService consumeMessageService;\n    private ConsumeMessageService consumeMessagePopService;\n    private long queueFlowControlTimes = 0;\n    private long queueMaxSpanFlowControlTimes = 0;\n\n    //10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h\n    private final int[] popDelayLevel = new int[] {10, 30, 60, 120, 180, 240, 300, 360, 420, 480, 540, 600, 1200, 1800, 3600, 7200};\n\n    private static final int MAX_POP_INVISIBLE_TIME = 300000;\n    private static final int MIN_POP_INVISIBLE_TIME = 5000;\n    private static final int ASYNC_TIMEOUT = 3000;\n\n    // only for test purpose, will be modified by reflection in unit test.\n    @SuppressWarnings(\"FieldMayBeFinal\")\n    private static boolean doNotUpdateTopicSubscribeInfoWhenSubscriptionChanged = false;\n\n    public DefaultMQPushConsumerImpl(DefaultMQPushConsumer defaultMQPushConsumer, RPCHook rpcHook) {\n        this.defaultMQPushConsumer = defaultMQPushConsumer;\n        this.rpcHook = rpcHook;\n        this.pullTimeDelayMillsWhenException = defaultMQPushConsumer.getPullTimeDelayMillsWhenException();\n    }\n\n    public void registerFilterMessageHook(final FilterMessageHook hook) {\n        this.filterMessageHookList.add(hook);\n        log.info(\"register FilterMessageHook Hook, {}\", hook.hookName());\n    }\n\n    public boolean hasHook() {\n        return !this.consumeMessageHookList.isEmpty();\n    }\n\n    public void registerConsumeMessageHook(final ConsumeMessageHook hook) {\n        this.consumeMessageHookList.add(hook);\n        log.info(\"register consumeMessageHook Hook, {}\", hook.hookName());\n    }\n\n    public void executeHookBefore(final ConsumeMessageContext context) {\n        if (!this.consumeMessageHookList.isEmpty()) {\n            for (ConsumeMessageHook hook : this.consumeMessageHookList) {\n                try {\n                    hook.consumeMessageBefore(context);\n                } catch (Throwable e) {\n                    log.warn(\"consumeMessageHook {} executeHookBefore exception\", hook.hookName(), e);\n                }\n            }\n        }\n    }\n\n    public void executeHookAfter(final ConsumeMessageContext context) {\n        if (!this.consumeMessageHookList.isEmpty()) {\n            for (ConsumeMessageHook hook : this.consumeMessageHookList) {\n                try {\n                    hook.consumeMessageAfter(context);\n                } catch (Throwable e) {\n                    log.warn(\"consumeMessageHook {} executeHookAfter exception\", hook.hookName(), e);\n                }\n            }\n        }\n    }\n\n    public void createTopic(String key, String newTopic, int queueNum) throws MQClientException {\n        createTopic(key, newTopic, queueNum, 0);\n    }\n\n    public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) throws MQClientException {\n        this.mQClientFactory.getMQAdminImpl().createTopic(key, newTopic, queueNum, topicSysFlag, null);\n    }\n\n    public Set<MessageQueue> fetchSubscribeMessageQueues(String topic) throws MQClientException {\n        Set<MessageQueue> result = this.rebalanceImpl.getTopicSubscribeInfoTable().get(topic);\n        if (null == result) {\n            this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic);\n            result = this.rebalanceImpl.getTopicSubscribeInfoTable().get(topic);\n        }\n\n        if (null == result) {\n            throw new MQClientException(\"The topic[\" + topic + \"] not exist\", null);\n        }\n\n        return parseSubscribeMessageQueues(result);\n    }\n\n    public Set<MessageQueue> parseSubscribeMessageQueues(Set<MessageQueue> messageQueueList) {\n        Set<MessageQueue> resultQueues = new HashSet<>();\n        for (MessageQueue queue : messageQueueList) {\n            String userTopic = NamespaceUtil.withoutNamespace(queue.getTopic(), this.defaultMQPushConsumer.getNamespace());\n            resultQueues.add(new MessageQueue(userTopic, queue.getBrokerName(), queue.getQueueId()));\n        }\n\n        return resultQueues;\n    }\n\n    public DefaultMQPushConsumer getDefaultMQPushConsumer() {\n        return defaultMQPushConsumer;\n    }\n\n    public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException {\n        return this.mQClientFactory.getMQAdminImpl().earliestMsgStoreTime(mq);\n    }\n\n    public long maxOffset(MessageQueue mq) throws MQClientException {\n        return this.mQClientFactory.getMQAdminImpl().maxOffset(mq);\n    }\n\n    public long minOffset(MessageQueue mq) throws MQClientException {\n        return this.mQClientFactory.getMQAdminImpl().minOffset(mq);\n    }\n\n    public OffsetStore getOffsetStore() {\n        return offsetStore;\n    }\n\n    public void setOffsetStore(OffsetStore offsetStore) {\n        this.offsetStore = offsetStore;\n    }\n\n    public void pullMessage(final PullRequest pullRequest) {\n        final ProcessQueue processQueue = pullRequest.getProcessQueue();\n        if (processQueue.isDropped()) {\n            log.info(\"the pull request[{}] is dropped.\", pullRequest.toString());\n            return;\n        }\n\n        pullRequest.getProcessQueue().setLastPullTimestamp(System.currentTimeMillis());\n\n        try {\n            this.makeSureStateOK();\n        } catch (MQClientException e) {\n            log.warn(\"pullMessage exception, consumer state not ok\", e);\n            this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException);\n            return;\n        }\n\n        if (this.isPause()) {\n            log.warn(\"consumer was paused, execute pull request later. instanceName={}, group={}\", this.defaultMQPushConsumer.getInstanceName(), this.defaultMQPushConsumer.getConsumerGroup());\n            this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_SUSPEND);\n            return;\n        }\n\n        long cachedMessageCount = processQueue.getMsgCount().get();\n        long cachedMessageSizeInMiB = processQueue.getMsgSize().get() / (1024 * 1024);\n\n        if (cachedMessageCount > this.defaultMQPushConsumer.getPullThresholdForQueue()) {\n            this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_CACHE_FLOW_CONTROL);\n            if ((queueFlowControlTimes++ % 1000) == 0) {\n                log.warn(\n                    \"the cached message count exceeds the threshold {}, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, pullRequest={}, flowControlTimes={}\",\n                    this.defaultMQPushConsumer.getPullThresholdForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, pullRequest, queueFlowControlTimes);\n            }\n            return;\n        }\n\n        if (cachedMessageSizeInMiB > this.defaultMQPushConsumer.getPullThresholdSizeForQueue()) {\n            this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_CACHE_FLOW_CONTROL);\n            if ((queueFlowControlTimes++ % 1000) == 0) {\n                log.warn(\n                    \"the cached message size exceeds the threshold {} MiB, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, pullRequest={}, flowControlTimes={}\",\n                    this.defaultMQPushConsumer.getPullThresholdSizeForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, pullRequest, queueFlowControlTimes);\n            }\n            return;\n        }\n\n        if (!this.consumeOrderly) {\n            if (processQueue.getMaxSpan() > this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan()) {\n                this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_CACHE_FLOW_CONTROL);\n                if ((queueMaxSpanFlowControlTimes++ % 1000) == 0) {\n                    log.warn(\n                        \"the queue's messages, span too long, so do flow control, minOffset={}, maxOffset={}, maxSpan={}, pullRequest={}, flowControlTimes={}\",\n                        processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), processQueue.getMaxSpan(),\n                        pullRequest, queueMaxSpanFlowControlTimes);\n                }\n                return;\n            }\n        } else {\n            if (processQueue.isLocked()) {\n                if (!pullRequest.isPreviouslyLocked()) {\n                    long offset = -1L;\n                    try {\n                        offset = this.rebalanceImpl.computePullFromWhereWithException(pullRequest.getMessageQueue());\n                        if (offset < 0) {\n                            throw new MQClientException(ResponseCode.SYSTEM_ERROR, \"Unexpected offset \" + offset);\n                        }\n                    } catch (Exception e) {\n                        this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException);\n                        log.error(\"Failed to compute pull offset, pullResult: {}\", pullRequest, e);\n                        return;\n                    }\n                    boolean brokerBusy = offset < pullRequest.getNextOffset();\n                    log.info(\"the first time to pull message, so fix offset from broker. pullRequest: {} NewOffset: {} brokerBusy: {}\",\n                        pullRequest, offset, brokerBusy);\n                    if (brokerBusy) {\n                        log.info(\"[NOTIFYME]the first time to pull message, but pull request offset larger than broker consume offset. pullRequest: {} NewOffset: {}\",\n                            pullRequest, offset);\n                    }\n\n                    pullRequest.setPreviouslyLocked(true);\n                    pullRequest.setNextOffset(offset);\n                }\n            } else {\n                this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException);\n                log.info(\"pull message later because not locked in broker, {}\", pullRequest);\n                return;\n            }\n        }\n\n        final MessageQueue messageQueue = pullRequest.getMessageQueue();\n        final SubscriptionData subscriptionData = this.rebalanceImpl.getSubscriptionInner().get(messageQueue.getTopic());\n        if (null == subscriptionData) {\n            this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException);\n            log.warn(\"find the consumer's subscription failed, {}\", pullRequest);\n            return;\n        }\n\n        final long beginTimestamp = System.currentTimeMillis();\n\n        PullCallback pullCallback = new PullCallback() {\n            @Override\n            public void onSuccess(PullResult pullResult) {\n                if (pullResult != null) {\n                    pullResult = DefaultMQPushConsumerImpl.this.pullAPIWrapper.processPullResult(pullRequest.getMessageQueue(), pullResult,\n                        subscriptionData);\n\n                    switch (pullResult.getPullStatus()) {\n                        case FOUND:\n                            long prevRequestOffset = pullRequest.getNextOffset();\n                            pullRequest.setNextOffset(pullResult.getNextBeginOffset());\n                            long pullRT = System.currentTimeMillis() - beginTimestamp;\n                            DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullRT(pullRequest.getConsumerGroup(),\n                                pullRequest.getMessageQueue().getTopic(), pullRT);\n\n                            long firstMsgOffset = Long.MAX_VALUE;\n                            if (pullResult.getMsgFoundList() == null || pullResult.getMsgFoundList().isEmpty()) {\n                                DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);\n                            } else {\n                                firstMsgOffset = pullResult.getMsgFoundList().get(0).getQueueOffset();\n\n                                DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullTPS(pullRequest.getConsumerGroup(),\n                                    pullRequest.getMessageQueue().getTopic(), pullResult.getMsgFoundList().size());\n\n                                boolean dispatchToConsume = processQueue.putMessage(pullResult.getMsgFoundList());\n                                DefaultMQPushConsumerImpl.this.consumeMessageService.submitConsumeRequest(\n                                    pullResult.getMsgFoundList(),\n                                    processQueue,\n                                    pullRequest.getMessageQueue(),\n                                    dispatchToConsume);\n\n                                if (DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval() > 0) {\n                                    DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest,\n                                        DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval());\n                                } else {\n                                    DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);\n                                }\n                            }\n\n                            if (pullResult.getNextBeginOffset() < prevRequestOffset\n                                || firstMsgOffset < prevRequestOffset) {\n                                log.warn(\n                                    \"[BUG] pull message result maybe data wrong, nextBeginOffset: {} firstMsgOffset: {} prevRequestOffset: {}\",\n                                    pullResult.getNextBeginOffset(),\n                                    firstMsgOffset,\n                                    prevRequestOffset);\n                            }\n\n                            break;\n                        case NO_NEW_MSG:\n                        case NO_MATCHED_MSG:\n                            pullRequest.setNextOffset(pullResult.getNextBeginOffset());\n\n                            DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest);\n\n                            DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);\n                            break;\n                        case OFFSET_ILLEGAL:\n                            log.warn(\"the pull request offset illegal, {} {}\",\n                                pullRequest.toString(), pullResult.toString());\n                            pullRequest.setNextOffset(pullResult.getNextBeginOffset());\n\n                            pullRequest.getProcessQueue().setDropped(true);\n                            DefaultMQPushConsumerImpl.this.executeTask(new Runnable() {\n\n                                @Override\n                                public void run() {\n                                    try {\n                                        DefaultMQPushConsumerImpl.this.offsetStore.updateAndFreezeOffset(pullRequest.getMessageQueue(),\n                                            pullRequest.getNextOffset());\n\n                                        DefaultMQPushConsumerImpl.this.offsetStore.persist(pullRequest.getMessageQueue());\n\n                                        // removeProcessQueue will also remove offset to cancel the frozen status.\n                                        DefaultMQPushConsumerImpl.this.rebalanceImpl.removeProcessQueue(pullRequest.getMessageQueue());\n                                        DefaultMQPushConsumerImpl.this.rebalanceImpl.getmQClientFactory().rebalanceImmediately();\n\n                                        log.warn(\"fix the pull request offset, {}\", pullRequest);\n                                    } catch (Throwable e) {\n                                        log.error(\"executeTaskLater Exception\", e);\n                                    }\n                                }\n                            });\n                            break;\n                        default:\n                            break;\n                    }\n                }\n            }\n\n            @Override\n            public void onException(Throwable e) {\n                if (!pullRequest.getMessageQueue().getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n                    if (e instanceof MQBrokerException && ((MQBrokerException) e).getResponseCode() == ResponseCode.SUBSCRIPTION_NOT_LATEST) {\n                        log.warn(\"the subscription is not latest, group={}, messageQueue={}\", groupName(), messageQueue);\n                    } else {\n                        log.warn(\"execute the pull request exception, group={}, messageQueue={}\", groupName(), messageQueue, e);\n                    }\n                }\n\n                if (e instanceof MQBrokerException && ((MQBrokerException) e).getResponseCode() == ResponseCode.FLOW_CONTROL) {\n                    DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_BROKER_FLOW_CONTROL);\n                } else {\n                    DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException);\n                }\n            }\n        };\n\n        boolean commitOffsetEnable = false;\n        long commitOffsetValue = 0L;\n        if (MessageModel.CLUSTERING == this.defaultMQPushConsumer.getMessageModel()) {\n            commitOffsetValue = this.offsetStore.readOffset(pullRequest.getMessageQueue(), ReadOffsetType.READ_FROM_MEMORY);\n            if (commitOffsetValue > 0) {\n                commitOffsetEnable = true;\n            }\n        }\n\n        String subExpression = null;\n        boolean classFilter = false;\n        SubscriptionData sd = this.rebalanceImpl.getSubscriptionInner().get(pullRequest.getMessageQueue().getTopic());\n        if (sd != null) {\n            if (this.defaultMQPushConsumer.isPostSubscriptionWhenPull() && !sd.isClassFilterMode()) {\n                subExpression = sd.getSubString();\n            }\n\n            classFilter = sd.isClassFilterMode();\n        }\n\n        int sysFlag = PullSysFlag.buildSysFlag(\n            commitOffsetEnable, // commitOffset\n            true, // suspend\n            subExpression != null, // subscription\n            classFilter // class filter\n        );\n        try {\n            this.pullAPIWrapper.pullKernelImpl(\n                pullRequest.getMessageQueue(),\n                subExpression,\n                subscriptionData.getExpressionType(),\n                subscriptionData.getSubVersion(),\n                pullRequest.getNextOffset(),\n                this.defaultMQPushConsumer.getPullBatchSize(),\n                this.defaultMQPushConsumer.getPullBatchSizeInBytes(),\n                sysFlag,\n                commitOffsetValue,\n                BROKER_SUSPEND_MAX_TIME_MILLIS,\n                CONSUMER_TIMEOUT_MILLIS_WHEN_SUSPEND,\n                CommunicationMode.ASYNC,\n                pullCallback\n            );\n        } catch (Exception e) {\n            log.error(\"pullKernelImpl exception\", e);\n            this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException);\n        }\n    }\n\n    void popMessage(final PopRequest popRequest) {\n        final PopProcessQueue processQueue = popRequest.getPopProcessQueue();\n        if (processQueue.isDropped()) {\n            log.info(\"the pop request[{}] is dropped.\", popRequest.toString());\n            return;\n        }\n\n        processQueue.setLastPopTimestamp(System.currentTimeMillis());\n\n        try {\n            this.makeSureStateOK();\n        } catch (MQClientException e) {\n            log.warn(\"popMessage exception, consumer state not ok\", e);\n            this.executePopPullRequestLater(popRequest, pullTimeDelayMillsWhenException);\n            return;\n        }\n\n        if (this.isPause()) {\n            log.warn(\"consumer was paused, execute pull request later. instanceName={}, group={}\", this.defaultMQPushConsumer.getInstanceName(), this.defaultMQPushConsumer.getConsumerGroup());\n            this.executePopPullRequestLater(popRequest, PULL_TIME_DELAY_MILLS_WHEN_SUSPEND);\n            return;\n        }\n\n        if (processQueue.getWaiAckMsgCount() > this.defaultMQPushConsumer.getPopThresholdForQueue()) {\n            this.executePopPullRequestLater(popRequest, PULL_TIME_DELAY_MILLS_WHEN_CACHE_FLOW_CONTROL);\n            if ((queueFlowControlTimes++ % 1000) == 0) {\n                log.warn(\"the messages waiting to ack exceeds the threshold {}, so do flow control, popRequest={}, flowControlTimes={}, wait count={}\",\n                    this.defaultMQPushConsumer.getPopThresholdForQueue(), popRequest, queueFlowControlTimes, processQueue.getWaiAckMsgCount());\n            }\n            return;\n        }\n\n        //POPTODO think of pop mode orderly implementation later.\n        final SubscriptionData subscriptionData = this.rebalanceImpl.getSubscriptionInner().get(popRequest.getMessageQueue().getTopic());\n        if (null == subscriptionData) {\n            this.executePopPullRequestLater(popRequest, pullTimeDelayMillsWhenException);\n            log.warn(\"find the consumer's subscription failed, {}\", popRequest);\n            return;\n        }\n\n        final long beginTimestamp = System.currentTimeMillis();\n\n        PopCallback popCallback = new PopCallback() {\n            @Override\n            public void onSuccess(PopResult popResult) {\n                if (popResult == null) {\n                    log.error(\"pop callback popResult is null\");\n                    DefaultMQPushConsumerImpl.this.executePopPullRequestImmediately(popRequest);\n                    return;\n                }\n\n                processPopResult(popResult, subscriptionData);\n\n                switch (popResult.getPopStatus()) {\n                    case FOUND:\n                        long pullRT = System.currentTimeMillis() - beginTimestamp;\n                        DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullRT(popRequest.getConsumerGroup(),\n                            popRequest.getMessageQueue().getTopic(), pullRT);\n                        if (popResult.getMsgFoundList() == null || popResult.getMsgFoundList().isEmpty()) {\n                            DefaultMQPushConsumerImpl.this.executePopPullRequestImmediately(popRequest);\n                        } else {\n                            DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullTPS(popRequest.getConsumerGroup(),\n                                popRequest.getMessageQueue().getTopic(), popResult.getMsgFoundList().size());\n                            popRequest.getPopProcessQueue().incFoundMsg(popResult.getMsgFoundList().size());\n\n                            DefaultMQPushConsumerImpl.this.consumeMessagePopService.submitPopConsumeRequest(\n                                popResult.getMsgFoundList(),\n                                processQueue,\n                                popRequest.getMessageQueue());\n\n                            if (DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval() > 0) {\n                                DefaultMQPushConsumerImpl.this.executePopPullRequestLater(popRequest,\n                                    DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval());\n                            } else {\n                                DefaultMQPushConsumerImpl.this.executePopPullRequestImmediately(popRequest);\n                            }\n                        }\n                        break;\n                    case NO_NEW_MSG:\n                    case POLLING_NOT_FOUND:\n                        DefaultMQPushConsumerImpl.this.executePopPullRequestImmediately(popRequest);\n                        break;\n                    case POLLING_FULL:\n                    default:\n                        DefaultMQPushConsumerImpl.this.executePopPullRequestLater(popRequest, pullTimeDelayMillsWhenException);\n                        break;\n                }\n\n            }\n\n            @Override\n            public void onException(Throwable e) {\n                if (!popRequest.getMessageQueue().getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n                    log.warn(\"execute the pull request exception: {}\", e);\n                }\n\n                if (e instanceof MQBrokerException && ((MQBrokerException) e).getResponseCode() == ResponseCode.FLOW_CONTROL) {\n                    DefaultMQPushConsumerImpl.this.executePopPullRequestLater(popRequest, PULL_TIME_DELAY_MILLS_WHEN_BROKER_FLOW_CONTROL);\n                } else {\n                    DefaultMQPushConsumerImpl.this.executePopPullRequestLater(popRequest, pullTimeDelayMillsWhenException);\n                }\n            }\n        };\n\n\n        try {\n\n            long invisibleTime = this.defaultMQPushConsumer.getPopInvisibleTime();\n            if (invisibleTime < MIN_POP_INVISIBLE_TIME || invisibleTime > MAX_POP_INVISIBLE_TIME) {\n                invisibleTime = 60000;\n            }\n            this.pullAPIWrapper.popAsync(popRequest.getMessageQueue(), invisibleTime, this.defaultMQPushConsumer.getPopBatchNums(),\n                popRequest.getConsumerGroup(), BROKER_SUSPEND_MAX_TIME_MILLIS, popCallback, true, popRequest.getInitMode(),\n                false, subscriptionData.getExpressionType(), subscriptionData.getSubString());\n        } catch (Exception e) {\n            log.error(\"popAsync exception\", e);\n            this.executePopPullRequestLater(popRequest, pullTimeDelayMillsWhenException);\n        }\n    }\n\n    private PopResult processPopResult(final PopResult popResult, final SubscriptionData subscriptionData) {\n        if (PopStatus.FOUND == popResult.getPopStatus()) {\n            List<MessageExt> msgFoundList = popResult.getMsgFoundList();\n            List<MessageExt> msgListFilterAgain = new ArrayList<>(popResult.getMsgFoundList().size());\n            if (!subscriptionData.getTagsSet().isEmpty() && !subscriptionData.isClassFilterMode()\n                && popResult.getMsgFoundList().size() > 0) {\n                for (MessageExt msg : popResult.getMsgFoundList()) {\n                    if (msg.getTags() != null) {\n                        if (subscriptionData.getTagsSet().contains(msg.getTags())) {\n                            msgListFilterAgain.add(msg);\n                        }\n                    }\n                }\n            } else {\n                msgListFilterAgain.addAll(msgFoundList);\n            }\n\n            if (!this.filterMessageHookList.isEmpty()) {\n                FilterMessageContext filterMessageContext = new FilterMessageContext();\n                filterMessageContext.setUnitMode(this.defaultMQPushConsumer.isUnitMode());\n                filterMessageContext.setMsgList(msgListFilterAgain);\n                if (!this.filterMessageHookList.isEmpty()) {\n                    for (FilterMessageHook hook : this.filterMessageHookList) {\n                        try {\n                            hook.filterMessage(filterMessageContext);\n                        } catch (Throwable e) {\n                            log.error(\"execute hook error. hookName={}\", hook.hookName());\n                        }\n                    }\n                }\n            }\n\n            Iterator<MessageExt> iterator = msgListFilterAgain.iterator();\n            while (iterator.hasNext()) {\n                MessageExt msg = iterator.next();\n                if (msg.getReconsumeTimes() > getMaxReconsumeTimes()) {\n                    iterator.remove();\n                    log.info(\"Reconsume times has reached {}, so ack msg={}\", msg.getReconsumeTimes(), msg);\n                }\n            }\n\n            if (msgFoundList.size() != msgListFilterAgain.size()) {\n                for (MessageExt msg : msgFoundList) {\n                    if (!msgListFilterAgain.contains(msg)) {\n                        ackAsync(msg, this.groupName());\n                    }\n                }\n            }\n\n            popResult.setMsgFoundList(msgListFilterAgain);\n        }\n\n        return popResult;\n    }\n\n    private void makeSureStateOK() throws MQClientException {\n        if (this.serviceState != ServiceState.RUNNING) {\n            throw new MQClientException(\"The consumer service state not OK, \"\n                + this.serviceState\n                + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK),\n                null);\n        }\n    }\n\n    void executePullRequestLater(final PullRequest pullRequest, final long timeDelay) {\n        this.mQClientFactory.getPullMessageService().executePullRequestLater(pullRequest, timeDelay);\n    }\n\n    public boolean isPause() {\n        return pause;\n    }\n\n    public void setPause(boolean pause) {\n        this.pause = pause;\n    }\n\n    public ConsumerStatsManager getConsumerStatsManager() {\n        return this.mQClientFactory.getConsumerStatsManager();\n    }\n\n    public void executePullRequestImmediately(final PullRequest pullRequest) {\n        this.mQClientFactory.getPullMessageService().executePullRequestImmediately(pullRequest);\n    }\n\n    void executePopPullRequestLater(final PopRequest pullRequest, final long timeDelay) {\n        this.mQClientFactory.getPullMessageService().executePopPullRequestLater(pullRequest, timeDelay);\n    }\n\n    void executePopPullRequestImmediately(final PopRequest pullRequest) {\n        this.mQClientFactory.getPullMessageService().executePopPullRequestImmediately(pullRequest);\n    }\n\n    private void correctTagsOffset(final PullRequest pullRequest) {\n        if (0L == pullRequest.getProcessQueue().getMsgCount().get()) {\n            this.offsetStore.updateOffset(pullRequest.getMessageQueue(), pullRequest.getNextOffset(), true);\n        }\n    }\n\n    public void executeTaskLater(final Runnable r, final long timeDelay) {\n        this.mQClientFactory.getPullMessageService().executeTaskLater(r, timeDelay);\n    }\n\n    public void executeTask(final Runnable r) {\n        this.mQClientFactory.getPullMessageService().executeTask(r);\n    }\n\n    public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end)\n        throws MQClientException, InterruptedException {\n        return this.mQClientFactory.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end);\n    }\n\n    public MessageExt queryMessageByUniqKey(String topic, String uniqKey) throws MQClientException,\n        InterruptedException {\n        return this.mQClientFactory.getMQAdminImpl().queryMessageByUniqKey(topic, uniqKey);\n    }\n\n    public void registerMessageListener(MessageListener messageListener) {\n        this.messageListenerInner = messageListener;\n    }\n\n    public void resume() {\n        this.pause = false;\n        doRebalance();\n        log.info(\"resume this consumer, {}\", this.defaultMQPushConsumer.getConsumerGroup());\n    }\n\n    @Deprecated\n    public void sendMessageBack(MessageExt msg, int delayLevel, final String brokerName)\n            throws RemotingException, MQBrokerException, InterruptedException, MQClientException {\n        sendMessageBack(msg, delayLevel, brokerName, null);\n    }\n\n    public void sendMessageBack(MessageExt msg, int delayLevel, final MessageQueue mq)\n            throws RemotingException, MQBrokerException, InterruptedException, MQClientException {\n        sendMessageBack(msg, delayLevel, msg.getBrokerName(), mq);\n    }\n\n\n    private void sendMessageBack(MessageExt msg, int delayLevel, final String brokerName, final MessageQueue mq)\n        throws RemotingException, MQBrokerException, InterruptedException, MQClientException {\n        boolean needRetry = true;\n        try {\n            if (brokerName != null && brokerName.startsWith(MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX)\n                || mq != null && mq.getBrokerName().startsWith(MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX)) {\n                needRetry = false;\n                sendMessageBackAsNormalMessage(msg);\n            } else {\n                String brokerAddr = (null != brokerName) ? this.mQClientFactory.findBrokerAddressInPublish(brokerName)\n                    : RemotingHelper.parseSocketAddressAddr(msg.getStoreHost());\n                if (UtilAll.isBlank(brokerAddr)) {\n                    throw new MQClientException(\"Broker[\" + brokerName + \"] master node does not exist\", null);\n                }\n                this.mQClientFactory.getMQClientAPIImpl().consumerSendMessageBack(brokerAddr, brokerName, msg,\n                    this.defaultMQPushConsumer.getConsumerGroup(), delayLevel, 5000, getMaxReconsumeTimes());\n            }\n        } catch (Throwable t) {\n            log.error(\"Failed to send message back, consumerGroup={}, brokerName={}, mq={}, message={}\",\n                this.defaultMQPushConsumer.getConsumerGroup(), brokerName, mq, msg, t);\n            if (needRetry) {\n                sendMessageBackAsNormalMessage(msg);\n            }\n        } finally {\n            msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQPushConsumer.getNamespace()));\n        }\n    }\n\n    private void sendMessageBackAsNormalMessage(MessageExt msg) throws  RemotingException, MQBrokerException, InterruptedException, MQClientException {\n        Message newMsg = new Message(MixAll.getRetryTopic(this.defaultMQPushConsumer.getConsumerGroup()), msg.getBody());\n        MessageAccessor.setProperties(newMsg, msg.getProperties());\n\n        String originMsgId = MessageAccessor.getOriginMessageId(msg);\n        MessageAccessor.setOriginMessageId(newMsg, UtilAll.isBlank(originMsgId) ? msg.getMsgId() : originMsgId);\n\n        newMsg.setFlag(msg.getFlag());\n        MessageAccessor.putProperty(newMsg, MessageConst.PROPERTY_RETRY_TOPIC, msg.getTopic());\n        MessageAccessor.setReconsumeTime(newMsg, String.valueOf(msg.getReconsumeTimes() + 1));\n        MessageAccessor.setMaxReconsumeTimes(newMsg, String.valueOf(getMaxReconsumeTimes()));\n        MessageAccessor.clearProperty(newMsg, MessageConst.PROPERTY_TRANSACTION_PREPARED);\n        newMsg.setDelayTimeLevel(3 + msg.getReconsumeTimes());\n\n        this.mQClientFactory.getDefaultMQProducer().send(newMsg);\n    }\n\n    void ackAsync(MessageExt message, String consumerGroup) {\n        final String extraInfo = message.getProperty(MessageConst.PROPERTY_POP_CK);\n\n        try {\n            String[] extraInfoStrs = ExtraInfoUtil.split(extraInfo);\n            String brokerName = ExtraInfoUtil.getBrokerName(extraInfoStrs);\n            int queueId = ExtraInfoUtil.getQueueId(extraInfoStrs);\n            long queueOffset = ExtraInfoUtil.getQueueOffset(extraInfoStrs);\n            String topic = message.getTopic();\n\n            String desBrokerName = brokerName;\n            if (brokerName != null && brokerName.startsWith(MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX)) {\n                desBrokerName = this.mQClientFactory.getBrokerNameFromMessageQueue(this.defaultMQPushConsumer.queueWithNamespace(new MessageQueue(topic, brokerName, queueId)));\n            }\n\n\n            FindBrokerResult\n                findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(desBrokerName, MixAll.MASTER_ID, true);\n            if (null == findBrokerResult) {\n                this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic);\n                findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(desBrokerName, MixAll.MASTER_ID, true);\n            }\n\n            if (findBrokerResult == null) {\n                log.error(\"The broker[\" + desBrokerName + \"] not exist\");\n                return;\n            }\n\n            AckMessageRequestHeader requestHeader = new AckMessageRequestHeader();\n            requestHeader.setTopic(ExtraInfoUtil.getRealTopic(extraInfoStrs, topic, consumerGroup));\n            requestHeader.setQueueId(queueId);\n            requestHeader.setOffset(queueOffset);\n            requestHeader.setConsumerGroup(consumerGroup);\n            requestHeader.setExtraInfo(extraInfo);\n            requestHeader.setBrokerName(brokerName);\n            this.mQClientFactory.getMQClientAPIImpl().ackMessageAsync(findBrokerResult.getBrokerAddr(), ASYNC_TIMEOUT, new AckCallback() {\n                @Override\n                public void onSuccess(AckResult ackResult) {\n                    if (ackResult != null && !AckStatus.OK.equals(ackResult.getStatus())) {\n                        log.warn(\"Ack message fail. ackResult: {}, extraInfo: {}\", ackResult, extraInfo);\n                    }\n                }\n                @Override\n                public void onException(Throwable e) {\n                    log.warn(\"Ack message fail. extraInfo: {}  error message: {}\", extraInfo, e.toString());\n                }\n            }, requestHeader);\n\n        } catch (Throwable t) {\n            log.error(\"ack async error.\", t);\n        }\n    }\n\n    void changePopInvisibleTimeAsync(String topic, String consumerGroup, String extraInfo, long invisibleTime, AckCallback callback)\n        throws MQClientException, RemotingException, InterruptedException, MQBrokerException {\n        String[] extraInfoStrs = ExtraInfoUtil.split(extraInfo);\n        String brokerName = ExtraInfoUtil.getBrokerName(extraInfoStrs);\n        int queueId = ExtraInfoUtil.getQueueId(extraInfoStrs);\n\n        String desBrokerName = brokerName;\n        if (brokerName != null && brokerName.startsWith(MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX)) {\n            desBrokerName = this.mQClientFactory.getBrokerNameFromMessageQueue(this.defaultMQPushConsumer.queueWithNamespace(new MessageQueue(topic, brokerName, queueId)));\n        }\n\n        FindBrokerResult\n            findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(desBrokerName, MixAll.MASTER_ID, true);\n        if (null == findBrokerResult) {\n            this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic);\n            findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(desBrokerName, MixAll.MASTER_ID, true);\n        }\n        if (findBrokerResult != null) {\n            ChangeInvisibleTimeRequestHeader requestHeader = new ChangeInvisibleTimeRequestHeader();\n            requestHeader.setTopic(ExtraInfoUtil.getRealTopic(extraInfoStrs, topic, consumerGroup));\n            requestHeader.setQueueId(queueId);\n            requestHeader.setOffset(ExtraInfoUtil.getQueueOffset(extraInfoStrs));\n            requestHeader.setConsumerGroup(consumerGroup);\n            requestHeader.setExtraInfo(extraInfo);\n            requestHeader.setInvisibleTime(invisibleTime);\n            requestHeader.setBrokerName(brokerName);\n            //here the broker should be polished\n            this.mQClientFactory.getMQClientAPIImpl().changeInvisibleTimeAsync(brokerName, findBrokerResult.getBrokerAddr(), requestHeader, ASYNC_TIMEOUT, callback);\n            return;\n        }\n        throw new MQClientException(\"The broker[\" + desBrokerName + \"] not exist\", null);\n    }\n\n    public int getMaxReconsumeTimes() {\n        // default reconsume times: 16\n        if (this.defaultMQPushConsumer.getMaxReconsumeTimes() == -1) {\n            return 16;\n        } else {\n            return this.defaultMQPushConsumer.getMaxReconsumeTimes();\n        }\n    }\n\n    public void shutdown() {\n        shutdown(0);\n    }\n\n    public synchronized void shutdown(long awaitTerminateMillis) {\n        switch (this.serviceState) {\n            case CREATE_JUST:\n                break;\n            case RUNNING:\n                this.consumeMessageService.shutdown(awaitTerminateMillis);\n                this.persistConsumerOffset();\n                this.mQClientFactory.unregisterConsumer(this.defaultMQPushConsumer.getConsumerGroup());\n                this.mQClientFactory.shutdown();\n                log.info(\"the consumer [{}] shutdown OK\", this.defaultMQPushConsumer.getConsumerGroup());\n                this.rebalanceImpl.destroy();\n                this.serviceState = ServiceState.SHUTDOWN_ALREADY;\n                break;\n            case SHUTDOWN_ALREADY:\n                break;\n            default:\n                break;\n        }\n    }\n\n    public synchronized void start() throws MQClientException {\n        switch (this.serviceState) {\n            case CREATE_JUST:\n                log.info(\"the consumer [{}] start beginning. messageModel={}, isUnitMode={}\", this.defaultMQPushConsumer.getConsumerGroup(),\n                    this.defaultMQPushConsumer.getMessageModel(), this.defaultMQPushConsumer.isUnitMode());\n                this.serviceState = ServiceState.START_FAILED;\n\n                this.checkConfig();\n\n                this.copySubscription();\n\n                if (this.defaultMQPushConsumer.getMessageModel() == MessageModel.CLUSTERING) {\n                    this.defaultMQPushConsumer.changeInstanceNameToPID();\n                }\n\n                this.mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultMQPushConsumer, this.rpcHook);\n\n                this.rebalanceImpl.setConsumerGroup(this.defaultMQPushConsumer.getConsumerGroup());\n                this.rebalanceImpl.setMessageModel(this.defaultMQPushConsumer.getMessageModel());\n                this.rebalanceImpl.setAllocateMessageQueueStrategy(this.defaultMQPushConsumer.getAllocateMessageQueueStrategy());\n                this.rebalanceImpl.setmQClientFactory(this.mQClientFactory);\n\n                if (this.pullAPIWrapper == null) {\n                    this.pullAPIWrapper = new PullAPIWrapper(\n                        mQClientFactory,\n                        this.defaultMQPushConsumer.getConsumerGroup(), isUnitMode());\n                }\n                this.pullAPIWrapper.registerFilterMessageHook(filterMessageHookList);\n\n                if (this.defaultMQPushConsumer.getOffsetStore() != null) {\n                    this.offsetStore = this.defaultMQPushConsumer.getOffsetStore();\n                } else {\n                    switch (this.defaultMQPushConsumer.getMessageModel()) {\n                        case BROADCASTING:\n                            this.offsetStore = new LocalFileOffsetStore(this.mQClientFactory, this.defaultMQPushConsumer.getConsumerGroup());\n                            break;\n                        case CLUSTERING:\n                            this.offsetStore = new RemoteBrokerOffsetStore(this.mQClientFactory, this.defaultMQPushConsumer.getConsumerGroup());\n                            break;\n                        default:\n                            break;\n                    }\n                    this.defaultMQPushConsumer.setOffsetStore(this.offsetStore);\n                }\n                this.offsetStore.load();\n\n                if (this.getMessageListenerInner() instanceof MessageListenerOrderly) {\n                    this.consumeOrderly = true;\n                    this.consumeMessageService =\n                        new ConsumeMessageOrderlyService(this, (MessageListenerOrderly) this.getMessageListenerInner());\n                    //POPTODO reuse Executor ?\n                    this.consumeMessagePopService = new ConsumeMessagePopOrderlyService(this, (MessageListenerOrderly) this.getMessageListenerInner());\n                } else if (this.getMessageListenerInner() instanceof MessageListenerConcurrently) {\n                    this.consumeOrderly = false;\n                    this.consumeMessageService =\n                        new ConsumeMessageConcurrentlyService(this, (MessageListenerConcurrently) this.getMessageListenerInner());\n                    //POPTODO reuse Executor ?\n                    this.consumeMessagePopService =\n                        new ConsumeMessagePopConcurrentlyService(this, (MessageListenerConcurrently) this.getMessageListenerInner());\n                }\n\n                this.consumeMessageService.start();\n                // POPTODO\n                this.consumeMessagePopService.start();\n\n                boolean registerOK = mQClientFactory.registerConsumer(this.defaultMQPushConsumer.getConsumerGroup(), this);\n                if (!registerOK) {\n                    this.serviceState = ServiceState.CREATE_JUST;\n                    this.consumeMessageService.shutdown(defaultMQPushConsumer.getAwaitTerminationMillisWhenShutdown());\n                    throw new MQClientException(\"The consumer group[\" + this.defaultMQPushConsumer.getConsumerGroup()\n                        + \"] has been created before, specify another name please.\" + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL),\n                        null);\n                }\n\n                mQClientFactory.start();\n                log.info(\"the consumer [{}] start OK.\", this.defaultMQPushConsumer.getConsumerGroup());\n                this.serviceState = ServiceState.RUNNING;\n                break;\n            case RUNNING:\n            case START_FAILED:\n            case SHUTDOWN_ALREADY:\n                throw new MQClientException(\"The PushConsumer service state not OK, maybe started once, \"\n                    + this.serviceState\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK),\n                    null);\n            default:\n                break;\n        }\n\n        try {\n            this.updateTopicSubscribeInfoWhenSubscriptionChanged();\n            this.mQClientFactory.checkClientInBroker();\n            if (this.mQClientFactory.sendHeartbeatToAllBrokerWithLock()) {\n                this.mQClientFactory.rebalanceImmediately();\n            }\n        } catch (Exception e) {\n            log.warn(\"Start the consumer {} fail.\", this.defaultMQPushConsumer.getConsumerGroup(), e);\n            shutdown();\n            throw e;\n        }\n    }\n\n    private void checkConfig() throws MQClientException {\n        Validators.checkGroup(this.defaultMQPushConsumer.getConsumerGroup());\n\n        if (null == this.defaultMQPushConsumer.getConsumerGroup()) {\n            throw new MQClientException(\n                \"consumerGroup is null\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        if (this.defaultMQPushConsumer.getConsumerGroup().equals(MixAll.DEFAULT_CONSUMER_GROUP)) {\n            throw new MQClientException(\n                \"consumerGroup can not equal \"\n                    + MixAll.DEFAULT_CONSUMER_GROUP\n                    + \", please specify another one.\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        if (null == this.defaultMQPushConsumer.getMessageModel()) {\n            throw new MQClientException(\n                \"messageModel is null\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        if (null == this.defaultMQPushConsumer.getConsumeFromWhere()) {\n            throw new MQClientException(\n                \"consumeFromWhere is null\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        Date dt = UtilAll.parseDate(this.defaultMQPushConsumer.getConsumeTimestamp(), UtilAll.YYYYMMDDHHMMSS);\n        if (null == dt) {\n            throw new MQClientException(\n                \"consumeTimestamp is invalid, the valid format is yyyyMMddHHmmss,but received \"\n                    + this.defaultMQPushConsumer.getConsumeTimestamp()\n                    + \" \" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null);\n        }\n\n        // allocateMessageQueueStrategy\n        if (null == this.defaultMQPushConsumer.getAllocateMessageQueueStrategy()) {\n            throw new MQClientException(\n                \"allocateMessageQueueStrategy is null\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        // subscription\n        if (null == this.defaultMQPushConsumer.getSubscription()) {\n            throw new MQClientException(\n                \"subscription is null\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        // messageListener\n        if (null == this.defaultMQPushConsumer.getMessageListener()) {\n            throw new MQClientException(\n                \"messageListener is null\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        boolean orderly = this.defaultMQPushConsumer.getMessageListener() instanceof MessageListenerOrderly;\n        boolean concurrently = this.defaultMQPushConsumer.getMessageListener() instanceof MessageListenerConcurrently;\n        if (!orderly && !concurrently) {\n            throw new MQClientException(\n                \"messageListener must be instanceof MessageListenerOrderly or MessageListenerConcurrently\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        // consumeThreadMin\n        if (this.defaultMQPushConsumer.getConsumeThreadMin() < 1\n            || this.defaultMQPushConsumer.getConsumeThreadMin() > 1000) {\n            throw new MQClientException(\n                \"consumeThreadMin Out of range [1, 1000]\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        // consumeThreadMax\n        if (this.defaultMQPushConsumer.getConsumeThreadMax() < 1 || this.defaultMQPushConsumer.getConsumeThreadMax() > 1000) {\n            throw new MQClientException(\n                \"consumeThreadMax Out of range [1, 1000]\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        // consumeThreadMin can't be larger than consumeThreadMax\n        if (this.defaultMQPushConsumer.getConsumeThreadMin() > this.defaultMQPushConsumer.getConsumeThreadMax()) {\n            throw new MQClientException(\n                \"consumeThreadMin (\" + this.defaultMQPushConsumer.getConsumeThreadMin() + \") \"\n                    + \"is larger than consumeThreadMax (\" + this.defaultMQPushConsumer.getConsumeThreadMax() + \")\",\n                null);\n        }\n\n        // consumeConcurrentlyMaxSpan\n        if (this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan() < 1\n            || this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan() > 65535) {\n            throw new MQClientException(\n                \"consumeConcurrentlyMaxSpan Out of range [1, 65535]\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        // pullThresholdForQueue\n        if (this.defaultMQPushConsumer.getPullThresholdForQueue() < 1 || this.defaultMQPushConsumer.getPullThresholdForQueue() > 65535) {\n            throw new MQClientException(\n                \"pullThresholdForQueue Out of range [1, 65535]\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        // pullThresholdForTopic\n        if (this.defaultMQPushConsumer.getPullThresholdForTopic() != -1) {\n            if (this.defaultMQPushConsumer.getPullThresholdForTopic() < 1 || this.defaultMQPushConsumer.getPullThresholdForTopic() > 6553500) {\n                throw new MQClientException(\n                    \"pullThresholdForTopic Out of range [1, 6553500]\"\n                        + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                    null);\n            }\n        }\n\n        // pullThresholdSizeForQueue\n        if (this.defaultMQPushConsumer.getPullThresholdSizeForQueue() < 1 || this.defaultMQPushConsumer.getPullThresholdSizeForQueue() > 1024) {\n            throw new MQClientException(\n                \"pullThresholdSizeForQueue Out of range [1, 1024]\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        if (this.defaultMQPushConsumer.getPullThresholdSizeForTopic() != -1) {\n            // pullThresholdSizeForTopic\n            if (this.defaultMQPushConsumer.getPullThresholdSizeForTopic() < 1 || this.defaultMQPushConsumer.getPullThresholdSizeForTopic() > 102400) {\n                throw new MQClientException(\n                    \"pullThresholdSizeForTopic Out of range [1, 102400]\"\n                        + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                    null);\n            }\n        }\n\n        // pullInterval\n        if (this.defaultMQPushConsumer.getPullInterval() < 0 || this.defaultMQPushConsumer.getPullInterval() > 65535) {\n            throw new MQClientException(\n                \"pullInterval Out of range [0, 65535]\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        // consumeMessageBatchMaxSize\n        if (this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize() < 1\n            || this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize() > 1024) {\n            throw new MQClientException(\n                \"consumeMessageBatchMaxSize Out of range [1, 1024]\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        // pullBatchSize\n        if (this.defaultMQPushConsumer.getPullBatchSize() < 1 || this.defaultMQPushConsumer.getPullBatchSize() > 1024) {\n            throw new MQClientException(\n                \"pullBatchSize Out of range [1, 1024]\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        // popInvisibleTime\n        if (this.defaultMQPushConsumer.getPopInvisibleTime() < MIN_POP_INVISIBLE_TIME\n            || this.defaultMQPushConsumer.getPopInvisibleTime() > MAX_POP_INVISIBLE_TIME) {\n            throw new MQClientException(\n                \"popInvisibleTime Out of range [\" + MIN_POP_INVISIBLE_TIME + \", \" + MAX_POP_INVISIBLE_TIME + \"]\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n\n        // popBatchNums\n        if (this.defaultMQPushConsumer.getPopBatchNums() <= 0 || this.defaultMQPushConsumer.getPopBatchNums() > 32) {\n            throw new MQClientException(\n                \"popBatchNums Out of range [1, 32]\"\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL),\n                null);\n        }\n    }\n\n    private void copySubscription() throws MQClientException {\n        try {\n            Map<String, String> sub = this.defaultMQPushConsumer.getSubscription();\n            if (sub != null) {\n                for (final Map.Entry<String, String> entry : sub.entrySet()) {\n                    final String topic = entry.getKey();\n                    final String subString = entry.getValue();\n                    SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(topic, subString);\n                    this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData);\n                }\n            }\n\n            if (null == this.messageListenerInner) {\n                this.messageListenerInner = this.defaultMQPushConsumer.getMessageListener();\n            }\n\n            switch (this.defaultMQPushConsumer.getMessageModel()) {\n                case BROADCASTING:\n                    break;\n                case CLUSTERING:\n                    final String retryTopic = MixAll.getRetryTopic(this.defaultMQPushConsumer.getConsumerGroup());\n                    SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(retryTopic, SubscriptionData.SUB_ALL);\n                    this.rebalanceImpl.getSubscriptionInner().put(retryTopic, subscriptionData);\n                    break;\n                default:\n                    break;\n            }\n        } catch (Exception e) {\n            throw new MQClientException(\"subscription exception\", e);\n        }\n    }\n\n    public MessageListener getMessageListenerInner() {\n        return messageListenerInner;\n    }\n\n    private void updateTopicSubscribeInfoWhenSubscriptionChanged() {\n        if (doNotUpdateTopicSubscribeInfoWhenSubscriptionChanged) {\n            return;\n        }\n        Map<String, SubscriptionData> subTable = this.getSubscriptionInner();\n        if (subTable != null) {\n            for (final Map.Entry<String, SubscriptionData> entry : subTable.entrySet()) {\n                final String topic = entry.getKey();\n                this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic);\n            }\n        }\n    }\n\n    public ConcurrentMap<String, SubscriptionData> getSubscriptionInner() {\n        return this.rebalanceImpl.getSubscriptionInner();\n    }\n\n    public void subscribe(String topic, String subExpression) throws MQClientException {\n        try {\n            SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(topic, subExpression);\n            this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData);\n            if (this.mQClientFactory != null) {\n                this.mQClientFactory.sendHeartbeatToAllBrokerWithLock();\n            }\n        } catch (Exception e) {\n            throw new MQClientException(\"subscription exception\", e);\n        }\n    }\n\n    public void subscribe(String topic, String fullClassName, String filterClassSource) throws MQClientException {\n        try {\n            SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(topic, SubscriptionData.SUB_ALL);\n            subscriptionData.setSubString(fullClassName);\n            subscriptionData.setClassFilterMode(true);\n            subscriptionData.setFilterClassSource(filterClassSource);\n            this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData);\n            if (this.mQClientFactory != null) {\n                this.mQClientFactory.sendHeartbeatToAllBrokerWithLock();\n            }\n\n        } catch (Exception e) {\n            throw new MQClientException(\"subscription exception\", e);\n        }\n    }\n\n    public void subscribe(final String topic, final MessageSelector messageSelector) throws MQClientException {\n        try {\n            if (messageSelector == null) {\n                subscribe(topic, SubscriptionData.SUB_ALL);\n                return;\n            }\n\n            SubscriptionData subscriptionData = FilterAPI.build(topic,\n                messageSelector.getExpression(), messageSelector.getExpressionType());\n\n            this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData);\n            if (this.mQClientFactory != null) {\n                this.mQClientFactory.sendHeartbeatToAllBrokerWithLock();\n            }\n        } catch (Exception e) {\n            throw new MQClientException(\"subscription exception\", e);\n        }\n    }\n\n    public void suspend() {\n        this.pause = true;\n        log.info(\"suspend this consumer, {}\", this.defaultMQPushConsumer.getConsumerGroup());\n    }\n\n    public void unsubscribe(String topic) {\n        this.rebalanceImpl.getSubscriptionInner().remove(topic);\n    }\n\n    public void updateConsumeOffset(MessageQueue mq, long offset) {\n        this.offsetStore.updateOffset(mq, offset, false);\n    }\n\n    public void updateCorePoolSize(int corePoolSize) {\n        this.consumeMessageService.updateCorePoolSize(corePoolSize);\n    }\n\n    public MessageExt viewMessage(String topic, String msgId)\n        throws RemotingException, MQBrokerException, InterruptedException, MQClientException {\n        return this.mQClientFactory.getMQAdminImpl().viewMessage(topic, msgId);\n    }\n\n    public RebalanceImpl getRebalanceImpl() {\n        return rebalanceImpl;\n    }\n\n    public boolean isConsumeOrderly() {\n        return consumeOrderly;\n    }\n\n    public void setConsumeOrderly(boolean consumeOrderly) {\n        this.consumeOrderly = consumeOrderly;\n    }\n\n    public void resetOffsetByTimeStamp(long timeStamp) throws MQClientException {\n        for (String topic : rebalanceImpl.getSubscriptionInner().keySet()) {\n            Set<MessageQueue> mqs = rebalanceImpl.getTopicSubscribeInfoTable().get(topic);\n            if (CollectionUtils.isNotEmpty(mqs)) {\n                Map<MessageQueue, Long> offsetTable = new HashMap<>(mqs.size(), 1);\n                for (MessageQueue mq : mqs) {\n                    long offset = searchOffset(mq, timeStamp);\n                    offsetTable.put(mq, offset);\n                }\n                this.mQClientFactory.resetOffset(topic, groupName(), offsetTable);\n            }\n        }\n    }\n\n    public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException {\n        return this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp);\n    }\n\n    @Override\n    public String groupName() {\n        return this.defaultMQPushConsumer.getConsumerGroup();\n    }\n\n    @Override\n    public MessageModel messageModel() {\n        return this.defaultMQPushConsumer.getMessageModel();\n    }\n\n    @Override\n    public ConsumeType consumeType() {\n        return ConsumeType.CONSUME_PASSIVELY;\n    }\n\n    @Override\n    public ConsumeFromWhere consumeFromWhere() {\n        return this.defaultMQPushConsumer.getConsumeFromWhere();\n    }\n\n    @Override\n    public Set<SubscriptionData> subscriptions() {\n        return new HashSet<>(this.rebalanceImpl.getSubscriptionInner().values());\n    }\n\n    @Override\n    public void doRebalance() {\n        if (!this.pause) {\n            this.rebalanceImpl.doRebalance(this.isConsumeOrderly());\n        }\n    }\n\n    @Override\n    public boolean tryRebalance() {\n        if (!this.pause) {\n            return this.rebalanceImpl.doRebalance(this.isConsumeOrderly());\n        }\n        return false;\n    }\n\n    @Override\n    public void persistConsumerOffset() {\n        try {\n            this.makeSureStateOK();\n            Set<MessageQueue> mqs = new HashSet<>();\n            Set<MessageQueue> allocateMq = this.rebalanceImpl.getProcessQueueTable().keySet();\n            mqs.addAll(allocateMq);\n\n            this.offsetStore.persistAll(mqs);\n        } catch (Exception e) {\n            log.error(\"group: \" + this.defaultMQPushConsumer.getConsumerGroup() + \" persistConsumerOffset exception\", e);\n        }\n    }\n\n    @Override\n    public void updateTopicSubscribeInfo(String topic, Set<MessageQueue> info) {\n        Map<String, SubscriptionData> subTable = this.getSubscriptionInner();\n        if (subTable != null) {\n            if (subTable.containsKey(topic)) {\n                this.rebalanceImpl.topicSubscribeInfoTable.put(topic, info);\n            }\n        }\n    }\n\n    @Override\n    public boolean isSubscribeTopicNeedUpdate(String topic) {\n        Map<String, SubscriptionData> subTable = this.getSubscriptionInner();\n        if (subTable != null) {\n            if (subTable.containsKey(topic)) {\n                return !this.rebalanceImpl.topicSubscribeInfoTable.containsKey(topic);\n            }\n        }\n\n        return false;\n    }\n\n    @Override\n    public boolean isUnitMode() {\n        return this.defaultMQPushConsumer.isUnitMode();\n    }\n\n    @Override\n    public ConsumerRunningInfo consumerRunningInfo() {\n        ConsumerRunningInfo info = new ConsumerRunningInfo();\n\n        Properties prop = MixAll.object2Properties(this.defaultMQPushConsumer);\n\n        prop.put(ConsumerRunningInfo.PROP_CONSUME_ORDERLY, String.valueOf(this.consumeOrderly));\n        prop.put(ConsumerRunningInfo.PROP_THREADPOOL_CORE_SIZE, String.valueOf(this.consumeMessageService.getCorePoolSize()));\n        prop.put(ConsumerRunningInfo.PROP_CONSUMER_START_TIMESTAMP, String.valueOf(this.consumerStartTimestamp));\n\n        info.setProperties(prop);\n\n        Set<SubscriptionData> subSet = this.subscriptions();\n        info.getSubscriptionSet().addAll(subSet);\n\n        Iterator<Entry<MessageQueue, ProcessQueue>> it = this.rebalanceImpl.getProcessQueueTable().entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<MessageQueue, ProcessQueue> next = it.next();\n            MessageQueue mq = next.getKey();\n            ProcessQueue pq = next.getValue();\n\n            ProcessQueueInfo pqinfo = new ProcessQueueInfo();\n            pqinfo.setCommitOffset(this.offsetStore.readOffset(mq, ReadOffsetType.MEMORY_FIRST_THEN_STORE));\n            pq.fillProcessQueueInfo(pqinfo);\n            info.getMqTable().put(mq, pqinfo);\n        }\n\n        Iterator<Entry<MessageQueue, PopProcessQueue>> popIt = this.rebalanceImpl.getPopProcessQueueTable().entrySet().iterator();\n        while (popIt.hasNext()) {\n            Entry<MessageQueue, PopProcessQueue> next = popIt.next();\n            MessageQueue mq = next.getKey();\n            PopProcessQueue pq = next.getValue();\n\n            PopProcessQueueInfo pqinfo = new PopProcessQueueInfo();\n            pq.fillPopProcessQueueInfo(pqinfo);\n            info.getMqPopTable().put(mq, pqinfo);\n        }\n\n        for (SubscriptionData sd : subSet) {\n            ConsumeStatus consumeStatus = this.mQClientFactory.getConsumerStatsManager().consumeStatus(this.groupName(), sd.getTopic());\n            info.getStatusTable().put(sd.getTopic(), consumeStatus);\n        }\n\n        return info;\n    }\n\n    public MQClientInstance getmQClientFactory() {\n        return mQClientFactory;\n    }\n\n    public void setmQClientFactory(MQClientInstance mQClientFactory) {\n        this.mQClientFactory = mQClientFactory;\n    }\n\n    public ServiceState getServiceState() {\n        return serviceState;\n    }\n\n    //Don't use this deprecated setter, which will be removed soon.\n    @Deprecated\n    public synchronized void setServiceState(ServiceState serviceState) {\n        this.serviceState = serviceState;\n    }\n\n    public void adjustThreadPool() {\n        long computeAccTotal = this.computeAccumulationTotal();\n        long adjustThreadPoolNumsThreshold = this.defaultMQPushConsumer.getAdjustThreadPoolNumsThreshold();\n\n        long incThreshold = (long) (adjustThreadPoolNumsThreshold * 1.0);\n\n        long decThreshold = (long) (adjustThreadPoolNumsThreshold * 0.8);\n\n        if (computeAccTotal >= incThreshold) {\n            this.consumeMessageService.incCorePoolSize();\n        }\n\n        if (computeAccTotal < decThreshold) {\n            this.consumeMessageService.decCorePoolSize();\n        }\n    }\n\n    private long computeAccumulationTotal() {\n        long msgAccTotal = 0;\n        ConcurrentMap<MessageQueue, ProcessQueue> processQueueTable = this.rebalanceImpl.getProcessQueueTable();\n        Iterator<Entry<MessageQueue, ProcessQueue>> it = processQueueTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<MessageQueue, ProcessQueue> next = it.next();\n            ProcessQueue value = next.getValue();\n            msgAccTotal += value.getMsgAccCnt();\n        }\n\n        return msgAccTotal;\n    }\n\n    public List<QueueTimeSpan> queryConsumeTimeSpan(final String topic)\n        throws RemotingException, MQClientException, InterruptedException, MQBrokerException {\n        List<QueueTimeSpan> queueTimeSpan = new ArrayList<>();\n        TopicRouteData routeData = this.mQClientFactory.getMQClientAPIImpl().getTopicRouteInfoFromNameServer(topic, 3000);\n        for (BrokerData brokerData : routeData.getBrokerDatas()) {\n            String addr = brokerData.selectBrokerAddr();\n            queueTimeSpan.addAll(this.mQClientFactory.getMQClientAPIImpl().queryConsumeTimeSpan(addr, topic, groupName(), 3000));\n        }\n\n        return queueTimeSpan;\n    }\n\n    public void tryResetPopRetryTopic(final List<MessageExt> msgs, String consumerGroup) {\n        String popRetryPrefix = MixAll.RETRY_GROUP_TOPIC_PREFIX + consumerGroup + \"_\";\n        for (MessageExt msg : msgs) {\n            if (msg.getTopic().startsWith(popRetryPrefix)) {\n                String normalTopic = KeyBuilder.parseNormalTopic(msg.getTopic(), consumerGroup);\n                if (normalTopic != null && !normalTopic.isEmpty()) {\n                    msg.setTopic(normalTopic);\n                }\n            }\n        }\n    }\n\n\n    public void resetRetryAndNamespace(final List<MessageExt> msgs, String consumerGroup) {\n        final String groupTopic = MixAll.getRetryTopic(consumerGroup);\n        for (MessageExt msg : msgs) {\n            String retryTopic = msg.getProperty(MessageConst.PROPERTY_RETRY_TOPIC);\n            if (retryTopic != null && groupTopic.equals(msg.getTopic())) {\n                msg.setTopic(retryTopic);\n            }\n\n            if (StringUtils.isNotEmpty(this.defaultMQPushConsumer.getNamespace())) {\n                msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQPushConsumer.getNamespace()));\n            }\n        }\n    }\n\n    public ConsumeMessageService getConsumeMessageService() {\n        return consumeMessageService;\n    }\n\n    public void setConsumeMessageService(ConsumeMessageService consumeMessageService) {\n        this.consumeMessageService = consumeMessageService;\n\n    }\n\n    public void setPullTimeDelayMillsWhenException(long pullTimeDelayMillsWhenException) {\n        this.pullTimeDelayMillsWhenException = pullTimeDelayMillsWhenException;\n    }\n\n    int[] getPopDelayLevel() {\n        return popDelayLevel;\n    }\n\n    public MessageQueueListener getMessageQueueListener() {\n        if (null == defaultMQPushConsumer) {\n            return null;\n        }\n        return defaultMQPushConsumer.getMessageQueueListener();\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/MQConsumerInner.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport java.util.Set;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\n/**\n * Consumer inner interface\n */\npublic interface MQConsumerInner {\n    String groupName();\n\n    MessageModel messageModel();\n\n    ConsumeType consumeType();\n\n    ConsumeFromWhere consumeFromWhere();\n\n    Set<SubscriptionData> subscriptions();\n\n    void doRebalance();\n\n    boolean tryRebalance();\n\n    void persistConsumerOffset();\n\n    void updateTopicSubscribeInfo(final String topic, final Set<MessageQueue> info);\n\n    boolean isSubscribeTopicNeedUpdate(final String topic);\n\n    boolean isUnitMode();\n\n    ConsumerRunningInfo consumerRunningInfo();\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/MessageQueueLock.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\n/**\n * Message lock,strictly ensure the single queue only one thread at a time consuming\n */\npublic class MessageQueueLock {\n    private ConcurrentMap<MessageQueue, ConcurrentMap<Integer, Object>> mqLockTable =\n        new ConcurrentHashMap<>(32);\n\n    public Object fetchLockObject(final MessageQueue mq) {\n        return fetchLockObject(mq, -1);\n    }\n\n    public Object fetchLockObject(final MessageQueue mq, final int shardingKeyIndex) {\n        ConcurrentMap<Integer, Object> objMap = this.mqLockTable.get(mq);\n        if (null == objMap) {\n            objMap = new ConcurrentHashMap<>(32);\n            ConcurrentMap<Integer, Object> prevObjMap = this.mqLockTable.putIfAbsent(mq, objMap);\n            if (prevObjMap != null) {\n                objMap = prevObjMap;\n            }\n        }\n\n        Object lock = objMap.get(shardingKeyIndex);\n        if (null == lock) {\n            lock = new Object();\n            Object prevLock = objMap.putIfAbsent(shardingKeyIndex, lock);\n            if (prevLock != null) {\n                lock = prevLock;\n            }\n        }\n\n        return lock;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/MessageRequest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport org.apache.rocketmq.common.message.MessageRequestMode;\n\npublic interface MessageRequest {\n    MessageRequestMode getMessageRequestMode();\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/PopProcessQueue.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.rocketmq.remoting.protocol.body.PopProcessQueueInfo;\n\n/**\n * Queue consumption snapshot\n */\npublic class PopProcessQueue {\n\n    private final static long PULL_MAX_IDLE_TIME = Long.parseLong(System.getProperty(\"rocketmq.client.pull.pullMaxIdleTime\", \"120000\"));\n\n    private long lastPopTimestamp = System.currentTimeMillis();\n    private AtomicInteger waitAckCounter = new AtomicInteger(0);\n    private volatile boolean dropped = false;\n\n    public long getLastPopTimestamp() {\n        return lastPopTimestamp;\n    }\n\n    public void setLastPopTimestamp(long lastPopTimestamp) {\n        this.lastPopTimestamp = lastPopTimestamp;\n    }\n\n    public void incFoundMsg(int count) {\n        this.waitAckCounter.getAndAdd(count);\n    }\n\n    /**\n     * @return the value before decrement.\n     */\n    public int ack() {\n        return this.waitAckCounter.getAndDecrement();\n    }\n\n    public void decFoundMsg(int count) {\n        this.waitAckCounter.addAndGet(count);\n    }\n\n    public int getWaiAckMsgCount() {\n        return this.waitAckCounter.get();\n    }\n\n    public boolean isDropped() {\n        return dropped;\n    }\n\n    public void setDropped(boolean dropped) {\n        this.dropped = dropped;\n    }\n\n    public void fillPopProcessQueueInfo(final PopProcessQueueInfo info) {\n        info.setWaitAckCount(getWaiAckMsgCount());\n        info.setDroped(isDropped());\n        info.setLastPopTimestamp(getLastPopTimestamp());\n    }\n\n    public boolean isPullExpired() {\n        return (System.currentTimeMillis() - this.lastPopTimestamp) > PULL_MAX_IDLE_TIME;\n    }\n\n    @Override\n    public String toString() {\n        return \"PopProcessQueue[waitAckCounter:\" + this.waitAckCounter.get()\n                + \", lastPopTimestamp:\" + getLastPopTimestamp()\n                + \", drop:\" + dropped +  \"]\";\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/PopRequest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport org.apache.rocketmq.common.constant.ConsumeInitMode;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.message.MessageRequestMode;\n\npublic class PopRequest implements MessageRequest {\n    private String topic;\n    private String consumerGroup;\n    private MessageQueue messageQueue;\n    private PopProcessQueue popProcessQueue;\n    private boolean lockedFirst = false;\n    private int initMode = ConsumeInitMode.MAX;\n\n    public boolean isLockedFirst() {\n        return lockedFirst;\n    }\n\n    public void setLockedFirst(boolean lockedFirst) {\n        this.lockedFirst = lockedFirst;\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public MessageQueue getMessageQueue() {\n        return messageQueue;\n    }\n\n    public void setMessageQueue(MessageQueue messageQueue) {\n        this.messageQueue = messageQueue;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public PopProcessQueue getPopProcessQueue() {\n        return popProcessQueue;\n    }\n\n    public void setPopProcessQueue(PopProcessQueue popProcessQueue) {\n        this.popProcessQueue = popProcessQueue;\n    }\n\n    public int getInitMode() {\n        return initMode;\n    }\n\n    public void setInitMode(int initMode) {\n        this.initMode = initMode;\n    }\n\n    @Override\n    public int hashCode() {\n        final int prime = 31;\n        int result = 1;\n        result = prime * result + ((topic == null) ? 0 : topic.hashCode());\n        result = prime * result + ((consumerGroup == null) ? 0 : consumerGroup.hashCode());\n        result = prime * result + ((messageQueue == null) ? 0 : messageQueue.hashCode());\n        return result;\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj)\n            return true;\n        if (obj == null)\n            return false;\n        if (getClass() != obj.getClass())\n            return false;\n\n        PopRequest other = (PopRequest) obj;\n\n        if (topic == null) {\n            if (other.topic != null)\n                return false;\n        } else if (!topic.equals(other.topic)) {\n            return false;\n        }\n\n        if (consumerGroup == null) {\n            if (other.consumerGroup != null)\n                return false;\n        } else if (!consumerGroup.equals(other.consumerGroup))\n            return false;\n\n        if (messageQueue == null) {\n            if (other.messageQueue != null)\n                return false;\n        } else if (!messageQueue.equals(other.messageQueue)) {\n            return false;\n        }\n        return true;\n    }\n\n    @Override\n    public String toString() {\n        return \"PopRequest [topic=\" + topic + \", consumerGroup=\" + consumerGroup + \", messageQueue=\" + messageQueue + \"]\";\n    }\n\n    @Override\n    public MessageRequestMode getMessageRequestMode() {\n        return MessageRequestMode.POP;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.TreeMap;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.concurrent.locks.ReadWriteLock;\nimport java.util.concurrent.locks.ReentrantReadWriteLock;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.body.ProcessQueueInfo;\n\n/**\n * Queue consumption snapshot\n */\npublic class ProcessQueue {\n    public final static long REBALANCE_LOCK_MAX_LIVE_TIME =\n        Long.parseLong(System.getProperty(\"rocketmq.client.rebalance.lockMaxLiveTime\", \"30000\"));\n    public final static long REBALANCE_LOCK_INTERVAL = Long.parseLong(System.getProperty(\"rocketmq.client.rebalance.lockInterval\", \"20000\"));\n    private final static long PULL_MAX_IDLE_TIME = Long.parseLong(System.getProperty(\"rocketmq.client.pull.pullMaxIdleTime\", \"120000\"));\n    private final Logger log = LoggerFactory.getLogger(ProcessQueue.class);\n    private final ReadWriteLock treeMapLock = new ReentrantReadWriteLock();\n    private final TreeMap<Long, MessageExt> msgTreeMap = new TreeMap<>();\n    private final AtomicLong msgCount = new AtomicLong();\n    private final AtomicLong msgSize = new AtomicLong();\n    private final ReadWriteLock consumeLock = new ReentrantReadWriteLock();\n    /**\n     * A subset of msgTreeMap, will only be used when orderly consume\n     */\n    private final TreeMap<Long, MessageExt> consumingMsgOrderlyTreeMap = new TreeMap<>();\n    private final AtomicLong tryUnlockTimes = new AtomicLong(0);\n    private volatile long queueOffsetMax = 0L;\n    private volatile boolean dropped = false;\n    private volatile long lastPullTimestamp = System.currentTimeMillis();\n    private volatile long lastConsumeTimestamp = System.currentTimeMillis();\n    private volatile boolean locked = false;\n    private volatile long lastLockTimestamp = System.currentTimeMillis();\n    private volatile boolean consuming = false;\n    private volatile long msgAccCnt = 0;\n\n    public boolean isLockExpired() {\n        return (System.currentTimeMillis() - this.lastLockTimestamp) > REBALANCE_LOCK_MAX_LIVE_TIME;\n    }\n\n    public boolean isPullExpired() {\n        return (System.currentTimeMillis() - this.lastPullTimestamp) > PULL_MAX_IDLE_TIME;\n    }\n\n    /**\n     * @param pushConsumer\n     */\n    public void cleanExpiredMsg(DefaultMQPushConsumer pushConsumer) {\n        if (pushConsumer.isConsumeOrderly()) {\n            return;\n        }\n\n        int loop = Math.min(msgTreeMap.size(), 16);\n        for (int i = 0; i < loop; i++) {\n            MessageExt msg = null;\n            try {\n                this.treeMapLock.readLock().lockInterruptibly();\n                try {\n                    if (!msgTreeMap.isEmpty()) {\n                        String consumeStartTimeStamp = MessageAccessor.getConsumeStartTimeStamp(msgTreeMap.firstEntry().getValue());\n                        if (StringUtils.isNotEmpty(consumeStartTimeStamp) && System.currentTimeMillis() - Long.parseLong(consumeStartTimeStamp) > pushConsumer.getConsumeTimeout() * 60 * 1000) {\n                            msg = msgTreeMap.firstEntry().getValue();\n                        }\n                    }\n                } finally {\n                    this.treeMapLock.readLock().unlock();\n                }\n            } catch (InterruptedException e) {\n                log.error(\"getExpiredMsg exception\", e);\n            }\n\n            if (msg == null) {\n                break;\n            }\n\n            try {\n\n                pushConsumer.sendMessageBack(msg, 3);\n                log.info(\"send expire msg back. topic={}, msgId={}, storeHost={}, queueId={}, queueOffset={}\", msg.getTopic(), msg.getMsgId(), msg.getStoreHost(), msg.getQueueId(), msg.getQueueOffset());\n                try {\n                    this.treeMapLock.writeLock().lockInterruptibly();\n                    try {\n                        if (!msgTreeMap.isEmpty() && msg.getQueueOffset() == msgTreeMap.firstKey()) {\n                            try {\n                                removeMessage(Collections.singletonList(msg));\n                            } catch (Exception e) {\n                                log.error(\"send expired msg exception\", e);\n                            }\n                        }\n                    } finally {\n                        this.treeMapLock.writeLock().unlock();\n                    }\n                } catch (InterruptedException e) {\n                    log.error(\"getExpiredMsg exception\", e);\n                }\n            } catch (Exception e) {\n                log.error(\"send expired msg exception\", e);\n            }\n        }\n    }\n\n    public boolean putMessage(final List<MessageExt> msgs) {\n        boolean dispatchToConsume = false;\n        try {\n            this.treeMapLock.writeLock().lockInterruptibly();\n            try {\n                int validMsgCnt = 0;\n                for (MessageExt msg : msgs) {\n                    MessageExt old = msgTreeMap.put(msg.getQueueOffset(), msg);\n                    if (null == old) {\n                        validMsgCnt++;\n                        this.queueOffsetMax = msg.getQueueOffset();\n                        msgSize.addAndGet(null == msg.getBody() ? 0 : msg.getBody().length);\n                    }\n                }\n                msgCount.addAndGet(validMsgCnt);\n\n                if (!msgTreeMap.isEmpty() && !this.consuming) {\n                    dispatchToConsume = true;\n                    this.consuming = true;\n                }\n\n                if (!msgs.isEmpty()) {\n                    MessageExt messageExt = msgs.get(msgs.size() - 1);\n                    String property = messageExt.getProperty(MessageConst.PROPERTY_MAX_OFFSET);\n                    if (property != null) {\n                        long accTotal = Long.parseLong(property) - messageExt.getQueueOffset();\n                        if (accTotal > 0) {\n                            this.msgAccCnt = accTotal;\n                        }\n                    }\n                }\n            } finally {\n                this.treeMapLock.writeLock().unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"putMessage exception\", e);\n        }\n\n        return dispatchToConsume;\n    }\n\n    public long getMaxSpan() {\n        try {\n            this.treeMapLock.readLock().lockInterruptibly();\n            try {\n                if (!this.msgTreeMap.isEmpty()) {\n                    return this.msgTreeMap.lastKey() - this.msgTreeMap.firstKey();\n                }\n            } finally {\n                this.treeMapLock.readLock().unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"getMaxSpan exception\", e);\n        }\n\n        return 0;\n    }\n\n    public long removeMessage(final List<MessageExt> msgs) {\n        long result = -1;\n        final long now = System.currentTimeMillis();\n        try {\n            this.treeMapLock.writeLock().lockInterruptibly();\n            this.lastConsumeTimestamp = now;\n            try {\n                if (!msgTreeMap.isEmpty()) {\n                    result = this.queueOffsetMax + 1;\n                    int removedCnt = 0;\n                    for (MessageExt msg : msgs) {\n                        MessageExt prev = msgTreeMap.remove(msg.getQueueOffset());\n                        if (prev != null) {\n                            removedCnt--;\n                            long bodySize = null == msg.getBody() ? 0 : msg.getBody().length;\n                            if (bodySize > 0) {\n                                msgSize.addAndGet(-bodySize);\n                            }\n                        }\n                    }\n                    if (msgCount.addAndGet(removedCnt) == 0) {\n                        msgSize.set(0);\n                    }\n\n                    if (!msgTreeMap.isEmpty()) {\n                        result = msgTreeMap.firstKey();\n                    }\n                }\n            } finally {\n                this.treeMapLock.writeLock().unlock();\n            }\n        } catch (Throwable t) {\n            log.error(\"removeMessage exception\", t);\n        }\n\n        return result;\n    }\n\n    public TreeMap<Long, MessageExt> getMsgTreeMap() {\n        return msgTreeMap;\n    }\n\n    public AtomicLong getMsgCount() {\n        return msgCount;\n    }\n\n    public AtomicLong getMsgSize() {\n        return msgSize;\n    }\n\n    public boolean isDropped() {\n        return dropped;\n    }\n\n    public void setDropped(boolean dropped) {\n        this.dropped = dropped;\n    }\n\n    public boolean isLocked() {\n        return locked;\n    }\n\n    public void setLocked(boolean locked) {\n        this.locked = locked;\n    }\n\n    public void rollback() {\n        try {\n            this.treeMapLock.writeLock().lockInterruptibly();\n            try {\n                this.msgTreeMap.putAll(this.consumingMsgOrderlyTreeMap);\n                this.consumingMsgOrderlyTreeMap.clear();\n            } finally {\n                this.treeMapLock.writeLock().unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"rollback exception\", e);\n        }\n    }\n\n    public long commit() {\n        try {\n            this.treeMapLock.writeLock().lockInterruptibly();\n            try {\n                Long offset = this.consumingMsgOrderlyTreeMap.lastKey();\n                if (msgCount.addAndGet(-this.consumingMsgOrderlyTreeMap.size()) == 0) {\n                    msgSize.set(0);\n                } else {\n                    for (MessageExt msg : this.consumingMsgOrderlyTreeMap.values()) {\n                        int bodySize = null == msg.getBody() ? 0 : msg.getBody().length;\n                        if (bodySize > 0) {\n                            msgSize.addAndGet(-bodySize);\n                        }\n                    }\n                }\n                this.consumingMsgOrderlyTreeMap.clear();\n                if (offset != null) {\n                    return offset + 1;\n                }\n            } finally {\n                this.treeMapLock.writeLock().unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"commit exception\", e);\n        }\n\n        return -1;\n    }\n\n    public void makeMessageToConsumeAgain(List<MessageExt> msgs) {\n        try {\n            this.treeMapLock.writeLock().lockInterruptibly();\n            try {\n                for (MessageExt msg : msgs) {\n                    this.consumingMsgOrderlyTreeMap.remove(msg.getQueueOffset());\n                    this.msgTreeMap.put(msg.getQueueOffset(), msg);\n                }\n            } finally {\n                this.treeMapLock.writeLock().unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"makeMessageToCosumeAgain exception\", e);\n        }\n    }\n\n    public List<MessageExt> takeMessages(final int batchSize) {\n        List<MessageExt> result = new ArrayList<>(batchSize);\n        final long now = System.currentTimeMillis();\n        try {\n            this.treeMapLock.writeLock().lockInterruptibly();\n            this.lastConsumeTimestamp = now;\n            try {\n                if (!this.msgTreeMap.isEmpty()) {\n                    for (int i = 0; i < batchSize; i++) {\n                        Map.Entry<Long, MessageExt> entry = this.msgTreeMap.pollFirstEntry();\n                        if (entry != null) {\n                            result.add(entry.getValue());\n                            consumingMsgOrderlyTreeMap.put(entry.getKey(), entry.getValue());\n                        } else {\n                            break;\n                        }\n                    }\n                }\n\n                if (result.isEmpty()) {\n                    consuming = false;\n                }\n            } finally {\n                this.treeMapLock.writeLock().unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"take Messages exception\", e);\n        }\n\n        return result;\n    }\n\n    /**\n     * Return the result that whether current message is exist in the process queue or not.\n     */\n    public boolean containsMessage(MessageExt message) {\n        if (message == null) {\n            // should never reach here.\n            return false;\n        }\n        try {\n            this.treeMapLock.readLock().lockInterruptibly();\n            try {\n                return this.msgTreeMap.containsKey(message.getQueueOffset());\n            } finally {\n                this.treeMapLock.readLock().unlock();\n            }\n        } catch (Throwable t) {\n            log.error(\"Failed to check message's existence in process queue, message={}\", message, t);\n        }\n        return false;\n    }\n\n    public boolean hasTempMessage() {\n        try {\n            this.treeMapLock.readLock().lockInterruptibly();\n            try {\n                return !this.msgTreeMap.isEmpty();\n            } finally {\n                this.treeMapLock.readLock().unlock();\n            }\n        } catch (InterruptedException e) {\n        }\n\n        return true;\n    }\n\n    public void clear() {\n        try {\n            this.treeMapLock.writeLock().lockInterruptibly();\n            try {\n                this.msgTreeMap.clear();\n                this.consumingMsgOrderlyTreeMap.clear();\n                this.msgCount.set(0);\n                this.msgSize.set(0);\n                this.queueOffsetMax = 0L;\n            } finally {\n                this.treeMapLock.writeLock().unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"rollback exception\", e);\n        }\n    }\n\n    public long getLastLockTimestamp() {\n        return lastLockTimestamp;\n    }\n\n    public void setLastLockTimestamp(long lastLockTimestamp) {\n        this.lastLockTimestamp = lastLockTimestamp;\n    }\n\n    public ReadWriteLock getConsumeLock() {\n        return consumeLock;\n    }\n\n    public long getLastPullTimestamp() {\n        return lastPullTimestamp;\n    }\n\n    public void setLastPullTimestamp(long lastPullTimestamp) {\n        this.lastPullTimestamp = lastPullTimestamp;\n    }\n\n    public long getMsgAccCnt() {\n        return msgAccCnt;\n    }\n\n    public void setMsgAccCnt(long msgAccCnt) {\n        this.msgAccCnt = msgAccCnt;\n    }\n\n    public long getTryUnlockTimes() {\n        return this.tryUnlockTimes.get();\n    }\n\n    public void incTryUnlockTimes() {\n        this.tryUnlockTimes.incrementAndGet();\n    }\n\n    public void fillProcessQueueInfo(final ProcessQueueInfo info) {\n        try {\n            this.treeMapLock.readLock().lockInterruptibly();\n\n            if (!this.msgTreeMap.isEmpty()) {\n                info.setCachedMsgMinOffset(this.msgTreeMap.firstKey());\n                info.setCachedMsgMaxOffset(this.msgTreeMap.lastKey());\n                info.setCachedMsgCount(this.msgTreeMap.size());\n            }\n            info.setCachedMsgSizeInMiB((int) (this.msgSize.get() / (1024 * 1024)));\n\n            if (!this.consumingMsgOrderlyTreeMap.isEmpty()) {\n                info.setTransactionMsgMinOffset(this.consumingMsgOrderlyTreeMap.firstKey());\n                info.setTransactionMsgMaxOffset(this.consumingMsgOrderlyTreeMap.lastKey());\n                info.setTransactionMsgCount(this.consumingMsgOrderlyTreeMap.size());\n            }\n\n            info.setLocked(this.locked);\n            info.setTryUnlockTimes(this.tryUnlockTimes.get());\n            info.setLastLockTimestamp(this.lastLockTimestamp);\n\n            info.setDroped(this.dropped);\n            info.setLastPullTimestamp(this.lastPullTimestamp);\n            info.setLastConsumeTimestamp(this.lastConsumeTimestamp);\n        } catch (Exception e) {\n        } finally {\n            this.treeMapLock.readLock().unlock();\n        }\n    }\n\n    public long getLastConsumeTimestamp() {\n        return lastConsumeTimestamp;\n    }\n\n    public void setLastConsumeTimestamp(long lastConsumeTimestamp) {\n        this.lastConsumeTimestamp = lastConsumeTimestamp;\n    }\n\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Random;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.client.consumer.PopCallback;\nimport org.apache.rocketmq.client.consumer.PullCallback;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.hook.FilterMessageContext;\nimport org.apache.rocketmq.client.hook.FilterMessageHook;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.FindBrokerResult;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.common.sysflag.PullSysFlag;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class PullAPIWrapper {\n    private static final Logger log = LoggerFactory.getLogger(PullAPIWrapper.class);\n    private final MQClientInstance mQClientFactory;\n    private final String consumerGroup;\n    private final boolean unitMode;\n    private ConcurrentMap<MessageQueue, AtomicLong/* brokerId */> pullFromWhichNodeTable =\n        new ConcurrentHashMap<>(32);\n    private volatile boolean connectBrokerByUser = false;\n    private volatile long defaultBrokerId = MixAll.MASTER_ID;\n    private Random random = new Random(System.nanoTime());\n    private ArrayList<FilterMessageHook> filterMessageHookList = new ArrayList<>();\n\n    public PullAPIWrapper(MQClientInstance mQClientFactory, String consumerGroup, boolean unitMode) {\n        this.mQClientFactory = mQClientFactory;\n        this.consumerGroup = consumerGroup;\n        this.unitMode = unitMode;\n    }\n\n    public PullResult processPullResult(final MessageQueue mq, final PullResult pullResult,\n        final SubscriptionData subscriptionData) {\n        PullResultExt pullResultExt = (PullResultExt) pullResult;\n\n        this.updatePullFromWhichNode(mq, pullResultExt.getSuggestWhichBrokerId());\n        if (PullStatus.FOUND == pullResult.getPullStatus()) {\n            ByteBuffer byteBuffer = ByteBuffer.wrap(pullResultExt.getMessageBinary());\n            List<MessageExt> msgList = MessageDecoder.decodesBatch(\n                byteBuffer,\n                this.mQClientFactory.getClientConfig().isDecodeReadBody(),\n                this.mQClientFactory.getClientConfig().isDecodeDecompressBody(),\n                true\n            );\n\n            boolean needDecodeInnerMessage = false;\n            for (MessageExt messageExt: msgList) {\n                if (MessageSysFlag.check(messageExt.getSysFlag(), MessageSysFlag.INNER_BATCH_FLAG)\n                    && MessageSysFlag.check(messageExt.getSysFlag(), MessageSysFlag.NEED_UNWRAP_FLAG)) {\n                    needDecodeInnerMessage = true;\n                    break;\n                }\n            }\n            if (needDecodeInnerMessage) {\n                List<MessageExt> innerMsgList = new ArrayList<>();\n                try {\n                    for (MessageExt messageExt: msgList) {\n                        if (MessageSysFlag.check(messageExt.getSysFlag(), MessageSysFlag.INNER_BATCH_FLAG)\n                            && MessageSysFlag.check(messageExt.getSysFlag(), MessageSysFlag.NEED_UNWRAP_FLAG)) {\n                            MessageDecoder.decodeMessage(messageExt, innerMsgList);\n                        } else {\n                            innerMsgList.add(messageExt);\n                        }\n                    }\n                    msgList = innerMsgList;\n                } catch (Throwable t) {\n                    log.error(\"Try to decode the inner batch failed for {}\", pullResult.toString(), t);\n                }\n            }\n\n            List<MessageExt> msgListFilterAgain = msgList;\n            if (!subscriptionData.getTagsSet().isEmpty() && !subscriptionData.isClassFilterMode()) {\n                msgListFilterAgain = new ArrayList<>(msgList.size());\n                for (MessageExt msg : msgList) {\n                    if (msg.getTags() != null) {\n                        if (subscriptionData.getTagsSet().contains(msg.getTags())) {\n                            msgListFilterAgain.add(msg);\n                        }\n                    }\n                }\n            }\n\n            if (this.hasHook()) {\n                FilterMessageContext filterMessageContext = new FilterMessageContext();\n                filterMessageContext.setUnitMode(unitMode);\n                filterMessageContext.setMsgList(msgListFilterAgain);\n                this.executeHook(filterMessageContext);\n            }\n\n            for (MessageExt msg : msgListFilterAgain) {\n                String traFlag = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED);\n                if (Boolean.parseBoolean(traFlag)) {\n                    msg.setTransactionId(msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));\n                }\n                MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MIN_OFFSET,\n                    Long.toString(pullResult.getMinOffset()));\n                MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MAX_OFFSET,\n                    Long.toString(pullResult.getMaxOffset()));\n                msg.setBrokerName(mq.getBrokerName());\n                msg.setQueueId(mq.getQueueId());\n                if (pullResultExt.getOffsetDelta() != null) {\n                    msg.setQueueOffset(pullResultExt.getOffsetDelta() + msg.getQueueOffset());\n                }\n            }\n\n            pullResultExt.setMsgFoundList(msgListFilterAgain);\n        }\n\n        pullResultExt.setMessageBinary(null);\n\n        return pullResult;\n    }\n\n    public void updatePullFromWhichNode(final MessageQueue mq, final long brokerId) {\n        AtomicLong suggest = this.pullFromWhichNodeTable.get(mq);\n        if (null == suggest) {\n            this.pullFromWhichNodeTable.put(mq, new AtomicLong(brokerId));\n        } else {\n            suggest.set(brokerId);\n        }\n    }\n\n    public boolean hasHook() {\n        return !this.filterMessageHookList.isEmpty();\n    }\n\n    public void executeHook(final FilterMessageContext context) {\n        if (!this.filterMessageHookList.isEmpty()) {\n            for (FilterMessageHook hook : this.filterMessageHookList) {\n                try {\n                    hook.filterMessage(context);\n                } catch (Throwable e) {\n                    log.error(\"execute hook error. hookName={}\", hook.hookName());\n                }\n            }\n        }\n    }\n\n    public PullResult pullKernelImpl(\n        final MessageQueue mq,\n        final String subExpression,\n        final String expressionType,\n        final long subVersion,\n        final long offset,\n        final int maxNums,\n        final int maxSizeInBytes,\n        final int sysFlag,\n        final long commitOffset,\n        final long brokerSuspendMaxTimeMillis,\n        final long timeoutMillis,\n        final CommunicationMode communicationMode,\n        final PullCallback pullCallback\n    ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        FindBrokerResult findBrokerResult =\n            this.mQClientFactory.findBrokerAddressInSubscribe(this.mQClientFactory.getBrokerNameFromMessageQueue(mq),\n                this.recalculatePullFromWhichNode(mq), false);\n        if (null == findBrokerResult) {\n            this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());\n            findBrokerResult =\n                this.mQClientFactory.findBrokerAddressInSubscribe(this.mQClientFactory.getBrokerNameFromMessageQueue(mq),\n                    this.recalculatePullFromWhichNode(mq), false);\n        }\n\n\n        if (findBrokerResult != null) {\n            {\n                // check version\n                if (!ExpressionType.isTagType(expressionType)\n                    && findBrokerResult.getBrokerVersion() < MQVersion.Version.V4_1_0_SNAPSHOT.ordinal()) {\n                    throw new MQClientException(\"The broker[\" + mq.getBrokerName() + \", \"\n                        + findBrokerResult.getBrokerVersion() + \"] does not upgrade to support for filter message by \" + expressionType, null);\n                }\n            }\n            int sysFlagInner = sysFlag;\n\n            if (findBrokerResult.isSlave()) {\n                sysFlagInner = PullSysFlag.clearCommitOffsetFlag(sysFlagInner);\n            }\n\n            PullMessageRequestHeader requestHeader = new PullMessageRequestHeader();\n            requestHeader.setConsumerGroup(this.consumerGroup);\n            requestHeader.setTopic(mq.getTopic());\n            requestHeader.setQueueId(mq.getQueueId());\n            requestHeader.setQueueOffset(offset);\n            requestHeader.setMaxMsgNums(maxNums);\n            requestHeader.setSysFlag(sysFlagInner);\n            requestHeader.setCommitOffset(commitOffset);\n            requestHeader.setSuspendTimeoutMillis(brokerSuspendMaxTimeMillis);\n            requestHeader.setSubscription(subExpression);\n            requestHeader.setSubVersion(subVersion);\n            requestHeader.setMaxMsgBytes(maxSizeInBytes);\n            requestHeader.setExpressionType(expressionType);\n            requestHeader.setBrokerName(mq.getBrokerName());\n\n            String brokerAddr = findBrokerResult.getBrokerAddr();\n            if (PullSysFlag.hasClassFilterFlag(sysFlagInner)) {\n                brokerAddr = computePullFromWhichFilterServer(mq.getTopic(), brokerAddr);\n            }\n\n            PullResult pullResult = this.mQClientFactory.getMQClientAPIImpl().pullMessage(\n                brokerAddr,\n                requestHeader,\n                timeoutMillis,\n                communicationMode,\n                pullCallback);\n\n            return pullResult;\n        }\n\n        throw new MQClientException(\"The broker[\" + mq.getBrokerName() + \"] not exist\", null);\n    }\n\n    public PullResult pullKernelImpl(\n        MessageQueue mq,\n        final String subExpression,\n        final String expressionType,\n        final long subVersion,\n        long offset,\n        final int maxNums,\n        final int sysFlag,\n        long commitOffset,\n        final long brokerSuspendMaxTimeMillis,\n        final long timeoutMillis,\n        final CommunicationMode communicationMode,\n        PullCallback pullCallback\n    ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        return pullKernelImpl(\n                mq,\n                subExpression,\n                expressionType,\n                subVersion, offset,\n                maxNums,\n                Integer.MAX_VALUE,\n                sysFlag,\n                commitOffset,\n                brokerSuspendMaxTimeMillis,\n                timeoutMillis,\n                communicationMode,\n                pullCallback\n        );\n    }\n\n    public long recalculatePullFromWhichNode(final MessageQueue mq) {\n        if (this.isConnectBrokerByUser()) {\n            return this.defaultBrokerId;\n        }\n\n        AtomicLong suggest = this.pullFromWhichNodeTable.get(mq);\n        if (suggest != null) {\n            return suggest.get();\n        }\n\n        return MixAll.MASTER_ID;\n    }\n\n    private String computePullFromWhichFilterServer(final String topic, final String brokerAddr)\n        throws MQClientException {\n        ConcurrentMap<String, TopicRouteData> topicRouteTable = this.mQClientFactory.getTopicRouteTable();\n        if (topicRouteTable != null) {\n            TopicRouteData topicRouteData = topicRouteTable.get(topic);\n            List<String> list = topicRouteData.getFilterServerTable().get(brokerAddr);\n\n            if (list != null && !list.isEmpty()) {\n                return list.get(randomNum() % list.size());\n            }\n        }\n\n        throw new MQClientException(\"Find Filter Server Failed, Broker Addr: \" + brokerAddr + \" topic: \"\n            + topic, null);\n    }\n\n    public boolean isConnectBrokerByUser() {\n        return connectBrokerByUser;\n    }\n\n    public void setConnectBrokerByUser(boolean connectBrokerByUser) {\n        this.connectBrokerByUser = connectBrokerByUser;\n\n    }\n\n    public int randomNum() {\n        int value = random.nextInt();\n        if (value < 0) {\n            value = Math.abs(value);\n            if (value < 0)\n                value = 0;\n        }\n        return value;\n    }\n\n    public void registerFilterMessageHook(ArrayList<FilterMessageHook> filterMessageHookList) {\n        this.filterMessageHookList = filterMessageHookList;\n    }\n\n    public long getDefaultBrokerId() {\n        return defaultBrokerId;\n    }\n\n    public void setDefaultBrokerId(long defaultBrokerId) {\n        this.defaultBrokerId = defaultBrokerId;\n    }\n\n\n    /**\n     *\n     * @param mq\n     * @param invisibleTime\n     * @param maxNums\n     * @param consumerGroup\n     * @param timeout\n     * @param popCallback\n     * @param poll\n     * @param initMode\n    //     * @param expressionType\n    //     * @param expression\n     * @param order\n     * @throws MQClientException\n     * @throws RemotingException\n     * @throws InterruptedException\n     */\n    public void popAsync(MessageQueue mq, long invisibleTime, int maxNums, String consumerGroup,\n                         long timeout, PopCallback popCallback, boolean poll, int initMode, boolean order, String expressionType, String expression)\n        throws MQClientException, RemotingException, InterruptedException {\n        FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), MixAll.MASTER_ID, true);\n        if (null == findBrokerResult) {\n            this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());\n            findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), MixAll.MASTER_ID, true);\n        }\n        if (findBrokerResult != null) {\n            PopMessageRequestHeader requestHeader = new PopMessageRequestHeader();\n            requestHeader.setConsumerGroup(consumerGroup);\n            requestHeader.setTopic(mq.getTopic());\n            requestHeader.setQueueId(mq.getQueueId());\n            requestHeader.setMaxMsgNums(maxNums);\n            requestHeader.setInvisibleTime(invisibleTime);\n            requestHeader.setInitMode(initMode);\n            requestHeader.setExpType(expressionType);\n            requestHeader.setExp(expression);\n            requestHeader.setOrder(order);\n            requestHeader.setBrokerName(mq.getBrokerName());\n            //give 1000 ms for server response\n            if (poll) {\n                requestHeader.setPollTime(timeout);\n                requestHeader.setBornTime(System.currentTimeMillis());\n                // timeout + 10s, fix the too earlier timeout of client when long polling.\n                timeout += 10 * 1000;\n            }\n            String brokerAddr = findBrokerResult.getBrokerAddr();\n            this.mQClientFactory.getMQClientAPIImpl().popMessageAsync(mq.getBrokerName(), brokerAddr, requestHeader, timeout, popCallback);\n            return;\n        }\n        throw new MQClientException(\"The broker[\" + mq.getBrokerName() + \"] not exist\", null);\n    }\n\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullMessageService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.message.MessageRequestMode;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class PullMessageService extends ServiceThread {\n    private final Logger logger = LoggerFactory.getLogger(PullMessageService.class);\n    private final LinkedBlockingQueue<MessageRequest> messageRequestQueue = new LinkedBlockingQueue<>();\n\n    private final MQClientInstance mQClientFactory;\n    private final ScheduledExecutorService scheduledExecutorService = Executors\n        .newSingleThreadScheduledExecutor(new ThreadFactoryImpl(\"PullMessageServiceScheduledThread\"));\n\n    public PullMessageService(MQClientInstance mQClientFactory) {\n        this.mQClientFactory = mQClientFactory;\n    }\n\n    public void executePullRequestLater(final PullRequest pullRequest, final long timeDelay) {\n        if (!isStopped()) {\n            this.scheduledExecutorService.schedule(new Runnable() {\n                @Override\n                public void run() {\n                    PullMessageService.this.executePullRequestImmediately(pullRequest);\n                }\n            }, timeDelay, TimeUnit.MILLISECONDS);\n        } else {\n            logger.warn(\"PullMessageServiceScheduledThread has shutdown\");\n        }\n    }\n\n    public void executePullRequestImmediately(final PullRequest pullRequest) {\n        try {\n            this.messageRequestQueue.put(pullRequest);\n        } catch (InterruptedException e) {\n            logger.error(\"executePullRequestImmediately pullRequestQueue.put\", e);\n        }\n    }\n\n    public void executePopPullRequestLater(final PopRequest popRequest, final long timeDelay) {\n        if (!isStopped()) {\n            this.scheduledExecutorService.schedule(new Runnable() {\n                @Override\n                public void run() {\n                    PullMessageService.this.executePopPullRequestImmediately(popRequest);\n                }\n            }, timeDelay, TimeUnit.MILLISECONDS);\n        } else {\n            logger.warn(\"PullMessageServiceScheduledThread has shutdown\");\n        }\n    }\n\n    public void executePopPullRequestImmediately(final PopRequest popRequest) {\n        try {\n            this.messageRequestQueue.put(popRequest);\n        } catch (InterruptedException e) {\n            logger.error(\"executePullRequestImmediately pullRequestQueue.put\", e);\n        }\n    }\n\n    public void executeTaskLater(final Runnable r, final long timeDelay) {\n        if (!isStopped()) {\n            this.scheduledExecutorService.schedule(r, timeDelay, TimeUnit.MILLISECONDS);\n        } else {\n            logger.warn(\"PullMessageServiceScheduledThread has shutdown\");\n        }\n    }\n\n    public void executeTask(final Runnable r) {\n        if (!isStopped()) {\n            this.scheduledExecutorService.execute(r);\n        } else {\n            logger.warn(\"PullMessageServiceScheduledThread has shutdown\");\n        }\n    }\n\n    public ScheduledExecutorService getScheduledExecutorService() {\n        return scheduledExecutorService;\n    }\n\n    private void pullMessage(final PullRequest pullRequest) {\n        final MQConsumerInner consumer = this.mQClientFactory.selectConsumer(pullRequest.getConsumerGroup());\n        if (consumer != null) {\n            DefaultMQPushConsumerImpl impl = (DefaultMQPushConsumerImpl) consumer;\n            impl.pullMessage(pullRequest);\n        } else {\n            logger.warn(\"No matched consumer for the PullRequest {}, drop it\", pullRequest);\n        }\n    }\n\n    private void popMessage(final PopRequest popRequest) {\n        final MQConsumerInner consumer = this.mQClientFactory.selectConsumer(popRequest.getConsumerGroup());\n        if (consumer != null) {\n            DefaultMQPushConsumerImpl impl = (DefaultMQPushConsumerImpl) consumer;\n            impl.popMessage(popRequest);\n        } else {\n            logger.warn(\"No matched consumer for the PopRequest {}, drop it\", popRequest);\n        }\n    }\n\n    @Override\n    public void run() {\n        logger.info(this.getServiceName() + \" service started\");\n\n        while (!this.isStopped()) {\n            try {\n                MessageRequest messageRequest = this.messageRequestQueue.take();\n                if (messageRequest.getMessageRequestMode() == MessageRequestMode.POP) {\n                    this.popMessage((PopRequest) messageRequest);\n                } else {\n                    this.pullMessage((PullRequest) messageRequest);\n                }\n            } catch (InterruptedException ignored) {\n            } catch (Throwable e) {\n                logger.error(\"Pull Message Service Run Method exception\", e);\n            }\n        }\n\n        logger.info(this.getServiceName() + \" service end\");\n    }\n\n    @Override\n    public void shutdown(boolean interrupt) {\n        super.shutdown(interrupt);\n        ThreadUtils.shutdownGracefully(this.scheduledExecutorService, 1000, TimeUnit.MILLISECONDS);\n    }\n\n    @Override\n    public String getServiceName() {\n        return PullMessageService.class.getSimpleName();\n    }\n\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullRequest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.message.MessageRequestMode;\n\npublic class PullRequest implements MessageRequest {\n    private String consumerGroup;\n    private MessageQueue messageQueue;\n    private ProcessQueue processQueue;\n    private long nextOffset;\n    private boolean previouslyLocked = false;\n\n    public boolean isPreviouslyLocked() {\n        return previouslyLocked;\n    }\n\n    public void setPreviouslyLocked(boolean previouslyLocked) {\n        this.previouslyLocked = previouslyLocked;\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public MessageQueue getMessageQueue() {\n        return messageQueue;\n    }\n\n    public void setMessageQueue(MessageQueue messageQueue) {\n        this.messageQueue = messageQueue;\n    }\n\n    public long getNextOffset() {\n        return nextOffset;\n    }\n\n    public void setNextOffset(long nextOffset) {\n        this.nextOffset = nextOffset;\n    }\n\n    @Override\n    public int hashCode() {\n        final int prime = 31;\n        int result = 1;\n        result = prime * result + ((consumerGroup == null) ? 0 : consumerGroup.hashCode());\n        result = prime * result + ((messageQueue == null) ? 0 : messageQueue.hashCode());\n        return result;\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj)\n            return true;\n        if (obj == null)\n            return false;\n        if (getClass() != obj.getClass())\n            return false;\n        PullRequest other = (PullRequest) obj;\n        if (consumerGroup == null) {\n            if (other.consumerGroup != null)\n                return false;\n        } else if (!consumerGroup.equals(other.consumerGroup))\n            return false;\n        if (messageQueue == null) {\n            if (other.messageQueue != null)\n                return false;\n        } else if (!messageQueue.equals(other.messageQueue))\n            return false;\n        return true;\n    }\n\n    @Override\n    public String toString() {\n        return \"PullRequest [consumerGroup=\" + consumerGroup + \", messageQueue=\" + messageQueue\n            + \", nextOffset=\" + nextOffset + \"]\";\n    }\n\n    public ProcessQueue getProcessQueue() {\n        return processQueue;\n    }\n\n    public void setProcessQueue(ProcessQueue processQueue) {\n        this.processQueue = processQueue;\n    }\n\n    @Override\n    public MessageRequestMode getMessageRequestMode() {\n        return MessageRequestMode.PULL;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullResultExt.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport java.util.List;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic class PullResultExt extends PullResult {\n    private final long suggestWhichBrokerId;\n    private byte[] messageBinary;\n\n    private final Long offsetDelta;\n\n    public PullResultExt(PullStatus pullStatus, long nextBeginOffset, long minOffset, long maxOffset,\n        List<MessageExt> msgFoundList, final long suggestWhichBrokerId, final byte[] messageBinary) {\n        this(pullStatus, nextBeginOffset, minOffset, maxOffset, msgFoundList, suggestWhichBrokerId, messageBinary, 0L);\n    }\n\n    public PullResultExt(PullStatus pullStatus, long nextBeginOffset, long minOffset, long maxOffset,\n                         List<MessageExt> msgFoundList, final long suggestWhichBrokerId, final byte[] messageBinary, final Long offsetDelta) {\n        super(pullStatus, nextBeginOffset, minOffset, maxOffset, msgFoundList);\n        this.suggestWhichBrokerId = suggestWhichBrokerId;\n        this.messageBinary = messageBinary;\n        this.offsetDelta = offsetDelta;\n    }\n\n    public Long getOffsetDelta() {\n        return offsetDelta;\n    }\n\n    public byte[] getMessageBinary() {\n        return messageBinary;\n    }\n\n    public void setMessageBinary(byte[] messageBinary) {\n        this.messageBinary = messageBinary;\n    }\n\n    public long getSuggestWhichBrokerId() {\n        return suggestWhichBrokerId;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Map.Entry;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.FindBrokerResult;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.message.MessageQueueAssignment;\nimport org.apache.rocketmq.common.message.MessageRequestMode;\nimport org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.filter.FilterAPI;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic abstract class RebalanceImpl {\n    protected static final Logger log = LoggerFactory.getLogger(RebalanceImpl.class);\n\n    protected final ConcurrentMap<MessageQueue, ProcessQueue> processQueueTable = new ConcurrentHashMap<>(64);\n    protected final ConcurrentMap<MessageQueue, PopProcessQueue> popProcessQueueTable = new ConcurrentHashMap<>(64);\n\n    protected final ConcurrentMap<String/* topic */, Set<MessageQueue>> topicSubscribeInfoTable =\n        new ConcurrentHashMap<>();\n    protected final ConcurrentMap<String /* topic */, SubscriptionData> subscriptionInner =\n        new ConcurrentHashMap<>();\n    protected String consumerGroup;\n    protected MessageModel messageModel;\n    protected AllocateMessageQueueStrategy allocateMessageQueueStrategy;\n    protected MQClientInstance mQClientFactory;\n    private static final int QUERY_ASSIGNMENT_TIMEOUT = 3000;\n\n    public RebalanceImpl(String consumerGroup, MessageModel messageModel,\n        AllocateMessageQueueStrategy allocateMessageQueueStrategy,\n        MQClientInstance mQClientFactory) {\n        this.consumerGroup = consumerGroup;\n        this.messageModel = messageModel;\n        this.allocateMessageQueueStrategy = allocateMessageQueueStrategy;\n        this.mQClientFactory = mQClientFactory;\n    }\n\n    public void unlock(final MessageQueue mq, final boolean oneway) {\n        FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(this.mQClientFactory.getBrokerNameFromMessageQueue(mq), MixAll.MASTER_ID, true);\n        if (findBrokerResult != null) {\n            UnlockBatchRequestBody requestBody = new UnlockBatchRequestBody();\n            requestBody.setConsumerGroup(this.consumerGroup);\n            requestBody.setClientId(this.mQClientFactory.getClientId());\n            requestBody.getMqSet().add(mq);\n\n            try {\n                this.mQClientFactory.getMQClientAPIImpl().unlockBatchMQ(findBrokerResult.getBrokerAddr(), requestBody, 1000, oneway);\n                log.warn(\"unlock messageQueue. group:{}, clientId:{}, mq:{}\",\n                    this.consumerGroup,\n                    this.mQClientFactory.getClientId(),\n                    mq);\n            } catch (Exception e) {\n                log.error(\"unlockBatchMQ exception, \" + mq, e);\n            }\n        }\n    }\n\n    public void unlockAll(final boolean oneway) {\n        HashMap<String, Set<MessageQueue>> brokerMqs = this.buildProcessQueueTableByBrokerName();\n\n        for (final Map.Entry<String, Set<MessageQueue>> entry : brokerMqs.entrySet()) {\n            final String brokerName = entry.getKey();\n            final Set<MessageQueue> mqs = entry.getValue();\n\n            if (mqs.isEmpty()) {\n                continue;\n            }\n\n            FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, true);\n            if (findBrokerResult != null) {\n                UnlockBatchRequestBody requestBody = new UnlockBatchRequestBody();\n                requestBody.setConsumerGroup(this.consumerGroup);\n                requestBody.setClientId(this.mQClientFactory.getClientId());\n                requestBody.setMqSet(mqs);\n\n                try {\n                    this.mQClientFactory.getMQClientAPIImpl().unlockBatchMQ(findBrokerResult.getBrokerAddr(), requestBody, 1000, oneway);\n\n                    for (MessageQueue mq : mqs) {\n                        ProcessQueue processQueue = this.processQueueTable.get(mq);\n                        if (processQueue != null) {\n                            processQueue.setLocked(false);\n                            log.info(\"the message queue unlock OK, Group: {} {}\", this.consumerGroup, mq);\n                        }\n                    }\n                } catch (Exception e) {\n                    log.error(\"unlockBatchMQ exception, \" + mqs, e);\n                }\n            }\n        }\n    }\n\n    private HashMap<String/* brokerName */, Set<MessageQueue>> buildProcessQueueTableByBrokerName() {\n        HashMap<String, Set<MessageQueue>> result = new HashMap<>();\n\n        for (Map.Entry<MessageQueue, ProcessQueue> entry : this.processQueueTable.entrySet()) {\n            MessageQueue mq = entry.getKey();\n            ProcessQueue pq = entry.getValue();\n\n            if (pq.isDropped()) {\n                continue;\n            }\n\n            String destBrokerName = this.mQClientFactory.getBrokerNameFromMessageQueue(mq);\n            Set<MessageQueue> mqs = result.get(destBrokerName);\n            if (null == mqs) {\n                mqs = new HashSet<>();\n                result.put(mq.getBrokerName(), mqs);\n            }\n\n            mqs.add(mq);\n        }\n\n        return result;\n    }\n\n    public boolean lock(final MessageQueue mq) {\n        FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(this.mQClientFactory.getBrokerNameFromMessageQueue(mq), MixAll.MASTER_ID, true);\n        if (findBrokerResult != null) {\n            LockBatchRequestBody requestBody = new LockBatchRequestBody();\n            requestBody.setConsumerGroup(this.consumerGroup);\n            requestBody.setClientId(this.mQClientFactory.getClientId());\n            requestBody.getMqSet().add(mq);\n\n            try {\n                Set<MessageQueue> lockedMq =\n                    this.mQClientFactory.getMQClientAPIImpl().lockBatchMQ(findBrokerResult.getBrokerAddr(), requestBody, 1000);\n                for (MessageQueue mmqq : lockedMq) {\n                    ProcessQueue processQueue = this.processQueueTable.get(mmqq);\n                    if (processQueue != null) {\n                        processQueue.setLocked(true);\n                        processQueue.setLastLockTimestamp(System.currentTimeMillis());\n                    }\n                }\n\n                boolean lockOK = lockedMq.contains(mq);\n                log.info(\"message queue lock {}, {} {}\", lockOK ? \"OK\" : \"Failed\", this.consumerGroup, mq);\n                return lockOK;\n            } catch (Exception e) {\n                log.error(\"lockBatchMQ exception, \" + mq, e);\n            }\n        }\n\n        return false;\n    }\n\n    public void lockAll() {\n        HashMap<String, Set<MessageQueue>> brokerMqs = this.buildProcessQueueTableByBrokerName();\n\n        Iterator<Entry<String, Set<MessageQueue>>> it = brokerMqs.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, Set<MessageQueue>> entry = it.next();\n            final String brokerName = entry.getKey();\n            final Set<MessageQueue> mqs = entry.getValue();\n\n            if (mqs.isEmpty()) {\n                continue;\n            }\n\n            FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, true);\n            if (findBrokerResult != null) {\n                LockBatchRequestBody requestBody = new LockBatchRequestBody();\n                requestBody.setConsumerGroup(this.consumerGroup);\n                requestBody.setClientId(this.mQClientFactory.getClientId());\n                requestBody.setMqSet(mqs);\n\n                try {\n                    Set<MessageQueue> lockOKMQSet =\n                        this.mQClientFactory.getMQClientAPIImpl().lockBatchMQ(findBrokerResult.getBrokerAddr(), requestBody, 1000);\n\n                    for (MessageQueue mq : mqs) {\n                        ProcessQueue processQueue = this.processQueueTable.get(mq);\n                        if (processQueue != null) {\n                            if (lockOKMQSet.contains(mq)) {\n                                if (!processQueue.isLocked()) {\n                                    log.info(\"the message queue locked OK, Group: {} {}\", this.consumerGroup, mq);\n                                }\n                                processQueue.setLocked(true);\n                                processQueue.setLastLockTimestamp(System.currentTimeMillis());\n                            } else {\n                                processQueue.setLocked(false);\n                                log.warn(\"the message queue locked Failed, Group: {} {}\", this.consumerGroup, mq);\n                            }\n                        }\n                    }\n                } catch (Exception e) {\n                    log.error(\"lockBatchMQ exception, \" + mqs, e);\n                }\n            }\n        }\n    }\n\n    public boolean clientRebalance(String topic) {\n        return true;\n    }\n\n    public boolean doRebalance(final boolean isOrder) {\n        boolean balanced = true;\n        Map<String, SubscriptionData> subTable = this.getSubscriptionInner();\n        if (subTable != null) {\n            for (final Map.Entry<String, SubscriptionData> entry : subTable.entrySet()) {\n                final String topic = entry.getKey();\n                try {\n                    if (!clientRebalance(topic)) {\n                        boolean result = this.getRebalanceResultFromBroker(topic, isOrder);\n                        if (!result) {\n                            balanced = false;\n                        }\n                    } else {\n                        boolean result = this.rebalanceByTopic(topic, isOrder);\n                        if (!result) {\n                            balanced = false;\n                        }\n                    }\n                } catch (Throwable e) {\n                    if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n                        log.warn(\"rebalance Exception\", e);\n                        balanced = false;\n                    }\n                }\n            }\n        }\n\n        this.truncateMessageQueueNotMyTopic();\n\n        return balanced;\n    }\n\n    public ConcurrentMap<String, SubscriptionData> getSubscriptionInner() {\n        return subscriptionInner;\n    }\n\n    private boolean rebalanceByTopic(final String topic, final boolean isOrder) {\n        boolean balanced = true;\n        switch (messageModel) {\n            case BROADCASTING: {\n                Set<MessageQueue> mqSet = this.topicSubscribeInfoTable.get(topic);\n                if (mqSet != null) {\n                    boolean changed = this.updateProcessQueueTableInRebalance(topic, mqSet, false);\n                    if (changed) {\n                        this.messageQueueChanged(topic, mqSet, mqSet);\n                        log.info(\"messageQueueChanged {} {} {} {}\", consumerGroup, topic, mqSet, mqSet);\n                    }\n\n                    balanced = mqSet.equals(getWorkingMessageQueue(topic));\n                } else {\n                    this.messageQueueChanged(topic, Collections.<MessageQueue>emptySet(), Collections.<MessageQueue>emptySet());\n                    log.warn(\"doRebalance, {}, but the topic[{}] not exist.\", consumerGroup, topic);\n                }\n                break;\n            }\n            case CLUSTERING: {\n                Set<MessageQueue> mqSet = this.topicSubscribeInfoTable.get(topic);\n                if (null == mqSet || mqSet.isEmpty()) {\n                    if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n                        this.messageQueueChanged(topic, Collections.<MessageQueue>emptySet(), Collections.<MessageQueue>emptySet());\n                        log.warn(\"doRebalance, {}, but the topic[{}] not exist.\", consumerGroup, topic);\n                    }\n                    break;\n                }\n\n                List<String> cidAll = this.mQClientFactory.findConsumerIdList(topic, consumerGroup);\n                if (null == cidAll || cidAll.isEmpty()) {\n                    log.warn(\"doRebalance, {} {}, get consumer id list failed\", consumerGroup, topic);\n                } else {\n                    List<MessageQueue> mqAll = new ArrayList<>(mqSet);\n\n                    Collections.sort(mqAll);\n                    Collections.sort(cidAll);\n\n                    AllocateMessageQueueStrategy strategy = this.allocateMessageQueueStrategy;\n\n                    List<MessageQueue> allocateResult = null;\n                    try {\n                        allocateResult = strategy.allocate(\n                            this.consumerGroup,\n                            this.mQClientFactory.getClientId(),\n                            mqAll,\n                            cidAll);\n                    } catch (Throwable e) {\n                        log.error(\"allocate message queue exception. strategy name: {}, ex: {}\", strategy.getName(), e);\n                        return false;\n                    }\n\n                    Set<MessageQueue> allocateResultSet = new HashSet<>();\n                    if (allocateResult != null) {\n                        allocateResultSet.addAll(allocateResult);\n                    }\n\n                    boolean changed = this.updateProcessQueueTableInRebalance(topic, allocateResultSet, isOrder);\n                    if (changed) {\n                        log.info(\n                            \"client rebalanced result changed. allocateMessageQueueStrategyName={}, group={}, topic={}, clientId={}, mqAllSize={}, cidAllSize={}, rebalanceResultSize={}, rebalanceResultSet={}\",\n                            strategy.getName(), consumerGroup, topic, this.mQClientFactory.getClientId(), mqSet.size(), cidAll.size(),\n                            allocateResultSet.size(), allocateResultSet);\n                        this.messageQueueChanged(topic, mqSet, allocateResultSet);\n                    }\n\n                    balanced = allocateResultSet.equals(getWorkingMessageQueue(topic));\n                }\n                break;\n            }\n            default:\n                break;\n        }\n\n        return balanced;\n    }\n\n    private boolean getRebalanceResultFromBroker(final String topic, final boolean isOrder) {\n        String strategyName = this.allocateMessageQueueStrategy.getName();\n        Set<MessageQueueAssignment> messageQueueAssignments;\n        try {\n            messageQueueAssignments = this.mQClientFactory.queryAssignment(topic, consumerGroup,\n                strategyName, messageModel, QUERY_ASSIGNMENT_TIMEOUT);\n        } catch (Exception e) {\n            log.error(\"allocate message queue exception. strategy name: {}, ex: {}\", strategyName, e);\n            return false;\n        }\n\n        // null means invalid result, we should skip the update logic\n        if (messageQueueAssignments == null) {\n            return false;\n        }\n        Set<MessageQueue> mqSet = new HashSet<>();\n        for (MessageQueueAssignment messageQueueAssignment : messageQueueAssignments) {\n            if (messageQueueAssignment.getMessageQueue() != null) {\n                mqSet.add(messageQueueAssignment.getMessageQueue());\n            }\n        }\n        Set<MessageQueue> mqAll = null;\n        boolean changed = this.updateMessageQueueAssignment(topic, messageQueueAssignments, isOrder);\n        if (changed) {\n            log.info(\"broker rebalanced result changed. allocateMessageQueueStrategyName={}, group={}, topic={}, clientId={}, assignmentSet={}\",\n                strategyName, consumerGroup, topic, this.mQClientFactory.getClientId(), messageQueueAssignments);\n            this.messageQueueChanged(topic, mqAll, mqSet);\n        }\n\n        return mqSet.equals(getWorkingMessageQueue(topic));\n    }\n\n    private Set<MessageQueue> getWorkingMessageQueue(String topic) {\n        Set<MessageQueue> queueSet = new HashSet<>();\n        for (Entry<MessageQueue, ProcessQueue> entry : this.processQueueTable.entrySet()) {\n            MessageQueue mq = entry.getKey();\n            ProcessQueue pq = entry.getValue();\n\n            if (mq.getTopic().equals(topic) && !pq.isDropped()) {\n                queueSet.add(mq);\n            }\n        }\n\n        for (Entry<MessageQueue, PopProcessQueue> entry : this.popProcessQueueTable.entrySet()) {\n            MessageQueue mq = entry.getKey();\n            PopProcessQueue pq = entry.getValue();\n\n            if (mq.getTopic().equals(topic) && !pq.isDropped()) {\n                queueSet.add(mq);\n            }\n        }\n\n        return queueSet;\n    }\n\n    private void truncateMessageQueueNotMyTopic() {\n        Map<String, SubscriptionData> subTable = this.getSubscriptionInner();\n\n        for (MessageQueue mq : this.processQueueTable.keySet()) {\n            if (!subTable.containsKey(mq.getTopic())) {\n\n                ProcessQueue pq = this.processQueueTable.remove(mq);\n                if (pq != null) {\n                    pq.setDropped(true);\n                    log.info(\"doRebalance, {}, truncateMessageQueueNotMyTopic remove unnecessary mq, {}\", consumerGroup, mq);\n                }\n            }\n        }\n\n        for (MessageQueue mq : this.popProcessQueueTable.keySet()) {\n            if (!subTable.containsKey(mq.getTopic())) {\n\n                PopProcessQueue pq = this.popProcessQueueTable.remove(mq);\n                if (pq != null) {\n                    pq.setDropped(true);\n                    log.info(\"doRebalance, {}, truncateMessageQueueNotMyTopic remove unnecessary pop mq, {}\", consumerGroup, mq);\n                }\n            }\n        }\n    }\n\n    private boolean updateProcessQueueTableInRebalance(final String topic, final Set<MessageQueue> mqSet,\n        final boolean needLockMq) {\n        boolean changed = false;\n\n        // drop process queues no longer belong me\n        HashMap<MessageQueue, ProcessQueue> removeQueueMap = new HashMap<>(this.processQueueTable.size());\n        Iterator<Entry<MessageQueue, ProcessQueue>> it = this.processQueueTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<MessageQueue, ProcessQueue> next = it.next();\n            MessageQueue mq = next.getKey();\n            ProcessQueue pq = next.getValue();\n\n            if (mq.getTopic().equals(topic)) {\n                if (!mqSet.contains(mq)) {\n                    pq.setDropped(true);\n                    removeQueueMap.put(mq, pq);\n                } else if (pq.isPullExpired() && this.consumeType() == ConsumeType.CONSUME_PASSIVELY) {\n                    pq.setDropped(true);\n                    removeQueueMap.put(mq, pq);\n                    log.error(\"[BUG]doRebalance, {}, try remove unnecessary mq, {}, because pull is pause, so try to fixed it\",\n                        consumerGroup, mq);\n                }\n            }\n        }\n\n        // remove message queues no longer belong me\n        for (Entry<MessageQueue, ProcessQueue> entry : removeQueueMap.entrySet()) {\n            MessageQueue mq = entry.getKey();\n            ProcessQueue pq = entry.getValue();\n\n            if (this.removeUnnecessaryMessageQueue(mq, pq)) {\n                this.processQueueTable.remove(mq);\n                changed = true;\n                log.info(\"doRebalance, {}, remove unnecessary mq, {}\", consumerGroup, mq);\n            }\n        }\n\n        // add new message queue\n        boolean allMQLocked = true;\n        List<PullRequest> pullRequestList = new ArrayList<>();\n        for (MessageQueue mq : mqSet) {\n            if (!this.processQueueTable.containsKey(mq)) {\n                if (needLockMq && !this.lock(mq)) {\n                    log.warn(\"doRebalance, {}, add a new mq failed, {}, because lock failed\", consumerGroup, mq);\n                    allMQLocked = false;\n                    continue;\n                }\n\n                this.removeDirtyOffset(mq);\n                ProcessQueue pq = createProcessQueue();\n                pq.setLocked(true);\n                long nextOffset = this.computePullFromWhere(mq);\n                if (nextOffset >= 0) {\n                    ProcessQueue pre = this.processQueueTable.putIfAbsent(mq, pq);\n                    if (pre != null) {\n                        log.info(\"doRebalance, {}, mq already exists, {}\", consumerGroup, mq);\n                    } else {\n                        log.info(\"doRebalance, {}, add a new mq, {}\", consumerGroup, mq);\n                        PullRequest pullRequest = new PullRequest();\n                        pullRequest.setConsumerGroup(consumerGroup);\n                        pullRequest.setNextOffset(nextOffset);\n                        pullRequest.setMessageQueue(mq);\n                        pullRequest.setProcessQueue(pq);\n                        pullRequestList.add(pullRequest);\n                        changed = true;\n                    }\n                } else {\n                    log.warn(\"doRebalance, {}, add new mq failed, {}\", consumerGroup, mq);\n                }\n            }\n\n        }\n\n        if (!allMQLocked) {\n            mQClientFactory.rebalanceLater(500);\n        }\n\n        this.dispatchPullRequest(pullRequestList, 500);\n\n        return changed;\n    }\n\n    private boolean updateMessageQueueAssignment(final String topic, final Set<MessageQueueAssignment> assignments,\n        final boolean isOrder) {\n        boolean changed = false;\n\n        Map<MessageQueue, MessageQueueAssignment> mq2PushAssignment = new HashMap<>();\n        Map<MessageQueue, MessageQueueAssignment> mq2PopAssignment = new HashMap<>();\n        for (MessageQueueAssignment assignment : assignments) {\n            MessageQueue messageQueue = assignment.getMessageQueue();\n            if (messageQueue == null) {\n                continue;\n            }\n            if (MessageRequestMode.POP == assignment.getMode()) {\n                mq2PopAssignment.put(messageQueue, assignment);\n            } else {\n                mq2PushAssignment.put(messageQueue, assignment);\n            }\n        }\n\n        if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n            if (mq2PopAssignment.isEmpty() && !mq2PushAssignment.isEmpty()) {\n                //pop switch to push\n                //subscribe pop retry topic\n                try {\n                    final String retryTopic = KeyBuilder.buildPopRetryTopic(topic, getConsumerGroup());\n                    SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(retryTopic, SubscriptionData.SUB_ALL);\n                    getSubscriptionInner().put(retryTopic, subscriptionData);\n                } catch (Exception ignored) {\n                }\n\n            } else if (!mq2PopAssignment.isEmpty() && mq2PushAssignment.isEmpty()) {\n                //push switch to pop\n                //unsubscribe pop retry topic\n                try {\n                    final String retryTopic = KeyBuilder.buildPopRetryTopic(topic, getConsumerGroup());\n                    getSubscriptionInner().remove(retryTopic);\n                } catch (Exception ignored) {\n                }\n\n            }\n        }\n\n        {\n            // drop process queues no longer belong me\n            HashMap<MessageQueue, ProcessQueue> removeQueueMap = new HashMap<>(this.processQueueTable.size());\n            Iterator<Entry<MessageQueue, ProcessQueue>> it = this.processQueueTable.entrySet().iterator();\n            while (it.hasNext()) {\n                Entry<MessageQueue, ProcessQueue> next = it.next();\n                MessageQueue mq = next.getKey();\n                ProcessQueue pq = next.getValue();\n\n                if (mq.getTopic().equals(topic)) {\n                    if (!mq2PushAssignment.containsKey(mq)) {\n                        pq.setDropped(true);\n                        removeQueueMap.put(mq, pq);\n                    } else if (pq.isPullExpired() && this.consumeType() == ConsumeType.CONSUME_PASSIVELY) {\n                        pq.setDropped(true);\n                        removeQueueMap.put(mq, pq);\n                        log.error(\"[BUG]doRebalance, {}, try remove unnecessary mq, {}, because pull is pause, so try to fixed it\",\n                            consumerGroup, mq);\n                    }\n                }\n            }\n            // remove message queues no longer belong me\n            for (Entry<MessageQueue, ProcessQueue> entry : removeQueueMap.entrySet()) {\n                MessageQueue mq = entry.getKey();\n                ProcessQueue pq = entry.getValue();\n\n                if (this.removeUnnecessaryMessageQueue(mq, pq)) {\n                    this.processQueueTable.remove(mq);\n                    changed = true;\n                    log.info(\"doRebalance, {}, remove unnecessary mq, {}\", consumerGroup, mq);\n                }\n            }\n        }\n\n        {\n            HashMap<MessageQueue, PopProcessQueue> removeQueueMap = new HashMap<>(this.popProcessQueueTable.size());\n            Iterator<Entry<MessageQueue, PopProcessQueue>> it = this.popProcessQueueTable.entrySet().iterator();\n            while (it.hasNext()) {\n                Entry<MessageQueue, PopProcessQueue> next = it.next();\n                MessageQueue mq = next.getKey();\n                PopProcessQueue pq = next.getValue();\n\n                if (mq.getTopic().equals(topic)) {\n                    if (!mq2PopAssignment.containsKey(mq)) {\n                        //the queue is no longer your assignment\n                        pq.setDropped(true);\n                        removeQueueMap.put(mq, pq);\n                    } else if (pq.isPullExpired() && this.consumeType() == ConsumeType.CONSUME_PASSIVELY) {\n                        pq.setDropped(true);\n                        removeQueueMap.put(mq, pq);\n                        log.error(\"[BUG]doRebalance, {}, try remove unnecessary pop mq, {}, because pop is pause, so try to fixed it\",\n                            consumerGroup, mq);\n                    }\n                }\n            }\n            // remove message queues no longer belong me\n            for (Entry<MessageQueue, PopProcessQueue> entry : removeQueueMap.entrySet()) {\n                MessageQueue mq = entry.getKey();\n                PopProcessQueue pq = entry.getValue();\n\n                if (this.removeUnnecessaryPopMessageQueue(mq, pq)) {\n                    this.popProcessQueueTable.remove(mq);\n                    changed = true;\n                    log.info(\"doRebalance, {}, remove unnecessary pop mq, {}\", consumerGroup, mq);\n                }\n            }\n        }\n\n        {\n            // add new message queue\n            boolean allMQLocked = true;\n            List<PullRequest> pullRequestList = new ArrayList<>();\n            for (MessageQueue mq : mq2PushAssignment.keySet()) {\n                if (!this.processQueueTable.containsKey(mq)) {\n                    if (isOrder && !this.lock(mq)) {\n                        log.warn(\"doRebalance, {}, add a new mq failed, {}, because lock failed\", consumerGroup, mq);\n                        allMQLocked = false;\n                        continue;\n                    }\n\n                    this.removeDirtyOffset(mq);\n                    ProcessQueue pq = createProcessQueue();\n                    pq.setLocked(true);\n                    long nextOffset = -1L;\n                    try {\n                        nextOffset = this.computePullFromWhereWithException(mq);\n                    } catch (Exception e) {\n                        log.info(\"doRebalance, {}, compute offset failed, {}\", consumerGroup, mq);\n                        continue;\n                    }\n\n                    if (nextOffset >= 0) {\n                        ProcessQueue pre = this.processQueueTable.putIfAbsent(mq, pq);\n                        if (pre != null) {\n                            log.info(\"doRebalance, {}, mq already exists, {}\", consumerGroup, mq);\n                        } else {\n                            log.info(\"doRebalance, {}, add a new mq, {}\", consumerGroup, mq);\n                            PullRequest pullRequest = new PullRequest();\n                            pullRequest.setConsumerGroup(consumerGroup);\n                            pullRequest.setNextOffset(nextOffset);\n                            pullRequest.setMessageQueue(mq);\n                            pullRequest.setProcessQueue(pq);\n                            pullRequestList.add(pullRequest);\n                            changed = true;\n                        }\n                    } else {\n                        log.warn(\"doRebalance, {}, add new mq failed, {}\", consumerGroup, mq);\n                    }\n                }\n            }\n\n            if (!allMQLocked) {\n                mQClientFactory.rebalanceLater(500);\n            }\n            this.dispatchPullRequest(pullRequestList, 500);\n        }\n\n        {\n            // add new message queue\n            List<PopRequest> popRequestList = new ArrayList<>();\n            for (MessageQueue mq : mq2PopAssignment.keySet()) {\n                if (!this.popProcessQueueTable.containsKey(mq)) {\n                    PopProcessQueue pq = createPopProcessQueue();\n                    PopProcessQueue pre = this.popProcessQueueTable.putIfAbsent(mq, pq);\n                    if (pre != null) {\n                        log.info(\"doRebalance, {}, mq pop already exists, {}\", consumerGroup, mq);\n                    } else {\n                        log.info(\"doRebalance, {}, add a new pop mq, {}\", consumerGroup, mq);\n                        PopRequest popRequest = new PopRequest();\n                        popRequest.setTopic(topic);\n                        popRequest.setConsumerGroup(consumerGroup);\n                        popRequest.setMessageQueue(mq);\n                        popRequest.setPopProcessQueue(pq);\n                        popRequest.setInitMode(getConsumeInitMode());\n                        popRequestList.add(popRequest);\n                        changed = true;\n                    }\n                }\n            }\n\n            this.dispatchPopPullRequest(popRequestList, 500);\n        }\n\n        return changed;\n    }\n\n    public abstract void messageQueueChanged(final String topic, final Set<MessageQueue> mqAll,\n        final Set<MessageQueue> mqDivided);\n\n    public abstract boolean removeUnnecessaryMessageQueue(final MessageQueue mq, final ProcessQueue pq);\n\n    public boolean removeUnnecessaryPopMessageQueue(final MessageQueue mq, final PopProcessQueue pq) {\n        return true;\n    }\n\n    public abstract ConsumeType consumeType();\n\n    public abstract void removeDirtyOffset(final MessageQueue mq);\n\n    /**\n     * When the network is unstable, using this interface may return wrong offset.\n     * It is recommended to use computePullFromWhereWithException instead.\n     * @param mq\n     * @return offset\n     */\n    @Deprecated\n    public abstract long computePullFromWhere(final MessageQueue mq);\n\n    public abstract long computePullFromWhereWithException(final MessageQueue mq) throws MQClientException;\n\n    public abstract int getConsumeInitMode();\n\n    public abstract void dispatchPullRequest(final List<PullRequest> pullRequestList, final long delay);\n\n    public abstract void dispatchPopPullRequest(final List<PopRequest> pullRequestList, final long delay);\n\n    public abstract ProcessQueue createProcessQueue();\n\n    public abstract PopProcessQueue createPopProcessQueue();\n\n    public void removeProcessQueue(final MessageQueue mq) {\n        ProcessQueue prev = this.processQueueTable.remove(mq);\n        if (prev != null) {\n            boolean droped = prev.isDropped();\n            prev.setDropped(true);\n            this.removeUnnecessaryMessageQueue(mq, prev);\n            log.info(\"Fix Offset, {}, remove unnecessary mq, {} Droped: {}\", consumerGroup, mq, droped);\n        }\n    }\n\n    public ConcurrentMap<MessageQueue, ProcessQueue> getProcessQueueTable() {\n        return processQueueTable;\n    }\n\n    public ConcurrentMap<MessageQueue, PopProcessQueue> getPopProcessQueueTable() {\n        return popProcessQueueTable;\n    }\n\n    public ConcurrentMap<String, Set<MessageQueue>> getTopicSubscribeInfoTable() {\n        return topicSubscribeInfoTable;\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public MessageModel getMessageModel() {\n        return messageModel;\n    }\n\n    public void setMessageModel(MessageModel messageModel) {\n        this.messageModel = messageModel;\n    }\n\n    public AllocateMessageQueueStrategy getAllocateMessageQueueStrategy() {\n        return allocateMessageQueueStrategy;\n    }\n\n    public void setAllocateMessageQueueStrategy(AllocateMessageQueueStrategy allocateMessageQueueStrategy) {\n        this.allocateMessageQueueStrategy = allocateMessageQueueStrategy;\n    }\n\n    public MQClientInstance getmQClientFactory() {\n        return mQClientFactory;\n    }\n\n    public void setmQClientFactory(MQClientInstance mQClientFactory) {\n        this.mQClientFactory = mQClientFactory;\n    }\n\n    public void destroy() {\n        Iterator<Entry<MessageQueue, ProcessQueue>> it = this.processQueueTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<MessageQueue, ProcessQueue> next = it.next();\n            next.getValue().setDropped(true);\n        }\n\n        this.processQueueTable.clear();\n\n        Iterator<Entry<MessageQueue, PopProcessQueue>> popIt = this.popProcessQueueTable.entrySet().iterator();\n        while (popIt.hasNext()) {\n            Entry<MessageQueue, PopProcessQueue> next = popIt.next();\n            next.getValue().setDropped(true);\n        }\n        this.popProcessQueueTable.clear();\n    }\n\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport java.util.List;\nimport java.util.Set;\nimport org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;\nimport org.apache.rocketmq.client.consumer.MessageQueueListener;\nimport org.apache.rocketmq.client.consumer.store.ReadOffsetType;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\n\npublic class RebalanceLitePullImpl extends RebalanceImpl {\n\n    private final DefaultLitePullConsumerImpl litePullConsumerImpl;\n\n    public RebalanceLitePullImpl(DefaultLitePullConsumerImpl litePullConsumerImpl) {\n        this(null, null, null, null, litePullConsumerImpl);\n    }\n\n    public RebalanceLitePullImpl(String consumerGroup, MessageModel messageModel,\n        AllocateMessageQueueStrategy allocateMessageQueueStrategy,\n        MQClientInstance mQClientFactory, DefaultLitePullConsumerImpl litePullConsumerImpl) {\n        super(consumerGroup, messageModel, allocateMessageQueueStrategy, mQClientFactory);\n        this.litePullConsumerImpl = litePullConsumerImpl;\n    }\n\n    @Override\n    public void messageQueueChanged(String topic, Set<MessageQueue> mqAll, Set<MessageQueue> mqDivided) {\n        MessageQueueListener messageQueueListener = this.litePullConsumerImpl.getDefaultLitePullConsumer().getMessageQueueListener();\n        if (messageQueueListener != null) {\n            try {\n                messageQueueListener.messageQueueChanged(topic, mqAll, mqDivided);\n            } catch (Throwable e) {\n                log.error(\"messageQueueChanged exception\", e);\n            }\n        }\n    }\n\n    @Override\n    public boolean removeUnnecessaryMessageQueue(MessageQueue mq, ProcessQueue pq) {\n        this.litePullConsumerImpl.getOffsetStore().persist(mq);\n        this.litePullConsumerImpl.getOffsetStore().removeOffset(mq);\n        return true;\n    }\n\n    @Override\n    public ConsumeType consumeType() {\n        return ConsumeType.CONSUME_ACTIVELY;\n    }\n\n    @Override\n    public void removeDirtyOffset(final MessageQueue mq) {\n        this.litePullConsumerImpl.getOffsetStore().removeOffset(mq);\n    }\n\n    @Deprecated\n    @Override\n    public long computePullFromWhere(MessageQueue mq) {\n        long result = -1L;\n        try {\n            result = computePullFromWhereWithException(mq);\n        } catch (MQClientException e) {\n            log.warn(\"Compute consume offset exception, mq={}\", mq);\n        }\n        return result;\n    }\n\n    @Override\n    public long computePullFromWhereWithException(MessageQueue mq) throws MQClientException {\n        ConsumeFromWhere consumeFromWhere = litePullConsumerImpl.getDefaultLitePullConsumer().getConsumeFromWhere();\n        long result = -1;\n        switch (consumeFromWhere) {\n            case CONSUME_FROM_LAST_OFFSET: {\n                long lastOffset = litePullConsumerImpl.getOffsetStore().readOffset(mq, ReadOffsetType.MEMORY_FIRST_THEN_STORE);\n                if (lastOffset >= 0) {\n                    result = lastOffset;\n                } else if (-1 == lastOffset) {\n                    if (mq.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { // First start, no offset\n                        result = 0L;\n                    } else {\n                        try {\n                            result = this.mQClientFactory.getMQAdminImpl().maxOffset(mq);\n                        } catch (MQClientException e) {\n                            log.warn(\"Compute consume offset from last offset exception, mq={}, exception={}\", mq, e);\n                            throw e;\n                        }\n                    }\n                } else {\n                    result = -1;\n                }\n                break;\n            }\n            case CONSUME_FROM_FIRST_OFFSET: {\n                long lastOffset = litePullConsumerImpl.getOffsetStore().readOffset(mq, ReadOffsetType.MEMORY_FIRST_THEN_STORE);\n                if (lastOffset >= 0) {\n                    result = lastOffset;\n                } else if (-1 == lastOffset) {\n                    result = 0L;\n                } else {\n                    result = -1;\n                }\n                break;\n            }\n            case CONSUME_FROM_TIMESTAMP: {\n                long lastOffset = litePullConsumerImpl.getOffsetStore().readOffset(mq, ReadOffsetType.MEMORY_FIRST_THEN_STORE);\n                if (lastOffset >= 0) {\n                    result = lastOffset;\n                } else if (-1 == lastOffset) {\n                    if (mq.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n                        try {\n                            result = this.mQClientFactory.getMQAdminImpl().maxOffset(mq);\n                        } catch (MQClientException e) {\n                            log.warn(\"Compute consume offset from last offset exception, mq={}, exception={}\", mq, e);\n                            throw e;\n                        }\n                    } else {\n                        try {\n                            long timestamp = UtilAll.parseDate(this.litePullConsumerImpl.getDefaultLitePullConsumer().getConsumeTimestamp(),\n                                UtilAll.YYYYMMDDHHMMSS).getTime();\n                            result = this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp);\n                        } catch (MQClientException e) {\n                            log.warn(\"Compute consume offset from last offset exception, mq={}, exception={}\", mq, e);\n                            throw e;\n                        }\n                    }\n                } else {\n                    result = -1;\n                }\n                break;\n            }\n        }\n        return result;\n    }\n\n    @Override\n    public int getConsumeInitMode() {\n        throw new UnsupportedOperationException(\"no initMode for Pull\");\n    }\n\n    @Override\n    public void dispatchPullRequest(final List<PullRequest> pullRequestList, final long delay) {\n    }\n\n    @Override\n    public void dispatchPopPullRequest(List<PopRequest> pullRequestList, long delay) {\n\n    }\n\n    @Override\n    public ProcessQueue createProcessQueue() {\n        return new ProcessQueue();\n    }\n\n    @Override\n    public PopProcessQueue createPopProcessQueue() {\n        return null;\n    }\n\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePullImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport java.util.List;\nimport java.util.Set;\nimport org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;\nimport org.apache.rocketmq.client.consumer.MessageQueueListener;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\n\npublic class RebalancePullImpl extends RebalanceImpl {\n    private final DefaultMQPullConsumerImpl defaultMQPullConsumerImpl;\n\n    public RebalancePullImpl(DefaultMQPullConsumerImpl defaultMQPullConsumerImpl) {\n        this(null, null, null, null, defaultMQPullConsumerImpl);\n    }\n\n    public RebalancePullImpl(String consumerGroup, MessageModel messageModel,\n        AllocateMessageQueueStrategy allocateMessageQueueStrategy,\n        MQClientInstance mQClientFactory, DefaultMQPullConsumerImpl defaultMQPullConsumerImpl) {\n        super(consumerGroup, messageModel, allocateMessageQueueStrategy, mQClientFactory);\n        this.defaultMQPullConsumerImpl = defaultMQPullConsumerImpl;\n    }\n\n    @Override\n    public void messageQueueChanged(String topic, Set<MessageQueue> mqAll, Set<MessageQueue> mqDivided) {\n        MessageQueueListener messageQueueListener = this.defaultMQPullConsumerImpl.getDefaultMQPullConsumer().getMessageQueueListener();\n        if (messageQueueListener != null) {\n            try {\n                messageQueueListener.messageQueueChanged(topic, mqAll, mqDivided);\n            } catch (Throwable e) {\n                log.error(\"messageQueueChanged exception\", e);\n            }\n        }\n    }\n\n    @Override\n    public boolean removeUnnecessaryMessageQueue(MessageQueue mq, ProcessQueue pq) {\n        this.defaultMQPullConsumerImpl.getOffsetStore().persist(mq);\n        this.defaultMQPullConsumerImpl.getOffsetStore().removeOffset(mq);\n        return true;\n    }\n\n    @Override\n    public ConsumeType consumeType() {\n        return ConsumeType.CONSUME_ACTIVELY;\n    }\n\n    @Override\n    public void removeDirtyOffset(final MessageQueue mq) {\n        this.defaultMQPullConsumerImpl.getOffsetStore().removeOffset(mq);\n    }\n\n    @Deprecated\n    @Override\n    public long computePullFromWhere(MessageQueue mq) {\n        return 0;\n    }\n\n    @Override\n    public long computePullFromWhereWithException(MessageQueue mq) throws MQClientException {\n        return 0;\n    }\n\n    @Override\n    public int getConsumeInitMode() {\n        throw new UnsupportedOperationException(\"no initMode for Pull\");\n    }\n\n    @Override\n    public void dispatchPullRequest(final List<PullRequest> pullRequestList, final long delay) {\n    }\n\n    @Override\n    public void dispatchPopPullRequest(final List<PopRequest> pullRequestList, final long delay) {\n    }\n\n    @Override\n    public ProcessQueue createProcessQueue() {\n        return new ProcessQueue();\n    }\n\n    @Override\n    public PopProcessQueue createPopProcessQueue() {\n        return null;\n    }\n\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;\nimport org.apache.rocketmq.client.consumer.MessageQueueListener;\nimport org.apache.rocketmq.client.consumer.store.OffsetStore;\nimport org.apache.rocketmq.client.consumer.store.ReadOffsetType;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.ConsumeInitMode;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\npublic class RebalancePushImpl extends RebalanceImpl {\n    private final static long UNLOCK_DELAY_TIME_MILLS = Long.parseLong(System.getProperty(\"rocketmq.client.unlockDelayTimeMills\", \"20000\"));\n    private final DefaultMQPushConsumerImpl defaultMQPushConsumerImpl;\n\n\n    public RebalancePushImpl(DefaultMQPushConsumerImpl defaultMQPushConsumerImpl) {\n        this(null, null, null, null, defaultMQPushConsumerImpl);\n    }\n\n    public RebalancePushImpl(String consumerGroup, MessageModel messageModel,\n        AllocateMessageQueueStrategy allocateMessageQueueStrategy,\n        MQClientInstance mQClientFactory, DefaultMQPushConsumerImpl defaultMQPushConsumerImpl) {\n        super(consumerGroup, messageModel, allocateMessageQueueStrategy, mQClientFactory);\n        this.defaultMQPushConsumerImpl = defaultMQPushConsumerImpl;\n    }\n\n    @Override\n    public void messageQueueChanged(String topic, Set<MessageQueue> mqAll, Set<MessageQueue> mqDivided) {\n        /*\n         * When rebalance result changed, should update subscription's version to notify broker.\n         * Fix: inconsistency subscription may lead to consumer miss messages.\n         */\n        SubscriptionData subscriptionData = this.subscriptionInner.get(topic);\n        long newVersion = System.currentTimeMillis();\n        log.info(\"{} Rebalance changed, also update version: {}, {}\", topic, subscriptionData.getSubVersion(), newVersion);\n        subscriptionData.setSubVersion(newVersion);\n\n        int currentQueueCount = this.processQueueTable.size();\n        if (currentQueueCount != 0) {\n            int pullThresholdForTopic = this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer().getPullThresholdForTopic();\n            if (pullThresholdForTopic != -1) {\n                int newVal = Math.max(1, pullThresholdForTopic / currentQueueCount);\n                log.info(\"The pullThresholdForQueue is changed from {} to {}\",\n                    this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer().getPullThresholdForQueue(), newVal);\n                this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer().setPullThresholdForQueue(newVal);\n            }\n\n            int pullThresholdSizeForTopic = this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer().getPullThresholdSizeForTopic();\n            if (pullThresholdSizeForTopic != -1) {\n                int newVal = Math.max(1, pullThresholdSizeForTopic / currentQueueCount);\n                log.info(\"The pullThresholdSizeForQueue is changed from {} to {}\",\n                    this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer().getPullThresholdSizeForQueue(), newVal);\n                this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer().setPullThresholdSizeForQueue(newVal);\n            }\n        }\n\n        // notify broker\n        this.getmQClientFactory().sendHeartbeatToAllBrokerWithLockV2(true);\n\n        MessageQueueListener messageQueueListener = defaultMQPushConsumerImpl.getMessageQueueListener();\n        if (null != messageQueueListener) {\n            messageQueueListener.messageQueueChanged(topic, mqAll, mqDivided);\n        }\n    }\n\n    @Override\n    public boolean removeUnnecessaryMessageQueue(final MessageQueue mq, final ProcessQueue pq) {\n        if (this.defaultMQPushConsumerImpl.isConsumeOrderly()\n            && MessageModel.CLUSTERING.equals(this.defaultMQPushConsumerImpl.messageModel())) {\n\n            // commit offset immediately\n            this.defaultMQPushConsumerImpl.getOffsetStore().persist(mq);\n\n            // remove order message queue: unlock & remove\n            return tryRemoveOrderMessageQueue(mq, pq);\n        } else {\n            this.defaultMQPushConsumerImpl.getOffsetStore().persist(mq);\n            this.defaultMQPushConsumerImpl.getOffsetStore().removeOffset(mq);\n            return true;\n        }\n    }\n\n    private boolean tryRemoveOrderMessageQueue(final MessageQueue mq, final ProcessQueue pq) {\n        try {\n            // unlock & remove when no message is consuming or UNLOCK_DELAY_TIME_MILLS timeout (Backwards compatibility)\n            boolean forceUnlock = pq.isDropped() && System.currentTimeMillis() > pq.getLastLockTimestamp() + UNLOCK_DELAY_TIME_MILLS;\n            if (forceUnlock || pq.getConsumeLock().writeLock().tryLock(500, TimeUnit.MILLISECONDS)) {\n                try {\n                    RebalancePushImpl.this.defaultMQPushConsumerImpl.getOffsetStore().persist(mq);\n                    RebalancePushImpl.this.defaultMQPushConsumerImpl.getOffsetStore().removeOffset(mq);\n\n                    pq.setLocked(false);\n                    RebalancePushImpl.this.unlock(mq, true);\n                    return true;\n                } finally {\n                    if (!forceUnlock) {\n                        pq.getConsumeLock().writeLock().unlock();\n                    }\n                }\n            } else {\n                pq.incTryUnlockTimes();\n            }\n        } catch (Exception e) {\n            pq.incTryUnlockTimes();\n        }\n\n        return false;\n    }\n\n    @Override\n    public boolean clientRebalance(String topic) {\n        // POPTODO order pop consume not implement yet\n        return defaultMQPushConsumerImpl.getDefaultMQPushConsumer().isClientRebalance() || defaultMQPushConsumerImpl.isConsumeOrderly() || MessageModel.BROADCASTING.equals(messageModel);\n    }\n\n    @Override\n    public ConsumeType consumeType() {\n        return ConsumeType.CONSUME_PASSIVELY;\n    }\n\n    @Override\n    public void removeDirtyOffset(final MessageQueue mq) {\n        this.defaultMQPushConsumerImpl.getOffsetStore().removeOffset(mq);\n    }\n\n    @Deprecated\n    @Override\n    public long computePullFromWhere(MessageQueue mq) {\n        long result = -1L;\n        try {\n            result = computePullFromWhereWithException(mq);\n        } catch (MQClientException e) {\n            log.warn(\"Compute consume offset exception, mq={}\", mq);\n        }\n        return result;\n    }\n\n    @Override\n    public long computePullFromWhereWithException(MessageQueue mq) throws MQClientException {\n        long result = -1;\n        final ConsumeFromWhere consumeFromWhere = this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer().getConsumeFromWhere();\n        final OffsetStore offsetStore = this.defaultMQPushConsumerImpl.getOffsetStore();\n        switch (consumeFromWhere) {\n            case CONSUME_FROM_LAST_OFFSET_AND_FROM_MIN_WHEN_BOOT_FIRST:\n            case CONSUME_FROM_MIN_OFFSET:\n            case CONSUME_FROM_MAX_OFFSET:\n            case CONSUME_FROM_LAST_OFFSET: {\n                long lastOffset = offsetStore.readOffset(mq, ReadOffsetType.READ_FROM_STORE);\n                if (lastOffset >= 0) {\n                    result = lastOffset;\n                }\n                // First start,no offset\n                else if (-1 == lastOffset) {\n                    if (mq.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n                        result = 0L;\n                    } else {\n                        try {\n                            result = this.mQClientFactory.getMQAdminImpl().maxOffset(mq);\n                        } catch (MQClientException e) {\n                            log.warn(\"Compute consume offset from last offset exception, mq={}, exception={}\", mq, e);\n                            throw e;\n                        }\n                    }\n                } else {\n                    throw new MQClientException(ResponseCode.QUERY_NOT_FOUND, \"Failed to query consume offset from \" +\n                            \"offset store\");\n                }\n                break;\n            }\n            case CONSUME_FROM_FIRST_OFFSET: {\n                long lastOffset = offsetStore.readOffset(mq, ReadOffsetType.READ_FROM_STORE);\n                if (lastOffset >= 0) {\n                    result = lastOffset;\n                } else if (-1 == lastOffset) {\n                    //the offset will be fixed by the OFFSET_ILLEGAL process\n                    result = 0L;\n                } else {\n                    throw new MQClientException(ResponseCode.QUERY_NOT_FOUND, \"Failed to query offset from offset \" +\n                            \"store\");\n                }\n                break;\n            }\n            case CONSUME_FROM_TIMESTAMP: {\n                long lastOffset = offsetStore.readOffset(mq, ReadOffsetType.READ_FROM_STORE);\n                if (lastOffset >= 0) {\n                    result = lastOffset;\n                } else if (-1 == lastOffset) {\n                    if (mq.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n                        try {\n                            result = this.mQClientFactory.getMQAdminImpl().maxOffset(mq);\n                        } catch (MQClientException e) {\n                            log.warn(\"Compute consume offset from last offset exception, mq={}, exception={}\", mq, e);\n                            throw e;\n                        }\n                    } else {\n                        try {\n                            long timestamp = UtilAll.parseDate(this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer().getConsumeTimestamp(),\n                                UtilAll.YYYYMMDDHHMMSS).getTime();\n                            result = this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp);\n                        } catch (MQClientException e) {\n                            log.warn(\"Compute consume offset from last offset exception, mq={}, exception={}\", mq, e);\n                            throw e;\n                        }\n                    }\n                } else {\n                    throw new MQClientException(ResponseCode.QUERY_NOT_FOUND, \"Failed to query offset from offset \" +\n                            \"store\");\n                }\n                break;\n            }\n\n            default:\n                break;\n        }\n\n        if (result < 0) {\n            throw new MQClientException(ResponseCode.SYSTEM_ERROR, \"Found unexpected result \" + result);\n        }\n\n        return result;\n    }\n\n    @Override\n    public int getConsumeInitMode() {\n        final ConsumeFromWhere consumeFromWhere = this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer().getConsumeFromWhere();\n        if (ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET == consumeFromWhere) {\n            return ConsumeInitMode.MIN;\n        } else {\n            return ConsumeInitMode.MAX;\n        }\n    }\n\n    @Override\n    public void dispatchPullRequest(final List<PullRequest> pullRequestList, final long delay) {\n        for (PullRequest pullRequest : pullRequestList) {\n            if (delay <= 0) {\n                this.defaultMQPushConsumerImpl.executePullRequestImmediately(pullRequest);\n            } else {\n                this.defaultMQPushConsumerImpl.executePullRequestLater(pullRequest, delay);\n            }\n        }\n    }\n\n    @Override\n    public void dispatchPopPullRequest(final List<PopRequest> pullRequestList, final long delay) {\n        for (PopRequest pullRequest : pullRequestList) {\n            if (delay <= 0) {\n                this.defaultMQPushConsumerImpl.executePopPullRequestImmediately(pullRequest);\n            } else {\n                this.defaultMQPushConsumerImpl.executePopPullRequestLater(pullRequest, delay);\n            }\n        }\n    }\n\n    @Override\n    public ProcessQueue createProcessQueue() {\n        return new ProcessQueue();\n    }\n\n    @Override\n    public PopProcessQueue createPopProcessQueue() {\n        return new PopProcessQueue();\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class RebalanceService extends ServiceThread {\n    private static long waitInterval =\n        Long.parseLong(System.getProperty(\n            \"rocketmq.client.rebalance.waitInterval\", \"20000\"));\n    private static long minInterval =\n        Long.parseLong(System.getProperty(\n            \"rocketmq.client.rebalance.minInterval\", \"1000\"));\n    private final Logger log = LoggerFactory.getLogger(RebalanceService.class);\n    private final MQClientInstance mqClientFactory;\n    private long lastRebalanceTimestamp = System.currentTimeMillis();\n\n    public RebalanceService(MQClientInstance mqClientFactory) {\n        this.mqClientFactory = mqClientFactory;\n    }\n\n    @Override\n    public void run() {\n        log.info(this.getServiceName() + \" service started\");\n\n        long realWaitInterval = waitInterval;\n        while (!this.isStopped()) {\n            this.waitForRunning(realWaitInterval);\n\n            long interval = System.currentTimeMillis() - lastRebalanceTimestamp;\n            if (interval < minInterval) {\n                realWaitInterval = minInterval - interval;\n            } else {\n                boolean balanced = this.mqClientFactory.doRebalance();\n                realWaitInterval = balanced ? waitInterval : minInterval;\n                lastRebalanceTimestamp = System.currentTimeMillis();\n            }\n        }\n\n        log.info(this.getServiceName() + \" service end\");\n    }\n\n    @Override\n    public String getServiceName() {\n        return RebalanceService.class.getSimpleName();\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.factory;\n\nimport com.alibaba.fastjson2.JSON;\nimport io.netty.channel.Channel;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.admin.MQAdminExtInner;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.ClientRemotingProcessor;\nimport org.apache.rocketmq.client.impl.FindBrokerResult;\nimport org.apache.rocketmq.client.impl.MQAdminImpl;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.MQClientManager;\nimport org.apache.rocketmq.client.impl.consumer.DefaultMQPullConsumerImpl;\nimport org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl;\nimport org.apache.rocketmq.client.impl.consumer.MQConsumerInner;\nimport org.apache.rocketmq.client.impl.consumer.ProcessQueue;\nimport org.apache.rocketmq.client.impl.consumer.PullMessageService;\nimport org.apache.rocketmq.client.impl.consumer.RebalanceService;\nimport org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;\nimport org.apache.rocketmq.client.impl.producer.MQProducerInner;\nimport org.apache.rocketmq.client.impl.producer.TopicPublishInfo;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.stat.ConsumerStatsManager;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ServiceState;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.message.MessageQueueAssignment;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.ChannelEventListener;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.common.HeartbeatV2Result;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumerData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ProducerData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Map.Entry;\nimport java.util.Random;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.RejectedExecutionException;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ThreadFactory;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.concurrent.locks.Lock;\nimport java.util.concurrent.locks.ReentrantLock;\n\nimport static org.apache.rocketmq.remoting.rpc.ClientMetadata.topicRouteData2EndpointsForStaticTopic;\n\npublic class MQClientInstance {\n    private final static long LOCK_TIMEOUT_MILLIS = 3000;\n    private final static Logger log = LoggerFactory.getLogger(MQClientInstance.class);\n    private final ClientConfig clientConfig;\n    private final String clientId;\n    private final long bootTimestamp = System.currentTimeMillis();\n\n    /**\n     * The container of the producer in the current client. The key is the name of producerGroup.\n     */\n    private final ConcurrentMap<String, MQProducerInner> producerTable = new ConcurrentHashMap<>();\n\n    /**\n     * The container of the consumer in the current client. The key is the name of consumerGroup.\n     */\n    private final ConcurrentMap<String, MQConsumerInner> consumerTable = new ConcurrentHashMap<>();\n\n    /**\n     * The container of the adminExt in the current client. The key is the name of adminExtGroup.\n     */\n    private final ConcurrentMap<String, MQAdminExtInner> adminExtTable = new ConcurrentHashMap<>();\n    private final NettyClientConfig nettyClientConfig;\n    private final MQClientAPIImpl mQClientAPIImpl;\n    private final MQAdminImpl mQAdminImpl;\n    private final ConcurrentMap<String/* Topic */, TopicRouteData> topicRouteTable = new ConcurrentHashMap<>();\n    private final ConcurrentMap<String/* Topic */, ConcurrentMap<MessageQueue, String/*brokerName*/>> topicEndPointsTable = new ConcurrentHashMap<>();\n    private final Lock lockNamesrv = new ReentrantLock();\n    private final Lock lockHeartbeat = new ReentrantLock();\n\n    /**\n     * The container which stores the brokerClusterInfo. The key of the map is the broker name.\n     * And the value is the broker instance list that belongs to the broker cluster.\n     * For the sub map, the key is the id of single broker instance, and the value is the address.\n     */\n    private final ConcurrentMap<String, HashMap<Long, String>> brokerAddrTable = new ConcurrentHashMap<>();\n\n    private final ConcurrentMap<String/* Broker Name */, ConcurrentHashMap<String/* address */, Integer>> brokerVersionTable = new ConcurrentHashMap<>();\n    private final Set<String/* Broker address */> brokerSupportV2HeartbeatSet = new HashSet<>();\n    private final ConcurrentMap<String, Integer> brokerAddrHeartbeatFingerprintTable = new ConcurrentHashMap<>();\n    private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(r -> new Thread(r, \"MQClientFactoryScheduledThread\"));\n    private final ScheduledExecutorService fetchRemoteConfigExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {\n        @Override\n        public Thread newThread(Runnable r) {\n            return new Thread(r, \"MQClientFactoryFetchRemoteConfigScheduledThread\");\n        }\n    });\n    private final PullMessageService pullMessageService;\n    private final RebalanceService rebalanceService;\n    private final DefaultMQProducer defaultMQProducer;\n    private final ConsumerStatsManager consumerStatsManager;\n    private final AtomicLong sendHeartbeatTimesTotal = new AtomicLong(0);\n    private ServiceState serviceState = ServiceState.CREATE_JUST;\n    private final Random random = new Random();\n    private ExecutorService concurrentHeartbeatExecutor;\n\n    public MQClientInstance(ClientConfig clientConfig, int instanceIndex, String clientId) {\n        this(clientConfig, instanceIndex, clientId, null);\n    }\n\n    public MQClientInstance(ClientConfig clientConfig, int instanceIndex, String clientId, RPCHook rpcHook) {\n        this.clientConfig = clientConfig;\n        this.nettyClientConfig = new NettyClientConfig();\n        this.nettyClientConfig.setClientCallbackExecutorThreads(clientConfig.getClientCallbackExecutorThreads());\n        this.nettyClientConfig.setUseTLS(clientConfig.isUseTLS());\n        this.nettyClientConfig.setSocksProxyConfig(clientConfig.getSocksProxyConfig());\n        this.nettyClientConfig.setScanAvailableNameSrv(false);\n        ClientRemotingProcessor clientRemotingProcessor = new ClientRemotingProcessor(this);\n        ChannelEventListener channelEventListener;\n        if (clientConfig.isEnableHeartbeatChannelEventListener()) {\n            channelEventListener = new ChannelEventListener() {\n                \n                private final ConcurrentMap<String, HashMap<Long, String>> brokerAddrTable = MQClientInstance.this.brokerAddrTable;\n                \n                @Override\n                public void onChannelConnect(String remoteAddr, Channel channel) {\n                }\n\n                @Override\n                public void onChannelClose(String remoteAddr, Channel channel) {\n                }\n\n                @Override\n                public void onChannelException(String remoteAddr, Channel channel) {\n                }\n\n                @Override\n                public void onChannelIdle(String remoteAddr, Channel channel) {\n                }\n\n                @Override\n                public void onChannelActive(String remoteAddr, Channel channel) {\n                    for (Map.Entry<String, HashMap<Long, String>> addressEntry : brokerAddrTable.entrySet()) {\n                        for (Map.Entry<Long, String> entry : addressEntry.getValue().entrySet()) {\n                            String addr = entry.getValue();\n                            if (addr.equals(remoteAddr)) {\n                                long id = entry.getKey();\n                                String brokerName = addressEntry.getKey();\n                                if (sendHeartbeatToBroker(id, brokerName, addr, false)) {\n                                    rebalanceImmediately();\n                                }\n                                break;\n                            }\n                        }\n                    }\n                }\n            };\n        } else {\n            channelEventListener = null;\n        }\n        this.mQClientAPIImpl = new MQClientAPIImpl(this.nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig, channelEventListener);\n\n        if (this.clientConfig.getNamesrvAddr() != null) {\n            this.mQClientAPIImpl.updateNameServerAddressList(this.clientConfig.getNamesrvAddr());\n            log.info(\"user specified name server address: {}\", this.clientConfig.getNamesrvAddr());\n        }\n\n        this.clientId = clientId;\n\n        this.mQAdminImpl = new MQAdminImpl(this);\n\n        this.pullMessageService = new PullMessageService(this);\n\n        this.rebalanceService = new RebalanceService(this);\n\n        this.defaultMQProducer = new DefaultMQProducer(MixAll.CLIENT_INNER_PRODUCER_GROUP);\n        this.defaultMQProducer.resetClientConfig(clientConfig);\n\n        this.consumerStatsManager = new ConsumerStatsManager(this.scheduledExecutorService);\n\n        if (this.clientConfig.isEnableConcurrentHeartbeat()) {\n            this.concurrentHeartbeatExecutor = Executors.newFixedThreadPool(\n                clientConfig.getConcurrentHeartbeatThreadPoolSize(),\n                new ThreadFactoryImpl(\"MQClientConcurrentHeartbeatThread_\", true));\n        }\n\n        log.info(\"Created a new client Instance, InstanceIndex:{}, ClientID:{}, ClientConfig:{}, ClientVersion:{}, SerializerType:{}\",\n            instanceIndex,\n            this.clientId,\n            this.clientConfig,\n            MQVersion.getVersionDesc(MQVersion.CURRENT_VERSION), RemotingCommand.getSerializeTypeConfigInThisServer());\n    }\n\n    public static TopicPublishInfo topicRouteData2TopicPublishInfo(final String topic, final TopicRouteData route) {\n        TopicPublishInfo info = new TopicPublishInfo();\n        // TO DO should check the usage of raw route, it is better to remove such field\n        info.setTopicRouteData(route);\n        if (route.getOrderTopicConf() != null && route.getOrderTopicConf().length() > 0) {\n            String[] brokers = route.getOrderTopicConf().split(\";\");\n            for (String broker : brokers) {\n                String[] item = broker.split(\":\");\n                int nums = Integer.parseInt(item[1]);\n                for (int i = 0; i < nums; i++) {\n                    MessageQueue mq = new MessageQueue(topic, item[0], i);\n                    info.getMessageQueueList().add(mq);\n                }\n            }\n\n            info.setOrderTopic(true);\n        } else if (route.getOrderTopicConf() == null\n            && route.getTopicQueueMappingByBroker() != null\n            && !route.getTopicQueueMappingByBroker().isEmpty()) {\n            info.setOrderTopic(false);\n            ConcurrentMap<MessageQueue, String> mqEndPoints = topicRouteData2EndpointsForStaticTopic(topic, route);\n            info.getMessageQueueList().addAll(mqEndPoints.keySet());\n            info.getMessageQueueList().sort((mq1, mq2) -> MixAll.compareInteger(mq1.getQueueId(), mq2.getQueueId()));\n        } else {\n            List<QueueData> qds = route.getQueueDatas();\n            Collections.sort(qds);\n            for (QueueData qd : qds) {\n                if (PermName.isWriteable(qd.getPerm())) {\n                    BrokerData brokerData = null;\n                    for (BrokerData bd : route.getBrokerDatas()) {\n                        if (bd.getBrokerName().equals(qd.getBrokerName())) {\n                            brokerData = bd;\n                            break;\n                        }\n                    }\n\n                    if (null == brokerData) {\n                        continue;\n                    }\n\n                    if (!brokerData.getBrokerAddrs().containsKey(MixAll.MASTER_ID)) {\n                        continue;\n                    }\n\n                    for (int i = 0; i < qd.getWriteQueueNums(); i++) {\n                        MessageQueue mq = new MessageQueue(topic, qd.getBrokerName(), i);\n                        info.getMessageQueueList().add(mq);\n                    }\n                }\n            }\n\n            info.setOrderTopic(false);\n        }\n\n        return info;\n    }\n\n    public static Set<MessageQueue> topicRouteData2TopicSubscribeInfo(final String topic, final TopicRouteData route) {\n        Set<MessageQueue> mqList = new HashSet<>();\n        if (route.getTopicQueueMappingByBroker() != null\n            && !route.getTopicQueueMappingByBroker().isEmpty()) {\n            ConcurrentMap<MessageQueue, String> mqEndPoints = topicRouteData2EndpointsForStaticTopic(topic, route);\n            return mqEndPoints.keySet();\n        }\n        List<QueueData> qds = route.getQueueDatas();\n        for (QueueData qd : qds) {\n            if (PermName.isReadable(qd.getPerm())) {\n                for (int i = 0; i < qd.getReadQueueNums(); i++) {\n                    MessageQueue mq = new MessageQueue(topic, qd.getBrokerName(), i);\n                    mqList.add(mq);\n                }\n            }\n        }\n\n        return mqList;\n    }\n\n    public void start() throws MQClientException {\n\n        synchronized (this) {\n            switch (this.serviceState) {\n                case CREATE_JUST:\n                    this.serviceState = ServiceState.START_FAILED;\n                    // If not specified,looking address from name server\n                    if (null == this.clientConfig.getNamesrvAddr()) {\n                        this.mQClientAPIImpl.fetchNameServerAddr();\n                    }\n                    // Start request-response channel\n                    this.mQClientAPIImpl.start();\n                    // Start various schedule tasks\n                    this.startScheduledTask();\n                    // Start pull service\n                    this.pullMessageService.start();\n                    // Start rebalance service\n                    this.rebalanceService.start();\n                    // Start push service\n                    this.defaultMQProducer.getDefaultMQProducerImpl().start(false);\n                    log.info(\"the client factory [{}] start OK\", this.clientId);\n                    this.serviceState = ServiceState.RUNNING;\n                    break;\n                case START_FAILED:\n                    throw new MQClientException(\"The Factory object[\" + this.getClientId() + \"] has been created before, and failed.\", null);\n                default:\n                    break;\n            }\n        }\n    }\n\n    private void startScheduledTask() {\n        if (null == this.clientConfig.getNamesrvAddr()) {\n            this.scheduledExecutorService.scheduleAtFixedRate(() -> {\n                try {\n                    MQClientInstance.this.mQClientAPIImpl.fetchNameServerAddr();\n                } catch (Throwable t) {\n                    log.error(\"ScheduledTask fetchNameServerAddr exception\", t);\n                }\n            }, 1000 * 10, 1000 * 60 * 2, TimeUnit.MILLISECONDS);\n        }\n\n        this.scheduledExecutorService.scheduleAtFixedRate(() -> {\n            try {\n                MQClientInstance.this.updateTopicRouteInfoFromNameServer();\n            } catch (Throwable t) {\n                log.error(\"ScheduledTask updateTopicRouteInfoFromNameServer exception\", t);\n            }\n        }, 10, this.clientConfig.getPollNameServerInterval(), TimeUnit.MILLISECONDS);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(() -> {\n            try {\n                MQClientInstance.this.cleanOfflineBroker();\n                MQClientInstance.this.sendHeartbeatToAllBrokerWithLock();\n            } catch (Throwable t) {\n                log.error(\"ScheduledTask sendHeartbeatToAllBroker exception\", t);\n            }\n        }, 1000, this.clientConfig.getHeartbeatBrokerInterval(), TimeUnit.MILLISECONDS);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(() -> {\n            try {\n                MQClientInstance.this.persistAllConsumerOffset();\n            } catch (Throwable t) {\n                log.error(\"ScheduledTask persistAllConsumerOffset exception\", t);\n            }\n        }, 1000 * 10, this.clientConfig.getPersistConsumerOffsetInterval(), TimeUnit.MILLISECONDS);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(() -> {\n            try {\n                MQClientInstance.this.adjustThreadPool();\n            } catch (Throwable t) {\n                log.error(\"ScheduledTask adjustThreadPool exception\", t);\n            }\n        }, 1, 1, TimeUnit.MINUTES);\n    }\n\n    public String getClientId() {\n        return clientId;\n    }\n\n    public void updateTopicRouteInfoFromNameServer() {\n        Set<String> topicList = new HashSet<>();\n\n        // Consumer\n        {\n            for (Entry<String, MQConsumerInner> entry : this.consumerTable.entrySet()) {\n                MQConsumerInner impl = entry.getValue();\n                if (impl != null) {\n                    Set<SubscriptionData> subList = impl.subscriptions();\n                    if (subList != null) {\n                        for (SubscriptionData subData : subList) {\n                            topicList.add(subData.getTopic());\n                        }\n                    }\n                }\n            }\n        }\n\n        // Producer\n        {\n            for (Entry<String, MQProducerInner> entry : this.producerTable.entrySet()) {\n                MQProducerInner impl = entry.getValue();\n                if (impl != null) {\n                    Set<String> lst = impl.getPublishTopicList();\n                    topicList.addAll(lst);\n                }\n            }\n        }\n\n        for (String topic : topicList) {\n            this.updateTopicRouteInfoFromNameServer(topic);\n        }\n    }\n\n    public Map<MessageQueue, Long> parseOffsetTableFromBroker(Map<MessageQueue, Long> offsetTable, String namespace) {\n        HashMap<MessageQueue, Long> newOffsetTable = new HashMap<>(offsetTable.size(), 1);\n        if (StringUtils.isNotEmpty(namespace)) {\n            for (Entry<MessageQueue, Long> entry : offsetTable.entrySet()) {\n                MessageQueue queue = entry.getKey();\n                queue.setTopic(NamespaceUtil.withoutNamespace(queue.getTopic(), namespace));\n                newOffsetTable.put(queue, entry.getValue());\n            }\n        } else {\n            newOffsetTable.putAll(offsetTable);\n        }\n\n        return newOffsetTable;\n    }\n\n    /**\n     * Remove offline broker\n     */\n    private void cleanOfflineBroker() {\n        try {\n            if (this.lockNamesrv.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS))\n                try {\n                    ConcurrentHashMap<String, HashMap<Long, String>> updatedTable = new ConcurrentHashMap<>(this.brokerAddrTable.size(), 1);\n\n                    Iterator<Entry<String, HashMap<Long, String>>> itBrokerTable = this.brokerAddrTable.entrySet().iterator();\n                    while (itBrokerTable.hasNext()) {\n                        Entry<String, HashMap<Long, String>> entry = itBrokerTable.next();\n                        String brokerName = entry.getKey();\n                        HashMap<Long, String> oneTable = entry.getValue();\n\n                        HashMap<Long, String> cloneAddrTable = new HashMap<>(oneTable.size(), 1);\n                        cloneAddrTable.putAll(oneTable);\n\n                        Iterator<Entry<Long, String>> it = cloneAddrTable.entrySet().iterator();\n                        while (it.hasNext()) {\n                            Entry<Long, String> ee = it.next();\n                            String addr = ee.getValue();\n                            if (!this.isBrokerAddrExistInTopicRouteTable(addr)) {\n                                it.remove();\n                                log.info(\"the broker addr[{} {}] is offline, remove it\", brokerName, addr);\n                            }\n                        }\n\n                        if (cloneAddrTable.isEmpty()) {\n                            itBrokerTable.remove();\n                            log.info(\"the broker[{}] name's host is offline, remove it\", brokerName);\n                        } else {\n                            updatedTable.put(brokerName, cloneAddrTable);\n                        }\n                    }\n\n                    if (!updatedTable.isEmpty()) {\n                        this.brokerAddrTable.putAll(updatedTable);\n                    }\n                } finally {\n                    this.lockNamesrv.unlock();\n                }\n        } catch (InterruptedException e) {\n            log.warn(\"cleanOfflineBroker Exception\", e);\n        }\n    }\n\n    public void checkClientInBroker() throws MQClientException {\n\n        for (Entry<String, MQConsumerInner> entry : this.consumerTable.entrySet()) {\n            Set<SubscriptionData> subscriptionInner = entry.getValue().subscriptions();\n            if (subscriptionInner == null || subscriptionInner.isEmpty()) {\n                return;\n            }\n\n            for (SubscriptionData subscriptionData : subscriptionInner) {\n                if (ExpressionType.isTagType(subscriptionData.getExpressionType())) {\n                    continue;\n                }\n                // may need to check one broker every cluster...\n                // assume that the configs of every broker in cluster are the same.\n                String addr = findBrokerAddrByTopic(subscriptionData.getTopic());\n\n                if (addr != null) {\n                    try {\n                        this.getMQClientAPIImpl().checkClientInBroker(\n                            addr, entry.getKey(), this.clientId, subscriptionData, clientConfig.getMqClientApiTimeout()\n                        );\n                    } catch (Exception e) {\n                        if (e instanceof MQClientException) {\n                            throw (MQClientException) e;\n                        } else {\n                            throw new MQClientException(\"Check client in broker error, maybe because you use \"\n                                + subscriptionData.getExpressionType() + \" to filter message, but server has not been upgraded to support!\"\n                                + \"This error would not affect the launch of consumer, but may has impact on message receiving if you \" +\n                                \"have use the new features which are not supported by server, please check the log!\", e);\n                        }\n                    }\n                }\n            }\n        }\n    }\n\n    public boolean sendHeartbeatToAllBrokerWithLockV2(boolean isRebalance) {\n        if (this.lockHeartbeat.tryLock()) {\n            try {\n                if (clientConfig.isUseHeartbeatV2()) {\n                    return this.sendHeartbeatToAllBrokerV2(isRebalance);\n                } else {\n                    return this.sendHeartbeatToAllBroker();\n                }\n            } catch (final Exception e) {\n                log.error(\"sendHeartbeatToAllBrokerWithLockV2 exception\", e);\n            } finally {\n                this.lockHeartbeat.unlock();\n            }\n        } else {\n            log.warn(\"sendHeartbeatToAllBrokerWithLockV2 lock heartBeat, but failed.\");\n        }\n        return false;\n    }\n\n    public boolean sendHeartbeatToAllBrokerWithLock() {\n        if (this.lockHeartbeat.tryLock()) {\n            try {\n                if (clientConfig.isUseHeartbeatV2()) {\n                    return this.sendHeartbeatToAllBrokerV2(false);\n                } else if (clientConfig.isEnableConcurrentHeartbeat()) {\n                    return this.sendHeartbeatToAllBrokerConcurrently();\n                } else {\n                    return this.sendHeartbeatToAllBroker();\n                }\n            } catch (final Exception e) {\n                log.error(\"sendHeartbeatToAllBroker exception\", e);\n            } finally {\n                this.lockHeartbeat.unlock();\n            }\n        } else {\n            log.warn(\"lock heartBeat, but failed. [{}]\", this.clientId);\n        }\n        return false;\n    }\n\n    private void persistAllConsumerOffset() {\n        for (Entry<String, MQConsumerInner> entry : this.consumerTable.entrySet()) {\n            MQConsumerInner impl = entry.getValue();\n            impl.persistConsumerOffset();\n        }\n    }\n\n    public void adjustThreadPool() {\n        for (Entry<String, MQConsumerInner> entry : this.consumerTable.entrySet()) {\n            MQConsumerInner impl = entry.getValue();\n            if (impl != null) {\n                try {\n                    if (impl instanceof DefaultMQPushConsumerImpl) {\n                        DefaultMQPushConsumerImpl dmq = (DefaultMQPushConsumerImpl) impl;\n                        dmq.adjustThreadPool();\n                    }\n                } catch (Exception ignored) {\n                }\n            }\n        }\n    }\n\n    public boolean updateTopicRouteInfoFromNameServer(final String topic) {\n        return updateTopicRouteInfoFromNameServer(topic, false, null);\n    }\n\n    private boolean isBrokerAddrExistInTopicRouteTable(final String addr) {\n        for (Entry<String, TopicRouteData> entry : this.topicRouteTable.entrySet()) {\n            TopicRouteData topicRouteData = entry.getValue();\n            List<BrokerData> bds = topicRouteData.getBrokerDatas();\n            for (BrokerData bd : bds) {\n                if (bd.getBrokerAddrs() != null) {\n                    boolean exist = bd.getBrokerAddrs().containsValue(addr);\n                    if (exist)\n                        return true;\n                }\n            }\n        }\n\n        return false;\n    }\n\n    public boolean sendHeartbeatToBroker(long id, String brokerName, String addr) {\n        return sendHeartbeatToBroker(id, brokerName, addr, true);\n    }\n\n    /**\n     * @param id\n     * @param brokerName\n     * @param addr\n     * @param strictLockMode When the connection is initially established, sending a heartbeat will simultaneously trigger the onChannelActive event to acquire the lock again, causing an exception. Therefore,\n     *                       the exception that occurs when sending the heartbeat during the initial onChannelActive event can be ignored.\n     * @return\n     */\n    public boolean sendHeartbeatToBroker(long id, String brokerName, String addr, boolean strictLockMode) {\n        if (this.lockHeartbeat.tryLock()) {\n            try {\n                final HeartbeatData heartbeatDataWithSub = this.prepareHeartbeatData(false);\n                final boolean producerEmpty = heartbeatDataWithSub.getProducerDataSet().isEmpty();\n                final boolean consumerEmpty = heartbeatDataWithSub.getConsumerDataSet().isEmpty();\n                if (producerEmpty && consumerEmpty) {\n                    log.warn(\"sendHeartbeatToBroker sending heartbeat, but no consumer and no producer. [{}]\", this.clientId);\n                    return false;\n                }\n                if (clientConfig.isUseHeartbeatV2()) {\n                    int currentHeartbeatFingerprint = heartbeatDataWithSub.computeHeartbeatFingerprint();\n                    heartbeatDataWithSub.setHeartbeatFingerprint(currentHeartbeatFingerprint);\n                    HeartbeatData heartbeatDataWithoutSub = this.prepareHeartbeatData(true);\n                    heartbeatDataWithoutSub.setHeartbeatFingerprint(currentHeartbeatFingerprint);\n                    return this.sendHeartbeatToBrokerV2(id, brokerName, addr, heartbeatDataWithSub, heartbeatDataWithoutSub, currentHeartbeatFingerprint);\n                } else {\n                    return this.sendHeartbeatToBroker(id, brokerName, addr, heartbeatDataWithSub);\n                }\n            } catch (final Exception e) {\n                log.error(\"sendHeartbeatToAllBroker exception\", e);\n            } finally {\n                this.lockHeartbeat.unlock();\n            }\n        } else {\n            if (strictLockMode) {\n                log.warn(\"lock heartBeat, but failed. [{}]\", this.clientId);\n            }\n        }\n        return false;\n    }\n\n    private boolean sendHeartbeatToBroker(long id, String brokerName, String addr, HeartbeatData heartbeatData) {\n        try {\n            int version = this.mQClientAPIImpl.sendHeartbeat(addr, heartbeatData, clientConfig.getMqClientApiTimeout());\n            if (!this.brokerVersionTable.containsKey(brokerName)) {\n                this.brokerVersionTable.put(brokerName, new ConcurrentHashMap<>(4));\n            }\n            this.brokerVersionTable.get(brokerName).put(addr, version);\n            long times = this.sendHeartbeatTimesTotal.getAndIncrement();\n            if (times % 20 == 0) {\n                log.info(\"send heart beat to broker[{} {} {}] success\", brokerName, id, addr);\n                log.info(heartbeatData.toString());\n            }\n            return true;\n        } catch (Exception e) {\n            if (this.isBrokerInNameServer(addr)) {\n                log.warn(\"send heart beat to broker[{} {} {}] failed\", brokerName, id, addr, e);\n            } else {\n                log.warn(\"send heart beat to broker[{} {} {}] exception, because the broker not up, forget it\", brokerName,\n                    id, addr, e);\n            }\n        }\n        return false;\n    }\n\n    private boolean sendHeartbeatToAllBroker() {\n        final HeartbeatData heartbeatData = this.prepareHeartbeatData(false);\n        final boolean producerEmpty = heartbeatData.getProducerDataSet().isEmpty();\n        final boolean consumerEmpty = heartbeatData.getConsumerDataSet().isEmpty();\n        if (producerEmpty && consumerEmpty) {\n            log.warn(\"sending heartbeat, but no consumer and no producer. [{}]\", this.clientId);\n            return false;\n        }\n\n        if (this.brokerAddrTable.isEmpty()) {\n            return false;\n        }\n        for (Entry<String, HashMap<Long, String>> brokerClusterInfo : this.brokerAddrTable.entrySet()) {\n            String brokerName = brokerClusterInfo.getKey();\n            HashMap<Long, String> oneTable = brokerClusterInfo.getValue();\n            if (oneTable == null) {\n                continue;\n            }\n            for (Entry<Long, String> singleBrokerInstance : oneTable.entrySet()) {\n                Long id = singleBrokerInstance.getKey();\n                String addr = singleBrokerInstance.getValue();\n                if (addr == null) {\n                    continue;\n                }\n                if (consumerEmpty && MixAll.MASTER_ID != id) {\n                    continue;\n                }\n\n                sendHeartbeatToBroker(id, brokerName, addr, heartbeatData);\n            }\n        }\n        return true;\n    }\n\n    private boolean sendHeartbeatToBrokerV2(long id, String brokerName, String addr, HeartbeatData heartbeatDataWithSub,\n        HeartbeatData heartbeatDataWithoutSub, int currentHeartbeatFingerprint) {\n        try {\n            int version = 0;\n            boolean isBrokerSupportV2 = brokerSupportV2HeartbeatSet.contains(addr);\n            HeartbeatV2Result heartbeatV2Result = null;\n            if (isBrokerSupportV2 && null != brokerAddrHeartbeatFingerprintTable.get(addr) && brokerAddrHeartbeatFingerprintTable.get(addr) == currentHeartbeatFingerprint) {\n                heartbeatV2Result = this.mQClientAPIImpl.sendHeartbeatV2(addr, heartbeatDataWithoutSub, clientConfig.getMqClientApiTimeout());\n                if (heartbeatV2Result.isSubChange()) {\n                    brokerAddrHeartbeatFingerprintTable.remove(addr);\n                }\n                log.info(\"sendHeartbeatToAllBrokerV2 simple brokerName: {} subChange: {} brokerAddrHeartbeatFingerprintTable: {}\", brokerName, heartbeatV2Result.isSubChange(), JSON.toJSONString(brokerAddrHeartbeatFingerprintTable));\n            } else {\n                heartbeatV2Result = this.mQClientAPIImpl.sendHeartbeatV2(addr, heartbeatDataWithSub, clientConfig.getMqClientApiTimeout());\n                if (heartbeatV2Result.isSupportV2()) {\n                    brokerSupportV2HeartbeatSet.add(addr);\n                    if (heartbeatV2Result.isSubChange()) {\n                        brokerAddrHeartbeatFingerprintTable.remove(addr);\n                    } else if (!brokerAddrHeartbeatFingerprintTable.containsKey(addr) || brokerAddrHeartbeatFingerprintTable.get(addr) != currentHeartbeatFingerprint) {\n                        brokerAddrHeartbeatFingerprintTable.put(addr, currentHeartbeatFingerprint);\n                    }\n                }\n                log.info(\"sendHeartbeatToAllBrokerV2 normal brokerName: {} subChange: {} brokerAddrHeartbeatFingerprintTable: {}\", brokerName, heartbeatV2Result.isSubChange(), JSON.toJSONString(brokerAddrHeartbeatFingerprintTable));\n            }\n            version = heartbeatV2Result.getVersion();\n            if (!this.brokerVersionTable.containsKey(brokerName)) {\n                this.brokerVersionTable.put(brokerName, new ConcurrentHashMap<>(4));\n            }\n            this.brokerVersionTable.get(brokerName).put(addr, version);\n            long times = this.sendHeartbeatTimesTotal.getAndIncrement();\n            if (times % 20 == 0) {\n                log.info(\"send heart beat to broker[{} {} {}] success\", brokerName, id, addr);\n                log.info(heartbeatDataWithSub.toString());\n            }\n            return true;\n        } catch (Exception e) {\n            if (this.isBrokerInNameServer(addr)) {\n                log.warn(\"sendHeartbeatToAllBrokerV2 send heart beat to broker[{} {} {}] failed\", brokerName, id, addr, e);\n            } else {\n                log.warn(\"sendHeartbeatToAllBrokerV2 send heart beat to broker[{} {} {}] exception, because the broker not up, forget it\", brokerName, id, addr, e);\n            }\n        }\n        return false;\n    }\n\n    private boolean sendHeartbeatToAllBrokerV2(boolean isRebalance) {\n        final HeartbeatData heartbeatDataWithSub = this.prepareHeartbeatData(false);\n        final boolean producerEmpty = heartbeatDataWithSub.getProducerDataSet().isEmpty();\n        final boolean consumerEmpty = heartbeatDataWithSub.getConsumerDataSet().isEmpty();\n        if (producerEmpty && consumerEmpty) {\n            log.warn(\"sendHeartbeatToAllBrokerV2 sending heartbeat, but no consumer and no producer. [{}]\", this.clientId);\n            return false;\n        }\n        if (this.brokerAddrTable.isEmpty()) {\n            return false;\n        }\n        if (isRebalance) {\n            resetBrokerAddrHeartbeatFingerprintMap();\n        }\n        int currentHeartbeatFingerprint = heartbeatDataWithSub.computeHeartbeatFingerprint();\n        heartbeatDataWithSub.setHeartbeatFingerprint(currentHeartbeatFingerprint);\n        HeartbeatData heartbeatDataWithoutSub = this.prepareHeartbeatData(true);\n        heartbeatDataWithoutSub.setHeartbeatFingerprint(currentHeartbeatFingerprint);\n\n        for (Entry<String, HashMap<Long, String>> brokerClusterInfo : this.brokerAddrTable.entrySet()) {\n            String brokerName = brokerClusterInfo.getKey();\n            HashMap<Long, String> oneTable = brokerClusterInfo.getValue();\n            if (oneTable == null) {\n                continue;\n            }\n            for (Entry<Long, String> singleBrokerInstance : oneTable.entrySet()) {\n                Long id = singleBrokerInstance.getKey();\n                String addr = singleBrokerInstance.getValue();\n                if (addr == null) {\n                    continue;\n                }\n                if (consumerEmpty && MixAll.MASTER_ID != id) {\n                    continue;\n                }\n                sendHeartbeatToBrokerV2(id, brokerName, addr, heartbeatDataWithSub, heartbeatDataWithoutSub, currentHeartbeatFingerprint);\n            }\n        }\n        return true;\n    }\n\n    private class ClientHeartBeatTask {\n        private final String brokerName;\n        private final Long brokerId;\n        private final String brokerAddr;\n        private final HeartbeatData heartbeatData;\n\n        public ClientHeartBeatTask(String brokerName, Long brokerId, String brokerAddr, HeartbeatData heartbeatData) {\n            this.brokerName = brokerName;\n            this.brokerId = brokerId;\n            this.brokerAddr = brokerAddr;\n            this.heartbeatData = heartbeatData;\n        }\n\n        public void execute() throws Exception {\n            int version = MQClientInstance.this.mQClientAPIImpl.sendHeartbeat(\n                brokerAddr, heartbeatData, MQClientInstance.this.clientConfig.getMqClientApiTimeout());\n\n            ConcurrentHashMap<String, Integer> inner = MQClientInstance.this.brokerVersionTable\n                .computeIfAbsent(brokerName, k -> new ConcurrentHashMap<>(4));\n            inner.put(brokerAddr, version);\n        }\n    }\n\n    private boolean sendHeartbeatToAllBrokerConcurrently() {\n        final HeartbeatData heartbeatData = this.prepareHeartbeatData(false);\n        final boolean producerEmpty = heartbeatData.getProducerDataSet().isEmpty();\n        final boolean consumerEmpty = heartbeatData.getConsumerDataSet().isEmpty();\n\n        if (producerEmpty && consumerEmpty) {\n            log.warn(\"sending heartbeat, but no consumer and no producer. [{}]\", this.clientId);\n            return false;\n        }\n\n        if (this.brokerAddrTable.isEmpty()) {\n            return false;\n        }\n\n        long times = this.sendHeartbeatTimesTotal.getAndIncrement();\n        List<ClientHeartBeatTask> tasks = new ArrayList<>();\n        for (Entry<String, HashMap<Long, String>> entry : this.brokerAddrTable.entrySet()) {\n            String brokerName = entry.getKey();\n            HashMap<Long, String> oneTable = entry.getValue();\n            if (oneTable != null) {\n                for (Map.Entry<Long, String> entry1 : oneTable.entrySet()) {\n                    Long id = entry1.getKey();\n                    String addr = entry1.getValue();\n                    if (addr == null) continue;\n                    if (consumerEmpty && id != MixAll.MASTER_ID) continue;\n                    tasks.add(new ClientHeartBeatTask(brokerName, id, addr, heartbeatData));\n                }\n            }\n        }\n\n        if (tasks.isEmpty()) {\n            return false;\n        }\n\n        final CountDownLatch latch = new CountDownLatch(tasks.size());\n\n        for (ClientHeartBeatTask task : tasks) {\n            try {\n                this.concurrentHeartbeatExecutor.execute(() -> {\n                    try {\n                        task.execute();\n                        if (times % 20 == 0) {\n                            log.info(\"send heart beat to broker[{} {} {}] success\", task.brokerName, task.brokerId, task.brokerAddr);\n                        }\n                    } catch (Exception e) {\n                        if (MQClientInstance.this.isBrokerInNameServer(task.brokerAddr)) {\n                            log.warn(\"send heart beat to broker[{} {} {}] failed\", task.brokerName, task.brokerId, task.brokerAddr, e);\n                        } else {\n                            log.warn(\"send heart beat to broker[{} {} {}] exception, because the broker not up, forget it\",\n                                task.brokerName, task.brokerId, task.brokerAddr, e);\n                        }\n                    } finally {\n                        latch.countDown();\n                    }\n                });\n            } catch (RejectedExecutionException rex) {\n                log.warn(\"heartbeat submission rejected for broker[{} {} {}], will skip this round\", task.brokerName, task.brokerId, task.brokerAddr, rex);\n                latch.countDown();\n            }\n        }\n\n        try {\n            // wait all tasks finish\n            latch.await();\n        } catch (InterruptedException ie) {\n            log.warn(\"Interrupted while waiting for broker heartbeat tasks to complete\", ie);\n            Thread.currentThread().interrupt();\n        }\n        return true;\n    }\n\n    public boolean updateTopicRouteInfoFromNameServer(final String topic, boolean isDefault,\n        DefaultMQProducer defaultMQProducer) {\n        try {\n            if (this.lockNamesrv.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {\n                try {\n                    TopicRouteData topicRouteData;\n                    if (isDefault && defaultMQProducer != null) {\n                        topicRouteData = this.mQClientAPIImpl.getDefaultTopicRouteInfoFromNameServer(clientConfig.getMqClientApiTimeout());\n                        if (topicRouteData != null) {\n                            for (QueueData data : topicRouteData.getQueueDatas()) {\n                                int queueNums = Math.min(defaultMQProducer.getDefaultTopicQueueNums(), data.getReadQueueNums());\n                                data.setReadQueueNums(queueNums);\n                                data.setWriteQueueNums(queueNums);\n                            }\n                        }\n                    } else {\n                        topicRouteData = this.mQClientAPIImpl.getTopicRouteInfoFromNameServer(topic, clientConfig.getMqClientApiTimeout());\n                    }\n                    if (topicRouteData != null) {\n                        TopicRouteData old = this.topicRouteTable.get(topic);\n                        boolean changed = topicRouteData.topicRouteDataChanged(old);\n                        if (!changed) {\n                            changed = this.isNeedUpdateTopicRouteInfo(topic);\n                        } else {\n                            log.info(\"the topic[{}] route info changed, old[{}] ,new[{}]\", topic, old, topicRouteData);\n                        }\n\n                        if (changed) {\n\n                            for (BrokerData bd : topicRouteData.getBrokerDatas()) {\n                                this.brokerAddrTable.put(bd.getBrokerName(), bd.getBrokerAddrs());\n                            }\n\n                            // Update endpoint map\n                            {\n                                ConcurrentMap<MessageQueue, String> mqEndPoints = topicRouteData2EndpointsForStaticTopic(topic, topicRouteData);\n                                if (!mqEndPoints.isEmpty()) {\n                                    topicEndPointsTable.put(topic, mqEndPoints);\n                                }\n                            }\n\n                            // Update Pub info\n                            {\n                                TopicPublishInfo publishInfo = topicRouteData2TopicPublishInfo(topic, topicRouteData);\n                                publishInfo.setHaveTopicRouterInfo(true);\n                                for (Entry<String, MQProducerInner> entry : this.producerTable.entrySet()) {\n                                    MQProducerInner impl = entry.getValue();\n                                    if (impl != null) {\n                                        impl.updateTopicPublishInfo(topic, publishInfo);\n                                    }\n                                }\n                            }\n\n                            // Update sub info\n                            if (!consumerTable.isEmpty()) {\n                                Set<MessageQueue> subscribeInfo = topicRouteData2TopicSubscribeInfo(topic, topicRouteData);\n                                for (Entry<String, MQConsumerInner> entry : this.consumerTable.entrySet()) {\n                                    MQConsumerInner impl = entry.getValue();\n                                    if (impl != null) {\n                                        impl.updateTopicSubscribeInfo(topic, subscribeInfo);\n                                    }\n                                }\n                            }\n                            TopicRouteData cloneTopicRouteData = new TopicRouteData(topicRouteData);\n                            log.info(\"topicRouteTable.put. Topic = {}, TopicRouteData[{}]\", topic, cloneTopicRouteData);\n                            this.topicRouteTable.put(topic, cloneTopicRouteData);\n                            return true;\n                        }\n                    } else {\n                        log.warn(\"updateTopicRouteInfoFromNameServer, getTopicRouteInfoFromNameServer return null, Topic: {}. [{}]\", topic, this.clientId);\n                    }\n                } catch (MQClientException e) {\n                    if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) && !topic.equals(TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC)) {\n                        log.warn(\"updateTopicRouteInfoFromNameServer Exception\", e);\n                    }\n                } catch (RemotingException e) {\n                    log.error(\"updateTopicRouteInfoFromNameServer Exception\", e);\n                    throw new IllegalStateException(e);\n                } finally {\n                    this.lockNamesrv.unlock();\n                }\n            } else {\n                log.warn(\"updateTopicRouteInfoFromNameServer tryLock timeout {}ms. [{}]\", LOCK_TIMEOUT_MILLIS, this.clientId);\n            }\n        } catch (InterruptedException e) {\n            log.warn(\"updateTopicRouteInfoFromNameServer Exception\", e);\n        }\n\n        return false;\n    }\n\n    private HeartbeatData prepareHeartbeatData(boolean isWithoutSub) {\n        HeartbeatData heartbeatData = new HeartbeatData();\n\n        // clientID\n        heartbeatData.setClientID(this.clientId);\n\n        // Consumer\n        for (Map.Entry<String, MQConsumerInner> entry : this.consumerTable.entrySet()) {\n            MQConsumerInner impl = entry.getValue();\n            if (impl != null) {\n                ConsumerData consumerData = new ConsumerData();\n                consumerData.setGroupName(impl.groupName());\n                consumerData.setConsumeType(impl.consumeType());\n                consumerData.setMessageModel(impl.messageModel());\n                consumerData.setConsumeFromWhere(impl.consumeFromWhere());\n                consumerData.setUnitMode(impl.isUnitMode());\n                if (!isWithoutSub) {\n                    consumerData.getSubscriptionDataSet().addAll(impl.subscriptions());\n                }\n                heartbeatData.getConsumerDataSet().add(consumerData);\n            }\n        }\n\n        // Producer\n        for (Map.Entry<String/* group */, MQProducerInner> entry : this.producerTable.entrySet()) {\n            MQProducerInner impl = entry.getValue();\n            if (impl != null) {\n                ProducerData producerData = new ProducerData();\n                producerData.setGroupName(entry.getKey());\n\n                heartbeatData.getProducerDataSet().add(producerData);\n            }\n        }\n        heartbeatData.setWithoutSub(isWithoutSub);\n        return heartbeatData;\n    }\n\n    private boolean isBrokerInNameServer(final String brokerAddr) {\n        for (Entry<String, TopicRouteData> itNext : this.topicRouteTable.entrySet()) {\n            List<BrokerData> brokerDatas = itNext.getValue().getBrokerDatas();\n            for (BrokerData bd : brokerDatas) {\n                boolean contain = bd.getBrokerAddrs().containsValue(brokerAddr);\n                if (contain)\n                    return true;\n            }\n        }\n\n        return false;\n    }\n\n    private boolean isNeedUpdateTopicRouteInfo(final String topic) {\n        boolean result = false;\n        Iterator<Entry<String, MQProducerInner>> producerIterator = this.producerTable.entrySet().iterator();\n        while (producerIterator.hasNext() && !result) {\n            Entry<String, MQProducerInner> entry = producerIterator.next();\n            MQProducerInner impl = entry.getValue();\n            if (impl != null) {\n                result = impl.isPublishTopicNeedUpdate(topic);\n            }\n        }\n\n        if (result) {\n            return true;\n        }\n\n        Iterator<Entry<String, MQConsumerInner>> consumerIterator = this.consumerTable.entrySet().iterator();\n        while (consumerIterator.hasNext() && !result) {\n            Entry<String, MQConsumerInner> entry = consumerIterator.next();\n            MQConsumerInner impl = entry.getValue();\n            if (impl != null) {\n                result = impl.isSubscribeTopicNeedUpdate(topic);\n            }\n        }\n\n        return result;\n    }\n\n    public void shutdown() {\n        // Consumer\n        if (!this.consumerTable.isEmpty())\n            return;\n\n        // AdminExt\n        if (!this.adminExtTable.isEmpty())\n            return;\n\n        // Producer\n        if (this.producerTable.size() > 1)\n            return;\n\n        synchronized (this) {\n            switch (this.serviceState) {\n                case RUNNING:\n                    this.defaultMQProducer.getDefaultMQProducerImpl().shutdown(false);\n\n                    this.serviceState = ServiceState.SHUTDOWN_ALREADY;\n                    this.pullMessageService.shutdown(true);\n                    this.scheduledExecutorService.shutdown();\n                    this.mQClientAPIImpl.shutdown();\n                    this.rebalanceService.shutdown();\n                    if (concurrentHeartbeatExecutor != null) {\n                        this.concurrentHeartbeatExecutor.shutdown();\n                    }\n\n                    MQClientManager.getInstance().removeClientFactory(this.clientId);\n                    log.info(\"the client factory [{}] shutdown OK\", this.clientId);\n                    break;\n                case CREATE_JUST:\n                case SHUTDOWN_ALREADY:\n                default:\n                    break;\n            }\n        }\n    }\n\n    public synchronized boolean registerConsumer(final String group, final MQConsumerInner consumer) {\n        if (null == group || null == consumer) {\n            return false;\n        }\n\n        MQConsumerInner prev = this.consumerTable.putIfAbsent(group, consumer);\n        if (prev != null) {\n            log.warn(\"the consumer group[\" + group + \"] exist already.\");\n            return false;\n        }\n\n        return true;\n    }\n\n    public synchronized void unregisterConsumer(final String group) {\n        this.consumerTable.remove(group);\n        this.unregisterClient(null, group);\n    }\n\n    private void unregisterClient(final String producerGroup, final String consumerGroup) {\n        for (Entry<String, HashMap<Long, String>> brokerClusterInfo : this.brokerAddrTable.entrySet()) {\n            String brokerName = brokerClusterInfo.getKey();\n            HashMap<Long, String> oneTable = brokerClusterInfo.getValue();\n\n            if (oneTable == null) {\n                continue;\n            }\n            for (Entry<Long, String> singleBrokerInstance : oneTable.entrySet()) {\n                String addr = singleBrokerInstance.getValue();\n                if (addr != null) {\n                    try {\n                        this.mQClientAPIImpl.unregisterClient(addr, this.clientId, producerGroup, consumerGroup, clientConfig.getMqClientApiTimeout());\n                        log.info(\"unregister client[Producer: {} Consumer: {}] from broker[{} {} {}] success\", producerGroup, consumerGroup, brokerName, singleBrokerInstance.getKey(), addr);\n                    } catch (RemotingException e) {\n                        log.warn(\"unregister client RemotingException from broker: {}, {}\", addr, e.getMessage());\n                    } catch (InterruptedException e) {\n                        log.warn(\"unregister client InterruptedException from broker: {}, {}\", addr, e.getMessage());\n                    } catch (MQBrokerException e) {\n                        log.warn(\"unregister client MQBrokerException from broker: {}, {}\", addr, e.getMessage());\n                    }\n                }\n            }\n        }\n    }\n\n    public synchronized boolean registerProducer(final String group, final DefaultMQProducerImpl producer) {\n        if (null == group || null == producer) {\n            return false;\n        }\n\n        MQProducerInner prev = this.producerTable.putIfAbsent(group, producer);\n        if (prev != null) {\n            log.warn(\"the producer group[{}] exist already.\", group);\n            return false;\n        }\n\n        return true;\n    }\n\n    public synchronized void unregisterProducer(final String group) {\n        this.producerTable.remove(group);\n        this.unregisterClient(group, null);\n    }\n\n    public boolean registerAdminExt(final String group, final MQAdminExtInner admin) {\n        if (null == group || null == admin) {\n            return false;\n        }\n\n        MQAdminExtInner prev = this.adminExtTable.putIfAbsent(group, admin);\n        if (prev != null) {\n            log.warn(\"the admin group[{}] exist already.\", group);\n            return false;\n        }\n\n        return true;\n    }\n\n    public void unregisterAdminExt(final String group) {\n        this.adminExtTable.remove(group);\n    }\n\n    public void rebalanceLater(long delayMillis) {\n        if (delayMillis <= 0) {\n            this.rebalanceService.wakeup();\n        } else {\n            this.scheduledExecutorService.schedule(MQClientInstance.this.rebalanceService::wakeup, delayMillis, TimeUnit.MILLISECONDS);\n        }\n    }\n\n    public void rebalanceImmediately() {\n        this.rebalanceService.wakeup();\n    }\n\n    public boolean doRebalance() {\n        boolean balanced = true;\n        for (Map.Entry<String, MQConsumerInner> entry : this.consumerTable.entrySet()) {\n            MQConsumerInner impl = entry.getValue();\n            if (impl != null) {\n                try {\n                    if (!impl.tryRebalance()) {\n                        balanced = false;\n                    }\n                } catch (Throwable e) {\n                    log.error(\"doRebalance for consumer group [{}] exception\", entry.getKey(), e);\n                }\n            }\n        }\n\n        return balanced;\n    }\n\n    public MQProducerInner selectProducer(final String group) {\n        return this.producerTable.get(group);\n    }\n\n    public MQConsumerInner selectConsumer(final String group) {\n        return this.consumerTable.get(group);\n    }\n\n    public String getBrokerNameFromMessageQueue(final MessageQueue mq) {\n        if (topicEndPointsTable.get(mq.getTopic()) != null && !topicEndPointsTable.get(mq.getTopic()).isEmpty()) {\n            return topicEndPointsTable.get(mq.getTopic()).get(mq);\n        }\n        return mq.getBrokerName();\n    }\n\n    public FindBrokerResult findBrokerAddressInAdmin(final String brokerName) {\n        if (brokerName == null) {\n            return null;\n        }\n        String brokerAddr = null;\n        boolean slave = false;\n        boolean found = false;\n\n        HashMap<Long/* brokerId */, String/* address */> map = this.brokerAddrTable.get(brokerName);\n        if (map != null && !map.isEmpty()) {\n            for (Map.Entry<Long, String> entry : map.entrySet()) {\n                Long id = entry.getKey();\n                brokerAddr = entry.getValue();\n                if (brokerAddr != null) {\n                    found = true;\n                    slave = MixAll.MASTER_ID != id;\n                    break;\n\n                }\n            } // end of for\n        }\n\n        if (found) {\n            return new FindBrokerResult(brokerAddr, slave, findBrokerVersion(brokerName, brokerAddr));\n        }\n\n        return null;\n    }\n\n    public String findBrokerAddressInPublish(final String brokerName) {\n        if (brokerName == null) {\n            return null;\n        }\n        HashMap<Long/* brokerId */, String/* address */> map = this.brokerAddrTable.get(brokerName);\n        if (map != null && !map.isEmpty()) {\n            return map.get(MixAll.MASTER_ID);\n        }\n\n        return null;\n    }\n\n    public FindBrokerResult findBrokerAddressInSubscribe(\n        final String brokerName,\n        final long brokerId,\n        final boolean onlyThisBroker\n    ) {\n        if (brokerName == null) {\n            return null;\n        }\n        String brokerAddr = null;\n        boolean slave = false;\n        boolean found = false;\n\n        HashMap<Long/* brokerId */, String/* address */> map = this.brokerAddrTable.get(brokerName);\n        if (map != null && !map.isEmpty()) {\n            brokerAddr = map.get(brokerId);\n            slave = brokerId != MixAll.MASTER_ID;\n            found = brokerAddr != null;\n\n            if (!found && slave) {\n                brokerAddr = map.get(brokerId + 1);\n                found = brokerAddr != null;\n            }\n\n            if (!found && !onlyThisBroker) {\n                Entry<Long, String> entry = map.entrySet().iterator().next();\n                brokerAddr = entry.getValue();\n                slave = entry.getKey() != MixAll.MASTER_ID;\n                found = brokerAddr != null;\n            }\n        }\n\n        if (found) {\n            return new FindBrokerResult(brokerAddr, slave, findBrokerVersion(brokerName, brokerAddr));\n        }\n\n        return null;\n    }\n\n    private int findBrokerVersion(String brokerName, String brokerAddr) {\n        if (this.brokerVersionTable.containsKey(brokerName)) {\n            if (this.brokerVersionTable.get(brokerName).containsKey(brokerAddr)) {\n                return this.brokerVersionTable.get(brokerName).get(brokerAddr);\n            }\n        }\n        //To do need to fresh the version\n        return 0;\n    }\n\n    public List<String> findConsumerIdList(final String topic, final String group) {\n        String brokerAddr = this.findBrokerAddrByTopic(topic);\n        if (null == brokerAddr) {\n            this.updateTopicRouteInfoFromNameServer(topic);\n            brokerAddr = this.findBrokerAddrByTopic(topic);\n        }\n\n        if (null != brokerAddr) {\n            try {\n                return this.mQClientAPIImpl.getConsumerIdListByGroup(brokerAddr, group, clientConfig.getMqClientApiTimeout());\n            } catch (Exception e) {\n                log.warn(\"getConsumerIdListByGroup exception, \" + brokerAddr + \" \" + group, e);\n            }\n        }\n\n        return null;\n    }\n\n    public Set<MessageQueueAssignment> queryAssignment(final String topic, final String consumerGroup,\n        final String strategyName, final MessageModel messageModel, int timeout)\n        throws RemotingException, InterruptedException, MQBrokerException {\n        String brokerAddr = this.findBrokerAddrByTopic(topic);\n        if (null == brokerAddr) {\n            this.updateTopicRouteInfoFromNameServer(topic);\n            brokerAddr = this.findBrokerAddrByTopic(topic);\n        }\n\n        if (null != brokerAddr) {\n            return this.mQClientAPIImpl.queryAssignment(brokerAddr, topic, consumerGroup, clientId, strategyName,\n                messageModel, timeout);\n        }\n\n        return null;\n    }\n\n    public String findBrokerAddrByTopic(final String topic) {\n        TopicRouteData topicRouteData = this.topicRouteTable.get(topic);\n        if (topicRouteData != null) {\n            List<BrokerData> brokers = topicRouteData.getBrokerDatas();\n            if (!brokers.isEmpty()) {\n                BrokerData bd = brokers.get(random.nextInt(brokers.size()));\n                return bd.selectBrokerAddr();\n            }\n        }\n\n        return null;\n    }\n\n    public synchronized void resetOffset(String topic, String group, Map<MessageQueue, Long> offsetTable) {\n        DefaultMQPushConsumerImpl consumer = null;\n        try {\n            MQConsumerInner impl = this.consumerTable.get(group);\n            if (impl instanceof DefaultMQPushConsumerImpl) {\n                consumer = (DefaultMQPushConsumerImpl) impl;\n            } else {\n                log.info(\"[reset-offset] consumer does not exist. group={}\", group);\n                return;\n            }\n            consumer.suspend();\n\n            ConcurrentMap<MessageQueue, ProcessQueue> processQueueTable = consumer.getRebalanceImpl().getProcessQueueTable();\n            for (Map.Entry<MessageQueue, ProcessQueue> entry : processQueueTable.entrySet()) {\n                MessageQueue mq = entry.getKey();\n                if (topic.equals(mq.getTopic()) && offsetTable.containsKey(mq)) {\n                    ProcessQueue pq = entry.getValue();\n                    pq.setDropped(true);\n                    pq.clear();\n                }\n            }\n\n            try {\n                TimeUnit.SECONDS.sleep(10);\n            } catch (InterruptedException ignored) {\n            }\n\n            Iterator<MessageQueue> iterator = processQueueTable.keySet().iterator();\n            while (iterator.hasNext()) {\n                MessageQueue mq = iterator.next();\n                Long offset = offsetTable.get(mq);\n                if (topic.equals(mq.getTopic()) && offset != null) {\n                    try {\n                        consumer.updateConsumeOffset(mq, offset);\n                        consumer.getRebalanceImpl().removeUnnecessaryMessageQueue(mq, processQueueTable.get(mq));\n                        iterator.remove();\n                    } catch (Exception e) {\n                        log.warn(\"reset offset failed. group={}, {}\", group, mq, e);\n                    }\n                }\n            }\n        } finally {\n            if (consumer != null) {\n                consumer.resume();\n            }\n        }\n    }\n\n    @SuppressWarnings(\"unchecked\")\n    public Map<MessageQueue, Long> getConsumerStatus(String topic, String group) {\n        MQConsumerInner impl = this.consumerTable.get(group);\n        if (impl instanceof DefaultMQPushConsumerImpl) {\n            DefaultMQPushConsumerImpl consumer = (DefaultMQPushConsumerImpl) impl;\n            return consumer.getOffsetStore().cloneOffsetTable(topic);\n        } else if (impl instanceof DefaultMQPullConsumerImpl) {\n            DefaultMQPullConsumerImpl consumer = (DefaultMQPullConsumerImpl) impl;\n            return consumer.getOffsetStore().cloneOffsetTable(topic);\n        } else {\n            return Collections.EMPTY_MAP;\n        }\n    }\n\n    public TopicRouteData getAnExistTopicRouteData(final String topic) {\n        return this.topicRouteTable.get(topic);\n    }\n\n    public MQClientAPIImpl getMQClientAPIImpl() {\n        return mQClientAPIImpl;\n    }\n\n    public MQAdminImpl getMQAdminImpl() {\n        return mQAdminImpl;\n    }\n\n    public long getBootTimestamp() {\n        return bootTimestamp;\n    }\n\n    public ScheduledExecutorService getScheduledExecutorService() {\n        return scheduledExecutorService;\n    }\n\n    public PullMessageService getPullMessageService() {\n        return pullMessageService;\n    }\n\n    public DefaultMQProducer getDefaultMQProducer() {\n        return defaultMQProducer;\n    }\n\n    public ConcurrentMap<String, TopicRouteData> getTopicRouteTable() {\n        return topicRouteTable;\n    }\n\n    public ConsumeMessageDirectlyResult consumeMessageDirectly(final MessageExt msg,\n        final String consumerGroup,\n        final String brokerName) {\n        MQConsumerInner mqConsumerInner = this.consumerTable.get(consumerGroup);\n        if (mqConsumerInner instanceof DefaultMQPushConsumerImpl) {\n            DefaultMQPushConsumerImpl consumer = (DefaultMQPushConsumerImpl) mqConsumerInner;\n\n            return consumer.getConsumeMessageService().consumeMessageDirectly(msg, brokerName);\n        }\n\n        return null;\n    }\n\n    public ConsumerRunningInfo consumerRunningInfo(final String consumerGroup) {\n        MQConsumerInner mqConsumerInner = this.consumerTable.get(consumerGroup);\n        if (mqConsumerInner == null) {\n            return null;\n        }\n\n        ConsumerRunningInfo consumerRunningInfo = mqConsumerInner.consumerRunningInfo();\n\n        List<String> nsList = this.mQClientAPIImpl.getRemotingClient().getNameServerAddressList();\n\n        StringBuilder strBuilder = new StringBuilder();\n        if (nsList != null) {\n            for (String addr : nsList) {\n                strBuilder.append(addr).append(\";\");\n            }\n        }\n\n        String nsAddr = strBuilder.toString();\n        consumerRunningInfo.getProperties().put(ConsumerRunningInfo.PROP_NAMESERVER_ADDR, nsAddr);\n        consumerRunningInfo.getProperties().put(ConsumerRunningInfo.PROP_CONSUME_TYPE, mqConsumerInner.consumeType().name());\n        consumerRunningInfo.getProperties().put(ConsumerRunningInfo.PROP_CLIENT_VERSION,\n            MQVersion.getVersionDesc(MQVersion.CURRENT_VERSION));\n\n        return consumerRunningInfo;\n    }\n\n    private void resetBrokerAddrHeartbeatFingerprintMap() {\n        brokerAddrHeartbeatFingerprintTable.clear();\n    }\n\n    public ConsumerStatsManager getConsumerStatsManager() {\n        return consumerStatsManager;\n    }\n\n    public NettyClientConfig getNettyClientConfig() {\n        return nettyClientConfig;\n    }\n\n    public ClientConfig getClientConfig() {\n        return clientConfig;\n    }\n\n    public ConcurrentMap<String, MQProducerInner> getProducerTable() {\n        return producerTable;\n    }\n\n    public ConcurrentMap<String, MQConsumerInner> getConsumerTable() {\n        return consumerTable;\n    }\n\n    public TopicRouteData queryTopicRouteData(String topic) {\n        TopicRouteData data = this.getAnExistTopicRouteData(topic);\n        if (data == null) {\n            this.updateTopicRouteInfoFromNameServer(topic);\n            data = this.getAnExistTopicRouteData(topic);\n        }\n        return data;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/mqclient/DoNothingClientRemotingProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.mqclient;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.client.impl.ClientRemotingProcessor;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic class DoNothingClientRemotingProcessor extends ClientRemotingProcessor {\n\n    public DoNothingClientRemotingProcessor(MQClientInstance mqClientFactory) {\n        super(mqClientFactory);\n    }\n\n    @Override\n    public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) {\n        return null;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.mqclient;\n\nimport java.nio.ByteBuffer;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.CompletionException;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.consumer.AckCallback;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.NotifyResult;\nimport org.apache.rocketmq.client.consumer.PopCallback;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PullCallback;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.OffsetNotFoundException;\nimport org.apache.rocketmq.client.impl.ClientRemotingProcessor;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.admin.MqClientAdminImpl;\nimport org.apache.rocketmq.client.impl.consumer.PullResultExt;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.ObjectCreator;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.lite.LiteSubscriptionDTO;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageBatch;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.InvokeCallback;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.RemotingClient;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.ResponseFuture;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.GetLiteTopicInfoResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.LiteSubscriptionCtlRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.LockBatchResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupResponseBody;\nimport org.apache.rocketmq.remoting.protocol.header.GetLiteTopicInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMinOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMinOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.HeartbeatRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.LiteSubscriptionCtlRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.LockBatchMqRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.NotificationRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.NotificationResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopLiteMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SearchOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SearchOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2;\nimport org.apache.rocketmq.remoting.protocol.header.UnlockBatchMqRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;\n\npublic class MQClientAPIExt extends MQClientAPIImpl {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    private final ClientConfig clientConfig;\n\n    private final MqClientAdminImpl mqClientAdmin;\n\n    public MQClientAPIExt(\n        ClientConfig clientConfig,\n        NettyClientConfig nettyClientConfig,\n        ClientRemotingProcessor clientRemotingProcessor,\n        RPCHook rpcHook\n    ) {\n        this(clientConfig, nettyClientConfig, clientRemotingProcessor, rpcHook, null);\n    }\n\n    public MQClientAPIExt(\n        ClientConfig clientConfig,\n        NettyClientConfig nettyClientConfig,\n        ClientRemotingProcessor clientRemotingProcessor,\n        RPCHook rpcHook,\n        ObjectCreator<RemotingClient> remotingClientCreator\n    ) {\n        super(nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig, null, remotingClientCreator);\n        this.clientConfig = clientConfig;\n        this.mqClientAdmin = new MqClientAdminImpl(getRemotingClient());\n    }\n\n    public boolean updateNameServerAddressList() {\n        if (this.clientConfig.getNamesrvAddr() != null) {\n            this.updateNameServerAddressList(this.clientConfig.getNamesrvAddr());\n            log.info(\"user specified name server address: {}\", this.clientConfig.getNamesrvAddr());\n            return true;\n        }\n        return false;\n    }\n\n    public CompletableFuture<Void> sendHeartbeatOneway(\n        String brokerAddr,\n        HeartbeatData heartbeatData,\n        long timeoutMillis\n    ) {\n        CompletableFuture<Void> future = new CompletableFuture<>();\n        try {\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.HEART_BEAT, new HeartbeatRequestHeader());\n            request.setLanguage(clientConfig.getLanguage());\n            request.setBody(heartbeatData.encode());\n            this.getRemotingClient().invokeOneway(brokerAddr, request, timeoutMillis);\n            future.complete(null);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    public CompletableFuture<Integer> sendHeartbeatAsync(\n        String brokerAddr,\n        HeartbeatData heartbeatData,\n        long timeoutMillis\n    ) {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.HEART_BEAT, new HeartbeatRequestHeader());\n        request.setLanguage(clientConfig.getLanguage());\n        request.setBody(heartbeatData.encode());\n\n        return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {\n            CompletableFuture<Integer> future0 = new CompletableFuture<>();\n            if (ResponseCode.SUCCESS == response.getCode()) {\n                future0.complete(response.getVersion());\n            } else {\n                future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr));\n            }\n            return future0;\n        });\n    }\n\n    public CompletableFuture<SendResult> sendMessageAsync(\n        String brokerAddr,\n        String brokerName,\n        Message msg,\n        SendMessageRequestHeader requestHeader,\n        long timeoutMillis\n    ) {\n        SendMessageRequestHeaderV2 requestHeaderV2 = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(requestHeader);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, requestHeaderV2);\n        request.setBody(msg.getBody());\n\n        return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {\n            CompletableFuture<SendResult> future0 = new CompletableFuture<>();\n            try {\n                future0.complete(this.processSendResponse(brokerName, msg, response, brokerAddr));\n            } catch (Exception e) {\n                future0.completeExceptionally(e);\n            }\n            return future0;\n        });\n    }\n\n    public CompletableFuture<SendResult> sendMessageAsync(\n        String brokerAddr,\n        String brokerName,\n        List<? extends Message> msgList,\n        SendMessageRequestHeader requestHeader,\n        long timeoutMillis\n    ) {\n        SendMessageRequestHeaderV2 requestHeaderV2 = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(requestHeader);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_BATCH_MESSAGE, requestHeaderV2);\n\n        CompletableFuture<SendResult> future = new CompletableFuture<>();\n        try {\n            requestHeader.setBatch(true);\n            MessageBatch msgBatch = MessageBatch.generateFromList(msgList);\n            MessageClientIDSetter.setUniqID(msgBatch);\n            byte[] body = msgBatch.encode();\n            msgBatch.setBody(body);\n\n            request.setBody(body);\n            return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {\n                CompletableFuture<SendResult> future0 = new CompletableFuture<>();\n                try {\n                    future0.complete(processSendResponse(brokerName, msgBatch, response, brokerAddr));\n                } catch (Exception e) {\n                    future0.completeExceptionally(e);\n                }\n                return future0;\n            });\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    public CompletableFuture<RemotingCommand> sendMessageBackAsync(\n        String brokerAddr,\n        ConsumerSendMsgBackRequestHeader requestHeader,\n        long timeoutMillis\n    ) {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, requestHeader);\n        return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis);\n    }\n\n    public CompletableFuture<PopResult> popMessageAsync(\n        String brokerAddr,\n        String brokerName,\n        PopMessageRequestHeader requestHeader,\n        long timeoutMillis\n    ) {\n        CompletableFuture<PopResult> future = new CompletableFuture<>();\n        try {\n            this.popMessageAsync(brokerName, brokerAddr, requestHeader, timeoutMillis, new PopCallback() {\n                @Override\n                public void onSuccess(PopResult popResult) {\n                    future.complete(popResult);\n                }\n\n                @Override\n                public void onException(Throwable t) {\n                    future.completeExceptionally(t);\n                }\n            });\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    public CompletableFuture<PopResult> popLiteMessageAsync(\n        String brokerAddr,\n        String brokerName,\n        PopLiteMessageRequestHeader requestHeader,\n        long timeoutMillis\n    ) {\n        CompletableFuture<PopResult> future = new CompletableFuture<>();\n        try {\n            this.popLiteMessageAsync(brokerName, brokerAddr, requestHeader, timeoutMillis, new PopCallback() {\n                @Override\n                public void onSuccess(PopResult popResult) {\n                    future.complete(popResult);\n                }\n\n                @Override\n                public void onException(Throwable t) {\n                    future.completeExceptionally(t);\n                }\n            });\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    public CompletableFuture<AckResult> ackMessageAsync(\n        String brokerAddr,\n        AckMessageRequestHeader requestHeader,\n        long timeoutMillis\n    ) {\n        CompletableFuture<AckResult> future = new CompletableFuture<>();\n        try {\n            this.ackMessageAsync(brokerAddr, timeoutMillis, new AckCallback() {\n                @Override\n                public void onSuccess(AckResult ackResult) {\n                    future.complete(ackResult);\n                }\n\n                @Override\n                public void onException(Throwable t) {\n                    future.completeExceptionally(t);\n                }\n            }, requestHeader);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    public CompletableFuture<AckResult> batchAckMessageAsync(\n        String brokerAddr,\n        String topic,\n        String consumerGroup,\n        List<String> extraInfoList,\n        long timeoutMillis\n    ) {\n        CompletableFuture<AckResult> future = new CompletableFuture<>();\n        try {\n            this.batchAckMessageAsync(brokerAddr, timeoutMillis, new AckCallback() {\n                @Override\n                public void onSuccess(AckResult ackResult) {\n                    future.complete(ackResult);\n                }\n\n                @Override\n                public void onException(Throwable t) {\n                    future.completeExceptionally(t);\n                }\n            }, topic, consumerGroup, extraInfoList);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    public CompletableFuture<AckResult> changeInvisibleTimeAsync(\n        String brokerAddr,\n        String brokerName,\n        ChangeInvisibleTimeRequestHeader requestHeader,\n        long timeoutMillis\n    ) {\n        CompletableFuture<AckResult> future = new CompletableFuture<>();\n        try {\n            this.changeInvisibleTimeAsync(brokerName, brokerAddr, requestHeader, timeoutMillis,\n                new AckCallback() {\n                    @Override\n                    public void onSuccess(AckResult ackResult) {\n                        future.complete(ackResult);\n                    }\n\n                    @Override\n                    public void onException(Throwable t) {\n                        future.completeExceptionally(t);\n                    }\n                }\n            );\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    public CompletableFuture<PullResult> pullMessageAsync(\n        String brokerAddr,\n        PullMessageRequestHeader requestHeader,\n        long timeoutMillis\n    ) {\n        CompletableFuture<PullResult> future = new CompletableFuture<>();\n        try {\n            this.pullMessage(brokerAddr, requestHeader, timeoutMillis, CommunicationMode.ASYNC,\n                new PullCallback() {\n                    @Override\n                    public void onSuccess(PullResult pullResult) {\n                        if (pullResult instanceof PullResultExt) {\n                            PullResultExt pullResultExt = (PullResultExt) pullResult;\n                            if (PullStatus.FOUND.equals(pullResult.getPullStatus())) {\n                                List<MessageExt> messageExtList = MessageDecoder.decodesBatch(\n                                    ByteBuffer.wrap(pullResultExt.getMessageBinary()),\n                                    true,\n                                    false,\n                                    true\n                                );\n                                pullResult.setMsgFoundList(messageExtList);\n                            }\n                        }\n                        future.complete(pullResult);\n                    }\n\n                    @Override\n                    public void onException(Throwable t) {\n                        future.completeExceptionally(t);\n                    }\n                }\n            );\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    public CompletableFuture<Long> queryConsumerOffsetWithFuture(\n        String brokerAddr,\n        QueryConsumerOffsetRequestHeader requestHeader,\n        long timeoutMillis\n    ) {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader);\n        return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {\n            CompletableFuture<Long> future0 = new CompletableFuture<>();\n            switch (response.getCode()) {\n                case ResponseCode.SUCCESS: {\n                    try {\n                        QueryConsumerOffsetResponseHeader responseHeader =\n                            (QueryConsumerOffsetResponseHeader) response.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class);\n                        future0.complete(responseHeader.getOffset());\n                    } catch (RemotingCommandException e) {\n                        future0.completeExceptionally(e);\n                    }\n                    break;\n                }\n                case ResponseCode.QUERY_NOT_FOUND: {\n                    future0.completeExceptionally(new OffsetNotFoundException(response.getCode(), response.getRemark(), brokerAddr));\n                    break;\n                }\n                default: {\n                    future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));\n                    break;\n                }\n            }\n            return future0;\n        });\n    }\n\n    public CompletableFuture<Void> updateConsumerOffsetOneWay(\n        String brokerAddr,\n        UpdateConsumerOffsetRequestHeader header,\n        long timeoutMillis\n    ) {\n        CompletableFuture<Void> future = new CompletableFuture<>();\n        try {\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_CONSUMER_OFFSET, header);\n            this.getRemotingClient().invokeOneway(brokerAddr, request, timeoutMillis);\n            future.complete(null);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    public CompletableFuture<Void> updateConsumerOffsetAsync(\n        String brokerAddr,\n        UpdateConsumerOffsetRequestHeader header,\n        long timeoutMillis\n    ) {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_CONSUMER_OFFSET, header);\n        CompletableFuture<Void> future = new CompletableFuture<>();\n        invoke(brokerAddr, request, timeoutMillis).whenComplete((response, t) -> {\n            if (t != null) {\n                log.error(\"updateConsumerOffsetAsync failed, brokerAddr={}, requestHeader={}\", brokerAddr, header, t);\n                future.completeExceptionally(t);\n                return;\n            }\n            switch (response.getCode()) {\n                case ResponseCode.SUCCESS:\n                    future.complete(null);\n                    break;\n                default:\n                    future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr));\n                    break;\n            }\n        });\n        return future;\n    }\n\n    public CompletableFuture<List<String>> getConsumerListByGroupAsync(\n        String brokerAddr,\n        GetConsumerListByGroupRequestHeader requestHeader,\n        long timeoutMillis\n    ) {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_LIST_BY_GROUP, requestHeader);\n\n        CompletableFuture<List<String>> future = new CompletableFuture<>();\n        try {\n            this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() {\n                @Override\n                public void operationComplete(ResponseFuture responseFuture) {\n\n                }\n\n                @Override\n                public void operationSucceed(RemotingCommand response) {\n                    switch (response.getCode()) {\n                        case ResponseCode.SUCCESS: {\n                            if (response.getBody() != null) {\n                                GetConsumerListByGroupResponseBody body =\n                                    GetConsumerListByGroupResponseBody.decode(response.getBody(), GetConsumerListByGroupResponseBody.class);\n                                future.complete(body.getConsumerIdList());\n                                return;\n                            }\n                        }\n                        /*\n                          @see org.apache.rocketmq.broker.processor.ConsumerManageProcessor#getConsumerListByGroup,\n                         * broker will return {@link ResponseCode.SYSTEM_ERROR} if there is no consumer.\n                         */\n                        case ResponseCode.SYSTEM_ERROR: {\n                            future.complete(Collections.emptyList());\n                            return;\n                        }\n                        default:\n                            break;\n                    }\n                    future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));\n                }\n\n                @Override\n                public void operationFail(Throwable throwable) {\n                    future.completeExceptionally(throwable);\n                }\n            });\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    public CompletableFuture<Long> getMaxOffset(String brokerAddr, GetMaxOffsetRequestHeader requestHeader,\n        long timeoutMillis) {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_MAX_OFFSET, requestHeader);\n\n        CompletableFuture<Long> future = new CompletableFuture<>();\n        try {\n            this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() {\n                @Override\n                public void operationComplete(ResponseFuture responseFuture) {\n\n                }\n\n                @Override\n                public void operationSucceed(RemotingCommand response) {\n                    if (ResponseCode.SUCCESS == response.getCode()) {\n                        try {\n                            GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class);\n                            future.complete(responseHeader.getOffset());\n                        } catch (Throwable t) {\n                            future.completeExceptionally(t);\n                        }\n                    }\n                    future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));\n                }\n\n                @Override\n                public void operationFail(Throwable throwable) {\n                    future.completeExceptionally(throwable);\n                }\n            });\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    public CompletableFuture<Long> getMinOffset(String brokerAddr, GetMinOffsetRequestHeader requestHeader,\n        long timeoutMillis) {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_MIN_OFFSET, requestHeader);\n\n        CompletableFuture<Long> future = new CompletableFuture<>();\n        try {\n            this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() {\n                @Override\n                public void operationComplete(ResponseFuture responseFuture) {\n\n                }\n\n                @Override\n                public void operationSucceed(RemotingCommand response) {\n                    if (ResponseCode.SUCCESS == response.getCode()) {\n                        try {\n                            GetMinOffsetResponseHeader responseHeader = (GetMinOffsetResponseHeader) response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class);\n                            future.complete(responseHeader.getOffset());\n                        } catch (Throwable t) {\n                            future.completeExceptionally(t);\n                        }\n                    }\n                    future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));\n                }\n\n                @Override\n                public void operationFail(Throwable throwable) {\n                    future.completeExceptionally(throwable);\n                }\n            });\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    public CompletableFuture<Long> searchOffset(String brokerAddr, SearchOffsetRequestHeader requestHeader,\n        long timeoutMillis) {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, requestHeader);\n\n        return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {\n            CompletableFuture<Long> future0 = new CompletableFuture<>();\n            if (response.getCode() == ResponseCode.SUCCESS) {\n                try {\n                    SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.decodeCommandCustomHeader(SearchOffsetResponseHeader.class);\n                    future0.complete(responseHeader.getOffset());\n                } catch (Throwable t) {\n                    future0.completeExceptionally(t);\n                }\n            } else {\n                future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));\n            }\n            return future0;\n        });\n    }\n\n    public CompletableFuture<Set<MessageQueue>> lockBatchMQWithFuture(String brokerAddr,\n        LockBatchRequestBody requestBody, long timeoutMillis) {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, new LockBatchMqRequestHeader());\n        request.setBody(requestBody.encode());\n        return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {\n            CompletableFuture<Set<MessageQueue>> future0 = new CompletableFuture<>();\n            if (response.getCode() == ResponseCode.SUCCESS) {\n                try {\n                    LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), LockBatchResponseBody.class);\n                    Set<MessageQueue> messageQueues = responseBody.getLockOKMQSet();\n                    future0.complete(messageQueues);\n                } catch (Throwable t) {\n                    future0.completeExceptionally(t);\n                }\n            } else {\n                future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));\n            }\n            return future0;\n        });\n    }\n\n    public CompletableFuture<Void> unlockBatchMQOneway(String brokerAddr,\n        UnlockBatchRequestBody requestBody, long timeoutMillis) {\n        CompletableFuture<Void> future = new CompletableFuture<>();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UNLOCK_BATCH_MQ, new UnlockBatchMqRequestHeader());\n        request.setBody(requestBody.encode());\n        try {\n            this.getRemotingClient().invokeOneway(brokerAddr, request, timeoutMillis);\n            future.complete(null);\n        } catch (Exception e) {\n            future.completeExceptionally(e);\n        }\n        return future;\n    }\n\n    public CompletableFuture<Boolean> notification(String brokerAddr, NotificationRequestHeader requestHeader,\n        long timeoutMillis) {\n        return notificationWithPollingStats(brokerAddr, requestHeader, timeoutMillis).thenApply(NotifyResult::isHasMsg);\n    }\n\n    public CompletableFuture<NotifyResult> notificationWithPollingStats(String brokerAddr,\n        NotificationRequestHeader requestHeader,\n        long timeoutMillis) {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.NOTIFICATION, requestHeader);\n        return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {\n            CompletableFuture<NotifyResult> future0 = new CompletableFuture<>();\n            if (response.getCode() == ResponseCode.SUCCESS) {\n                try {\n                    NotificationResponseHeader responseHeader = (NotificationResponseHeader) response.decodeCommandCustomHeader(NotificationResponseHeader.class);\n                    NotifyResult notifyResult = new NotifyResult();\n                    notifyResult.setHasMsg(responseHeader.isHasMsg());\n                    notifyResult.setPollingFull(responseHeader.isPollingFull());\n                    future0.complete(notifyResult);\n                } catch (Throwable t) {\n                    future0.completeExceptionally(t);\n                }\n            } else {\n                future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));\n            }\n            return future0;\n        });\n    }\n\n    public CompletableFuture<String> recallMessageAsync(String brokerAddr,\n        RecallMessageRequestHeader requestHeader, long timeoutMillis) {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RECALL_MESSAGE, requestHeader);\n        return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {\n            CompletableFuture<String> future = new CompletableFuture<>();\n            if (ResponseCode.SUCCESS == response.getCode()) {\n                try {\n                    RecallMessageResponseHeader responseHeader =\n                        response.decodeCommandCustomHeader(RecallMessageResponseHeader.class);\n                    future.complete(responseHeader.getMsgId());\n                } catch (Throwable t) {\n                    future.completeExceptionally(t);\n                }\n            } else {\n                future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr));\n            }\n            return future;\n        });\n    }\n\n    public CompletableFuture<Void> syncLiteSubscriptionAsync(\n        String brokerAddr,\n        LiteSubscriptionDTO liteSubscriptionDTO,\n        long timeoutMillis\n    ) {\n        LiteSubscriptionCtlRequestBody requestBody = new LiteSubscriptionCtlRequestBody();\n        requestBody.setSubscriptionSet(Collections.singleton(liteSubscriptionDTO));\n        RemotingCommand request = RemotingCommand\n            .createRequestCommand(RequestCode.LITE_SUBSCRIPTION_CTL, new LiteSubscriptionCtlRequestHeader());\n        request.setBody(requestBody.encode());\n\n        return getRemotingClient()\n            .invoke(brokerAddr, request, timeoutMillis)\n            .thenCompose(response -> {\n                if (ResponseCode.SUCCESS == response.getCode()) {\n                    return CompletableFuture.completedFuture(null);\n                } else {\n                    CompletableFuture<Void> future = new CompletableFuture<>();\n                    future.completeExceptionally(\n                        new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr)\n                    );\n                    return future;\n                }\n            });\n    }\n\n    public CompletableFuture<GetLiteTopicInfoResponseBody> getLiteTopicInfoAsync(\n        String addr,\n        String parentTopic,\n        String liteTopic,\n        long timeoutMillis\n    ) {\n        GetLiteTopicInfoRequestHeader requestHeader = new GetLiteTopicInfoRequestHeader();\n        requestHeader.setParentTopic(parentTopic);\n        requestHeader.setLiteTopic(liteTopic);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_LITE_TOPIC_INFO, requestHeader);\n\n        return this.getRemotingClient()\n            .invoke(addr, request, timeoutMillis)\n            .thenApply(response -> {\n                if (ResponseCode.SUCCESS == response.getCode()) {\n                    try {\n                        return GetLiteTopicInfoResponseBody.decode(response.getBody(), GetLiteTopicInfoResponseBody.class);\n                    } catch (Exception e) {\n                        throw new CompletionException(e);\n                    }\n                } else {\n                    throw new CompletionException(new MQBrokerException(response.getCode(), response.getRemark()));\n                }\n            });\n    }\n\n    public CompletableFuture<RemotingCommand> invoke(String brokerAddr, RemotingCommand request, long timeoutMillis) {\n        return getRemotingClient().invoke(brokerAddr, request, timeoutMillis);\n    }\n\n    public CompletableFuture<Void> invokeOneway(String brokerAddr, RemotingCommand request, long timeoutMillis) {\n        CompletableFuture<Void> future = new CompletableFuture<>();\n        try {\n            this.getRemotingClient().invokeOneway(brokerAddr, request, timeoutMillis);\n            future.complete(null);\n        } catch (Exception e) {\n            future.completeExceptionally(e);\n        }\n        return future;\n    }\n\n    public MqClientAdminImpl getMqClientAdmin() {\n        return mqClientAdmin;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIFactory.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.mqclient;\n\nimport com.google.common.base.Strings;\nimport java.time.Duration;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ThreadLocalRandom;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.common.NameserverAccessConfig;\nimport org.apache.rocketmq.client.impl.ClientRemotingProcessor;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ObjectCreator;\nimport org.apache.rocketmq.common.utils.AsyncShutdownHelper;\nimport org.apache.rocketmq.common.utils.StartAndShutdown;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.RemotingClient;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\n\npublic class MQClientAPIFactory implements StartAndShutdown {\n\n    private MQClientAPIExt[] clients;\n    private final String namePrefix;\n    private final int clientNum;\n    private final ClientRemotingProcessor clientRemotingProcessor;\n    private final RPCHook rpcHook;\n    private final ScheduledExecutorService scheduledExecutorService;\n    private final NameserverAccessConfig nameserverAccessConfig;\n    private final ObjectCreator<RemotingClient> remotingClientCreator;\n\n    public MQClientAPIFactory(\n        NameserverAccessConfig nameserverAccessConfig,\n        String namePrefix,\n        int clientNum,\n        ClientRemotingProcessor clientRemotingProcessor,\n        RPCHook rpcHook,\n        ScheduledExecutorService scheduledExecutorService\n    ) {\n        this(nameserverAccessConfig, namePrefix, clientNum, clientRemotingProcessor, rpcHook, scheduledExecutorService, null);\n    }\n\n    public MQClientAPIFactory(\n        NameserverAccessConfig nameserverAccessConfig,\n        String namePrefix,\n        int clientNum,\n        ClientRemotingProcessor clientRemotingProcessor,\n        RPCHook rpcHook,\n        ScheduledExecutorService scheduledExecutorService,\n        ObjectCreator<RemotingClient> remotingClientCreator\n    ) {\n        this.nameserverAccessConfig = nameserverAccessConfig;\n        this.namePrefix = namePrefix;\n        this.clientNum = clientNum;\n        this.clientRemotingProcessor = clientRemotingProcessor;\n        this.rpcHook = rpcHook;\n        this.scheduledExecutorService = scheduledExecutorService;\n        this.remotingClientCreator = remotingClientCreator;\n\n        this.init();\n    }\n\n    protected void init() {\n        System.setProperty(ClientConfig.SEND_MESSAGE_WITH_VIP_CHANNEL_PROPERTY, \"false\");\n        if (StringUtils.isEmpty(nameserverAccessConfig.getNamesrvDomain())) {\n            if (Strings.isNullOrEmpty(nameserverAccessConfig.getNamesrvAddr())) {\n                throw new RuntimeException(\"The configuration item NamesrvAddr is not configured\");\n            }\n            System.setProperty(MixAll.NAMESRV_ADDR_PROPERTY, nameserverAccessConfig.getNamesrvAddr());\n        } else {\n            System.setProperty(\"rocketmq.namesrv.domain\", nameserverAccessConfig.getNamesrvDomain());\n            System.setProperty(\"rocketmq.namesrv.domain.subgroup\", nameserverAccessConfig.getNamesrvDomainSubgroup());\n        }\n    }\n\n    public MQClientAPIExt getClient() {\n        if (clients.length == 1) {\n            return this.clients[0];\n        }\n        int index = ThreadLocalRandom.current().nextInt(this.clients.length);\n        return this.clients[index];\n    }\n\n    @Override\n    public void start() throws Exception {\n        this.clients = new MQClientAPIExt[this.clientNum];\n\n        for (int i = 0; i < this.clientNum; i++) {\n            clients[i] = createAndStart(this.namePrefix + \"N_\" + i);\n        }\n    }\n\n    @Override\n    public void shutdown() throws Exception {\n        AsyncShutdownHelper helper = new AsyncShutdownHelper();\n        for (int i = 0; i < this.clientNum; i++) {\n            helper.addTarget(clients[i]);\n        }\n        helper.shutdown().await(Integer.MAX_VALUE, TimeUnit.SECONDS);\n    }\n\n    protected MQClientAPIExt createAndStart(String instanceName) {\n        ClientConfig clientConfig = new ClientConfig();\n        clientConfig.setInstanceName(instanceName);\n        clientConfig.setDecodeReadBody(true);\n        clientConfig.setDecodeDecompressBody(false);\n\n        NettyClientConfig nettyClientConfig = new NettyClientConfig();\n        nettyClientConfig.setDisableCallbackExecutor(true);\n\n        MQClientAPIExt mqClientAPIExt = new MQClientAPIExt(\n            clientConfig,\n            nettyClientConfig,\n            clientRemotingProcessor,\n            rpcHook,\n            remotingClientCreator\n        );\n\n        if (StringUtils.isEmpty(nameserverAccessConfig.getNamesrvDomain())) {\n            mqClientAPIExt.updateNameServerAddressList(nameserverAccessConfig.getNamesrvAddr());\n        } else {\n            mqClientAPIExt.fetchNameServerAddr();\n            this.scheduledExecutorService.scheduleAtFixedRate(\n                mqClientAPIExt::fetchNameServerAddr,\n                Duration.ofSeconds(10).toMillis(),\n                Duration.ofMinutes(2).toMillis(),\n                TimeUnit.MILLISECONDS\n            );\n        }\n\n        mqClientAPIExt.start();\n        return mqClientAPIExt;\n    }\n\n    public void onNameServerAddressChange(String namesrvAddress) {\n        for (MQClientAPIExt client : clients) {\n            client.onNameServerAddressChange(namesrvAddress);\n        }\n    }\n\n    public MQClientAPIExt[] getClients() {\n        return clients;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.producer;\n\nimport java.io.IOException;\nimport java.net.UnknownHostException;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Random;\nimport java.util.Set;\nimport java.util.Optional;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.RejectedExecutionException;\nimport java.util.concurrent.Semaphore;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.QueryResult;\nimport org.apache.rocketmq.client.Validators;\nimport org.apache.rocketmq.client.common.ClientErrorCode;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.exception.RequestTimeoutException;\nimport org.apache.rocketmq.client.hook.CheckForbiddenContext;\nimport org.apache.rocketmq.client.hook.CheckForbiddenHook;\nimport org.apache.rocketmq.client.hook.EndTransactionContext;\nimport org.apache.rocketmq.client.hook.EndTransactionHook;\nimport org.apache.rocketmq.client.hook.SendMessageContext;\nimport org.apache.rocketmq.client.hook.SendMessageHook;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.MQClientManager;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.client.latency.MQFaultStrategy;\nimport org.apache.rocketmq.client.latency.Resolver;\nimport org.apache.rocketmq.client.latency.ServiceDetector;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.LocalTransactionState;\nimport org.apache.rocketmq.client.producer.MessageQueueSelector;\nimport org.apache.rocketmq.client.producer.RequestCallback;\nimport org.apache.rocketmq.client.producer.RequestFutureHolder;\nimport org.apache.rocketmq.client.producer.RequestResponseFuture;\nimport org.apache.rocketmq.client.producer.SendCallback;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.client.producer.TransactionCheckListener;\nimport org.apache.rocketmq.client.producer.TransactionListener;\nimport org.apache.rocketmq.client.producer.TransactionMQProducer;\nimport org.apache.rocketmq.client.producer.TransactionSendResult;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ServiceState;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.help.FAQUrl;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageBatch;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageId;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.message.MessageType;\nimport org.apache.rocketmq.common.producer.RecallMessageHandle;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.common.utils.CorrelationIdUtil;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.exception.RemotingConnectException;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class DefaultMQProducerImpl implements MQProducerInner {\n\n    private final Logger log = LoggerFactory.getLogger(DefaultMQProducerImpl.class);\n    private final Random random = new Random();\n    private final DefaultMQProducer defaultMQProducer;\n    private final ConcurrentMap<String/* topic */, TopicPublishInfo> topicPublishInfoTable =\n        new ConcurrentHashMap<>();\n    private final ArrayList<SendMessageHook> sendMessageHookList = new ArrayList<>();\n    private final ArrayList<EndTransactionHook> endTransactionHookList = new ArrayList<>();\n    private final RPCHook rpcHook;\n    private final BlockingQueue<Runnable> asyncSenderThreadPoolQueue;\n    private final ExecutorService defaultAsyncSenderExecutor;\n    protected BlockingQueue<Runnable> checkRequestQueue;\n    protected ExecutorService checkExecutor;\n    private ServiceState serviceState = ServiceState.CREATE_JUST;\n    private MQClientInstance mQClientFactory;\n    private ArrayList<CheckForbiddenHook> checkForbiddenHookList = new ArrayList<>();\n    private MQFaultStrategy mqFaultStrategy;\n    private ExecutorService asyncSenderExecutor;\n\n    // backpressure related\n    private Semaphore semaphoreAsyncSendNum;\n    private Semaphore semaphoreAsyncSendSize;\n\n    public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer) {\n        this(defaultMQProducer, null);\n    }\n\n    public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer, RPCHook rpcHook) {\n        this.defaultMQProducer = defaultMQProducer;\n        this.rpcHook = rpcHook;\n\n        this.asyncSenderThreadPoolQueue = new LinkedBlockingQueue<>(50000);\n        this.defaultAsyncSenderExecutor = new ThreadPoolExecutor(\n            Runtime.getRuntime().availableProcessors(),\n            Runtime.getRuntime().availableProcessors(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            this.asyncSenderThreadPoolQueue,\n            new ThreadFactoryImpl(\"AsyncSenderExecutor_\"));\n        if (defaultMQProducer.getBackPressureForAsyncSendNum() > 10) {\n            semaphoreAsyncSendNum = new Semaphore(Math.max(defaultMQProducer.getBackPressureForAsyncSendNum(), 10), true);\n        } else {\n            semaphoreAsyncSendNum = new Semaphore(10, true);\n            log.info(\"semaphoreAsyncSendNum can not be smaller than 10.\");\n        }\n\n        if (defaultMQProducer.getBackPressureForAsyncSendSize() > 1024 * 1024) {\n            semaphoreAsyncSendSize = new Semaphore(Math.max(defaultMQProducer.getBackPressureForAsyncSendSize(), 1024 * 1024), true);\n        } else {\n            semaphoreAsyncSendSize = new Semaphore(1024 * 1024, true);\n            log.info(\"semaphoreAsyncSendSize can not be smaller than 1M.\");\n        }\n\n        ServiceDetector serviceDetector = new ServiceDetector() {\n            @Override\n            public boolean detect(String endpoint, long timeoutMillis) {\n                Optional<String> candidateTopic = pickTopic();\n                if (!candidateTopic.isPresent()) {\n                    return false;\n                }\n                try {\n                    MessageQueue mq = new MessageQueue(candidateTopic.get(), null, 0);\n                    mQClientFactory.getMQClientAPIImpl()\n                            .getMaxOffset(endpoint, mq, timeoutMillis);\n                    return true;\n                } catch (Exception e) {\n                    return false;\n                }\n            }\n        };\n\n        this.mqFaultStrategy = new MQFaultStrategy(defaultMQProducer.cloneClientConfig(), new Resolver() {\n            @Override\n            public String resolve(String name) {\n                return DefaultMQProducerImpl.this.mQClientFactory.findBrokerAddressInPublish(name);\n            }\n        }, serviceDetector);\n    }\n    private Optional<String> pickTopic() {\n        if (topicPublishInfoTable.isEmpty()) {\n            return Optional.empty();\n        }\n        return Optional.of(topicPublishInfoTable.keySet().iterator().next());\n    }\n    public void registerCheckForbiddenHook(CheckForbiddenHook checkForbiddenHook) {\n        this.checkForbiddenHookList.add(checkForbiddenHook);\n        log.info(\"register a new checkForbiddenHook. hookName={}, allHookSize={}\", checkForbiddenHook.hookName(),\n            checkForbiddenHookList.size());\n    }\n\n    public void setSemaphoreAsyncSendNum(int num) {\n        semaphoreAsyncSendNum = new Semaphore(num, true);\n    }\n\n    public void setSemaphoreAsyncSendSize(int size) {\n        semaphoreAsyncSendSize = new Semaphore(size, true);\n    }\n\n    public int getSemaphoreAsyncSendNumAvailablePermits() {\n        return semaphoreAsyncSendNum == null ? 0 : semaphoreAsyncSendNum.availablePermits();\n    }\n\n    public int getSemaphoreAsyncSendSizeAvailablePermits() {\n        return semaphoreAsyncSendSize == null ? 0 : semaphoreAsyncSendSize.availablePermits();\n    }\n\n    public void initTransactionEnv() {\n        TransactionMQProducer producer = (TransactionMQProducer) this.defaultMQProducer;\n        if (producer.getExecutorService() != null) {\n            this.checkExecutor = producer.getExecutorService();\n        } else {\n            this.checkRequestQueue = new LinkedBlockingQueue<>(producer.getCheckRequestHoldMax());\n            this.checkExecutor = new ThreadPoolExecutor(\n                producer.getCheckThreadPoolMinSize(),\n                producer.getCheckThreadPoolMaxSize(),\n                1000 * 60,\n                TimeUnit.MILLISECONDS,\n                this.checkRequestQueue);\n        }\n    }\n\n    public void destroyTransactionEnv() {\n        if (this.checkExecutor != null) {\n            this.checkExecutor.shutdown();\n        }\n    }\n\n    public void registerSendMessageHook(final SendMessageHook hook) {\n        this.sendMessageHookList.add(hook);\n        log.info(\"register sendMessage Hook, {}\", hook.hookName());\n    }\n\n    public void registerEndTransactionHook(final EndTransactionHook hook) {\n        this.endTransactionHookList.add(hook);\n        log.info(\"register endTransaction Hook, {}\", hook.hookName());\n    }\n\n    public void start() throws MQClientException {\n        this.start(true);\n    }\n\n    public void start(final boolean startFactory) throws MQClientException {\n        switch (this.serviceState) {\n            case CREATE_JUST:\n                this.serviceState = ServiceState.START_FAILED;\n\n                this.checkConfig();\n\n                if (!this.defaultMQProducer.getProducerGroup().equals(MixAll.CLIENT_INNER_PRODUCER_GROUP)) {\n                    this.defaultMQProducer.changeInstanceNameToPID();\n                }\n\n                this.mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultMQProducer, rpcHook);\n\n                defaultMQProducer.initProduceAccumulator();\n\n                boolean registerOK = mQClientFactory.registerProducer(this.defaultMQProducer.getProducerGroup(), this);\n                if (!registerOK) {\n                    this.serviceState = ServiceState.CREATE_JUST;\n                    throw new MQClientException(\"The producer group[\" + this.defaultMQProducer.getProducerGroup()\n                        + \"] has been created before, specify another name please.\" + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL),\n                        null);\n                }\n\n                if (startFactory) {\n                    mQClientFactory.start();\n                }\n\n                this.initTopicRoute();\n\n                this.mqFaultStrategy.startDetector();\n\n                log.info(\"the producer [{}] start OK. sendMessageWithVIPChannel={}\", this.defaultMQProducer.getProducerGroup(),\n                    this.defaultMQProducer.isSendMessageWithVIPChannel());\n                this.serviceState = ServiceState.RUNNING;\n                break;\n            case RUNNING:\n            case START_FAILED:\n            case SHUTDOWN_ALREADY:\n                throw new MQClientException(\"The producer service state not OK, maybe started once, \"\n                    + this.serviceState\n                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK),\n                    null);\n            default:\n                break;\n        }\n\n        this.mQClientFactory.sendHeartbeatToAllBrokerWithLock();\n\n        RequestFutureHolder.getInstance().startScheduledTask(this);\n\n    }\n\n    private void checkConfig() throws MQClientException {\n        Validators.checkGroup(this.defaultMQProducer.getProducerGroup());\n\n        if (this.defaultMQProducer.getProducerGroup().equals(MixAll.DEFAULT_PRODUCER_GROUP)) {\n            throw new MQClientException(\"producerGroup can not equal \" + MixAll.DEFAULT_PRODUCER_GROUP + \", please specify another one.\",\n                null);\n        }\n    }\n\n    public void shutdown() {\n        this.shutdown(true);\n    }\n\n    public void shutdown(final boolean shutdownFactory) {\n        switch (this.serviceState) {\n            case CREATE_JUST:\n                break;\n            case RUNNING:\n                this.mQClientFactory.unregisterProducer(this.defaultMQProducer.getProducerGroup());\n                this.defaultAsyncSenderExecutor.shutdown();\n                if (shutdownFactory) {\n                    this.mQClientFactory.shutdown();\n                }\n                this.mqFaultStrategy.shutdown();\n                RequestFutureHolder.getInstance().shutdown(this);\n                log.info(\"the producer [{}] shutdown OK\", this.defaultMQProducer.getProducerGroup());\n                this.serviceState = ServiceState.SHUTDOWN_ALREADY;\n                break;\n            case SHUTDOWN_ALREADY:\n                break;\n            default:\n                break;\n        }\n    }\n\n    @Override\n    public Set<String> getPublishTopicList() {\n        return new HashSet<>(this.topicPublishInfoTable.keySet());\n    }\n\n    @Override\n    public boolean isPublishTopicNeedUpdate(String topic) {\n        TopicPublishInfo prev = this.topicPublishInfoTable.get(topic);\n\n        return null == prev || !prev.ok();\n    }\n\n    /**\n     * @deprecated This method will be removed in the version 5.0.0 and {@link DefaultMQProducerImpl#getCheckListener} is recommended.\n     */\n    @Override\n    @Deprecated\n    public TransactionCheckListener checkListener() {\n        if (this.defaultMQProducer instanceof TransactionMQProducer) {\n            TransactionMQProducer producer = (TransactionMQProducer) defaultMQProducer;\n            return producer.getTransactionCheckListener();\n        }\n\n        return null;\n    }\n\n    @Override\n    public TransactionListener getCheckListener() {\n        if (this.defaultMQProducer instanceof TransactionMQProducer) {\n            TransactionMQProducer producer = (TransactionMQProducer) defaultMQProducer;\n            return producer.getTransactionListener();\n        }\n        return null;\n    }\n\n    @Override\n    public void checkTransactionState(final String addr, final MessageExt msg,\n        final CheckTransactionStateRequestHeader header) {\n        Runnable request = new Runnable() {\n            private final String brokerAddr = addr;\n            private final MessageExt message = msg;\n            private final CheckTransactionStateRequestHeader checkRequestHeader = header;\n            private final String group = DefaultMQProducerImpl.this.defaultMQProducer.getProducerGroup();\n\n            @Override\n            public void run() {\n                TransactionCheckListener transactionCheckListener = DefaultMQProducerImpl.this.checkListener();\n                TransactionListener transactionListener = getCheckListener();\n                if (transactionCheckListener != null || transactionListener != null) {\n                    LocalTransactionState localTransactionState = LocalTransactionState.UNKNOW;\n                    Throwable exception = null;\n                    try {\n                        if (transactionCheckListener != null) {\n                            localTransactionState = transactionCheckListener.checkLocalTransactionState(message);\n                        } else {\n                            log.debug(\"TransactionCheckListener is null, used new check API, producerGroup={}\", group);\n                            localTransactionState = transactionListener.checkLocalTransaction(message);\n                        }\n                    } catch (Throwable e) {\n                        log.error(\"Broker call checkTransactionState, but checkLocalTransactionState exception\", e);\n                        exception = e;\n                    }\n\n                    this.processTransactionState(\n                        checkRequestHeader.getTopic(),\n                        localTransactionState,\n                        group,\n                        exception);\n                } else {\n                    log.warn(\"CheckTransactionState, pick transactionCheckListener by group[{}] failed\", group);\n                }\n            }\n\n            private void processTransactionState(\n                final String topic,\n                final LocalTransactionState localTransactionState,\n                final String producerGroup,\n                final Throwable exception) {\n                final EndTransactionRequestHeader thisHeader = new EndTransactionRequestHeader();\n                thisHeader.setTopic(topic);\n                thisHeader.setCommitLogOffset(checkRequestHeader.getCommitLogOffset());\n                thisHeader.setProducerGroup(producerGroup);\n                thisHeader.setTranStateTableOffset(checkRequestHeader.getTranStateTableOffset());\n                thisHeader.setFromTransactionCheck(true);\n                thisHeader.setBrokerName(checkRequestHeader.getBrokerName());\n\n                String uniqueKey = message.getProperties().get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX);\n                if (uniqueKey == null) {\n                    uniqueKey = message.getMsgId();\n                }\n                thisHeader.setMsgId(uniqueKey);\n                thisHeader.setTransactionId(checkRequestHeader.getTransactionId());\n                switch (localTransactionState) {\n                    case COMMIT_MESSAGE:\n                        thisHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_COMMIT_TYPE);\n                        break;\n                    case ROLLBACK_MESSAGE:\n                        thisHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE);\n                        log.warn(\"when broker check, client rollback this transaction, {}\", thisHeader);\n                        break;\n                    case UNKNOW:\n                        thisHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_NOT_TYPE);\n                        log.warn(\"when broker check, client does not know this transaction state, {}\", thisHeader);\n                        break;\n                    default:\n                        break;\n                }\n\n                String remark = null;\n                if (exception != null) {\n                    remark = \"checkLocalTransactionState Exception: \" + UtilAll.exceptionSimpleDesc(exception);\n                }\n                doExecuteEndTransactionHook(msg, uniqueKey, brokerAddr, localTransactionState, true);\n\n                try {\n                    DefaultMQProducerImpl.this.mQClientFactory.getMQClientAPIImpl().endTransactionOneway(brokerAddr, thisHeader, remark,\n                        3000);\n                } catch (Exception e) {\n                    log.error(\"endTransactionOneway exception\", e);\n                }\n            }\n        };\n\n        this.checkExecutor.submit(request);\n    }\n\n    @Override\n    public void updateTopicPublishInfo(final String topic, final TopicPublishInfo info) {\n        if (info != null && topic != null) {\n            TopicPublishInfo prev = this.topicPublishInfoTable.put(topic, info);\n            if (prev != null) {\n                log.info(\"updateTopicPublishInfo prev is not null, \" + prev);\n            }\n        }\n    }\n\n    @Override\n    public boolean isUnitMode() {\n        return this.defaultMQProducer.isUnitMode();\n    }\n\n    public void createTopic(String key, String newTopic, int queueNum) throws MQClientException {\n        createTopic(key, newTopic, queueNum, 0);\n    }\n\n    public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) throws MQClientException {\n        this.makeSureStateOK();\n        Validators.checkTopic(newTopic);\n        Validators.isSystemTopic(newTopic);\n\n        this.mQClientFactory.getMQAdminImpl().createTopic(key, newTopic, queueNum, topicSysFlag, null);\n    }\n\n    private void makeSureStateOK() throws MQClientException {\n        if (this.serviceState != ServiceState.RUNNING) {\n            throw new MQClientException(\"The producer service state not OK, \"\n                + this.serviceState\n                + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK),\n                null);\n        }\n    }\n\n    public List<MessageQueue> fetchPublishMessageQueues(String topic) throws MQClientException {\n        this.makeSureStateOK();\n        return this.mQClientFactory.getMQAdminImpl().fetchPublishMessageQueues(topic);\n    }\n\n    public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException {\n        this.makeSureStateOK();\n        return this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp);\n    }\n\n    public long maxOffset(MessageQueue mq) throws MQClientException {\n        this.makeSureStateOK();\n        return this.mQClientFactory.getMQAdminImpl().maxOffset(mq);\n    }\n\n    public long minOffset(MessageQueue mq) throws MQClientException {\n        this.makeSureStateOK();\n        return this.mQClientFactory.getMQAdminImpl().minOffset(mq);\n    }\n\n    public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException {\n        this.makeSureStateOK();\n        return this.mQClientFactory.getMQAdminImpl().earliestMsgStoreTime(mq);\n    }\n\n    public MessageExt viewMessage(String topic,\n        String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {\n        this.makeSureStateOK();\n\n        return this.mQClientFactory.getMQAdminImpl().viewMessage(topic, msgId);\n    }\n\n    public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end)\n        throws MQClientException, InterruptedException {\n        this.makeSureStateOK();\n        return this.mQClientFactory.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end);\n    }\n\n    public MessageExt queryMessageByUniqKey(String topic, String uniqKey)\n        throws MQClientException, InterruptedException {\n        this.makeSureStateOK();\n        return this.mQClientFactory.getMQAdminImpl().queryMessageByUniqKey(topic, uniqKey);\n    }\n\n    /**\n     * DEFAULT ASYNC -------------------------------------------------------\n     */\n    public void send(Message msg,\n        SendCallback sendCallback) throws MQClientException, RemotingException, InterruptedException {\n        send(msg, sendCallback, this.defaultMQProducer.getSendMsgTimeout());\n    }\n\n    /**\n     * @param msg\n     * @param sendCallback\n     * @param timeout      the <code>sendCallback</code> will be invoked at most time\n     * @throws RejectedExecutionException\n     * @deprecated It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. A new one will be\n     * provided in next version\n     */\n    @Deprecated\n    public void send(final Message msg, final SendCallback sendCallback, final long timeout)\n        throws MQClientException, RemotingException, InterruptedException {\n        BackpressureSendCallBack newCallBack = new BackpressureSendCallBack(sendCallback);\n\n        final long beginStartTime = System.currentTimeMillis();\n        Runnable runnable = new Runnable() {\n            @Override\n            public void run() {\n                long costTime = System.currentTimeMillis() - beginStartTime;\n                if (timeout > costTime) {\n                    try {\n                        sendDefaultImpl(msg, CommunicationMode.ASYNC, newCallBack, timeout - costTime);\n                    } catch (Exception e) {\n                        newCallBack.onException(e);\n                    }\n                } else {\n                    newCallBack.onException(\n                        new RemotingTooMuchRequestException(\"DEFAULT ASYNC send call timeout\"));\n                }\n            }\n        };\n        executeAsyncMessageSend(runnable, msg, newCallBack, timeout, beginStartTime);\n    }\n\n    class BackpressureSendCallBack implements SendCallback {\n        public boolean isSemaphoreAsyncSizeAcquired = false;\n        public boolean isSemaphoreAsyncNumAcquired = false;\n        public int msgLen;\n        private final SendCallback sendCallback;\n\n        public BackpressureSendCallBack(final SendCallback sendCallback) {\n            this.sendCallback = sendCallback;\n        }\n\n        @Override\n        public void onSuccess(SendResult sendResult) {\n            semaphoreProcessor();\n            sendCallback.onSuccess(sendResult);\n        }\n\n        @Override\n        public void onException(Throwable e) {\n            semaphoreProcessor();\n            sendCallback.onException(e);\n        }\n\n        public void semaphoreProcessor() {\n            if (isSemaphoreAsyncSizeAcquired) {\n                defaultMQProducer.acquireBackPressureForAsyncSendSizeLock();\n                semaphoreAsyncSendSize.release(msgLen);\n                defaultMQProducer.releaseBackPressureForAsyncSendSizeLock();\n            }\n            if (isSemaphoreAsyncNumAcquired) {\n                defaultMQProducer.acquireBackPressureForAsyncSendNumLock();\n                semaphoreAsyncSendNum.release();\n                defaultMQProducer.releaseBackPressureForAsyncSendNumLock();\n            }\n        }\n\n        public void semaphoreAsyncAdjust(int semaphoreAsyncNum, int semaphoreAsyncSize) throws InterruptedException {\n            defaultMQProducer.acquireBackPressureForAsyncSendNumLock();\n            if (semaphoreAsyncNum > 0) {\n                semaphoreAsyncSendNum.release(semaphoreAsyncNum);\n            } else {\n                semaphoreAsyncSendNum.acquire(- semaphoreAsyncNum);\n            }\n            defaultMQProducer.setBackPressureForAsyncSendNumInsideAdjust(defaultMQProducer.getBackPressureForAsyncSendNum()\n                    + semaphoreAsyncNum);\n            defaultMQProducer.releaseBackPressureForAsyncSendNumLock();\n\n            defaultMQProducer.acquireBackPressureForAsyncSendSizeLock();\n            if (semaphoreAsyncSize > 0) {\n                semaphoreAsyncSendSize.release(semaphoreAsyncSize);\n            } else {\n                semaphoreAsyncSendSize.acquire(- semaphoreAsyncSize);\n            }\n            defaultMQProducer.setBackPressureForAsyncSendSizeInsideAdjust(defaultMQProducer.getBackPressureForAsyncSendSize()\n                    + semaphoreAsyncSize);\n            defaultMQProducer.releaseBackPressureForAsyncSendSizeLock();\n        }\n    }\n\n    public void executeAsyncMessageSend(Runnable runnable, final Message msg, final BackpressureSendCallBack sendCallback,\n        final long timeout, final long beginStartTime)\n        throws MQClientException, InterruptedException {\n        ExecutorService executor = this.getAsyncSenderExecutor();\n        boolean isEnableBackpressureForAsyncMode = this.getDefaultMQProducer().isEnableBackpressureForAsyncMode();\n        boolean isSemaphoreAsyncNumAcquired = false;\n        boolean isSemaphoreAsyncSizeAcquired = false;\n        int msgLen = msg.getBody() == null ? 1 : msg.getBody().length;\n        sendCallback.msgLen = msgLen;\n\n        try {\n            if (isEnableBackpressureForAsyncMode) {\n                defaultMQProducer.acquireBackPressureForAsyncSendNumLock();\n                long costTime = System.currentTimeMillis() - beginStartTime;\n\n                isSemaphoreAsyncNumAcquired = timeout - costTime > 0\n                    && semaphoreAsyncSendNum.tryAcquire(timeout - costTime, TimeUnit.MILLISECONDS);\n                sendCallback.isSemaphoreAsyncNumAcquired = isSemaphoreAsyncNumAcquired;\n                defaultMQProducer.releaseBackPressureForAsyncSendNumLock();\n                if (!isSemaphoreAsyncNumAcquired) {\n                    sendCallback.onException(\n                        new RemotingTooMuchRequestException(\"send message tryAcquire semaphoreAsyncNum timeout\"));\n                    return;\n                }\n\n                defaultMQProducer.acquireBackPressureForAsyncSendSizeLock();\n                costTime = System.currentTimeMillis() - beginStartTime;\n\n                isSemaphoreAsyncSizeAcquired = timeout - costTime > 0\n                    && semaphoreAsyncSendSize.tryAcquire(msgLen, timeout - costTime, TimeUnit.MILLISECONDS);\n                sendCallback.isSemaphoreAsyncSizeAcquired = isSemaphoreAsyncSizeAcquired;\n                defaultMQProducer.releaseBackPressureForAsyncSendSizeLock();\n                if (!isSemaphoreAsyncSizeAcquired) {\n                    sendCallback.onException(\n                        new RemotingTooMuchRequestException(\"send message tryAcquire semaphoreAsyncSize timeout\"));\n                    return;\n                }\n            }\n\n            executor.submit(runnable);\n        } catch (RejectedExecutionException e) {\n            if (isEnableBackpressureForAsyncMode) {\n                runnable.run();\n            } else {\n                throw new MQClientException(\"executor rejected \", e);\n            }\n        }\n    }\n\n    public MessageQueue invokeMessageQueueSelector(Message msg, MessageQueueSelector selector, Object arg,\n                                                   final long timeout) throws MQClientException, RemotingTooMuchRequestException {\n        long beginStartTime = System.currentTimeMillis();\n        this.makeSureStateOK();\n        Validators.checkMessage(msg, this.defaultMQProducer);\n\n        TopicPublishInfo topicPublishInfo = this.tryToFindTopicPublishInfo(msg.getTopic());\n        if (topicPublishInfo != null && topicPublishInfo.ok()) {\n            MessageQueue mq = null;\n            try {\n                List<MessageQueue> messageQueueList =\n                        mQClientFactory.getMQAdminImpl().parsePublishMessageQueues(topicPublishInfo.getMessageQueueList());\n                Message userMessage = MessageAccessor.cloneMessage(msg);\n                String userTopic = NamespaceUtil.withoutNamespace(userMessage.getTopic(), mQClientFactory.getClientConfig().getNamespace());\n                userMessage.setTopic(userTopic);\n\n                mq = mQClientFactory.getClientConfig().queueWithNamespace(selector.select(messageQueueList, userMessage, arg));\n            } catch (Throwable e) {\n                throw new MQClientException(\"select message queue threw exception.\", e);\n            }\n\n            long costTime = System.currentTimeMillis() - beginStartTime;\n            if (timeout < costTime) {\n                throw new RemotingTooMuchRequestException(\"sendSelectImpl call timeout\");\n            }\n            if (mq != null) {\n                return mq;\n            } else {\n                throw new MQClientException(\"select message queue return null.\", null);\n            }\n        }\n\n        validateNameServerSetting();\n        throw new MQClientException(\"No route info for this topic, \" + msg.getTopic(), null);\n    }\n\n    public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName, final boolean resetIndex) {\n        return this.mqFaultStrategy.selectOneMessageQueue(tpInfo, lastBrokerName, resetIndex);\n    }\n\n    public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation,\n                                boolean reachable) {\n        this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation, reachable);\n    }\n\n    private void validateNameServerSetting() throws MQClientException {\n        List<String> nsList = this.getMqClientFactory().getMQClientAPIImpl().getNameServerAddressList();\n        if (null == nsList || nsList.isEmpty()) {\n            throw new MQClientException(\n                \"No name server address, please set it.\" + FAQUrl.suggestTodo(FAQUrl.NAME_SERVER_ADDR_NOT_EXIST_URL), null).setResponseCode(ClientErrorCode.NO_NAME_SERVER_EXCEPTION);\n        }\n\n    }\n\n    private SendResult sendDefaultImpl(\n        Message msg,\n        final CommunicationMode communicationMode,\n        final SendCallback sendCallback,\n        final long timeout\n    ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        this.makeSureStateOK();\n        Validators.checkMessage(msg, this.defaultMQProducer);\n        final long invokeID = random.nextLong();\n        long beginTimestampFirst = System.currentTimeMillis();\n        long beginTimestampPrev = beginTimestampFirst;\n        long endTimestamp = beginTimestampFirst;\n        TopicPublishInfo topicPublishInfo = this.tryToFindTopicPublishInfo(msg.getTopic());\n        if (topicPublishInfo != null && topicPublishInfo.ok()) {\n            boolean callTimeout = false;\n            MessageQueue mq = null;\n            Exception exception = null;\n            SendResult sendResult = null;\n            int timesTotal = communicationMode == CommunicationMode.SYNC ? 1 + this.defaultMQProducer.getRetryTimesWhenSendFailed() : 1;\n            int times = 0;\n            String[] brokersSent = new String[timesTotal];\n            boolean resetIndex = false;\n            for (; times < timesTotal; times++) {\n                String lastBrokerName = null == mq ? null : mq.getBrokerName();\n                if (times > 0) {\n                    resetIndex = true;\n                }\n                MessageQueue mqSelected = this.selectOneMessageQueue(topicPublishInfo, lastBrokerName, resetIndex);\n                if (mqSelected != null) {\n                    mq = mqSelected;\n                    brokersSent[times] = mq.getBrokerName();\n                    try {\n                        beginTimestampPrev = System.currentTimeMillis();\n                        if (times > 0) {\n                            //Reset topic with namespace during resend.\n                            msg.setTopic(this.defaultMQProducer.withNamespace(msg.getTopic()));\n                        }\n                        long costTime = beginTimestampPrev - beginTimestampFirst;\n                        if (timeout < costTime) {\n                            callTimeout = true;\n                            break;\n                        }\n                        long curTimeout = timeout - costTime;\n                        // Get the maximum timeout allowed per request\n                        long maxSendTimeoutPerRequest = defaultMQProducer.getSendMsgMaxTimeoutPerRequest();\n                        // Determine if retries are still possible\n                        boolean canRetryAgain = times + 1 < timesTotal;\n                        // If retries are possible, and the current timeout exceeds the max allowed timeout, set the current timeout to the max allowed\n                        if (maxSendTimeoutPerRequest > -1 && canRetryAgain && curTimeout > maxSendTimeoutPerRequest) {\n                            curTimeout = maxSendTimeoutPerRequest;\n                        }\n                        sendResult = this.sendKernelImpl(msg, mq, communicationMode, sendCallback, topicPublishInfo, curTimeout);\n                        endTimestamp = System.currentTimeMillis();\n                        this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false, true);\n                        switch (communicationMode) {\n                            case ASYNC:\n                                return null;\n                            case ONEWAY:\n                                return null;\n                            case SYNC:\n                                if (sendResult.getSendStatus() != SendStatus.SEND_OK) {\n                                    if (this.defaultMQProducer.isRetryAnotherBrokerWhenNotStoreOK()) {\n                                        continue;\n                                    }\n                                }\n\n                                return sendResult;\n                            default:\n                                break;\n                        }\n                    } catch (MQClientException e) {\n                        endTimestamp = System.currentTimeMillis();\n                        this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false, true);\n                        log.warn(\"sendKernelImpl exception, resend at once, InvokeID: {}, RT: {}ms, Broker: {}\", invokeID, endTimestamp - beginTimestampPrev, mq, e);\n                        if (log.isDebugEnabled()) {\n                            log.debug(msg.toString());\n                        }\n                        exception = e;\n                        continue;\n                    } catch (RemotingException e) {\n                        endTimestamp = System.currentTimeMillis();\n                        // Set this broker unreachable when detecting schedule task is running for RemotingException.\n                        // Otherwise, isolate this broker.\n                        this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true, !this.mqFaultStrategy.isStartDetectorEnable());\n                        log.warn(\"sendKernelImpl exception, resend at once, InvokeID: {}, RT: {}ms, Broker: {}\", invokeID, endTimestamp - beginTimestampPrev, mq, e);\n                        if (log.isDebugEnabled()) {\n                            log.debug(msg.toString());\n                        }\n                        exception = e;\n                        continue;\n                    } catch (MQBrokerException e) {\n                        endTimestamp = System.currentTimeMillis();\n                        this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true, false);\n                        log.warn(\"sendKernelImpl exception, resend at once, InvokeID: {}, RT: {}ms, Broker: {}\", invokeID, endTimestamp - beginTimestampPrev, mq, e);\n                        if (log.isDebugEnabled()) {\n                            log.debug(msg.toString());\n                        }\n                        exception = e;\n                        if (this.defaultMQProducer.getRetryResponseCodes().contains(e.getResponseCode())) {\n                            continue;\n                        } else {\n                            if (sendResult != null) {\n                                return sendResult;\n                            }\n\n                            throw e;\n                        }\n                    } catch (InterruptedException e) {\n                        endTimestamp = System.currentTimeMillis();\n                        this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false, true);\n                        log.warn(\"sendKernelImpl exception, throw exception, InvokeID: {}, RT: {}ms, Broker: {}\", invokeID, endTimestamp - beginTimestampPrev, mq, e);\n                        if (log.isDebugEnabled()) {\n                            log.debug(msg.toString());\n                        }\n                        throw e;\n                    }\n                } else {\n                    break;\n                }\n            }\n\n            if (sendResult != null) {\n                return sendResult;\n            }\n            String info = String.format(\"Send [%d] times, still failed, cost [%d]ms, Topic: %s, BrokersSent: %s\",\n                times,\n                System.currentTimeMillis() - beginTimestampFirst,\n                msg.getTopic(),\n                Arrays.toString(brokersSent));\n\n            info += FAQUrl.suggestTodo(FAQUrl.SEND_MSG_FAILED);\n\n            MQClientException mqClientException = new MQClientException(info, exception);\n            if (callTimeout) {\n                throw new RemotingTooMuchRequestException(\"sendDefaultImpl call timeout\");\n            }\n\n            if (exception instanceof MQBrokerException) {\n                mqClientException.setResponseCode(((MQBrokerException) exception).getResponseCode());\n            } else if (exception instanceof RemotingConnectException) {\n                mqClientException.setResponseCode(ClientErrorCode.CONNECT_BROKER_EXCEPTION);\n            } else if (exception instanceof RemotingTimeoutException) {\n                mqClientException.setResponseCode(ClientErrorCode.ACCESS_BROKER_TIMEOUT);\n            } else if (exception instanceof MQClientException) {\n                mqClientException.setResponseCode(ClientErrorCode.BROKER_NOT_EXIST_EXCEPTION);\n            }\n\n            throw mqClientException;\n        }\n\n        validateNameServerSetting();\n\n        throw new MQClientException(\"No route info of this topic: \" + msg.getTopic() + FAQUrl.suggestTodo(FAQUrl.NO_TOPIC_ROUTE_INFO),\n            null).setResponseCode(ClientErrorCode.NOT_FOUND_TOPIC_EXCEPTION);\n    }\n\n    private TopicPublishInfo tryToFindTopicPublishInfo(final String topic) {\n        TopicPublishInfo topicPublishInfo = this.topicPublishInfoTable.get(topic);\n        if (null == topicPublishInfo || !topicPublishInfo.ok()) {\n            this.topicPublishInfoTable.putIfAbsent(topic, new TopicPublishInfo());\n            this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic);\n            topicPublishInfo = this.topicPublishInfoTable.get(topic);\n        }\n\n        if (topicPublishInfo.isHaveTopicRouterInfo() || topicPublishInfo.ok()) {\n            return topicPublishInfo;\n        } else {\n            this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic, true, this.defaultMQProducer);\n            topicPublishInfo = this.topicPublishInfoTable.get(topic);\n            return topicPublishInfo;\n        }\n    }\n\n    private SendResult sendKernelImpl(final Message msg,\n        final MessageQueue mq,\n        final CommunicationMode communicationMode,\n        final SendCallback sendCallback,\n        final TopicPublishInfo topicPublishInfo,\n        final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        long beginStartTime = System.currentTimeMillis();\n        String brokerName = this.mQClientFactory.getBrokerNameFromMessageQueue(mq);\n        String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(brokerName);\n        if (null == brokerAddr) {\n            tryToFindTopicPublishInfo(mq.getTopic());\n            brokerName = this.mQClientFactory.getBrokerNameFromMessageQueue(mq);\n            brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(brokerName);\n        }\n\n        SendMessageContext context = null;\n        if (brokerAddr != null) {\n            brokerAddr = MixAll.brokerVIPChannel(this.defaultMQProducer.isSendMessageWithVIPChannel(), brokerAddr);\n\n            byte[] prevBody = msg.getBody();\n            try {\n                //for MessageBatch,ID has been set in the generating process\n                if (!(msg instanceof MessageBatch)) {\n                    MessageClientIDSetter.setUniqID(msg);\n                }\n\n                boolean topicWithNamespace = false;\n                if (null != this.mQClientFactory.getClientConfig().getNamespace()) {\n                    msg.setInstanceId(this.mQClientFactory.getClientConfig().getNamespace());\n                    topicWithNamespace = true;\n                }\n\n                int sysFlag = 0;\n                boolean msgBodyCompressed = false;\n                if (this.tryToCompressMessage(msg)) {\n                    sysFlag |= MessageSysFlag.COMPRESSED_FLAG;\n                    sysFlag |= this.defaultMQProducer.getCompressType().getCompressionFlag();\n                    msgBodyCompressed = true;\n                }\n\n                final String tranMsg = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED);\n                if (Boolean.parseBoolean(tranMsg)) {\n                    sysFlag |= MessageSysFlag.TRANSACTION_PREPARED_TYPE;\n                }\n\n                if (hasCheckForbiddenHook()) {\n                    CheckForbiddenContext checkForbiddenContext = new CheckForbiddenContext();\n                    checkForbiddenContext.setNameSrvAddr(this.defaultMQProducer.getNamesrvAddr());\n                    checkForbiddenContext.setGroup(this.defaultMQProducer.getProducerGroup());\n                    checkForbiddenContext.setCommunicationMode(communicationMode);\n                    checkForbiddenContext.setBrokerAddr(brokerAddr);\n                    checkForbiddenContext.setMessage(msg);\n                    checkForbiddenContext.setMq(mq);\n                    checkForbiddenContext.setUnitMode(this.isUnitMode());\n                    this.executeCheckForbiddenHook(checkForbiddenContext);\n                }\n\n                if (this.hasSendMessageHook()) {\n                    context = new SendMessageContext();\n                    context.setProducer(this);\n                    context.setProducerGroup(this.defaultMQProducer.getProducerGroup());\n                    context.setCommunicationMode(communicationMode);\n                    context.setBornHost(this.defaultMQProducer.getClientIP());\n                    context.setBrokerAddr(brokerAddr);\n                    context.setMessage(msg);\n                    context.setMq(mq);\n                    context.setNamespace(this.defaultMQProducer.getNamespace());\n                    String isTrans = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED);\n                    if (isTrans != null && isTrans.equals(\"true\")) {\n                        context.setMsgType(MessageType.Trans_Msg_Half);\n                    }\n\n                    if (msg.getProperty(\"__STARTDELIVERTIME\") != null\n                            || msg.getProperty(MessageConst.PROPERTY_DELAY_TIME_LEVEL) != null\n                            || msg.getProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS) != null\n                            || msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC) != null\n                            || msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_MS) != null) {\n                        context.setMsgType(MessageType.Delay_Msg);\n                    }\n                    this.executeSendMessageHookBefore(context);\n                }\n\n                SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();\n                requestHeader.setProducerGroup(this.defaultMQProducer.getProducerGroup());\n                requestHeader.setTopic(msg.getTopic());\n                requestHeader.setDefaultTopic(this.defaultMQProducer.getCreateTopicKey());\n                requestHeader.setDefaultTopicQueueNums(this.defaultMQProducer.getDefaultTopicQueueNums());\n                requestHeader.setQueueId(mq.getQueueId());\n                requestHeader.setSysFlag(sysFlag);\n                requestHeader.setBornTimestamp(System.currentTimeMillis());\n                requestHeader.setFlag(msg.getFlag());\n                requestHeader.setProperties(MessageDecoder.messageProperties2String(msg.getProperties()));\n                requestHeader.setReconsumeTimes(0);\n                requestHeader.setUnitMode(this.isUnitMode());\n                requestHeader.setBatch(msg instanceof MessageBatch);\n                requestHeader.setBrokerName(brokerName);\n                if (requestHeader.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n                    String reconsumeTimes = MessageAccessor.getReconsumeTime(msg);\n                    if (reconsumeTimes != null) {\n                        requestHeader.setReconsumeTimes(Integer.valueOf(reconsumeTimes));\n                        MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_RECONSUME_TIME);\n                    }\n\n                    String maxReconsumeTimes = MessageAccessor.getMaxReconsumeTimes(msg);\n                    if (maxReconsumeTimes != null) {\n                        requestHeader.setMaxReconsumeTimes(Integer.valueOf(maxReconsumeTimes));\n                        MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_MAX_RECONSUME_TIMES);\n                    }\n                }\n\n                SendResult sendResult = null;\n                switch (communicationMode) {\n                    case ASYNC:\n                        Message tmpMessage = msg;\n                        boolean messageCloned = false;\n                        if (msgBodyCompressed) {\n                            //If msg body was compressed, msgbody should be reset using prevBody.\n                            //Clone new message using compressed message body and recover origin massage.\n                            //Fix bug:https://github.com/apache/rocketmq-externals/issues/66\n                            tmpMessage = MessageAccessor.cloneMessage(msg);\n                            messageCloned = true;\n                            msg.setBody(prevBody);\n                        }\n\n                        if (topicWithNamespace) {\n                            if (!messageCloned) {\n                                tmpMessage = MessageAccessor.cloneMessage(msg);\n                                messageCloned = true;\n                            }\n                            msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQProducer.getNamespace()));\n                        }\n\n                        long costTimeAsync = System.currentTimeMillis() - beginStartTime;\n                        if (timeout < costTimeAsync) {\n                            throw new RemotingTooMuchRequestException(\"sendKernelImpl call timeout\");\n                        }\n                        sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage(\n                            brokerAddr,\n                            brokerName,\n                            tmpMessage,\n                            requestHeader,\n                            timeout - costTimeAsync,\n                            communicationMode,\n                            sendCallback,\n                            topicPublishInfo,\n                            this.mQClientFactory,\n                            this.defaultMQProducer.getRetryTimesWhenSendAsyncFailed(),\n                            context,\n                            this);\n                        break;\n                    case ONEWAY:\n                    case SYNC:\n                        long costTimeSync = System.currentTimeMillis() - beginStartTime;\n                        if (timeout < costTimeSync) {\n                            throw new RemotingTooMuchRequestException(\"sendKernelImpl call timeout\");\n                        }\n                        sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage(\n                            brokerAddr,\n                            brokerName,\n                            msg,\n                            requestHeader,\n                            timeout - costTimeSync,\n                            communicationMode,\n                            context,\n                            this);\n                        break;\n                    default:\n                        assert false;\n                        break;\n                }\n\n                if (this.hasSendMessageHook()) {\n                    context.setSendResult(sendResult);\n                    this.executeSendMessageHookAfter(context);\n                }\n\n                return sendResult;\n            } catch (RemotingException | InterruptedException | MQBrokerException e) {\n                if (this.hasSendMessageHook()) {\n                    context.setException(e);\n                    this.executeSendMessageHookAfter(context);\n                }\n                throw e;\n            } finally {\n                msg.setBody(prevBody);\n                msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQProducer.getNamespace()));\n            }\n        }\n\n        throw new MQClientException(\"The broker[\" + brokerName + \"] not exist\", null);\n    }\n\n    public MQClientInstance getMqClientFactory() {\n        return mQClientFactory;\n    }\n\n    @Deprecated\n    public MQClientInstance getmQClientFactory() {\n        return mQClientFactory;\n    }\n\n    private boolean tryToCompressMessage(final Message msg) {\n        if (msg instanceof MessageBatch) {\n            //batch does not support compressing right now\n            return false;\n        }\n        byte[] body = msg.getBody();\n        if (body != null) {\n            if (body.length >= this.defaultMQProducer.getCompressMsgBodyOverHowmuch()) {\n                try {\n                    byte[] data = this.defaultMQProducer.getCompressor().compress(body, this.defaultMQProducer.getCompressLevel());\n                    if (data != null) {\n                        msg.setBody(data);\n                        return true;\n                    }\n                } catch (IOException e) {\n                    log.error(\"tryToCompressMessage exception\", e);\n                    if (log.isDebugEnabled()) {\n                        log.debug(msg.toString());\n                    }\n                }\n            }\n        }\n\n        return false;\n    }\n\n    public boolean hasCheckForbiddenHook() {\n        return !checkForbiddenHookList.isEmpty();\n    }\n\n    public void executeCheckForbiddenHook(final CheckForbiddenContext context) throws MQClientException {\n        if (hasCheckForbiddenHook()) {\n            for (CheckForbiddenHook hook : checkForbiddenHookList) {\n                hook.checkForbidden(context);\n            }\n        }\n    }\n\n    public boolean hasSendMessageHook() {\n        return !this.sendMessageHookList.isEmpty();\n    }\n\n    public void executeSendMessageHookBefore(final SendMessageContext context) {\n        if (!this.sendMessageHookList.isEmpty()) {\n            for (SendMessageHook hook : this.sendMessageHookList) {\n                try {\n                    hook.sendMessageBefore(context);\n                } catch (Throwable e) {\n                    log.warn(\"failed to executeSendMessageHookBefore\", e);\n                }\n            }\n        }\n    }\n\n    public void executeSendMessageHookAfter(final SendMessageContext context) {\n        if (!this.sendMessageHookList.isEmpty()) {\n            for (SendMessageHook hook : this.sendMessageHookList) {\n                try {\n                    hook.sendMessageAfter(context);\n                } catch (Throwable e) {\n                    log.warn(\"failed to executeSendMessageHookAfter\", e);\n                }\n            }\n        }\n    }\n\n    public boolean hasEndTransactionHook() {\n        return !this.endTransactionHookList.isEmpty();\n    }\n\n    public void executeEndTransactionHook(final EndTransactionContext context) {\n        if (!this.endTransactionHookList.isEmpty()) {\n            for (EndTransactionHook hook : this.endTransactionHookList) {\n                try {\n                    hook.endTransaction(context);\n                } catch (Throwable e) {\n                    log.warn(\"failed to executeEndTransactionHook\", e);\n                }\n            }\n        }\n    }\n\n    public void doExecuteEndTransactionHook(Message msg, String msgId, String brokerAddr, LocalTransactionState state,\n        boolean fromTransactionCheck) {\n        if (hasEndTransactionHook()) {\n            EndTransactionContext context = new EndTransactionContext();\n            context.setProducerGroup(defaultMQProducer.getProducerGroup());\n            context.setBrokerAddr(brokerAddr);\n            context.setMessage(msg);\n            context.setMsgId(msgId);\n            context.setTransactionId(msg.getTransactionId());\n            context.setTransactionState(state);\n            context.setFromTransactionCheck(fromTransactionCheck);\n            executeEndTransactionHook(context);\n        }\n    }\n\n    /**\n     * DEFAULT ONEWAY -------------------------------------------------------\n     */\n    public void sendOneway(Message msg) throws MQClientException, RemotingException, InterruptedException {\n        try {\n            this.sendDefaultImpl(msg, CommunicationMode.ONEWAY, null, this.defaultMQProducer.getSendMsgTimeout());\n        } catch (MQBrokerException e) {\n            throw new MQClientException(\"unknown exception\", e);\n        }\n    }\n\n    /**\n     * KERNEL SYNC -------------------------------------------------------\n     */\n    public SendResult send(Message msg, MessageQueue mq)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        return send(msg, mq, this.defaultMQProducer.getSendMsgTimeout());\n    }\n\n    public SendResult send(Message msg, MessageQueue mq, long timeout)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        long beginStartTime = System.currentTimeMillis();\n        this.makeSureStateOK();\n        Validators.checkMessage(msg, this.defaultMQProducer);\n\n        if (!msg.getTopic().equals(mq.getTopic())) {\n            throw new MQClientException(\"message's topic not equal mq's topic\", null);\n        }\n\n        long costTime = System.currentTimeMillis() - beginStartTime;\n        if (timeout < costTime) {\n            throw new RemotingTooMuchRequestException(\"call timeout\");\n        }\n\n        return this.sendKernelImpl(msg, mq, CommunicationMode.SYNC, null, null, timeout);\n    }\n\n    /**\n     * KERNEL ASYNC -------------------------------------------------------\n     */\n    public void send(Message msg, MessageQueue mq, SendCallback sendCallback)\n        throws MQClientException, RemotingException, InterruptedException {\n        send(msg, mq, sendCallback, this.defaultMQProducer.getSendMsgTimeout());\n    }\n\n    /**\n     * @param msg\n     * @param mq\n     * @param sendCallback\n     * @param timeout      the <code>sendCallback</code> will be invoked at most time\n     * @throws MQClientException\n     * @throws RemotingException\n     * @throws InterruptedException\n     * @deprecated It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. A new one will be\n     * provided in next version\n     */\n    @Deprecated\n    public void send(final Message msg, final MessageQueue mq, final SendCallback sendCallback, final long timeout)\n        throws MQClientException, RemotingException, InterruptedException {\n        BackpressureSendCallBack newCallBack = new BackpressureSendCallBack(sendCallback);\n        final long beginStartTime = System.currentTimeMillis();\n        Runnable runnable = new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    makeSureStateOK();\n                    Validators.checkMessage(msg, defaultMQProducer);\n\n                    if (!msg.getTopic().equals(mq.getTopic())) {\n                        throw new MQClientException(\"Topic of the message does not match its target message queue\", null);\n                    }\n                    long costTime = System.currentTimeMillis() - beginStartTime;\n                    if (timeout > costTime) {\n                        try {\n                            sendKernelImpl(msg, mq, CommunicationMode.ASYNC, newCallBack, null,\n                                timeout - costTime);\n                        } catch (MQBrokerException e) {\n                            throw new MQClientException(\"unknown exception\", e);\n                        }\n                    } else {\n                        newCallBack.onException(new RemotingTooMuchRequestException(\"call timeout\"));\n                    }\n                } catch (Exception e) {\n                    newCallBack.onException(e);\n                }\n            }\n\n        };\n\n        executeAsyncMessageSend(runnable, msg, newCallBack, timeout, beginStartTime);\n    }\n\n    /**\n     * KERNEL ONEWAY -------------------------------------------------------\n     */\n    public void sendOneway(Message msg,\n        MessageQueue mq) throws MQClientException, RemotingException, InterruptedException {\n        this.makeSureStateOK();\n        Validators.checkMessage(msg, this.defaultMQProducer);\n\n        try {\n            this.sendKernelImpl(msg, mq, CommunicationMode.ONEWAY, null, null, this.defaultMQProducer.getSendMsgTimeout());\n        } catch (MQBrokerException e) {\n            throw new MQClientException(\"unknown exception\", e);\n        }\n    }\n\n    /**\n     * SELECT SYNC -------------------------------------------------------\n     */\n    public SendResult send(Message msg, MessageQueueSelector selector, Object arg)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        return send(msg, selector, arg, this.defaultMQProducer.getSendMsgTimeout());\n    }\n\n    public SendResult send(Message msg, MessageQueueSelector selector, Object arg, long timeout)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        return this.sendSelectImpl(msg, selector, arg, CommunicationMode.SYNC, null, timeout);\n    }\n\n    private SendResult sendSelectImpl(\n        Message msg,\n        MessageQueueSelector selector,\n        Object arg,\n        final CommunicationMode communicationMode,\n        final SendCallback sendCallback, final long timeout\n    ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        long beginStartTime = System.currentTimeMillis();\n        this.makeSureStateOK();\n        Validators.checkMessage(msg, this.defaultMQProducer);\n\n        TopicPublishInfo topicPublishInfo = this.tryToFindTopicPublishInfo(msg.getTopic());\n        if (topicPublishInfo != null && topicPublishInfo.ok()) {\n            MessageQueue mq = null;\n            try {\n                List<MessageQueue> messageQueueList =\n                    mQClientFactory.getMQAdminImpl().parsePublishMessageQueues(topicPublishInfo.getMessageQueueList());\n                Message userMessage = MessageAccessor.cloneMessage(msg);\n                String userTopic = NamespaceUtil.withoutNamespace(userMessage.getTopic(), mQClientFactory.getClientConfig().getNamespace());\n                userMessage.setTopic(userTopic);\n\n                mq = mQClientFactory.getClientConfig().queueWithNamespace(selector.select(messageQueueList, userMessage, arg));\n            } catch (Throwable e) {\n                throw new MQClientException(\"select message queue threw exception.\", e);\n            }\n\n            long costTime = System.currentTimeMillis() - beginStartTime;\n            if (timeout < costTime) {\n                throw new RemotingTooMuchRequestException(\"sendSelectImpl call timeout\");\n            }\n            if (mq != null) {\n                return this.sendKernelImpl(msg, mq, communicationMode, sendCallback, null, timeout - costTime);\n            } else {\n                throw new MQClientException(\"select message queue return null.\", null);\n            }\n        }\n\n        validateNameServerSetting();\n        throw new MQClientException(\"No route info for this topic, \" + msg.getTopic(), null);\n    }\n\n    /**\n     * SELECT ASYNC -------------------------------------------------------\n     */\n    public void send(Message msg, MessageQueueSelector selector, Object arg, SendCallback sendCallback)\n        throws MQClientException, RemotingException, InterruptedException {\n        send(msg, selector, arg, sendCallback, this.defaultMQProducer.getSendMsgTimeout());\n    }\n\n    /**\n     * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. A new one will be\n     * provided in next version\n     *\n     * @param msg\n     * @param selector\n     * @param arg\n     * @param sendCallback\n     * @param timeout      the <code>sendCallback</code> will be invoked at most time\n     * @throws MQClientException\n     * @throws RemotingException\n     * @throws InterruptedException\n     */\n    @Deprecated\n    public void send(final Message msg, final MessageQueueSelector selector, final Object arg,\n        final SendCallback sendCallback, final long timeout)\n        throws MQClientException, RemotingException, InterruptedException {\n        BackpressureSendCallBack newCallBack = new BackpressureSendCallBack(sendCallback);\n        final long beginStartTime = System.currentTimeMillis();\n        Runnable runnable = new Runnable() {\n            @Override\n            public void run() {\n                long costTime = System.currentTimeMillis() - beginStartTime;\n                if (timeout > costTime) {\n                    try {\n                        try {\n                            sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, newCallBack,\n                                timeout - costTime);\n                        } catch (MQBrokerException e) {\n                            throw new MQClientException(\"unknown exception\", e);\n                        }\n                    } catch (Exception e) {\n                        newCallBack.onException(e);\n                    }\n                } else {\n                    newCallBack.onException(new RemotingTooMuchRequestException(\"call timeout\"));\n                }\n            }\n\n        };\n        executeAsyncMessageSend(runnable, msg, newCallBack, timeout, beginStartTime);\n    }\n\n    /**\n     * SELECT ONEWAY -------------------------------------------------------\n     */\n    public void sendOneway(Message msg, MessageQueueSelector selector, Object arg)\n        throws MQClientException, RemotingException, InterruptedException {\n        try {\n            this.sendSelectImpl(msg, selector, arg, CommunicationMode.ONEWAY, null, this.defaultMQProducer.getSendMsgTimeout());\n        } catch (MQBrokerException e) {\n            throw new MQClientException(\"unknown exception\", e);\n        }\n    }\n\n    public TransactionSendResult sendMessageInTransaction(final Message msg,\n        final TransactionListener localTransactionListener, final Object arg)\n        throws MQClientException {\n        TransactionListener transactionListener = getCheckListener();\n        if (null == localTransactionListener && null == transactionListener) {\n            throw new MQClientException(\"tranExecutor is null\", null);\n        }\n\n        ensureNotDelayedForTransactional(msg);\n        Validators.checkMessage(msg, this.defaultMQProducer);\n\n        SendResult sendResult = null;\n        MessageAccessor.putProperty(msg, MessageConst.PROPERTY_TRANSACTION_PREPARED, \"true\");\n        MessageAccessor.putProperty(msg, MessageConst.PROPERTY_PRODUCER_GROUP, this.defaultMQProducer.getProducerGroup());\n        try {\n            sendResult = this.send(msg);\n        } catch (Exception e) {\n            throw new MQClientException(\"send message Exception\", e);\n        }\n\n        LocalTransactionState localTransactionState = LocalTransactionState.UNKNOW;\n        Throwable localException = null;\n        switch (sendResult.getSendStatus()) {\n            case SEND_OK: {\n                try {\n                    if (sendResult.getTransactionId() != null) {\n                        msg.putUserProperty(\"__transactionId__\", sendResult.getTransactionId());\n                    }\n                    String transactionId = msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX);\n                    if (null != transactionId && !\"\".equals(transactionId)) {\n                        msg.setTransactionId(transactionId);\n                    }\n                    if (null != localTransactionListener) {\n                        localTransactionState = localTransactionListener.executeLocalTransaction(msg, arg);\n                    } else {\n                        log.debug(\"Used new transaction API\");\n                        localTransactionState = transactionListener.executeLocalTransaction(msg, arg);\n                    }\n                    if (null == localTransactionState) {\n                        localTransactionState = LocalTransactionState.UNKNOW;\n                    }\n\n                    if (localTransactionState != LocalTransactionState.COMMIT_MESSAGE) {\n                        log.info(\"executeLocalTransactionBranch return: {} messageTopic: {} transactionId: {} tag: {} key: {}\",\n                            localTransactionState, msg.getTopic(), msg.getTransactionId(), msg.getTags(), msg.getKeys());\n                    }\n                } catch (Throwable e) {\n                    log.error(\"executeLocalTransactionBranch exception, messageTopic: {} transactionId: {} tag: {} key: {}\",\n                        msg.getTopic(), msg.getTransactionId(), msg.getTags(), msg.getKeys(), e);\n                    localException = e;\n                }\n            }\n            break;\n            case FLUSH_DISK_TIMEOUT:\n            case FLUSH_SLAVE_TIMEOUT:\n            case SLAVE_NOT_AVAILABLE:\n                localTransactionState = LocalTransactionState.ROLLBACK_MESSAGE;\n                break;\n            default:\n                break;\n        }\n\n        try {\n            this.endTransaction(msg, sendResult, localTransactionState, localException);\n        } catch (Exception e) {\n            log.warn(\"local transaction execute {}, but end broker transaction failed\", localTransactionState, e);\n        }\n\n        TransactionSendResult transactionSendResult = new TransactionSendResult();\n        transactionSendResult.setSendStatus(sendResult.getSendStatus());\n        transactionSendResult.setMessageQueue(sendResult.getMessageQueue());\n        transactionSendResult.setMsgId(sendResult.getMsgId());\n        transactionSendResult.setQueueOffset(sendResult.getQueueOffset());\n        transactionSendResult.setTransactionId(sendResult.getTransactionId());\n        transactionSendResult.setLocalTransactionState(localTransactionState);\n        return transactionSendResult;\n    }\n\n    private void ensureNotDelayedForTransactional(final Message msg) throws MQClientException {\n        if (msg.getProperty(MessageConst.PROPERTY_DELAY_TIME_LEVEL) != null\n                || msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_MS) != null\n                || msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC) != null\n                || msg.getProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS) != null) {\n            throw new MQClientException(\"Transactional messages do not support delayed delivery\", null);\n        }\n    }\n\n    /**\n     * DEFAULT SYNC -------------------------------------------------------\n     */\n    public SendResult send(\n        Message msg) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        return send(msg, this.defaultMQProducer.getSendMsgTimeout());\n    }\n\n    public void endTransaction(\n        final Message msg,\n        final SendResult sendResult,\n        final LocalTransactionState localTransactionState,\n        final Throwable localException) throws RemotingException, MQBrokerException, InterruptedException, UnknownHostException {\n        final MessageId id;\n        if (sendResult.getOffsetMsgId() != null) {\n            id = MessageDecoder.decodeMessageId(sendResult.getOffsetMsgId());\n        } else {\n            id = MessageDecoder.decodeMessageId(sendResult.getMsgId());\n        }\n        String transactionId = sendResult.getTransactionId();\n        final String destBrokerName = this.mQClientFactory.getBrokerNameFromMessageQueue(defaultMQProducer.queueWithNamespace(sendResult.getMessageQueue()));\n        final String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(destBrokerName);\n        EndTransactionRequestHeader requestHeader = new EndTransactionRequestHeader();\n        requestHeader.setTopic(msg.getTopic());\n        requestHeader.setTransactionId(transactionId);\n        requestHeader.setCommitLogOffset(id.getOffset());\n        requestHeader.setBrokerName(destBrokerName);\n        switch (localTransactionState) {\n            case COMMIT_MESSAGE:\n                requestHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_COMMIT_TYPE);\n                break;\n            case ROLLBACK_MESSAGE:\n                requestHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE);\n                break;\n            case UNKNOW:\n                requestHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_NOT_TYPE);\n                break;\n            default:\n                break;\n        }\n\n        doExecuteEndTransactionHook(msg, sendResult.getMsgId(), brokerAddr, localTransactionState, false);\n        requestHeader.setProducerGroup(this.defaultMQProducer.getProducerGroup());\n        requestHeader.setTranStateTableOffset(sendResult.getQueueOffset());\n        requestHeader.setMsgId(sendResult.getMsgId());\n        String remark = localException != null ? (\"executeLocalTransactionBranch exception: \" + localException.toString()) : null;\n        this.mQClientFactory.getMQClientAPIImpl().endTransactionOneway(brokerAddr, requestHeader, remark,\n            this.defaultMQProducer.getSendMsgTimeout());\n    }\n\n    public String recallMessage(\n        String topic,\n        String recallHandle) throws RemotingException, MQClientException, MQBrokerException, InterruptedException {\n        makeSureStateOK();\n        Validators.checkTopic(topic);\n        if (NamespaceUtil.isRetryTopic(topic) || NamespaceUtil.isDLQTopic(topic)) {\n            throw new MQClientException(\"topic is not supported\", null);\n        }\n        RecallMessageHandle.HandleV1 handleEntity;\n        try {\n            handleEntity = (RecallMessageHandle.HandleV1) RecallMessageHandle.decodeHandle(recallHandle);\n        } catch (Exception e) {\n            throw new MQClientException(e.getMessage(), null);\n        }\n\n        tryToFindTopicPublishInfo(topic);\n        String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(handleEntity.getBrokerName());\n        brokerAddr = StringUtils.isNotEmpty(brokerAddr) ?\n            // find another address to support multi proxy endpoints,\n            // may cause failure request in proxy-less mode when the broker is temporarily unavailable\n            brokerAddr : this.mQClientFactory.findBrokerAddrByTopic(topic);\n        if (StringUtils.isEmpty(brokerAddr)) {\n            log.warn(\"can't find broker service address. {}\", handleEntity.getBrokerName());\n            throw new MQClientException(\"The broker service address not found\", null);\n        }\n        RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader();\n        requestHeader.setProducerGroup(this.defaultMQProducer.getProducerGroup());\n        requestHeader.setTopic(topic);\n        requestHeader.setRecallHandle(recallHandle);\n        requestHeader.setBrokerName(handleEntity.getBrokerName());\n        return this.mQClientFactory.getMQClientAPIImpl().recallMessage(brokerAddr,\n            requestHeader, this.defaultMQProducer.getSendMsgTimeout());\n    }\n\n    public void setCallbackExecutor(final ExecutorService callbackExecutor) {\n        this.mQClientFactory.getMQClientAPIImpl().getRemotingClient().setCallbackExecutor(callbackExecutor);\n    }\n\n    public ExecutorService getAsyncSenderExecutor() {\n        return null == asyncSenderExecutor ? defaultAsyncSenderExecutor : asyncSenderExecutor;\n    }\n\n    public void setAsyncSenderExecutor(ExecutorService asyncSenderExecutor) {\n        this.asyncSenderExecutor = asyncSenderExecutor;\n    }\n\n    public SendResult send(Message msg,\n        long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        return this.sendDefaultImpl(msg, CommunicationMode.SYNC, null, timeout);\n    }\n\n    public Message request(final Message msg,\n        long timeout) throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        long beginTimestamp = System.currentTimeMillis();\n        prepareSendRequest(msg, timeout);\n        final String correlationId = msg.getProperty(MessageConst.PROPERTY_CORRELATION_ID);\n\n        try {\n            final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(correlationId, timeout, null);\n            RequestFutureHolder.getInstance().getRequestFutureTable().put(correlationId, requestResponseFuture);\n\n            long cost = System.currentTimeMillis() - beginTimestamp;\n            this.sendDefaultImpl(msg, CommunicationMode.ASYNC, new SendCallback() {\n                @Override\n                public void onSuccess(SendResult sendResult) {\n                    requestResponseFuture.setSendRequestOk(true);\n                }\n\n                @Override\n                public void onException(Throwable e) {\n                    requestResponseFuture.setSendRequestOk(false);\n                    requestResponseFuture.putResponseMessage(null);\n                    requestResponseFuture.setCause(e);\n                }\n            }, timeout - cost);\n\n            return waitResponse(msg, timeout, requestResponseFuture, cost);\n        } finally {\n            RequestFutureHolder.getInstance().getRequestFutureTable().remove(correlationId);\n        }\n    }\n\n    public void request(Message msg, final RequestCallback requestCallback, long timeout)\n        throws RemotingException, InterruptedException, MQClientException, MQBrokerException {\n        long beginTimestamp = System.currentTimeMillis();\n        prepareSendRequest(msg, timeout);\n        final String correlationId = msg.getProperty(MessageConst.PROPERTY_CORRELATION_ID);\n\n        final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(correlationId, timeout, requestCallback);\n        RequestFutureHolder.getInstance().getRequestFutureTable().put(correlationId, requestResponseFuture);\n\n        long cost = System.currentTimeMillis() - beginTimestamp;\n        this.sendDefaultImpl(msg, CommunicationMode.ASYNC, new SendCallback() {\n            @Override\n            public void onSuccess(SendResult sendResult) {\n                requestResponseFuture.setSendRequestOk(true);\n                requestResponseFuture.executeRequestCallback();\n            }\n\n            @Override\n            public void onException(Throwable e) {\n                requestResponseFuture.setCause(e);\n                requestFail(correlationId);\n            }\n        }, timeout - cost);\n    }\n\n    public Message request(final Message msg, final MessageQueueSelector selector, final Object arg,\n        final long timeout) throws MQClientException, RemotingException, MQBrokerException,\n        InterruptedException, RequestTimeoutException {\n        long beginTimestamp = System.currentTimeMillis();\n        prepareSendRequest(msg, timeout);\n        final String correlationId = msg.getProperty(MessageConst.PROPERTY_CORRELATION_ID);\n\n        try {\n            final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(correlationId, timeout, null);\n            RequestFutureHolder.getInstance().getRequestFutureTable().put(correlationId, requestResponseFuture);\n\n            long cost = System.currentTimeMillis() - beginTimestamp;\n            this.sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, new SendCallback() {\n                @Override\n                public void onSuccess(SendResult sendResult) {\n                    requestResponseFuture.setSendRequestOk(true);\n                }\n\n                @Override\n                public void onException(Throwable e) {\n                    requestResponseFuture.setSendRequestOk(false);\n                    requestResponseFuture.putResponseMessage(null);\n                    requestResponseFuture.setCause(e);\n                }\n            }, timeout - cost);\n\n            return waitResponse(msg, timeout, requestResponseFuture, cost);\n        } finally {\n            RequestFutureHolder.getInstance().getRequestFutureTable().remove(correlationId);\n        }\n    }\n\n    public void request(final Message msg, final MessageQueueSelector selector, final Object arg,\n        final RequestCallback requestCallback, final long timeout)\n        throws RemotingException, InterruptedException, MQClientException, MQBrokerException {\n        long beginTimestamp = System.currentTimeMillis();\n        prepareSendRequest(msg, timeout);\n        final String correlationId = msg.getProperty(MessageConst.PROPERTY_CORRELATION_ID);\n\n        final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(correlationId, timeout, requestCallback);\n        RequestFutureHolder.getInstance().getRequestFutureTable().put(correlationId, requestResponseFuture);\n\n        long cost = System.currentTimeMillis() - beginTimestamp;\n        this.sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, new SendCallback() {\n            @Override\n            public void onSuccess(SendResult sendResult) {\n                requestResponseFuture.setSendRequestOk(true);\n            }\n\n            @Override\n            public void onException(Throwable e) {\n                requestResponseFuture.setCause(e);\n                requestFail(correlationId);\n            }\n        }, timeout - cost);\n\n    }\n\n    public Message request(final Message msg, final MessageQueue mq, final long timeout)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException, RequestTimeoutException {\n        long beginTimestamp = System.currentTimeMillis();\n        prepareSendRequest(msg, timeout);\n        final String correlationId = msg.getProperty(MessageConst.PROPERTY_CORRELATION_ID);\n\n        try {\n            final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(correlationId, timeout, null);\n            RequestFutureHolder.getInstance().getRequestFutureTable().put(correlationId, requestResponseFuture);\n\n            long cost = System.currentTimeMillis() - beginTimestamp;\n            this.sendKernelImpl(msg, mq, CommunicationMode.ASYNC, new SendCallback() {\n                @Override\n                public void onSuccess(SendResult sendResult) {\n                    requestResponseFuture.setSendRequestOk(true);\n                }\n\n                @Override\n                public void onException(Throwable e) {\n                    requestResponseFuture.setSendRequestOk(false);\n                    requestResponseFuture.putResponseMessage(null);\n                    requestResponseFuture.setCause(e);\n                }\n            }, null, timeout - cost);\n\n            return waitResponse(msg, timeout, requestResponseFuture, cost);\n        } finally {\n            RequestFutureHolder.getInstance().getRequestFutureTable().remove(correlationId);\n        }\n    }\n\n    private Message waitResponse(Message msg, long timeout, RequestResponseFuture requestResponseFuture,\n        long cost) throws InterruptedException, RequestTimeoutException, MQClientException {\n        Message responseMessage = requestResponseFuture.waitResponseMessage(timeout - cost);\n        if (responseMessage == null) {\n            if (requestResponseFuture.isSendRequestOk()) {\n                throw new RequestTimeoutException(ClientErrorCode.REQUEST_TIMEOUT_EXCEPTION,\n                    \"send request message to <\" + msg.getTopic() + \"> OK, but wait reply message timeout, \" + timeout + \" ms.\");\n            } else {\n                throw new MQClientException(\"send request message to <\" + msg.getTopic() + \"> fail\", requestResponseFuture.getCause());\n            }\n        }\n        return responseMessage;\n    }\n\n    public void request(final Message msg, final MessageQueue mq, final RequestCallback requestCallback, long timeout)\n        throws RemotingException, InterruptedException, MQClientException, MQBrokerException {\n        long beginTimestamp = System.currentTimeMillis();\n        prepareSendRequest(msg, timeout);\n        final String correlationId = msg.getProperty(MessageConst.PROPERTY_CORRELATION_ID);\n\n        final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(correlationId, timeout, requestCallback);\n        RequestFutureHolder.getInstance().getRequestFutureTable().put(correlationId, requestResponseFuture);\n\n        long cost = System.currentTimeMillis() - beginTimestamp;\n        this.sendKernelImpl(msg, mq, CommunicationMode.ASYNC, new SendCallback() {\n            @Override\n            public void onSuccess(SendResult sendResult) {\n                requestResponseFuture.setSendRequestOk(true);\n            }\n\n            @Override\n            public void onException(Throwable e) {\n                requestResponseFuture.setCause(e);\n                requestFail(correlationId);\n            }\n        }, null, timeout - cost);\n    }\n\n    private void requestFail(final String correlationId) {\n        RequestResponseFuture responseFuture = RequestFutureHolder.getInstance().getRequestFutureTable().remove(correlationId);\n        if (responseFuture != null) {\n            responseFuture.setSendRequestOk(false);\n            responseFuture.putResponseMessage(null);\n            try {\n                responseFuture.executeRequestCallback();\n            } catch (Exception e) {\n                log.warn(\"execute requestCallback in requestFail, and callback throw\", e);\n            }\n        }\n    }\n\n    private void prepareSendRequest(final Message msg, long timeout) {\n        String correlationId = CorrelationIdUtil.createCorrelationId();\n        String requestClientId = this.getMqClientFactory().getClientId();\n        MessageAccessor.putProperty(msg, MessageConst.PROPERTY_CORRELATION_ID, correlationId);\n        MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT, requestClientId);\n        MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MESSAGE_TTL, String.valueOf(timeout));\n\n        boolean hasRouteData = this.getMqClientFactory().getTopicRouteTable().containsKey(msg.getTopic());\n        if (!hasRouteData) {\n            long beginTimestamp = System.currentTimeMillis();\n            this.tryToFindTopicPublishInfo(msg.getTopic());\n            this.getMqClientFactory().sendHeartbeatToAllBrokerWithLock();\n            long cost = System.currentTimeMillis() - beginTimestamp;\n            if (cost > 500) {\n                log.warn(\"prepare send request for <{}> cost {} ms\", msg.getTopic(), cost);\n            }\n        }\n    }\n\n    private void initTopicRoute() {\n        List<String> topics = this.defaultMQProducer.getTopics();\n        if (topics != null && topics.size() > 0) {\n            topics.forEach(topic -> {\n                String newTopic = NamespaceUtil.wrapNamespace(this.defaultMQProducer.getNamespace(), topic);\n                TopicPublishInfo topicPublishInfo = tryToFindTopicPublishInfo(newTopic);\n                if (topicPublishInfo == null || !topicPublishInfo.ok()) {\n                    log.warn(\"No route info of this topic: \" + newTopic + FAQUrl.suggestTodo(FAQUrl.NO_TOPIC_ROUTE_INFO));\n                }\n            });\n        }\n    }\n\n    public ConcurrentMap<String, TopicPublishInfo> getTopicPublishInfoTable() {\n        return topicPublishInfoTable;\n    }\n\n    public ServiceState getServiceState() {\n        return serviceState;\n    }\n\n    public void setServiceState(ServiceState serviceState) {\n        this.serviceState = serviceState;\n    }\n\n    public long[] getNotAvailableDuration() {\n        return this.mqFaultStrategy.getNotAvailableDuration();\n    }\n\n    public void setNotAvailableDuration(final long[] notAvailableDuration) {\n        this.mqFaultStrategy.setNotAvailableDuration(notAvailableDuration);\n    }\n\n    public long[] getLatencyMax() {\n        return this.mqFaultStrategy.getLatencyMax();\n    }\n\n    public void setLatencyMax(final long[] latencyMax) {\n        this.mqFaultStrategy.setLatencyMax(latencyMax);\n    }\n\n    public boolean isSendLatencyFaultEnable() {\n        return this.mqFaultStrategy.isSendLatencyFaultEnable();\n    }\n\n    public void setSendLatencyFaultEnable(final boolean sendLatencyFaultEnable) {\n        this.mqFaultStrategy.setSendLatencyFaultEnable(sendLatencyFaultEnable);\n    }\n\n    public DefaultMQProducer getDefaultMQProducer() {\n        return defaultMQProducer;\n    }\n\n    public MQFaultStrategy getMqFaultStrategy() {\n        return mqFaultStrategy;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/producer/MQProducerInner.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.producer;\n\nimport java.util.Set;\nimport org.apache.rocketmq.client.producer.TransactionCheckListener;\nimport org.apache.rocketmq.client.producer.TransactionListener;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;\n\npublic interface MQProducerInner {\n    Set<String> getPublishTopicList();\n\n    boolean isPublishTopicNeedUpdate(final String topic);\n\n    TransactionCheckListener checkListener();\n    TransactionListener getCheckListener();\n\n    void checkTransactionState(\n        final String addr,\n        final MessageExt msg,\n        final CheckTransactionStateRequestHeader checkRequestHeader);\n\n    void updateTopicPublishInfo(final String topic, final TopicPublishInfo info);\n\n    boolean isUnitMode();\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.producer;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\nimport com.google.common.base.Preconditions;\nimport org.apache.rocketmq.client.common.ThreadLocalIndex;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\n\npublic class TopicPublishInfo {\n    private boolean orderTopic = false;\n    private boolean haveTopicRouterInfo = false;\n    private List<MessageQueue> messageQueueList = new ArrayList<>();\n    private volatile ThreadLocalIndex sendWhichQueue = new ThreadLocalIndex();\n    private TopicRouteData topicRouteData;\n\n    public interface QueueFilter {\n        boolean filter(MessageQueue mq);\n    }\n\n    public boolean isOrderTopic() {\n        return orderTopic;\n    }\n\n    public void setOrderTopic(boolean orderTopic) {\n        this.orderTopic = orderTopic;\n    }\n\n    public boolean ok() {\n        return null != this.messageQueueList && !this.messageQueueList.isEmpty();\n    }\n\n    public List<MessageQueue> getMessageQueueList() {\n        return messageQueueList;\n    }\n\n    public void setMessageQueueList(List<MessageQueue> messageQueueList) {\n        this.messageQueueList = messageQueueList;\n    }\n\n    public ThreadLocalIndex getSendWhichQueue() {\n        return sendWhichQueue;\n    }\n\n    public void setSendWhichQueue(ThreadLocalIndex sendWhichQueue) {\n        this.sendWhichQueue = sendWhichQueue;\n    }\n\n    public boolean isHaveTopicRouterInfo() {\n        return haveTopicRouterInfo;\n    }\n\n    public void setHaveTopicRouterInfo(boolean haveTopicRouterInfo) {\n        this.haveTopicRouterInfo = haveTopicRouterInfo;\n    }\n\n    public MessageQueue selectOneMessageQueue(QueueFilter ...filter) {\n        return selectOneMessageQueue(this.messageQueueList, this.sendWhichQueue, filter);\n    }\n\n    private MessageQueue selectOneMessageQueue(List<MessageQueue> messageQueueList, ThreadLocalIndex sendQueue, QueueFilter ...filter) {\n        if (messageQueueList == null || messageQueueList.isEmpty()) {\n            return null;\n        }\n\n        if (filter != null && filter.length != 0) {\n            for (int i = 0; i < messageQueueList.size(); i++) {\n                int index = Math.abs(sendQueue.incrementAndGet() % messageQueueList.size());\n                MessageQueue mq = messageQueueList.get(index);\n                boolean filterResult = true;\n                for (QueueFilter f: filter) {\n                    Preconditions.checkNotNull(f);\n                    filterResult &= f.filter(mq);\n                }\n                if (filterResult) {\n                    return mq;\n                }\n            }\n\n            return null;\n        }\n\n        int index = Math.abs(sendQueue.incrementAndGet() % messageQueueList.size());\n        return messageQueueList.get(index);\n    }\n\n    public void resetIndex() {\n        this.sendWhichQueue.reset();\n    }\n\n    public MessageQueue selectOneMessageQueue(final String lastBrokerName) {\n        if (lastBrokerName == null) {\n            return selectOneMessageQueue();\n        } else {\n            for (int i = 0; i < this.messageQueueList.size(); i++) {\n                MessageQueue mq = selectOneMessageQueue();\n                if (!mq.getBrokerName().equals(lastBrokerName)) {\n                    return mq;\n                }\n            }\n            return selectOneMessageQueue();\n        }\n    }\n\n    public MessageQueue selectOneMessageQueue() {\n        int index = this.sendWhichQueue.incrementAndGet();\n        int pos = index % this.messageQueueList.size();\n\n        return this.messageQueueList.get(pos);\n    }\n\n    public int getWriteQueueNumsByBroker(final String brokerName) {\n        for (int i = 0; i < topicRouteData.getQueueDatas().size(); i++) {\n            final QueueData queueData = this.topicRouteData.getQueueDatas().get(i);\n            if (queueData.getBrokerName().equals(brokerName)) {\n                return queueData.getWriteQueueNums();\n            }\n        }\n\n        return -1;\n    }\n\n    @Override\n    public String toString() {\n        return \"TopicPublishInfo [orderTopic=\" + orderTopic + \", messageQueueList=\" + messageQueueList\n            + \", sendWhichQueue=\" + sendWhichQueue + \", haveTopicRouterInfo=\" + haveTopicRouterInfo + \"]\";\n    }\n\n    public TopicRouteData getTopicRouteData() {\n        return topicRouteData;\n    }\n\n    public void setTopicRouteData(final TopicRouteData topicRouteData) {\n        this.topicRouteData = topicRouteData;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.latency;\n\npublic interface LatencyFaultTolerance<T> {\n    /**\n     * Update brokers' states, to decide if they are good or not.\n     *\n     * @param name Broker's name.\n     * @param currentLatency Current message sending process's latency.\n     * @param notAvailableDuration Corresponding not available time, ms. The broker will be not available until it\n     * spends such time.\n     * @param reachable To decide if this broker is reachable or not.\n     */\n    void updateFaultItem(final T name, final long currentLatency, final long notAvailableDuration,\n                         final boolean reachable);\n\n    /**\n     * To check if this broker is available.\n     *\n     * @param name Broker's name.\n     * @return boolean variable, if this is true, then the broker is available.\n     */\n    boolean isAvailable(final T name);\n\n    /**\n     * To check if this broker is reachable.\n     *\n     * @param name Broker's name.\n     * @return boolean variable, if this is true, then the broker is reachable.\n     */\n    boolean isReachable(final T name);\n\n    /**\n     * Remove the broker in this fault item table.\n     *\n     * @param name broker's name.\n     */\n    void remove(final T name);\n\n    /**\n     * The worst situation, no broker can be available. Then choose random one.\n     *\n     * @return A random mq will be returned.\n     */\n    T pickOneAtLeast();\n\n    /**\n     * Start a new thread, to detect the broker's reachable tag.\n     */\n    void startDetector();\n\n    /**\n     * Shutdown threads that started by LatencyFaultTolerance.\n     */\n    void shutdown();\n\n    /**\n     * A function reserved, just detect by once, won't create a new thread.\n     */\n    void detectByOneRound();\n\n    /**\n     * Use it to set the detect timeout bound.\n     *\n     * @param detectTimeout timeout bound\n     */\n    void setDetectTimeout(final int detectTimeout);\n\n    /**\n     * Use it to set the detector's detector interval for each broker (each broker will be detected once during this\n     * time)\n     *\n     * @param detectInterval each broker's detecting interval\n     */\n    void setDetectInterval(final int detectInterval);\n\n    /**\n     * Use it to set the detector work or not.\n     *\n     * @param startDetectorEnable set the detector's work status\n     */\n    void setStartDetectorEnable(final boolean startDetectorEnable);\n\n    /**\n     * Use it to judge if the detector enabled.\n     *\n     * @return is the detector should be started.\n     */\n    boolean isStartDetectorEnable();\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.latency;\n\nimport java.util.Collections;\nimport java.util.Enumeration;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ThreadFactory;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.client.common.ThreadLocalIndex;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class LatencyFaultToleranceImpl implements LatencyFaultTolerance<String> {\n    private final static Logger log = LoggerFactory.getLogger(MQFaultStrategy.class);\n    private final ConcurrentHashMap<String, FaultItem> faultItemTable = new ConcurrentHashMap<String, FaultItem>(16);\n    private int detectTimeout = 200;\n    private int detectInterval = 2000;\n    private final ThreadLocalIndex whichItemWorst = new ThreadLocalIndex();\n\n    private volatile boolean startDetectorEnable = false;\n    private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {\n        @Override\n        public Thread newThread(Runnable r) {\n            return new Thread(r, \"LatencyFaultToleranceScheduledThread\");\n        }\n    });\n\n    private final Resolver resolver;\n\n    private final ServiceDetector serviceDetector;\n\n    public LatencyFaultToleranceImpl(Resolver resolver, ServiceDetector serviceDetector) {\n        this.resolver = resolver;\n        this.serviceDetector = serviceDetector;\n    }\n\n    @Override\n    public void detectByOneRound() {\n        for (Map.Entry<String, FaultItem> item : this.faultItemTable.entrySet()) {\n            FaultItem brokerItem = item.getValue();\n            if (System.currentTimeMillis() - brokerItem.checkStamp >= 0) {\n                brokerItem.checkStamp = System.currentTimeMillis() + this.detectInterval;\n                String brokerAddr = resolver.resolve(brokerItem.getName());\n                if (brokerAddr == null) {\n                    faultItemTable.remove(item.getKey());\n                    continue;\n                }\n                if (null == serviceDetector) {\n                    continue;\n                }\n                boolean serviceOK = serviceDetector.detect(brokerAddr, detectTimeout);\n                if (serviceOK && !brokerItem.reachableFlag) {\n                    log.info(brokerItem.name + \" is reachable now, then it can be used.\");\n                    brokerItem.reachableFlag = true;\n                }\n            }\n        }\n    }\n\n    @Override\n    public void startDetector() {\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    if (startDetectorEnable) {\n                        detectByOneRound();\n                    }\n                } catch (Exception e) {\n                    log.warn(\"Unexpected exception raised while detecting service reachability\", e);\n                }\n            }\n        }, 3, 3, TimeUnit.SECONDS);\n    }\n\n    @Override\n    public void shutdown() {\n        this.scheduledExecutorService.shutdown();\n    }\n\n    @Override\n    public void updateFaultItem(final String name, final long currentLatency, final long notAvailableDuration,\n                                final boolean reachable) {\n        FaultItem old = this.faultItemTable.get(name);\n        if (null == old) {\n            final FaultItem faultItem = new FaultItem(name);\n            faultItem.setCurrentLatency(currentLatency);\n            faultItem.updateNotAvailableDuration(notAvailableDuration);\n            faultItem.setReachable(reachable);\n            old = this.faultItemTable.putIfAbsent(name, faultItem);\n        }\n\n        if (null != old) {\n            old.setCurrentLatency(currentLatency);\n            old.updateNotAvailableDuration(notAvailableDuration);\n            old.setReachable(reachable);\n        }\n\n        if (!reachable) {\n            log.info(name + \" is unreachable, it will not be used until it's reachable\");\n        }\n    }\n\n    @Override\n    public boolean isAvailable(final String name) {\n        final FaultItem faultItem = this.faultItemTable.get(name);\n        if (faultItem != null) {\n            return faultItem.isAvailable();\n        }\n        return true;\n    }\n\n    @Override\n    public boolean isReachable(final String name) {\n        final FaultItem faultItem = this.faultItemTable.get(name);\n        if (faultItem != null) {\n            return faultItem.isReachable();\n        }\n        return true;\n    }\n\n    @Override\n    public void remove(final String name) {\n        this.faultItemTable.remove(name);\n    }\n\n    @Override\n    public boolean isStartDetectorEnable() {\n        return startDetectorEnable;\n    }\n\n    @Override\n    public void setStartDetectorEnable(boolean startDetectorEnable) {\n        this.startDetectorEnable = startDetectorEnable;\n    }\n    @Override\n    public String pickOneAtLeast() {\n        final Enumeration<FaultItem> elements = this.faultItemTable.elements();\n        List<FaultItem> tmpList = new LinkedList<FaultItem>();\n        while (elements.hasMoreElements()) {\n            final FaultItem faultItem = elements.nextElement();\n            tmpList.add(faultItem);\n        }\n\n        if (!tmpList.isEmpty()) {\n            Collections.shuffle(tmpList);\n            for (FaultItem faultItem : tmpList) {\n                if (faultItem.reachableFlag) {\n                    return faultItem.name;\n                }\n            }\n        }\n\n        return null;\n    }\n\n    @Override\n    public String toString() {\n        return \"LatencyFaultToleranceImpl{\" +\n                \"faultItemTable=\" + faultItemTable +\n                \", whichItemWorst=\" + whichItemWorst +\n                '}';\n    }\n\n    @Override\n    public void setDetectTimeout(final int detectTimeout) {\n        this.detectTimeout = detectTimeout;\n    }\n\n    @Override\n    public void setDetectInterval(final int detectInterval) {\n        this.detectInterval = detectInterval;\n    }\n\n    public class FaultItem implements Comparable<FaultItem> {\n        private final String name;\n        private volatile long currentLatency;\n        private volatile long startTimestamp;\n        private volatile long checkStamp;\n        private volatile boolean reachableFlag;\n\n        public FaultItem(final String name) {\n            this.name = name;\n        }\n\n        public void updateNotAvailableDuration(long notAvailableDuration) {\n            if (notAvailableDuration > 0 && System.currentTimeMillis() + notAvailableDuration > this.startTimestamp) {\n                this.startTimestamp = System.currentTimeMillis() + notAvailableDuration;\n                log.info(name + \" will be isolated for \" + notAvailableDuration + \" ms.\");\n            }\n        }\n\n        @Override\n        public int compareTo(final FaultItem other) {\n            if (this.isAvailable() != other.isAvailable()) {\n                if (this.isAvailable()) {\n                    return -1;\n                }\n\n                if (other.isAvailable()) {\n                    return 1;\n                }\n            }\n\n            if (this.currentLatency < other.currentLatency) {\n                return -1;\n            } else if (this.currentLatency > other.currentLatency) {\n                return 1;\n            }\n\n            if (this.startTimestamp < other.startTimestamp) {\n                return -1;\n            } else if (this.startTimestamp > other.startTimestamp) {\n                return 1;\n            }\n            return 0;\n        }\n\n        public void setReachable(boolean reachableFlag) {\n            this.reachableFlag = reachableFlag;\n        }\n\n        public void setCheckStamp(long checkStamp) {\n            this.checkStamp = checkStamp;\n        }\n\n        public boolean isAvailable() {\n            return System.currentTimeMillis() >= startTimestamp;\n        }\n\n        public boolean isReachable() {\n            return reachableFlag;\n        }\n\n        @Override\n        public int hashCode() {\n            int result = getName() != null ? getName().hashCode() : 0;\n            result = 31 * result + (int) (getCurrentLatency() ^ (getCurrentLatency() >>> 32));\n            result = 31 * result + (int) (getStartTimestamp() ^ (getStartTimestamp() >>> 32));\n            return result;\n        }\n\n        @Override\n        public boolean equals(final Object o) {\n            if (this == o) {\n                return true;\n            }\n            if (!(o instanceof FaultItem)) {\n                return false;\n            }\n\n            final FaultItem faultItem = (FaultItem) o;\n\n            if (getCurrentLatency() != faultItem.getCurrentLatency()) {\n                return false;\n            }\n            if (getStartTimestamp() != faultItem.getStartTimestamp()) {\n                return false;\n            }\n            return getName() != null ? getName().equals(faultItem.getName()) : faultItem.getName() == null;\n        }\n\n        @Override\n        public String toString() {\n            return \"FaultItem{\" +\n                    \"name='\" + name + '\\'' +\n                    \", currentLatency=\" + currentLatency +\n                    \", startTimestamp=\" + startTimestamp +\n                    \", reachableFlag=\" + reachableFlag +\n                    '}';\n        }\n\n        public String getName() {\n            return name;\n        }\n\n        public long getCurrentLatency() {\n            return currentLatency;\n        }\n\n        public void setCurrentLatency(final long currentLatency) {\n            this.currentLatency = currentLatency;\n        }\n\n        public long getStartTimestamp() {\n            return startTimestamp;\n        }\n\n        public void setStartTimestamp(final long startTimestamp) {\n            this.startTimestamp = startTimestamp;\n        }\n\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.latency;\n\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.impl.producer.TopicPublishInfo;\nimport org.apache.rocketmq.client.impl.producer.TopicPublishInfo.QueueFilter;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.utils.StartAndShutdown;\n\npublic class MQFaultStrategy implements StartAndShutdown {\n    private LatencyFaultTolerance<String> latencyFaultTolerance;\n    private volatile boolean sendLatencyFaultEnable;\n    private volatile boolean startDetectorEnable;\n    private long[] latencyMax = {50L, 100L, 550L, 1800L, 3000L, 5000L, 15000L};\n    private long[] notAvailableDuration = {0L, 0L, 2000L, 5000L, 6000L, 10000L, 30000L};\n\n    public static class BrokerFilter implements QueueFilter {\n        private String lastBrokerName;\n\n        public void setLastBrokerName(String lastBrokerName) {\n            this.lastBrokerName = lastBrokerName;\n        }\n\n        @Override public boolean filter(MessageQueue mq) {\n            if (lastBrokerName != null) {\n                return !mq.getBrokerName().equals(lastBrokerName);\n            }\n            return true;\n        }\n    }\n\n    private ThreadLocal<BrokerFilter> threadBrokerFilter = new ThreadLocal<BrokerFilter>() {\n        @Override protected BrokerFilter initialValue() {\n            return new BrokerFilter();\n        }\n    };\n\n    private QueueFilter reachableFilter = new QueueFilter() {\n        @Override public boolean filter(MessageQueue mq) {\n            return latencyFaultTolerance.isReachable(mq.getBrokerName());\n        }\n    };\n\n    private QueueFilter availableFilter = new QueueFilter() {\n        @Override public boolean filter(MessageQueue mq) {\n            return latencyFaultTolerance.isAvailable(mq.getBrokerName());\n        }\n    };\n\n\n    public MQFaultStrategy(ClientConfig cc, Resolver fetcher, ServiceDetector serviceDetector) {\n        this.latencyFaultTolerance = new LatencyFaultToleranceImpl(fetcher, serviceDetector);\n        this.latencyFaultTolerance.setDetectInterval(cc.getDetectInterval());\n        this.latencyFaultTolerance.setDetectTimeout(cc.getDetectTimeout());\n        this.setStartDetectorEnable(cc.isStartDetectorEnable());\n        this.setSendLatencyFaultEnable(cc.isSendLatencyEnable());\n    }\n\n    // For unit test.\n    public MQFaultStrategy(ClientConfig cc, LatencyFaultTolerance<String> tolerance) {\n        this.setStartDetectorEnable(cc.isStartDetectorEnable());\n        this.setSendLatencyFaultEnable(cc.isSendLatencyEnable());\n        this.latencyFaultTolerance = tolerance;\n        this.latencyFaultTolerance.setDetectInterval(cc.getDetectInterval());\n        this.latencyFaultTolerance.setDetectTimeout(cc.getDetectTimeout());\n    }\n\n\n    public long[] getNotAvailableDuration() {\n        return notAvailableDuration;\n    }\n\n    public QueueFilter getAvailableFilter() {\n        return availableFilter;\n    }\n\n    public QueueFilter getReachableFilter() {\n        return reachableFilter;\n    }\n\n    public ThreadLocal<BrokerFilter> getThreadBrokerFilter() {\n        return threadBrokerFilter;\n    }\n\n    public void setNotAvailableDuration(final long[] notAvailableDuration) {\n        this.notAvailableDuration = notAvailableDuration;\n    }\n\n    public long[] getLatencyMax() {\n        return latencyMax;\n    }\n\n    public void setLatencyMax(final long[] latencyMax) {\n        this.latencyMax = latencyMax;\n    }\n\n    public boolean isSendLatencyFaultEnable() {\n        return sendLatencyFaultEnable;\n    }\n\n    public void setSendLatencyFaultEnable(final boolean sendLatencyFaultEnable) {\n        this.sendLatencyFaultEnable = sendLatencyFaultEnable;\n    }\n\n    public boolean isStartDetectorEnable() {\n        return startDetectorEnable;\n    }\n\n    public void setStartDetectorEnable(boolean startDetectorEnable) {\n        this.startDetectorEnable = startDetectorEnable;\n        this.latencyFaultTolerance.setStartDetectorEnable(startDetectorEnable);\n    }\n\n    public void startDetector() {\n        this.latencyFaultTolerance.startDetector();\n    }\n\n    @Override\n    public void start() throws Exception {\n        this.startDetector();\n    }\n\n    public void shutdown() {\n        this.latencyFaultTolerance.shutdown();\n    }\n\n    public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName, final boolean resetIndex) {\n        BrokerFilter brokerFilter = threadBrokerFilter.get();\n        brokerFilter.setLastBrokerName(lastBrokerName);\n        if (this.sendLatencyFaultEnable) {\n            if (resetIndex) {\n                tpInfo.resetIndex();\n            }\n            MessageQueue mq = tpInfo.selectOneMessageQueue(availableFilter, brokerFilter);\n            if (mq != null) {\n                return mq;\n            }\n\n            mq = tpInfo.selectOneMessageQueue(reachableFilter, brokerFilter);\n            if (mq != null) {\n                return mq;\n            }\n\n            return tpInfo.selectOneMessageQueue();\n        }\n\n        MessageQueue mq = tpInfo.selectOneMessageQueue(brokerFilter);\n        if (mq != null) {\n            return mq;\n        }\n        return tpInfo.selectOneMessageQueue();\n    }\n\n    public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation,\n                                final boolean reachable) {\n        if (this.sendLatencyFaultEnable) {\n            long duration = computeNotAvailableDuration(isolation ? 10000 : currentLatency);\n            this.latencyFaultTolerance.updateFaultItem(brokerName, currentLatency, duration, reachable);\n        }\n    }\n\n    private long computeNotAvailableDuration(final long currentLatency) {\n        for (int i = latencyMax.length - 1; i >= 0; i--) {\n            if (currentLatency >= latencyMax[i]) {\n                return this.notAvailableDuration[i];\n            }\n        }\n\n        return 0;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/latency/Resolver.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.latency;\n\npublic interface Resolver {\n\n    String resolve(String name);\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/latency/ServiceDetector.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.latency;\n\n/**\n * Detect whether the remote service state is normal.\n */\npublic interface ServiceDetector {\n\n    /**\n     * Check if the remote service is normal.\n     * @param endpoint Service endpoint to check against\n     * @return true if the service is back to normal; false otherwise.\n     */\n    boolean detect(String endpoint, long timeoutMillis);\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/lock/ReadWriteCASLock.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.lock;\n\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.concurrent.atomic.AtomicInteger;\n\npublic class ReadWriteCASLock {\n    //true : can lock ; false : not lock\n    private final AtomicBoolean writeLock = new AtomicBoolean(true);\n\n    private final AtomicInteger readLock = new AtomicInteger(0);\n\n    public void acquireWriteLock() {\n        boolean isLock = false;\n        do {\n            isLock = writeLock.compareAndSet(true, false);\n        } while (!isLock);\n\n        do {\n            isLock = readLock.get() == 0;\n        } while (!isLock);\n    }\n\n    public void releaseWriteLock() {\n        this.writeLock.compareAndSet(false, true);\n    }\n\n    public void acquireReadLock() {\n        boolean isLock = false;\n        do {\n            isLock = writeLock.get();\n        } while (!isLock);\n        readLock.getAndIncrement();\n    }\n\n    public void releaseReadLock() {\n        this.readLock.getAndDecrement();\n    }\n\n    public boolean getWriteLock() {\n        return this.writeLock.get() && this.readLock.get() == 0;\n    }\n\n    public boolean getReadLock() {\n        return this.writeLock.get();\n    }\n\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.producer;\n\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.QueryResult;\nimport org.apache.rocketmq.client.Validators;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.exception.RequestTimeoutException;\nimport org.apache.rocketmq.client.impl.MQClientManager;\nimport org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;\nimport org.apache.rocketmq.client.lock.ReadWriteCASLock;\nimport org.apache.rocketmq.client.trace.hook.DefaultRecallMessageTraceHook;\nimport org.apache.rocketmq.client.trace.AsyncTraceDispatcher;\nimport org.apache.rocketmq.client.trace.TraceDispatcher;\nimport org.apache.rocketmq.client.trace.hook.EndTransactionTraceHookImpl;\nimport org.apache.rocketmq.client.trace.hook.SendMessageTraceHookImpl;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.compression.CompressionType;\nimport org.apache.rocketmq.common.compression.Compressor;\nimport org.apache.rocketmq.common.compression.CompressorFactory;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageBatch;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\n\nimport java.util.Arrays;\nimport java.util.Collection;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.CopyOnWriteArraySet;\nimport java.util.concurrent.ExecutorService;\n\n/**\n * This class is the entry point for applications intending to send messages. </p>\n * <p>\n * It's fine to tune fields which exposes getter/setter methods, but keep in mind, all of them should work well out of\n * box for most scenarios. </p>\n * <p>\n * This class aggregates various <code>send</code> methods to deliver messages to broker(s). Each of them has pros and\n * cons; you'd better understand strengths and weakness of them before actually coding. </p>\n *\n * <p> <strong>Thread Safety:</strong> After configuring and starting process, this class can be regarded as thread-safe\n * and used among multiple threads context. </p>\n */\npublic class DefaultMQProducer extends ClientConfig implements MQProducer {\n\n    /**\n     * Wrapping internal implementations for virtually all methods presented in this class.\n     */\n    protected final transient DefaultMQProducerImpl defaultMQProducerImpl;\n    private final Logger logger = LoggerFactory.getLogger(DefaultMQProducer.class);\n    private final Set<Integer> retryResponseCodes = new CopyOnWriteArraySet<>(Arrays.asList(\n        ResponseCode.TOPIC_NOT_EXIST,\n        ResponseCode.SERVICE_NOT_AVAILABLE,\n        ResponseCode.SYSTEM_ERROR,\n        ResponseCode.SYSTEM_BUSY,\n        ResponseCode.NO_PERMISSION,\n        ResponseCode.NO_BUYER_ID,\n        ResponseCode.NOT_IN_CURRENT_UNIT,\n        ResponseCode.GO_AWAY\n    ));\n\n    /**\n     * Producer group conceptually aggregates all producer instances of exactly same role, which is particularly\n     * important when transactional messages are involved. </p>\n     * <p>\n     * For non-transactional messages, it does not matter as long as it's unique per process. </p>\n     * <p>\n     * See <a href=\"https://rocketmq.apache.org/docs/introduction/02concepts\">core concepts</a> for more discussion.\n     */\n    private String producerGroup;\n\n    /**\n     * Topics that need to be initialized for transaction producer\n     */\n    private List<String> topics;\n\n    /**\n     * Just for testing or demo program\n     */\n    private String createTopicKey = TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC;\n\n    /**\n     * Number of queues to create per default topic.\n     */\n    private volatile int defaultTopicQueueNums = 4;\n\n    /**\n     * Timeout for sending messages.\n     */\n    private int sendMsgTimeout = 3000;\n\n    /**\n     * Max timeout for sending messages per request.\n     */\n    private int sendMsgMaxTimeoutPerRequest = -1;\n\n    /**\n     * Compress message body threshold, namely, message body larger than 4k will be compressed on default.\n     */\n    private int compressMsgBodyOverHowmuch = 1024 * 4;\n\n    /**\n     * Maximum number of retry to perform internally before claiming sending failure in synchronous mode. </p>\n     * <p>\n     * This may potentially cause message duplication which is up to application developers to resolve.\n     */\n    private int retryTimesWhenSendFailed = 2;\n\n    /**\n     * Maximum number of retry to perform internally before claiming sending failure in asynchronous mode. </p>\n     * <p>\n     * This may potentially cause message duplication which is up to application developers to resolve.\n     */\n    private int retryTimesWhenSendAsyncFailed = 2;\n\n    /**\n     * Indicate whether to retry another broker on sending failure internally.\n     */\n    private boolean retryAnotherBrokerWhenNotStoreOK = false;\n\n    /**\n     * Maximum allowed message body size in bytes.\n     */\n    private int maxMessageSize = 1024 * 1024 * 4; // 4M\n\n    /**\n     * Interface of asynchronous transfer data\n     */\n    private TraceDispatcher traceDispatcher = null;\n\n    /**\n     * Switch flag instance for automatic batch message\n     */\n    private boolean autoBatch = false;\n    /**\n     * Instance for batching message automatically\n     */\n    private ProduceAccumulator produceAccumulator = null;\n\n    /**\n     * Indicate whether to block message when asynchronous sending traffic is too heavy.\n     */\n    private boolean enableBackpressureForAsyncMode = false;\n\n    /**\n     * on BackpressureForAsyncMode, limit maximum number of on-going sending async messages\n     * default is 1024\n     */\n    private int backPressureForAsyncSendNum = 1024;\n\n    /**\n     * on BackpressureForAsyncMode, limit maximum message size of on-going sending async messages\n     * default is 100M\n     */\n    private int backPressureForAsyncSendSize = 100 * 1024 * 1024;\n\n    /**\n     * Maximum hold time of accumulator.\n     */\n    private int batchMaxDelayMs = -1;\n\n    /**\n     * Maximum accumulation message body size for a single messageAccumulation.\n     */\n    private long batchMaxBytes = -1;\n\n    /**\n     * Maximum message body size for produceAccumulator.\n     */\n    private long totalBatchMaxBytes = -1;\n\n    private RPCHook rpcHook = null;\n\n    /**\n     *  backPressureForAsyncSendNum is guaranteed to be modified at runtime and no new requests are allowed\n     */\n    private final ReadWriteCASLock backPressureForAsyncSendNumLock = new ReadWriteCASLock();\n\n    /**\n     * backPressureForAsyncSendSize is guaranteed to be modified at runtime and no new requests are allowed\n     */\n    private final ReadWriteCASLock backPressureForAsyncSendSizeLock = new ReadWriteCASLock();\n\n    /**\n     * Compress level of compress algorithm.\n     */\n    private int compressLevel = Integer.parseInt(System.getProperty(MixAll.MESSAGE_COMPRESS_LEVEL, \"5\"));\n\n    /**\n     * Compress type of compress algorithm, default using ZLIB.\n     */\n    private CompressionType compressType = CompressionType.of(System.getProperty(MixAll.MESSAGE_COMPRESS_TYPE, \"ZLIB\"));\n\n    /**\n     * Compressor of compress algorithm.\n     */\n    private Compressor compressor = CompressorFactory.getCompressor(compressType);\n\n    /**\n     * Default constructor.\n     */\n    public DefaultMQProducer() {\n        this(MixAll.DEFAULT_PRODUCER_GROUP);\n    }\n\n    /**\n     * Constructor specifying the RPC hook.\n     *\n     * @param rpcHook RPC hook to execute per each remoting command execution.\n     */\n    public DefaultMQProducer(RPCHook rpcHook) {\n        this(MixAll.DEFAULT_PRODUCER_GROUP, rpcHook);\n    }\n\n    /**\n     * Constructor specifying producer group.\n     *\n     * @param producerGroup Producer group, see the name-sake field.\n     */\n    public DefaultMQProducer(final String producerGroup) {\n        this(producerGroup, (RPCHook) null);\n    }\n\n    /**\n     * Constructor specifying both producer group and RPC hook.\n     *\n     * @param producerGroup Producer group, see the name-sake field.\n     * @param rpcHook       RPC hook to execute per each remoting command execution.\n     */\n    public DefaultMQProducer(final String producerGroup, RPCHook rpcHook) {\n        this(producerGroup, rpcHook, null);\n    }\n\n    /**\n     * Constructor specifying namespace, producer group, topics and RPC hook.\n     *\n     * @param producerGroup Producer group, see the name-sake field.\n     * @param rpcHook       RPC hook to execute per each remoting command execution.\n     * @param topics        Topic that needs to be initialized for routing\n     */\n    public DefaultMQProducer(final String producerGroup, RPCHook rpcHook,\n        final List<String> topics) {\n        this(producerGroup, rpcHook, topics, false, null);\n    }\n\n    /**\n     * Constructor specifying producer group, enabled msgTrace flag and customized trace topic name.\n     *\n     * @param producerGroup        Producer group, see the name-sake field.\n     * @param enableMsgTrace       Switch flag instance for message trace.\n     * @param customizedTraceTopic The name value of message trace topic.If you don't config,you can use the default\n     *                             trace topic name.\n     */\n    public DefaultMQProducer(final String producerGroup, boolean enableMsgTrace, final String customizedTraceTopic) {\n        this(producerGroup, null, enableMsgTrace, customizedTraceTopic);\n    }\n\n    /**\n     * Constructor specifying producer group.\n     *\n     * @param producerGroup        Producer group, see the name-sake field.\n     * @param rpcHook              RPC hook to execute per each remoting command execution.\n     * @param enableMsgTrace       Switch flag instance for message trace.\n     * @param customizedTraceTopic The name value of message trace topic.If you don't config,you can use the default\n     *                             trace topic name.\n     */\n    public DefaultMQProducer(final String producerGroup, RPCHook rpcHook, boolean enableMsgTrace,\n        final String customizedTraceTopic) {\n        this(producerGroup, rpcHook, null, enableMsgTrace, customizedTraceTopic);\n    }\n\n    /**\n     * Constructor specifying namespace, producer group, topics, RPC hook, enabled msgTrace flag and customized trace topic\n     * name.\n     *\n     * @param producerGroup        Producer group, see the name-sake field.\n     * @param rpcHook              RPC hook to execute per each remoting command execution.\n     * @param topics               Topic that needs to be initialized for routing\n     * @param enableMsgTrace       Switch flag instance for message trace.\n     * @param customizedTraceTopic The name value of message trace topic.If you don't config,you can use the default\n     *                             trace topic name.\n     */\n    public DefaultMQProducer(final String producerGroup, RPCHook rpcHook, final List<String> topics,\n        boolean enableMsgTrace, final String customizedTraceTopic) {\n        this.producerGroup = producerGroup;\n        this.rpcHook = rpcHook;\n        this.topics = topics;\n        this.enableTrace = enableMsgTrace;\n        this.traceTopic = customizedTraceTopic;\n        defaultMQProducerImpl = new DefaultMQProducerImpl(this, rpcHook);\n    }\n\n    /**\n     * Constructor specifying producer group.\n     *\n     * @param namespace     Namespace for this MQ Producer instance.\n     * @param producerGroup Producer group, see the name-sake field.\n     */\n    @Deprecated\n    public DefaultMQProducer(final String namespace, final String producerGroup) {\n        this(namespace, producerGroup, null);\n    }\n\n    /**\n     * Constructor specifying namespace, producer group and RPC hook.\n     *\n     * @param namespace     Namespace for this MQ Producer instance.\n     * @param producerGroup Producer group, see the name-sake field.\n     * @param rpcHook       RPC hook to execute per each remoting command execution.\n     */\n    @Deprecated\n    public DefaultMQProducer(final String namespace, final String producerGroup, RPCHook rpcHook) {\n        this.namespace = namespace;\n        this.producerGroup = producerGroup;\n        this.rpcHook = rpcHook;\n        defaultMQProducerImpl = new DefaultMQProducerImpl(this, rpcHook);\n    }\n\n    /**\n     * Constructor specifying namespace, producer group, RPC hook, enabled msgTrace flag and customized trace topic\n     * name.\n     *\n     * @param namespace            Namespace for this MQ Producer instance.\n     * @param producerGroup        Producer group, see the name-sake field.\n     * @param rpcHook              RPC hook to execute per each remoting command execution.\n     * @param enableMsgTrace       Switch flag instance for message trace.\n     * @param customizedTraceTopic The name value of message trace topic.If you don't config,you can use the default\n     *                             trace topic name.\n     */\n    @Deprecated\n    public DefaultMQProducer(final String namespace, final String producerGroup, RPCHook rpcHook,\n        boolean enableMsgTrace, final String customizedTraceTopic) {\n        this(namespace, producerGroup, rpcHook);\n        //if client open the message trace feature\n        this.enableTrace = enableMsgTrace;\n        this.traceTopic = customizedTraceTopic;\n    }\n\n    /**\n     * Start this producer instance. </p>\n     *\n     * <strong> Much internal initializing procedures are carried out to make this instance prepared, thus, it's a must\n     * to invoke this method before sending or querying messages. </strong> </p>\n     *\n     * @throws MQClientException if there is any unexpected error.\n     */\n    @Override\n    public void start() throws MQClientException {\n        this.setProducerGroup(withNamespace(this.producerGroup));\n        this.defaultMQProducerImpl.start();\n        if (this.produceAccumulator != null) {\n            this.produceAccumulator.start();\n        }\n        if (enableTrace) {\n            try {\n                AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(producerGroup, TraceDispatcher.Type.PRODUCE, getTraceMsgBatchNum(), traceTopic, rpcHook);\n                dispatcher.setHostProducer(this.defaultMQProducerImpl);\n                dispatcher.setNamespaceV2(this.namespaceV2);\n                traceDispatcher = dispatcher;\n                this.defaultMQProducerImpl.registerSendMessageHook(\n                    new SendMessageTraceHookImpl(traceDispatcher));\n                this.defaultMQProducerImpl.registerEndTransactionHook(\n                    new EndTransactionTraceHookImpl(traceDispatcher));\n                this.defaultMQProducerImpl.getMqClientFactory().getMQClientAPIImpl().getRemotingClient()\n                    .registerRPCHook(new DefaultRecallMessageTraceHook(traceDispatcher));\n            } catch (Throwable e) {\n                logger.error(\"system mqtrace hook init failed ,maybe can't send msg trace data\");\n            }\n        }\n        if (null != traceDispatcher) {\n            if (traceDispatcher instanceof AsyncTraceDispatcher) {\n                ((AsyncTraceDispatcher) traceDispatcher).getTraceProducer().setUseTLS(isUseTLS());\n            }\n            try {\n                traceDispatcher.start(this.getNamesrvAddr(), this.getAccessChannel());\n            } catch (MQClientException e) {\n                logger.warn(\"trace dispatcher start failed \", e);\n            }\n        }\n    }\n\n    /**\n     * This method shuts down this producer instance and releases related resources.\n     */\n    @Override\n    public void shutdown() {\n        this.defaultMQProducerImpl.shutdown();\n        if (this.produceAccumulator != null) {\n            this.produceAccumulator.shutdown();\n        }\n        if (null != traceDispatcher) {\n            traceDispatcher.shutdown();\n        }\n    }\n\n    /**\n     * Fetch message queues of topic <code>topic</code>, to which we may send/publish messages.\n     *\n     * @param topic Topic to fetch.\n     * @return List of message queues readily to send messages to\n     * @throws MQClientException if there is any client error.\n     */\n    @Override\n    public List<MessageQueue> fetchPublishMessageQueues(String topic) throws MQClientException {\n        return this.defaultMQProducerImpl.fetchPublishMessageQueues(withNamespace(topic));\n    }\n\n    private boolean canBatch(Message msg) {\n        // produceAccumulator is full\n        if (!produceAccumulator.tryAddMessage(msg)) {\n            return false;\n        }\n        // delay message do not support batch processing\n        if (msg.getDelayTimeLevel() > 0 || msg.getDelayTimeMs() > 0 || msg.getDelayTimeSec() > 0 || msg.getDeliverTimeMs() > 0) {\n            return false;\n        }\n        // retry message do not support batch processing\n        if (msg.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n            return false;\n        }\n        // message which have been assigned to producer group do not support batch processing\n        if (msg.getProperties().containsKey(MessageConst.PROPERTY_PRODUCER_GROUP)) {\n            return false;\n        }\n        return true;\n    }\n\n    /**\n     * Send message in synchronous mode. This method returns only when the sending procedure totally completes. </p>\n     *\n     * <strong>Warn:</strong> this method has internal retry-mechanism, that is, internal implementation will retry\n     * {@link #retryTimesWhenSendFailed} times before claiming failure. As a result, multiple messages may be potentially\n     * delivered to broker(s). It's up to the application developers to resolve potential duplication issue.\n     *\n     * @param msg Message to send.\n     * @return {@link SendResult} instance to inform senders details of the deliverable, say Message ID of the message,\n     * {@link SendStatus} indicating broker storage/replication status, message queue sent to, etc.\n     * @throws MQClientException    if there is any client error.\n     * @throws RemotingException    if there is any network-tier error.\n     * @throws MQBrokerException    if there is any error with broker.\n     * @throws InterruptedException if the sending thread is interrupted.\n     */\n    @Override\n    public SendResult send(\n        Message msg) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        if (this.getAutoBatch() && !(msg instanceof MessageBatch)) {\n            return sendByAccumulator(msg, null, null);\n        } else {\n            return sendDirect(msg, null, null);\n        }\n    }\n\n    /**\n     * Same to {@link #send(Message)} with send timeout specified in addition.\n     *\n     * @param msg     Message to send.\n     * @param timeout send timeout.\n     * @return {@link SendResult} instance to inform senders details of the deliverable, say Message ID of the message,\n     * {@link SendStatus} indicating broker storage/replication status, message queue sent to, etc.\n     * @throws MQClientException    if there is any client error.\n     * @throws RemotingException    if there is any network-tier error.\n     * @throws MQBrokerException    if there is any error with broker.\n     * @throws InterruptedException if the sending thread is interrupted.\n     */\n    @Override\n    public SendResult send(Message msg,\n        long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        return this.defaultMQProducerImpl.send(msg, timeout);\n    }\n\n    /**\n     * Send message to broker asynchronously. </p>\n     * <p>\n     * This method returns immediately. On sending completion, <code>sendCallback</code> will be executed. </p>\n     * <p>\n     * Similar to {@link #send(Message)}, internal implementation would potentially retry up to {@link\n     * #retryTimesWhenSendAsyncFailed} times before claiming sending failure, which may yield message duplication and\n     * application developers are the one to resolve this potential issue.\n     *\n     * @param msg          Message to send.\n     * @param sendCallback Callback to execute on sending completed, either successful or unsuccessful.\n     * @throws MQClientException    if there is any client error.\n     * @throws RemotingException    if there is any network-tier error.\n     * @throws InterruptedException if the sending thread is interrupted.\n     */\n    @Override\n    public void send(Message msg,\n        SendCallback sendCallback) throws MQClientException, RemotingException, InterruptedException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        try {\n            if (this.getAutoBatch() && !(msg instanceof MessageBatch)) {\n                sendByAccumulator(msg, null, sendCallback);\n            } else {\n                sendDirect(msg, null, sendCallback);\n            }\n        } catch (Throwable e) {\n            sendCallback.onException(e);\n        }\n    }\n\n    /**\n     * Same to {@link #send(Message, SendCallback)} with send timeout specified in addition.\n     *\n     * @param msg          message to send.\n     * @param sendCallback Callback to execute.\n     * @param timeout      send timeout.\n     * @throws MQClientException    if there is any client error.\n     * @throws RemotingException    if there is any network-tier error.\n     * @throws InterruptedException if the sending thread is interrupted.\n     */\n    @Override\n    public void send(Message msg, SendCallback sendCallback, long timeout)\n        throws MQClientException, RemotingException, InterruptedException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        this.defaultMQProducerImpl.send(msg, sendCallback, timeout);\n    }\n\n    /**\n     * Similar to <a href=\"https://en.wikipedia.org/wiki/User_Datagram_Protocol\">UDP</a>, this method won't wait for\n     * acknowledgement from broker before return. Obviously, it has maximums throughput yet potentials of message loss.\n     *\n     * @param msg Message to send.\n     * @throws MQClientException    if there is any client error.\n     * @throws RemotingException    if there is any network-tier error.\n     * @throws InterruptedException if the sending thread is interrupted.\n     */\n    @Override\n    public void sendOneway(Message msg) throws MQClientException, RemotingException, InterruptedException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        this.defaultMQProducerImpl.sendOneway(msg);\n    }\n\n    /**\n     * Same to {@link #send(Message)} with target message queue specified in addition.\n     *\n     * @param msg Message to send.\n     * @param mq  Target message queue.\n     * @return {@link SendResult} instance to inform senders details of the deliverable, say Message ID of the message,\n     * {@link SendStatus} indicating broker storage/replication status, message queue sent to, etc.\n     * @throws MQClientException    if there is any client error.\n     * @throws RemotingException    if there is any network-tier error.\n     * @throws MQBrokerException    if there is any error with broker.\n     * @throws InterruptedException if the sending thread is interrupted.\n     */\n    @Override\n    public SendResult send(Message msg, MessageQueue mq)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        mq = queueWithNamespace(mq);\n        if (this.getAutoBatch() && !(msg instanceof MessageBatch)) {\n            return sendByAccumulator(msg, mq, null);\n        } else {\n            return sendDirect(msg, mq, null);\n        }\n    }\n\n    /**\n     * Same to {@link #send(Message)} with target message queue and send timeout specified.\n     *\n     * @param msg     Message to send.\n     * @param mq      Target message queue.\n     * @param timeout send timeout.\n     * @return {@link SendResult} instance to inform senders details of the deliverable, say Message ID of the message,\n     * {@link SendStatus} indicating broker storage/replication status, message queue sent to, etc.\n     * @throws MQClientException    if there is any client error.\n     * @throws RemotingException    if there is any network-tier error.\n     * @throws MQBrokerException    if there is any error with broker.\n     * @throws InterruptedException if the sending thread is interrupted.\n     */\n    @Override\n    public SendResult send(Message msg, MessageQueue mq, long timeout)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        return this.defaultMQProducerImpl.send(msg, queueWithNamespace(mq), timeout);\n    }\n\n    /**\n     * Same to {@link #send(Message, SendCallback)} with target message queue specified.\n     *\n     * @param msg          Message to send.\n     * @param mq           Target message queue.\n     * @param sendCallback Callback to execute on sending completed, either successful or unsuccessful.\n     * @throws MQClientException    if there is any client error.\n     * @throws RemotingException    if there is any network-tier error.\n     * @throws InterruptedException if the sending thread is interrupted.\n     */\n    @Override\n    public void send(Message msg, MessageQueue mq, SendCallback sendCallback)\n        throws MQClientException, RemotingException, InterruptedException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        mq = queueWithNamespace(mq);\n        try {\n            if (this.getAutoBatch() && !(msg instanceof MessageBatch)) {\n                sendByAccumulator(msg, mq, sendCallback);\n            } else {\n                sendDirect(msg, mq, sendCallback);\n            }\n        } catch (MQBrokerException e) {\n            // ignore\n        }\n    }\n\n    /**\n     * Same to {@link #send(Message, SendCallback)} with target message queue and send timeout specified.\n     *\n     * @param msg          Message to send.\n     * @param mq           Target message queue.\n     * @param sendCallback Callback to execute on sending completed, either successful or unsuccessful.\n     * @param timeout      Send timeout.\n     * @throws MQClientException    if there is any client error.\n     * @throws RemotingException    if there is any network-tier error.\n     * @throws InterruptedException if the sending thread is interrupted.\n     */\n    @Override\n    public void send(Message msg, MessageQueue mq, SendCallback sendCallback, long timeout)\n        throws MQClientException, RemotingException, InterruptedException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        this.defaultMQProducerImpl.send(msg, queueWithNamespace(mq), sendCallback, timeout);\n    }\n\n    /**\n     * Same to {@link #sendOneway(Message)} with target message queue specified.\n     *\n     * @param msg Message to send.\n     * @param mq  Target message queue.\n     * @throws MQClientException    if there is any client error.\n     * @throws RemotingException    if there is any network-tier error.\n     * @throws InterruptedException if the sending thread is interrupted.\n     */\n    @Override\n    public void sendOneway(Message msg,\n        MessageQueue mq) throws MQClientException, RemotingException, InterruptedException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        this.defaultMQProducerImpl.sendOneway(msg, queueWithNamespace(mq));\n    }\n\n    /**\n     * Same to {@link #send(Message)} with message queue selector specified.\n     *\n     * @param msg      Message to send.\n     * @param selector Message queue selector, through which we get target message queue to deliver message to.\n     * @param arg      Argument to work along with message queue selector.\n     * @return {@link SendResult} instance to inform senders details of the deliverable, say Message ID of the message,\n     * {@link SendStatus} indicating broker storage/replication status, message queue sent to, etc.\n     * @throws MQClientException    if there is any client error.\n     * @throws RemotingException    if there is any network-tier error.\n     * @throws MQBrokerException    if there is any error with broker.\n     * @throws InterruptedException if the sending thread is interrupted.\n     */\n    @Override\n    public SendResult send(Message msg, MessageQueueSelector selector, Object arg)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        MessageQueue mq = this.defaultMQProducerImpl.invokeMessageQueueSelector(msg, selector, arg, this.getSendMsgTimeout());\n        mq = queueWithNamespace(mq);\n        if (this.getAutoBatch() && !(msg instanceof MessageBatch)) {\n            return sendByAccumulator(msg, mq, null);\n        } else {\n            return sendDirect(msg, mq, null);\n        }\n    }\n\n    /**\n     * Same to {@link #send(Message, MessageQueueSelector, Object)} with send timeout specified.\n     *\n     * @param msg      Message to send.\n     * @param selector Message queue selector, through which we get target message queue to deliver message to.\n     * @param arg      Argument to work along with message queue selector.\n     * @param timeout  Send timeout.\n     * @return {@link SendResult} instance to inform senders details of the deliverable, say Message ID of the message,\n     * {@link SendStatus} indicating broker storage/replication status, message queue sent to, etc.\n     * @throws MQClientException    if there is any client error.\n     * @throws RemotingException    if there is any network-tier error.\n     * @throws MQBrokerException    if there is any error with broker.\n     * @throws InterruptedException if the sending thread is interrupted.\n     */\n    @Override\n    public SendResult send(Message msg, MessageQueueSelector selector, Object arg, long timeout)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        return this.defaultMQProducerImpl.send(msg, selector, arg, timeout);\n    }\n\n    /**\n     * Same to {@link #send(Message, SendCallback)} with message queue selector specified.\n     *\n     * @param msg          Message to send.\n     * @param selector     Message selector through which to get target message queue.\n     * @param arg          Argument used along with message queue selector.\n     * @param sendCallback callback to execute on sending completion.\n     * @throws MQClientException    if there is any client error.\n     * @throws RemotingException    if there is any network-tier error.\n     * @throws InterruptedException if the sending thread is interrupted.\n     */\n    @Override\n    public void send(Message msg, MessageQueueSelector selector, Object arg, SendCallback sendCallback)\n        throws MQClientException, RemotingException, InterruptedException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        try {\n            MessageQueue mq = this.defaultMQProducerImpl.invokeMessageQueueSelector(msg, selector, arg, this.getSendMsgTimeout());\n            mq = queueWithNamespace(mq);\n            if (this.getAutoBatch() && !(msg instanceof MessageBatch)) {\n                sendByAccumulator(msg, mq, sendCallback);\n            } else {\n                sendDirect(msg, mq, sendCallback);\n            }\n        } catch (Throwable e) {\n            sendCallback.onException(e);\n        }\n    }\n\n    /**\n     * Same to {@link #send(Message, MessageQueueSelector, Object, SendCallback)} with timeout specified.\n     *\n     * @param msg          Message to send.\n     * @param selector     Message selector through which to get target message queue.\n     * @param arg          Argument used along with message queue selector.\n     * @param sendCallback callback to execute on sending completion.\n     * @param timeout      Send timeout.\n     * @throws MQClientException    if there is any client error.\n     * @throws RemotingException    if there is any network-tier error.\n     * @throws InterruptedException if the sending thread is interrupted.\n     */\n    @Override\n    public void send(Message msg, MessageQueueSelector selector, Object arg, SendCallback sendCallback, long timeout)\n        throws MQClientException, RemotingException, InterruptedException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        this.defaultMQProducerImpl.send(msg, selector, arg, sendCallback, timeout);\n    }\n\n    public SendResult sendDirect(Message msg, MessageQueue mq,\n        SendCallback sendCallback) throws MQClientException, RemotingException, InterruptedException, MQBrokerException {\n        // send in sync mode\n        if (sendCallback == null) {\n            if (mq == null) {\n                return this.defaultMQProducerImpl.send(msg);\n            } else {\n                return this.defaultMQProducerImpl.send(msg, mq);\n            }\n        } else {\n            if (mq == null) {\n                this.defaultMQProducerImpl.send(msg, sendCallback);\n            } else {\n                this.defaultMQProducerImpl.send(msg, mq, sendCallback);\n            }\n            return null;\n        }\n    }\n\n    public SendResult sendByAccumulator(Message msg, MessageQueue mq,\n        SendCallback sendCallback) throws MQClientException, RemotingException, InterruptedException, MQBrokerException {\n        // check whether it can batch\n        if (!canBatch(msg)) {\n            return sendDirect(msg, mq, sendCallback);\n        } else {\n            Validators.checkMessage(msg, this);\n            MessageClientIDSetter.setUniqID(msg);\n            if (sendCallback == null) {\n                return this.produceAccumulator.send(msg, mq, this);\n            } else {\n                this.produceAccumulator.send(msg, mq, sendCallback, this);\n                return null;\n            }\n        }\n    }\n\n    /**\n     * Send request message in synchronous mode. This method returns only when the consumer consume the request message and reply a message. </p>\n     *\n     * <strong>Warn:</strong> this method has internal retry-mechanism, that is, internal implementation will retry\n     * {@link #retryTimesWhenSendFailed} times before claiming failure. As a result, multiple messages may be potentially\n     * delivered to broker(s). It's up to the application developers to resolve potential duplication issue.\n     *\n     * @param msg     request message to send\n     * @param timeout request timeout\n     * @return reply message\n     * @throws MQClientException       if there is any client error.\n     * @throws RemotingException       if there is any network-tier error.\n     * @throws MQBrokerException       if there is any broker error.\n     * @throws InterruptedException    if the thread is interrupted.\n     * @throws RequestTimeoutException if request timeout.\n     */\n    @Override\n    public Message request(final Message msg, final long timeout) throws RequestTimeoutException, MQClientException,\n        RemotingException, MQBrokerException, InterruptedException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        return this.defaultMQProducerImpl.request(msg, timeout);\n    }\n\n    /**\n     * Request asynchronously. </p>\n     * This method returns immediately. On receiving reply message, <code>requestCallback</code> will be executed. </p>\n     * <p>\n     * Similar to {@link #request(Message, long)}, internal implementation would potentially retry up to {@link\n     * #retryTimesWhenSendAsyncFailed} times before claiming sending failure, which may yield message duplication and\n     * application developers are the one to resolve this potential issue.\n     *\n     * @param msg             request message to send\n     * @param requestCallback callback to execute on request completion.\n     * @param timeout         request timeout\n     * @throws MQClientException    if there is any client error.\n     * @throws RemotingException    if there is any network-tier error.\n     * @throws InterruptedException if the thread is interrupted.\n     * @throws MQBrokerException    if there is any broker error.\n     */\n    @Override\n    public void request(final Message msg, final RequestCallback requestCallback, final long timeout)\n        throws MQClientException, RemotingException, InterruptedException, MQBrokerException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        this.defaultMQProducerImpl.request(msg, requestCallback, timeout);\n    }\n\n    /**\n     * Same to {@link #request(Message, long)}  with message queue selector specified.\n     *\n     * @param msg      request message to send\n     * @param selector message queue selector, through which we get target message queue to deliver message to.\n     * @param arg      argument to work along with message queue selector.\n     * @param timeout  timeout of request.\n     * @return reply message\n     * @throws MQClientException       if there is any client error.\n     * @throws RemotingException       if there is any network-tier error.\n     * @throws MQBrokerException       if there is any broker error.\n     * @throws InterruptedException    if the thread is interrupted.\n     * @throws RequestTimeoutException if request timeout.\n     */\n    @Override\n    public Message request(final Message msg, final MessageQueueSelector selector, final Object arg,\n        final long timeout) throws MQClientException, RemotingException, MQBrokerException,\n        InterruptedException, RequestTimeoutException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        return this.defaultMQProducerImpl.request(msg, selector, arg, timeout);\n    }\n\n    /**\n     * Same to {@link #request(Message, RequestCallback, long)} with target message selector specified.\n     *\n     * @param msg             request message to send\n     * @param selector        message queue selector, through which we get target message queue to deliver message to.\n     * @param arg             argument to work along with message queue selector.\n     * @param requestCallback callback to execute on request completion.\n     * @param timeout         timeout of request.\n     * @throws MQClientException    if there is any client error.\n     * @throws RemotingException    if there is any network-tier error.\n     * @throws InterruptedException if the thread is interrupted.\n     * @throws MQBrokerException    if there is any broker error.\n     */\n    @Override\n    public void request(final Message msg, final MessageQueueSelector selector, final Object arg,\n        final RequestCallback requestCallback, final long timeout) throws MQClientException, RemotingException,\n        InterruptedException, MQBrokerException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        this.defaultMQProducerImpl.request(msg, selector, arg, requestCallback, timeout);\n    }\n\n    /**\n     * Same to {@link #request(Message, long)}  with target message queue specified in addition.\n     *\n     * @param msg     request message to send\n     * @param mq      target message queue.\n     * @param timeout request timeout\n     * @throws MQClientException       if there is any client error.\n     * @throws RemotingException       if there is any network-tier error.\n     * @throws MQBrokerException       if there is any broker error.\n     * @throws InterruptedException    if the thread is interrupted.\n     * @throws RequestTimeoutException if request timeout.\n     */\n    @Override\n    public Message request(final Message msg, final MessageQueue mq, final long timeout)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException, RequestTimeoutException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        return this.defaultMQProducerImpl.request(msg, mq, timeout);\n    }\n\n    /**\n     * Same to {@link #request(Message, RequestCallback, long)} with target message queue specified.\n     *\n     * @param msg             request message to send\n     * @param mq              target message queue.\n     * @param requestCallback callback to execute on request completion.\n     * @param timeout         timeout of request.\n     * @throws MQClientException    if there is any client error.\n     * @throws RemotingException    if there is any network-tier error.\n     * @throws InterruptedException if the thread is interrupted.\n     * @throws MQBrokerException    if there is any broker error.\n     */\n    @Override\n    public void request(final Message msg, final MessageQueue mq, final RequestCallback requestCallback, long timeout)\n        throws MQClientException, RemotingException, InterruptedException, MQBrokerException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        this.defaultMQProducerImpl.request(msg, mq, requestCallback, timeout);\n    }\n\n    /**\n     * Same to {@link #sendOneway(Message)} with message queue selector specified.\n     *\n     * @param msg      Message to send.\n     * @param selector Message queue selector, through which to determine target message queue to deliver message\n     * @param arg      Argument used along with message queue selector.\n     * @throws MQClientException    if there is any client error.\n     * @throws RemotingException    if there is any network-tier error.\n     * @throws InterruptedException if the sending thread is interrupted.\n     */\n    @Override\n    public void sendOneway(Message msg, MessageQueueSelector selector, Object arg)\n        throws MQClientException, RemotingException, InterruptedException {\n        msg.setTopic(withNamespace(msg.getTopic()));\n        this.defaultMQProducerImpl.sendOneway(msg, selector, arg);\n    }\n\n    /**\n     * This method is used to send transactional messages.\n     *\n     * @param msg Transactional message to send.\n     * @param arg Argument used along with local transaction executor.\n     * @return Transaction result.\n     * @throws MQClientException\n     */\n    @Override\n    public TransactionSendResult sendMessageInTransaction(Message msg,\n        Object arg) throws MQClientException {\n        throw new RuntimeException(\"sendMessageInTransaction not implement, please use TransactionMQProducer class\");\n    }\n\n    /**\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     *\n     * @param key        accessKey\n     * @param newTopic   topic name\n     * @param queueNum   topic's queue number\n     * @param attributes\n     * @throws MQClientException if there is any client error.\n     */\n    @Deprecated\n    @Override\n    public void createTopic(String key, String newTopic, int queueNum,\n        Map<String, String> attributes) throws MQClientException {\n        createTopic(key, withNamespace(newTopic), queueNum, 0, null);\n    }\n\n    /**\n     * Create a topic on broker. This method will be removed in a certain version after April 5, 2020, so please do not\n     * use this method.\n     *\n     * @param key          accessKey\n     * @param newTopic     topic name\n     * @param queueNum     topic's queue number\n     * @param topicSysFlag topic system flag\n     * @param attributes\n     * @throws MQClientException if there is any client error.\n     */\n    @Deprecated\n    @Override\n    public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag,\n        Map<String, String> attributes) throws MQClientException {\n        this.defaultMQProducerImpl.createTopic(key, withNamespace(newTopic), queueNum, topicSysFlag);\n    }\n\n    /**\n     * Search consume queue offset of the given time stamp.\n     *\n     * @param mq        Instance of MessageQueue\n     * @param timestamp from when in milliseconds.\n     * @return Consume queue offset.\n     * @throws MQClientException if there is any client error.\n     */\n    @Override\n    public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException {\n        return this.defaultMQProducerImpl.searchOffset(queueWithNamespace(mq), timestamp);\n    }\n\n    /**\n     * Query maximum offset of the given message queue.\n     * <p>\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     *\n     * @param mq Instance of MessageQueue\n     * @return maximum offset of the given consume queue.\n     * @throws MQClientException if there is any client error.\n     */\n    @Deprecated\n    @Override\n    public long maxOffset(MessageQueue mq) throws MQClientException {\n        return this.defaultMQProducerImpl.maxOffset(queueWithNamespace(mq));\n    }\n\n    /**\n     * Query minimum offset of the given message queue.\n     * <p>\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     *\n     * @param mq Instance of MessageQueue\n     * @return minimum offset of the given message queue.\n     * @throws MQClientException if there is any client error.\n     */\n    @Deprecated\n    @Override\n    public long minOffset(MessageQueue mq) throws MQClientException {\n        return this.defaultMQProducerImpl.minOffset(queueWithNamespace(mq));\n    }\n\n    /**\n     * Query the earliest message store time.\n     * <p>\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     *\n     * @param mq Instance of MessageQueue\n     * @return earliest message store time.\n     * @throws MQClientException if there is any client error.\n     */\n    @Deprecated\n    @Override\n    public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException {\n        return this.defaultMQProducerImpl.earliestMsgStoreTime(queueWithNamespace(mq));\n    }\n\n    /**\n     * Query message by key.\n     * <p>\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     *\n     * @param topic  message topic\n     * @param key    message key index word\n     * @param maxNum max message number\n     * @param begin  from when\n     * @param end    to when\n     * @return QueryResult instance contains matched messages.\n     * @throws MQClientException    if there is any client error.\n     * @throws InterruptedException if the thread is interrupted.\n     */\n    @Deprecated\n    @Override\n    public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end)\n        throws MQClientException, InterruptedException {\n        return this.defaultMQProducerImpl.queryMessage(withNamespace(topic), key, maxNum, begin, end);\n    }\n\n    /**\n     * Query message of the given message ID.\n     * <p>\n     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.\n     *\n     * @param topic Topic\n     * @param msgId Message ID\n     * @return Message specified.\n     * @throws MQBrokerException    if there is any broker error.\n     * @throws MQClientException    if there is any client error.\n     * @throws RemotingException    if there is any network-tier error.\n     * @throws InterruptedException if the sending thread is interrupted.\n     */\n    @Deprecated\n    @Override\n    public MessageExt viewMessage(String topic,\n        String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {\n        try {\n            return this.defaultMQProducerImpl.viewMessage(topic, msgId);\n        } catch (Exception ignored) {\n        }\n        return this.defaultMQProducerImpl.queryMessageByUniqKey(withNamespace(topic), msgId);\n    }\n\n    @Override\n    public SendResult send(\n        Collection<Message> msgs) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        return this.defaultMQProducerImpl.send(batch(msgs));\n    }\n\n    @Override\n    public SendResult send(Collection<Message> msgs,\n        long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        return this.defaultMQProducerImpl.send(batch(msgs), timeout);\n    }\n\n    @Override\n    public SendResult send(Collection<Message> msgs,\n        MessageQueue messageQueue) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        return this.defaultMQProducerImpl.send(batch(msgs), messageQueue);\n    }\n\n    @Override\n    public SendResult send(Collection<Message> msgs, MessageQueue messageQueue,\n        long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        return this.defaultMQProducerImpl.send(batch(msgs), messageQueue, timeout);\n    }\n\n    @Override\n    public void send(Collection<Message> msgs,\n        SendCallback sendCallback) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        this.defaultMQProducerImpl.send(batch(msgs), sendCallback);\n    }\n\n    @Override\n    public void send(Collection<Message> msgs, SendCallback sendCallback,\n        long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        this.defaultMQProducerImpl.send(batch(msgs), sendCallback, timeout);\n    }\n\n    @Override\n    public void send(Collection<Message> msgs, MessageQueue mq,\n        SendCallback sendCallback) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        this.defaultMQProducerImpl.send(batch(msgs), queueWithNamespace(mq), sendCallback);\n    }\n\n    @Override\n    public void send(Collection<Message> msgs, MessageQueue mq,\n        SendCallback sendCallback,\n        long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        this.defaultMQProducerImpl.send(batch(msgs), queueWithNamespace(mq), sendCallback, timeout);\n    }\n\n    @Override\n    public String recallMessage(String topic, String recallHandle)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {\n        return this.defaultMQProducerImpl.recallMessage(withNamespace(topic), recallHandle);\n    }\n\n    /**\n     * Sets an Executor to be used for executing callback methods.\n     *\n     * @param callbackExecutor the instance of Executor\n     */\n    public void setCallbackExecutor(final ExecutorService callbackExecutor) {\n        this.defaultMQProducerImpl.setCallbackExecutor(callbackExecutor);\n    }\n\n    /**\n     * Sets an Executor to be used for executing asynchronous send.\n     *\n     * @param asyncSenderExecutor the instance of Executor\n     */\n    public void setAsyncSenderExecutor(final ExecutorService asyncSenderExecutor) {\n        this.defaultMQProducerImpl.setAsyncSenderExecutor(asyncSenderExecutor);\n    }\n\n    /**\n     * Add response code for retrying.\n     *\n     * @param responseCode response code, {@link ResponseCode}\n     */\n    public void addRetryResponseCode(int responseCode) {\n        this.retryResponseCodes.add(responseCode);\n    }\n\n    private MessageBatch batch(Collection<Message> msgs) throws MQClientException {\n        MessageBatch msgBatch;\n        try {\n            msgBatch = MessageBatch.generateFromList(msgs);\n            for (Message message : msgBatch) {\n                Validators.checkMessage(message, this);\n                MessageClientIDSetter.setUniqID(message);\n                message.setTopic(withNamespace(message.getTopic()));\n            }\n            MessageClientIDSetter.setUniqID(msgBatch);\n            msgBatch.setBody(msgBatch.encode());\n        } catch (Exception e) {\n            throw new MQClientException(\"Failed to initiate the MessageBatch\", e);\n        }\n        msgBatch.setTopic(withNamespace(msgBatch.getTopic()));\n        return msgBatch;\n    }\n\n    public int getBatchMaxDelayMs() {\n        if (this.produceAccumulator == null) {\n            return 0;\n        }\n        return produceAccumulator.getBatchMaxDelayMs();\n    }\n\n    public void batchMaxDelayMs(int holdMs) {\n        this.batchMaxDelayMs = holdMs;\n        if (this.produceAccumulator != null) {\n            this.produceAccumulator.batchMaxDelayMs(holdMs);\n        }\n    }\n\n    public long getBatchMaxBytes() {\n        if (this.produceAccumulator == null) {\n            return 0;\n        }\n        return produceAccumulator.getBatchMaxBytes();\n    }\n\n    public void batchMaxBytes(long holdSize) {\n        this.batchMaxBytes = holdSize;\n        if (this.produceAccumulator != null) {\n            this.produceAccumulator.batchMaxBytes(holdSize);\n        }\n    }\n\n    public long getTotalBatchMaxBytes() {\n        if (this.produceAccumulator == null) {\n            return 0;\n        }\n        return produceAccumulator.getTotalBatchMaxBytes();\n    }\n\n    public void totalBatchMaxBytes(long totalHoldSize) {\n        this.totalBatchMaxBytes = totalHoldSize;\n        if (this.produceAccumulator != null) {\n            this.produceAccumulator.totalBatchMaxBytes(totalHoldSize);\n        }\n    }\n\n    public boolean getAutoBatch() {\n        if (this.produceAccumulator == null) {\n            return false;\n        }\n        return this.autoBatch;\n    }\n\n    public void setAutoBatch(boolean autoBatch) {\n        this.autoBatch = autoBatch;\n    }\n\n    public String getProducerGroup() {\n        return producerGroup;\n    }\n\n    public void setProducerGroup(String producerGroup) {\n        this.producerGroup = producerGroup;\n    }\n\n    public String getCreateTopicKey() {\n        return createTopicKey;\n    }\n\n    public void setCreateTopicKey(String createTopicKey) {\n        this.createTopicKey = createTopicKey;\n    }\n\n    public int getSendMsgTimeout() {\n        return sendMsgTimeout;\n    }\n\n    public void setSendMsgTimeout(int sendMsgTimeout) {\n        this.sendMsgTimeout = sendMsgTimeout;\n    }\n\n    public int getSendMsgMaxTimeoutPerRequest() {\n        return sendMsgMaxTimeoutPerRequest;\n    }\n\n    public void setSendMsgMaxTimeoutPerRequest(int sendMsgMaxTimeoutPerRequest) {\n        this.sendMsgMaxTimeoutPerRequest = sendMsgMaxTimeoutPerRequest;\n    }\n\n    public int getCompressMsgBodyOverHowmuch() {\n        return compressMsgBodyOverHowmuch;\n    }\n\n    public void setCompressMsgBodyOverHowmuch(int compressMsgBodyOverHowmuch) {\n        this.compressMsgBodyOverHowmuch = compressMsgBodyOverHowmuch;\n    }\n\n    @Deprecated\n    public DefaultMQProducerImpl getDefaultMQProducerImpl() {\n        return defaultMQProducerImpl;\n    }\n\n    public boolean isRetryAnotherBrokerWhenNotStoreOK() {\n        return retryAnotherBrokerWhenNotStoreOK;\n    }\n\n    public void setRetryAnotherBrokerWhenNotStoreOK(boolean retryAnotherBrokerWhenNotStoreOK) {\n        this.retryAnotherBrokerWhenNotStoreOK = retryAnotherBrokerWhenNotStoreOK;\n    }\n\n    public int getMaxMessageSize() {\n        return maxMessageSize;\n    }\n\n    public void setMaxMessageSize(int maxMessageSize) {\n        this.maxMessageSize = maxMessageSize;\n    }\n\n    public int getDefaultTopicQueueNums() {\n        return defaultTopicQueueNums;\n    }\n\n    public void setDefaultTopicQueueNums(int defaultTopicQueueNums) {\n        this.defaultTopicQueueNums = defaultTopicQueueNums;\n    }\n\n    public int getRetryTimesWhenSendFailed() {\n        return retryTimesWhenSendFailed;\n    }\n\n    public void setRetryTimesWhenSendFailed(int retryTimesWhenSendFailed) {\n        this.retryTimesWhenSendFailed = retryTimesWhenSendFailed;\n    }\n\n    public boolean isSendMessageWithVIPChannel() {\n        return isVipChannelEnabled();\n    }\n\n    public void setSendMessageWithVIPChannel(final boolean sendMessageWithVIPChannel) {\n        this.setVipChannelEnabled(sendMessageWithVIPChannel);\n    }\n\n    public long[] getNotAvailableDuration() {\n        return this.defaultMQProducerImpl.getNotAvailableDuration();\n    }\n\n    public void setNotAvailableDuration(final long[] notAvailableDuration) {\n        this.defaultMQProducerImpl.setNotAvailableDuration(notAvailableDuration);\n    }\n\n    public long[] getLatencyMax() {\n        return this.defaultMQProducerImpl.getLatencyMax();\n    }\n\n    public void setLatencyMax(final long[] latencyMax) {\n        this.defaultMQProducerImpl.setLatencyMax(latencyMax);\n    }\n\n    public boolean isSendLatencyFaultEnable() {\n        return this.defaultMQProducerImpl.isSendLatencyFaultEnable();\n    }\n\n    public void setSendLatencyFaultEnable(final boolean sendLatencyFaultEnable) {\n        this.defaultMQProducerImpl.setSendLatencyFaultEnable(sendLatencyFaultEnable);\n    }\n\n    public int getRetryTimesWhenSendAsyncFailed() {\n        return retryTimesWhenSendAsyncFailed;\n    }\n\n    public void setRetryTimesWhenSendAsyncFailed(final int retryTimesWhenSendAsyncFailed) {\n        this.retryTimesWhenSendAsyncFailed = retryTimesWhenSendAsyncFailed;\n    }\n\n    public TraceDispatcher getTraceDispatcher() {\n        return traceDispatcher;\n    }\n\n    public Set<Integer> getRetryResponseCodes() {\n        return retryResponseCodes;\n    }\n\n    public boolean isEnableBackpressureForAsyncMode() {\n        return enableBackpressureForAsyncMode;\n    }\n\n    public void setEnableBackpressureForAsyncMode(boolean enableBackpressureForAsyncMode) {\n        this.enableBackpressureForAsyncMode = enableBackpressureForAsyncMode;\n    }\n\n    public int getBackPressureForAsyncSendNum() {\n        return backPressureForAsyncSendNum;\n    }\n\n    /**\n     * For user modify backPressureForAsyncSendNum at runtime\n     */\n    public void setBackPressureForAsyncSendNum(int backPressureForAsyncSendNum) {\n        this.backPressureForAsyncSendNumLock.acquireWriteLock();\n        backPressureForAsyncSendNum = Math.max(backPressureForAsyncSendNum, 10);\n        int acquiredBackPressureForAsyncSendNum = this.backPressureForAsyncSendNum\n                - defaultMQProducerImpl.getSemaphoreAsyncSendNumAvailablePermits();\n        this.backPressureForAsyncSendNum = backPressureForAsyncSendNum;\n        defaultMQProducerImpl.setSemaphoreAsyncSendNum(backPressureForAsyncSendNum - acquiredBackPressureForAsyncSendNum);\n        this.backPressureForAsyncSendNumLock.releaseWriteLock();\n    }\n\n    public int getBackPressureForAsyncSendSize() {\n        return backPressureForAsyncSendSize;\n    }\n\n    /**\n    * For user modify backPressureForAsyncSendSize at runtime\n     */\n    public void setBackPressureForAsyncSendSize(int backPressureForAsyncSendSize) {\n        this.backPressureForAsyncSendSizeLock.acquireWriteLock();\n        backPressureForAsyncSendSize = Math.max(backPressureForAsyncSendSize, 1024 * 1024);\n        int acquiredBackPressureForAsyncSendSize = this.backPressureForAsyncSendSize\n                - defaultMQProducerImpl.getSemaphoreAsyncSendSizeAvailablePermits();\n        this.backPressureForAsyncSendSize = backPressureForAsyncSendSize;\n        defaultMQProducerImpl.setSemaphoreAsyncSendSize(backPressureForAsyncSendSize - acquiredBackPressureForAsyncSendSize);\n        this.backPressureForAsyncSendSizeLock.releaseWriteLock();\n    }\n\n    /**\n     * Used for system internal adjust backPressureForAsyncSendSize\n     */\n    public void setBackPressureForAsyncSendSizeInsideAdjust(int backPressureForAsyncSendSize) {\n        this.backPressureForAsyncSendSize = backPressureForAsyncSendSize;\n    }\n\n    /**\n     * Used for system internal adjust backPressureForAsyncSendNum\n     */\n    public void setBackPressureForAsyncSendNumInsideAdjust(int backPressureForAsyncSendNum) {\n        this.backPressureForAsyncSendNum = backPressureForAsyncSendNum;\n    }\n\n    public void acquireBackPressureForAsyncSendSizeLock() {\n        this.backPressureForAsyncSendSizeLock.acquireReadLock();\n    }\n\n    public void releaseBackPressureForAsyncSendSizeLock() {\n        this.backPressureForAsyncSendSizeLock.releaseReadLock();\n    }\n\n    public void acquireBackPressureForAsyncSendNumLock() {\n        this.backPressureForAsyncSendNumLock.acquireReadLock();\n    }\n\n    public void releaseBackPressureForAsyncSendNumLock() {\n        this.backPressureForAsyncSendNumLock.releaseReadLock();\n    }\n\n    public List<String> getTopics() {\n        return topics;\n    }\n\n    public void setTopics(List<String> topics) {\n        this.topics = topics;\n    }\n\n    @Override\n    public void setStartDetectorEnable(boolean startDetectorEnable) {\n        super.setStartDetectorEnable(startDetectorEnable);\n        this.defaultMQProducerImpl.getMqFaultStrategy().setStartDetectorEnable(startDetectorEnable);\n    }\n\n    public int getCompressLevel() {\n        return compressLevel;\n    }\n\n    public void setCompressLevel(int compressLevel) {\n        this.compressLevel = compressLevel;\n    }\n\n    public CompressionType getCompressType() {\n        return compressType;\n    }\n\n    public void setCompressType(CompressionType compressType) {\n        this.compressType = compressType;\n        this.compressor = CompressorFactory.getCompressor(compressType);\n    }\n\n    public Compressor getCompressor() {\n        return compressor;\n    }\n\n    public void initProduceAccumulator() {\n        this.produceAccumulator = MQClientManager.getInstance().getOrCreateProduceAccumulator(this);\n\n        if (this.batchMaxDelayMs > -1) {\n            this.produceAccumulator.batchMaxDelayMs(this.batchMaxDelayMs);\n        }\n\n        if (this.batchMaxBytes > -1) {\n            this.produceAccumulator.batchMaxBytes(this.batchMaxBytes);\n        }\n\n        if (this.totalBatchMaxBytes > -1) {\n            this.produceAccumulator.totalBatchMaxBytes(this.totalBatchMaxBytes);\n        }\n\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/producer/LocalTransactionState.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.producer;\n\npublic enum LocalTransactionState {\n    COMMIT_MESSAGE,\n    ROLLBACK_MESSAGE,\n    UNKNOW,\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.producer;\n\nimport java.util.Collection;\nimport java.util.List;\nimport org.apache.rocketmq.client.MQAdmin;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.exception.RequestTimeoutException;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\n\npublic interface MQProducer extends MQAdmin {\n    void start() throws MQClientException;\n\n    void shutdown();\n\n    List<MessageQueue> fetchPublishMessageQueues(final String topic) throws MQClientException;\n\n    SendResult send(final Message msg) throws MQClientException, RemotingException, MQBrokerException,\n        InterruptedException;\n\n    SendResult send(final Message msg, final long timeout) throws MQClientException,\n        RemotingException, MQBrokerException, InterruptedException;\n\n    void send(final Message msg, final SendCallback sendCallback) throws MQClientException,\n        RemotingException, InterruptedException, MQBrokerException;\n\n    void send(final Message msg, final SendCallback sendCallback, final long timeout)\n        throws MQClientException, RemotingException, InterruptedException;\n\n    void sendOneway(final Message msg) throws MQClientException, RemotingException,\n        InterruptedException;\n\n    SendResult send(final Message msg, final MessageQueue mq) throws MQClientException,\n        RemotingException, MQBrokerException, InterruptedException;\n\n    SendResult send(final Message msg, final MessageQueue mq, final long timeout)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException;\n\n    void send(final Message msg, final MessageQueue mq, final SendCallback sendCallback)\n        throws MQClientException, RemotingException, InterruptedException;\n\n    void send(final Message msg, final MessageQueue mq, final SendCallback sendCallback, long timeout)\n        throws MQClientException, RemotingException, InterruptedException;\n\n    void sendOneway(final Message msg, final MessageQueue mq) throws MQClientException,\n        RemotingException, InterruptedException;\n\n    SendResult send(final Message msg, final MessageQueueSelector selector, final Object arg)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException;\n\n    SendResult send(final Message msg, final MessageQueueSelector selector, final Object arg,\n        final long timeout) throws MQClientException, RemotingException, MQBrokerException,\n        InterruptedException;\n\n    void send(final Message msg, final MessageQueueSelector selector, final Object arg,\n        final SendCallback sendCallback) throws MQClientException, RemotingException,\n        InterruptedException;\n\n    void send(final Message msg, final MessageQueueSelector selector, final Object arg,\n        final SendCallback sendCallback, final long timeout) throws MQClientException, RemotingException,\n        InterruptedException;\n\n    void sendOneway(final Message msg, final MessageQueueSelector selector, final Object arg)\n        throws MQClientException, RemotingException, InterruptedException;\n\n    TransactionSendResult sendMessageInTransaction(final Message msg,\n        final Object arg) throws MQClientException;\n\n    //for batch\n    SendResult send(final Collection<Message> msgs) throws MQClientException, RemotingException, MQBrokerException,\n        InterruptedException;\n\n    SendResult send(final Collection<Message> msgs, final long timeout) throws MQClientException,\n        RemotingException, MQBrokerException, InterruptedException;\n\n    SendResult send(final Collection<Message> msgs, final MessageQueue mq) throws MQClientException,\n        RemotingException, MQBrokerException, InterruptedException;\n\n    SendResult send(final Collection<Message> msgs, final MessageQueue mq, final long timeout)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException;\n\n    void send(final Collection<Message> msgs,\n        final SendCallback sendCallback) throws MQClientException, RemotingException, MQBrokerException,\n        InterruptedException;\n\n    void send(final Collection<Message> msgs, final SendCallback sendCallback,\n        final long timeout) throws MQClientException, RemotingException,\n        MQBrokerException, InterruptedException;\n\n    void send(final Collection<Message> msgs, final MessageQueue mq,\n        final SendCallback sendCallback) throws MQClientException, RemotingException,\n        MQBrokerException, InterruptedException;\n\n    void send(final Collection<Message> msgs, final MessageQueue mq, final SendCallback sendCallback,\n        final long timeout) throws MQClientException,\n        RemotingException, MQBrokerException, InterruptedException;\n\n    String recallMessage(String topic, String recallHandle)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException;\n\n    //for rpc\n    Message request(final Message msg, final long timeout) throws RequestTimeoutException, MQClientException,\n        RemotingException, MQBrokerException, InterruptedException;\n\n    void request(final Message msg, final RequestCallback requestCallback, final long timeout)\n        throws MQClientException, RemotingException, InterruptedException, MQBrokerException;\n\n    Message request(final Message msg, final MessageQueueSelector selector, final Object arg,\n        final long timeout) throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException,\n        InterruptedException;\n\n    void request(final Message msg, final MessageQueueSelector selector, final Object arg,\n        final RequestCallback requestCallback,\n        final long timeout) throws MQClientException, RemotingException,\n        InterruptedException, MQBrokerException;\n\n    Message request(final Message msg, final MessageQueue mq, final long timeout)\n        throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException, InterruptedException;\n\n    void request(final Message msg, final MessageQueue mq, final RequestCallback requestCallback, long timeout)\n        throws MQClientException, RemotingException, MQBrokerException, InterruptedException;\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/producer/MessageQueueSelector.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.producer;\n\nimport java.util.List;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\npublic interface MessageQueueSelector {\n    MessageQueue select(final List<MessageQueue> mqs, final Message msg, final Object arg);\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/producer/ProduceAccumulator.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.producer;\n\nimport java.util.Arrays;\nimport java.util.Collection;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.LinkedList;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageBatch;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\n\npublic class ProduceAccumulator {\n    // totalHoldSize normal value\n    private long totalHoldSize = 32 * 1024 * 1024;\n    // holdSize normal value\n    private long holdSize = 32 * 1024;\n    // holdMs normal value\n    private int holdMs = 10;\n    private final Logger log = LoggerFactory.getLogger(DefaultMQProducer.class);\n    private final GuardForSyncSendService guardThreadForSyncSend;\n    private final GuardForAsyncSendService guardThreadForAsyncSend;\n    private final Map<AggregateKey, MessageAccumulation> syncSendBatchs = new ConcurrentHashMap<AggregateKey, MessageAccumulation>();\n    private final Map<AggregateKey, MessageAccumulation> asyncSendBatchs = new ConcurrentHashMap<AggregateKey, MessageAccumulation>();\n    private final AtomicLong currentlyHoldSize = new AtomicLong(0);\n    private final String instanceName;\n\n    public ProduceAccumulator(String instanceName) {\n        this.instanceName = instanceName;\n        this.guardThreadForSyncSend = new GuardForSyncSendService(this.instanceName);\n        this.guardThreadForAsyncSend = new GuardForAsyncSendService(this.instanceName);\n    }\n\n    private class GuardForSyncSendService extends ServiceThread {\n        private final String serviceName;\n\n        public GuardForSyncSendService(String clientInstanceName) {\n            serviceName = String.format(\"Client_%s_GuardForSyncSend\", clientInstanceName);\n        }\n\n        @Override\n        public String getServiceName() {\n            return serviceName;\n        }\n\n        @Override\n        public void run() {\n            log.info(this.getServiceName() + \" service started\");\n\n            while (!this.isStopped()) {\n                try {\n                    this.doWork();\n                } catch (Exception e) {\n                    log.warn(this.getServiceName() + \" service has exception. \", e);\n                }\n            }\n\n            log.info(this.getServiceName() + \" service end\");\n        }\n\n        private void doWork() throws InterruptedException {\n            Collection<MessageAccumulation> values = syncSendBatchs.values();\n            final int sleepTime = Math.max(1, holdMs / 2);\n            for (MessageAccumulation v : values) {\n                v.wakeup();\n                synchronized (v) {\n                    synchronized (v.closed) {\n                        if (v.messagesSize.get() == 0) {\n                            v.closed.set(true);\n                            syncSendBatchs.remove(v.aggregateKey, v);\n                        } else {\n                            v.notify();\n                        }\n                    }\n                }\n            }\n            Thread.sleep(sleepTime);\n        }\n    }\n\n    private class GuardForAsyncSendService extends ServiceThread {\n        private final String serviceName;\n\n        public GuardForAsyncSendService(String clientInstanceName) {\n            serviceName = String.format(\"Client_%s_GuardForAsyncSend\", clientInstanceName);\n        }\n\n        @Override\n        public String getServiceName() {\n            return serviceName;\n        }\n\n        @Override\n        public void run() {\n            log.info(this.getServiceName() + \" service started\");\n\n            while (!this.isStopped()) {\n                try {\n                    this.doWork();\n                } catch (Exception e) {\n                    log.warn(this.getServiceName() + \" service has exception. \", e);\n                }\n            }\n\n            log.info(this.getServiceName() + \" service end\");\n        }\n\n        private void doWork() throws Exception {\n            Collection<MessageAccumulation> values = asyncSendBatchs.values();\n            final int sleepTime = Math.max(1, holdMs / 2);\n            for (MessageAccumulation v : values) {\n                if (v.readyToSend()) {\n                    v.send(null);\n                }\n                synchronized (v.closed) {\n                    if (v.messagesSize.get() == 0) {\n                        v.closed.set(true);\n                        asyncSendBatchs.remove(v.aggregateKey, v);\n                    }\n                }\n            }\n            Thread.sleep(sleepTime);\n        }\n    }\n\n    void start() {\n        guardThreadForSyncSend.start();\n        guardThreadForAsyncSend.start();\n    }\n\n    void shutdown() {\n        guardThreadForSyncSend.shutdown();\n        guardThreadForAsyncSend.shutdown();\n    }\n\n    int getBatchMaxDelayMs() {\n        return holdMs;\n    }\n\n    void batchMaxDelayMs(int holdMs) {\n        if (holdMs <= 0 || holdMs > 30 * 1000) {\n            throw new IllegalArgumentException(String.format(\"batchMaxDelayMs expect between 1ms and 30s, but get %d!\", holdMs));\n        }\n        this.holdMs = holdMs;\n    }\n\n    long getBatchMaxBytes() {\n        return holdSize;\n    }\n\n    void batchMaxBytes(long holdSize) {\n        if (holdSize <= 0 || holdSize > 2 * 1024 * 1024) {\n            throw new IllegalArgumentException(String.format(\"batchMaxBytes expect between 1B and 2MB, but get %d!\", holdSize));\n        }\n        this.holdSize = holdSize;\n    }\n\n    long getTotalBatchMaxBytes() {\n        return holdSize;\n    }\n\n    void totalBatchMaxBytes(long totalHoldSize) {\n        if (totalHoldSize <= 0) {\n            throw new IllegalArgumentException(String.format(\"totalBatchMaxBytes must bigger then 0, but get %d!\", totalHoldSize));\n        }\n        this.totalHoldSize = totalHoldSize;\n    }\n\n    private MessageAccumulation getOrCreateSyncSendBatch(AggregateKey aggregateKey,\n        DefaultMQProducer defaultMQProducer) {\n        MessageAccumulation batch = syncSendBatchs.get(aggregateKey);\n        if (batch != null) {\n            return batch;\n        }\n        batch = new MessageAccumulation(aggregateKey, defaultMQProducer);\n        MessageAccumulation previous = syncSendBatchs.putIfAbsent(aggregateKey, batch);\n\n        return previous == null ? batch : previous;\n    }\n\n    private MessageAccumulation getOrCreateAsyncSendBatch(AggregateKey aggregateKey,\n        DefaultMQProducer defaultMQProducer) {\n        MessageAccumulation batch = asyncSendBatchs.get(aggregateKey);\n        if (batch != null) {\n            return batch;\n        }\n        batch = new MessageAccumulation(aggregateKey, defaultMQProducer);\n        MessageAccumulation previous = asyncSendBatchs.putIfAbsent(aggregateKey, batch);\n\n        return previous == null ? batch : previous;\n    }\n\n    SendResult send(Message msg,\n        DefaultMQProducer defaultMQProducer) throws InterruptedException, MQBrokerException, RemotingException, MQClientException {\n        AggregateKey partitionKey = new AggregateKey(msg);\n        while (true) {\n            MessageAccumulation batch = getOrCreateSyncSendBatch(partitionKey, defaultMQProducer);\n            int index = batch.add(msg);\n            if (index == -1) {\n                syncSendBatchs.remove(partitionKey, batch);\n            } else {\n                return batch.sendResults[index];\n            }\n        }\n    }\n\n    SendResult send(Message msg, MessageQueue mq,\n        DefaultMQProducer defaultMQProducer) throws InterruptedException, MQBrokerException, RemotingException, MQClientException {\n        AggregateKey partitionKey = new AggregateKey(msg, mq);\n        while (true) {\n            MessageAccumulation batch = getOrCreateSyncSendBatch(partitionKey, defaultMQProducer);\n            int index = batch.add(msg);\n            if (index == -1) {\n                syncSendBatchs.remove(partitionKey, batch);\n            } else {\n                return batch.sendResults[index];\n            }\n        }\n    }\n\n    void send(Message msg, SendCallback sendCallback,\n        DefaultMQProducer defaultMQProducer) throws InterruptedException, RemotingException, MQClientException {\n        AggregateKey partitionKey = new AggregateKey(msg);\n        while (true) {\n            MessageAccumulation batch = getOrCreateAsyncSendBatch(partitionKey, defaultMQProducer);\n            if (!batch.add(msg, sendCallback)) {\n                asyncSendBatchs.remove(partitionKey, batch);\n            } else {\n                return;\n            }\n        }\n    }\n\n    void send(Message msg, MessageQueue mq,\n        SendCallback sendCallback,\n        DefaultMQProducer defaultMQProducer) throws InterruptedException, RemotingException, MQClientException {\n        AggregateKey partitionKey = new AggregateKey(msg, mq);\n        while (true) {\n            MessageAccumulation batch = getOrCreateAsyncSendBatch(partitionKey, defaultMQProducer);\n            if (!batch.add(msg, sendCallback)) {\n                asyncSendBatchs.remove(partitionKey, batch);\n            } else {\n                return;\n            }\n        }\n    }\n\n    boolean tryAddMessage(Message message) {\n        synchronized (currentlyHoldSize) {\n            if (currentlyHoldSize.get() < totalHoldSize) {\n                int bodySize = null == message.getBody() ? 0 : message.getBody().length;\n                if (bodySize > 0) {\n                    currentlyHoldSize.addAndGet(bodySize);\n                }\n                return true;\n            } else {\n                return false;\n            }\n        }\n    }\n\n    private class AggregateKey {\n        public String topic = null;\n        public MessageQueue mq = null;\n        public boolean waitStoreMsgOK = false;\n        public String tag = null;\n\n        public AggregateKey(Message message) {\n            this(message.getTopic(), null, message.isWaitStoreMsgOK(), message.getTags());\n        }\n\n        public AggregateKey(Message message, MessageQueue mq) {\n            this(message.getTopic(), mq, message.isWaitStoreMsgOK(), message.getTags());\n        }\n\n        public AggregateKey(String topic, MessageQueue mq, boolean waitStoreMsgOK, String tag) {\n            this.topic = topic;\n            this.mq = mq;\n            this.waitStoreMsgOK = waitStoreMsgOK;\n            this.tag = tag;\n        }\n\n        @Override\n        public boolean equals(Object o) {\n            if (this == o)\n                return true;\n            if (o == null || getClass() != o.getClass())\n                return false;\n            AggregateKey key = (AggregateKey) o;\n            return waitStoreMsgOK == key.waitStoreMsgOK && topic.equals(key.topic) && Objects.equals(mq, key.mq) && Objects.equals(tag, key.tag);\n        }\n\n        @Override\n        public int hashCode() {\n            return Objects.hash(topic, mq, waitStoreMsgOK, tag);\n        }\n    }\n\n    private class MessageAccumulation {\n        private final DefaultMQProducer defaultMQProducer;\n        private LinkedList<Message> messages;\n        private LinkedList<SendCallback> sendCallbacks;\n        private Set<String> keys;\n        private final AtomicBoolean closed;\n        private SendResult[] sendResults;\n        private AggregateKey aggregateKey;\n        private AtomicInteger messagesSize;\n        private int count;\n        private long createTime;\n\n        public MessageAccumulation(AggregateKey aggregateKey, DefaultMQProducer defaultMQProducer) {\n            this.defaultMQProducer = defaultMQProducer;\n            this.messages = new LinkedList<Message>();\n            this.sendCallbacks = new LinkedList<SendCallback>();\n            this.keys = new HashSet<String>();\n            this.closed = new AtomicBoolean(false);\n            this.messagesSize = new AtomicInteger(0);\n            this.aggregateKey = aggregateKey;\n            this.count = 0;\n            this.createTime = System.currentTimeMillis();\n        }\n\n        private boolean readyToSend() {\n            if (this.messagesSize.get() > holdSize\n                || System.currentTimeMillis() >= this.createTime + holdMs) {\n                return true;\n            }\n            return false;\n        }\n\n        public int add(Message msg) throws InterruptedException, MQBrokerException, RemotingException, MQClientException {\n            int ret = -1;\n            synchronized (this.closed) {\n                if (this.closed.get()) {\n                    return ret;\n                }\n                ret = this.count++;\n                this.messages.add(msg);\n                int bodySize = null == msg.getBody() ? 0 : msg.getBody().length;\n                if (bodySize > 0) {\n                    messagesSize.addAndGet(bodySize);\n                }\n                String msgKeys = msg.getKeys();\n                if (msgKeys != null) {\n                    this.keys.addAll(Arrays.asList(msgKeys.split(MessageConst.KEY_SEPARATOR)));\n                }\n            }\n            synchronized (this) {\n                while (!this.closed.get()) {\n                    if (readyToSend()) {\n                        this.send();\n                        break;\n                    } else {\n                        this.wait();\n                    }\n                }\n                return ret;\n            }\n        }\n\n        public boolean add(Message msg,\n            SendCallback sendCallback) throws InterruptedException, RemotingException, MQClientException {\n            synchronized (this.closed) {\n                if (this.closed.get()) {\n                    return false;\n                }\n                this.count++;\n                this.messages.add(msg);\n                this.sendCallbacks.add(sendCallback);\n                int bodySize = null == msg.getBody() ? 0 : msg.getBody().length;\n                if (bodySize > 0) {\n                    messagesSize.addAndGet(bodySize);\n                }\n            }\n            if (readyToSend()) {\n                this.send(sendCallback);\n            }\n            return true;\n\n        }\n\n        public synchronized void wakeup() {\n            if (this.closed.get()) {\n                return;\n            }\n            this.notify();\n        }\n\n        private MessageBatch batch() {\n            MessageBatch messageBatch = new MessageBatch(this.messages);\n            messageBatch.setTopic(this.aggregateKey.topic);\n            messageBatch.setWaitStoreMsgOK(this.aggregateKey.waitStoreMsgOK);\n            messageBatch.setKeys(this.keys);\n            messageBatch.setTags(this.aggregateKey.tag);\n            MessageClientIDSetter.setUniqID(messageBatch);\n            messageBatch.setBody(MessageDecoder.encodeMessages(this.messages));\n            return messageBatch;\n        }\n\n        private void splitSendResults(SendResult sendResult) {\n            if (sendResult == null) {\n                throw new IllegalArgumentException(\"sendResult is null\");\n            }\n            boolean isBatchConsumerQueue = !sendResult.getMsgId().contains(\",\");\n            this.sendResults = new SendResult[this.count];\n            if (!isBatchConsumerQueue) {\n                String[] msgIds = sendResult.getMsgId().split(\",\");\n                String[] offsetMsgIds = sendResult.getOffsetMsgId().split(\",\");\n                if (offsetMsgIds.length != this.count || msgIds.length != this.count) {\n                    throw new IllegalArgumentException(\"sendResult is illegal\");\n                }\n                for (int i = 0; i < this.count; i++) {\n                    this.sendResults[i] = new SendResult(sendResult.getSendStatus(), msgIds[i],\n                        sendResult.getMessageQueue(), sendResult.getQueueOffset() + i,\n                        sendResult.getTransactionId(), offsetMsgIds[i], sendResult.getRegionId());\n                }\n            } else {\n                for (int i = 0; i < this.count; i++) {\n                    this.sendResults[i] = sendResult;\n                }\n            }\n        }\n\n        private void send() throws InterruptedException, MQClientException, MQBrokerException, RemotingException {\n            synchronized (this.closed) {\n                if (this.closed.getAndSet(true)) {\n                    return;\n                }\n            }\n            MessageBatch messageBatch = this.batch();\n            SendResult sendResult = null;\n            try {\n                if (defaultMQProducer != null) {\n                    sendResult = defaultMQProducer.sendDirect(messageBatch, aggregateKey.mq, null);\n                    this.splitSendResults(sendResult);\n                } else {\n                    throw new IllegalArgumentException(\"defaultMQProducer is null, can not send message\");\n                }\n            } finally {\n                currentlyHoldSize.addAndGet(-messagesSize.get());\n                this.notifyAll();\n            }\n        }\n\n        private void send(SendCallback sendCallback) {\n            synchronized (this.closed) {\n                if (this.closed.getAndSet(true)) {\n                    return;\n                }\n            }\n            MessageBatch messageBatch = this.batch();\n            SendResult sendResult = null;\n            try {\n                if (defaultMQProducer != null) {\n                    final int size = messagesSize.get();\n                    defaultMQProducer.sendDirect(messageBatch, aggregateKey.mq, new SendCallback() {\n                        @Override\n                        public void onSuccess(SendResult sendResult) {\n                            try {\n                                splitSendResults(sendResult);\n                                int i = 0;\n                                Iterator<SendCallback> it = sendCallbacks.iterator();\n                                while (it.hasNext()) {\n                                    SendCallback v = it.next();\n                                    v.onSuccess(sendResults[i++]);\n                                }\n                                if (i != count) {\n                                    throw new IllegalArgumentException(\"sendResult is illegal\");\n                                }\n                                currentlyHoldSize.addAndGet(-size);\n                            } catch (Exception e) {\n                                onException(e);\n                            }\n                        }\n\n                        @Override\n                        public void onException(Throwable e) {\n                            for (SendCallback v : sendCallbacks) {\n                                v.onException(e);\n                            }\n                            currentlyHoldSize.addAndGet(-size);\n                        }\n                    });\n                } else {\n                    throw new IllegalArgumentException(\"defaultMQProducer is null, can not send message\");\n                }\n            } catch (Exception e) {\n                for (SendCallback v : sendCallbacks) {\n                    v.onException(e);\n                }\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/producer/RequestCallback.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.producer;\n\nimport org.apache.rocketmq.common.message.Message;\n\npublic interface RequestCallback {\n    void onSuccess(final Message message);\n\n    void onException(final Throwable e);\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/producer/RequestFutureHolder.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.producer;\n\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\n\nimport org.apache.rocketmq.client.common.ClientErrorCode;\nimport org.apache.rocketmq.client.exception.RequestTimeoutException;\nimport org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class RequestFutureHolder {\n    private static final Logger log = LoggerFactory.getLogger(RequestFutureHolder.class);\n    private static final RequestFutureHolder INSTANCE = new RequestFutureHolder();\n    private ConcurrentHashMap<String, RequestResponseFuture> requestFutureTable = new ConcurrentHashMap<>();\n    private final Set<DefaultMQProducerImpl> producerSet = new HashSet<>();\n    private ScheduledExecutorService scheduledExecutorService = null;\n\n    public ConcurrentHashMap<String, RequestResponseFuture> getRequestFutureTable() {\n        return requestFutureTable;\n    }\n\n    private void scanExpiredRequest() {\n        final List<RequestResponseFuture> rfList = new LinkedList<>();\n        Iterator<Map.Entry<String, RequestResponseFuture>> it = requestFutureTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Map.Entry<String, RequestResponseFuture> next = it.next();\n            RequestResponseFuture rep = next.getValue();\n\n            if (rep.isTimeout()) {\n                it.remove();\n                rfList.add(rep);\n                log.warn(\"remove timeout request, CorrelationId={}\" + rep.getCorrelationId());\n            }\n        }\n\n        for (RequestResponseFuture rf : rfList) {\n            try {\n                Throwable cause = new RequestTimeoutException(ClientErrorCode.REQUEST_TIMEOUT_EXCEPTION, \"request timeout, no reply message.\");\n                rf.setCause(cause);\n                rf.executeRequestCallback();\n            } catch (Throwable e) {\n                log.warn(\"scanResponseTable, operationComplete Exception\", e);\n            }\n        }\n    }\n\n    public synchronized void startScheduledTask(DefaultMQProducerImpl producer) {\n        this.producerSet.add(producer);\n        if (null == scheduledExecutorService) {\n            this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(\"RequestHouseKeepingService\"));\n\n            this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n                @Override\n                public void run() {\n                    try {\n                        RequestFutureHolder.getInstance().scanExpiredRequest();\n                    } catch (Throwable e) {\n                        log.error(\"scan RequestFutureTable exception\", e);\n                    }\n                }\n            }, 1000 * 3, 1000, TimeUnit.MILLISECONDS);\n\n        }\n    }\n\n    public synchronized void shutdown(DefaultMQProducerImpl producer) {\n        this.producerSet.remove(producer);\n        if (this.producerSet.size() <= 0 && null != this.scheduledExecutorService) {\n            ScheduledExecutorService executorService = this.scheduledExecutorService;\n            this.scheduledExecutorService = null;\n            executorService.shutdown();\n        }\n    }\n\n    private RequestFutureHolder() {}\n\n    public static RequestFutureHolder getInstance() {\n        return INSTANCE;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.producer;\n\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.common.message.Message;\n\npublic class RequestResponseFuture {\n    private final String correlationId;\n    private final RequestCallback requestCallback;\n    private final long beginTimestamp = System.currentTimeMillis();\n    private final Message requestMsg = null;\n    private long timeoutMillis;\n    private CountDownLatch countDownLatch = new CountDownLatch(1);\n    private volatile Message responseMsg = null;\n    private volatile boolean sendRequestOk = true;\n    private volatile Throwable cause = null;\n\n    public RequestResponseFuture(String correlationId, long timeoutMillis, RequestCallback requestCallback) {\n        this.correlationId = correlationId;\n        this.timeoutMillis = timeoutMillis;\n        this.requestCallback = requestCallback;\n    }\n\n    public void executeRequestCallback() {\n        if (requestCallback != null) {\n            if (sendRequestOk && cause == null) {\n                requestCallback.onSuccess(responseMsg);\n            } else {\n                requestCallback.onException(cause);\n            }\n        }\n    }\n\n    public boolean isTimeout() {\n        long diff = System.currentTimeMillis() - this.beginTimestamp;\n        return diff > this.timeoutMillis;\n    }\n\n    public Message waitResponseMessage(final long timeout) throws InterruptedException {\n        this.countDownLatch.await(timeout, TimeUnit.MILLISECONDS);\n        return this.responseMsg;\n    }\n\n    public void putResponseMessage(final Message responseMsg) {\n        this.responseMsg = responseMsg;\n        this.countDownLatch.countDown();\n    }\n\n    public String getCorrelationId() {\n        return correlationId;\n    }\n\n    public long getTimeoutMillis() {\n        return timeoutMillis;\n    }\n\n    public void setTimeoutMillis(long timeoutMillis) {\n        this.timeoutMillis = timeoutMillis;\n    }\n\n    public RequestCallback getRequestCallback() {\n        return requestCallback;\n    }\n\n    public long getBeginTimestamp() {\n        return beginTimestamp;\n    }\n\n    public CountDownLatch getCountDownLatch() {\n        return countDownLatch;\n    }\n\n    public void setCountDownLatch(CountDownLatch countDownLatch) {\n        this.countDownLatch = countDownLatch;\n    }\n\n    public Message getResponseMsg() {\n        return responseMsg;\n    }\n\n    public void setResponseMsg(Message responseMsg) {\n        this.responseMsg = responseMsg;\n    }\n\n    public boolean isSendRequestOk() {\n        return sendRequestOk;\n    }\n\n    public void setSendRequestOk(boolean sendRequestOk) {\n        this.sendRequestOk = sendRequestOk;\n    }\n\n    public Message getRequestMsg() {\n        return requestMsg;\n    }\n\n    public Throwable getCause() {\n        return cause;\n    }\n\n    public void setCause(Throwable cause) {\n        this.cause = cause;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/producer/SendCallback.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.producer;\n\npublic interface SendCallback {\n    void onSuccess(final SendResult sendResult);\n\n    void onException(final Throwable e);\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.producer;\n\nimport com.alibaba.fastjson2.JSON;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\npublic class SendResult {\n    private SendStatus sendStatus;\n    private String msgId;\n    private MessageQueue messageQueue;\n    private long queueOffset;\n    private String transactionId;\n    private String offsetMsgId;\n    private String regionId;\n    private boolean traceOn = true;\n    private byte[] rawRespBody;\n    private String recallHandle;\n\n    public SendResult() {\n    }\n\n    public SendResult(SendStatus sendStatus, String msgId, String offsetMsgId, MessageQueue messageQueue,\n        long queueOffset) {\n        this.sendStatus = sendStatus;\n        this.msgId = msgId;\n        this.offsetMsgId = offsetMsgId;\n        this.messageQueue = messageQueue;\n        this.queueOffset = queueOffset;\n    }\n\n    public SendResult(final SendStatus sendStatus, final String msgId, final MessageQueue messageQueue,\n        final long queueOffset, final String transactionId,\n        final String offsetMsgId, final String regionId) {\n        this.sendStatus = sendStatus;\n        this.msgId = msgId;\n        this.messageQueue = messageQueue;\n        this.queueOffset = queueOffset;\n        this.transactionId = transactionId;\n        this.offsetMsgId = offsetMsgId;\n        this.regionId = regionId;\n    }\n\n    public static String encoderSendResultToJson(final Object obj) {\n        return JSON.toJSONString(obj);\n    }\n\n    public static SendResult decoderSendResultFromJson(String json) {\n        return JSON.parseObject(json, SendResult.class);\n    }\n\n    public boolean isTraceOn() {\n        return traceOn;\n    }\n\n    public void setTraceOn(final boolean traceOn) {\n        this.traceOn = traceOn;\n    }\n\n    public String getRegionId() {\n        return regionId;\n    }\n\n    public void setRegionId(final String regionId) {\n        this.regionId = regionId;\n    }\n\n    public String getMsgId() {\n        return msgId;\n    }\n\n    public void setMsgId(String msgId) {\n        this.msgId = msgId;\n    }\n\n    public SendStatus getSendStatus() {\n        return sendStatus;\n    }\n\n    public void setSendStatus(SendStatus sendStatus) {\n        this.sendStatus = sendStatus;\n    }\n\n    public MessageQueue getMessageQueue() {\n        return messageQueue;\n    }\n\n    public void setMessageQueue(MessageQueue messageQueue) {\n        this.messageQueue = messageQueue;\n    }\n\n    public long getQueueOffset() {\n        return queueOffset;\n    }\n\n    public void setQueueOffset(long queueOffset) {\n        this.queueOffset = queueOffset;\n    }\n\n    public String getTransactionId() {\n        return transactionId;\n    }\n\n    public void setTransactionId(String transactionId) {\n        this.transactionId = transactionId;\n    }\n\n    public String getOffsetMsgId() {\n        return offsetMsgId;\n    }\n\n    public void setOffsetMsgId(String offsetMsgId) {\n        this.offsetMsgId = offsetMsgId;\n    }\n\n    public String getRecallHandle() {\n        return recallHandle;\n    }\n\n    public void setRecallHandle(String recallHandle) {\n        this.recallHandle = recallHandle;\n    }\n\n    @Override\n    public String toString() {\n        return \"SendResult [sendStatus=\" + sendStatus + \", msgId=\" + msgId + \", offsetMsgId=\" + offsetMsgId + \", messageQueue=\" + messageQueue\n            + \", queueOffset=\" + queueOffset + \", recallHandle=\" + recallHandle + \"]\";\n    }\n\n    public void setRawRespBody(byte[] body) {\n        this.rawRespBody = body;\n    }\n\n    public byte[] getRawRespBody() {\n        return rawRespBody;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/producer/SendStatus.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.producer;\n\npublic enum SendStatus {\n    SEND_OK,\n    FLUSH_DISK_TIMEOUT,\n    FLUSH_SLAVE_TIMEOUT,\n    SLAVE_NOT_AVAILABLE,\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/producer/TransactionCheckListener.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.producer;\n\nimport org.apache.rocketmq.common.message.MessageExt;\n\n/**\n * @deprecated This interface will be removed in the version 5.0.0, interface {@link TransactionListener} is recommended.\n */\n@Deprecated\npublic interface TransactionCheckListener {\n    LocalTransactionState checkLocalTransactionState(final MessageExt msg);\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/producer/TransactionListener.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.producer;\n\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic interface TransactionListener {\n    /**\n     * When send transactional prepare(half) message succeed, this method will be invoked to execute local transaction.\n     *\n     * @param msg Half(prepare) message\n     * @param arg Custom business parameter\n     * @return Transaction state\n     */\n    LocalTransactionState executeLocalTransaction(final Message msg, final Object arg);\n\n    /**\n     * When no response to prepare(half) message. broker will send check message to check the transaction status, and this\n     * method will be invoked to get local transaction status.\n     *\n     * @param msg Check message\n     * @return Transaction state\n     */\n    LocalTransactionState checkLocalTransaction(final MessageExt msg);\n}"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/producer/TransactionMQProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.producer;\n\nimport java.util.List;\nimport java.util.concurrent.ExecutorService;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\n\npublic class TransactionMQProducer extends DefaultMQProducer {\n    private TransactionCheckListener transactionCheckListener;\n    private int checkThreadPoolMinSize = 1;\n    private int checkThreadPoolMaxSize = 1;\n    private int checkRequestHoldMax = 2000;\n\n    private ExecutorService executorService;\n\n    private TransactionListener transactionListener;\n\n    public TransactionMQProducer() {\n    }\n\n    public TransactionMQProducer(final String producerGroup) {\n        super(producerGroup);\n    }\n\n    public TransactionMQProducer(final String producerGroup, final List<String> topics) {\n        super(producerGroup, null, topics);\n    }\n\n    public TransactionMQProducer(final String producerGroup, RPCHook rpcHook) {\n        super(producerGroup, rpcHook, null);\n    }\n\n    public TransactionMQProducer(final String producerGroup, RPCHook rpcHook, final List<String> topics) {\n        super(producerGroup, rpcHook, topics);\n    }\n\n    public TransactionMQProducer(final String producerGroup, RPCHook rpcHook, boolean enableMsgTrace, final String customizedTraceTopic) {\n        super(producerGroup, rpcHook, enableMsgTrace, customizedTraceTopic);\n    }\n\n    @Deprecated\n    public TransactionMQProducer(final String namespace, final String producerGroup) {\n        super(namespace, producerGroup);\n    }\n\n    @Deprecated\n    public TransactionMQProducer(final String namespace, final String producerGroup, RPCHook rpcHook, boolean enableMsgTrace, final String customizedTraceTopic) {\n        super(namespace, producerGroup, rpcHook, enableMsgTrace, customizedTraceTopic);\n    }\n\n    @Override\n    public void start() throws MQClientException {\n        this.defaultMQProducerImpl.initTransactionEnv();\n        super.start();\n    }\n\n    @Override\n    public void shutdown() {\n        super.shutdown();\n        this.defaultMQProducerImpl.destroyTransactionEnv();\n    }\n\n    @Override\n    public TransactionSendResult sendMessageInTransaction(final Message msg,\n        final Object arg) throws MQClientException {\n        if (null == this.transactionListener) {\n            throw new MQClientException(\"TransactionListener is null\", null);\n        }\n\n        msg.setTopic(NamespaceUtil.wrapNamespace(this.getNamespace(), msg.getTopic()));\n        return this.defaultMQProducerImpl.sendMessageInTransaction(msg, null, arg);\n    }\n\n    public TransactionCheckListener getTransactionCheckListener() {\n        return transactionCheckListener;\n    }\n\n    /**\n     * This method will be removed in the version 5.0.0 and set a custom thread pool is recommended.\n     */\n    @Deprecated\n    public void setTransactionCheckListener(TransactionCheckListener transactionCheckListener) {\n        this.transactionCheckListener = transactionCheckListener;\n    }\n\n    public int getCheckThreadPoolMinSize() {\n        return checkThreadPoolMinSize;\n    }\n\n    /**\n     * This method will be removed in the version 5.0.0 and set a custom thread pool is recommended.\n     */\n    @Deprecated\n    public void setCheckThreadPoolMinSize(int checkThreadPoolMinSize) {\n        this.checkThreadPoolMinSize = checkThreadPoolMinSize;\n    }\n\n    public int getCheckThreadPoolMaxSize() {\n        return checkThreadPoolMaxSize;\n    }\n\n    /**\n     * This method will be removed in the version 5.0.0 and set a custom thread pool is recommended.\n     */\n    @Deprecated\n    public void setCheckThreadPoolMaxSize(int checkThreadPoolMaxSize) {\n        this.checkThreadPoolMaxSize = checkThreadPoolMaxSize;\n    }\n\n    public int getCheckRequestHoldMax() {\n        return checkRequestHoldMax;\n    }\n\n    /**\n     * This method will be removed in the version 5.0.0 and set a custom thread pool is recommended.\n     */\n    @Deprecated\n    public void setCheckRequestHoldMax(int checkRequestHoldMax) {\n        this.checkRequestHoldMax = checkRequestHoldMax;\n    }\n\n    public ExecutorService getExecutorService() {\n        return executorService;\n    }\n\n    public void setExecutorService(ExecutorService executorService) {\n        this.executorService = executorService;\n    }\n\n    public TransactionListener getTransactionListener() {\n        return transactionListener;\n    }\n\n    public void setTransactionListener(TransactionListener transactionListener) {\n        this.transactionListener = transactionListener;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/producer/TransactionSendResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.producer;\n\npublic class TransactionSendResult extends SendResult {\n    private LocalTransactionState localTransactionState;\n\n    public TransactionSendResult() {\n    }\n\n    public LocalTransactionState getLocalTransactionState() {\n        return localTransactionState;\n    }\n\n    public void setLocalTransactionState(LocalTransactionState localTransactionState) {\n        this.localTransactionState = localTransactionState;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByHash.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.producer.selector;\n\nimport java.util.List;\nimport org.apache.rocketmq.client.producer.MessageQueueSelector;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\npublic class SelectMessageQueueByHash implements MessageQueueSelector {\n\n    @Override\n    public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {\n        int value = arg.hashCode() % mqs.size();\n        if (value < 0) {\n            value = Math.abs(value);\n        }\n        return mqs.get(value);\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByMachineRoom.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.producer.selector;\n\nimport java.util.List;\nimport java.util.Set;\nimport org.apache.rocketmq.client.producer.MessageQueueSelector;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\npublic class SelectMessageQueueByMachineRoom implements MessageQueueSelector {\n    private Set<String> consumeridcs;\n\n    @Override\n    public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {\n        return null;\n    }\n\n    public Set<String> getConsumeridcs() {\n        return consumeridcs;\n    }\n\n    public void setConsumeridcs(Set<String> consumeridcs) {\n        this.consumeridcs = consumeridcs;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByRandom.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.producer.selector;\n\nimport java.util.List;\nimport java.util.Random;\nimport org.apache.rocketmq.client.producer.MessageQueueSelector;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\npublic class SelectMessageQueueByRandom implements MessageQueueSelector {\n    private Random random = new Random(System.currentTimeMillis());\n\n    @Override\n    public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {\n        int value = random.nextInt(mqs.size());\n        return mqs.get(value);\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/rpchook/NamespaceRpcHook.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.rpchook;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic class NamespaceRpcHook implements RPCHook {\n    private final ClientConfig clientConfig;\n\n    public NamespaceRpcHook(ClientConfig clientConfig) {\n        this.clientConfig = clientConfig;\n    }\n\n    @Override\n    public void doBeforeRequest(String remoteAddr, RemotingCommand request) {\n        if (StringUtils.isNotEmpty(clientConfig.getNamespaceV2())) {\n            request.addExtField(MixAll.RPC_REQUEST_HEADER_NAMESPACED_FIELD, \"true\");\n            request.addExtField(MixAll.RPC_REQUEST_HEADER_NAMESPACE_FIELD, clientConfig.getNamespaceV2());\n        }\n    }\n\n    @Override\n    public void doAfterResponse(String remoteAddr, RemotingCommand request,\n        RemotingCommand response) {\n\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/stat/ConsumerStatsManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.stat;\n\nimport java.util.concurrent.ScheduledExecutorService;\nimport org.apache.rocketmq.common.stats.StatsItemSet;\nimport org.apache.rocketmq.common.stats.StatsSnapshot;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeStatus;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class ConsumerStatsManager {\n    private static final Logger log = LoggerFactory.getLogger(ConsumerStatsManager.class);\n\n    private static final String TOPIC_AND_GROUP_CONSUME_OK_TPS = \"CONSUME_OK_TPS\";\n    private static final String TOPIC_AND_GROUP_CONSUME_FAILED_TPS = \"CONSUME_FAILED_TPS\";\n    private static final String TOPIC_AND_GROUP_CONSUME_RT = \"CONSUME_RT\";\n    private static final String TOPIC_AND_GROUP_PULL_TPS = \"PULL_TPS\";\n    private static final String TOPIC_AND_GROUP_PULL_RT = \"PULL_RT\";\n\n    private final StatsItemSet topicAndGroupConsumeOKTPS;\n    private final StatsItemSet topicAndGroupConsumeRT;\n    private final StatsItemSet topicAndGroupConsumeFailedTPS;\n    private final StatsItemSet topicAndGroupPullTPS;\n    private final StatsItemSet topicAndGroupPullRT;\n\n    public ConsumerStatsManager(final ScheduledExecutorService scheduledExecutorService) {\n        this.topicAndGroupConsumeOKTPS =\n            new StatsItemSet(TOPIC_AND_GROUP_CONSUME_OK_TPS, scheduledExecutorService, log);\n\n        this.topicAndGroupConsumeRT =\n            new StatsItemSet(TOPIC_AND_GROUP_CONSUME_RT, scheduledExecutorService, log);\n\n        this.topicAndGroupConsumeFailedTPS =\n            new StatsItemSet(TOPIC_AND_GROUP_CONSUME_FAILED_TPS, scheduledExecutorService, log);\n\n        this.topicAndGroupPullTPS = new StatsItemSet(TOPIC_AND_GROUP_PULL_TPS, scheduledExecutorService, log);\n\n        this.topicAndGroupPullRT = new StatsItemSet(TOPIC_AND_GROUP_PULL_RT, scheduledExecutorService, log);\n    }\n\n    public void start() {\n    }\n\n    public void shutdown() {\n    }\n\n    public void incPullRT(final String group, final String topic, final long rt) {\n        this.topicAndGroupPullRT.addRTValue(topic + \"@\" + group, (int) rt, 1);\n    }\n\n    public void incPullTPS(final String group, final String topic, final long msgs) {\n        this.topicAndGroupPullTPS.addValue(topic + \"@\" + group, (int) msgs, 1);\n    }\n\n    public void incConsumeRT(final String group, final String topic, final long rt) {\n        this.topicAndGroupConsumeRT.addRTValue(topic + \"@\" + group, (int) rt, 1);\n    }\n\n    public void incConsumeOKTPS(final String group, final String topic, final long msgs) {\n        this.topicAndGroupConsumeOKTPS.addValue(topic + \"@\" + group, (int) msgs, 1);\n    }\n\n    public void incConsumeFailedTPS(final String group, final String topic, final long msgs) {\n        this.topicAndGroupConsumeFailedTPS.addValue(topic + \"@\" + group, (int) msgs, 1);\n    }\n\n    public ConsumeStatus consumeStatus(final String group, final String topic) {\n        ConsumeStatus cs = new ConsumeStatus();\n        {\n            StatsSnapshot ss = this.getPullRT(group, topic);\n            if (ss != null) {\n                cs.setPullRT(ss.getAvgpt());\n            }\n        }\n\n        {\n            StatsSnapshot ss = this.getPullTPS(group, topic);\n            if (ss != null) {\n                cs.setPullTPS(ss.getTps());\n            }\n        }\n\n        {\n            StatsSnapshot ss = this.getConsumeRT(group, topic);\n            if (ss != null) {\n                cs.setConsumeRT(ss.getAvgpt());\n            }\n        }\n\n        {\n            StatsSnapshot ss = this.getConsumeOKTPS(group, topic);\n            if (ss != null) {\n                cs.setConsumeOKTPS(ss.getTps());\n            }\n        }\n\n        {\n            StatsSnapshot ss = this.getConsumeFailedTPS(group, topic);\n            if (ss != null) {\n                cs.setConsumeFailedTPS(ss.getTps());\n            }\n        }\n\n        {\n            StatsSnapshot ss = this.topicAndGroupConsumeFailedTPS.getStatsDataInHour(topic + \"@\" + group);\n            if (ss != null) {\n                cs.setConsumeFailedMsgs(ss.getSum());\n            }\n        }\n\n        return cs;\n    }\n\n    private StatsSnapshot getPullRT(final String group, final String topic) {\n        return this.topicAndGroupPullRT.getStatsDataInMinute(topic + \"@\" + group);\n    }\n\n    private StatsSnapshot getPullTPS(final String group, final String topic) {\n        return this.topicAndGroupPullTPS.getStatsDataInMinute(topic + \"@\" + group);\n    }\n\n    private StatsSnapshot getConsumeRT(final String group, final String topic) {\n        StatsSnapshot statsData = this.topicAndGroupConsumeRT.getStatsDataInMinute(topic + \"@\" + group);\n        if (0 == statsData.getSum()) {\n            statsData = this.topicAndGroupConsumeRT.getStatsDataInHour(topic + \"@\" + group);\n        }\n\n        return statsData;\n    }\n\n    private StatsSnapshot getConsumeOKTPS(final String group, final String topic) {\n        return this.topicAndGroupConsumeOKTPS.getStatsDataInMinute(topic + \"@\" + group);\n    }\n\n    private StatsSnapshot getConsumeFailedTPS(final String group, final String topic) {\n        return this.topicAndGroupConsumeFailedTPS.getStatsDataInMinute(topic + \"@\" + group);\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.trace;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ArrayBlockingQueue;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.client.AccessChannel;\nimport org.apache.rocketmq.client.common.ThreadLocalIndex;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl;\nimport org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;\nimport org.apache.rocketmq.client.impl.producer.TopicPublishInfo;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.MessageQueueSelector;\nimport org.apache.rocketmq.client.producer.SendCallback;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.RPCHook;\n\nimport static org.apache.rocketmq.client.trace.TraceConstants.TRACE_INSTANCE_NAME;\n\npublic class AsyncTraceDispatcher implements TraceDispatcher {\n    private static final Logger log = LoggerFactory.getLogger(AsyncTraceDispatcher.class);\n    private static final AtomicInteger COUNTER = new AtomicInteger();\n    private static final AtomicInteger INSTANCE_NUM = new AtomicInteger(0);\n    private static final long WAIT_FOR_SHUTDOWN = 5000L;\n    private volatile boolean stopped = false;\n    private final int traceInstanceId = INSTANCE_NUM.getAndIncrement();\n    private final int batchNum;\n    private final int maxMsgSize;\n    private final DefaultMQProducer traceProducer;\n    private AtomicLong discardCount;\n    private Thread worker;\n    private final ThreadPoolExecutor traceExecutor;\n    private final ArrayBlockingQueue<TraceContext> traceContextQueue;\n    private final ArrayBlockingQueue<Runnable> appenderQueue;\n    private volatile Thread shutDownHook;\n\n    private DefaultMQProducerImpl hostProducer;\n    private DefaultMQPushConsumerImpl hostConsumer;\n    private volatile ThreadLocalIndex sendWhichQueue = new ThreadLocalIndex();\n    private volatile String traceTopicName;\n    private AtomicBoolean isStarted = new AtomicBoolean(false);\n    private volatile AccessChannel accessChannel = AccessChannel.LOCAL;\n    private String group;\n    private Type type;\n    private String namespaceV2;\n    private final int flushTraceInterval = 5000;\n\n    private long lastFlushTime = System.currentTimeMillis();\n\n    public AsyncTraceDispatcher(String group, Type type, int batchNum, String traceTopicName, RPCHook rpcHook) {\n        this.batchNum = Math.min(batchNum, 20);/* max value 20*/\n        this.maxMsgSize = 128000;\n        this.discardCount = new AtomicLong(0L);\n        this.traceContextQueue = new ArrayBlockingQueue<>(2048);\n        this.group = group;\n        this.type = type;\n        this.appenderQueue = new ArrayBlockingQueue<>(2048);\n        if (!UtilAll.isBlank(traceTopicName)) {\n            this.traceTopicName = traceTopicName;\n        } else {\n            this.traceTopicName = TopicValidator.RMQ_SYS_TRACE_TOPIC;\n        }\n        this.traceExecutor = new ThreadPoolExecutor(//\n            2, //\n            4, //\n            1000 * 60, //\n            TimeUnit.MILLISECONDS, //\n            this.appenderQueue, //\n            new ThreadFactoryImpl(\"MQTraceSendThread_\" + traceInstanceId + \"_\"));\n        traceProducer = getAndCreateTraceProducer(rpcHook);\n    }\n\n    public AccessChannel getAccessChannel() {\n        return accessChannel;\n    }\n\n    public void setAccessChannel(AccessChannel accessChannel) {\n        this.accessChannel = accessChannel;\n    }\n\n    public String getTraceTopicName() {\n        return traceTopicName;\n    }\n\n    public void setTraceTopicName(String traceTopicName) {\n        this.traceTopicName = traceTopicName;\n    }\n\n    public DefaultMQProducer getTraceProducer() {\n        return traceProducer;\n    }\n\n    public DefaultMQProducerImpl getHostProducer() {\n        return hostProducer;\n    }\n\n    public void setHostProducer(DefaultMQProducerImpl hostProducer) {\n        this.hostProducer = hostProducer;\n    }\n\n    public DefaultMQPushConsumerImpl getHostConsumer() {\n        return hostConsumer;\n    }\n\n    public void setHostConsumer(DefaultMQPushConsumerImpl hostConsumer) {\n        this.hostConsumer = hostConsumer;\n    }\n\n    public String getNamespaceV2() {\n        return namespaceV2;\n    }\n\n    public void setNamespaceV2(String namespaceV2) {\n        this.namespaceV2 = namespaceV2;\n    }\n\n    public void start(String nameSrvAddr, AccessChannel accessChannel) throws MQClientException {\n        if (isStarted.compareAndSet(false, true)) {\n            traceProducer.setNamesrvAddr(nameSrvAddr);\n            traceProducer.setInstanceName(TRACE_INSTANCE_NAME + \"_\" + nameSrvAddr);\n            traceProducer.setNamespaceV2(namespaceV2);\n            traceProducer.setEnableTrace(false);\n            traceProducer.start();\n        }\n        this.accessChannel = accessChannel;\n        this.worker = new ThreadFactoryImpl(\"MQ-AsyncArrayDispatcher-Thread\" + traceInstanceId, true)\n            .newThread(new AsyncRunnable());\n        this.worker.setDaemon(true);\n        this.worker.start();\n        this.registerShutDownHook();\n    }\n\n    private DefaultMQProducer getAndCreateTraceProducer(RPCHook rpcHook) {\n        DefaultMQProducer traceProducerInstance = this.traceProducer;\n        if (traceProducerInstance == null) {\n            traceProducerInstance = new DefaultMQProducer(rpcHook);\n            traceProducerInstance.setProducerGroup(genGroupNameForTrace());\n            traceProducerInstance.setSendMsgTimeout(5000);\n            traceProducerInstance.setVipChannelEnabled(false);\n            // The max size of message is 128K\n            traceProducerInstance.setMaxMessageSize(maxMsgSize);\n        }\n        return traceProducerInstance;\n    }\n\n    private String genGroupNameForTrace() {\n        return TraceConstants.GROUP_NAME_PREFIX + \"-\" + this.group + \"-\" + this.type + \"-\" + COUNTER.incrementAndGet();\n    }\n\n    @Override\n    public boolean append(final Object ctx) {\n        boolean result = traceContextQueue.offer((TraceContext) ctx);\n        if (!result) {\n            log.info(\"buffer full\" + discardCount.incrementAndGet() + \" ,context is \" + ctx);\n        }\n        return result;\n    }\n\n    @Override\n    public void flush() {\n        while (traceContextQueue.size() > 0) {\n            try {\n                flushTraceContext(true);\n            } catch (Throwable throwable) {\n                log.error(\"flushTraceContext error\", throwable);\n            }\n        }\n    }\n\n    @Override\n    public void shutdown() {\n        flush();\n        ThreadUtils.shutdownGracefully(this.traceExecutor, WAIT_FOR_SHUTDOWN, TimeUnit.MILLISECONDS);\n        if (isStarted.get()) {\n            traceProducer.shutdown();\n        }\n        this.removeShutdownHook();\n        stopped = true;\n    }\n\n    public void registerShutDownHook() {\n        if (shutDownHook == null) {\n            shutDownHook = new Thread(new Runnable() {\n                private volatile boolean hasShutdown = false;\n\n                @Override\n                public void run() {\n                    synchronized (this) {\n                        if (!this.hasShutdown) {\n                            flush();\n                        }\n                    }\n                }\n            }, \"ShutdownHookMQTrace\");\n            \n\n            try {\n                Runtime.getRuntime().addShutdownHook(shutDownHook);\n            } catch (IllegalStateException e) {\n                // ignore - VM is already shutting down\n            }\n        }\n    }\n\n    public void removeShutdownHook() {\n        if (shutDownHook != null) {\n            try {\n                Runtime.getRuntime().removeShutdownHook(shutDownHook);\n            } catch (IllegalStateException e) {\n                // ignore - VM is already shutting down\n            }\n        }\n    }\n\n    class AsyncRunnable implements Runnable {\n        private volatile boolean stopped = false;\n\n        @Override\n        public void run() {\n            while (!stopped) {\n                try {\n                    flushTraceContext(false);\n                } catch (Throwable e) {\n                    log.error(\"flushTraceContext error\", e);\n                }\n\n                if (AsyncTraceDispatcher.this.stopped) {\n                    this.stopped = true;\n                }\n            }\n        }\n    }\n\n    private void flushTraceContext(boolean forceFlush) throws InterruptedException {\n        List<TraceContext> contextList = new ArrayList<>(batchNum);\n        int size = traceContextQueue.size();\n        if (size != 0) {\n            if (forceFlush || size >= batchNum || System.currentTimeMillis() - lastFlushTime > flushTraceInterval) {\n                for (int i = 0; i < batchNum; i++) {\n                    TraceContext context = traceContextQueue.poll();\n                    if (context != null) {\n                        contextList.add(context);\n                    } else {\n                        break;\n                    }\n                }\n                asyncSendTraceMessage(contextList);\n                return;\n            }\n        }\n        // To prevent an infinite loop, add a wait time between each two task executions\n        Thread.sleep(5);\n    }\n\n    private void asyncSendTraceMessage(List<TraceContext> contextList) {\n        AsyncDataSendTask request = new AsyncDataSendTask(contextList);\n        traceExecutor.submit(request);\n        lastFlushTime = System.currentTimeMillis();\n    }\n\n    class AsyncDataSendTask implements Runnable {\n        private final List<TraceContext> contextList;\n\n        public AsyncDataSendTask(List<TraceContext> contextList) {\n            this.contextList = contextList;\n        }\n\n        @Override\n        public void run() {\n            sendTraceData(contextList);\n        }\n\n        public void sendTraceData(List<TraceContext> contextList) {\n            Map<String, List<TraceTransferBean>> transBeanMap = new HashMap<>(16);\n            String traceTopic;\n            for (TraceContext context : contextList) {\n                AccessChannel accessChannel = context.getAccessChannel();\n                if (accessChannel == null) {\n                    accessChannel = AsyncTraceDispatcher.this.accessChannel;\n                }\n                String currentRegionId = context.getRegionId();\n                if (currentRegionId == null || context.getTraceBeans().isEmpty()) {\n                    continue;\n                }\n                if (AccessChannel.CLOUD == accessChannel) {\n                    traceTopic = TraceConstants.TRACE_TOPIC_PREFIX + currentRegionId;\n                } else {\n                    traceTopic = traceTopicName;\n                }\n\n                String topic = context.getTraceBeans().get(0).getTopic();\n                String key = topic + TraceConstants.CONTENT_SPLITOR + traceTopic;\n                List<TraceTransferBean> transBeanList = transBeanMap.computeIfAbsent(key, k -> new ArrayList<>());\n                TraceTransferBean traceData = TraceDataEncoder.encoderFromContextBean(context);\n                transBeanList.add(traceData);\n            }\n            for (Map.Entry<String, List<TraceTransferBean>> entry : transBeanMap.entrySet()) {\n                String[] key = entry.getKey().split(String.valueOf(TraceConstants.CONTENT_SPLITOR));\n                flushData(entry.getValue(), key[0], key[1]);\n            }\n        }\n\n        private void flushData(List<TraceTransferBean> transBeanList, String topic, String traceTopic) {\n            if (transBeanList.size() == 0) {\n                return;\n            }\n            StringBuilder buffer = new StringBuilder(1024);\n            int count = 0;\n            Set<String> keySet = new HashSet<String>();\n            for (TraceTransferBean bean : transBeanList) {\n                keySet.addAll(bean.getTransKey());\n                buffer.append(bean.getTransData());\n                count++;\n                if (buffer.length() >= traceProducer.getMaxMessageSize()) {\n                    sendTraceDataByMQ(keySet, buffer.toString(), traceTopic);\n                    buffer.delete(0, buffer.length());\n                    keySet.clear();\n                    count = 0;\n                }\n            }\n            if (count > 0) {\n                sendTraceDataByMQ(keySet, buffer.toString(), traceTopic);\n            }\n            transBeanList.clear();\n        }\n\n        /**\n         * Send message trace data\n         *\n         * @param keySet     the keyset in this batch(including msgId in original message not offsetMsgId)\n         * @param data       the message trace data in this batch\n         * @param traceTopic the topic which message trace data will send to\n         */\n        private void sendTraceDataByMQ(Set<String> keySet, final String data, String traceTopic) {\n            final Message message = new Message(traceTopic, data.getBytes(StandardCharsets.UTF_8));\n            // Keyset of message trace includes msgId of or original message\n            message.setKeys(keySet);\n            try {\n                Set<String> traceBrokerSet = tryGetMessageQueueBrokerSet(traceProducer.getDefaultMQProducerImpl(), traceTopic);\n                SendCallback callback = new SendCallback() {\n                    @Override\n                    public void onSuccess(SendResult sendResult) {\n\n                    }\n\n                    @Override\n                    public void onException(Throwable e) {\n                        log.error(\"send trace data failed, the traceData is {}\", data, e);\n                    }\n                };\n                if (traceBrokerSet.isEmpty()) {\n                    // No cross set\n                    traceProducer.send(message, callback, 5000);\n                } else {\n                    traceProducer.send(message, new MessageQueueSelector() {\n                        @Override\n                        public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {\n                            Set<String> brokerSet = (Set<String>) arg;\n                            List<MessageQueue> filterMqs = new ArrayList<>();\n                            for (MessageQueue queue : mqs) {\n                                if (brokerSet.contains(queue.getBrokerName())) {\n                                    filterMqs.add(queue);\n                                }\n                            }\n                            int index = sendWhichQueue.incrementAndGet();\n                            int pos = index % filterMqs.size();\n                            return filterMqs.get(pos);\n                        }\n                    }, traceBrokerSet, callback);\n                }\n\n            } catch (Exception e) {\n                log.error(\"send trace data failed, the traceData is {}\", data, e);\n            }\n        }\n\n        private Set<String> tryGetMessageQueueBrokerSet(DefaultMQProducerImpl producer, String topic) {\n            Set<String> brokerSet = new HashSet<>();\n            TopicPublishInfo topicPublishInfo = producer.getTopicPublishInfoTable().get(topic);\n            if (null == topicPublishInfo || !topicPublishInfo.ok()) {\n                producer.getTopicPublishInfoTable().putIfAbsent(topic, new TopicPublishInfo());\n                producer.getMqClientFactory().updateTopicRouteInfoFromNameServer(topic);\n                topicPublishInfo = producer.getTopicPublishInfoTable().get(topic);\n            }\n            if (topicPublishInfo.isHaveTopicRouterInfo() || topicPublishInfo.ok()) {\n                for (MessageQueue queue : topicPublishInfo.getMessageQueueList()) {\n                    brokerSet.add(queue.getBrokerName());\n                }\n            }\n            return brokerSet;\n        }\n    }\n\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/trace/TraceBean.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.trace;\n\nimport org.apache.rocketmq.client.producer.LocalTransactionState;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.MessageType;\n\npublic class TraceBean {\n    private static final String LOCAL_ADDRESS;\n    private String topic = \"\";\n    private String msgId = \"\";\n    private String offsetMsgId = \"\";\n    private String tags = \"\";\n    private String keys = \"\";\n    private String storeHost = LOCAL_ADDRESS;\n    private String clientHost = LOCAL_ADDRESS;\n    private long storeTime;\n    private int retryTimes;\n    private int bodyLength;\n    private MessageType msgType;\n    private LocalTransactionState transactionState;\n    private String transactionId;\n    private boolean fromTransactionCheck;\n\n    static {\n        byte[] ip = UtilAll.getIP();\n        if (ip.length == 4) {\n            LOCAL_ADDRESS = UtilAll.ipToIPv4Str(ip);\n        } else {\n            LOCAL_ADDRESS = UtilAll.ipToIPv6Str(ip);\n        }\n    }\n\n    public MessageType getMsgType() {\n        return msgType;\n    }\n\n\n    public void setMsgType(final MessageType msgType) {\n        this.msgType = msgType;\n    }\n\n\n    public String getOffsetMsgId() {\n        return offsetMsgId;\n    }\n\n\n    public void setOffsetMsgId(final String offsetMsgId) {\n        this.offsetMsgId = offsetMsgId;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n\n    public String getMsgId() {\n        return msgId;\n    }\n\n\n    public void setMsgId(String msgId) {\n        this.msgId = msgId;\n    }\n\n\n    public String getTags() {\n        return tags;\n    }\n\n\n    public void setTags(String tags) {\n        this.tags = tags;\n    }\n\n\n    public String getKeys() {\n        return keys;\n    }\n\n\n    public void setKeys(String keys) {\n        this.keys = keys;\n    }\n\n\n    public String getStoreHost() {\n        return storeHost;\n    }\n\n\n    public void setStoreHost(String storeHost) {\n        this.storeHost = storeHost;\n    }\n\n\n    public String getClientHost() {\n        return clientHost;\n    }\n\n\n    public void setClientHost(String clientHost) {\n        this.clientHost = clientHost;\n    }\n\n\n    public long getStoreTime() {\n        return storeTime;\n    }\n\n\n    public void setStoreTime(long storeTime) {\n        this.storeTime = storeTime;\n    }\n\n\n    public int getRetryTimes() {\n        return retryTimes;\n    }\n\n\n    public void setRetryTimes(int retryTimes) {\n        this.retryTimes = retryTimes;\n    }\n\n\n    public int getBodyLength() {\n        return bodyLength;\n    }\n\n\n    public void setBodyLength(int bodyLength) {\n        this.bodyLength = bodyLength;\n    }\n\n    public LocalTransactionState getTransactionState() {\n        return transactionState;\n    }\n\n    public void setTransactionState(LocalTransactionState transactionState) {\n        this.transactionState = transactionState;\n    }\n\n    public String getTransactionId() {\n        return transactionId;\n    }\n\n    public void setTransactionId(String transactionId) {\n        this.transactionId = transactionId;\n    }\n\n    public boolean isFromTransactionCheck() {\n        return fromTransactionCheck;\n    }\n\n    public void setFromTransactionCheck(boolean fromTransactionCheck) {\n        this.fromTransactionCheck = fromTransactionCheck;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/trace/TraceConstants.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.trace;\n\nimport org.apache.rocketmq.common.topic.TopicValidator;\n\npublic class TraceConstants {\n\n    public static final String GROUP_NAME_PREFIX = \"_INNER_TRACE_PRODUCER\";\n    public static final char CONTENT_SPLITOR = (char) 1;\n    public static final char FIELD_SPLITOR = (char) 2;\n    public static final String TRACE_INSTANCE_NAME = \"PID_CLIENT_INNER_TRACE_PRODUCER\";\n    public static final String TRACE_TOPIC_PREFIX = TopicValidator.SYSTEM_TOPIC_PREFIX + \"TRACE_DATA_\";\n    public static final String TO_PREFIX = \"To_\";\n    public static final String FROM_PREFIX = \"From_\";\n    public static final String END_TRANSACTION = \"EndTransaction\";\n    public static final String ROCKETMQ_SERVICE = \"rocketmq\";\n    public static final String ROCKETMQ_SUCCESS = \"rocketmq.success\";\n    public static final String ROCKETMQ_TAGS = \"rocketmq.tags\";\n    public static final String ROCKETMQ_KEYS = \"rocketmq.keys\";\n    public static final String ROCKETMQ_STORE_HOST = \"rocketmq.store_host\";\n    public static final String ROCKETMQ_BODY_LENGTH = \"rocketmq.body_length\";\n    public static final String ROCKETMQ_MSG_ID = \"rocketmq.mgs_id\";\n    public static final String ROCKETMQ_MSG_TYPE = \"rocketmq.mgs_type\";\n    public static final String ROCKETMQ_REGION_ID = \"rocketmq.region_id\";\n    public static final String ROCKETMQ_TRANSACTION_ID = \"rocketmq.transaction_id\";\n    public static final String ROCKETMQ_TRANSACTION_STATE = \"rocketmq.transaction_state\";\n    public static final String ROCKETMQ_IS_FROM_TRANSACTION_CHECK = \"rocketmq.is_from_transaction_check\";\n    public static final String ROCKETMQ_RETRY_TIMERS = \"rocketmq.retry_times\";\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/trace/TraceContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.trace;\n\nimport org.apache.rocketmq.client.AccessChannel;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\n\nimport java.util.List;\n\n/**\n * The context of Trace\n */\npublic class TraceContext implements Comparable<TraceContext> {\n\n    private TraceType traceType;\n    private long timeStamp = System.currentTimeMillis();\n    private String regionId = \"\";\n    private String regionName = \"\";\n    private String groupName = \"\";\n    private int costTime = 0;\n    private boolean isSuccess = true;\n    private String requestId = MessageClientIDSetter.createUniqID();\n    private int contextCode = 0;\n    private AccessChannel accessChannel;\n    private List<TraceBean> traceBeans;\n\n    public int getContextCode() {\n        return contextCode;\n    }\n\n    public void setContextCode(final int contextCode) {\n        this.contextCode = contextCode;\n    }\n\n    public List<TraceBean> getTraceBeans() {\n        return traceBeans;\n    }\n\n    public void setTraceBeans(List<TraceBean> traceBeans) {\n        this.traceBeans = traceBeans;\n    }\n\n    public String getRegionId() {\n        return regionId;\n    }\n\n    public void setRegionId(String regionId) {\n        this.regionId = regionId;\n    }\n\n    public TraceType getTraceType() {\n        return traceType;\n    }\n\n    public void setTraceType(TraceType traceType) {\n        this.traceType = traceType;\n    }\n\n    public long getTimeStamp() {\n        return timeStamp;\n    }\n\n    public void setTimeStamp(long timeStamp) {\n        this.timeStamp = timeStamp;\n    }\n\n    public String getGroupName() {\n        return groupName;\n    }\n\n    public void setGroupName(String groupName) {\n        this.groupName = groupName;\n    }\n\n    public int getCostTime() {\n        return costTime;\n    }\n\n    public void setCostTime(int costTime) {\n        this.costTime = costTime;\n    }\n\n    public boolean isSuccess() {\n        return isSuccess;\n    }\n\n    public void setSuccess(boolean success) {\n        isSuccess = success;\n    }\n\n    public String getRequestId() {\n        return requestId;\n    }\n\n    public void setRequestId(String requestId) {\n        this.requestId = requestId;\n    }\n\n    public String getRegionName() {\n        return regionName;\n    }\n\n    public void setRegionName(String regionName) {\n        this.regionName = regionName;\n    }\n\n    public AccessChannel getAccessChannel() {\n        return accessChannel;\n    }\n\n    public void setAccessChannel(AccessChannel accessChannel) {\n        this.accessChannel = accessChannel;\n    }\n\n    @Override\n    public int compareTo(TraceContext o) {\n        return Long.compare(this.timeStamp, o.getTimeStamp());\n    }\n\n    @Override\n    public String toString() {\n        StringBuilder sb = new StringBuilder(1024);\n        sb.append(\"TraceContext{\").append(traceType).append(\"_\").append(groupName).append(\"_\")\n            .append(regionId).append(\"_\").append(isSuccess).append(\"_\");\n        if (traceBeans != null && traceBeans.size() > 0) {\n            for (TraceBean bean : traceBeans) {\n                sb.append(bean.getMsgId()).append(\"_\").append(bean.getTopic()).append(\"_\");\n            }\n        }\n        sb.append('}');\n        return sb.toString();\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.trace;\n\nimport org.apache.rocketmq.client.AccessChannel;\nimport org.apache.rocketmq.client.producer.LocalTransactionState;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageType;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.List;\n\n/**\n * Encode/decode for Trace Data\n */\npublic class TraceDataEncoder {\n\n    /**\n     * Resolving traceContext list From trace data String\n     *\n     * @param traceData\n     * @return\n     */\n    public static List<TraceContext> decoderFromTraceDataString(String traceData) {\n        List<TraceContext> resList = new ArrayList<>();\n        if (traceData == null || traceData.length() <= 0) {\n            return resList;\n        }\n        String[] contextList = traceData.split(String.valueOf(TraceConstants.FIELD_SPLITOR));\n        for (String context : contextList) {\n            String[] line = context.split(String.valueOf(TraceConstants.CONTENT_SPLITOR));\n            if (line[0].equals(TraceType.Pub.name())) {\n                TraceContext pubContext = new TraceContext();\n                pubContext.setTraceType(TraceType.Pub);\n                pubContext.setTimeStamp(Long.parseLong(line[1]));\n                pubContext.setRegionId(line[2]);\n                pubContext.setGroupName(line[3]);\n                TraceBean bean = new TraceBean();\n                bean.setTopic(line[4]);\n                bean.setMsgId(line[5]);\n                bean.setTags(line[6]);\n                bean.setKeys(line[7]);\n                bean.setStoreHost(line[8]);\n                bean.setBodyLength(Integer.parseInt(line[9]));\n                pubContext.setCostTime(Integer.parseInt(line[10]));\n                bean.setMsgType(MessageType.values()[Integer.parseInt(line[11])]);\n\n                if (line.length == 13) {\n                    pubContext.setSuccess(Boolean.parseBoolean(line[12]));\n                } else if (line.length == 14) {\n                    bean.setOffsetMsgId(line[12]);\n                    pubContext.setSuccess(Boolean.parseBoolean(line[13]));\n                }\n\n                // compatible with the old version\n                if (line.length >= 15) {\n                    bean.setOffsetMsgId(line[12]);\n                    pubContext.setSuccess(Boolean.parseBoolean(line[13]));\n                    bean.setClientHost(line[14]);\n                }\n\n                pubContext.setTraceBeans(new ArrayList<>(1));\n                pubContext.getTraceBeans().add(bean);\n                resList.add(pubContext);\n            } else if (line[0].equals(TraceType.SubBefore.name())) {\n                TraceContext subBeforeContext = new TraceContext();\n                subBeforeContext.setTraceType(TraceType.SubBefore);\n                subBeforeContext.setTimeStamp(Long.parseLong(line[1]));\n                subBeforeContext.setRegionId(line[2]);\n                subBeforeContext.setGroupName(line[3]);\n                subBeforeContext.setRequestId(line[4]);\n                TraceBean bean = new TraceBean();\n                bean.setMsgId(line[5]);\n                bean.setRetryTimes(Integer.parseInt(line[6]));\n                bean.setKeys(line[7]);\n                subBeforeContext.setTraceBeans(new ArrayList<>(1));\n                subBeforeContext.getTraceBeans().add(bean);\n                resList.add(subBeforeContext);\n            } else if (line[0].equals(TraceType.SubAfter.name())) {\n                TraceContext subAfterContext = new TraceContext();\n                subAfterContext.setTraceType(TraceType.SubAfter);\n                subAfterContext.setRequestId(line[1]);\n                TraceBean bean = new TraceBean();\n                bean.setMsgId(line[2]);\n                bean.setKeys(line[5]);\n                subAfterContext.setTraceBeans(new ArrayList<>(1));\n                subAfterContext.getTraceBeans().add(bean);\n                subAfterContext.setCostTime(Integer.parseInt(line[3]));\n                subAfterContext.setSuccess(Boolean.parseBoolean(line[4]));\n                if (line.length >= 7) {\n                    // add the context type\n                    subAfterContext.setContextCode(Integer.parseInt(line[6]));\n                }\n                // compatible with the old version\n                if (line.length >= 9) {\n                    subAfterContext.setTimeStamp(Long.parseLong(line[7]));\n                    subAfterContext.setGroupName(line[8]);\n                }\n                resList.add(subAfterContext);\n            } else if (line[0].equals(TraceType.EndTransaction.name())) {\n                TraceContext endTransactionContext = new TraceContext();\n                endTransactionContext.setTraceType(TraceType.EndTransaction);\n                endTransactionContext.setTimeStamp(Long.parseLong(line[1]));\n                endTransactionContext.setRegionId(line[2]);\n                endTransactionContext.setGroupName(line[3]);\n                TraceBean bean = new TraceBean();\n                bean.setTopic(line[4]);\n                bean.setMsgId(line[5]);\n                bean.setTags(line[6]);\n                bean.setKeys(line[7]);\n                bean.setStoreHost(line[8]);\n                bean.setMsgType(MessageType.values()[Integer.parseInt(line[9])]);\n                bean.setTransactionId(line[10]);\n                bean.setTransactionState(LocalTransactionState.valueOf(line[11]));\n                bean.setFromTransactionCheck(Boolean.parseBoolean(line[12]));\n\n                endTransactionContext.setTraceBeans(new ArrayList<>(1));\n                endTransactionContext.getTraceBeans().add(bean);\n                resList.add(endTransactionContext);\n            } else if (line[0].equals(TraceType.Recall.name())) {\n                TraceContext recallContext = new TraceContext();\n                recallContext.setTraceType(TraceType.Recall);\n                recallContext.setTimeStamp(Long.parseLong(line[1]));\n                recallContext.setRegionId(line[2]);\n                recallContext.setGroupName(line[3]);\n                TraceBean bean = new TraceBean();\n                bean.setTopic(line[4]);\n                bean.setMsgId(line[5]);\n                recallContext.setSuccess(Boolean.parseBoolean(line[6]));\n                recallContext.setTraceBeans(new ArrayList<>(1));\n                recallContext.getTraceBeans().add(bean);\n                resList.add(recallContext);\n            }\n        }\n        return resList;\n    }\n\n    /**\n     * Encoding the trace context into data strings and keyset sets\n     *\n     * @param ctx\n     * @return\n     */\n    public static TraceTransferBean encoderFromContextBean(TraceContext ctx) {\n        if (ctx == null) {\n            return null;\n        }\n        //build message trace of the transferring entity content bean\n        TraceTransferBean transferBean = new TraceTransferBean();\n        StringBuilder sb = new StringBuilder(256);\n        switch (ctx.getTraceType()) {\n            case Pub: {\n                TraceBean bean = ctx.getTraceBeans().get(0);\n                //append the content of context and traceBean to transferBean's TransData\n                sb.append(ctx.getTraceType()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(ctx.getTimeStamp()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(ctx.getRegionId()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(ctx.getGroupName()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(bean.getTopic()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(bean.getMsgId()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(bean.getTags()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(bean.getKeys()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(bean.getStoreHost()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(bean.getBodyLength()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(ctx.getCostTime()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(bean.getMsgType().ordinal()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(bean.getOffsetMsgId()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(ctx.isSuccess()).append(TraceConstants.FIELD_SPLITOR);//\n            }\n            break;\n            case SubBefore: {\n                for (TraceBean bean : ctx.getTraceBeans()) {\n                    sb.append(ctx.getTraceType()).append(TraceConstants.CONTENT_SPLITOR)//\n                        .append(ctx.getTimeStamp()).append(TraceConstants.CONTENT_SPLITOR)//\n                        .append(ctx.getRegionId()).append(TraceConstants.CONTENT_SPLITOR)//\n                        .append(ctx.getGroupName()).append(TraceConstants.CONTENT_SPLITOR)//\n                        .append(ctx.getRequestId()).append(TraceConstants.CONTENT_SPLITOR)//\n                        .append(bean.getMsgId()).append(TraceConstants.CONTENT_SPLITOR)//\n                        .append(bean.getRetryTimes()).append(TraceConstants.CONTENT_SPLITOR)//\n                        .append(bean.getKeys()).append(TraceConstants.FIELD_SPLITOR);//\n                }\n            }\n            break;\n            case SubAfter: {\n                for (TraceBean bean : ctx.getTraceBeans()) {\n                    sb.append(ctx.getTraceType()).append(TraceConstants.CONTENT_SPLITOR)//\n                        .append(ctx.getRequestId()).append(TraceConstants.CONTENT_SPLITOR)//\n                        .append(bean.getMsgId()).append(TraceConstants.CONTENT_SPLITOR)//\n                        .append(ctx.getCostTime()).append(TraceConstants.CONTENT_SPLITOR)//\n                        .append(ctx.isSuccess()).append(TraceConstants.CONTENT_SPLITOR)//\n                        .append(bean.getKeys()).append(TraceConstants.CONTENT_SPLITOR)//\n                        .append(ctx.getContextCode()).append(TraceConstants.CONTENT_SPLITOR);\n                    if (!ctx.getAccessChannel().equals(AccessChannel.CLOUD)) {\n                        sb.append(ctx.getTimeStamp()).append(TraceConstants.CONTENT_SPLITOR);\n                        sb.append(ctx.getGroupName());\n                    }\n                    sb.append(TraceConstants.FIELD_SPLITOR);\n                }\n            }\n            break;\n            case EndTransaction: {\n                TraceBean bean = ctx.getTraceBeans().get(0);\n                sb.append(ctx.getTraceType()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(ctx.getTimeStamp()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(ctx.getRegionId()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(ctx.getGroupName()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(bean.getTopic()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(bean.getMsgId()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(bean.getTags()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(bean.getKeys()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(bean.getStoreHost()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(bean.getMsgType().ordinal()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(bean.getTransactionId()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(bean.getTransactionState().name()).append(TraceConstants.CONTENT_SPLITOR)//\n                    .append(bean.isFromTransactionCheck()).append(TraceConstants.FIELD_SPLITOR);\n            }\n            break;\n            case Recall: {\n                TraceBean bean = ctx.getTraceBeans().get(0);\n                sb.append(ctx.getTraceType()).append(TraceConstants.CONTENT_SPLITOR)\n                    .append(ctx.getTimeStamp()).append(TraceConstants.CONTENT_SPLITOR)\n                    .append(ctx.getRegionId()).append(TraceConstants.CONTENT_SPLITOR)\n                    .append(ctx.getGroupName()).append(TraceConstants.CONTENT_SPLITOR)\n                    .append(bean.getTopic()).append(TraceConstants.CONTENT_SPLITOR)\n                    .append(bean.getMsgId()).append(TraceConstants.CONTENT_SPLITOR)\n                    .append(ctx.isSuccess()).append(TraceConstants.FIELD_SPLITOR);//\n            }\n            break;\n            default:\n        }\n        transferBean.setTransData(sb.toString());\n        for (TraceBean bean : ctx.getTraceBeans()) {\n\n            transferBean.getTransKey().add(bean.getMsgId());\n            if (bean.getKeys() != null && bean.getKeys().length() > 0) {\n                String[] keys = bean.getKeys().split(MessageConst.KEY_SEPARATOR);\n                transferBean.getTransKey().addAll(Arrays.asList(keys));\n            }\n        }\n        return transferBean;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/trace/TraceDispatcher.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.trace;\n\nimport org.apache.rocketmq.client.AccessChannel;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport java.io.IOException;\n\n/**\n * Interface of asynchronous transfer data\n */\npublic interface TraceDispatcher {\n    enum Type {\n        PRODUCE,\n        CONSUME\n    }\n    /**\n     * Initialize asynchronous transfer data module\n     */\n    void start(String nameSrvAddr, AccessChannel accessChannel) throws MQClientException;\n\n    /**\n     * Append the transferring data\n     * @param ctx data information\n     * @return\n     */\n    boolean append(Object ctx);\n\n    /**\n     * Write flush action\n     *\n     * @throws IOException\n     */\n    void flush() throws IOException;\n\n    /**\n     * Close the trace Hook\n     */\n    void shutdown();\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/trace/TraceDispatcherType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.trace;\n\npublic enum TraceDispatcherType {\n    PRODUCER,\n    CONSUMER\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/trace/TraceTransferBean.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.trace;\n\nimport java.util.HashSet;\nimport java.util.Set;\n\n/**\n * Trace transferring bean\n */\npublic class TraceTransferBean {\n    private String transData;\n    private Set<String> transKey = new HashSet<>();\n\n    public String getTransData() {\n        return transData;\n    }\n\n    public void setTransData(String transData) {\n        this.transData = transData;\n    }\n\n    public Set<String> getTransKey() {\n        return transKey;\n    }\n\n    public void setTransKey(Set<String> transKey) {\n        this.transKey = transKey;\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/trace/TraceType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.trace;\n\npublic enum TraceType {\n    Pub,\n    Recall,\n    SubBefore,\n    SubAfter,\n    EndTransaction,\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/trace/TraceView.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.trace;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic class TraceView {\n\n    private String msgId;\n    private String tags;\n    private String keys;\n    private String storeHost;\n    private String clientHost;\n    private int costTime;\n    private String msgType;\n    private String offSetMsgId;\n    private long timeStamp;\n    private long bornTime;\n    private String topic;\n    private String groupName;\n    private String status;\n\n    public static List<TraceView> decodeFromTraceTransData(String key, MessageExt messageExt) {\n        List<TraceView> messageTraceViewList = new ArrayList<>();\n        String messageBody = new String(messageExt.getBody(), StandardCharsets.UTF_8);\n        if (messageBody == null || messageBody.length() <= 0) {\n            return messageTraceViewList;\n        }\n\n        List<TraceContext> traceContextList = TraceDataEncoder.decoderFromTraceDataString(messageBody);\n\n        for (TraceContext context : traceContextList) {\n            TraceView messageTraceView = new TraceView();\n            TraceBean traceBean = context.getTraceBeans().get(0);\n            if (!traceBean.getMsgId().equals(key)) {\n                continue;\n            }\n            messageTraceView.setCostTime(context.getCostTime());\n            messageTraceView.setGroupName(context.getGroupName());\n            if (context.isSuccess()) {\n                messageTraceView.setStatus(\"success\");\n            } else {\n                messageTraceView.setStatus(\"failed\");\n            }\n            messageTraceView.setKeys(traceBean.getKeys());\n            messageTraceView.setMsgId(traceBean.getMsgId());\n            messageTraceView.setTags(traceBean.getTags());\n            messageTraceView.setTopic(traceBean.getTopic());\n            messageTraceView.setMsgType(context.getTraceType().name());\n            messageTraceView.setOffSetMsgId(traceBean.getOffsetMsgId());\n            messageTraceView.setTimeStamp(context.getTimeStamp());\n            messageTraceView.setStoreHost(traceBean.getStoreHost());\n            messageTraceView.setClientHost(messageExt.getBornHostString());\n            messageTraceViewList.add(messageTraceView);\n        }\n        return messageTraceViewList;\n    }\n\n    public String getMsgId() {\n        return msgId;\n    }\n\n    public void setMsgId(String msgId) {\n        this.msgId = msgId;\n    }\n\n    public String getTags() {\n        return tags;\n    }\n\n    public void setTags(String tags) {\n        this.tags = tags;\n    }\n\n    public String getKeys() {\n        return keys;\n    }\n\n    public void setKeys(String keys) {\n        this.keys = keys;\n    }\n\n    public String getStoreHost() {\n        return storeHost;\n    }\n\n    public void setStoreHost(String storeHost) {\n        this.storeHost = storeHost;\n    }\n\n    public String getClientHost() {\n        return clientHost;\n    }\n\n    public void setClientHost(String clientHost) {\n        this.clientHost = clientHost;\n    }\n\n    public int getCostTime() {\n        return costTime;\n    }\n\n    public void setCostTime(int costTime) {\n        this.costTime = costTime;\n    }\n\n    public String getMsgType() {\n        return msgType;\n    }\n\n    public void setMsgType(String msgType) {\n        this.msgType = msgType;\n    }\n\n    public String getOffSetMsgId() {\n        return offSetMsgId;\n    }\n\n    public void setOffSetMsgId(String offSetMsgId) {\n        this.offSetMsgId = offSetMsgId;\n    }\n\n    public long getTimeStamp() {\n        return timeStamp;\n    }\n\n    public void setTimeStamp(long timeStamp) {\n        this.timeStamp = timeStamp;\n    }\n\n    public long getBornTime() {\n        return bornTime;\n    }\n\n    public void setBornTime(long bornTime) {\n        this.bornTime = bornTime;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getGroupName() {\n        return groupName;\n    }\n\n    public void setGroupName(String groupName) {\n        this.groupName = groupName;\n    }\n\n    public String getStatus() {\n        return status;\n    }\n\n    public void setStatus(String status) {\n        this.status = status;\n    }\n}"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageOpenTracingHookImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.trace.hook;\n\nimport io.opentracing.Span;\nimport io.opentracing.SpanContext;\nimport io.opentracing.Tracer;\nimport io.opentracing.propagation.Format;\nimport io.opentracing.propagation.TextMapAdapter;\nimport io.opentracing.tag.Tags;\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.rocketmq.client.hook.ConsumeMessageContext;\nimport org.apache.rocketmq.client.hook.ConsumeMessageHook;\nimport org.apache.rocketmq.client.trace.TraceConstants;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\n\n\npublic class ConsumeMessageOpenTracingHookImpl implements ConsumeMessageHook {\n\n    private Tracer tracer;\n\n    public ConsumeMessageOpenTracingHookImpl(Tracer tracer) {\n        this.tracer = tracer;\n    }\n\n    @Override\n    public String hookName() {\n        return \"ConsumeMessageOpenTracingHook\";\n    }\n\n    @Override\n    public void consumeMessageBefore(ConsumeMessageContext context) {\n        if (context == null || context.getMsgList() == null || context.getMsgList().isEmpty()) {\n            return;\n        }\n        List<Span> spanList = new ArrayList<>();\n        for (MessageExt msg : context.getMsgList()) {\n            if (msg == null) {\n                continue;\n            }\n            Tracer.SpanBuilder spanBuilder = tracer\n                    .buildSpan(TraceConstants.FROM_PREFIX + msg.getTopic())\n                    .withTag(Tags.SPAN_KIND, Tags.SPAN_KIND_CONSUMER);\n            SpanContext spanContext = tracer.extract(Format.Builtin.TEXT_MAP, new TextMapAdapter(msg.getProperties()));\n            if (spanContext != null) {\n                spanBuilder.asChildOf(spanContext);\n            }\n            Span span = spanBuilder.start();\n\n            span.setTag(Tags.PEER_SERVICE, TraceConstants.ROCKETMQ_SERVICE);\n            span.setTag(Tags.MESSAGE_BUS_DESTINATION, NamespaceUtil.withoutNamespace(msg.getTopic()));\n            span.setTag(TraceConstants.ROCKETMQ_MSG_ID, msg.getMsgId());\n            span.setTag(TraceConstants.ROCKETMQ_TAGS, msg.getTags());\n            span.setTag(TraceConstants.ROCKETMQ_KEYS, msg.getKeys());\n            span.setTag(TraceConstants.ROCKETMQ_BODY_LENGTH, msg.getStoreSize());\n            span.setTag(TraceConstants.ROCKETMQ_RETRY_TIMERS, msg.getReconsumeTimes());\n            span.setTag(TraceConstants.ROCKETMQ_REGION_ID, msg.getProperty(MessageConst.PROPERTY_MSG_REGION));\n            spanList.add(span);\n        }\n        context.setMqTraceContext(spanList);\n    }\n\n    @Override\n    public void consumeMessageAfter(ConsumeMessageContext context) {\n        if (context == null || context.getMsgList() == null || context.getMsgList().isEmpty()) {\n            return;\n        }\n        List<Span> spanList = (List<Span>) context.getMqTraceContext();\n        if (spanList == null) {\n            return;\n        }\n        for (Span span : spanList) {\n            span.setTag(TraceConstants.ROCKETMQ_SUCCESS, context.isSuccess());\n            span.finish();\n        }\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageTraceHookImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.trace.hook;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeReturnType;\nimport org.apache.rocketmq.client.hook.ConsumeMessageContext;\nimport org.apache.rocketmq.client.hook.ConsumeMessageHook;\nimport org.apache.rocketmq.client.trace.TraceBean;\nimport org.apache.rocketmq.client.trace.TraceContext;\nimport org.apache.rocketmq.client.trace.TraceDispatcher;\nimport org.apache.rocketmq.client.trace.TraceType;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\n\npublic class ConsumeMessageTraceHookImpl implements ConsumeMessageHook {\n\n    private TraceDispatcher localDispatcher;\n\n    public ConsumeMessageTraceHookImpl(TraceDispatcher localDispatcher) {\n        this.localDispatcher = localDispatcher;\n    }\n\n    @Override\n    public String hookName() {\n        return \"ConsumeMessageTraceHook\";\n    }\n\n    @Override\n    public void consumeMessageBefore(ConsumeMessageContext context) {\n        if (context == null || context.getMsgList() == null || context.getMsgList().isEmpty()) {\n            return;\n        }\n        TraceContext traceContext = new TraceContext();\n        context.setMqTraceContext(traceContext);\n        traceContext.setTraceType(TraceType.SubBefore);\n        traceContext.setGroupName(NamespaceUtil.withoutNamespace(context.getConsumerGroup()));\n        List<TraceBean> beans = new ArrayList<>();\n        for (MessageExt msg : context.getMsgList()) {\n            if (msg == null) {\n                continue;\n            }\n            String regionId = msg.getProperty(MessageConst.PROPERTY_MSG_REGION);\n            String traceOn = msg.getProperty(MessageConst.PROPERTY_TRACE_SWITCH);\n\n            if (traceOn != null && traceOn.equals(\"false\")) {\n                // If trace switch is false ,skip it\n                continue;\n            }\n            TraceBean traceBean = new TraceBean();\n            traceBean.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic()));\n            traceBean.setMsgId(msg.getMsgId());\n            traceBean.setTags(msg.getTags());\n            traceBean.setKeys(msg.getKeys());\n            traceBean.setStoreTime(msg.getStoreTimestamp());\n            traceBean.setBodyLength(msg.getStoreSize());\n            traceBean.setRetryTimes(msg.getReconsumeTimes());\n            traceContext.setRegionId(regionId);\n            beans.add(traceBean);\n        }\n        if (beans.size() > 0) {\n            traceContext.setTraceBeans(beans);\n            traceContext.setTimeStamp(System.currentTimeMillis());\n            localDispatcher.append(traceContext);\n        }\n    }\n\n    @Override\n    public void consumeMessageAfter(ConsumeMessageContext context) {\n        if (context == null || context.getMsgList() == null || context.getMsgList().isEmpty()) {\n            return;\n        }\n        TraceContext subBeforeContext = (TraceContext) context.getMqTraceContext();\n\n        if (subBeforeContext.getTraceBeans() == null || subBeforeContext.getTraceBeans().size() < 1) {\n            // If subBefore bean is null ,skip it\n            return;\n        }\n        TraceContext subAfterContext = new TraceContext();\n        subAfterContext.setTraceType(TraceType.SubAfter);\n        subAfterContext.setRegionId(subBeforeContext.getRegionId());\n        subAfterContext.setGroupName(NamespaceUtil.withoutNamespace(subBeforeContext.getGroupName()));\n        subAfterContext.setRequestId(subBeforeContext.getRequestId());\n        subAfterContext.setAccessChannel(context.getAccessChannel());\n        subAfterContext.setSuccess(context.isSuccess());\n\n        // Calculate the cost time for processing messages\n        int costTime = (int) ((System.currentTimeMillis() - subBeforeContext.getTimeStamp()) / context.getMsgList().size());\n        subAfterContext.setCostTime(costTime);\n        subAfterContext.setTraceBeans(subBeforeContext.getTraceBeans());\n        Map<String, String> props = context.getProps();\n        if (props != null) {\n            String contextType = props.get(MixAll.CONSUME_CONTEXT_TYPE);\n            if (contextType != null) {\n                subAfterContext.setContextCode(ConsumeReturnType.valueOf(contextType).ordinal());\n            }\n        }\n        localDispatcher.append(subAfterContext);\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/trace/hook/DefaultRecallMessageTraceHook.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.trace.hook;\n\nimport org.apache.rocketmq.client.trace.TraceBean;\nimport org.apache.rocketmq.client.trace.TraceContext;\nimport org.apache.rocketmq.client.trace.TraceDispatcher;\nimport org.apache.rocketmq.client.trace.TraceType;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.producer.RecallMessageHandle;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;\n\nimport java.util.ArrayList;\n\npublic class DefaultRecallMessageTraceHook implements RPCHook {\n\n    private static final String RECALL_TRACE_ENABLE_KEY = \"com.rocketmq.recall.default.trace.enable\";\n    private boolean enableDefaultTrace = Boolean.parseBoolean(System.getProperty(RECALL_TRACE_ENABLE_KEY, \"false\"));\n    private TraceDispatcher traceDispatcher;\n\n    public DefaultRecallMessageTraceHook(TraceDispatcher traceDispatcher) {\n        this.traceDispatcher = traceDispatcher;\n    }\n\n    @Override\n    public void doBeforeRequest(String remoteAddr, RemotingCommand request) {\n    }\n\n    @Override\n    public void doAfterResponse(String remoteAddr, RemotingCommand request, RemotingCommand response) {\n        if (request.getCode() != RequestCode.RECALL_MESSAGE\n            || !enableDefaultTrace\n            || null == response.getExtFields()\n            || null == response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION)\n            || null == traceDispatcher) {\n            return;\n        }\n\n        try {\n            String regionId = response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION);\n            RecallMessageRequestHeader requestHeader =\n                request.decodeCommandCustomHeader(RecallMessageRequestHeader.class);\n            String topic = NamespaceUtil.withoutNamespace(requestHeader.getTopic());\n            String group = NamespaceUtil.withoutNamespace(requestHeader.getProducerGroup());\n            String recallHandle = requestHeader.getRecallHandle();\n            RecallMessageHandle.HandleV1 handleV1 =\n                (RecallMessageHandle.HandleV1) RecallMessageHandle.decodeHandle(recallHandle);\n\n            TraceBean traceBean = new TraceBean();\n            traceBean.setTopic(topic);\n            traceBean.setMsgId(handleV1.getMessageId());\n\n            TraceContext traceContext = new TraceContext();\n            traceContext.setRegionId(regionId);\n            traceContext.setTraceBeans(new ArrayList<>(1));\n            traceContext.setTraceType(TraceType.Recall);\n            traceContext.setGroupName(group);\n            traceContext.getTraceBeans().add(traceBean);\n            traceContext.setSuccess(ResponseCode.SUCCESS == response.getCode());\n\n            traceDispatcher.append(traceContext);\n        } catch (Exception e) {\n        }\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/trace/hook/EndTransactionOpenTracingHookImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.trace.hook;\n\nimport io.opentracing.Span;\nimport io.opentracing.SpanContext;\nimport io.opentracing.Tracer;\nimport io.opentracing.propagation.Format;\nimport io.opentracing.propagation.TextMapAdapter;\nimport io.opentracing.tag.Tags;\nimport org.apache.rocketmq.client.hook.EndTransactionContext;\nimport org.apache.rocketmq.client.hook.EndTransactionHook;\nimport org.apache.rocketmq.client.trace.TraceConstants;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageType;\n\npublic class EndTransactionOpenTracingHookImpl implements EndTransactionHook {\n\n    private Tracer tracer;\n\n    public EndTransactionOpenTracingHookImpl(Tracer tracer) {\n        this.tracer = tracer;\n    }\n\n    @Override\n    public String hookName() {\n        return \"EndTransactionOpenTracingHook\";\n    }\n\n    @Override\n    public void endTransaction(EndTransactionContext context) {\n        if (context == null) {\n            return;\n        }\n        Message msg = context.getMessage();\n        Tracer.SpanBuilder spanBuilder = tracer\n                .buildSpan(TraceConstants.END_TRANSACTION)\n                .withTag(Tags.SPAN_KIND, Tags.SPAN_KIND_PRODUCER);\n        SpanContext spanContext = tracer.extract(Format.Builtin.TEXT_MAP, new TextMapAdapter(msg.getProperties()));\n        if (spanContext != null) {\n            spanBuilder.asChildOf(spanContext);\n        }\n\n        Span span = spanBuilder.start();\n        span.setTag(Tags.PEER_SERVICE, TraceConstants.ROCKETMQ_SERVICE);\n        span.setTag(Tags.MESSAGE_BUS_DESTINATION, msg.getTopic());\n        span.setTag(TraceConstants.ROCKETMQ_TAGS, msg.getTags());\n        span.setTag(TraceConstants.ROCKETMQ_KEYS, msg.getKeys());\n        span.setTag(TraceConstants.ROCKETMQ_STORE_HOST, context.getBrokerAddr());\n        span.setTag(TraceConstants.ROCKETMQ_MSG_ID, context.getMsgId());\n        span.setTag(TraceConstants.ROCKETMQ_MSG_TYPE, MessageType.Trans_msg_Commit.name());\n        span.setTag(TraceConstants.ROCKETMQ_TRANSACTION_ID, context.getTransactionId());\n        span.setTag(TraceConstants.ROCKETMQ_TRANSACTION_STATE, context.getTransactionState().name());\n        span.setTag(TraceConstants.ROCKETMQ_IS_FROM_TRANSACTION_CHECK, context.isFromTransactionCheck());\n        span.finish();\n    }\n\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/trace/hook/EndTransactionTraceHookImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.trace.hook;\n\nimport java.util.ArrayList;\nimport org.apache.rocketmq.client.hook.EndTransactionContext;\nimport org.apache.rocketmq.client.hook.EndTransactionHook;\nimport org.apache.rocketmq.client.trace.AsyncTraceDispatcher;\nimport org.apache.rocketmq.client.trace.TraceBean;\nimport org.apache.rocketmq.client.trace.TraceContext;\nimport org.apache.rocketmq.client.trace.TraceDispatcher;\nimport org.apache.rocketmq.client.trace.TraceType;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageType;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\n\npublic class EndTransactionTraceHookImpl implements EndTransactionHook {\n\n    private TraceDispatcher localDispatcher;\n\n    public EndTransactionTraceHookImpl(TraceDispatcher localDispatcher) {\n        this.localDispatcher = localDispatcher;\n    }\n\n    @Override\n    public String hookName() {\n        return \"EndTransactionTraceHook\";\n    }\n\n    @Override\n    public void endTransaction(EndTransactionContext context) {\n        //if it is message trace data,then it doesn't recorded\n        if (context == null || context.getMessage().getTopic().startsWith(((AsyncTraceDispatcher) localDispatcher).getTraceTopicName())) {\n            return;\n        }\n        Message msg = context.getMessage();\n        //build the context content of TuxeTraceContext\n        TraceContext tuxeContext = new TraceContext();\n        tuxeContext.setTraceBeans(new ArrayList<>(1));\n        tuxeContext.setTraceType(TraceType.EndTransaction);\n        tuxeContext.setGroupName(NamespaceUtil.withoutNamespace(context.getProducerGroup()));\n        //build the data bean object of message trace\n        TraceBean traceBean = new TraceBean();\n        traceBean.setTopic(NamespaceUtil.withoutNamespace(context.getMessage().getTopic()));\n        traceBean.setTags(context.getMessage().getTags());\n        traceBean.setKeys(context.getMessage().getKeys());\n        traceBean.setStoreHost(context.getBrokerAddr());\n        traceBean.setMsgType(MessageType.Trans_msg_Commit);\n        traceBean.setClientHost(((AsyncTraceDispatcher)localDispatcher).getHostProducer().getMqClientFactory().getClientId());\n        traceBean.setMsgId(context.getMsgId());\n        traceBean.setTransactionState(context.getTransactionState());\n        traceBean.setTransactionId(context.getTransactionId());\n        traceBean.setFromTransactionCheck(context.isFromTransactionCheck());\n        String regionId = msg.getProperty(MessageConst.PROPERTY_MSG_REGION);\n        if (regionId == null || regionId.isEmpty()) {\n            regionId = MixAll.DEFAULT_TRACE_REGION_ID;\n        }\n        tuxeContext.setRegionId(regionId);\n        tuxeContext.getTraceBeans().add(traceBean);\n        tuxeContext.setTimeStamp(System.currentTimeMillis());\n        localDispatcher.append(tuxeContext);\n    }\n\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageOpenTracingHookImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.trace.hook;\n\nimport io.opentracing.Span;\nimport io.opentracing.SpanContext;\nimport io.opentracing.Tracer;\nimport io.opentracing.propagation.Format;\nimport io.opentracing.propagation.TextMapAdapter;\nimport io.opentracing.tag.Tags;\nimport org.apache.rocketmq.client.hook.SendMessageContext;\nimport org.apache.rocketmq.client.hook.SendMessageHook;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.client.trace.TraceConstants;\nimport org.apache.rocketmq.common.message.Message;\n\npublic class SendMessageOpenTracingHookImpl implements SendMessageHook {\n\n    private Tracer tracer;\n\n    public SendMessageOpenTracingHookImpl(Tracer tracer) {\n        this.tracer = tracer;\n    }\n\n    @Override\n    public String hookName() {\n        return \"SendMessageOpenTracingHook\";\n    }\n\n    @Override\n    public void sendMessageBefore(SendMessageContext context) {\n        if (context == null) {\n            return;\n        }\n        Message msg = context.getMessage();\n        Tracer.SpanBuilder spanBuilder = tracer\n            .buildSpan(TraceConstants.TO_PREFIX + msg.getTopic())\n            .withTag(Tags.SPAN_KIND, Tags.SPAN_KIND_PRODUCER);\n        SpanContext spanContext = tracer.extract(Format.Builtin.TEXT_MAP, new TextMapAdapter(msg.getProperties()));\n        if (spanContext != null) {\n            spanBuilder.asChildOf(spanContext);\n        }\n        Span span = spanBuilder.start();\n        tracer.inject(span.context(), Format.Builtin.TEXT_MAP, new TextMapAdapter(msg.getProperties()));\n        span.setTag(Tags.PEER_SERVICE, TraceConstants.ROCKETMQ_SERVICE);\n        span.setTag(Tags.MESSAGE_BUS_DESTINATION, msg.getTopic());\n        span.setTag(TraceConstants.ROCKETMQ_TAGS, msg.getTags());\n        span.setTag(TraceConstants.ROCKETMQ_KEYS, msg.getKeys());\n        span.setTag(TraceConstants.ROCKETMQ_STORE_HOST, context.getBrokerAddr());\n        span.setTag(TraceConstants.ROCKETMQ_MSG_TYPE, context.getMsgType().name());\n        span.setTag(TraceConstants.ROCKETMQ_BODY_LENGTH, null == msg.getBody() ? 0 : msg.getBody().length);\n        context.setMqTraceContext(span);\n    }\n\n    @Override\n    public void sendMessageAfter(SendMessageContext context) {\n        if (context == null || context.getMqTraceContext() == null) {\n            return;\n        }\n        if (context.getSendResult() == null) {\n            return;\n        }\n\n        if (context.getSendResult().getRegionId() == null) {\n            return;\n        }\n\n        Span span = (Span) context.getMqTraceContext();\n        span.setTag(TraceConstants.ROCKETMQ_SUCCESS, context.getSendResult().getSendStatus().equals(SendStatus.SEND_OK));\n        span.setTag(TraceConstants.ROCKETMQ_MSG_ID, context.getSendResult().getMsgId());\n        span.setTag(TraceConstants.ROCKETMQ_REGION_ID, context.getSendResult().getRegionId());\n        span.finish();\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageTraceHookImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.trace.hook;\n\nimport java.util.ArrayList;\nimport org.apache.rocketmq.client.hook.SendMessageContext;\nimport org.apache.rocketmq.client.hook.SendMessageHook;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.client.trace.AsyncTraceDispatcher;\nimport org.apache.rocketmq.client.trace.TraceBean;\nimport org.apache.rocketmq.client.trace.TraceContext;\nimport org.apache.rocketmq.client.trace.TraceDispatcher;\nimport org.apache.rocketmq.client.trace.TraceType;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\n\npublic class SendMessageTraceHookImpl implements SendMessageHook {\n\n    private TraceDispatcher localDispatcher;\n\n    public SendMessageTraceHookImpl(TraceDispatcher localDispatcher) {\n        this.localDispatcher = localDispatcher;\n    }\n\n    @Override\n    public String hookName() {\n        return \"SendMessageTraceHook\";\n    }\n\n    @Override\n    public void sendMessageBefore(SendMessageContext context) {\n        //if it is message trace data,then it doesn't recorded\n        if (context == null || context.getMessage().getTopic().startsWith(((AsyncTraceDispatcher) localDispatcher).getTraceTopicName())) {\n            return;\n        }\n        //build the context content of TraceContext\n        TraceContext traceContext = new TraceContext();\n        traceContext.setTraceBeans(new ArrayList<>(1));\n        context.setMqTraceContext(traceContext);\n        traceContext.setTraceType(TraceType.Pub);\n        traceContext.setGroupName(NamespaceUtil.withoutNamespace(context.getProducerGroup()));\n        //build the data bean object of message trace\n        TraceBean traceBean = new TraceBean();\n        traceBean.setTopic(NamespaceUtil.withoutNamespace(context.getMessage().getTopic()));\n        traceBean.setTags(context.getMessage().getTags());\n        traceBean.setKeys(context.getMessage().getKeys());\n        traceBean.setStoreHost(context.getBrokerAddr());\n        int bodyLength = null == context.getMessage().getBody() ? 0 : context.getMessage().getBody().length;\n        traceBean.setBodyLength(bodyLength);\n        traceBean.setMsgType(context.getMsgType());\n        traceContext.getTraceBeans().add(traceBean);\n    }\n\n    @Override\n    public void sendMessageAfter(SendMessageContext context) {\n        //if it is message trace data,then it doesn't recorded\n        if (context == null || context.getMessage().getTopic().startsWith(((AsyncTraceDispatcher) localDispatcher).getTraceTopicName())\n            || context.getMqTraceContext() == null) {\n            return;\n        }\n        if (context.getSendResult() == null) {\n            return;\n        }\n\n        if (context.getSendResult().getRegionId() == null\n            || !context.getSendResult().isTraceOn()) {\n            // if switch is false,skip it\n            return;\n        }\n\n        TraceContext traceContext = (TraceContext) context.getMqTraceContext();\n        TraceBean traceBean = traceContext.getTraceBeans().get(0);\n        int costTime = (int) ((System.currentTimeMillis() - traceContext.getTimeStamp()) / traceContext.getTraceBeans().size());\n        traceContext.setCostTime(costTime);\n        if (context.getSendResult().getSendStatus().equals(SendStatus.SEND_OK)) {\n            traceContext.setSuccess(true);\n        } else {\n            traceContext.setSuccess(false);\n        }\n        traceContext.setRegionId(context.getSendResult().getRegionId());\n        traceBean.setMsgId(context.getSendResult().getMsgId());\n        traceBean.setOffsetMsgId(context.getSendResult().getOffsetMsgId());\n        traceBean.setStoreTime(traceContext.getTimeStamp() + costTime / 2);\n        localDispatcher.append(traceContext);\n    }\n}\n"
  },
  {
    "path": "client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.utils;\n\nimport org.apache.rocketmq.client.common.ClientErrorCode;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\n\npublic class MessageUtil {\n    public static Message createReplyMessage(final Message requestMessage, final byte[] body) throws MQClientException {\n        if (requestMessage != null) {\n            Message replyMessage = new Message();\n            String cluster = requestMessage.getProperty(MessageConst.PROPERTY_CLUSTER);\n            String replyTo = requestMessage.getProperty(MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT);\n            String correlationId = requestMessage.getProperty(MessageConst.PROPERTY_CORRELATION_ID);\n            String ttl = requestMessage.getProperty(MessageConst.PROPERTY_MESSAGE_TTL);\n            replyMessage.setBody(body);\n            if (cluster != null) {\n                String replyTopic = MixAll.getReplyTopic(cluster);\n                replyMessage.setTopic(replyTopic);\n                MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_TYPE, MixAll.REPLY_MESSAGE_FLAG);\n                MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_CORRELATION_ID, correlationId);\n                MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT, replyTo);\n                MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_TTL, ttl);\n\n                return replyMessage;\n            } else {\n                throw new MQClientException(ClientErrorCode.CREATE_REPLY_MESSAGE_EXCEPTION, \"create reply message fail, requestMessage error, property[\" + MessageConst.PROPERTY_CLUSTER + \"] is null.\");\n            }\n        }\n        throw new MQClientException(ClientErrorCode.CREATE_REPLY_MESSAGE_EXCEPTION, \"create reply message fail, requestMessage cannot be null.\");\n    }\n\n    public static String getReplyToClient(final Message msg) {\n        return msg.getProperty(MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT);\n    }\n}\n"
  },
  {
    "path": "client/src/main/resources/rmq.client.logback.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n      http://www.apache.org/licenses/LICENSE-2.0\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n<configuration>\n    <conversionRule conversionWord=\"pid\" converterClass=\"io.github.aliyunmq.logback.extensions.ProcessIdConverter\"/>\n    <appender name=\"CustomConsoleAppender\" class=\"io.github.aliyunmq.logback.extensions.CustomConsoleAppender\">\n        <encoder>\n            <pattern>%yellow(%d{yyy-MM-dd HH:mm:ss.SSS,GMT+8}) %highlight(%-5p) %boldWhite([%pid]) %magenta([%t]) %boldGreen([%logger{12}#%M:%L]) - %m%n\n            </pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"DefaultAppenderInner\" class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <append>true</append>\n        <File>\n            ${rocketmq.log.root:-${user.home}${file.separator}logs${file.separator}rocketmqlogs}${file.separator}rocketmq_client.log\n        </File>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <FileNamePattern>\n                ${rocketmq.log.root:-${user.home}${file.separator}logs${file.separator}rocketmqlogs}${file.separator}other_days${file.separator}rocketmq_client-%i.log.gz\n            </FileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>${rocketmq.log.file.maxIndex:-10}</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>64MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder class=\"ch.qos.logback.classic.encoder.PatternLayoutEncoder\">\n            <pattern>%d{yyy-MM-dd HH:mm:ss.SSS,GMT+8} %-5p [%pid] [%t] [%logger{12}#%M:%L] - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"DefaultAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"DefaultAppenderInner\"/>\n    </appender>\n    <root level=\"${rocketmq.log.level:-info}\">\n        <appender-ref ref=\"CustomConsoleAppender\"/>\n        <appender-ref ref=\"DefaultAppender\" additivity=\"false\"/>\n    </root>\n    <!-- ref: https://github.com/grpc/grpc-java/issues/3033 -->\n    <!-- <logger name=\"io.grpc\" level=\"warn\"/> -->\n</configuration>\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/acl/common/AclClientRPCHookTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.acl.common;\n\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.RequestType;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.junit.Test;\n\nimport java.lang.reflect.Field;\nimport java.util.SortedMap;\nimport java.util.TreeMap;\nimport java.util.concurrent.ConcurrentHashMap;\n\nimport static org.apache.rocketmq.acl.common.SessionCredentials.ACCESS_KEY;\nimport static org.apache.rocketmq.acl.common.SessionCredentials.SECURITY_TOKEN;\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class AclClientRPCHookTest {\n    protected ConcurrentHashMap<Class<? extends CommandCustomHeader>, Field[]> fieldCache =\n        new ConcurrentHashMap<>();\n    private AclClientRPCHook aclClientRPCHook = new AclClientRPCHook(null);\n\n    @Test\n    public void testParseRequestContent() {\n        PullMessageRequestHeader requestHeader = new PullMessageRequestHeader();\n        requestHeader.setConsumerGroup(\"group\");\n        requestHeader.setTopic(\"topic\");\n        requestHeader.setQueueId(1);\n        requestHeader.setQueueOffset(2L);\n        requestHeader.setMaxMsgNums(32);\n        requestHeader.setSysFlag(0);\n        requestHeader.setCommitOffset(0L);\n        requestHeader.setSuspendTimeoutMillis(15000L);\n        requestHeader.setSubVersion(0L);\n        RemotingCommand testPullRemotingCommand = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, requestHeader);\n        SortedMap<String, String> oldContent = oldVersionParseRequestContent(testPullRemotingCommand, \"ak\", null);\n        byte[] oldBytes = AclUtils.combineRequestContent(testPullRemotingCommand, oldContent);\n        testPullRemotingCommand.addExtField(ACCESS_KEY, \"ak\");\n        SortedMap<String, String> content = aclClientRPCHook.parseRequestContent(testPullRemotingCommand);\n        byte[] newBytes = AclUtils.combineRequestContent(testPullRemotingCommand, content);\n        assertThat(newBytes).isEqualTo(oldBytes);\n    }\n\n    @Test\n    public void testParseRequestContentWithStreamRequestType() {\n        PullMessageRequestHeader requestHeader = new PullMessageRequestHeader();\n        requestHeader.setConsumerGroup(\"group\");\n        requestHeader.setTopic(\"topic\");\n        requestHeader.setQueueId(1);\n        requestHeader.setQueueOffset(2L);\n        requestHeader.setMaxMsgNums(32);\n        requestHeader.setSysFlag(0);\n        requestHeader.setCommitOffset(0L);\n        requestHeader.setSuspendTimeoutMillis(15000L);\n        requestHeader.setSubVersion(0L);\n        RemotingCommand testPullRemotingCommand = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, requestHeader);\n        testPullRemotingCommand.addExtField(MixAll.REQ_T, String.valueOf(RequestType.STREAM.getCode()));\n        testPullRemotingCommand.addExtField(ACCESS_KEY, \"ak\");\n        SortedMap<String, String> content = aclClientRPCHook.parseRequestContent(testPullRemotingCommand);\n        assertThat(content.get(MixAll.REQ_T)).isEqualTo(String.valueOf(RequestType.STREAM.getCode()));\n    }\n\n    private SortedMap<String, String> oldVersionParseRequestContent(RemotingCommand request, String ak, String securityToken) {\n        CommandCustomHeader header = request.readCustomHeader();\n        // Sort property\n        SortedMap<String, String> map = new TreeMap<>();\n        map.put(ACCESS_KEY, ak);\n        if (securityToken != null) {\n            map.put(SECURITY_TOKEN, securityToken);\n        }\n        try {\n            // Add header properties\n            if (null != header) {\n                Field[] fields = fieldCache.get(header.getClass());\n                if (null == fields) {\n                    fields = header.getClass().getDeclaredFields();\n                    for (Field field : fields) {\n                        field.setAccessible(true);\n                    }\n                    Field[] tmp = fieldCache.putIfAbsent(header.getClass(), fields);\n                    if (null != tmp) {\n                        fields = tmp;\n                    }\n                }\n\n                for (Field field : fields) {\n                    Object value = field.get(header);\n                    if (null != value && !field.isSynthetic()) {\n                        map.put(field.getName(), value.toString());\n                    }\n                }\n            }\n            return map;\n        } catch (Exception e) {\n            throw new RuntimeException(\"incompatible exception.\", e);\n        }\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/acl/common/AclSignerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.acl.common;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class AclSignerTest {\n\n    @Test(expected = Exception.class)\n    public void calSignatureExceptionTest() {\n        AclSigner.calSignature(new byte[]{},\"\");\n    }\n\n    @Test\n    public void calSignatureTest() {\n        String expectedSignature = \"IUc8rrO/0gDch8CjObLQsW2rsiA=\";\n        Assert.assertEquals(expectedSignature, AclSigner.calSignature(\"RocketMQ\", \"12345678\"));\n        Assert.assertEquals(expectedSignature, AclSigner.calSignature(\"RocketMQ\".getBytes(), \"12345678\"));\n    }\n\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/acl/common/AclUtilsTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.acl.common;\n\nimport com.alibaba.fastjson2.JSONObject;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.junit.Assert;\nimport org.junit.Assume;\nimport org.junit.Test;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.UUID;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertTrue;\n\npublic class AclUtilsTest {\n\n    @Test\n    public void testGetAddresses() {\n        String address = \"1.1.1.{1,2,3,4}\";\n        String[] addressArray = AclUtils.getAddresses(address, \"{1,2,3,4}\");\n        List<String> newAddressList = new ArrayList<>(Arrays.asList(addressArray));\n\n        List<String> addressList = new ArrayList<>();\n        addressList.add(\"1.1.1.1\");\n        addressList.add(\"1.1.1.2\");\n        addressList.add(\"1.1.1.3\");\n        addressList.add(\"1.1.1.4\");\n        Assert.assertEquals(newAddressList, addressList);\n\n        // IPv6 test\n        String ipv6Address = \"1:ac41:9987::bb22:666:{1,2,3,4}\";\n        String[] ipv6AddressArray = AclUtils.getAddresses(ipv6Address, \"{1,2,3,4}\");\n        List<String> newIPv6AddressList = new ArrayList<>();\n        Collections.addAll(newIPv6AddressList, ipv6AddressArray);\n\n        List<String> ipv6AddressList = new ArrayList<>();\n        ipv6AddressList.add(\"1:ac41:9987::bb22:666:1\");\n        ipv6AddressList.add(\"1:ac41:9987::bb22:666:2\");\n        ipv6AddressList.add(\"1:ac41:9987::bb22:666:3\");\n        ipv6AddressList.add(\"1:ac41:9987::bb22:666:4\");\n        Assert.assertEquals(newIPv6AddressList, ipv6AddressList);\n    }\n\n    @Test\n    public void testIsScope_StringArray() {\n        String address = \"12\";\n\n        for (int i = 0; i < 6; i++) {\n            boolean isScope = AclUtils.isScope(address, 4);\n            if (i == 3) {\n                Assert.assertTrue(isScope);\n            } else {\n                Assert.assertFalse(isScope);\n            }\n            address = address + \".12\";\n        }\n    }\n\n    @Test\n    public void testIsScope_Array() {\n        String[] address = StringUtils.split(\"12.12.12.12\", \".\");\n        boolean isScope = AclUtils.isScope(address, 4);\n        Assert.assertTrue(isScope);\n        isScope = AclUtils.isScope(address, 3);\n        Assert.assertTrue(isScope);\n\n        address = StringUtils.split(\"12.12.1222.1222\", \".\");\n        isScope = AclUtils.isScope(address, 4);\n        Assert.assertFalse(isScope);\n        isScope = AclUtils.isScope(address, 3);\n        Assert.assertFalse(isScope);\n\n        // IPv6 test\n        address = StringUtils.split(\"1050:0000:0000:0000:0005:0600:300c:326b\", \":\");\n        isScope = AclUtils.isIPv6Scope(address, 8);\n        Assert.assertTrue(isScope);\n        isScope = AclUtils.isIPv6Scope(address, 4);\n        Assert.assertTrue(isScope);\n\n        address = StringUtils.split(\"1050:9876:0000:0000:0005:akkg:300c:326b\", \":\");\n        isScope = AclUtils.isIPv6Scope(address, 8);\n        Assert.assertFalse(isScope);\n        isScope = AclUtils.isIPv6Scope(address, 4);\n        Assert.assertTrue(isScope);\n\n        address = StringUtils.split(AclUtils.expandIP(\"1050::0005:akkg:300c:326b\", 8), \":\");\n        isScope = AclUtils.isIPv6Scope(address, 8);\n        Assert.assertFalse(isScope);\n        isScope = AclUtils.isIPv6Scope(address, 4);\n        Assert.assertTrue(isScope);\n    }\n\n    @Test\n    public void testIsScope_String() {\n        for (int i = 0; i < 256; i++) {\n            boolean isScope = AclUtils.isScope(i + \"\");\n            Assert.assertTrue(isScope);\n        }\n        boolean isScope = AclUtils.isScope(\"-1\");\n        Assert.assertFalse(isScope);\n        isScope = AclUtils.isScope(\"256\");\n        Assert.assertFalse(isScope);\n    }\n\n    @Test\n    public void testIsScope_Integral() {\n        for (int i = 0; i < 256; i++) {\n            boolean isScope = AclUtils.isScope(i);\n            Assert.assertTrue(isScope);\n        }\n        boolean isScope = AclUtils.isScope(-1);\n        Assert.assertFalse(isScope);\n        isScope = AclUtils.isScope(256);\n        Assert.assertFalse(isScope);\n\n        // IPv6 test\n        int min = Integer.parseInt(\"0\", 16);\n        int max = Integer.parseInt(\"ffff\", 16);\n        for (int i = min; i < max + 1; i++) {\n            isScope = AclUtils.isIPv6Scope(i);\n            Assert.assertTrue(isScope);\n        }\n        isScope = AclUtils.isIPv6Scope(-1);\n        Assert.assertFalse(isScope);\n        isScope = AclUtils.isIPv6Scope(max + 1);\n        Assert.assertFalse(isScope);\n    }\n\n    @Test\n    public void testIsAsterisk() {\n        boolean isAsterisk = AclUtils.isAsterisk(\"*\");\n        Assert.assertTrue(isAsterisk);\n\n        isAsterisk = AclUtils.isAsterisk(\",\");\n        Assert.assertFalse(isAsterisk);\n    }\n\n    @Test\n    public void testIsComma() {\n        boolean isColon = AclUtils.isComma(\",\");\n        Assert.assertTrue(isColon);\n\n        isColon = AclUtils.isComma(\"-\");\n        Assert.assertFalse(isColon);\n    }\n\n    @Test\n    public void testIsMinus() {\n        boolean isMinus = AclUtils.isMinus(\"-\");\n        Assert.assertTrue(isMinus);\n\n        isMinus = AclUtils.isMinus(\"*\");\n        Assert.assertFalse(isMinus);\n    }\n\n    @Test\n    public void testV6ipProcess() {\n        String remoteAddr = \"5::7:6:1-200:*\";\n        Assert.assertEquals(AclUtils.v6ipProcess(remoteAddr), \"0005:0000:0000:0000:0007:0006\");\n\n        remoteAddr = \"5::7:6:1-200\";\n        Assert.assertEquals(AclUtils.v6ipProcess(remoteAddr), \"0005:0000:0000:0000:0000:0007:0006\");\n        remoteAddr = \"5::7:6:*\";\n        Assert.assertEquals(AclUtils.v6ipProcess(remoteAddr), \"0005:0000:0000:0000:0000:0007:0006\");\n\n        remoteAddr = \"5:7:6:*\";\n        Assert.assertEquals(AclUtils.v6ipProcess(remoteAddr), \"0005:0007:0006\");\n    }\n\n    @Test\n    public void testExpandIP() {\n        Assert.assertEquals(AclUtils.expandIP(\"::\", 8), \"0000:0000:0000:0000:0000:0000:0000:0000\");\n        Assert.assertEquals(AclUtils.expandIP(\"::1\", 8), \"0000:0000:0000:0000:0000:0000:0000:0001\");\n        Assert.assertEquals(AclUtils.expandIP(\"3::\", 8), \"0003:0000:0000:0000:0000:0000:0000:0000\");\n        Assert.assertEquals(AclUtils.expandIP(\"2::2\", 8), \"0002:0000:0000:0000:0000:0000:0000:0002\");\n        Assert.assertEquals(AclUtils.expandIP(\"4::aac4:92\", 8), \"0004:0000:0000:0000:0000:0000:AAC4:0092\");\n        Assert.assertEquals(AclUtils.expandIP(\"ab23:56:901a::cc6:765:bb:9011\", 8), \"AB23:0056:901A:0000:0CC6:0765:00BB:9011\");\n        Assert.assertEquals(AclUtils.expandIP(\"ab23:56:901a:1:cc6:765:bb:9011\", 8), \"AB23:0056:901A:0001:0CC6:0765:00BB:9011\");\n        Assert.assertEquals(AclUtils.expandIP(\"5::7:6\", 6), \"0005:0000:0000:0000:0007:0006\");\n    }\n\n    private static String randomTmpFile() {\n        String tmpFileName = System.getProperty(\"java.io.tmpdir\");\n        // https://rationalpi.wordpress.com/2007/01/26/javaiotmpdir-inconsitency/\n        if (!tmpFileName.endsWith(File.separator)) {\n            tmpFileName += File.separator;\n        }\n        tmpFileName += UUID.randomUUID() + \".yml\";\n        return tmpFileName;\n    }\n\n    @Test\n    public void getYamlDataIgnoreFileNotFoundExceptionTest() {\n\n        JSONObject yamlDataObject = AclUtils.getYamlDataObject(\"plain_acl.yml\", JSONObject.class);\n        Assert.assertNull(yamlDataObject);\n    }\n\n    @Test\n    public void getAclRPCHookTest() throws IOException {\n        try (InputStream is = AclUtilsTest.class.getClassLoader().getResourceAsStream(\"conf/plain_acl_incomplete.yml\")) {\n            RPCHook incompleteContRPCHook = AclUtils.getAclRPCHook(is);\n            Assert.assertNull(incompleteContRPCHook);\n        }\n    }\n\n    @Test\n    public void testGetAclRPCHookByFileName() {\n        // Skip this test if running in Bazel, as the resource path is a path inside the JAR.\n        Assume.assumeTrue(System.getProperty(\"build.bazel\") == null);\n        RPCHook actual = AclUtils.getAclRPCHook(Objects.requireNonNull(AclUtilsTest.class.getResource(\"/acl_hook/plain_acl.yml\")).getPath());\n        assertNotNull(actual);\n        assertTrue(actual instanceof AclClientRPCHook);\n        assertAclClientRPCHook((AclClientRPCHook) actual);\n    }\n\n    @Test\n    public void testGetAclRPCHookByInputStream() {\n        RPCHook actual = AclUtils.getAclRPCHook(Objects.requireNonNull(AclUtilsTest.class.getResourceAsStream(\"/acl_hook/plain_acl.yml\")));\n        assertNotNull(actual);\n        assertTrue(actual instanceof AclClientRPCHook);\n        assertAclClientRPCHook((AclClientRPCHook) actual);\n    }\n\n    private void assertAclClientRPCHook(final AclClientRPCHook actual) {\n        assertEquals(\"rocketmq2\", actual.getSessionCredentials().getAccessKey());\n        assertEquals(\"12345678\", actual.getSessionCredentials().getSecretKey());\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/acl/common/PermissionTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.acl.common;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class PermissionTest {\n\n    @Test\n    public void fromStringGetPermissionTest() {\n        byte perm = Permission.parsePermFromString(\"PUB\");\n        Assert.assertEquals(perm, Permission.PUB);\n\n        perm = Permission.parsePermFromString(\"SUB\");\n        Assert.assertEquals(perm, Permission.SUB);\n\n        perm = Permission.parsePermFromString(\"PUB|SUB\");\n        Assert.assertEquals(perm, Permission.PUB | Permission.SUB);\n\n        perm = Permission.parsePermFromString(\"SUB|PUB\");\n        Assert.assertEquals(perm, Permission.PUB | Permission.SUB);\n\n        perm = Permission.parsePermFromString(\"DENY\");\n        Assert.assertEquals(perm, Permission.DENY);\n\n        perm = Permission.parsePermFromString(\"1\");\n        Assert.assertEquals(perm, Permission.DENY);\n\n        perm = Permission.parsePermFromString(null);\n        Assert.assertEquals(perm, Permission.DENY);\n\n    }\n\n    @Test\n    public void AclExceptionTest() {\n        AclException aclException = new AclException(\"CAL_SIGNATURE_FAILED\",10015);\n        AclException aclExceptionWithMessage = new AclException(\"CAL_SIGNATURE_FAILED\",10015,\"CAL_SIGNATURE_FAILED Exception\");\n        Assert.assertEquals(aclException.getCode(),10015);\n        Assert.assertEquals(aclExceptionWithMessage.getStatus(),\"CAL_SIGNATURE_FAILED\");\n        aclException.setCode(10016);\n        Assert.assertEquals(aclException.getCode(),10016);\n        aclException.setStatus(\"netAddress examine scope Exception netAddress\");\n        Assert.assertEquals(aclException.getStatus(),\"netAddress examine scope Exception netAddress\");\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/acl/common/SessionCredentialsTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.acl.common;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport java.util.Properties;\n\npublic class SessionCredentialsTest {\n\n    @Test\n    public void equalsTest() {\n        SessionCredentials sessionCredentials = new SessionCredentials(\"RocketMQ\",\"12345678\");\n        sessionCredentials.setSecurityToken(\"abcd\");\n        SessionCredentials other = new SessionCredentials(\"RocketMQ\",\"12345678\",\"abcd\");\n        Assert.assertTrue(sessionCredentials.equals(other));\n    }\n\n    @Test\n    public void updateContentTest() {\n        SessionCredentials sessionCredentials = new SessionCredentials();\n        Properties properties = new Properties();\n        properties.setProperty(SessionCredentials.ACCESS_KEY,\"RocketMQ\");\n        properties.setProperty(SessionCredentials.SECRET_KEY,\"12345678\");\n        properties.setProperty(SessionCredentials.SECURITY_TOKEN,\"abcd\");\n        sessionCredentials.updateContent(properties);\n    }\n\n    @Test\n    public void SessionCredentialHashCodeTest() {\n        SessionCredentials sessionCredentials = new SessionCredentials();\n        Properties properties = new Properties();\n        properties.setProperty(SessionCredentials.ACCESS_KEY,\"RocketMQ\");\n        properties.setProperty(SessionCredentials.SECRET_KEY,\"12345678\");\n        properties.setProperty(SessionCredentials.SECURITY_TOKEN,\"abcd\");\n        sessionCredentials.updateContent(properties);\n        Assert.assertEquals(sessionCredentials.hashCode(),353652211);\n    }\n\n    @Test\n    public void SessionCredentialEqualsTest() {\n        SessionCredentials sessionCredential1  = new SessionCredentials();\n        Properties properties1 = new Properties();\n        properties1.setProperty(SessionCredentials.ACCESS_KEY,\"RocketMQ\");\n        properties1.setProperty(SessionCredentials.SECRET_KEY,\"12345678\");\n        properties1.setProperty(SessionCredentials.SECURITY_TOKEN,\"abcd\");\n        sessionCredential1.updateContent(properties1);\n\n        SessionCredentials sessionCredential2 = new SessionCredentials();\n        Properties properties2 = new Properties();\n        properties2.setProperty(SessionCredentials.ACCESS_KEY,\"RocketMQ\");\n        properties2.setProperty(SessionCredentials.SECRET_KEY,\"12345678\");\n        properties2.setProperty(SessionCredentials.SECURITY_TOKEN,\"abcd\");\n        sessionCredential2.updateContent(properties2);\n\n        Assert.assertTrue(sessionCredential2.equals(sessionCredential1));\n        sessionCredential2.setSecretKey(\"1234567899\");\n        sessionCredential2.setSignature(\"1234567899\");\n        Assert.assertFalse(sessionCredential2.equals(sessionCredential1));\n    }\n\n    @Test\n    public void SessionCredentialToStringTest() {\n        SessionCredentials sessionCredential1 = new SessionCredentials();\n        Properties properties1 = new Properties();\n        properties1.setProperty(SessionCredentials.ACCESS_KEY,\"RocketMQ\");\n        properties1.setProperty(SessionCredentials.SECRET_KEY,\"12345678\");\n        properties1.setProperty(SessionCredentials.SECURITY_TOKEN,\"abcd\");\n        sessionCredential1.updateContent(properties1);\n\n        Assert.assertEquals(sessionCredential1.toString(),\n            \"SessionCredentials [accessKey=RocketMQ, secretKey=12345678, signature=null, SecurityToken=abcd]\");\n    }\n\n\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/ClientConfigTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client;\n\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.util.Collection;\nimport java.util.Collections;\nimport java.util.Set;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ClientConfigTest {\n\n    private ClientConfig clientConfig;\n\n    private final String resource = \"resource\";\n\n    @Before\n    public void init() {\n        clientConfig = createClientConfig();\n    }\n\n    @Test\n    public void testWithNamespace() {\n        Set<String> resources = clientConfig.withNamespace(Collections.singleton(resource));\n        assertTrue(resources.contains(\"lmq%resource\"));\n    }\n\n    @Test\n    public void testWithoutNamespace() {\n        String actual = clientConfig.withoutNamespace(resource);\n        assertEquals(resource, actual);\n        Set<String> resources = clientConfig.withoutNamespace(Collections.singleton(resource));\n        assertTrue(resources.contains(resource));\n    }\n\n    @Test\n    public void testQueuesWithNamespace() {\n        MessageQueue messageQueue = new MessageQueue();\n        messageQueue.setTopic(\"defaultTopic\");\n        Collection<MessageQueue> messageQueues = clientConfig.queuesWithNamespace(Collections.singleton(messageQueue));\n        assertTrue(messageQueues.contains(messageQueue));\n        assertEquals(\"lmq%defaultTopic\", messageQueues.iterator().next().getTopic());\n    }\n\n    private ClientConfig createClientConfig() {\n        ClientConfig result = new ClientConfig();\n        result.setUnitName(\"unitName\");\n        result.setClientIP(\"127.0.0.1\");\n        result.setClientCallbackExecutorThreads(1);\n        result.setPollNameServerInterval(1000 * 30);\n        result.setHeartbeatBrokerInterval(1000 * 30);\n        result.setPersistConsumerOffsetInterval(1000 * 5);\n        result.setPullTimeDelayMillsWhenException(1000);\n        result.setUnitMode(true);\n        result.setSocksProxyConfig(\"{}\");\n        result.setLanguage(LanguageCode.JAVA);\n        result.setDecodeReadBody(true);\n        result.setDecodeDecompressBody(true);\n        result.setAccessChannel(AccessChannel.LOCAL);\n        result.setMqClientApiTimeout(1000 * 3);\n        result.setEnableStreamRequestType(true);\n        result.setSendLatencyEnable(true);\n        result.setEnableHeartbeatChannelEventListener(true);\n        result.setDetectTimeout(200);\n        result.setDetectInterval(1000 * 2);\n        result.setUseHeartbeatV2(false);\n        result.buildMQClientId();\n        result.setNamespace(\"lmq\");\n        return result;\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/ValidatorsTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client;\n\nimport java.util.Properties;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown;\nimport static org.junit.Assert.fail;\n\npublic class ValidatorsTest {\n\n    @Test\n    public void testGroupNameBlank() {\n        try {\n            Validators.checkGroup(null);\n            fail(\"excepted MQClientException for group name is blank\");\n        } catch (MQClientException e) {\n            assertThat(e.getErrorMessage()).isEqualTo(\"the specified group is blank\");\n        }\n    }\n\n    @Test\n    public void testCheckTopic_Success() throws MQClientException {\n        Validators.checkTopic(\"Hello\");\n        Validators.checkTopic(\"%RETRY%Hello\");\n        Validators.checkTopic(\"_%RETRY%Hello\");\n        Validators.checkTopic(\"-%RETRY%Hello\");\n        Validators.checkTopic(\"223-%RETRY%Hello\");\n    }\n\n    @Test\n    public void testCheckTopic_HasIllegalCharacters() {\n        String illegalTopic = \"TOPIC&*^\";\n        try {\n            Validators.checkTopic(illegalTopic);\n            failBecauseExceptionWasNotThrown(MQClientException.class);\n        } catch (MQClientException e) {\n            assertThat(e).hasMessageStartingWith(String.format(\"The specified topic[%s] contains illegal characters, allowing only %s\", illegalTopic, \"^[%|a-zA-Z0-9_-]+$\"));\n        }\n    }\n\n    @Test\n    public void testCheckTopic_BlankTopic() {\n        String blankTopic = \"\";\n        try {\n            Validators.checkTopic(blankTopic);\n            failBecauseExceptionWasNotThrown(MQClientException.class);\n        } catch (MQClientException e) {\n            assertThat(e).hasMessageStartingWith(\"The specified topic is blank\");\n        }\n    }\n\n    @Test\n    public void testCheckTopic_TooLongTopic() {\n        String tooLongTopic = StringUtils.rightPad(\"TooLongTopic\", Validators.TOPIC_MAX_LENGTH + 1, \"_\");\n        assertThat(tooLongTopic.length()).isGreaterThan(Validators.TOPIC_MAX_LENGTH);\n        try {\n            Validators.checkTopic(tooLongTopic);\n            failBecauseExceptionWasNotThrown(MQClientException.class);\n        } catch (MQClientException e) {\n            assertThat(e).hasMessageStartingWith(\"The specified topic is longer than topic max length\");\n        }\n    }\n\n    @Test\n    public void testIsSystemTopic() {\n        for (String topic : TopicValidator.getSystemTopicSet()) {\n            try {\n                Validators.isSystemTopic(topic);\n                fail(\"excepted MQClientException for system topic\");\n            } catch (MQClientException e) {\n                assertThat(e.getResponseCode()).isEqualTo(-1);\n                assertThat(e.getErrorMessage()).isEqualTo(String.format(\"The topic[%s] is conflict with system topic.\", topic));\n            }\n        }\n    }\n\n    @Test\n    public void testIsNotAllowedSendTopic() {\n        for (String topic : TopicValidator.getNotAllowedSendTopicSet()) {\n            try {\n                Validators.isNotAllowedSendTopic(topic);\n                fail(\"excepted MQClientException for blacklist topic\");\n            } catch (MQClientException e) {\n                assertThat(e.getResponseCode()).isEqualTo(-1);\n                assertThat(e.getErrorMessage()).isEqualTo(String.format(\"Sending message to topic[%s] is forbidden.\", topic));\n            }\n        }\n    }\n\n    @Test\n    public void testTopicConfigValid() throws MQClientException {\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setPerm(PermName.PERM_INHERIT | PermName.PERM_WRITE | PermName.PERM_READ);\n        Validators.checkTopicConfig(topicConfig);\n\n        topicConfig.setPerm(PermName.PERM_WRITE | PermName.PERM_READ);\n        Validators.checkTopicConfig(topicConfig);\n\n        topicConfig.setPerm(PermName.PERM_READ);\n        Validators.checkTopicConfig(topicConfig);\n\n        try {\n            topicConfig.setPerm(PermName.PERM_PRIORITY);\n            Validators.checkTopicConfig(topicConfig);\n        } catch (MQClientException e) {\n            assertThat(e.getResponseCode()).isEqualTo(ResponseCode.NO_PERMISSION);\n            assertThat(e.getErrorMessage()).isEqualTo(String.format(\"topicPermission value: %s is invalid.\", topicConfig.getPerm()));\n        }\n\n        try {\n            topicConfig.setPerm(PermName.PERM_PRIORITY | PermName.PERM_WRITE);\n            Validators.checkTopicConfig(topicConfig);\n        } catch (MQClientException e) {\n            assertThat(e.getResponseCode()).isEqualTo(ResponseCode.NO_PERMISSION);\n            assertThat(e.getErrorMessage()).isEqualTo(String.format(\"topicPermission value: %s is invalid.\", topicConfig.getPerm()));\n        }\n    }\n\n    @Test\n    public void testBrokerConfigValid() throws MQClientException {\n        Properties brokerConfig = new Properties();\n        brokerConfig.setProperty(\"brokerPermission\",\n            String.valueOf(PermName.PERM_INHERIT | PermName.PERM_WRITE | PermName.PERM_READ));\n        Validators.checkBrokerConfig(brokerConfig);\n\n        brokerConfig.setProperty(\"brokerPermission\", String.valueOf(PermName.PERM_WRITE | PermName.PERM_READ));\n        Validators.checkBrokerConfig(brokerConfig);\n\n        brokerConfig.setProperty(\"brokerPermission\", String.valueOf(PermName.PERM_READ));\n        Validators.checkBrokerConfig(brokerConfig);\n\n        try {\n            brokerConfig.setProperty(\"brokerPermission\", String.valueOf(PermName.PERM_PRIORITY));\n            Validators.checkBrokerConfig(brokerConfig);\n        } catch (MQClientException e) {\n            assertThat(e.getResponseCode()).isEqualTo(ResponseCode.NO_PERMISSION);\n            assertThat(e.getErrorMessage()).isEqualTo(String.format(\"brokerPermission value: %s is invalid.\", brokerConfig.getProperty(\"brokerPermission\")));\n        }\n\n        try {\n            brokerConfig.setProperty(\"brokerPermission\", String.valueOf(PermName.PERM_PRIORITY | PermName.PERM_INHERIT));\n            Validators.checkBrokerConfig(brokerConfig);\n        } catch (MQClientException e) {\n            assertThat(e.getResponseCode()).isEqualTo(ResponseCode.NO_PERMISSION);\n            assertThat(e.getErrorMessage()).isEqualTo(String.format(\"brokerPermission value: %s is invalid.\", brokerConfig.getProperty(\"brokerPermission\")));\n        }\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/common/ThreadLocalIndexTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.common;\n\nimport java.lang.reflect.Field;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class ThreadLocalIndexTest {\n    @Test\n    public void testIncrementAndGet() throws Exception {\n        ThreadLocalIndex localIndex = new ThreadLocalIndex();\n        int initialVal = localIndex.incrementAndGet();\n\n        assertThat(localIndex.incrementAndGet()).isEqualTo(initialVal + 1);\n    }\n\n    @Test\n    public void testIncrementAndGet2() throws Exception {\n        ThreadLocalIndex localIndex = new ThreadLocalIndex();\n        int initialVal = localIndex.incrementAndGet();\n        assertThat(initialVal >= 0).isTrue();\n    }\n\n    @Test\n    public void testIncrementAndGet3() throws Exception {\n        ThreadLocalIndex localIndex = new ThreadLocalIndex();\n        Field threadLocalIndexField = ThreadLocalIndex.class.getDeclaredField(\"threadLocalIndex\");\n        ThreadLocal<Integer> mockThreadLocal = new ThreadLocal<>();\n        mockThreadLocal.set(Integer.MAX_VALUE);\n\n        threadLocalIndexField.setAccessible(true);\n        threadLocalIndexField.set(localIndex, mockThreadLocal);\n\n        int initialVal = localIndex.incrementAndGet();\n        assertThat(initialVal >= 0).isTrue();\n    }\n\n    @Test\n    public void testResultOfResetIsGreaterThanOrEqualToZero() {\n        ThreadLocalIndex localIndex = new ThreadLocalIndex();\n        localIndex.reset();\n        assertThat(localIndex.incrementAndGet() > 0).isTrue();\n    }\n\n}"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.consumer;\n\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.consumer.store.OffsetStore;\nimport org.apache.rocketmq.client.consumer.store.ReadOffsetType;\nimport org.apache.rocketmq.client.consumer.store.RemoteBrokerOffsetStore;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.FindBrokerResult;\nimport org.apache.rocketmq.client.impl.MQAdminImpl;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.MQClientManager;\nimport org.apache.rocketmq.client.impl.consumer.AssignedMessageQueue;\nimport org.apache.rocketmq.client.impl.consumer.DefaultLitePullConsumerImpl;\nimport org.apache.rocketmq.client.impl.consumer.PullAPIWrapper;\nimport org.apache.rocketmq.client.impl.consumer.PullResultExt;\nimport org.apache.rocketmq.client.impl.consumer.RebalanceImpl;\nimport org.apache.rocketmq.client.impl.consumer.RebalanceService;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.MessageClientExt;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.BeforeClass;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.invocation.InvocationOnMock;\nimport org.mockito.junit.MockitoJUnitRunner;\nimport org.mockito.stubbing.Answer;\n\nimport java.io.ByteArrayOutputStream;\nimport java.lang.reflect.Field;\nimport java.net.InetSocketAddress;\nimport java.time.Duration;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentMap;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown;\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertEquals;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyBoolean;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.nullable;\nimport static org.mockito.Mockito.doAnswer;\nimport static org.mockito.Mockito.doNothing;\nimport static org.mockito.Mockito.doReturn;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.Silent.class)\npublic class DefaultLitePullConsumerTest {\n    @Spy\n    private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig());\n\n    private MQClientInstance mqClientInstance;\n\n    @Mock\n    private MQClientAPIImpl mQClientAPIImpl;\n    @Mock\n    private MQAdminImpl mQAdminImpl;\n    @Mock\n    private AssignedMessageQueue assignedMQ;\n\n    private RebalanceImpl rebalanceImpl;\n    private OffsetStore offsetStore;\n    private DefaultLitePullConsumerImpl litePullConsumerImpl;\n    private String consumerGroup = \"LitePullConsumerGroup\";\n    private String topic = \"LitePullConsumerTest\";\n    private String brokerName = \"BrokerA\";\n    private boolean flag = false;\n\n    @BeforeClass\n    public static void setEnv() {\n        System.setProperty(\"rocketmq.client.logRoot\", System.getProperty(\"java.io.tmpdir\"));\n    }\n\n    @Before\n    public void init() throws Exception {\n        ConcurrentMap<String, MQClientInstance> factoryTable = (ConcurrentMap<String, MQClientInstance>) FieldUtils.readDeclaredField(MQClientManager.getInstance(), \"factoryTable\", true);\n        factoryTable.forEach((s, instance) -> instance.shutdown());\n        factoryTable.clear();\n\n        Field field = MQClientInstance.class.getDeclaredField(\"rebalanceService\");\n        field.setAccessible(true);\n        RebalanceService rebalanceService = (RebalanceService) field.get(mQClientFactory);\n        field = RebalanceService.class.getDeclaredField(\"waitInterval\");\n        field.setAccessible(true);\n        field.set(rebalanceService, 100);\n\n        field = DefaultLitePullConsumerImpl.class.getDeclaredField(\"doNotUpdateTopicSubscribeInfoWhenSubscriptionChanged\");\n        field.setAccessible(true);\n        field.set(null, true);\n    }\n\n    @After\n    public void destroy() {\n        if (mqClientInstance != null) {\n            mqClientInstance.unregisterConsumer(litePullConsumerImpl.groupName());\n            mqClientInstance.shutdown();\n        }\n    }\n\n    @Test\n    public void testAssign_PollMessageSuccess() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer();\n        try {\n            MessageQueue messageQueue = createMessageQueue();\n            litePullConsumer.assign(Collections.singletonList(messageQueue));\n            List<MessageExt> result = litePullConsumer.poll();\n            assertThat(result.get(0).getTopic()).isEqualTo(topic);\n            assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'});\n        } finally {\n            litePullConsumer.shutdown();\n        }\n    }\n\n    @Test\n    public void testSubscribeWithListener_PollMessageSuccess() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = createSubscribeLitePullConsumerWithListener();\n        try {\n            Set<MessageQueue> messageQueueSet = new HashSet<>();\n            messageQueueSet.add(createMessageQueue());\n            litePullConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet);\n            litePullConsumer.setPollTimeoutMillis(20 * 1000);\n            List<MessageExt> result = litePullConsumer.poll();\n            assertThat(result.get(0).getTopic()).isEqualTo(topic);\n            assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'});\n\n            Set<MessageQueue> assignment = litePullConsumer.assignment();\n            assertThat(assignment.stream().findFirst().get()).isEqualTo(messageQueueSet.stream().findFirst().get());\n        } finally {\n            litePullConsumer.shutdown();\n        }\n    }\n\n    @Test\n    public void testAssign_PollMessageWithTagSuccess() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumerWithTag();\n        try {\n            MessageQueue messageQueue = createMessageQueue();\n            litePullConsumer.assign(Collections.singletonList(messageQueue));\n            List<MessageExt> result = litePullConsumer.poll();\n            assertThat(result.get(0).getTopic()).isEqualTo(topic);\n            assertThat(result.get(0).getTags()).isEqualTo(\"tagA\");\n            assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'});\n        } finally {\n            litePullConsumer.shutdown();\n        }\n    }\n\n    @Test\n    public void testConsumerCommitSyncWithMQOffset() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = createNotStartLitePullConsumer();\n        try {\n            RemoteBrokerOffsetStore store = new RemoteBrokerOffsetStore(mQClientFactory, consumerGroup);\n            litePullConsumer.setOffsetStore(store);\n            litePullConsumer.start();\n            initDefaultLitePullConsumer(litePullConsumer);\n\n            //replace with real offsetStore.\n            Field offsetStore = litePullConsumerImpl.getClass().getDeclaredField(\"offsetStore\");\n            offsetStore.setAccessible(true);\n            offsetStore.set(litePullConsumerImpl, store);\n\n            MessageQueue messageQueue = createMessageQueue();\n            HashSet<MessageQueue> set = new HashSet<>();\n            set.add(messageQueue);\n\n            //mock assign and reset offset\n            litePullConsumer.assign(set);\n            litePullConsumer.seek(messageQueue, 0);\n            await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> assertThat(litePullConsumer.committed(messageQueue)).isEqualTo(0));\n            //commit offset 1\n            Map<MessageQueue, Long> commitOffset = new HashMap<>();\n            commitOffset.put(messageQueue, 1L);\n            litePullConsumer.commit(commitOffset, true);\n\n            assertThat(litePullConsumer.committed(messageQueue)).isEqualTo(1);\n        } finally {\n            litePullConsumer.shutdown();\n        }\n    }\n\n    @Test\n    public void testSubscribe_PollMessageSuccess() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = createSubscribeLitePullConsumer();\n        try {\n            Set<MessageQueue> messageQueueSet = new HashSet<>();\n            messageQueueSet.add(createMessageQueue());\n            litePullConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet);\n            litePullConsumer.setPollTimeoutMillis(20 * 1000);\n            List<MessageExt> result = litePullConsumer.poll();\n            assertThat(result.get(0).getTopic()).isEqualTo(topic);\n            assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'});\n        } finally {\n            litePullConsumer.shutdown();\n        }\n    }\n\n    @Test\n    public void testSubscribe_BroadcastPollMessageSuccess() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = createBroadcastLitePullConsumer();\n        try {\n            Set<MessageQueue> messageQueueSet = new HashSet<>();\n            messageQueueSet.add(createMessageQueue());\n            litePullConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet);\n            litePullConsumer.setPollTimeoutMillis(20 * 1000);\n            List<MessageExt> result = litePullConsumer.poll();\n            assertThat(result.get(0).getTopic()).isEqualTo(topic);\n            assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'});\n        } finally {\n            litePullConsumer.shutdown();\n        }\n    }\n\n    @Test\n    public void testSubscriptionType_AssignAndSubscribeExclusive() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer();\n        try {\n            litePullConsumer.subscribe(topic, \"*\");\n            litePullConsumer.assign(Collections.singletonList(createMessageQueue()));\n            failBecauseExceptionWasNotThrown(IllegalStateException.class);\n        } catch (IllegalStateException e) {\n            assertThat(e).hasMessageContaining(\"Subscribe and assign are mutually exclusive.\");\n        } finally {\n            litePullConsumer.shutdown();\n        }\n    }\n\n    @Test\n    public void testFetchMessageQueues_FetchMessageQueuesBeforeStart() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = createNotStartLitePullConsumer();\n        try {\n            litePullConsumer.fetchMessageQueues(topic);\n            failBecauseExceptionWasNotThrown(IllegalStateException.class);\n        } catch (IllegalStateException e) {\n            assertThat(e).hasMessageContaining(\"The consumer not running, please start it first.\");\n        } finally {\n            litePullConsumer.shutdown();\n        }\n    }\n\n    @Test\n    public void testSeek_SeekOffsetSuccess() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer();\n        when(mQAdminImpl.minOffset(any(MessageQueue.class))).thenReturn(0L);\n        when(mQAdminImpl.maxOffset(any(MessageQueue.class))).thenReturn(500L);\n        MessageQueue messageQueue = createMessageQueue();\n        List<MessageQueue> messageQueues = Collections.singletonList(messageQueue);\n        litePullConsumer.assign(messageQueues);\n        litePullConsumer.pause(messageQueues);\n        litePullConsumer.pause(Collections.singletonList(messageQueue));\n        long offset = litePullConsumer.committed(messageQueue);\n        litePullConsumer.seek(messageQueue, offset);\n        Field field = DefaultLitePullConsumerImpl.class.getDeclaredField(\"assignedMessageQueue\");\n        field.setAccessible(true);\n        AssignedMessageQueue assignedMessageQueue = (AssignedMessageQueue) field.get(litePullConsumerImpl);\n        assertEquals(assignedMessageQueue.getSeekOffset(messageQueue), offset);\n        litePullConsumer.shutdown();\n    }\n\n    @Test\n    public void testSeek_SeekToBegin() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer();\n        when(mQAdminImpl.minOffset(any(MessageQueue.class))).thenReturn(0L);\n        when(mQAdminImpl.maxOffset(any(MessageQueue.class))).thenReturn(500L);\n        MessageQueue messageQueue = createMessageQueue();\n        List<MessageQueue> messageQueues = Collections.singletonList(messageQueue);\n        litePullConsumer.assign(messageQueues);\n        litePullConsumer.pause(messageQueues);\n        litePullConsumer.pause(Collections.singletonList(messageQueue));\n        litePullConsumer.seekToBegin(messageQueue);\n        Field field = DefaultLitePullConsumerImpl.class.getDeclaredField(\"assignedMessageQueue\");\n        field.setAccessible(true);\n        AssignedMessageQueue assignedMessageQueue = (AssignedMessageQueue) field.get(litePullConsumerImpl);\n        assertEquals(assignedMessageQueue.getSeekOffset(messageQueue), 0L);\n        litePullConsumer.shutdown();\n    }\n\n    @Test\n    public void testSeek_SeekToEnd() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer();\n        when(mQAdminImpl.minOffset(any(MessageQueue.class))).thenReturn(0L);\n        when(mQAdminImpl.maxOffset(any(MessageQueue.class))).thenReturn(500L);\n        MessageQueue messageQueue = createMessageQueue();\n        List<MessageQueue> messageQueues = Collections.singletonList(messageQueue);\n        litePullConsumer.assign(messageQueues);\n        litePullConsumer.pause(messageQueues);\n        litePullConsumer.pause(Collections.singletonList(messageQueue));\n        litePullConsumer.seekToEnd(messageQueue);\n        Field field = DefaultLitePullConsumerImpl.class.getDeclaredField(\"assignedMessageQueue\");\n        field.setAccessible(true);\n        AssignedMessageQueue assignedMessageQueue = (AssignedMessageQueue) field.get(litePullConsumerImpl);\n        assertEquals(assignedMessageQueue.getSeekOffset(messageQueue), 500L);\n        litePullConsumer.shutdown();\n    }\n\n    @Test\n    public void testSeek_SeekOffsetIllegal() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer();\n        when(mQAdminImpl.minOffset(any(MessageQueue.class))).thenReturn(0L);\n        when(mQAdminImpl.maxOffset(any(MessageQueue.class))).thenReturn(100L);\n        MessageQueue messageQueue = createMessageQueue();\n        List<MessageQueue> messageQueues = Collections.singletonList(messageQueue);\n        litePullConsumer.assign(messageQueues);\n        litePullConsumer.pause(messageQueues);\n        litePullConsumer.pause(Collections.singletonList(messageQueue));\n        try {\n            litePullConsumer.seek(messageQueue, -1);\n            failBecauseExceptionWasNotThrown(MQClientException.class);\n        } catch (MQClientException e) {\n            assertThat(e).hasMessageContaining(\"min offset = 0\");\n        }\n\n        try {\n            litePullConsumer.seek(messageQueue, 1000);\n            failBecauseExceptionWasNotThrown(MQClientException.class);\n        } catch (MQClientException e) {\n            assertThat(e).hasMessageContaining(\"max offset = 100\");\n        }\n        litePullConsumer.shutdown();\n    }\n\n    @Test\n    public void testSeek_MessageQueueNotInAssignList() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer();\n        try {\n            litePullConsumer.seek(createMessageQueue(), 0);\n            failBecauseExceptionWasNotThrown(MQClientException.class);\n        } catch (MQClientException e) {\n            assertThat(e).hasMessageContaining(\"The message queue is not in assigned list\");\n        } finally {\n            litePullConsumer.shutdown();\n        }\n\n        litePullConsumer = createSubscribeLitePullConsumer();\n        try {\n            litePullConsumer.seek(createMessageQueue(), 0);\n            failBecauseExceptionWasNotThrown(MQClientException.class);\n        } catch (MQClientException e) {\n            assertThat(e).hasMessageContaining(\"The message queue is not in assigned list, may be rebalancing\");\n        } finally {\n            litePullConsumer.shutdown();\n        }\n    }\n\n    @Test\n    public void testOffsetForTimestamp_FailedAndSuccess() throws Exception {\n        MessageQueue messageQueue = createMessageQueue();\n        DefaultLitePullConsumer litePullConsumer = createNotStartLitePullConsumer();\n        try {\n            litePullConsumer.offsetForTimestamp(messageQueue, 123456L);\n            failBecauseExceptionWasNotThrown(IllegalStateException.class);\n        } catch (IllegalStateException e) {\n            assertThat(e).hasMessageContaining(\"The consumer not running, please start it first.\");\n        } finally {\n            litePullConsumer.shutdown();\n        }\n        doReturn(123L).when(mQAdminImpl).searchOffset(any(MessageQueue.class), anyLong());\n        litePullConsumer = createStartLitePullConsumer();\n        try {\n            long offset = litePullConsumer.offsetForTimestamp(messageQueue, 123456L);\n            assertThat(offset).isEqualTo(123L);\n        } finally {\n            litePullConsumer.shutdown();\n        }\n    }\n\n    @Test\n    public void testPauseAndResume_Success() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = createNotStartLitePullConsumer();\n        try {\n            MessageQueue messageQueue = createMessageQueue();\n            litePullConsumer.assign(Collections.singletonList(messageQueue));\n            litePullConsumer.pause(Collections.singletonList(messageQueue));\n            litePullConsumer.start();\n            initDefaultLitePullConsumer(litePullConsumer);\n            List<MessageExt> result = litePullConsumer.poll();\n            assertThat(result.isEmpty()).isTrue();\n            litePullConsumer.resume(Collections.singletonList(messageQueue));\n            result = litePullConsumer.poll();\n            assertThat(result.get(0).getTopic()).isEqualTo(topic);\n            assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'});\n        } finally {\n            litePullConsumer.shutdown();\n        }\n    }\n\n    @Test\n    public void testPullTaskImpl_ProcessQueueNull() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = createNotStartLitePullConsumer();\n        try {\n            MessageQueue messageQueue = createMessageQueue();\n            litePullConsumer.assign(Collections.singletonList(messageQueue));\n            Field field = DefaultLitePullConsumer.class.getDeclaredField(\"defaultLitePullConsumerImpl\");\n            field.setAccessible(true);\n            // set ProcessQueue dropped = true\n            DefaultLitePullConsumerImpl localLitePullConsumerImpl = (DefaultLitePullConsumerImpl) field.get(litePullConsumer);\n            field = DefaultLitePullConsumerImpl.class.getDeclaredField(\"assignedMessageQueue\");\n            field.setAccessible(true);\n            when(assignedMQ.isPaused(any(MessageQueue.class))).thenReturn(false);\n            when(assignedMQ.getProcessQueue(any(MessageQueue.class))).thenReturn(null);\n            litePullConsumer.start();\n            field.set(localLitePullConsumerImpl, assignedMQ);\n\n            List<MessageExt> result = litePullConsumer.poll(100);\n            assertThat(result.isEmpty()).isTrue();\n        } finally {\n            litePullConsumer.shutdown();\n        }\n    }\n\n    @Test\n    public void testPullTaskImpl_ProcessQueueDropped() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = createNotStartLitePullConsumer();\n        try {\n            MessageQueue messageQueue = createMessageQueue();\n            litePullConsumer.assign(Collections.singletonList(messageQueue));\n            Field field = DefaultLitePullConsumer.class.getDeclaredField(\"defaultLitePullConsumerImpl\");\n            field.setAccessible(true);\n            // set ProcessQueue dropped = true\n            DefaultLitePullConsumerImpl localLitePullConsumerImpl = (DefaultLitePullConsumerImpl) field.get(litePullConsumer);\n            field = DefaultLitePullConsumerImpl.class.getDeclaredField(\"assignedMessageQueue\");\n            field.setAccessible(true);\n            AssignedMessageQueue assignedMessageQueue = (AssignedMessageQueue) field.get(localLitePullConsumerImpl);\n            assignedMessageQueue.getProcessQueue(messageQueue).setDropped(true);\n            litePullConsumer.start();\n\n            List<MessageExt> result = litePullConsumer.poll(100);\n            assertThat(result.isEmpty()).isTrue();\n        } finally {\n            litePullConsumer.shutdown();\n        }\n    }\n\n    @Test\n    public void testRegisterTopicMessageQueueChangeListener_Success() throws Exception {\n        flag = false;\n        DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer();\n        try {\n            doReturn(Collections.emptySet()).when(mQAdminImpl).fetchSubscribeMessageQueues(anyString());\n            litePullConsumer.setTopicMetadataCheckIntervalMillis(10);\n            litePullConsumer.registerTopicMessageQueueChangeListener(topic, new TopicMessageQueueChangeListener() {\n                @Override\n                public void onChanged(String topic, Set<MessageQueue> messageQueues) {\n                    flag = true;\n                }\n            });\n            Set<MessageQueue> set = new HashSet<>();\n            set.add(createMessageQueue());\n            doReturn(set).when(mQAdminImpl).fetchSubscribeMessageQueues(anyString());\n            Thread.sleep(11 * 1000);\n            assertThat(flag).isTrue();\n        } finally {\n            litePullConsumer.shutdown();\n        }\n    }\n\n    @Test\n    public void testFlowControl_Success() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer();\n        try {\n            MessageQueue messageQueue = createMessageQueue();\n            litePullConsumer.setPullThresholdForAll(-1);\n            litePullConsumer.assign(Collections.singletonList(messageQueue));\n            litePullConsumer.setPollTimeoutMillis(500);\n            List<MessageExt> result = litePullConsumer.poll();\n            assertThat(result).isEmpty();\n        } finally {\n            litePullConsumer.shutdown();\n        }\n\n        litePullConsumer = createStartLitePullConsumer();\n        try {\n            MessageQueue messageQueue = createMessageQueue();\n            litePullConsumer.setPullThresholdForQueue(-1);\n            litePullConsumer.assign(Collections.singletonList(messageQueue));\n            litePullConsumer.setPollTimeoutMillis(500);\n            List<MessageExt> result = litePullConsumer.poll();\n            assertThat(result).isEmpty();\n        } finally {\n            litePullConsumer.shutdown();\n        }\n\n        litePullConsumer = createStartLitePullConsumer();\n        try {\n            MessageQueue messageQueue = createMessageQueue();\n            litePullConsumer.setPullThresholdSizeForQueue(-1);\n            litePullConsumer.assign(Collections.singletonList(messageQueue));\n            litePullConsumer.setPollTimeoutMillis(500);\n            List<MessageExt> result = litePullConsumer.poll();\n            assertThat(result).isEmpty();\n        } finally {\n            litePullConsumer.shutdown();\n        }\n\n        litePullConsumer = createStartLitePullConsumer();\n        try {\n            MessageQueue messageQueue = createMessageQueue();\n            litePullConsumer.setConsumeMaxSpan(-1);\n            litePullConsumer.assign(Collections.singletonList(messageQueue));\n            litePullConsumer.setPollTimeoutMillis(500);\n            List<MessageExt> result = litePullConsumer.poll();\n            assertThat(result).isEmpty();\n        } finally {\n            litePullConsumer.shutdown();\n        }\n    }\n\n    @Test\n    public void testCheckConfig_Exception() {\n        DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(MixAll.DEFAULT_CONSUMER_GROUP);\n        try {\n            litePullConsumer.start();\n            failBecauseExceptionWasNotThrown(MQClientException.class);\n        } catch (MQClientException e) {\n            assertThat(e).hasMessageContaining(\"consumerGroup can not equal\");\n        } finally {\n            litePullConsumer.shutdown();\n        }\n\n        litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis());\n        litePullConsumer.setMessageModel(null);\n        try {\n            litePullConsumer.start();\n            failBecauseExceptionWasNotThrown(MQClientException.class);\n        } catch (MQClientException e) {\n            assertThat(e).hasMessageContaining(\"messageModel is null\");\n        } finally {\n            litePullConsumer.shutdown();\n        }\n\n        litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis());\n        litePullConsumer.setAllocateMessageQueueStrategy(null);\n        try {\n            litePullConsumer.start();\n            failBecauseExceptionWasNotThrown(MQClientException.class);\n        } catch (MQClientException e) {\n            assertThat(e).hasMessageContaining(\"allocateMessageQueueStrategy is null\");\n        } finally {\n            litePullConsumer.shutdown();\n        }\n\n        litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis());\n        litePullConsumer.setConsumerTimeoutMillisWhenSuspend(1);\n        try {\n            litePullConsumer.start();\n            failBecauseExceptionWasNotThrown(MQClientException.class);\n        } catch (MQClientException e) {\n            assertThat(e).hasMessageContaining(\"Long polling mode, the consumer consumerTimeoutMillisWhenSuspend must greater than brokerSuspendMaxTimeMillis\");\n        } finally {\n            litePullConsumer.shutdown();\n        }\n\n    }\n\n    @Test\n    public void testComputePullFromWhereReturnedNotFound() throws Exception {\n        DefaultLitePullConsumer defaultLitePullConsumer = createStartLitePullConsumer();\n        try {\n            defaultLitePullConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n            MessageQueue messageQueue = createMessageQueue();\n            when(offsetStore.readOffset(any(MessageQueue.class), any(ReadOffsetType.class))).thenReturn(-1L);\n            long offset = rebalanceImpl.computePullFromWhere(messageQueue);\n            assertThat(offset).isEqualTo(0);\n        } finally {\n            defaultLitePullConsumer.shutdown();\n        }\n    }\n\n    @Test\n    public void testComputePullFromWhereReturned() throws Exception {\n        DefaultLitePullConsumer defaultLitePullConsumer = createStartLitePullConsumer();\n        defaultLitePullConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        MessageQueue messageQueue = createMessageQueue();\n        when(offsetStore.readOffset(any(MessageQueue.class), any(ReadOffsetType.class))).thenReturn(100L);\n        long offset = rebalanceImpl.computePullFromWhere(messageQueue);\n        assertThat(offset).isEqualTo(100);\n        defaultLitePullConsumer.shutdown();\n    }\n\n    @Test\n    public void testComputePullFromLast() throws Exception {\n        DefaultLitePullConsumer defaultLitePullConsumer = createStartLitePullConsumer();\n        defaultLitePullConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);\n        MessageQueue messageQueue = createMessageQueue();\n        when(offsetStore.readOffset(any(MessageQueue.class), any(ReadOffsetType.class))).thenReturn(-1L);\n        when(mQClientFactory.getMQAdminImpl().maxOffset(any(MessageQueue.class))).thenReturn(100L);\n        long offset = rebalanceImpl.computePullFromWhere(messageQueue);\n        assertThat(offset).isEqualTo(100);\n        defaultLitePullConsumer.shutdown();\n    }\n\n    @Test\n    public void testComputePullByTimeStamp() throws Exception {\n        DefaultLitePullConsumer defaultLitePullConsumer = createStartLitePullConsumer();\n        try {\n            defaultLitePullConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_TIMESTAMP);\n            defaultLitePullConsumer.setConsumeTimestamp(\"20191024171201\");\n            MessageQueue messageQueue = createMessageQueue();\n            when(offsetStore.readOffset(any(MessageQueue.class), any(ReadOffsetType.class))).thenReturn(-1L);\n            when(mQClientFactory.getMQAdminImpl().searchOffset(any(MessageQueue.class), anyLong())).thenReturn(100L);\n            long offset = rebalanceImpl.computePullFromWhere(messageQueue);\n            assertThat(offset).isEqualTo(100);\n        } finally {\n            defaultLitePullConsumer.shutdown();\n        }\n    }\n\n    @Test\n    public void testConsumerAfterShutdown() throws Exception {\n        DefaultLitePullConsumer defaultLitePullConsumer = createSubscribeLitePullConsumer();\n\n        new AsyncConsumer().executeAsync(defaultLitePullConsumer);\n\n        Thread.sleep(100);\n        defaultLitePullConsumer.shutdown();\n        assertThat(defaultLitePullConsumer.isRunning()).isFalse();\n    }\n\n    @Test\n    public void testConsumerCommitWithMQ() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = createNotStartLitePullConsumer();\n        try {\n            RemoteBrokerOffsetStore store = new RemoteBrokerOffsetStore(mQClientFactory, consumerGroup);\n            litePullConsumer.setOffsetStore(store);\n            litePullConsumer.start();\n            initDefaultLitePullConsumer(litePullConsumer);\n\n            //replace with real offsetStore.\n            Field offsetStore = litePullConsumerImpl.getClass().getDeclaredField(\"offsetStore\");\n            offsetStore.setAccessible(true);\n            offsetStore.set(litePullConsumerImpl, store);\n\n            MessageQueue messageQueue = createMessageQueue();\n            HashSet<MessageQueue> set = new HashSet<>();\n            set.add(messageQueue);\n\n            //mock assign and reset offset\n            litePullConsumer.assign(set);\n            litePullConsumer.seek(messageQueue, 0);\n\n            //commit\n            litePullConsumer.commit(set, true);\n\n            assertThat(litePullConsumer.committed(messageQueue)).isEqualTo(0);\n        } finally {\n            litePullConsumer.shutdown();\n        }\n    }\n\n    static class AsyncConsumer {\n        public void executeAsync(final DefaultLitePullConsumer consumer) {\n            new Thread(() -> {\n                while (consumer.isRunning()) {\n                    consumer.poll(2 * 1000);\n                }\n            }).start();\n        }\n    }\n\n    private void initDefaultLitePullConsumer(DefaultLitePullConsumer litePullConsumer) throws Exception {\n        Field field = DefaultLitePullConsumer.class.getDeclaredField(\"defaultLitePullConsumerImpl\");\n        field.setAccessible(true);\n        litePullConsumerImpl = (DefaultLitePullConsumerImpl) field.get(litePullConsumer);\n        field = DefaultLitePullConsumerImpl.class.getDeclaredField(\"mQClientFactory\");\n        field.setAccessible(true);\n        mqClientInstance = (MQClientInstance) field.get(litePullConsumerImpl);\n        field.set(litePullConsumerImpl, mQClientFactory);\n\n        PullAPIWrapper pullAPIWrapper = litePullConsumerImpl.getPullAPIWrapper();\n        field = PullAPIWrapper.class.getDeclaredField(\"mQClientFactory\");\n        field.setAccessible(true);\n        field.set(pullAPIWrapper, mQClientFactory);\n\n        field = MQClientInstance.class.getDeclaredField(\"mQClientAPIImpl\");\n        field.setAccessible(true);\n        field.set(mQClientFactory, mQClientAPIImpl);\n\n        field = MQClientInstance.class.getDeclaredField(\"mQAdminImpl\");\n        field.setAccessible(true);\n        field.set(mQClientFactory, mQAdminImpl);\n\n        field = DefaultLitePullConsumerImpl.class.getDeclaredField(\"rebalanceImpl\");\n        field.setAccessible(true);\n        rebalanceImpl = (RebalanceImpl) field.get(litePullConsumerImpl);\n        field = RebalanceImpl.class.getDeclaredField(\"mQClientFactory\");\n        field.setAccessible(true);\n        field.set(rebalanceImpl, mQClientFactory);\n\n        offsetStore = spy(litePullConsumerImpl.getOffsetStore());\n        field = DefaultLitePullConsumerImpl.class.getDeclaredField(\"offsetStore\");\n        field.setAccessible(true);\n        field.set(litePullConsumerImpl, offsetStore);\n\n        when(mQClientFactory.getMQClientAPIImpl().pullMessage(anyString(), any(PullMessageRequestHeader.class),\n            anyLong(), any(CommunicationMode.class), nullable(PullCallback.class)))\n            .thenAnswer(new Answer<PullResult>() {\n                @Override\n                public PullResult answer(InvocationOnMock mock) throws Throwable {\n                    PullMessageRequestHeader requestHeader = mock.getArgument(1);\n                    MessageClientExt messageClientExt = new MessageClientExt();\n                    messageClientExt.setTopic(topic);\n                    messageClientExt.setQueueId(0);\n                    messageClientExt.setMsgId(\"123\");\n                    messageClientExt.setBody(new byte[] {'a'});\n                    messageClientExt.setOffsetMsgId(\"234\");\n                    messageClientExt.setBornHost(new InetSocketAddress(8080));\n                    messageClientExt.setStoreHost(new InetSocketAddress(8080));\n                    PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(messageClientExt));\n                    return pullResult;\n                }\n            });\n\n        doAnswer(x -> new FindBrokerResult(\"127.0.0.1:10911\", false)).when(mQClientFactory).findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean());\n\n        doReturn(Collections.singletonList(mQClientFactory.getClientId())).when(mQClientFactory).findConsumerIdList(anyString(), anyString());\n\n        doReturn(123L).when(offsetStore).readOffset(any(MessageQueue.class), any(ReadOffsetType.class));\n    }\n\n    private void initDefaultLitePullConsumerWithTag(DefaultLitePullConsumer litePullConsumer) throws Exception {\n\n        Field field = DefaultLitePullConsumer.class.getDeclaredField(\"defaultLitePullConsumerImpl\");\n        field.setAccessible(true);\n        litePullConsumerImpl = (DefaultLitePullConsumerImpl) field.get(litePullConsumer);\n        field = DefaultLitePullConsumerImpl.class.getDeclaredField(\"mQClientFactory\");\n        field.setAccessible(true);\n        field.set(litePullConsumerImpl, mQClientFactory);\n\n        PullAPIWrapper pullAPIWrapper = litePullConsumerImpl.getPullAPIWrapper();\n        field = PullAPIWrapper.class.getDeclaredField(\"mQClientFactory\");\n        field.setAccessible(true);\n        field.set(pullAPIWrapper, mQClientFactory);\n\n        field = MQClientInstance.class.getDeclaredField(\"mQClientAPIImpl\");\n        field.setAccessible(true);\n        field.set(mQClientFactory, mQClientAPIImpl);\n\n        field = MQClientInstance.class.getDeclaredField(\"mQAdminImpl\");\n        field.setAccessible(true);\n        field.set(mQClientFactory, mQAdminImpl);\n\n        field = DefaultLitePullConsumerImpl.class.getDeclaredField(\"rebalanceImpl\");\n        field.setAccessible(true);\n        rebalanceImpl = (RebalanceImpl) field.get(litePullConsumerImpl);\n        field = RebalanceImpl.class.getDeclaredField(\"mQClientFactory\");\n        field.setAccessible(true);\n        field.set(rebalanceImpl, mQClientFactory);\n\n        offsetStore = spy(litePullConsumerImpl.getOffsetStore());\n        field = DefaultLitePullConsumerImpl.class.getDeclaredField(\"offsetStore\");\n        field.setAccessible(true);\n        field.set(litePullConsumerImpl, offsetStore);\n\n        when(mQClientFactory.getMQClientAPIImpl().pullMessage(anyString(), any(PullMessageRequestHeader.class),\n            anyLong(), any(CommunicationMode.class), nullable(PullCallback.class)))\n            .thenAnswer(new Answer<PullResult>() {\n                @Override\n                public PullResult answer(InvocationOnMock mock) throws Throwable {\n                    PullMessageRequestHeader requestHeader = mock.getArgument(1);\n                    MessageClientExt messageClientExt = new MessageClientExt();\n                    messageClientExt.setTopic(topic);\n                    messageClientExt.setTags(\"tagA\");\n                    messageClientExt.setQueueId(0);\n                    messageClientExt.setMsgId(\"123\");\n                    messageClientExt.setBody(new byte[] {'a'});\n                    messageClientExt.setOffsetMsgId(\"234\");\n                    messageClientExt.setBornHost(new InetSocketAddress(8080));\n                    messageClientExt.setStoreHost(new InetSocketAddress(8080));\n                    PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(messageClientExt));\n                    return pullResult;\n                }\n            });\n\n        when(mQClientFactory.findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean())).thenReturn(new FindBrokerResult(\"127.0.0.1:10911\", false));\n\n        doReturn(123L).when(offsetStore).readOffset(any(MessageQueue.class), any(ReadOffsetType.class));\n    }\n\n    private DefaultLitePullConsumer createSubscribeLitePullConsumer() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis());\n        litePullConsumer.setNamesrvAddr(\"127.0.0.1:9876\");\n        litePullConsumer.subscribe(topic, \"*\");\n        suppressUpdateTopicRouteInfoFromNameServer(litePullConsumer);\n        litePullConsumer.start();\n        initDefaultLitePullConsumer(litePullConsumer);\n        return litePullConsumer;\n    }\n\n    private DefaultLitePullConsumer createSubscribeLitePullConsumerWithListener() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis());\n        litePullConsumer.setNamesrvAddr(\"127.0.0.1:9876\");\n        litePullConsumer.subscribe(topic, \"*\", new MessageQueueListener() {\n            @Override\n            public void messageQueueChanged(String topic, Set<MessageQueue> mqAll, Set<MessageQueue> mqDivided) {\n                assertThat(mqAll.stream().findFirst().get().getTopic()).isEqualTo(mqDivided.stream().findFirst().get().getTopic());\n                assertThat(mqAll.stream().findFirst().get().getBrokerName()).isEqualTo(mqDivided.stream().findFirst().get().getBrokerName());\n                assertThat(mqAll.stream().findFirst().get().getQueueId()).isEqualTo(mqDivided.stream().findFirst().get().getQueueId());\n            }\n        });\n        suppressUpdateTopicRouteInfoFromNameServer(litePullConsumer);\n        litePullConsumer.start();\n        initDefaultLitePullConsumer(litePullConsumer);\n        return litePullConsumer;\n    }\n\n    private DefaultLitePullConsumer createStartLitePullConsumer() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis());\n        litePullConsumer.setNamesrvAddr(\"127.0.0.1:9876\");\n        suppressUpdateTopicRouteInfoFromNameServer(litePullConsumer);\n        litePullConsumer.start();\n        initDefaultLitePullConsumer(litePullConsumer);\n        return litePullConsumer;\n    }\n\n    private DefaultLitePullConsumer createStartLitePullConsumerWithTag() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis());\n        litePullConsumer.setNamesrvAddr(\"127.0.0.1:9876\");\n        suppressUpdateTopicRouteInfoFromNameServer(litePullConsumer);\n        litePullConsumer.setSubExpressionForAssign(topic, \"tagA\");\n        litePullConsumer.start();\n        initDefaultLitePullConsumerWithTag(litePullConsumer);\n        return litePullConsumer;\n    }\n\n    private DefaultLitePullConsumer createNotStartLitePullConsumer() {\n        DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis());\n        return litePullConsumer;\n    }\n\n    private DefaultLitePullConsumer createBroadcastLitePullConsumer() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis());\n        litePullConsumer.setNamesrvAddr(\"127.0.0.1:9876\");\n        litePullConsumer.setMessageModel(MessageModel.BROADCASTING);\n        litePullConsumer.subscribe(topic, \"*\");\n        suppressUpdateTopicRouteInfoFromNameServer(litePullConsumer);\n        litePullConsumer.start();\n        initDefaultLitePullConsumer(litePullConsumer);\n        return litePullConsumer;\n    }\n\n    private MessageQueue createMessageQueue() {\n        MessageQueue messageQueue = new MessageQueue();\n        messageQueue.setBrokerName(brokerName);\n        messageQueue.setQueueId(0);\n        messageQueue.setTopic(topic);\n        return messageQueue;\n    }\n\n    private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus,\n        List<MessageExt> messageExtList) throws Exception {\n        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();\n        for (MessageExt messageExt : messageExtList) {\n            outputStream.write(MessageDecoder.encode(messageExt, false));\n        }\n        return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray());\n    }\n\n    private void suppressUpdateTopicRouteInfoFromNameServer(\n        DefaultLitePullConsumer litePullConsumer) throws IllegalAccessException {\n        if (litePullConsumer.getMessageModel() == MessageModel.CLUSTERING) {\n            litePullConsumer.changeInstanceNameToPID();\n        }\n\n        ConcurrentMap<String, MQClientInstance> factoryTable = (ConcurrentMap<String, MQClientInstance>) FieldUtils.readDeclaredField(MQClientManager.getInstance(), \"factoryTable\", true);\n        factoryTable.put(litePullConsumer.buildMQClientId(), mQClientFactory);\n        doReturn(false).when(mQClientFactory).updateTopicRouteInfoFromNameServer(anyString());\n        doNothing().when(mQClientFactory).updateTopicRouteInfoFromNameServer();\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\nimport java.lang.reflect.Field;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.FindBrokerResult;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.MQClientManager;\nimport org.apache.rocketmq.client.impl.consumer.PullAPIWrapper;\nimport org.apache.rocketmq.client.impl.consumer.PullResultExt;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.invocation.InvocationOnMock;\nimport org.mockito.junit.MockitoJUnitRunner;\nimport org.mockito.stubbing.Answer;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyBoolean;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.nullable;\nimport static org.mockito.Mockito.doAnswer;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class DefaultMQPullConsumerTest {\n    @Spy\n    private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig());\n    @Mock\n    private MQClientAPIImpl mQClientAPIImpl;\n    private DefaultMQPullConsumer pullConsumer;\n    private String consumerGroup = \"FooBarGroup\";\n    private String topic = \"FooBar\";\n    private String brokerName = \"BrokerA\";\n\n    @Before\n    public void init() throws Exception {\n        pullConsumer = new DefaultMQPullConsumer(consumerGroup);\n        pullConsumer.setNamesrvAddr(\"127.0.0.1:9876\");\n        pullConsumer.start();\n        PullAPIWrapper pullAPIWrapper = pullConsumer.getDefaultMQPullConsumerImpl().getPullAPIWrapper();\n        Field field = PullAPIWrapper.class.getDeclaredField(\"mQClientFactory\");\n        field.setAccessible(true);\n        field.set(pullAPIWrapper, mQClientFactory);\n\n        field = MQClientInstance.class.getDeclaredField(\"mQClientAPIImpl\");\n        field.setAccessible(true);\n        field.set(mQClientFactory, mQClientAPIImpl);\n\n        when(mQClientFactory.findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean())).thenReturn(new FindBrokerResult(\"127.0.0.1:10911\", false));\n    }\n\n    @After\n    public void terminate() {\n        pullConsumer.shutdown();\n    }\n\n    @Test\n    public void testStart_OffsetShouldNotNUllAfterStart() {\n        Assert.assertNotNull(pullConsumer.getOffsetStore());\n    }\n\n    @Test\n    public void testPullMessage_Success() throws Exception {\n        doAnswer(new Answer() {\n            @Override\n            public Object answer(InvocationOnMock mock) throws Throwable {\n                PullMessageRequestHeader requestHeader = mock.getArgument(1);\n                return createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(new MessageExt()));\n            }\n        }).when(mQClientAPIImpl).pullMessage(anyString(), any(PullMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), nullable(PullCallback.class));\n\n        MessageQueue messageQueue = new MessageQueue(topic, brokerName, 0);\n        PullResult pullResult = pullConsumer.pull(messageQueue, \"*\", 1024, 3);\n        assertThat(pullResult).isNotNull();\n        assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND);\n        assertThat(pullResult.getNextBeginOffset()).isEqualTo(1024 + 1);\n        assertThat(pullResult.getMinOffset()).isEqualTo(123);\n        assertThat(pullResult.getMaxOffset()).isEqualTo(2048);\n        assertThat(pullResult.getMsgFoundList()).isEqualTo(new ArrayList<>());\n    }\n\n    @Test\n    public void testPullMessage_NotFound() throws Exception {\n        doAnswer(new Answer() {\n            @Override\n            public Object answer(InvocationOnMock mock) throws Throwable {\n                PullMessageRequestHeader requestHeader = mock.getArgument(1);\n                return createPullResult(requestHeader, PullStatus.NO_NEW_MSG, new ArrayList<>());\n            }\n        }).when(mQClientAPIImpl).pullMessage(anyString(), any(PullMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), nullable(PullCallback.class));\n\n        MessageQueue messageQueue = new MessageQueue(topic, brokerName, 0);\n        PullResult pullResult = pullConsumer.pull(messageQueue, \"*\", 1024, 3);\n        assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.NO_NEW_MSG);\n    }\n\n    @Test\n    public void testPullMessageAsync_Success() throws Exception {\n        doAnswer(new Answer() {\n            @Override\n            public Object answer(InvocationOnMock mock) throws Throwable {\n                PullMessageRequestHeader requestHeader = mock.getArgument(1);\n                PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(new MessageExt()));\n\n                PullCallback pullCallback = mock.getArgument(4);\n                pullCallback.onSuccess(pullResult);\n                return null;\n            }\n        }).when(mQClientAPIImpl).pullMessage(anyString(), any(PullMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), nullable(PullCallback.class));\n\n        MessageQueue messageQueue = new MessageQueue(topic, brokerName, 0);\n        pullConsumer.pull(messageQueue, \"*\", 1024, 3, new PullCallback() {\n            @Override\n            public void onSuccess(PullResult pullResult) {\n                assertThat(pullResult).isNotNull();\n                assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND);\n                assertThat(pullResult.getNextBeginOffset()).isEqualTo(1024 + 1);\n                assertThat(pullResult.getMinOffset()).isEqualTo(123);\n                assertThat(pullResult.getMaxOffset()).isEqualTo(2048);\n                assertThat(pullResult.getMsgFoundList()).isEqualTo(new ArrayList<>());\n            }\n\n            @Override\n            public void onException(Throwable e) {\n\n            }\n        });\n    }\n\n    private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus,\n        List<MessageExt> messageExtList) throws Exception {\n        return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, new byte[] {});\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer;\n\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;\nimport org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely;\nimport org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragelyByCircle;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.FindBrokerResult;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.MQClientManager;\nimport org.apache.rocketmq.client.impl.consumer.ConsumeMessageConcurrentlyService;\nimport org.apache.rocketmq.client.impl.consumer.ConsumeMessageOrderlyService;\nimport org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl;\nimport org.apache.rocketmq.client.impl.consumer.ProcessQueue;\nimport org.apache.rocketmq.client.impl.consumer.PullAPIWrapper;\nimport org.apache.rocketmq.client.impl.consumer.PullMessageService;\nimport org.apache.rocketmq.client.impl.consumer.PullRequest;\nimport org.apache.rocketmq.client.impl.consumer.PullResultExt;\nimport org.apache.rocketmq.client.impl.consumer.RebalanceImpl;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.MessageClientExt;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.junit.AfterClass;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.invocation.InvocationOnMock;\nimport org.mockito.junit.MockitoJUnitRunner;\nimport org.mockito.stubbing.Answer;\n\nimport java.io.ByteArrayOutputStream;\nimport java.lang.reflect.Field;\nimport java.net.InetSocketAddress;\nimport java.util.Collections;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.concurrent.atomic.AtomicReference;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyBoolean;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.nullable;\nimport static org.mockito.Mockito.doReturn;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.Silent.class)\npublic class DefaultMQPushConsumerTest {\n    private String consumerGroup;\n    private String topic = \"FooBar\";\n    private String brokerName = \"BrokerA\";\n    private MQClientInstance mQClientFactory;\n    private final byte[] msgBody = Long.toString(System.currentTimeMillis()).getBytes();\n\n    @Mock\n    private MQClientAPIImpl mQClientAPIImpl;\n    private PullAPIWrapper pullAPIWrapper;\n    private RebalanceImpl rebalanceImpl;\n    private static DefaultMQPushConsumer pushConsumer;\n    private AtomicLong queueOffset = new AtomicLong(1024);\n\n    @Before\n    public void init() throws Exception {\n        ConcurrentMap<String, MQClientInstance> factoryTable = (ConcurrentMap<String, MQClientInstance>) FieldUtils.readDeclaredField(MQClientManager.getInstance(), \"factoryTable\", true);\n        for (Map.Entry<String, MQClientInstance> entry : factoryTable.entrySet()) {\n            entry.getValue().shutdown();\n        }\n        factoryTable.clear();\n\n        when(mQClientAPIImpl.pullMessage(anyString(), any(PullMessageRequestHeader.class),\n            anyLong(), any(CommunicationMode.class), nullable(PullCallback.class)))\n            .thenAnswer(new Answer<PullResult>() {\n                @Override\n                public PullResult answer(InvocationOnMock mock) throws Throwable {\n                    PullMessageRequestHeader requestHeader = mock.getArgument(1);\n                    MessageClientExt messageClientExt = new MessageClientExt();\n                    messageClientExt.setTopic(topic);\n                    messageClientExt.setQueueId(0);\n                    messageClientExt.setMsgId(\"123\");\n                    messageClientExt.setBody(new byte[] {'a'});\n                    messageClientExt.setOffsetMsgId(\"234\");\n                    messageClientExt.setBornHost(new InetSocketAddress(8080));\n                    messageClientExt.setStoreHost(new InetSocketAddress(8080));\n                    PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.<MessageExt>singletonList(messageClientExt));\n                    ((PullCallback) mock.getArgument(4)).onSuccess(pullResult);\n                    return pullResult;\n                }\n            });\n\n\n        consumerGroup = \"FooBarGroup\" + System.currentTimeMillis();\n        pushConsumer = new DefaultMQPushConsumer(consumerGroup);\n        pushConsumer.setNamesrvAddr(\"127.0.0.1:9876\");\n        pushConsumer.setPullInterval(60 * 1000);\n        pushConsumer.setClientRebalance(false);\n\n        pushConsumer.registerMessageListener(new MessageListenerConcurrently() {\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,\n                ConsumeConcurrentlyContext context) {\n                return null;\n            }\n        });\n\n        DefaultMQPushConsumerImpl pushConsumerImpl = pushConsumer.getDefaultMQPushConsumerImpl();\n\n        // suppress updateTopicRouteInfoFromNameServer\n        pushConsumer.changeInstanceNameToPID();\n        mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(pushConsumer, (RPCHook) FieldUtils.readDeclaredField(pushConsumerImpl, \"rpcHook\", true));\n        FieldUtils.writeDeclaredField(mQClientFactory, \"mQClientAPIImpl\", mQClientAPIImpl, true);\n        mQClientFactory = spy(mQClientFactory);\n        factoryTable.put(pushConsumer.buildMQClientId(), mQClientFactory);\n        doReturn(false).when(mQClientFactory).updateTopicRouteInfoFromNameServer(anyString());\n        doReturn(null).when(mQClientFactory).queryAssignment(anyString(), anyString(), anyString(), any(MessageModel.class), anyInt());\n        doReturn(new FindBrokerResult(\"127.0.0.1:10911\", false)).when(mQClientFactory).findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean());\n\n        rebalanceImpl = spy(pushConsumerImpl.getRebalanceImpl());\n        doReturn(123L).when(rebalanceImpl).computePullFromWhereWithException(any(MessageQueue.class));\n        Field field = DefaultMQPushConsumerImpl.class.getDeclaredField(\"rebalanceImpl\");\n        field.setAccessible(true);\n        field.set(pushConsumerImpl, rebalanceImpl);\n\n        field = DefaultMQPushConsumerImpl.class.getDeclaredField(\"doNotUpdateTopicSubscribeInfoWhenSubscriptionChanged\");\n        field.setAccessible(true);\n        field.set(null, true);\n\n        Set<MessageQueue> messageQueueSet = new HashSet<>();\n        messageQueueSet.add(createPullRequest().getMessageQueue());\n        pushConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet);\n\n        pushConsumerImpl.setmQClientFactory(mQClientFactory);\n\n        pullAPIWrapper = spy(new PullAPIWrapper(mQClientFactory, consumerGroup, false));\n        FieldUtils.writeDeclaredField(pushConsumerImpl, \"pullAPIWrapper\", pullAPIWrapper, true);\n\n        when(mQClientAPIImpl.pullMessage(anyString(), any(PullMessageRequestHeader.class),\n            anyLong(), any(CommunicationMode.class), nullable(PullCallback.class)))\n            .thenAnswer(new Answer<PullResult>() {\n                @Override\n                public PullResult answer(InvocationOnMock mock) throws Throwable {\n                    PullMessageRequestHeader requestHeader = mock.getArgument(1);\n                    MessageClientExt messageClientExt = new MessageClientExt();\n                    messageClientExt.setTopic(topic);\n                    messageClientExt.setQueueId(0);\n                    messageClientExt.setQueueOffset(queueOffset.getAndIncrement());\n                    messageClientExt.setMsgId(\"1024\");\n                    messageClientExt.setBody(msgBody);\n                    messageClientExt.setOffsetMsgId(\"234\");\n                    messageClientExt.setBornHost(new InetSocketAddress(8080));\n                    messageClientExt.setStoreHost(new InetSocketAddress(8080));\n                    PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.<MessageExt>singletonList(messageClientExt));\n                    ((PullCallback) mock.getArgument(4)).onSuccess(pullResult);\n                    return pullResult;\n                }\n            });\n\n        pushConsumer.subscribe(topic, \"*\");\n        pushConsumer.start();\n\n        mQClientFactory.registerConsumer(consumerGroup, pushConsumerImpl);\n    }\n\n    @AfterClass\n    public static void terminate() {\n        if (pushConsumer != null) {\n            pushConsumer.shutdown();\n        }\n    }\n\n    @Test\n    public void testStart_OffsetShouldNotNUllAfterStart() {\n        assertNotNull(pushConsumer.getOffsetStore());\n    }\n\n    @Test\n    public void testPullMessage_Success() throws InterruptedException, RemotingException, MQBrokerException {\n        final CountDownLatch countDownLatch = new CountDownLatch(1);\n        final AtomicReference<MessageExt> messageAtomic = new AtomicReference<>();\n        pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() {\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,\n                ConsumeConcurrentlyContext context) {\n                messageAtomic.set(msgs.get(0));\n                countDownLatch.countDown();\n                return null;\n            }\n        }));\n\n        PullMessageService pullMessageService = mQClientFactory.getPullMessageService();\n        pullMessageService.executePullRequestImmediately(createPullRequest());\n        countDownLatch.await(10, TimeUnit.SECONDS);\n        MessageExt msg = messageAtomic.get();\n        assertThat(msg).isNotNull();\n        assertThat(msg.getTopic()).isEqualTo(topic);\n        assertThat(msg.getBody()).isEqualTo(msgBody);\n    }\n\n    @Test(timeout = 20000)\n    public void testPullMessage_SuccessWithOrderlyService() throws Exception {\n        final CountDownLatch countDownLatch = new CountDownLatch(1);\n        final AtomicReference<MessageExt> messageAtomic = new AtomicReference<>();\n\n        MessageListenerOrderly listenerOrderly = new MessageListenerOrderly() {\n            @Override\n            public ConsumeOrderlyStatus consumeMessage(List<MessageExt> msgs, ConsumeOrderlyContext context) {\n                messageAtomic.set(msgs.get(0));\n                countDownLatch.countDown();\n                return null;\n            }\n        };\n        pushConsumer.registerMessageListener(listenerOrderly);\n        pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(new ConsumeMessageOrderlyService(pushConsumer.getDefaultMQPushConsumerImpl(), listenerOrderly));\n        pushConsumer.getDefaultMQPushConsumerImpl().setConsumeOrderly(true);\n        pushConsumer.getDefaultMQPushConsumerImpl().doRebalance();\n        PullMessageService pullMessageService = mQClientFactory.getPullMessageService();\n        pullMessageService.executePullRequestLater(createPullRequest(), 100);\n\n        countDownLatch.await();\n        MessageExt msg = messageAtomic.get();\n        assertThat(msg).isNotNull();\n        assertThat(msg.getTopic()).isEqualTo(topic);\n        assertThat(msg.getBody()).isEqualTo(msgBody);\n    }\n\n    @Test\n    public void testCheckConfig() {\n        DefaultMQPushConsumer pushConsumer = createPushConsumer();\n\n        pushConsumer.setPullThresholdForQueue(65535 + 1);\n        try {\n            pushConsumer.start();\n            failBecauseExceptionWasNotThrown(MQClientException.class);\n        } catch (MQClientException e) {\n            assertThat(e).hasMessageContaining(\"pullThresholdForQueue Out of range [1, 65535]\");\n        }\n\n        pushConsumer = createPushConsumer();\n        pushConsumer.setPullThresholdForTopic(65535 * 100 + 1);\n\n        try {\n            pushConsumer.start();\n            failBecauseExceptionWasNotThrown(MQClientException.class);\n        } catch (MQClientException e) {\n            assertThat(e).hasMessageContaining(\"pullThresholdForTopic Out of range [1, 6553500]\");\n        }\n\n        pushConsumer = createPushConsumer();\n        pushConsumer.setPullThresholdSizeForQueue(1024 + 1);\n        try {\n            pushConsumer.start();\n            failBecauseExceptionWasNotThrown(MQClientException.class);\n        } catch (MQClientException e) {\n            assertThat(e).hasMessageContaining(\"pullThresholdSizeForQueue Out of range [1, 1024]\");\n        }\n\n        pushConsumer = createPushConsumer();\n        pushConsumer.setPullThresholdSizeForTopic(1024 * 100 + 1);\n        try {\n            pushConsumer.start();\n            failBecauseExceptionWasNotThrown(MQClientException.class);\n        } catch (MQClientException e) {\n            assertThat(e).hasMessageContaining(\"pullThresholdSizeForTopic Out of range [1, 102400]\");\n        }\n    }\n\n    @Test(timeout = 20000)\n    public void testGracefulShutdown() throws InterruptedException, RemotingException, MQBrokerException, MQClientException {\n        final CountDownLatch countDownLatch = new CountDownLatch(1);\n        pushConsumer.setAwaitTerminationMillisWhenShutdown(2000);\n        final AtomicBoolean messageConsumedFlag = new AtomicBoolean(false);\n        pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() {\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,\n                ConsumeConcurrentlyContext context) {\n                assertThat(msgs.get(0).getBody()).isEqualTo(msgBody);\n                countDownLatch.countDown();\n                try {\n                    Thread.sleep(1000);\n                    messageConsumedFlag.set(true);\n                } catch (InterruptedException e) {\n                }\n\n                return null;\n            }\n        }));\n\n        PullMessageService pullMessageService = mQClientFactory.getPullMessageService();\n        pullMessageService.executePullRequestImmediately(createPullRequest());\n        assertThat(countDownLatch.await(30, TimeUnit.SECONDS)).isTrue();\n\n        pushConsumer.shutdown();\n        assertThat(messageConsumedFlag.get()).isTrue();\n    }\n\n    private DefaultMQPushConsumer createPushConsumer() {\n        DefaultMQPushConsumer pushConsumer = new DefaultMQPushConsumer(consumerGroup);\n        pushConsumer.registerMessageListener(new MessageListenerConcurrently() {\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,\n                ConsumeConcurrentlyContext context) {\n                return null;\n            }\n        });\n        return pushConsumer;\n    }\n\n    private PullRequest createPullRequest() {\n        PullRequest pullRequest = new PullRequest();\n        pullRequest.setConsumerGroup(consumerGroup);\n        pullRequest.setNextOffset(queueOffset.get());\n\n        MessageQueue messageQueue = new MessageQueue();\n        messageQueue.setBrokerName(brokerName);\n        messageQueue.setQueueId(0);\n        messageQueue.setTopic(topic);\n        pullRequest.setMessageQueue(messageQueue);\n        ProcessQueue processQueue = new ProcessQueue();\n        processQueue.setLocked(true);\n        processQueue.setLastLockTimestamp(System.currentTimeMillis());\n        pullRequest.setProcessQueue(processQueue);\n\n        return pullRequest;\n    }\n\n    private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus,\n        List<MessageExt> messageExtList) throws Exception {\n        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();\n        for (MessageExt messageExt : messageExtList) {\n            outputStream.write(MessageDecoder.encode(messageExt, false));\n        }\n        return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray());\n    }\n\n    @Test\n    public void testPullMessage_ExceptionOccursWhenComputePullFromWhere() throws MQClientException {\n        final CountDownLatch countDownLatch = new CountDownLatch(1);\n        final MessageExt[] messageExts = new MessageExt[1];\n        pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(\n                new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(),\n                    new MessageListenerConcurrently() {\n                        @Override\n                        public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,\n                            ConsumeConcurrentlyContext context) {\n                            messageExts[0] = msgs.get(0);\n                            return null;\n                        }\n                    }));\n\n        pushConsumer.getDefaultMQPushConsumerImpl().setConsumeOrderly(true);\n        PullMessageService pullMessageService = mQClientFactory.getPullMessageService();\n        pullMessageService.executePullRequestImmediately(createPullRequest());\n        assertThat(messageExts[0]).isNull();\n    }\n\n    @Test\n    public void assertCreatePushConsumer() {\n        DefaultMQPushConsumer pushConsumer1 = new DefaultMQPushConsumer(consumerGroup, mock(RPCHook.class));\n        assertNotNull(pushConsumer1);\n        assertEquals(consumerGroup, pushConsumer1.getConsumerGroup());\n        assertTrue(pushConsumer1.getAllocateMessageQueueStrategy() instanceof AllocateMessageQueueAveragely);\n        assertNotNull(pushConsumer1.defaultMQPushConsumerImpl);\n        assertFalse(pushConsumer1.isEnableTrace());\n        assertTrue(UtilAll.isBlank(pushConsumer1.getTraceTopic()));\n        DefaultMQPushConsumer pushConsumer2 = new DefaultMQPushConsumer(consumerGroup, mock(RPCHook.class), new AllocateMessageQueueAveragelyByCircle());\n        assertNotNull(pushConsumer2);\n        assertEquals(consumerGroup, pushConsumer2.getConsumerGroup());\n        assertTrue(pushConsumer2.getAllocateMessageQueueStrategy() instanceof AllocateMessageQueueAveragelyByCircle);\n        assertNotNull(pushConsumer2.defaultMQPushConsumerImpl);\n        assertFalse(pushConsumer2.isEnableTrace());\n        assertTrue(UtilAll.isBlank(pushConsumer2.getTraceTopic()));\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMachineRoomNearByTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.rebalance;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Random;\nimport java.util.Set;\nimport java.util.TreeMap;\nimport java.util.TreeSet;\nimport org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\n\npublic class AllocateMachineRoomNearByTest {\n\n    private static final String CID_PREFIX = \"CID-\";\n\n    private final String topic = \"topic_test\";\n    private final AllocateMachineRoomNearby.MachineRoomResolver machineRoomResolver =  new AllocateMachineRoomNearby.MachineRoomResolver() {\n        @Override\n        public String brokerDeployIn(MessageQueue messageQueue) {\n            return messageQueue.getBrokerName().split(\"-\")[0];\n        }\n\n        @Override\n        public String consumerDeployIn(String clientID) {\n            return clientID.split(\"-\")[0];\n        }\n    };\n    private final AllocateMessageQueueStrategy allocateMessageQueueStrategy = new AllocateMachineRoomNearby(new AllocateMessageQueueAveragely(), machineRoomResolver);\n\n\n    @Before\n    public void init() {\n    }\n\n\n    @Test\n    public void test1() {\n        testWhenIDCSizeEquals(5,20,10);\n        testWhenIDCSizeEquals(5,20,20);\n        testWhenIDCSizeEquals(5,20,30);\n        testWhenIDCSizeEquals(5,20,0);\n    }\n\n    @Test\n    public void test2() {\n        testWhenConsumerIDCIsMore(5,1,10, 10, false);\n        testWhenConsumerIDCIsMore(5,1,10, 5, false);\n        testWhenConsumerIDCIsMore(5,1,10, 20, false);\n        testWhenConsumerIDCIsMore(5,1,10, 0, false);\n    }\n\n    @Test\n    public void test3() {\n        testWhenConsumerIDCIsLess(5,2,10, 10, false);\n        testWhenConsumerIDCIsLess(5,2,10, 5, false);\n        testWhenConsumerIDCIsLess(5,2,10, 20, false);\n        testWhenConsumerIDCIsLess(5,2,10, 0, false);\n    }\n\n\n    @Test\n    public void testRun10RandomCase() {\n        for (int i = 0; i < 10; i++) {\n            int consumerSize = new Random().nextInt(200) + 1;//1-200\n            int queueSize = new Random().nextInt(100) + 1;//1-100\n            int brokerIDCSize = new Random().nextInt(10) + 1;//1-10\n            int consumerIDCSize = new Random().nextInt(10) + 1;//1-10\n\n            if (brokerIDCSize == consumerIDCSize) {\n                testWhenIDCSizeEquals(brokerIDCSize,queueSize,consumerSize);\n            }\n            else if (brokerIDCSize > consumerIDCSize) {\n                testWhenConsumerIDCIsLess(brokerIDCSize,brokerIDCSize - consumerIDCSize, queueSize, consumerSize, false);\n            } else {\n                testWhenConsumerIDCIsMore(brokerIDCSize, consumerIDCSize - brokerIDCSize, queueSize, consumerSize, false);\n            }\n        }\n    }\n\n\n\n\n    public void testWhenIDCSizeEquals(int idcSize, int queueSize, int consumerSize) {\n        List<String> cidAll = prepareConsumer(idcSize, consumerSize);\n        List<MessageQueue> mqAll = prepareMQ(idcSize, queueSize);\n        List<MessageQueue> resAll = new ArrayList<>();\n        for (String currentID : cidAll) {\n            List<MessageQueue> res = allocateMessageQueueStrategy.allocate(\"Test-C-G\",currentID,mqAll,cidAll);\n            for (MessageQueue mq : res) {\n                Assert.assertTrue(machineRoomResolver.brokerDeployIn(mq).equals(machineRoomResolver.consumerDeployIn(currentID)));\n            }\n            resAll.addAll(res);\n        }\n        Assert.assertTrue(hasAllocateAllQ(cidAll,mqAll,resAll));\n    }\n\n    public void testWhenConsumerIDCIsMore(int brokerIDCSize, int consumerMore, int queueSize, int consumerSize, boolean print) {\n        Set<String> brokerIDCWithConsumer = new TreeSet<>();\n        List<String> cidAll = prepareConsumer(brokerIDCSize + consumerMore, consumerSize);\n        List<MessageQueue> mqAll = prepareMQ(brokerIDCSize, queueSize);\n        for (MessageQueue mq : mqAll) {\n            brokerIDCWithConsumer.add(machineRoomResolver.brokerDeployIn(mq));\n        }\n\n        List<MessageQueue> resAll = new ArrayList<>();\n        for (String currentID : cidAll) {\n            List<MessageQueue> res = allocateMessageQueueStrategy.allocate(\"Test-C-G\",currentID,mqAll,cidAll);\n            for (MessageQueue mq : res) {\n                if (brokerIDCWithConsumer.contains(machineRoomResolver.brokerDeployIn(mq))) { //healthy idc, so only consumer in this idc should be allocated\n                    Assert.assertTrue(machineRoomResolver.brokerDeployIn(mq).equals(machineRoomResolver.consumerDeployIn(currentID)));\n                }\n            }\n            resAll.addAll(res);\n        }\n\n        Assert.assertTrue(hasAllocateAllQ(cidAll,mqAll,resAll));\n    }\n\n    public void testWhenConsumerIDCIsLess(int brokerIDCSize, int consumerIDCLess, int queueSize, int consumerSize, boolean print) {\n        Set<String> healthyIDC = new TreeSet<>();\n        List<String> cidAll = prepareConsumer(brokerIDCSize - consumerIDCLess, consumerSize);\n        List<MessageQueue> mqAll = prepareMQ(brokerIDCSize, queueSize);\n        for (String cid : cidAll) {\n            healthyIDC.add(machineRoomResolver.consumerDeployIn(cid));\n        }\n\n        List<MessageQueue> resAll = new ArrayList<>();\n        Map<String, List<MessageQueue>> idc2Res = new TreeMap<>();\n        for (String currentID : cidAll) {\n            String currentIDC = machineRoomResolver.consumerDeployIn(currentID);\n            List<MessageQueue> res = allocateMessageQueueStrategy.allocate(\"Test-C-G\",currentID,mqAll,cidAll);\n            if (!idc2Res.containsKey(currentIDC)) {\n                idc2Res.put(currentIDC, new ArrayList<>());\n            }\n            idc2Res.get(currentIDC).addAll(res);\n            resAll.addAll(res);\n        }\n\n        for (String consumerIDC : healthyIDC) {\n            List<MessageQueue> resInOneIDC = idc2Res.get(consumerIDC);\n            List<MessageQueue> mqInThisIDC = createMessageQueueList(consumerIDC,queueSize);\n            Assert.assertTrue(resInOneIDC.containsAll(mqInThisIDC));\n        }\n\n        Assert.assertTrue(hasAllocateAllQ(cidAll,mqAll,resAll));\n    }\n\n\n    private boolean hasAllocateAllQ(List<String> cidAll,List<MessageQueue> mqAll, List<MessageQueue> allocatedResAll) {\n        if (cidAll.isEmpty()) {\n            return allocatedResAll.isEmpty();\n        }\n        return mqAll.containsAll(allocatedResAll) && allocatedResAll.containsAll(mqAll) && mqAll.size() == allocatedResAll.size();\n    }\n\n\n    private List<String> createConsumerIdList(String machineRoom, int size) {\n        List<String> consumerIdList = new ArrayList<>(size);\n        for (int i = 0; i < size; i++) {\n            consumerIdList.add(machineRoom + \"-\" + CID_PREFIX + String.valueOf(i));\n        }\n        return consumerIdList;\n    }\n\n    private List<MessageQueue> createMessageQueueList(String machineRoom, int size) {\n        List<MessageQueue> messageQueueList = new ArrayList<>(size);\n        for (int i = 0; i < size; i++) {\n            MessageQueue mq = new MessageQueue(topic, machineRoom + \"-brokerName\", i);\n            messageQueueList.add(mq);\n        }\n        return messageQueueList;\n    }\n\n    private List<MessageQueue> prepareMQ(int brokerIDCSize, int queueSize) {\n        List<MessageQueue> mqAll = new ArrayList<>();\n        for (int i = 1; i <= brokerIDCSize; i++) {\n            mqAll.addAll(createMessageQueueList(\"IDC\" + i, queueSize));\n        }\n\n        return mqAll;\n    }\n\n    private List<String> prepareConsumer(int idcSize, int consumerSize) {\n        List<String> cidAll = new ArrayList<>();\n        for (int i = 1; i <= idcSize; i++) {\n            cidAll.addAll(createConsumerIdList(\"IDC\" + i, consumerSize));\n        }\n        return cidAll;\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircleTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.rebalance;\n\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport junit.framework.TestCase;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.junit.Assert;\n\npublic class AllocateMessageQueueAveragelyByCircleTest extends TestCase {\n\n    public void testAllocateMessageQueueAveragelyByCircle() {\n        List<String> consumerIdList = createConsumerIdList(4);\n        List<MessageQueue> messageQueueList = createMessageQueueList(10);\n        // the consumerId not in cidAll\n        List<MessageQueue> allocateQueues = new AllocateMessageQueueAveragelyByCircle().allocate(\"\", \"CID_PREFIX\", messageQueueList, consumerIdList);\n        Assert.assertEquals(0, allocateQueues.size());\n\n        Map<String, int[]> consumerAllocateQueue = new HashMap<>(consumerIdList.size());\n        for (String consumerId : consumerIdList) {\n            List<MessageQueue> queues = new AllocateMessageQueueAveragelyByCircle().allocate(\"\", consumerId, messageQueueList, consumerIdList);\n            int[] queueIds = new int[queues.size()];\n            for (int i = 0; i < queues.size(); i++) {\n                queueIds[i] = queues.get(i).getQueueId();\n            }\n            consumerAllocateQueue.put(consumerId, queueIds);\n        }\n        Assert.assertArrayEquals(new int[] {0, 4, 8}, consumerAllocateQueue.get(\"CID_PREFIX0\"));\n        Assert.assertArrayEquals(new int[] {1, 5, 9}, consumerAllocateQueue.get(\"CID_PREFIX1\"));\n        Assert.assertArrayEquals(new int[] {2, 6}, consumerAllocateQueue.get(\"CID_PREFIX2\"));\n        Assert.assertArrayEquals(new int[] {3, 7}, consumerAllocateQueue.get(\"CID_PREFIX3\"));\n    }\n\n    private List<String> createConsumerIdList(int size) {\n        List<String> consumerIdList = new ArrayList<>(size);\n        for (int i = 0; i < size; i++) {\n            consumerIdList.add(\"CID_PREFIX\" + i);\n        }\n        return consumerIdList;\n    }\n\n    private List<MessageQueue> createMessageQueueList(int size) {\n        List<MessageQueue> messageQueueList = new ArrayList<>(size);\n        for (int i = 0; i < size; i++) {\n            MessageQueue mq = new MessageQueue(\"topic\", \"brokerName\", i);\n            messageQueueList.add(mq);\n        }\n        return messageQueueList;\n    }\n}\n\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.rebalance;\n\nimport junit.framework.TestCase;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.junit.Assert;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\npublic class AllocateMessageQueueAveragelyTest extends TestCase {\n\n    public void testAllocateMessageQueueAveragely() {\n        List<String> consumerIdList = createConsumerIdList(4);\n        List<MessageQueue> messageQueueList = createMessageQueueList(10);\n        int[] results = new int[consumerIdList.size()];\n        for (int i = 0; i < consumerIdList.size(); i++) {\n            List<MessageQueue> result = new AllocateMessageQueueAveragely().allocate(\"\", consumerIdList.get(i), messageQueueList, consumerIdList);\n            results[i] = result.size();\n        }\n        Assert.assertArrayEquals(new int[]{3, 3, 2, 2}, results);\n    }\n\n    private List<String> createConsumerIdList(int size) {\n        List<String> consumerIdList = new ArrayList<>(size);\n        for (int i = 0; i < size; i++) {\n            consumerIdList.add(\"CID_PREFIX\" + i);\n        }\n        return consumerIdList;\n    }\n\n    private List<MessageQueue> createMessageQueueList(int size) {\n        List<MessageQueue> messageQueueList = new ArrayList<>(size);\n        for (int i = 0; i < size; i++) {\n            MessageQueue mq = new MessageQueue(\"topic\", \"brokerName\", i);\n            messageQueueList.add(mq);\n        }\n        return messageQueueList;\n    }\n\n\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueByConfigTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.rebalance;\n\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport junit.framework.TestCase;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.junit.Assert;\n\npublic class AllocateMessageQueueByConfigTest extends TestCase {\n\n    public void testAllocateMessageQueueByConfig() {\n        List<String> consumerIdList = createConsumerIdList(2);\n        List<MessageQueue> messageQueueList = createMessageQueueList(4);\n        AllocateMessageQueueByConfig allocateStrategy = new AllocateMessageQueueByConfig();\n        allocateStrategy.setMessageQueueList(messageQueueList);\n\n        Map<String, int[]> consumerAllocateQueue = new HashMap<>(consumerIdList.size());\n        for (String consumerId : consumerIdList) {\n            List<MessageQueue> queues = allocateStrategy.allocate(\"\", consumerId, messageQueueList, consumerIdList);\n            int[] queueIds = new int[queues.size()];\n            for (int i = 0; i < queues.size(); i++) {\n                queueIds[i] = queues.get(i).getQueueId();\n            }\n            consumerAllocateQueue.put(consumerId, queueIds);\n        }\n        Assert.assertArrayEquals(new int[] {0, 1, 2, 3}, consumerAllocateQueue.get(\"CID_PREFIX0\"));\n        Assert.assertArrayEquals(new int[] {0, 1, 2, 3}, consumerAllocateQueue.get(\"CID_PREFIX1\"));\n    }\n\n    private List<String> createConsumerIdList(int size) {\n        List<String> consumerIdList = new ArrayList<>(size);\n        for (int i = 0; i < size; i++) {\n            consumerIdList.add(\"CID_PREFIX\" + i);\n        }\n        return consumerIdList;\n    }\n\n    private List<MessageQueue> createMessageQueueList(int size) {\n        List<MessageQueue> messageQueueList = new ArrayList<>(size);\n        for (int i = 0; i < size; i++) {\n            MessageQueue mq = new MessageQueue(\"topic\", \"brokerName\", i);\n            messageQueueList.add(mq);\n        }\n        return messageQueueList;\n    }\n}\n\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueByMachineRoomTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.rebalance;\n\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport junit.framework.TestCase;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.junit.Assert;\n\npublic class AllocateMessageQueueByMachineRoomTest extends TestCase {\n\n    public void testAllocateMessageQueueByMachineRoom() {\n        List<String> consumerIdList = createConsumerIdList(2);\n        List<MessageQueue> messageQueueList = createMessageQueueList(10);\n        Set<String> consumeridcs = new HashSet<>();\n        consumeridcs.add(\"room1\");\n        AllocateMessageQueueByMachineRoom allocateStrategy = new AllocateMessageQueueByMachineRoom();\n        allocateStrategy.setConsumeridcs(consumeridcs);\n\n        // mqAll is null or mqAll empty\n        try {\n            allocateStrategy.allocate(\"\", consumerIdList.get(0), new ArrayList<>(), consumerIdList);\n        } catch (Exception e) {\n            assert e instanceof IllegalArgumentException;\n            Assert.assertEquals(\"mqAll is null or mqAll empty\", e.getMessage());\n        }\n\n        Map<String, int[]> consumerAllocateQueue = new HashMap<>(consumerIdList.size());\n        for (String consumerId : consumerIdList) {\n            List<MessageQueue> queues = allocateStrategy.allocate(\"\", consumerId, messageQueueList, consumerIdList);\n            int[] queueIds = new int[queues.size()];\n            for (int i = 0; i < queues.size(); i++) {\n                queueIds[i] = queues.get(i).getQueueId();\n            }\n            consumerAllocateQueue.put(consumerId, queueIds);\n        }\n        Assert.assertArrayEquals(new int[] {0, 1, 4}, consumerAllocateQueue.get(\"CID_PREFIX0\"));\n        Assert.assertArrayEquals(new int[] {2, 3}, consumerAllocateQueue.get(\"CID_PREFIX1\"));\n    }\n\n    private List<String> createConsumerIdList(int size) {\n        List<String> consumerIdList = new ArrayList<>(size);\n        for (int i = 0; i < size; i++) {\n            consumerIdList.add(\"CID_PREFIX\" + i);\n        }\n        return consumerIdList;\n    }\n\n    private List<MessageQueue> createMessageQueueList(int size) {\n        List<MessageQueue> messageQueueList = new ArrayList<>(size);\n        for (int i = 0; i < size; i++) {\n            MessageQueue mq;\n            if (i < size / 2) {\n                mq = new MessageQueue(\"topic\", \"room1@broker-a\", i);\n            } else {\n                mq = new MessageQueue(\"topic\", \"room2@broker-b\", i);\n            }\n            messageQueueList.add(mq);\n        }\n        return messageQueueList;\n    }\n}\n\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueConsitentHashTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.rebalance;\n\nimport java.util.ArrayList;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Random;\nimport java.util.TreeMap;\nimport org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\npublic class AllocateMessageQueueConsitentHashTest {\n\n    private String topic;\n    private static final String CID_PREFIX = \"CID-\";\n\n    @Before\n    public void init() {\n        topic = \"topic_test\";\n    }\n\n    @Test\n    public void testCurrentCIDNotExists() {\n        String currentCID = String.valueOf(Integer.MAX_VALUE);\n        List<String> consumerIdList = createConsumerIdList(2);\n        List<MessageQueue> messageQueueList = createMessageQueueList(6);\n        List<MessageQueue> result = new AllocateMessageQueueConsistentHash().allocate(\"\", currentCID, messageQueueList, consumerIdList);\n        Assert.assertEquals(result.size(), 0);\n    }\n\n    @Test(expected = IllegalArgumentException.class)\n    public void testCurrentCIDIllegalArgument() {\n        List<String> consumerIdList = createConsumerIdList(2);\n        List<MessageQueue> messageQueueList = createMessageQueueList(6);\n        new AllocateMessageQueueConsistentHash().allocate(\"\", \"\", messageQueueList, consumerIdList);\n    }\n\n    @Test(expected = IllegalArgumentException.class)\n    public void testMessageQueueIllegalArgument() {\n        String currentCID = \"0\";\n        List<String> consumerIdList = createConsumerIdList(2);\n        new AllocateMessageQueueConsistentHash().allocate(\"\", currentCID, null, consumerIdList);\n    }\n\n    @Test(expected = IllegalArgumentException.class)\n    public void testConsumerIdIllegalArgument() {\n        String currentCID = \"0\";\n        List<MessageQueue> messageQueueList = createMessageQueueList(6);\n        new AllocateMessageQueueConsistentHash().allocate(\"\", currentCID, messageQueueList, null);\n    }\n\n    @Test\n    public void testAllocate1() {\n        testAllocate(20, 10);\n    }\n\n    @Test\n    public void testAllocate2() {\n        testAllocate(10, 20);\n    }\n\n    @Test\n    public void testRun100RandomCase() {\n        for (int i = 0; i < 10; i++) {\n            int consumerSize = new Random().nextInt(20) + 1;//1-20\n            int queueSize = new Random().nextInt(20) + 1;//1-20\n            testAllocate(queueSize, consumerSize);\n            try {\n                Thread.sleep(1);\n            } catch (InterruptedException e) {\n            }\n        }\n    }\n\n    public void testAllocate(int queueSize, int consumerSize) {\n        AllocateMessageQueueStrategy allocateMessageQueueConsistentHash = new AllocateMessageQueueConsistentHash(3);\n\n        List<MessageQueue> mqAll = createMessageQueueList(queueSize);\n\n        List<String> cidAll = createConsumerIdList(consumerSize);\n        List<MessageQueue> allocatedResAll = new ArrayList<>();\n\n        Map<MessageQueue, String> allocateToAllOrigin = new TreeMap<>();\n        //test allocate all\n        {\n\n            List<String> cidBegin = new ArrayList<>(cidAll);\n\n            for (String cid : cidBegin) {\n                List<MessageQueue> rs = allocateMessageQueueConsistentHash.allocate(\"testConsumerGroup\", cid, mqAll, cidBegin);\n                for (MessageQueue mq : rs) {\n                    allocateToAllOrigin.put(mq, cid);\n                }\n                allocatedResAll.addAll(rs);\n            }\n\n            Assert.assertTrue(\n                verifyAllocateAll(cidBegin, mqAll, allocatedResAll));\n        }\n\n        Map<MessageQueue, String> allocateToAllAfterRemoveOne = new TreeMap<>();\n        List<String> cidAfterRemoveOne = new ArrayList<>(cidAll);\n        //test allocate remove one cid\n        {\n            String removeCID = cidAfterRemoveOne.remove(0);\n            List<MessageQueue> mqShouldOnlyChanged = new ArrayList<>();\n            Iterator<Map.Entry<MessageQueue, String>> it = allocateToAllOrigin.entrySet().iterator();\n            while (it.hasNext()) {\n                Map.Entry<MessageQueue, String> entry = it.next();\n                if (entry.getValue().equals(removeCID)) {\n                    mqShouldOnlyChanged.add(entry.getKey());\n                }\n            }\n\n            List<MessageQueue> allocatedResAllAfterRemove = new ArrayList<>();\n            for (String cid : cidAfterRemoveOne) {\n                List<MessageQueue> rs = allocateMessageQueueConsistentHash.allocate(\"testConsumerGroup\", cid, mqAll, cidAfterRemoveOne);\n                allocatedResAllAfterRemove.addAll(rs);\n                for (MessageQueue mq : rs) {\n                    allocateToAllAfterRemoveOne.put(mq, cid);\n                }\n            }\n\n            Assert.assertTrue(\"queueSize\" + queueSize + \"consumerSize:\" + consumerSize + \"\\nmqAll:\" + mqAll + \"\\nallocatedResAllAfterRemove\" + allocatedResAllAfterRemove,\n                verifyAllocateAll(cidAfterRemoveOne, mqAll, allocatedResAllAfterRemove));\n            verifyAfterRemove(allocateToAllOrigin, allocateToAllAfterRemoveOne, removeCID);\n        }\n\n        List<String> cidAfterAdd = new ArrayList<>(cidAfterRemoveOne);\n        //test allocate add one more cid\n        {\n            String newCid = CID_PREFIX + \"NEW\";\n            cidAfterAdd.add(newCid);\n            List<MessageQueue> mqShouldOnlyChanged = new ArrayList<>();\n            List<MessageQueue> allocatedResAllAfterAdd = new ArrayList<>();\n            Map<MessageQueue, String> allocateToAll3 = new TreeMap<>();\n            for (String cid : cidAfterAdd) {\n                List<MessageQueue> rs = allocateMessageQueueConsistentHash.allocate(\"testConsumerGroup\", cid, mqAll, cidAfterAdd);\n                allocatedResAllAfterAdd.addAll(rs);\n                for (MessageQueue mq : rs) {\n                    allocateToAll3.put(mq, cid);\n                    if (cid.equals(newCid)) {\n                        mqShouldOnlyChanged.add(mq);\n                    }\n                }\n            }\n\n            Assert.assertTrue(\n                verifyAllocateAll(cidAfterAdd, mqAll, allocatedResAllAfterAdd));\n            verifyAfterAdd(allocateToAllAfterRemoveOne, allocateToAll3, newCid);\n        }\n    }\n\n    private boolean verifyAllocateAll(List<String> cidAll, List<MessageQueue> mqAll,\n        List<MessageQueue> allocatedResAll) {\n        if (cidAll.isEmpty()) {\n            return allocatedResAll.isEmpty();\n        }\n        return mqAll.containsAll(allocatedResAll) && allocatedResAll.containsAll(mqAll);\n    }\n\n    private void verifyAfterRemove(Map<MessageQueue, String> allocateToBefore, Map<MessageQueue, String> allocateAfter,\n        String removeCID) {\n        for (MessageQueue mq : allocateToBefore.keySet()) {\n            String allocateToOrigin = allocateToBefore.get(mq);\n            if (allocateToOrigin.equals(removeCID)) {\n\n            } else { //the rest queue should be the same\n                Assert.assertTrue(allocateAfter.get(mq).equals(allocateToOrigin));//should be the same\n            }\n        }\n    }\n\n    private void verifyAfterAdd(Map<MessageQueue, String> allocateBefore, Map<MessageQueue, String> allocateAfter,\n        String newCID) {\n        for (MessageQueue mq : allocateAfter.keySet()) {\n            String allocateToOrigin = allocateBefore.get(mq);\n            String allocateToAfter = allocateAfter.get(mq);\n            if (allocateToAfter.equals(newCID)) {\n\n            } else { //the rest queue should be the same\n                Assert.assertTrue(\"it was allocated to \" + allocateToOrigin + \". Now, it is to \" + allocateAfter.get(mq) + \" mq:\" + mq, allocateAfter.get(mq).equals(allocateToOrigin));//should be the same\n            }\n        }\n    }\n\n    private List<String> createConsumerIdList(int size) {\n        List<String> consumerIdList = new ArrayList<>(size);\n        for (int i = 0; i < size; i++) {\n            consumerIdList.add(CID_PREFIX + String.valueOf(i));\n        }\n        return consumerIdList;\n    }\n\n    private List<MessageQueue> createMessageQueueList(int size) {\n        List<MessageQueue> messageQueueList = new ArrayList<>(size);\n        for (int i = 0; i < size; i++) {\n            MessageQueue mq = new MessageQueue(topic, \"brokerName\", i);\n            messageQueueList.add(mq);\n        }\n        return messageQueueList;\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/consumer/store/ControllableOffsetTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.consumer.store;\n\n\nimport org.junit.Before;\nimport org.junit.Test;\nimport static org.junit.Assert.assertEquals;\n\npublic class ControllableOffsetTest {\n\n    private ControllableOffset controllableOffset;\n\n    @Before\n    public void setUp() {\n        controllableOffset = new ControllableOffset(0);\n    }\n\n    @Test\n    public void testUpdateAndFreeze_ShouldFreezeOffsetAtTargetValue() {\n        controllableOffset.updateAndFreeze(100);\n        assertEquals(100, controllableOffset.getOffset());\n        controllableOffset.update(200);\n        assertEquals(100, controllableOffset.getOffset());\n    }\n\n    @Test\n    public void testUpdate_ShouldUpdateOffsetWhenNotFrozen() {\n        controllableOffset.update(200);\n        assertEquals(200, controllableOffset.getOffset());\n    }\n\n    @Test\n    public void testUpdate_ShouldNotUpdateOffsetWhenFrozen() {\n        controllableOffset.updateAndFreeze(100);\n        controllableOffset.update(200);\n        assertEquals(100, controllableOffset.getOffset());\n    }\n\n    @Test\n    public void testUpdate_ShouldNotDecreaseOffsetWhenIncreaseOnly() {\n        controllableOffset.update(200);\n        controllableOffset.update(100, true);\n        assertEquals(200, controllableOffset.getOffset());\n    }\n\n    @Test\n    public void testUpdate_ShouldUpdateOffsetToGreaterValueWhenIncreaseOnly() {\n        controllableOffset.update(100);\n        controllableOffset.update(200, true);\n        assertEquals(200, controllableOffset.getOffset());\n    }\n\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/consumer/store/LocalFileOffsetStoreTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.store;\n\nimport java.io.File;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.HashSet;\nimport java.util.Map;\n\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class LocalFileOffsetStoreTest {\n    @Mock\n    private MQClientInstance mQClientFactory;\n    private String group = \"FooBarGroup\";\n    private String topic = \"FooBar\";\n    private String brokerName = \"DefaultBrokerName\";\n\n    @Before\n    public void init() {\n        System.setProperty(\"rocketmq.client.localOffsetStoreDir\", System.getProperty(\"java.io.tmpdir\") + File.separator + \".rocketmq_offsets\");\n        String clientId = new ClientConfig().buildMQClientId() + \"#TestNamespace\" + System.currentTimeMillis();\n        when(mQClientFactory.getClientId()).thenReturn(clientId);\n    }\n\n    @Test\n    public void testUpdateOffset() throws Exception {\n        OffsetStore offsetStore = new LocalFileOffsetStore(mQClientFactory, group);\n        MessageQueue messageQueue = new MessageQueue(topic, brokerName, 1);\n        offsetStore.updateOffset(messageQueue, 1024, false);\n\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_MEMORY)).isEqualTo(1024);\n\n        offsetStore.updateOffset(messageQueue, 1023, false);\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_MEMORY)).isEqualTo(1023);\n\n        offsetStore.updateOffset(messageQueue, 1022, true);\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_MEMORY)).isEqualTo(1023);\n    }\n\n    @Test\n    public void testReadOffset_FromStore() throws Exception {\n        OffsetStore offsetStore = new LocalFileOffsetStore(mQClientFactory, group);\n        MessageQueue messageQueue = new MessageQueue(topic, brokerName, 2);\n\n        offsetStore.updateOffset(messageQueue, 1024, false);\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE)).isEqualTo(-1);\n\n        offsetStore.persistAll(new HashSet<>(Collections.singletonList(messageQueue)));\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE)).isEqualTo(1024);\n    }\n\n    @Test\n    public void testCloneOffset() throws Exception {\n        OffsetStore offsetStore = new LocalFileOffsetStore(mQClientFactory, group);\n        MessageQueue messageQueue = new MessageQueue(topic, brokerName, 3);\n        offsetStore.updateOffset(messageQueue, 1024, false);\n        Map<MessageQueue, Long> cloneOffsetTable = offsetStore.cloneOffsetTable(topic);\n\n        assertThat(cloneOffsetTable.size()).isEqualTo(1);\n        assertThat(cloneOffsetTable.get(messageQueue)).isEqualTo(1024);\n    }\n\n    @Test\n    public void testPersist() throws Exception {\n        OffsetStore offsetStore = new LocalFileOffsetStore(mQClientFactory, group);\n\n        MessageQueue messageQueue0 = new MessageQueue(topic, brokerName, 0);\n        offsetStore.updateOffset(messageQueue0, 1024, false);\n        offsetStore.persist(messageQueue0);\n        assertThat(offsetStore.readOffset(messageQueue0, ReadOffsetType.READ_FROM_STORE)).isEqualTo(1024);\n\n        MessageQueue messageQueue1 = new MessageQueue(topic, brokerName, 1);\n        assertThat(offsetStore.readOffset(messageQueue1, ReadOffsetType.READ_FROM_STORE)).isEqualTo(-1);\n    }\n\n    @Test\n    public void testPersistAll() throws Exception {\n        OffsetStore offsetStore = new LocalFileOffsetStore(mQClientFactory, group);\n\n        MessageQueue messageQueue0 = new MessageQueue(topic, brokerName, 0);\n        offsetStore.updateOffset(messageQueue0, 1024, false);\n        offsetStore.persistAll(new HashSet<MessageQueue>(Collections.singletonList(messageQueue0)));\n        assertThat(offsetStore.readOffset(messageQueue0, ReadOffsetType.READ_FROM_STORE)).isEqualTo(1024);\n\n        MessageQueue messageQueue1 = new MessageQueue(topic, brokerName, 1);\n        MessageQueue messageQueue2 = new MessageQueue(topic, brokerName, 2);\n        offsetStore.updateOffset(messageQueue1, 1025, false);\n        offsetStore.updateOffset(messageQueue2, 1026, false);\n        offsetStore.persistAll(new HashSet<MessageQueue>(Arrays.asList(messageQueue1, messageQueue2)));\n\n        assertThat(offsetStore.readOffset(messageQueue0, ReadOffsetType.READ_FROM_STORE)).isEqualTo(1024);\n        assertThat(offsetStore.readOffset(messageQueue1, ReadOffsetType.READ_FROM_STORE)).isEqualTo(1025);\n        assertThat(offsetStore.readOffset(messageQueue2, ReadOffsetType.READ_FROM_STORE)).isEqualTo(1026);\n    }\n\n    @Test\n    public void testRemoveOffset() throws Exception {\n        OffsetStore offsetStore = new LocalFileOffsetStore(mQClientFactory, group);\n        MessageQueue messageQueue = new MessageQueue(topic, brokerName, 0);\n        offsetStore.updateOffset(messageQueue, 1024, false);\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_MEMORY)).isEqualTo(1024);\n\n        offsetStore.removeOffset(messageQueue);\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_MEMORY)).isEqualTo(-1);\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStoreTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.consumer.store;\n\nimport java.util.Collections;\nimport java.util.HashSet;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.OffsetNotFoundException;\nimport org.apache.rocketmq.client.impl.FindBrokerResult;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.invocation.InvocationOnMock;\nimport org.mockito.junit.MockitoJUnitRunner;\nimport org.mockito.stubbing.Answer;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.doAnswer;\nimport static org.mockito.Mockito.doThrow;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class RemoteBrokerOffsetStoreTest {\n    @Mock\n    private MQClientInstance mQClientFactory;\n    @Mock\n    private MQClientAPIImpl mqClientAPI;\n    private String group = \"FooBarGroup\";\n    private String topic = \"FooBar\";\n    private String brokerName = \"DefaultBrokerName\";\n\n    @Before\n    public void init() {\n        System.setProperty(\"rocketmq.client.localOffsetStoreDir\", System.getProperty(\"java.io.tmpdir\") + \".rocketmq_offsets\");\n        String clientId = new ClientConfig().buildMQClientId() + \"#TestNamespace\" + System.currentTimeMillis();\n        when(mQClientFactory.getClientId()).thenReturn(clientId);\n        when(mQClientFactory.findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, false)).thenReturn(new FindBrokerResult(\"127.0.0.1\", false));\n        when(mQClientFactory.getMQClientAPIImpl()).thenReturn(mqClientAPI);\n        when(mQClientFactory.getBrokerNameFromMessageQueue(any())).thenReturn(brokerName);\n    }\n\n    @Test\n    public void testUpdateOffset() throws Exception {\n        OffsetStore offsetStore = new RemoteBrokerOffsetStore(mQClientFactory, group);\n        MessageQueue messageQueue = new MessageQueue(topic, brokerName, 1);\n\n        offsetStore.updateOffset(messageQueue, 1024, false);\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_MEMORY)).isEqualTo(1024);\n\n        offsetStore.updateOffset(messageQueue, 1023, false);\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_MEMORY)).isEqualTo(1023);\n\n        offsetStore.updateOffset(messageQueue, 1022, true);\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_MEMORY)).isEqualTo(1023);\n    }\n\n    @Test\n    public void testUpdateAndFreezeOffset() throws Exception {\n        OffsetStore offsetStore = new RemoteBrokerOffsetStore(mQClientFactory, group);\n        MessageQueue messageQueue = new MessageQueue(topic, brokerName, 1);\n\n        offsetStore.updateAndFreezeOffset(messageQueue, 1024);\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_MEMORY)).isEqualTo(1024);\n\n        offsetStore.updateOffset(messageQueue, 1023, false);\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_MEMORY)).isEqualTo(1024);\n\n        offsetStore.updateOffset(messageQueue, 1022, true);\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_MEMORY)).isEqualTo(1024);\n    }\n\n    @Test\n    public void testUpdateAndFreezeOffsetWithRemove() throws Exception {\n        OffsetStore offsetStore = new RemoteBrokerOffsetStore(mQClientFactory, group);\n        MessageQueue messageQueue = new MessageQueue(topic, brokerName, 1);\n\n        offsetStore.updateAndFreezeOffset(messageQueue, 1024);\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_MEMORY)).isEqualTo(1024);\n\n        offsetStore.updateOffset(messageQueue, 1023, false);\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_MEMORY)).isEqualTo(1024);\n\n        offsetStore.removeOffset(messageQueue);\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_MEMORY)).isEqualTo(-1);\n        offsetStore.updateOffset(messageQueue, 1023, false);\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_MEMORY)).isEqualTo(1023);\n    }\n\n    @Test\n    public void testReadOffset_WithException() throws Exception {\n        OffsetStore offsetStore = new RemoteBrokerOffsetStore(mQClientFactory, group);\n        MessageQueue messageQueue = new MessageQueue(topic, brokerName, 2);\n\n        offsetStore.updateOffset(messageQueue, 1024, false);\n\n        doThrow(new OffsetNotFoundException(ResponseCode.QUERY_NOT_FOUND, \"\", null))\n            .when(mqClientAPI).queryConsumerOffset(anyString(), any(QueryConsumerOffsetRequestHeader.class), anyLong());\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE)).isEqualTo(-1);\n\n\n        doThrow(new MQBrokerException(-1, \"\", null))\n            .when(mqClientAPI).queryConsumerOffset(anyString(), any(QueryConsumerOffsetRequestHeader.class), anyLong());\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE)).isEqualTo(-2);\n\n        doThrow(new RemotingException(\"\", null))\n            .when(mqClientAPI).queryConsumerOffset(anyString(), any(QueryConsumerOffsetRequestHeader.class), anyLong());\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE)).isEqualTo(-2);\n    }\n\n    @Test\n    public void testReadOffset_Success() throws Exception {\n        OffsetStore offsetStore = new RemoteBrokerOffsetStore(mQClientFactory, group);\n        final MessageQueue messageQueue = new MessageQueue(topic, brokerName, 3);\n\n        doAnswer(new Answer() {\n            @Override\n            public Object answer(InvocationOnMock mock) throws Throwable {\n                UpdateConsumerOffsetRequestHeader updateRequestHeader = mock.getArgument(1);\n                when(mqClientAPI.queryConsumerOffset(anyString(), any(QueryConsumerOffsetRequestHeader.class), anyLong())).thenReturn(updateRequestHeader.getCommitOffset());\n                return null;\n            }\n        }).when(mqClientAPI).updateConsumerOffsetOneway(any(String.class), any(UpdateConsumerOffsetRequestHeader.class), any(Long.class));\n\n        offsetStore.updateOffset(messageQueue, 1024, false);\n        offsetStore.persist(messageQueue);\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE)).isEqualTo(1024);\n\n        offsetStore.updateOffset(messageQueue, 1023, false);\n        offsetStore.persist(messageQueue);\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE)).isEqualTo(1023);\n\n        offsetStore.updateOffset(messageQueue, 1022, true);\n        offsetStore.persist(messageQueue);\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE)).isEqualTo(1023);\n\n        offsetStore.updateOffset(messageQueue, 1025, false);\n        offsetStore.persistAll(new HashSet<>(Collections.singletonList(messageQueue)));\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE)).isEqualTo(1025);\n    }\n\n    @Test\n    public void testRemoveOffset() throws Exception {\n        OffsetStore offsetStore = new RemoteBrokerOffsetStore(mQClientFactory, group);\n        final MessageQueue messageQueue = new MessageQueue(topic, brokerName, 4);\n\n        offsetStore.updateOffset(messageQueue, 1024, false);\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_MEMORY)).isEqualTo(1024);\n\n        offsetStore.removeOffset(messageQueue);\n        assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_MEMORY)).isEqualTo(-1);\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/impl/ClientRemotingProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.client.impl.producer.MQProducerInner;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.body.ResetOffsetBody;\nimport org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerStatusRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.NotifyConsumerIdsChangedRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ReplyMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\n\nimport static org.apache.rocketmq.common.message.MessageDecoder.NAME_VALUE_SEPARATOR;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ClientRemotingProcessorTest {\n\n    @Mock\n    private MQClientInstance mQClientFactory;\n\n    private ClientRemotingProcessor processor;\n\n    private final String defaultTopic = \"defaultTopic\";\n\n    private final String defaultBroker = \"defaultBroker\";\n\n    private final String defaultGroup = \"defaultGroup\";\n\n    @Before\n    public void init() throws RemotingException, InterruptedException, MQClientException {\n        processor = new ClientRemotingProcessor(mQClientFactory);\n        ClientConfig clientConfig = mock(ClientConfig.class);\n        when(clientConfig.getNamespace()).thenReturn(\"namespace\");\n        when(mQClientFactory.getClientConfig()).thenReturn(clientConfig);\n        MQProducerInner producerInner = mock(MQProducerInner.class);\n        when(mQClientFactory.selectProducer(defaultGroup)).thenReturn(producerInner);\n    }\n\n    @Test\n    public void testCheckTransactionState() throws Exception {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        RemotingCommand request = mock(RemotingCommand.class);\n        when(request.getCode()).thenReturn(RequestCode.CHECK_TRANSACTION_STATE);\n        when(request.getBody()).thenReturn(getMessageResult());\n        CheckTransactionStateRequestHeader requestHeader = new CheckTransactionStateRequestHeader();\n        when(request.decodeCommandCustomHeader(CheckTransactionStateRequestHeader.class)).thenReturn(requestHeader);\n        assertNull(processor.processRequest(ctx, request));\n    }\n\n    @Test\n    public void testNotifyConsumerIdsChanged() throws Exception {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        RemotingCommand request = mock(RemotingCommand.class);\n        when(request.getCode()).thenReturn(RequestCode.NOTIFY_CONSUMER_IDS_CHANGED);\n        NotifyConsumerIdsChangedRequestHeader requestHeader = new NotifyConsumerIdsChangedRequestHeader();\n        when(request.decodeCommandCustomHeader(NotifyConsumerIdsChangedRequestHeader.class)).thenReturn(requestHeader);\n        assertNull(processor.processRequest(ctx, request));\n    }\n\n    @Test\n    public void testResetOffset() throws Exception {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        RemotingCommand request = mock(RemotingCommand.class);\n        when(request.getCode()).thenReturn(RequestCode.RESET_CONSUMER_CLIENT_OFFSET);\n        ResetOffsetBody offsetBody = new ResetOffsetBody();\n        when(request.getBody()).thenReturn(RemotingSerializable.encode(offsetBody));\n        ResetOffsetRequestHeader requestHeader = new ResetOffsetRequestHeader();\n        when(request.decodeCommandCustomHeader(ResetOffsetRequestHeader.class)).thenReturn(requestHeader);\n        assertNull(processor.processRequest(ctx, request));\n    }\n\n    @Test\n    public void testGetConsumeStatus() throws Exception {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        RemotingCommand request = mock(RemotingCommand.class);\n        when(request.getCode()).thenReturn(RequestCode.GET_CONSUMER_STATUS_FROM_CLIENT);\n        GetConsumerStatusRequestHeader requestHeader = new GetConsumerStatusRequestHeader();\n        when(request.decodeCommandCustomHeader(GetConsumerStatusRequestHeader.class)).thenReturn(requestHeader);\n        assertNotNull(processor.processRequest(ctx, request));\n    }\n\n    @Test\n    public void testGetConsumerRunningInfo() throws Exception {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        RemotingCommand request = mock(RemotingCommand.class);\n        when(request.getCode()).thenReturn(RequestCode.GET_CONSUMER_RUNNING_INFO);\n        ConsumerRunningInfo consumerRunningInfo = new ConsumerRunningInfo();\n        consumerRunningInfo.setJstack(\"jstack\");\n        when(mQClientFactory.consumerRunningInfo(anyString())).thenReturn(consumerRunningInfo);\n        GetConsumerRunningInfoRequestHeader requestHeader = new GetConsumerRunningInfoRequestHeader();\n        requestHeader.setJstackEnable(true);\n        requestHeader.setConsumerGroup(defaultGroup);\n        when(request.decodeCommandCustomHeader(GetConsumerRunningInfoRequestHeader.class)).thenReturn(requestHeader);\n        RemotingCommand command = processor.processRequest(ctx, request);\n        assertNotNull(command);\n        assertEquals(ResponseCode.SUCCESS, command.getCode());\n    }\n\n    @Test\n    public void testConsumeMessageDirectly() throws Exception {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        RemotingCommand request = mock(RemotingCommand.class);\n        when(request.getCode()).thenReturn(RequestCode.CONSUME_MESSAGE_DIRECTLY);\n        when(request.getBody()).thenReturn(getMessageResult());\n        ConsumeMessageDirectlyResult directlyResult = mock(ConsumeMessageDirectlyResult.class);\n        when(mQClientFactory.consumeMessageDirectly(any(MessageExt.class), anyString(), anyString())).thenReturn(directlyResult);\n        ConsumeMessageDirectlyResultRequestHeader requestHeader = new ConsumeMessageDirectlyResultRequestHeader();\n        requestHeader.setConsumerGroup(defaultGroup);\n        requestHeader.setBrokerName(defaultBroker);\n        when(request.decodeCommandCustomHeader(ConsumeMessageDirectlyResultRequestHeader.class)).thenReturn(requestHeader);\n        RemotingCommand command = processor.processRequest(ctx, request);\n        assertNotNull(command);\n        assertEquals(ResponseCode.SUCCESS, command.getCode());\n    }\n\n    @Test\n    public void testReceiveReplyMessage() throws Exception {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        RemotingCommand request = mock(RemotingCommand.class);\n        when(request.getCode()).thenReturn(RequestCode.PUSH_REPLY_MESSAGE_TO_CLIENT);\n        when(request.getBody()).thenReturn(getMessageResult());\n        when(request.decodeCommandCustomHeader(ReplyMessageRequestHeader.class)).thenReturn(createReplyMessageRequestHeader());\n        when(request.getBody()).thenReturn(new byte[1]);\n        RemotingCommand command = processor.processRequest(ctx, request);\n        assertNotNull(command);\n        assertEquals(ResponseCode.SUCCESS, command.getCode());\n    }\n\n    private ReplyMessageRequestHeader createReplyMessageRequestHeader() {\n        ReplyMessageRequestHeader result = new ReplyMessageRequestHeader();\n        result.setTopic(defaultTopic);\n        result.setQueueId(0);\n        result.setStoreTimestamp(System.currentTimeMillis());\n        result.setBornTimestamp(System.currentTimeMillis());\n        result.setReconsumeTimes(1);\n        result.setBornHost(\"127.0.0.1:12911\");\n        result.setStoreHost(\"127.0.0.1:10911\");\n        result.setSysFlag(1);\n        result.setFlag(1);\n        result.setProperties(\"CORRELATION_ID\" + NAME_VALUE_SEPARATOR + \"1\");\n        return result;\n    }\n\n    private byte[] getMessageResult() throws Exception {\n        byte[] bytes = MessageDecoder.encode(createMessageExt(), false);\n        ByteBuffer byteBuffer = ByteBuffer.allocate(bytes.length);\n        byteBuffer.put(bytes);\n        return byteBuffer.array();\n    }\n\n    private MessageExt createMessageExt() {\n        MessageExt result = new MessageExt();\n        result.setBody(\"body\".getBytes(StandardCharsets.UTF_8));\n        result.setTopic(defaultTopic);\n        result.setBrokerName(defaultBroker);\n        result.putUserProperty(\"key\", \"value\");\n        result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, defaultGroup);\n        result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, \"TX1\");\n        result.setKeys(\"keys\");\n        SocketAddress bornHost = new InetSocketAddress(\"127.0.0.1\", 12911);\n        SocketAddress storeHost = new InetSocketAddress(\"127.0.0.1\", 10911);\n        result.setBornHost(bornHost);\n        result.setStoreHost(storeHost);\n        return result;\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/impl/MQAdminImplTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl;\n\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.QueryResult;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.remoting.InvokeCallback;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.QueryMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Set;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.doAnswer;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class MQAdminImplTest {\n\n    @Mock\n    private MQClientInstance mQClientFactory;\n\n    @Mock\n    private MQClientAPIImpl mQClientAPIImpl;\n\n    private MQAdminImpl mqAdminImpl;\n\n    private final String defaultTopic = \"defaultTopic\";\n\n    private final String defaultBroker = \"defaultBroker\";\n\n    private final String defaultCluster = \"defaultCluster\";\n\n    private final String defaultBrokerAddr = \"127.0.0.1:10911\";\n\n    private final long defaultTimeout = 3000L;\n\n    @Before\n    public void init() throws RemotingException, InterruptedException, MQClientException {\n        when(mQClientFactory.getMQClientAPIImpl()).thenReturn(mQClientAPIImpl);\n        when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createRouteData());\n        ClientConfig clientConfig = mock(ClientConfig.class);\n        when(clientConfig.getNamespace()).thenReturn(\"namespace\");\n        when(mQClientFactory.getClientConfig()).thenReturn(clientConfig);\n        when(mQClientFactory.findBrokerAddressInPublish(any())).thenReturn(defaultBrokerAddr);\n        when(mQClientFactory.getAnExistTopicRouteData(any())).thenReturn(createRouteData());\n        mqAdminImpl = new MQAdminImpl(mQClientFactory);\n    }\n\n    @Test\n    public void assertTimeoutMillis() {\n        assertEquals(6000L, mqAdminImpl.getTimeoutMillis());\n        mqAdminImpl.setTimeoutMillis(defaultTimeout);\n        assertEquals(defaultTimeout, mqAdminImpl.getTimeoutMillis());\n    }\n\n    @Test\n    public void testCreateTopic() throws MQClientException {\n        mqAdminImpl.createTopic(\"\", defaultTopic, 6);\n    }\n\n    @Test\n    public void assertFetchPublishMessageQueues() throws MQClientException {\n        List<MessageQueue> queueList = mqAdminImpl.fetchPublishMessageQueues(defaultTopic);\n        assertNotNull(queueList);\n        assertEquals(6, queueList.size());\n        for (MessageQueue each : queueList) {\n            assertEquals(defaultTopic, each.getTopic());\n            assertEquals(defaultBroker, each.getBrokerName());\n        }\n    }\n\n    @Test\n    public void assertFetchSubscribeMessageQueues() throws MQClientException {\n        Set<MessageQueue> queueList = mqAdminImpl.fetchSubscribeMessageQueues(defaultTopic);\n        assertNotNull(queueList);\n        assertEquals(6, queueList.size());\n        for (MessageQueue each : queueList) {\n            assertEquals(defaultTopic, each.getTopic());\n            assertEquals(defaultBroker, each.getBrokerName());\n        }\n    }\n\n    @Test\n    public void assertSearchOffset() throws MQClientException {\n        assertEquals(0, mqAdminImpl.searchOffset(new MessageQueue(), defaultTimeout));\n    }\n\n    @Test\n    public void assertMaxOffset() throws MQClientException {\n        assertEquals(0, mqAdminImpl.maxOffset(new MessageQueue()));\n    }\n\n    @Test\n    public void assertMinOffset() throws MQClientException {\n        assertEquals(0, mqAdminImpl.minOffset(new MessageQueue()));\n    }\n\n    @Test\n    public void assertEarliestMsgStoreTime() throws MQClientException {\n        assertEquals(0, mqAdminImpl.earliestMsgStoreTime(new MessageQueue()));\n    }\n\n    @Test(expected = MQClientException.class)\n    public void assertViewMessage() throws MQBrokerException, RemotingException, InterruptedException, MQClientException {\n        MessageExt actual = mqAdminImpl.viewMessage(defaultTopic, \"1\");\n        assertNotNull(actual);\n    }\n\n    @Test\n    public void assertQueryMessage() throws InterruptedException, MQClientException, MQBrokerException, RemotingException {\n        doAnswer(invocation -> {\n            InvokeCallback callback = invocation.getArgument(3);\n            QueryMessageResponseHeader responseHeader = new QueryMessageResponseHeader();\n            responseHeader.setIndexLastUpdatePhyoffset(1L);\n            responseHeader.setIndexLastUpdateTimestamp(System.currentTimeMillis());\n            RemotingCommand response = mock(RemotingCommand.class);\n            when(response.decodeCommandCustomHeader(QueryMessageResponseHeader.class)).thenReturn(responseHeader);\n            when(response.getBody()).thenReturn(getMessageResult());\n            when(response.getCode()).thenReturn(ResponseCode.SUCCESS);\n            callback.operationSucceed(response);\n            return null;\n        }).when(mQClientAPIImpl).queryMessage(anyString(), any(), anyLong(), any(InvokeCallback.class), any());\n        QueryResult actual = mqAdminImpl.queryMessage(defaultTopic, \"keys\", 100, 1L, 50L);\n        assertNotNull(actual);\n        assertEquals(1, actual.getMessageList().size());\n        assertEquals(defaultTopic, actual.getMessageList().get(0).getTopic());\n    }\n\n    @Test\n    public void assertQueryMessageByUniqKey() throws InterruptedException, MQClientException, MQBrokerException, RemotingException {\n        doAnswer(invocation -> {\n            InvokeCallback callback = invocation.getArgument(3);\n            QueryMessageResponseHeader responseHeader = new QueryMessageResponseHeader();\n            responseHeader.setIndexLastUpdatePhyoffset(1L);\n            responseHeader.setIndexLastUpdateTimestamp(System.currentTimeMillis());\n            RemotingCommand response = mock(RemotingCommand.class);\n            when(response.decodeCommandCustomHeader(QueryMessageResponseHeader.class)).thenReturn(responseHeader);\n            when(response.getBody()).thenReturn(getMessageResult());\n            when(response.getCode()).thenReturn(ResponseCode.SUCCESS);\n            callback.operationSucceed(response);\n            return null;\n        }).when(mQClientAPIImpl).queryMessage(anyString(), any(), anyLong(), any(InvokeCallback.class), any());\n        String msgId = buildMsgId();\n        MessageExt actual = mqAdminImpl.queryMessageByUniqKey(defaultTopic, msgId);\n        assertNotNull(actual);\n        assertEquals(msgId, actual.getMsgId());\n        assertEquals(defaultTopic, actual.getTopic());\n        actual = mqAdminImpl.queryMessageByUniqKey(defaultCluster, defaultTopic, msgId);\n        assertNotNull(actual);\n        assertEquals(msgId, actual.getMsgId());\n        assertEquals(defaultTopic, actual.getTopic());\n        QueryResult queryResult = mqAdminImpl.queryMessageByUniqKey(defaultTopic, msgId, 1, 0L, 1L);\n        assertNotNull(queryResult);\n        assertEquals(1, queryResult.getMessageList().size());\n        assertEquals(defaultTopic, queryResult.getMessageList().get(0).getTopic());\n    }\n\n    private String buildMsgId() {\n        MessageExt msgExt = createMessageExt();\n        int storeHostIPLength = (msgExt.getFlag() & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 : 16;\n        int msgIDLength = storeHostIPLength + 4 + 8;\n        ByteBuffer byteBufferMsgId = ByteBuffer.allocate(msgIDLength);\n        return MessageDecoder.createMessageId(byteBufferMsgId, msgExt.getStoreHostBytes(), msgExt.getCommitLogOffset());\n    }\n\n    private TopicRouteData createRouteData() {\n        TopicRouteData result = new TopicRouteData();\n        result.setBrokerDatas(createBrokerData());\n        result.setQueueDatas(createQueueData());\n        return result;\n    }\n\n    private List<BrokerData> createBrokerData() {\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(MixAll.MASTER_ID, defaultBrokerAddr);\n        return Collections.singletonList(new BrokerData(defaultCluster, defaultBroker, brokerAddrs));\n    }\n\n    private List<QueueData> createQueueData() {\n        QueueData queueData = new QueueData();\n        queueData.setPerm(6);\n        queueData.setBrokerName(defaultBroker);\n        queueData.setReadQueueNums(6);\n        queueData.setWriteQueueNums(6);\n        return Collections.singletonList(queueData);\n    }\n\n    private byte[] getMessageResult() throws Exception {\n        byte[] bytes = MessageDecoder.encode(createMessageExt(), false);\n        ByteBuffer byteBuffer = ByteBuffer.allocate(bytes.length);\n        byteBuffer.put(bytes);\n        return byteBuffer.array();\n    }\n\n    private MessageExt createMessageExt() {\n        MessageExt result = new MessageExt();\n        result.setBody(\"body\".getBytes(StandardCharsets.UTF_8));\n        result.setTopic(defaultTopic);\n        result.setBrokerName(defaultBroker);\n        result.putUserProperty(\"key\", \"value\");\n        result.setKeys(\"keys\");\n        SocketAddress bornHost = new InetSocketAddress(\"127.0.0.1\", 12911);\n        SocketAddress storeHost = new InetSocketAddress(\"127.0.0.1\", 10911);\n        result.setBornHost(bornHost);\n        result.setStoreHost(storeHost);\n        return result;\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl;\n\nimport com.alibaba.fastjson2.JSON;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.consumer.AckCallback;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.AckStatus;\nimport org.apache.rocketmq.client.consumer.PopCallback;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PopStatus;\nimport org.apache.rocketmq.client.consumer.PullCallback;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.hook.SendMessageContext;\nimport org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendCallback;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.CheckRocksdbCqWriteResult;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ObjectCreator;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.message.MessageQueueAssignment;\nimport org.apache.rocketmq.common.message.MessageRequestMode;\nimport org.apache.rocketmq.common.namesrv.TopAddressing;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.InvokeCallback;\nimport org.apache.rocketmq.remoting.RemotingClient;\nimport org.apache.rocketmq.remoting.common.HeartbeatV2Result;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.exception.RemotingConnectException;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingClient;\nimport org.apache.rocketmq.remoting.netty.ResponseFuture;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.admin.ConsumeStats;\nimport org.apache.rocketmq.remoting.protocol.admin.TopicOffset;\nimport org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;\nimport org.apache.rocketmq.remoting.protocol.body.AclInfo;\nimport org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;\nimport org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo;\nimport org.apache.rocketmq.remoting.protocol.body.BrokerStatsData;\nimport org.apache.rocketmq.remoting.protocol.body.BrokerStatsItem;\nimport org.apache.rocketmq.remoting.protocol.body.ClusterInfo;\nimport org.apache.rocketmq.remoting.protocol.body.Connection;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeQueueData;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerConnection;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.body.EpochEntryCache;\nimport org.apache.rocketmq.remoting.protocol.body.GetConsumerStatusBody;\nimport org.apache.rocketmq.remoting.protocol.body.GroupList;\nimport org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;\nimport org.apache.rocketmq.remoting.protocol.body.KVTable;\nimport org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.ProducerConnection;\nimport org.apache.rocketmq.remoting.protocol.body.ProducerInfo;\nimport org.apache.rocketmq.remoting.protocol.body.ProducerTableInfo;\nimport org.apache.rocketmq.remoting.protocol.body.QueryAssignmentResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.QueryConsumeQueueResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.QueryConsumeTimeSpanBody;\nimport org.apache.rocketmq.remoting.protocol.body.QueryCorrectionOffsetBody;\nimport org.apache.rocketmq.remoting.protocol.body.QuerySubscriptionResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan;\nimport org.apache.rocketmq.remoting.protocol.body.ResetOffsetBody;\nimport org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.TopicList;\nimport org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.UserInfo;\nimport org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupResponseBody;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetEarliestMsgStoretimeResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMinOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SearchOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateGroupForbiddenRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.AddWritePermOfBrokerResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.GetKVConfigResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.WipeWritePermOfBrokerResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicConfigAndQueueMapping;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingDetail;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo;\nimport org.apache.rocketmq.remoting.protocol.subscription.GroupForbidden;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.assertj.core.api.Assertions;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.ArgumentMatchers;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\nimport org.mockito.stubbing.Answer;\n\nimport java.io.UnsupportedEncodingException;\nimport java.lang.reflect.Field;\nimport java.net.InetSocketAddress;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Properties;\nimport java.util.Set;\nimport java.util.concurrent.CountDownLatch;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertThrows;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.doAnswer;\nimport static org.mockito.Mockito.doNothing;\nimport static org.mockito.Mockito.doThrow;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class MQClientAPIImplTest {\n\n    private MQClientAPIImpl mqClientAPI = new MQClientAPIImpl(new NettyClientConfig(), null, null, new ClientConfig());\n\n    @Mock\n    private RemotingClient remotingClient;\n\n    @Mock\n    private DefaultMQProducerImpl defaultMQProducerImpl;\n\n    @Mock\n    private RemotingCommand response;\n\n    private final String brokerAddr = \"127.0.0.1\";\n\n    private final String brokerName = \"DefaultBroker\";\n\n    private final String clusterName = \"DefaultCluster\";\n\n    private final String group = \"FooBarGroup\";\n\n    private final String topic = \"FooBar\";\n\n    private final Message msg = new Message(\"FooBar\", new byte[]{});\n\n    private final String clientId = \"127.0.0.2@UnitTest\";\n\n    private final String defaultTopic = \"defaultTopic\";\n\n    private final String defaultBrokerAddr = \"127.0.0.1:10911\";\n\n    private final String defaultNsAddr = \"127.0.0.1:9876\";\n\n    private final long defaultTimeout = 3000L;\n\n    @Before\n    public void init() throws Exception {\n        Field field = MQClientAPIImpl.class.getDeclaredField(\"remotingClient\");\n        field.setAccessible(true);\n        field.set(mqClientAPI, remotingClient);\n    }\n\n    @Test\n    public void testSendMessageOneWay_Success() throws RemotingException, InterruptedException, MQBrokerException {\n        doNothing().when(remotingClient).invokeOneway(anyString(), any(RemotingCommand.class), anyLong());\n        SendResult sendResult = mqClientAPI.sendMessage(brokerAddr, brokerName, msg, new SendMessageRequestHeader(),\n            3 * 1000, CommunicationMode.ONEWAY, new SendMessageContext(), defaultMQProducerImpl);\n        assertThat(sendResult).isNull();\n    }\n\n    @Test\n    public void testSendMessageOneWay_WithException() throws RemotingException, InterruptedException, MQBrokerException {\n        doThrow(new RemotingTimeoutException(\"Remoting Exception in Test\")).when(remotingClient).invokeOneway(anyString(), any(RemotingCommand.class), anyLong());\n        try {\n            mqClientAPI.sendMessage(brokerAddr, brokerName, msg, new SendMessageRequestHeader(),\n                3 * 1000, CommunicationMode.ONEWAY, new SendMessageContext(), defaultMQProducerImpl);\n            failBecauseExceptionWasNotThrown(RemotingException.class);\n        } catch (RemotingException e) {\n            assertThat(e).hasMessage(\"Remoting Exception in Test\");\n        }\n\n        doThrow(new InterruptedException(\"Interrupted Exception in Test\")).when(remotingClient).invokeOneway(anyString(), any(RemotingCommand.class), anyLong());\n        try {\n            mqClientAPI.sendMessage(brokerAddr, brokerName, msg, new SendMessageRequestHeader(),\n                3 * 1000, CommunicationMode.ONEWAY, new SendMessageContext(), defaultMQProducerImpl);\n            failBecauseExceptionWasNotThrown(InterruptedException.class);\n        } catch (InterruptedException e) {\n            assertThat(e).hasMessage(\"Interrupted Exception in Test\");\n        }\n    }\n\n    @Test\n    public void testSendMessageSync_Success() throws InterruptedException, RemotingException, MQBrokerException {\n        doAnswer(mock -> {\n            RemotingCommand request = mock.getArgument(1);\n            return createSendMessageSuccessResponse(request);\n        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());\n\n        SendMessageRequestHeader requestHeader = createSendMessageRequestHeader();\n\n        SendResult sendResult = mqClientAPI.sendMessage(brokerAddr, brokerName, msg, requestHeader,\n            3 * 1000, CommunicationMode.SYNC, new SendMessageContext(), defaultMQProducerImpl);\n\n        assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n        assertThat(sendResult.getOffsetMsgId()).isEqualTo(\"123\");\n        assertThat(sendResult.getQueueOffset()).isEqualTo(123L);\n        assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(1);\n    }\n\n    @Test\n    public void testSendMessageSync_WithException() throws InterruptedException, RemotingException {\n        doAnswer(mock -> {\n            RemotingCommand request = mock.getArgument(1);\n            RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setOpaque(request.getOpaque());\n            response.setRemark(\"Broker is broken.\");\n            return response;\n        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());\n\n        SendMessageRequestHeader requestHeader = createSendMessageRequestHeader();\n\n        try {\n            mqClientAPI.sendMessage(brokerAddr, brokerName, msg, requestHeader,\n                3 * 1000, CommunicationMode.SYNC, new SendMessageContext(), defaultMQProducerImpl);\n            failBecauseExceptionWasNotThrown(MQBrokerException.class);\n        } catch (MQBrokerException e) {\n            assertThat(e).hasMessageContaining(\"Broker is broken.\");\n        }\n    }\n\n    @Test\n    public void testSendMessageAsync_Success() throws RemotingException, InterruptedException, MQBrokerException {\n        doNothing().when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));\n        SendResult sendResult = mqClientAPI.sendMessage(brokerAddr, brokerName, msg, new SendMessageRequestHeader(),\n            3 * 1000, CommunicationMode.ASYNC, new SendMessageContext(), defaultMQProducerImpl);\n        assertThat(sendResult).isNull();\n\n        doAnswer(mock -> {\n            InvokeCallback callback = mock.getArgument(3);\n            RemotingCommand request = mock.getArgument(1);\n            ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);\n            responseFuture.setResponseCommand(createSendMessageSuccessResponse(request));\n            callback.operationSucceed(responseFuture.getResponseCommand());\n            return null;\n        }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));\n        SendMessageContext sendMessageContext = new SendMessageContext();\n        sendMessageContext.setProducer(new DefaultMQProducerImpl(new DefaultMQProducer()));\n        mqClientAPI.sendMessage(brokerAddr, brokerName, msg, new SendMessageRequestHeader(), 3 * 1000, CommunicationMode.ASYNC,\n            new SendCallback() {\n                @Override\n                public void onSuccess(SendResult sendResult) {\n                    assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n                    assertThat(sendResult.getOffsetMsgId()).isEqualTo(\"123\");\n                    assertThat(sendResult.getQueueOffset()).isEqualTo(123L);\n                    assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(1);\n                }\n\n                @Override\n                public void onException(Throwable e) {\n                }\n            },\n            null, null, 0, sendMessageContext, defaultMQProducerImpl);\n    }\n\n    @Test\n    public void testSendMessageAsync_WithException() throws RemotingException, InterruptedException, MQBrokerException {\n        doThrow(new RemotingTimeoutException(\"Remoting Exception in Test\")).when(remotingClient)\n            .invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));\n        SendMessageContext sendMessageContext = new SendMessageContext();\n        sendMessageContext.setProducer(new DefaultMQProducerImpl(new DefaultMQProducer()));\n        mqClientAPI.sendMessage(brokerAddr, brokerName, msg, new SendMessageRequestHeader(), 3 * 1000, CommunicationMode.ASYNC,\n                new SendCallback() {\n                    @Override\n                    public void onSuccess(SendResult sendResult) {\n                    }\n                    @Override\n                    public void onException(Throwable e) {\n                        assertThat(e).hasMessage(\"Remoting Exception in Test\");\n                    }\n                }, null, null, 0, sendMessageContext, defaultMQProducerImpl);\n\n        doThrow(new InterruptedException(\"Interrupted Exception in Test\")).when(remotingClient)\n            .invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));\n        mqClientAPI.sendMessage(brokerAddr, brokerName, msg, new SendMessageRequestHeader(), 3 * 1000, CommunicationMode.ASYNC,\n                new SendCallback() {\n                    @Override\n                    public void onSuccess(SendResult sendResult) {\n                    }\n                    @Override\n                    public void onException(Throwable e) {\n                        assertThat(e).hasMessage(\"Interrupted Exception in Test\");\n                    }\n                }, null, null, 0, sendMessageContext, defaultMQProducerImpl);\n    }\n\n    @Test\n    public void testResumeCheckHalfMessage_WithException() throws RemotingException, InterruptedException {\n        doAnswer(mock -> {\n            RemotingCommand request = mock.getArgument(1);\n            RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setOpaque(request.getOpaque());\n            response.setRemark(\"Put message back to RMQ_SYS_TRANS_HALF_TOPIC failed.\");\n            return response;\n        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());\n\n        boolean result = mqClientAPI.resumeCheckHalfMessage(brokerAddr, \"topic,\", \"test\", 3000);\n        assertThat(result).isEqualTo(false);\n    }\n\n    @Test\n    public void testResumeCheckHalfMessage_Success() throws InterruptedException, RemotingException {\n        doAnswer(mock -> {\n            RemotingCommand request = mock.getArgument(1);\n            return createResumeSuccessResponse(request);\n        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());\n\n        boolean result = mqClientAPI.resumeCheckHalfMessage(brokerAddr, \"topic\", \"test\", 3000);\n\n        assertThat(result).isEqualTo(true);\n    }\n\n    @Test\n    public void testSendMessageTypeofReply() throws Exception {\n        doAnswer(mock -> {\n            InvokeCallback callback = mock.getArgument(3);\n            RemotingCommand request = mock.getArgument(1);\n            ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);\n            responseFuture.setResponseCommand(createSendMessageSuccessResponse(request));\n            callback.operationSucceed(responseFuture.getResponseCommand());\n            return null;\n        }).when(remotingClient).invokeAsync(ArgumentMatchers.anyString(), ArgumentMatchers.any(RemotingCommand.class), ArgumentMatchers.anyLong(), ArgumentMatchers.any(InvokeCallback.class));\n        SendMessageContext sendMessageContext = new SendMessageContext();\n        sendMessageContext.setProducer(new DefaultMQProducerImpl(new DefaultMQProducer()));\n        msg.getProperties().put(\"MSG_TYPE\", \"reply\");\n        mqClientAPI.sendMessage(brokerAddr, brokerName, msg, new SendMessageRequestHeader(), 3 * 1000, CommunicationMode.ASYNC,\n            new SendCallback() {\n                @Override\n                public void onSuccess(SendResult sendResult) {\n                    assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n                    assertThat(sendResult.getOffsetMsgId()).isEqualTo(\"123\");\n                    assertThat(sendResult.getQueueOffset()).isEqualTo(123L);\n                    assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(1);\n                }\n\n                @Override\n                public void onException(Throwable e) {\n                }\n            }, null, null, 0, sendMessageContext, defaultMQProducerImpl);\n    }\n\n    @Test\n    public void testQueryAssignment_Success() throws Exception {\n        doAnswer((Answer<RemotingCommand>) mock -> {\n            RemotingCommand request = mock.getArgument(1);\n\n            RemotingCommand response = RemotingCommand.createResponseCommand(null);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setOpaque(request.getOpaque());\n            QueryAssignmentResponseBody b = new QueryAssignmentResponseBody();\n            b.setMessageQueueAssignments(Collections.singleton(new MessageQueueAssignment()));\n            response.setBody(b.encode());\n            return response;\n        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());\n        Set<MessageQueueAssignment> assignments = mqClientAPI.queryAssignment(brokerAddr, topic, group, clientId, null, MessageModel.CLUSTERING, 10 * 1000);\n        assertThat(assignments).size().isEqualTo(1);\n    }\n\n    @Test\n    public void testPopMessageAsync_Success() throws Exception {\n        final long popTime = System.currentTimeMillis();\n        final int invisibleTime = 10 * 1000;\n        doAnswer((Answer<Void>) mock -> {\n            InvokeCallback callback = mock.getArgument(3);\n            RemotingCommand request = mock.getArgument(1);\n            ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);\n            RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setOpaque(request.getOpaque());\n\n            PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader();\n            responseHeader.setInvisibleTime(invisibleTime);\n            responseHeader.setPopTime(popTime);\n            responseHeader.setReviveQid(0);\n            responseHeader.setRestNum(1);\n            StringBuilder startOffsetInfo = new StringBuilder(64);\n            ExtraInfoUtil.buildStartOffsetInfo(startOffsetInfo, topic, 0, 0L);\n            responseHeader.setStartOffsetInfo(startOffsetInfo.toString());\n            StringBuilder msgOffsetInfo = new StringBuilder(64);\n            ExtraInfoUtil.buildMsgOffsetInfo(msgOffsetInfo, topic, 0, Collections.singletonList(0L));\n            responseHeader.setMsgOffsetInfo(msgOffsetInfo.toString());\n            response.setRemark(\"FOUND\");\n            response.makeCustomHeaderToNet();\n\n            MessageExt message = new MessageExt();\n            message.setQueueId(0);\n            message.setFlag(12);\n            message.setQueueOffset(0L);\n            message.setCommitLogOffset(100L);\n            message.setSysFlag(0);\n            message.setBornTimestamp(System.currentTimeMillis());\n            message.setBornHost(new InetSocketAddress(\"127.0.0.1\", 10));\n            message.setStoreTimestamp(System.currentTimeMillis());\n            message.setStoreHost(new InetSocketAddress(\"127.0.0.1\", 11));\n            message.setBody(\"body\".getBytes());\n            message.setTopic(topic);\n            message.putUserProperty(\"key\", \"value\");\n            response.setBody(MessageDecoder.encode(message, false));\n            responseFuture.setResponseCommand(response);\n            callback.operationSucceed(responseFuture.getResponseCommand());\n            return null;\n        }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));\n        final CountDownLatch done = new CountDownLatch(1);\n        mqClientAPI.popMessageAsync(brokerName, brokerAddr, new PopMessageRequestHeader(), 10 * 1000, new PopCallback() {\n            @Override\n            public void onSuccess(PopResult popResult) {\n                assertThat(popResult.getPopStatus()).isEqualTo(PopStatus.FOUND);\n                assertThat(popResult.getRestNum()).isEqualTo(1);\n                assertThat(popResult.getInvisibleTime()).isEqualTo(invisibleTime);\n                assertThat(popResult.getPopTime()).isEqualTo(popTime);\n                assertThat(popResult.getMsgFoundList()).size().isEqualTo(1);\n                done.countDown();\n            }\n\n            @Override\n            public void onException(Throwable e) {\n                Assertions.fail(\"want no exception but got one\", e);\n                done.countDown();\n            }\n        });\n        done.await();\n    }\n\n    @Test\n    public void testPopLmqMessage_async() throws Exception {\n        final long popTime = System.currentTimeMillis();\n        final int invisibleTime = 10 * 1000;\n        final String lmqTopic = MixAll.LMQ_PREFIX + \"lmq1\";\n        doAnswer((Answer<Void>) mock -> {\n            InvokeCallback callback = mock.getArgument(3);\n            RemotingCommand request = mock.getArgument(1);\n            ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);\n            RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setOpaque(request.getOpaque());\n\n            PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader();\n            responseHeader.setInvisibleTime(invisibleTime);\n            responseHeader.setPopTime(popTime);\n            responseHeader.setReviveQid(0);\n            responseHeader.setRestNum(1);\n            StringBuilder startOffsetInfo = new StringBuilder(64);\n            ExtraInfoUtil.buildStartOffsetInfo(startOffsetInfo, topic, 0, 0L);\n            responseHeader.setStartOffsetInfo(startOffsetInfo.toString());\n            StringBuilder msgOffsetInfo = new StringBuilder(64);\n            ExtraInfoUtil.buildMsgOffsetInfo(msgOffsetInfo, topic, 0, Collections.singletonList(0L));\n            responseHeader.setMsgOffsetInfo(msgOffsetInfo.toString());\n            response.setRemark(\"FOUND\");\n            response.makeCustomHeaderToNet();\n\n            MessageExt message = new MessageExt();\n            message.setQueueId(3);\n            message.setFlag(0);\n            message.setQueueOffset(5L);\n            message.setCommitLogOffset(11111L);\n            message.setSysFlag(0);\n            message.setBornTimestamp(System.currentTimeMillis());\n            message.setBornHost(new InetSocketAddress(\"127.0.0.1\", 10));\n            message.setStoreTimestamp(System.currentTimeMillis());\n            message.setStoreHost(new InetSocketAddress(\"127.0.0.1\", 11));\n            message.setBody(\"body\".getBytes());\n            message.setTopic(topic);\n            message.putUserProperty(\"key\", \"value\");\n            MessageAccessor.putProperty(message, MessageConst.PROPERTY_INNER_MULTI_DISPATCH, lmqTopic);\n            message.getProperties().put(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, String.valueOf(0));\n            response.setBody(MessageDecoder.encode(message, false));\n            responseFuture.setResponseCommand(response);\n            callback.operationSucceed(responseFuture.getResponseCommand());\n            return null;\n        }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));\n        final CountDownLatch done = new CountDownLatch(1);\n        final PopMessageRequestHeader requestHeader = new PopMessageRequestHeader();\n        requestHeader.setTopic(lmqTopic);\n        mqClientAPI.popMessageAsync(brokerName, brokerAddr, requestHeader, 10 * 1000, new PopCallback() {\n            @Override\n            public void onSuccess(PopResult popResult) {\n                assertThat(popResult.getPopStatus()).isEqualTo(PopStatus.FOUND);\n                assertThat(popResult.getRestNum()).isEqualTo(1);\n                assertThat(popResult.getInvisibleTime()).isEqualTo(invisibleTime);\n                assertThat(popResult.getPopTime()).isEqualTo(popTime);\n                assertThat(popResult.getMsgFoundList()).size().isEqualTo(1);\n                assertThat(popResult.getMsgFoundList().get(0).getTopic()).isEqualTo(lmqTopic);\n                assertThat(popResult.getMsgFoundList().get(0).getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH))\n                    .isEqualTo(lmqTopic);\n                done.countDown();\n            }\n\n            @Override\n            public void onException(Throwable e) {\n                Assertions.fail(\"want no exception but got one\", e);\n                done.countDown();\n            }\n        });\n        done.await();\n    }\n\n    @Test\n    public void testPopMultiLmqMessage_async() throws Exception {\n        final long popTime = System.currentTimeMillis();\n        final int invisibleTime = 10 * 1000;\n        final String lmqTopic = MixAll.LMQ_PREFIX + \"lmq1\";\n        final String lmqTopic2 = MixAll.LMQ_PREFIX + \"lmq2\";\n        final String multiDispatch = String.join(MixAll.LMQ_DISPATCH_SEPARATOR, lmqTopic, lmqTopic2);\n        final String multiOffset = String.join(MixAll.LMQ_DISPATCH_SEPARATOR, \"0\", \"0\");\n        doAnswer((Answer<Void>) mock -> {\n            InvokeCallback callback = mock.getArgument(3);\n            RemotingCommand request = mock.getArgument(1);\n            ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);\n            RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setOpaque(request.getOpaque());\n\n            PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader();\n            responseHeader.setInvisibleTime(invisibleTime);\n            responseHeader.setPopTime(popTime);\n            responseHeader.setReviveQid(0);\n            responseHeader.setRestNum(1);\n            StringBuilder startOffsetInfo = new StringBuilder(64);\n            ExtraInfoUtil.buildStartOffsetInfo(startOffsetInfo, topic, 0, 0L);\n            responseHeader.setStartOffsetInfo(startOffsetInfo.toString());\n            StringBuilder msgOffsetInfo = new StringBuilder(64);\n            ExtraInfoUtil.buildMsgOffsetInfo(msgOffsetInfo, topic, 0, Collections.singletonList(0L));\n            responseHeader.setMsgOffsetInfo(msgOffsetInfo.toString());\n            response.setRemark(\"FOUND\");\n            response.makeCustomHeaderToNet();\n\n            MessageExt message = new MessageExt();\n            message.setQueueId(0);\n            message.setFlag(0);\n            message.setQueueOffset(10L);\n            message.setCommitLogOffset(10000L);\n            message.setSysFlag(0);\n            message.setBornTimestamp(System.currentTimeMillis());\n            message.setBornHost(new InetSocketAddress(\"127.0.0.1\", 10));\n            message.setStoreTimestamp(System.currentTimeMillis());\n            message.setStoreHost(new InetSocketAddress(\"127.0.0.1\", 11));\n            message.setBody(\"body\".getBytes());\n            message.setTopic(topic);\n            MessageAccessor.putProperty(message, MessageConst.PROPERTY_INNER_MULTI_DISPATCH, multiDispatch);\n            MessageAccessor.putProperty(message, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, multiOffset);\n            response.setBody(MessageDecoder.encode(message, false));\n            responseFuture.setResponseCommand(response);\n            callback.operationSucceed(responseFuture.getResponseCommand());\n            return null;\n        }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));\n        final CountDownLatch done = new CountDownLatch(1);\n        final PopMessageRequestHeader requestHeader = new PopMessageRequestHeader();\n        requestHeader.setTopic(lmqTopic);\n        mqClientAPI.popMessageAsync(brokerName, brokerAddr, requestHeader, 10 * 1000, new PopCallback() {\n            @Override\n            public void onSuccess(PopResult popResult) {\n                assertThat(popResult.getPopStatus()).isEqualTo(PopStatus.FOUND);\n                assertThat(popResult.getRestNum()).isEqualTo(1);\n                assertThat(popResult.getInvisibleTime()).isEqualTo(invisibleTime);\n                assertThat(popResult.getPopTime()).isEqualTo(popTime);\n                assertThat(popResult.getMsgFoundList()).size().isEqualTo(1);\n                assertThat(popResult.getMsgFoundList().get(0).getTopic()).isEqualTo(lmqTopic);\n                assertThat(popResult.getMsgFoundList().get(0).getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH))\n                    .isEqualTo(multiDispatch);\n                assertThat(popResult.getMsgFoundList().get(0).getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET))\n                    .isEqualTo(multiOffset);\n                done.countDown();\n            }\n\n            @Override\n            public void onException(Throwable e) {\n                Assertions.fail(\"want no exception but got one\", e);\n                done.countDown();\n            }\n        });\n        done.await();\n    }\n\n    @Test\n    public void testAckMessageAsync_Success() throws Exception {\n        doAnswer((Answer<Void>) mock -> {\n            InvokeCallback callback = mock.getArgument(3);\n            RemotingCommand request = mock.getArgument(1);\n            ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);\n            RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, null);\n            response.setOpaque(request.getOpaque());\n            response.setCode(ResponseCode.SUCCESS);\n            responseFuture.setResponseCommand(response);\n            callback.operationSucceed(responseFuture.getResponseCommand());\n            return null;\n        }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));\n\n        final CountDownLatch done = new CountDownLatch(1);\n        mqClientAPI.ackMessageAsync(brokerAddr, 10 * 1000, new AckCallback() {\n            @Override\n            public void onSuccess(AckResult ackResult) {\n                assertThat(ackResult.getStatus()).isEqualTo(AckStatus.OK);\n                done.countDown();\n            }\n\n            @Override\n            public void onException(Throwable e) {\n                Assertions.fail(\"want no exception but got one\", e);\n                done.countDown();\n            }\n        }, new AckMessageRequestHeader());\n        done.await();\n    }\n\n    @Test\n    public void testChangeInvisibleTimeAsync_Success() throws Exception {\n        doAnswer((Answer<Void>) mock -> {\n            InvokeCallback callback = mock.getArgument(3);\n            RemotingCommand request = mock.getArgument(1);\n            ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);\n            RemotingCommand response = RemotingCommand.createResponseCommand(ChangeInvisibleTimeResponseHeader.class);\n            response.setOpaque(request.getOpaque());\n            response.setCode(ResponseCode.SUCCESS);\n            ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.readCustomHeader();\n            responseHeader.setPopTime(System.currentTimeMillis());\n            responseHeader.setInvisibleTime(10 * 1000L);\n            responseFuture.setResponseCommand(response);\n            callback.operationSucceed(responseFuture.getResponseCommand());\n            return null;\n        }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));\n\n        final CountDownLatch done = new CountDownLatch(1);\n        ChangeInvisibleTimeRequestHeader requestHeader = new ChangeInvisibleTimeRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setQueueId(0);\n        requestHeader.setOffset(0L);\n        requestHeader.setInvisibleTime(10 * 1000L);\n        mqClientAPI.changeInvisibleTimeAsync(brokerName, brokerAddr, requestHeader, 10 * 1000, new AckCallback() {\n            @Override\n            public void onSuccess(AckResult ackResult) {\n                assertThat(ackResult.getStatus()).isEqualTo(AckStatus.OK);\n                done.countDown();\n            }\n\n            @Override\n            public void onException(Throwable e) {\n                Assertions.fail(\"want no exception but got one\", e);\n                done.countDown();\n            }\n        });\n        done.await();\n    }\n\n    @Test\n    public void testSetMessageRequestMode_Success() throws Exception {\n        doAnswer((Answer<RemotingCommand>) mock -> {\n            RemotingCommand request = mock.getArgument(1);\n\n            RemotingCommand response = RemotingCommand.createResponseCommand(null);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setOpaque(request.getOpaque());\n            return response;\n        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());\n\n        mqClientAPI.setMessageRequestMode(brokerAddr, topic, group, MessageRequestMode.POP, 8, 10 * 1000L);\n    }\n\n    @Test\n    public void testCreateSubscriptionGroup_Success() throws Exception {\n        doAnswer((Answer<RemotingCommand>) mock -> {\n            RemotingCommand request = mock.getArgument(1);\n\n            RemotingCommand response = RemotingCommand.createResponseCommand(null);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setOpaque(request.getOpaque());\n            return response;\n        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());\n\n        mqClientAPI.createSubscriptionGroup(brokerAddr, new SubscriptionGroupConfig(), 10000);\n    }\n\n    @Test\n    public void testCreateTopic_Success() throws Exception {\n        doAnswer((Answer<RemotingCommand>) mock -> {\n            RemotingCommand request = mock.getArgument(1);\n\n            RemotingCommand response = RemotingCommand.createResponseCommand(null);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setOpaque(request.getOpaque());\n            return response;\n        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());\n\n        mqClientAPI.createTopic(brokerAddr, topic, new TopicConfig(), 10000);\n    }\n\n    @Test\n    public void testViewMessage() throws Exception {\n        doAnswer((Answer<RemotingCommand>) mock -> {\n            RemotingCommand request = mock.getArgument(1);\n\n            RemotingCommand response = RemotingCommand.createResponseCommand(null);\n            MessageExt message = new MessageExt();\n            message.setQueueId(0);\n            message.setFlag(12);\n            message.setQueueOffset(0L);\n            message.setCommitLogOffset(100L);\n            message.setSysFlag(0);\n            message.setBornTimestamp(System.currentTimeMillis());\n            message.setBornHost(new InetSocketAddress(\"127.0.0.1\", 10));\n            message.setStoreTimestamp(System.currentTimeMillis());\n            message.setStoreHost(new InetSocketAddress(\"127.0.0.1\", 11));\n            message.setBody(\"body\".getBytes());\n            message.setTopic(topic);\n            message.putUserProperty(\"key\", \"value\");\n            response.setBody(MessageDecoder.encode(message, false));\n            response.makeCustomHeaderToNet();\n            response.setCode(ResponseCode.SUCCESS);\n            response.setOpaque(request.getOpaque());\n            return response;\n        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());\n\n        MessageExt messageExt = mqClientAPI.viewMessage(brokerAddr, \"topic\", 100L, 10000);\n        assertThat(messageExt.getTopic()).isEqualTo(topic);\n    }\n\n    @Test\n    public void testSearchOffset() throws Exception {\n        doAnswer((Answer<RemotingCommand>) mock -> {\n            RemotingCommand request = mock.getArgument(1);\n\n            final RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class);\n            final SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader();\n            responseHeader.setOffset(100L);\n            response.makeCustomHeaderToNet();\n            response.setCode(ResponseCode.SUCCESS);\n            response.setOpaque(request.getOpaque());\n            return response;\n        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());\n\n        long offset = mqClientAPI.searchOffset(brokerAddr, topic, 0, System.currentTimeMillis() - 1000, 10000);\n        assertThat(offset).isEqualTo(100L);\n    }\n\n    @Test\n    public void testGetMaxOffset() throws Exception {\n        doAnswer((Answer<RemotingCommand>) mock -> {\n            RemotingCommand request = mock.getArgument(1);\n\n            final RemotingCommand response = RemotingCommand.createResponseCommand(GetMaxOffsetResponseHeader.class);\n            final GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.readCustomHeader();\n            responseHeader.setOffset(100L);\n            response.makeCustomHeaderToNet();\n            response.setCode(ResponseCode.SUCCESS);\n            response.setOpaque(request.getOpaque());\n            return response;\n        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());\n\n        long offset = mqClientAPI.getMaxOffset(brokerAddr, new MessageQueue(topic, brokerName, 0), 10000);\n        assertThat(offset).isEqualTo(100L);\n    }\n\n    @Test\n    public void testGetMinOffset() throws Exception {\n        doAnswer((Answer<RemotingCommand>) mock -> {\n            RemotingCommand request = mock.getArgument(1);\n\n            final RemotingCommand response = RemotingCommand.createResponseCommand(GetMinOffsetResponseHeader.class);\n            final GetMinOffsetResponseHeader responseHeader = (GetMinOffsetResponseHeader) response.readCustomHeader();\n            responseHeader.setOffset(100L);\n            response.makeCustomHeaderToNet();\n            response.setCode(ResponseCode.SUCCESS);\n            response.setOpaque(request.getOpaque());\n            return response;\n        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());\n\n        long offset = mqClientAPI.getMinOffset(brokerAddr, new MessageQueue(topic, brokerName, 0), 10000);\n        assertThat(offset).isEqualTo(100L);\n    }\n\n    @Test\n    public void testGetEarliestMsgStoretime() throws Exception {\n        doAnswer((Answer<RemotingCommand>) mock -> {\n            RemotingCommand request = mock.getArgument(1);\n\n            final RemotingCommand response = RemotingCommand.createResponseCommand(GetEarliestMsgStoretimeResponseHeader.class);\n            final GetEarliestMsgStoretimeResponseHeader responseHeader = (GetEarliestMsgStoretimeResponseHeader) response.readCustomHeader();\n            responseHeader.setTimestamp(100L);\n            response.makeCustomHeaderToNet();\n            response.setCode(ResponseCode.SUCCESS);\n            response.setOpaque(request.getOpaque());\n            return response;\n        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());\n\n        long t = mqClientAPI.getEarliestMsgStoretime(brokerAddr, new MessageQueue(topic, brokerName, 0), 10000);\n        assertThat(t).isEqualTo(100L);\n    }\n\n    @Test\n    public void testQueryConsumerOffset() throws Exception {\n        doAnswer((Answer<RemotingCommand>) mock -> {\n            RemotingCommand request = mock.getArgument(1);\n\n            final RemotingCommand response =\n                    RemotingCommand.createResponseCommand(QueryConsumerOffsetResponseHeader.class);\n            final QueryConsumerOffsetResponseHeader responseHeader =\n                    (QueryConsumerOffsetResponseHeader) response.readCustomHeader();\n            responseHeader.setOffset(100L);\n            response.makeCustomHeaderToNet();\n            response.setCode(ResponseCode.SUCCESS);\n            response.setOpaque(request.getOpaque());\n            return response;\n        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());\n\n        long t = mqClientAPI.queryConsumerOffset(brokerAddr, new QueryConsumerOffsetRequestHeader(), 1000);\n        assertThat(t).isEqualTo(100L);\n    }\n\n    @Test\n    public void testUpdateConsumerOffset() throws Exception {\n        doAnswer((Answer<RemotingCommand>) mock -> {\n            RemotingCommand request = mock.getArgument(1);\n\n            final RemotingCommand response =\n                    RemotingCommand.createResponseCommand(UpdateConsumerOffsetResponseHeader.class);\n            response.makeCustomHeaderToNet();\n            response.setCode(ResponseCode.SUCCESS);\n            response.setOpaque(request.getOpaque());\n            return response;\n        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());\n\n        mqClientAPI.updateConsumerOffset(brokerAddr, new UpdateConsumerOffsetRequestHeader(), 1000);\n    }\n\n    @Test\n    public void testGetConsumerIdListByGroup() throws Exception {\n        doAnswer((Answer<RemotingCommand>) mock -> {\n            RemotingCommand request = mock.getArgument(1);\n\n            final RemotingCommand response =\n                    RemotingCommand.createResponseCommand(GetConsumerListByGroupResponseHeader.class);\n            GetConsumerListByGroupResponseBody body = new GetConsumerListByGroupResponseBody();\n            body.setConsumerIdList(Collections.singletonList(\"consumer1\"));\n            response.setBody(body.encode());\n            response.makeCustomHeaderToNet();\n            response.setCode(ResponseCode.SUCCESS);\n            response.setOpaque(request.getOpaque());\n            return response;\n        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());\n        List<String> consumerIdList = mqClientAPI.getConsumerIdListByGroup(brokerAddr, group, 10000);\n        assertThat(consumerIdList).size().isGreaterThan(0);\n    }\n\n    private RemotingCommand createResumeSuccessResponse(RemotingCommand request) {\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setOpaque(request.getOpaque());\n        return response;\n    }\n\n    private RemotingCommand createSendMessageSuccessResponse(RemotingCommand request) {\n        RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setOpaque(request.getOpaque());\n\n        SendMessageResponseHeader responseHeader = (SendMessageResponseHeader) response.readCustomHeader();\n        responseHeader.setMsgId(\"123\");\n        responseHeader.setQueueId(1);\n        responseHeader.setQueueOffset(123L);\n\n        response.addExtField(MessageConst.PROPERTY_MSG_REGION, \"RegionHZ\");\n        response.addExtField(MessageConst.PROPERTY_TRACE_SWITCH, \"true\");\n        response.addExtField(\"queueId\", String.valueOf(responseHeader.getQueueId()));\n        response.addExtField(\"msgId\", responseHeader.getMsgId());\n        response.addExtField(\"queueOffset\", String.valueOf(responseHeader.getQueueOffset()));\n        return response;\n    }\n\n    private RemotingCommand createSuccessResponse4UpdateAclConfig(RemotingCommand request) {\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setOpaque(request.getOpaque());\n        response.markResponseType();\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand createSuccessResponse4DeleteAclConfig(RemotingCommand request) {\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setOpaque(request.getOpaque());\n        response.markResponseType();\n        response.setRemark(null);\n        return response;\n    }\n\n    private SendMessageRequestHeader createSendMessageRequestHeader() {\n        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();\n        requestHeader.setBornTimestamp(System.currentTimeMillis());\n        requestHeader.setTopic(topic);\n        requestHeader.setProducerGroup(group);\n        requestHeader.setQueueId(1);\n        requestHeader.setMaxReconsumeTimes(10);\n        return requestHeader;\n    }\n\n    @Test\n    public void testAddWritePermOfBroker() throws Exception {\n        doAnswer(invocationOnMock -> {\n            RemotingCommand request = invocationOnMock.getArgument(1);\n            if (request.getCode() != RequestCode.ADD_WRITE_PERM_OF_BROKER) {\n                return null;\n            }\n\n            RemotingCommand response = RemotingCommand.createResponseCommand(AddWritePermOfBrokerResponseHeader.class);\n            AddWritePermOfBrokerResponseHeader responseHeader = (AddWritePermOfBrokerResponseHeader) response.readCustomHeader();\n            response.setCode(ResponseCode.SUCCESS);\n            responseHeader.setAddTopicCount(7);\n            response.addExtField(\"addTopicCount\", String.valueOf(responseHeader.getAddTopicCount()));\n            return response;\n        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());\n\n        int topicCnt = mqClientAPI.addWritePermOfBroker(\"127.0.0.1\", \"default-broker\", 1000);\n        assertThat(topicCnt).isEqualTo(7);\n    }\n\n    @Test\n    public void testCreateTopicList_Success() throws RemotingException, InterruptedException, MQClientException {\n        doAnswer((Answer<RemotingCommand>) mock -> {\n            RemotingCommand request = mock.getArgument(1);\n\n            RemotingCommand response = RemotingCommand.createResponseCommand(null);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setOpaque(request.getOpaque());\n            return response;\n        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());\n\n        final List<TopicConfig> topicConfigList = new LinkedList<>();\n        for (int i = 0; i < 16; i++) {\n            TopicConfig topicConfig = new TopicConfig();\n            topicConfig.setTopicName(\"Topic\" + i);\n            topicConfigList.add(topicConfig);\n        }\n\n        mqClientAPI.createTopicList(brokerAddr, topicConfigList, 10000);\n    }\n\n    @Test\n    public void assertFetchNameServerAddr() throws NoSuchFieldException, IllegalAccessException {\n        setTopAddressing();\n        assertEquals(defaultNsAddr, mqClientAPI.fetchNameServerAddr());\n    }\n\n    @Test\n    public void assertOnNameServerAddressChange() {\n        assertEquals(defaultNsAddr, mqClientAPI.onNameServerAddressChange(defaultNsAddr));\n    }\n\n    @Test\n    public void assertPullMessage() throws MQBrokerException, RemotingException, InterruptedException {\n        PullMessageRequestHeader requestHeader = mock(PullMessageRequestHeader.class);\n        mockInvokeSync();\n        PullCallback callback = mock(PullCallback.class);\n        PullMessageResponseHeader responseHeader = mock(PullMessageResponseHeader.class);\n        setResponseHeader(responseHeader);\n        when(responseHeader.getNextBeginOffset()).thenReturn(1L);\n        when(responseHeader.getMinOffset()).thenReturn(1L);\n        when(responseHeader.getMaxOffset()).thenReturn(10L);\n        when(responseHeader.getSuggestWhichBrokerId()).thenReturn(MixAll.MASTER_ID);\n        PullResult actual = mqClientAPI.pullMessage(defaultBrokerAddr, requestHeader, defaultTimeout, CommunicationMode.SYNC, callback);\n        assertNotNull(actual);\n        assertEquals(1L, actual.getNextBeginOffset());\n        assertEquals(1L, actual.getMinOffset());\n        assertEquals(10L, actual.getMaxOffset());\n        assertEquals(PullStatus.FOUND, actual.getPullStatus());\n        assertNull(actual.getMsgFoundList());\n    }\n\n    @Test\n    public void testBatchAckMessageAsync() throws MQBrokerException, RemotingException, InterruptedException {\n        AckCallback callback = mock(AckCallback.class);\n        List<String> extraInfoList = new ArrayList<>();\n        extraInfoList.add(String.format(\"%s %s %s %s %s %s %d %d\", \"1\", \"2\", \"3\", \"4\", \"5\", brokerName, 7, 8));\n        mqClientAPI.batchAckMessageAsync(defaultBrokerAddr, defaultTimeout, callback, defaultTopic, \"\", extraInfoList);\n    }\n\n    @Test\n    public void assertSearchOffset() throws MQBrokerException, RemotingException, InterruptedException {\n        mockInvokeSync();\n        SearchOffsetResponseHeader responseHeader = mock(SearchOffsetResponseHeader.class);\n        when(responseHeader.getOffset()).thenReturn(1L);\n        setResponseHeader(responseHeader);\n        assertEquals(1L, mqClientAPI.searchOffset(defaultBrokerAddr, new MessageQueue(), System.currentTimeMillis(), defaultTimeout));\n    }\n\n    @Test\n    public void testUpdateConsumerOffsetOneway() throws RemotingException, InterruptedException {\n        UpdateConsumerOffsetRequestHeader requestHeader = mock(UpdateConsumerOffsetRequestHeader.class);\n        mqClientAPI.updateConsumerOffsetOneway(defaultBrokerAddr, requestHeader, defaultTimeout);\n    }\n\n    @Test\n    public void assertSendHeartbeat() throws MQBrokerException, RemotingException, InterruptedException {\n        mockInvokeSync();\n        HeartbeatData heartbeatData = new HeartbeatData();\n        assertEquals(1, mqClientAPI.sendHeartbeat(defaultBrokerAddr, heartbeatData, defaultTimeout));\n    }\n\n    @Test\n    public void assertSendHeartbeatV2() throws MQBrokerException, RemotingException, InterruptedException {\n        mockInvokeSync();\n        HeartbeatData heartbeatData = new HeartbeatData();\n        HeartbeatV2Result actual = mqClientAPI.sendHeartbeatV2(defaultBrokerAddr, heartbeatData, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.getVersion());\n        assertFalse(actual.isSubChange());\n        assertFalse(actual.isSupportV2());\n    }\n\n    @Test\n    public void testUnregisterClient() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        mqClientAPI.unregisterClient(defaultBrokerAddr, \"\", \"\", \"\", defaultTimeout);\n    }\n\n    @Test\n    public void testEndTransactionOneway() throws RemotingException, InterruptedException {\n        mockInvokeSync();\n        EndTransactionRequestHeader requestHeader = mock(EndTransactionRequestHeader.class);\n        mqClientAPI.endTransactionOneway(defaultBrokerAddr, requestHeader, \"\", defaultTimeout);\n    }\n\n    @Test\n    public void testQueryMessage() throws MQBrokerException, RemotingException, InterruptedException {\n        QueryMessageRequestHeader requestHeader = mock(QueryMessageRequestHeader.class);\n        InvokeCallback callback = mock(InvokeCallback.class);\n        mqClientAPI.queryMessage(defaultBrokerAddr, requestHeader, defaultTimeout, callback, false);\n    }\n\n    @Test\n    public void testRegisterClient() throws RemotingException, InterruptedException {\n        mockInvokeSync();\n        HeartbeatData heartbeatData = new HeartbeatData();\n        assertTrue(mqClientAPI.registerClient(defaultBrokerAddr, heartbeatData, defaultTimeout));\n    }\n\n    @Test\n    public void testConsumerSendMessageBack() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        MessageExt messageExt = mock(MessageExt.class);\n        mqClientAPI.consumerSendMessageBack(defaultBrokerAddr, brokerName, messageExt, \"\", 1, defaultTimeout, 1000);\n    }\n\n    @Test\n    public void assertLockBatchMQ() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        LockBatchRequestBody responseBody = new LockBatchRequestBody();\n        setResponseBody(responseBody);\n        Set<MessageQueue> actual = mqClientAPI.lockBatchMQ(defaultBrokerAddr, responseBody, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(0, actual.size());\n    }\n\n    @Test\n    public void testUnlockBatchMQ() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        UnlockBatchRequestBody unlockBatchRequestBody = new UnlockBatchRequestBody();\n        mqClientAPI.unlockBatchMQ(defaultBrokerAddr, unlockBatchRequestBody, defaultTimeout, false);\n    }\n\n    @Test\n    public void assertGetTopicStatsInfo() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        TopicStatsTable responseBody = new TopicStatsTable();\n        MessageQueue messageQueue = new MessageQueue();\n        TopicOffset topicOffset = new TopicOffset();\n        responseBody.getOffsetTable().put(messageQueue, topicOffset);\n        setResponseBody(responseBody);\n        TopicStatsTable actual = mqClientAPI.getTopicStatsInfo(defaultBrokerAddr, defaultTopic, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.getOffsetTable().size());\n    }\n\n    @Test\n    public void assertGetConsumeStats() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        ConsumeStats responseBody = new ConsumeStats();\n        responseBody.setConsumeTps(1000);\n        setResponseBody(responseBody);\n        ConsumeStats actual = mqClientAPI.getConsumeStats(defaultBrokerAddr, \"\", defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1000, actual.getConsumeTps(), 0.0);\n    }\n\n    @Test\n    public void assertGetProducerConnectionList() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        ProducerConnection responseBody = new ProducerConnection();\n        responseBody.getConnectionSet().add(new Connection());\n        setResponseBody(responseBody);\n        ProducerConnection actual = mqClientAPI.getProducerConnectionList(defaultBrokerAddr, \"\", defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.getConnectionSet().size());\n    }\n\n    @Test\n    public void assertGetAllProducerInfo() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        Map<String, List<ProducerInfo>> data = new HashMap<>();\n        data.put(\"key\", Collections.emptyList());\n        ProducerTableInfo responseBody = new ProducerTableInfo(data);\n        setResponseBody(responseBody);\n        ProducerTableInfo actual = mqClientAPI.getAllProducerInfo(defaultBrokerAddr, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.getData().size());\n    }\n\n    @Test\n    public void assertGetConsumerConnectionList() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        ConsumerConnection responseBody = new ConsumerConnection();\n        responseBody.setConsumeType(ConsumeType.CONSUME_ACTIVELY);\n        responseBody.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);\n        responseBody.setMessageModel(MessageModel.CLUSTERING);\n        setResponseBody(responseBody);\n        ConsumerConnection actual = mqClientAPI.getConsumerConnectionList(defaultBrokerAddr, \"\", defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(ConsumeType.CONSUME_ACTIVELY, actual.getConsumeType());\n        assertEquals(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET, actual.getConsumeFromWhere());\n        assertEquals(MessageModel.CLUSTERING, actual.getMessageModel());\n    }\n\n    @Test\n    public void assertGetBrokerRuntimeInfo() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        KVTable responseBody = new KVTable();\n        responseBody.getTable().put(\"key\", \"value\");\n        setResponseBody(responseBody);\n        KVTable actual = mqClientAPI.getBrokerRuntimeInfo(defaultBrokerAddr, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.getTable().size());\n    }\n\n    @Test\n    public void testAddBroker() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        mqClientAPI.addBroker(defaultBrokerAddr, \"\", defaultTimeout);\n    }\n\n    @Test\n    public void testRemoveBroker() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        mqClientAPI.removeBroker(defaultBrokerAddr, clusterName, brokerName, MixAll.MASTER_ID, defaultTimeout);\n    }\n\n    @Test\n    public void testUpdateBrokerConfig() throws RemotingException, InterruptedException, MQBrokerException, UnsupportedEncodingException, MQClientException {\n        mockInvokeSync();\n        mqClientAPI.updateBrokerConfig(defaultBrokerAddr, createProperties(), defaultTimeout);\n    }\n\n    @Test\n    public void assertGetBrokerConfig() throws RemotingException, InterruptedException, MQBrokerException, UnsupportedEncodingException {\n        mockInvokeSync();\n        setResponseBody(\"{\\\"key\\\":\\\"value\\\"}\");\n        Properties actual = mqClientAPI.getBrokerConfig(defaultBrokerAddr, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.size());\n    }\n\n    @Test\n    public void testUpdateColdDataFlowCtrGroupConfig() throws RemotingException, InterruptedException, MQBrokerException, UnsupportedEncodingException {\n        mockInvokeSync();\n        Properties props = new Properties();\n        mqClientAPI.updateColdDataFlowCtrGroupConfig(defaultBrokerAddr, props, defaultTimeout);\n    }\n\n    @Test\n    public void testRemoveColdDataFlowCtrGroupConfig() throws RemotingException, InterruptedException, MQBrokerException, UnsupportedEncodingException {\n        mockInvokeSync();\n        mqClientAPI.removeColdDataFlowCtrGroupConfig(defaultBrokerAddr, \"\", defaultTimeout);\n    }\n\n    @Test\n    public void assertGetColdDataFlowCtrInfo() throws RemotingException, InterruptedException, MQBrokerException, UnsupportedEncodingException {\n        mockInvokeSync();\n        setResponseBody(\"{\\\"key\\\":\\\"value\\\"}\");\n        String actual = mqClientAPI.getColdDataFlowCtrInfo(defaultBrokerAddr, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(\"\\\"{\\\\\\\"key\\\\\\\":\\\\\\\"value\\\\\\\"}\\\"\", actual);\n    }\n\n    @Test\n    public void assertSetCommitLogReadAheadMode() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        when(response.getRemark()).thenReturn(\"remark\");\n        String actual = mqClientAPI.setCommitLogReadAheadMode(defaultBrokerAddr, \"\", defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(\"remark\", actual);\n    }\n\n    @Test\n    public void assertGetBrokerClusterInfo() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        ClusterInfo responseBody = new ClusterInfo();\n        Map<String, Set<String>> clusterAddrTable = new HashMap<>();\n        clusterAddrTable.put(clusterName, new HashSet<>());\n        Map<String, BrokerData> brokerAddrTable = new HashMap<>();\n        brokerAddrTable.put(brokerName, new BrokerData());\n        responseBody.setClusterAddrTable(clusterAddrTable);\n        responseBody.setBrokerAddrTable(brokerAddrTable);\n        setResponseBody(responseBody);\n        ClusterInfo actual = mqClientAPI.getBrokerClusterInfo(defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.getClusterAddrTable().size());\n        assertEquals(1, actual.getBrokerAddrTable().size());\n    }\n\n    @Test\n    public void assertGetDefaultTopicRouteInfoFromNameServer() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        TopicRouteData responseBody = new TopicRouteData();\n        responseBody.getQueueDatas().add(new QueueData());\n        responseBody.getBrokerDatas().add(new BrokerData());\n        responseBody.getFilterServerTable().put(\"key\", Collections.emptyList());\n        Map<String, TopicQueueMappingInfo> topicQueueMappingByBroker = new HashMap<>();\n        topicQueueMappingByBroker.put(\"key\", new TopicQueueMappingInfo());\n        responseBody.setTopicQueueMappingByBroker(topicQueueMappingByBroker);\n        setResponseBody(responseBody);\n        TopicRouteData actual = mqClientAPI.getDefaultTopicRouteInfoFromNameServer(defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.getQueueDatas().size());\n        assertEquals(1, actual.getBrokerDatas().size());\n        assertEquals(1, actual.getFilterServerTable().size());\n        assertEquals(1, actual.getTopicQueueMappingByBroker().size());\n    }\n\n    @Test\n    public void assertGetTopicRouteInfoFromNameServer() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        TopicRouteData responseBody = new TopicRouteData();\n        responseBody.getQueueDatas().add(new QueueData());\n        responseBody.getBrokerDatas().add(new BrokerData());\n        responseBody.getFilterServerTable().put(\"key\", Collections.emptyList());\n        Map<String, TopicQueueMappingInfo> topicQueueMappingByBroker = new HashMap<>();\n        topicQueueMappingByBroker.put(\"key\", new TopicQueueMappingInfo());\n        responseBody.setTopicQueueMappingByBroker(topicQueueMappingByBroker);\n        setResponseBody(responseBody);\n        TopicRouteData actual = mqClientAPI.getTopicRouteInfoFromNameServer(defaultTopic, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.getQueueDatas().size());\n        assertEquals(1, actual.getBrokerDatas().size());\n        assertEquals(1, actual.getFilterServerTable().size());\n        assertEquals(1, actual.getTopicQueueMappingByBroker().size());\n    }\n\n    @Test\n    public void assertGetTopicListFromNameServer() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        TopicList responseBody = new TopicList();\n        responseBody.setBrokerAddr(defaultBrokerAddr);\n        responseBody.getTopicList().add(defaultTopic);\n        setResponseBody(responseBody);\n        TopicList actual = mqClientAPI.getTopicListFromNameServer(defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.getTopicList().size());\n        assertEquals(defaultBrokerAddr, actual.getBrokerAddr());\n    }\n\n    @Test\n    public void assertWipeWritePermOfBroker() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        WipeWritePermOfBrokerResponseHeader responseHeader = mock(WipeWritePermOfBrokerResponseHeader.class);\n        when(responseHeader.getWipeTopicCount()).thenReturn(1);\n        setResponseHeader(responseHeader);\n        assertEquals(1, mqClientAPI.wipeWritePermOfBroker(defaultNsAddr, brokerName, defaultTimeout));\n    }\n\n    @Test\n    public void testDeleteTopicInBroker() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        mqClientAPI.deleteTopicInBroker(defaultBrokerAddr, defaultTopic, defaultTimeout);\n    }\n\n    @Test\n    public void testDeleteTopicInNameServer() throws RemotingException, InterruptedException, MQClientException, MQBrokerException {\n        mockInvokeSync();\n        mqClientAPI.deleteTopicInNameServer(defaultNsAddr, defaultTopic, defaultTimeout);\n        mqClientAPI.deleteTopicInNameServer(defaultNsAddr, clusterName, defaultTopic, defaultTimeout);\n    }\n\n    @Test\n    public void testDeleteSubscriptionGroup() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        mqClientAPI.deleteSubscriptionGroup(defaultBrokerAddr, \"\", true, defaultTimeout);\n    }\n\n    @Test\n    public void assertGetKVConfigValue() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        GetKVConfigResponseHeader responseHeader = mock(GetKVConfigResponseHeader.class);\n        when(responseHeader.getValue()).thenReturn(\"value\");\n        setResponseHeader(responseHeader);\n        assertEquals(\"value\", mqClientAPI.getKVConfigValue(\"\", \"\", defaultTimeout));\n    }\n\n    @Test\n    public void testPutKVConfigValue() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        mqClientAPI.putKVConfigValue(\"\", \"\", \"\", defaultTimeout);\n    }\n\n    @Test\n    public void testDeleteKVConfigValue() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        mqClientAPI.deleteKVConfigValue(\"\", \"\", defaultTimeout);\n    }\n\n    @Test\n    public void assertGetKVListByNamespace() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        KVTable responseBody = new KVTable();\n        responseBody.getTable().put(\"key\", \"value\");\n        setResponseBody(responseBody);\n        KVTable actual = mqClientAPI.getKVListByNamespace(\"\", defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.getTable().size());\n    }\n\n    @Test\n    public void assertInvokeBrokerToResetOffset() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        ResetOffsetBody responseBody = new ResetOffsetBody();\n        responseBody.getOffsetTable().put(new MessageQueue(), 1L);\n        setResponseBody(responseBody);\n        Map<MessageQueue, Long> actual = mqClientAPI.invokeBrokerToResetOffset(defaultBrokerAddr, defaultTopic, \"\", System.currentTimeMillis(), false, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.size());\n        actual = mqClientAPI.invokeBrokerToResetOffset(defaultBrokerAddr, defaultTopic, \"\", System.currentTimeMillis(), 1, 1L, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.size());\n    }\n\n    @Test\n    public void assertInvokeBrokerToGetConsumerStatus() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        GetConsumerStatusBody responseBody = new GetConsumerStatusBody();\n        responseBody.getConsumerTable().put(\"key\", new HashMap<>());\n        responseBody.getMessageQueueTable().put(new MessageQueue(), 1L);\n        setResponseBody(responseBody);\n        Map<String, Map<MessageQueue, Long>> actual = mqClientAPI.invokeBrokerToGetConsumerStatus(defaultBrokerAddr, defaultTopic, \"\", \"\", defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.size());\n    }\n\n    @Test\n    public void assertQueryTopicConsumeByWho() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        GroupList responseBody = new GroupList();\n        responseBody.getGroupList().add(\"\");\n        setResponseBody(responseBody);\n        GroupList actual = mqClientAPI.queryTopicConsumeByWho(defaultBrokerAddr, defaultTopic, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.getGroupList().size());\n    }\n\n    @Test\n    public void assertQueryTopicsByConsumer() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        TopicList responseBody = new TopicList();\n        responseBody.getTopicList().add(defaultTopic);\n        responseBody.setBrokerAddr(defaultBrokerAddr);\n        setResponseBody(responseBody);\n        TopicList actual = mqClientAPI.queryTopicsByConsumer(defaultBrokerAddr, \"\", defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.getTopicList().size());\n        assertEquals(defaultBrokerAddr, actual.getBrokerAddr());\n    }\n\n    @Test\n    public void assertQuerySubscriptionByConsumer() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        QuerySubscriptionResponseBody responseBody = new QuerySubscriptionResponseBody();\n        SubscriptionData subscriptionData = new SubscriptionData();\n        subscriptionData.setTopic(defaultTopic);\n        responseBody.setSubscriptionData(subscriptionData);\n        setResponseBody(responseBody);\n        SubscriptionData actual = mqClientAPI.querySubscriptionByConsumer(defaultBrokerAddr, group, defaultTopic, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(defaultTopic, actual.getTopic());\n    }\n\n    @Test\n    public void assertQueryConsumeTimeSpan() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        QueryConsumeTimeSpanBody responseBody = new QueryConsumeTimeSpanBody();\n        responseBody.getConsumeTimeSpanSet().add(new QueueTimeSpan());\n        setResponseBody(responseBody);\n        List<QueueTimeSpan> actual = mqClientAPI.queryConsumeTimeSpan(defaultBrokerAddr, defaultTopic, group, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.size());\n    }\n\n    @Test\n    public void assertGetTopicsByCluster() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        TopicList responseBody = new TopicList();\n        responseBody.setBrokerAddr(defaultBrokerAddr);\n        responseBody.setTopicList(Collections.singleton(defaultTopic));\n        setResponseBody(responseBody);\n        TopicList actual = mqClientAPI.getTopicsByCluster(clusterName, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(defaultBrokerAddr, actual.getBrokerAddr());\n        assertEquals(1, actual.getTopicList().size());\n        assertTrue(actual.getTopicList().contains(defaultTopic));\n    }\n\n    @Test\n    public void assertGetSystemTopicList() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        TopicList responseBody = new TopicList();\n        responseBody.setBrokerAddr(defaultBrokerAddr);\n        responseBody.setTopicList(Collections.singleton(defaultTopic));\n        setResponseBody(responseBody);\n        TopicList actual = mqClientAPI.getSystemTopicList(defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(defaultBrokerAddr, actual.getBrokerAddr());\n        assertEquals(1, actual.getTopicList().size());\n        assertTrue(actual.getTopicList().contains(defaultTopic));\n    }\n\n    @Test\n    public void assertGetSystemTopicListFromBroker() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        TopicList responseBody = new TopicList();\n        responseBody.setBrokerAddr(defaultBrokerAddr);\n        responseBody.setTopicList(Collections.singleton(defaultTopic));\n        setResponseBody(responseBody);\n        TopicList actual = mqClientAPI.getSystemTopicListFromBroker(defaultBrokerAddr, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(defaultBrokerAddr, actual.getBrokerAddr());\n        assertEquals(1, actual.getTopicList().size());\n        assertTrue(actual.getTopicList().contains(defaultTopic));\n    }\n\n    @Test\n    public void assertCleanExpiredConsumeQueue() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        assertTrue(mqClientAPI.cleanExpiredConsumeQueue(defaultBrokerAddr, defaultTimeout));\n    }\n\n    @Test\n    public void assertDeleteExpiredCommitLog() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        assertTrue(mqClientAPI.deleteExpiredCommitLog(defaultBrokerAddr, defaultTimeout));\n    }\n\n    @Test\n    public void assertCleanUnusedTopicByAddr() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        assertTrue(mqClientAPI.cleanUnusedTopicByAddr(defaultBrokerAddr, defaultTimeout));\n    }\n\n    @Test\n    public void assertGetConsumerRunningInfo() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        ConsumerRunningInfo responseBody = new ConsumerRunningInfo();\n        responseBody.setJstack(\"jstack\");\n        responseBody.getUserConsumerInfo().put(\"key\", \"value\");\n        setResponseBody(responseBody);\n        ConsumerRunningInfo actual = mqClientAPI.getConsumerRunningInfo(defaultBrokerAddr, group, clientId, false, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(\"jstack\", actual.getJstack());\n        assertEquals(1, actual.getUserConsumerInfo().size());\n    }\n\n    @Test\n    public void assertConsumeMessageDirectly() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        ConsumeMessageDirectlyResult responseBody = new ConsumeMessageDirectlyResult();\n        responseBody.setAutoCommit(true);\n        responseBody.setRemark(\"remark\");\n        setResponseBody(responseBody);\n        ConsumeMessageDirectlyResult actual = mqClientAPI.consumeMessageDirectly(defaultBrokerAddr, group, clientId, topic, \"\", defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(\"remark\", actual.getRemark());\n        assertTrue(actual.isAutoCommit());\n    }\n\n    @Test\n    public void assertQueryCorrectionOffset() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        QueryCorrectionOffsetBody responseBody = new QueryCorrectionOffsetBody();\n        responseBody.getCorrectionOffsets().put(1, 1L);\n        setResponseBody(responseBody);\n        Map<Integer, Long> actual = mqClientAPI.queryCorrectionOffset(defaultBrokerAddr, topic, group, new HashSet<>(), defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.size());\n        assertTrue(actual.containsKey(1));\n        assertTrue(actual.containsValue(1L));\n    }\n\n    @Test\n    public void assertGetUnitTopicList() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        TopicList responseBody = new TopicList();\n        responseBody.getTopicList().add(defaultTopic);\n        setResponseBody(responseBody);\n        TopicList actual = mqClientAPI.getUnitTopicList(false, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.getTopicList().size());\n    }\n\n    @Test\n    public void assertGetHasUnitSubTopicList() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        TopicList responseBody = new TopicList();\n        responseBody.getTopicList().add(defaultTopic);\n        setResponseBody(responseBody);\n        TopicList actual = mqClientAPI.getHasUnitSubTopicList(false, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.getTopicList().size());\n    }\n\n    @Test\n    public void assertGetHasUnitSubUnUnitTopicList() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        TopicList responseBody = new TopicList();\n        responseBody.getTopicList().add(defaultTopic);\n        setResponseBody(responseBody);\n        TopicList actual = mqClientAPI.getHasUnitSubUnUnitTopicList(false, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.getTopicList().size());\n    }\n\n    @Test\n    public void testCloneGroupOffset() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        mqClientAPI.cloneGroupOffset(defaultBrokerAddr, \"\", \"\", defaultTopic, false, defaultTimeout);\n    }\n\n    @Test\n    public void assertViewBrokerStatsData() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        BrokerStatsData responseBody = new BrokerStatsData();\n        responseBody.setStatsDay(new BrokerStatsItem());\n        setResponseBody(responseBody);\n        BrokerStatsData actual = mqClientAPI.viewBrokerStatsData(defaultBrokerAddr, \"\", \"\", defaultTimeout);\n        assertNotNull(actual);\n        assertNotNull(actual.getStatsDay());\n    }\n\n    @Test\n    public void assertGetClusterList() {\n        Set<String> actual = mqClientAPI.getClusterList(topic, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(0, actual.size());\n    }\n\n    @Test\n    public void assertFetchConsumeStatsInBroker() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        ConsumeStatsList responseBody = new ConsumeStatsList();\n        responseBody.setBrokerAddr(defaultBrokerAddr);\n        responseBody.getConsumeStatsList().add(new HashMap<>());\n        setResponseBody(responseBody);\n        ConsumeStatsList actual = mqClientAPI.fetchConsumeStatsInBroker(defaultBrokerAddr, false, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.getConsumeStatsList().size());\n        assertEquals(defaultBrokerAddr, actual.getBrokerAddr());\n    }\n\n    @Test\n    public void assertGetAllSubscriptionGroupForSubscriptionGroupWrapper() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        SubscriptionGroupWrapper responseBody = new SubscriptionGroupWrapper();\n        responseBody.getSubscriptionGroupTable().put(\"key\", new SubscriptionGroupConfig());\n        setResponseBody(responseBody);\n        SubscriptionGroupWrapper actual = mqClientAPI.getAllSubscriptionGroup(defaultBrokerAddr, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.getSubscriptionGroupTable().size());\n        assertNotNull(actual.getDataVersion());\n        assertEquals(0, actual.getDataVersion().getStateVersion());\n    }\n\n    @Test\n    public void assertGetAllSubscriptionGroupForSubscriptionGroupConfig() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        SubscriptionGroupConfig responseBody = new SubscriptionGroupConfig();\n        responseBody.setGroupName(group);\n        responseBody.setBrokerId(MixAll.MASTER_ID);\n        setResponseBody(responseBody);\n        SubscriptionGroupConfig actual = mqClientAPI.getSubscriptionGroupConfig(defaultBrokerAddr, group, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(group, actual.getGroupName());\n        assertEquals(MixAll.MASTER_ID, actual.getBrokerId());\n    }\n\n    @Test\n    public void assertGetAllTopicConfig() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        TopicConfigSerializeWrapper responseBody = new TopicConfigSerializeWrapper();\n        responseBody.getTopicConfigTable().put(\"key\", new TopicConfig());\n        setResponseBody(responseBody);\n        TopicConfigSerializeWrapper actual = mqClientAPI.getAllTopicConfig(defaultBrokerAddr, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.getTopicConfigTable().size());\n        assertNotNull(actual.getDataVersion());\n        assertEquals(0, actual.getDataVersion().getStateVersion());\n    }\n\n    @Test\n    public void testUpdateNameServerConfig() throws RemotingException, InterruptedException, MQClientException, UnsupportedEncodingException {\n        mockInvokeSync();\n        mqClientAPI.updateNameServerConfig(createProperties(), Collections.singletonList(defaultNsAddr), defaultTimeout);\n    }\n\n    @Test\n    public void assertGetNameServerConfig() throws RemotingException, InterruptedException, UnsupportedEncodingException, MQClientException {\n        mockInvokeSync();\n        setResponseBody(\"{\\\"key\\\":\\\"value\\\"}\");\n        Map<String, Properties> actual = mqClientAPI.getNameServerConfig(Collections.singletonList(defaultNsAddr), defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.size());\n        assertTrue(actual.containsKey(defaultNsAddr));\n    }\n\n    @Test\n    public void assertQueryConsumeQueue() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        QueryConsumeQueueResponseBody responseBody = new QueryConsumeQueueResponseBody();\n        responseBody.setQueueData(Collections.singletonList(new ConsumeQueueData()));\n        setResponseBody(responseBody);\n        QueryConsumeQueueResponseBody actual = mqClientAPI.queryConsumeQueue(defaultBrokerAddr, defaultTopic, 1, 1, 1, group, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1, actual.getQueueData().size());\n    }\n\n    @Test\n    public void testCheckClientInBroker() throws RemotingException, InterruptedException, MQClientException {\n        mockInvokeSync();\n        mqClientAPI.checkClientInBroker(defaultBrokerAddr, group, clientId, new SubscriptionData(), defaultTimeout);\n    }\n\n    @Test\n    public void assertGetTopicConfig() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        TopicConfigAndQueueMapping responseBody = new TopicConfigAndQueueMapping(new TopicConfig(), new TopicQueueMappingDetail());\n        setResponseBody(responseBody);\n        TopicConfigAndQueueMapping actual = mqClientAPI.getTopicConfig(defaultBrokerAddr, defaultTopic, defaultTimeout);\n        assertNotNull(actual);\n        assertNotNull(actual.getMappingDetail());\n    }\n\n    @Test\n    public void testCreateStaticTopic() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        mqClientAPI.createStaticTopic(defaultBrokerAddr, defaultTopic, new TopicConfig(), new TopicQueueMappingDetail(), false, defaultTimeout);\n    }\n\n    @Test\n    public void assertUpdateAndGetGroupForbidden() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        GroupForbidden responseBody = new GroupForbidden();\n        responseBody.setGroup(group);\n        responseBody.setTopic(defaultTopic);\n        setResponseBody(responseBody);\n        GroupForbidden actual = mqClientAPI.updateAndGetGroupForbidden(defaultBrokerAddr, new UpdateGroupForbiddenRequestHeader(), defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(group, actual.getGroup());\n        assertEquals(defaultTopic, actual.getTopic());\n    }\n\n    @Test\n    public void testResetMasterFlushOffset() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        mqClientAPI.resetMasterFlushOffset(defaultBrokerAddr, 1L);\n    }\n\n    @Test\n    public void assertGetBrokerHAStatus() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        HARuntimeInfo responseBody = new HARuntimeInfo();\n        responseBody.setMaster(true);\n        responseBody.setMasterCommitLogMaxOffset(1L);\n        setResponseBody(responseBody);\n        HARuntimeInfo actual = mqClientAPI.getBrokerHAStatus(defaultBrokerAddr, defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1L, actual.getMasterCommitLogMaxOffset());\n        assertTrue(actual.isMaster());\n    }\n\n    @Test\n    public void assertGetControllerMetaData() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        GetMetaDataResponseHeader responseHeader = new GetMetaDataResponseHeader();\n        responseHeader.setGroup(group);\n        responseHeader.setIsLeader(true);\n        setResponseHeader(responseHeader);\n        GetMetaDataResponseHeader actual = mqClientAPI.getControllerMetaData(defaultBrokerAddr);\n        assertNotNull(actual);\n        assertEquals(group, actual.getGroup());\n        assertTrue(actual.isLeader());\n    }\n\n    @Test\n    public void assertGetInSyncStateData() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        BrokerReplicasInfo responseBody = new BrokerReplicasInfo();\n        BrokerReplicasInfo.ReplicasInfo replicasInfo = new BrokerReplicasInfo.ReplicasInfo(MixAll.MASTER_ID, defaultBrokerAddr, 1, 1, Collections.emptyList(), Collections.emptyList());\n        responseBody.getReplicasInfoTable().put(\"key\", replicasInfo);\n        GetMetaDataResponseHeader responseHeader = new GetMetaDataResponseHeader();\n        responseHeader.setControllerLeaderAddress(defaultBrokerAddr);\n        setResponseHeader(responseHeader);\n        setResponseBody(responseBody);\n        BrokerReplicasInfo actual = mqClientAPI.getInSyncStateData(defaultBrokerAddr, Collections.singletonList(defaultBrokerAddr));\n        assertNotNull(actual);\n        assertEquals(1L, actual.getReplicasInfoTable().size());\n    }\n\n    @Test\n    public void assertGetBrokerEpochCache() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        EpochEntryCache responseBody = new EpochEntryCache(clusterName, brokerName, MixAll.MASTER_ID, Collections.emptyList(), 1);\n        setResponseBody(responseBody);\n        EpochEntryCache actual = mqClientAPI.getBrokerEpochCache(defaultBrokerAddr);\n        assertNotNull(actual);\n        assertEquals(1L, actual.getMaxOffset());\n        assertEquals(MixAll.MASTER_ID, actual.getBrokerId());\n        assertEquals(brokerName, actual.getBrokerName());\n        assertEquals(clusterName, actual.getClusterName());\n    }\n\n    @Test\n    public void assertGetControllerConfig() throws RemotingException, InterruptedException, UnsupportedEncodingException, MQClientException {\n        mockInvokeSync();\n        setResponseBody(\"{\\\"key\\\":\\\"value\\\"}\");\n        Map<String, Properties> actual = mqClientAPI.getControllerConfig(Collections.singletonList(defaultBrokerAddr), defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(1L, actual.size());\n    }\n\n    @Test\n    public void testUpdateControllerConfig() throws RemotingException, InterruptedException, UnsupportedEncodingException, MQClientException {\n        mockInvokeSync();\n        mqClientAPI.updateControllerConfig(createProperties(), Collections.singletonList(defaultBrokerAddr), defaultTimeout);\n    }\n\n    @Test\n    public void assertElectMaster() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        BrokerMemberGroup responseBody = new BrokerMemberGroup();\n        setResponseBody(responseBody);\n        GetMetaDataResponseHeader getMetaDataResponseHeader = new GetMetaDataResponseHeader();\n        getMetaDataResponseHeader.setControllerLeaderAddress(defaultBrokerAddr);\n        when(response.decodeCommandCustomHeader(GetMetaDataResponseHeader.class)).thenReturn(getMetaDataResponseHeader);\n        ElectMasterResponseHeader responseHeader = new ElectMasterResponseHeader();\n        when(response.decodeCommandCustomHeader(ElectMasterResponseHeader.class)).thenReturn(responseHeader);\n        Pair<ElectMasterResponseHeader, BrokerMemberGroup> actual = mqClientAPI.electMaster(defaultBrokerAddr, clusterName, brokerName, MixAll.MASTER_ID);\n        assertNotNull(actual);\n        assertEquals(responseHeader, actual.getObject1());\n        assertEquals(responseBody, actual.getObject2());\n    }\n\n    @Test\n    public void testCleanControllerBrokerData() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        GetMetaDataResponseHeader responseHeader = new GetMetaDataResponseHeader();\n        responseHeader.setControllerLeaderAddress(defaultBrokerAddr);\n        setResponseHeader(responseHeader);\n        mqClientAPI.cleanControllerBrokerData(defaultBrokerAddr, clusterName, brokerName, \"\", false);\n    }\n\n    @Test\n    public void testCreateUser() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        mqClientAPI.createUser(defaultBrokerAddr, new UserInfo(), defaultTimeout);\n    }\n\n    @Test\n    public void testUpdateUser() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        mqClientAPI.updateUser(defaultBrokerAddr, new UserInfo(), defaultTimeout);\n    }\n\n    @Test\n    public void testDeleteUser() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        mqClientAPI.deleteUser(defaultBrokerAddr, \"\", defaultTimeout);\n    }\n\n    @Test\n    public void assertGetUser() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        setResponseBody(createUserInfo());\n        UserInfo actual = mqClientAPI.getUser(defaultBrokerAddr, \"\", defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(\"username\", actual.getUsername());\n        assertEquals(\"password\", actual.getPassword());\n        assertEquals(\"userStatus\", actual.getUserStatus());\n        assertEquals(\"userType\", actual.getUserType());\n    }\n\n    @Test\n    public void assertListUser() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        setResponseBody(Collections.singletonList(createUserInfo()));\n        List<UserInfo> actual = mqClientAPI.listUser(defaultBrokerAddr, \"\", defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(\"username\", actual.get(0).getUsername());\n        assertEquals(\"password\", actual.get(0).getPassword());\n        assertEquals(\"userStatus\", actual.get(0).getUserStatus());\n        assertEquals(\"userType\", actual.get(0).getUserType());\n    }\n\n    @Test\n    public void testCreateAcl() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        mqClientAPI.createAcl(defaultBrokerAddr, new AclInfo(), defaultTimeout);\n    }\n\n    @Test\n    public void testUpdateAcl() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        mqClientAPI.updateAcl(defaultBrokerAddr, new AclInfo(), defaultTimeout);\n    }\n\n    @Test\n    public void testDeleteAcl() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        mqClientAPI.deleteAcl(defaultBrokerAddr, \"\", \"\", defaultTimeout);\n    }\n\n    @Test\n    public void assertGetAcl() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        setResponseBody(createAclInfo());\n        AclInfo actual = mqClientAPI.getAcl(defaultBrokerAddr, \"\", defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(\"subject\", actual.getSubject());\n        assertEquals(1, actual.getPolicies().size());\n    }\n\n    @Test\n    public void assertListAcl() throws RemotingException, InterruptedException, MQBrokerException {\n        mockInvokeSync();\n        setResponseBody(Collections.singletonList(createAclInfo()));\n        List<AclInfo> actual = mqClientAPI.listAcl(defaultBrokerAddr, \"\", \"\", defaultTimeout);\n        assertNotNull(actual);\n        assertEquals(\"subject\", actual.get(0).getSubject());\n        assertEquals(1, actual.get(0).getPolicies().size());\n    }\n\n    @Test\n    public void testRecallMessage() throws RemotingException, InterruptedException, MQBrokerException {\n        RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader();\n        requestHeader.setProducerGroup(group);\n        requestHeader.setTopic(topic);\n        requestHeader.setRecallHandle(\"handle\");\n        requestHeader.setBrokerName(brokerName);\n\n        // success\n        mockInvokeSync();\n        String msgId = MessageClientIDSetter.createUniqID();\n        RecallMessageResponseHeader responseHeader = new RecallMessageResponseHeader();\n        responseHeader.setMsgId(msgId);\n        setResponseHeader(responseHeader);\n        String result = mqClientAPI.recallMessage(defaultBrokerAddr, requestHeader, defaultTimeout);\n        assertEquals(msgId, result);\n\n        // error\n        when(response.getCode()).thenReturn(ResponseCode.SYSTEM_ERROR);\n        when(response.getRemark()).thenReturn(\"error\");\n        MQBrokerException e = assertThrows(MQBrokerException.class, () -> {\n            mqClientAPI.recallMessage(defaultBrokerAddr, requestHeader, defaultTimeout);\n        });\n        assertEquals(ResponseCode.SYSTEM_ERROR, e.getResponseCode());\n        assertEquals(\"error\", e.getErrorMessage());\n        assertEquals(defaultBrokerAddr, e.getBrokerAddr());\n    }\n\n    @Test\n    public void testRecallMessageAsync() throws RemotingException, InterruptedException {\n        RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader();\n        requestHeader.setProducerGroup(group);\n        requestHeader.setTopic(topic);\n        requestHeader.setRecallHandle(\"handle\");\n        requestHeader.setBrokerName(brokerName);\n        String msgId = \"msgId\";\n        doAnswer((Answer<Void>) mock -> {\n            InvokeCallback callback = mock.getArgument(3);\n            RemotingCommand request = mock.getArgument(1);\n            RemotingCommand response = RemotingCommand.createResponseCommand(RecallMessageResponseHeader.class);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setOpaque(request.getOpaque());\n            RecallMessageResponseHeader responseHeader = (RecallMessageResponseHeader) response.readCustomHeader();\n            responseHeader.setMsgId(msgId);\n            ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);\n            responseFuture.setResponseCommand(response);\n            callback.operationSucceed(responseFuture.getResponseCommand());\n            return null;\n        }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));\n\n        final CountDownLatch done = new CountDownLatch(1);\n        mqClientAPI.recallMessageAsync(defaultBrokerAddr, requestHeader,\n            defaultTimeout, new InvokeCallback() {\n                @Override\n                public void operationComplete(ResponseFuture responseFuture) {\n                }\n\n                @Override\n                public void operationSucceed(RemotingCommand response) {\n                    RecallMessageResponseHeader responseHeader = (RecallMessageResponseHeader) response.readCustomHeader();\n                    Assert.assertEquals(msgId, responseHeader.getMsgId());\n                    done.countDown();\n                }\n\n                @Override\n                public void operationFail(Throwable throwable) {\n                }\n            });\n        done.await();\n    }\n\n    @Test\n    public void testMQClientAPIImplWithoutObjectCreator() {\n        MQClientAPIImpl clientAPI = new MQClientAPIImpl(\n            new NettyClientConfig(),\n            null,\n            null,\n            new ClientConfig(),\n            null,\n            null\n        );\n        RemotingClient remotingClient1 = clientAPI.getRemotingClient();\n        Assert.assertTrue(remotingClient1 instanceof NettyRemotingClient);\n    }\n\n    @Test\n    public void testMQClientAPIImplWithObjectCreator() {\n        ObjectCreator<RemotingClient> clientObjectCreator = args -> new MockRemotingClientTest((NettyClientConfig) args[0]);\n        final NettyClientConfig nettyClientConfig = new NettyClientConfig();\n        MQClientAPIImpl clientAPI = new MQClientAPIImpl(\n            nettyClientConfig,\n            null,\n            null,\n            new ClientConfig(),\n            null,\n            clientObjectCreator\n        );\n        RemotingClient remotingClient1 = clientAPI.getRemotingClient();\n        Assert.assertTrue(remotingClient1 instanceof MockRemotingClientTest);\n        MockRemotingClientTest remotingClientTest = (MockRemotingClientTest) remotingClient1;\n        Assert.assertSame(remotingClientTest.getNettyClientConfig(), nettyClientConfig);\n    }\n\n    private static class MockRemotingClientTest extends NettyRemotingClient {\n        public MockRemotingClientTest(NettyClientConfig nettyClientConfig) {\n            super(nettyClientConfig);\n        }\n\n        public NettyClientConfig getNettyClientConfig() {\n            return nettyClientConfig;\n        }\n    }\n\n    @Test\n    public void testCheckRocksdbCqWriteProgress() throws Exception {\n        RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"Success\");\n        CheckRocksdbCqWriteResult expectedResult = new CheckRocksdbCqWriteResult();\n        expectedResult.setCheckStatus(CheckRocksdbCqWriteResult.CheckStatus.CHECK_OK.getValue());\n        response.setBody(JSON.toJSONString(expectedResult).getBytes());\n\n        when(remotingClient.invokeSync(any(String.class), any(RemotingCommand.class), any(Long.class)))\n                .thenReturn(response);\n\n        CheckRocksdbCqWriteResult result = mqClientAPI.checkRocksdbCqWriteProgress(\n                \"brokerAddr\", \"testTopic\", 12345L, 3000L);\n\n        assertEquals(CheckRocksdbCqWriteResult.CheckStatus.CHECK_OK.getValue(), result.getCheckStatus());\n    }\n\n    private Properties createProperties() {\n        Properties result = new Properties();\n        result.put(\"key\", \"value\");\n        return result;\n    }\n\n    private AclInfo createAclInfo() {\n        return AclInfo.of(\"subject\", Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), \"\");\n    }\n\n    private UserInfo createUserInfo() {\n        UserInfo result = new UserInfo();\n        result.setUsername(\"username\");\n        result.setPassword(\"password\");\n        result.setUserStatus(\"userStatus\");\n        result.setUserType(\"userType\");\n        return result;\n    }\n\n    private void setResponseHeader(CommandCustomHeader responseHeader) throws RemotingCommandException {\n        when(response.decodeCommandCustomHeader(any())).thenReturn(responseHeader);\n    }\n\n    private void setResponseBody(Object responseBody) {\n        when(response.getBody()).thenReturn(RemotingSerializable.encode(responseBody));\n    }\n\n    private void mockInvokeSync() throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException {\n        when(response.getCode()).thenReturn(ResponseCode.SUCCESS);\n        when(response.getVersion()).thenReturn(1);\n        when(remotingClient.invokeSync(any(), any(), anyLong())).thenReturn(response);\n        when(remotingClient.getNameServerAddressList()).thenReturn(Collections.singletonList(defaultNsAddr));\n    }\n\n    private void setTopAddressing() throws NoSuchFieldException, IllegalAccessException {\n        TopAddressing topAddressing = mock(TopAddressing.class);\n        setField(mqClientAPI, \"topAddressing\", topAddressing);\n        when(topAddressing.fetchNSAddr()).thenReturn(defaultNsAddr);\n    }\n\n    private void setField(final Object target, final String fieldName, final Object newValue) throws NoSuchFieldException, IllegalAccessException {\n        Class<?> clazz = target.getClass();\n        Field field = clazz.getDeclaredField(fieldName);\n        field.setAccessible(true);\n        field.set(target, newValue);\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/impl/admin/MqClientAdminImplTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.admin;\n\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.RemotingClient;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.admin.ConsumeStats;\nimport org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;\nimport org.apache.rocketmq.remoting.protocol.body.ClusterInfo;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerConnection;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.body.GroupList;\nimport org.apache.rocketmq.remoting.protocol.body.QueryConsumeTimeSpanBody;\nimport org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan;\nimport org.apache.rocketmq.remoting.protocol.body.ResetOffsetBody;\nimport org.apache.rocketmq.remoting.protocol.body.TopicList;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.CreateTopicRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.DeleteSubscriptionGroupRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.DeleteTopicRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumeStatsRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerConnectionListRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetTopicStatsInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumeTimeSpanRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QuerySubscriptionByConsumerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryTopicConsumeByWhoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryTopicsByConsumerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ViewMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.DeleteKVConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.DeleteTopicFromNamesrvRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ExecutionException;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertThrows;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class MqClientAdminImplTest {\n\n    @Mock\n    private RemotingClient remotingClient;\n\n    @Mock\n    private RemotingCommand response;\n\n    private MqClientAdminImpl mqClientAdminImpl;\n\n    private final String defaultTopic = \"defaultTopic\";\n\n    private final String defaultBrokerAddr = \"127.0.0.1:10911\";\n\n    private final long defaultTimeout = 3000L;\n\n    @Before\n    public void init() throws RemotingException, InterruptedException, MQClientException {\n        mqClientAdminImpl = new MqClientAdminImpl(remotingClient);\n        when(remotingClient.invoke(any(String.class), any(RemotingCommand.class), any(Long.class))).thenReturn(CompletableFuture.completedFuture(response));\n    }\n\n    @Test\n    public void assertQueryMessageWithSuccess() throws Exception {\n        setResponseSuccess(getMessageResult());\n        QueryMessageRequestHeader requestHeader = mock(QueryMessageRequestHeader.class);\n        when(requestHeader.getTopic()).thenReturn(defaultTopic);\n        when(requestHeader.getKey()).thenReturn(\"keys\");\n        CompletableFuture<List<MessageExt>> actual = mqClientAdminImpl.queryMessage(defaultBrokerAddr, false, false, requestHeader, defaultTimeout);\n        List<MessageExt> messageExtList = actual.get();\n        assertNotNull(messageExtList);\n        assertEquals(1, messageExtList.size());\n    }\n\n    @Test\n    public void assertQueryMessageWithNotFound() throws Exception {\n        when(response.getCode()).thenReturn(ResponseCode.QUERY_NOT_FOUND);\n        QueryMessageRequestHeader requestHeader = mock(QueryMessageRequestHeader.class);\n        CompletableFuture<List<MessageExt>> actual = mqClientAdminImpl.queryMessage(defaultBrokerAddr, false, false, requestHeader, defaultTimeout);\n        List<MessageExt> messageExtList = actual.get();\n        assertNotNull(messageExtList);\n        assertEquals(0, messageExtList.size());\n    }\n\n    @Test\n    public void assertQueryMessageWithError() {\n        setResponseError();\n        QueryMessageRequestHeader requestHeader = mock(QueryMessageRequestHeader.class);\n        CompletableFuture<List<MessageExt>> actual = mqClientAdminImpl.queryMessage(defaultBrokerAddr, false, false, requestHeader, defaultTimeout);\n        Throwable thrown = assertThrows(ExecutionException.class, actual::get);\n        assertTrue(thrown.getCause() instanceof MQClientException);\n        MQClientException mqException = (MQClientException) thrown.getCause();\n        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());\n        assertTrue(mqException.getMessage().contains(\"CODE: 1  DESC: null\"));\n    }\n\n    @Test\n    public void assertGetTopicStatsInfoWithSuccess() throws Exception {\n        TopicStatsTable responseBody = new TopicStatsTable();\n        setResponseSuccess(RemotingSerializable.encode(responseBody));\n        GetTopicStatsInfoRequestHeader requestHeader = mock(GetTopicStatsInfoRequestHeader.class);\n        CompletableFuture<TopicStatsTable> actual = mqClientAdminImpl.getTopicStatsInfo(defaultBrokerAddr, requestHeader, defaultTimeout);\n        TopicStatsTable topicStatsTable = actual.get();\n        assertNotNull(topicStatsTable);\n        assertEquals(0, topicStatsTable.getOffsetTable().size());\n    }\n\n    @Test\n    public void assertGetTopicStatsInfoWithError() {\n        setResponseError();\n        GetTopicStatsInfoRequestHeader requestHeader = mock(GetTopicStatsInfoRequestHeader.class);\n        CompletableFuture<TopicStatsTable> actual = mqClientAdminImpl.getTopicStatsInfo(defaultBrokerAddr, requestHeader, defaultTimeout);\n        Throwable thrown = assertThrows(ExecutionException.class, actual::get);\n        assertTrue(thrown.getCause() instanceof MQClientException);\n        MQClientException mqException = (MQClientException) thrown.getCause();\n        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());\n        assertTrue(mqException.getMessage().contains(\"CODE: 1  DESC: null\"));\n    }\n\n    @Test\n    public void assertQueryConsumeTimeSpanWithSuccess() throws Exception {\n        QueryConsumeTimeSpanBody responseBody = new QueryConsumeTimeSpanBody();\n        setResponseSuccess(RemotingSerializable.encode(responseBody));\n        QueryConsumeTimeSpanRequestHeader requestHeader = mock(QueryConsumeTimeSpanRequestHeader.class);\n        CompletableFuture<List<QueueTimeSpan>> actual = mqClientAdminImpl.queryConsumeTimeSpan(defaultBrokerAddr, requestHeader, defaultTimeout);\n        List<QueueTimeSpan> queueTimeSpans = actual.get();\n        assertNotNull(queueTimeSpans);\n        assertEquals(0, queueTimeSpans.size());\n    }\n\n    @Test\n    public void assertQueryConsumeTimeSpanWithError() {\n        setResponseError();\n        QueryConsumeTimeSpanRequestHeader requestHeader = mock(QueryConsumeTimeSpanRequestHeader.class);\n        CompletableFuture<List<QueueTimeSpan>> actual = mqClientAdminImpl.queryConsumeTimeSpan(defaultBrokerAddr, requestHeader, defaultTimeout);\n        Throwable thrown = assertThrows(ExecutionException.class, actual::get);\n        assertTrue(thrown.getCause() instanceof MQClientException);\n        MQClientException mqException = (MQClientException) thrown.getCause();\n        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());\n        assertTrue(mqException.getMessage().contains(\"CODE: 1  DESC: null\"));\n    }\n\n    @Test\n    public void assertUpdateOrCreateTopicWithSuccess() throws Exception {\n        setResponseSuccess(null);\n        CreateTopicRequestHeader requestHeader = mock(CreateTopicRequestHeader.class);\n        CompletableFuture<Void> actual = mqClientAdminImpl.updateOrCreateTopic(defaultBrokerAddr, requestHeader, defaultTimeout);\n        assertNull(actual.get());\n    }\n\n    @Test\n    public void assertUpdateOrCreateTopicWithError() {\n        setResponseError();\n        CreateTopicRequestHeader requestHeader = mock(CreateTopicRequestHeader.class);\n        CompletableFuture<Void> actual = mqClientAdminImpl.updateOrCreateTopic(defaultBrokerAddr, requestHeader, defaultTimeout);\n        Throwable thrown = assertThrows(ExecutionException.class, actual::get);\n        assertTrue(thrown.getCause() instanceof MQClientException);\n        MQClientException mqException = (MQClientException) thrown.getCause();\n        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());\n        assertTrue(mqException.getMessage().contains(\"CODE: 1  DESC: null\"));\n    }\n\n    @Test\n    public void assertUpdateOrCreateSubscriptionGroupWithSuccess() throws Exception {\n        setResponseSuccess(null);\n        SubscriptionGroupConfig config = new SubscriptionGroupConfig();\n        CompletableFuture<Void> actual = mqClientAdminImpl.updateOrCreateSubscriptionGroup(defaultBrokerAddr, config, defaultTimeout);\n        assertNull(actual.get());\n    }\n\n    @Test\n    public void assertUpdateOrCreateSubscriptionGroupWithError() {\n        setResponseError();\n        SubscriptionGroupConfig config = new SubscriptionGroupConfig();\n        CompletableFuture<Void> actual = mqClientAdminImpl.updateOrCreateSubscriptionGroup(defaultBrokerAddr, config, defaultTimeout);\n        Throwable thrown = assertThrows(ExecutionException.class, actual::get);\n        assertTrue(thrown.getCause() instanceof MQClientException);\n        MQClientException mqException = (MQClientException) thrown.getCause();\n        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());\n        assertTrue(mqException.getMessage().contains(\"CODE: 1  DESC: null\"));\n    }\n\n    @Test\n    public void assertDeleteTopicInBrokerWithSuccess() throws Exception {\n        setResponseSuccess(null);\n        DeleteTopicRequestHeader requestHeader = mock(DeleteTopicRequestHeader.class);\n        CompletableFuture<Void> actual = mqClientAdminImpl.deleteTopicInBroker(defaultBrokerAddr, requestHeader, defaultTimeout);\n        assertNull(actual.get());\n    }\n\n    @Test\n    public void assertDeleteTopicInBrokerWithError() {\n        setResponseError();\n        DeleteTopicRequestHeader requestHeader = mock(DeleteTopicRequestHeader.class);\n        CompletableFuture<Void> actual = mqClientAdminImpl.deleteTopicInBroker(defaultBrokerAddr, requestHeader, defaultTimeout);\n        Throwable thrown = assertThrows(ExecutionException.class, actual::get);\n        assertTrue(thrown.getCause() instanceof MQClientException);\n        MQClientException mqException = (MQClientException) thrown.getCause();\n        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());\n        assertTrue(mqException.getMessage().contains(\"CODE: 1  DESC: null\"));\n    }\n\n    @Test\n    public void assertDeleteTopicInNameserverWithSuccess() throws Exception {\n        setResponseSuccess(null);\n        DeleteTopicFromNamesrvRequestHeader requestHeader = mock(DeleteTopicFromNamesrvRequestHeader.class);\n        CompletableFuture<Void> actual = mqClientAdminImpl.deleteTopicInNameserver(defaultBrokerAddr, requestHeader, defaultTimeout);\n        assertNull(actual.get());\n    }\n\n    @Test\n    public void assertDeleteTopicInNameserverWithError() {\n        setResponseError();\n        DeleteTopicFromNamesrvRequestHeader requestHeader = mock(DeleteTopicFromNamesrvRequestHeader.class);\n        CompletableFuture<Void> actual = mqClientAdminImpl.deleteTopicInNameserver(defaultBrokerAddr, requestHeader, defaultTimeout);\n        Throwable thrown = assertThrows(ExecutionException.class, actual::get);\n        assertTrue(thrown.getCause() instanceof MQClientException);\n        MQClientException mqException = (MQClientException) thrown.getCause();\n        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());\n        assertTrue(mqException.getMessage().contains(\"CODE: 1  DESC: null\"));\n    }\n\n    @Test\n    public void assertDeleteKvConfigWithSuccess() throws Exception {\n        setResponseSuccess(null);\n        DeleteKVConfigRequestHeader requestHeader = mock(DeleteKVConfigRequestHeader.class);\n        CompletableFuture<Void> actual = mqClientAdminImpl.deleteKvConfig(defaultBrokerAddr, requestHeader, defaultTimeout);\n        assertNull(actual.get());\n    }\n\n    @Test\n    public void assertDeleteKvConfigWithError() {\n        setResponseError();\n        DeleteKVConfigRequestHeader requestHeader = mock(DeleteKVConfigRequestHeader.class);\n        CompletableFuture<Void> actual = mqClientAdminImpl.deleteKvConfig(defaultBrokerAddr, requestHeader, defaultTimeout);\n        Throwable thrown = assertThrows(ExecutionException.class, actual::get);\n        assertTrue(thrown.getCause() instanceof MQClientException);\n        MQClientException mqException = (MQClientException) thrown.getCause();\n        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());\n        assertTrue(mqException.getMessage().contains(\"CODE: 1  DESC: null\"));\n    }\n\n    @Test\n    public void assertDeleteSubscriptionGroupWithSuccess() throws Exception {\n        setResponseSuccess(null);\n        DeleteSubscriptionGroupRequestHeader requestHeader = mock(DeleteSubscriptionGroupRequestHeader.class);\n        CompletableFuture<Void> actual = mqClientAdminImpl.deleteSubscriptionGroup(defaultBrokerAddr, requestHeader, defaultTimeout);\n        assertNull(actual.get());\n    }\n\n    @Test\n    public void assertDeleteSubscriptionGroupWithError() {\n        setResponseError();\n        DeleteSubscriptionGroupRequestHeader requestHeader = mock(DeleteSubscriptionGroupRequestHeader.class);\n        CompletableFuture<Void> actual = mqClientAdminImpl.deleteSubscriptionGroup(defaultBrokerAddr, requestHeader, defaultTimeout);\n        Throwable thrown = assertThrows(ExecutionException.class, actual::get);\n        assertTrue(thrown.getCause() instanceof MQClientException);\n        MQClientException mqException = (MQClientException) thrown.getCause();\n        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());\n        assertTrue(mqException.getMessage().contains(\"CODE: 1  DESC: null\"));\n    }\n\n    @Test\n    public void assertInvokeBrokerToResetOffsetWithSuccess() throws Exception {\n        ResetOffsetBody responseBody = new ResetOffsetBody();\n        setResponseSuccess(RemotingSerializable.encode(responseBody));\n        ResetOffsetRequestHeader requestHeader = mock(ResetOffsetRequestHeader.class);\n        CompletableFuture<Map<MessageQueue, Long>> actual = mqClientAdminImpl.invokeBrokerToResetOffset(defaultBrokerAddr, requestHeader, defaultTimeout);\n        assertEquals(0, actual.get().size());\n    }\n\n    @Test\n    public void assertInvokeBrokerToResetOffsetWithError() {\n        setResponseError();\n        ResetOffsetRequestHeader requestHeader = mock(ResetOffsetRequestHeader.class);\n        CompletableFuture<Map<MessageQueue, Long>> actual = mqClientAdminImpl.invokeBrokerToResetOffset(defaultBrokerAddr, requestHeader, defaultTimeout);\n        Throwable thrown = assertThrows(ExecutionException.class, actual::get);\n        assertTrue(thrown.getCause() instanceof MQClientException);\n        MQClientException mqException = (MQClientException) thrown.getCause();\n        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());\n        assertTrue(mqException.getMessage().contains(\"CODE: 1  DESC: null\"));\n    }\n\n    @Test\n    public void assertViewMessageWithSuccess() throws Exception {\n        setResponseSuccess(getMessageResult());\n        ViewMessageRequestHeader requestHeader = mock(ViewMessageRequestHeader.class);\n        CompletableFuture<MessageExt> actual = mqClientAdminImpl.viewMessage(defaultBrokerAddr, requestHeader, defaultTimeout);\n        MessageExt result = actual.get();\n        assertNotNull(result);\n        assertEquals(defaultTopic, result.getTopic());\n    }\n\n    @Test\n    public void assertViewMessageWithError() {\n        setResponseError();\n        ViewMessageRequestHeader requestHeader = mock(ViewMessageRequestHeader.class);\n        CompletableFuture<MessageExt> actual = mqClientAdminImpl.viewMessage(defaultBrokerAddr, requestHeader, defaultTimeout);\n        Throwable thrown = assertThrows(ExecutionException.class, actual::get);\n        assertTrue(thrown.getCause() instanceof MQClientException);\n        MQClientException mqException = (MQClientException) thrown.getCause();\n        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());\n        assertTrue(mqException.getMessage().contains(\"CODE: 1  DESC: null\"));\n    }\n\n    @Test\n    public void assertGetBrokerClusterInfoWithSuccess() throws Exception {\n        ClusterInfo responseBody = new ClusterInfo();\n        setResponseSuccess(RemotingSerializable.encode(responseBody));\n        CompletableFuture<ClusterInfo> actual = mqClientAdminImpl.getBrokerClusterInfo(defaultBrokerAddr, defaultTimeout);\n        ClusterInfo result = actual.get();\n        assertNotNull(result);\n    }\n\n    @Test\n    public void assertGetBrokerClusterInfoWithError() {\n        setResponseError();\n        CompletableFuture<ClusterInfo> actual = mqClientAdminImpl.getBrokerClusterInfo(defaultBrokerAddr, defaultTimeout);\n        Throwable thrown = assertThrows(ExecutionException.class, actual::get);\n        assertTrue(thrown.getCause() instanceof MQClientException);\n        MQClientException mqException = (MQClientException) thrown.getCause();\n        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());\n        assertTrue(mqException.getMessage().contains(\"CODE: 1  DESC: null\"));\n    }\n\n    @Test\n    public void assertGetConsumerConnectionListWithSuccess() throws Exception {\n        ConsumerConnection responseBody = new ConsumerConnection();\n        setResponseSuccess(RemotingSerializable.encode(responseBody));\n        GetConsumerConnectionListRequestHeader requestHeader = mock(GetConsumerConnectionListRequestHeader.class);\n        CompletableFuture<ConsumerConnection> actual = mqClientAdminImpl.getConsumerConnectionList(defaultBrokerAddr, requestHeader, defaultTimeout);\n        ConsumerConnection result = actual.get();\n        assertNotNull(result);\n        assertEquals(0, result.getConnectionSet().size());\n    }\n\n    @Test\n    public void assertGetConsumerConnectionListWithError() {\n        setResponseError();\n        GetConsumerConnectionListRequestHeader requestHeader = mock(GetConsumerConnectionListRequestHeader.class);\n        CompletableFuture<ConsumerConnection> actual = mqClientAdminImpl.getConsumerConnectionList(defaultBrokerAddr, requestHeader, defaultTimeout);\n        Throwable thrown = assertThrows(ExecutionException.class, actual::get);\n        assertTrue(thrown.getCause() instanceof MQClientException);\n        MQClientException mqException = (MQClientException) thrown.getCause();\n        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());\n        assertTrue(mqException.getMessage().contains(\"CODE: 1  DESC: null\"));\n    }\n\n    @Test\n    public void assertQueryTopicsByConsumerWithSuccess() throws Exception {\n        TopicList responseBody = new TopicList();\n        setResponseSuccess(RemotingSerializable.encode(responseBody));\n        QueryTopicsByConsumerRequestHeader requestHeader = mock(QueryTopicsByConsumerRequestHeader.class);\n        CompletableFuture<TopicList> actual = mqClientAdminImpl.queryTopicsByConsumer(defaultBrokerAddr, requestHeader, defaultTimeout);\n        TopicList result = actual.get();\n        assertNotNull(result);\n        assertEquals(0, result.getTopicList().size());\n    }\n\n    @Test\n    public void assertQueryTopicsByConsumerWithError() {\n        setResponseError();\n        QueryTopicsByConsumerRequestHeader requestHeader = mock(QueryTopicsByConsumerRequestHeader.class);\n        CompletableFuture<TopicList> actual = mqClientAdminImpl.queryTopicsByConsumer(defaultBrokerAddr, requestHeader, defaultTimeout);\n        Throwable thrown = assertThrows(ExecutionException.class, actual::get);\n        assertTrue(thrown.getCause() instanceof MQClientException);\n        MQClientException mqException = (MQClientException) thrown.getCause();\n        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());\n        assertTrue(mqException.getMessage().contains(\"CODE: 1  DESC: null\"));\n    }\n\n    @Test\n    public void assertQuerySubscriptionByConsumerWithSuccess() throws Exception {\n        SubscriptionData responseBody = new SubscriptionData();\n        setResponseSuccess(RemotingSerializable.encode(responseBody));\n        QuerySubscriptionByConsumerRequestHeader requestHeader = mock(QuerySubscriptionByConsumerRequestHeader.class);\n        CompletableFuture<SubscriptionData> actual = mqClientAdminImpl.querySubscriptionByConsumer(defaultBrokerAddr, requestHeader, defaultTimeout);\n        assertNull(actual.get());\n    }\n\n    @Test\n    public void assertQuerySubscriptionByConsumerWithError() {\n        setResponseError();\n        QuerySubscriptionByConsumerRequestHeader requestHeader = mock(QuerySubscriptionByConsumerRequestHeader.class);\n        CompletableFuture<SubscriptionData> actual = mqClientAdminImpl.querySubscriptionByConsumer(defaultBrokerAddr, requestHeader, defaultTimeout);\n        Throwable thrown = assertThrows(ExecutionException.class, actual::get);\n        assertTrue(thrown.getCause() instanceof MQClientException);\n        MQClientException mqException = (MQClientException) thrown.getCause();\n        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());\n        assertTrue(mqException.getMessage().contains(\"CODE: 1  DESC: null\"));\n    }\n\n    @Test\n    public void assertGetConsumeStatsWithSuccess() throws Exception {\n        ConsumeStats responseBody = new ConsumeStats();\n        setResponseSuccess(RemotingSerializable.encode(responseBody));\n        GetConsumeStatsRequestHeader requestHeader = mock(GetConsumeStatsRequestHeader.class);\n        CompletableFuture<ConsumeStats> actual = mqClientAdminImpl.getConsumeStats(defaultBrokerAddr, requestHeader, defaultTimeout);\n        ConsumeStats result = actual.get();\n        assertNotNull(result);\n        assertEquals(0, result.getOffsetTable().size());\n    }\n\n    @Test\n    public void assertGetConsumeStatsWithError() {\n        setResponseError();\n        GetConsumeStatsRequestHeader requestHeader = mock(GetConsumeStatsRequestHeader.class);\n        CompletableFuture<ConsumeStats> actual = mqClientAdminImpl.getConsumeStats(defaultBrokerAddr, requestHeader, defaultTimeout);\n        Throwable thrown = assertThrows(ExecutionException.class, actual::get);\n        assertTrue(thrown.getCause() instanceof MQClientException);\n        MQClientException mqException = (MQClientException) thrown.getCause();\n        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());\n        assertTrue(mqException.getMessage().contains(\"CODE: 1  DESC: null\"));\n    }\n\n    @Test\n    public void assertQueryTopicConsumeByWhoWithSuccess() throws Exception {\n        GroupList responseBody = new GroupList();\n        setResponseSuccess(RemotingSerializable.encode(responseBody));\n        QueryTopicConsumeByWhoRequestHeader requestHeader = mock(QueryTopicConsumeByWhoRequestHeader.class);\n        CompletableFuture<GroupList> actual = mqClientAdminImpl.queryTopicConsumeByWho(defaultBrokerAddr, requestHeader, defaultTimeout);\n        GroupList result = actual.get();\n        assertNotNull(result);\n        assertEquals(0, result.getGroupList().size());\n    }\n\n    @Test\n    public void assertQueryTopicConsumeByWhoWithError() {\n        setResponseError();\n        QueryTopicConsumeByWhoRequestHeader requestHeader = mock(QueryTopicConsumeByWhoRequestHeader.class);\n        CompletableFuture<GroupList> actual = mqClientAdminImpl.queryTopicConsumeByWho(defaultBrokerAddr, requestHeader, defaultTimeout);\n        Throwable thrown = assertThrows(ExecutionException.class, actual::get);\n        assertTrue(thrown.getCause() instanceof MQClientException);\n        MQClientException mqException = (MQClientException) thrown.getCause();\n        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());\n        assertTrue(mqException.getMessage().contains(\"CODE: 1  DESC: null\"));\n    }\n\n    @Test\n    public void assertGetConsumerRunningInfoWithSuccess() throws Exception {\n        ConsumerRunningInfo responseBody = new ConsumerRunningInfo();\n        setResponseSuccess(RemotingSerializable.encode(responseBody));\n        GetConsumerRunningInfoRequestHeader requestHeader = mock(GetConsumerRunningInfoRequestHeader.class);\n        CompletableFuture<ConsumerRunningInfo> actual = mqClientAdminImpl.getConsumerRunningInfo(defaultBrokerAddr, requestHeader, defaultTimeout);\n        ConsumerRunningInfo result = actual.get();\n        assertNotNull(result);\n        assertEquals(0, result.getProperties().size());\n    }\n\n    @Test\n    public void assertGetConsumerRunningInfoWithError() {\n        setResponseError();\n        GetConsumerRunningInfoRequestHeader requestHeader = mock(GetConsumerRunningInfoRequestHeader.class);\n        CompletableFuture<ConsumerRunningInfo> actual = mqClientAdminImpl.getConsumerRunningInfo(defaultBrokerAddr, requestHeader, defaultTimeout);\n        Throwable thrown = assertThrows(ExecutionException.class, actual::get);\n        assertTrue(thrown.getCause() instanceof MQClientException);\n        MQClientException mqException = (MQClientException) thrown.getCause();\n        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());\n        assertTrue(mqException.getMessage().contains(\"CODE: 1  DESC: null\"));\n    }\n\n    @Test\n    public void assertConsumeMessageDirectlyWithSuccess() throws Exception {\n        ConsumeMessageDirectlyResult responseBody = new ConsumeMessageDirectlyResult();\n        setResponseSuccess(RemotingSerializable.encode(responseBody));\n        ConsumeMessageDirectlyResultRequestHeader requestHeader = mock(ConsumeMessageDirectlyResultRequestHeader.class);\n        CompletableFuture<ConsumeMessageDirectlyResult> actual = mqClientAdminImpl.consumeMessageDirectly(defaultBrokerAddr, requestHeader, defaultTimeout);\n        ConsumeMessageDirectlyResult result = actual.get();\n        assertNotNull(result);\n        assertTrue(result.isAutoCommit());\n    }\n\n    @Test\n    public void assertConsumeMessageDirectlyWithError() {\n        setResponseError();\n        ConsumeMessageDirectlyResultRequestHeader requestHeader = mock(ConsumeMessageDirectlyResultRequestHeader.class);\n        CompletableFuture<ConsumeMessageDirectlyResult> actual = mqClientAdminImpl.consumeMessageDirectly(defaultBrokerAddr, requestHeader, defaultTimeout);\n        Throwable thrown = assertThrows(ExecutionException.class, actual::get);\n        assertTrue(thrown.getCause() instanceof MQClientException);\n        MQClientException mqException = (MQClientException) thrown.getCause();\n        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());\n        assertTrue(mqException.getMessage().contains(\"CODE: 1  DESC: null\"));\n    }\n\n    private byte[] getMessageResult() throws Exception {\n        byte[] bytes = MessageDecoder.encode(createMessageExt(), false);\n        ByteBuffer byteBuffer = ByteBuffer.allocate(bytes.length);\n        byteBuffer.put(bytes);\n        return byteBuffer.array();\n    }\n\n    private MessageExt createMessageExt() {\n        MessageExt result = new MessageExt();\n        result.setBody(\"body\".getBytes(StandardCharsets.UTF_8));\n        result.setTopic(defaultTopic);\n        result.setBrokerName(\"defaultBroker\");\n        result.putUserProperty(\"key\", \"value\");\n        result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, \"defaultGroup\");\n        result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, \"TX1\");\n        result.setKeys(\"keys\");\n        SocketAddress bornHost = new InetSocketAddress(\"127.0.0.1\", 12911);\n        SocketAddress storeHost = new InetSocketAddress(\"127.0.0.1\", 10911);\n        result.setBornHost(bornHost);\n        result.setStoreHost(storeHost);\n        return result;\n    }\n\n    private void setResponseSuccess(byte[] body) {\n        when(response.getCode()).thenReturn(ResponseCode.SUCCESS);\n        when(response.getBody()).thenReturn(body);\n    }\n\n    private void setResponseError() {\n        when(response.getCode()).thenReturn(ResponseCode.SYSTEM_ERROR);\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport java.io.ByteArrayOutputStream;\nimport java.lang.reflect.Field;\nimport java.net.InetSocketAddress;\nimport java.util.Collection;\nimport java.util.Collections;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.atomic.AtomicReference;\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.PullCallback;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.FindBrokerResult;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.MQClientManager;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.client.stat.ConsumerStatsManager;\nimport org.apache.rocketmq.common.message.MessageClientExt;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.stats.StatsItem;\nimport org.apache.rocketmq.common.stats.StatsItemSet;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeStatus;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.junit.AfterClass;\nimport org.junit.BeforeClass;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.invocation.InvocationOnMock;\nimport org.mockito.junit.MockitoJUnitRunner;\nimport org.mockito.stubbing.Answer;\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyBoolean;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.nullable;\nimport static org.mockito.Mockito.doReturn;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.when;\nimport org.mockito.Mockito;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ConsumeMessageConcurrentlyServiceTest {\n\n    private static String consumerGroup;\n\n    private static String topic = \"FooBar\";\n\n    private static String brokerName = \"BrokerA\";\n\n    private static MQClientInstance mQClientFactory;\n\n    @Mock\n    private static MQClientAPIImpl mQClientAPIImpl;\n\n    private static PullAPIWrapper pullAPIWrapper;\n\n    private static RebalancePushImpl rebalancePushImpl;\n\n    private static DefaultMQPushConsumer pushConsumer;\n\n    @BeforeClass\n    public static void init() throws Exception {\n        mQClientAPIImpl = Mockito.mock(MQClientAPIImpl.class);\n        ConcurrentMap<String, MQClientInstance> factoryTable = (ConcurrentMap<String, MQClientInstance>) FieldUtils.readDeclaredField(MQClientManager.getInstance(), \"factoryTable\", true);\n        Collection<MQClientInstance> instances = factoryTable.values();\n        for (MQClientInstance instance : instances) {\n            instance.shutdown();\n        }\n        factoryTable.clear();\n        consumerGroup = \"FooBarGroup\" + System.currentTimeMillis();\n        pushConsumer = new DefaultMQPushConsumer(consumerGroup);\n        pushConsumer.setNamesrvAddr(\"127.0.0.1:9876\");\n        pushConsumer.setPullInterval(60 * 1000);\n        pushConsumer.registerMessageListener(new MessageListenerConcurrently() {\n\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {\n                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n            }\n        });\n        DefaultMQPushConsumerImpl pushConsumerImpl = pushConsumer.getDefaultMQPushConsumerImpl();\n        rebalancePushImpl = spy(new RebalancePushImpl(pushConsumer.getDefaultMQPushConsumerImpl()));\n        Field field = DefaultMQPushConsumerImpl.class.getDeclaredField(\"rebalanceImpl\");\n        field.setAccessible(true);\n        field.set(pushConsumerImpl, rebalancePushImpl);\n        pushConsumer.subscribe(topic, \"*\");\n        // suppress updateTopicRouteInfoFromNameServer\n        pushConsumer.changeInstanceNameToPID();\n        mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(pushConsumer, (RPCHook) FieldUtils.readDeclaredField(pushConsumerImpl, \"rpcHook\", true));\n        mQClientFactory = spy(mQClientFactory);\n        field = DefaultMQPushConsumerImpl.class.getDeclaredField(\"mQClientFactory\");\n        field.setAccessible(true);\n        field.set(pushConsumerImpl, mQClientFactory);\n        factoryTable.put(pushConsumer.buildMQClientId(), mQClientFactory);\n        field = MQClientInstance.class.getDeclaredField(\"mQClientAPIImpl\");\n        field.setAccessible(true);\n        field.set(mQClientFactory, mQClientAPIImpl);\n        pullAPIWrapper = spy(new PullAPIWrapper(mQClientFactory, consumerGroup, false));\n        field = DefaultMQPushConsumerImpl.class.getDeclaredField(\"pullAPIWrapper\");\n        field.setAccessible(true);\n        field.set(pushConsumerImpl, pullAPIWrapper);\n        pushConsumer.getDefaultMQPushConsumerImpl().getRebalanceImpl().setmQClientFactory(mQClientFactory);\n        when(mQClientFactory.getMQClientAPIImpl().pullMessage(anyString(), any(PullMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), nullable(PullCallback.class))).thenAnswer(new Answer<PullResult>() {\n\n            @Override\n            public PullResult answer(InvocationOnMock mock) throws Throwable {\n                PullMessageRequestHeader requestHeader = mock.getArgument(1);\n                MessageClientExt messageClientExt = new MessageClientExt();\n                messageClientExt.setTopic(topic);\n                messageClientExt.setQueueId(0);\n                messageClientExt.setMsgId(\"123\");\n                messageClientExt.setBody(new byte[] { 'a' });\n                messageClientExt.setOffsetMsgId(\"234\");\n                messageClientExt.setBornHost(new InetSocketAddress(8080));\n                messageClientExt.setStoreHost(new InetSocketAddress(8080));\n                PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.<MessageExt>singletonList(messageClientExt));\n                ((PullCallback) mock.getArgument(4)).onSuccess(pullResult);\n                return pullResult;\n            }\n        });\n        doReturn(new FindBrokerResult(\"127.0.0.1:10912\", false)).when(mQClientFactory).findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean());\n        doReturn(false).when(mQClientFactory).updateTopicRouteInfoFromNameServer(anyString());\n        Set<MessageQueue> messageQueueSet = new HashSet<>();\n        messageQueueSet.add(createPullRequest().getMessageQueue());\n        pushConsumer.getDefaultMQPushConsumerImpl().updateTopicSubscribeInfo(topic, messageQueueSet);\n        pushConsumer.start();\n    }\n\n    @Test\n    public void testPullMessage_ConsumeSuccess() throws InterruptedException, RemotingException, MQBrokerException, NoSuchFieldException, Exception {\n        final CountDownLatch countDownLatch = new CountDownLatch(1);\n        final AtomicReference<MessageExt> messageAtomic = new AtomicReference<>();\n        ConsumeMessageConcurrentlyService normalServie = new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() {\n\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {\n                messageAtomic.set(msgs.get(0));\n                countDownLatch.countDown();\n                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n            }\n        });\n        pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(normalServie);\n        PullMessageService pullMessageService = mQClientFactory.getPullMessageService();\n        pullMessageService.executePullRequestImmediately(createPullRequest());\n        countDownLatch.await();\n        Thread.sleep(1000);\n        ConsumeStatus stats = normalServie.getConsumerStatsManager().consumeStatus(pushConsumer.getDefaultMQPushConsumerImpl().groupName(), topic);\n        ConsumerStatsManager mgr = normalServie.getConsumerStatsManager();\n        Field statItmeSetField = mgr.getClass().getDeclaredField(\"topicAndGroupConsumeOKTPS\");\n        statItmeSetField.setAccessible(true);\n        StatsItemSet itemSet = (StatsItemSet) statItmeSetField.get(mgr);\n        StatsItem item = itemSet.getAndCreateStatsItem(topic + \"@\" + pushConsumer.getDefaultMQPushConsumerImpl().groupName());\n        assertThat(item.getValue().sum()).isGreaterThan(0L);\n        MessageExt msg = messageAtomic.get();\n        assertThat(msg).isNotNull();\n        assertThat(msg.getTopic()).isEqualTo(topic);\n        assertThat(msg.getBody()).isEqualTo(new byte[] { 'a' });\n    }\n\n    @AfterClass\n    public static void terminate() {\n        pushConsumer.shutdown();\n    }\n\n    private static PullRequest createPullRequest() {\n        PullRequest pullRequest = new PullRequest();\n        pullRequest.setConsumerGroup(consumerGroup);\n        pullRequest.setNextOffset(1024);\n        MessageQueue messageQueue = new MessageQueue();\n        messageQueue.setBrokerName(brokerName);\n        messageQueue.setQueueId(0);\n        messageQueue.setTopic(topic);\n        pullRequest.setMessageQueue(messageQueue);\n        ProcessQueue processQueue = new ProcessQueue();\n        processQueue.setLocked(true);\n        processQueue.setLastLockTimestamp(System.currentTimeMillis());\n        pullRequest.setProcessQueue(processQueue);\n        return pullRequest;\n    }\n\n    private static PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus, List<MessageExt> messageExtList) throws Exception {\n        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();\n        for (MessageExt messageExt : messageExtList) {\n            outputStream.write(MessageDecoder.encode(messageExt, false));\n        }\n        return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray());\n    }\n\n    @Test\n    public void testConsumeThreadName() throws Exception {\n        final CountDownLatch countDownLatch = new CountDownLatch(1);\n        final AtomicReference<String> consumeThreadName = new AtomicReference<>();\n        StringBuilder consumeGroup2 = new StringBuilder();\n        for (int i = 0; i < 101; i++) {\n            consumeGroup2.append(i).append(\"#\");\n        }\n        pushConsumer.setConsumerGroup(consumeGroup2.toString());\n        ConsumeMessageConcurrentlyService normalServie = new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() {\n\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {\n                consumeThreadName.set(Thread.currentThread().getName());\n                countDownLatch.countDown();\n                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n            }\n        });\n        pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(normalServie);\n        PullMessageService pullMessageService = mQClientFactory.getPullMessageService();\n        pullMessageService.executePullRequestImmediately(createPullRequest());\n        countDownLatch.await();\n        if (consumeGroup2.length() <= 100) {\n            assertThat(consumeThreadName.get()).startsWith(\"ConsumeMessageThread_\" + consumeGroup2 + \"_\");\n        } else {\n            assertThat(consumeThreadName.get()).startsWith(\"ConsumeMessageThread_\" + consumeGroup2.substring(0, 100) + \"_\");\n        }\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport java.io.ByteArrayOutputStream;\nimport java.lang.reflect.Field;\nimport java.net.InetSocketAddress;\nimport java.util.Collection;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.atomic.AtomicReference;\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.PullCallback;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.FindBrokerResult;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.MQClientManager;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.message.MessageClientExt;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.protocol.body.CMResult;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.Mock;\nimport org.mockito.invocation.InvocationOnMock;\nimport org.mockito.stubbing.Answer;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyBoolean;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.nullable;\nimport static org.mockito.Mockito.doReturn;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.when;\n\npublic class ConsumeMessageOrderlyServiceTest {\n    private String consumerGroup;\n    private String topic = \"FooBar\";\n    private String brokerName = \"BrokerA\";\n    private DefaultMQPushConsumer pushConsumer;\n    private MQClientInstance mQClientFactory;\n\n    @Mock\n    private MQClientAPIImpl mQClientAPIImpl;\n    private PullAPIWrapper pullAPIWrapper;\n    private RebalancePushImpl rebalancePushImpl;\n    @Before\n    public void init() throws Exception {\n        ConcurrentMap<String, MQClientInstance> factoryTable = (ConcurrentMap<String, MQClientInstance>) FieldUtils.readDeclaredField(MQClientManager.getInstance(), \"factoryTable\", true);\n        Collection<MQClientInstance> instances = factoryTable.values();\n        for (MQClientInstance instance : instances) {\n            instance.shutdown();\n        }\n        factoryTable.clear();\n        consumerGroup = \"FooBarGroup\" + System.currentTimeMillis();\n        pushConsumer = new DefaultMQPushConsumer(consumerGroup);\n\n        pushConsumer.setNamesrvAddr(\"127.0.0.1:9876\");\n        pushConsumer.setPullInterval(60 * 1000);\n\n        pushConsumer.registerMessageListener(new MessageListenerOrderly() {\n\n            @Override\n            public ConsumeOrderlyStatus consumeMessage(List<MessageExt> msgs, ConsumeOrderlyContext context) {\n                return ConsumeOrderlyStatus.SUCCESS;\n            }\n        });\n\n\n        DefaultMQPushConsumerImpl pushConsumerImpl = pushConsumer.getDefaultMQPushConsumerImpl();\n        rebalancePushImpl = spy(new RebalancePushImpl(pushConsumer.getDefaultMQPushConsumerImpl()));\n        Field field = DefaultMQPushConsumerImpl.class.getDeclaredField(\"rebalanceImpl\");\n        field.setAccessible(true);\n        field.set(pushConsumerImpl, rebalancePushImpl);\n        pushConsumer.subscribe(topic, \"*\");\n\n        // suppress updateTopicRouteInfoFromNameServer\n        pushConsumer.changeInstanceNameToPID();\n        mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(pushConsumer, (RPCHook) FieldUtils.readDeclaredField(pushConsumerImpl, \"rpcHook\", true));\n        mQClientFactory = spy(mQClientFactory);\n        field = DefaultMQPushConsumerImpl.class.getDeclaredField(\"mQClientFactory\");\n        field.setAccessible(true);\n        field.set(pushConsumerImpl, mQClientFactory);\n        factoryTable.put(pushConsumer.buildMQClientId(), mQClientFactory);\n\n        mQClientAPIImpl = mock(MQClientAPIImpl.class);\n        field = MQClientInstance.class.getDeclaredField(\"mQClientAPIImpl\");\n        field.setAccessible(true);\n        field.set(mQClientFactory, mQClientAPIImpl);\n\n        pullAPIWrapper = spy(new PullAPIWrapper(mQClientFactory, consumerGroup, false));\n        field = DefaultMQPushConsumerImpl.class.getDeclaredField(\"pullAPIWrapper\");\n        field.setAccessible(true);\n        field.set(pushConsumerImpl, pullAPIWrapper);\n\n        pushConsumer.getDefaultMQPushConsumerImpl().getRebalanceImpl().setmQClientFactory(mQClientFactory);\n\n        when(mQClientFactory.getMQClientAPIImpl().pullMessage(anyString(), any(PullMessageRequestHeader.class),\n                anyLong(), any(CommunicationMode.class), nullable(PullCallback.class)))\n                .thenAnswer(new Answer<PullResult>() {\n                    @Override\n                    public PullResult answer(InvocationOnMock mock) throws Throwable {\n                        PullMessageRequestHeader requestHeader = mock.getArgument(1);\n                        MessageClientExt messageClientExt = new MessageClientExt();\n                        messageClientExt.setTopic(topic);\n                        messageClientExt.setQueueId(0);\n                        messageClientExt.setMsgId(\"123\");\n                        messageClientExt.setBody(new byte[] {'a'});\n                        messageClientExt.setOffsetMsgId(\"234\");\n                        messageClientExt.setBornHost(new InetSocketAddress(8080));\n                        messageClientExt.setStoreHost(new InetSocketAddress(8080));\n                        PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.<MessageExt>singletonList(messageClientExt));\n                        ((PullCallback) mock.getArgument(4)).onSuccess(pullResult);\n                        return pullResult;\n                    }\n                });\n\n        doReturn(new FindBrokerResult(\"127.0.0.1:10912\", false)).when(mQClientFactory).findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean());\n        doReturn(false).when(mQClientFactory).updateTopicRouteInfoFromNameServer(anyString());\n        Set<MessageQueue> messageQueueSet = new HashSet<>();\n        messageQueueSet.add(createPullRequest().getMessageQueue());\n        pushConsumer.getDefaultMQPushConsumerImpl().updateTopicSubscribeInfo(topic, messageQueueSet);\n        pushConsumer.start();\n    }\n\n    @Test\n    public void testConsumeMessageDirectly_WithNoException() {\n        Map<ConsumeOrderlyStatus, CMResult> map = new HashMap();\n        map.put(ConsumeOrderlyStatus.SUCCESS, CMResult.CR_SUCCESS);\n        map.put(ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT, CMResult.CR_LATER);\n        map.put(ConsumeOrderlyStatus.COMMIT, CMResult.CR_COMMIT);\n        map.put(ConsumeOrderlyStatus.ROLLBACK, CMResult.CR_ROLLBACK);\n        map.put(null, CMResult.CR_RETURN_NULL);\n\n        for (ConsumeOrderlyStatus consumeOrderlyStatus : map.keySet()) {\n            final ConsumeOrderlyStatus status = consumeOrderlyStatus;\n            MessageListenerOrderly listenerOrderly = new MessageListenerOrderly() {\n                @Override\n                public ConsumeOrderlyStatus consumeMessage(List<MessageExt> msgs, ConsumeOrderlyContext context) {\n                    return status;\n                }\n            };\n\n            ConsumeMessageOrderlyService consumeMessageOrderlyService = new ConsumeMessageOrderlyService(pushConsumer.getDefaultMQPushConsumerImpl(), listenerOrderly);\n            MessageExt msg = new MessageExt();\n            msg.setTopic(topic);\n            assertTrue(consumeMessageOrderlyService.consumeMessageDirectly(msg, brokerName).getConsumeResult().equals(map.get(consumeOrderlyStatus)));\n        }\n\n    }\n\n    @Test\n    public void testConsumeMessageDirectly_WithException() {\n        MessageListenerOrderly listenerOrderly = new MessageListenerOrderly() {\n            @Override\n            public ConsumeOrderlyStatus consumeMessage(List<MessageExt> msgs, ConsumeOrderlyContext context) {\n                throw new RuntimeException();\n            }\n        };\n\n        ConsumeMessageOrderlyService consumeMessageOrderlyService = new ConsumeMessageOrderlyService(pushConsumer.getDefaultMQPushConsumerImpl(), listenerOrderly);\n        MessageExt msg = new MessageExt();\n        msg.setTopic(topic);\n        assertTrue(consumeMessageOrderlyService.consumeMessageDirectly(msg, brokerName).getConsumeResult().equals(CMResult.CR_THROW_EXCEPTION));\n    }\n\n    @Test\n    public void testConsumeThreadName() throws Exception {\n        final CountDownLatch countDownLatch = new CountDownLatch(1);\n        final AtomicReference<String> consumeThreadName = new AtomicReference<>();\n\n        StringBuilder consumeGroup2 = new StringBuilder();\n        for (int i = 0; i < 101; i++) {\n            consumeGroup2.append(i).append(\"#\");\n        }\n        pushConsumer.setConsumerGroup(consumeGroup2.toString());\n\n        MessageListenerOrderly listenerOrderly = new MessageListenerOrderly() {\n            @Override\n            public ConsumeOrderlyStatus consumeMessage(List<MessageExt> msgs, ConsumeOrderlyContext context) {\n                consumeThreadName.set(Thread.currentThread().getName());\n                countDownLatch.countDown();\n                return ConsumeOrderlyStatus.SUCCESS;\n            }\n        };\n        ConsumeMessageOrderlyService consumeMessageOrderlyService = new ConsumeMessageOrderlyService(pushConsumer.getDefaultMQPushConsumerImpl(), listenerOrderly);\n        pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(consumeMessageOrderlyService);\n\n\n        PullMessageService pullMessageService = mQClientFactory.getPullMessageService();\n        pullMessageService.executePullRequestImmediately(createPullRequest());\n        countDownLatch.await();\n        if (consumeGroup2.length() <= 100) {\n            assertThat(consumeThreadName.get()).startsWith(\"ConsumeMessageThread_\" + consumeGroup2 + \"_\");\n        } else {\n            assertThat(consumeThreadName.get()).startsWith(\"ConsumeMessageThread_\" + consumeGroup2.substring(0, 100) + \"_\");\n        }\n    }\n\n    private PullRequest createPullRequest() {\n        PullRequest pullRequest = new PullRequest();\n        pullRequest.setConsumerGroup(consumerGroup);\n        pullRequest.setNextOffset(1024);\n\n        MessageQueue messageQueue = new MessageQueue();\n        messageQueue.setBrokerName(brokerName);\n        messageQueue.setQueueId(0);\n        messageQueue.setTopic(topic);\n        pullRequest.setMessageQueue(messageQueue);\n        ProcessQueue processQueue = new ProcessQueue();\n        processQueue.setLocked(true);\n        processQueue.setLastLockTimestamp(System.currentTimeMillis());\n        pullRequest.setProcessQueue(processQueue);\n\n        return pullRequest;\n    }\n\n    private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus,\n                                           List<MessageExt> messageExtList) throws Exception {\n        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();\n        for (MessageExt messageExt : messageExtList) {\n            outputStream.write(MessageDecoder.encode(messageExt, false));\n        }\n        return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray());\n    }\n\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopConcurrentlyServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.stat.ConsumerStatsManager;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.body.CMResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.lang.reflect.Field;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.nio.charset.StandardCharsets;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ThreadPoolExecutor;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertThrows;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ConsumeMessagePopConcurrentlyServiceTest {\n\n    @Mock\n    private DefaultMQPushConsumerImpl defaultMQPushConsumerImpl;\n\n    @Mock\n    private MessageListenerConcurrently messageListener;\n\n    @Mock\n    private DefaultMQPushConsumer defaultMQPushConsumer;\n\n    private ConsumeMessagePopConcurrentlyService popService;\n\n    private final String defaultGroup = \"defaultGroup\";\n\n    private final String defaultBroker = \"defaultBroker\";\n\n    private final String defaultTopic = \"defaultTopic\";\n\n    @Before\n    public void init() throws Exception {\n        when(defaultMQPushConsumer.getConsumerGroup()).thenReturn(defaultGroup);\n        when(defaultMQPushConsumer.getConsumeThreadMin()).thenReturn(1);\n        when(defaultMQPushConsumer.getConsumeThreadMax()).thenReturn(3);\n        when(defaultMQPushConsumer.getConsumeMessageBatchMaxSize()).thenReturn(32);\n        when(defaultMQPushConsumerImpl.getDefaultMQPushConsumer()).thenReturn(defaultMQPushConsumer);\n        ConsumerStatsManager consumerStatsManager = mock(ConsumerStatsManager.class);\n        when(defaultMQPushConsumerImpl.getConsumerStatsManager()).thenReturn(consumerStatsManager);\n        popService = new ConsumeMessagePopConcurrentlyService(defaultMQPushConsumerImpl, messageListener);\n    }\n\n    @Test\n    public void testUpdateCorePoolSize() {\n        popService.updateCorePoolSize(2);\n        popService.incCorePoolSize();\n        popService.decCorePoolSize();\n        assertEquals(2, popService.getCorePoolSize());\n    }\n\n    @Test\n    public void testConsumeMessageDirectly() {\n        when(messageListener.consumeMessage(any(), any(ConsumeConcurrentlyContext.class))).thenReturn(ConsumeConcurrentlyStatus.CONSUME_SUCCESS);\n        ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker);\n        assertEquals(CMResult.CR_SUCCESS, actual.getConsumeResult());\n    }\n\n    @Test\n    public void testConsumeMessageDirectlyWithCrLater() {\n        when(messageListener.consumeMessage(any(), any(ConsumeConcurrentlyContext.class))).thenReturn(ConsumeConcurrentlyStatus.RECONSUME_LATER);\n        ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker);\n        assertEquals(CMResult.CR_LATER, actual.getConsumeResult());\n    }\n\n    @Test\n    public void testConsumeMessageDirectlyWithCrReturnNull() {\n        ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker);\n        assertEquals(CMResult.CR_RETURN_NULL, actual.getConsumeResult());\n    }\n\n    @Test\n    public void testConsumeMessageDirectlyWithCrThrowException() {\n        when(messageListener.consumeMessage(any(), any(ConsumeConcurrentlyContext.class))).thenThrow(new RuntimeException(\"exception\"));\n        ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker);\n        assertEquals(CMResult.CR_THROW_EXCEPTION, actual.getConsumeResult());\n    }\n\n    @Test\n    public void testShutdown() throws IllegalAccessException {\n        popService.shutdown(3000L);\n        Field scheduledExecutorServiceField = FieldUtils.getDeclaredField(popService.getClass(), \"scheduledExecutorService\", true);\n        Field consumeExecutorField = FieldUtils.getDeclaredField(popService.getClass(), \"consumeExecutor\", true);\n        ScheduledExecutorService scheduledExecutorService = (ScheduledExecutorService) scheduledExecutorServiceField.get(popService);\n        ThreadPoolExecutor consumeExecutor = (ThreadPoolExecutor) consumeExecutorField.get(popService);\n        assertTrue(scheduledExecutorService.isShutdown());\n        assertTrue(scheduledExecutorService.isTerminated());\n        assertTrue(consumeExecutor.isShutdown());\n        assertTrue(consumeExecutor.isTerminated());\n    }\n\n    @Test\n    public void testSubmitConsumeRequest() {\n        assertThrows(UnsupportedOperationException.class, () -> {\n            List<MessageExt> msgs = mock(List.class);\n            ProcessQueue processQueue = mock(ProcessQueue.class);\n            MessageQueue messageQueue = mock(MessageQueue.class);\n            popService.submitConsumeRequest(msgs, processQueue, messageQueue, false);\n        });\n    }\n\n    @Test\n    public void testSubmitPopConsumeRequest() throws IllegalAccessException {\n        List<MessageExt> msgs = Collections.singletonList(createMessageExt());\n        PopProcessQueue processQueue = mock(PopProcessQueue.class);\n        MessageQueue messageQueue = mock(MessageQueue.class);\n        ThreadPoolExecutor consumeExecutor = mock(ThreadPoolExecutor.class);\n        FieldUtils.writeDeclaredField(popService, \"consumeExecutor\", consumeExecutor, true);\n        popService.submitPopConsumeRequest(msgs, processQueue, messageQueue);\n        verify(consumeExecutor, times(1)).submit(any(Runnable.class));\n    }\n\n    @Test\n    public void testSubmitPopConsumeRequestWithMultiMsg() throws IllegalAccessException {\n        List<MessageExt> msgs = Arrays.asList(createMessageExt(), createMessageExt());\n        PopProcessQueue processQueue = mock(PopProcessQueue.class);\n        MessageQueue messageQueue = mock(MessageQueue.class);\n        ThreadPoolExecutor consumeExecutor = mock(ThreadPoolExecutor.class);\n        FieldUtils.writeDeclaredField(popService, \"consumeExecutor\", consumeExecutor, true);\n        when(defaultMQPushConsumer.getConsumeMessageBatchMaxSize()).thenReturn(1);\n        popService.submitPopConsumeRequest(msgs, processQueue, messageQueue);\n        verify(consumeExecutor, times(2)).submit(any(Runnable.class));\n    }\n\n    @Test\n    public void testProcessConsumeResult() {\n        ConsumeConcurrentlyContext context = mock(ConsumeConcurrentlyContext.class);\n        ConsumeMessagePopConcurrentlyService.ConsumeRequest consumeRequest = mock(ConsumeMessagePopConcurrentlyService.ConsumeRequest.class);\n        when(consumeRequest.getMsgs()).thenReturn(Arrays.asList(createMessageExt(), createMessageExt()));\n        MessageQueue messageQueue = mock(MessageQueue.class);\n        when(messageQueue.getTopic()).thenReturn(defaultTopic);\n        when(consumeRequest.getMessageQueue()).thenReturn(messageQueue);\n        PopProcessQueue processQueue = mock(PopProcessQueue.class);\n        when(processQueue.ack()).thenReturn(0);\n        when(consumeRequest.getPopProcessQueue()).thenReturn(processQueue);\n        when(defaultMQPushConsumerImpl.getPopDelayLevel()).thenReturn(new int[]{1, 10});\n        popService.processConsumeResult(ConsumeConcurrentlyStatus.CONSUME_SUCCESS, context, consumeRequest);\n        verify(defaultMQPushConsumerImpl, times(1)).ackAsync(any(MessageExt.class), any());\n    }\n\n    private MessageExt createMessageExt() {\n        MessageExt result = new MessageExt();\n        result.setBody(\"body\".getBytes(StandardCharsets.UTF_8));\n        result.setTopic(defaultTopic);\n        result.setBrokerName(defaultBroker);\n        result.putUserProperty(\"key\", \"value\");\n        result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, defaultGroup);\n        result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, \"TX1\");\n        long curTime = System.currentTimeMillis();\n        result.setBornTimestamp(curTime - 1000);\n        result.getProperties().put(MessageConst.PROPERTY_POP_CK, curTime + \" \" + curTime + \" \" + curTime + \" \" + curTime);\n        result.setKeys(\"keys\");\n        SocketAddress bornHost = new InetSocketAddress(\"127.0.0.1\", 12911);\n        SocketAddress storeHost = new InetSocketAddress(\"127.0.0.1\", 10911);\n        result.setBornHost(bornHost);\n        result.setStoreHost(storeHost);\n        return result;\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopOrderlyServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.stat.ConsumerStatsManager;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.body.CMResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.lang.reflect.Field;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.nio.charset.StandardCharsets;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ThreadPoolExecutor;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertThrows;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ConsumeMessagePopOrderlyServiceTest {\n\n    @Mock\n    private DefaultMQPushConsumerImpl defaultMQPushConsumerImpl;\n\n    @Mock\n    private MessageListenerOrderly messageListener;\n\n    @Mock\n    private DefaultMQPushConsumer defaultMQPushConsumer;\n\n    @Mock\n    private ConsumerStatsManager consumerStatsManager;\n\n    @Mock\n    private RebalanceImpl rebalanceImpl;\n\n    private ConsumeMessagePopOrderlyService popService;\n\n    private final String defaultGroup = \"defaultGroup\";\n\n    private final String defaultBroker = \"defaultBroker\";\n\n    private final String defaultTopic = \"defaultTopic\";\n\n    @Before\n    public void init() throws Exception {\n        when(defaultMQPushConsumer.getConsumerGroup()).thenReturn(defaultGroup);\n        when(defaultMQPushConsumer.getConsumeThreadMin()).thenReturn(1);\n        when(defaultMQPushConsumer.getConsumeThreadMax()).thenReturn(3);\n        when(defaultMQPushConsumerImpl.getDefaultMQPushConsumer()).thenReturn(defaultMQPushConsumer);\n        when(defaultMQPushConsumerImpl.getRebalanceImpl()).thenReturn(rebalanceImpl);\n        when(defaultMQPushConsumerImpl.getConsumerStatsManager()).thenReturn(consumerStatsManager);\n        MQClientInstance mQClientFactory = mock(MQClientInstance.class);\n        DefaultMQProducer defaultMQProducer = mock(DefaultMQProducer.class);\n        when(mQClientFactory.getDefaultMQProducer()).thenReturn(defaultMQProducer);\n        when(defaultMQPushConsumerImpl.getmQClientFactory()).thenReturn(mQClientFactory);\n        popService = new ConsumeMessagePopOrderlyService(defaultMQPushConsumerImpl, messageListener);\n    }\n\n    @Test\n    public void testShutdown() throws IllegalAccessException {\n        popService.shutdown(3000L);\n        Field scheduledExecutorServiceField = FieldUtils.getDeclaredField(popService.getClass(), \"scheduledExecutorService\", true);\n        Field consumeExecutorField = FieldUtils.getDeclaredField(popService.getClass(), \"consumeExecutor\", true);\n        ScheduledExecutorService scheduledExecutorService = (ScheduledExecutorService) scheduledExecutorServiceField.get(popService);\n        ThreadPoolExecutor consumeExecutor = (ThreadPoolExecutor) consumeExecutorField.get(popService);\n        assertTrue(scheduledExecutorService.isShutdown());\n        assertTrue(scheduledExecutorService.isTerminated());\n        assertTrue(consumeExecutor.isShutdown());\n        assertTrue(consumeExecutor.isTerminated());\n    }\n\n    @Test\n    public void testUnlockAllMessageQueues() {\n        popService.unlockAllMessageQueues();\n        verify(rebalanceImpl, times(1)).unlockAll(eq(false));\n    }\n\n    @Test\n    public void testUpdateCorePoolSize() {\n        popService.updateCorePoolSize(2);\n        popService.incCorePoolSize();\n        popService.decCorePoolSize();\n        assertEquals(2, popService.getCorePoolSize());\n    }\n\n    @Test\n    public void testConsumeMessageDirectly() {\n        when(messageListener.consumeMessage(any(), any(ConsumeOrderlyContext.class))).thenReturn(ConsumeOrderlyStatus.SUCCESS);\n        ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker);\n        assertEquals(CMResult.CR_SUCCESS, actual.getConsumeResult());\n        assertTrue(actual.isOrder());\n    }\n\n    @Test\n    public void testConsumeMessageDirectlyWithCommit() {\n        when(messageListener.consumeMessage(any(), any(ConsumeOrderlyContext.class))).thenReturn(ConsumeOrderlyStatus.COMMIT);\n        ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker);\n        assertEquals(CMResult.CR_COMMIT, actual.getConsumeResult());\n        assertTrue(actual.isOrder());\n    }\n\n    @Test\n    public void testConsumeMessageDirectlyWithRollback() {\n        when(messageListener.consumeMessage(any(), any(ConsumeOrderlyContext.class))).thenReturn(ConsumeOrderlyStatus.ROLLBACK);\n        ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker);\n        assertEquals(CMResult.CR_ROLLBACK, actual.getConsumeResult());\n        assertTrue(actual.isOrder());\n    }\n\n    @Test\n    public void testConsumeMessageDirectlyWithCrLater() {\n        when(messageListener.consumeMessage(any(), any(ConsumeOrderlyContext.class))).thenReturn(ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT);\n        ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker);\n        assertEquals(CMResult.CR_LATER, actual.getConsumeResult());\n    }\n\n    @Test\n    public void testConsumeMessageDirectlyWithCrReturnNull() {\n        ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker);\n        assertEquals(CMResult.CR_RETURN_NULL, actual.getConsumeResult());\n    }\n\n    @Test\n    public void testConsumeMessageDirectlyWithCrThrowException() {\n        when(messageListener.consumeMessage(any(), any(ConsumeOrderlyContext.class))).thenThrow(new RuntimeException(\"exception\"));\n        ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker);\n        assertEquals(CMResult.CR_THROW_EXCEPTION, actual.getConsumeResult());\n    }\n\n    @Test\n    public void testSubmitConsumeRequest() {\n        assertThrows(UnsupportedOperationException.class, () -> {\n            List<MessageExt> msgs = mock(List.class);\n            ProcessQueue processQueue = mock(ProcessQueue.class);\n            MessageQueue messageQueue = mock(MessageQueue.class);\n            popService.submitConsumeRequest(msgs, processQueue, messageQueue, false);\n        });\n    }\n\n    @Test\n    public void testSubmitPopConsumeRequest() throws IllegalAccessException {\n        List<MessageExt> msgs = Collections.singletonList(createMessageExt());\n        PopProcessQueue processQueue = mock(PopProcessQueue.class);\n        MessageQueue messageQueue = mock(MessageQueue.class);\n        ThreadPoolExecutor consumeExecutor = mock(ThreadPoolExecutor.class);\n        FieldUtils.writeDeclaredField(popService, \"consumeExecutor\", consumeExecutor, true);\n        popService.submitPopConsumeRequest(msgs, processQueue, messageQueue);\n        verify(consumeExecutor, times(1)).submit(any(Runnable.class));\n    }\n\n    @Test\n    public void testLockMQPeriodically() {\n        popService.lockMQPeriodically();\n        verify(defaultMQPushConsumerImpl, times(1)).getRebalanceImpl();\n        verify(rebalanceImpl, times(1)).lockAll();\n    }\n\n    @Test\n    public void testGetConsumerStatsManager() {\n        ConsumerStatsManager actual = popService.getConsumerStatsManager();\n        assertNotNull(actual);\n        assertEquals(consumerStatsManager, actual);\n    }\n\n    @Test\n    public void testSendMessageBack() {\n        assertTrue(popService.sendMessageBack(createMessageExt()));\n    }\n\n    @Test\n    public void testProcessConsumeResult() {\n        ConsumeOrderlyContext context = mock(ConsumeOrderlyContext.class);\n        ConsumeMessagePopOrderlyService.ConsumeRequest consumeRequest = mock(ConsumeMessagePopOrderlyService.ConsumeRequest.class);\n        assertTrue(popService.processConsumeResult(Collections.singletonList(createMessageExt()), ConsumeOrderlyStatus.SUCCESS, context, consumeRequest));\n    }\n\n    @Test\n    public void testResetNamespace() {\n        when(defaultMQPushConsumer.getNamespace()).thenReturn(\"defaultNamespace\");\n        List<MessageExt> msgs = Collections.singletonList(createMessageExt());\n        popService.resetNamespace(msgs);\n        assertEquals(defaultTopic, msgs.get(0).getTopic());\n    }\n\n    private MessageExt createMessageExt() {\n        MessageExt result = new MessageExt();\n        result.setBody(\"body\".getBytes(StandardCharsets.UTF_8));\n        result.setTopic(defaultTopic);\n        result.setBrokerName(defaultBroker);\n        result.putUserProperty(\"key\", \"value\");\n        result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, defaultGroup);\n        result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, \"TX1\");\n        long curTime = System.currentTimeMillis();\n        result.setBornTimestamp(curTime - 1000);\n        result.getProperties().put(MessageConst.PROPERTY_POP_CK, curTime + \" \" + curTime + \" \" + curTime + \" \" + curTime);\n        result.setKeys(\"keys\");\n        SocketAddress bornHost = new InetSocketAddress(\"127.0.0.1\", 12911);\n        SocketAddress storeHost = new InetSocketAddress(\"127.0.0.1\", 10911);\n        result.setBornHost(bornHost);\n        result.setStoreHost(storeHost);\n        return result;\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImplTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport org.apache.rocketmq.client.consumer.DefaultLitePullConsumer;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.junit.Assert;\nimport org.junit.BeforeClass;\nimport org.junit.Test;\n\nimport java.lang.reflect.InvocationTargetException;\nimport java.lang.reflect.Method;\nimport java.util.HashSet;\nimport java.util.Set;\n\n\npublic class DefaultLitePullConsumerImplTest {\n    private final DefaultLitePullConsumerImpl consumer = new DefaultLitePullConsumerImpl(new DefaultLitePullConsumer(), null);\n\n    private static Method isSetEqualMethod;\n\n    @BeforeClass\n    public static void initReflectionMethod() throws NoSuchMethodException {\n        Class<DefaultLitePullConsumerImpl> consumerClass = DefaultLitePullConsumerImpl.class;\n        Method testMethod = consumerClass.getDeclaredMethod(\"isSetEqual\", Set.class, Set.class);\n        testMethod.setAccessible(true);\n        isSetEqualMethod = testMethod;\n    }\n\n\n    /**\n     * The two empty sets should be equal\n     */\n    @Test\n    public void testIsSetEqual1() throws InvocationTargetException, IllegalAccessException {\n        Set<MessageQueue> set1 = new HashSet<>();\n        Set<MessageQueue> set2 = new HashSet<>();\n        boolean equalResult = (boolean) isSetEqualMethod.invoke(consumer, set1, set2);\n        Assert.assertTrue(equalResult);\n    }\n\n\n    /**\n     * When a set has elements and one does not, the two sets are not equal\n     */\n    @Test\n    public void testIsSetEqual2() throws InvocationTargetException, IllegalAccessException {\n        Set<MessageQueue> set1 = new HashSet<>();\n        set1.add(new MessageQueue(\"testTopic\",\"testBroker\",111));\n        Set<MessageQueue> set2 = new HashSet<>();\n        boolean equalResult = (boolean) isSetEqualMethod.invoke(consumer, set1, set2);\n        Assert.assertFalse(equalResult);\n    }\n\n    /**\n     * The two null sets should be equal\n     */\n    @Test\n    public void testIsSetEqual3() throws InvocationTargetException, IllegalAccessException {\n        Set<MessageQueue> set1 = null;\n        Set<MessageQueue> set2 = null;\n        boolean equalResult = (boolean) isSetEqualMethod.invoke(consumer, set1, set2);\n        Assert.assertTrue(equalResult);\n    }\n\n    @Test\n    public void testIsSetEqual4() throws InvocationTargetException, IllegalAccessException {\n        Set<MessageQueue> set1 = null;\n        Set<MessageQueue> set2 = new HashSet<>();\n        boolean equalResult = (boolean) isSetEqualMethod.invoke(consumer, set1, set2);\n        Assert.assertFalse(equalResult);\n    }\n\n    @Test\n    public void testIsSetEqual5() throws InvocationTargetException, IllegalAccessException {\n        Set<MessageQueue> set1 = new HashSet<>();\n        set1.add(new MessageQueue(\"testTopic\",\"testBroker\",111));\n        Set<MessageQueue> set2 = new HashSet<>();\n        set2.add(new MessageQueue(\"testTopic\",\"testBroker\",111));\n        boolean equalResult = (boolean) isSetEqualMethod.invoke(consumer, set1, set2);\n        Assert.assertTrue(equalResult);\n    }\n\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImplTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.client.consumer.AckCallback;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.AckStatus;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.MessageSelector;\nimport org.apache.rocketmq.client.consumer.PopCallback;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PopStatus;\nimport org.apache.rocketmq.client.consumer.PullCallback;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.consumer.store.OffsetStore;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.hook.ConsumeMessageContext;\nimport org.apache.rocketmq.client.hook.ConsumeMessageHook;\nimport org.apache.rocketmq.client.hook.FilterMessageContext;\nimport org.apache.rocketmq.client.hook.FilterMessageHook;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.FindBrokerResult;\nimport org.apache.rocketmq.client.impl.MQAdminImpl;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.client.stat.ConsumerStatsManager;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ServiceState;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeStatus;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan;\nimport org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Ignore;\nimport org.junit.Rule;\nimport org.junit.Test;\nimport org.junit.rules.ExpectedException;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.TreeMap;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.atomic.AtomicLong;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.junit.Assert.assertArrayEquals;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyBoolean;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.doAnswer;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class DefaultMQPushConsumerImplTest {\n\n    @Mock\n    private DefaultMQPushConsumer defaultMQPushConsumer;\n\n    @Mock\n    private MQClientInstance mQClientFactory;\n\n    @Mock\n    private RebalanceImpl rebalanceImpl;\n\n    @Mock\n    private PullAPIWrapper pullAPIWrapper;\n\n    @Mock\n    private PullRequest pullRequest;\n\n    @Mock\n    private PopRequest popRequest;\n\n    @Mock\n    private ProcessQueue processQueue;\n\n    @Mock\n    private PopProcessQueue popProcessQueue;\n\n    @Mock\n    private MQClientAPIImpl mqClientAPIImpl;\n\n    @Mock\n    private OffsetStore offsetStore;\n\n    private DefaultMQPushConsumerImpl defaultMQPushConsumerImpl;\n\n    @Rule\n    public ExpectedException thrown = ExpectedException.none();\n\n    private final String defaultKey = \"defaultKey\";\n\n    private final String defaultTopic = \"defaultTopic\";\n\n    private final String defaultBroker = \"defaultBroker\";\n\n    private final String defaultBrokerAddr = \"127.0.0.1:10911\";\n\n    private final String defaultGroup = \"defaultGroup\";\n\n    private final long defaultTimeout = 3000L;\n\n    @Test\n    public void checkConfigTest() throws MQClientException {\n\n        //test type\n        thrown.expect(MQClientException.class);\n\n        //test message\n        thrown.expectMessage(\"consumeThreadMin (10) is larger than consumeThreadMax (9)\");\n\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"test_consumer_group\");\n\n        consumer.setConsumeThreadMin(10);\n        consumer.setConsumeThreadMax(9);\n\n        consumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> ConsumeConcurrentlyStatus.CONSUME_SUCCESS);\n\n        DefaultMQPushConsumerImpl defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(consumer, null);\n        defaultMQPushConsumerImpl.start();\n    }\n\n    @Test\n    public void testHook() {\n        DefaultMQPushConsumerImpl defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(defaultMQPushConsumer, null);\n        defaultMQPushConsumerImpl.registerConsumeMessageHook(new ConsumeMessageHook() {\n            @Override\n            public String hookName() {\n                return \"consumerHook\";\n            }\n\n            @Override\n            public void consumeMessageBefore(ConsumeMessageContext context) {\n                assertThat(context).isNotNull();\n            }\n\n            @Override\n            public void consumeMessageAfter(ConsumeMessageContext context) {\n                assertThat(context).isNotNull();\n            }\n        });\n        defaultMQPushConsumerImpl.registerFilterMessageHook(new FilterMessageHook() {\n            @Override\n            public String hookName() {\n                return \"filterHook\";\n            }\n\n            @Override\n            public void filterMessage(FilterMessageContext context) {\n                assertThat(context).isNotNull();\n            }\n        });\n        defaultMQPushConsumerImpl.executeHookBefore(new ConsumeMessageContext());\n        defaultMQPushConsumerImpl.executeHookAfter(new ConsumeMessageContext());\n    }\n\n    @Ignore\n    @Test\n    public void testPush() throws Exception {\n        when(defaultMQPushConsumer.getMessageListener()).thenReturn((MessageListenerConcurrently) (msgs, context) -> {\n            assertThat(msgs).size().isGreaterThan(0);\n            assertThat(context).isNotNull();\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        DefaultMQPushConsumerImpl defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(defaultMQPushConsumer, null);\n        try {\n            defaultMQPushConsumerImpl.start();\n        } finally {\n            defaultMQPushConsumerImpl.shutdown();\n        }\n    }\n\n    @Before\n    public void init() throws NoSuchFieldException, IllegalAccessException {\n        MQAdminImpl mqAdminImpl = mock(MQAdminImpl.class);\n        when(mQClientFactory.getMQAdminImpl()).thenReturn(mqAdminImpl);\n        ConsumerStatsManager consumerStatsManager = mock(ConsumerStatsManager.class);\n        ConsumeStatus consumeStatus = mock(ConsumeStatus.class);\n        when(consumerStatsManager.consumeStatus(any(), any())).thenReturn(consumeStatus);\n        when(mQClientFactory.getConsumerStatsManager()).thenReturn(consumerStatsManager);\n        when(mQClientFactory.getPullMessageService()).thenReturn(mock(PullMessageService.class));\n        when(mQClientFactory.getMQClientAPIImpl()).thenReturn(mqClientAPIImpl);\n        FindBrokerResult findBrokerResult = mock(FindBrokerResult.class);\n        when(findBrokerResult.getBrokerAddr()).thenReturn(defaultBrokerAddr);\n        when(mQClientFactory.findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean())).thenReturn(findBrokerResult);\n        Set<MessageQueue> messageQueueSet = Collections.singleton(createMessageQueue());\n        ConcurrentMap<String, Set<MessageQueue>> topicMessageQueueMap = new ConcurrentHashMap<>();\n        topicMessageQueueMap.put(defaultTopic, messageQueueSet);\n        when(rebalanceImpl.getTopicSubscribeInfoTable()).thenReturn(topicMessageQueueMap);\n        ConcurrentMap<MessageQueue, ProcessQueue> processQueueTable = new ConcurrentHashMap<>();\n        when(rebalanceImpl.getProcessQueueTable()).thenReturn(processQueueTable);\n        RPCHook rpcHook = mock(RPCHook.class);\n        defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(defaultMQPushConsumer, rpcHook);\n        defaultMQPushConsumerImpl.setOffsetStore(offsetStore);\n        FieldUtils.writeDeclaredField(defaultMQPushConsumerImpl, \"mQClientFactory\", mQClientFactory, true);\n        FieldUtils.writeDeclaredField(defaultMQPushConsumerImpl, \"rebalanceImpl\", rebalanceImpl, true);\n        FieldUtils.writeDeclaredField(defaultMQPushConsumerImpl, \"pullAPIWrapper\", pullAPIWrapper, true);\n        FilterMessageHook filterMessageHook = mock(FilterMessageHook.class);\n        ArrayList<FilterMessageHook> filterMessageHookList = new ArrayList<>();\n        filterMessageHookList.add(filterMessageHook);\n        ConsumeMessageService consumeMessagePopService = mock(ConsumeMessageService.class);\n        ConsumeMessageService consumeMessageService = mock(ConsumeMessageService.class);\n        FieldUtils.writeDeclaredField(defaultMQPushConsumerImpl, \"filterMessageHookList\", filterMessageHookList, true);\n        FieldUtils.writeDeclaredField(defaultMQPushConsumerImpl, \"consumeMessageService\", consumeMessageService, true);\n        FieldUtils.writeDeclaredField(defaultMQPushConsumerImpl, \"consumeMessagePopService\", consumeMessagePopService, true);\n        ConcurrentMap<String, SubscriptionData> subscriptionDataMap = new ConcurrentHashMap<>();\n        SubscriptionData subscriptionData = new SubscriptionData();\n        subscriptionData.setTopic(defaultTopic);\n        subscriptionDataMap.put(defaultTopic, subscriptionData);\n        when(rebalanceImpl.getSubscriptionInner()).thenReturn(subscriptionDataMap);\n    }\n\n    @Test\n    public void testFetchSubscribeMessageQueues() throws MQClientException {\n        Set<MessageQueue> actual = defaultMQPushConsumerImpl.fetchSubscribeMessageQueues(defaultTopic);\n        assertNotNull(actual);\n        Assert.assertEquals(1, actual.size());\n        MessageQueue next = actual.iterator().next();\n        assertEquals(defaultTopic, next.getTopic());\n        assertEquals(defaultBroker, next.getBrokerName());\n        assertEquals(0, next.getQueueId());\n    }\n\n    @Test\n    public void testEarliestMsgStoreTime() throws MQClientException {\n        assertEquals(0, defaultMQPushConsumerImpl.earliestMsgStoreTime(createMessageQueue()));\n    }\n\n    @Test\n    public void testMaxOffset() throws MQClientException {\n        assertEquals(0, defaultMQPushConsumerImpl.maxOffset(createMessageQueue()));\n    }\n\n    @Test\n    public void testMinOffset() throws MQClientException {\n        assertEquals(0, defaultMQPushConsumerImpl.minOffset(createMessageQueue()));\n    }\n\n    @Test\n    public void testGetOffsetStore() {\n        assertEquals(offsetStore, defaultMQPushConsumerImpl.getOffsetStore());\n    }\n\n    @Test\n    public void testPullMessageWithStateNotOk() {\n        when(pullRequest.getProcessQueue()).thenReturn(processQueue);\n        defaultMQPushConsumerImpl.pullMessage(pullRequest);\n    }\n\n    @Test\n    public void testPullMessageWithIsPause() {\n        when(pullRequest.getProcessQueue()).thenReturn(processQueue);\n        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);\n        defaultMQPushConsumerImpl.setPause(true);\n        defaultMQPushConsumerImpl.pullMessage(pullRequest);\n    }\n\n    @Test\n    public void testPullMessageWithMsgCountFlowControl() {\n        when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2));\n        when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024));\n        TreeMap<Long, MessageExt> treeMap = new TreeMap<>();\n        treeMap.put(1L, new MessageExt());\n        when(processQueue.getMsgTreeMap()).thenReturn(treeMap);\n        when(pullRequest.getProcessQueue()).thenReturn(processQueue);\n        when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(1);\n        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);\n        defaultMQPushConsumerImpl.pullMessage(pullRequest);\n    }\n\n    @Test\n    public void testPullMessageWithMsgSizeFlowControl() {\n        when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2));\n        when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024));\n        TreeMap<Long, MessageExt> treeMap = new TreeMap<>();\n        treeMap.put(1L, new MessageExt());\n        when(processQueue.getMsgTreeMap()).thenReturn(treeMap);\n        when(pullRequest.getProcessQueue()).thenReturn(processQueue);\n        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);\n        when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(3);\n        when(defaultMQPushConsumer.getPullThresholdSizeForQueue()).thenReturn(1);\n        defaultMQPushConsumerImpl.pullMessage(pullRequest);\n    }\n\n    @Test\n    public void testPullMessageWithMaxSpanFlowControl() {\n        when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2));\n        when(processQueue.getMaxSpan()).thenReturn(2L);\n        when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024));\n        TreeMap<Long, MessageExt> treeMap = new TreeMap<>();\n        treeMap.put(1L, new MessageExt());\n        when(processQueue.getMsgTreeMap()).thenReturn(treeMap);\n        when(pullRequest.getProcessQueue()).thenReturn(processQueue);\n        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);\n        when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(3);\n        when(defaultMQPushConsumer.getPullThresholdSizeForQueue()).thenReturn(10);\n        defaultMQPushConsumerImpl.pullMessage(pullRequest);\n    }\n\n    @Test\n    public void testPullMessageWithNotLocked() {\n        when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2));\n        when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024));\n        when(pullRequest.getProcessQueue()).thenReturn(processQueue);\n        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);\n        defaultMQPushConsumerImpl.setConsumeOrderly(true);\n        when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(3);\n        when(defaultMQPushConsumer.getPullThresholdSizeForQueue()).thenReturn(10);\n        defaultMQPushConsumerImpl.pullMessage(pullRequest);\n    }\n\n    @Test\n    public void testPullMessageWithSubscriptionDataIsNull() {\n        when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2));\n        when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024));\n        when(pullRequest.getMessageQueue()).thenReturn(createMessageQueue());\n        when(pullRequest.getProcessQueue()).thenReturn(processQueue);\n        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);\n        when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(3);\n        when(defaultMQPushConsumer.getPullThresholdSizeForQueue()).thenReturn(10);\n        defaultMQPushConsumerImpl.pullMessage(pullRequest);\n    }\n\n    @Test\n    public void testPullMessageWithNoMatchedMsg() throws MQBrokerException, RemotingException, InterruptedException, MQClientException {\n        when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2));\n        when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024));\n        when(pullRequest.getMessageQueue()).thenReturn(createMessageQueue());\n        when(pullRequest.getProcessQueue()).thenReturn(processQueue);\n        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);\n        when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(3);\n        when(defaultMQPushConsumer.getPullThresholdSizeForQueue()).thenReturn(10);\n        PullResult pullResultMock = mock(PullResult.class);\n        when(pullAPIWrapper.processPullResult(any(MessageQueue.class), any(PullResult.class), any(SubscriptionData.class))).thenReturn(pullResultMock);\n        when(pullResultMock.getPullStatus()).thenReturn(PullStatus.NO_MATCHED_MSG);\n        doAnswer(invocation -> {\n            PullCallback callback = invocation.getArgument(12);\n            PullResult pullResult = mock(PullResult.class);\n            callback.onSuccess(pullResult);\n            return null;\n        }).when(pullAPIWrapper).pullKernelImpl(\n                any(MessageQueue.class),\n                any(),\n                any(),\n                anyLong(),\n                anyLong(),\n                anyInt(),\n                anyInt(),\n                anyInt(),\n                anyLong(),\n                anyLong(),\n                anyLong(),\n                any(CommunicationMode.class),\n                any(PullCallback.class));\n        defaultMQPushConsumerImpl.pullMessage(pullRequest);\n    }\n\n    @Test\n    public void testPullMessageWithOffsetIllegal() throws MQBrokerException, RemotingException, InterruptedException, MQClientException {\n        when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2));\n        when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024));\n        when(pullRequest.getMessageQueue()).thenReturn(createMessageQueue());\n        when(pullRequest.getProcessQueue()).thenReturn(processQueue);\n        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);\n        when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(3);\n        when(defaultMQPushConsumer.getPullThresholdSizeForQueue()).thenReturn(10);\n        PullResult pullResultMock = mock(PullResult.class);\n        when(pullAPIWrapper.processPullResult(any(MessageQueue.class), any(PullResult.class), any(SubscriptionData.class))).thenReturn(pullResultMock);\n        when(pullResultMock.getPullStatus()).thenReturn(PullStatus.OFFSET_ILLEGAL);\n        doAnswer(invocation -> {\n            PullCallback callback = invocation.getArgument(12);\n            PullResult pullResult = mock(PullResult.class);\n            callback.onSuccess(pullResult);\n            return null;\n        }).when(pullAPIWrapper).pullKernelImpl(\n                any(MessageQueue.class),\n                any(),\n                any(),\n                anyLong(),\n                anyLong(),\n                anyInt(),\n                anyInt(),\n                anyInt(),\n                anyLong(),\n                anyLong(),\n                anyLong(),\n                any(CommunicationMode.class),\n                any(PullCallback.class));\n        defaultMQPushConsumerImpl.pullMessage(pullRequest);\n    }\n\n    @Test\n    public void testPullMessageWithException() throws MQBrokerException, RemotingException, InterruptedException, MQClientException {\n        when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2));\n        when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024));\n        when(pullRequest.getMessageQueue()).thenReturn(createMessageQueue());\n        when(pullRequest.getProcessQueue()).thenReturn(processQueue);\n        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);\n        when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(3);\n        when(defaultMQPushConsumer.getPullThresholdSizeForQueue()).thenReturn(10);\n        doAnswer(invocation -> {\n            PullCallback callback = invocation.getArgument(12);\n            callback.onException(new RuntimeException(\"exception\"));\n            return null;\n        }).when(pullAPIWrapper).pullKernelImpl(\n                any(MessageQueue.class),\n                any(),\n                any(),\n                anyLong(),\n                anyLong(),\n                anyInt(),\n                anyInt(),\n                anyInt(),\n                anyLong(),\n                anyLong(),\n                anyLong(),\n                any(CommunicationMode.class),\n                any(PullCallback.class));\n        defaultMQPushConsumerImpl.pullMessage(pullRequest);\n    }\n\n    @Test\n    public void testPopMessageWithFound() throws RemotingException, InterruptedException, MQClientException {\n        when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue);\n        when(popRequest.getMessageQueue()).thenReturn(createMessageQueue());\n        when(popRequest.getConsumerGroup()).thenReturn(defaultGroup);\n        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);\n        ConcurrentMap<String, SubscriptionData> subscriptionDataMap = new ConcurrentHashMap<>();\n        SubscriptionData subscriptionData = new SubscriptionData();\n        subscriptionData.setTagsSet(Collections.singleton(\"*\"));\n        subscriptionDataMap.put(defaultTopic, subscriptionData);\n        when(rebalanceImpl.getSubscriptionInner()).thenReturn(subscriptionDataMap);\n        doAnswer(invocation -> {\n            PopCallback callback = invocation.getArgument(5);\n            PopResult popResult = mock(PopResult.class);\n            when(popResult.getPopStatus()).thenReturn(PopStatus.FOUND);\n            when(popResult.getMsgFoundList()).thenReturn(Collections.singletonList(createMessageExt()));\n            callback.onSuccess(popResult);\n            return null;\n        }).when(pullAPIWrapper).popAsync(\n                any(MessageQueue.class),\n                anyLong(),\n                anyInt(),\n                any(),\n                anyLong(),\n                any(PopCallback.class),\n                anyBoolean(),\n                anyInt(),\n                anyBoolean(),\n                any(),\n                any());\n        defaultMQPushConsumerImpl.popMessage(popRequest);\n    }\n\n    @Test\n    public void testPopMessageWithException() throws RemotingException, InterruptedException, MQClientException {\n        when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue);\n        when(popRequest.getMessageQueue()).thenReturn(createMessageQueue());\n        when(popRequest.getConsumerGroup()).thenReturn(defaultGroup);\n        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);\n        ConcurrentMap<String, SubscriptionData> subscriptionDataMap = new ConcurrentHashMap<>();\n        SubscriptionData subscriptionData = new SubscriptionData();\n        subscriptionData.setTagsSet(Collections.singleton(\"*\"));\n        subscriptionDataMap.put(defaultTopic, subscriptionData);\n        when(rebalanceImpl.getSubscriptionInner()).thenReturn(subscriptionDataMap);\n        doAnswer(invocation -> {\n            PopCallback callback = invocation.getArgument(5);\n            callback.onException(new RuntimeException(\"exception\"));\n            return null;\n        }).when(pullAPIWrapper).popAsync(\n                any(MessageQueue.class),\n                anyLong(),\n                anyInt(),\n                any(),\n                anyLong(),\n                any(PopCallback.class),\n                anyBoolean(),\n                anyInt(),\n                anyBoolean(),\n                any(),\n                any());\n        defaultMQPushConsumerImpl.popMessage(popRequest);\n    }\n\n    @Test\n    public void testPopMessageWithNoNewMsg() throws RemotingException, InterruptedException, MQClientException {\n        when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue);\n        when(popRequest.getMessageQueue()).thenReturn(createMessageQueue());\n        when(popRequest.getConsumerGroup()).thenReturn(defaultGroup);\n        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);\n        ConcurrentMap<String, SubscriptionData> subscriptionDataMap = new ConcurrentHashMap<>();\n        SubscriptionData subscriptionData = new SubscriptionData();\n        subscriptionData.setTagsSet(Collections.singleton(\"*\"));\n        subscriptionDataMap.put(defaultTopic, subscriptionData);\n        when(rebalanceImpl.getSubscriptionInner()).thenReturn(subscriptionDataMap);\n        doAnswer(invocation -> {\n            PopCallback callback = invocation.getArgument(5);\n            PopResult popResult = mock(PopResult.class);\n            when(popResult.getPopStatus()).thenReturn(PopStatus.NO_NEW_MSG);\n            callback.onSuccess(popResult);\n            return null;\n        }).when(pullAPIWrapper).popAsync(\n                any(MessageQueue.class),\n                anyLong(),\n                anyInt(),\n                any(),\n                anyLong(),\n                any(PopCallback.class),\n                anyBoolean(),\n                anyInt(),\n                anyBoolean(),\n                any(),\n                any());\n        defaultMQPushConsumerImpl.popMessage(popRequest);\n    }\n\n    @Test\n    public void testPopMessageWithPollingFull() throws RemotingException, InterruptedException, MQClientException {\n        when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue);\n        when(popRequest.getMessageQueue()).thenReturn(createMessageQueue());\n        when(popRequest.getConsumerGroup()).thenReturn(defaultGroup);\n        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);\n        ConcurrentMap<String, SubscriptionData> subscriptionDataMap = new ConcurrentHashMap<>();\n        SubscriptionData subscriptionData = new SubscriptionData();\n        subscriptionData.setTagsSet(Collections.singleton(\"*\"));\n        subscriptionDataMap.put(defaultTopic, subscriptionData);\n        when(rebalanceImpl.getSubscriptionInner()).thenReturn(subscriptionDataMap);\n        doAnswer(invocation -> {\n            PopCallback callback = invocation.getArgument(5);\n            PopResult popResult = mock(PopResult.class);\n            when(popResult.getPopStatus()).thenReturn(PopStatus.POLLING_FULL);\n            callback.onSuccess(popResult);\n            return null;\n        }).when(pullAPIWrapper).popAsync(any(\n                        MessageQueue.class),\n                anyLong(),\n                anyInt(),\n                any(),\n                anyLong(),\n                any(PopCallback.class),\n                anyBoolean(),\n                anyInt(),\n                anyBoolean(),\n                any(),\n                any());\n        defaultMQPushConsumerImpl.popMessage(popRequest);\n    }\n\n    @Test\n    public void testPopMessageWithStateNotOk() {\n        when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue);\n        defaultMQPushConsumerImpl.popMessage(popRequest);\n    }\n\n    @Test\n    public void testPopMessageWithIsPause() {\n        when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue);\n        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);\n        defaultMQPushConsumerImpl.setPause(true);\n        defaultMQPushConsumerImpl.popMessage(popRequest);\n    }\n\n    @Test\n    public void testPopMessageWithWaiAckMsgCountFlowControl() {\n        when(popProcessQueue.getWaiAckMsgCount()).thenReturn(2);\n        when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue);\n        when(defaultMQPushConsumer.getPopThresholdForQueue()).thenReturn(1);\n        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);\n        defaultMQPushConsumerImpl.popMessage(popRequest);\n    }\n\n    @Test\n    public void testPopMessageWithSubscriptionDataIsNull() throws RemotingException, InterruptedException, MQClientException {\n        when(popProcessQueue.getWaiAckMsgCount()).thenReturn(2);\n        when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue);\n        when(popRequest.getMessageQueue()).thenReturn(createMessageQueue());\n        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);\n        when(defaultMQPushConsumer.getPopThresholdForQueue()).thenReturn(3);\n        defaultMQPushConsumerImpl.popMessage(popRequest);\n        verify(pullAPIWrapper).popAsync(any(MessageQueue.class),\n                eq(60000L),\n                eq(0),\n                any(),\n                eq(15000L),\n                any(PopCallback.class),\n                eq(true),\n                eq(0),\n                eq(false),\n                any(),\n                any());\n    }\n\n    @Test\n    public void testQueryMessage() throws InterruptedException, MQClientException {\n        assertNull(defaultMQPushConsumerImpl.queryMessage(defaultTopic, defaultKey, 1, 0, 1));\n    }\n\n    @Test\n    public void testQueryMessageByUniqKey() throws InterruptedException, MQClientException {\n        assertNull(defaultMQPushConsumerImpl.queryMessageByUniqKey(defaultTopic, defaultKey));\n    }\n\n    @Test\n    public void testSendMessageBack() throws InterruptedException, MQClientException, MQBrokerException, RemotingException {\n        when(mQClientFactory.findBrokerAddressInPublish(anyString())).thenReturn(defaultBrokerAddr);\n        defaultMQPushConsumerImpl.sendMessageBack(createMessageExt(), 1, createMessageQueue());\n        verify(mqClientAPIImpl).consumerSendMessageBack(\n                eq(defaultBrokerAddr),\n                eq(defaultBroker),\n                any(MessageExt.class),\n                any(),\n                eq(1),\n                eq(5000L),\n                eq(0));\n    }\n\n    @Test\n    public void testAckAsync() throws MQBrokerException, RemotingException, InterruptedException {\n        doAnswer(invocation -> {\n            AckCallback callback = invocation.getArgument(2);\n            AckResult result = mock(AckResult.class);\n            when(result.getStatus()).thenReturn(AckStatus.OK);\n            callback.onSuccess(result);\n            return null;\n        }).when(mqClientAPIImpl).ackMessageAsync(any(),\n                anyLong(),\n                any(AckCallback.class),\n                any(AckMessageRequestHeader.class));\n        defaultMQPushConsumerImpl.ackAsync(createMessageExt(), defaultGroup);\n        verify(mqClientAPIImpl).ackMessageAsync(eq(defaultBrokerAddr),\n                eq(3000L),\n                any(AckCallback.class),\n                any(AckMessageRequestHeader.class));\n    }\n\n    @Test\n    public void testChangePopInvisibleTimeAsync() throws MQBrokerException, RemotingException, InterruptedException, MQClientException {\n        AckCallback callback = mock(AckCallback.class);\n        String extraInfo = createMessageExt().getProperty(MessageConst.PROPERTY_POP_CK);\n        defaultMQPushConsumerImpl.changePopInvisibleTimeAsync(defaultTopic, defaultGroup, extraInfo, defaultTimeout, callback);\n        verify(mqClientAPIImpl).changeInvisibleTimeAsync(eq(defaultBroker),\n                eq(defaultBrokerAddr),\n                any(ChangeInvisibleTimeRequestHeader.class),\n                eq(defaultTimeout),\n                any(AckCallback.class));\n    }\n\n    @Test\n    public void testShutdown() {\n        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);\n        defaultMQPushConsumerImpl.shutdown();\n        assertEquals(ServiceState.SHUTDOWN_ALREADY, defaultMQPushConsumerImpl.getServiceState());\n    }\n\n    @Test\n    public void testSubscribe() throws MQClientException {\n        defaultMQPushConsumerImpl.subscribe(defaultTopic, \"fullClassname\", \"filterClassSource\");\n        RebalanceImpl actual = defaultMQPushConsumerImpl.getRebalanceImpl();\n        assertEquals(1, actual.getSubscriptionInner().size());\n    }\n\n    @Test\n    public void testSubscribeByMessageSelector() throws MQClientException {\n        MessageSelector messageSelector = mock(MessageSelector.class);\n        defaultMQPushConsumerImpl.subscribe(defaultTopic, messageSelector);\n        RebalanceImpl actual = defaultMQPushConsumerImpl.getRebalanceImpl();\n        assertEquals(1, actual.getSubscriptionInner().size());\n    }\n\n    @Test\n    public void testSuspend() {\n        defaultMQPushConsumerImpl.suspend();\n        assertTrue(defaultMQPushConsumerImpl.isPause());\n    }\n\n    @Test\n    public void testViewMessage() throws InterruptedException, MQClientException, MQBrokerException, RemotingException {\n        assertNull(defaultMQPushConsumerImpl.viewMessage(defaultTopic, createMessageExt().getMsgId()));\n    }\n\n    @Test\n    public void testResetOffsetByTimeStamp() throws MQClientException {\n        ConcurrentMap<String, SubscriptionData> subscriptionDataMap = new ConcurrentHashMap<>();\n        subscriptionDataMap.put(defaultTopic, new SubscriptionData());\n        when(rebalanceImpl.getSubscriptionInner()).thenReturn(subscriptionDataMap);\n        defaultMQPushConsumerImpl.resetOffsetByTimeStamp(System.currentTimeMillis());\n        verify(mQClientFactory).resetOffset(eq(defaultTopic), any(), any());\n    }\n\n    @Test\n    public void testSearchOffset() throws MQClientException {\n        assertEquals(0, defaultMQPushConsumerImpl.searchOffset(createMessageQueue(), System.currentTimeMillis()));\n    }\n\n    @Test\n    public void testQueryConsumeTimeSpan() throws InterruptedException, MQClientException, MQBrokerException, RemotingException {\n        TopicRouteData topicRouteData = new TopicRouteData();\n        topicRouteData.getBrokerDatas().add(createBrokerData());\n        when(mqClientAPIImpl.getTopicRouteInfoFromNameServer(any(), anyLong())).thenReturn(topicRouteData);\n        List<QueueTimeSpan> actual = defaultMQPushConsumerImpl.queryConsumeTimeSpan(defaultTopic);\n        assertNotNull(actual);\n        assertEquals(0, actual.size());\n    }\n\n    @Test\n    public void testTryResetPopRetryTopic() {\n        TopicRouteData topicRouteData = new TopicRouteData();\n        topicRouteData.getBrokerDatas().add(createBrokerData());\n        MessageExt messageExt = createMessageExt();\n        List<MessageExt> msgs = new ArrayList<>();\n        messageExt.setTopic(MixAll.RETRY_GROUP_TOPIC_PREFIX + defaultGroup + \"_\" + defaultTopic);\n        msgs.add(messageExt);\n        defaultMQPushConsumerImpl.tryResetPopRetryTopic(msgs, defaultGroup);\n        assertEquals(defaultTopic, msgs.get(0).getTopic());\n    }\n\n    @Test\n    public void testGetPopDelayLevel() {\n        int[] actual = defaultMQPushConsumerImpl.getPopDelayLevel();\n        int[] expected = new int[]{10, 30, 60, 120, 180, 240, 300, 360, 420, 480, 540, 600, 1200, 1800, 3600, 7200};\n        assertArrayEquals(expected, actual);\n    }\n\n    @Test\n    public void testGetMessageQueueListener() {\n        assertNull(defaultMQPushConsumerImpl.getMessageQueueListener());\n    }\n\n    @Test\n    public void testConsumerRunningInfo() {\n        ConcurrentMap<MessageQueue, ProcessQueue> processQueueMap = new ConcurrentHashMap<>();\n        ConcurrentMap<MessageQueue, PopProcessQueue> popProcessQueueMap = new ConcurrentHashMap<>();\n        processQueueMap.put(createMessageQueue(), new ProcessQueue());\n        popProcessQueueMap.put(createMessageQueue(), new PopProcessQueue());\n        when(rebalanceImpl.getProcessQueueTable()).thenReturn(processQueueMap);\n        when(rebalanceImpl.getPopProcessQueueTable()).thenReturn(popProcessQueueMap);\n        ConsumerRunningInfo actual = defaultMQPushConsumerImpl.consumerRunningInfo();\n        assertNotNull(actual);\n        assertEquals(1, actual.getSubscriptionSet().size());\n        assertEquals(defaultTopic, actual.getSubscriptionSet().iterator().next().getTopic());\n        assertEquals(1, actual.getMqTable().size());\n        assertEquals(1, actual.getMqPopTable().size());\n        assertEquals(1, actual.getStatusTable().size());\n    }\n\n    private BrokerData createBrokerData() {\n        BrokerData result = new BrokerData();\n        HashMap<Long, String> brokerAddrMap = new HashMap<>();\n        brokerAddrMap.put(MixAll.MASTER_ID, defaultBrokerAddr);\n        result.setBrokerAddrs(brokerAddrMap);\n        result.setBrokerName(defaultBroker);\n        return result;\n    }\n\n    private MessageQueue createMessageQueue() {\n        MessageQueue result = new MessageQueue();\n        result.setQueueId(0);\n        result.setBrokerName(defaultBroker);\n        result.setTopic(defaultTopic);\n        return result;\n    }\n\n    private MessageExt createMessageExt() {\n        MessageExt result = new MessageExt();\n        result.setBody(\"body\".getBytes(StandardCharsets.UTF_8));\n        result.setTopic(defaultTopic);\n        result.setBrokerName(defaultBroker);\n        result.putUserProperty(\"key\", \"value\");\n        result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, defaultGroup);\n        result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, \"TX1\");\n        long curTime = System.currentTimeMillis();\n        result.setBornTimestamp(curTime - 1000);\n        String popProps = String.format(\"%d %d %d %d %d %s %d %d %d\", curTime, curTime, curTime, curTime, curTime, defaultBroker, 1, 0L, 1L);\n        result.getProperties().put(MessageConst.PROPERTY_POP_CK, popProps);\n        result.setKeys(\"keys\");\n        result.setTags(\"*\");\n        SocketAddress bornHost = new InetSocketAddress(\"127.0.0.1\", 12911);\n        SocketAddress storeHost = new InetSocketAddress(\"127.0.0.1\", 10911);\n        result.setBornHost(bornHost);\n        result.setStoreHost(storeHost);\n        return result;\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/impl/consumer/PopProcessQueueTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport org.apache.rocketmq.remoting.protocol.body.PopProcessQueueInfo;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.junit.Assert.assertEquals;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class PopProcessQueueTest {\n\n    private final PopProcessQueueInfo popProcessQueueInfo = new PopProcessQueueInfo();\n\n    @Test\n    public void testPopProcessQueue() {\n        long currentTime = System.currentTimeMillis();\n        PopProcessQueue popRequest1 = createPopProcessQueue(currentTime);\n        PopProcessQueue popRequest2 = createPopProcessQueue(currentTime);\n        assertEquals(popRequest1.getLastPopTimestamp(), popRequest2.getLastPopTimestamp());\n        assertEquals(popRequest1.toString(), popRequest2.toString());\n        assertEquals(popRequest1.getWaiAckMsgCount(), popRequest2.getWaiAckMsgCount());\n        assertEquals(popRequest1.ack(), popRequest2.ack());\n        assertEquals(popRequest1.isPullExpired(), popRequest2.isPullExpired());\n        assertEquals(popProcessQueueInfo.getLastPopTimestamp(), popRequest1.getLastPopTimestamp());\n        assertEquals(popProcessQueueInfo.isDroped(), popRequest1.isDropped());\n        assertEquals(popProcessQueueInfo.getWaitAckCount(), popRequest1.getWaiAckMsgCount() + popRequest2.getWaiAckMsgCount());\n    }\n\n    private PopProcessQueue createPopProcessQueue(final long currentTime) {\n        PopProcessQueue result = new PopProcessQueue();\n        long curTime = System.currentTimeMillis();\n        result.setLastPopTimestamp(curTime);\n        result.incFoundMsg(1);\n        result.decFoundMsg(1);\n        result.setLastPopTimestamp(currentTime);\n        result.fillPopProcessQueueInfo(popProcessQueueInfo);\n        return result;\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/impl/consumer/ProcessQueueTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.body.ProcessQueueInfo;\nimport org.assertj.core.util.Lists;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.lang.reflect.Field;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.TreeMap;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.verify;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ProcessQueueTest {\n\n    @Test\n    public void testCachedMessageCount() {\n        ProcessQueue pq = new ProcessQueue();\n\n        pq.putMessage(createMessageList());\n\n        assertThat(pq.getMsgCount().get()).isEqualTo(100);\n\n        pq.takeMessages(10);\n        pq.commit();\n\n        assertThat(pq.getMsgCount().get()).isEqualTo(90);\n\n        pq.removeMessage(Collections.singletonList(pq.getMsgTreeMap().lastEntry().getValue()));\n        assertThat(pq.getMsgCount().get()).isEqualTo(89);\n    }\n\n    @Test\n    public void testCachedMessageSize() {\n        ProcessQueue pq = new ProcessQueue();\n\n        pq.putMessage(createMessageList());\n\n        assertThat(pq.getMsgSize().get()).isEqualTo(100 * 123);\n\n        pq.takeMessages(10);\n        pq.commit();\n\n        assertThat(pq.getMsgSize().get()).isEqualTo(90 * 123);\n\n        pq.removeMessage(Collections.singletonList(pq.getMsgTreeMap().lastEntry().getValue()));\n        assertThat(pq.getMsgSize().get()).isEqualTo(89 * 123);\n    }\n\n    @Test\n    public void testContainsMessage() {\n        ProcessQueue pq = new ProcessQueue();\n        final List<MessageExt> messageList = createMessageList(2);\n        final MessageExt message0 = messageList.get(0);\n        final MessageExt message1 = messageList.get(1);\n\n        pq.putMessage(Lists.list(message0));\n        assertThat(pq.containsMessage(message0)).isTrue();\n        assertThat(pq.containsMessage(message1)).isFalse();\n    }\n\n    @Test\n    public void testFillProcessQueueInfo() throws IllegalAccessException {\n        ProcessQueue pq = new ProcessQueue();\n        pq.putMessage(createMessageList(102400));\n\n        ProcessQueueInfo processQueueInfo = new ProcessQueueInfo();\n        pq.fillProcessQueueInfo(processQueueInfo);\n\n        assertThat(processQueueInfo.getCachedMsgSizeInMiB()).isEqualTo(12);\n\n        pq.takeMessages(10000);\n        pq.commit();\n        pq.fillProcessQueueInfo(processQueueInfo);\n        assertThat(processQueueInfo.getCachedMsgSizeInMiB()).isEqualTo(10);\n\n        pq.takeMessages(10000);\n        pq.commit();\n        pq.fillProcessQueueInfo(processQueueInfo);\n        assertThat(processQueueInfo.getCachedMsgSizeInMiB()).isEqualTo(9);\n\n        pq.takeMessages(80000);\n        pq.commit();\n        pq.fillProcessQueueInfo(processQueueInfo);\n        assertThat(processQueueInfo.getCachedMsgSizeInMiB()).isEqualTo(0);\n\n        TreeMap<Long, MessageExt> consumingMsgOrderlyTreeMap = new TreeMap<>();\n        consumingMsgOrderlyTreeMap.put(0L, createMessageList(1).get(0));\n        FieldUtils.writeDeclaredField(pq, \"consumingMsgOrderlyTreeMap\", consumingMsgOrderlyTreeMap, true);\n        pq.fillProcessQueueInfo(processQueueInfo);\n        assertEquals(0, processQueueInfo.getTransactionMsgMinOffset());\n        assertEquals(0, processQueueInfo.getTransactionMsgMaxOffset());\n        assertEquals(1, processQueueInfo.getTransactionMsgCount());\n    }\n\n    @Test\n    public void testPopRequest() throws MQBrokerException, RemotingException, InterruptedException, MQClientException {\n        ProcessQueue processQueue = createProcessQueue();\n        MessageExt messageExt = createMessageList(1).get(0);\n        messageExt.getProperties().put(MessageConst.PROPERTY_CONSUME_START_TIMESTAMP, System.currentTimeMillis() - 20 * 60 * 1000L + \"\");\n        processQueue.getMsgTreeMap().put(0L, messageExt);\n        DefaultMQPushConsumer pushConsumer = mock(DefaultMQPushConsumer.class);\n        processQueue.cleanExpiredMsg(pushConsumer);\n        verify(pushConsumer).sendMessageBack(any(MessageExt.class), eq(3));\n    }\n\n    @Test\n    public void testRollback() throws IllegalAccessException {\n        ProcessQueue processQueue = createProcessQueue();\n        processQueue.rollback();\n        Field consumingMsgOrderlyTreeMapField = FieldUtils.getDeclaredField(processQueue.getClass(), \"consumingMsgOrderlyTreeMap\", true);\n        TreeMap<Long, MessageExt> consumingMsgOrderlyTreeMap = (TreeMap<Long, MessageExt>) consumingMsgOrderlyTreeMapField.get(processQueue);\n        assertEquals(0, consumingMsgOrderlyTreeMap.size());\n    }\n\n    @Test\n    public void testHasTempMessage() {\n        ProcessQueue processQueue = createProcessQueue();\n        assertFalse(processQueue.hasTempMessage());\n    }\n\n    @Test\n    public void testProcessQueue() {\n        ProcessQueue processQueue1 = createProcessQueue();\n        ProcessQueue processQueue2 = createProcessQueue();\n        assertEquals(processQueue1.getMsgAccCnt(), processQueue2.getMsgAccCnt());\n        assertEquals(processQueue1.getTryUnlockTimes(), processQueue2.getTryUnlockTimes());\n        assertEquals(processQueue1.getLastPullTimestamp(), processQueue2.getLastPullTimestamp());\n    }\n\n    private ProcessQueue createProcessQueue() {\n        ProcessQueue result = new ProcessQueue();\n        result.setMsgAccCnt(1);\n        result.incTryUnlockTimes();\n        result.setLastPullTimestamp(10000L);\n        return result;\n    }\n\n    private List<MessageExt> createMessageList() {\n        return createMessageList(100);\n    }\n\n    private List<MessageExt> createMessageList(int count) {\n        List<MessageExt> result = new ArrayList<>();\n        for (int i = 0; i < count; i++) {\n            MessageExt messageExt = new MessageExt();\n            messageExt.setQueueOffset(i);\n            messageExt.setBody(new byte[123]);\n            messageExt.setKeys(\"keys\" + i);\n            messageExt.getProperties().put(MessageConst.PROPERTY_CONSUME_START_TIMESTAMP, System.currentTimeMillis() + \"\");\n            result.add(messageExt);\n        }\n        return result;\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapperTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.consumer.PopCallback;\nimport org.apache.rocketmq.client.consumer.PullCallback;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.hook.FilterMessageContext;\nimport org.apache.rocketmq.client.hook.FilterMessageHook;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.FindBrokerResult;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.common.sysflag.PullSysFlag;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyBoolean;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class PullAPIWrapperTest {\n\n    @Mock\n    private MQClientInstance mQClientFactory;\n\n    @Mock\n    private MQClientAPIImpl mqClientAPIImpl;\n\n    private PullAPIWrapper pullAPIWrapper;\n\n    private final String defaultGroup = \"defaultGroup\";\n\n    private final String defaultBroker = \"defaultBroker\";\n\n    private final String defaultTopic = \"defaultTopic\";\n\n    private final String defaultBrokerAddr = \"127.0.0.1:10911\";\n\n    private final long defaultTimeout = 3000L;\n\n    @Before\n    public void init() throws Exception {\n        ClientConfig clientConfig = mock(ClientConfig.class);\n        when(mQClientFactory.getClientConfig()).thenReturn(clientConfig);\n        MQClientAPIImpl mqClientAPIImpl = mock(MQClientAPIImpl.class);\n        when(mQClientFactory.getMQClientAPIImpl()).thenReturn(mqClientAPIImpl);\n        when(mQClientFactory.getTopicRouteTable()).thenReturn(createTopicRouteTable());\n        FindBrokerResult findBrokerResult = mock(FindBrokerResult.class);\n        when(findBrokerResult.getBrokerAddr()).thenReturn(defaultBrokerAddr);\n        when(mQClientFactory.findBrokerAddressInSubscribe(any(), anyLong(), anyBoolean())).thenReturn(findBrokerResult);\n        pullAPIWrapper = new PullAPIWrapper(mQClientFactory, defaultGroup, false);\n        ArrayList<FilterMessageHook> filterMessageHookList = new ArrayList<>();\n        filterMessageHookList.add(mock(FilterMessageHook.class));\n        FieldUtils.writeDeclaredField(pullAPIWrapper, \"filterMessageHookList\", filterMessageHookList, true);\n    }\n\n    @Test\n    public void testProcessPullResult() throws Exception {\n        PullResultExt pullResult = mock(PullResultExt.class);\n        when(pullResult.getPullStatus()).thenReturn(PullStatus.FOUND);\n        when(pullResult.getMessageBinary()).thenReturn(MessageDecoder.encode(createMessageExt(), false));\n        SubscriptionData subscriptionData = mock(SubscriptionData.class);\n        PullResult actual = pullAPIWrapper.processPullResult(createMessageQueue(), pullResult, subscriptionData);\n        assertNotNull(actual);\n        assertEquals(0, actual.getNextBeginOffset());\n        assertEquals(0, actual.getMsgFoundList().size());\n    }\n\n    @Test\n    public void testExecuteHook() throws IllegalAccessException {\n        FilterMessageContext filterMessageContext = mock(FilterMessageContext.class);\n        ArrayList<FilterMessageHook> filterMessageHookList = new ArrayList<>();\n        FilterMessageHook filterMessageHook = mock(FilterMessageHook.class);\n        filterMessageHookList.add(filterMessageHook);\n        FieldUtils.writeDeclaredField(pullAPIWrapper, \"filterMessageHookList\", filterMessageHookList, true);\n        pullAPIWrapper.executeHook(filterMessageContext);\n        verify(filterMessageHook, times(1)).filterMessage(any(FilterMessageContext.class));\n    }\n\n    @Test\n    public void testPullKernelImpl() throws Exception {\n        PullCallback pullCallback = mock(PullCallback.class);\n        when(mQClientFactory.getMQClientAPIImpl()).thenReturn(mqClientAPIImpl);\n        PullResult actual = pullAPIWrapper.pullKernelImpl(createMessageQueue(),\n                \"\",\n                \"\",\n                1L,\n                1L,\n                1,\n                1,\n                PullSysFlag.buildSysFlag(false, false, false, true),\n                1L,\n                System.currentTimeMillis(),\n                defaultTimeout, CommunicationMode.ASYNC, pullCallback);\n        assertNull(actual);\n        verify(mqClientAPIImpl, times(1)).pullMessage(eq(defaultBroker),\n                any(PullMessageRequestHeader.class),\n                eq(defaultTimeout),\n                any(CommunicationMode.class),\n                any(PullCallback.class));\n    }\n\n    @Test\n    public void testSetConnectBrokerByUser() {\n        pullAPIWrapper.setConnectBrokerByUser(true);\n        assertTrue(pullAPIWrapper.isConnectBrokerByUser());\n    }\n\n    @Test\n    public void testRandomNum() {\n        int randomNum = pullAPIWrapper.randomNum();\n        assertTrue(randomNum > 0);\n    }\n\n    @Test\n    public void testSetDefaultBrokerId() {\n        pullAPIWrapper.setDefaultBrokerId(MixAll.MASTER_ID);\n        assertEquals(MixAll.MASTER_ID, pullAPIWrapper.getDefaultBrokerId());\n    }\n\n    @Test\n    public void testPopAsync() throws RemotingException, InterruptedException, MQClientException {\n        PopCallback popCallback = mock(PopCallback.class);\n        when(mQClientFactory.getMQClientAPIImpl()).thenReturn(mqClientAPIImpl);\n        pullAPIWrapper.popAsync(createMessageQueue(),\n                System.currentTimeMillis(),\n                1,\n                defaultGroup,\n                defaultTimeout,\n                popCallback,\n                true,\n                1,\n                false,\n                \"\",\n                \"\");\n        verify(mqClientAPIImpl, times(1)).popMessageAsync(eq(defaultBroker),\n                eq(defaultBrokerAddr),\n                any(PopMessageRequestHeader.class),\n                eq(13000L),\n                any(PopCallback.class));\n    }\n\n    private ConcurrentMap<String, TopicRouteData> createTopicRouteTable() {\n        TopicRouteData topicRouteData = new TopicRouteData();\n        List<BrokerData> brokerDatas = new ArrayList<>();\n        BrokerData brokerData = new BrokerData();\n        brokerData.setBrokerName(defaultBroker);\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(MixAll.MASTER_ID, defaultBroker);\n        brokerData.setBrokerAddrs(brokerAddrs);\n        brokerDatas.add(brokerData);\n        topicRouteData.setBrokerDatas(brokerDatas);\n        HashMap<String, List<String>> filterServerTable = new HashMap<>();\n        List<String> filterServers = new ArrayList<>();\n        filterServers.add(defaultBroker);\n        filterServerTable.put(defaultBrokerAddr, filterServers);\n        topicRouteData.setFilterServerTable(filterServerTable);\n        ConcurrentMap<String, TopicRouteData> result = new ConcurrentHashMap<>();\n        result.put(defaultTopic, topicRouteData);\n        return result;\n    }\n\n    private MessageQueue createMessageQueue() {\n        MessageQueue result = new MessageQueue();\n        result.setQueueId(0);\n        result.setBrokerName(defaultBroker);\n        result.setTopic(defaultTopic);\n        return result;\n    }\n\n    private MessageExt createMessageExt() {\n        MessageExt result = new MessageExt();\n        result.setBody(\"body\".getBytes(StandardCharsets.UTF_8));\n        result.setTopic(defaultTopic);\n        result.setBrokerName(defaultBroker);\n        result.putUserProperty(\"key\", \"value\");\n        result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, defaultGroup);\n        result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, \"TX1\");\n        long curTime = System.currentTimeMillis();\n        result.setBornTimestamp(curTime - 1000);\n        result.getProperties().put(MessageConst.PROPERTY_POP_CK, curTime + \" \" + curTime + \" \" + curTime + \" \" + curTime);\n        result.setKeys(\"keys\");\n        result.setSysFlag(MessageSysFlag.INNER_BATCH_FLAG);\n        result.setSysFlag(result.getSysFlag() | MessageSysFlag.NEED_UNWRAP_FLAG);\n        SocketAddress bornHost = new InetSocketAddress(\"127.0.0.1\", 12911);\n        SocketAddress storeHost = new InetSocketAddress(\"127.0.0.1\", 10911);\n        result.setBornHost(bornHost);\n        result.setStoreHost(storeHost);\n        return result;\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/impl/consumer/PullMessageServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.message.MessageRequestMode;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class PullMessageServiceTest {\n\n    @Mock\n    private MQClientInstance mQClientFactory;\n\n    @Mock\n    private ScheduledExecutorService executorService;\n\n    private PullMessageService pullMessageService;\n\n    private final long defaultTimeout = 3000L;\n\n    private final String defaultGroup = \"defaultGroup\";\n\n    @Before\n    public void init() throws Exception {\n        pullMessageService = new PullMessageService(mQClientFactory);\n        FieldUtils.writeDeclaredField(pullMessageService, \"scheduledExecutorService\", executorService, true);\n        pullMessageService.start();\n    }\n\n    @Test\n    public void testProcessPullResult() {\n        PopRequest popRequest = mock(PopRequest.class);\n        pullMessageService.executePopPullRequestLater(popRequest, defaultTimeout);\n        pullMessageService.makeStop();\n        pullMessageService.executePopPullRequestLater(popRequest, defaultTimeout);\n        verify(executorService, times(1))\n                .schedule(any(Runnable.class),\n                        eq(defaultTimeout),\n                        eq(TimeUnit.MILLISECONDS));\n    }\n\n    @Test\n    public void testExecutePopPullRequestImmediately() throws IllegalAccessException, InterruptedException {\n        PopRequest popRequest = mock(PopRequest.class);\n        LinkedBlockingQueue<MessageRequest> messageRequestQueue = mock(LinkedBlockingQueue.class);\n        FieldUtils.writeDeclaredField(pullMessageService, \"messageRequestQueue\", messageRequestQueue, true);\n        pullMessageService.executePopPullRequestImmediately(popRequest);\n        verify(messageRequestQueue, times(1)).put(any(PopRequest.class));\n    }\n\n    @Test\n    public void testExecuteTaskLater() {\n        Runnable runnable = mock(Runnable.class);\n        pullMessageService.executeTaskLater(runnable, defaultTimeout);\n        pullMessageService.makeStop();\n        pullMessageService.executeTaskLater(runnable, defaultTimeout);\n        verify(executorService, times(1))\n                .schedule(any(Runnable.class),\n                        eq(defaultTimeout),\n                        eq(TimeUnit.MILLISECONDS));\n    }\n\n    @Test\n    public void testExecuteTask() {\n        Runnable runnable = mock(Runnable.class);\n        pullMessageService.executeTask(runnable);\n        pullMessageService.makeStop();\n        pullMessageService.executeTask(runnable);\n        verify(executorService, times(1)).execute(any(Runnable.class));\n    }\n\n    @Test\n    public void testGetScheduledExecutorService() {\n        assertEquals(executorService, pullMessageService.getScheduledExecutorService());\n    }\n\n    @Test\n    public void testRun() throws InterruptedException, IllegalAccessException {\n        LinkedBlockingQueue<MessageRequest> messageRequestQueue = new LinkedBlockingQueue<>();\n        PopRequest popRequest = mock(PopRequest.class);\n        when(popRequest.getMessageRequestMode()).thenReturn(MessageRequestMode.POP);\n        when(popRequest.getConsumerGroup()).thenReturn(defaultGroup);\n        messageRequestQueue.put(popRequest);\n        DefaultMQPushConsumerImpl defaultMQPushConsumerImpl = mock(DefaultMQPushConsumerImpl.class);\n        when(mQClientFactory.selectConsumer(any())).thenReturn(defaultMQPushConsumerImpl);\n        FieldUtils.writeDeclaredField(pullMessageService, \"messageRequestQueue\", messageRequestQueue, true);\n        new Thread(() -> pullMessageService.run()).start();\n        TimeUnit.SECONDS.sleep(1);\n        pullMessageService.makeStop();\n        verify(mQClientFactory, times(1)).selectConsumer(eq(defaultGroup));\n        verify(defaultMQPushConsumerImpl).popMessage(any(PopRequest.class));\n    }\n\n    @Test\n    public void testRunWithNullConsumer() throws InterruptedException, IllegalAccessException {\n        LinkedBlockingQueue<MessageRequest> messageRequestQueue = new LinkedBlockingQueue<>();\n        PopRequest popRequest = mock(PopRequest.class);\n        when(popRequest.getMessageRequestMode()).thenReturn(MessageRequestMode.POP);\n        when(popRequest.getConsumerGroup()).thenReturn(defaultGroup);\n        messageRequestQueue.put(popRequest);\n        FieldUtils.writeDeclaredField(pullMessageService, \"messageRequestQueue\", messageRequestQueue, true);\n        new Thread(() -> pullMessageService.run()).start();\n        TimeUnit.SECONDS.sleep(1);\n        pullMessageService.makeStop();\n        verify(mQClientFactory, times(1)).selectConsumer(eq(defaultGroup));\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImplTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport org.apache.rocketmq.client.consumer.DefaultLitePullConsumer;\nimport org.apache.rocketmq.client.consumer.store.OffsetStore;\nimport org.apache.rocketmq.client.consumer.store.ReadOffsetType;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.MQAdminImpl;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\npublic class RebalanceLitePullImplTest {\n    private MessageQueue mq = new MessageQueue(\"topic1\", \"broker1\", 0);\n    private MessageQueue retryMq = new MessageQueue(MixAll.RETRY_GROUP_TOPIC_PREFIX + \"group\", \"broker1\", 0);\n    private DefaultLitePullConsumerImpl consumerImpl = mock(DefaultLitePullConsumerImpl.class);\n    private RebalanceLitePullImpl rebalanceImpl = new RebalanceLitePullImpl(consumerImpl);\n    private OffsetStore offsetStore = mock(OffsetStore.class);\n    private DefaultLitePullConsumer consumer = new DefaultLitePullConsumer();\n    private MQClientInstance client = mock(MQClientInstance.class);\n    private MQAdminImpl admin = mock(MQAdminImpl.class);\n\n    public RebalanceLitePullImplTest() {\n        when(consumerImpl.getDefaultLitePullConsumer()).thenReturn(consumer);\n        when(consumerImpl.getOffsetStore()).thenReturn(offsetStore);\n        rebalanceImpl.setmQClientFactory(client);\n        when(client.getMQAdminImpl()).thenReturn(admin);\n    }\n\n    @Test\n    public void testComputePullFromWhereWithException_ne_minus1() throws MQClientException {\n        for (ConsumeFromWhere where : new ConsumeFromWhere[]{\n            ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET,\n            ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET,\n            ConsumeFromWhere.CONSUME_FROM_TIMESTAMP}) {\n            consumer.setConsumeFromWhere(where);\n\n            when(offsetStore.readOffset(any(MessageQueue.class), any(ReadOffsetType.class))).thenReturn(0L);\n            assertEquals(0, rebalanceImpl.computePullFromWhereWithException(mq));\n\n            when(offsetStore.readOffset(any(MessageQueue.class), any(ReadOffsetType.class))).thenReturn(-2L);\n            assertEquals(-1, rebalanceImpl.computePullFromWhereWithException(mq));\n        }\n    }\n\n    @Test\n    public void testComputePullFromWhereWithException_eq_minus1_last() throws MQClientException {\n        when(offsetStore.readOffset(any(MessageQueue.class), any(ReadOffsetType.class))).thenReturn(-1L);\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);\n        when(admin.maxOffset(any(MessageQueue.class))).thenReturn(12345L);\n\n        assertEquals(12345L, rebalanceImpl.computePullFromWhereWithException(mq));\n\n        assertEquals(0L, rebalanceImpl.computePullFromWhereWithException(retryMq));\n    }\n\n    @Test\n    public void testComputePullFromWhereWithException_eq_minus1_first() throws MQClientException {\n        when(offsetStore.readOffset(any(MessageQueue.class), any(ReadOffsetType.class))).thenReturn(-1L);\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        assertEquals(0, rebalanceImpl.computePullFromWhereWithException(mq));\n    }\n\n    @Test\n    public void testComputePullFromWhereWithException_eq_minus1_timestamp() throws MQClientException {\n        when(offsetStore.readOffset(any(MessageQueue.class), any(ReadOffsetType.class))).thenReturn(-1L);\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_TIMESTAMP);\n        when(admin.searchOffset(any(MessageQueue.class), anyLong())).thenReturn(12345L);\n        when(admin.maxOffset(any(MessageQueue.class))).thenReturn(23456L);\n\n        assertEquals(12345L, rebalanceImpl.computePullFromWhereWithException(mq));\n\n        assertEquals(23456L, rebalanceImpl.computePullFromWhereWithException(retryMq));\n    }\n\n\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImplTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.consumer;\n\nimport java.util.Collections;\nimport java.util.HashSet;\nimport java.util.Set;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely;\nimport org.apache.rocketmq.client.consumer.store.OffsetStore;\nimport org.apache.rocketmq.client.consumer.store.ReadOffsetType;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.MQAdminImpl;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.junit.Assert.assertEquals;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class RebalancePushImplTest {\n    @Spy\n    private DefaultMQPushConsumerImpl defaultMQPushConsumer = new DefaultMQPushConsumerImpl(new DefaultMQPushConsumer(\"RebalancePushImplTest\"), null);\n    @Mock\n    private MQClientInstance mqClientInstance;\n    private OffsetStore offsetStore = mock(OffsetStore.class);\n    private String consumerGroup = \"CID_RebalancePushImplTest\";\n    private String topic = \"TopicA\";\n    private MessageQueue mq = new MessageQueue(\"topic1\", \"broker1\", 0);\n    private MessageQueue retryMq = new MessageQueue(MixAll.RETRY_GROUP_TOPIC_PREFIX + \"group\", \"broker1\", 0);\n    private DefaultMQPushConsumerImpl consumerImpl = mock(DefaultMQPushConsumerImpl.class);\n    private RebalancePushImpl rebalanceImpl = new RebalancePushImpl(consumerImpl);\n    private DefaultMQPushConsumer consumer = new DefaultMQPushConsumer();\n    private MQClientInstance client = mock(MQClientInstance.class);\n    private MQAdminImpl admin = mock(MQAdminImpl.class);\n\n    public RebalancePushImplTest() {\n        when(consumerImpl.getDefaultMQPushConsumer()).thenReturn(consumer);\n        when(consumerImpl.getOffsetStore()).thenReturn(offsetStore);\n        rebalanceImpl.setmQClientFactory(client);\n        when(client.getMQAdminImpl()).thenReturn(admin);\n    }\n\n    @Test\n    public void testMessageQueueChanged_CountThreshold() {\n        RebalancePushImpl rebalancePush = new RebalancePushImpl(consumerGroup, MessageModel.CLUSTERING,\n            new AllocateMessageQueueAveragely(), mqClientInstance, defaultMQPushConsumer);\n        init(rebalancePush);\n\n        // Just set pullThresholdForQueue\n        defaultMQPushConsumer.getDefaultMQPushConsumer().setPullThresholdForQueue(1024);\n        Set<MessageQueue> allocateResultSet = new HashSet<>();\n        allocateResultSet.add(new MessageQueue(topic, \"BrokerA\", 0));\n        allocateResultSet.add(new MessageQueue(topic, \"BrokerA\", 1));\n        doRebalanceForcibly(rebalancePush, allocateResultSet);\n        assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdForQueue()).isEqualTo(1024);\n\n        // Set pullThresholdForTopic\n        defaultMQPushConsumer.getDefaultMQPushConsumer().setPullThresholdForTopic(1024);\n        doRebalanceForcibly(rebalancePush, allocateResultSet);\n        assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdForQueue()).isEqualTo(512);\n\n        // Change message queue allocate result\n        allocateResultSet.add(new MessageQueue(topic, \"BrokerA\", 2));\n        doRebalanceForcibly(rebalancePush, allocateResultSet);\n        assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdForQueue()).isEqualTo(341);\n    }\n\n    private void doRebalanceForcibly(RebalancePushImpl rebalancePush, Set<MessageQueue> allocateResultSet) {\n        rebalancePush.topicSubscribeInfoTable.put(topic, allocateResultSet);\n        rebalancePush.doRebalance(false);\n        rebalancePush.messageQueueChanged(topic, allocateResultSet, allocateResultSet);\n    }\n\n    private void init(final RebalancePushImpl rebalancePush) {\n        rebalancePush.getSubscriptionInner().putIfAbsent(topic, new SubscriptionData());\n\n        rebalancePush.subscriptionInner.putIfAbsent(topic, new SubscriptionData());\n\n        when(mqClientInstance.findConsumerIdList(anyString(), anyString())).thenReturn(Collections.singletonList(consumerGroup));\n        when(mqClientInstance.getClientId()).thenReturn(consumerGroup);\n        when(defaultMQPushConsumer.getOffsetStore()).thenReturn(offsetStore);\n    }\n\n    @Test\n    public void testMessageQueueChanged_SizeThreshold() {\n        RebalancePushImpl rebalancePush = new RebalancePushImpl(consumerGroup, MessageModel.CLUSTERING,\n            new AllocateMessageQueueAveragely(), mqClientInstance, defaultMQPushConsumer);\n        init(rebalancePush);\n\n        // Just set pullThresholdSizeForQueue\n        defaultMQPushConsumer.getDefaultMQPushConsumer().setPullThresholdSizeForQueue(1024);\n        Set<MessageQueue> allocateResultSet = new HashSet<>();\n        allocateResultSet.add(new MessageQueue(topic, \"BrokerA\", 0));\n        allocateResultSet.add(new MessageQueue(topic, \"BrokerA\", 1));\n        doRebalanceForcibly(rebalancePush, allocateResultSet);\n        assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdSizeForQueue()).isEqualTo(1024);\n\n        // Set pullThresholdSizeForTopic\n        defaultMQPushConsumer.getDefaultMQPushConsumer().setPullThresholdSizeForTopic(1024);\n        doRebalanceForcibly(rebalancePush, allocateResultSet);\n        assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdSizeForQueue()).isEqualTo(512);\n\n        // Change message queue allocate result\n        allocateResultSet.add(new MessageQueue(topic, \"BrokerA\", 2));\n        doRebalanceForcibly(rebalancePush, allocateResultSet);\n        assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdSizeForQueue()).isEqualTo(341);\n    }\n\n    @Test\n    public void testMessageQueueChanged_ConsumerRuntimeInfo() throws MQClientException {\n        RebalancePushImpl rebalancePush = new RebalancePushImpl(consumerGroup, MessageModel.CLUSTERING,\n            new AllocateMessageQueueAveragely(), mqClientInstance, defaultMQPushConsumer);\n        init(rebalancePush);\n\n        defaultMQPushConsumer.getDefaultMQPushConsumer().setPullThresholdSizeForQueue(1024);\n        defaultMQPushConsumer.getDefaultMQPushConsumer().setPullThresholdForQueue(1024);\n        Set<MessageQueue> allocateResultSet = new HashSet<>();\n        allocateResultSet.add(new MessageQueue(topic, \"BrokerA\", 0));\n        allocateResultSet.add(new MessageQueue(topic, \"BrokerA\", 1));\n        doRebalanceForcibly(rebalancePush, allocateResultSet);\n\n        defaultMQPushConsumer.setConsumeMessageService(new ConsumeMessageConcurrentlyService(defaultMQPushConsumer, null));\n        assertThat(defaultMQPushConsumer.consumerRunningInfo().getProperties().get(\"pullThresholdSizeForQueue\")).isEqualTo(\"1024\");\n        assertThat(defaultMQPushConsumer.consumerRunningInfo().getProperties().get(\"pullThresholdForQueue\")).isEqualTo(\"1024\");\n        assertThat(defaultMQPushConsumer.consumerRunningInfo().getProperties().get(\"pullThresholdSizeForTopic\")).isEqualTo(\"-1\");\n        assertThat(defaultMQPushConsumer.consumerRunningInfo().getProperties().get(\"pullThresholdForTopic\")).isEqualTo(\"-1\");\n\n        defaultMQPushConsumer.getDefaultMQPushConsumer().setPullThresholdSizeForTopic(1024);\n        defaultMQPushConsumer.getDefaultMQPushConsumer().setPullThresholdForTopic(1024);\n        doRebalanceForcibly(rebalancePush, allocateResultSet);\n        assertThat(defaultMQPushConsumer.consumerRunningInfo().getProperties().get(\"pullThresholdSizeForQueue\")).isEqualTo(\"512\");\n        assertThat(defaultMQPushConsumer.consumerRunningInfo().getProperties().get(\"pullThresholdForQueue\")).isEqualTo(\"512\");\n        assertThat(defaultMQPushConsumer.consumerRunningInfo().getProperties().get(\"pullThresholdSizeForTopic\")).isEqualTo(\"1024\");\n        assertThat(defaultMQPushConsumer.consumerRunningInfo().getProperties().get(\"pullThresholdForTopic\")).isEqualTo(\"1024\");\n\n        // Change message queue allocate result\n        allocateResultSet.add(new MessageQueue(topic, \"BrokerA\", 2));\n        doRebalanceForcibly(rebalancePush, allocateResultSet);\n        assertThat(defaultMQPushConsumer.consumerRunningInfo().getProperties().get(\"pullThresholdSizeForQueue\")).isEqualTo(\"341\");\n        assertThat(defaultMQPushConsumer.consumerRunningInfo().getProperties().get(\"pullThresholdForQueue\")).isEqualTo(\"341\");\n        assertThat(defaultMQPushConsumer.consumerRunningInfo().getProperties().get(\"pullThresholdSizeForTopic\")).isEqualTo(\"1024\");\n        assertThat(defaultMQPushConsumer.consumerRunningInfo().getProperties().get(\"pullThresholdForTopic\")).isEqualTo(\"1024\");\n    }\n\n    @Test\n    public void testComputePullFromWhereWithException_ne_minus1() throws MQClientException {\n        for (ConsumeFromWhere where : new ConsumeFromWhere[]{\n            ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET,\n            ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET,\n            ConsumeFromWhere.CONSUME_FROM_TIMESTAMP}) {\n            consumer.setConsumeFromWhere(where);\n\n            when(offsetStore.readOffset(any(MessageQueue.class), any(ReadOffsetType.class))).thenReturn(0L);\n            assertEquals(0, rebalanceImpl.computePullFromWhereWithException(mq));\n        }\n    }\n\n    @Test\n    public void testComputePullFromWhereWithException_eq_minus1_last() throws MQClientException {\n        when(offsetStore.readOffset(any(MessageQueue.class), any(ReadOffsetType.class))).thenReturn(-1L);\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);\n        when(admin.maxOffset(any(MessageQueue.class))).thenReturn(12345L);\n\n        assertEquals(12345L, rebalanceImpl.computePullFromWhereWithException(mq));\n\n        assertEquals(0L, rebalanceImpl.computePullFromWhereWithException(retryMq));\n    }\n\n    @Test\n    public void testComputePullFromWhereWithException_eq_minus1_first() throws MQClientException {\n        when(offsetStore.readOffset(any(MessageQueue.class), any(ReadOffsetType.class))).thenReturn(-1L);\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        assertEquals(0, rebalanceImpl.computePullFromWhereWithException(mq));\n    }\n\n    @Test\n    public void testComputePullFromWhereWithException_eq_minus1_timestamp() throws MQClientException {\n        when(offsetStore.readOffset(any(MessageQueue.class), any(ReadOffsetType.class))).thenReturn(-1L);\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_TIMESTAMP);\n        when(admin.searchOffset(any(MessageQueue.class), anyLong())).thenReturn(12345L);\n        when(admin.maxOffset(any(MessageQueue.class))).thenReturn(23456L);\n\n        assertEquals(12345L, rebalanceImpl.computePullFromWhereWithException(mq));\n\n        assertEquals(23456L, rebalanceImpl.computePullFromWhereWithException(retryMq));\n    }\n\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/impl/factory/MQClientInstanceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.impl.factory;\n\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.admin.MQAdminExtInner;\nimport org.apache.rocketmq.client.consumer.store.OffsetStore;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.FindBrokerResult;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.MQClientManager;\nimport org.apache.rocketmq.client.impl.consumer.ConsumeMessageService;\nimport org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl;\nimport org.apache.rocketmq.client.impl.consumer.MQConsumerInner;\nimport org.apache.rocketmq.client.impl.consumer.ProcessQueue;\nimport org.apache.rocketmq.client.impl.consumer.RebalanceImpl;\nimport org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;\nimport org.apache.rocketmq.client.impl.producer.TopicPublishInfo;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.message.MessageQueueAssignment;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.remoting.RemotingClient;\nimport org.apache.rocketmq.remoting.common.HeartbeatV2Result;\nimport org.apache.rocketmq.remoting.exception.RemotingConnectException;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.lang.reflect.Field;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.nio.charset.StandardCharsets;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Properties;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.ExecutorService;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertThrows;\nimport static org.junit.Assert.assertTrue;\nimport static org.junit.Assert.fail;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.doAnswer;\nimport static org.mockito.Mockito.doThrow;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class MQClientInstanceTest {\n\n    @Mock\n    private MQClientAPIImpl mQClientAPIImpl;\n\n    @Mock\n    private RemotingClient remotingClient;\n\n    @Mock\n    private ClientConfig clientConfig;\n\n    private final MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig());\n\n    private final String topic = \"FooBar\";\n\n    private final String group = \"FooBarGroup\";\n\n    private final String defaultBrokerAddr = \"127.0.0.1:10911\";\n\n    private final String defaultBroker = \"BrokerA\";\n\n    private final ConcurrentMap<String, HashMap<Long, String>> brokerAddrTable = new ConcurrentHashMap<>();\n\n    private final ConcurrentMap<String, MQConsumerInner> consumerTable = new ConcurrentHashMap<>();\n\n    private final ConcurrentMap<String, TopicRouteData> topicRouteTable = new ConcurrentHashMap<>();\n\n    @Before\n    public void init() throws Exception {\n        when(mQClientAPIImpl.getRemotingClient()).thenReturn(remotingClient);\n        FieldUtils.writeDeclaredField(mqClientInstance, \"brokerAddrTable\", brokerAddrTable, true);\n        FieldUtils.writeDeclaredField(mqClientInstance, \"mQClientAPIImpl\", mQClientAPIImpl, true);\n        FieldUtils.writeDeclaredField(mqClientInstance, \"consumerTable\", consumerTable, true);\n        FieldUtils.writeDeclaredField(mqClientInstance, \"clientConfig\", clientConfig, true);\n        FieldUtils.writeDeclaredField(mqClientInstance, \"topicRouteTable\", topicRouteTable, true);\n    }\n\n    @After\n    public void tearDown() throws Exception {\n        brokerAddrTable.clear();\n        consumerTable.clear();\n        topicRouteTable.clear();\n    }\n\n    @Test\n    public void testFindBrokerAddressInSubscribe() {\n        // dledger normal case\n        String brokerName = \"BrokerA\";\n        HashMap<Long, String> addrMap = new HashMap<>();\n        addrMap.put(0L, \"127.0.0.1:10911\");\n        addrMap.put(1L, \"127.0.0.1:10912\");\n        addrMap.put(2L, \"127.0.0.1:10913\");\n        brokerAddrTable.put(brokerName, addrMap);\n        long brokerId = 1;\n        FindBrokerResult brokerResult = mqClientInstance.findBrokerAddressInSubscribe(brokerName, brokerId, false);\n        assertThat(brokerResult).isNotNull();\n        assertThat(brokerResult.getBrokerAddr()).isEqualTo(\"127.0.0.1:10912\");\n        assertThat(brokerResult.isSlave()).isTrue();\n\n        // dledger case, when node n0 was voted as the leader\n        brokerName = \"BrokerB\";\n        HashMap<Long, String> addrMapNew = new HashMap<>();\n        addrMapNew.put(0L, \"127.0.0.1:10911\");\n        addrMapNew.put(2L, \"127.0.0.1:10912\");\n        addrMapNew.put(3L, \"127.0.0.1:10913\");\n        brokerAddrTable.put(brokerName, addrMapNew);\n        brokerResult = mqClientInstance.findBrokerAddressInSubscribe(brokerName, brokerId, false);\n        assertThat(brokerResult).isNotNull();\n        assertThat(brokerResult.getBrokerAddr()).isEqualTo(\"127.0.0.1:10912\");\n        assertThat(brokerResult.isSlave()).isTrue();\n    }\n\n    @Test\n    public void testRegisterProducer() {\n        boolean flag = mqClientInstance.registerProducer(group, mock(DefaultMQProducerImpl.class));\n        assertThat(flag).isTrue();\n\n        flag = mqClientInstance.registerProducer(group, mock(DefaultMQProducerImpl.class));\n        assertThat(flag).isFalse();\n\n        mqClientInstance.unregisterProducer(group);\n        flag = mqClientInstance.registerProducer(group, mock(DefaultMQProducerImpl.class));\n        assertThat(flag).isTrue();\n    }\n\n    @Test\n    public void testRegisterConsumer() {\n        boolean flag = mqClientInstance.registerConsumer(group, mock(MQConsumerInner.class));\n        assertThat(flag).isTrue();\n\n        flag = mqClientInstance.registerConsumer(group, mock(MQConsumerInner.class));\n        assertThat(flag).isFalse();\n\n        mqClientInstance.unregisterConsumer(group);\n        flag = mqClientInstance.registerConsumer(group, mock(MQConsumerInner.class));\n        assertThat(flag).isTrue();\n    }\n\n    @Test\n    public void testConsumerRunningInfoWhenConsumersIsEmptyOrNot() throws RemotingException, InterruptedException, MQBrokerException {\n        MQConsumerInner mockConsumerInner = mock(MQConsumerInner.class);\n        ConsumerRunningInfo mockConsumerRunningInfo = mock(ConsumerRunningInfo.class);\n        when(mockConsumerInner.consumerRunningInfo()).thenReturn(mockConsumerRunningInfo);\n        when(mockConsumerInner.consumeType()).thenReturn(ConsumeType.CONSUME_PASSIVELY);\n        Properties properties = new Properties();\n        when(mockConsumerRunningInfo.getProperties()).thenReturn(properties);\n        mqClientInstance.unregisterConsumer(group);\n\n        ConsumerRunningInfo runningInfo = mqClientInstance.consumerRunningInfo(group);\n        assertThat(runningInfo).isNull();\n        boolean flag = mqClientInstance.registerConsumer(group, mockConsumerInner);\n        assertThat(flag).isTrue();\n\n        runningInfo = mqClientInstance.consumerRunningInfo(group);\n        assertThat(runningInfo).isNotNull();\n        assertThat(mockConsumerInner.consumerRunningInfo().getProperties().get(ConsumerRunningInfo.PROP_CONSUME_TYPE)).isNotNull();\n\n        mqClientInstance.unregisterConsumer(group);\n        flag = mqClientInstance.registerConsumer(group, mock(MQConsumerInner.class));\n        assertThat(flag).isTrue();\n    }\n\n    @Test\n    public void testRegisterAdminExt() {\n        boolean flag = mqClientInstance.registerAdminExt(group, mock(MQAdminExtInner.class));\n        assertThat(flag).isTrue();\n\n        flag = mqClientInstance.registerAdminExt(group, mock(MQAdminExtInner.class));\n        assertThat(flag).isFalse();\n\n        mqClientInstance.unregisterAdminExt(group);\n        flag = mqClientInstance.registerAdminExt(group, mock(MQAdminExtInner.class));\n        assertThat(flag).isTrue();\n    }\n\n    @Test\n    public void testTopicRouteData2TopicPublishInfo() {\n        TopicPublishInfo actual = MQClientInstance.topicRouteData2TopicPublishInfo(topic, createTopicRouteData());\n        assertThat(actual.isHaveTopicRouterInfo()).isFalse();\n        assertThat(actual.getMessageQueueList().size()).isEqualTo(4);\n    }\n\n    @Test\n    public void testTopicRouteData2TopicPublishInfoWithOrderTopicConf() {\n        TopicRouteData topicRouteData = createTopicRouteData();\n        topicRouteData.setOrderTopicConf(\"127.0.0.1:4\");\n        TopicPublishInfo actual = MQClientInstance.topicRouteData2TopicPublishInfo(topic, topicRouteData);\n        assertFalse(actual.isHaveTopicRouterInfo());\n        assertEquals(4, actual.getMessageQueueList().size());\n    }\n\n    @Test\n    public void testTopicRouteData2TopicPublishInfoWithTopicQueueMappingByBroker() {\n        TopicRouteData topicRouteData = createTopicRouteData();\n        topicRouteData.setTopicQueueMappingByBroker(Collections.singletonMap(topic, new TopicQueueMappingInfo()));\n        TopicPublishInfo actual = MQClientInstance.topicRouteData2TopicPublishInfo(topic, topicRouteData);\n        assertFalse(actual.isHaveTopicRouterInfo());\n        assertEquals(0, actual.getMessageQueueList().size());\n    }\n\n    @Test\n    public void testTopicRouteData2TopicSubscribeInfo() {\n        TopicRouteData topicRouteData = createTopicRouteData();\n        topicRouteData.setTopicQueueMappingByBroker(Collections.singletonMap(topic, new TopicQueueMappingInfo()));\n        Set<MessageQueue> actual = MQClientInstance.topicRouteData2TopicSubscribeInfo(topic, topicRouteData);\n        assertNotNull(actual);\n        assertEquals(0, actual.size());\n    }\n\n    @Test\n    public void testParseOffsetTableFromBroker() {\n        Map<MessageQueue, Long> offsetTable = new HashMap<>();\n        offsetTable.put(new MessageQueue(), 0L);\n        Map<MessageQueue, Long> actual = mqClientInstance.parseOffsetTableFromBroker(offsetTable, \"defaultNamespace\");\n        assertNotNull(actual);\n        assertEquals(1, actual.size());\n    }\n\n    @Test\n    public void testCheckClientInBroker() throws MQClientException, RemotingSendRequestException, RemotingConnectException, RemotingTimeoutException, InterruptedException {\n        doThrow(new MQClientException(\"checkClientInBroker exception\", null)).when(mQClientAPIImpl).checkClientInBroker(\n                any(),\n                any(),\n                any(),\n                any(SubscriptionData.class),\n                anyLong());\n        topicRouteTable.put(topic, createTopicRouteData());\n        MQConsumerInner mqConsumerInner = createMQConsumerInner();\n        mqConsumerInner.subscriptions().clear();\n        SubscriptionData subscriptionData = new SubscriptionData();\n        subscriptionData.setTopic(topic);\n        subscriptionData.setExpressionType(\"type\");\n        mqConsumerInner.subscriptions().add(subscriptionData);\n        consumerTable.put(group, mqConsumerInner);\n        Throwable thrown = assertThrows(MQClientException.class, mqClientInstance::checkClientInBroker);\n        assertTrue(thrown.getMessage().contains(\"checkClientInBroker exception\"));\n    }\n\n    @Test\n    public void testSendHeartbeatToBrokerV1() {\n        consumerTable.put(group, createMQConsumerInner());\n        assertTrue(mqClientInstance.sendHeartbeatToBroker(0L, defaultBroker, defaultBrokerAddr));\n    }\n\n    @Test\n    public void testSendHeartbeatToBrokerV2() throws MQBrokerException, RemotingException, InterruptedException {\n        consumerTable.put(group, createMQConsumerInner());\n        when(clientConfig.isUseHeartbeatV2()).thenReturn(true);\n        HeartbeatV2Result heartbeatV2Result = mock(HeartbeatV2Result.class);\n        when(heartbeatV2Result.isSupportV2()).thenReturn(true);\n        when(mQClientAPIImpl.sendHeartbeatV2(any(), any(HeartbeatData.class), anyLong())).thenReturn(heartbeatV2Result);\n        assertTrue(mqClientInstance.sendHeartbeatToBroker(0L, defaultBroker, defaultBrokerAddr));\n    }\n\n    @Test\n    public void testSendHeartbeatToAllBrokerWithLockV1() {\n        brokerAddrTable.put(defaultBroker, createBrokerAddrMap());\n        consumerTable.put(group, createMQConsumerInner());\n        assertTrue(mqClientInstance.sendHeartbeatToAllBrokerWithLock());\n    }\n\n    @Test\n    public void testSendHeartbeatToAllBrokerWithLockV2() {\n        brokerAddrTable.put(defaultBroker, createBrokerAddrMap());\n        consumerTable.put(group, createMQConsumerInner());\n        when(clientConfig.isUseHeartbeatV2()).thenReturn(true);\n        assertTrue(mqClientInstance.sendHeartbeatToAllBrokerWithLock());\n    }\n\n    @Test\n    public void testUpdateTopicRouteInfoFromNameServer() throws RemotingException, InterruptedException, MQClientException {\n        brokerAddrTable.put(defaultBroker, createBrokerAddrMap());\n        consumerTable.put(group, createMQConsumerInner());\n        DefaultMQProducer defaultMQProducer = mock(DefaultMQProducer.class);\n        TopicRouteData topicRouteData = createTopicRouteData();\n        when(mQClientAPIImpl.getDefaultTopicRouteInfoFromNameServer(anyLong())).thenReturn(topicRouteData);\n        assertTrue(mqClientInstance.updateTopicRouteInfoFromNameServer(topic, true, defaultMQProducer));\n        assertEquals(topicRouteData, topicRouteTable.get(topic));\n    }\n\n    @Test\n    public void testFindBrokerAddressInAdmin() {\n        brokerAddrTable.put(defaultBroker, createBrokerAddrMap());\n        consumerTable.put(group, createMQConsumerInner());\n        FindBrokerResult actual = mqClientInstance.findBrokerAddressInAdmin(defaultBroker);\n        assertNotNull(actual);\n        assertEquals(defaultBrokerAddr, actual.getBrokerAddr());\n    }\n\n    @Test\n    public void testFindBrokerAddressInSubscribeWithOneBroker() throws IllegalAccessException {\n        brokerAddrTable.put(defaultBroker, createBrokerAddrMap());\n        consumerTable.put(group, createMQConsumerInner());\n        ConcurrentMap<String, ConcurrentHashMap<String, Integer>> brokerVersionTable = new ConcurrentHashMap<>();\n        ConcurrentHashMap<String, Integer> addressMap = new ConcurrentHashMap<>();\n        addressMap.put(defaultBrokerAddr, 0);\n        brokerVersionTable.put(defaultBroker, addressMap);\n        FieldUtils.writeDeclaredField(mqClientInstance, \"brokerVersionTable\", brokerVersionTable, true);\n        FindBrokerResult actual = mqClientInstance.findBrokerAddressInSubscribe(defaultBroker, 1L, false);\n        assertNotNull(actual);\n        assertEquals(defaultBrokerAddr, actual.getBrokerAddr());\n    }\n\n    @Test\n    public void testFindConsumerIdList() {\n        topicRouteTable.put(topic, createTopicRouteData());\n        brokerAddrTable.put(defaultBroker, createBrokerAddrMap());\n        consumerTable.put(group, createMQConsumerInner());\n        List<String> actual = mqClientInstance.findConsumerIdList(topic, group);\n        assertNotNull(actual);\n        assertEquals(0, actual.size());\n    }\n\n    @Test\n    public void testQueryAssignment() throws MQBrokerException, RemotingException, InterruptedException {\n        topicRouteTable.put(topic, createTopicRouteData());\n        brokerAddrTable.put(defaultBroker, createBrokerAddrMap());\n        consumerTable.put(group, createMQConsumerInner());\n        Set<MessageQueueAssignment> actual = mqClientInstance.queryAssignment(topic, group, \"\", MessageModel.CLUSTERING, 1000);\n        assertNotNull(actual);\n        assertEquals(0, actual.size());\n    }\n\n    @Test\n    public void testResetOffset() throws IllegalAccessException {\n        topicRouteTable.put(topic, createTopicRouteData());\n        brokerAddrTable.put(defaultBroker, createBrokerAddrMap());\n        consumerTable.put(group, createMQConsumerInner());\n        Map<MessageQueue, Long> offsetTable = new HashMap<>();\n        offsetTable.put(createMessageQueue(), 0L);\n        mqClientInstance.resetOffset(topic, group, offsetTable);\n        Field consumerTableField = FieldUtils.getDeclaredField(mqClientInstance.getClass(), \"consumerTable\", true);\n        ConcurrentMap<String, MQConsumerInner> consumerTable = (ConcurrentMap<String, MQConsumerInner>) consumerTableField.get(mqClientInstance);\n        DefaultMQPushConsumerImpl consumer = (DefaultMQPushConsumerImpl) consumerTable.get(group);\n        verify(consumer).suspend();\n        verify(consumer).resume();\n        verify(consumer, times(1))\n                .updateConsumeOffset(\n                        any(MessageQueue.class),\n                        eq(0L));\n    }\n\n    @Test\n    public void testGetConsumerStatus() {\n        topicRouteTable.put(topic, createTopicRouteData());\n        brokerAddrTable.put(defaultBroker, createBrokerAddrMap());\n        consumerTable.put(group, createMQConsumerInner());\n        Map<MessageQueue, Long> actual = mqClientInstance.getConsumerStatus(topic, group);\n        assertNotNull(actual);\n        assertEquals(0, actual.size());\n    }\n\n    @Test\n    public void testGetAnExistTopicRouteData() {\n        topicRouteTable.put(topic, createTopicRouteData());\n        TopicRouteData actual = mqClientInstance.getAnExistTopicRouteData(topic);\n        assertNotNull(actual);\n        assertNotNull(actual.getQueueDatas());\n        assertNotNull(actual.getBrokerDatas());\n    }\n\n    @Test\n    public void testConsumeMessageDirectly() {\n        consumerTable.put(group, createMQConsumerInner());\n        assertNull(mqClientInstance.consumeMessageDirectly(createMessageExt(), group, defaultBroker));\n    }\n\n    @Test\n    public void testQueryTopicRouteData() {\n        consumerTable.put(group, createMQConsumerInner());\n        topicRouteTable.put(topic, createTopicRouteData());\n        TopicRouteData actual = mqClientInstance.queryTopicRouteData(topic);\n        assertNotNull(actual);\n        assertNotNull(actual.getQueueDatas());\n        assertNotNull(actual.getBrokerDatas());\n    }\n\n    private MessageExt createMessageExt() {\n        MessageExt result = new MessageExt();\n        result.setBody(\"body\".getBytes(StandardCharsets.UTF_8));\n        result.setTopic(topic);\n        result.setBrokerName(defaultBroker);\n        result.putUserProperty(\"key\", \"value\");\n        result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, group);\n        result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, \"TX1\");\n        long curTime = System.currentTimeMillis();\n        result.setBornTimestamp(curTime - 1000);\n        result.getProperties().put(MessageConst.PROPERTY_POP_CK, curTime + \" \" + curTime + \" \" + curTime + \" \" + curTime);\n        result.setKeys(\"keys\");\n        result.setSysFlag(MessageSysFlag.INNER_BATCH_FLAG);\n        result.setSysFlag(result.getSysFlag() | MessageSysFlag.NEED_UNWRAP_FLAG);\n        SocketAddress bornHost = new InetSocketAddress(\"127.0.0.1\", 12911);\n        SocketAddress storeHost = new InetSocketAddress(\"127.0.0.1\", 10911);\n        result.setBornHost(bornHost);\n        result.setStoreHost(storeHost);\n        return result;\n    }\n\n    private MessageQueue createMessageQueue() {\n        MessageQueue result = new MessageQueue();\n        result.setQueueId(0);\n        result.setBrokerName(defaultBroker);\n        result.setTopic(topic);\n        return result;\n    }\n\n    private TopicRouteData createTopicRouteData() {\n        TopicRouteData result = new TopicRouteData();\n        result.setBrokerDatas(createBrokerDatas());\n        result.setQueueDatas(createQueueDatas());\n        return result;\n    }\n\n    private HashMap<Long, String> createBrokerAddrMap() {\n        HashMap<Long, String> result = new HashMap<>();\n        result.put(0L, defaultBrokerAddr);\n        return result;\n    }\n\n    private MQConsumerInner createMQConsumerInner() {\n        DefaultMQPushConsumerImpl result = mock(DefaultMQPushConsumerImpl.class);\n        Set<SubscriptionData> subscriptionDataSet = new HashSet<>();\n        SubscriptionData subscriptionData = mock(SubscriptionData.class);\n        subscriptionDataSet.add(subscriptionData);\n        when(result.subscriptions()).thenReturn(subscriptionDataSet);\n        RebalanceImpl rebalanceImpl = mock(RebalanceImpl.class);\n        ConcurrentMap<MessageQueue, ProcessQueue> processQueueMap = new ConcurrentHashMap<>();\n        ProcessQueue processQueue = new ProcessQueue();\n        processQueueMap.put(createMessageQueue(), processQueue);\n        when(rebalanceImpl.getProcessQueueTable()).thenReturn(processQueueMap);\n        when(result.getRebalanceImpl()).thenReturn(rebalanceImpl);\n        OffsetStore offsetStore = mock(OffsetStore.class);\n        when(result.getOffsetStore()).thenReturn(offsetStore);\n        ConsumeMessageService consumeMessageService = mock(ConsumeMessageService.class);\n        when(result.getConsumeMessageService()).thenReturn(consumeMessageService);\n        return result;\n    }\n\n    private List<QueueData> createQueueDatas() {\n        QueueData queueData = new QueueData();\n        queueData.setBrokerName(defaultBroker);\n        queueData.setPerm(6);\n        queueData.setReadQueueNums(3);\n        queueData.setWriteQueueNums(4);\n        queueData.setTopicSysFlag(0);\n        return Collections.singletonList(queueData);\n    }\n\n    private List<BrokerData> createBrokerDatas() {\n        BrokerData brokerData = new BrokerData();\n        brokerData.setBrokerName(defaultBroker);\n        String defaultCluster = \"defaultCluster\";\n        brokerData.setCluster(defaultCluster);\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(MixAll.MASTER_ID, defaultBrokerAddr);\n        brokerData.setBrokerAddrs(brokerAddrs);\n        return Collections.singletonList(brokerData);\n    }\n\n    @Test\n    public void testSendHeartbeatToAllBrokerConcurrently() {\n        try {\n            String brokerName = \"BrokerA\";\n            HashMap<Long, String> addrMap = new HashMap<>();\n            addrMap.put(0L, \"127.0.0.1:10911\");\n            addrMap.put(1L, \"127.0.0.1:10912\");\n            addrMap.put(2L, \"127.0.0.1:10913\");\n            brokerAddrTable.put(brokerName, addrMap);\n\n            DefaultMQPushConsumerImpl mockConsumer = mock(DefaultMQPushConsumerImpl.class);\n            when(mockConsumer.subscriptions()).thenReturn(Collections.singleton(new SubscriptionData()));\n            mqClientInstance.registerConsumer(\"TestConsumerGroup\", mockConsumer);\n\n            ClientConfig clientConfig = new ClientConfig();\n            FieldUtils.writeDeclaredField(clientConfig, \"enableConcurrentHeartbeat\", true, true);\n            FieldUtils.writeDeclaredField(mqClientInstance, \"clientConfig\", clientConfig, true);\n\n            ExecutorService mockExecutor = mock(ExecutorService.class);\n            doAnswer(invocation -> {\n                try {\n                    Runnable task = invocation.getArgument(0);\n                    task.run();\n                } catch (Exception e) {\n                    // ignore\n                }\n                return null;\n            }).when(mockExecutor).execute(any(Runnable.class));\n            FieldUtils.writeDeclaredField(mqClientInstance, \"concurrentHeartbeatExecutor\", mockExecutor, true);\n            MQClientAPIImpl mockMqClientAPIImpl = mock(MQClientAPIImpl.class);\n            FieldUtils.writeDeclaredField(mqClientInstance, \"mQClientAPIImpl\", mockMqClientAPIImpl, true);\n\n            mqClientInstance.sendHeartbeatToAllBrokerWithLock();\n\n            assertTrue(true);\n\n        } catch (Exception e) {\n            fail(\"failed: \" + e.getMessage());\n        }\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExtTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.impl.mqclient;\n\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.CompletionException;\nimport java.util.concurrent.ExecutionException;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.utils.FutureUtils;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingClient;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThatThrownBy;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNull;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.doReturn;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class MQClientAPIExtTest {\n    MQClientAPIExt mqClientAPIExt;\n    @Mock\n    NettyRemotingClient remotingClientMock;\n\n    @Before\n    public void before() {\n        mqClientAPIExt = Mockito.spy(new MQClientAPIExt(new ClientConfig(), new NettyClientConfig(), null, null));\n        Mockito.when(mqClientAPIExt.getRemotingClient()).thenReturn(remotingClientMock);\n        Mockito.when(remotingClientMock.invoke(anyString(), any(), anyLong())).thenReturn(FutureUtils.completeExceptionally(new RemotingTimeoutException(\"addr\")));\n    }\n\n    @Test\n    public void sendMessageAsync() {\n        String topic = \"test\";\n        Message msg = new Message(topic, \"test\".getBytes());\n        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setProducerGroup(\"test\");\n        requestHeader.setDefaultTopic(\"test\");\n        requestHeader.setDefaultTopicQueueNums(1);\n        requestHeader.setQueueId(0);\n        requestHeader.setSysFlag(0);\n        requestHeader.setBornTimestamp(0L);\n        requestHeader.setFlag(0);\n        requestHeader.setProperties(\"test\");\n        requestHeader.setReconsumeTimes(0);\n        requestHeader.setUnitMode(false);\n        requestHeader.setBatch(false);\n        CompletableFuture<SendResult> future = mqClientAPIExt.sendMessageAsync(\"127.0.0.1:10911\", \"test\", msg, requestHeader, 10);\n        assertThatThrownBy(future::get).getCause().isInstanceOf(RemotingTimeoutException.class);\n    }\n\n    @Test\n    public void testUpdateConsumerOffsetAsync_Success() throws ExecutionException, InterruptedException {\n        CompletableFuture<RemotingCommand> remotingFuture = new CompletableFuture<>();\n        remotingFuture.complete(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"\"));\n        doReturn(remotingFuture).when(remotingClientMock).invoke(anyString(), any(RemotingCommand.class), anyLong());\n\n        CompletableFuture<Void> future = mqClientAPIExt.updateConsumerOffsetAsync(\"brokerAddr\", new UpdateConsumerOffsetRequestHeader(), 3000L);\n\n        assertNull(\"Future should be completed without exception\", future.get());\n    }\n\n    @Test\n    public void testUpdateConsumerOffsetAsync_Fail() throws InterruptedException {\n\n        CompletableFuture<RemotingCommand> remotingFuture = new CompletableFuture<>();\n        remotingFuture.complete(RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, \"QueueId is null, topic is testTopic\"));\n        doReturn(remotingFuture).when(remotingClientMock).invoke(anyString(), any(RemotingCommand.class), anyLong());\n\n        CompletableFuture<Void> future = mqClientAPIExt.updateConsumerOffsetAsync(\"brokerAddr\", new UpdateConsumerOffsetRequestHeader(), 3000L);\n\n        try {\n            future.get();\n        } catch (ExecutionException e) {\n            MQBrokerException customEx = (MQBrokerException) e.getCause();\n            assertEquals(customEx.getResponseCode(), ResponseCode.SYSTEM_ERROR);\n            assertEquals(customEx.getErrorMessage(), \"QueueId is null, topic is testTopic\");\n        }\n    }\n\n    @Test\n    public void testRecallMessageAsync_success() {\n        String msgId = \"msgId\";\n        RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader();\n        requestHeader.setProducerGroup(\"group\");\n        requestHeader.setTopic(\"topic\");\n        requestHeader.setRecallHandle(\"handle\");\n        requestHeader.setBrokerName(\"brokerName\");\n\n        RemotingCommand response = RemotingCommand.createResponseCommand(RecallMessageResponseHeader.class);\n        response.setCode(ResponseCode.SUCCESS);\n        RecallMessageResponseHeader responseHeader = (RecallMessageResponseHeader) response.readCustomHeader();\n        responseHeader.setMsgId(msgId);\n        response.makeCustomHeaderToNet();\n        CompletableFuture<RemotingCommand> remotingFuture = new CompletableFuture<>();\n        remotingFuture.complete(response);\n        doReturn(remotingFuture).when(remotingClientMock).invoke(anyString(), any(RemotingCommand.class), anyLong());\n\n        String resultId =\n            mqClientAPIExt.recallMessageAsync(\"brokerAddr\", requestHeader, 3000L).join();\n        Assert.assertEquals(msgId, resultId);\n    }\n\n    @Test\n    public void testRecallMessageAsync_fail() {\n        RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader();\n        requestHeader.setProducerGroup(\"group\");\n        requestHeader.setTopic(\"topic\");\n        requestHeader.setRecallHandle(\"handle\");\n        requestHeader.setBrokerName(\"brokerName\");\n\n        CompletableFuture<RemotingCommand> remotingFuture = new CompletableFuture<>();\n        remotingFuture.complete(RemotingCommand.createResponseCommand(ResponseCode.SERVICE_NOT_AVAILABLE, \"\"));\n        doReturn(remotingFuture).when(remotingClientMock).invoke(anyString(), any(RemotingCommand.class), anyLong());\n\n        CompletionException exception = Assert.assertThrows(CompletionException.class, () -> {\n            mqClientAPIExt.recallMessageAsync(\"brokerAddr\", requestHeader, 3000L).join();\n        });\n        Assert.assertTrue(exception.getCause() instanceof MQBrokerException);\n        MQBrokerException cause = (MQBrokerException) exception.getCause();\n        Assert.assertEquals(ResponseCode.SERVICE_NOT_AVAILABLE, cause.getResponseCode());\n    }\n}"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPITest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.impl.mqclient;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertThrows;\nimport static org.junit.Assert.assertTrue;\n\nimport java.util.List;\nimport java.util.concurrent.ScheduledExecutorService;\nimport org.apache.rocketmq.client.common.NameserverAccessConfig;\nimport org.apache.rocketmq.client.impl.ClientRemotingProcessor;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\npublic class MQClientAPITest {\n\n    private NameserverAccessConfig nameserverAccessConfig;\n    private final ClientRemotingProcessor clientRemotingProcessor = new DoNothingClientRemotingProcessor(null);\n    private final RPCHook rpcHook = null;\n    private ScheduledExecutorService scheduledExecutorService;\n    private MQClientAPIFactory mqClientAPIFactory;\n\n    @Before\n    public void setUp() {\n        scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor(\"TestScheduledExecutorService\", true);\n    }\n\n    @After\n    public void tearDown() {\n        scheduledExecutorService.shutdownNow();\n    }\n\n    @Test\n    public void testInitWithNamesrvAddr() {\n        nameserverAccessConfig = new NameserverAccessConfig(\"127.0.0.1:9876\", \"\", \"\");\n\n        mqClientAPIFactory = new MQClientAPIFactory(\n            nameserverAccessConfig,\n            \"TestPrefix\",\n            2,\n            clientRemotingProcessor,\n            rpcHook,\n            scheduledExecutorService\n        );\n\n        assertEquals(\"127.0.0.1:9876\", System.getProperty(\"rocketmq.namesrv.addr\"));\n    }\n\n    @Test\n    public void testInitWithNamesrvDomain() {\n        nameserverAccessConfig = new NameserverAccessConfig(\"\", \"test-domain\", \"\");\n\n        mqClientAPIFactory = new MQClientAPIFactory(\n            nameserverAccessConfig,\n            \"TestPrefix\",\n            2,\n            clientRemotingProcessor,\n            rpcHook,\n            scheduledExecutorService\n        );\n\n        assertEquals(\"test-domain\", System.getProperty(\"rocketmq.namesrv.domain\"));\n    }\n\n    @Test\n    public void testInitThrowsExceptionWhenBothEmpty() {\n        nameserverAccessConfig = new NameserverAccessConfig(\"\", \"\", \"\");\n\n        RuntimeException exception = assertThrows(RuntimeException.class, () -> new MQClientAPIFactory(\n            nameserverAccessConfig,\n            \"TestPrefix\",\n            2,\n            clientRemotingProcessor,\n            rpcHook,\n            scheduledExecutorService\n        ));\n\n        assertEquals(\"The configuration item NamesrvAddr is not configured\", exception.getMessage());\n    }\n\n    @Test\n    public void testStartCreatesClients() throws Exception {\n        nameserverAccessConfig = new NameserverAccessConfig(\"127.0.0.1:9876\", \"\", \"\");\n\n        mqClientAPIFactory = new MQClientAPIFactory(\n            nameserverAccessConfig,\n            \"TestPrefix\",\n            2,\n            clientRemotingProcessor,\n            rpcHook,\n            scheduledExecutorService\n        );\n\n        System.setProperty(MixAll.NAMESRV_ADDR_PROPERTY, \"127.0.0.1:123\");\n\n        mqClientAPIFactory.start();\n\n        // Assert\n        MQClientAPIExt client = mqClientAPIFactory.getClient();\n        List<String> nameServerAddressList = client.getNameServerAddressList();\n        assertEquals(1, nameServerAddressList.size());\n        assertEquals(\"127.0.0.1:9876\", nameServerAddressList.get(0));\n    }\n\n    @Test\n    public void testOnNameServerAddressChangeUpdatesAllClients() throws Exception {\n        nameserverAccessConfig = new NameserverAccessConfig(\"127.0.0.1:9876\", \"\", \"\");\n\n        mqClientAPIFactory = new MQClientAPIFactory(\n            nameserverAccessConfig,\n            \"TestPrefix\",\n            2,\n            clientRemotingProcessor,\n            rpcHook,\n            scheduledExecutorService\n        );\n        mqClientAPIFactory.start();\n\n        // Act\n        mqClientAPIFactory.onNameServerAddressChange(\"new-address0;new-address1\");\n\n        MQClientAPIExt client = mqClientAPIFactory.getClient();\n        List<String> nameServerAddressList = client.getNameServerAddressList();\n        assertEquals(2, nameServerAddressList.size());\n        assertTrue(nameServerAddressList.contains(\"new-address0\"));\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImplTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.latency;\n\nimport org.awaitility.core.ThrowingRunnable;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport java.util.concurrent.TimeUnit;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.awaitility.Awaitility.await;\n\npublic class LatencyFaultToleranceImplTest {\n    private LatencyFaultTolerance<String> latencyFaultTolerance;\n    private String brokerName = \"BrokerA\";\n    private String anotherBrokerName = \"BrokerB\";\n\n    @Before\n    public void init() {\n        latencyFaultTolerance = new LatencyFaultToleranceImpl(null, null);\n    }\n\n    @Test\n    public void testUpdateFaultItem() throws Exception {\n        latencyFaultTolerance.updateFaultItem(brokerName, 3000, 3000, true);\n        assertThat(latencyFaultTolerance.isAvailable(brokerName)).isFalse();\n        assertThat(latencyFaultTolerance.isAvailable(anotherBrokerName)).isTrue();\n    }\n\n    @Test\n    public void testIsAvailable() throws Exception {\n        latencyFaultTolerance.updateFaultItem(brokerName, 3000, 50, true);\n        assertThat(latencyFaultTolerance.isAvailable(brokerName)).isFalse();\n\n        await().atMost(500, TimeUnit.MILLISECONDS).untilAsserted(new ThrowingRunnable() {\n            @Override public void run() throws Throwable {\n                assertThat(latencyFaultTolerance.isAvailable(brokerName)).isTrue();\n            }\n        });\n    }\n\n    @Test\n    public void testRemove() throws Exception {\n        latencyFaultTolerance.updateFaultItem(brokerName, 3000, 3000, true);\n        assertThat(latencyFaultTolerance.isAvailable(brokerName)).isFalse();\n        latencyFaultTolerance.remove(brokerName);\n        assertThat(latencyFaultTolerance.isAvailable(brokerName)).isTrue();\n    }\n\n    @Test\n    public void testPickOneAtLeast() throws Exception {\n        latencyFaultTolerance.updateFaultItem(brokerName, 1000, 3000, true);\n        assertThat(latencyFaultTolerance.pickOneAtLeast()).isEqualTo(brokerName);\n\n        // Bad case, since pickOneAtLeast's behavior becomes random\n        // latencyFaultTolerance.updateFaultItem(anotherBrokerName, 1001, 3000, \"127.0.0.1:12011\", true);\n        // assertThat(latencyFaultTolerance.pickOneAtLeast()).isEqualTo(brokerName);\n    }\n\n    @Test\n    public void testIsReachable() throws Exception {\n        latencyFaultTolerance.updateFaultItem(brokerName, 1000, 3000, true);\n        assertThat(latencyFaultTolerance.isReachable(brokerName)).isEqualTo(true);\n\n        latencyFaultTolerance.updateFaultItem(anotherBrokerName, 1001, 3000, false);\n        assertThat(latencyFaultTolerance.isReachable(anotherBrokerName)).isEqualTo(false);\n    }\n}"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.producer;\n\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.exception.RequestTimeoutException;\nimport org.apache.rocketmq.client.hook.SendMessageContext;\nimport org.apache.rocketmq.client.hook.SendMessageHook;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.MQClientManager;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;\nimport org.apache.rocketmq.client.impl.producer.TopicPublishInfo;\nimport org.apache.rocketmq.client.latency.MQFaultStrategy;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.compression.CompressionType;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingClient;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.lang.reflect.Field;\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.concurrent.atomic.AtomicInteger;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertNotEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.nullable;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class DefaultMQProducerTest {\n    @Spy\n    private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig());\n    @Mock\n    private MQClientAPIImpl mQClientAPIImpl;\n\n    private DefaultMQProducer producer;\n    private Message message;\n    private Message zeroMsg;\n    private Message bigMessage;\n    private final String topic = \"FooBar\";\n    private final String producerGroupPrefix = \"FooBar_PID\";\n    private final long defaultTimeout = 3000L;\n\n    @Before\n    public void init() throws Exception {\n        String producerGroupTemp = producerGroupPrefix + System.currentTimeMillis();\n        producer = new DefaultMQProducer(producerGroupTemp);\n        producer.setNamesrvAddr(\"127.0.0.1:9876\");\n        producer.setCompressMsgBodyOverHowmuch(16);\n        message = new Message(topic, new byte[] {'a'});\n        zeroMsg = new Message(topic, new byte[] {});\n        bigMessage = new Message(topic, \"This is a very huge message!\".getBytes());\n\n        producer.start();\n\n        Field field = DefaultMQProducerImpl.class.getDeclaredField(\"mQClientFactory\");\n        field.setAccessible(true);\n        field.set(producer.getDefaultMQProducerImpl(), mQClientFactory);\n\n        field = MQClientInstance.class.getDeclaredField(\"mQClientAPIImpl\");\n        field.setAccessible(true);\n        field.set(mQClientFactory, mQClientAPIImpl);\n\n        producer.getDefaultMQProducerImpl().getMqClientFactory().registerProducer(producerGroupTemp, producer.getDefaultMQProducerImpl());\n\n        when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class),\n            nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))).thenCallRealMethod();\n        when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class),\n            nullable(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class)))\n            .thenReturn(createSendResult(SendStatus.SEND_OK));\n    }\n\n    @After\n    public void terminate() {\n        producer.shutdown();\n    }\n\n    @Test\n    public void testSendMessage_ZeroMessage() throws InterruptedException, RemotingException, MQBrokerException {\n        try {\n            producer.send(zeroMsg);\n            failBecauseExceptionWasNotThrown(MQClientException.class);\n        } catch (MQClientException e) {\n            assertThat(e).hasMessageContaining(\"message body length is zero\");\n        }\n    }\n\n    @Test\n    public void testSendMessage_NoNameSrv() throws RemotingException, InterruptedException, MQBrokerException {\n        when(mQClientAPIImpl.getNameServerAddressList()).thenReturn(new ArrayList<>());\n        try {\n            producer.send(message);\n            failBecauseExceptionWasNotThrown(MQClientException.class);\n        } catch (MQClientException e) {\n            assertThat(e).hasMessageContaining(\"No name server address\");\n        }\n    }\n\n    @Test\n    public void testSendMessage_NoRoute() throws RemotingException, InterruptedException, MQBrokerException {\n        when(mQClientAPIImpl.getNameServerAddressList()).thenReturn(Collections.singletonList(\"127.0.0.1:9876\"));\n        try {\n            producer.send(message);\n            failBecauseExceptionWasNotThrown(MQClientException.class);\n        } catch (MQClientException e) {\n            assertThat(e).hasMessageContaining(\"No route info of this topic\");\n        }\n    }\n\n    @Test\n    public void testSendMessageSync_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {\n        when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());\n        SendResult sendResult = producer.send(message);\n\n        assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n        assertThat(sendResult.getOffsetMsgId()).isEqualTo(\"123\");\n        assertThat(sendResult.getQueueOffset()).isEqualTo(456L);\n    }\n\n    @Test\n    public void testSendMessageSync_WithBodyCompressed() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {\n        when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());\n        SendResult sendResult = producer.send(bigMessage);\n\n        assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n        assertThat(sendResult.getOffsetMsgId()).isEqualTo(\"123\");\n        assertThat(sendResult.getQueueOffset()).isEqualTo(456L);\n    }\n\n    @Test\n    public void testSendMessageAsync_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {\n        final CountDownLatch countDownLatch = new CountDownLatch(1);\n        when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());\n        producer.send(message, new SendCallback() {\n            @Override\n            public void onSuccess(SendResult sendResult) {\n                assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n                assertThat(sendResult.getOffsetMsgId()).isEqualTo(\"123\");\n                assertThat(sendResult.getQueueOffset()).isEqualTo(456L);\n                countDownLatch.countDown();\n            }\n\n            @Override\n            public void onException(Throwable e) {\n                countDownLatch.countDown();\n            }\n        });\n        countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS);\n    }\n\n    @Test\n    public void testSendMessageAsync() throws RemotingException, MQClientException, InterruptedException {\n        final AtomicInteger cc = new AtomicInteger(0);\n        final CountDownLatch countDownLatch = new CountDownLatch(12);\n\n        when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());\n        SendCallback sendCallback = new SendCallback() {\n            @Override\n            public void onSuccess(SendResult sendResult) {\n                countDownLatch.countDown();\n            }\n\n            @Override\n            public void onException(Throwable e) {\n                e.printStackTrace();\n                cc.incrementAndGet();\n                countDownLatch.countDown();\n            }\n        };\n        MessageQueueSelector messageQueueSelector = new MessageQueueSelector() {\n            @Override\n            public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {\n                return null;\n            }\n        };\n\n        // on enableBackpressureForAsyncMode\n        producer.setEnableBackpressureForAsyncMode(true);\n        producer.setBackPressureForAsyncSendNum(5000);\n        producer.setBackPressureForAsyncSendSize(50 * 1024 * 1024);\n        Message message = new Message();\n        message.setTopic(\"test\");\n        message.setBody(\"hello world\".getBytes());\n        producer.send(new Message(), sendCallback);\n        producer.send(message, new MessageQueue(), sendCallback);\n        producer.send(new Message(), new MessageQueue(), sendCallback, 1000);\n        producer.send(new Message(), messageQueueSelector, null, sendCallback);\n        producer.send(message, messageQueueSelector, null, sendCallback, 1000);\n        //this message is send success\n        producer.send(message, sendCallback, 1000);\n\n        countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS);\n        assertThat(cc.get()).isEqualTo(5);\n\n        // off enableBackpressureForAsyncMode\n        producer.setEnableBackpressureForAsyncMode(false);\n        producer.send(new Message(), sendCallback);\n        producer.send(message, new MessageQueue(), sendCallback);\n        producer.send(new Message(), new MessageQueue(), sendCallback, 1000);\n        producer.send(new Message(), messageQueueSelector, null, sendCallback);\n        producer.send(message, messageQueueSelector, null, sendCallback, 1000);\n        //this message is send success\n        producer.send(message, sendCallback, 1000);\n\n        countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS);\n        assertThat(cc.get()).isEqualTo(10);\n    }\n\n    @Test\n    public void testBatchSendMessageAsync()\n        throws RemotingException, MQClientException, InterruptedException, MQBrokerException {\n        final AtomicInteger cc = new AtomicInteger(0);\n        final CountDownLatch countDownLatch = new CountDownLatch(4);\n\n        when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());\n        SendCallback sendCallback = new SendCallback() {\n            @Override\n            public void onSuccess(SendResult sendResult) {\n                countDownLatch.countDown();\n            }\n\n            @Override\n            public void onException(Throwable e) {\n                e.printStackTrace();\n                cc.incrementAndGet();\n                countDownLatch.countDown();\n            }\n        };\n        MessageQueueSelector messageQueueSelector = new MessageQueueSelector() {\n            @Override\n            public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {\n                return null;\n            }\n        };\n\n        List<Message> msgs = new ArrayList<>();\n        for (int i = 0; i < 5; i++) {\n            Message message = new Message();\n            message.setTopic(\"test\");\n            message.setBody((\"hello world\" + i).getBytes());\n            msgs.add(message);\n        }\n\n        // on enableBackpressureForAsyncMode\n        producer.setEnableBackpressureForAsyncMode(true);\n        producer.send(msgs, sendCallback);\n        producer.send(msgs, sendCallback, 1000);\n        MessageQueue mq = new MessageQueue(\"test\", \"BrokerA\", 1);\n        producer.send(msgs, mq, sendCallback);\n        // this message is send failed\n        producer.send(msgs, new MessageQueue(), sendCallback, 1000);\n\n        countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS);\n        assertThat(cc.get()).isEqualTo(1);\n\n        // off enableBackpressureForAsyncMode\n        producer.setEnableBackpressureForAsyncMode(false);\n        producer.send(msgs, sendCallback);\n        producer.send(msgs, sendCallback, 1000);\n        producer.send(msgs, mq, sendCallback);\n        // this message is send failed\n        producer.send(msgs, new MessageQueue(), sendCallback, 1000);\n\n        countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS);\n        assertThat(cc.get()).isEqualTo(2);\n    }\n\n    @Test\n    public void testSendMessageAsync_BodyCompressed() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {\n        final CountDownLatch countDownLatch = new CountDownLatch(1);\n        when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());\n        producer.send(bigMessage, new SendCallback() {\n            @Override\n            public void onSuccess(SendResult sendResult) {\n                assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n                assertThat(sendResult.getOffsetMsgId()).isEqualTo(\"123\");\n                assertThat(sendResult.getQueueOffset()).isEqualTo(456L);\n                countDownLatch.countDown();\n            }\n\n            @Override\n            public void onException(Throwable e) {\n            }\n        });\n        countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS);\n    }\n\n    @Test\n    public void testSendMessageSync_SuccessWithHook() throws Throwable {\n        when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());\n        final Throwable[] assertionErrors = new Throwable[1];\n        final CountDownLatch countDownLatch = new CountDownLatch(2);\n        producer.getDefaultMQProducerImpl().registerSendMessageHook(new SendMessageHook() {\n            @Override\n            public String hookName() {\n                return \"TestHook\";\n            }\n\n            @Override\n            public void sendMessageBefore(final SendMessageContext context) {\n                assertionErrors[0] = assertInOtherThread(new Runnable() {\n                    @Override\n                    public void run() {\n                        assertThat(context.getMessage()).isEqualTo(message);\n                        assertThat(context.getProducer()).isEqualTo(producer);\n                        assertThat(context.getCommunicationMode()).isEqualTo(CommunicationMode.SYNC);\n                        assertThat(context.getSendResult()).isNull();\n                    }\n                });\n                countDownLatch.countDown();\n            }\n\n            @Override\n            public void sendMessageAfter(final SendMessageContext context) {\n                assertionErrors[0] = assertInOtherThread(new Runnable() {\n                    @Override\n                    public void run() {\n                        assertThat(context.getMessage()).isEqualTo(message);\n                        assertThat(context.getProducer()).isEqualTo(producer.getDefaultMQProducerImpl());\n                        assertThat(context.getCommunicationMode()).isEqualTo(CommunicationMode.SYNC);\n                        assertThat(context.getSendResult()).isNotNull();\n                    }\n                });\n                countDownLatch.countDown();\n            }\n        });\n        SendResult sendResult = producer.send(message);\n\n        assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n        assertThat(sendResult.getOffsetMsgId()).isEqualTo(\"123\");\n        assertThat(sendResult.getQueueOffset()).isEqualTo(456L);\n\n        countDownLatch.await();\n\n        if (assertionErrors[0] != null) {\n            throw assertionErrors[0];\n        }\n    }\n\n    @Test\n    public void testSetCallbackExecutor() throws MQClientException {\n        String producerGroupTemp = \"testSetCallbackExecutor_\" + System.currentTimeMillis();\n        producer = new DefaultMQProducer(producerGroupTemp);\n        producer.setNamesrvAddr(\"127.0.0.1:9876\");\n        producer.start();\n\n        ExecutorService customized = Executors.newCachedThreadPool();\n        producer.setCallbackExecutor(customized);\n\n        NettyRemotingClient remotingClient = (NettyRemotingClient) producer.getDefaultMQProducerImpl()\n            .getMqClientFactory().getMQClientAPIImpl().getRemotingClient();\n\n        assertThat(remotingClient.getCallbackExecutor()).isEqualTo(customized);\n    }\n\n    @Test\n    public void testRequestMessage() throws RemotingException, RequestTimeoutException, MQClientException, InterruptedException, MQBrokerException {\n        when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());\n        final AtomicBoolean finish = new AtomicBoolean(false);\n        new Thread(new Runnable() {\n            @Override\n            public void run() {\n                ConcurrentHashMap<String, RequestResponseFuture> responseMap = RequestFutureHolder.getInstance().getRequestFutureTable();\n                assertThat(responseMap).isNotNull();\n                while (!finish.get()) {\n                    try {\n                        Thread.sleep(10);\n                    } catch (InterruptedException e) {\n                    }\n                    MessageExt responseMsg = new MessageExt();\n                    responseMsg.setTopic(message.getTopic());\n                    responseMsg.setBody(message.getBody());\n                    for (Map.Entry<String, RequestResponseFuture> entry : responseMap.entrySet()) {\n                        RequestResponseFuture future = entry.getValue();\n                        future.putResponseMessage(responseMsg);\n                    }\n                }\n            }\n        }).start();\n        Message result = producer.request(message, 3 * 1000L);\n        finish.getAndSet(true);\n        assertThat(result).isExactlyInstanceOf(MessageExt.class);\n        assertThat(result.getTopic()).isEqualTo(\"FooBar\");\n        assertThat(result.getBody()).isEqualTo(new byte[] {'a'});\n    }\n\n    @Test(expected = RequestTimeoutException.class)\n    public void testRequestMessage_RequestTimeoutException() throws RemotingException, RequestTimeoutException, MQClientException, InterruptedException, MQBrokerException {\n        when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());\n        Message result = producer.request(message, 3 * 1000L);\n    }\n\n    @Test\n    public void testAsyncRequest_OnSuccess() throws Exception {\n        when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());\n        final CountDownLatch countDownLatch = new CountDownLatch(1);\n        RequestCallback requestCallback = new RequestCallback() {\n            @Override\n            public void onSuccess(Message message) {\n                assertThat(message).isExactlyInstanceOf(MessageExt.class);\n                assertThat(message.getTopic()).isEqualTo(\"FooBar\");\n                assertThat(message.getBody()).isEqualTo(new byte[] {'a'});\n                assertThat(message.getFlag()).isEqualTo(1);\n                countDownLatch.countDown();\n            }\n\n            @Override\n            public void onException(Throwable e) {\n            }\n        };\n        producer.request(message, requestCallback, 3 * 1000L);\n        ConcurrentHashMap<String, RequestResponseFuture> responseMap = RequestFutureHolder.getInstance().getRequestFutureTable();\n        assertThat(responseMap).isNotNull();\n\n        MessageExt responseMsg = new MessageExt();\n        responseMsg.setTopic(message.getTopic());\n        responseMsg.setBody(message.getBody());\n        responseMsg.setFlag(1);\n        for (Map.Entry<String, RequestResponseFuture> entry : responseMap.entrySet()) {\n            RequestResponseFuture future = entry.getValue();\n            future.setSendRequestOk(true);\n            future.getRequestCallback().onSuccess(responseMsg);\n        }\n        countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS);\n    }\n\n    @Test\n    public void testAsyncRequest_OnException() throws Exception {\n        final AtomicInteger cc = new AtomicInteger(0);\n        final CountDownLatch countDownLatch = new CountDownLatch(1);\n        RequestCallback requestCallback = new RequestCallback() {\n            @Override\n            public void onSuccess(Message message) {\n\n            }\n\n            @Override\n            public void onException(Throwable e) {\n                cc.incrementAndGet();\n                countDownLatch.countDown();\n            }\n        };\n        MessageQueueSelector messageQueueSelector = new MessageQueueSelector() {\n            @Override\n            public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {\n                return null;\n            }\n        };\n\n        try {\n            producer.request(message, requestCallback, 3 * 1000L);\n            failBecauseExceptionWasNotThrown(Exception.class);\n        } catch (Exception e) {\n            ConcurrentHashMap<String, RequestResponseFuture> responseMap = RequestFutureHolder.getInstance().getRequestFutureTable();\n            assertThat(responseMap).isNotNull();\n            for (Map.Entry<String, RequestResponseFuture> entry : responseMap.entrySet()) {\n                RequestResponseFuture future = entry.getValue();\n                future.getRequestCallback().onException(e);\n            }\n        }\n        countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS);\n        assertThat(cc.get()).isEqualTo(1);\n    }\n\n    @Test\n    public void testBatchSendMessageAsync_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {\n        final CountDownLatch countDownLatch = new CountDownLatch(1);\n        when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());\n        producer.setAutoBatch(true);\n        producer.send(message, new SendCallback() {\n            @Override\n            public void onSuccess(SendResult sendResult) {\n                assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n                assertThat(sendResult.getOffsetMsgId()).isEqualTo(\"123\");\n                assertThat(sendResult.getQueueOffset()).isEqualTo(456L);\n                countDownLatch.countDown();\n            }\n\n            @Override\n            public void onException(Throwable e) {\n                countDownLatch.countDown();\n            }\n        });\n\n        countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS);\n        producer.setAutoBatch(false);\n    }\n\n    @Test\n    public void testBatchSendMessageSync_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {\n        producer.setAutoBatch(true);\n        when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());\n        SendResult sendResult = producer.send(message);\n\n        assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n        assertThat(sendResult.getOffsetMsgId()).isEqualTo(\"123\");\n        assertThat(sendResult.getQueueOffset()).isEqualTo(456L);\n        producer.setAutoBatch(false);\n    }\n\n\n    @Test\n    public void testRunningSetBackCompress() throws RemotingException, InterruptedException, MQClientException {\n        final CountDownLatch countDownLatch = new CountDownLatch(5);\n        SendCallback sendCallback = new SendCallback() {\n            @Override\n            public void onSuccess(SendResult sendResult) {\n                countDownLatch.countDown();\n            }\n\n            @Override\n            public void onException(Throwable e) {\n                e.printStackTrace();\n                countDownLatch.countDown();\n            }\n        };\n\n        // on enableBackpressureForAsyncMode\n        producer.setEnableBackpressureForAsyncMode(true);\n        producer.setBackPressureForAsyncSendNum(10);\n        producer.setBackPressureForAsyncSendSize(50 * 1024 * 1024);\n        Message message = new Message();\n        message.setTopic(\"test\");\n        message.setBody(\"hello world\".getBytes());\n        MessageQueue mq = new MessageQueue(\"test\", \"BrokerA\", 1);\n        //this message is send success\n        for (int i = 0; i < 5; i++) {\n            new Thread(new Runnable() {\n                @Override\n                public void run() {\n                    try {\n                        producer.send(message, mq, sendCallback);\n                    } catch (MQClientException | RemotingException | InterruptedException e) {\n                        throw new RuntimeException(e);\n                    }\n                }\n            }).start();\n        }\n        producer.setBackPressureForAsyncSendNum(15);\n        countDownLatch.await(3000L, TimeUnit.MILLISECONDS);\n        assertThat(producer.defaultMQProducerImpl.getSemaphoreAsyncSendNumAvailablePermits() + countDownLatch.getCount()).isEqualTo(15);\n        producer.setEnableBackpressureForAsyncMode(false);\n    }\n\n    public static TopicRouteData createTopicRoute() {\n        TopicRouteData topicRouteData = new TopicRouteData();\n\n        topicRouteData.setFilterServerTable(new HashMap<>());\n        List<BrokerData> brokerDataList = new ArrayList<>();\n        BrokerData brokerData = new BrokerData();\n        brokerData.setBrokerName(\"BrokerA\");\n        brokerData.setCluster(\"DefaultCluster\");\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(0L, \"127.0.0.1:10911\");\n        brokerData.setBrokerAddrs(brokerAddrs);\n        brokerDataList.add(brokerData);\n        topicRouteData.setBrokerDatas(brokerDataList);\n\n        List<QueueData> queueDataList = new ArrayList<>();\n        QueueData queueData = new QueueData();\n        queueData.setBrokerName(\"BrokerA\");\n        queueData.setPerm(6);\n        queueData.setReadQueueNums(3);\n        queueData.setWriteQueueNums(4);\n        queueData.setTopicSysFlag(0);\n        queueDataList.add(queueData);\n        topicRouteData.setQueueDatas(queueDataList);\n        return topicRouteData;\n    }\n\n    private SendResult createSendResult(SendStatus sendStatus) {\n        SendResult sendResult = new SendResult();\n        sendResult.setMsgId(\"123\");\n        sendResult.setOffsetMsgId(\"123\");\n        sendResult.setQueueOffset(456);\n        sendResult.setSendStatus(sendStatus);\n        sendResult.setRegionId(\"HZ\");\n        return sendResult;\n    }\n\n    private Throwable assertInOtherThread(final Runnable runnable) {\n        final Throwable[] assertionErrors = new Throwable[1];\n        Thread thread = new Thread(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    runnable.run();\n                } catch (AssertionError e) {\n                    assertionErrors[0] = e;\n                }\n            }\n        });\n        thread.start();\n        try {\n            thread.join();\n        } catch (InterruptedException e) {\n            assertionErrors[0] = e;\n        }\n        return assertionErrors[0];\n    }\n\n    @Test\n    public void assertCreateDefaultMQProducer() {\n        String producerGroupTemp = producerGroupPrefix + System.currentTimeMillis();\n        DefaultMQProducer producer1 = new DefaultMQProducer(producerGroupTemp);\n        assertNotNull(producer1);\n        assertEquals(producerGroupTemp, producer1.getProducerGroup());\n        assertNotNull(producer1.getDefaultMQProducerImpl());\n        assertEquals(0, producer1.getTotalBatchMaxBytes());\n        assertEquals(0, producer1.getBatchMaxBytes());\n        assertEquals(0, producer1.getBatchMaxDelayMs());\n        assertNull(producer1.getTopics());\n        assertFalse(producer1.isEnableTrace());\n        assertTrue(UtilAll.isBlank(producer1.getTraceTopic()));\n        DefaultMQProducer producer2 = new DefaultMQProducer(producerGroupTemp, mock(RPCHook.class));\n        assertNotNull(producer2);\n        assertEquals(producerGroupTemp, producer2.getProducerGroup());\n        assertNotNull(producer2.getDefaultMQProducerImpl());\n        assertEquals(0, producer2.getTotalBatchMaxBytes());\n        assertEquals(0, producer2.getBatchMaxBytes());\n        assertEquals(0, producer2.getBatchMaxDelayMs());\n        assertNull(producer2.getTopics());\n        assertFalse(producer2.isEnableTrace());\n        assertTrue(UtilAll.isBlank(producer2.getTraceTopic()));\n        DefaultMQProducer producer3 = new DefaultMQProducer(producerGroupTemp, mock(RPCHook.class), Collections.singletonList(\"custom_topic\"));\n        assertNotNull(producer3);\n        assertEquals(producerGroupTemp, producer3.getProducerGroup());\n        assertNotNull(producer3.getDefaultMQProducerImpl());\n        assertEquals(0, producer3.getTotalBatchMaxBytes());\n        assertEquals(0, producer3.getBatchMaxBytes());\n        assertEquals(0, producer3.getBatchMaxDelayMs());\n        assertNotNull(producer3.getTopics());\n        assertEquals(1, producer3.getTopics().size());\n        assertFalse(producer3.isEnableTrace());\n        assertTrue(UtilAll.isBlank(producer3.getTraceTopic()));\n        DefaultMQProducer producer4 = new DefaultMQProducer(producerGroupTemp, mock(RPCHook.class), true, \"custom_trace_topic\");\n        assertNotNull(producer4);\n        assertEquals(producerGroupTemp, producer4.getProducerGroup());\n        assertNotNull(producer4.getDefaultMQProducerImpl());\n        assertEquals(0, producer4.getTotalBatchMaxBytes());\n        assertEquals(0, producer4.getBatchMaxBytes());\n        assertEquals(0, producer4.getBatchMaxDelayMs());\n        assertNull(producer4.getTopics());\n        assertTrue(producer4.isEnableTrace());\n        assertEquals(\"custom_trace_topic\", producer4.getTraceTopic());\n        DefaultMQProducer producer5 = new DefaultMQProducer(producerGroupTemp, mock(RPCHook.class), Collections.singletonList(\"custom_topic\"), true, \"custom_trace_topic\");\n        assertNotNull(producer5);\n        assertEquals(producerGroupTemp, producer5.getProducerGroup());\n        assertNotNull(producer5.getDefaultMQProducerImpl());\n        assertEquals(0, producer5.getTotalBatchMaxBytes());\n        assertEquals(0, producer5.getBatchMaxBytes());\n        assertEquals(0, producer5.getBatchMaxDelayMs());\n        assertNotNull(producer5.getTopics());\n        assertEquals(1, producer5.getTopics().size());\n        assertTrue(producer5.isEnableTrace());\n        assertEquals(\"custom_trace_topic\", producer5.getTraceTopic());\n    }\n\n    @Test\n    public void assertSend() throws MQBrokerException, RemotingException, InterruptedException, MQClientException, NoSuchFieldException, IllegalAccessException {\n        setDefaultMQProducerImpl();\n        setOtherParam();\n        SendResult send = producer.send(message, defaultTimeout);\n        assertNull(send);\n        Collection<Message> msgs = Collections.singletonList(message);\n        send = producer.send(msgs);\n        assertNull(send);\n        send = producer.send(msgs, defaultTimeout);\n        assertNull(send);\n    }\n\n    @Test\n    public void assertSendOneway() throws RemotingException, InterruptedException, MQClientException, NoSuchFieldException, IllegalAccessException {\n        setDefaultMQProducerImpl();\n        producer.sendOneway(message);\n        MessageQueue mq = mock(MessageQueue.class);\n        producer.sendOneway(message, mq);\n        MessageQueueSelector selector = mock(MessageQueueSelector.class);\n        producer.sendOneway(message, selector, 1);\n    }\n\n    @Test\n    public void assertSendByQueue() throws MQBrokerException, RemotingException, InterruptedException, MQClientException, NoSuchFieldException, IllegalAccessException {\n        setDefaultMQProducerImpl();\n        MessageQueue mq = mock(MessageQueue.class);\n        SendResult send = producer.send(message, mq);\n        assertNull(send);\n        send = producer.send(message, mq, defaultTimeout);\n        assertNull(send);\n        Collection<Message> msgs = Collections.singletonList(message);\n        send = producer.send(msgs, mq);\n        assertNull(send);\n        send = producer.send(msgs, mq, defaultTimeout);\n        assertNull(send);\n    }\n\n    @Test\n    public void assertSendByQueueSelector() throws MQBrokerException, RemotingException, InterruptedException, MQClientException, NoSuchFieldException, IllegalAccessException {\n        setDefaultMQProducerImpl();\n        MessageQueueSelector selector = mock(MessageQueueSelector.class);\n        SendResult send = producer.send(message, selector, 1);\n        assertNull(send);\n        send = producer.send(message, selector, 1, defaultTimeout);\n        assertNull(send);\n    }\n\n    @Test\n    public void assertRequest() throws MQBrokerException, RemotingException, InterruptedException, MQClientException, NoSuchFieldException, IllegalAccessException, RequestTimeoutException {\n        setDefaultMQProducerImpl();\n        MessageQueueSelector selector = mock(MessageQueueSelector.class);\n        Message replyNsg = producer.request(message, selector, 1, defaultTimeout);\n        assertNull(replyNsg);\n        RequestCallback requestCallback = mock(RequestCallback.class);\n        producer.request(message, selector, 1, requestCallback, defaultTimeout);\n        MessageQueue mq = mock(MessageQueue.class);\n        producer.request(message, mq, defaultTimeout);\n        producer.request(message, mq, requestCallback, defaultTimeout);\n    }\n\n    @Test(expected = RuntimeException.class)\n    public void assertSendMessageInTransaction() throws MQClientException {\n        TransactionSendResult result = producer.sendMessageInTransaction(message, 1);\n        assertNull(result);\n    }\n\n    @Test\n    public void assertSearchOffset() throws MQClientException, NoSuchFieldException, IllegalAccessException {\n        setDefaultMQProducerImpl();\n        MessageQueue mq = mock(MessageQueue.class);\n        long result = producer.searchOffset(mq, System.currentTimeMillis());\n        assertEquals(0L, result);\n    }\n\n    @Test\n    public void assertBatchMaxDelayMs() throws NoSuchFieldException, IllegalAccessException {\n        setProduceAccumulator(true);\n        assertEquals(0, producer.getBatchMaxDelayMs());\n        setProduceAccumulator(false);\n        assertEquals(10, producer.getBatchMaxDelayMs());\n        producer.batchMaxDelayMs(1000);\n        assertEquals(1000, producer.getBatchMaxDelayMs());\n    }\n\n    @Test\n    public void assertBatchMaxBytes() throws NoSuchFieldException, IllegalAccessException {\n        setProduceAccumulator(true);\n        assertEquals(0L, producer.getBatchMaxBytes());\n        setProduceAccumulator(false);\n        assertEquals(32 * 1024L, producer.getBatchMaxBytes());\n        producer.batchMaxBytes(64 * 1024L);\n        assertEquals(64 * 1024L, producer.getBatchMaxBytes());\n    }\n\n    @Test\n    public void assertTotalBatchMaxBytes() throws NoSuchFieldException, IllegalAccessException {\n        setProduceAccumulator(true);\n        assertEquals(0L, producer.getTotalBatchMaxBytes());\n    }\n\n    @Test\n    public void assertProduceAccumulatorStart() throws NoSuchFieldException, IllegalAccessException, MQClientException {\n        String producerGroupTemp = producerGroupPrefix + System.nanoTime();\n        DefaultMQProducer producer = new DefaultMQProducer(producerGroupTemp);\n        assertEquals(0, producer.getTotalBatchMaxBytes());\n        assertEquals(0, producer.getBatchMaxBytes());\n        assertEquals(0, producer.getBatchMaxDelayMs());\n        assertNull(getField(producer, \"produceAccumulator\", ProduceAccumulator.class));\n        producer.start();\n        assertTrue(producer.getTotalBatchMaxBytes() > 0);\n        assertTrue(producer.getBatchMaxBytes() > 0);\n        assertTrue(producer.getBatchMaxDelayMs() > 0);\n        assertNotNull(getField(producer, \"produceAccumulator\", ProduceAccumulator.class));\n    }\n\n    @Test\n    public void assertProduceAccumulatorBeforeStartSet() throws NoSuchFieldException, IllegalAccessException, MQClientException {\n        String producerGroupTemp = producerGroupPrefix + System.nanoTime();\n        DefaultMQProducer producer = new DefaultMQProducer(producerGroupTemp);\n        producer.totalBatchMaxBytes(64 * 1024 * 100);\n        producer.batchMaxBytes(64 * 1024);\n        producer.batchMaxDelayMs(10);\n\n        producer.start();\n        assertEquals(64 * 1024, producer.getBatchMaxBytes());\n        assertEquals(10, producer.getBatchMaxDelayMs());\n        assertNotNull(getField(producer, \"produceAccumulator\", ProduceAccumulator.class));\n    }\n\n    @Test\n    public void assertProduceAccumulatorAfterStartSet() throws NoSuchFieldException, IllegalAccessException, MQClientException {\n        String producerGroupTemp = producerGroupPrefix + System.nanoTime();\n        DefaultMQProducer producer = new DefaultMQProducer(producerGroupTemp);\n        producer.start();\n\n        assertNotNull(getField(producer, \"produceAccumulator\", ProduceAccumulator.class));\n\n        producer.totalBatchMaxBytes(64 * 1024 * 100);\n        producer.batchMaxBytes(64 * 1024);\n        producer.batchMaxDelayMs(10);\n\n        assertEquals(64 * 1024, producer.getBatchMaxBytes());\n        assertEquals(10, producer.getBatchMaxDelayMs());\n    }\n\n    @Test\n    public void assertProduceAccumulatorUnit() throws NoSuchFieldException, IllegalAccessException, MQClientException {\n        String producerGroupTemp = producerGroupPrefix + System.nanoTime();\n        DefaultMQProducer producer1 = new DefaultMQProducer(producerGroupTemp);\n        producer1.setUnitName(\"unit1\");\n        DefaultMQProducer producer2 = new DefaultMQProducer(producerGroupTemp);\n        producer2.setUnitName(\"unit2\");\n\n        producer1.start();\n        producer2.start();\n\n        ProduceAccumulator producer1Accumulator = getField(producer1, \"produceAccumulator\", ProduceAccumulator.class);\n        ProduceAccumulator producer2Accumulator = getField(producer2, \"produceAccumulator\", ProduceAccumulator.class);\n\n        assertNotNull(producer1Accumulator);\n        assertNotNull(producer2Accumulator);\n\n        assertNotEquals(producer1Accumulator, producer2Accumulator);\n    }\n\n    @Test\n    public void assertProduceAccumulator() throws NoSuchFieldException, IllegalAccessException, MQClientException {\n        String producerGroupTemp1 = producerGroupPrefix + System.nanoTime();\n        DefaultMQProducer producer1 = new DefaultMQProducer(producerGroupTemp1);\n        producer1.setInstanceName(\"instanceName1\");\n        String producerGroupTemp2 = producerGroupPrefix + System.nanoTime();\n        DefaultMQProducer producer2 = new DefaultMQProducer(producerGroupTemp2);\n        producer2.setInstanceName(\"instanceName2\");\n\n        producer1.start();\n        producer2.start();\n\n        ProduceAccumulator producer1Accumulator = getField(producer1, \"produceAccumulator\", ProduceAccumulator.class);\n        ProduceAccumulator producer2Accumulator = getField(producer2, \"produceAccumulator\", ProduceAccumulator.class);\n\n        assertNotNull(producer1Accumulator);\n        assertNotNull(producer2Accumulator);\n\n        assertNotEquals(producer1Accumulator, producer2Accumulator);\n    }\n\n    @Test\n    public void assertProduceAccumulatorInstanceEqual() throws NoSuchFieldException, IllegalAccessException, MQClientException {\n        String producerGroupTemp1 = producerGroupPrefix + System.nanoTime();\n        DefaultMQProducer producer1 = new DefaultMQProducer(producerGroupTemp1);\n        producer1.setInstanceName(\"equalInstance\");\n        String producerGroupTemp2 = producerGroupPrefix + System.nanoTime();\n        DefaultMQProducer producer2 = new DefaultMQProducer(producerGroupTemp2);\n        producer2.setInstanceName(\"equalInstance\");\n\n        producer1.start();\n        producer2.start();\n\n        ProduceAccumulator producer1Accumulator = getField(producer1, \"produceAccumulator\", ProduceAccumulator.class);\n        ProduceAccumulator producer2Accumulator = getField(producer2, \"produceAccumulator\", ProduceAccumulator.class);\n\n        assertNotNull(producer1Accumulator);\n        assertNotNull(producer2Accumulator);\n\n        assertEquals(producer1Accumulator, producer2Accumulator);\n    }\n\n    @Test\n    public void assertProduceAccumulatorInstanceAndUnitNameEqual() throws NoSuchFieldException, IllegalAccessException, MQClientException {\n        String producerGroupTemp1 = producerGroupPrefix + System.nanoTime();\n        DefaultMQProducer producer1 = new DefaultMQProducer(producerGroupTemp1);\n        producer1.setInstanceName(\"equalInstance\");\n        producer1.setUnitName(\"equalUnitName\");\n        String producerGroupTemp2 = producerGroupPrefix + System.nanoTime();\n        DefaultMQProducer producer2 = new DefaultMQProducer(producerGroupTemp2);\n        producer2.setInstanceName(\"equalInstance\");\n        producer2.setUnitName(\"equalUnitName\");\n\n        producer1.start();\n        producer2.start();\n\n        ProduceAccumulator producer1Accumulator = getField(producer1, \"produceAccumulator\", ProduceAccumulator.class);\n        ProduceAccumulator producer2Accumulator = getField(producer2, \"produceAccumulator\", ProduceAccumulator.class);\n\n        assertNotNull(producer1Accumulator);\n        assertNotNull(producer2Accumulator);\n\n        assertEquals(producer1Accumulator, producer2Accumulator);\n    }\n\n    @Test\n    public void assertGetRetryResponseCodes() {\n        assertNotNull(producer.getRetryResponseCodes());\n        assertEquals(8, producer.getRetryResponseCodes().size());\n    }\n\n    @Test\n    public void assertIsSendLatencyFaultEnable() {\n        assertFalse(producer.isSendLatencyFaultEnable());\n    }\n\n    @Test\n    public void assertGetLatencyMax() {\n        assertNotNull(producer.getLatencyMax());\n    }\n\n    @Test\n    public void assertGetNotAvailableDuration() {\n        assertNotNull(producer.getNotAvailableDuration());\n    }\n\n    @Test\n    public void assertIsRetryAnotherBrokerWhenNotStoreOK() {\n        assertFalse(producer.isRetryAnotherBrokerWhenNotStoreOK());\n    }\n\n    private void setOtherParam() {\n        producer.setCreateTopicKey(\"createTopicKey\");\n        producer.setRetryAnotherBrokerWhenNotStoreOK(false);\n        producer.setDefaultTopicQueueNums(6);\n        producer.setRetryTimesWhenSendFailed(1);\n        producer.setSendMessageWithVIPChannel(false);\n        producer.setNotAvailableDuration(new long[1]);\n        producer.setLatencyMax(new long[1]);\n        producer.setSendLatencyFaultEnable(false);\n        producer.setRetryTimesWhenSendAsyncFailed(1);\n        producer.setTopics(Collections.singletonList(topic));\n        producer.setStartDetectorEnable(false);\n        producer.setCompressLevel(5);\n        producer.setCompressType(CompressionType.LZ4);\n        producer.addRetryResponseCode(0);\n        ExecutorService executorService = mock(ExecutorService.class);\n        producer.setAsyncSenderExecutor(executorService);\n    }\n\n    private void setProduceAccumulator(final boolean isDefault) throws NoSuchFieldException, IllegalAccessException {\n        ProduceAccumulator accumulator = null;\n        if (!isDefault) {\n            accumulator = new ProduceAccumulator(\"instanceName\");\n        }\n        setField(producer, \"produceAccumulator\", accumulator);\n    }\n\n    private void setDefaultMQProducerImpl() throws NoSuchFieldException, IllegalAccessException {\n        DefaultMQProducerImpl producerImpl = mock(DefaultMQProducerImpl.class);\n        setField(producer, \"defaultMQProducerImpl\", producerImpl);\n        when(producerImpl.getMqFaultStrategy()).thenReturn(mock(MQFaultStrategy.class));\n    }\n\n    private void setField(final Object target, final String fieldName, final Object newValue) throws NoSuchFieldException, IllegalAccessException {\n        Class<?> clazz = target.getClass();\n        Field field = clazz.getDeclaredField(fieldName);\n        field.setAccessible(true);\n        field.set(target, newValue);\n    }\n\n    private <T> T getField(final Object target, final String fieldName, final Class<T> fieldClassType) throws NoSuchFieldException, IllegalAccessException {\n        Class<?> targetClazz = target.getClass();\n        Field field = targetClazz.getDeclaredField(fieldName);\n        field.setAccessible(true);\n        return fieldClassType.cast(field.get(target));\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/producer/ProduceAccumulatorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.producer;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageBatch;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class ProduceAccumulatorTest {\n    private boolean compareMessageBatch(MessageBatch a, MessageBatch b) {\n        if (!a.getTopic().equals(b.getTopic())) {\n            return false;\n        }\n        if (!Arrays.equals(a.getBody(), b.getBody())) {\n            return false;\n        }\n        return true;\n    }\n\n    private class MockMQProducer extends DefaultMQProducer {\n        private Message beSendMessage = null;\n        private MessageQueue beSendMessageQueue = null;\n\n        @Override\n        public SendResult sendDirect(Message msg, MessageQueue mq,\n            SendCallback sendCallback) {\n            this.beSendMessage = msg;\n            this.beSendMessageQueue = mq;\n\n            SendResult sendResult = new SendResult();\n            sendResult.setMsgId(\"123\");\n            if (sendCallback != null) {\n                sendCallback.onSuccess(sendResult);\n            }\n            return sendResult;\n        }\n    }\n\n    @Test\n    public void testProduceAccumulator_async() throws MQBrokerException, RemotingException, InterruptedException, MQClientException {\n        MockMQProducer mockMQProducer = new MockMQProducer();\n\n        ProduceAccumulator produceAccumulator = new ProduceAccumulator(\"test\");\n        produceAccumulator.start();\n\n        final CountDownLatch countDownLatch = new CountDownLatch(1);\n        List<Message> messages = new ArrayList<Message>();\n        messages.add(new Message(\"testTopic\", \"1\".getBytes()));\n        messages.add(new Message(\"testTopic\", \"22\".getBytes()));\n        messages.add(new Message(\"testTopic\", \"333\".getBytes()));\n        messages.add(new Message(\"testTopic\", \"4444\".getBytes()));\n        messages.add(new Message(\"testTopic\", \"55555\".getBytes()));\n        for (Message message : messages) {\n            produceAccumulator.send(message, new SendCallback() {\n                final CountDownLatch finalCountDownLatch = countDownLatch;\n\n                @Override\n                public void onSuccess(SendResult sendResult) {\n                    finalCountDownLatch.countDown();\n                }\n\n                @Override\n                public void onException(Throwable e) {\n                    finalCountDownLatch.countDown();\n                }\n            }, mockMQProducer);\n        }\n        assertThat(countDownLatch.await(3000L, TimeUnit.MILLISECONDS)).isTrue();\n        assertThat(mockMQProducer.beSendMessage instanceof MessageBatch).isTrue();\n\n        MessageBatch messageBatch1 = (MessageBatch) mockMQProducer.beSendMessage;\n        MessageBatch messageBatch2 = MessageBatch.generateFromList(messages);\n        messageBatch2.setBody(messageBatch2.encode());\n\n        assertThat(compareMessageBatch(messageBatch1, messageBatch2)).isTrue();\n    }\n\n    @Test\n    public void testProduceAccumulator_sync() throws MQBrokerException, RemotingException, InterruptedException, MQClientException {\n        final MockMQProducer mockMQProducer = new MockMQProducer();\n\n        final ProduceAccumulator produceAccumulator = new ProduceAccumulator(\"test\");\n        produceAccumulator.batchMaxDelayMs(3000);\n        produceAccumulator.start();\n\n        List<Message> messages = new ArrayList<Message>();\n        messages.add(new Message(\"testTopic\", \"1\".getBytes()));\n        messages.add(new Message(\"testTopic\", \"22\".getBytes()));\n        messages.add(new Message(\"testTopic\", \"333\".getBytes()));\n        messages.add(new Message(\"testTopic\", \"4444\".getBytes()));\n        messages.add(new Message(\"testTopic\", \"55555\".getBytes()));\n        final CountDownLatch countDownLatch = new CountDownLatch(messages.size());\n\n        for (final Message message : messages) {\n            new Thread(new Runnable() {\n                final ProduceAccumulator finalProduceAccumulator = produceAccumulator;\n                final CountDownLatch finalCountDownLatch = countDownLatch;\n                final MockMQProducer finalMockMQProducer = mockMQProducer;\n                final Message finalMessage = message;\n\n                @Override\n                public void run() {\n                    try {\n                        finalProduceAccumulator.send(finalMessage, finalMockMQProducer);\n                        finalCountDownLatch.countDown();\n                    } catch (Exception e) {\n                        e.printStackTrace();\n                    }\n                }\n            }).start();\n        }\n        assertThat(countDownLatch.await(5000L, TimeUnit.MILLISECONDS)).isTrue();\n        assertThat(mockMQProducer.beSendMessage instanceof MessageBatch).isTrue();\n\n        MessageBatch messageBatch1 = (MessageBatch) mockMQProducer.beSendMessage;\n        MessageBatch messageBatch2 = MessageBatch.generateFromList(messages);\n        messageBatch2.setBody(messageBatch2.encode());\n\n        assertThat(messageBatch1.getTopic()).isEqualTo(messageBatch2.getTopic());\n        // The execution order is uncertain, just compare the length\n        assertThat(messageBatch1.getBody().length).isEqualTo(messageBatch2.getBody().length);\n    }\n\n    @Test\n    public void testProduceAccumulator_sendWithMessageQueue() throws MQBrokerException, RemotingException, InterruptedException, MQClientException {\n        MockMQProducer mockMQProducer = new MockMQProducer();\n\n        MessageQueue messageQueue = new MessageQueue(\"topicTest\", \"brokerTest\", 0);\n        final ProduceAccumulator produceAccumulator = new ProduceAccumulator(\"test\");\n        produceAccumulator.start();\n\n        Message message = new Message(\"testTopic\", \"1\".getBytes());\n        produceAccumulator.send(message, messageQueue, mockMQProducer);\n        assertThat(mockMQProducer.beSendMessageQueue).isEqualTo(messageQueue);\n\n        final CountDownLatch countDownLatch = new CountDownLatch(1);\n        produceAccumulator.send(message, messageQueue, new SendCallback() {\n            @Override\n            public void onSuccess(SendResult sendResult) {\n                countDownLatch.countDown();\n            }\n\n            @Override\n            public void onException(Throwable e) {\n                countDownLatch.countDown();\n            }\n        }, mockMQProducer);\n        assertThat(countDownLatch.await(3000L, TimeUnit.MILLISECONDS)).isTrue();\n        assertThat(mockMQProducer.beSendMessageQueue).isEqualTo(messageQueue);\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/producer/RequestResponseFutureTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.producer;\n\nimport java.util.UUID;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.rocketmq.common.message.Message;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class RequestResponseFutureTest {\n\n    @Test\n    public void testExecuteRequestCallback() throws Exception {\n        final AtomicInteger cc = new AtomicInteger(0);\n        RequestResponseFuture future = new RequestResponseFuture(UUID.randomUUID().toString(), 3 * 1000L, new RequestCallback() {\n            @Override\n            public void onSuccess(Message message) {\n                cc.incrementAndGet();\n            }\n\n            @Override\n            public void onException(Throwable e) {\n            }\n        });\n        future.setSendRequestOk(true);\n        future.executeRequestCallback();\n        assertThat(cc.get()).isEqualTo(1);\n    }\n\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/producer/SendResultTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.producer;\n\nimport com.alibaba.fastjson2.JSON;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\n\npublic class SendResultTest {\n\n    @Test\n    public void testEncoderSendResultToJson() {\n        SendResult sendResult = new SendResult();\n        sendResult.setSendStatus(SendStatus.SEND_OK);\n        sendResult.setMsgId(\"12345\");\n        sendResult.setQueueOffset(100L);\n        MessageQueue messageQueue = new MessageQueue(\"TestTopic\", \"BrokerA\", 1);\n        sendResult.setMessageQueue(messageQueue);\n\n        String json = SendResult.encoderSendResultToJson(sendResult);\n\n        SendResult decodedResult = JSON.parseObject(json, SendResult.class);\n        assertEquals(sendResult.getSendStatus(), decodedResult.getSendStatus());\n        assertEquals(sendResult.getMsgId(), decodedResult.getMsgId());\n        assertEquals(sendResult.getQueueOffset(), decodedResult.getQueueOffset());\n        assertEquals(sendResult.getMessageQueue(), decodedResult.getMessageQueue());\n    }\n\n    @Test\n    public void testDecoderSendResultFromJson() {\n        String json = \"{\\\"sendStatus\\\":\\\"SEND_OK\\\",\\\"msgId\\\":\\\"12345\\\",\\\"queueOffset\\\":100,\\\"messageQueue\\\":{\\\"topic\\\":\\\"TestTopic\\\",\\\"brokerName\\\":\\\"BrokerA\\\",\\\"queueId\\\":1}}\";\n\n        SendResult sendResult = SendResult.decoderSendResultFromJson(json);\n\n        assertEquals(SendStatus.SEND_OK, sendResult.getSendStatus());\n        assertEquals(\"12345\", sendResult.getMsgId());\n        assertEquals(100L, sendResult.getQueueOffset());\n        assertEquals(\"TestTopic\", sendResult.getMessageQueue().getTopic());\n        assertEquals(\"BrokerA\", sendResult.getMessageQueue().getBrokerName());\n        assertEquals(1, sendResult.getMessageQueue().getQueueId());\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/producer/selector/DefaultMQProducerImplTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.producer.selector;\n\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.exception.RequestTimeoutException;\nimport org.apache.rocketmq.client.hook.CheckForbiddenContext;\nimport org.apache.rocketmq.client.hook.CheckForbiddenHook;\nimport org.apache.rocketmq.client.impl.MQAdminImpl;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;\nimport org.apache.rocketmq.client.impl.producer.TopicPublishInfo;\nimport org.apache.rocketmq.client.producer.MessageQueueSelector;\nimport org.apache.rocketmq.client.producer.RequestCallback;\nimport org.apache.rocketmq.client.producer.SendCallback;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.TransactionListener;\nimport org.apache.rocketmq.client.producer.TransactionMQProducer;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ServiceState;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.producer.RecallMessageHandle;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.lang.reflect.Field;\nimport java.lang.reflect.InvocationTargetException;\nimport java.lang.reflect.Method;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.ExecutorService;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertTrue;\nimport static org.junit.Assert.assertThrows;\nimport static org.mockito.AdditionalMatchers.or;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.isNull;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class DefaultMQProducerImplTest {\n\n    @Mock\n    private Message message;\n\n    @Mock\n    private MessageQueue messageQueue;\n\n    @Mock\n    private MessageQueueSelector queueSelector;\n\n    @Mock\n    private RequestCallback requestCallback;\n\n    @Mock\n    private MQClientInstance mQClientFactory;\n\n    @Mock\n    private MQClientAPIImpl mQClientAPIImpl;\n\n    private DefaultMQProducerImpl defaultMQProducerImpl;\n\n    private final long defaultTimeout = 30000L;\n\n    private final String defaultBrokerName = \"broker-0\";\n\n    private final String defaultBrokerAddr = \"127.0.0.1:10911\";\n\n    private final String defaultTopic = \"testTopic\";\n\n    @Before\n    public void init() throws Exception {\n        when(mQClientFactory.getTopicRouteTable()).thenReturn(mock(ConcurrentMap.class));\n        when(mQClientFactory.getClientId()).thenReturn(\"client-id\");\n        when(mQClientFactory.getMQAdminImpl()).thenReturn(mock(MQAdminImpl.class));\n        ClientConfig clientConfig = mock(ClientConfig.class);\n        when(messageQueue.getTopic()).thenReturn(defaultTopic);\n        when(clientConfig.queueWithNamespace(any())).thenReturn(messageQueue);\n        when(mQClientFactory.getClientConfig()).thenReturn(clientConfig);\n        when(mQClientFactory.getTopicRouteTable()).thenReturn(mock(ConcurrentMap.class));\n        when(mQClientFactory.getMQClientAPIImpl()).thenReturn(mQClientAPIImpl);\n        when(mQClientFactory.findBrokerAddressInPublish(or(isNull(), anyString()))).thenReturn(defaultBrokerAddr);\n        when(message.getTopic()).thenReturn(defaultTopic);\n        when(message.getProperty(MessageConst.PROPERTY_CORRELATION_ID)).thenReturn(\"correlation-id\");\n        when(message.getBody()).thenReturn(new byte[1]);\n        TransactionMQProducer producer = new TransactionMQProducer(\"test-producer-group\");\n        producer.setTransactionListener(mock(TransactionListener.class));\n        producer.setTopics(Collections.singletonList(defaultTopic));\n        defaultMQProducerImpl = new DefaultMQProducerImpl(producer);\n        setMQClientFactory();\n        setCheckExecutor();\n        setCheckForbiddenHookList();\n        setTopicPublishInfoTable();\n        defaultMQProducerImpl.setServiceState(ServiceState.RUNNING);\n    }\n\n    @Test\n    public void testRequest() throws Exception {\n        defaultMQProducerImpl.request(message, messageQueue, requestCallback, defaultTimeout);\n        defaultMQProducerImpl.request(message, queueSelector, 1, requestCallback, defaultTimeout);\n    }\n\n    @Test(expected = MQClientException.class)\n    public void testRequestMQClientExceptionByVoid() throws Exception {\n        defaultMQProducerImpl.request(message, requestCallback, defaultTimeout);\n    }\n\n    @Test\n    public void testCheckTransactionState() {\n        defaultMQProducerImpl.checkTransactionState(defaultBrokerAddr, mock(MessageExt.class), mock(CheckTransactionStateRequestHeader.class));\n    }\n\n    @Test\n    public void testCreateTopic() throws MQClientException {\n        defaultMQProducerImpl.createTopic(\"key\", defaultTopic, 0);\n    }\n\n    @Test\n    public void testExecuteCheckForbiddenHook() throws MQClientException {\n        defaultMQProducerImpl.executeCheckForbiddenHook(mock(CheckForbiddenContext.class));\n    }\n\n    @Test(expected = MQClientException.class)\n    public void testSendOneway() throws MQClientException, InterruptedException, RemotingException {\n        defaultMQProducerImpl.sendOneway(message);\n    }\n\n    @Test\n    public void testSendOnewayByQueueSelector() throws MQClientException, InterruptedException, RemotingException {\n        defaultMQProducerImpl.sendOneway(message, mock(MessageQueueSelector.class), 1);\n    }\n\n    @Test\n    public void testSendOnewayByQueue() throws MQClientException, InterruptedException, RemotingException {\n        defaultMQProducerImpl.sendOneway(message, messageQueue);\n    }\n\n    @Test(expected = MQClientException.class)\n    public void testSend() throws RemotingException, InterruptedException, MQClientException, MQBrokerException {\n        assertNull(defaultMQProducerImpl.send(message));\n    }\n\n    @Test\n    public void assertSendByQueue() throws InterruptedException, MQBrokerException, RemotingException, MQClientException {\n        SendResult actual = defaultMQProducerImpl.send(message, messageQueue);\n        assertNull(actual);\n        actual = defaultMQProducerImpl.send(message, messageQueue, defaultTimeout);\n        assertNull(actual);\n    }\n\n    @Test\n    public void assertSendByQueueSelector() throws InterruptedException, MQBrokerException, RemotingException, MQClientException {\n        SendCallback sendCallback = mock(SendCallback.class);\n        defaultMQProducerImpl.send(message, queueSelector, 1, sendCallback);\n        SendResult actual = defaultMQProducerImpl.send(message, queueSelector, 1);\n        assertNull(actual);\n        actual = defaultMQProducerImpl.send(message, queueSelector, 1, defaultTimeout);\n        assertNull(actual);\n    }\n\n    @Test(expected = MQClientException.class)\n    public void assertMQClientException() throws Exception {\n        assertNull(defaultMQProducerImpl.request(message, defaultTimeout));\n    }\n\n    @Test(expected = RequestTimeoutException.class)\n    public void assertRequestRequestTimeoutByQueueSelector() throws Exception {\n        assertNull(defaultMQProducerImpl.request(message, queueSelector, 1, 3000L));\n    }\n\n    @Test(expected = Exception.class)\n    public void assertRequestTimeoutExceptionByQueue() throws Exception {\n        assertNull(defaultMQProducerImpl.request(message, messageQueue, 3000L));\n    }\n\n    @Test\n    public void testRegisterCheckForbiddenHook() {\n        CheckForbiddenHook checkForbiddenHook = mock(CheckForbiddenHook.class);\n        defaultMQProducerImpl.registerCheckForbiddenHook(checkForbiddenHook);\n    }\n\n    @Test\n    public void testInitTopicRoute() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {\n        Class<?> clazz = defaultMQProducerImpl.getClass();\n        Method method = clazz.getDeclaredMethod(\"initTopicRoute\");\n        method.setAccessible(true);\n        method.invoke(defaultMQProducerImpl);\n    }\n\n    @Test\n    public void assertFetchPublishMessageQueues() throws MQClientException {\n        List<MessageQueue> actual = defaultMQProducerImpl.fetchPublishMessageQueues(defaultTopic);\n        assertNotNull(actual);\n        assertEquals(0, actual.size());\n    }\n\n    @Test\n    public void assertSearchOffset() throws MQClientException {\n        assertEquals(0, defaultMQProducerImpl.searchOffset(messageQueue, System.currentTimeMillis()));\n    }\n\n    @Test\n    public void assertMaxOffset() throws MQClientException {\n        assertEquals(0, defaultMQProducerImpl.maxOffset(messageQueue));\n    }\n\n    @Test\n    public void assertMinOffset() throws MQClientException {\n        assertEquals(0, defaultMQProducerImpl.minOffset(messageQueue));\n    }\n\n    @Test\n    public void assertEarliestMsgStoreTime() throws MQClientException {\n        assertEquals(0, defaultMQProducerImpl.earliestMsgStoreTime(messageQueue));\n    }\n\n    @Test\n    public void assertViewMessage() throws MQClientException, MQBrokerException, RemotingException, InterruptedException {\n        assertNull(defaultMQProducerImpl.viewMessage(defaultTopic, \"msgId\"));\n    }\n\n    @Test\n    public void assertQueryMessage() throws MQClientException, InterruptedException {\n        assertNull(defaultMQProducerImpl.queryMessage(defaultTopic, \"key\", 1, 0L, 10L));\n    }\n\n    @Test\n    public void assertQueryMessageByUniqKey() throws MQClientException, InterruptedException {\n        assertNull(defaultMQProducerImpl.queryMessageByUniqKey(defaultTopic, \"key\"));\n    }\n\n    @Test\n    public void assertSetAsyncSenderExecutor() {\n        ExecutorService asyncSenderExecutor = mock(ExecutorService.class);\n        defaultMQProducerImpl.setAsyncSenderExecutor(asyncSenderExecutor);\n        assertEquals(asyncSenderExecutor, defaultMQProducerImpl.getAsyncSenderExecutor());\n    }\n\n    @Test\n    public void assertServiceState() {\n        ServiceState serviceState = defaultMQProducerImpl.getServiceState();\n        assertNotNull(serviceState);\n        assertEquals(ServiceState.RUNNING, serviceState);\n        defaultMQProducerImpl.setServiceState(ServiceState.SHUTDOWN_ALREADY);\n        serviceState = defaultMQProducerImpl.getServiceState();\n        assertNotNull(serviceState);\n        assertEquals(ServiceState.SHUTDOWN_ALREADY, serviceState);\n    }\n\n    @Test\n    public void assertGetNotAvailableDuration() {\n        long[] notAvailableDuration = defaultMQProducerImpl.getNotAvailableDuration();\n        assertNotNull(notAvailableDuration);\n        defaultMQProducerImpl.setNotAvailableDuration(new long[1]);\n        notAvailableDuration = defaultMQProducerImpl.getNotAvailableDuration();\n        assertNotNull(notAvailableDuration);\n        assertEquals(1, notAvailableDuration.length);\n    }\n\n    @Test\n    public void assertGetLatencyMax() {\n        long[] actual = defaultMQProducerImpl.getLatencyMax();\n        assertNotNull(actual);\n        defaultMQProducerImpl.setLatencyMax(new long[1]);\n        actual = defaultMQProducerImpl.getLatencyMax();\n        assertNotNull(actual);\n        assertEquals(1, actual.length);\n    }\n\n    @Test\n    public void assertIsSendLatencyFaultEnable() {\n        boolean actual = defaultMQProducerImpl.isSendLatencyFaultEnable();\n        assertFalse(actual);\n        defaultMQProducerImpl.setSendLatencyFaultEnable(true);\n        actual = defaultMQProducerImpl.isSendLatencyFaultEnable();\n        assertTrue(actual);\n    }\n\n    @Test\n    public void assertGetMqFaultStrategy() {\n        assertNotNull(defaultMQProducerImpl.getMqFaultStrategy());\n    }\n\n    @Test\n    public void assertCheckListener() {\n        assertNull(defaultMQProducerImpl.checkListener());\n    }\n\n    @Test\n    public void testRecallMessage_invalid() {\n        assertThrows(MQClientException.class, () -> {\n            defaultMQProducerImpl.recallMessage(MixAll.REPLY_TOPIC_POSTFIX + defaultTopic, \"handle\");\n        });\n        assertThrows(MQClientException.class, () -> {\n            defaultMQProducerImpl.recallMessage(MixAll.DLQ_GROUP_TOPIC_PREFIX + defaultTopic, \"handle\");\n        });\n        assertThrows(MQClientException.class, () -> {\n            defaultMQProducerImpl.recallMessage(defaultTopic, \"handle\");\n        });\n    }\n\n    @Test\n    public void testRecallMessage_addressNotFound() {\n        String handle = RecallMessageHandle.HandleV1.buildHandle(defaultTopic, defaultBrokerName, \"1\", \"id\");\n        when(mQClientFactory.findBrokerAddressInPublish(defaultBrokerName)).thenReturn(null);\n        MQClientException e = assertThrows(MQClientException.class, () -> {\n            defaultMQProducerImpl.recallMessage(defaultTopic, handle);\n        });\n        assertEquals(\"The broker service address not found\", e.getErrorMessage());\n    }\n\n    @Test\n    public void testRecallMessage_success()\n        throws RemotingException, MQClientException, MQBrokerException, InterruptedException {\n        String handle = RecallMessageHandle.HandleV1.buildHandle(defaultTopic, defaultBrokerName, \"1\", \"id\");\n        when(mQClientFactory.findBrokerAddressInPublish(defaultBrokerName)).thenReturn(defaultBrokerAddr);\n        when(mQClientAPIImpl.recallMessage(any(), any(), anyLong())).thenReturn(\"id\");\n        String result = defaultMQProducerImpl.recallMessage(defaultTopic, handle);\n        assertEquals(\"id\", result);\n    }\n\n    private void setMQClientFactory() throws IllegalAccessException, NoSuchFieldException {\n        setField(defaultMQProducerImpl, \"mQClientFactory\", mQClientFactory);\n    }\n\n    private void setTopicPublishInfoTable() throws IllegalAccessException, NoSuchFieldException {\n        ConcurrentMap<String, TopicPublishInfo> topicPublishInfoTable = new ConcurrentHashMap<>();\n        TopicPublishInfo topicPublishInfo = mock(TopicPublishInfo.class);\n        when(topicPublishInfo.ok()).thenReturn(true);\n        topicPublishInfoTable.put(defaultTopic, topicPublishInfo);\n        setField(defaultMQProducerImpl, \"topicPublishInfoTable\", topicPublishInfoTable);\n    }\n\n    private void setCheckExecutor() throws NoSuchFieldException, IllegalAccessException {\n        setField(defaultMQProducerImpl, \"checkExecutor\", mock(ExecutorService.class));\n    }\n\n    private void setCheckForbiddenHookList() throws NoSuchFieldException, IllegalAccessException {\n        ArrayList<CheckForbiddenHook> checkForbiddenHookList = new ArrayList<>();\n        checkForbiddenHookList.add(mock(CheckForbiddenHook.class));\n        setField(defaultMQProducerImpl, \"checkForbiddenHookList\", checkForbiddenHookList);\n    }\n\n    private void setField(final Object target, final String fieldName, final Object newValue) throws NoSuchFieldException, IllegalAccessException {\n        Class<?> clazz = target.getClass();\n        Field field = clazz.getDeclaredField(fieldName);\n        field.setAccessible(true);\n        field.set(target, newValue);\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByHashTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.producer.selector;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class SelectMessageQueueByHashTest {\n\n    private String topic = \"FooBar\";\n\n    @Test\n    public void testSelect() throws Exception {\n        SelectMessageQueueByHash selector = new SelectMessageQueueByHash();\n\n        Message message = new Message(topic, new byte[] {});\n\n        List<MessageQueue> messageQueues = new ArrayList<>();\n        for (int i = 0; i < 10; i++) {\n            MessageQueue messageQueue = new MessageQueue(topic, \"DefaultBroker\", i);\n            messageQueues.add(messageQueue);\n        }\n\n        String orderId = \"123\";\n        String anotherOrderId = \"234\";\n        MessageQueue selected = selector.select(messageQueues, message, orderId);\n        assertThat(selector.select(messageQueues, message, anotherOrderId)).isNotEqualTo(selected);\n\n        //No exception is thrown while order Id hashcode is Integer.MIN\n        anotherOrderId = \"polygenelubricants\";\n        selector.select(messageQueues, message, anotherOrderId);\n        anotherOrderId = \"GydZG_\";\n        selector.select(messageQueues, message, anotherOrderId);\n        anotherOrderId = \"DESIGNING WORKHOUSES\";\n        selector.select(messageQueues, message, anotherOrderId);\n    }\n\n}"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByRandomTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.producer.selector;\n\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.junit.Test;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertThrows;\n\npublic class SelectMessageQueueByRandomTest {\n\n    private final SelectMessageQueueByRandom selector = new SelectMessageQueueByRandom();\n\n    private final String defaultBroker = \"defaultBroker\";\n\n    private final String defaultTopic = \"defaultTopic\";\n\n    @Test\n    public void testSelectRandomMessageQueue() {\n        List<MessageQueue> messageQueues = createMessageQueues(10);\n        Message message = new Message(defaultTopic, \"tag\", \"key\", \"body\".getBytes());\n        MessageQueue selectedQueue = selector.select(messageQueues, message, null);\n        assertNotNull(selectedQueue);\n        assertEquals(messageQueues.size(), 10);\n        assertEquals(defaultTopic, selectedQueue.getTopic());\n        assertEquals(defaultBroker, selectedQueue.getBrokerName());\n    }\n\n    @Test\n    public void testSelectEmptyMessageQueue() {\n        List<MessageQueue> emptyQueues = new ArrayList<>();\n        Message message = new Message(defaultTopic, \"tag\", \"key\", \"body\".getBytes());\n        assertThrows(IllegalArgumentException.class, () -> selector.select(emptyQueues, message, null));\n    }\n\n    @Test\n    public void testSelectSingleMessageQueue() {\n        List<MessageQueue> singleQueueList = createMessageQueues(1);\n        Message message = new Message(defaultTopic, \"tag\", \"key\", \"body\".getBytes());\n        MessageQueue selectedQueue = selector.select(singleQueueList, message, null);\n        assertNotNull(selectedQueue);\n        assertEquals(defaultTopic, selectedQueue.getTopic());\n        assertEquals(defaultBroker, selectedQueue.getBrokerName());\n        assertEquals(singleQueueList.get(0).getQueueId(), selectedQueue.getQueueId());\n    }\n\n    private List<MessageQueue> createMessageQueues(final int count) {\n        List<MessageQueue> result = new ArrayList<>();\n        for (int i = 0; i < count; i++) {\n            result.add(new MessageQueue(defaultTopic, defaultBroker, i));\n        }\n        return result;\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueRetryTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.producer.selector;\n\nimport org.apache.rocketmq.client.impl.producer.TopicPublishInfo;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.junit.Test;\n\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class SelectMessageQueueRetryTest {\n\n    private String topic = \"TEST\";\n\n    @Test\n    public void testSelect() throws Exception {\n\n        TopicPublishInfo topicPublishInfo = new TopicPublishInfo();\n        List<MessageQueue> messageQueueList = new ArrayList();\n        for (int i = 0; i < 3; i++) {\n            MessageQueue mq = new MessageQueue();\n            mq.setBrokerName(\"broker-\" + i);\n            mq.setQueueId(0);\n            mq.setTopic(topic);\n            messageQueueList.add(mq);\n        }\n\n        topicPublishInfo.setMessageQueueList(messageQueueList);\n\n        Set<String> retryBrokerNameSet = retryBroker(topicPublishInfo);\n        //always in Set (broker-0, broker-1, broker-2)\n        assertThat(retryBroker(topicPublishInfo)).isEqualTo(retryBrokerNameSet);\n    }\n\n    private Set<String> retryBroker(TopicPublishInfo topicPublishInfo) {\n        MessageQueue mqTmp = null;\n        Set<String> retryBrokerNameSet = new HashSet();\n        for (int times = 0; times < 3; times++) {\n            String lastBrokerName = null == mqTmp ? null : mqTmp.getBrokerName();\n            mqTmp = topicPublishInfo.selectOneMessageQueue(lastBrokerName);\n            retryBrokerNameSet.add(mqTmp.getBrokerName());\n        }\n        return retryBrokerNameSet;\n    }\n\n}"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/rpchook/NamespaceRpcHookTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.rpchook;\n\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class NamespaceRpcHookTest {\n    private NamespaceRpcHook namespaceRpcHook;\n    private ClientConfig clientConfig;\n    private String namespace = \"namespace\";\n\n\n    @Test\n    public void testDoBeforeRequestWithNamespace() {\n        clientConfig = new ClientConfig();\n        clientConfig.setNamespaceV2(namespace);\n        namespaceRpcHook = new NamespaceRpcHook(clientConfig);\n        PullMessageRequestHeader pullMessageRequestHeader = new PullMessageRequestHeader();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, pullMessageRequestHeader);\n        namespaceRpcHook.doBeforeRequest(\"\", request);\n        assertThat(request.getExtFields().get(MixAll.RPC_REQUEST_HEADER_NAMESPACED_FIELD)).isEqualTo(\"true\");\n        assertThat(request.getExtFields().get(MixAll.RPC_REQUEST_HEADER_NAMESPACE_FIELD)).isEqualTo(namespace);\n    }\n\n    @Test\n    public void testDoBeforeRequestWithoutNamespace() {\n        clientConfig = new ClientConfig();\n        namespaceRpcHook = new NamespaceRpcHook(clientConfig);\n        PullMessageRequestHeader pullMessageRequestHeader = new PullMessageRequestHeader();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, pullMessageRequestHeader);\n        namespaceRpcHook.doBeforeRequest(\"\", request);\n        assertThat(request.getExtFields()).isNull();\n    }\n}"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithOpenTracingTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.trace;\n\nimport io.opentracing.mock.MockSpan;\nimport io.opentracing.mock.MockTracer;\nimport io.opentracing.tag.Tags;\nimport java.io.ByteArrayOutputStream;\nimport java.net.InetSocketAddress;\nimport java.util.Collections;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.Callable;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicReference;\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.PullCallback;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.consumer.store.OffsetStore;\nimport org.apache.rocketmq.client.consumer.store.ReadOffsetType;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.FindBrokerResult;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.MQClientManager;\nimport org.apache.rocketmq.client.impl.consumer.ConsumeMessageConcurrentlyService;\nimport org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl;\nimport org.apache.rocketmq.client.impl.consumer.ProcessQueue;\nimport org.apache.rocketmq.client.impl.consumer.PullAPIWrapper;\nimport org.apache.rocketmq.client.impl.consumer.PullMessageService;\nimport org.apache.rocketmq.client.impl.consumer.PullRequest;\nimport org.apache.rocketmq.client.impl.consumer.PullResultExt;\nimport org.apache.rocketmq.client.impl.consumer.RebalancePushImpl;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.client.trace.hook.ConsumeMessageOpenTracingHookImpl;\nimport org.apache.rocketmq.common.message.MessageClientExt;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.invocation.InvocationOnMock;\nimport org.mockito.junit.MockitoJUnitRunner;\nimport org.mockito.stubbing.Answer;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.awaitility.Awaitility.waitAtMost;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyBoolean;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.nullable;\nimport static org.mockito.Mockito.doReturn;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class DefaultMQConsumerWithOpenTracingTest {\n    private String consumerGroup;\n\n    private String topic = \"FooBar\";\n    private String brokerName = \"BrokerA\";\n    private MQClientInstance mQClientFactory;\n\n    @Mock\n    private MQClientAPIImpl mQClientAPIImpl;\n    private PullAPIWrapper pullAPIWrapper;\n    private RebalancePushImpl rebalancePushImpl;\n    private DefaultMQPushConsumer pushConsumer;\n    private final MockTracer tracer = new MockTracer();\n\n    @Before\n    public void init() throws Exception {\n        ConcurrentMap<String, MQClientInstance> factoryTable = (ConcurrentMap<String, MQClientInstance>) FieldUtils.readDeclaredField(MQClientManager.getInstance(), \"factoryTable\", true);\n        for (Map.Entry<String, MQClientInstance> entry : factoryTable.entrySet()) {\n            entry.getValue().shutdown();\n        }\n        factoryTable.clear();\n\n        when(mQClientAPIImpl.pullMessage(anyString(), any(PullMessageRequestHeader.class),\n            anyLong(), any(CommunicationMode.class), nullable(PullCallback.class)))\n            .thenAnswer(new Answer<PullResult>() {\n                @Override\n                public PullResult answer(InvocationOnMock mock) throws Throwable {\n                    PullMessageRequestHeader requestHeader = mock.getArgument(1);\n                    MessageClientExt messageClientExt = new MessageClientExt();\n                    messageClientExt.setTopic(topic);\n                    messageClientExt.setQueueId(0);\n                    messageClientExt.setMsgId(\"123\");\n                    messageClientExt.setBody(new byte[] {'a'});\n                    messageClientExt.setOffsetMsgId(\"234\");\n                    messageClientExt.setBornHost(new InetSocketAddress(8080));\n                    messageClientExt.setStoreHost(new InetSocketAddress(8080));\n                    PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.<MessageExt>singletonList(messageClientExt));\n                    ((PullCallback) mock.getArgument(4)).onSuccess(pullResult);\n                    return pullResult;\n                }\n            });\n\n        consumerGroup = \"FooBarGroup\" + System.currentTimeMillis();\n        pushConsumer = new DefaultMQPushConsumer(consumerGroup);\n        pushConsumer.getDefaultMQPushConsumerImpl().registerConsumeMessageHook(\n            new ConsumeMessageOpenTracingHookImpl(tracer));\n        pushConsumer.setNamesrvAddr(\"127.0.0.1:9876\");\n        pushConsumer.setPullInterval(60 * 1000);\n        // disable trace to let mock trace work\n        pushConsumer.setEnableTrace(false);\n\n        OffsetStore offsetStore = Mockito.mock(OffsetStore.class);\n        Mockito.when(offsetStore.readOffset(any(MessageQueue.class), any(ReadOffsetType.class))).thenReturn(0L);\n        pushConsumer.setOffsetStore(offsetStore);\n\n        pushConsumer.registerMessageListener(new MessageListenerConcurrently() {\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,\n                ConsumeConcurrentlyContext context) {\n                return null;\n            }\n        });\n\n        DefaultMQPushConsumerImpl pushConsumerImpl = pushConsumer.getDefaultMQPushConsumerImpl();\n\n        // suppress updateTopicRouteInfoFromNameServer\n        pushConsumer.changeInstanceNameToPID();\n        mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(pushConsumer, (RPCHook) FieldUtils.readDeclaredField(pushConsumerImpl, \"rpcHook\", true));\n        FieldUtils.writeDeclaredField(mQClientFactory, \"mQClientAPIImpl\", mQClientAPIImpl, true);\n        mQClientFactory = spy(mQClientFactory);\n        factoryTable.put(pushConsumer.buildMQClientId(), mQClientFactory);\n        doReturn(false).when(mQClientFactory).updateTopicRouteInfoFromNameServer(anyString());\n\n        doReturn(new FindBrokerResult(\"127.0.0.1:10911\", false)).when(mQClientFactory).findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean());\n\n        Set<MessageQueue> messageQueueSet = new HashSet<>();\n        messageQueueSet.add(createPullRequest().getMessageQueue());\n        pushConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet);\n\n        pushConsumer.subscribe(topic, \"*\");\n        pushConsumer.start();\n    }\n\n    @After\n    public void terminate() {\n        pushConsumer.shutdown();\n    }\n\n    @Test\n    public void testPullMessage_WithTrace_Success() throws InterruptedException, RemotingException, MQBrokerException, MQClientException {\n        final CountDownLatch countDownLatch = new CountDownLatch(1);\n        final AtomicReference<MessageExt> messageAtomic = new AtomicReference<>();\n        pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() {\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,\n                ConsumeConcurrentlyContext context) {\n                messageAtomic.set(msgs.get(0));\n                countDownLatch.countDown();\n                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n            }\n        }));\n\n        PullMessageService pullMessageService = mQClientFactory.getPullMessageService();\n        pullMessageService.executePullRequestImmediately(createPullRequest());\n        countDownLatch.await(30, TimeUnit.SECONDS);\n        MessageExt msg = messageAtomic.get();\n        assertThat(msg).isNotNull();\n        assertThat(msg.getTopic()).isEqualTo(topic);\n        assertThat(msg.getBody()).isEqualTo(new byte[] {'a'});\n\n        // wait until consumeMessageAfter hook of tracer is done surely.\n        waitAtMost(1, TimeUnit.SECONDS).until(new Callable() {\n            @Override\n            public Object call() throws Exception {\n                return tracer.finishedSpans().size() == 1;\n            }\n        });\n\n        MockSpan span = tracer.finishedSpans().get(0);\n        assertThat(span.tags().get(Tags.MESSAGE_BUS_DESTINATION.getKey())).isEqualTo(topic);\n        assertThat(span.tags().get(Tags.SPAN_KIND.getKey())).isEqualTo(Tags.SPAN_KIND_CONSUMER);\n        assertThat(span.tags().get(TraceConstants.ROCKETMQ_SUCCESS)).isEqualTo(true);\n    }\n\n    private PullRequest createPullRequest() {\n        PullRequest pullRequest = new PullRequest();\n        pullRequest.setConsumerGroup(consumerGroup);\n        pullRequest.setNextOffset(1024);\n\n        MessageQueue messageQueue = new MessageQueue();\n        messageQueue.setBrokerName(brokerName);\n        messageQueue.setQueueId(0);\n        messageQueue.setTopic(topic);\n        pullRequest.setMessageQueue(messageQueue);\n        ProcessQueue processQueue = new ProcessQueue();\n        processQueue.setLocked(true);\n        processQueue.setLastLockTimestamp(System.currentTimeMillis());\n        pullRequest.setProcessQueue(processQueue);\n\n        return pullRequest;\n    }\n\n    private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus,\n        List<MessageExt> messageExtList) throws Exception {\n        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();\n        for (MessageExt messageExt : messageExtList) {\n            outputStream.write(MessageDecoder.encode(messageExt, false));\n        }\n        return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray());\n    }\n\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithTraceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.trace;\n\nimport java.io.ByteArrayOutputStream;\nimport java.lang.reflect.Field;\nimport java.net.InetSocketAddress;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicReference;\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.PullCallback;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.FindBrokerResult;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.MQClientManager;\nimport org.apache.rocketmq.client.impl.consumer.ConsumeMessageConcurrentlyService;\nimport org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl;\nimport org.apache.rocketmq.client.impl.consumer.ProcessQueue;\nimport org.apache.rocketmq.client.impl.consumer.PullAPIWrapper;\nimport org.apache.rocketmq.client.impl.consumer.PullMessageService;\nimport org.apache.rocketmq.client.impl.consumer.PullRequest;\nimport org.apache.rocketmq.client.impl.consumer.PullResultExt;\nimport org.apache.rocketmq.client.impl.consumer.RebalancePushImpl;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.message.MessageClientExt;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.invocation.InvocationOnMock;\nimport org.mockito.junit.MockitoJUnitRunner;\nimport org.mockito.stubbing.Answer;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyBoolean;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.nullable;\nimport static org.mockito.Mockito.doReturn;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.Silent.class)\npublic class DefaultMQConsumerWithTraceTest {\n    private String consumerGroup;\n    private String consumerGroupNormal;\n    private String producerGroupTraceTemp = TopicValidator.RMQ_SYS_TRACE_TOPIC + System.currentTimeMillis();\n\n    private String topic = \"FooBar\";\n    private String brokerName = \"BrokerA\";\n    private MQClientInstance mQClientFactory;\n\n    @Mock\n    private MQClientAPIImpl mQClientAPIImpl;\n    private PullAPIWrapper pullAPIWrapper;\n    private RebalancePushImpl rebalancePushImpl;\n    private DefaultMQPushConsumer pushConsumer;\n    private DefaultMQPushConsumer normalPushConsumer;\n    private DefaultMQPushConsumer customTraceTopicPushConsumer;\n\n    private AsyncTraceDispatcher asyncTraceDispatcher;\n    private MQClientInstance mQClientTraceFactory;\n    @Mock\n    private MQClientAPIImpl mQClientTraceAPIImpl;\n    private DefaultMQProducer traceProducer;\n    private String customerTraceTopic = \"rmq_trace_topic_12345\";\n\n    @Before\n    public void init() throws Exception {\n        ConcurrentMap<String, MQClientInstance> factoryTable = (ConcurrentMap<String, MQClientInstance>) FieldUtils.readDeclaredField(MQClientManager.getInstance(), \"factoryTable\", true);\n        for (Map.Entry<String, MQClientInstance> entry : factoryTable.entrySet()) {\n            entry.getValue().shutdown();\n        }\n        factoryTable.clear();\n\n        consumerGroup = \"FooBarGroup\" + System.currentTimeMillis();\n        pushConsumer = new DefaultMQPushConsumer(consumerGroup, true, \"\");\n        consumerGroupNormal = \"FooBarGroup\" + System.currentTimeMillis();\n        normalPushConsumer = new DefaultMQPushConsumer(consumerGroupNormal, false, \"\");\n        customTraceTopicPushConsumer = new DefaultMQPushConsumer(consumerGroup, true, customerTraceTopic);\n        pushConsumer.setNamesrvAddr(\"127.0.0.1:9876\");\n        pushConsumer.setUseTLS(true);\n        pushConsumer.setPullInterval(60 * 1000);\n\n        pushConsumer.registerMessageListener(new MessageListenerConcurrently() {\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,\n                ConsumeConcurrentlyContext context) {\n                return null;\n            }\n        });\n\n        DefaultMQPushConsumerImpl pushConsumerImpl = pushConsumer.getDefaultMQPushConsumerImpl();\n\n        // suppress updateTopicRouteInfoFromNameServer\n        pushConsumer.changeInstanceNameToPID();\n        mQClientFactory = spy(MQClientManager.getInstance().getOrCreateMQClientInstance(pushConsumer, (RPCHook) FieldUtils.readDeclaredField(pushConsumerImpl, \"rpcHook\", true)));\n        factoryTable.put(pushConsumer.buildMQClientId(), mQClientFactory);\n        doReturn(false).when(mQClientFactory).updateTopicRouteInfoFromNameServer(anyString());\n\n        rebalancePushImpl = spy(new RebalancePushImpl(pushConsumer.getDefaultMQPushConsumerImpl()));\n        Field field = DefaultMQPushConsumerImpl.class.getDeclaredField(\"rebalanceImpl\");\n        field.setAccessible(true);\n        field.set(pushConsumerImpl, rebalancePushImpl);\n        pushConsumer.subscribe(topic, \"*\");\n\n        pushConsumer.start();\n\n        asyncTraceDispatcher = (AsyncTraceDispatcher) pushConsumer.getTraceDispatcher();\n        traceProducer = asyncTraceDispatcher.getTraceProducer();\n\n        mQClientFactory = spy(pushConsumerImpl.getmQClientFactory());\n        mQClientTraceFactory = spy(pushConsumerImpl.getmQClientFactory());\n\n        field = DefaultMQPushConsumerImpl.class.getDeclaredField(\"mQClientFactory\");\n        field.setAccessible(true);\n        field.set(pushConsumerImpl, mQClientFactory);\n\n        field = MQClientInstance.class.getDeclaredField(\"mQClientAPIImpl\");\n        field.setAccessible(true);\n        field.set(mQClientFactory, mQClientAPIImpl);\n\n        Field fieldTrace = DefaultMQProducerImpl.class.getDeclaredField(\"mQClientFactory\");\n        fieldTrace.setAccessible(true);\n        fieldTrace.set(traceProducer.getDefaultMQProducerImpl(), mQClientTraceFactory);\n\n        fieldTrace = MQClientInstance.class.getDeclaredField(\"mQClientAPIImpl\");\n        fieldTrace.setAccessible(true);\n        fieldTrace.set(mQClientTraceFactory, mQClientTraceAPIImpl);\n\n        pullAPIWrapper = spy(new PullAPIWrapper(mQClientFactory, consumerGroup, false));\n        field = DefaultMQPushConsumerImpl.class.getDeclaredField(\"pullAPIWrapper\");\n        field.setAccessible(true);\n        field.set(pushConsumerImpl, pullAPIWrapper);\n\n        pushConsumer.getDefaultMQPushConsumerImpl().getRebalanceImpl().setmQClientFactory(mQClientFactory);\n        mQClientFactory.registerConsumer(consumerGroup, pushConsumerImpl);\n\n        when(mQClientFactory.getMQClientAPIImpl().pullMessage(anyString(), any(PullMessageRequestHeader.class),\n            anyLong(), any(CommunicationMode.class), nullable(PullCallback.class)))\n            .thenAnswer(new Answer<PullResult>() {\n                @Override\n                public PullResult answer(InvocationOnMock mock) throws Throwable {\n                    PullMessageRequestHeader requestHeader = mock.getArgument(1);\n                    MessageClientExt messageClientExt = new MessageClientExt();\n                    messageClientExt.setTopic(topic);\n                    messageClientExt.setQueueId(0);\n                    messageClientExt.setMsgId(\"123\");\n                    messageClientExt.setBody(new byte[] {'a'});\n                    messageClientExt.setOffsetMsgId(\"234\");\n                    messageClientExt.setBornHost(new InetSocketAddress(8080));\n                    messageClientExt.setStoreHost(new InetSocketAddress(8080));\n                    PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.<MessageExt>singletonList(messageClientExt));\n                    ((PullCallback) mock.getArgument(4)).onSuccess(pullResult);\n                    return pullResult;\n                }\n            });\n\n        doReturn(new FindBrokerResult(\"127.0.0.1:10911\", false)).when(mQClientFactory).findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean());\n        Set<MessageQueue> messageQueueSet = new HashSet<>();\n        messageQueueSet.add(createPullRequest().getMessageQueue());\n        pushConsumer.getDefaultMQPushConsumerImpl().updateTopicSubscribeInfo(topic, messageQueueSet);\n    }\n\n    @After\n    public void terminate() {\n        pushConsumer.shutdown();\n    }\n\n    @Test\n    public void testPullMessage_WithTrace_Success() throws InterruptedException, RemotingException, MQBrokerException, MQClientException {\n        traceProducer.getDefaultMQProducerImpl().getMqClientFactory().registerProducer(producerGroupTraceTemp, traceProducer.getDefaultMQProducerImpl());\n\n        final CountDownLatch countDownLatch = new CountDownLatch(1);\n        final AtomicReference<MessageExt> messageAtomic = new AtomicReference<>();\n        pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() {\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,\n                ConsumeConcurrentlyContext context) {\n                messageAtomic.set(msgs.get(0));\n                countDownLatch.countDown();\n                return null;\n            }\n        }));\n\n        PullMessageService pullMessageService = mQClientFactory.getPullMessageService();\n        pullMessageService.executePullRequestImmediately(createPullRequest());\n        countDownLatch.await(30, TimeUnit.SECONDS);\n        MessageExt msg = messageAtomic.get();\n        assertThat(msg).isNotNull();\n        assertThat(msg.getTopic()).isEqualTo(topic);\n        assertThat(msg.getBody()).isEqualTo(new byte[] {'a'});\n    }\n\n    @Test\n    public void testPushConsumerWithTraceTLS() {\n        Assert.assertTrue(asyncTraceDispatcher.getTraceProducer().isUseTLS());\n    }\n\n    private PullRequest createPullRequest() {\n        PullRequest pullRequest = new PullRequest();\n        pullRequest.setConsumerGroup(consumerGroup);\n        pullRequest.setNextOffset(1024);\n\n        MessageQueue messageQueue = new MessageQueue();\n        messageQueue.setBrokerName(brokerName);\n        messageQueue.setQueueId(0);\n        messageQueue.setTopic(topic);\n        pullRequest.setMessageQueue(messageQueue);\n        ProcessQueue processQueue = new ProcessQueue();\n        processQueue.setLocked(true);\n        processQueue.setLastLockTimestamp(System.currentTimeMillis());\n        pullRequest.setProcessQueue(processQueue);\n\n        return pullRequest;\n    }\n\n    private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus,\n        List<MessageExt> messageExtList) throws Exception {\n        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();\n        for (MessageExt messageExt : messageExtList) {\n            outputStream.write(MessageDecoder.encode(messageExt, false));\n        }\n        return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray());\n    }\n\n    public static TopicRouteData createTopicRoute() {\n        TopicRouteData topicRouteData = new TopicRouteData();\n\n        topicRouteData.setFilterServerTable(new HashMap<>());\n        List<BrokerData> brokerDataList = new ArrayList<>();\n        BrokerData brokerData = new BrokerData();\n        brokerData.setBrokerName(\"BrokerA\");\n        brokerData.setCluster(\"DefaultCluster\");\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(0L, \"127.0.0.1:10911\");\n        brokerData.setBrokerAddrs(brokerAddrs);\n        brokerDataList.add(brokerData);\n        topicRouteData.setBrokerDatas(brokerDataList);\n\n        List<QueueData> queueDataList = new ArrayList<>();\n        QueueData queueData = new QueueData();\n        queueData.setBrokerName(\"BrokerA\");\n        queueData.setPerm(6);\n        queueData.setReadQueueNums(3);\n        queueData.setWriteQueueNums(4);\n        queueData.setTopicSysFlag(0);\n        queueDataList.add(queueData);\n        topicRouteData.setQueueDatas(queueDataList);\n        return topicRouteData;\n    }\n\n    private SendResult createSendResult(SendStatus sendStatus) {\n        SendResult sendResult = new SendResult();\n        sendResult.setMsgId(\"123\");\n        sendResult.setOffsetMsgId(\"123\");\n        sendResult.setQueueOffset(456);\n        sendResult.setSendStatus(sendStatus);\n        sendResult.setRegionId(\"HZ\");\n        return sendResult;\n    }\n\n    public static TopicRouteData createTraceTopicRoute() {\n        TopicRouteData topicRouteData = new TopicRouteData();\n\n        topicRouteData.setFilterServerTable(new HashMap<>());\n        List<BrokerData> brokerDataList = new ArrayList<>();\n        BrokerData brokerData = new BrokerData();\n        brokerData.setBrokerName(\"broker-trace\");\n        brokerData.setCluster(\"DefaultCluster\");\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(0L, \"127.0.0.1:10912\");\n        brokerData.setBrokerAddrs(brokerAddrs);\n        brokerDataList.add(brokerData);\n        topicRouteData.setBrokerDatas(brokerDataList);\n\n        List<QueueData> queueDataList = new ArrayList<>();\n        QueueData queueData = new QueueData();\n        queueData.setBrokerName(\"broker-trace\");\n        queueData.setPerm(6);\n        queueData.setReadQueueNums(1);\n        queueData.setWriteQueueNums(1);\n        queueData.setTopicSysFlag(1);\n        queueDataList.add(queueData);\n        topicRouteData.setQueueDatas(queueDataList);\n        return topicRouteData;\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQLitePullConsumerWithTraceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.client.trace;\n\nimport java.io.ByteArrayOutputStream;\nimport java.lang.reflect.Field;\nimport java.net.InetSocketAddress;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.client.consumer.DefaultLitePullConsumer;\nimport org.apache.rocketmq.client.consumer.PullCallback;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.client.consumer.store.OffsetStore;\nimport org.apache.rocketmq.client.consumer.store.ReadOffsetType;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.FindBrokerResult;\nimport org.apache.rocketmq.client.impl.MQAdminImpl;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.MQClientManager;\nimport org.apache.rocketmq.client.impl.consumer.DefaultLitePullConsumerImpl;\nimport org.apache.rocketmq.client.impl.consumer.PullAPIWrapper;\nimport org.apache.rocketmq.client.impl.consumer.PullResultExt;\nimport org.apache.rocketmq.client.impl.consumer.RebalanceImpl;\nimport org.apache.rocketmq.client.impl.consumer.RebalanceService;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.message.MessageClientExt;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.BeforeClass;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.invocation.InvocationOnMock;\nimport org.mockito.junit.MockitoJUnitRunner;\nimport org.mockito.stubbing.Answer;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyBoolean;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.nullable;\nimport static org.mockito.Mockito.doReturn;\nimport static org.mockito.Mockito.lenient;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class DefaultMQLitePullConsumerWithTraceTest {\n\n    private MQClientInstance mQClientFactory;\n    private MQClientInstance mqClientInstance;\n    private MQClientInstance traceMqClientInstance;\n\n    @Mock\n    private MQClientAPIImpl mQClientAPIImpl;\n    @Mock\n    private MQAdminImpl mQAdminImpl;\n\n    private AsyncTraceDispatcher asyncTraceDispatcher;\n    private DefaultMQProducer traceProducer;\n    private RebalanceImpl rebalanceImpl;\n    private OffsetStore offsetStore;\n    private DefaultLitePullConsumerImpl litePullConsumerImpl;\n    private String consumerGroup = \"LitePullConsumerGroup\";\n    private String topic = \"LitePullConsumerTest\";\n    private String brokerName = \"BrokerA\";\n    private String producerGroupTraceTemp = TopicValidator.RMQ_SYS_TRACE_TOPIC + System.currentTimeMillis();\n\n    private String customerTraceTopic = \"rmq_trace_topic_12345\";\n\n    @BeforeClass\n    public static void setUpEnv() {\n        System.setProperty(\"rocketmq.client.logRoot\", System.getProperty(\"java.io.tmpdir\"));\n    }\n\n    @Before\n    public void init() throws Exception {\n        ConcurrentMap<String, MQClientInstance> factoryTable =\n            (ConcurrentMap<String, MQClientInstance>) FieldUtils.readDeclaredField(\n                MQClientManager.getInstance(), \"factoryTable\", true);\n        factoryTable.forEach((clientId, instance) -> instance.shutdown());\n        factoryTable.clear();\n        mQClientFactory = null;\n        mqClientInstance = null;\n        traceMqClientInstance = null;\n        asyncTraceDispatcher = null;\n        traceProducer = null;\n        rebalanceImpl = null;\n        offsetStore = null;\n        litePullConsumerImpl = null;\n    }\n\n    @After\n    public void destroy() {\n        if (traceProducer != null) {\n            MQClientInstance traceClientFactory = traceProducer.getDefaultMQProducerImpl().getMqClientFactory();\n            traceClientFactory.unregisterProducer(producerGroupTraceTemp);\n            traceClientFactory.unregisterProducer(traceProducer.getProducerGroup());\n        }\n\n        if (traceMqClientInstance != null && traceProducer != null) {\n            traceMqClientInstance.unregisterProducer(traceProducer.getProducerGroup());\n            traceMqClientInstance.shutdown();\n        }\n\n        if (litePullConsumerImpl != null) {\n            if (mQClientFactory != null) {\n                mQClientFactory.unregisterConsumer(litePullConsumerImpl.groupName());\n                mQClientFactory.shutdown();\n            }\n\n            if (mqClientInstance != null && mqClientInstance != mQClientFactory) {\n                mqClientInstance.unregisterConsumer(litePullConsumerImpl.groupName());\n                mqClientInstance.shutdown();\n            }\n        }\n    }\n\n    @Test\n    public void testSubscribe_PollMessageSuccess_WithDefaultTraceTopic() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = createLitePullConsumerWithDefaultTraceTopic();\n        try {\n            Set<MessageQueue> messageQueueSet = new HashSet<>();\n            messageQueueSet.add(createMessageQueue());\n            litePullConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet);\n            List<MessageExt> result = pollUntilFound(litePullConsumer);\n            assertThat(result).isNotEmpty();\n            assertThat(result.get(0).getTopic()).isEqualTo(topic);\n            assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'});\n        } finally {\n            litePullConsumer.shutdown();\n        }\n    }\n\n    @Test\n    public void testSubscribe_PollMessageSuccess_WithCustomizedTraceTopic() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = createLitePullConsumerWithCustomizedTraceTopic();\n        try {\n            Set<MessageQueue> messageQueueSet = new HashSet<>();\n            messageQueueSet.add(createMessageQueue());\n            litePullConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet);\n            List<MessageExt> result = pollUntilFound(litePullConsumer);\n            assertThat(result).isNotEmpty();\n            assertThat(result.get(0).getTopic()).isEqualTo(topic);\n            assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'});\n        } finally {\n            litePullConsumer.shutdown();\n        }\n    }\n\n    @Test\n    public void testLitePullConsumerWithTraceTLS() throws Exception {\n        DefaultLitePullConsumer consumer = new DefaultLitePullConsumer(\"consumerGroup\");\n        try {\n            consumer.setUseTLS(true);\n            consumer.setEnableMsgTrace(true);\n            consumer.start();\n            AsyncTraceDispatcher asyncTraceDispatcher = (AsyncTraceDispatcher) consumer.getTraceDispatcher();\n            Assert.assertTrue(asyncTraceDispatcher.getTraceProducer().isUseTLS());\n        } finally {\n            consumer.shutdown();\n        }\n    }\n\n    private DefaultLitePullConsumer createLitePullConsumerWithDefaultTraceTopic() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis());\n        litePullConsumer.setEnableMsgTrace(true);\n        litePullConsumer.setNamesrvAddr(\"127.0.0.1:9876\");\n        litePullConsumer.subscribe(topic, \"*\");\n        suppressUpdateTopicRouteInfoFromNameServer(litePullConsumer);\n        litePullConsumer.start();\n        initDefaultLitePullConsumer(litePullConsumer);\n        return litePullConsumer;\n    }\n\n    private DefaultLitePullConsumer createLitePullConsumerWithCustomizedTraceTopic() throws Exception {\n        DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis());\n        litePullConsumer.setEnableMsgTrace(true);\n        litePullConsumer.setCustomizedTraceTopic(customerTraceTopic);\n        litePullConsumer.setNamesrvAddr(\"127.0.0.1:9876\");\n        litePullConsumer.subscribe(topic, \"*\");\n        suppressUpdateTopicRouteInfoFromNameServer(litePullConsumer);\n        litePullConsumer.start();\n        initDefaultLitePullConsumer(litePullConsumer);\n        return litePullConsumer;\n    }\n\n    private void initDefaultLitePullConsumer(DefaultLitePullConsumer litePullConsumer) throws Exception {\n        asyncTraceDispatcher = (AsyncTraceDispatcher) litePullConsumer.getTraceDispatcher();\n        traceProducer = asyncTraceDispatcher.getTraceProducer();\n        Field field = DefaultLitePullConsumer.class.getDeclaredField(\"defaultLitePullConsumerImpl\");\n        field.setAccessible(true);\n        litePullConsumerImpl = (DefaultLitePullConsumerImpl) field.get(litePullConsumer);\n        field = DefaultLitePullConsumerImpl.class.getDeclaredField(\"mQClientFactory\");\n        field.setAccessible(true);\n        mqClientInstance = (MQClientInstance) field.get(litePullConsumerImpl);\n        mQClientFactory = spy(mqClientInstance);\n        mQClientFactory.getClientConfig().setDecodeReadBody(true);\n        field.set(litePullConsumerImpl, mQClientFactory);\n\n        field = MQClientInstance.class.getDeclaredField(\"rebalanceService\");\n        field.setAccessible(true);\n        RebalanceService rebalanceService = (RebalanceService) field.get(mQClientFactory);\n        field = RebalanceService.class.getDeclaredField(\"waitInterval\");\n        field.setAccessible(true);\n        field.set(rebalanceService, 100);\n\n        PullAPIWrapper pullAPIWrapper = litePullConsumerImpl.getPullAPIWrapper();\n        field = PullAPIWrapper.class.getDeclaredField(\"mQClientFactory\");\n        field.setAccessible(true);\n        field.set(pullAPIWrapper, mQClientFactory);\n\n        Field fieldTrace = DefaultMQProducerImpl.class.getDeclaredField(\"mQClientFactory\");\n        fieldTrace.setAccessible(true);\n        traceMqClientInstance = traceProducer.getDefaultMQProducerImpl().getMqClientFactory();\n        fieldTrace.set(traceProducer.getDefaultMQProducerImpl(), mQClientFactory);\n\n        field = MQClientInstance.class.getDeclaredField(\"mQClientAPIImpl\");\n        field.setAccessible(true);\n        field.set(mQClientFactory, mQClientAPIImpl);\n\n        field = MQClientInstance.class.getDeclaredField(\"mQAdminImpl\");\n        field.setAccessible(true);\n        field.set(mQClientFactory, mQAdminImpl);\n\n        field = DefaultLitePullConsumerImpl.class.getDeclaredField(\"rebalanceImpl\");\n        field.setAccessible(true);\n        rebalanceImpl = (RebalanceImpl) field.get(litePullConsumerImpl);\n        field = RebalanceImpl.class.getDeclaredField(\"mQClientFactory\");\n        field.setAccessible(true);\n        field.set(rebalanceImpl, mQClientFactory);\n\n        offsetStore = spy(litePullConsumerImpl.getOffsetStore());\n        field = DefaultLitePullConsumerImpl.class.getDeclaredField(\"offsetStore\");\n        field.setAccessible(true);\n        field.set(litePullConsumerImpl, offsetStore);\n\n        traceProducer.getDefaultMQProducerImpl().getMqClientFactory().registerProducer(producerGroupTraceTemp, traceProducer.getDefaultMQProducerImpl());\n\n        lenient().when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());\n\n        when(mQClientFactory.getMQClientAPIImpl().pullMessage(anyString(), any(PullMessageRequestHeader.class),\n            anyLong(), any(CommunicationMode.class), nullable(PullCallback.class)))\n            .thenAnswer(new Answer<Object>() {\n                @Override\n                public Object answer(InvocationOnMock mock) throws Throwable {\n                    PullMessageRequestHeader requestHeader = mock.getArgument(1);\n                    MessageClientExt messageClientExt = new MessageClientExt();\n                    messageClientExt.setTopic(topic);\n                    messageClientExt.setQueueId(0);\n                    messageClientExt.setMsgId(\"123\");\n                    messageClientExt.setBody(new byte[] {'a'});\n                    messageClientExt.setOffsetMsgId(\"234\");\n                    messageClientExt.setBornHost(new InetSocketAddress(8080));\n                    messageClientExt.setStoreHost(new InetSocketAddress(8080));\n                    PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.<MessageExt>singletonList(messageClientExt));\n                    return pullResult;\n                }\n            });\n\n        when(mQClientFactory.findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean())).thenReturn(new FindBrokerResult(\"127.0.0.1:10911\", false));\n\n        doReturn(Collections.singletonList(mQClientFactory.getClientId())).when(mQClientFactory).findConsumerIdList(anyString(), anyString());\n\n        doReturn(123L).when(offsetStore).readOffset(any(MessageQueue.class), any(ReadOffsetType.class));\n\n    }\n\n    private List<MessageExt> pollUntilFound(DefaultLitePullConsumer litePullConsumer) {\n        litePullConsumer.setPollTimeoutMillis(1000);\n        long deadline = System.currentTimeMillis() + 20 * 1000;\n        List<MessageExt> result = Collections.emptyList();\n        while (System.currentTimeMillis() < deadline) {\n            result = litePullConsumer.poll();\n            if (!result.isEmpty()) {\n                return result;\n            }\n        }\n        return result;\n    }\n\n    private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus,\n        List<MessageExt> messageExtList) throws Exception {\n        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();\n        for (MessageExt messageExt : messageExtList) {\n            outputStream.write(MessageDecoder.encode(messageExt, false));\n        }\n        return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray());\n    }\n\n    private MessageQueue createMessageQueue() {\n        MessageQueue messageQueue = new MessageQueue();\n        messageQueue.setBrokerName(brokerName);\n        messageQueue.setQueueId(0);\n        messageQueue.setTopic(topic);\n        return messageQueue;\n    }\n\n    private TopicRouteData createTopicRoute() {\n        TopicRouteData topicRouteData = new TopicRouteData();\n\n        topicRouteData.setFilterServerTable(new HashMap<>());\n        List<BrokerData> brokerDataList = new ArrayList<>();\n        BrokerData brokerData = new BrokerData();\n        brokerData.setBrokerName(\"BrokerA\");\n        brokerData.setCluster(\"DefaultCluster\");\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(0L, \"127.0.0.1:10911\");\n        brokerData.setBrokerAddrs(brokerAddrs);\n        brokerDataList.add(brokerData);\n        topicRouteData.setBrokerDatas(brokerDataList);\n\n        List<QueueData> queueDataList = new ArrayList<>();\n        QueueData queueData = new QueueData();\n        queueData.setBrokerName(\"BrokerA\");\n        queueData.setPerm(6);\n        queueData.setReadQueueNums(3);\n        queueData.setWriteQueueNums(4);\n        queueData.setTopicSysFlag(0);\n        queueDataList.add(queueData);\n        topicRouteData.setQueueDatas(queueDataList);\n        return topicRouteData;\n    }\n\n    private SendResult createSendResult(SendStatus sendStatus) {\n        SendResult sendResult = new SendResult();\n        sendResult.setMsgId(\"123\");\n        sendResult.setOffsetMsgId(\"123\");\n        sendResult.setQueueOffset(456);\n        sendResult.setSendStatus(sendStatus);\n        sendResult.setRegionId(\"HZ\");\n        return sendResult;\n    }\n\n    private static void suppressUpdateTopicRouteInfoFromNameServer(DefaultLitePullConsumer litePullConsumer) throws IllegalAccessException {\n        DefaultLitePullConsumerImpl defaultLitePullConsumerImpl = (DefaultLitePullConsumerImpl) FieldUtils.readDeclaredField(litePullConsumer, \"defaultLitePullConsumerImpl\", true);\n        if (litePullConsumer.getMessageModel() == MessageModel.CLUSTERING) {\n            litePullConsumer.changeInstanceNameToPID();\n        }\n        MQClientInstance mQClientFactory = spy(MQClientManager.getInstance().getOrCreateMQClientInstance(litePullConsumer, (RPCHook) FieldUtils.readDeclaredField(defaultLitePullConsumerImpl, \"rpcHook\", true)));\n        ConcurrentMap<String, MQClientInstance> factoryTable = (ConcurrentMap<String, MQClientInstance>) FieldUtils.readDeclaredField(MQClientManager.getInstance(), \"factoryTable\", true);\n        factoryTable.put(litePullConsumer.buildMQClientId(), mQClientFactory);\n        doReturn(false).when(mQClientFactory).updateTopicRouteInfoFromNameServer(anyString());\n    }\n\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithOpenTracingTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.trace;\n\nimport io.opentracing.mock.MockSpan;\nimport io.opentracing.mock.MockTracer;\nimport io.opentracing.tag.Tags;\nimport java.lang.reflect.Field;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.hook.SendMessageContext;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.MQClientManager;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;\nimport org.apache.rocketmq.client.impl.producer.TopicPublishInfo;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendCallback;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.client.trace.hook.SendMessageOpenTracingHookImpl;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageType;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.nullable;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class DefaultMQProducerWithOpenTracingTest {\n\n    @Spy\n    private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig());\n    @Mock\n    private MQClientAPIImpl mQClientAPIImpl;\n\n    private DefaultMQProducer producer;\n\n    private Message message;\n    private String topic = \"FooBar\";\n    private String producerGroupPrefix = \"FooBar_PID\";\n    private String producerGroupTemp = producerGroupPrefix + System.currentTimeMillis();\n    private String producerGroupTraceTemp = TopicValidator.RMQ_SYS_TRACE_TOPIC + System.currentTimeMillis();\n    private MockTracer tracer = new MockTracer();\n\n    @Before\n    public void init() throws Exception {\n\n        producer = new DefaultMQProducer(producerGroupTemp);\n        producer.getDefaultMQProducerImpl().registerSendMessageHook(\n                new SendMessageOpenTracingHookImpl(tracer));\n        producer.setNamesrvAddr(\"127.0.0.1:9876\");\n        message = new Message(topic, new byte[] {'a', 'b', 'c'});\n        // disable trace to let mock trace work\n        producer.setEnableTrace(false);\n\n        producer.start();\n\n        Field field = DefaultMQProducerImpl.class.getDeclaredField(\"mQClientFactory\");\n        field.setAccessible(true);\n        field.set(producer.getDefaultMQProducerImpl(), mQClientFactory);\n\n        field = MQClientInstance.class.getDeclaredField(\"mQClientAPIImpl\");\n        field.setAccessible(true);\n        field.set(mQClientFactory, mQClientAPIImpl);\n\n        producer.getDefaultMQProducerImpl().getMqClientFactory().registerProducer(producerGroupTemp, producer.getDefaultMQProducerImpl());\n\n        when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class),\n            nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))).thenCallRealMethod();\n        when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class),\n            nullable(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class)))\n            .thenReturn(createSendResult(SendStatus.SEND_OK));\n\n    }\n\n    @Test\n    public void testSendMessageSync_WithTrace_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {\n        producer.getDefaultMQProducerImpl().getMqClientFactory().registerProducer(producerGroupTraceTemp, producer.getDefaultMQProducerImpl());\n        when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());\n        producer.send(message);\n        assertThat(tracer.finishedSpans().size()).isEqualTo(1);\n        MockSpan span = tracer.finishedSpans().get(0);\n        assertThat(span.tags().get(Tags.MESSAGE_BUS_DESTINATION.getKey())).isEqualTo(topic);\n        assertThat(span.tags().get(Tags.SPAN_KIND.getKey())).isEqualTo(Tags.SPAN_KIND_PRODUCER);\n        assertThat(span.tags().get(TraceConstants.ROCKETMQ_MSG_ID)).isEqualTo(\"123\");\n        assertThat(span.tags().get(TraceConstants.ROCKETMQ_BODY_LENGTH)).isEqualTo(3);\n        assertThat(span.tags().get(TraceConstants.ROCKETMQ_REGION_ID)).isEqualTo(\"HZ\");\n        assertThat(span.tags().get(TraceConstants.ROCKETMQ_MSG_TYPE)).isEqualTo(MessageType.Normal_Msg.name());\n        assertThat(span.tags().get(TraceConstants.ROCKETMQ_STORE_HOST)).isEqualTo(\"127.0.0.1:10911\");\n    }\n\n    @After\n    public void terminate() {\n        producer.shutdown();\n    }\n\n    public static TopicRouteData createTopicRoute() {\n        TopicRouteData topicRouteData = new TopicRouteData();\n\n        topicRouteData.setFilterServerTable(new HashMap<>());\n        List<BrokerData> brokerDataList = new ArrayList<>();\n        BrokerData brokerData = new BrokerData();\n        brokerData.setBrokerName(\"BrokerA\");\n        brokerData.setCluster(\"DefaultCluster\");\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(0L, \"127.0.0.1:10911\");\n        brokerData.setBrokerAddrs(brokerAddrs);\n        brokerDataList.add(brokerData);\n        topicRouteData.setBrokerDatas(brokerDataList);\n\n        List<QueueData> queueDataList = new ArrayList<>();\n        QueueData queueData = new QueueData();\n        queueData.setBrokerName(\"BrokerA\");\n        queueData.setPerm(6);\n        queueData.setReadQueueNums(3);\n        queueData.setWriteQueueNums(4);\n        queueData.setTopicSysFlag(0);\n        queueDataList.add(queueData);\n        topicRouteData.setQueueDatas(queueDataList);\n        return topicRouteData;\n    }\n\n    private SendResult createSendResult(SendStatus sendStatus) {\n        SendResult sendResult = new SendResult();\n        sendResult.setMsgId(\"123\");\n        sendResult.setOffsetMsgId(\"123\");\n        sendResult.setQueueOffset(456);\n        sendResult.setSendStatus(sendStatus);\n        sendResult.setRegionId(\"HZ\");\n        return sendResult;\n    }\n\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.trace;\n\nimport java.lang.reflect.Field;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.hook.SendMessageContext;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.MQClientManager;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;\nimport org.apache.rocketmq.client.impl.producer.TopicPublishInfo;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendCallback;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.nullable;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class DefaultMQProducerWithTraceTest {\n\n    @Spy\n    private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig());\n    @Mock\n    private MQClientAPIImpl mQClientAPIImpl;\n\n    private AsyncTraceDispatcher asyncTraceDispatcher;\n\n    private DefaultMQProducer producer;\n    private DefaultMQProducer customTraceTopicproducer;\n    private DefaultMQProducer traceProducer;\n    private DefaultMQProducer normalProducer;\n\n    private Message message;\n    private String topic = \"FooBar\";\n    private String producerGroupPrefix = \"FooBar_PID\";\n    private String producerGroupTemp = producerGroupPrefix + System.currentTimeMillis();\n    private String producerGroupTraceTemp = TopicValidator.RMQ_SYS_TRACE_TOPIC + System.currentTimeMillis();\n    private String customerTraceTopic = \"rmq_trace_topic_12345\";\n\n    @Before\n    public void init() throws Exception {\n\n        customTraceTopicproducer = new DefaultMQProducer(producerGroupTemp, false, customerTraceTopic);\n        normalProducer = new DefaultMQProducer(producerGroupTemp, false, \"\");\n        producer = new DefaultMQProducer(producerGroupTemp, true, \"\");\n        producer.setNamesrvAddr(\"127.0.0.1:9876\");\n        normalProducer.setNamesrvAddr(\"127.0.0.1:9877\");\n        customTraceTopicproducer.setNamesrvAddr(\"127.0.0.1:9878\");\n        message = new Message(topic, new byte[] {'a', 'b', 'c'});\n        producer.setTraceTopic(customerTraceTopic);\n        producer.setUseTLS(true);\n\n        producer.start();\n\n        asyncTraceDispatcher = (AsyncTraceDispatcher) producer.getTraceDispatcher();\n        traceProducer = asyncTraceDispatcher.getTraceProducer();\n\n        Field field = DefaultMQProducerImpl.class.getDeclaredField(\"mQClientFactory\");\n        field.setAccessible(true);\n        field.set(producer.getDefaultMQProducerImpl(), mQClientFactory);\n\n        Field fieldTrace = DefaultMQProducerImpl.class.getDeclaredField(\"mQClientFactory\");\n        fieldTrace.setAccessible(true);\n        fieldTrace.set(traceProducer.getDefaultMQProducerImpl(), mQClientFactory);\n\n        field = MQClientInstance.class.getDeclaredField(\"mQClientAPIImpl\");\n        field.setAccessible(true);\n        field.set(mQClientFactory, mQClientAPIImpl);\n\n        producer.getDefaultMQProducerImpl().getMqClientFactory().registerProducer(producerGroupTemp, producer.getDefaultMQProducerImpl());\n\n        when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class),\n            nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))).thenCallRealMethod();\n        when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class),\n            nullable(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class)))\n            .thenReturn(createSendResult(SendStatus.SEND_OK));\n\n    }\n\n    @Test\n    public void testSendMessageSync_WithTrace_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {\n        traceProducer.getDefaultMQProducerImpl().getMqClientFactory().registerProducer(producerGroupTraceTemp, traceProducer.getDefaultMQProducerImpl());\n        when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());\n        final CountDownLatch countDownLatch = new CountDownLatch(1);\n        try {\n            producer.send(message);\n        } catch (MQClientException e) {\n        }\n        countDownLatch.await(3000L, TimeUnit.MILLISECONDS);\n\n    }\n\n    @Test\n    public void testSendMessageSync_WithTrace_NoBrokerSet_Exception() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {\n        when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());\n        final CountDownLatch countDownLatch = new CountDownLatch(1);\n        try {\n            producer.send(message);\n        } catch (MQClientException e) {\n        }\n        countDownLatch.await(3000L, TimeUnit.MILLISECONDS);\n\n    }\n\n\n    @Test\n    public void testProducerWithTraceTLS() {\n        Assert.assertTrue(asyncTraceDispatcher.getTraceProducer().isUseTLS());\n    }\n\n    @After\n    public void terminate() {\n        producer.shutdown();\n    }\n\n    public static TopicRouteData createTopicRoute() {\n        TopicRouteData topicRouteData = new TopicRouteData();\n\n        topicRouteData.setFilterServerTable(new HashMap<>());\n        List<BrokerData> brokerDataList = new ArrayList<>();\n        BrokerData brokerData = new BrokerData();\n        brokerData.setBrokerName(\"BrokerA\");\n        brokerData.setCluster(\"DefaultCluster\");\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(0L, \"127.0.0.1:10911\");\n        brokerData.setBrokerAddrs(brokerAddrs);\n        brokerDataList.add(brokerData);\n        topicRouteData.setBrokerDatas(brokerDataList);\n\n        List<QueueData> queueDataList = new ArrayList<>();\n        QueueData queueData = new QueueData();\n        queueData.setBrokerName(\"BrokerA\");\n        queueData.setPerm(6);\n        queueData.setReadQueueNums(3);\n        queueData.setWriteQueueNums(4);\n        queueData.setTopicSysFlag(0);\n        queueDataList.add(queueData);\n        topicRouteData.setQueueDatas(queueDataList);\n        return topicRouteData;\n    }\n\n    private SendResult createSendResult(SendStatus sendStatus) {\n        SendResult sendResult = new SendResult();\n        sendResult.setMsgId(\"123\");\n        sendResult.setOffsetMsgId(\"123\");\n        sendResult.setQueueOffset(456);\n        sendResult.setSendStatus(sendStatus);\n        sendResult.setRegionId(\"HZ\");\n        return sendResult;\n    }\n\n    public static TopicRouteData createTraceTopicRoute() {\n        TopicRouteData topicRouteData = new TopicRouteData();\n\n        topicRouteData.setFilterServerTable(new HashMap<>());\n        List<BrokerData> brokerDataList = new ArrayList<>();\n        BrokerData brokerData = new BrokerData();\n        brokerData.setBrokerName(\"broker-trace\");\n        brokerData.setCluster(\"DefaultCluster\");\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(0L, \"127.0.0.1:10912\");\n        brokerData.setBrokerAddrs(brokerAddrs);\n        brokerDataList.add(brokerData);\n        topicRouteData.setBrokerDatas(brokerDataList);\n\n        List<QueueData> queueDataList = new ArrayList<>();\n        QueueData queueData = new QueueData();\n        queueData.setBrokerName(\"broker-trace\");\n        queueData.setPerm(6);\n        queueData.setReadQueueNums(1);\n        queueData.setWriteQueueNums(1);\n        queueData.setTopicSysFlag(1);\n        queueDataList.add(queueData);\n        topicRouteData.setQueueDatas(queueDataList);\n        return topicRouteData;\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/trace/TraceDataEncoderTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.trace;\n\nimport org.apache.rocketmq.client.AccessChannel;\nimport org.apache.rocketmq.client.producer.LocalTransactionState;\nimport org.apache.rocketmq.common.message.MessageType;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Set;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class TraceDataEncoderTest {\n\n    private String traceData;\n\n    private long time;\n\n    @Before\n    public void init() {\n        time = System.currentTimeMillis();\n        traceData = new StringBuilder()\n            .append(\"Pub\").append(TraceConstants.CONTENT_SPLITOR)\n            .append(time).append(TraceConstants.CONTENT_SPLITOR)\n            .append(\"DefaultRegion\").append(TraceConstants.CONTENT_SPLITOR)\n            .append(\"PID-test\").append(TraceConstants.CONTENT_SPLITOR)\n            .append(\"topic-test\").append(TraceConstants.CONTENT_SPLITOR)\n            .append(\"AC1415116D1418B4AAC217FE1B4E0000\").append(TraceConstants.CONTENT_SPLITOR)\n            .append(\"Tags\").append(TraceConstants.CONTENT_SPLITOR)\n            .append(\"Keys\").append(TraceConstants.CONTENT_SPLITOR)\n            .append(\"127.0.0.1:10911\").append(TraceConstants.CONTENT_SPLITOR)\n            .append(26).append(TraceConstants.CONTENT_SPLITOR)\n            .append(245).append(TraceConstants.CONTENT_SPLITOR)\n            .append(MessageType.Normal_Msg.ordinal()).append(TraceConstants.CONTENT_SPLITOR)\n            .append(\"0A9A002600002A9F0000000000002329\").append(TraceConstants.CONTENT_SPLITOR)\n            .append(true).append(TraceConstants.FIELD_SPLITOR)\n            .toString();\n    }\n\n    @Test\n    public void testDecoderFromTraceDataString() {\n        List<TraceContext> contexts = TraceDataEncoder.decoderFromTraceDataString(traceData);\n        Assert.assertEquals(contexts.size(), 1);\n        Assert.assertEquals(contexts.get(0).getTraceType(), TraceType.Pub);\n    }\n\n    @Test\n    public void testEncoderFromContextBean() {\n        TraceContext context = new TraceContext();\n        context.setTraceType(TraceType.Pub);\n        context.setGroupName(\"PID-test\");\n        context.setRegionId(\"DefaultRegion\");\n        context.setCostTime(245);\n        context.setSuccess(true);\n        context.setTimeStamp(time);\n        TraceBean traceBean = new TraceBean();\n        traceBean.setTopic(\"topic-test\");\n        traceBean.setKeys(\"Keys\");\n        traceBean.setTags(\"Tags\");\n        traceBean.setMsgId(\"AC1415116D1418B4AAC217FE1B4E0000\");\n        traceBean.setOffsetMsgId(\"0A9A002600002A9F0000000000002329\");\n        traceBean.setStoreHost(\"127.0.0.1:10911\");\n        traceBean.setStoreTime(time);\n        traceBean.setMsgType(MessageType.Normal_Msg);\n        traceBean.setBodyLength(26);\n        List<TraceBean> traceBeans = new ArrayList<>();\n        traceBeans.add(traceBean);\n        context.setTraceBeans(traceBeans);\n        TraceTransferBean traceTransferBean = TraceDataEncoder.encoderFromContextBean(context);\n\n        Assert.assertEquals(traceTransferBean.getTransData(), traceData);\n        Assert.assertEquals(traceTransferBean.getTransKey().size(), 2);\n    }\n\n    @Test\n    public void testEncoderFromContextBean_EndTransaction() {\n        TraceContext context = new TraceContext();\n        context.setTraceType(TraceType.EndTransaction);\n        context.setGroupName(\"PID-test\");\n        context.setRegionId(\"DefaultRegion\");\n        context.setTimeStamp(time);\n        TraceBean traceBean = new TraceBean();\n        traceBean.setTopic(\"topic-test\");\n        traceBean.setKeys(\"Keys\");\n        traceBean.setTags(\"Tags\");\n        traceBean.setMsgId(\"AC1415116D1418B4AAC217FE1B4E0000\");\n        traceBean.setStoreHost(\"127.0.0.1:10911\");\n        traceBean.setMsgType(MessageType.Trans_msg_Commit);\n        traceBean.setTransactionId(\"transactionId\");\n        traceBean.setTransactionState(LocalTransactionState.COMMIT_MESSAGE);\n        traceBean.setFromTransactionCheck(false);\n        List<TraceBean> traceBeans = new ArrayList<>();\n        traceBeans.add(traceBean);\n        context.setTraceBeans(traceBeans);\n        TraceTransferBean traceTransferBean = TraceDataEncoder.encoderFromContextBean(context);\n\n        Assert.assertEquals(traceTransferBean.getTransKey().size(), 2);\n        String traceData = traceTransferBean.getTransData();\n        TraceContext contextAfter = TraceDataEncoder.decoderFromTraceDataString(traceData).get(0);\n        Assert.assertEquals(context.getTraceType(), contextAfter.getTraceType());\n        Assert.assertEquals(context.getTimeStamp(), contextAfter.getTimeStamp());\n        Assert.assertEquals(context.getGroupName(), contextAfter.getGroupName());\n        TraceBean before = context.getTraceBeans().get(0);\n        TraceBean after = contextAfter.getTraceBeans().get(0);\n        Assert.assertEquals(before.getTopic(), after.getTopic());\n        Assert.assertEquals(before.getMsgId(), after.getMsgId());\n        Assert.assertEquals(before.getTags(), after.getTags());\n        Assert.assertEquals(before.getKeys(), after.getKeys());\n        Assert.assertEquals(before.getStoreHost(), after.getStoreHost());\n        Assert.assertEquals(before.getMsgType(), after.getMsgType());\n        Assert.assertEquals(before.getClientHost(), after.getClientHost());\n        Assert.assertEquals(before.getTransactionId(), after.getTransactionId());\n        Assert.assertEquals(before.getTransactionState(), after.getTransactionState());\n        Assert.assertEquals(before.isFromTransactionCheck(), after.isFromTransactionCheck());\n    }\n\n    @Test\n    public void testPubTraceDataFormatTest() {\n        TraceContext pubContext = new TraceContext();\n        pubContext.setTraceType(TraceType.Pub);\n        pubContext.setTimeStamp(time);\n        pubContext.setRegionId(\"Default-region\");\n        pubContext.setGroupName(\"GroupName-test\");\n        pubContext.setCostTime(34);\n        pubContext.setSuccess(true);\n        TraceBean bean = new TraceBean();\n        bean.setTopic(\"topic-test\");\n        bean.setMsgId(\"AC1415116D1418B4AAC217FE1B4E0000\");\n        bean.setTags(\"tags\");\n        bean.setKeys(\"keys\");\n        bean.setStoreHost(\"127.0.0.1:10911\");\n        bean.setBodyLength(100);\n        bean.setMsgType(MessageType.Normal_Msg);\n        bean.setOffsetMsgId(\"AC1415116D1418B4AAC217FE1B4E0000\");\n        pubContext.setTraceBeans(new ArrayList<>(1));\n        pubContext.getTraceBeans().add(bean);\n\n        TraceTransferBean traceTransferBean = TraceDataEncoder.encoderFromContextBean(pubContext);\n        String transData = traceTransferBean.getTransData();\n        Assert.assertNotNull(transData);\n        String[] items = transData.split(String.valueOf(TraceConstants.CONTENT_SPLITOR));\n        Assert.assertEquals(14, items.length);\n\n    }\n\n    @Test\n    public void testSubBeforeTraceDataFormatTest() {\n        TraceContext subBeforeContext = new TraceContext();\n        subBeforeContext.setTraceType(TraceType.SubBefore);\n        subBeforeContext.setTimeStamp(time);\n        subBeforeContext.setRegionId(\"Default-region\");\n        subBeforeContext.setGroupName(\"GroupName-test\");\n        subBeforeContext.setRequestId(\"3455848576927\");\n        TraceBean bean = new TraceBean();\n        bean.setMsgId(\"AC1415116D1418B4AAC217FE1B4E0000\");\n        bean.setRetryTimes(0);\n        bean.setKeys(\"keys\");\n        subBeforeContext.setTraceBeans(new ArrayList<>(1));\n        subBeforeContext.getTraceBeans().add(bean);\n\n        TraceTransferBean traceTransferBean = TraceDataEncoder.encoderFromContextBean(subBeforeContext);\n        String transData = traceTransferBean.getTransData();\n        Assert.assertNotNull(transData);\n        String[] items = transData.split(String.valueOf(TraceConstants.CONTENT_SPLITOR));\n        Assert.assertEquals(8, items.length);\n\n    }\n\n    @Test\n    public void testSubAfterTraceDataFormatTest() {\n        TraceContext subAfterContext = new TraceContext();\n        subAfterContext.setTraceType(TraceType.SubAfter);\n        subAfterContext.setRequestId(\"3455848576927\");\n        subAfterContext.setCostTime(20);\n        subAfterContext.setSuccess(true);\n        subAfterContext.setTimeStamp(1625883640000L);\n        subAfterContext.setGroupName(\"GroupName-test\");\n        subAfterContext.setContextCode(98623046);\n        subAfterContext.setAccessChannel(AccessChannel.LOCAL);\n        TraceBean bean = new TraceBean();\n        bean.setMsgId(\"AC1415116D1418B4AAC217FE1B4E0000\");\n        bean.setKeys(\"keys\");\n        subAfterContext.setTraceBeans(new ArrayList<>(1));\n        subAfterContext.getTraceBeans().add(bean);\n\n        TraceTransferBean traceTransferBean = TraceDataEncoder.encoderFromContextBean(subAfterContext);\n        String transData = traceTransferBean.getTransData();\n        Assert.assertNotNull(transData);\n        String[] items = transData.split(String.valueOf(TraceConstants.CONTENT_SPLITOR));\n        Assert.assertEquals(9, items.length);\n\n    }\n\n    @Test\n    public void testEndTrxTraceDataFormatTest() {\n        TraceContext endTrxContext = new TraceContext();\n        endTrxContext.setTraceType(TraceType.EndTransaction);\n        endTrxContext.setGroupName(\"PID-test\");\n        endTrxContext.setRegionId(\"DefaultRegion\");\n        endTrxContext.setTimeStamp(time);\n        TraceBean endTrxTraceBean = new TraceBean();\n        endTrxTraceBean.setTopic(\"topic-test\");\n        endTrxTraceBean.setKeys(\"Keys\");\n        endTrxTraceBean.setTags(\"Tags\");\n        endTrxTraceBean.setMsgId(\"AC1415116D1418B4AAC217FE1B4E0000\");\n        endTrxTraceBean.setStoreHost(\"127.0.0.1:10911\");\n        endTrxTraceBean.setMsgType(MessageType.Trans_msg_Commit);\n        endTrxTraceBean.setTransactionId(\"transactionId\");\n        endTrxTraceBean.setTransactionState(LocalTransactionState.COMMIT_MESSAGE);\n        endTrxTraceBean.setFromTransactionCheck(false);\n        List<TraceBean> traceBeans = new ArrayList<>();\n        traceBeans.add(endTrxTraceBean);\n        endTrxContext.setTraceBeans(traceBeans);\n\n        TraceTransferBean traceTransferBean = TraceDataEncoder.encoderFromContextBean(endTrxContext);\n        String transData = traceTransferBean.getTransData();\n        Assert.assertNotNull(transData);\n        String[] items = transData.split(String.valueOf(TraceConstants.CONTENT_SPLITOR));\n        Assert.assertEquals(13, items.length);\n\n    }\n\n    @Test\n    public void testTraceKeys() {\n        TraceContext endTrxContext = new TraceContext();\n        endTrxContext.setTraceType(TraceType.EndTransaction);\n        endTrxContext.setGroupName(\"PID-test\");\n        endTrxContext.setRegionId(\"DefaultRegion\");\n        endTrxContext.setTimeStamp(time);\n        TraceBean endTrxTraceBean = new TraceBean();\n        endTrxTraceBean.setTopic(\"topic-test\");\n        endTrxTraceBean.setKeys(\"Keys Keys2\");\n        endTrxTraceBean.setTags(\"Tags\");\n        endTrxTraceBean.setMsgId(\"AC1415116D1418B4AAC217FE1B4E0000\");\n        endTrxTraceBean.setStoreHost(\"127.0.0.1:10911\");\n        endTrxTraceBean.setMsgType(MessageType.Trans_msg_Commit);\n        endTrxTraceBean.setTransactionId(\"transactionId\");\n        endTrxTraceBean.setTransactionState(LocalTransactionState.COMMIT_MESSAGE);\n        endTrxTraceBean.setFromTransactionCheck(false);\n        List<TraceBean> traceBeans = new ArrayList<>();\n        traceBeans.add(endTrxTraceBean);\n        endTrxContext.setTraceBeans(traceBeans);\n\n        TraceTransferBean traceTransferBean = TraceDataEncoder.encoderFromContextBean(endTrxContext);\n\n        Set<String> keys = traceTransferBean.getTransKey();\n        assertThat(keys).contains(\"Keys\");\n        assertThat(keys).contains(\"Keys2\");\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/trace/TraceViewTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.trace;\n\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageType;\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.List;\n\npublic class TraceViewTest {\n\n    @Test\n    public void testDecodeFromTraceTransData() {\n        String messageBody = new StringBuilder()\n            .append(\"Pub\").append(TraceConstants.CONTENT_SPLITOR)\n            .append(System.currentTimeMillis()).append(TraceConstants.CONTENT_SPLITOR)\n            .append(\"DefaultRegion\").append(TraceConstants.CONTENT_SPLITOR)\n            .append(\"PID-test\").append(TraceConstants.CONTENT_SPLITOR)\n            .append(\"topic-test\").append(TraceConstants.CONTENT_SPLITOR)\n            .append(\"AC1415116D1418B4AAC217FE1B4E0000\").append(TraceConstants.CONTENT_SPLITOR)\n            .append(\"Tags\").append(TraceConstants.CONTENT_SPLITOR)\n            .append(\"Keys\").append(TraceConstants.CONTENT_SPLITOR)\n            .append(\"127.0.0.1:10911\").append(TraceConstants.CONTENT_SPLITOR)\n            .append(26).append(TraceConstants.CONTENT_SPLITOR)\n            .append(245).append(TraceConstants.CONTENT_SPLITOR)\n            .append(MessageType.Normal_Msg.ordinal()).append(TraceConstants.CONTENT_SPLITOR)\n            .append(\"0A9A002600002A9F0000000000002329\").append(TraceConstants.CONTENT_SPLITOR)\n            .append(true).append(TraceConstants.FIELD_SPLITOR)\n            .toString();\n        MessageExt message = new MessageExt();\n        message.setBody(messageBody.getBytes(StandardCharsets.UTF_8));\n        String key = \"AC1415116D1418B4AAC217FE1B4E0000\";\n        List<TraceView> traceViews = TraceView.decodeFromTraceTransData(key, message);\n        Assert.assertEquals(traceViews.size(), 1);\n        Assert.assertEquals(traceViews.get(0).getMsgId(), key);\n\n        key = \"AD4233434334AAC217FEFFD0000\";\n        traceViews = TraceView.decodeFromTraceTransData(key, message);\n        Assert.assertEquals(traceViews.size(), 0);\n    }\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithOpenTracingTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.trace;\n\nimport io.opentracing.mock.MockSpan;\nimport io.opentracing.mock.MockTracer;\nimport io.opentracing.tag.Tags;\nimport java.lang.reflect.Field;\nimport java.net.InetSocketAddress;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.hook.SendMessageContext;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.MQClientManager;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;\nimport org.apache.rocketmq.client.impl.producer.TopicPublishInfo;\nimport org.apache.rocketmq.client.producer.LocalTransactionState;\nimport org.apache.rocketmq.client.producer.SendCallback;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.client.producer.TransactionListener;\nimport org.apache.rocketmq.client.producer.TransactionMQProducer;\nimport org.apache.rocketmq.client.trace.hook.EndTransactionOpenTracingHookImpl;\nimport org.apache.rocketmq.client.trace.hook.SendMessageOpenTracingHookImpl;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.message.MessageType;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.nullable;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class TransactionMQProducerWithOpenTracingTest {\n\n    @Spy\n    private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig());\n    @Mock\n    private MQClientAPIImpl mQClientAPIImpl;\n\n    private TransactionMQProducer producer;\n\n    private Message message;\n    private String topic = \"FooBar\";\n    private String producerGroupPrefix = \"FooBar_PID\";\n    private String producerGroupTemp = producerGroupPrefix + System.currentTimeMillis();\n    private String producerGroupTraceTemp = TopicValidator.RMQ_SYS_TRACE_TOPIC + System.currentTimeMillis();\n    private MockTracer tracer = new MockTracer();\n    @Before\n    public void init() throws Exception {\n        TransactionListener transactionListener = new TransactionListener() {\n            @Override\n            public LocalTransactionState executeLocalTransaction(Message msg, Object arg) {\n                return LocalTransactionState.COMMIT_MESSAGE;\n            }\n\n            @Override\n            public LocalTransactionState checkLocalTransaction(MessageExt msg) {\n                return LocalTransactionState.COMMIT_MESSAGE;\n            }\n        };\n        producer = new TransactionMQProducer(producerGroupTemp);\n        producer.getDefaultMQProducerImpl().registerSendMessageHook(new SendMessageOpenTracingHookImpl(tracer));\n        producer.getDefaultMQProducerImpl().registerEndTransactionHook(new EndTransactionOpenTracingHookImpl(tracer));\n        producer.setTransactionListener(transactionListener);\n        // disable trace to let mock trace work\n        producer.setEnableTrace(false);\n\n        producer.setNamesrvAddr(\"127.0.0.1:9876\");\n        message = new Message(topic, new byte[] {'a', 'b', 'c'});\n\n        producer.start();\n\n        Field field = DefaultMQProducerImpl.class.getDeclaredField(\"mQClientFactory\");\n        field.setAccessible(true);\n        field.set(producer.getDefaultMQProducerImpl(), mQClientFactory);\n\n        field = MQClientInstance.class.getDeclaredField(\"mQClientAPIImpl\");\n        field.setAccessible(true);\n        field.set(mQClientFactory, mQClientAPIImpl);\n\n        producer.getDefaultMQProducerImpl().getMqClientFactory().registerProducer(producerGroupTemp, producer.getDefaultMQProducerImpl());\n\n        when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class),\n            nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))).thenCallRealMethod();\n        when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class),\n            nullable(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class)))\n            .thenReturn(createSendResult(SendStatus.SEND_OK));\n\n    }\n\n    @Test\n    public void testSendMessageSync_WithTrace_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {\n        producer.getDefaultMQProducerImpl().getMqClientFactory().registerProducer(producerGroupTraceTemp, producer.getDefaultMQProducerImpl());\n        when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());\n        producer.sendMessageInTransaction(message, null);\n\n        assertThat(tracer.finishedSpans().size()).isEqualTo(2);\n        MockSpan span = tracer.finishedSpans().get(1);\n        assertThat(span.tags().get(Tags.MESSAGE_BUS_DESTINATION.getKey())).isEqualTo(topic);\n        assertThat(span.tags().get(Tags.SPAN_KIND.getKey())).isEqualTo(Tags.SPAN_KIND_PRODUCER);\n        assertThat(span.tags().get(TraceConstants.ROCKETMQ_MSG_ID)).isEqualTo(\"123\");\n        assertThat(span.tags().get(TraceConstants.ROCKETMQ_MSG_TYPE)).isEqualTo(MessageType.Trans_msg_Commit.name());\n        assertThat(span.tags().get(TraceConstants.ROCKETMQ_TRANSACTION_STATE)).isEqualTo(LocalTransactionState.COMMIT_MESSAGE.name());\n        assertThat(span.tags().get(TraceConstants.ROCKETMQ_IS_FROM_TRANSACTION_CHECK)).isEqualTo(false);\n    }\n\n    @After\n    public void terminate() {\n        producer.shutdown();\n    }\n\n    public static TopicRouteData createTopicRoute() {\n        TopicRouteData topicRouteData = new TopicRouteData();\n\n        topicRouteData.setFilterServerTable(new HashMap<>());\n        List<BrokerData> brokerDataList = new ArrayList<>();\n        BrokerData brokerData = new BrokerData();\n        brokerData.setBrokerName(\"BrokerA\");\n        brokerData.setCluster(\"DefaultCluster\");\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(0L, \"127.0.0.1:10911\");\n        brokerData.setBrokerAddrs(brokerAddrs);\n        brokerDataList.add(brokerData);\n        topicRouteData.setBrokerDatas(brokerDataList);\n\n        List<QueueData> queueDataList = new ArrayList<>();\n        QueueData queueData = new QueueData();\n        queueData.setBrokerName(\"BrokerA\");\n        queueData.setPerm(6);\n        queueData.setReadQueueNums(3);\n        queueData.setWriteQueueNums(4);\n        queueData.setTopicSysFlag(0);\n        queueDataList.add(queueData);\n        topicRouteData.setQueueDatas(queueDataList);\n        return topicRouteData;\n    }\n\n    private SendResult createSendResult(SendStatus sendStatus) {\n        SendResult sendResult = new SendResult();\n        sendResult.setMsgId(\"123\");\n        sendResult.setOffsetMsgId(MessageDecoder.createMessageId(new InetSocketAddress(\"127.0.0.1\", 12), 1));\n        sendResult.setQueueOffset(456);\n        sendResult.setSendStatus(sendStatus);\n        sendResult.setRegionId(\"HZ\");\n        sendResult.setMessageQueue(new MessageQueue(topic, \"broker-trace\", 0));\n        return sendResult;\n    }\n\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithTraceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.trace;\n\nimport java.lang.reflect.Field;\nimport java.net.InetSocketAddress;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.concurrent.atomic.AtomicReference;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.hook.EndTransactionContext;\nimport org.apache.rocketmq.client.hook.EndTransactionHook;\nimport org.apache.rocketmq.client.hook.SendMessageContext;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.MQClientManager;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;\nimport org.apache.rocketmq.client.impl.producer.TopicPublishInfo;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.LocalTransactionState;\nimport org.apache.rocketmq.client.producer.SendCallback;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.client.producer.TransactionListener;\nimport org.apache.rocketmq.client.producer.TransactionMQProducer;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.invocation.InvocationOnMock;\nimport org.mockito.junit.MockitoJUnitRunner;\nimport org.mockito.stubbing.Answer;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.nullable;\nimport static org.mockito.Mockito.doAnswer;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class TransactionMQProducerWithTraceTest {\n\n    @Spy\n    private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig());\n    @Mock\n    private MQClientAPIImpl mQClientAPIImpl;\n    @Mock\n    private EndTransactionHook endTransactionHook;\n\n    private AsyncTraceDispatcher asyncTraceDispatcher;\n\n    private TransactionMQProducer producer;\n    private DefaultMQProducer traceProducer;\n\n    private Message message;\n    private String topic = \"FooBar\";\n    private String producerGroupPrefix = \"FooBar_PID\";\n    private String producerGroupTemp = producerGroupPrefix + System.currentTimeMillis();\n    private String producerGroupTraceTemp = TopicValidator.RMQ_SYS_TRACE_TOPIC + System.currentTimeMillis();\n    private String customerTraceTopic = \"rmq_trace_topic_12345\";\n\n    @Before\n    public void init() throws Exception {\n        TransactionListener transactionListener = new TransactionListener() {\n            @Override\n            public LocalTransactionState executeLocalTransaction(Message msg, Object arg) {\n                return LocalTransactionState.COMMIT_MESSAGE;\n            }\n\n            @Override\n            public LocalTransactionState checkLocalTransaction(MessageExt msg) {\n                return LocalTransactionState.COMMIT_MESSAGE;\n            }\n        };\n        producer = new TransactionMQProducer(producerGroupTemp, null, true, null);\n        producer.setTransactionListener(transactionListener);\n\n        producer.setNamesrvAddr(\"127.0.0.1:9876\");\n        message = new Message(topic, new byte[] {'a', 'b', 'c'});\n\n        producer.start();\n\n        asyncTraceDispatcher = (AsyncTraceDispatcher) producer.getTraceDispatcher();\n        traceProducer = asyncTraceDispatcher.getTraceProducer();\n\n        Field field = DefaultMQProducerImpl.class.getDeclaredField(\"mQClientFactory\");\n        field.setAccessible(true);\n        field.set(producer.getDefaultMQProducerImpl(), mQClientFactory);\n\n        Field fieldTrace = DefaultMQProducerImpl.class.getDeclaredField(\"mQClientFactory\");\n        fieldTrace.setAccessible(true);\n        fieldTrace.set(traceProducer.getDefaultMQProducerImpl(), mQClientFactory);\n\n        field = MQClientInstance.class.getDeclaredField(\"mQClientAPIImpl\");\n        field.setAccessible(true);\n        field.set(mQClientFactory, mQClientAPIImpl);\n\n        producer.getDefaultMQProducerImpl().getMqClientFactory().registerProducer(producerGroupTemp, producer.getDefaultMQProducerImpl());\n\n        Field fieldHooks = DefaultMQProducerImpl.class.getDeclaredField(\"endTransactionHookList\");\n        fieldHooks.setAccessible(true);\n        List<EndTransactionHook> hooks = new ArrayList<>();\n        hooks.add(endTransactionHook);\n        fieldHooks.set(producer.getDefaultMQProducerImpl(), hooks);\n\n        when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class),\n            nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))).thenCallRealMethod();\n        when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class),\n            nullable(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class)))\n            .thenReturn(createSendResult(SendStatus.SEND_OK));\n\n    }\n\n    @Test\n    public void testSendMessageSync_WithTrace_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {\n        traceProducer.getDefaultMQProducerImpl().getMqClientFactory().registerProducer(producerGroupTraceTemp, traceProducer.getDefaultMQProducerImpl());\n        when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());\n        final AtomicReference<EndTransactionContext> context = new AtomicReference<>();\n        doAnswer(new Answer() {\n            @Override\n            public Object answer(InvocationOnMock mock) throws Throwable {\n                context.set((EndTransactionContext) mock.getArgument(0));\n                return null;\n            }\n\n        }).when(endTransactionHook).endTransaction(any(EndTransactionContext.class));\n        producer.sendMessageInTransaction(message, null);\n\n        EndTransactionContext ctx = context.get();\n        assertThat(ctx.getProducerGroup()).isEqualTo(producerGroupTemp);\n        assertThat(ctx.getMsgId()).isEqualTo(\"123\");\n        assertThat(ctx.isFromTransactionCheck()).isFalse();\n        assertThat(new String(ctx.getMessage().getBody())).isEqualTo(new String(message.getBody()));\n        assertThat(ctx.getMessage().getTopic()).isEqualTo(topic);\n    }\n\n    @Test(expected = MQClientException.class)\n    public void testSendMessageInTransaction_NoListener_ThrowsException() throws MQClientException {\n        producer.setTransactionListener(null);\n        producer.sendMessageInTransaction(message, null);\n    }\n\n    @Test(expected = MQClientException.class)\n    public void testSendMessageInTransaction_DelayMsg_ThrowsException() throws MQClientException {\n        message.setDelayTimeLevel(3);\n        producer.sendMessageInTransaction(message, null);\n    }\n\n    @After\n    public void terminate() {\n        producer.shutdown();\n    }\n\n    public static TopicRouteData createTopicRoute() {\n        TopicRouteData topicRouteData = new TopicRouteData();\n\n        topicRouteData.setFilterServerTable(new HashMap<>());\n        List<BrokerData> brokerDataList = new ArrayList<>();\n        BrokerData brokerData = new BrokerData();\n        brokerData.setBrokerName(\"BrokerA\");\n        brokerData.setCluster(\"DefaultCluster\");\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(0L, \"127.0.0.1:10911\");\n        brokerData.setBrokerAddrs(brokerAddrs);\n        brokerDataList.add(brokerData);\n        topicRouteData.setBrokerDatas(brokerDataList);\n\n        List<QueueData> queueDataList = new ArrayList<>();\n        QueueData queueData = new QueueData();\n        queueData.setBrokerName(\"BrokerA\");\n        queueData.setPerm(6);\n        queueData.setReadQueueNums(3);\n        queueData.setWriteQueueNums(4);\n        queueData.setTopicSysFlag(0);\n        queueDataList.add(queueData);\n        topicRouteData.setQueueDatas(queueDataList);\n        return topicRouteData;\n    }\n\n    private SendResult createSendResult(SendStatus sendStatus) {\n        SendResult sendResult = new SendResult();\n        sendResult.setMsgId(\"123\");\n        sendResult.setOffsetMsgId(MessageDecoder.createMessageId(new InetSocketAddress(\"127.0.0.1\", 12), 1));\n        sendResult.setQueueOffset(456);\n        sendResult.setSendStatus(sendStatus);\n        sendResult.setRegionId(\"HZ\");\n        sendResult.setMessageQueue(new MessageQueue(topic, \"broker-trace\", 0));\n        return sendResult;\n    }\n\n}\n"
  },
  {
    "path": "client/src/test/java/org/apache/rocketmq/client/utils/MessageUtilsTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.client.utils;\n\nimport java.util.HashMap;\nimport java.util.Map;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown;\n\npublic class MessageUtilsTest {\n\n    @Test\n    public void testCreateReplyMessage() throws MQClientException {\n        Message msg = MessageUtil.createReplyMessage(createReplyMessage(\"clusterName\"), new byte[] {'a'});\n        assertThat(msg.getTopic()).isEqualTo(\"clusterName\" + \"_\" + MixAll.REPLY_TOPIC_POSTFIX);\n        assertThat(msg.getProperty(MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT)).isEqualTo(\"127.0.0.1\");\n        assertThat(msg.getProperty(MessageConst.PROPERTY_MESSAGE_TTL)).isEqualTo(\"3000\");\n    }\n\n    @Test\n    public void testCreateReplyMessage_Exception() throws MQClientException {\n        try {\n            Message msg = MessageUtil.createReplyMessage(createReplyMessage(null), new byte[] {'a'});\n            failBecauseExceptionWasNotThrown(MQClientException.class);\n        } catch (MQClientException e) {\n            assertThat(e).hasMessageContaining(\"create reply message fail, requestMessage error, property[\" + MessageConst.PROPERTY_CLUSTER + \"] is null.\");\n        }\n    }\n\n    @Test\n    public void testCreateReplyMessage_reqMsgIsNull() throws MQClientException {\n        try {\n            Message msg = MessageUtil.createReplyMessage(null, new byte[] {'a'});\n            failBecauseExceptionWasNotThrown(MQClientException.class);\n        } catch (MQClientException e) {\n            assertThat(e).hasMessageContaining(\"create reply message fail, requestMessage cannot be null.\");\n        }\n    }\n\n    @Test\n    public void testGetReplyToClient() throws MQClientException {\n        Message msg = createReplyMessage(\"clusterName\");\n        String replyToClient = MessageUtil.getReplyToClient(msg);\n        assertThat(replyToClient).isNotNull();\n        assertThat(replyToClient).isEqualTo(\"127.0.0.1\");\n    }\n\n    private Message createReplyMessage(String clusterName) {\n        Message requestMessage = new Message();\n        Map map = new HashMap<String, String>();\n        map.put(MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT, \"127.0.0.1\");\n        map.put(MessageConst.PROPERTY_CLUSTER, clusterName);\n        map.put(MessageConst.PROPERTY_MESSAGE_TTL, \"3000\");\n        MessageAccessor.setProperties(requestMessage, map);\n        return requestMessage;\n    }\n\n}\n"
  },
  {
    "path": "client/src/test/resources/acl_hook/plain_acl.yml",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\n## suggested format\n\n- accessKey: rocketmq2\n  secretKey: 12345678\n  whiteRemoteAddress: 192.168.1.*\n  admin: true\n"
  },
  {
    "path": "client/src/test/resources/conf/plain_acl_incomplete.yml",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\n## suggested format\n\n- accessKey: rocketmq2\n  secretKey:\n  whiteRemoteAddress: 192.168.1.*\n  # if it is admin, it could access all resources\n  admin: true"
  },
  {
    "path": "client/src/test/resources/org/powermock/extensions/configuration.properties",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\npowermock.global-ignore=javax.management.*"
  },
  {
    "path": "client/src/test/resources/rmq.logback-test.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n<configuration>\n\n    <appender name=\"CONSOLE\" class=\"ch.qos.logback.core.ConsoleAppender\">\n        <layout class=\"ch.qos.logback.classic.PatternLayout\">\n            <Pattern>\n                %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n\n            </Pattern>\n        </layout>\n    </appender>\n\n    <logger name=\"org.apache.rocketmq\" level=\"error\" additivity=\"false\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </logger>\n\n    <root level=\"error\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </root>\n\n</configuration>"
  },
  {
    "path": "common/BUILD.bazel",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\nload(\"//bazel:GenTestRules.bzl\", \"GenTestRules\")\n\njava_library(\n    name = \"common\",\n    srcs = glob([\"src/main/java/**/*.java\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \"@maven//:com_alibaba_fastjson2_fastjson2\",\n        \"@maven//:com_github_luben_zstd_jni\",\n        \"@maven//:com_google_guava_guava\",\n        \"@maven//:commons_collections_commons_collections\",\n        \"@maven//:commons_codec_commons_codec\",\n        \"@maven//:commons_validator_commons_validator\",\n        \"@maven//:io_netty_netty_all\",\n        \"@maven//:io_opentelemetry_opentelemetry_api\",\n        \"@maven//:io_opentelemetry_opentelemetry_context\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_otlp\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_prometheus\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk_common\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk_metrics\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp\",\n        \"@maven//:io_grpc_grpc_api\",\n        \"@maven//:io_grpc_grpc_context\",\n        \"@maven//:org_apache_commons_commons_lang3\",\n        \"@maven//:org_lz4_lz4_java\",\n        \"@maven//:io_github_aliyunmq_rocketmq_slf4j_api\",\n        \"@maven//:io_github_aliyunmq_rocketmq_logback_classic\",\n        \"@maven//:org_apache_rocketmq_rocketmq_rocksdb\",\n    ],\n)\n\njava_library(\n    name = \"tests\",\n    srcs = glob([\"src/test/java/**/*.java\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \":common\",\n        \"//:test_deps\",\n        \"@maven//:com_google_guava_guava\",\n        \"@maven//:com_alibaba_fastjson2_fastjson2\",\n        \"@maven//:commons_codec_commons_codec\",\n        \"@maven//:io_netty_netty_all\",\n        \"@maven//:io_opentelemetry_opentelemetry_api\",\n        \"@maven//:io_opentelemetry_opentelemetry_context\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_otlp\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_prometheus\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk_common\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk_metrics\",\n        \"@maven//:io_grpc_grpc_api\",\n        \"@maven//:io_grpc_grpc_context\",\n        \"@maven//:org_apache_commons_commons_lang3\",\n    ],\n    resources = glob([\"src/test/resources/certs/*.pem\"]) + glob([\"src/test/resources/certs/*.key\"])\n)\n\nGenTestRules(\n    name = \"GeneratedTestRules\",\n    test_files = glob([\"src/test/java/**/*Test.java\"]),\n    deps = [\n        \":tests\",\n    ],\n)\n"
  },
  {
    "path": "common/pom.xml",
    "content": "<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n    <parent>\n        <groupId>org.apache.rocketmq</groupId>\n        <artifactId>rocketmq-all</artifactId>\n        <version>${revision}</version>\n    </parent>\n\n    <modelVersion>4.0.0</modelVersion>\n    <packaging>jar</packaging>\n    <artifactId>rocketmq-common</artifactId>\n    <name>rocketmq-common ${project.version}</name>\n\n    <properties>\n        <project.root>${basedir}/..</project.root>\n    </properties>\n\n    <dependencies>\n        <dependency>\n            <groupId>com.alibaba</groupId>\n            <artifactId>fastjson</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>com.alibaba.fastjson2</groupId>\n            <artifactId>fastjson2</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.netty</groupId>\n            <artifactId>netty-all</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.apache.commons</groupId>\n            <artifactId>commons-lang3</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>commons-validator</groupId>\n            <artifactId>commons-validator</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>com.github.luben</groupId>\n            <artifactId>zstd-jni</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>at.yawk.lz4</groupId>\n            <artifactId>lz4-java</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>com.google.guava</groupId>\n            <artifactId>guava</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>commons-codec</groupId>\n            <artifactId>commons-codec</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.opentelemetry</groupId>\n            <artifactId>opentelemetry-exporter-otlp</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.opentelemetry</groupId>\n            <artifactId>opentelemetry-exporter-prometheus</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.opentelemetry</groupId>\n            <artifactId>opentelemetry-exporter-logging</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.opentelemetry</groupId>\n            <artifactId>opentelemetry-sdk</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.opentelemetry</groupId>\n            <artifactId>opentelemetry-exporter-logging-otlp</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.grpc</groupId>\n            <artifactId>grpc-stub</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.grpc</groupId>\n            <artifactId>grpc-netty-shaded</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>com.squareup.okio</groupId>\n            <artifactId>okio-jvm</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.apache.tomcat</groupId>\n            <artifactId>annotations-api</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.github.aliyunmq</groupId>\n            <artifactId>rocketmq-slf4j-api</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.github.aliyunmq</groupId>\n            <artifactId>rocketmq-logback-classic</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.apache.rocketmq</groupId>\n            <artifactId>rocketmq-rocksdb</artifactId>\n        </dependency>\n    </dependencies>\n</project>\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/AbortProcessException.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\nimport org.apache.rocketmq.common.help.FAQUrl;\n\n/**\n *\n * This exception is used for broker hooks only : SendMessageHook, ConsumeMessageHook, RPCHook\n * This exception is not ignored while executing hooks and it means that\n * certain processor should return an immediate error response to the client. The\n * error response code is included in AbortProcessException.  it's naming might\n * be confusing, so feel free to refactor this class. Also when any class implements\n * the 3 hook interface mentioned above we should be careful if we want to throw\n * an AbortProcessException, because it will change the control flow of broker\n * and cause a RemotingCommand return error immediately. So be aware of the side\n * effect before throw AbortProcessException in your implementation.\n *\n */\npublic class AbortProcessException extends RuntimeException {\n    private static final long serialVersionUID = -5728810933841185841L;\n    private int responseCode;\n    private String errorMessage;\n\n    public AbortProcessException(String errorMessage, Throwable cause) {\n        super(FAQUrl.attachDefaultURL(errorMessage), cause);\n        this.responseCode = -1;\n        this.errorMessage = errorMessage;\n    }\n\n    public AbortProcessException(int responseCode, String errorMessage) {\n        super(FAQUrl.attachDefaultURL(\"CODE: \" + UtilAll.responseCode2String(responseCode) + \"  DESC: \"\n            + errorMessage));\n        this.responseCode = responseCode;\n        this.errorMessage = errorMessage;\n    }\n\n    public int getResponseCode() {\n        return responseCode;\n    }\n\n    public AbortProcessException setResponseCode(final int responseCode) {\n        this.responseCode = responseCode;\n        return this;\n    }\n\n    public String getErrorMessage() {\n        return errorMessage;\n    }\n\n    public void setErrorMessage(final String errorMessage) {\n        this.errorMessage = errorMessage;\n    }\n}"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/BoundaryType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\npublic enum BoundaryType {\n    /**\n     * Indicate that lower boundary is expected.\n     */\n    LOWER(\"lower\"),\n\n    /**\n     * Indicate that upper boundary is expected.\n     */\n    UPPER(\"upper\");\n\n    private String name;\n\n    BoundaryType(String name) {\n        this.name = name;\n    }\n\n    public String getName() {\n        return name;\n    }\n\n    public static BoundaryType getType(String name) {\n        if (BoundaryType.UPPER.getName().equalsIgnoreCase(name)) {\n            return UPPER;\n        }\n        return LOWER;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.common.annotation.ImportantField;\nimport org.apache.rocketmq.common.config.ConfigManagerVersion;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.message.MessageRequestMode;\nimport org.apache.rocketmq.common.metrics.MetricsExporterType;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\n\npublic class BrokerConfig extends BrokerIdentity {\n\n    private String brokerConfigPath = null;\n\n    private String rocketmqHome = MixAll.ROCKETMQ_HOME_DIR;\n    @ImportantField\n    private String namesrvAddr = System.getProperty(MixAll.NAMESRV_ADDR_PROPERTY, System.getenv(MixAll.NAMESRV_ADDR_ENV));\n\n    /**\n     * Listen port for single broker\n     */\n    @ImportantField\n    private int listenPort = 6888;\n\n    @ImportantField\n    private String brokerIP1 = NetworkUtil.getLocalAddress();\n    private String brokerIP2 = NetworkUtil.getLocalAddress();\n\n    @ImportantField\n    private boolean recoverConcurrently = false;\n\n    private int brokerPermission = PermName.PERM_READ | PermName.PERM_WRITE;\n    private int defaultTopicQueueNums = 8;\n    @ImportantField\n    private boolean autoCreateTopicEnable = true;\n\n    private boolean clusterTopicEnable = true;\n\n    private boolean brokerTopicEnable = true;\n    @ImportantField\n    private boolean autoCreateSubscriptionGroup = true;\n    private String messageStorePlugIn = \"\";\n\n    private static final int PROCESSOR_NUMBER = Runtime.getRuntime().availableProcessors();\n    @ImportantField\n    private String msgTraceTopicName = TopicValidator.RMQ_SYS_TRACE_TOPIC;\n    @ImportantField\n    private boolean traceTopicEnable = false;\n    /**\n     * thread numbers for send message thread pool.\n     */\n    private int sendMessageThreadPoolNums = Math.min(PROCESSOR_NUMBER, 4);\n    private int putMessageFutureThreadPoolNums = Math.min(PROCESSOR_NUMBER, 4);\n    private int pullMessageThreadPoolNums = 16 + PROCESSOR_NUMBER * 2;\n    private int litePullMessageThreadPoolNums = 16 + PROCESSOR_NUMBER * 2;\n    private int ackMessageThreadPoolNums = 16;\n    private int processReplyMessageThreadPoolNums = 16 + PROCESSOR_NUMBER * 2;\n    private int queryMessageThreadPoolNums = 8 + PROCESSOR_NUMBER;\n\n    private int adminBrokerThreadPoolNums = 16;\n    private int clientManageThreadPoolNums = 32;\n    private int consumerManageThreadPoolNums = 32;\n    private int loadBalanceProcessorThreadPoolNums = 32;\n    private int heartbeatThreadPoolNums = Math.min(32, PROCESSOR_NUMBER);\n    private int recoverThreadPoolNums = 32;\n\n    /**\n     * Thread numbers for EndTransactionProcessor\n     */\n    private int endTransactionThreadPoolNums = Math.max(8 + PROCESSOR_NUMBER * 2,\n            sendMessageThreadPoolNums * 4);\n\n    private int flushConsumerOffsetInterval = 1000 * 5;\n\n    private int flushConsumerOffsetHistoryInterval = 1000 * 60;\n\n    @ImportantField\n    private boolean rejectTransactionMessage = false;\n\n    @ImportantField\n    private boolean fetchNameSrvAddrByDnsLookup = false;\n\n    @ImportantField\n    private boolean fetchNamesrvAddrByAddressServer = false;\n\n    private int sendThreadPoolQueueCapacity = 10000;\n    private int putThreadPoolQueueCapacity = 10000;\n    private int pullThreadPoolQueueCapacity = 100000;\n    private int litePullThreadPoolQueueCapacity = 100000;\n    private int ackThreadPoolQueueCapacity = 100000;\n    private int replyThreadPoolQueueCapacity = 10000;\n    private int queryThreadPoolQueueCapacity = 20000;\n    private int clientManagerThreadPoolQueueCapacity = 1000000;\n    private int consumerManagerThreadPoolQueueCapacity = 1000000;\n    private int heartbeatThreadPoolQueueCapacity = 50000;\n    private int endTransactionPoolQueueCapacity = 100000;\n    private int adminBrokerThreadPoolQueueCapacity = 10000;\n    private int loadBalanceThreadPoolQueueCapacity = 100000;\n\n    private boolean longPollingEnable = true;\n\n    private long shortPollingTimeMills = 1000;\n\n    private boolean notifyConsumerIdsChangedEnable = true;\n\n    private boolean highSpeedMode = false;\n\n    private int commercialBaseCount = 1;\n\n    private int commercialSizePerMsg = 4 * 1024;\n\n    private boolean accountStatsEnable = true;\n    private boolean accountStatsPrintZeroValues = true;\n\n    private int maxStatsIdleTimeInMinutes = -1;\n\n    private boolean transferMsgByHeap = true;\n\n    private String regionId = MixAll.DEFAULT_TRACE_REGION_ID;\n    private int registerBrokerTimeoutMills = 24000;\n\n    private int sendHeartbeatTimeoutMillis = 1000;\n\n    private boolean slaveReadEnable = false;\n\n    private boolean disableConsumeIfConsumerReadSlowly = false;\n    private long consumerFallbehindThreshold = 1024L * 1024 * 1024 * 16;\n\n    private boolean brokerFastFailureEnable = true;\n    private long waitTimeMillsInSendQueue = 200;\n    private long waitTimeMillsInPullQueue = 5 * 1000;\n    private long waitTimeMillsInLitePullQueue = 5 * 1000;\n    private long waitTimeMillsInHeartbeatQueue = 31 * 1000;\n    private long waitTimeMillsInTransactionQueue = 3 * 1000;\n    private long waitTimeMillsInAckQueue = 3000;\n    private long waitTimeMillsInAdminBrokerQueue = 5 * 1000;\n    private long startAcceptSendRequestTimeStamp = 0L;\n\n    private boolean traceOn = true;\n\n    // Switch of filter bit map calculation.\n    // If switch on:\n    // 1. Calculate filter bit map when construct queue.\n    // 2. Filter bit map will be saved to consume queue extend file if allowed.\n    private boolean enableCalcFilterBitMap = false;\n\n    //Reject the pull consumer instance to pull messages from broker.\n    private boolean rejectPullConsumerEnable = false;\n\n    // Expect num of consumers will use filter.\n    private int expectConsumerNumUseFilter = 32;\n\n    // Error rate of bloom filter, 1~100.\n    private int maxErrorRateOfBloomFilter = 20;\n\n    //how long to clean filter data after dead.Default: 24h\n    private long filterDataCleanTimeSpan = 24 * 3600 * 1000;\n\n    // whether do filter when retry.\n    private boolean filterSupportRetry = false;\n    private boolean enablePropertyFilter = false;\n\n    private boolean compressedRegister = false;\n\n    private boolean forceRegister = true;\n\n    /**\n     * This configurable item defines interval of topics registration of broker to name server. Allowing values are\n     * between 10,000 and 60,000 milliseconds.\n     */\n    private int registerNameServerPeriod = 1000 * 30;\n\n    /**\n     * This configurable item defines interval of update name server address. Default: 120 * 1000 milliseconds\n     */\n    private int updateNameServerAddrPeriod = 1000 * 120;\n\n    /**\n     * the interval to send heartbeat to name server for liveness detection.\n     */\n    private int brokerHeartbeatInterval = 1000;\n\n    /**\n     * How long the broker will be considered as inactive by nameserver since last heartbeat. Effective only if\n     * enableSlaveActingMaster is true\n     */\n    private long brokerNotActiveTimeoutMillis = 10 * 1000;\n\n    private boolean enableNetWorkFlowControl = false;\n\n    private boolean enableBroadcastOffsetStore = true;\n\n    private long broadcastOffsetExpireSecond = 2 * 60;\n\n    private long broadcastOffsetExpireMaxSecond = 5 * 60;\n\n    private int popPollingSize = 1024;\n    private int popPollingMapSize = 100000;\n\n    private int popPollingMapExpireTimeSeconds = 60 * 10;\n    // 20w cost 200M heap memory.\n    private long maxPopPollingSize = 100000;\n    private int reviveQueueNum = 8;\n    private long reviveInterval = 1000;\n    private long reviveMaxSlow = 3;\n    private long reviveScanTime = 10000;\n    private boolean enableSkipLongAwaitingAck = false;\n    private long reviveAckWaitMs = TimeUnit.MINUTES.toMillis(3);\n    private boolean enablePopLog = false;\n    private boolean enablePopBufferMerge = false;\n    private int popCkStayBufferTime = 10 * 1000;\n    private int popCkStayBufferTimeOut = 3 * 1000;\n    private int popCkMaxBufferSize = 200000;\n    private int popCkOffsetMaxQueueSize = 20000;\n    private boolean enablePopBatchAck = false;\n    // set the interval to the maxFilterMessageSize in MessageStoreConfig divided by the cq unit size\n    private long popLongPollingForceNotifyInterval = 800;\n    private boolean enableNotifyBeforePopCalculateLag = true;\n    private boolean enableNotifyAfterPopOrderLockRelease = true;\n    private boolean initPopOffsetByCheckMsgInMem = true;\n    // read message from pop retry topic v1, for the compatibility, will be removed in the future version\n    private boolean retrieveMessageFromPopRetryTopicV1 = true;\n    private boolean enableRetryTopicV2 = false;\n    private int popFromRetryProbability = 20;\n    // pop retry probability for priority mode\n    private int popFromRetryProbabilityForPriority = 0;\n    // 0 as the lowest priority if true\n    private boolean priorityOrderAsc = true;\n    private boolean popConsumerFSServiceInit = true;\n    private boolean popConsumerKVServiceLog = false;\n    private boolean popConsumerKVServiceInit = false;\n    private boolean popConsumerKVServiceEnable = false;\n    private int popReviveMaxReturnSizePerRead = 16 * 1024;\n    private int popReviveConcurrency = 32;\n    private int popReviveMaxAttemptTimes = 16;\n    private boolean popReviveSkipIfGroupAbsent = true;\n    // each message queue will have a corresponding retry queue\n    private boolean useSeparateRetryQueue = false;\n    private boolean realTimeNotifyConsumerChange = true;\n\n    private boolean useMessageFilterForNotification = true;\n    private int maxMessageFilterNumForNotification = 64;\n\n    private boolean litePullMessageEnable = true;\n\n    // The period to sync broker member group from namesrv, default value is 1 second\n    private int syncBrokerMemberGroupPeriod = 1000;\n\n    /**\n     * the interval of pulling topic information from the named server\n     */\n    private long loadBalancePollNameServerInterval = 1000 * 30;\n\n    /**\n     * the interval of cleaning\n     */\n    private int cleanOfflineBrokerInterval = 1000 * 30;\n\n    private boolean serverLoadBalancerEnable = true;\n\n    private MessageRequestMode defaultMessageRequestMode = MessageRequestMode.PULL;\n\n    private int defaultPopShareQueueNum = -1;\n\n    /**\n     * The minimum time of the transactional message  to be checked firstly, one message only exceed this time interval\n     * that can be checked.\n     */\n    @ImportantField\n    private long transactionTimeOut = 6 * 1000;\n\n    /**\n     * The maximum number of times the message was checked, if exceed this value, this message will be discarded.\n     */\n    @ImportantField\n    private int transactionCheckMax = 15;\n\n    /**\n     * Transaction message check interval.\n     */\n    @ImportantField\n    private long transactionCheckInterval = 30 * 1000;\n\n    private long transactionMetricFlushInterval = 10 * 1000;\n\n    private int transactionCheckRocksdbCoreThreads = 2;\n\n    private int transactionCheckRocksdbMaxThreads = 5;\n\n    private int transactionCheckRocksdbQueueCapacity = 2000;\n\n    /**\n     * transaction batch op message\n     */\n    private int transactionOpMsgMaxSize = 4096;\n\n    private int transactionOpBatchInterval = 3000;\n\n    /**\n     * Acl feature switch\n     */\n    @ImportantField\n    private boolean aclEnable = false;\n\n    private boolean storeReplyMessageEnable = true;\n\n    private boolean enableDetailStat = true;\n\n    private boolean autoDeleteUnusedStats = true;\n\n    /**\n     * Whether to distinguish log paths when multiple brokers are deployed on the same machine\n     */\n    private boolean isolateLogEnable = false;\n\n    private long forwardTimeout = 3 * 1000;\n\n    /**\n     * Slave will act master when failover. For example, if master down, timer or transaction message which is expire in slave will\n     * put to master (master of the same process in broker container mode or other masters in cluster when enableFailoverRemotingActing is true)\n     * when enableSlaveActingMaster is true\n     */\n    private boolean enableSlaveActingMaster = false;\n\n    private boolean enableRemoteEscape = false;\n\n    private boolean skipPreOnline = false;\n\n    private boolean asyncSendEnable = true;\n\n    private boolean useServerSideResetOffset = true;\n\n    private long consumerOffsetUpdateVersionStep = 500;\n\n    private long delayOffsetUpdateVersionStep = 200;\n\n    /**\n     * Whether to lock quorum replicas.\n     *\n     * True: need to lock quorum replicas succeed. False: only need to lock one replica succeed.\n     */\n    private boolean lockInStrictMode = false;\n\n    private boolean compatibleWithOldNameSrv = true;\n\n    /**\n     * Is startup controller mode, which support auto switch broker's role.\n     */\n    private boolean enableControllerMode = false;\n\n    private String controllerAddr = \"\";\n\n    private boolean fetchControllerAddrByDnsLookup = false;\n\n    private long syncBrokerMetadataPeriod = 5 * 1000;\n\n    private long checkSyncStateSetPeriod = 5 * 1000;\n\n    private long syncControllerMetadataPeriod = 10 * 1000;\n\n    private long controllerHeartBeatTimeoutMills = 10 * 1000;\n\n    private boolean validateSystemTopicWhenUpdateTopic = true;\n\n    /**\n     * It is an important basis for the controller to choose the broker master.\n     * The lower the value of brokerElectionPriority, the higher the priority of the broker being selected as the master.\n     * You can set a lower priority for the broker with better machine conditions.\n     */\n    private int brokerElectionPriority = Integer.MAX_VALUE;\n\n    private boolean useStaticSubscription = false;\n\n    private MetricsExporterType metricsExporterType = MetricsExporterType.DISABLE;\n\n    private int metricsOtelCardinalityLimit = 50 * 1000;\n    private String metricsGrpcExporterTarget = \"\";\n    private String metricsGrpcExporterHeader = \"\";\n    private long metricGrpcExporterTimeOutInMills = 3 * 1000;\n    private long metricGrpcExporterIntervalInMills = 60 * 1000;\n    private long metricLoggingExporterIntervalInMills = 10 * 1000;\n\n    private int metricsPromExporterPort = 5557;\n    private String metricsPromExporterHost = \"\";\n\n    // Label pairs in CSV. Each label follows pattern of Key:Value. eg: instance_id:xxx,uid:xxx\n    private String metricsLabel = \"\";\n\n    private boolean metricsInDelta = false;\n\n    private boolean enableRemotingMetrics = true;\n    private boolean enableMessageStoreMetrics = true;\n    private boolean enablePopMetrics = true;\n    private boolean enableConnectionMetrics = true;\n    private boolean enableTransactionMetrics = true;\n    private boolean enableStatsMetrics = true;\n    private boolean enableRequestMetrics = true;\n    private boolean enableLagAndDlqMetrics = true;\n\n    private long channelExpiredTimeout = 1000 * 120;\n    private long subscriptionExpiredTimeout = 1000 * 60 * 10;\n\n    /**\n     * Estimate accumulation or not when subscription filter type is tag and is not SUB_ALL.\n     */\n    private boolean estimateAccumulation = true;\n\n    private boolean coldCtrStrategyEnable = false;\n    private boolean usePIDColdCtrStrategy = true;\n    private long cgColdReadThreshold = 3 * 1024 * 1024;\n    private long globalColdReadThreshold = 100 * 1024 * 1024;\n    \n    /**\n     * The interval to fetch namesrv addr, default value is 10 second\n     */\n    private long fetchNamesrvAddrInterval = 10 * 1000;\n\n    /**\n     * Pop response returns the actual retry topic rather than tampering with the original topic\n     */\n    private boolean popResponseReturnActualRetryTopic = false;\n\n    /**\n     * If both the deleteTopicWithBrokerRegistration flag in the NameServer configuration and this flag are set to true,\n     * it guarantees the ultimate consistency of data between the broker and the nameserver during topic deletion.\n     */\n    private boolean enableSingleTopicRegister = false;\n\n    private boolean enableMixedMessageType = false;\n\n    /**\n     * This flag and deleteTopicWithBrokerRegistration flag in the NameServer cannot be set to true at the same time,\n     * otherwise there will be a loss of routing\n     */\n    private boolean enableSplitRegistration = false;\n\n    private boolean enableSplitMetadata = true;\n    private int splitMetadataSize = 2000;\n\n    private long popInflightMessageThreshold = 10000;\n    private boolean enablePopMessageThreshold = false;\n\n    private boolean enableFastChannelEventProcess = false;\n    private boolean printChannelGroups = false;\n    private int printChannelGroupsMinNum = 5;\n\n    private int splitRegistrationSize = 800;\n\n    /**\n     * Config in this black list will be not allowed to update by command.\n     * Try to update this config black list by restart process.\n     * Try to update configures in black list by restart process.\n     */\n    private String configBlackList = \"configBlackList;brokerConfigPath\";\n\n    // if false, will still rewrite ck after max times 17\n    private boolean skipWhenCKRePutReachMaxTimes = false;\n\n    private boolean appendAckAsync = false;\n\n    private boolean appendCkAsync = false;\n\n    private boolean clearRetryTopicWhenDeleteTopic = true;\n\n    private boolean enableLmqStats = false;\n\n    /**\n     * V2 is recommended in cases where LMQ feature is extensively used.\n     */\n    private String configManagerVersion = ConfigManagerVersion.V1.getVersion();\n\n    /**\n     * Whether to use a single RocksDB instance with multiple column families for all configs\n     * instead of separate RocksDB instances for Topic, Group, and Offset configs\n     */\n    private boolean useSingleRocksDBForAllConfigs = false;\n\n    private boolean allowRecallWhenBrokerNotWriteable = true;\n\n    private boolean recallMessageEnable = false;\n\n    private boolean enableRegisterProducer = true;\n\n    private boolean enableCreateSysGroup = true;\n\n    private boolean enableLiteEventMode = true;\n\n    private long liteEventCheckInterval = 10 * 1000;\n\n    private long liteTtlCheckInterval = 120 * 1000;\n\n    private long minLiteTTl = 15 * 60 * 1000;\n\n    private long liteSubscriptionCheckInterval = TimeUnit.MINUTES.toMillis(2);\n\n    private long liteSubscriptionCheckTimeoutMills = TimeUnit.MINUTES.toMillis(3);\n\n    // make sense for rocksdb store\n    private boolean persistConsumerOffsetIncrementally = false;\n\n    private long maxLiteSubscriptionCount = 100000;\n\n    private boolean enableLitePopLog = false;\n\n    private int maxClientEventCount = 100;\n\n    private long liteEventFullDispatchDelayTime = 10 * 1000;\n\n    // lite metrics\n    // whether to collect storeTime in popLiteProcessor\n    private boolean liteLagLatencyCollectEnable = false;\n\n    private boolean liteLagLatencyMetricsEnable = false;\n\n    private boolean liteLagCountMetricsEnable = false;\n\n    private int liteLagLatencyTopK = 50;\n\n    public String getConfigBlackList() {\n        return configBlackList;\n    }\n\n    public void setConfigBlackList(String configBlackList) {\n        this.configBlackList = configBlackList;\n    }\n\n    public long getMaxPopPollingSize() {\n        return maxPopPollingSize;\n    }\n\n    public void setMaxPopPollingSize(long maxPopPollingSize) {\n        this.maxPopPollingSize = maxPopPollingSize;\n    }\n\n    public int getReviveQueueNum() {\n        return reviveQueueNum;\n    }\n\n    public void setReviveQueueNum(int reviveQueueNum) {\n        this.reviveQueueNum = reviveQueueNum;\n    }\n\n    public long getReviveInterval() {\n        return reviveInterval;\n    }\n\n    public void setReviveInterval(long reviveInterval) {\n        this.reviveInterval = reviveInterval;\n    }\n\n    public int getPopCkStayBufferTime() {\n        return popCkStayBufferTime;\n    }\n\n    public void setPopCkStayBufferTime(int popCkStayBufferTime) {\n        this.popCkStayBufferTime = popCkStayBufferTime;\n    }\n\n    public int getPopCkStayBufferTimeOut() {\n        return popCkStayBufferTimeOut;\n    }\n\n    public void setPopCkStayBufferTimeOut(int popCkStayBufferTimeOut) {\n        this.popCkStayBufferTimeOut = popCkStayBufferTimeOut;\n    }\n\n    public int getPopPollingMapSize() {\n        return popPollingMapSize;\n    }\n\n    public void setPopPollingMapSize(int popPollingMapSize) {\n        this.popPollingMapSize = popPollingMapSize;\n    }\n\n    public int getPopPollingMapExpireTimeSeconds() {\n        return popPollingMapExpireTimeSeconds;\n    }\n\n    public void setPopPollingMapExpireTimeSeconds(int popPollingMapExpireTimeSeconds) {\n        this.popPollingMapExpireTimeSeconds = popPollingMapExpireTimeSeconds;\n    }\n\n    public long getReviveScanTime() {\n        return reviveScanTime;\n    }\n\n    public void setReviveScanTime(long reviveScanTime) {\n        this.reviveScanTime = reviveScanTime;\n    }\n\n    public long getReviveMaxSlow() {\n        return reviveMaxSlow;\n    }\n\n    public void setReviveMaxSlow(long reviveMaxSlow) {\n        this.reviveMaxSlow = reviveMaxSlow;\n    }\n\n    public int getPopPollingSize() {\n        return popPollingSize;\n    }\n\n    public void setPopPollingSize(int popPollingSize) {\n        this.popPollingSize = popPollingSize;\n    }\n\n    public boolean isEnablePopBufferMerge() {\n        return enablePopBufferMerge;\n    }\n\n    public void setEnablePopBufferMerge(boolean enablePopBufferMerge) {\n        this.enablePopBufferMerge = enablePopBufferMerge;\n    }\n\n    public int getPopCkMaxBufferSize() {\n        return popCkMaxBufferSize;\n    }\n\n    public void setPopCkMaxBufferSize(int popCkMaxBufferSize) {\n        this.popCkMaxBufferSize = popCkMaxBufferSize;\n    }\n\n    public int getPopCkOffsetMaxQueueSize() {\n        return popCkOffsetMaxQueueSize;\n    }\n\n    public void setPopCkOffsetMaxQueueSize(int popCkOffsetMaxQueueSize) {\n        this.popCkOffsetMaxQueueSize = popCkOffsetMaxQueueSize;\n    }\n\n    public boolean isEnablePopBatchAck() {\n        return enablePopBatchAck;\n    }\n\n    public void setEnablePopBatchAck(boolean enablePopBatchAck) {\n        this.enablePopBatchAck = enablePopBatchAck;\n    }\n\n    public boolean isEnableSkipLongAwaitingAck() {\n        return enableSkipLongAwaitingAck;\n    }\n\n    public void setEnableSkipLongAwaitingAck(boolean enableSkipLongAwaitingAck) {\n        this.enableSkipLongAwaitingAck = enableSkipLongAwaitingAck;\n    }\n\n    public long getReviveAckWaitMs() {\n        return reviveAckWaitMs;\n    }\n\n    public void setReviveAckWaitMs(long reviveAckWaitMs) {\n        this.reviveAckWaitMs = reviveAckWaitMs;\n    }\n\n    public boolean isEnablePopLog() {\n        return enablePopLog;\n    }\n\n    public void setEnablePopLog(boolean enablePopLog) {\n        this.enablePopLog = enablePopLog;\n    }\n\n    public int getPopFromRetryProbability() {\n        return popFromRetryProbability;\n    }\n\n    public void setPopFromRetryProbability(int popFromRetryProbability) {\n        this.popFromRetryProbability = popFromRetryProbability;\n    }\n\n    public boolean isPopConsumerFSServiceInit() {\n        return popConsumerFSServiceInit;\n    }\n\n    public void setPopConsumerFSServiceInit(boolean popConsumerFSServiceInit) {\n        this.popConsumerFSServiceInit = popConsumerFSServiceInit;\n    }\n\n    public boolean isPopConsumerKVServiceLog() {\n        return popConsumerKVServiceLog;\n    }\n\n    public void setPopConsumerKVServiceLog(boolean popConsumerKVServiceLog) {\n        this.popConsumerKVServiceLog = popConsumerKVServiceLog;\n    }\n\n    public boolean isPopConsumerKVServiceInit() {\n        return popConsumerKVServiceInit;\n    }\n\n    public void setPopConsumerKVServiceInit(boolean popConsumerKVServiceInit) {\n        this.popConsumerKVServiceInit = popConsumerKVServiceInit;\n    }\n\n    public boolean isPopConsumerKVServiceEnable() {\n        return popConsumerKVServiceEnable;\n    }\n\n    public void setPopConsumerKVServiceEnable(boolean popConsumerKVServiceEnable) {\n        this.popConsumerKVServiceEnable = popConsumerKVServiceEnable;\n    }\n\n    public int getPopReviveConcurrency() {\n        return popReviveConcurrency;\n    }\n\n    public void setPopReviveConcurrency(int popReviveConcurrency) {\n        this.popReviveConcurrency = popReviveConcurrency;\n    }\n\n    public int getPopReviveMaxReturnSizePerRead() {\n        return popReviveMaxReturnSizePerRead;\n    }\n\n    public void setPopReviveMaxReturnSizePerRead(int popReviveMaxReturnSizePerRead) {\n        this.popReviveMaxReturnSizePerRead = popReviveMaxReturnSizePerRead;\n    }\n\n    public int getPopReviveMaxAttemptTimes() {\n        return popReviveMaxAttemptTimes;\n    }\n\n    public void setPopReviveMaxAttemptTimes(int popReviveMaxAttemptTimes) {\n        this.popReviveMaxAttemptTimes = popReviveMaxAttemptTimes;\n    }\n\n    public boolean isPopReviveSkipIfGroupAbsent() {\n        return popReviveSkipIfGroupAbsent;\n    }\n\n    public void setPopReviveSkipIfGroupAbsent(boolean popReviveSkipIfGroupAbsent) {\n        this.popReviveSkipIfGroupAbsent = popReviveSkipIfGroupAbsent;\n    }\n\n    public boolean isTraceOn() {\n        return traceOn;\n    }\n\n    public void setTraceOn(final boolean traceOn) {\n        this.traceOn = traceOn;\n    }\n\n    public long getStartAcceptSendRequestTimeStamp() {\n        return startAcceptSendRequestTimeStamp;\n    }\n\n    public void setStartAcceptSendRequestTimeStamp(final long startAcceptSendRequestTimeStamp) {\n        this.startAcceptSendRequestTimeStamp = startAcceptSendRequestTimeStamp;\n    }\n\n    public long getWaitTimeMillsInSendQueue() {\n        return waitTimeMillsInSendQueue;\n    }\n\n    public void setWaitTimeMillsInSendQueue(final long waitTimeMillsInSendQueue) {\n        this.waitTimeMillsInSendQueue = waitTimeMillsInSendQueue;\n    }\n\n    public long getConsumerFallbehindThreshold() {\n        return consumerFallbehindThreshold;\n    }\n\n    public void setConsumerFallbehindThreshold(final long consumerFallbehindThreshold) {\n        this.consumerFallbehindThreshold = consumerFallbehindThreshold;\n    }\n\n    public boolean isBrokerFastFailureEnable() {\n        return brokerFastFailureEnable;\n    }\n\n    public void setBrokerFastFailureEnable(final boolean brokerFastFailureEnable) {\n        this.brokerFastFailureEnable = brokerFastFailureEnable;\n    }\n\n    public long getWaitTimeMillsInPullQueue() {\n        return waitTimeMillsInPullQueue;\n    }\n\n    public void setWaitTimeMillsInPullQueue(final long waitTimeMillsInPullQueue) {\n        this.waitTimeMillsInPullQueue = waitTimeMillsInPullQueue;\n    }\n\n    public boolean isDisableConsumeIfConsumerReadSlowly() {\n        return disableConsumeIfConsumerReadSlowly;\n    }\n\n    public void setDisableConsumeIfConsumerReadSlowly(final boolean disableConsumeIfConsumerReadSlowly) {\n        this.disableConsumeIfConsumerReadSlowly = disableConsumeIfConsumerReadSlowly;\n    }\n\n    public boolean isSlaveReadEnable() {\n        return slaveReadEnable;\n    }\n\n    public void setSlaveReadEnable(final boolean slaveReadEnable) {\n        this.slaveReadEnable = slaveReadEnable;\n    }\n\n    public int getRegisterBrokerTimeoutMills() {\n        return registerBrokerTimeoutMills;\n    }\n\n    public void setRegisterBrokerTimeoutMills(final int registerBrokerTimeoutMills) {\n        this.registerBrokerTimeoutMills = registerBrokerTimeoutMills;\n    }\n\n    public String getRegionId() {\n        return regionId;\n    }\n\n    public void setRegionId(final String regionId) {\n        this.regionId = regionId;\n    }\n\n    public boolean isTransferMsgByHeap() {\n        return transferMsgByHeap;\n    }\n\n    public void setTransferMsgByHeap(final boolean transferMsgByHeap) {\n        this.transferMsgByHeap = transferMsgByHeap;\n    }\n\n    public String getMessageStorePlugIn() {\n        return messageStorePlugIn;\n    }\n\n    public void setMessageStorePlugIn(String messageStorePlugIn) {\n        this.messageStorePlugIn = messageStorePlugIn;\n    }\n\n    public boolean isHighSpeedMode() {\n        return highSpeedMode;\n    }\n\n    public void setHighSpeedMode(final boolean highSpeedMode) {\n        this.highSpeedMode = highSpeedMode;\n    }\n\n    public int getBrokerPermission() {\n        return brokerPermission;\n    }\n\n    public void setBrokerPermission(int brokerPermission) {\n        this.brokerPermission = brokerPermission;\n    }\n\n    public int getDefaultTopicQueueNums() {\n        return defaultTopicQueueNums;\n    }\n\n    public void setDefaultTopicQueueNums(int defaultTopicQueueNums) {\n        this.defaultTopicQueueNums = defaultTopicQueueNums;\n    }\n\n    public boolean isAutoCreateTopicEnable() {\n        return autoCreateTopicEnable;\n    }\n\n    public void setAutoCreateTopicEnable(boolean autoCreateTopic) {\n        this.autoCreateTopicEnable = autoCreateTopic;\n    }\n\n    public String getBrokerIP1() {\n        return brokerIP1;\n    }\n\n    public void setBrokerIP1(String brokerIP1) {\n        this.brokerIP1 = brokerIP1;\n    }\n\n    public String getBrokerIP2() {\n        return brokerIP2;\n    }\n\n    public void setBrokerIP2(String brokerIP2) {\n        this.brokerIP2 = brokerIP2;\n    }\n\n    public int getSendMessageThreadPoolNums() {\n        return sendMessageThreadPoolNums;\n    }\n\n    public void setSendMessageThreadPoolNums(int sendMessageThreadPoolNums) {\n        this.sendMessageThreadPoolNums = sendMessageThreadPoolNums;\n    }\n\n    public int getPutMessageFutureThreadPoolNums() {\n        return putMessageFutureThreadPoolNums;\n    }\n\n    public void setPutMessageFutureThreadPoolNums(int putMessageFutureThreadPoolNums) {\n        this.putMessageFutureThreadPoolNums = putMessageFutureThreadPoolNums;\n    }\n\n    public int getPullMessageThreadPoolNums() {\n        return pullMessageThreadPoolNums;\n    }\n\n    public void setPullMessageThreadPoolNums(int pullMessageThreadPoolNums) {\n        this.pullMessageThreadPoolNums = pullMessageThreadPoolNums;\n    }\n\n    public int getAckMessageThreadPoolNums() {\n        return ackMessageThreadPoolNums;\n    }\n\n    public void setAckMessageThreadPoolNums(int ackMessageThreadPoolNums) {\n        this.ackMessageThreadPoolNums = ackMessageThreadPoolNums;\n    }\n\n    public int getProcessReplyMessageThreadPoolNums() {\n        return processReplyMessageThreadPoolNums;\n    }\n\n    public void setProcessReplyMessageThreadPoolNums(int processReplyMessageThreadPoolNums) {\n        this.processReplyMessageThreadPoolNums = processReplyMessageThreadPoolNums;\n    }\n\n    public int getQueryMessageThreadPoolNums() {\n        return queryMessageThreadPoolNums;\n    }\n\n    public void setQueryMessageThreadPoolNums(final int queryMessageThreadPoolNums) {\n        this.queryMessageThreadPoolNums = queryMessageThreadPoolNums;\n    }\n\n    public int getAdminBrokerThreadPoolNums() {\n        return adminBrokerThreadPoolNums;\n    }\n\n    public void setAdminBrokerThreadPoolNums(int adminBrokerThreadPoolNums) {\n        this.adminBrokerThreadPoolNums = adminBrokerThreadPoolNums;\n    }\n\n    public int getFlushConsumerOffsetInterval() {\n        return flushConsumerOffsetInterval;\n    }\n\n    public void setFlushConsumerOffsetInterval(int flushConsumerOffsetInterval) {\n        this.flushConsumerOffsetInterval = flushConsumerOffsetInterval;\n    }\n\n    public int getFlushConsumerOffsetHistoryInterval() {\n        return flushConsumerOffsetHistoryInterval;\n    }\n\n    public void setFlushConsumerOffsetHistoryInterval(int flushConsumerOffsetHistoryInterval) {\n        this.flushConsumerOffsetHistoryInterval = flushConsumerOffsetHistoryInterval;\n    }\n\n    public boolean isClusterTopicEnable() {\n        return clusterTopicEnable;\n    }\n\n    public void setClusterTopicEnable(boolean clusterTopicEnable) {\n        this.clusterTopicEnable = clusterTopicEnable;\n    }\n\n    public String getNamesrvAddr() {\n        return namesrvAddr;\n    }\n\n    public void setNamesrvAddr(String namesrvAddr) {\n        this.namesrvAddr = namesrvAddr;\n    }\n\n    public boolean isAutoCreateSubscriptionGroup() {\n        return autoCreateSubscriptionGroup;\n    }\n\n    public void setAutoCreateSubscriptionGroup(boolean autoCreateSubscriptionGroup) {\n        this.autoCreateSubscriptionGroup = autoCreateSubscriptionGroup;\n    }\n\n    public String getBrokerConfigPath() {\n        return brokerConfigPath;\n    }\n\n    public void setBrokerConfigPath(String brokerConfigPath) {\n        this.brokerConfigPath = brokerConfigPath;\n    }\n\n    public String getRocketmqHome() {\n        return rocketmqHome;\n    }\n\n    public void setRocketmqHome(String rocketmqHome) {\n        this.rocketmqHome = rocketmqHome;\n    }\n\n    public int getListenPort() {\n        return listenPort;\n    }\n\n    public void setListenPort(int listenPort) {\n        this.listenPort = listenPort;\n    }\n\n    public int getLitePullMessageThreadPoolNums() {\n        return litePullMessageThreadPoolNums;\n    }\n\n    public void setLitePullMessageThreadPoolNums(int litePullMessageThreadPoolNums) {\n        this.litePullMessageThreadPoolNums = litePullMessageThreadPoolNums;\n    }\n\n    public int getLitePullThreadPoolQueueCapacity() {\n        return litePullThreadPoolQueueCapacity;\n    }\n\n    public void setLitePullThreadPoolQueueCapacity(int litePullThreadPoolQueueCapacity) {\n        this.litePullThreadPoolQueueCapacity = litePullThreadPoolQueueCapacity;\n    }\n\n    public int getAdminBrokerThreadPoolQueueCapacity() {\n        return adminBrokerThreadPoolQueueCapacity;\n    }\n\n    public void setAdminBrokerThreadPoolQueueCapacity(int adminBrokerThreadPoolQueueCapacity) {\n        this.adminBrokerThreadPoolQueueCapacity = adminBrokerThreadPoolQueueCapacity;\n    }\n\n    public int getLoadBalanceThreadPoolQueueCapacity() {\n        return loadBalanceThreadPoolQueueCapacity;\n    }\n\n    public void setLoadBalanceThreadPoolQueueCapacity(int loadBalanceThreadPoolQueueCapacity) {\n        this.loadBalanceThreadPoolQueueCapacity = loadBalanceThreadPoolQueueCapacity;\n    }\n\n    public int getSendHeartbeatTimeoutMillis() {\n        return sendHeartbeatTimeoutMillis;\n    }\n\n    public void setSendHeartbeatTimeoutMillis(int sendHeartbeatTimeoutMillis) {\n        this.sendHeartbeatTimeoutMillis = sendHeartbeatTimeoutMillis;\n    }\n\n    public long getWaitTimeMillsInLitePullQueue() {\n        return waitTimeMillsInLitePullQueue;\n    }\n\n    public void setWaitTimeMillsInLitePullQueue(long waitTimeMillsInLitePullQueue) {\n        this.waitTimeMillsInLitePullQueue = waitTimeMillsInLitePullQueue;\n    }\n\n    public boolean isLitePullMessageEnable() {\n        return litePullMessageEnable;\n    }\n\n    public void setLitePullMessageEnable(boolean litePullMessageEnable) {\n        this.litePullMessageEnable = litePullMessageEnable;\n    }\n\n    public int getSyncBrokerMemberGroupPeriod() {\n        return syncBrokerMemberGroupPeriod;\n    }\n\n    public void setSyncBrokerMemberGroupPeriod(int syncBrokerMemberGroupPeriod) {\n        this.syncBrokerMemberGroupPeriod = syncBrokerMemberGroupPeriod;\n    }\n\n    public boolean isRejectTransactionMessage() {\n        return rejectTransactionMessage;\n    }\n\n    public void setRejectTransactionMessage(boolean rejectTransactionMessage) {\n        this.rejectTransactionMessage = rejectTransactionMessage;\n    }\n\n    public boolean isFetchNamesrvAddrByAddressServer() {\n        return fetchNamesrvAddrByAddressServer;\n    }\n\n    public void setFetchNamesrvAddrByAddressServer(boolean fetchNamesrvAddrByAddressServer) {\n        this.fetchNamesrvAddrByAddressServer = fetchNamesrvAddrByAddressServer;\n    }\n\n    public int getSendThreadPoolQueueCapacity() {\n        return sendThreadPoolQueueCapacity;\n    }\n\n    public void setSendThreadPoolQueueCapacity(int sendThreadPoolQueueCapacity) {\n        this.sendThreadPoolQueueCapacity = sendThreadPoolQueueCapacity;\n    }\n\n    public int getPutThreadPoolQueueCapacity() {\n        return putThreadPoolQueueCapacity;\n    }\n\n    public void setPutThreadPoolQueueCapacity(int putThreadPoolQueueCapacity) {\n        this.putThreadPoolQueueCapacity = putThreadPoolQueueCapacity;\n    }\n\n    public int getPullThreadPoolQueueCapacity() {\n        return pullThreadPoolQueueCapacity;\n    }\n\n    public void setPullThreadPoolQueueCapacity(int pullThreadPoolQueueCapacity) {\n        this.pullThreadPoolQueueCapacity = pullThreadPoolQueueCapacity;\n    }\n\n    public int getAckThreadPoolQueueCapacity() {\n        return ackThreadPoolQueueCapacity;\n    }\n\n    public void setAckThreadPoolQueueCapacity(int ackThreadPoolQueueCapacity) {\n        this.ackThreadPoolQueueCapacity = ackThreadPoolQueueCapacity;\n    }\n\n    public int getReplyThreadPoolQueueCapacity() {\n        return replyThreadPoolQueueCapacity;\n    }\n\n    public void setReplyThreadPoolQueueCapacity(int replyThreadPoolQueueCapacity) {\n        this.replyThreadPoolQueueCapacity = replyThreadPoolQueueCapacity;\n    }\n\n    public int getQueryThreadPoolQueueCapacity() {\n        return queryThreadPoolQueueCapacity;\n    }\n\n    public void setQueryThreadPoolQueueCapacity(final int queryThreadPoolQueueCapacity) {\n        this.queryThreadPoolQueueCapacity = queryThreadPoolQueueCapacity;\n    }\n\n    public boolean isBrokerTopicEnable() {\n        return brokerTopicEnable;\n    }\n\n    public void setBrokerTopicEnable(boolean brokerTopicEnable) {\n        this.brokerTopicEnable = brokerTopicEnable;\n    }\n\n    public boolean isLongPollingEnable() {\n        return longPollingEnable;\n    }\n\n    public void setLongPollingEnable(boolean longPollingEnable) {\n        this.longPollingEnable = longPollingEnable;\n    }\n\n    public boolean isNotifyConsumerIdsChangedEnable() {\n        return notifyConsumerIdsChangedEnable;\n    }\n\n    public void setNotifyConsumerIdsChangedEnable(boolean notifyConsumerIdsChangedEnable) {\n        this.notifyConsumerIdsChangedEnable = notifyConsumerIdsChangedEnable;\n    }\n\n    public long getShortPollingTimeMills() {\n        return shortPollingTimeMills;\n    }\n\n    public void setShortPollingTimeMills(long shortPollingTimeMills) {\n        this.shortPollingTimeMills = shortPollingTimeMills;\n    }\n\n    public int getClientManageThreadPoolNums() {\n        return clientManageThreadPoolNums;\n    }\n\n    public void setClientManageThreadPoolNums(int clientManageThreadPoolNums) {\n        this.clientManageThreadPoolNums = clientManageThreadPoolNums;\n    }\n\n    public int getClientManagerThreadPoolQueueCapacity() {\n        return clientManagerThreadPoolQueueCapacity;\n    }\n\n    public void setClientManagerThreadPoolQueueCapacity(int clientManagerThreadPoolQueueCapacity) {\n        this.clientManagerThreadPoolQueueCapacity = clientManagerThreadPoolQueueCapacity;\n    }\n\n    public int getConsumerManagerThreadPoolQueueCapacity() {\n        return consumerManagerThreadPoolQueueCapacity;\n    }\n\n    public void setConsumerManagerThreadPoolQueueCapacity(int consumerManagerThreadPoolQueueCapacity) {\n        this.consumerManagerThreadPoolQueueCapacity = consumerManagerThreadPoolQueueCapacity;\n    }\n\n    public int getConsumerManageThreadPoolNums() {\n        return consumerManageThreadPoolNums;\n    }\n\n    public void setConsumerManageThreadPoolNums(int consumerManageThreadPoolNums) {\n        this.consumerManageThreadPoolNums = consumerManageThreadPoolNums;\n    }\n\n    public int getCommercialBaseCount() {\n        return commercialBaseCount;\n    }\n\n    public void setCommercialBaseCount(int commercialBaseCount) {\n        this.commercialBaseCount = commercialBaseCount;\n    }\n\n    public boolean isEnableCalcFilterBitMap() {\n        return enableCalcFilterBitMap;\n    }\n\n    public void setEnableCalcFilterBitMap(boolean enableCalcFilterBitMap) {\n        this.enableCalcFilterBitMap = enableCalcFilterBitMap;\n    }\n\n    public int getExpectConsumerNumUseFilter() {\n        return expectConsumerNumUseFilter;\n    }\n\n    public void setExpectConsumerNumUseFilter(int expectConsumerNumUseFilter) {\n        this.expectConsumerNumUseFilter = expectConsumerNumUseFilter;\n    }\n\n    public int getMaxErrorRateOfBloomFilter() {\n        return maxErrorRateOfBloomFilter;\n    }\n\n    public void setMaxErrorRateOfBloomFilter(int maxErrorRateOfBloomFilter) {\n        this.maxErrorRateOfBloomFilter = maxErrorRateOfBloomFilter;\n    }\n\n    public long getFilterDataCleanTimeSpan() {\n        return filterDataCleanTimeSpan;\n    }\n\n    public void setFilterDataCleanTimeSpan(long filterDataCleanTimeSpan) {\n        this.filterDataCleanTimeSpan = filterDataCleanTimeSpan;\n    }\n\n    public boolean isFilterSupportRetry() {\n        return filterSupportRetry;\n    }\n\n    public void setFilterSupportRetry(boolean filterSupportRetry) {\n        this.filterSupportRetry = filterSupportRetry;\n    }\n\n    public boolean isEnablePropertyFilter() {\n        return enablePropertyFilter;\n    }\n\n    public void setEnablePropertyFilter(boolean enablePropertyFilter) {\n        this.enablePropertyFilter = enablePropertyFilter;\n    }\n\n    public boolean isCompressedRegister() {\n        return compressedRegister;\n    }\n\n    public void setCompressedRegister(boolean compressedRegister) {\n        this.compressedRegister = compressedRegister;\n    }\n\n    public boolean isForceRegister() {\n        return forceRegister;\n    }\n\n    public void setForceRegister(boolean forceRegister) {\n        this.forceRegister = forceRegister;\n    }\n\n    public int getHeartbeatThreadPoolQueueCapacity() {\n        return heartbeatThreadPoolQueueCapacity;\n    }\n\n    public void setHeartbeatThreadPoolQueueCapacity(int heartbeatThreadPoolQueueCapacity) {\n        this.heartbeatThreadPoolQueueCapacity = heartbeatThreadPoolQueueCapacity;\n    }\n\n    public int getHeartbeatThreadPoolNums() {\n        return heartbeatThreadPoolNums;\n    }\n\n    public void setHeartbeatThreadPoolNums(int heartbeatThreadPoolNums) {\n        this.heartbeatThreadPoolNums = heartbeatThreadPoolNums;\n    }\n\n    public long getWaitTimeMillsInHeartbeatQueue() {\n        return waitTimeMillsInHeartbeatQueue;\n    }\n\n    public void setWaitTimeMillsInHeartbeatQueue(long waitTimeMillsInHeartbeatQueue) {\n        this.waitTimeMillsInHeartbeatQueue = waitTimeMillsInHeartbeatQueue;\n    }\n\n    public int getRegisterNameServerPeriod() {\n        return registerNameServerPeriod;\n    }\n\n    public void setRegisterNameServerPeriod(int registerNameServerPeriod) {\n        this.registerNameServerPeriod = registerNameServerPeriod;\n    }\n\n    public long getTransactionTimeOut() {\n        return transactionTimeOut;\n    }\n\n    public void setTransactionTimeOut(long transactionTimeOut) {\n        this.transactionTimeOut = transactionTimeOut;\n    }\n\n    public int getTransactionCheckMax() {\n        return transactionCheckMax;\n    }\n\n    public void setTransactionCheckMax(int transactionCheckMax) {\n        this.transactionCheckMax = transactionCheckMax;\n    }\n\n    public long getTransactionCheckInterval() {\n        return transactionCheckInterval;\n    }\n\n    public void setTransactionCheckInterval(long transactionCheckInterval) {\n        this.transactionCheckInterval = transactionCheckInterval;\n    }\n\n    public int getEndTransactionThreadPoolNums() {\n        return endTransactionThreadPoolNums;\n    }\n\n    public void setEndTransactionThreadPoolNums(int endTransactionThreadPoolNums) {\n        this.endTransactionThreadPoolNums = endTransactionThreadPoolNums;\n    }\n\n    public int getEndTransactionPoolQueueCapacity() {\n        return endTransactionPoolQueueCapacity;\n    }\n\n    public void setEndTransactionPoolQueueCapacity(int endTransactionPoolQueueCapacity) {\n        this.endTransactionPoolQueueCapacity = endTransactionPoolQueueCapacity;\n    }\n\n    public long getWaitTimeMillsInTransactionQueue() {\n        return waitTimeMillsInTransactionQueue;\n    }\n\n    public void setWaitTimeMillsInTransactionQueue(long waitTimeMillsInTransactionQueue) {\n        this.waitTimeMillsInTransactionQueue = waitTimeMillsInTransactionQueue;\n    }\n\n    public String getMsgTraceTopicName() {\n        return msgTraceTopicName;\n    }\n\n    public long getWaitTimeMillsInAdminBrokerQueue() {\n        return waitTimeMillsInAdminBrokerQueue;\n    }\n\n    public void setWaitTimeMillsInAdminBrokerQueue(long waitTimeMillsInAdminBrokerQueue) {\n        this.waitTimeMillsInAdminBrokerQueue = waitTimeMillsInAdminBrokerQueue;\n    }\n\n    public void setMsgTraceTopicName(String msgTraceTopicName) {\n        this.msgTraceTopicName = msgTraceTopicName;\n    }\n\n    public boolean isTraceTopicEnable() {\n        return traceTopicEnable;\n    }\n\n    public void setTraceTopicEnable(boolean traceTopicEnable) {\n        this.traceTopicEnable = traceTopicEnable;\n    }\n\n    public void setAclEnable(boolean aclEnable) {\n        this.aclEnable = aclEnable;\n    }\n\n    public boolean isStoreReplyMessageEnable() {\n        return storeReplyMessageEnable;\n    }\n\n    public void setStoreReplyMessageEnable(boolean storeReplyMessageEnable) {\n        this.storeReplyMessageEnable = storeReplyMessageEnable;\n    }\n\n    public boolean isEnableDetailStat() {\n        return enableDetailStat;\n    }\n\n    public void setEnableDetailStat(boolean enableDetailStat) {\n        this.enableDetailStat = enableDetailStat;\n    }\n\n    public boolean isAutoDeleteUnusedStats() {\n        return autoDeleteUnusedStats;\n    }\n\n    public void setAutoDeleteUnusedStats(boolean autoDeleteUnusedStats) {\n        this.autoDeleteUnusedStats = autoDeleteUnusedStats;\n    }\n\n    public long getLoadBalancePollNameServerInterval() {\n        return loadBalancePollNameServerInterval;\n    }\n\n    public void setLoadBalancePollNameServerInterval(long loadBalancePollNameServerInterval) {\n        this.loadBalancePollNameServerInterval = loadBalancePollNameServerInterval;\n    }\n\n    public int getCleanOfflineBrokerInterval() {\n        return cleanOfflineBrokerInterval;\n    }\n\n    public void setCleanOfflineBrokerInterval(int cleanOfflineBrokerInterval) {\n        this.cleanOfflineBrokerInterval = cleanOfflineBrokerInterval;\n    }\n\n    public int getLoadBalanceProcessorThreadPoolNums() {\n        return loadBalanceProcessorThreadPoolNums;\n    }\n\n    public void setLoadBalanceProcessorThreadPoolNums(int loadBalanceProcessorThreadPoolNums) {\n        this.loadBalanceProcessorThreadPoolNums = loadBalanceProcessorThreadPoolNums;\n    }\n\n    public boolean isServerLoadBalancerEnable() {\n        return serverLoadBalancerEnable;\n    }\n\n    public void setServerLoadBalancerEnable(boolean serverLoadBalancerEnable) {\n        this.serverLoadBalancerEnable = serverLoadBalancerEnable;\n    }\n\n    public MessageRequestMode getDefaultMessageRequestMode() {\n        return defaultMessageRequestMode;\n    }\n\n    public void setDefaultMessageRequestMode(String defaultMessageRequestMode) {\n        this.defaultMessageRequestMode = MessageRequestMode.valueOf(defaultMessageRequestMode);\n    }\n\n    public int getDefaultPopShareQueueNum() {\n        return defaultPopShareQueueNum;\n    }\n\n    public void setDefaultPopShareQueueNum(int defaultPopShareQueueNum) {\n        this.defaultPopShareQueueNum = defaultPopShareQueueNum;\n    }\n\n    public long getForwardTimeout() {\n        return forwardTimeout;\n    }\n\n    public void setForwardTimeout(long timeout) {\n        this.forwardTimeout = timeout;\n    }\n\n    public int getBrokerHeartbeatInterval() {\n        return brokerHeartbeatInterval;\n    }\n\n    public void setBrokerHeartbeatInterval(int brokerHeartbeatInterval) {\n        this.brokerHeartbeatInterval = brokerHeartbeatInterval;\n    }\n\n    public long getBrokerNotActiveTimeoutMillis() {\n        return brokerNotActiveTimeoutMillis;\n    }\n\n    public void setBrokerNotActiveTimeoutMillis(long brokerNotActiveTimeoutMillis) {\n        this.brokerNotActiveTimeoutMillis = brokerNotActiveTimeoutMillis;\n    }\n\n    public boolean isEnableNetWorkFlowControl() {\n        return enableNetWorkFlowControl;\n    }\n\n    public void setEnableNetWorkFlowControl(boolean enableNetWorkFlowControl) {\n        this.enableNetWorkFlowControl = enableNetWorkFlowControl;\n    }\n\n    public long getPopLongPollingForceNotifyInterval() {\n        return popLongPollingForceNotifyInterval;\n    }\n\n    public void setPopLongPollingForceNotifyInterval(long popLongPollingForceNotifyInterval) {\n        this.popLongPollingForceNotifyInterval = popLongPollingForceNotifyInterval;\n    }\n\n    public boolean isEnableNotifyBeforePopCalculateLag() {\n        return enableNotifyBeforePopCalculateLag;\n    }\n\n    public void setEnableNotifyBeforePopCalculateLag(boolean enableNotifyBeforePopCalculateLag) {\n        this.enableNotifyBeforePopCalculateLag = enableNotifyBeforePopCalculateLag;\n    }\n\n    public boolean isEnableNotifyAfterPopOrderLockRelease() {\n        return enableNotifyAfterPopOrderLockRelease;\n    }\n\n    public void setEnableNotifyAfterPopOrderLockRelease(boolean enableNotifyAfterPopOrderLockRelease) {\n        this.enableNotifyAfterPopOrderLockRelease = enableNotifyAfterPopOrderLockRelease;\n    }\n\n    public boolean isInitPopOffsetByCheckMsgInMem() {\n        return initPopOffsetByCheckMsgInMem;\n    }\n\n    public void setInitPopOffsetByCheckMsgInMem(boolean initPopOffsetByCheckMsgInMem) {\n        this.initPopOffsetByCheckMsgInMem = initPopOffsetByCheckMsgInMem;\n    }\n\n    public boolean isRetrieveMessageFromPopRetryTopicV1() {\n        return retrieveMessageFromPopRetryTopicV1;\n    }\n\n    public void setRetrieveMessageFromPopRetryTopicV1(boolean retrieveMessageFromPopRetryTopicV1) {\n        this.retrieveMessageFromPopRetryTopicV1 = retrieveMessageFromPopRetryTopicV1;\n    }\n\n    public boolean isEnableRetryTopicV2() {\n        return enableRetryTopicV2;\n    }\n\n    public void setEnableRetryTopicV2(boolean enableRetryTopicV2) {\n        this.enableRetryTopicV2 = enableRetryTopicV2;\n    }\n\n    public boolean isRealTimeNotifyConsumerChange() {\n        return realTimeNotifyConsumerChange;\n    }\n\n    public void setRealTimeNotifyConsumerChange(boolean realTimeNotifyConsumerChange) {\n        this.realTimeNotifyConsumerChange = realTimeNotifyConsumerChange;\n    }\n\n    public boolean isEnableSlaveActingMaster() {\n        return enableSlaveActingMaster;\n    }\n\n    public void setEnableSlaveActingMaster(boolean enableSlaveActingMaster) {\n        this.enableSlaveActingMaster = enableSlaveActingMaster;\n    }\n\n    public boolean isEnableRemoteEscape() {\n        return enableRemoteEscape;\n    }\n\n    public void setEnableRemoteEscape(boolean enableRemoteEscape) {\n        this.enableRemoteEscape = enableRemoteEscape;\n    }\n\n    public boolean isSkipPreOnline() {\n        return skipPreOnline;\n    }\n\n    public void setSkipPreOnline(boolean skipPreOnline) {\n        this.skipPreOnline = skipPreOnline;\n    }\n\n    public boolean isAsyncSendEnable() {\n        return asyncSendEnable;\n    }\n\n    public void setAsyncSendEnable(boolean asyncSendEnable) {\n        this.asyncSendEnable = asyncSendEnable;\n    }\n\n    public long getConsumerOffsetUpdateVersionStep() {\n        return consumerOffsetUpdateVersionStep;\n    }\n\n    public void setConsumerOffsetUpdateVersionStep(long consumerOffsetUpdateVersionStep) {\n        this.consumerOffsetUpdateVersionStep = consumerOffsetUpdateVersionStep;\n    }\n\n    public long getDelayOffsetUpdateVersionStep() {\n        return delayOffsetUpdateVersionStep;\n    }\n\n    public void setDelayOffsetUpdateVersionStep(long delayOffsetUpdateVersionStep) {\n        this.delayOffsetUpdateVersionStep = delayOffsetUpdateVersionStep;\n    }\n\n    public int getCommercialSizePerMsg() {\n        return commercialSizePerMsg;\n    }\n\n    public void setCommercialSizePerMsg(int commercialSizePerMsg) {\n        this.commercialSizePerMsg = commercialSizePerMsg;\n    }\n\n    public long getWaitTimeMillsInAckQueue() {\n        return waitTimeMillsInAckQueue;\n    }\n\n    public void setWaitTimeMillsInAckQueue(long waitTimeMillsInAckQueue) {\n        this.waitTimeMillsInAckQueue = waitTimeMillsInAckQueue;\n    }\n\n    public boolean isRejectPullConsumerEnable() {\n        return rejectPullConsumerEnable;\n    }\n\n    public void setRejectPullConsumerEnable(boolean rejectPullConsumerEnable) {\n        this.rejectPullConsumerEnable = rejectPullConsumerEnable;\n    }\n\n    public boolean isAccountStatsEnable() {\n        return accountStatsEnable;\n    }\n\n    public void setAccountStatsEnable(boolean accountStatsEnable) {\n        this.accountStatsEnable = accountStatsEnable;\n    }\n\n    public boolean isAccountStatsPrintZeroValues() {\n        return accountStatsPrintZeroValues;\n    }\n\n    public void setAccountStatsPrintZeroValues(boolean accountStatsPrintZeroValues) {\n        this.accountStatsPrintZeroValues = accountStatsPrintZeroValues;\n    }\n\n    public int getMaxStatsIdleTimeInMinutes() {\n        return maxStatsIdleTimeInMinutes;\n    }\n\n    public void setMaxStatsIdleTimeInMinutes(int maxStatsIdleTimeInMinutes) {\n        this.maxStatsIdleTimeInMinutes = maxStatsIdleTimeInMinutes;\n    }\n\n    public boolean isLockInStrictMode() {\n        return lockInStrictMode;\n    }\n\n    public void setLockInStrictMode(boolean lockInStrictMode) {\n        this.lockInStrictMode = lockInStrictMode;\n    }\n\n    public boolean isIsolateLogEnable() {\n        return isolateLogEnable;\n    }\n\n    public void setIsolateLogEnable(boolean isolateLogEnable) {\n        this.isolateLogEnable = isolateLogEnable;\n    }\n\n    public boolean isCompatibleWithOldNameSrv() {\n        return compatibleWithOldNameSrv;\n    }\n\n    public void setCompatibleWithOldNameSrv(boolean compatibleWithOldNameSrv) {\n        this.compatibleWithOldNameSrv = compatibleWithOldNameSrv;\n    }\n\n    public boolean isEnableControllerMode() {\n        return enableControllerMode;\n    }\n\n    public void setEnableControllerMode(boolean enableControllerMode) {\n        this.enableControllerMode = enableControllerMode;\n    }\n\n    public String getControllerAddr() {\n        return controllerAddr;\n    }\n\n    public void setControllerAddr(String controllerAddr) {\n        this.controllerAddr = controllerAddr;\n    }\n\n    public boolean isFetchControllerAddrByDnsLookup() {\n        return fetchControllerAddrByDnsLookup;\n    }\n\n    public void setFetchControllerAddrByDnsLookup(boolean fetchControllerAddrByDnsLookup) {\n        this.fetchControllerAddrByDnsLookup = fetchControllerAddrByDnsLookup;\n    }\n\n    public long getSyncBrokerMetadataPeriod() {\n        return syncBrokerMetadataPeriod;\n    }\n\n    public void setSyncBrokerMetadataPeriod(long syncBrokerMetadataPeriod) {\n        this.syncBrokerMetadataPeriod = syncBrokerMetadataPeriod;\n    }\n\n    public long getCheckSyncStateSetPeriod() {\n        return checkSyncStateSetPeriod;\n    }\n\n    public void setCheckSyncStateSetPeriod(long checkSyncStateSetPeriod) {\n        this.checkSyncStateSetPeriod = checkSyncStateSetPeriod;\n    }\n\n    public long getSyncControllerMetadataPeriod() {\n        return syncControllerMetadataPeriod;\n    }\n\n    public void setSyncControllerMetadataPeriod(long syncControllerMetadataPeriod) {\n        this.syncControllerMetadataPeriod = syncControllerMetadataPeriod;\n    }\n\n    public int getBrokerElectionPriority() {\n        return brokerElectionPriority;\n    }\n\n    public void setBrokerElectionPriority(int brokerElectionPriority) {\n        this.brokerElectionPriority = brokerElectionPriority;\n    }\n\n    public long getControllerHeartBeatTimeoutMills() {\n        return controllerHeartBeatTimeoutMills;\n    }\n\n    public void setControllerHeartBeatTimeoutMills(long controllerHeartBeatTimeoutMills) {\n        this.controllerHeartBeatTimeoutMills = controllerHeartBeatTimeoutMills;\n    }\n\n    public boolean isRecoverConcurrently() {\n        return recoverConcurrently;\n    }\n\n    public void setRecoverConcurrently(boolean recoverConcurrently) {\n        this.recoverConcurrently = recoverConcurrently;\n    }\n\n    public int getRecoverThreadPoolNums() {\n        return recoverThreadPoolNums;\n    }\n\n    public void setRecoverThreadPoolNums(int recoverThreadPoolNums) {\n        this.recoverThreadPoolNums = recoverThreadPoolNums;\n    }\n\n    public boolean isFetchNameSrvAddrByDnsLookup() {\n        return fetchNameSrvAddrByDnsLookup;\n    }\n\n    public void setFetchNameSrvAddrByDnsLookup(boolean fetchNameSrvAddrByDnsLookup) {\n        this.fetchNameSrvAddrByDnsLookup = fetchNameSrvAddrByDnsLookup;\n    }\n\n    public boolean isUseServerSideResetOffset() {\n        return useServerSideResetOffset;\n    }\n\n    public void setUseServerSideResetOffset(boolean useServerSideResetOffset) {\n        this.useServerSideResetOffset = useServerSideResetOffset;\n    }\n\n    public boolean isEnableBroadcastOffsetStore() {\n        return enableBroadcastOffsetStore;\n    }\n\n    public void setEnableBroadcastOffsetStore(boolean enableBroadcastOffsetStore) {\n        this.enableBroadcastOffsetStore = enableBroadcastOffsetStore;\n    }\n\n    public long getBroadcastOffsetExpireSecond() {\n        return broadcastOffsetExpireSecond;\n    }\n\n    public void setBroadcastOffsetExpireSecond(long broadcastOffsetExpireSecond) {\n        this.broadcastOffsetExpireSecond = broadcastOffsetExpireSecond;\n    }\n\n    public long getBroadcastOffsetExpireMaxSecond() {\n        return broadcastOffsetExpireMaxSecond;\n    }\n\n    public void setBroadcastOffsetExpireMaxSecond(long broadcastOffsetExpireMaxSecond) {\n        this.broadcastOffsetExpireMaxSecond = broadcastOffsetExpireMaxSecond;\n    }\n\n    public MetricsExporterType getMetricsExporterType() {\n        return metricsExporterType;\n    }\n\n    public void setMetricsExporterType(MetricsExporterType metricsExporterType) {\n        this.metricsExporterType = metricsExporterType;\n    }\n\n    public void setMetricsExporterType(int metricsExporterType) {\n        this.metricsExporterType = MetricsExporterType.valueOf(metricsExporterType);\n    }\n\n    public void setMetricsExporterType(String metricsExporterType) {\n        this.metricsExporterType = MetricsExporterType.valueOf(metricsExporterType);\n    }\n\n    public int getMetricsOtelCardinalityLimit() {\n        return metricsOtelCardinalityLimit;\n    }\n\n    public void setMetricsOtelCardinalityLimit(int metricsOtelCardinalityLimit) {\n        this.metricsOtelCardinalityLimit = metricsOtelCardinalityLimit;\n    }\n\n    public String getMetricsGrpcExporterTarget() {\n        return metricsGrpcExporterTarget;\n    }\n\n    public void setMetricsGrpcExporterTarget(String metricsGrpcExporterTarget) {\n        this.metricsGrpcExporterTarget = metricsGrpcExporterTarget;\n    }\n\n    public String getMetricsGrpcExporterHeader() {\n        return metricsGrpcExporterHeader;\n    }\n\n    public void setMetricsGrpcExporterHeader(String metricsGrpcExporterHeader) {\n        this.metricsGrpcExporterHeader = metricsGrpcExporterHeader;\n    }\n\n    public long getMetricGrpcExporterTimeOutInMills() {\n        return metricGrpcExporterTimeOutInMills;\n    }\n\n    public void setMetricGrpcExporterTimeOutInMills(long metricGrpcExporterTimeOutInMills) {\n        this.metricGrpcExporterTimeOutInMills = metricGrpcExporterTimeOutInMills;\n    }\n\n    public long getMetricGrpcExporterIntervalInMills() {\n        return metricGrpcExporterIntervalInMills;\n    }\n\n    public void setMetricGrpcExporterIntervalInMills(long metricGrpcExporterIntervalInMills) {\n        this.metricGrpcExporterIntervalInMills = metricGrpcExporterIntervalInMills;\n    }\n\n    public long getMetricLoggingExporterIntervalInMills() {\n        return metricLoggingExporterIntervalInMills;\n    }\n\n    public void setMetricLoggingExporterIntervalInMills(long metricLoggingExporterIntervalInMills) {\n        this.metricLoggingExporterIntervalInMills = metricLoggingExporterIntervalInMills;\n    }\n\n    public String getMetricsLabel() {\n        return metricsLabel;\n    }\n\n    public void setMetricsLabel(String metricsLabel) {\n        this.metricsLabel = metricsLabel;\n    }\n\n    public boolean isMetricsInDelta() {\n        return metricsInDelta;\n    }\n\n    public void setMetricsInDelta(boolean metricsInDelta) {\n        this.metricsInDelta = metricsInDelta;\n    }\n\n    public int getMetricsPromExporterPort() {\n        return metricsPromExporterPort;\n    }\n\n    public void setMetricsPromExporterPort(int metricsPromExporterPort) {\n        this.metricsPromExporterPort = metricsPromExporterPort;\n    }\n\n    public String getMetricsPromExporterHost() {\n        return metricsPromExporterHost;\n    }\n\n    public void setMetricsPromExporterHost(String metricsPromExporterHost) {\n        this.metricsPromExporterHost = metricsPromExporterHost;\n    }\n\n    public boolean isEnablePopMetrics() {\n        return enablePopMetrics;\n    }\n\n    public void setEnablePopMetrics(boolean enablePopMetrics) {\n        this.enablePopMetrics = enablePopMetrics;\n    }\n\n    public boolean isEnableConnectionMetrics() {\n        return enableConnectionMetrics;\n    }\n\n    public void setEnableConnectionMetrics(boolean enableConnectionMetrics) {\n        this.enableConnectionMetrics = enableConnectionMetrics;\n    }\n\n    public boolean isEnableTransactionMetrics() {\n        return enableTransactionMetrics;\n    }\n\n    public void setEnableTransactionMetrics(boolean enableTransactionMetrics) {\n        this.enableTransactionMetrics = enableTransactionMetrics;\n    }\n\n    public boolean isEnableStatsMetrics() {\n        return enableStatsMetrics;\n    }\n\n    public void setEnableStatsMetrics(boolean enableStatsMetrics) {\n        this.enableStatsMetrics = enableStatsMetrics;\n    }\n\n    public boolean isEnableRequestMetrics() {\n        return enableRequestMetrics;\n    }\n\n    public void setEnableRequestMetrics(boolean enableRequestMetrics) {\n        this.enableRequestMetrics = enableRequestMetrics;\n    }\n\n\n    public boolean isEnableLagAndDlqMetrics() {\n        return enableLagAndDlqMetrics;\n    }\n\n    public void setEnableLagAndDlqMetrics(boolean enableLagAndDlqMetrics) {\n        this.enableLagAndDlqMetrics = enableLagAndDlqMetrics;\n    }\n\n    public boolean isEnableRemotingMetrics() {\n        return enableRemotingMetrics;\n    }\n\n    public void setEnableRemotingMetrics(boolean enableRemotingMetrics) {\n        this.enableRemotingMetrics = enableRemotingMetrics;\n    }\n\n    public boolean isEnableMessageStoreMetrics() {\n        return enableMessageStoreMetrics;\n    }\n\n    public void setEnableMessageStoreMetrics(boolean enableMessageStoreMetrics) {\n        this.enableMessageStoreMetrics = enableMessageStoreMetrics;\n    }\n\n    public int getTransactionOpMsgMaxSize() {\n        return transactionOpMsgMaxSize;\n    }\n\n    public void setTransactionOpMsgMaxSize(int transactionOpMsgMaxSize) {\n        this.transactionOpMsgMaxSize = transactionOpMsgMaxSize;\n    }\n\n    public int getTransactionOpBatchInterval() {\n        return transactionOpBatchInterval;\n    }\n\n    public void setTransactionOpBatchInterval(int transactionOpBatchInterval) {\n        this.transactionOpBatchInterval = transactionOpBatchInterval;\n    }\n\n    public long getChannelExpiredTimeout() {\n        return channelExpiredTimeout;\n    }\n\n    public void setChannelExpiredTimeout(long channelExpiredTimeout) {\n        this.channelExpiredTimeout = channelExpiredTimeout;\n    }\n\n    public long getSubscriptionExpiredTimeout() {\n        return subscriptionExpiredTimeout;\n    }\n\n    public void setSubscriptionExpiredTimeout(long subscriptionExpiredTimeout) {\n        this.subscriptionExpiredTimeout = subscriptionExpiredTimeout;\n    }\n\n    public boolean isValidateSystemTopicWhenUpdateTopic() {\n        return validateSystemTopicWhenUpdateTopic;\n    }\n\n    public void setValidateSystemTopicWhenUpdateTopic(boolean validateSystemTopicWhenUpdateTopic) {\n        this.validateSystemTopicWhenUpdateTopic = validateSystemTopicWhenUpdateTopic;\n    }\n\n    public boolean isEstimateAccumulation() {\n        return estimateAccumulation;\n    }\n\n    public void setEstimateAccumulation(boolean estimateAccumulation) {\n        this.estimateAccumulation = estimateAccumulation;\n    }\n\n    public boolean isColdCtrStrategyEnable() {\n        return coldCtrStrategyEnable;\n    }\n\n    public void setColdCtrStrategyEnable(boolean coldCtrStrategyEnable) {\n        this.coldCtrStrategyEnable = coldCtrStrategyEnable;\n    }\n\n    public boolean isUsePIDColdCtrStrategy() {\n        return usePIDColdCtrStrategy;\n    }\n\n    public void setUsePIDColdCtrStrategy(boolean usePIDColdCtrStrategy) {\n        this.usePIDColdCtrStrategy = usePIDColdCtrStrategy;\n    }\n\n    public long getCgColdReadThreshold() {\n        return cgColdReadThreshold;\n    }\n\n    public void setCgColdReadThreshold(long cgColdReadThreshold) {\n        this.cgColdReadThreshold = cgColdReadThreshold;\n    }\n\n    public long getGlobalColdReadThreshold() {\n        return globalColdReadThreshold;\n    }\n\n    public void setGlobalColdReadThreshold(long globalColdReadThreshold) {\n        this.globalColdReadThreshold = globalColdReadThreshold;\n    }\n\n    public boolean isUseStaticSubscription() {\n        return useStaticSubscription;\n    }\n\n    public void setUseStaticSubscription(boolean useStaticSubscription) {\n        this.useStaticSubscription = useStaticSubscription;\n    }\n    \n    public long getFetchNamesrvAddrInterval() {\n        return fetchNamesrvAddrInterval;\n    }\n    \n    public void setFetchNamesrvAddrInterval(final long fetchNamesrvAddrInterval) {\n        this.fetchNamesrvAddrInterval = fetchNamesrvAddrInterval;\n    }\n\n    public boolean isPopResponseReturnActualRetryTopic() {\n        return popResponseReturnActualRetryTopic;\n    }\n\n    public void setPopResponseReturnActualRetryTopic(boolean popResponseReturnActualRetryTopic) {\n        this.popResponseReturnActualRetryTopic = popResponseReturnActualRetryTopic;\n    }\n\n    public boolean isEnableSingleTopicRegister() {\n        return enableSingleTopicRegister;\n    }\n\n    public void setEnableSingleTopicRegister(boolean enableSingleTopicRegister) {\n        this.enableSingleTopicRegister = enableSingleTopicRegister;\n    }\n\n    public boolean isEnableMixedMessageType() {\n        return enableMixedMessageType;\n    }\n\n    public void setEnableMixedMessageType(boolean enableMixedMessageType) {\n        this.enableMixedMessageType = enableMixedMessageType;\n    }\n\n    public boolean isEnableSplitRegistration() {\n        return enableSplitRegistration;\n    }\n\n    public void setEnableSplitRegistration(boolean enableSplitRegistration) {\n        this.enableSplitRegistration = enableSplitRegistration;\n    }\n\n    public boolean isEnableFastChannelEventProcess() {\n        return enableFastChannelEventProcess;\n    }\n\n    public void setEnableFastChannelEventProcess(boolean enableFastChannelEventProcess) {\n        this.enableFastChannelEventProcess = enableFastChannelEventProcess;\n    }\n\n    public boolean isPrintChannelGroups() {\n        return printChannelGroups;\n    }\n\n    public void setPrintChannelGroups(boolean printChannelGroups) {\n        this.printChannelGroups = printChannelGroups;\n    }\n\n    public int getPrintChannelGroupsMinNum() {\n        return printChannelGroupsMinNum;\n    }\n\n    public void setPrintChannelGroupsMinNum(int printChannelGroupsMinNum) {\n        this.printChannelGroupsMinNum = printChannelGroupsMinNum;\n    }\n\n    public int getSplitRegistrationSize() {\n        return splitRegistrationSize;\n    }\n\n    public void setSplitRegistrationSize(int splitRegistrationSize) {\n        this.splitRegistrationSize = splitRegistrationSize;\n    }\n\n    public long getTransactionMetricFlushInterval() {\n        return transactionMetricFlushInterval;\n    }\n\n    public void setTransactionMetricFlushInterval(long transactionMetricFlushInterval) {\n        this.transactionMetricFlushInterval = transactionMetricFlushInterval;\n    }\n\n    public void setTransactionCheckRocksdbCoreThreads(int transactionCheckRocksdbCoreThreads) {\n        this.transactionCheckRocksdbCoreThreads = transactionCheckRocksdbCoreThreads;\n    }\n\n    public int getTransactionCheckRocksdbCoreThreads() {\n        return transactionCheckRocksdbCoreThreads;\n    }\n\n    public int getTransactionCheckRocksdbMaxThreads() {\n        return transactionCheckRocksdbMaxThreads;\n    }\n\n    public void setTransactionCheckRocksdbMaxThreads(int transactionCheckRocksdbMaxThreads) {\n        this.transactionCheckRocksdbMaxThreads = transactionCheckRocksdbMaxThreads;\n    }\n\n    public int getTransactionCheckRocksdbQueueCapacity() {\n        return transactionCheckRocksdbQueueCapacity;\n    }\n\n    public void setTransactionCheckRocksdbQueueCapacity(int transactionCheckRocksdbQueueCapacity) {\n        this.transactionCheckRocksdbQueueCapacity = transactionCheckRocksdbQueueCapacity;\n    }\n\n    public long getPopInflightMessageThreshold() {\n        return popInflightMessageThreshold;\n    }\n\n    public void setPopInflightMessageThreshold(long popInflightMessageThreshold) {\n        this.popInflightMessageThreshold = popInflightMessageThreshold;\n    }\n\n    public boolean isEnablePopMessageThreshold() {\n        return enablePopMessageThreshold;\n    }\n\n    public void setEnablePopMessageThreshold(boolean enablePopMessageThreshold) {\n        this.enablePopMessageThreshold = enablePopMessageThreshold;\n    }\n\n    public boolean isSkipWhenCKRePutReachMaxTimes() {\n        return skipWhenCKRePutReachMaxTimes;\n    }\n\n    public void setSkipWhenCKRePutReachMaxTimes(boolean skipWhenCKRePutReachMaxTimes) {\n        this.skipWhenCKRePutReachMaxTimes = skipWhenCKRePutReachMaxTimes;\n    }\n\n    public int getUpdateNameServerAddrPeriod() {\n        return updateNameServerAddrPeriod;\n    }\n\n    public void setUpdateNameServerAddrPeriod(int updateNameServerAddrPeriod) {\n        this.updateNameServerAddrPeriod = updateNameServerAddrPeriod;\n    }\n\n    public boolean isAppendAckAsync() {\n        return appendAckAsync;\n    }\n\n    public void setAppendAckAsync(boolean appendAckAsync) {\n        this.appendAckAsync = appendAckAsync;\n    }\n\n    public boolean isAppendCkAsync() {\n        return appendCkAsync;\n    }\n\n    public void setAppendCkAsync(boolean appendCkAsync) {\n        this.appendCkAsync = appendCkAsync;\n    }\n\n    public boolean isClearRetryTopicWhenDeleteTopic() {\n        return clearRetryTopicWhenDeleteTopic;\n    }\n\n    public void setClearRetryTopicWhenDeleteTopic(boolean clearRetryTopicWhenDeleteTopic) {\n        this.clearRetryTopicWhenDeleteTopic = clearRetryTopicWhenDeleteTopic;\n    }\n\n    public boolean isEnableLmqStats() {\n        return enableLmqStats;\n    }\n\n    public void setEnableLmqStats(boolean enableLmqStats) {\n        this.enableLmqStats = enableLmqStats;\n    }\n\n    public String getConfigManagerVersion() {\n        return configManagerVersion;\n    }\n\n    public void setConfigManagerVersion(String configManagerVersion) {\n        this.configManagerVersion = configManagerVersion;\n    }\n\n    public boolean isUseSingleRocksDBForAllConfigs() {\n        return useSingleRocksDBForAllConfigs;\n    }\n\n    public void setUseSingleRocksDBForAllConfigs(boolean useSingleRocksDBForAllConfigs) {\n        this.useSingleRocksDBForAllConfigs = useSingleRocksDBForAllConfigs;\n    }\n\n    public boolean isAllowRecallWhenBrokerNotWriteable() {\n        return allowRecallWhenBrokerNotWriteable;\n    }\n\n    public void setAllowRecallWhenBrokerNotWriteable(boolean allowRecallWhenBrokerNotWriteable) {\n        this.allowRecallWhenBrokerNotWriteable = allowRecallWhenBrokerNotWriteable;\n    }\n\n    public boolean isRecallMessageEnable() {\n        return recallMessageEnable;\n    }\n\n    public void setRecallMessageEnable(boolean recallMessageEnable) {\n        this.recallMessageEnable = recallMessageEnable;\n    }\n\n    public boolean isEnableRegisterProducer() {\n        return enableRegisterProducer;\n    }\n\n    public void setEnableRegisterProducer(boolean enableRegisterProducer) {\n        this.enableRegisterProducer = enableRegisterProducer;\n    }\n\n    public boolean isEnableCreateSysGroup() {\n        return enableCreateSysGroup;\n    }\n\n    public void setEnableCreateSysGroup(boolean enableCreateSysGroup) {\n        this.enableCreateSysGroup = enableCreateSysGroup;\n    }\n\n    public boolean isEnableSplitMetadata() {\n        return enableSplitMetadata;\n    }\n\n    public void setEnableSplitMetadata(boolean enableSplitMetadata) {\n        this.enableSplitMetadata = enableSplitMetadata;\n    }\n\n    public int getSplitMetadataSize() {\n        return splitMetadataSize;\n    }\n\n    public void setSplitMetadataSize(int splitMetadataSize) {\n        this.splitMetadataSize = splitMetadataSize;\n    }\n\n    public int getPopFromRetryProbabilityForPriority() {\n        return popFromRetryProbabilityForPriority;\n    }\n\n    public void setPopFromRetryProbabilityForPriority(int popFromRetryProbabilityForPriority) {\n        this.popFromRetryProbabilityForPriority = popFromRetryProbabilityForPriority;\n    }\n\n    public boolean isPriorityOrderAsc() {\n        return priorityOrderAsc;\n    }\n\n    public void setPriorityOrderAsc(boolean priorityOrderAsc) {\n        this.priorityOrderAsc = priorityOrderAsc;\n    }\n\n    public boolean isUseSeparateRetryQueue() {\n        return useSeparateRetryQueue;\n    }\n\n    public void setUseSeparateRetryQueue(boolean useSeparateRetryQueue) {\n        this.useSeparateRetryQueue = useSeparateRetryQueue;\n    }\n\n    public boolean isEnableLiteEventMode() {\n        return enableLiteEventMode;\n    }\n\n    public void setEnableLiteEventMode(boolean enableLiteEventMode) {\n        this.enableLiteEventMode = enableLiteEventMode;\n    }\n\n    public long getLiteEventCheckInterval() {\n        return liteEventCheckInterval;\n    }\n\n    public void setLiteEventCheckInterval(long liteEventCheckInterval) {\n        this.liteEventCheckInterval = liteEventCheckInterval;\n    }\n\n    public long getLiteTtlCheckInterval() {\n        return liteTtlCheckInterval;\n    }\n\n    public void setLiteTtlCheckInterval(long liteTtlCheckInterval) {\n        this.liteTtlCheckInterval = liteTtlCheckInterval;\n    }\n\n    public long getMinLiteTTl() {\n        return minLiteTTl;\n    }\n\n    public void setMinLiteTTl(long minLiteTTl) {\n        this.minLiteTTl = minLiteTTl;\n    }\n\n    public long getLiteSubscriptionCheckInterval() {\n        return liteSubscriptionCheckInterval;\n    }\n\n    public void setLiteSubscriptionCheckInterval(long liteSubscriptionCheckInterval) {\n        this.liteSubscriptionCheckInterval = liteSubscriptionCheckInterval;\n    }\n\n    public long getLiteSubscriptionCheckTimeoutMills() {\n        return liteSubscriptionCheckTimeoutMills;\n    }\n\n    public void setLiteSubscriptionCheckTimeoutMills(long liteSubscriptionCheckTimeoutMills) {\n        this.liteSubscriptionCheckTimeoutMills = liteSubscriptionCheckTimeoutMills;\n    }\n\n    public boolean isPersistConsumerOffsetIncrementally() {\n        return persistConsumerOffsetIncrementally;\n    }\n\n    public void setPersistConsumerOffsetIncrementally(boolean persistConsumerOffsetIncrementally) {\n        this.persistConsumerOffsetIncrementally = persistConsumerOffsetIncrementally;\n    }\n\n    public long getMaxLiteSubscriptionCount() {\n        return maxLiteSubscriptionCount;\n    }\n\n    public void setMaxLiteSubscriptionCount(long maxLiteSubscriptionCount) {\n        this.maxLiteSubscriptionCount = maxLiteSubscriptionCount;\n    }\n\n    public boolean isEnableLitePopLog() {\n        return enableLitePopLog;\n    }\n\n    public void setEnableLitePopLog(boolean enableLitePopLog) {\n        this.enableLitePopLog = enableLitePopLog;\n    }\n\n    public int getMaxClientEventCount() {\n        return maxClientEventCount;\n    }\n\n    public void setMaxClientEventCount(int maxClientEventCount) {\n        this.maxClientEventCount = maxClientEventCount;\n    }\n\n    public long getLiteEventFullDispatchDelayTime() {\n        return liteEventFullDispatchDelayTime;\n    }\n\n    public void setLiteEventFullDispatchDelayTime(long liteEventFullDispatchDelayTime) {\n        this.liteEventFullDispatchDelayTime = liteEventFullDispatchDelayTime;\n    }\n\n    public boolean isLiteLagLatencyCollectEnable() {\n        return liteLagLatencyCollectEnable;\n    }\n\n    public void setLiteLagLatencyCollectEnable(boolean liteLagLatencyCollectEnable) {\n        this.liteLagLatencyCollectEnable = liteLagLatencyCollectEnable;\n    }\n\n    public boolean isLiteLagLatencyMetricsEnable() {\n        return liteLagLatencyMetricsEnable;\n    }\n\n    public void setLiteLagLatencyMetricsEnable(boolean liteLagLatencyMetricsEnable) {\n        this.liteLagLatencyMetricsEnable = liteLagLatencyMetricsEnable;\n    }\n\n    public boolean isLiteLagCountMetricsEnable() {\n        return liteLagCountMetricsEnable;\n    }\n\n    public void setLiteLagCountMetricsEnable(boolean liteLagCountMetricsEnable) {\n        this.liteLagCountMetricsEnable = liteLagCountMetricsEnable;\n    }\n\n    public int getLiteLagLatencyTopK() {\n        return liteLagLatencyTopK;\n    }\n\n    public void setLiteLagLatencyTopK(int liteLagLatencyTopK) {\n        this.liteLagLatencyTopK = liteLagLatencyTopK;\n    }\n\n    public boolean isUseMessageFilterForNotification() {\n        return useMessageFilterForNotification;\n    }\n\n    public void setUseMessageFilterForNotification(boolean useMessageFilterForNotification) {\n        this.useMessageFilterForNotification = useMessageFilterForNotification;\n    }\n\n    public int getMaxMessageFilterNumForNotification() {\n        return maxMessageFilterNumForNotification;\n    }\n\n    public void setMaxMessageFilterNumForNotification(int maxMessageFilterNumForNotification) {\n        this.maxMessageFilterNumForNotification = maxMessageFilterNumForNotification;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/BrokerConfigSingleton.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common;\n\nimport java.util.concurrent.atomic.AtomicBoolean;\n\npublic class BrokerConfigSingleton {\n    private static AtomicBoolean isInit = new AtomicBoolean();\n    private static BrokerConfig brokerConfig;\n\n    public static BrokerConfig getBrokerConfig() {\n        if (brokerConfig == null) {\n            throw new IllegalArgumentException(\"brokerConfig Cannot be null !\");\n        }\n        return brokerConfig;\n    }\n\n    public static void setBrokerConfig(BrokerConfig brokerConfig) {\n        if (!isInit.compareAndSet(false, true)) {\n            throw new IllegalArgumentException(\"broker config have inited !\");\n        }\n        BrokerConfigSingleton.brokerConfig = brokerConfig;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/BrokerIdentity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.commons.lang3.builder.EqualsBuilder;\nimport org.apache.commons.lang3.builder.HashCodeBuilder;\nimport org.apache.rocketmq.common.annotation.ImportantField;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\nimport java.net.InetAddress;\nimport java.net.UnknownHostException;\n\npublic class BrokerIdentity {\n    private static final String DEFAULT_CLUSTER_NAME = \"DefaultCluster\";\n\n    protected static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);\n\n    private static String localHostName;\n\n    static {\n        try {\n            localHostName = InetAddress.getLocalHost().getHostName();\n        } catch (UnknownHostException e) {\n            LOGGER.error(\"Failed to obtain the host name\", e);\n        }\n    }\n\n    // load it after the localHostName is initialized\n    public static final BrokerIdentity BROKER_CONTAINER_IDENTITY = new BrokerIdentity(true);\n\n    @ImportantField\n    private String brokerName = defaultBrokerName();\n    @ImportantField\n    private String brokerClusterName = DEFAULT_CLUSTER_NAME;\n    @ImportantField\n    private volatile long brokerId = MixAll.MASTER_ID;\n\n    private boolean isBrokerContainer = false;\n\n    // Do not set it manually, it depends on the startup mode\n    // Broker start by BrokerStartup is false, start or add by BrokerContainer is true\n    private boolean isInBrokerContainer = false;\n\n    public BrokerIdentity() {\n    }\n\n    public BrokerIdentity(boolean isBrokerContainer) {\n        this.isBrokerContainer = isBrokerContainer;\n    }\n\n    public BrokerIdentity(String brokerClusterName, String brokerName, long brokerId) {\n        this.brokerName = brokerName;\n        this.brokerClusterName = brokerClusterName;\n        this.brokerId = brokerId;\n    }\n\n    public BrokerIdentity(String brokerClusterName, String brokerName, long brokerId, boolean isInBrokerContainer) {\n        this.brokerName = brokerName;\n        this.brokerClusterName = brokerClusterName;\n        this.brokerId = brokerId;\n        this.isInBrokerContainer = isInBrokerContainer;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(final String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public String getBrokerClusterName() {\n        return brokerClusterName;\n    }\n\n    public void setBrokerClusterName(final String brokerClusterName) {\n        this.brokerClusterName = brokerClusterName;\n    }\n\n    public long getBrokerId() {\n        return brokerId;\n    }\n\n    public void setBrokerId(final long brokerId) {\n        this.brokerId = brokerId;\n    }\n\n    public boolean isInBrokerContainer() {\n        return isInBrokerContainer;\n    }\n\n    public void setInBrokerContainer(boolean inBrokerContainer) {\n        isInBrokerContainer = inBrokerContainer;\n    }\n\n    private String defaultBrokerName() {\n        return StringUtils.isEmpty(localHostName) ? \"DEFAULT_BROKER\" : localHostName;\n    }\n\n    public String getCanonicalName() {\n        return isBrokerContainer ? \"BrokerContainer\" : String.format(\"%s_%s_%d\", brokerClusterName, brokerName,\n            brokerId);\n    }\n\n    public String getIdentifier() {\n        return \"#\" + getCanonicalName() + \"#\";\n    }\n\n    @Override\n    public boolean equals(final Object o) {\n        if (this == o) {\n            return true;\n        }\n\n        if (o == null || getClass() != o.getClass()) {\n            return false;\n        }\n\n        final BrokerIdentity identity = (BrokerIdentity) o;\n\n        return new EqualsBuilder()\n            .append(brokerId, identity.brokerId)\n            .append(brokerName, identity.brokerName)\n            .append(brokerClusterName, identity.brokerClusterName)\n            .isEquals();\n    }\n\n    @Override\n    public int hashCode() {\n        return new HashCodeBuilder(17, 37)\n            .append(brokerName)\n            .append(brokerClusterName)\n            .append(brokerId)\n            .toHashCode();\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/CheckRocksdbCqWriteResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common;\n\npublic class CheckRocksdbCqWriteResult {\n    String checkResult;\n\n    int checkStatus;\n\n    public enum CheckStatus {\n        CHECK_OK(0),\n        CHECK_NOT_OK(1),\n        CHECK_IN_PROGRESS(2),\n        CHECK_ERROR(3);\n\n        private int value;\n\n        CheckStatus(int value) {\n            this.value = value;\n        }\n\n        public int getValue() {\n            return value;\n        }\n    }\n\n    public String getCheckResult() {\n        return checkResult;\n    }\n\n    public void setCheckResult(String checkResult) {\n        this.checkResult = checkResult;\n    }\n\n    public int getCheckStatus() {\n        return checkStatus;\n    }\n\n    public void setCheckStatus(int checkStatus) {\n        this.checkStatus = checkStatus;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/ConfigManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\nimport java.io.File;\nimport java.io.RandomAccessFile;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.nio.file.Paths;\nimport java.nio.file.StandardCopyOption;\nimport java.util.Map;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic abstract class ConfigManager {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);\n\n    public boolean load() {\n        String fileName = null;\n        try {\n            fileName = this.configFilePath();\n            String jsonString = MixAll.file2String(fileName);\n\n            if (null == jsonString || jsonString.length() == 0) {\n                // delete invalid file\n                Files.deleteIfExists(Paths.get(fileName));\n                return this.loadBak();\n            } else {\n                this.decode(jsonString);\n                log.info(\"load \" + fileName + \" OK\");\n                return true;\n            }\n        } catch (Exception e) {\n            log.error(\"load \" + fileName + \" failed, and try to load backup file\", e);\n            try {\n                if (fileName != null) {\n                    // delete invalid file\n                    Files.deleteIfExists(Paths.get(fileName));\n                }\n            } catch (Throwable t) {\n                log.error(\"load \" + fileName + \" failed, and delete invalid file errr\", e);\n            }\n            return this.loadBak();\n        }\n    }\n\n    private boolean loadBak() {\n        String fileName = null;\n        try {\n            fileName = this.configFilePath() + \".bak\";\n            String jsonString = MixAll.file2String(fileName);\n            if (jsonString != null && jsonString.length() > 0) {\n                this.decode(jsonString);\n                log.info(\"load \" + fileName + \" OK\");\n                return true;\n            }\n        } catch (Exception e) {\n            log.error(\"load \" + fileName + \" Failed\", e);\n            return false;\n        }\n\n        return true;\n    }\n\n    public synchronized <T> void persist(String topicName, T t) {\n        // stub for future\n        this.persist();\n    }\n\n    public synchronized <T> void persist(Map<String, T> m) {\n        // stub for future\n        this.persist();\n    }\n\n    public synchronized void persist() {\n        String jsonString = this.encode(true);\n        if (jsonString != null) {\n            try {\n                // bak metrics file\n                String config = configFilePath();\n                String backup = config + \".bak\";\n                File configFile = new File(config);\n                File bakFile = new File(backup);\n\n                if (configFile.exists()) {\n                    // atomic move\n                    Files.move(configFile.toPath(), bakFile.toPath(), StandardCopyOption.ATOMIC_MOVE);\n\n                    // sync the directory, ensure that the bak file is visible\n                    MixAll.fsyncDirectory(Paths.get(bakFile.getParent()));\n                }\n\n                File dir = new File(configFile.getParent());\n                if (!dir.exists()) {\n                    Files.createDirectories(dir.toPath());\n                }\n\n                try (RandomAccessFile randomAccessFile = new RandomAccessFile(config, \"rw\")) {\n                    randomAccessFile.write(jsonString.getBytes(StandardCharsets.UTF_8));\n                    randomAccessFile.getChannel().force(true);\n                    // sync the directory, ensure that the config file is visible\n                    MixAll.fsyncDirectory(Paths.get(configFile.getParent()));\n                }\n            } catch (Throwable t) {\n                log.error(\"Failed to persist\", t);\n            }\n        }\n    }\n\n    public boolean stop() {\n        return true;\n    }\n\n    public void shutdown() {\n        stop();\n    }\n\n    public abstract String configFilePath();\n\n    public abstract String encode();\n\n    public abstract String encode(final boolean prettyFormat);\n\n    public abstract void decode(final String jsonString);\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/ControllerConfig.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\nimport java.io.File;\nimport java.util.Arrays;\nimport org.apache.rocketmq.common.metrics.MetricsExporterType;\n\npublic class ControllerConfig {\n    private String rocketmqHome = MixAll.ROCKETMQ_HOME_DIR;\n    private String configStorePath = System.getProperty(\"user.home\") + File.separator + \"controller\" + File.separator + \"controller.properties\";\n    public static final String DLEDGER_CONTROLLER = \"DLedger\";\n    public static final String JRAFT_CONTROLLER = \"jRaft\";\n\n    private JraftConfig jraftConfig = new JraftConfig();\n\n    private String controllerType = DLEDGER_CONTROLLER;\n    /**\n     * Interval of periodic scanning for non-active broker;\n     * Unit: millisecond\n     */\n    private long scanNotActiveBrokerInterval = 5 * 1000;\n\n    /**\n     * Indicates the nums of thread to handle broker or operation requests, like REGISTER_BROKER.\n     */\n    private int controllerThreadPoolNums = 16;\n\n    /**\n     * Indicates the capacity of queue to hold client requests.\n     */\n    private int controllerRequestThreadPoolQueueCapacity = 50000;\n\n    private String controllerDLegerGroup;\n    private String controllerDLegerPeers;\n    private String controllerDLegerSelfId;\n    private int mappedFileSize = 1024 * 1024 * 1024;\n    private String controllerStorePath = \"\";\n\n    /**\n     * Max retry count for electing master when failed because of network or system error.\n     */\n    private int electMasterMaxRetryCount = 3;\n\n\n    /**\n     * Whether the controller can elect a master which is not in the syncStateSet.\n     */\n    private boolean enableElectUncleanMaster = false;\n\n    /**\n     * Whether process read event\n     */\n    private boolean isProcessReadEvent = false;\n\n    /**\n     * Whether notify broker when its role changed\n     */\n    private volatile boolean notifyBrokerRoleChanged = true;\n    /**\n     * Interval of periodic scanning for non-active master in each broker-set;\n     * Unit: millisecond\n     */\n    private long scanInactiveMasterInterval = 5 * 1000;\n\n    private MetricsExporterType metricsExporterType = MetricsExporterType.DISABLE;\n\n    private String metricsGrpcExporterTarget = \"\";\n    private String metricsGrpcExporterHeader = \"\";\n    private long metricGrpcExporterTimeOutInMills = 3 * 1000;\n    private long metricGrpcExporterIntervalInMills = 60 * 1000;\n    private long metricLoggingExporterIntervalInMills = 10 * 1000;\n\n    private int metricsPromExporterPort = 5557;\n    private String metricsPromExporterHost = \"\";\n\n    // Label pairs in CSV. Each label follows pattern of Key:Value. eg: instance_id:xxx,uid:xxx\n    private String metricsLabel = \"\";\n\n    private boolean metricsInDelta = false;\n\n    /**\n     * Config in this black list will be not allowed to update by command.\n     * Try to update this config black list by restart process.\n     * Try to update configures in black list by restart process.\n     */\n    private String configBlackList = \"configBlackList;configStorePath\";\n\n    public String getConfigBlackList() {\n        return configBlackList;\n    }\n\n    public void setConfigBlackList(String configBlackList) {\n        this.configBlackList = configBlackList;\n    }\n\n    public String getRocketmqHome() {\n        return rocketmqHome;\n    }\n\n    public void setRocketmqHome(String rocketmqHome) {\n        this.rocketmqHome = rocketmqHome;\n    }\n\n    public String getConfigStorePath() {\n        return configStorePath;\n    }\n\n    public void setConfigStorePath(String configStorePath) {\n        this.configStorePath = configStorePath;\n    }\n\n    public long getScanNotActiveBrokerInterval() {\n        return scanNotActiveBrokerInterval;\n    }\n\n    public void setScanNotActiveBrokerInterval(long scanNotActiveBrokerInterval) {\n        this.scanNotActiveBrokerInterval = scanNotActiveBrokerInterval;\n    }\n\n    public int getControllerThreadPoolNums() {\n        return controllerThreadPoolNums;\n    }\n\n    public void setControllerThreadPoolNums(int controllerThreadPoolNums) {\n        this.controllerThreadPoolNums = controllerThreadPoolNums;\n    }\n\n    public int getControllerRequestThreadPoolQueueCapacity() {\n        return controllerRequestThreadPoolQueueCapacity;\n    }\n\n    public void setControllerRequestThreadPoolQueueCapacity(int controllerRequestThreadPoolQueueCapacity) {\n        this.controllerRequestThreadPoolQueueCapacity = controllerRequestThreadPoolQueueCapacity;\n    }\n\n    public String getControllerDLegerGroup() {\n        return controllerDLegerGroup;\n    }\n\n    public void setControllerDLegerGroup(String controllerDLegerGroup) {\n        this.controllerDLegerGroup = controllerDLegerGroup;\n    }\n\n    public String getControllerDLegerPeers() {\n        return controllerDLegerPeers;\n    }\n\n    public void setControllerDLegerPeers(String controllerDLegerPeers) {\n        this.controllerDLegerPeers = controllerDLegerPeers;\n    }\n\n    public String getControllerDLegerSelfId() {\n        return controllerDLegerSelfId;\n    }\n\n    public void setControllerDLegerSelfId(String controllerDLegerSelfId) {\n        this.controllerDLegerSelfId = controllerDLegerSelfId;\n    }\n\n    public int getMappedFileSize() {\n        return mappedFileSize;\n    }\n\n    public void setMappedFileSize(int mappedFileSize) {\n        this.mappedFileSize = mappedFileSize;\n    }\n\n    public String getControllerStorePath() {\n        if (controllerStorePath.isEmpty()) {\n            controllerStorePath = System.getProperty(\"user.home\") + File.separator + controllerType + \"Controller\";\n        }\n        return controllerStorePath;\n    }\n\n    public void setControllerStorePath(String controllerStorePath) {\n        this.controllerStorePath = controllerStorePath;\n    }\n\n    public boolean isEnableElectUncleanMaster() {\n        return enableElectUncleanMaster;\n    }\n\n    public void setEnableElectUncleanMaster(boolean enableElectUncleanMaster) {\n        this.enableElectUncleanMaster = enableElectUncleanMaster;\n    }\n\n    public boolean isProcessReadEvent() {\n        return isProcessReadEvent;\n    }\n\n    public void setProcessReadEvent(boolean processReadEvent) {\n        isProcessReadEvent = processReadEvent;\n    }\n\n    public boolean isNotifyBrokerRoleChanged() {\n        return notifyBrokerRoleChanged;\n    }\n\n    public void setNotifyBrokerRoleChanged(boolean notifyBrokerRoleChanged) {\n        this.notifyBrokerRoleChanged = notifyBrokerRoleChanged;\n    }\n\n    public long getScanInactiveMasterInterval() {\n        return scanInactiveMasterInterval;\n    }\n\n    public void setScanInactiveMasterInterval(long scanInactiveMasterInterval) {\n        this.scanInactiveMasterInterval = scanInactiveMasterInterval;\n    }\n\n    public String getDLedgerAddress() {\n        return Arrays.stream(this.controllerDLegerPeers.split(\";\"))\n            .filter(x -> this.controllerDLegerSelfId.equals(x.split(\"-\")[0]))\n            .map(x -> x.split(\"-\")[1]).findFirst().get();\n    }\n\n    public MetricsExporterType getMetricsExporterType() {\n        return metricsExporterType;\n    }\n\n    public void setMetricsExporterType(MetricsExporterType metricsExporterType) {\n        this.metricsExporterType = metricsExporterType;\n    }\n\n    public void setMetricsExporterType(int metricsExporterType) {\n        this.metricsExporterType = MetricsExporterType.valueOf(metricsExporterType);\n    }\n\n    public void setMetricsExporterType(String metricsExporterType) {\n        this.metricsExporterType = MetricsExporterType.valueOf(metricsExporterType);\n    }\n\n    public String getMetricsGrpcExporterTarget() {\n        return metricsGrpcExporterTarget;\n    }\n\n    public void setMetricsGrpcExporterTarget(String metricsGrpcExporterTarget) {\n        this.metricsGrpcExporterTarget = metricsGrpcExporterTarget;\n    }\n\n    public String getMetricsGrpcExporterHeader() {\n        return metricsGrpcExporterHeader;\n    }\n\n    public void setMetricsGrpcExporterHeader(String metricsGrpcExporterHeader) {\n        this.metricsGrpcExporterHeader = metricsGrpcExporterHeader;\n    }\n\n    public long getMetricGrpcExporterTimeOutInMills() {\n        return metricGrpcExporterTimeOutInMills;\n    }\n\n    public void setMetricGrpcExporterTimeOutInMills(long metricGrpcExporterTimeOutInMills) {\n        this.metricGrpcExporterTimeOutInMills = metricGrpcExporterTimeOutInMills;\n    }\n\n    public long getMetricGrpcExporterIntervalInMills() {\n        return metricGrpcExporterIntervalInMills;\n    }\n\n    public void setMetricGrpcExporterIntervalInMills(long metricGrpcExporterIntervalInMills) {\n        this.metricGrpcExporterIntervalInMills = metricGrpcExporterIntervalInMills;\n    }\n\n    public long getMetricLoggingExporterIntervalInMills() {\n        return metricLoggingExporterIntervalInMills;\n    }\n\n    public void setMetricLoggingExporterIntervalInMills(long metricLoggingExporterIntervalInMills) {\n        this.metricLoggingExporterIntervalInMills = metricLoggingExporterIntervalInMills;\n    }\n\n    public int getMetricsPromExporterPort() {\n        return metricsPromExporterPort;\n    }\n\n    public void setMetricsPromExporterPort(int metricsPromExporterPort) {\n        this.metricsPromExporterPort = metricsPromExporterPort;\n    }\n\n    public String getMetricsPromExporterHost() {\n        return metricsPromExporterHost;\n    }\n\n    public void setMetricsPromExporterHost(String metricsPromExporterHost) {\n        this.metricsPromExporterHost = metricsPromExporterHost;\n    }\n\n    public String getMetricsLabel() {\n        return metricsLabel;\n    }\n\n    public void setMetricsLabel(String metricsLabel) {\n        this.metricsLabel = metricsLabel;\n    }\n\n    public boolean isMetricsInDelta() {\n        return metricsInDelta;\n    }\n\n    public void setMetricsInDelta(boolean metricsInDelta) {\n        this.metricsInDelta = metricsInDelta;\n    }\n\n    public String getControllerType() {\n        return controllerType;\n    }\n\n    public void setControllerType(String controllerType) {\n        this.controllerType = controllerType;\n    }\n\n    public JraftConfig getJraftConfig() {\n        return jraftConfig;\n    }\n\n    public void setJraftConfig(JraftConfig jraftConfig) {\n        this.jraftConfig = jraftConfig;\n    }\n\n    public int getElectMasterMaxRetryCount() {\n        return this.electMasterMaxRetryCount;\n    }\n\n    public void setElectMasterMaxRetryCount(int electMasterMaxRetryCount) {\n        this.electMasterMaxRetryCount = electMasterMaxRetryCount;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/CountDownLatch2.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common;\n\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.locks.AbstractQueuedSynchronizer;\n\n/**\n * Add reset feature for @see java.util.concurrent.CountDownLatch\n */\npublic class CountDownLatch2 {\n    private final Sync sync;\n\n    /**\n     * Constructs a {@code CountDownLatch2} initialized with the given count.\n     *\n     * @param count the number of times {@link #countDown} must be invoked before threads can pass through {@link\n     * #await}\n     * @throws IllegalArgumentException if {@code count} is negative\n     */\n    public CountDownLatch2(int count) {\n        if (count < 0)\n            throw new IllegalArgumentException(\"count < 0\");\n        this.sync = new Sync(count);\n    }\n\n    /**\n     * Causes the current thread to wait until the latch has counted down to\n     * zero, unless the thread is {@linkplain Thread#interrupt interrupted}.\n     *\n     * <p>If the current count is zero then this method returns immediately.\n     *\n     * <p>If the current count is greater than zero then the current\n     * thread becomes disabled for thread scheduling purposes and lies\n     * dormant until one of two things happen:\n     * <ul>\n     * <li>The count reaches zero due to invocations of the\n     * {@link #countDown} method; or\n     * <li>Some other thread {@linkplain Thread#interrupt interrupts}\n     * the current thread.\n     * </ul>\n     *\n     * <p>If the current thread:\n     * <ul>\n     * <li>has its interrupted status set on entry to this method; or\n     * <li>is {@linkplain Thread#interrupt interrupted} while waiting,\n     * </ul>\n     * then {@link InterruptedException} is thrown and the current thread's\n     * interrupted status is cleared.\n     *\n     * @throws InterruptedException if the current thread is interrupted while waiting\n     */\n    public void await() throws InterruptedException {\n        sync.acquireSharedInterruptibly(1);\n    }\n\n    /**\n     * Causes the current thread to wait until the latch has counted down to\n     * zero, unless the thread is {@linkplain Thread#interrupt interrupted},\n     * or the specified waiting time elapses.\n     *\n     * <p>If the current count is zero then this method returns immediately\n     * with the value {@code true}.\n     *\n     * <p>If the current count is greater than zero then the current\n     * thread becomes disabled for thread scheduling purposes and lies\n     * dormant until one of three things happen:\n     * <ul>\n     * <li>The count reaches zero due to invocations of the\n     * {@link #countDown} method; or\n     * <li>Some other thread {@linkplain Thread#interrupt interrupts}\n     * the current thread; or\n     * <li>The specified waiting time elapses.\n     * </ul>\n     *\n     * <p>If the count reaches zero then the method returns with the\n     * value {@code true}.\n     *\n     * <p>If the current thread:\n     * <ul>\n     * <li>has its interrupted status set on entry to this method; or\n     * <li>is {@linkplain Thread#interrupt interrupted} while waiting,\n     * </ul>\n     * then {@link InterruptedException} is thrown and the current thread's\n     * interrupted status is cleared.\n     *\n     * <p>If the specified waiting time elapses then the value {@code false}\n     * is returned.  If the time is less than or equal to zero, the method\n     * will not wait at all.\n     *\n     * @param timeout the maximum time to wait\n     * @param unit the time unit of the {@code timeout} argument\n     * @return {@code true} if the count reached zero and {@code false} if the waiting time elapsed before the count\n     * reached zero\n     * @throws InterruptedException if the current thread is interrupted while waiting\n     */\n    public boolean await(long timeout, TimeUnit unit)\n        throws InterruptedException {\n        return sync.tryAcquireSharedNanos(1, unit.toNanos(timeout));\n    }\n\n    /**\n     * Decrements the count of the latch, releasing all waiting threads if\n     * the count reaches zero.\n     *\n     * <p>If the current count is greater than zero then it is decremented.\n     * If the new count is zero then all waiting threads are re-enabled for\n     * thread scheduling purposes.\n     *\n     * <p>If the current count equals zero then nothing happens.\n     */\n    public void countDown() {\n        sync.releaseShared(1);\n    }\n\n    /**\n     * Returns the current count.\n     *\n     * <p>This method is typically used for debugging and testing purposes.\n     *\n     * @return the current count\n     */\n    public long getCount() {\n        return sync.getCount();\n    }\n\n    public void reset() {\n        sync.reset();\n    }\n\n    /**\n     * Returns a string identifying this latch, as well as its state.\n     * The state, in brackets, includes the String {@code \"Count =\"}\n     * followed by the current count.\n     *\n     * @return a string identifying this latch, as well as its state\n     */\n    public String toString() {\n        return super.toString() + \"[Count = \" + sync.getCount() + \"]\";\n    }\n\n    /**\n     * Synchronization control For CountDownLatch2.\n     * Uses AQS state to represent count.\n     */\n    private static final class Sync extends AbstractQueuedSynchronizer {\n        private static final long serialVersionUID = 4982264981922014374L;\n\n        private final int startCount;\n\n        Sync(int count) {\n            this.startCount = count;\n            setState(count);\n        }\n\n        int getCount() {\n            return getState();\n        }\n\n        @Override\n        protected int tryAcquireShared(int acquires) {\n            return (getState() == 0) ? 1 : -1;\n        }\n\n        @Override\n        protected boolean tryReleaseShared(int releases) {\n            // Decrement count; signal when transition to zero\n            for (; ; ) {\n                int c = getState();\n                if (c == 0)\n                    return false;\n                int nextc = c - 1;\n                if (compareAndSetState(c, nextc))\n                    return nextc == 0;\n            }\n        }\n\n        protected void reset() {\n            setState(startCount);\n        }\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/JraftConfig.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\npublic class JraftConfig {\n    private int jRaftElectionTimeoutMs = 1000;\n\n    private int jRaftScanWaitTimeoutMs = 1000;\n    private int jRaftSnapshotIntervalSecs = 3600;\n    private String jRaftGroupId = \"jRaft-Controller\";\n    private String jRaftServerId = \"localhost:9880\";\n    private String jRaftInitConf = \"localhost:9880,localhost:9881,localhost:9882\";\n    private String jRaftControllerRPCAddr = \"localhost:9770,localhost:9771,localhost:9772\";\n\n    public int getjRaftElectionTimeoutMs() {\n        return jRaftElectionTimeoutMs;\n    }\n\n    public void setjRaftElectionTimeoutMs(int jRaftElectionTimeoutMs) {\n        this.jRaftElectionTimeoutMs = jRaftElectionTimeoutMs;\n    }\n\n    public int getjRaftSnapshotIntervalSecs() {\n        return jRaftSnapshotIntervalSecs;\n    }\n\n    public void setjRaftSnapshotIntervalSecs(int jRaftSnapshotIntervalSecs) {\n        this.jRaftSnapshotIntervalSecs = jRaftSnapshotIntervalSecs;\n    }\n\n    public String getjRaftGroupId() {\n        return jRaftGroupId;\n    }\n\n    public void setjRaftGroupId(String jRaftGroupId) {\n        this.jRaftGroupId = jRaftGroupId;\n    }\n\n    public String getjRaftServerId() {\n        return jRaftServerId;\n    }\n\n    public void setjRaftServerId(String jRaftServerId) {\n        this.jRaftServerId = jRaftServerId;\n    }\n\n    public String getjRaftInitConf() {\n        return jRaftInitConf;\n    }\n\n    public void setjRaftInitConf(String jRaftInitConf) {\n        this.jRaftInitConf = jRaftInitConf;\n    }\n\n    public String getjRaftControllerRPCAddr() {\n        return jRaftControllerRPCAddr;\n    }\n\n    public void setjRaftControllerRPCAddr(String jRaftControllerRPCAddr) {\n        this.jRaftControllerRPCAddr = jRaftControllerRPCAddr;\n    }\n\n    public String getjRaftAddress() {\n        return this.jRaftServerId;\n    }\n\n    public int getjRaftScanWaitTimeoutMs() {\n        return jRaftScanWaitTimeoutMs;\n    }\n\n    public void setjRaftScanWaitTimeoutMs(int jRaftScanWaitTimeoutMs) {\n        this.jRaftScanWaitTimeoutMs = jRaftScanWaitTimeoutMs;\n    }\n}"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/KeyBuilder.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\npublic class KeyBuilder {\n    public static final int POP_ORDER_REVIVE_QUEUE = 999;\n    private static final char POP_RETRY_SEPARATOR_V1 = '_';\n    private static final char POP_RETRY_SEPARATOR_V2 = '+';\n    private static final String POP_RETRY_REGEX_SEPARATOR_V2 = \"\\\\+\";\n\n    public static String buildPopRetryTopic(String topic, String cid, boolean enableRetryV2) {\n        if (enableRetryV2) {\n            return buildPopRetryTopicV2(topic, cid);\n        }\n        return buildPopRetryTopicV1(topic, cid);\n    }\n\n    public static String buildPopRetryTopic(String topic, String cid) {\n        return MixAll.RETRY_GROUP_TOPIC_PREFIX + cid + POP_RETRY_SEPARATOR_V1 + topic;\n    }\n\n    public static String buildPopRetryTopicV2(String topic, String cid) {\n        return MixAll.RETRY_GROUP_TOPIC_PREFIX + cid + POP_RETRY_SEPARATOR_V2 + topic;\n    }\n\n    public static String buildPopRetryTopicV1(String topic, String cid) {\n        return MixAll.RETRY_GROUP_TOPIC_PREFIX + cid + POP_RETRY_SEPARATOR_V1 + topic;\n    }\n\n    public static String parseNormalTopic(String topic, String cid) {\n        if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n            if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX + cid + POP_RETRY_SEPARATOR_V2)) {\n                return topic.substring((MixAll.RETRY_GROUP_TOPIC_PREFIX + cid + POP_RETRY_SEPARATOR_V2).length());\n            }\n            return topic.substring((MixAll.RETRY_GROUP_TOPIC_PREFIX + cid + POP_RETRY_SEPARATOR_V1).length());\n        } else {\n            return topic;\n        }\n    }\n\n    public static String parseNormalTopic(String retryTopic) {\n        if (isPopRetryTopicV2(retryTopic)) {\n            String[] result = retryTopic.split(POP_RETRY_REGEX_SEPARATOR_V2);\n            if (result.length == 2) {\n                return result[1];\n            }\n        }\n        return retryTopic;\n    }\n\n    public static String parseGroup(String retryTopic) {\n        if (isPopRetryTopicV2(retryTopic)) {\n            String[] result = retryTopic.split(POP_RETRY_REGEX_SEPARATOR_V2);\n            if (result.length == 2) {\n                return result[0].substring(MixAll.RETRY_GROUP_TOPIC_PREFIX.length());\n            }\n        }\n        return retryTopic.substring(MixAll.RETRY_GROUP_TOPIC_PREFIX.length());\n    }\n\n    public static String buildPollingKey(String topic, String cid, int queueId) {\n        return topic + PopAckConstants.SPLIT + cid + PopAckConstants.SPLIT + queueId;\n    }\n\n    public static boolean isPopRetryTopicV2(String retryTopic) {\n        return retryTopic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) && retryTopic.contains(String.valueOf(POP_RETRY_SEPARATOR_V2));\n    }\n\n    public static String buildPopLiteLockKey(String group, String lmqName) {\n        return group + PopAckConstants.SPLIT + lmqName;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/LifecycleAwareServiceThread.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicBoolean;\n\npublic abstract class LifecycleAwareServiceThread extends ServiceThread {\n\n    private final AtomicBoolean started = new AtomicBoolean(false);\n\n    @Override\n    public void run() {\n        started.set(true);\n        synchronized (started) {\n            started.notifyAll();\n        }\n\n        run0();\n    }\n\n    public abstract void run0();\n\n    /**\n     * Take spurious wakeup into account.\n     *\n     * @param timeout amount of time in milliseconds\n     * @throws InterruptedException if interrupted\n     */\n    public void awaitStarted(long timeout) throws InterruptedException {\n        long expire = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeout);\n        synchronized (started) {\n            while (!started.get()) {\n                long duration = expire - System.nanoTime();\n                if (duration < TimeUnit.MILLISECONDS.toNanos(1)) {\n                    break;\n                }\n                started.wait(TimeUnit.NANOSECONDS.toMillis(duration));\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/LockCallback.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\nimport java.util.Set;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\npublic interface LockCallback {\n    void onSuccess(final Set<MessageQueue> lockOKMQSet);\n\n    void onException(final Throwable e);\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/MQVersion.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\npublic class MQVersion {\n\n    public static final int CURRENT_VERSION = Version.V5_4_0.ordinal();\n\n    public static String getVersionDesc(int value) {\n        int length = Version.values().length;\n        if (value >= length) {\n            return Version.values()[length - 1].name();\n        }\n\n        return Version.values()[value].name();\n    }\n\n    public static Version value2Version(int value) {\n        int length = Version.values().length;\n        if (value >= length) {\n            return Version.values()[length - 1];\n        }\n\n        return Version.values()[value];\n    }\n\n    public enum Version {\n        V3_0_0_SNAPSHOT,\n        V3_0_0_ALPHA1,\n        V3_0_0_BETA1,\n        V3_0_0_BETA2,\n        V3_0_0_BETA3,\n        V3_0_0_BETA4,\n        V3_0_0_BETA5,\n        V3_0_0_BETA6_SNAPSHOT,\n        V3_0_0_BETA6,\n        V3_0_0_BETA7_SNAPSHOT,\n        V3_0_0_BETA7,\n        V3_0_0_BETA8_SNAPSHOT,\n        V3_0_0_BETA8,\n        V3_0_0_BETA9_SNAPSHOT,\n        V3_0_0_BETA9,\n        V3_0_0_FINAL,\n        V3_0_1_SNAPSHOT,\n        V3_0_1,\n        V3_0_2_SNAPSHOT,\n        V3_0_2,\n        V3_0_3_SNAPSHOT,\n        V3_0_3,\n        V3_0_4_SNAPSHOT,\n        V3_0_4,\n        V3_0_5_SNAPSHOT,\n        V3_0_5,\n        V3_0_6_SNAPSHOT,\n        V3_0_6,\n        V3_0_7_SNAPSHOT,\n        V3_0_7,\n        V3_0_8_SNAPSHOT,\n        V3_0_8,\n        V3_0_9_SNAPSHOT,\n        V3_0_9,\n\n        V3_0_10_SNAPSHOT,\n        V3_0_10,\n\n        V3_0_11_SNAPSHOT,\n        V3_0_11,\n\n        V3_0_12_SNAPSHOT,\n        V3_0_12,\n\n        V3_0_13_SNAPSHOT,\n        V3_0_13,\n\n        V3_0_14_SNAPSHOT,\n        V3_0_14,\n\n        V3_0_15_SNAPSHOT,\n        V3_0_15,\n\n        V3_1_0_SNAPSHOT,\n        V3_1_0,\n\n        V3_1_1_SNAPSHOT,\n        V3_1_1,\n\n        V3_1_2_SNAPSHOT,\n        V3_1_2,\n\n        V3_1_3_SNAPSHOT,\n        V3_1_3,\n\n        V3_1_4_SNAPSHOT,\n        V3_1_4,\n\n        V3_1_5_SNAPSHOT,\n        V3_1_5,\n\n        V3_1_6_SNAPSHOT,\n        V3_1_6,\n\n        V3_1_7_SNAPSHOT,\n        V3_1_7,\n\n        V3_1_8_SNAPSHOT,\n        V3_1_8,\n\n        V3_1_9_SNAPSHOT,\n        V3_1_9,\n\n        V3_2_0_SNAPSHOT,\n        V3_2_0,\n\n        V3_2_1_SNAPSHOT,\n        V3_2_1,\n\n        V3_2_2_SNAPSHOT,\n        V3_2_2,\n\n        V3_2_3_SNAPSHOT,\n        V3_2_3,\n\n        V3_2_4_SNAPSHOT,\n        V3_2_4,\n\n        V3_2_5_SNAPSHOT,\n        V3_2_5,\n\n        V3_2_6_SNAPSHOT,\n        V3_2_6,\n\n        V3_2_7_SNAPSHOT,\n        V3_2_7,\n\n        V3_2_8_SNAPSHOT,\n        V3_2_8,\n\n        V3_2_9_SNAPSHOT,\n        V3_2_9,\n\n        V3_3_1_SNAPSHOT,\n        V3_3_1,\n\n        V3_3_2_SNAPSHOT,\n        V3_3_2,\n\n        V3_3_3_SNAPSHOT,\n        V3_3_3,\n\n        V3_3_4_SNAPSHOT,\n        V3_3_4,\n\n        V3_3_5_SNAPSHOT,\n        V3_3_5,\n\n        V3_3_6_SNAPSHOT,\n        V3_3_6,\n\n        V3_3_7_SNAPSHOT,\n        V3_3_7,\n\n        V3_3_8_SNAPSHOT,\n        V3_3_8,\n\n        V3_3_9_SNAPSHOT,\n        V3_3_9,\n\n        V3_4_1_SNAPSHOT,\n        V3_4_1,\n\n        V3_4_2_SNAPSHOT,\n        V3_4_2,\n\n        V3_4_3_SNAPSHOT,\n        V3_4_3,\n\n        V3_4_4_SNAPSHOT,\n        V3_4_4,\n\n        V3_4_5_SNAPSHOT,\n        V3_4_5,\n\n        V3_4_6_SNAPSHOT,\n        V3_4_6,\n\n        V3_4_7_SNAPSHOT,\n        V3_4_7,\n\n        V3_4_8_SNAPSHOT,\n        V3_4_8,\n\n        V3_4_9_SNAPSHOT,\n        V3_4_9,\n        V3_5_1_SNAPSHOT,\n        V3_5_1,\n\n        V3_5_2_SNAPSHOT,\n        V3_5_2,\n\n        V3_5_3_SNAPSHOT,\n        V3_5_3,\n\n        V3_5_4_SNAPSHOT,\n        V3_5_4,\n\n        V3_5_5_SNAPSHOT,\n        V3_5_5,\n\n        V3_5_6_SNAPSHOT,\n        V3_5_6,\n\n        V3_5_7_SNAPSHOT,\n        V3_5_7,\n\n        V3_5_8_SNAPSHOT,\n        V3_5_8,\n\n        V3_5_9_SNAPSHOT,\n        V3_5_9,\n\n        V3_6_1_SNAPSHOT,\n        V3_6_1,\n\n        V3_6_2_SNAPSHOT,\n        V3_6_2,\n\n        V3_6_3_SNAPSHOT,\n        V3_6_3,\n\n        V3_6_4_SNAPSHOT,\n        V3_6_4,\n\n        V3_6_5_SNAPSHOT,\n        V3_6_5,\n\n        V3_6_6_SNAPSHOT,\n        V3_6_6,\n\n        V3_6_7_SNAPSHOT,\n        V3_6_7,\n\n        V3_6_8_SNAPSHOT,\n        V3_6_8,\n\n        V3_6_9_SNAPSHOT,\n        V3_6_9,\n\n        V3_7_1_SNAPSHOT,\n        V3_7_1,\n\n        V3_7_2_SNAPSHOT,\n        V3_7_2,\n\n        V3_7_3_SNAPSHOT,\n        V3_7_3,\n\n        V3_7_4_SNAPSHOT,\n        V3_7_4,\n\n        V3_7_5_SNAPSHOT,\n        V3_7_5,\n\n        V3_7_6_SNAPSHOT,\n        V3_7_6,\n\n        V3_7_7_SNAPSHOT,\n        V3_7_7,\n\n        V3_7_8_SNAPSHOT,\n        V3_7_8,\n\n        V3_7_9_SNAPSHOT,\n        V3_7_9,\n\n        V3_8_1_SNAPSHOT,\n        V3_8_1,\n\n        V3_8_2_SNAPSHOT,\n        V3_8_2,\n\n        V3_8_3_SNAPSHOT,\n        V3_8_3,\n\n        V3_8_4_SNAPSHOT,\n        V3_8_4,\n\n        V3_8_5_SNAPSHOT,\n        V3_8_5,\n\n        V3_8_6_SNAPSHOT,\n        V3_8_6,\n\n        V3_8_7_SNAPSHOT,\n        V3_8_7,\n\n        V3_8_8_SNAPSHOT,\n        V3_8_8,\n\n        V3_8_9_SNAPSHOT,\n        V3_8_9,\n\n        V3_9_1_SNAPSHOT,\n        V3_9_1,\n\n        V3_9_2_SNAPSHOT,\n        V3_9_2,\n\n        V3_9_3_SNAPSHOT,\n        V3_9_3,\n\n        V3_9_4_SNAPSHOT,\n        V3_9_4,\n\n        V3_9_5_SNAPSHOT,\n        V3_9_5,\n\n        V3_9_6_SNAPSHOT,\n        V3_9_6,\n\n        V3_9_7_SNAPSHOT,\n        V3_9_7,\n\n        V3_9_8_SNAPSHOT,\n        V3_9_8,\n\n        V3_9_9_SNAPSHOT,\n        V3_9_9,\n\n        V4_0_0_SNAPSHOT,\n        V4_0_0,\n\n        V4_0_1_SNAPSHOT,\n        V4_0_1,\n\n        V4_0_2_SNAPSHOT,\n        V4_0_2,\n\n        V4_0_3_SNAPSHOT,\n        V4_0_3,\n\n        V4_0_4_SNAPSHOT,\n        V4_0_4,\n\n        V4_0_5_SNAPSHOT,\n        V4_0_5,\n\n        V4_0_6_SNAPSHOT,\n        V4_0_6,\n\n        V4_0_7_SNAPSHOT,\n        V4_0_7,\n\n        V4_0_8_SNAPSHOT,\n        V4_0_8,\n\n        V4_0_9_SNAPSHOT,\n        V4_0_9,\n\n        V4_1_0_SNAPSHOT,\n        V4_1_0,\n\n        V4_1_1_SNAPSHOT,\n        V4_1_1,\n\n        V4_1_2_SNAPSHOT,\n        V4_1_2,\n\n        V4_1_3_SNAPSHOT,\n        V4_1_3,\n\n        V4_1_4_SNAPSHOT,\n        V4_1_4,\n\n        V4_1_5_SNAPSHOT,\n        V4_1_5,\n\n        V4_1_6_SNAPSHOT,\n        V4_1_6,\n\n        V4_1_7_SNAPSHOT,\n        V4_1_7,\n\n        V4_1_8_SNAPSHOT,\n        V4_1_8,\n\n        V4_1_9_SNAPSHOT,\n        V4_1_9,\n\n        V4_2_0_SNAPSHOT,\n        V4_2_0,\n\n        V4_2_1_SNAPSHOT,\n        V4_2_1,\n\n        V4_2_2_SNAPSHOT,\n        V4_2_2,\n\n        V4_2_3_SNAPSHOT,\n        V4_2_3,\n\n        V4_2_4_SNAPSHOT,\n        V4_2_4,\n\n        V4_2_5_SNAPSHOT,\n        V4_2_5,\n\n        V4_2_6_SNAPSHOT,\n        V4_2_6,\n\n        V4_2_7_SNAPSHOT,\n        V4_2_7,\n\n        V4_2_8_SNAPSHOT,\n        V4_2_8,\n\n        V4_2_9_SNAPSHOT,\n        V4_2_9,\n\n        V4_3_0_SNAPSHOT,\n        V4_3_0,\n\n        V4_3_1_SNAPSHOT,\n        V4_3_1,\n\n        V4_3_2_SNAPSHOT,\n        V4_3_2,\n\n        V4_3_3_SNAPSHOT,\n        V4_3_3,\n\n        V4_3_4_SNAPSHOT,\n        V4_3_4,\n\n        V4_3_5_SNAPSHOT,\n        V4_3_5,\n\n        V4_3_6_SNAPSHOT,\n        V4_3_6,\n\n        V4_3_7_SNAPSHOT,\n        V4_3_7,\n\n        V4_3_8_SNAPSHOT,\n        V4_3_8,\n\n        V4_3_9_SNAPSHOT,\n        V4_3_9,\n\n        V4_4_0_SNAPSHOT,\n        V4_4_0,\n\n        V4_4_1_SNAPSHOT,\n        V4_4_1,\n\n        V4_4_2_SNAPSHOT,\n        V4_4_2,\n\n        V4_4_3_SNAPSHOT,\n        V4_4_3,\n\n        V4_4_4_SNAPSHOT,\n        V4_4_4,\n\n        V4_4_5_SNAPSHOT,\n        V4_4_5,\n\n        V4_4_6_SNAPSHOT,\n        V4_4_6,\n\n        V4_4_7_SNAPSHOT,\n        V4_4_7,\n\n        V4_4_8_SNAPSHOT,\n        V4_4_8,\n\n        V4_4_9_SNAPSHOT,\n        V4_4_9,\n\n        V4_5_0_SNAPSHOT,\n        V4_5_0,\n\n        V4_5_1_SNAPSHOT,\n        V4_5_1,\n\n        V4_5_2_SNAPSHOT,\n        V4_5_2,\n\n        V4_5_3_SNAPSHOT,\n        V4_5_3,\n\n        V4_5_4_SNAPSHOT,\n        V4_5_4,\n\n        V4_5_5_SNAPSHOT,\n        V4_5_5,\n\n        V4_5_6_SNAPSHOT,\n        V4_5_6,\n\n        V4_5_7_SNAPSHOT,\n        V4_5_7,\n\n        V4_5_8_SNAPSHOT,\n        V4_5_8,\n\n        V4_5_9_SNAPSHOT,\n        V4_5_9,\n\n        V4_6_0_SNAPSHOT,\n        V4_6_0,\n\n        V4_6_1_SNAPSHOT,\n        V4_6_1,\n\n        V4_6_2_SNAPSHOT,\n        V4_6_2,\n\n        V4_6_3_SNAPSHOT,\n        V4_6_3,\n\n        V4_6_4_SNAPSHOT,\n        V4_6_4,\n\n        V4_6_5_SNAPSHOT,\n        V4_6_5,\n\n        V4_6_6_SNAPSHOT,\n        V4_6_6,\n\n        V4_6_7_SNAPSHOT,\n        V4_6_7,\n\n        V4_6_8_SNAPSHOT,\n        V4_6_8,\n\n        V4_6_9_SNAPSHOT,\n        V4_6_9,\n\n        V4_7_0_SNAPSHOT,\n        V4_7_0,\n\n        V4_7_1_SNAPSHOT,\n        V4_7_1,\n\n        V4_7_2_SNAPSHOT,\n        V4_7_2,\n\n        V4_7_3_SNAPSHOT,\n        V4_7_3,\n\n        V4_7_4_SNAPSHOT,\n        V4_7_4,\n\n        V4_7_5_SNAPSHOT,\n        V4_7_5,\n\n        V4_7_6_SNAPSHOT,\n        V4_7_6,\n\n        V4_7_7_SNAPSHOT,\n        V4_7_7,\n\n        V4_7_8_SNAPSHOT,\n        V4_7_8,\n\n        V4_7_9_SNAPSHOT,\n        V4_7_9,\n\n        V4_8_0_SNAPSHOT,\n        V4_8_0,\n\n        V4_8_1_SNAPSHOT,\n        V4_8_1,\n\n        V4_8_2_SNAPSHOT,\n        V4_8_2,\n\n        V4_8_3_SNAPSHOT,\n        V4_8_3,\n\n        V4_8_4_SNAPSHOT,\n        V4_8_4,\n\n        V4_8_5_SNAPSHOT,\n        V4_8_5,\n\n        V4_8_6_SNAPSHOT,\n        V4_8_6,\n\n        V4_8_7_SNAPSHOT,\n        V4_8_7,\n\n        V4_8_8_SNAPSHOT,\n        V4_8_8,\n\n        V4_8_9_SNAPSHOT,\n        V4_8_9,\n\n        V4_9_0_SNAPSHOT,\n        V4_9_0,\n\n        V4_9_1_SNAPSHOT,\n        V4_9_1,\n\n        V4_9_2_SNAPSHOT,\n        V4_9_2,\n\n        V4_9_3_SNAPSHOT,\n        V4_9_3,\n\n        V4_9_4_SNAPSHOT,\n        V4_9_4,\n\n        V4_9_5_SNAPSHOT,\n        V4_9_5,\n\n        V4_9_6_SNAPSHOT,\n        V4_9_6,\n\n        V4_9_7_SNAPSHOT,\n        V4_9_7,\n\n        V4_9_8_SNAPSHOT,\n        V4_9_8,\n\n        V4_9_9_SNAPSHOT,\n        V4_9_9,\n\n        V5_0_0_SNAPSHOT,\n        V5_0_0,\n\n        V5_0_1_SNAPSHOT,\n        V5_0_1,\n\n        V5_0_2_SNAPSHOT,\n        V5_0_2,\n\n        V5_0_3_SNAPSHOT,\n        V5_0_3,\n\n        V5_0_4_SNAPSHOT,\n        V5_0_4,\n\n        V5_0_5_SNAPSHOT,\n        V5_0_5,\n\n        V5_0_6_SNAPSHOT,\n        V5_0_6,\n\n        V5_0_7_SNAPSHOT,\n        V5_0_7,\n\n        V5_0_8_SNAPSHOT,\n        V5_0_8,\n\n        V5_0_9_SNAPSHOT,\n        V5_0_9,\n\n        V5_1_0_SNAPSHOT,\n        V5_1_0,\n\n        V5_1_1_SNAPSHOT,\n        V5_1_1,\n\n        V5_1_2_SNAPSHOT,\n        V5_1_2,\n\n        V5_1_3_SNAPSHOT,\n        V5_1_3,\n\n        V5_1_4_SNAPSHOT,\n        V5_1_4,\n\n        V5_1_5_SNAPSHOT,\n        V5_1_5,\n\n        V5_1_6_SNAPSHOT,\n        V5_1_6,\n\n        V5_1_7_SNAPSHOT,\n        V5_1_7,\n\n        V5_1_8_SNAPSHOT,\n        V5_1_8,\n\n        V5_1_9_SNAPSHOT,\n        V5_1_9,\n\n        V5_2_0_SNAPSHOT,\n        V5_2_0,\n\n        V5_2_1_SNAPSHOT,\n        V5_2_1,\n\n        V5_2_2_SNAPSHOT,\n        V5_2_2,\n\n        V5_2_3_SNAPSHOT,\n        V5_2_3,\n\n        V5_2_4_SNAPSHOT,\n        V5_2_4,\n\n        V5_2_5_SNAPSHOT,\n        V5_2_5,\n\n        V5_2_6_SNAPSHOT,\n        V5_2_6,\n\n        V5_2_7_SNAPSHOT,\n        V5_2_7,\n\n        V5_2_8_SNAPSHOT,\n        V5_2_8,\n\n        V5_2_9_SNAPSHOT,\n        V5_2_9,\n\n        V5_3_0_SNAPSHOT,\n        V5_3_0,\n\n        V5_3_1_SNAPSHOT,\n        V5_3_1,\n\n        V5_3_2_SNAPSHOT,\n        V5_3_2,\n\n        V5_3_3_SNAPSHOT,\n        V5_3_3,\n\n        V5_3_4_SNAPSHOT,\n        V5_3_4,\n\n        V5_3_5_SNAPSHOT,\n        V5_3_5,\n\n        V5_3_6_SNAPSHOT,\n        V5_3_6,\n\n        V5_3_7_SNAPSHOT,\n        V5_3_7,\n\n        V5_3_8_SNAPSHOT,\n        V5_3_8,\n\n        V5_3_9_SNAPSHOT,\n        V5_3_9,\n\n        V5_4_0_SNAPSHOT,\n        V5_4_0,\n\n        V5_4_1_SNAPSHOT,\n        V5_4_1,\n\n        V5_4_2_SNAPSHOT,\n        V5_4_2,\n\n        V5_4_3_SNAPSHOT,\n        V5_4_3,\n\n        V5_4_4_SNAPSHOT,\n        V5_4_4,\n\n        V5_4_5_SNAPSHOT,\n        V5_4_5,\n\n        V5_4_6_SNAPSHOT,\n        V5_4_6,\n\n        V5_4_7_SNAPSHOT,\n        V5_4_7,\n\n        V5_4_8_SNAPSHOT,\n        V5_4_8,\n\n        V5_4_9_SNAPSHOT,\n        V5_4_9,\n\n        V5_5_0_SNAPSHOT,\n        V5_5_0,\n\n        V5_5_1_SNAPSHOT,\n        V5_5_1,\n\n        V5_5_2_SNAPSHOT,\n        V5_5_2,\n\n        V5_5_3_SNAPSHOT,\n        V5_5_3,\n\n        V5_5_4_SNAPSHOT,\n        V5_5_4,\n\n        V5_5_5_SNAPSHOT,\n        V5_5_5,\n\n        V5_5_6_SNAPSHOT,\n        V5_5_6,\n\n        V5_5_7_SNAPSHOT,\n        V5_5_7,\n\n        V5_5_8_SNAPSHOT,\n        V5_5_8,\n\n        V5_5_9_SNAPSHOT,\n        V5_5_9,\n\n        V5_6_0_SNAPSHOT,\n        V5_6_0,\n\n        V5_6_1_SNAPSHOT,\n        V5_6_1,\n\n        V5_6_2_SNAPSHOT,\n        V5_6_2,\n\n        V5_6_3_SNAPSHOT,\n        V5_6_3,\n\n        V5_6_4_SNAPSHOT,\n        V5_6_4,\n\n        V5_6_5_SNAPSHOT,\n        V5_6_5,\n\n        V5_6_6_SNAPSHOT,\n        V5_6_6,\n\n        V5_6_7_SNAPSHOT,\n        V5_6_7,\n\n        V5_6_8_SNAPSHOT,\n        V5_6_8,\n\n        V5_6_9_SNAPSHOT,\n        V5_6_9,\n\n        V5_7_0_SNAPSHOT,\n        V5_7_0,\n\n        V5_7_1_SNAPSHOT,\n        V5_7_1,\n\n        V5_7_2_SNAPSHOT,\n        V5_7_2,\n\n        V5_7_3_SNAPSHOT,\n        V5_7_3,\n\n        V5_7_4_SNAPSHOT,\n        V5_7_4,\n\n        V5_7_5_SNAPSHOT,\n        V5_7_5,\n\n        V5_7_6_SNAPSHOT,\n        V5_7_6,\n\n        V5_7_7_SNAPSHOT,\n        V5_7_7,\n\n        V5_7_8_SNAPSHOT,\n        V5_7_8,\n\n        V5_7_9_SNAPSHOT,\n        V5_7_9,\n\n        V5_8_0_SNAPSHOT,\n        V5_8_0,\n\n        V5_8_1_SNAPSHOT,\n        V5_8_1,\n\n        V5_8_2_SNAPSHOT,\n        V5_8_2,\n\n        V5_8_3_SNAPSHOT,\n        V5_8_3,\n\n        V5_8_4_SNAPSHOT,\n        V5_8_4,\n\n        V5_8_5_SNAPSHOT,\n        V5_8_5,\n\n        V5_8_6_SNAPSHOT,\n        V5_8_6,\n\n        V5_8_7_SNAPSHOT,\n        V5_8_7,\n\n        V5_8_8_SNAPSHOT,\n        V5_8_8,\n\n        V5_8_9_SNAPSHOT,\n        V5_8_9,\n\n        V5_9_0_SNAPSHOT,\n        V5_9_0,\n\n        V5_9_1_SNAPSHOT,\n        V5_9_1,\n\n        V5_9_2_SNAPSHOT,\n        V5_9_2,\n\n        V5_9_3_SNAPSHOT,\n        V5_9_3,\n\n        V5_9_4_SNAPSHOT,\n        V5_9_4,\n\n        V5_9_5_SNAPSHOT,\n        V5_9_5,\n\n        V5_9_6_SNAPSHOT,\n        V5_9_6,\n\n        V5_9_7_SNAPSHOT,\n        V5_9_7,\n\n        V5_9_8_SNAPSHOT,\n        V5_9_8,\n\n        V5_9_9_SNAPSHOT,\n        V5_9_9,\n\n        HIGHER_VERSION\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/MixAll.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\nimport java.io.ByteArrayInputStream;\nimport java.io.File;\nimport java.io.FileInputStream;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.Field;\nimport java.lang.reflect.Method;\nimport java.lang.reflect.Modifier;\nimport java.net.Inet6Address;\nimport java.net.InetAddress;\nimport java.net.NetworkInterface;\nimport java.net.SocketException;\nimport java.net.URL;\nimport java.net.URLConnection;\nimport java.nio.channels.FileChannel;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.nio.file.NotDirectoryException;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.nio.file.StandardOpenOption;\nimport java.util.ArrayList;\nimport java.util.Enumeration;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Properties;\nimport java.util.Set;\nimport java.util.TreeMap;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.function.Predicate;\n\nimport com.google.common.collect.ImmutableSet;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.annotation.ImportantField;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.help.FAQUrl;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.common.utils.IOTinyUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class MixAll {\n    public static final String ROCKETMQ_HOME_ENV = \"ROCKETMQ_HOME\";\n    public static final String ROCKETMQ_HOME_PROPERTY = \"rocketmq.home.dir\";\n    /**\n     * unify the home dir\n     */\n    public static final String ROCKETMQ_HOME_DIR = System.getProperty(ROCKETMQ_HOME_PROPERTY, System.getenv(ROCKETMQ_HOME_ENV));\n    public static final String NAMESRV_ADDR_ENV = \"NAMESRV_ADDR\";\n    public static final String NAMESRV_ADDR_PROPERTY = \"rocketmq.namesrv.addr\";\n    public static final String MESSAGE_COMPRESS_TYPE = \"rocketmq.message.compressType\";\n    public static final String MESSAGE_COMPRESS_LEVEL = \"rocketmq.message.compressLevel\";\n    public static final String DEFAULT_NAMESRV_ADDR_LOOKUP = \"jmenv.tbsite.net\";\n    public static final String WS_DOMAIN_NAME = System.getProperty(\"rocketmq.namesrv.domain\", DEFAULT_NAMESRV_ADDR_LOOKUP);\n    public static final String WS_DOMAIN_SUBGROUP = System.getProperty(\"rocketmq.namesrv.domain.subgroup\", \"nsaddr\");\n    public static final String DEFAULT_PRODUCER_GROUP = \"DEFAULT_PRODUCER\";\n    public static final String DEFAULT_CONSUMER_GROUP = \"DEFAULT_CONSUMER\";\n    public static final String TOOLS_CONSUMER_GROUP = \"TOOLS_CONSUMER\";\n    public static final String SCHEDULE_CONSUMER_GROUP = \"SCHEDULE_CONSUMER\";\n    public static final String FILTERSRV_CONSUMER_GROUP = \"FILTERSRV_CONSUMER\";\n    public static final String MONITOR_CONSUMER_GROUP = \"__MONITOR_CONSUMER\";\n    public static final String CLIENT_INNER_PRODUCER_GROUP = \"CLIENT_INNER_PRODUCER\";\n    public static final String SELF_TEST_PRODUCER_GROUP = \"SELF_TEST_P_GROUP\";\n    public static final String SELF_TEST_CONSUMER_GROUP = \"SELF_TEST_C_GROUP\";\n    public static final String ONS_HTTP_PROXY_GROUP = \"CID_ONS-HTTP-PROXY\";\n    public static final String CID_ONSAPI_PERMISSION_GROUP = \"CID_ONSAPI_PERMISSION\";\n    public static final String CID_ONSAPI_OWNER_GROUP = \"CID_ONSAPI_OWNER\";\n    public static final String CID_ONSAPI_PULL_GROUP = \"CID_ONSAPI_PULL\";\n    public static final String CID_RMQ_SYS_PREFIX = \"CID_RMQ_SYS_\";\n    public static final String IS_SUPPORT_HEART_BEAT_V2 = \"IS_SUPPORT_HEART_BEAT_V2\";\n    public static final String IS_SUB_CHANGE = \"IS_SUB_CHANGE\";\n    public static final List<String> LOCAL_INET_ADDRESS = getLocalInetAddress();\n    public static final String LOCALHOST = localhost();\n    public static final String DEFAULT_CHARSET = \"UTF-8\";\n    public static final long MASTER_ID = 0L;\n    public static final long FIRST_SLAVE_ID = 1L;\n\n    public static final long FIRST_BROKER_CONTROLLER_ID = 1L;\n    public static final long CURRENT_JVM_PID = getPID();\n    public final static int UNIT_PRE_SIZE_FOR_MSG = 28;\n    public final static int ALL_ACK_IN_SYNC_STATE_SET = -1;\n\n    public static final String RETRY_GROUP_TOPIC_PREFIX = \"%RETRY%\";\n    public static final String DLQ_GROUP_TOPIC_PREFIX = \"%DLQ%\";\n    public static final String REPLY_TOPIC_POSTFIX = \"REPLY_TOPIC\";\n    public static final String UNIQUE_MSG_QUERY_FLAG = \"_UNIQUE_KEY_QUERY\";\n    public static final String DEFAULT_TRACE_REGION_ID = \"DefaultRegion\";\n    public static final String CONSUME_CONTEXT_TYPE = \"ConsumeContextType\";\n    public static final String CID_SYS_RMQ_TRANS = \"CID_RMQ_SYS_TRANS\";\n    public static final String ACL_CONF_TOOLS_FILE = \"/conf/tools.yml\";\n    public static final String REPLY_MESSAGE_FLAG = \"reply\";\n    public static final String LMQ_PREFIX = \"%LMQ%\";\n    public static final int LMQ_QUEUE_ID = 0;\n    public static final String LMQ_DISPATCH_SEPARATOR = \",\";\n    public static final String REQ_T = \"ReqT\";\n    public static final String ROCKETMQ_ZONE_ENV = \"ROCKETMQ_ZONE\";\n    public static final String ROCKETMQ_ZONE_PROPERTY = \"rocketmq.zone\";\n    public static final String ROCKETMQ_ZONE_MODE_ENV = \"ROCKETMQ_ZONE_MODE\";\n    public static final String ROCKETMQ_ZONE_MODE_PROPERTY = \"rocketmq.zone.mode\";\n    public static final String ZONE_NAME = \"__ZONE_NAME\";\n    public static final String ZONE_MODE = \"__ZONE_MODE\";\n    public final static String RPC_REQUEST_HEADER_NAMESPACED_FIELD = \"nsd\";\n    public final static String RPC_REQUEST_HEADER_NAMESPACE_FIELD = \"ns\";\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);\n    public static final String LOGICAL_QUEUE_MOCK_BROKER_PREFIX = \"__syslo__\";\n    public static final String METADATA_SCOPE_GLOBAL = \"__global__\";\n    public static final String LOGICAL_QUEUE_MOCK_BROKER_NAME_NOT_EXIST = \"__syslo__none__\";\n    public static final String MULTI_PATH_SPLITTER = System.getProperty(\"rocketmq.broker.multiPathSplitter\", \",\");\n\n    private static final String OS = System.getProperty(\"os.name\").toLowerCase();\n    public static final long MILLS_FOR_HOUR = TimeUnit.HOURS.toMillis(1);\n\n    private static final Set<String> PREDEFINE_GROUP_SET = ImmutableSet.of(\n        DEFAULT_CONSUMER_GROUP,\n        DEFAULT_PRODUCER_GROUP,\n        TOOLS_CONSUMER_GROUP,\n        SCHEDULE_CONSUMER_GROUP,\n        FILTERSRV_CONSUMER_GROUP,\n        MONITOR_CONSUMER_GROUP,\n        CLIENT_INNER_PRODUCER_GROUP,\n        SELF_TEST_PRODUCER_GROUP,\n        SELF_TEST_CONSUMER_GROUP,\n        ONS_HTTP_PROXY_GROUP,\n        CID_ONSAPI_PERMISSION_GROUP,\n        CID_ONSAPI_OWNER_GROUP,\n        CID_ONSAPI_PULL_GROUP,\n        CID_SYS_RMQ_TRANS\n    );\n\n    public static boolean isWindows() {\n        return OS.contains(\"win\");\n    }\n\n    public static boolean isMac() {\n        return OS.contains(\"mac\");\n    }\n\n    public static boolean isUnix() {\n        return OS.contains(\"nix\")\n            || OS.contains(\"nux\")\n            || OS.contains(\"aix\");\n    }\n\n    public static boolean isSolaris() {\n        return OS.contains(\"sunos\");\n    }\n\n    public static String getWSAddr() {\n        String wsDomainName = System.getProperty(\"rocketmq.namesrv.domain\", DEFAULT_NAMESRV_ADDR_LOOKUP);\n        String wsDomainSubgroup = System.getProperty(\"rocketmq.namesrv.domain.subgroup\", \"nsaddr\");\n        String wsAddr = \"http://\" + wsDomainName + \":8080/rocketmq/\" + wsDomainSubgroup;\n        if (wsDomainName.indexOf(\":\") > 0) {\n            wsAddr = \"http://\" + wsDomainName + \"/rocketmq/\" + wsDomainSubgroup;\n        }\n        return wsAddr;\n    }\n\n    public static String getRetryTopic(final String consumerGroup) {\n        return RETRY_GROUP_TOPIC_PREFIX + consumerGroup;\n    }\n\n    public static String getReplyTopic(final String clusterName) {\n        return clusterName + \"_\" + REPLY_TOPIC_POSTFIX;\n    }\n\n    public static boolean isSysConsumerGroup(final String consumerGroup) {\n        return consumerGroup.startsWith(CID_RMQ_SYS_PREFIX);\n    }\n\n    public static boolean isSysConsumerGroupAndEnableCreate(final String consumerGroup, final boolean isEnableCreateSysGroup) {\n        return isEnableCreateSysGroup && isSysConsumerGroup(consumerGroup);\n    }\n\n    public static boolean isPredefinedGroup(final String consumerGroup) {\n        return PREDEFINE_GROUP_SET.contains(consumerGroup);\n    }\n\n    public static String getDLQTopic(final String consumerGroup) {\n        return DLQ_GROUP_TOPIC_PREFIX + consumerGroup;\n    }\n\n    public static String brokerVIPChannel(final boolean isChange, final String brokerAddr) {\n        if (isChange) {\n            int split = brokerAddr.lastIndexOf(\":\");\n            String ip = brokerAddr.substring(0, split);\n            String port = brokerAddr.substring(split + 1);\n            String brokerAddrNew = ip + \":\" + (Integer.parseInt(port) - 2);\n            return brokerAddrNew;\n        } else {\n            return brokerAddr;\n        }\n    }\n\n    public static long getPID() {\n        String processName = java.lang.management.ManagementFactory.getRuntimeMXBean().getName();\n        if (StringUtils.isNotEmpty(processName)) {\n            try {\n                return Long.parseLong(processName.split(\"@\")[0]);\n            } catch (Exception e) {\n                return 0;\n            }\n        }\n\n        return 0;\n    }\n\n    public static synchronized void string2File(final String str, final String fileName) throws IOException {\n\n        String bakFile = fileName + \".bak\";\n        String prevContent = file2String(fileName);\n        if (prevContent != null) {\n            string2FileNotSafe(prevContent, bakFile);\n        }\n\n        string2FileNotSafe(str, fileName);\n    }\n\n    public static void string2FileNotSafe(final String str, final String fileName) throws IOException {\n        File file = new File(fileName);\n        File fileParent = file.getParentFile();\n        if (fileParent != null) {\n            fileParent.mkdirs();\n        }\n        IOTinyUtils.writeStringToFile(file, str, DEFAULT_CHARSET);\n    }\n\n    public static synchronized void fsyncDirectory(Path dir) throws IOException {\n        if (!Files.isDirectory(dir)) {\n            throw new NotDirectoryException(dir.toString());\n        }\n        if (isWindows()) {\n            return;\n        }\n        try (FileChannel fc = FileChannel.open(dir, StandardOpenOption.READ)) {\n            fc.force(true);\n        }\n    }\n\n    public static String file2String(final String fileName) throws IOException {\n        File file = new File(fileName);\n        return file2String(file);\n    }\n\n    public static String file2String(final File file) throws IOException {\n        if (file.exists()) {\n            byte[] data = new byte[(int) file.length()];\n            boolean result;\n\n            try (FileInputStream inputStream = new FileInputStream(file)) {\n                int len = inputStream.read(data);\n                result = len == data.length;\n            }\n\n            if (result) {\n                return new String(data, DEFAULT_CHARSET);\n            }\n        }\n        return null;\n    }\n\n    public static String file2String(final URL url) {\n        InputStream in = null;\n        try {\n            URLConnection urlConnection = url.openConnection();\n            urlConnection.setUseCaches(false);\n            in = urlConnection.getInputStream();\n            int len = in.available();\n            byte[] data = new byte[len];\n            in.read(data, 0, len);\n            return new String(data, StandardCharsets.UTF_8);\n        } catch (Exception ignored) {\n        } finally {\n            if (null != in) {\n                try {\n                    in.close();\n                } catch (IOException ignored) {\n                }\n            }\n        }\n\n        return null;\n    }\n\n    public static void printObjectProperties(final Logger logger, final Object object) {\n        printObjectProperties(logger, object, false);\n    }\n\n    public static void printObjectProperties(final Logger logger, final Object object,\n        final boolean onlyImportantField) {\n        Field[] fields = object.getClass().getDeclaredFields();\n        for (Field field : fields) {\n            if (!Modifier.isStatic(field.getModifiers())) {\n                String name = field.getName();\n                if (!name.startsWith(\"this\")) {\n                    if (onlyImportantField) {\n                        Annotation annotation = field.getAnnotation(ImportantField.class);\n                        if (null == annotation) {\n                            continue;\n                        }\n                    }\n\n                    Object value = null;\n                    try {\n                        field.setAccessible(true);\n                        value = field.get(object);\n                        if (null == value) {\n                            value = \"\";\n                        }\n                    } catch (IllegalAccessException e) {\n                        log.error(\"Failed to obtain object properties\", e);\n                    }\n\n                    if (logger != null) {\n                        logger.info(name + \"=\" + value);\n                    }\n                }\n            }\n        }\n    }\n\n    public static String properties2String(final Properties properties) {\n        return properties2String(properties, false);\n    }\n\n    public static String properties2String(final Properties properties, final boolean isSort) {\n        StringBuilder sb = new StringBuilder();\n        Set<Map.Entry<Object, Object>> entrySet = isSort ? new TreeMap<>(properties).entrySet() : properties.entrySet();\n        for (Map.Entry<Object, Object> entry : entrySet) {\n            if (entry.getValue() != null) {\n                sb.append(entry.getKey().toString() + \"=\" + entry.getValue().toString() + \"\\n\");\n            }\n        }\n        return sb.toString();\n    }\n\n    public static Properties string2Properties(final String str) {\n        Properties properties = new Properties();\n        try {\n            InputStream in = new ByteArrayInputStream(str.getBytes(DEFAULT_CHARSET));\n            properties.load(in);\n        } catch (Exception e) {\n            log.error(\"Failed to handle properties\", e);\n            return null;\n        }\n\n        return properties;\n    }\n\n    public static Properties object2Properties(final Object object) {\n        Properties properties = new Properties();\n\n        Class<?> objectClass = object.getClass();\n        while (true) {\n            Field[] fields = objectClass.getDeclaredFields();\n            for (Field field : fields) {\n                if (!Modifier.isStatic(field.getModifiers())) {\n                    String name = field.getName();\n                    if (!name.startsWith(\"this\")) {\n                        Object value = null;\n                        try {\n                            field.setAccessible(true);\n                            value = field.get(object);\n                        } catch (IllegalAccessException e) {\n                            log.error(\"Failed to handle properties\", e);\n                        }\n\n                        if (value != null) {\n                            properties.setProperty(name, value.toString());\n                        }\n                    }\n                }\n            }\n            if (objectClass == Object.class || objectClass.getSuperclass() == Object.class) {\n                break;\n            }\n            objectClass = objectClass.getSuperclass();\n        }\n\n        return properties;\n    }\n\n    public static void properties2Object(final Properties p, final Object object) {\n        Method[] methods = object.getClass().getMethods();\n        for (Method method : methods) {\n            String mn = method.getName();\n            if (mn.startsWith(\"set\")) {\n                try {\n                    String tmp = mn.substring(4);\n                    String first = mn.substring(3, 4);\n\n                    String key = first.toLowerCase() + tmp;\n                    String property = p.getProperty(key);\n                    if (property != null) {\n                        Class<?>[] pt = method.getParameterTypes();\n                        if (pt.length > 0) {\n                            String cn = pt[0].getSimpleName();\n                            Object arg;\n                            if (cn.equals(\"int\") || cn.equals(\"Integer\")) {\n                                arg = Integer.parseInt(property);\n                            } else if (cn.equals(\"long\") || cn.equals(\"Long\")) {\n                                arg = Long.parseLong(property);\n                            } else if (cn.equals(\"double\") || cn.equals(\"Double\")) {\n                                arg = Double.parseDouble(property);\n                            } else if (cn.equals(\"boolean\") || cn.equals(\"Boolean\")) {\n                                arg = Boolean.parseBoolean(property);\n                            } else if (cn.equals(\"float\") || cn.equals(\"Float\")) {\n                                arg = Float.parseFloat(property);\n                            } else if (cn.equals(\"String\")) {\n                                property = property.trim();\n                                arg = property;\n                            } else {\n                                continue;\n                            }\n                            method.invoke(object, arg);\n                        }\n                    }\n                } catch (Throwable ignored) {\n                }\n            }\n        }\n    }\n\n    public static boolean isPropertiesEqual(final Properties p1, final Properties p2) {\n        return p1.equals(p2);\n    }\n\n    public static boolean isPropertyValid(Properties props, String key, Predicate<String> validator) {\n        return validator.test(props.getProperty(key));\n    }\n\n    public static List<String> getLocalInetAddress() {\n        List<String> inetAddressList = new ArrayList<>();\n        try {\n            Enumeration<NetworkInterface> enumeration = NetworkInterface.getNetworkInterfaces();\n            while (enumeration.hasMoreElements()) {\n                NetworkInterface networkInterface = enumeration.nextElement();\n                Enumeration<InetAddress> addrs = networkInterface.getInetAddresses();\n                while (addrs.hasMoreElements()) {\n                    inetAddressList.add(addrs.nextElement().getHostAddress());\n                }\n            }\n        } catch (SocketException e) {\n            throw new RuntimeException(\"get local inet address fail\", e);\n        }\n\n        return inetAddressList;\n    }\n\n    private static String localhost() {\n        try {\n            return InetAddress.getLocalHost().getHostAddress();\n        } catch (Throwable e) {\n            try {\n                String candidatesHost = getLocalhostByNetworkInterface();\n                if (candidatesHost != null)\n                    return candidatesHost;\n\n            } catch (Exception ignored) {\n            }\n\n            throw new RuntimeException(\"InetAddress java.net.InetAddress.getLocalHost() throws UnknownHostException\" + FAQUrl.suggestTodo(FAQUrl.UNKNOWN_HOST_EXCEPTION), e);\n        }\n    }\n\n    //Reverse logic comparing to RemotingUtil method, consider refactor in RocketMQ 5.0\n    public static String getLocalhostByNetworkInterface() throws SocketException {\n        List<String> candidatesHost = new ArrayList<>();\n        Enumeration<NetworkInterface> enumeration = NetworkInterface.getNetworkInterfaces();\n\n        while (enumeration.hasMoreElements()) {\n            NetworkInterface networkInterface = enumeration.nextElement();\n            // Workaround for docker0 bridge\n            if (\"docker0\".equals(networkInterface.getName()) || !networkInterface.isUp()) {\n                continue;\n            }\n            Enumeration<InetAddress> addrs = networkInterface.getInetAddresses();\n            while (addrs.hasMoreElements()) {\n                InetAddress address = addrs.nextElement();\n                if (address.isLoopbackAddress()) {\n                    continue;\n                }\n                //ip4 higher priority\n                if (address instanceof Inet6Address) {\n                    candidatesHost.add(address.getHostAddress());\n                    continue;\n                }\n                return address.getHostAddress();\n            }\n        }\n\n        if (!candidatesHost.isEmpty()) {\n            return candidatesHost.get(0);\n        }\n\n        // Fallback to loopback\n        return localhost();\n    }\n\n    public static boolean compareAndIncreaseOnly(final AtomicLong target, final long value) {\n        long prev = target.get();\n        while (value > prev) {\n            boolean updated = target.compareAndSet(prev, value);\n            if (updated)\n                return true;\n\n            prev = target.get();\n        }\n\n        return false;\n    }\n\n    public static String humanReadableByteCount(long bytes, boolean si) {\n        int unit = si ? 1000 : 1024;\n        if (bytes < unit)\n            return bytes + \" B\";\n        int exp = (int) (Math.log(bytes) / Math.log(unit));\n        String pre = (si ? \"kMGTPE\" : \"KMGTPE\").charAt(exp - 1) + (si ? \"\" : \"i\");\n        return String.format(\"%.1f %sB\", bytes / Math.pow(unit, exp), pre);\n    }\n\n    public static int compareInteger(int x, int y) {\n        return Integer.compare(x, y);\n    }\n\n    public static int compareLong(long x, long y) {\n        return Long.compare(x, y);\n    }\n\n    public static boolean isLmq(String lmqMetaData) {\n        return lmqMetaData != null && lmqMetaData.startsWith(LMQ_PREFIX);\n    }\n\n    public static String dealFilePath(String aclFilePath) {\n        Path path = Paths.get(aclFilePath);\n        return path.normalize().toString();\n    }\n\n    public static boolean isSysConsumerGroupPullMessage(String consumerGroup) {\n        if (DEFAULT_CONSUMER_GROUP.equals(consumerGroup)\n            || TOOLS_CONSUMER_GROUP.equals(consumerGroup)\n            || SCHEDULE_CONSUMER_GROUP.equals(consumerGroup)\n            || FILTERSRV_CONSUMER_GROUP.equals(consumerGroup)\n            || MONITOR_CONSUMER_GROUP.equals(consumerGroup)\n            || SELF_TEST_CONSUMER_GROUP.equals(consumerGroup)\n            || ONS_HTTP_PROXY_GROUP.equals(consumerGroup)\n            || CID_ONSAPI_PERMISSION_GROUP.equals(consumerGroup)\n            || CID_ONSAPI_OWNER_GROUP.equals(consumerGroup)\n            || CID_ONSAPI_PULL_GROUP.equals(consumerGroup)\n            || CID_SYS_RMQ_TRANS.equals(consumerGroup)\n            || consumerGroup.startsWith(CID_RMQ_SYS_PREFIX)) {\n            return true;\n        }\n        return false;\n    }\n\n    public static boolean topicAllowsLMQ(String topic) {\n        return !topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)\n            && !topic.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX)\n            && !topic.startsWith(TopicValidator.SYSTEM_TOPIC_PREFIX)\n            && !topic.equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC);\n    }\n\n    public static String adjustConfigForPlatform(String config) {\n        if (StringUtils.isNotBlank(config)) {\n            if (isWindows()) {\n                config = StringUtils.replace(config, \"\\\\\", \"\\\\\\\\\");\n            }\n        }\n        return config;\n    }\n\n    public static long dealTimeToHourStamps(long timeStamp) {\n        if (timeStamp <= 0L) {\n            return timeStamp;\n        }\n        return (timeStamp / MILLS_FOR_HOUR) * MILLS_FOR_HOUR;\n    }\n\n    public static boolean isHourTime(Long timeStamp) {\n        if (null == timeStamp) {\n            return false;\n        }\n        if (timeStamp <= 0L) {\n            return false;\n        }\n        return timeStamp % MILLS_FOR_HOUR == 0;\n    }\n\n    public static List<Long> getHours(long startTimeMillis, long endTimeMillis) {\n        if (startTimeMillis > endTimeMillis || startTimeMillis <= 0L || endTimeMillis <= 0L) {\n            return null;\n        }\n        List<Long> result = new ArrayList<>();\n        long startHour = dealTimeToHourStamps(startTimeMillis);\n        long endHour = dealTimeToHourStamps(endTimeMillis);\n        long current = startHour;\n        while (current <= endHour) {\n            result.add(current);\n            //protect system self 30 * 24\n            if (result.size() >= 720) {\n                return result;\n            }\n            current += MILLS_FOR_HOUR;\n        }\n        return result;\n    }\n\n    public static boolean isByteArrayEqual(byte[] array1, int offset1, int length1, byte[] array2, int offset2, int length2) {\n        if (null == array1 || null == array2) {\n            return false;\n        }\n        if (length1 != length2) {\n            return false;\n        }\n        if (offset1 < 0 || offset1 + length1 > array1.length ||\n            offset2 < 0 || offset2 + length2 > array2.length) {\n            throw new ArrayIndexOutOfBoundsException(\"Invalid array index\");\n        }\n        for (int i = 0; i < length1; i++) {\n            if (array1[offset1 + i] != array2[offset2 + i]) {\n                return false;\n            }\n        }\n        return true;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/ObjectCreator.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\npublic interface ObjectCreator<T> {\n    T create(Object... args);\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/OrderedConsumptionLevel.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\npublic enum OrderedConsumptionLevel {\n    QUEUE(0),\n    SHARDING_KEY(1);\n\n    private final int value;\n\n    OrderedConsumptionLevel(int value) {\n        this.value = value;\n    }\n\n    public int getValue() {\n        return value;\n    }\n\n    public static OrderedConsumptionLevel valueOf(int value) {\n        if (value == 1) {\n            return SHARDING_KEY;\n        }\n        return QUEUE;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/Pair.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\nimport java.io.Serializable;\n\npublic class Pair<T1, T2> implements Serializable {\n    private T1 object1;\n    private T2 object2;\n\n    public Pair(T1 object1, T2 object2) {\n        this.object1 = object1;\n        this.object2 = object2;\n    }\n\n    public static <T1, T2> Pair<T1, T2> of(T1 object1, T2 object2) {\n        return new Pair<>(object1, object2);\n    }\n\n    public T1 getObject1() {\n        return object1;\n    }\n\n    public void setObject1(T1 object1) {\n        this.object1 = object1;\n    }\n\n    public T2 getObject2() {\n        return object2;\n    }\n\n    public void setObject2(T2 object2) {\n        this.object2 = object2;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/PopAckConstants.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\nimport org.apache.rocketmq.common.topic.TopicValidator;\n\npublic class PopAckConstants {\n    public static long ackTimeInterval = 1000;\n    public static final long SECOND = 1000;\n\n    public static long lockTime = 5000;\n    public static int retryQueueNum = 1;\n\n    public static final String REVIVE_GROUP = MixAll.CID_RMQ_SYS_PREFIX + \"REVIVE_GROUP\";\n    public static final String LOCAL_HOST = \"127.0.0.1\";\n    public static final String REVIVE_TOPIC = TopicValidator.SYSTEM_TOPIC_PREFIX + \"REVIVE_LOG_\";\n    public static final String CK_TAG = \"ck\";\n    public static final String ACK_TAG = \"ack\";\n    public static final String BATCH_ACK_TAG = \"bAck\";\n    public static final String SPLIT = \"@\";\n\n    /**\n     * Build cluster revive topic\n     *\n     * @param clusterName cluster name\n     * @return revive topic\n     */\n    public static String buildClusterReviveTopic(String clusterName) {\n        return PopAckConstants.REVIVE_TOPIC + clusterName;\n    }\n\n    public static boolean isStartWithRevivePrefix(String topicName) {\n        return topicName != null && topicName.startsWith(REVIVE_TOPIC);\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/ServiceState.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\npublic enum ServiceState {\n    /**\n     * Service just created,not start\n     */\n    CREATE_JUST,\n    /**\n     * Service Running\n     */\n    RUNNING,\n    /**\n     * Service shutdown\n     */\n    SHUTDOWN_ALREADY,\n    /**\n     * Service Start failure\n     */\n    START_FAILED;\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/ServiceThread.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicBoolean;\n\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic abstract class ServiceThread implements Runnable {\n    protected static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);\n\n    private static final long JOIN_TIME = 90 * 1000;\n\n    protected Thread thread;\n    protected final CountDownLatch2 waitPoint = new CountDownLatch2(1);\n    protected volatile AtomicBoolean hasNotified = new AtomicBoolean(false);\n    protected volatile boolean stopped = false;\n    protected boolean isDaemon = false;\n\n    //Make it able to restart the thread\n    private final AtomicBoolean started = new AtomicBoolean(false);\n\n    public ServiceThread() {\n\n    }\n\n    public String getServiceName() {\n        return this.getClass().getSimpleName();\n    }\n\n    public void start() {\n        log.info(\"Try to start service thread:{} started:{} lastThread:{}\", getServiceName(), started.get(), thread);\n        if (!started.compareAndSet(false, true)) {\n            return;\n        }\n        stopped = false;\n        this.thread = new Thread(this, getServiceName());\n        this.thread.setDaemon(isDaemon);\n        this.thread.start();\n        log.info(\"Start service thread:{} started:{} lastThread:{}\", getServiceName(), started.get(), thread);\n    }\n\n    public void shutdown() {\n        this.shutdown(false);\n    }\n\n    public void shutdown(final boolean interrupt) {\n        log.info(\"Try to shutdown service thread:{} started:{} lastThread:{}\", getServiceName(), started.get(), thread);\n        if (!started.compareAndSet(true, false)) {\n            return;\n        }\n        this.stopped = true;\n        log.info(\"shutdown thread[{}] interrupt={} \", getServiceName(), interrupt);\n\n        //if thead is waiting, wakeup it\n        wakeup();\n\n        try {\n            if (interrupt) {\n                this.thread.interrupt();\n            }\n\n            long beginTime = System.currentTimeMillis();\n            if (!this.thread.isDaemon()) {\n                this.thread.join(this.getJoinTime());\n            }\n            long elapsedTime = System.currentTimeMillis() - beginTime;\n            log.info(\"join thread[{}], elapsed time: {}ms, join time:{}ms\", getServiceName(), elapsedTime, this.getJoinTime());\n        } catch (InterruptedException e) {\n            log.error(\"Interrupted\", e);\n        }\n    }\n\n    public long getJoinTime() {\n        return JOIN_TIME;\n    }\n\n    public void makeStop() {\n        if (!started.get()) {\n            return;\n        }\n        this.stopped = true;\n        log.info(\"makestop thread[{}] \", this.getServiceName());\n    }\n\n    public void wakeup() {\n        if (hasNotified.compareAndSet(false, true)) {\n            waitPoint.countDown(); // notify\n        }\n    }\n\n    protected void waitForRunning(long interval) {\n        if (hasNotified.compareAndSet(true, false)) {\n            this.onWaitEnd();\n            return;\n        }\n\n        //entry to wait\n        waitPoint.reset();\n\n        try {\n            waitPoint.await(interval, TimeUnit.MILLISECONDS);\n        } catch (InterruptedException e) {\n            log.error(\"Interrupted\", e);\n        } finally {\n            hasNotified.set(false);\n            this.onWaitEnd();\n        }\n    }\n\n    protected void onWaitEnd() {\n    }\n\n    public boolean isStopped() {\n        return stopped;\n    }\n\n    public boolean isDaemon() {\n        return isDaemon;\n    }\n\n    public void setDaemon(boolean daemon) {\n        isDaemon = daemon;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/SubscriptionGroupAttributes.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\nimport static com.google.common.collect.Sets.newHashSet;\n\nimport java.util.HashMap;\nimport java.util.Map;\nimport org.apache.rocketmq.common.attribute.Attribute;\nimport org.apache.rocketmq.common.attribute.BooleanAttribute;\nimport org.apache.rocketmq.common.attribute.EnumAttribute;\nimport org.apache.rocketmq.common.attribute.LongRangeAttribute;\nimport org.apache.rocketmq.common.attribute.StringAttribute;\nimport org.apache.rocketmq.common.attribute.LiteSubModel;\n\npublic class SubscriptionGroupAttributes {\n\n    public static final Map<String, Attribute> ALL;\n    public static final LongRangeAttribute PRIORITY_FACTOR_ATTRIBUTE = new LongRangeAttribute(\n        \"priority.factor\",\n        true,\n        0, // disable priority mode\n        100, // enable priority mode\n        100\n    );\n\n    public static final StringAttribute LITE_BIND_TOPIC_ATTRIBUTE = new StringAttribute(\n        \"lite.bind.topic\",\n        true\n    );\n\n    public static final EnumAttribute LITE_SUB_MODEL_ATTRIBUTE = new EnumAttribute(\n        \"lite.sub.model\",\n        true,\n        newHashSet(LiteSubModel.Shared.name(), LiteSubModel.Exclusive.name()),\n        LiteSubModel.Shared.name()\n    );\n\n    public static final BooleanAttribute LITE_SUB_RESET_OFFSET_EXCLUSIVE_ATTRIBUTE = new BooleanAttribute(\n        \"lite.sub.reset.offset.exclusive\",\n        true,\n        false\n    );\n\n    public static final BooleanAttribute LITE_SUB_RESET_OFFSET_UNSUBSCRIBE_ATTRIBUTE = new BooleanAttribute(\n        \"lite.sub.reset.offset.unsubscribe\",\n        true,\n        false\n    );\n\n    /**\n     * client-side lite subscription quota limit\n     */\n    public static final LongRangeAttribute LITE_SUB_CLIENT_QUOTA_ATTRIBUTE = new LongRangeAttribute(\n        \"lite.sub.client.quota\",\n        true,\n        -1,\n        Long.MAX_VALUE,\n        2000\n    );\n\n    public static final LongRangeAttribute LITE_SUB_CLIENT_MAX_EVENT_COUNT = new LongRangeAttribute(\n        \"lite.sub.client.max.event.cnt\",\n        true,\n        10,\n        Long.MAX_VALUE,\n        400\n    );\n\n    static {\n        ALL = new HashMap<>();\n        ALL.put(PRIORITY_FACTOR_ATTRIBUTE.getName(), PRIORITY_FACTOR_ATTRIBUTE);\n        ALL.put(LITE_BIND_TOPIC_ATTRIBUTE.getName(), LITE_BIND_TOPIC_ATTRIBUTE);\n        ALL.put(LITE_SUB_CLIENT_QUOTA_ATTRIBUTE.getName(), LITE_SUB_CLIENT_QUOTA_ATTRIBUTE);\n        ALL.put(LITE_SUB_MODEL_ATTRIBUTE.getName(), LITE_SUB_MODEL_ATTRIBUTE);\n        ALL.put(LITE_SUB_RESET_OFFSET_EXCLUSIVE_ATTRIBUTE.getName(), LITE_SUB_RESET_OFFSET_EXCLUSIVE_ATTRIBUTE);\n        ALL.put(LITE_SUB_RESET_OFFSET_UNSUBSCRIBE_ATTRIBUTE.getName(), LITE_SUB_RESET_OFFSET_UNSUBSCRIBE_ATTRIBUTE);\n        ALL.put(LITE_SUB_CLIENT_MAX_EVENT_COUNT.getName(), LITE_SUB_CLIENT_MAX_EVENT_COUNT);\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/SystemClock.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\npublic class SystemClock {\n    public long now() {\n        return System.currentTimeMillis();\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/ThreadFactoryImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common;\n\nimport java.util.concurrent.ThreadFactory;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class ThreadFactoryImpl implements ThreadFactory {\n\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);\n\n    private final AtomicLong threadIndex = new AtomicLong(0);\n    private final String threadNamePrefix;\n    private final boolean daemon;\n\n    public ThreadFactoryImpl(final String threadNamePrefix) {\n        this(threadNamePrefix, false);\n    }\n\n    public ThreadFactoryImpl(final String threadNamePrefix, boolean daemon) {\n        this.threadNamePrefix = threadNamePrefix;\n        this.daemon = daemon;\n    }\n\n    public ThreadFactoryImpl(final String threadNamePrefix, BrokerIdentity brokerIdentity) {\n        this(threadNamePrefix, false, brokerIdentity);\n    }\n\n    public ThreadFactoryImpl(final String threadNamePrefix, boolean daemon, BrokerIdentity brokerIdentity) {\n        this.daemon = daemon;\n        if (brokerIdentity != null && brokerIdentity.isInBrokerContainer()) {\n            this.threadNamePrefix = brokerIdentity.getIdentifier() + threadNamePrefix;\n        } else {\n            this.threadNamePrefix = threadNamePrefix;\n        }\n    }\n\n    @Override\n    public Thread newThread(Runnable r) {\n        Thread thread = new Thread(r, threadNamePrefix + this.threadIndex.incrementAndGet());\n        thread.setDaemon(daemon);\n\n        // Log all uncaught exception\n        thread.setUncaughtExceptionHandler((t, e) ->\n            LOGGER.error(\"[BUG] Thread has an uncaught exception, threadId={}, threadName={}\",\n                t.getId(), t.getName(), e));\n\n        return thread;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/TopicAttributes.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.common.attribute.Attribute;\nimport org.apache.rocketmq.common.attribute.EnumAttribute;\nimport org.apache.rocketmq.common.attribute.LongRangeAttribute;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\n\nimport static com.google.common.collect.Sets.newHashSet;\n\npublic class TopicAttributes {\n    public static final EnumAttribute QUEUE_TYPE_ATTRIBUTE = new EnumAttribute(\n        \"queue.type\",\n        false,\n        newHashSet(\"BatchCQ\", \"SimpleCQ\"),\n        \"SimpleCQ\"\n    );\n    public static final EnumAttribute CLEANUP_POLICY_ATTRIBUTE = new EnumAttribute(\n        \"cleanup.policy\",\n        false,\n        newHashSet(\"DELETE\", \"COMPACTION\"),\n        \"DELETE\"\n    );\n    public static final EnumAttribute TOPIC_MESSAGE_TYPE_ATTRIBUTE = new EnumAttribute(\n        \"message.type\",\n        true,\n        TopicMessageType.topicMessageTypeSet(),\n        TopicMessageType.NORMAL.getValue()\n    );\n    public static final LongRangeAttribute TOPIC_RESERVE_TIME_ATTRIBUTE = new LongRangeAttribute(\n        \"reserve.time\",\n        true,\n        -1,\n        Long.MAX_VALUE,\n        -1\n    );\n\n    public static final LongRangeAttribute LITE_EXPIRATION_ATTRIBUTE = new LongRangeAttribute(\n        \"lite.topic.expiration\",\n        true,\n        -1,\n        TimeUnit.DAYS.toMinutes(30),\n        -1\n    );\n\n    public static final Map<String, Attribute> ALL;\n\n    static {\n        ALL = new HashMap<>();\n        ALL.put(QUEUE_TYPE_ATTRIBUTE.getName(), QUEUE_TYPE_ATTRIBUTE);\n        ALL.put(CLEANUP_POLICY_ATTRIBUTE.getName(), CLEANUP_POLICY_ATTRIBUTE);\n        ALL.put(TOPIC_MESSAGE_TYPE_ATTRIBUTE.getName(), TOPIC_MESSAGE_TYPE_ATTRIBUTE);\n        ALL.put(TOPIC_RESERVE_TIME_ATTRIBUTE.getName(), TOPIC_RESERVE_TIME_ATTRIBUTE);\n        ALL.put(LITE_EXPIRATION_ATTRIBUTE.getName(), LITE_EXPIRATION_ATTRIBUTE);\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/TopicConfig.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.alibaba.fastjson2.TypeReference;\nimport com.alibaba.fastjson2.annotation.JSONField;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.Objects;\nimport org.apache.commons.lang3.math.NumberUtils;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.PermName;\n\nimport static org.apache.rocketmq.common.TopicAttributes.LITE_EXPIRATION_ATTRIBUTE;\nimport static org.apache.rocketmq.common.TopicAttributes.TOPIC_MESSAGE_TYPE_ATTRIBUTE;\n\npublic class TopicConfig {\n    private static final String SEPARATOR = \" \";\n    public static int defaultReadQueueNums = 16;\n    public static int defaultWriteQueueNums = 16;\n    private static final TypeReference<Map<String, String>> ATTRIBUTES_TYPE_REFERENCE = new TypeReference<Map<String, String>>() {\n    };\n    private String topicName;\n    private int readQueueNums = defaultReadQueueNums;\n    private int writeQueueNums = defaultWriteQueueNums;\n    private int perm = PermName.PERM_READ | PermName.PERM_WRITE;\n    private TopicFilterType topicFilterType = TopicFilterType.SINGLE_TAG;\n    private int topicSysFlag = 0;\n    private boolean order = false;\n    // Field attributes should not have ' ' char in key or value, otherwise will lead to decode failure.\n    private Map<String, String> attributes = new HashMap<>();\n\n    public TopicConfig() {\n    }\n\n    public TopicConfig(String topicName) {\n        this.topicName = topicName;\n    }\n\n    public TopicConfig(String topicName, int readQueueNums, int writeQueueNums) {\n        this.topicName = topicName;\n        this.readQueueNums = readQueueNums;\n        this.writeQueueNums = writeQueueNums;\n    }\n\n    public TopicConfig(String topicName, int readQueueNums, int writeQueueNums, int perm) {\n        this.topicName = topicName;\n        this.readQueueNums = readQueueNums;\n        this.writeQueueNums = writeQueueNums;\n        this.perm = perm;\n    }\n\n    public TopicConfig(String topicName, int readQueueNums, int writeQueueNums, int perm, int topicSysFlag) {\n        this.topicName = topicName;\n        this.readQueueNums = readQueueNums;\n        this.writeQueueNums = writeQueueNums;\n        this.perm = perm;\n        this.topicSysFlag = topicSysFlag;\n    }\n\n    public TopicConfig(TopicConfig other) {\n        this.topicName = other.topicName;\n        this.readQueueNums = other.readQueueNums;\n        this.writeQueueNums = other.writeQueueNums;\n        this.perm = other.perm;\n        this.topicFilterType = other.topicFilterType;\n        this.topicSysFlag = other.topicSysFlag;\n        this.order = other.order;\n        this.attributes = other.attributes;\n    }\n\n    public String encode() {\n        StringBuilder sb = new StringBuilder();\n        //[0]\n        sb.append(this.topicName);\n        sb.append(SEPARATOR);\n        //[1]\n        sb.append(this.readQueueNums);\n        sb.append(SEPARATOR);\n        //[2]\n        sb.append(this.writeQueueNums);\n        sb.append(SEPARATOR);\n        //[3]\n        sb.append(this.perm);\n        sb.append(SEPARATOR);\n        //[4]\n        sb.append(this.topicFilterType);\n        sb.append(SEPARATOR);\n        //[5]\n        if (attributes != null) {\n            sb.append(JSON.toJSONString(attributes));\n        }\n\n        return sb.toString();\n    }\n\n    public boolean decode(final String in) {\n        String[] strs = in.split(SEPARATOR);\n        if (strs.length >= 5) {\n            this.topicName = strs[0];\n\n            this.readQueueNums = Integer.parseInt(strs[1]);\n\n            this.writeQueueNums = Integer.parseInt(strs[2]);\n\n            this.perm = Integer.parseInt(strs[3]);\n\n            this.topicFilterType = TopicFilterType.valueOf(strs[4]);\n\n            if (strs.length >= 6) {\n                try {\n                    this.attributes = JSON.parseObject(strs[5], ATTRIBUTES_TYPE_REFERENCE.getType());\n                } catch (Exception e) {\n                    // ignore exception when parse failed, cause map's key/value can have ' ' char.\n                }\n            }\n\n            return true;\n        }\n\n        return false;\n    }\n\n    public String getTopicName() {\n        return topicName;\n    }\n\n    public void setTopicName(String topicName) {\n        this.topicName = topicName;\n    }\n\n    public int getReadQueueNums() {\n        return readQueueNums;\n    }\n\n    public void setReadQueueNums(int readQueueNums) {\n        this.readQueueNums = readQueueNums;\n    }\n\n    public int getWriteQueueNums() {\n        return writeQueueNums;\n    }\n\n    public void setWriteQueueNums(int writeQueueNums) {\n        this.writeQueueNums = writeQueueNums;\n    }\n\n    public int getPerm() {\n        return perm;\n    }\n\n    public void setPerm(int perm) {\n        this.perm = perm;\n    }\n\n    public TopicFilterType getTopicFilterType() {\n        return topicFilterType;\n    }\n\n    public void setTopicFilterType(TopicFilterType topicFilterType) {\n        this.topicFilterType = topicFilterType;\n    }\n\n    public int getTopicSysFlag() {\n        return topicSysFlag;\n    }\n\n    public void setTopicSysFlag(int topicSysFlag) {\n        this.topicSysFlag = topicSysFlag;\n    }\n\n    public boolean isOrder() {\n        return order;\n    }\n\n    public void setOrder(boolean isOrder) {\n        this.order = isOrder;\n    }\n\n    public Map<String, String> getAttributes() {\n        return attributes;\n    }\n\n    public void setAttributes(Map<String, String> attributes) {\n        this.attributes = attributes;\n    }\n\n    @JSONField(serialize = false, deserialize = false)\n    public TopicMessageType getTopicMessageType() {\n        if (attributes == null) {\n            return TopicMessageType.NORMAL;\n        }\n        String content = attributes.get(TOPIC_MESSAGE_TYPE_ATTRIBUTE.getName());\n        if (content == null) {\n            return TopicMessageType.NORMAL;\n        }\n        return TopicMessageType.valueOf(content);\n    }\n\n    @JSONField(serialize = false, deserialize = false)\n    public void setTopicMessageType(TopicMessageType topicMessageType) {\n        attributes.put(TOPIC_MESSAGE_TYPE_ATTRIBUTE.getName(), topicMessageType.getValue());\n    }\n\n    @JSONField(serialize = false, deserialize = false)\n    public void setLiteTopicExpiration(int liteTopicExpiration) {\n        if (!TopicMessageType.LITE.equals(getTopicMessageType())) {\n            return;\n        }\n        attributes.put(LITE_EXPIRATION_ATTRIBUTE.getName(), String.valueOf(liteTopicExpiration));\n    }\n\n    @JSONField(serialize = false, deserialize = false)\n    public int getLiteTopicExpiration() {\n        if (!TopicMessageType.LITE.equals(getTopicMessageType())) {\n            return -1;\n        }\n        String content = attributes.get(LITE_EXPIRATION_ATTRIBUTE.getName());\n        if (content == null) {\n            return -1;\n        }\n        return NumberUtils.toInt(content, -1);\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) {\n            return true;\n        }\n        if (o == null || getClass() != o.getClass()) {\n            return false;\n        }\n\n        TopicConfig that = (TopicConfig) o;\n\n        if (readQueueNums != that.readQueueNums) {\n            return false;\n        }\n        if (writeQueueNums != that.writeQueueNums) {\n            return false;\n        }\n        if (perm != that.perm) {\n            return false;\n        }\n        if (topicSysFlag != that.topicSysFlag) {\n            return false;\n        }\n        if (order != that.order) {\n            return false;\n        }\n        if (!Objects.equals(topicName, that.topicName)) {\n            return false;\n        }\n        if (topicFilterType != that.topicFilterType) {\n            return false;\n        }\n        return Objects.equals(attributes, that.attributes);\n    }\n\n    @Override\n    public int hashCode() {\n        int result = topicName != null ? topicName.hashCode() : 0;\n        result = 31 * result + readQueueNums;\n        result = 31 * result + writeQueueNums;\n        result = 31 * result + perm;\n        result = 31 * result + (topicFilterType != null ? topicFilterType.hashCode() : 0);\n        result = 31 * result + topicSysFlag;\n        result = 31 * result + (order ? 1 : 0);\n        result = 31 * result + (attributes != null ? attributes.hashCode() : 0);\n        return result;\n    }\n\n    @Override\n    public String toString() {\n        return \"TopicConfig [topicName=\" + topicName + \", readQueueNums=\" + readQueueNums\n            + \", writeQueueNums=\" + writeQueueNums + \", perm=\" + PermName.perm2String(perm)\n            + \", topicFilterType=\" + topicFilterType + \", topicSysFlag=\" + topicSysFlag + \", order=\" + order\n            + \", attributes=\" + attributes + \"]\";\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/TopicFilterType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\npublic enum TopicFilterType {\n    SINGLE_TAG,\n    MULTI_TAG\n\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/TopicQueueId.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\nimport com.google.common.base.Objects;\n\npublic class TopicQueueId {\n    private final String topic;\n    private final int queueId;\n\n    private final int hash;\n\n    public TopicQueueId(String topic, int queueId) {\n        this.topic = topic;\n        this.queueId = queueId;\n\n        this.hash = Objects.hashCode(topic, queueId);\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o)\n            return true;\n        if (o == null || getClass() != o.getClass())\n            return false;\n        TopicQueueId broker = (TopicQueueId) o;\n        return queueId == broker.queueId && Objects.equal(topic, broker.topic);\n    }\n\n    @Override\n    public int hashCode() {\n        return hash;\n    }\n\n    @Override\n    public String toString() {\n        final StringBuilder sb = new StringBuilder(\"MessageQueueInBroker{\");\n        sb.append(\"topic='\").append(topic).append('\\'');\n        sb.append(\", queueId=\").append(queueId);\n        sb.append('}');\n        return sb.toString();\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/UnlockCallback.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\npublic interface UnlockCallback {\n    void onSuccess();\n\n    void onException(final Throwable e);\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/UtilAll.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\nimport io.netty.util.internal.PlatformDependent;\nimport java.io.ByteArrayInputStream;\nimport java.io.ByteArrayOutputStream;\nimport java.io.File;\nimport java.io.IOException;\nimport java.lang.management.ManagementFactory;\nimport java.net.Inet4Address;\nimport java.net.Inet6Address;\nimport java.net.InetAddress;\nimport java.net.NetworkInterface;\nimport java.nio.ByteBuffer;\nimport java.nio.file.Files;\nimport java.text.NumberFormat;\nimport java.text.ParseException;\nimport java.text.SimpleDateFormat;\nimport java.util.Arrays;\nimport java.util.Calendar;\nimport java.util.Date;\nimport java.util.Enumeration;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.TimeUnit;\nimport java.util.function.Supplier;\nimport java.util.zip.CRC32;\nimport java.util.zip.DeflaterOutputStream;\nimport java.util.zip.InflaterInputStream;\nimport java.util.Collections;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.commons.validator.routines.InetAddressValidator;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class UtilAll {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);\n    private static final Logger STORE_LOG = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n\n    public static final String YYYY_MM_DD_HH_MM_SS = \"yyyy-MM-dd HH:mm:ss\";\n    public static final String YYYY_MM_DD_HH_MM_SS_SSS = \"yyyy-MM-dd#HH:mm:ss:SSS\";\n    public static final String YYYYMMDDHHMMSS = \"yyyyMMddHHmmss\";\n    private final static char[] HEX_ARRAY;\n    private final static int PID;\n\n    static {\n        HEX_ARRAY = \"0123456789ABCDEF\".toCharArray();\n        Supplier<Integer> supplier = () -> {\n            // format: \"pid@hostname\"\n            String currentJVM = ManagementFactory.getRuntimeMXBean().getName();\n            try {\n                return Integer.parseInt(currentJVM.substring(0, currentJVM.indexOf('@')));\n            } catch (Exception e) {\n                return -1;\n            }\n        };\n        PID = supplier.get();\n    }\n\n    public static int getPid() {\n        return PID;\n    }\n\n    public static void sleep(long sleepMs) {\n        sleep(sleepMs, TimeUnit.MILLISECONDS);\n    }\n\n    public static void sleep(long timeOut, TimeUnit timeUnit) {\n        if (null == timeUnit) {\n            return;\n        }\n        try {\n            timeUnit.sleep(timeOut);\n        } catch (Throwable ignored) {\n\n        }\n    }\n\n    public static String currentStackTrace() {\n        StringBuilder sb = new StringBuilder();\n        StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();\n        for (StackTraceElement ste : stackTrace) {\n            sb.append(\"\\n\\t\");\n            sb.append(ste.toString());\n        }\n\n        return sb.toString();\n    }\n\n    public static String offset2FileName(final long offset) {\n        final NumberFormat nf = NumberFormat.getInstance();\n        nf.setMinimumIntegerDigits(20);\n        nf.setMaximumFractionDigits(0);\n        nf.setGroupingUsed(false);\n        return nf.format(offset);\n    }\n\n    public static long computeElapsedTimeMilliseconds(final long beginTime) {\n        return System.currentTimeMillis() - beginTime;\n    }\n\n    public static boolean isItTimeToDo(final String when) {\n        String[] whiles = when.split(\";\");\n        if (whiles.length > 0) {\n            Calendar now = Calendar.getInstance();\n            for (String w : whiles) {\n                int nowHour = Integer.parseInt(w);\n                if (nowHour == now.get(Calendar.HOUR_OF_DAY)) {\n                    return true;\n                }\n            }\n        }\n\n        return false;\n    }\n\n    public static String timeMillisToHumanString() {\n        return timeMillisToHumanString(System.currentTimeMillis());\n    }\n\n    public static String timeMillisToHumanString(final long t) {\n        Calendar cal = Calendar.getInstance();\n        cal.setTimeInMillis(t);\n        return String.format(\"%04d%02d%02d%02d%02d%02d%03d\", cal.get(Calendar.YEAR), cal.get(Calendar.MONTH) + 1,\n            cal.get(Calendar.DAY_OF_MONTH), cal.get(Calendar.HOUR_OF_DAY), cal.get(Calendar.MINUTE), cal.get(Calendar.SECOND),\n            cal.get(Calendar.MILLISECOND));\n    }\n\n    public static long computeNextMorningTimeMillis() {\n        Calendar cal = Calendar.getInstance();\n        cal.setTimeInMillis(System.currentTimeMillis());\n        cal.add(Calendar.DAY_OF_MONTH, 1);\n        cal.set(Calendar.HOUR_OF_DAY, 0);\n        cal.set(Calendar.MINUTE, 0);\n        cal.set(Calendar.SECOND, 0);\n        cal.set(Calendar.MILLISECOND, 0);\n\n        return cal.getTimeInMillis();\n    }\n\n    public static long computeNextMinutesTimeMillis() {\n        Calendar cal = Calendar.getInstance();\n        cal.setTimeInMillis(System.currentTimeMillis());\n        cal.add(Calendar.DAY_OF_MONTH, 0);\n        cal.add(Calendar.HOUR_OF_DAY, 0);\n        cal.add(Calendar.MINUTE, 1);\n        cal.set(Calendar.SECOND, 0);\n        cal.set(Calendar.MILLISECOND, 0);\n\n        return cal.getTimeInMillis();\n    }\n\n    public static long computeNextHourTimeMillis() {\n        Calendar cal = Calendar.getInstance();\n        cal.setTimeInMillis(System.currentTimeMillis());\n        cal.add(Calendar.DAY_OF_MONTH, 0);\n        cal.add(Calendar.HOUR_OF_DAY, 1);\n        cal.set(Calendar.MINUTE, 0);\n        cal.set(Calendar.SECOND, 0);\n        cal.set(Calendar.MILLISECOND, 0);\n\n        return cal.getTimeInMillis();\n    }\n\n    public static long computeNextHalfHourTimeMillis() {\n        Calendar cal = Calendar.getInstance();\n        cal.setTimeInMillis(System.currentTimeMillis());\n        cal.add(Calendar.DAY_OF_MONTH, 0);\n        cal.add(Calendar.HOUR_OF_DAY, 1);\n        cal.set(Calendar.MINUTE, 30);\n        cal.set(Calendar.SECOND, 0);\n        cal.set(Calendar.MILLISECOND, 0);\n\n        return cal.getTimeInMillis();\n    }\n\n    public static String timeMillisToHumanString2(final long t) {\n        Calendar cal = Calendar.getInstance();\n        cal.setTimeInMillis(t);\n        return String.format(\"%04d-%02d-%02d %02d:%02d:%02d,%03d\",\n            cal.get(Calendar.YEAR),\n            cal.get(Calendar.MONTH) + 1,\n            cal.get(Calendar.DAY_OF_MONTH),\n            cal.get(Calendar.HOUR_OF_DAY),\n            cal.get(Calendar.MINUTE),\n            cal.get(Calendar.SECOND),\n            cal.get(Calendar.MILLISECOND));\n    }\n\n    public static String timeMillisToHumanString3(final long t) {\n        Calendar cal = Calendar.getInstance();\n        cal.setTimeInMillis(t);\n        return String.format(\"%04d%02d%02d%02d%02d%02d\",\n            cal.get(Calendar.YEAR),\n            cal.get(Calendar.MONTH) + 1,\n            cal.get(Calendar.DAY_OF_MONTH),\n            cal.get(Calendar.HOUR_OF_DAY),\n            cal.get(Calendar.MINUTE),\n            cal.get(Calendar.SECOND));\n    }\n\n    public static long getTotalSpace(final String path) {\n        if (null == path || path.isEmpty())\n            return -1;\n        try {\n            File file = new File(path);\n            if (!file.exists())\n                return -1;\n            return file.getTotalSpace();\n        } catch (Exception e) {\n            return -1;\n        }\n    }\n\n    public static boolean isPathExists(final String path) {\n        File file = new File(path);\n        return file.exists();\n    }\n\n    public static double getDiskPartitionSpaceUsedPercent(final String path) {\n        if (null == path || path.isEmpty()) {\n            STORE_LOG.error(\"Error when measuring disk space usage, path is null or empty, path : {}\", path);\n            return -1;\n        }\n\n        try {\n            File file = new File(path);\n\n            if (!file.exists()) {\n                STORE_LOG.error(\"Error when measuring disk space usage, file doesn't exist on this path: {}\", path);\n                return -1;\n            }\n\n            long totalSpace = file.getTotalSpace();\n\n            if (totalSpace > 0) {\n                long usedSpace = totalSpace - file.getFreeSpace();\n                long usableSpace = file.getUsableSpace();\n                long entireSpace = usedSpace + usableSpace;\n                long roundNum = 0;\n                if (usedSpace * 100 % entireSpace != 0) {\n                    roundNum = 1;\n                }\n                long result = usedSpace * 100 / entireSpace + roundNum;\n                return result / 100.0;\n            }\n        } catch (Exception e) {\n            STORE_LOG.error(\"Error when measuring disk space usage, got exception: :\", e);\n            return -1;\n        }\n\n        return -1;\n    }\n\n    public static long getDiskPartitionTotalSpace(final String path) {\n        if (null == path || path.isEmpty()) {\n            return -1;\n        }\n\n        try {\n            File file = new File(path);\n\n            if (!file.exists()) {\n                return -1;\n            }\n\n            return file.getTotalSpace() - file.getFreeSpace() + file.getUsableSpace();\n        } catch (Exception e) {\n            return -1;\n        }\n    }\n\n    public static int crc32(byte[] array) {\n        if (array != null) {\n            return crc32(array, 0, array.length);\n        }\n\n        return 0;\n    }\n\n    public static int crc32(byte[] array, int offset, int length) {\n        CRC32 crc32 = new CRC32();\n        crc32.update(array, offset, length);\n        return (int) (crc32.getValue() & 0x7FFFFFFF);\n    }\n\n    public static int crc32(ByteBuffer byteBuffer) {\n        CRC32 crc32 = new CRC32();\n        crc32.update(byteBuffer);\n        return (int) (crc32.getValue() & 0x7FFFFFFF);\n    }\n\n    public static int crc32(ByteBuffer[] byteBuffers) {\n        CRC32 crc32 = new CRC32();\n        for (ByteBuffer buffer : byteBuffers) {\n            crc32.update(buffer);\n        }\n        return (int) (crc32.getValue() & 0x7FFFFFFF);\n    }\n\n    public static String bytes2string(byte[] src) {\n        char[] hexChars = new char[src.length * 2];\n        for (int j = 0; j < src.length; j++) {\n            int v = src[j] & 0xFF;\n            hexChars[j * 2] = HEX_ARRAY[v >>> 4];\n            hexChars[j * 2 + 1] = HEX_ARRAY[v & 0x0F];\n        }\n        return new String(hexChars);\n    }\n\n    public static void writeInt(char[] buffer, int pos, int value) {\n        for (int moveBits = 28; moveBits >= 0; moveBits -= 4) {\n            buffer[pos++] = HEX_ARRAY[(value >>> moveBits) & 0x0F];\n        }\n    }\n\n    public static void writeShort(char[] buffer, int pos, int value) {\n        for (int moveBits = 12; moveBits >= 0; moveBits -= 4) {\n            buffer[pos++] = HEX_ARRAY[(value >>> moveBits) & 0x0F];\n        }\n    }\n\n    public static byte[] string2bytes(String hexString) {\n        if (hexString == null || hexString.equals(\"\")) {\n            return null;\n        }\n        hexString = hexString.toUpperCase();\n        int length = hexString.length() / 2;\n        char[] hexChars = hexString.toCharArray();\n        byte[] d = new byte[length];\n        for (int i = 0; i < length; i++) {\n            int pos = i * 2;\n            d[i] = (byte) (charToByte(hexChars[pos]) << 4 | charToByte(hexChars[pos + 1]));\n        }\n        return d;\n    }\n\n    private static byte charToByte(char c) {\n        return (byte) \"0123456789ABCDEF\".indexOf(c);\n    }\n\n    /**\n     * use {@link org.apache.rocketmq.common.compression.Compressor#decompress(byte[])} instead.\n     */\n    @Deprecated\n    public static byte[] uncompress(final byte[] src) throws IOException {\n        byte[] result = src;\n        byte[] uncompressData = new byte[src.length];\n        ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(src);\n        InflaterInputStream inflaterInputStream = new InflaterInputStream(byteArrayInputStream);\n        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(src.length);\n\n        try {\n            while (true) {\n                int len = inflaterInputStream.read(uncompressData, 0, uncompressData.length);\n                if (len <= 0) {\n                    break;\n                }\n                byteArrayOutputStream.write(uncompressData, 0, len);\n            }\n            byteArrayOutputStream.flush();\n            result = byteArrayOutputStream.toByteArray();\n        } catch (IOException e) {\n            throw e;\n        } finally {\n            try {\n                byteArrayInputStream.close();\n            } catch (IOException e) {\n                log.error(\"Failed to close the stream\", e);\n            }\n            try {\n                inflaterInputStream.close();\n            } catch (IOException e) {\n                log.error(\"Failed to close the stream\", e);\n            }\n            try {\n                byteArrayOutputStream.close();\n            } catch (IOException e) {\n                log.error(\"Failed to close the stream\", e);\n            }\n        }\n\n        return result;\n    }\n\n    /**\n     * use {@link org.apache.rocketmq.common.compression.Compressor#compress(byte[], int)} instead.\n     */\n    @Deprecated\n    public static byte[] compress(final byte[] src, final int level) throws IOException {\n        byte[] result = src;\n        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(src.length);\n        java.util.zip.Deflater defeater = new java.util.zip.Deflater(level);\n        DeflaterOutputStream deflaterOutputStream = new DeflaterOutputStream(byteArrayOutputStream, defeater);\n        try {\n            deflaterOutputStream.write(src);\n            deflaterOutputStream.finish();\n            deflaterOutputStream.close();\n            result = byteArrayOutputStream.toByteArray();\n        } catch (IOException e) {\n            defeater.end();\n            throw e;\n        } finally {\n            try {\n                byteArrayOutputStream.close();\n            } catch (IOException ignored) {\n            }\n\n            defeater.end();\n        }\n\n        return result;\n    }\n\n    public static int asInt(String str, int defaultValue) {\n        try {\n            return Integer.parseInt(str);\n        } catch (Exception e) {\n            return defaultValue;\n        }\n    }\n\n    public static long asLong(String str, long defaultValue) {\n        try {\n            return Long.parseLong(str);\n        } catch (Exception e) {\n            return defaultValue;\n        }\n    }\n\n    public static String formatDate(Date date, String pattern) {\n        SimpleDateFormat df = new SimpleDateFormat(pattern);\n        return df.format(date);\n    }\n\n    public static Date parseDate(String date, String pattern) {\n        SimpleDateFormat df = new SimpleDateFormat(pattern);\n        try {\n            return df.parse(date);\n        } catch (ParseException e) {\n            return null;\n        }\n    }\n\n    public static String responseCode2String(final int code) {\n        return Integer.toString(code);\n    }\n\n    public static String frontStringAtLeast(final String str, final int size) {\n        if (str != null) {\n            if (str.length() > size) {\n                return str.substring(0, size);\n            }\n        }\n\n        return str;\n    }\n\n    public static boolean isBlank(String str) {\n        return StringUtils.isBlank(str);\n    }\n\n    public static String jstack() {\n        return jstack(Thread.getAllStackTraces());\n    }\n\n    public static String jstack(Map<Thread, StackTraceElement[]> map) {\n        StringBuilder result = new StringBuilder();\n        try {\n            Iterator<Map.Entry<Thread, StackTraceElement[]>> ite = map.entrySet().iterator();\n            while (ite.hasNext()) {\n                Map.Entry<Thread, StackTraceElement[]> entry = ite.next();\n                StackTraceElement[] elements = entry.getValue();\n                Thread thread = entry.getKey();\n                if (elements != null && elements.length > 0) {\n                    String threadName = entry.getKey().getName();\n                    result.append(String.format(\"%-40sTID: %d STATE: %s%n\", threadName, thread.getId(), thread.getState()));\n                    for (StackTraceElement el : elements) {\n                        result.append(String.format(\"%-40s%s%n\", threadName, el.toString()));\n                    }\n                    result.append(\"\\n\");\n                }\n            }\n        } catch (Throwable e) {\n            result.append(exceptionSimpleDesc(e));\n        }\n\n        return result.toString();\n    }\n\n    public static String exceptionSimpleDesc(final Throwable e) {\n        StringBuilder sb = new StringBuilder();\n        if (e != null) {\n            sb.append(e);\n\n            StackTraceElement[] stackTrace = e.getStackTrace();\n            if (stackTrace != null && stackTrace.length > 0) {\n                StackTraceElement element = stackTrace[0];\n                sb.append(\", \");\n                sb.append(element.toString());\n            }\n        }\n        return sb.toString();\n    }\n\n    public static boolean isInternalIP(byte[] ip) {\n        if (ip.length != 4) {\n            throw new RuntimeException(\"illegal ipv4 bytes\");\n        }\n\n        //10.0.0.0~10.255.255.255\n        //172.16.0.0~172.31.255.255\n        //192.168.0.0~192.168.255.255\n        //127.0.0.0~127.255.255.255\n        if (ip[0] == (byte) 10) {\n            return true;\n        } else if (ip[0] == (byte) 127) {\n            return true;\n        } else if (ip[0] == (byte) 172) {\n            return ip[1] >= (byte) 16 && ip[1] <= (byte) 31;\n        } else if (ip[0] == (byte) 192) {\n            return ip[1] == (byte) 168;\n        }\n        return false;\n    }\n\n    public static boolean isInternalV6IP(InetAddress inetAddr) {\n        return inetAddr.isAnyLocalAddress() // Wild card ipv6\n            || inetAddr.isLinkLocalAddress() // Single broadcast ipv6 address: fe80:xx:xx...\n            || inetAddr.isLoopbackAddress() //Loopback ipv6 address\n            || inetAddr.isSiteLocalAddress();// Site local ipv6 address: fec0:xx:xx...\n    }\n\n    private static boolean ipCheck(byte[] ip) {\n        if (ip.length != 4) {\n            throw new RuntimeException(\"illegal ipv4 bytes\");\n        }\n\n        InetAddressValidator validator = InetAddressValidator.getInstance();\n        return validator.isValidInet4Address(ipToIPv4Str(ip));\n    }\n\n    private static boolean ipV6Check(byte[] ip) {\n        if (ip.length != 16) {\n            throw new RuntimeException(\"illegal ipv6 bytes\");\n        }\n\n        InetAddressValidator validator = InetAddressValidator.getInstance();\n        return validator.isValidInet6Address(ipToIPv6Str(ip));\n    }\n\n    public static String ipToIPv4Str(byte[] ip) {\n        if (ip.length != 4) {\n            return null;\n        }\n        return new StringBuilder().append(ip[0] & 0xFF).append(\".\").append(\n                ip[1] & 0xFF).append(\".\").append(ip[2] & 0xFF)\n            .append(\".\").append(ip[3] & 0xFF).toString();\n    }\n\n    public static String ipToIPv6Str(byte[] ip) {\n        if (ip.length != 16) {\n            return null;\n        }\n\n        StringBuilder sb = new StringBuilder();\n        for (int i = 0; i < ip.length; i++) {\n            String hex = Integer.toHexString(ip[i] & 0xFF);\n            if (hex.length() < 2) {\n                sb.append(0);\n            }\n            sb.append(hex);\n            if (i % 2 == 1 && i < ip.length - 1) {\n                sb.append(\":\");\n            }\n        }\n        return sb.toString();\n    }\n\n    public static byte[] getIP() {\n        try {\n            Enumeration<NetworkInterface> allNetInterfaces = NetworkInterface.getNetworkInterfaces();\n            InetAddress ip;\n            byte[] internalIP = null;\n            while (allNetInterfaces.hasMoreElements()) {\n                NetworkInterface netInterface = allNetInterfaces.nextElement();\n                Enumeration<InetAddress> addresses = netInterface.getInetAddresses();\n                while (addresses.hasMoreElements()) {\n                    ip =  addresses.nextElement();\n                    if (ip instanceof Inet4Address) {\n                        byte[] ipByte = ip.getAddress();\n                        if (ipByte.length == 4) {\n                            if (ipCheck(ipByte)) {\n                                if (!isInternalIP(ipByte)) {\n                                    return ipByte;\n                                } else if (internalIP == null || internalIP[0] == (byte) 127) {\n                                    internalIP = ipByte;\n                                }\n                            }\n                        }\n                    } else if (ip instanceof Inet6Address) {\n                        byte[] ipByte = ip.getAddress();\n                        if (ipByte.length == 16) {\n                            if (ipV6Check(ipByte)) {\n                                if (!isInternalV6IP(ip)) {\n                                    return ipByte;\n                                }\n                            }\n                        }\n                    }\n                }\n            }\n            if (internalIP != null) {\n                return internalIP;\n            } else {\n                throw new RuntimeException(\"Can not get local ip\");\n            }\n        } catch (Exception e) {\n            throw new RuntimeException(\"Can not get local ip\", e);\n        }\n    }\n\n    public static void deleteFile(File file) {\n        if (!file.exists()) {\n            return;\n        }\n        if (file.isFile()) {\n            file.delete();\n        } else if (file.isDirectory()) {\n            File[] files = file.listFiles();\n            if (files != null) {\n                for (File file1 : files) {\n                    deleteFile(file1);\n                }\n            }\n            file.delete();\n        }\n    }\n\n    public static String join(List<String> list, String splitter) {\n        if (list == null) {\n            return null;\n        }\n\n        StringBuilder str = new StringBuilder();\n        for (int i = 0; i < list.size(); i++) {\n            str.append(list.get(i));\n            if (i == list.size() - 1) {\n                break;\n            }\n            str.append(splitter);\n        }\n        return str.toString();\n    }\n\n    public static List<String> split(String str, String splitter) {\n        if (str == null) {\n            return null;\n        }\n\n        if (StringUtils.isBlank(str)) {\n            return Collections.EMPTY_LIST;\n        }\n\n        String[] addrArray = str.split(splitter);\n        return Arrays.asList(addrArray);\n    }\n\n    public static void deleteEmptyDirectory(File file) {\n        if (file == null || !file.exists()) {\n            return;\n        }\n        if (!file.isDirectory()) {\n            return;\n        }\n        File[] files = file.listFiles();\n        if (files == null || files.length <= 0) {\n            file.delete();\n            STORE_LOG.info(\"delete empty direct, {}\", file.getPath());\n        }\n    }\n\n    /**\n     * Free direct-buffer's memory actively.\n     * @param buffer Direct buffer to free.\n     */\n    public static void cleanBuffer(final ByteBuffer buffer) {\n        if (null == buffer) {\n            return;\n        }\n\n        if (!buffer.isDirect()) {\n            return;\n        }\n\n        PlatformDependent.freeDirectBuffer(buffer);\n    }\n\n    public static void ensureDirOK(final String dirName) {\n        if (dirName != null) {\n            if (dirName.contains(MixAll.MULTI_PATH_SPLITTER)) {\n                String[] dirs = dirName.trim().split(MixAll.MULTI_PATH_SPLITTER);\n                for (String dir : dirs) {\n                    createDirIfNotExist(dir);\n                }\n            } else {\n                createDirIfNotExist(dirName);\n            }\n        }\n    }\n\n    private static void createDirIfNotExist(String dirName) {\n        File f = new File(dirName);\n        if (!f.exists()) {\n            boolean result = f.mkdirs();\n            STORE_LOG.info(dirName + \" mkdir \" + (result ? \"OK\" : \"Failed\"));\n        }\n    }\n\n    public static long calculateFileSizeInPath(File path) {\n        long size = 0;\n        try {\n            if (!path.exists() || Files.isSymbolicLink(path.toPath())) {\n                return 0;\n            }\n            if (path.isFile()) {\n                return path.length();\n            }\n            if (path.isDirectory()) {\n                File[] files = path.listFiles();\n                if (files != null && files.length > 0) {\n                    for (File file : files) {\n                        long fileSize = calculateFileSizeInPath(file);\n                        if (fileSize == -1) return -1;\n                        size += fileSize;\n                    }\n                }\n            }\n        } catch (Exception e) {\n            log.error(\"calculate all file size in: {} error\", path.getAbsolutePath(), e);\n            return -1;\n        }\n        return size;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/action/Action.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.action;\n\nimport com.alibaba.fastjson2.annotation.JSONField;\nimport org.apache.commons.lang3.StringUtils;\n\npublic enum Action {\n\n    UNKNOWN((byte) 0, \"Unknown\"),\n\n    ALL((byte) 1, \"All\"),\n\n    ANY((byte) 2, \"Any\"),\n\n    PUB((byte) 3, \"Pub\"),\n\n    SUB((byte) 4, \"Sub\"),\n\n    CREATE((byte) 5, \"Create\"),\n\n    UPDATE((byte) 6, \"Update\"),\n\n    DELETE((byte) 7, \"Delete\"),\n\n    GET((byte) 8, \"Get\"),\n\n    LIST((byte) 9, \"List\");\n\n    @JSONField(value = true)\n    private final byte code;\n    private final String name;\n\n    Action(byte code, String name) {\n        this.code = code;\n        this.name = name;\n    }\n\n    public static Action getByName(String name) {\n        for (Action action : Action.values()) {\n            if (StringUtils.equalsIgnoreCase(action.getName(), name)) {\n                return action;\n            }\n        }\n        return null;\n    }\n\n    public byte getCode() {\n        return code;\n    }\n\n    public String getName() {\n        return name;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/action/RocketMQAction.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.action;\n\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.RetentionPolicy;\nimport org.apache.rocketmq.common.resource.ResourceType;\n\n@Retention(RetentionPolicy.RUNTIME)\npublic @interface RocketMQAction {\n\n    int value();\n\n    ResourceType resource() default ResourceType.UNKNOWN;\n\n    Action[] action();\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/annotation/ImportantField.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.annotation;\n\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.ElementType;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.RetentionPolicy;\nimport java.lang.annotation.Target;\n\n@Documented\n@Retention(RetentionPolicy.RUNTIME)\n@Target({ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER, ElementType.LOCAL_VARIABLE})\npublic @interface ImportantField {\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/attribute/Attribute.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.attribute;\n\npublic abstract class Attribute {\n    protected String name;\n    protected boolean changeable;\n\n    public abstract void verify(String value);\n\n    public Attribute(String name, boolean changeable) {\n        this.name = name;\n        this.changeable = changeable;\n    }\n\n    public String getName() {\n        return name;\n    }\n\n    public void setName(String name) {\n        this.name = name;\n    }\n\n    public boolean isChangeable() {\n        return changeable;\n    }\n\n    public void setChangeable(boolean changeable) {\n        this.changeable = changeable;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/attribute/AttributeParser.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.attribute;\n\nimport com.google.common.base.Strings;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\npublic class AttributeParser {\n\n    public static final String ATTR_ARRAY_SEPARATOR_COMMA = \",\";\n\n    public static final String ATTR_KEY_VALUE_EQUAL_SIGN = \"=\";\n\n    public static final String ATTR_ADD_PLUS_SIGN = \"+\";\n\n    private static final String ATTR_DELETE_MINUS_SIGN = \"-\";\n\n    public static Map<String, String> parseToMap(String attributesModification) {\n        if (Strings.isNullOrEmpty(attributesModification)) {\n            return new HashMap<>();\n        }\n\n        // format: +key1=value1,+key2=value2,-key3,+key4=value4\n        Map<String, String> attributes = new HashMap<>();\n        String[] kvs = attributesModification.split(ATTR_ARRAY_SEPARATOR_COMMA);\n        for (String kv : kvs) {\n            String key;\n            String value;\n            if (kv.contains(ATTR_KEY_VALUE_EQUAL_SIGN)) {\n                String[] splits = kv.split(ATTR_KEY_VALUE_EQUAL_SIGN);\n                key = splits[0];\n                value = splits[1];\n                if (!key.contains(ATTR_ADD_PLUS_SIGN)) {\n                    throw new RuntimeException(\"add/alter attribute format is wrong: \" + key);\n                }\n            } else {\n                key = kv;\n                value = \"\";\n                if (!key.contains(ATTR_DELETE_MINUS_SIGN)) {\n                    throw new RuntimeException(\"delete attribute format is wrong: \" + key);\n                }\n            }\n            String old = attributes.put(key, value);\n            if (old != null) {\n                throw new RuntimeException(\"key duplication: \" + key);\n            }\n        }\n        return attributes;\n    }\n\n    public static String parseToString(Map<String, String> attributes) {\n        if (attributes == null || attributes.size() == 0) {\n            return \"\";\n        }\n\n        List<String> kvs = new ArrayList<>();\n        for (Map.Entry<String, String> entry : attributes.entrySet()) {\n\n            String value = entry.getValue();\n            if (Strings.isNullOrEmpty(value)) {\n                kvs.add(entry.getKey());\n            } else {\n                kvs.add(entry.getKey() + ATTR_KEY_VALUE_EQUAL_SIGN + entry.getValue());\n            }\n        }\n        return String.join(ATTR_ARRAY_SEPARATOR_COMMA, kvs);\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/attribute/AttributeUtil.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.attribute;\n\nimport com.google.common.base.Strings;\nimport com.google.common.collect.ImmutableMap;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Map;\nimport java.util.Set;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class AttributeUtil {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);\n    public static Map<String, String> alterCurrentAttributes(boolean create, Map<String, Attribute> all,\n        ImmutableMap<String, String> currentAttributes, ImmutableMap<String, String> newAttributes) {\n\n        Map<String, String> init = new HashMap<>();\n        Map<String, String> add = new HashMap<>();\n        Map<String, String> update = new HashMap<>();\n        Map<String, String> delete = new HashMap<>();\n        Set<String> keys = new HashSet<>();\n\n        for (Map.Entry<String, String> attribute : newAttributes.entrySet()) {\n            String key = attribute.getKey();\n            String realKey = realKey(key);\n            String value = attribute.getValue();\n\n            validate(realKey);\n            duplicationCheck(keys, realKey);\n\n            if (create) {\n                if (key.startsWith(\"+\")) {\n                    init.put(realKey, value);\n                } else {\n                    throw new RuntimeException(\"only add attribute is supported while creating topic. key: \" + realKey);\n                }\n            } else {\n                if (key.startsWith(\"+\")) {\n                    if (!currentAttributes.containsKey(realKey)) {\n                        add.put(realKey, value);\n                    } else {\n                        update.put(realKey, value);\n                    }\n                } else if (key.startsWith(\"-\")) {\n                    if (!currentAttributes.containsKey(realKey)) {\n                        throw new RuntimeException(\"attempt to delete a nonexistent key: \" + realKey);\n                    }\n                    delete.put(realKey, value);\n                } else {\n                    throw new RuntimeException(\"wrong format key: \" + realKey);\n                }\n            }\n        }\n\n        validateAlter(all, init, true, false);\n        validateAlter(all, add, false, false);\n        validateAlter(all, update, false, false);\n        validateAlter(all, delete, false, true);\n\n        log.info(\"add: {}, update: {}, delete: {}\", add, update, delete);\n        HashMap<String, String> finalAttributes = new HashMap<>(currentAttributes);\n        finalAttributes.putAll(init);\n        finalAttributes.putAll(add);\n        finalAttributes.putAll(update);\n        for (String s : delete.keySet()) {\n            finalAttributes.remove(s);\n        }\n        return finalAttributes;\n    }\n\n    private static void duplicationCheck(Set<String> keys, String key) {\n        boolean notExist = keys.add(key);\n        if (!notExist) {\n            throw new RuntimeException(\"alter duplication key. key: \" + key);\n        }\n    }\n\n    private static void validate(String kvAttribute) {\n        if (Strings.isNullOrEmpty(kvAttribute)) {\n            throw new RuntimeException(\"kv string format wrong.\");\n        }\n\n        if (kvAttribute.contains(\"+\")) {\n            throw new RuntimeException(\"kv string format wrong.\");\n        }\n\n        if (kvAttribute.contains(\"-\")) {\n            throw new RuntimeException(\"kv string format wrong.\");\n        }\n    }\n\n    private static void validateAlter(Map<String, Attribute> all, Map<String, String> alter, boolean init, boolean delete) {\n        for (Map.Entry<String, String> entry : alter.entrySet()) {\n            String key = entry.getKey();\n            String value = entry.getValue();\n\n            Attribute attribute = all.get(key);\n            if (attribute == null) {\n                throw new RuntimeException(\"unsupported key: \" + key);\n            }\n            if (!init && !attribute.isChangeable()) {\n                throw new RuntimeException(\"attempt to update an unchangeable attribute. key: \" + key);\n            }\n\n            if (!delete) {\n                attribute.verify(value);\n            }\n        }\n    }\n\n    private static String realKey(String key) {\n        return key.substring(1);\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/attribute/BooleanAttribute.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.attribute;\n\nimport static com.google.common.base.Preconditions.checkNotNull;\n\npublic class BooleanAttribute extends Attribute {\n    private final boolean defaultValue;\n\n    public BooleanAttribute(String name, boolean changeable, boolean defaultValue) {\n        super(name, changeable);\n        this.defaultValue = defaultValue;\n    }\n\n    @Override\n    public void verify(String value) {\n        checkNotNull(value);\n\n        if (!\"false\".equalsIgnoreCase(value) && !\"true\".equalsIgnoreCase(value)) {\n            throw new RuntimeException(\"boolean attribute format is wrong.\");\n        }\n    }\n\n    public boolean getDefaultValue() {\n        return defaultValue;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/attribute/CQType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.attribute;\n\npublic enum CQType {\n    SimpleCQ,\n    BatchCQ,\n    RocksDBCQ\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/attribute/CleanupPolicy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\npackage org.apache.rocketmq.common.attribute;\n\npublic enum CleanupPolicy {\n    DELETE,\n    COMPACTION\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/attribute/EnumAttribute.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.attribute;\n\nimport java.util.Set;\n\npublic class EnumAttribute extends Attribute {\n    private final Set<String> universe;\n    private final String defaultValue;\n\n    public EnumAttribute(String name, boolean changeable, Set<String> universe, String defaultValue) {\n        super(name, changeable);\n        this.universe = universe;\n        this.defaultValue = defaultValue;\n    }\n\n    @Override\n    public void verify(String value) {\n        if (!this.universe.contains(value)) {\n            throw new RuntimeException(\"value is not in set: \" + this.universe);\n        }\n    }\n\n    public String getDefaultValue() {\n        return defaultValue;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/attribute/LiteSubModel.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.attribute;\n\npublic enum LiteSubModel {\n    Shared,\n    Exclusive\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/attribute/LongRangeAttribute.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.attribute;\n\nimport static java.lang.String.format;\n\npublic class LongRangeAttribute extends Attribute {\n    private final long min;\n    private final long max;\n    private final long defaultValue;\n\n    public LongRangeAttribute(String name, boolean changeable, long min, long max, long defaultValue) {\n        super(name, changeable);\n        this.min = min;\n        this.max = max;\n        this.defaultValue = defaultValue;\n    }\n\n    @Override\n    public void verify(String value) {\n        long l = Long.parseLong(value);\n        if (l < min || l > max) {\n            throw new RuntimeException(format(\"value is not in range(%d, %d)\", min, max));\n        }\n    }\n\n    public long getDefaultValue() {\n        return defaultValue;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/attribute/StringAttribute.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.attribute;\n\nimport static com.google.common.base.Preconditions.checkNotNull;\n\npublic class StringAttribute extends Attribute {\n\n    public StringAttribute(String name, boolean changeable) {\n        super(name, changeable);\n    }\n\n    @Override\n    public void verify(String value) {\n        checkNotNull(value);\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/attribute/TopicMessageType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.attribute;\n\nimport com.google.common.collect.Sets;\nimport java.util.Map;\nimport java.util.Set;\nimport org.apache.rocketmq.common.message.MessageConst;\n\npublic enum TopicMessageType {\n    UNSPECIFIED(\"UNSPECIFIED\"),\n    NORMAL(\"NORMAL\"),\n    FIFO(\"FIFO\"),\n    DELAY(\"DELAY\"),\n    TRANSACTION(\"TRANSACTION\"),\n    PRIORITY(\"PRIORITY\"),\n    LITE(\"LITE\"),\n    MIXED(\"MIXED\");\n\n    private final String value;\n\n    TopicMessageType(String value) {\n        this.value = value;\n    }\n\n    public static Set<String> topicMessageTypeSet() {\n        return Sets.newHashSet(UNSPECIFIED.value, NORMAL.value, FIFO.value, DELAY.value, TRANSACTION.value,\n            PRIORITY.value, LITE.value, MIXED.value);\n    }\n\n    public String getValue() {\n        return value;\n    }\n\n    public static TopicMessageType parseFromMessageProperty(Map<String, String> messageProperty) {\n        // the parse order keeps message types mutually exclusive\n        if (Boolean.parseBoolean(messageProperty.get(MessageConst.PROPERTY_TRANSACTION_PREPARED))) {\n            return TopicMessageType.TRANSACTION;\n        } else if (messageProperty.get(MessageConst.PROPERTY_DELAY_TIME_LEVEL) != null\n            || messageProperty.get(MessageConst.PROPERTY_TIMER_DELIVER_MS) != null\n            || messageProperty.get(MessageConst.PROPERTY_TIMER_DELAY_SEC) != null\n            || messageProperty.get(MessageConst.PROPERTY_TIMER_DELAY_MS) != null) {\n            return TopicMessageType.DELAY;\n        } else if (messageProperty.get(MessageConst.PROPERTY_SHARDING_KEY) != null) {\n            return TopicMessageType.FIFO;\n        } else if (messageProperty.get(MessageConst.PROPERTY_PRIORITY) != null) {\n            return TopicMessageType.PRIORITY;\n        } else if (messageProperty.get(MessageConst.PROPERTY_LITE_TOPIC) != null) {\n            return TopicMessageType.LITE;\n        }\n        return TopicMessageType.NORMAL;\n    }\n\n    public String getMetricsValue() {\n        return value.toLowerCase();\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/chain/Handler.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.chain;\n\npublic interface Handler<T, R> {\n\n    R handle(T t, HandlerChain<T, R> chain);\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/chain/HandlerChain.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.chain;\n\nimport java.util.ArrayList;\nimport java.util.Iterator;\nimport java.util.List;\n\npublic class HandlerChain<T, R> {\n\n    private List<Handler<T, R>> handlers;\n    private Iterator<Handler<T, R>> iterator;\n\n    public static <T, R> HandlerChain<T, R> create() {\n        return new HandlerChain<>();\n    }\n\n    public HandlerChain<T, R> addNext(Handler<T, R> handler) {\n        if (this.handlers == null) {\n            this.handlers = new ArrayList<>();\n        }\n        this.handlers.add(handler);\n        return this;\n    }\n\n    public R handle(T t) {\n        if (iterator == null) {\n            iterator = handlers.iterator();\n        }\n        if (iterator.hasNext()) {\n            Handler<T, R> handler = iterator.next();\n            return handler.handle(t, this);\n        }\n        return null;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/coldctr/AccAndTimeStamp.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.coldctr;\n\nimport java.util.concurrent.atomic.AtomicLong;\n\npublic class AccAndTimeStamp {\n\n    public AtomicLong coldAcc = new AtomicLong(0L);\n    public Long lastColdReadTimeMills = System.currentTimeMillis();\n    public Long createTimeMills = System.currentTimeMillis();\n\n    public AccAndTimeStamp(AtomicLong coldAcc) {\n        this.coldAcc = coldAcc;\n    }\n\n    public AtomicLong getColdAcc() {\n        return coldAcc;\n    }\n\n    public void setColdAcc(AtomicLong coldAcc) {\n        this.coldAcc = coldAcc;\n    }\n\n    public Long getLastColdReadTimeMills() {\n        return lastColdReadTimeMills;\n    }\n\n    public void setLastColdReadTimeMills(Long lastColdReadTimeMills) {\n        this.lastColdReadTimeMills = lastColdReadTimeMills;\n    }\n\n    public Long getCreateTimeMills() {\n        return createTimeMills;\n    }\n\n    public void setCreateTimeMills(Long createTimeMills) {\n        this.createTimeMills = createTimeMills;\n    }\n\n    @Override\n    public String toString() {\n        return \"AccAndTimeStamp{\" +\n            \"coldAcc=\" + coldAcc +\n            \", lastColdReadTimeMills=\" + lastColdReadTimeMills +\n            \", createTimeMills=\" + createTimeMills +\n            '}';\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/compression/CompressionType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.compression;\n\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\n\npublic enum CompressionType {\n\n    /**\n     *    Compression types number can be extended to seven {@link MessageSysFlag}\n     *\n     *    Benchmarks from https://github.com/facebook/zstd\n     *\n     *    |   Compressor   |  Ratio  | Compression | Decompress |\n     *    |----------------|---------|-------------|------------|\n     *    |   zstd 1.5.1   |  2.887  |   530 MB/s  |  1700 MB/s |\n     *    |  zlib 1.2.11   |  2.743  |    95 MB/s  |   400 MB/s |\n     *    |    lz4 1.9.3   |  2.101  |   740 MB/s  |  4500 MB/s |\n     *\n     */\n\n    LZ4(1),\n    ZSTD(2),\n    ZLIB(3);\n\n    private final int value;\n\n    CompressionType(int value) {\n        this.value = value;\n    }\n\n    public int getValue() {\n        return value;\n    }\n\n    public static CompressionType of(String name) {\n        switch (name.trim().toUpperCase()) {\n            case \"LZ4\":\n                return CompressionType.LZ4;\n            case \"ZSTD\":\n                return CompressionType.ZSTD;\n            case \"ZLIB\":\n                return CompressionType.ZLIB;\n            default:\n                throw new RuntimeException(\"Unsupported compress type name: \" + name);\n        }\n    }\n\n    public static CompressionType findByValue(int value) {\n        switch (value) {\n            case 1:\n                return LZ4;\n            case 2:\n                return ZSTD;\n            case 0: // To be compatible for older versions without compression type\n            case 3:\n                return ZLIB;\n            default:\n                throw new RuntimeException(\"Unknown compress type value: \" + value);\n        }\n    }\n\n    public int getCompressionFlag() {\n        switch (value) {\n            case 1:\n                return MessageSysFlag.COMPRESSION_LZ4_TYPE;\n            case 2:\n                return MessageSysFlag.COMPRESSION_ZSTD_TYPE;\n            case 3:\n                return MessageSysFlag.COMPRESSION_ZLIB_TYPE;\n            default:\n                throw new RuntimeException(\"Unsupported compress type flag: \" + value);\n        }\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/compression/Compressor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.compression;\n\nimport java.io.IOException;\n\npublic interface Compressor {\n\n    /**\n     * Compress message by different compressor.\n     *\n     * @param src bytes ready to compress\n     * @param level compression level used to balance compression rate and time consumption\n     * @return compressed byte data\n     * @throws IOException\n     */\n    byte[] compress(byte[] src, int level) throws IOException;\n\n    /**\n     * Decompress message by different compressor.\n     *\n     * @param src bytes ready to decompress\n     * @return decompressed byte data\n     * @throws IOException\n     */\n    byte[] decompress(byte[] src) throws IOException;\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/compression/CompressorFactory.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.compression;\n\nimport java.util.EnumMap;\n\npublic class CompressorFactory {\n    private static final EnumMap<CompressionType, Compressor> COMPRESSORS;\n\n    static {\n        COMPRESSORS = new EnumMap<>(CompressionType.class);\n        COMPRESSORS.put(CompressionType.LZ4, new Lz4Compressor());\n        COMPRESSORS.put(CompressionType.ZSTD, new ZstdCompressor());\n        COMPRESSORS.put(CompressionType.ZLIB, new ZlibCompressor());\n    }\n\n    public static Compressor getCompressor(CompressionType type) {\n        return COMPRESSORS.get(type);\n    }\n\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/compression/Lz4Compressor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.compression;\n\nimport java.io.ByteArrayInputStream;\nimport java.io.ByteArrayOutputStream;\nimport java.io.IOException;\nimport net.jpountz.lz4.LZ4FrameInputStream;\nimport net.jpountz.lz4.LZ4FrameOutputStream;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class Lz4Compressor implements Compressor {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);\n\n    @Override\n    public byte[] compress(byte[] src, int level) throws IOException {\n        byte[] result = src;\n        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(src.length);\n        LZ4FrameOutputStream outputStream = new LZ4FrameOutputStream(byteArrayOutputStream);\n        try {\n            outputStream.write(src);\n            outputStream.flush();\n            outputStream.close();\n            result = byteArrayOutputStream.toByteArray();\n        } catch (IOException e) {\n            log.error(\"Failed to compress data by lz4\", e);\n            throw e;\n        } finally {\n            try {\n                byteArrayOutputStream.close();\n            } catch (IOException ignored) {\n            }\n        }\n        return result;\n    }\n\n    @Override\n    public byte[] decompress(byte[] src) throws IOException {\n        byte[] result = src;\n        byte[] uncompressData = new byte[src.length];\n        ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(src);\n        LZ4FrameInputStream lz4InputStream = new LZ4FrameInputStream(byteArrayInputStream);\n        ByteArrayOutputStream resultOutputStream = new ByteArrayOutputStream(src.length);\n\n        try {\n            while (true) {\n                int len = lz4InputStream.read(uncompressData, 0, uncompressData.length);\n                if (len <= 0) {\n                    break;\n                }\n                resultOutputStream.write(uncompressData, 0, len);\n            }\n            resultOutputStream.flush();\n            resultOutputStream.close();\n            result = resultOutputStream.toByteArray();\n        } catch (IOException e) {\n            throw e;\n        } finally {\n            try {\n                lz4InputStream.close();\n                byteArrayInputStream.close();\n            } catch (IOException e) {\n                log.warn(\"Failed to close the lz4 compress stream \", e);\n            }\n        }\n\n        return result;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/compression/ZlibCompressor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.compression;\n\nimport java.io.ByteArrayInputStream;\nimport java.io.ByteArrayOutputStream;\nimport java.io.IOException;\nimport java.util.zip.DeflaterOutputStream;\nimport java.util.zip.InflaterInputStream;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class ZlibCompressor implements Compressor {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);\n\n    @Override\n    public byte[] compress(byte[] src, int level) throws IOException {\n        byte[] result = src;\n        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(src.length);\n        java.util.zip.Deflater defeater = new java.util.zip.Deflater(level);\n        DeflaterOutputStream deflaterOutputStream = new DeflaterOutputStream(byteArrayOutputStream, defeater);\n        try {\n            deflaterOutputStream.write(src);\n            deflaterOutputStream.finish();\n            deflaterOutputStream.close();\n            result = byteArrayOutputStream.toByteArray();\n        } catch (IOException e) {\n            throw e;\n        } finally {\n            try {\n                byteArrayOutputStream.close();\n            } catch (IOException ignored) {\n            }\n\n            defeater.end();\n        }\n\n        return result;\n    }\n\n    @Override\n    public byte[] decompress(byte[] src) throws IOException {\n        byte[] result = src;\n        byte[] uncompressData = new byte[src.length];\n        ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(src);\n        InflaterInputStream inflaterInputStream = new InflaterInputStream(byteArrayInputStream);\n        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(src.length);\n\n        try {\n            while (true) {\n                int len = inflaterInputStream.read(uncompressData, 0, uncompressData.length);\n                if (len <= 0) {\n                    break;\n                }\n                byteArrayOutputStream.write(uncompressData, 0, len);\n            }\n            byteArrayOutputStream.flush();\n            result = byteArrayOutputStream.toByteArray();\n        } catch (IOException e) {\n            throw e;\n        } finally {\n            try {\n                byteArrayInputStream.close();\n            } catch (IOException e) {\n                log.error(\"Failed to close the stream\", e);\n            }\n            try {\n                inflaterInputStream.close();\n            } catch (IOException e) {\n                log.error(\"Failed to close the stream\", e);\n            }\n            try {\n                byteArrayOutputStream.close();\n            } catch (IOException e) {\n                log.error(\"Failed to close the stream\", e);\n            }\n        }\n\n        return result;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/compression/ZstdCompressor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.compression;\n\nimport com.github.luben.zstd.ZstdInputStream;\nimport com.github.luben.zstd.ZstdOutputStream;\nimport java.io.ByteArrayInputStream;\nimport java.io.ByteArrayOutputStream;\nimport java.io.IOException;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class ZstdCompressor implements Compressor {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);\n\n    @Override\n    public byte[] compress(byte[] src, int level) throws IOException {\n        byte[] result = src;\n        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(src.length);\n        ZstdOutputStream outputStream = new ZstdOutputStream(byteArrayOutputStream, level);\n        try {\n            outputStream.write(src);\n            outputStream.flush();\n            outputStream.close();\n            result = byteArrayOutputStream.toByteArray();\n        } catch (IOException e) {\n            log.error(\"Failed to compress data by zstd\", e);\n            throw e;\n        } finally {\n            try {\n                byteArrayOutputStream.close();\n            } catch (IOException ignored) {\n            }\n        }\n        return result;\n    }\n\n    @Override\n    public byte[] decompress(byte[] src) throws IOException {\n        byte[] result = src;\n        byte[] uncompressData = new byte[src.length];\n        ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(src);\n        ZstdInputStream zstdInputStream = new ZstdInputStream(byteArrayInputStream);\n        ByteArrayOutputStream resultOutputStream = new ByteArrayOutputStream(src.length);\n\n        try {\n            while (true) {\n                int len = zstdInputStream.read(uncompressData, 0, uncompressData.length);\n                if (len <= 0) {\n                    break;\n                }\n                resultOutputStream.write(uncompressData, 0, len);\n            }\n            resultOutputStream.flush();\n            resultOutputStream.close();\n            result = resultOutputStream.toByteArray();\n        } catch (IOException e) {\n            throw e;\n        } finally {\n            try {\n                zstdInputStream.close();\n                byteArrayInputStream.close();\n            } catch (IOException e) {\n                log.warn(\"Failed to close the zstd compress stream\", e);\n            }\n        }\n\n        return result;\n    }\n}"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.config;\n\nimport com.google.common.collect.Maps;\nimport io.netty.buffer.PooledByteBufAllocator;\nimport java.io.File;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ArrayBlockingQueue;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.Semaphore;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport java.util.function.BiConsumer;\nimport org.apache.commons.lang3.ArrayUtils;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.rocksdb.ColumnFamilyDescriptor;\nimport org.rocksdb.ColumnFamilyHandle;\nimport org.rocksdb.ColumnFamilyOptions;\nimport org.rocksdb.CompactRangeOptions;\nimport org.rocksdb.CompactionOptions;\nimport org.rocksdb.CompressionType;\nimport org.rocksdb.DBOptions;\nimport org.rocksdb.Env;\nimport org.rocksdb.FlushOptions;\nimport org.rocksdb.LiveFileMetaData;\nimport org.rocksdb.Priority;\nimport org.rocksdb.ReadOptions;\nimport org.rocksdb.RocksDB;\nimport org.rocksdb.RocksDBException;\nimport org.rocksdb.RocksIterator;\nimport org.rocksdb.Slice;\nimport org.rocksdb.Statistics;\nimport org.rocksdb.Status;\nimport org.rocksdb.WriteBatch;\nimport org.rocksdb.WriteOptions;\n\npublic abstract class AbstractRocksDBStorage {\n    protected static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME);\n\n    /**\n     * Direct Jemalloc allocator\n     */\n    public static final PooledByteBufAllocator POOLED_ALLOCATOR = new PooledByteBufAllocator(true);\n\n    public static final byte CTRL_0 = '\\u0000';\n    public static final byte CTRL_1 = '\\u0001';\n    public static final byte CTRL_2 = '\\u0002';\n\n    private static final String SPACE = \" | \";\n\n    protected final String dbPath;\n    protected boolean readOnly;\n    protected RocksDB db;\n    protected DBOptions options;\n\n    protected WriteOptions writeOptions;\n    protected WriteOptions ableWalWriteOptions;\n\n    protected ReadOptions readOptions;\n    protected ReadOptions totalOrderReadOptions;\n\n    protected CompactionOptions compactionOptions;\n    protected CompactRangeOptions compactRangeOptions;\n\n    protected FlushOptions flushOptions;\n\n    protected ColumnFamilyHandle defaultCFHandle;\n    protected final List<ColumnFamilyOptions> cfOptions = new ArrayList<>();\n    protected final List<ColumnFamilyHandle> cfHandles = new ArrayList<>();\n\n    protected volatile boolean loaded;\n    protected CompressionType compressionType = CompressionType.LZ4_COMPRESSION;\n    private volatile boolean closed;\n\n    private final Semaphore reloadPermit = new Semaphore(1);\n    private final ScheduledExecutorService reloadScheduler = ThreadUtils.newScheduledThreadPool(1, new ThreadFactoryImpl(\"RocksDBStorageReloadService_\"));\n    private final ThreadPoolExecutor manualCompactionThread = (ThreadPoolExecutor) ThreadUtils.newThreadPoolExecutor(\n        1, 1, 1000 * 60, TimeUnit.MILLISECONDS,\n        new ArrayBlockingQueue<>(1),\n        new ThreadFactoryImpl(\"RocksDBManualCompactionService_\"),\n        new ThreadPoolExecutor.DiscardOldestPolicy());\n\n    static {\n        RocksDB.loadLibrary();\n    }\n\n    public AbstractRocksDBStorage(String dbPath) {\n        this.dbPath = dbPath;\n    }\n\n    protected void initOptions() {\n        initWriteOptions();\n        initAbleWalWriteOptions();\n        initReadOptions();\n        initTotalOrderReadOptions();\n        initCompactRangeOptions();\n        initCompactionOptions();\n        initFlushOptions();\n    }\n\n    /**\n     * Write options for <a href=\"https://github.com/facebook/rocksdb/wiki/Atomic-flush\">Atomic Flush</a>\n     */\n    protected void initWriteOptions() {\n        this.writeOptions = new WriteOptions();\n        this.writeOptions.setSync(false);\n        this.writeOptions.setDisableWAL(true);\n        // https://github.com/facebook/rocksdb/wiki/Write-Stalls\n        this.writeOptions.setNoSlowdown(false);\n    }\n\n    protected void initAbleWalWriteOptions() {\n        this.ableWalWriteOptions = new WriteOptions();\n        this.ableWalWriteOptions.setSync(false);\n        this.ableWalWriteOptions.setDisableWAL(false);\n        // https://github.com/facebook/rocksdb/wiki/Write-Stalls\n        this.ableWalWriteOptions.setNoSlowdown(false);\n    }\n\n    protected void initReadOptions() {\n        this.readOptions = new ReadOptions();\n        this.readOptions.setPrefixSameAsStart(true);\n        this.readOptions.setTotalOrderSeek(false);\n        this.readOptions.setTailing(false);\n    }\n\n    protected void initTotalOrderReadOptions() {\n        this.totalOrderReadOptions = new ReadOptions();\n        this.totalOrderReadOptions.setPrefixSameAsStart(false);\n        this.totalOrderReadOptions.setTotalOrderSeek(true);\n        this.totalOrderReadOptions.setTailing(false);\n    }\n\n    protected void initCompactRangeOptions() {\n        this.compactRangeOptions = new CompactRangeOptions();\n        this.compactRangeOptions.setBottommostLevelCompaction(CompactRangeOptions.BottommostLevelCompaction.kForce);\n        this.compactRangeOptions.setAllowWriteStall(true);\n        this.compactRangeOptions.setExclusiveManualCompaction(false);\n        this.compactRangeOptions.setChangeLevel(true);\n        this.compactRangeOptions.setTargetLevel(-1);\n        this.compactRangeOptions.setMaxSubcompactions(4);\n    }\n\n    protected void initCompactionOptions() {\n        this.compactionOptions = new CompactionOptions();\n        this.compactionOptions.setCompression(compressionType);\n        this.compactionOptions.setMaxSubcompactions(4);\n        this.compactionOptions.setOutputFileSizeLimit(4 * 1024 * 1024 * 1024L);\n    }\n\n    protected void initFlushOptions() {\n        this.flushOptions = new FlushOptions();\n    }\n\n    public boolean hold() {\n        if (!this.loaded || this.db == null || this.closed) {\n            LOGGER.error(\"hold rocksdb Failed. {}\", this.dbPath);\n            return false;\n        } else {\n            return true;\n        }\n    }\n\n    public void release() {\n    }\n\n    protected void put(ColumnFamilyHandle cfHandle, WriteOptions writeOptions,\n        final byte[] keyBytes, final int keyLen,\n        final byte[] valueBytes, final int valueLen) throws RocksDBException {\n        if (!hold()) {\n            throw new IllegalStateException(\"rocksDB:\" + this + \" is not ready\");\n        }\n        try {\n            this.db.put(cfHandle, writeOptions, keyBytes, 0, keyLen, valueBytes, 0, valueLen);\n        } catch (RocksDBException e) {\n            scheduleReloadRocksdb(e);\n            LOGGER.error(\"put Failed. {}, {}\", this.dbPath, getStatusError(e));\n            throw e;\n        } finally {\n            release();\n        }\n    }\n\n    protected void put(ColumnFamilyHandle cfHandle, WriteOptions writeOptions,\n        final ByteBuffer keyBB, final ByteBuffer valueBB) throws RocksDBException {\n        if (!hold()) {\n            throw new IllegalStateException(\"rocksDB:\" + this + \" is not ready\");\n        }\n        try {\n            this.db.put(cfHandle, writeOptions, keyBB, valueBB);\n        } catch (RocksDBException e) {\n            scheduleReloadRocksdb(e);\n            LOGGER.error(\"put Failed. {}, {}\", this.dbPath, getStatusError(e));\n            throw e;\n        } finally {\n            release();\n        }\n    }\n\n    protected void batchPut(WriteOptions writeOptions, final WriteBatch batch) throws RocksDBException {\n        try {\n            this.db.write(writeOptions, batch);\n        } catch (RocksDBException e) {\n            scheduleReloadRocksdb(e);\n            LOGGER.error(\"batchPut Failed. {}, {}\", this.dbPath, getStatusError(e));\n            throw e;\n        } finally {\n            batch.clear();\n        }\n    }\n\n    protected byte[] get(ColumnFamilyHandle cfHandle, ReadOptions readOptions, byte[] keyBytes) throws RocksDBException {\n        if (!hold()) {\n            throw new IllegalStateException(\"rocksDB:\" + this + \" is not ready\");\n        }\n        try {\n            return this.db.get(cfHandle, readOptions, keyBytes);\n        } catch (RocksDBException e) {\n            LOGGER.error(\"get Failed. {}, {}\", this.dbPath, getStatusError(e));\n            throw e;\n        } finally {\n            release();\n        }\n    }\n\n    protected int get(ColumnFamilyHandle cfHandle, ReadOptions readOptions, final ByteBuffer keyBB,\n        final ByteBuffer valueBB) throws RocksDBException {\n        if (!hold()) {\n            throw new IllegalStateException(\"rocksDB:\" + this + \" is not ready\");\n        }\n        try {\n            return this.db.get(cfHandle, readOptions, keyBB, valueBB);\n        } catch (RocksDBException e) {\n            LOGGER.error(\"get Failed. {}, {}\", this.dbPath, getStatusError(e));\n            throw e;\n        } finally {\n            release();\n        }\n    }\n\n    protected List<byte[]> multiGet(final ReadOptions readOptions,\n        final List<ColumnFamilyHandle> columnFamilyHandleList,\n        final List<byte[]> keys) throws RocksDBException {\n        if (!hold()) {\n            throw new IllegalStateException(\"rocksDB:\" + this + \" is not ready\");\n        }\n        try {\n            return this.db.multiGetAsList(readOptions, columnFamilyHandleList, keys);\n        } catch (RocksDBException e) {\n            LOGGER.error(\"multiGet Failed. {}, {}\", this.dbPath, getStatusError(e));\n            throw e;\n        } finally {\n            release();\n        }\n    }\n\n    protected void delete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions,\n        byte[] keyBytes) throws RocksDBException {\n        if (!hold()) {\n            throw new IllegalStateException(\"rocksDB:\" + this + \" is not ready\");\n        }\n        try {\n            this.db.delete(cfHandle, writeOptions, keyBytes);\n        } catch (RocksDBException e) {\n            LOGGER.error(\"delete Failed. {}, {}\", this.dbPath, getStatusError(e));\n            throw e;\n        } finally {\n            release();\n        }\n    }\n\n    protected void delete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, ByteBuffer keyBB)\n        throws RocksDBException {\n        if (!hold()) {\n            throw new IllegalStateException(\"rocksDB:\" + this + \" is not ready\");\n        }\n        try {\n            this.db.delete(cfHandle, writeOptions, keyBB);\n        } catch (RocksDBException e) {\n            LOGGER.error(\"delete Failed. {}, {}\", this.dbPath, getStatusError(e));\n            throw e;\n        } finally {\n            release();\n        }\n    }\n\n    protected void rangeDelete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, final byte[] startKey,\n        final byte[] endKey) throws RocksDBException {\n        if (!hold()) {\n            throw new IllegalStateException(\"rocksDB:\" + this + \" is not ready\");\n        }\n        try {\n            this.db.deleteRange(cfHandle, writeOptions, startKey, endKey);\n        } catch (RocksDBException e) {\n            scheduleReloadRocksdb(e);\n            LOGGER.error(\"rangeDelete Failed. {}, {}\", this.dbPath, getStatusError(e));\n            throw e;\n        } finally {\n            release();\n        }\n    }\n\n    public void iterate(ColumnFamilyHandle columnFamilyHandle, final byte[] prefix, BiConsumer<byte[], byte[]> callback)\n        throws RocksDBException {\n\n        if (ArrayUtils.isEmpty(prefix)) {\n            throw new RocksDBException(\"Prefix is not allowed to be null\");\n        }\n\n        iterate(columnFamilyHandle, prefix, null, null, callback);\n    }\n\n    public void iterate(ColumnFamilyHandle columnFamilyHandle, byte[] prefix,\n        final byte[] start, final byte[] end, BiConsumer<byte[], byte[]> callback) throws RocksDBException {\n\n        if (ArrayUtils.isEmpty(prefix) && ArrayUtils.isEmpty(start)) {\n            throw new RocksDBException(\n                \"To determine lower boundary, prefix and start may not be null at the same time.\");\n        }\n\n        if (ArrayUtils.isEmpty(prefix) && ArrayUtils.isEmpty(end)) {\n            throw new RocksDBException(\n                \"To determine upper boundary, prefix and end may not be null at the same time.\");\n        }\n\n        if (columnFamilyHandle == null) {\n            return;\n        }\n\n        ReadOptions readOptions = null;\n        Slice startSlice = null;\n        Slice endSlice = null;\n        Slice prefixSlice = null;\n        RocksIterator iterator = null;\n        try {\n            readOptions = new ReadOptions();\n            readOptions.setTotalOrderSeek(true);\n            readOptions.setReadaheadSize(4L * 1024 * 1024);\n            boolean hasStart = !ArrayUtils.isEmpty(start);\n            boolean hasPrefix = !ArrayUtils.isEmpty(prefix);\n\n            if (hasStart) {\n                startSlice = new Slice(start);\n                readOptions.setIterateLowerBound(startSlice);\n            }\n\n            if (!ArrayUtils.isEmpty(end)) {\n                endSlice = new Slice(end);\n                readOptions.setIterateUpperBound(endSlice);\n            }\n\n            if (!hasStart && hasPrefix) {\n                prefixSlice = new Slice(prefix);\n                readOptions.setIterateLowerBound(prefixSlice);\n            }\n\n            iterator = db.newIterator(columnFamilyHandle, readOptions);\n            if (hasStart) {\n                iterator.seek(start);\n            } else if (hasPrefix) {\n                iterator.seek(prefix);\n            }\n\n            while (iterator.isValid()) {\n                byte[] key = iterator.key();\n                if (hasPrefix && !checkPrefix(key, prefix)) {\n                    break;\n                }\n                callback.accept(iterator.key(), iterator.value());\n                iterator.next();\n            }\n        } finally {\n            if (startSlice != null) {\n                startSlice.close();\n            }\n            if (endSlice != null) {\n                endSlice.close();\n            }\n            if (prefixSlice != null) {\n                prefixSlice.close();\n            }\n            if (readOptions != null) {\n                readOptions.close();\n            }\n            if (iterator != null) {\n                iterator.close();\n            }\n        }\n    }\n\n    private boolean checkPrefix(byte[] key, byte[] upperBound) {\n        if (key.length < upperBound.length) {\n            return false;\n        }\n        for (int i = 0; i < upperBound.length; i++) {\n            if (key[i] > upperBound[i]) {\n                return false;\n            }\n        }\n        return true;\n    }\n\n    protected void manualCompactionDefaultCfRange(CompactRangeOptions compactRangeOptions) {\n        if (!hold()) {\n            return;\n        }\n        long s1 = System.currentTimeMillis();\n        boolean result = true;\n        try {\n            LOGGER.info(\"manualCompaction Start. {}\", this.dbPath);\n            this.db.compactRange(this.defaultCFHandle, null, null, compactRangeOptions);\n        } catch (RocksDBException e) {\n            result = false;\n            scheduleReloadRocksdb(e);\n            LOGGER.error(\"manualCompaction Failed. {}, {}\", this.dbPath, getStatusError(e));\n        } finally {\n            release();\n            LOGGER.info(\"manualCompaction End. {}, rt: {}(ms), result: {}\", this.dbPath, System.currentTimeMillis() - s1, result);\n        }\n    }\n\n    protected void manualCompaction(long minPhyOffset, final CompactRangeOptions compactRangeOptions) {\n        this.manualCompactionThread.submit(new Runnable() {\n            @Override\n            public void run() {\n                manualCompactionDefaultCfRange(compactRangeOptions);\n            }\n        });\n    }\n\n    protected void open(final List<ColumnFamilyDescriptor> cfDescriptors) throws RocksDBException {\n        this.cfHandles.clear();\n        if (this.readOnly) {\n            this.db = RocksDB.openReadOnly(this.options, this.dbPath, cfDescriptors, cfHandles);\n        } else {\n            this.db = RocksDB.open(this.options, this.dbPath, cfDescriptors, cfHandles);\n        }\n        assert cfDescriptors.size() == cfHandles.size();\n\n        if (this.db == null) {\n            throw new RocksDBException(\"open rocksdb null\");\n        }\n        try (Env env = this.db.getEnv()) {\n            env.setBackgroundThreads(8, Priority.LOW);\n        }\n    }\n\n    protected abstract boolean postLoad();\n\n    public synchronized boolean start() {\n        if (this.loaded) {\n            return true;\n        }\n        if (postLoad()) {\n            this.loaded = true;\n            LOGGER.info(\"RocksDB [{}] starts OK\", this.dbPath);\n            this.closed = false;\n            return true;\n        } else {\n            return false;\n        }\n    }\n\n    /**\n     * Close column family handles except the default column family\n     */\n    protected abstract void preShutdown();\n\n    public boolean isLoaded() {\n        return loaded;\n    }\n\n    public synchronized boolean shutdown() {\n        try {\n            if (!this.loaded) {\n                LOGGER.info(\"RocksDBStorage is not loaded, shutdown OK. dbPath={}, readOnly={}\", this.dbPath, this.readOnly);\n                return true;\n            }\n\n            manualCompactionThread.shutdownNow();\n\n            manualCompactionThread.awaitTermination(30, TimeUnit.SECONDS);\n\n            final FlushOptions flushOptions = new FlushOptions();\n            flushOptions.setWaitForFlush(true);\n            try {\n                flush(flushOptions);\n            } catch (Throwable e) {\n                LOGGER.error(\"flush rocksdb wal failed when shutdown\", e);\n            } finally {\n                flushOptions.close();\n            }\n            this.db.cancelAllBackgroundWork(true);\n            this.db.pauseBackgroundWork();\n            //The close order matters.\n            //1. close column family handles\n            preShutdown();\n\n            if (this.defaultCFHandle.isOwningHandle()) {\n                this.defaultCFHandle.close();\n            }\n\n            //2. close column family options.\n            for (final ColumnFamilyOptions opt : this.cfOptions) {\n                opt.close();\n            }\n            //3. close options\n            if (this.writeOptions != null) {\n                this.writeOptions.close();\n            }\n            if (this.ableWalWriteOptions != null) {\n                this.ableWalWriteOptions.close();\n            }\n            if (this.readOptions != null) {\n                this.readOptions.close();\n            }\n            if (this.totalOrderReadOptions != null) {\n                this.totalOrderReadOptions.close();\n            }\n            if (this.flushOptions != null) {\n                this.flushOptions.close();\n            }\n            //4. close db.\n            if (db != null && !this.readOnly) {\n                try {\n                    this.db.syncWal();\n                } catch (Throwable e) {\n                    LOGGER.error(\"rocksdb sync wal failed when shutdown\", e);\n                } finally {\n                    flushOptions.close();\n                }\n\n            }\n            if (db != null) {\n                try {\n                    this.db.closeE();\n                } catch (Throwable e) {\n                    LOGGER.error(\"rocksdb db closeE failed when shutdown\", e);\n                }\n\n            }\n            // Close DBOptions after RocksDB instance is closed.\n            if (this.options != null) {\n                this.options.close();\n            }\n            //5. help gc.\n            this.cfOptions.clear();\n            this.db = null;\n            this.readOptions = null;\n            this.totalOrderReadOptions = null;\n            this.flushOptions = null;\n            this.writeOptions = null;\n            this.ableWalWriteOptions = null;\n            this.options = null;\n\n            this.loaded = false;\n            LOGGER.info(\"RocksDB shutdown OK. {}\", this.dbPath);\n        } catch (Exception e) {\n            LOGGER.error(\"RocksDB shutdown failed. {}\", this.dbPath, e);\n            return false;\n        }\n        return true;\n    }\n\n    public void flush(final FlushOptions flushOptions) throws RocksDBException {\n        flush(flushOptions, this.cfHandles);\n    }\n\n    public void flush(final FlushOptions flushOptions, List<ColumnFamilyHandle> columnFamilyHandles) throws RocksDBException {\n        if (!this.loaded || this.readOnly || closed) {\n            return;\n        }\n\n        try {\n            if (db != null) {\n                // For atomic-flush, we have to explicitly specify column family handles\n                // See https://github.com/rust-rocksdb/rust-rocksdb/pull/793\n                // and https://github.com/facebook/rocksdb/blob/8ad4c7efc48d301f5e85467105d7019a49984dc8/include/rocksdb/db.h#L1667\n                this.db.flush(flushOptions, columnFamilyHandles);\n            }\n        } catch (RocksDBException e) {\n            scheduleReloadRocksdb(e);\n            LOGGER.error(\"flush Failed. {}, {}\", this.dbPath, getStatusError(e));\n            throw e;\n        }\n    }\n\n    public void flushWAL() throws RocksDBException {\n        this.db.flushWal(true);\n    }\n\n    public Statistics getStatistics() {\n        return this.options.statistics();\n    }\n\n    public ColumnFamilyHandle getDefaultCFHandle() {\n        return defaultCFHandle;\n    }\n\n    public List<LiveFileMetaData> getCompactionStatus() {\n        if (!hold()) {\n            return null;\n        }\n        try {\n            return this.db.getLiveFilesMetaData();\n        } finally {\n            release();\n        }\n    }\n\n    private void scheduleReloadRocksdb(RocksDBException rocksDBException) {\n        if (rocksDBException == null || rocksDBException.getStatus() == null) {\n            return;\n        }\n        Status status = rocksDBException.getStatus();\n        Status.Code code = status.getCode();\n        // Status.Code.Incomplete == code\n        if (Status.Code.Aborted == code || Status.Code.Corruption == code || Status.Code.Undefined == code) {\n            LOGGER.error(\"scheduleReloadRocksdb. {}, {}\", this.dbPath, getStatusError(rocksDBException));\n            scheduleReloadRocksdb0();\n        }\n    }\n\n    private void scheduleReloadRocksdb0() {\n        if (!this.reloadPermit.tryAcquire()) {\n            return;\n        }\n        this.closed = true;\n        this.reloadScheduler.schedule(new Runnable() {\n            @Override\n            public void run() {\n                boolean result = true;\n                try {\n                    reloadRocksdb();\n                } catch (Exception e) {\n                    result = false;\n                } finally {\n                    reloadPermit.release();\n                }\n                // try to reload rocksdb next time\n                if (!result) {\n                    LOGGER.info(\"reload rocksdb Retry. {}\", dbPath);\n                    scheduleReloadRocksdb0();\n                }\n            }\n        }, 10, TimeUnit.SECONDS);\n    }\n\n    private void reloadRocksdb() throws Exception {\n        LOGGER.info(\"reload rocksdb Start. {}\", this.dbPath);\n        if (!shutdown() || !start()) {\n            LOGGER.error(\"reload rocksdb Failed. {}\", dbPath);\n            throw new Exception(\"reload rocksdb Error\");\n        }\n        LOGGER.info(\"reload rocksdb OK. {}\", this.dbPath);\n    }\n\n    private String getStatusError(RocksDBException e) {\n        if (e == null || e.getStatus() == null) {\n            return \"null\";\n        }\n        Status status = e.getStatus();\n        StringBuilder sb = new StringBuilder(64);\n        sb.append(\"code: \");\n        if (status.getCode() != null) {\n            sb.append(status.getCode().name());\n        } else {\n            sb.append(\"null\");\n        }\n        sb.append(\", \").append(\"subCode: \");\n        if (status.getSubCode() != null) {\n            sb.append(status.getSubCode().name());\n        } else {\n            sb.append(\"null\");\n        }\n        sb.append(\", \").append(\"state: \").append(status.getState());\n        return sb.toString();\n    }\n\n    public void statRocksdb(Logger logger) {\n        try {\n            // Log Memory Usage\n            String blockCacheMemUsage = this.db.getProperty(\"rocksdb.block-cache-usage\");\n            String indexesAndFilterBlockMemUsage = this.db.getProperty(\"rocksdb.estimate-table-readers-mem\");\n            String memTableMemUsage = this.db.getProperty(\"rocksdb.cur-size-all-mem-tables\");\n            String blocksPinnedByIteratorMemUsage = this.db.getProperty(\"rocksdb.block-cache-pinned-usage\");\n            logger.info(\"RocksDB Memory Usage: BlockCache: {}, IndexesAndFilterBlock: {}, MemTable: {}, BlocksPinnedByIterator: {}\",\n                blockCacheMemUsage, indexesAndFilterBlockMemUsage, memTableMemUsage, blocksPinnedByIteratorMemUsage);\n\n            // Log file metadata by level\n            List<LiveFileMetaData> liveFileMetaDataList = this.getCompactionStatus();\n            if (liveFileMetaDataList == null || liveFileMetaDataList.isEmpty()) {\n                return;\n            }\n            Map<Integer, StringBuilder> map = Maps.newHashMap();\n            for (LiveFileMetaData metaData : liveFileMetaDataList) {\n                StringBuilder sb = map.computeIfAbsent(metaData.level(), k -> new StringBuilder(256));\n                sb.append(new String(metaData.columnFamilyName(), StandardCharsets.UTF_8)).append(SPACE).\n                    append(metaData.fileName()).append(SPACE).\n                    append(\"file-size: \").append(metaData.size()).append(SPACE).\n                    append(\"number-of-entries: \").append(metaData.numEntries()).append(SPACE).\n                    append(\"file-read-times: \").append(metaData.numReadsSampled()).append(SPACE).\n                    append(\"deletions: \").append(metaData.numDeletions()).append(SPACE).\n                    append(\"being-compacted: \").append(metaData.beingCompacted()).append(\"\\n\");\n            }\n            map.forEach((key, value) -> logger.info(\"level: {}\\n{}\", key, value.toString()));\n        } catch (Exception ignored) {\n        }\n    }\n\n    public void destroy() {\n        recursiveDelete(new File(dbPath));\n    }\n\n    void recursiveDelete(File file) {\n        if (file.isFile()) {\n            if (file.delete()) {\n                LOGGER.info(\"Delete rocksdb file={}\", file.getAbsolutePath());\n            }\n        } else {\n            File[] files = file.listFiles();\n            for (File f : files) {\n                recursiveDelete(f);\n            }\n            file.delete();\n        }\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/config/ConfigHelper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.config;\n\nimport com.google.common.base.Strings;\nimport java.io.File;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.rocksdb.BlockBasedTableConfig;\nimport org.rocksdb.BloomFilter;\nimport org.rocksdb.ColumnFamilyOptions;\nimport org.rocksdb.CompactionStyle;\nimport org.rocksdb.CompressionType;\nimport org.rocksdb.DBOptions;\nimport org.rocksdb.DataBlockIndexType;\nimport org.rocksdb.IndexType;\nimport org.rocksdb.InfoLogLevel;\nimport org.rocksdb.LRUCache;\nimport org.rocksdb.RateLimiter;\nimport org.rocksdb.SkipListMemTableConfig;\nimport org.rocksdb.Statistics;\nimport org.rocksdb.StatsLevel;\nimport org.rocksdb.StringAppendOperator;\nimport org.rocksdb.WALRecoveryMode;\nimport org.rocksdb.util.SizeUnit;\n\npublic class ConfigHelper {\n    public static ColumnFamilyOptions createConfigColumnFamilyOptions() {\n        BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig().\n            setFormatVersion(5).\n            setIndexType(IndexType.kBinarySearch).\n            setDataBlockIndexType(DataBlockIndexType.kDataBlockBinarySearch).\n            setBlockSize(32 * SizeUnit.KB).\n            setFilterPolicy(new BloomFilter(16, false)).\n            // Indicating if we'd put index/filter blocks to the block cache.\n            setCacheIndexAndFilterBlocks(true).\n            setCacheIndexAndFilterBlocksWithHighPriority(true).\n            setPinL0FilterAndIndexBlocksInCache(false).\n            setPinTopLevelIndexAndFilter(true).\n            setBlockCache(new LRUCache(4 * SizeUnit.MB, 8, false)).\n            setWholeKeyFiltering(true);\n\n        ColumnFamilyOptions options = new ColumnFamilyOptions();\n        return options.setMaxWriteBufferNumber(4).\n            setWriteBufferSize(64 * SizeUnit.MB).\n            setMinWriteBufferNumberToMerge(1).\n            setTableFormatConfig(blockBasedTableConfig).\n            setMemTableConfig(new SkipListMemTableConfig()).\n            setCompressionType(CompressionType.NO_COMPRESSION).\n            setNumLevels(7).\n            setCompactionStyle(CompactionStyle.LEVEL).\n            setLevel0FileNumCompactionTrigger(4).\n            setLevel0SlowdownWritesTrigger(8).\n            setLevel0StopWritesTrigger(12).\n            // The target file size for compaction.\n            setTargetFileSizeBase(64 * SizeUnit.MB).\n            setTargetFileSizeMultiplier(2).\n            // The upper-bound of the total size of L1 files in bytes\n            setMaxBytesForLevelBase(256 * SizeUnit.MB).\n            setMaxBytesForLevelMultiplier(2).\n            setMergeOperator(new StringAppendOperator()).\n            setInplaceUpdateSupport(true);\n    }\n\n    public static DBOptions createConfigDBOptions() {\n        // Tune based on https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide\n        // and http://gitlab.alibaba-inc.com/aloha/aloha/blob/branch_2_5_0/jstorm-core/src/main/java/com/alibaba/jstorm/cache/rocksdb/RocksDbOptionsFactory.java\n        DBOptions options = new DBOptions();\n        Statistics statistics = new Statistics();\n        statistics.setStatsLevel(StatsLevel.EXCEPT_DETAILED_TIMERS);\n        return options.\n            setDbLogDir(getDBLogDir()).\n            setInfoLogLevel(InfoLogLevel.INFO_LEVEL).\n            setWalRecoveryMode(WALRecoveryMode.SkipAnyCorruptedRecords).\n            /*\n             * We use manual flush to achieve desired balance between reliability and performance:\n             * for metadata that matters, including {topic, subscription}-config changes, each write incurs a\n             * flush-and-sync to ensure reliability; for {commit, pull}-offset advancements, group-flush are offered for\n             * every N(configurable, 1024 by default) writes or aging of writes, similar to OS page-cache flush\n             * mechanism.\n             */\n            setManualWalFlush(true).\n            // This option takes effect only when we have multiple column families\n            // https://github.com/facebook/rocksdb/issues/4180\n            // setMaxTotalWalSize(1024 * SizeUnit.MB).\n            setDbWriteBufferSize(128 * SizeUnit.MB).\n            setBytesPerSync(SizeUnit.MB).\n            setWalBytesPerSync(SizeUnit.MB).\n            setCreateIfMissing(true).\n            setCreateMissingColumnFamilies(true).\n            setMaxOpenFiles(-1).\n            setMaxLogFileSize(SizeUnit.GB).\n            setKeepLogFileNum(5).\n            setMaxManifestFileSize(SizeUnit.GB).\n            setAllowConcurrentMemtableWrite(false).\n            setStatistics(statistics).\n            setStatsDumpPeriodSec(600).\n            setMaxBackgroundJobs(32).\n            setMaxSubcompactions(4).\n            setParanoidChecks(true).\n            setDelayedWriteRate(16 * SizeUnit.MB).\n            setRateLimiter(new RateLimiter(100 * SizeUnit.MB)).\n            setUseDirectIoForFlushAndCompaction(true).\n            setUseDirectReads(true);\n    }\n\n    public static String getDBLogDir() {\n        String[] rootPaths = new String[] {\n            System.getProperty(\"user.home\"),\n            System.getProperty(\"java.io.tmpdir\"),\n            File.separator + \"data\"\n        };\n        for (String rootPath : rootPaths) {\n            // Refer bazel test encyclopedia: https://bazel.build/reference/test-encyclopedia\n            // Not all directories is available\n            if (Strings.isNullOrEmpty(rootPath)) {\n                continue;\n            }\n            File rootPathFile = new File(rootPath);\n            if (!rootPathFile.exists() || !rootPathFile.canWrite()) {\n                continue;\n            }\n            String logDirectory = rootPath + File.separator + \"logs\" + File.separator + \"rocketmqlogs\";\n            // Create directories recursively.\n            UtilAll.ensureDirOK(logDirectory);\n            return logDirectory;\n        }\n        throw new RuntimeException(\"Failed to get log directory\");\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/config/ConfigManagerVersion.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.config;\n\npublic enum ConfigManagerVersion {\n    V1(\"v1\"),\n    V2(\"v2\"),\n    ;\n    private final String version;\n\n    ConfigManagerVersion(String version) {\n        this.version = version;\n    }\n\n    public String getVersion() {\n        return version;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.config;\n\nimport java.nio.ByteBuffer;\nimport java.nio.charset.Charset;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.function.BiConsumer;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.utils.ConcurrentHashMapUtils;\nimport org.rocksdb.ColumnFamilyDescriptor;\nimport org.rocksdb.ColumnFamilyHandle;\nimport org.rocksdb.ColumnFamilyOptions;\nimport org.rocksdb.CompressionType;\nimport org.rocksdb.Options;\nimport org.rocksdb.RocksDB;\nimport org.rocksdb.RocksDBException;\nimport org.rocksdb.RocksIterator;\nimport org.rocksdb.WriteBatch;\n\npublic class ConfigRocksDBStorage extends AbstractRocksDBStorage {\n    public static final Charset CHARSET = StandardCharsets.UTF_8;\n    public static final ConcurrentMap<String, ConfigRocksDBStorage> STORE_MAP = new ConcurrentHashMap<>();\n\n    private final ConcurrentHashMap<String, ColumnFamilyHandle> columnFamilyNameHandleMap;\n    private ColumnFamilyOptions columnFamilyOptions;\n\n    private ConfigRocksDBStorage(final String dbPath, boolean readOnly, CompressionType compressionType) {\n        super(dbPath);\n        this.readOnly = readOnly;\n        if (compressionType != null) {\n            this.compressionType = compressionType;\n        }\n        this.columnFamilyNameHandleMap = new ConcurrentHashMap<>();\n    }\n\n    public ConfigRocksDBStorage(final String dbPath, boolean readOnly) {\n        this(dbPath, readOnly, null);\n    }\n\n    protected void initOptions() {\n        this.options = ConfigHelper.createConfigDBOptions();\n        this.columnFamilyOptions = ConfigHelper.createConfigColumnFamilyOptions();\n        this.cfOptions.add(columnFamilyOptions);\n        super.initOptions();\n    }\n\n    @Override\n    protected boolean postLoad() {\n        try {\n            UtilAll.ensureDirOK(this.dbPath);\n\n            initOptions();\n\n            List<byte[]> columnFamilyNames = new ArrayList<>(RocksDB.listColumnFamilies(\n                new Options(options, columnFamilyOptions), dbPath));\n            addIfNotExists(columnFamilyNames, RocksDB.DEFAULT_COLUMN_FAMILY);\n\n            List<ColumnFamilyDescriptor> cfDescriptors = new ArrayList<>();\n            for (byte[] columnFamilyName : columnFamilyNames) {\n                cfDescriptors.add(new ColumnFamilyDescriptor(columnFamilyName, columnFamilyOptions));\n            }\n\n            this.open(cfDescriptors);\n            for (int i = 0; i < columnFamilyNames.size(); i++) {\n                columnFamilyNameHandleMap.put(new String(columnFamilyNames.get(i), CHARSET), cfHandles.get(i));\n            }\n            this.defaultCFHandle = columnFamilyNameHandleMap.get(new String(RocksDB.DEFAULT_COLUMN_FAMILY, CHARSET));\n        } catch (final Exception e) {\n            AbstractRocksDBStorage.LOGGER.error(\"postLoad Failed. {}\", this.dbPath, e);\n            return false;\n        }\n        return true;\n    }\n\n    @Override\n    protected void preShutdown() {\n        for (final ColumnFamilyHandle columnFamilyHandle : this.columnFamilyNameHandleMap.values()) {\n            if (columnFamilyHandle.isOwningHandle()) {\n                columnFamilyHandle.close();\n            }\n        }\n    }\n\n    // batch operations\n    public void writeBatchPutOperation(String cf, WriteBatch writeBatch, final byte[] key, final byte[] value) throws RocksDBException {\n        writeBatch.put(getOrCreateColumnFamily(cf), key, value);\n    }\n\n    public void batchPut(final WriteBatch batch) throws RocksDBException {\n        batchPut(this.writeOptions, batch);\n    }\n\n    public void batchPutWithWal(final WriteBatch batch) throws RocksDBException {\n        batchPut(this.ableWalWriteOptions, batch);\n    }\n\n\n    // operations with the specified cf\n    public void put(String cf, final byte[] keyBytes, final int keyLen, final byte[] valueBytes) throws Exception {\n        put(getOrCreateColumnFamily(cf), this.ableWalWriteOptions, keyBytes, keyLen, valueBytes, valueBytes.length);\n    }\n\n    public void put(String cf, final ByteBuffer keyBB, final ByteBuffer valueBB) throws Exception {\n        put(getOrCreateColumnFamily(cf), this.ableWalWriteOptions, keyBB, valueBB);\n    }\n\n    public byte[] get(String cf, final byte[] keyBytes) throws Exception {\n        ColumnFamilyHandle columnFamilyHandle = columnFamilyNameHandleMap.get(cf);\n        if (columnFamilyHandle == null) {\n            return null;\n        }\n        return get(columnFamilyHandle, this.totalOrderReadOptions, keyBytes);\n    }\n\n    public void delete(String cf, final byte[] keyBytes) throws Exception {\n        ColumnFamilyHandle columnFamilyHandle = columnFamilyNameHandleMap.get(cf);\n        if (columnFamilyHandle == null) {\n            return;\n        }\n        delete(columnFamilyHandle, this.ableWalWriteOptions, keyBytes);\n    }\n\n    public void iterate(final String cf, BiConsumer<byte[], byte[]> biConsumer) throws RocksDBException {\n        if (!hold()) {\n            LOGGER.warn(\"RocksDBKvStore[path={}] has been shut down\", dbPath);\n            return;\n        }\n        ColumnFamilyHandle columnFamilyHandle = columnFamilyNameHandleMap.get(cf);\n        if (columnFamilyHandle == null) {\n            return;\n        }\n        try (RocksIterator iterator = this.db.newIterator(columnFamilyHandle)) {\n            for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) {\n                biConsumer.accept(iterator.key(), iterator.value());\n            }\n            iterator.status();\n        }\n    }\n\n    public RocksIterator iterator() {\n        return this.db.newIterator(this.defaultCFHandle, this.totalOrderReadOptions);\n    }\n\n    public ColumnFamilyHandle getOrCreateColumnFamily(String cf) throws RocksDBException {\n        if (!columnFamilyNameHandleMap.containsKey(cf)) {\n            if (readOnly) {\n                String errInfo = String.format(\"RocksDBKvStore[path=%s] is open as read-only\", dbPath);\n                LOGGER.warn(errInfo);\n                throw new RocksDBException(errInfo);\n            }\n            synchronized (this) {\n                if (!columnFamilyNameHandleMap.containsKey(cf)) {\n                    ColumnFamilyDescriptor columnFamilyDescriptor =\n                        new ColumnFamilyDescriptor(cf.getBytes(CHARSET), columnFamilyOptions);\n                    ColumnFamilyHandle columnFamilyHandle = db.createColumnFamily(columnFamilyDescriptor);\n                    columnFamilyNameHandleMap.putIfAbsent(cf, columnFamilyHandle);\n                    cfHandles.add(columnFamilyHandle);\n                }\n            }\n        }\n        return columnFamilyNameHandleMap.get(cf);\n    }\n\n    public void addIfNotExists(List<byte[]> columnFamilyNames, byte[] byteArray) {\n        if (columnFamilyNames.stream().noneMatch(array -> Arrays.equals(array, byteArray))) {\n            columnFamilyNames.add(byteArray);\n        }\n    }\n\n    public static ConfigRocksDBStorage getStore(String path, boolean readOnly, CompressionType compressionType) {\n        return ConcurrentHashMapUtils.computeIfAbsent(STORE_MAP, path,\n            k -> new ConfigRocksDBStorage(path, readOnly, compressionType));\n    }\n\n    public static ConfigRocksDBStorage getStore(String path, boolean readOnly) {\n        return getStore(path, readOnly, null);\n    }\n\n    public static void shutdown(String path) {\n        ConfigRocksDBStorage kvStore = STORE_MAP.remove(path);\n        if (kvStore != null) {\n            kvStore.shutdown();\n        }\n    }\n\n    public static void destroy(String path) {\n        ConfigRocksDBStorage kvStore = STORE_MAP.remove(path);\n        if (kvStore != null) {\n            kvStore.shutdown();\n            kvStore.destroy();\n        }\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/consistenthash/ConsistentHashRouter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.consistenthash;\n\nimport java.nio.charset.StandardCharsets;\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.util.Collection;\nimport java.util.Iterator;\nimport java.util.SortedMap;\nimport java.util.TreeMap;\n\n/**\n * To hash Node objects to a hash ring with a certain amount of virtual node.\n * Method routeNode will return a Node instance which the object key should be allocated to according to consistent hash\n * algorithm\n */\npublic class ConsistentHashRouter<T extends Node> {\n    private final SortedMap<Long, VirtualNode<T>> ring = new TreeMap<>();\n    private final HashFunction hashFunction;\n\n    public ConsistentHashRouter(Collection<T> pNodes, int vNodeCount) {\n        this(pNodes, vNodeCount, new MD5Hash());\n    }\n\n    /**\n     * @param pNodes collections of physical nodes\n     * @param vNodeCount amounts of virtual nodes\n     * @param hashFunction hash Function to hash Node instances\n     */\n    public ConsistentHashRouter(Collection<T> pNodes, int vNodeCount, HashFunction hashFunction) {\n        if (hashFunction == null) {\n            throw new NullPointerException(\"Hash Function is null\");\n        }\n        this.hashFunction = hashFunction;\n        if (pNodes != null) {\n            for (T pNode : pNodes) {\n                addNode(pNode, vNodeCount);\n            }\n        }\n    }\n\n    /**\n     * add physic node to the hash ring with some virtual nodes\n     *\n     * @param pNode physical node needs added to hash ring\n     * @param vNodeCount the number of virtual node of the physical node. Value should be greater than or equals to 0\n     */\n    public void addNode(T pNode, int vNodeCount) {\n        if (vNodeCount < 0)\n            throw new IllegalArgumentException(\"illegal virtual node counts :\" + vNodeCount);\n        int existingReplicas = getExistingReplicas(pNode);\n        for (int i = 0; i < vNodeCount; i++) {\n            VirtualNode<T> vNode = new VirtualNode<>(pNode, i + existingReplicas);\n            ring.put(hashFunction.hash(vNode.getKey()), vNode);\n        }\n    }\n\n    /**\n     * remove the physical node from the hash ring\n     */\n    public void removeNode(T pNode) {\n        Iterator<Long> it = ring.keySet().iterator();\n        while (it.hasNext()) {\n            Long key = it.next();\n            VirtualNode<T> virtualNode = ring.get(key);\n            if (virtualNode.isVirtualNodeOf(pNode)) {\n                it.remove();\n            }\n        }\n    }\n\n    /**\n     * with a specified key, route the nearest Node instance in the current hash ring\n     *\n     * @param objectKey the object key to find a nearest Node\n     */\n    public T routeNode(String objectKey) {\n        if (ring.isEmpty()) {\n            return null;\n        }\n        Long hashVal = hashFunction.hash(objectKey);\n        SortedMap<Long, VirtualNode<T>> tailMap = ring.tailMap(hashVal);\n        Long nodeHashVal = !tailMap.isEmpty() ? tailMap.firstKey() : ring.firstKey();\n        return ring.get(nodeHashVal).getPhysicalNode();\n    }\n\n    public int getExistingReplicas(T pNode) {\n        int replicas = 0;\n        for (VirtualNode<T> vNode : ring.values()) {\n            if (vNode.isVirtualNodeOf(pNode)) {\n                replicas++;\n            }\n        }\n        return replicas;\n    }\n\n    //default hash function\n    private static class MD5Hash implements HashFunction {\n        MessageDigest instance;\n\n        public MD5Hash() {\n            try {\n                instance = MessageDigest.getInstance(\"MD5\");\n            } catch (NoSuchAlgorithmException e) {\n            }\n        }\n\n        @Override\n        public long hash(String key) {\n            instance.reset();\n            instance.update(key.getBytes(StandardCharsets.UTF_8));\n            byte[] digest = instance.digest();\n\n            long h = 0;\n            for (int i = 0; i < 4; i++) {\n                h <<= 8;\n                h |= ((int) digest[i]) & 0xFF;\n            }\n            return h;\n        }\n    }\n\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/consistenthash/HashFunction.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.consistenthash;\n\n/**\n * Hash String to long value\n */\npublic interface HashFunction {\n    long hash(String key);\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/consistenthash/Node.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.consistenthash;\n\n/**\n * Represent a node which should be mapped to a hash ring\n */\npublic interface Node {\n    /**\n     * @return the key which will be used for hash mapping\n     */\n    String getKey();\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/consistenthash/VirtualNode.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.consistenthash;\n\npublic class VirtualNode<T extends Node> implements Node {\n    final T physicalNode;\n    final int replicaIndex;\n\n    public VirtualNode(T physicalNode, int replicaIndex) {\n        this.replicaIndex = replicaIndex;\n        this.physicalNode = physicalNode;\n    }\n\n    @Override\n    public String getKey() {\n        return physicalNode.getKey() + \"-\" + replicaIndex;\n    }\n\n    public boolean isVirtualNodeOf(T pNode) {\n        return physicalNode.getKey().equals(pNode.getKey());\n    }\n\n    public T getPhysicalNode() {\n        return physicalNode;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/constant/CommonConstants.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.constant;\n\npublic class CommonConstants {\n\n    public static final String COLON = \":\";\n\n    public static final String ASTERISK = \"*\";\n\n    public static final String COMMA = \",\";\n\n    public static final String EQUAL = \"=\";\n\n    public static final String SLASH = \"/\";\n\n    public static final String SPACE = \" \";\n\n    public static final String HYPHEN = \"-\";\n\n    public static final String POUND = \"#\";\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/constant/ConsumeInitMode.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.constant;\n\npublic class ConsumeInitMode {\n    public static final int MIN = 0;\n    public static final int MAX = 1;\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/constant/DBMsgConstants.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.constant;\n\npublic class DBMsgConstants {\n    public static final int MAX_BODY_SIZE = 64 * 1024 * 1024; //64KB\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/constant/FIleReadaheadMode.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.constant;\n\npublic class FIleReadaheadMode {\n    public static final String READ_AHEAD_MODE = \"READ_AHEAD_MODE\";\n}"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/constant/GrpcConstants.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.constant;\n\nimport io.grpc.Context;\nimport io.grpc.Metadata;\n\npublic class GrpcConstants {\n    public static final Context.Key<Metadata> METADATA = Context.key(\"rpc-metadata\");\n\n    /**\n     * Remote address key in attributes of call\n     */\n    public static final Metadata.Key<String> REMOTE_ADDRESS\n        = Metadata.Key.of(\"rpc-remote-address\", Metadata.ASCII_STRING_MARSHALLER);\n\n    /**\n     * Local address key in attributes of call\n     */\n    public static final Metadata.Key<String> LOCAL_ADDRESS\n        = Metadata.Key.of(\"rpc-local-address\", Metadata.ASCII_STRING_MARSHALLER);\n\n    public static final Metadata.Key<String> AUTHORIZATION\n        = Metadata.Key.of(\"authorization\", Metadata.ASCII_STRING_MARSHALLER);\n\n    public static final Metadata.Key<String> NAMESPACE_ID\n        = Metadata.Key.of(\"x-mq-namespace\", Metadata.ASCII_STRING_MARSHALLER);\n\n    public static final Metadata.Key<String> DATE_TIME\n        = Metadata.Key.of(\"x-mq-date-time\", Metadata.ASCII_STRING_MARSHALLER);\n\n    public static final Metadata.Key<String> REQUEST_ID\n        = Metadata.Key.of(\"x-mq-request-id\", Metadata.ASCII_STRING_MARSHALLER);\n\n    public static final Metadata.Key<String> LANGUAGE\n        = Metadata.Key.of(\"x-mq-language\", Metadata.ASCII_STRING_MARSHALLER);\n\n    public static final Metadata.Key<String> CLIENT_VERSION\n        = Metadata.Key.of(\"x-mq-client-version\", Metadata.ASCII_STRING_MARSHALLER);\n\n    public static final Metadata.Key<String> PROTOCOL_VERSION\n        = Metadata.Key.of(\"x-mq-protocol\", Metadata.ASCII_STRING_MARSHALLER);\n\n    public static final Metadata.Key<String> RPC_NAME\n        = Metadata.Key.of(\"x-mq-rpc-name\", Metadata.ASCII_STRING_MARSHALLER);\n\n    public static final Metadata.Key<String> SIMPLE_RPC_NAME\n            = Metadata.Key.of(\"x-mq-simple-rpc-name\", Metadata.ASCII_STRING_MARSHALLER);\n\n    public static final Metadata.Key<String> SESSION_TOKEN\n        = Metadata.Key.of(\"x-mq-session-token\", Metadata.ASCII_STRING_MARSHALLER);\n\n    public static final Metadata.Key<String> CLIENT_ID\n        = Metadata.Key.of(\"x-mq-client-id\", Metadata.ASCII_STRING_MARSHALLER);\n\n    public static final Metadata.Key<String> AUTHORIZATION_AK\n        = Metadata.Key.of(\"x-mq-authorization-ak\", Metadata.ASCII_STRING_MARSHALLER);\n\n    public static final Metadata.Key<String> CHANNEL_ID\n        = Metadata.Key.of(\"x-mq-channel-id\", Metadata.ASCII_STRING_MARSHALLER);\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/constant/HAProxyConstants.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.constant;\n\npublic class HAProxyConstants {\n\n    public static final String CHANNEL_ID = \"channel_id\";\n    public static final String PROXY_PROTOCOL_PREFIX = \"proxy_protocol_\";\n    public static final String PROXY_PROTOCOL_ADDR = PROXY_PROTOCOL_PREFIX + \"addr\";\n    public static final String PROXY_PROTOCOL_PORT = PROXY_PROTOCOL_PREFIX + \"port\";\n    public static final String PROXY_PROTOCOL_SERVER_ADDR = PROXY_PROTOCOL_PREFIX + \"server_addr\";\n    public static final String PROXY_PROTOCOL_SERVER_PORT = PROXY_PROTOCOL_PREFIX + \"server_port\";\n    public static final String PROXY_PROTOCOL_TLV_PREFIX = PROXY_PROTOCOL_PREFIX + \"tlv_0x\";\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java",
    "content": "/*\r\n * Licensed to the Apache Software Foundation (ASF) under one or more\r\n * contributor license agreements.  See the NOTICE file distributed with\r\n * this work for additional information regarding copyright ownership.\r\n * The ASF licenses this file to You under the Apache License, Version 2.0\r\n * (the \"License\"); you may not use this file except in compliance with\r\n * the License.  You may obtain a copy of the License at\r\n *\r\n *     http://www.apache.org/licenses/LICENSE-2.0\r\n *\r\n * Unless required by applicable law or agreed to in writing, software\r\n * distributed under the License is distributed on an \"AS IS\" BASIS,\r\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r\n * See the License for the specific language governing permissions and\r\n * limitations under the License.\r\n */\r\npackage org.apache.rocketmq.common.constant;\r\n\r\npublic class LoggerName {\r\n    public static final String FILTERSRV_LOGGER_NAME = \"RocketmqFiltersrv\";\r\n    public static final String NAMESRV_LOGGER_NAME = \"RocketmqNamesrv\";\r\n    public static final String NAMESRV_CONSOLE_LOGGER_NAME = \"RocketmqNamesrvConsole\";\r\n    public static final String CONTROLLER_LOGGER_NAME = \"RocketmqController\";\r\n    public static final String CONTROLLER_CONSOLE_NAME = \"RocketmqControllerConsole\";\r\n    public static final String NAMESRV_WATER_MARK_LOGGER_NAME = \"RocketmqNamesrvWaterMark\";\r\n    public static final String BROKER_LOGGER_NAME = \"RocketmqBroker\";\r\n    public static final String BROKER_CONSOLE_NAME = \"RocketmqConsole\";\r\n    public static final String CLIENT_LOGGER_NAME = \"RocketmqClient\";\r\n    public static final String ROCKETMQ_TRAFFIC_NAME = \"RocketmqTraffic\";\r\n    public static final String ROCKETMQ_REMOTING_NAME = \"RocketmqRemoting\";\r\n    public static final String TOOLS_LOGGER_NAME = \"RocketmqTools\";\r\n    public static final String COMMON_LOGGER_NAME = \"RocketmqCommon\";\r\n    public static final String STORE_LOGGER_NAME = \"RocketmqStore\";\r\n    public static final String STORE_ERROR_LOGGER_NAME = \"RocketmqStoreError\";\r\n    public static final String TRANSACTION_LOGGER_NAME = \"RocketmqTransaction\";\r\n    public static final String REBALANCE_LOCK_LOGGER_NAME = \"RocketmqRebalanceLock\";\r\n    public static final String ROCKETMQ_STATS_LOGGER_NAME = \"RocketmqStats\";\r\n    public static final String DLQ_STATS_LOGGER_NAME = \"RocketmqDLQStats\";\r\n    public static final String DLQ_LOGGER_NAME = \"RocketmqDLQ\";\r\n    public static final String CONSUMER_STATS_LOGGER_NAME = \"RocketmqConsumerStats\";\r\n    public static final String COMMERCIAL_LOGGER_NAME = \"RocketmqCommercial\";\r\n    public static final String ACCOUNT_LOGGER_NAME = \"RocketmqAccount\";\r\n    public static final String FLOW_CONTROL_LOGGER_NAME = \"RocketmqFlowControl\";\r\n    public static final String ROCKETMQ_AUTHORIZE_LOGGER_NAME = \"RocketmqAuthorize\";\r\n    public static final String DUPLICATION_LOGGER_NAME = \"RocketmqDuplication\";\r\n    public static final String PROTECTION_LOGGER_NAME = \"RocketmqProtection\";\r\n    public static final String WATER_MARK_LOGGER_NAME = \"RocketmqWaterMark\";\r\n    public static final String FILTER_LOGGER_NAME = \"RocketmqFilter\";\r\n    public static final String ROCKETMQ_POP_LOGGER_NAME = \"RocketmqPop\";\r\n    public static final String ROCKETMQ_POP_LITE_LOGGER_NAME = \"RocketmqPopLite\";\r\n    public static final String FAILOVER_LOGGER_NAME = \"RocketmqFailover\";\r\n    public static final String STDOUT_LOGGER_NAME = \"STDOUT\";\r\n    public static final String PROXY_LOGGER_NAME = \"RocketmqProxy\";\r\n    public static final String PROXY_WATER_MARK_LOGGER_NAME = \"RocketmqProxyWatermark\";\r\n    public static final String ROCKETMQ_COLDCTR_LOGGER_NAME = \"RocketmqColdCtr\";\r\n    public static final String ROCKSDB_LOGGER_NAME = \"RocketmqRocksDB\";\r\n\r\n    public static final String ROCKETMQ_AUTH_AUDIT_LOGGER_NAME = \"RocketmqAuthAudit\";\r\n}\r\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/constant/PermName.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.constant;\n\npublic class PermName {\n    public static final int INDEX_PERM_PRIORITY = 3;\n    public static final int INDEX_PERM_READ = 2;\n    public static final int INDEX_PERM_WRITE = 1;\n    public static final int INDEX_PERM_INHERIT = 0;\n\n\n    public static final int PERM_PRIORITY = 0x1 << INDEX_PERM_PRIORITY;\n    public static final int PERM_READ = 0x1 << INDEX_PERM_READ;\n    public static final int PERM_WRITE = 0x1 << INDEX_PERM_WRITE;\n    public static final int PERM_INHERIT = 0x1 << INDEX_PERM_INHERIT;\n\n    public static String perm2String(final int perm) {\n        final StringBuilder sb = new StringBuilder(\"---\");\n        if (isReadable(perm)) {\n            sb.replace(0, 1, \"R\");\n        }\n\n        if (isWriteable(perm)) {\n            sb.replace(1, 2, \"W\");\n        }\n\n        if (isInherited(perm)) {\n            sb.replace(2, 3, \"X\");\n        }\n\n        return sb.toString();\n    }\n\n    public static boolean isReadable(final int perm) {\n        return (perm & PERM_READ) == PERM_READ;\n    }\n\n    public static boolean isWriteable(final int perm) {\n        return (perm & PERM_WRITE) == PERM_WRITE;\n    }\n\n    public static boolean isInherited(final int perm) {\n        return (perm & PERM_INHERIT) == PERM_INHERIT;\n    }\n\n    public static boolean isValid(final String perm) {\n        return isValid(Integer.parseInt(perm));\n    }\n\n    public static boolean isValid(final int perm) {\n        return perm >= 0 && perm < PERM_PRIORITY;\n    }\n    \n    public static boolean isPriority(final int perm) {\n        return (perm & PERM_PRIORITY) == PERM_PRIORITY;\n    }\n\n    public static boolean isAccessible(final int perm) {\n        return isReadable(perm) || isWriteable(perm);\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/consumer/ConsumeFromWhere.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.consumer;\n\npublic enum ConsumeFromWhere {\n    CONSUME_FROM_LAST_OFFSET,\n\n    @Deprecated\n    CONSUME_FROM_LAST_OFFSET_AND_FROM_MIN_WHEN_BOOT_FIRST,\n    @Deprecated\n    CONSUME_FROM_MIN_OFFSET,\n    @Deprecated\n    CONSUME_FROM_MAX_OFFSET,\n    CONSUME_FROM_FIRST_OFFSET,\n    CONSUME_FROM_TIMESTAMP,\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/consumer/ReceiptHandle.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.consumer;\n\nimport java.util.Arrays;\nimport java.util.List;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.message.MessageConst;\n\npublic class ReceiptHandle {\n    private static final String SEPARATOR = MessageConst.KEY_SEPARATOR;\n    public static final String NORMAL_TOPIC = \"0\";\n    public static final String RETRY_TOPIC = \"1\";\n\n    public static final String RETRY_TOPIC_V2 = \"2\";\n    private final long startOffset;\n    private final long retrieveTime;\n    private final long invisibleTime;\n    private final long nextVisibleTime;\n    private final int reviveQueueId;\n    private final String topicType;\n    private final String brokerName;\n    private final int queueId;\n    private final long offset;\n    private final long commitLogOffset;\n    private final String receiptHandle;\n\n    public String encode() {\n        return startOffset + SEPARATOR + retrieveTime + SEPARATOR + invisibleTime + SEPARATOR + reviveQueueId\n            + SEPARATOR + topicType + SEPARATOR + brokerName + SEPARATOR + queueId + SEPARATOR + offset + SEPARATOR\n            + commitLogOffset;\n    }\n\n    public boolean isExpired() {\n        return nextVisibleTime <= System.currentTimeMillis();\n    }\n\n    public static ReceiptHandle decode(String receiptHandle) {\n        List<String> dataList = Arrays.asList(receiptHandle.split(SEPARATOR));\n        if (dataList.size() < 8) {\n            throw new IllegalArgumentException(\"Parse failed, dataList size \" + dataList.size());\n        }\n        long startOffset = Long.parseLong(dataList.get(0));\n        long retrieveTime = Long.parseLong(dataList.get(1));\n        long invisibleTime = Long.parseLong(dataList.get(2));\n        int reviveQueueId = Integer.parseInt(dataList.get(3));\n        String topicType = dataList.get(4);\n        String brokerName = dataList.get(5);\n        int queueId = Integer.parseInt(dataList.get(6));\n        long offset = Long.parseLong(dataList.get(7));\n        long commitLogOffset = -1L;\n        if (dataList.size() >= 9) {\n            commitLogOffset = Long.parseLong(dataList.get(8));\n        }\n\n        return new ReceiptHandleBuilder()\n            .startOffset(startOffset)\n            .retrieveTime(retrieveTime)\n            .invisibleTime(invisibleTime)\n            .reviveQueueId(reviveQueueId)\n            .topicType(topicType)\n            .brokerName(brokerName)\n            .queueId(queueId)\n            .offset(offset)\n            .commitLogOffset(commitLogOffset)\n            .receiptHandle(receiptHandle).build();\n    }\n\n    ReceiptHandle(final long startOffset, final long retrieveTime, final long invisibleTime, final long nextVisibleTime,\n        final int reviveQueueId, final String topicType, final String brokerName, final int queueId, final long offset,\n        final long commitLogOffset, final String receiptHandle) {\n        this.startOffset = startOffset;\n        this.retrieveTime = retrieveTime;\n        this.invisibleTime = invisibleTime;\n        this.nextVisibleTime = nextVisibleTime;\n        this.reviveQueueId = reviveQueueId;\n        this.topicType = topicType;\n        this.brokerName = brokerName;\n        this.queueId = queueId;\n        this.offset = offset;\n        this.commitLogOffset = commitLogOffset;\n        this.receiptHandle = receiptHandle;\n    }\n\n    public static class ReceiptHandleBuilder {\n        private long startOffset;\n        private long retrieveTime;\n        private long invisibleTime;\n        private int reviveQueueId;\n        private String topicType;\n        private String brokerName;\n        private int queueId;\n        private long offset;\n        private long commitLogOffset;\n        private String receiptHandle;\n\n        ReceiptHandleBuilder() {\n        }\n\n        public ReceiptHandle.ReceiptHandleBuilder startOffset(final long startOffset) {\n            this.startOffset = startOffset;\n            return this;\n        }\n\n        public ReceiptHandle.ReceiptHandleBuilder retrieveTime(final long retrieveTime) {\n            this.retrieveTime = retrieveTime;\n            return this;\n        }\n\n        public ReceiptHandle.ReceiptHandleBuilder invisibleTime(final long invisibleTime) {\n            this.invisibleTime = invisibleTime;\n            return this;\n        }\n\n        public ReceiptHandle.ReceiptHandleBuilder reviveQueueId(final int reviveQueueId) {\n            this.reviveQueueId = reviveQueueId;\n            return this;\n        }\n\n        public ReceiptHandle.ReceiptHandleBuilder topicType(final String topicType) {\n            this.topicType = topicType;\n            return this;\n        }\n\n        public ReceiptHandle.ReceiptHandleBuilder brokerName(final String brokerName) {\n            this.brokerName = brokerName;\n            return this;\n        }\n\n        public ReceiptHandle.ReceiptHandleBuilder queueId(final int queueId) {\n            this.queueId = queueId;\n            return this;\n        }\n\n        public ReceiptHandle.ReceiptHandleBuilder offset(final long offset) {\n            this.offset = offset;\n            return this;\n        }\n\n        public ReceiptHandle.ReceiptHandleBuilder commitLogOffset(final long commitLogOffset) {\n            this.commitLogOffset = commitLogOffset;\n            return this;\n        }\n\n        public ReceiptHandle.ReceiptHandleBuilder receiptHandle(final String receiptHandle) {\n            this.receiptHandle = receiptHandle;\n            return this;\n        }\n\n        public ReceiptHandle build() {\n            return new ReceiptHandle(this.startOffset, this.retrieveTime, this.invisibleTime, this.retrieveTime + this.invisibleTime,\n                this.reviveQueueId, this.topicType, this.brokerName, this.queueId, this.offset, this.commitLogOffset, this.receiptHandle);\n        }\n\n        @Override\n        public String toString() {\n            return \"ReceiptHandle.ReceiptHandleBuilder(startOffset=\" + this.startOffset + \", retrieveTime=\" + this.retrieveTime + \", invisibleTime=\" + this.invisibleTime + \", reviveQueueId=\" + this.reviveQueueId + \", topic=\" + this.topicType + \", brokerName=\" + this.brokerName + \", queueId=\" + this.queueId + \", offset=\" + this.offset + \", commitLogOffset=\" + this.commitLogOffset + \", receiptHandle=\" + this.receiptHandle + \")\";\n        }\n    }\n\n    public static ReceiptHandle.ReceiptHandleBuilder builder() {\n        return new ReceiptHandle.ReceiptHandleBuilder();\n    }\n\n    public long getStartOffset() {\n        return this.startOffset;\n    }\n\n    public long getRetrieveTime() {\n        return this.retrieveTime;\n    }\n\n    public long getInvisibleTime() {\n        return this.invisibleTime;\n    }\n\n    public long getNextVisibleTime() {\n        return this.nextVisibleTime;\n    }\n\n    public int getReviveQueueId() {\n        return this.reviveQueueId;\n    }\n\n    public String getTopicType() {\n        return this.topicType;\n    }\n\n    public String getBrokerName() {\n        return this.brokerName;\n    }\n\n    public int getQueueId() {\n        return this.queueId;\n    }\n\n    public long getOffset() {\n        return this.offset;\n    }\n\n    public long getCommitLogOffset() {\n        return commitLogOffset;\n    }\n\n    public String getReceiptHandle() {\n        return this.receiptHandle;\n    }\n\n    public boolean isRetryTopic() {\n        return RETRY_TOPIC.equals(topicType) || RETRY_TOPIC_V2.equals(topicType);\n    }\n\n    public String getRealTopic(String topic, String groupName) {\n        if (RETRY_TOPIC.equals(topicType)) {\n            return KeyBuilder.buildPopRetryTopicV1(topic, groupName);\n        }\n        if (RETRY_TOPIC_V2.equals(topicType)) {\n            return KeyBuilder.buildPopRetryTopicV2(topic, groupName);\n        }\n        return topic;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/entity/ClientGroup.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.entity;\n\nimport java.util.Objects;\n\npublic class ClientGroup {\n\n    public final String clientId;\n    public final String group;\n    /**\n     * Cache the hash code for the object\n     */\n    private int hash; // Default to 0\n\n    public ClientGroup(String clientId, String group) {\n        this.clientId = clientId;\n        this.group = group;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (o == null || getClass() != o.getClass()) {\n            return false;\n        }\n        ClientGroup that = (ClientGroup) o;\n        return Objects.equals(clientId, that.clientId)\n            && Objects.equals(group, that.group);\n    }\n\n    @Override\n    public int hashCode() {\n        if (hash == 0) {\n            hash = Objects.hash(clientId, group);\n        }\n        return hash;\n    }\n\n    @Override\n    public String toString() {\n        return \"ClientGroup{\" +\n            \"clientId='\" + clientId + '\\'' +\n            \", group='\" + group + '\\'' +\n            '}';\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/entity/TopicGroup.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.entity;\n\nimport java.util.Objects;\n\npublic class TopicGroup {\n\n    public final String topic;\n    public final String group;\n    /**\n     * Cache the hash code for the object\n     */\n    private int hash; // Default to 0\n\n    public TopicGroup(String topic, String group) {\n        this.topic = topic;\n        this.group = group;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (o == null || getClass() != o.getClass()) {\n            return false;\n        }\n        TopicGroup that = (TopicGroup) o;\n        return Objects.equals(topic, that.topic) && Objects.equals(group, that.group);\n    }\n\n    @Override\n    public int hashCode() {\n        if (hash == 0) {\n            hash = Objects.hash(topic, group);\n        }\n        return hash;\n    }\n\n    @Override\n    public String toString() {\n        return \"TopicGroup{\" +\n            \"topic='\" + topic + '\\'' +\n            \", group='\" + group + '\\'' +\n            '}';\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/fastjson/GenericMapSuperclassDeserializer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.fastjson;\n\nimport com.alibaba.fastjson2.JSONException;\nimport com.alibaba.fastjson2.JSONReader;\nimport com.alibaba.fastjson2.reader.ObjectReader;\n\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport java.util.Map;\n\n/**\n * workaround https://github.com/alibaba/fastjson/issues/3730\n */\npublic class GenericMapSuperclassDeserializer implements ObjectReader<Object> {\n    public static final GenericMapSuperclassDeserializer INSTANCE = new GenericMapSuperclassDeserializer();\n    @SuppressWarnings({\"unchecked\", \"rawtypes\"})\n    @Override\n    public Object readObject(JSONReader reader, Type type, Object fieldName, long features) {\n        Class<?> clz = (Class<?>) type;\n        Type genericSuperclass = clz.getGenericSuperclass();\n        Map map;\n        try {\n            map = (Map) clz.getDeclaredConstructor().newInstance();\n        } catch (Exception e) {\n            throw new JSONException(\"unsupport type \" + type, e);\n        }\n        ParameterizedType parameterizedType = (ParameterizedType) genericSuperclass;\n        Type keyType = parameterizedType.getActualTypeArguments()[0];\n        Type valueType = parameterizedType.getActualTypeArguments()[1];\n\n        if (!reader.nextIfObjectStart()) {\n            throw new JSONException(reader.info(\"expect '{', but \" + reader.current()));\n        }\n\n        while (!reader.nextIfObjectEnd()) {\n            Object key;\n            if (keyType == String.class) {\n                key = reader.readFieldName();\n            } else {\n                key = reader.getContext().getProvider().getObjectReader(keyType).readObject(reader, keyType, fieldName, features);\n                reader.nextIfMatch(':');\n            }\n\n            Object value = reader.getContext().getProvider().getObjectReader(valueType).readObject(reader, valueType, fieldName, features);\n            map.put(key, value);\n            reader.nextIfComma();\n        }\n        return map;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/filter/ExpressionType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.filter;\n\npublic class ExpressionType {\n\n    /**\n     * <ul>\n     * Keywords:\n     * <li>{@code AND, OR, NOT, BETWEEN, IN, TRUE, FALSE, IS, NULL}</li>\n     * </ul>\n     * <p/>\n     * <ul>\n     * Data type:\n     * <li>Boolean, like: TRUE, FALSE</li>\n     * <li>String, like: 'abc'</li>\n     * <li>Decimal, like: 123</li>\n     * <li>Float number, like: 3.1415</li>\n     * </ul>\n     * <p/>\n     * <ul>\n     * Grammar:\n     * <li>{@code AND, OR}</li>\n     * <li>{@code >, >=, <, <=, =}</li>\n     * <li>{@code BETWEEN A AND B}, equals to {@code >=A AND <=B}</li>\n     * <li>{@code NOT BETWEEN A AND B}, equals to {@code >B OR <A}</li>\n     * <li>{@code IN ('a', 'b')}, equals to {@code ='a' OR ='b'}, this operation only support String type.</li>\n     * <li>{@code IS NULL}, {@code IS NOT NULL}, check parameter whether is null, or not.</li>\n     * <li>{@code =TRUE}, {@code =FALSE}, check parameter whether is true, or false.</li>\n     * </ul>\n     * <p/>\n     * <p>\n     * Example:\n     * (a > 10 AND a < 100) OR (b IS NOT NULL AND b=TRUE)\n     * </p>\n     */\n    public static final String SQL92 = \"SQL92\";\n\n    /**\n     * Only support or operation such as\n     * \"tag1 || tag2 || tag3\", <br>\n     * If null or * expression,meaning subscribe all.\n     */\n    public static final String TAG = \"TAG\";\n\n    public static boolean isTagType(String type) {\n        if (type == null || \"\".equals(type) || TAG.equals(type)) {\n            return true;\n        }\n        return false;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/filter/FilterContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.filter;\n\npublic class FilterContext {\n    private String consumerGroup;\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/filter/MessageFilter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.filter;\n\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic interface MessageFilter {\n    boolean match(final MessageExt msg, final FilterContext context);\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/filter/impl/Op.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.filter.impl;\n\npublic abstract class Op {\n\n    private String symbol;\n\n    protected Op(String symbol) {\n        this.symbol = symbol;\n    }\n\n    public String getSymbol() {\n        return symbol;\n    }\n\n    public String toString() {\n        return symbol;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/filter/impl/Operand.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.filter.impl;\n\npublic class Operand extends Op {\n\n    public Operand(String symbol) {\n        super(symbol);\n    }\n\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/filter/impl/Operator.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.filter.impl;\n\npublic class Operator extends Op {\n\n    public static final Operator LEFTPARENTHESIS = new Operator(\"(\", 30, false);\n    public static final Operator RIGHTPARENTHESIS = new Operator(\")\", 30, false);\n    public static final Operator AND = new Operator(\"&&\", 20, true);\n    public static final Operator OR = new Operator(\"||\", 15, true);\n\n    private int priority;\n    private boolean compareable;\n\n    private Operator(String symbol, int priority, boolean compareable) {\n        super(symbol);\n        this.priority = priority;\n        this.compareable = compareable;\n    }\n\n    public static Operator createOperator(String operator) {\n        if (LEFTPARENTHESIS.getSymbol().equals(operator))\n            return LEFTPARENTHESIS;\n        else if (RIGHTPARENTHESIS.getSymbol().equals(operator))\n            return RIGHTPARENTHESIS;\n        else if (AND.getSymbol().equals(operator))\n            return AND;\n        else if (OR.getSymbol().equals(operator))\n            return OR;\n        else\n            throw new IllegalArgumentException(\"unsupport operator \" + operator);\n    }\n\n    public int getPriority() {\n        return priority;\n    }\n\n    public boolean isCompareable() {\n        return compareable;\n    }\n\n    public int compare(Operator operator) {\n        if (this.priority > operator.priority)\n            return 1;\n        else if (this.priority == operator.priority)\n            return 0;\n        else\n            return -1;\n    }\n\n    public boolean isSpecifiedOp(String operator) {\n        return this.getSymbol().equals(operator);\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/filter/impl/PolishExpr.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.filter.impl;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Stack;\n\nimport static org.apache.rocketmq.common.filter.impl.Operator.LEFTPARENTHESIS;\nimport static org.apache.rocketmq.common.filter.impl.Operator.RIGHTPARENTHESIS;\nimport static org.apache.rocketmq.common.filter.impl.Operator.createOperator;\n\npublic class PolishExpr {\n\n    public static List<Op> reversePolish(String expression) {\n        return reversePolish(participle(expression));\n    }\n\n    /**\n     * Shunting-yard algorithm <br/>\n     * http://en.wikipedia.org/wiki/Shunting_yard_algorithm\n     *\n     * @return the compute result of Shunting-yard algorithm\n     */\n    public static List<Op> reversePolish(List<Op> tokens) {\n        List<Op> segments = new ArrayList<>();\n        Stack<Operator> operatorStack = new Stack<>();\n\n        for (int i = 0; i < tokens.size(); i++) {\n            Op token = tokens.get(i);\n            if (isOperand(token)) {\n\n                segments.add(token);\n            } else if (isLeftParenthesis(token)) {\n\n                operatorStack.push((Operator) token);\n            } else if (isRightParenthesis(token)) {\n\n                Operator opNew = null;\n                while (!operatorStack.empty() && LEFTPARENTHESIS != (opNew = operatorStack.pop())) {\n                    segments.add(opNew);\n                }\n                if (null == opNew || LEFTPARENTHESIS != opNew)\n                    throw new IllegalArgumentException(\"mismatched parentheses\");\n            } else if (isOperator(token)) {\n\n                Operator opNew = (Operator) token;\n                if (!operatorStack.empty()) {\n                    Operator opOld = operatorStack.peek();\n                    if (opOld.isCompareable() && opNew.compare(opOld) != 1) {\n                        segments.add(operatorStack.pop());\n                    }\n                }\n                operatorStack.push(opNew);\n            } else\n                throw new IllegalArgumentException(\"illegal token \" + token);\n        }\n\n        while (!operatorStack.empty()) {\n            Operator operator = operatorStack.pop();\n            if (LEFTPARENTHESIS == operator || RIGHTPARENTHESIS == operator)\n                throw new IllegalArgumentException(\"mismatched parentheses \" + operator);\n            segments.add(operator);\n        }\n\n        return segments;\n    }\n\n    /**\n     * @param expression\n     * @return\n     * @throws Exception\n     */\n    private static List<Op> participle(String expression) {\n        List<Op> segments = new ArrayList<>();\n\n        int size = expression.length();\n        int wordStartIndex = -1;\n        int wordLen = 0;\n        Type preType = Type.NULL;\n\n        for (int i = 0; i < size; i++) {\n            int chValue = (int) expression.charAt(i);\n\n            if (97 <= chValue && chValue <= 122 || 65 <= chValue && chValue <= 90\n                || 49 <= chValue && chValue <= 57 || 95 == chValue) {\n\n                if (Type.OPERATOR == preType || Type.SEPAERATOR == preType || Type.NULL == preType\n                    || Type.PARENTHESIS == preType) {\n                    if (Type.OPERATOR == preType) {\n                        segments.add(createOperator(expression.substring(wordStartIndex, wordStartIndex\n                            + wordLen)));\n                    }\n                    wordStartIndex = i;\n                    wordLen = 0;\n                }\n                preType = Type.OPERAND;\n                wordLen++;\n            } else if (40 == chValue || 41 == chValue) {\n\n                if (Type.OPERATOR == preType) {\n                    segments.add(createOperator(expression\n                        .substring(wordStartIndex, wordStartIndex + wordLen)));\n                    wordStartIndex = -1;\n                    wordLen = 0;\n                } else if (Type.OPERAND == preType) {\n                    segments.add(new Operand(expression.substring(wordStartIndex, wordStartIndex + wordLen)));\n                    wordStartIndex = -1;\n                    wordLen = 0;\n                }\n\n                preType = Type.PARENTHESIS;\n                segments.add(createOperator((char) chValue + \"\"));\n            } else if (38 == chValue || 124 == chValue) {\n\n                if (Type.OPERAND == preType || Type.SEPAERATOR == preType || Type.PARENTHESIS == preType) {\n                    if (Type.OPERAND == preType) {\n                        segments.add(new Operand(expression.substring(wordStartIndex, wordStartIndex\n                            + wordLen)));\n                    }\n                    wordStartIndex = i;\n                    wordLen = 0;\n                }\n                preType = Type.OPERATOR;\n                wordLen++;\n            } else if (32 == chValue || 9 == chValue) {\n\n                if (Type.OPERATOR == preType) {\n                    segments.add(createOperator(expression\n                        .substring(wordStartIndex, wordStartIndex + wordLen)));\n                    wordStartIndex = -1;\n                    wordLen = 0;\n                } else if (Type.OPERAND == preType) {\n                    segments.add(new Operand(expression.substring(wordStartIndex, wordStartIndex + wordLen)));\n                    wordStartIndex = -1;\n                    wordLen = 0;\n                }\n                preType = Type.SEPAERATOR;\n            } else {\n\n                throw new IllegalArgumentException(\"illegal expression, at index \" + i + \" \" + (char) chValue);\n            }\n\n        }\n\n        if (wordLen > 0) {\n            segments.add(new Operand(expression.substring(wordStartIndex, wordStartIndex + wordLen)));\n        }\n        return segments;\n    }\n\n    public static boolean isOperand(Op token) {\n        return token instanceof Operand;\n    }\n\n    public static boolean isLeftParenthesis(Op token) {\n        return token instanceof Operator && LEFTPARENTHESIS == (Operator) token;\n    }\n\n    public static boolean isRightParenthesis(Op token) {\n        return token instanceof Operator && RIGHTPARENTHESIS == (Operator) token;\n    }\n\n    public static boolean isOperator(Op token) {\n        return token instanceof Operator;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/filter/impl/Type.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.filter.impl;\n\npublic enum Type {\n    NULL,\n    OPERAND,\n    OPERATOR,\n    PARENTHESIS,\n    SEPAERATOR;\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/future/FutureTaskExt.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.future;\n\nimport java.util.concurrent.Callable;\nimport java.util.concurrent.FutureTask;\n\npublic class FutureTaskExt<V> extends FutureTask<V> {\n    private final Runnable runnable;\n\n    public FutureTaskExt(final Callable<V> callable) {\n        super(callable);\n        this.runnable = null;\n    }\n\n    public FutureTaskExt(final Runnable runnable, final V result) {\n        super(runnable, result);\n        this.runnable = runnable;\n    }\n\n    public Runnable getRunnable() {\n        return runnable;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/help/FAQUrl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.help;\n\npublic class FAQUrl {\n\n    public static final String DEFAULT_FAQ_URL = \"https://rocketmq.apache.org/docs/bestPractice/06FAQ\";\n\n    public static final String APPLY_TOPIC_URL = DEFAULT_FAQ_URL;\n\n    public static final String NAME_SERVER_ADDR_NOT_EXIST_URL = DEFAULT_FAQ_URL;\n\n    public static final String GROUP_NAME_DUPLICATE_URL = DEFAULT_FAQ_URL;\n\n    public static final String CLIENT_PARAMETER_CHECK_URL = DEFAULT_FAQ_URL;\n\n    public static final String SUBSCRIPTION_GROUP_NOT_EXIST = DEFAULT_FAQ_URL;\n\n    public static final String CLIENT_SERVICE_NOT_OK = DEFAULT_FAQ_URL;\n\n    // FAQ: No route info of this topic, TopicABC\n    public static final String NO_TOPIC_ROUTE_INFO = DEFAULT_FAQ_URL;\n\n    public static final String LOAD_JSON_EXCEPTION = DEFAULT_FAQ_URL;\n\n    public static final String SAME_GROUP_DIFFERENT_TOPIC = DEFAULT_FAQ_URL;\n\n    public static final String MQLIST_NOT_EXIST = DEFAULT_FAQ_URL;\n\n    public static final String UNEXPECTED_EXCEPTION_URL = DEFAULT_FAQ_URL;\n\n    public static final String SEND_MSG_FAILED = DEFAULT_FAQ_URL;\n\n    public static final String UNKNOWN_HOST_EXCEPTION = DEFAULT_FAQ_URL;\n\n    private static final String TIP_STRING_BEGIN = \"\\nSee \";\n    private static final String TIP_STRING_END = \" for further details.\";\n    private static final String MORE_INFORMATION = \"For more information, please visit the url, \";\n\n    public static String suggestTodo(final String url) {\n        StringBuilder sb = new StringBuilder(TIP_STRING_BEGIN.length() + url.length() + TIP_STRING_END.length());\n        sb.append(TIP_STRING_BEGIN);\n        sb.append(url);\n        sb.append(TIP_STRING_END);\n        return sb.toString();\n    }\n\n    public static String attachDefaultURL(final String errorMessage) {\n        if (errorMessage != null) {\n            int index = errorMessage.indexOf(TIP_STRING_BEGIN);\n            if (-1 == index) {\n                StringBuilder sb = new StringBuilder(errorMessage.length() + UNEXPECTED_EXCEPTION_URL.length() + MORE_INFORMATION.length() + 1);\n                sb.append(errorMessage);\n                sb.append(\"\\n\");\n                sb.append(MORE_INFORMATION);\n                sb.append(UNEXPECTED_EXCEPTION_URL);\n                return sb.toString();\n            }\n        }\n\n        return errorMessage;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/hook/FilterCheckHook.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.hook;\n\nimport java.nio.ByteBuffer;\n\npublic interface FilterCheckHook {\n    String hookName();\n\n    boolean isFilterMatched(final boolean isUnitMode, final ByteBuffer byteBuffer);\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/lite/LiteLagInfo.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.lite;\n\npublic class LiteLagInfo {\n    private String liteTopic;\n    private long lagCount;\n    // earliest unconsumed timestamp\n    private long earliestUnconsumedTimestamp = -1;\n\n    public String getLiteTopic() {\n        return liteTopic;\n    }\n\n    public void setLiteTopic(String liteTopic) {\n        this.liteTopic = liteTopic;\n    }\n\n    public long getLagCount() {\n        return lagCount;\n    }\n\n    public void setLagCount(long lagCount) {\n        this.lagCount = lagCount;\n    }\n\n    public long getEarliestUnconsumedTimestamp() {\n        return earliestUnconsumedTimestamp;\n    }\n\n    public void setEarliestUnconsumedTimestamp(long earliestUnconsumedTimestamp) {\n        this.earliestUnconsumedTimestamp = earliestUnconsumedTimestamp;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/lite/LiteSubscription.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.lite;\n\nimport java.util.Collection;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\n\npublic class LiteSubscription {\n    private String group;\n    private String topic;\n    private final Set<String> liteTopicSet = ConcurrentHashMap.newKeySet();\n    private volatile long updateTime = System.currentTimeMillis();\n\n    public boolean addLiteTopic(String liteTopic) {\n        updateTime();\n        return this.liteTopicSet.add(liteTopic);\n    }\n\n    public void addLiteTopic(Collection<String> set) {\n        updateTime();\n        this.liteTopicSet.addAll(set);\n    }\n\n    public boolean removeLiteTopic(String liteTopic) {\n        updateTime();\n        return this.liteTopicSet.remove(liteTopic);\n    }\n\n    public void removeLiteTopic(Collection<String> set) {\n        updateTime();\n        this.liteTopicSet.removeAll(set);\n    }\n\n    public String getGroup() {\n        return group;\n    }\n\n    public LiteSubscription setGroup(String group) {\n        this.group = group;\n        return this;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public LiteSubscription setTopic(String topic) {\n        this.topic = topic;\n        return this;\n    }\n\n    public Set<String> getLiteTopicSet() {\n        return liteTopicSet;\n    }\n\n    public LiteSubscription setLiteTopicSet(Set<String> liteTopicSet) {\n        this.liteTopicSet.addAll(liteTopicSet);\n        return this;\n    }\n\n    public long getUpdateTime() {\n        return updateTime;\n    }\n\n    public void setUpdateTime(long updateTime) {\n        this.updateTime = updateTime;\n    }\n\n    private void updateTime() {\n        this.updateTime = System.currentTimeMillis();\n    }\n\n    @Override\n    public String toString() {\n        return \"LiteSubscription{\" +\n            \"group='\" + group + '\\'' +\n            \", topic='\" + topic + '\\'' +\n            \", liteTopicSet=\" + liteTopicSet +\n            \", updateTime=\" + updateTime +\n            '}';\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/lite/LiteSubscriptionAction.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.lite;\n\npublic enum LiteSubscriptionAction {\n    PARTIAL_ADD,\n    PARTIAL_REMOVE,\n    COMPLETE_ADD,\n    COMPLETE_REMOVE\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/lite/LiteSubscriptionDTO.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.lite;\n\nimport java.util.Set;\n\npublic class LiteSubscriptionDTO {\n    private LiteSubscriptionAction action;\n    private String clientId;\n    private String group;\n    private String topic;\n    private Set<String> liteTopicSet;\n    private OffsetOption offsetOption;\n    private long version;\n\n    public LiteSubscriptionAction getAction() {\n        return action;\n    }\n\n    public LiteSubscriptionDTO setAction(LiteSubscriptionAction action) {\n        this.action = action;\n        return this;\n    }\n\n    public String getClientId() {\n        return clientId;\n    }\n\n    public LiteSubscriptionDTO setClientId(String clientId) {\n        this.clientId = clientId;\n        return this;\n    }\n\n    public String getGroup() {\n        return group;\n    }\n\n    public LiteSubscriptionDTO setGroup(String group) {\n        this.group = group;\n        return this;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public LiteSubscriptionDTO setTopic(String topic) {\n        this.topic = topic;\n        return this;\n    }\n\n    public Set<String> getLiteTopicSet() {\n        return liteTopicSet;\n    }\n\n    public LiteSubscriptionDTO setLiteTopicSet(Set<String> liteTopicSet) {\n        this.liteTopicSet = liteTopicSet;\n        return this;\n    }\n\n    public OffsetOption getOffsetOption() {\n        return offsetOption;\n    }\n\n    public void setOffsetOption(OffsetOption offsetOption) {\n        this.offsetOption = offsetOption;\n    }\n\n    public long getVersion() {\n        return version;\n    }\n\n    public LiteSubscriptionDTO setVersion(long version) {\n        this.version = version;\n        return this;\n    }\n\n    @Override\n    public String toString() {\n        return \"LiteSubscriptionDTO{\" + \"action=\" + action +\n            \", clientId='\" + clientId + '\\'' +\n            \", group='\" + group + '\\'' +\n            \", topic='\" + topic + '\\'' +\n            \", liteTopicSet=\" + liteTopicSet +\n            \", offsetOption=\" + offsetOption +\n            \", version=\" + version +\n            '}';\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/lite/LiteUtil.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.lite;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.Pair;\n\npublic class LiteUtil {\n\n    public static final char SEPARATOR = '$';\n    public static final String LITE_TOPIC_PREFIX = MixAll.LMQ_PREFIX + SEPARATOR;\n\n    /**\n     * Lite Topic: A specific type of message topic implemented based on LMQ, which has no retry topic.\n     * A lite topic's underlying storage is a lmq (Light Message Queue),\n     * but the reverse is not true: lmq is not necessarily a lite topic,\n     * we use \"$\" as a separator to achieve the distinction and assume \"$\" is not allowed for topic name.\n     * pattern like: %LMQ%$parentTopic$liteTopic\n     *\n     * @param parentTopic act as namespace\n     * @param liteTopic here means child topic string\n     * @return lmqName\n     */\n    public static String toLmqName(String parentTopic, String liteTopic) {\n        if (StringUtils.isEmpty(parentTopic) || StringUtils.isEmpty(liteTopic)) {\n            return null;\n        }\n        return LITE_TOPIC_PREFIX + parentTopic + SEPARATOR + liteTopic;\n    }\n\n    /**\n     * whether lmqName is queue of a lite topic, here we only check the prefix.\n     * @param lmqName\n     * @return\n     */\n    public static boolean isLiteTopicQueue(String lmqName) {\n        return lmqName != null && lmqName.startsWith(LITE_TOPIC_PREFIX);\n    }\n\n    public static String getParentTopic(String lmqName) {\n        if (!isLiteTopicQueue(lmqName)) {\n            return null;\n        }\n        int index = lmqName.indexOf(SEPARATOR, LITE_TOPIC_PREFIX.length());\n        if (index == -1 || index == lmqName.length() - 1 || index == LITE_TOPIC_PREFIX.length()) {\n            return null;\n        }\n        if (lmqName.indexOf(SEPARATOR, index + 1) != -1) {\n            return null;\n        }\n        return lmqName.substring(LITE_TOPIC_PREFIX.length(), index);\n    }\n\n    public static String getLiteTopic(String lmqName) {\n        if (!isLiteTopicQueue(lmqName)) {\n            return null;\n        }\n        int index = lmqName.indexOf(SEPARATOR, LITE_TOPIC_PREFIX.length());\n        if (index == -1 || index == lmqName.length() - 1 || index == LITE_TOPIC_PREFIX.length()) {\n            return null;\n        }\n        if (lmqName.indexOf(SEPARATOR, index + 1) != -1) {\n            return null;\n        }\n        return lmqName.substring(index + 1);\n    }\n\n    /**\n     * %LMQ%${parentTopic}${liteTopic}\n     * parse parent topic and child topic from lmqName\n     * @param lmqName\n     * @return\n     */\n    public static Pair<String, String> getParentAndLiteTopic(String lmqName) {\n        if (null == lmqName || !lmqName.startsWith(LITE_TOPIC_PREFIX)) {\n            return null;\n        }\n        String[] array = StringUtils.split(lmqName, SEPARATOR);\n        if (array.length != 3) {\n            return null;\n        }\n        return new Pair<>(array[1], array[2]);\n    }\n\n    /**\n     * whether lmqName is queue of a lite topic and belongs to the specified parent,\n     * here we only check the prefix.\n     * @param lmqName\n     * @param parentTopic\n     * @return\n     */\n    public static boolean belongsTo(String lmqName, String parentTopic) {\n        return lmqName != null && lmqName.startsWith(LITE_TOPIC_PREFIX + parentTopic + SEPARATOR);\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/lite/OffsetOption.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.lite;\n\nimport java.util.Objects;\n\npublic class OffsetOption {\n\n    public static final long POLICY_LAST_VALUE = 0L;\n    public static final long POLICY_MIN_VALUE = 1L;\n    public static final long POLICY_MAX_VALUE = 2L;\n\n    private Type type;\n    private long value;\n\n    public OffsetOption() {\n    }\n\n    public OffsetOption(Type type, long value) {\n        this.type = type;\n        this.value = value;\n    }\n\n    public Type getType() {\n        return type;\n    }\n\n    public void setType(Type type) {\n        this.type = type;\n    }\n\n    public long getValue() {\n        return value;\n    }\n\n    public void setValue(long value) {\n        this.value = value;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (o == null || getClass() != o.getClass()) {\n            return false;\n        }\n\n        OffsetOption option = (OffsetOption) o;\n        return value == option.value && type == option.type;\n    }\n\n    @Override\n    public int hashCode() {\n        int result = Objects.hashCode(type);\n        result = 31 * result + Long.hashCode(value);\n        return result;\n    }\n\n    @Override\n    public String toString() {\n        return \"OffsetOption{\" + \"type=\" + type +\n            \", value=\" + value +\n            '}';\n    }\n\n    public enum Type {\n        POLICY,\n        OFFSET,\n        TAIL_N,\n        TIMESTAMP\n    }\n\n}"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/logging/DefaultJoranConfiguratorExt.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.logging;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.net.MalformedURLException;\nimport java.net.URL;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Set;\nimport org.apache.rocketmq.logging.ch.qos.logback.classic.ClassicConstants;\nimport org.apache.rocketmq.logging.ch.qos.logback.classic.LoggerContext;\nimport org.apache.rocketmq.logging.ch.qos.logback.classic.util.DefaultJoranConfigurator;\nimport org.apache.rocketmq.logging.ch.qos.logback.core.LogbackException;\nimport org.apache.rocketmq.logging.ch.qos.logback.core.joran.spi.JoranException;\nimport org.apache.rocketmq.logging.ch.qos.logback.core.status.InfoStatus;\nimport org.apache.rocketmq.logging.ch.qos.logback.core.status.StatusManager;\nimport org.apache.rocketmq.logging.ch.qos.logback.core.util.Loader;\nimport org.apache.rocketmq.logging.ch.qos.logback.core.util.OptionHelper;\n\npublic class DefaultJoranConfiguratorExt extends DefaultJoranConfigurator {\n\n    final public static String TEST_AUTOCONFIG_FILE = \"rmq.logback-test.xml\";\n    final public static String AUTOCONFIG_FILE = \"rmq.logback.xml\";\n\n    final public static String PROXY_AUTOCONFIG_FILE = \"rmq.proxy.logback.xml\";\n    final public static String BROKER_AUTOCONFIG_FILE = \"rmq.broker.logback.xml\";\n\n    final public static String NAMESRV_AUTOCONFIG_FILE = \"rmq.namesrv.logback.xml\";\n    final public static String CONTROLLER_AUTOCONFIG_FILE = \"rmq.controller.logback.xml\";\n    final public static String TOOLS_AUTOCONFIG_FILE = \"rmq.tools.logback.xml\";\n\n    final public static String CLIENT_AUTOCONFIG_FILE = \"rmq.client.logback.xml\";\n\n    private final List<String> configFiles;\n\n    public DefaultJoranConfiguratorExt() {\n        this.configFiles = new ArrayList<>();\n        configFiles.add(TEST_AUTOCONFIG_FILE);\n        configFiles.add(AUTOCONFIG_FILE);\n        configFiles.add(PROXY_AUTOCONFIG_FILE);\n        configFiles.add(BROKER_AUTOCONFIG_FILE);\n        configFiles.add(NAMESRV_AUTOCONFIG_FILE);\n        configFiles.add(CONTROLLER_AUTOCONFIG_FILE);\n        configFiles.add(TOOLS_AUTOCONFIG_FILE);\n        configFiles.add(CLIENT_AUTOCONFIG_FILE);\n    }\n\n    @Override\n    public ExecutionStatus configure(LoggerContext loggerContext) {\n        URL url = findURLOfDefaultConfigurationFile(true);\n        if (url != null) {\n            try {\n                configureByResource(url);\n            } catch (JoranException e) {\n                e.printStackTrace();\n            }\n        }\n        // skip other configurator on purpose.\n        return ExecutionStatus.DO_NOT_INVOKE_NEXT_IF_ANY;\n    }\n\n    public void configureByResource(URL url) throws JoranException {\n        if (url == null) {\n            throw new IllegalArgumentException(\"URL argument cannot be null\");\n        }\n        final String urlString = url.toString();\n        if (urlString.endsWith(\"xml\")) {\n            JoranConfiguratorExt configurator = new JoranConfiguratorExt();\n            configurator.setContext(context);\n            configurator.doConfigure0(url);\n        } else {\n            throw new LogbackException(\n                \"Unexpected filename extension of file [\" + url + \"]. Should be .xml\");\n        }\n    }\n\n    public URL findURLOfDefaultConfigurationFile(boolean updateStatus) {\n        ClassLoader myClassLoader = Loader.getClassLoaderOfObject(this);\n        URL url = findConfigFileURLFromSystemProperties(myClassLoader, updateStatus);\n        if (url != null) {\n            return url;\n        }\n\n        for (String configFile : configFiles) {\n            url = getResource(configFile, myClassLoader, updateStatus);\n            if (url != null) {\n                return url;\n            }\n        }\n        return null;\n    }\n\n    private URL findConfigFileURLFromSystemProperties(ClassLoader classLoader, boolean updateStatus) {\n        String logbackConfigFile = OptionHelper.getSystemProperty(ClassicConstants.CONFIG_FILE_PROPERTY);\n        if (logbackConfigFile != null) {\n            URL result = null;\n            try {\n                result = new URL(logbackConfigFile);\n                return result;\n            } catch (MalformedURLException e) {\n                // so, resource is not a URL:\n                // attempt to get the resource from the class path\n                result = Loader.getResource(logbackConfigFile, classLoader);\n                if (result != null) {\n                    return result;\n                }\n                File f = new File(logbackConfigFile);\n                if (f.exists() && f.isFile()) {\n                    try {\n                        result = f.toURI().toURL();\n                        return result;\n                    } catch (MalformedURLException ignored) {\n                    }\n                }\n            } finally {\n                if (updateStatus) {\n                    statusOnResourceSearch(logbackConfigFile, classLoader, result);\n                }\n            }\n        }\n        return null;\n    }\n\n    private URL getResource(String filename, ClassLoader myClassLoader, boolean updateStatus) {\n        URL url = Loader.getResource(filename, myClassLoader);\n        if (updateStatus) {\n            statusOnResourceSearch(filename, myClassLoader, url);\n        }\n        return url;\n    }\n\n    private void statusOnResourceSearch(String resourceName, ClassLoader classLoader, URL url) {\n        StatusManager sm = context.getStatusManager();\n        if (url == null) {\n            sm.add(new InfoStatus(\"Could NOT find resource [\" + resourceName + \"]\", context));\n        } else {\n            sm.add(new InfoStatus(\"Found resource [\" + resourceName + \"] at [\" + url.toString() + \"]\", context));\n            multiplicityWarning(resourceName, classLoader);\n        }\n    }\n\n    private void multiplicityWarning(String resourceName, ClassLoader classLoader) {\n        Set<URL> urlSet = null;\n        try {\n            urlSet = Loader.getResources(resourceName, classLoader);\n        } catch (IOException e) {\n            addError(\"Failed to get url list for resource [\" + resourceName + \"]\", e);\n        }\n        if (urlSet != null && urlSet.size() > 1) {\n            addWarn(\"Resource [\" + resourceName + \"] occurs multiple times on the classpath.\");\n            for (URL url : urlSet) {\n                addWarn(\"Resource [\" + resourceName + \"] occurs at [\" + url.toString() + \"]\");\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/logging/JoranConfiguratorExt.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.logging;\n\nimport com.google.common.io.CharStreams;\nimport java.io.ByteArrayInputStream;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.InputStreamReader;\nimport java.net.URL;\nimport java.net.URLConnection;\nimport java.nio.charset.StandardCharsets;\nimport org.apache.rocketmq.logging.ch.qos.logback.classic.joran.JoranConfigurator;\nimport org.apache.rocketmq.logging.ch.qos.logback.core.joran.spi.JoranException;\n\npublic class JoranConfiguratorExt extends JoranConfigurator {\n    private InputStream transformXml(InputStream in) throws IOException {\n        try {\n            String str = CharStreams.toString(new InputStreamReader(in, StandardCharsets.UTF_8));\n            str = str.replace(\"\\\"ch.qos.logback\", \"\\\"org.apache.rocketmq.logging.ch.qos.logback\");\n            return new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8));\n        } finally {\n            if (null != in) {\n                in.close();\n            }\n        }\n    }\n\n    public final void doConfigure0(URL url) throws JoranException {\n        InputStream in = null;\n        try {\n            informContextOfURLUsedForConfiguration(getContext(), url);\n            URLConnection urlConnection = url.openConnection();\n            // per http://jira.qos.ch/browse/LBCORE-105\n            // per http://jira.qos.ch/browse/LBCORE-127\n            urlConnection.setUseCaches(false);\n\n            InputStream temp = urlConnection.getInputStream();\n            in = transformXml(temp);\n\n            doConfigure(in, url.toExternalForm());\n        } catch (IOException ioe) {\n            String errMsg = \"Could not open URL [\" + url + \"].\";\n            addError(errMsg, ioe);\n            throw new JoranException(errMsg, ioe);\n        } finally {\n            if (in != null) {\n                try {\n                    in.close();\n                } catch (IOException ioe) {\n                    String errMsg = \"Could not close input stream\";\n                    addError(errMsg, ioe);\n                    throw new JoranException(errMsg, ioe);\n                }\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/message/Message.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.message;\n\nimport org.apache.commons.lang3.math.NumberUtils;\n\nimport java.io.Serializable;\nimport java.util.Arrays;\nimport java.util.Collection;\nimport java.util.HashMap;\nimport java.util.Map;\n\npublic class Message implements Serializable {\n    private static final long serialVersionUID = 8445773977080406428L;\n\n    private String topic;\n    private int flag;\n    private Map<String, String> properties;\n    private byte[] body;\n    private String transactionId;\n\n    public Message() {\n    }\n\n    public Message(String topic, byte[] body) {\n        this(topic, \"\", \"\", 0, body, true);\n    }\n\n    public Message(String topic, String tags, String keys, int flag, byte[] body, boolean waitStoreMsgOK) {\n        this.topic = topic;\n        this.flag = flag;\n        this.body = body;\n\n        if (tags != null && tags.length() > 0) {\n            this.setTags(tags);\n        }\n\n        if (keys != null && keys.length() > 0) {\n            this.setKeys(keys);\n        }\n\n        this.setWaitStoreMsgOK(waitStoreMsgOK);\n    }\n\n    public Message(String topic, String tags, byte[] body) {\n        this(topic, tags, \"\", 0, body, true);\n    }\n\n    public Message(String topic, String tags, String keys, byte[] body) {\n        this(topic, tags, keys, 0, body, true);\n    }\n\n    public void setKeys(String keys) {\n        this.putProperty(MessageConst.PROPERTY_KEYS, keys);\n    }\n\n    void putProperty(final String name, final String value) {\n        if (null == this.properties) {\n            this.properties = new HashMap<>();\n        }\n\n        this.properties.put(name, value);\n    }\n\n    void clearProperty(final String name) {\n        if (null != this.properties) {\n            this.properties.remove(name);\n        }\n    }\n\n    public void putUserProperty(final String name, final String value) {\n        if (MessageConst.STRING_HASH_SET.contains(name)) {\n            throw new RuntimeException(String.format(\n                \"The Property<%s> is used by system, input another please\", name));\n        }\n\n        if (value == null || value.trim().isEmpty()\n            || name == null || name.trim().isEmpty()) {\n            throw new IllegalArgumentException(\n                \"The name or value of property can not be null or blank string!\"\n            );\n        }\n\n        this.putProperty(name, value);\n    }\n\n    public String getUserProperty(final String name) {\n        return this.getProperty(name);\n    }\n\n    public String getProperty(final String name) {\n        if (null == this.properties) {\n            this.properties = new HashMap<>();\n        }\n\n        return this.properties.get(name);\n    }\n\n    public boolean hasProperty(final String name) {\n        if (null == this.properties) {\n            return false;\n        }\n        return this.properties.containsKey(name);\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getTags() {\n        return this.getProperty(MessageConst.PROPERTY_TAGS);\n    }\n\n    public void setTags(String tags) {\n        this.putProperty(MessageConst.PROPERTY_TAGS, tags);\n    }\n\n    public String getKeys() {\n        return this.getProperty(MessageConst.PROPERTY_KEYS);\n    }\n\n    public void setKeys(Collection<String> keyCollection) {\n        String keys = String.join(MessageConst.KEY_SEPARATOR, keyCollection);\n\n        this.setKeys(keys);\n    }\n\n    public int getDelayTimeLevel() {\n        String t = this.getProperty(MessageConst.PROPERTY_DELAY_TIME_LEVEL);\n        if (t != null) {\n            return Integer.parseInt(t);\n        }\n\n        return 0;\n    }\n\n    public void setDelayTimeLevel(int level) {\n        this.putProperty(MessageConst.PROPERTY_DELAY_TIME_LEVEL, String.valueOf(level));\n    }\n\n    public void setPriority(int priority) {\n        if (priority < 0) {\n            throw new IllegalArgumentException(\"The priority must be greater than or equal to 0\");\n        }\n        this.putProperty(MessageConst.PROPERTY_PRIORITY, String.valueOf(priority));\n    }\n\n    public int getPriority() {\n        return NumberUtils.toInt(this.getProperty(MessageConst.PROPERTY_PRIORITY), -1);\n    }\n\n    public boolean isWaitStoreMsgOK() {\n        String result = this.getProperty(MessageConst.PROPERTY_WAIT_STORE_MSG_OK);\n        if (null == result) {\n            return true;\n        }\n\n        return Boolean.parseBoolean(result);\n    }\n\n    public void setWaitStoreMsgOK(boolean waitStoreMsgOK) {\n        this.putProperty(MessageConst.PROPERTY_WAIT_STORE_MSG_OK, Boolean.toString(waitStoreMsgOK));\n    }\n\n    public void setInstanceId(String instanceId) {\n        this.putProperty(MessageConst.PROPERTY_INSTANCE_ID, instanceId);\n    }\n\n    public int getFlag() {\n        return flag;\n    }\n\n    public void setFlag(int flag) {\n        this.flag = flag;\n    }\n\n    public byte[] getBody() {\n        return body;\n    }\n\n    public void setBody(byte[] body) {\n        this.body = body;\n    }\n\n    public Map<String, String> getProperties() {\n        return properties;\n    }\n\n    void setProperties(Map<String, String> properties) {\n        this.properties = properties;\n    }\n\n    public String getBuyerId() {\n        return getProperty(MessageConst.PROPERTY_BUYER_ID);\n    }\n\n    public void setBuyerId(String buyerId) {\n        putProperty(MessageConst.PROPERTY_BUYER_ID, buyerId);\n    }\n\n    public String getTransactionId() {\n        return transactionId;\n    }\n\n    public void setTransactionId(String transactionId) {\n        this.transactionId = transactionId;\n    }\n\n    @Override\n    public String toString() {\n        return \"Message{\" +\n            \"topic='\" + topic + '\\'' +\n            \", flag=\" + flag +\n            \", properties=\" + properties +\n            \", body=\" + Arrays.toString(body) +\n            \", transactionId='\" + transactionId + '\\'' +\n            '}';\n    }\n\n    public void setDelayTimeSec(long sec) {\n        this.putProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC, String.valueOf(sec));\n    }\n\n    public long getDelayTimeSec() {\n        String t = this.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC);\n        if (t != null) {\n            return Long.parseLong(t);\n        }\n        return 0;\n    }\n\n    public void setDelayTimeMs(long timeMs) {\n        this.putProperty(MessageConst.PROPERTY_TIMER_DELAY_MS, String.valueOf(timeMs));\n    }\n\n    public long getDelayTimeMs() {\n        String t = this.getProperty(MessageConst.PROPERTY_TIMER_DELAY_MS);\n        if (t != null) {\n            return Long.parseLong(t);\n        }\n        return 0;\n    }\n\n    public void setDeliverTimeMs(long timeMs) {\n        this.putProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS, String.valueOf(timeMs));\n    }\n\n    public long getDeliverTimeMs() {\n        String t = this.getProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS);\n        if (t != null) {\n            return Long.parseLong(t);\n        }\n        return 0;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/message/MessageAccessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.message;\n\nimport java.util.HashMap;\nimport java.util.Map;\n\npublic class MessageAccessor {\n\n    public static void clearProperty(final Message msg, final String name) {\n        msg.clearProperty(name);\n    }\n\n    public static void setProperties(final Message msg, Map<String, String> properties) {\n        msg.setProperties(properties);\n    }\n\n    public static void setTransferFlag(final Message msg, String unit) {\n        putProperty(msg, MessageConst.PROPERTY_TRANSFER_FLAG, unit);\n    }\n\n    public static void putProperty(final Message msg, final String name, final String value) {\n        msg.putProperty(name, value);\n    }\n\n    public static String getTransferFlag(final Message msg) {\n        return msg.getProperty(MessageConst.PROPERTY_TRANSFER_FLAG);\n    }\n\n    public static void setCorrectionFlag(final Message msg, String unit) {\n        putProperty(msg, MessageConst.PROPERTY_CORRECTION_FLAG, unit);\n    }\n\n    public static String getCorrectionFlag(final Message msg) {\n        return msg.getProperty(MessageConst.PROPERTY_CORRECTION_FLAG);\n    }\n\n    public static void setOriginMessageId(final Message msg, String originMessageId) {\n        putProperty(msg, MessageConst.PROPERTY_ORIGIN_MESSAGE_ID, originMessageId);\n    }\n\n    public static String getOriginMessageId(final Message msg) {\n        return msg.getProperty(MessageConst.PROPERTY_ORIGIN_MESSAGE_ID);\n    }\n\n    public static void setMQ2Flag(final Message msg, String flag) {\n        putProperty(msg, MessageConst.PROPERTY_MQ2_FLAG, flag);\n    }\n\n    public static String getMQ2Flag(final Message msg) {\n        return msg.getProperty(MessageConst.PROPERTY_MQ2_FLAG);\n    }\n\n    public static void setReconsumeTime(final Message msg, String reconsumeTimes) {\n        putProperty(msg, MessageConst.PROPERTY_RECONSUME_TIME, reconsumeTimes);\n    }\n\n    public static String getReconsumeTime(final Message msg) {\n        return msg.getProperty(MessageConst.PROPERTY_RECONSUME_TIME);\n    }\n\n    public static void setMaxReconsumeTimes(final Message msg, String maxReconsumeTimes) {\n        putProperty(msg, MessageConst.PROPERTY_MAX_RECONSUME_TIMES, maxReconsumeTimes);\n    }\n\n    public static String getMaxReconsumeTimes(final Message msg) {\n        return msg.getProperty(MessageConst.PROPERTY_MAX_RECONSUME_TIMES);\n    }\n\n    public static void setConsumeStartTimeStamp(final Message msg, String propertyConsumeStartTimeStamp) {\n        putProperty(msg, MessageConst.PROPERTY_CONSUME_START_TIMESTAMP, propertyConsumeStartTimeStamp);\n    }\n\n    public static String getConsumeStartTimeStamp(final Message msg) {\n        return msg.getProperty(MessageConst.PROPERTY_CONSUME_START_TIMESTAMP);\n    }\n\n    public static void setLiteTopic(final Message msg, String liteTopic) {\n        MessageAccessor.putProperty(msg, MessageConst.PROPERTY_LITE_TOPIC, liteTopic);\n    }\n\n    public static Message cloneMessage(final Message msg) {\n        Message newMsg = new Message(msg.getTopic(), msg.getBody());\n        newMsg.setFlag(msg.getFlag());\n        newMsg.setProperties(msg.getProperties());\n        return newMsg;\n    }\n\n    public static Map<String, String> deepCopyProperties(Map<String, String> properties) {\n        if (properties == null) {\n            return null;\n        }\n        return new HashMap<>(properties);\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/message/MessageBatch.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.message;\n\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.Iterator;\nimport java.util.List;\nimport org.apache.rocketmq.common.MixAll;\n\npublic class MessageBatch extends Message implements Iterable<Message> {\n\n    private static final long serialVersionUID = 621335151046335557L;\n    private final List<Message> messages;\n\n    public MessageBatch(List<Message> messages) {\n        this.messages = messages;\n    }\n\n    public byte[] encode() {\n        return MessageDecoder.encodeMessages(messages);\n    }\n\n    public Iterator<Message> iterator() {\n        return messages.iterator();\n    }\n\n    public static MessageBatch generateFromList(Collection<? extends Message> messages) {\n        assert messages != null;\n        assert messages.size() > 0;\n        List<Message> messageList = new ArrayList<>(messages.size());\n        Message first = null;\n        for (Message message : messages) {\n            if (message.getDelayTimeLevel() > 0 || message.getDelayTimeMs() > 0 || message.getDelayTimeSec() > 0 || message.getDeliverTimeMs() > 0) {\n                throw new UnsupportedOperationException(\"Delayed messages are not supported for batching\");\n            }\n            if (message.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n                throw new UnsupportedOperationException(\"Retry Group is not supported for batching\");\n            }\n            if (first == null) {\n                first = message;\n            } else {\n                if (!first.getTopic().equals(message.getTopic())) {\n                    throw new UnsupportedOperationException(\"The topic of the messages in one batch should be the same\");\n                }\n                if (first.isWaitStoreMsgOK() != message.isWaitStoreMsgOK()) {\n                    throw new UnsupportedOperationException(\"The waitStoreMsgOK of the messages in one batch should the same\");\n                }\n            }\n            messageList.add(message);\n        }\n        MessageBatch messageBatch = new MessageBatch(messageList);\n\n        messageBatch.setTopic(first.getTopic());\n        messageBatch.setWaitStoreMsgOK(first.isWaitStoreMsgOK());\n        return messageBatch;\n    }\n\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/message/MessageClientExt.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.message;\n\npublic class MessageClientExt extends MessageExt {\n\n    public String getOffsetMsgId() {\n        return super.getMsgId();\n    }\n\n    public void setOffsetMsgId(String offsetMsgId) {\n        super.setMsgId(offsetMsgId);\n    }\n\n    @Override\n    public String getMsgId() {\n        String uniqID = MessageClientIDSetter.getUniqID(this);\n        if (uniqID == null) {\n            return this.getOffsetMsgId();\n        } else {\n            return uniqID;\n        }\n    }\n\n    public void setMsgId(String msgId) {\n        //DO NOTHING\n        //MessageClientIDSetter.setUniqID(this);\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/message/MessageClientIDSetter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.message;\n\nimport java.nio.ByteBuffer;\nimport java.util.Calendar;\nimport java.util.Date;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.rocketmq.common.UtilAll;\n\npublic class MessageClientIDSetter {\n    \n    private static final int LEN;\n    private static final char[] FIX_STRING;\n    private static final AtomicInteger COUNTER;\n    private static long startTime;\n    private static long nextStartTime;\n\n    static {\n        byte[] ip;\n        try {\n            ip = UtilAll.getIP();\n        } catch (Exception e) {\n            ip = createFakeIP();\n        }\n        LEN = ip.length + 2 + 4 + 4 + 2;\n        ByteBuffer tempBuffer = ByteBuffer.allocate(ip.length + 2 + 4);\n        tempBuffer.put(ip);\n        tempBuffer.putShort((short) UtilAll.getPid());\n        tempBuffer.putInt(MessageClientIDSetter.class.getClassLoader().hashCode());\n        FIX_STRING = UtilAll.bytes2string(tempBuffer.array()).toCharArray();\n        setStartTime(System.currentTimeMillis());\n        COUNTER = new AtomicInteger(0);\n    }\n\n    private synchronized static void setStartTime(long millis) {\n        Calendar cal = Calendar.getInstance();\n        cal.setTimeInMillis(millis);\n        cal.set(Calendar.DAY_OF_MONTH, 1);\n        cal.set(Calendar.HOUR_OF_DAY, 0);\n        cal.set(Calendar.MINUTE, 0);\n        cal.set(Calendar.SECOND, 0);\n        cal.set(Calendar.MILLISECOND, 0);\n        startTime = cal.getTimeInMillis();\n        cal.add(Calendar.MONTH, 1);\n        nextStartTime = cal.getTimeInMillis();\n    }\n\n    public static Date getNearlyTimeFromID(String msgID) {\n        ByteBuffer buf = ByteBuffer.allocate(8);\n        byte[] bytes = UtilAll.string2bytes(msgID);\n        int ipLength = bytes.length == 28 ? 16 : 4;\n        buf.put((byte) 0);\n        buf.put((byte) 0);\n        buf.put((byte) 0);\n        buf.put((byte) 0);\n        buf.put(bytes, ipLength + 2 + 4, 4);\n        buf.position(0);\n        long spanMS = buf.getLong();\n        Calendar cal = Calendar.getInstance();\n        long now = cal.getTimeInMillis();\n        cal.set(Calendar.DAY_OF_MONTH, 1);\n        cal.set(Calendar.HOUR_OF_DAY, 0);\n        cal.set(Calendar.MINUTE, 0);\n        cal.set(Calendar.SECOND, 0);\n        cal.set(Calendar.MILLISECOND, 0);\n        long monStartTime = cal.getTimeInMillis();\n        if (monStartTime + spanMS >= now) {\n            cal.add(Calendar.MONTH, -1);\n            monStartTime = cal.getTimeInMillis();\n        }\n        cal.setTimeInMillis(monStartTime + spanMS);\n        return cal.getTime();\n    }\n\n    public static String getIPStrFromID(String msgID) {\n        byte[] ipBytes = getIPFromID(msgID);\n        if (ipBytes.length == 16) {\n            return UtilAll.ipToIPv6Str(ipBytes);\n        } else {\n            return UtilAll.ipToIPv4Str(ipBytes);\n        }\n    }\n\n    public static byte[] getIPFromID(String msgID) {\n        byte[] bytes = UtilAll.string2bytes(msgID);\n        int ipLength = bytes.length == 28 ? 16 : 4;\n        byte[] result = new byte[ipLength];\n        System.arraycopy(bytes, 0, result, 0, ipLength);\n        return result;\n    }\n\n    public static int getPidFromID(String msgID) {\n        byte[] bytes = UtilAll.string2bytes(msgID);\n        ByteBuffer wrap = ByteBuffer.wrap(bytes);\n        int value = wrap.getShort(bytes.length - 2 - 4 - 4 - 2);\n        return value & 0x0000FFFF;\n    }\n\n    public static String createUniqID() {\n        char[] sb = new char[LEN * 2];\n        System.arraycopy(FIX_STRING, 0, sb, 0, FIX_STRING.length);\n        long current = System.currentTimeMillis();\n        if (current >= nextStartTime) {\n            setStartTime(current);\n        }\n        int diff = (int)(current - startTime);\n        if (diff < 0 && diff > -1000_000) {\n            // may cause by NTP\n            diff = 0;\n        }\n        int pos = FIX_STRING.length;\n        UtilAll.writeInt(sb, pos, diff);\n        pos += 8;\n        UtilAll.writeShort(sb, pos, COUNTER.getAndIncrement());\n        return new String(sb);\n    }\n\n    public static void setUniqID(final Message msg) {\n        if (msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX) == null) {\n            msg.putProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, createUniqID());\n        }\n    }\n\n    public static String getUniqID(final Message msg) {\n        return msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX);\n    }\n\n    public static byte[] createFakeIP() {\n        ByteBuffer bb = ByteBuffer.allocate(8);\n        bb.putLong(System.currentTimeMillis());\n        bb.position(4);\n        byte[] fakeIP = new byte[4];\n        bb.get(fakeIP);\n        return fakeIP;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.message;\n\nimport java.util.HashSet;\n\npublic class MessageConst {\n    public static final String PROPERTY_KEYS = \"KEYS\";\n    public static final String PROPERTY_TAGS = \"TAGS\";\n    public static final String PROPERTY_WAIT_STORE_MSG_OK = \"WAIT\";\n    public static final String PROPERTY_DELAY_TIME_LEVEL = \"DELAY\";\n    public static final String PROPERTY_RETRY_TOPIC = \"RETRY_TOPIC\";\n    public static final String PROPERTY_ORIGIN_GROUP = \"ORIGIN_GROUP\";\n    public static final String PROPERTY_REAL_TOPIC = \"REAL_TOPIC\";\n    public static final String PROPERTY_REAL_QUEUE_ID = \"REAL_QID\";\n    public static final String PROPERTY_TRANSACTION_PREPARED = \"TRAN_MSG\";\n    public static final String PROPERTY_PRODUCER_GROUP = \"PGROUP\";\n    public static final String PROPERTY_MIN_OFFSET = \"MIN_OFFSET\";\n    public static final String PROPERTY_MAX_OFFSET = \"MAX_OFFSET\";\n    public static final String PROPERTY_BUYER_ID = \"BUYER_ID\";\n    public static final String PROPERTY_ORIGIN_MESSAGE_ID = \"ORIGIN_MESSAGE_ID\";\n    public static final String PROPERTY_TRANSFER_FLAG = \"TRANSFER_FLAG\";\n    public static final String PROPERTY_CORRECTION_FLAG = \"CORRECTION_FLAG\";\n    public static final String PROPERTY_MQ2_FLAG = \"MQ2_FLAG\";\n    public static final String PROPERTY_RECONSUME_TIME = \"RECONSUME_TIME\";\n    public static final String PROPERTY_MSG_REGION = \"MSG_REGION\";\n    public static final String PROPERTY_TRACE_SWITCH = \"TRACE_ON\";\n    public static final String PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX = \"UNIQ_KEY\";\n    public static final String PROPERTY_TRANS_OFFSET = \"TRANS_OFFSET\";\n    public static final String PROPERTY_EXTEND_UNIQ_INFO = \"EXTEND_UNIQ_INFO\";\n    public static final String PROPERTY_MAX_RECONSUME_TIMES = \"MAX_RECONSUME_TIMES\";\n    public static final String PROPERTY_CONSUME_START_TIMESTAMP = \"CONSUME_START_TIME\";\n    public static final String PROPERTY_PRIORITY = \"_SYS_MSG_PRIORITY_\";\n    public static final String PROPERTY_INNER_NUM = \"INNER_NUM\";\n    public static final String PROPERTY_INNER_BASE = \"INNER_BASE\";\n    public static final String DUP_INFO = \"DUP_INFO\";\n    public static final String PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS = \"CHECK_IMMUNITY_TIME_IN_SECONDS\";\n    public static final String PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET = \"TRAN_PREPARED_QUEUE_OFFSET\";\n    public static final String PROPERTY_TRANSACTION_ID = \"__transactionId__\";\n    public static final String PROPERTY_TRANSACTION_CHECK_TIMES = \"TRANSACTION_CHECK_TIMES\";\n    public static final String PROPERTY_INSTANCE_ID = \"INSTANCE_ID\";\n    public static final String PROPERTY_CORRELATION_ID = \"CORRELATION_ID\";\n    public static final String PROPERTY_MESSAGE_REPLY_TO_CLIENT = \"REPLY_TO_CLIENT\";\n    public static final String PROPERTY_MESSAGE_TTL = \"TTL\";\n    public static final String PROPERTY_REPLY_MESSAGE_ARRIVE_TIME = \"ARRIVE_TIME\";\n    public static final String PROPERTY_PUSH_REPLY_TIME = \"PUSH_REPLY_TIME\";\n    public static final String PROPERTY_CLUSTER = \"CLUSTER\";\n    public static final String PROPERTY_MESSAGE_TYPE = \"MSG_TYPE\";\n    public static final String PROPERTY_POP_CK = \"POP_CK\";\n    public static final String PROPERTY_POP_CK_OFFSET = \"POP_CK_OFFSET\";\n    public static final String PROPERTY_FIRST_POP_TIME = \"1ST_POP_TIME\";\n    public static final String PROPERTY_SHARDING_KEY = \"__SHARDINGKEY\";\n    public static final String PROPERTY_LITE_TOPIC = \"__LITE_TOPIC\";\n    public static final String PROPERTY_FORWARD_QUEUE_ID = \"PROPERTY_FORWARD_QUEUE_ID\";\n    public static final String PROPERTY_REDIRECT = \"REDIRECT\";\n    public static final String PROPERTY_INNER_MULTI_DISPATCH = \"INNER_MULTI_DISPATCH\";\n    public static final String PROPERTY_INNER_MULTI_QUEUE_OFFSET = \"INNER_MULTI_QUEUE_OFFSET\";\n    public static final String PROPERTY_TRACE_CONTEXT = \"TRACE_CONTEXT\";\n    public static final String PROPERTY_TIMER_DELAY_SEC = \"TIMER_DELAY_SEC\";\n    public static final String PROPERTY_TIMER_DELIVER_MS = \"TIMER_DELIVER_MS\";\n    public static final String PROPERTY_BORN_HOST = \"__BORNHOST\";\n    public static final String PROPERTY_BORN_TIMESTAMP = \"BORN_TIMESTAMP\";\n\n    /**\n     * property which name starts with \"__RMQ.TRANSIENT.\" is called transient one that will not stored in broker disks.\n     */\n    public static final String PROPERTY_TRANSIENT_PREFIX = \"__RMQ.TRANSIENT.\";\n\n    /**\n     * the transient property key of topicSysFlag (set by client when pulling messages)\n     */\n    public static final String PROPERTY_TRANSIENT_TOPIC_CONFIG = PROPERTY_TRANSIENT_PREFIX + \"TOPIC_SYS_FLAG\";\n\n    /**\n     * the transient property key of groupSysFlag (set by client when pulling messages)\n     */\n    public static final String PROPERTY_TRANSIENT_GROUP_CONFIG = PROPERTY_TRANSIENT_PREFIX + \"GROUP_SYS_FLAG\";\n\n    public static final String KEY_SEPARATOR = \" \";\n\n    public final static String INDEX_KEY_TYPE = \"K\";\n    public final static String INDEX_UNIQUE_TYPE = \"U\";\n    public final static String INDEX_TAG_TYPE = \"T\";\n\n    public final static String TIMER_ENGINE_TYPE = \"E_T\";\n    public final static String TIMER_ENGINE_ROCKSDB_TIMELINE = \"R\";\n    public final static String TIMER_ENGINE_FILE_TIME_WHEEL = \"F\";\n\n    public static final HashSet<String> STRING_HASH_SET = new HashSet<>(64);\n\n    public static final String PROPERTY_TIMER_ENQUEUE_MS = \"TIMER_ENQUEUE_MS\";\n    public static final String PROPERTY_TIMER_DEQUEUE_MS = \"TIMER_DEQUEUE_MS\";\n    public static final String PROPERTY_TIMER_ROLL_TIMES = \"TIMER_ROLL_TIMES\";\n    public static final String PROPERTY_TIMER_OUT_MS = \"TIMER_OUT_MS\";\n    public static final String PROPERTY_TIMER_DEL_UNIQKEY = \"TIMER_DEL_UNIQKEY\";\n    public static final String PROPERTY_TIMER_ROLL_LABEL = \"TIMER_ROLL_LABEL\";\n    public static final String PROPERTY_TIMER_DELAY_LEVEL = \"TIMER_DELAY_LEVEL\";\n    public static final String PROPERTY_TIMER_DELAY_MS = \"TIMER_DELAY_MS\";\n    public static final String PROPERTY_CRC32 = \"__CRC32#\";\n\n    /**\n     * properties for DLQ\n     */\n    public static final String PROPERTY_DLQ_ORIGIN_TOPIC = \"DLQ_ORIGIN_TOPIC\";\n    public static final String PROPERTY_DLQ_ORIGIN_MESSAGE_ID = \"DLQ_ORIGIN_MESSAGE_ID\";\n\n    static {\n        STRING_HASH_SET.add(PROPERTY_TRACE_SWITCH);\n        STRING_HASH_SET.add(PROPERTY_MSG_REGION);\n        STRING_HASH_SET.add(PROPERTY_KEYS);\n        STRING_HASH_SET.add(PROPERTY_TAGS);\n        STRING_HASH_SET.add(PROPERTY_WAIT_STORE_MSG_OK);\n        STRING_HASH_SET.add(PROPERTY_DELAY_TIME_LEVEL);\n        STRING_HASH_SET.add(PROPERTY_RETRY_TOPIC);\n        STRING_HASH_SET.add(PROPERTY_ORIGIN_GROUP);\n        STRING_HASH_SET.add(PROPERTY_REAL_TOPIC);\n        STRING_HASH_SET.add(PROPERTY_REAL_QUEUE_ID);\n        STRING_HASH_SET.add(PROPERTY_TRANSACTION_PREPARED);\n        STRING_HASH_SET.add(PROPERTY_PRODUCER_GROUP);\n        STRING_HASH_SET.add(PROPERTY_MIN_OFFSET);\n        STRING_HASH_SET.add(PROPERTY_MAX_OFFSET);\n        STRING_HASH_SET.add(PROPERTY_BUYER_ID);\n        STRING_HASH_SET.add(PROPERTY_ORIGIN_MESSAGE_ID);\n        STRING_HASH_SET.add(PROPERTY_TRANSFER_FLAG);\n        STRING_HASH_SET.add(PROPERTY_CORRECTION_FLAG);\n        STRING_HASH_SET.add(PROPERTY_MQ2_FLAG);\n        STRING_HASH_SET.add(PROPERTY_RECONSUME_TIME);\n        STRING_HASH_SET.add(PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX);\n        STRING_HASH_SET.add(PROPERTY_MAX_RECONSUME_TIMES);\n        STRING_HASH_SET.add(PROPERTY_CONSUME_START_TIMESTAMP);\n        STRING_HASH_SET.add(PROPERTY_POP_CK);\n        STRING_HASH_SET.add(PROPERTY_POP_CK_OFFSET);\n        STRING_HASH_SET.add(PROPERTY_FIRST_POP_TIME);\n        STRING_HASH_SET.add(PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET);\n        STRING_HASH_SET.add(DUP_INFO);\n        STRING_HASH_SET.add(PROPERTY_EXTEND_UNIQ_INFO);\n        STRING_HASH_SET.add(PROPERTY_INSTANCE_ID);\n        STRING_HASH_SET.add(PROPERTY_CORRELATION_ID);\n        STRING_HASH_SET.add(PROPERTY_MESSAGE_REPLY_TO_CLIENT);\n        STRING_HASH_SET.add(PROPERTY_MESSAGE_TTL);\n        STRING_HASH_SET.add(PROPERTY_REPLY_MESSAGE_ARRIVE_TIME);\n        STRING_HASH_SET.add(PROPERTY_PUSH_REPLY_TIME);\n        STRING_HASH_SET.add(PROPERTY_CLUSTER);\n        STRING_HASH_SET.add(PROPERTY_MESSAGE_TYPE);\n        STRING_HASH_SET.add(PROPERTY_INNER_MULTI_QUEUE_OFFSET);\n        STRING_HASH_SET.add(PROPERTY_TIMER_DELAY_MS);\n        STRING_HASH_SET.add(PROPERTY_TIMER_DELAY_SEC);\n        STRING_HASH_SET.add(PROPERTY_TIMER_DELIVER_MS);\n        STRING_HASH_SET.add(PROPERTY_TIMER_ENQUEUE_MS);\n        STRING_HASH_SET.add(PROPERTY_TIMER_DEQUEUE_MS);\n        STRING_HASH_SET.add(PROPERTY_TIMER_ROLL_TIMES);\n        STRING_HASH_SET.add(PROPERTY_TIMER_OUT_MS);\n        STRING_HASH_SET.add(PROPERTY_TIMER_DEL_UNIQKEY);\n        STRING_HASH_SET.add(PROPERTY_TIMER_DELAY_LEVEL);\n        STRING_HASH_SET.add(PROPERTY_BORN_HOST);\n        STRING_HASH_SET.add(PROPERTY_BORN_TIMESTAMP);\n        STRING_HASH_SET.add(PROPERTY_DLQ_ORIGIN_TOPIC);\n        STRING_HASH_SET.add(PROPERTY_DLQ_ORIGIN_MESSAGE_ID);\n        STRING_HASH_SET.add(PROPERTY_CRC32);\n        STRING_HASH_SET.add(PROPERTY_PRIORITY);\n        STRING_HASH_SET.add(PROPERTY_LITE_TOPIC);\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.message;\n\nimport io.netty.buffer.ByteBuf;\nimport java.io.IOException;\nimport java.net.Inet4Address;\nimport java.net.InetAddress;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.net.UnknownHostException;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.Charset;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.compression.Compressor;\nimport org.apache.rocketmq.common.compression.CompressorFactory;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\n\npublic class MessageDecoder {\n//    public final static int MSG_ID_LENGTH = 8 + 8;\n\n    public final static Charset CHARSET_UTF8 = StandardCharsets.UTF_8;\n    public final static int MESSAGE_MAGIC_CODE_POSITION = 4;\n    public final static int MESSAGE_FLAG_POSITION = 16;\n    public final static int MESSAGE_PHYSIC_OFFSET_POSITION = 28;\n    public final static int MESSAGE_STORE_TIMESTAMP_POSITION = 56;\n\n    // Set message magic code v2 if topic length > 127\n    public final static int MESSAGE_MAGIC_CODE = -626843481;\n    public final static int MESSAGE_MAGIC_CODE_V2 = -626843477;\n\n    // End of file empty MAGIC CODE cbd43194\n    public final static int BLANK_MAGIC_CODE = -875286124;\n    public static final char NAME_VALUE_SEPARATOR = 1;\n    public static final char PROPERTY_SEPARATOR = 2;\n    public static final int PHY_POS_POSITION = 4 + 4 + 4 + 4 + 4 + 8;\n    public static final int QUEUE_OFFSET_POSITION = 4 + 4 + 4 + 4 + 4;\n    public static final int SYSFLAG_POSITION = 4 + 4 + 4 + 4 + 4 + 8 + 8;\n//    public static final int BODY_SIZE_POSITION = 4 // 1 TOTALSIZE\n//        + 4 // 2 MAGICCODE\n//        + 4 // 3 BODYCRC\n//        + 4 // 4 QUEUEID\n//        + 4 // 5 FLAG\n//        + 8 // 6 QUEUEOFFSET\n//        + 8 // 7 PHYSICALOFFSET\n//        + 4 // 8 SYSFLAG\n//        + 8 // 9 BORNTIMESTAMP\n//        + 8 // 10 BORNHOST\n//        + 8 // 11 STORETIMESTAMP\n//        + 8 // 12 STOREHOSTADDRESS\n//        + 4 // 13 RECONSUMETIMES\n//        + 8; // 14 Prepared Transaction Offset\n\n    public static String createMessageId(final ByteBuffer input, final ByteBuffer addr, final long offset) {\n        input.flip();\n        int msgIDLength = addr.limit() == 8 ? 16 : 28;\n        input.limit(msgIDLength);\n\n        input.put(addr);\n        input.putLong(offset);\n\n        return UtilAll.bytes2string(input.array());\n    }\n\n    public static String createMessageId(SocketAddress socketAddress, long transactionIdhashCode) {\n        InetSocketAddress inetSocketAddress = (InetSocketAddress) socketAddress;\n        int msgIDLength = inetSocketAddress.getAddress() instanceof Inet4Address ? 16 : 28;\n        ByteBuffer byteBuffer = ByteBuffer.allocate(msgIDLength);\n        byteBuffer.put(inetSocketAddress.getAddress().getAddress());\n        byteBuffer.putInt(inetSocketAddress.getPort());\n        byteBuffer.putLong(transactionIdhashCode);\n        byteBuffer.flip();\n        return UtilAll.bytes2string(byteBuffer.array());\n    }\n\n    public static MessageId decodeMessageId(final String msgId) throws UnknownHostException {\n        byte[] bytes = UtilAll.string2bytes(msgId);\n        ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);\n\n        // address(ip+port)\n        byte[] ip = new byte[msgId.length() == 32 ? 4 : 16];\n        byteBuffer.get(ip);\n        int port = byteBuffer.getInt();\n        SocketAddress address = new InetSocketAddress(InetAddress.getByAddress(ip), port);\n\n        // offset\n        long offset = byteBuffer.getLong();\n\n        return new MessageId(address, offset);\n    }\n\n    /**\n     * Just decode properties from msg buffer.\n     *\n     * @param byteBuffer msg commit log buffer.\n     */\n    public static Map<String, String> decodeProperties(ByteBuffer byteBuffer) {\n        int sysFlag = byteBuffer.getInt(SYSFLAG_POSITION);\n        int magicCode = byteBuffer.getInt(MESSAGE_MAGIC_CODE_POSITION);\n        MessageVersion version = MessageVersion.valueOfMagicCode(magicCode);\n\n        int bornhostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20;\n        int storehostAddressLength = (sysFlag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 8 : 20;\n        int bodySizePosition = 4 // 1 TOTALSIZE\n            + 4 // 2 MAGICCODE\n            + 4 // 3 BODYCRC\n            + 4 // 4 QUEUEID\n            + 4 // 5 FLAG\n            + 8 // 6 QUEUEOFFSET\n            + 8 // 7 PHYSICALOFFSET\n            + 4 // 8 SYSFLAG\n            + 8 // 9 BORNTIMESTAMP\n            + bornhostLength // 10 BORNHOST\n            + 8 // 11 STORETIMESTAMP\n            + storehostAddressLength // 12 STOREHOSTADDRESS\n            + 4 // 13 RECONSUMETIMES\n            + 8; // 14 Prepared Transaction Offset\n\n        int topicLengthPosition = bodySizePosition + 4 + byteBuffer.getInt(bodySizePosition);\n        byteBuffer.position(topicLengthPosition);\n        int topicLengthSize = version.getTopicLengthSize();\n        int topicLength = version.getTopicLength(byteBuffer);\n\n        int propertiesPosition = topicLengthPosition + topicLengthSize + topicLength;\n        short propertiesLength = byteBuffer.getShort(propertiesPosition);\n        byteBuffer.position(propertiesPosition + 2);\n\n        if (propertiesLength > 0) {\n            byte[] properties = new byte[propertiesLength];\n            byteBuffer.get(properties);\n            String propertiesString = new String(properties, CHARSET_UTF8);\n            return string2messageProperties(propertiesString);\n        }\n        return null;\n    }\n\n    public static void createCrc32(final ByteBuffer input, int crc32) {\n        input.put(MessageConst.PROPERTY_CRC32.getBytes(StandardCharsets.UTF_8));\n        input.put((byte) NAME_VALUE_SEPARATOR);\n        for (int i = 0; i < 10; i++) {\n            byte b = '0';\n            if (crc32 > 0) {\n                b += (byte) (crc32 % 10);\n                crc32 /= 10;\n            }\n            input.put(b);\n        }\n        input.put((byte) PROPERTY_SEPARATOR);\n    }\n\n    public static void createCrc32(final ByteBuf input, int crc32) {\n        input.writeBytes(MessageConst.PROPERTY_CRC32.getBytes(StandardCharsets.UTF_8));\n        input.writeByte((byte) NAME_VALUE_SEPARATOR);\n        for (int i = 0; i < 10; i++) {\n            byte b = '0';\n            if (crc32 > 0) {\n                b += (byte) (crc32 % 10);\n                crc32 /= 10;\n            }\n            input.writeByte(b);\n        }\n        input.writeByte((byte) PROPERTY_SEPARATOR);\n    }\n\n    public static MessageExt decode(ByteBuffer byteBuffer) {\n        return decode(byteBuffer, true, true, false);\n    }\n\n    public static MessageExt clientDecode(ByteBuffer byteBuffer, final boolean readBody) {\n        return decode(byteBuffer, readBody, true, true);\n    }\n\n    public static MessageExt decode(ByteBuffer byteBuffer, final boolean readBody) {\n        return decode(byteBuffer, readBody, true, false);\n    }\n\n    public static byte[] encode(MessageExt messageExt, boolean needCompress) throws Exception {\n        byte[] body = messageExt.getBody();\n        byte[] topics = messageExt.getTopic().getBytes(CHARSET_UTF8);\n        byte topicLen = (byte) topics.length;\n        String properties = messageProperties2String(messageExt.getProperties());\n        byte[] propertiesBytes = properties.getBytes(CHARSET_UTF8);\n        short propertiesLength = (short) propertiesBytes.length;\n        int sysFlag = messageExt.getSysFlag();\n        int bornhostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20;\n        int storehostAddressLength = (sysFlag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 8 : 20;\n        byte[] newBody = messageExt.getBody();\n        if (needCompress && (sysFlag & MessageSysFlag.COMPRESSED_FLAG) == MessageSysFlag.COMPRESSED_FLAG) {\n            Compressor compressor = CompressorFactory.getCompressor(MessageSysFlag.getCompressionType(sysFlag));\n            newBody = compressor.compress(body, 5);\n        }\n        int bodyLength = newBody.length;\n        int storeSize = messageExt.getStoreSize();\n        ByteBuffer byteBuffer;\n        if (storeSize > 0) {\n            byteBuffer = ByteBuffer.allocate(storeSize);\n        } else {\n            storeSize = 4 // 1 TOTALSIZE\n                + 4 // 2 MAGICCODE\n                + 4 // 3 BODYCRC\n                + 4 // 4 QUEUEID\n                + 4 // 5 FLAG\n                + 8 // 6 QUEUEOFFSET\n                + 8 // 7 PHYSICALOFFSET\n                + 4 // 8 SYSFLAG\n                + 8 // 9 BORNTIMESTAMP\n                + bornhostLength // 10 BORNHOST\n                + 8 // 11 STORETIMESTAMP\n                + storehostAddressLength // 12 STOREHOSTADDRESS\n                + 4 // 13 RECONSUMETIMES\n                + 8 // 14 Prepared Transaction Offset\n                + 4 + bodyLength // 14 BODY\n                + 1 + topicLen // 15 TOPIC\n                + 2 + propertiesLength // 16 propertiesLength\n                + 0;\n            byteBuffer = ByteBuffer.allocate(storeSize);\n        }\n        // 1 TOTALSIZE\n        byteBuffer.putInt(storeSize);\n\n        // 2 MAGICCODE\n        byteBuffer.putInt(MESSAGE_MAGIC_CODE);\n\n        // 3 BODYCRC\n        int bodyCRC = messageExt.getBodyCRC();\n        byteBuffer.putInt(bodyCRC);\n\n        // 4 QUEUEID\n        int queueId = messageExt.getQueueId();\n        byteBuffer.putInt(queueId);\n\n        // 5 FLAG\n        int flag = messageExt.getFlag();\n        byteBuffer.putInt(flag);\n\n        // 6 QUEUEOFFSET\n        long queueOffset = messageExt.getQueueOffset();\n        byteBuffer.putLong(queueOffset);\n\n        // 7 PHYSICALOFFSET\n        long physicOffset = messageExt.getCommitLogOffset();\n        byteBuffer.putLong(physicOffset);\n\n        // 8 SYSFLAG\n        byteBuffer.putInt(sysFlag);\n\n        // 9 BORNTIMESTAMP\n        long bornTimeStamp = messageExt.getBornTimestamp();\n        byteBuffer.putLong(bornTimeStamp);\n\n        // 10 BORNHOST\n        InetSocketAddress bornHost = (InetSocketAddress) messageExt.getBornHost();\n        byteBuffer.put(bornHost.getAddress().getAddress());\n        byteBuffer.putInt(bornHost.getPort());\n\n        // 11 STORETIMESTAMP\n        long storeTimestamp = messageExt.getStoreTimestamp();\n        byteBuffer.putLong(storeTimestamp);\n\n        // 12 STOREHOST\n        InetSocketAddress serverHost = (InetSocketAddress) messageExt.getStoreHost();\n        byteBuffer.put(serverHost.getAddress().getAddress());\n        byteBuffer.putInt(serverHost.getPort());\n\n        // 13 RECONSUMETIMES\n        int reconsumeTimes = messageExt.getReconsumeTimes();\n        byteBuffer.putInt(reconsumeTimes);\n\n        // 14 Prepared Transaction Offset\n        long preparedTransactionOffset = messageExt.getPreparedTransactionOffset();\n        byteBuffer.putLong(preparedTransactionOffset);\n\n        // 15 BODY\n        byteBuffer.putInt(bodyLength);\n        byteBuffer.put(newBody);\n\n        // 16 TOPIC\n        byteBuffer.put(topicLen);\n        byteBuffer.put(topics);\n\n        // 17 properties\n        byteBuffer.putShort(propertiesLength);\n        byteBuffer.put(propertiesBytes);\n\n        return byteBuffer.array();\n    }\n\n    /**\n     * Encode without store timestamp and store host, skip blank msg.\n     *\n     * @param messageExt   msg\n     * @param needCompress need compress or not\n     * @return byte array\n     * @throws IOException when compress failed\n     */\n    public static byte[] encodeUniquely(MessageExt messageExt, boolean needCompress) throws IOException {\n        byte[] body = messageExt.getBody();\n        byte[] topics = messageExt.getTopic().getBytes(CHARSET_UTF8);\n        byte topicLen = (byte) topics.length;\n        String properties = messageProperties2String(messageExt.getProperties());\n        byte[] propertiesBytes = properties.getBytes(CHARSET_UTF8);\n        short propertiesLength = (short) propertiesBytes.length;\n        int sysFlag = messageExt.getSysFlag();\n        int bornhostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20;\n        byte[] newBody = messageExt.getBody();\n        if (needCompress && (sysFlag & MessageSysFlag.COMPRESSED_FLAG) == MessageSysFlag.COMPRESSED_FLAG) {\n            newBody = UtilAll.compress(body, 5);\n        }\n        int bodyLength = newBody.length;\n        int storeSize = messageExt.getStoreSize();\n        ByteBuffer byteBuffer;\n        if (storeSize > 0) {\n            byteBuffer = ByteBuffer.allocate(storeSize - 8); // except size for store timestamp\n        } else {\n            storeSize = 4 +  // 1 TOTALSIZE\n                4 +  // 2 MAGICCODE\n                4 +  // 3 BODYCRC\n                4 +  // 4 QUEUEID\n                4 +  // 5 FLAG\n                8 +  // 6 QUEUEOFFSET\n                8 +  // 7 PHYSICALOFFSET\n                4 +  // 8 SYSFLAG\n                8 +  // 9 BORNTIMESTAMP\n                bornhostLength + // 10 BORNHOST\n                4 +  // 11 RECONSUMETIMES\n                8 +  // 12 Prepared Transaction Offset\n                4 + bodyLength +  // 13 BODY\n                +1 + topicLen +  // 14 TOPIC\n                2 + propertiesLength // 15 propertiesLength\n            ;\n            byteBuffer = ByteBuffer.allocate(storeSize);\n        }\n\n        // 1 TOTALSIZE\n        byteBuffer.putInt(storeSize);\n\n        // 2 MAGICCODE\n        byteBuffer.putInt(MESSAGE_MAGIC_CODE);\n\n        // 3 BODYCRC\n        int bodyCRC = messageExt.getBodyCRC();\n        byteBuffer.putInt(bodyCRC);\n\n        // 4 QUEUEID\n        int queueId = messageExt.getQueueId();\n        byteBuffer.putInt(queueId);\n\n        // 5 FLAG\n        int flag = messageExt.getFlag();\n        byteBuffer.putInt(flag);\n\n        // 6 QUEUEOFFSET\n        long queueOffset = messageExt.getQueueOffset();\n        byteBuffer.putLong(queueOffset);\n\n        // 7 PHYSICALOFFSET\n        long physicOffset = messageExt.getCommitLogOffset();\n        byteBuffer.putLong(physicOffset);\n\n        // 8 SYSFLAG\n        byteBuffer.putInt(sysFlag);\n\n        // 9 BORNTIMESTAMP\n        long bornTimeStamp = messageExt.getBornTimestamp();\n        byteBuffer.putLong(bornTimeStamp);\n\n        // 10 BORNHOST\n        InetSocketAddress bornHost = (InetSocketAddress) messageExt.getBornHost();\n        byteBuffer.put(bornHost.getAddress().getAddress());\n        byteBuffer.putInt(bornHost.getPort());\n\n        // 11 RECONSUMETIMES\n        int reconsumeTimes = messageExt.getReconsumeTimes();\n        byteBuffer.putInt(reconsumeTimes);\n\n        // 12 Prepared Transaction Offset\n        long preparedTransactionOffset = messageExt.getPreparedTransactionOffset();\n        byteBuffer.putLong(preparedTransactionOffset);\n\n        // 13 BODY\n        byteBuffer.putInt(bodyLength);\n        byteBuffer.put(newBody);\n\n        // 14 TOPIC\n        byteBuffer.put(topicLen);\n        byteBuffer.put(topics);\n\n        // 15 properties\n        byteBuffer.putShort(propertiesLength);\n        byteBuffer.put(propertiesBytes);\n\n        return byteBuffer.array();\n    }\n\n    public static MessageExt decode(\n        ByteBuffer byteBuffer, final boolean readBody, final boolean deCompressBody) {\n        return decode(byteBuffer, readBody, deCompressBody, false);\n    }\n\n    public static MessageExt decode(\n        java.nio.ByteBuffer byteBuffer, final boolean readBody, final boolean deCompressBody, final boolean isClient) {\n        return decode(byteBuffer, readBody, deCompressBody, isClient, false, false);\n    }\n\n    public static MessageExt decode(\n        java.nio.ByteBuffer byteBuffer, final boolean readBody, final boolean deCompressBody, final boolean isClient,\n        final boolean isSetPropertiesString) {\n        return decode(byteBuffer, readBody, deCompressBody, isClient, isSetPropertiesString, false);\n    }\n\n    public static MessageExt decode(\n        java.nio.ByteBuffer byteBuffer, final boolean readBody, final boolean deCompressBody, final boolean isClient,\n        final boolean isSetPropertiesString, final boolean checkCRC) {\n        try {\n\n            MessageExt msgExt;\n            if (isClient) {\n                msgExt = new MessageClientExt();\n            } else {\n                msgExt = new MessageExt();\n            }\n\n            // 1 TOTALSIZE\n            int storeSize = byteBuffer.getInt();\n            msgExt.setStoreSize(storeSize);\n\n            // 2 MAGICCODE\n            int magicCode = byteBuffer.getInt();\n            MessageVersion version = MessageVersion.valueOfMagicCode(magicCode);\n\n            // 3 BODYCRC\n            int bodyCRC = byteBuffer.getInt();\n            msgExt.setBodyCRC(bodyCRC);\n\n            // 4 QUEUEID\n            int queueId = byteBuffer.getInt();\n            msgExt.setQueueId(queueId);\n\n            // 5 FLAG\n            int flag = byteBuffer.getInt();\n            msgExt.setFlag(flag);\n\n            // 6 QUEUEOFFSET\n            long queueOffset = byteBuffer.getLong();\n            msgExt.setQueueOffset(queueOffset);\n\n            // 7 PHYSICALOFFSET\n            long physicOffset = byteBuffer.getLong();\n            msgExt.setCommitLogOffset(physicOffset);\n\n            // 8 SYSFLAG\n            int sysFlag = byteBuffer.getInt();\n            msgExt.setSysFlag(sysFlag);\n\n            // 9 BORNTIMESTAMP\n            long bornTimeStamp = byteBuffer.getLong();\n            msgExt.setBornTimestamp(bornTimeStamp);\n\n            // 10 BORNHOST\n            int bornhostIPLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 4 : 16;\n            byte[] bornHost = new byte[bornhostIPLength];\n            byteBuffer.get(bornHost, 0, bornhostIPLength);\n            int port = byteBuffer.getInt();\n            msgExt.setBornHost(new InetSocketAddress(InetAddress.getByAddress(bornHost), port));\n\n            // 11 STORETIMESTAMP\n            long storeTimestamp = byteBuffer.getLong();\n            msgExt.setStoreTimestamp(storeTimestamp);\n\n            // 12 STOREHOST\n            int storehostIPLength = (sysFlag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 : 16;\n            byte[] storeHost = new byte[storehostIPLength];\n            byteBuffer.get(storeHost, 0, storehostIPLength);\n            port = byteBuffer.getInt();\n            msgExt.setStoreHost(new InetSocketAddress(InetAddress.getByAddress(storeHost), port));\n\n            // 13 RECONSUMETIMES\n            int reconsumeTimes = byteBuffer.getInt();\n            msgExt.setReconsumeTimes(reconsumeTimes);\n\n            // 14 Prepared Transaction Offset\n            long preparedTransactionOffset = byteBuffer.getLong();\n            msgExt.setPreparedTransactionOffset(preparedTransactionOffset);\n\n            // 15 BODY\n            int bodyLen = byteBuffer.getInt();\n            if (bodyLen > 0) {\n                if (readBody) {\n                    byte[] body = new byte[bodyLen];\n                    byteBuffer.get(body);\n\n                    if (checkCRC) {\n                        //crc body\n                        int crc = UtilAll.crc32(body, 0, bodyLen);\n                        if (crc != bodyCRC) {\n                            throw new Exception(\"Msg crc is error!\");\n                        }\n                    }\n\n                    // inflate body\n                    if (deCompressBody && (sysFlag & MessageSysFlag.COMPRESSED_FLAG) == MessageSysFlag.COMPRESSED_FLAG) {\n                        Compressor compressor = CompressorFactory.getCompressor(MessageSysFlag.getCompressionType(sysFlag));\n                        body = compressor.decompress(body);\n                        sysFlag &= ~MessageSysFlag.COMPRESSED_FLAG;\n                    }\n\n                    msgExt.setBody(body);\n                    msgExt.setSysFlag(sysFlag);\n                } else {\n                    byteBuffer.position(byteBuffer.position() + bodyLen);\n                }\n            }\n\n            // 16 TOPIC\n            int topicLen = version.getTopicLength(byteBuffer);\n            byte[] topic = new byte[topicLen];\n            byteBuffer.get(topic);\n            msgExt.setTopic(new String(topic, CHARSET_UTF8));\n\n            // 17 properties\n            short propertiesLength = byteBuffer.getShort();\n            if (propertiesLength > 0) {\n                byte[] properties = new byte[propertiesLength];\n                byteBuffer.get(properties);\n                String propertiesString = new String(properties, CHARSET_UTF8);\n                if (!isSetPropertiesString) {\n                    Map<String, String> map = string2messageProperties(propertiesString);\n                    msgExt.setProperties(map);\n                } else {\n                    Map<String, String> map = string2messageProperties(propertiesString);\n                    map.put(\"propertiesString\", propertiesString);\n                    msgExt.setProperties(map);\n                }\n            }\n\n            int msgIDLength = storehostIPLength + 4 + 8;\n            ByteBuffer byteBufferMsgId = ByteBuffer.allocate(msgIDLength);\n            String msgId = createMessageId(byteBufferMsgId, msgExt.getStoreHostBytes(), msgExt.getCommitLogOffset());\n            msgExt.setMsgId(msgId);\n\n            if (isClient) {\n                ((MessageClientExt) msgExt).setOffsetMsgId(msgId);\n            }\n\n            return msgExt;\n        } catch (Exception e) {\n            byteBuffer.position(byteBuffer.limit());\n        }\n\n        return null;\n    }\n\n    public static List<MessageExt> decodes(ByteBuffer byteBuffer) {\n        return decodes(byteBuffer, true);\n    }\n\n    public static List<MessageExt> decodesBatch(ByteBuffer byteBuffer,\n        final boolean readBody,\n        final boolean decompressBody,\n        final boolean isClient) {\n        List<MessageExt> msgExts = new ArrayList<>();\n        while (byteBuffer.hasRemaining()) {\n            MessageExt msgExt = decode(byteBuffer, readBody, decompressBody, isClient);\n            if (null != msgExt) {\n                msgExts.add(msgExt);\n            } else {\n                break;\n            }\n        }\n        return msgExts;\n    }\n\n    public static List<MessageExt> decodes(ByteBuffer byteBuffer, final boolean readBody) {\n        List<MessageExt> msgExts = new ArrayList<>();\n        while (byteBuffer.hasRemaining()) {\n            MessageExt msgExt = clientDecode(byteBuffer, readBody);\n            if (null != msgExt) {\n                msgExts.add(msgExt);\n            } else {\n                break;\n            }\n        }\n        return msgExts;\n    }\n\n    public static String messageProperties2String(Map<String, String> properties) {\n        if (properties == null) {\n            return \"\";\n        }\n        int len = 0;\n        for (final Map.Entry<String, String> entry : properties.entrySet()) {\n            final String name = entry.getKey();\n            final String value = entry.getValue();\n            if (value == null) {\n                continue;\n            }\n            if (name != null) {\n                len += name.length();\n            }\n            len += value.length();\n            len += 2; // separator\n        }\n        StringBuilder sb = new StringBuilder(len);\n        for (final Map.Entry<String, String> entry : properties.entrySet()) {\n            final String name = entry.getKey();\n            final String value = entry.getValue();\n\n            if (value == null) {\n                continue;\n            }\n            sb.append(name);\n            sb.append(NAME_VALUE_SEPARATOR);\n            sb.append(value);\n            sb.append(PROPERTY_SEPARATOR);\n        }\n        return sb.toString();\n    }\n\n    public static Map<String, String> string2messageProperties(final String properties) {\n        Map<String, String> map = new HashMap<>(128);\n        if (properties != null) {\n            int len = properties.length();\n            int index = 0;\n            while (index < len) {\n                int newIndex = properties.indexOf(PROPERTY_SEPARATOR, index);\n                if (newIndex < 0) {\n                    newIndex = len;\n                }\n                if (newIndex - index >= 3) {\n                    int kvSepIndex = properties.indexOf(NAME_VALUE_SEPARATOR, index);\n                    if (kvSepIndex > index && kvSepIndex < newIndex - 1) {\n                        String k = properties.substring(index, kvSepIndex);\n                        String v = properties.substring(kvSepIndex + 1, newIndex);\n                        map.put(k, v);\n                    }\n                }\n                index = newIndex + 1;\n            }\n        }\n\n        return map;\n    }\n\n    public static byte[] encodeMessage(Message message) {\n        //only need flag, body, properties\n        byte[] body = message.getBody();\n        int bodyLen = body.length;\n        String properties = messageProperties2String(message.getProperties());\n        byte[] propertiesBytes = properties.getBytes(CHARSET_UTF8);\n        //note properties length must not more than Short.MAX\n        short propertiesLength = (short) propertiesBytes.length;\n        int storeSize = 4 // 1 TOTALSIZE\n            + 4 // 2 MAGICCOD\n            + 4 // 3 BODYCRC\n            + 4 // 4 FLAG\n            + 4 + bodyLen // 4 BODY\n            + 2 + propertiesLength;\n        ByteBuffer byteBuffer = ByteBuffer.allocate(storeSize);\n        // 1 TOTALSIZE\n        byteBuffer.putInt(storeSize);\n\n        // 2 MAGICCODE\n        byteBuffer.putInt(0);\n\n        // 3 BODYCRC\n        byteBuffer.putInt(0);\n\n        // 4 FLAG\n        int flag = message.getFlag();\n        byteBuffer.putInt(flag);\n\n        // 5 BODY\n        byteBuffer.putInt(bodyLen);\n        byteBuffer.put(body);\n\n        // 6 properties\n        byteBuffer.putShort(propertiesLength);\n        byteBuffer.put(propertiesBytes);\n\n        return byteBuffer.array();\n    }\n\n    public static Message decodeMessage(ByteBuffer byteBuffer) throws Exception {\n        Message message = new Message();\n\n        // 1 TOTALSIZE\n        byteBuffer.getInt();\n\n        // 2 MAGICCODE\n        byteBuffer.getInt();\n\n        // 3 BODYCRC\n        byteBuffer.getInt();\n\n        // 4 FLAG\n        int flag = byteBuffer.getInt();\n        message.setFlag(flag);\n\n        // 5 BODY\n        int bodyLen = byteBuffer.getInt();\n        byte[] body = new byte[bodyLen];\n        byteBuffer.get(body);\n        message.setBody(body);\n\n        // 6 properties\n        short propertiesLen = byteBuffer.getShort();\n        byte[] propertiesBytes = new byte[propertiesLen];\n        byteBuffer.get(propertiesBytes);\n        message.setProperties(string2messageProperties(new String(propertiesBytes, CHARSET_UTF8)));\n\n        return message;\n    }\n\n    public static byte[] encodeMessages(List<Message> messages) {\n        //TO DO refactor, accumulate in one buffer, avoid copies\n        List<byte[]> encodedMessages = new ArrayList<>(messages.size());\n        int allSize = 0;\n        for (Message message : messages) {\n            byte[] tmp = encodeMessage(message);\n            encodedMessages.add(tmp);\n            allSize += tmp.length;\n        }\n        byte[] allBytes = new byte[allSize];\n        int pos = 0;\n        for (byte[] bytes : encodedMessages) {\n            System.arraycopy(bytes, 0, allBytes, pos, bytes.length);\n            pos += bytes.length;\n        }\n        return allBytes;\n    }\n\n    public static List<Message> decodeMessages(ByteBuffer byteBuffer) throws Exception {\n        //TO DO add a callback for processing,  avoid creating lists\n        List<Message> msgs = new ArrayList<>();\n        while (byteBuffer.hasRemaining()) {\n            Message msg = decodeMessage(byteBuffer);\n            msgs.add(msg);\n        }\n        return msgs;\n    }\n\n    public static void decodeMessage(MessageExt messageExt, List<MessageExt> list) throws Exception {\n        List<Message> messages = MessageDecoder.decodeMessages(ByteBuffer.wrap(messageExt.getBody()));\n        for (int i = 0; i < messages.size(); i++) {\n            Message message = messages.get(i);\n            MessageClientExt messageClientExt = new MessageClientExt();\n            messageClientExt.setTopic(messageExt.getTopic());\n            messageClientExt.setQueueOffset(messageExt.getQueueOffset() + i);\n            messageClientExt.setQueueId(messageExt.getQueueId());\n            messageClientExt.setFlag(message.getFlag());\n            MessageAccessor.setProperties(messageClientExt, message.getProperties());\n            messageClientExt.setBody(message.getBody());\n            messageClientExt.setStoreHost(messageExt.getStoreHost());\n            messageClientExt.setBornHost(messageExt.getBornHost());\n            messageClientExt.setBornTimestamp(messageExt.getBornTimestamp());\n            messageClientExt.setStoreTimestamp(messageExt.getStoreTimestamp());\n            messageClientExt.setSysFlag(messageExt.getSysFlag());\n            messageClientExt.setCommitLogOffset(messageExt.getCommitLogOffset());\n            messageClientExt.setWaitStoreMsgOK(messageExt.isWaitStoreMsgOK());\n            list.add(messageClientExt);\n        }\n    }\n\n    public static int countInnerMsgNum(ByteBuffer buffer) {\n        int count = 0;\n        while (buffer.hasRemaining()) {\n            count++;\n            int currPos = buffer.position();\n            int size = buffer.getInt();\n            buffer.position(currPos + size);\n        }\n        return count;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/message/MessageExt.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.message;\n\nimport java.net.Inet4Address;\nimport java.net.InetAddress;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.nio.ByteBuffer;\nimport org.apache.rocketmq.common.TopicFilterType;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\n\npublic class MessageExt extends Message {\n    private static final long serialVersionUID = 5720810158625748049L;\n\n    private String brokerName;\n\n    private int queueId;\n\n    private int storeSize;\n\n    private long queueOffset;\n    private int sysFlag;\n    private long bornTimestamp;\n    private SocketAddress bornHost;\n\n    private long storeTimestamp;\n    private SocketAddress storeHost;\n    private String msgId;\n    private long commitLogOffset;\n    private int bodyCRC;\n    private int reconsumeTimes;\n\n    private long preparedTransactionOffset;\n\n    public MessageExt() {\n    }\n\n    public MessageExt(int queueId, long bornTimestamp, SocketAddress bornHost, long storeTimestamp,\n        SocketAddress storeHost, String msgId) {\n        this.queueId = queueId;\n        this.bornTimestamp = bornTimestamp;\n        this.bornHost = bornHost;\n        this.storeTimestamp = storeTimestamp;\n        this.storeHost = storeHost;\n        this.msgId = msgId;\n    }\n\n    public static TopicFilterType parseTopicFilterType(final int sysFlag) {\n        if ((sysFlag & MessageSysFlag.MULTI_TAGS_FLAG) == MessageSysFlag.MULTI_TAGS_FLAG) {\n            return TopicFilterType.MULTI_TAG;\n        }\n\n        return TopicFilterType.SINGLE_TAG;\n    }\n\n    public static ByteBuffer socketAddress2ByteBuffer(final SocketAddress socketAddress, final ByteBuffer byteBuffer) {\n        InetSocketAddress inetSocketAddress = (InetSocketAddress) socketAddress;\n        InetAddress address = inetSocketAddress.getAddress();\n        if (address instanceof Inet4Address) {\n            byteBuffer.put(inetSocketAddress.getAddress().getAddress(), 0, 4);\n        } else {\n            byteBuffer.put(inetSocketAddress.getAddress().getAddress(), 0, 16);\n        }\n        byteBuffer.putInt(inetSocketAddress.getPort());\n        byteBuffer.flip();\n        return byteBuffer;\n    }\n\n    public static ByteBuffer socketAddress2ByteBuffer(SocketAddress socketAddress) {\n        InetSocketAddress inetSocketAddress = (InetSocketAddress) socketAddress;\n        InetAddress address = inetSocketAddress.getAddress();\n        ByteBuffer byteBuffer;\n        if (address instanceof Inet4Address) {\n            byteBuffer = ByteBuffer.allocate(4 + 4);\n        } else {\n            byteBuffer = ByteBuffer.allocate(16 + 4);\n        }\n        return socketAddress2ByteBuffer(socketAddress, byteBuffer);\n    }\n\n    public ByteBuffer getBornHostBytes() {\n        return socketAddress2ByteBuffer(this.bornHost);\n    }\n\n    public ByteBuffer getBornHostBytes(ByteBuffer byteBuffer) {\n        return socketAddress2ByteBuffer(this.bornHost, byteBuffer);\n    }\n\n    public ByteBuffer getStoreHostBytes() {\n        return socketAddress2ByteBuffer(this.storeHost);\n    }\n\n    public ByteBuffer getStoreHostBytes(ByteBuffer byteBuffer) {\n        return socketAddress2ByteBuffer(this.storeHost, byteBuffer);\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public int getQueueId() {\n        return queueId;\n    }\n\n    public void setQueueId(int queueId) {\n        this.queueId = queueId;\n    }\n\n    public long getBornTimestamp() {\n        return bornTimestamp;\n    }\n\n    public void setBornTimestamp(long bornTimestamp) {\n        this.bornTimestamp = bornTimestamp;\n    }\n\n    public SocketAddress getBornHost() {\n        return bornHost;\n    }\n\n    public void setBornHost(SocketAddress bornHost) {\n        this.bornHost = bornHost;\n    }\n\n    public String getBornHostString() {\n        if (null != this.bornHost) {\n            InetAddress inetAddress = ((InetSocketAddress) this.bornHost).getAddress();\n\n            return null != inetAddress ? inetAddress.getHostAddress() : null;\n        }\n\n        return null;\n    }\n\n    public String getBornHostNameString() {\n        if (null != this.bornHost) {\n            if (bornHost instanceof InetSocketAddress) {\n                // without reverse dns lookup\n                return ((InetSocketAddress) bornHost).getHostString();\n            }\n            InetAddress inetAddress = ((InetSocketAddress) this.bornHost).getAddress();\n\n            return null != inetAddress ? inetAddress.getHostName() : null;\n        }\n\n        return null;\n    }\n\n    public long getStoreTimestamp() {\n        return storeTimestamp;\n    }\n\n    public void setStoreTimestamp(long storeTimestamp) {\n        this.storeTimestamp = storeTimestamp;\n    }\n\n    public SocketAddress getStoreHost() {\n        return storeHost;\n    }\n\n    public void setStoreHost(SocketAddress storeHost) {\n        this.storeHost = storeHost;\n    }\n\n    public String getMsgId() {\n        return msgId;\n    }\n\n    public void setMsgId(String msgId) {\n        this.msgId = msgId;\n    }\n\n    public int getSysFlag() {\n        return sysFlag;\n    }\n\n    public void setSysFlag(int sysFlag) {\n        this.sysFlag = sysFlag;\n    }\n\n    public void setStoreHostAddressV6Flag() { this.sysFlag = this.sysFlag | MessageSysFlag.STOREHOSTADDRESS_V6_FLAG; }\n\n    public void setBornHostV6Flag() { this.sysFlag = this.sysFlag | MessageSysFlag.BORNHOST_V6_FLAG; }\n\n    public int getBodyCRC() {\n        return bodyCRC;\n    }\n\n    public void setBodyCRC(int bodyCRC) {\n        this.bodyCRC = bodyCRC;\n    }\n\n    public long getQueueOffset() {\n        return queueOffset;\n    }\n\n    public void setQueueOffset(long queueOffset) {\n        this.queueOffset = queueOffset;\n    }\n\n    public long getCommitLogOffset() {\n        return commitLogOffset;\n    }\n\n    public void setCommitLogOffset(long physicOffset) {\n        this.commitLogOffset = physicOffset;\n    }\n\n    public int getStoreSize() {\n        return storeSize;\n    }\n\n    public void setStoreSize(int storeSize) {\n        this.storeSize = storeSize;\n    }\n\n    public int getReconsumeTimes() {\n        return reconsumeTimes;\n    }\n\n    public void setReconsumeTimes(int reconsumeTimes) {\n        this.reconsumeTimes = reconsumeTimes;\n    }\n\n    public long getPreparedTransactionOffset() {\n        return preparedTransactionOffset;\n    }\n\n    public void setPreparedTransactionOffset(long preparedTransactionOffset) {\n        this.preparedTransactionOffset = preparedTransactionOffset;\n    }\n\n    /**\n     *\n     * achieves topicSysFlag value from transient properties\n     *\n     * @return\n     */\n    public Integer getTopicSysFlag() {\n        String topicSysFlagString = getProperty(MessageConst.PROPERTY_TRANSIENT_TOPIC_CONFIG);\n        if (topicSysFlagString != null && topicSysFlagString.length() > 0) {\n            return Integer.valueOf(topicSysFlagString);\n        }\n        return null;\n    }\n\n    /**\n     * set topicSysFlag to transient properties, or clear it\n     *\n     * @param topicSysFlag\n     */\n    public void setTopicSysFlag(Integer topicSysFlag) {\n        if (topicSysFlag == null) {\n            clearProperty(MessageConst.PROPERTY_TRANSIENT_TOPIC_CONFIG);\n        } else {\n            putProperty(MessageConst.PROPERTY_TRANSIENT_TOPIC_CONFIG, String.valueOf(topicSysFlag));\n        }\n    }\n\n    /**\n     *\n     * achieves groupSysFlag value from transient properties\n     *\n     * @return\n     */\n    public Integer getGroupSysFlag() {\n        String groupSysFlagString = getProperty(MessageConst.PROPERTY_TRANSIENT_GROUP_CONFIG);\n        if (groupSysFlagString != null && groupSysFlagString.length() > 0) {\n            return Integer.valueOf(groupSysFlagString);\n        }\n        return null;\n    }\n\n    /**\n     *\n     * set groupSysFlag to transient properties, or clear it\n     *\n     * @param groupSysFlag\n     */\n    public void setGroupSysFlag(Integer groupSysFlag) {\n        if (groupSysFlag == null) {\n            clearProperty(MessageConst.PROPERTY_TRANSIENT_GROUP_CONFIG);\n        } else {\n            putProperty(MessageConst.PROPERTY_TRANSIENT_GROUP_CONFIG, String.valueOf(groupSysFlag));\n        }\n    }\n\n    @Override\n    public String toString() {\n        return \"MessageExt [brokerName=\" + brokerName + \", queueId=\" + queueId + \", storeSize=\" + storeSize + \", queueOffset=\" + queueOffset\n            + \", sysFlag=\" + sysFlag + \", bornTimestamp=\" + bornTimestamp + \", bornHost=\" + bornHost\n            + \", storeTimestamp=\" + storeTimestamp + \", storeHost=\" + storeHost + \", msgId=\" + msgId\n            + \", commitLogOffset=\" + commitLogOffset + \", bodyCRC=\" + bodyCRC + \", reconsumeTimes=\"\n            + reconsumeTimes + \", preparedTransactionOffset=\" + preparedTransactionOffset\n            + \", toString()=\" + super.toString() + \"]\";\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/message/MessageExtBatch.java",
    "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.message;\n\nimport java.nio.ByteBuffer;\n\npublic class MessageExtBatch extends MessageExtBrokerInner {\n\n    private static final long serialVersionUID = -2353110995348498537L;\n\n    /**\n     * Inner batch means the batch does not need to be unwrapped\n     */\n    private boolean isInnerBatch = false;\n\n    public ByteBuffer wrap() {\n        assert getBody() != null;\n        return ByteBuffer.wrap(getBody(), 0, getBody().length);\n    }\n\n    public boolean isInnerBatch() {\n        return isInnerBatch;\n    }\n\n    public void setInnerBatch(boolean innerBatch) {\n        isInnerBatch = innerBatch;\n    }\n\n    private ByteBuffer encodedBuff;\n\n    public ByteBuffer getEncodedBuff() {\n        return encodedBuff;\n    }\n\n    public void setEncodedBuff(ByteBuffer encodedBuff) {\n        this.encodedBuff = encodedBuff;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.message;\n\nimport com.google.common.base.Strings;\nimport java.nio.ByteBuffer;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicFilterType;\nimport org.apache.rocketmq.common.utils.MessageUtils;\n\npublic class MessageExtBrokerInner extends MessageExt {\n    private static final long serialVersionUID = 7256001576878700634L;\n    private String propertiesString;\n    private long tagsCode;\n\n    private ByteBuffer encodedBuff;\n\n    private volatile boolean encodeCompleted;\n\n    private MessageVersion version = MessageVersion.MESSAGE_VERSION_V1;\n\n    public ByteBuffer getEncodedBuff() {\n        return encodedBuff;\n    }\n\n    public void setEncodedBuff(ByteBuffer encodedBuff) {\n        this.encodedBuff = encodedBuff;\n    }\n\n    public static long tagsString2tagsCode(final TopicFilterType filter, final String tags) {\n        if (Strings.isNullOrEmpty(tags)) { return 0; }\n\n        return tags.hashCode();\n    }\n\n    public static long tagsString2tagsCode(final String tags) {\n        return tagsString2tagsCode(null, tags);\n    }\n\n    public String getPropertiesString() {\n        return propertiesString;\n    }\n\n    public void setPropertiesString(String propertiesString) {\n        this.propertiesString = propertiesString;\n    }\n\n\n    public void deleteProperty(String name) {\n        super.clearProperty(name);\n        if (propertiesString != null) {\n            this.setPropertiesString(MessageUtils.deleteProperty(propertiesString, name));\n        }\n    }\n\n    public long getTagsCode() {\n        return tagsCode;\n    }\n\n    public void setTagsCode(long tagsCode) {\n        this.tagsCode = tagsCode;\n    }\n\n    public MessageVersion getVersion() {\n        return version;\n    }\n\n    public void setVersion(MessageVersion version) {\n        this.version = version;\n    }\n\n    public void removeWaitStorePropertyString() {\n        if (this.getProperties().containsKey(MessageConst.PROPERTY_WAIT_STORE_MSG_OK)) {\n            // There is no need to store \"WAIT=true\", remove it from propertiesString to save 9 bytes for each message.\n            // It works for most case. In some cases msgInner.setPropertiesString invoked later and replace it.\n            String waitStoreMsgOKValue = this.getProperties().remove(MessageConst.PROPERTY_WAIT_STORE_MSG_OK);\n            this.setPropertiesString(MessageDecoder.messageProperties2String(this.getProperties()));\n            // Reput to properties, since msgInner.isWaitStoreMsgOK() will be invoked later\n            this.getProperties().put(MessageConst.PROPERTY_WAIT_STORE_MSG_OK, waitStoreMsgOKValue);\n        } else {\n            this.setPropertiesString(MessageDecoder.messageProperties2String(this.getProperties()));\n        }\n    }\n\n    public boolean isEncodeCompleted() {\n        return encodeCompleted;\n    }\n\n    public void setEncodeCompleted(boolean encodeCompleted) {\n        this.encodeCompleted = encodeCompleted;\n    }\n\n    public boolean needDispatchLMQ() {\n        return StringUtils.isNoneBlank(getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH))\n            && MixAll.topicAllowsLMQ(getTopic());\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/message/MessageId.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.message;\n\nimport java.net.SocketAddress;\n\npublic class MessageId {\n    private SocketAddress address;\n    private long offset;\n\n    public MessageId(SocketAddress address, long offset) {\n        this.address = address;\n        this.offset = offset;\n    }\n\n    public SocketAddress getAddress() {\n        return address;\n    }\n\n    public void setAddress(SocketAddress address) {\n        this.address = address;\n    }\n\n    public long getOffset() {\n        return offset;\n    }\n\n    public void setOffset(long offset) {\n        this.offset = offset;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/message/MessageQueue.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.message;\n\nimport java.io.Serializable;\n\npublic class MessageQueue implements Comparable<MessageQueue>, Serializable {\n    private static final long serialVersionUID = 6191200464116433425L;\n    private String topic;\n    private String brokerName;\n    private int queueId;\n\n    public MessageQueue() {\n\n    }\n\n    public MessageQueue(MessageQueue other) {\n        this.topic = other.topic;\n        this.brokerName = other.brokerName;\n        this.queueId = other.queueId;\n    }\n\n    public MessageQueue(String topic, String brokerName, int queueId) {\n        this.topic = topic;\n        this.brokerName = brokerName;\n        this.queueId = queueId;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public int getQueueId() {\n        return queueId;\n    }\n\n    public void setQueueId(int queueId) {\n        this.queueId = queueId;\n    }\n\n    @Override\n    public int hashCode() {\n        final int prime = 31;\n        int result = 1;\n        result = prime * result + ((brokerName == null) ? 0 : brokerName.hashCode());\n        result = prime * result + queueId;\n        result = prime * result + ((topic == null) ? 0 : topic.hashCode());\n        return result;\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj)\n            return true;\n        if (obj == null)\n            return false;\n        if (getClass() != obj.getClass())\n            return false;\n        MessageQueue other = (MessageQueue) obj;\n        if (brokerName == null) {\n            if (other.brokerName != null)\n                return false;\n        } else if (!brokerName.equals(other.brokerName))\n            return false;\n        if (queueId != other.queueId)\n            return false;\n        if (topic == null) {\n            if (other.topic != null)\n                return false;\n        } else if (!topic.equals(other.topic))\n            return false;\n        return true;\n    }\n\n    @Override\n    public String toString() {\n        return \"MessageQueue [topic=\" + topic + \", brokerName=\" + brokerName + \", queueId=\" + queueId + \"]\";\n    }\n\n    @Override\n    public int compareTo(MessageQueue o) {\n        {\n            int result = this.topic.compareTo(o.topic);\n            if (result != 0) {\n                return result;\n            }\n        }\n\n        {\n            int result = this.brokerName.compareTo(o.brokerName);\n            if (result != 0) {\n                return result;\n            }\n        }\n\n        return this.queueId - o.queueId;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/message/MessageQueueAssignment.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.message;\n\nimport java.io.Serializable;\nimport java.util.Map;\n\npublic class MessageQueueAssignment implements Serializable {\n\n    private static final long serialVersionUID = 8092600270527861645L;\n\n    private MessageQueue messageQueue;\n\n    private MessageRequestMode mode = MessageRequestMode.PULL;\n\n    private Map<String, String> attachments;\n\n    @Override\n    public int hashCode() {\n        final int prime = 31;\n        int result = 1;\n        result = prime * result + ((messageQueue == null) ? 0 : messageQueue.hashCode());\n        result = prime * result + ((mode == null) ? 0 : mode.hashCode());\n        result = prime * result + ((attachments == null) ? 0 : attachments.hashCode());\n        return result;\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj)\n            return true;\n        if (obj == null)\n            return false;\n        if (getClass() != obj.getClass())\n            return false;\n        MessageQueueAssignment other = (MessageQueueAssignment) obj;\n        return messageQueue.equals(other.messageQueue);\n    }\n\n    @Override\n    public String toString() {\n        return \"MessageQueueAssignment [MessageQueue=\" + messageQueue + \", Mode=\" + mode + \"]\";\n    }\n\n    public MessageQueue getMessageQueue() {\n        return messageQueue;\n    }\n\n    public void setMessageQueue(MessageQueue messageQueue) {\n        this.messageQueue = messageQueue;\n    }\n\n    public MessageRequestMode getMode() {\n        return mode;\n    }\n\n    public void setMode(MessageRequestMode mode) {\n        this.mode = mode;\n    }\n\n    public Map<String, String> getAttachments() {\n        return attachments;\n    }\n\n    public void setAttachments(Map<String, String> attachments) {\n        this.attachments = attachments;\n    }\n\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/message/MessageQueueForC.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.message;\n\nimport java.io.Serializable;\n\npublic class MessageQueueForC implements Comparable<MessageQueueForC>, Serializable {\n\n    private static final long serialVersionUID = 5320967846569962104L;\n    private String topic;\n    private String brokerName;\n    private int queueId;\n    private long offset;\n\n    public MessageQueueForC(String topic, String brokerName, int queueId, long offset) {\n        this.topic = topic;\n        this.brokerName = brokerName;\n        this.queueId = queueId;\n        this.offset = offset;\n    }\n\n    @Override\n    public int compareTo(MessageQueueForC o) {\n        int result = this.topic.compareTo(o.topic);\n        if (result != 0) {\n            return result;\n        }\n        result = this.brokerName.compareTo(o.brokerName);\n        if (result != 0) {\n            return result;\n        }\n        result = this.queueId - o.queueId;\n        if (result != 0) {\n            return result;\n        }\n        if ((this.offset - o.offset) > 0) {\n            return 1;\n        } else if ((this.offset - o.offset) == 0) {\n            return 0;\n        } else {\n            return -1;\n        }\n    }\n\n    @Override\n    public int hashCode() {\n        final int prime = 31;\n        int result = 1;\n        result = prime * result + ((brokerName == null) ? 0 : brokerName.hashCode());\n        result = prime * result + queueId;\n        result = prime * result + ((topic == null) ? 0 : topic.hashCode());\n        return result;\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj)\n            return true;\n        if (obj == null)\n            return false;\n        if (getClass() != obj.getClass())\n            return false;\n        MessageQueueForC other = (MessageQueueForC) obj;\n        if (brokerName == null) {\n            if (other.brokerName != null)\n                return false;\n        } else if (!brokerName.equals(other.brokerName))\n            return false;\n        if (queueId != other.queueId)\n            return false;\n        if (topic == null) {\n            if (other.topic != null)\n                return false;\n        } else if (!topic.equals(other.topic))\n            return false;\n\n        if (offset != other.offset) {\n            return false;\n        }\n        return true;\n    }\n\n    @Override\n    public String toString() {\n        return \"MessageQueueForC [topic=\" + topic + \", brokerName=\" + brokerName + \", queueId=\" + queueId\n            + \", offset=\" + offset + \"]\";\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public int getQueueId() {\n        return queueId;\n    }\n\n    public void setQueueId(int queueId) {\n        this.queueId = queueId;\n    }\n\n    public long getOffset() {\n        return offset;\n    }\n\n    public void setOffset(long offset) {\n        this.offset = offset;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/message/MessageRequestMode.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.message;\n\n/**\n * Message Request Mode\n */\npublic enum MessageRequestMode {\n\n    /**\n     * pull\n     */\n    PULL(\"PULL\"),\n\n    /**\n     * pop, consumer working in pop mode could share MessageQueue\n     */\n    POP(\"POP\");\n\n    private String name;\n\n    MessageRequestMode(String name) {\n        this.name = name;\n    }\n\n    public String getName() {\n        return name;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/message/MessageType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.message;\n\npublic enum MessageType {\n    Normal_Msg(\"Normal\"),\n    Trans_Msg_Half(\"Trans\"),\n    Trans_msg_Commit(\"TransCommit\"),\n    Delay_Msg(\"Delay\"),\n    Order_Msg(\"Order\");\n\n    private final String shortName;\n\n    MessageType(String shortName) {\n        this.shortName = shortName;\n    }\n\n    public String getShortName() {\n        return shortName;\n    }\n\n    public static MessageType getByShortName(String shortName) {\n        for (MessageType msgType : MessageType.values()) {\n            if (msgType.getShortName().equals(shortName)) {\n                return msgType;\n            }\n        }\n        return Normal_Msg;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/message/MessageVersion.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.message;\n\nimport java.nio.ByteBuffer;\n\npublic enum MessageVersion {\n\n    MESSAGE_VERSION_V1(MessageDecoder.MESSAGE_MAGIC_CODE) {\n        @Override\n        public int getTopicLengthSize() {\n            return 1;\n        }\n\n        @Override\n        public int getTopicLength(ByteBuffer buffer) {\n            return buffer.get();\n        }\n\n        @Override\n        public int getTopicLength(ByteBuffer buffer, int index) {\n            return buffer.get(index);\n        }\n\n        @Override\n        public void putTopicLength(ByteBuffer buffer, int topicLength) {\n            buffer.put((byte) topicLength);\n        }\n    },\n\n    MESSAGE_VERSION_V2(MessageDecoder.MESSAGE_MAGIC_CODE_V2) {\n        @Override\n        public int getTopicLengthSize() {\n            return 2;\n        }\n\n        @Override\n        public int getTopicLength(ByteBuffer buffer) {\n            return buffer.getShort();\n        }\n\n        @Override\n        public int getTopicLength(ByteBuffer buffer, int index) {\n            return buffer.getShort(index);\n        }\n\n        @Override\n        public void putTopicLength(ByteBuffer buffer, int topicLength) {\n            buffer.putShort((short) topicLength);\n        }\n    };\n\n    private final int magicCode;\n\n    MessageVersion(int magicCode) {\n        this.magicCode = magicCode;\n    }\n\n    public static MessageVersion valueOfMagicCode(int magicCode) {\n        for (MessageVersion version : MessageVersion.values()) {\n            if (version.getMagicCode() == magicCode) {\n                return version;\n            }\n        }\n\n        throw new IllegalArgumentException(\"Invalid magicCode \" + magicCode);\n    }\n\n    public int getMagicCode() {\n        return magicCode;\n    }\n\n    public abstract int getTopicLengthSize();\n\n    public abstract int getTopicLength(java.nio.ByteBuffer buffer);\n    public abstract int getTopicLength(java.nio.ByteBuffer buffer, int index);\n    public abstract void putTopicLength(java.nio.ByteBuffer buffer, int topicLength);\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/metrics/MetricsExporterType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.metrics;\n\n\npublic enum MetricsExporterType {\n    DISABLE(0),\n    OTLP_GRPC(1),\n    PROM(2),\n    LOG(3);\n\n    private final int value;\n\n    MetricsExporterType(int value) {\n        this.value = value;\n    }\n\n    public int getValue() {\n        return value;\n    }\n\n    public static MetricsExporterType valueOf(int value) {\n        switch (value) {\n            case 1:\n                return OTLP_GRPC;\n            case 2:\n                return PROM;\n            case 3:\n                return LOG;\n            default:\n                return DISABLE;\n        }\n    }\n\n    public boolean isEnable() {\n        return this.value > 0;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/metrics/NopLongCounter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.metrics;\n\nimport io.opentelemetry.api.common.Attributes;\nimport io.opentelemetry.api.metrics.LongCounter;\nimport io.opentelemetry.context.Context;\n\npublic class NopLongCounter implements LongCounter {\n    @Override public void add(long l) {\n\n    }\n\n    @Override public void add(long l, Attributes attributes) {\n\n    }\n\n    @Override public void add(long l, Attributes attributes, Context context) {\n\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/metrics/NopLongHistogram.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.metrics;\n\nimport io.opentelemetry.api.common.Attributes;\nimport io.opentelemetry.api.metrics.LongHistogram;\nimport io.opentelemetry.context.Context;\n\npublic class NopLongHistogram implements LongHistogram {\n    @Override public void record(long l) {\n\n    }\n\n    @Override public void record(long l, Attributes attributes) {\n\n    }\n\n    @Override public void record(long l, Attributes attributes, Context context) {\n\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/metrics/NopLongUpDownCounter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.metrics;\n\nimport io.opentelemetry.api.common.Attributes;\nimport io.opentelemetry.api.metrics.LongUpDownCounter;\nimport io.opentelemetry.context.Context;\n\npublic class NopLongUpDownCounter implements LongUpDownCounter {\n    @Override public void add(long l) {\n\n    }\n\n    @Override public void add(long l, Attributes attributes) {\n\n    }\n\n    @Override public void add(long l, Attributes attributes, Context context) {\n\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/metrics/NopObservableDoubleGauge.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.metrics;\n\nimport io.opentelemetry.api.metrics.ObservableDoubleGauge;\n\npublic class NopObservableDoubleGauge implements ObservableDoubleGauge {\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/metrics/NopObservableLongGauge.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.metrics;\n\nimport io.opentelemetry.api.metrics.ObservableLongGauge;\n\npublic class NopObservableLongGauge implements ObservableLongGauge {\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/namesrv/DefaultTopAddressing.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.namesrv;\n\nimport com.google.common.base.Strings;\nimport java.io.IOException;\nimport java.util.ArrayList;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.ServiceLoader;\nimport java.util.Map;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.help.FAQUrl;\nimport org.apache.rocketmq.common.utils.HttpTinyClient;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class DefaultTopAddressing implements TopAddressing {\n\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);\n\n    private String nsAddr;\n    private String wsAddr;\n    private String unitName;\n    private Map<String, String> para;\n    private List<TopAddressing> topAddressingList;\n\n    public DefaultTopAddressing(final String wsAddr) {\n        this(wsAddr, null);\n    }\n\n    public DefaultTopAddressing(final String wsAddr, final String unitName) {\n        this.wsAddr = wsAddr;\n        this.unitName = unitName;\n        this.topAddressingList = loadCustomTopAddressing();\n    }\n\n    public DefaultTopAddressing(final String unitName, final Map<String, String> para, final String wsAddr) {\n        this.wsAddr = wsAddr;\n        this.unitName = unitName;\n        this.para = para;\n        this.topAddressingList = loadCustomTopAddressing();\n    }\n\n    private static String clearNewLine(final String str) {\n        String newString = str.trim();\n        int index = newString.indexOf(\"\\r\");\n        if (index != -1) {\n            return newString.substring(0, index);\n        }\n\n        index = newString.indexOf(\"\\n\");\n        if (index != -1) {\n            return newString.substring(0, index);\n        }\n\n        return newString;\n    }\n\n    private List<TopAddressing> loadCustomTopAddressing() {\n        ServiceLoader<TopAddressing> serviceLoader = ServiceLoader.load(TopAddressing.class);\n        Iterator<TopAddressing> iterator = serviceLoader.iterator();\n        List<TopAddressing> topAddressingList = new ArrayList<>();\n        if (iterator.hasNext()) {\n            topAddressingList.add(iterator.next());\n        }\n        return topAddressingList;\n    }\n\n    @Override\n    public final String fetchNSAddr() {\n        if (!topAddressingList.isEmpty()) {\n            for (TopAddressing topAddressing : topAddressingList) {\n                String nsAddress = topAddressing.fetchNSAddr();\n                if (!Strings.isNullOrEmpty(nsAddress)) {\n                    return nsAddress;\n                }\n            }\n        }\n        // Return result of default implementation\n        return fetchNSAddr(true, 3000);\n    }\n\n    @Override\n    public void registerChangeCallBack(NameServerUpdateCallback changeCallBack) {\n        if (!topAddressingList.isEmpty()) {\n            for (TopAddressing topAddressing : topAddressingList) {\n                topAddressing.registerChangeCallBack(changeCallBack);\n            }\n        }\n    }\n\n    public final String fetchNSAddr(boolean verbose, long timeoutMills) {\n        StringBuilder url = new StringBuilder(this.wsAddr);\n        try {\n            if (null != para && para.size() > 0) {\n                if (!UtilAll.isBlank(this.unitName)) {\n                    url.append(\"-\").append(this.unitName).append(\"?nofix=1&\");\n                }\n                else {\n                    url.append(\"?\");\n                }\n                for (Map.Entry<String, String> entry : this.para.entrySet()) {\n                    url.append(entry.getKey()).append(\"=\").append(entry.getValue()).append(\"&\");\n                }\n                url = new StringBuilder(url.substring(0, url.length() - 1));\n            }\n            else {\n                if (!UtilAll.isBlank(this.unitName)) {\n                    url.append(\"-\").append(this.unitName).append(\"?nofix=1\");\n                }\n            }\n\n            HttpTinyClient.HttpResult result = HttpTinyClient.httpGet(url.toString(), null, null, \"UTF-8\", timeoutMills);\n            if (200 == result.code) {\n                String responseStr = result.content;\n                if (responseStr != null) {\n                    return clearNewLine(responseStr);\n                } else {\n                    LOGGER.error(\"fetch nameserver address is null\");\n                }\n            } else {\n                LOGGER.error(\"fetch nameserver address failed. statusCode=\" + result.code);\n            }\n        } catch (IOException e) {\n            if (verbose) {\n                LOGGER.error(\"fetch name server address exception\", e);\n            }\n        }\n\n        if (verbose) {\n            String errorMsg =\n                \"connect to \" + url + \" failed, maybe the domain name \" + MixAll.getWSAddr() + \" not bind in /etc/hosts\";\n            errorMsg += FAQUrl.suggestTodo(FAQUrl.NAME_SERVER_ADDR_NOT_EXIST_URL);\n\n            LOGGER.warn(errorMsg);\n        }\n        return null;\n    }\n\n    public String getNsAddr() {\n        return nsAddr;\n    }\n\n    public void setNsAddr(String nsAddr) {\n        this.nsAddr = nsAddr;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/namesrv/NameServerUpdateCallback.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.namesrv;\n\npublic interface NameServerUpdateCallback {\n    String onNameServerAddressChange(String namesrvAddress);\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/namesrv/NamesrvConfig.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: NamesrvConfig.java 1839 2013-05-16 02:12:02Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.common.namesrv;\n\nimport java.io.File;\nimport org.apache.rocketmq.common.MixAll;\n\npublic class NamesrvConfig {\n\n    private String rocketmqHome = MixAll.ROCKETMQ_HOME_DIR;\n    private String kvConfigPath = System.getProperty(\"user.home\") + File.separator + \"namesrv\" + File.separator + \"kvConfig.json\";\n    private String configStorePath = System.getProperty(\"user.home\") + File.separator + \"namesrv\" + File.separator + \"namesrv.properties\";\n    private String productEnvName = \"center\";\n    private boolean clusterTest = false;\n    private boolean orderMessageEnable = false;\n    private boolean returnOrderTopicConfigToBroker = true;\n\n    /**\n     * Indicates the nums of thread to handle client requests, like GET_ROUTEINTO_BY_TOPIC.\n     */\n    private int clientRequestThreadPoolNums = 8;\n    /**\n     * Indicates the nums of thread to handle broker or operation requests, like REGISTER_BROKER.\n     */\n    private int defaultThreadPoolNums = 16;\n    /**\n     * Indicates the capacity of queue to hold client requests.\n     */\n    private int clientRequestThreadPoolQueueCapacity = 50000;\n    /**\n     * Indicates the capacity of queue to hold broker or operation requests.\n     */\n    private int defaultThreadPoolQueueCapacity = 10000;\n    /**\n     * Interval of periodic scanning for non-active broker;\n     */\n    private long scanNotActiveBrokerInterval = 5 * 1000;\n\n    private int unRegisterBrokerQueueCapacity = 3000;\n\n    /**\n     * Support acting master or not.\n     *\n     * The slave can be an acting master when master node is down to support following operations:\n     * 1. support lock/unlock message queue operation.\n     * 2. support searchOffset, query maxOffset/minOffset operation.\n     * 3. support query earliest msg store time.\n     */\n    private boolean supportActingMaster = false;\n\n    private volatile boolean enableAllTopicList = true;\n\n\n    private volatile boolean enableTopicList = true;\n\n    private volatile boolean notifyMinBrokerIdChanged = false;\n\n    /**\n     * Is startup the controller in this name-srv\n     */\n    private boolean enableControllerInNamesrv = false;\n\n    private volatile boolean needWaitForService = false;\n\n    private int waitSecondsForService = 45;\n\n    /**\n     * If enable this flag, the topics that don't exist in broker registration payload will be deleted from name server.\n     *\n     * WARNING:\n     * 1. Enable this flag and \"enableSingleTopicRegister\" of broker config meanwhile to avoid losing topic route info unexpectedly.\n     * 2. This flag does not support static topic currently.\n     */\n    private boolean deleteTopicWithBrokerRegistration = false;\n    /**\n     * Config in this black list will be not allowed to update by command.\n     * Try to update this config black list by restart process.\n     * Try to update configures in black list by restart process.\n     */\n    private String configBlackList = \"configBlackList;configStorePath;kvConfigPath\";\n\n    public String getConfigBlackList() {\n        return configBlackList;\n    }\n\n    public void setConfigBlackList(String configBlackList) {\n        this.configBlackList = configBlackList;\n    }\n\n    public boolean isOrderMessageEnable() {\n        return orderMessageEnable;\n    }\n\n    public void setOrderMessageEnable(boolean orderMessageEnable) {\n        this.orderMessageEnable = orderMessageEnable;\n    }\n\n    public String getRocketmqHome() {\n        return rocketmqHome;\n    }\n\n    public void setRocketmqHome(String rocketmqHome) {\n        this.rocketmqHome = rocketmqHome;\n    }\n\n    public String getKvConfigPath() {\n        return kvConfigPath;\n    }\n\n    public void setKvConfigPath(String kvConfigPath) {\n        this.kvConfigPath = kvConfigPath;\n    }\n\n    public String getProductEnvName() {\n        return productEnvName;\n    }\n\n    public void setProductEnvName(String productEnvName) {\n        this.productEnvName = productEnvName;\n    }\n\n    public boolean isClusterTest() {\n        return clusterTest;\n    }\n\n    public void setClusterTest(boolean clusterTest) {\n        this.clusterTest = clusterTest;\n    }\n\n    public String getConfigStorePath() {\n        return configStorePath;\n    }\n\n    public void setConfigStorePath(final String configStorePath) {\n        this.configStorePath = configStorePath;\n    }\n\n    public boolean isReturnOrderTopicConfigToBroker() {\n        return returnOrderTopicConfigToBroker;\n    }\n\n    public void setReturnOrderTopicConfigToBroker(boolean returnOrderTopicConfigToBroker) {\n        this.returnOrderTopicConfigToBroker = returnOrderTopicConfigToBroker;\n    }\n\n    public int getClientRequestThreadPoolNums() {\n        return clientRequestThreadPoolNums;\n    }\n\n    public void setClientRequestThreadPoolNums(final int clientRequestThreadPoolNums) {\n        this.clientRequestThreadPoolNums = clientRequestThreadPoolNums;\n    }\n\n    public int getDefaultThreadPoolNums() {\n        return defaultThreadPoolNums;\n    }\n\n    public void setDefaultThreadPoolNums(final int defaultThreadPoolNums) {\n        this.defaultThreadPoolNums = defaultThreadPoolNums;\n    }\n\n    public int getClientRequestThreadPoolQueueCapacity() {\n        return clientRequestThreadPoolQueueCapacity;\n    }\n\n    public void setClientRequestThreadPoolQueueCapacity(final int clientRequestThreadPoolQueueCapacity) {\n        this.clientRequestThreadPoolQueueCapacity = clientRequestThreadPoolQueueCapacity;\n    }\n\n    public int getDefaultThreadPoolQueueCapacity() {\n        return defaultThreadPoolQueueCapacity;\n    }\n\n    public void setDefaultThreadPoolQueueCapacity(final int defaultThreadPoolQueueCapacity) {\n        this.defaultThreadPoolQueueCapacity = defaultThreadPoolQueueCapacity;\n    }\n\n    public long getScanNotActiveBrokerInterval() {\n        return scanNotActiveBrokerInterval;\n    }\n\n    public void setScanNotActiveBrokerInterval(long scanNotActiveBrokerInterval) {\n        this.scanNotActiveBrokerInterval = scanNotActiveBrokerInterval;\n    }\n\n    public int getUnRegisterBrokerQueueCapacity() {\n        return unRegisterBrokerQueueCapacity;\n    }\n\n    public void setUnRegisterBrokerQueueCapacity(final int unRegisterBrokerQueueCapacity) {\n        this.unRegisterBrokerQueueCapacity = unRegisterBrokerQueueCapacity;\n    }\n\n    public boolean isSupportActingMaster() {\n        return supportActingMaster;\n    }\n\n    public void setSupportActingMaster(final boolean supportActingMaster) {\n        this.supportActingMaster = supportActingMaster;\n    }\n\n    public boolean isEnableAllTopicList() {\n        return enableAllTopicList;\n    }\n\n    public void setEnableAllTopicList(boolean enableAllTopicList) {\n        this.enableAllTopicList = enableAllTopicList;\n    }\n\n    public boolean isEnableTopicList() {\n        return enableTopicList;\n    }\n\n    public void setEnableTopicList(boolean enableTopicList) {\n        this.enableTopicList = enableTopicList;\n    }\n\n    public boolean isNotifyMinBrokerIdChanged() {\n        return notifyMinBrokerIdChanged;\n    }\n\n    public void setNotifyMinBrokerIdChanged(boolean notifyMinBrokerIdChanged) {\n        this.notifyMinBrokerIdChanged = notifyMinBrokerIdChanged;\n    }\n\n    public boolean isEnableControllerInNamesrv() {\n        return enableControllerInNamesrv;\n    }\n\n    public void setEnableControllerInNamesrv(boolean enableControllerInNamesrv) {\n        this.enableControllerInNamesrv = enableControllerInNamesrv;\n    }\n\n    public boolean isNeedWaitForService() {\n        return needWaitForService;\n    }\n\n    public void setNeedWaitForService(boolean needWaitForService) {\n        this.needWaitForService = needWaitForService;\n    }\n\n    public int getWaitSecondsForService() {\n        return waitSecondsForService;\n    }\n\n    public void setWaitSecondsForService(int waitSecondsForService) {\n        this.waitSecondsForService = waitSecondsForService;\n    }\n\n    public boolean isDeleteTopicWithBrokerRegistration() {\n        return deleteTopicWithBrokerRegistration;\n    }\n\n    public void setDeleteTopicWithBrokerRegistration(boolean deleteTopicWithBrokerRegistration) {\n        this.deleteTopicWithBrokerRegistration = deleteTopicWithBrokerRegistration;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/namesrv/NamesrvUtil.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.namesrv;\n\npublic class NamesrvUtil {\n    public static final String NAMESPACE_ORDER_TOPIC_CONFIG = \"ORDER_TOPIC_CONFIG\";\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/namesrv/TopAddressing.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.namesrv;\n\n\npublic interface TopAddressing {\n\n    String fetchNSAddr();\n\n    void registerChangeCallBack(NameServerUpdateCallback changeCallBack);\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/producer/RecallMessageHandle.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.producer;\n\nimport org.apache.commons.codec.DecoderException;\nimport org.apache.commons.lang3.StringUtils;\n\nimport java.util.Base64;\n\nimport static java.nio.charset.StandardCharsets.UTF_8;\n\n/**\n * handle to recall a message, only support delay message for now\n * v1 pattern like this:\n * version topic brokerName timestamp messageId\n * use Base64 to encode it\n */\npublic class RecallMessageHandle {\n    private static final String SEPARATOR = \" \";\n    private static final String VERSION_1 = \"v1\";\n\n    public static class HandleV1 extends RecallMessageHandle {\n        private String version;\n        private String topic;\n        private String brokerName;\n        private String timestampStr;\n        private String messageId; // id of unique key\n\n        public HandleV1(String topic, String brokerName, String timestamp, String messageId) {\n            this.version = VERSION_1;\n            this.topic = topic;\n            this.brokerName = brokerName;\n            this.timestampStr = timestamp;\n            this.messageId = messageId;\n        }\n\n        // no param check\n        public static String buildHandle(String topic, String brokerName, String timestampStr, String messageId) {\n            String rawString = String.join(SEPARATOR, VERSION_1, topic, brokerName, timestampStr, messageId);\n            return Base64.getUrlEncoder().encodeToString(rawString.getBytes(UTF_8));\n        }\n\n        public String getTopic() {\n            return topic;\n        }\n\n        public String getBrokerName() {\n            return brokerName;\n        }\n\n        public String getTimestampStr() {\n            return timestampStr;\n        }\n\n        public String getMessageId() {\n            return messageId;\n        }\n\n        public String getVersion() {\n            return version;\n        }\n    }\n\n    public static RecallMessageHandle decodeHandle(String handle) throws DecoderException {\n        if (StringUtils.isEmpty(handle)) {\n            throw new DecoderException(\"recall handle is invalid\");\n        }\n        String rawString;\n        try {\n            rawString =\n                new String(Base64.getUrlDecoder().decode(handle.getBytes(UTF_8)), UTF_8);\n        } catch (IllegalArgumentException e) {\n            throw new DecoderException(\"recall handle is invalid\");\n        }\n        String[] items = rawString.split(SEPARATOR);\n        if (!VERSION_1.equals(items[0]) || items.length < 5) {\n            throw new DecoderException(\"recall handle is invalid\");\n        }\n        return new HandleV1(items[1], items[2], items[3], items[4]);\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/queue/ConcurrentTreeMap.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.queue;\n\nimport java.util.Comparator;\nimport java.util.Map;\nimport java.util.TreeMap;\nimport java.util.concurrent.locks.ReentrantLock;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\n/**\n * thread safe\n */\npublic class ConcurrentTreeMap<K, V> {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private final ReentrantLock lock;\n    private TreeMap<K, V> tree;\n    private RoundQueue<K> roundQueue;\n\n    public ConcurrentTreeMap(int capacity, Comparator<? super K> comparator) {\n        tree = new TreeMap<>(comparator);\n        roundQueue = new RoundQueue<>(capacity);\n        lock = new ReentrantLock(true);\n    }\n\n    public Map.Entry<K, V> pollFirstEntry() {\n        lock.lock();\n        try {\n            return tree.pollFirstEntry();\n        } finally {\n            lock.unlock();\n        }\n    }\n\n    public V putIfAbsentAndRetExsit(K key, V value) {\n        lock.lock();\n        try {\n            if (roundQueue.put(key)) {\n                V exist = tree.get(key);\n                if (null == exist) {\n                    tree.put(key, value);\n                    exist = value;\n                }\n                log.warn(\"putIfAbsentAndRetExsit success. \" + key);\n                return exist;\n            } else {\n                V exist = tree.get(key);\n                return exist;\n            }\n        } finally {\n            lock.unlock();\n        }\n    }\n\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/queue/RoundQueue.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.queue;\n\nimport java.util.LinkedList;\nimport java.util.Queue;\n\n/**\n * not thread safe\n */\npublic class RoundQueue<E> {\n\n    private Queue<E> queue;\n    private int capacity;\n\n    public RoundQueue(int capacity) {\n        this.capacity = capacity;\n        queue = new LinkedList<>();\n    }\n\n    public boolean put(E e) {\n        boolean ok = false;\n        if (!queue.contains(e)) {\n            if (queue.size() >= capacity) {\n                queue.poll();\n            }\n            queue.add(e);\n            ok = true;\n        }\n\n        return ok;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/resource/ResourcePattern.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.resource;\n\nimport com.alibaba.fastjson2.annotation.JSONField;\n\npublic enum ResourcePattern {\n\n    ANY((byte) 1, \"ANY\"),\n\n    LITERAL((byte) 2, \"LITERAL\"),\n\n    PREFIXED((byte) 3, \"PREFIXED\");\n\n    @JSONField(value = true)\n    private final byte code;\n    private final String name;\n\n    ResourcePattern(byte code, String name) {\n        this.code = code;\n        this.name = name;\n    }\n\n    public byte getCode() {\n        return code;\n    }\n\n    public String getName() {\n        return name;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/resource/ResourceType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.resource;\n\nimport com.alibaba.fastjson2.annotation.JSONField;\nimport org.apache.commons.lang3.StringUtils;\n\npublic enum ResourceType {\n\n    UNKNOWN((byte) 0, \"Unknown\"),\n\n    ANY((byte) 1, \"Any\"),\n\n    CLUSTER((byte) 2, \"Cluster\"),\n\n    NAMESPACE((byte) 3, \"Namespace\"),\n\n    TOPIC((byte) 4, \"Topic\"),\n\n    GROUP((byte) 5, \"Group\");\n\n    @JSONField(value = true)\n    private final byte code;\n    private final String name;\n\n    ResourceType(byte code, String name) {\n        this.code = code;\n        this.name = name;\n    }\n\n    public static ResourceType getByName(String name) {\n        for (ResourceType resourceType : ResourceType.values()) {\n            if (StringUtils.equalsIgnoreCase(resourceType.getName(), name)) {\n                return resourceType;\n            }\n        }\n        return null;\n    }\n\n    public byte getCode() {\n        return code;\n    }\n\n    public String getName() {\n        return name;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/resource/RocketMQResource.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.resource;\n\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.RetentionPolicy;\n\n@Retention(RetentionPolicy.RUNTIME)\npublic @interface RocketMQResource {\n\n    ResourceType value();\n\n    String splitter() default \"\";\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/running/RunningStats.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.running;\n\npublic enum RunningStats {\n    commitLogMaxOffset,\n    commitLogMinOffset,\n    commitLogDiskRatio,\n    consumeQueueDiskRatio,\n    scheduleMessageOffset,\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/state/StateEventListener.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.state;\n\npublic interface StateEventListener<T> {\n    void fireEvent(T event);\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/statistics/FutureHolder.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.statistics;\n\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.Future;\nimport java.util.concurrent.LinkedBlockingQueue;\n\npublic class FutureHolder<T> {\n    private ConcurrentMap<T, BlockingQueue<Future>> futureMap = new ConcurrentHashMap<>(8);\n\n    public void addFuture(T t, Future future) {\n        BlockingQueue<Future> list = futureMap.get(t);\n        if (list == null) {\n            list = new LinkedBlockingQueue<>();\n            BlockingQueue<Future> old = futureMap.putIfAbsent(t, list);\n            if (old != null) {\n                list = old;\n            }\n        }\n        list.add(future);\n    }\n\n    public void removeAllFuture(T t) {\n        cancelAll(t, false);\n        futureMap.remove(t);\n    }\n\n    private void cancelAll(T t, boolean mayInterruptIfRunning) {\n        BlockingQueue<Future> list = futureMap.get(t);\n        if (list != null) {\n            for (Future future : list) {\n                future.cancel(mayInterruptIfRunning);\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/statistics/Interceptor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.statistics;\n\n/**\n * interceptor\n */\npublic interface Interceptor {\n    /**\n     * increase multiple values\n     *\n     * @param deltas\n     */\n    void inc(long... deltas);\n\n    void reset();\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsBrief.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.statistics;\n\nimport org.apache.commons.lang3.ArrayUtils;\n\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicLong;\n\npublic class StatisticsBrief {\n    public static final int META_RANGE_INDEX = 0;\n    public static final int META_SLOT_NUM_INDEX = 1;\n\n    // TopPercentile\n    private long[][] topPercentileMeta;\n    private AtomicInteger[] counts;\n    private AtomicLong totalCount;\n\n    // max min avg total\n    private long max;\n    private long min;\n    private long total;\n\n    public StatisticsBrief(long[][] topPercentileMeta) {\n        if (!isLegalMeta(topPercentileMeta)) {\n            throw new IllegalArgumentException(\"illegal topPercentileMeta\");\n        }\n\n        this.topPercentileMeta = topPercentileMeta;\n        this.counts = new AtomicInteger[slotNum(topPercentileMeta)];\n        this.totalCount = new AtomicLong(0);\n        reset();\n    }\n\n    public void reset() {\n        for (int i = 0; i < counts.length; i++) {\n            if (counts[i] == null) {\n                counts[i] = new AtomicInteger(0);\n            } else {\n                counts[i].set(0);\n            }\n        }\n        totalCount.set(0);\n\n        synchronized (this) {\n            max = 0;\n            min = Long.MAX_VALUE;\n            total = 0;\n        }\n    }\n\n    private static boolean isLegalMeta(long[][] meta) {\n        if (ArrayUtils.isEmpty(meta)) {\n            return false;\n        }\n\n        for (long[] line : meta) {\n            if (ArrayUtils.isEmpty(line) || line.length != 2) {\n                return false;\n            }\n        }\n\n        return true;\n    }\n\n    private static int slotNum(long[][] meta) {\n        int ret = 1;\n        for (long[] line : meta) {\n            ret += line[META_SLOT_NUM_INDEX];\n        }\n        return ret;\n    }\n\n    public void sample(long value) {\n        int index = getSlotIndex(value);\n        counts[index].incrementAndGet();\n        totalCount.incrementAndGet();\n\n        synchronized (this) {\n            max = Math.max(max, value);\n            min = Math.min(min, value);\n            total += value;\n        }\n    }\n\n    public long tp999() {\n        return getTPValue(0.999f);\n    }\n\n    public long getTPValue(float ratio) {\n        if (ratio <= 0 || ratio >= 1) {\n            ratio = 0.99f;\n        }\n        long count = totalCount.get();\n        long excludes = (long)(count - count * ratio);\n        if (excludes == 0) {\n            return getMax();\n        }\n\n        int tmp = 0;\n        for (int i = counts.length - 1; i > 0; i--) {\n            tmp += counts[i].get();\n            if (tmp > excludes) {\n                return Math.min(getSlotTPValue(i), getMax());\n            }\n        }\n        return 0;\n    }\n\n    private long getSlotTPValue(int index) {\n        int slotNumLeft = index;\n        for (int i = 0; i < topPercentileMeta.length; i++) {\n            int slotNum = (int)topPercentileMeta[i][META_SLOT_NUM_INDEX];\n            if (slotNumLeft < slotNum) {\n                long metaRangeMax = topPercentileMeta[i][META_RANGE_INDEX];\n                long metaRangeMin = 0;\n                if (i > 0) {\n                    metaRangeMin = topPercentileMeta[i - 1][META_RANGE_INDEX];\n                }\n\n                return metaRangeMin + (metaRangeMax - metaRangeMin) / slotNum * (slotNumLeft + 1);\n            } else {\n                slotNumLeft -= slotNum;\n            }\n        }\n        // MAX_VALUE: the last slot\n        return Integer.MAX_VALUE;\n    }\n\n    private int getSlotIndex(long num) {\n        int index = 0;\n        for (int i = 0; i < topPercentileMeta.length; i++) {\n            long rangeMax = topPercentileMeta[i][META_RANGE_INDEX];\n            int slotNum = (int)topPercentileMeta[i][META_SLOT_NUM_INDEX];\n            long rangeMin = (i > 0) ? topPercentileMeta[i - 1][META_RANGE_INDEX] : 0;\n            if (rangeMin <= num && num < rangeMax) {\n                index += (num - rangeMin) / ((rangeMax - rangeMin) / slotNum);\n                break;\n            }\n\n            index += slotNum;\n        }\n        return index;\n    }\n\n    /**\n     * Getters\n     *\n     * @return\n     */\n    public long getMax() {\n        return max;\n    }\n\n    public long getMin() {\n        return totalCount.get() > 0 ? min : 0;\n    }\n\n    public long getTotal() {\n        return total;\n    }\n\n    public long getCnt() {\n        return totalCount.get();\n    }\n\n    public double getAvg() {\n        return totalCount.get() != 0 ? ((double)total) / totalCount.get() : 0;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsBriefInterceptor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.statistics;\n\nimport org.apache.commons.lang3.ArrayUtils;\nimport org.apache.commons.lang3.tuple.Pair;\n\n/**\n * interceptor to generate statistics brief\n */\npublic class StatisticsBriefInterceptor implements Interceptor {\n    private int[] indexOfItems;\n\n    private StatisticsBrief[] statisticsBriefs;\n\n    public StatisticsBriefInterceptor(StatisticsItem item, Pair<String, long[][]>[] briefMetas) {\n        indexOfItems = new int[briefMetas.length];\n        statisticsBriefs = new StatisticsBrief[briefMetas.length];\n        for (int i = 0; i < briefMetas.length; i++) {\n            String name = briefMetas[i].getKey();\n            int index = ArrayUtils.indexOf(item.getItemNames(), name);\n            if (index < 0) {\n                throw new IllegalArgumentException(\"illegal briefItemName: \" + name);\n            }\n            indexOfItems[i] = index;\n            statisticsBriefs[i] = new StatisticsBrief(briefMetas[i].getValue());\n        }\n    }\n\n    @Override\n    public void inc(long... itemValues) {\n        for (int i = 0; i < indexOfItems.length; i++) {\n            int indexOfItem = indexOfItems[i];\n            if (indexOfItem < itemValues.length) {\n                statisticsBriefs[i].sample(itemValues[indexOfItem]);\n            }\n        }\n    }\n\n    @Override\n    public void reset() {\n        for (StatisticsBrief brief : statisticsBriefs) {\n            brief.reset();\n        }\n    }\n\n    public int[] getIndexOfItems() {\n        return indexOfItems;\n    }\n\n    public void setIndexOfItems(int[] indexOfItems) {\n        this.indexOfItems = indexOfItems;\n    }\n\n    public StatisticsBrief[] getStatisticsBriefs() {\n        return statisticsBriefs;\n    }\n\n    public void setStatisticsBriefs(StatisticsBrief[] statisticsBriefs) {\n        this.statisticsBriefs = statisticsBriefs;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItem.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.statistics;\n\nimport java.security.InvalidParameterException;\nimport java.util.Arrays;\nimport java.util.concurrent.atomic.AtomicLong;\n\nimport org.apache.commons.lang3.ArrayUtils;\n\n/**\n * Statistics Item\n */\npublic class StatisticsItem {\n    private String statKind;\n    private String statObject;\n\n    private String[] itemNames;\n    private AtomicLong[] itemAccumulates;\n    private AtomicLong invokeTimes;\n\n    private Interceptor interceptor;\n\n    /**\n     * last timestamp when the item was updated\n     */\n    private AtomicLong lastTimeStamp;\n\n    public StatisticsItem(String statKind, String statObject, String... itemNames) {\n        if (itemNames == null || itemNames.length <= 0) {\n            throw new InvalidParameterException(\"StatisticsItem \\\"itemNames\\\" is empty\");\n        }\n\n        this.statKind = statKind;\n        this.statObject = statObject;\n        this.itemNames = itemNames;\n\n        AtomicLong[] accs = new AtomicLong[itemNames.length];\n        for (int i = 0; i < itemNames.length; i++) {\n            accs[i] = new AtomicLong(0);\n        }\n\n        this.itemAccumulates = accs;\n        this.invokeTimes = new AtomicLong();\n        this.lastTimeStamp = new AtomicLong(System.currentTimeMillis());\n    }\n\n    public void incItems(long... itemIncs) {\n        int len = Math.min(itemIncs.length, itemAccumulates.length);\n        for (int i = 0; i < len; i++) {\n            itemAccumulates[i].addAndGet(itemIncs[i]);\n        }\n\n        invokeTimes.addAndGet(1);\n        lastTimeStamp.set(System.currentTimeMillis());\n\n        if (interceptor != null) {\n            interceptor.inc(itemIncs);\n        }\n    }\n\n    public boolean allZeros() {\n        if (invokeTimes.get() == 0) {\n            return true;\n        }\n\n        for (AtomicLong acc : itemAccumulates) {\n            if (acc.get() != 0) {\n                return false;\n            }\n        }\n        return true;\n    }\n\n    public String getStatKind() {\n        return statKind;\n    }\n\n    public String getStatObject() {\n        return statObject;\n    }\n\n    public String[] getItemNames() {\n        return itemNames;\n    }\n\n    public AtomicLong[] getItemAccumulates() {\n        return itemAccumulates;\n    }\n\n    public AtomicLong getInvokeTimes() {\n        return invokeTimes;\n    }\n\n    public AtomicLong getLastTimeStamp() {\n        return lastTimeStamp;\n    }\n\n    public AtomicLong getItemAccumulate(String itemName) {\n        int index = ArrayUtils.indexOf(itemNames, itemName);\n        if (index < 0) {\n            return new AtomicLong(0);\n        }\n        return itemAccumulates[index];\n    }\n\n    /**\n     * get snapshot\n     * <p>\n     * Warning: no guarantee of itemAccumulates consistency\n     *\n     * @return\n     */\n    public StatisticsItem snapshot() {\n        StatisticsItem ret = new StatisticsItem(statKind, statObject, itemNames);\n\n        ret.itemAccumulates = new AtomicLong[itemAccumulates.length];\n        for (int i = 0; i < itemAccumulates.length; i++) {\n            ret.itemAccumulates[i] = new AtomicLong(itemAccumulates[i].get());\n        }\n\n        ret.invokeTimes = new AtomicLong(invokeTimes.longValue());\n        ret.lastTimeStamp = new AtomicLong(lastTimeStamp.longValue());\n\n        return ret;\n    }\n\n    /**\n     * subtract another StatisticsItem\n     *\n     * @param item\n     * @return\n     */\n    public StatisticsItem subtract(StatisticsItem item) {\n        if (item == null) {\n            return snapshot();\n        }\n\n        if (!statKind.equals(item.statKind) || !statObject.equals(item.statObject) || !Arrays.equals(itemNames,\n            item.itemNames)) {\n            throw new IllegalArgumentException(\"StatisticsItem's kind, key and itemNames must be exactly the same\");\n        }\n\n        StatisticsItem ret = new StatisticsItem(statKind, statObject, itemNames);\n        ret.invokeTimes = new AtomicLong(invokeTimes.get() - item.invokeTimes.get());\n        ret.itemAccumulates = new AtomicLong[itemAccumulates.length];\n        for (int i = 0; i < itemAccumulates.length; i++) {\n            ret.itemAccumulates[i] = new AtomicLong(itemAccumulates[i].get() - item.itemAccumulates[i].get());\n        }\n        return ret;\n    }\n\n    public Interceptor getInterceptor() {\n        return interceptor;\n    }\n\n    public void setInterceptor(Interceptor interceptor) {\n        this.interceptor = interceptor;\n    }\n\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemFormatter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.statistics;\n\nimport java.util.concurrent.atomic.AtomicLong;\n\npublic class StatisticsItemFormatter {\n    public String format(StatisticsItem statItem) {\n        final String separator = \"|\";\n        StringBuilder sb = new StringBuilder();\n        sb.append(statItem.getStatKind()).append(separator);\n        sb.append(statItem.getStatObject()).append(separator);\n        for (AtomicLong acc : statItem.getItemAccumulates()) {\n            sb.append(acc.get()).append(separator);\n        }\n        sb.append(statItem.getInvokeTimes());\n        return sb.toString();\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemPrinter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.statistics;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\n\npublic class StatisticsItemPrinter {\n    private Logger log;\n\n    private StatisticsItemFormatter formatter;\n\n    public StatisticsItemPrinter(StatisticsItemFormatter formatter, Logger log) {\n        this.formatter = formatter;\n        this.log = log;\n    }\n\n    public void log(Logger log) {\n        this.log = log;\n    }\n\n    public void formatter(StatisticsItemFormatter formatter) {\n        this.formatter = formatter;\n    }\n\n    public void print(String prefix, StatisticsItem statItem, String... suffixs) {\n        StringBuilder suffix = new StringBuilder();\n        for (String str : suffixs) {\n            suffix.append(str);\n        }\n\n        if (log != null) {\n            log.info(\"{}{}{}\", prefix, formatter.format(statItem), suffix.toString());\n        }\n        // System.out.printf(\"%s %s%s%s\\n\", new Date().toString(), prefix, formatter.format(statItem), suffix.toString());\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemScheduledIncrementPrinter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.statistics;\n\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ScheduledFuture;\nimport java.util.concurrent.TimeUnit;\n\npublic class StatisticsItemScheduledIncrementPrinter extends StatisticsItemScheduledPrinter {\n\n    private String[] tpsItemNames;\n\n    public static final int TPS_INITIAL_DELAY = 0;\n    public static final int TPS_INTREVAL = 1000;\n    public static final String SEPARATOR = \"|\";\n\n    /**\n     * last snapshots of all scheduled items\n     */\n    private final ConcurrentHashMap<String, ConcurrentHashMap<String, StatisticsItem>> lastItemSnapshots\n        = new ConcurrentHashMap<>();\n\n    private final ConcurrentHashMap<String, ConcurrentHashMap<String, StatisticsItemSampleBrief>> sampleBriefs\n        = new ConcurrentHashMap<>();\n\n    public StatisticsItemScheduledIncrementPrinter(String name, StatisticsItemPrinter printer,\n                                                   ScheduledExecutorService executor, InitialDelay initialDelay,\n                                                   long interval, String[] tpsItemNames, Valve valve) {\n        super(name, printer, executor, initialDelay, interval, valve);\n        this.tpsItemNames = tpsItemNames;\n    }\n\n    /**\n     * schedule a StatisticsItem to print the Increments periodically\n     */\n    @Override\n    public void schedule(final StatisticsItem item) {\n        setItemSampleBrief(item.getStatKind(), item.getStatObject(), new StatisticsItemSampleBrief(item, tpsItemNames));\n\n        // print log every ${interval} milliseconds\n        ScheduledFuture future = executor.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                if (!enabled()) {\n                    return;\n                }\n\n                StatisticsItem snapshot = item.snapshot();\n                StatisticsItem lastSnapshot = getItemSnapshot(lastItemSnapshots, item.getStatKind(),\n                    item.getStatObject());\n                StatisticsItem increment = snapshot.subtract(lastSnapshot);\n\n                Interceptor interceptor = item.getInterceptor();\n                String interceptorStr = formatInterceptor(interceptor);\n                if (interceptor != null) {\n                    interceptor.reset();\n                }\n\n                StatisticsItemSampleBrief brief = getSampleBrief(item.getStatKind(), item.getStatObject());\n                if (brief != null && (!increment.allZeros() || printZeroLine())) {\n                    printer.print(name, increment, interceptorStr, brief.toString());\n                }\n\n                setItemSnapshot(lastItemSnapshots, snapshot);\n\n                if (brief != null) {\n                    brief.reset();\n                }\n            }\n        }, getInitialDelay(), interval, TimeUnit.MILLISECONDS);\n        addFuture(item, future);\n\n        // sample every TPS_INTERVAL\n        ScheduledFuture futureSample = executor.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                if (!enabled()) {\n                    return;\n                }\n\n                StatisticsItem snapshot = item.snapshot();\n                StatisticsItemSampleBrief brief = getSampleBrief(item.getStatKind(), item.getStatObject());\n                if (brief != null) {\n                    brief.sample(snapshot);\n                }\n            }\n        }, TPS_INTREVAL, TPS_INTREVAL, TimeUnit.MILLISECONDS);\n        addFuture(item, futureSample);\n    }\n\n    @Override\n    public void remove(StatisticsItem item) {\n        // remove task\n        removeAllFuture(item);\n\n        String kind = item.getStatKind();\n        String key = item.getStatObject();\n\n        ConcurrentHashMap<String, StatisticsItem> lastItemMap = lastItemSnapshots.get(kind);\n        if (lastItemMap != null) {\n            lastItemMap.remove(key);\n        }\n\n        ConcurrentHashMap<String, StatisticsItemSampleBrief> briefMap = sampleBriefs.get(kind);\n        if (briefMap != null) {\n            briefMap.remove(key);\n        }\n    }\n\n    private StatisticsItem getItemSnapshot(\n        ConcurrentHashMap<String, ConcurrentHashMap<String, StatisticsItem>> snapshots,\n        String kind, String key) {\n        ConcurrentHashMap<String, StatisticsItem> itemMap = snapshots.get(kind);\n        return (itemMap != null) ? itemMap.get(key) : null;\n    }\n\n    private StatisticsItemSampleBrief getSampleBrief(String kind, String key) {\n        ConcurrentHashMap<String, StatisticsItemSampleBrief> itemMap = sampleBriefs.get(kind);\n        return (itemMap != null) ? itemMap.get(key) : null;\n    }\n\n    private void setItemSnapshot(ConcurrentHashMap<String, ConcurrentHashMap<String, StatisticsItem>> snapshots,\n                                 StatisticsItem item) {\n        String kind = item.getStatKind();\n        String key = item.getStatObject();\n        ConcurrentHashMap<String, StatisticsItem> itemMap = snapshots.get(kind);\n        if (itemMap == null) {\n            itemMap = new ConcurrentHashMap<>();\n            ConcurrentHashMap<String, StatisticsItem> oldItemMap = snapshots.putIfAbsent(kind, itemMap);\n            if (oldItemMap != null) {\n                itemMap = oldItemMap;\n            }\n        }\n\n        itemMap.put(key, item);\n    }\n\n    private void setItemSampleBrief(String kind, String key,\n                                    StatisticsItemSampleBrief brief) {\n        ConcurrentHashMap<String, StatisticsItemSampleBrief> itemMap = sampleBriefs.get(kind);\n        if (itemMap == null) {\n            itemMap = new ConcurrentHashMap<>();\n            ConcurrentHashMap<String, StatisticsItemSampleBrief> oldItemMap = sampleBriefs.putIfAbsent(kind, itemMap);\n            if (oldItemMap != null) {\n                itemMap = oldItemMap;\n            }\n        }\n\n        itemMap.put(key, brief);\n    }\n\n    private String formatInterceptor(Interceptor interceptor) {\n        if (interceptor == null) {\n            return \"\";\n        }\n\n        if (interceptor instanceof StatisticsBriefInterceptor) {\n            StringBuilder sb = new StringBuilder();\n            StatisticsBriefInterceptor briefInterceptor = (StatisticsBriefInterceptor)interceptor;\n            for (StatisticsBrief brief : briefInterceptor.getStatisticsBriefs()) {\n                long max = brief.getMax();\n                long tp999 = Math.min(brief.tp999(), max);\n                //sb.append(SEPARATOR).append(brief.getTotal());\n                sb.append(SEPARATOR).append(max);\n                //sb.append(SEPARATOR).append(brief.getMin());\n                sb.append(SEPARATOR).append(String.format(\"%.2f\", brief.getAvg()));\n                sb.append(SEPARATOR).append(tp999);\n            }\n            return sb.toString();\n        }\n        return \"\";\n    }\n\n    public static class StatisticsItemSampleBrief {\n        private StatisticsItem lastSnapshot;\n\n        public String[] itemNames;\n        public ItemSampleBrief[] briefs;\n\n        public StatisticsItemSampleBrief(StatisticsItem statItem, String[] itemNames) {\n            this.lastSnapshot = statItem.snapshot();\n            this.itemNames = itemNames;\n            this.briefs = new ItemSampleBrief[itemNames.length];\n            for (int i = 0; i < itemNames.length; i++) {\n                this.briefs[i] = new ItemSampleBrief();\n            }\n        }\n\n        public synchronized void reset() {\n            for (ItemSampleBrief brief : briefs) {\n                brief.reset();\n            }\n        }\n\n        public synchronized void sample(StatisticsItem snapshot) {\n            if (snapshot == null) {\n                return;\n            }\n\n            for (int i = 0; i < itemNames.length; i++) {\n                String name = itemNames[i];\n\n                long lastValue = lastSnapshot != null ? lastSnapshot.getItemAccumulate(name).get() : 0;\n                long increment = snapshot.getItemAccumulate(name).get() - lastValue;\n                briefs[i].sample(increment);\n            }\n            lastSnapshot = snapshot;\n        }\n\n        @Override\n        public String toString() {\n            StringBuilder sb = new StringBuilder();\n            for (int i = 0; i < briefs.length; i++) {\n                ItemSampleBrief brief = briefs[i];\n                sb.append(SEPARATOR).append(brief.getMax());\n                //sb.append(SEPARATOR).append(brief.getMin());\n                sb.append(SEPARATOR).append(String.format(\"%.2f\", brief.getAvg()));\n            }\n            return sb.toString();\n        }\n    }\n\n    /**\n     * sample brief of a item for a period of time\n     */\n    public static class ItemSampleBrief {\n        private long max;\n        private long min;\n        private long total;\n        private long cnt;\n\n        public ItemSampleBrief() {\n            reset();\n        }\n\n        public void sample(long value) {\n            max = Math.max(max, value);\n            min = Math.min(min, value);\n            total += value;\n            cnt++;\n        }\n\n        public void reset() {\n            max = 0;\n            min = Long.MAX_VALUE;\n            total = 0;\n            cnt = 0;\n        }\n\n        /**\n         * Getters\n         *\n         * @return\n         */\n        public long getMax() {\n            return max;\n        }\n\n        public long getMin() {\n            return cnt > 0 ? min : 0;\n        }\n\n        public long getTotal() {\n            return total;\n        }\n\n        public long getCnt() {\n            return cnt;\n        }\n\n        public double getAvg() {\n            return cnt != 0 ? ((double)total) / cnt : 0;\n        }\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemScheduledPrinter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.statistics;\n\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ScheduledFuture;\nimport java.util.concurrent.TimeUnit;\n\npublic class StatisticsItemScheduledPrinter extends FutureHolder {\n    protected String name;\n\n    protected StatisticsItemPrinter printer;\n    protected ScheduledExecutorService executor;\n    protected long interval;\n    protected InitialDelay initialDelay;\n    protected Valve valve;\n\n    public StatisticsItemScheduledPrinter(String name, StatisticsItemPrinter printer,\n                                          ScheduledExecutorService executor, InitialDelay initialDelay,\n                                          long interval, Valve valve) {\n        this.name = name;\n        this.printer = printer;\n        this.executor = executor;\n        this.initialDelay = initialDelay;\n        this.interval = interval;\n        this.valve = valve;\n    }\n\n    /**\n     * schedule a StatisticsItem to print all the values periodically\n     */\n    public void schedule(final StatisticsItem statisticsItem) {\n        ScheduledFuture future = executor.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                if (enabled()) {\n                    printer.print(name, statisticsItem);\n                }\n            }\n        }, getInitialDelay(), interval, TimeUnit.MILLISECONDS);\n\n        addFuture(statisticsItem, future);\n    }\n\n    public void remove(final StatisticsItem statisticsItem) {\n        removeAllFuture(statisticsItem);\n    }\n\n    public interface InitialDelay {\n        /**\n         * Get initial delay value\n         * @return\n         */\n        long get();\n    }\n\n    public interface Valve {\n        /**\n         * whether enabled\n         * @return\n         */\n        boolean enabled();\n\n        /**\n         * whether print zero lines\n         * @return\n         */\n        boolean printZeroLine();\n    }\n\n    protected long getInitialDelay() {\n        return initialDelay != null ? initialDelay.get() : 0;\n    }\n\n    protected boolean enabled() {\n        return valve != null ? valve.enabled() : false;\n    }\n\n    protected boolean printZeroLine() {\n        return valve != null ? valve.printZeroLine() : false;\n    }\n\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemStateGetter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.statistics;\n\npublic interface StatisticsItemStateGetter {\n    boolean online(StatisticsItem item);\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsKindMeta.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.statistics;\n\n/**\n * Statistics Kind Metadata\n */\npublic class StatisticsKindMeta {\n    private String name;\n    private String[] itemNames;\n    private StatisticsItemScheduledPrinter scheduledPrinter;\n\n    public String getName() {\n        return name;\n    }\n\n    public void setName(String name) {\n        this.name = name;\n    }\n\n    public String[] getItemNames() {\n        return itemNames;\n    }\n\n    public void setItemNames(String[] itemNames) {\n        this.itemNames = itemNames;\n    }\n\n    public StatisticsItemScheduledPrinter getScheduledPrinter() {\n        return scheduledPrinter;\n    }\n\n    public void setScheduledPrinter(StatisticsItemScheduledPrinter scheduledPrinter) {\n        this.scheduledPrinter = scheduledPrinter;\n    }\n}"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.statistics;\n\nimport java.util.HashMap;\nimport java.util.Iterator;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\n\nimport org.apache.commons.lang3.tuple.Pair;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\n\npublic class StatisticsManager {\n\n    /**\n     * Set of Statistics Kind Metadata\n     */\n    private Map<String, StatisticsKindMeta> kindMetaMap;\n\n    /**\n     * item names to calculate statistics brief\n     */\n    private Pair<String, long[][]>[] briefMetas;\n\n    /**\n     * Statistics\n     */\n    private final ConcurrentHashMap<String, ConcurrentHashMap<String, StatisticsItem>> statsTable\n        = new ConcurrentHashMap<>();\n\n    private static final int MAX_IDLE_TIME = 10 * 60 * 1000;\n    private final ScheduledExecutorService executor = ThreadUtils.newSingleThreadScheduledExecutor(\n        \"StatisticsManagerCleaner\", true);\n\n    private StatisticsItemStateGetter statisticsItemStateGetter;\n\n    public StatisticsManager() {\n        kindMetaMap = new HashMap<>();\n        start();\n    }\n\n    public StatisticsManager(Map<String, StatisticsKindMeta> kindMeta) {\n        this.kindMetaMap = kindMeta;\n        start();\n    }\n\n    public void addStatisticsKindMeta(StatisticsKindMeta kindMeta) {\n        kindMetaMap.put(kindMeta.getName(), kindMeta);\n        statsTable.putIfAbsent(kindMeta.getName(), new ConcurrentHashMap<>(16));\n    }\n\n    public void setBriefMeta(Pair<String, long[][]>[] briefMetas) {\n        this.briefMetas = briefMetas;\n    }\n\n    private void start() {\n        int maxIdleTime = MAX_IDLE_TIME;\n        executor.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                Iterator<Map.Entry<String, ConcurrentHashMap<String, StatisticsItem>>> iter\n                    = statsTable.entrySet().iterator();\n                while (iter.hasNext()) {\n                    Map.Entry<String, ConcurrentHashMap<String, StatisticsItem>> entry = iter.next();\n                    String kind = entry.getKey();\n                    ConcurrentHashMap<String, StatisticsItem> itemMap = entry.getValue();\n\n                    if (itemMap == null || itemMap.isEmpty()) {\n                        continue;\n                    }\n\n                    HashMap<String, StatisticsItem> tmpItemMap = new HashMap<>(itemMap);\n                    for (StatisticsItem item : tmpItemMap.values()) {\n                        // remove when expired\n                        if (System.currentTimeMillis() - item.getLastTimeStamp().get() > MAX_IDLE_TIME\n                            && (statisticsItemStateGetter == null || !statisticsItemStateGetter.online(item))) {\n                            remove(item);\n                        }\n                    }\n                }\n            }\n        }, maxIdleTime, maxIdleTime / 3, TimeUnit.MILLISECONDS);\n    }\n\n    /**\n     * Increment a StatisticsItem\n     *\n     * @param kind\n     * @param key\n     * @param itemAccumulates\n     */\n    public boolean inc(String kind, String key, long... itemAccumulates) {\n        ConcurrentHashMap<String, StatisticsItem> itemMap = statsTable.get(kind);\n        if (itemMap != null) {\n            StatisticsItem item = itemMap.get(key);\n\n            // if not exist, create and schedule\n            if (item == null) {\n                item = new StatisticsItem(kind, key, kindMetaMap.get(kind).getItemNames());\n                item.setInterceptor(new StatisticsBriefInterceptor(item, briefMetas));\n                StatisticsItem oldItem = itemMap.putIfAbsent(key, item);\n                if (oldItem != null) {\n                    item = oldItem;\n                } else {\n                    scheduleStatisticsItem(item);\n                }\n            }\n\n            // do increment\n            item.incItems(itemAccumulates);\n\n            return true;\n        }\n\n        return false;\n    }\n\n    private void scheduleStatisticsItem(StatisticsItem item) {\n        kindMetaMap.get(item.getStatKind()).getScheduledPrinter().schedule(item);\n    }\n\n    public void remove(StatisticsItem item) {\n        ConcurrentHashMap<String, StatisticsItem> itemMap = statsTable.get(item.getStatKind());\n        if (itemMap != null) {\n            itemMap.remove(item.getStatObject(), item);\n        }\n\n        StatisticsKindMeta kindMeta = kindMetaMap.get(item.getStatKind());\n        if (kindMeta != null && kindMeta.getScheduledPrinter() != null) {\n            kindMeta.getScheduledPrinter().remove(item);\n        }\n    }\n\n    public StatisticsItemStateGetter getStatisticsItemStateGetter() {\n        return statisticsItemStateGetter;\n    }\n\n    public void setStatisticsItemStateGetter(StatisticsItemStateGetter statisticsItemStateGetter) {\n        this.statisticsItemStateGetter = statisticsItemStateGetter;\n    }\n\n    public void shutdown() {\n        executor.shutdown();\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItem.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.stats;\n\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\n\npublic class MomentStatsItem {\n\n    private final AtomicLong value = new AtomicLong(0);\n\n    private final String statsName;\n    private final String statsKey;\n    private final ScheduledExecutorService scheduledExecutorService;\n    private final Logger log;\n    private long lastUpdateTimestamp = System.currentTimeMillis();\n\n    public MomentStatsItem(String statsName, String statsKey,\n        ScheduledExecutorService scheduledExecutorService, Logger log) {\n        this.statsName = statsName;\n        this.statsKey = statsKey;\n        this.scheduledExecutorService = scheduledExecutorService;\n        this.log = log;\n    }\n\n    public void init() {\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    printAtMinutes();\n\n                    MomentStatsItem.this.value.set(0);\n                } catch (Throwable e) {\n                }\n            }\n        }, Math.abs(UtilAll.computeNextMinutesTimeMillis() - System.currentTimeMillis()), 1000 * 60 * 5, TimeUnit.MILLISECONDS);\n    }\n\n    public void printAtMinutes() {\n        log.info(\"[{}] [{}] Stats Every 5 Minutes, Value: {}\",\n            this.statsName,\n            this.statsKey,\n            this.value.get());\n    }\n\n    public AtomicLong getValue() {\n        return value;\n    }\n\n    public String getStatsKey() {\n        return statsKey;\n    }\n\n    public String getStatsName() {\n        return statsName;\n    }\n\n    public long getLastUpdateTimestamp() {\n        return lastUpdateTimestamp;\n    }\n\n    public void setLastUpdateTimestamp(long lastUpdateTimestamp) {\n        this.lastUpdateTimestamp = lastUpdateTimestamp;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItemSet.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.stats;\n\nimport java.util.Iterator;\nimport java.util.Map.Entry;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class MomentStatsItemSet {\n    private static final Logger COMMERCIAL_LOG = LoggerFactory.getLogger(LoggerName.COMMERCIAL_LOGGER_NAME);\n    private final ConcurrentMap<String/* key */, MomentStatsItem> statsItemTable =\n        new ConcurrentHashMap<>(128);\n    private final String statsName;\n    private final ScheduledExecutorService scheduledExecutorService;\n    private final Logger log;\n\n    public MomentStatsItemSet(String statsName, ScheduledExecutorService scheduledExecutorService, Logger log) {\n        this.statsName = statsName;\n        this.scheduledExecutorService = scheduledExecutorService;\n        this.log = log;\n        this.init();\n    }\n\n    public ConcurrentMap<String, MomentStatsItem> getStatsItemTable() {\n        return statsItemTable;\n    }\n\n    public String getStatsName() {\n        return statsName;\n    }\n\n    public void init() {\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    printAtMinutes();\n                } catch (Throwable ignored) {\n                }\n            }\n        }, Math.abs(UtilAll.computeNextMinutesTimeMillis() - System.currentTimeMillis()), 1000 * 60 * 5, TimeUnit.MILLISECONDS);\n    }\n\n    private void printAtMinutes() {\n        Iterator<Entry<String, MomentStatsItem>> it = this.statsItemTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, MomentStatsItem> next = it.next();\n            next.getValue().printAtMinutes();\n        }\n    }\n\n    public void setValue(final String statsKey, final int value) {\n        MomentStatsItem statsItem = this.getAndCreateStatsItem(statsKey);\n        statsItem.getValue().set(value);\n        statsItem.setLastUpdateTimestamp(System.currentTimeMillis());\n    }\n\n    public void setValue(final String statsKey, final long value) {\n        MomentStatsItem statsItem = this.getAndCreateStatsItem(statsKey);\n        statsItem.getValue().set(value);\n        statsItem.setLastUpdateTimestamp(System.currentTimeMillis());\n    }\n\n    public void delValueByInfixKey(final String statsKey, String separator) {\n        Iterator<Entry<String, MomentStatsItem>> it = this.statsItemTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, MomentStatsItem> next = it.next();\n            if (next.getKey().contains(separator + statsKey + separator)) {\n                it.remove();\n            }\n        }\n    }\n\n    public void delValueBySuffixKey(final String statsKey, String separator) {\n        Iterator<Entry<String, MomentStatsItem>> it = this.statsItemTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, MomentStatsItem> next = it.next();\n            if (next.getKey().endsWith(separator + statsKey)) {\n                it.remove();\n            }\n        }\n    }\n\n    public MomentStatsItem getAndCreateStatsItem(final String statsKey) {\n        MomentStatsItem statsItem = this.statsItemTable.get(statsKey);\n        if (null == statsItem) {\n            statsItem =\n                new MomentStatsItem(this.statsName, statsKey, this.scheduledExecutorService, this.log);\n            MomentStatsItem prev = this.statsItemTable.putIfAbsent(statsKey, statsItem);\n\n            if (null != prev) {\n                statsItem = prev;\n                // statsItem.init();\n            }\n        }\n\n        return statsItem;\n    }\n\n    public void cleanResource(int maxStatsIdleTimeInMinutes) {\n        COMMERCIAL_LOG.info(\"CleanStatisticItem: kind:{}, size:{}\", statsName, this.statsItemTable.size());\n        Iterator<Entry<String, MomentStatsItem>> it = this.statsItemTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, MomentStatsItem> next = it.next();\n            MomentStatsItem statsItem = next.getValue();\n            if (System.currentTimeMillis() - statsItem.getLastUpdateTimestamp() > maxStatsIdleTimeInMinutes * 60 * 1000L) {\n                it.remove();\n                COMMERCIAL_LOG.info(\"CleanStatisticItem: removeKind:{}, removeKey:{}\", statsName, statsItem.getStatsKey());\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/stats/RTStatsItem.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.stats;\n\nimport java.util.concurrent.ScheduledExecutorService;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\n\n/**\n * A StatItem for response time, the only difference between from StatsItem is it has a different log output.\n */\npublic class RTStatsItem extends StatsItem {\n\n    public RTStatsItem(String statsName, String statsKey, ScheduledExecutorService scheduledExecutorService,\n        Logger logger) {\n        super(statsName, statsKey, scheduledExecutorService, logger);\n    }\n\n    /**\n     *   For Response Time stat Item, the print detail should be a little different, TPS and SUM makes no sense.\n     *   And we give a name \"AVGRT\" rather than AVGPT for value getAvgpt()\n      */\n    @Override\n    protected String statPrintDetail(StatsSnapshot ss) {\n        return String.format(\"TIMES: %d AVGRT: %.2f\", ss.getTimes(), ss.getAvgpt());\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/stats/Stats.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.stats;\n\npublic class Stats {\n\n    public static final String QUEUE_PUT_NUMS = \"QUEUE_PUT_NUMS\";\n    public static final String QUEUE_PUT_SIZE = \"QUEUE_PUT_SIZE\";\n    public static final String QUEUE_GET_NUMS = \"QUEUE_GET_NUMS\";\n    public static final String QUEUE_GET_SIZE = \"QUEUE_GET_SIZE\";\n    public static final String TOPIC_PUT_NUMS = \"TOPIC_PUT_NUMS\";\n    public static final String TOPIC_PUT_SIZE = \"TOPIC_PUT_SIZE\";\n    public static final String GROUP_GET_NUMS = \"GROUP_GET_NUMS\";\n    public static final String GROUP_GET_SIZE = \"GROUP_GET_SIZE\";\n    public static final String SNDBCK_PUT_NUMS = \"SNDBCK_PUT_NUMS\";\n    public static final String BROKER_PUT_NUMS = \"BROKER_PUT_NUMS\";\n    public static final String BROKER_GET_NUMS = \"BROKER_GET_NUMS\";\n    public static final String GROUP_GET_FROM_DISK_NUMS = \"GROUP_GET_FROM_DISK_NUMS\";\n    public static final String GROUP_GET_FROM_DISK_SIZE = \"GROUP_GET_FROM_DISK_SIZE\";\n    public static final String BROKER_GET_FROM_DISK_NUMS = \"BROKER_GET_FROM_DISK_NUMS\";\n    public static final String BROKER_GET_FROM_DISK_SIZE = \"BROKER_GET_FROM_DISK_SIZE\";\n    public static final String COMMERCIAL_SEND_TIMES = \"COMMERCIAL_SEND_TIMES\";\n    public static final String COMMERCIAL_SNDBCK_TIMES = \"COMMERCIAL_SNDBCK_TIMES\";\n    public static final String COMMERCIAL_RCV_TIMES = \"COMMERCIAL_RCV_TIMES\";\n    public static final String COMMERCIAL_RCV_EPOLLS = \"COMMERCIAL_RCV_EPOLLS\";\n    public static final String COMMERCIAL_SEND_SIZE = \"COMMERCIAL_SEND_SIZE\";\n    public static final String COMMERCIAL_RCV_SIZE = \"COMMERCIAL_RCV_SIZE\";\n    public static final String COMMERCIAL_PERM_FAILURES = \"COMMERCIAL_PERM_FAILURES\";\n\n    public static final String GROUP_GET_FALL_SIZE = \"GROUP_GET_FALL_SIZE\";\n    public static final String GROUP_GET_FALL_TIME = \"GROUP_GET_FALL_TIME\";\n    public static final String GROUP_GET_LATENCY = \"GROUP_GET_LATENCY\";\n    public static final String TOPIC_PUT_LATENCY = \"TOPIC_PUT_LATENCY\";\n    public static final String GROUP_ACK_NUMS = \"GROUP_ACK_NUMS\";\n    public static final String GROUP_CK_NUMS = \"GROUP_CK_NUMS\";\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/stats/StatsItem.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.stats;\n\nimport java.util.LinkedList;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.LongAdder;\n\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\n\npublic class StatsItem {\n    private final LongAdder value = new LongAdder();\n\n    private final LongAdder times = new LongAdder();\n\n    private final LinkedList<CallSnapshot> csListMinute = new LinkedList<>();\n\n    private final LinkedList<CallSnapshot> csListHour = new LinkedList<>();\n\n    private final LinkedList<CallSnapshot> csListDay = new LinkedList<>();\n\n    private final String statsName;\n    private final String statsKey;\n    private long lastUpdateTimestamp = System.currentTimeMillis();\n    private final ScheduledExecutorService scheduledExecutorService;\n\n    private final Logger logger;\n\n    public StatsItem(String statsName, String statsKey, ScheduledExecutorService scheduledExecutorService, Logger logger) {\n        this.statsName = statsName;\n        this.statsKey = statsKey;\n        this.scheduledExecutorService = scheduledExecutorService;\n        this.logger = logger;\n    }\n\n    private static StatsSnapshot computeStatsData(final LinkedList<CallSnapshot> csList) {\n        StatsSnapshot statsSnapshot = new StatsSnapshot();\n        synchronized (csList) {\n            double tps = 0;\n            double avgpt = 0;\n            long sum = 0;\n            long timesDiff = 0;\n            if (!csList.isEmpty()) {\n                CallSnapshot first = csList.getFirst();\n                CallSnapshot last = csList.getLast();\n                sum = last.getValue() - first.getValue();\n                tps = (sum * 1000.0d) / (last.getTimestamp() - first.getTimestamp());\n\n                timesDiff = last.getTimes() - first.getTimes();\n                if (timesDiff > 0) {\n                    avgpt = (sum * 1.0d) / timesDiff;\n                }\n            }\n\n            statsSnapshot.setSum(sum);\n            statsSnapshot.setTps(tps);\n            statsSnapshot.setAvgpt(avgpt);\n            statsSnapshot.setTimes(timesDiff);\n        }\n\n        return statsSnapshot;\n    }\n\n    public StatsSnapshot getStatsDataInMinute() {\n        return computeStatsData(this.csListMinute);\n    }\n\n    public StatsSnapshot getStatsDataInHour() {\n        return computeStatsData(this.csListHour);\n    }\n\n    public StatsSnapshot getStatsDataInDay() {\n        return computeStatsData(this.csListDay);\n    }\n\n    public void init() {\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    samplingInSeconds();\n                } catch (Throwable ignored) {\n                }\n            }\n        }, 0, 10, TimeUnit.SECONDS);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    samplingInMinutes();\n                } catch (Throwable ignored) {\n                }\n            }\n        }, 0, 10, TimeUnit.MINUTES);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    samplingInHour();\n                } catch (Throwable ignored) {\n                }\n            }\n        }, 0, 1, TimeUnit.HOURS);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    printAtMinutes();\n                } catch (Throwable ignored) {\n                }\n            }\n        }, Math.abs(UtilAll.computeNextMinutesTimeMillis() - System.currentTimeMillis()), 1000 * 60, TimeUnit.MILLISECONDS);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    printAtHour();\n                } catch (Throwable ignored) {\n                }\n            }\n        }, Math.abs(UtilAll.computeNextHourTimeMillis() - System.currentTimeMillis()), 1000 * 60 * 60, TimeUnit.MILLISECONDS);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    printAtDay();\n                } catch (Throwable ignored) {\n                }\n            }\n        }, Math.abs(UtilAll.computeNextMorningTimeMillis() - System.currentTimeMillis()) - 2000, 1000 * 60 * 60 * 24, TimeUnit.MILLISECONDS);\n    }\n\n    public void samplingInSeconds() {\n        synchronized (this.csListMinute) {\n            if (this.csListMinute.size() == 0) {\n                this.csListMinute.add(new CallSnapshot(System.currentTimeMillis() - 10 * 1000, 0, 0));\n            }\n            this.csListMinute.add(new CallSnapshot(System.currentTimeMillis(), this.times.sum(), this.value\n                .sum()));\n            if (this.csListMinute.size() > 7) {\n                this.csListMinute.removeFirst();\n            }\n        }\n    }\n\n    public void samplingInMinutes() {\n        synchronized (this.csListHour) {\n            if (this.csListHour.size() == 0) {\n                this.csListHour.add(new CallSnapshot(System.currentTimeMillis() - 10 * 60 * 1000, 0, 0));\n            }\n            this.csListHour.add(new CallSnapshot(System.currentTimeMillis(), this.times.sum(), this.value\n                .sum()));\n            if (this.csListHour.size() > 7) {\n                this.csListHour.removeFirst();\n            }\n        }\n    }\n\n    public void samplingInHour() {\n        synchronized (this.csListDay) {\n            if (this.csListDay.size() == 0) {\n                this.csListDay.add(new CallSnapshot(System.currentTimeMillis() - 1 * 60 * 60 * 1000, 0, 0));\n            }\n            this.csListDay.add(new CallSnapshot(System.currentTimeMillis(), this.times.sum(), this.value\n                .sum()));\n            if (this.csListDay.size() > 25) {\n                this.csListDay.removeFirst();\n            }\n        }\n    }\n\n    public void printAtMinutes() {\n        StatsSnapshot ss = computeStatsData(this.csListMinute);\n        logger.info(String.format(\"[%s] [%s] Stats In One Minute, \", this.statsName, this.statsKey) + statPrintDetail(ss));\n    }\n\n    public void printAtHour() {\n        StatsSnapshot ss = computeStatsData(this.csListHour);\n        logger.info(String.format(\"[%s] [%s] Stats In One Hour, \", this.statsName, this.statsKey) + statPrintDetail(ss));\n\n    }\n\n    public void printAtDay() {\n        StatsSnapshot ss = computeStatsData(this.csListDay);\n        logger.info(String.format(\"[%s] [%s] Stats In One Day, \", this.statsName, this.statsKey) + statPrintDetail(ss));\n    }\n\n    protected String statPrintDetail(StatsSnapshot ss) {\n        return String.format(\"SUM: %d TPS: %.2f AVGPT: %.2f\",\n                ss.getSum(),\n                ss.getTps(),\n                ss.getAvgpt());\n    }\n\n    public LongAdder getValue() {\n        return value;\n    }\n\n    public String getStatsKey() {\n        return statsKey;\n    }\n\n    public String getStatsName() {\n        return statsName;\n    }\n\n    public LongAdder getTimes() {\n        return times;\n    }\n\n    public long getLastUpdateTimestamp() {\n        return lastUpdateTimestamp;\n    }\n\n    public void setLastUpdateTimestamp(long lastUpdateTimestamp) {\n        this.lastUpdateTimestamp = lastUpdateTimestamp;\n    }\n}\n\nclass CallSnapshot {\n    private final long timestamp;\n    private final long times;\n\n    private final long value;\n\n    public CallSnapshot(long timestamp, long times, long value) {\n        super();\n        this.timestamp = timestamp;\n        this.times = times;\n        this.value = value;\n    }\n\n    public long getTimestamp() {\n        return timestamp;\n    }\n\n    public long getTimes() {\n        return times;\n    }\n\n    public long getValue() {\n        return value;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/stats/StatsItemSet.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.stats;\n\nimport java.util.Iterator;\nimport java.util.Map.Entry;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class StatsItemSet {\n    private static final Logger COMMERCIAL_LOG = LoggerFactory.getLogger(LoggerName.COMMERCIAL_LOGGER_NAME);\n    private final ConcurrentMap<String/* key */, StatsItem> statsItemTable =\n        new ConcurrentHashMap<>(128);\n\n    private final String statsName;\n    private final ScheduledExecutorService scheduledExecutorService;\n\n    private final Logger logger;\n\n    public StatsItemSet(String statsName, ScheduledExecutorService scheduledExecutorService, Logger logger) {\n        this.logger = logger;\n        this.statsName = statsName;\n        this.scheduledExecutorService = scheduledExecutorService;\n        this.init();\n    }\n\n    public void init() {\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    samplingInSeconds();\n                } catch (Throwable ignored) {\n                }\n            }\n        }, 0, 10, TimeUnit.SECONDS);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    samplingInMinutes();\n                } catch (Throwable ignored) {\n                }\n            }\n        }, 0, 10, TimeUnit.MINUTES);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    samplingInHour();\n                } catch (Throwable ignored) {\n                }\n            }\n        }, 0, 1, TimeUnit.HOURS);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    printAtMinutes();\n                } catch (Throwable ignored) {\n                }\n            }\n        }, Math.abs(UtilAll.computeNextMinutesTimeMillis() - System.currentTimeMillis()), 1000 * 60, TimeUnit.MILLISECONDS);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    printAtHour();\n                } catch (Throwable ignored) {\n                }\n            }\n        }, Math.abs(UtilAll.computeNextHourTimeMillis() - System.currentTimeMillis()), 1000 * 60 * 60, TimeUnit.MILLISECONDS);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    printAtDay();\n                } catch (Throwable ignored) {\n                }\n            }\n        }, Math.abs(UtilAll.computeNextMorningTimeMillis() - System.currentTimeMillis()), 1000 * 60 * 60 * 24, TimeUnit.MILLISECONDS);\n    }\n\n    private void samplingInSeconds() {\n        Iterator<Entry<String, StatsItem>> it = this.statsItemTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, StatsItem> next = it.next();\n            next.getValue().samplingInSeconds();\n        }\n    }\n\n    private void samplingInMinutes() {\n        Iterator<Entry<String, StatsItem>> it = this.statsItemTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, StatsItem> next = it.next();\n            next.getValue().samplingInMinutes();\n        }\n    }\n\n    private void samplingInHour() {\n        Iterator<Entry<String, StatsItem>> it = this.statsItemTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, StatsItem> next = it.next();\n            next.getValue().samplingInHour();\n        }\n    }\n\n    private void printAtMinutes() {\n        Iterator<Entry<String, StatsItem>> it = this.statsItemTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, StatsItem> next = it.next();\n            next.getValue().printAtMinutes();\n        }\n    }\n\n    private void printAtHour() {\n        Iterator<Entry<String, StatsItem>> it = this.statsItemTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, StatsItem> next = it.next();\n            next.getValue().printAtHour();\n        }\n    }\n\n    private void printAtDay() {\n        Iterator<Entry<String, StatsItem>> it = this.statsItemTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, StatsItem> next = it.next();\n            next.getValue().printAtDay();\n        }\n    }\n\n    public void addValue(final String statsKey, final int incValue, final int incTimes) {\n        StatsItem statsItem = this.getAndCreateStatsItem(statsKey);\n        statsItem.getValue().add(incValue);\n        statsItem.getTimes().add(incTimes);\n        statsItem.setLastUpdateTimestamp(System.currentTimeMillis());\n    }\n\n    public void addRTValue(final String statsKey, final int incValue, final int incTimes) {\n        StatsItem statsItem = this.getAndCreateRTStatsItem(statsKey);\n        statsItem.getValue().add(incValue);\n        statsItem.getTimes().add(incTimes);\n        statsItem.setLastUpdateTimestamp(System.currentTimeMillis());\n    }\n\n    public void delValue(final String statsKey) {\n        StatsItem statsItem = this.statsItemTable.get(statsKey);\n        if (null != statsItem) {\n            this.statsItemTable.remove(statsKey);\n        }\n    }\n\n    public void delValueByPrefixKey(final String statsKey, String separator) {\n        Iterator<Entry<String, StatsItem>> it = this.statsItemTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, StatsItem> next = it.next();\n            if (next.getKey().startsWith(statsKey + separator)) {\n                it.remove();\n            }\n        }\n    }\n\n    public void delValueByInfixKey(final String statsKey, String separator) {\n        Iterator<Entry<String, StatsItem>> it = this.statsItemTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, StatsItem> next = it.next();\n            if (next.getKey().contains(separator + statsKey + separator)) {\n                it.remove();\n            }\n        }\n    }\n\n    public void delValueBySuffixKey(final String statsKey, String separator) {\n        Iterator<Entry<String, StatsItem>> it = this.statsItemTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, StatsItem> next = it.next();\n            if (next.getKey().endsWith(separator + statsKey)) {\n                it.remove();\n            }\n        }\n    }\n\n    public StatsItem getAndCreateStatsItem(final String statsKey) {\n        return getAndCreateItem(statsKey, false);\n    }\n\n    public StatsItem getAndCreateRTStatsItem(final String statsKey) {\n        return getAndCreateItem(statsKey, true);\n    }\n\n    public StatsItem getAndCreateItem(final String statsKey, boolean rtItem) {\n        StatsItem statsItem = this.statsItemTable.get(statsKey);\n        if (null == statsItem) {\n            if (rtItem) {\n                statsItem = new RTStatsItem(this.statsName, statsKey, this.scheduledExecutorService, logger);\n            } else {\n                statsItem = new StatsItem(this.statsName, statsKey, this.scheduledExecutorService, logger);\n            }\n            StatsItem prev = this.statsItemTable.putIfAbsent(statsKey, statsItem);\n\n            if (null != prev) {\n                statsItem = prev;\n                // statsItem.init();\n            }\n        }\n\n        return statsItem;\n    }\n\n    public StatsSnapshot getStatsDataInMinute(final String statsKey) {\n        StatsItem statsItem = this.statsItemTable.get(statsKey);\n        if (null != statsItem) {\n            return statsItem.getStatsDataInMinute();\n        }\n        return new StatsSnapshot();\n    }\n\n    public StatsSnapshot getStatsDataInHour(final String statsKey) {\n        StatsItem statsItem = this.statsItemTable.get(statsKey);\n        if (null != statsItem) {\n            return statsItem.getStatsDataInHour();\n        }\n        return new StatsSnapshot();\n    }\n\n    public StatsSnapshot getStatsDataInDay(final String statsKey) {\n        StatsItem statsItem = this.statsItemTable.get(statsKey);\n        if (null != statsItem) {\n            return statsItem.getStatsDataInDay();\n        }\n        return new StatsSnapshot();\n    }\n\n    public StatsItem getStatsItem(final String statsKey) {\n        return this.statsItemTable.get(statsKey);\n    }\n\n\n    public void cleanResource(int maxStatsIdleTimeInMinutes) {\n        COMMERCIAL_LOG.info(\"CleanStatisticItemOld: kind:{}, size:{}\", statsName, this.statsItemTable.size());\n        Iterator<Entry<String, StatsItem>> it = this.statsItemTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<String, StatsItem> next = it.next();\n            StatsItem statsItem = next.getValue();\n            if (System.currentTimeMillis() - statsItem.getLastUpdateTimestamp() > maxStatsIdleTimeInMinutes * 60 * 1000L) {\n                it.remove();\n                COMMERCIAL_LOG.info(\"CleanStatisticItemOld: removeKind:{}, removeKey:{}\", statsName, statsItem.getStatsKey());\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/stats/StatsSnapshot.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.stats;\n\npublic class StatsSnapshot {\n    private long sum;\n    private double tps;\n\n    private long times;\n    private double avgpt;\n\n    public long getSum() {\n        return sum;\n    }\n\n    public void setSum(long sum) {\n        this.sum = sum;\n    }\n\n    public double getTps() {\n        return tps;\n    }\n\n    public void setTps(double tps) {\n        this.tps = tps;\n    }\n\n    public double getAvgpt() {\n        return avgpt;\n    }\n\n    public void setAvgpt(double avgpt) {\n        this.avgpt = avgpt;\n    }\n\n    public long getTimes() {\n        return times;\n    }\n\n    public void setTimes(long times) {\n        this.times = times;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.sysflag;\n\nimport org.apache.rocketmq.common.compression.CompressionType;\n\npublic class MessageSysFlag {\n\n    /**\n     * Meaning of each bit in the system flag\n     *\n     * | bit    | 7 | 6 | 5         | 4        | 3           | 2                | 1                | 0                |\n     * |--------|---|---|-----------|----------|-------------|------------------|------------------|------------------|\n     * | byte 1 |   |   | STOREHOST | BORNHOST | TRANSACTION | TRANSACTION      | MULTI_TAGS       | COMPRESSED       |\n     * | byte 2 |   |   |           |          |             | COMPRESSION_TYPE | COMPRESSION_TYPE | COMPRESSION_TYPE |\n     * | byte 3 |   |   |           |          |             |                  |                  |                  |\n     * | byte 4 |   |   |           |          |             |                  |                  |                  |\n     */\n    public final static int COMPRESSED_FLAG = 0x1;\n    public final static int MULTI_TAGS_FLAG = 0x1 << 1;\n    public final static int TRANSACTION_NOT_TYPE = 0;\n    public final static int TRANSACTION_PREPARED_TYPE = 0x1 << 2;\n    public final static int TRANSACTION_COMMIT_TYPE = 0x2 << 2;\n    public final static int TRANSACTION_ROLLBACK_TYPE = 0x3 << 2;\n    public final static int BORNHOST_V6_FLAG = 0x1 << 4;\n    public final static int STOREHOSTADDRESS_V6_FLAG = 0x1 << 5;\n    //Mark the flag for batch to avoid conflict\n    public final static int NEED_UNWRAP_FLAG = 0x1 << 6;\n    public final static int INNER_BATCH_FLAG = 0x1 << 7;\n\n    // COMPRESSION_TYPE\n    public final static int COMPRESSION_LZ4_TYPE = 0x1 << 8;\n    public final static int COMPRESSION_ZSTD_TYPE = 0x2 << 8;\n    public final static int COMPRESSION_ZLIB_TYPE = 0x3 << 8;\n    public final static int COMPRESSION_TYPE_COMPARATOR = 0x7 << 8;\n\n    public static int getTransactionValue(final int flag) {\n        return flag & TRANSACTION_ROLLBACK_TYPE;\n    }\n\n    public static int resetTransactionValue(final int flag, final int type) {\n        return (flag & (~TRANSACTION_ROLLBACK_TYPE)) | type;\n    }\n\n    public static int clearCompressedFlag(final int flag) {\n        return flag & (~COMPRESSED_FLAG);\n    }\n\n    // To match the compression type\n    public static CompressionType getCompressionType(final int flag) {\n        return CompressionType.findByValue((flag & COMPRESSION_TYPE_COMPARATOR) >> 8);\n    }\n\n    public static boolean check(int flag, int expectedFlag) {\n        return (flag & expectedFlag) != 0;\n    }\n\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/sysflag/PullSysFlag.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.sysflag;\n\npublic class PullSysFlag {\n    private final static int FLAG_COMMIT_OFFSET = 0x1;\n    private final static int FLAG_SUSPEND = 0x1 << 1;\n    private final static int FLAG_SUBSCRIPTION = 0x1 << 2;\n    private final static int FLAG_CLASS_FILTER = 0x1 << 3;\n    private final static int FLAG_LITE_PULL_MESSAGE = 0x1 << 4;\n\n    public static int buildSysFlag(final boolean commitOffset, final boolean suspend,\n        final boolean subscription, final boolean classFilter) {\n        int flag = 0;\n\n        if (commitOffset) {\n            flag |= FLAG_COMMIT_OFFSET;\n        }\n\n        if (suspend) {\n            flag |= FLAG_SUSPEND;\n        }\n\n        if (subscription) {\n            flag |= FLAG_SUBSCRIPTION;\n        }\n\n        if (classFilter) {\n            flag |= FLAG_CLASS_FILTER;\n        }\n\n        return flag;\n    }\n\n    public static int buildSysFlag(final boolean commitOffset, final boolean suspend,\n        final boolean subscription, final boolean classFilter, final boolean litePull) {\n        int flag = buildSysFlag(commitOffset, suspend, subscription, classFilter);\n\n        if (litePull) {\n            flag |= FLAG_LITE_PULL_MESSAGE;\n        }\n\n        return flag;\n    }\n\n    public static int clearCommitOffsetFlag(final int sysFlag) {\n        return sysFlag & (~FLAG_COMMIT_OFFSET);\n    }\n\n    public static boolean hasCommitOffsetFlag(final int sysFlag) {\n        return (sysFlag & FLAG_COMMIT_OFFSET) == FLAG_COMMIT_OFFSET;\n    }\n\n    public static boolean hasSuspendFlag(final int sysFlag) {\n        return (sysFlag & FLAG_SUSPEND) == FLAG_SUSPEND;\n    }\n\n    public static int clearSuspendFlag(final int sysFlag) {\n        return sysFlag & (~FLAG_SUSPEND);\n    }\n\n    public static boolean hasSubscriptionFlag(final int sysFlag) {\n        return (sysFlag & FLAG_SUBSCRIPTION) == FLAG_SUBSCRIPTION;\n    }\n\n    public static int buildSysFlagWithSubscription(final int sysFlag) {\n        return sysFlag | FLAG_SUBSCRIPTION;\n    }\n\n    public static boolean hasClassFilterFlag(final int sysFlag) {\n        return (sysFlag & FLAG_CLASS_FILTER) == FLAG_CLASS_FILTER;\n    }\n\n    public static boolean hasLitePullFlag(final int sysFlag) {\n        return (sysFlag & FLAG_LITE_PULL_MESSAGE) == FLAG_LITE_PULL_MESSAGE;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/sysflag/SubscriptionSysFlag.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.sysflag;\n\npublic class SubscriptionSysFlag {\n\n    private final static int FLAG_UNIT = 0x1 << 0;\n\n    public static int buildSysFlag(final boolean unit) {\n        int sysFlag = 0;\n\n        if (unit) {\n            sysFlag |= FLAG_UNIT;\n        }\n\n        return sysFlag;\n    }\n\n    public static int setUnitFlag(final int sysFlag) {\n        return sysFlag | FLAG_UNIT;\n    }\n\n    public static int clearUnitFlag(final int sysFlag) {\n        return sysFlag & (~FLAG_UNIT);\n    }\n\n    public static boolean hasUnitFlag(final int sysFlag) {\n        return (sysFlag & FLAG_UNIT) == FLAG_UNIT;\n    }\n\n    public static void main(String[] args) {\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/sysflag/TopicSysFlag.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.sysflag;\n\npublic class TopicSysFlag {\n\n    private final static int FLAG_UNIT = 0x1 << 0;\n\n    private final static int FLAG_UNIT_SUB = 0x1 << 1;\n\n    public static int buildSysFlag(final boolean unit, final boolean hasUnitSub) {\n        int sysFlag = 0;\n\n        if (unit) {\n            sysFlag |= FLAG_UNIT;\n        }\n\n        if (hasUnitSub) {\n            sysFlag |= FLAG_UNIT_SUB;\n        }\n\n        return sysFlag;\n    }\n\n    public static int setUnitFlag(final int sysFlag) {\n        return sysFlag | FLAG_UNIT;\n    }\n\n    public static int clearUnitFlag(final int sysFlag) {\n        return sysFlag & (~FLAG_UNIT);\n    }\n\n    public static boolean hasUnitFlag(final int sysFlag) {\n        return (sysFlag & FLAG_UNIT) == FLAG_UNIT;\n    }\n\n    public static int setUnitSubFlag(final int sysFlag) {\n        return sysFlag | FLAG_UNIT_SUB;\n    }\n\n    public static int clearUnitSubFlag(final int sysFlag) {\n        return sysFlag & (~FLAG_UNIT_SUB);\n    }\n\n    public static boolean hasUnitSubFlag(final int sysFlag) {\n        return (sysFlag & FLAG_UNIT_SUB) == FLAG_UNIT_SUB;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.thread;\n\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.RejectedExecutionHandler;\nimport java.util.concurrent.RunnableFuture;\nimport java.util.concurrent.ThreadFactory;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.common.future.FutureTaskExt;\n\npublic class FutureTaskExtThreadPoolExecutor extends ThreadPoolExecutor {\n\n    public FutureTaskExtThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime,\n        TimeUnit unit,\n        BlockingQueue<Runnable> workQueue,\n        ThreadFactory threadFactory,\n        RejectedExecutionHandler handler) {\n        super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler);\n    }\n\n    @Override\n    protected <T> RunnableFuture<T> newTaskFor(final Runnable runnable, final T value) {\n        return new FutureTaskExt<>(runnable, value);\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.thread;\n\nimport com.google.common.collect.Lists;\nimport com.google.common.util.concurrent.ThreadFactoryBuilder;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.concurrent.CopyOnWriteArrayList;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class ThreadPoolMonitor {\n    private static Logger jstackLogger = LoggerFactory.getLogger(ThreadPoolMonitor.class);\n    private static Logger waterMarkLogger = LoggerFactory.getLogger(ThreadPoolMonitor.class);\n\n    private static final List<ThreadPoolWrapper> MONITOR_EXECUTOR = new CopyOnWriteArrayList<>();\n    private static final ScheduledExecutorService MONITOR_SCHEDULED = ThreadUtils.newSingleThreadScheduledExecutor(\n        new ThreadFactoryBuilder().setNameFormat(\"ThreadPoolMonitor-%d\").build()\n    );\n\n    private static volatile long threadPoolStatusPeriodTime = TimeUnit.SECONDS.toMillis(3);\n    private static volatile boolean enablePrintJstack = true;\n    private static volatile long jstackPeriodTime = 60000;\n    private static volatile long jstackTime = System.currentTimeMillis();\n\n    public static void config(Logger jstackLoggerConfig, Logger waterMarkLoggerConfig,\n        boolean enablePrintJstack, long jstackPeriodTimeConfig, long threadPoolStatusPeriodTimeConfig) {\n        jstackLogger = jstackLoggerConfig;\n        waterMarkLogger = waterMarkLoggerConfig;\n        threadPoolStatusPeriodTime = threadPoolStatusPeriodTimeConfig;\n        ThreadPoolMonitor.enablePrintJstack = enablePrintJstack;\n        jstackPeriodTime = jstackPeriodTimeConfig;\n    }\n\n    public static ThreadPoolExecutor createAndMonitor(int corePoolSize,\n        int maximumPoolSize,\n        long keepAliveTime,\n        TimeUnit unit,\n        String name,\n        int queueCapacity) {\n        return createAndMonitor(corePoolSize, maximumPoolSize, keepAliveTime, unit, name, queueCapacity, Collections.emptyList());\n    }\n\n    public static ThreadPoolExecutor createAndMonitor(int corePoolSize,\n        int maximumPoolSize,\n        long keepAliveTime,\n        TimeUnit unit,\n        String name,\n        int queueCapacity,\n        ThreadPoolStatusMonitor... threadPoolStatusMonitors) {\n        return createAndMonitor(corePoolSize, maximumPoolSize, keepAliveTime, unit, name, queueCapacity,\n            Lists.newArrayList(threadPoolStatusMonitors));\n    }\n\n    public static ThreadPoolExecutor createAndMonitor(int corePoolSize,\n        int maximumPoolSize,\n        long keepAliveTime,\n        TimeUnit unit,\n        String name,\n        int queueCapacity,\n        List<ThreadPoolStatusMonitor> threadPoolStatusMonitors) {\n        ThreadPoolExecutor executor = (ThreadPoolExecutor) ThreadUtils.newThreadPoolExecutor(\n            corePoolSize,\n            maximumPoolSize,\n            keepAliveTime,\n            unit,\n            new LinkedBlockingQueue<>(queueCapacity),\n            new ThreadFactoryBuilder().setNameFormat(name + \"-%d\").build(),\n            new ThreadPoolExecutor.DiscardOldestPolicy());\n        List<ThreadPoolStatusMonitor> printers = Lists.newArrayList(new ThreadPoolQueueSizeMonitor(queueCapacity));\n        printers.addAll(threadPoolStatusMonitors);\n\n        MONITOR_EXECUTOR.add(ThreadPoolWrapper.builder()\n            .name(name)\n            .threadPoolExecutor(executor)\n            .statusPrinters(printers)\n            .build());\n        return executor;\n    }\n\n    public static void logThreadPoolStatus() {\n        for (ThreadPoolWrapper threadPoolWrapper : MONITOR_EXECUTOR) {\n            List<ThreadPoolStatusMonitor> monitors = threadPoolWrapper.getStatusPrinters();\n            for (ThreadPoolStatusMonitor monitor : monitors) {\n                double value = monitor.value(threadPoolWrapper.getThreadPoolExecutor());\n                String nameFormatted = String.format(\"%-40s\", threadPoolWrapper.getName());\n                String descFormatted = String.format(\"%-12s\", monitor.describe());\n                waterMarkLogger.info(\"{}{}{}\", nameFormatted, descFormatted, value);\n                if (enablePrintJstack) {\n                    if (monitor.needPrintJstack(threadPoolWrapper.getThreadPoolExecutor(), value) &&\n                        System.currentTimeMillis() - jstackTime > jstackPeriodTime) {\n                        jstackTime = System.currentTimeMillis();\n                        jstackLogger.warn(\"jstack start\\n{}\", UtilAll.jstack());\n                    }\n                }\n            }\n        }\n    }\n\n    public static void init() {\n        MONITOR_SCHEDULED.scheduleAtFixedRate(ThreadPoolMonitor::logThreadPoolStatus, 20,\n            threadPoolStatusPeriodTime, TimeUnit.MILLISECONDS);\n    }\n\n    public static void shutdown() {\n        MONITOR_SCHEDULED.shutdown();\n    }\n}"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolQueueSizeMonitor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.thread;\n\nimport java.util.concurrent.ThreadPoolExecutor;\n\npublic class ThreadPoolQueueSizeMonitor implements ThreadPoolStatusMonitor {\n\n    private final int maxQueueCapacity;\n\n    public ThreadPoolQueueSizeMonitor(int maxQueueCapacity) {\n        this.maxQueueCapacity = maxQueueCapacity;\n    }\n\n    @Override\n    public String describe() {\n        return \"queueSize\";\n    }\n\n    @Override\n    public double value(ThreadPoolExecutor executor) {\n        return executor.getQueue().size();\n    }\n\n    @Override\n    public boolean needPrintJstack(ThreadPoolExecutor executor, double value) {\n        return value > maxQueueCapacity * 0.85;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolStatusMonitor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.thread;\n\nimport java.util.concurrent.ThreadPoolExecutor;\n\npublic interface ThreadPoolStatusMonitor {\n\n    String describe();\n\n    double value(ThreadPoolExecutor executor);\n\n    boolean needPrintJstack(ThreadPoolExecutor executor, double value);\n}"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolWrapper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.thread;\n\nimport com.google.common.base.MoreObjects;\nimport com.google.common.base.Objects;\nimport java.util.List;\nimport java.util.concurrent.ThreadPoolExecutor;\n\npublic class ThreadPoolWrapper {\n    private String name;\n    private ThreadPoolExecutor threadPoolExecutor;\n    private List<ThreadPoolStatusMonitor> statusPrinters;\n\n    ThreadPoolWrapper(final String name, final ThreadPoolExecutor threadPoolExecutor,\n        final List<ThreadPoolStatusMonitor> statusPrinters) {\n        this.name = name;\n        this.threadPoolExecutor = threadPoolExecutor;\n        this.statusPrinters = statusPrinters;\n    }\n\n    public static class ThreadPoolWrapperBuilder {\n        private String name;\n        private ThreadPoolExecutor threadPoolExecutor;\n        private List<ThreadPoolStatusMonitor> statusPrinters;\n\n        ThreadPoolWrapperBuilder() {\n        }\n\n        public ThreadPoolWrapper.ThreadPoolWrapperBuilder name(final String name) {\n            this.name = name;\n            return this;\n        }\n\n        public ThreadPoolWrapper.ThreadPoolWrapperBuilder threadPoolExecutor(\n            final ThreadPoolExecutor threadPoolExecutor) {\n            this.threadPoolExecutor = threadPoolExecutor;\n            return this;\n        }\n\n        public ThreadPoolWrapper.ThreadPoolWrapperBuilder statusPrinters(\n            final List<ThreadPoolStatusMonitor> statusPrinters) {\n            this.statusPrinters = statusPrinters;\n            return this;\n        }\n\n        public ThreadPoolWrapper build() {\n            return new ThreadPoolWrapper(this.name, this.threadPoolExecutor, this.statusPrinters);\n        }\n\n        @java.lang.Override\n        public java.lang.String toString() {\n            return \"ThreadPoolWrapper.ThreadPoolWrapperBuilder(name=\" + this.name + \", threadPoolExecutor=\" + this.threadPoolExecutor + \", statusPrinters=\" + this.statusPrinters + \")\";\n        }\n    }\n\n    public static ThreadPoolWrapper.ThreadPoolWrapperBuilder builder() {\n        return new ThreadPoolWrapper.ThreadPoolWrapperBuilder();\n    }\n\n    public String getName() {\n        return this.name;\n    }\n\n    public ThreadPoolExecutor getThreadPoolExecutor() {\n        return this.threadPoolExecutor;\n    }\n\n    public List<ThreadPoolStatusMonitor> getStatusPrinters() {\n        return this.statusPrinters;\n    }\n\n    public void setName(final String name) {\n        this.name = name;\n    }\n\n    public void setThreadPoolExecutor(final ThreadPoolExecutor threadPoolExecutor) {\n        this.threadPoolExecutor = threadPoolExecutor;\n    }\n\n    public void setStatusPrinters(final List<ThreadPoolStatusMonitor> statusPrinters) {\n        this.statusPrinters = statusPrinters;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o)\n            return true;\n        if (o == null || getClass() != o.getClass())\n            return false;\n        ThreadPoolWrapper wrapper = (ThreadPoolWrapper) o;\n        return Objects.equal(name, wrapper.name) && Objects.equal(threadPoolExecutor, wrapper.threadPoolExecutor) && Objects.equal(statusPrinters, wrapper.statusPrinters);\n    }\n\n    @Override\n    public int hashCode() {\n        return Objects.hashCode(name, threadPoolExecutor, statusPrinters);\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"name\", name)\n            .add(\"threadPoolExecutor\", threadPoolExecutor)\n            .add(\"statusPrinters\", statusPrinters)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.topic;\n\nimport java.util.HashSet;\nimport java.util.Set;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\n\npublic class TopicValidator {\n\n    public static final String AUTO_CREATE_TOPIC_KEY_TOPIC = \"TBW102\"; // Will be created at broker when isAutoCreateTopicEnable\n    public static final String RMQ_SYS_SCHEDULE_TOPIC = \"SCHEDULE_TOPIC_XXXX\";\n    public static final String RMQ_SYS_BENCHMARK_TOPIC = \"BenchmarkTest\";\n    public static final String RMQ_SYS_TRANS_HALF_TOPIC = \"RMQ_SYS_TRANS_HALF_TOPIC\";\n    public static final String RMQ_SYS_ROCKSDB_TRANS_HALF_TOPIC = \"RMQ_SYS_ROCKSDB_TRANS_HALF_TOPIC\";\n    public static final String RMQ_SYS_TRACE_TOPIC = \"RMQ_SYS_TRACE_TOPIC\";\n    public static final String RMQ_SYS_TRANS_OP_HALF_TOPIC = \"RMQ_SYS_TRANS_OP_HALF_TOPIC\";\n    public static final String RMQ_SYS_ROCKSDB_TRANS_OP_HALF_TOPIC = \"RMQ_SYS_ROCKSDB_TRANS_OP_HALF_TOPIC\";\n    public static final String RMQ_SYS_TRANS_CHECK_MAX_TIME_TOPIC = \"TRANS_CHECK_MAX_TIME_TOPIC\";\n    public static final String RMQ_SYS_SELF_TEST_TOPIC = \"SELF_TEST_TOPIC\";\n    public static final String RMQ_SYS_OFFSET_MOVED_EVENT = \"OFFSET_MOVED_EVENT\";\n    public static final String RMQ_SYS_ROCKSDB_OFFSET_TOPIC = \"CHECKPOINT_TOPIC\";\n\n    public static final String SYSTEM_TOPIC_PREFIX = \"rmq_sys_\";\n    public static final String SYNC_BROKER_MEMBER_GROUP_PREFIX = SYSTEM_TOPIC_PREFIX + \"SYNC_BROKER_MEMBER_\";\n\n    public static final boolean[] VALID_CHAR_BIT_MAP = new boolean[128];\n    private static final int TOPIC_MAX_LENGTH = 127;\n    /*\n     * Group name max length is 120, for it will be used to make up retry and DLQ topic,\n     * like pull retry: %RETRY%group_topic and pop retry: %RETRY%group_topic.\n     */\n    private static final int GROUP_MAX_LENGTH = 120;\n    private static final int RETRY_OR_DLQ_TOPIC_MAX_LENGTH = 255;\n\n    private static final Set<String> SYSTEM_TOPIC_SET = new HashSet<>();\n\n    /**\n     * Topic set which client can not send msg!\n     */\n    private static final Set<String> NOT_ALLOWED_SEND_TOPIC_SET = new HashSet<>();\n\n    static {\n        SYSTEM_TOPIC_SET.add(AUTO_CREATE_TOPIC_KEY_TOPIC);\n        SYSTEM_TOPIC_SET.add(RMQ_SYS_SCHEDULE_TOPIC);\n        SYSTEM_TOPIC_SET.add(RMQ_SYS_BENCHMARK_TOPIC);\n        SYSTEM_TOPIC_SET.add(RMQ_SYS_TRANS_HALF_TOPIC);\n        SYSTEM_TOPIC_SET.add(RMQ_SYS_TRACE_TOPIC);\n        SYSTEM_TOPIC_SET.add(RMQ_SYS_TRANS_OP_HALF_TOPIC);\n        SYSTEM_TOPIC_SET.add(RMQ_SYS_TRANS_CHECK_MAX_TIME_TOPIC);\n        SYSTEM_TOPIC_SET.add(RMQ_SYS_SELF_TEST_TOPIC);\n        SYSTEM_TOPIC_SET.add(RMQ_SYS_OFFSET_MOVED_EVENT);\n        SYSTEM_TOPIC_SET.add(RMQ_SYS_ROCKSDB_OFFSET_TOPIC);\n        SYSTEM_TOPIC_SET.add(RMQ_SYS_ROCKSDB_TRANS_HALF_TOPIC);\n        SYSTEM_TOPIC_SET.add(RMQ_SYS_ROCKSDB_TRANS_OP_HALF_TOPIC);\n\n        NOT_ALLOWED_SEND_TOPIC_SET.add(RMQ_SYS_SCHEDULE_TOPIC);\n        NOT_ALLOWED_SEND_TOPIC_SET.add(RMQ_SYS_TRANS_HALF_TOPIC);\n        NOT_ALLOWED_SEND_TOPIC_SET.add(RMQ_SYS_TRANS_OP_HALF_TOPIC);\n        NOT_ALLOWED_SEND_TOPIC_SET.add(RMQ_SYS_TRANS_CHECK_MAX_TIME_TOPIC);\n        NOT_ALLOWED_SEND_TOPIC_SET.add(RMQ_SYS_SELF_TEST_TOPIC);\n        NOT_ALLOWED_SEND_TOPIC_SET.add(RMQ_SYS_OFFSET_MOVED_EVENT);\n        NOT_ALLOWED_SEND_TOPIC_SET.add(RMQ_SYS_ROCKSDB_TRANS_HALF_TOPIC);\n        NOT_ALLOWED_SEND_TOPIC_SET.add(RMQ_SYS_ROCKSDB_TRANS_OP_HALF_TOPIC);\n\n        // regex: ^[%|a-zA-Z0-9_-]+$\n        // %\n        VALID_CHAR_BIT_MAP['%'] = true;\n        // -\n        VALID_CHAR_BIT_MAP['-'] = true;\n        // _\n        VALID_CHAR_BIT_MAP['_'] = true;\n        // |\n        VALID_CHAR_BIT_MAP['|'] = true;\n        for (int i = 0; i < VALID_CHAR_BIT_MAP.length; i++) {\n            if (i >= '0' && i <= '9') {\n                // 0-9\n                VALID_CHAR_BIT_MAP[i] = true;\n            } else if (i >= 'A' && i <= 'Z') {\n                // A-Z\n                VALID_CHAR_BIT_MAP[i] = true;\n            } else if (i >= 'a' && i <= 'z') {\n                // a-z\n                VALID_CHAR_BIT_MAP[i] = true;\n            }\n        }\n    }\n\n    public static boolean isTopicOrGroupIllegal(String str) {\n        int strLen = str.length();\n        int len = VALID_CHAR_BIT_MAP.length;\n        for (int i = 0; i < strLen; i++) {\n            char ch = str.charAt(i);\n            if (ch >= len || !VALID_CHAR_BIT_MAP[ch]) {\n                return true;\n            }\n        }\n        return false;\n    }\n\n    public static ValidateResult validateTopic(String topic) {\n\n        if (UtilAll.isBlank(topic)) {\n            return new ValidateResult(false, \"The specified topic is blank.\");\n        }\n\n        if (isTopicOrGroupIllegal(topic)) {\n            String falseRemark = \"The specified topic: \" + topic + \", contains illegal characters, allowing only ^[%|a-zA-Z0-9_-]+$\";\n            return new ValidateResult(false, falseRemark);\n        }\n\n        if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) || topic.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX)) {\n            if (topic.length() > RETRY_OR_DLQ_TOPIC_MAX_LENGTH) {\n                String falseRemark = \"The specified topic is DLQ or Retry topic: \" + topic + \", and it's longer than topic max length: \" + RETRY_OR_DLQ_TOPIC_MAX_LENGTH;\n                return new ValidateResult(false, falseRemark);\n            }\n        } else {\n            if (topic.length() > TOPIC_MAX_LENGTH) {\n                String falseRemark = \"The specified topic: \" + topic + \", is longer than topic max length: \" + TOPIC_MAX_LENGTH;\n                return new ValidateResult(false, falseRemark);\n            }\n        }\n\n        return new ValidateResult(true, \"\");\n    }\n\n    public static ValidateResult validateGroup(String group) {\n\n        if (UtilAll.isBlank(group)) {\n            return new ValidateResult(false, \"The specified group is blank.\");\n        }\n\n        if (isTopicOrGroupIllegal(group)) {\n            String falseRemark = \"The specified group: \" + group + \", contains illegal characters, allowing only ^[%|a-zA-Z0-9_-]+$\";\n            return new ValidateResult(false, falseRemark);\n        }\n\n        if (group.length() > GROUP_MAX_LENGTH) {\n            String falseRemark = \"The specified group: \" + group + \", is longer than group max length: \" + GROUP_MAX_LENGTH;\n            return new ValidateResult(false, falseRemark);\n        }\n\n        return new ValidateResult(true, \"\");\n    }\n\n    public static class ValidateResult {\n        private final boolean valid;\n        private final String remark;\n\n        public ValidateResult(boolean valid, String remark) {\n            this.valid = valid;\n            this.remark = remark;\n        }\n\n        public boolean isValid() {\n            return valid;\n        }\n\n        public String getRemark() {\n            return remark;\n        }\n    }\n\n    public static boolean isSystemTopic(String topic) {\n        return SYSTEM_TOPIC_SET.contains(topic) || topic.startsWith(SYSTEM_TOPIC_PREFIX);\n    }\n\n    public static boolean isNotAllowedSendTopic(String topic) {\n        return NOT_ALLOWED_SEND_TOPIC_SET.contains(topic);\n    }\n\n    public static void addSystemTopic(String systemTopic) {\n        SYSTEM_TOPIC_SET.add(systemTopic);\n    }\n\n    public static Set<String> getSystemTopicSet() {\n        return SYSTEM_TOPIC_SET;\n    }\n\n    public static Set<String> getNotAllowedSendTopicSet() {\n        return NOT_ALLOWED_SEND_TOPIC_SET;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/AbstractStartAndShutdown.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.utils;\n\nimport java.util.List;\nimport java.util.concurrent.CopyOnWriteArrayList;\n\npublic abstract class AbstractStartAndShutdown implements StartAndShutdown {\n\n    protected List<StartAndShutdown> startAndShutdownList = new CopyOnWriteArrayList<>();\n\n    protected void appendStartAndShutdown(StartAndShutdown startAndShutdown) {\n        this.startAndShutdownList.add(startAndShutdown);\n    }\n\n    @Override\n    public void start() throws Exception {\n        for (StartAndShutdown startAndShutdown : startAndShutdownList) {\n            startAndShutdown.start();\n        }\n    }\n\n    @Override\n    public void shutdown() throws Exception {\n        int index = startAndShutdownList.size() - 1;\n        for (; index >= 0; index--) {\n            startAndShutdownList.get(index).shutdown();\n        }\n    }\n\n    @Override\n    public void preShutdown() throws Exception {\n        int index = startAndShutdownList.size() - 1;\n        for (; index >= 0; index--) {\n            startAndShutdownList.get(index).preShutdown();\n        }\n    }\n\n    public void appendStart(Start start) {\n        this.appendStartAndShutdown(new StartAndShutdown() {\n            @Override\n            public void shutdown() throws Exception {\n\n            }\n\n            @Override\n            public void start() throws Exception {\n                start.start();\n            }\n        });\n    }\n\n    public void appendShutdown(Shutdown shutdown) {\n        this.appendStartAndShutdown(new StartAndShutdown() {\n            @Override\n            public void shutdown() throws Exception {\n                shutdown.shutdown();\n            }\n\n            @Override\n            public void start() throws Exception {\n\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/AsyncShutdownHelper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.utils;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicBoolean;\n\npublic class AsyncShutdownHelper {\n    private final AtomicBoolean shutdown;\n    private final List<Shutdown> targetList;\n\n    private CountDownLatch countDownLatch;\n\n    public AsyncShutdownHelper() {\n        this.targetList = new ArrayList<>();\n        this.shutdown = new AtomicBoolean(false);\n    }\n\n    public void addTarget(Shutdown target) {\n        if (shutdown.get()) {\n            return;\n        }\n        targetList.add(target);\n    }\n\n    public AsyncShutdownHelper shutdown() {\n        if (shutdown.get()) {\n            return this;\n        }\n        if (targetList.isEmpty()) {\n            return this;\n        }\n        this.countDownLatch = new CountDownLatch(targetList.size());\n        for (Shutdown target : targetList) {\n            Runnable runnable = () -> {\n                try {\n                    target.shutdown();\n                } catch (Exception ignored) {\n\n                } finally {\n                    countDownLatch.countDown();\n                }\n            };\n            new Thread(runnable).start();\n        }\n        return this;\n    }\n\n    public boolean await(long time, TimeUnit unit) throws InterruptedException {\n        if (shutdown.get()) {\n            return false;\n        }\n        try {\n            return this.countDownLatch.await(time, unit);\n        } finally {\n            shutdown.compareAndSet(false, true);\n        }\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.utils;\n\nimport java.nio.charset.Charset;\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport org.apache.commons.codec.binary.Hex;\n\npublic class BinaryUtil {\n    public static byte[] calculateMd5(byte[] binaryData) {\n        MessageDigest messageDigest = null;\n        try {\n            messageDigest = MessageDigest.getInstance(\"MD5\");\n        } catch (NoSuchAlgorithmException e) {\n            throw new RuntimeException(\"MD5 algorithm not found.\");\n        }\n        messageDigest.update(binaryData);\n        return messageDigest.digest();\n    }\n\n    public static String generateMd5(String bodyStr) {\n        byte[] bytes = calculateMd5(bodyStr.getBytes(Charset.forName(\"UTF-8\")));\n        return Hex.encodeHexString(bytes, false);\n    }\n\n    public static String generateMd5(byte[] content) {\n        byte[] bytes = calculateMd5(content);\n        return Hex.encodeHexString(bytes, false);\n    }\n\n    /**\n     * Returns true if subject contains only bytes that are spec-compliant ASCII characters.\n     * @param subject\n     * @return\n     */\n    public static boolean isAscii(byte[] subject) {\n        if (subject == null) {\n            return false;\n        }\n        for (byte b : subject) {\n            if (b < 32 || b > 126) {\n                return false;\n            }\n        }\n        return true;\n    }\n}"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/ChannelUtil.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.utils;\n\nimport io.netty.channel.Channel;\nimport java.net.InetAddress;\nimport java.net.InetSocketAddress;\n\npublic class ChannelUtil {\n    public static String getRemoteIp(Channel channel) {\n        InetSocketAddress inetSocketAddress = (InetSocketAddress) channel.remoteAddress();\n        if (inetSocketAddress == null) {\n            return \"\";\n        }\n        final InetAddress inetAddr = inetSocketAddress.getAddress();\n        return inetAddr != null ? inetAddr.getHostAddress() : inetSocketAddress.getHostName();\n    }\n\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/CheckpointFile.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.utils;\n\nimport java.io.BufferedReader;\nimport java.io.File;\nimport java.io.IOException;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\n\n/**\n * Entry Checkpoint file util\n * Format:\n * <li>First line:  Entries size\n * <li>Second line: Entries crc32\n * <li>Next: Entry data per line\n * <p>\n * Example:\n * <li>2 (size)\n * <li>773307083 (crc32)\n * <li>7-7000 (entry data)\n * <li>8-8000 (entry data)\n */\npublic class CheckpointFile<T> {\n\n    /**\n     * Not check crc32 when value is 0\n     */\n    private static final int NOT_CHECK_CRC_MAGIC_CODE = 0;\n    private final String filePath;\n    private final CheckpointSerializer<T> serializer;\n\n    public interface CheckpointSerializer<T> {\n        /**\n         * Serialize entry to line\n         */\n        String toLine(final T entry);\n\n        /**\n         * DeSerialize line to entry\n         */\n        T fromLine(final String line);\n    }\n\n    public CheckpointFile(final String filePath, final CheckpointSerializer<T> serializer) {\n        this.filePath = filePath;\n        this.serializer = serializer;\n    }\n\n    public String getBackFilePath() {\n        return this.filePath + \".bak\";\n    }\n\n    /**\n     * Write entries to file\n     */\n    public void write(final List<T> entries) throws IOException {\n        if (entries.isEmpty()) {\n            return;\n        }\n        synchronized (this) {\n            StringBuilder entryContent = new StringBuilder();\n            for (T entry : entries) {\n                final String line = this.serializer.toLine(entry);\n                if (line != null && !line.isEmpty()) {\n                    entryContent.append(line);\n                    entryContent.append(System.lineSeparator());\n                }\n            }\n            int crc32 = UtilAll.crc32(entryContent.toString().getBytes(StandardCharsets.UTF_8));\n\n            String content = entries.size() + System.lineSeparator() +\n                crc32 + System.lineSeparator() + entryContent;\n            MixAll.string2File(content, this.filePath);\n        }\n    }\n\n    private List<T> read(String filePath) throws IOException {\n        final ArrayList<T> result = new ArrayList<>();\n        synchronized (this) {\n            final File file = new File(filePath);\n            if (!file.exists()) {\n                return result;\n            }\n            try (BufferedReader reader = Files.newBufferedReader(file.toPath())) {\n                // Read size\n                int expectedLines = Integer.parseInt(reader.readLine());\n\n                // Read block crc\n                int expectedCrc32 = Integer.parseInt(reader.readLine());\n\n                // Read entries\n                StringBuilder sb = new StringBuilder();\n                String line = reader.readLine();\n                while (line != null) {\n                    sb.append(line).append(System.lineSeparator());\n                    final T entry = this.serializer.fromLine(line);\n                    if (entry != null) {\n                        result.add(entry);\n                    }\n                    line = reader.readLine();\n                }\n                int truthCrc32 = UtilAll.crc32(sb.toString().getBytes(StandardCharsets.UTF_8));\n\n                if (result.size() != expectedLines) {\n                    final String err = String.format(\n                        \"Expect %d entries, only found %d entries\", expectedLines, result.size());\n                    throw new IOException(err);\n                }\n\n                if (NOT_CHECK_CRC_MAGIC_CODE != expectedCrc32 && truthCrc32 != expectedCrc32) {\n                    final String err = String.format(\n                        \"Entries crc32 not match, file=%s, truth=%s\", expectedCrc32, truthCrc32);\n                    throw new IOException(err);\n                }\n                return result;\n            }\n        }\n    }\n\n    /**\n     * Read entries from file\n     */\n    public List<T> read() throws IOException {\n        try {\n            List<T> result = this.read(this.filePath);\n            if (CollectionUtils.isEmpty(result)) {\n                result = this.read(this.getBackFilePath());\n            }\n            return result;\n        } catch (IOException e) {\n            return this.read(this.getBackFilePath());\n        }\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/CleanupPolicyUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.utils;\n\nimport org.apache.rocketmq.common.TopicAttributes;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.attribute.CleanupPolicy;\n\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Optional;\n\npublic class CleanupPolicyUtils {\n    public static boolean isCompaction(Optional<TopicConfig> topicConfig) {\n        return Objects.equals(CleanupPolicy.COMPACTION, getDeletePolicy(topicConfig));\n    }\n\n    public static CleanupPolicy getDeletePolicy(Optional<TopicConfig> topicConfig) {\n        if (!topicConfig.isPresent()) {\n            return CleanupPolicy.valueOf(TopicAttributes.CLEANUP_POLICY_ATTRIBUTE.getDefaultValue());\n        }\n\n        String attributeName = TopicAttributes.CLEANUP_POLICY_ATTRIBUTE.getName();\n\n        Map<String, String> attributes = topicConfig.get().getAttributes();\n        if (attributes == null || attributes.size() == 0) {\n            return CleanupPolicy.valueOf(TopicAttributes.CLEANUP_POLICY_ATTRIBUTE.getDefaultValue());\n        }\n\n        if (attributes.containsKey(attributeName)) {\n            return CleanupPolicy.valueOf(attributes.get(attributeName));\n        } else {\n            return CleanupPolicy.valueOf(TopicAttributes.CLEANUP_POLICY_ATTRIBUTE.getDefaultValue());\n        }\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/ConcurrentHashMapUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.utils;\n\nimport java.util.Objects;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.function.Function;\n\npublic abstract class ConcurrentHashMapUtils {\n\n    private static boolean isJdk8;\n\n    static {\n        // Java 8\n        // Java 9+: 9,11,17\n        try {\n            isJdk8 = System.getProperty(\"java.version\").startsWith(\"1.8.\");\n        } catch (Exception ignore) {\n            isJdk8 = true;\n        }\n    }\n\n    /**\n     * A temporary workaround for Java 8 specific performance issue JDK-8161372 .<br> Use implementation of\n     * ConcurrentMap.computeIfAbsent instead.\n     * \n     * Requirement: <strong>The mapping function should not modify this map during computation.</strong>\n     *\n     * @see <a href=\"https://bugs.openjdk.java.net/browse/JDK-8161372\">https://bugs.openjdk.java.net/browse/JDK-8161372</a>\n     */\n    public static <K, V> V computeIfAbsent(ConcurrentMap<K, V> map, K key, Function<? super K, ? extends V> func) {\n        Objects.requireNonNull(func);\n        if (isJdk8) {\n            V v = map.get(key);\n            if (null == v) {\n                // this bug fix methods maybe cause `func.apply` multiple calls.\n                v = func.apply(key);\n                if (null == v) {\n                    return null;\n                }\n                final V res = map.putIfAbsent(key, v);\n                if (null != res) {\n                    // if pre value present, means other thread put value already, and putIfAbsent not effect\n                    // return exist value\n                    return res;\n                }\n            }\n            return v;\n        } else {\n            return map.computeIfAbsent(key, func);\n        }\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/CorrelationIdUtil.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.utils;\n\nimport java.util.UUID;\n\npublic class CorrelationIdUtil {\n    public static String createCorrelationId() {\n        return UUID.randomUUID().toString();\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/DataConverter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.utils;\n\nimport java.nio.ByteBuffer;\nimport java.nio.charset.Charset;\n\npublic class DataConverter {\n    public static final Charset CHARSET_UTF8 = Charset.forName(\"UTF-8\");\n\n    public static byte[] Long2Byte(Long v) {\n        ByteBuffer tmp = ByteBuffer.allocate(8);\n        tmp.putLong(v);\n        return tmp.array();\n    }\n\n    public static int setBit(int value, int index, boolean flag) {\n        if (flag) {\n            return (int) (value | (1L << index));\n        } else {\n            return (int) (value & ~(1L << index));\n        }\n    }\n\n    public static boolean getBit(int value, int index) {\n        return (value & (1L << index)) != 0;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/ExceptionUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.utils;\n\nimport java.util.concurrent.CompletionException;\nimport java.util.concurrent.ExecutionException;\n\npublic class ExceptionUtils {\n\n    public static Throwable getRealException(Throwable throwable) {\n        if (throwable instanceof CompletionException || throwable instanceof ExecutionException) {\n            if (throwable.getCause() != null) {\n                throwable = throwable.getCause();\n            }\n        }\n        return throwable;\n    }\n\n    public static String getErrorDetailMessage(Throwable t) {\n        if (t == null) {\n            return null;\n        }\n        StringBuilder sb = new StringBuilder();\n        sb.append(t.getMessage()).append(\". \").append(t.getClass().getSimpleName());\n\n        if (t.getStackTrace().length > 0) {\n            sb.append(\". \").append(t.getStackTrace()[0]);\n        }\n        return sb.toString();\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/FastJsonSerializer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.utils;\n\nimport com.alibaba.fastjson2.JSON;\nimport org.apache.commons.lang3.SerializationException;\n\n/**\n * The object serializer based on fastJson\n */\npublic class FastJsonSerializer implements Serializer {\n\n    @Override\n    public <T> byte[] serialize(T t) throws SerializationException {\n        if (t == null) {\n            return new byte[0];\n        } else {\n            try {\n                return JSON.toJSONBytes(t);\n            } catch (Exception var3) {\n                throw new SerializationException(\"Could not serialize: \" + var3.getMessage(), var3);\n            }\n        }\n    }\n\n    @Override\n    public <T> T deserialize(byte[] bytes, Class<T> type) throws SerializationException {\n        if (bytes != null && bytes.length != 0) {\n            try {\n                return JSON.parseObject(bytes, type);\n            } catch (Exception var3) {\n                throw new SerializationException(\"Could not deserialize: \" + var3.getMessage(), var3);\n            }\n        } else {\n            return null;\n        }\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/FutureUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.utils;\n\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ExecutorService;\n\npublic class FutureUtils {\n\n    public static <T> CompletableFuture<T> appendNextFuture(CompletableFuture<T> future,\n        CompletableFuture<T> nextFuture, ExecutorService executor) {\n        future.whenCompleteAsync((t, throwable) -> {\n            if (throwable != null) {\n                nextFuture.completeExceptionally(throwable);\n            } else {\n                nextFuture.complete(t);\n            }\n        }, executor);\n        return nextFuture;\n    }\n\n    public static <T> CompletableFuture<T> addExecutor(CompletableFuture<T> future, ExecutorService executor) {\n        return appendNextFuture(future, new CompletableFuture<>(), executor);\n    }\n\n    public static <T> CompletableFuture<T> completeExceptionally(Throwable t) {\n        CompletableFuture<T> future = new CompletableFuture<>();\n        future.completeExceptionally(t);\n        return future;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/HttpTinyClient.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.utils;\n\nimport java.io.IOException;\nimport java.io.UnsupportedEncodingException;\nimport java.net.HttpURLConnection;\nimport java.net.URL;\nimport java.net.URLEncoder;\nimport java.util.Iterator;\nimport java.util.List;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.MixAll;\n\npublic class HttpTinyClient {\n\n    static public HttpResult httpGet(String url, List<String> headers, List<String> paramValues,\n        String encoding, long readTimeoutMs) throws IOException {\n        String encodedContent = encodingParams(paramValues, encoding);\n        url += (null == encodedContent) ? \"\" : (\"?\" + encodedContent);\n\n        HttpURLConnection conn = null;\n        try {\n            conn = (HttpURLConnection) new URL(url).openConnection();\n            conn.setRequestMethod(\"GET\");\n            conn.setConnectTimeout((int) readTimeoutMs);\n            conn.setReadTimeout((int) readTimeoutMs);\n            setHeaders(conn, headers, encoding);\n\n            conn.connect();\n            int respCode = conn.getResponseCode();\n            String resp = null;\n\n            if (HttpURLConnection.HTTP_OK == respCode) {\n                resp = IOTinyUtils.toString(conn.getInputStream(), encoding);\n            } else {\n                resp = IOTinyUtils.toString(conn.getErrorStream(), encoding);\n            }\n            return new HttpResult(respCode, resp);\n        } finally {\n            if (conn != null) {\n                conn.disconnect();\n            }\n        }\n    }\n\n    static private String encodingParams(List<String> paramValues, String encoding)\n        throws UnsupportedEncodingException {\n        StringBuilder sb = new StringBuilder();\n        if (null == paramValues) {\n            return null;\n        }\n\n        for (Iterator<String> iter = paramValues.iterator(); iter.hasNext(); ) {\n            sb.append(iter.next()).append(\"=\");\n            sb.append(URLEncoder.encode(iter.next(), encoding));\n            if (iter.hasNext()) {\n                sb.append(\"&\");\n            }\n        }\n        return sb.toString();\n    }\n\n    static private void setHeaders(HttpURLConnection conn, List<String> headers, String encoding) {\n        if (null != headers) {\n            for (Iterator<String> iter = headers.iterator(); iter.hasNext(); ) {\n                conn.addRequestProperty(iter.next(), iter.next());\n            }\n        }\n        conn.addRequestProperty(\"Client-Version\", MQVersion.getVersionDesc(MQVersion.CURRENT_VERSION));\n        conn.addRequestProperty(\"Content-Type\", \"application/x-www-form-urlencoded;charset=\" + encoding);\n\n        String ts = String.valueOf(System.currentTimeMillis());\n        conn.addRequestProperty(\"Metaq-Client-RequestTS\", ts);\n    }\n\n    /**\n     * @return the http response of given http post request\n     */\n    static public HttpResult httpPost(String url, List<String> headers, List<String> paramValues,\n        String encoding, long readTimeoutMs) throws IOException {\n        String encodedContent = encodingParams(paramValues, encoding);\n\n        HttpURLConnection conn = null;\n        try {\n            conn = (HttpURLConnection) new URL(url).openConnection();\n            conn.setRequestMethod(\"POST\");\n            conn.setConnectTimeout(3000);\n            conn.setReadTimeout((int) readTimeoutMs);\n            conn.setDoOutput(true);\n            conn.setDoInput(true);\n            setHeaders(conn, headers, encoding);\n\n            conn.getOutputStream().write(encodedContent.getBytes(MixAll.DEFAULT_CHARSET));\n\n            int respCode = conn.getResponseCode();\n            String resp = null;\n\n            if (HttpURLConnection.HTTP_OK == respCode) {\n                resp = IOTinyUtils.toString(conn.getInputStream(), encoding);\n            } else {\n                resp = IOTinyUtils.toString(conn.getErrorStream(), encoding);\n            }\n            return new HttpResult(respCode, resp);\n        } finally {\n            if (null != conn) {\n                conn.disconnect();\n            }\n        }\n    }\n\n    static public class HttpResult {\n        final public int code;\n        final public String content;\n\n        public HttpResult(int code, String content) {\n            this.code = code;\n            this.content = content;\n        }\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/IOTinyUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.utils;\n\nimport java.io.BufferedReader;\nimport java.io.CharArrayWriter;\nimport java.io.File;\nimport java.io.FileInputStream;\nimport java.io.FileOutputStream;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.InputStreamReader;\nimport java.io.OutputStream;\nimport java.io.Reader;\nimport java.io.Writer;\nimport java.nio.channels.FileChannel;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.List;\n\npublic class IOTinyUtils {\n\n    static public String toString(InputStream input, String encoding) throws IOException {\n        return (null == encoding) ? toString(new InputStreamReader(input, StandardCharsets.UTF_8)) : toString(new InputStreamReader(\n            input, encoding));\n    }\n\n    static public String toString(Reader reader) throws IOException {\n        CharArrayWriter sw = new CharArrayWriter();\n        copy(reader, sw);\n        return sw.toString();\n    }\n\n    static public long copy(Reader input, Writer output) throws IOException {\n        char[] buffer = new char[1 << 12];\n        long count = 0;\n        for (int n = 0; (n = input.read(buffer)) >= 0; ) {\n            output.write(buffer, 0, n);\n            count += n;\n        }\n        return count;\n    }\n\n    static public List<String> readLines(Reader input) throws IOException {\n        BufferedReader reader = toBufferedReader(input);\n        List<String> list = new ArrayList<>();\n        String line;\n        for (; ; ) {\n            line = reader.readLine();\n            if (null != line) {\n                list.add(line);\n            } else {\n                break;\n            }\n        }\n        return list;\n    }\n\n    static private BufferedReader toBufferedReader(Reader reader) {\n        return reader instanceof BufferedReader ? (BufferedReader) reader : new BufferedReader(reader);\n    }\n\n    static public void copyFile(String source, String target) throws IOException {\n        File sf = new File(source);\n        if (!sf.exists()) {\n            throw new IllegalArgumentException(\"source file does not exist.\");\n        }\n        File tf = new File(target);\n        tf.getParentFile().mkdirs();\n        if (!tf.exists() && !tf.createNewFile()) {\n            throw new RuntimeException(\"failed to create target file.\");\n        }\n\n        FileChannel sc = null;\n        FileChannel tc = null;\n        try {\n            tc = new FileOutputStream(tf).getChannel();\n            sc = new FileInputStream(sf).getChannel();\n            sc.transferTo(0, sc.size(), tc);\n        } finally {\n            if (null != sc) {\n                sc.close();\n            }\n            if (null != tc) {\n                tc.close();\n            }\n        }\n    }\n\n    public static void delete(File fileOrDir) throws IOException {\n        if (fileOrDir == null) {\n            return;\n        }\n\n        if (fileOrDir.isDirectory()) {\n            cleanDirectory(fileOrDir);\n        }\n\n        fileOrDir.delete();\n    }\n\n    public static void cleanDirectory(File directory) throws IOException {\n        if (!directory.exists()) {\n            String message = directory + \" does not exist\";\n            throw new IllegalArgumentException(message);\n        }\n\n        if (!directory.isDirectory()) {\n            String message = directory + \" is not a directory\";\n            throw new IllegalArgumentException(message);\n        }\n\n        File[] files = directory.listFiles();\n        if (files == null) { // null if security restricted\n            throw new IOException(\"Failed to list contents of \" + directory);\n        }\n\n        IOException exception = null;\n        for (File file : files) {\n            try {\n                delete(file);\n            } catch (IOException ioe) {\n                exception = ioe;\n            }\n        }\n\n        if (null != exception) {\n            throw exception;\n        }\n    }\n\n    public static void writeStringToFile(File file, String data, String encoding) throws IOException {\n        OutputStream os = null;\n        try {\n            os = new FileOutputStream(file);\n            os.write(data.getBytes(encoding));\n        } finally {\n            if (null != os) {\n                os.close();\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/IPAddressUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.utils;\n\nimport java.math.BigInteger;\nimport java.net.InetAddress;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.commons.validator.routines.InetAddressValidator;\n\npublic class IPAddressUtils {\n\n    private static final String SLASH = \"/\";\n\n    private static final InetAddressValidator VALIDATOR = InetAddressValidator.getInstance();\n\n    public static boolean isValidIPOrCidr(String ipOrCidr) {\n        return isValidIp(ipOrCidr) || isValidCidr(ipOrCidr);\n    }\n\n    public static boolean isValidIp(String ip) {\n        return VALIDATOR.isValid(ip);\n    }\n\n    public static boolean isValidIPv4(String ip) {\n        return VALIDATOR.isValidInet4Address(ip);\n    }\n\n    public static boolean isValidIPv6(String ip) {\n        return VALIDATOR.isValidInet6Address(ip);\n    }\n\n    public static boolean isValidCidr(String cidr) {\n        return isValidIPv4Cidr(cidr) || isValidIPv6Cidr(cidr);\n    }\n\n    public static boolean isValidIPv4Cidr(String cidr) {\n        try {\n            String[] parts = cidr.split(SLASH);\n            if (parts.length != 2) {\n                return false;\n            }\n            InetAddress ip = InetAddress.getByName(parts[0]);\n            if (ip.getAddress().length != 4) {\n                return false;\n            }\n            int prefix = Integer.parseInt(parts[1]);\n            return prefix >= 0 && prefix <= 32;\n        } catch (Exception e) {\n            return false;\n        }\n    }\n\n    public static boolean isValidIPv6Cidr(String cidr) {\n        try {\n            String[] parts = cidr.split(SLASH);\n            if (parts.length != 2) {\n                return false;\n            }\n            InetAddress ip = InetAddress.getByName(parts[0]);\n            if (ip.getAddress().length != 16) {\n                return false;\n            }\n            int prefix = Integer.parseInt(parts[1]);\n            return prefix >= 0 && prefix <= 128;\n        } catch (Exception e) {\n            return false;\n        }\n    }\n\n    public static boolean isIPInRange(String ip, String cidr) {\n        try {\n            String[] parts = cidr.split(SLASH);\n            if (parts.length == 1) {\n                return StringUtils.equals(ip, cidr);\n            }\n            if (parts.length != 2) {\n                return false;\n            }\n            InetAddress cidrIp = InetAddress.getByName(parts[0]);\n            int prefixLength = Integer.parseInt(parts[1]);\n\n            BigInteger cidrIpBigInt = new BigInteger(1, cidrIp.getAddress());\n            BigInteger ipBigInt = new BigInteger(1, InetAddress.getByName(ip).getAddress());\n\n            BigInteger mask = BigInteger.valueOf(-1).shiftLeft(cidrIp.getAddress().length * 8 - prefixLength);\n            BigInteger cidrIpLower = cidrIpBigInt.and(mask);\n            BigInteger cidrIpUpper = cidrIpLower.add(mask.not());\n\n            return ipBigInt.compareTo(cidrIpLower) >= 0 && ipBigInt.compareTo(cidrIpUpper) <= 0;\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/MessageUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.utils;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.Collection;\nimport java.util.HashSet;\nimport java.util.Set;\n\nimport com.google.common.hash.Hashing;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\n\nimport static org.apache.rocketmq.common.message.MessageDecoder.NAME_VALUE_SEPARATOR;\nimport static org.apache.rocketmq.common.message.MessageDecoder.PROPERTY_SEPARATOR;\n\npublic class MessageUtils {\n\n    public static int getShardingKeyIndex(String shardingKey, int indexSize) {\n        return Math.abs(Hashing.murmur3_32().hashBytes(shardingKey.getBytes(StandardCharsets.UTF_8)).asInt() % indexSize);\n    }\n\n    public static int getShardingKeyIndexByMsg(MessageExt msg, int indexSize) {\n        String shardingKey = msg.getProperty(MessageConst.PROPERTY_SHARDING_KEY);\n        if (shardingKey == null) {\n            shardingKey = \"\";\n        }\n\n        return getShardingKeyIndex(shardingKey, indexSize);\n    }\n\n    public static Set<Integer> getShardingKeyIndexes(Collection<MessageExt> msgs, int indexSize) {\n        Set<Integer> indexSet = new HashSet<>(indexSize);\n        for (MessageExt msg : msgs) {\n            indexSet.add(getShardingKeyIndexByMsg(msg, indexSize));\n        }\n        return indexSet;\n    }\n\n    public static String deleteProperty(String propertiesString, String name) {\n        if (propertiesString != null) {\n            int idx0 = 0;\n            int idx1;\n            int idx2;\n            idx1 = propertiesString.indexOf(name, idx0);\n            if (idx1 != -1) {\n                // cropping may be required\n                StringBuilder stringBuilder = new StringBuilder(propertiesString.length());\n                while (true) {\n                    int startIdx = idx0;\n                    while (true) {\n                        idx1 = propertiesString.indexOf(name, startIdx);\n                        if (idx1 == -1) {\n                            break;\n                        }\n                        startIdx = idx1 + name.length();\n                        if (idx1 == 0 || propertiesString.charAt(idx1 - 1) == PROPERTY_SEPARATOR) {\n                            if (propertiesString.length() > idx1 + name.length()\n                                && propertiesString.charAt(idx1 + name.length()) == NAME_VALUE_SEPARATOR) {\n                                break;\n                            }\n                        }\n                    }\n                    if (idx1 == -1) {\n                        // there are no characters that need to be skipped. Append all remaining characters.\n                        stringBuilder.append(propertiesString, idx0, propertiesString.length());\n                        break;\n                    }\n                    // there are characters that need to be cropped\n                    stringBuilder.append(propertiesString, idx0, idx1);\n                    // move idx2 to the end of the cropped character\n                    idx2 = propertiesString.indexOf(PROPERTY_SEPARATOR, idx1 + name.length() + 1);\n                    // all subsequent characters will be cropped\n                    if (idx2 == -1) {\n                        break;\n                    }\n                    idx0 = idx2 + 1;\n                }\n                return stringBuilder.toString();\n            }\n        }\n        return propertiesString;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/NameServerAddressUtils.java",
    "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE\n * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file\n * to You under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the\n * License. You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on\n * an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations under the License.\n */\npackage org.apache.rocketmq.common.utils;\n\nimport java.util.regex.Pattern;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.MixAll;\n\npublic class NameServerAddressUtils {\n    public static final String INSTANCE_PREFIX = \"MQ_INST_\";\n    public static final String INSTANCE_REGEX = INSTANCE_PREFIX + \"\\\\w+_\\\\w+\";\n    public static final String ENDPOINT_PREFIX = \"(\\\\w+://|)\";\n    public static final Pattern NAMESRV_ENDPOINT_PATTERN = Pattern.compile(\"^http://.*\");\n    public static final Pattern INST_ENDPOINT_PATTERN = Pattern.compile(\"^\" + ENDPOINT_PREFIX + INSTANCE_REGEX + \"\\\\..*\");\n\n    public static String getNameServerAddresses() {\n        return System.getProperty(MixAll.NAMESRV_ADDR_PROPERTY, System.getenv(MixAll.NAMESRV_ADDR_ENV));\n    }\n\n    public static boolean validateInstanceEndpoint(String endpoint) {\n        return INST_ENDPOINT_PATTERN.matcher(endpoint).matches();\n    }\n\n    public static String parseInstanceIdFromEndpoint(String endpoint) {\n        if (StringUtils.isEmpty(endpoint)) {\n            return null;\n        }\n        return endpoint.substring(endpoint.lastIndexOf(\"/\") + 1, endpoint.indexOf('.'));\n    }\n\n    public static String getNameSrvAddrFromNamesrvEndpoint(String nameSrvEndpoint) {\n        if (StringUtils.isEmpty(nameSrvEndpoint)) {\n            return null;\n        }\n        return nameSrvEndpoint.substring(nameSrvEndpoint.lastIndexOf('/') + 1);\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/NetworkUtil.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.utils;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.lang.reflect.Method;\nimport java.net.Inet4Address;\nimport java.net.Inet6Address;\nimport java.net.InetAddress;\nimport java.net.InetSocketAddress;\nimport java.net.NetworkInterface;\nimport java.net.SocketAddress;\nimport java.net.SocketException;\nimport java.nio.channels.Selector;\nimport java.nio.channels.spi.SelectorProvider;\nimport java.util.ArrayList;\nimport java.util.Enumeration;\nimport java.util.List;\n\nimport org.apache.commons.validator.routines.InetAddressValidator;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class NetworkUtil {\n    public static final String OS_NAME = System.getProperty(\"os.name\");\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);\n    private static boolean isLinuxPlatform = false;\n    private static boolean isWindowsPlatform = false;\n\n    static {\n        if (OS_NAME != null && OS_NAME.toLowerCase().contains(\"linux\")) {\n            isLinuxPlatform = true;\n        }\n\n        if (OS_NAME != null && OS_NAME.toLowerCase().contains(\"windows\")) {\n            isWindowsPlatform = true;\n        }\n    }\n\n    public static boolean isWindowsPlatform() {\n        return isWindowsPlatform;\n    }\n\n    public static Selector openSelector() throws IOException {\n        Selector result = null;\n\n        if (isLinuxPlatform()) {\n            try {\n                final Class<?> providerClazz = Class.forName(\"sun.nio.ch.EPollSelectorProvider\");\n                try {\n                    final Method method = providerClazz.getMethod(\"provider\");\n                    final SelectorProvider selectorProvider = (SelectorProvider) method.invoke(null);\n                    if (selectorProvider != null) {\n                        result = selectorProvider.openSelector();\n                    }\n                } catch (final Exception e) {\n                    log.warn(\"Open ePoll Selector for linux platform exception\", e);\n                }\n            } catch (final Exception e) {\n                // ignore\n            }\n        }\n\n        if (result == null) {\n            result = Selector.open();\n        }\n\n        return result;\n    }\n\n    public static boolean isLinuxPlatform() {\n        return isLinuxPlatform;\n    }\n\n    public static List<InetAddress> getLocalInetAddressList() throws SocketException {\n        Enumeration<NetworkInterface> enumeration = NetworkInterface.getNetworkInterfaces();\n        List<InetAddress> inetAddressList = new ArrayList<>();\n        // Traversal Network interface to get the non-bridge and non-virtual and non-ppp and up address\n        while (enumeration.hasMoreElements()) {\n            final NetworkInterface nif = enumeration.nextElement();\n            if (isBridge(nif) || nif.isVirtual() || nif.isPointToPoint() || !nif.isUp()) {\n                continue;\n            }\n            InetAddressValidator validator = InetAddressValidator.getInstance();\n            final Enumeration<InetAddress> en = nif.getInetAddresses();\n            while (en.hasMoreElements()) {\n                final InetAddress address = en.nextElement();\n                if (address instanceof Inet4Address) {\n                    byte[] ipByte = address.getAddress();\n                    if (ipByte.length == 4) {\n                        if (validator.isValidInet4Address(UtilAll.ipToIPv4Str(ipByte))) {\n                            inetAddressList.add(address);\n                        }\n                    }\n                } else if (address instanceof Inet6Address) {\n                    byte[] ipByte = address.getAddress();\n                    if (ipByte.length == 16) {\n                        if (validator.isValidInet6Address(UtilAll.ipToIPv6Str(ipByte))) {\n                            inetAddressList.add(address);\n                        }\n                    }\n                }\n            }\n        }\n        return inetAddressList;\n    }\n\n    public static InetAddress getLocalInetAddress() {\n        try {\n            ArrayList<InetAddress> ipv4Result = new ArrayList<>();\n            ArrayList<InetAddress> ipv6Result = new ArrayList<>();\n            List<InetAddress> localInetAddressList = getLocalInetAddressList();\n            for (InetAddress inetAddress : localInetAddressList) {\n                // Skip loopback addresses\n                if (inetAddress.isLoopbackAddress()) {\n                    continue;\n                }\n                if (inetAddress instanceof Inet6Address) {\n                    ipv6Result.add(inetAddress);\n                } else {\n                    ipv4Result.add(inetAddress);\n                }\n            }\n            // prefer ipv4 and prefer external ip\n            if (!ipv4Result.isEmpty()) {\n                for (InetAddress ip : ipv4Result) {\n                    if (UtilAll.isInternalIP(ip.getAddress())) {\n                        continue;\n                    }\n                    return ip;\n                }\n                return ipv4Result.get(ipv4Result.size() - 1);\n            } else if (!ipv6Result.isEmpty()) {\n                for (InetAddress ip : ipv6Result) {\n                    if (UtilAll.isInternalV6IP(ip)) {\n                        continue;\n                    }\n                    return ip;\n                }\n                return ipv6Result.get(0);\n            }\n            //If failed to find,fall back to localhost\n            return InetAddress.getLocalHost();\n        } catch (Exception e) {\n            log.error(\"Failed to obtain local address\", e);\n        }\n\n        return null;\n    }\n\n    public static String getLocalAddress() {\n        InetAddress localHost = getLocalInetAddress();\n        return normalizeHostAddress(localHost);\n    }\n\n    public static String normalizeHostAddress(final InetAddress localHost) {\n        if (localHost instanceof Inet6Address) {\n            return \"[\" + localHost.getHostAddress() + \"]\";\n        } else {\n            return localHost.getHostAddress();\n        }\n    }\n\n    public static String denormalizeHostAddress(final String bracketedAddress) {\n        if (bracketedAddress == null) {\n            return null;\n        }\n\n        if (bracketedAddress.startsWith(\"[\") && bracketedAddress.endsWith(\"]\")) {\n            return bracketedAddress.substring(1, bracketedAddress.length() - 1);\n        } else {\n            return bracketedAddress;\n        }\n    }\n\n    public static SocketAddress string2SocketAddress(final String addr) {\n        int split = addr.lastIndexOf(\":\");\n        String host = addr.substring(0, split);\n        String port = addr.substring(split + 1);\n        return new InetSocketAddress(host, Integer.parseInt(port));\n    }\n\n    public static String socketAddress2String(final SocketAddress addr) {\n        StringBuilder sb = new StringBuilder();\n        InetSocketAddress inetSocketAddress = (InetSocketAddress) addr;\n        sb.append(inetSocketAddress.getAddress().getHostAddress());\n        sb.append(\":\");\n        sb.append(inetSocketAddress.getPort());\n        return sb.toString();\n    }\n\n    public static String convert2IpString(final String addr) {\n        return socketAddress2String(string2SocketAddress(addr));\n    }\n\n    private static boolean isBridge(NetworkInterface networkInterface) {\n        try {\n            if (isLinuxPlatform()) {\n                String interfaceName = networkInterface.getName();\n                File file = new File(\"/sys/class/net/\" + interfaceName + \"/bridge\");\n                return file.exists();\n            }\n        } catch (SecurityException e) {\n            //Ignore\n        }\n        return false;\n    }\n\n    // valid various ipv6 format like:\n    // with scope 2001:0db8:85a3:0000:0000:8a2e:0370:7334%eth0\n    // with bracketed [2001:0db8:85a3:0000:0000:8a2e:0370:7334]\n    public static boolean validCommonInet6Address(String ipOrCidr) {\n        String  ipWithoutBracketed = denormalizeHostAddress(ipOrCidr);\n        if (ipWithoutBracketed != null && ipWithoutBracketed.length() != 0) {\n            InetAddressValidator validator = InetAddressValidator.getInstance();\n            if (validator.isValidInet6Address(ipWithoutBracketed.split(\"%\")[0])) {\n                return true;\n            }\n        }\n        return false;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/PositiveAtomicCounter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.utils;\n\nimport java.util.concurrent.atomic.AtomicInteger;\n\npublic class PositiveAtomicCounter {\n    private static final int MASK = 0x7FFFFFFF;\n    private final AtomicInteger atom;\n\n\n    public PositiveAtomicCounter() {\n        atom = new AtomicInteger(0);\n    }\n\n\n    public final int incrementAndGet() {\n        final int rt = atom.incrementAndGet();\n        return rt & MASK;\n    }\n\n\n    public int intValue() {\n        return atom.intValue();\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/QueueTypeUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\npackage org.apache.rocketmq.common.utils;\n\nimport org.apache.rocketmq.common.TopicAttributes;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.attribute.CQType;\n\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Optional;\n\npublic class QueueTypeUtils {\n\n    public static boolean isBatchCq(Optional<TopicConfig> topicConfig) {\n        return Objects.equals(CQType.BatchCQ, getCQType(topicConfig));\n    }\n\n    public static CQType getCQType(Optional<TopicConfig> topicConfig) {\n        if (!topicConfig.isPresent()) {\n            return CQType.valueOf(TopicAttributes.QUEUE_TYPE_ATTRIBUTE.getDefaultValue());\n        }\n\n        String attributeName = TopicAttributes.QUEUE_TYPE_ATTRIBUTE.getName();\n\n        Map<String, String> attributes = topicConfig.get().getAttributes();\n        if (attributes == null || attributes.size() == 0) {\n            return CQType.valueOf(TopicAttributes.QUEUE_TYPE_ATTRIBUTE.getDefaultValue());\n        }\n\n        if (attributes.containsKey(attributeName)) {\n            return CQType.valueOf(attributes.get(attributeName));\n        } else {\n            return CQType.valueOf(TopicAttributes.QUEUE_TYPE_ATTRIBUTE.getDefaultValue());\n        }\n    }\n}"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/Serializer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.utils;\n\nimport org.apache.commons.lang3.SerializationException;\n\n/**\n * Serializer\n */\npublic interface Serializer {\n\n    /**\n     * Serialize object t to byte[]\n     */\n    <T> byte[] serialize(T t) throws SerializationException;\n\n    /**\n     * De-serialize bytes to T\n     */\n    <T> T deserialize(byte[] bytes, Class<T> type) throws SerializationException;\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/ServiceProvider.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.utils;\n\nimport java.nio.charset.StandardCharsets;\n\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\nimport java.io.BufferedReader;\nimport java.io.InputStream;\nimport java.io.InputStreamReader;\nimport java.util.ArrayList;\nimport java.util.List;\n\npublic class ServiceProvider {\n    private static final Logger LOG = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);\n    /**\n     * A reference to the classloader that loaded this class. It's more efficient to compute it once and cache it here.\n     */\n    private static ClassLoader thisClassLoader;\n    \n    /**\n     * JDK1.3+ <a href= \"http://java.sun.com/j2se/1.3/docs/guide/jar/jar.html#Service%20Provider\" > 'Service Provider'\n     * specification</a>.\n     */\n    public static final String PREFIX = \"META-INF/service/\";\n    \n    static {\n        thisClassLoader = getClassLoader(ServiceProvider.class);\n    }\n    \n    /**\n     * Returns a string that uniquely identifies the specified object, including its class.\n     * <p>\n     * The returned string is of form \"classname@hashcode\", ie is the same as the return value of the Object.toString()\n     * method, but works even when the specified object's class has overridden the toString method.\n     *\n     * @param o may be null.\n     * @return a string of form classname@hashcode, or \"null\" if param o is null.\n     */\n    protected static String objectId(Object o) {\n        if (o == null) {\n            return \"null\";\n        } else {\n            return o.getClass().getName() + \"@\" + System.identityHashCode(o);\n        }\n    }\n    \n    protected static ClassLoader getClassLoader(Class<?> clazz) {\n        try {\n            return clazz.getClassLoader();\n        } catch (SecurityException e) {\n            LOG.error(\"Unable to get classloader for class {} due to security restrictions , error info {}\",\n                clazz, e.getMessage());\n            throw e;\n        }\n    }\n    \n    protected static ClassLoader getContextClassLoader() {\n        ClassLoader classLoader = null;\n        try {\n            classLoader = Thread.currentThread().getContextClassLoader();\n        } catch (SecurityException ex) {\n            /**\n             * The getContextClassLoader() method throws SecurityException when the context\n             * class loader isn't an ancestor of the calling class's class\n             * loader, or if security permissions are restricted.\n             */\n        }\n        return classLoader;\n    }\n    \n    protected static InputStream getResourceAsStream(ClassLoader loader, String name) {\n        if (loader != null) {\n            return loader.getResourceAsStream(name);\n        } else {\n            return ClassLoader.getSystemResourceAsStream(name);\n        }\n    }\n    \n    public static <T> List<T> load(Class<?> clazz) {\n        String fullName = PREFIX + clazz.getName();\n        return load(fullName, clazz);\n    }\n    \n    public static <T> List<T> load(String name, Class<?> clazz) {\n        LOG.info(\"Looking for a resource file of name [{}] ...\", name);\n        List<T> services = new ArrayList<>();\n        InputStream is = getResourceAsStream(getContextClassLoader(), name);\n        if (is == null) {\n            LOG.warn(\"No resource file with name [{}] found.\", name);\n            return services;\n        }\n        try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) {\n            String serviceName = reader.readLine();\n            List<String> names = new ArrayList<>();\n            while (serviceName != null && !\"\".equals(serviceName)) {\n                LOG.info(\n                    \"Creating an instance as specified by file {} which was present in the path of the context classloader.\",\n                    name);\n                if (!names.contains(serviceName)) {\n                    names.add(serviceName);\n                    services.add(initService(getContextClassLoader(), serviceName, clazz));\n                }\n                serviceName = reader.readLine();\n            }\n        } catch (Exception e) {\n            LOG.error(\"Error occurred when looking for resource file \" + name, e);\n        }\n        return services;\n    }\n    \n    public static <T> T loadClass(Class<?> clazz) {\n        String fullName = PREFIX + clazz.getName();\n        return loadClass(fullName, clazz);\n    }\n    \n    public static <T> T loadClass(String name, Class<?> clazz) {\n        LOG.info(\"Looking for a resource file of name [{}] ...\", name);\n        T s = null;\n        InputStream is = getResourceAsStream(getContextClassLoader(), name);\n        if (is == null) {\n            LOG.warn(\"No resource file with name [{}] found.\", name);\n            return null;\n        }\n        try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) {\n            String serviceName = reader.readLine();\n            if (serviceName != null && !\"\".equals(serviceName)) {\n                s = initService(getContextClassLoader(), serviceName, clazz);\n            } else {\n                LOG.warn(\"ServiceName is empty!\");\n            }\n        } catch (Exception e) {\n            LOG.warn(\"Error occurred when looking for resource file \" + name, e);\n        }\n        return s;\n    }\n\n    protected static <T> T initService(ClassLoader classLoader, String serviceName, Class<?> clazz) {\n        Class<?> serviceClazz = null;\n        try {\n            if (classLoader != null) {\n                try {\n                    // Warning: must typecast here & allow exception to be generated/caught & recast properly\n                    serviceClazz = classLoader.loadClass(serviceName);\n                    if (clazz.isAssignableFrom(serviceClazz)) {\n                        LOG.info(\"Loaded class {} from classloader {}\", serviceClazz.getName(),\n                            objectId(classLoader));\n                    } else {\n                        // This indicates a problem with the ClassLoader tree. An incompatible ClassLoader was used to load the implementation.\n                        LOG.error(\n                            \"Class {} loaded from classloader {} does not extend {} as loaded by this classloader.\",\n                            serviceClazz.getName(),\n                            objectId(serviceClazz.getClassLoader()), clazz.getName());\n                    }\n                    return (T) serviceClazz.getDeclaredConstructor().newInstance();\n                } catch (ClassNotFoundException ex) {\n                    if (classLoader == thisClassLoader) {\n                        // Nothing more to try, onwards.\n                        LOG.warn(\"Unable to locate any class {} via classloader {}\", serviceName,\n                            objectId(classLoader));\n                        throw ex;\n                    }\n                    // Ignore exception, continue\n                } catch (NoClassDefFoundError e) {\n                    if (classLoader == thisClassLoader) {\n                        // Nothing more to try, onwards.\n                        LOG.warn(\n                            \"Class {} cannot be loaded via classloader {}.it depends on some other class that cannot be found.\",\n                            serviceClazz, objectId(classLoader));\n                        throw e;\n                    }\n                    // Ignore exception, continue\n                }\n            }\n        } catch (Exception e) {\n            LOG.error(\"Unable to init service.\", e);\n        }\n        return (T) serviceClazz;\n    }\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/Shutdown.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.utils;\n\npublic interface Shutdown {\n    void shutdown() throws Exception;\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/Start.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.utils;\n\npublic interface Start {\n    void start() throws Exception;\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/StartAndShutdown.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.utils;\n\npublic interface StartAndShutdown extends Start, Shutdown {\n    default void preShutdown() throws Exception {}\n}\n"
  },
  {
    "path": "common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.utils;\n\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.RejectedExecutionHandler;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ScheduledThreadPoolExecutor;\nimport java.util.concurrent.ThreadFactory;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.thread.FutureTaskExtThreadPoolExecutor;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic final class ThreadUtils {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.TOOLS_LOGGER_NAME);\n\n    public static ExecutorService newSingleThreadExecutor(String processName, boolean isDaemon) {\n        return ThreadUtils.newSingleThreadExecutor(newThreadFactory(processName, isDaemon));\n    }\n\n    public static ExecutorService newSingleThreadExecutor(ThreadFactory threadFactory) {\n        return ThreadUtils.newThreadPoolExecutor(1, threadFactory);\n    }\n\n    public static ExecutorService newThreadPoolExecutor(int corePoolSize, ThreadFactory threadFactory) {\n        return ThreadUtils.newThreadPoolExecutor(corePoolSize, corePoolSize,\n            0L, TimeUnit.MILLISECONDS,\n            new LinkedBlockingQueue<>(),\n            threadFactory);\n    }\n\n    public static ExecutorService newThreadPoolExecutor(int corePoolSize,\n        int maximumPoolSize,\n        long keepAliveTime,\n        TimeUnit unit, BlockingQueue<Runnable> workQueue,\n        String processName,\n        boolean isDaemon) {\n        return ThreadUtils.newThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, newThreadFactory(processName, isDaemon));\n    }\n\n    public static ExecutorService newThreadPoolExecutor(final int corePoolSize,\n        final int maximumPoolSize,\n        final long keepAliveTime,\n        final TimeUnit unit,\n        final BlockingQueue<Runnable> workQueue,\n        final ThreadFactory threadFactory) {\n        return ThreadUtils.newThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, new ThreadPoolExecutor.AbortPolicy());\n    }\n\n    public static ExecutorService newThreadPoolExecutor(int corePoolSize,\n        int maximumPoolSize,\n        long keepAliveTime,\n        TimeUnit unit,\n        BlockingQueue<Runnable> workQueue,\n        ThreadFactory threadFactory,\n        RejectedExecutionHandler handler) {\n        return new FutureTaskExtThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler);\n    }\n\n    public static ScheduledExecutorService newSingleThreadScheduledExecutor(String processName, boolean isDaemon) {\n        return ThreadUtils.newScheduledThreadPool(1, processName, isDaemon);\n    }\n\n    public static ScheduledExecutorService newSingleThreadScheduledExecutor(ThreadFactory threadFactory) {\n        return ThreadUtils.newScheduledThreadPool(1, threadFactory);\n    }\n\n    public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize) {\n        return ThreadUtils.newScheduledThreadPool(corePoolSize, Executors.defaultThreadFactory());\n    }\n\n    public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize, String processName,\n        boolean isDaemon) {\n        return ThreadUtils.newScheduledThreadPool(corePoolSize, newThreadFactory(processName, isDaemon));\n    }\n\n    public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize, ThreadFactory threadFactory) {\n        return ThreadUtils.newScheduledThreadPool(corePoolSize, threadFactory, new ThreadPoolExecutor.AbortPolicy());\n    }\n\n    public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize,\n        ThreadFactory threadFactory,\n        RejectedExecutionHandler handler) {\n        return new ScheduledThreadPoolExecutor(corePoolSize, threadFactory, handler);\n    }\n\n    public static ThreadFactory newThreadFactory(String processName, boolean isDaemon) {\n        return newGenericThreadFactory(\"ThreadUtils-\" + processName, isDaemon);\n    }\n\n    public static ThreadFactory newGenericThreadFactory(String processName) {\n        return newGenericThreadFactory(processName, false);\n    }\n\n    public static ThreadFactory newGenericThreadFactory(String processName, int threads) {\n        return newGenericThreadFactory(processName, threads, false);\n    }\n\n    public static ThreadFactory newGenericThreadFactory(final String processName, final boolean isDaemon) {\n        return new ThreadFactoryImpl(processName + \"_\", isDaemon);\n    }\n\n    public static ThreadFactory newGenericThreadFactory(final String processName, final int threads,\n        final boolean isDaemon) {\n        return new ThreadFactoryImpl(String.format(\"%s_%d_\", processName, threads), isDaemon);\n    }\n\n    /**\n     * Create a new thread\n     *\n     * @param name     The name of the thread\n     * @param runnable The work for the thread to do\n     * @param daemon   Should the thread block JVM stop?\n     * @return The unstarted thread\n     */\n    public static Thread newThread(String name, Runnable runnable, boolean daemon) {\n        Thread thread = new Thread(runnable, name);\n        thread.setDaemon(daemon);\n        thread.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {\n            public void uncaughtException(Thread t, Throwable e) {\n                LOGGER.error(\"Uncaught exception in thread '\" + t.getName() + \"':\", e);\n            }\n        });\n        return thread;\n    }\n\n    /**\n     * Shutdown passed thread using isAlive and join.\n     *\n     * @param t Thread to stop\n     */\n    public static void shutdownGracefully(final Thread t) {\n        shutdownGracefully(t, 0);\n    }\n\n    /**\n     * Shutdown passed thread using isAlive and join.\n     *\n     * @param millis Pass 0 if we're to wait forever.\n     * @param t      Thread to stop\n     */\n    public static void shutdownGracefully(final Thread t, final long millis) {\n        if (t == null)\n            return;\n        while (t.isAlive()) {\n            try {\n                t.interrupt();\n                t.join(millis);\n            } catch (InterruptedException e) {\n                Thread.currentThread().interrupt();\n            }\n        }\n    }\n\n    /**\n     * An implementation of the graceful stop sequence recommended by\n     * {@link ExecutorService}.\n     *\n     * @param executor executor\n     * @param timeout  timeout\n     * @param timeUnit timeUnit\n     */\n    public static void shutdownGracefully(ExecutorService executor, long timeout, TimeUnit timeUnit) {\n        // Disable new tasks from being submitted.\n        executor.shutdown();\n        try {\n            // Wait a while for existing tasks to terminate.\n            if (!executor.awaitTermination(timeout, timeUnit)) {\n                executor.shutdownNow();\n                // Wait a while for tasks to respond to being cancelled.\n                if (!executor.awaitTermination(timeout, timeUnit)) {\n                    LOGGER.warn(String.format(\"%s didn't terminate!\", executor));\n                }\n            }\n        } catch (InterruptedException ie) {\n            // (Re-)Cancel if current thread also interrupted.\n            executor.shutdownNow();\n            // Preserve interrupt status.\n            Thread.currentThread().interrupt();\n        }\n    }\n\n    /**\n     * Shutdown the specific ExecutorService\n     *\n     * @param executorService the executor\n     */\n    public static void shutdown(ExecutorService executorService) {\n        if (executorService != null) {\n            executorService.shutdown();\n        }\n    }\n\n    /**\n     * A constructor to stop this class being constructed.\n     */\n    private ThreadUtils() {\n        // Unused\n\n    }\n}\n"
  },
  {
    "path": "common/src/main/resources/META-INF/services/org.apache.rocketmq.logging.ch.qos.logback.classic.spi.Configurator",
    "content": "org.apache.rocketmq.common.logging.DefaultJoranConfiguratorExt"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/BrokerConfigSingletonTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class BrokerConfigSingletonTest {\n\n    /**\n     * Tests the behavior of getting the broker configuration when it has not been initialized.\n     * Expects an IllegalArgumentException to be thrown, ensuring that the configuration cannot be obtained without initialization.\n     */\n    @Test(expected = IllegalArgumentException.class)\n    public void getBrokerConfig_NullConfiguration_ThrowsException() {\n        BrokerConfigSingleton.getBrokerConfig();\n    }\n\n    /**\n     * Tests the behavior of setting the broker configuration after it has already been initialized.\n     * Expects an IllegalArgumentException to be thrown, ensuring that the configuration cannot be reset once set.\n     * Also asserts that the returned brokerConfig instance is the same as the one set, confirming the singleton property.\n     */\n    @Test(expected = IllegalArgumentException.class)\n    public void setBrokerConfig_AlreadyInitialized_ThrowsException() {\n        BrokerConfig config = new BrokerConfig();\n        BrokerConfigSingleton.setBrokerConfig(config);\n        Assert.assertSame(\"Expected brokerConfig instance is not returned\", config, BrokerConfigSingleton.getBrokerConfig());\n        BrokerConfigSingleton.setBrokerConfig(config);\n    }\n\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/BrokerConfigTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class BrokerConfigTest {\n\n    @Test\n    public void testConsumerFallBehindThresholdOverflow() {\n        long expect = 1024L * 1024 * 1024 * 16;\n        assertThat(new BrokerConfig().getConsumerFallbehindThreshold()).isEqualTo(expect);\n    }\n\n    @Test\n    public void testBrokerConfigAttribute() {\n        BrokerConfig brokerConfig = new BrokerConfig();\n        brokerConfig.setNamesrvAddr(\"127.0.0.1:9876\");\n        brokerConfig.setAutoCreateTopicEnable(false);\n        brokerConfig.setBrokerName(\"broker-a\");\n        brokerConfig.setBrokerId(0);\n        brokerConfig.setBrokerClusterName(\"DefaultCluster\");\n        brokerConfig.setMsgTraceTopicName(\"RMQ_SYS_TRACE_TOPIC4\");\n        brokerConfig.setAutoDeleteUnusedStats(true);\n        assertThat(brokerConfig.getBrokerClusterName()).isEqualTo(\"DefaultCluster\");\n        assertThat(brokerConfig.getNamesrvAddr()).isEqualTo(\"127.0.0.1:9876\");\n        assertThat(brokerConfig.getMsgTraceTopicName()).isEqualTo(\"RMQ_SYS_TRACE_TOPIC4\");\n        assertThat(brokerConfig.getBrokerId()).isEqualTo(0);\n        assertThat(brokerConfig.getBrokerName()).isEqualTo(\"broker-a\");\n        assertThat(brokerConfig.isAutoCreateTopicEnable()).isEqualTo(false);\n        assertThat(brokerConfig.isAutoDeleteUnusedStats()).isEqualTo(true);\n    }\n}"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/ConfigManagerTest.java",
    "content": "package org.apache.rocketmq.common;/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport java.io.File;\nimport java.io.PrintWriter;\nimport java.lang.reflect.Method;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\n\npublic class ConfigManagerTest {\n    private static final String PATH_FILE = System.getProperty(\"java.io.tmpdir\") + File.separator + \"org.apache.rocketmq.common.ConfigManagerTest\";\n    private static final String CONTENT_ENCODE = \"Encode content for ConfigManager\";\n\n    @Test\n    public void testLoad() throws Exception {\n        ConfigManager testConfigManager = buildTestConfigManager();\n        File file = createAndWriteFile(testConfigManager.configFilePath());\n        assertTrue(testConfigManager.load());\n        file.delete();\n        File fileBak = createAndWriteFile(testConfigManager.configFilePath() + \".bak\");\n        assertTrue(testConfigManager.load());\n        fileBak.delete();\n    }\n\n    @Test\n    public void testLoadBak() throws Exception {\n        ConfigManager testConfigManager = buildTestConfigManager();\n        File file = createAndWriteFile(testConfigManager.configFilePath() + \".bak\");\n        // invoke private method \"loadBak()\"\n        Method declaredMethod = ConfigManager.class.getDeclaredMethod(\"loadBak\");\n        declaredMethod.setAccessible(true);\n        Boolean loadBakResult = (Boolean) declaredMethod.invoke(testConfigManager);\n        assertTrue(loadBakResult);\n        file.delete();\n\n        Boolean loadBakResult2 = (Boolean) declaredMethod.invoke(testConfigManager);\n        assertTrue(loadBakResult2);\n        declaredMethod.setAccessible(false);\n    }\n\n    @Test\n    public void testPersist() throws Exception {\n        ConfigManager testConfigManager = buildTestConfigManager();\n        testConfigManager.persist();\n        File file = new File(testConfigManager.configFilePath());\n        assertEquals(CONTENT_ENCODE, MixAll.file2String(file));\n    }\n\n    private ConfigManager buildTestConfigManager() {\n        return new ConfigManager() {\n            @Override\n            public String encode() {\n                return encode(false);\n            }\n\n            @Override\n            public String configFilePath() {\n                return PATH_FILE;\n            }\n\n            @Override\n            public void decode(String jsonString) {\n\n            }\n\n            @Override\n            public String encode(boolean prettyFormat) {\n                return CONTENT_ENCODE;\n            }\n        };\n    }\n\n    private File createAndWriteFile(String fileName) throws Exception {\n        File file = new File(fileName);\n        if (file.exists()) {\n            file.delete();\n        }\n        file.createNewFile();\n        PrintWriter out = new PrintWriter(fileName);\n        out.write(\"TestForConfigManager\");\n        out.close();\n        return file;\n    }\n}"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/CountDownLatch2Test.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.common;\n\nimport org.junit.Test;\n\nimport java.util.concurrent.TimeUnit;\n\nimport static org.hamcrest.core.Is.is;\nimport static org.hamcrest.core.StringContains.containsString;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertThat;\nimport static org.junit.Assert.assertTrue;\n\n/**\n * CountDownLatch2 Unit Test\n *\n * @see CountDownLatch2\n */\npublic class CountDownLatch2Test {\n\n    /**\n     * test constructor with invalid init param\n     *\n     * @see CountDownLatch2#CountDownLatch2(int)\n     */\n    @Test\n    public void testConstructorError() {\n        int count = -1;\n        try {\n            CountDownLatch2 latch = new CountDownLatch2(count);\n        } catch (IllegalArgumentException e) {\n            assertThat(e.getMessage(), is(\"count < 0\"));\n        }\n    }\n\n    /**\n     * test constructor with valid init param\n     *\n     * @see CountDownLatch2#CountDownLatch2(int)\n     */\n    @Test\n    public void testConstructor() {\n        int count = 10;\n        CountDownLatch2 latch = new CountDownLatch2(count);\n        assertEquals(\"Expected equal\", count, latch.getCount());\n        assertThat(\"Expected contain\", latch.toString(), containsString(\"[Count = \" + count + \"]\"));\n    }\n\n    /**\n     * test await timeout\n     *\n     * @see CountDownLatch2#await(long, TimeUnit)\n     */\n    @Test\n    public void testAwaitTimeout() throws InterruptedException {\n        int count = 1;\n        CountDownLatch2 latch = new CountDownLatch2(count);\n        boolean await = latch.await(10, TimeUnit.MILLISECONDS);\n        assertFalse(\"Expected false\", await);\n\n        latch.countDown();\n        boolean await2 = latch.await(10, TimeUnit.MILLISECONDS);\n        assertTrue(\"Expected true\", await2);\n    }\n\n\n    /**\n     * test reset\n     *\n     * @see CountDownLatch2#countDown()\n     */\n    @Test(timeout = 1000)\n    public void testCountDownAndGetCount() throws InterruptedException {\n        int count = 2;\n        CountDownLatch2 latch = new CountDownLatch2(count);\n        assertEquals(\"Expected equal\", count, latch.getCount());\n        latch.countDown();\n        assertEquals(\"Expected equal\", count - 1, latch.getCount());\n        latch.countDown();\n        latch.await();\n        assertEquals(\"Expected equal\", 0, latch.getCount());\n    }\n\n\n    /**\n     * test reset\n     *\n     * @see CountDownLatch2#reset()\n     */\n    @Test\n    public void testReset() throws InterruptedException {\n        int count = 2;\n        CountDownLatch2 latch = new CountDownLatch2(count);\n        latch.countDown();\n        assertEquals(\"Expected equal\", count - 1, latch.getCount());\n        latch.reset();\n        assertEquals(\"Expected equal\", count, latch.getCount());\n        latch.countDown();\n        latch.countDown();\n        latch.await();\n        assertEquals(\"Expected equal\", 0, latch.getCount());\n        // coverage Sync#tryReleaseShared, c==0\n        latch.countDown();\n        assertEquals(\"Expected equal\", 0, latch.getCount());\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/KeyBuilderTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common;\n\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class KeyBuilderTest {\n    String topic = \"test-topic\";\n    String group = \"test-group\";\n\n    @Test\n    public void testBuildPopRetryTopic() {\n        assertThat(KeyBuilder.buildPopRetryTopicV2(topic, group)).isEqualTo(MixAll.RETRY_GROUP_TOPIC_PREFIX + group + \"+\" + topic);\n    }\n\n    @Test\n    public void testBuildPopRetryTopicV1() {\n        assertThat(KeyBuilder.buildPopRetryTopicV1(topic, group)).isEqualTo(MixAll.RETRY_GROUP_TOPIC_PREFIX + group + \"_\" + topic);\n    }\n\n    @Test\n    public void testParseNormalTopic() {\n        String popRetryTopic = KeyBuilder.buildPopRetryTopicV2(topic, group);\n        assertThat(KeyBuilder.parseNormalTopic(popRetryTopic, group)).isEqualTo(topic);\n\n        String popRetryTopicV1 = KeyBuilder.buildPopRetryTopicV1(topic, group);\n        assertThat(KeyBuilder.parseNormalTopic(popRetryTopicV1, group)).isEqualTo(topic);\n\n        popRetryTopic = KeyBuilder.buildPopRetryTopicV2(topic, group);\n        assertThat(KeyBuilder.parseNormalTopic(popRetryTopic)).isEqualTo(topic);\n    }\n\n    @Test\n    public void testParseGroup() {\n        String popRetryTopic = KeyBuilder.buildPopRetryTopicV2(topic, group);\n        assertThat(KeyBuilder.parseGroup(popRetryTopic)).isEqualTo(group);\n    }\n\n    @Test\n    public void testIsPopRetryTopicV2() {\n        String popRetryTopic = KeyBuilder.buildPopRetryTopicV2(topic, group);\n        assertThat(KeyBuilder.isPopRetryTopicV2(popRetryTopic)).isEqualTo(true);\n        String popRetryTopicV1 = KeyBuilder.buildPopRetryTopicV1(topic, group);\n        assertThat(KeyBuilder.isPopRetryTopicV2(popRetryTopicV1)).isEqualTo(false);\n    }\n}"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/MQVersionTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common;\n\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class MQVersionTest {\n\n    @Test\n    public void testGetVersionDesc() throws Exception {\n        String desc = \"V3_0_0_SNAPSHOT\";\n        assertThat(MQVersion.getVersionDesc(0)).isEqualTo(desc);\n    }\n\n    @Test\n    public void testGetVersionDesc_higherVersion() throws Exception {\n        String desc = \"HIGHER_VERSION\";\n        assertThat(MQVersion.getVersionDesc(Integer.MAX_VALUE)).isEqualTo(desc);\n    }\n\n    @Test\n    public void testValue2Version() throws Exception {\n        assertThat(MQVersion.value2Version(0)).isEqualTo(MQVersion.Version.V3_0_0_SNAPSHOT);\n    }\n\n    @Test\n    public void testValue2Version_HigherVersion() throws Exception {\n        assertThat(MQVersion.value2Version(Integer.MAX_VALUE)).isEqualTo(MQVersion.Version.HIGHER_VERSION);\n    }\n}"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/MessageBatchTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.common;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageBatch;\nimport org.junit.Test;\n\npublic class MessageBatchTest {\n\n    public List<Message> generateMessages() {\n        List<Message> messages = new ArrayList<>();\n        Message message1 = new Message(\"topic1\", \"body\".getBytes());\n        Message message2 = new Message(\"topic1\", \"body\".getBytes());\n\n        messages.add(message1);\n        messages.add(message2);\n        return messages;\n    }\n\n    @Test\n    public void testGenerate_OK() throws Exception {\n        List<Message> messages = generateMessages();\n        MessageBatch.generateFromList(messages);\n    }\n\n    @Test(expected = UnsupportedOperationException.class)\n    public void testGenerate_DiffTopic() throws Exception {\n        List<Message> messages = generateMessages();\n        messages.get(1).setTopic(\"topic2\");\n        MessageBatch.generateFromList(messages);\n    }\n\n    @Test(expected = UnsupportedOperationException.class)\n    public void testGenerate_DiffWaitOK() throws Exception {\n        List<Message> messages = generateMessages();\n        messages.get(1).setWaitStoreMsgOK(false);\n        MessageBatch.generateFromList(messages);\n    }\n\n    @Test(expected = UnsupportedOperationException.class)\n    public void testGenerate_Delay() throws Exception {\n        List<Message> messages = generateMessages();\n        messages.get(1).setDelayTimeLevel(1);\n        MessageBatch.generateFromList(messages);\n    }\n\n    @Test(expected = UnsupportedOperationException.class)\n    public void testGenerate_Retry() throws Exception {\n        List<Message> messages = generateMessages();\n        messages.get(1).setTopic(MixAll.RETRY_GROUP_TOPIC_PREFIX + \"topic\");\n        MessageBatch.generateFromList(messages);\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/MessageEncodeDecodeTest.java",
    "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.List;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertTrue;\n\npublic class MessageEncodeDecodeTest {\n\n    @Test\n    public void testEncodeDecodeSingle() throws Exception {\n        Message message = new Message(\"topic\", \"body\".getBytes());\n        message.setFlag(12);\n        message.putUserProperty(\"key\", \"value\");\n        byte[] bytes = MessageDecoder.encodeMessage(message);\n        ByteBuffer buffer = ByteBuffer.allocate(bytes.length);\n        buffer.put(bytes);\n        buffer.flip();\n        Message newMessage = MessageDecoder.decodeMessage(buffer);\n\n        assertTrue(message.getFlag() == newMessage.getFlag());\n        assertTrue(newMessage.getProperty(\"key\").equals(newMessage.getProperty(\"key\")));\n        assertTrue(Arrays.equals(newMessage.getBody(), message.getBody()));\n    }\n\n    @Test\n    public void testEncodeDecodeList() throws Exception {\n        List<Message> messages = new ArrayList<>(128);\n        for (int i = 0; i < 100; i++) {\n            Message message = new Message(\"topic\", (\"body\" + i).getBytes());\n            message.setFlag(i);\n            message.putUserProperty(\"key\", \"value\" + i);\n            messages.add(message);\n        }\n        byte[] bytes = MessageDecoder.encodeMessages(messages);\n\n        ByteBuffer buffer = ByteBuffer.allocate(bytes.length);\n        buffer.put(bytes);\n        buffer.flip();\n\n        List<Message> newMsgs = MessageDecoder.decodeMessages(buffer);\n\n        assertTrue(newMsgs.size() == messages.size());\n\n        for (int i = 0; i < newMsgs.size(); i++) {\n            Message message = messages.get(i);\n            Message newMessage = newMsgs.get(i);\n            assertTrue(message.getFlag() == newMessage.getFlag());\n            assertTrue(newMessage.getProperty(\"key\").equals(newMessage.getProperty(\"key\")));\n            assertTrue(Arrays.equals(newMessage.getBody(), message.getBody()));\n\n        }\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/MessageExtBrokerInnerTest.java",
    "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class MessageExtBrokerInnerTest {\n    @Test\n    public void testDeleteProperty() {\n        MessageExtBrokerInner messageExtBrokerInner = new MessageExtBrokerInner();\n        String propertiesString = \"\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"\");\n\n        propertiesString = \"KeyA\\u0001ValueA\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"\");\n\n        propertiesString = \"KeyA\\u0001ValueA\\u0002\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"\");\n\n        propertiesString = \"KeyA\\u0001ValueA\\u0002KeyA\\u0001ValueA\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"\");\n\n        propertiesString = \"KeyA\\u0001ValueA\\u0002KeyA\\u0001ValueA\\u0002\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"\");\n\n        propertiesString = \"KeyB\\u0001ValueB\\u0002KeyA\\u0001ValueA\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"KeyB\\u0001ValueB\\u0002\");\n\n        propertiesString = \"KeyB\\u0001ValueB\\u0002KeyA\\u0001ValueA\\u0002\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"KeyB\\u0001ValueB\\u0002\");\n\n        propertiesString = \"KeyB\\u0001ValueB\\u0002KeyA\\u0001ValueA\\u0002KeyB\\u0001ValueB\\u0002\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"KeyB\\u0001ValueB\\u0002KeyB\\u0001ValueB\\u0002\");\n\n        propertiesString = \"KeyA\\u0001ValueA\\u0002KeyB\\u0001ValueB\\u0002\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"KeyB\\u0001ValueB\\u0002\");\n\n        propertiesString = \"KeyA\\u0001ValueA\\u0002KeyB\\u0001ValueB\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"KeyB\\u0001ValueB\");\n\n        propertiesString = \"KeyA\\u0001ValueA\\u0002KeyB\\u0001ValueBKeyA\\u0001ValueA\\u0002\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"KeyB\\u0001ValueBKeyA\\u0001ValueA\\u0002\");\n\n        propertiesString = \"KeyA\\u0001ValueA\\u0002KeyB\\u0001ValueBKeyA\\u0001\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"KeyB\\u0001ValueBKeyA\\u0001\");\n\n        propertiesString = \"KeyA\\u0001ValueA\\u0002KeyB\\u0001ValueBKeyA\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"KeyB\\u0001ValueBKeyA\");\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/MixAllTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.io.PrintWriter;\nimport java.net.InetAddress;\nimport java.util.List;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class MixAllTest {\n    @Test\n    public void testGetLocalInetAddress() throws Exception {\n        List<String> localInetAddress = MixAll.getLocalInetAddress();\n        String local = InetAddress.getLocalHost().getHostAddress();\n        assertThat(localInetAddress).contains(\"127.0.0.1\");\n        assertThat(local).isNotNull();\n    }\n\n    @Test\n    public void testBrokerVIPChannel() {\n        assertThat(MixAll.brokerVIPChannel(true, \"127.0.0.1:10911\")).isEqualTo(\"127.0.0.1:10909\");\n    }\n\n    @Test\n    public void testCompareAndIncreaseOnly() {\n        AtomicLong target = new AtomicLong(5);\n        assertThat(MixAll.compareAndIncreaseOnly(target, 6)).isTrue();\n        assertThat(target.get()).isEqualTo(6);\n\n        assertThat(MixAll.compareAndIncreaseOnly(target, 4)).isFalse();\n        assertThat(target.get()).isEqualTo(6);\n    }\n\n    @Test\n    public void testFile2String() throws IOException {\n        String fileName = System.getProperty(\"java.io.tmpdir\") + File.separator + \"MixAllTest\" + System.currentTimeMillis();\n        File file = new File(fileName);\n        if (file.exists()) {\n            file.delete();\n        }\n        file.createNewFile();\n        PrintWriter out = new PrintWriter(fileName);\n        out.write(\"TestForMixAll\");\n        out.close();\n        String string = MixAll.file2String(fileName);\n        assertThat(string).isEqualTo(\"TestForMixAll\");\n        file.delete();\n    }\n\n    @Test\n    public void testString2File() throws IOException {\n        String fileName = System.getProperty(\"java.io.tmpdir\") + File.separator + \"MixAllTest\" + System.currentTimeMillis();\n        MixAll.string2File(\"MixAll_testString2File\", fileName);\n        assertThat(MixAll.file2String(fileName)).isEqualTo(\"MixAll_testString2File\");\n    }\n\n    @Test\n    public void testIsLmq() {\n        String testLmq = null;\n        assertThat(MixAll.isLmq(testLmq)).isFalse();\n        testLmq = \"lmq\";\n        assertThat(MixAll.isLmq(testLmq)).isFalse();\n        testLmq = \"%LMQ%queue123\";\n        assertThat(MixAll.isLmq(testLmq)).isTrue();\n        testLmq = \"%LMQ%GID_TEST\";\n        assertThat(MixAll.isLmq(testLmq)).isTrue();\n    }\n\n    @Test\n    public void testAdjustConfigForPlatform_OnWindows() {\n        if (MixAll.isWindows()) {\n            String configWithSingleBackslash = \"data\\\\path\\\\config\\\\file.properties\";\n            String adjusted = MixAll.adjustConfigForPlatform(configWithSingleBackslash);\n            assertThat(adjusted).isEqualTo(\"data\\\\\\\\path\\\\\\\\config\\\\\\\\file.properties\");\n\n            String configWithMultipleBackslashes = \"C:\\\\\\\\RocketMQ\\\\\\\\logs\\\\\\\\broker.log\";\n            adjusted = MixAll.adjustConfigForPlatform(configWithMultipleBackslashes);\n            assertThat(adjusted).isEqualTo(\"C:\\\\\\\\\\\\\\\\RocketMQ\\\\\\\\\\\\\\\\logs\\\\\\\\\\\\\\\\broker.log\");\n\n            String configWithoutBackslash = \"listenPort=10911\";\n            adjusted = MixAll.adjustConfigForPlatform(configWithoutBackslash);\n            assertThat(adjusted).isEqualTo(\"listenPort=10911\");\n\n            String emptyConfig = \"\";\n            adjusted = MixAll.adjustConfigForPlatform(emptyConfig);\n            assertThat(adjusted).isEqualTo(\"\");\n\n            adjusted = MixAll.adjustConfigForPlatform(null);\n            assertThat(adjusted).isNull();\n        } else {\n            String configWithSingleBackslash = \"/home/rocketmq/conf/broker.conf\";\n            String adjusted = MixAll.adjustConfigForPlatform(configWithSingleBackslash);\n            assertThat(adjusted).isEqualTo(\"/home/rocketmq/conf/broker.conf\");\n\n            String linuxPathWithBackslash = \"some\\\\directory\\\\file.txt\";\n            adjusted = MixAll.adjustConfigForPlatform(linuxPathWithBackslash);\n            assertThat(adjusted).isEqualTo(\"some\\\\directory\\\\file.txt\");\n\n            String emptyConfig = \"\";\n            adjusted = MixAll.adjustConfigForPlatform(emptyConfig);\n            assertThat(adjusted).isEqualTo(\"\");\n\n            adjusted = MixAll.adjustConfigForPlatform(null);\n            assertThat(adjusted).isNull();\n        }\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/NetworkUtilTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common;\n\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class NetworkUtilTest {\n    @Test\n    public void testGetLocalAddress() {\n        String localAddress = NetworkUtil.getLocalAddress();\n        assertThat(localAddress).isNotNull();\n        assertThat(localAddress.length()).isGreaterThan(0);\n    }\n\n    @Test\n    public void testConvert2IpStringWithIp() {\n        String result = NetworkUtil.convert2IpString(\"127.0.0.1:9876\");\n        assertThat(result).isEqualTo(\"127.0.0.1:9876\");\n    }\n\n    @Test\n    public void testConvert2IpStringWithHost() {\n        String result = NetworkUtil.convert2IpString(\"localhost:9876\");\n        assertThat(result).isEqualTo(\"127.0.0.1:9876\");\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/ServiceThreadTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common;\n\nimport static org.junit.Assert.assertEquals;\n\nimport org.junit.Test;\n\npublic class ServiceThreadTest {\n\n    @Test\n    public void testShutdown() {\n        shutdown(false, false);\n        shutdown(false, true);\n        shutdown(true, false);\n        shutdown(true, true);\n    }\n\n    @Test\n    public void testMakeStop() {\n        ServiceThread testServiceThread = startTestServiceThread();\n        testServiceThread.makeStop();\n        assertEquals(true, testServiceThread.isStopped());\n    }\n\n    @Test\n    public void testWakeup() {\n        ServiceThread testServiceThread = startTestServiceThread();\n        testServiceThread.wakeup();\n        assertEquals(true, testServiceThread.hasNotified.get());\n        assertEquals(0, testServiceThread.waitPoint.getCount());\n    }\n\n    @Test\n    public void testWaitForRunning() {\n        ServiceThread testServiceThread = startTestServiceThread();\n        // test waitForRunning\n        testServiceThread.waitForRunning(1000);\n        assertEquals(false, testServiceThread.hasNotified.get());\n        assertEquals(1, testServiceThread.waitPoint.getCount());\n        // test wake up\n        testServiceThread.wakeup();\n        assertEquals(true, testServiceThread.hasNotified.get());\n        assertEquals(0, testServiceThread.waitPoint.getCount());\n        // repeat waitForRunning\n        testServiceThread.waitForRunning(1000);\n        assertEquals(false, testServiceThread.hasNotified.get());\n        assertEquals(0, testServiceThread.waitPoint.getCount());\n        // repeat waitForRunning again\n        testServiceThread.waitForRunning(1000);\n        assertEquals(false, testServiceThread.hasNotified.get());\n        assertEquals(1, testServiceThread.waitPoint.getCount());\n    }\n\n    private ServiceThread startTestServiceThread() {\n        return startTestServiceThread(false);\n    }\n\n    private ServiceThread startTestServiceThread(boolean daemon) {\n        ServiceThread testServiceThread = new ServiceThread() {\n\n            @Override\n            public void run() {\n                doNothing();\n            }\n\n            private void doNothing() {}\n\n            @Override\n            public String getServiceName() {\n                return \"TestServiceThread\";\n            }\n        };\n        testServiceThread.setDaemon(daemon);\n        // test start\n        testServiceThread.start();\n        assertEquals(false, testServiceThread.isStopped());\n        return testServiceThread;\n    }\n\n    public void shutdown(boolean daemon, boolean interrupt) {\n        ServiceThread testServiceThread = startTestServiceThread(daemon);\n        shutdown0(interrupt, testServiceThread);\n        // repeat\n        shutdown0(interrupt, testServiceThread);\n    }\n\n    private void shutdown0(boolean interrupt, ServiceThread testServiceThread) {\n        if (interrupt) {\n            testServiceThread.shutdown(true);\n        } else {\n            testServiceThread.shutdown();\n        }\n        assertEquals(true, testServiceThread.isStopped());\n        assertEquals(true, testServiceThread.hasNotified.get());\n        assertEquals(0, testServiceThread.waitPoint.getCount());\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/TopicConfigTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common;\n\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class TopicConfigTest {\n    String topicName = \"topic\";\n    int queueNums = 8;\n    int perm = PermName.PERM_READ | PermName.PERM_WRITE;\n    TopicFilterType topicFilterType = TopicFilterType.SINGLE_TAG;\n\n    @Test\n    public void testEncode() {\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(topicName);\n        topicConfig.setReadQueueNums(queueNums);\n        topicConfig.setWriteQueueNums(queueNums);\n        topicConfig.setPerm(perm);\n        topicConfig.setTopicFilterType(topicFilterType);\n        topicConfig.setTopicMessageType(TopicMessageType.FIFO);\n\n        String encode = topicConfig.encode();\n        assertThat(encode).isEqualTo(\"topic 8 8 6 SINGLE_TAG {\\\"message.type\\\":\\\"FIFO\\\"}\");\n    }\n\n    @Test\n    public void testDecode() {\n        String encode = \"topic 8 8 6 SINGLE_TAG {\\\"message.type\\\":\\\"FIFO\\\"}\";\n        TopicConfig decodeTopicConfig = new TopicConfig();\n        decodeTopicConfig.decode(encode);\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(topicName);\n        topicConfig.setReadQueueNums(queueNums);\n        topicConfig.setWriteQueueNums(queueNums);\n        topicConfig.setPerm(perm);\n        topicConfig.setTopicFilterType(topicFilterType);\n        topicConfig.setTopicMessageType(TopicMessageType.FIFO);\n\n        assertThat(decodeTopicConfig).isEqualTo(topicConfig);\n    }\n\n    @Test\n    public void testDecodeWhenCompatible() {\n        String encode = \"topic 8 8 6 SINGLE_TAG\";\n        TopicConfig decodeTopicConfig = new TopicConfig();\n        decodeTopicConfig.decode(encode);\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(topicName);\n        topicConfig.setReadQueueNums(queueNums);\n        topicConfig.setWriteQueueNums(queueNums);\n        topicConfig.setPerm(perm);\n        topicConfig.setTopicFilterType(topicFilterType);\n\n        assertThat(decodeTopicConfig).isEqualTo(topicConfig);\n    }\n}"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common;\n\nimport java.io.File;\nimport java.io.FileOutputStream;\nimport java.net.InetAddress;\nimport java.net.UnknownHostException;\nimport java.nio.ByteBuffer;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Properties;\n\nimport org.junit.Rule;\nimport org.junit.Test;\nimport org.junit.rules.TemporaryFolder;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.assertj.core.api.Assertions.within;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\n\npublic class UtilAllTest {\n    @Rule\n    public TemporaryFolder tempDir = new TemporaryFolder();\n\n    @Test\n    public void testCurrentStackTrace() {\n        String currentStackTrace = UtilAll.currentStackTrace();\n        assertThat(currentStackTrace).contains(\"UtilAll.currentStackTrace\");\n        assertThat(currentStackTrace).contains(\"UtilAllTest.testCurrentStackTrace(\");\n    }\n\n    @Test\n    public void testProperties2Object() {\n        DemoConfig demoConfig = new DemoConfig();\n        Properties properties = new Properties();\n        properties.setProperty(\"demoWidth\", \"123\");\n        properties.setProperty(\"demoLength\", \"456\");\n        properties.setProperty(\"demoOK\", \"true\");\n        properties.setProperty(\"demoName\", \"TestDemo\");\n        MixAll.properties2Object(properties, demoConfig);\n        assertThat(demoConfig.getDemoLength()).isEqualTo(456);\n        assertThat(demoConfig.getDemoWidth()).isEqualTo(123);\n        assertThat(demoConfig.isDemoOK()).isTrue();\n        assertThat(demoConfig.getDemoName()).isEqualTo(\"TestDemo\");\n    }\n\n    @Test\n    public void testProperties2String() {\n        DemoSubConfig demoConfig = new DemoSubConfig();\n        demoConfig.setDemoLength(123);\n        demoConfig.setDemoWidth(456);\n        demoConfig.setDemoName(\"TestDemo\");\n        demoConfig.setDemoOK(true);\n\n        demoConfig.setSubField0(\"1\");\n        demoConfig.setSubField1(false);\n\n        Properties properties = MixAll.object2Properties(demoConfig);\n        assertThat(properties.getProperty(\"demoLength\")).isEqualTo(\"123\");\n        assertThat(properties.getProperty(\"demoWidth\")).isEqualTo(\"456\");\n        assertThat(properties.getProperty(\"demoOK\")).isEqualTo(\"true\");\n        assertThat(properties.getProperty(\"demoName\")).isEqualTo(\"TestDemo\");\n\n        assertThat(properties.getProperty(\"subField0\")).isEqualTo(\"1\");\n        assertThat(properties.getProperty(\"subField1\")).isEqualTo(\"false\");\n\n        properties = MixAll.object2Properties(new Object());\n        assertEquals(0, properties.size());\n    }\n\n    @Test\n    public void testIsPropertiesEqual() {\n        final Properties p1 = new Properties();\n        final Properties p2 = new Properties();\n\n        p1.setProperty(\"a\", \"1\");\n        p1.setProperty(\"b\", \"2\");\n        p2.setProperty(\"a\", \"1\");\n        p2.setProperty(\"b\", \"2\");\n\n        assertThat(MixAll.isPropertiesEqual(p1, p2)).isTrue();\n    }\n\n    @Test\n    public void testGetPid() {\n        assertThat(UtilAll.getPid()).isGreaterThan(0);\n    }\n\n    @Test\n    public void testGetDiskPartitionSpaceUsedPercent() {\n        String tmpDir = System.getProperty(\"java.io.tmpdir\");\n\n        assertThat(UtilAll.getDiskPartitionSpaceUsedPercent(null)).isCloseTo(-1, within(0.000001));\n        assertThat(UtilAll.getDiskPartitionSpaceUsedPercent(\"\")).isCloseTo(-1, within(0.000001));\n        assertThat(UtilAll.getDiskPartitionSpaceUsedPercent(\"nonExistingPath\")).isCloseTo(-1, within(0.000001));\n        assertThat(UtilAll.getDiskPartitionSpaceUsedPercent(tmpDir)).isNotCloseTo(-1, within(0.000001));\n    }\n\n    @Test\n    public void testIsBlank() {\n        assertThat(UtilAll.isBlank(\"Hello \")).isFalse();\n        assertThat(UtilAll.isBlank(\" Hello\")).isFalse();\n        assertThat(UtilAll.isBlank(\"He llo\")).isFalse();\n        assertThat(UtilAll.isBlank(\"  \")).isTrue();\n        assertThat(UtilAll.isBlank(\"Hello\")).isFalse();\n    }\n\n    @Test\n    public void testIPv6Check() throws UnknownHostException {\n        InetAddress nonInternal = InetAddress.getByName(\"2408:4004:0180:8100:3FAA:1DDE:2B3F:898A\");\n        InetAddress internal = InetAddress.getByName(\"FE80:0000:0000:0000:0000:0000:0000:FFFF\");\n        assertThat(UtilAll.isInternalV6IP(nonInternal)).isFalse();\n        assertThat(UtilAll.isInternalV6IP(internal)).isTrue();\n        assertThat(UtilAll.ipToIPv6Str(nonInternal.getAddress()).toUpperCase()).isEqualTo(\"2408:4004:0180:8100:3FAA:1DDE:2B3F:898A\");\n    }\n\n    @Test\n    public void testJoin() {\n        List<String> list = Arrays.asList(\"groupA=DENY\", \"groupB=PUB|SUB\", \"groupC=SUB\");\n        String comma = \",\";\n        assertEquals(\"groupA=DENY,groupB=PUB|SUB,groupC=SUB\", UtilAll.join(list, comma));\n        assertEquals(null, UtilAll.join(null, comma));\n        List<String> objects = Collections.emptyList();\n        assertEquals(\"\", UtilAll.join(objects, comma));\n    }\n\n    @Test\n    public void testSplit() {\n        List<String> list = Arrays.asList(\"groupA=DENY\", \"groupB=PUB|SUB\", \"groupC=SUB\");\n        String comma = \",\";\n        assertEquals(list, UtilAll.split(\"groupA=DENY,groupB=PUB|SUB,groupC=SUB\", comma));\n        assertEquals(null, UtilAll.split(null, comma));\n        assertEquals(Collections.EMPTY_LIST, UtilAll.split(\"\", comma));\n    }\n\n    static class DemoConfig {\n        private int demoWidth = 0;\n        private int demoLength = 0;\n        private boolean demoOK = false;\n        private String demoName = \"haha\";\n\n        int getDemoWidth() {\n            return demoWidth;\n        }\n\n        public void setDemoWidth(int demoWidth) {\n            this.demoWidth = demoWidth;\n        }\n\n        public int getDemoLength() {\n            return demoLength;\n        }\n\n        public void setDemoLength(int demoLength) {\n            this.demoLength = demoLength;\n        }\n\n        public boolean isDemoOK() {\n            return demoOK;\n        }\n\n        public void setDemoOK(boolean demoOK) {\n            this.demoOK = demoOK;\n        }\n\n        public String getDemoName() {\n            return demoName;\n        }\n\n        public void setDemoName(String demoName) {\n            this.demoName = demoName;\n        }\n\n        @Override\n        public String toString() {\n            return \"DemoConfig{\" +\n                \"demoWidth=\" + demoWidth +\n                \", demoLength=\" + demoLength +\n                \", demoOK=\" + demoOK +\n                \", demoName='\" + demoName + '\\'' +\n                '}';\n        }\n    }\n\n    static class DemoSubConfig extends DemoConfig {\n        private String subField0 = \"0\";\n        public boolean subField1 = true;\n\n        public String getSubField0() {\n            return subField0;\n        }\n\n        public void setSubField0(String subField0) {\n            this.subField0 = subField0;\n        }\n\n        public boolean isSubField1() {\n            return subField1;\n        }\n\n        public void setSubField1(boolean subField1) {\n            this.subField1 = subField1;\n        }\n    }\n\n    @Test\n    public void testCleanBuffer() {\n        UtilAll.cleanBuffer(null);\n        UtilAll.cleanBuffer(ByteBuffer.allocateDirect(10));\n        UtilAll.cleanBuffer(ByteBuffer.allocateDirect(0));\n        UtilAll.cleanBuffer(ByteBuffer.allocate(10));\n    }\n\n    @Test\n    public void testCalculateFileSizeInPath() throws Exception {\n        /**\n         * testCalculateFileSizeInPath\n         *  - file_0\n         *  - dir_1\n         *      - file_1_0\n         *      - file_1_1\n         *      - dir_1_2\n         *          - file_1_2_0\n         *  - dir_2\n         */\n        File baseFile = tempDir.getRoot();\n\n        // test empty path\n        assertEquals(0, UtilAll.calculateFileSizeInPath(baseFile));\n\n        File file0 = new File(baseFile, \"file_0\");\n        assertTrue(file0.createNewFile());\n        writeFixedBytesToFile(file0, 1313);\n\n        assertEquals(1313, UtilAll.calculateFileSizeInPath(baseFile));\n\n        // build a file tree like above\n        File dir1 = new File(baseFile, \"dir_1\");\n        dir1.mkdirs();\n        File file10 = new File(dir1, \"file_1_0\");\n        File file11 = new File(dir1, \"file_1_1\");\n        File dir12 = new File(dir1, \"dir_1_2\");\n        dir12.mkdirs();\n        File file120 = new File(dir12, \"file_1_2_0\");\n        File dir2 = new File(baseFile, \"dir_2\");\n        dir2.mkdirs();\n\n        // write all file with 1313 bytes data\n        assertTrue(file10.createNewFile());\n        writeFixedBytesToFile(file10, 1313);\n        assertTrue(file11.createNewFile());\n        writeFixedBytesToFile(file11, 1313);\n        assertTrue(file120.createNewFile());\n        writeFixedBytesToFile(file120, 1313);\n\n        assertEquals(1313 * 4, UtilAll.calculateFileSizeInPath(baseFile));\n    }\n\n    private void writeFixedBytesToFile(File file, int size) throws Exception {\n        FileOutputStream outputStream = new FileOutputStream(file);\n        byte[] bytes = new byte[size];\n        outputStream.write(bytes, 0, size);\n        outputStream.close();\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/action/ActionTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.action;\n\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNull;\n\npublic class ActionTest {\n\n    @Test\n    public void getByName_NullName_ReturnsNull() {\n        Action result = Action.getByName(\"null\");\n        assertNull(result);\n    }\n\n    @Test\n    public void getByName_UnknownName_ReturnsUnknown() {\n        Action result = Action.getByName(\"unknown\");\n        assertEquals(Action.UNKNOWN, result);\n    }\n\n    @Test\n    public void getByName_AllName_ReturnsAll() {\n        Action result = Action.getByName(\"All\");\n        assertEquals(Action.ALL, result);\n    }\n\n    @Test\n    public void getByName_AnyName_ReturnsAny() {\n        Action result = Action.getByName(\"Any\");\n        assertEquals(Action.ANY, result);\n    }\n\n    @Test\n    public void getByName_PubName_ReturnsPub() {\n        Action result = Action.getByName(\"Pub\");\n        assertEquals(Action.PUB, result);\n    }\n\n    @Test\n    public void getByName_SubName_ReturnsSub() {\n        Action result = Action.getByName(\"Sub\");\n        assertEquals(Action.SUB, result);\n    }\n\n    @Test\n    public void getByName_CreateName_ReturnsCreate() {\n        Action result = Action.getByName(\"Create\");\n        assertEquals(Action.CREATE, result);\n    }\n\n    @Test\n    public void getByName_UpdateName_ReturnsUpdate() {\n        Action result = Action.getByName(\"Update\");\n        assertEquals(Action.UPDATE, result);\n    }\n\n    @Test\n    public void getByName_DeleteName_ReturnsDelete() {\n        Action result = Action.getByName(\"Delete\");\n        assertEquals(Action.DELETE, result);\n    }\n\n    @Test\n    public void getByName_GetName_ReturnsGet() {\n        Action result = Action.getByName(\"Get\");\n        assertEquals(Action.GET, result);\n    }\n\n    @Test\n    public void getByName_ListName_ReturnsList() {\n        Action result = Action.getByName(\"List\");\n        assertEquals(Action.LIST, result);\n    }\n\n    @Test\n    public void getCode_ReturnsCorrectCode() {\n        assertEquals((byte) 1, Action.ALL.getCode());\n        assertEquals((byte) 2, Action.ANY.getCode());\n        assertEquals((byte) 3, Action.PUB.getCode());\n        assertEquals((byte) 4, Action.SUB.getCode());\n        assertEquals((byte) 5, Action.CREATE.getCode());\n        assertEquals((byte) 6, Action.UPDATE.getCode());\n        assertEquals((byte) 7, Action.DELETE.getCode());\n        assertEquals((byte) 8, Action.GET.getCode());\n        assertEquals((byte) 9, Action.LIST.getCode());\n    }\n\n    @Test\n    public void getName_ReturnsCorrectName() {\n        assertEquals(\"All\", Action.ALL.getName());\n        assertEquals(\"Any\", Action.ANY.getName());\n        assertEquals(\"Pub\", Action.PUB.getName());\n        assertEquals(\"Sub\", Action.SUB.getName());\n        assertEquals(\"Create\", Action.CREATE.getName());\n        assertEquals(\"Update\", Action.UPDATE.getName());\n        assertEquals(\"Delete\", Action.DELETE.getName());\n        assertEquals(\"Get\", Action.GET.getName());\n        assertEquals(\"List\", Action.LIST.getName());\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/action/RocketMQActionTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.action;\n\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertArrayEquals;\nimport static org.junit.Assert.assertEquals;\n\npublic class RocketMQActionTest {\n\n    @Test\n    public void testRocketMQAction_DefaultResourceType_CustomisedValueAndActionArray() {\n        RocketMQAction annotation = DemoClass.class.getAnnotation(RocketMQAction.class);\n        assertEquals(0, annotation.value());\n        assertEquals(ResourceType.UNKNOWN, annotation.resource());\n        assertArrayEquals(new Action[] {}, annotation.action());\n    }\n\n    @Test\n    public void testRocketMQAction_CustomisedValueAndResourceTypeAndActionArray() {\n        RocketMQAction annotation = CustomisedDemoClass.class.getAnnotation(RocketMQAction.class);\n        assertEquals(1, annotation.value());\n        assertEquals(ResourceType.TOPIC, annotation.resource());\n        assertArrayEquals(new Action[] {Action.CREATE, Action.DELETE}, annotation.action());\n    }\n\n    @RocketMQAction(value = 0, resource = ResourceType.UNKNOWN, action = {})\n    private static class DemoClass {\n    }\n\n    @RocketMQAction(value = 1, resource = ResourceType.TOPIC, action = {Action.CREATE, Action.DELETE})\n    private static class CustomisedDemoClass {\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/attribute/AttributeParserTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.attribute;\n\nimport com.google.common.collect.Maps;\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport static com.google.common.collect.Maps.newHashMap;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\n\npublic class AttributeParserTest {\n\n    @Test\n    public void parseToMap_EmptyString_ReturnsEmptyMap() {\n        String attributesModification = \"\";\n        Map<String, String> result = AttributeParser.parseToMap(attributesModification);\n        assertTrue(result.isEmpty());\n    }\n\n    @Test\n    public void parseToMap_NullString_ReturnsEmptyMap() {\n        String attributesModification = null;\n        Map<String, String> result = AttributeParser.parseToMap(attributesModification);\n        assertTrue(result.isEmpty());\n    }\n\n    @Test\n    public void parseToMap_ValidAttributesModification_ReturnsExpectedMap() {\n        String attributesModification = \"+key1=value1,+key2=value2,-key3,+key4=value4\";\n        Map<String, String> result = AttributeParser.parseToMap(attributesModification);\n\n        Map<String, String> expectedMap = new HashMap<>();\n        expectedMap.put(\"+key1\", \"value1\");\n        expectedMap.put(\"+key2\", \"value2\");\n        expectedMap.put(\"-key3\", \"\");\n        expectedMap.put(\"+key4\", \"value4\");\n\n        assertEquals(expectedMap, result);\n    }\n\n    @Test(expected = RuntimeException.class)\n    public void parseToMap_InvalidAddAttributeFormat_ThrowsRuntimeException() {\n        String attributesModification = \"+key1=value1,key2=value2,-key3,+key4=value4\";\n        AttributeParser.parseToMap(attributesModification);\n    }\n\n    @Test(expected = RuntimeException.class)\n    public void parseToMap_InvalidDeleteAttributeFormat_ThrowsRuntimeException() {\n        String attributesModification = \"+key1=value1,+key2=value2,key3,+key4=value4\";\n        AttributeParser.parseToMap(attributesModification);\n    }\n\n    @Test(expected = RuntimeException.class)\n    public void parseToMap_DuplicateKey_ThrowsRuntimeException() {\n        String attributesModification = \"+key1=value1,+key1=value2\";\n        AttributeParser.parseToMap(attributesModification);\n    }\n\n    @Test\n    public void parseToString_EmptyMap_ReturnsEmptyString() {\n        Map<String, String> attributes = new HashMap<>();\n        String result = AttributeParser.parseToString(attributes);\n        assertEquals(\"\", result);\n    }\n\n    @Test\n    public void parseToString_ValidAttributes_ReturnsExpectedString() {\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(\"key1\", \"value1\");\n        attributes.put(\"key2\", \"value2\");\n        attributes.put(\"key3\", \"\");\n\n        String result = AttributeParser.parseToString(attributes);\n        String expectedString = \"key1=value1,key2=value2,key3\";\n        assertEquals(expectedString, result);\n    }\n\n    @Test\n    public void testParseToMap() {\n        Assert.assertEquals(0, AttributeParser.parseToMap(null).size());\n        AttributeParser.parseToMap(\"++=++\");\n        AttributeParser.parseToMap(\"--\");\n        Assert.assertThrows(RuntimeException.class, () -> AttributeParser.parseToMap(\"x\"));\n        Assert.assertThrows(RuntimeException.class, () -> AttributeParser.parseToMap(\"+\"));\n        Assert.assertThrows(RuntimeException.class, () -> AttributeParser.parseToMap(\"++\"));\n    }\n\n    @Test\n    public void testParseToString() {\n        Assert.assertEquals(\"\", AttributeParser.parseToString(null));\n        Assert.assertEquals(\"\", AttributeParser.parseToString(newHashMap()));\n        HashMap<String, String> map = new HashMap<>();\n        int addSize = 10;\n        for (int i = 0; i < addSize; i++) {\n            map.put(\"+add.key\" + i, \"value\" + i);\n        }\n        int deleteSize = 10;\n        for (int i = 0; i < deleteSize; i++) {\n            map.put(\"-delete.key\" + i, \"\");\n        }\n        Assert.assertEquals(addSize + deleteSize, AttributeParser.parseToString(map).split(\",\").length);\n    }\n\n    @Test\n    public void testParseBetweenStringAndMapWithoutDistortion() {\n        List<String> testCases = Arrays.asList(\"-a\", \"+a=b,+c=d,+z=z,+e=e\", \"+a=b,-d\", \"+a=b\", \"-a,-b\");\n        for (String testCase : testCases) {\n            assertTrue(Maps.difference(AttributeParser.parseToMap(testCase), AttributeParser.parseToMap(parse(testCase))).areEqual());\n        }\n    }\n\n    private String parse(String original) {\n        Map<String, String> stringStringMap = AttributeParser.parseToMap(original);\n        return AttributeParser.parseToString(stringStringMap);\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/attribute/AttributeTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.attribute;\n\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.collect.Sets.newHashSet;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertTrue;\n\npublic class AttributeTest {\n\n    private Attribute attribute;\n\n    @Before\n    public void setUp() {\n        attribute = new Attribute(\"testAttribute\", true) {\n            @Override\n            public void verify(String value) {\n                throw new UnsupportedOperationException();\n            }\n        };\n    }\n\n    @Test\n    public void testGetName_ShouldReturnCorrectName() {\n        assertEquals(\"testAttribute\", attribute.getName());\n    }\n\n    @Test\n    public void testSetName_ShouldSetCorrectName() {\n        attribute.setName(\"newTestAttribute\");\n        assertEquals(\"newTestAttribute\", attribute.getName());\n    }\n\n    @Test\n    public void testIsChangeable_ShouldReturnCorrectChangeableStatus() {\n        assertTrue(attribute.isChangeable());\n    }\n\n    @Test\n    public void testSetChangeable_ShouldSetCorrectChangeableStatus() {\n        attribute.setChangeable(false);\n        assertFalse(attribute.isChangeable());\n    }\n\n    @Test(expected = UnsupportedOperationException.class)\n    public void testVerify_ShouldThrowUnsupportedOperationException() {\n        attribute.verify(\"testValue\");\n    }\n\n    @Test\n    public void testEnumAttribute() {\n        EnumAttribute enumAttribute = new EnumAttribute(\"enum.key\", true, newHashSet(\"enum-1\", \"enum-2\", \"enum-3\"), \"enum-1\");\n\n        Assert.assertThrows(RuntimeException.class, () -> enumAttribute.verify(\"\"));\n        Assert.assertThrows(RuntimeException.class, () -> enumAttribute.verify(\"x\"));\n        Assert.assertThrows(RuntimeException.class, () -> enumAttribute.verify(\"enum-4\"));\n\n        enumAttribute.verify(\"enum-1\");\n        enumAttribute.verify(\"enum-2\");\n        enumAttribute.verify(\"enum-3\");\n    }\n\n    @Test\n    public void testLongRangeAttribute() {\n        LongRangeAttribute longRangeAttribute = new LongRangeAttribute(\"long.range.key\", true, 10, 20, 15);\n        Assert.assertThrows(RuntimeException.class, () -> longRangeAttribute.verify(\"\"));\n        Assert.assertThrows(RuntimeException.class, () -> longRangeAttribute.verify(\",\"));\n        Assert.assertThrows(RuntimeException.class, () -> longRangeAttribute.verify(\"a\"));\n        Assert.assertThrows(RuntimeException.class, () -> longRangeAttribute.verify(\"-1\"));\n        Assert.assertThrows(RuntimeException.class, () -> longRangeAttribute.verify(\"21\"));\n\n        longRangeAttribute.verify(\"11\");\n        longRangeAttribute.verify(\"10\");\n        longRangeAttribute.verify(\"20\");\n    }\n\n    @Test\n    public void testBooleanAttribute() {\n        BooleanAttribute booleanAttribute = new BooleanAttribute(\"bool.key\", false, false);\n\n        Assert.assertThrows(RuntimeException.class, () -> booleanAttribute.verify(\"\"));\n        Assert.assertThrows(RuntimeException.class, () -> booleanAttribute.verify(\"a\"));\n        Assert.assertThrows(RuntimeException.class, () -> booleanAttribute.verify(\",\"));\n        Assert.assertThrows(RuntimeException.class, () -> booleanAttribute.verify(\"checked\"));\n        Assert.assertThrows(RuntimeException.class, () -> booleanAttribute.verify(\"1\"));\n        Assert.assertThrows(RuntimeException.class, () -> booleanAttribute.verify(\"0\"));\n        Assert.assertThrows(RuntimeException.class, () -> booleanAttribute.verify(\"-1\"));\n\n        booleanAttribute.verify(\"true\");\n        booleanAttribute.verify(\"tRue\");\n        booleanAttribute.verify(\"false\");\n        booleanAttribute.verify(\"falSe\");\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/attribute/AttributeUtilTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.attribute;\n\nimport com.google.common.collect.ImmutableMap;\nimport java.util.HashMap;\nimport java.util.Map;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertTrue;\n\npublic class AttributeUtilTest {\n\n    private Map<String, Attribute> allAttributes;\n    private ImmutableMap<String, String> currentAttributes;\n\n    @Before\n    public void setUp() {\n        allAttributes = new HashMap<>();\n        allAttributes.put(\"attr1\", new TestAttribute(\"value1\", true, value -> true));\n        allAttributes.put(\"attr2\", new TestAttribute(\"value2\", true, value -> true));\n        allAttributes.put(\"attr3\", new TestAttribute(\"value3\", true, value -> value.equals(\"valid\")));\n\n        currentAttributes = ImmutableMap.of(\"attr1\", \"value1\", \"attr2\", \"value2\");\n    }\n\n    @Test\n    public void alterCurrentAttributes_CreateMode_ShouldReturnOnlyAddedAttributes() {\n        ImmutableMap<String, String> newAttributes = ImmutableMap.of(\"+attr1\", \"new_value1\", \"+attr2\", \"value2\", \"+attr3\", \"value3\");\n\n        Map<String, String> result = AttributeUtil.alterCurrentAttributes(true, allAttributes, currentAttributes, newAttributes);\n\n        assertEquals(3, result.size());\n        assertTrue(result.containsKey(\"attr1\"));\n        assertEquals(\"new_value1\", result.get(\"attr1\"));\n        assertTrue(result.containsKey(\"attr3\"));\n        assertEquals(\"value3\", result.get(\"attr3\"));\n        assertTrue(result.containsKey(\"attr2\"));\n    }\n\n    @Test(expected = RuntimeException.class)\n    public void alterCurrentAttributes_CreateMode_AddNonAddableAttribute_ShouldThrowException() {\n        ImmutableMap<String, String> newAttributes = ImmutableMap.of(\"attr1\", \"value1\");\n        AttributeUtil.alterCurrentAttributes(true, allAttributes, currentAttributes, newAttributes);\n    }\n\n    @Test\n    public void alterCurrentAttributes_UpdateMode_ShouldReturnUpdatedAndAddedAttributes() {\n        ImmutableMap<String, String> newAttributes = ImmutableMap.of(\"+attr1\", \"new_value1\", \"-attr2\", \"value2\", \"+attr3\", \"value3\");\n\n        Map<String, String> result = AttributeUtil.alterCurrentAttributes(false, allAttributes, currentAttributes, newAttributes);\n\n        assertEquals(2, result.size());\n        assertTrue(result.containsKey(\"attr1\"));\n        assertEquals(\"new_value1\", result.get(\"attr1\"));\n        assertTrue(result.containsKey(\"attr3\"));\n        assertEquals(\"value3\", result.get(\"attr3\"));\n        assertFalse(result.containsKey(\"attr2\"));\n    }\n\n    @Test(expected = RuntimeException.class)\n    public void alterCurrentAttributes_UpdateMode_DeleteNonExistentAttribute_ShouldThrowException() {\n        ImmutableMap<String, String> newAttributes = ImmutableMap.of(\"-attr4\", \"value4\");\n        AttributeUtil.alterCurrentAttributes(false, allAttributes, currentAttributes, newAttributes);\n    }\n\n    @Test(expected = RuntimeException.class)\n    public void alterCurrentAttributes_UpdateMode_WrongFormatKey_ShouldThrowException() {\n        ImmutableMap<String, String> newAttributes = ImmutableMap.of(\"attr1\", \"+value1\");\n        AttributeUtil.alterCurrentAttributes(false, allAttributes, currentAttributes, newAttributes);\n    }\n\n    @Test(expected = RuntimeException.class)\n    public void alterCurrentAttributes_UnsupportedKey_ShouldThrowException() {\n        ImmutableMap<String, String> newAttributes = ImmutableMap.of(\"unsupported_attr\", \"value\");\n        AttributeUtil.alterCurrentAttributes(false, allAttributes, currentAttributes, newAttributes);\n    }\n\n    @Test(expected = RuntimeException.class)\n    public void alterCurrentAttributes_AttemptToUpdateUnchangeableAttribute_ShouldThrowException() {\n        ImmutableMap<String, String> newAttributes = ImmutableMap.of(\"attr2\", \"new_value2\");\n        AttributeUtil.alterCurrentAttributes(false, allAttributes, currentAttributes, newAttributes);\n    }\n\n    private static class TestAttribute extends Attribute {\n        private final AttributeValidator validator;\n\n        public TestAttribute(String name, boolean changeable, AttributeValidator validator) {\n            super(name, changeable);\n            this.validator = validator;\n        }\n\n        @Override\n        public void verify(String value) {\n            validator.validate(value);\n        }\n    }\n\n    private interface AttributeValidator {\n        boolean validate(String value);\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/attribute/BooleanAttributeTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.attribute;\n\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertThrows;\n\npublic class BooleanAttributeTest {\n\n    private BooleanAttribute booleanAttribute;\n\n    @Before\n    public void setUp() {\n        booleanAttribute = new BooleanAttribute(\"testAttribute\", true, false);\n    }\n\n    @Test\n    public void testVerify_ValidValue_NoExceptionThrown() {\n        booleanAttribute.verify(\"true\");\n        booleanAttribute.verify(\"false\");\n    }\n\n    @Test\n    public void testVerify_InvalidValue_ExceptionThrown() {\n        assertThrows(RuntimeException.class, () -> booleanAttribute.verify(\"invalid\"));\n        assertThrows(RuntimeException.class, () -> booleanAttribute.verify(\"1\"));\n        assertThrows(RuntimeException.class, () -> booleanAttribute.verify(\"0\"));\n        assertThrows(RuntimeException.class, () -> booleanAttribute.verify(\"\"));\n    }\n\n    @Test\n    public void testGetDefaultValue() {\n        assertFalse(booleanAttribute.getDefaultValue());\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/attribute/CQTypeTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.attribute;\n\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\n\npublic class CQTypeTest {\n\n    @Test\n    public void testValues() {\n        CQType[] values = CQType.values();\n        assertEquals(3, values.length);\n        assertEquals(CQType.SimpleCQ, values[0]);\n        assertEquals(CQType.BatchCQ, values[1]);\n        assertEquals(CQType.RocksDBCQ, values[2]);\n    }\n\n    @Test\n    public void testValueOf() {\n        assertEquals(CQType.SimpleCQ, CQType.valueOf(\"SimpleCQ\"));\n        assertEquals(CQType.BatchCQ, CQType.valueOf(\"BatchCQ\"));\n        assertEquals(CQType.RocksDBCQ, CQType.valueOf(\"RocksDBCQ\"));\n    }\n\n    @Test(expected = IllegalArgumentException.class)\n    public void testValueOf_InvalidName() {\n        CQType.valueOf(\"InvalidCQ\");\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/attribute/CleanupPolicyTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.attribute;\n\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\n\npublic class CleanupPolicyTest {\n\n    @Test\n    public void testCleanupPolicy_Delete() {\n        CleanupPolicy cleanupPolicy = CleanupPolicy.DELETE;\n        assertEquals(\"DELETE\", cleanupPolicy.toString());\n    }\n\n    @Test\n    public void testCleanupPolicy_Compaction() {\n        CleanupPolicy cleanupPolicy = CleanupPolicy.COMPACTION;\n        assertEquals(\"COMPACTION\", cleanupPolicy.toString());\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/attribute/EnumAttributeTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.attribute;\n\nimport org.junit.Before;\nimport org.junit.Test;\nimport java.util.HashSet;\nimport java.util.Set;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertThrows;\nimport static org.junit.Assert.assertTrue;\n\npublic class EnumAttributeTest {\n\n    private EnumAttribute enumAttribute;\n\n    @Before\n    public void setUp() {\n        Set<String> universe = new HashSet<>();\n        universe.add(\"value1\");\n        universe.add(\"value2\");\n        universe.add(\"value3\");\n\n        enumAttribute = new EnumAttribute(\"testAttribute\", true, universe, \"value1\");\n    }\n\n    @Test\n    public void verify_ValidValue_NoExceptionThrown() {\n        enumAttribute.verify(\"value1\");\n        enumAttribute.verify(\"value2\");\n        enumAttribute.verify(\"value3\");\n    }\n\n    @Test\n    public void verify_InvalidValue_ExceptionThrown() {\n        RuntimeException exception = assertThrows(RuntimeException.class, () -> {\n            enumAttribute.verify(\"invalidValue\");\n        });\n\n        assertTrue(exception.getMessage().startsWith(\"value is not in set:\"));\n    }\n\n    @Test\n    public void getDefaultValue_ReturnsDefaultValue() {\n        assertEquals(\"value1\", enumAttribute.getDefaultValue());\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/attribute/LongRangeAttributeTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.attribute;\n\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertThrows;\n\npublic class LongRangeAttributeTest {\n\n    private LongRangeAttribute longRangeAttribute;\n\n    @Before\n    public void setUp() {\n        longRangeAttribute = new LongRangeAttribute(\"testAttribute\", true, 0, 100, 50);\n    }\n\n    @Test\n    public void verify_ValidValue_NoExceptionThrown() {\n        longRangeAttribute.verify(\"50\");\n    }\n\n    @Test\n    public void verify_MinValue_NoExceptionThrown() {\n        longRangeAttribute.verify(\"0\");\n    }\n\n    @Test\n    public void verify_MaxValue_NoExceptionThrown() {\n        longRangeAttribute.verify(\"100\");\n    }\n\n    @Test\n    public void verify_ValueLessThanMin_ThrowsRuntimeException() {\n        RuntimeException exception = assertThrows(RuntimeException.class, () -> longRangeAttribute.verify(\"-1\"));\n        assertEquals(\"value is not in range(0, 100)\", exception.getMessage());\n    }\n\n    @Test\n    public void verify_ValueGreaterThanMax_ThrowsRuntimeException() {\n        RuntimeException exception = assertThrows(RuntimeException.class, () -> longRangeAttribute.verify(\"101\"));\n        assertEquals(\"value is not in range(0, 100)\", exception.getMessage());\n    }\n\n    @Test\n    public void getDefaultValue_ReturnsDefaultValue() {\n        assertEquals(50, longRangeAttribute.getDefaultValue());\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/attribute/TopicMessageTypeTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.attribute;\n\nimport com.google.common.collect.Sets;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.Set;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\n\npublic class TopicMessageTypeTest {\n\n    private Map<String, String> normalMessageProperty;\n    private Map<String, String> transactionMessageProperty;\n    private Map<String, String> delayMessageProperty;\n    private Map<String, String> fifoMessageProperty;\n    private Map<String, String> priorityMessageProperty;\n\n    @Before\n    public void setUp() {\n        normalMessageProperty = new HashMap<>();\n        transactionMessageProperty = new HashMap<>();\n        delayMessageProperty = new HashMap<>();\n        fifoMessageProperty = new HashMap<>();\n        priorityMessageProperty = new HashMap<>();\n\n        transactionMessageProperty.put(MessageConst.PROPERTY_TRANSACTION_PREPARED, \"true\");\n        delayMessageProperty.put(MessageConst.PROPERTY_DELAY_TIME_LEVEL, \"1\");\n        fifoMessageProperty.put(MessageConst.PROPERTY_SHARDING_KEY, \"shardingKey\");\n        priorityMessageProperty.put(MessageConst.PROPERTY_PRIORITY, \"3\");\n    }\n\n    @Test\n    public void testTopicMessageTypeSet() {\n        Set<String> expectedSet\n            = Sets.newHashSet(\"UNSPECIFIED\", \"NORMAL\", \"FIFO\", \"DELAY\", \"TRANSACTION\", \"PRIORITY\", \"LITE\", \"MIXED\");\n        Set<String> actualSet = TopicMessageType.topicMessageTypeSet();\n        assertEquals(expectedSet, actualSet);\n    }\n\n    @Test\n    public void testParseFromMessageProperty_Normal() {\n        TopicMessageType actual = TopicMessageType.parseFromMessageProperty(normalMessageProperty);\n        assertEquals(TopicMessageType.NORMAL, actual);\n    }\n\n    @Test\n    public void testParseFromMessageProperty_Transaction() {\n        TopicMessageType actual = TopicMessageType.parseFromMessageProperty(transactionMessageProperty);\n        assertEquals(TopicMessageType.TRANSACTION, actual);\n    }\n\n    @Test\n    public void testParseFromMessageProperty_Delay() {\n        TopicMessageType actual = TopicMessageType.parseFromMessageProperty(delayMessageProperty);\n        assertEquals(TopicMessageType.DELAY, actual);\n    }\n\n    @Test\n    public void testParseFromMessageProperty_Fifo() {\n        TopicMessageType actual = TopicMessageType.parseFromMessageProperty(fifoMessageProperty);\n        assertEquals(TopicMessageType.FIFO, actual);\n    }\n\n    @Test\n    public void testParseFromMessageProperty_Priority() {\n        TopicMessageType actual = TopicMessageType.parseFromMessageProperty(priorityMessageProperty);\n        assertEquals(TopicMessageType.PRIORITY, actual);\n    }\n\n    @Test\n    public void testGetMetricsValue() {\n        for (TopicMessageType type : TopicMessageType.values()) {\n            String expected = type.getValue().toLowerCase();\n            String actual = type.getMetricsValue();\n            assertEquals(expected, actual);\n        }\n    }\n\n    @Test\n    public void testParseFromMessageProperty() {\n        Map<String, String> properties = new HashMap<>();\n\n        // TRANSACTION\n        properties.put(MessageConst.PROPERTY_TRANSACTION_PREPARED, \"true\");\n        Assert.assertEquals(TopicMessageType.TRANSACTION, TopicMessageType.parseFromMessageProperty(properties));\n\n        // DELAY\n        properties.clear();\n        properties.put(MessageConst.PROPERTY_DELAY_TIME_LEVEL, \"3\");\n        Assert.assertEquals(TopicMessageType.DELAY, TopicMessageType.parseFromMessageProperty(properties));\n\n        properties.clear();\n        properties.put(MessageConst.PROPERTY_TIMER_DELIVER_MS, System.currentTimeMillis() + 10000 + \"\");\n        Assert.assertEquals(TopicMessageType.DELAY, TopicMessageType.parseFromMessageProperty(properties));\n\n        properties.clear();\n        properties.put(MessageConst.PROPERTY_TIMER_DELAY_SEC, 10 + \"\");\n        Assert.assertEquals(TopicMessageType.DELAY, TopicMessageType.parseFromMessageProperty(properties));\n\n        properties.clear();\n        properties.put(MessageConst.PROPERTY_TIMER_DELAY_MS, 10000 + \"\");\n        Assert.assertEquals(TopicMessageType.DELAY, TopicMessageType.parseFromMessageProperty(properties));\n\n        // FIFO\n        properties.clear();\n        properties.put(MessageConst.PROPERTY_SHARDING_KEY, \"sharding_key\");\n        Assert.assertEquals(TopicMessageType.FIFO, TopicMessageType.parseFromMessageProperty(properties));\n\n        // PRIORITY\n        properties.clear();\n        properties.put(MessageConst.PROPERTY_PRIORITY, \"3\");\n        Assert.assertEquals(TopicMessageType.PRIORITY, TopicMessageType.parseFromMessageProperty(properties));\n        properties.put(MessageConst.PROPERTY_DELAY_TIME_LEVEL, \"3\");\n        Assert.assertEquals(TopicMessageType.DELAY, TopicMessageType.parseFromMessageProperty(properties));\n\n        // NORMAL\n        properties.clear();\n        Assert.assertEquals(TopicMessageType.NORMAL, TopicMessageType.parseFromMessageProperty(properties));\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/chain/HandlerChainTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.chain;\n\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNull;\n\npublic class HandlerChainTest {\n\n    private HandlerChain<Integer, String> handlerChain;\n    private Handler<Integer, String> handler1;\n    private Handler<Integer, String> handler2;\n\n    @Before\n    public void setUp() {\n        handlerChain = HandlerChain.create();\n        handler1 = (t, chain) -> \"Handler1\";\n        handler2 = (t, chain) -> null;\n    }\n\n    @Test\n    public void testHandle_withEmptyChain() {\n        handlerChain.addNext(handler1);\n        handlerChain.handle(1);\n        assertNull(\"Expected null since the handler chain is empty\", handlerChain.handle(2));\n    }\n\n    @Test\n    public void testHandle_withNonEmptyChain() {\n        handlerChain.addNext(handler1);\n\n        String result = handlerChain.handle(1);\n\n        assertEquals(\"Handler1\", result);\n    }\n\n    @Test\n    public void testHandle_withMultipleHandlers() {\n        handlerChain.addNext(handler1);\n        handlerChain.addNext(handler2);\n\n        String result1 = handlerChain.handle(1);\n        String result2 = handlerChain.handle(2);\n\n        assertEquals(\"Handler1\", result1);\n        assertNull(\"Expected null since there are no more handlers\", result2);\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/coldctr/AccAndTimeStampTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.coldctr;\n\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\n\npublic class AccAndTimeStampTest {\n\n    private AccAndTimeStamp accAndTimeStamp;\n\n    @Before\n    public void setUp() {\n        accAndTimeStamp = new AccAndTimeStamp(new AtomicLong());\n    }\n\n    @Test\n    public void testInitialValues() {\n        assertEquals(\"Cold accumulator should be initialized to 0\", 0, accAndTimeStamp.getColdAcc().get());\n        assertTrue(\"Last cold read time should be initialized to current time\", accAndTimeStamp.getLastColdReadTimeMills() >= System.currentTimeMillis() - 1000);\n        assertTrue(\"Create time should be initialized to current time\", accAndTimeStamp.getCreateTimeMills() >= System.currentTimeMillis() - 1000);\n    }\n\n    @Test\n    public void testSetColdAcc() {\n        AtomicLong newColdAcc = new AtomicLong(100L);\n        accAndTimeStamp.setColdAcc(newColdAcc);\n        assertEquals(\"Cold accumulator should be set to new value\", newColdAcc, accAndTimeStamp.getColdAcc());\n    }\n\n    @Test\n    public void testSetLastColdReadTimeMills() {\n        long newLastColdReadTimeMills = System.currentTimeMillis() + 1000;\n        accAndTimeStamp.setLastColdReadTimeMills(newLastColdReadTimeMills);\n        assertEquals(\"Last cold read time should be set to new value\", newLastColdReadTimeMills, accAndTimeStamp.getLastColdReadTimeMills().longValue());\n    }\n\n    @Test\n    public void testSetCreateTimeMills() {\n        long newCreateTimeMills = System.currentTimeMillis() + 2000;\n        accAndTimeStamp.setCreateTimeMills(newCreateTimeMills);\n        assertEquals(\"Create time should be set to new value\", newCreateTimeMills, accAndTimeStamp.getCreateTimeMills().longValue());\n    }\n\n    @Test\n    public void testToStringContainsCorrectInformation() {\n        String toStringOutput = accAndTimeStamp.toString();\n        assertTrue(\"ToString should contain cold accumulator value\", toStringOutput.contains(\"coldAcc=\" + accAndTimeStamp.getColdAcc()));\n        assertTrue(\"ToString should contain last cold read time\", toStringOutput.contains(\"lastColdReadTimeMills=\" + accAndTimeStamp.getLastColdReadTimeMills()));\n        assertTrue(\"ToString should contain create time\", toStringOutput.contains(\"createTimeMills=\" + accAndTimeStamp.getCreateTimeMills()));\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/compression/CompressionTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.compression;\n\nimport java.io.IOException;\nimport java.nio.charset.StandardCharsets;\nimport java.util.Random;\nimport org.apache.commons.lang3.RandomStringUtils;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class CompressionTest {\n\n    private int level;\n    private Compressor zstd;\n    private Compressor zlib;\n    private Compressor lz4;\n\n    @Before\n    public void setUp() {\n        level = 5;\n        zstd = CompressorFactory.getCompressor(CompressionType.ZSTD);\n        zlib = CompressorFactory.getCompressor(CompressionType.ZLIB);\n        lz4 = CompressorFactory.getCompressor(CompressionType.LZ4);\n    }\n\n    @Test\n    public void testCompressionZlib() throws IOException {\n        assertThat(CompressionType.of(\"zlib\")).isEqualTo(CompressionType.ZLIB);\n        assertThat(CompressionType.of(\" ZLiB \")).isEqualTo(CompressionType.ZLIB);\n        assertThat(CompressionType.of(\"ZLIB\")).isEqualTo(CompressionType.ZLIB);\n\n        int randomKB = 4096;\n        Random random = new Random();\n        for (int i = 0; i < 5; ++i) {\n            String message = RandomStringUtils.randomAlphanumeric(random.nextInt(randomKB) * 1024);\n            byte[] srcBytes = message.getBytes(StandardCharsets.UTF_8);\n            byte[] compressed = zlib.compress(srcBytes, level);\n            byte[] decompressed = zlib.decompress(compressed);\n            // compression ratio may be negative for some random string data\n            // assertThat(compressed.length).isLessThan(srcBytes.length);\n            assertThat(decompressed).isEqualTo(srcBytes);\n            assertThat(new String(decompressed)).isEqualTo(message);\n        }\n    }\n\n    @Test\n    public void testCompressionZstd() throws IOException {\n        assertThat(CompressionType.of(\"zstd\")).isEqualTo(CompressionType.ZSTD);\n        assertThat(CompressionType.of(\"ZStd \")).isEqualTo(CompressionType.ZSTD);\n        assertThat(CompressionType.of(\"ZSTD\")).isEqualTo(CompressionType.ZSTD);\n\n        int randomKB = 4096;\n        Random random = new Random();\n        for (int i = 0; i < 5; ++i) {\n            String message = RandomStringUtils.randomAlphanumeric(random.nextInt(randomKB) * 1024);\n            byte[] srcBytes = message.getBytes(StandardCharsets.UTF_8);\n            byte[] compressed = zstd.compress(srcBytes, level);\n            byte[] decompressed = zstd.decompress(compressed);\n            // compression ratio may be negative for some random string data\n            // assertThat(compressed.length).isLessThan(srcBytes.length);\n            assertThat(decompressed).isEqualTo(srcBytes);\n            assertThat(new String(decompressed)).isEqualTo(message);\n        }\n    }\n\n    @Test\n    public void testCompressionLz4() throws IOException {\n        assertThat(CompressionType.of(\"lz4\")).isEqualTo(CompressionType.LZ4);\n\n        int randomKB = 4096;\n        Random random = new Random();\n        for (int i = 0; i < 5; ++i) {\n            String message = RandomStringUtils.randomAlphanumeric(random.nextInt(randomKB) * 1024);\n            byte[] srcBytes = message.getBytes(StandardCharsets.UTF_8);\n            byte[] compressed = lz4.compress(srcBytes, level);\n            byte[] decompressed = lz4.decompress(compressed);\n            // compression ratio may be negative for some random string data\n            // assertThat(compressed.length).isLessThan(srcBytes.length);\n            assertThat(decompressed).isEqualTo(srcBytes);\n            assertThat(new String(decompressed)).isEqualTo(message);\n        }\n    }\n\n    @Test(expected = RuntimeException.class)\n    public void testCompressionUnsupportedType() {\n        CompressionType.of(\"snappy\");\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/compression/CompressionTypeTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.compression;\n\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertThrows;\n\npublic class CompressionTypeTest {\n\n    @Test\n    public void testCompressionTypeValues() {\n        assertEquals(1, CompressionType.LZ4.getValue());\n        assertEquals(2, CompressionType.ZSTD.getValue());\n        assertEquals(3, CompressionType.ZLIB.getValue());\n    }\n\n    @Test\n    public void testCompressionTypeOf() {\n        assertEquals(CompressionType.LZ4, CompressionType.of(\"LZ4\"));\n        assertEquals(CompressionType.ZSTD, CompressionType.of(\"ZSTD\"));\n        assertEquals(CompressionType.ZLIB, CompressionType.of(\"ZLIB\"));\n        assertThrows(RuntimeException.class, () -> CompressionType.of(\"UNKNOWN\"));\n    }\n\n    @Test\n    public void testCompressionTypeFindByValue() {\n        assertEquals(CompressionType.LZ4, CompressionType.findByValue(1));\n        assertEquals(CompressionType.ZSTD, CompressionType.findByValue(2));\n        assertEquals(CompressionType.ZLIB, CompressionType.findByValue(3));\n        assertEquals(CompressionType.ZLIB, CompressionType.findByValue(0));\n        assertThrows(RuntimeException.class, () -> CompressionType.findByValue(99));\n    }\n\n    @Test\n    public void testCompressionFlag() {\n        assertEquals(MessageSysFlag.COMPRESSION_LZ4_TYPE, CompressionType.LZ4.getCompressionFlag());\n        assertEquals(MessageSysFlag.COMPRESSION_ZSTD_TYPE, CompressionType.ZSTD.getCompressionFlag());\n        assertEquals(MessageSysFlag.COMPRESSION_ZLIB_TYPE, CompressionType.ZLIB.getCompressionFlag());\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/compression/CompressorFactoryTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.compression;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class CompressorFactoryTest {\n\n    @Test\n    public void testGetCompressor_ReturnsNonNull() {\n        for (CompressionType type : CompressionType.values()) {\n            Compressor compressor = CompressorFactory.getCompressor(type);\n            Assert.assertNotNull(\"Compressor should not be null for type \" + type, compressor);\n        }\n    }\n\n    @Test\n    public void testGetCompressor_ReturnsCorrectType() {\n        for (CompressionType type : CompressionType.values()) {\n            Compressor compressor = CompressorFactory.getCompressor(type);\n            Assert.assertTrue(\"Compressor type mismatch for \" + type,\n                compressor instanceof Lz4Compressor && type == CompressionType.LZ4 ||\n                    compressor instanceof ZstdCompressor && type == CompressionType.ZSTD ||\n                    compressor instanceof ZlibCompressor && type == CompressionType.ZLIB);\n        }\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/compression/Lz4CompressorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.compression;\n\nimport static org.junit.Assert.assertArrayEquals;\nimport static org.junit.Assert.assertTrue;\n\nimport java.io.IOException;\nimport org.junit.Test;\n\npublic class Lz4CompressorTest {\n\n    private static final String TEST_STRING = \"The quick brown fox jumps over the lazy dog\";\n\n    @Test\n    public void testCompressAndDecompress() throws Exception {\n        byte[] originalData = TEST_STRING.getBytes();\n        Compressor compressor = new Lz4Compressor();\n        byte[] compressedData = compressor.compress(originalData, 1);\n        assertTrue(\"Compressed data should be bigger than original\", compressedData.length > originalData.length);\n\n        byte[] decompressedData = compressor.decompress(compressedData);\n        assertArrayEquals(\"Decompressed data should match original data\", originalData, decompressedData);\n    }\n\n    @Test\n    public void testCompressWithIOException() throws Exception {\n        byte[] originalData = new byte[] {1, 2, 3};\n        Compressor compressor = new Lz4Compressor();\n        compressor.compress(originalData, 1);\n    }\n\n    @Test(expected = IOException.class)\n    public void testDecompressWithIOException() throws Exception {\n        byte[] compressedData = new byte[] {1, 2, 3};\n        Compressor compressor = new Lz4Compressor();\n        compressor.decompress(compressedData);\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/compression/ZlibCompressorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.compression;\n\nimport static org.junit.Assert.assertArrayEquals;\nimport static org.junit.Assert.assertTrue;\n\nimport java.io.IOException;\nimport org.junit.Test;\n\npublic class ZlibCompressorTest {\n\n    private static final String TEST_STRING = \"The quick brown fox jumps over the lazy dog\";\n\n    @Test\n    public void testCompressionAndDecompression() throws Exception {\n        byte[] originalData = TEST_STRING.getBytes();\n        ZlibCompressor compressor = new ZlibCompressor();\n        byte[] compressedData = compressor.compress(originalData, 0);\n        assertTrue(\"Compressed data should be bigger than original\", compressedData.length > originalData.length);\n\n        byte[] decompressedData = compressor.decompress(compressedData);\n        assertArrayEquals(\"Decompressed data should match original\", originalData, decompressedData);\n    }\n\n    @Test\n    public void testCompressionFailureWithInvalidData() throws Exception {\n        byte[] originalData = new byte[] {0, 1, 2, 3, 4};\n        ZlibCompressor compressor = new ZlibCompressor();\n        compressor.compress(originalData, 0);\n    }\n\n    @Test(expected = IOException.class)\n    public void testDecompressionFailureWithInvalidData() throws Exception {\n        byte[] compressedData = new byte[] {0, 1, 2, 3, 4};\n        ZlibCompressor compressor = new ZlibCompressor();\n        compressor.decompress(compressedData); // Invalid compressed data\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/compression/ZstdCompressorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.compression;\n\nimport java.io.IOException;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertArrayEquals;\nimport static org.junit.Assert.assertTrue;\n\npublic class ZstdCompressorTest {\n\n    @Test\n    public void testCompressAndDecompress() throws IOException {\n        byte[] originalData = \"RocketMQ is awesome!\".getBytes();\n        ZstdCompressor compressor = new ZstdCompressor();\n        byte[] compressedData = compressor.compress(originalData, 1);\n        assertTrue(\"Compressed data should be bigger than original\", compressedData.length > originalData.length);\n\n        byte[] decompressedData = compressor.decompress(compressedData);\n        assertArrayEquals(\"Decompressed data should match original data\", originalData, decompressedData);\n    }\n\n    @Test\n    public void testCompressWithInvalidData() throws IOException {\n        byte[] invalidData = new byte[] {-1, -1, -1, -1};\n        ZstdCompressor compressor = new ZstdCompressor();\n        compressor.compress(invalidData, 1);\n    }\n\n    @Test(expected = IOException.class)\n    public void testDecompressWithInvalidData() throws IOException {\n        byte[] invalidData = new byte[] {-1, -1, -1, -1};\n        ZstdCompressor compressor = new ZstdCompressor();\n        compressor.decompress(invalidData);\n    }\n\n    @Test\n    public void testCompressAndDecompressEmptyString() throws IOException {\n        byte[] originalData = \"\".getBytes();\n        ZstdCompressor compressor = new ZstdCompressor();\n        byte[] compressedData = compressor.compress(originalData, 1);\n        assertTrue(\"Compressed data for empty string should not be empty\", compressedData.length > 0);\n\n        byte[] decompressedData = compressor.decompress(compressedData);\n        assertArrayEquals(\"Decompressed data for empty string should match original\", originalData, decompressedData);\n    }\n\n    @Test\n    public void testCompressAndDecompressLargeData() throws IOException {\n        StringBuilder largeStringBuilder = new StringBuilder();\n        for (int i = 0; i < 10000; i++) {\n            largeStringBuilder.append(\"RocketMQ is awesome! \");\n        }\n        byte[] originalData = largeStringBuilder.toString().getBytes();\n\n        ZstdCompressor compressor = new ZstdCompressor();\n        byte[] compressedData = compressor.compress(originalData, 1);\n        assertTrue(\"Compressed data for large data should be smaller than original\", compressedData.length < originalData.length);\n\n        byte[] decompressedData = compressor.decompress(compressedData);\n        assertArrayEquals(\"Decompressed data for large data should match original\", originalData, decompressedData);\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/config/ConfigHelperTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.config;\n\nimport org.junit.Test;\n\npublic class ConfigHelperTest {\n\n    @Test\n    public void testGetDBLogDir() {\n        // Should not raise exception.\n        ConfigHelper.getDBLogDir();\n    }\n\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/consumer/ReceiptHandleTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.consumer;\n\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\n\npublic class ReceiptHandleTest {\n\n    @Test\n    public void testEncodeAndDecode() {\n        long startOffset = 1000L;\n        long retrieveTime = System.currentTimeMillis();\n        long invisibleTime = 1000L;\n        int reviveQueueId = 1;\n        String topicType = \"NORMAL\";\n        String brokerName = \"BrokerA\";\n        int queueId = 2;\n        long offset = 2000L;\n        long commitLogOffset = 3000L;\n        ReceiptHandle receiptHandle = ReceiptHandle.builder()\n            .startOffset(startOffset)\n            .retrieveTime(retrieveTime)\n            .invisibleTime(invisibleTime)\n            .reviveQueueId(reviveQueueId)\n            .topicType(topicType)\n            .brokerName(brokerName)\n            .queueId(queueId)\n            .offset(offset)\n            .commitLogOffset(commitLogOffset)\n            .build();\n\n        String encoded = receiptHandle.encode();\n        ReceiptHandle decoded = ReceiptHandle.decode(encoded);\n\n        assertEquals(receiptHandle.getStartOffset(), decoded.getStartOffset());\n        assertEquals(receiptHandle.getRetrieveTime(), decoded.getRetrieveTime());\n        assertEquals(receiptHandle.getInvisibleTime(), decoded.getInvisibleTime());\n        assertEquals(receiptHandle.getReviveQueueId(), decoded.getReviveQueueId());\n        assertEquals(receiptHandle.getTopicType(), decoded.getTopicType());\n        assertEquals(receiptHandle.getBrokerName(), decoded.getBrokerName());\n        assertEquals(receiptHandle.getQueueId(), decoded.getQueueId());\n        assertEquals(receiptHandle.getOffset(), decoded.getOffset());\n        assertEquals(receiptHandle.getCommitLogOffset(), decoded.getCommitLogOffset());\n    }\n\n    @Test(expected = IllegalArgumentException.class)\n    public void testDecodeWithInvalidString() {\n        String invalidReceiptHandle = \"invalid_data\";\n\n        ReceiptHandle.decode(invalidReceiptHandle);\n    }\n\n    @Test\n    public void testIsExpired() {\n        long startOffset = 1000L;\n        long retrieveTime = System.currentTimeMillis();\n        long invisibleTime = 1000L;\n        int reviveQueueId = 1;\n        String topicType = \"NORMAL\";\n        String brokerName = \"BrokerA\";\n        int queueId = 2;\n        long offset = 2000L;\n        long commitLogOffset = 3000L;\n        long pastTime = System.currentTimeMillis() - 1000L;\n        ReceiptHandle receiptHandle = new ReceiptHandle(startOffset, retrieveTime, invisibleTime, pastTime, reviveQueueId, topicType, brokerName, queueId, offset, commitLogOffset, \"\");\n\n        boolean isExpired = receiptHandle.isExpired();\n\n        assertTrue(isExpired);\n    }\n\n    @Test\n    public void testGetRealTopic() {\n        // Arrange\n        String topic = \"TestTopic\";\n        String groupName = \"TestGroup\";\n        ReceiptHandle receiptHandle = ReceiptHandle.builder()\n            .topicType(ReceiptHandle.RETRY_TOPIC)\n            .build();\n\n        String realTopic = receiptHandle.getRealTopic(topic, groupName);\n\n        assertEquals(KeyBuilder.buildPopRetryTopicV1(topic, groupName), realTopic);\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/fastjson/GenericMapSuperclassDeserializerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.fastjson;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.alibaba.fastjson2.JSONException;\nimport org.junit.Test;\n\nimport java.util.HashMap;\nimport java.util.Map;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertTrue;\n\npublic class GenericMapSuperclassDeserializerTest {\n\n    public static class CustomMap extends HashMap<String, Object> {\n        private static final long serialVersionUID = 1L;\n    }\n\n    public static class IntKeyMap extends HashMap<Integer, String> {\n        private static final long serialVersionUID = 1L;\n    }\n\n    @Test\n    public void testBasicDeserialization() {\n        JSON.registerIfAbsent(CustomMap.class, GenericMapSuperclassDeserializer.INSTANCE);\n        String json = \"{\\\"key1\\\":\\\"value1\\\",\\\"key2\\\":42,\\\"key3\\\":true}\";\n        CustomMap map = JSON.parseObject(json, CustomMap.class);\n\n        assertNotNull(map);\n        assertEquals(3, map.size());\n        assertEquals(\"value1\", map.get(\"key1\"));\n        assertEquals(42, map.get(\"key2\"));\n        assertEquals(true, map.get(\"key3\"));\n    }\n\n    @Test\n    public void testNestedObjects() {\n        JSON.registerIfAbsent(CustomMap.class, GenericMapSuperclassDeserializer.INSTANCE);\n        String json = \"{\\\"simple\\\":\\\"value\\\",\\\"nested\\\":{\\\"inner\\\":123},\\\"array\\\":[1,2,3]}\";\n        CustomMap map = JSON.parseObject(json, CustomMap.class);\n\n        assertNotNull(map);\n        assertEquals(3, map.size());\n        assertEquals(\"value\", map.get(\"simple\"));\n\n        assertTrue(map.get(\"nested\") instanceof Map);\n        Map<?, ?> nestedMap = (Map<?, ?>) map.get(\"nested\");\n        assertEquals(123, nestedMap.get(\"inner\"));\n\n        assertTrue(map.get(\"array\") instanceof java.util.List);\n        java.util.List<?> array = (java.util.List<?>) map.get(\"array\");\n        assertEquals(3, array.size());\n        assertEquals(1, array.get(0));\n        assertEquals(2, array.get(1));\n        assertEquals(3, array.get(2));\n    }\n\n    @Test\n    public void testEmptyObject() {\n        JSON.registerIfAbsent(CustomMap.class, GenericMapSuperclassDeserializer.INSTANCE);\n        String json = \"{}\";\n        CustomMap map = JSON.parseObject(json, CustomMap.class);\n\n        assertNotNull(map);\n        assertEquals(0, map.size());\n    }\n\n    @Test\n    public void testNonStringKey() {\n        JSON.registerIfAbsent(IntKeyMap.class, GenericMapSuperclassDeserializer.INSTANCE);\n        String json = \"{1:\\\"one\\\",2:\\\"two\\\",3:\\\"three\\\"}\";\n        IntKeyMap map = JSON.parseObject(json, IntKeyMap.class);\n\n        assertNotNull(map);\n        assertEquals(3, map.size());\n        assertEquals(\"one\", map.get(1));\n        assertEquals(\"two\", map.get(2));\n        assertEquals(\"three\", map.get(3));\n    }\n\n    @Test(expected = JSONException.class)\n    public void testMalformedJson() {\n        JSON.registerIfAbsent(CustomMap.class, GenericMapSuperclassDeserializer.INSTANCE);\n        String json = \"{\\\"key\\\":\\\"missing closing brace\\\"\";\n        JSON.parseObject(json, CustomMap.class);\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/help/FAQUrlTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.help;\n\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\n\npublic class FAQUrlTest {\n\n    @Test\n    public void testSuggestTodo() {\n        String expected = \"\\nSee \" + FAQUrl.DEFAULT_FAQ_URL + \" for further details.\";\n        String actual = FAQUrl.suggestTodo(FAQUrl.DEFAULT_FAQ_URL);\n        assertEquals(expected, actual);\n    }\n\n    @Test\n    public void testAttachDefaultURL() {\n        String errorMsg = \"errorMsg\";\n        String expected = errorMsg\n                + \"\\nFor more information, please visit the url, \"\n                + FAQUrl.UNEXPECTED_EXCEPTION_URL;\n        String actual = FAQUrl.attachDefaultURL(errorMsg);\n        assertEquals(expected, actual);\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/message/MessageClientIDSetterTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.message;\n\nimport org.apache.rocketmq.common.UtilAll;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class MessageClientIDSetterTest {\n\n    @Test\n    public void testGetTimeFromID() {\n        long t = System.currentTimeMillis();\n        String uniqID = MessageClientIDSetter.createUniqID();\n        long t2 = MessageClientIDSetter.getNearlyTimeFromID(uniqID).getTime();\n        assertThat(t2 - t < 20).isTrue();\n    }\n\n    @Test\n    public void testGetCountFromID() {\n        String uniqID = MessageClientIDSetter.createUniqID();\n        String uniqID2 = MessageClientIDSetter.createUniqID();\n        String idHex = uniqID.substring(uniqID.length() - 4);\n        String idHex2 = uniqID2.substring(uniqID2.length() - 4);\n        int s1 = Integer.parseInt(idHex, 16);\n        int s2 = Integer.parseInt(idHex2, 16);\n        assertThat(s1 == s2 - 1).isTrue();\n    }\n\n\n    @Test\n    public void testGetIPStrFromID() {\n        byte[] ip = UtilAll.getIP();\n        String ipStr = (4 == ip.length) ? UtilAll.ipToIPv4Str(ip) : UtilAll.ipToIPv6Str(ip);\n\n        String uniqID = MessageClientIDSetter.createUniqID();\n        String ipStrFromID = MessageClientIDSetter.getIPStrFromID(uniqID);\n\n        assertThat(ipStr).isEqualTo(ipStrFromID);\n    }\n\n\n    @Test\n    public void testGetPidFromID() {\n        // Temporary fix on MacOS\n        short pid = (short) UtilAll.getPid();\n\n        String uniqID = MessageClientIDSetter.createUniqID();\n        short pidFromID = (short) MessageClientIDSetter.getPidFromID(uniqID);\n\n        assertThat(pid).isEqualTo(pidFromID);\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/message/MessageDecoderTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.message;\n\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.junit.Test;\n\nimport java.net.InetAddress;\nimport java.net.InetSocketAddress;\nimport java.net.UnknownHostException;\nimport java.nio.ByteBuffer;\nimport java.util.Map;\n\nimport static org.apache.rocketmq.common.message.MessageDecoder.NAME_VALUE_SEPARATOR;\nimport static org.apache.rocketmq.common.message.MessageDecoder.PROPERTY_SEPARATOR;\nimport static org.apache.rocketmq.common.message.MessageDecoder.createMessageId;\nimport static org.apache.rocketmq.common.message.MessageDecoder.decodeMessageId;\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class MessageDecoderTest {\n\n    @Test\n    public void testDecodeProperties() {\n        MessageExt messageExt = new MessageExt();\n\n        messageExt.setMsgId(\"645100FA00002A9F000000489A3AA09E\");\n        messageExt.setTopic(\"abc\");\n        messageExt.setBody(\"hello!q!\".getBytes());\n        try {\n            messageExt.setBornHost(new InetSocketAddress(InetAddress.getByName(\"127.0.0.1\"), 0));\n        } catch (UnknownHostException e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n        messageExt.setBornTimestamp(System.currentTimeMillis());\n        messageExt.setCommitLogOffset(123456);\n        messageExt.setPreparedTransactionOffset(0);\n        messageExt.setQueueId(0);\n        messageExt.setQueueOffset(123);\n        messageExt.setReconsumeTimes(0);\n        try {\n            messageExt.setStoreHost(new InetSocketAddress(InetAddress.getLocalHost(), 0));\n        } catch (UnknownHostException e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n\n        messageExt.putUserProperty(\"a\", \"123\");\n        messageExt.putUserProperty(\"b\", \"hello\");\n        messageExt.putUserProperty(\"c\", \"3.14\");\n\n        {\n            byte[] msgBytes = new byte[0];\n            try {\n                msgBytes = MessageDecoder.encode(messageExt, false);\n            } catch (Exception e) {\n                e.printStackTrace();\n                assertThat(Boolean.FALSE).isTrue();\n            }\n\n            ByteBuffer byteBuffer = ByteBuffer.allocate(msgBytes.length);\n            byteBuffer.put(msgBytes);\n\n            Map<String, String> properties = MessageDecoder.decodeProperties(byteBuffer);\n\n            assertThat(properties).isNotNull();\n            assertThat(\"123\").isEqualTo(properties.get(\"a\"));\n            assertThat(\"hello\").isEqualTo(properties.get(\"b\"));\n            assertThat(\"3.14\").isEqualTo(properties.get(\"c\"));\n        }\n\n        {\n            byte[] msgBytes = new byte[0];\n            try {\n                msgBytes = MessageDecoder.encode(messageExt, false);\n            } catch (Exception e) {\n                e.printStackTrace();\n                assertThat(Boolean.FALSE).isTrue();\n            }\n\n            ByteBuffer byteBuffer = ByteBuffer.allocate(msgBytes.length);\n            byteBuffer.put(msgBytes);\n\n            Map<String, String> properties = MessageDecoder.decodeProperties(byteBuffer);\n\n            assertThat(properties).isNotNull();\n            assertThat(\"123\").isEqualTo(properties.get(\"a\"));\n            assertThat(\"hello\").isEqualTo(properties.get(\"b\"));\n            assertThat(\"3.14\").isEqualTo(properties.get(\"c\"));\n        }\n    }\n\n    @Test\n    public void testDecodePropertiesOnIPv6Host() {\n        MessageExt messageExt = new MessageExt();\n\n        messageExt.setMsgId(\"24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0\");\n        messageExt.setBornHostV6Flag();\n        messageExt.setStoreHostAddressV6Flag();\n        messageExt.setTopic(\"abc\");\n        messageExt.setBody(\"hello!q!\".getBytes());\n        try {\n            messageExt.setBornHost(new InetSocketAddress(InetAddress.getByName(\"1050:0000:0000:0000:0005:0600:300c:326b\"), 0));\n        } catch (UnknownHostException e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n        messageExt.setBornTimestamp(System.currentTimeMillis());\n        messageExt.setCommitLogOffset(123456);\n        messageExt.setPreparedTransactionOffset(0);\n        messageExt.setQueueId(0);\n        messageExt.setQueueOffset(123);\n        messageExt.setReconsumeTimes(0);\n        try {\n            messageExt.setStoreHost(new InetSocketAddress(InetAddress.getByName(\"::1\"), 0));\n        } catch (UnknownHostException e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n\n        messageExt.putUserProperty(\"a\", \"123\");\n        messageExt.putUserProperty(\"b\", \"hello\");\n        messageExt.putUserProperty(\"c\", \"3.14\");\n\n        byte[] msgBytes = new byte[0];\n        try {\n            msgBytes = MessageDecoder.encode(messageExt, false);\n        } catch (Exception e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n\n        ByteBuffer byteBuffer = ByteBuffer.allocate(msgBytes.length);\n        byteBuffer.put(msgBytes);\n\n        Map<String, String> properties = MessageDecoder.decodeProperties(byteBuffer);\n\n        assertThat(properties).isNotNull();\n        assertThat(\"123\").isEqualTo(properties.get(\"a\"));\n        assertThat(\"hello\").isEqualTo(properties.get(\"b\"));\n        assertThat(\"3.14\").isEqualTo(properties.get(\"c\"));\n    }\n\n    @Test\n    public void testEncodeAndDecode() {\n        MessageExt messageExt = new MessageExt();\n\n        messageExt.setMsgId(\"645100FA00002A9F000000489A3AA09E\");\n        messageExt.setTopic(\"abc\");\n        messageExt.setBody(\"hello!q!\".getBytes());\n        try {\n            messageExt.setBornHost(new InetSocketAddress(InetAddress.getByName(\"127.0.0.1\"), 0));\n        } catch (UnknownHostException e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n        messageExt.setBornTimestamp(System.currentTimeMillis());\n        messageExt.setCommitLogOffset(123456);\n        messageExt.setPreparedTransactionOffset(0);\n        messageExt.setQueueId(1);\n        messageExt.setQueueOffset(123);\n        messageExt.setReconsumeTimes(0);\n        try {\n            messageExt.setStoreHost(new InetSocketAddress(InetAddress.getLocalHost(), 0));\n        } catch (UnknownHostException e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n\n        messageExt.putUserProperty(\"a\", \"123\");\n        messageExt.putUserProperty(\"b\", \"hello\");\n        messageExt.putUserProperty(\"c\", \"3.14\");\n\n        messageExt.setBodyCRC(UtilAll.crc32(messageExt.getBody()));\n\n        byte[] msgBytes = new byte[0];\n        try {\n            msgBytes = MessageDecoder.encode(messageExt, false);\n        } catch (Exception e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n\n        ByteBuffer byteBuffer = ByteBuffer.allocate(msgBytes.length);\n        byteBuffer.put(msgBytes);\n\n        byteBuffer.flip();\n        MessageExt decodedMsg = MessageDecoder.decode(byteBuffer);\n\n        assertThat(decodedMsg).isNotNull();\n        assertThat(1).isEqualTo(decodedMsg.getQueueId());\n        assertThat(123456L).isEqualTo(decodedMsg.getCommitLogOffset());\n        assertThat(\"hello!q!\".getBytes()).isEqualTo(decodedMsg.getBody());\n\n        int msgIDLength = 4 + 4 + 8;\n        ByteBuffer byteBufferMsgId = ByteBuffer.allocate(msgIDLength);\n        String msgId = createMessageId(byteBufferMsgId, messageExt.getStoreHostBytes(), messageExt.getCommitLogOffset());\n        assertThat(msgId).isEqualTo(decodedMsg.getMsgId());\n\n        assertThat(\"abc\").isEqualTo(decodedMsg.getTopic());\n    }\n\n    @Test\n    public void testEncodeAndDecodeOnIPv6Host() {\n        MessageExt messageExt = new MessageExt();\n\n        messageExt.setMsgId(\"24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0\");\n        messageExt.setBornHostV6Flag();\n        messageExt.setStoreHostAddressV6Flag();\n        messageExt.setTopic(\"abc\");\n        messageExt.setBody(\"hello!q!\".getBytes());\n        try {\n            messageExt.setBornHost(new InetSocketAddress(InetAddress.getByName(\"1050:0000:0000:0000:0005:0600:300c:326b\"), 0));\n        } catch (UnknownHostException e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n        messageExt.setBornTimestamp(System.currentTimeMillis());\n        messageExt.setCommitLogOffset(123456);\n        messageExt.setPreparedTransactionOffset(0);\n        messageExt.setQueueId(1);\n        messageExt.setQueueOffset(123);\n        messageExt.setReconsumeTimes(0);\n        try {\n            messageExt.setStoreHost(new InetSocketAddress(InetAddress.getByName(\"::1\"), 0));\n        } catch (UnknownHostException e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n\n        messageExt.putUserProperty(\"a\", \"123\");\n        messageExt.putUserProperty(\"b\", \"hello\");\n        messageExt.putUserProperty(\"c\", \"3.14\");\n\n        messageExt.setBodyCRC(UtilAll.crc32(messageExt.getBody()));\n\n        byte[] msgBytes = new byte[0];\n        try {\n            msgBytes = MessageDecoder.encode(messageExt, false);\n        } catch (Exception e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n\n        ByteBuffer byteBuffer = ByteBuffer.allocate(msgBytes.length);\n        byteBuffer.put(msgBytes);\n\n        byteBuffer.flip();\n        MessageExt decodedMsg = MessageDecoder.decode(byteBuffer);\n\n        assertThat(decodedMsg).isNotNull();\n        assertThat(1).isEqualTo(decodedMsg.getQueueId());\n        assertThat(123456L).isEqualTo(decodedMsg.getCommitLogOffset());\n        assertThat(\"hello!q!\".getBytes()).isEqualTo(decodedMsg.getBody());\n        // assertThat(48).isEqualTo(decodedMsg.getSysFlag());\n        assertThat(MessageSysFlag.check(messageExt.getSysFlag(), MessageSysFlag.STOREHOSTADDRESS_V6_FLAG)).isTrue();\n\n        int msgIDLength = 16 + 4 + 8;\n        ByteBuffer byteBufferMsgId = ByteBuffer.allocate(msgIDLength);\n        String msgId = createMessageId(byteBufferMsgId, messageExt.getStoreHostBytes(), messageExt.getCommitLogOffset());\n        assertThat(msgId).isEqualTo(decodedMsg.getMsgId());\n\n        assertThat(\"abc\").isEqualTo(decodedMsg.getTopic());\n    }\n\n    @Test\n    public void testNullValueProperty() throws Exception {\n        MessageExt msg = new MessageExt();\n        msg.setBody(\"x\".getBytes());\n        msg.setTopic(\"x\");\n        msg.setBornHost(new InetSocketAddress(\"127.0.0.1\", 9000));\n        msg.setStoreHost(new InetSocketAddress(\"127.0.0.1\", 9000));\n        String key = \"NullValueKey\";\n        msg.putProperty(key, null);\n        try {\n            byte[] encode = MessageDecoder.encode(msg, false);\n            MessageExt decode = MessageDecoder.decode(ByteBuffer.wrap(encode));\n            assertThat(decode.getProperty(key)).isNull();\n        } catch (Exception e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n    }\n\n    @Test\n    public void testString2messageProperties() {\n        StringBuilder sb = new StringBuilder();\n        sb.append(\"k1\").append(NAME_VALUE_SEPARATOR).append(\"v1\");\n        Map<String,String> m = MessageDecoder.string2messageProperties(sb.toString());\n        assertThat(m).size().isEqualTo(1);\n        assertThat(m.get(\"k1\")).isEqualTo(\"v1\");\n\n        m = MessageDecoder.string2messageProperties(\"\");\n        assertThat(m).size().isEqualTo(0);\n\n        m = MessageDecoder.string2messageProperties(\" \");\n        assertThat(m).size().isEqualTo(0);\n\n        m = MessageDecoder.string2messageProperties(\"aaa\");\n        assertThat(m).size().isEqualTo(0);\n\n        sb.setLength(0);\n        sb.append(\"k1\").append(NAME_VALUE_SEPARATOR);\n        m = MessageDecoder.string2messageProperties(sb.toString());\n        assertThat(m).size().isEqualTo(0);\n\n        sb.setLength(0);\n        sb.append(NAME_VALUE_SEPARATOR).append(\"v1\");\n        m = MessageDecoder.string2messageProperties(sb.toString());\n        assertThat(m).size().isEqualTo(0);\n\n        sb.setLength(0);\n        sb.append(\"k1\").append(NAME_VALUE_SEPARATOR).append(\"v1\").append(PROPERTY_SEPARATOR);\n        m = MessageDecoder.string2messageProperties(sb.toString());\n        assertThat(m).size().isEqualTo(1);\n        assertThat(m.get(\"k1\")).isEqualTo(\"v1\");\n\n        sb.setLength(0);\n        sb.append(\"k1\").append(NAME_VALUE_SEPARATOR).append(\"v1\").append(PROPERTY_SEPARATOR)\n                .append(\"k2\").append(NAME_VALUE_SEPARATOR).append(\"v2\");\n        m = MessageDecoder.string2messageProperties(sb.toString());\n        assertThat(m).size().isEqualTo(2);\n        assertThat(m.get(\"k1\")).isEqualTo(\"v1\");\n        assertThat(m.get(\"k2\")).isEqualTo(\"v2\");\n\n        sb.setLength(0);\n        sb.append(\"k1\").append(NAME_VALUE_SEPARATOR).append(\"v1\").append(PROPERTY_SEPARATOR)\n                .append(NAME_VALUE_SEPARATOR).append(\"v2\");\n        m = MessageDecoder.string2messageProperties(sb.toString());\n        assertThat(m).size().isEqualTo(1);\n        assertThat(m.get(\"k1\")).isEqualTo(\"v1\");\n\n        sb.setLength(0);\n        sb.append(\"k1\").append(NAME_VALUE_SEPARATOR).append(\"v1\").append(PROPERTY_SEPARATOR)\n                .append(\"k2\").append(NAME_VALUE_SEPARATOR);\n        m = MessageDecoder.string2messageProperties(sb.toString());\n        assertThat(m).size().isEqualTo(1);\n        assertThat(m.get(\"k1\")).isEqualTo(\"v1\");\n\n        sb.setLength(0);\n        sb.append(NAME_VALUE_SEPARATOR).append(\"v1\").append(PROPERTY_SEPARATOR)\n                .append(\"k2\").append(NAME_VALUE_SEPARATOR).append(\"v2\");\n        m = MessageDecoder.string2messageProperties(sb.toString());\n        assertThat(m).size().isEqualTo(1);\n        assertThat(m.get(\"k2\")).isEqualTo(\"v2\");\n\n        sb.setLength(0);\n        sb.append(\"k1\").append(NAME_VALUE_SEPARATOR).append(PROPERTY_SEPARATOR)\n                .append(\"k2\").append(NAME_VALUE_SEPARATOR).append(\"v2\");\n        m = MessageDecoder.string2messageProperties(sb.toString());\n        assertThat(m).size().isEqualTo(1);\n        assertThat(m.get(\"k2\")).isEqualTo(\"v2\");\n\n        sb.setLength(0);\n        sb.append(\"1\").append(NAME_VALUE_SEPARATOR).append(\"1\").append(PROPERTY_SEPARATOR)\n                .append(\"2\").append(NAME_VALUE_SEPARATOR).append(\"2\");\n        m = MessageDecoder.string2messageProperties(sb.toString());\n        assertThat(m).size().isEqualTo(2);\n        assertThat(m.get(\"1\")).isEqualTo(\"1\");\n        assertThat(m.get(\"2\")).isEqualTo(\"2\");\n\n        sb.setLength(0);\n        sb.append(\"1\").append(NAME_VALUE_SEPARATOR).append(PROPERTY_SEPARATOR)\n                .append(\"2\").append(NAME_VALUE_SEPARATOR).append(\"2\");\n        m = MessageDecoder.string2messageProperties(sb.toString());\n        assertThat(m).size().isEqualTo(1);\n        assertThat(m.get(\"2\")).isEqualTo(\"2\");\n\n        sb.setLength(0);\n        sb.append(NAME_VALUE_SEPARATOR).append(\"1\").append(PROPERTY_SEPARATOR)\n                .append(\"2\").append(NAME_VALUE_SEPARATOR).append(\"2\");\n        m = MessageDecoder.string2messageProperties(sb.toString());\n        assertThat(m).size().isEqualTo(1);\n        assertThat(m.get(\"2\")).isEqualTo(\"2\");\n\n        sb.setLength(0);\n        sb.append(\"1\").append(NAME_VALUE_SEPARATOR).append(\"1\").append(PROPERTY_SEPARATOR)\n                .append(\"2\").append(NAME_VALUE_SEPARATOR);\n        m = MessageDecoder.string2messageProperties(sb.toString());\n        assertThat(m).size().isEqualTo(1);\n        assertThat(m.get(\"1\")).isEqualTo(\"1\");\n\n        sb.setLength(0);\n        sb.append(\"1\").append(NAME_VALUE_SEPARATOR).append(\"1\").append(PROPERTY_SEPARATOR)\n                .append(NAME_VALUE_SEPARATOR).append(\"2\");\n        m = MessageDecoder.string2messageProperties(sb.toString());\n        assertThat(m).size().isEqualTo(1);\n        assertThat(m.get(\"1\")).isEqualTo(\"1\");\n    }\n\n    @Test\n    public void testMessageId() throws Exception {\n        // ipv4 messageId test\n        MessageExt msgExt = new MessageExt();\n        msgExt.setStoreHost(new InetSocketAddress(\"127.0.0.1\", 9103));\n        msgExt.setCommitLogOffset(123456);\n        verifyMessageId(msgExt);\n\n        // ipv6 messageId test\n        msgExt.setStoreHostAddressV6Flag();\n        msgExt.setStoreHost(new InetSocketAddress(InetAddress.getByName(\"::1\"), 0));\n        verifyMessageId(msgExt);\n    }\n\n    private void verifyMessageId(MessageExt msgExt) throws UnknownHostException {\n        int storehostIPLength = (msgExt.getSysFlag() & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 : 16;\n        int msgIDLength = storehostIPLength + 4 + 8;\n        ByteBuffer byteBufferMsgId = ByteBuffer.allocate(msgIDLength);\n        String msgId = createMessageId(byteBufferMsgId, msgExt.getStoreHostBytes(), msgExt.getCommitLogOffset());\n\n        MessageId messageId = decodeMessageId(msgId);\n        assertThat(messageId.getAddress()).isEqualTo(msgExt.getStoreHost());\n        assertThat(messageId.getOffset()).isEqualTo(msgExt.getCommitLogOffset());\n    }\n}"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/message/MessageTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.message;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport static org.apache.rocketmq.common.message.MessageConst.PROPERTY_TRACE_SWITCH;\n\npublic class MessageTest {\n    @Test(expected = RuntimeException.class)\n    public void putUserPropertyWithRuntimeException() throws Exception {\n        Message m = new Message();\n\n        m.putUserProperty(PROPERTY_TRACE_SWITCH, \"\");\n    }\n\n    @Test(expected = IllegalArgumentException.class)\n    public void putUserNullValuePropertyWithException() throws Exception {\n        Message m = new Message();\n\n        m.putUserProperty(\"prop1\", null);\n    }\n\n    @Test(expected = IllegalArgumentException.class)\n    public void putUserEmptyValuePropertyWithException() throws Exception {\n        Message m = new Message();\n\n        m.putUserProperty(\"prop1\", \"   \");\n    }\n\n    @Test(expected = IllegalArgumentException.class)\n    public void putUserNullNamePropertyWithException() throws Exception {\n        Message m = new Message();\n\n        m.putUserProperty(null, \"val1\");\n    }\n\n    @Test(expected = IllegalArgumentException.class)\n    public void putUserEmptyNamePropertyWithException() throws Exception {\n        Message m = new Message();\n\n        m.putUserProperty(\"   \", \"val1\");\n    }\n\n    @Test\n    public void putUserProperty() throws Exception {\n        Message m = new Message();\n\n        m.putUserProperty(\"prop1\", \"val1\");\n        Assert.assertEquals(\"val1\", m.getUserProperty(\"prop1\"));\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/producer/RecallMessageHandleTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.producer;\n\nimport org.apache.commons.codec.DecoderException;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.Base64;\n\npublic class RecallMessageHandleTest {\n    @Test\n    public void testHandleInvalid() {\n        Assert.assertThrows(DecoderException.class, () -> {\n            RecallMessageHandle.decodeHandle(\"\");\n        });\n        Assert.assertThrows(DecoderException.class, () -> {\n            RecallMessageHandle.decodeHandle(null);\n        });\n\n        Assert.assertThrows(DecoderException.class, () -> {\n            String invalidHandle = Base64.getUrlEncoder().encodeToString(\"v1 a b c\".getBytes(StandardCharsets.UTF_8));\n            RecallMessageHandle.decodeHandle(invalidHandle);\n        });\n        Assert.assertThrows(DecoderException.class, () -> {\n            String invalidHandle = Base64.getUrlEncoder().encodeToString(\"v2 a b c d\".getBytes(StandardCharsets.UTF_8));\n            RecallMessageHandle.decodeHandle(invalidHandle);\n        });\n        Assert.assertThrows(DecoderException.class, () -> {\n            String invalidHandle = \"v1 a b c d\";\n            RecallMessageHandle.decodeHandle(invalidHandle);\n        });\n    }\n\n    @Test\n    public void testEncodeAndDecodeV1() throws DecoderException {\n        String topic = \"topic\";\n        String brokerName = \"broker-0\";\n        String timestampStr = String.valueOf(System.currentTimeMillis());\n        String messageId = MessageClientIDSetter.createUniqID();\n        String handle = RecallMessageHandle.HandleV1.buildHandle(topic, brokerName, timestampStr, messageId);\n        RecallMessageHandle handleEntity = RecallMessageHandle.decodeHandle(handle);\n        Assert.assertTrue(handleEntity instanceof RecallMessageHandle.HandleV1);\n        RecallMessageHandle.HandleV1 handleV1 = (RecallMessageHandle.HandleV1) handleEntity;\n        Assert.assertEquals(handleV1.getVersion(), \"v1\");\n        Assert.assertEquals(handleV1.getTopic(), topic);\n        Assert.assertEquals(handleV1.getBrokerName(), brokerName);\n        Assert.assertEquals(handleV1.getTimestampStr(), timestampStr);\n        Assert.assertEquals(handleV1.getMessageId(), messageId);\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/stats/StatsItemSetTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.stats;\n\nimport java.util.concurrent.ArrayBlockingQueue;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.concurrent.atomic.LongAdder;\n\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.junit.After;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\n\npublic class StatsItemSetTest {\n\n    private ThreadPoolExecutor executor;\n    private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);\n\n    @Test\n    public void test_getAndCreateStatsItem_multiThread() throws InterruptedException {\n        assertEquals(20L, test_unit().longValue());\n    }\n\n    @Test\n    public void test_getAndCreateMomentStatsItem_multiThread() throws InterruptedException {\n        assertEquals(10, test_unit_moment().longValue());\n    }\n\n    @Test\n    public void test_statsOfFirstStatisticsCycle() throws InterruptedException {\n        final String tpsStatKey = \"tpsTest\";\n        final String rtStatKey = \"rtTest\";\n        final StatsItemSet statsItemSet = new StatsItemSet(tpsStatKey, scheduler, null);\n        executor = new ThreadPoolExecutor(10, 20, 10, TimeUnit.SECONDS,\n            new ArrayBlockingQueue<>(100), new ThreadFactoryImpl(\"testMultiThread\"));\n        for (int i = 0; i < 10; i++) {\n            executor.submit(new Runnable() {\n                @Override\n                public void run() {\n                    statsItemSet.addValue(tpsStatKey, 2, 1);\n                    statsItemSet.addRTValue(rtStatKey, 2, 1);\n                }\n            });\n        }\n        while (true) {\n            if (executor.getCompletedTaskCount() == 10) {\n                break;\n            }\n            Thread.sleep(1000);\n        }\n        // simulate schedule task execution , tps stat\n        {\n            statsItemSet.getStatsItem(tpsStatKey).samplingInSeconds();\n            statsItemSet.getStatsItem(tpsStatKey).samplingInMinutes();\n            statsItemSet.getStatsItem(tpsStatKey).samplingInHour();\n\n            assertEquals(20L, statsItemSet.getStatsDataInMinute(tpsStatKey).getSum());\n            assertEquals(20L, statsItemSet.getStatsDataInHour(tpsStatKey).getSum());\n            assertEquals(20L, statsItemSet.getStatsDataInDay(tpsStatKey).getSum());\n            assertEquals(10L, statsItemSet.getStatsDataInDay(tpsStatKey).getTimes());\n            assertEquals(10L, statsItemSet.getStatsDataInHour(tpsStatKey).getTimes());\n            assertEquals(10L, statsItemSet.getStatsDataInDay(tpsStatKey).getTimes());\n        }\n\n        // simulate schedule task execution , rt stat\n        {\n            statsItemSet.getStatsItem(rtStatKey).samplingInSeconds();\n            statsItemSet.getStatsItem(rtStatKey).samplingInMinutes();\n            statsItemSet.getStatsItem(rtStatKey).samplingInHour();\n\n            assertEquals(20L, statsItemSet.getStatsDataInMinute(rtStatKey).getSum());\n            assertEquals(20L, statsItemSet.getStatsDataInHour(rtStatKey).getSum());\n            assertEquals(20L, statsItemSet.getStatsDataInDay(rtStatKey).getSum());\n            assertEquals(10L, statsItemSet.getStatsDataInDay(rtStatKey).getTimes());\n            assertEquals(10L, statsItemSet.getStatsDataInHour(rtStatKey).getTimes());\n            assertEquals(10L, statsItemSet.getStatsDataInDay(rtStatKey).getTimes());\n        }\n    }\n\n    private LongAdder test_unit() throws InterruptedException {\n        final StatsItemSet statsItemSet = new StatsItemSet(\"topicTest\", scheduler, null);\n        executor = new ThreadPoolExecutor(10, 20, 10, TimeUnit.SECONDS,\n            new ArrayBlockingQueue<>(100), new ThreadFactoryImpl(\"testMultiThread\"));\n        for (int i = 0; i < 10; i++) {\n            executor.submit(new Runnable() {\n                @Override\n                public void run() {\n                    statsItemSet.addValue(\"topicTest\", 2, 1);\n                }\n            });\n        }\n        while (true) {\n            if (executor.getCompletedTaskCount() == 10) {\n                break;\n            }\n            Thread.sleep(1000);\n        }\n        return statsItemSet.getStatsItem(\"topicTest\").getValue();\n    }\n\n    private AtomicLong test_unit_moment() throws InterruptedException {\n        final MomentStatsItemSet statsItemSet = new MomentStatsItemSet(\"topicTest\", scheduler, null);\n        executor = new ThreadPoolExecutor(10, 20, 10, TimeUnit.SECONDS,\n            new ArrayBlockingQueue<>(100), new ThreadFactoryImpl(\"testMultiThread\"));\n        for (int i = 0; i < 10; i++) {\n            executor.submit(new Runnable() {\n                @Override\n                public void run() {\n                    statsItemSet.setValue(\"test\", 10);\n                }\n            });\n        }\n        while (true) {\n            if (executor.getCompletedTaskCount() == 10) {\n                break;\n            }\n            Thread.sleep(1000);\n        }\n        return statsItemSet.getAndCreateStatsItem(\"test\").getValue();\n    }\n\n    @After\n    public void shutdown() {\n        executor.shutdown();\n    }\n}"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/sysflag/CompressionFlagTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.sysflag;\n\nimport org.apache.rocketmq.common.compression.CompressionType;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class CompressionFlagTest {\n\n    @Test\n    public void testCompressionFlag() {\n        int flag = 0;\n        flag |= MessageSysFlag.COMPRESSED_FLAG;\n\n        assertThat(MessageSysFlag.getCompressionType(flag)).isEqualTo(CompressionType.ZLIB);\n\n        flag |= MessageSysFlag.COMPRESSION_LZ4_TYPE;\n        assertThat(MessageSysFlag.getCompressionType(flag)).isEqualTo(CompressionType.LZ4);\n\n        flag &= ~MessageSysFlag.COMPRESSION_TYPE_COMPARATOR;\n        flag |= MessageSysFlag.COMPRESSION_ZSTD_TYPE;\n        assertThat(MessageSysFlag.getCompressionType(flag)).isEqualTo(CompressionType.ZSTD);\n\n\n        flag &= ~MessageSysFlag.COMPRESSION_TYPE_COMPARATOR;\n        flag |= MessageSysFlag.COMPRESSION_ZLIB_TYPE;\n        assertThat(MessageSysFlag.getCompressionType(flag)).isEqualTo(CompressionType.ZLIB);\n    }\n\n    @Test(expected = RuntimeException.class)\n    public void testCompressionFlagNotMatch() {\n        int flag = 0;\n        flag |= MessageSysFlag.COMPRESSED_FLAG;\n        flag |= 0x4 << 8;\n\n        MessageSysFlag.getCompressionType(flag);\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/sysflag/PullSysFlagTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.sysflag;\n\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class PullSysFlagTest {\n\n    @Test\n    public void testLitePullFlag() {\n        int flag = PullSysFlag.buildSysFlag(false, false, false, false, true);\n        assertThat(PullSysFlag.hasLitePullFlag(flag)).isTrue();\n    }\n\n    @Test\n    public void testLitePullFlagFalse() {\n        int flag = PullSysFlag.buildSysFlag(false, false, false, false, false);\n        assertThat(PullSysFlag.hasLitePullFlag(flag)).isFalse();\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/topic/TopicValidatorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.topic;\n\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class TopicValidatorTest {\n\n    @Test\n    public void testTopicValidator_NotPass() {\n        TopicValidator.ValidateResult res = TopicValidator.validateTopic(\"\");\n        assertThat(res.isValid()).isFalse();\n        assertThat(res.getRemark()).contains(\"The specified topic is blank\");\n\n        res = TopicValidator.validateTopic(\"../TopicTest\");\n        assertThat(res.isValid()).isFalse();\n\n        res = TopicValidator.validateTopic(generateString(128));\n        assertThat(res.isValid()).isFalse();\n\n        res = TopicValidator.validateTopic(generateRetryTopic(256));\n        assertThat(res.isValid()).isFalse();\n\n        res = TopicValidator.validateTopic(generateDlqTopic(256));\n        assertThat(res.isValid()).isFalse();\n    }\n\n    @Test\n    public void testTopicValidator_Pass() {\n        TopicValidator.ValidateResult res = TopicValidator.validateTopic(\"TestTopic\");\n        assertThat(res.isValid()).isTrue();\n        assertThat(res.getRemark()).isEmpty();\n\n        res = TopicValidator.validateTopic(generateString2(127));\n        assertThat(res.isValid()).isTrue();\n        assertThat(res.getRemark()).isEmpty();\n\n        res = TopicValidator.validateTopic(generateRetryTopic(255));\n        assertThat(res.isValid()).isTrue();\n        assertThat(res.getRemark()).isEmpty();\n\n        res = TopicValidator.validateTopic(generateDlqTopic(255));\n        assertThat(res.isValid()).isTrue();\n        assertThat(res.getRemark()).isEmpty();\n    }\n\n    @Test\n    public void testGroupValidator_Pass() {\n        TopicValidator.ValidateResult res = TopicValidator.validateGroup(\"TestGroup\");\n        assertThat(res.isValid()).isTrue();\n        assertThat(res.getRemark()).isEmpty();\n\n        res = TopicValidator.validateGroup(generateString2(120));\n        assertThat(res.isValid()).isTrue();\n        assertThat(res.getRemark()).isEmpty();\n    }\n\n    @Test\n    public void testGroupValidator__NotPass() {\n        TopicValidator.ValidateResult res = TopicValidator.validateGroup(\"\");\n        assertThat(res.isValid()).isFalse();\n        assertThat(res.getRemark()).contains(\"The specified group is blank\");\n\n        res = TopicValidator.validateGroup(\"../GroupTest\");\n        assertThat(res.isValid()).isFalse();\n\n        res = TopicValidator.validateGroup(generateString(120));\n        assertThat(res.isValid()).isFalse();\n\n        res = TopicValidator.validateGroup(generateString2(121));\n        assertThat(res.isValid()).isFalse();\n    }\n\n    @Test\n    public void testAddSystemTopic() {\n        String topic = \"SYSTEM_TOPIC_TEST\";\n        TopicValidator.addSystemTopic(topic);\n        assertThat(TopicValidator.getSystemTopicSet()).contains(topic);\n    }\n\n    @Test\n    public void testIsSystemTopic() {\n        boolean res;\n        for (String topic : TopicValidator.getSystemTopicSet()) {\n            res = TopicValidator.isSystemTopic(topic);\n            assertThat(res).isTrue();\n        }\n\n        String topic = TopicValidator.SYSTEM_TOPIC_PREFIX + \"_test\";\n        res = TopicValidator.isSystemTopic(topic);\n        assertThat(res).isTrue();\n\n        topic = \"test_not_system_topic\";\n        res = TopicValidator.isSystemTopic(topic);\n        assertThat(res).isFalse();\n    }\n\n    @Test\n    public void testIsSystemTopicWithResponse() {\n        boolean res;\n        for (String topic : TopicValidator.getSystemTopicSet()) {\n            res = TopicValidator.isSystemTopic(topic);\n            assertThat(res).isTrue();\n        }\n\n        String topic = \"test_not_system_topic\";\n        res = TopicValidator.isSystemTopic(topic);\n        assertThat(res).isFalse();\n    }\n\n    @Test\n    public void testIsNotAllowedSendTopic() {\n        boolean res;\n        for (String topic : TopicValidator.getNotAllowedSendTopicSet()) {\n            res = TopicValidator.isNotAllowedSendTopic(topic);\n            assertThat(res).isTrue();\n        }\n\n        String topic = \"test_allowed_send_topic\";\n        res = TopicValidator.isNotAllowedSendTopic(topic);\n        assertThat(res).isFalse();\n    }\n\n    @Test\n    public void testIsNotAllowedSendTopicWithResponse() {\n        boolean res;\n        for (String topic : TopicValidator.getNotAllowedSendTopicSet()) {\n            res = TopicValidator.isNotAllowedSendTopic(topic);\n            assertThat(res).isTrue();\n        }\n\n        String topic = \"test_allowed_send_topic\";\n        res = TopicValidator.isNotAllowedSendTopic(topic);\n        assertThat(res).isFalse();\n    }\n\n    private static String generateString(int length) {\n        StringBuilder stringBuffer = new StringBuilder();\n        String tmpStr = \"0123456789\";\n        for (int i = 0; i < length; i++) {\n            stringBuffer.append(tmpStr);\n        }\n        return stringBuffer.toString();\n    }\n\n    private static String generateString2(int length) {\n        StringBuilder stringBuilder = new StringBuilder();\n        for (int i = 0; i < length; i++) {\n            stringBuilder.append(\"a\");\n        }\n        return stringBuilder.toString();\n    }\n\n    private static String generateRetryTopic(int length) {\n        StringBuilder stringBuilder = new StringBuilder();\n        stringBuilder.append(\"%RETRY%\");\n        for (int i = 0; i < length - 7; i++) {\n            stringBuilder.append(\"a\");\n        }\n        return stringBuilder.toString();\n    }\n\n    private static String generateDlqTopic(int length) {\n        StringBuilder stringBuilder = new StringBuilder();\n        stringBuilder.append(\"%DLQ%\");\n        for (int i = 0; i < length - 5; i++) {\n            stringBuilder.append(\"a\");\n        }\n        return stringBuilder.toString();\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/utils/ConcurrentHashMapUtilsTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.utils;\n\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\n\npublic class ConcurrentHashMapUtilsTest {\n\n    @Test\n    public void computeIfAbsent() {\n        ConcurrentHashMap<String, String> map = new ConcurrentHashMap<>();\n        map.put(\"123\", \"1111\");\n        String value = ConcurrentHashMapUtils.computeIfAbsent(map, \"123\", k -> \"234\");\n        assertEquals(\"1111\", value);\n        String value1 = ConcurrentHashMapUtils.computeIfAbsent(map, \"1232\", k -> \"2342\");\n        assertEquals(\"2342\", value1);\n        String value2 = ConcurrentHashMapUtils.computeIfAbsent(map, \"123\", k -> \"2342\");\n        assertEquals(\"1111\", value2);\n    }\n}"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/utils/IOTinyUtilsTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.utils;\n\nimport java.io.BufferedReader;\nimport java.io.ByteArrayInputStream;\nimport java.io.CharArrayReader;\nimport java.io.CharArrayWriter;\nimport java.io.File;\nimport java.io.InputStream;\nimport java.io.InputStreamReader;\nimport java.io.Reader;\nimport java.io.StringReader;\nimport java.io.Writer;\nimport java.lang.reflect.Method;\nimport java.nio.charset.StandardCharsets;\nimport java.util.List;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\n\npublic class IOTinyUtilsTest {\n\n    /**\n     * https://bazel.build/reference/test-encyclopedia#filesystem\n     */\n    private String testRootDir = System.getProperty(\"java.io.tmpdir\") + File.separator + \"iotinyutilstest\";\n\n    @Before\n    public void init() {\n        File dir = new File(testRootDir);\n        if (dir.exists()) {\n            UtilAll.deleteFile(dir);\n        }\n\n        dir.mkdirs();\n    }\n\n    @After\n    public void destroy() {\n        File file = new File(testRootDir);\n        UtilAll.deleteFile(file);\n    }\n\n    @Test\n    public void testToString() throws Exception {\n        byte[] b = \"testToString\".getBytes(StandardCharsets.UTF_8);\n        InputStream is = new ByteArrayInputStream(b);\n\n        String str = IOTinyUtils.toString(is, null);\n        assertEquals(\"testToString\", str);\n\n        is = new ByteArrayInputStream(b);\n        str = IOTinyUtils.toString(is, StandardCharsets.UTF_8.name());\n        assertEquals(\"testToString\", str);\n\n        is = new ByteArrayInputStream(b);\n        Reader isr = new InputStreamReader(is, StandardCharsets.UTF_8);\n        str = IOTinyUtils.toString(isr);\n        assertEquals(\"testToString\", str);\n    }\n\n\n    @Test\n    public void testCopy() throws Exception {\n        char[] arr = \"testToString\".toCharArray();\n        Reader reader = new CharArrayReader(arr);\n        Writer writer = new CharArrayWriter();\n\n        long count = IOTinyUtils.copy(reader, writer);\n        assertEquals(arr.length, count);\n    }\n\n    @Test\n    public void testReadLines() throws Exception {\n        StringBuilder sb = new StringBuilder();\n        for (int i = 0; i < 10; i++) {\n            sb.append(\"testReadLines\").append(\"\\n\");\n        }\n\n        StringReader reader = new StringReader(sb.toString());\n        List<String> lines = IOTinyUtils.readLines(reader);\n\n        assertEquals(10, lines.size());\n    }\n\n    @Test\n    public void testToBufferedReader() throws Exception {\n        StringBuilder sb = new StringBuilder();\n        for (int i = 0; i < 10; i++) {\n            sb.append(\"testToBufferedReader\").append(\"\\n\");\n        }\n\n        StringReader reader = new StringReader(sb.toString());\n        Method method = IOTinyUtils.class.getDeclaredMethod(\"toBufferedReader\", new Class[]{Reader.class});\n        method.setAccessible(true);\n        Object bReader = method.invoke(IOTinyUtils.class, reader);\n\n        assertTrue(bReader instanceof BufferedReader);\n    }\n\n    @Test\n    public void testWriteStringToFile() throws Exception {\n        File file = new File(testRootDir, \"testWriteStringToFile\");\n        assertTrue(!file.exists());\n\n        IOTinyUtils.writeStringToFile(file, \"testWriteStringToFile\", StandardCharsets.UTF_8.name());\n\n        assertTrue(file.exists());\n    }\n\n    @Test\n    public void testCleanDirectory() throws Exception {\n        for (int i = 0; i < 10; i++) {\n            IOTinyUtils.writeStringToFile(new File(testRootDir, \"testCleanDirectory\" + i), \"testCleanDirectory\", StandardCharsets.UTF_8.name());\n        }\n\n        File dir = new File(testRootDir);\n        assertTrue(dir.exists() && dir.isDirectory());\n        assertTrue(dir.listFiles().length > 0);\n\n        IOTinyUtils.cleanDirectory(new File(testRootDir));\n\n        assertTrue(dir.listFiles().length == 0);\n    }\n\n    @Test\n    public void testDelete() throws Exception {\n        for (int i = 0; i < 10; i++) {\n            IOTinyUtils.writeStringToFile(new File(testRootDir, \"testDelete\" + i), \"testCleanDirectory\", StandardCharsets.UTF_8.name());\n        }\n\n        File dir = new File(testRootDir);\n        assertTrue(dir.exists() && dir.isDirectory());\n        assertTrue(dir.listFiles().length > 0);\n\n        IOTinyUtils.delete(new File(testRootDir));\n\n        assertTrue(!dir.exists());\n    }\n\n    @Test\n    public void testCopyFile() throws Exception {\n        File source = new File(testRootDir, \"source\");\n        String target = testRootDir + File.separator + \"dest\";\n\n        IOTinyUtils.writeStringToFile(source, \"testCopyFile\", StandardCharsets.UTF_8.name());\n\n        IOTinyUtils.copyFile(source.getCanonicalPath(), target);\n\n        File dest = new File(target);\n        assertTrue(dest.exists());\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/utils/IPAddressUtilsTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.common.utils;\n\nimport org.junit.Test;\n\nimport static org.apache.rocketmq.common.utils.NetworkUtil.validCommonInet6Address;\n\npublic class IPAddressUtilsTest {\n\n    @Test\n    public void isIPInRange() {\n\n        // IPv4 test\n        String ipv4Address = \"192.168.1.10\";\n        String ipv4Cidr = \"192.168.1.0/24\";\n        assert IPAddressUtils.isIPInRange(ipv4Address, ipv4Cidr);\n\n        ipv4Address = \"192.168.2.10\";\n        assert !IPAddressUtils.isIPInRange(ipv4Address, ipv4Cidr);\n\n        // IPv6 test\n        String ipv6Address = \"2001:0db8:85a3:0000:0000:8a2e:0370:7334\";\n        String ipv6Cidr = \"2001:0db8:85a3::/48\";\n        assert IPAddressUtils.isIPInRange(ipv6Address, ipv6Cidr);\n    }\n\n    @Test\n    public void isValidCidr() {\n        String ipv4Cidr = \"192.168.1.0/24\";\n        String ipv6Cidr = \"2001:0db8:1234:5678::/64\";\n        String invalidCidr = \"192.168.1.0\";\n\n        assert IPAddressUtils.isValidCidr(ipv4Cidr);\n        assert IPAddressUtils.isValidCidr(ipv6Cidr);\n        assert !IPAddressUtils.isValidCidr(invalidCidr);\n    }\n\n    @Test\n    public void isValidIp() {\n        String ipv4 = \"192.168.1.0\";\n        String ipv6 = \"2001:0db8:85a3:0000:0000:8a2e:0370:7334\";\n        String invalidIp = \"192.168.1.256\";\n        String ipv4Cidr = \"192.168.1.0/24\";\n\n        assert IPAddressUtils.isValidIp(ipv4);\n        assert IPAddressUtils.isValidIp(ipv6);\n        assert !IPAddressUtils.isValidIp(invalidIp);\n        assert !IPAddressUtils.isValidIp(ipv4Cidr);\n    }\n\n    @Test\n    public void isValidIPOrCidr() {\n        String ipv4 = \"192.168.1.0\";\n        String ipv6 = \"2001:0db8:85a3:0000:0000:8a2e:0370:7334\";\n        String ipv4Cidr = \"192.168.1.0/24\";\n        String ipv6Cidr = \"2001:0db8:1234:5678::/64\";\n        assert IPAddressUtils.isValidIPOrCidr(ipv4);\n        assert IPAddressUtils.isValidIPOrCidr(ipv6);\n        assert IPAddressUtils.isValidIPOrCidr(ipv4Cidr);\n        assert IPAddressUtils.isValidIPOrCidr(ipv6Cidr);\n    }\n\n    @Test\n    public void isValidIPv6Common() {\n        String ipv6WithoutScope = \"2001:0db8:85a3:0000:0000:8a2e:0370:7334\";\n        assert validCommonInet6Address(ipv6WithoutScope);\n        String ipv6WithScope = \"2001:0db8:85a3:0000:0000:8a2e:0370:7334%eth0\";\n        assert validCommonInet6Address(ipv6WithScope);\n        String ipv6WithBracketedAndScope = \"[2001:0db8:85a3:0000:0000:8a2e:0370:7334%eth0]\";\n        assert validCommonInet6Address(ipv6WithBracketedAndScope);\n        String ipv4 = \"192.168.1.0\";\n        assert !validCommonInet6Address(ipv4);\n        String ipv4Cidr = \"192.168.1.0/24\";\n        assert !validCommonInet6Address(ipv4Cidr);\n    }\n\n}"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/utils/LiteUtilTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.utils;\n\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.lite.LiteUtil;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertTrue;\n\npublic class LiteUtilTest {\n\n    @Test\n    public void testToLmqName() {\n        String result = LiteUtil.toLmqName(\"parentTopic\", \"liteTopic\");\n        String expected = LiteUtil.LITE_TOPIC_PREFIX + \"parentTopic\" + LiteUtil.SEPARATOR + \"liteTopic\";\n        assertEquals(expected, result);\n\n        assertNull(LiteUtil.toLmqName(null, \"liteTopic\"));\n        assertNull(LiteUtil.toLmqName(\"parentTopic\", null));\n        assertNull(LiteUtil.toLmqName(\"\", \"liteTopic\"));\n        assertNull(LiteUtil.toLmqName(\"parentTopic\", \"\"));\n    }\n\n    @Test\n    public void testIsLiteTopicQueue() {\n        assertTrue(LiteUtil.isLiteTopicQueue(\"%LMQ%$parentTopic$liteTopic\"));\n\n        assertFalse(LiteUtil.isLiteTopicQueue(\"%LMQ%parentTopic\"));\n        assertFalse(LiteUtil.isLiteTopicQueue(\"parentTopic\"));\n        assertFalse(LiteUtil.isLiteTopicQueue(null));\n        assertFalse(LiteUtil.isLiteTopicQueue(\"%LMQ$\"));\n    }\n\n    @Test\n    public void testGetParentTopic() {\n        assertEquals(\"parentTopic\", LiteUtil.getParentTopic(\"%LMQ%$parentTopic$liteTopic\"));\n\n        assertNull(LiteUtil.getParentTopic(null));\n        assertNull(LiteUtil.getParentTopic(\"parentTopic\"));\n        assertNull(LiteUtil.getParentTopic(\"%LMQ%parentTopic$liteTopic\"));\n        assertNull(LiteUtil.getParentTopic(\"%LMQ%$$\"));\n        assertNull(LiteUtil.getParentTopic(\"%LMQ%$parentTopic\"));\n        assertNull(LiteUtil.getParentTopic(\"%LMQ%$parentTopic$\"));\n        assertNull(LiteUtil.getParentTopic(\"%LMQ%$$liteTopic\"));\n        assertNull(LiteUtil.getParentTopic(\"%LMQ%$parent$lite$extra\"));\n    }\n\n    @Test\n    public void testGetLiteTopic() {\n        assertEquals(\"liteTopic\", LiteUtil.getLiteTopic(\"%LMQ%$parentTopic$liteTopic\"));\n\n        assertNull(LiteUtil.getLiteTopic(null));\n        assertNull(LiteUtil.getLiteTopic(\"parentTopic\"));\n        assertNull(LiteUtil.getParentTopic(\"%LMQ%parentTopic$liteTopic\"));\n        assertNull(LiteUtil.getParentTopic(\"%LMQ%$$\"));\n        assertNull(LiteUtil.getLiteTopic(\"%LMQ%$parentTopic\"));\n        assertNull(LiteUtil.getLiteTopic(\"%LMQ%$parentTopic$\"));\n        assertNull(LiteUtil.getLiteTopic(\"%LMQ%$$liteTopic\"));\n        assertNull(LiteUtil.getLiteTopic(\"%LMQ%$parent$lite$extra\"));\n    }\n\n    @Test\n    public void testGetParentAndLiteTopic() {\n        Pair<String, String> result = LiteUtil.getParentAndLiteTopic(\"%LMQ%$parentTopic$liteTopic\");\n        assertNotNull(result);\n        assertEquals(\"parentTopic\", result.getObject1());\n        assertEquals(\"liteTopic\", result.getObject2());\n\n        assertNull(LiteUtil.getParentTopic(null));\n        assertNull(LiteUtil.getParentTopic(\"parentTopic\"));\n        assertNull(LiteUtil.getParentTopic(\"%LMQ%parentTopic$liteTopic\"));\n        assertNull(LiteUtil.getParentTopic(\"%LMQ%$$\"));\n        assertNull(LiteUtil.getParentTopic(\"%LMQ%$parentTopic\"));\n        assertNull(LiteUtil.getParentTopic(\"%LMQ%$parentTopic$\"));\n        assertNull(LiteUtil.getParentTopic(\"%LMQ%$$liteTopic\"));\n        assertNull(LiteUtil.getParentTopic(\"%LMQ%$parent$lite$extra\"));\n    }\n\n    @Test\n    public void testBelongsTo() {\n        assertTrue(LiteUtil.belongsTo(\"%LMQ%$parentTopic$liteTopic\", \"parentTopic\"));\n        assertTrue(LiteUtil.belongsTo(\"%LMQ%$parentTopic$\", \"parentTopic\")); // only check prefix\n        assertTrue(LiteUtil.belongsTo(\"%LMQ%$parentTopic$liteTopic$xxx\", \"parentTopic\")); // only check prefix\n\n        assertFalse(LiteUtil.belongsTo(\"%LMQ%$parentTopic$liteTopic\", \"otherParent\"));\n        assertFalse(LiteUtil.belongsTo(\"parentTopic\", \"parentTopic\"));\n        assertFalse(LiteUtil.belongsTo(null, \"parentTopic\"));\n        assertFalse(LiteUtil.belongsTo(\"%LMQ%$parentTopic$liteTopic\", null));\n    }\n}\n"
  },
  {
    "path": "common/src/test/java/org/apache/rocketmq/common/utils/NameServerAddressUtilsTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.common.utils;\n\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class NameServerAddressUtilsTest {\n\n    private static String endpoint1 = \"http://127.0.0.1:9876\";\n    private static String endpoint2 = \"127.0.0.1:9876\";\n    private static String endpoint3\n        = \"http://MQ_INST_123456789_BXXUzaee.xxx:80\";\n    private static String endpoint4 = \"MQ_INST_123456789_BXXUzaee.xxx:80\";\n\n    @Test\n    public void testValidateInstanceEndpoint() {\n        assertThat(NameServerAddressUtils.validateInstanceEndpoint(endpoint1)).isEqualTo(false);\n        assertThat(NameServerAddressUtils.validateInstanceEndpoint(endpoint2)).isEqualTo(false);\n        assertThat(NameServerAddressUtils.validateInstanceEndpoint(endpoint3)).isEqualTo(true);\n        assertThat(NameServerAddressUtils.validateInstanceEndpoint(endpoint4)).isEqualTo(true);\n    }\n\n    @Test\n    public void testParseInstanceIdFromEndpoint() {\n        assertThat(NameServerAddressUtils.parseInstanceIdFromEndpoint(endpoint3)).isEqualTo(\n            \"MQ_INST_123456789_BXXUzaee\");\n        assertThat(NameServerAddressUtils.parseInstanceIdFromEndpoint(endpoint4)).isEqualTo(\n            \"MQ_INST_123456789_BXXUzaee\");\n    }\n\n    @Test\n    public void testGetNameSrvAddrFromNamesrvEndpoint() {\n        assertThat(NameServerAddressUtils.getNameSrvAddrFromNamesrvEndpoint(endpoint1))\n            .isEqualTo(\"127.0.0.1:9876\");\n        assertThat(NameServerAddressUtils.getNameSrvAddrFromNamesrvEndpoint(endpoint2))\n            .isEqualTo(\"127.0.0.1:9876\");\n        assertThat(NameServerAddressUtils.getNameSrvAddrFromNamesrvEndpoint(endpoint3))\n            .isEqualTo(\"MQ_INST_123456789_BXXUzaee.xxx:80\");\n        assertThat(NameServerAddressUtils.getNameSrvAddrFromNamesrvEndpoint(endpoint4))\n            .isEqualTo(\"MQ_INST_123456789_BXXUzaee.xxx:80\");\n    }\n}\n"
  },
  {
    "path": "common/src/test/resources/rmq.logback-test.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n<configuration>\n\n    <appender name=\"CONSOLE\" class=\"ch.qos.logback.core.ConsoleAppender\">\n        <layout class=\"ch.qos.logback.classic.PatternLayout\">\n            <Pattern>\n                %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n\n            </Pattern>\n        </layout>\n    </appender>\n\n    <logger name=\"org.apache.rocketmq\" level=\"error\" additivity=\"false\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </logger>\n\n    <root level=\"error\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </root>\n\n</configuration>"
  },
  {
    "path": "container/BUILD.bazel",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\nload(\"//bazel:GenTestRules.bzl\", \"GenTestRules\")\n\njava_library(\n    name = \"container\",\n    srcs = glob([\"src/main/java/**/*.java\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \"//auth\",\n        \"//broker\",\n        \"//common\",\n        \"//remoting\",\n        \"//client\",\n        \"//srvutil\", \n        \"//store\",\n        \"@maven//:io_openmessaging_storage_dledger\",\n        \"@maven//:org_apache_commons_commons_lang3\",\n        \"@maven//:commons_validator_commons_validator\",\n        \"@maven//:commons_collections_commons_collections\",\n        \"@maven//:commons_codec_commons_codec\",\n        \"@maven//:com_github_luben_zstd_jni\",\n        \"@maven//:org_lz4_lz4_java\",\n        \"@maven//:io_netty_netty_all\",\n        \"@maven//:com_google_guava_guava\",\n        \"@maven//:org_slf4j_slf4j_api\",\n        \"@maven//:ch_qos_logback_logback_core\",\n        \"@maven//:ch_qos_logback_logback_classic\",\n        \"@maven//:commons_cli_commons_cli\",\n        \"@maven//:io_github_aliyunmq_rocketmq_slf4j_api\",\n        \"@maven//:io_github_aliyunmq_rocketmq_logback_classic\",\n    ],\n)\n\njava_library(\n    name = \"tests\",\n    srcs = glob([\"src/test/java/**/*.java\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \":container\",\n        \"//auth\",\n        \"//broker\",\n        \"//common\",\n        \"//remoting\",\n        \"//client\",\n        \"//srvutil\", \n        \"//store\",\n        \"@maven//:io_openmessaging_storage_dledger\",\n        \"//:test_deps\",\n        \"@maven//:org_apache_commons_commons_lang3\", \n        \"@maven//:io_netty_netty_all\",    \n        \"@maven//:com_google_guava_guava\",  \n    ],\n    resources = glob([\"src/test/resources/certs/*.pem\"]) + glob([\"src/test/resources/certs/*.key\"])\n)\n\nGenTestRules(\n    name = \"GeneratedTestRules\",\n    test_files = glob([\"src/test/java/**/*Test.java\"]),\n    deps = [\n        \":tests\",\n    ],\n# The following tests are flaky, fix them later.\n    exclude_tests = [\n        \"src/test/java/org/apache/rocketmq/container/BrokerContainerStartupTest\",\n        \"src/test/java/org/apache/rocketmq/container/BrokerContainerTest\",\n    ],\n)\n"
  },
  {
    "path": "container/pom.xml",
    "content": "<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n    <parent>\n        <groupId>org.apache.rocketmq</groupId>\n        <artifactId>rocketmq-all</artifactId>\n        <version>${revision}</version>\n    </parent>\n\n    <modelVersion>4.0.0</modelVersion>\n    <packaging>jar</packaging>\n    <artifactId>rocketmq-container</artifactId>\n    <name>rocketmq-container ${project.version}</name>\n\n    <properties>\n        <project.root>${basedir}/..</project.root>\n    </properties>\n\n    <dependencies>\n        <dependency>\n            <groupId>org.apache.rocketmq</groupId>\n            <artifactId>rocketmq-broker</artifactId>\n        </dependency>\n    </dependencies>\n</project>\n"
  },
  {
    "path": "container/src/main/java/org/apache/rocketmq/container/BrokerBootHook.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.container;\n\nimport java.util.Properties;\n\npublic interface BrokerBootHook {\n    /**\n     * Name of the hook.\n     *\n     * @return name of the hook\n     */\n    String hookName();\n\n    /**\n     * Code to execute before broker start.\n     *\n     * @param innerBrokerController inner broker to start\n     * @param properties broker properties\n     * @throws Exception when execute hook\n     */\n    void executeBeforeStart(InnerBrokerController innerBrokerController, Properties properties) throws Exception;\n\n    /**\n     * Code to execute after broker start.\n     *\n     * @param innerBrokerController inner broker to start\n     * @param properties broker properties\n     * @throws Exception when execute hook\n     */\n    void executeAfterStart(InnerBrokerController innerBrokerController, Properties properties) throws Exception;\n}\n\n"
  },
  {
    "path": "container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.container;\n\nimport org.apache.commons.lang3.concurrent.BasicThreadFactory;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.BrokerPathConfigHelper;\nimport org.apache.rocketmq.broker.ConfigContext;\nimport org.apache.rocketmq.broker.out.BrokerOuterAPI;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.BrokerIdentity;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.Configuration;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.RemotingServer;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingServer;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\n\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.List;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\n\npublic class BrokerContainer implements IBrokerContainer {\n    private static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n\n    private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1,\n        new BasicThreadFactory.Builder()\n            .namingPattern(\"BrokerContainerScheduledThread\")\n            .daemon(true)\n            .build());\n    protected final NettyServerConfig nettyServerConfig;\n    protected final NettyClientConfig nettyClientConfig;\n    protected final BrokerOuterAPI brokerOuterAPI;\n    protected final ContainerClientHouseKeepingService containerClientHouseKeepingService;\n\n    protected final ConcurrentMap<BrokerIdentity, InnerSalveBrokerController> slaveBrokerControllers = new ConcurrentHashMap<>();\n    protected final ConcurrentMap<BrokerIdentity, InnerBrokerController> masterBrokerControllers = new ConcurrentHashMap<>();\n    protected final ConcurrentMap<BrokerIdentity, InnerBrokerController> dLedgerBrokerControllers = new ConcurrentHashMap<>();\n    protected final List<BrokerBootHook> brokerBootHookList = new ArrayList<>();\n    protected final BrokerContainerProcessor brokerContainerProcessor;\n    protected final Configuration configuration;\n    protected final BrokerContainerConfig brokerContainerConfig;\n\n    protected RemotingServer remotingServer;\n    protected RemotingServer fastRemotingServer;\n    protected ExecutorService brokerContainerExecutor;\n\n    public BrokerContainer(\n        final BrokerContainerConfig brokerContainerConfig,\n        final NettyServerConfig nettyServerConfig,\n        final NettyClientConfig nettyClientConfig\n    ) {\n        this.brokerContainerConfig = brokerContainerConfig;\n        this.nettyServerConfig = nettyServerConfig;\n        this.nettyClientConfig = nettyClientConfig;\n\n        this.brokerOuterAPI = new BrokerOuterAPI(nettyClientConfig, null);\n\n        this.brokerContainerProcessor = new BrokerContainerProcessor(this);\n        this.brokerContainerProcessor.registerBrokerBootHook(this.brokerBootHookList);\n        this.containerClientHouseKeepingService = new ContainerClientHouseKeepingService(this);\n\n        this.configuration = new Configuration(\n            LOG,\n            BrokerPathConfigHelper.getBrokerConfigPath(),\n            this.brokerContainerConfig, this.nettyServerConfig, this.nettyClientConfig);\n    }\n\n    @Override\n    public String getBrokerContainerAddr() {\n        return this.brokerContainerConfig.getBrokerContainerIP() + \":\" + this.nettyServerConfig.getListenPort();\n    }\n\n    @Override\n    public BrokerContainerConfig getBrokerContainerConfig() {\n        return brokerContainerConfig;\n    }\n\n    @Override\n    public NettyServerConfig getNettyServerConfig() {\n        return nettyServerConfig;\n    }\n\n    @Override\n    public NettyClientConfig getNettyClientConfig() {\n        return nettyClientConfig;\n    }\n\n    @Override\n    public BrokerOuterAPI getBrokerOuterAPI() {\n        return brokerOuterAPI;\n    }\n\n    @Override\n    public RemotingServer getRemotingServer() {\n        return remotingServer;\n    }\n\n    public Configuration getConfiguration() {\n        return this.configuration;\n    }\n\n    private void updateNamesrvAddr() {\n        if (this.brokerContainerConfig.isFetchNameSrvAddrByDnsLookup()) {\n            this.brokerOuterAPI.updateNameServerAddressListByDnsLookup(this.brokerContainerConfig.getNamesrvAddr());\n        } else {\n            this.brokerOuterAPI.updateNameServerAddressList(this.brokerContainerConfig.getNamesrvAddr());\n        }\n    }\n\n    public boolean initialize() {\n        this.remotingServer = new NettyRemotingServer(this.nettyServerConfig, this.containerClientHouseKeepingService);\n        this.fastRemotingServer = this.remotingServer.newRemotingServer(this.nettyServerConfig.getListenPort() - 2);\n\n        this.brokerContainerExecutor = ThreadUtils.newThreadPoolExecutor(\n            1,\n            1,\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            new LinkedBlockingQueue<>(10000),\n            new ThreadFactoryImpl(\"SharedBrokerThread_\"));\n\n        this.registerProcessor();\n\n        if (this.brokerContainerConfig.getNamesrvAddr() != null) {\n            this.updateNamesrvAddr();\n            LOG.info(\"Set user specified name server address: {}\", this.brokerContainerConfig.getNamesrvAddr());\n            // also auto update namesrv if specify\n            this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n                @Override\n                public void run() {\n                    try {\n                        BrokerContainer.this.updateNamesrvAddr();\n                    } catch (Throwable e) {\n                        LOG.error(\"ScheduledTask fetchNameServerAddr exception\", e);\n                    }\n                }\n            }, 1000 * 10, this.brokerContainerConfig.getUpdateNamesrvAddrInterval(), TimeUnit.MILLISECONDS);\n        } else if (this.brokerContainerConfig.isFetchNamesrvAddrByAddressServer()) {\n            this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n\n                @Override\n                public void run() {\n                    try {\n                        BrokerContainer.this.brokerOuterAPI.fetchNameServerAddr();\n                    } catch (Throwable e) {\n                        LOG.error(\"ScheduledTask fetchNameServerAddr exception\", e);\n                    }\n                }\n            }, 1000 * 10, this.brokerContainerConfig.getFetchNamesrvAddrInterval(), TimeUnit.MILLISECONDS);\n        }\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    BrokerContainer.this.brokerOuterAPI.refreshMetadata();\n                } catch (Exception e) {\n                    LOG.error(\"ScheduledTask refresh metadata exception\", e);\n                }\n            }\n        }, 10, 5, TimeUnit.SECONDS);\n\n        return true;\n    }\n\n    public void registerProcessor() {\n        remotingServer.registerDefaultProcessor(brokerContainerProcessor, this.brokerContainerExecutor);\n        fastRemotingServer.registerDefaultProcessor(brokerContainerProcessor, this.brokerContainerExecutor);\n    }\n\n    @Override\n    public void start() throws Exception {\n        if (this.remotingServer != null) {\n            this.remotingServer.start();\n        }\n\n        if (this.fastRemotingServer != null) {\n            this.fastRemotingServer.start();\n        }\n\n        if (this.brokerOuterAPI != null) {\n            this.brokerOuterAPI.start();\n        }\n    }\n\n    @Override\n    public void shutdown() {\n        // Shutdown slave brokers\n        for (InnerSalveBrokerController slaveBrokerController : slaveBrokerControllers.values()) {\n            slaveBrokerController.shutdown();\n        }\n\n        slaveBrokerControllers.clear();\n\n        // Shutdown master brokers\n        for (BrokerController masterBrokerController : masterBrokerControllers.values()) {\n            masterBrokerController.shutdown();\n        }\n\n        masterBrokerControllers.clear();\n\n        // Shutdown dLedger brokers\n        dLedgerBrokerControllers.values().forEach(InnerBrokerController::shutdown);\n        dLedgerBrokerControllers.clear();\n\n        // Shutdown the remoting server with a high priority to avoid further traffic\n        if (this.remotingServer != null) {\n            this.remotingServer.shutdown();\n        }\n\n        if (this.fastRemotingServer != null) {\n            this.fastRemotingServer.shutdown();\n        }\n\n        // Shutdown the request executors\n        ThreadUtils.shutdown(this.brokerContainerExecutor);\n\n        if (this.brokerOuterAPI != null) {\n            this.brokerOuterAPI.shutdown();\n        }\n    }\n\n    public void registerClientRPCHook(RPCHook rpcHook) {\n        this.getBrokerOuterAPI().registerRPCHook(rpcHook);\n    }\n\n    public void clearClientRPCHook() {\n        this.getBrokerOuterAPI().clearRPCHook();\n    }\n\n    public List<BrokerBootHook> getBrokerBootHookList() {\n        return brokerBootHookList;\n    }\n\n    public void registerBrokerBootHook(BrokerBootHook brokerBootHook) {\n        this.brokerBootHookList.add(brokerBootHook);\n        LOG.info(\"register BrokerBootHook, {}\", brokerBootHook.hookName());\n    }\n\n    @Override\n    public InnerBrokerController addBroker(ConfigContext configContext) throws Exception {\n\n        BrokerConfig brokerConfig = configContext.getBrokerConfig();\n        MessageStoreConfig storeConfig = configContext.getMessageStoreConfig();\n        AuthConfig authConfig = configContext.getAuthConfig();\n\n        if (storeConfig.isEnableDLegerCommitLog()) {\n            return this.addDLedgerBroker(brokerConfig, storeConfig, authConfig);\n        } else {\n            if (brokerConfig.getBrokerId() == MixAll.MASTER_ID && storeConfig.getBrokerRole() != BrokerRole.SLAVE) {\n                return this.addMasterBroker(brokerConfig, storeConfig, authConfig);\n            }\n            if (brokerConfig.getBrokerId() != MixAll.MASTER_ID && storeConfig.getBrokerRole() == BrokerRole.SLAVE) {\n                return this.addSlaveBroker(brokerConfig, storeConfig, authConfig);\n            }\n        }\n\n        return null;\n    }\n\n    public InnerBrokerController addDLedgerBroker(final BrokerConfig brokerConfig, final MessageStoreConfig storeConfig,\n        final AuthConfig authConfig) throws Exception {\n        brokerConfig.setInBrokerContainer(true);\n        if (storeConfig.isDuplicationEnable()) {\n            LOG.error(\"Can not add broker to container when duplicationEnable is true currently\");\n            throw new Exception(\"Can not add broker to container when duplicationEnable is true currently\");\n        }\n        InnerBrokerController brokerController = new InnerBrokerController(this, brokerConfig, storeConfig, authConfig);\n        BrokerIdentity brokerIdentity = brokerController.getBrokerIdentity();\n        final BrokerController previousBroker = dLedgerBrokerControllers.putIfAbsent(brokerIdentity, brokerController);\n        if (previousBroker == null) {\n            // New dLedger broker added, start it\n            try {\n                final boolean initResult = brokerController.initialize();\n                if (!initResult) {\n                    dLedgerBrokerControllers.remove(brokerIdentity);\n                    brokerController.shutdown();\n                    throw new Exception(\"Failed to init dLedger broker \" + brokerIdentity.getCanonicalName());\n                }\n            } catch (Exception e) {\n                // Remove the failed dLedger broker and throw the exception\n                dLedgerBrokerControllers.remove(brokerIdentity);\n                brokerController.shutdown();\n                throw new Exception(\"Failed to initialize dLedger broker \" + brokerIdentity.getCanonicalName(), e);\n            }\n            return brokerController;\n        }\n        throw new Exception(brokerIdentity.getCanonicalName() + \" has already been added to current broker container\");\n    }\n\n    public InnerBrokerController addMasterBroker(final BrokerConfig masterBrokerConfig,\n        final MessageStoreConfig storeConfig, final AuthConfig authConfig) throws Exception {\n\n        masterBrokerConfig.setInBrokerContainer(true);\n        if (storeConfig.isDuplicationEnable()) {\n            LOG.error(\"Can not add broker to container when duplicationEnable is true currently\");\n            throw new Exception(\"Can not add broker to container when duplicationEnable is true currently\");\n        }\n        InnerBrokerController masterBroker = new InnerBrokerController(this, masterBrokerConfig, storeConfig, authConfig);\n        BrokerIdentity brokerIdentity = masterBroker.getBrokerIdentity();\n        final BrokerController previousBroker = masterBrokerControllers.putIfAbsent(brokerIdentity, masterBroker);\n        if (previousBroker == null) {\n            // New master broker added, start it\n            try {\n                final boolean initResult = masterBroker.initialize();\n                if (!initResult) {\n                    masterBrokerControllers.remove(brokerIdentity);\n                    masterBroker.shutdown();\n                    throw new Exception(\"Failed to init master broker \" + masterBrokerConfig.getCanonicalName());\n                }\n\n                for (InnerSalveBrokerController slaveBroker : this.getSlaveBrokers()) {\n                    if (slaveBroker.getMessageStore().getMasterStoreInProcess() == null) {\n                        slaveBroker.getMessageStore().setMasterStoreInProcess(masterBroker.getMessageStore());\n                    }\n                }\n            } catch (Exception e) {\n                // Remove the failed master broker and throw the exception\n                masterBrokerControllers.remove(brokerIdentity);\n                masterBroker.shutdown();\n                throw new Exception(\"Failed to initialize master broker \" + masterBrokerConfig.getCanonicalName(), e);\n            }\n            return masterBroker;\n        }\n        throw new Exception(masterBrokerConfig.getCanonicalName() + \" has already been added to current broker container\");\n    }\n\n    /**\n     * This function will create a slave broker along with the main broker, and start it with a different port.\n     *\n     * @param slaveBrokerConfig the specific slave broker config\n     * @throws Exception is thrown if an error occurs\n     */\n    public InnerSalveBrokerController addSlaveBroker(final BrokerConfig slaveBrokerConfig,\n        final MessageStoreConfig storeConfig, final AuthConfig authConfig) throws Exception {\n\n        slaveBrokerConfig.setInBrokerContainer(true);\n        if (storeConfig.isDuplicationEnable()) {\n            LOG.error(\"Can not add broker to container when duplicationEnable is true currently\");\n            throw new Exception(\"Can not add broker to container when duplicationEnable is true currently\");\n        }\n\n        int ratio = storeConfig.getAccessMessageInMemoryMaxRatio() - 10;\n        storeConfig.setAccessMessageInMemoryMaxRatio(Math.max(ratio, 0));\n        InnerSalveBrokerController slaveBroker = new InnerSalveBrokerController(this, slaveBrokerConfig, storeConfig, authConfig);\n        BrokerIdentity brokerIdentity = slaveBroker.getBrokerIdentity();\n        final InnerSalveBrokerController previousBroker = slaveBrokerControllers.putIfAbsent(brokerIdentity, slaveBroker);\n        if (previousBroker == null) {\n            // New slave broker added, start it\n            try {\n                final boolean initResult = slaveBroker.initialize();\n                if (!initResult) {\n                    slaveBrokerControllers.remove(brokerIdentity);\n                    slaveBroker.shutdown();\n                    throw new Exception(\"Failed to init slave broker \" + slaveBrokerConfig.getCanonicalName());\n                }\n                BrokerController masterBroker = this.peekMasterBroker();\n                if (slaveBroker.getMessageStore().getMasterStoreInProcess() == null && masterBroker != null) {\n                    slaveBroker.getMessageStore().setMasterStoreInProcess(masterBroker.getMessageStore());\n                }\n            } catch (Exception e) {\n                // Remove the failed slave broker and throw the exception\n                slaveBrokerControllers.remove(brokerIdentity);\n                slaveBroker.shutdown();\n                throw new Exception(\"Failed to initialize slave broker \" + slaveBrokerConfig.getCanonicalName(), e);\n            }\n            return slaveBroker;\n        }\n        throw new Exception(slaveBrokerConfig.getCanonicalName() + \" has already been added to current broker container\");\n    }\n\n    @Override\n    public BrokerController removeBroker(final BrokerIdentity brokerIdentity) throws Exception {\n\n        InnerBrokerController dLedgerController = dLedgerBrokerControllers.remove(brokerIdentity);\n        if (dLedgerController != null) {\n            dLedgerController.shutdown();\n            return dLedgerController;\n        }\n\n        InnerSalveBrokerController slaveBroker = slaveBrokerControllers.remove(brokerIdentity);\n        if (slaveBroker != null) {\n            slaveBroker.shutdown();\n            return slaveBroker;\n        }\n\n        BrokerController masterBroker = masterBrokerControllers.remove(brokerIdentity);\n\n        BrokerController nextMasterBroker = this.peekMasterBroker();\n        for (InnerSalveBrokerController slave : this.getSlaveBrokers()) {\n            if (nextMasterBroker == null) {\n                slave.getMessageStore().setMasterStoreInProcess(null);\n            } else {\n                slave.getMessageStore().setMasterStoreInProcess(nextMasterBroker.getMessageStore());\n            }\n\n        }\n\n        if (masterBroker != null) {\n            masterBroker.shutdown();\n            return masterBroker;\n        }\n\n        return null;\n    }\n\n    @Override\n    public BrokerController getBroker(final BrokerIdentity brokerIdentity) {\n        InnerSalveBrokerController slaveBroker = slaveBrokerControllers.get(brokerIdentity);\n        if (slaveBroker != null) {\n            return slaveBroker;\n        }\n\n        return masterBrokerControllers.get(brokerIdentity);\n    }\n\n    @Override\n    public Collection<InnerBrokerController> getMasterBrokers() {\n        return masterBrokerControllers.values();\n    }\n\n    @Override\n    public Collection<InnerSalveBrokerController> getSlaveBrokers() {\n        return slaveBrokerControllers.values();\n    }\n\n    @Override\n    public List<BrokerController> getBrokerControllers() {\n        List<BrokerController> brokerControllers = new ArrayList<>();\n        brokerControllers.addAll(this.getMasterBrokers());\n        brokerControllers.addAll(this.getSlaveBrokers());\n        return brokerControllers;\n    }\n\n    @Override\n    public BrokerController peekMasterBroker() {\n        if (!masterBrokerControllers.isEmpty()) {\n            return masterBrokerControllers.values().iterator().next();\n        }\n        return null;\n    }\n\n    public BrokerController findBrokerControllerByBrokerName(String brokerName) {\n        for (BrokerController brokerController : masterBrokerControllers.values()) {\n            if (brokerController.getBrokerConfig().getBrokerName().equals(brokerName)) {\n                return brokerController;\n            }\n        }\n\n        for (BrokerController brokerController : slaveBrokerControllers.values()) {\n            if (brokerController.getBrokerConfig().getBrokerName().equals(brokerName)) {\n                return brokerController;\n            }\n        }\n        return null;\n    }\n\n    public ExecutorService getBrokerContainerExecutor() {\n        return brokerContainerExecutor;\n    }\n}\n"
  },
  {
    "path": "container/src/main/java/org/apache/rocketmq/container/BrokerContainerConfig.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.container;\n\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.annotation.ImportantField;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\n\npublic class BrokerContainerConfig {\n\n    private String rocketmqHome = MixAll.ROCKETMQ_HOME_DIR;\n\n    @ImportantField\n    private String namesrvAddr = System.getProperty(MixAll.NAMESRV_ADDR_PROPERTY, System.getenv(MixAll.NAMESRV_ADDR_ENV));\n\n    @ImportantField\n    private boolean fetchNameSrvAddrByDnsLookup = false;\n\n    @ImportantField\n    private boolean fetchNamesrvAddrByAddressServer = false;\n\n    @ImportantField\n    private String brokerContainerIP = NetworkUtil.getLocalAddress();\n\n    private String brokerConfigPaths = null;\n    \n    /**\n     * The interval to fetch namesrv addr, default value is 10 second\n     */\n    private long fetchNamesrvAddrInterval = 10 * 1000;\n\n    /**\n     * The interval to update namesrv addr, default value is 120 second\n     */\n    private long updateNamesrvAddrInterval = 60 * 2 * 1000;\n\n\n    /**\n     * Config in this black list will be not allowed to update by command.\n     * Try to update this config black list by restart process.\n     * Try to update configures in black list by restart process.\n     */\n    private String configBlackList = \"configBlackList;brokerConfigPaths\";\n\n    public String getRocketmqHome() {\n        return rocketmqHome;\n    }\n\n    public void setRocketmqHome(String rocketmqHome) {\n        this.rocketmqHome = rocketmqHome;\n    }\n\n    public String getNamesrvAddr() {\n        return namesrvAddr;\n    }\n\n    public void setNamesrvAddr(String namesrvAddr) {\n        this.namesrvAddr = namesrvAddr;\n    }\n\n    public boolean isFetchNameSrvAddrByDnsLookup() {\n        return fetchNameSrvAddrByDnsLookup;\n    }\n\n    public void setFetchNameSrvAddrByDnsLookup(boolean fetchNameSrvAddrByDnsLookup) {\n        this.fetchNameSrvAddrByDnsLookup = fetchNameSrvAddrByDnsLookup;\n    }\n\n    public boolean isFetchNamesrvAddrByAddressServer() {\n        return fetchNamesrvAddrByAddressServer;\n    }\n\n    public void setFetchNamesrvAddrByAddressServer(boolean fetchNamesrvAddrByAddressServer) {\n        this.fetchNamesrvAddrByAddressServer = fetchNamesrvAddrByAddressServer;\n    }\n\n    public String getBrokerContainerIP() {\n        return brokerContainerIP;\n    }\n\n    public String getBrokerConfigPaths() {\n        return brokerConfigPaths;\n    }\n\n    public void setBrokerConfigPaths(String brokerConfigPaths) {\n        this.brokerConfigPaths = brokerConfigPaths;\n    }\n    \n    public long getFetchNamesrvAddrInterval() {\n        return fetchNamesrvAddrInterval;\n    }\n    \n    public void setFetchNamesrvAddrInterval(final long fetchNamesrvAddrInterval) {\n        this.fetchNamesrvAddrInterval = fetchNamesrvAddrInterval;\n    }\n\n    public long getUpdateNamesrvAddrInterval() {\n        return updateNamesrvAddrInterval;\n    }\n\n    public void setUpdateNamesrvAddrInterval(long updateNamesrvAddrInterval) {\n        this.updateNamesrvAddrInterval = updateNamesrvAddrInterval;\n    }\n\n    public String getConfigBlackList() {\n        return configBlackList;\n    }\n\n    public void setConfigBlackList(String configBlackList) {\n        this.configBlackList = configBlackList;\n    }\n}\n"
  },
  {
    "path": "container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.container;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport java.io.UnsupportedEncodingException;\nimport java.util.Arrays;\nimport java.util.HashSet;\nimport java.util.Set;\nimport java.util.List;\nimport java.util.Properties;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.BrokerStartup;\nimport org.apache.rocketmq.broker.ConfigContext;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.BrokerIdentity;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.AddBrokerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetBrokerConfigResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.RemoveBrokerRequestHeader;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\n\npublic class BrokerContainerProcessor implements NettyRequestProcessor {\n    protected static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    protected final BrokerContainer brokerContainer;\n    protected List<BrokerBootHook> brokerBootHookList;\n\n    protected final Set<String> configBlackList = new HashSet<>();\n\n    public BrokerContainerProcessor(BrokerContainer brokerContainer) {\n        this.brokerContainer = brokerContainer;\n        initConfigBlackList();\n    }\n\n    private void initConfigBlackList() {\n        configBlackList.add(\"brokerConfigPaths\");\n        configBlackList.add(\"rocketmqHome\");\n        configBlackList.add(\"configBlackList\");\n        String[] configArray = brokerContainer.getBrokerContainerConfig().getConfigBlackList().split(\";\");\n        configBlackList.addAll(Arrays.asList(configArray));\n    }\n\n    @Override\n    public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws Exception {\n        switch (request.getCode()) {\n            case RequestCode.ADD_BROKER:\n                return this.addBroker(ctx, request);\n            case RequestCode.REMOVE_BROKER:\n                return this.removeBroker(ctx, request);\n            case RequestCode.GET_BROKER_CONFIG:\n                return this.getBrokerConfig(ctx, request);\n            case RequestCode.UPDATE_BROKER_CONFIG:\n                return this.updateBrokerConfig(ctx, request);\n            default:\n                break;\n        }\n        return null;\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n\n    protected synchronized RemotingCommand addBroker(ChannelHandlerContext ctx,\n        RemotingCommand request) throws Exception {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final AddBrokerRequestHeader requestHeader = (AddBrokerRequestHeader) request.decodeCommandCustomHeader(AddBrokerRequestHeader.class);\n\n        LOGGER.info(\"addBroker called by {}\", RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n\n        Properties brokerProperties = null;\n        String configPath = requestHeader.getConfigPath();\n\n        if (configPath != null && !configPath.isEmpty()) {\n            BrokerStartup.SystemConfigFileHelper configFileHelper = new BrokerStartup.SystemConfigFileHelper();\n            configFileHelper.setFile(configPath);\n\n            try {\n                brokerProperties = configFileHelper.loadConfig();\n            } catch (Exception e) {\n                LOGGER.error(\"addBroker load config from {} failed, {}\", configPath, e);\n            }\n        } else {\n            LOGGER.error(\"addBroker config path is empty\");\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"addBroker config path is empty\");\n            return response;\n        }\n\n        if (brokerProperties == null) {\n            LOGGER.error(\"addBroker properties empty\");\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"addBroker properties empty\");\n            return response;\n        }\n\n        BrokerConfig brokerConfig = new BrokerConfig();\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        AuthConfig authConfig = new AuthConfig();\n        MixAll.properties2Object(brokerProperties, brokerConfig);\n        MixAll.properties2Object(brokerProperties, messageStoreConfig);\n        MixAll.properties2Object(brokerProperties, authConfig);\n\n        messageStoreConfig.setHaListenPort(brokerConfig.getListenPort() + 1);\n        brokerConfig.setBrokerConfigPath(configPath);\n\n        if (!messageStoreConfig.isEnableDLegerCommitLog()) {\n            if (!brokerConfig.isEnableControllerMode()) {\n                switch (messageStoreConfig.getBrokerRole()) {\n                    case ASYNC_MASTER:\n                    case SYNC_MASTER:\n                        brokerConfig.setBrokerId(MixAll.MASTER_ID);\n                        break;\n                    case SLAVE:\n                        if (brokerConfig.getBrokerId() <= 0) {\n                            response.setCode(ResponseCode.SYSTEM_ERROR);\n                            response.setRemark(\"slave broker id must be > 0\");\n                            return response;\n                        }\n                        break;\n                    default:\n                        break;\n\n                }\n            }\n\n            if (messageStoreConfig.getTotalReplicas() < messageStoreConfig.getInSyncReplicas()\n                || messageStoreConfig.getTotalReplicas() < messageStoreConfig.getMinInSyncReplicas()\n                || messageStoreConfig.getInSyncReplicas() < messageStoreConfig.getMinInSyncReplicas()) {\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"invalid replicas number\");\n                return response;\n            }\n        }\n\n        ConfigContext configContext = new ConfigContext.Builder().\n            brokerConfig(brokerConfig).\n            messageStoreConfig(messageStoreConfig).\n            authConfig(authConfig).\n            properties(brokerProperties).\n            build();\n\n        InnerBrokerController innerBrokerController;\n        try {\n            innerBrokerController = this.brokerContainer.addBroker(configContext);\n        } catch (Exception e) {\n            LOGGER.error(\"addBroker exception\", e);\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(e.getMessage());\n            return response;\n        }\n        if (innerBrokerController != null) {\n            innerBrokerController.getConfiguration().registerConfig(brokerProperties);\n            try {\n                for (BrokerBootHook brokerBootHook : brokerBootHookList) {\n                    brokerBootHook.executeBeforeStart(innerBrokerController, brokerProperties);\n                }\n                innerBrokerController.start();\n\n                for (BrokerBootHook brokerBootHook : brokerBootHookList) {\n                    brokerBootHook.executeAfterStart(innerBrokerController, brokerProperties);\n                }\n            } catch (Exception e) {\n                LOGGER.error(\"start broker exception\", e);\n                BrokerIdentity brokerIdentity;\n                if (messageStoreConfig.isEnableDLegerCommitLog()) {\n                    brokerIdentity = new BrokerIdentity(brokerConfig.getBrokerClusterName(),\n                        brokerConfig.getBrokerName(), Integer.parseInt(messageStoreConfig.getdLegerSelfId().substring(1)));\n                } else {\n                    brokerIdentity = new BrokerIdentity(brokerConfig.getBrokerClusterName(),\n                        brokerConfig.getBrokerName(), brokerConfig.getBrokerId());\n                }\n                this.brokerContainer.removeBroker(brokerIdentity);\n                innerBrokerController.shutdown();\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"start broker failed\" + e);\n                return response;\n            }\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n        } else {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"add broker return null\");\n        }\n\n        return response;\n    }\n\n    protected synchronized RemotingCommand removeBroker(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final RemoveBrokerRequestHeader requestHeader = (RemoveBrokerRequestHeader) request.decodeCommandCustomHeader(RemoveBrokerRequestHeader.class);\n\n        LOGGER.info(\"removeBroker called by {}\", RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n\n        BrokerIdentity brokerIdentity = new BrokerIdentity(requestHeader.getBrokerClusterName(), requestHeader.getBrokerName(), requestHeader.getBrokerId());\n\n        BrokerController brokerController;\n        try {\n            brokerController = this.brokerContainer.removeBroker(brokerIdentity);\n        } catch (Exception e) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(e.getMessage());\n            return response;\n        }\n\n        if (brokerController != null) {\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n        } else {\n            response.setCode(ResponseCode.BROKER_NOT_EXIST);\n            response.setRemark(\"Broker not exist\");\n        }\n        return response;\n    }\n\n    public void registerBrokerBootHook(List<BrokerBootHook> brokerBootHookList) {\n        this.brokerBootHookList = brokerBootHookList;\n    }\n\n    private RemotingCommand updateBrokerConfig(ChannelHandlerContext ctx, RemotingCommand request) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        LOGGER.info(\"updateSharedBrokerConfig called by {}\", RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n\n        byte[] body = request.getBody();\n        if (body != null) {\n            try {\n                String bodyStr = new String(body, MixAll.DEFAULT_CHARSET);\n                Properties properties = MixAll.string2Properties(bodyStr);\n\n                if (properties == null) {\n                    LOGGER.error(\"string2Properties error\");\n                    response.setCode(ResponseCode.SYSTEM_ERROR);\n                    response.setRemark(\"string2Properties error\");\n                    return response;\n                }\n\n                if (validateBlackListConfigExist(properties)) {\n                    response.setCode(ResponseCode.NO_PERMISSION);\n                    response.setRemark(\"Can not update config in black list.\");\n                    return response;\n                }\n\n\n                LOGGER.info(\"updateBrokerContainerConfig, new config: [{}] client: {} \", properties, ctx.channel().remoteAddress());\n                this.brokerContainer.getConfiguration().update(properties);\n\n            } catch (UnsupportedEncodingException e) {\n                LOGGER.error(\"\", e);\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"UnsupportedEncodingException \" + e);\n                return response;\n            }\n        }\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private boolean validateBlackListConfigExist(Properties properties) {\n        for (String blackConfig : configBlackList) {\n            if (properties.containsKey(blackConfig)) {\n                return true;\n            }\n        }\n        return false;\n    }\n\n    private RemotingCommand getBrokerConfig(ChannelHandlerContext ctx, RemotingCommand request) {\n\n        final RemotingCommand response = RemotingCommand.createResponseCommand(GetBrokerConfigResponseHeader.class);\n        final GetBrokerConfigResponseHeader responseHeader = (GetBrokerConfigResponseHeader) response.readCustomHeader();\n\n        String content = this.brokerContainer.getConfiguration().getAllConfigsFormatString();\n        if (content != null && content.length() > 0) {\n            try {\n                content = MixAll.adjustConfigForPlatform(content);\n                response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET));\n            } catch (UnsupportedEncodingException e) {\n                LOGGER.error(\"\", e);\n\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"UnsupportedEncodingException \" + e);\n                return response;\n            }\n        }\n\n        responseHeader.setVersion(this.brokerContainer.getConfiguration().getDataVersionJson());\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n}\n"
  },
  {
    "path": "container/src/main/java/org/apache/rocketmq/container/BrokerContainerStartup.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.container;\n\nimport java.io.BufferedInputStream;\nimport java.io.FileInputStream;\nimport java.io.InputStream;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Properties;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.commons.cli.CommandLine;\nimport org.apache.commons.cli.DefaultParser;\nimport org.apache.commons.cli.Option;\nimport org.apache.commons.cli.Options;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.BrokerPathConfigHelper;\nimport org.apache.rocketmq.broker.ConfigContext;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.netty.NettySystemConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.srvutil.ServerUtil;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\n\npublic class BrokerContainerStartup {\n    private static final String BROKER_CONTAINER_CONFIG_OPTION = \"c\";\n    private static final String BROKER_CONFIG_OPTION = \"b\";\n    private static final String PRINT_PROPERTIES_OPTION = \"p\";\n    private static final String PRINT_IMPORTANT_PROPERTIES_OPTION = \"m\";\n    public static Properties properties = null;\n    public static CommandLine commandLine = null;\n    public static String configFile = null;\n    public static Logger log;\n    public static final SystemConfigFileHelper CONFIG_FILE_HELPER = new SystemConfigFileHelper();\n    public static String rocketmqHome = null;\n\n    public static void main(String[] args) {\n        final BrokerContainerConfig containerConfig = new BrokerContainerConfig();\n        final NettyServerConfig nettyServerConfig = new NettyServerConfig();\n        final NettyClientConfig nettyClientConfig = new NettyClientConfig();\n        parseCmdLineToConfig(args, containerConfig, nettyServerConfig, nettyClientConfig);\n        final BrokerContainer brokerContainer = startBrokerContainer(createBrokerContainer(containerConfig, nettyServerConfig, nettyClientConfig));\n        createAndStartBrokers(brokerContainer);\n    }\n\n    public static List<BrokerController> createAndStartBrokers(BrokerContainer brokerContainer) {\n        String[] configPaths = parseBrokerConfigPath();\n        List<BrokerController> brokerControllerList = new ArrayList<>();\n\n        if (configPaths != null && configPaths.length > 0) {\n            SystemConfigFileHelper configFileHelper = new SystemConfigFileHelper();\n            for (String configPath : configPaths) {\n                System.out.printf(\"Start broker from config file path %s%n\", configPath);\n                configFileHelper.setFile(configPath);\n\n                Properties brokerProperties = null;\n                try {\n                    brokerProperties = configFileHelper.loadConfig();\n                } catch (Exception e) {\n                    e.printStackTrace();\n                    System.exit(-1);\n                }\n\n                final InnerBrokerController innerBrokerController = createAndInitializeBroker(brokerContainer, configPath, brokerProperties);\n                if (innerBrokerController != null) {\n                    brokerControllerList.add(innerBrokerController);\n                    startBrokerController(brokerContainer, innerBrokerController, brokerProperties);\n                }\n            }\n        }\n\n        return brokerControllerList;\n    }\n\n    public static String[] parseBrokerConfigPath() {\n        String brokerConfigList = null;\n        if (commandLine.hasOption(BROKER_CONFIG_OPTION)) {\n            brokerConfigList = commandLine.getOptionValue(BROKER_CONFIG_OPTION);\n\n        } else if (commandLine.hasOption(BROKER_CONTAINER_CONFIG_OPTION)) {\n            String brokerContainerConfigPath = commandLine.getOptionValue(BROKER_CONTAINER_CONFIG_OPTION);\n            if (brokerContainerConfigPath != null) {\n                BrokerContainerConfig brokerContainerConfig = new BrokerContainerConfig();\n                SystemConfigFileHelper configFileHelper = new SystemConfigFileHelper();\n                configFileHelper.setFile(brokerContainerConfigPath);\n                Properties brokerContainerProperties = null;\n                try {\n                    brokerContainerProperties = configFileHelper.loadConfig();\n                } catch (Exception e) {\n                    e.printStackTrace();\n                    System.exit(-1);\n                }\n                if (brokerContainerProperties != null) {\n                    MixAll.properties2Object(brokerContainerProperties, brokerContainerConfig);\n                }\n                brokerConfigList = brokerContainerConfig.getBrokerConfigPaths();\n            }\n        }\n\n        if (brokerConfigList != null) {\n            return brokerConfigList.split(\":\");\n        }\n        return null;\n    }\n\n    public static InnerBrokerController createAndInitializeBroker(BrokerContainer brokerContainer,\n        String filePath, Properties brokerProperties) {\n\n        final BrokerConfig brokerConfig = new BrokerConfig();\n        final MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        final AuthConfig authConfig = new AuthConfig();\n\n        if (brokerProperties != null) {\n            properties2SystemEnv(brokerProperties);\n            MixAll.properties2Object(brokerProperties, brokerConfig);\n            MixAll.properties2Object(brokerProperties, messageStoreConfig);\n            MixAll.properties2Object(brokerProperties, authConfig);\n        }\n\n        messageStoreConfig.setHaListenPort(brokerConfig.getListenPort() + 1);\n\n        MixAll.properties2Object(ServerUtil.commandLine2Properties(commandLine), brokerConfig);\n\n        if (!brokerConfig.isEnableControllerMode()) {\n            switch (messageStoreConfig.getBrokerRole()) {\n                case ASYNC_MASTER:\n                case SYNC_MASTER:\n                    brokerConfig.setBrokerId(MixAll.MASTER_ID);\n                    break;\n                case SLAVE:\n                    if (brokerConfig.getBrokerId() <= 0) {\n                        System.out.printf(\"Slave's brokerId must be > 0%n\");\n                        System.exit(-3);\n                    }\n\n                    break;\n                default:\n                    break;\n            }\n        }\n\n        if (messageStoreConfig.getTotalReplicas() < messageStoreConfig.getInSyncReplicas()\n            || messageStoreConfig.getTotalReplicas() < messageStoreConfig.getMinInSyncReplicas()\n            || messageStoreConfig.getInSyncReplicas() < messageStoreConfig.getMinInSyncReplicas()) {\n            System.out.printf(\"invalid replicas number%n\");\n            System.exit(-3);\n        }\n\n        brokerConfig.setBrokerConfigPath(filePath);\n\n        log = LoggerFactory.getLogger(brokerConfig.getIdentifier() + LoggerName.BROKER_LOGGER_NAME);\n        MixAll.printObjectProperties(log, brokerConfig);\n        MixAll.printObjectProperties(log, messageStoreConfig);\n\n        ConfigContext configContext = new ConfigContext.Builder()\n            .brokerConfig(brokerConfig)\n            .messageStoreConfig(messageStoreConfig)\n            .authConfig(authConfig)\n            .properties(brokerProperties)\n            .build();\n\n        try {\n            InnerBrokerController innerBrokerController = brokerContainer.addBroker(configContext);\n            if (innerBrokerController != null) {\n                innerBrokerController.getConfiguration().registerConfig(brokerProperties);\n                return innerBrokerController;\n            } else {\n                System.out.printf(\"Add broker [%s-%s] failed.%n\", brokerConfig.getBrokerName(), brokerConfig.getBrokerId());\n            }\n        } catch (Exception e) {\n            e.printStackTrace();\n            System.exit(-1);\n        }\n        return null;\n    }\n\n    public static BrokerContainer startBrokerContainer(BrokerContainer brokerContainer) {\n        try {\n\n            brokerContainer.start();\n\n            String tip = \"The broker container boot success. serializeType=\" + RemotingCommand.getSerializeTypeConfigInThisServer();\n\n            if (null != brokerContainer.getBrokerContainerConfig().getNamesrvAddr()) {\n                tip += \" and name server is \" + brokerContainer.getBrokerContainerConfig().getNamesrvAddr();\n            }\n\n            log.info(tip);\n            System.out.printf(\"%s%n\", tip);\n            return brokerContainer;\n        } catch (Throwable e) {\n            e.printStackTrace();\n            System.exit(-1);\n        }\n\n        return null;\n    }\n\n    public static void startBrokerController(BrokerContainer brokerContainer,\n        InnerBrokerController innerBrokerController, Properties brokerProperties) {\n        try {\n            for (BrokerBootHook hook : brokerContainer.getBrokerBootHookList()) {\n                hook.executeBeforeStart(innerBrokerController, brokerProperties);\n            }\n\n            innerBrokerController.start();\n\n            for (BrokerBootHook hook : brokerContainer.getBrokerBootHookList()) {\n                hook.executeAfterStart(innerBrokerController, brokerProperties);\n            }\n\n            String tip = String.format(\"Broker [%s-%s] boot success. serializeType=%s\",\n                innerBrokerController.getBrokerConfig().getBrokerName(),\n                innerBrokerController.getBrokerConfig().getBrokerId(),\n                RemotingCommand.getSerializeTypeConfigInThisServer());\n\n            log.info(tip);\n            System.out.printf(\"%s%n\", tip);\n        } catch (Throwable e) {\n            e.printStackTrace();\n            System.exit(-1);\n        }\n    }\n\n    public static void shutdown(final BrokerContainer controller) {\n        if (null != controller) {\n            controller.shutdown();\n        }\n    }\n\n    public static Properties parseCmdLineToConfig(String[] args,\n        BrokerContainerConfig containerConfig,\n        NettyServerConfig nettyServerConfig,\n        NettyClientConfig nettyClientConfig) {\n        System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION));\n\n        if (null == System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_SNDBUF_SIZE)) {\n            NettySystemConfig.socketSndbufSize = 131072;\n        }\n\n        if (null == System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_RCVBUF_SIZE)) {\n            NettySystemConfig.socketRcvbufSize = 131072;\n        }\n\n        try {\n            //PackageConflictDetect.detectFastjson();\n            Options options = ServerUtil.buildCommandlineOptions(new Options());\n            commandLine = ServerUtil.parseCmdLine(\"mqbroker\", args, buildCommandlineOptions(options),\n                new DefaultParser());\n            if (null == commandLine) {\n                System.exit(-1);\n            }\n\n            nettyServerConfig.setListenPort(10811);\n\n            if (commandLine.hasOption(BROKER_CONTAINER_CONFIG_OPTION)) {\n                String file = commandLine.getOptionValue(BROKER_CONTAINER_CONFIG_OPTION);\n                if (file != null) {\n                    CONFIG_FILE_HELPER.setFile(file);\n                    configFile = file;\n                    BrokerPathConfigHelper.setBrokerConfigPath(file);\n                }\n            }\n\n            properties = CONFIG_FILE_HELPER.loadConfig();\n            if (properties != null) {\n                properties2SystemEnv(properties);\n                MixAll.properties2Object(properties, containerConfig);\n                MixAll.properties2Object(properties, nettyServerConfig);\n                MixAll.properties2Object(properties, nettyClientConfig);\n            }\n\n            MixAll.properties2Object(ServerUtil.commandLine2Properties(commandLine), containerConfig);\n\n            if (null == containerConfig.getRocketmqHome()) {\n                System.out.printf(\"Please set the %s variable in your environment to match the location of the RocketMQ installation\", MixAll.ROCKETMQ_HOME_ENV);\n                System.exit(-2);\n            }\n            rocketmqHome = containerConfig.getRocketmqHome();\n\n            String namesrvAddr = containerConfig.getNamesrvAddr();\n            if (null != namesrvAddr) {\n                try {\n                    String[] addrArray = namesrvAddr.split(\";\");\n                    for (String addr : addrArray) {\n                        NetworkUtil.string2SocketAddress(addr);\n                    }\n                } catch (Exception e) {\n                    System.out.printf(\n                        \"The Name Server Address[%s] illegal, please set it as follows, \\\"127.0.0.1:9876;192.168.0.1:9876\\\"%n\",\n                        namesrvAddr);\n                    System.exit(-3);\n                }\n            }\n\n            if (commandLine.hasOption(PRINT_PROPERTIES_OPTION)) {\n                Logger console = LoggerFactory.getLogger(LoggerName.BROKER_CONSOLE_NAME);\n                MixAll.printObjectProperties(console, containerConfig);\n                MixAll.printObjectProperties(console, nettyServerConfig);\n                MixAll.printObjectProperties(console, nettyClientConfig);\n                System.exit(0);\n            } else if (commandLine.hasOption(PRINT_IMPORTANT_PROPERTIES_OPTION)) {\n                Logger console = LoggerFactory.getLogger(LoggerName.BROKER_CONSOLE_NAME);\n                MixAll.printObjectProperties(console, containerConfig, true);\n                MixAll.printObjectProperties(console, nettyServerConfig, true);\n                MixAll.printObjectProperties(console, nettyClientConfig, true);\n                System.exit(0);\n            }\n        } catch (Throwable e) {\n            e.printStackTrace();\n            System.exit(-1);\n        }\n\n        return properties;\n    }\n\n    public static BrokerContainer createBrokerContainer(BrokerContainerConfig containerConfig,\n        NettyServerConfig nettyServerConfig,\n        NettyClientConfig nettyClientConfig) {\n\n        log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n        MixAll.printObjectProperties(log, containerConfig);\n        MixAll.printObjectProperties(log, nettyServerConfig);\n        MixAll.printObjectProperties(log, nettyClientConfig);\n\n        final BrokerContainer brokerContainer = new BrokerContainer(\n            containerConfig,\n            nettyServerConfig,\n            nettyClientConfig);\n        // remember all configs to prevent discard\n        brokerContainer.getConfiguration().registerConfig(properties);\n\n        boolean initResult = brokerContainer.initialize();\n        if (!initResult) {\n            brokerContainer.shutdown();\n            System.exit(-3);\n        }\n\n        setupShutdownHook(brokerContainer);\n\n        return brokerContainer;\n\n    }\n\n    public static void setupShutdownHook(final BrokerContainer brokerContainer) {\n        Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {\n            private volatile boolean hasShutdown = false;\n            private AtomicInteger shutdownTimes = new AtomicInteger(0);\n\n            @Override\n            public void run() {\n                synchronized (this) {\n                    log.info(\"Shutdown hook was invoked, {}\", this.shutdownTimes.incrementAndGet());\n                    if (!this.hasShutdown) {\n                        this.hasShutdown = true;\n                        long beginTime = System.currentTimeMillis();\n                        brokerContainer.shutdown();\n                        long consumingTimeTotal = System.currentTimeMillis() - beginTime;\n                        log.info(\"Shutdown hook over, consuming total time(ms): {}\", consumingTimeTotal);\n                    }\n                }\n            }\n        }, \"ShutdownHook\"));\n    }\n\n    private static void properties2SystemEnv(Properties properties) {\n        if (properties == null) {\n            return;\n        }\n        String rmqAddressServerDomain = properties.getProperty(\"rmqAddressServerDomain\", MixAll.WS_DOMAIN_NAME);\n        String rmqAddressServerSubGroup = properties.getProperty(\"rmqAddressServerSubGroup\", MixAll.WS_DOMAIN_SUBGROUP);\n        System.setProperty(\"rocketmq.namesrv.domain\", rmqAddressServerDomain);\n        System.setProperty(\"rocketmq.namesrv.domain.subgroup\", rmqAddressServerSubGroup);\n    }\n\n    private static Options buildCommandlineOptions(final Options options) {\n        Option opt = new Option(BROKER_CONTAINER_CONFIG_OPTION, \"configFile\", true, \"Config file for shared broker\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(PRINT_PROPERTIES_OPTION, \"printConfigItem\", false, \"Print all config item\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(PRINT_IMPORTANT_PROPERTIES_OPTION, \"printImportantConfig\", false, \"Print important config item\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(BROKER_CONFIG_OPTION, \"brokerConfigFiles\", true, \"The path of broker config files, split by ':'\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        return options;\n    }\n\n    public static class SystemConfigFileHelper {\n        private static final Logger LOGGER = LoggerFactory.getLogger(SystemConfigFileHelper.class);\n\n        private String file;\n\n        public SystemConfigFileHelper() {\n        }\n\n        public Properties loadConfig() throws Exception {\n            Properties properties = new Properties();\n\n            try (InputStream in = new BufferedInputStream(new FileInputStream(file))) {\n                properties.load(in);\n            }\n            return properties;\n        }\n\n        public void update(Properties properties) throws Exception {\n            LOGGER.error(\"[SystemConfigFileHelper] update no thing.\");\n        }\n\n        public void setFile(String file) {\n            this.file = file;\n        }\n\n        public String getFile() {\n            return file;\n        }\n    }\n\n}\n"
  },
  {
    "path": "container/src/main/java/org/apache/rocketmq/container/ContainerClientHouseKeepingService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.container;\n\nimport io.netty.channel.Channel;\nimport java.util.Collection;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.remoting.ChannelEventListener;\n\npublic class ContainerClientHouseKeepingService implements ChannelEventListener {\n    private final IBrokerContainer brokerContainer;\n\n    public ContainerClientHouseKeepingService(final IBrokerContainer brokerContainer) {\n        this.brokerContainer = brokerContainer;\n    }\n\n    @Override\n    public void onChannelConnect(String remoteAddr, Channel channel) {\n        onChannelOperation(CallbackCode.CONNECT, remoteAddr, channel);\n    }\n\n    @Override\n    public void onChannelClose(String remoteAddr, Channel channel) {\n        onChannelOperation(CallbackCode.CLOSE, remoteAddr, channel);\n    }\n\n    @Override\n    public void onChannelException(String remoteAddr, Channel channel) {\n        onChannelOperation(CallbackCode.EXCEPTION, remoteAddr, channel);\n    }\n\n    @Override\n    public void onChannelIdle(String remoteAddr, Channel channel) {\n        onChannelOperation(CallbackCode.IDLE, remoteAddr, channel);\n    }\n\n    @Override\n    public void onChannelActive(String remoteAddr, Channel channel) {\n        onChannelOperation(CallbackCode.ACTIVE, remoteAddr, channel);\n    }\n\n    private void onChannelOperation(CallbackCode callbackCode, String remoteAddr, Channel channel) {\n        Collection<InnerBrokerController> masterBrokers = this.brokerContainer.getMasterBrokers();\n        Collection<InnerSalveBrokerController> slaveBrokers = this.brokerContainer.getSlaveBrokers();\n\n        for (BrokerController masterBroker : masterBrokers) {\n            brokerOperation(masterBroker, callbackCode, remoteAddr, channel);\n        }\n\n        for (InnerSalveBrokerController slaveBroker : slaveBrokers) {\n            brokerOperation(slaveBroker, callbackCode, remoteAddr, channel);\n        }\n    }\n\n    private void brokerOperation(BrokerController brokerController, CallbackCode callbackCode, String remoteAddr,\n        Channel channel) {\n        if (callbackCode == CallbackCode.CONNECT) {\n            brokerController.getBrokerStatsManager().incChannelConnectNum();\n            return;\n        }\n        boolean removed = brokerController.getProducerManager().doChannelCloseEvent(remoteAddr, channel);\n        removed &= brokerController.getConsumerManager().doChannelCloseEvent(remoteAddr, channel);\n        if (removed) {\n            switch (callbackCode) {\n                case CLOSE:\n                    brokerController.getBrokerStatsManager().incChannelCloseNum();\n                    break;\n                case EXCEPTION:\n                    brokerController.getBrokerStatsManager().incChannelExceptionNum();\n                    break;\n                case IDLE:\n                    brokerController.getBrokerStatsManager().incChannelIdleNum();\n                    break;\n                default:\n                    break;\n            }\n        }\n    }\n\n    public enum CallbackCode {\n        /**\n         * onChannelConnect\n         */\n        CONNECT,\n        /**\n         * onChannelClose\n         */\n        CLOSE,\n        /**\n         * onChannelException\n         */\n        EXCEPTION,\n        /**\n         * onChannelIdle\n         */\n        IDLE,\n        /**\n         * onChannelActive\n         */\n        ACTIVE\n    }\n}\n"
  },
  {
    "path": "container/src/main/java/org/apache/rocketmq/container/IBrokerContainer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.container;\n\nimport java.util.Collection;\nimport java.util.List;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.ConfigContext;\nimport org.apache.rocketmq.broker.out.BrokerOuterAPI;\nimport org.apache.rocketmq.common.BrokerIdentity;\nimport org.apache.rocketmq.remoting.RemotingServer;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\n\n/**\n * An interface for broker container to hold multiple master and slave brokers.\n */\npublic interface IBrokerContainer {\n\n    /**\n     * Start broker container\n     */\n    void start() throws Exception;\n\n    /**\n     * Shutdown broker container and all the brokers inside.\n     */\n    void shutdown();\n\n    /**\n     * Add a broker to this container with specific broker config.\n     *\n     * @param configContext the specified config context\n     * @return the added BrokerController or null if the broker already exists\n     * @throws Exception when initialize broker\n     */\n    BrokerController addBroker(ConfigContext configContext) throws Exception;\n\n    /**\n     * Remove the broker from this container associated with the specific broker identity\n     *\n     * @param brokerIdentity the specific broker identity\n     * @return the removed BrokerController or null if the broker doesn't exists\n     */\n    BrokerController removeBroker(BrokerIdentity brokerIdentity) throws Exception;\n\n    /**\n     * Return the broker controller associated with the specific broker identity\n     *\n     * @param brokerIdentity the specific broker identity\n     * @return the associated messaging broker or null\n     */\n    BrokerController getBroker(BrokerIdentity brokerIdentity);\n\n    /**\n     * Return all the master brokers belong to this container\n     *\n     * @return the master broker list\n     */\n    Collection<InnerBrokerController> getMasterBrokers();\n\n    /**\n     * Return all the slave brokers belong to this container\n     *\n     * @return the slave broker list\n     */\n    Collection<InnerSalveBrokerController> getSlaveBrokers();\n\n    /**\n     * Return all broker controller in this container\n     *\n     * @return all broker controller\n     */\n    List<BrokerController> getBrokerControllers();\n\n    /**\n     * Return the address of broker container.\n     *\n     * @return broker container address.\n     */\n    String getBrokerContainerAddr();\n\n    /**\n     * Peek the first master broker in container.\n     *\n     * @return the first master broker in container\n     */\n    BrokerController peekMasterBroker();\n\n    /**\n     * Return the config of the broker container\n     *\n     * @return the broker container config\n     */\n    BrokerContainerConfig getBrokerContainerConfig();\n\n    /**\n     * Get netty server config.\n     *\n     * @return netty server config\n     */\n    NettyServerConfig getNettyServerConfig();\n\n    /**\n     * Get netty client config.\n     *\n     * @return netty client config\n     */\n    NettyClientConfig getNettyClientConfig();\n\n    /**\n     * Return the shared BrokerOuterAPI\n     *\n     * @return the shared BrokerOuterAPI\n     */\n    BrokerOuterAPI getBrokerOuterAPI();\n\n    /**\n     * Return the shared RemotingServer\n     *\n     * @return the shared RemotingServer\n     */\n    RemotingServer getRemotingServer();\n}\n"
  },
  {
    "path": "container/src/main/java/org/apache/rocketmq/container/InnerBrokerController.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.container;\n\nimport java.util.concurrent.ScheduledFuture;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.out.BrokerOuterAPI;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.remoting.RemotingServer;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingServer;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\n\npublic class InnerBrokerController extends BrokerController {\n    protected BrokerContainer brokerContainer;\n\n    public InnerBrokerController(\n        final BrokerContainer brokerContainer,\n        final BrokerConfig brokerConfig,\n        final MessageStoreConfig messageStoreConfig,\n        final AuthConfig authConfig\n    ) {\n        super(brokerConfig, messageStoreConfig, authConfig);\n        this.brokerContainer = brokerContainer;\n        this.brokerOuterAPI = this.brokerContainer.getBrokerOuterAPI();\n    }\n\n    @Override\n    protected void initializeRemotingServer() {\n        RemotingServer remotingServer = this.brokerContainer.getRemotingServer().newRemotingServer(brokerConfig.getListenPort());\n        RemotingServer fastRemotingServer = this.brokerContainer.getRemotingServer().newRemotingServer(brokerConfig.getListenPort() - 2);\n\n        if (this.brokerMetricsManager != null && remotingServer instanceof NettyRemotingServer) {\n            ((NettyRemotingServer) remotingServer).setRemotingMetricsManager(this.brokerMetricsManager.getRemotingMetricsManager());\n        }\n\n        if (this.brokerMetricsManager != null && fastRemotingServer instanceof NettyRemotingServer) {\n            ((NettyRemotingServer) fastRemotingServer).setRemotingMetricsManager(this.brokerMetricsManager.getRemotingMetricsManager());\n        }\n\n        setRemotingServer(remotingServer);\n        setFastRemotingServer(fastRemotingServer);\n    }\n\n    @Override\n    protected void initializeScheduledTasks() {\n        initializeBrokerScheduledTasks();\n    }\n\n    @Override\n    public void start() throws Exception {\n        this.shouldStartTime = System.currentTimeMillis() + messageStoreConfig.getDisappearTimeAfterStart();\n\n        if (messageStoreConfig.getTotalReplicas() > 1 && this.brokerConfig.isEnableSlaveActingMaster()) {\n            isIsolated = true;\n        }\n\n        startBasicService();\n\n        if (!isIsolated && !this.messageStoreConfig.isEnableDLegerCommitLog() && !this.messageStoreConfig.isDuplicationEnable()) {\n            changeSpecialServiceStatus(this.brokerConfig.getBrokerId() == MixAll.MASTER_ID);\n            this.registerBrokerAll(true, false, true);\n        }\n\n        scheduledFutures.add(this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    if (System.currentTimeMillis() < shouldStartTime) {\n                        BrokerController.LOG.info(\"Register to namesrv after {}\", shouldStartTime);\n                        return;\n                    }\n                    if (isIsolated) {\n                        BrokerController.LOG.info(\"Skip register for broker is isolated\");\n                        return;\n                    }\n                    InnerBrokerController.this.registerBrokerAll(true, false, brokerConfig.isForceRegister());\n                } catch (Throwable e) {\n                    BrokerController.LOG.error(\"registerBrokerAll Exception\", e);\n                }\n            }\n        }, 1000 * 10, Math.max(10000, Math.min(brokerConfig.getRegisterNameServerPeriod(), 60000)), TimeUnit.MILLISECONDS));\n\n        if (this.brokerConfig.isEnableSlaveActingMaster()) {\n            scheduleSendHeartbeat();\n\n            scheduledFutures.add(this.syncBrokerMemberGroupExecutorService.scheduleAtFixedRate(new Runnable() {\n                @Override\n                public void run() {\n                    try {\n                        InnerBrokerController.this.syncBrokerMemberGroup();\n                    } catch (Throwable e) {\n                        BrokerController.LOG.error(\"sync BrokerMemberGroup error. \", e);\n                    }\n                }\n            }, 1000, this.brokerConfig.getSyncBrokerMemberGroupPeriod(), TimeUnit.MILLISECONDS));\n        }\n\n        if (this.brokerConfig.isEnableControllerMode()) {\n            scheduleSendHeartbeat();\n        }\n\n        if (brokerConfig.isSkipPreOnline()) {\n            startServiceWithoutCondition();\n        }\n    }\n\n    @Override\n    public void shutdown() {\n\n        shutdownBasicService();\n\n        for (ScheduledFuture<?> scheduledFuture : scheduledFutures) {\n            scheduledFuture.cancel(true);\n        }\n\n        if (getRemotingServer() != null) {\n            this.brokerContainer.getRemotingServer().removeRemotingServer(brokerConfig.getListenPort());\n        }\n\n        if (getFastRemotingServer() != null) {\n            this.brokerContainer.getRemotingServer().removeRemotingServer(brokerConfig.getListenPort() - 2);\n        }\n    }\n\n    @Override\n    public String getBrokerAddr() {\n        return this.brokerConfig.getBrokerIP1() + \":\" + this.brokerConfig.getListenPort();\n    }\n\n    @Override\n    public String getHAServerAddr() {\n        return this.brokerConfig.getBrokerIP2() + \":\" + this.messageStoreConfig.getHaListenPort();\n    }\n\n    @Override\n    public long getMinBrokerIdInGroup() {\n        return this.minBrokerIdInGroup;\n    }\n\n    @Override\n    public int getListenPort() {\n        return this.brokerConfig.getListenPort();\n    }\n\n    public BrokerOuterAPI getBrokerOuterAPI() {\n        return brokerContainer.getBrokerOuterAPI();\n    }\n\n    public BrokerContainer getBrokerContainer() {\n        return this.brokerContainer;\n    }\n\n    public NettyServerConfig getNettyServerConfig() {\n        return brokerContainer.getNettyServerConfig();\n    }\n\n    public NettyClientConfig getNettyClientConfig() {\n        return brokerContainer.getNettyClientConfig();\n    }\n\n    public MessageStore getMessageStoreByBrokerName(String brokerName) {\n        if (this.brokerConfig.getBrokerName().equals(brokerName)) {\n            return this.getMessageStore();\n        }\n        BrokerController brokerController = this.brokerContainer.findBrokerControllerByBrokerName(brokerName);\n        if (brokerController != null) {\n            return brokerController.getMessageStore();\n        }\n        return null;\n    }\n\n    @Override\n    public BrokerController peekMasterBroker() {\n        if (brokerConfig.getBrokerId() == MixAll.MASTER_ID) {\n            return this;\n        }\n        return this.brokerContainer.peekMasterBroker();\n    }\n}\n"
  },
  {
    "path": "container/src/main/java/org/apache/rocketmq/container/InnerSalveBrokerController.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.container;\n\nimport com.google.common.base.Preconditions;\n\n\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\n\npublic class InnerSalveBrokerController extends InnerBrokerController {\n\n    public InnerSalveBrokerController(final BrokerContainer brokerContainer,\n        final BrokerConfig brokerConfig,\n        final MessageStoreConfig storeConfig,\n        final AuthConfig authConfig) {\n        super(brokerContainer, brokerConfig, storeConfig, authConfig);\n        // Check configs\n        checkSlaveBrokerConfig();\n    }\n\n    private void checkSlaveBrokerConfig() {\n        Preconditions.checkNotNull(brokerConfig.getBrokerClusterName());\n        Preconditions.checkNotNull(brokerConfig.getBrokerName());\n        Preconditions.checkArgument(brokerConfig.getBrokerId() != MixAll.MASTER_ID);\n    }\n}\n"
  },
  {
    "path": "container/src/test/java/org/apache/rocketmq/container/BrokerContainerExtensibilityTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.container;\n\nimport java.io.File;\nimport java.util.Properties;\nimport java.util.UUID;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.concurrent.atomic.AtomicInteger;\n\nimport org.apache.rocketmq.broker.ConfigContext;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class BrokerContainerExtensibilityTest {\n\n    private BrokerContainer brokerContainer;\n    private BrokerContainerConfig containerConfig;\n    private NettyServerConfig nettyServerConfig;\n    private NettyClientConfig nettyClientConfig;\n    private File tempDir;\n\n    @Before\n    public void setUp() {\n        tempDir = new File(System.getProperty(\"java.io.tmpdir\") + File.separator + \"container-test-\" + UUID.randomUUID());\n        tempDir.mkdirs();\n\n        containerConfig = new BrokerContainerConfig();\n        // Note: brokerContainerIP is automatically set to local address\n        \n        nettyServerConfig = new NettyServerConfig();\n        nettyServerConfig.setListenPort(0); // Random port\n        \n        nettyClientConfig = new NettyClientConfig();\n\n        brokerContainer = new BrokerContainer(containerConfig, nettyServerConfig, nettyClientConfig);\n    }\n\n    @After\n    public void tearDown() {\n        if (brokerContainer != null) {\n            try {\n                brokerContainer.shutdown();\n            } catch (Exception e) {\n                // Ignore cleanup errors\n            }\n        }\n        UtilAll.deleteFile(tempDir);\n    }\n\n    @Test\n    public void testBrokerBootHookExtensibility() throws Exception {\n        // Test that BrokerBootHook system provides proper extensibility\n        // This test verifies that hooks can be registered and executed correctly\n        // during the broker lifecycle (before and after start)\n        brokerContainer.initialize();\n\n        // Create a test hook\n        AtomicInteger beforeStartCount = new AtomicInteger(0);\n        AtomicInteger afterStartCount = new AtomicInteger(0);\n        AtomicBoolean hookExecuted = new AtomicBoolean(false);\n\n        BrokerBootHook testHook = new BrokerBootHook() {\n            @Override\n            public String hookName() {\n                return \"TestBrokerBootHook\";\n            }\n            \n            @Override\n            public void executeBeforeStart(InnerBrokerController innerBrokerController, Properties properties) throws Exception {\n                beforeStartCount.incrementAndGet();\n                hookExecuted.set(true);\n                assertThat(innerBrokerController).isNotNull();\n            }\n            \n            @Override\n            public void executeAfterStart(InnerBrokerController innerBrokerController, Properties properties) throws Exception {\n                afterStartCount.incrementAndGet();\n                assertThat(innerBrokerController).isNotNull();\n            }\n        };\n        \n        // Register the hook\n        brokerContainer.getBrokerBootHookList().add(testHook);\n        \n        // Verify hook is registered\n        assertThat(brokerContainer.getBrokerBootHookList()).contains(testHook);\n        assertThat(brokerContainer.getBrokerBootHookList().size()).isGreaterThan(0);\n        \n        // Start container\n        brokerContainer.start();\n\n        // Create a broker to trigger hooks\n        BrokerConfig brokerConfig = new BrokerConfig();\n        brokerConfig.setBrokerClusterName(\"test-cluster\");\n        brokerConfig.setBrokerName(\"test-broker\");\n        brokerConfig.setBrokerId(0);\n\n        MessageStoreConfig storeConfig = new MessageStoreConfig();\n        storeConfig.setStorePathRootDir(tempDir.getAbsolutePath());\n        storeConfig.setStorePathCommitLog(tempDir.getAbsolutePath() + File.separator + \"commitlog\");\n\n        Properties testProperties = new Properties();\n        testProperties.setProperty(\"brokerName\", \"test-broker\");\n\n        ConfigContext configContext = new ConfigContext.Builder()\n            .brokerConfig(brokerConfig)\n            .messageStoreConfig(storeConfig)\n            .nettyServerConfig(new NettyServerConfig())\n            .nettyClientConfig(new NettyClientConfig())\n            .properties(testProperties)\n            .build();\n\n        // Add broker should trigger hooks\n        try {\n            InnerBrokerController innerBroker = brokerContainer.addBroker(configContext);\n\n            // Manually execute hooks as they would be executed during broker startup\n            // This simulates the real scenario where hooks are executed before broker.start()\n            for (BrokerBootHook brokerBootHook : brokerContainer.getBrokerBootHookList()) {\n                brokerBootHook.executeBeforeStart(innerBroker, testProperties);\n            }\n\n            // Verify hook was executed\n            assertThat(hookExecuted.get()).isTrue();\n            assertThat(beforeStartCount.get()).isEqualTo(1);\n\n            // Test executeAfterStart hook as well\n            for (BrokerBootHook brokerBootHook : brokerContainer.getBrokerBootHookList()) {\n                brokerBootHook.executeAfterStart(innerBroker, testProperties);\n            }\n            assertThat(afterStartCount.get()).isEqualTo(1);\n\n            // Cleanup\n            brokerContainer.removeBroker(innerBroker.getBrokerIdentity());\n        } catch (Exception e) {\n            // Expected for some configurations in test environment\n        }\n    }\n\n    @Test\n    public void testContainerConfigurationExtensibility() throws Exception {\n        // Test that container configuration is properly accessible and modifiable\n        assertThat(brokerContainer.getBrokerContainerConfig()).isNotNull();\n        assertThat(brokerContainer.getBrokerContainerConfig()).isSameAs(containerConfig);\n        \n        // Test address configuration\n        String expectedAddr = containerConfig.getBrokerContainerIP() + \":\" + nettyServerConfig.getListenPort();\n        assertThat(brokerContainer.getBrokerContainerAddr()).isEqualTo(expectedAddr);\n        \n        // Test network configuration access\n        assertThat(brokerContainer.getNettyServerConfig()).isSameAs(nettyServerConfig);\n        assertThat(brokerContainer.getNettyClientConfig()).isSameAs(nettyClientConfig);\n        assertThat(brokerContainer.getBrokerOuterAPI()).isNotNull();\n    }\n\n    @Test\n    public void testContainerInitialization() throws Exception {\n        // Test container initialization process\n        assertThat(brokerContainer.initialize()).isTrue();\n        \n        // Verify essential components are initialized\n        assertThat(brokerContainer.getRemotingServer()).isNotNull();\n        assertThat(brokerContainer.getBrokerOuterAPI()).isNotNull();\n        assertThat(brokerContainer.getConfiguration()).isNotNull();\n        \n        // Test that container can be started and stopped\n        brokerContainer.start();\n        assertThat(brokerContainer.getRemotingServer()).isNotNull();\n        \n        brokerContainer.shutdown();\n    }\n\n    @Test\n    public void testBrokerContainerProcessor() throws Exception {\n        // Test that BrokerContainerProcessor is properly integrated\n        brokerContainer.initialize();\n        brokerContainer.start();\n        \n        // Verify processor is registered\n        assertThat(brokerContainer.getRemotingServer()).isNotNull();\n        \n        // The processor should handle container-specific requests\n        // (Implementation details are tested in the processor's own tests)\n        assertThat(true).isTrue(); // Placeholder for successful integration\n    }\n\n    @Test\n    public void testContainerStartupAndShutdownSequence() throws Exception {\n        // Test proper startup and shutdown sequence\n        assertThat(brokerContainer.initialize()).isTrue();\n        \n        // Start should succeed\n        brokerContainer.start();\n        \n        // Shutdown should clean up properly\n        brokerContainer.shutdown();\n        \n        // Multiple shutdown calls should be safe\n        brokerContainer.shutdown();\n    }\n\n    @Test\n    public void testContainerExtensibilityPoints() throws Exception {\n        // Test that the container provides proper extension points\n        brokerContainer.initialize();\n        \n        // Verify that hook list is accessible and modifiable\n        int initialHookCount = brokerContainer.getBrokerBootHookList().size();\n        \n        BrokerBootHook extensionHook = new BrokerBootHook() {\n            @Override\n            public String hookName() {\n                return \"ExtensionTestHook\";\n            }\n            \n            @Override\n            public void executeBeforeStart(InnerBrokerController innerBrokerController, Properties properties) {\n                // Extension point for before start\n            }\n            \n            @Override\n            public void executeAfterStart(InnerBrokerController innerBrokerController, Properties properties) {\n                // Extension point for after start\n            }\n        };\n        \n        brokerContainer.getBrokerBootHookList().add(extensionHook);\n        assertThat(brokerContainer.getBrokerBootHookList().size()).isEqualTo(initialHookCount + 1);\n        \n        // Verify hook name is accessible\n        assertThat(extensionHook.hookName()).isEqualTo(\"ExtensionTestHook\");\n    }\n}"
  },
  {
    "path": "container/src/test/java/org/apache/rocketmq/container/BrokerContainerStartupTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.container;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.nio.file.Files;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Properties;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.assertj.core.util.Arrays;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.apache.rocketmq.container.BrokerContainerStartup.parseCmdLineToConfig;\nimport static org.assertj.core.api.Java6Assertions.assertThat;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class BrokerContainerStartupTest {\n    private static final List<File> TMP_FILE_LIST = new ArrayList<>();\n    private static final String BROKER_NAME_PREFIX = \"TestBroker\";\n    private static final String SHARED_BROKER_NAME_PREFIX = \"TestBrokerContainer\";\n    private static String brokerConfigPath;\n    private static String brokerContainerConfigPath;\n\n    @Mock\n    private BrokerConfig brokerConfig;\n    private String storePathRootDir = \"store/test\";\n    @Mock\n    private NettyClientConfig nettyClientConfig;\n    @Mock\n    private NettyServerConfig nettyServerConfig;\n\n    @Before\n    public void init() throws IOException {\n        String brokerName = BROKER_NAME_PREFIX + \"_\" + System.currentTimeMillis();\n        BrokerConfig brokerConfig = new BrokerConfig();\n        brokerConfig.setBrokerName(brokerName);\n        if (brokerConfig.getRocketmqHome() == null) {\n            brokerConfig.setRocketmqHome(\"../distribution\");\n        }\n        MessageStoreConfig storeConfig = new MessageStoreConfig();\n        String baseDir = createBaseDir(brokerConfig.getBrokerName() + \"_\" + brokerConfig.getBrokerId()).getAbsolutePath();\n        storeConfig.setStorePathRootDir(baseDir);\n        storeConfig.setStorePathCommitLog(baseDir + File.separator + \"commitlog\");\n\n        brokerConfigPath = \"/tmp/\" + brokerName;\n        brokerConfig.setBrokerConfigPath(brokerConfigPath);\n        File file = new File(brokerConfigPath);\n        TMP_FILE_LIST.add(file);\n        Properties brokerConfigProp = MixAll.object2Properties(brokerConfig);\n        Properties storeConfigProp = MixAll.object2Properties(storeConfig);\n\n        for (Object key : storeConfigProp.keySet()) {\n            brokerConfigProp.put(key, storeConfigProp.get(key));\n        }\n        MixAll.string2File(MixAll.properties2String(brokerConfigProp), brokerConfigPath);\n\n        brokerContainerConfigPath = \"/tmp/\" + SHARED_BROKER_NAME_PREFIX + System.currentTimeMillis();\n        BrokerContainerConfig brokerContainerConfig = new BrokerContainerConfig();\n        brokerContainerConfig.setBrokerConfigPaths(brokerConfigPath);\n        if (brokerContainerConfig.getRocketmqHome() == null) {\n            brokerContainerConfig.setRocketmqHome(\"../distribution\");\n        }\n        File file1 = new File(brokerContainerConfigPath);\n        TMP_FILE_LIST.add(file1);\n        Properties brokerContainerConfigProp = MixAll.object2Properties(brokerContainerConfig);\n        MixAll.string2File(MixAll.properties2String(brokerContainerConfigProp), brokerContainerConfigPath);\n    }\n\n    @After\n    public void destroy() {\n        for (File file : TMP_FILE_LIST) {\n            UtilAll.deleteFile(file);\n        }\n    }\n\n    @Test\n    public void testStartBrokerContainer() {\n        final BrokerContainerConfig containerConfig = new BrokerContainerConfig();\n        final NettyServerConfig nettyServerConfig = new NettyServerConfig();\n        final NettyClientConfig nettyClientConfig = new NettyClientConfig();\n        parseCmdLineToConfig(Arrays.array(\"-c\", brokerContainerConfigPath), containerConfig, nettyServerConfig, nettyClientConfig);\n        BrokerContainer brokerContainer = BrokerContainerStartup.startBrokerContainer(\n            BrokerContainerStartup.createBrokerContainer(containerConfig, nettyServerConfig, nettyClientConfig));\n        assertThat(brokerContainer).isNotNull();\n        List<BrokerController> brokers = BrokerContainerStartup.createAndStartBrokers(brokerContainer);\n        assertThat(brokers.size()).isEqualTo(1);\n\n        brokerContainer.shutdown();\n        assertThat(brokerContainer.getBrokerControllers().size()).isEqualTo(0);\n    }\n\n    private static File createBaseDir(String prefix) {\n        final File file;\n        try {\n            file = Files.createTempDirectory(prefix).toFile();\n            TMP_FILE_LIST.add(file);\n            System.out.printf(\"create file at %s%n\", file.getAbsolutePath());\n            return file;\n        } catch (IOException e) {\n            throw new RuntimeException(\"Couldn't create tmp folder\", e);\n        }\n    }\n\n    @Before\n    public void clear() {\n        UtilAll.deleteFile(new File(storePathRootDir));\n    }\n\n    @After\n    public void tearDown() {\n        File configFile = new File(storePathRootDir);\n        UtilAll.deleteFile(configFile);\n        UtilAll.deleteEmptyDirectory(configFile);\n        UtilAll.deleteEmptyDirectory(configFile.getParentFile());\n    }\n}"
  },
  {
    "path": "container/src/test/java/org/apache/rocketmq/container/BrokerContainerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.container;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.lang.reflect.Field;\nimport java.nio.file.Files;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Random;\nimport java.util.Set;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.ConfigContext;\nimport org.apache.rocketmq.broker.out.BrokerOuterAPI;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.BrokerIdentity;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.After;\nimport org.junit.Test;\nimport org.mockito.ArgumentCaptor;\nimport org.mockito.ArgumentMatchers;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.assertj.core.api.Assertions.entry;\nimport static org.mockito.Mockito.anyBoolean;\nimport static org.mockito.Mockito.anyInt;\nimport static org.mockito.Mockito.anyLong;\nimport static org.mockito.Mockito.anyString;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\n\npublic class BrokerContainerTest {\n    private static final List<File> TMP_FILE_LIST = new ArrayList<>();\n    private static final Random RANDOM = new Random();\n    private static final Set<Integer> PORTS_IN_USE = new HashSet<>();\n\n    /**\n     * Tests if the controller can be properly stopped and started.\n     *\n     * @throws Exception If fails.\n     */\n    @Test\n    public void testBrokerContainerRestart() throws Exception {\n        BrokerContainer brokerController = new BrokerContainer(\n            new BrokerContainerConfig(),\n            new NettyServerConfig(),\n            new NettyClientConfig());\n        assertThat(brokerController.initialize()).isTrue();\n        brokerController.start();\n        brokerController.shutdown();\n    }\n\n    @Test\n    public void testRegisterIncrementBrokerData() throws Exception {\n        BrokerController brokerController = new BrokerController(\n            new BrokerConfig(),\n            new NettyServerConfig(),\n            new NettyClientConfig(),\n            new MessageStoreConfig());\n\n        brokerController.getBrokerConfig().setEnableSlaveActingMaster(true);\n\n        BrokerOuterAPI brokerOuterAPI = mock(BrokerOuterAPI.class);\n        Field field = BrokerController.class.getDeclaredField(\"brokerOuterAPI\");\n        field.setAccessible(true);\n        field.set(brokerController, brokerOuterAPI);\n\n        List<TopicConfig> topicConfigList = new ArrayList<>(2);\n        for (int i = 0; i < 2; i++) {\n            topicConfigList.add(new TopicConfig(\"topic-\" + i));\n        }\n        DataVersion dataVersion = new DataVersion();\n\n        // Check normal condition.\n        testRegisterIncrementBrokerDataWithPerm(brokerController, brokerOuterAPI,\n            topicConfigList, dataVersion, PermName.PERM_READ | PermName.PERM_WRITE, 1);\n        // Check unwritable broker.\n        testRegisterIncrementBrokerDataWithPerm(brokerController, brokerOuterAPI,\n            topicConfigList, dataVersion, PermName.PERM_READ, 2);\n        // Check unreadable broker.\n        testRegisterIncrementBrokerDataWithPerm(brokerController, brokerOuterAPI,\n            topicConfigList, dataVersion, PermName.PERM_WRITE, 3);\n    }\n\n    @Test\n    public void testRegisterIncrementBrokerDataPerm() throws Exception {\n        BrokerController brokerController = new BrokerController(\n            new BrokerConfig(),\n            new NettyServerConfig(),\n            new NettyClientConfig(),\n            new MessageStoreConfig());\n\n        brokerController.getBrokerConfig().setEnableSlaveActingMaster(true);\n\n        BrokerOuterAPI brokerOuterAPI = mock(BrokerOuterAPI.class);\n        Field field = BrokerController.class.getDeclaredField(\"brokerOuterAPI\");\n        field.setAccessible(true);\n        field.set(brokerController, brokerOuterAPI);\n\n        List<TopicConfig> topicConfigList = new ArrayList<>(2);\n        for (int i = 0; i < 2; i++) {\n            topicConfigList.add(new TopicConfig(\"topic-\" + i));\n        }\n        DataVersion dataVersion = new DataVersion();\n\n        brokerController.getBrokerConfig().setBrokerPermission(4);\n\n        brokerController.registerIncrementBrokerData(topicConfigList, dataVersion);\n        // Get topicConfigSerializeWrapper created by registerIncrementBrokerData() from brokerOuterAPI.registerBrokerAll()\n        ArgumentCaptor<TopicConfigSerializeWrapper> captor = ArgumentCaptor.forClass(TopicConfigSerializeWrapper.class);\n        ArgumentCaptor<BrokerIdentity> brokerIdentityCaptor = ArgumentCaptor.forClass(BrokerIdentity.class);\n        verify(brokerOuterAPI).registerBrokerAll(anyString(), anyString(), anyString(), anyLong(), anyString(),\n            captor.capture(), ArgumentMatchers.anyList(), anyBoolean(), anyInt(), anyBoolean(), anyBoolean(), anyLong(), brokerIdentityCaptor.capture());\n        TopicConfigSerializeWrapper wrapper = captor.getValue();\n        for (Map.Entry<String, TopicConfig> entry : wrapper.getTopicConfigTable().entrySet()) {\n            assertThat(entry.getValue().getPerm()).isEqualTo(brokerController.getBrokerConfig().getBrokerPermission());\n        }\n\n    }\n\n    @Test\n    public void testMasterScaleOut() throws Exception {\n        BrokerContainer brokerContainer = new BrokerContainer(\n            new BrokerContainerConfig(),\n            new NettyServerConfig(),\n            new NettyClientConfig());\n        assertThat(brokerContainer.initialize()).isTrue();\n        brokerContainer.getBrokerContainerConfig().setNamesrvAddr(\"127.0.0.1:9876\");\n        brokerContainer.start();\n\n        BrokerConfig masterBrokerConfig = new BrokerConfig();\n\n        String baseDir = createBaseDir(\"unnittest-master\").getAbsolutePath();\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setStorePathRootDir(baseDir);\n        messageStoreConfig.setStorePathCommitLog(baseDir + File.separator + \"commitlog\");\n        ConfigContext configContext = new ConfigContext.Builder()\n            .brokerConfig(masterBrokerConfig)\n            .messageStoreConfig(messageStoreConfig)\n            .build();\n        InnerBrokerController brokerController = brokerContainer.addBroker(configContext);\n        assertThat(brokerController.isIsolated()).isFalse();\n\n        brokerContainer.shutdown();\n        brokerController.getMessageStore().destroy();\n    }\n\n    @Test\n    public void testAddMasterFailed() throws Exception {\n        BrokerContainer brokerContainer = new BrokerContainer(\n            new BrokerContainerConfig(),\n            new NettyServerConfig(),\n            new NettyClientConfig());\n        assertThat(brokerContainer.initialize()).isTrue();\n        brokerContainer.start();\n\n        BrokerConfig masterBrokerConfig = new BrokerConfig();\n        masterBrokerConfig.setListenPort(brokerContainer.getNettyServerConfig().getListenPort());\n        boolean exceptionCaught = false;\n        try {\n            String baseDir = createBaseDir(\"unnittest-master\").getAbsolutePath();\n            MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n            messageStoreConfig.setStorePathRootDir(baseDir);\n            messageStoreConfig.setStorePathCommitLog(baseDir + File.separator + \"commitlog\");\n            ConfigContext configContext = new ConfigContext.Builder()\n                .brokerConfig(masterBrokerConfig)\n                .messageStoreConfig(messageStoreConfig)\n                .build();\n            brokerContainer.addBroker(configContext);\n        } catch (Exception e) {\n            exceptionCaught = true;\n        } finally {\n            brokerContainer.shutdown();\n\n        }\n\n        assertThat(exceptionCaught).isTrue();\n    }\n\n    @Test\n    public void testAddSlaveFailed() throws Exception {\n        BrokerContainer sharedBrokerController = new BrokerContainer(\n            new BrokerContainerConfig(),\n            new NettyServerConfig(),\n            new NettyClientConfig());\n        assertThat(sharedBrokerController.initialize()).isTrue();\n        sharedBrokerController.start();\n\n        BrokerConfig slaveBrokerConfig = new BrokerConfig();\n        slaveBrokerConfig.setBrokerId(1);\n        slaveBrokerConfig.setListenPort(sharedBrokerController.getNettyServerConfig().getListenPort());\n        MessageStoreConfig slaveMessageStoreConfig = new MessageStoreConfig();\n        slaveMessageStoreConfig.setBrokerRole(BrokerRole.SLAVE);\n        String baseDir = createBaseDir(\"unnittest-slave\").getAbsolutePath();\n        slaveMessageStoreConfig.setStorePathRootDir(baseDir);\n        slaveMessageStoreConfig.setStorePathCommitLog(baseDir + File.separator + \"commitlog\");\n        boolean exceptionCaught = false;\n        ConfigContext configContext = new ConfigContext.Builder()\n            .brokerConfig(slaveBrokerConfig)\n            .messageStoreConfig(slaveMessageStoreConfig)\n            .build();\n        try {\n            sharedBrokerController.addBroker(configContext);\n        } catch (Exception e) {\n            exceptionCaught = true;\n        } finally {\n            sharedBrokerController.shutdown();\n        }\n\n        assertThat(exceptionCaught).isTrue();\n    }\n\n    @Test\n    public void testAddAndRemoveMaster() throws Exception {\n        BrokerContainer brokerContainer = new BrokerContainer(\n            new BrokerContainerConfig(),\n            new NettyServerConfig(),\n            new NettyClientConfig());\n        assertThat(brokerContainer.initialize()).isTrue();\n        brokerContainer.start();\n\n        BrokerConfig masterBrokerConfig = new BrokerConfig();\n        String baseDir = createBaseDir(\"unnittest-master\").getAbsolutePath();\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setStorePathRootDir(baseDir);\n        messageStoreConfig.setStorePathCommitLog(baseDir + File.separator + \"commitlog\");\n        ConfigContext configContext = new ConfigContext.Builder()\n            .brokerConfig(masterBrokerConfig)\n            .messageStoreConfig(messageStoreConfig)\n            .build();\n        InnerBrokerController master = brokerContainer.addBroker(configContext);\n        assertThat(master).isNotNull();\n        master.start();\n        assertThat(master.isIsolated()).isFalse();\n\n        brokerContainer.removeBroker(new BrokerIdentity(masterBrokerConfig.getBrokerClusterName(), masterBrokerConfig.getBrokerName(), masterBrokerConfig.getBrokerId()));\n        assertThat(brokerContainer.getMasterBrokers().size()).isEqualTo(0);\n\n        brokerContainer.shutdown();\n        master.getMessageStore().destroy();\n    }\n\n    @Test\n    public void testAddAndRemoveDLedgerBroker() throws Exception {\n        BrokerContainer brokerContainer = new BrokerContainer(\n            new BrokerContainerConfig(),\n            new NettyServerConfig(),\n            new NettyClientConfig());\n        assertThat(brokerContainer.initialize()).isTrue();\n        brokerContainer.start();\n\n        BrokerConfig dLedgerBrokerConfig = new BrokerConfig();\n        String baseDir = createBaseDir(\"unnittest-dLedger\").getAbsolutePath();\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setStorePathRootDir(baseDir);\n        messageStoreConfig.setStorePathCommitLog(baseDir + File.separator + \"commitlog\");\n        messageStoreConfig.setEnableDLegerCommitLog(true);\n        messageStoreConfig.setdLegerSelfId(\"n0\");\n        messageStoreConfig.setdLegerGroup(\"group\");\n        messageStoreConfig.setdLegerPeers(String.format(\"n0-localhost:%d\", generatePort(30900, 10000)));\n        ConfigContext configContext = new ConfigContext.Builder()\n            .brokerConfig(dLedgerBrokerConfig)\n            .messageStoreConfig(messageStoreConfig)\n            .build();\n        InnerBrokerController dLedger = brokerContainer.addBroker(configContext);\n        assertThat(dLedger).isNotNull();\n        dLedger.start();\n        assertThat(dLedger.isIsolated()).isFalse();\n\n        brokerContainer.removeBroker(new BrokerIdentity(dLedgerBrokerConfig.getBrokerClusterName(), dLedgerBrokerConfig.getBrokerName(), Integer.parseInt(messageStoreConfig.getdLegerSelfId().substring(1))));\n        assertThat(brokerContainer.getMasterBrokers().size()).isEqualTo(0);\n\n        brokerContainer.shutdown();\n        dLedger.getMessageStore().destroy();\n    }\n\n    @Test\n    public void testAddAndRemoveSlaveSuccess() throws Exception {\n        BrokerContainer brokerContainer = new BrokerContainer(\n            new BrokerContainerConfig(),\n            new NettyServerConfig(),\n            new NettyClientConfig());\n        assertThat(brokerContainer.initialize()).isTrue();\n        brokerContainer.start();\n\n        BrokerConfig masterBrokerConfig = new BrokerConfig();\n        String baseDir = createBaseDir(\"unnittest-master\").getAbsolutePath();\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setStorePathRootDir(baseDir);\n        messageStoreConfig.setStorePathCommitLog(baseDir + File.separator + \"commitlog\");\n        ConfigContext masterBrokerConfigContext = new ConfigContext.Builder()\n            .brokerConfig(masterBrokerConfig)\n            .messageStoreConfig(messageStoreConfig)\n            .build();\n        InnerBrokerController master = brokerContainer.addBroker(masterBrokerConfigContext);\n        assertThat(master).isNotNull();\n        master.start();\n        assertThat(master.isIsolated()).isFalse();\n\n        BrokerConfig slaveBrokerConfig = new BrokerConfig();\n        slaveBrokerConfig.setListenPort(generatePort(masterBrokerConfig.getListenPort(), 10000));\n        slaveBrokerConfig.setBrokerId(1);\n        MessageStoreConfig slaveMessageStoreConfig = new MessageStoreConfig();\n        slaveMessageStoreConfig.setBrokerRole(BrokerRole.SLAVE);\n        slaveMessageStoreConfig.setHaListenPort(generatePort(messageStoreConfig.getHaListenPort(), 10000));\n        baseDir = createBaseDir(\"unnittest-slave\").getAbsolutePath();\n        slaveMessageStoreConfig.setStorePathRootDir(baseDir);\n        slaveMessageStoreConfig.setStorePathCommitLog(baseDir + File.separator + \"commitlog\");\n        ConfigContext slaveBrokerConfigContext = new ConfigContext.Builder()\n            .brokerConfig(slaveBrokerConfig)\n            .messageStoreConfig(slaveMessageStoreConfig)\n            .build();\n        InnerBrokerController slave = brokerContainer.addBroker(slaveBrokerConfigContext);\n        assertThat(slave).isNotNull();\n        slave.start();\n        assertThat(slave.isIsolated()).isFalse();\n\n        brokerContainer.removeBroker(new BrokerIdentity(slaveBrokerConfig.getBrokerClusterName(), slaveBrokerConfig.getBrokerName(), slaveBrokerConfig.getBrokerId()));\n        assertThat(brokerContainer.getSlaveBrokers().size()).isEqualTo(0);\n\n        brokerContainer.removeBroker(new BrokerIdentity(masterBrokerConfig.getBrokerClusterName(), masterBrokerConfig.getBrokerName(), masterBrokerConfig.getBrokerId()));\n        assertThat(brokerContainer.getMasterBrokers().size()).isEqualTo(0);\n\n        brokerContainer.shutdown();\n        slave.getMessageStore().destroy();\n        master.getMessageStore().destroy();\n    }\n\n    private static File createBaseDir(String prefix) {\n        final File file;\n        try {\n            file = Files.createTempDirectory(prefix).toFile();\n            TMP_FILE_LIST.add(file);\n            return file;\n        } catch (IOException e) {\n            throw new RuntimeException(\"Couldn't create tmp folder\", e);\n        }\n    }\n\n    public static int generatePort(int base, int range) {\n        int result = base + RANDOM.nextInt(range);\n        while (PORTS_IN_USE.contains(result) || PORTS_IN_USE.contains(result - 2)) {\n            result = base + RANDOM.nextInt(range);\n        }\n        PORTS_IN_USE.add(result);\n        PORTS_IN_USE.add(result - 2);\n        return result;\n    }\n\n    @After\n    public void destroy() {\n        for (File file : TMP_FILE_LIST) {\n            UtilAll.deleteFile(file);\n        }\n    }\n\n    private void testRegisterIncrementBrokerDataWithPerm(BrokerController brokerController,\n        BrokerOuterAPI brokerOuterAPI,\n        List<TopicConfig> topicConfigList, DataVersion dataVersion, int perm, int times) {\n        brokerController.getBrokerConfig().setBrokerPermission(perm);\n\n        brokerController.registerIncrementBrokerData(topicConfigList, dataVersion);\n        // Get topicConfigSerializeWrapper created by registerIncrementBrokerData() from brokerOuterAPI.registerBrokerAll()\n        ArgumentCaptor<TopicConfigSerializeWrapper> captor = ArgumentCaptor.forClass(TopicConfigSerializeWrapper.class);\n        ArgumentCaptor<BrokerIdentity> brokerIdentityCaptor = ArgumentCaptor.forClass(BrokerIdentity.class);\n        verify(brokerOuterAPI, times(times)).registerBrokerAll(anyString(), anyString(), anyString(), anyLong(),\n            anyString(), captor.capture(), ArgumentMatchers.anyList(), anyBoolean(), anyInt(), anyBoolean(), anyBoolean(), anyLong(), brokerIdentityCaptor.capture());\n        TopicConfigSerializeWrapper wrapper = captor.getValue();\n\n        for (TopicConfig topicConfig : topicConfigList) {\n            topicConfig.setPerm(perm);\n        }\n        assertThat(wrapper.getDataVersion()).isEqualTo(dataVersion);\n        assertThat(wrapper.getTopicConfigTable()).containsExactly(\n            entry(\"topic-0\", topicConfigList.get(0)),\n            entry(\"topic-1\", topicConfigList.get(1)));\n        for (TopicConfig topicConfig : topicConfigList) {\n            topicConfig.setPerm(PermName.PERM_READ | PermName.PERM_WRITE);\n        }\n    }\n}\n"
  },
  {
    "path": "container/src/test/java/org/apache/rocketmq/container/BrokerPreOnlineServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.container;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.BrokerPreOnlineService;\nimport org.apache.rocketmq.broker.out.BrokerOuterAPI;\nimport org.apache.rocketmq.broker.transaction.TransactionalMessageCheckService;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.remoting.protocol.BrokerSyncInfo;\nimport org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.lang.reflect.Field;\nimport java.time.Duration;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.concurrent.TimeUnit;\n\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertEquals;\nimport static org.mockito.ArgumentMatchers.anyBoolean;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class BrokerPreOnlineServiceTest {\n    @Mock\n    private BrokerContainer brokerContainer;\n\n    private InnerBrokerController innerBrokerController;\n\n    @Mock\n    private BrokerOuterAPI brokerOuterAPI;\n\n    public void init(final long brokerId) throws Exception {\n        when(brokerContainer.getBrokerOuterAPI()).thenReturn(brokerOuterAPI);\n\n        BrokerMemberGroup brokerMemberGroup1 = new BrokerMemberGroup();\n        Map<Long, String> brokerAddrMap = new HashMap<>();\n        brokerAddrMap.put(0L, \"127.0.0.1:10911\");\n        brokerMemberGroup1.setBrokerAddrs(brokerAddrMap);\n\n        BrokerMemberGroup brokerMemberGroup2 = new BrokerMemberGroup();\n        brokerMemberGroup2.setBrokerAddrs(new HashMap<>());\n        when(brokerOuterAPI.syncBrokerMemberGroup(anyString(), anyString(), anyBoolean()))\n                .thenReturn(brokerMemberGroup1)\n                .thenReturn(brokerMemberGroup2);\n        BrokerSyncInfo brokerSyncInfo = mock(BrokerSyncInfo.class);\n        when(brokerOuterAPI.retrieveBrokerHaInfo(anyString())).thenReturn(brokerSyncInfo);\n\n        DefaultMessageStore defaultMessageStore = mock(DefaultMessageStore.class);\n        when(defaultMessageStore.getMessageStoreConfig()).thenReturn(new MessageStoreConfig());\n        BrokerConfig brokerConfig = new BrokerConfig();\n        brokerConfig.setBrokerId(brokerId);\n        when(defaultMessageStore.getBrokerConfig()).thenReturn(brokerConfig);\n\n//        HAService haService = new DefaultHAService();\n//        haService.init(defaultMessageStore);\n//        haService.start();\n//\n//        when(defaultMessageStore.getHaService()).thenReturn(haService);\n\n        innerBrokerController = new InnerBrokerController(brokerContainer,\n            defaultMessageStore.getBrokerConfig(),\n            defaultMessageStore.getMessageStoreConfig(), null);\n\n        innerBrokerController.setTransactionalMessageCheckService(new TransactionalMessageCheckService(innerBrokerController));\n\n        Field field = BrokerController.class.getDeclaredField(\"isIsolated\");\n        field.setAccessible(true);\n        field.set(innerBrokerController, true);\n\n        field = BrokerController.class.getDeclaredField(\"messageStore\");\n        field.setAccessible(true);\n        field.set(innerBrokerController, defaultMessageStore);\n    }\n\n    @Test\n    public void testMasterOnlineConnTimeout() throws Exception {\n        init(0);\n        BrokerPreOnlineService brokerPreOnlineService = new BrokerPreOnlineService(innerBrokerController);\n\n        brokerPreOnlineService.start();\n\n        await().atMost(Duration.ofSeconds(30)).until(() -> !innerBrokerController.isIsolated());\n    }\n\n    @Test\n    public void testMasterOnlineNormal() throws Exception {\n        init(0);\n        BrokerPreOnlineService brokerPreOnlineService = new BrokerPreOnlineService(innerBrokerController);\n        brokerPreOnlineService.start();\n        TimeUnit.SECONDS.sleep(1);\n        brokerPreOnlineService.shutdown();\n        await().atMost(Duration.ofSeconds(30)).until(brokerPreOnlineService::isStopped);\n    }\n\n    @Test\n    public void testSlaveOnlineNormal() throws Exception {\n        init(1);\n        BrokerPreOnlineService brokerPreOnlineService = new BrokerPreOnlineService(innerBrokerController);\n        brokerPreOnlineService.start();\n        TimeUnit.SECONDS.sleep(1);\n        brokerPreOnlineService.shutdown();\n        await().atMost(Duration.ofSeconds(30)).until(brokerPreOnlineService::isStopped);\n    }\n\n    @Test\n    public void testGetServiceName() throws Exception {\n        init(1);\n        BrokerPreOnlineService brokerPreOnlineService = new BrokerPreOnlineService(innerBrokerController);\n        assertEquals(BrokerPreOnlineService.class.getSimpleName(), brokerPreOnlineService.getServiceName());\n    }\n}\n"
  },
  {
    "path": "container/src/test/resources/rmq.logback-test.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n<configuration>\n\n    <appender name=\"CONSOLE\" class=\"ch.qos.logback.core.ConsoleAppender\">\n        <layout class=\"ch.qos.logback.classic.PatternLayout\">\n            <Pattern>\n                %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n\n            </Pattern>\n        </layout>\n    </appender>\n\n    <logger name=\"org.apache.rocketmq\" level=\"error\" additivity=\"false\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </logger>\n\n    <root level=\"error\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </root>\n\n</configuration>"
  },
  {
    "path": "controller/BUILD.bazel",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\nload(\"//bazel:GenTestRules.bzl\", \"GenTestRules\")\n\njava_library(\n    name = \"controller\",\n    srcs = glob([\"src/main/java/**/*.java\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \"//common\",\n        \"//remoting\",\n        \"//srvutil\", \n        \"@maven//:io_openmessaging_storage_dledger\",\n        \"@maven//:org_apache_commons_commons_lang3\",\n        \"@maven//:commons_validator_commons_validator\",\n        \"@maven//:commons_collections_commons_collections\",\n        \"@maven//:commons_codec_commons_codec\",\n        \"@maven//:com_github_luben_zstd_jni\",\n        \"@maven//:org_lz4_lz4_java\",\n        \"@maven//:com_alibaba_fastjson2_fastjson2\",\n        \"@maven//:io_netty_netty_all\",\n        \"@maven//:com_google_guava_guava\",\n        \"@maven//:org_slf4j_slf4j_api\",\n        \"@maven//:ch_qos_logback_logback_core\",\n        \"@maven//:ch_qos_logback_logback_classic\",\n        \"@maven//:commons_cli_commons_cli\",\n        \"@maven//:io_github_aliyunmq_rocketmq_slf4j_api\",\n        \"@maven//:io_github_aliyunmq_rocketmq_logback_classic\",\n        \"@maven//:io_opentelemetry_opentelemetry_api\",\n        \"@maven//:io_opentelemetry_opentelemetry_context\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_otlp\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_prometheus\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk_common\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk_metrics\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_logging\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp\",\n        \"@maven//:org_slf4j_jul_to_slf4j\",\n        \"@maven//:com_alipay_sofa_jraft_core\",\n        \"@maven//:com_alipay_sofa_hessian\",\n        \"@maven//:commons_io_commons_io\",\n    ],\n)\n\njava_library(\n    name = \"tests\",\n    srcs = glob([\"src/test/java/**/*.java\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \":controller\",\n        \"//common\",\n        \"//remoting\",\n        \"//srvutil\", \n        \"@maven//:io_openmessaging_storage_dledger\",\n        \"//:test_deps\",\n        \"@maven//:org_apache_commons_commons_lang3\", \n        \"@maven//:io_netty_netty_all\",    \n        \"@maven//:com_google_guava_guava\",  \n        \"@maven//:com_alibaba_fastjson2_fastjson2\",\n    ],\n    resources = glob([\"src/test/resources/certs/*.pem\"]) + glob([\"src/test/resources/certs/*.key\"])\n)\n\nGenTestRules(\n    name = \"GeneratedTestRules\",\n    test_files = glob([\"src/test/java/**/*Test.java\"]),\n    deps = [\n        \":tests\",\n    ],\n    exclude_tests = [\n        # This test is buggy, exclude it.\n        \"src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest\",\n    ],\n    medium_tests = [\n        \"src/test/java/org/apache/rocketmq/controller/impl/controller/impl/DLedgerControllerTest\",\n    ],\n)\n"
  },
  {
    "path": "controller/pom.xml",
    "content": "<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n    <parent>\n        <artifactId>rocketmq-all</artifactId>\n        <groupId>org.apache.rocketmq</groupId>\n        <version>${revision}</version>\n    </parent>\n    <modelVersion>4.0.0</modelVersion>\n    <packaging>jar</packaging>\n    <artifactId>rocketmq-controller</artifactId>\n    <name>rocketmq-controller ${project.version}</name>\n\n    <properties>\n        <project.root>${basedir}/..</project.root>\n    </properties>\n\n    <dependencies>\n        <dependency>\n            <groupId>io.openmessaging.storage</groupId>\n            <artifactId>dledger</artifactId>\n            <exclusions>\n                <exclusion>\n                    <groupId>org.apache.rocketmq</groupId>\n                    <artifactId>rocketmq-remoting</artifactId>\n                </exclusion>\n            </exclusions>\n        </dependency>\n        <dependency>\n            <groupId>org.slf4j</groupId>\n            <artifactId>slf4j-api</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.github.aliyunmq</groupId>\n            <artifactId>rocketmq-shaded-slf4j-api-bridge</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-client</artifactId>\n            <scope>test</scope>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-srvutil</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.slf4j</groupId>\n            <artifactId>jul-to-slf4j</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>com.alipay.sofa</groupId>\n            <artifactId>jraft-core</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>com.google.protobuf</groupId>\n            <artifactId>protobuf-java-util</artifactId>\n        </dependency>\n    </dependencies>\n</project>\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/BrokerHeartbeatManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller;\n\nimport io.netty.channel.Channel;\nimport org.apache.rocketmq.common.ControllerConfig;\nimport org.apache.rocketmq.controller.helper.BrokerLifecycleListener;\nimport org.apache.rocketmq.controller.impl.heartbeat.BrokerLiveInfo;\nimport org.apache.rocketmq.controller.impl.heartbeat.DefaultBrokerHeartbeatManager;\nimport org.apache.rocketmq.controller.impl.heartbeat.RaftBrokerHeartBeatManager;\n\nimport java.util.Map;\n\npublic interface BrokerHeartbeatManager {\n    long DEFAULT_BROKER_CHANNEL_EXPIRED_TIME = 1000 * 10;\n\n    static BrokerHeartbeatManager newBrokerHeartbeatManager(ControllerConfig controllerConfig) {\n        if (controllerConfig.getControllerType().equals(ControllerConfig.JRAFT_CONTROLLER)) {\n            return new RaftBrokerHeartBeatManager(controllerConfig);\n        } else {\n            return new DefaultBrokerHeartbeatManager(controllerConfig);\n        }\n    }\n\n    /**\n     * Initialize the resources\n     */\n    void initialize();\n\n    /**\n     * Broker new heartbeat.\n     */\n    void onBrokerHeartbeat(final String clusterName, final String brokerName, final String brokerAddr,\n        final Long brokerId, final Long timeoutMillis, final Channel channel, final Integer epoch,\n        final Long maxOffset, final Long confirmOffset, final Integer electionPriority);\n\n    /**\n     * Start heartbeat manager.\n     */\n    void start();\n\n    /**\n     * Shutdown heartbeat manager.\n     */\n    void shutdown();\n\n    /**\n     * Add BrokerLifecycleListener.\n     */\n    void registerBrokerLifecycleListener(final BrokerLifecycleListener listener);\n\n    /**\n     * Broker channel close\n     */\n    void onBrokerChannelClose(final Channel channel);\n\n    /**\n     * Get broker live information by clusterName and brokerAddr\n     *\n     * @return broker live information or null if not found\n     */\n    BrokerLiveInfo getBrokerLiveInfo(String clusterName, String brokerName, Long brokerId);\n\n    /**\n     * Check whether broker active\n     */\n    boolean isBrokerActive(final String clusterName, final String brokerName, final Long brokerId);\n\n    /**\n     * Count the number of active brokers in each broker-set of each cluster\n     *\n     * @return active brokers count\n     */\n    Map<String/*cluster*/, Map<String/*broker-set*/, Integer/*active broker num*/>> getActiveBrokersNum();\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/BrokerHousekeepingService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller;\n\nimport io.netty.channel.Channel;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.ChannelEventListener;\n\npublic class BrokerHousekeepingService implements ChannelEventListener {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME);\n    private final ControllerManager controllerManager;\n\n    public BrokerHousekeepingService(ControllerManager controllerManager) {\n        this.controllerManager = controllerManager;\n    }\n\n    @Override\n    public void onChannelConnect(String remoteAddr, Channel channel) {\n    }\n\n    @Override\n    public void onChannelClose(String remoteAddr, Channel channel) {\n        this.controllerManager.getHeartbeatManager().onBrokerChannelClose(channel);\n    }\n\n    @Override\n    public void onChannelException(String remoteAddr, Channel channel) {\n        this.controllerManager.getHeartbeatManager().onBrokerChannelClose(channel);\n    }\n\n    @Override\n    public void onChannelIdle(String remoteAddr, Channel channel) {\n        this.controllerManager.getHeartbeatManager().onBrokerChannelClose(channel);\n    }\n\n    @Override\n    public void onChannelActive(String remoteAddr, Channel channel) {\n\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/Controller.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.controller;\n\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\n\nimport org.apache.rocketmq.controller.helper.BrokerLifecycleListener;\nimport org.apache.rocketmq.remoting.RemotingServer;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.body.SyncStateSet;\nimport org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerRequestHeader;\n\n/**\n * The api for controller\n */\npublic interface Controller {\n\n    /**\n     * Startup controller\n     */\n    void startup();\n\n    /**\n     * Shutdown controller\n     */\n    void shutdown();\n\n    /**\n     * Start scheduling controller events, this function only will be triggered when the controller becomes leader.\n     */\n    void startScheduling();\n\n    /**\n     * Stop scheduling controller events, this function only will be triggered when the controller lose leadership.\n     */\n    void stopScheduling();\n\n    /**\n     * Whether this controller is in leader state.\n     */\n    boolean isLeaderState();\n\n    /**\n     * Alter SyncStateSet of broker replicas.\n     *\n     * @param request AlterSyncStateSetRequestHeader\n     * @return RemotingCommand(AlterSyncStateSetResponseHeader)\n     */\n    CompletableFuture<RemotingCommand> alterSyncStateSet(\n        final AlterSyncStateSetRequestHeader request, final SyncStateSet syncStateSet);\n\n    /**\n     * Elect new master for a broker.\n     *\n     * @param request ElectMasterRequest\n     * @return RemotingCommand(ElectMasterResponseHeader)\n     */\n    CompletableFuture<RemotingCommand> electMaster(final ElectMasterRequestHeader request);\n\n    CompletableFuture<RemotingCommand> getNextBrokerId(final GetNextBrokerIdRequestHeader request);\n\n    CompletableFuture<RemotingCommand> applyBrokerId(final ApplyBrokerIdRequestHeader request);\n\n    /**\n     * Register broker with unique brokerId and now broker address\n     *\n     * @param request RegisterBrokerToControllerRequest\n     * @return RemotingCommand(RegisterBrokerToControllerResponseHeader)\n     */\n    CompletableFuture<RemotingCommand> registerBroker(final RegisterBrokerToControllerRequestHeader request);\n\n    /**\n     * Get the Replica Info for a target broker.\n     *\n     * @param request GetRouteInfoRequest\n     * @return RemotingCommand(GetReplicaInfoResponseHeader)\n     */\n    CompletableFuture<RemotingCommand> getReplicaInfo(final GetReplicaInfoRequestHeader request);\n\n    /**\n     * Get Metadata of controller\n     *\n     * @return RemotingCommand(GetControllerMetadataResponseHeader)\n     */\n    RemotingCommand getControllerMetadata();\n\n    /**\n     * Get SyncStateData for target brokers, this api is used for admin tools.\n     */\n    CompletableFuture<RemotingCommand> getSyncStateData(final List<String> brokerNames);\n\n    /**\n     * Add broker's lifecycle listener\n     * @param listener listener\n     */\n    void registerBrokerLifecycleListener(final BrokerLifecycleListener listener);\n\n    /**\n     * Get the remotingServer used by the controller, the upper layer will reuse this remotingServer.\n     */\n    RemotingServer getRemotingServer();\n\n    /**\n     * Clean controller broker data\n     *\n     */\n    CompletableFuture<RemotingCommand> cleanBrokerData(final CleanControllerBrokerDataRequestHeader requestHeader);\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/ControllerManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller;\n\nimport java.io.IOException;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.Future;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.ControllerConfig;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.controller.elect.impl.DefaultElectPolicy;\nimport org.apache.rocketmq.controller.impl.DLedgerController;\nimport org.apache.rocketmq.controller.impl.JRaftController;\nimport org.apache.rocketmq.controller.impl.heartbeat.RaftBrokerHeartBeatManager;\nimport org.apache.rocketmq.controller.metrics.ControllerMetricsManager;\nimport org.apache.rocketmq.controller.processor.ControllerRequestProcessor;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.Configuration;\nimport org.apache.rocketmq.remoting.RemotingClient;\nimport org.apache.rocketmq.remoting.RemotingServer;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingClient;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;\nimport org.apache.rocketmq.remoting.protocol.body.RoleChangeNotifyEntry;\nimport org.apache.rocketmq.remoting.protocol.body.SyncStateSet;\nimport org.apache.rocketmq.remoting.protocol.header.NotifyBrokerRoleChangedRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoResponseHeader;\n\npublic class ControllerManager {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.CONTROLLER_LOGGER_NAME);\n\n    private final ControllerConfig controllerConfig;\n    private final NettyServerConfig nettyServerConfig;\n    private final NettyClientConfig nettyClientConfig;\n    private final BrokerHousekeepingService brokerHousekeepingService;\n    private final Configuration configuration;\n    private final RemotingClient remotingClient;\n    private Controller controller;\n    private final BrokerHeartbeatManager heartbeatManager;\n    private ExecutorService controllerRequestExecutor;\n    private BlockingQueue<Runnable> controllerRequestThreadPoolQueue;\n    private final NotifyService notifyService;\n    private ControllerMetricsManager controllerMetricsManager;\n\n    public ControllerManager(ControllerConfig controllerConfig, NettyServerConfig nettyServerConfig,\n        NettyClientConfig nettyClientConfig) {\n        this.controllerConfig = controllerConfig;\n        this.nettyServerConfig = nettyServerConfig;\n        this.nettyClientConfig = nettyClientConfig;\n        this.brokerHousekeepingService = new BrokerHousekeepingService(this);\n        this.configuration = new Configuration(log, this.controllerConfig, this.nettyServerConfig);\n        this.configuration.setStorePathFromConfig(this.controllerConfig, \"configStorePath\");\n        this.remotingClient = new NettyRemotingClient(nettyClientConfig);\n        this.heartbeatManager = BrokerHeartbeatManager.newBrokerHeartbeatManager(controllerConfig);\n        this.notifyService = new NotifyService();\n    }\n\n    public boolean initialize() {\n        this.controllerRequestThreadPoolQueue = new LinkedBlockingQueue<>(this.controllerConfig.getControllerRequestThreadPoolQueueCapacity());\n        this.controllerRequestExecutor = ThreadUtils.newThreadPoolExecutor(\n            this.controllerConfig.getControllerThreadPoolNums(),\n            this.controllerConfig.getControllerThreadPoolNums(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            this.controllerRequestThreadPoolQueue,\n            new ThreadFactoryImpl(\"ControllerRequestExecutorThread_\"));\n\n        this.notifyService.initialize();\n\n        if (controllerConfig.getControllerType().equals(ControllerConfig.JRAFT_CONTROLLER)) {\n            if (StringUtils.isEmpty(this.controllerConfig.getJraftConfig().getjRaftInitConf())) {\n                throw new IllegalArgumentException(\"Attribute value jRaftInitConf of ControllerConfig is null or empty\");\n            }\n            if (StringUtils.isEmpty(this.controllerConfig.getJraftConfig().getjRaftServerId())) {\n                throw new IllegalArgumentException(\"Attribute value jRaftServerId of ControllerConfig is null or empty\");\n            }\n            try {\n                this.controller = new JRaftController(controllerConfig, this.brokerHousekeepingService);\n                ((RaftBrokerHeartBeatManager) this.heartbeatManager).setController((JRaftController) this.controller);\n            } catch (IOException e) {\n                throw new RuntimeException(e);\n            }\n        } else {\n            if (StringUtils.isEmpty(this.controllerConfig.getControllerDLegerPeers())) {\n                throw new IllegalArgumentException(\"Attribute value controllerDLegerPeers of ControllerConfig is null or empty\");\n            }\n            if (StringUtils.isEmpty(this.controllerConfig.getControllerDLegerSelfId())) {\n                throw new IllegalArgumentException(\"Attribute value controllerDLegerSelfId of ControllerConfig is null or empty\");\n            }\n            this.controller = new DLedgerController(this.controllerConfig, this.heartbeatManager::isBrokerActive,\n                this.nettyServerConfig, this.nettyClientConfig, this.brokerHousekeepingService,\n                new DefaultElectPolicy(this.heartbeatManager::isBrokerActive, this.heartbeatManager::getBrokerLiveInfo));\n        }\n\n        // Initialize the basic resources\n        this.heartbeatManager.initialize();\n\n        // Register broker inactive listener\n        this.heartbeatManager.registerBrokerLifecycleListener(this::onBrokerInactive);\n        this.controller.registerBrokerLifecycleListener(this::onBrokerInactive);\n        registerProcessor();\n        this.controllerMetricsManager = ControllerMetricsManager.getInstance(this);\n        return true;\n    }\n\n    /**\n     * When the heartbeatManager detects the \"Broker is not active\", we call this method to elect a master and do\n     * something else.\n     *\n     * @param clusterName The cluster name of this inactive broker\n     * @param brokerName  The inactive broker name\n     * @param brokerId    The inactive broker id, null means that the election forced to be triggered\n     */\n    private void onBrokerInactive(String clusterName, String brokerName, Long brokerId) {\n        log.info(\"Controller Manager received broker inactive event, clusterName: {}, brokerName: {}, brokerId: {}\",\n            clusterName, brokerName, brokerId);\n        if (controller.isLeaderState()) {\n            if (brokerId == null) {\n                // Means that force triggering election for this broker-set\n                triggerElectMaster(brokerName);\n                return;\n            }\n            final CompletableFuture<RemotingCommand> replicaInfoFuture = controller.getReplicaInfo(new GetReplicaInfoRequestHeader(brokerName));\n            replicaInfoFuture.whenCompleteAsync((replicaInfoResponse, err) -> {\n                if (err != null || replicaInfoResponse == null) {\n                    log.error(\"Failed to get replica-info for broker-set: {} when OnBrokerInactive\", brokerName, err);\n                    return;\n                }\n                final GetReplicaInfoResponseHeader replicaInfoResponseHeader = (GetReplicaInfoResponseHeader) replicaInfoResponse.readCustomHeader();\n                // Not master broker offline\n                if (!brokerId.equals(replicaInfoResponseHeader.getMasterBrokerId())) {\n                    log.warn(\"The broker with brokerId: {} in broker-set: {} has been inactive\", brokerId, brokerName);\n                    return;\n                }\n                // Trigger election\n                triggerElectMaster(brokerName);\n            });\n        } else {\n            log.warn(\"The broker with brokerId: {} in broker-set: {} has been inactive\", brokerId, brokerName);\n        }\n    }\n\n    private CompletableFuture<Boolean> triggerElectMaster0(String brokerName) {\n        final CompletableFuture<RemotingCommand> electMasterFuture = controller.electMaster(ElectMasterRequestHeader.ofControllerTrigger(brokerName));\n        return electMasterFuture.handleAsync((electMasterResponse, err) -> {\n            if (err != null || electMasterResponse == null || electMasterResponse.getCode() != ResponseCode.SUCCESS) {\n                log.error(\"Failed to trigger elect-master in broker-set: {}\", brokerName, err);\n                return false;\n            }\n            if (electMasterResponse.getCode() == ResponseCode.SUCCESS) {\n                log.info(\"Elect a new master in broker-set: {} done, result: {}\", brokerName, electMasterResponse);\n                if (controllerConfig.isNotifyBrokerRoleChanged()) {\n                    notifyBrokerRoleChanged(RoleChangeNotifyEntry.convert(electMasterResponse));\n                }\n                return true;\n            }\n            //default is false\n            return false;\n        });\n    }\n\n    private void triggerElectMaster(String brokerName) {\n        int maxRetryCount = controllerConfig.getElectMasterMaxRetryCount();\n        for (int i = 0; i < maxRetryCount; i++) {\n            try {\n                Boolean electResult = triggerElectMaster0(brokerName).get(3, TimeUnit.SECONDS);\n                if (electResult) {\n                    return;\n                }\n            } catch (Exception e) {\n                log.warn(\"Failed to trigger elect-master in broker-set: {}, retryCount: {}\", brokerName, i, e);\n            }\n        }\n    }\n\n    /**\n     * Notify master and all slaves for a broker that the master role changed.\n     */\n    public void notifyBrokerRoleChanged(final RoleChangeNotifyEntry entry) {\n        final BrokerMemberGroup memberGroup = entry.getBrokerMemberGroup();\n        if (memberGroup != null) {\n            final Long masterBrokerId = entry.getMasterBrokerId();\n            String clusterName = memberGroup.getCluster();\n            String brokerName = memberGroup.getBrokerName();\n            if (masterBrokerId == null) {\n                log.warn(\"Notify broker role change failed, because member group is not null but the new master brokerId is empty, entry:{}\", entry);\n                return;\n            }\n            // Inform all active brokers\n            final Map<Long, String> brokerAddrs = memberGroup.getBrokerAddrs();\n            brokerAddrs.entrySet().stream().filter(x -> this.heartbeatManager.isBrokerActive(clusterName, brokerName, x.getKey()))\n                .forEach(x -> this.notifyService.notifyBroker(x.getValue(), entry));\n        }\n    }\n\n    /**\n     * Notify broker that there are roles-changing in controller\n     *\n     * @param brokerAddr target broker's address to notify\n     * @param entry      role change entry\n     */\n    public void doNotifyBrokerRoleChanged(final String brokerAddr, final RoleChangeNotifyEntry entry) {\n        if (StringUtils.isNoneEmpty(brokerAddr)) {\n            log.info(\"Try notify broker {} that role changed, RoleChangeNotifyEntry:{}\", brokerAddr, entry);\n            final NotifyBrokerRoleChangedRequestHeader requestHeader = new NotifyBrokerRoleChangedRequestHeader(entry.getMasterAddress(), entry.getMasterBrokerId(),\n                entry.getMasterEpoch(), entry.getSyncStateSetEpoch());\n            final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.NOTIFY_BROKER_ROLE_CHANGED, requestHeader);\n            request.setBody(new SyncStateSet(entry.getSyncStateSet(), entry.getSyncStateSetEpoch()).encode());\n            try {\n                this.remotingClient.invokeOneway(brokerAddr, request, 3000);\n            } catch (final Exception e) {\n                log.error(\"Failed to notify broker {} that role changed\", brokerAddr, e);\n            }\n        }\n    }\n\n    public void registerProcessor() {\n        final ControllerRequestProcessor controllerRequestProcessor = new ControllerRequestProcessor(this);\n        RemotingServer controllerRemotingServer = this.controller.getRemotingServer();\n        assert controllerRemotingServer != null;\n        controllerRemotingServer.registerProcessor(RequestCode.CONTROLLER_ALTER_SYNC_STATE_SET, controllerRequestProcessor, this.controllerRequestExecutor);\n        controllerRemotingServer.registerProcessor(RequestCode.CONTROLLER_ELECT_MASTER, controllerRequestProcessor, this.controllerRequestExecutor);\n        controllerRemotingServer.registerProcessor(RequestCode.CONTROLLER_REGISTER_BROKER, controllerRequestProcessor, this.controllerRequestExecutor);\n        controllerRemotingServer.registerProcessor(RequestCode.CONTROLLER_GET_REPLICA_INFO, controllerRequestProcessor, this.controllerRequestExecutor);\n        controllerRemotingServer.registerProcessor(RequestCode.CONTROLLER_GET_METADATA_INFO, controllerRequestProcessor, this.controllerRequestExecutor);\n        controllerRemotingServer.registerProcessor(RequestCode.CONTROLLER_GET_SYNC_STATE_DATA, controllerRequestProcessor, this.controllerRequestExecutor);\n        controllerRemotingServer.registerProcessor(RequestCode.BROKER_HEARTBEAT, controllerRequestProcessor, this.controllerRequestExecutor);\n        controllerRemotingServer.registerProcessor(RequestCode.UPDATE_CONTROLLER_CONFIG, controllerRequestProcessor, this.controllerRequestExecutor);\n        controllerRemotingServer.registerProcessor(RequestCode.GET_CONTROLLER_CONFIG, controllerRequestProcessor, this.controllerRequestExecutor);\n        controllerRemotingServer.registerProcessor(RequestCode.CLEAN_BROKER_DATA, controllerRequestProcessor, this.controllerRequestExecutor);\n        controllerRemotingServer.registerProcessor(RequestCode.CONTROLLER_GET_NEXT_BROKER_ID, controllerRequestProcessor, this.controllerRequestExecutor);\n        controllerRemotingServer.registerProcessor(RequestCode.CONTROLLER_APPLY_BROKER_ID, controllerRequestProcessor, this.controllerRequestExecutor);\n    }\n\n    public void start() {\n        this.controller.startup();\n        this.heartbeatManager.start();\n        this.remotingClient.start();\n    }\n\n    public void shutdown() {\n        this.heartbeatManager.shutdown();\n        this.controllerRequestExecutor.shutdown();\n        this.notifyService.shutdown();\n        this.controller.shutdown();\n        this.remotingClient.shutdown();\n    }\n\n    public BrokerHeartbeatManager getHeartbeatManager() {\n        return heartbeatManager;\n    }\n\n    public ControllerConfig getControllerConfig() {\n        return controllerConfig;\n    }\n\n    public Controller getController() {\n        return controller;\n    }\n\n    public NettyServerConfig getNettyServerConfig() {\n        return nettyServerConfig;\n    }\n\n    public NettyClientConfig getNettyClientConfig() {\n        return nettyClientConfig;\n    }\n\n    public BrokerHousekeepingService getBrokerHousekeepingService() {\n        return brokerHousekeepingService;\n    }\n\n    public Configuration getConfiguration() {\n        return configuration;\n    }\n\n    class NotifyService {\n        private ExecutorService executorService;\n\n        private Map<String/*brokerAddress*/, NotifyTask/*currentNotifyTask*/> currentNotifyFutures;\n\n        public NotifyService() {\n        }\n\n        public void initialize() {\n            this.executorService = Executors.newFixedThreadPool(3, new ThreadFactoryImpl(\"ControllerManager_NotifyService_\"));\n            this.currentNotifyFutures = new ConcurrentHashMap<>();\n        }\n\n        public void notifyBroker(String brokerAddress, RoleChangeNotifyEntry entry) {\n            int masterEpoch = entry.getMasterEpoch();\n            NotifyTask oldTask = this.currentNotifyFutures.get(brokerAddress);\n            if (oldTask != null && masterEpoch > oldTask.getMasterEpoch()) {\n                // cancel current future\n                Future oldFuture = oldTask.getFuture();\n                if (oldFuture != null && !oldFuture.isDone()) {\n                    oldFuture.cancel(true);\n                }\n            }\n            final NotifyTask task = new NotifyTask(masterEpoch, null);\n            Runnable runnable = () -> {\n                doNotifyBrokerRoleChanged(brokerAddress, entry);\n                this.currentNotifyFutures.remove(brokerAddress, task);\n            };\n            this.currentNotifyFutures.put(brokerAddress, task);\n            Future<?> future = this.executorService.submit(runnable);\n            task.setFuture(future);\n        }\n\n        public void shutdown() {\n            if (!this.executorService.isShutdown()) {\n                this.executorService.shutdownNow();\n            }\n        }\n\n        class NotifyTask extends Pair<Integer/*epochMaster*/, Future/*notifyFuture*/> {\n            public NotifyTask(Integer masterEpoch, Future future) {\n                super(masterEpoch, future);\n            }\n\n            public Integer getMasterEpoch() {\n                return super.getObject1();\n            }\n\n            public Future getFuture() {\n                return super.getObject2();\n            }\n\n            public void setFuture(Future future) {\n                super.setObject2(future);\n            }\n\n            @Override\n            public int hashCode() {\n                return Objects.hashCode(super.getObject1());\n            }\n\n            @Override\n            public boolean equals(Object obj) {\n                if (this == obj) {\n                    return true;\n                }\n                if (!(obj instanceof NotifyTask)) {\n                    return false;\n                }\n                NotifyTask task = (NotifyTask) obj;\n                return super.getObject1().equals(task.getObject1());\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/ControllerStartup.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller;\n\nimport java.io.BufferedInputStream;\nimport java.io.FileInputStream;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.util.Properties;\nimport java.util.concurrent.Callable;\nimport org.apache.commons.cli.CommandLine;\nimport org.apache.commons.cli.DefaultParser;\nimport org.apache.commons.cli.Option;\nimport org.apache.commons.cli.Options;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.ControllerConfig;\nimport org.apache.rocketmq.common.JraftConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.srvutil.ServerUtil;\nimport org.apache.rocketmq.srvutil.ShutdownHookThread;\n\npublic class ControllerStartup {\n\n    private static Logger log;\n    private static Properties properties = null;\n    private static CommandLine commandLine = null;\n\n    public static void main(String[] args) {\n        main0(args);\n    }\n\n    public static ControllerManager main0(String[] args) {\n\n        try {\n            ControllerManager controller = createControllerManager(args);\n            start(controller);\n            String tip = \"The Controller Server boot success. serializeType=\" + RemotingCommand.getSerializeTypeConfigInThisServer();\n            log.info(tip);\n            System.out.printf(\"%s%n\", tip);\n            return controller;\n        } catch (Throwable e) {\n            e.printStackTrace();\n            System.exit(-1);\n        }\n\n        return null;\n    }\n\n    public static ControllerManager createControllerManager(String[] args) throws IOException {\n        Options options = ServerUtil.buildCommandlineOptions(new Options());\n        commandLine = ServerUtil.parseCmdLine(\"mqcontroller\", args, buildCommandlineOptions(options), new DefaultParser());\n        if (null == commandLine) {\n            System.exit(-1);\n            return null;\n        }\n\n        final ControllerConfig controllerConfig = new ControllerConfig();\n        final JraftConfig jraftConfig = new JraftConfig();\n        controllerConfig.setJraftConfig(jraftConfig);\n        final NettyServerConfig nettyServerConfig = new NettyServerConfig();\n        final NettyClientConfig nettyClientConfig = new NettyClientConfig();\n        nettyServerConfig.setListenPort(19876);\n\n        if (commandLine.hasOption('c')) {\n            String file = commandLine.getOptionValue('c');\n            if (file != null) {\n                InputStream in = new BufferedInputStream(new FileInputStream(file));\n                properties = new Properties();\n                properties.load(in);\n                MixAll.properties2Object(properties, controllerConfig);\n                MixAll.properties2Object(properties, jraftConfig);\n                MixAll.properties2Object(properties, nettyServerConfig);\n                MixAll.properties2Object(properties, nettyClientConfig);\n\n                System.out.printf(\"load config properties file OK, %s%n\", file);\n                in.close();\n            }\n        }\n\n        if (commandLine.hasOption('p')) {\n            Logger console = LoggerFactory.getLogger(LoggerName.CONTROLLER_CONSOLE_NAME);\n            MixAll.printObjectProperties(console, controllerConfig);\n            MixAll.printObjectProperties(console, jraftConfig);\n            MixAll.printObjectProperties(console, nettyServerConfig);\n            MixAll.printObjectProperties(console, nettyClientConfig);\n            System.exit(0);\n        }\n\n        MixAll.properties2Object(ServerUtil.commandLine2Properties(commandLine), controllerConfig);\n\n        if (StringUtils.isEmpty(controllerConfig.getRocketmqHome())) {\n            System.out.printf(\"Please set the %s or %s variable in your environment!%n\", MixAll.ROCKETMQ_HOME_ENV, MixAll.ROCKETMQ_HOME_PROPERTY);\n            System.exit(-1);\n        }\n\n        log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME);\n\n        MixAll.printObjectProperties(log, controllerConfig);\n        MixAll.printObjectProperties(log, nettyServerConfig);\n\n        final ControllerManager controllerManager = new ControllerManager(controllerConfig, nettyServerConfig, nettyClientConfig);\n        // remember all configs to prevent discard\n        controllerManager.getConfiguration().registerConfig(properties);\n\n        return controllerManager;\n    }\n\n    public static ControllerManager start(final ControllerManager controller) throws Exception {\n\n        if (null == controller) {\n            throw new IllegalArgumentException(\"ControllerManager is null\");\n        }\n\n        boolean initResult = controller.initialize();\n        if (!initResult) {\n            controller.shutdown();\n            System.exit(-3);\n        }\n\n        Runtime.getRuntime().addShutdownHook(new ShutdownHookThread(log, (Callable<Void>) () -> {\n            controller.shutdown();\n            return null;\n        }));\n\n        controller.start();\n\n        return controller;\n    }\n\n    public static void shutdown(final ControllerManager controller) {\n        controller.shutdown();\n    }\n\n    public static Options buildCommandlineOptions(final Options options) {\n        Option opt = new Option(\"c\", \"configFile\", true, \"Controller config properties file\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"p\", \"printConfigItem\", false, \"Print all config items\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        return options;\n    }\n\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/elect/ElectPolicy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.elect;\n\nimport java.util.Set;\n\npublic interface ElectPolicy {\n\n    /**\n     * elect a master\n     *\n     * @param clusterName       the broker group belongs to\n     * @param brokerName        the broker group name\n     * @param syncStateBrokers  all broker replicas in syncStateSet\n     * @param allReplicaBrokers all broker replicas\n     * @param oldMaster         old master\n     * @param brokerId          broker id(can be used as prefer or assigned in some elect policy)\n     * @return new master's broker id\n     */\n    Long elect(String clusterName, String brokerName, Set<Long> syncStateBrokers, Set<Long> allReplicaBrokers,\n        Long oldMaster, Long brokerId);\n\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/elect/impl/DefaultElectPolicy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.elect.impl;\n\nimport org.apache.rocketmq.controller.elect.ElectPolicy;\nimport org.apache.rocketmq.controller.impl.heartbeat.BrokerLiveInfo;\nimport org.apache.rocketmq.controller.helper.BrokerLiveInfoGetter;\nimport org.apache.rocketmq.controller.helper.BrokerValidPredicate;\n\nimport java.util.Comparator;\nimport java.util.Set;\nimport java.util.TreeSet;\nimport java.util.stream.Collectors;\n\npublic class DefaultElectPolicy implements ElectPolicy {\n\n    // <clusterName, brokerName, brokerAddr>, Used to judge whether a broker\n    // has preliminary qualification to be selected as master\n    private BrokerValidPredicate validPredicate;\n\n    // <clusterName, brokerName, brokerAddr, BrokerLiveInfo>, Used to obtain the BrokerLiveInfo information of a broker\n    private BrokerLiveInfoGetter brokerLiveInfoGetter;\n\n    // Sort in descending order according to<epoch, offset>, and sort in ascending order according to priority\n    private final Comparator<BrokerLiveInfo> comparator = (o1, o2) -> {\n        if (o1.getEpoch() == o2.getEpoch()) {\n            return o1.getMaxOffset() == o2.getMaxOffset() ? o1.getElectionPriority() - o2.getElectionPriority() :\n                (int) (o2.getMaxOffset() - o1.getMaxOffset());\n        } else {\n            return o2.getEpoch() - o1.getEpoch();\n        }\n    };\n\n    public DefaultElectPolicy(BrokerValidPredicate validPredicate, BrokerLiveInfoGetter brokerLiveInfoGetter) {\n        this.validPredicate = validPredicate;\n        this.brokerLiveInfoGetter = brokerLiveInfoGetter;\n    }\n\n    public DefaultElectPolicy() {\n\n    }\n\n    /**\n     * We will try to select a new master from syncStateBrokers and allReplicaBrokers in turn.\n     * The strategies are as follows:\n     *    - Filter alive brokers by 'validPredicate'.\n     *    - Check whether the old master is still valid.\n     *    - If preferBrokerAddr is not empty and valid, select it as master.\n     *    - Otherwise, we will sort the array of 'brokerLiveInfo' according to (epoch, offset, electionPriority), and select the best candidate as the new master.\n     *\n     * @param clusterName       the brokerGroup belongs\n     * @param syncStateBrokers  all broker replicas in syncStateSet\n     * @param allReplicaBrokers all broker replicas\n     * @param oldMaster         old master's broker id\n     * @param preferBrokerId    the broker id prefer to be elected\n     * @return master elected by our own policy\n     */\n    @Override\n    public Long elect(String clusterName, String brokerName, Set<Long> syncStateBrokers, Set<Long> allReplicaBrokers,\n        Long oldMaster, Long preferBrokerId) {\n        Long newMaster = null;\n        // try to elect in syncStateBrokers\n        if (syncStateBrokers != null) {\n            newMaster = tryElect(clusterName, brokerName, syncStateBrokers, oldMaster, preferBrokerId);\n        }\n        if (newMaster != null) {\n            return newMaster;\n        }\n\n        // try to elect in all allReplicaBrokers\n        if (allReplicaBrokers != null) {\n            newMaster = tryElect(clusterName, brokerName, allReplicaBrokers, oldMaster, preferBrokerId);\n        }\n        return newMaster;\n    }\n\n    private Long tryElect(String clusterName, String brokerName, Set<Long> brokers, Long oldMaster,\n        Long preferBrokerId) {\n        if (this.validPredicate != null) {\n            brokers = brokers.stream().filter(brokerAddr -> this.validPredicate.check(clusterName, brokerName, brokerAddr)).collect(Collectors.toSet());\n        }\n        if (!brokers.isEmpty()) {\n            // if old master is still valid, and preferBrokerAddr is blank or is equals to oldMaster\n            if (brokers.contains(oldMaster) && (preferBrokerId == null || preferBrokerId.equals(oldMaster))) {\n                return oldMaster;\n            }\n\n            // if preferBrokerAddr is valid, we choose it, otherwise we choose nothing\n            if (preferBrokerId != null) {\n                return brokers.contains(preferBrokerId) ? preferBrokerId : null;\n            }\n\n            if (this.brokerLiveInfoGetter != null) {\n                // sort brokerLiveInfos by (epoch,maxOffset)\n                TreeSet<BrokerLiveInfo> brokerLiveInfos = new TreeSet<>(this.comparator);\n                brokers.forEach(brokerAddr -> brokerLiveInfos.add(this.brokerLiveInfoGetter.get(clusterName, brokerName, brokerAddr)));\n                if (brokerLiveInfos.size() >= 1) {\n                    return brokerLiveInfos.first().getBrokerId();\n                }\n            }\n            // elect random\n            return brokers.iterator().next();\n        }\n        return null;\n    }\n\n\n    public void setBrokerLiveInfoGetter(BrokerLiveInfoGetter brokerLiveInfoGetter) {\n        this.brokerLiveInfoGetter = brokerLiveInfoGetter;\n    }\n\n    public void setValidPredicate(BrokerValidPredicate validPredicate) {\n        this.validPredicate = validPredicate;\n    }\n\n    public BrokerLiveInfoGetter getBrokerLiveInfoGetter() {\n        return brokerLiveInfoGetter;\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/helper/BrokerLifecycleListener.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.controller.helper;\n\npublic interface BrokerLifecycleListener {\n    /**\n     * Trigger when broker inactive.\n     */\n    void onBrokerInactive(final String clusterName, final String brokerName, final Long brokerId);\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/helper/BrokerLiveInfoGetter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.controller.helper;\n\nimport org.apache.rocketmq.controller.impl.heartbeat.BrokerLiveInfo;\n\npublic interface BrokerLiveInfoGetter {\n\n    BrokerLiveInfo get(String clusterName, String brokerName, Long brokerId);\n\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/helper/BrokerValidPredicate.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.helper;\n\npublic interface BrokerValidPredicate {\n\n    boolean check(String clusterName, String brokerName, Long brokerId);\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl;\n\nimport com.google.common.base.Stopwatch;\nimport io.openmessaging.storage.dledger.AppendFuture;\nimport io.openmessaging.storage.dledger.DLedgerConfig;\nimport io.openmessaging.storage.dledger.DLedgerLeaderElector;\nimport io.openmessaging.storage.dledger.DLedgerServer;\nimport io.openmessaging.storage.dledger.MemberState;\nimport io.openmessaging.storage.dledger.protocol.AppendEntryRequest;\nimport io.openmessaging.storage.dledger.protocol.AppendEntryResponse;\nimport io.openmessaging.storage.dledger.protocol.BatchAppendEntryRequest;\nimport io.opentelemetry.api.common.AttributesBuilder;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ScheduledFuture;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.TimeoutException;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.function.Supplier;\nimport org.apache.rocketmq.common.ControllerConfig;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.controller.Controller;\nimport org.apache.rocketmq.controller.elect.ElectPolicy;\nimport org.apache.rocketmq.controller.elect.impl.DefaultElectPolicy;\nimport org.apache.rocketmq.controller.helper.BrokerLifecycleListener;\nimport org.apache.rocketmq.controller.helper.BrokerValidPredicate;\nimport org.apache.rocketmq.controller.impl.event.ControllerResult;\nimport org.apache.rocketmq.controller.impl.event.EventMessage;\nimport org.apache.rocketmq.controller.impl.event.EventSerializer;\nimport org.apache.rocketmq.controller.impl.manager.ReplicasInfoManager;\nimport org.apache.rocketmq.controller.metrics.ControllerMetricsConstant;\nimport org.apache.rocketmq.controller.metrics.ControllerMetricsManager;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.ChannelEventListener;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.RemotingServer;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.SyncStateSet;\nimport org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerRequestHeader;\n\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.LABEL_BROKER_SET;\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.LABEL_CLUSTER_NAME;\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.LABEL_DLEDGER_OPERATION;\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.LABEL_DLEDGER_OPERATION_STATUS;\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.LABEL_ELECTION_RESULT;\n\n/**\n * The implementation of controller, based on DLedger (raft).\n */\npublic class DLedgerController implements Controller {\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.CONTROLLER_LOGGER_NAME);\n    private final DLedgerServer dLedgerServer;\n    private final ControllerConfig controllerConfig;\n    private final DLedgerConfig dLedgerConfig;\n    private final ReplicasInfoManager replicasInfoManager;\n    private final EventScheduler scheduler;\n    private final EventSerializer eventSerializer;\n    private final RoleChangeHandler roleHandler;\n    private final DLedgerControllerStateMachine statemachine;\n    private final ScheduledExecutorService scanInactiveMasterService;\n\n    private ScheduledFuture scanInactiveMasterFuture;\n\n    private final List<BrokerLifecycleListener> brokerLifecycleListeners;\n\n    // use for checking whether the broker is alive\n    private BrokerValidPredicate brokerAlivePredicate;\n    // use for elect a master\n    private ElectPolicy electPolicy;\n\n    private final AtomicBoolean isScheduling = new AtomicBoolean(false);\n\n    public DLedgerController(final ControllerConfig config, final BrokerValidPredicate brokerAlivePredicate) {\n        this(config, brokerAlivePredicate, null, null, null, null);\n    }\n\n    public DLedgerController(final ControllerConfig controllerConfig,\n        final BrokerValidPredicate brokerAlivePredicate, final NettyServerConfig nettyServerConfig,\n        final NettyClientConfig nettyClientConfig, final ChannelEventListener channelEventListener,\n        final ElectPolicy electPolicy) {\n        this.controllerConfig = controllerConfig;\n        this.eventSerializer = new EventSerializer();\n        this.scheduler = new EventScheduler();\n        this.brokerAlivePredicate = brokerAlivePredicate;\n        this.electPolicy = electPolicy == null ? new DefaultElectPolicy() : electPolicy;\n        this.dLedgerConfig = new DLedgerConfig();\n        this.dLedgerConfig.setGroup(controllerConfig.getControllerDLegerGroup());\n        this.dLedgerConfig.setPeers(controllerConfig.getControllerDLegerPeers());\n        this.dLedgerConfig.setSelfId(controllerConfig.getControllerDLegerSelfId());\n        this.dLedgerConfig.setStoreBaseDir(controllerConfig.getControllerStorePath());\n        this.dLedgerConfig.setMappedFileSizeForEntryData(controllerConfig.getMappedFileSize());\n\n        this.roleHandler = new RoleChangeHandler(dLedgerConfig.getSelfId());\n        this.replicasInfoManager = new ReplicasInfoManager(controllerConfig);\n        this.statemachine = new DLedgerControllerStateMachine(replicasInfoManager, this.eventSerializer, dLedgerConfig.getGroup(), dLedgerConfig.getSelfId());\n\n        // Register statemachine and role handler.\n        this.dLedgerServer = new DLedgerServer(dLedgerConfig, nettyServerConfig, nettyClientConfig, channelEventListener);\n        this.dLedgerServer.registerStateMachine(this.statemachine);\n        this.dLedgerServer.getDLedgerLeaderElector().addRoleChangeHandler(this.roleHandler);\n        this.scanInactiveMasterService = ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(\"DLedgerController_scanInactiveService_\"));\n        this.brokerLifecycleListeners = new ArrayList<>();\n    }\n\n    @Override\n    public void startup() {\n        this.dLedgerServer.startup();\n    }\n\n    @Override\n    public void shutdown() {\n        this.cancelScanInactiveFuture();\n        this.dLedgerServer.shutdown();\n    }\n\n    @Override\n    public void startScheduling() {\n        if (this.isScheduling.compareAndSet(false, true)) {\n            log.info(\"Start scheduling controller events\");\n            this.scheduler.start();\n        }\n    }\n\n    @Override\n    public void stopScheduling() {\n        if (this.isScheduling.compareAndSet(true, false)) {\n            log.info(\"Stop scheduling controller events\");\n            this.scheduler.shutdown(true);\n        }\n    }\n\n    @Override\n    public boolean isLeaderState() {\n        return this.roleHandler.isLeaderState();\n    }\n\n    public ControllerConfig getControllerConfig() {\n        return controllerConfig;\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> alterSyncStateSet(AlterSyncStateSetRequestHeader request,\n        final SyncStateSet syncStateSet) {\n        return this.scheduler.appendEvent(\"alterSyncStateSet\",\n            () -> this.replicasInfoManager.alterSyncStateSet(request, syncStateSet, this.brokerAlivePredicate), true);\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> electMaster(final ElectMasterRequestHeader request) {\n        return this.scheduler.appendEvent(\"electMaster\",\n            () -> {\n                ControllerResult<ElectMasterResponseHeader> electResult = this.replicasInfoManager.electMaster(request, this.electPolicy);\n                AttributesBuilder attributesBuilder = ControllerMetricsManager.newAttributesBuilder()\n                    .put(LABEL_CLUSTER_NAME, request.getClusterName())\n                    .put(LABEL_BROKER_SET, request.getBrokerName());\n                switch (electResult.getResponseCode()) {\n                    case ResponseCode.SUCCESS:\n                        ControllerMetricsManager.electionTotal.add(1,\n                            attributesBuilder.put(LABEL_ELECTION_RESULT, ControllerMetricsConstant.ElectionResult.NEW_MASTER_ELECTED.getLowerCaseName()).build());\n                        break;\n                    case ResponseCode.CONTROLLER_MASTER_STILL_EXIST:\n                        ControllerMetricsManager.electionTotal.add(1,\n                            attributesBuilder.put(LABEL_ELECTION_RESULT, ControllerMetricsConstant.ElectionResult.KEEP_CURRENT_MASTER.getLowerCaseName()).build());\n                        break;\n                    case ResponseCode.CONTROLLER_MASTER_NOT_AVAILABLE:\n                    case ResponseCode.CONTROLLER_ELECT_MASTER_FAILED:\n                        ControllerMetricsManager.electionTotal.add(1,\n                            attributesBuilder.put(LABEL_ELECTION_RESULT, ControllerMetricsConstant.ElectionResult.NO_MASTER_ELECTED.getLowerCaseName()).build());\n                        break;\n                    default:\n                        break;\n                }\n                return electResult;\n            }, true);\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> getNextBrokerId(GetNextBrokerIdRequestHeader request) {\n        return this.scheduler.appendEvent(\"getNextBrokerId\", () -> this.replicasInfoManager.getNextBrokerId(request), false);\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> applyBrokerId(ApplyBrokerIdRequestHeader request) {\n        return this.scheduler.appendEvent(\"applyBrokerId\", () -> this.replicasInfoManager.applyBrokerId(request), true);\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> registerBroker(RegisterBrokerToControllerRequestHeader request) {\n        return this.scheduler.appendEvent(\"registerSuccess\", () -> this.replicasInfoManager.registerBroker(request, brokerAlivePredicate), true);\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> getReplicaInfo(final GetReplicaInfoRequestHeader request) {\n        return this.scheduler.appendEvent(\"getReplicaInfo\",\n            () -> this.replicasInfoManager.getReplicaInfo(request), false);\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> getSyncStateData(List<String> brokerNames) {\n        return this.scheduler.appendEvent(\"getSyncStateData\",\n            () -> this.replicasInfoManager.getSyncStateData(brokerNames, brokerAlivePredicate), false);\n    }\n\n    @Override\n    public void registerBrokerLifecycleListener(BrokerLifecycleListener listener) {\n        this.brokerLifecycleListeners.add(listener);\n    }\n\n    @Override\n    public RemotingCommand getControllerMetadata() {\n        final MemberState state = getMemberState();\n        final Map<String, String> peers = state.getPeerMap();\n        final StringBuilder sb = new StringBuilder();\n        for (Map.Entry<String, String> entry : peers.entrySet()) {\n            final String peer = entry.getKey() + \":\" + entry.getValue();\n            sb.append(peer).append(\";\");\n        }\n        return RemotingCommand.createResponseCommandWithHeader(ResponseCode.SUCCESS, new GetMetaDataResponseHeader(\n            state.getGroup(), state.getLeaderId(), state.getLeaderAddr(), state.isLeader(), sb.toString()));\n    }\n\n    @Override\n    public RemotingServer getRemotingServer() {\n        return this.dLedgerServer.getRemotingServer();\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> cleanBrokerData(\n        final CleanControllerBrokerDataRequestHeader requestHeader) {\n        return this.scheduler.appendEvent(\"cleanBrokerData\",\n            () -> this.replicasInfoManager.cleanBrokerData(requestHeader, this.brokerAlivePredicate), true);\n    }\n\n    /**\n     * Scan all broker-set in statemachine, find that the broker-set which\n     * its master has been timeout but still has at least one broker keep alive with controller,\n     * and we trigger an election to update its state.\n     */\n    private void scanInactiveMasterAndTriggerReelect() {\n        if (!this.roleHandler.isLeaderState()) {\n            cancelScanInactiveFuture();\n            return;\n        }\n        List<String> brokerSets = this.replicasInfoManager.scanNeedReelectBrokerSets(this.brokerAlivePredicate);\n        for (String brokerName : brokerSets) {\n            // Notify ControllerManager\n            this.brokerLifecycleListeners.forEach(listener -> listener.onBrokerInactive(null, brokerName, null));\n        }\n    }\n\n    /**\n     * Append the request to DLedger, and wait for DLedger to commit the request.\n     */\n    private boolean appendToDLedgerAndWait(final AppendEntryRequest request) {\n        if (request != null) {\n            request.setGroup(this.dLedgerConfig.getGroup());\n            request.setRemoteId(this.dLedgerConfig.getSelfId());\n            Stopwatch stopwatch = Stopwatch.createStarted();\n            AttributesBuilder attributesBuilder = ControllerMetricsManager.newAttributesBuilder()\n                .put(LABEL_DLEDGER_OPERATION, ControllerMetricsConstant.DLedgerOperation.APPEND.getLowerCaseName());\n            try {\n                final AppendFuture<AppendEntryResponse> dLedgerFuture = (AppendFuture<AppendEntryResponse>) dLedgerServer.handleAppend(request);\n                if (dLedgerFuture.getPos() == -1) {\n                    ControllerMetricsManager.dLedgerOpTotal.add(1,\n                        attributesBuilder.put(LABEL_DLEDGER_OPERATION_STATUS, ControllerMetricsConstant.DLedgerOperationStatus.FAILED.getLowerCaseName()).build());\n                    return false;\n                }\n                dLedgerFuture.get(5, TimeUnit.SECONDS);\n                ControllerMetricsManager.dLedgerOpTotal.add(1,\n                    attributesBuilder.put(LABEL_DLEDGER_OPERATION_STATUS, ControllerMetricsConstant.DLedgerOperationStatus.SUCCESS.getLowerCaseName()).build());\n                ControllerMetricsManager.dLedgerOpLatency.record(stopwatch.elapsed(TimeUnit.MICROSECONDS),\n                    attributesBuilder.build());\n            } catch (Exception e) {\n                log.error(\"Failed to append entry to DLedger\", e);\n                if (e instanceof TimeoutException) {\n                    ControllerMetricsManager.dLedgerOpTotal.add(1,\n                        attributesBuilder.put(LABEL_DLEDGER_OPERATION_STATUS, ControllerMetricsConstant.DLedgerOperationStatus.TIMEOUT.getLowerCaseName()).build());\n                } else {\n                    ControllerMetricsManager.dLedgerOpTotal.add(1,\n                        attributesBuilder.put(LABEL_DLEDGER_OPERATION_STATUS, ControllerMetricsConstant.DLedgerOperationStatus.FAILED.getLowerCaseName()).build());\n                }\n                return false;\n            }\n            return true;\n        }\n        return false;\n    }\n\n    // Only for test\n    public MemberState getMemberState() {\n        return this.dLedgerServer.getMemberState();\n    }\n\n    public void setBrokerAlivePredicate(BrokerValidPredicate brokerAlivePredicate) {\n        this.brokerAlivePredicate = brokerAlivePredicate;\n    }\n\n    public void setElectPolicy(ElectPolicy electPolicy) {\n        this.electPolicy = electPolicy;\n    }\n\n    private void cancelScanInactiveFuture() {\n        if (this.scanInactiveMasterFuture != null) {\n            this.scanInactiveMasterFuture.cancel(true);\n            this.scanInactiveMasterFuture = null;\n        }\n    }\n\n    /**\n     * Event handler that handle event\n     */\n    interface EventHandler<T> {\n        /**\n         * Run the controller event\n         */\n        void run() throws Throwable;\n\n        /**\n         * Return the completableFuture\n         */\n        CompletableFuture<RemotingCommand> future();\n\n        /**\n         * Handle Exception.\n         */\n        void handleException(final Throwable t);\n    }\n\n    /**\n     * Event scheduler, schedule event handler from event queue\n     */\n    class EventScheduler extends ServiceThread {\n        private final BlockingQueue<EventHandler> eventQueue;\n\n        public EventScheduler() {\n            this.eventQueue = new LinkedBlockingQueue<>(1024);\n        }\n\n        @Override\n        public String getServiceName() {\n            return EventScheduler.class.getName();\n        }\n\n        @Override\n        public void run() {\n            log.info(\"Start event scheduler.\");\n            while (!isStopped()) {\n                EventHandler handler;\n                try {\n                    handler = this.eventQueue.poll(5, TimeUnit.SECONDS);\n                } catch (final InterruptedException e) {\n                    continue;\n                }\n                try {\n                    if (handler != null) {\n                        handler.run();\n                    }\n                } catch (final Throwable e) {\n                    handler.handleException(e);\n                }\n            }\n        }\n\n        public <T> CompletableFuture<RemotingCommand> appendEvent(final String name,\n            final Supplier<ControllerResult<T>> supplier, boolean isWriteEvent) {\n            if (isStopped() || !DLedgerController.this.roleHandler.isLeaderState()) {\n                final RemotingCommand command = RemotingCommand.createResponseCommand(ResponseCode.CONTROLLER_NOT_LEADER, \"The controller is not in leader state\");\n                final CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n                future.complete(command);\n                return future;\n            }\n\n            final EventHandler<T> event = new ControllerEventHandler<>(name, supplier, isWriteEvent);\n            int tryTimes = 0;\n            while (true) {\n                try {\n                    if (!this.eventQueue.offer(event, 5, TimeUnit.SECONDS)) {\n                        continue;\n                    }\n                    return event.future();\n                } catch (final InterruptedException e) {\n                    log.error(\"Error happen in EventScheduler when append event\", e);\n                    tryTimes++;\n                    if (tryTimes > 3) {\n                        return null;\n                    }\n                }\n            }\n        }\n    }\n\n    /**\n     * Event handler, get events from supplier, and append events to DLedger\n     */\n    class ControllerEventHandler<T> implements EventHandler<T> {\n        private final String name;\n        private final Supplier<ControllerResult<T>> supplier;\n        private final CompletableFuture<RemotingCommand> future;\n        private final boolean isWriteEvent;\n\n        ControllerEventHandler(final String name, final Supplier<ControllerResult<T>> supplier,\n            final boolean isWriteEvent) {\n            this.name = name;\n            this.supplier = supplier;\n            this.future = new CompletableFuture<>();\n            this.isWriteEvent = isWriteEvent;\n        }\n\n        @Override\n        public void run() throws Throwable {\n            final ControllerResult<T> result = this.supplier.get();\n            log.info(\"Event queue run event {}, get the result {}\", this.name, result);\n            boolean appendSuccess = true;\n\n            if (!this.isWriteEvent || result.getEvents() == null || result.getEvents().isEmpty()) {\n                // read event, or write event with empty events in response which also equals to read event\n                if (DLedgerController.this.controllerConfig.isProcessReadEvent()) {\n                    // Now the DLedger don't have the function of Read-Index or Lease-Read,\n                    // So we still need to propose an empty request to DLedger.\n                    final AppendEntryRequest request = new AppendEntryRequest();\n                    request.setBody(new byte[0]);\n                    appendSuccess = appendToDLedgerAndWait(request);\n                }\n            } else {\n                // write event\n                final List<EventMessage> events = result.getEvents();\n                final List<byte[]> eventBytes = new ArrayList<>(events.size());\n                for (final EventMessage event : events) {\n                    if (event != null) {\n                        final byte[] data = DLedgerController.this.eventSerializer.serialize(event);\n                        if (data != null && data.length > 0) {\n                            eventBytes.add(data);\n                        }\n                    }\n                }\n                // Append events to DLedger\n                if (!eventBytes.isEmpty()) {\n                    // batch append events\n                    final BatchAppendEntryRequest request = new BatchAppendEntryRequest();\n                    request.setBatchMsgs(eventBytes);\n                    appendSuccess = appendToDLedgerAndWait(request);\n                }\n            }\n\n            if (appendSuccess) {\n                final RemotingCommand response = RemotingCommand.createResponseCommandWithHeader(result.getResponseCode(), (CommandCustomHeader) result.getResponse());\n                if (result.getBody() != null) {\n                    response.setBody(result.getBody());\n                }\n                if (result.getRemark() != null) {\n                    response.setRemark(result.getRemark());\n                }\n                this.future.complete(response);\n            } else {\n                log.error(\"Failed to append event to DLedger, the response is {}, try cancel the future\", result.getResponse());\n                this.future.cancel(true);\n            }\n        }\n\n        @Override\n        public CompletableFuture<RemotingCommand> future() {\n            return this.future;\n        }\n\n        @Override\n        public void handleException(final Throwable t) {\n            log.error(\"Error happen when handle event {}\", this.name, t);\n            this.future.completeExceptionally(t);\n        }\n    }\n\n    /**\n     * Role change handler, trigger the startScheduling() and stopScheduling() when role change.\n     */\n    class RoleChangeHandler implements DLedgerLeaderElector.RoleChangeHandler {\n\n        private final String selfId;\n        private final ExecutorService executorService = ThreadUtils.newSingleThreadExecutor(new ThreadFactoryImpl(\"DLedgerControllerRoleChangeHandler_\"));\n        private volatile MemberState.Role currentRole = MemberState.Role.FOLLOWER;\n\n        public RoleChangeHandler(final String selfId) {\n            this.selfId = selfId;\n        }\n\n        @Override\n        public void handle(long term, MemberState.Role role) {\n            Runnable runnable = () -> {\n                switch (role) {\n                    case CANDIDATE:\n                        ControllerMetricsManager.recordRole(role, this.currentRole);\n                        this.currentRole = MemberState.Role.CANDIDATE;\n                        log.info(\"Controller {} change role to candidate\", this.selfId);\n                        DLedgerController.this.stopScheduling();\n                        DLedgerController.this.cancelScanInactiveFuture();\n                        break;\n                    case FOLLOWER:\n                        ControllerMetricsManager.recordRole(role, this.currentRole);\n                        this.currentRole = MemberState.Role.FOLLOWER;\n                        log.info(\"Controller {} change role to Follower, leaderId:{}\", this.selfId, getMemberState().getLeaderId());\n                        DLedgerController.this.stopScheduling();\n                        DLedgerController.this.cancelScanInactiveFuture();\n                        break;\n                    case LEADER: {\n                        log.info(\"Controller {} change role to leader, try process a initial proposal\", this.selfId);\n                        // Because the role becomes to leader, but the memory statemachine of the controller is still in the old point,\n                        // some committed logs have not been applied. Therefore, we must first process an empty request to DLedger,\n                        // and after the request is committed, the controller can provide services(startScheduling).\n                        int tryTimes = 0;\n                        while (true) {\n                            final AppendEntryRequest request = new AppendEntryRequest();\n                            request.setBody(new byte[0]);\n                            try {\n                                if (appendToDLedgerAndWait(request)) {\n                                    ControllerMetricsManager.recordRole(role, this.currentRole);\n                                    this.currentRole = MemberState.Role.LEADER;\n                                    DLedgerController.this.startScheduling();\n                                    if (DLedgerController.this.scanInactiveMasterFuture == null) {\n                                        long scanInactiveMasterInterval = DLedgerController.this.controllerConfig.getScanInactiveMasterInterval();\n                                        DLedgerController.this.scanInactiveMasterFuture =\n                                            DLedgerController.this.scanInactiveMasterService.scheduleAtFixedRate(DLedgerController.this::scanInactiveMasterAndTriggerReelect,\n                                                scanInactiveMasterInterval, scanInactiveMasterInterval, TimeUnit.MILLISECONDS);\n                                    }\n                                    break;\n                                }\n                            } catch (final Throwable e) {\n                                log.error(\"Error happen when controller leader append initial request to DLedger\", e);\n                            }\n                            if (!DLedgerController.this.getMemberState().isLeader()) {\n                                // now is not a leader\n                                log.error(\"Append a initial log failed because current state is not leader\");\n                                break;\n                            }\n                            tryTimes++;\n                            log.error(\"Controller leader append initial log failed, try {} times\", tryTimes);\n                            if (tryTimes % 3 == 0) {\n                                log.warn(\"Controller leader append initial log failed too many times, please wait a while\");\n                            }\n                        }\n                        break;\n                    }\n                }\n            };\n            this.executorService.submit(runnable);\n        }\n\n        @Override\n        public void startup() {\n        }\n\n        @Override\n        public void shutdown() {\n            if (this.currentRole == MemberState.Role.LEADER) {\n                DLedgerController.this.stopScheduling();\n            }\n            this.executorService.shutdown();\n        }\n\n        public boolean isLeaderState() {\n            return this.currentRole == MemberState.Role.LEADER;\n        }\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerControllerStateMachine.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl;\n\nimport io.openmessaging.storage.dledger.entry.DLedgerEntry;\nimport io.openmessaging.storage.dledger.exception.DLedgerException;\nimport io.openmessaging.storage.dledger.snapshot.SnapshotReader;\nimport io.openmessaging.storage.dledger.snapshot.SnapshotWriter;\nimport io.openmessaging.storage.dledger.statemachine.CommittedEntryIterator;\nimport io.openmessaging.storage.dledger.statemachine.StateMachine;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.controller.impl.event.EventMessage;\nimport org.apache.rocketmq.controller.impl.event.EventSerializer;\nimport org.apache.rocketmq.controller.impl.manager.ReplicasInfoManager;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\n/**\n * The state machine implementation of the dledger controller\n */\npublic class DLedgerControllerStateMachine implements StateMachine {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.CONTROLLER_LOGGER_NAME);\n    private final ReplicasInfoManager replicasInfoManager;\n    private final EventSerializer eventSerializer;\n    private final String dLedgerId;\n\n    public DLedgerControllerStateMachine(final ReplicasInfoManager replicasInfoManager,\n        final EventSerializer eventSerializer, final String dLedgerGroupId, final String dLedgerSelfId) {\n        this.replicasInfoManager = replicasInfoManager;\n        this.eventSerializer = eventSerializer;\n        this.dLedgerId = generateDLedgerId(dLedgerGroupId, dLedgerSelfId);\n    }\n\n    @Override\n    public String generateDLedgerId(String dLedgerGroupId, String dLedgerSelfId) {\n        return new StringBuilder(20).append(dLedgerGroupId).append(\"#\").append(dLedgerSelfId).toString();\n    }\n\n    @Override\n    public void onApply(CommittedEntryIterator iterator) {\n        int applyingSize = 0;\n        long firstApplyIndex = -1;\n        long lastApplyIndex = -1;\n        while (iterator.hasNext()) {\n            final DLedgerEntry entry = iterator.next();\n            final byte[] body = entry.getBody();\n            if (body != null && body.length > 0) {\n                final EventMessage event = this.eventSerializer.deserialize(body);\n                this.replicasInfoManager.applyEvent(event);\n            }\n            firstApplyIndex = firstApplyIndex == -1 ? entry.getIndex() : firstApplyIndex;\n            lastApplyIndex = entry.getIndex();\n            applyingSize++;\n        }\n        log.info(\"Apply {} events index from {} to {} on controller {}\", applyingSize, firstApplyIndex, lastApplyIndex, this.dLedgerId);\n    }\n\n    @Override\n    public boolean onSnapshotSave(SnapshotWriter writer) {\n        return true;\n    }\n\n    @Override\n    public boolean onSnapshotLoad(SnapshotReader reader) {\n        return false;\n    }\n\n    @Override\n    public void onShutdown() {\n        log.info(\"StateMachine {} onShutdown\", this.dLedgerId);\n    }\n\n    @Override\n    public void onError(DLedgerException exception) {\n        log.error(\"Encountered an error on StateMachine {}, dLedger may stop working since some error occurs, you should figure out the cause and repair or remove this node.\", this.dLedgerId, exception);\n    }\n\n    @Override\n    public String getBindDLedgerId() {\n        return this.dLedgerId;\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/JRaftController.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl;\n\nimport com.alipay.sofa.jraft.Node;\nimport com.alipay.sofa.jraft.RaftGroupService;\nimport com.alipay.sofa.jraft.Status;\nimport com.alipay.sofa.jraft.conf.Configuration;\nimport com.alipay.sofa.jraft.entity.NodeId;\nimport com.alipay.sofa.jraft.entity.PeerId;\nimport com.alipay.sofa.jraft.entity.Task;\nimport com.alipay.sofa.jraft.option.NodeOptions;\nimport org.apache.commons.io.FileUtils;\nimport org.apache.rocketmq.common.ControllerConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.controller.Controller;\nimport org.apache.rocketmq.controller.helper.BrokerLifecycleListener;\nimport org.apache.rocketmq.controller.impl.closure.ControllerClosure;\nimport org.apache.rocketmq.controller.impl.task.BrokerCloseChannelRequest;\nimport org.apache.rocketmq.controller.impl.task.CheckNotActiveBrokerRequest;\nimport org.apache.rocketmq.controller.impl.task.GetBrokerLiveInfoRequest;\nimport org.apache.rocketmq.controller.impl.task.GetSyncStateDataRequest;\nimport org.apache.rocketmq.controller.impl.task.RaftBrokerHeartBeatEventRequest;\nimport org.apache.rocketmq.remoting.ChannelEventListener;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.RemotingServer;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingServer;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.SyncStateSet;\nimport org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerRequestHeader;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.CompletableFuture;\n\npublic class JRaftController implements Controller {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.CONTROLLER_LOGGER_NAME);\n    private final RaftGroupService raftGroupService;\n    private Node node;\n    private final JRaftControllerStateMachine stateMachine;\n    private final ControllerConfig controllerConfig;\n    private final List<BrokerLifecycleListener> brokerLifecycleListeners;\n    private final Map<PeerId/* jRaft peerId */, String/* Controller RPC Server Addr */> peerIdToAddr;\n    private final NettyRemotingServer remotingServer;\n\n    public JRaftController(ControllerConfig controllerConfig,\n        final ChannelEventListener channelEventListener) throws IOException {\n        this.controllerConfig = controllerConfig;\n        this.brokerLifecycleListeners = new ArrayList<>();\n\n        final NodeOptions nodeOptions = new NodeOptions();\n        nodeOptions.setElectionTimeoutMs(controllerConfig.getJraftConfig().getjRaftElectionTimeoutMs());\n        nodeOptions.setSnapshotIntervalSecs(controllerConfig.getJraftConfig().getjRaftSnapshotIntervalSecs());\n        final PeerId serverId = new PeerId();\n        if (!serverId.parse(controllerConfig.getJraftConfig().getjRaftServerId())) {\n            throw new IllegalArgumentException(\"Fail to parse serverId:\" + controllerConfig.getJraftConfig().getjRaftServerId());\n        }\n        final Configuration initConf = new Configuration();\n        if (!initConf.parse(controllerConfig.getJraftConfig().getjRaftInitConf())) {\n            throw new IllegalArgumentException(\"Fail to parse initConf:\" + controllerConfig.getJraftConfig().getjRaftInitConf());\n        }\n        nodeOptions.setInitialConf(initConf);\n\n        FileUtils.forceMkdir(new File(controllerConfig.getControllerStorePath()));\n        nodeOptions.setLogUri(controllerConfig.getControllerStorePath() + File.separator + \"log\");\n        nodeOptions.setRaftMetaUri(controllerConfig.getControllerStorePath() + File.separator + \"raft_meta\");\n        nodeOptions.setSnapshotUri(controllerConfig.getControllerStorePath() + File.separator + \"snapshot\");\n\n        this.stateMachine = new JRaftControllerStateMachine(controllerConfig, new NodeId(controllerConfig.getJraftConfig().getjRaftGroupId(), serverId));\n        this.stateMachine.registerOnLeaderStart(this::onLeaderStart);\n        this.stateMachine.registerOnLeaderStop(this::onLeaderStop);\n        nodeOptions.setFsm(this.stateMachine);\n\n        this.raftGroupService = new RaftGroupService(controllerConfig.getJraftConfig().getjRaftGroupId(), serverId, nodeOptions);\n\n        this.peerIdToAddr = new HashMap<>();\n        initPeerIdMap();\n\n        NettyServerConfig nettyServerConfig = new NettyServerConfig();\n        nettyServerConfig.setListenPort(Integer.parseInt(this.peerIdToAddr.get(serverId).split(\":\")[1]));\n        remotingServer = new NettyRemotingServer(nettyServerConfig, channelEventListener);\n    }\n\n    private void initPeerIdMap() {\n        String[] peers = this.controllerConfig.getJraftConfig().getjRaftInitConf().split(\",\");\n        String[] rpcAddrs = this.controllerConfig.getJraftConfig().getjRaftControllerRPCAddr().split(\",\");\n        for (int i = 0; i < peers.length; i++) {\n            PeerId peerId = new PeerId();\n            if (!peerId.parse(peers[i])) {\n                throw new IllegalArgumentException(\"Fail to parse peerId:\" + peers[i]);\n            }\n            this.peerIdToAddr.put(peerId, rpcAddrs[i]);\n        }\n    }\n\n    @Override\n    public void startup() {\n        this.remotingServer.start();\n        this.node = this.raftGroupService.start();\n        log.info(\"Controller {} started.\", node.getNodeId());\n    }\n\n    @Override\n    public void shutdown() {\n        this.stopScheduling();\n        this.raftGroupService.shutdown();\n        this.remotingServer.shutdown();\n        log.info(\"Controller {} stopped.\", node.getNodeId());\n    }\n\n    @Override\n    public void startScheduling() {\n    }\n\n    @Override\n    public void stopScheduling() {\n    }\n\n    @Override\n    public boolean isLeaderState() {\n        return node.isLeader();\n    }\n\n    private <T extends CommandCustomHeader> CompletableFuture<RemotingCommand> applyToJRaft(RemotingCommand request) {\n        if (!isLeaderState()) {\n            final RemotingCommand command = RemotingCommand.createResponseCommand(ResponseCode.CONTROLLER_NOT_LEADER, \"The controller is not in leader state\");\n            final CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n            future.complete(command);\n            log.warn(\"Apply to none leader controller, controller state is {}\", node.getNodeState());\n            return future;\n        }\n        ControllerClosure closure = new ControllerClosure(request);\n        Task task = closure.taskWithThisClosure();\n        if (task != null) {\n            node.apply(task);\n            return closure.getFuture();\n        } else {\n            log.error(\"Apply task failed, task is null.\");\n            return CompletableFuture.completedFuture(RemotingCommand.createResponseCommand(ResponseCode.CONTROLLER_JRAFT_INTERNAL_ERROR, \"Apply task failed, Please see the server log.\"));\n        }\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> alterSyncStateSet(AlterSyncStateSetRequestHeader request,\n        SyncStateSet syncStateSet) {\n        final RemotingCommand requestCommand = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_ALTER_SYNC_STATE_SET, request);\n        requestCommand.setBody(syncStateSet.encode());\n        return applyToJRaft(requestCommand);\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> electMaster(ElectMasterRequestHeader request) {\n        final RemotingCommand requestCommand = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_ELECT_MASTER, request);\n        return applyToJRaft(requestCommand);\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> getNextBrokerId(GetNextBrokerIdRequestHeader request) {\n        final RemotingCommand requestCommand = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_GET_NEXT_BROKER_ID, request);\n        return applyToJRaft(requestCommand);\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> applyBrokerId(ApplyBrokerIdRequestHeader request) {\n        final RemotingCommand requestCommand = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_APPLY_BROKER_ID, request);\n        return applyToJRaft(requestCommand);\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> registerBroker(RegisterBrokerToControllerRequestHeader request) {\n        final RemotingCommand requestCommand = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_REGISTER_BROKER, request);\n        return applyToJRaft(requestCommand);\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> getReplicaInfo(GetReplicaInfoRequestHeader request) {\n        final RemotingCommand requestCommand = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_GET_REPLICA_INFO, request);\n        return applyToJRaft(requestCommand);\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> getSyncStateData(List<String> brokerNames) {\n        final RemotingCommand requestCommand = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_GET_SYNC_STATE_DATA, new GetSyncStateDataRequest());\n        requestCommand.setBody(RemotingSerializable.encode(brokerNames));\n        return applyToJRaft(requestCommand);\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> cleanBrokerData(CleanControllerBrokerDataRequestHeader requestHeader) {\n        final RemotingCommand requestCommand = RemotingCommand.createRequestCommand(RequestCode.CLEAN_BROKER_DATA, requestHeader);\n        return applyToJRaft(requestCommand);\n    }\n\n    @Override\n    public void registerBrokerLifecycleListener(BrokerLifecycleListener listener) {\n        this.brokerLifecycleListeners.add(listener);\n    }\n\n    @Override\n    public RemotingCommand getControllerMetadata() {\n        List<PeerId> peers = node.getOptions().getInitialConf().getPeers();\n        final StringBuilder sb = new StringBuilder();\n        for (PeerId peer : peers) {\n            sb.append(peerIdToAddr.get(peer)).append(\";\");\n        }\n        return RemotingCommand.createResponseCommandWithHeader(ResponseCode.SUCCESS, new GetMetaDataResponseHeader(\n            node.getGroupId(),\n            node.getLeaderId() == null ? \"\" : node.getLeaderId().toString(),\n            this.peerIdToAddr.get(node.getLeaderId()),\n            node.isLeader(),\n            sb.toString()\n        ));\n    }\n\n    @Override\n    public RemotingServer getRemotingServer() {\n        return remotingServer;\n    }\n\n    public void onLeaderStart(long term) {\n        log.info(\"Controller start leadership, term: {}.\", term);\n    }\n\n    public void onLeaderStop(Status status) {\n        log.info(\"Controller {} stop leadership, status: {}.\", node.getNodeId(), status);\n        this.stopScheduling();\n    }\n\n    public CompletableFuture<RemotingCommand> getBrokerLiveInfo(GetBrokerLiveInfoRequest requestHeader) {\n        final RemotingCommand requestCommand = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_LIVE_INFO_REQUEST, requestHeader);\n        return applyToJRaft(requestCommand);\n    }\n\n    public CompletableFuture<RemotingCommand> onBrokerHeartBeat(RaftBrokerHeartBeatEventRequest requestHeader) {\n        final RemotingCommand requestCommand = RemotingCommand.createRequestCommand(RequestCode.RAFT_BROKER_HEART_BEAT_EVENT_REQUEST, requestHeader);\n        return applyToJRaft(requestCommand);\n    }\n\n    public CompletableFuture<RemotingCommand> onBrokerCloseChannel(BrokerCloseChannelRequest requestHeader) {\n        final RemotingCommand requestCommand = RemotingCommand.createRequestCommand(RequestCode.BROKER_CLOSE_CHANNEL_REQUEST, requestHeader);\n        return applyToJRaft(requestCommand);\n    }\n\n    public CompletableFuture<RemotingCommand> checkNotActiveBroker(CheckNotActiveBrokerRequest requestHeader) {\n        final RemotingCommand requestCommand = RemotingCommand.createRequestCommand(RequestCode.CHECK_NOT_ACTIVE_BROKER_REQUEST, requestHeader);\n        return applyToJRaft(requestCommand);\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/JRaftControllerStateMachine.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl;\n\nimport com.alipay.sofa.jraft.Closure;\nimport com.alipay.sofa.jraft.Iterator;\nimport com.alipay.sofa.jraft.StateMachine;\nimport com.alipay.sofa.jraft.Status;\nimport com.alipay.sofa.jraft.conf.Configuration;\nimport com.alipay.sofa.jraft.entity.LeaderChangeContext;\nimport com.alipay.sofa.jraft.entity.NodeId;\nimport com.alipay.sofa.jraft.error.RaftError;\nimport com.alipay.sofa.jraft.error.RaftException;\nimport com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;\nimport com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter;\nimport com.alipay.sofa.jraft.util.Utils;\nimport io.opentelemetry.api.common.AttributesBuilder;\nimport org.apache.commons.io.FileUtils;\nimport org.apache.rocketmq.common.ControllerConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.controller.elect.impl.DefaultElectPolicy;\nimport org.apache.rocketmq.controller.impl.closure.ControllerClosure;\nimport org.apache.rocketmq.controller.impl.event.ControllerResult;\nimport org.apache.rocketmq.controller.impl.manager.RaftReplicasInfoManager;\nimport org.apache.rocketmq.controller.impl.task.BrokerCloseChannelRequest;\nimport org.apache.rocketmq.controller.impl.task.CheckNotActiveBrokerRequest;\nimport org.apache.rocketmq.controller.impl.task.GetBrokerLiveInfoRequest;\nimport org.apache.rocketmq.controller.impl.task.GetSyncStateDataRequest;\nimport org.apache.rocketmq.controller.impl.task.RaftBrokerHeartBeatEventRequest;\nimport org.apache.rocketmq.controller.metrics.ControllerMetricsConstant;\nimport org.apache.rocketmq.controller.metrics.ControllerMetricsManager;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.SyncStateSet;\nimport org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerRequestHeader;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.function.Consumer;\n\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.LABEL_BROKER_SET;\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.LABEL_CLUSTER_NAME;\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.LABEL_ELECTION_RESULT;\n\npublic class JRaftControllerStateMachine implements StateMachine {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.CONTROLLER_LOGGER_NAME);\n    private final List<Consumer<Long>> onLeaderStartCallbacks;\n    private final List<Consumer<Status>> onLeaderStopCallbacks;\n    private final RaftReplicasInfoManager replicasInfoManager;\n    private final NodeId nodeId;\n\n    public JRaftControllerStateMachine(ControllerConfig controllerConfig, NodeId nodeId) {\n        this.replicasInfoManager = new RaftReplicasInfoManager(controllerConfig);\n        this.nodeId = nodeId;\n        this.onLeaderStartCallbacks = new ArrayList<>();\n        this.onLeaderStopCallbacks = new ArrayList<>();\n    }\n\n    @Override\n    public void onApply(Iterator iter) {\n        while (iter.hasNext()) {\n            byte[] data = iter.getData().array();\n            ControllerClosure controllerClosure = (ControllerClosure) iter.done();\n            processEvent(controllerClosure, data, iter.getTerm(), iter.getIndex());\n\n            iter.next();\n        }\n    }\n\n    private void processEvent(ControllerClosure controllerClosure, byte[] data, long term, long index) {\n        RemotingCommand request;\n        ControllerResult<?> result;\n        try {\n            if (controllerClosure != null) {\n                request = controllerClosure.getRequestEvent();\n            } else {\n                request = RemotingCommand.decode(Arrays.copyOfRange(data, 4, data.length));\n            }\n            log.info(\"process event: term {}, index {}, request code {}\", term, index, request.getCode());\n            switch (request.getCode()) {\n                case RequestCode.CONTROLLER_ALTER_SYNC_STATE_SET:\n                    AlterSyncStateSetRequestHeader requestHeader = (AlterSyncStateSetRequestHeader) request.decodeCommandCustomHeader(AlterSyncStateSetRequestHeader.class);\n                    SyncStateSet syncStateSet = RemotingSerializable.decode(request.getBody(), SyncStateSet.class);\n                    result = alterSyncStateSet(requestHeader, syncStateSet);\n                    break;\n                case RequestCode.CONTROLLER_ELECT_MASTER:\n                    ElectMasterRequestHeader electMasterRequestHeader = (ElectMasterRequestHeader) request.decodeCommandCustomHeader(ElectMasterRequestHeader.class);\n                    result = electMaster(electMasterRequestHeader);\n                    break;\n                case RequestCode.CONTROLLER_GET_NEXT_BROKER_ID:\n                    GetNextBrokerIdRequestHeader getNextBrokerIdRequestHeader = (GetNextBrokerIdRequestHeader) request.decodeCommandCustomHeader(GetNextBrokerIdRequestHeader.class);\n                    result = getNextBrokerId(getNextBrokerIdRequestHeader);\n                    break;\n                case RequestCode.CONTROLLER_APPLY_BROKER_ID:\n                    ApplyBrokerIdRequestHeader applyBrokerIdRequestHeader = (ApplyBrokerIdRequestHeader) request.decodeCommandCustomHeader(ApplyBrokerIdRequestHeader.class);\n                    result = applyBrokerId(applyBrokerIdRequestHeader);\n                    break;\n                case RequestCode.CONTROLLER_REGISTER_BROKER:\n                    RegisterBrokerToControllerRequestHeader registerBrokerToControllerRequestHeader = (RegisterBrokerToControllerRequestHeader) request.decodeCommandCustomHeader(RegisterBrokerToControllerRequestHeader.class);\n                    result = registerBroker(registerBrokerToControllerRequestHeader);\n                    break;\n                case RequestCode.CONTROLLER_GET_REPLICA_INFO:\n                    GetReplicaInfoRequestHeader getReplicaInfoRequestHeader = (GetReplicaInfoRequestHeader) request.decodeCommandCustomHeader(GetReplicaInfoRequestHeader.class);\n                    result = getReplicaInfo(getReplicaInfoRequestHeader);\n                    break;\n                case RequestCode.CONTROLLER_GET_SYNC_STATE_DATA:\n                    List<String> brokerNames = RemotingSerializable.decode(request.getBody(), List.class);\n                    GetSyncStateDataRequest getSyncStateDataRequest = (GetSyncStateDataRequest) request.decodeCommandCustomHeader(GetSyncStateDataRequest.class);\n                    result = getSyncStateData(brokerNames, getSyncStateDataRequest.getInvokeTime());\n                    break;\n                case RequestCode.CLEAN_BROKER_DATA:\n                    CleanControllerBrokerDataRequestHeader cleanBrokerDataRequestHeader = (CleanControllerBrokerDataRequestHeader) request.decodeCommandCustomHeader(CleanControllerBrokerDataRequestHeader.class);\n                    result = cleanBrokerData(cleanBrokerDataRequestHeader);\n                    break;\n                case RequestCode.GET_BROKER_LIVE_INFO_REQUEST:\n                    GetBrokerLiveInfoRequest getBrokerLiveInfoRequest = (GetBrokerLiveInfoRequest) request.decodeCommandCustomHeader(GetBrokerLiveInfoRequest.class);\n                    result = replicasInfoManager.getBrokerLiveInfo(getBrokerLiveInfoRequest);\n                    break;\n                case RequestCode.RAFT_BROKER_HEART_BEAT_EVENT_REQUEST:\n                    RaftBrokerHeartBeatEventRequest brokerHeartbeatRequestHeader = (RaftBrokerHeartBeatEventRequest) request.decodeCommandCustomHeader(RaftBrokerHeartBeatEventRequest.class);\n                    result = replicasInfoManager.onBrokerHeartBeat(brokerHeartbeatRequestHeader);\n                    break;\n                case RequestCode.BROKER_CLOSE_CHANNEL_REQUEST:\n                    BrokerCloseChannelRequest brokerCloseChannelRequest = (BrokerCloseChannelRequest) request.decodeCommandCustomHeader(BrokerCloseChannelRequest.class);\n                    result = replicasInfoManager.onBrokerCloseChannel(brokerCloseChannelRequest);\n                    break;\n                case RequestCode.CHECK_NOT_ACTIVE_BROKER_REQUEST:\n                    CheckNotActiveBrokerRequest checkNotActiveBrokerRequest = (CheckNotActiveBrokerRequest) request.decodeCommandCustomHeader(CheckNotActiveBrokerRequest.class);\n                    result = replicasInfoManager.checkNotActiveBroker(checkNotActiveBrokerRequest);\n                    break;\n                default:\n                    throw new RemotingCommandException(\"Unknown request code: \" + request.getCode());\n            }\n            result.getEvents().forEach(replicasInfoManager::applyEvent);\n        } catch (RemotingCommandException e) {\n            log.error(\"Fail to process event\", e);\n            if (controllerClosure != null) {\n                controllerClosure.run(new Status(RaftError.EINTERNAL, e.getMessage()));\n            }\n            return;\n        }\n        log.info(\"process event: term {}, index {}, request code {} success with result {}\", term, index, request.getCode(), result.toString());\n        if (controllerClosure != null) {\n            controllerClosure.setControllerResult(result);\n            controllerClosure.run(Status.OK());\n        }\n    }\n\n    private ControllerResult<AlterSyncStateSetResponseHeader> alterSyncStateSet(\n        AlterSyncStateSetRequestHeader requestHeader, SyncStateSet syncStateSet) {\n        return replicasInfoManager.alterSyncStateSet(requestHeader, syncStateSet, new RaftReplicasInfoManager.BrokerValidPredicateWithInvokeTime(requestHeader.getInvokeTime(), this.replicasInfoManager));\n    }\n\n    private ControllerResult<ElectMasterResponseHeader> electMaster(ElectMasterRequestHeader request) {\n        ControllerResult<ElectMasterResponseHeader> electResult = this.replicasInfoManager.electMaster(request, new DefaultElectPolicy(\n            (clusterName, brokerName, brokerId) -> replicasInfoManager.isBrokerActive(clusterName, brokerName, brokerId, request.getInvokeTime()),\n            replicasInfoManager::getBrokerLiveInfo\n        ));\n        log.info(\"elect master, request :{}, result: {}\", request.toString(), electResult.toString());\n        AttributesBuilder attributesBuilder = ControllerMetricsManager.newAttributesBuilder()\n            .put(LABEL_CLUSTER_NAME, request.getClusterName())\n            .put(LABEL_BROKER_SET, request.getBrokerName());\n        switch (electResult.getResponseCode()) {\n            case ResponseCode.SUCCESS:\n                ControllerMetricsManager.electionTotal.add(1,\n                    attributesBuilder.put(LABEL_ELECTION_RESULT, ControllerMetricsConstant.ElectionResult.NEW_MASTER_ELECTED.getLowerCaseName()).build());\n                break;\n            case ResponseCode.CONTROLLER_MASTER_STILL_EXIST:\n                ControllerMetricsManager.electionTotal.add(1,\n                    attributesBuilder.put(LABEL_ELECTION_RESULT, ControllerMetricsConstant.ElectionResult.KEEP_CURRENT_MASTER.getLowerCaseName()).build());\n                break;\n            case ResponseCode.CONTROLLER_MASTER_NOT_AVAILABLE:\n            case ResponseCode.CONTROLLER_ELECT_MASTER_FAILED:\n                ControllerMetricsManager.electionTotal.add(1,\n                    attributesBuilder.put(LABEL_ELECTION_RESULT, ControllerMetricsConstant.ElectionResult.NO_MASTER_ELECTED.getLowerCaseName()).build());\n                break;\n            default:\n                break;\n        }\n        return electResult;\n    }\n\n    private ControllerResult<GetNextBrokerIdResponseHeader> getNextBrokerId(\n        GetNextBrokerIdRequestHeader requestHeader) {\n        return replicasInfoManager.getNextBrokerId(requestHeader);\n    }\n\n    private ControllerResult<ApplyBrokerIdResponseHeader> applyBrokerId(ApplyBrokerIdRequestHeader requestHeader) {\n        return replicasInfoManager.applyBrokerId(requestHeader);\n    }\n\n    private ControllerResult<?> registerBroker(RegisterBrokerToControllerRequestHeader request) {\n        return replicasInfoManager.registerBroker(request, new RaftReplicasInfoManager.BrokerValidPredicateWithInvokeTime(request.getInvokeTime(), this.replicasInfoManager));\n    }\n\n    private ControllerResult<GetReplicaInfoResponseHeader> getReplicaInfo(GetReplicaInfoRequestHeader request) {\n        return replicasInfoManager.getReplicaInfo(request);\n    }\n\n    private ControllerResult<Void> getSyncStateData(List<String> brokerNames, long invokeTile) {\n        return replicasInfoManager.getSyncStateData(brokerNames, new RaftReplicasInfoManager.BrokerValidPredicateWithInvokeTime(invokeTile, this.replicasInfoManager));\n    }\n\n    private ControllerResult<Void> cleanBrokerData(CleanControllerBrokerDataRequestHeader requestHeader) {\n        return replicasInfoManager.cleanBrokerData(requestHeader, new RaftReplicasInfoManager.BrokerValidPredicateWithInvokeTime(requestHeader.getInvokeTime(), this.replicasInfoManager));\n    }\n\n    @Override\n    public void onShutdown() {\n        log.info(\"StateMachine {} node {} onShutdown\", getClass().getName(), nodeId.toString());\n    }\n\n    @Override\n    public void onSnapshotSave(SnapshotWriter writer, Closure done) {\n        byte[] data;\n        try {\n            data = this.replicasInfoManager.serialize();\n        } catch (Throwable e) {\n            done.run(new Status(RaftError.EIO, \"Fail to serialize replicasInfoManager state machine data\"));\n            return;\n        }\n        Utils.runInThread(() -> {\n            try {\n                FileUtils.writeByteArrayToFile(new File(writer.getPath() + File.separator + \"data\"), data);\n                if (writer.addFile(\"data\")) {\n                    log.info(\"Save snapshot, path={}\", writer.getPath());\n                    done.run(Status.OK());\n                } else {\n                    throw new IOException(\"Fail to add file to writer\");\n                }\n            } catch (IOException e) {\n                log.error(\"Fail to save snapshot\", e);\n                done.run(new Status(RaftError.EIO, \"Fail to save snapshot\"));\n            }\n        });\n    }\n\n    @Override\n    public boolean onSnapshotLoad(SnapshotReader reader) {\n        if (reader.getFileMeta(\"data\") == null) {\n            log.error(\"Fail to find data file in {}\", reader.getPath());\n            return false;\n        }\n        try {\n            byte[] data = FileUtils.readFileToByteArray(new File(reader.getPath() + File.separator + \"data\"));\n            this.replicasInfoManager.deserializeFrom(data);\n            log.info(\"Load snapshot from {}\", reader.getPath());\n            return true;\n        } catch (Throwable e) {\n            log.error(\"Fail to load snapshot from {}\", reader.getPath(), e);\n            return false;\n        }\n    }\n\n    @Override\n    public void onLeaderStart(long term) {\n        for (Consumer<Long> callback : onLeaderStartCallbacks) {\n            callback.accept(term);\n        }\n        log.info(\"node {} Start Leader, term={}\", nodeId.toString(), term);\n    }\n\n    @Override\n    public void onLeaderStop(Status status) {\n        for (Consumer<Status> callback : onLeaderStopCallbacks) {\n            callback.accept(status);\n        }\n        log.info(\"node {} Stop Leader, status={}\", nodeId.toString(), status);\n    }\n\n    public void registerOnLeaderStart(Consumer<Long> callback) {\n        onLeaderStartCallbacks.add(callback);\n    }\n\n    public void registerOnLeaderStop(Consumer<Status> callback) {\n        onLeaderStopCallbacks.add(callback);\n    }\n\n    @Override\n    public void onError(RaftException e) {\n        log.error(\"Encountered an error={} on StateMachine {}, node {}, raft may stop working since some error occurs, you should figure out the cause and repair or remove this node.\", e.getStatus(), this.getClass().getName(), nodeId.toString(), e);\n    }\n\n    @Override\n    public void onConfigurationCommitted(Configuration conf) {\n        log.info(\"Configuration committed, conf={}\", conf);\n    }\n\n    @Override\n    public void onStopFollowing(LeaderChangeContext ctx) {\n        log.info(\"Stop following, ctx={}\", ctx);\n    }\n\n    @Override\n    public void onStartFollowing(LeaderChangeContext ctx) {\n        log.info(\"Start following, ctx={}\", ctx);\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/closure/ControllerClosure.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.closure;\n\nimport com.alipay.sofa.jraft.Closure;\nimport com.alipay.sofa.jraft.Status;\nimport com.alipay.sofa.jraft.entity.Task;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.controller.impl.event.ControllerResult;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\n\nimport java.util.concurrent.CompletableFuture;\n\npublic class ControllerClosure implements Closure {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.CONTROLLER_LOGGER_NAME);\n    private final RemotingCommand requestEvent;\n    private final CompletableFuture<RemotingCommand> future;\n    private ControllerResult<?> controllerResult;\n    private Task task;\n\n    public ControllerClosure(RemotingCommand requestEvent) {\n        this.requestEvent = requestEvent;\n        this.future = new CompletableFuture<>();\n        this.task = null;\n    }\n\n    public CompletableFuture<RemotingCommand> getFuture() {\n        return future;\n    }\n\n    public void setControllerResult(ControllerResult<?> controllerResult) {\n        this.controllerResult = controllerResult;\n    }\n\n    @Override\n    public void run(Status status) {\n        if (status.isOk()) {\n            final RemotingCommand response = RemotingCommand.createResponseCommandWithHeader(controllerResult.getResponseCode(), (CommandCustomHeader) controllerResult.getResponse());\n            if (controllerResult.getBody() != null) {\n                response.setBody(controllerResult.getBody());\n            }\n            if (controllerResult.getRemark() != null) {\n                response.setRemark(controllerResult.getRemark());\n            }\n            future.complete(response);\n        } else {\n            log.error(\"Failed to append to jRaft node, error is: {}.\", status);\n            future.complete(RemotingCommand.createResponseCommand(ResponseCode.CONTROLLER_JRAFT_INTERNAL_ERROR, status.getErrorMsg()));\n        }\n    }\n\n    public Task taskWithThisClosure() {\n        if (task != null) {\n            return task;\n        }\n        task = new Task();\n        task.setDone(this);\n        task.setData(requestEvent.encode());\n        return task;\n    }\n\n    public RemotingCommand getRequestEvent() {\n        return requestEvent;\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/event/AlterSyncStateSetEvent.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.event;\n\nimport java.util.HashSet;\nimport java.util.Set;\n\n/**\n * The event alters the syncStateSet of target broker.\n * Triggered by the AlterSyncStateSetApi.\n */\npublic class AlterSyncStateSetEvent implements EventMessage {\n\n    private final String brokerName;\n    private final Set<Long/*BrokerId*/> newSyncStateSet;\n\n    public AlterSyncStateSetEvent(String brokerName, Set<Long> newSyncStateSet) {\n        this.brokerName = brokerName;\n        this.newSyncStateSet = new HashSet<>(newSyncStateSet);\n    }\n\n    @Override\n    public EventType getEventType() {\n        return EventType.ALTER_SYNC_STATE_SET_EVENT;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public Set<Long> getNewSyncStateSet() {\n        return new HashSet<>(newSyncStateSet);\n    }\n\n    @Override\n    public String toString() {\n        return \"AlterSyncStateSetEvent{\" +\n            \"brokerName='\" + brokerName + '\\'' +\n            \", newSyncStateSet=\" + newSyncStateSet +\n            '}';\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/event/ApplyBrokerIdEvent.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.event;\n\n/**\n * The event trys to apply a new id for a new broker.\n * Triggered by the RegisterBrokerApi.\n */\npublic class ApplyBrokerIdEvent implements EventMessage {\n    private final String clusterName;\n    private final String brokerName;\n    private final String brokerAddress;\n\n    private final String registerCheckCode;\n\n    private final long newBrokerId;\n\n    public ApplyBrokerIdEvent(String clusterName, String brokerName, String brokerAddress, long newBrokerId,\n        String registerCheckCode) {\n        this.clusterName = clusterName;\n        this.brokerName = brokerName;\n        this.brokerAddress = brokerAddress;\n        this.newBrokerId = newBrokerId;\n        this.registerCheckCode = registerCheckCode;\n    }\n\n    @Override\n    public EventType getEventType() {\n        return EventType.APPLY_BROKER_ID_EVENT;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public String getBrokerAddress() {\n        return brokerAddress;\n    }\n\n    public long getNewBrokerId() {\n        return newBrokerId;\n    }\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    public String getRegisterCheckCode() {\n        return registerCheckCode;\n    }\n\n    @Override\n    public String toString() {\n        return \"ApplyBrokerIdEvent{\" +\n            \"clusterName='\" + clusterName + '\\'' +\n            \", brokerName='\" + brokerName + '\\'' +\n            \", brokerAddress='\" + brokerAddress + '\\'' +\n            \", registerCheckCode='\" + registerCheckCode + '\\'' +\n            \", newBrokerId=\" + newBrokerId +\n            '}';\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/event/CleanBrokerDataEvent.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.controller.impl.event;\n\nimport java.util.Set;\n\npublic class CleanBrokerDataEvent implements EventMessage {\n\n    private String brokerName;\n\n    private Set<Long> brokerIdSetToClean;\n\n    public CleanBrokerDataEvent(String brokerName, Set<Long> brokerIdSetToClean) {\n        this.brokerName = brokerName;\n        this.brokerIdSetToClean = brokerIdSetToClean;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public void setBrokerIdSetToClean(Set<Long> brokerIdSetToClean) {\n        this.brokerIdSetToClean = brokerIdSetToClean;\n    }\n\n    public Set<Long> getBrokerIdSetToClean() {\n        return brokerIdSetToClean;\n    }\n\n    /**\n     * Returns the event type of this message\n     */\n    @Override\n    public EventType getEventType() {\n        return EventType.CLEAN_BROKER_DATA_EVENT;\n    }\n\n    @Override\n    public String toString() {\n        return \"CleanBrokerDataEvent{\" +\n            \"brokerName='\" + brokerName + '\\'' +\n            \", brokerIdSetToClean=\" + brokerIdSetToClean +\n            '}';\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/event/ControllerResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.event;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\n\npublic class ControllerResult<T> {\n    private final List<EventMessage> events;\n    private final T response;\n    private byte[] body;\n    private int responseCode = ResponseCode.SUCCESS;\n    private String remark;\n\n    public ControllerResult() {\n        this(null);\n    }\n\n    public ControllerResult(T response) {\n        this.events = new ArrayList<>();\n        this.response = response;\n    }\n\n    public ControllerResult(List<EventMessage> events, T response) {\n        this.events = new ArrayList<>(events);\n        this.response = response;\n    }\n\n    public static <T> ControllerResult<T> of(List<EventMessage> events, T response) {\n        return new ControllerResult<>(events, response);\n    }\n\n    public List<EventMessage> getEvents() {\n        return new ArrayList<>(events);\n    }\n\n    public T getResponse() {\n        return response;\n    }\n\n    public byte[] getBody() {\n        return body;\n    }\n\n    public void setBody(byte[] body) {\n        this.body = body;\n    }\n\n    public void setCodeAndRemark(int responseCode, String remark) {\n        this.responseCode = responseCode;\n        this.remark = remark;\n    }\n\n    public int getResponseCode() {\n        return responseCode;\n    }\n\n    public String getRemark() {\n        return remark;\n    }\n\n    public void addEvent(EventMessage event) {\n        this.events.add(event);\n    }\n\n    @Override\n    public String toString() {\n        return \"ControllerResult{\" +\n            \"events=\" + events +\n            \", response=\" + response +\n            '}';\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/event/ElectMasterEvent.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.event;\n\n/**\n * The event trys to elect a new master for target broker.\n * Triggered by the ElectMasterApi.\n */\npublic class ElectMasterEvent implements EventMessage {\n    // Mark whether a new master was elected.\n    private final boolean newMasterElected;\n    private final String brokerName;\n    private final Long newMasterBrokerId;\n\n    public ElectMasterEvent(boolean newMasterElected, String brokerName) {\n        this(newMasterElected, brokerName, null);\n    }\n\n    public ElectMasterEvent(String brokerName, Long newMasterBrokerId) {\n        this(true, brokerName, newMasterBrokerId);\n    }\n\n    public ElectMasterEvent(boolean newMasterElected, String brokerName, Long newMasterBrokerId) {\n        this.newMasterElected = newMasterElected;\n        this.brokerName = brokerName;\n        this.newMasterBrokerId = newMasterBrokerId;\n    }\n\n    @Override\n    public EventType getEventType() {\n        return EventType.ELECT_MASTER_EVENT;\n    }\n\n    public boolean getNewMasterElected() {\n        return newMasterElected;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public Long getNewMasterBrokerId() {\n        return newMasterBrokerId;\n    }\n\n    @Override\n    public String toString() {\n        return \"ElectMasterEvent{\" +\n            \"newMasterElected=\" + newMasterElected +\n            \", brokerName='\" + brokerName + '\\'' +\n            \", newMasterBrokerId=\" + newMasterBrokerId +\n            '}';\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/event/EventMessage.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.event;\n\n/**\n * The parent class of Event, the subclass needs to indicate eventType.\n */\npublic interface EventMessage {\n\n    /**\n     * Returns the event type of this message\n     */\n    EventType getEventType();\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/event/EventSerializer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.event;\n\nimport org.apache.commons.lang3.SerializationException;\nimport org.apache.rocketmq.common.utils.FastJsonSerializer;\n\n/**\n * EventMessage serializer\n */\npublic class EventSerializer {\n    private final FastJsonSerializer serializer;\n\n    public EventSerializer() {\n        this.serializer = new FastJsonSerializer();\n    }\n\n    private void putShort(byte[] memory, int index, int value) {\n        memory[index] = (byte) (value >>> 8);\n        memory[index + 1] = (byte) value;\n    }\n\n    private short getShort(byte[] memory, int index) {\n        return (short) (memory[index] << 8 | memory[index + 1] & 0xFF);\n    }\n\n    public byte[] serialize(EventMessage message) throws SerializationException {\n        final short eventType = message.getEventType().getId();\n        final byte[] data = this.serializer.serialize(message);\n        if (data != null && data.length > 0) {\n            final byte[] result = new byte[2 + data.length];\n            putShort(result, 0, eventType);\n            System.arraycopy(data, 0, result, 2, data.length);\n            return result;\n        }\n        return null;\n    }\n\n    public EventMessage deserialize(byte[] bytes) throws SerializationException {\n        if (bytes.length < 2) {\n            return null;\n        }\n        final short eventId = getShort(bytes, 0);\n        if (eventId > 0) {\n            final byte[] data = new byte[bytes.length - 2];\n            System.arraycopy(bytes, 2, data, 0, data.length);\n            final EventType eventType = EventType.from(eventId);\n            if (eventType != null) {\n                switch (eventType) {\n                    case ALTER_SYNC_STATE_SET_EVENT:\n                        return this.serializer.deserialize(data, AlterSyncStateSetEvent.class);\n                    case APPLY_BROKER_ID_EVENT:\n                        return this.serializer.deserialize(data, ApplyBrokerIdEvent.class);\n                    case ELECT_MASTER_EVENT:\n                        return this.serializer.deserialize(data, ElectMasterEvent.class);\n                    case CLEAN_BROKER_DATA_EVENT:\n                        return this.serializer.deserialize(data, CleanBrokerDataEvent.class);\n                    case UPDATE_BROKER_ADDRESS:\n                        return this.serializer.deserialize(data, UpdateBrokerAddressEvent.class);\n                    default:\n                        break;\n                }\n            }\n        }\n        return null;\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/event/EventType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.event;\n\n/**\n * Event type (name, id);\n */\npublic enum EventType {\n    ALTER_SYNC_STATE_SET_EVENT(\"AlterSyncStateSetEvent\", (short) 1),\n    APPLY_BROKER_ID_EVENT(\"ApplyBrokerIdEvent\", (short) 2),\n    ELECT_MASTER_EVENT(\"ElectMasterEvent\", (short) 3),\n    READ_EVENT(\"ReadEvent\", (short) 4),\n    CLEAN_BROKER_DATA_EVENT(\"CleanBrokerDataEvent\", (short) 5),\n\n    UPDATE_BROKER_ADDRESS(\"UpdateBrokerAddressEvent\", (short) 6);\n\n    private final String name;\n    private final short id;\n\n    EventType(String name, short id) {\n        this.name = name;\n        this.id = id;\n    }\n\n    public static EventType from(short id) {\n        switch (id) {\n            case 1:\n                return ALTER_SYNC_STATE_SET_EVENT;\n            case 2:\n                return APPLY_BROKER_ID_EVENT;\n            case 3:\n                return ELECT_MASTER_EVENT;\n            case 4:\n                return READ_EVENT;\n            case 5:\n                return CLEAN_BROKER_DATA_EVENT;\n            case 6:\n                return UPDATE_BROKER_ADDRESS;\n        }\n        return null;\n    }\n\n    public String getName() {\n        return name;\n    }\n\n    public short getId() {\n        return id;\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/event/ListEventSerializer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.event;\n\nimport org.apache.commons.lang3.SerializationException;\nimport org.apache.rocketmq.common.utils.FastJsonSerializer;\nimport org.apache.rocketmq.common.utils.Serializer;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\n\nimport java.io.ByteArrayOutputStream;\nimport java.util.ArrayList;\nimport java.util.List;\n\npublic class ListEventSerializer {\n    private ListEventSerializer() {\n    }\n\n    private static final Serializer SERIALIZER = new FastJsonSerializer();\n\n    private static void putShort(byte[] memory, int index, int value) {\n        memory[index] = (byte) (value >>> 8);\n        memory[index + 1] = (byte) value;\n    }\n\n    private static void putShort(ByteArrayOutputStream outputStream, int value) {\n        outputStream.write((byte) (value >>> 8));\n        outputStream.write((byte) value);\n    }\n\n    private static short getShort(byte[] memory, int index) {\n        return (short) (memory[index] << 8 | memory[index + 1] & 0xFF);\n    }\n\n    private static void putInt(byte[] memory, int index, int value) {\n        memory[index] = (byte) (value >>> 24);\n        memory[index + 1] = (byte) (value >>> 16);\n        memory[index + 2] = (byte) (value >>> 8);\n        memory[index + 3] = (byte) value;\n    }\n\n    private static void putInt(ByteArrayOutputStream outputStream, int value) {\n        outputStream.write((byte) (value >>> 24));\n        outputStream.write((byte) (value >>> 16));\n        outputStream.write((byte) (value >>> 8));\n        outputStream.write((byte) value);\n    }\n\n    private static int getInt(byte[] memory, int index) {\n        return memory[index] << 24 | (memory[index + 1] & 0xFF) << 16 | (memory[index + 2] & 0xFF) << 8 | memory[index + 3] & 0xFF;\n    }\n\n    public static byte[] serialize(List<EventMessage> message, Logger log) throws SerializationException {\n        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();\n        for (EventMessage eventMessage : message) {\n            final short eventType = eventMessage.getEventType().getId();\n            final byte[] data = SERIALIZER.serialize(eventMessage);\n            if (data != null && data.length > 0) {\n                putShort(outputStream, eventType);\n                putInt(outputStream, data.length);\n                outputStream.write(data, 0, data.length);\n            } else {\n                log.error(\"serialize event message error, event: {}, this event will be discard\", eventMessage);\n            }\n        }\n        return outputStream.toByteArray();\n    }\n\n    public static List<EventMessage> deserialize(byte[] bytes, Logger log) throws SerializationException {\n        List<EventMessage> eventMessages = new ArrayList<>();\n        if (bytes == null || bytes.length <= 6) {\n            return eventMessages;\n        }\n        int index = 0;\n        while (index < bytes.length) {\n            final short eventId = getShort(bytes, index);\n            index += 2;\n            final int dataLength = getInt(bytes, index);\n            index += 4;\n            if (dataLength > 0) {\n                final byte[] data = new byte[dataLength];\n                System.arraycopy(bytes, index, data, 0, dataLength);\n                final EventType eventType = EventType.from(eventId);\n                if (eventType != null) {\n                    switch (eventType) {\n                        case ALTER_SYNC_STATE_SET_EVENT:\n                            eventMessages.add(SERIALIZER.deserialize(data, AlterSyncStateSetEvent.class));\n                            break;\n                        case APPLY_BROKER_ID_EVENT:\n                            eventMessages.add(SERIALIZER.deserialize(data, ApplyBrokerIdEvent.class));\n                            break;\n                        case ELECT_MASTER_EVENT:\n                            eventMessages.add(SERIALIZER.deserialize(data, ElectMasterEvent.class));\n                            break;\n                        case CLEAN_BROKER_DATA_EVENT:\n                            eventMessages.add(SERIALIZER.deserialize(data, CleanBrokerDataEvent.class));\n                            break;\n                        case UPDATE_BROKER_ADDRESS:\n                            eventMessages.add(SERIALIZER.deserialize(data, UpdateBrokerAddressEvent.class));\n                            break;\n                        default:\n                            log.error(\"deserialize event message error, event id: {}, data: {}\", eventId, data);\n                            break;\n                    }\n                } else {\n                    log.error(\"deserialize event message error, event id: {}, data: {}\", eventId, data);\n                }\n                index += dataLength;\n            } else {\n                log.error(\"deserialize event message error, event id: {}, data length: {}\", eventId, dataLength);\n            }\n        }\n        return eventMessages;\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/event/UpdateBrokerAddressEvent.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.controller.impl.event;\n\npublic class UpdateBrokerAddressEvent implements EventMessage {\n\n    private String clusterName;\n\n    private String brokerName;\n\n    private String brokerAddress;\n\n    private Long brokerId;\n\n    public UpdateBrokerAddressEvent(String clusterName, String brokerName, String brokerAddress, Long brokerId) {\n        this.clusterName = clusterName;\n        this.brokerName = brokerName;\n        this.brokerAddress = brokerAddress;\n        this.brokerId = brokerId;\n    }\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public String getBrokerAddress() {\n        return brokerAddress;\n    }\n\n    public Long getBrokerId() {\n        return brokerId;\n    }\n\n    @Override\n    public String toString() {\n        return \"UpdateBrokerAddressEvent{\" +\n            \"clusterName='\" + clusterName + '\\'' +\n            \", brokerName='\" + brokerName + '\\'' +\n            \", brokerAddress='\" + brokerAddress + '\\'' +\n            \", brokerId=\" + brokerId +\n            '}';\n    }\n\n    @Override\n    public EventType getEventType() {\n        return EventType.UPDATE_BROKER_ADDRESS;\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/BrokerIdentityInfo.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.heartbeat;\n\nimport java.io.Serializable;\nimport org.apache.rocketmq.common.UtilAll;\n\nimport java.util.Objects;\n\npublic class BrokerIdentityInfo implements Serializable {\n\n    private static final long serialVersionUID = 883597359635995567L;\n    private final String clusterName;\n\n    private final String brokerName;\n\n    private final Long brokerId;\n\n    public BrokerIdentityInfo(String clusterName, String brokerName, Long brokerId) {\n        this.clusterName = clusterName;\n        this.brokerName = brokerName;\n        this.brokerId = brokerId;\n    }\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    public Long getBrokerId() {\n        return brokerId;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public boolean isEmpty() {\n        return UtilAll.isBlank(clusterName) && UtilAll.isBlank(brokerName) && brokerId == null;\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj) {\n            return true;\n        }\n        if (obj == null) {\n            return false;\n        }\n\n        if (obj instanceof BrokerIdentityInfo) {\n            BrokerIdentityInfo addr = (BrokerIdentityInfo) obj;\n            return clusterName.equals(addr.clusterName) && brokerName.equals(addr.brokerName) && brokerId.equals(addr.brokerId);\n        }\n        return false;\n    }\n\n    @Override\n    public int hashCode() {\n        return Objects.hash(this.clusterName, this.brokerName, this.brokerId);\n    }\n\n    @Override\n    public String toString() {\n        return \"BrokerIdentityInfo{\" +\n            \"clusterName='\" + clusterName + '\\'' +\n            \", brokerName='\" + brokerName + '\\'' +\n            \", brokerId=\" + brokerId +\n            '}';\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/BrokerLiveInfo.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.heartbeat;\n\nimport io.netty.channel.Channel;\nimport java.io.Serializable;\n\npublic class BrokerLiveInfo implements Serializable {\n    private static final long serialVersionUID = 3612173344946510993L;\n    private final String brokerName;\n\n    private String brokerAddr;\n    private long heartbeatTimeoutMillis;\n    private Channel channel;\n    private long brokerId;\n    private long lastUpdateTimestamp;\n    private int epoch;\n    private long maxOffset;\n    private long confirmOffset;\n    private Integer electionPriority;\n\n    public BrokerLiveInfo(String brokerName, String brokerAddr, long brokerId, long lastUpdateTimestamp,\n        long heartbeatTimeoutMillis, Channel channel, int epoch, long maxOffset, Integer electionPriority) {\n        this.brokerName = brokerName;\n        this.brokerAddr = brokerAddr;\n        this.brokerId = brokerId;\n        this.lastUpdateTimestamp = lastUpdateTimestamp;\n        this.heartbeatTimeoutMillis = heartbeatTimeoutMillis;\n        this.channel = channel;\n        this.epoch = epoch;\n        this.electionPriority = electionPriority;\n        this.maxOffset = maxOffset;\n    }\n\n    public BrokerLiveInfo(String brokerName, String brokerAddr, long brokerId, long lastUpdateTimestamp,\n        long heartbeatTimeoutMillis, Channel channel, int epoch, long maxOffset, Integer electionPriority,\n        long confirmOffset) {\n        this.brokerName = brokerName;\n        this.brokerAddr = brokerAddr;\n        this.brokerId = brokerId;\n        this.lastUpdateTimestamp = lastUpdateTimestamp;\n        this.heartbeatTimeoutMillis = heartbeatTimeoutMillis;\n        this.channel = channel;\n        this.epoch = epoch;\n        this.maxOffset = maxOffset;\n        this.electionPriority = electionPriority;\n        this.confirmOffset = confirmOffset;\n    }\n\n    @Override\n    public String toString() {\n        return \"BrokerLiveInfo{\" +\n            \"brokerName='\" + brokerName + '\\'' +\n            \", brokerAddr='\" + brokerAddr + '\\'' +\n            \", heartbeatTimeoutMillis=\" + heartbeatTimeoutMillis +\n            \", channel=\" + channel +\n            \", brokerId=\" + brokerId +\n            \", lastUpdateTimestamp=\" + lastUpdateTimestamp +\n            \", epoch=\" + epoch +\n            \", maxOffset=\" + maxOffset +\n            \", confirmOffset=\" + confirmOffset +\n            '}';\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public long getHeartbeatTimeoutMillis() {\n        return heartbeatTimeoutMillis;\n    }\n\n    public void setHeartbeatTimeoutMillis(long heartbeatTimeoutMillis) {\n        this.heartbeatTimeoutMillis = heartbeatTimeoutMillis;\n    }\n\n    public Channel getChannel() {\n        return channel;\n    }\n\n    public long getBrokerId() {\n        return brokerId;\n    }\n\n    public void setBrokerId(long brokerId) {\n        this.brokerId = brokerId;\n    }\n\n    public long getLastUpdateTimestamp() {\n        return lastUpdateTimestamp;\n    }\n\n    public void setLastUpdateTimestamp(long lastUpdateTimestamp) {\n        this.lastUpdateTimestamp = lastUpdateTimestamp;\n    }\n\n    public int getEpoch() {\n        return epoch;\n    }\n\n    public void setEpoch(int epoch) {\n        this.epoch = epoch;\n    }\n\n    public long getMaxOffset() {\n        return maxOffset;\n    }\n\n    public void setMaxOffset(long maxOffset) {\n        this.maxOffset = maxOffset;\n    }\n\n    public String getBrokerAddr() {\n        return brokerAddr;\n    }\n\n    public void setConfirmOffset(long confirmOffset) {\n        this.confirmOffset = confirmOffset;\n    }\n\n    public void setElectionPriority(Integer electionPriority) {\n        this.electionPriority = electionPriority;\n    }\n\n    public Integer getElectionPriority() {\n        return electionPriority;\n    }\n\n    public long getConfirmOffset() {\n        return confirmOffset;\n    }\n\n    public void setBrokerAddr(String brokerAddr) {\n        this.brokerAddr = brokerAddr;\n    }\n\n    public void setChannel(Channel channel) {\n        this.channel = channel;\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.heartbeat;\n\nimport io.netty.channel.Channel;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Optional;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.common.ControllerConfig;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.controller.BrokerHeartbeatManager;\nimport org.apache.rocketmq.controller.helper.BrokerLifecycleListener;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\n\npublic class DefaultBrokerHeartbeatManager implements BrokerHeartbeatManager {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.CONTROLLER_LOGGER_NAME);\n\n    private ScheduledExecutorService scheduledService;\n    private ExecutorService executor;\n\n    private final ControllerConfig controllerConfig;\n    private final Map<BrokerIdentityInfo/* brokerIdentity*/, BrokerLiveInfo> brokerLiveTable;\n    private final List<BrokerLifecycleListener> brokerLifecycleListeners;\n\n    public DefaultBrokerHeartbeatManager(final ControllerConfig controllerConfig) {\n        this.controllerConfig = controllerConfig;\n        this.brokerLiveTable = new ConcurrentHashMap<>(256);\n        this.brokerLifecycleListeners = new ArrayList<>();\n    }\n\n    @Override\n    public void start() {\n        this.scheduledService.scheduleAtFixedRate(this::scanNotActiveBroker, 2000, this.controllerConfig.getScanNotActiveBrokerInterval(), TimeUnit.MILLISECONDS);\n    }\n\n    @Override\n    public void shutdown() {\n        this.scheduledService.shutdown();\n        this.executor.shutdown();\n    }\n\n    @Override\n    public void initialize() {\n        this.scheduledService = ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(\"DefaultBrokerHeartbeatManager_scheduledService_\"));\n        this.executor = Executors.newFixedThreadPool(2, new ThreadFactoryImpl(\"DefaultBrokerHeartbeatManager_executorService_\"));\n    }\n\n    public void scanNotActiveBroker() {\n        try {\n            log.info(\"start scanNotActiveBroker\");\n            final Iterator<Map.Entry<BrokerIdentityInfo, BrokerLiveInfo>> iterator = this.brokerLiveTable.entrySet().iterator();\n            while (iterator.hasNext()) {\n                final Map.Entry<BrokerIdentityInfo, BrokerLiveInfo> next = iterator.next();\n                long last = next.getValue().getLastUpdateTimestamp();\n                long timeoutMillis = next.getValue().getHeartbeatTimeoutMillis();\n                if (System.currentTimeMillis() - last > timeoutMillis) {\n                    final Channel channel = next.getValue().getChannel();\n                    iterator.remove();\n                    if (channel != null) {\n                        RemotingHelper.closeChannel(channel);\n                    }\n                    this.executor.submit(() ->\n                        notifyBrokerInActive(next.getKey().getClusterName(), next.getValue().getBrokerName(), next.getValue().getBrokerId()));\n                    log.warn(\"The broker channel {} expired, brokerInfo {}, expired {}ms\", next.getValue().getChannel(), next.getKey(), timeoutMillis);\n                }\n            }\n        } catch (Exception e) {\n            log.error(\"scanNotActiveBroker exception\", e);\n        }\n    }\n\n    private void notifyBrokerInActive(String clusterName, String brokerName, Long brokerId) {\n        for (BrokerLifecycleListener listener : this.brokerLifecycleListeners) {\n            listener.onBrokerInactive(clusterName, brokerName, brokerId);\n        }\n    }\n\n    @Override\n    public void registerBrokerLifecycleListener(BrokerLifecycleListener listener) {\n        this.brokerLifecycleListeners.add(listener);\n    }\n\n    @Override\n    public void onBrokerHeartbeat(String clusterName, String brokerName, String brokerAddr, Long brokerId,\n        Long timeoutMillis, Channel channel, Integer epoch, Long maxOffset, Long confirmOffset,\n        Integer electionPriority) {\n        BrokerIdentityInfo brokerIdentityInfo = new BrokerIdentityInfo(clusterName, brokerName, brokerId);\n        BrokerLiveInfo prev = this.brokerLiveTable.get(brokerIdentityInfo);\n        int realEpoch = Optional.ofNullable(epoch).orElse(-1);\n        long realBrokerId = Optional.ofNullable(brokerId).orElse(-1L);\n        long realMaxOffset = Optional.ofNullable(maxOffset).orElse(-1L);\n        long realConfirmOffset = Optional.ofNullable(confirmOffset).orElse(-1L);\n        long realTimeoutMillis = Optional.ofNullable(timeoutMillis).orElse(DEFAULT_BROKER_CHANNEL_EXPIRED_TIME);\n        int realElectionPriority = Optional.ofNullable(electionPriority).orElse(Integer.MAX_VALUE);\n        if (null == prev) {\n            this.brokerLiveTable.put(brokerIdentityInfo,\n                new BrokerLiveInfo(brokerName,\n                    brokerAddr,\n                    realBrokerId,\n                    System.currentTimeMillis(),\n                    realTimeoutMillis,\n                    channel,\n                    realEpoch,\n                    realMaxOffset,\n                    realElectionPriority));\n            log.info(\"new broker registered, {}, brokerId:{}\", brokerIdentityInfo, realBrokerId);\n        } else {\n            prev.setLastUpdateTimestamp(System.currentTimeMillis());\n            prev.setHeartbeatTimeoutMillis(realTimeoutMillis);\n            prev.setElectionPriority(realElectionPriority);\n            if (realEpoch > prev.getEpoch() || realEpoch == prev.getEpoch() && realMaxOffset > prev.getMaxOffset()) {\n                prev.setEpoch(realEpoch);\n                prev.setMaxOffset(realMaxOffset);\n                prev.setConfirmOffset(realConfirmOffset);\n            }\n        }\n\n    }\n\n    @Override\n    public void onBrokerChannelClose(Channel channel) {\n        BrokerIdentityInfo addrInfo = null;\n        for (Map.Entry<BrokerIdentityInfo, BrokerLiveInfo> entry : this.brokerLiveTable.entrySet()) {\n            if (entry.getValue().getChannel() == channel) {\n                log.info(\"Channel {} inactive, broker {}, addr:{}, id:{}\", entry.getValue().getChannel(), entry.getValue().getBrokerName(), entry.getValue().getBrokerAddr(), entry.getValue().getBrokerId());\n                addrInfo = entry.getKey();\n                this.executor.submit(() ->\n                    notifyBrokerInActive(entry.getKey().getClusterName(), entry.getValue().getBrokerName(), entry.getValue().getBrokerId()));\n                break;\n            }\n        }\n        if (addrInfo != null) {\n            this.brokerLiveTable.remove(addrInfo);\n        }\n    }\n\n    @Override\n    public BrokerLiveInfo getBrokerLiveInfo(String clusterName, String brokerName, Long brokerId) {\n        return this.brokerLiveTable.get(new BrokerIdentityInfo(clusterName, brokerName, brokerId));\n    }\n\n    @Override\n    public boolean isBrokerActive(String clusterName, String brokerName, Long brokerId) {\n        final BrokerLiveInfo info = this.brokerLiveTable.get(new BrokerIdentityInfo(clusterName, brokerName, brokerId));\n        if (info != null) {\n            long last = info.getLastUpdateTimestamp();\n            long timeoutMillis = info.getHeartbeatTimeoutMillis();\n            return (last + timeoutMillis) >= System.currentTimeMillis();\n        }\n        return false;\n    }\n\n    @Override\n    public Map<String, Map<String, Integer>> getActiveBrokersNum() {\n        Map<String, Map<String, Integer>> map = new HashMap<>();\n        this.brokerLiveTable.keySet().stream()\n            .filter(brokerIdentity -> this.isBrokerActive(brokerIdentity.getClusterName(), brokerIdentity.getBrokerName(), brokerIdentity.getBrokerId()))\n            .forEach(id -> {\n                map.computeIfAbsent(id.getClusterName(), k -> new HashMap<>());\n                map.get(id.getClusterName()).compute(id.getBrokerName(), (broker, num) ->\n                    num == null ? 1 : num + 1\n                );\n            });\n        return map;\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/RaftBrokerHeartBeatManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.heartbeat;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.alibaba.fastjson2.TypeReference;\nimport io.netty.channel.Channel;\nimport org.apache.rocketmq.common.ControllerConfig;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.controller.BrokerHeartbeatManager;\nimport org.apache.rocketmq.controller.helper.BrokerLifecycleListener;\nimport org.apache.rocketmq.controller.impl.JRaftController;\nimport org.apache.rocketmq.controller.impl.task.BrokerCloseChannelRequest;\nimport org.apache.rocketmq.controller.impl.task.CheckNotActiveBrokerRequest;\nimport org.apache.rocketmq.controller.impl.task.GetBrokerLiveInfoRequest;\nimport org.apache.rocketmq.controller.impl.task.GetBrokerLiveInfoResponse;\nimport org.apache.rocketmq.controller.impl.task.RaftBrokerHeartBeatEventRequest;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\n\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Optional;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.TimeoutException;\n\npublic class RaftBrokerHeartBeatManager implements BrokerHeartbeatManager {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.CONTROLLER_LOGGER_NAME);\n    private JRaftController controller;\n    private final List<BrokerLifecycleListener> brokerLifecycleListeners = new ArrayList<>();\n    private final ScheduledExecutorService scheduledService;\n    private final ExecutorService executor;\n    private final ControllerConfig controllerConfig;\n\n    private final Map<Channel, BrokerIdentityInfo> brokerChannelIdentityInfoMap = new HashMap<>();\n\n\n    // resolve the scene\n    // when controller all down and startup again, we wait for some time to avoid electing a new leader,which is not necessary\n    private volatile long firstReceivedHeartbeatTime = -1;\n\n    public RaftBrokerHeartBeatManager(ControllerConfig controllerConfig) {\n        this.scheduledService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(\"RaftBrokerHeartbeatManager_scheduledService_\"));\n        this.executor = Executors.newFixedThreadPool(2, new ThreadFactoryImpl(\"RaftBrokerHeartbeatManager_executorService_\"));\n        this.controllerConfig = controllerConfig;\n    }\n\n    public void setController(JRaftController controller) {\n        this.controller = controller;\n    }\n\n    @Override\n    public void initialize() {\n\n    }\n\n    @Override\n    public void start() {\n        this.scheduledService.scheduleAtFixedRate(this::scanNotActiveBroker, 2000, this.controllerConfig.getScanNotActiveBrokerInterval(), TimeUnit.MILLISECONDS);\n    }\n\n    @Override\n    public void shutdown() {\n        this.scheduledService.shutdown();\n        this.executor.shutdown();\n    }\n\n    @Override\n    public void registerBrokerLifecycleListener(BrokerLifecycleListener listener) {\n        brokerLifecycleListeners.add(listener);\n    }\n\n    @Override\n    public void onBrokerHeartbeat(String clusterName, String brokerName, String brokerAddr, Long brokerId,\n        Long timeoutMillis, Channel channel, Integer epoch, Long maxOffset, Long confirmOffset,\n        Integer electionPriority) {\n\n        if (firstReceivedHeartbeatTime == -1) {\n            firstReceivedHeartbeatTime = System.currentTimeMillis();\n        }\n\n        BrokerIdentityInfo brokerIdentityInfo = new BrokerIdentityInfo(clusterName, brokerName, brokerId);\n        int realEpoch = Optional.ofNullable(epoch).orElse(-1);\n        long realBrokerId = Optional.ofNullable(brokerId).orElse(-1L);\n        long realMaxOffset = Optional.ofNullable(maxOffset).orElse(-1L);\n        long realConfirmOffset = Optional.ofNullable(confirmOffset).orElse(-1L);\n        long realTimeoutMillis = Optional.ofNullable(timeoutMillis).orElse(DEFAULT_BROKER_CHANNEL_EXPIRED_TIME);\n        int realElectionPriority = Optional.ofNullable(electionPriority).orElse(Integer.MAX_VALUE);\n        BrokerLiveInfo liveInfo = new BrokerLiveInfo(brokerName,\n            brokerAddr,\n            realBrokerId,\n            System.currentTimeMillis(),\n            realTimeoutMillis,\n            null,\n            realEpoch,\n            realMaxOffset,\n            realElectionPriority,\n            realConfirmOffset);\n        log.info(\"broker {} heart beat\", brokerIdentityInfo);\n        RaftBrokerHeartBeatEventRequest requestHeader = new RaftBrokerHeartBeatEventRequest(brokerIdentityInfo, liveInfo);\n        CompletableFuture<RemotingCommand> future = controller.onBrokerHeartBeat(requestHeader);\n        try {\n            RemotingCommand remotingCommand = future.get(5, java.util.concurrent.TimeUnit.SECONDS);\n            if (remotingCommand.getCode() != ResponseCode.SUCCESS && remotingCommand.getCode() != ResponseCode.CONTROLLER_NOT_LEADER) {\n                throw new RuntimeException(\"on broker heartbeat return invalid code, code: \" + remotingCommand.getCode());\n            }\n        } catch (ExecutionException | InterruptedException | TimeoutException | RuntimeException e) {\n            log.error(\"on broker heartbeat through raft failed\", e);\n        }\n        brokerChannelIdentityInfoMap.put(channel, brokerIdentityInfo);\n    }\n\n    @Override\n    public void onBrokerChannelClose(Channel channel) {\n        BrokerIdentityInfo brokerIdentityInfo = brokerChannelIdentityInfoMap.get(channel);\n        log.info(\"Channel {} inactive, broker identity info: {}\", channel, brokerIdentityInfo);\n        if (brokerIdentityInfo != null) {\n            BrokerCloseChannelRequest requestHeader = new BrokerCloseChannelRequest(brokerIdentityInfo);\n            CompletableFuture<RemotingCommand> future = controller.onBrokerCloseChannel(requestHeader);\n            try {\n                RemotingCommand remotingCommand = future.get(5, java.util.concurrent.TimeUnit.SECONDS);\n                if (remotingCommand.getCode() != ResponseCode.SUCCESS) {\n                    throw new RuntimeException(\"on broker close channel return invalid code, code: \" + remotingCommand.getCode());\n                }\n                this.executor.submit(() -> notifyBrokerInActive(brokerIdentityInfo.getClusterName(), brokerIdentityInfo.getBrokerName(), brokerIdentityInfo.getBrokerId()));\n                brokerChannelIdentityInfoMap.remove(channel);\n            } catch (ExecutionException | InterruptedException | TimeoutException | RuntimeException e) {\n                log.error(\"on broker close channel through raft failed\", e);\n            }\n        }\n    }\n\n    /**\n     * @param brokerIdentityInfo null means get broker live info of all brokers\n     */\n    private Map<BrokerIdentityInfo, BrokerLiveInfo> getBrokerLiveInfo(BrokerIdentityInfo brokerIdentityInfo) {\n        GetBrokerLiveInfoRequest requestHeader;\n        if (brokerIdentityInfo == null) {\n            requestHeader = new GetBrokerLiveInfoRequest();\n        } else {\n            requestHeader = new GetBrokerLiveInfoRequest(brokerIdentityInfo);\n        }\n        CompletableFuture<RemotingCommand> future = controller.getBrokerLiveInfo(requestHeader);\n        try {\n            RemotingCommand remotingCommand = future.get(5, java.util.concurrent.TimeUnit.SECONDS);\n            if (remotingCommand.getCode() != ResponseCode.SUCCESS) {\n                throw new RuntimeException(\"get broker live info return invalid code, code: \" + remotingCommand.getCode());\n            }\n            GetBrokerLiveInfoResponse getBrokerLiveInfoResponse = (GetBrokerLiveInfoResponse) remotingCommand.decodeCommandCustomHeader(GetBrokerLiveInfoResponse.class);\n            return JSON.parseObject(remotingCommand.getBody(), new TypeReference<Map<BrokerIdentityInfo, BrokerLiveInfo>>() {\n            }.getType());\n        } catch (Throwable e) {\n            log.error(\"get broker live info through raft failed\", e);\n        }\n        return new HashMap<>();\n    }\n\n    private void scanNotActiveBroker() {\n        if (!controller.isLeaderState()) {\n            log.info(\"current node is not leader, skip scan not active broker\");\n            return;\n        }\n\n        // if has not received any heartbeat from broker, we do not need to scan\n        if (this.firstReceivedHeartbeatTime == -1 ||\n            this.firstReceivedHeartbeatTime + controllerConfig.getJraftConfig().getjRaftScanWaitTimeoutMs() > System.currentTimeMillis()) {\n            log.info(\"has not received any heartbeat from broker, skip scan not active broker\");\n            return;\n        }\n\n        log.info(\"start scan not active broker\");\n        CheckNotActiveBrokerRequest requestHeader = new CheckNotActiveBrokerRequest();\n        CompletableFuture<RemotingCommand> future = this.controller.checkNotActiveBroker(requestHeader);\n        try {\n            RemotingCommand remotingCommand = future.get(5, java.util.concurrent.TimeUnit.SECONDS);\n            if (remotingCommand.getCode() != ResponseCode.SUCCESS) {\n                throw new RuntimeException(\"check not active broker return invalid code, code: \" + remotingCommand.getCode());\n            }\n            List<BrokerIdentityInfo> notActiveAndNeedReElectBrokerIdentityInfoList = JSON.parseObject(remotingCommand.getBody(), new TypeReference<List<BrokerIdentityInfo>>() {\n            }.getType());\n            if (notActiveAndNeedReElectBrokerIdentityInfoList != null && !notActiveAndNeedReElectBrokerIdentityInfoList.isEmpty()) {\n                notActiveAndNeedReElectBrokerIdentityInfoList.forEach(brokerIdentityInfo -> {\n                    Iterator<Map.Entry<Channel, BrokerIdentityInfo>> iterator = brokerChannelIdentityInfoMap.entrySet().iterator();\n                    Channel channel = null;\n                    while (iterator.hasNext()) {\n                        Map.Entry<Channel, BrokerIdentityInfo> entry = iterator.next();\n                        if (entry.getValue().getBrokerId() == null) {\n                            continue;\n                        }\n                        if (entry.getValue().equals(brokerIdentityInfo)) {\n                            channel = entry.getKey();\n                            RemotingHelper.closeChannel(entry.getKey());\n                            iterator.remove();\n                            break;\n                        }\n                    }\n                    this.executor.submit(() -> notifyBrokerInActive(brokerIdentityInfo.getClusterName(), brokerIdentityInfo.getBrokerName(), brokerIdentityInfo.getBrokerId()));\n                    log.warn(\"The broker channel {} expired, brokerInfo {}\", channel, brokerIdentityInfo);\n                });\n            }\n        } catch (Throwable e) {\n            log.error(\"check not active broker through raft failed\", e);\n        }\n    }\n\n    @Override\n    public BrokerLiveInfo getBrokerLiveInfo(String clusterName, String brokerName, Long brokerId) {\n        log.info(\"get broker live info, clusterName: {}, brokerName: {}, brokerId: {}\", clusterName, brokerName, brokerId);\n        BrokerIdentityInfo brokerIdentityInfo = new BrokerIdentityInfo(clusterName, brokerName, brokerId);\n        Map<BrokerIdentityInfo, BrokerLiveInfo> brokerLiveInfoMap = getBrokerLiveInfo(brokerIdentityInfo);\n        return brokerLiveInfoMap.get(brokerIdentityInfo);\n    }\n\n    @Override\n    public boolean isBrokerActive(String clusterName, String brokerName, Long brokerId) {\n        BrokerLiveInfo info = null;\n        try {\n            info = getBrokerLiveInfo(clusterName, brokerName, brokerId);\n        } catch (RuntimeException e) {\n            log.error(\"get broker live info failed\", e);\n            return false;\n        }\n\n        if (info != null) {\n            long last = info.getLastUpdateTimestamp();\n            long timeoutMillis = info.getHeartbeatTimeoutMillis();\n            return (last + timeoutMillis) >= System.currentTimeMillis();\n        }\n        return false;\n    }\n\n    @Override\n    public Map<String, Map<String, Integer>> getActiveBrokersNum() {\n        Map<String, Map<String, Integer>> map = new HashMap<>();\n        Map<BrokerIdentityInfo, BrokerLiveInfo> brokerLiveInfoMap = getBrokerLiveInfo(null);\n        brokerLiveInfoMap.keySet().stream()\n            .filter(brokerIdentity -> this.isBrokerActive(brokerIdentity.getClusterName(), brokerIdentity.getBrokerName(), brokerIdentity.getBrokerId()))\n            .forEach(id -> {\n                map.computeIfAbsent(id.getClusterName(), k -> new HashMap<>());\n                map.get(id.getClusterName()).compute(id.getBrokerName(), (broker, num) ->\n                    num == null ? 1 : num + 1\n                );\n            });\n        return map;\n    }\n\n    private void notifyBrokerInActive(String clusterName, String brokerName, Long brokerId) {\n        log.info(\"Broker {}-{}-{} inactive\", clusterName, brokerName, brokerId);\n        for (BrokerLifecycleListener listener : this.brokerLifecycleListeners) {\n            listener.onBrokerInactive(clusterName, brokerName, brokerId);\n        }\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/manager/BrokerReplicaInfo.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.manager;\n\nimport java.io.Serializable;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.Pair;\n\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.atomic.AtomicLong;\n\n/**\n * Broker replicas info, mapping from brokerAddress to {brokerId, brokerHaAddress}.\n */\npublic class BrokerReplicaInfo implements Serializable {\n    private final String clusterName;\n\n    private final String brokerName;\n\n    // Start from 1\n    private final AtomicLong nextAssignBrokerId;\n\n    private final Map<Long/*brokerId*/, Pair<String/*ipAddress*/, String/*registerCheckCode*/>> brokerIdInfo;\n\n    public BrokerReplicaInfo(String clusterName, String brokerName) {\n        this.clusterName = clusterName;\n        this.brokerName = brokerName;\n        this.nextAssignBrokerId = new AtomicLong(MixAll.FIRST_BROKER_CONTROLLER_ID);\n        this.brokerIdInfo = new ConcurrentHashMap<>();\n    }\n\n    public void removeBrokerId(final Long brokerId) {\n        this.brokerIdInfo.remove(brokerId);\n    }\n\n    public Long getNextAssignBrokerId() {\n        return nextAssignBrokerId.get();\n    }\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void addBroker(final Long brokerId, final String ipAddress, final String registerCheckCode) {\n        this.brokerIdInfo.put(brokerId, new Pair<>(ipAddress, registerCheckCode));\n        this.nextAssignBrokerId.incrementAndGet();\n    }\n\n    public boolean isBrokerExist(final Long brokerId) {\n        return this.brokerIdInfo.containsKey(brokerId);\n    }\n\n    public Set<Long> getAllBroker() {\n        return new HashSet<>(this.brokerIdInfo.keySet());\n    }\n\n    public Map<Long, String> getBrokerIdTable() {\n        Map<Long/*brokerId*/, String/*address*/> map = new HashMap<>(this.brokerIdInfo.size());\n        this.brokerIdInfo.forEach((id, pair) -> {\n            map.put(id, pair.getObject1());\n        });\n        return map;\n    }\n\n    public String getBrokerAddress(final Long brokerId) {\n        if (brokerId == null) {\n            return null;\n        }\n        Pair<String, String> pair = this.brokerIdInfo.get(brokerId);\n        if (pair != null) {\n            return pair.getObject1();\n        }\n        return null;\n    }\n\n    public String getBrokerRegisterCheckCode(final Long brokerId) {\n        if (brokerId == null) {\n            return null;\n        }\n        Pair<String, String> pair = this.brokerIdInfo.get(brokerId);\n        if (pair != null) {\n            return pair.getObject2();\n        }\n        return null;\n    }\n\n    public void updateBrokerAddress(final Long brokerId, final String brokerAddress) {\n        if (brokerId == null)\n            return;\n        Pair<String, String> oldPair = this.brokerIdInfo.get(brokerId);\n        if (oldPair != null) {\n            this.brokerIdInfo.put(brokerId, new Pair<>(brokerAddress, oldPair.getObject2()));\n        }\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/manager/RaftReplicasInfoManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.manager;\n\nimport com.alibaba.fastjson2.JSON;\nimport org.apache.rocketmq.common.ControllerConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.controller.helper.BrokerValidPredicate;\nimport org.apache.rocketmq.controller.impl.event.ControllerResult;\nimport org.apache.rocketmq.controller.impl.heartbeat.BrokerIdentityInfo;\nimport org.apache.rocketmq.controller.impl.heartbeat.BrokerLiveInfo;\nimport org.apache.rocketmq.controller.impl.task.BrokerCloseChannelRequest;\nimport org.apache.rocketmq.controller.impl.task.BrokerCloseChannelResponse;\nimport org.apache.rocketmq.controller.impl.task.CheckNotActiveBrokerRequest;\nimport org.apache.rocketmq.controller.impl.task.CheckNotActiveBrokerResponse;\nimport org.apache.rocketmq.controller.impl.task.GetBrokerLiveInfoRequest;\nimport org.apache.rocketmq.controller.impl.task.GetBrokerLiveInfoResponse;\nimport org.apache.rocketmq.controller.impl.task.RaftBrokerHeartBeatEventRequest;\nimport org.apache.rocketmq.controller.impl.task.RaftBrokerHeartBeatEventResponse;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\n\nimport java.io.ByteArrayOutputStream;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.stream.Collectors;\n\npublic class RaftReplicasInfoManager extends ReplicasInfoManager {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.CONTROLLER_LOGGER_NAME);\n    private final Map<BrokerIdentityInfo/* brokerIdentity*/, BrokerLiveInfo> brokerLiveTable = new ConcurrentHashMap<>(256);\n\n    public RaftReplicasInfoManager(ControllerConfig controllerConfig) {\n        super(controllerConfig);\n    }\n\n    public ControllerResult<GetBrokerLiveInfoResponse> getBrokerLiveInfo(final GetBrokerLiveInfoRequest request) {\n        BrokerIdentityInfo brokerIdentityInfo = request.getBrokerIdentity();\n        ControllerResult<GetBrokerLiveInfoResponse> result = new ControllerResult<>(new GetBrokerLiveInfoResponse());\n        Map<BrokerIdentityInfo/* brokerIdentity*/, BrokerLiveInfo> resBrokerLiveTable = new HashMap<>();\n        if (brokerIdentityInfo == null || brokerIdentityInfo.isEmpty()) {\n            resBrokerLiveTable.putAll(this.brokerLiveTable);\n        } else {\n            if (brokerLiveTable.containsKey(brokerIdentityInfo)) {\n                resBrokerLiveTable.put(brokerIdentityInfo, brokerLiveTable.get(brokerIdentityInfo));\n            } else {\n                log.warn(\"GetBrokerLiveInfo failed, brokerIdentityInfo: {} not exist\", brokerIdentityInfo);\n                result.setCodeAndRemark(ResponseCode.CONTROLLER_BROKER_LIVE_INFO_NOT_EXISTS, \"brokerIdentityInfo not exist\");\n            }\n        }\n        try {\n            result.setBody(JSON.toJSONBytes(resBrokerLiveTable));\n        } catch (Throwable e) {\n            log.error(\"json serialize resBrokerLiveTable {} error\", resBrokerLiveTable, e);\n            result.setCodeAndRemark(ResponseCode.SYSTEM_ERROR, \"serialize error\");\n        }\n\n        return result;\n    }\n\n    public ControllerResult<RaftBrokerHeartBeatEventResponse> onBrokerHeartBeat(\n        RaftBrokerHeartBeatEventRequest request) {\n        BrokerIdentityInfo brokerIdentityInfo = request.getBrokerIdentityInfo();\n        BrokerLiveInfo brokerLiveInfo = request.getBrokerLiveInfo();\n        ControllerResult<RaftBrokerHeartBeatEventResponse> result = new ControllerResult<>(new RaftBrokerHeartBeatEventResponse());\n        BrokerLiveInfo prev = brokerLiveTable.computeIfAbsent(brokerIdentityInfo, identityInfo -> {\n            log.info(\"new broker registered, brokerIdentityInfo: {}\", identityInfo);\n            return brokerLiveInfo;\n        });\n        prev.setLastUpdateTimestamp(brokerLiveInfo.getLastUpdateTimestamp());\n        prev.setHeartbeatTimeoutMillis(brokerLiveInfo.getHeartbeatTimeoutMillis());\n        prev.setElectionPriority(brokerLiveInfo.getElectionPriority());\n        if (brokerLiveInfo.getEpoch() > prev.getEpoch() || brokerLiveInfo.getEpoch() == prev.getEpoch() && brokerLiveInfo.getMaxOffset() > prev.getMaxOffset()) {\n            prev.setEpoch(brokerLiveInfo.getEpoch());\n            prev.setMaxOffset(brokerLiveInfo.getMaxOffset());\n            prev.setConfirmOffset(brokerLiveInfo.getConfirmOffset());\n        }\n        return result;\n    }\n\n    public ControllerResult<BrokerCloseChannelResponse> onBrokerCloseChannel(BrokerCloseChannelRequest request) {\n        BrokerIdentityInfo brokerIdentityInfo = request.getBrokerIdentityInfo();\n        ControllerResult<BrokerCloseChannelResponse> result = new ControllerResult<>(new BrokerCloseChannelResponse());\n        if (brokerIdentityInfo == null || brokerIdentityInfo.isEmpty()) {\n            log.warn(\"onBrokerCloseChannel failed, brokerIdentityInfo is null\");\n        } else {\n            brokerLiveTable.remove(brokerIdentityInfo);\n            log.info(\"onBrokerCloseChannel success, brokerIdentityInfo: {}\", brokerIdentityInfo);\n        }\n        return result;\n    }\n\n    public ControllerResult<CheckNotActiveBrokerResponse> checkNotActiveBroker(CheckNotActiveBrokerRequest request) {\n        List<BrokerIdentityInfo> notActiveBrokerIdentityInfoList = new ArrayList<>();\n        long checkTime = request.getCheckTimeMillis();\n        final Iterator<Map.Entry<BrokerIdentityInfo, BrokerLiveInfo>> iterator = this.brokerLiveTable.entrySet().iterator();\n        while (iterator.hasNext()) {\n            final Map.Entry<BrokerIdentityInfo, BrokerLiveInfo> next = iterator.next();\n            long last = next.getValue().getLastUpdateTimestamp();\n            long timeoutMillis = next.getValue().getHeartbeatTimeoutMillis();\n            if (checkTime - last > timeoutMillis) {\n                notActiveBrokerIdentityInfoList.add(next.getKey());\n                iterator.remove();\n                log.warn(\"Broker expired, brokerInfo {}, expired {}ms\", next.getKey(), timeoutMillis);\n            }\n        }\n        List<String> needReElectBrokerNames = scanNeedReelectBrokerSets(new BrokerValidPredicate() {\n            @Override\n            public boolean check(String clusterName, String brokerName, Long brokerId) {\n                return !isBrokerActive(clusterName, brokerName, brokerId, checkTime);\n            }\n        });\n        Set<String> alreadyReportedBrokerName = notActiveBrokerIdentityInfoList.stream()\n            .map(BrokerIdentityInfo::getBrokerName)\n            .collect(Collectors.toSet());\n        // avoid to duplicate report, filter by name,\n        // because BrokerIdentityInfo in needReElectBrokerNames does not have brokerId or clusterName\n        notActiveBrokerIdentityInfoList.addAll(needReElectBrokerNames.stream()\n            .filter(brokerName -> !alreadyReportedBrokerName.contains(brokerName))\n            .map(brokerName -> new BrokerIdentityInfo(null, brokerName, null))\n            .collect(Collectors.toList()));\n        ControllerResult<CheckNotActiveBrokerResponse> result = new ControllerResult<>(new CheckNotActiveBrokerResponse());\n        try {\n            result.setBody(JSON.toJSONBytes(notActiveBrokerIdentityInfoList));\n        } catch (Throwable e) {\n            log.error(\"json serialize notActiveBrokerIdentityInfoList {} error\", notActiveBrokerIdentityInfoList, e);\n            result.setCodeAndRemark(ResponseCode.SYSTEM_ERROR, \"serialize error\");\n        }\n        return result;\n    }\n\n    public boolean isBrokerActive(String clusterName, String brokerName, Long brokerId, long invokeTime) {\n        final BrokerLiveInfo info = this.brokerLiveTable.get(new BrokerIdentityInfo(clusterName, brokerName, brokerId));\n        if (info != null) {\n            long last = info.getLastUpdateTimestamp();\n            long timeoutMillis = info.getHeartbeatTimeoutMillis();\n            return (last + timeoutMillis) >= invokeTime;\n        }\n        return false;\n    }\n\n    public BrokerLiveInfo getBrokerLiveInfo(String clusterName, String brokerName, Long brokerId) {\n        return this.brokerLiveTable.get(new BrokerIdentityInfo(clusterName, brokerName, brokerId));\n    }\n\n    @Override\n    public byte[] serialize() throws Throwable {\n        try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {\n            final byte[] superSerialize = super.serialize();\n            putInt(outputStream, superSerialize.length);\n            outputStream.write(superSerialize);\n            putInt(outputStream, this.brokerLiveTable.size());\n            for (Map.Entry<BrokerIdentityInfo, BrokerLiveInfo> entry : brokerLiveTable.entrySet()) {\n                final byte[] brokerIdentityInfo = hessianSerialize(entry.getKey());\n                final byte[] brokerLiveInfo = hessianSerialize(entry.getValue());\n                putInt(outputStream, brokerIdentityInfo.length);\n                outputStream.write(brokerIdentityInfo);\n                putInt(outputStream, brokerLiveInfo.length);\n                outputStream.write(brokerLiveInfo);\n            }\n            return outputStream.toByteArray();\n        } catch (Throwable e) {\n            log.error(\"serialize replicaInfoTable or syncStateSetInfoTable error\", e);\n            throw e;\n        }\n    }\n\n    @Override\n    public void deserializeFrom(byte[] data) throws Throwable {\n        int index = 0;\n        this.brokerLiveTable.clear();\n\n        try {\n            int superTableSize = getInt(data, index);\n            index += 4;\n            byte[] superTableData = new byte[superTableSize];\n            System.arraycopy(data, index, superTableData, 0, superTableSize);\n            super.deserializeFrom(superTableData);\n            index += superTableSize;\n            int brokerLiveTableSize = getInt(data, index);\n            index += 4;\n            for (int i = 0; i < brokerLiveTableSize; i++) {\n                int brokerIdentityInfoLength = getInt(data, index);\n                index += 4;\n                byte[] brokerIdentityInfoArray = new byte[brokerIdentityInfoLength];\n                System.arraycopy(data, index, brokerIdentityInfoArray, 0, brokerIdentityInfoLength);\n                BrokerIdentityInfo brokerIdentityInfo = (BrokerIdentityInfo) hessianDeserialize(brokerIdentityInfoArray);\n                index += brokerIdentityInfoLength;\n                int brokerLiveInfoLength = getInt(data, index);\n                index += 4;\n                byte[] brokerLiveInfoArray = new byte[brokerLiveInfoLength];\n                System.arraycopy(data, index, brokerLiveInfoArray, 0, brokerLiveInfoLength);\n                BrokerLiveInfo brokerLiveInfo = (BrokerLiveInfo) hessianDeserialize(brokerLiveInfoArray);\n                index += brokerLiveInfoLength;\n                this.brokerLiveTable.put(brokerIdentityInfo, brokerLiveInfo);\n            }\n        } catch (Throwable e) {\n            log.error(\"deserialize replicaInfoTable or syncStateSetInfoTable error\", e);\n            throw e;\n        }\n    }\n\n    public static class BrokerValidPredicateWithInvokeTime implements BrokerValidPredicate {\n        private final long invokeTime;\n        private final RaftReplicasInfoManager raftBrokerHeartBeatManager;\n\n        public BrokerValidPredicateWithInvokeTime(long invokeTime, RaftReplicasInfoManager raftBrokerHeartBeatManager) {\n            this.invokeTime = invokeTime;\n            this.raftBrokerHeartBeatManager = raftBrokerHeartBeatManager;\n        }\n\n        @Override\n        public boolean check(String clusterName, String brokerName, Long brokerId) {\n            return raftBrokerHeartBeatManager.isBrokerActive(clusterName, brokerName, brokerId, invokeTime);\n        }\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/manager/ReplicasInfoManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.manager;\n\nimport com.caucho.hessian.io.Hessian2Input;\nimport com.caucho.hessian.io.Hessian2Output;\nimport com.caucho.hessian.io.SerializerFactory;\nimport java.io.ByteArrayInputStream;\nimport java.io.IOException;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.ControllerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.controller.elect.ElectPolicy;\nimport org.apache.rocketmq.controller.helper.BrokerValidPredicate;\nimport org.apache.rocketmq.controller.impl.event.AlterSyncStateSetEvent;\nimport org.apache.rocketmq.controller.impl.event.ApplyBrokerIdEvent;\nimport org.apache.rocketmq.controller.impl.event.CleanBrokerDataEvent;\nimport org.apache.rocketmq.controller.impl.event.ControllerResult;\nimport org.apache.rocketmq.controller.impl.event.ElectMasterEvent;\nimport org.apache.rocketmq.controller.impl.event.EventMessage;\nimport org.apache.rocketmq.controller.impl.event.EventType;\nimport org.apache.rocketmq.controller.impl.event.UpdateBrokerAddressEvent;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;\nimport org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo;\nimport org.apache.rocketmq.remoting.protocol.body.ElectMasterResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.SyncStateSet;\nimport org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerResponseHeader;\n\nimport java.io.ByteArrayOutputStream;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.stream.Collectors;\nimport java.util.stream.Stream;\n\n/**\n * The manager that manages the replicas info for all brokers. We can think of this class as the controller's memory\n * state machine. If the upper layer want to update the statemachine, it must sequentially call its methods.\n */\npublic class ReplicasInfoManager {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.CONTROLLER_LOGGER_NAME);\n\n    protected static final SerializerFactory SERIALIZER_FACTORY = new SerializerFactory();\n    protected final ControllerConfig controllerConfig;\n    private final Map<String/* brokerName */, BrokerReplicaInfo> replicaInfoTable;\n    private final Map<String/* brokerName */, SyncStateInfo> syncStateSetInfoTable;\n\n    protected static byte[] hessianSerialize(Object object) throws IOException {\n        try (ByteArrayOutputStream bout = new ByteArrayOutputStream()) {\n            Hessian2Output hessianOut = new Hessian2Output(bout);\n            hessianOut.setSerializerFactory(SERIALIZER_FACTORY);\n            hessianOut.writeObject(object);\n            hessianOut.close();\n            return bout.toByteArray();\n        }\n    }\n\n    protected static Object hessianDeserialize(byte[] data) throws IOException {\n        try (ByteArrayInputStream bin = new ByteArrayInputStream(data, 0, data.length)) {\n            Hessian2Input hin = new Hessian2Input(bin);\n            hin.setSerializerFactory(new SerializerFactory());\n            Object o =  hin.readObject();\n            hin.close();\n            return o;\n        }\n    }\n\n    public ReplicasInfoManager(final ControllerConfig config) {\n        this.controllerConfig = config;\n        this.replicaInfoTable = new ConcurrentHashMap<String, BrokerReplicaInfo>();\n        this.syncStateSetInfoTable = new ConcurrentHashMap<String, SyncStateInfo>();\n    }\n\n    public ControllerResult<AlterSyncStateSetResponseHeader> alterSyncStateSet(\n        final AlterSyncStateSetRequestHeader request, final SyncStateSet syncStateSet,\n        final BrokerValidPredicate brokerAlivePredicate) {\n        final String brokerName = request.getBrokerName();\n        final ControllerResult<AlterSyncStateSetResponseHeader> result = new ControllerResult<>(new AlterSyncStateSetResponseHeader());\n        final AlterSyncStateSetResponseHeader response = result.getResponse();\n\n        if (!isContainsBroker(brokerName)) {\n            result.setCodeAndRemark(ResponseCode.CONTROLLER_ALTER_SYNC_STATE_SET_FAILED, \"Broker metadata is not existed\");\n            return result;\n        }\n        final Set<Long> newSyncStateSet = syncStateSet.getSyncStateSet();\n        final SyncStateInfo syncStateInfo = this.syncStateSetInfoTable.get(brokerName);\n        final BrokerReplicaInfo brokerReplicaInfo = this.replicaInfoTable.get(brokerName);\n\n        // Check whether the oldSyncStateSet is equal with newSyncStateSet\n        final Set<Long> oldSyncStateSet = syncStateInfo.getSyncStateSet();\n        if (oldSyncStateSet.size() == newSyncStateSet.size() && oldSyncStateSet.containsAll(newSyncStateSet)) {\n            String err = \"The newSyncStateSet is equal with oldSyncStateSet, no needed to update syncStateSet\";\n            LOGGER.warn(\"{}\", err);\n            result.setCodeAndRemark(ResponseCode.CONTROLLER_ALTER_SYNC_STATE_SET_FAILED, err);\n            return result;\n        }\n\n        // Check master\n        if (syncStateInfo.getMasterBrokerId() == null || !syncStateInfo.getMasterBrokerId().equals(request.getMasterBrokerId())) {\n            String err = String.format(\"Rejecting alter syncStateSet request because the current leader is:{%s}, not {%s}\",\n                syncStateInfo.getMasterBrokerId(), request.getMasterBrokerId());\n            LOGGER.error(\"{}\", err);\n            result.setCodeAndRemark(ResponseCode.CONTROLLER_INVALID_MASTER, err);\n            return result;\n        }\n\n        // Check master epoch\n        if (request.getMasterEpoch() != syncStateInfo.getMasterEpoch()) {\n            String err = String.format(\"Rejecting alter syncStateSet request because the current master epoch is:{%d}, not {%d}\",\n                syncStateInfo.getMasterEpoch(), request.getMasterEpoch());\n            LOGGER.error(\"{}\", err);\n            result.setCodeAndRemark(ResponseCode.CONTROLLER_FENCED_MASTER_EPOCH, err);\n            return result;\n        }\n\n        // Check syncStateSet epoch\n        if (syncStateSet.getSyncStateSetEpoch() != syncStateInfo.getSyncStateSetEpoch()) {\n            String err = String.format(\"Rejecting alter syncStateSet request because the current syncStateSet epoch is:{%d}, not {%d}\",\n                syncStateInfo.getSyncStateSetEpoch(), syncStateSet.getSyncStateSetEpoch());\n            LOGGER.error(\"{}\", err);\n            result.setCodeAndRemark(ResponseCode.CONTROLLER_FENCED_SYNC_STATE_SET_EPOCH, err);\n            return result;\n        }\n\n        // Check newSyncStateSet correctness\n        for (Long replica : newSyncStateSet) {\n            if (!brokerReplicaInfo.isBrokerExist(replica)) {\n                String err = String.format(\"Rejecting alter syncStateSet request because the replicas {%s} don't exist\", replica);\n                LOGGER.error(\"{}\", err);\n                result.setCodeAndRemark(ResponseCode.CONTROLLER_INVALID_REPLICAS, err);\n                return result;\n            }\n            if (!brokerAlivePredicate.check(brokerReplicaInfo.getClusterName(), brokerReplicaInfo.getBrokerName(), replica)) {\n                String err = String.format(\"Rejecting alter syncStateSet request because the replicas {%s} don't alive\", replica);\n                LOGGER.error(err);\n                result.setCodeAndRemark(ResponseCode.CONTROLLER_BROKER_NOT_ALIVE, err);\n                return result;\n            }\n        }\n\n        if (!newSyncStateSet.contains(syncStateInfo.getMasterBrokerId())) {\n            String err = String.format(\"Rejecting alter syncStateSet request because the newSyncStateSet don't contains origin leader {%s}\", syncStateInfo.getMasterBrokerId());\n            LOGGER.error(err);\n            result.setCodeAndRemark(ResponseCode.CONTROLLER_ALTER_SYNC_STATE_SET_FAILED, err);\n            return result;\n        }\n\n        // Generate event\n        int epoch = syncStateInfo.getSyncStateSetEpoch() + 1;\n        response.setNewSyncStateSetEpoch(epoch);\n        result.setBody(new SyncStateSet(newSyncStateSet, epoch).encode());\n        final AlterSyncStateSetEvent event = new AlterSyncStateSetEvent(brokerName, newSyncStateSet);\n        result.addEvent(event);\n        return result;\n    }\n\n    public ControllerResult<ElectMasterResponseHeader> electMaster(final ElectMasterRequestHeader request,\n        final ElectPolicy electPolicy) {\n        final String brokerName = request.getBrokerName();\n        final Long brokerId = request.getBrokerId();\n        final ControllerResult<ElectMasterResponseHeader> result = new ControllerResult<>(new ElectMasterResponseHeader());\n        final ElectMasterResponseHeader response = result.getResponse();\n        if (!isContainsBroker(brokerName)) {\n            // this broker set hasn't been registered\n            result.setCodeAndRemark(ResponseCode.CONTROLLER_BROKER_NEED_TO_BE_REGISTERED, \"Broker hasn't been registered\");\n            return result;\n        }\n\n        final SyncStateInfo syncStateInfo = this.syncStateSetInfoTable.get(brokerName);\n        final BrokerReplicaInfo brokerReplicaInfo = this.replicaInfoTable.get(brokerName);\n        final Set<Long> syncStateSet = syncStateInfo.getSyncStateSet();\n        final Long oldMaster = syncStateInfo.getMasterBrokerId();\n        Set<Long> allReplicaBrokers = controllerConfig.isEnableElectUncleanMaster() ? brokerReplicaInfo.getAllBroker() : null;\n        Long newMaster = null;\n\n        if (syncStateInfo.isFirstTimeForElect()) {\n            // If never have a master in this broker set, in other words, it is the first time to elect a master\n            // elect it as the first master\n            newMaster = brokerId;\n        }\n\n        // elect by policy\n        if (newMaster == null || newMaster == -1) {\n            // we should assign this assignedBrokerId when the brokerAddress need to be elected by force\n            Long assignedBrokerId = request.getDesignateElect() ? brokerId : null;\n            newMaster = electPolicy.elect(brokerReplicaInfo.getClusterName(), brokerReplicaInfo.getBrokerName(), syncStateSet, allReplicaBrokers, oldMaster, assignedBrokerId);\n        }\n\n        if (newMaster != null && newMaster.equals(oldMaster)) {\n            // old master still valid, change nothing\n            String err = String.format(\"The old master %s is still alive, not need to elect new master for broker %s\", oldMaster, brokerReplicaInfo.getBrokerName());\n            LOGGER.warn(\"{}\", err);\n            // the master still exist\n            response.setMasterEpoch(syncStateInfo.getMasterEpoch());\n            response.setSyncStateSetEpoch(syncStateInfo.getSyncStateSetEpoch());\n            response.setMasterBrokerId(oldMaster);\n            response.setMasterAddress(brokerReplicaInfo.getBrokerAddress(oldMaster));\n\n            result.setBody(new ElectMasterResponseBody(syncStateSet).encode());\n            result.setCodeAndRemark(ResponseCode.CONTROLLER_MASTER_STILL_EXIST, err);\n            return result;\n        }\n\n        // a new master is elected\n        if (newMaster != null) {\n            final int masterEpoch = syncStateInfo.getMasterEpoch();\n            final int syncStateSetEpoch = syncStateInfo.getSyncStateSetEpoch();\n            final HashSet<Long> newSyncStateSet = new HashSet<>();\n            newSyncStateSet.add(newMaster);\n\n            response.setMasterBrokerId(newMaster);\n            response.setMasterAddress(brokerReplicaInfo.getBrokerAddress(newMaster));\n            response.setMasterEpoch(masterEpoch + 1);\n            response.setSyncStateSetEpoch(syncStateSetEpoch + 1);\n            ElectMasterResponseBody responseBody = new ElectMasterResponseBody(newSyncStateSet);\n\n            BrokerMemberGroup brokerMemberGroup = buildBrokerMemberGroup(brokerReplicaInfo);\n            if (null != brokerMemberGroup) {\n                responseBody.setBrokerMemberGroup(brokerMemberGroup);\n            }\n\n            result.setBody(responseBody.encode());\n            final ElectMasterEvent event = new ElectMasterEvent(brokerName, newMaster);\n            result.addEvent(event);\n            LOGGER.info(\"Elect new master {} for broker {}\", newMaster, brokerName);\n            return result;\n        }\n        // If elect failed and the electMaster is triggered by controller (we can figure it out by brokerAddress),\n        // we still need to apply an ElectMasterEvent to tell the statemachine\n        // that the master was shutdown and no new master was elected.\n        if (request.getBrokerId() == null || request.getBrokerId() == -1) {\n            final ElectMasterEvent event = new ElectMasterEvent(false, brokerName);\n            result.addEvent(event);\n            result.setCodeAndRemark(ResponseCode.CONTROLLER_MASTER_NOT_AVAILABLE, \"Old master has down and failed to elect a new broker master\");\n        } else {\n            result.setCodeAndRemark(ResponseCode.CONTROLLER_ELECT_MASTER_FAILED, \"Failed to elect a new master\");\n        }\n        LOGGER.warn(\"Failed to elect a new master for broker {}\", brokerName);\n        return result;\n    }\n\n    private BrokerMemberGroup buildBrokerMemberGroup(final BrokerReplicaInfo brokerReplicaInfo) {\n        if (brokerReplicaInfo != null) {\n            final BrokerMemberGroup group = new BrokerMemberGroup(brokerReplicaInfo.getClusterName(), brokerReplicaInfo.getBrokerName());\n            final Map<Long, String> brokerIdTable = brokerReplicaInfo.getBrokerIdTable();\n            final Map<Long, String> memberGroup = new HashMap<>();\n            brokerIdTable.forEach((id, addr) -> memberGroup.put(id, addr));\n            group.setBrokerAddrs(memberGroup);\n            return group;\n        }\n        return null;\n    }\n\n    public ControllerResult<GetNextBrokerIdResponseHeader> getNextBrokerId(final GetNextBrokerIdRequestHeader request) {\n        final String clusterName = request.getClusterName();\n        final String brokerName = request.getBrokerName();\n        BrokerReplicaInfo brokerReplicaInfo = this.replicaInfoTable.get(brokerName);\n        final ControllerResult<GetNextBrokerIdResponseHeader> result = new ControllerResult<>(new GetNextBrokerIdResponseHeader(clusterName, brokerName));\n        final GetNextBrokerIdResponseHeader response = result.getResponse();\n        if (brokerReplicaInfo == null) {\n            // means that none of brokers in this broker-set are registered\n            response.setNextBrokerId(MixAll.FIRST_BROKER_CONTROLLER_ID);\n        } else {\n            response.setNextBrokerId(brokerReplicaInfo.getNextAssignBrokerId());\n        }\n        return result;\n    }\n\n    public ControllerResult<ApplyBrokerIdResponseHeader> applyBrokerId(final ApplyBrokerIdRequestHeader request) {\n        final String clusterName = request.getClusterName();\n        final String brokerName = request.getBrokerName();\n        final Long brokerId = request.getAppliedBrokerId();\n        final String registerCheckCode = request.getRegisterCheckCode();\n        final String brokerAddress = registerCheckCode.split(\";\")[0];\n        BrokerReplicaInfo brokerReplicaInfo = this.replicaInfoTable.get(brokerName);\n        final ControllerResult<ApplyBrokerIdResponseHeader> result = new ControllerResult<>(new ApplyBrokerIdResponseHeader(clusterName, brokerName));\n        final ApplyBrokerIdEvent event = new ApplyBrokerIdEvent(clusterName, brokerName, brokerAddress, brokerId, registerCheckCode);\n        // broker-set unregistered\n        if (brokerReplicaInfo == null) {\n            // first brokerId\n            if (brokerId == MixAll.FIRST_BROKER_CONTROLLER_ID) {\n                result.addEvent(event);\n            } else {\n                result.setCodeAndRemark(ResponseCode.CONTROLLER_BROKER_ID_INVALID, String.format(\"Broker-set: %s hasn't been registered in controller, but broker try to apply brokerId: %d\", brokerName, brokerId));\n            }\n            return result;\n        }\n        // broker-set registered\n        if (!brokerReplicaInfo.isBrokerExist(brokerId) || registerCheckCode.equals(brokerReplicaInfo.getBrokerRegisterCheckCode(brokerId))) {\n            // if brokerId hasn't been assigned or brokerId was assigned to this broker\n            result.addEvent(event);\n            return result;\n        }\n        result.setCodeAndRemark(ResponseCode.CONTROLLER_BROKER_ID_INVALID, String.format(\"Fail to apply brokerId: %d in broker-set: %s\", brokerId, brokerName));\n        return result;\n    }\n\n    public ControllerResult<RegisterBrokerToControllerResponseHeader> registerBroker(\n        final RegisterBrokerToControllerRequestHeader request, final BrokerValidPredicate alivePredicate) {\n        final String brokerAddress = request.getBrokerAddress();\n        final String brokerName = request.getBrokerName();\n        final String clusterName = request.getClusterName();\n        final Long brokerId = request.getBrokerId();\n        final ControllerResult<RegisterBrokerToControllerResponseHeader> result = new ControllerResult<>(new RegisterBrokerToControllerResponseHeader(clusterName, brokerName));\n        final RegisterBrokerToControllerResponseHeader response = result.getResponse();\n        if (!isContainsBroker(brokerName)) {\n            result.setCodeAndRemark(ResponseCode.CONTROLLER_BROKER_NEED_TO_BE_REGISTERED, String.format(\"Broker-set: %s hasn't been registered in controller\", brokerName));\n            return result;\n        }\n        final BrokerReplicaInfo brokerReplicaInfo = this.replicaInfoTable.get(brokerName);\n        final SyncStateInfo syncStateInfo = this.syncStateSetInfoTable.get(brokerName);\n        if (!brokerReplicaInfo.isBrokerExist(brokerId)) {\n            result.setCodeAndRemark(ResponseCode.CONTROLLER_BROKER_NEED_TO_BE_REGISTERED, String.format(\"BrokerId: %d hasn't been registered in broker-set: %s\", brokerId, brokerName));\n            return result;\n        }\n        if (syncStateInfo.isMasterExist() && alivePredicate.check(clusterName, brokerName, syncStateInfo.getMasterBrokerId())) {\n            // if master still exist\n            response.setMasterBrokerId(syncStateInfo.getMasterBrokerId());\n            response.setMasterAddress(brokerReplicaInfo.getBrokerAddress(response.getMasterBrokerId()));\n            response.setMasterEpoch(syncStateInfo.getMasterEpoch());\n            response.setSyncStateSetEpoch(syncStateInfo.getSyncStateSetEpoch());\n        }\n        result.setBody(new SyncStateSet(syncStateInfo.getSyncStateSet(), syncStateInfo.getSyncStateSetEpoch()).encode());\n        // if this broker's address has been changed, we need to update it\n        if (!brokerAddress.equals(brokerReplicaInfo.getBrokerAddress(brokerId))) {\n            final UpdateBrokerAddressEvent event = new UpdateBrokerAddressEvent(clusterName, brokerName, brokerAddress, brokerId);\n            result.addEvent(event);\n        }\n        return result;\n    }\n\n    public ControllerResult<GetReplicaInfoResponseHeader> getReplicaInfo(final GetReplicaInfoRequestHeader request) {\n        final String brokerName = request.getBrokerName();\n        final ControllerResult<GetReplicaInfoResponseHeader> result = new ControllerResult<>(new GetReplicaInfoResponseHeader());\n        final GetReplicaInfoResponseHeader response = result.getResponse();\n        if (isContainsBroker(brokerName)) {\n            // If exist broker metadata, just return metadata\n            final SyncStateInfo syncStateInfo = this.syncStateSetInfoTable.get(brokerName);\n            final BrokerReplicaInfo brokerReplicaInfo = this.replicaInfoTable.get(brokerName);\n            final Long masterBrokerId = syncStateInfo.getMasterBrokerId();\n            response.setMasterBrokerId(masterBrokerId);\n            response.setMasterAddress(brokerReplicaInfo.getBrokerAddress(masterBrokerId));\n            response.setMasterEpoch(syncStateInfo.getMasterEpoch());\n            result.setBody(new SyncStateSet(syncStateInfo.getSyncStateSet(), syncStateInfo.getSyncStateSetEpoch()).encode());\n            return result;\n        }\n        result.setCodeAndRemark(ResponseCode.CONTROLLER_BROKER_METADATA_NOT_EXIST, \"Broker metadata is not existed\");\n        return result;\n    }\n\n    public ControllerResult<Void> getSyncStateData(final List<String> brokerNames,\n        final BrokerValidPredicate brokerAlivePredicate) {\n        final ControllerResult<Void> result = new ControllerResult<>();\n        final BrokerReplicasInfo brokerReplicasInfo = new BrokerReplicasInfo();\n        for (String brokerName : brokerNames) {\n            if (isContainsBroker(brokerName)) {\n                // If exist broker metadata, just return metadata\n                final SyncStateInfo syncStateInfo = this.syncStateSetInfoTable.get(brokerName);\n                final BrokerReplicaInfo brokerReplicaInfo = this.replicaInfoTable.get(brokerName);\n                final Set<Long> syncStateSet = syncStateInfo.getSyncStateSet();\n                final Long masterBrokerId = syncStateInfo.getMasterBrokerId();\n                final ArrayList<BrokerReplicasInfo.ReplicaIdentity> inSyncReplicas = new ArrayList<>();\n                final ArrayList<BrokerReplicasInfo.ReplicaIdentity> notInSyncReplicas = new ArrayList<>();\n\n                if (brokerReplicaInfo == null) {\n                    continue;\n                }\n\n                brokerReplicaInfo.getBrokerIdTable().forEach((brokerId, brokerAddress) -> {\n                    Boolean isAlive = brokerAlivePredicate.check(brokerReplicaInfo.getClusterName(), brokerName, brokerId);\n                    BrokerReplicasInfo.ReplicaIdentity replica = new BrokerReplicasInfo.ReplicaIdentity(brokerName, brokerId, brokerAddress);\n                    replica.setAlive(isAlive);\n                    if (syncStateSet.contains(brokerId)) {\n                        inSyncReplicas.add(replica);\n                    } else {\n                        notInSyncReplicas.add(replica);\n                    }\n                });\n\n                final BrokerReplicasInfo.ReplicasInfo inSyncState = new BrokerReplicasInfo.ReplicasInfo(masterBrokerId, brokerReplicaInfo.getBrokerAddress(masterBrokerId), syncStateInfo.getMasterEpoch(), syncStateInfo.getSyncStateSetEpoch(),\n                    inSyncReplicas, notInSyncReplicas);\n                brokerReplicasInfo.addReplicaInfo(brokerName, inSyncState);\n            }\n        }\n        result.setBody(brokerReplicasInfo.encode());\n        return result;\n    }\n\n    public ControllerResult<Void> cleanBrokerData(final CleanControllerBrokerDataRequestHeader requestHeader,\n        final BrokerValidPredicate validPredicate) {\n        final ControllerResult<Void> result = new ControllerResult<>();\n\n        final String clusterName = requestHeader.getClusterName();\n        final String brokerName = requestHeader.getBrokerName();\n        final String brokerControllerIdsToClean = requestHeader.getBrokerControllerIdsToClean();\n\n        Set<Long> brokerIdSet = null;\n        if (!requestHeader.isCleanLivingBroker()) {\n            //if SyncStateInfo.masterAddress is not empty, at least one broker with the same BrokerName is alive\n            SyncStateInfo syncStateInfo = this.syncStateSetInfoTable.get(brokerName);\n            if (StringUtils.isBlank(brokerControllerIdsToClean) && null != syncStateInfo && syncStateInfo.getMasterBrokerId() != null) {\n                String remark = String.format(\"Broker %s is still alive, clean up failure\", requestHeader.getBrokerName());\n                result.setCodeAndRemark(ResponseCode.CONTROLLER_INVALID_CLEAN_BROKER_METADATA, remark);\n                return result;\n            }\n            if (StringUtils.isNotBlank(brokerControllerIdsToClean)) {\n                try {\n                    brokerIdSet = Stream.of(brokerControllerIdsToClean.split(\";\")).map(idStr -> Long.valueOf(idStr)).collect(Collectors.toSet());\n                } catch (NumberFormatException numberFormatException) {\n                    String remark = String.format(\"Please set the option <brokerControllerIdsToClean> according to the format, exception: %s\", numberFormatException);\n                    result.setCodeAndRemark(ResponseCode.CONTROLLER_INVALID_CLEAN_BROKER_METADATA, remark);\n                    return result;\n                }\n                for (Long brokerId : brokerIdSet) {\n                    if (validPredicate.check(clusterName, brokerName, brokerId)) {\n                        String remark = String.format(\"Broker [%s,  %s] is still alive, clean up failure\", requestHeader.getBrokerName(), brokerId);\n                        result.setCodeAndRemark(ResponseCode.CONTROLLER_INVALID_CLEAN_BROKER_METADATA, remark);\n                        return result;\n                    }\n                }\n            }\n        }\n        if (isContainsBroker(brokerName)) {\n            final CleanBrokerDataEvent event = new CleanBrokerDataEvent(brokerName, brokerIdSet);\n            result.addEvent(event);\n            return result;\n        }\n        result.setCodeAndRemark(ResponseCode.CONTROLLER_INVALID_CLEAN_BROKER_METADATA, String.format(\"Broker %s is not existed,clean broker data failure.\", brokerName));\n        return result;\n    }\n\n    public List<String/*BrokerName*/> scanNeedReelectBrokerSets(final BrokerValidPredicate validPredicate) {\n        List<String> needReelectBrokerSets = new LinkedList<>();\n        this.syncStateSetInfoTable.forEach((brokerName, syncStateInfo) -> {\n            Long masterBrokerId = syncStateInfo.getMasterBrokerId();\n            String clusterName = syncStateInfo.getClusterName();\n            // Now master is inactive\n            if (masterBrokerId != null && !validPredicate.check(clusterName, brokerName, masterBrokerId)) {\n                // Still at least one broker alive\n                Set<Long> brokerIds = this.replicaInfoTable.get(brokerName).getBrokerIdTable().keySet();\n                boolean alive = brokerIds.stream().anyMatch(id -> validPredicate.check(clusterName, brokerName, id));\n                if (alive) {\n                    needReelectBrokerSets.add(brokerName);\n                }\n            }\n        });\n        return needReelectBrokerSets;\n    }\n\n    /**\n     * Apply events to memory statemachine.\n     *\n     * @param event event message\n     */\n    public void applyEvent(final EventMessage event) {\n        final EventType type = event.getEventType();\n        switch (type) {\n            case ALTER_SYNC_STATE_SET_EVENT:\n                handleAlterSyncStateSet((AlterSyncStateSetEvent) event);\n                break;\n            case APPLY_BROKER_ID_EVENT:\n                handleApplyBrokerId((ApplyBrokerIdEvent) event);\n                break;\n            case ELECT_MASTER_EVENT:\n                handleElectMaster((ElectMasterEvent) event);\n                break;\n            case CLEAN_BROKER_DATA_EVENT:\n                handleCleanBrokerDataEvent((CleanBrokerDataEvent) event);\n                break;\n            case UPDATE_BROKER_ADDRESS:\n                handleUpdateBrokerAddress((UpdateBrokerAddressEvent) event);\n                break;\n            default:\n                break;\n        }\n    }\n\n    private void handleAlterSyncStateSet(final AlterSyncStateSetEvent event) {\n        final String brokerName = event.getBrokerName();\n        if (isContainsBroker(brokerName)) {\n            final SyncStateInfo syncStateInfo = this.syncStateSetInfoTable.get(brokerName);\n            syncStateInfo.updateSyncStateSetInfo(event.getNewSyncStateSet());\n        }\n    }\n\n    private void handleApplyBrokerId(final ApplyBrokerIdEvent event) {\n        final String brokerName = event.getBrokerName();\n        if (isContainsBroker(brokerName)) {\n            final BrokerReplicaInfo brokerReplicaInfo = this.replicaInfoTable.get(brokerName);\n            if (!brokerReplicaInfo.isBrokerExist(event.getNewBrokerId())) {\n                brokerReplicaInfo.addBroker(event.getNewBrokerId(), event.getBrokerAddress(), event.getRegisterCheckCode());\n            }\n        } else {\n            // First time to register in this broker set\n            // Initialize the replicaInfo about this broker set\n            final String clusterName = event.getClusterName();\n            final BrokerReplicaInfo brokerReplicaInfo = new BrokerReplicaInfo(clusterName, brokerName);\n            brokerReplicaInfo.addBroker(event.getNewBrokerId(), event.getBrokerAddress(), event.getRegisterCheckCode());\n            this.replicaInfoTable.put(brokerName, brokerReplicaInfo);\n            final SyncStateInfo syncStateInfo = new SyncStateInfo(clusterName, brokerName);\n            // Initialize an empty syncStateInfo for this broker set\n            this.syncStateSetInfoTable.put(brokerName, syncStateInfo);\n        }\n    }\n\n    private void handleUpdateBrokerAddress(final UpdateBrokerAddressEvent event) {\n        final String brokerName = event.getBrokerName();\n        final String brokerAddress = event.getBrokerAddress();\n        final Long brokerId = event.getBrokerId();\n        BrokerReplicaInfo brokerReplicaInfo = this.replicaInfoTable.get(brokerName);\n        brokerReplicaInfo.updateBrokerAddress(brokerId, brokerAddress);\n    }\n\n    private void handleElectMaster(final ElectMasterEvent event) {\n        final String brokerName = event.getBrokerName();\n        final Long newMaster = event.getNewMasterBrokerId();\n        if (isContainsBroker(brokerName)) {\n            final SyncStateInfo syncStateInfo = this.syncStateSetInfoTable.get(brokerName);\n\n            if (event.getNewMasterElected()) {\n                // Record new master\n                syncStateInfo.updateMasterInfo(newMaster);\n\n                // Record new newSyncStateSet list\n                final HashSet<Long> newSyncStateSet = new HashSet<>();\n                newSyncStateSet.add(newMaster);\n                syncStateInfo.updateSyncStateSetInfo(newSyncStateSet);\n            } else {\n                // If new master was not elected, which means old master was shutdown and the newSyncStateSet list had no more replicas\n                // So we should delete old master, but retain newSyncStateSet list.\n                syncStateInfo.updateMasterInfo(null);\n            }\n            return;\n        }\n        LOGGER.error(\"Receive an ElectMasterEvent which contains the un-registered broker, event = {}\", event);\n    }\n\n    private void handleCleanBrokerDataEvent(final CleanBrokerDataEvent event) {\n\n        final String brokerName = event.getBrokerName();\n        final Set<Long> brokerIdSetToClean = event.getBrokerIdSetToClean();\n\n        if (null == brokerIdSetToClean || brokerIdSetToClean.isEmpty()) {\n            this.replicaInfoTable.remove(brokerName);\n            this.syncStateSetInfoTable.remove(brokerName);\n            return;\n        }\n        if (!isContainsBroker(brokerName)) {\n            return;\n        }\n        final BrokerReplicaInfo brokerReplicaInfo = this.replicaInfoTable.get(brokerName);\n        final SyncStateInfo syncStateInfo = this.syncStateSetInfoTable.get(brokerName);\n        for (Long brokerId : brokerIdSetToClean) {\n            brokerReplicaInfo.removeBrokerId(brokerId);\n            syncStateInfo.removeFromSyncState(brokerId);\n        }\n        if (brokerReplicaInfo.getBrokerIdTable().isEmpty()) {\n            this.replicaInfoTable.remove(brokerName);\n        }\n        if (syncStateInfo.getSyncStateSet().isEmpty()) {\n            this.syncStateSetInfoTable.remove(brokerName);\n        }\n    }\n\n    /**\n     * Is the broker existed in the memory metadata\n     *\n     * @return true if both existed in replicaInfoTable and inSyncReplicasInfoTable\n     */\n    private boolean isContainsBroker(final String brokerName) {\n        return this.replicaInfoTable.containsKey(brokerName) && this.syncStateSetInfoTable.containsKey(brokerName);\n    }\n\n    protected void putInt(ByteArrayOutputStream outputStream, int value) {\n        outputStream.write((byte) (value >>> 24));\n        outputStream.write((byte) (value >>> 16));\n        outputStream.write((byte) (value >>> 8));\n        outputStream.write((byte) value);\n    }\n\n    protected int getInt(byte[] memory, int index) {\n        return memory[index] << 24 | (memory[index + 1] & 0xFF) << 16 | (memory[index + 2] & 0xFF) << 8 | memory[index + 3] & 0xFF;\n    }\n\n    public byte[] serialize() throws Throwable {\n        try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {\n            putInt(outputStream, this.replicaInfoTable.size());\n            for (Map.Entry<String, BrokerReplicaInfo> entry : replicaInfoTable.entrySet()) {\n                final byte[] brokerName = entry.getKey().getBytes(StandardCharsets.UTF_8);\n                byte[] brokerReplicaInfo = hessianSerialize(entry.getValue());\n                putInt(outputStream, brokerName.length);\n                outputStream.write(brokerName);\n                putInt(outputStream, brokerReplicaInfo.length);\n                outputStream.write(brokerReplicaInfo);\n            }\n            putInt(outputStream, this.syncStateSetInfoTable.size());\n            for (Map.Entry<String, SyncStateInfo> entry : syncStateSetInfoTable.entrySet()) {\n                final byte[] brokerName = entry.getKey().getBytes(StandardCharsets.UTF_8);\n                byte[] syncStateInfo = hessianSerialize(entry.getValue());\n                putInt(outputStream, brokerName.length);\n                outputStream.write(brokerName);\n                putInt(outputStream, syncStateInfo.length);\n                outputStream.write(syncStateInfo);\n            }\n            return outputStream.toByteArray();\n        } catch (Throwable e) {\n            LOGGER.error(\"serialize replicaInfoTable or syncStateSetInfoTable error\", e);\n            throw e;\n        }\n    }\n\n    public void deserializeFrom(byte[] data) throws Throwable {\n        int index = 0;\n        this.replicaInfoTable.clear();\n        this.syncStateSetInfoTable.clear();\n\n        try {\n            int replicaInfoTableSize = getInt(data, index);\n            index += 4;\n            for (int i = 0; i < replicaInfoTableSize; i++) {\n                int brokerNameLength = getInt(data, index);\n                index += 4;\n                String brokerName = new String(data, index, brokerNameLength, StandardCharsets.UTF_8);\n                index += brokerNameLength;\n                int brokerReplicaInfoLength = getInt(data, index);\n                index += 4;\n                byte[] brokerReplicaInfoArray = new byte[brokerReplicaInfoLength];\n                System.arraycopy(data, index, brokerReplicaInfoArray, 0, brokerReplicaInfoLength);\n                BrokerReplicaInfo brokerReplicaInfo = (BrokerReplicaInfo) hessianDeserialize(brokerReplicaInfoArray);\n                index += brokerReplicaInfoLength;\n                this.replicaInfoTable.put(brokerName, brokerReplicaInfo);\n            }\n            int syncStateSetInfoTableSize = getInt(data, index);\n            index += 4;\n            for (int i = 0; i < syncStateSetInfoTableSize; i++) {\n                int brokerNameLength = getInt(data, index);\n                index += 4;\n                String brokerName = new String(data, index, brokerNameLength, StandardCharsets.UTF_8);\n                index += brokerNameLength;\n                int syncStateInfoLength = getInt(data, index);\n                index += 4;\n                byte[] syncStateInfoArray = new byte[syncStateInfoLength];\n                System.arraycopy(data, index, syncStateInfoArray, 0, syncStateInfoLength);\n                SyncStateInfo syncStateInfo = (SyncStateInfo) hessianDeserialize(syncStateInfoArray);\n                index += syncStateInfoLength;\n                this.syncStateSetInfoTable.put(brokerName, syncStateInfo);\n            }\n        } catch (Throwable e) {\n            LOGGER.error(\"deserialize replicaInfoTable or syncStateSetInfoTable error\", e);\n            throw e;\n        }\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/manager/SyncStateInfo.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.manager;\n\nimport java.io.Serializable;\nimport java.util.Collections;\nimport java.util.HashSet;\nimport java.util.Set;\nimport java.util.concurrent.atomic.AtomicInteger;\n\n/**\n * Manages the syncStateSet of broker replicas.\n */\npublic class SyncStateInfo implements Serializable {\n    private final String clusterName;\n    private final String brokerName;\n    private final AtomicInteger masterEpoch;\n    private final AtomicInteger syncStateSetEpoch;\n\n    private Set<Long/*brokerId*/> syncStateSet;\n\n    private Long masterBrokerId;\n\n    public SyncStateInfo(String clusterName, String brokerName) {\n        this.clusterName = clusterName;\n        this.brokerName = brokerName;\n        this.masterEpoch = new AtomicInteger(0);\n        this.syncStateSetEpoch = new AtomicInteger(0);\n        this.syncStateSet = Collections.emptySet();\n    }\n\n    public void updateMasterInfo(Long masterBrokerId) {\n        this.masterBrokerId = masterBrokerId;\n        this.masterEpoch.incrementAndGet();\n    }\n\n    public void updateSyncStateSetInfo(Set<Long> newSyncStateSet) {\n        this.syncStateSet = new HashSet<>(newSyncStateSet);\n        this.syncStateSetEpoch.incrementAndGet();\n    }\n\n    public boolean isFirstTimeForElect() {\n        return this.masterEpoch.get() == 0;\n    }\n\n    public boolean isMasterExist() {\n        return masterBrokerId != null;\n    }\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public Set<Long> getSyncStateSet() {\n        return new HashSet<>(syncStateSet);\n    }\n\n    public int getSyncStateSetEpoch() {\n        return syncStateSetEpoch.get();\n    }\n\n    public Long getMasterBrokerId() {\n        return masterBrokerId;\n    }\n\n    public int getMasterEpoch() {\n        return masterEpoch.get();\n    }\n\n    public void removeFromSyncState(final Long brokerId) {\n        syncStateSet.remove(brokerId);\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/task/BrokerCloseChannelRequest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.task;\n\nimport org.apache.rocketmq.controller.impl.heartbeat.BrokerIdentityInfo;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class BrokerCloseChannelRequest implements CommandCustomHeader {\n    @CFNullable\n    private String clusterName;\n\n    @CFNullable\n    private String brokerName;\n\n    @CFNullable\n    private Long brokerId;\n\n    public BrokerCloseChannelRequest() {\n        this.clusterName = null;\n        this.brokerName = null;\n        this.brokerId = null;\n    }\n\n    public BrokerCloseChannelRequest(BrokerIdentityInfo brokerIdentityInfo) {\n        this.clusterName = brokerIdentityInfo.getClusterName();\n        this.brokerName = brokerIdentityInfo.getBrokerName();\n        this.brokerId = brokerIdentityInfo.getBrokerId();\n    }\n\n    public BrokerIdentityInfo getBrokerIdentityInfo() {\n        return new BrokerIdentityInfo(this.clusterName, this.brokerName, this.brokerId);\n    }\n\n    public void setBrokerIdentityInfo(BrokerIdentityInfo brokerIdentityInfo) {\n        this.clusterName = brokerIdentityInfo.getClusterName();\n        this.brokerName = brokerIdentityInfo.getBrokerName();\n        this.brokerId = brokerIdentityInfo.getBrokerId();\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    @Override\n    public String toString() {\n        return \"BrokerCloseChannelRequest{\" +\n            \"clusterName='\" + clusterName + '\\'' +\n            \", brokerName='\" + brokerName + '\\'' +\n            \", brokerId=\" + brokerId +\n            '}';\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/task/BrokerCloseChannelResponse.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.task;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class BrokerCloseChannelResponse implements CommandCustomHeader {\n    public BrokerCloseChannelResponse() {\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    @Override\n    public String toString() {\n        return \"BrokerCloseChannelResponse{}\";\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/task/CheckNotActiveBrokerRequest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.task;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class CheckNotActiveBrokerRequest implements CommandCustomHeader {\n    private final Long checkTimeMillis = System.currentTimeMillis();\n\n    public CheckNotActiveBrokerRequest() {\n    }\n\n    public Long getCheckTimeMillis() {\n        return checkTimeMillis;\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    @Override\n    public String toString() {\n        return \"CheckNotActiveBrokerRequest{\" +\n            \"checkTimeMillis=\" + checkTimeMillis +\n            '}';\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/task/CheckNotActiveBrokerResponse.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.task;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class CheckNotActiveBrokerResponse implements CommandCustomHeader {\n    public CheckNotActiveBrokerResponse() {\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    @Override\n    public String toString() {\n        return \"CheckNotActiveBrokerResponse{}\";\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/task/GetBrokerLiveInfoRequest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.task;\n\nimport org.apache.rocketmq.controller.impl.heartbeat.BrokerIdentityInfo;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class GetBrokerLiveInfoRequest implements CommandCustomHeader {\n    private String clusterName;\n\n    private String brokerName;\n\n    private Long brokerId;\n\n    public GetBrokerLiveInfoRequest() {\n        this.clusterName = null;\n        this.brokerName = null;\n        this.brokerId = null;\n    }\n\n    /**\n     * @param brokerIdentity The BrokerIdentityInfo that needs to be queried, if it is null, it means obtaining BrokerLiveInfo for all brokers\n     */\n    public GetBrokerLiveInfoRequest(BrokerIdentityInfo brokerIdentity) {\n        this.clusterName = brokerIdentity.getClusterName();\n        this.brokerName = brokerIdentity.getBrokerName();\n        this.brokerId = brokerIdentity.getBrokerId();\n    }\n\n    public BrokerIdentityInfo getBrokerIdentity() {\n        return new BrokerIdentityInfo(this.clusterName, this.brokerName, this.brokerId);\n    }\n\n    public void setBrokerIdentity(BrokerIdentityInfo brokerIdentity) {\n        this.clusterName = brokerIdentity.getClusterName();\n        this.brokerName = brokerIdentity.getBrokerName();\n        this.brokerId = brokerIdentity.getBrokerId();\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    @Override\n    public String toString() {\n        return \"GetBrokerLiveInfoRequest{\" +\n            \"brokerIdentity=\" + getBrokerIdentity() +\n            '}';\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/task/GetBrokerLiveInfoResponse.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.task;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class GetBrokerLiveInfoResponse implements CommandCustomHeader {\n    public GetBrokerLiveInfoResponse() {\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    @Override\n    public String toString() {\n        return \"GetBrokerLiveInfoResponse{}\";\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/task/GetSyncStateDataRequest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.task;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class GetSyncStateDataRequest implements CommandCustomHeader {\n    private final Long invokeTime = System.currentTimeMillis();\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public GetSyncStateDataRequest() {\n\n    }\n\n    public Long getInvokeTime() {\n        return invokeTime;\n    }\n\n    @Override\n    public String toString() {\n        return \"GetSyncStateDataRequest{\" +\n            \"invokeTime=\" + invokeTime +\n            '}';\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/task/RaftBrokerHeartBeatEventRequest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.task;\n\nimport org.apache.rocketmq.controller.impl.heartbeat.BrokerIdentityInfo;\nimport org.apache.rocketmq.controller.impl.heartbeat.BrokerLiveInfo;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class RaftBrokerHeartBeatEventRequest implements CommandCustomHeader {\n    // brokerIdentityInfo\n    private String clusterNameIdentityInfo;\n\n    private String brokerNameIdentityInfo;\n\n    private Long brokerIdIdentityInfo;\n\n    // brokerLiveInfo\n    private String brokerName;\n    private String brokerAddr;\n    private Long heartbeatTimeoutMillis;\n    private Long brokerId;\n    private Long lastUpdateTimestamp;\n    private Integer epoch;\n    private Long maxOffset;\n    private Long confirmOffset;\n    private Integer electionPriority;\n\n    public RaftBrokerHeartBeatEventRequest() {\n    }\n\n    public RaftBrokerHeartBeatEventRequest(BrokerIdentityInfo brokerIdentityInfo, BrokerLiveInfo brokerLiveInfo) {\n        this.clusterNameIdentityInfo = brokerIdentityInfo.getClusterName();\n        this.brokerNameIdentityInfo = brokerIdentityInfo.getBrokerName();\n        this.brokerIdIdentityInfo = brokerIdentityInfo.getBrokerId();\n\n        this.brokerName = brokerLiveInfo.getBrokerName();\n        this.brokerAddr = brokerLiveInfo.getBrokerAddr();\n        this.heartbeatTimeoutMillis = brokerLiveInfo.getHeartbeatTimeoutMillis();\n        this.brokerId = brokerLiveInfo.getBrokerId();\n        this.lastUpdateTimestamp = brokerLiveInfo.getLastUpdateTimestamp();\n        this.epoch = brokerLiveInfo.getEpoch();\n        this.maxOffset = brokerLiveInfo.getMaxOffset();\n        this.confirmOffset = brokerLiveInfo.getConfirmOffset();\n        this.electionPriority = brokerLiveInfo.getElectionPriority();\n    }\n\n    public BrokerIdentityInfo getBrokerIdentityInfo() {\n        return new BrokerIdentityInfo(clusterNameIdentityInfo, brokerNameIdentityInfo, brokerIdIdentityInfo);\n    }\n\n    public void setBrokerIdentityInfo(BrokerIdentityInfo brokerIdentityInfo) {\n        this.clusterNameIdentityInfo = brokerIdentityInfo.getClusterName();\n        this.brokerNameIdentityInfo = brokerIdentityInfo.getBrokerName();\n        this.brokerIdIdentityInfo = brokerIdentityInfo.getBrokerId();\n    }\n\n    public BrokerLiveInfo getBrokerLiveInfo() {\n        return new BrokerLiveInfo(brokerName, brokerAddr, brokerId, lastUpdateTimestamp, heartbeatTimeoutMillis, null, epoch, maxOffset, electionPriority, confirmOffset);\n    }\n\n    public void setBrokerLiveInfo(BrokerLiveInfo brokerLiveInfo) {\n        this.brokerName = brokerLiveInfo.getBrokerName();\n        this.brokerAddr = brokerLiveInfo.getBrokerAddr();\n        this.heartbeatTimeoutMillis = brokerLiveInfo.getHeartbeatTimeoutMillis();\n        this.brokerId = brokerLiveInfo.getBrokerId();\n        this.lastUpdateTimestamp = brokerLiveInfo.getLastUpdateTimestamp();\n        this.epoch = brokerLiveInfo.getEpoch();\n        this.maxOffset = brokerLiveInfo.getMaxOffset();\n        this.confirmOffset = brokerLiveInfo.getConfirmOffset();\n        this.electionPriority = brokerLiveInfo.getElectionPriority();\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    @Override\n    public String toString() {\n        return \"RaftBrokerHeartBeatEventRequest{\" +\n            \"brokerIdentityInfo=\" + getBrokerIdentityInfo() +\n            \", brokerLiveInfo=\" + getBrokerLiveInfo() +\n            \"}\";\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/impl/task/RaftBrokerHeartBeatEventResponse.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.task;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class RaftBrokerHeartBeatEventResponse implements CommandCustomHeader {\n    public RaftBrokerHeartBeatEventResponse() {\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    @Override\n    public String toString() {\n        return \"RaftBrokerHeartBeatEventResponse{}\";\n    }\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsConstant.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.controller.metrics;\n\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\npublic class ControllerMetricsConstant {\n\n    public static final String LABEL_ADDRESS = \"address\";\n    public static final String LABEL_GROUP = \"group\";\n    public static final String LABEL_PEER_ID = \"peer_id\";\n    public static final String LABEL_AGGREGATION = \"aggregation\";\n    public static final String AGGREGATION_DELTA = \"delta\";\n\n    public static final String OPEN_TELEMETRY_METER_NAME = \"controller\";\n\n    public static final String GAUGE_ROLE = \"role\";\n\n    // unit: B\n    public static final String GAUGE_DLEDGER_DISK_USAGE = \"dledger_disk_usage\";\n\n    public static final String GAUGE_ACTIVE_BROKER_NUM = \"active_broker_num\";\n\n    public static final String COUNTER_REQUEST_TOTAL = \"request_total\";\n\n    public static final String COUNTER_DLEDGER_OP_TOTAL = \"dledger_op_total\";\n\n    public static final String COUNTER_ELECTION_TOTAL = \"election_total\";\n\n    // unit: us\n    public static final String HISTOGRAM_REQUEST_LATENCY = \"request_latency\";\n\n    // unit: us\n    public static final String HISTOGRAM_DLEDGER_OP_LATENCY = \"dledger_op_latency\";\n\n    public static final String LABEL_CLUSTER_NAME = \"cluster\";\n\n    public static final String LABEL_BROKER_SET = \"broker_set\";\n\n    public static final String LABEL_REQUEST_TYPE = \"request_type\";\n\n    public static final String LABEL_REQUEST_HANDLE_STATUS = \"request_handle_status\";\n\n    public static final String LABEL_DLEDGER_OPERATION = \"dledger_operation\";\n\n    public static final String LABEL_DLEDGER_OPERATION_STATUS = \"dLedger_operation_status\";\n\n    public static final String LABEL_ELECTION_RESULT = \"election_result\";\n\n    public enum RequestType {\n        CONTROLLER_ALTER_SYNC_STATE_SET(RequestCode.CONTROLLER_ALTER_SYNC_STATE_SET),\n\n        CONTROLLER_ELECT_MASTER(RequestCode.CONTROLLER_ELECT_MASTER),\n\n        CONTROLLER_REGISTER_BROKER(RequestCode.CONTROLLER_REGISTER_BROKER),\n\n        CONTROLLER_GET_REPLICA_INFO(RequestCode.CONTROLLER_GET_REPLICA_INFO),\n\n        CONTROLLER_GET_METADATA_INFO(RequestCode.CONTROLLER_GET_METADATA_INFO),\n\n        CONTROLLER_GET_SYNC_STATE_DATA(RequestCode.CONTROLLER_GET_SYNC_STATE_DATA),\n\n        CONTROLLER_GET_BROKER_EPOCH_CACHE(RequestCode.GET_BROKER_EPOCH_CACHE),\n\n        CONTROLLER_NOTIFY_BROKER_ROLE_CHANGED(RequestCode.NOTIFY_BROKER_ROLE_CHANGED),\n\n        CONTROLLER_BROKER_HEARTBEAT(RequestCode.BROKER_HEARTBEAT),\n\n        CONTROLLER_UPDATE_CONTROLLER_CONFIG(RequestCode.UPDATE_CONTROLLER_CONFIG),\n\n        CONTROLLER_GET_CONTROLLER_CONFIG(RequestCode.GET_CONTROLLER_CONFIG),\n\n        CONTROLLER_CLEAN_BROKER_DATA(RequestCode.CLEAN_BROKER_DATA),\n\n        CONTROLLER_GET_NEXT_BROKER_ID(RequestCode.CONTROLLER_GET_NEXT_BROKER_ID),\n\n        CONTROLLER_APPLY_BROKER_ID(RequestCode.CONTROLLER_APPLY_BROKER_ID);\n\n        private final int code;\n\n        RequestType(int code) {\n            this.code = code;\n        }\n\n        public static String getLowerCaseNameByCode(int code) {\n            for (RequestType requestType : RequestType.values()) {\n                if (requestType.code == code) {\n                    return requestType.name();\n                }\n            }\n            return null;\n        }\n    }\n\n    public enum RequestHandleStatus {\n        SUCCESS,\n        FAILED,\n        TIMEOUT;\n\n        public String getLowerCaseName() {\n            return this.name().toLowerCase();\n        }\n    }\n\n    public enum DLedgerOperation {\n        APPEND;\n\n        public String getLowerCaseName() {\n            return this.name().toLowerCase();\n        }\n    }\n\n    public enum DLedgerOperationStatus {\n        SUCCESS,\n        FAILED,\n        TIMEOUT;\n\n        public String getLowerCaseName() {\n            return this.name().toLowerCase();\n        }\n    }\n\n    public enum ElectionResult {\n        NEW_MASTER_ELECTED,\n        KEEP_CURRENT_MASTER,\n        NO_MASTER_ELECTED;\n\n        public String getLowerCaseName() {\n            return this.name().toLowerCase();\n        }\n    }\n\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.controller.metrics;\n\nimport com.google.common.base.Splitter;\nimport io.openmessaging.storage.dledger.MemberState;\nimport io.opentelemetry.api.common.Attributes;\nimport io.opentelemetry.api.common.AttributesBuilder;\nimport io.opentelemetry.api.metrics.LongCounter;\nimport io.opentelemetry.api.metrics.LongHistogram;\nimport io.opentelemetry.api.metrics.LongUpDownCounter;\nimport io.opentelemetry.api.metrics.Meter;\nimport io.opentelemetry.api.metrics.ObservableLongGauge;\nimport io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter;\nimport io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter;\nimport io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporterBuilder;\nimport io.opentelemetry.exporter.prometheus.PrometheusHttpServer;\nimport io.opentelemetry.sdk.OpenTelemetrySdk;\nimport io.opentelemetry.sdk.metrics.Aggregation;\nimport io.opentelemetry.sdk.metrics.InstrumentSelector;\nimport io.opentelemetry.sdk.metrics.InstrumentType;\nimport io.opentelemetry.sdk.metrics.SdkMeterProvider;\nimport io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder;\nimport io.opentelemetry.sdk.metrics.View;\nimport io.opentelemetry.sdk.metrics.data.AggregationTemporality;\nimport io.opentelemetry.sdk.metrics.export.MetricExporter;\nimport io.opentelemetry.sdk.metrics.export.PeriodicMetricReader;\nimport io.opentelemetry.sdk.resources.Resource;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.ControllerConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.metrics.MetricsExporterType;\nimport org.apache.rocketmq.common.metrics.NopLongCounter;\nimport org.apache.rocketmq.common.metrics.NopLongHistogram;\nimport org.apache.rocketmq.common.metrics.NopLongUpDownCounter;\nimport org.apache.rocketmq.common.metrics.NopObservableLongGauge;\nimport org.apache.rocketmq.controller.ControllerManager;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.slf4j.bridge.SLF4JBridgeHandler;\n\nimport java.io.File;\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.TimeUnit;\n\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.AGGREGATION_DELTA;\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.COUNTER_DLEDGER_OP_TOTAL;\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.COUNTER_ELECTION_TOTAL;\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.COUNTER_REQUEST_TOTAL;\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.GAUGE_ACTIVE_BROKER_NUM;\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.GAUGE_DLEDGER_DISK_USAGE;\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.GAUGE_ROLE;\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.HISTOGRAM_DLEDGER_OP_LATENCY;\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.HISTOGRAM_REQUEST_LATENCY;\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.LABEL_ADDRESS;\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.LABEL_AGGREGATION;\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.LABEL_BROKER_SET;\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.LABEL_CLUSTER_NAME;\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.LABEL_GROUP;\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.LABEL_PEER_ID;\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.OPEN_TELEMETRY_METER_NAME;\n\npublic class ControllerMetricsManager {\n\n    private static final Logger logger = LoggerFactory.getLogger(LoggerName.CONTROLLER_LOGGER_NAME);\n\n    private static volatile ControllerMetricsManager instance;\n\n    private static final Map<String, String> LABEL_MAP = new HashMap<>();\n\n    // metrics about node status\n    public static LongUpDownCounter role = new NopLongUpDownCounter();\n\n    public static ObservableLongGauge dLedgerDiskUsage = new NopObservableLongGauge();\n\n    public static ObservableLongGauge activeBrokerNum = new NopObservableLongGauge();\n\n    public static LongCounter requestTotal = new NopLongCounter();\n\n    public static LongCounter dLedgerOpTotal = new NopLongCounter();\n\n    public static LongCounter electionTotal = new NopLongCounter();\n\n    // metrics about latency\n    public static LongHistogram requestLatency = new NopLongHistogram();\n\n    public static LongHistogram dLedgerOpLatency = new NopLongHistogram();\n\n    private static double us = 1d;\n\n    private static double ms = 1000 * us;\n\n    private static double s = 1000 * ms;\n\n    private final ControllerManager controllerManager;\n\n    private final ControllerConfig config;\n\n    private Meter controllerMeter;\n\n    private OtlpGrpcMetricExporter metricExporter;\n\n    private PeriodicMetricReader periodicMetricReader;\n\n    private PrometheusHttpServer prometheusHttpServer;\n\n    private MetricExporter loggingMetricExporter;\n\n    public static ControllerMetricsManager getInstance(ControllerManager controllerManager) {\n        if (instance == null) {\n            synchronized (ControllerMetricsManager.class) {\n                if (instance == null) {\n                    instance = new ControllerMetricsManager(controllerManager);\n                }\n            }\n        }\n        return instance;\n    }\n\n    public static AttributesBuilder newAttributesBuilder() {\n        AttributesBuilder builder = Attributes.builder();\n        LABEL_MAP.forEach(builder::put);\n        return builder;\n    }\n\n    public static void recordRole(MemberState.Role newRole, MemberState.Role oldRole) {\n        role.add(getRoleValue(newRole) - getRoleValue(oldRole),\n            newAttributesBuilder().build());\n    }\n\n    private static int getRoleValue(MemberState.Role role) {\n        switch (role) {\n            case UNKNOWN:\n                return 0;\n            case CANDIDATE:\n                return 1;\n            case FOLLOWER:\n                return 2;\n            case LEADER:\n                return 3;\n            default:\n                logger.error(\"Unknown role {}\", role);\n                return 0;\n        }\n    }\n\n    private ControllerMetricsManager(ControllerManager controllerManager) {\n        this.controllerManager = controllerManager;\n        this.config = this.controllerManager.getControllerConfig();\n        if (config.getControllerType().equals(ControllerConfig.JRAFT_CONTROLLER)) {\n            this.LABEL_MAP.put(LABEL_ADDRESS, this.config.getJraftConfig().getjRaftAddress());\n            this.LABEL_MAP.put(LABEL_GROUP, this.config.getJraftConfig().getjRaftGroupId());\n            this.LABEL_MAP.put(LABEL_PEER_ID, this.config.getJraftConfig().getjRaftServerId());\n        } else {\n            this.LABEL_MAP.put(LABEL_ADDRESS, this.config.getDLedgerAddress());\n            this.LABEL_MAP.put(LABEL_GROUP, this.config.getControllerDLegerGroup());\n            this.LABEL_MAP.put(LABEL_PEER_ID, this.config.getControllerDLegerSelfId());\n        }\n        this.init();\n    }\n\n    private boolean checkConfig() {\n        if (config == null) {\n            return false;\n        }\n        MetricsExporterType exporterType = config.getMetricsExporterType();\n        if (!exporterType.isEnable()) {\n            return false;\n        }\n\n        switch (exporterType) {\n            case OTLP_GRPC:\n                return StringUtils.isNotBlank(config.getMetricsGrpcExporterTarget());\n            case PROM:\n                return true;\n            case LOG:\n                return true;\n        }\n        return false;\n    }\n\n    private void registerMetricsView(SdkMeterProviderBuilder providerBuilder) {\n        // define latency bucket\n        List<Double> latencyBuckets = Arrays.asList(\n            1 * us, 3 * us, 5 * us,\n            10 * us, 30 * us, 50 * us,\n            100 * us, 300 * us, 500 * us,\n            1 * ms, 3 * ms, 5 * ms,\n            10 * ms, 30 * ms, 50 * ms,\n            100 * ms, 300 * ms, 500 * ms,\n            1 * s, 3 * s, 5 * s,\n            10 * s\n        );\n\n        View latencyView = View.builder()\n            .setAggregation(Aggregation.explicitBucketHistogram(latencyBuckets))\n            .build();\n\n        InstrumentSelector requestLatencySelector = InstrumentSelector.builder()\n            .setType(InstrumentType.HISTOGRAM)\n            .setName(HISTOGRAM_REQUEST_LATENCY)\n            .build();\n\n        InstrumentSelector dLedgerOpLatencySelector = InstrumentSelector.builder()\n            .setType(InstrumentType.HISTOGRAM)\n            .setName(HISTOGRAM_DLEDGER_OP_LATENCY)\n            .build();\n\n        providerBuilder.registerView(requestLatencySelector, latencyView);\n        providerBuilder.registerView(dLedgerOpLatencySelector, latencyView);\n    }\n\n    private void initMetric(Meter meter) {\n        role = meter.upDownCounterBuilder(GAUGE_ROLE)\n            .setDescription(\"role of current node\")\n            .build();\n\n        dLedgerDiskUsage = meter.gaugeBuilder(GAUGE_DLEDGER_DISK_USAGE)\n            .setDescription(\"disk usage of dledger\")\n            .setUnit(\"bytes\")\n            .ofLongs()\n            .buildWithCallback(measurement -> {\n                String path = config.getControllerStorePath();\n                if (!UtilAll.isPathExists(path)) {\n                    return;\n                }\n                File file = new File(path);\n                Long diskUsage = UtilAll.calculateFileSizeInPath(file);\n                if (diskUsage == -1) {\n                    logger.error(\"calculateFileSizeInPath error, path: {}\", path);\n                    return;\n                }\n                measurement.record(diskUsage, newAttributesBuilder().build());\n            });\n\n        activeBrokerNum = meter.gaugeBuilder(GAUGE_ACTIVE_BROKER_NUM)\n            .setDescription(\"now active brokers num\")\n            .ofLongs()\n            .buildWithCallback(measurement -> {\n                Map<String, Map<String, Integer>> activeBrokersNum = controllerManager.getHeartbeatManager().getActiveBrokersNum();\n                activeBrokersNum.forEach((cluster, brokerSetAndNum) -> {\n                    brokerSetAndNum.forEach((brokerSet, num) -> measurement.record(num,\n                        newAttributesBuilder().put(LABEL_CLUSTER_NAME, cluster).put(LABEL_BROKER_SET, brokerSet).build()));\n                });\n            });\n\n        requestTotal = meter.counterBuilder(COUNTER_REQUEST_TOTAL)\n            .setDescription(\"total request num\")\n            .build();\n\n        dLedgerOpTotal = meter.counterBuilder(COUNTER_DLEDGER_OP_TOTAL)\n            .setDescription(\"total dledger operation num\")\n            .build();\n\n        electionTotal = meter.counterBuilder(COUNTER_ELECTION_TOTAL)\n            .setDescription(\"total elect num\")\n            .build();\n\n        requestLatency = meter.histogramBuilder(HISTOGRAM_REQUEST_LATENCY)\n            .setDescription(\"request latency\")\n            .setUnit(\"us\")\n            .ofLongs()\n            .build();\n\n        dLedgerOpLatency = meter.histogramBuilder(HISTOGRAM_DLEDGER_OP_LATENCY)\n            .setDescription(\"dledger operation latency\")\n            .setUnit(\"us\")\n            .ofLongs()\n            .build();\n\n    }\n\n    public void init() {\n        MetricsExporterType type = this.config.getMetricsExporterType();\n        if (type == MetricsExporterType.DISABLE) {\n            return;\n        }\n        if (!checkConfig()) {\n            logger.error(\"check metric config failed, will not export metrics\");\n            return;\n        }\n\n        String labels = config.getMetricsLabel();\n        if (StringUtils.isNotBlank(labels)) {\n            List<String> labelList = Splitter.on(',').omitEmptyStrings().splitToList(labels);\n            for (String label : labelList) {\n                String[] pair = label.split(\":\");\n                if (pair.length != 2) {\n                    logger.warn(\"metrics label is not valid: {}\", label);\n                    continue;\n                }\n                LABEL_MAP.put(pair[0], pair[1]);\n            }\n        }\n        if (config.isMetricsInDelta()) {\n            LABEL_MAP.put(LABEL_AGGREGATION, AGGREGATION_DELTA);\n        }\n\n        SdkMeterProviderBuilder providerBuilder = SdkMeterProvider.builder().setResource(Resource.empty());\n\n        if (type == MetricsExporterType.OTLP_GRPC) {\n            String endpoint = config.getMetricsGrpcExporterTarget();\n            if (!endpoint.startsWith(\"http\")) {\n                endpoint = \"https://\" + endpoint;\n            }\n            OtlpGrpcMetricExporterBuilder metricExporterBuilder = OtlpGrpcMetricExporter.builder()\n                .setEndpoint(endpoint)\n                .setTimeout(config.getMetricGrpcExporterTimeOutInMills(), TimeUnit.MILLISECONDS)\n                .setAggregationTemporalitySelector(x -> {\n                    if (config.isMetricsInDelta() &&\n                        (x == InstrumentType.COUNTER || x == InstrumentType.OBSERVABLE_COUNTER || x == InstrumentType.HISTOGRAM)) {\n                        return AggregationTemporality.DELTA;\n                    }\n                    return AggregationTemporality.CUMULATIVE;\n                });\n\n            String headers = config.getMetricsGrpcExporterHeader();\n            if (StringUtils.isNotBlank(headers)) {\n                Map<String, String> headerMap = new HashMap<>();\n                List<String> headerList = Splitter.on(',').omitEmptyStrings().splitToList(headers);\n                for (String header : headerList) {\n                    String[] pair = header.split(\":\");\n                    if (pair.length != 2) {\n                        logger.warn(\"metricsGrpcExporterHeader is not valid: {}\", headers);\n                        continue;\n                    }\n                    headerMap.put(pair[0], pair[1]);\n                }\n                headerMap.forEach(metricExporterBuilder::addHeader);\n            }\n\n            metricExporter = metricExporterBuilder.build();\n\n            periodicMetricReader = PeriodicMetricReader.builder(metricExporter)\n                .setInterval(config.getMetricGrpcExporterIntervalInMills(), TimeUnit.MILLISECONDS)\n                .build();\n\n            providerBuilder.registerMetricReader(periodicMetricReader);\n        }\n\n        if (type == MetricsExporterType.PROM) {\n            String promExporterHost = config.getMetricsPromExporterHost();\n            if (StringUtils.isBlank(promExporterHost)) {\n                promExporterHost = \"0.0.0.0\";\n            }\n            prometheusHttpServer = PrometheusHttpServer.builder()\n                .setHost(promExporterHost)\n                .setPort(config.getMetricsPromExporterPort())\n                .build();\n            providerBuilder.registerMetricReader(prometheusHttpServer);\n        }\n\n        if (type == MetricsExporterType.LOG) {\n            SLF4JBridgeHandler.removeHandlersForRootLogger();\n            SLF4JBridgeHandler.install();\n            loggingMetricExporter = OtlpJsonLoggingMetricExporter.create(config.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE);\n            java.util.logging.Logger.getLogger(OtlpJsonLoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST);\n            periodicMetricReader = PeriodicMetricReader.builder(loggingMetricExporter)\n                .setInterval(config.getMetricLoggingExporterIntervalInMills(), TimeUnit.MILLISECONDS)\n                .build();\n            providerBuilder.registerMetricReader(periodicMetricReader);\n        }\n\n        registerMetricsView(providerBuilder);\n\n        controllerMeter = OpenTelemetrySdk.builder().setMeterProvider(providerBuilder.build())\n            .build().getMeter(OPEN_TELEMETRY_METER_NAME);\n\n        initMetric(controllerMeter);\n    }\n\n}\n"
  },
  {
    "path": "controller/src/main/java/org/apache/rocketmq/controller/processor/ControllerRequestProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.processor;\n\nimport com.google.common.base.Stopwatch;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.opentelemetry.api.common.Attributes;\nimport java.io.UnsupportedEncodingException;\nimport java.util.Arrays;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Properties;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.TimeUnit;\n\nimport java.util.concurrent.TimeoutException;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.controller.BrokerHeartbeatManager;\nimport org.apache.rocketmq.controller.ControllerManager;\nimport org.apache.rocketmq.controller.metrics.ControllerMetricsConstant;\nimport org.apache.rocketmq.controller.metrics.ControllerMetricsManager;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.RoleChangeNotifyEntry;\nimport org.apache.rocketmq.remoting.protocol.body.SyncStateSet;\nimport org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.BrokerHeartbeatRequestHeader;\n\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.LABEL_REQUEST_HANDLE_STATUS;\nimport static org.apache.rocketmq.controller.metrics.ControllerMetricsConstant.LABEL_REQUEST_TYPE;\nimport static org.apache.rocketmq.remoting.protocol.RequestCode.CONTROLLER_APPLY_BROKER_ID;\nimport static org.apache.rocketmq.remoting.protocol.RequestCode.BROKER_HEARTBEAT;\nimport static org.apache.rocketmq.remoting.protocol.RequestCode.CLEAN_BROKER_DATA;\nimport static org.apache.rocketmq.remoting.protocol.RequestCode.CONTROLLER_ALTER_SYNC_STATE_SET;\nimport static org.apache.rocketmq.remoting.protocol.RequestCode.CONTROLLER_ELECT_MASTER;\nimport static org.apache.rocketmq.remoting.protocol.RequestCode.CONTROLLER_GET_METADATA_INFO;\nimport static org.apache.rocketmq.remoting.protocol.RequestCode.CONTROLLER_GET_REPLICA_INFO;\nimport static org.apache.rocketmq.remoting.protocol.RequestCode.CONTROLLER_GET_SYNC_STATE_DATA;\nimport static org.apache.rocketmq.remoting.protocol.RequestCode.CONTROLLER_REGISTER_BROKER;\nimport static org.apache.rocketmq.remoting.protocol.RequestCode.GET_CONTROLLER_CONFIG;\nimport static org.apache.rocketmq.remoting.protocol.RequestCode.CONTROLLER_GET_NEXT_BROKER_ID;\nimport static org.apache.rocketmq.remoting.protocol.RequestCode.UPDATE_CONTROLLER_CONFIG;\n\n/**\n * Processor for controller request\n */\npublic class ControllerRequestProcessor implements NettyRequestProcessor {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.CONTROLLER_LOGGER_NAME);\n    private static final int WAIT_TIMEOUT_OUT = 5;\n    private final ControllerManager controllerManager;\n    private final BrokerHeartbeatManager heartbeatManager;\n    protected Set<String> configBlackList = new HashSet<>();\n\n    public ControllerRequestProcessor(final ControllerManager controllerManager) {\n        this.controllerManager = controllerManager;\n        this.heartbeatManager = controllerManager.getHeartbeatManager();\n        initConfigBlackList();\n    }\n    private void initConfigBlackList() {\n        configBlackList.add(\"configBlackList\");\n        configBlackList.add(\"configStorePath\");\n        configBlackList.add(\"rocketmqHome\");\n        String[] configArray = controllerManager.getControllerConfig().getConfigBlackList().split(\";\");\n        configBlackList.addAll(Arrays.asList(configArray));\n    }\n    @Override\n    public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws Exception {\n        if (ctx != null) {\n            log.debug(\"Receive request, {} {} {}\",\n                request.getCode(),\n                RemotingHelper.parseChannelRemoteAddr(ctx.channel()),\n                request);\n        }\n        Stopwatch stopwatch = Stopwatch.createStarted();\n        try {\n            RemotingCommand resp = handleRequest(ctx, request);\n            Attributes attributes = ControllerMetricsManager.newAttributesBuilder()\n                .put(LABEL_REQUEST_TYPE, ControllerMetricsConstant.RequestType.getLowerCaseNameByCode(request.getCode()))\n                .put(LABEL_REQUEST_HANDLE_STATUS, ControllerMetricsConstant.RequestHandleStatus.SUCCESS.getLowerCaseName())\n                .build();\n            ControllerMetricsManager.requestTotal.add(1, attributes);\n            attributes = ControllerMetricsManager.newAttributesBuilder()\n                .put(LABEL_REQUEST_TYPE, ControllerMetricsConstant.RequestType.getLowerCaseNameByCode(request.getCode()))\n                .build();\n            ControllerMetricsManager.requestLatency.record(stopwatch.elapsed(TimeUnit.MICROSECONDS), attributes);\n            return resp;\n        } catch (Exception e) {\n            log.error(\"process request: {} error, \", request, e);\n            Attributes attributes;\n            if (e instanceof TimeoutException) {\n                attributes = ControllerMetricsManager.newAttributesBuilder()\n                    .put(LABEL_REQUEST_TYPE, ControllerMetricsConstant.RequestType.getLowerCaseNameByCode(request.getCode()))\n                    .put(LABEL_REQUEST_HANDLE_STATUS, ControllerMetricsConstant.RequestHandleStatus.TIMEOUT.getLowerCaseName())\n                    .build();\n            } else {\n                attributes = ControllerMetricsManager.newAttributesBuilder()\n                    .put(LABEL_REQUEST_TYPE, ControllerMetricsConstant.RequestType.getLowerCaseNameByCode(request.getCode()))\n                    .put(LABEL_REQUEST_HANDLE_STATUS, ControllerMetricsConstant.RequestHandleStatus.FAILED.getLowerCaseName())\n                    .build();\n            }\n            ControllerMetricsManager.requestTotal.add(1, attributes);\n            throw e;\n        }\n    }\n\n    private RemotingCommand handleRequest(ChannelHandlerContext ctx, RemotingCommand request) throws Exception {\n        switch (request.getCode()) {\n            case CONTROLLER_ALTER_SYNC_STATE_SET:\n                return this.handleAlterSyncStateSet(ctx, request);\n            case CONTROLLER_ELECT_MASTER:\n                return this.handleControllerElectMaster(ctx, request);\n            case CONTROLLER_GET_REPLICA_INFO:\n                return this.handleControllerGetReplicaInfo(ctx, request);\n            case CONTROLLER_GET_METADATA_INFO:\n                return this.handleControllerGetMetadataInfo(ctx, request);\n            case BROKER_HEARTBEAT:\n                return this.handleBrokerHeartbeat(ctx, request);\n            case CONTROLLER_GET_SYNC_STATE_DATA:\n                return this.handleControllerGetSyncStateData(ctx, request);\n            case UPDATE_CONTROLLER_CONFIG:\n                return this.handleUpdateControllerConfig(ctx, request);\n            case GET_CONTROLLER_CONFIG:\n                return this.handleGetControllerConfig(ctx, request);\n            case CLEAN_BROKER_DATA:\n                return this.handleCleanBrokerData(ctx, request);\n            case CONTROLLER_GET_NEXT_BROKER_ID:\n                return this.handleGetNextBrokerId(ctx, request);\n            case CONTROLLER_APPLY_BROKER_ID:\n                return this.handleApplyBrokerId(ctx, request);\n            case CONTROLLER_REGISTER_BROKER:\n                return this.handleRegisterBroker(ctx, request);\n            default: {\n                final String error = \" request type \" + request.getCode() + \" not supported\";\n                return RemotingCommand.createResponseCommand(ResponseCode.REQUEST_CODE_NOT_SUPPORTED, error);\n            }\n        }\n    }\n\n    private RemotingCommand handleAlterSyncStateSet(ChannelHandlerContext ctx,\n        RemotingCommand request) throws Exception {\n        final AlterSyncStateSetRequestHeader controllerRequest = (AlterSyncStateSetRequestHeader) request.decodeCommandCustomHeader(AlterSyncStateSetRequestHeader.class);\n        final SyncStateSet syncStateSet = RemotingSerializable.decode(request.getBody(), SyncStateSet.class);\n        final CompletableFuture<RemotingCommand> future = this.controllerManager.getController().alterSyncStateSet(controllerRequest, syncStateSet);\n        if (future != null) {\n            return future.get(WAIT_TIMEOUT_OUT, TimeUnit.SECONDS);\n        }\n        return RemotingCommand.createResponseCommand(null);\n    }\n\n    private RemotingCommand handleControllerElectMaster(ChannelHandlerContext ctx,\n        RemotingCommand request) throws Exception {\n        final ElectMasterRequestHeader electMasterRequest = (ElectMasterRequestHeader) request.decodeCommandCustomHeader(ElectMasterRequestHeader.class);\n        final CompletableFuture<RemotingCommand> future = this.controllerManager.getController().electMaster(electMasterRequest);\n        if (future != null) {\n            final RemotingCommand response = future.get(WAIT_TIMEOUT_OUT, TimeUnit.SECONDS);\n\n            if (response.getCode() == ResponseCode.SUCCESS) {\n                if (this.controllerManager.getControllerConfig().isNotifyBrokerRoleChanged()) {\n                    this.controllerManager.notifyBrokerRoleChanged(RoleChangeNotifyEntry.convert(response));\n                }\n            }\n            return response;\n        }\n        return RemotingCommand.createResponseCommand(null);\n    }\n\n    private RemotingCommand handleControllerGetReplicaInfo(ChannelHandlerContext ctx,\n                                                           RemotingCommand request) throws Exception {\n        final GetReplicaInfoRequestHeader controllerRequest = (GetReplicaInfoRequestHeader) request.decodeCommandCustomHeader(GetReplicaInfoRequestHeader.class);\n        final CompletableFuture<RemotingCommand> future = this.controllerManager.getController().getReplicaInfo(controllerRequest);\n        if (future != null) {\n            return future.get(WAIT_TIMEOUT_OUT, TimeUnit.SECONDS);\n        }\n        return RemotingCommand.createResponseCommand(null);\n    }\n\n    private RemotingCommand handleControllerGetMetadataInfo(ChannelHandlerContext ctx, RemotingCommand request) {\n        return this.controllerManager.getController().getControllerMetadata();\n    }\n\n    private RemotingCommand handleBrokerHeartbeat(ChannelHandlerContext ctx, RemotingCommand request) throws Exception {\n        final BrokerHeartbeatRequestHeader requestHeader = (BrokerHeartbeatRequestHeader) request.decodeCommandCustomHeader(BrokerHeartbeatRequestHeader.class);\n        if (requestHeader.getBrokerId() == null) {\n            return RemotingCommand.createResponseCommand(ResponseCode.CONTROLLER_INVALID_REQUEST, \"Heart beat with empty brokerId\");\n        }\n        this.heartbeatManager.onBrokerHeartbeat(requestHeader.getClusterName(), requestHeader.getBrokerName(), requestHeader.getBrokerAddr(), requestHeader.getBrokerId(),\n                requestHeader.getHeartbeatTimeoutMills(), ctx.channel(), requestHeader.getEpoch(), requestHeader.getMaxOffset(), requestHeader.getConfirmOffset(), requestHeader.getElectionPriority());\n        return RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"Heart beat success\");\n    }\n\n    private RemotingCommand handleControllerGetSyncStateData(ChannelHandlerContext ctx,\n                                                             RemotingCommand request) throws Exception {\n        if (request.getBody() != null) {\n            final List<String> brokerNames = RemotingSerializable.decode(request.getBody(), List.class);\n            if (brokerNames != null && brokerNames.size() > 0) {\n                final CompletableFuture<RemotingCommand> future = this.controllerManager.getController().getSyncStateData(brokerNames);\n                if (future != null) {\n                    return future.get(WAIT_TIMEOUT_OUT, TimeUnit.SECONDS);\n                }\n            }\n        }\n        return RemotingCommand.createResponseCommand(null);\n    }\n\n    private RemotingCommand handleCleanBrokerData(ChannelHandlerContext ctx, RemotingCommand request) throws Exception {\n        final CleanControllerBrokerDataRequestHeader requestHeader = (CleanControllerBrokerDataRequestHeader) request.decodeCommandCustomHeader(CleanControllerBrokerDataRequestHeader.class);\n        final CompletableFuture<RemotingCommand> future = this.controllerManager.getController().cleanBrokerData(requestHeader);\n        if (null != future) {\n            return future.get(WAIT_TIMEOUT_OUT, TimeUnit.SECONDS);\n        }\n        return RemotingCommand.createResponseCommand(null);\n    }\n\n    private RemotingCommand handleGetNextBrokerId(ChannelHandlerContext ctx, RemotingCommand request) throws Exception {\n        final GetNextBrokerIdRequestHeader requestHeader = (GetNextBrokerIdRequestHeader) request.decodeCommandCustomHeader(GetNextBrokerIdRequestHeader.class);\n        CompletableFuture<RemotingCommand> future = this.controllerManager.getController().getNextBrokerId(requestHeader);\n        if (future != null) {\n            return future.get(WAIT_TIMEOUT_OUT, TimeUnit.SECONDS);\n        }\n        return RemotingCommand.createResponseCommand(null);\n    }\n\n    private RemotingCommand handleApplyBrokerId(ChannelHandlerContext ctx, RemotingCommand request) throws Exception {\n        final ApplyBrokerIdRequestHeader requestHeader = (ApplyBrokerIdRequestHeader) request.decodeCommandCustomHeader(ApplyBrokerIdRequestHeader.class);\n        CompletableFuture<RemotingCommand> future = this.controllerManager.getController().applyBrokerId(requestHeader);\n        if (future != null) {\n            return future.get(WAIT_TIMEOUT_OUT, TimeUnit.SECONDS);\n        }\n        return RemotingCommand.createResponseCommand(null);\n    }\n\n    private RemotingCommand handleRegisterBroker(ChannelHandlerContext ctx, RemotingCommand request) throws Exception {\n        RegisterBrokerToControllerRequestHeader requestHeader = (RegisterBrokerToControllerRequestHeader) request.decodeCommandCustomHeader(RegisterBrokerToControllerRequestHeader.class);\n        CompletableFuture<RemotingCommand> future = this.controllerManager.getController().registerBroker(requestHeader);\n        if (future != null) {\n            return future.get(WAIT_TIMEOUT_OUT, TimeUnit.SECONDS);\n        }\n        return RemotingCommand.createResponseCommand(null);\n    }\n\n    private RemotingCommand handleUpdateControllerConfig(ChannelHandlerContext ctx, RemotingCommand request) {\n        if (ctx != null) {\n            log.info(\"updateConfig called by {}\", RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n        }\n\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        byte[] body = request.getBody();\n        if (body != null) {\n            String bodyStr;\n            try {\n                bodyStr = new String(body, MixAll.DEFAULT_CHARSET);\n            } catch (UnsupportedEncodingException e) {\n                log.error(\"updateConfig byte array to string error: \", e);\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"UnsupportedEncodingException \" + e);\n                return response;\n            }\n\n            Properties properties = MixAll.string2Properties(bodyStr);\n            if (properties == null) {\n                log.error(\"updateConfig MixAll.string2Properties error {}\", bodyStr);\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"string2Properties error\");\n                return response;\n            }\n            if (validateBlackListConfigExist(properties)) {\n                response.setCode(ResponseCode.NO_PERMISSION);\n                response.setRemark(\"Can not update config in black list.\");\n                return response;\n            }\n\n            this.controllerManager.getConfiguration().update(properties);\n        }\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand handleGetControllerConfig(ChannelHandlerContext ctx, RemotingCommand request) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        String content = this.controllerManager.getConfiguration().getAllConfigsFormatString();\n        if (content != null && content.length() > 0) {\n            try {\n                content = MixAll.adjustConfigForPlatform(content);\n                response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET));\n            } catch (UnsupportedEncodingException e) {\n                log.error(\"getConfig error, \", e);\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"UnsupportedEncodingException \" + e);\n                return response;\n            }\n        }\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n    private boolean validateBlackListConfigExist(Properties properties) {\n        for (String blackConfig : configBlackList) {\n            if (properties.containsKey(blackConfig)) {\n                return true;\n            }\n        }\n        return false;\n    }\n}\n"
  },
  {
    "path": "controller/src/main/resources/rmq.controller.logback.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n<configuration scan=\"true\" scanPeriod=\"30 seconds\">\n    <appender name=\"DefaultAppender\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}controller_default.log\n        </file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>\n                ${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}controller_default.%i.log.gz\n            </fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>5</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy\n            class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n\n    <appender name=\"DLedgerAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}dledger.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>\n                ${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}dledger.%i.log.gz\n            </fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>5</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy\n            class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n\n    <appender name=\"DLedgerAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"DLedgerAppender_inner\"/>\n        <discardingThreshold>0</discardingThreshold>\n    </appender>\n\n    <appender name=\"JRaftAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}jraft.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>\n                ${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}jraft.%i.log.gz\n            </fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>5</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy\n            class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n\n    <appender name=\"JRaftAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"JRaftAppender_inner\"/>\n        <discardingThreshold>0</discardingThreshold>\n    </appender>\n\n    <appender name=\"RocketmqTrafficAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}controller_traffic.log\n        </file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>\n                ${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}controller_traffic.%i.log.gz\n            </fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n\n    <appender name=\"RocketmqTrafficAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqTrafficAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqControllerAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}controller.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>\n                ${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}controller.%i.log.gz\n            </fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>5</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy\n            class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqControllerAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqControllerAppender_inner\"/>\n        <discardingThreshold>0</discardingThreshold>\n    </appender>\n\n    <appender name=\"STDOUT\" class=\"ch.qos.logback.core.ConsoleAppender\">\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n\n    <logger name=\"RocketmqController\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqControllerAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqControllerConsole\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"STDOUT\"/>\n    </logger>\n\n    <logger name=\"RocketmqCommon\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqControllerAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqRemoting\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqControllerAppender\"/>\n    </logger>\n\n    <logger name=\"io.openmessaging.storage.dledger\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"DLedgerAppender\"/>\n    </logger>\n\n    <logger name=\"com.alipay.sofa.jraft\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"JRaftAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqControllerConsole\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"STDOUT\"/>\n    </logger>\n\n    <logger name=\"RocketmqTraffic\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqTrafficAppender\"/>\n    </logger>\n\n    <root level=\"INFO\">\n        <appender-ref ref=\"DefaultAppender\"/>\n    </root>\n</configuration>\n"
  },
  {
    "path": "controller/src/test/java/org/apache/rocketmq/controller/ControllerManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller;\n\nimport org.apache.rocketmq.common.ControllerConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.controller.impl.DLedgerController;\nimport org.apache.rocketmq.remoting.RemotingClient;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingClient;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.BrokerHeartbeatRequestHeader;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport java.io.File;\nimport java.time.Duration;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.UUID;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\n\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertTrue;\n\npublic class ControllerManagerTest {\n\n    public static final String STORE_BASE_PATH = System.getProperty(\"java.io.tmpdir\") + File.separator + \"ControllerManagerTest\";\n\n    public static final String STORE_PATH = STORE_BASE_PATH + File.separator + UUID.randomUUID();\n\n    private List<ControllerManager> controllers;\n    private NettyRemotingClient remotingClient;\n    private NettyRemotingClient remotingClient1;\n\n    public ControllerManager launchManager(final String group, final String peers, final String selfId) {\n        final String path = STORE_PATH + File.separator + group + File.separator + selfId;\n        final ControllerConfig config = new ControllerConfig();\n        config.setControllerDLegerGroup(group);\n        config.setControllerDLegerPeers(peers);\n        config.setControllerDLegerSelfId(selfId);\n        config.setControllerStorePath(path);\n        config.setMappedFileSize(10 * 1024 * 1024);\n        config.setEnableElectUncleanMaster(true);\n        config.setScanNotActiveBrokerInterval(1000L);\n        config.setNotifyBrokerRoleChanged(false);\n\n        final NettyServerConfig serverConfig = new NettyServerConfig();\n\n        final ControllerManager manager = new ControllerManager(config, serverConfig, new NettyClientConfig());\n        manager.initialize();\n        manager.start();\n        this.controllers.add(manager);\n        return manager;\n    }\n\n    @Before\n    public void startup() {\n        UtilAll.deleteFile(new File(STORE_BASE_PATH));\n        this.controllers = new ArrayList<>();\n        this.remotingClient = new NettyRemotingClient(new NettyClientConfig());\n        this.remotingClient.start();\n        this.remotingClient1 = new NettyRemotingClient(new NettyClientConfig());\n        this.remotingClient1.start();\n    }\n\n    public ControllerManager waitLeader(final List<ControllerManager> controllers) throws Exception {\n        if (controllers.isEmpty()) {\n            return null;\n        }\n        DLedgerController c1 = (DLedgerController) controllers.get(0).getController();\n\n        ControllerManager manager = await().atMost(Duration.ofSeconds(10)).until(() -> {\n            String leaderId = c1.getMemberState().getLeaderId();\n            if (null == leaderId) {\n                return null;\n            }\n            for (ControllerManager controllerManager : controllers) {\n                final DLedgerController controller = (DLedgerController) controllerManager.getController();\n                if (controller.getMemberState().getSelfId().equals(leaderId) && controller.isLeaderState()) {\n                    return controllerManager;\n                }\n            }\n            return null;\n        }, item -> item != null);\n        return manager;\n    }\n\n    public void mockData() {\n        String group = UUID.randomUUID().toString();\n        String peers = String.format(\"n0-localhost:%d;n1-localhost:%d;n2-localhost:%d\", 30000, 30001, 30002);\n        launchManager(group, peers, \"n0\");\n        launchManager(group, peers, \"n1\");\n        launchManager(group, peers, \"n2\");\n    }\n\n    /**\n     * Register broker to controller\n     */\n    public void registerBroker(\n        final String controllerAddress, final String clusterName,\n        final String brokerName, final Long brokerId, final String brokerAddress, final Long expectMasterBrokerId,\n        final RemotingClient client) throws Exception {\n        // Get next brokerId;\n        final GetNextBrokerIdRequestHeader getNextBrokerIdRequestHeader = new GetNextBrokerIdRequestHeader(clusterName, brokerName);\n        final RemotingCommand getNextBrokerIdRequest = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_GET_NEXT_BROKER_ID, getNextBrokerIdRequestHeader);\n        final RemotingCommand getNextBrokerIdResponse = client.invokeSync(controllerAddress, getNextBrokerIdRequest, 3000);\n        final GetNextBrokerIdResponseHeader getNextBrokerIdResponseHeader = (GetNextBrokerIdResponseHeader) getNextBrokerIdResponse.decodeCommandCustomHeader(GetNextBrokerIdResponseHeader.class);\n        String registerCheckCode = brokerAddress + \";\" + System.currentTimeMillis();\n        assertEquals(ResponseCode.SUCCESS, getNextBrokerIdResponse.getCode());\n        assertEquals(brokerId, getNextBrokerIdResponseHeader.getNextBrokerId());\n\n        // Apply brokerId\n        final ApplyBrokerIdRequestHeader applyBrokerIdRequestHeader = new ApplyBrokerIdRequestHeader(clusterName, brokerName, brokerId, registerCheckCode);\n        final RemotingCommand applyBrokerIdRequest = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_APPLY_BROKER_ID, applyBrokerIdRequestHeader);\n        final RemotingCommand applyBrokerIdResponse = client.invokeSync(controllerAddress, applyBrokerIdRequest, 3000);\n        final ApplyBrokerIdResponseHeader applyBrokerIdResponseHeader = (ApplyBrokerIdResponseHeader) applyBrokerIdResponse.decodeCommandCustomHeader(ApplyBrokerIdResponseHeader.class);\n        assertEquals(ResponseCode.SUCCESS, applyBrokerIdResponse.getCode());\n\n        // Register success\n        final RegisterBrokerToControllerRequestHeader registerBrokerToControllerRequestHeader = new RegisterBrokerToControllerRequestHeader(clusterName, brokerName, brokerId, brokerAddress);\n        final RemotingCommand registerSuccessRequest = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_REGISTER_BROKER, registerBrokerToControllerRequestHeader);\n        final RemotingCommand registerSuccessResponse = client.invokeSync(controllerAddress, registerSuccessRequest, 3000);\n        final RegisterBrokerToControllerResponseHeader registerBrokerToControllerResponseHeader = (RegisterBrokerToControllerResponseHeader) registerSuccessResponse.decodeCommandCustomHeader(RegisterBrokerToControllerResponseHeader.class);\n        assertEquals(ResponseCode.SUCCESS, registerSuccessResponse.getCode());\n        assertEquals(expectMasterBrokerId, registerBrokerToControllerResponseHeader.getMasterBrokerId());\n    }\n\n    public RemotingCommand brokerTryElect(final String controllerAddress, final String clusterName,\n        final String brokerName, final Long brokerId, final RemotingClient client) throws Exception {\n        final ElectMasterRequestHeader requestHeader = ElectMasterRequestHeader.ofBrokerTrigger(clusterName, brokerName, brokerId);\n        final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_ELECT_MASTER, requestHeader);\n        RemotingCommand response = client.invokeSync(controllerAddress, request, 10000);\n        assertNotNull(response);\n        return response;\n    }\n\n    public void sendHeartbeat(final String controllerAddress, final String clusterName, final String brokerName,\n        final Long brokerId,\n        final String brokerAddress, final Long timeout, final RemotingClient client) throws Exception {\n        final BrokerHeartbeatRequestHeader heartbeatRequestHeader0 = new BrokerHeartbeatRequestHeader();\n        heartbeatRequestHeader0.setBrokerId(brokerId);\n        heartbeatRequestHeader0.setClusterName(clusterName);\n        heartbeatRequestHeader0.setBrokerName(brokerName);\n        heartbeatRequestHeader0.setBrokerAddr(brokerAddress);\n        heartbeatRequestHeader0.setHeartbeatTimeoutMills(timeout);\n        final RemotingCommand heartbeatRequest = RemotingCommand.createRequestCommand(RequestCode.BROKER_HEARTBEAT, heartbeatRequestHeader0);\n        RemotingCommand remotingCommand = client.invokeSync(controllerAddress, heartbeatRequest, 3000);\n        assertEquals(ResponseCode.SUCCESS, remotingCommand.getCode());\n    }\n\n    @Test\n    public void testSomeApi() throws Exception {\n        mockData();\n        final ControllerManager leader = waitLeader(this.controllers);\n        String leaderAddr = \"localhost\" + \":\" + leader.getController().getRemotingServer().localListenPort();\n\n        // Register two broker\n        registerBroker(leaderAddr, \"cluster1\", \"broker1\", 1L, \"127.0.0.1:8000\", null, this.remotingClient);\n\n        registerBroker(leaderAddr, \"cluster1\", \"broker1\", 2L, \"127.0.0.1:8001\", null, this.remotingClient1);\n\n        // Send heartbeat\n        sendHeartbeat(leaderAddr, \"cluster1\", \"broker1\", 1L, \"127.0.0.1:8000\", 3000L, remotingClient);\n        sendHeartbeat(leaderAddr, \"cluster1\", \"broker1\", 2L, \"127.0.0.1:8001\", 3000L, remotingClient1);\n\n        // Two all try elect itself as master, but only the first can be the master\n        RemotingCommand tryElectCommand1 = brokerTryElect(leaderAddr, \"cluster1\", \"broker1\", 1L, this.remotingClient);\n        ElectMasterResponseHeader brokerTryElectResponseHeader1 = (ElectMasterResponseHeader) tryElectCommand1.decodeCommandCustomHeader(ElectMasterResponseHeader.class);\n        RemotingCommand tryElectCommand2 = brokerTryElect(leaderAddr, \"cluster1\", \"broker1\", 2L, this.remotingClient1);\n        ElectMasterResponseHeader brokerTryElectResponseHeader2 = (ElectMasterResponseHeader) tryElectCommand2.decodeCommandCustomHeader(ElectMasterResponseHeader.class);\n\n        assertEquals(ResponseCode.SUCCESS, tryElectCommand1.getCode());\n        assertEquals(1L, brokerTryElectResponseHeader1.getMasterBrokerId().longValue());\n        assertEquals(\"127.0.0.1:8000\", brokerTryElectResponseHeader1.getMasterAddress());\n        assertEquals(1, brokerTryElectResponseHeader1.getMasterEpoch().intValue());\n        assertEquals(1, brokerTryElectResponseHeader1.getSyncStateSetEpoch().intValue());\n\n        assertEquals(ResponseCode.CONTROLLER_MASTER_STILL_EXIST, tryElectCommand2.getCode());\n        assertEquals(1L, brokerTryElectResponseHeader2.getMasterBrokerId().longValue());\n        assertEquals(\"127.0.0.1:8000\", brokerTryElectResponseHeader2.getMasterAddress());\n        assertEquals(1, brokerTryElectResponseHeader2.getMasterEpoch().intValue());\n        assertEquals(1, brokerTryElectResponseHeader2.getSyncStateSetEpoch().intValue());\n\n        // Send heartbeat for broker2 every one second\n        ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();\n        executor.scheduleAtFixedRate(() -> {\n            final BrokerHeartbeatRequestHeader heartbeatRequestHeader = new BrokerHeartbeatRequestHeader();\n            heartbeatRequestHeader.setClusterName(\"cluster1\");\n            heartbeatRequestHeader.setBrokerName(\"broker1\");\n            heartbeatRequestHeader.setBrokerAddr(\"127.0.0.1:8001\");\n            heartbeatRequestHeader.setBrokerId(2L);\n            heartbeatRequestHeader.setHeartbeatTimeoutMills(3000L);\n            final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.BROKER_HEARTBEAT, heartbeatRequestHeader);\n            try {\n                final RemotingCommand remotingCommand = this.remotingClient1.invokeSync(leaderAddr, request, 3000);\n            } catch (Exception e) {\n                e.printStackTrace();\n            }\n        }, 0, 1000L, TimeUnit.MILLISECONDS);\n        Boolean flag = await().atMost(Duration.ofSeconds(10)).until(() -> {\n            final GetReplicaInfoRequestHeader requestHeader = new GetReplicaInfoRequestHeader(\"broker1\");\n            final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_GET_REPLICA_INFO, requestHeader);\n            final RemotingCommand response = this.remotingClient1.invokeSync(leaderAddr, request, 3000);\n            final GetReplicaInfoResponseHeader responseHeader = (GetReplicaInfoResponseHeader) response.decodeCommandCustomHeader(GetReplicaInfoResponseHeader.class);\n            return responseHeader.getMasterBrokerId().equals(2L);\n        }, item -> item);\n\n        // The new master should be broker2.\n        assertTrue(flag);\n\n        executor.shutdown();\n    }\n\n    @After\n    public void tearDown() {\n        for (ControllerManager controller : this.controllers) {\n            controller.shutdown();\n        }\n        UtilAll.deleteFile(new File(STORE_BASE_PATH));\n        this.remotingClient.shutdown();\n        this.remotingClient1.shutdown();\n    }\n}\n"
  },
  {
    "path": "controller/src/test/java/org/apache/rocketmq/controller/ControllerRequestProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.controller;\n\nimport org.apache.rocketmq.common.ControllerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.controller.processor.ControllerRequestProcessor;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.Properties;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class ControllerRequestProcessorTest {\n\n    private ControllerRequestProcessor controllerRequestProcessor;\n\n    @Before\n    public void init() throws Exception {\n        controllerRequestProcessor = new ControllerRequestProcessor(new ControllerManager(new ControllerConfig(), new NettyServerConfig(), new NettyClientConfig()));\n    }\n\n    @Test\n    public void testProcessRequest_UpdateConfigPath() throws Exception {\n        final RemotingCommand updateConfigRequest = RemotingCommand.createRequestCommand(RequestCode.UPDATE_CONTROLLER_CONFIG, null);\n        Properties properties = new Properties();\n\n        // Update allowed value\n        properties.setProperty(\"notifyBrokerRoleChanged\", \"true\");\n        updateConfigRequest.setBody(MixAll.properties2String(properties).getBytes(StandardCharsets.UTF_8));\n\n        RemotingCommand response = controllerRequestProcessor.processRequest(null, updateConfigRequest);\n\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        // Update disallowed value\n        properties.clear();\n        properties.setProperty(\"configStorePath\", \"test/path\");\n        updateConfigRequest.setBody(MixAll.properties2String(properties).getBytes(StandardCharsets.UTF_8));\n\n        response = controllerRequestProcessor.processRequest(null, updateConfigRequest);\n\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);\n        assertThat(response.getRemark()).contains(\"Can not update config in black list.\");\n\n        // Update disallowed value\n        properties.clear();\n        properties.setProperty(\"rocketmqHome\", \"test/path\");\n        updateConfigRequest.setBody(MixAll.properties2String(properties).getBytes(StandardCharsets.UTF_8));\n\n        response = controllerRequestProcessor.processRequest(null, updateConfigRequest);\n\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);\n        assertThat(response.getRemark()).contains(\"Can not update config in black list.\");\n\n        // Update disallowed value\n        properties.clear();\n        properties.setProperty(\"configBlackList\", \"test;path\");\n        updateConfigRequest.setBody(MixAll.properties2String(properties).getBytes(StandardCharsets.UTF_8));\n\n        response = controllerRequestProcessor.processRequest(null, updateConfigRequest);\n\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);\n        assertThat(response.getRemark()).contains(\"Can not update config in black list.\");\n    }\n}\n"
  },
  {
    "path": "controller/src/test/java/org/apache/rocketmq/controller/ControllerTestBase.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.controller;\n\npublic class ControllerTestBase {\n\n    public final static String DEFAULT_CLUSTER_NAME = \"cluster-a\";\n\n    public final static String DEFAULT_BROKER_NAME = \"broker-set-a\";\n\n    public final static String[] DEFAULT_IP = {\"127.0.0.1:9000\", \"127.0.0.1:9001\", \"127.0.0.1:9002\"};\n}\n"
  },
  {
    "path": "controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.ControllerConfig;\nimport org.apache.rocketmq.controller.Controller;\nimport org.apache.rocketmq.controller.elect.impl.DefaultElectPolicy;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.SyncStateSet;\nimport org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerRequestHeader;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport java.io.File;\nimport java.time.Duration;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.UUID;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.stream.Collectors;\n\nimport static org.apache.rocketmq.controller.ControllerTestBase.DEFAULT_BROKER_NAME;\nimport static org.apache.rocketmq.controller.ControllerTestBase.DEFAULT_CLUSTER_NAME;\nimport static org.apache.rocketmq.controller.ControllerTestBase.DEFAULT_IP;\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertArrayEquals;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertTrue;\n\npublic class DLedgerControllerTest {\n    private List<String> baseDirs;\n    private List<DLedgerController> controllers;\n\n    public DLedgerController launchController(final String group, final String peers, final String selfId,\n        final boolean isEnableElectUncleanMaster) {\n        String tmpdir = System.getProperty(\"java.io.tmpdir\");\n        final String path = (StringUtils.endsWith(tmpdir, File.separator) ? tmpdir : tmpdir + File.separator) + group + File.separator + selfId;\n        baseDirs.add(path);\n\n        final ControllerConfig config = new ControllerConfig();\n        config.setControllerDLegerGroup(group);\n        config.setControllerDLegerPeers(peers);\n        config.setControllerDLegerSelfId(selfId);\n        config.setControllerStorePath(path);\n        config.setMappedFileSize(10 * 1024 * 1024);\n        config.setEnableElectUncleanMaster(isEnableElectUncleanMaster);\n        config.setScanInactiveMasterInterval(1000);\n        final DLedgerController controller = new DLedgerController(config, (str1, str2, str3) -> true);\n\n        controller.startup();\n        return controller;\n    }\n\n    @Before\n    public void startup() {\n        this.baseDirs = new ArrayList<>();\n        this.controllers = new ArrayList<>();\n    }\n\n    @After\n    public void tearDown() {\n        for (Controller controller : this.controllers) {\n            controller.shutdown();\n        }\n        for (String dir : this.baseDirs) {\n            new File(dir).delete();\n        }\n    }\n\n    public void registerNewBroker(Controller leader, String clusterName, String brokerName, String brokerAddress,\n        Long expectBrokerId) throws Exception {\n        // Get next brokerId\n        final GetNextBrokerIdRequestHeader getNextBrokerIdRequest = new GetNextBrokerIdRequestHeader(clusterName, brokerName);\n        RemotingCommand remotingCommand = leader.getNextBrokerId(getNextBrokerIdRequest).get(2, TimeUnit.SECONDS);\n        GetNextBrokerIdResponseHeader getNextBrokerIdResp = (GetNextBrokerIdResponseHeader) remotingCommand.readCustomHeader();\n        Long nextBrokerId = getNextBrokerIdResp.getNextBrokerId();\n        String registerCheckCode = brokerAddress + \";\" + System.currentTimeMillis();\n\n        // Check response\n        assertEquals(expectBrokerId, nextBrokerId);\n\n        // Apply brokerId\n        final ApplyBrokerIdRequestHeader applyBrokerIdRequestHeader = new ApplyBrokerIdRequestHeader(clusterName, brokerName, nextBrokerId, registerCheckCode);\n        RemotingCommand remotingCommand1 = leader.applyBrokerId(applyBrokerIdRequestHeader).get(2, TimeUnit.SECONDS);\n\n        // Check response\n        assertEquals(ResponseCode.SUCCESS, remotingCommand1.getCode());\n\n        // Register success\n        final RegisterBrokerToControllerRequestHeader registerBrokerToControllerRequestHeader = new RegisterBrokerToControllerRequestHeader(clusterName, brokerName, nextBrokerId, brokerAddress);\n        RemotingCommand remotingCommand2 = leader.registerBroker(registerBrokerToControllerRequestHeader).get(2, TimeUnit.SECONDS);\n\n        assertEquals(ResponseCode.SUCCESS, remotingCommand2.getCode());\n    }\n\n    public void brokerTryElectMaster(Controller leader, String clusterName, String brokerName, String brokerAddress,\n        Long brokerId,\n        boolean exceptSuccess) throws Exception {\n        final ElectMasterRequestHeader electMasterRequestHeader = ElectMasterRequestHeader.ofBrokerTrigger(clusterName, brokerName, brokerId);\n        RemotingCommand command = leader.electMaster(electMasterRequestHeader).get(2, TimeUnit.SECONDS);\n        ElectMasterResponseHeader header = (ElectMasterResponseHeader) command.readCustomHeader();\n        assertEquals(exceptSuccess, ResponseCode.SUCCESS == command.getCode());\n    }\n\n    private boolean alterNewInSyncSet(Controller leader, String brokerName, Long masterBrokerId, Integer masterEpoch,\n        Set<Long> newSyncStateSet, Integer syncStateSetEpoch) throws Exception {\n        final AlterSyncStateSetRequestHeader alterRequest =\n            new AlterSyncStateSetRequestHeader(brokerName, masterBrokerId, masterEpoch);\n        final RemotingCommand response = leader.alterSyncStateSet(alterRequest, new SyncStateSet(newSyncStateSet, syncStateSetEpoch)).get(10, TimeUnit.SECONDS);\n        if (null == response || response.getCode() != ResponseCode.SUCCESS) {\n            return false;\n        }\n        final RemotingCommand getInfoResponse = leader.getReplicaInfo(new GetReplicaInfoRequestHeader(brokerName)).get(10, TimeUnit.SECONDS);\n        final GetReplicaInfoResponseHeader replicaInfo = (GetReplicaInfoResponseHeader) getInfoResponse.readCustomHeader();\n        final SyncStateSet syncStateSet = RemotingSerializable.decode(getInfoResponse.getBody(), SyncStateSet.class);\n        assertArrayEquals(syncStateSet.getSyncStateSet().toArray(), newSyncStateSet.toArray());\n        assertEquals(syncStateSet.getSyncStateSetEpoch(), syncStateSetEpoch + 1);\n        return true;\n    }\n\n    public DLedgerController waitLeader(final List<DLedgerController> controllers) throws Exception {\n        if (controllers.isEmpty()) {\n            return null;\n        }\n        DLedgerController c1 = controllers.get(0);\n        DLedgerController dLedgerController = await().atMost(Duration.ofSeconds(10)).until(() -> {\n            String leaderId = c1.getMemberState().getLeaderId();\n            if (null == leaderId) {\n                return null;\n            }\n            for (DLedgerController controller : controllers) {\n                if (controller.getMemberState().getSelfId().equals(leaderId) && controller.isLeaderState()) {\n                    return controller;\n                }\n            }\n            return null;\n        }, item -> item != null);\n        return dLedgerController;\n    }\n\n    public DLedgerController mockMetaData(boolean enableElectUncleanMaster) throws Exception {\n        String group = UUID.randomUUID().toString();\n        String peers = String.format(\"n0-localhost:%d;n1-localhost:%d;n2-localhost:%d\", 30000, 30001, 30002);\n        DLedgerController c0 = launchController(group, peers, \"n0\", enableElectUncleanMaster);\n        DLedgerController c1 = launchController(group, peers, \"n1\", enableElectUncleanMaster);\n        DLedgerController c2 = launchController(group, peers, \"n2\", enableElectUncleanMaster);\n        controllers.add(c0);\n        controllers.add(c1);\n        controllers.add(c2);\n\n        DLedgerController leader = waitLeader(controllers);\n\n        // register\n        registerNewBroker(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[0], 1L);\n        registerNewBroker(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L);\n        registerNewBroker(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L);\n        // try elect\n        brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[0], 1L, true);\n        brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L, false);\n        brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L, false);\n        final RemotingCommand getInfoResponse = leader.getReplicaInfo(new GetReplicaInfoRequestHeader(DEFAULT_BROKER_NAME)).get(10, TimeUnit.SECONDS);\n        final GetReplicaInfoResponseHeader replicaInfo = (GetReplicaInfoResponseHeader) getInfoResponse.readCustomHeader();\n        assertEquals(1, replicaInfo.getMasterEpoch().intValue());\n        assertEquals(DEFAULT_IP[0], replicaInfo.getMasterAddress());\n        // Try alter SyncStateSet\n        final HashSet<Long> newSyncStateSet = new HashSet<>();\n        newSyncStateSet.add(1L);\n        newSyncStateSet.add(2L);\n        newSyncStateSet.add(3L);\n        assertTrue(alterNewInSyncSet(leader, DEFAULT_BROKER_NAME, 1L, 1, newSyncStateSet, 1));\n        return leader;\n    }\n\n    public void setBrokerAlivePredicate(DLedgerController controller, Long... deathBroker) {\n        controller.setBrokerAlivePredicate((clusterName, brokerName, brokerId) -> {\n            for (Long broker : deathBroker) {\n                if (broker.equals(brokerId)) {\n                    return false;\n                }\n            }\n            return true;\n        });\n    }\n\n    public void setBrokerElectPolicy(DLedgerController controller, Long... deathBroker) {\n        controller.setElectPolicy(new DefaultElectPolicy((clusterName, brokerName, brokerId) -> {\n            for (Long broker : deathBroker) {\n                if (broker.equals(brokerId)) {\n                    return false;\n                }\n            }\n            return true;\n        }, null));\n    }\n\n    @Test\n    public void testElectMaster() throws Exception {\n        final DLedgerController leader = mockMetaData(false);\n        final ElectMasterRequestHeader request = ElectMasterRequestHeader.ofControllerTrigger(DEFAULT_BROKER_NAME);\n        setBrokerElectPolicy(leader, 1L);\n        final RemotingCommand resp = leader.electMaster(request).get(10, TimeUnit.SECONDS);\n        final ElectMasterResponseHeader response = (ElectMasterResponseHeader) resp.readCustomHeader();\n        assertEquals(2, response.getMasterEpoch().intValue());\n        assertNotEquals(1L, response.getMasterBrokerId().longValue());\n        assertNotEquals(DEFAULT_IP[0], response.getMasterAddress());\n    }\n\n    @Test\n    public void testBrokerLifecycleListener() throws Exception {\n        final DLedgerController leader = mockMetaData(false);\n\n        assertTrue(leader.isLeaderState());\n        // Mock that master broker has been inactive, and try to elect a new master from sync-state-set\n        // But we shut down two controller, so the ElectMasterEvent will be appended to DLedger failed.\n        // So the statemachine still keep the stale master's information\n        List<DLedgerController> removed = controllers.stream().filter(controller -> controller != leader).collect(Collectors.toList());\n        for (DLedgerController dLedgerController : removed) {\n            dLedgerController.shutdown();\n            controllers.remove(dLedgerController);\n        }\n\n        final ElectMasterRequestHeader request = ElectMasterRequestHeader.ofControllerTrigger(DEFAULT_BROKER_NAME);\n        setBrokerElectPolicy(leader, 1L);\n        Exception exception = null;\n        RemotingCommand remotingCommand = null;\n        try {\n            remotingCommand = leader.electMaster(request).get(5, TimeUnit.SECONDS);\n        } catch (Exception e) {\n            exception = e;\n        }\n\n        assertTrue(exception != null ||\n            remotingCommand != null && remotingCommand.getCode() == ResponseCode.CONTROLLER_NOT_LEADER);\n\n        // Shut down leader controller\n        leader.shutdown();\n        controllers.remove(leader);\n        // Restart two controller\n        for (DLedgerController controller : removed) {\n            if (controller != leader) {\n                ControllerConfig config = controller.getControllerConfig();\n                DLedgerController newController = launchController(config.getControllerDLegerGroup(), config.getControllerDLegerPeers(), config.getControllerDLegerSelfId(), false);\n                controllers.add(newController);\n                newController.startup();\n            }\n        }\n        DLedgerController newLeader = waitLeader(controllers);\n        setBrokerAlivePredicate(newLeader, 1L);\n        // Check if the statemachine is stale\n        final RemotingCommand resp = newLeader.getReplicaInfo(new GetReplicaInfoRequestHeader(DEFAULT_BROKER_NAME)).\n            get(10, TimeUnit.SECONDS);\n        final GetReplicaInfoResponseHeader replicaInfo = (GetReplicaInfoResponseHeader) resp.readCustomHeader();\n        assertEquals(1, replicaInfo.getMasterBrokerId().longValue());\n        assertEquals(1, replicaInfo.getMasterEpoch().intValue());\n\n        // Register broker's lifecycle listener\n        AtomicBoolean atomicBoolean = new AtomicBoolean(false);\n        newLeader.registerBrokerLifecycleListener((clusterName, brokerName, brokerId) -> {\n            assertEquals(DEFAULT_BROKER_NAME, brokerName);\n            atomicBoolean.set(true);\n        });\n        Thread.sleep(2000);\n        assertTrue(atomicBoolean.get());\n    }\n\n    @Test\n    public void testAllReplicasShutdownAndRestartWithUnEnableElectUnCleanMaster() throws Exception {\n        final DLedgerController leader = mockMetaData(false);\n        final HashSet<Long> newSyncStateSet = new HashSet<>();\n        newSyncStateSet.add(1L);\n\n        assertTrue(alterNewInSyncSet(leader, DEFAULT_BROKER_NAME, 1L, 1, newSyncStateSet, 2));\n\n        // Now we trigger electMaster api, which means the old master is shutdown and want to elect a new master.\n        // However, the syncStateSet in statemachine is {1}, not more replicas can be elected as master, it will be failed.\n        final ElectMasterRequestHeader electRequest = ElectMasterRequestHeader.ofControllerTrigger(DEFAULT_BROKER_NAME);\n        setBrokerElectPolicy(leader, 1L);\n        leader.electMaster(electRequest).get(10, TimeUnit.SECONDS);\n\n        final RemotingCommand resp = leader.getReplicaInfo(new GetReplicaInfoRequestHeader(DEFAULT_BROKER_NAME)).\n            get(10, TimeUnit.SECONDS);\n        final GetReplicaInfoResponseHeader replicaInfo = (GetReplicaInfoResponseHeader) resp.readCustomHeader();\n        final SyncStateSet syncStateSet = RemotingSerializable.decode(resp.getBody(), SyncStateSet.class);\n        assertEquals(syncStateSet.getSyncStateSet(), newSyncStateSet);\n        assertEquals(null, replicaInfo.getMasterAddress());\n        assertEquals(2, replicaInfo.getMasterEpoch().intValue());\n\n        // Now, we start broker - id[2]address[127.0.0.1:9001] to try elect, but it was not in syncStateSet, so it will not be elected as master.\n        final ElectMasterRequestHeader request1 =\n            ElectMasterRequestHeader.ofBrokerTrigger(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, 2L);\n        final ElectMasterResponseHeader r1 = (ElectMasterResponseHeader) leader.electMaster(request1).get(10, TimeUnit.SECONDS).readCustomHeader();\n        assertEquals(null, r1.getMasterBrokerId());\n        assertEquals(null, r1.getMasterAddress());\n\n        // Now, we start broker - id[1]address[127.0.0.1:9000] to try elect, it will be elected as master\n        setBrokerElectPolicy(leader);\n        final ElectMasterRequestHeader request2 =\n            ElectMasterRequestHeader.ofBrokerTrigger(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, 1L);\n        final ElectMasterResponseHeader r2 = (ElectMasterResponseHeader) leader.electMaster(request2).get(10, TimeUnit.SECONDS).readCustomHeader();\n        assertEquals(1L, r2.getMasterBrokerId().longValue());\n        assertEquals(DEFAULT_IP[0], r2.getMasterAddress());\n        assertEquals(3, r2.getMasterEpoch().intValue());\n    }\n\n    @Test\n    public void testEnableElectUnCleanMaster() throws Exception {\n        final DLedgerController leader = mockMetaData(true);\n        final HashSet<Long> newSyncStateSet = new HashSet<>();\n        newSyncStateSet.add(1L);\n\n        assertTrue(alterNewInSyncSet(leader, DEFAULT_BROKER_NAME, 1L, 1, newSyncStateSet, 2));\n\n        // Now we trigger electMaster api, which means the old master is shutdown and want to elect a new master.\n        // However, event if the syncStateSet in statemachine is {DEFAULT_IP[0]}\n        // the option {enableElectUncleanMaster = true}, so the controller sill can elect a new master\n        final ElectMasterRequestHeader electRequest = ElectMasterRequestHeader.ofControllerTrigger(DEFAULT_BROKER_NAME);\n        setBrokerElectPolicy(leader, 1L);\n        final CompletableFuture<RemotingCommand> future = leader.electMaster(electRequest);\n        future.get(10, TimeUnit.SECONDS);\n\n        final RemotingCommand resp = leader.getReplicaInfo(new GetReplicaInfoRequestHeader(DEFAULT_BROKER_NAME)).get(10, TimeUnit.SECONDS);\n        final GetReplicaInfoResponseHeader replicaInfo = (GetReplicaInfoResponseHeader) resp.readCustomHeader();\n        final SyncStateSet syncStateSet = RemotingSerializable.decode(resp.getBody(), SyncStateSet.class);\n\n        final HashSet<Long> newSyncStateSet2 = new HashSet<>();\n        newSyncStateSet2.add(replicaInfo.getMasterBrokerId());\n        assertEquals(syncStateSet.getSyncStateSet(), newSyncStateSet2);\n        assertNotEquals(1L, replicaInfo.getMasterBrokerId().longValue());\n        assertNotEquals(DEFAULT_IP[0], replicaInfo.getMasterAddress());\n        assertEquals(2, replicaInfo.getMasterEpoch().intValue());\n    }\n\n    @Test\n    public void testChangeControllerLeader() throws Exception {\n        final DLedgerController leader = mockMetaData(false);\n        leader.shutdown();\n        this.controllers.remove(leader);\n        // Wait leader again\n        final DLedgerController newLeader = waitLeader(this.controllers);\n        assertNotNull(newLeader);\n\n        RemotingCommand response = await().atMost(Duration.ofSeconds(10)).until(() -> {\n            final RemotingCommand resp = newLeader.getReplicaInfo(new GetReplicaInfoRequestHeader(DEFAULT_BROKER_NAME)).get(10, TimeUnit.SECONDS);\n            if (resp.getCode() == ResponseCode.SUCCESS) {\n\n                return resp;\n            }\n            return null;\n\n        }, item -> item != null);\n        final GetReplicaInfoResponseHeader replicaInfo = (GetReplicaInfoResponseHeader) response.readCustomHeader();\n        final SyncStateSet syncStateSetResult = RemotingSerializable.decode(response.getBody(), SyncStateSet.class);\n        assertEquals(replicaInfo.getMasterAddress(), DEFAULT_IP[0]);\n        assertEquals(1, replicaInfo.getMasterEpoch().intValue());\n\n        final HashSet<Long> syncStateSet = new HashSet<>();\n        syncStateSet.add(1L);\n        syncStateSet.add(2L);\n        syncStateSet.add(3L);\n        assertEquals(syncStateSetResult.getSyncStateSet(), syncStateSet);\n    }\n}\n"
  },
  {
    "path": "controller/src/test/java/org/apache/rocketmq/controller/impl/DefaultBrokerHeartbeatManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl;\n\nimport org.apache.rocketmq.common.ControllerConfig;\nimport org.apache.rocketmq.controller.BrokerHeartbeatManager;\nimport org.apache.rocketmq.controller.impl.heartbeat.DefaultBrokerHeartbeatManager;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.TimeUnit;\n\nimport static org.junit.Assert.assertTrue;\n\npublic class DefaultBrokerHeartbeatManagerTest {\n    private BrokerHeartbeatManager heartbeatManager;\n\n    @Before\n    public void init() {\n        final ControllerConfig config = new ControllerConfig();\n        config.setScanNotActiveBrokerInterval(2000);\n        this.heartbeatManager = new DefaultBrokerHeartbeatManager(config);\n        this.heartbeatManager.initialize();\n        this.heartbeatManager.start();\n    }\n\n    @Test\n    public void testDetectBrokerAlive() throws InterruptedException {\n        final CountDownLatch latch = new CountDownLatch(1);\n        this.heartbeatManager.registerBrokerLifecycleListener((clusterName, brokerName, brokerId) -> {\n            latch.countDown();\n        });\n        this.heartbeatManager.onBrokerHeartbeat(\"cluster1\", \"broker1\", \"127.0.0.1:7000\", 1L, 3000L, null,\n            1, 1L, -1L, 0);\n        assertTrue(latch.await(5000, TimeUnit.MILLISECONDS));\n        this.heartbeatManager.shutdown();\n    }\n\n}"
  },
  {
    "path": "controller/src/test/java/org/apache/rocketmq/controller/impl/RaftBrokerHeartBeatManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl;\n\nimport com.alibaba.fastjson2.JSON;\nimport io.netty.channel.Channel;\nimport io.netty.channel.DefaultChannelPromise;\nimport org.apache.rocketmq.common.ControllerConfig;\nimport org.apache.rocketmq.controller.impl.heartbeat.BrokerIdentityInfo;\nimport org.apache.rocketmq.controller.impl.heartbeat.RaftBrokerHeartBeatManager;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.util.Collections;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.TimeUnit;\n\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class RaftBrokerHeartBeatManagerTest {\n    @Mock\n    private JRaftController jRaftController;\n    @Mock\n    private Channel brokerChannel;\n    private RaftBrokerHeartBeatManager heartbeatManager;\n    private final ControllerConfig config = new ControllerConfig();\n\n    @Before\n    public void init() {\n        when(jRaftController.isLeaderState()).thenReturn(true);\n        config.setScanNotActiveBrokerInterval(1000);\n        this.heartbeatManager = new RaftBrokerHeartBeatManager(config);\n        this.heartbeatManager.setController(jRaftController);\n        this.heartbeatManager.initialize();\n        this.heartbeatManager.start();\n    }\n\n    @Test\n    public void testDetectBrokerAlive() throws InterruptedException {\n        final CountDownLatch latch = new CountDownLatch(1);\n        this.heartbeatManager.registerBrokerLifecycleListener((clusterName, brokerName, brokerId) -> {\n            latch.countDown(); // onBrokerInactive\n        });\n        String clusterName = \"cluster-1\";\n        String brokerName = \"broker-1\";\n        String brokerAddr = \"127.0.0.1:10911\";\n        long brokerId = 1L;\n        RemotingCommand onBrokerHeartbeat = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"\");\n        RemotingCommand checkNotActiveResp1 = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"\");\n        checkNotActiveResp1.setBody(JSON.toJSONBytes(Collections.emptyList()));\n        RemotingCommand checkNotActiveResp2 = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"\");\n        checkNotActiveResp2.setBody(JSON.toJSONBytes(Collections.singletonList(new BrokerIdentityInfo(clusterName, brokerName, brokerId))));\n        when(jRaftController.onBrokerHeartBeat(any()))\n            .thenReturn(CompletableFuture.completedFuture(onBrokerHeartbeat));\n        when(jRaftController.checkNotActiveBroker(any()))\n            .thenReturn(CompletableFuture.completedFuture(checkNotActiveResp1))\n            .thenReturn(CompletableFuture.completedFuture(checkNotActiveResp2));\n        DefaultChannelPromise channelPromise = new DefaultChannelPromise(brokerChannel);\n        channelPromise.setSuccess();\n        when(brokerChannel.close()).thenReturn(channelPromise);\n        this.heartbeatManager.onBrokerHeartbeat(clusterName, brokerName, brokerAddr, brokerId, 3000L, brokerChannel,\n            1, 1L, -1L, 0);\n        assertTrue(latch.await(5000, TimeUnit.MILLISECONDS));\n        this.heartbeatManager.shutdown();\n    }\n}\n"
  },
  {
    "path": "controller/src/test/java/org/apache/rocketmq/controller/impl/event/EventSerializerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.event;\n\nimport org.apache.commons.lang3.SerializationException;\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.common.utils.FastJsonSerializer;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertThrows;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class EventSerializerTest {\n\n    @Mock\n    private FastJsonSerializer serializer;\n\n    private final EventSerializer eventSerializer = new EventSerializer();\n\n    @Before\n    public void init() throws IllegalAccessException {\n        FieldUtils.writeDeclaredField(eventSerializer, \"serializer\", serializer, true);\n    }\n\n    @Test\n    public void testSerializeValidEventMessageShouldReturnSerializedData() {\n        EventMessage eventMessage = mock(EventMessage.class);\n        EventType eventType = EventType.APPLY_BROKER_ID_EVENT;\n        when(eventMessage.getEventType()).thenReturn(eventType);\n        when(serializer.serialize(eventMessage)).thenReturn(\"{\\\"event\\\":\\\"APPLY_BROKER_ID_EVENT\\\"}\".getBytes());\n        byte[] result = eventSerializer.serialize(eventMessage);\n        assertNotNull(result);\n    }\n\n    @Test\n    public void testSerializeEventMessageWithNoEventType() {\n        EventMessage eventMessage = mock(EventMessage.class);\n        when(eventMessage.getEventType()).thenReturn(null);\n        assertThrows(NullPointerException.class, () -> eventSerializer.serialize(eventMessage));\n    }\n\n    @Test\n    public void testSerializeSerializerReturnsNullShouldReturnNull() {\n        EventMessage eventMessage = mock(EventMessage.class);\n        EventType eventType = EventType.READ_EVENT;\n        when(eventMessage.getEventType()).thenReturn(eventType);\n        when(serializer.serialize(eventMessage)).thenReturn(null);\n        byte[] result = eventSerializer.serialize(eventMessage);\n        assertNull(result);\n    }\n\n    @Test\n    public void testSerializeSerializerThrowsException() {\n        EventMessage eventMessage = mock(EventMessage.class);\n        EventType eventType = EventType.ELECT_MASTER_EVENT;\n        when(eventMessage.getEventType()).thenReturn(eventType);\n        when(serializer.serialize(eventMessage)).thenThrow(new RuntimeException(\"Serialization error\"));\n        assertThrows(RuntimeException.class, () -> eventSerializer.serialize(eventMessage));\n    }\n\n    @Test\n    public void testDeserializeBytesLessThanTwoReturnsNull() {\n        byte[] bytes = new byte[1];\n        assertNull(eventSerializer.deserialize(bytes));\n    }\n\n    @Test\n    public void testDeserializeInvalidEventIdReturnsNull() {\n        assertNull(eventSerializer.deserialize(new byte[]{0, 0xF}));\n    }\n\n    @Test\n    public void testDeserializeValidEventTypeReturnsEventMessage() throws SerializationException {\n        byte[] data = new byte[]{0, 0xF};\n        byte[] bytes = new byte[]{0, (byte) EventType.ALTER_SYNC_STATE_SET_EVENT.getId(), data[0], data[1]};\n        AlterSyncStateSetEvent alterSyncStateSetEvent = mock(AlterSyncStateSetEvent.class);\n        when(serializer.deserialize(any(byte[].class), eq(AlterSyncStateSetEvent.class))).thenReturn(alterSyncStateSetEvent);\n        EventMessage result = eventSerializer.deserialize(bytes);\n        assertNotNull(result);\n        assertTrue(result instanceof AlterSyncStateSetEvent);\n    }\n\n    @Test\n    public void testDeserializeSerializerThrowsException() throws SerializationException {\n        byte[] data = new byte[]{0, 0xF};\n        byte[] bytes = new byte[]{0, (byte) EventType.ALTER_SYNC_STATE_SET_EVENT.getId(), data[0], data[1]};\n        when(serializer.deserialize(any(byte[].class), eq(AlterSyncStateSetEvent.class))).thenThrow(new SerializationException(\"Deserialization failed\"));\n        assertThrows(SerializationException.class, () -> eventSerializer.deserialize(bytes));\n    }\n\n    @Test\n    public void testDeserializeValidEventTypeUnknownEventReturnsNull() throws SerializationException {\n        byte[] data = new byte[]{0, 0xF};\n        byte[] bytes = new byte[]{0, (short) 99, data[0], data[1]};\n        assertNull(eventSerializer.deserialize(bytes));\n    }\n}"
  },
  {
    "path": "controller/src/test/java/org/apache/rocketmq/controller/impl/event/ListEventSerializerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.event;\n\nimport org.apache.commons.lang3.SerializationException;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.util.Collections;\nimport java.util.List;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertThrows;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ListEventSerializerTest {\n\n    @Mock\n    private Logger logger;\n\n    @Test\n    public void testSerializeEmptyList() {\n        List<EventMessage> events = Collections.emptyList();\n        byte[] result = ListEventSerializer.serialize(events, null);\n        assertNotNull(result);\n        assertEquals(0, result.length);\n    }\n\n    @Test\n    public void testSerializeValidEventMessage() {\n        EventMessage eventMessage = new ElectMasterEvent(\"brokerA\", 0L);\n        List<EventMessage> events = Collections.singletonList(eventMessage);\n        byte[] result = ListEventSerializer.serialize(events, null);\n        assertNotNull(result);\n        assertTrue(result.length > 0);\n    }\n\n    @Test\n    public void testSerializeEventMessageWithNullEventType() {\n        EventMessage eventMessage = mock(EventMessage.class);\n        when(eventMessage.getEventType()).thenReturn(null);\n        List<EventMessage> events = Collections.singletonList(eventMessage);\n        assertThrows(NullPointerException.class, () -> ListEventSerializer.serialize(events, logger));\n    }\n\n    @Test\n    public void testDeserializeBytesIsNull() throws SerializationException {\n        List<EventMessage> result = ListEventSerializer.deserialize(null, logger);\n        assertNotNull(result);\n        assertTrue(result.isEmpty());\n    }\n\n    @Test\n    public void testDeserializeBytesLengthLessThanSix() throws SerializationException {\n        byte[] bytes = new byte[5];\n        List<EventMessage> result = ListEventSerializer.deserialize(bytes, logger);\n        assertNotNull(result);\n        assertTrue(result.isEmpty());\n    }\n\n    @Test\n    public void testDeserializeValidBytesWithKnownEventType() throws SerializationException {\n        byte[] bytes = new byte[]{0x01, 0x00, 0x06, 0x00, 0x00, 0x00};\n        assertNotNull(ListEventSerializer.deserialize(bytes, logger));\n    }\n\n    @Test\n    public void testDeserializeException() throws SerializationException {\n        byte[] bytes = new byte[]{0x01, 0x00, 0x06, 0x00, 0x00, 0x00, 0x02, 0x00, 0x06, 0x00, 0x00, 0x00};\n        assertThrows(ArrayIndexOutOfBoundsException.class, () -> ListEventSerializer.deserialize(bytes, logger));\n    }\n}"
  },
  {
    "path": "controller/src/test/java/org/apache/rocketmq/controller/impl/heartbeat/RaftBrokerHeartBeatManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.heartbeat;\n\nimport com.alibaba.fastjson2.JSON;\nimport io.netty.channel.Channel;\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.common.ControllerConfig;\nimport org.apache.rocketmq.common.JraftConfig;\nimport org.apache.rocketmq.controller.impl.JRaftController;\nimport org.apache.rocketmq.controller.impl.task.BrokerCloseChannelRequest;\nimport org.apache.rocketmq.controller.impl.task.CheckNotActiveBrokerRequest;\nimport org.apache.rocketmq.controller.impl.task.GetBrokerLiveInfoResponse;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.lang.reflect.Method;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ExecutionException;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertThrows;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.never;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class RaftBrokerHeartBeatManagerTest {\n\n    @Mock\n    private JRaftController controller;\n\n    private RaftBrokerHeartBeatManager raftBrokerHeartBeatManager;\n\n    @Before\n    public void init() throws IllegalAccessException {\n        ControllerConfig controllerConfig = new ControllerConfig();\n        raftBrokerHeartBeatManager = new RaftBrokerHeartBeatManager(controllerConfig);\n        FieldUtils.writeDeclaredField(raftBrokerHeartBeatManager, \"controller\", controller, true);\n    }\n\n    @Test\n    public void testOnBrokerHeartbeatSuccess() {\n        Channel channel = mock(Channel.class);\n        CompletableFuture<RemotingCommand> future = CompletableFuture.completedFuture(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"Success\"));\n        when(controller.onBrokerHeartBeat(any())).thenReturn(future);\n        raftBrokerHeartBeatManager.onBrokerHeartbeat(\"cluster1\", \"broker1\", \"127.0.0.1:10911\", 1L, 3000L, channel, 1, 1000L, 500L, 1);\n        verify(channel, never()).close();\n    }\n\n    @Test\n    public void testOnBrokerHeartbeatLeaderNotAvailable() {\n        Channel channel = mock(Channel.class);\n        CompletableFuture<RemotingCommand> future = CompletableFuture.completedFuture(RemotingCommand.createResponseCommand(ResponseCode.CONTROLLER_NOT_LEADER, \"Not Leader\"));\n        when(controller.onBrokerHeartBeat(any())).thenReturn(future);\n        raftBrokerHeartBeatManager.onBrokerHeartbeat(\"cluster1\", \"broker1\", \"127.0.0.1:10911\", 1L, 3000L, channel, 1, 1000L, 500L, 1);\n        verify(channel, never()).close();\n    }\n\n    @Test\n    public void testOnBrokerHeartbeatException() throws Exception {\n        Channel channel = mock(Channel.class);\n        CompletableFuture<RemotingCommand> future = CompletableFuture.completedFuture(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"Success\"));\n        when(controller.onBrokerHeartBeat(any())).thenReturn(future);\n        FieldUtils.writeDeclaredField(raftBrokerHeartBeatManager, \"brokerChannelIdentityInfoMap\", null, true);\n        assertThrows(NullPointerException.class, () -> raftBrokerHeartBeatManager.onBrokerHeartbeat(\"cluster1\", \"broker1\", \"127.0.0.1:10911\", 1L, 3000L, channel, 1, 1000L, 500L, 1));\n    }\n\n    @Test\n    public void testOnBrokerChannelCloseBrokerIdentityInfoNotNullSuccess() throws Exception {\n        Channel channel = mock(Channel.class);\n        BrokerIdentityInfo brokerIdentityInfo = new BrokerIdentityInfo(\"cluster1\", \"broker1\", 1L);\n        Map<Channel, BrokerIdentityInfo> brokerChannelIdentityInfoMap = new HashMap<>();\n        brokerChannelIdentityInfoMap.put(channel, brokerIdentityInfo);\n        FieldUtils.writeDeclaredField(raftBrokerHeartBeatManager, \"brokerChannelIdentityInfoMap\", brokerChannelIdentityInfoMap, true);\n        CompletableFuture<RemotingCommand> future = CompletableFuture.completedFuture(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"Success\"));\n        when(controller.onBrokerCloseChannel(any(BrokerCloseChannelRequest.class))).thenReturn(future);\n        raftBrokerHeartBeatManager.onBrokerChannelClose(channel);\n        verify(controller).onBrokerCloseChannel(any(BrokerCloseChannelRequest.class));\n    }\n\n    @Test\n    public void testOnBrokerChannelCloseBrokerIdentityInfoNotNullException() throws Exception {\n        Channel channel = mock(Channel.class);\n        BrokerIdentityInfo brokerIdentityInfo = new BrokerIdentityInfo(\"cluster1\", \"broker1\", 1L);\n        Map<Channel, BrokerIdentityInfo> brokerChannelIdentityInfoMap = new HashMap<>();\n        brokerChannelIdentityInfoMap.put(channel, brokerIdentityInfo);\n        FieldUtils.writeDeclaredField(raftBrokerHeartBeatManager, \"brokerChannelIdentityInfoMap\", brokerChannelIdentityInfoMap, true);\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        future.completeExceptionally(new ExecutionException(new RuntimeException(\"Test Exception\")));\n        when(controller.onBrokerCloseChannel(any(BrokerCloseChannelRequest.class))).thenReturn(future);\n        raftBrokerHeartBeatManager.onBrokerChannelClose(channel);\n        verify(controller).onBrokerCloseChannel(any(BrokerCloseChannelRequest.class));\n    }\n\n    @Test\n    public void testOnBrokerChannelCloseBrokerIdentityInfoNull() {\n        Channel channel = mock(Channel.class);\n        raftBrokerHeartBeatManager.onBrokerChannelClose(channel);\n        verify(controller, never()).onBrokerCloseChannel(any(BrokerCloseChannelRequest.class));\n    }\n\n    @Test\n    public void testOnBrokerChannelCloseBrokerIdentityInfoNotNullTimeoutException() throws Exception {\n        Channel channel = mock(Channel.class);\n        BrokerIdentityInfo brokerIdentityInfo = new BrokerIdentityInfo(\"cluster1\", \"broker1\", 1L);\n        Map<Channel, BrokerIdentityInfo> brokerChannelIdentityInfoMap = new HashMap<>();\n        brokerChannelIdentityInfoMap.put(channel, brokerIdentityInfo);\n        FieldUtils.writeDeclaredField(raftBrokerHeartBeatManager, \"brokerChannelIdentityInfoMap\", brokerChannelIdentityInfoMap, true);\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        when(controller.onBrokerCloseChannel(any(BrokerCloseChannelRequest.class))).thenReturn(future);\n        raftBrokerHeartBeatManager.onBrokerChannelClose(channel);\n        verify(controller).onBrokerCloseChannel(any(BrokerCloseChannelRequest.class));\n    }\n\n    @Test\n    public void testScanNotActiveBrokerSuccess() throws Exception {\n        ControllerConfig controllerConfig = new ControllerConfig();\n        JraftConfig jraftConfig = new JraftConfig();\n        jraftConfig.setjRaftScanWaitTimeoutMs(10000);\n        controllerConfig.setJraftConfig(jraftConfig);\n        raftBrokerHeartBeatManager = new RaftBrokerHeartBeatManager(controllerConfig);\n        controller = mock(JRaftController.class);\n        FieldUtils.writeDeclaredField(raftBrokerHeartBeatManager, \"controller\", controller, true);\n        when(controller.isLeaderState()).thenReturn(true);\n        FieldUtils.writeDeclaredField(raftBrokerHeartBeatManager, \"firstReceivedHeartbeatTime\", 1000, true);\n\n        List<BrokerIdentityInfo> inactiveBrokers = new ArrayList<>();\n        BrokerIdentityInfo brokerInfo = new BrokerIdentityInfo(\"testCluster\", \"testBroker\", 1L);\n        inactiveBrokers.add(brokerInfo);\n\n        RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"Success\");\n        response.setBody(JSON.toJSONString(inactiveBrokers).getBytes());\n        CompletableFuture<RemotingCommand> future = CompletableFuture.completedFuture(response);\n        when(controller.checkNotActiveBroker(any())).thenReturn(future);\n\n        Channel channel = mock(Channel.class);\n        Map<Channel, BrokerIdentityInfo> brokerChannelMap = new HashMap<>();\n        brokerChannelMap.put(channel, brokerInfo);\n        FieldUtils.writeDeclaredField(raftBrokerHeartBeatManager, \"brokerChannelIdentityInfoMap\", brokerChannelMap, true);\n\n        Method method = RaftBrokerHeartBeatManager.class.getDeclaredMethod(\"scanNotActiveBroker\");\n        method.setAccessible(true);\n        method.invoke(raftBrokerHeartBeatManager);\n\n        verify(controller).checkNotActiveBroker(any(CheckNotActiveBrokerRequest.class));\n    }\n\n    @Test\n    public void testGetBrokerLiveInfoSuccess() throws Exception {\n        String clusterName = \"cluster1\";\n        String brokerName = \"broker1\";\n        Long brokerId = 1L;\n        BrokerIdentityInfo brokerIdentityInfo = new BrokerIdentityInfo(clusterName, brokerName, brokerId);\n        BrokerLiveInfo expectedBrokerLiveInfo = new BrokerLiveInfo(brokerName, \"127.0.0.1:10911\", brokerId, System.currentTimeMillis(), 3000L, null, 1, 1000L, 500);\n        Map<BrokerIdentityInfo, BrokerLiveInfo> expectedResponse = new HashMap<>();\n        expectedResponse.put(brokerIdentityInfo, expectedBrokerLiveInfo);\n        CompletableFuture<RemotingCommand> future = CompletableFuture.completedFuture(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"Success\"));\n        future.get().writeCustomHeader(new GetBrokerLiveInfoResponse());\n        future.get().setBody(JSON.toJSONString(expectedResponse).getBytes());\n        when(controller.getBrokerLiveInfo(any())).thenReturn(future);\n        BrokerLiveInfo brokerLiveInfo = raftBrokerHeartBeatManager.getBrokerLiveInfo(clusterName, brokerName, brokerId);\n        assertEquals(expectedBrokerLiveInfo.getBrokerName(), brokerLiveInfo.getBrokerName());\n        assertEquals(expectedBrokerLiveInfo.getBrokerAddr(), brokerLiveInfo.getBrokerAddr());\n        assertEquals(expectedBrokerLiveInfo.getBrokerId(), brokerLiveInfo.getBrokerId());\n    }\n\n    @Test\n    public void testGetBrokerLiveInfoAllBrokers() throws Exception {\n        String clusterName = \"cluster1\";\n        String brokerName = \"broker1\";\n        Long brokerId = 1L;\n        BrokerIdentityInfo brokerIdentityInfo = new BrokerIdentityInfo(clusterName, brokerName, brokerId);\n        BrokerLiveInfo expectedBrokerLiveInfo = new BrokerLiveInfo(brokerName, \"127.0.0.1:10911\", brokerId, System.currentTimeMillis(), 3000L, null, 1, 1000L, 500);\n        Map<BrokerIdentityInfo, BrokerLiveInfo> expectedResponse = new HashMap<>();\n        expectedResponse.put(brokerIdentityInfo, expectedBrokerLiveInfo);\n        CompletableFuture<RemotingCommand> future = CompletableFuture.completedFuture(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"Success\"));\n        future.get().writeCustomHeader(new GetBrokerLiveInfoResponse());\n        future.get().setBody(JSON.toJSONString(expectedResponse).getBytes());\n        when(controller.getBrokerLiveInfo(any())).thenReturn(future);\n        BrokerLiveInfo brokerLiveInfo = raftBrokerHeartBeatManager.getBrokerLiveInfo(null, null, null);\n        assertNull(brokerLiveInfo);\n    }\n\n    @Test\n    public void testIsBrokerActiveBrokerActive() throws Exception {\n        String clusterName = \"cluster1\";\n        String brokerName = \"broker1\";\n        Long brokerId = 1L;\n        BrokerLiveInfo brokerLiveInfo = new BrokerLiveInfo(brokerName, \"127.0.0.1:10911\", brokerId, System.currentTimeMillis(), 3000L, null, 1, 1000L, 500);\n        Map<BrokerIdentityInfo, BrokerLiveInfo> responseMap = new HashMap<>();\n        responseMap.put(new BrokerIdentityInfo(clusterName, brokerName, brokerId), brokerLiveInfo);\n        CompletableFuture<RemotingCommand> future = CompletableFuture.completedFuture(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"Success\"));\n        future.get().writeCustomHeader(new GetBrokerLiveInfoResponse());\n        future.get().setBody(JSON.toJSONString(responseMap).getBytes());\n        when(controller.getBrokerLiveInfo(any())).thenReturn(future);\n        assertTrue(raftBrokerHeartBeatManager.isBrokerActive(clusterName, brokerName, brokerId));\n    }\n\n    @Test\n    public void testIsBrokerActiveBrokerNotActive() throws Exception {\n        String clusterName = \"cluster1\";\n        String brokerName = \"broker1\";\n        Long brokerId = 1L;\n        BrokerLiveInfo brokerLiveInfo = new BrokerLiveInfo(brokerName, \"127.0.0.1:10911\", brokerId, System.currentTimeMillis() - 4000L, 3000L, null, 1, 1000L, 500);\n        Map<BrokerIdentityInfo, BrokerLiveInfo> responseMap = new HashMap<>();\n        responseMap.put(new BrokerIdentityInfo(clusterName, brokerName, brokerId), brokerLiveInfo);\n        CompletableFuture<RemotingCommand> future = CompletableFuture.completedFuture(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"Success\"));\n        future.get().writeCustomHeader(new GetBrokerLiveInfoResponse());\n        future.get().setBody(JSON.toJSONString(responseMap).getBytes());\n        when(controller.getBrokerLiveInfo(any())).thenReturn(future);\n        assertFalse(raftBrokerHeartBeatManager.isBrokerActive(clusterName, brokerName, brokerId));\n    }\n\n    @Test\n    public void testIsBrokerActiveException() {\n        String clusterName = \"cluster1\";\n        String brokerName = \"broker1\";\n        Long brokerId = 1L;\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        future.completeExceptionally(new ExecutionException(new RuntimeException(\"Test Exception\")));\n        when(controller.getBrokerLiveInfo(any())).thenReturn(future);\n        assertFalse(raftBrokerHeartBeatManager.isBrokerActive(clusterName, brokerName, brokerId));\n    }\n\n    @Test\n    public void testIsBrokerActiveNoInfo() throws Exception {\n        String clusterName = \"cluster1\";\n        String brokerName = \"broker1\";\n        Long brokerId = 1L;\n        CompletableFuture<RemotingCommand> future = CompletableFuture.completedFuture(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"Success\"));\n        future.get().writeCustomHeader(new GetBrokerLiveInfoResponse());\n        future.get().setBody(JSON.toJSONString(new HashMap<BrokerIdentityInfo, BrokerLiveInfo>()).getBytes());\n        when(controller.getBrokerLiveInfo(any())).thenReturn(future);\n        assertFalse(raftBrokerHeartBeatManager.isBrokerActive(clusterName, brokerName, brokerId));\n    }\n\n    @Test\n    public void testIsBrokerActiveInvalidResponseCode() {\n        String clusterName = \"cluster1\";\n        String brokerName = \"broker1\";\n        Long brokerId = 1L;\n        CompletableFuture<RemotingCommand> future = CompletableFuture.completedFuture(RemotingCommand.createResponseCommand(ResponseCode.RPC_TIME_OUT, \"Timeout\"));\n        when(controller.getBrokerLiveInfo(any())).thenReturn(future);\n        assertFalse(raftBrokerHeartBeatManager.isBrokerActive(clusterName, brokerName, brokerId));\n    }\n\n    @Test\n    public void testGetActiveBrokersNumAllBrokers() throws Exception {\n        String clusterName1 = \"cluster1\";\n        String brokerName1 = \"broker1\";\n        Long brokerId1 = 1L;\n        String clusterName2 = \"cluster2\";\n        String brokerName2 = \"broker2\";\n        Long brokerId2 = 2L;\n        BrokerIdentityInfo brokerIdentityInfo1 = new BrokerIdentityInfo(clusterName1, brokerName1, brokerId1);\n        BrokerIdentityInfo brokerIdentityInfo2 = new BrokerIdentityInfo(clusterName2, brokerName2, brokerId2);\n        BrokerLiveInfo brokerLiveInfo1 = new BrokerLiveInfo(brokerName1, \"127.0.0.1:10911\", brokerId1, System.currentTimeMillis(), 3000L, null, 1, 1000L, 500);\n        BrokerLiveInfo brokerLiveInfo2 = new BrokerLiveInfo(brokerName2, \"127.0.0.1:10912\", brokerId2, System.currentTimeMillis(), 3000L, null, 1, 1000L, 500);\n        Map<BrokerIdentityInfo, BrokerLiveInfo> responseMap = new HashMap<>();\n        responseMap.put(brokerIdentityInfo1, brokerLiveInfo1);\n        responseMap.put(brokerIdentityInfo2, brokerLiveInfo2);\n        CompletableFuture<RemotingCommand> future = CompletableFuture.completedFuture(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"Success\"));\n        future.get().writeCustomHeader(new GetBrokerLiveInfoResponse());\n        future.get().setBody(JSON.toJSONString(responseMap).getBytes());\n        when(controller.getBrokerLiveInfo(any())).thenReturn(future);\n        Map<String, Map<String, Integer>> activeBrokersNum = raftBrokerHeartBeatManager.getActiveBrokersNum();\n        assertEquals(2, activeBrokersNum.size());\n        assertEquals(1, activeBrokersNum.get(clusterName1).size());\n        assertEquals(1, activeBrokersNum.get(clusterName2).size());\n        assertEquals(1, (int) activeBrokersNum.get(clusterName1).get(brokerName1));\n        assertEquals(1, (int) activeBrokersNum.get(clusterName2).get(brokerName2));\n    }\n\n    @Test\n    public void testGetActiveBrokersNum() throws Exception {\n        String clusterName1 = \"cluster1\";\n        String brokerName1 = \"broker1\";\n        Long brokerId1 = 1L;\n        String clusterName2 = \"cluster2\";\n        String brokerName2 = \"broker2\";\n        Long brokerId2 = 2L;\n        BrokerIdentityInfo brokerIdentityInfo1 = new BrokerIdentityInfo(clusterName1, brokerName1, brokerId1);\n        BrokerIdentityInfo brokerIdentityInfo2 = new BrokerIdentityInfo(clusterName2, brokerName2, brokerId2);\n        BrokerLiveInfo brokerLiveInfo1 = new BrokerLiveInfo(brokerName1, \"127.0.0.1:10911\", brokerId1, System.currentTimeMillis(), 3000L, null, 1, 1000L, 500);\n        BrokerLiveInfo brokerLiveInfo2 = new BrokerLiveInfo(brokerName2, \"127.0.0.1:10912\", brokerId2, System.currentTimeMillis() - 4000L, 3000L, null, 1, 1000L, 500);\n        Map<BrokerIdentityInfo, BrokerLiveInfo> responseMap = new HashMap<>();\n        responseMap.put(brokerIdentityInfo1, brokerLiveInfo1);\n        responseMap.put(brokerIdentityInfo2, brokerLiveInfo2);\n        CompletableFuture<RemotingCommand> future = CompletableFuture.completedFuture(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"Success\"));\n        future.get().writeCustomHeader(new GetBrokerLiveInfoResponse());\n        future.get().setBody(JSON.toJSONString(responseMap).getBytes());\n        when(controller.getBrokerLiveInfo(any())).thenReturn(future);\n        Map<String, Map<String, Integer>> activeBrokersNum = raftBrokerHeartBeatManager.getActiveBrokersNum();\n        assertEquals(1, activeBrokersNum.size());\n    }\n\n    @Test\n    public void testGetActiveBrokersNumException() {\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        future.completeExceptionally(new ExecutionException(new RuntimeException(\"Test Exception\")));\n        when(controller.getBrokerLiveInfo(any())).thenReturn(future);\n        Map<String, Map<String, Integer>> activeBrokersNum = raftBrokerHeartBeatManager.getActiveBrokersNum();\n        assertTrue(activeBrokersNum.isEmpty());\n    }\n\n    @Test\n    public void testGetActiveBrokersNumNoBrokers() throws Exception {\n        CompletableFuture<RemotingCommand> future = CompletableFuture.completedFuture(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"Success\"));\n        future.get().writeCustomHeader(new GetBrokerLiveInfoResponse());\n        future.get().setBody(JSON.toJSONString(new HashMap<BrokerIdentityInfo, BrokerLiveInfo>()).getBytes());\n        when(controller.getBrokerLiveInfo(any())).thenReturn(future);\n        Map<String, Map<String, Integer>> activeBrokersNum = raftBrokerHeartBeatManager.getActiveBrokersNum();\n        assertTrue(activeBrokersNum.isEmpty());\n    }\n\n    @Test\n    public void testGetActiveBrokersNumInvalidResponseCode() {\n        CompletableFuture<RemotingCommand> future = CompletableFuture.completedFuture(RemotingCommand.createResponseCommand(ResponseCode.RPC_TIME_OUT, \"Timeout\"));\n        when(controller.getBrokerLiveInfo(any())).thenReturn(future);\n        Map<String, Map<String, Integer>> activeBrokersNum = raftBrokerHeartBeatManager.getActiveBrokersNum();\n        assertTrue(activeBrokersNum.isEmpty());\n    }\n}\n"
  },
  {
    "path": "controller/src/test/java/org/apache/rocketmq/controller/impl/manager/RaftReplicasInfoManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.manager;\n\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.common.ControllerConfig;\nimport org.apache.rocketmq.controller.impl.event.ControllerResult;\nimport org.apache.rocketmq.controller.impl.heartbeat.BrokerIdentityInfo;\nimport org.apache.rocketmq.controller.impl.heartbeat.BrokerLiveInfo;\nimport org.apache.rocketmq.controller.impl.task.BrokerCloseChannelRequest;\nimport org.apache.rocketmq.controller.impl.task.CheckNotActiveBrokerRequest;\nimport org.apache.rocketmq.controller.impl.task.CheckNotActiveBrokerResponse;\nimport org.apache.rocketmq.controller.impl.task.GetBrokerLiveInfoRequest;\nimport org.apache.rocketmq.controller.impl.task.GetBrokerLiveInfoResponse;\nimport org.apache.rocketmq.controller.impl.task.RaftBrokerHeartBeatEventRequest;\nimport org.apache.rocketmq.controller.impl.task.RaftBrokerHeartBeatEventResponse;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertTrue;\nimport static org.junit.Assert.fail;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class RaftReplicasInfoManagerTest {\n\n    @Mock\n    private ControllerConfig controllerConfig;\n\n    private RaftReplicasInfoManager raftReplicasInfoManager;\n\n    @Before\n    public void init() {\n        raftReplicasInfoManager = new RaftReplicasInfoManager(controllerConfig);\n    }\n\n    @Test\n    public void testGetBrokerLiveInfoBrokerIdentityInfoIsNullReturnsAllBrokersInfo() throws IllegalAccessException {\n        List<BrokerIdentityInfo> brokerIdentityInfos = createBrokerIdentityInfos(2);\n        List<BrokerLiveInfo> brokerLiveInfos = createBrokerLiveInfos(2);\n        Map<BrokerIdentityInfo, BrokerLiveInfo> brokerLiveTable = new HashMap<>();\n        brokerLiveTable.put(brokerIdentityInfos.get(0), brokerLiveInfos.get(0));\n        brokerLiveTable.put(brokerIdentityInfos.get(1), brokerLiveInfos.get(1));\n        FieldUtils.writeDeclaredField(raftReplicasInfoManager, \"brokerLiveTable\", brokerLiveTable, true);\n        GetBrokerLiveInfoRequest request = new GetBrokerLiveInfoRequest();\n        ControllerResult<GetBrokerLiveInfoResponse> result = raftReplicasInfoManager.getBrokerLiveInfo(request);\n        assertNotNull(result);\n        assertEquals(ResponseCode.SUCCESS, result.getResponseCode());\n    }\n\n    @Test\n    public void testGetBrokerLiveInfoBrokerIdentityInfoExistsReturnsBrokerInfo() throws IllegalAccessException {\n        BrokerIdentityInfo brokerIdentityInfo = createBrokerIdentityInfo();\n        Map<BrokerIdentityInfo, BrokerLiveInfo> brokerLiveTable = new HashMap<>();\n        brokerLiveTable.put(brokerIdentityInfo, createBrokerLiveInfo());\n        FieldUtils.writeDeclaredField(raftReplicasInfoManager, \"brokerLiveTable\", brokerLiveTable, true);\n        GetBrokerLiveInfoRequest request = new GetBrokerLiveInfoRequest(brokerIdentityInfo);\n        ControllerResult<GetBrokerLiveInfoResponse> result = raftReplicasInfoManager.getBrokerLiveInfo(request);\n        assertNotNull(result);\n        assertEquals(ResponseCode.SUCCESS, result.getResponseCode());\n    }\n\n    @Test\n    public void testGetBrokerLiveInfoBrokerIdentityInfoNotExistsReturnsError() {\n        GetBrokerLiveInfoRequest request = new GetBrokerLiveInfoRequest(createBrokerIdentityInfo());\n        ControllerResult<GetBrokerLiveInfoResponse> result = raftReplicasInfoManager.getBrokerLiveInfo(request);\n        assertNotNull(result);\n        assertEquals(ResponseCode.CONTROLLER_BROKER_LIVE_INFO_NOT_EXISTS, result.getResponseCode());\n    }\n\n    @Test\n    public void testOnBrokerHeartBeatNewBrokerRegistered() {\n        RaftBrokerHeartBeatEventRequest request = new RaftBrokerHeartBeatEventRequest(createBrokerIdentityInfo(), createBrokerLiveInfo());\n        ControllerResult<RaftBrokerHeartBeatEventResponse> result = raftReplicasInfoManager.onBrokerHeartBeat(request);\n        assertNotNull(result);\n        assertEquals(ResponseCode.SUCCESS, result.getResponseCode());\n    }\n\n    @Test\n    public void testOnBrokerHeartBeatExistingBrokerUpdate() throws IllegalAccessException {\n        BrokerIdentityInfo brokerIdentityInfo = createBrokerIdentityInfo();\n        Map<BrokerIdentityInfo, BrokerLiveInfo> brokerLiveTable = new HashMap<>();\n        brokerLiveTable.put(brokerIdentityInfo, createBrokerLiveInfo());\n        FieldUtils.writeDeclaredField(raftReplicasInfoManager, \"brokerLiveTable\", brokerLiveTable, true);\n        BrokerLiveInfo updatedInfo = new BrokerLiveInfo(\"brokerName1\", \"brokerAddr1\", 1L, System.currentTimeMillis(), 2000L, null, 2, 200L, 2);\n        RaftBrokerHeartBeatEventRequest request = new RaftBrokerHeartBeatEventRequest(brokerIdentityInfo, updatedInfo);\n        ControllerResult<RaftBrokerHeartBeatEventResponse> result = raftReplicasInfoManager.onBrokerHeartBeat(request);\n        assertNotNull(result);\n        assertEquals(ResponseCode.SUCCESS, result.getResponseCode());\n    }\n\n    @Test\n    public void testOnBrokerCloseChannelBrokerIdentityInfoIsNullLogsWarningAndReturnsResult() {\n        assertNotNull(raftReplicasInfoManager.onBrokerCloseChannel(new BrokerCloseChannelRequest()));\n    }\n\n    @Test\n    public void testCheckNotActiveBrokerNoBrokersInTableReturnsEmptyList() {\n        CheckNotActiveBrokerRequest request = new CheckNotActiveBrokerRequest();\n        ControllerResult<CheckNotActiveBrokerResponse> result = raftReplicasInfoManager.checkNotActiveBroker(request);\n        assertNotNull(result);\n        assertEquals(ResponseCode.SUCCESS, result.getResponseCode());\n    }\n\n    @Test\n    public void testCheckNotActiveBrokerBrokerLiveTableNotEmptyIdentifiesNotActiveBrokers() throws IllegalAccessException {\n        List<BrokerIdentityInfo> brokerIdentityInfos = createBrokerIdentityInfos(2);\n        List<BrokerLiveInfo> brokerLiveInfos = createBrokerLiveInfos(2);\n        Map<BrokerIdentityInfo, BrokerLiveInfo> brokerLiveTable = new HashMap<>();\n        brokerLiveTable.put(brokerIdentityInfos.get(0), brokerLiveInfos.get(0));\n        brokerLiveTable.put(brokerIdentityInfos.get(1), brokerLiveInfos.get(1));\n        FieldUtils.writeDeclaredField(raftReplicasInfoManager, \"brokerLiveTable\", brokerLiveTable, true);\n        CheckNotActiveBrokerRequest request = new CheckNotActiveBrokerRequest();\n        ControllerResult<CheckNotActiveBrokerResponse> result = raftReplicasInfoManager.checkNotActiveBroker(request);\n        assertNotNull(result);\n        assertEquals(ResponseCode.SUCCESS, result.getResponseCode());\n        assertNotNull(result.getBody());\n    }\n\n    @Test\n    public void testCheckNotActiveBrokerSerializeErrorSetsErrorRemark() throws IllegalAccessException {\n        Map<BrokerIdentityInfo, BrokerLiveInfo> brokerLiveTable = new HashMap<>();\n        brokerLiveTable.put(createBrokerIdentityInfo(), createBrokerLiveInfo());\n        FieldUtils.writeDeclaredField(raftReplicasInfoManager, \"brokerLiveTable\", brokerLiveTable, true);\n        CheckNotActiveBrokerRequest request = new CheckNotActiveBrokerRequest();\n        ControllerResult<CheckNotActiveBrokerResponse> result = raftReplicasInfoManager.checkNotActiveBroker(request);\n        assertNotNull(result);\n    }\n\n    @Test\n    public void testIsBrokerActiveBrokerLiveInfoNotNullAndActiveReturnsTrue() throws IllegalAccessException {\n        Map<BrokerIdentityInfo, BrokerLiveInfo> brokerLiveTable = new HashMap<>();\n        brokerLiveTable.put(createBrokerIdentityInfo(), createBrokerLiveInfo());\n        FieldUtils.writeDeclaredField(raftReplicasInfoManager, \"brokerLiveTable\", brokerLiveTable, true);\n        long invokeTime = System.currentTimeMillis() + 500;\n        boolean brokerActive = raftReplicasInfoManager.isBrokerActive(\"cluster0\", \"broker0\", 0L, invokeTime);\n        assertTrue(brokerActive);\n    }\n\n    @Test\n    public void testIsBrokerActiveBrokerLiveInfoNotNullAndNotActiveReturnsFalse() throws IllegalAccessException {\n        Map<BrokerIdentityInfo, BrokerLiveInfo> brokerLiveTable = new HashMap<>();\n        brokerLiveTable.put(createBrokerIdentityInfo(), createBrokerLiveInfo());\n        FieldUtils.writeDeclaredField(raftReplicasInfoManager, \"brokerLiveTable\", brokerLiveTable, true);\n        long invokeTime = System.currentTimeMillis();\n        assertFalse(raftReplicasInfoManager.isBrokerActive(\"cluster1\", \"broker1\", 1L, invokeTime));\n    }\n\n    @Test\n    public void testIsBrokerActiveBrokerLiveInfoNullReturnsFalse() {\n        assertFalse(raftReplicasInfoManager.isBrokerActive(\"cluster1\", \"broker1\", 1L, System.currentTimeMillis()));\n    }\n\n    @Test\n    public void testSerializeWithPopulatedTablesReturnsByteArray() throws Throwable {\n        Map<BrokerIdentityInfo, BrokerLiveInfo> brokerLiveTable = new HashMap<>();\n        brokerLiveTable.put(createBrokerIdentityInfo(), createBrokerLiveInfo());\n        FieldUtils.writeDeclaredField(raftReplicasInfoManager, \"brokerLiveTable\", brokerLiveTable, true);\n        byte[] result = raftReplicasInfoManager.serialize();\n        assertNotNull(result);\n        assertTrue(result.length > 0);\n    }\n\n    @Test\n    public void testDeserializeFromValidDataSuccess() throws Throwable {\n        BrokerIdentityInfo brokerIdentityInfo = createBrokerIdentityInfo();\n        Map<BrokerIdentityInfo, BrokerLiveInfo> brokerLiveTable = new HashMap<>();\n        brokerLiveTable.put(brokerIdentityInfo, createBrokerLiveInfo());\n        FieldUtils.writeDeclaredField(raftReplicasInfoManager, \"brokerLiveTable\", brokerLiveTable, true);\n        raftReplicasInfoManager.deserializeFrom(raftReplicasInfoManager.serialize());\n        assertNotNull(brokerLiveTable);\n        assertEquals(1, brokerLiveTable.size());\n        assertTrue(brokerLiveTable.containsKey(brokerIdentityInfo));\n    }\n\n    @Test\n    public void testDeserializeFromInvalidDataExceptionThrown() {\n        byte[] invalidData = new byte[]{0x00, 0x01, 0x02, 0x03};\n        try {\n            raftReplicasInfoManager.deserializeFrom(invalidData);\n            fail(\"Expected an exception to be thrown\");\n        } catch (Throwable e) {\n            assertTrue(e instanceof ArrayIndexOutOfBoundsException);\n        }\n    }\n\n    private BrokerIdentityInfo createBrokerIdentityInfo() {\n        return createBrokerIdentityInfos(1).get(0);\n    }\n\n    private List<BrokerIdentityInfo> createBrokerIdentityInfos(final int count) {\n        List<BrokerIdentityInfo> result = new ArrayList<>();\n        for (int i = 0; i < count; i++) {\n            result.add(new BrokerIdentityInfo(\"cluster\" + i, \"broker\" + i, (long) i));\n        }\n        return result;\n    }\n\n    private BrokerLiveInfo createBrokerLiveInfo() {\n        return createBrokerLiveInfos(1).get(0);\n    }\n\n    private List<BrokerLiveInfo> createBrokerLiveInfos(final int count) {\n        List<BrokerLiveInfo> result = new ArrayList<>();\n        for (int i = 0; i < count; i++) {\n            result.add(new BrokerLiveInfo(\"brokerName\" + i,\n                    \"brokerAddr\" + i,\n                    i,\n                    System.currentTimeMillis(),\n                    1000L,\n                    null,\n                    1,\n                    100L,\n                    1));\n        }\n        return result;\n    }\n}\n"
  },
  {
    "path": "controller/src/test/java/org/apache/rocketmq/controller/impl/manager/ReplicasInfoManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.controller.impl.manager;\n\nimport org.apache.rocketmq.common.ControllerConfig;\nimport org.apache.rocketmq.controller.elect.ElectPolicy;\nimport org.apache.rocketmq.controller.elect.impl.DefaultElectPolicy;\nimport org.apache.rocketmq.controller.helper.BrokerValidPredicate;\nimport org.apache.rocketmq.controller.impl.event.ControllerResult;\nimport org.apache.rocketmq.controller.impl.event.ElectMasterEvent;\nimport org.apache.rocketmq.controller.impl.event.EventMessage;\nimport org.apache.rocketmq.controller.impl.heartbeat.DefaultBrokerHeartbeatManager;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo;\nimport org.apache.rocketmq.remoting.protocol.body.SyncStateSet;\nimport org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerResponseHeader;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport java.lang.reflect.Field;\nimport java.util.Arrays;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.TreeMap;\n\nimport static org.apache.rocketmq.controller.ControllerTestBase.DEFAULT_BROKER_NAME;\nimport static org.apache.rocketmq.controller.ControllerTestBase.DEFAULT_CLUSTER_NAME;\nimport static org.apache.rocketmq.controller.ControllerTestBase.DEFAULT_IP;\nimport static org.junit.Assert.assertArrayEquals;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertNotEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertTrue;\n\npublic class ReplicasInfoManagerTest {\n    private ReplicasInfoManager replicasInfoManager;\n\n    private DefaultBrokerHeartbeatManager heartbeatManager;\n\n    private ControllerConfig config;\n\n    @Before\n    public void init() {\n        this.config = new ControllerConfig();\n        this.config.setEnableElectUncleanMaster(false);\n        this.config.setScanNotActiveBrokerInterval(300000000);\n        this.replicasInfoManager = new ReplicasInfoManager(config);\n        this.heartbeatManager = new DefaultBrokerHeartbeatManager(config);\n        this.heartbeatManager.initialize();\n        this.heartbeatManager.start();\n    }\n\n    @After\n    public void destroy() {\n        this.replicasInfoManager = null;\n        this.heartbeatManager.shutdown();\n        this.heartbeatManager = null;\n    }\n\n    private BrokerReplicasInfo.ReplicasInfo getReplicasInfo(String brokerName) {\n        ControllerResult<Void> syncStateData = this.replicasInfoManager.getSyncStateData(Arrays.asList(brokerName), (a, b, c) -> true);\n        BrokerReplicasInfo replicasInfo = RemotingSerializable.decode(syncStateData.getBody(), BrokerReplicasInfo.class);\n        return replicasInfo.getReplicasInfoTable().get(brokerName);\n    }\n\n    public void registerNewBroker(String clusterName, String brokerName, String brokerAddress,\n        Long exceptBrokerId, Long exceptMasterBrokerId) {\n\n        // Get next brokerId\n        final GetNextBrokerIdRequestHeader getNextBrokerIdRequestHeader = new GetNextBrokerIdRequestHeader(clusterName, brokerName);\n        final ControllerResult<GetNextBrokerIdResponseHeader> nextBrokerIdResult = this.replicasInfoManager.getNextBrokerId(getNextBrokerIdRequestHeader);\n        Long nextBrokerId = nextBrokerIdResult.getResponse().getNextBrokerId();\n        String registerCheckCode = brokerAddress + \";\" + System.currentTimeMillis();\n\n        // check response\n        assertEquals(ResponseCode.SUCCESS, nextBrokerIdResult.getResponseCode());\n        assertEquals(exceptBrokerId, nextBrokerId);\n\n        // Apply brokerId\n        final ApplyBrokerIdRequestHeader applyBrokerIdRequestHeader = new ApplyBrokerIdRequestHeader(clusterName, brokerName, nextBrokerId, registerCheckCode);\n        final ControllerResult<ApplyBrokerIdResponseHeader> applyBrokerIdResult = this.replicasInfoManager.applyBrokerId(applyBrokerIdRequestHeader);\n        apply(applyBrokerIdResult.getEvents());\n\n        // check response\n        assertEquals(ResponseCode.SUCCESS, applyBrokerIdResult.getResponseCode());\n\n        // check it in state machine\n        BrokerReplicasInfo.ReplicasInfo replicasInfo = getReplicasInfo(brokerName);\n        BrokerReplicasInfo.ReplicaIdentity replicaIdentity = replicasInfo.getNotInSyncReplicas().stream().filter(x -> x.getBrokerId().equals(nextBrokerId)).findFirst().get();\n        assertNotNull(replicaIdentity);\n        assertEquals(brokerName, replicaIdentity.getBrokerName());\n        assertEquals(exceptBrokerId, replicaIdentity.getBrokerId());\n        assertEquals(brokerAddress, replicaIdentity.getBrokerAddress());\n\n        // register success\n        final RegisterBrokerToControllerRequestHeader registerBrokerToControllerRequestHeader = new RegisterBrokerToControllerRequestHeader(clusterName, brokerName, exceptBrokerId, brokerAddress);\n        ControllerResult<RegisterBrokerToControllerResponseHeader> registerSuccessResult = this.replicasInfoManager.registerBroker(registerBrokerToControllerRequestHeader, (a, b, c) -> true);\n        apply(registerSuccessResult.getEvents());\n\n        // check response\n        assertEquals(ResponseCode.SUCCESS, registerSuccessResult.getResponseCode());\n        assertEquals(exceptMasterBrokerId, registerSuccessResult.getResponse().getMasterBrokerId());\n\n    }\n\n    public void brokerElectMaster(String clusterName, Long brokerId, String brokerName, String brokerAddress,\n        boolean isFirstTryElect, boolean expectToBeElected) {\n        this.brokerElectMaster(clusterName, brokerId, brokerName, brokerAddress, isFirstTryElect, expectToBeElected, (a, b, c) -> true);\n    }\n\n    public void brokerElectMaster(String clusterName, Long brokerId, String brokerName, String brokerAddress,\n        boolean isFirstTryElect, boolean expectToBeElected, BrokerValidPredicate validPredicate) {\n\n        final GetReplicaInfoResponseHeader replicaInfoBefore = this.replicasInfoManager.getReplicaInfo(new GetReplicaInfoRequestHeader(brokerName)).getResponse();\n        BrokerReplicasInfo.ReplicasInfo syncStateSetInfo = getReplicasInfo(brokerName);\n        // Try elect itself as a master\n        ElectMasterRequestHeader requestHeader = ElectMasterRequestHeader.ofBrokerTrigger(clusterName, brokerName, brokerId);\n        final ControllerResult<ElectMasterResponseHeader> result = this.replicasInfoManager.electMaster(requestHeader, new DefaultElectPolicy(validPredicate, null));\n        apply(result.getEvents());\n\n        final GetReplicaInfoResponseHeader replicaInfoAfter = this.replicasInfoManager.getReplicaInfo(new GetReplicaInfoRequestHeader(brokerName)).getResponse();\n        final ElectMasterResponseHeader response = result.getResponse();\n\n        if (isFirstTryElect) {\n            // it should be elected\n            // check response\n            assertEquals(ResponseCode.SUCCESS, result.getResponseCode());\n            assertEquals(1, response.getMasterEpoch().intValue());\n            assertEquals(1, response.getSyncStateSetEpoch().intValue());\n            assertEquals(brokerAddress, response.getMasterAddress());\n            assertEquals(brokerId, response.getMasterBrokerId());\n            // check it in state machine\n            assertEquals(brokerAddress, replicaInfoAfter.getMasterAddress());\n            assertEquals(1, replicaInfoAfter.getMasterEpoch().intValue());\n            assertEquals(brokerId, replicaInfoAfter.getMasterBrokerId());\n        } else {\n\n            // failed because now master still exist\n            if (replicaInfoBefore.getMasterBrokerId() != null && validPredicate.check(clusterName, brokerName, replicaInfoBefore.getMasterBrokerId())) {\n                assertEquals(ResponseCode.CONTROLLER_MASTER_STILL_EXIST, result.getResponseCode());\n                assertEquals(replicaInfoBefore.getMasterAddress(), response.getMasterAddress());\n                assertEquals(replicaInfoBefore.getMasterEpoch(), response.getMasterEpoch());\n                assertEquals(replicaInfoBefore.getMasterBrokerId(), response.getMasterBrokerId());\n                assertEquals(replicaInfoBefore.getMasterBrokerId(), replicaInfoAfter.getMasterBrokerId());\n                return;\n            }\n            if (syncStateSetInfo.isExistInSync(brokerName, brokerId, brokerAddress) || this.config.isEnableElectUncleanMaster()) {\n                // a new master can be elected successfully\n                assertEquals(ResponseCode.SUCCESS, result.getResponseCode());\n                assertEquals(replicaInfoBefore.getMasterEpoch() + 1, replicaInfoAfter.getMasterEpoch().intValue());\n\n                if (expectToBeElected) {\n                    assertEquals(brokerAddress, response.getMasterAddress());\n                    assertEquals(brokerId, response.getMasterBrokerId());\n                    assertEquals(brokerAddress, replicaInfoAfter.getMasterAddress());\n                    assertEquals(brokerId, replicaInfoAfter.getMasterBrokerId());\n                }\n\n            } else {\n                // failed because elect nothing\n                assertEquals(ResponseCode.CONTROLLER_ELECT_MASTER_FAILED, result.getResponseCode());\n            }\n        }\n    }\n\n    private boolean alterNewInSyncSet(String brokerName, Long brokerId, Integer masterEpoch,\n        Set<Long> newSyncStateSet, Integer syncStateSetEpoch) {\n        final AlterSyncStateSetRequestHeader alterRequest =\n            new AlterSyncStateSetRequestHeader(brokerName, brokerId, masterEpoch);\n        final ControllerResult<AlterSyncStateSetResponseHeader> result = this.replicasInfoManager.alterSyncStateSet(alterRequest,\n            new SyncStateSet(newSyncStateSet, syncStateSetEpoch), (cluster, brokerName1, brokerId1) -> true);\n        apply(result.getEvents());\n\n        final ControllerResult<GetReplicaInfoResponseHeader> resp = this.replicasInfoManager.getReplicaInfo(new GetReplicaInfoRequestHeader(brokerName));\n        final GetReplicaInfoResponseHeader replicaInfo = resp.getResponse();\n        final SyncStateSet syncStateSet = RemotingSerializable.decode(resp.getBody(), SyncStateSet.class);\n\n        assertArrayEquals(syncStateSet.getSyncStateSet().toArray(), newSyncStateSet.toArray());\n        assertEquals(syncStateSet.getSyncStateSetEpoch(), syncStateSetEpoch + 1);\n        return true;\n    }\n\n    private void apply(final List<EventMessage> events) {\n        for (EventMessage event : events) {\n            this.replicasInfoManager.applyEvent(event);\n        }\n    }\n\n    public void mockMetaData() {\n        registerNewBroker(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[0], 1L, null);\n        registerNewBroker(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L, null);\n        registerNewBroker(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L, null);\n        brokerElectMaster(DEFAULT_CLUSTER_NAME, 1L, DEFAULT_BROKER_NAME, DEFAULT_IP[0], true, true);\n        brokerElectMaster(DEFAULT_CLUSTER_NAME, 2L, DEFAULT_BROKER_NAME, DEFAULT_IP[1], false, false);\n        brokerElectMaster(DEFAULT_CLUSTER_NAME, 3L, DEFAULT_BROKER_NAME, DEFAULT_IP[2], false, false);\n        final HashSet<Long> newSyncStateSet = new HashSet<>();\n        newSyncStateSet.add(1L);\n        newSyncStateSet.add(2L);\n        newSyncStateSet.add(3L);\n        assertTrue(alterNewInSyncSet(DEFAULT_BROKER_NAME, 1L, 1, newSyncStateSet, 1));\n    }\n\n    public void mockHeartbeatDataMasterStillAlive() {\n        this.heartbeatManager.onBrokerHeartbeat(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[0], 1L, 10000000000L, null,\n            1, 1L, -1L, 0);\n        this.heartbeatManager.onBrokerHeartbeat(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L, 10000000000L, null,\n            1, 2L, -1L, 0);\n        this.heartbeatManager.onBrokerHeartbeat(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L, 10000000000L, null,\n            1, 3L, -1L, 0);\n    }\n\n    public void mockHeartbeatDataHigherEpoch() {\n        this.heartbeatManager.onBrokerHeartbeat(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[0], 1L, -10000L, null,\n            1, 3L, -1L, 0);\n        this.heartbeatManager.onBrokerHeartbeat(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L, 10000000000L, null,\n            1, 2L, -1L, 0);\n        this.heartbeatManager.onBrokerHeartbeat(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L, 10000000000L, null,\n            0, 3L, -1L, 0);\n    }\n\n    public void mockHeartbeatDataHigherOffset() {\n        this.heartbeatManager.onBrokerHeartbeat(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[0], 1L, -10000L, null,\n            1, 3L, -1L, 0);\n        this.heartbeatManager.onBrokerHeartbeat(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L, 10000000000L, null,\n            1, 2L, -1L, 0);\n        this.heartbeatManager.onBrokerHeartbeat(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L, 10000000000L, null,\n            1, 3L, -1L, 0);\n    }\n\n    public void mockHeartbeatDataHigherPriority() {\n        this.heartbeatManager.onBrokerHeartbeat(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[0], 1L, -10000L, null,\n            1, 3L, -1L, 3);\n        this.heartbeatManager.onBrokerHeartbeat(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L, 10000000000L, null,\n            1, 3L, -1L, 2);\n        this.heartbeatManager.onBrokerHeartbeat(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L, 10000000000L, null,\n            1, 3L, -1L, 1);\n    }\n\n    @Test\n    public void testRegisterBrokerSuccess() {\n        mockMetaData();\n\n        BrokerReplicasInfo.ReplicasInfo replicasInfo = getReplicasInfo(DEFAULT_BROKER_NAME);\n        assertEquals(1L, replicasInfo.getMasterBrokerId().longValue());\n        assertEquals(DEFAULT_IP[0], replicasInfo.getMasterAddress());\n        assertEquals(1, replicasInfo.getMasterEpoch());\n        assertEquals(2, replicasInfo.getSyncStateSetEpoch());\n        assertEquals(3, replicasInfo.getInSyncReplicas().size());\n        assertEquals(0, replicasInfo.getNotInSyncReplicas().size());\n    }\n\n    @Test\n    public void testRegisterWithMasterExistResp() {\n        registerNewBroker(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[0], 1L, null);\n        registerNewBroker(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L, null);\n        brokerElectMaster(DEFAULT_CLUSTER_NAME, 1L, DEFAULT_BROKER_NAME, DEFAULT_IP[0], true, true);\n        brokerElectMaster(DEFAULT_CLUSTER_NAME, 2L, DEFAULT_BROKER_NAME, DEFAULT_IP[1], false, false);\n        registerNewBroker(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L, 1L);\n        brokerElectMaster(DEFAULT_CLUSTER_NAME, 3L, DEFAULT_BROKER_NAME, DEFAULT_IP[2], false, false);\n\n        BrokerReplicasInfo.ReplicasInfo replicasInfo = getReplicasInfo(DEFAULT_BROKER_NAME);\n        assertEquals(1L, replicasInfo.getMasterBrokerId().longValue());\n        assertEquals(DEFAULT_IP[0], replicasInfo.getMasterAddress());\n        assertEquals(1, replicasInfo.getMasterEpoch());\n        assertEquals(1, replicasInfo.getSyncStateSetEpoch());\n        assertEquals(1, replicasInfo.getInSyncReplicas().size());\n        assertEquals(2, replicasInfo.getNotInSyncReplicas().size());\n    }\n\n    @Test\n    public void testRegisterWithOldMasterInactive() {\n        mockMetaData();\n        // If now only broker-3 alive, it will be elected to be a new master\n        brokerElectMaster(DEFAULT_CLUSTER_NAME, 3L, DEFAULT_BROKER_NAME, DEFAULT_IP[2], false, true, (a, b, c) -> c.equals(3L));\n\n        // Check in statemachine\n        BrokerReplicasInfo.ReplicasInfo replicasInfo = getReplicasInfo(DEFAULT_BROKER_NAME);\n        assertEquals(3L, replicasInfo.getMasterBrokerId().longValue());\n        assertEquals(DEFAULT_IP[2], replicasInfo.getMasterAddress());\n        assertEquals(2, replicasInfo.getMasterEpoch());\n        assertEquals(3, replicasInfo.getSyncStateSetEpoch());\n        assertEquals(1, replicasInfo.getInSyncReplicas().size());\n        assertEquals(2, replicasInfo.getNotInSyncReplicas().size());\n    }\n\n    @Test\n    public void testElectMasterOldMasterStillAlive() {\n        mockMetaData();\n        final ElectMasterRequestHeader request = ElectMasterRequestHeader.ofControllerTrigger(DEFAULT_BROKER_NAME);\n        ElectPolicy electPolicy = new DefaultElectPolicy(this.heartbeatManager::isBrokerActive, this.heartbeatManager::getBrokerLiveInfo);\n        mockHeartbeatDataMasterStillAlive();\n        final ControllerResult<ElectMasterResponseHeader> cResult = this.replicasInfoManager.electMaster(request,\n            electPolicy);\n        assertEquals(ResponseCode.CONTROLLER_MASTER_STILL_EXIST, cResult.getResponseCode());\n    }\n\n    @Test\n    public void testElectMasterPreferHigherEpoch() {\n        mockMetaData();\n        final ElectMasterRequestHeader request = ElectMasterRequestHeader.ofControllerTrigger(DEFAULT_BROKER_NAME);\n        ElectPolicy electPolicy = new DefaultElectPolicy(this.heartbeatManager::isBrokerActive, this.heartbeatManager::getBrokerLiveInfo);\n        mockHeartbeatDataHigherEpoch();\n        final ControllerResult<ElectMasterResponseHeader> cResult = this.replicasInfoManager.electMaster(request,\n            electPolicy);\n        final ElectMasterResponseHeader response = cResult.getResponse();\n        assertEquals(DEFAULT_IP[1], response.getMasterAddress());\n        assertEquals(2L, response.getMasterBrokerId().longValue());\n        assertEquals(2, response.getMasterEpoch().intValue());\n    }\n\n    @Test\n    public void testElectMasterPreferHigherOffsetWhenEpochEquals() {\n        mockMetaData();\n        final ElectMasterRequestHeader request = ElectMasterRequestHeader.ofControllerTrigger(DEFAULT_BROKER_NAME);\n        ElectPolicy electPolicy = new DefaultElectPolicy(this.heartbeatManager::isBrokerActive, this.heartbeatManager::getBrokerLiveInfo);\n        mockHeartbeatDataHigherOffset();\n        final ControllerResult<ElectMasterResponseHeader> cResult = this.replicasInfoManager.electMaster(request,\n            electPolicy);\n        final ElectMasterResponseHeader response = cResult.getResponse();\n        assertEquals(DEFAULT_IP[2], response.getMasterAddress());\n        assertEquals(3L, response.getMasterBrokerId().longValue());\n        assertEquals(2, response.getMasterEpoch().intValue());\n    }\n\n    @Test\n    public void testElectMasterPreferHigherPriorityWhenEpochAndOffsetEquals() {\n        mockMetaData();\n        final ElectMasterRequestHeader request = new ElectMasterRequestHeader(DEFAULT_BROKER_NAME);\n        ElectPolicy electPolicy = new DefaultElectPolicy(this.heartbeatManager::isBrokerActive, this.heartbeatManager::getBrokerLiveInfo);\n        mockHeartbeatDataHigherPriority();\n        final ControllerResult<ElectMasterResponseHeader> cResult = this.replicasInfoManager.electMaster(request,\n            electPolicy);\n        final ElectMasterResponseHeader response = cResult.getResponse();\n        assertEquals(DEFAULT_IP[2], response.getMasterAddress());\n        assertEquals(3L, response.getMasterBrokerId().longValue());\n        assertEquals(2, response.getMasterEpoch().intValue());\n    }\n\n    @Test\n    public void testElectMaster() {\n        mockMetaData();\n        final ElectMasterRequestHeader request = ElectMasterRequestHeader.ofControllerTrigger(DEFAULT_BROKER_NAME);\n        final ControllerResult<ElectMasterResponseHeader> cResult = this.replicasInfoManager.electMaster(request,\n            new DefaultElectPolicy((cluster, brokerName, brokerId) -> !brokerId.equals(1L), null));\n        final ElectMasterResponseHeader response = cResult.getResponse();\n        assertEquals(2, response.getMasterEpoch().intValue());\n        assertNotEquals(1L, response.getMasterBrokerId().longValue());\n        assertNotEquals(DEFAULT_IP[0], response.getMasterAddress());\n        apply(cResult.getEvents());\n\n        final Set<Long> brokerSet = new HashSet<>();\n        brokerSet.add(1L);\n        brokerSet.add(2L);\n        brokerSet.add(3L);\n        assertTrue(alterNewInSyncSet(DEFAULT_BROKER_NAME, response.getMasterBrokerId(), response.getMasterEpoch(), brokerSet, response.getSyncStateSetEpoch()));\n\n        // test admin try to elect a assignedMaster, but it isn't alive\n        final ElectMasterRequestHeader assignRequest = ElectMasterRequestHeader.ofAdminTrigger(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, 1L);\n        final ControllerResult<ElectMasterResponseHeader> cResult1 = this.replicasInfoManager.electMaster(assignRequest,\n            new DefaultElectPolicy((cluster, brokerName, brokerId) -> !brokerId.equals(1L), null));\n\n        assertEquals(cResult1.getResponseCode(), ResponseCode.CONTROLLER_ELECT_MASTER_FAILED);\n\n        // test admin try to elect a assignedMaster but old master still alive, and the old master is equals to assignedMaster\n        final ElectMasterRequestHeader assignRequest1 = ElectMasterRequestHeader.ofAdminTrigger(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, response.getMasterBrokerId());\n        final ControllerResult<ElectMasterResponseHeader> cResult2 = this.replicasInfoManager.electMaster(assignRequest1,\n            new DefaultElectPolicy((cluster, brokerName, brokerId) -> true, null));\n        assertEquals(cResult2.getResponseCode(), ResponseCode.CONTROLLER_MASTER_STILL_EXIST);\n\n        // admin successful elect a assignedMaster.\n        final ElectMasterRequestHeader assignRequest2 = ElectMasterRequestHeader.ofAdminTrigger(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, 1L);\n        final ControllerResult<ElectMasterResponseHeader> cResult3 = this.replicasInfoManager.electMaster(assignRequest2,\n            new DefaultElectPolicy((cluster, brokerName, brokerId) -> !brokerId.equals(response.getMasterBrokerId()), null));\n        assertEquals(cResult3.getResponseCode(), ResponseCode.SUCCESS);\n\n        final ElectMasterResponseHeader response3 = cResult3.getResponse();\n        assertEquals(1L, response3.getMasterBrokerId().longValue());\n        assertEquals(DEFAULT_IP[0], response3.getMasterAddress());\n        assertEquals(3, response3.getMasterEpoch().intValue());\n    }\n\n    @Test\n    public void testAllReplicasShutdownAndRestart() {\n        mockMetaData();\n        final HashSet<Long> newSyncStateSet = new HashSet<>();\n        newSyncStateSet.add(1L);\n        assertTrue(alterNewInSyncSet(DEFAULT_BROKER_NAME, 1L, 1, newSyncStateSet, 2));\n\n        // Now we trigger electMaster api, which means the old master is shutdown and want to elect a new master.\n        // However, the syncStateSet in statemachine is {DEFAULT_IP[0]}, not more replicas can be elected as master, it will be failed.\n        final ElectMasterRequestHeader electRequest = ElectMasterRequestHeader.ofControllerTrigger(DEFAULT_BROKER_NAME);\n        final ControllerResult<ElectMasterResponseHeader> cResult = this.replicasInfoManager.electMaster(electRequest,\n            new DefaultElectPolicy((cluster, brokerName, brokerId) -> !brokerId.equals(1L), null));\n        final List<EventMessage> events = cResult.getEvents();\n        assertEquals(events.size(), 1);\n        final ElectMasterEvent event = (ElectMasterEvent) events.get(0);\n        assertFalse(event.getNewMasterElected());\n\n        apply(cResult.getEvents());\n\n        final GetReplicaInfoResponseHeader replicaInfo = this.replicasInfoManager.getReplicaInfo(new GetReplicaInfoRequestHeader(DEFAULT_BROKER_NAME)).getResponse();\n        assertEquals(replicaInfo.getMasterAddress(), null);\n        assertEquals(2, replicaInfo.getMasterEpoch().intValue());\n    }\n\n    @Test\n    public void testCleanBrokerData() {\n        mockMetaData();\n        CleanControllerBrokerDataRequestHeader header1 = new CleanControllerBrokerDataRequestHeader(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, \"1\");\n        ControllerResult<Void> result1 = this.replicasInfoManager.cleanBrokerData(header1, (cluster, brokerName, brokerId) -> true);\n        assertEquals(ResponseCode.CONTROLLER_INVALID_CLEAN_BROKER_METADATA, result1.getResponseCode());\n\n        CleanControllerBrokerDataRequestHeader header2 = new CleanControllerBrokerDataRequestHeader(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, null);\n        ControllerResult<Void> result2 = this.replicasInfoManager.cleanBrokerData(header2, (cluster, brokerName, brokerId) -> true);\n        assertEquals(ResponseCode.CONTROLLER_INVALID_CLEAN_BROKER_METADATA, result2.getResponseCode());\n        assertEquals(\"Broker broker-set-a is still alive, clean up failure\", result2.getRemark());\n\n        CleanControllerBrokerDataRequestHeader header3 = new CleanControllerBrokerDataRequestHeader(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, \"1\");\n        ControllerResult<Void> result3 = this.replicasInfoManager.cleanBrokerData(header3, (cluster, brokerName, brokerId) -> false);\n        assertEquals(ResponseCode.SUCCESS, result3.getResponseCode());\n\n        CleanControllerBrokerDataRequestHeader header4 = new CleanControllerBrokerDataRequestHeader(DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, \"1;2;3\");\n        ControllerResult<Void> result4 = this.replicasInfoManager.cleanBrokerData(header4, (cluster, brokerName, brokerId) -> false);\n        assertEquals(ResponseCode.SUCCESS, result4.getResponseCode());\n\n        CleanControllerBrokerDataRequestHeader header5 = new CleanControllerBrokerDataRequestHeader(DEFAULT_CLUSTER_NAME, \"broker12\", \"1;2;3\", true);\n        ControllerResult<Void> result5 = this.replicasInfoManager.cleanBrokerData(header5, (cluster, brokerName, brokerId) -> false);\n        assertEquals(ResponseCode.CONTROLLER_INVALID_CLEAN_BROKER_METADATA, result5.getResponseCode());\n        assertEquals(\"Broker broker12 is not existed,clean broker data failure.\", result5.getRemark());\n\n        CleanControllerBrokerDataRequestHeader header6 = new CleanControllerBrokerDataRequestHeader(null, \"broker12\", \"1;2;3\", true);\n        ControllerResult<Void> result6 = this.replicasInfoManager.cleanBrokerData(header6, (cluster, brokerName, brokerId) -> cluster != null);\n        assertEquals(ResponseCode.CONTROLLER_INVALID_CLEAN_BROKER_METADATA, result6.getResponseCode());\n\n        CleanControllerBrokerDataRequestHeader header7 = new CleanControllerBrokerDataRequestHeader(null, DEFAULT_BROKER_NAME, \"1;2;3\", true);\n        ControllerResult<Void> result7 = this.replicasInfoManager.cleanBrokerData(header7, (cluster, brokerName, brokerId) -> false);\n        assertEquals(ResponseCode.SUCCESS, result7.getResponseCode());\n\n    }\n\n    @Test\n    public void testSerialize() {\n        mockMetaData();\n        byte[] data;\n        try {\n            data = this.replicasInfoManager.serialize();\n        } catch (Throwable e) {\n            throw new RuntimeException(e);\n        }\n        final ReplicasInfoManager newReplicasInfoManager = new ReplicasInfoManager(config);\n        try {\n            newReplicasInfoManager.deserializeFrom(data);\n        } catch (Throwable e) {\n            throw new RuntimeException(e);\n        }\n        Map<String, BrokerReplicaInfo> oldReplicaInfoTable = new TreeMap<>();\n        Map<String, BrokerReplicaInfo> newReplicaInfoTable = new TreeMap<>();\n        Map<String/* brokerName */, SyncStateInfo> oldSyncStateTable = new TreeMap<>();\n        Map<String/* brokerName */, SyncStateInfo> newSyncStateTable = new TreeMap<>();\n        try {\n            Field field = ReplicasInfoManager.class.getDeclaredField(\"replicaInfoTable\");\n            field.setAccessible(true);\n            oldReplicaInfoTable.putAll((Map<String, BrokerReplicaInfo>) field.get(this.replicasInfoManager));\n            newReplicaInfoTable.putAll((Map<String, BrokerReplicaInfo>) field.get(newReplicasInfoManager));\n            field = ReplicasInfoManager.class.getDeclaredField(\"syncStateSetInfoTable\");\n            field.setAccessible(true);\n            oldSyncStateTable.putAll((Map<String, SyncStateInfo>) field.get(this.replicasInfoManager));\n            newSyncStateTable.putAll((Map<String, SyncStateInfo>) field.get(newReplicasInfoManager));\n        } catch (NoSuchFieldException | IllegalAccessException e) {\n            throw new RuntimeException(e);\n        }\n        assertArrayEquals(oldReplicaInfoTable.keySet().toArray(), newReplicaInfoTable.keySet().toArray());\n        assertArrayEquals(oldSyncStateTable.keySet().toArray(), newSyncStateTable.keySet().toArray());\n        for (String brokerName : oldReplicaInfoTable.keySet()) {\n            BrokerReplicaInfo oldReplicaInfo = oldReplicaInfoTable.get(brokerName);\n            BrokerReplicaInfo newReplicaInfo = newReplicaInfoTable.get(brokerName);\n            Field[] fields = oldReplicaInfo.getClass().getFields();\n        }\n    }\n}\n"
  },
  {
    "path": "controller/src/test/resources/rmq.logback-test.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n<configuration>\n\n    <appender name=\"CONSOLE\" class=\"ch.qos.logback.core.ConsoleAppender\">\n        <layout class=\"ch.qos.logback.classic.PatternLayout\">\n            <Pattern>\n                %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n\n            </Pattern>\n        </layout>\n    </appender>\n\n    <logger name=\"org.apache.rocketmq\" level=\"error\" additivity=\"false\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </logger>\n\n    <root level=\"error\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </root>\n\n</configuration>"
  },
  {
    "path": "dev/merge_rocketmq_pr.py",
    "content": "#!/usr/bin/env python\n\n#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n# This script is a modified version of the one created by the RocketMQ\n# project (https://github.com/apache/rocketmq/blob/master/dev/merge_rocketmq_pr.py).\n\n# Utility for creating well-formed pull request merges and pushing them to Apache.\n#   usage: ./merge_rocketmq_pr.py    (see config env vars below)\n#\n# This utility assumes you already have local a RocketMQ git folder and that you\n# have added remotes corresponding to both (i) the github apache RocketMQ\n# mirror and (ii) the apache git repo.\n\nimport json\nimport os\nimport re\nimport subprocess\nimport sys\nimport urllib2\n\ntry:\n    import jira.client\n    JIRA_IMPORTED = True\nexcept ImportError:\n    JIRA_IMPORTED = False\n\n# Location of your RocketMQ git development area\nROCKETMQ_HOME = os.environ.get(\"ROCKETMQ_HOME\", os.getcwd())\n# Remote name which points to the Gihub site\nPR_REMOTE_NAME = os.environ.get(\"PR_REMOTE_NAME\", \"apache-github\")\n# Remote name which points to Apache git\nPUSH_REMOTE_NAME = os.environ.get(\"PUSH_REMOTE_NAME\", \"origin\")\n# ASF JIRA username\nJIRA_USERNAME = os.environ.get(\"JIRA_USERNAME\", \"\")\n# ASF JIRA password\nJIRA_PASSWORD = os.environ.get(\"JIRA_PASSWORD\", \"\")\n# OAuth key used for issuing requests against the GitHub API. If this is not defined, then requests\n# will be unauthenticated. You should only need to configure this if you find yourself regularly\n# exceeding your IP's unauthenticated request rate limit. You can create an OAuth key at\n# https://github.com/settings/tokens. This script only requires the \"public_repo\" scope.\nGITHUB_OAUTH_KEY = os.environ.get(\"GITHUB_OAUTH_KEY\")\n\n\nGITHUB_BASE = \"https://github.com/apache/rocketmq/pull\"\nGITHUB_API_BASE = \"https://api.github.com/repos/apache/rocketmq\"\nJIRA_BASE = \"https://issues.apache.org/jira/browse\"\nJIRA_API_BASE = \"https://issues.apache.org/jira\"\n# Prefix added to temporary branches\nBRANCH_PREFIX = \"PR_TOOL\"\nDEVELOP_BRANCH = \"develop\"\n\n\ndef get_json(url):\n    try:\n        request = urllib2.Request(url)\n        if GITHUB_OAUTH_KEY:\n            request.add_header('Authorization', 'token %s' % GITHUB_OAUTH_KEY)\n        return json.load(urllib2.urlopen(request))\n    except urllib2.HTTPError as e:\n        if \"X-RateLimit-Remaining\" in e.headers and e.headers[\"X-RateLimit-Remaining\"] == '0':\n            print(\"Exceeded the GitHub API rate limit; see the instructions in \" +\n                  \"dev/merge_rocketmq_pr.py to configure an OAuth token for making authenticated \" +\n                  \"GitHub requests.\")\n        else:\n            print(\"Unable to fetch URL, exiting: %s\" % url)\n        sys.exit(-1)\n\n\ndef fail(msg):\n    print(msg)\n    clean_up()\n    sys.exit(-1)\n\n\ndef run_cmd(cmd):\n    print(cmd)\n    if isinstance(cmd, list):\n        return subprocess.check_output(cmd)\n    else:\n        return subprocess.check_output(cmd.split(\" \"))\n\n\ndef continue_maybe(prompt):\n    result = raw_input(\"\\n%s (y/n): \" % prompt)\n    if result.lower() != \"y\":\n        fail(\"Okay, exiting\")\n\n\ndef clean_up():\n    print(\"Restoring head pointer to %s\" % original_head)\n    run_cmd(\"git checkout %s\" % original_head)\n\n    branches = run_cmd(\"git branch\").replace(\" \", \"\").split(\"\\n\")\n\n    for branch in filter(lambda x: x.startswith(BRANCH_PREFIX), branches):\n        print(\"Deleting local branch %s\" % branch)\n        run_cmd(\"git branch -D %s\" % branch)\n\n\n# merge the requested PR and return the merge hash\ndef merge_pr(pr_num, target_ref, title, body, pr_repo_desc):\n    pr_branch_name = \"%s_MERGE_PR_%s\" % (BRANCH_PREFIX, pr_num)\n    target_branch_name = \"%s_MERGE_PR_%s_%s\" % (BRANCH_PREFIX, pr_num, target_ref.upper())\n    run_cmd(\"git fetch %s pull/%s/head:%s\" % (PR_REMOTE_NAME, pr_num, pr_branch_name))\n    run_cmd(\"git fetch %s %s:%s\" % (PUSH_REMOTE_NAME, target_ref, target_branch_name))\n    run_cmd(\"git checkout %s\" % target_branch_name)\n\n    had_conflicts = False\n    try:\n        run_cmd(['git', 'merge', pr_branch_name, '--squash'])\n    except Exception as e:\n        msg = \"Error merging: %s\\nWould you like to manually fix-up this merge?\" % e\n        continue_maybe(msg)\n        msg = \"Okay, please fix any conflicts and 'git add' conflicting files... Finished?\"\n        continue_maybe(msg)\n        had_conflicts = True\n\n    commit_authors = run_cmd(['git', 'log', 'HEAD..%s' % pr_branch_name,\n                             '--pretty=format:%an <%ae>']).split(\"\\n\")\n    distinct_authors = sorted(set(commit_authors),\n                              key=lambda x: commit_authors.count(x), reverse=True)\n    primary_author = raw_input(\n        \"Enter primary author in the format of \\\"name <email>\\\" [%s]: \" %\n        distinct_authors[0])\n    if primary_author == \"\":\n        primary_author = distinct_authors[0]\n\n    commits = run_cmd(['git', 'log', 'HEAD..%s' % pr_branch_name,\n                      '--pretty=format:%h [%an] %s']).split(\"\\n\\n\")\n\n    merge_message_flags = []\n\n    modified_title = raw_input(\"Modify commit log [%s]: \" % title)\n    if modified_title == \"\":\n        modified_title = title\n    merge_message_flags += [\"-m\", modified_title]\n\n    authors = \"\\n\".join([\"Author: %s\" % a for a in distinct_authors])\n\n    merge_message_flags += [\"-m\", authors]\n\n    if had_conflicts:\n        committer_name = run_cmd(\"git config --get user.name\").strip()\n        committer_email = run_cmd(\"git config --get user.email\").strip()\n        message = \"This patch had conflicts when merged, resolved by\\nCommitter: %s <%s>\" % (\n            committer_name, committer_email)\n        merge_message_flags += [\"-m\", message]\n\n    # The string \"Closes #%s\" string is required for GitHub to correctly close the PR\n    merge_message_flags += [\"-m\", \"Closes #%s from %s.\" % (pr_num, pr_repo_desc)]\n\n    run_cmd(['git', 'commit', '--author=\"%s\"' % primary_author] + merge_message_flags)\n\n    continue_maybe(\"Merge complete (local ref %s). Push to %s?\" % (\n        target_branch_name, PUSH_REMOTE_NAME))\n\n    try:\n        run_cmd('git push %s %s:%s' % (PUSH_REMOTE_NAME, target_branch_name, target_ref))\n    except Exception as e:\n        clean_up()\n        fail(\"Exception while pushing: %s\" % e)\n\n    merge_hash = run_cmd(\"git rev-parse %s\" % target_branch_name)[:8]\n    clean_up()\n    print(\"Pull request #%s merged!\" % pr_num)\n    print(\"Merge hash: %s\" % merge_hash)\n    return merge_hash\n\n\ndef cherry_pick(pr_num, merge_hash, default_branch):\n    pick_ref = raw_input(\"Enter a branch name [%s]: \" % default_branch)\n    if pick_ref == \"\":\n        pick_ref = default_branch\n\n    pick_branch_name = \"%s_PICK_PR_%s_%s\" % (BRANCH_PREFIX, pr_num, pick_ref.upper())\n\n    run_cmd(\"git fetch %s %s:%s\" % (PUSH_REMOTE_NAME, pick_ref, pick_branch_name))\n    run_cmd(\"git checkout %s\" % pick_branch_name)\n\n    try:\n        run_cmd(\"git cherry-pick -sx %s\" % merge_hash)\n    except Exception as e:\n        msg = \"Error cherry-picking: %s\\nWould you like to manually fix-up this merge?\" % e\n        continue_maybe(msg)\n        msg = \"Okay, please fix any conflicts and finish the cherry-pick. Finished?\"\n        continue_maybe(msg)\n\n    continue_maybe(\"Pick complete (local ref %s). Push to %s?\" % (\n        pick_branch_name, PUSH_REMOTE_NAME))\n\n    try:\n        run_cmd('git push %s %s:%s' % (PUSH_REMOTE_NAME, pick_branch_name, pick_ref))\n    except Exception as e:\n        clean_up()\n        fail(\"Exception while pushing: %s\" % e)\n\n    pick_hash = run_cmd(\"git rev-parse %s\" % pick_branch_name)[:8]\n    clean_up()\n\n    print(\"Pull request #%s picked into %s!\" % (pr_num, pick_ref))\n    print(\"Pick hash: %s\" % pick_hash)\n    return pick_ref\n\n\ndef fix_version_from_branch(branch, versions):\n    # Note: Assumes this is a sorted (newest->oldest) list of un-released versions\n    if branch == \"master\":\n        return versions[0]\n    else:\n        branch_ver = branch.replace(\"branch-\", \"\")\n        return filter(lambda x: x.name.startswith(branch_ver), versions)[-1]\n\n\ndef resolve_jira_issue(merge_branches, comment, default_jira_id=\"\"):\n    asf_jira = jira.client.JIRA({'server': JIRA_API_BASE},\n                                basic_auth=(JIRA_USERNAME, JIRA_PASSWORD))\n\n    jira_id = raw_input(\"Enter a JIRA id [%s]: \" % default_jira_id)\n    if jira_id == \"\":\n        jira_id = default_jira_id\n\n    try:\n        issue = asf_jira.issue(jira_id)\n    except Exception as e:\n        fail(\"ASF JIRA could not find %s\\n%s\" % (jira_id, e))\n\n    cur_status = issue.fields.status.name\n    cur_summary = issue.fields.summary\n    cur_assignee = issue.fields.assignee\n    if cur_assignee is None:\n        cur_assignee = \"NOT ASSIGNED!!!\"\n    else:\n        cur_assignee = cur_assignee.displayName\n\n    if cur_status == \"Resolved\" or cur_status == \"Closed\":\n        fail(\"JIRA issue %s already has status '%s'\" % (jira_id, cur_status))\n    print(\"=== JIRA %s ===\" % jira_id)\n    print(\"summary\\t\\t%s\\nassignee\\t%s\\nstatus\\t\\t%s\\nurl\\t\\t%s/%s\\n\" %\n          (cur_summary, cur_assignee, cur_status, JIRA_BASE, jira_id))\n\n    versions = asf_jira.project_versions(\"ROCKETMQ\")\n    versions = sorted(versions, key=lambda x: x.name, reverse=True)\n    versions = filter(lambda x: x.raw['released'] is False, versions)\n    # Consider only x.y.z versions\n    versions = filter(lambda x: re.match('\\d+\\.\\d+\\.\\d+', x.name), versions)\n\n    default_fix_versions = map(lambda x: fix_version_from_branch(x, versions).name, merge_branches)\n    for v in default_fix_versions:\n        # Handles the case where we have forked a release branch but not yet made the release.\n        # In this case, if the PR is committed to the master branch and the release branch, we\n        # only consider the release branch to be the fix version. E.g. it is not valid to have\n        # both 1.1.0 and 1.0.0 as fix versions.\n        (major, minor, patch) = v.split(\".\")\n        if patch == \"0\":\n            previous = \"%s.%s.%s\" % (major, int(minor) - 1, 0)\n            if previous in default_fix_versions:\n                default_fix_versions = filter(lambda x: x != v, default_fix_versions)\n    default_fix_versions = \",\".join(default_fix_versions)\n\n    fix_versions = raw_input(\"Enter comma-separated fix version(s) [%s]: \" % default_fix_versions)\n    if fix_versions == \"\":\n        fix_versions = default_fix_versions\n    fix_versions = fix_versions.replace(\" \", \"\").split(\",\")\n\n    def get_version_json(version_str):\n        return filter(lambda v: v.name == version_str, versions)[0].raw\n\n    jira_fix_versions = map(lambda v: get_version_json(v), fix_versions)\n\n    resolve = filter(lambda a: a['name'] == \"Resolve Issue\", asf_jira.transitions(jira_id))[0]\n    resolution = filter(lambda r: r.raw['name'] == \"Fixed\", asf_jira.resolutions())[0]\n    asf_jira.transition_issue(\n        jira_id, resolve[\"id\"], fixVersions=jira_fix_versions,\n        comment=comment, resolution={'id': resolution.raw['id']})\n\n    print(\"Successfully resolved %s with fixVersions=%s!\" % (jira_id, fix_versions))\n\n\ndef resolve_jira_issues(title, merge_branches, comment):\n    jira_ids = re.findall(\"ROCKETMQ-[0-9]{4,5}\", title)\n\n    if len(jira_ids) == 0:\n        resolve_jira_issue(merge_branches, comment)\n    for jira_id in jira_ids:\n        resolve_jira_issue(merge_branches, comment, jira_id)\n\n\ndef standardize_jira_ref(text):\n    \"\"\"\n    Standardize the [ROCKETMQ-XXXXX] [MODULE] prefix\n    Converts \"[ROCKETMQ-XXX][mllib] Issue\", \"[MLLib] ROCKETMQ-XXX. Issue\" or \"ROCKETMQ XXX [MLLIB]: Issue\" to\n    \"[ROCKETMQ-XXX][MLLIB] Issue\"\n    \"\"\"\n    jira_refs = []\n    components = []\n\n    # If the string is compliant, no need to process any further\n    if (re.search(r'^\\[ROCKETMQ-[0-9]{3,6}\\](\\[[A-Z0-9_\\s,]+\\] )+\\S+', text)):\n        return text\n\n    # Extract JIRA ref(s):\n    pattern = re.compile(r'(ROCKETMQ[-\\s]*[0-9]{3,6})+', re.IGNORECASE)\n    for ref in pattern.findall(text):\n        # Add brackets, replace spaces with a dash, & convert to uppercase\n        jira_refs.append('[' + re.sub(r'\\s+', '-', ref.upper()) + ']')\n        text = text.replace(ref, '')\n\n    # Extract rocketmq component(s):\n    # Look for alphanumeric chars, spaces, dashes, periods, and/or commas\n    pattern = re.compile(r'(\\[[\\w\\s,-\\.]+\\])', re.IGNORECASE)\n    for component in pattern.findall(text):\n        components.append(component.upper())\n        text = text.replace(component, '')\n\n    # Cleanup any remaining symbols:\n    pattern = re.compile(r'^\\W+(.*)', re.IGNORECASE)\n    if (pattern.search(text) is not None):\n        text = pattern.search(text).groups()[0]\n\n    # Assemble full text (JIRA ref(s), module(s), remaining text)\n    clean_text = ''.join(jira_refs).strip() + ''.join(components).strip() + \" \" + text.strip()\n\n    # Replace multiple spaces with a single space, e.g. if no jira refs and/or components were\n    # included\n    clean_text = re.sub(r'\\s+', ' ', clean_text.strip())\n\n    return clean_text\n\n\ndef get_current_ref():\n    ref = run_cmd(\"git rev-parse --abbrev-ref HEAD\").strip()\n    if ref == 'HEAD':\n        # The current ref is a detached HEAD, so grab its SHA.\n        return run_cmd(\"git rev-parse HEAD\").strip()\n    else:\n        return ref\n\n\ndef main():\n    global original_head\n\n    os.chdir(ROCKETMQ_HOME)\n    original_head = get_current_ref()\n\n    latest_branch = DEVELOP_BRANCH\n\n    pr_num = raw_input(\"Which pull request would you like to merge? (e.g. 34): \")\n    pr = get_json(\"%s/pulls/%s\" % (GITHUB_API_BASE, pr_num))\n    pr_events = get_json(\"%s/issues/%s/events\" % (GITHUB_API_BASE, pr_num))\n\n    url = pr[\"url\"]\n\n    # Decide whether to use the modified title or not\n    modified_title = standardize_jira_ref(pr[\"title\"])\n    if modified_title != pr[\"title\"]:\n        print(\"I've re-written the title as follows to match the standard format:\")\n        print(\"Original: %s\" % pr[\"title\"])\n        print(\"Modified: %s\" % modified_title)\n        result = raw_input(\"Would you like to use the modified title? (y/n): \")\n        if result.lower() == \"y\":\n            title = modified_title\n            print(\"Using modified title:\")\n        else:\n            title = pr[\"title\"]\n            print(\"Using original title:\")\n        print(title)\n    else:\n        title = pr[\"title\"]\n\n    body = pr[\"body\"]\n    target_ref = pr[\"base\"][\"ref\"]\n    user_login = pr[\"user\"][\"login\"]\n    base_ref = pr[\"head\"][\"ref\"]\n    pr_repo_desc = \"%s/%s\" % (user_login, base_ref)\n\n    # Merged pull requests don't appear as merged in the GitHub API;\n    # Instead, they're closed by asfgit.\n    merge_commits = \\\n        [e for e in pr_events if e[\"actor\"][\"login\"] == \"asfgit\" and e[\"event\"] == \"closed\"]\n\n    if merge_commits:\n        merge_hash = merge_commits[0][\"commit_id\"]\n        message = get_json(\"%s/commits/%s\" % (GITHUB_API_BASE, merge_hash))[\"commit\"][\"message\"]\n\n        print(\"Pull request %s has already been merged, assuming you want to backport\" % pr_num)\n        commit_is_downloaded = run_cmd(['git', 'rev-parse', '--quiet', '--verify',\n                                        \"%s^{commit}\" % merge_hash]).strip() != \"\"\n        if not commit_is_downloaded:\n            fail(\"Couldn't find any merge commit for #%s, you may need to update HEAD.\" % pr_num)\n\n        print(\"Found commit %s:\\n%s\" % (merge_hash, message))\n        cherry_pick(pr_num, merge_hash, latest_branch)\n        sys.exit(0)\n\n    if not bool(pr[\"mergeable\"]):\n        msg = \"Pull request %s is not mergeable in its current form.\\n\" % pr_num + \\\n            \"Continue? (experts only!)\"\n        continue_maybe(msg)\n\n    print(\"\\n=== Pull Request #%s ===\" % pr_num)\n    print(\"title\\t%s\\nsource\\t%s\\ntarget\\t%s\\nurl\\t%s\" %\n          (title, pr_repo_desc, target_ref, url))\n    continue_maybe(\"Proceed with merging pull request #%s?\" % pr_num)\n\n    merged_refs = [target_ref]\n\n    merge_hash = merge_pr(pr_num, target_ref, title, body, pr_repo_desc)\n\n    pick_prompt = \"Would you like to pick %s into another branch?\" % merge_hash\n    while raw_input(\"\\n%s (y/n): \" % pick_prompt).lower() == \"y\":\n        merged_refs = merged_refs + [cherry_pick(pr_num, merge_hash, latest_branch)]\n\n    if JIRA_IMPORTED:\n        if JIRA_USERNAME and JIRA_PASSWORD:\n            continue_maybe(\"Would you like to update an associated JIRA?\")\n            jira_comment = \"Issue resolved by pull request %s\\n[%s/%s]\" % \\\n                           (pr_num, GITHUB_BASE, pr_num)\n            resolve_jira_issues(title, merged_refs, jira_comment)\n        else:\n            print(\"JIRA_USERNAME and JIRA_PASSWORD not set\")\n            print(\"Exiting without trying to close the associated JIRA.\")\n    else:\n        print(\"Could not find jira-python library. Run 'sudo pip install jira' to install.\")\n        print(\"Exiting without trying to close the associated JIRA.\")\n\nif __name__ == \"__main__\":\n    import doctest\n    (failure_count, test_count) = doctest.testmod()\n    if failure_count:\n        exit(-1)\n    try:\n        main()\n    except:\n        clean_up()\n        raise\n"
  },
  {
    "path": "distribution/LICENSE-BIN",
    "content": "                                 Apache License\n                           Version 2.0, January 2004\n                        http://www.apache.org/licenses/\n\n   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n   1. Definitions.\n\n      \"License\" shall mean the terms and conditions for use, reproduction,\n      and distribution as defined by Sections 1 through 9 of this document.\n\n      \"Licensor\" shall mean the copyright owner or entity authorized by\n      the copyright owner that is granting the License.\n\n      \"Legal Entity\" shall mean the union of the acting entity and all\n      other entities that control, are controlled by, or are under common\n      control with that entity. For the purposes of this definition,\n      \"control\" means (i) the power, direct or indirect, to cause the\n      direction or management of such entity, whether by contract or\n      otherwise, or (ii) ownership of fifty percent (50%) or more of the\n      outstanding shares, or (iii) beneficial ownership of such entity.\n\n      \"You\" (or \"Your\") shall mean an individual or Legal Entity\n      exercising permissions granted by this License.\n\n      \"Source\" form shall mean the preferred form for making modifications,\n      including but not limited to software source code, documentation\n      source, and configuration files.\n\n      \"Object\" form shall mean any form resulting from mechanical\n      transformation or translation of a Source form, including but\n      not limited to compiled object code, generated documentation,\n      and conversions to other media types.\n\n      \"Work\" shall mean the work of authorship, whether in Source or\n      Object form, made available under the License, as indicated by a\n      copyright notice that is included in or attached to the work\n      (an example is provided in the Appendix below).\n\n      \"Derivative Works\" shall mean any work, whether in Source or Object\n      form, that is based on (or derived from) the Work and for which the\n      editorial revisions, annotations, elaborations, or other modifications\n      represent, as a whole, an original work of authorship. For the purposes\n      of this License, Derivative Works shall not include works that remain\n      separable from, or merely link (or bind by name) to the interfaces of,\n      the Work and Derivative Works thereof.\n\n      \"Contribution\" shall mean any work of authorship, including\n      the original version of the Work and any modifications or additions\n      to that Work or Derivative Works thereof, that is intentionally\n      submitted to Licensor for inclusion in the Work by the copyright owner\n      or by an individual or Legal Entity authorized to submit on behalf of\n      the copyright owner. For the purposes of this definition, \"submitted\"\n      means any form of electronic, verbal, or written communication sent\n      to the Licensor or its representatives, including but not limited to\n      communication on electronic mailing lists, source code control systems,\n      and issue tracking systems that are managed by, or on behalf of, the\n      Licensor for the purpose of discussing and improving the Work, but\n      excluding communication that is conspicuously marked or otherwise\n      designated in writing by the copyright owner as \"Not a Contribution.\"\n\n      \"Contributor\" shall mean Licensor and any individual or Legal Entity\n      on behalf of whom a Contribution has been received by Licensor and\n      subsequently incorporated within the Work.\n\n   2. Grant of Copyright License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      copyright license to reproduce, prepare Derivative Works of,\n      publicly display, publicly perform, sublicense, and distribute the\n      Work and such Derivative Works in Source or Object form.\n\n   3. Grant of Patent License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      (except as stated in this section) patent license to make, have made,\n      use, offer to sell, sell, import, and otherwise transfer the Work,\n      where such license applies only to those patent claims licensable\n      by such Contributor that are necessarily infringed by their\n      Contribution(s) alone or by combination of their Contribution(s)\n      with the Work to which such Contribution(s) was submitted. If You\n      institute patent litigation against any entity (including a\n      cross-claim or counterclaim in a lawsuit) alleging that the Work\n      or a Contribution incorporated within the Work constitutes direct\n      or contributory patent infringement, then any patent licenses\n      granted to You under this License for that Work shall terminate\n      as of the date such litigation is filed.\n\n   4. Redistribution. You may reproduce and distribute copies of the\n      Work or Derivative Works thereof in any medium, with or without\n      modifications, and in Source or Object form, provided that You\n      meet the following conditions:\n\n      (a) You must give any other recipients of the Work or\n          Derivative Works a copy of this License; and\n\n      (b) You must cause any modified files to carry prominent notices\n          stating that You changed the files; and\n\n      (c) You must retain, in the Source form of any Derivative Works\n          that You distribute, all copyright, patent, trademark, and\n          attribution notices from the Source form of the Work,\n          excluding those notices that do not pertain to any part of\n          the Derivative Works; and\n\n      (d) If the Work includes a \"NOTICE\" text file as part of its\n          distribution, then any Derivative Works that You distribute must\n          include a readable copy of the attribution notices contained\n          within such NOTICE file, excluding those notices that do not\n          pertain to any part of the Derivative Works, in at least one\n          of the following places: within a NOTICE text file distributed\n          as part of the Derivative Works; within the Source form or\n          documentation, if provided along with the Derivative Works; or,\n          within a display generated by the Derivative Works, if and\n          wherever such third-party notices normally appear. The contents\n          of the NOTICE file are for informational purposes only and\n          do not modify the License. You may add Your own attribution\n          notices within Derivative Works that You distribute, alongside\n          or as an addendum to the NOTICE text from the Work, provided\n          that such additional attribution notices cannot be construed\n          as modifying the License.\n\n      You may add Your own copyright statement to Your modifications and\n      may provide additional or different license terms and conditions\n      for use, reproduction, or distribution of Your modifications, or\n      for any such Derivative Works as a whole, provided Your use,\n      reproduction, and distribution of the Work otherwise complies with\n      the conditions stated in this License.\n\n   5. Submission of Contributions. Unless You explicitly state otherwise,\n      any Contribution intentionally submitted for inclusion in the Work\n      by You to the Licensor shall be under the terms and conditions of\n      this License, without any additional terms or conditions.\n      Notwithstanding the above, nothing herein shall supersede or modify\n      the terms of any separate license agreement you may have executed\n      with Licensor regarding such Contributions.\n\n   6. Trademarks. This License does not grant permission to use the trade\n      names, trademarks, service marks, or product names of the Licensor,\n      except as required for reasonable and customary use in describing the\n      origin of the Work and reproducing the content of the NOTICE file.\n\n   7. Disclaimer of Warranty. Unless required by applicable law or\n      agreed to in writing, Licensor provides the Work (and each\n      Contributor provides its Contributions) on an \"AS IS\" BASIS,\n      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n      implied, including, without limitation, any warranties or conditions\n      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n      PARTICULAR PURPOSE. You are solely responsible for determining the\n      appropriateness of using or redistributing the Work and assume any\n      risks associated with Your exercise of permissions under this License.\n\n   8. Limitation of Liability. In no event and under no legal theory,\n      whether in tort (including negligence), contract, or otherwise,\n      unless required by applicable law (such as deliberate and grossly\n      negligent acts) or agreed to in writing, shall any Contributor be\n      liable to You for damages, including any direct, indirect, special,\n      incidental, or consequential damages of any character arising as a\n      result of this License or out of the use or inability to use the\n      Work (including but not limited to damages for loss of goodwill,\n      work stoppage, computer failure or malfunction, or any and all\n      other commercial damages or losses), even if such Contributor\n      has been advised of the possibility of such damages.\n\n   9. Accepting Warranty or Additional Liability. While redistributing\n      the Work or Derivative Works thereof, You may choose to offer,\n      and charge a fee for, acceptance of support, warranty, indemnity,\n      or other liability obligations and/or rights consistent with this\n      License. However, in accepting such obligations, You may act only\n      on Your own behalf and on Your sole responsibility, not on behalf\n      of any other Contributor, and only if You agree to indemnify,\n      defend, and hold each Contributor harmless for any liability\n      incurred by, or claims asserted against, such Contributor by reason\n      of your accepting any such warranty or additional liability.\n\n   END OF TERMS AND CONDITIONS\n\n   APPENDIX: How to apply the Apache License to your work.\n\n      To apply the Apache License to your work, attach the following\n      boilerplate notice, with the fields enclosed by brackets \"[]\"\n      replaced with your own identifying information. (Don't include\n      the brackets!)  The text should be enclosed in the appropriate\n      comment syntax for the file format. We also recommend that a\n      file or class name and description of purpose be included on the\n      same \"printed page\" as the copyright notice for easier\n      identification within third-party archives.\n\n   Copyright [yyyy] [name of copyright owner]\n\n   Licensed under the Apache License, Version 2.0 (the \"License\");\n   you may not use this file except in compliance with the License.\n   You may obtain a copy of the License at\n\n       http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing, software\n   distributed under the License is distributed on an \"AS IS\" BASIS,\n   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n   See the License for the specific language governing permissions and\n   limitations under the License.\n\n------\nThis product has a bundle logback, which is available under the EPL v1.0 License.\nThe source code of logback can be found at https://github.com/qos-ch/logback.\n\nLogback LICENSE\n---------------\n\nLogback: the reliable, generic, fast and flexible logging framework.\nCopyright (C) 1999-2015, QOS.ch. All rights reserved.\n\nThis program and the accompanying materials are dual-licensed under\neither the terms of the Eclipse Public License v1.0 as published by\nthe Eclipse Foundation\n\n  or (per the licensee's choosing)\n\nunder the terms of the GNU Lesser General Public License version 2.1\nas published by the Free Software Foundation.\n\n------\nThis product has a bundle slf4j, which is available under the MIT License.\nThe source code of slf4j can be found at https://github.com/qos-ch/slf4j.\n\n Copyright (c) 2004-2017 QOS.ch\n All rights reserved.\n\n Permission is hereby granted, free  of charge, to any person obtaining\n a  copy  of this  software  and  associated  documentation files  (the\n \"Software\"), to  deal in  the Software without  restriction, including\n without limitation  the rights to  use, copy, modify,  merge, publish,\n distribute,  sublicense, and/or sell  copies of  the Software,  and to\n permit persons to whom the Software  is furnished to do so, subject to\n the following conditions:\n\n The  above  copyright  notice  and  this permission  notice  shall  be\n included in all copies or substantial portions of the Software.\n\n THE  SOFTWARE IS  PROVIDED  \"AS  IS\", WITHOUT  WARRANTY  OF ANY  KIND,\n EXPRESS OR  IMPLIED, INCLUDING  BUT NOT LIMITED  TO THE  WARRANTIES OF\n MERCHANTABILITY,    FITNESS    FOR    A   PARTICULAR    PURPOSE    AND\n NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE\n LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\n OF CONTRACT, TORT OR OTHERWISE,  ARISING FROM, OUT OF OR IN CONNECTION\n WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n\n------\nThis product has a bundle fastjson, which is available under the ASL2 License.\nThe source code of fastjson can be found at https://github.com/alibaba/fastjson.\n\n Copyright 1999-2016 Alibaba Group Holding Ltd.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n------\nThis product has a bundle javassist, which is available under the ASL2 License.\nThe source code of javassist can be found at https://github.com/jboss-javassist/javassist.\n\n Copyright (C) 1999- by Shigeru Chiba, All rights reserved.\n\n Javassist (JAVA programming ASSISTant) makes Java bytecode manipulation simple.\n It is a class library for editing bytecodes in Java; it enables Java programs to define a new class\n at runtime and to modify a class file when the JVM loads it. Unlike other similar bytecode editors,\n Javassist provides two levels of API: source level and bytecode level. If the users use the source- level API,\n they can edit a class file without knowledge of the specifications of the Java bytecode.\n The whole API is designed with only the vocabulary of the Java language.\n You can even specify inserted bytecode in the form of source text; Javassist compiles it on the fly.\n On the other hand, the bytecode-level API allows the users to directly edit a class file as other editors.\n\n This software is distributed under the Mozilla Public License Version 1.1,\n the GNU Lesser General Public License Version 2.1 or later, or the Apache License Version 2.0.\n\n------\nThis product has a bundle jna, which is available under the ASL2 License.\nThe source code of jna can be found at https://github.com/java-native-access/jna.\n\n This copy of JNA is licensed under the\n Apache (Software) License, version 2.0 (\"the License\").\n See the License for details about distribution rights, and the\n specific rights regarding derivate works.\n\n You may obtain a copy of the License at:\n\n http://www.apache.org/licenses/\n\n A copy is also included in the downloadable source code package\n containing JNA, in file \"AL2.0\", under the same directory\n as this file.\n------\nThis product has a bundle guava, which is available under the ASL2 License.\nThe source code of guava can be found at https://github.com/google/guava.\n\n Copyright (C) 2007 The Guava authors \n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n------\nThis product has a bundle OpenMessaging, which is available under the ASL2 License.\nThe source code of OpenMessaging can be found at https://github.com/openmessaging/openmessaging.\n\n Copyright (C) 2017 The OpenMessaging authors.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n"
  },
  {
    "path": "distribution/NOTICE-BIN",
    "content": "Apache RocketMQ\nCopyright 2016-2022 The Apache Software Foundation\n\nThis product includes software developed at\nThe Apache Software Foundation (http://www.apache.org/).\n\n------\nThis product has a bundle netty:\n                            The Netty Project\n                            =================\n\nPlease visit the Netty web site for more information:\n\n  * http://netty.io/\n\nCopyright 2014 The Netty Project\n\nThe Netty Project licenses this file to you under the Apache License,\nversion 2.0 (the \"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at:\n\n  http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\nWARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\nLicense for the specific language governing permissions and limitations\nunder the License.\n\nAlso, please refer to each LICENSE.<component>.txt file, which is located in\nthe 'license' directory of the distribution file, for the license terms of the\ncomponents that this product depends on.\n\n------\nThis product has a bundle commons-lang, which includes software from the Spring Framework,\nunder the Apache License 2.0 (see: StringUtils.containsWhitespace())\n"
  },
  {
    "path": "distribution/benchmark/batchproducer.sh",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nsh ./runclass.sh org.apache.rocketmq.example.benchmark.BatchProducer $@ &\n"
  },
  {
    "path": "distribution/benchmark/consumer.sh",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nsh ./runclass.sh org.apache.rocketmq.example.benchmark.Consumer $@ &\n"
  },
  {
    "path": "distribution/benchmark/producer.sh",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nsh ./runclass.sh -Dorg.apache.rocketmq.client.sendSmartMsg=true org.apache.rocketmq.example.benchmark.Producer $@ &\n"
  },
  {
    "path": "distribution/benchmark/runclass.sh",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nif [ $# -lt 1 ];\nthen\n  echo \"USAGE: $0 classname opts\"\n  exit 1\nfi\n\nBASE_DIR=$(dirname $0)/..\nCLASSPATH=.:${BASE_DIR}/conf:${BASE_DIR}/lib/*:${CLASSPATH}\n\n# The RAMDisk initializing size in MB on Darwin OS for gc-log\nDIR_SIZE_IN_MB=600\n\nchoose_gc_log_directory()\n{\n    case \"`uname`\" in\n        Darwin)\n            if [ ! -d \"/Volumes/RAMDisk\" ]; then\n                # create ram disk on Darwin systems as gc-log directory\n                DEV=`hdiutil attach -nomount ram://$((2 * 1024 * DIR_SIZE_IN_MB))` > /dev/null\n                diskutil eraseVolume HFS+ RAMDisk ${DEV} > /dev/null\n                echo \"Create RAMDisk /Volumes/RAMDisk for gc logging on Darwin OS.\"\n            fi\n            GC_LOG_DIR=\"/Volumes/RAMDisk\"\n        ;;\n        *)\n            # check if /dev/shm exists on other systems\n            if [ -d \"/dev/shm\" ]; then\n                GC_LOG_DIR=\"/dev/shm\"\n            else\n                GC_LOG_DIR=${BASE_DIR}\n            fi\n        ;;\n    esac\n}\n\nchoose_gc_log_directory\n\nJAVA_OPT=\"${JAVA_OPT} -server -Xms1g -Xmx1g -Xmn256m -XX:PermSize=128m -XX:MaxPermSize=320m\"\nJAVA_OPT=\"${JAVA_OPT} -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:SurvivorRatio=8 -XX:+DisableExplicitGC\"\nJAVA_OPT=\"${JAVA_OPT} -verbose:gc -Xloggc:${GC_LOG_DIR}/rmq_run_class_gc_%p_%t.log -XX:+PrintGCDetails\"\nJAVA_OPT=\"${JAVA_OPT} -XX:-OmitStackTraceInFastThrow\"\nJAVA_OPT=\"${JAVA_OPT} -XX:-UseLargePages\"\nJAVA_OPT=\"${JAVA_OPT} -XX:+PerfDisableSharedMem\"\n#JAVA_OPT=\"${JAVA_OPT} -Xdebug -Xrunjdwp:transport=dt_socket,address=9555,server=y,suspend=n\"\nJAVA_OPT=\"${JAVA_OPT} -cp ${CLASSPATH}\"\nJAVA_OPT=\"${JAVA_OPT} -Djava.ext.dirs=${JAVA_HOME}/jre/lib/ext:${BASE_DIR}/lib:${JAVA_HOME}/lib/ext\"\nJAVA_OPT=\"${JAVA_OPT} -Drmq.logback.configurationFile=${BASE_DIR}/conf/rmq.client.logback.xml\"\n\nif [ -z \"$JAVA_HOME\" ]; then\n  JAVA_HOME=/usr/java\nfi\n\nJAVA=\"$JAVA_HOME/bin/java\"\n\n$JAVA ${JAVA_OPT} $@\n"
  },
  {
    "path": "distribution/benchmark/shutdown.sh",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\ncase $1 in\n    producer)\n\n    pid=`ps ax | grep -i 'org.apache.rocketmq.example.benchmark.Producer' |grep java | grep -v grep | awk '{print $1}'`\n    if [ -z \"$pid\" ] ; then\n            echo \"No benchmark producer running.\"\n            exit -1;\n    fi\n\n    echo \"The benchmark producer(${pid}) is running...\"\n\n    kill ${pid}\n\n    echo \"Send shutdown request to benchmark producer(${pid}) OK\"\n    ;;\n    consumer)\n\n    pid=`ps ax | grep -i 'org.apache.rocketmq.example.benchmark.Consumer' |grep java | grep -v grep | awk '{print $1}'`\n    if [ -z \"$pid\" ] ; then\n            echo \"No benchmark consumer running.\"\n            exit -1;\n    fi\n\n    echo \"The benchmark consumer(${pid}) is running...\"\n\n    kill ${pid}\n\n    echo \"Send shutdown request to benchmark consumer(${pid}) OK\"\n    ;;\n    tproducer)\n\n    pid=`ps ax | grep -i 'org.apache.rocketmq.example.benchmark.TransactionProducer' |grep java | grep -v grep | awk '{print $1}'`\n    if [ -z \"$pid\" ] ; then\n            echo \"No benchmark transaction producer running.\"\n            exit -1;\n    fi\n\n    echo \"The benchmark transaction producer(${pid}) is running...\"\n\n    kill ${pid}\n\n    echo \"Send shutdown request to benchmark transaction producer(${pid}) OK\"\n    ;;\n    bproducer)\n\n    pid=`ps ax | grep -i 'org.apache.rocketmq.example.benchmark.BatchProducer' |grep java | grep -v grep | awk '{print $1}'`\n    if [ -z \"$pid\" ] ; then\n            echo \"No benchmark batch producer running.\"\n            exit -1;\n    fi\n\n    echo \"The benchmark batch producer(${pid}) is running...\"\n\n    kill ${pid}\n\n    echo \"Send shutdown request to benchmark batch producer(${pid}) OK\"\n    ;;\n    *)\n    echo \"Usage: shutdown producer | consumer | tproducer | bproducer\"\nesac\n"
  },
  {
    "path": "distribution/benchmark/tproducer.sh",
    "content": "#!/bin/sh\r\n\r\n# Licensed to the Apache Software Foundation (ASF) under one or more\r\n# contributor license agreements.  See the NOTICE file distributed with\r\n# this work for additional information regarding copyright ownership.\r\n# The ASF licenses this file to You under the Apache License, Version 2.0\r\n# (the \"License\"); you may not use this file except in compliance with\r\n# the License.  You may obtain a copy of the License at\r\n#\r\n#     http://www.apache.org/licenses/LICENSE-2.0\r\n#\r\n# Unless required by applicable law or agreed to in writing, software\r\n# distributed under the License is distributed on an \"AS IS\" BASIS,\r\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r\n# See the License for the specific language governing permissions and\r\n# limitations under the License.\r\n\r\nsh ./runclass.sh org.apache.rocketmq.example.benchmark.TransactionProducer $@ &\r\n"
  },
  {
    "path": "distribution/bin/README.md",
    "content": "### Operating system tuning\nBefore deploying broker servers, it is highly recommended to run **os.sh**, which optimizes your operating system for better performance.\n\n## Notice\n### os.sh should be executed only once with root permission.\n### os.sh parameter settings are for reference purposes only. You can tune them according to your target host configurations.\n\n\n### Start broker\n* Unix platform\n\n  `nohup sh mqbroker &`\n\n### Shutdown broker\n  sh mqshutdown broker\n\n### Start Nameserver\n* Unix platform\n\n  `nohup sh mqnamesrv &`\n\n### Shutdown Nameserver\n    sh mqshutdown namesrv\n\n### Update or create Topic\n    sh mqadmin updateTopic -b 127.0.0.1:10911 -t TopicA\n\n### Update or create subscription group\n    sh mqadmin updateSubGroup -b 127.0.0.1:10911 -g SubGroupA"
  },
  {
    "path": "distribution/bin/cachedog.sh",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nexport PATH=$PATH:/sbin\n\nwhile true; do\n    nr_free_pages=`fgrep -A 10 Normal /proc/zoneinfo  |grep nr_free_pages  |awk -F ' ' '{print $2}'`\n    high=`fgrep -A 10 Normal /proc/zoneinfo  |grep high  |awk -F ' ' '{print $2}'`\n\n    NOW_DATE=`date +%D`\n    NOW_TIME=`date +%T`\n\n    if [ ${nr_free_pages} -le ${high} ]; then\n        sysctl -w vm.drop_caches=3\n        nr_free_pages_new=`fgrep -A 10 Normal /proc/zoneinfo  |grep nr_free_pages  |awk -F ' ' '{print $2}'`\n\n        printf \"%s %s [CLEAN] nr_free_pages < high, clean cache. nr_free_pages=%s ====> nr_free_pages=%s\\n\" \"${NOW_DATE}\" \"${NOW_TIME}\" ${nr_free_pages} ${nr_free_pages_new}\n\n        sysctl -w vm.drop_caches=1\n        echo\n        echo\n        echo\n    else\n        printf \"%s %s [NOTHING] nr_free_pages=%s high=%s\\n\" \"${NOW_DATE}\" \"${NOW_TIME}\" ${nr_free_pages} ${high}\n    fi\n\n    sleep 1\ndone\n"
  },
  {
    "path": "distribution/bin/cleancache.sh",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nexport PATH=$PATH:/sbin\n\nsysctl -w vm.drop_caches=3\n"
  },
  {
    "path": "distribution/bin/cleancache.v1.sh",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nexport PATH=$PATH:/sbin\n\n#\n# GB\n#\nfunction changeFreeCache()\n{\n    EXTRA=$1\n    MIN=$2\n    sysctl -w vm.extra_free_kbytes=${EXTRA}000000\n    sysctl -w vm.min_free_kbytes=${MIN}000000\n}\n\n\nif [ $# -ne 1 ]\nthen\n    echo \"Usage: $0 freecache(GB)\"\n    echo \"Example: $0 15\"\n    exit\nfi\n\nchangeFreeCache 3 $1\nchangeFreeCache 3 1\n"
  },
  {
    "path": "distribution/bin/controller/fast-try-independent-deployment.cmd",
    "content": "@echo off\nrem Licensed to the Apache Software Foundation (ASF) under one or more\nrem contributor license agreements.  See the NOTICE file distributed with\nrem this work for additional information regarding copyright ownership.\nrem The ASF licenses this file to You under the Apache License, Version 2.0\nrem (the \"License\"); you may not use this file except in compliance with\nrem the License.  You may obtain a copy of the License at\nrem\nrem     http://www.apache.org/licenses/LICENSE-2.0\nrem\nrem Unless required by applicable law or agreed to in writing, software\nrem distributed under the License is distributed on an \"AS IS\" BASIS,\nrem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nrem See the License for the specific language governing permissions and\nrem limitations under the License.\n\nset ROCKETMQ_HOME=%cd%\nif not exist \"%ROCKETMQ_HOME%\\conf\\controller\\cluster-3n-independent\\controller-n0.conf\" echo Make sure the %ROCKETMQ_HOME%\\conf\\controller\\cluster-3n-independent\\controller-n0.conf exists & EXIT /B 1\nif not exist \"%ROCKETMQ_HOME%\\conf\\controller\\cluster-3n-independent\\controller-n1.conf\" echo Make sure the %ROCKETMQ_HOME%\\conf\\controller\\cluster-3n-independent\\controller-n1.conf exists & EXIT /B 1\nif not exist \"%ROCKETMQ_HOME%\\conf\\controller\\cluster-3n-independent\\controller-n2.conf\" echo Make sure the %ROCKETMQ_HOME%\\conf\\controller\\cluster-3n-independent\\controller-n2.conf exists & EXIT /B 1\n\nset \"JAVA_OPT_EXT= -server -Xms512m -Xmx512m\"\nstart call \"%ROCKETMQ_HOME%\\bin\\runserver.cmd\" org.apache.rocketmq.controller.ControllerStartup -c %ROCKETMQ_HOME%\\conf\\controller\\cluster-3n-independent\\controller-n0.conf\nIF %ERRORLEVEL% EQU 0 (\n   ECHO \"Controller start OK\"\n)\ntimeout /T 3 /NOBREAK\nstart call \"%ROCKETMQ_HOME%\\bin\\runserver.cmd\" org.apache.rocketmq.controller.ControllerStartup -c %ROCKETMQ_HOME%\\conf\\controller\\cluster-3n-independent\\controller-n1.conf\nIF %ERRORLEVEL% EQU 0 (\n   ECHO \"Controller start OK\"\n)\ntimeout /T 3 /NOBREAK\nstart call \"%ROCKETMQ_HOME%\\bin\\runserver.cmd\" org.apache.rocketmq.controller.ControllerStartup -c %ROCKETMQ_HOME%\\conf\\controller\\cluster-3n-independent\\controller-n2.conf\nIF %ERRORLEVEL% EQU 0 (\n   ECHO \"Controller start OK\"\n)\n"
  },
  {
    "path": "distribution/bin/controller/fast-try-independent-deployment.sh",
    "content": "#!/usr/bin/env bash\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n## Revise the base dir\nCURRENT_DIR=\"$(cd \"$(dirname \"$0\")\"; pwd)\"\nRMQ_DIR=$CURRENT_DIR/../..\ncd $RMQ_DIR\n\nstartController() {\n    export JAVA_OPT_EXT=\" -Xms512m -Xmx512m  \"\n    conf_name=$1\n    nohup bin/mqcontroller -c $conf_name &\n}\n\nstopController() {\n    PIDS=$(ps -ef|grep java|grep ControllerStartup|grep -v grep|awk '{print $2}')\n    if [ ! -z \"$PIDS\" ]; then\n        kill -s TERM $PIDS\n    fi\n}\n\nstopAll() {\n    stopController\n}\n\nstartAll() {\n    startController ./conf/controller/cluster-3n-independent/controller-n0.conf\n    startController ./conf/controller/cluster-3n-independent/controller-n1.conf\n    startController ./conf/controller/cluster-3n-independent/controller-n2.conf\n}\n\ncheckConf() {\n    if [ ! -f ./conf/controller/cluster-3n-independent/controller-n0.conf -o ! -f ./conf/controller/cluster-3n-independent/controller-n1.conf -o ! -f ./conf/controller/cluster-3n-independent/controller-n2.conf ]; then\n        echo \"Make sure the ./conf/controller/cluster-3n-independent/controller-n0.conf, ./conf/controller/cluster-3n-independent/controller-n1.conf, ./conf/controller/cluster-3n-independent/controller-n2.conf exists\"\n        exit 1\n    fi\n}\n\n\n\n## Main\nif [ $# -lt 1 ]; then\n    echo \"Usage: sh $0 start|stop\"\n    exit 1\nfi\naction=$1\ncheckConf\ncase $action in\n    \"start\")\n        startAll\n        exit\n        ;;\n    \"stop\")\n        stopAll\n        exit\n        ;;\n    *)\n        echo \"Usage: sh $0 start|stop\"\n        exit\n        ;;\nesac\n\n"
  },
  {
    "path": "distribution/bin/controller/fast-try-namesrv-plugin.cmd",
    "content": "@echo off\nrem Licensed to the Apache Software Foundation (ASF) under one or more\nrem contributor license agreements.  See the NOTICE file distributed with\nrem this work for additional information regarding copyright ownership.\nrem The ASF licenses this file to You under the Apache License, Version 2.0\nrem (the \"License\"); you may not use this file except in compliance with\nrem the License.  You may obtain a copy of the License at\nrem\nrem     http://www.apache.org/licenses/LICENSE-2.0\nrem\nrem Unless required by applicable law or agreed to in writing, software\nrem distributed under the License is distributed on an \"AS IS\" BASIS,\nrem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nrem See the License for the specific language governing permissions and\nrem limitations under the License.\n\nset ROCKETMQ_HOME=%cd%\nif not exist \"%ROCKETMQ_HOME%\\conf\\controller\\cluster-3n-namesrv-plugin\\namesrv-n0.conf\" echo Make sure the %ROCKETMQ_HOME%\\conf\\controller\\cluster-3n-independent\\controller-n0.conf exists & EXIT /B 1\nif not exist \"%ROCKETMQ_HOME%\\conf\\controller\\cluster-3n-namesrv-plugin\\namesrv-n1.conf\" echo Make sure the %ROCKETMQ_HOME%\\conf\\controller\\cluster-3n-independent\\controller-n1.conf exists & EXIT /B 1\nif not exist \"%ROCKETMQ_HOME%\\conf\\controller\\cluster-3n-namesrv-plugin\\namesrv-n2.conf\" echo Make sure the %ROCKETMQ_HOME%\\conf\\controller\\cluster-3n-independent\\controller-n2.conf exists & EXIT /B 1\n\nset \"JAVA_OPT_EXT= -server -Xms512m -Xmx512m\"\nstart call \"%ROCKETMQ_HOME%\\bin\\runserver.cmd\" org.apache.rocketmq.namesrv.NamesrvStartup -c %ROCKETMQ_HOME%\\conf\\controller\\cluster-3n-namesrv-plugin\\namesrv-n0.conf\nIF %ERRORLEVEL% EQU 0 (\n   ECHO \"Controller start OK\"\n)\ntimeout /T 3 /NOBREAK\nstart call \"%ROCKETMQ_HOME%\\bin\\runserver.cmd\" org.apache.rocketmq.namesrv.NamesrvStartup -c %ROCKETMQ_HOME%\\conf\\controller\\cluster-3n-namesrv-plugin\\namesrv-n1.conf\nIF %ERRORLEVEL% EQU 0 (\n   ECHO \"Controller start OK\"\n)\ntimeout /T 3 /NOBREAK\nstart call \"%ROCKETMQ_HOME%\\bin\\runserver.cmd\" org.apache.rocketmq.namesrv.NamesrvStartup -c %ROCKETMQ_HOME%\\conf\\controller\\cluster-3n-namesrv-plugin\\namesrv-n2.conf\nIF %ERRORLEVEL% EQU 0 (\n   ECHO \"Controller start OK\"\n)"
  },
  {
    "path": "distribution/bin/controller/fast-try-namesrv-plugin.sh",
    "content": "#!/usr/bin/env bash\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n## Revise the base dir\nCURRENT_DIR=\"$(cd \"$(dirname \"$0\")\"; pwd)\"\nRMQ_DIR=$CURRENT_DIR/../..\ncd $RMQ_DIR\n\nstartNameserver() {\n    export JAVA_OPT_EXT=\" -Xms512m -Xmx512m  \"\n    conf_name=$1\n    nohup bin/mqnamesrv -c $conf_name &\n}\n\nstopNameserver() {\n    PIDS=$(ps -ef|grep java|grep NamesrvStartup|grep -v grep|awk '{print $2}')\n    if [ ! -z \"$PIDS\" ]; then\n        kill -s TERM $PIDS\n    fi\n}\n\nstopAll() {\n    stopNameserver\n}\n\nstartAll() {\n    startNameserver ./conf/controller/cluster-3n-namesrv-plugin/namesrv-n0.conf\n    startNameserver ./conf/controller/cluster-3n-namesrv-plugin/namesrv-n1.conf\n    startNameserver ./conf/controller/cluster-3n-namesrv-plugin/namesrv-n2.conf\n}\n\ncheckConf() {\n    if [ ! -f ./conf/controller/cluster-3n-namesrv-plugin/namesrv-n0.conf -o ! -f ./conf/controller/cluster-3n-namesrv-plugin/namesrv-n1.conf -o ! -f ./conf/controller/cluster-3n-namesrv-plugin/namesrv-n2.conf ]; then\n        echo \"Make sure the ./conf/controller/cluster-3n-namesrv-plugin/namesrv-n0.conf, ./conf/controller/cluster-3n-namesrv-plugin/namesrv-n1.conf, ./conf/controller/cluster-3n-namesrv-plugin/namesrv-n2.conf exists\"\n        exit 1\n    fi\n}\n\n\n\n## Main\nif [ $# -lt 1 ]; then\n    echo \"Usage: sh $0 start|stop\"\n    exit 1\nfi\naction=$1\ncheckConf\ncase $action in\n    \"start\")\n        startAll\n        exit\n        ;;\n    \"stop\")\n        stopAll\n        exit\n        ;;\n    *)\n        echo \"Usage: sh $0 start|stop\"\n        exit\n        ;;\nesac\n\n"
  },
  {
    "path": "distribution/bin/controller/fast-try.cmd",
    "content": "@echo off\nrem Licensed to the Apache Software Foundation (ASF) under one or more\nrem contributor license agreements.  See the NOTICE file distributed with\nrem this work for additional information regarding copyright ownership.\nrem The ASF licenses this file to You under the Apache License, Version 2.0\nrem (the \"License\"); you may not use this file except in compliance with\nrem the License.  You may obtain a copy of the License at\nrem\nrem     http://www.apache.org/licenses/LICENSE-2.0\nrem\nrem Unless required by applicable law or agreed to in writing, software\nrem distributed under the License is distributed on an \"AS IS\" BASIS,\nrem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nrem See the License for the specific language governing permissions and\nrem limitations under the License.\n\nset ROCKETMQ_HOME=%cd%\nif not exist \"%ROCKETMQ_HOME%\\conf\\controller\\quick-start\\namesrv.conf\" echo Make sure the %ROCKETMQ_HOME%\\conf\\controller\\quick-start\\namesrv.conf exists & EXIT /B 1\nif not exist \"%ROCKETMQ_HOME%\\conf\\controller\\quick-start\\broker-n0.conf\" echo Make sure the %ROCKETMQ_HOME%\\conf\\controller\\quick-start\\broker-n0.conf exists & EXIT /B 1\nif not exist \"%ROCKETMQ_HOME%\\conf\\controller\\quick-start\\broker-n1.conf\" echo Make sure the %ROCKETMQ_HOME%\\conf\\controller\\quick-start\\broker-n1.conf exists & EXIT /B 1\n\nset \"JAVA_OPT_EXT= -server -Xms512m -Xmx512m\"\nstart call \"%ROCKETMQ_HOME%\\bin\\runserver.cmd\" org.apache.rocketmq.namesrv.NamesrvStartup -c %ROCKETMQ_HOME%\\conf\\controller\\quick-start\\namesrv.conf\nIF %ERRORLEVEL% EQU 0 (\n   ECHO \"Namesrv start OK\"\n)\ntimeout /T 3 /NOBREAK\n\nset \"JAVA_OPT_EXT= -server -Xms1g -Xmx1g\"\nstart call \"%ROCKETMQ_HOME%\\bin\\runbroker.cmd\" org.apache.rocketmq.broker.BrokerStartup -c %ROCKETMQ_HOME%\\conf\\controller\\quick-start\\broker-n0.conf\ntimeout /T 1 /NOBREAK\nstart call \"%ROCKETMQ_HOME%\\bin\\runbroker.cmd\" org.apache.rocketmq.broker.BrokerStartup -c %ROCKETMQ_HOME%\\conf\\controller\\quick-start\\broker-n1.conf\ntimeout /T 1 /NOBREAK\n\nIF %ERRORLEVEL% EQU 0 (\n   ECHO \"Broker starts OK\"\n)"
  },
  {
    "path": "distribution/bin/controller/fast-try.sh",
    "content": "#!/usr/bin/env bash\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n## Revise the base dir\nCURRENT_DIR=\"$(cd \"$(dirname \"$0\")\"; pwd)\"\nRMQ_DIR=$CURRENT_DIR/../..\ncd $RMQ_DIR\n\nstartNameserver() {\n    export JAVA_OPT_EXT=\" -Xms512m -Xmx512m  \"\n    conf_name=$1\n    nohup bin/mqnamesrv -c $conf_name &\n}\n\nstartBroker() {\n    export JAVA_OPT_EXT=\" -Xms1g -Xmx1g  \"\n    conf_name=$1\n    nohup bin/mqbroker -c $conf_name &\n}\n\nstopNameserver() {\n    PIDS=$(ps -ef|grep java|grep NamesrvStartup|grep -v grep|awk '{print $2}')\n    if [ ! -z \"$PIDS\" ]; then\n        kill -s TERM $PIDS\n    fi\n}\n\nstopBroker() {\n    conf_name=$1\n    PIDS=$(ps -ef|grep java|grep BrokerStartup|grep $conf_name|grep -v grep|awk '{print $2}')\n    i=1\n    while [ ! -z \"$PIDS\" -a $i -lt 5 ]\n    do\n        echo \"Waiting to kill ...\"\n        kill -s TERM $PIDS\n        i=`expr $i + 1`\n        sleep 2\n        PIDS=$(ps -ef|grep java|grep BrokerStartup|grep $conf_name|grep -v grep|awk '{print $2}')\n    done\n    PIDS=$(ps -ef|grep java|grep BrokerStartup|grep $conf_name|grep -v grep|awk '{print $2}')\n    if [ ! -z \"$PIDS\" ]; then\n        kill -9 $PIDS\n    fi\n}\n\nstopAll() {\n    ps -ef|grep java|grep BrokerStartup|grep -v grep|awk '{print $2}'|xargs kill\n    stopNameserver\n    stopBroker ./conf/controller/quick-start/broker-n0.conf\n    stopBroker ./conf/controller/quick-start/broker-n1.conf\n}\n\nstartAll() {\n    startNameserver ./conf/controller/quick-start/namesrv.conf\n    startBroker ./conf/controller/quick-start/broker-n0.conf\n    startBroker ./conf/controller/quick-start/broker-n1.conf\n}\n\ncheckConf() {\n    if [ ! -f ./conf/controller/quick-start/broker-n0.conf -o ! -f ./conf/controller/quick-start/broker-n1.conf -o ! -f ./conf/controller/quick-start/namesrv.conf ]; then\n        echo \"Make sure the ./conf/controller/quick-start/broker-n0.conf, ./conf/controller/quick-start/broker-n1.conf, ./conf/controller/quick-start/namesrv.conf exists\"\n        exit 1\n    fi\n}\n\n\n\n## Main\nif [ $# -lt 1 ]; then\n    echo \"Usage: sh $0 start|stop\"\n    exit 1\nfi\naction=$1\ncheckConf\ncase $action in\n    \"start\")\n        startAll\n        exit\n        ;;\n    \"stop\")\n        stopAll\n        ;;\n    *)\n        echo \"Usage: sh $0 start|stop\"\n        ;;\nesac\n\n"
  },
  {
    "path": "distribution/bin/dledger/fast-try.sh",
    "content": "#!/usr/bin/env bash\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n## Revise the base dir\nCURRENT_DIR=\"$(cd \"$(dirname \"$0\")\"; pwd)\"\nRMQ_DIR=$CURRENT_DIR/../..\ncd $RMQ_DIR\n\nstartNameserver() {\n    export JAVA_OPT_EXT=\" -Xms512m -Xmx512m  \"\n    nohup bin/mqnamesrv &\n}\n\nstartBroker() {\n    export JAVA_OPT_EXT=\" -Xms1g -Xmx1g  \"\n    conf_name=$1\n    nohup bin/mqbroker -c $conf_name &\n}\n\nstopNameserver() {\n    PIDS=$(ps -ef|grep java|grep NamesrvStartup|grep -v grep|awk '{print $2}')\n    if [ ! -z \"$PIDS\" ]; then\n        kill -s TERM $PIDS\n    fi\n}\n\nstopBroker() {\n    conf_name=$1\n    PIDS=$(ps -ef|grep java|grep BrokerStartup|grep $conf_name|grep -v grep|awk '{print $2}')\n    i=1\n    while [ ! -z \"$PIDS\" -a $i -lt 5 ]\n    do\n        echo \"Waiting to kill ...\"\n        kill -s TERM $PIDS\n        i=`expr $i + 1`\n        sleep 2\n        PIDS=$(ps -ef|grep java|grep BrokerStartup|grep $conf_name|grep -v grep|awk '{print $2}')\n    done\n    PIDS=$(ps -ef|grep java|grep BrokerStartup|grep $conf_name|grep -v grep|awk '{print $2}')\n    if [ ! -z \"$PIDS\" ]; then\n        kill -9 $PIDS\n    fi\n}\n\nstopAll() {\n    ps -ef|grep java|grep BrokerStartup|grep -v grep|awk '{print $2}'|xargs kill\n    stopNameserver\n    stopBroker ./conf/dledger/broker-n0.conf\n    stopBroker ./conf/dledger/broker-n1.conf\n    stopBroker ./conf/dledger/broker-n2.conf\n}\n\nstartAll() {\n    startNameserver\n    startBroker ./conf/dledger/broker-n0.conf\n    startBroker ./conf/dledger/broker-n1.conf\n    startBroker ./conf/dledger/broker-n2.conf\n}\n\ncheckConf() {\n    if [ ! -f ./conf/dledger/broker-n0.conf -o ! -f ./conf/dledger/broker-n1.conf -o ! -f ./conf/dledger/broker-n2.conf ]; then\n        echo \"Make sure the ./conf/dledger/broker-n0.conf, ./conf/dledger/broker-n1.conf, ./conf/dledger/broker-n2.conf exists\"\n        exit 1\n    fi\n}\n\n\n\n## Main\nif [ $# -lt 1 ]; then\n    echo \"Usage: sh $0 start|stop\"\n    exit 1\nfi\naction=$1\ncheckConf\ncase $action in\n    \"start\")\n        startAll\n        exit\n        ;;\n    \"stop\")\n        stopAll\n        ;;\n    *)\n        echo \"Usage: sh $0 start|stop\"\n        ;;\nesac\n\n"
  },
  {
    "path": "distribution/bin/export.sh",
    "content": "#!/bin/bash\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nif [ -z \"$ROCKETMQ_HOME\" ]; then\n  ## resolve links - $0 may be a link to maven's home\n  PRG=\"$0\"\n\n  # need this for relative symlinks\n  while [ -h \"$PRG\" ]; do\n    ls=$(ls -ld \"$PRG\")\n    link=$(expr \"$ls\" : '.*-> \\(.*\\)$')\n    if expr \"$link\" : '/.*' >/dev/null; then\n      PRG=\"$link\"\n    else\n      PRG=\"$(dirname \"$PRG\")/$link\"\n    fi\n  done\n\n  saveddir=$(pwd)\n\n  ROCKETMQ_HOME=$(dirname \"$PRG\")/..\n\n  # make it fully qualified\n  ROCKETMQ_HOME=$(cd \"$ROCKETMQ_HOME\" && pwd)\n\n  cd \"$saveddir\"\nfi\n\nexport ROCKETMQ_HOME\n\nnamesrvAddr=\nwhile [ -z \"${namesrvAddr}\" ]; do\n  read -p \"Enter name server address list:\" namesrvAddr\ndone\n\nclusterName=\nwhile [ -z \"${clusterName}\" ]; do\n  read -p \"Choose a cluster to export:\" clusterName\ndone\n\nread -p \"Enter file path to export [default /tmp/rocketmq/export]:\" filePath\nif [ -z \"${filePath}\" ]; then\n  filePath=\"/tmp/rocketmq/config\"\nfi\n\nif [[ -e ${filePath} ]]; then\n  rm -rf ${filePath}\nfi\n\nsh ${ROCKETMQ_HOME}/bin/mqadmin exportMetrics -c ${clusterName} -n ${namesrvAddr} -f ${filePath}\nsh ${ROCKETMQ_HOME}/bin/mqadmin exportConfigs -c ${clusterName} -n ${namesrvAddr} -f ${filePath}\nsh ${ROCKETMQ_HOME}/bin/mqadmin exportMetadata -c ${clusterName} -n ${namesrvAddr} -f ${filePath}\n\ncd ${filePath} || exit\n\nconfigs=$(cat ./configs.json)\nif [ -z \"$configs\" ]; then\n  configs=\"{}\"\nfi\nmetadata=$(cat ./metadata.json)\nif [ -z \"$metadata\" ]; then\n  metadata=\"{}\"\nfi\nmetrics=$(cat ./metrics.json)\nif [ -z \"$metrics\" ]; then\n  metrics=\"{}\"\nfi\n\necho \"{\n    \\\"configs\\\": ${configs},\n    \\\"metadata\\\": ${metadata},\n    \\\"metrics\\\": ${metrics}\n  }\" >rocketmq-metadata-export.json\n\necho -e \"[INFO] The RocketMQ metadata has been exported to the file:${filePath}/rocketmq-metadata-export.json\"\n"
  },
  {
    "path": "distribution/bin/mqadmin",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nif [ -z \"$ROCKETMQ_HOME\" ] ; then\n  ## resolve links - $0 may be a link to maven's home\n  PRG=\"$0\"\n\n  # need this for relative symlinks\n  while [ -h \"$PRG\" ] ; do\n    ls=`ls -ld \"$PRG\"`\n    link=`expr \"$ls\" : '.*-> \\(.*\\)$'`\n    if expr \"$link\" : '/.*' > /dev/null; then\n      PRG=\"$link\"\n    else\n      PRG=\"`dirname \"$PRG\"`/$link\"\n    fi\n  done\n\n  saveddir=`pwd`\n\n  ROCKETMQ_HOME=`dirname \"$PRG\"`/..\n\n  # make it fully qualified\n  ROCKETMQ_HOME=`cd \"$ROCKETMQ_HOME\" && pwd`\n\n  cd \"$saveddir\"\nfi\n\nexport ROCKETMQ_HOME\n\nsh ${ROCKETMQ_HOME}/bin/tools.sh -Drmq.logback.configurationFile=$ROCKETMQ_HOME/conf/rmq.tools.logback.xml org.apache.rocketmq.tools.command.MQAdminStartup \"$@\"\n"
  },
  {
    "path": "distribution/bin/mqadmin.cmd",
    "content": "@echo off\nrem Licensed to the Apache Software Foundation (ASF) under one or more\nrem contributor license agreements.  See the NOTICE file distributed with\nrem this work for additional information regarding copyright ownership.\nrem The ASF licenses this file to You under the Apache License, Version 2.0\nrem (the \"License\"); you may not use this file except in compliance with\nrem the License.  You may obtain a copy of the License at\nrem\nrem     http://www.apache.org/licenses/LICENSE-2.0\nrem\nrem Unless required by applicable law or agreed to in writing, software\nrem distributed under the License is distributed on an \"AS IS\" BASIS,\nrem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nrem See the License for the specific language governing permissions and\nrem limitations under the License.\n\nif not exist \"%ROCKETMQ_HOME%\\bin\\tools.cmd\" echo Please set the ROCKETMQ_HOME variable in your environment! & EXIT /B 1\ncall \"%ROCKETMQ_HOME%\\bin\\tools.cmd\" -Drmq.logback.configurationFile=%ROCKETMQ_HOME%\\conf\\rmq.tools.logback.xml org.apache.rocketmq.tools.command.MQAdminStartup %*"
  },
  {
    "path": "distribution/bin/mqbroker",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nif [ -z \"$ROCKETMQ_HOME\" ] ; then\n  ## resolve links - $0 may be a link to maven's home\n  PRG=\"$0\"\n\n  # need this for relative symlinks\n  while [ -h \"$PRG\" ] ; do\n    ls=`ls -ld \"$PRG\"`\n    link=`expr \"$ls\" : '.*-> \\(.*\\)$'`\n    if expr \"$link\" : '/.*' > /dev/null; then\n      PRG=\"$link\"\n    else\n      PRG=\"`dirname \"$PRG\"`/$link\"\n    fi\n  done\n\n  saveddir=`pwd`\n\n  ROCKETMQ_HOME=`dirname \"$PRG\"`/..\n\n  # make it fully qualified\n  ROCKETMQ_HOME=`cd \"$ROCKETMQ_HOME\" && pwd`\n\n  cd \"$saveddir\"\nfi\n\nexport ROCKETMQ_HOME\n\nother_args=\" \"\nenable_proxy=false\n\nwhile [ $# -gt 0 ]; do\n  case $1 in\n    --enable-proxy)\n      enable_proxy=true\n      shift\n      ;;\n    -c|--configFile)\n      broker_config=\"$2\"\n      shift\n      shift\n      ;;\n    *)\n      other_args=${other_args}\" \"${1}\n      shift\n      ;;\n  esac\ndone\n\nif [ \"$enable_proxy\" = true ]; then\n  args_for_proxy=$other_args\" -pm local\"\n  if [ \"$broker_config\" != \"\" ]; then\n      args_for_proxy=${args_for_proxy}\" -bc \"${broker_config}\n  fi\n  sh ${ROCKETMQ_HOME}/bin/runbroker.sh -Drmq.logback.configurationFile=$ROCKETMQ_HOME/conf/rmq.proxy.logback.xml org.apache.rocketmq.proxy.ProxyStartup ${args_for_proxy}\nelse\n  args_for_broker=$other_args\n  if [ \"$broker_config\" != \"\" ]; then\n      args_for_broker=${args_for_broker}\" -c \"${broker_config}\n  fi\n  sh ${ROCKETMQ_HOME}/bin/runbroker.sh -Drmq.logback.configurationFile=$ROCKETMQ_HOME/conf/rmq.broker.logback.xml org.apache.rocketmq.broker.BrokerStartup ${args_for_broker}\nfi\n"
  },
  {
    "path": "distribution/bin/mqbroker.cmd",
    "content": "@echo off\nrem Licensed to the Apache Software Foundation (ASF) under one or more\nrem contributor license agreements.  See the NOTICE file distributed with\nrem this work for additional information regarding copyright ownership.\nrem The ASF licenses this file to You under the Apache License, Version 2.0\nrem (the \"License\"); you may not use this file except in compliance with\nrem the License.  You may obtain a copy of the License at\nrem\nrem     http://www.apache.org/licenses/LICENSE-2.0\nrem\nrem Unless required by applicable law or agreed to in writing, software\nrem distributed under the License is distributed on an \"AS IS\" BASIS,\nrem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nrem See the License for the specific language governing permissions and\nrem limitations under the License.\n\nif not exist \"%ROCKETMQ_HOME%\\bin\\runbroker.cmd\" echo Please set the ROCKETMQ_HOME variable in your environment! & EXIT /B 1\n\ncall \"%ROCKETMQ_HOME%\\bin\\runbroker.cmd\" -Drmq.logback.configurationFile=%ROCKETMQ_HOME%\\conf\\rmq.broker.logback.xml org.apache.rocketmq.broker.BrokerStartup %*\n\nIF %ERRORLEVEL% EQU 0 (\n   ECHO \"Broker starts OK\"\n)"
  },
  {
    "path": "distribution/bin/mqbroker.numanode0",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nif [ -z \"$ROCKETMQ_HOME\" ] ; then\n  ## resolve links - $0 may be a link to maven's home\n  PRG=\"$0\"\n\n  # need this for relative symlinks\n  while [ -h \"$PRG\" ] ; do\n    ls=`ls -ld \"$PRG\"`\n    link=`expr \"$ls\" : '.*-> \\(.*\\)$'`\n    if expr \"$link\" : '/.*' > /dev/null; then\n      PRG=\"$link\"\n    else\n      PRG=\"`dirname \"$PRG\"`/$link\"\n    fi\n  done\n\n  saveddir=`pwd`\n\n  ROCKETMQ_HOME=`dirname \"$PRG\"`/..\n\n  # make it fully qualified\n  ROCKETMQ_HOME=`cd \"$ROCKETMQ_HOME\" && pwd`\n\n  cd \"$saveddir\"\nfi\n\nexport ROCKETMQ_HOME\n\nexport RMQ_NUMA_NODE=0\n\nsh ${ROCKETMQ_HOME}/bin/mqbroker $@\n"
  },
  {
    "path": "distribution/bin/mqbroker.numanode1",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nif [ -z \"$ROCKETMQ_HOME\" ] ; then\n  ## resolve links - $0 may be a link to maven's home\n  PRG=\"$0\"\n\n  # need this for relative symlinks\n  while [ -h \"$PRG\" ] ; do\n    ls=`ls -ld \"$PRG\"`\n    link=`expr \"$ls\" : '.*-> \\(.*\\)$'`\n    if expr \"$link\" : '/.*' > /dev/null; then\n      PRG=\"$link\"\n    else\n      PRG=\"`dirname \"$PRG\"`/$link\"\n    fi\n  done\n\n  saveddir=`pwd`\n\n  ROCKETMQ_HOME=`dirname \"$PRG\"`/..\n\n  # make it fully qualified\n  ROCKETMQ_HOME=`cd \"$ROCKETMQ_HOME\" && pwd`\n\n  cd \"$saveddir\"\nfi\n\nexport ROCKETMQ_HOME\n\nexport RMQ_NUMA_NODE=1\n\nsh ${ROCKETMQ_HOME}/bin/mqbroker $@\n"
  },
  {
    "path": "distribution/bin/mqbroker.numanode2",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nif [ -z \"$ROCKETMQ_HOME\" ] ; then\n  ## resolve links - $0 may be a link to maven's home\n  PRG=\"$0\"\n\n  # need this for relative symlinks\n  while [ -h \"$PRG\" ] ; do\n    ls=`ls -ld \"$PRG\"`\n    link=`expr \"$ls\" : '.*-> \\(.*\\)$'`\n    if expr \"$link\" : '/.*' > /dev/null; then\n      PRG=\"$link\"\n    else\n      PRG=\"`dirname \"$PRG\"`/$link\"\n    fi\n  done\n\n  saveddir=`pwd`\n\n  ROCKETMQ_HOME=`dirname \"$PRG\"`/..\n\n  # make it fully qualified\n  ROCKETMQ_HOME=`cd \"$ROCKETMQ_HOME\" && pwd`\n\n  cd \"$saveddir\"\nfi\n\nexport ROCKETMQ_HOME\n\nexport RMQ_NUMA_NODE=2\n\nsh ${ROCKETMQ_HOME}/bin/mqbroker $@\n"
  },
  {
    "path": "distribution/bin/mqbroker.numanode3",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nif [ -z \"$ROCKETMQ_HOME\" ] ; then\n  ## resolve links - $0 may be a link to maven's home\n  PRG=\"$0\"\n\n  # need this for relative symlinks\n  while [ -h \"$PRG\" ] ; do\n    ls=`ls -ld \"$PRG\"`\n    link=`expr \"$ls\" : '.*-> \\(.*\\)$'`\n    if expr \"$link\" : '/.*' > /dev/null; then\n      PRG=\"$link\"\n    else\n      PRG=\"`dirname \"$PRG\"`/$link\"\n    fi\n  done\n\n  saveddir=`pwd`\n\n  ROCKETMQ_HOME=`dirname \"$PRG\"`/..\n\n  # make it fully qualified\n  ROCKETMQ_HOME=`cd \"$ROCKETMQ_HOME\" && pwd`\n\n  cd \"$saveddir\"\nfi\n\nexport ROCKETMQ_HOME\n\nexport RMQ_NUMA_NODE=3\n\nsh ${ROCKETMQ_HOME}/bin/mqbroker $@\n"
  },
  {
    "path": "distribution/bin/mqbrokercontainer",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nif [ -z \"$ROCKETMQ_HOME\" ] ; then\n  ## resolve links - $0 may be a link to maven's home\n  PRG=\"$0\"\n\n  # need this for relative symlinks\n  while [ -h \"$PRG\" ] ; do\n    ls=`ls -ld \"$PRG\"`\n    link=`expr \"$ls\" : '.*-> \\(.*\\)$'`\n    if expr \"$link\" : '/.*' > /dev/null; then\n      PRG=\"$link\"\n    else\n      PRG=\"`dirname \"$PRG\"`/$link\"\n    fi\n  done\n\n  saveddir=`pwd`\n\n  ROCKETMQ_HOME=`dirname \"$PRG\"`/..\n\n  # make it fully qualified\n  ROCKETMQ_HOME=`cd \"$ROCKETMQ_HOME\" && pwd`\n\n  cd \"$saveddir\"\nfi\n\nexport ROCKETMQ_HOME\n\nsh ${ROCKETMQ_HOME}/bin/runbroker.sh org.apache.rocketmq.container.BrokerContainerStartup $@\n"
  },
  {
    "path": "distribution/bin/mqcontroller",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nif [ -z \"$ROCKETMQ_HOME\" ] ; then\n  ## resolve links - $0 may be a link to maven's home\n  PRG=\"$0\"\n\n  # need this for relative symlinks\n  while [ -h \"$PRG\" ] ; do\n    ls=`ls -ld \"$PRG\"`\n    link=`expr \"$ls\" : '.*-> \\(.*\\)$'`\n    if expr \"$link\" : '/.*' > /dev/null; then\n      PRG=\"$link\"\n    else\n      PRG=\"`dirname \"$PRG\"`/$link\"\n    fi\n  done\n\n  saveddir=`pwd`\n\n  ROCKETMQ_HOME=`dirname \"$PRG\"`/..\n\n  # make it fully qualified\n  ROCKETMQ_HOME=`cd \"$ROCKETMQ_HOME\" && pwd`\n\n  cd \"$saveddir\"\nfi\n\nexport ROCKETMQ_HOME\n\nsh ${ROCKETMQ_HOME}/bin/runserver.sh -Drmq.logback.configurationFile=$ROCKETMQ_HOME/conf/rmq.controller.logback.xml org.apache.rocketmq.controller.ControllerStartup $@\n"
  },
  {
    "path": "distribution/bin/mqcontroller.cmd",
    "content": "@echo off\nrem Licensed to the Apache Software Foundation (ASF) under one or more\nrem contributor license agreements.  See the NOTICE file distributed with\nrem this work for additional information regarding copyright ownership.\nrem The ASF licenses this file to You under the Apache License, Version 2.0\nrem (the \"License\"); you may not use this file except in compliance with\nrem the License.  You may obtain a copy of the License at\nrem\nrem     http://www.apache.org/licenses/LICENSE-2.0\nrem\nrem Unless required by applicable law or agreed to in writing, software\nrem distributed under the License is distributed on an \"AS IS\" BASIS,\nrem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nrem See the License for the specific language governing permissions and\nrem limitations under the License.\n\nif not exist \"%ROCKETMQ_HOME%\\bin\\runserver.cmd\" echo Please set the ROCKETMQ_HOME variable in your environment! & EXIT /B 1\n\ncall \"%ROCKETMQ_HOME%\\bin\\runserver.cmd\" -Drmq.logback.configurationFile=%ROCKETMQ_HOME%\\conf\\rmq.controller.logback.xml org.apache.rocketmq.controller.ControllerStartup %*\n\nIF %ERRORLEVEL% EQU 0 (\n    ECHO \"Controller starts OK\"\n)"
  },
  {
    "path": "distribution/bin/mqnamesrv",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nif [ -z \"$ROCKETMQ_HOME\" ] ; then\n  ## resolve links - $0 may be a link to maven's home\n  PRG=\"$0\"\n\n  # need this for relative symlinks\n  while [ -h \"$PRG\" ] ; do\n    ls=`ls -ld \"$PRG\"`\n    link=`expr \"$ls\" : '.*-> \\(.*\\)$'`\n    if expr \"$link\" : '/.*' > /dev/null; then\n      PRG=\"$link\"\n    else\n      PRG=\"`dirname \"$PRG\"`/$link\"\n    fi\n  done\n\n  saveddir=`pwd`\n\n  ROCKETMQ_HOME=`dirname \"$PRG\"`/..\n\n  # make it fully qualified\n  ROCKETMQ_HOME=`cd \"$ROCKETMQ_HOME\" && pwd`\n\n  cd \"$saveddir\"\nfi\n\nexport ROCKETMQ_HOME\n\nsh ${ROCKETMQ_HOME}/bin/runserver.sh -Drmq.logback.configurationFile=$ROCKETMQ_HOME/conf/rmq.namesrv.logback.xml org.apache.rocketmq.namesrv.NamesrvStartup $@\n"
  },
  {
    "path": "distribution/bin/mqnamesrv.cmd",
    "content": "@echo off\nrem Licensed to the Apache Software Foundation (ASF) under one or more\nrem contributor license agreements.  See the NOTICE file distributed with\nrem this work for additional information regarding copyright ownership.\nrem The ASF licenses this file to You under the Apache License, Version 2.0\nrem (the \"License\"); you may not use this file except in compliance with\nrem the License.  You may obtain a copy of the License at\nrem\nrem     http://www.apache.org/licenses/LICENSE-2.0\nrem\nrem Unless required by applicable law or agreed to in writing, software\nrem distributed under the License is distributed on an \"AS IS\" BASIS,\nrem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nrem See the License for the specific language governing permissions and\nrem limitations under the License.\n\nif not exist \"%ROCKETMQ_HOME%\\bin\\runserver.cmd\" echo Please set the ROCKETMQ_HOME variable in your environment! & EXIT /B 1\n\ncall \"%ROCKETMQ_HOME%\\bin\\runserver.cmd\" -Drmq.logback.configurationFile=%ROCKETMQ_HOME%\\conf\\rmq.namesrv.logback.xml org.apache.rocketmq.namesrv.NamesrvStartup %*\n\nIF %ERRORLEVEL% EQU 0 (\n    ECHO \"Namesrv starts OK\"\n)"
  },
  {
    "path": "distribution/bin/mqproxy",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nif [ -z \"$ROCKETMQ_HOME\" ] ; then\n  ## resolve links - $0 may be a link to maven's home\n  PRG=\"$0\"\n\n  # need this for relative symlinks\n  while [ -h \"$PRG\" ] ; do\n    ls=`ls -ld \"$PRG\"`\n    link=`expr \"$ls\" : '.*-> \\(.*\\)$'`\n    if expr \"$link\" : '/.*' > /dev/null; then\n      PRG=\"$link\"\n    else\n      PRG=\"`dirname \"$PRG\"`/$link\"\n    fi\n  done\n\n  saveddir=`pwd`\n\n  ROCKETMQ_HOME=`dirname \"$PRG\"`/..\n\n  # make it fully qualified\n  ROCKETMQ_HOME=`cd \"$ROCKETMQ_HOME\" && pwd`\n\n  cd \"$saveddir\"\nfi\n\nexport ROCKETMQ_HOME\n\nsh ${ROCKETMQ_HOME}/bin/runserver.sh -Drmq.logback.configurationFile=$ROCKETMQ_HOME/conf/rmq.proxy.logback.xml org.apache.rocketmq.proxy.ProxyStartup $@\n"
  },
  {
    "path": "distribution/bin/mqproxy.cmd",
    "content": "@echo off\nrem Licensed to the Apache Software Foundation (ASF) under one or more\nrem contributor license agreements.  See the NOTICE file distributed with\nrem this work for additional information regarding copyright ownership.\nrem The ASF licenses this file to You under the Apache License, Version 2.0\nrem (the \"License\"); you may not use this file except in compliance with\nrem the License.  You may obtain a copy of the License at\nrem\nrem     http://www.apache.org/licenses/LICENSE-2.0\nrem\nrem Unless required by applicable law or agreed to in writing, software\nrem distributed under the License is distributed on an \"AS IS\" BASIS,\nrem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nrem See the License for the specific language governing permissions and\nrem limitations under the License.\n\nif not exist \"%ROCKETMQ_HOME%\\bin\\runserver.cmd\" echo Please set the ROCKETMQ_HOME variable in your environment! & EXIT /B 1\n\ncall \"%ROCKETMQ_HOME%\\bin\\runserver.cmd\" -Drmq.logback.configurationFile=%ROCKETMQ_HOME%\\conf\\rmq.proxy.logback.xml org.apache.rocketmq.proxy.ProxyStartup %*\n\nIF %ERRORLEVEL% EQU 0 (\n    ECHO \"Proxy starts OK\"\n)"
  },
  {
    "path": "distribution/bin/mqshutdown",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\ncase $1 in\n    broker)\n    pid=`ps ax | grep -i 'org.apache.rocketmq.proxy.ProxyStartup' | grep '\\-pm local' |grep java | grep -v grep | awk '{print $1}'`\n    if [ \"$pid\" != \"\" ] ; then\n            echo \"The mqbroker with proxy enable is running(${pid})...\"\n            kill ${pid}\n            echo \"Send shutdown request to mqbroker with proxy enable OK(${pid})\"\n    fi\n    pid=`ps ax | grep -i 'org.apache.rocketmq.broker.BrokerStartup' |grep java | grep -v grep | awk '{print $1}'`\n    if [ -z \"$pid\" ] ; then\n            echo \"No mqbroker running.\"\n            exit 1;\n    fi\n\n    echo \"The mqbroker(${pid}) is running...\"\n\n    kill ${pid}\n\n    echo \"Send shutdown request to mqbroker(${pid}) OK\"\n    ;;\n    brokerContainer)\n\n    pid=`ps ax | grep -i 'org.apache.rocketmq.container.BrokerContainerStartup' |grep java | grep -v grep | awk '{print $1}'`\n    if [ -z \"$pid\" ] ; then\n            echo \"No broker container running.\"\n            exit 1;\n    fi\n\n    echo \"The broker container(${pid}) is running...\"\n\n    kill ${pid}\n\n    echo \"Send shutdown request to broker container(${pid}) OK\"\n    ;;\n    namesrv)\n\n    pid=`ps ax | grep -i 'org.apache.rocketmq.namesrv.NamesrvStartup' |grep java | grep -v grep | awk '{print $1}'`\n    if [ -z \"$pid\" ] ; then\n            echo \"No mqnamesrv running.\"\n            exit 1;\n    fi\n\n    echo \"The mqnamesrv(${pid}) is running...\"\n\n    kill ${pid}\n\n    echo \"Send shutdown request to mqnamesrv(${pid}) OK\"\n    ;;\n    controller)\n\n    pid=`ps ax | grep -i 'org.apache.rocketmq.controller.ControllerStartup' |grep java | grep -v grep | awk '{print $1}'`\n    if [ -z \"$pid\" ] ; then\n            echo \"No mqcontroller running.\"\n            exit 1;\n    fi\n\n    echo \"The mqcontroller(${pid}) is running...\"\n\n    kill ${pid}\n\n    echo \"Send shutdown request to mqcontroller(${pid}) OK\"\n    ;;\n    proxy)\n\n    pid=`ps ax | grep -i 'org.apache.rocketmq.proxy.ProxyStartup' |grep java | grep -v grep | awk '{print $1}'`\n    if [ -z \"$pid\" ] ; then\n            echo \"No mqproxy running.\"\n            exit 1;\n    fi\n\n    echo \"The mqproxy(${pid}) is running...\"\n\n    kill ${pid}\n\n    echo \"Send shutdown request to mqproxy(${pid}) OK\"\n    ;;\n    *)\n    echo \"Usage: mqshutdown broker | namesrv | controller | proxy\"\nesac\n"
  },
  {
    "path": "distribution/bin/mqshutdown.cmd",
    "content": "@echo off\nrem Licensed to the Apache Software Foundation (ASF) under one or more\nrem contributor license agreements.  See the NOTICE file distributed with\nrem this work for additional information regarding copyright ownership.\nrem The ASF licenses this file to You under the Apache License, Version 2.0\nrem (the \"License\"); you may not use this file except in compliance with\nrem the License.  You may obtain a copy of the License at\nrem\nrem     http://www.apache.org/licenses/LICENSE-2.0\nrem\nrem Unless required by applicable law or agreed to in writing, software\nrem distributed under the License is distributed on an \"AS IS\" BASIS,\nrem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nrem See the License for the specific language governing permissions and\nrem limitations under the License.\n\nif not exist \"%JAVA_HOME%\\bin\\jps.exe\" echo Please set the JAVA_HOME variable in your environment, We need java(x64)! & EXIT /B 1\n\nsetlocal\n\nset \"PATH=%JAVA_HOME%\\bin;%PATH%\"\n\nif /I \"%1\" == \"broker\" (\n    echo killing broker\n    for /f \"tokens=1\" %%i in ('jps -m ^| find \"BrokerStartup\"') do ( taskkill /F /PID %%i )\n    echo Done!\n) else if /I \"%1\" == \"namesrv\" (\n    echo killing name server\n\n    for /f \"tokens=1\" %%i in ('jps -m ^| find \"NamesrvStartup\"') do ( taskkill /F /PID %%i )\n\n    echo Done!\n) else if /I \"%1\" == \"controller\" (\n    echo killing controller server\n\n    for /f \"tokens=1\" %%i in ('jps -m ^| find \"ControllerStartup\"') do ( taskkill /F /PID %%i )\n\n    echo Done!\n) else (\n    echo Unknown role to kill, please specify broker or namesrv or controller\n)"
  },
  {
    "path": "distribution/bin/os.sh",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nexport PATH=$PATH:/sbin\n\n# sudo sysctl -w vm.extra_free_kbytes=2000000\n# sudo sysctl -w vm.min_free_kbytes=1000000\nsudo sysctl -w vm.overcommit_memory=1\nsudo sysctl -w vm.drop_caches=1\nsudo sysctl -w vm.zone_reclaim_mode=0\nsudo sysctl -w vm.max_map_count=655360\nsudo sysctl -w vm.dirty_background_ratio=50\nsudo sysctl -w vm.dirty_ratio=50\nsudo sysctl -w vm.dirty_writeback_centisecs=360000\nsudo sysctl -w vm.page-cluster=3\nsudo sysctl -w vm.swappiness=1\n\necho 'ulimit -n 655350' >> /etc/profile\necho '* hard nofile 655350' >> /etc/security/limits.conf\n\necho '* hard memlock      unlimited' >> /etc/security/limits.conf\necho '* soft memlock      unlimited' >> /etc/security/limits.conf\n\nDISK=`df -k | sort -n -r -k 2 | awk -F/ 'NR==1 {gsub(/[0-9].*/,\"\",$3); print $3}'`\n[ \"$DISK\" = 'cciss' ] && DISK='cciss!c0d0'\necho 'deadline' > /sys/block/${DISK}/queue/scheduler\n\n\necho \"---------------------------------------------------------------\"\nsysctl vm.extra_free_kbytes\nsysctl vm.min_free_kbytes\nsysctl vm.overcommit_memory\nsysctl vm.drop_caches\nsysctl vm.zone_reclaim_mode\nsysctl vm.max_map_count\nsysctl vm.dirty_background_ratio\nsysctl vm.dirty_ratio\nsysctl vm.dirty_writeback_centisecs\nsysctl vm.page-cluster\nsysctl vm.swappiness\n\nsu - admin -c 'ulimit -n'\ncat /sys/block/$DISK/queue/scheduler\n\nif [ -d ${HOME}/tmpfs ] ; then\n    echo \"tmpfs exist, do nothing.\"\nelse\n    ln -s /dev/shm ${HOME}/tmpfs\n    echo \"create tmpfs ok\"\nfi\n"
  },
  {
    "path": "distribution/bin/play.cmd",
    "content": "@echo off\nrem Licensed to the Apache Software Foundation (ASF) under one or more\nrem contributor license agreements.  See the NOTICE file distributed with\nrem this work for additional information regarding copyright ownership.\nrem The ASF licenses this file to You under the Apache License, Version 2.0\nrem (the \"License\"); you may not use this file except in compliance with\nrem the License.  You may obtain a copy of the License at\nrem\nrem     http://www.apache.org/licenses/LICENSE-2.0\nrem\nrem Unless required by applicable law or agreed to in writing, software\nrem distributed under the License is distributed on an \"AS IS\" BASIS,\nrem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nrem See the License for the specific language governing permissions and\nrem limitations under the License.\n\nSTART /B mqnamesrv > ns.log 2>&1\nIF %ERRORLEVEL% NEQ 0 (\n    echo \"Failed to start name server. Please check ns.log\"\n    EXIT /B 1\n)\n\nSTART /B mqbroker -n localhost:9876 > bk.log 2>&1\n\nIF %ERRORLEVEL% NEQ 0 (\n   ECHO \"Failed to start broker. Please check bk.log\"\n   EXIT /B 1\n)\n\necho \"Start Name Server and Broker Successfully.\""
  },
  {
    "path": "distribution/bin/play.sh",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n#\n# Name Server\n#\nnohup sh mqnamesrv > ns.log 2>&1 &\n\n#\n# Service Addr\n#\nADDR=`hostname -i`:9876\n\n#\n# Broker\n#\nnohup sh mqbroker -n ${ADDR} > bk.log 2>&1 &\n\necho \"Start Name Server and Broker Successfully, ${ADDR}\"\n"
  },
  {
    "path": "distribution/bin/runbroker.cmd",
    "content": "@echo off\nrem Licensed to the Apache Software Foundation (ASF) under one or more\nrem contributor license agreements.  See the NOTICE file distributed with\nrem this work for additional information regarding copyright ownership.\nrem The ASF licenses this file to You under the Apache License, Version 2.0\nrem (the \"License\"); you may not use this file except in compliance with\nrem the License.  You may obtain a copy of the License at\nrem\nrem     http://www.apache.org/licenses/LICENSE-2.0\nrem\nrem Unless required by applicable law or agreed to in writing, software\nrem distributed under the License is distributed on an \"AS IS\" BASIS,\nrem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nrem See the License for the specific language governing permissions and\nrem limitations under the License.\n\nif not exist \"%JAVA_HOME%\\bin\\java.exe\" echo Please set the JAVA_HOME variable in your environment, We need java(x64)! & EXIT /B 1\nset \"JAVA=%JAVA_HOME%\\bin\\java.exe\"\n\nsetlocal\n\nset BASE_DIR=%~dp0\nset BASE_DIR=%BASE_DIR:~0,-1%\nfor %%d in (%BASE_DIR%) do set BASE_DIR=%%~dpd\n\nset CLASSPATH=.;%BASE_DIR%conf;%BASE_DIR%lib\\*;\"%CLASSPATH%\"\n\nrem ===========================================================================================\nrem  JVM Configuration\nrem ===========================================================================================\nfor /f \"tokens=2 delims=\" %%v in ('java -version 2^>^&1 ^| findstr /i \"version\"') do (\n    for /f \"tokens=1 delims=.\" %%m in (\"%%v\") do set \"JAVA_MAJOR_VERSION=%%m\"\n)\n\nif \"%JAVA_MAJOR_VERSION%\"==\"\" (\n    set \"JAVA_MAJOR_VERSION=0\"\n)\nif %JAVA_MAJOR_VERSION% lss 17 (\n   set \"JAVA_OPT=%JAVA_OPT% -server -Xms2g -Xmx2g\"\n   set \"JAVA_OPT=%JAVA_OPT% -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0 -XX:SurvivorRatio=8\"\n   set \"JAVA_OPT=%JAVA_OPT% -verbose:gc -Xloggc:%USERPROFILE%\\mq_gc.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintAdaptiveSizePolicy\"\n   set \"JAVA_OPT=%JAVA_OPT% -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m\"\n   set \"JAVA_OPT=%JAVA_OPT% -XX:-OmitStackTraceInFastThrow\"\n   set \"JAVA_OPT=%JAVA_OPT% -XX:+AlwaysPreTouch\"\n   set \"JAVA_OPT=%JAVA_OPT% -XX:MaxDirectMemorySize=15g\"\n   set \"JAVA_OPT=%JAVA_OPT% -XX:-UseLargePages -XX:-UseBiasedLocking\"\n   set \"JAVA_OPT=%JAVA_OPT% %JAVA_OPT_EXT% -cp %CLASSPATH%\"\n) else (\n   set \"JAVA_OPT=%JAVA_OPT% -server -Xms2g -Xmx2g\"\n   set \"JAVA_OPT=%JAVA_OPT% -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0 -XX:SurvivorRatio=8\"\n   rem set \"JAVA_OPT=%JAVA_OPT% -verbose:gc -Xloggc:%USERPROFILE%\\mq_gc.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintAdaptiveSizePolicy\"\n   rem set \"JAVA_OPT=%JAVA_OPT% -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m\"\n   set \"JAVA_OPT=%JAVA_OPT% -XX:-OmitStackTraceInFastThrow\"\n   set \"JAVA_OPT=%JAVA_OPT% -XX:+AlwaysPreTouch\"\n   set \"JAVA_OPT=%JAVA_OPT% -XX:MaxDirectMemorySize=15g\"\n   set \"JAVA_OPT=%JAVA_OPT% -XX:-UseLargePages -XX:-UseBiasedLocking -XX:+IgnoreUnrecognizedVMOptions\"\n   set \"JAVA_OPT=%JAVA_OPT% %JAVA_OPT_EXT% -cp \"%CLASSPATH%\"\"\n)\n\n\"%JAVA%\" %JAVA_OPT% %*\n"
  },
  {
    "path": "distribution/bin/runbroker.sh",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n#===========================================================================================\n# Java Environment Setting\n#===========================================================================================\nerror_exit ()\n{\n    echo \"ERROR: $1 !!\"\n    exit 1\n}\n\nfind_java_home()\n{\n    case \"`uname`\" in\n        Darwin)\n          if [ -n \"$JAVA_HOME\" ]; then\n            JAVA_HOME=$JAVA_HOME\n            return\n          fi\n            JAVA_HOME=$(/usr/libexec/java_home)\n        ;;\n        *)\n            JAVA_HOME=$(dirname $(dirname $(readlink -f $(which javac))))\n        ;;\n    esac\n}\n\nfind_java_home\n\n[ ! -e \"$JAVA_HOME/bin/java\" ] && JAVA_HOME=$HOME/jdk/java\n[ ! -e \"$JAVA_HOME/bin/java\" ] && JAVA_HOME=/usr/java\n[ ! -e \"$JAVA_HOME/bin/java\" ] && error_exit \"Please set the JAVA_HOME variable in your environment, We need java(x64)!\"\n\nexport JAVA_HOME\nexport JAVA=\"$JAVA_HOME/bin/java\"\nexport BASE_DIR=$(dirname $0)/..\nexport CLASSPATH=.:${BASE_DIR}/conf:${BASE_DIR}/lib/*:${CLASSPATH}\n\n#===========================================================================================\n# JVM Configuration\n#===========================================================================================\n# The RAMDisk initializing size in MB on Darwin OS for gc-log\nDIR_SIZE_IN_MB=600\n\nchoose_gc_log_directory()\n{\n    case \"`uname`\" in\n        Darwin)\n            if [ ! -d \"/Volumes/RAMDisk\" ]; then\n                # create ram disk on Darwin systems as gc-log directory\n                DEV=`hdiutil attach -nomount ram://$((2 * 1024 * DIR_SIZE_IN_MB))` > /dev/null\n                diskutil eraseVolume HFS+ RAMDisk ${DEV} > /dev/null\n                echo \"Create RAMDisk /Volumes/RAMDisk for gc logging on Darwin OS.\"\n            fi\n            GC_LOG_DIR=\"/Volumes/RAMDisk\"\n        ;;\n        *)\n            # check if /dev/shm exists on other systems\n            if [ -d \"/dev/shm\" ]; then\n                GC_LOG_DIR=\"/dev/shm\"\n            else\n                GC_LOG_DIR=${BASE_DIR}\n            fi\n        ;;\n    esac\n}\n\nchoose_gc_options()\n{\n    JAVA_MAJOR_VERSION=$(\"$JAVA\" -version 2>&1 | head -1 | cut -d'\"' -f2 | sed 's/^1\\.//' | cut -d'.' -f1)\n    if [ -z \"$JAVA_MAJOR_VERSION\" ] || [ \"$JAVA_MAJOR_VERSION\" -lt \"8\" ] ; then\n      JAVA_OPT=\"${JAVA_OPT} -Xmn4g -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:SurvivorRatio=8 -XX:-UseParNewGC\"\n    else\n      JAVA_OPT=\"${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0\"\n    fi\n\n    if [ -z \"$JAVA_MAJOR_VERSION\" ] || [ \"$JAVA_MAJOR_VERSION\" -lt \"9\" ] ; then\n      JAVA_OPT=\"${JAVA_OPT} -verbose:gc -Xloggc:${GC_LOG_DIR}/rmq_srv_gc_%p_%t.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintAdaptiveSizePolicy\"\n      JAVA_OPT=\"${JAVA_OPT} -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m\"\n    else\n      JAVA_OPT=\"${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0\"\n      JAVA_OPT=\"${JAVA_OPT} -Xlog:gc*:file=${GC_LOG_DIR}/rmq_srv_gc_%p_%t.log:time,tags:filecount=5,filesize=30M\"\n    fi\n}\n\nchoose_gc_log_directory\n\nJAVA_OPT=\"${JAVA_OPT} -server -Xms8g -Xmx8g\"\nchoose_gc_options\nJAVA_OPT=\"${JAVA_OPT} -XX:-OmitStackTraceInFastThrow\"\nJAVA_OPT=\"${JAVA_OPT} -XX:+AlwaysPreTouch\"\nJAVA_OPT=\"${JAVA_OPT} -XX:MaxDirectMemorySize=15g\"\nJAVA_OPT=\"${JAVA_OPT} -XX:-UseLargePages -XX:-UseBiasedLocking -XX:+IgnoreUnrecognizedVMOptions\"\n#JAVA_OPT=\"${JAVA_OPT} -Xdebug -Xrunjdwp:transport=dt_socket,address=9555,server=y,suspend=n\"\nJAVA_OPT=\"${JAVA_OPT} ${JAVA_OPT_EXT}\"\nJAVA_OPT=\"${JAVA_OPT} -cp ${CLASSPATH}\"\n\nnumactl --interleave=all pwd > /dev/null 2>&1\nif [ $? -eq 0 ]\nthen\n\tif [ -z \"$RMQ_NUMA_NODE\" ] ; then\n\t\tnumactl --interleave=all $JAVA ${JAVA_OPT} $@\n\telse\n\t\tnumactl --cpunodebind=$RMQ_NUMA_NODE --membind=$RMQ_NUMA_NODE $JAVA ${JAVA_OPT} $@\n\tfi\nelse\n\t\"$JAVA\" ${JAVA_OPT} $@\nfi\n"
  },
  {
    "path": "distribution/bin/runserver.cmd",
    "content": "@echo off\nrem Licensed to the Apache Software Foundation (ASF) under one or more\nrem contributor license agreements.  See the NOTICE file distributed with\nrem this work for additional information regarding copyright ownership.\nrem The ASF licenses this file to You under the Apache License, Version 2.0\nrem (the \"License\"); you may not use this file except in compliance with\nrem the License.  You may obtain a copy of the License at\nrem\nrem     http://www.apache.org/licenses/LICENSE-2.0\nrem\nrem Unless required by applicable law or agreed to in writing, software\nrem distributed under the License is distributed on an \"AS IS\" BASIS,\nrem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nrem See the License for the specific language governing permissions and\nrem limitations under the License.\n\n\nif not exist \"%JAVA_HOME%\\bin\\java.exe\" echo Please set the JAVA_HOME variable in your environment, We need java(x64)! & EXIT /B 1\nset \"JAVA=%JAVA_HOME%\\bin\\java.exe\"\n\nsetlocal\n\nset BASE_DIR=%~dp0\nset BASE_DIR=%BASE_DIR:~0,-1%\nfor %%d in (%BASE_DIR%) do set BASE_DIR=%%~dpd\n\nset CLASSPATH=.;%BASE_DIR%conf;%BASE_DIR%lib\\*;%CLASSPATH%\n\nREM Example of JAVA_MAJOR_VERSION value: '1', '9', '10', '11', ...\nREM '1' means releases before Java 9\n\nfor /f \"tokens=2 delims=\" %%v in ('java -version 2^>^&1 ^| findstr /i \"version\"') do (\n    for /f \"tokens=1 delims=.\" %%m in (\"%%v\") do set \"JAVA_MAJOR_VERSION=%%m\"\n)\n\nif \"%JAVA_MAJOR_VERSION%\"==\"\" (\n    set \"JAVA_MAJOR_VERSION=0\"\n)\n\nif %JAVA_MAJOR_VERSION% lss 17 (\n   set \"JAVA_OPT=%JAVA_OPT% -server -Xms2g -Xmx2g -Xmn1g -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m\"\n   set \"JAVA_OPT=%JAVA_OPT% -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:SurvivorRatio=8 -XX:-UseParNewGC\"\n   set \"JAVA_OPT=%JAVA_OPT% -verbose:gc -Xloggc:\"%USERPROFILE%\\rmq_srv_gc.log\" -XX:+PrintGCDetails -XX:+PrintGCDateStamps\"\n   set \"JAVA_OPT=%JAVA_OPT% -XX:-OmitStackTraceInFastThrow\"\n   set \"JAVA_OPT=%JAVA_OPT% -XX:-UseLargePages\"\n   set \"JAVA_OPT=%JAVA_OPT% %JAVA_OPT_EXT% -cp \"%CLASSPATH%\"\"\n) else (\n   set \"JAVA_OPT=%JAVA_OPT% -server -Xms2g -Xmx2g -Xmn1g -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m\"\n   rem set \"JAVA_OPT=%JAVA_OPT% -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:SurvivorRatio=8 -XX:-UseParNewGC\"\n   rem set \"JAVA_OPT=%JAVA_OPT% -verbose:gc -Xloggc:\"%USERPROFILE%\\rmq_srv_gc.log\" -XX:+PrintGCDetails -XX:+PrintGCDateStamps\"\n   set \"JAVA_OPT=%JAVA_OPT% -XX:-OmitStackTraceInFastThrow\"\n   set \"JAVA_OPT=%JAVA_OPT% -XX:-UseLargePages\"\n   set \"JAVA_OPT=%JAVA_OPT% %JAVA_OPT_EXT% -cp \"%CLASSPATH%\"\"\n)\n\n\"%JAVA%\" %JAVA_OPT% %*"
  },
  {
    "path": "distribution/bin/runserver.sh",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n#===========================================================================================\n# Java Environment Setting\n#===========================================================================================\nerror_exit ()\n{\n    echo \"ERROR: $1 !!\"\n    exit 1\n}\n\nfind_java_home()\n{\n    case \"`uname`\" in\n        Darwin)\n          if [ -n \"$JAVA_HOME\" ]; then\n              JAVA_HOME=$JAVA_HOME\n              return\n          fi\n            JAVA_HOME=$(/usr/libexec/java_home)\n        ;;\n        *)\n            JAVA_HOME=$(dirname $(dirname $(readlink -f $(which javac))))\n        ;;\n    esac\n}\n\nfind_java_home\n\n[ ! -e \"$JAVA_HOME/bin/java\" ] && JAVA_HOME=$HOME/jdk/java\n[ ! -e \"$JAVA_HOME/bin/java\" ] && JAVA_HOME=/usr/java\n[ ! -e \"$JAVA_HOME/bin/java\" ] && error_exit \"Please set the JAVA_HOME variable in your environment, We need java(x64)!\"\n\nexport JAVA_HOME\nexport JAVA=\"$JAVA_HOME/bin/java\"\nexport BASE_DIR=$(dirname $0)/..\nexport CLASSPATH=.:${BASE_DIR}/conf:${BASE_DIR}/lib/*:${CLASSPATH}\n\n#===========================================================================================\n# JVM Configuration\n#===========================================================================================\n# The RAMDisk initializing size in MB on Darwin OS for gc-log\nDIR_SIZE_IN_MB=600\n\nchoose_gc_log_directory()\n{\n    case \"`uname`\" in\n        Darwin)\n            if [ ! -d \"/Volumes/RAMDisk\" ]; then\n                # create ram disk on Darwin systems as gc-log directory\n                DEV=`hdiutil attach -nomount ram://$((2 * 1024 * DIR_SIZE_IN_MB))` > /dev/null\n                diskutil eraseVolume HFS+ RAMDisk ${DEV} > /dev/null\n                echo \"Create RAMDisk /Volumes/RAMDisk for gc logging on Darwin OS.\"\n            fi\n            GC_LOG_DIR=\"/Volumes/RAMDisk\"\n        ;;\n        *)\n            # check if /dev/shm exists on other systems\n            if [ -d \"/dev/shm\" ]; then\n                GC_LOG_DIR=\"/dev/shm\"\n            else\n                GC_LOG_DIR=${BASE_DIR}\n            fi\n        ;;\n    esac\n}\n\nchoose_gc_options()\n{\n    # Example of JAVA_MAJOR_VERSION value : '1', '9', '10', '11', ...\n    # '1' means releases before Java 9\n    JAVA_MAJOR_VERSION=$(\"$JAVA\" -version 2>&1 | awk -F '\"' '/version/ {print $2}' | awk -F '.' '{print $1}')\n    if [ -z \"$JAVA_MAJOR_VERSION\" ] || [ \"$JAVA_MAJOR_VERSION\" -lt \"9\" ] ; then\n      JAVA_OPT=\"${JAVA_OPT} -server -Xms4g -Xmx4g -Xmn2g -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m\"\n      JAVA_OPT=\"${JAVA_OPT} -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:SurvivorRatio=8 -XX:-UseParNewGC\"\n      JAVA_OPT=\"${JAVA_OPT} -verbose:gc -Xloggc:${GC_LOG_DIR}/rmq_srv_gc_%p_%t.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps\"\n      JAVA_OPT=\"${JAVA_OPT} -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m\"\n    else\n      JAVA_OPT=\"${JAVA_OPT} -server -Xms4g -Xmx4g -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m\"\n      JAVA_OPT=\"${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0\"\n      JAVA_OPT=\"${JAVA_OPT} -Xlog:gc*:file=${GC_LOG_DIR}/rmq_srv_gc_%p_%t.log:time,tags:filecount=5,filesize=30M\"\n    fi\n}\n\nchoose_gc_log_directory\nchoose_gc_options\nJAVA_OPT=\"${JAVA_OPT} -XX:-OmitStackTraceInFastThrow\"\nJAVA_OPT=\"${JAVA_OPT} -XX:-UseLargePages\"\n#JAVA_OPT=\"${JAVA_OPT} -Xdebug -Xrunjdwp:transport=dt_socket,address=9555,server=y,suspend=n\"\nJAVA_OPT=\"${JAVA_OPT} ${JAVA_OPT_EXT}\"\nJAVA_OPT=\"${JAVA_OPT} -cp ${CLASSPATH}\"\n\n\"$JAVA\" ${JAVA_OPT} $@\n"
  },
  {
    "path": "distribution/bin/setcache.sh",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nexport PATH=$PATH:/sbin\n\n#\n# GB\n#\nfunction changeFreeCache()\n{\n    EXTRA=$1\n    MIN=$2\n    sysctl -w vm.extra_free_kbytes=${EXTRA}000000\n    sysctl -w vm.min_free_kbytes=${MIN}000000\n    sysctl -w vm.swappiness=0\n}\n\n\nif [ $# -ne 2 ]\nthen\n    echo \"Usage: $0 extra_free_kbytes(GB) min_free_kbytes(GB)\"\n    echo \"Example: $0 3 1\"\n    exit\nfi\n\nchangeFreeCache $1 $2\n"
  },
  {
    "path": "distribution/bin/startfsrv.sh",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nif [ -z \"$ROCKETMQ_HOME\" ] ; then\n  ## resolve links - $0 may be a link to maven's home\n  PRG=\"$0\"\n\n  # need this for relative symlinks\n  while [ -h \"$PRG\" ] ; do\n    ls=`ls -ld \"$PRG\"`\n    link=`expr \"$ls\" : '.*-> \\(.*\\)$'`\n    if expr \"$link\" : '/.*' > /dev/null; then\n      PRG=\"$link\"\n    else\n      PRG=\"`dirname \"$PRG\"`/$link\"\n    fi\n  done\n\n  saveddir=`pwd`\n\n  ROCKETMQ_HOME=`dirname \"$PRG\"`/..\n\n  # make it fully qualified\n  ROCKETMQ_HOME=`cd \"$ROCKETMQ_HOME\" && pwd`\n\n  cd \"$saveddir\"\nfi\n\nexport ROCKETMQ_HOME\n\nnohup sh ${ROCKETMQ_HOME}/bin/runserver.sh org.apache.rocketmq.filtersrv.FiltersrvStartup $@ &\n"
  },
  {
    "path": "distribution/bin/tools.cmd",
    "content": "@echo off\nrem Licensed to the Apache Software Foundation (ASF) under one or more\nrem contributor license agreements.  See the NOTICE file distributed with\nrem this work for additional information regarding copyright ownership.\nrem The ASF licenses this file to You under the Apache License, Version 2.0\nrem (the \"License\"); you may not use this file except in compliance with\nrem the License.  You may obtain a copy of the License at\nrem\nrem     http://www.apache.org/licenses/LICENSE-2.0\nrem\nrem Unless required by applicable law or agreed to in writing, software\nrem distributed under the License is distributed on an \"AS IS\" BASIS,\nrem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nrem See the License for the specific language governing permissions and\nrem limitations under the License.\n\nif not exist \"%JAVA_HOME%\\bin\\java.exe\" echo Please set the JAVA_HOME variable in your environment, We need java(x64)! & EXIT /B 1\n\nset \"JAVA=%JAVA_HOME%\\bin\\java.exe\"\n\nsetlocal\nset BASE_DIR=%~dp0\nset BASE_DIR=%BASE_DIR:~0,-1%\nfor %%d in (%BASE_DIR%) do set BASE_DIR=%%~dpd\n\nset CLASSPATH=.;%BASE_DIR%conf;%BASE_DIR%lib\\*;%CLASSPATH%\n\nrem ===========================================================================================\nrem JVM Configuration\nrem ===========================================================================================\nset \"JAVA_OPT=%JAVA_OPT% -server -Xms1g -Xmx1g -Xmn256m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m\"\nset \"JAVA_OPT=%JAVA_OPT% -cp \"%CLASSPATH%\"\"\n\n\"%JAVA%\" %JAVA_OPT% %*\n"
  },
  {
    "path": "distribution/bin/tools.sh",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n#===========================================================================================\n# Java Environment Setting\n#===========================================================================================\nerror_exit ()\n{\n    echo \"ERROR: $1 !!\"\n    exit 1\n}\n\nfind_java_home()\n{\n    if [ -n \"$JAVA_HOME\" ]; then\n        JAVA_HOME=$JAVA_HOME\n        return\n    fi\n    case \"`uname`\" in\n        Darwin)\n            JAVA_HOME=$(/usr/libexec/java_home)\n        ;;\n        *)\n            JAVA_HOME=$(dirname $(dirname $(readlink -f $(which javac))))\n        ;;\n    esac\n}\n\nfind_java_home\n\n[ ! -e \"$JAVA_HOME/bin/java\" ] && JAVA_HOME=$HOME/jdk/java\n[ ! -e \"$JAVA_HOME/bin/java\" ] && JAVA_HOME=/usr/java\n[ ! -e \"$JAVA_HOME/bin/java\" ] && error_exit \"Please set the JAVA_HOME variable in your environment, We need java(x64)!\"\n\nexport JAVA_HOME\nexport JAVA=\"$JAVA_HOME/bin/java\"\nexport BASE_DIR=$(dirname $0)/..\nexport CLASSPATH=.:${BASE_DIR}/conf:${BASE_DIR}/lib/*:${CLASSPATH}\n\n#===========================================================================================\n# JVM Configuration\n#===========================================================================================\nJAVA_OPT=\"${JAVA_OPT} -server -Xms1g -Xmx1g -Xmn256m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m\"\nJAVA_OPT=\"${JAVA_OPT} -cp ${CLASSPATH}\"\n\n\"$JAVA\" ${JAVA_OPT} \"$@\"\n"
  },
  {
    "path": "distribution/conf/2m-2s-async/broker-a-s.properties",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nbrokerClusterName=DefaultCluster\nbrokerName=broker-a\nbrokerId=1\ndeleteWhen=04\nfileReservedTime=48\nbrokerRole=SLAVE\nflushDiskType=ASYNC_FLUSH\n"
  },
  {
    "path": "distribution/conf/2m-2s-async/broker-a.properties",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nbrokerClusterName=DefaultCluster\nbrokerName=broker-a\nbrokerId=0\ndeleteWhen=04\nfileReservedTime=48\nbrokerRole=ASYNC_MASTER\nflushDiskType=ASYNC_FLUSH\n"
  },
  {
    "path": "distribution/conf/2m-2s-async/broker-b-s.properties",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nbrokerClusterName=DefaultCluster\nbrokerName=broker-b\nbrokerId=1\ndeleteWhen=04\nfileReservedTime=48\nbrokerRole=SLAVE\nflushDiskType=ASYNC_FLUSH\n"
  },
  {
    "path": "distribution/conf/2m-2s-async/broker-b.properties",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nbrokerClusterName=DefaultCluster\nbrokerName=broker-b\nbrokerId=0\ndeleteWhen=04\nfileReservedTime=48\nbrokerRole=ASYNC_MASTER\nflushDiskType=ASYNC_FLUSH\n"
  },
  {
    "path": "distribution/conf/2m-2s-sync/broker-a-s.properties",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nbrokerClusterName=DefaultCluster\nbrokerName=broker-a\nbrokerId=1\ndeleteWhen=04\nfileReservedTime=48\nbrokerRole=SLAVE\nflushDiskType=ASYNC_FLUSH\n"
  },
  {
    "path": "distribution/conf/2m-2s-sync/broker-a.properties",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nbrokerClusterName=DefaultCluster\nbrokerName=broker-a\nbrokerId=0\ndeleteWhen=04\nfileReservedTime=48\nbrokerRole=SYNC_MASTER\nflushDiskType=ASYNC_FLUSH\n"
  },
  {
    "path": "distribution/conf/2m-2s-sync/broker-b-s.properties",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nbrokerClusterName=DefaultCluster\nbrokerName=broker-b\nbrokerId=1\ndeleteWhen=04\nfileReservedTime=48\nbrokerRole=SLAVE\nflushDiskType=ASYNC_FLUSH\n"
  },
  {
    "path": "distribution/conf/2m-2s-sync/broker-b.properties",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nbrokerClusterName=DefaultCluster\nbrokerName=broker-b\nbrokerId=0\ndeleteWhen=04\nfileReservedTime=48\nbrokerRole=SYNC_MASTER\nflushDiskType=ASYNC_FLUSH\n"
  },
  {
    "path": "distribution/conf/2m-noslave/broker-a.properties",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nbrokerClusterName=DefaultCluster\nbrokerName=broker-a\nbrokerId=0\ndeleteWhen=04\nfileReservedTime=48\nbrokerRole=ASYNC_MASTER\nflushDiskType=ASYNC_FLUSH\n"
  },
  {
    "path": "distribution/conf/2m-noslave/broker-b.properties",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nbrokerClusterName=DefaultCluster\nbrokerName=broker-b\nbrokerId=0\ndeleteWhen=04\nfileReservedTime=48\nbrokerRole=ASYNC_MASTER\nflushDiskType=ASYNC_FLUSH\n"
  },
  {
    "path": "distribution/conf/2m-noslave/broker-trace.properties",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\nbrokerClusterName=DefaultCluster\nbrokerName=broker-trace\nbrokerId=0\ndeleteWhen=04\nfileReservedTime=48\nbrokerRole=ASYNC_MASTER\nflushDiskType=ASYNC_FLUSH\n"
  },
  {
    "path": "distribution/conf/broker.conf",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\nbrokerClusterName = DefaultCluster\nbrokerName = broker-a\nbrokerId = 0\ndeleteWhen = 04\nfileReservedTime = 48\nbrokerRole = ASYNC_MASTER\nflushDiskType = ASYNC_FLUSH\n"
  },
  {
    "path": "distribution/conf/container/2container-2m-2s/broker-a-in-container1.conf",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\n#Master配置\nbrokerClusterName=DefaultCluster\nbrokerName=broker-a\nbrokerId=0\nbrokerRole=SYNC_MASTER\nflushDiskType=ASYNC_FLUSH\nstorePathRootDir=/root/broker-a/store\nstorePathCommitLog=/root/broker-a/store/commitlog\nlistenPort=10911\nhaListenPort=10912\ntotalReplicas=2\ninSyncReplicas=2\nminInSyncReplicas=1\nenableAutoInSyncReplicas=true\nslaveReadEnable=true\nbrokerHeartbeatInterval=1000\nbrokerNotActiveTimeoutMillis=5000\nsendHeartbeatTimeoutMillis=1000\nenableSlaveActingMaster=true"
  },
  {
    "path": "distribution/conf/container/2container-2m-2s/broker-a-in-container2.conf",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\n#Master配置\nbrokerClusterName=DefaultCluster\nbrokerName=broker-a\nbrokerId=1\nbrokerRole=SLAVE\nflushDiskType=ASYNC_FLUSH\nstorePathRootDir=/root/broker-a/store\nstorePathCommitLog=/root/broker-a/store/commitlog\nlistenPort=10911\nhaListenPort=10912\ntotalReplicas=2\ninSyncReplicas=2\nminInSyncReplicas=1\nenableAutoInSyncReplicas=true\nslaveReadEnable=true\nbrokerHeartbeatInterval=1000\nbrokerNotActiveTimeoutMillis=5000\nsendHeartbeatTimeoutMillis=1000\nenableSlaveActingMaster=true"
  },
  {
    "path": "distribution/conf/container/2container-2m-2s/broker-b-in-container1.conf",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\n#Slave配置\nbrokerClusterName=DefaultCluster\nbrokerName=broker-b\nbrokerId=1\nbrokerRole=SLAVE\nflushDiskType=ASYNC_FLUSH\nstorePathRootDir=/root/broker-b/store\nstorePathCommitLog=/root/broker-b/store/commitlog\nlistenPort=20911\nhaListenPort=20912\ntotalReplicas=2\ninSyncReplicas=2\nminInSyncReplicas=1\nenableAutoInSyncReplicas=true\nslaveReadEnable=true\nbrokerHeartbeatInterval=1000\nbrokerNotActiveTimeoutMillis=5000\nsendHeartbeatTimeoutMillis=1000\nenableSlaveActingMaster=true"
  },
  {
    "path": "distribution/conf/container/2container-2m-2s/broker-b-in-container2.conf",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\n#Slave配置\nbrokerClusterName=DefaultCluster\nbrokerName=broker-b\nbrokerId=0\nbrokerRole=SYNC_MASTER\nflushDiskType=ASYNC_FLUSH\nstorePathRootDir=/root/broker-b/store\nstorePathCommitLog=/root/broker-b/store/commitlog\nlistenPort=20911\nhaListenPort=20912\ntotalReplicas=2\ninSyncReplicas=2\nminInSyncReplicas=1\nenableAutoInSyncReplicas=true\nslaveReadEnable=true\nbrokerHeartbeatInterval=1000\nbrokerNotActiveTimeoutMillis=5000\nsendHeartbeatTimeoutMillis=1000\nenableSlaveActingMaster=true"
  },
  {
    "path": "distribution/conf/container/2container-2m-2s/broker-container1.conf",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\n#配置端口，用于接收mqadmin命令\nlistenPort=10811\n#指定namesrv\nnamesrvAddr=172.22.144.49:9876\n#或指定自动获取namesrv\nfetchNamesrvAddrByAddressServer=false\n#指定要向BrokerContainer内添加的brokerConfig路径，多个config间用“:”分隔；\n#不指定则只启动BrokerConainer，具体broker可通过mqadmin工具添加\nbrokerConfigPaths=/root/2container-2m-2s/broker-a-in-container1.conf:/root/2container-2m-2s/broker-b-in-container1.conf"
  },
  {
    "path": "distribution/conf/container/2container-2m-2s/broker-container2.conf",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\n#配置端口，用于接收mqadmin命令\nlistenPort=10811\n#指定namesrv\nnamesrvAddr=172.22.144.49:9876\n#或指定自动获取namesrv\nfetchNamesrvAddrByAddressServer=false\n#指定要向BrokerContainer内添加的brokerConfig路径，多个config间用“:”分隔；\n#不指定则只启动BrokerConainer，具体broker可通过mqadmin工具添加\nbrokerConfigPaths=/root/2container-2m-2s/broker-a-in-container2.conf:/root/2container-2m-2s/broker-b-in-container2.conf"
  },
  {
    "path": "distribution/conf/container/2container-2m-2s/nameserver.conf",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\nsupportActingMaster=true"
  },
  {
    "path": "distribution/conf/controller/cluster-3n-independent/controller-n0.conf",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\ncontrollerDLegerGroup = group1\ncontrollerDLegerPeers = n0-127.0.0.1:9878;n1-127.0.0.1:9868;n2-127.0.0.1:9858\ncontrollerDLegerSelfId = n0\n\n"
  },
  {
    "path": "distribution/conf/controller/cluster-3n-independent/controller-n1.conf",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\ncontrollerDLegerGroup = group1\ncontrollerDLegerPeers = n0-127.0.0.1:9878;n1-127.0.0.1:9868;n2-127.0.0.1:9858\ncontrollerDLegerSelfId = n1\n\n"
  },
  {
    "path": "distribution/conf/controller/cluster-3n-independent/controller-n2.conf",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\ncontrollerDLegerGroup = group1\ncontrollerDLegerPeers = n0-127.0.0.1:9878;n1-127.0.0.1:9868;n2-127.0.0.1:9858\ncontrollerDLegerSelfId = n2\n\n"
  },
  {
    "path": "distribution/conf/controller/cluster-3n-namesrv-plugin/namesrv-n0.conf",
    "content": "<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n#Namesrv config\nlistenPort = 9876\nenableControllerInNamesrv = true\n\n#controller config\ncontrollerDLegerGroup = group1\ncontrollerDLegerPeers = n0-127.0.0.1:9878;n1-127.0.0.1:9868;n2-127.0.0.1:9858\ncontrollerDLegerSelfId = n0"
  },
  {
    "path": "distribution/conf/controller/cluster-3n-namesrv-plugin/namesrv-n1.conf",
    "content": "<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n#Namesrv config\nlistenPort = 9886\nenableControllerInNamesrv = true\n\n#controller config\ncontrollerDLegerGroup = group1\ncontrollerDLegerPeers = n0-127.0.0.1:9878;n1-127.0.0.1:9868;n2-127.0.0.1:9858\ncontrollerDLegerSelfId = n1"
  },
  {
    "path": "distribution/conf/controller/cluster-3n-namesrv-plugin/namesrv-n2.conf",
    "content": "<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n#Namesrv config\nlistenPort = 9896\nenableControllerInNamesrv = true\n\n#controller config\ncontrollerDLegerGroup = group1\ncontrollerDLegerPeers = n0-127.0.0.1:9878;n1-127.0.0.1:9868;n2-127.0.0.1:9858\ncontrollerDLegerSelfId = n2"
  },
  {
    "path": "distribution/conf/controller/controller-standalone.conf",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\ncontrollerDLegerGroup = group1\ncontrollerDLegerPeers = n0-127.0.0.1:9878\ncontrollerDLegerSelfId = n0\n\n"
  },
  {
    "path": "distribution/conf/controller/quick-start/broker-n0.conf",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\nbrokerClusterName = DefaultCluster\nbrokerName = broker-a\nbrokerId = -1\nbrokerRole = SLAVE\ndeleteWhen = 04\nfileReservedTime = 48\nenableControllerMode = true\ncontrollerAddr = 127.0.0.1:9878\nnamesrvAddr = 127.0.0.1:9876\nallAckInSyncStateSet=true\nlistenPort=30911\nstorePathRootDir=/tmp/rmqstore/node00\nstorePathCommitLog=/tmp/rmqstore/node00/commitlog"
  },
  {
    "path": "distribution/conf/controller/quick-start/broker-n1.conf",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\nbrokerClusterName = DefaultCluster\nbrokerName = broker-a\nbrokerId = -1\nbrokerRole = SLAVE\ndeleteWhen = 04\nfileReservedTime = 48\nenableControllerMode = true\ncontrollerAddr = 127.0.0.1:9878\nnamesrvAddr = 127.0.0.1:9876\nallAckInSyncStateSet=true\nlistenPort=30921\nstorePathRootDir=/tmp/rmqstore/node01\nstorePathCommitLog=/tmp/rmqstore/node01/commitlog"
  },
  {
    "path": "distribution/conf/controller/quick-start/namesrv.conf",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\nenableControllerInNamesrv = true\ncontrollerDLegerGroup = group1\ncontrollerDLegerPeers = n0-127.0.0.1:9878\ncontrollerDLegerSelfId = n0"
  },
  {
    "path": "distribution/conf/dledger/broker-n0.conf",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\nbrokerClusterName = RaftCluster\nbrokerName=RaftNode00\nlistenPort=30911\nnamesrvAddr=127.0.0.1:9876\nstorePathRootDir=/tmp/rmqstore/node00\nstorePathCommitLog=/tmp/rmqstore/node00/commitlog\nenableDLegerCommitLog=true\ndLegerGroup=RaftNode00\ndLegerPeers=n0-127.0.0.1:40911;n1-127.0.0.1:40912;n2-127.0.0.1:40913\n## must be unique\ndLegerSelfId=n0\nsendMessageThreadPoolNums=16\n"
  },
  {
    "path": "distribution/conf/dledger/broker-n1.conf",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\nbrokerClusterName = RaftCluster\nbrokerName=RaftNode00\nlistenPort=30921\nnamesrvAddr=127.0.0.1:9876\nstorePathRootDir=/tmp/rmqstore/node01\nstorePathCommitLog=/tmp/rmqstore/node01/commitlog\nenableDLegerCommitLog=true\ndLegerGroup=RaftNode00\ndLegerPeers=n0-127.0.0.1:40911;n1-127.0.0.1:40912;n2-127.0.0.1:40913\n## must be unique\ndLegerSelfId=n1\nsendMessageThreadPoolNums=16\n"
  },
  {
    "path": "distribution/conf/dledger/broker-n2.conf",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\nbrokerClusterName = RaftCluster\nbrokerName=RaftNode00\nlistenPort=30931\nnamesrvAddr=127.0.0.1:9876\nstorePathRootDir=/tmp/rmqstore/node02\nstorePathCommitLog=/tmp/rmqstore/node02/commitlog\nenableDLegerCommitLog=true\ndLegerGroup=RaftNode00\ndLegerPeers=n0-127.0.0.1:40911;n1-127.0.0.1:40912;n2-127.0.0.1:40913\n## must be unique\ndLegerSelfId=n2\nsendMessageThreadPoolNums=16\n"
  },
  {
    "path": "distribution/conf/rmq-proxy.json",
    "content": "{\n  \"rocketMQClusterName\": \"DefaultCluster\"\n}"
  },
  {
    "path": "distribution/conf/tools.yml",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\n\naccessKey: rocketmq2\nsecretKey: 12345678\n    \n"
  },
  {
    "path": "distribution/pom.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n    <modelVersion>4.0.0</modelVersion>\n    <parent>\n        <groupId>org.apache.rocketmq</groupId>\n        <artifactId>rocketmq-all</artifactId>\n        <version>${revision}</version>\n    </parent>\n    <artifactId>rocketmq-distribution</artifactId>\n    <name>rocketmq-distribution ${project.version}</name>\n    <packaging>pom</packaging>\n\n    <properties>\n        <project.root>${basedir}/..</project.root>\n    </properties>\n\n    <profiles>\n        <profile>\n            <id>release-all</id>\n            <dependencies>\n                <dependency>\n                    <groupId>org.apache.rocketmq</groupId>\n                    <artifactId>rocketmq-container</artifactId>\n                </dependency>\n                <dependency>\n                    <groupId>org.apache.rocketmq</groupId>\n                    <artifactId>rocketmq-controller</artifactId>\n                </dependency>\n                <dependency>\n                    <groupId>org.apache.rocketmq</groupId>\n                    <artifactId>rocketmq-broker</artifactId>\n                </dependency>\n                <dependency>\n                    <groupId>org.apache.rocketmq</groupId>\n                    <artifactId>rocketmq-proxy</artifactId>\n                </dependency>\n                <dependency>\n                    <groupId>org.apache.rocketmq</groupId>\n                    <artifactId>rocketmq-client</artifactId>\n                </dependency>\n                <dependency>\n                    <groupId>org.apache.rocketmq</groupId>\n                    <artifactId>rocketmq-tools</artifactId>\n                </dependency>\n                <dependency>\n                    <groupId>org.apache.rocketmq</groupId>\n                    <artifactId>rocketmq-example</artifactId>\n                </dependency>\n            </dependencies>\n\n            <build>\n                <plugins>\n                    <plugin>\n                        <artifactId>maven-assembly-plugin</artifactId>\n                        <executions>\n                            <execution>\n                                <id>release-all</id>\n                                <goals>\n                                    <goal>single</goal>\n                                </goals>\n                                <phase>package</phase>\n                                <configuration>\n                                    <descriptors>\n                                        <descriptor>release.xml</descriptor>\n                                    </descriptors>\n                                    <appendAssemblyId>false</appendAssemblyId>\n                                </configuration>\n                            </execution>\n                        </executions>\n                    </plugin>\n                </plugins>\n                <finalName>rocketmq-${project.version}</finalName>\n            </build>\n        </profile>\n\n        <profile>\n            <id>release-client</id>\n            <dependencies>\n                <dependency>\n                    <groupId>org.apache.rocketmq</groupId>\n                    <artifactId>rocketmq-client</artifactId>\n                </dependency>\n            </dependencies>\n\n            <build>\n                <plugins>\n                    <plugin>\n                        <artifactId>maven-assembly-plugin</artifactId>\n                        <executions>\n                            <execution>\n                                <id>release-client</id>\n                                <goals>\n                                    <goal>single</goal>\n                                </goals>\n                                <phase>package</phase>\n                                <configuration>\n                                    <descriptors>\n                                        <descriptor>release-client.xml</descriptor>\n                                    </descriptors>\n                                    <appendAssemblyId>false</appendAssemblyId>\n                                </configuration>\n                            </execution>\n                        </executions>\n                    </plugin>\n                </plugins>\n                <finalName>rocketmq-client-${project.version}</finalName>\n            </build>\n        </profile>\n    </profiles>\n\n</project>\n"
  },
  {
    "path": "distribution/release-client.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n<assembly>\n    <id>client</id>\n    <includeBaseDirectory>true</includeBaseDirectory>\n    <formats>\n        <format>dir</format>\n        <format>tar.gz</format>\n        <format>zip</format>\n    </formats>\n\n    <fileSets>\n        <fileSet>\n            <directory>../</directory>\n            <includes>\n                <include>README.md</include>\n            </includes>\n        </fileSet>\n    </fileSets>\n\n    <files>\n        <file>\n            <source>LICENSE-BIN</source>\n            <destName>LICENSE</destName>\n        </file>\n        <file>\n            <source>NOTICE-BIN</source>\n            <destName>NOTICE</destName>\n        </file>\n    </files>\n\n    <moduleSets>\n        <moduleSet>\n            <useAllReactorProjects>true</useAllReactorProjects>\n            <includes>\n                <include>org.apache.rocketmq:rocketmq-client</include>\n                <include>org.apache.rocketmq:rocketmq-openmessaging</include>\n            </includes>\n            <binaries>\n                <outputDirectory>./</outputDirectory>\n                <unpack>false</unpack>\n                <dependencySets>\n                    <dependencySet>\n                        <outputDirectory>./</outputDirectory>\n                    </dependencySet>\n                </dependencySets>\n            </binaries>\n        </moduleSet>\n    </moduleSets>\n</assembly>\n"
  },
  {
    "path": "distribution/release.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n<assembly>\n    <id>all</id>\n    <includeBaseDirectory>true</includeBaseDirectory>\n    <formats>\n        <format>dir</format>\n        <format>tar.gz</format>\n        <format>zip</format>\n    </formats>\n    <fileSets>\n        <fileSet>\n            <directory>../</directory>\n            <includes>\n                <include>README.md</include>\n            </includes>\n        </fileSet>\n\n        <fileSet>\n            <includes>\n                <include>conf/**</include>\n                <include>benchmark/*</include>\n            </includes>\n        </fileSet>\n\n        <fileSet>\n            <includes>\n                <include>bin/**</include>\n            </includes>\n            <fileMode>0755</fileMode>\n        </fileSet>\n    </fileSets>\n\n    <files>\n        <file>\n            <source>LICENSE-BIN</source>\n            <destName>LICENSE</destName>\n        </file>\n        <file>\n            <source>NOTICE-BIN</source>\n            <destName>NOTICE</destName>\n        </file>\n        <file>\n            <source>../broker/src/main/resources/rmq.broker.logback.xml</source>\n            <destName>conf/rmq.broker.logback.xml</destName>\n        </file>\n        <file>\n            <source>../client/src/main/resources/rmq.client.logback.xml</source>\n            <destName>conf/rmq.client.logback.xml</destName>\n        </file>\n        <file>\n            <source>../controller/src/main/resources/rmq.controller.logback.xml</source>\n            <destName>conf/rmq.controller.logback.xml</destName>\n        </file>\n        <file>\n            <source>../namesrv/src/main/resources/rmq.namesrv.logback.xml</source>\n            <destName>conf/rmq.namesrv.logback.xml</destName>\n        </file>\n        <file>\n            <source>../tools/src/main/resources/rmq.tools.logback.xml</source>\n            <destName>conf/rmq.tools.logback.xml</destName>\n        </file>\n        <file>\n            <source>../proxy/src/main/resources/rmq.proxy.logback.xml</source>\n            <destName>conf/rmq.proxy.logback.xml</destName>\n        </file>\n    </files>\n\n    <moduleSets>\n        <moduleSet>\n            <useAllReactorProjects>true</useAllReactorProjects>\n            <includes>\n                <include>org.apache.rocketmq:rocketmq-container</include>\n                <include>org.apache.rocketmq:rocketmq-broker</include>\n                <include>org.apache.rocketmq:rocketmq-tools</include>\n                <include>org.apache.rocketmq:rocketmq-client</include>\n                <include>org.apache.rocketmq:rocketmq-namesrv</include>\n                <include>org.apache.rocketmq:rocketmq-example</include>\n                <include>org.apache.rocketmq:rocketmq-openmessaging</include>\n                <include>org.apache.rocketmq:rocketmq-controller</include>\n            </includes>\n            <binaries>\n                <outputDirectory>lib/</outputDirectory>\n                <unpack>false</unpack>\n                <dependencySets>\n                    <dependencySet>\n                        <outputDirectory>lib/</outputDirectory>\n                        <excludes>\n                            <exclude>io.jaegertracing:jaeger-core</exclude>\n                            <exclude>io.jaegertracing:jaeger-client</exclude>\n                        </excludes>\n                    </dependencySet>\n                </dependencySets>\n            </binaries>\n        </moduleSet>\n    </moduleSets>\n</assembly>\n"
  },
  {
    "path": "docs/cn/BrokerContainer.md",
    "content": "# BrokerContainer\n\n## 背景\n\n在RocketMQ 4.x 版本中，一个进程只有一个broker，通常会以主备或者DLedger（Raft）的形式部署，但是一个进程中只有一个broker，而slave一般只承担冷备或热备的作用，节点之间角色的不对等导致slave节点资源没有充分被利用。\n因此在RocketMQ 5.x 版本中，提供一种新的模式BrokerContainer，在一个BrokerContainer进程中可以加入多个Broker（Master Broker、Slave Broker、DLedger Broker），来提高单个节点的资源利用率，并且可以通过各种形式的交叉部署来实现节点之间的对等部署。\n该特性的优点包括：\n\n1. 一个BrokerContainer进程中可以加入多个broker，通过进程内混部来提高单个节点的资源利用率\n2. 通过各种形式的交叉部署来实现节点之间的对等部署，增强单节点的高可用能力\n3. 利用BrokerContainer可以实现单进程内多CommitLog写入，也可以实现单机的多磁盘写入\n4. BrokerContainer中的CommitLog天然隔离的，不同的CommitLog（broker）可以采取不同作用，比如可以用来比如创建单独的broker做不同TTL的CommitLog。\n\n## 架构\n\n### 单进程视图\n\n![](https://s4.ax1x.com/2022/01/26/7LMZHP.png)\n\n相比于原来一个Broker一个进程，RocketMQ 5.0将增加BrokerContainer概念，一个BrokerContainer可以存放多个Broker，每个Broker拥有不同的端口，但它们共享同一个传输层（remoting层），而每一个broker在功能上是完全独立的。BrokerContainer也拥有自己端口，在运行时可以通过admin命令来增加或减少Broker。\n\n### 对等部署形态\n\n在BrokerContainer模式下，可以通过各种形式的交叉部署完成节点的对等部署\n\n- 二副本对等部署\n\n![](https://s4.ax1x.com/2022/01/26/7LQi5T.png)\n\n二副本对等部署情况下，每个节点都会有一主一备，资源利用率均等。另外假设图中Node1宕机，由于Node2的broker_2可读可写，broker_1可以备读，因此普通消息的收发不会收到影响，单节点的高可用能力得到了增强。\n\n- 三副本对等部署\n\n![](https://s4.ax1x.com/2022/01/26/7LQMa6.png)\n\n三副本对等部署情况下，每个节点都会有一主两备，资源利用率均等。此外，和二副本一样，任意一个节点的宕机也不会影响到普通消息的收发。\n\n### 传输层共享\n\n![](https://s4.ax1x.com/2022/02/07/HMNIVs.png)\n\nBrokerContainer中的所有broker共享同一个传输层，就像RocketMQ客户端中同进程的Consumer和Producer共享同一个传输层一样。\n\n这里为NettyRemotingServer提供SubRemotingServer支持，通过为一个RemotingServer绑定另一个端口即可生成SubRemotingServer，其共享NettyRemotingServer的Netty实例、计算资源、以及协议栈等，但拥有不同的端口以及ProcessorTable。另外同一个BrokerContainer中的所有的broker也会共享同一个BrokerOutAPI（RemotingClient）。\n\n## 启动方式和配置\n\n![](https://s4.ax1x.com/2022/01/26/7LQ1PO.png)\n\n像Broker启动利用BrokerStartup一样，使用BrokerContainerStartup来启动BrokerContainer。我们可以通过两种方式向BrokerContainer中增加broker，一种是通过启动时通过在配置文件中指定\n\nBrokerContainer配置文件内容主要是Netty网络层参数（由于传输层共享），BrokerContainer的监听端口、namesrv配置，以及最重要的brokerConfigPaths参数，brokerConfigPaths是指需要向BrokerContainer内添加的brokerConfig路径，多个config间用“:”分隔，不指定则只启动BrokerConainer，具体broker可通过mqadmin工具添加\n\nbroker-container.conf（distribution/conf/container/broker-container.conf）:\n\n```\n#配置端口，用于接收mqadmin命令\nlistenPort=10811\n#指定namesrv\nnamesrvAddr=127.0.0.1:9876\n#或指定自动获取namesrv\nfetchNamesrvAddrByAddressServer=true\n#指定要向BrokerContainer内添加的brokerConfig路径，多个config间用“:”分隔；\n#不指定则只启动BrokerConainer，具体broker可通过mqadmin工具添加\nbrokerConfigPaths=/home/admin/broker-a.conf:/home/admin/broker-b.conf\n```\nbroker的配置和以前一样，但在BrokerContainer模式下broker配置文件中下Netty网络层参数和nameserver参数不生效，均使用BrokerContainer的配置参数。\n\n完成配置文件后，可以以如下命令启动\n```\nsh mqbrokercontainer -c broker-container.conf\n```\nmqbrokercontainer脚本路径为distribution/bin/mqbrokercontainer。\n\n## 运行时增加或减少Broker\n\n当BrokerContainer进程启动后，也可以通过Admin命令来增加或减少Broker。\n\nAddBrokerCommand\n```\nusage: mqadmin addBroker -b <arg> -c <arg> [-h] [-n <arg>]\n -b,--brokerConfigPath <arg>      Broker config path\n -c,--brokerContainerAddr <arg>   Broker container address\n -h,--help                        Print help\n -n,--namesrvAddr <arg>           Name server address list, eg: 192.168.0.1:9876;192.168.0.2:9876\n```\n\nRemoveBroker Command\n```\nusage: mqadmin removeBroker -b <arg> -c <arg> [-h] [-n <arg>]\n -b,--brokerIdentity <arg>        Information to identify a broker: clusterName:brokerName:brokerId\n -c,--brokerContainerAddr <arg>   Broker container address\n -h,--help                        Print help\n -n,--namesrvAddr <arg>           Name server address list, eg: 192.168.0.1:9876;192.168.0.2:9876\n```\n\n## 存储变化\n\nstorePathRootDir，storePathCommitLog路径依然为MessageStoreConfig中配置值，需要注意的是同一个brokerContainer中的broker不能使用相同的storePathRootDir，storePathCommitLog，否则不同的broker占用同一个存储目录，发生数据混乱。\n\n在文件删除策略上，仍然单个Broker的视角来进行删除，但MessageStoreConfig新增replicasPerDiskPartition参数和logicalDiskSpaceCleanForciblyThreshold。\n\nreplicasPerDiskPartition表示同一磁盘分区上有多少个副本，即该broker的存储目录所在的磁盘分区被几个broker共享，默认值为1。该配置用于计算当同一节点上的多个broker共享同一磁盘分区时，各broker的磁盘配额\n\ne.g. replicasPerDiskPartition==2且broker所在磁盘空间为1T时，则该broker磁盘配额为512G，该broker的逻辑磁盘空间利用率基于512G的空间进行计算。\n\nlogicalDiskSpaceCleanForciblyThreshold，该值只在replicasPerDiskPartition大于1时生效，表示逻辑磁盘空间强制清理阈值，默认为0.80（80%）， 逻辑磁盘空间利用率为该broker在自身磁盘配额内的空间利用率，物理磁盘空间利用率为该磁盘分区总空间利用率。由于在BrokerContainer实现中，考虑计算效率的情况下，仅统计了commitLog+consumeQueue（+ BCQ）+indexFile作为broker的存储空间占用，其余文件如元数据、消费进度、磁盘脏数据等未统计在内，故在多个broker存储空间达到动态平衡时，各broker所占空间可能有相差，以一个BrokerContainer中有两个broker为例，两broker存储空间差异可表示为：\n![](https://s4.ax1x.com/2022/01/26/7L14v4.png)\n其中，R_logical为logicalDiskSpaceCleanForciblyThreshold，R_phy为diskSpaceCleanForciblyRatio，T为磁盘分区总空间，x为除上述计算的broker存储空间外的其他文件所占磁盘总空间比例，可见，当\n![](https://s4.ax1x.com/2022/01/26/7L1TbR.png)\n时，可保证BrokerContainer各Broker存储空间在达到动态平衡时相差无几。\n\neg.假设broker获取到的配额是500g（根据replicasPerDiskPartition计算获得），logicalDiskSpaceCleanForciblyThreshold为默认值0.8，则默认commitLog+consumeQueue（+ BCQ）+indexFile总量超过400g就会强制清理文件。\n\n其他清理阈值（diskSpaceCleanForciblyRatio、diskSpaceWarningLevelRatio），文件保存时间（fileReservedTime）等逻辑与之前保持一致。\n\n注意：当以普通broker方式启动而非brokerContainer启动时，且replicasPerDiskPartition=1（默认值）时，清理逻辑与之前完全一致。replicasPerDiskPartition>1时，逻辑磁盘空间强制清理阈值logicalDiskSpaceCleanForciblyThreshold将会生效。\n\n\n## 日志变化\n\n在BrokerContainer模式下日志的默认输出路径将发生变化，具体为：\n\n```\n{user.home}/logs/rocketmqlogs/${brokerCanonicalName}/\n```\n\n其中 `brokerCanonicalName` 为 `{BrokerClusterName_BrokerName_BrokerId}`。"
  },
  {
    "path": "docs/cn/Configuration_System.md",
    "content": "# 系统配置\n\n本节重点介绍系统（JVM/OS）的配置 \n---\n\n## **1 JVM 选项** ##\n\n建议使用最新发布的 JDK 1.8 版本。设置相同的 Xms 和 Xmx 值以防止 JVM 调整堆大小，并获得更好的性能。一种通用的JVM配置如下： \n\n    -server -Xms8g -Xmx8g -Xmn4g\n\n设置 Direct ByteBuffer 内存大小。当 Direct ByteBuffer 达到指定大小时，将触发 Full GC：\n\n    -XXMaxDirectMemorySize=15g\n\n如果你不在乎 RocketMQ broker 的启动时间，建议启用预分配 Java 堆以确保在 JVM 初始化期间为每个页面分配内存。你可以通过以下方式启用它： \n    \n    -XX+AlwaysPreTouch\n\n禁用偏向锁定可以减少 JVM 停顿： \n\n    -XX-UseBiasedLocking\n\n关于垃圾收集器，推荐使用 JDK 1.8 的 G1 收集器： \n\n    -XX+UseG1GC -XXG1HeapRegionSize=16m \n    -XXG1ReservePercent=25\n    -XXInitiatingHeapOccupancyPercent=30\n\n这些 GC 选项看起来有点激进，但事实证明它在生产环境中具有良好的性能 \n\n不要把-XXMaxGCPauseMillis 的值设置太小，否则JVM会使用一个小的新生代来实现这个目标，从而导致频繁发生minor GC。因此，建议使用滚动 GC 日志文件：\n    \n    -XX+UseGCLogFileRotation \n    -XXNumberOfGCLogFiles=5 \n    -XXGCLogFileSize=30m\n    \n写 GC 文件会增加 broker 的延迟，因此可以考虑将 GC 日志文件重定向到内存文件系统：\n    \n    -Xloggcdevshmmq_gc_%p.log123\n\n## 2 Linux 内核参数 ##\n\n在 bin 文件夹里，有一个 os.sh 脚本，里面列出了许多的内核参数，只需稍作更改即可用于生产用途。需特别关注以下参数，如想了解更多细节，请参考文档/proc/sys/vm/*。 \n\n\n\n- **vm.extra_free_kbytes**, 控制VM在后台回收（kswapd）开始的阈值和直接回收（通过分配进程）开始的阈值之间保留额外的空闲内存。通过使用这个参数，RocketMQ 可以避免在内存分配过程中出现高延迟。（与内核版本有关）\n\n\n\n- **vm.min_free_kbytes**, 该值不应设置低于1024KB，否则系统将遭到破坏，并且在高负载环境下容易出现死锁。 \n\n\n\n\n\n- **vm.max_map_count**, 规定进程可以拥有的最大内存映射区域数。 RocketMQ 使用 mmap 来加载 CommitLog 和 ConsumeQueue，因此建议将此参数设置为较大的值。 \n\n\n\n- **vm.swappiness**, 定义内核交换内存页的频率。该值若较大，则会导致频繁交换，较小则会减少交换量。为了避免交换延迟，建议将此值设为 10。 \n\n\n\n- **File descriptor limits**, RocketMQ 需要给文件（CommitLog 和 ConsumeQueue）和网络连接分配文件描述符。因此建议将该值设置为 655350。 \n\n\n\n- **Disk scheduler**, 推荐使用deadline IO 调度器，它可以为请求提供有保证的延迟。 "
  },
  {
    "path": "docs/cn/Configuration_TLS.md",
    "content": "# TLS配置\n本节介绍TLS相关配置\n\n## 1 生成证书\n开发、测试的证书可以自行安装OpenSSL进行生成.建议在Linux环境下安装Open SSL并进行证书生成。\n\n### 1.1 生成ca.pem\n```shell\nopenssl req -newkey rsa:2048 -keyout ca_rsa_private.pem -x509 -days 365 -out ca.pem\n```\n### 1.2 生成server.csr\n```shell\nopenssl req -newkey rsa:2048 -keyout server_rsa.key  -out server.csr\n```\n### 1.3 生成server.pem\n```shell\nopenssl x509 -req -days 365 -in server.csr -CA ca.pem -CAkey ca_rsa_private.pem -CAcreateserial -out server.pem\n```\n### 1.4 生成client.csr\n```shell\nopenssl req -newkey rsa:2048 -keyout client_rsa.key -out client.csr\n```\n### 1.5 生成client.pem\n```shell\nopenssl x509 -req -days 365 -in client.csr -CA ca.pem -CAkey ca_rsa_private.pem -CAcreateserial -out client.pem\n```\n### 1.6 生成server.key\n```shell\nopenssl pkcs8 -topk8 -v1 PBE-SHA1-RC4-128 -in  server_rsa.key -out server.key\n```\n### 1.7 生成client.key\n```shell\nopenssl pkcs8 -topk8 -v1 PBE-SHA1-RC4-128 -in client_rsa.key -out client.key\n```\n\n## 2 创建tls.properties\n创建tls.properties文件，并将生成证书的路径和密码进行正确的配置.\n\n\n```properties\n# The flag to determine whether use test mode when initialize TLS context. default is true\ntls.test.mode.enable=false                     \n# Indicates how SSL engine respect to client authentication, default is none\ntls.server.need.client.auth=require   \n# The store path of server-side private key\ntls.server.keyPath=/opt/certFiles/server.key\n# The password of the server-side private key\ntls.server.keyPassword=123456\n# The store path of server-side X.509 certificate chain in PEM format\ntls.server.certPath=/opt/certFiles/server.pem\n# To determine whether verify the client endpoint's certificate strictly. default is false\ntls.server.authClient=false\n# The store path of trusted certificates for verifying the client endpoint's certificate\ntls.server.trustCertPath=/opt/certFiles/ca.pem\n# The ciphers in TLS\n# tls.ciphers=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256\n# The protocols in TLS\n# tls.protocols=TLSv1.2,TLSv1.3\n```\n\n如果需要客户端连接时也进行认证，则还需要在该文件中增加以下内容\n```properties\n# The store path of client-side private key \ntls.client.keyPath=/opt/certFiles/client.key\n# The password of the client-side private key\ntls.client.keyPassword=123456\n# The store path of client-side X.509 certificate chain in PEM format\ntls.client.certPath=/opt/certFiles/client.pem\n# To determine whether verify the server endpoint's certificate strictly\ntls.client.authServer=false                    \n# The store path of trusted certificates for verifying the server endpoint's certificate\ntls.client.trustCertPath=/opt/certFiles/ca.pem\n# The ciphers in TLS\n# tls.ciphers=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256\n# The protocols in TLS\n# tls.protocols=TLSv1.2,TLSv1.3\n```\n\n\n## 3 配置Rocketmq启动参数\n\n编辑rocketmq/bin路径下的配置文件，使tls.properties配置生效.-Dtls.config.file的值需要替换为步骤2中创建的tls.peoperties文件的路径\n\n### 3.1 编辑runserver.sh，在JAVA_OPT中增加以下内容：\n```shell\nJAVA_OPT=\"${JAVA_OPT} -Dtls.server.mode=enforcing -Dtls.config.file=/opt/rocketmq-4.9.3/conf/tls.properties\"\n```\n\n### 3.2 编辑runbroker.sh，在JAVA_OPT中增加以下内容：\n\n```shell\nJAVA_OPT=\"${JAVA_OPT} -Dorg.apache.rocketmq.remoting.ssl.mode=enforcing -Dtls.config.file=/opt/rocketmq-4.9.3/conf/tls.properties  -Dtls.enable=true\"\n```\n\n# 4 客户端连接\n\n创建客户端使用的tlsclient.properties，并加入以下内容：\n```properties\n# The store path of client-side private key \ntls.client.keyPath=/opt/certFiles/client.key\n# The password of the client-side private key\ntls.client.keyPassword=123456\n# The store path of client-side X.509 certificate chain in PEM format\ntls.client.certPath=/opt/certFiles/client.pem               \n# The store path of trusted certificates for verifying the server endpoint's certificate\ntls.client.trustCertPath=/opt/certFiles/ca.pem\n# The ciphers in TLS\n# tls.ciphers=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256\n# The protocols in TLS\n# tls.protocols=TLSv1.2,TLSv1.3\n```\n\nJVM中需要加以下参数.tls.config.file的值需要使用之前创建的文件：\n```shell\n-Dtls.client.authServer=true -Dtls.enable=true  -Dtls.test.mode.enable=false  -Dtls.config.file=/opt/certs/tlsclient.properties\n```\n\n在客户端连接的代码中，需要将setUseTLS设置为true：\n```java\npublic class ExampleProducer {\n    public static void main(String[] args) throws Exception {\n        DefaultMQProducer producer = new DefaultMQProducer(\"please_rename_unique_group_name\");\n        //setUseTLS should be true\n        producer.setUseTLS(true);\n        producer.start();\n\n        // Send messages as usual.\n        producer.shutdown();\n    }\n}\n```\n\n## 5 Proxy TLS 配置\n\nRocketMQ Proxy 使用 `rmq-proxy.json`（而非 `tls.properties`）进行 TLS 配置。Proxy 的 gRPC 和 Remoting 协议端口均支持 TLS。\n\n### 5.1 配置 rmq-proxy.json\n\n在 `distribution/conf/rmq-proxy.json` 中添加 TLS 相关字段：\n\n```json\n{\n  \"rocketMQClusterName\": \"DefaultCluster\",\n  \"tlsTestModeEnable\": false,\n  \"tlsKeyPath\": \"/opt/certFiles/server.key\",\n  \"tlsKeyPassword\": \"123456\",\n  \"tlsCertPath\": \"/opt/certFiles/server.pem\"\n}\n```\n\n| 字段 | 类型 | 默认值 | 说明 |\n|------|------|--------|------|\n| `tlsTestModeEnable` | boolean | `true` | 是否使用自签名测试证书，生产环境需设为 `false` |\n| `tlsKeyPath` | string | `${PROXY_HOME}/conf/tls/rocketmq.key` | 服务端私钥文件路径（PKCS#8 PEM 格式） |\n| `tlsKeyPassword` | string | `\"\"` | 加密私钥的密码，私钥未加密时留空 |\n| `tlsCertPath` | string | `${PROXY_HOME}/conf/tls/rocketmq.crt` | 服务端证书链文件路径（X.509 PEM 格式） |\n| `tlsCertWatchIntervalMs` | int | `3600000` | 证书文件变更检测间隔（毫秒） |\n\n### 5.2 配置 Proxy 启动参数\n\n编辑 `runproxy.sh`（或启动 Proxy 的脚本），启用 TLS enforcing 模式：\n\n```shell\nJAVA_OPT=\"${JAVA_OPT} -Dtls.server.mode=enforcing\"\n```\n\n三种 TLS 模式说明：\n- `disabled` - 不支持 TLS，拒绝所有 TLS 握手请求\n- `permissive` - TLS 可选，同时接受 TLS 和非 TLS 连接\n- `enforcing` - 强制 TLS，拒绝非 TLS 连接"
  },
  {
    "path": "docs/cn/Debug_In_Idea.md",
    "content": "## 本地调试RocketMQ\n\n### Step0: 解决依赖问题\n1. 运行前下载RocketMQ需要的maven依赖，可以使用`mvn clean install -Dmaven.test.skip=true`\n2. 确保本地能够编译通过\n\n### Step1: 启动NameServer\n1. NamerServer的启动类在`org.apache.rocketmq.namesrv.NamesrvStartup`\n2. `Idea-Edit Configurations`中添加环境变量 `ROCKETMQ_HOME=<rocketmq仓库目录>`\n![Idea_config_nameserver.png](image/Idea_config_nameserver.png)\n3. 运行NameServer，观察到如下日志输出则启动成功\n```shell\nThe Name Server boot success. serializeType=JSON, address 0.0.0.0:9876\n```\n\n### Step2: 启动Broker\n1. Broker的启动类在`org.apache.rocketmq.broker.BrokerStartup`\n2. 创建`/rocketmq/conf/broker.conf`文件或直接在官方release发布包中拷贝即可\n```shell\n# broker.conf\n\nbrokerClusterName = DefaultCluster\nbrokerName = broker-a\nbrokerId = 0\ndeleteWhen = 04\nfileReservedTime = 48\nbrokerRole = ASYNC_MASTER\nflushDiskType = ASYNC_FLUSH\nnamesrvAddr = 127.0.0.1:9876\n```\n3. `Idea-Edit Configurations`中添加运行参数 `-c /Users/xxx/rocketmq/conf/broker.conf` 以及环境变量 `ROCKETMQ_HOME=<rocketmq仓库目录>`\n![Idea_config_broker.png](image/Idea_config_broker.png)\n4. 运行Broker，观察到如下日志则启动成功\n```shell\nThe broker[broker-a,192.169.1.2:10911] boot success...\n```\n\n### Step3: 发送或消费消息\n至此已经完成了RocketMQ的启动，可以使用`/example`里的示例进行收发消息\n\n### 补充：本地启动Proxy\n1. RocketMQ5.x支持了Proxy模式，使用`LOCAL`模式可以免去`Step2`，启动类在`org.apache.rocketmq.proxy.ProxyStartup`\n2. `Idea-Edit Configurations`中添加环境变量 `ROCKETMQ_HOME=<rocketmq仓库目录>`\n3. 在`/conf/`下新建配置文件`rmq-proxy.json`\n```json\n{\n  \"rocketMQClusterName\": \"DefaultCluster\",\n  \"nameSrvAddr\": \"127.0.0.1:9876\",\n  \"proxyMode\": \"local\"\n}\n```\n4. 运行Proxy，观察到如下日志则启动成功\n```shell\nSat Aug 26 15:29:33 CST 2023 rocketmq-proxy startup successfully\n```"
  },
  {
    "path": "docs/cn/Deployment.md",
    "content": "# 部署架构和设置步骤\n\n## 集群的设置\n\n### 1 单master模式\n\n这是最简单但也是最危险的模式，一旦broker服务器重启或宕机，整个服务将不可用。 建议在生产环境中不要使用这种部署方式，在本地测试和开发可以选择这种模式。 以下是构建的步骤。\n\n**1）启动NameServer**\n\n```shell\n### 第一步启动namesrv\n$ nohup sh mqnamesrv &\n \n### 验证namesrv是否启动成功\n$ tail -f ~/logs/rocketmqlogs/namesrv.log\nThe Name Server boot success...\n```\n\n我们可以在namesrv.log 中看到'The Name Server boot success..'，表示NameServer 已成功启动。\n\n**2）启动Broker**\n\n```shell\n### 第一步先启动broker\n$ nohup sh bin/mqbroker -n localhost:9876 &\n\n### 验证broker是否启动成功，比如，broker的ip是192.168.1.2 然后名字是broker-a\n$ tail -f ~/logs/rocketmqlogs/Broker.log \nThe broker[broker-a,192.169.1.2:10911] boot success...\n```\n\n我们可以在 Broker.log 中看到“The broker[brokerName,ip:port] boot success..”，这表明 broker 已成功启动。\n\n### 2 多Master模式\n\n该模式是指所有节点都是master主节点（比如2个或3个主节点），没有slave从节点的模式。 这种模式的优缺点如下：\n\n- 优点： \n  1. 配置简单。\n  2. 一个master节点的宕机或者重启（维护）对应用程序没有影响。\n  3. 当磁盘配置为RAID10时，消息不会丢失，因为RAID10磁盘非常可靠，即使机器不可恢复（消息异步刷盘模式的情况下，会丢失少量消息；如果消息是同步刷盘模式，不会丢失任何消息）。\n  4. 在这种模式下，性能是最高的。\n- 缺点：\n  1. 单台机器宕机时，本机未消费的消息，直到机器恢复后才会订阅，影响消息实时性。\n\n多Master模式的启动步骤如下：\n\n**1）启动 NameServer**\n\n```shell\n### 第一步先启动namesrv\n$ nohup sh mqnamesrv &\n \n### 验证namesrv是否启动成功\n$ tail -f ~/logs/rocketmqlogs/namesrv.log\nThe Name Server boot success...\n```\n\n**2）启动 Broker 集群**\n\n```shell\n### 比如在A机器上启动第一个Master，假设配置的NameServer IP为：192.168.1.1\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-noslave/broker-a.properties &\n \n### 然后在机器B上启动第二个Master，假设配置的NameServer IP是：192.168.1.1\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-noslave/broker-b.properties &\n\n...\n```\n\n上面显示的启动命令用于单个NameServer的情况。对于多个NameServer的集群，broker 启动命令中-n参数后面的地址列表用分号隔开，例如 192.168.1.1:9876;192.168.1.2:9876\n\n### 3 多Master多Slave模式-异步复制\n\n每个主节点配置多个从节点，多对主从。HA采用异步复制，主节点和从节点之间有短消息延迟（毫秒）。这种模式的优缺点如下：\n\n- 优点：\n  1. 即使磁盘损坏，也只会丢失极少的消息，不影响消息的实时性能。\n  2. 同时，当主节点宕机时，消费者仍然可以消费从节点的消息，这个过程对应用本身是透明的，不需要人为干预。\n  3. 性能几乎与多Master模式一样高。\n- 缺点：\n  1. 主节点宕机、磁盘损坏时，会丢失少量消息。\n\n多主多从模式的启动步骤如下：\n\n**1）启动 NameServer**\n\n```shell\n### 第一步先启动namesrv\n$ nohup sh mqnamesrv &\n \n### 验证namesrv是否启动成功\n$ tail -f ~/logs/rocketmqlogs/namesrv.log\nThe Name Server boot success...\n```\n\n**2）启动 Broker 集群**\n\n```shell\n### 例如在A机器上启动第一个Master，假设配置的NameServer IP为：192.168.1.1，端口为9876。\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-a.properties &\n \n### 然后在机器B上启动第二个Master，假设配置的NameServer IP为：192.168.1.1，端口为9876。\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-b.properties &\n \n### 然后在C机器上启动第一个Slave，假设配置的NameServer IP为：192.168.1.1，端口为9876。\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-a-s.properties &\n \n### 最后在D机启动第二个Slave，假设配置的NameServer IP为：192.168.1.1，端口为9876。\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-b-s.properties &\n```\n\n上图显示了 2M-2S-Async 模式的启动命令，类似于其他 nM-nS-Async 模式。\n\n### 4 多Master多Slave模式-同步双写\n\n这种模式下，每个master节点配置多个slave节点，有多对Master-Slave。HA采用同步双写，即只有消息成功写入到主节点并复制到多个从节点，才会返回成功响应给应用程序。\n\n这种模式的优缺点如下：\n\n- 优点： \n  1. 数据和服务都没有单点故障。\n  2. 在master节点关闭的情况下，消息也没有延迟。\n  3. 服务可用性和数据可用性非常高；\n- 缺点：\n  1. 这种模式下的性能略低于异步复制模式（大约低 10%）。\n  2. 发送单条消息的RT略高，目前版本，master节点宕机后，slave节点无法自动切换到master。\n\n启动步骤如下：\n\n**1）启动NameServer**\n\n```shell\n### 第一步启动namesrv\n$ nohup sh mqnamesrv &\n \n### 验证namesrv是否启动成功\n$ tail -f ~/logs/rocketmqlogs/namesrv.log\nThe Name Server boot success...\n```\n\n**2）启动 Broker 集群**\n\n```shell\n### 例如在A机器上启动第一个Master，假设配置的NameServer IP为：192.168.1.1，端口为9876。\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-a.properties &\n \n### 然后在B机器上启动第二个Master，假设配置的NameServer IP为：192.168.1.1，端口为9876。\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-b.properties &\n \n### 然后在C机器上启动第一个Slave，假设配置的NameServer IP为：192.168.1.1，端口为9876。\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-a-s.properties &\n \n### 最后在D机启动第二个Slave，假设配置的NameServer IP为：192.168.1.1，端口为9876。\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-b-s.properties &\n```\n\n上述Master和Slave是通过指定相同的config命名为“brokerName”来配对的，master节点的brokerId必须为0，slave节点的brokerId必须大于0。\n\n### 5 RocketMQ 5.0 自动主从切换\n\nRocketMQ 5.0 开始支持自动主从切换的模式，可参考以下文档\n\n[快速开始](controller/quick_start.md)\n\n[部署文档](controller/deploy.md)\n\n[设计思想](controller/design.md)\n\n"
  },
  {
    "path": "docs/cn/Example_Batch.md",
    "content": "# 批量消息发送\n批量消息发送能够提高发送效率，提升系统吞吐量。同一批批量消息的topic、waitStoreMsgOK属性必须保持一致，批量消息不支持延迟消息。批量消息发送一次最多可以发送 4MiB 的消息，但是如果需要发送更大的消息，建议将较大的消息分成多个不超过 1MiB 的小消息。\n\n### 1 发送批量消息\n如果你一次只发送不超过 4MiB 的消息，使用批处理很容易：\n```java\nString topic = \"BatchTest\";\nList<Message> messages = new ArrayList<>();\nmessages.add(new Message(topic, \"TagA\", \"OrderID001\", \"Hello world 0\".getBytes()));\nmessages.add(new Message(topic, \"TagA\", \"OrderID002\", \"Hello world 1\".getBytes()));\nmessages.add(new Message(topic, \"TagA\", \"OrderID003\", \"Hello world 2\".getBytes()));\ntry {\n    producer.send(messages);\n} catch (Exception e) {\n    e.printStackTrace();\n    //handle the error\n}\n```\n### 2 拆分\n当您发送较大的消息时，复杂性会增加，如果您不确定它是否超过 4MiB的限制。 这时候，您最好将较大的消息分成多个不超过 1MiB 的小消息：\n\n```java\npublic class ListSplitter implements Iterator<List<Message>> { \n    private final int SIZE_LIMIT = 1024 * 1024 * 4;\n    private final List<Message> messages;\n    private int currIndex;\n    public ListSplitter(List<Message> messages) { \n        this.messages = messages;\n    }\n    @Override \n    public boolean hasNext() {\n        return currIndex < messages.size(); \n    }\n    @Override \n    public List<Message> next() { \n        int startIndex = getStartIndex();\n        int nextIndex = startIndex;\n        int totalSize = 0;\n        for (; nextIndex < messages.size(); nextIndex++) {\n            Message message = messages.get(nextIndex); \n            int tmpSize = calcMessageSize(message);\n            if (tmpSize + totalSize > SIZE_LIMIT) {\n                break; \n            } else {\n                totalSize += tmpSize; \n            }\n        }\n        List<Message> subList = messages.subList(startIndex, nextIndex); \n        currIndex = nextIndex;\n        return subList;\n    }\n    private int getStartIndex() {\n        Message currMessage = messages.get(currIndex); \n        int tmpSize = calcMessageSize(currMessage); \n        while(tmpSize > SIZE_LIMIT) {\n            currIndex += 1;\n            Message message = messages.get(curIndex); \n            tmpSize = calcMessageSize(message);\n        }\n        return currIndex; \n    }\n    private int calcMessageSize(Message message) {\n        int tmpSize = message.getTopic().length() + message.getBody().length(); \n        Map<String, String> properties = message.getProperties();\n        for (Map.Entry<String, String> entry : properties.entrySet()) {\n            tmpSize += entry.getKey().length() + entry.getValue().length(); \n        }\n        tmpSize = tmpSize + 20; // Increase the log overhead by 20 bytes\n        return tmpSize; \n    }\n}\n\n// then you could split the large list into small ones:\nListSplitter splitter = new ListSplitter(messages);\nwhile (splitter.hasNext()) {\n   try {\n       List<Message>  listItem = splitter.next();\n       producer.send(listItem);\n   } catch (Exception e) {\n       e.printStackTrace();\n       // handle the error\n   }\n}\n```"
  },
  {
    "path": "docs/cn/Example_Compaction_Topic_cn.md",
    "content": "# Compaction Topic\n\n## 使用方式\n\n### 打开namesrv上支持顺序消息的开关\nCompactionTopic依赖顺序消息来保障一致性\n```shell\n$ bin/mqadmin updateNamesrvConfig -k orderMessageEnable -v true\n```\n\n### 创建compaction topic\n\n```shell\n$ bin/mqadmin updateTopic -w 8 -r 8 -a +cleanup.policy=COMPACTION -n localhost:9876 -t ctopic -o true -c DefaultCluster\ncreate topic to 127.0.0.1:10911 success.\nTopicConfig [topicName=ctopic, readQueueNums=8, writeQueueNums=8, perm=RW-, topicFilterType=SINGLE_TAG, topicSysFlag=0, order=false, attributes={+cleanup.policy=COMPACTION}]\n```\n\n### 生产数据\n\n与普通消息一样\n\n```java\nDefaultMQProducer producer = new DefaultMQProducer(\"CompactionTestGroup\");\nproducer.setNamesrvAddr(\"localhost:9876\");\nproducer.start();\n\nString topic = \"ctopic\";\nString tag = \"tag1\";\nString key = \"key1\";\nMessage msg = new Message(topic, tag, key, \"bodys\".getBytes(StandardCharsets.UTF_8));\nSendResult sendResult = producer.send(msg, (mqs, message, shardingKey) -> {\n    int select = Math.abs(shardingKey.hashCode());\n    if (select < 0) {\n        select = 0;\n    }\n    return mqs.get(select % mqs.size());\n}, key);\n\nSystem.out.printf(\"%s%n\", sendResult);\n``` \n\n### 消费数据\n\n消费offset与compaction之前保持不变，如果指定offset消费，当指定的offset不存在时，返回后面最近的一条数据\n在compaction场景下，大部分消费都是从0开始消费完整的数据\n\n```java\nDefaultLitePullConsumer consumer = new DefaultLitePullConsumer(\"compactionTestGroup\");\nconsumer.setNamesrvAddr(\"localhost:9876\");\nconsumer.setPullThreadNums(4);\nconsumer.start();\n\nCollection<MessageQueue> messageQueueList = consumer.fetchMessageQueues(\"ctopic\");\nconsumer.assign(messageQueueList);\nmessageQueueList.forEach(mq -> {\n    try {\n        consumer.seekToBegin(mq);\n    } catch (MQClientException e) {\n        e.printStackTrace();\n    }\n});\n\nMap<String, byte[]> kvStore = Maps.newHashMap();\nwhile (true) {\n    List<MessageExt> msgList = consumer.poll(1000);\n    if (CollectionUtils.isNotEmpty(msgList)) {\n        msgList.forEach(msg -> kvStore.put(msg.getKeys(), msg.getBody()));\n    }\n}\n\n//use the kvStore\n```"
  },
  {
    "path": "docs/cn/Example_CreateTopic.md",
    "content": "# 创建主题\n\n## 背景\n\nRocketMQ 5.0 引入了 `TopicMessageType` 的概念，并且使用了现有的主题属性功能来实现它。\n\n主题的创建是通过 `mqadmin` 工具来申明 `message.type` 属性。\n\n## 使用案例\n\n```shell\n# default\nsh ./mqadmin updateTopic -n <nameserver_address> -t <topic_name> -c DefaultCluster\n\n# normal topic\nsh ./mqadmin updateTopic -n <nameserver_address> -t <topic_name> -c DefaultCluster -a +message.type=NORMAL\n\n# fifo topic\nsh ./mqadmin updateTopic -n <nameserver_address> -t <topic_name> -c DefaultCluster -a +message.type=FIFO\n\n# delay topic\nsh ./mqadmin updateTopic -n <nameserver_address> -t <topic_name> -c DefaultCluster -a +message.type=DELAY\n\n# transaction topic\nsh ./mqadmin updateTopic -n <nameserver_address> -t <topic_name> -c DefaultCluster -a +message.type=TRANSACTION\n```\n"
  },
  {
    "path": "docs/cn/Example_Delay.md",
    "content": "# Schedule example\n\n### 1 启动消费者等待传入的订阅消息\n\n```java\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport java.util.List;\n\npublic class ScheduledMessageConsumer {\n\n    public static void main(String[] args) throws Exception {\n        // Instantiate message consumer\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"ExampleConsumer\");\n        // Subscribe topics\n        consumer.subscribe(\"TestTopic\", \"*\");\n        // Register message listener\n        consumer.registerMessageListener(new MessageListenerConcurrently() {\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> messages, ConsumeConcurrentlyContext context) {\n                for (MessageExt message : messages) {\n                    // Print approximate delay time period\n                    System.out.println(\"Receive message[msgId=\" + message.getMsgId() + \"] \"\n                                       + (System.currentTimeMillis() - message.getStoreTimestamp()) + \"ms later\");\n                }\n                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n            }\n        });\n        // Launch consumer\n        consumer.start();\n    }\n}\n```\n\n### 2 发送延迟消息 \n\n```java\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.common.message.Message;\n\npublic class ScheduledMessageProducer {\n\n    public static void main(String[] args) throws Exception {\n        // Instantiate a producer to send scheduled messages\n        DefaultMQProducer producer = new DefaultMQProducer(\"ExampleProducerGroup\");\n        // Launch producer\n        producer.start();\n        int totalMessagesToSend = 100;\n        for (int i = 0; i < totalMessagesToSend; i++) {\n            Message message = new Message(\"TestTopic\", (\"Hello scheduled message \" + i).getBytes());\n            // This message will be delivered to consumer 10 seconds later.\n            message.setDelayTimeLevel(3);\n            // Send the message\n            producer.send(message);\n        }\n\n        // Shutdown producer after use.\n        producer.shutdown();\n    }\n\n}\n```\n\n### 3 确认\n\n您应该会看到消息在其存储时间后大约 10 秒被消耗。\n\n### 4 延迟消息的使用场景\n\n例如在电子商务中，如果提交订单，可以发送延迟消息，1小时后可以查看订单状态。 如果订单仍未付款，则可以取消订单并释放库存。\n\n### 5 使用延迟消息的限制\n\n```java \n// org/apache/rocketmq/store/config/MessageStoreConfig.java\n\nprivate String messageDelayLevel = \"1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h\";\n```\n\n当前 RocketMQ 不支持任意时间的延迟。 生产者发送延迟消息前需要设置几个固定的延迟级别，分别对应1s到2h的1到18个延迟级，消息消费失败会进入延迟消息队列，消息发送时间与设置的延迟级别和重试次数有关。\n\n See `SendMessageProcessor.java` \n"
  },
  {
    "path": "docs/cn/Example_LMQ.md",
    "content": "# Light message queue (LMQ)\nLMQ采用的读放大的策略，写一份数据，多个LMQ队列分发，\n因为存储的成本和效率对用户的体感最明显。写多份不仅加大了存储成本，同时也对性能和数据准确一致性提出了挑战。\n\n![](image/LMQ_1.png)\n\n上图描述的是LMQ的队列存储模型，消息可以来自各个接入场景\n（如服务端的MQ/AMQP，客户端的MQTT），但只会写一份存到commitlog里面，然后分发出多个需求场景的队列索引（ConsumerQueue），如服务端场景（MQ/AMQP）可以按照一级Topic队列进行传统的服务端消费，客户端MQTT场景可以按照MQTT多级Topic(也即 LMQ)进行消费消息。\n\n## 一、broker启动配置\n\n\nbroker.conf文件需要增加以下的配置项，开启LMQ开关，这样就可以识别LMQ相关属性的消息，进行原子分发消息到LMQ队列\n```properties\nenableLmq = true\nenableMultiDispatch = true\n```\n## 二、发送消息\n发送消息的时候通过设置 INNER_MULTI_DISPATCH 属性，LMQ queue使用逗号分割，queue前缀必须是 %LMQ%，这样broker就可以识别LMQ queue.\n以下代码只是demo伪代码 具体逻辑参照执行即可\n```java\nDefaultMQProducer producer = new DefaultMQProducer(\"please_rename_unique_group_name\");\nproducer.setNamesrvAddr(\"name-server1-ip:9876;name-server2-ip:9876\");\nproducer.start();\n\n\n/*\n* Create a message instance, specifying topic, tag and message body.\n*/\nMessage msg = new Message(\"TopicTest\" /* Topic */,\n                          \"TagA\" /* Tag */,\n                          (\"Hello RocketMQ \" + i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */\n                         );\n/*\n* INNER_MULTI_DISPATCH property and PREFIX must start as \"%LMQ%\",\n* If it is multiple LMQ, need to use “,” split\n*/\nmessage.putUserProperty(\"INNER_MULTI_DISPATCH\", \"%LMQ%123,%LMQ%456\");\n/*\n* Call send message to deliver message to one of brokers.\n*/\nSendResult sendResult = producer.send(msg);\n```\n## 三、拉取消息\nLMQ queue在每个broker上只有一个queue，也即queueId为0， 指明轻量级的MessageQueue，就可以拉取消息进行消费。\n以下代码只是demo伪代码 具体逻辑参照执行即可\n```java\nDefaultMQPullConsumer defaultMQPullConsumer = new DefaultMQPullConsumer();\ndefaultMQPullConsumer.setNamesrvAddr(\"name-server1-ip:9876;name-server2-ip:9876\");\ndefaultMQPullConsumer.setVipChannelEnabled(false);\ndefaultMQPullConsumer.setConsumerGroup(\"CID_RMQ_SYS_LMQ_TEST\");\ndefaultMQPullConsumer.setInstanceName(\"CID_RMQ_SYS_LMQ_TEST\");\ndefaultMQPullConsumer.setRegisterTopics(new HashSet<>(Arrays.asList(\"TopicTest\")));\ndefaultMQPullConsumer.setBrokerSuspendMaxTimeMillis(2000);\ndefaultMQPullConsumer.setConsumerTimeoutMillisWhenSuspend(3000);\ndefaultMQPullConsumer.start();\n\nString brokerName = \"set broker Name\";\nMessageQueue mq = new MessageQueue(\"%LMQ%123\", brokerName, 0);\ndefaultMQPullConsumer.getDefaultMQPullConsumerImpl().getRebalanceImpl().getmQClientFactory().updateTopicRouteInfoFromNameServer(\"TopicTest\");\n\nThread.sleep(30000);\nLong offset = defaultMQPullConsumer.maxOffset(mq);\n\ndefaultMQPullConsumer.pullBlockIfNotFound(\n                mq, \"*\", offset, 32,\n                new PullCallback() {\n                    @Override\n                    public void onSuccess(PullResult pullResult) {\n                        List<MessageExt> list = pullResult.getMsgFoundList();\n                        if (list == null || list.isEmpty()) {\n                            return;\n                        }\n                        for (MessageExt messageExt : list) {\n                            System.out.println(messageExt);\n                        }    \n                    }\n                    @Override\n                    public void onException(Throwable e) {\n                       \n                    }\n});\n```\n​\n\n"
  },
  {
    "path": "docs/cn/Example_Simple_cn.md",
    "content": "# Basic Sample \n------\n基本示例中提供了以下两个功能\n* RocketMQ可用于以三种方式发送消息：可靠的同步、可靠的异步和单向传输。前两种消息类型是可靠的，因为无论它们是否成功发送都有响应。\n* RocketMQ可以用来消费消息。\n### 1 添加依赖\nmaven:\n``` java\n<dependency>\n  <groupId>org.apache.rocketmq</groupId>\n  <artifactId>rocketmq-client</artifactId>\n  <version>4.3.0</version>\n</dependency>\n```\ngradle: \n``` java \ncompile 'org.apache.rocketmq:rocketmq-client:4.3.0'\n```\n### 2 发送消息\n##### 2.1 使用Producer发送同步消息\n可靠的同步传输被广泛应用于各种场景，如重要的通知消息、短消息通知等。\n``` java\npublic class SyncProducer {\n  public static void main(String[] args) throws Exception {\n    // Instantiate with a producer group name\n    DefaultMQProducer producer = new DefaultMQProducer(\"please_rename_unique_group_name\");\n    // Specify name server addresses\n    producer.setNamesrvAddr(\"localhost:9876\");\n    // Launch the producer instance\n    producer.start();\n    for (int i = 0; i < 100; i++) {\n      // Create a message instance with specifying topic, tag and message body\n      Message msg = new Message(\"TopicTest\" /* Topic */,\n        \"TagA\" /* Tag */,\n        (\"Hello RocketMQ \" + i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */\n        );\n      // Send message to one of brokers\n      SendResult sendResult = producer.send(msg);\n      // Check whether the message has been delivered by the callback of sendResult\n      System.out.printf(\"%s%n\", sendResult);\n    }\n    // Shut down once the producer instance is not longer in use\n    producer.shutdown();\n  }\n}\n```\n##### 2.2 发送异步消息\n异步传输通常用于响应时间敏感的业务场景。这意味着发送方无法等待代理的响应太长时间。\n``` java\npublic class AsyncProducer {\n  public static void main(String[] args) throws Exception {\n    // Instantiate with a producer group name\n    DefaultMQProducer producer = new DefaultMQProducer(\"please_rename_unique_group_name\");\n    // Specify name server addresses\n    producer.setNamesrvAddr(\"localhost:9876\");\n    // Launch the producer instance\n    producer.start();\n    producer.setRetryTimesWhenSendAsyncFailed(0);\n    for (int i = 0; i < 100; i++) {\n      final int index = i;\n      // Create a message instance with specifying topic, tag and message body\n      Message msg = new Message(\"TopicTest\",\n        \"TagA\",\n        \"OrderID188\",\n        \"Hello world\".getBytes(RemotingHelper.DEFAULT_CHARSET));\n      // SendCallback: receive the callback of the asynchronous return result.\n      producer.send(msg, new SendCallback() {\n        @Override\n        public void onSuccess(SendResult sendResult) {\n          System.out.printf(\"%-10d OK %s %n\", index,\n            sendResult.getMsgId());\n        }\n        @Override\n        public void onException(Throwable e) {\n          System.out.printf(\"%-10d Exception %s %n\", index, e);\n          e.printStackTrace();\n        }\n      });\n    }\n    // Shut down once the producer instance is not longer in use\n    producer.shutdown();\n  }\n}\n```\n##### 2.3 以单向模式发送消息\n单向传输用于需要中等可靠性的情况，如日志收集。\n``` java\npublic class OnewayProducer {\n  public static void main(String[] args) throws Exception{\n    // Instantiate with a producer group name\n    DefaultMQProducer producer = new DefaultMQProducer(\"please_rename_unique_group_name\");\n    // Specify name server addresses\n    producer.setNamesrvAddr(\"localhost:9876\");\n    // Launch the producer instance\n    producer.start();\n    for (int i = 0; i < 100; i++) {\n      // Create a message instance with specifying topic, tag and message body\n      Message msg = new Message(\"TopicTest\" /* Topic */,\n        \"TagA\" /* Tag */,\n        (\"Hello RocketMQ \" + i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */\n      );\n      // Send in one-way mode, no return result\n      producer.sendOneway(msg);\n    }\n    // Shut down once the producer instance is not longer in use\n     producer.shutdown();\n  }\n}\n```\n### 3 消费消息\n``` java\npublic class Consumer {\n  public static void main(String[] args) throws InterruptedException, MQClientException {\n    // Instantiate with specified consumer group name\n    DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"please_rename_unique_group_name\");\n    \n    // Specify name server addresses\n    consumer.setNamesrvAddr(\"localhost:9876\");\n\n    // Subscribe one or more topics and tags for finding those messages need to be consumed\n    consumer.subscribe(\"TopicTest\", \"*\");\n    // Register callback to execute on arrival of messages fetched from brokers\n    consumer.registerMessageListener(new MessageListenerConcurrently() {\n      @Override\n      public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {\n        System.out.printf(\"%s Receive New Messages: %s %n\", Thread.currentThread().getName(), msgs);\n        // Mark the message that have been consumed successfully\n        return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n      }\n    });\n    // Launch the consumer instance\n    consumer.start();\n    System.out.printf(\"Consumer Started.%n\");\n  }\n}\n```"
  },
  {
    "path": "docs/cn/FAQ.md",
    "content": "# 经常被问到的问题\n\n以下是关于RocketMQ项目的常见问题\n\n## 1 基本\n\n1. **为什么我们要使用RocketMQ而不是选择其他的产品？**\n\n   请参考[为什么要选择RocketMQ](http://rocketmq.apache.org/docs/motivation/)\n\n2. **我是否需要安装其他的软件才能使用RocketMQ，例如zookeeper？**\n\n   不需要，RocketMQ可以独立的运行。\n\n## 2 使用\n\n1. **新创建的Consumer ID从哪里开始消费消息？**\n\n   1）如果发送的消息在三天之内，那么消费者会从服务器中保存的第一条消息开始消费。\n\n   2）如果发送的消息已经超过三天，则消费者会从服务器中的最新消息开始消费，也就是从队列的尾部开始消费。\n\n   3）如果消费者重新启动，那么它会从最后一个消费位置开始消费消息。\n\n2. **当消费失败的时候如何重新消费消息？**\n\n   1）在集群模式下，消费的业务逻辑代码会返回Action.ReconsumerLater，NULL，或者抛出异常，如果一条消息消费失败，最多会重试16次，之后该消息会被丢弃。\n\n   2）在广播消费模式下，广播消费仍然保证消息至少被消费一次，但不提供重发的选项。\n\n3. **当消费失败的时候如何找到失败的消息？**\n\n   1）使用按时间的主题查询，可以查询到一段时间内的消息。\n\n   2）使用主题和消息ID来准确查询消息。\n\n   3）使用主题和消息的Key来准确查询所有消息Key相同的消息。\n\n4. **消息只会被传递一次吗？**\n\n   RocketMQ 确保所有消息至少传递一次。 在大多数情况下，消息不会重复。\n\n5. **如何增加一个新的Broker？**\n\n   1）启动一个新的Broker并将其注册到name server中的Broker列表里。\n\n   2）默认只自动创建内部系统topic和consumer group。 如果您希望在新节点上拥有您的业务主题和消费者组，请从现有的Broker中复制它们。 我们提供了管理工具和命令行来处理此问题。\n\n## 3 配置相关\n\n以下回答均为默认值，可通过配置修改。\n\n1. **消息在服务器上可以保存多长时间？**\n\n   存储的消息将最多保存 3 天，超过 3 天未使用的消息将被删除。\n\n2. **消息体的大小限制是多少？**\n\n   通常是256KB\n\n3. **怎么设置消费者线程数？**\n\n   当你启动消费者的时候，可以设置 ConsumeThreadNums属性的值，举例如下：\n\n   ```java\n   consumer.setConsumeThreadMin(20);\n   consumer.setConsumeThreadMax(20);\n   ```\n\n## 4 错误\n\n1. **当你启动一个生产者或消费者的过程失败了并且错误信息是生产者组或消费者重复**\n\n   原因：使用同一个Producer/Consumer Group在同一个JVM中启动多个Producer/Consumer实例可能会导致客户端无法启动。\n\n   解决方案：确保一个 Producer/Consumer Group 对应的 JVM 只启动一个 Producer/Consumer 实例。\n\n2. **消费者无法在广播模式下开始加载 json 文件**\n\n   原因：fastjson 版本太低，无法让广播消费者加载本地 offsets.json，导致消费者启动失败。 损坏的 fastjson 文件也会导致同样的问题。\n\n   解决方案：Fastjson 版本必须升级到 RocketMQ 客户端依赖版本，以确保可以加载本地 offsets.json。 默认情况下，offsets.json 文件在 /home/{user}/.rocketmq_offsets 中。 或者检查fastjson的完整性。\n\n3. **Broker崩溃以后有什么影响？**\n\n   1）Master节点崩溃\n\n   消息不能再发送到该Broker集群，但是如果您有另一个可用的Broker集群，那么在主题存在的条件下仍然可以发送消息。消息仍然可以从Slave节点消费。\n\n   2）一些Slave节点崩溃\n\n   只要有另一个工作的slave，就不会影响发送消息。 对消费消息也不会产生影响，除非消费者组设置为优先从该Slave消费。 默认情况下，消费者组从 master 消费。\n\n   3）所有Slave节点崩溃\n\n   向master发送消息不会有任何影响，但是，如果master是SYNC_MASTER，producer会得到一个SLAVE_NOT_AVAILABLE，表示消息没有发送给任何slave。 对消费消息也没有影响，除非消费者组设置为优先从slave消费。 默认情况下，消费者组从master消费。\n\n4. **Producer提示“No Topic Route Info”，如何诊断？**\n\n   当您尝试将消息发送到一个路由信息对生产者不可用的主题时，就会发生这种情况。\n\n   1）确保生产者可以连接到名称服务器并且能够从中获取路由元信息。\n\n   2）确保名称服务器确实包含主题的路由元信息。 您可以使用管理工具或 Web 控制台通过 topicRoute 从名称服务器查询路由元信息。\n\n   3）确保您的Broker将心跳发送到您的生产者正在连接的同一name server列表。\n\n   4）确保主题的权限为6(rw-)，或至少为2(-w-)。\n\n   如果找不到此主题，请通过管理工具命令updateTopic或Web控制台在Broker上创建它。"
  },
  {
    "path": "docs/cn/QuorumACK.md",
    "content": "# Quorum Write和自动降级\n\n## 背景\n\n![](https://s4.ax1x.com/2022/02/05/HnWo2d.png)\n\n在RocketMQ中，主备之间的复制模式主要有同步复制和异步复制，如上图所示，Slave1的复制是同步的，在向Producer报告成功写入之前，Master需要等待Slave1成功复制该消息并确认，Slave2的复制是异步的，Master不需要等待Slave2的响应。在RocketMQ中，发送一条消息，如果一切都顺利，那最后会返回给Producer客户端一个PUT_OK的状态，如果是Slave同步超时则返回FLUSH_SLAVE_TIMEOUT状态，如果是Slave不可用或者Slave与Master之间CommitLog差距超过一定的值（默认是256MB），则返回SLAVE_NOT_AVAILABLE，后面两个状态并不会导致系统异常而无法写入下一条消息。\n\n同步复制可以保证Master失效后，数据仍然能在Slave中找到，适合可靠性要求较高的场景。异步复制虽然消息可能会丢失，但是由于无需等待Slave的确认，效率上要高于同步复制，适合对效率有一定要求的场景。但是只有两种模式仍然不够灵活，比如在三副本甚至五副本且对可靠性要求高场景中，采用异步复制无法满足需求，但采用同步复制则需要每一个副本确认后才会返回，在副本数多的情况下严重影响效率。另一方面，在同步复制的模式下，如果副本组中的某一个Slave出现假死，整个发送将一直失败直到进行手动处理。\n\n因此，RocketMQ 5 提出了副本组的quorum write，在同步复制的模式下，用户可以在broker端指定发送后至少需要写入多少副本数后才能返回，并且提供自适应降级的方式，可以根据存活的副本数以及CommitLog差距自动完成降级。\n\n## 架构和参数\n\n### Quorum Write\n\n通过增加两个参数来支持quorum write。\n\n- **totalReplicas**：副本组broker总数。默认为1。\n- **inSyncReplicas**：正常情况需保持同步的副本组数量。默认为1。\n\n通过这两个参数，可以在同步复制的模式下，灵活指定需要ACK的副本数。\n\n![](https://s4.ax1x.com/2022/02/05/HnWHKI.png)\n\n如上图所示，在两副本情况下，如果inSyncReplicas为2，则该条消息需要在Master和Slave中均复制完成后才会返回给客户端；在三副本情况下，如果inSyncReplicas为2，则该条消息除了需要复制在Master上，还需要复制到任意一个slave上，才会返回给客户端。在四副本情况下，如果inSyncReplicas为3，则条消息除了需要复制在Master上，还需要复制到任意两个slave上，才会返回给客户端。通过灵活设置totalReplicas和inSyncReplicas，可以满足用户各类场景的需求。\n\n### 自动降级\n\n自动降级的标准是\n\n- 当前副本组的存活副本数\n- Master Commitlog和Slave CommitLog的高度差\n\n> **注意：自动降级只在slaveActingMaster模式开启后才生效**\n\n通过Nameserver的反向通知以及GetBrokerMemberGroup请求可以获取当前副本组的存活信息，而Master与Slave的Commitlog高度差也可以通过HA服务中的位点记录计算出来。将增加以下参数完成自动降级：\n\n- **minInSyncReplicas**：最小需保持同步的副本组数量，仅在enableAutoInSyncReplicas为true时生效，默认为1\n- **enableAutoInSyncReplicas**：自动同步降级开关，开启后，若当前副本组处于同步状态的broker数量（包括master自身）不满足inSyncReplicas指定的数量，则按照minInSyncReplicas进行同步。同步状态判断条件为：slave commitLog落后master长度不超过haSlaveFallBehindMax。默认为false。\n- **haMaxGapNotInSync**：slave是否与master处于in-sync状态的判断值，slave commitLog落后master长度超过该值则认为slave已处于非同步状态。当enableAutoInSyncReplicas打开时，该值越小，越容易触发master的自动降级，当enableAutoInSyncReplicas关闭，且totalReplicas==inSyncReplicas时，该值越小，越容易导致在大流量时发送请求失败，故在该情况下可适当调大haMaxGapNotInSync。默认为256K。\n\n注意：在RocketMQ 4.x中存在haSlaveFallbehindMax参数，默认256MB，表明Slave与Master的CommitLog高度差多少后判定其为不可用，在[RIP-34](https://github.com/apache/rocketmq/wiki/RIP-34-Support-quorum-write-and-adaptive-degradation-in-master-slave-architecture)中该参数被取消。\n\n```java\n//计算needAckNums\nint inSyncReplicas = Math.min(this.defaultMessageStore.getAliveReplicaNumInGroup(),\n                              this.defaultMessageStore.getHaService().inSyncSlaveNums(currOffset) + 1);\nneedAckNums = calcNeedAckNums(inSyncReplicas);\nif (needAckNums > inSyncReplicas) {\n    // Tell the producer, don't have enough slaves to handle the send request\n    return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH, null));\n}\n\nprivate int calcNeedAckNums(int inSyncReplicas) {\n    int needAckNums = this.defaultMessageStore.getMessageStoreConfig().getInSyncReplicas();\n    if (this.defaultMessageStore.getMessageStoreConfig().isEnableAutoInSyncReplicas()) {\n        needAckNums = Math.min(needAckNums, inSyncReplicas);\n        needAckNums = Math.max(needAckNums, this.defaultMessageStore.getMessageStoreConfig().getMinInSyncReplicas());\n    }\n    return needAckNums;\n}\n```\n\n当enableAutoInSyncReplicas=true是开启自适应降级模式，当副本组中存活的副本数减少或Master和Slave Commitlog高度差过大时，都会进行自动降级，最小降级到minInSyncReplicas副本数。比如在两副本中，如果设置totalReplicas=2，InSyncReplicas=2，minInSyncReplicas=1，enableAutoInSyncReplicas=true，正常情况下，两个副本均会处于同步复制，当Slave下线或假死时，会进行自适应降级，producer只需要发送到master即成功。\n\n## 兼容性\n\n用户需要设置正确的参数才能完成正确的向后兼容。举个例子，假设用户原集群为两副本同步复制，在没有修改任何参数的情况下，升级到RocketMQ 5的版本，由于totalReplicas、inSyncReplicas默认都为1，将降级为异步复制，如果需要和以前行为保持一致，则需要将totalReplicas和inSyncReplicas均设置为2。\n\n参考文档：\n\n- [RIP-34](https://github.com/apache/rocketmq/wiki/RIP-34-Support-quorum-write-and-adaptive-degradation-in-master-slave-architecture)"
  },
  {
    "path": "docs/cn/README.md",
    "content": "Apache RocketMQ开发者指南\n--------\n\n##### 这个开发者指南旨在帮助您快速了解并使用 Apache RocketMQ\n\n### 1. 概念和特性\n\n- [概念(Concept)](concept.md)：介绍RocketMQ的基本概念模型。\n\n- [特性(Features)](features.md)：介绍RocketMQ实现的功能特性。 \n\n\n### 2. 架构设计\n\n- [架构(Architecture)](architecture.md)：介绍RocketMQ部署架构和技术架构。\n\n- [设计(Design)](design.md)：介绍RocketMQ关键机制的设计原理，主要包括消息存储、通信机制、消息过滤、负载均衡、事务消息等。\n\n\n### 3. 样例\n\n- [样例(Example)](RocketMQ_Example.md) ：介绍RocketMQ的常见用法，包括基本样例、顺序消息样例、延时消息样例、批量消息样例、过滤消息样例、事务消息样例等。\n\n### 4. 最佳实践\n- [最佳实践（Best Practice）](best_practice.md)：介绍RocketMQ的最佳实践，包括生产者、消费者、Broker以及NameServer的最佳实践，客户端的配置方式以及JVM和linux的最佳参数配置。\n- [消息轨迹指南(Message Trace)](msg_trace/user_guide.md)：介绍RocketMQ消息轨迹的使用方法。\n- [权限管理(Auth Management)](acl/user_guide.md)：介绍如何快速部署和使用支持权限控制特性的RocketMQ集群。\n- [自动主从切换快速开始](controller/quick_start.md)：RocketMQ 5.0 自动主从切换快速开始。\n- [自动主从切换部署升级指南](controller/deploy.md)：RocketMQ 5.0 自动主从切换部署升级指南。\n- [Proxy 部署指南](proxy/deploy_guide.md)：介绍如何部署Proxy (包括 `Local` 模式和 `Cluster` 模式).\n\n### 5. 运维管理\n- [集群部署(Operation)](operation.md)：介绍单Master模式、多Master模式、多Master多slave模式等RocketMQ集群各种形式的部署方法以及运维工具mqadmin的使用方式。\n\n### 6. RocketMQ 5.0 新特性\n\n- [POP消费](https://github.com/apache/rocketmq/wiki/%5BRIP-19%5D-Server-side-rebalance,--lightweight-consumer-client-support)\n- [StaticTopic](statictopic/RocketMQ_Static_Topic_Logic_Queue_设计.md)\n- [BatchConsumeQueue](https://github.com/apache/rocketmq/wiki/RIP-26-Improve-Batch-Message-Processing-Throughput)\n- [自动主从切换](controller/design.md)\n- [BrokerContainer](BrokerContainer.md)\n- [SlaveActingMaster模式](SlaveActingMasterMode.md)\n- [Grpc Proxy](../../proxy/README.md)\n\n### 7. API Reference（待补充）\n\n- [DefaultMQProducer API Reference](client/java/API_Reference_DefaultMQProducer.md)\n\n\n\n\n\n\n\n"
  },
  {
    "path": "docs/cn/RocketMQ_Example.md",
    "content": "# 样例\n-----\n * [目录](#样例)\n      * [1 基本样例](#1-基本样例)\n         * [1.1 加入依赖：](#11-加入依赖)\n         * [1.2 消息发送](#12-消息发送)\n            * [1、Producer端发送同步消息](#1producer端发送同步消息)\n            * [2、发送异步消息](#2发送异步消息)\n            * [3、单向发送消息](#3单向发送消息)\n         * [1.3 消费消息](#13-消费消息)\n      * [2 顺序消息样例](#2-顺序消息样例)\n         * [2.1 顺序消息生产](#21-顺序消息生产)\n         * [2.2 顺序消费消息](#22-顺序消费消息)\n      * [3 延时消息样例](#3-延时消息样例)\n         * [3.1 启动消费者等待传入订阅消息](#31-启动消费者等待传入订阅消息)\n         * [3.2 发送延时消息](#32-发送延时消息)\n         * [3.3 验证](#33-验证)\n         * [3.4 延时消息的使用场景](#34-延时消息的使用场景)\n         * [3.5 延时消息的使用限制](#35-延时消息的使用限制)\n      * [4 批量消息样例](#4-批量消息样例)\n         * [4.1 发送批量消息](#41-发送批量消息)\n         * [4.2 消息列表分割](#42-消息列表分割)\n      * [5 过滤消息样例](#5-过滤消息样例)\n         * [5.1 基本语法](#51-基本语法)\n         * [5.2 使用样例](#52-使用样例)\n            * [1、生产者样例](#1生产者样例)\n            * [2、消费者样例](#2消费者样例)\n      * [6 消息事务样例](#6-消息事务样例)\n         * [6.1 发送事务消息样例](#61-发送事务消息样例)\n            * [1、创建事务性生产者](#1创建事务性生产者)\n            * [2、实现事务的监听接口](#2实现事务的监听接口)\n         * [6.2 事务消息使用上的限制](#62-事务消息使用上的限制)\n      * [7 Logappender样例](#7-logappender样例)\n         * [7.1 log4j样例](#71-log4j样例)\n         * [7.2 log4j2样例](#72-log4j2样例)\n         * [7.3 logback样例](#73-logback样例)\n      * [8 OpenMessaging样例](#8-openmessaging样例)\n         * [8.1 OMSProducer样例](#81-omsproducer样例)\n         * [8.2 OMSPullConsumer](#82-omspullconsumer)\n         * [8.3 OMSPushConsumer](#83-omspushconsumer)\n-----\n## 1 基本样例\n\n\n在基本样例中我们提供如下的功能场景：\n\n* 使用RocketMQ发送三种类型的消息：同步消息、异步消息和单向消息。其中前两种消息是可靠的，因为会有发送是否成功的应答。\n* 使用RocketMQ来消费接收到的消息。\n\n### 1.1 加入依赖：\n\n`maven:`\n```\n<dependency>\n    <groupId>org.apache.rocketmq</groupId>\n    <artifactId>rocketmq-client</artifactId>\n    <version>4.9.1</version>\n</dependency>\n```\n`gradle`\n```\ncompile 'org.apache.rocketmq:rocketmq-client:4.3.0'\n```\n### 1.2 消息发送\n\n#### 1、Producer端发送同步消息\n\n这种可靠性同步地发送方式使用的比较广泛，比如：重要的消息通知，短信通知。\n```java\npublic class SyncProducer {\n\tpublic static void main(String[] args) throws Exception {\n    \t// 实例化消息生产者Producer\n        DefaultMQProducer producer = new DefaultMQProducer(\"please_rename_unique_group_name\");\n    \t// 设置NameServer的地址\n    \tproducer.setNamesrvAddr(\"localhost:9876\");\n    \t// 启动Producer实例\n        producer.start();\n    \tfor (int i = 0; i < 100; i++) {\n    \t    // 创建消息，并指定Topic，Tag和消息体\n    \t    Message msg = new Message(\"TopicTest\" /* Topic */,\n        \t\"TagA\" /* Tag */,\n        \t(\"Hello RocketMQ \" + i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */\n        \t);\n        \t// 发送消息到一个Broker\n            SendResult sendResult = producer.send(msg);\n            // 通过sendResult返回消息是否成功送达\n            System.out.printf(\"%s%n\", sendResult);\n    \t}\n    \t// 如果不再发送消息，关闭Producer实例。\n    \tproducer.shutdown();\n    }\n}\n```\n#### 2、发送异步消息\n\n异步消息通常用在对响应时间敏感的业务场景，即发送端不能容忍长时间地等待Broker的响应。\n\n```java\npublic class AsyncProducer {\n\tpublic static void main(String[] args) throws Exception {\n    \t// 实例化消息生产者Producer\n        DefaultMQProducer producer = new DefaultMQProducer(\"please_rename_unique_group_name\");\n    \t// 设置NameServer的地址\n        producer.setNamesrvAddr(\"localhost:9876\");\n    \t// 启动Producer实例\n        producer.start();\n        producer.setRetryTimesWhenSendAsyncFailed(0);\n\t\n\tint messageCount = 100;\n        // 根据消息数量实例化倒计时计算器\n\tfinal CountDownLatch2 countDownLatch = new CountDownLatch2(messageCount);\n    \tfor (int i = 0; i < messageCount; i++) {\n                final int index = i;\n            \t// 创建消息，并指定Topic，Tag和消息体\n                Message msg = new Message(\"TopicTest\",\n                    \"TagA\",\n                    \"OrderID188\",\n                    \"Hello world\".getBytes(RemotingHelper.DEFAULT_CHARSET));\n                // SendCallback接收异步返回结果的回调\n                producer.send(msg, new SendCallback() {\n                    @Override\n                    public void onSuccess(SendResult sendResult) {\n                        countDownLatch.countDown();\n                        System.out.printf(\"%-10d OK %s %n\", index,\n                            sendResult.getMsgId());\n                    }\n                    @Override\n                    public void onException(Throwable e) {\n                        countDownLatch.countDown();\n      \t                System.out.printf(\"%-10d Exception %s %n\", index, e);\n      \t                e.printStackTrace();\n                    }\n            \t});\n    \t}\n\t// 等待5s\n\tcountDownLatch.await(5, TimeUnit.SECONDS);\n    \t// 如果不再发送消息，关闭Producer实例。\n    \tproducer.shutdown();\n    }\n}\n```\n\n#### 3、单向发送消息\n\n这种方式主要用在不特别关心发送结果的场景，例如日志发送。\n\n```java\npublic class OnewayProducer {\n\tpublic static void main(String[] args) throws Exception{\n    \t// 实例化消息生产者Producer\n        DefaultMQProducer producer = new DefaultMQProducer(\"please_rename_unique_group_name\");\n    \t// 设置NameServer的地址\n        producer.setNamesrvAddr(\"localhost:9876\");\n    \t// 启动Producer实例\n        producer.start();\n    \tfor (int i = 0; i < 100; i++) {\n        \t// 创建消息，并指定Topic，Tag和消息体\n        \tMessage msg = new Message(\"TopicTest\" /* Topic */,\n                \"TagA\" /* Tag */,\n                (\"Hello RocketMQ \" + i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */\n        \t);\n        \t// 发送单向消息，没有任何返回结果\n        \tproducer.sendOneway(msg);\n\n    \t}\n    \t// 如果不再发送消息，关闭Producer实例。\n    \tproducer.shutdown();\n    }\n}\n```\n\n### 1.3 消费消息\n\n```java\npublic class Consumer {\n\n\tpublic static void main(String[] args) throws InterruptedException, MQClientException {\n\n    \t// 实例化消费者\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"please_rename_unique_group_name\");\n\n    \t// 设置NameServer的地址\n        consumer.setNamesrvAddr(\"localhost:9876\");\n\n    \t// 订阅一个或者多个Topic，以及Tag来过滤需要消费的消息\n        consumer.subscribe(\"TopicTest\", \"*\");\n    \t// 注册回调实现类来处理从broker拉取回来的消息\n        consumer.registerMessageListener(new MessageListenerConcurrently() {\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {\n                System.out.printf(\"%s Receive New Messages: %s %n\", Thread.currentThread().getName(), msgs);\n                // 标记该消息已经被成功消费\n                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n            }\n        });\n        // 启动消费者实例\n        consumer.start();\n        System.out.printf(\"Consumer Started.%n\");\n\t}\n}\n```\n\n2 顺序消息样例\n----------\n\n消息有序指的是可以按照消息的发送顺序来消费(FIFO)。RocketMQ可以严格的保证消息有序，可以分为分区有序或者全局有序。\n\n顺序消费的原理解析，在默认的情况下消息发送会采取Round Robin轮询方式把消息发送到不同的queue(分区队列)；而消费消息的时候从多个queue上拉取消息，这种情况发送和消费是不能保证顺序。但是如果控制发送的顺序消息只依次发送到同一个queue中，消费的时候只从这个queue上依次拉取，则就保证了顺序。当发送和消费参与的queue只有一个，则是全局有序；如果多个queue参与，则为分区有序，即相对每个queue，消息都是有序的。\n\n下面用订单进行分区有序的示例。一个订单的顺序流程是：创建、付款、推送、完成。订单号相同的消息会被先后发送到同一个队列中，消费时，同一个OrderId获取到的肯定是同一个队列。\n\n### 2.1 顺序消息生产\n\n```java\npackage org.apache.rocketmq.example.order2;\n\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.MessageQueueSelector;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\nimport java.text.SimpleDateFormat;\nimport java.util.ArrayList;\nimport java.util.Date;\nimport java.util.List;\n\n/**\n* Producer，发送顺序消息\n*/\npublic class Producer {\n\n   public static void main(String[] args) throws Exception {\n       DefaultMQProducer producer = new DefaultMQProducer(\"please_rename_unique_group_name\");\n\n       producer.setNamesrvAddr(\"127.0.0.1:9876\");\n\n       producer.start();\n\n       String[] tags = new String[]{\"TagA\", \"TagC\", \"TagD\"};\n\n       // 订单列表\n       List<OrderStep> orderList = new Producer().buildOrders();\n\n       Date date = new Date();\n       SimpleDateFormat sdf = new SimpleDateFormat(\"yyyy-MM-dd HH:mm:ss\");\n       String dateStr = sdf.format(date);\n       for (int i = 0; i < 10; i++) {\n           // 加个时间前缀\n           String body = dateStr + \" Hello RocketMQ \" + orderList.get(i);\n           Message msg = new Message(\"TopicTest\", tags[i % tags.length], \"KEY\" + i, body.getBytes());\n\n           SendResult sendResult = producer.send(msg, new MessageQueueSelector() {\n               @Override\n               public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {\n                   Long id = (Long) arg;  //根据订单id选择发送queue\n                   long index = id % mqs.size();\n                   return mqs.get((int) index);\n               }\n           }, orderList.get(i).getOrderId());//订单id\n\n           System.out.println(String.format(\"SendResult status:%s, queueId:%d, body:%s\",\n               sendResult.getSendStatus(),\n               sendResult.getMessageQueue().getQueueId(),\n               body));\n       }\n\n       producer.shutdown();\n   }\n\n   /**\n    * 订单的步骤\n    */\n   private static class OrderStep {\n       private long orderId;\n       private String desc;\n\n       public long getOrderId() {\n           return orderId;\n       }\n\n       public void setOrderId(long orderId) {\n           this.orderId = orderId;\n       }\n\n       public String getDesc() {\n           return desc;\n       }\n\n       public void setDesc(String desc) {\n           this.desc = desc;\n       }\n\n       @Override\n       public String toString() {\n           return \"OrderStep{\" +\n               \"orderId=\" + orderId +\n               \", desc='\" + desc + '\\'' +\n               '}';\n       }\n   }\n\n   /**\n    * 生成模拟订单数据\n    */\n   private List<OrderStep> buildOrders() {\n       List<OrderStep> orderList = new ArrayList<OrderStep>();\n\n       OrderStep orderDemo = new OrderStep();\n       orderDemo.setOrderId(15103111039L);\n       orderDemo.setDesc(\"创建\");\n       orderList.add(orderDemo);\n\n       orderDemo = new OrderStep();\n       orderDemo.setOrderId(15103111065L);\n       orderDemo.setDesc(\"创建\");\n       orderList.add(orderDemo);\n\n       orderDemo = new OrderStep();\n       orderDemo.setOrderId(15103111039L);\n       orderDemo.setDesc(\"付款\");\n       orderList.add(orderDemo);\n\n       orderDemo = new OrderStep();\n       orderDemo.setOrderId(15103117235L);\n       orderDemo.setDesc(\"创建\");\n       orderList.add(orderDemo);\n\n       orderDemo = new OrderStep();\n       orderDemo.setOrderId(15103111065L);\n       orderDemo.setDesc(\"付款\");\n       orderList.add(orderDemo);\n\n       orderDemo = new OrderStep();\n       orderDemo.setOrderId(15103117235L);\n       orderDemo.setDesc(\"付款\");\n       orderList.add(orderDemo);\n\n       orderDemo = new OrderStep();\n       orderDemo.setOrderId(15103111065L);\n       orderDemo.setDesc(\"完成\");\n       orderList.add(orderDemo);\n\n       orderDemo = new OrderStep();\n       orderDemo.setOrderId(15103111039L);\n       orderDemo.setDesc(\"推送\");\n       orderList.add(orderDemo);\n\n       orderDemo = new OrderStep();\n       orderDemo.setOrderId(15103117235L);\n       orderDemo.setDesc(\"完成\");\n       orderList.add(orderDemo);\n\n       orderDemo = new OrderStep();\n       orderDemo.setOrderId(15103111039L);\n       orderDemo.setDesc(\"完成\");\n       orderList.add(orderDemo);\n\n       return orderList;\n   }\n}\n```\n\n### 2.2 顺序消费消息\n\n```java\npackage org.apache.rocketmq.example.order2;\n\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.MessageExt;\n\nimport java.util.List;\nimport java.util.Random;\nimport java.util.concurrent.TimeUnit;\n\n/**\n* 顺序消息消费，带事务方式（应用可控制Offset什么时候提交）\n*/\npublic class ConsumerInOrder {\n\n   public static void main(String[] args) throws Exception {\n       DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"please_rename_unique_group_name_3\");\n       consumer.setNamesrvAddr(\"127.0.0.1:9876\");\n       /**\n        * 设置Consumer第一次启动是从队列头部开始消费还是队列尾部开始消费<br>\n        * 如果非第一次启动，那么按照上次消费的位置继续消费\n        */\n       consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n\n       consumer.subscribe(\"TopicTest\", \"TagA || TagC || TagD\");\n\n       consumer.registerMessageListener(new MessageListenerOrderly() {\n\n           Random random = new Random();\n\n           @Override\n           public ConsumeOrderlyStatus consumeMessage(List<MessageExt> msgs, ConsumeOrderlyContext context) {\n               context.setAutoCommit(true);\n               for (MessageExt msg : msgs) {\n                   // 可以看到每个queue有唯一的consume线程来消费, 订单对每个queue(分区)有序\n                   System.out.println(\"consumeThread=\" + Thread.currentThread().getName() + \"queueId=\" + msg.getQueueId() + \", content:\" + new String(msg.getBody()));\n               }\n\n               try {\n                   //模拟业务逻辑处理中...\n                   TimeUnit.SECONDS.sleep(random.nextInt(10));\n               } catch (Exception e) {\n                   e.printStackTrace();\n               }\n               return ConsumeOrderlyStatus.SUCCESS;\n           }\n       });\n\n       consumer.start();\n\n       System.out.println(\"Consumer Started.\");\n   }\n}\n```\n\n3 延时消息样例\n----------\n\n### 3.1 启动消费者等待传入订阅消息\n\n```java\n\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport java.util.List;\n\npublic class ScheduledMessageConsumer {\n   public static void main(String[] args) throws Exception {\n      // 实例化消费者\n      DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"ExampleConsumer\");\n      // 设置NameServer的地址\n      consumer.setNamesrvAddr(\"localhost:9876\");\n      // 订阅Topics\n      consumer.subscribe(\"TestTopic\", \"*\");\n      // 注册消息监听者\n      consumer.registerMessageListener(new MessageListenerConcurrently() {\n          @Override\n          public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> messages, ConsumeConcurrentlyContext context) {\n              for (MessageExt message : messages) {\n                  // Print approximate delay time period\n                  System.out.println(\"Receive message[msgId=\" + message.getMsgId() + \"] \" + (System.currentTimeMillis() - message.getBornTimestamp()) + \"ms later\");\n              }\n              return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n          }\n      });\n      // 启动消费者\n      consumer.start();\n  }\n}\n\n```\n\n### 3.2 发送延时消息\n\n```java\n\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.common.message.Message;\n\npublic class ScheduledMessageProducer {\n   public static void main(String[] args) throws Exception {\n      // 实例化一个生产者来产生延时消息\n      DefaultMQProducer producer = new DefaultMQProducer(\"ExampleProducerGroup\");\n      // 设置NameServer的地址\n      producer.setNamesrvAddr(\"localhost:9876\");\n      // 启动生产者\n      producer.start();\n      int totalMessagesToSend = 100;\n      for (int i = 0; i < totalMessagesToSend; i++) {\n          Message message = new Message(\"TestTopic\", (\"Hello scheduled message \" + i).getBytes());\n          // 设置延时等级3,这个消息将在10s之后发送(现在只支持固定的几个时间,详看delayTimeLevel)\n          message.setDelayTimeLevel(3);\n          // 发送消息\n          producer.send(message);\n      }\n       // 关闭生产者\n      producer.shutdown();\n  }\n}\n```\n\n### 3.3 验证\n\n您将会看到消息的消费比存储时间晚10秒。\n\n### 3.4 延时消息的使用场景\n比如电商里，提交了一个订单就可以发送一个延时消息，1h后去检查这个订单的状态，如果还是未付款就取消订单释放库存。\n\n### 3.5 延时消息的使用限制\n\n```java\n// org/apache/rocketmq/store/config/MessageStoreConfig.java\n\nprivate String messageDelayLevel = \"1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h\";\n```\n现在RocketMq并不支持任意时间的延时，需要设置几个固定的延时等级，从1s到2h分别对应着等级1到18\n消息消费失败会进入延时消息队列，消息发送时间与设置的延时等级和重试次数有关，详见代码`SendMessageProcessor.java`\n\n\n4 批量消息样例\n----------\n\n批量发送消息能显著提高传递小消息的性能。限制是这些批量消息应该有相同的topic，相同的waitStoreMsgOK，而且不能是延时消息。此外，这一批消息的总大小不应超过4MB。\n\n### 4.1 发送批量消息\n\n如果您每次只发送不超过4MB的消息，则很容易使用批处理，样例如下：\n\n```java\nString topic = \"BatchTest\";\nList<Message> messages = new ArrayList<>();\nmessages.add(new Message(topic, \"TagA\", \"OrderID001\", \"Hello world 0\".getBytes()));\nmessages.add(new Message(topic, \"TagA\", \"OrderID002\", \"Hello world 1\".getBytes()));\nmessages.add(new Message(topic, \"TagA\", \"OrderID003\", \"Hello world 2\".getBytes()));\ntry {\n   producer.send(messages);\n} catch (Exception e) {\n   e.printStackTrace();\n   //处理error\n}\n\n```\n\n### 4.2 消息列表分割\n\n复杂度只有当你发送大批量时才会增长，你可能不确定它是否超过了大小限制（4MB）。这时候你最好把你的消息列表分割一下：\n\n```java\npublic class ListSplitter implements Iterator<List<Message>> { \n    private final int SIZE_LIMIT = 1024 * 1024 * 4;\n    private final List<Message> messages;\n    private int currIndex;\n    public ListSplitter(List<Message> messages) { \n        this.messages = messages;\n    }\n    @Override \n    public boolean hasNext() {\n        return currIndex < messages.size(); \n    }\n    @Override \n    public List<Message> next() { \n        int startIndex = getStartIndex();\n        int nextIndex = startIndex;\n        int totalSize = 0;\n        for (; nextIndex < messages.size(); nextIndex++) {\n            Message message = messages.get(nextIndex); \n            int tmpSize = calcMessageSize(message);\n            if (tmpSize + totalSize > SIZE_LIMIT) {\n                break; \n            } else {\n                totalSize += tmpSize; \n            }\n        }\n        List<Message> subList = messages.subList(startIndex, nextIndex); \n        currIndex = nextIndex;\n        return subList;\n    }\n    private int getStartIndex() {\n        Message currMessage = messages.get(currIndex); \n        int tmpSize = calcMessageSize(currMessage); \n        while(tmpSize > SIZE_LIMIT) {\n            currIndex += 1;\n            Message message = messages.get(currIndex); \n            tmpSize = calcMessageSize(message);\n        }\n        return currIndex; \n    }\n    private int calcMessageSize(Message message) {\n        int tmpSize = message.getTopic().length() + message.getBody().length; \n        Map<String, String> properties = message.getProperties();\n        for (Map.Entry<String, String> entry : properties.entrySet()) {\n            tmpSize += entry.getKey().length() + entry.getValue().length(); \n        }\n        tmpSize = tmpSize + 20; // 增加⽇日志的开销20字节\n        return tmpSize; \n    }\n}\n//把大的消息分裂成若干个小的消息\nListSplitter splitter = new ListSplitter(messages);\nwhile (splitter.hasNext()) {\n  try {\n      List<Message>  listItem = splitter.next();\n      producer.send(listItem);\n  } catch (Exception e) {\n      e.printStackTrace();\n      //处理error\n  }\n}\n```\n\n5 过滤消息样例\n----------\n\n在大多数情况下，TAG是一个简单而有用的设计，其可以来选择您想要的消息。例如：\n\n```java\nDefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"CID_EXAMPLE\");\nconsumer.subscribe(\"TOPIC\", \"TAGA || TAGB || TAGC\");\n```\n\n消费者将接收包含TAGA或TAGB或TAGC的消息。但是限制是一个消息只能有一个标签，这对于复杂的场景可能不起作用。在这种情况下，可以使用SQL表达式筛选消息。SQL特性可以通过发送消息时的属性来进行计算。在RocketMQ定义的语法下，可以实现一些简单的逻辑。下面是一个例子：\n```\n------------\n| message  |\n|----------|  a > 5 AND b = 'abc'\n| a = 10   |  --------------------> Gotten\n| b = 'abc'|\n| c = true |\n------------\n------------\n| message  |\n|----------|   a > 5 AND b = 'abc'\n| a = 1    |  --------------------> Missed\n| b = 'abc'|\n| c = true |\n------------\n```\n### 5.1 基本语法\n\nRocketMQ只定义了一些基本语法来支持这个特性。你也可以很容易地扩展它。\n\n- 数值比较，比如：**>，>=，<，<=，BETWEEN，=；**\n- 字符比较，比如：**=，<>，IN；**\n- **IS NULL** 或者 **IS NOT NULL；**\n- 逻辑符号 **AND，OR，NOT；**\n\n常量支持类型为：\n\n- 数值，比如：**123，3.1415；**\n- 字符，比如：**'abc'，必须用单引号包裹起来；**\n- **NULL**，特殊的常量\n- 布尔值，**TRUE** 或 **FALSE**\n\n只有使用push模式的消费者才能用使用SQL92标准的sql语句，接口如下：\n```\npublic void subscribe(final String topic, final MessageSelector messageSelector)\n```\n\n### 5.2 使用样例\n\n#### 1、生产者样例\n\n发送消息时，你能通过`putUserProperty`来设置消息的属性\n\n```java\nDefaultMQProducer producer = new DefaultMQProducer(\"please_rename_unique_group_name\");\nproducer.start();\nMessage msg = new Message(\"TopicTest\",\n   tag,\n   (\"Hello RocketMQ \" + i).getBytes(RemotingHelper.DEFAULT_CHARSET)\n);\n// 设置一些属性\nmsg.putUserProperty(\"a\", String.valueOf(i));\nSendResult sendResult = producer.send(msg);\n\nproducer.shutdown();\n```\n\n#### 2、消费者样例\n\n用MessageSelector.bySql来使用sql筛选消息\n\n```java\nDefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"please_rename_unique_group_name_4\");\n// 只有订阅的消息有这个属性a, a >=0 and a <= 3\nconsumer.subscribe(\"TopicTest\", MessageSelector.bySql(\"a between 0 and 3\");\nconsumer.registerMessageListener(new MessageListenerConcurrently() {\n   @Override\n   public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {\n       return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n   }\n});\nconsumer.start();\n\n```\n\n6 消息事务样例\n----------\n\n事务消息共有三种状态，提交状态、回滚状态、中间状态：\n\n- TransactionStatus.CommitTransaction: 提交事务，它允许消费者消费此消息。\n- TransactionStatus.RollbackTransaction: 回滚事务，它代表该消息将被删除，不允许被消费。\n- TransactionStatus.Unknown: 中间状态，它代表需要检查消息队列来确定状态。\n\n### 6.1 发送事务消息样例\n\n#### 1、创建事务性生产者\n\n使用 `TransactionMQProducer`类创建生产者，并指定唯一的 `ProducerGroup`，就可以设置自定义线程池来处理这些检查请求。执行本地事务后、需要根据执行结果对消息队列进行回复。回传的事务状态在请参考前一节。\n\n```java\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport java.util.List;\npublic class TransactionProducer {\n   public static void main(String[] args) throws MQClientException, InterruptedException {\n       TransactionListener transactionListener = new TransactionListenerImpl();\n       TransactionMQProducer producer = new TransactionMQProducer(\"please_rename_unique_group_name\");\n       ExecutorService executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(2000), new ThreadFactory() {\n           @Override\n           public Thread newThread(Runnable r) {\n               Thread thread = new Thread(r);\n               thread.setName(\"client-transaction-msg-check-thread\");\n               return thread;\n           }\n       });\n       producer.setExecutorService(executorService);\n       producer.setTransactionListener(transactionListener);\n       producer.start();\n       String[] tags = new String[] {\"TagA\", \"TagB\", \"TagC\", \"TagD\", \"TagE\"};\n       for (int i = 0; i < 10; i++) {\n           try {\n               Message msg =\n                   new Message(\"TopicTest1234\", tags[i % tags.length], \"KEY\" + i,\n                       (\"Hello RocketMQ \" + i).getBytes(RemotingHelper.DEFAULT_CHARSET));\n               SendResult sendResult = producer.sendMessageInTransaction(msg, null);\n               System.out.printf(\"%s%n\", sendResult);\n               Thread.sleep(10);\n           } catch (MQClientException | UnsupportedEncodingException e) {\n               e.printStackTrace();\n           }\n       }\n       for (int i = 0; i < 100000; i++) {\n           Thread.sleep(1000);\n       }\n       producer.shutdown();\n   }\n}\n\n```\n#### 2、实现事务的监听接口\n\n当发送半消息成功时，我们使用 `executeLocalTransaction` 方法来执行本地事务。它返回前一节中提到的三个事务状态之一。`checkLocalTransaction` 方法用于检查本地事务状态，并回应消息队列的检查请求。它也是返回前一节中提到的三个事务状态之一。\n\n```java\npublic class TransactionListenerImpl implements TransactionListener {\n  private AtomicInteger transactionIndex = new AtomicInteger(0);\n  private ConcurrentHashMap<String, Integer> localTrans = new ConcurrentHashMap<>();\n  @Override\n  public LocalTransactionState executeLocalTransaction(Message msg, Object arg) {\n      int value = transactionIndex.getAndIncrement();\n      int status = value % 3;\n      localTrans.put(msg.getTransactionId(), status);\n      return LocalTransactionState.UNKNOW;\n  }\n  @Override\n  public LocalTransactionState checkLocalTransaction(MessageExt msg) {\n      Integer status = localTrans.get(msg.getTransactionId());\n      if (null != status) {\n          switch (status) {\n              case 0:\n                  return LocalTransactionState.UNKNOW;\n              case 1:\n                  return LocalTransactionState.COMMIT_MESSAGE;\n              case 2:\n                  return LocalTransactionState.ROLLBACK_MESSAGE;\n          }\n      }\n      return LocalTransactionState.COMMIT_MESSAGE;\n  }\n}\n\n```\n\n### 6.2 事务消息使用上的限制\n\n1. 事务消息不支持延时消息和批量消息。\n2. 为了避免单个消息被检查太多次而导致半队列消息累积，我们默认将单个消息的检查次数限制为 15 次，但是用户可以通过 Broker 配置文件的 `transactionCheckMax`参数来修改此限制。如果已经检查某条消息超过 N 次的话（ N = `transactionCheckMax` ） 则 Broker 将丢弃此消息，并在默认情况下同时打印错误日志。用户可以通过重写 `AbstractTransactionalMessageCheckListener` 类来修改这个行为。\n3. 事务消息将在 Broker 配置文件中的参数 transactionTimeout 这样的特定时间长度之后被检查。当发送事务消息时，用户还可以通过设置用户属性 CHECK_IMMUNITY_TIME_IN_SECONDS 来改变这个限制，该参数优先于 `transactionTimeout` 参数。\n4. 事务性消息可能不止一次被检查或消费。\n5. 提交给用户的目标主题消息可能会失败，目前这依日志的记录而定。它的高可用性通过 RocketMQ 本身的高可用性机制来保证，如果希望确保事务消息不丢失、并且事务完整性得到保证，建议使用同步的双重写入机制。\n6. 事务消息的生产者 GroupName 不能与其他类型消息的生产者 GroupName 共享。与其他类型的消息不同，事务消息允许反向查询、MQ服务器能通过它们的生产者 GroupName 查询到生产者。\n\n7 Logappender样例\n-----------------\n\nRocketMQ日志提供log4j、log4j2和logback日志框架作为业务应用，下面是配置样例\n\n### 7.1 log4j样例\n\n按下面样例使用log4j属性配置\n```\nlog4j.appender.mq=org.apache.rocketmq.logappender.log4j.RocketmqLog4jAppender\nlog4j.appender.mq.Tag=yourTag\nlog4j.appender.mq.Topic=yourLogTopic\nlog4j.appender.mq.ProducerGroup=yourLogGroup\nlog4j.appender.mq.NameServerAddress=yourRocketmqNameserverAddress\nlog4j.appender.mq.layout=org.apache.log4j.PatternLayout\nlog4j.appender.mq.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-4r [%t] (%F:%L) %-5p - %m%n\n```\n按下面样例使用log4j xml配置来使用异步添加日志\n```\n<appender name=\"mqAppender1\"class=\"org.apache.rocketmq.logappender.log4j.RocketmqLog4jAppender\">\n  <param name=\"Tag\" value=\"yourTag\" />\n  <param name=\"Topic\" value=\"yourLogTopic\" />\n  <param name=\"ProducerGroup\" value=\"yourLogGroup\" />\n  <param name=\"NameServerAddress\" value=\"yourRocketmqNameserverAddress\"/>\n  <layout class=\"org.apache.log4j.PatternLayout\">\n      <param name=\"ConversionPattern\" value=\"%d{yyyy-MM-dd HH:mm:ss}-%p %t %c - %m%n\" />\n  </layout>\n</appender>\n<appender name=\"mqAsyncAppender1\"class=\"org.apache.log4j.AsyncAppender\">\n  <param name=\"BufferSize\" value=\"1024\" />\n  <param name=\"Blocking\" value=\"false\" />\n  <appender-ref ref=\"mqAppender1\"/>\n</appender>\n```\n### 7.2 log4j2样例\n\n用log4j2时，配置如下，如果想要非阻塞，只需要使用异步添加引用即可\n```\n<RocketMQ name=\"rocketmqAppender\" producerGroup=\"yourLogGroup\" nameServerAddress=\"yourRocketmqNameserverAddress\"\n   topic=\"yourLogTopic\" tag=\"yourTag\">\n  <PatternLayout pattern=\"%d [%p] hahahah %c %m%n\"/>\n</RocketMQ>\n```\n### 7.3 logback样例\n```\n<appender name=\"mqAppender1\"class=\"org.apache.rocketmq.logappender.logback.RocketmqLogbackAppender\">\n  <tag>yourTag</tag>\n  <topic>yourLogTopic</topic>\n  <producerGroup>yourLogGroup</producerGroup>\n  <nameServerAddress>yourRocketmqNameserverAddress</nameServerAddress>\n  <layout>\n      <pattern>%date %p %t - %m%n</pattern>\n  </layout>\n</appender>\n<appender name=\"mqAsyncAppender1\"class=\"ch.qos.logback.classic.AsyncAppender\">\n  <queueSize>1024</queueSize>\n  <discardingThreshold>80</discardingThreshold>\n  <maxFlushTime>2000</maxFlushTime>\n  <neverBlock>true</neverBlock>\n  <appender-ref ref=\"mqAppender1\"/>\n</appender>\n```\n\n8 OpenMessaging样例\n---------------\n\n [OpenMessaging](https://www.google.com/url?q=http://openmessaging.cloud/&sa=D&ust=1546524111089000)旨在建立消息和流处理规范，以为金融、电子商务、物联网和大数据领域提供通用框架及工业级指导方案。在分布式异构环境中，设计原则是面向云、简单、灵活和独立于语言。符合这些规范将帮助企业方便的开发跨平台和操作系统的异构消息传递应用程序。提供了openmessaging-api 0.3.0-alpha的部分实现，下面的示例演示如何基于OpenMessaging访问RocketMQ。\n\n### 8.1 OMSProducer样例\n\n下面的示例演示如何在同步、异步或单向传输中向RocketMQ代理发送消息。\n\n```java\nimport io.openmessaging.Future;\nimport io.openmessaging.FutureListener;\nimport io.openmessaging.Message;\nimport io.openmessaging.MessagingAccessPoint;\nimport io.openmessaging.OMS;\nimport io.openmessaging.producer.Producer;\nimport io.openmessaging.producer.SendResult;\n\nimport java.nio.charset.Charset;\nimport java.nio.charset.StandardCharsets;\nimport java.util.concurrent.CountDownLatch;\n\npublic class SimpleProducer {\n    public static void main(String[] args) {\n        final MessagingAccessPoint messagingAccessPoint =\n                OMS.getMessagingAccessPoint(\"oms:rocketmq://localhost:9876/default:default\");\n        final Producer producer = messagingAccessPoint.createProducer();\n        messagingAccessPoint.startup();\n        System.out.printf(\"MessagingAccessPoint startup OK%n\");\n        producer.startup();\n        System.out.printf(\"Producer startup OK%n\");\n        {\n            Message message = producer.createBytesMessage(\"OMS_HELLO_TOPIC\", \"OMS_HELLO_BODY\".getBytes(StandardCharsets.UTF_8));\n            SendResult sendResult = producer.send(message);\n            //final Void aVoid = result.get(3000L);\n            System.out.printf(\"Send async message OK, msgId: %s%n\", sendResult.messageId());\n        }\n        final CountDownLatch countDownLatch = new CountDownLatch(1);\n        {\n            final Future<SendResult> result = producer.sendAsync(producer.createBytesMessage(\"OMS_HELLO_TOPIC\", \"OMS_HELLO_BODY\".getBytes(StandardCharsets.UTF_8)));\n            result.addListener(new FutureListener<SendResult>() {\n                @Override\n                public void operationComplete(Future<SendResult> future) {\n                    if (future.getThrowable() != null) {\n                        System.out.printf(\"Send async message Failed, error: %s%n\", future.getThrowable().getMessage());\n                    } else {\n                        System.out.printf(\"Send async message OK, msgId: %s%n\", future.get().messageId());\n                    }\n                    countDownLatch.countDown();\n                }\n            });\n        }\n        {\n            producer.sendOneway(producer.createBytesMessage(\"OMS_HELLO_TOPIC\", \"OMS_HELLO_BODY\".getBytes(StandardCharsets.UTF_8)));\n            System.out.printf(\"Send oneway message OK%n\");\n        }\n        try {\n            countDownLatch.await();\n            Thread.sleep(500); // 等一些时间来发送消息\n        } catch (InterruptedException ignore) {\n        }\n        producer.shutdown();\n    }\n}\n```\n\n### 8.2 OMSPullConsumer\n\n用OMS PullConsumer 来从指定的队列中拉取消息\n\n```java\nimport io.openmessaging.Message;\nimport io.openmessaging.MessagingAccessPoint;\nimport io.openmessaging.OMS;\nimport io.openmessaging.OMSBuiltinKeys;\nimport io.openmessaging.consumer.PullConsumer;\nimport io.openmessaging.producer.Producer;\nimport io.openmessaging.producer.SendResult;\n\npublic class SimplePullConsumer {\n    public static void main(String[] args) {\n       final MessagingAccessPoint messagingAccessPoint =\n           OMS.getMessagingAccessPoint(\"oms:rocketmq://localhost:9876/default:default\");\n       messagingAccessPoint.startup();\n       final Producer producer = messagingAccessPoint.createProducer();\n       final PullConsumer consumer = messagingAccessPoint.createPullConsumer(\n           OMS.newKeyValue().put(OMSBuiltinKeys.CONSUMER_ID, \"OMS_CONSUMER\"));\n       messagingAccessPoint.startup();\n       System.out.printf(\"MessagingAccessPoint startup OK%n\");\n       final String queueName = \"TopicTest\";\n       producer.startup();\n       Message msg = producer.createBytesMessage(queueName, \"Hello Open Messaging\".getBytes());\n       SendResult sendResult = producer.send(msg);\n       System.out.printf(\"Send Message OK. MsgId: %s%n\", sendResult.messageId());\n       producer.shutdown();\n       consumer.attachQueue(queueName);\n       consumer.startup();\n       System.out.printf(\"Consumer startup OK%n\");\n       // 运行直到发现一个消息被发送了\n       boolean stop = false;\n       while (!stop) {\n           Message message = consumer.receive();\n           if (message != null) {\n               String msgId = message.sysHeaders().getString(Message.BuiltinKeys.MESSAGE_ID);\n               System.out.printf(\"Received one message: %s%n\", msgId);\n               consumer.ack(msgId);\n               if (!stop) {\n                   stop = msgId.equalsIgnoreCase(sendResult.messageId());\n               }\n           } else {\n               System.out.printf(\"Return without any message%n\");\n           }\n       }\n       consumer.shutdown();\n       messagingAccessPoint.shutdown();\n   }\n}\n```\n\n### 8.3 OMSPushConsumer\n\n以下示范如何将 OMS PushConsumer 添加到指定的队列，并通过 MessageListener 消费这些消息。\n\n```java\nimport io.openmessaging.Message;\nimport io.openmessaging.MessagingAccessPoint;\nimport io.openmessaging.OMS;\nimport io.openmessaging.OMSBuiltinKeys;\nimport io.openmessaging.consumer.MessageListener;\nimport io.openmessaging.consumer.PushConsumer;\n\npublic class SimplePushConsumer {\n    public static void main(String[] args) {\n       final MessagingAccessPoint messagingAccessPoint = OMS\n           .getMessagingAccessPoint(\"oms:rocketmq://localhost:9876/default:default\");\n       final PushConsumer consumer = messagingAccessPoint.\n           createPushConsumer(OMS.newKeyValue().put(OMSBuiltinKeys.CONSUMER_ID, \"OMS_CONSUMER\"));\n       messagingAccessPoint.startup();\n       System.out.printf(\"MessagingAccessPoint startup OK%n\");\n       Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {\n           @Override\n           public void run() {\n               consumer.shutdown();\n               messagingAccessPoint.shutdown();\n           }\n       }));\n       consumer.attachQueue(\"OMS_HELLO_TOPIC\", new MessageListener() {\n           @Override\n           public void onReceived(Message message, Context context) {\n               System.out.printf(\"Received one message: %s%n\", message.sysHeaders().getString(Message.BuiltinKeys.MESSAGE_ID));\n               context.ack();\n           }\n       });\n       consumer.startup();\n       System.out.printf(\"Consumer startup OK%n\");\n   }\n}\n```\n"
  },
  {
    "path": "docs/cn/SlaveActingMasterMode.md",
    "content": "# Slave Acting Master模式\n\n## 背景\n\n![](https://s4.ax1x.com/2022/02/05/HnW3CQ.png)\n\n上图为当前RocketMQ Master-Slave冷备部署，在该部署方式下，即使一个Master掉线，发送端仍然可以向其他Master发送消息，对于消费端而言，若开启备读，Consumer会自动重连到对应的Slave机器，不会出现消费停滞的情况。但也存在以下问题：\n\n1. 一些仅限于在Master上进行的操作将无法进行，包括且不限于：\n\n- searchOffset \n- maxOffset \n- minOffset \n- earliestMsgStoreTime \n- endTransaction\n\n所有锁MQ相关操作，包括lock，unlock，lockBatch，unlockAll\n\n具体影响为：\n- 客户端无法获取位于该副本组的mq的锁，故当本地锁过期后，将无法消费该组的顺序消息 \n- 客户端无法主动结束处于半状态的事务消息，只能等待broker回查事务状态 \n- Admin tools或控制中依赖查询offset及earliestMsgStoreTime等操作在该组上无法生效\n\n2. 故障Broker组上的二级消息消费将会中断，该类消息特点依赖Master Broker上的线程扫描CommitLog上的特殊Topic，并将满足要求的消息投放回CommitLog，如果Master Broker下线，会出现二级消息的消费延迟或丢失。具体会影响到当前版本的延迟消息消费、事务消息消费、Pop消费。\n\n3. 没有元数据的反向同步。Master重新被人工拉起后，容易造成元数据的回退，如Master上线后将落后的消费位点同步给备，该组broker的消费位点回退，造成大量消费重复。\n\n![](https://s4.ax1x.com/2022/02/05/HnWwUU.png)\n\n上图为DLedger（Raft）架构，其可以通过选主一定程度上规避上述存在的问题，但可以看到DLedger模式下当前需要强制三副本及以上。\n\n提出一个新的方案，Slave代理Master模式，作为Master-Slave部署模式的升级。在原先Master-Slave部署模式下，通过备代理主、轻量级心跳、副本组信息获取、broker预上线机制、二级消息逃逸等方式，当同组Master发生故障时，Slave将承担更加重要的作用，包括：\n\n- 当Master下线后，该组中brokerId最小的Slave会承担备读 以及 一些 客户端和管控会访问 但却只能在Master节点上完成的任务。包括且不限于searchOffset、maxOffset、minOffset、earliestMsgStoreTime、endTransaction以及所有锁MQ相关操作lock，unlock，lockBatch，unlockAll。\n- 当Master下线后，故障Broker组上的二级消息消费将不会中断，由该组中该组中brokerId最小的Slave承担起该任务，定时消息、Pop消息、事务消息等仍然可以正常运行。\n- 当Master下线后，在Slave代理Master一段时间主后，然后当Master再次上线后，通过预上线机制，Master会自动完成元数据的反向同步后再上线，不会出现元数据回退，造成消息大量重复消费或二级消息大量重放。\n\n## 架构\n\n### 备代理主\n\nMaster下线后Slave能正常消费，且在不修改客户端代码情况下完成只能在Master完成的操作源自于Namesrv对“代理”Master的支持。此处“代理”Master指的是，当副本组处于无主状态时，Namesrv将把brokerId最小的存活Slave视为“代理”Master，具体表现为在构建TopicRouteData时，将该Slave的brokerId替换为0，并将brokerPermission修改为4（Read-Only），从而使得该Slave在客户端视图中充当只读模式的Master的角色。\n\n此外，当Master下线后，brokerId最小的Slave会承担起二级消息的扫描和重新投递功能，这也是“代理”的一部分。\n\n```java\n//改变二级消息扫描状态\npublic void changeSpecialServiceStatus(boolean shouldStart) {\n\t……\n\n    //改变延迟消息服务的状态\n    changeScheduleServiceStatus(shouldStart);\n\n    //改变事务消息服务的状态\n    changeTransactionCheckServiceStatus(shouldStart);\n\n    //改变Pop消息服务状态\n    if (this.ackMessageProcessor != null) {\n        LOG.info(\"Set PopReviveService Status to {}\", shouldStart);\n        this.ackMessageProcessor.setPopReviveServiceStatus(shouldStart);\n    }\n}\n```\n\n### 轻量级心跳\n\n如上文所述，brokerId最小的存活Slave在Master故障后开启自动代理Master模式，因此需要一种机制，这个机制需要保证：\n\n1. Nameserver能及时发现broker上下线并完成路由替换以及下线broker的路由剔除。\n\n2. Broker能及时感知到同组Broker的上下线情况。\n\n针对1，Nameserver原本就存在判活机制，定时会扫描不活跃的broker使其下线，而原本broker与nameserver的“心跳”则依赖于registerBroker操作，而这个操作涉及到topic信息上报，过于“重”，而且注册间隔过于长，因此需要一个轻量级的心跳机制，RocketMQ 5.0在nameserver和broker间新增BrokerHeartbeat请求，broker会定时向nameserver发送心跳，若nameserver定时任务扫描发现超过心跳超时时间仍未收到该broker的心跳，将unregister该broker。registerBroker时会完成心跳超时时间的设置，并且注册时如果发现broker组内最小brokerId发生变化，将反向通知该组所有broker，并在路由获取时将最小brokerId的Slave路由替换使其充当只读模式的Master的角色\n\n针对2，通过两个机制来及时感知同组broker上下线情况，1是上文中介绍的当nameserver发现该broker组内最小brokerId发生变化，反向通知该组所有broker。2是broker自身会有定时任务，向nameserver同步本broker组存活broker的信息，RocketMQ 5.0会新增GetBrokerMemberGroup请求来完成该工作。\n\nSlave Broker发现自己是该组中最小的brokerId，将会开启代理模式，而一旦Master Broker重新上线，Slave Broker同样会通过Nameserver反向通知或自身定时任务同步同组broker的信息感知到，并自动结束代理模式。\n\n### 二级消息逃逸\n\n代理模式开启后，brokerId最小的Slave会承担起二级消息的扫描和重新投递功能。\n\n二级消息一般分为两个阶段，发送或者消费时会发送到一个特殊topic中，后台会有线程会扫描，最终的满足要求的消息会被重新投递到Commitlog中。我们可以让brokerId最小的Slave进行扫描，但如果扫描之后的消息重新投递到本Commitlog，那将会破坏Slave不可写的语义，造成Commitlog分叉。因此RocketMQ 5.0提出一种逃逸机制，将重放的二级消息远程或本地投放到其他Master的Commitlog中。\n\n- 远程逃逸\n\n![](https://s4.ax1x.com/2022/02/05/HnWWVK.png)\n\n如上图所示，假设Region A发生故障，Region B中的节点2将会承担二级消息的扫描任务，同时将最终的满足要求的消息通过EscapeBridge远程发送到当前Broker集群中仍然存活的Master上。\n\n- 本地逃逸\n\n![](https://s4.ax1x.com/2022/02/05/HnWfUO.png)\n\n本地逃逸需要在BrokerContainer下进行，如果BrokerContainer中存在存活的Master，会优先向同进程的Master Commitlog中逃逸，避免远程RPC。\n\n#### 各类二级消息变化\n\n**延迟消息**\n\nSlave代理Master时，ScheduleMessageService将启动，时间到期的延迟消息将通过EscapeBridge优先往本地Master逃逸，若没有则向远程的Master逃逸。该broker上存量的时间未到期的消息将会被逃逸到存活的其他Master上，数据量上如果该broker上有大量的延迟消息未到期，远程逃逸会造成集群内部会有较大数据流转，但基本可控。\n\n\n**POP消息**\n\n1. CK/ACK拼key的时候增加brokerName属性。这样每个broker能在扫描自身commitlog的revive topic时抵消其他broker的CK/ACK消息。\n\n2. Slave上的CK/ACK消息将被逃逸到其他指定的Master A上（需要同一个Master，否则CK/ACK无法抵消，造成消息重复），Master A扫描自身Commitlog revive消息并进行抵消，若超时，则将根据CK消息中的信息向Slave拉取消息（若本地有则拉取本地，否则远程拉取），然后投放到本地的retry topic中。\n\n数据量上，如果是远程投递或拉取，且有消费者大量通过Pop消费存量的Slave消息，并且长时间不ACK，则在集群内部会有较大数据流转。\n\n### 预上线机制\n\n![](https://s4.ax1x.com/2022/02/05/HnW5Pe.png)\n\n当Master Broker下线后，Slave Broker将承担备读的作用，并对二级消息进行代理，因此Slave Broker中的部分元数据包括消费位点、定时消息进度等会比下线的Master Broker更加超前。如果Master Broker重新上线，Slave Broker元数据将被Master Broker覆盖，该组Broker元数据将发生回退，可能造成大量消息重复。因此，需要一套预上线机制来完成元数据的反向同步。\n\n需要为consumerOffset和delayOffset等元数据增加版本号（DataVersion）的概念，并且为了防止版本号更新太频繁，增加更新步长的概念，比如对于消费位点来说，默认每更新位点超过500次，版本号增加到下一个版本。\n\n如上图所示，Master Broker启动前会进行预上线，再预上线之前，对外不可见（Broker会有isIsolated标记自己的状态，当其为true时，不会像nameserver注册和发送心跳），因此也不会对外提供服务，二级消息的扫描流程也不会进行启动，具体预上线机制如下：\n\n1. Master Broker向NameServer获取Slave Broker地址（GetBrokerMemberGroup请求），但不注册\n2. Master Broker向Slave Broker发送自己的状态信息和地址\n3. Slave Broker得到Master Broker地址后和状态信息后，建立HA连接，并完成握手，进入Transfer状态\n4. Master Broker再完成握手后，反向获取备的元数据，包括消费位点、定时消息进度等，根据版本号决定是否更新。\n5. Master Broker对broker组内所有Slave Broker都完成1-4步操作后，正式上线，向NameServer注册，正式对外提供服务。\n\n### 锁Quorum\n\n当Slave代理Master时，外部看到的是“只读”的Master，因此顺序消息仍然可以对队列上锁，消费不会中断。但当真的Master重新上线后，在一定的时间差内可能会造成多个consumer锁定同一个队列，比如一个consumer仍然锁着代理的备某一个队列，一个consumer锁刚上线的主的同一队列，造成顺序消息的乱序和重复。\n\n因此在lock操作时要求，需锁broker副本组的大多数成员（quorum原则）均成功才算锁成功。但两副本下达不到quorum的原则，所以提供了lockInStrictMode参数，表示消费端消费顺序消息锁队列时是否使用严格模式。严格模式即对单个队列而言，需锁副本组的大多数成员（quorum原则）均成功才算锁成功，非严格模式即锁任意一副本成功就算锁成功，该参数默认为false。当对消息顺序性高于可用性时，需将该参数设置为false。\n\n## 配置更新\n\nNameserver\n\n- scanNotActiveBrokerInterval：扫描不活跃broker间隔，每次扫描将判断broker心跳是否超时，默认5s。\n- supportActingMaster：nameserver端是否支持Slave代理Master模式，开启后，副本组在无master状态下，brokerId==1的slave将在TopicRoute中被替换成master（即brokerId=0），并以只读模式对客户端提供服务，默认为false。\n\nBroker\n\n- enableSlaveActingMaster：broker端开启slave代理master模式总开关，默认为false。\n- enableRemoteEscape：是否允许远程逃逸，默认为false。\n- brokerHeartbeatInterval：broker向nameserver发送心跳间隔（不同于注册间隔），默认1s。\n- brokerNotActiveTimeoutMillis：broker不活跃超时时间，超过此时间nameserver仍未收到broker心跳，则判定broker下线，默认10s。\n- sendHeartbeatTimeoutMillis：broker发送心跳请求超时时间，默认1s。\n- lockInStrictMode：消费端消费顺序消息锁队列时是否使用严格模式，默认为false，上文已介绍。\n- skipPreOnline：broker跳过预上线流程，默认为false。\n- compatibleWithOldNameSrv：是否以兼容模式访问旧nameserver，默认为true。\n\n## 兼容性方案\n\n新版nameserver和旧版broker：新版nameserver可以完全兼容旧版broker，无兼容问题。\n\n旧版nameserver和新版Broker：新版Broker开启Slave代理Master，会向Nameserver发送 BROKER_HEARTBEAT以及GET_BROKER_MEMBER_GROUP请求，但由于旧版本nameserver无法处理这些请求。因此需要在brokerConfig中配置compatibleWithOldNameSrv=true，开启对旧版nameserver的兼容模式，在该模式下，broker的一些新增RPC将通过复用原有RequestCode实现，具体为：\n新增轻量级心跳将通过复用QUERY_DATA_VERSION实现\n新增获取BrokerMemberGroup数据将通过复用GET_ROUTEINFO_BY_TOPIC实现，具体实现方式是每个broker都会新增rmq_sys_{brokerName}的系统topic，通过获取该系统topic的路由来获取该副本组的存活信息。\n但旧版nameserver无法提供代理功能，Slave代理Master的功能将无法生效，但不影响其他功能。\n\n客户端对新旧版本的nameserver和broker均无兼容性问题。\n\n\n参考文档：[原RIP](https://github.com/apache/rocketmq/wiki/RIP-32-Slave-Acting-Master-Mode)"
  },
  {
    "path": "docs/cn/acl/RocketMQ_Multiple_ACL_Files_设计.md",
    "content": "# Version记录\n| 时间 | 主要内容 | 作者 |\n| --- | --- | --- |\n| 2022-01-27 | 初版，包括需求背景、兼容性影响、重要业务逻辑和后续扩展性考虑 | sunxi92 |\n\n中文文档在描述特定专业术语时，仍然使用英文。\n# 需求背景\nRocketMQ ACL特性目前只支持单个ACL配置文件，当存在很多用户时该配置文件会非常大，因此提出支持多ACL配置文件的想法。\n如果支持该特性那么也方便对RocketMQ用户进行分类。\n\n# 兼容性影响\n当前在支持多ACL配置文件特性的设计上是向前兼容的。\n\n# 重要业务逻辑\n## 1. ACL配置文件存储路径\nACL配置文件夹是在RocketMQ安装目录下的conf/acl目录中，也可以在该路径新建子目录并在子目录中新建ACL配置文件，同时也保留了之前默认的配置文件conf/plain_acl.yml。\n注意：目前用户还不能自定义配置文件目录。\n## 2. ACL配置文件更新\n热感知：当检测到ACL配置文件改动会自动刷新数据，判断ACL配置文件是否发生变化的依据是文件的修改时间是否发生变化\n## 3. RocketMQ Broker缓存ACL配置信息数据结构设计\n- aclPlainAccessResourceMap\n\naclPlainAccessResourceMap是个Map类型，用来缓存所有ACL配置文件的权限数据，其中key表示ACL配置文件的绝对路径，\nvalue表示相应配置文件中的权限数据，需要注意的是value也是一个Map类型，其中key是String类型表示AccessKey，value是PlainAccessResource类型。\n- accessKeyTable\n\naccessKeyTable是个Map类型，用来缓存AccessKey和ACL配置文件的映射关系，其中key表示AccessKey，value表示ACL配置文件的绝对路径。\n- globalWhiteRemoteAddressStrategy\n\nglobalWhiteRemoteAddressStrategy用来缓存所有ACL配置文件的全局白名单。\n- globalWhiteRemoteAddressStrategyMap\n\nglobalWhiteRemoteAddressStrategyMap是个Map类型，用来缓存ACL配置文件和全局白名单的映射关系\n- dataVersionMap\n\ndataVersionMap是个Map类型，用来缓存所有ACL配置文件的DataVersion，其中key表示ACL配置文件的绝对路径，value表示该配置文件对应的DataVersion。\n## 4.加载和监控ACL配置文件\n### 4.1 加载ACL配置文件\n- load()\n\nload()方法会获取\"RocketMQ安装目录/conf/acl\"目录（包括该目录的子目录）和\"rocketmq.acl.plain.file\"下所有ACL配置文件，然后遍历这些文件读取权限数据和全局白名单。\n- load(String aclFilePath)\n\nload(String aclFilePath)方法完成加载指定ACL配置文件内容的功能，将配置文件中的全局白名单globalWhiteRemoteAddresses和用户权限accounts加载到缓存中，\n这里需要注意以下几点：\n\n（1）判断缓存中该配置文件的全局白名单globalWhiteRemoteAddresses和用户权限accounts数据是否为空，如果不为空则需要注意删除文件原有数据\n\n（2）相同的accessKey只允许存在在一个ACL配置文件中\n### 4.2 监控ACL配置文件\nwatch()方法用来监控\"RocketMQ安装目录/conf/acl\"目录下所有ACL配置文件和\"rocketmq.acl.plain.file\"是否发生变化，变化考虑两种情况：一种是ACL配置文件的数量发生变化，\n此时会调用load()方法重新加载所有配置文件的数据；一种是配置文件的内容发生变化；具体完成监控ACL配置文件变化的是AclFileWatchService服务，\n该服务是一个线程，当启动该服务后它会以WATCH_INTERVAL（该参数目前设置为5秒，目前还不能在Broker配置文件中设置）的时间间隔来执行其核心逻辑。在该服务中会记录其监控的ACL配置文件目录aclPath、\nACL配置文件的数量aclFilesNum、所有ACL配置文件绝对路径fileList以及每个ACL配置文件最近一次修改的时间fileLastModifiedTime\n（Map类型，key为ACL配置文件的绝对路径，value为其最近一次修改时间）。\n该服务的核心逻辑如下：\n获取ACL配置文件数量并和aclFilesNum进行比较是否相等，如果不相等则更新aclFilesNum和fileList并调用load()方法重新加载所有配置文件；\n如果相等则遍历每个ACL配置文件，获取其最近一次修改的时间，并将该时间与fileLastModifiedTime中记录的时间进行比较，如果不相等则表示该文件发生过修改，\n此时调用load(String aclFilePath)方法重新加载该配置文件。\n\n## 5. 权限数据相关操作修改\n（1） updateAclConfigFileVersion(Map<String, Object> updateAclConfigMap)\n\n添加对缓存dataVersionMap的修改\n\n（2）updateAccessConfig(PlainAccessConfig plainAccessConfig)\n\n将该方法原有的逻辑修改为：首先判断accessKeyTable中是否包含待修改的accessKey，如果包含则根据accessKey来获取其对应的ACL配置文件绝对路径，\n再根据该路径更新aclPlainAccessResourceMap中缓存的数据，最后将该ACL配置文件中的数据写回原文件；如果不包含则会将数据写到\"rocketmq.acl.plain.file\"配置文件中，\n然后更新accessKeyTable和aclPlainAccessResourceMap，最后最后将该ACL配置文件中的数据写回原文件。\n\n（3）deleteAccessConfig(String accessKey)\n\n将该方法原有的逻辑修改为：判断accessKeyTable中是否存在accessKey，如果不存在则返回false，否则将其删除并将修改后的数据写回原文件。\n\n（4）getAllAclConfig()\n\nfileList中存储了所有ACL配置文件的绝对路径，遍历fileList分别从各ACL配置文件中读取数据并组装返回\n\n（5）updateGlobalWhiteAddrsConfig(List<String> globalWhiteAddrsList, String fileName)\n\n该方法是新增的，完成功能是修改指定ACL配置文件的全局白名单，为后续添加相关运维命令做准备\n## 6. ACL相关运维命令修改\n（1）ClusterAclConfigVersionListSubCommand\n\n将printClusterBaseInfo(final DefaultMQAdminExt defaultMQAdminExt, final String addr)方法原有的逻辑修改为：\n获取全部的ACL配置文件的DataVersion并输出。注意：获取的全部ACL配置文件的DataVersion集合可能为空，这里需要添加判断\n\n（2）GetBrokerAclConfigResponseHeader\n\n在GetBrokerAclConfigResponseHeader中新增allAclFileVersion字段，它是个Map类型，其key表示ACL配置文件的绝对路径，value表示对应ACL配置文件的DataVersion\n\n（3）ClusterAclVersionInfo\n\n在ClusterAclVersionInfo中废弃了aclConfigDataVersion属性，增加了allAclConfigDataVersion属性，该属性是个Map类型，用来存储所有ACL配置文件的版本数据，\n其中key表示ACL配置文件的绝对路径，value表示对应ACL配置文件的DataVersion\n\n## 7. 关于ACL配置文件DataVersion存储修改\n\n在原来版本中ACL权限数据存储在一个配置文件中，所以只记录了该配置文件的DataVersion，而现在需要支持多个配置文件特性，每个配置文件都有自己的DataVersion，\n为了能够准确记录所有配置文件的DataVersion，需要调整相关类型的属性、接口及方法。\n\n（1）PlainPermissionManager\n\n对PlainPermissionManager属性的修改具体如下：\n\n- 废弃dataVersion属性，该属性在历史版本中是用来存来存储默认ACL配置文件的DataVersion\n\n- 新增dataVersionMap属性用来缓存所有ACL配置文件的DataVersion，它是一个Map类型，其key表示ACL配置文件的绝对路径，value表示对应配置文件的DataVersion\n\n（2）AccessValidator\n\n对AccessValidator的修改如下：\n\n- 废弃String getAclConfigVersion();，该接口原来是获取ACL配置文件文件的版本数据\n\n- 新增Map<String, DataVersion> getAllAclConfigVersion();该接口是用来获取所有ACL配置文件的版本数据，接口会返回一个Map类型数据，\nkey表示各ACL配置文件的绝对路径，value表示对应配置文件的版本数据\n\n（3）PlainAccessValidator\n\n由于PlainAccessValidator实现了AccessValidator接口，所以相应地增加了getAllAclConfigVersion()方法\n\n# 后续扩展性考虑\n1.目前的修改只支持ACL配置文件存储在\"RocketMQ安装目录/conf/acl\"目录下，后续可以考虑支持多目录；\n\n2.目前ACL配置文件路径是不支持让用户指定，后续可以考虑让用户指定指定ACL配置文件的存储路径\n\n3.当前updateGlobalWhiteAddrsConfig命令只支持修改\"rocketmq.acl.plain.file\"文件中全局白名单，\n后续可以扩展为修改指定ACL配置文件的全局白名单（如果参数中没有传ACL配置文件则会修改\"rocketmq.acl.plain.file\"文件）\n\n4.目前ACL数据中的secretKey是以明文形式存储在文件中，在一些对此类信息敏感的行业是不允许以明文落地，后续可以考虑安全性问题\n\n5.目前ACL数据存储只支持文件形式存储，后续可以考虑增加数据库存储\n\n\n\n"
  },
  {
    "path": "docs/cn/acl/user_guide.md",
    "content": "# 权限控制\n----\n\n\n## 1.权限控制特性介绍\n权限控制（ACL）主要为RocketMQ提供Topic资源级别的用户访问控制。用户在使用RocketMQ权限控制时，可以在Client客户端通过 RPCHook注入AccessKey和SecretKey签名；同时，将对应的权限控制属性（包括Topic访问权限、IP白名单和AccessKey和SecretKey签名等）设置在distribution/conf/plain_acl.yml的配置文件中。Broker端对AccessKey所拥有的权限进行校验，校验不过，抛出异常；\nACL客户端可以参考：**org.apache.rocketmq.example.simple**包下面的**AclClient**代码。\n\n## 2. 权限控制的定义与属性值\n### 2.1权限定义\n对RocketMQ的Topic资源访问权限控制定义主要如下表所示，分为以下四种\n\n\n| 权限 | 含义 |\n| --- | --- |\n| DENY | 拒绝 |\n| ANY | PUB 或者 SUB 权限 |\n| PUB | 发送权限 |\n| SUB | 订阅权限 |\n\n### 2.2 权限定义的关键属性\n| 字段 | 取值 | 含义 |\n| --- | --- | --- |\n| globalWhiteRemoteAddresses | \\*;192.168.\\*.\\*;192.168.0.1 | 全局IP白名单 |\n| accessKey | 字符串 | Access Key |\n| secretKey | 字符串 | Secret Key |\n| whiteRemoteAddress | \\*;192.168.\\*.\\*;192.168.0.1 | 用户IP白名单 |\n| admin | true;false | 是否管理员账户 |\n| defaultTopicPerm | DENY;PUB;SUB;PUB\\|SUB | 默认的Topic权限 |\n| defaultGroupPerm | DENY;PUB;SUB;PUB\\|SUB | 默认的ConsumerGroup权限 |\n| topicPerms | topic=权限 | 各个Topic的权限 |\n| groupPerms | group=权限 | 各个ConsumerGroup的权限 |\n\n具体可以参考**distribution/conf/plain_acl.yml**配置文件\n\n## 3. 支持权限控制的集群部署\n在**distribution/conf/plain_acl.yml**配置文件中按照上述说明定义好权限属性后，打开**aclEnable**开关变量即可开启RocketMQ集群的ACL特性。这里贴出Broker端开启ACL特性的properties配置文件内容：\n```\nbrokerClusterName=DefaultCluster\nbrokerName=broker-a\nbrokerId=0\ndeleteWhen=04\nfileReservedTime=48\nbrokerRole=ASYNC_MASTER\nflushDiskType=ASYNC_FLUSH\nstorePathRootDir=/data/rocketmq/rootdir-a-m\nstorePathCommitLog=/data/rocketmq/commitlog-a-m\nautoCreateSubscriptionGroup=true\n## if acl is open,the flag will be true\naclEnable=true\nlistenPort=10911\nbrokerIP1=XX.XX.XX.XX1\nnamesrvAddr=XX.XX.XX.XX:9876\n```\n\n## 4. 权限控制主要流程\nACL主要流程分为两部分，主要包括权限解析和权限校验。\n\n### 4.1 权限解析\nBroker端对客户端的RequestCommand请求进行解析，拿到需要鉴权的属性字段。\n主要包括：\n（1）AccessKey：类似于用户名，代指用户主体，权限数据与之对应；\n（2）Signature：客户根据 SecretKey 签名得到的串，服务端再用SecretKey进行签名验证；\n\n### 4.2 权限校验\nBroker端对权限的校验逻辑主要分为以下几步：\n（1）检查是否命中全局 IP 白名单；如果是，则认为校验通过；否则走 2；\n（2）检查是否命中用户 IP 白名单；如果是，则认为校验通过；否则走 3；\n（3）校验签名，校验不通过，抛出异常；校验通过，则走 4；\n（4）对用户请求所需的权限 和 用户所拥有的权限进行校验；不通过，抛出异常； \n用户所需权限的校验需要注意已下内容：\n（1）特殊的请求例如 UPDATE_AND_CREATE_TOPIC 等，只能由 admin 账户进行操作；\n（2）对于某个资源，如果有显性配置权限，则采用配置的权限；如果没有显性配置权限，则采用默认的权限；\n\n## 5. 热加载修改后权限控制定义\nRocketMQ的权限控制存储的默认实现是基于yml配置文件。用户可以动态修改权限控制定义的属性，而不需重新启动Broker服务节点。\n\n## 6. 权限控制的使用限制\n(1)如果ACL与高可用部署(Master/Slave架构)同时启用，那么需要在Broker Master节点的distribution/conf/plain_acl.yml配置文件中\n设置全局白名单信息，即为将Slave节点的ip地址设置至Master节点plain_acl.yml配置文件的全局白名单中。\n\n(2)如果ACL与高可用部署(多副本Dledger架构)同时启用，由于出现节点宕机时，Dledger Group组内会自动选主，那么就需要将Dledger Group组\n内所有Broker节点的plain_acl.yml配置文件的白名单设置所有Broker节点的ip地址。\n\n## 7. ACL mqadmin配置管理命令\n\n### 7.1 更新ACL配置文件中“account”的属性值\n\n该命令的示例如下：\n\nsh mqadmin updateAclConfig -n 192.168.1.2:9876 -b 192.168.12.134:10911 -a RocketMQ -s 1234567809123 \n-t topicA=DENY,topicD=SUB -g groupD=DENY,groupB=SUB\n\n说明：如果不存在则会在ACL Config YAML配置文件中创建；若存在，则会更新对应的“accounts”的属性值;\n如果指定的是集群名称，则会在集群中各个broker节点执行该命令；否则会在单个broker节点执行该命令。\n\n| 参数 | 取值 | 含义 |\n| --- | --- | --- |\n| n | eg:192.168.1.2:9876 | namesrv地址(必填) |\n| c | eg:DefaultCluster | 指定集群名称(与broker地址二选一) |\n| b | eg:192.168.12.134:10911 | 指定broker地址(与集群名称二选一) |\n| a | eg:RocketMQ | Access Key值(必填) |\n| s | eg:1234567809123 | Secret Key值(可选) |\n| m | eg:true | 是否管理员账户(可选) |\n| w | eg:192.168.0.* | whiteRemoteAddress,用户IP白名单(可选) |\n| i | eg:DENY;PUB;SUB;PUB\\|SUB | defaultTopicPerm,默认Topic权限(可选) |\n| u | eg:DENY;PUB;SUB;PUB\\|SUB | defaultGroupPerm,默认ConsumerGroup权限(可选) |\n| t | eg:topicA=DENY,topicD=SUB | topicPerms,各个Topic的权限(可选) |\n| g | eg:groupD=DENY,groupB=SUB | groupPerms,各个ConsumerGroup的权限(可选) |\n\n### 7.2 删除ACL配置文件里面的对应“account”\n该命令的示例如下：\n\nsh mqadmin deleteAccessConfig -n 192.168.1.2:9876 -c DefaultCluster -a RocketMQ\n\n说明：如果指定的是集群名称，则会在集群中各个broker节点执行该命令；否则会在单个broker节点执行该命令。\n其中，参数\"a\"为Access Key的值，用以标识唯一账户id，因此该命令的参数中指定账户id即可。\n\n| 参数 | 取值 | 含义 |\n| --- | --- | --- |\n| n | eg:192.168.1.2:9876 | namesrv地址(必填) |\n| c | eg:DefaultCluster | 指定集群名称(与broker地址二选一) |\n| b | eg:192.168.12.134:10911 | 指定broker地址(与集群名称二选一) |\n| a | eg:RocketMQ | Access Key的值(必填) |\n\n\n### 7.3 更新ACL配置文件里面中的全局白名单\n该命令的示例如下：\n\nsh mqadmin updateGlobalWhiteAddr -n 192.168.1.2:9876 -b 192.168.12.134:10911 -g 10.10.154.1,10.10.154.2\n\n说明：如果指定的是集群名称，则会在集群中各个broker节点执行该命令；否则会在单个broker节点执行该命令。\n其中，参数\"g\"为全局IP白名的值，用以更新ACL配置文件中的“globalWhiteRemoteAddresses”字段的属性值。\n\n| 参数 | 取值 | 含义 |\n| --- | --- | --- |\n| n | eg:192.168.1.2:9876 | namesrv地址(必填) |\n| c | eg:DefaultCluster | 指定集群名称(与broker地址二选一) |\n| b | eg:192.168.12.134:10911 | 指定broker地址(与集群名称二选一) |\n| g | eg:10.10.154.1,10.10.154.2 | 全局IP白名单(必填) |\n\n### 7.4 查询集群/Broker的ACL配置文件版本信息\n该命令的示例如下：\n\nsh mqadmin clusterAclConfigVersion -n 192.168.1.2:9876 -c DefaultCluster\n\n说明：如果指定的是集群名称，则会在集群中各个broker节点执行该命令；否则会在单个broker节点执行该命令。\n\n| 参数 | 取值 | 含义 |\n| --- | --- | --- |\n| n | eg:192.168.1.2:9876 | namesrv地址(必填) |\n| c | eg:DefaultCluster | 指定集群名称(与broker地址二选一) |\n| b | eg:192.168.12.134:10911 | 指定broker地址(与集群名称二选一) |\n\n### 7.5 查询集群/Broker的ACL配置文件全部内容\n该命令的示例如下：\n\nsh mqadmin getAclConfig -n 192.168.1.2:9876 -c DefaultCluster\n\n说明：如果指定的是集群名称，则会在集群中各个broker节点执行该命令；否则会在单个broker节点执行该命令。\n\n| 参数 | 取值 | 含义 |\n| --- | --- | --- |\n| n | eg:192.168.1.2:9876 | namesrv地址(必填) |\n| c | eg:DefaultCluster | 指定集群名称(与broker地址二选一) |\n| b | eg:192.168.12.134:10911 | 指定broker地址(与集群名称二选一) |\n\n**特别注意**开启Acl鉴权认证后导致Master/Slave和Dledger模式下Broker同步数据异常的问题，\n在社区[4.5.1]版本中已经修复，具体的PR链接为：#1149。\n"
  },
  {
    "path": "docs/cn/architecture.md",
    "content": "# 架构设计\n---\n## 1 技术架构\n![](image/rocketmq_architecture_1.png)\n\nRocketMQ架构上主要分为四部分，如上图所示：\n\n- Producer：消息发布的角色，支持分布式集群方式部署。Producer通过MQ的负载均衡模块选择相应的Broker集群队列进行消息投递，投递的过程支持快速失败并且低延迟。\n\n- Consumer：消息消费的角色，支持分布式集群方式部署。支持以push推，pull拉两种模式对消息进行消费。同时也支持集群方式和广播方式的消费，它提供实时消息订阅机制，可以满足大多数用户的需求。\n\n- NameServer：NameServer是一个非常简单的Topic路由注册中心，其角色类似Dubbo中的zookeeper，支持Broker的动态注册与发现。主要包括两个功能：Broker管理，NameServer接受Broker集群的注册信息并且保存下来作为路由信息的基本数据。然后提供心跳检测机制，检查Broker是否还存活；路由信息管理，每个NameServer将保存关于Broker集群的整个路由信息和用于客户端查询的队列信息。然后Producer和Consumer通过NameServer就可以知道整个Broker集群的路由信息，从而进行消息的投递和消费。NameServer通常也是集群的方式部署，各实例间相互不进行信息通讯。Broker是向每一台NameServer注册自己的路由信息，所以每一个NameServer实例上面都保存一份完整的路由信息。当某个NameServer因某种原因下线了，Broker仍然可以向其它NameServer同步其路由信息，Producer和Consumer仍然可以动态感知Broker的路由的信息。 \n\n- BrokerServer：Broker主要负责消息的存储、投递和查询以及服务高可用保证，为了实现这些功能，Broker包含了以下几个重要子模块。\n    1. Remoting Module：整个Broker的实体，负责处理来自Client端的请求。\n    2. Client Manager：负责管理客户端(Producer/Consumer)和维护Consumer的Topic订阅信息。\n    3. Store Service：提供方便简单的API接口处理消息存储到物理硬盘和查询功能。\n    4. HA Service：高可用服务，提供Master Broker 和 Slave Broker之间的数据同步功能。\n    5. Index Service：根据特定的Message key对投递到Broker的消息进行索引服务，以提供消息的快速查询。\n\n![](image/rocketmq_architecture_2.png)\n\n## 2 部署架构\n\n\n![](image/rocketmq_architecture_3.png)\n\n\n### RocketMQ 网络部署特点\n\n- NameServer是一个几乎无状态节点，可集群部署，节点之间无任何信息同步。\n\n- Broker部署相对复杂，Broker分为Master与Slave，一个Master可以对应多个Slave，但是一个Slave只能对应一个Master，Master与Slave 的对应关系通过指定相同的BrokerName，不同的BrokerId 来定义，BrokerId为0表示Master，非0表示Slave。Master也可以部署多个。每个Broker与NameServer集群中的所有节点建立长连接，定时注册Topic信息到所有NameServer。 注意：当前RocketMQ版本在部署架构上支持一Master多Slave，但只有BrokerId=1的从服务器才会参与消息的读负载。\n\n- Producer与NameServer集群中的其中一个节点（随机选择）建立长连接，定期从NameServer获取Topic路由信息，并向提供Topic 服务的Master建立长连接，且定时向Master发送心跳。Producer完全无状态，可集群部署。\n\n- Consumer与NameServer集群中的其中一个节点（随机选择）建立长连接，定期从NameServer获取Topic路由信息，并向提供Topic服务的Master、Slave建立长连接，且定时向Master、Slave发送心跳。Consumer既可以从Master订阅消息，也可以从Slave订阅消息，消费者在向Master拉取消息时，Master服务器会根据拉取偏移量与最大偏移量的距离（判断是否读老消息，产生读I/O），以及从服务器是否可读等因素建议下一次是从Master还是Slave拉取。\n\n结合部署架构图，描述集群工作流程：\n\n- 启动NameServer，NameServer起来后监听端口，等待Broker、Producer、Consumer连上来，相当于一个路由控制中心。\n- Broker启动，跟所有的NameServer保持长连接，定时发送心跳包。心跳包中包含当前Broker信息(IP+端口等)以及存储所有Topic信息。注册成功后，NameServer集群中就有Topic跟Broker的映射关系。\n- 收发消息前，先创建Topic，创建Topic时需要指定该Topic要存储在哪些Broker上，也可以在发送消息时自动创建Topic。\n- Producer发送消息，启动时先跟NameServer集群中的其中一台建立长连接，并从NameServer中获取当前发送的Topic存在哪些Broker上，轮询从队列列表中选择一个队列，然后与队列所在的Broker建立长连接从而向Broker发消息。\n- Consumer跟Producer类似，跟其中一台NameServer建立长连接，获取当前订阅Topic存在哪些Broker上，然后直接跟Broker建立连接通道，开始消费消息。\n"
  },
  {
    "path": "docs/cn/best_practice.md",
    "content": "#  最佳实践\r\n\r\n---\r\n## 1   生产者\r\n\r\n### 1.1 发送消息注意事项\r\n\r\n#### 1  Tags的使用\r\n\r\n一个应用尽可能用一个Topic，而消息子类型则可以用tags来标识。tags可以由应用自由设置，只有生产者在发送消息设置了tags，消费方在订阅消息时才可以利用tags通过broker做消息过滤：message.setTags(\"TagA\")。  \r\n \r\n#### 2 Keys的使用\r\n\r\n每个消息在业务层面的唯一标识码要设置到keys字段，方便将来定位消息丢失问题。服务器会为每个消息创建索引（哈希索引），应用可以通过topic、key来查询这条消息内容，以及消息被谁消费。由于是哈希索引，请务必保证key尽可能唯一，这样可以避免潜在的哈希冲突。\r\n\r\n\r\n```java\r\n   // 订单Id   \r\n   String orderId = \"20034568923546\";   \r\n   message.setKeys(orderId);   \r\n```\r\n#### 3 日志的打印\r\n\r\n​消息发送成功或者失败要打印消息日志，务必要打印SendResult和key字段。send消息方法只要不抛异常，就代表发送成功。发送成功会有多个状态，在sendResult里定义。以下对每个状态进行说明：     \r\n\r\n- **SEND_OK**\r\n\r\n消息发送成功。要注意的是消息发送成功也不意味着它是可靠的。要确保不会丢失任何消息，还应启用同步Master服务器或同步刷盘，即SYNC_MASTER或SYNC_FLUSH。\r\n\r\n\r\n- **FLUSH_DISK_TIMEOUT**\r\n\r\n消息发送成功但是服务器刷盘超时。此时消息已经进入服务器队列（内存），只有服务器宕机，消息才会丢失。消息存储配置参数中可以设置刷盘方式和同步刷盘时间长度，如果Broker服务器设置了刷盘方式为同步刷盘，即FlushDiskType=SYNC_FLUSH（默认为异步刷盘方式），当Broker服务器未在同步刷盘时间内（默认为5s）完成刷盘，则将返回该状态——刷盘超时。\r\n\r\n- **FLUSH_SLAVE_TIMEOUT**\r\n\r\n消息发送成功，但是服务器同步到Slave时超时。此时消息已经进入服务器队列，只有服务器宕机，消息才会丢失。如果Broker服务器的角色是同步Master，即SYNC_MASTER（默认是异步Master即ASYNC_MASTER），并且从Broker服务器未在同步刷盘时间（默认为5秒）内完成与主服务器的同步，则将返回该状态——数据同步到Slave服务器超时。\r\n\r\n- **SLAVE_NOT_AVAILABLE**\r\n\r\n消息发送成功，但是此时Slave不可用。如果Broker服务器的角色是同步Master，即SYNC_MASTER（默认是异步Master服务器即ASYNC_MASTER），但没有配置slave Broker服务器，则将返回该状态——无Slave服务器可用。\r\n\r\n\r\n### 1.2 消息发送失败处理方式\r\n\r\nProducer的send方法本身支持内部重试，重试逻辑如下：\r\n\r\n- 至多重试2次。\r\n- 如果同步模式发送失败，则轮转到下一个Broker，如果异步模式发送失败，则只会在当前Broker进行重试。这个方法的总耗时时间不超过sendMsgTimeout设置的值，默认10s。\r\n- 如果本身向broker发送消息产生超时异常，就不会再重试。\r\n\r\n以上策略也是在一定程度上保证了消息可以发送成功。如果业务对消息可靠性要求比较高，建议应用增加相应的重试逻辑：比如调用send同步方法发送失败时，则尝试将消息存储到db，然后由后台线程定时重试，确保消息一定到达Broker。\r\n\r\n上述db重试方式为什么没有集成到MQ客户端内部做，而是要求应用自己去完成，主要基于以下几点考虑：首先，MQ的客户端设计为无状态模式，方便任意的水平扩展，且对机器资源的消耗仅仅是cpu、内存、网络。其次，如果MQ客户端内部集成一个KV存储模块，那么数据只有同步落盘才能较可靠，而同步落盘本身性能开销较大，所以通常会采用异步落盘，又由于应用关闭过程不受MQ运维人员控制，可能经常会发生 kill -9 这样暴力方式关闭，造成数据没有及时落盘而丢失。第三，Producer所在机器的可靠性较低，一般为虚拟机，不适合存储重要数据。综上，建议重试过程交由应用来控制。\r\n\r\n### 1.3选择oneway形式发送\r\n通常消息的发送是这样一个过程：\r\n\r\n- 客户端发送请求到服务器\r\n- 服务器处理请求\r\n- 服务器向客户端返回应答\r\n\r\n所以，一次消息发送的耗时时间是上述三个步骤的总和，而某些场景要求耗时非常短，但是对可靠性要求并不高，例如日志收集类应用，此类应用可以采用oneway形式调用，oneway形式只发送请求不等待应答，而发送请求在客户端实现层面仅仅是一个操作系统系统调用的开销，即将数据写入客户端的socket缓冲区，此过程耗时通常在微秒级。\r\n\r\n\r\n## 2   消费者\r\n\r\n### 2.1 消费过程幂等\r\n\r\nRocketMQ无法避免消息重复（Exactly-Once），所以如果业务对消费重复非常敏感，务必要在业务层面进行去重处理。可以借助关系数据库进行去重。首先需要确定消息的唯一键，可以是msgId，也可以是消息内容中的唯一标识字段，例如订单Id等。在消费之前判断唯一键是否在关系数据库中存在。如果不存在则插入，并消费，否则跳过。（实际过程要考虑原子性问题，判断是否存在可以尝试插入，如果报主键冲突，则插入失败，直接跳过）\r\n\r\nmsgId一定是全局唯一标识符，但是实际使用中，可能会存在相同的消息有两个不同msgId的情况（消费者主动重发、因客户端重投机制导致的重复等），这种情况就需要使业务字段进行重复消费。\r\n\r\n### 2.2 消费速度慢的处理方式\r\n\r\n#### 1 提高消费并行度\r\n\r\n绝大部分消息消费行为都属于 IO 密集型，即可能是操作数据库，或者调用 RPC，这类消费行为的消费速度在于后端数据库或者外系统的吞吐量，通过增加消费并行度，可以提高总的消费吞吐量，但是并行度增加到一定程度，反而会下降。所以，应用必须要设置合理的并行度。 如下有几种修改消费并行度的方法：\r\n\r\n- 同一个 ConsumerGroup 下，通过增加 Consumer 实例数量来提高并行度（需要注意的是超过订阅队列数的 Consumer 实例无效）。可以通过加机器，或者在已有机器启动多个进程的方式。\r\n- 提高单个 Consumer 的消费并行线程，通过修改参数 consumeThreadMin、consumeThreadMax实现。\r\n\r\n#### 2   批量方式消费\r\n\r\n某些业务流程如果支持批量方式消费，则可以很大程度上提高消费吞吐量，例如订单扣款类应用，一次处理一个订单耗时 1 s，一次处理 10 个订单可能也只耗时 2 s，这样即可大幅度提高消费的吞吐量，通过设置 consumer的 consumeMessageBatchMaxSize 返个参数，默认是 1，即一次只消费一条消息，例如设置为 N，那么每次消费的消息数小于等于 N。\r\n\r\n#### 3   跳过非重要消息\r\n\r\n发生消息堆积时，如果消费速度一直追不上发送速度，如果业务对数据要求不高的话，可以选择丢弃不重要的消息。例如，当某个队列的消息数堆积到100000条以上，则尝试丢弃部分或全部消息，这样就可以快速追上发送消息的速度。示例代码如下：\r\n\r\n```java\r\n    public ConsumeConcurrentlyStatus consumeMessage(\r\n            List<MessageExt> msgs,\r\n            ConsumeConcurrentlyContext context) {\r\n        long offset = msgs.get(0).getQueueOffset();\r\n        String maxOffset =\r\n                msgs.get(0).getProperty(Message.PROPERTY_MAX_OFFSET);\r\n        long diff = Long.parseLong(maxOffset) - offset;\r\n        if (diff > 100000) {\r\n            // TODO 消息堆积情况的特殊处理\r\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\r\n        }\r\n        // TODO 正常消费过程\r\n        return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\r\n    }    \r\n```\r\n\r\n\r\n#### 4 优化每条消息消费过程     \r\n \r\n举例如下，某条消息的消费过程如下：\r\n\r\n- 根据消息从 DB 查询【数据 1】\r\n- 根据消息从 DB 查询【数据 2】\r\n- 复杂的业务计算\r\n- 向 DB 插入【数据 3】\r\n- 向 DB 插入【数据 4】\r\n\r\n这条消息的消费过程中有4次与 DB的 交互，如果按照每次 5ms 计算，那么总共耗时 20ms，假设业务计算耗时 5ms，那么总过耗时 25ms，所以如果能把 4 次 DB 交互优化为 2 次，那么总耗时就可以优化到 15ms，即总体性能提高了 40%。所以应用如果对时延敏感的话，可以把DB部署在SSD硬盘，相比于SCSI磁盘，前者的RT会小很多。\r\n\r\n### 2.3 消费打印日志\r\n\r\n如果消息量较少，建议在消费入口方法打印消息，消费耗时等，方便后续排查问题。\r\n\r\n\r\n```java\r\n   public ConsumeConcurrentlyStatus consumeMessage(\r\n            List<MessageExt> msgs,\r\n            ConsumeConcurrentlyContext context) {\r\n        log.info(\"RECEIVE_MSG_BEGIN: \" + msgs.toString());\r\n        // TODO 正常消费过程\r\n        return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\r\n    }   \r\n```\r\n\r\n如果能打印每条消息消费耗时，那么在排查消费慢等线上问题时，会更方便。\r\n\r\n### 2.4 其他消费建议\r\n\r\n#### 1 关于消费者和订阅\r\n\r\n​第一件需要注意的事情是，不同的消费者组可以独立的消费一些 topic，并且每个消费者组都有自己的消费偏移量，请确保同一组内的每个消费者订阅信息保持一致。\r\n\r\n#### 2 关于有序消息\r\n\r\n消费者将锁定每个消息队列，以确保他们被逐个消费，虽然这将会导致性能下降，但是当你关心消息顺序的时候会很有用。我们不建议抛出异常，你可以返回 ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT 作为替代。\r\n\r\n#### 3 关于并发消费\r\n\r\n顾名思义，消费者将并发消费这些消息，建议你使用它来获得良好性能，我们不建议抛出异常，你可以返回 ConsumeConcurrentlyStatus.RECONSUME_LATER 作为替代。\r\n\r\n#### 4 关于消费状态Consume Status\r\n\r\n对于并发的消费监听器，你可以返回 RECONSUME_LATER 来通知消费者现在不能消费这条消息，并且希望可以稍后重新消费它。然后，你可以继续消费其他消息。对于有序的消息监听器，因为你关心它的顺序，所以不能跳过消息，但是你可以返回SUSPEND_CURRENT_QUEUE_A_MOMENT 告诉消费者等待片刻。\r\n\r\n#### 5 关于Blocking\r\n\r\n不建议阻塞监听器，因为它会阻塞线程池，并最终可能会终止消费进程\r\n\r\n#### 6 关于线程数设置     \r\n\r\n消费者使用 ThreadPoolExecutor 在内部对消息进行消费，所以你可以通过设置 setConsumeThreadMin 或 setConsumeThreadMax 来改变它。\r\n\r\n#### 7 关于消费位点\r\n\r\n当建立一个新的消费者组时，需要决定是否需要消费已经存在于 Broker 中的历史消息CONSUME_FROM_LAST_OFFSET 将会忽略历史消息，并消费之后生成的任何消息。CONSUME_FROM_FIRST_OFFSET 将会消费每个存在于 Broker 中的信息。你也可以使用 CONSUME_FROM_TIMESTAMP 来消费在指定时间戳后产生的消息。\r\n\r\n\r\n\r\n\r\n\r\n## 3   Broker\r\n\r\n### 3.1 Broker 角色\r\n​  Broker 角色分为 ASYNC_MASTER（异步主机）、SYNC_MASTER（同步主机）以及SLAVE（从机）。如果对消息的可靠性要求比较严格，可以采用 SYNC_MASTER加SLAVE的部署方式。如果对消息可靠性要求不高，可以采用ASYNC_MASTER加SLAVE的部署方式。如果只是测试方便，则可以选择仅ASYNC_MASTER或仅SYNC_MASTER的部署方式。\r\n### 3.2 FlushDiskType\r\n​ SYNC_FLUSH（同步刷新）相比于ASYNC_FLUSH（异步处理）会损失很多性能，但是也更可靠，所以需要根据实际的业务场景做好权衡。\r\n### 3.3 Broker 配置\r\n\r\n| 参数名                           | 默认值                        | 说明                                                         |\r\n| -------------------------------- | ----------------------------- | ------------------------------------------------------------ |\r\n| listenPort                    | 10911              | 接受客户端连接的监听端口 |\r\n| namesrvAddr       | null                         | nameServer 地址     |\r\n| brokerIP1 | 网卡的 InetAddress                         | 当前 broker 监听的 IP  |\r\n| brokerIP2 | 跟 brokerIP1 一样                         | 存在主从 broker 时，如果在 broker 主节点上配置了 brokerIP2 属性，broker 从节点会连接主节点配置的 brokerIP2 进行同步  |\r\n| brokerName        | null                         | broker 的名称                           |\r\n| brokerClusterName                     | DefaultCluster                  | 本 broker 所属的 Cluster 名称           |\r\n| brokerId             | 0                              | broker id，0 表示 master，其他的正整数表示 slave                                                 |\r\n| storePathRootDir                         | $HOME/store/                   | 存储根路径                                            |\r\n| storePathCommitLog                      | $HOME/store/commitlog/                              | 存储 commit log 的路径                                                |\r\n| mappedFileSizeCommitLog     | 1024 * 1024 * 1024(1G) | commit log 的映射文件大小                                       |​ \r\n| deleteWhen     | 04 | 在每天的什么时间删除已经超过文件保留时间的 commit log                                        |​ \r\n| fileReservedTime     | 72 | 以小时计算的文件保留时间                                        |​ \r\n| brokerRole     | ASYNC_MASTER | SYNC_MASTER/ASYNC_MASTER/SLAVE                                        |​ \r\n| flushDiskType     | ASYNC_FLUSH | SYNC_FLUSH/ASYNC_FLUSH SYNC_FLUSH 模式下的 broker 保证在收到确认生产者之前将消息刷盘。ASYNC_FLUSH 模式下的 broker 则利用刷盘一组消息的模式，可以取得更好的性能。                                       |​\r\n\r\n## 4  NameServer\r\n\r\n​RocketMQ 中，Name Servers 被设计用来做简单的路由管理。其职责包括：\r\n\r\n- Brokers 定期向每个名称服务器注册路由数据。\r\n- 名称服务器为客户端，包括生产者，消费者和命令行客户端提供最新的路由信息。\r\n​     \r\n​\r\n\r\n## 5 客户端配置\r\n\r\n​ 相对于RocketMQ的Broker集群，生产者和消费者都是客户端。本小节主要描述生产者和消费者公共的行为配置。\r\n\r\n### 5.1 客户端寻址方式\r\n\r\nRocketMQ可以令客户端找到Name Server，然后通过Name Server再找到Broker。如下所示有多种配置方式，优先级由高到低，高优先级会覆盖低优先级。\r\n\r\n- 代码中指定Name Server地址，多个namesrv地址之间用分号分割   \r\n\r\n```java\r\nproducer.setNamesrvAddr(\"192.168.0.1:9876;192.168.0.2:9876\");  \r\n\r\nconsumer.setNamesrvAddr(\"192.168.0.1:9876;192.168.0.2:9876\");\r\n```\r\n- Java启动参数中指定Name Server地址\r\n\r\n```text\r\n-Drocketmq.namesrv.addr=192.168.0.1:9876;192.168.0.2:9876  \r\n```\r\n- 环境变量指定Name Server地址\r\n\r\n```text\r\nexport   NAMESRV_ADDR=192.168.0.1:9876;192.168.0.2:9876   \r\n```\r\n- HTTP静态服务器寻址（默认）\r\n\r\n客户端启动后，会定时访问一个静态HTTP服务器，地址如下：<http://jmenv.tbsite.net:8080/rocketmq/nsaddr>，这个URL的返回内容如下：\r\n\r\n```text\r\n192.168.0.1:9876;192.168.0.2:9876   \r\n```\r\n客户端默认每隔2分钟访问一次这个HTTP服务器，并更新本地的Name Server地址。URL已经在代码中硬编码，可通过修改/etc/hosts文件来改变要访问的服务器，例如在/etc/hosts增加如下配置：\r\n```text\r\n10.232.22.67    jmenv.tbsite.net   \r\n```\r\n推荐使用HTTP静态服务器寻址方式，好处是客户端部署简单，且Name Server集群可以热升级。\r\n\r\n### 5.2 客户端配置\r\n\r\nDefaultMQProducer、TransactionMQProducer、DefaultMQPushConsumer、DefaultMQPullConsumer都继承于ClientConfig类，ClientConfig为客户端的公共配置类。客户端的配置都是get、set形式，每个参数都可以用spring来配置，也可以在代码中配置，例如namesrvAddr这个参数可以这样配置，producer.setNamesrvAddr(\"192.168.0.1:9876\")，其他参数同理。\r\n\r\n#### 1  客户端的公共配置\r\n\r\n| 参数名                        | 默认值  | 说明                                                         |\r\n| ----------------------------- | ------- | ------------------------------------------------------------ |\r\n| namesrvAddr                   |         | Name Server地址列表，多个NameServer地址用分号隔开            |\r\n| clientIP                      | 本机IP  | 客户端本机IP地址，某些机器会发生无法识别客户端IP地址情况，需要应用在代码中强制指定 |\r\n| instanceName                  | DEFAULT | 客户端实例名称，客户端创建的多个Producer、Consumer实际是共用一个内部实例（这个实例包含网络连接、线程资源等） |\r\n| clientCallbackExecutorThreads | 4       | 通信层异步回调线程数                                         |\r\n| pollNameServerInterval        | 30000   | 轮询Name Server间隔时间，单位毫秒                            |\r\n| heartbeatBrokerInterval       | 30000   | 向Broker发送心跳间隔时间，单位毫秒                           |\r\n| persistConsumerOffsetInterval | 5000    | 持久化Consumer消费进度间隔时间，单位毫秒                     |\r\n\r\n#### 2  Producer配置\r\n\r\n| 参数名                           | 默认值           | 说明                                                         |\r\n| -------------------------------- | ---------------- | ------------------------------------------------------------ |\r\n| producerGroup                    | DEFAULT_PRODUCER | Producer组名，多个Producer如果属于一个应用，发送同样的消息，则应该将它们归为同一组 |\r\n| createTopicKey                   | TBW102           | 在发送消息时，自动创建服务器不存在的topic，需要指定Key，该Key可用于配置发送消息所在topic的默认路由。 |\r\n| defaultTopicQueueNums            | 4                | 在发送消息，自动创建服务器不存在的topic时，默认创建的队列数  |\r\n| sendMsgTimeout                   | 3000             | 发送消息超时时间，单位毫秒                                   |\r\n| compressMsgBodyOverHowmuch       | 4096             | 消息Body超过多大开始压缩（Consumer收到消息会自动解压缩），单位字节 |\r\n| retryAnotherBrokerWhenNotStoreOK | FALSE            | 如果发送消息返回sendResult，但是sendStatus!=SEND_OK，是否重试发送 |\r\n| retryTimesWhenSendFailed         | 2                | 如果消息发送失败，最大重试次数，该参数只对同步发送模式起作用 |\r\n| maxMessageSize                   | 4MB              | 客户端限制的消息体大小，超过报错，同时服务端也会限制，所以需要跟服务端配合使用。 |\r\n| transactionCheckListener         |                  | 事务消息回查监听器，如果发送事务消息，必须设置               |\r\n| checkThreadPoolMinSize           | 1                | Broker回查Producer事务状态时，线程池最小线程数                     |\r\n| checkThreadPoolMaxSize           | 1                | Broker回查Producer事务状态时，线程池最大线程数                     |\r\n| checkRequestHoldMax              | 2000             | Broker回查Producer事务状态时，Producer本地缓冲请求队列大小   |\r\n| RPCHook                          | null             | 该参数是在Producer创建时传入的，包含消息发送前的预处理和消息响应后的处理两个接口，用户可以在第一个接口中做一些安全控制或者其他操作。 |\r\n\r\n#### 3  PushConsumer配置\r\n\r\n| 参数名                       | 默认值                        | 说明                                                         |\r\n| ---------------------------- | ----------------------------- | ------------------------------------------------------------ |\r\n| consumerGroup                | DEFAULT_CONSUMER              | Consumer组名，多个Consumer如果属于一个应用，订阅同样的消息，且消费逻辑一致，则应该将它们归为同一组 |\r\n| messageModel                 | CLUSTERING                    | 消费模型支持集群消费和广播消费两种                           |\r\n| consumeFromWhere             | CONSUME_FROM_LAST_OFFSET      | Consumer启动后，默认从上次消费的位置开始消费，这包含两种情况：一种是上次消费的位置未过期，则消费从上次中止的位置进行；一种是上次消费位置已经过期，则从当前队列第一条消息开始消费 |\r\n| consumeTimestamp             | 半个小时前                    | 只有当consumeFromWhere值为CONSUME_FROM_TIMESTAMP时才起作用。 |\r\n| allocateMessageQueueStrategy | AllocateMessageQueueAveragely | Rebalance算法实现策略                                        |\r\n| subscription                 |                               | 订阅关系                                                     |\r\n| messageListener              |                               | 消息监听器                                                   |\r\n| offsetStore                  |                               | 消费进度存储                                                 |\r\n| consumeThreadMin             | 20                            | 消费线程池最小线程数                                               |\r\n| consumeThreadMax             | 20                            | 消费线程池最大线程数                                               |\r\n| consumeConcurrentlyMaxSpan   | 2000                          | 单队列并行消费允许的最大跨度                                 |\r\n| pullThresholdForQueue        | 1000                          | 拉消息本地队列缓存消息最大数                                 |\r\n| pullInterval                 | 0                             | 拉消息间隔，由于是长轮询，所以为0，但是如果应用为了流控，也可以设置大于0的值，单位毫秒 |\r\n| consumeMessageBatchMaxSize   | 1                             | 批量消费，一次消费多少条消息                                 |\r\n| pullBatchSize                | 32                            | 批量拉消息，一次最多拉多少条                                 |\r\n\r\n#### 4  PullConsumer配置\r\n\r\n| 参数名                           | 默认值                        | 说明                                                         |\r\n| -------------------------------- | ----------------------------- | ------------------------------------------------------------ |\r\n| consumerGroup                    | DEFAULT_CONSUMER              | Consumer组名，多个Consumer如果属于一个应用，订阅同样的消息，且消费逻辑一致，则应该将它们归为同一组 |\r\n| brokerSuspendMaxTimeMillis       | 20000                         | 长轮询，Consumer拉消息请求在Broker挂起最长时间，单位毫秒     |\r\n| consumerTimeoutMillisWhenSuspend | 30000                         | 长轮询，Consumer拉消息请求在Broker挂起超过指定时间，客户端认为超时，单位毫秒 |\r\n| consumerPullTimeoutMillis        | 10000                         | 非长轮询，拉消息超时时间，单位毫秒                           |\r\n| messageModel                     | BROADCASTING                  | 消息支持两种模式：集群消费和广播消费           |\r\n| messageQueueListener             |                               | 监听队列变化                                                 |\r\n| offsetStore                      |                               | 消费进度存储                                                 |\r\n| registerTopics                   |                               | 注册的topic集合                                              |\r\n| allocateMessageQueueStrategy     | AllocateMessageQueueAveragely | Rebalance算法实现策略                                        |\r\n\r\n#### 5  Message数据结构\r\n\r\n| 字段名         | 默认值 | 说明                                                         |\r\n| -------------- | ------ | ------------------------------------------------------------ |\r\n| Topic          | null   | 必填，消息所属topic的名称                                        |\r\n| Body           | null   | 必填，消息体                                                 |\r\n| Tags           | null   | 选填，消息标签，方便服务器过滤使用。目前只支持每个消息设置一个tag |\r\n| Keys           | null   | 选填，代表这条消息的业务关键词，服务器会根据keys创建哈希索引，设置后，可以在Console系统根据Topic、Keys来查询消息，由于是哈希索引，请尽可能保证key唯一，例如订单号，商品Id等。 |\r\n| Flag           | 0      | 选填，完全由应用来设置，RocketMQ不做干预                     |\r\n| DelayTimeLevel | 0      | 选填，消息延时级别，0表示不延时，大于0会延时特定的时间才会被消费 |\r\n| WaitStoreMsgOK | TRUE   | 选填，表示消息是否在服务器落盘后才返回应答。                 |\r\n\r\n## 6  系统配置\r\n\r\n本小节主要介绍系统（JVM/OS）相关的配置。\r\n\r\n### 6.1 JVM选项\r\n\r\n​ 推荐使用最新发布的JDK 1.8版本。通过设置相同的Xms和Xmx值来防止JVM调整堆大小以获得更好的性能。简单的JVM配置如下所示：\r\n​  \r\n​```\r\n​\r\n​-server -Xms8g -Xmx8g -Xmn4g   \r\n​ ```\r\n​ \r\n​  \r\n如果您不关心RocketMQ Broker的启动时间，还有一种更好的选择，就是通过“预触摸”Java堆以确保在JVM初始化期间每个页面都将被分配。那些不关心启动时间的人可以启用它：\r\n​ -XX:+AlwaysPreTouch   \r\n禁用偏置锁定可能会减少JVM暂停，\r\n​ -XX:-UseBiasedLocking   \r\n至于垃圾回收，建议使用带JDK 1.8的G1收集器。\r\n\r\n```text\r\n-XX:+UseG1GC -XX:G1HeapRegionSize=16m   \r\n-XX:G1ReservePercent=25 \r\n-XX:InitiatingHeapOccupancyPercent=30\r\n```\r\n\r\n​ 这些GC选项看起来有点激进，但事实证明它在我们的生产环境中具有良好的性能。另外不要把-XX:MaxGCPauseMillis的值设置太小，否则JVM将使用一个小的年轻代来实现这个目标，这将导致非常频繁的minor GC，所以建议使用rolling GC日志文件：\r\n\r\n```text\r\n-XX:+UseGCLogFileRotation   \r\n-XX:NumberOfGCLogFiles=5 \r\n-XX:GCLogFileSize=30m\r\n```\r\n\r\n如果写入GC文件会增加代理的延迟，可以考虑将GC日志文件重定向到内存文件系统：\r\n\r\n```text\r\n-Xloggc:/dev/shm/mq_gc_%p.log123   \r\n```\r\n### 6.2 Linux内核参数\r\n\r\n​ os.sh脚本在bin文件夹中列出了许多内核参数，可以进行微小的更改然后用于生产用途。下面的参数需要注意，更多细节请参考/proc/sys/vm/*的[文档](https://www.kernel.org/doc/Documentation/sysctl/vm.txt)\r\n\r\n- **vm.extra_free_kbytes**，告诉VM在后台回收（kswapd）启动的阈值与直接回收（通过分配进程）的阈值之间保留额外的可用内存。RocketMQ使用此参数来避免内存分配中的长延迟。（与具体内核版本相关）\r\n- **vm.min_free_kbytes**，如果将其设置为低于1024KB，将会巧妙的将系统破坏，并且系统在高负载下容易出现死锁。\r\n- **vm.max_map_count**，限制一个进程可能具有的最大内存映射区域数。RocketMQ将使用mmap加载CommitLog和ConsumeQueue，因此建议将为此参数设置较大的值。（agressiveness --> aggressiveness）\r\n- **vm.swappiness**，定义内核交换内存页面的积极程度。较高的值会增加攻击性，较低的值会减少交换量。建议将值设置为10来避免交换延迟。\r\n- **File descriptor limits**，RocketMQ需要为文件（CommitLog和ConsumeQueue）和网络连接打开文件描述符。我们建议设置文件描述符的值为655350。\r\n- [Disk scheduler](https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/6/html/Performance_Tuning_Guide/ch06s04s02.html)，RocketMQ建议使用I/O截止时间调度器，它试图为请求提供有保证的延迟。\r\n[]([]())\r\n"
  },
  {
    "path": "docs/cn/client/java/API_Reference_ DefaultPullConsumer.md",
    "content": "## DefaultPullConsumer\n---\n### 类简介\n\n1. `DefaultMQPullConsumer extends ClientConfig implements MQPullConsumer`\n\n2. `DefaultMQPullConsumer`主动的从Broker拉取消息，主动权由应用控制，可以实现批量的消费消息。Pull方式取消息的过程需要用户自己写，首先通过打算消费的Topic拿到MessageQueue的集合，遍历MessageQueue集合，然后针对每个MessageQueue批量取消息，也可以自定义与控制offset位置。\n                        \n3. 优势：consumer可以按需消费，不用担心自己处理能力，而broker堆积消息也会相对简单，无需记录每一个要发送消息的状态，只需要维护所有消息的队列和偏移量就可以了。所以对于慢消费，消息量有限且到来的速度不均匀的情况，pull模式比较合适消息延迟与忙等。\n                        \n4. 缺点：由于主动权在消费方，消费方无法及时获取最新的消息。比较适合不及时批处理场景。\n                        \n``` java \n\n \nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\n \nimport org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\n \npublic class MQPullConsumer {\n \n\tprivate static final Map<MessageQueue,Long> OFFSE_TABLE = new HashMap<MessageQueue,Long>();\n\t\n\tpublic static void main(String[] args) throws MQClientException {\n\t\tDefaultMQPullConsumer consumer = new DefaultMQPullConsumer(\"groupName\");\n\t\tconsumer.setNamesrvAddr(\"127.0.0.1:9876\");\n\t\tconsumer.start();\n\t\t// 从指定topic中拉取所有消息队列\n\t\tSet<MessageQueue> mqs = consumer.fetchSubscribeMessageQueues(\"order-topic\");\n\t\tfor(MessageQueue mq:mqs){\n\t\t\ttry {\n\t\t\t\t// 获取消息的offset，指定从store中获取\n\t\t\t\tlong offset = consumer.fetchConsumeOffset(mq,true);\n\t\t\t\tSystem.out.println(\"consumer from the queue:\"+mq+\":\"+offset);\n\t\t\t\twhile(true){\n\t\t\t\t\tPullResult pullResult = consumer.pullBlockIfNotFound(mq, null, \n\t\t\t\t\t\t\tgetMessageQueueOffset(mq), 32);\n\t\t\t\t\tputMessageQueueOffset(mq,pullResult.getNextBeginOffset());\n\t\t\t\t\tswitch(pullResult.getPullStatus()){\n\t\t\t\t\tcase FOUND:\n\t\t\t\t\t\tList<MessageExt> messageExtList = pullResult.getMsgFoundList();\n                        for (MessageExt m : messageExtList) {\n                            System.out.println(new String(m.getBody()));\n                        }\n\t\t\t\t\t\tbreak;\n\t\t\t\t\tcase NO_MATCHED_MSG:\n\t\t\t\t\t\tbreak;\n\t\t\t\t\tcase NO_NEW_MSG:\n\t\t\t\t\t\tbreak;\n\t\t\t\t\tcase OFFSET_ILLEGAL:\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} catch (Exception e) {\n\t\t\t\te.printStackTrace();\n\t\t\t}\n\t\t}\n\t\tconsumer.shutdown();\n\t}\n \n\t// 保存上次消费的消息下标\n\tprivate static void putMessageQueueOffset(MessageQueue mq,\n\t\t\tlong nextBeginOffset) {\n\t\tOFFSE_TABLE.put(mq, nextBeginOffset);\n\t}\n\t\n\t// 获取上次消费的消息的下标\n\tprivate static Long getMessageQueueOffset(MessageQueue mq) {\n\t\tLong offset = OFFSE_TABLE.get(mq);\n\t\tif(offset != null){\n\t\t\treturn offset;\n\t\t}\n\t\treturn 0l;\n\t}\n\t\n \n}\n```\n\n\n\n### 字段摘要\n|类型|字段名称|描述|\n|------|-------|-------|\n|DefaultMQPullConsumerImpl|defaultMQPullConsumerImpl|DefaultMQPullConsumer的内部核心处理默认实现|\n|String|consumerGroup|消费的唯一分组|\n|long|brokerSuspendMaxTimeMillis|consumer取连接broker的最大延迟时间，不建议修改|\n|long|consumerTimeoutMillisWhenSuspend|pull取连接的最大超时时间，必须大于brokerSuspendMaxTimeMillis，不建议修改|\n|long|consumerPullTimeoutMillis|socket连接的最大超时时间，不建议修改|\n|String|messageModel|默认cluster模式|\n|int|messageQueueListener|消息queue监听器，用来获取topic的queue变化|\n|int|offsetStore|RemoteBrokerOffsetStore 远程与本地offset存储器|\n|int|registerTopics|注册到该consumer的topic集合|\n|int|allocateMessageQueueStrategy|consumer的默认获取queue的负载分配策略算法|\n\n### 构造方法摘要\n\n|方法名称|方法描述|\n|-------|------------|\n|DefaultMQPullConsumer()|由默认参数值创建一个Pull消费者 |\n|DefaultMQPullConsumer(final String consumerGroup, RPCHook rpcHook)|使用指定的分组名，hook创建一个消费者|\n|DefaultMQPullConsumer(final String consumerGroup)|使用指定的分组名消费者|\n|DefaultMQPullConsumer(RPCHook rpcHook)|使用指定的hook创建一个生产者|\n\n\n### 使用方法摘要\n\n|返回值|方法名称| 方法描述                                                                                   |\n|-------|-------|----------------------------------------------------------------------------------------|\n|MQAdmin接口method|-------| ------------                                                                           |\n|void|createTopic(String key, String newTopic, int queueNum)| 在broker上创建指定的topic                                                                     |\n|void|createTopic(String key, String newTopic, int queueNum, int topicSysFlag)| 在broker上创建指定的topic                                                                     |\n|long|earliestMsgStoreTime(MessageQueue mq)| 查询最早的消息存储时间                                                                            |\n|long|maxOffset(MessageQueue mq)| 查询给定消息队列的最大offset                                                                      |\n|long|minOffset(MessageQueue mq)| 查询给定消息队列的最小offset                                                                      |\n|QueryResult|queryMessage(String topic, String key, int maxNum, long begin, long end)| 按关键字查询消息                                                                               |\n|long|searchOffset(MessageQueue mq, long timestamp)| 查找指定时间的消息队列的物理offset                                                                   |\n|MessageExt|viewMessage(String offsetMsgId)| 根据给定的msgId查询消息                                                                         |\n|MessageExt|public MessageExt viewMessage(String topic, String msgId)| 根据给定的msgId查询消息，并指定topic                                                                |\n|MQConsumer接口method|-------| ------------                                                                           |\n|Set<MessageQueue>|fetchSubscribeMessageQueues(String topic)| 根据topic获取订阅的Queue                                                                      |\n|void|sendMessageBack(final MessageExt msg, final int delayLevel)| 如果消息出来失败，可以发送回去延迟消费，delayLevel=DelayConf.DELAY_LEVEL                                   |\n|void|sendMessageBack(final MessageExt msg, final int delayLevel, final String brokerName)| 如果消息出来失败，可以发送回去延迟消费，delayLevel=DelayConf.DELAY_LEVEL                                   |\n|MQPullConsumer接口method|-------| ------------                                                                           |\n|long|fetchConsumeOffset(MessageQueue mq, boolean fromStore)| 查询给定消息队列的最大offset                                                                      |\n|PullResult |pull(final MessageQueue mq, final String subExpression, final long offset,final int maxNums)| 异步拉取制定匹配的消息                                                                            |\n|PullResult| pull(final MessageQueue mq, final String subExpression, final long offset,final int maxNums, final long timeout)| 异步拉取制定匹配的消息                                                                            |\n|PullResult|pull(final MessageQueue mq, final MessageSelector selector, final long offset,final int maxNums)| 异步拉取制定匹配的消息，通过MessageSelector器来过滤消息，参考org.apache.rocketmq.common.filter.ExpressionType |\n|PullResult|pullBlockIfNotFound(final MessageQueue mq, final String subExpression,final long offset, final int maxNums)| 异步拉取制定匹配的消息，如果没有消息讲block住，并指定超时时间consumerPullTimeoutMillis                             |\n|void|pullBlockIfNotFound(final MessageQueue mq, final String subExpression, final long offset,final int maxNums, final PullCallback pullCallback)| 异步拉取制定匹配的消息，如果没有消息讲block住，并指定超时时间consumerPullTimeoutMillis，通过回调pullCallback来消费         |    \n|void|updateConsumeOffset(final MessageQueue mq, final long offset)| 更新指定mq的offset                                                                          |\n|long|fetchMessageQueuesInBalance(String topic)| 根据topic获取订阅的Queue(是balance分配后的)                                                        |\n|void|void sendMessageBack(MessageExt msg, int delayLevel, String brokerName, String consumerGroup)| 如果消息出来失败，可以发送回去延迟消费，delayLevel=DelayConf.DELAY_LEVEL，消息可能在同一个consumerGroup消费           |\n|void|shutdown()| 关闭当前消费者实例并释放相关资源                                                                       |\n|void|start()| 启动消费者                                                                                  |\n\n"
  },
  {
    "path": "docs/cn/client/java/API_Reference_DefaultMQProducer.md",
    "content": "## DefaultMQProducer\n---\n### 类简介\n\n`public class DefaultMQProducer \nextends ClientConfig \nimplements MQProducer`\n\n>`DefaultMQProducer`类是应用用来投递消息的入口，开箱即用，可通过无参构造方法快速创建一个生产者。主要负责消息的发送，支持同步/异步/oneway的发送方式，这些发送方式均支持批量发送。可以通过该类提供的getter/setter方法，调整发送者的参数。`DefaultMQProducer`提供了多个send方法，每个send方法略有不同，在使用前务必详细了解其意图。下面给出一个生产者示例代码，[点击查看更多示例代码](https://github.com/apache/rocketmq/blob/master/example/src/main/java/org/apache/rocketmq/example/)。\n\n``` java \npublic class Producer {\n    public static void main(String[] args) throws MQClientException {\n        // 创建指定分组名的生产者\n        DefaultMQProducer producer = new DefaultMQProducer(\"ProducerGroupName\");\n\n        // 启动生产者\n        producer.start();\n\n        for (int i = 0; i < 128; i++)\n            try {\n            \t// 构建消息\n                Message msg = new Message(\"TopicTest\",\n                        \"TagA\",\n                        \"OrderID188\",\n                        \"Hello world\".getBytes(RemotingHelper.DEFAULT_CHARSET));\n\n                // 同步发送\n                SendResult sendResult = producer.send(msg);\n\n                // 打印发送结果\n                System.out.printf(\"%s%n\", sendResult);\n            } catch (Exception e) {\n                e.printStackTrace();\n            }\n\n        producer.shutdown();\n    }\n}\n```\n\n**注意**：该类是线程安全的。在配置并启动完成后可在多个线程间安全共享。\n\n### 字段摘要\n|类型|字段名称|描述|\n|------|-------|-------|\n|DefaultMQProducerImpl|defaultMQProducerImpl|生产者的内部默认实现|\n|String|producerGroup|生产者分组|\n|String|createTopicKey|在发送消息时，自动创建服务器不存在的topic|\n|int|defaultTopicQueueNums|创建topic时默认的队列数量|\n|int|sendMsgTimeout|发送消息的超时时间|\n|int|compressMsgBodyOverHowmuch|压缩消息体的阈值|\n|int|retryTimesWhenSendFailed|同步模式下内部尝试发送消息的最大次数|\n|int|retryTimesWhenSendAsyncFailed|异步模式下内部尝试发送消息的最大次数|\n|boolean|retryAnotherBrokerWhenNotStoreOK|是否在内部发送失败时重试另一个broker|\n|int|maxMessageSize|消息体的最大长度|\n|TraceDispatcher|traceDispatcher|基于RPCHooK实现的消息轨迹插件|\n\n### 构造方法摘要\n\n|方法名称|方法描述|\n|-------|------------|\n|DefaultMQProducer()|由默认参数值创建一个生产者 |\n|DefaultMQProducer(final String producerGroup)|使用指定的分组名创建一个生产者|\n|DefaultMQProducer(final String producerGroup, boolean enableMsgTrace)|使用指定的分组名创建一个生产者，并设置是否开启消息轨迹|\n|DefaultMQProducer(final String producerGroup, boolean enableMsgTrace, final String customizedTraceTopic)|使用指定的分组名创建一个生产者，并设置是否开启消息轨迹及追踪topic的名称|\n|DefaultMQProducer(RPCHook rpcHook)|使用指定的hook创建一个生产者|\n|DefaultMQProducer(final String producerGroup, RPCHook rpcHook)|使用指定的分组名及自定义hook创建一个生产者|\n|DefaultMQProducer(final String producerGroup, RPCHook rpcHook, boolean enableMsgTrace,final String customizedTraceTopic)|使用指定的分组名及自定义hook创建一个生产者，并设置是否开启消息轨迹及追踪topic的名称|\n\n### 使用方法摘要\n\n|返回值|方法名称|方法描述|\n|-------|-------|------------|\n|void|createTopic(String key, String newTopic, int queueNum)|在broker上创建指定的topic|\n|void|createTopic(String key, String newTopic, int queueNum, int topicSysFlag)|在broker上创建指定的topic|\n|long|earliestMsgStoreTime(MessageQueue mq)|查询最早的消息存储时间|\n|List<MessageQueue>|fetchPublishMessageQueues(String topic)|获取topic的消息队列|\n|long|maxOffset(MessageQueue mq)|查询给定消息队列的最大offset|\n|long|minOffset(MessageQueue mq)|查询给定消息队列的最小offset|\n|QueryResult|queryMessage(String topic, String key, int maxNum, long begin, long end)|按关键字查询消息|\n|long|searchOffset(MessageQueue mq, long timestamp)|查找指定时间的消息队列的物理offset|\n|SendResult|send(Collection<Message> msgs)|同步批量发送消息|\n|SendResult|send(Collection<Message> msgs, long timeout)|同步批量发送消息|\n|SendResult|send(Collection<Message> msgs, MessageQueue messageQueue)|向指定的消息队列同步批量发送消息|\n|SendResult|send(Collection<Message> msgs, MessageQueue messageQueue, long timeout)|向指定的消息队列同步批量发送消息，并指定超时时间|\n|SendResult|send(Message msg)|同步单条发送消息|\n|SendResult|send(Message msg, long timeout)|同步发送单条消息，并指定超时时间|\n|SendResult|send(Message msg, MessageQueue mq)|向指定的消息队列同步发送单条消息|\n|SendResult|send(Message msg, MessageQueue mq, long timeout)|向指定的消息队列同步单条发送消息，并指定超时时间|\n|void|send(Message msg, MessageQueue mq, SendCallback sendCallback)|向指定的消息队列异步单条发送消息，并指定回调方法|\n|void|send(Message msg, MessageQueue mq, SendCallback sendCallback, long timeout)|向指定的消息队列异步单条发送消息，并指定回调方法和超时时间|\n|SendResult|send(Message msg, MessageQueueSelector selector, Object arg)|向消息队列同步单条发送消息，并指定发送队列选择器|\n|SendResult|send(Message msg, MessageQueueSelector selector, Object arg, long timeout)|向消息队列同步单条发送消息，并指定发送队列选择器与超时时间|\n|void|send(Message msg, MessageQueueSelector selector, Object arg, SendCallback sendCallback)|向指定的消息队列异步单条发送消息|\n|void|send(Message msg, MessageQueueSelector selector, Object arg, SendCallback sendCallback, long timeout)|向指定的消息队列异步单条发送消息，并指定超时时间|\n|void|send(Message msg, SendCallback sendCallback)|异步发送消息|\n|void|send(Message msg, SendCallback sendCallback, long timeout)|异步发送消息，并指定回调方法和超时时间|\n|TransactionSendResult|sendMessageInTransaction(Message msg, LocalTransactionExecuter tranExecuter, final Object arg)|发送事务消息，并指定本地执行事务实例|\n|TransactionSendResult|sendMessageInTransaction(Message msg, Object arg)|发送事务消息|\n|void|sendOneway(Message msg)|单向发送消息，不等待broker响应|\n|void|sendOneway(Message msg, MessageQueue mq) |单向发送消息到指定队列，不等待broker响应|\n|void|sendOneway(Message msg, MessageQueueSelector selector, Object arg)|单向发送消息到队列选择器的选中的队列，不等待broker响应|\n|void|shutdown()|关闭当前生产者实例并释放相关资源|\n|void|start()|启动生产者|\n|MessageExt|viewMessage(String offsetMsgId)|根据给定的msgId查询消息|\n|MessageExt|public MessageExt viewMessage(String topic, String msgId)|根据给定的msgId查询消息，并指定topic|\n\n### 字段详细信息\n\n- [producerGroup](https://rocketmq.apache.org/docs/introduction/02concepts)\n\n\t`private String producerGroup`\n\t\n\t生产者的分组名称。相同的分组名称表明生产者实例在概念上归属于同一分组。这对事务消息十分重要，如果原始生产者在事务之后崩溃，那么broker可以联系同一生产者分组的不同生产者实例来提交或回滚事务。\n\n\t默认值：DEFAULT_PRODUCER\n\n\t注意： 由数字、字母、下划线、横杠（-）、竖线（|）或百分号组成；不能为空；长度不能超过255。\n\n- defaultMQProducerImpl\n\n\t`protected final transient DefaultMQProducerImpl defaultMQProducerImpl`\n\n\t生产者的内部默认实现，在构造生产者时内部自动初始化，提供了大部分方法的内部实现。\n\n- createTopicKey\n\n\t`private String createTopicKey = MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC`\n\n\t在发送消息时，自动创建服务器不存在的topic，需要指定Key，该Key可用于配置发送消息所在topic的默认路由。\n\n\t默认值：TBW102\n\n\t建议：测试或者demo使用，生产环境下不建议打开自动创建配置。\n\n- defaultTopicQueueNums\n\n\t`private volatile int defaultTopicQueueNums = 4`\n\n\t创建topic时默认的队列数量。\n\n\t默认值：4\n\n- sendMsgTimeout\n\n\t`private int sendMsgTimeout = 3000`\n\n\t发送消息时的超时时间。\n\n\t默认值：3000，单位：毫秒 \n\n\t建议：不建议修改该值，该值应该与broker配置中的sendTimeout一致，发送超时，可临时修改该值，建议解决超时问题，提高broker集群的Tps。\n\n- compressMsgBodyOverHowmuch\n\n\t`private int compressMsgBodyOverHowmuch = 1024 * 4`\n\n\t压缩消息体阈值。大于4K的消息体将默认进行压缩。\n\n\t默认值：1024 * 4，单位：字节\n\n\t建议：可通过DefaultMQProducerImpl.setZipCompressLevel方法设置压缩率（默认为5，可选范围[0,9]）；可通过DefaultMQProducerImpl.tryToCompressMessage方法测试出compressLevel与compressMsgBodyOverHowmuch最佳值。\n\n- retryTimesWhenSendFailed\n\n\t`private int retryTimesWhenSendFailed = 2`\n\n\t同步模式下，在返回发送失败之前，内部尝试重新发送消息的最大次数。\n\n\t默认值：2，即：默认情况下一条消息最多会被投递3次。\n\n\t注意：在极端情况下，这可能会导致消息的重复。\n\n- retryTimesWhenSendAsyncFailed\n\n\t`private int retryTimesWhenSendAsyncFailed = 2`\n\n\t异步模式下，在发送失败之前，内部尝试重新发送消息的最大次数。\n\n\t默认值：2，即：默认情况下一条消息最多会被投递3次。\n\n\t注意：在极端情况下，这可能会导致消息的重复。\n\n- retryAnotherBrokerWhenNotStoreOK\n\n\t`private boolean retryAnotherBrokerWhenNotStoreOK = false`\n\n\t同步模式下，消息保存失败时是否重试其他broker。\n\n\t默认值：false\n\n\t注意：此配置关闭时，非投递时产生异常情况下，会忽略retryTimesWhenSendFailed配置。\n\n- maxMessageSize\n\n\t`private int maxMessageSize = 1024 * 1024 * 4`\n\n\t消息体的最大大小。当消息体的字节数超过maxMessageSize就发送失败。\n\n\t默认值：1024 * 1024 * 4，单位：字节\n\n- [traceDispatcher](https://github.com/apache/rocketmq/wiki/RIP-6-Message-Trace)\n\n\t`private TraceDispatcher traceDispatcher = null`\n\n\t在开启消息轨迹后，该类通过hook的方式把消息生产者，消息存储的broker和消费者消费消息的信息像链路一样记录下来。在构造生产者时根据构造入参enableMsgTrace来决定是否创建该对象。\n\n### 构造方法详细信息\n\n1. DefaultMQProducer\n\t\n\t`public DefaultMQProducer()`\n\n\t创建一个新的生产者。\n\n2. DefaultMQProducer\n\t\n\t`DefaultMQProducer(final String producerGroup)`\n\n\t使用指定的分组名创建一个生产者。\n\n\t- 入参描述：\n\n\t\t参数名 | 类型 | 是否必须 | 缺省值 |描述\n\t\t---|---|---|---|---\n\t\tproducerGroup | String | 是 | DEFAULT_PRODUCER | 生产者的分组名称\n\n3. DefaultMQProducer\n\t\n\t`DefaultMQProducer(final String producerGroup, boolean enableMsgTrace)`\n\n\t使用指定的分组名创建一个生产者，并设置是否开启消息轨迹。\n\n\t- 入参描述：\n\n\t\t参数名 | 类型 | 是否必须 | 缺省值 |描述\n\t\t---|---|---|---|---\n\t\tproducerGroup | String | 是 | DEFAULT_PRODUCER | 生产者的分组名称\n\t\tenableMsgTrace | boolean | 是 | false |是否开启消息轨迹\n\n4. DefaultMQProducer\n\t\n\t`DefaultMQProducer(final String producerGroup, boolean enableMsgTrace, final String customizedTraceTopic)`\n\n\t使用指定的分组名创建一个生产者，并设置是否开启消息轨迹及追踪topic的名称。\n\n\t- 入参描述：\n\n\t\t参数名 | 类型 | 是否必须 | 缺省值 |描述\n\t\t---|---|---|---|---\n\t\tproducerGroup | String | 是 | DEFAULT_PRODUCER | 生产者的分组名称\n\t\trpcHook | RPCHook | 否 | null |每个远程命令执行后会回调rpcHook\n\t\tenableMsgTrace | boolean | 是 | false |是否开启消息轨迹\n\t\tcustomizedTraceTopic | String | 否 | RMQ_SYS_TRACE_TOPIC | 消息轨迹topic的名称\n\n5. DefaultMQProducer\n\t\n\t`DefaultMQProducer(RPCHook rpcHook)`\n\n\t使用指定的hook创建一个生产者。\n\n\t- 入参描述：\n\n\t\t参数名 | 类型 | 是否必须 | 缺省值 |描述\n\t\t---|---|---|---|---\n\t\trpcHook | RPCHook | 否 | null |每个远程命令执行后会回调rpcHook\n\n6. DefaultMQProducer\n\n\t`DefaultMQProducer(final String producerGroup, RPCHook rpcHook)`\n\n\t使用指定的分组名及自定义hook创建一个生产者。\n\n\t- 入参描述：\n\n\t\t参数名 | 类型 | 是否必须 | 缺省值 |描述\n\t\t---|---|---|---|---\n\t\tproducerGroup | String | 是 | DEFAULT_PRODUCER | 生产者的分组名称\n\t\trpcHook | RPCHook | 否 | null |每个远程命令执行后会回调rpcHook\n\n7. DefaultMQProducer\n\n\t`DefaultMQProducer(final String producerGroup, RPCHook rpcHook, boolean enableMsgTrace,final String customizedTraceTopic)`\n\n\t使用指定的分组名及自定义hook创建一个生产者，并设置是否开启消息轨迹及追踪topic的名称。\n\n\t- 入参描述：\n\n\t参数名 | 类型 | 是否必须 | 缺省值 |描述\n\t---|---|---|---|---\n\tproducerGroup | String | 是 | DEFAULT_PRODUCER | 生产者的分组名称\n\trpcHook | RPCHook | 否 | null |每个远程命令执行后会回调rpcHook\n\tenableMsgTrace | boolean | 是 | false |是否开启消息轨迹\n\tcustomizedTraceTopic | String | 否 | RMQ_SYS_TRACE_TOPIC | 消息轨迹topic的名称\n\n### 使用方法详细信息\n\n1.  createTopic\n\n\t`public void createTopic(String key, String newTopic, int queueNum)`\n\n\t在broker上创建一个topic。\n\n\t- 入参描述：\n\n\t\t参数名 | 类型 | 是否必须 | 默认值 |值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tkey | String | 是 | | | 访问密钥。\n\t\tnewTopic | String | 是 | |  | 新建topic的名称。由数字、字母、下划线（_）、横杠（-）、竖线（&#124;）或百分号（%）组成；<br>长度小于255；不能为TBW102或空。\n\t\tqueueNum | int | 是 | 0 | (0, maxIntValue] | topic的队列数量。\n\n\t- 返回值描述：\n\n\t\tvoid\n\n\t- 异常描述：\n\n\t\tMQClientException - 生产者状态非Running；未找到broker等客户端异常。\n\n2.  createTopic\n\n\t`public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag)`\n\n\t在broker上创建一个topic。\n\n\t- 入参描述：\n\n\t\t参数名 | 类型 | 是否必须 | 默认值 |值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tkey | String | 是 | | | 访问密钥。\n\t\tnewTopic | String | 是 | |  | 新建topic的名称。\n\t\tqueueNum | int | 是 | 0 | (0, maxIntValue] | topic的队列数量。\n\t\ttopicSysFlag | int | 是 | 0 | | 保留字段，暂未使用。\n\n\t- 返回值描述：\n\n\t\tvoid\n\n\t- 异常描述：\n\n\t\tMQClientException - 生产者状态非Running；未找到broker等客户端异常。\n\n3. earliestMsgStoreTime\n\n\t`public long earliestMsgStoreTime(MessageQueue mq)`\n\n\t查询最早的消息存储时间。\n\n\t- 入参描述：\n\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmq | MessageQueue | 是 | | | 要查询的消息队列\n\t\t\n\t- 返回值描述：\n\n\t\t指定队列最早的消息存储时间。单位：毫秒。\n\n\t- 异常描述：\n\n\t\tMQClientException - 生产者状态非Running；没有找到broker；broker返回失败；网络异常；线程中断等客户端异常。\n\n4. fetchPublishMessageQueues\n\n\t`public List<MessageQueue> fetchPublishMessageQueues(String topic)`\n\n\t获取topic的消息队列。\n\n\t- 入参描述：\n\t\t\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\ttopic | String | 是 | | | topic名称\n\n\t- 返回值描述：\n\n\t\t传入topic下的消息队列。\n\n\t- 异常描述：\n\n\t\tMQClientException - 生产者状态非Running；没有找到broker；broker返回失败；网络异常；线程中断等客户端异常。\n\n5. maxOffset\n\n\t`public long maxOffset(MessageQueue mq)`\n\n\t查询消息队列的最大物理偏移量。\n\n\t- 入参描述：\n\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmq | MessageQueue | 是 | | | 要查询的消息队列\n\n\t- 返回值描述：\n\n\t\t给定消息队列的最大物理偏移量。\n\n\t- 异常描述：\n\n\t\tMQClientException - 生产者状态非Running；没有找到broker；broker返回失败；网络异常；线程中断等客户端异常。\n\n6. minOffset\n\n\t`public long minOffset(MessageQueue mq)`\n\n\t查询给定消息队列的最小物理偏移量。\n\n\t- 入参描述：\n\t\t\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmq | MessageQueue | 是 | | | 要查询的消息队列\n\n\t- 返回值描述：\n\n\t\t给定消息队列的最小物理偏移量。\n\n\t- 异常描述：\n\n\t\tMQClientException - 生产者状态非Running；没有找到broker；broker返回失败；网络异常；线程中断等客户端异常。\n\n7. queryMessage\n\n\t`public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end)`\n\n\t按关键字查询消息。\n\n\t- 入参描述：\n\t\t\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\ttopic | String | 是 | | | topic名称\n\t\tkey | String | 否 | null | | 查找的关键字\n\t\tmaxNum | int | 是 | | | 返回消息的最大数量\n\t\tbegin | long | 是 | | | 开始时间戳，单位：毫秒\n\t\tend | long | 是 | | | 结束时间戳，单位：毫秒\n\n\t- 返回值描述：\n\n\t\t查询到的消息集合。\n\n\t- 异常描述：\n\n\t\tMQClientException - 生产者状态非Running；没有找到broker；broker返回失败；网络异常等客户端异常客户端异常。<br>\n\t\tInterruptedException - 线程中断。\n\n8. searchOffset\n\n\t`public long searchOffset(MessageQueue mq, long timestamp)`\n\n\t查找指定时间的消息队列的物理偏移量。\n\n\t- 入参描述：\n\t\t\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmq | MessageQueue | 是 | | | 要查询的消息队列。\n\t\ttimestamp | long | 是 | | | 指定要查询时间的时间戳。单位：毫秒。\n\n\t- 返回值描述：\n\n\t\t指定时间的消息队列的物理偏移量。\n\n\t- 异常描述：\n\n\t\tMQClientException - 生产者状态非Running；没有找到broker；broker返回失败；网络异常；线程中断等客户端异常。\n\n9. send\n\n\t`public SendResult send(Collection<Message> msgs)`\n\n\t同步批量发送消息。在返回发送失败之前，内部尝试重新发送消息的最大次数（参见*retryTimesWhenSendFailed*属性）。未明确指定发送队列，默认采取轮询策略发送。\n\n\t- 入参描述：\n\t\t\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmsgs | Collection<Message> | 是 | | | 待发送的消息集合。集合内的消息必须属同一个topic。\n\n\t- 返回值描述：\n\n\t\t批量消息的发送结果，包含msgId，发送状态等信息。\n\n\t- 异常描述：\n\n\t\tMQClientException - broker不存在或未找到；namesrv地址为空；未找到topic的路由信息等客户端异常。<br>\n\t\tRemotingException - 网络异常。<br>\n\t\tMQBrokerException - broker发生错误。<br>\n\t\tInterruptedException - 发送线程中断。<br>\n\t\tRemotingTooMuchRequestException - 发送超时。\n\n10. send\n\n\t`public SendResult send(Collection<Message> msgs, long timeout)`\n\n\t同步批量发送消息，如果在指定的超时时间内未完成消息投递，会抛出*RemotingTooMuchRequestException*。\n\t在返回发送失败之前，内部尝试重新发送消息的最大次数（参见*retryTimesWhenSendFailed*属性）。未明确指定发送队列，默认采取轮询策略发送。\n\n\t- 入参描述：\n\t\t\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmsgs | Collection<Message> | 是 | | | 待发送的消息集合。集合内的消息必须属同一个topic。\n\t\ttimeout | long | 是 | 参见*sendMsgTimeout*属性 | | 发送超时时间，单位：毫秒。\n\n\t- 返回值描述：\n\n\t\t批量消息的发送结果，包含msgId，发送状态等信息。\n\n\t- 异常描述：\n\n\t\tMQClientException - broker不存在或未找到；namesrv地址为空；未找到topic的路由信息等客户端异常。<br>\n\t\tRemotingException - 网络异常。<br>\n\t\tMQBrokerException - broker发生错误。<br>\n\t\tInterruptedException - 发送线程中断。<br>\n\t\tRemotingTooMuchRequestException - 发送超时。\n\n11. send\n\n\t`public SendResult send(Collection<Message> msgs, MessageQueue messageQueue)`\n\n\t向给定队列同步批量发送消息。\n\t\n\t注意：指定队列意味着所有消息均为同一个topic。\n\n\t- 入参描述：\n\t\t\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmsgs | Collection<Message> | 是 | | | 待发送的消息集合。集合内的消息必须属同一个topic。\n\t\tmessageQueue | MessageQueue | 是 | | | 待投递的消息队列。指定队列意味着待投递消息均为同一个topic。\n\n\t- 返回值描述：\n\n\t\t批量消息的发送结果，包含msgId，发送状态等信息。\n\n\t- 异常描述：\n\n\t\tMQClientException - broker不存在或未找到；namesrv地址为空；未找到topic的路由信息等客户端异常。<br>\n\t\tRemotingException - 网络异常。<br>\n\t\tMQBrokerException - broker发生错误。<br>\n\t\tInterruptedException - 发送线程中断。<br>\n\t\tRemotingTooMuchRequestException - 发送超时。\n\n12. send\n\n\t`public SendResult send(Collection<Message> msgs, MessageQueue messageQueue, long timeout)`\n\n\t向给定队列同步批量发送消息，如果在指定的超时时间内未完成消息投递，会抛出*RemotingTooMuchRequestException*。\n\t\n\t注意：指定队列意味着所有消息均为同一个topic。\n\n\t- 入参描述：\n\t\t\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmsgs | Collection<Message> | 是 | | | 待发送的消息集合。集合内的消息必须属同一个topic。\n\t\ttimeout | long | 是 | 参见*sendMsgTimeout*属性 | | 发送超时时间，单位：毫秒。\n\t\tmessageQueue | MessageQueue | 是 | | | 待投递的消息队列。指定队列意味着待投递消息均为同一个topic。\n\n\t- 返回值描述：\n\n\t\t批量消息的发送结果，包含msgId，发送状态等信息。\n\n\t- 异常描述：\n\n\t\tMQClientException - broker不存在或未找到；namesrv地址为空；未找到topic的路由信息等客户端异常。<br>\n\t\tRemotingException - 网络异常。<br>\n\t\tMQBrokerException - broker发生错误。<br>\n\t\tInterruptedException - 发送线程中断。<br>\n\t\tRemotingTooMuchRequestException - 发送超时。\n\n13. send\n\n\t`public SendResult send(Message msg)`\n\n\t以同步模式发送消息，仅当发送过程完全完成时，此方法才会返回。\n\t在返回发送失败之前，内部尝试重新发送消息的最大次数（参见*retryTimesWhenSendFailed*属性）。未明确指定发送队列，默认采取轮询策略发送。\n\n\t- 入参描述：\n\t\t\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmsg | Message | 是 | | | 待发送的消息。\n\n\t- 返回值描述：\n\n\t\t消息的发送结果，包含msgId，发送状态等信息。\n\n\t- 异常描述：\n\n\t\tMQClientException - broker不存在或未找到；namesrv地址为空；未找到topic的路由信息等客户端异常。<br>\n\t\tRemotingException - 网络异常。<br>\n\t\tMQBrokerException - broker发生错误。<br>\n\t\tInterruptedException - 发送线程中断。<br>\n\t\tRemotingTooMuchRequestException - 发送超时。\n\n14. send\n\n\t`public SendResult send(Message msg, long timeout)`\n\n\t以同步模式发送消息，如果在指定的超时时间内未完成消息投递，会抛出*RemotingTooMuchRequestException*。仅当发送过程完全完成时，此方法才会返回。\n\t在返回发送失败之前，内部尝试重新发送消息的最大次数（参见*retryTimesWhenSendFailed*属性）。未明确指定发送队列，默认采取轮询策略发送。\n\n\t- 入参描述：\n\t\t\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmsg | Message | 是 | | | 待发送的消息。\n\t\ttimeout | long | 是 | 参见*sendMsgTimeout*属性 | | 发送超时时间，单位：毫秒。\n\n\t- 返回值描述：\n\n\t\t消息的发送结果，包含msgId，发送状态等信息。\n\n\t- 异常描述：\n\n\t\tMQClientException - broker不存在或未找到；namesrv地址为空；未找到topic的路由信息等客户端异常。<br>\n\t\tRemotingException - 网络异常。<br>\n\t\tMQBrokerException - broker发生错误。<br>\n\t\tInterruptedException - 发送线程中断。<br>\n\t\tRemotingTooMuchRequestException - 发送超时。\n\n15. send\n\n\t`public SendResult send(Message msg, MessageQueue mq)`\n\n\t向指定的消息队列同步发送单条消息。仅当发送过程完全完成时，此方法才会返回。\n\n\t- 入参描述：\n\t\t\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmsg | Message | 是 | | | 待发送的消息。\n\t\tmq | MessageQueue | 是 | | | 待投递的消息队列。\n\n\t- 返回值描述：\n\n\t\t消息的发送结果，包含msgId，发送状态等信息。\n\n\t- 异常描述：\n\n\t\tMQClientException - broker不存在或未找到；namesrv地址为空；未找到topic的路由信息等客户端异常。<br>\n\t\tRemotingException - 网络异常。<br>\n\t\tMQBrokerException - broker发生错误。<br>\n\t\tInterruptedException - 发送线程中断。<br>\n\t\tRemotingTooMuchRequestException - 发送超时。\n\n16. send\n\n\t`public SendResult send(Message msg, MessageQueue mq, long timeout)`\n\n\t向指定的消息队列同步发送单条消息，如果在指定的超时时间内未完成消息投递，会抛出*RemotingTooMuchRequestException*。仅当发送过程完全完成时，此方法才会返回。\n\n\t- 入参描述：\n\t\t\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmsg | Message | 是 | | | 待发送的消息。\n\t\ttimeout | long | 是 | 参见*sendMsgTimeout*属性 | | 发送超时时间，单位：毫秒。\n\t\tmq | MessageQueue | 是 | | | 待投递的消息队列。指定队列意味着待投递消息均为同一个topic。\n\n\t- 返回值描述：\n\n\t\t消息的发送结果，包含msgId，发送状态等信息。\n\n\t- 异常描述：\n\n\t\tMQClientException - broker不存在或未找到；namesrv地址为空；未找到topic的路由信息等客户端异常。<br>\n\t\tRemotingException - 网络异常。<br>\n\t\tMQBrokerException - broker发生错误。<br>\n\t\tInterruptedException - 发送线程中断。<br>\n\t\tRemotingTooMuchRequestException - 发送超时。\n\n17. send\n\n\t`public void send(Message msg, MessageQueue mq, SendCallback sendCallback)`\n\n\t向指定的消息队列异步发送单条消息，异步发送调用后直接返回，并在在发送成功或者异常时回调`sendCallback`，所以异步发送时`sendCallback`参数不能为null，否则在回调时会抛出`NullPointerException`。\n\t异步发送时，在成功发送前，其内部会尝试重新发送消息的最大次数（参见*retryTimesWhenSendAsyncFailed*属性）。\n\n\t- 入参描述：\n\t\t\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmsg | Message | 是 | | | 待发送的消息。\n\t\tmq | MessageQueue | 是 | | | 待投递的消息队列。指定队列意味着待投递消息均为同一个topic。\n\t\tsendCallback | SendCallback | 是 | | | 回调接口的实现。\n\n\t- 返回值描述：\n\n\t\tvoid\n\n\t- 异常描述：\n\n\t\tMQClientException - broker不存在或未找到；namesrv地址为空；未找到topic的路由信息等客户端异常。<br>\n\t\tRemotingException - 网络异常。<br>\n\t\tInterruptedException - 发送线程中断。\n\n18. send\n\n\t`public void send(Message msg, MessageQueue mq, SendCallback sendCallback, long timeout)`\n\n\t向指定的消息队列异步发送单条消息，异步发送调用后直接返回，并在在发送成功或者异常时回调`sendCallback`，所以异步发送时`sendCallback`参数不能为null，否则在回调时会抛出`NullPointerException`。\n\t若在指定时间内消息未发送成功，回调方法会收到*RemotingTooMuchRequestException*异常。\n\t异步发送时，在成功发送前，其内部会尝试重新发送消息的最大次数（参见*retryTimesWhenSendAsyncFailed*属性）。\n\n\t- 入参描述：\n\t\t\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmsg | Message | 是 | | | 待发送的消息。\n\t\tmq | MessageQueue | 是 | | | 待投递的消息队列。\n\t\tsendCallback | SendCallback | 是 | | | 回调接口的实现。\n\t\ttimeout | long | 是 | 参见*sendMsgTimeout*属性 | | 发送超时时间，单位：毫秒。\n\n\t- 返回值描述：\n\t\tvoid\n\n\t- 异常描述：\n\n\t\tMQClientException - broker不存在或未找到；namesrv地址为空；未找到topic的路由信息等客户端异常。<br>\n\t\tRemotingException - 网络异常。<br>\n\t\tInterruptedException - 发送线程中断。\n\n19. send\n\n\t`public SendResult send(Message msg, MessageQueueSelector selector, Object arg)`\n\n\t向通过`MessageQueueSelector`计算出的队列同步发送消息。\n\n\t可以通过自实现`MessageQueueSelector`接口，将某一类消息发送至固定的队列。比如：将同一个订单的状态变更消息投递至固定的队列。\n\n    注意：此消息发送失败内部不会重试。\n    \n\t- 入参描述：\n\t\t\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmsg | Message | 是 | | | 待发送的消息。\n\t\tselector | MessageQueueSelector | 是 | | | 队列选择器。\n\t\targ | Object | 否 | | | 供队列选择器使用的参数对象。\n\n\t- 返回值描述：\n\n\t\t消息的发送结果，包含msgId，发送状态等信息。\n\n\t- 异常描述：\n\n\t\tMQClientException - broker不存在或未找到；namesrv地址为空；未找到topic的路由信息等客户端异常。<br>\n\t\tRemotingException - 网络异常。<br>\n\t\tMQBrokerException - broker发生错误。<br>\n\t\tInterruptedException - 发送线程中断。<br>\n\t\tRemotingTooMuchRequestException - 发送超时。\n\n20. send\n\n\t`public SendResult send(Message msg, MessageQueueSelector selector, Object arg, long timeout)`\n\n\t向通过`MessageQueueSelector`计算出的队列同步发送消息，并指定发送超时时间。\n\n\t可以通过自实现`MessageQueueSelector`接口，将某一类消息发送至固定的队列。比如：将同一个订单的状态变更消息投递至固定的队列。\n\t\n\t注意：此消息发送失败内部不会重试。\n\n\t- 入参描述：\n\t\t\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmsg | Message | 是 | | | 待发送的消息。\n\t\tselector | MessageQueueSelector | 是 | | | 队列选择器。\n\t\targ | Object | 否 | | | 供队列选择器使用的参数对象。\n\t\ttimeout | long | 是 | 参见*sendMsgTimeout*属性 | | 发送超时时间，单位：毫秒。\n\n\t- 返回值描述：\n\n\t\t消息的发送结果，包含msgId，发送状态等信息。\n\n\t- 异常描述：\n\t\tMQClientException - broker不存在或未找到；namesrv地址为空；未找到topic的路由信息等客户端异常。<br>\n\t\tRemotingException - 网络异常。<br>\n\t\tMQBrokerException - broker发生错误。<br>\n\t\tInterruptedException - 发送线程中断。<br>\n\t\tRemotingTooMuchRequestException - 发送超时。\n\n21. send\n\n\t`public void send(Message msg, MessageQueueSelector selector, Object arg, SendCallback sendCallback)`\n\n\t向通过`MessageQueueSelector`计算出的队列异步发送单条消息，异步发送调用后直接返回，并在在发送成功或者异常时回调`sendCallback`，所以异步发送时`sendCallback`参数不能为null，否则在回调时会抛出`NullPointerException`。\n\t异步发送时，在成功发送前，其内部会尝试重新发送消息的最大次数（参见*retryTimesWhenSendAsyncFailed*属性）。\n\n\t可以通过自实现`MessageQueueSelector`接口，将某一类消息发送至固定的队列。比如：将同一个订单的状态变更消息投递至固定的队列。\n\n\t- 入参描述：\n\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmsg | Message | 是 | | | 待发送的消息。\n\t\tselector | MessageQueueSelector | 是 | | | 队列选择器。\n\t\targ | Object | 否 | | | 供队列选择器使用的参数对象。\n\t\tsendCallback | SendCallback | 是 | | | 回调接口的实现。\n\n\t- 返回值描述：\n\n\t\tvoid\n\n\t- 异常描述：\n\n\t\tMQClientException - broker不存在或未找到；namesrv地址为空；未找到topic的路由信息等客户端异常。<br>\n\t\tRemotingException - 网络异常。<br>\n\t\tInterruptedException - 发送线程中断。\n\n22. send\n\n\t`public void send(Message msg, MessageQueueSelector selector, Object arg, SendCallback sendCallback, long timeout)`\n\n\t向通过`MessageQueueSelector`计算出的队列异步发送单条消息，异步发送调用后直接返回，并在在发送成功或者异常时回调`sendCallback`，所以异步发送时`sendCallback`参数不能为null，否则在回调时会抛出`NullPointerException`。\n\t异步发送时，在成功发送前，其内部会尝试重新发送消息的最大次数（参见*retryTimesWhenSendAsyncFailed*属性）。\n\n\t可以通过自实现`MessageQueueSelector`接口，将某一类消息发送至固定的队列。比如：将同一个订单的状态变更消息投递至固定的队列。\n\n\t- 入参描述：\n\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmsg | Message | 是 | | | 待发送的消息。\n\t\tselector | MessageQueueSelector | 是 | | | 队列选择器。\n\t\targ | Object | 否 | | | 供队列选择器使用的参数对象。\n\t\tsendCallback | SendCallback | 是 | | | 回调接口的实现。\n\t\ttimeout | long | 是 | 参见*sendMsgTimeout*属性 | | 发送超时时间，单位：毫秒。\n\n\t- 返回值描述：\n\n\t\tvoid\n\n\t- 异常描述：\n\n\t\tMQClientException - broker不存在或未找到；namesrv地址为空；未找到topic的路由信息等客户端异常。<br>\n\t\tRemotingException - 网络异常。<br>\n\t\tInterruptedException - 发送线程中断。\n\n23. send\n\n\t`public void send(Message msg, SendCallback sendCallback)`\n\n\t异步发送单条消息，异步发送调用后直接返回，并在在发送成功或者异常时回调`sendCallback`，所以异步发送时`sendCallback`参数不能为null，否则在回调时会抛出`NullPointerException`。\n\t异步发送时，在成功发送前，其内部会尝试重新发送消息的最大次数（参见*retryTimesWhenSendAsyncFailed*属性）。\n\n\t- 入参描述：\n\t\t\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmsg | Message | 是 | | | 待发送的消息。\n\t\tsendCallback | SendCallback | 是 | | | 回调接口的实现。\n\n\t- 返回值描述：\n\n\t\tvoid\n\n\t- 异常描述：\n\n\t\tMQClientException - broker不存在或未找到；namesrv地址为空；未找到topic的路由信息等客户端异常。<br>\n\t\tRemotingException - 网络异常。<br>\n\t\tInterruptedException - 发送线程中断。\n\n24. send\n\n\t`public void send(Message msg, SendCallback sendCallback, long timeout)`\n\n\t异步发送单条消息，异步发送调用后直接返回，并在在发送成功或者异常时回调`sendCallback`，所以异步发送时`sendCallback`参数不能为null，否则在回调时会抛出`NullPointerException`。\n\t异步发送时，在成功发送前，其内部会尝试重新发送消息的最大次数（参见*retryTimesWhenSendAsyncFailed*属性）。\n\n\t- 入参描述：\n\t\t\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmsg | Message | 是 | | | 待发送的消息。\n\t\tsendCallback | SendCallback | 是 | | | 回调接口的实现。\n\t\ttimeout | long | 是 | 参见*sendMsgTimeout*属性 | | 发送超时时间，单位：毫秒。\n\n\t- 返回值描述：\n\n\t\tvoid\n\n\t- 异常描述：\n\n\t\tMQClientException - broker不存在或未找到；namesrv地址为空；未找到topic的路由信息等客户端异常。<br>\n\t\tRemotingException - 网络异常。<br>\n\t\tInterruptedException - 发送线程中断。\n\n25. sendMessageInTransaction\n\n\t`public TransactionSendResult sendMessageInTransaction(Message msg, LocalTransactionExecuter tranExecuter, final Object arg)`\n\n\t发送事务消息。该类不做默认实现，抛出`RuntimeException`异常。参见：`TransactionMQProducer`类。\n\n\t- 入参描述：\n\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmsg | Message | 是 | | | 待投递的事务消息\n\t\ttranExecuter | `LocalTransactionExecuter` | 是 | | | 本地事务执行器。该类*已过期*，将在5.0.0版本中移除。请勿使用该方法。\n\t\targ | Object | 是 | | | 供本地事务执行程序使用的参数对象\n\n\t- 返回值描述：\n\n\t\t事务结果，参见：`LocalTransactionState`类。\n\n\t- 异常描述：\n\n\t\tRuntimeException - 永远抛出该异常。\n\n26. sendMessageInTransaction\n\n\t`public TransactionSendResult sendMessageInTransaction(Message msg, final Object arg)`\n\n\t发送事务消息。该类不做默认实现，抛出`RuntimeException`异常。参见：`TransactionMQProducer`类。\n\n\t- 入参描述：\n\t\t\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmsg | Message | 是 | | | 待投递的事务消息\n\t\targ | Object | 是 | | | 供本地事务执行程序使用的参数对象\n\n\t- 返回值描述：\n\t\t\n\t\t事务结果，参见：`LocalTransactionState`类。\n\n\t- 异常描述：\n\n\t\tRuntimeException - 永远抛出该异常。\n\n27. sendOneway\n\n\t`public void sendOneway(Message msg)`\n\n\t  以oneway形式发送消息，broker不会响应任何执行结果，和[UDP](https://en.wikipedia.org/wiki/User_Datagram_Protocol)类似。它具有最大的吞吐量但消息可能会丢失。\n\n\t  可在消息量大，追求高吞吐量并允许消息丢失的情况下使用该方式。\n\n\t- 入参描述：\n\t\t\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmsg | Message | 是 | | | 待投递的消息\n\n\t- 返回值描述：\n\t\t\n\t\tvoid\n\n\t- 异常描述：\n\n\t\tMQClientException - broker不存在或未找到；namesrv地址为空；未找到topic的路由信息等客户端异常。<br>\n\t\tRemotingException - 网络异常。<br>\n\t\tInterruptedException - 发送线程中断。\n\n28. sendOneway\n\n\t`public void sendOneway(Message msg, MessageQueue mq)`\n\n\t  向指定队列以oneway形式发送消息，broker不会响应任何执行结果，和[UDP](https://en.wikipedia.org/wiki/User_Datagram_Protocol)类似。它具有最大的吞吐量但消息可能会丢失。\n\n\t  可在消息量大，追求高吞吐量并允许消息丢失的情况下使用该方式。\n\n\t- 入参描述：\n\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmsg | Message | 是 | | | 待投递的消息\n\t\tmq | MessageQueue | 是 | | | 待投递的消息队列\n\n\t- 返回值描述：\n\t\tvoid\n\t- 异常描述：\n\t\tMQClientException - broker不存在或未找到；namesrv地址为空；未找到topic的路由信息等客户端异常。<br>\n\t\tRemotingException - 网络异常。<br>\n\t\tInterruptedException - 发送线程中断。\n\n29. sendOneway\n\n\t`public void sendOneway(Message msg, MessageQueueSelector selector, Object arg)`\n\n\t  向通过`MessageQueueSelector`计算出的队列以oneway形式发送消息，broker不会响应任何执行结果，和[UDP](https://en.wikipedia.org/wiki/User_Datagram_Protocol)类似。它具有最大的吞吐量但消息可能会丢失。\n\n\t  可在消息量大，追求高吞吐量并允许消息丢失的情况下使用该方式。\n\n\t- 入参描述：\n\t\t\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmsg | Message | 是 | | | 待发送的消息。\n\t\tselector | MessageQueueSelector | 是 | | | 队列选择器。\n\t\targ | Object | 否 | | | 供队列选择器使用的参数对象。\n\n\t- 返回值描述：\n\t\t\n\t\tvoid\n\n\t- 异常描述：\n\n\t\tMQClientException - broker不存在或未找到；namesrv地址为空；未找到topic的路由信息等客户端异常。<br>\n\t\tRemotingException - 网络异常。<br>\n\t\tInterruptedException - 发送线程中断。\n\n30. shutdown\n\n\t`public void shutdown()`\n\n\t关闭当前生产者实例并释放相关资源。\n\n\t- 入参描述：\n\n\t\t无。\n\n\t- 返回值描述：\n\n\t\tvoid\n\n\t- 异常描述：\n\n31. start\n\n\t`public void start()`\n\n\t启动生产者实例。在发送或查询消息之前必须调用此方法。它执行了许多内部初始化，比如：检查配置、与namesrv建立连接、启动一系列心跳等定时任务等。\n\n\t- 入参描述：\n\n\t\t无。\n\n\t- 返回值描述：\n\n\t\tvoid\n\n\t- 异常描述：\n\n\t\tMQClientException - 初始化过程中出现失败。\n\n32. viewMessage\n\n\t`public MessageExt viewMessage(String offsetMsgId)`\n\n\t根据给定的msgId查询消息。\n\n\t- 入参描述：\n\t\t\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\toffsetMsgId | String | 是 | | | offsetMsgId\n\n\t- 返回值描述：\n\n\t\t返回`MessageExt`，包含：topic名称，消息题，消息ID，消费次数，生产者host等信息。\n\n\t- 异常描述：\n\n\t\tRemotingException - 网络层发生错误。<br>\n\t\tMQBrokerException - broker发生错误。<br>\n\t\tInterruptedException - 线程被中断。<br>\n\t\tMQClientException - 生产者状态非Running；msgId非法等。\n\n33. viewMessage\n\n\t`public MessageExt viewMessage(String topic, String msgId)`\n\n\t根据给定的msgId查询消息，并指定topic。\n\n\t- 入参描述：\n\n\t\t参数名 | 类型 | 是否必须 | 默认值 | 值范围 | 说明\n\t\t---|---|---|---|---|---\n\t\tmsgId | String | 是 | | | msgId\n\t\ttopic | String | 是 | | | topic名称\n\n\t- 返回值描述：\n\n\t\t返回`MessageExt`，包含：topic名称，消息题，消息ID，消费次数，生产者host等信息。\n\n\t- 异常描述：\n\n\t\tRemotingException - 网络层发生错误。<br>\n\t\tMQBrokerException - broker发生错误。<br>\n\t\tInterruptedException - 线程被中断。<br>\n\t\tMQClientException - 生产者状态非Running；msgId非法等。"
  },
  {
    "path": "docs/cn/concept.md",
    "content": "# 基本概念\n----\n## 1 消息模型（Message Model）\n\nRocketMQ主要由 Producer、Broker、Consumer 三部分组成，其中Producer 负责生产消息，Consumer 负责消费消息，Broker 负责存储消息。Broker 在实际部署过程中对应一台服务器，每个 Broker 可以存储多个Topic的消息，每个Topic的消息也可以分片存储于不同的 Broker。Message Queue 用于存储消息的物理地址，每个Topic中的消息地址存储于多个 Message Queue 中。ConsumerGroup 由多个Consumer 实例构成。\n\n## 2 消息生产者（Producer）\n 负责生产消息，一般由业务系统负责生产消息。一个消息生产者会把业务应用系统里产生的消息发送到broker服务器。RocketMQ提供多种发送方式，同步发送、异步发送、顺序发送、单向发送。同步和异步方式均需要Broker返回确认信息，单向发送不需要。\n \n## 3 消息消费者（Consumer）\n 负责消费消息，一般是后台系统负责异步消费。一个消息消费者会从Broker服务器拉取消息、并将其提供给应用程序。从用户应用的角度而言提供了两种消费形式：拉取式消费、推动式消费。\n \n## 4 主题（Topic）\n  表示一类消息的集合，每个主题包含若干条消息，每条消息只能属于一个主题，是RocketMQ进行消息订阅的基本单位。\n  \n## 5 代理服务器（Broker Server）\n消息中转角色，负责存储消息、转发消息。代理服务器在RocketMQ系统中负责接收从生产者发送来的消息并存储、同时为消费者的拉取请求作准备。代理服务器也存储消息相关的元数据，包括消费者组、消费进度偏移和主题和队列消息等。\n\n## 6 名字服务（Name Server）\n名字服务充当路由消息的提供者。生产者或消费者能够通过名字服务查找各主题相应的Broker IP列表。多个Namesrv实例组成集群，但相互独立，没有信息交换。\n \n## 7 拉取式消费（Pull Consumer）\n  Consumer消费的一种类型，应用通常主动调用Consumer的拉消息方法从Broker服务器拉消息、主动权由应用控制。一旦获取了批量消息，应用就会启动消费过程。\n  \n## 8 推动式消费（Push Consumer）\n Consumer消费的一种类型，应用不需要主动调用Consumer的拉消息方法，在底层已经封装了拉取的调用逻辑，在用户层面看来是broker把消息推送过来的，其实底层还是consumer去broker主动拉取消息。\n \n## 9 生产者组（Producer Group）\n  同一类Producer的集合，这类Producer发送同一类消息且发送逻辑一致。如果发送的是事务消息且原始生产者在发送之后崩溃，则Broker服务器会联系同一生产者组的其他生产者实例以提交或回溯消费。\n  \n## 10 消费者组（Consumer Group）\n  同一类Consumer的集合，这类Consumer通常消费同一类消息且消费逻辑一致。消费者组使得在消息消费方面，实现负载均衡和容错的目标变得非常容易。要注意的是，消费者组的消费者实例必须订阅完全相同的Topic。RocketMQ 支持两种消息模式：集群消费（Clustering）和广播消费（Broadcasting）。\n  \n## 11 集群消费（Clustering）\n集群消费模式下，相同Consumer Group的每个Consumer实例平均分摊消息。\n\n## 12 广播消费（Broadcasting）\n广播消费模式下，相同Consumer Group的每个Consumer实例都接收全量的消息。\n\n## 13 普通顺序消息（Normal Ordered Message）\n普通顺序消费模式下，消费者通过同一个消息队列（ Topic 分区，称作 Message Queue） 收到的消息是有顺序的，不同消息队列收到的消息则可能是无顺序的。\n\n## 14 严格顺序消息（Strictly Ordered Message）\n严格顺序消息模式下，消费者收到的所有消息均是有顺序的。\n\n## 15 消息（Message）\n消息系统所传输信息的物理载体，生产和消费数据的最小单位，每条消息必须属于一个主题。RocketMQ中每个消息拥有唯一的Message ID，且可以携带具有业务标识的Key。系统提供了通过Message ID和Key查询消息的功能。\n## 16 标签（Tag）\n 为消息设置的标志，用于同一主题下区分不同类型的消息。来自同一业务单元的消息，可以根据不同业务目的在同一主题下设置不同标签。标签能够有效地保持代码的清晰度和连贯性，并优化RocketMQ提供的查询系统。消费者可以根据Tag实现对不同子主题的不同消费逻辑，实现更好的扩展性。\n\n"
  },
  {
    "path": "docs/cn/controller/deploy.md",
    "content": "# 部署和升级指南\n\n## Controller部署\n\n若需要保证Controller具备容错能力，Controller部署需要三副本及以上（遵循Raft的多数派协议）。\n\n> Controller若只部署单副本也能完成Broker Failover，但若该单点Controller故障，会影响切换能力，但不会影响存量集群的正常收发。\n\nController部署有两种方式。一种是嵌入于NameServer进行部署，可以通过配置enableControllerInNamesrv打开（可以选择性打开，并不强制要求每一台NameServer都打开），在该模式下，NameServer本身能力仍然是无状态的，也就是内嵌模式下若NameServer挂掉多数派，只影响切换能力，不影响原来路由获取等功能。另一种是独立部署，需要单独部署Controller组件。\n\n### 嵌入NameServer部署\n\n嵌入NameServer部署时只需要在NameServer的配置文件中设置enableControllerInNamesrv=true，并填上Controller的配置即可。\n\n```\nenableControllerInNamesrv = true\ncontrollerDLegerGroup = group1\ncontrollerDLegerPeers = n0-127.0.0.1:9877;n1-127.0.0.1:9878;n2-127.0.0.1:9879\ncontrollerDLegerSelfId = n0\ncontrollerStorePath = /home/admin/DledgerController\nenableElectUncleanMaster = false\nnotifyBrokerRoleChanged = true\n```\n\n参数解释：\n\n- enableControllerInNamesrv：Nameserver中是否开启controller，默认false。\n- controllerDLegerGroup：DLedger Raft Group的名字，同一个DLedger Raft Group保持一致即可。\n- controllerDLegerPeers：DLedger Group 内各节点的地址信息，同一个 Group 内的各个节点配置必须要保证一致。\n- controllerDLegerSelfId：节点 id，必须属于 controllerDLegerPeers 中的一个；同 Group 内各个节点要唯一。\n- controllerStorePath：controller日志存储位置。controller是有状态的，controller重启或宕机需要依靠日志来恢复数据，该目录非常重要，不可以轻易删除。\n- enableElectUncleanMaster：是否可以从SyncStateSet以外选举Master，若为true，可能会选取数据落后的副本作为Master而丢失消息，默认为false。\n- notifyBrokerRoleChanged：当broker副本组上角色发生变化时是否主动通知，默认为true。\n- scanNotActiveBrokerInterval：扫描 Broker是否存活的时间间隔。\n\n- 其他一些参数可以参考ControllerConfig代码。\n\n> 5.2.0 Controller 开始支持 jRaft 内核启动，不支持 DLedger 内核到 jRaft 内核原地升级\n\n```\ncontrollerType = jRaft\njRaftGroupId = jRaft-Controller\njRaftServerId = localhost:9880\njRaftInitConf = localhost:9880,localhost:9881,localhost:9882\njRaftControllerRPCAddr = localhost:9770,localhost:9771,localhost:9772\njRaftSnapshotIntervalSecs = 3600\n```\n\njRaft 版本相关参数\n- controllerType：controllerType=jRaft的时候内核启动使用jRaft，默认为DLedger。\n- jRaftGroupId：jRaft Group的名字，同一个jRaft Group保持一致即可。\n- jRaftServerId：标志自己节点的ServerId，必须出现在 jRaftInitConf 中。\n- jRaftInitConf：jRaft Group 内部通信各节点的地址信息，用逗号分隔，是 jRaft 内部通信来做选举和复制所用的地址。\n- jRaftControllerRPCAddr：Controller 外部通信的各节点的地址信息，用逗号分隔，比如 Controller 与 Broker 通信会使用该地址。\n- jRaftSnapshotIntervalSecs：Raft Snapshot 持久化时间间隔。\n\n参数设置完成后，指定配置文件启动Nameserver即可。\n\n### 独立部署\n\n独立部署执行以下脚本即可\n\n```shell\nsh bin/mqcontroller -c controller.conf\n```\nmqcontroller脚本在distribution/bin/mqcontroller，配置参数与内嵌模式相同。\n\n## Broker Controller模式部署\n\nBroker启动方法与之前相同，增加以下参数\n\n- enableControllerMode：Broker controller模式的总开关，只有该值为true，controller模式才会打开。默认为false。\n- controllerAddr：controller的地址，两种方式填写。\n  - 直接填写多个Controller IP地址，多个controller中间用分号隔开，例如`controllerAddr = 127.0.0.1:9877;127.0.0.1:9878;127.0.0.1:9879`。注意由于Broker需要向所有controller发送心跳，因此请填上所有的controller地址。\n  - 填写域名，然后设置fetchControllerAddrByDnsLookup为true，则Broker去自动解析域名后面的多个真实controller地址。\n- fetchControllerAddrByDnsLookup：controllerAddr填写域名时，如果设置该参数为true，会自动获取所有controller的地址。默认为false。\n- controllerHeartBeatTimeoutMills：Broker和controller之间心跳超时时间，心跳超过该时间判断Broker不在线。\n- syncBrokerMetadataPeriod：向controller同步Broker副本信息的时间间隔。默认5000（5s）。\n- checkSyncStateSetPeriod：检查SyncStateSet的时间间隔，检查SyncStateSet可能会shrink SyncState。默认5000（5s）。\n- syncControllerMetadataPeriod：同步controller元数据的时间间隔，主要是获取active controller的地址。默认10000（10s）。\n- haMaxTimeSlaveNotCatchup：表示slave没有跟上Master的最大时间间隔，若在SyncStateSet中的slave超过该时间间隔会将其从SyncStateSet移除。默认为15000（15s）。\n- storePathEpochFile：存储epoch文件的位置。epoch文件非常重要，不可以随意删除。默认在store目录下。\n- allAckInSyncStateSet：若该值为true，则一条消息需要复制到SyncStateSet中的每一个副本才会向客户端返回成功，可以保证消息不丢失。默认为false。\n- syncFromLastFile：若slave是空盘启动，是否从最后一个文件进行复制。默认为false。\n- asyncLearner：若该值为true，则该副本不会进入SyncStateSet，也就是不会被选举成Master，而是一直作为一个learner副本进行异步复制。默认为false。\n- inSyncReplicas：需保持同步的副本组数量，默认为1，allAckInSyncStateSet=true时该参数无效。\n- minInSyncReplicas：最小需保持同步的副本组数量，若SyncStateSet中副本个数小于minInSyncReplicas则putMessage直接返回PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH，默认为1。\n\n在Controller模式下，Broker配置必须设置enableControllerMode=true，并填写controllerAddr。\n\n### 重要参数解析\n\n1.写入副本参数\n\n其中inSyncReplicas、minInSyncReplicas等参数在普通Master-Salve部署、SlaveActingMaster模式、自动主从切换架构有重叠和不同含义，具体区别如下\n\n|                      | inSyncReplicas                                                      | minInSyncReplicas                                                        | enableAutoInSyncReplicas                    | allAckInSyncStateSet                                          | haMaxGapNotInSync                     | haMaxTimeSlaveNotCatchup                          |\n|----------------------|---------------------------------------------------------------------|--------------------------------------------------------------------------|---------------------------------------------|---------------------------------------------------------------|---------------------------------------|---------------------------------------------------|\n| 普通Master-Salve部署     | 同步复制下需要ACK的副本数，异步复制无效                                               | 无效                                                                       | 无效                                          | 无效                                                            | 无效                                    | 无效                                                |\n| 开启SlaveActingMaster （slaveActingMaster=true） | 不自动降级情况下同步复制下需要ACK的副本数                                              | 自动降级后，需要ACK最小副本数                                                         | 是否开启自动降级，自动降级后，ACK最小副本数降级到minInSyncReplicas | 无效                                                            | 判断降级依据：Slave与Master Commitlog差距值，单位字节 | 无效                                                |\n| 自动主从切换架构（enableControllerMode=true） | 不开启allAckInSyncStateSet下，同步复制下需要ACK的副本数，开启allAckInSyncStateSet后该值无效 | SyncStateSet可以降低到最小的副本数，如果SyncStateSet中副本个数小于minInSyncReplicas则直接返回副本数不足 | 无效                                          | 若该值为true，则一条消息需要复制到SyncStateSet中的每一个副本才会向客户端返回成功，该参数可以保证消息不丢失 | 无效                             | SyncStateSet收缩时，Slave最小未跟上Master的时间差，详见[RIP-44](https://shimo.im/docs/N2A1Mz9QZltQZoAD) |\n\n总结来说：\n- 普通Master-Slave下无自动降级能力，除了inSyncReplicas其他参数均无效，inSyncReplicas表示同步复制下需要ACK的副本数。\n- slaveActingMaster模式下开启enableAutoInSyncReplicas有降级能力，最小可降级到minInSyncReplicas副本数，降级判断依据是主备Commitlog高度差（haMaxGapNotInSync）以及副本存活情况，参考[slaveActingMaster模式自动降级](../QuorumACK.md)。\n> SlaveActingMaster为其他高可用部署方式，该模式下如果不使用可不参考\n- 自动主从切换（Controller模式）依赖SyncStateSet的收缩进行自动降级，SyncStateSet副本数最小收缩到minInSyncReplicas仍能正常工作，小于minInSyncReplicas直接返回副本数不足，收缩依据之一是Slave跟上的时间间隔（haMaxTimeSlaveNotCatchup）而非Commitlog高度。\n- 自动主从切换（Controller模式）正常情况是要求保证不丢消息的，只需设置allAckInSyncStateSet = true 即可，不需要考虑inSyncReplicas参数（该参数无效），如果副本较多、距离较远对延迟有要求，可以参考设置部分副本设置为asyncLearner。\n\n2.SyncStateSet收缩检查配置\n\ncheckSyncStateSetPeriod 参数决定定时检查SyncStateSet是否需要收缩的时间间隔\nhaMaxTimeSlaveNotCatchup 参数决定备跟不上主的时间\n\n当allAckInSyncState = true时（保证不丢消息），\n- haMaxTimeSlaveNotCatchup 值越小，对SyncStateSet收缩越敏感，比如主备之间网络抖动就可能导致SyncStateSet收缩，造成不必要的集群抖动。\n- haMaxTimeSlaveNotCatchup 值越大，对SyncStateSet收缩虽然不敏感，但是可能加大SyncStateSet收缩时的RTO时间。该RTO时间可以按照 checkSyncStateSetPeriod/2 + haMaxTimeSlaveNotCatchup 估算。\n\n3.消息可靠性配置\n\n保证 allAckInSyncStateSet = true 以及 enableElectUncleanMaster = false\n\n4.延迟\n\n当 allAckInSyncStateSet = true 后，一条消息要复制到SyncStateSet所有副本才能确认返回，假设SyncStateSet有3副本，其中1副本距离较远，则会影响到消息延迟。可以设置延迟最高距离最远的副本为asyncLearner，该副本不会进入SyncStateSet，只会进行异步复制，该副本作为冗余副本。\n\n## 兼容性\n\n该模式未对任何客户端层面 API 进行新增或修改，不存在客户端的兼容性问题。\n\nNameserver本身能力未做任何修改，Nameserver不存在兼容性问题。如开启enableControllerInNamesrv且controller参数配置正确，则开启controller功能。\n\nBroker若设置enableControllerMode=false，则仍然以之前方式运行。若设置enableControllerMode=true，则需要部署controller且参数配置正确才能正常运行。\n\n具体行为如下表所示：\n\n|                         | 旧版Nameserver | 旧版Nameserver+独立部署Controller | 新版Nameserver开启controller功能 | 新版Nameserver关闭controller功能 |\n|-------------------------|--------------|-----------------------------|----------------------------|----------------------------|\n| 旧版Broker                | 正常运行，无法切换    | 正常运行，无法切换                   | 正常运行，无法切换                  | 正常运行，无法切换                  |\n| 新版Broker开启Controller模式  | 无法正常上线       | 正常运行，可以切换                   | 正常运行，可以切换                  | 无法正常上线                        |\n| 新版Broker不开启Controller模式 | 正常运行，无法切换    | 正常运行，无法切换                   |正常运行，无法切换 | 正常运行，无法切换                  |\n\n## 升级注意事项\n\n从上述兼容性表述可以看出，NameServer正常升级即可，无兼容性问题。在不想升级Nameserver情况，可以独立部署Controller组件来获得切换能力。\n\n针对Broker升级，分为两种情况：\n\n（1）Master-Slave部署升级成Controller切换架构\n\n可以带数据进行原地升级，对于每组Broker，停机主、备Broker，**保证主、备的Commitlog对齐**（可以在升级前禁写该组Broker一段时间，或则通过拷贝方式保证一致），升级包后重新启动即可。\n\n> 若主备commitlog不对齐，需要保证主上线以后再上线备，否则可能会因为数据截断而丢失消息。\n\n（2）原DLedger模式升级到Controller切换架构\n\n由于原DLedger模式消息数据格式与Master-Slave下数据格式存在区别，不提供带数据原地升级的路径。在部署多组Broker的情况下，可以禁写某一组Broker一段时间（只要确认存量消息被全部消费即可，比如根据消息的保存时间来决定），然后清空store目录下除config/topics.json、subscriptionGroup.json下（保留topic和订阅关系的元数据）的其他文件后，进行空盘升级。\n\n### 持久化BrokerID版本的升级注意事项\n\n目前版本支持采用了新的持久化BrokerID版本，详情可以参考[该文档](persistent_unique_broker_id.md)，从该版本前的5.x升级到当前版本需要注意如下事项。\n\n4.x版本升级遵守上述正常流程即可。\n5.x非持久化BrokerID版本升级到持久化BrokerID版本按照如下流程:\n\n**升级Controller**\n\n1. 将旧版本Controller组停机。\n2. 清除Controller数据，即默认在`~/DLedgerController`下的数据文件。\n3. 上线新版Controller组。\n\n> 在上述升级Controller流程中，Broker仍可正常运行，但无法切换。\n\n**升级Broker**\n\n1. 将Broker从节点停机。\n2. 将Broker主节点停机。\n3. 将所有的Broker的Epoch文件删除，即默认为`~/store/epochFileCheckpoint`和`~/store/epochFileCheckpoint.bak`。\n4. 将原先的主Broker先上线，等待该Broker当选为master。(可使用`admin`命令的`getSyncStateSet`来观察)\n5. 将原来的从Broker全部上线。\n\n> 建议停机时先停从再停主，上线时先上原先的主再上原先的从，这样可以保证原来的主备关系。\n> 若需要改变升级前后主备关系，则需要停机时保证主、备的CommitLog对齐，否则可能导致数据被截断而丢失。"
  },
  {
    "path": "docs/cn/controller/design.md",
    "content": "# 背景\n\n当前 RocketMQ Raft 模式主要是利用 DLedger Commitlog 替换原来的 Commitlog，使 Commitlog 拥有选举复制能力，但这也造成了一些问题：\n\n- Raft 模式下，Broker组内副本数必须是三副本及以上，副本的ACK也必须遵循多数派协议。\n- RocketMQ 存在两套 HA 复制流程，且 Raft 模式下的复制无法利用 RocketMQ 原生的存储能力。\n\n因此我们希望利用 DLedger 实现一个基于 Raft 的一致性模块（DLedger Controller），并当作一个可选的选主组件，支持独立部署，也可以嵌入在 Nameserver 中，Broker 通过与 Controller 的交互完成 Master 的选举，从而解决上述问题，我们将该新模式称为 Controller 模式。\n\n# 架构\n\n## 核心思想\n\n![架构图](../image/controller/controller_design_1.png)\n\n如图是 Controller 模式的核心架构，介绍如下：\n\n- DledgerController：利⽤ DLedger ，构建⼀个保证元数据强⼀致性的 DLedger Controller 控制器，利⽤ Raft 选举会选出⼀个 Active DLedger Controller 作为主控制器，DLedger Controller 可以内嵌在 Nameserver中，也可以独立的部署。其主要作用是，用来存储和管理 Broker 的 SyncStateSet 列表，并在某个 Broker 的 Master Broker 下线或⽹络隔离时，主动发出调度指令来切换 Broker 的 Master。\n- SyncStateSet：主要表示⼀个 broker 副本组中跟上 Master 的 Slave 副本加上 Master 的集合。主要判断标准是 Master 和 Slave 之间的差距。当 Master 下线时，我们会从 SyncStateSet 列表中选出新的 Master。 SyncStateSet 列表的变更主要由 Master Broker 发起。Master通过定时任务判断和同步过程中完成 SyncStateSet 的Shrink 和 Expand，并向选举组件 Controller 发起 Alter SyncStateSet 请求。\n- AutoSwitchHAService：一个新的 HAService，在 DefaultHAService 的基础上，支持 BrokerRole 的切换，支持 Master 和 Slave 之间互相转换 (在 Controller 的控制下) 。此外，该 HAService 统一了日志复制流程，会在 HA HandShake 阶段进行日志的截断。\n- ReplicasManager：作为一个中间组件，起到承上启下的作用。对上，可以定期同步来自 Controller 的控制指令，对下，可以定期监控 HAService 的状态，并在合适的时间修改 SyncStateSet。ReplicasManager 会定期同步 Controller 中关于该 Broker 的元数据，当 Controller 选举出一个新的 Master 的时候，ReplicasManager 能够感知到元数据的变化，并进行 BrokerRole 的切换。\n\n## DLedgerController 核心设计\n\n![image-20220605213143645](../image/controller/quick-start/controller.png)\n\n如图是 DledgerController 的核心设计：\n\n- DLedgerController 可以内嵌在 Namesrv 中，也可以独立的部署。\n- Active DLedgerController 是 DLedger 选举出来的 Leader，其会接受来自客户端的事件请求，并通过 DLedger 发起共识，最后应用到内存元数据状态机中。\n- Not Active DLedgerController，也即 Follower 角色，其会通过 DLedger 复制来自 Active DLedgerController 的事件日志，然后直接运用到状态机中。\n\n## 日志复制\n\n### 基本概念与流程\n\n为了统一日志复制流程，区分每一任 Master 的日志复制边界，方便日志截断，引入了 MasterEpoch 的概念，代表当前 Master 的任期号 (类似 Raft Term 的含义)。\n\n对于每一任 Master，其都有 MasterEpoch 与 StartOffset，分别代表该 Master 的任期号与起始日志位移。\n\n需要注意的是，MasterEpoch 是由 Controller 决定的，且其是单调递增的。\n\n此外，我们还引入了 EpochFile，用于存放 <Epoch, StartOffset> 序列。\n\n**当⼀个 Broker 成为 Master，其会：**\n\n- 将 Commitlog 截断到最后⼀条消息的边界。\n\n- 同时最新将 <MasterEpoch , startoffset> 持久化到 EpochFile，startOffset 也即当前 CommitLog 的 MaxPhyOffset 。\n\n- 然后 HAService 监听连接，创建 HAConnection，配合 Slave 完成流程交互。\n\n**当一个 Broker 成为 Slave，其会：**\n\nReady 阶段：\n\n- 将Commitlog截断到最后⼀条消息的边界。\n\n- 与Master建⽴连接。\n\nHandshake 阶段：\n\n- 进⾏⽇志截断，这⾥关键在于 Slave 利⽤本地的 epoch 与 startOffset 和 Master 对⽐，找到⽇志截断点，进⾏⽇志截断。\n\nTransfer 阶段：\n\n- 从 Master 同步日志。\n\n### 截断算法\n\n具体的日志截断算法流程如下：\n\n- 在 HandShake 阶段， Slave 会从 Master 处获取 Master 的 EpochCache 。\n\n- Slave ⽐较获取到的 Master EpochCahce <Startoffset，Endoffset>，从后往前依次和本地进行比对，如果二者的 Epoch 与 StartOffset 相等， 则该 Epoch 有效，截断位点为两者中较⼩的 Endoffset，截断后修正⾃⼰的<Epoch , Startoffset> 信息，进⼊Transfer 阶 段；如果不相等，对比 Slave 前⼀个epoch，直到找到截断位点。\n\n```java\nslave：TreeMap<Epoch, Pair<startOffset,endOffset>> epochMap;\nIterator iterator = epochMap.entrySet().iterator();\ntruncateOffset = -1;\n\n//Epoch为从⼤到⼩排序\nwhile (iterator.hasNext()) {\n    Map.Entry<Epoch, Pair<startOffset,endOffset>> curEntry = iterator.next();\n    Pair<startOffset,endOffset> masterOffset=\n    findMasterOffsetByEpoch(curEntry.getKey());\n    \n    if(masterOffset != null && \n            curEntry.getKey().getObejct1() == masterOffset.getObejct1()) {\n        truncateOffset = Math.min(curEntry.getKey().getObejct2(), masterOffset.getObejct2());\n        break;\n   }\n}\n```\n\n### 复制流程\n\n由于 Ha 是基于流进行日志复制的，我们无法分清日志的边界 (也即传输的一批日志可能横跨多个 MasterEpoch)，Slave 无法感知到 MasterEpoch 的变化，也就无法及时修改 EpochFile。\n\n因此，我们做了如下改进：\n\nMaster 传输⽇志时，保证⼀次发送的⼀个 batch 是同⼀个 epoch 中的，⽽不能横跨多个 epoch。可以在WriteSocketService 中新增两个变量：\n\n- currentTransferEpoch：代表当前 WriteSocketService.nextTransferFromWhere 对应在哪个 epoch 中\n\n- currentTransferEpochEndOffset： 对应 currentTransferEpoch 的 end offset.。如果 currentTransferEpoch == MaxEpoch，则 currentTransferEpochEndOffset= -1，表示没有界限。\n\nWriteSocketService 传输下⼀批⽇志时 (假设这⼀批⽇志总⼤⼩为 size)，如果发现\n\nnextTransferFromWhere + size > currentTransferEpochEndOffset，则将 selectMappedBufferResult limit ⾄ currentTransferEpochEndOffset。 最后，修改 currentTransferEpoch 和 currentTransferEpochEndOffset ⾄下⼀个 epoch。\n\n相应的， Slave 接受⽇志时，如果从 header 中发现 epoch 变化，则记录到本地 epoch⽂件中。\n\n### 复制协议\n\n根据上文我们可以知道，AutoSwitchHaService 对日志复制划分为多个阶段，下面介绍是该 HaService 的协议。\n\n#### Handshake 阶段\n\n1.AutoSwitchHaClient (Slave) 会向 Master 发送 HandShake 包，如下：\n\n![示意图](../image/controller/controller_design_3.png)\n\n`current state(4byte) + Two flags(4byte) + slaveBrokerId(8byte)`\n\n- Current state 代表当前的 HAConnectionState，也即 HANDSHAKE。\n\n- Two flags 是两个状态标志位，其中，isSyncFromLastFile 代表是否要从 Master 的最后一个文件开始复制，isAsyncLearner 代表该 Slave 是否是异步复制，并以 Learner 的形式接入 Master。\n\n- slaveBrokerId 代表了该 Slave 的 brokerId，用于后续加入 SyncStateSet 。\n\n2.AutoSwitchHaConnection (Master) 会向 Slave 回送 HandShake 包，如下：\n\n![示意图](../image/controller/controller_design_4.png)\n\n`current state(4byte) + body size(4byte) + offset(8byte) + epoch(4byte) + body`\n\n- Current state 代表当前的 HAConnectionState，也即 HANDSHAKE。\n- Body size 代表了 body 的长度。\n- Offset 代表 Master 端日志的最大偏移量。\n- Epoch 代表了 Master 的 Epoch 。\n- Body 中传输的是 Master 端的 EpochEntryList 。\n\nSlave 收到 Master 回送的包后，就会在本地进行上文阐述的日志截断流程。\n\n#### Transfer 阶段\n\n1.AutoSwitchHaConnection (Master) 会不断的往 Slave 发送日志包，如下：\n\n![示意图](../image/controller/controller_design_5.png)\n\n`current state(4byte) + body size(4byte) + offset(8byte) + epoch(4byte)  + epochStartOffset(8byte) + additionalInfo(confirmOffset) (8byte)+ body`\n\n- Current state：代表当前的 HAConnectionState，也即 Transfer 。\n- Body size：代表了 body 的长度。\n- Offset：当前这一批次的日志的起始偏移量。\n- Epoch：代表当前这一批次日志所属的 MasterEpoch。\n- epochStartOffset：代表当前这一批次日志的 MasterEpoch 对应的 StartOffset。\n- confirmOffset：代表在 SyncStateSet 中的副本的最小偏移量。\n- Body：日志。\n\n2.AutoSwitchHaClient (Slave) 会向 Master 发送 ACK 包：\n\n![示意图](../image/controller/controller_design_6.png)\n\n` current state(4byte) + maxOffset(8byte)`\n\n- Current state：代表当前的 HAConnectionState，也即 Transfer 。\n- MaxOffset：代表当前 Slave 的最大日志偏移量。\n\n## Master 选举\n\n### 基本流程\n\nELectMaster 主要是在某 Broker 副本组的 Master 下线或不可访问时，重新从 SyncStateSet 列表⾥⾯选出⼀个新的 Master，该事件由 Controller ⾃身或者通过运维命令`electMaster` 发起Master选举。\n\n无论 Controller 是独立部署，还是嵌入在 Namesrv 中，其都会监听每个 Broker 的连接通道，如果某个 Broker channel inActive 了，就会判断该 Broker 是否为 Master，如果是，则会触发选主的流程。\n\n选举 Master 的⽅式⽐较简单，我们只需要在该组 Broker 所对应的 SyncStateSet 列表中，挑选⼀个出来成为新的 Master 即可，并通过 DLedger 共识后应⽤到内存元数据，最后将结果通知对应的Broker副本组。\n\n### SyncStateSet 变更\n\nSyncStateSet 是选主的重要依据，SyncStateSet 列表的变更主要由 Master Broker 发起。Master通过定时任务判断和同步过程中完成 SyncStateSet 的Shrink 和 Expand，并向选举组件 Controller 发起 Alter SyncStateSet 请求。\n\n#### Shrink\n\nShrink SyncStateSet ，指把 SyncStateSet 副本集合中那些与Master差距过⼤的副本移除，判断依据如下：\n\n- 增加 haMaxTimeSlaveNotCatchUp 参数 。\n\n- HaConnection 中记录 Slave 上⼀次跟上 Master 的时间戳 lastCaughtUpTimeMs，该时间戳含义是：每次Master 向 Slave 发送数据（transferData）时记录⾃⼰当前的 MaxOffset 为 lastMasterMaxOffset 以及当前时间戳 lastTransferTimeMs。\n\n- ReadSocketService 接收到 slaveAckOffset 时若 slaveAckOffset >= lastMasterMaxOffset 则将lastCaughtUpTimeMs 更新为 lastTransferTimeMs。\n\n- Master 端通过定时任务扫描每一个 HaConnection，如果 (cur_time - connection.lastCaughtUpTimeMs) > haMaxTimeSlaveNotCatchUp，则该 Slave 是 Out-of-sync 的。\n\n- 如果检测到 Slave out of sync ，master 会立刻向 Controller 上报SyncStateSet，从而 Shrink SyncStateSet。\n\n#### Expand\n\n如果⼀个 Slave 副本追赶上了 Master，Master 需要及时向Controller Alter SyncStateSet 。加⼊SyncStateSet 的条件是 slaveAckOffset >= ConfirmOffset（当前 SyncStateSet  中所有副本的 MaxOffset 的最⼩值）。\n\n## 参考资料\n\n[RIP-44原文](https://github.com/apache/rocketmq/wiki/RIP-44-Support-DLedger-Controller)\n"
  },
  {
    "path": "docs/cn/controller/persistent_unique_broker_id.md",
    "content": "# 持久化的唯一BrokerId\n\n## 现阶段问题\n\n在 RocketMQ 5.0.0 和 5.1.0 版本中，采用`BrokerAddress`作为Broker在Controller模式下的唯一标识。导致如下情景出现问题：\n\n- 在容器或者K8s环境下，每次Broker的重启或升级都可能会导致IP发生变化，导致之前的`BrokerAddress`留下的记录没办法和重启后的Broker联系起来，比如说`ReplicaInfo`, `SyncStateSet`等数据。\n\n## 改进方案\n\n在Controller侧采用`BrokerName:BrokerId`作为唯一标识，不再以`BrokerAddress`作为唯一标识。并且需要对`BrokerId`进行持久化存储，由于`ClusterName`和`BrokerName`都是启动的时候在配置文件中配置好的，所以只需要处理`BrokerId`的分配和持久化问题。\nBroker第一次上线的时候，只有配置文件中配置的`ClusterName`和`BrokerName`，以及自身的`BrokerAddress`。那么我们需要和`Controller`协商出一个在整个集群生命周期中都唯一确定的标识：`BrokerId`，该`BrokerId`从1开始分配。当某一个Broker被选为Master的时候，在向Name Server中重新注册时，将更改为`BrokerId`为0 (兼容之前逻辑 brokerId为0代表着Broker是Master身份)。\n\n### 上线流程\n\n![register process](../image/controller/persistent_unique_broker_id/register_process.png)\n\n#### 1. GetNextBrokerId Request\n\n这时候发起一个`GetNextBrokerId`的请求到Controller，为了拿到当前的下一个待分配的`BrokerId`(从1开始分配)。\n\n#### 1.1 ReadFromDLedger\n\n此时Controller接收到请求，然后走DLedger去获取到状态机的`NextBrokerId`数据。\n\n#### 2. GetNextBrokerId Response\n\nController将`NextBrokerId`返回给Broker。\n\n#### 2.1 CreateTempMetaFile\n\nBroker拿到`NextBrokerId`之后，创建一个临时文件`.broker.meta.temp`，里面记录了`NextBrokerId`(也就是期望应用的`BrokerId`)，以及自己生成一个`RegisterCode`(用于之后的身份校验)也持久化到临时文件中。\n\n#### 3. ApplyBrokerId Request\n\nBroker携带着当前自己的基本数据(`ClusterName`、`BrokerName`和`BrokerAddress`)以及此时期望应用的`BrokerId`和`RegisterCode`，发送一个`ApplyBrokerId`的请求到Controller。\n\n#### 3.1 CASApplyBrokerId\n\nController通过DLedger写入该事件，当该事件(日志)被应用到状态机的时候，判断此时是否可以应用该`BrokerId`(若`BrokerId`已被分配并且也不是分配给该Broker时则失败)。并且此时会记录下来该`BrokerId`和`RegisterCode`之间的关系。\n\n#### 4. ApplyBrokerId Response\n\n若上一步成功应用了该`BrokerId`，此时则返回成功给Broker，若失败则返回当前的`NextBrokerId`。\n\n#### 4.1 CreateMetaFileFromTemp\n\n若上一步成功的应用了该`BrokerId`，那么此时可以视为Broker侧成功的分配了该BrokerId，那么此时我们也需要彻底将这个BrokerId的信息持久化，那么我们就可以直接原子删除`.broker.meta.temp`并创建`.broker.meta`。删除和创建这两步需为原子操作。\n\n> 经过上述流程，第一次上线的broker和controller成功协商出一个双方都认同的brokeId并持久化保存起来。\n\n#### 5. RegisterBrokerToController Request\n\n之前的步骤已经正确协商出了`BrokerId`，但是这时候有可能Controller侧保存的`BrokerAddress`是上次Broker上线的时候的`BrokerAddress`，所以现在需要更新一下`BrokerAddress`，发送一个`RegisterBrokerToController` 请求并带上当前的`BrokerAddress`。\n\n#### 5.1 UpdateBrokerAddress\n\nController比对当前该Broker在Controller状态机中保存的`BrokerAddress`，若和请求中携带的不一致则更新为请求中的`BrokerAddress`。\n\n#### 6. RegisterBrokerToController Response\n\nController侧在更新完`BrokerAddress`之后可携带着当前该Broker所在的`Broker-set`的主从信息返回，用于通知Broker进行相应的身份转变。\n\n### 注册状态轮转\n\n![register state transfer](../image/controller/persistent_unique_broker_id/register_state_transfer.png)\n\n### 故障容错\n\n> 如果在正常上线流程中出现了各种情况的宕机，则以下流程保证正确的`BrokerId`分配\n\n#### 正常重启后的节点上线\n\n若是正常重启，那么则已经在双方协商出唯一的`BrokerId`，并且本地也在`broker.meta`中有该`BrokerId`的数据，那么就该注册流程不需要进行，直接继续后面的流程即可。即从`RegisterBrokerToController`处继续上线即可。\n\n![restart normally](../image/controller/persistent_unique_broker_id/normal_restart.png)\n\n#### CreateTempMetaFile失败\n\n![fail at creating temp metadata file](../image/controller/persistent_unique_broker_id/fail_create_temp_metadata_file.png)\n\n如果是上图中的流程失败的话，那么Broker重启后，Controller侧的状态机本身也没有分配任何`BrokerId`。Broker自身也没有任何数据被保存。因此直接重新按照上述流程从头开始走即可。\n\n#### CreateTempMetaFile成功，ApplyBrokerId未成功\n\n若是Controller侧已经认为本次`ApplyBrokerId`请求不对(请求去分配一个已被分配的`BrokerId`并且该 `RegisterCode`不相等)，并且此时返回当前的`NextBrokerId`给Broker，那么此时Broker直接删除`.broker.meta.temp`文件，接下来回到第2步，重新开始该流程以及后续流程。\n\n![fail at applying broker id](../image/controller/persistent_unique_broker_id/fail_apply_broker_id.png)\n\n#### ApplyBrokerId成功，CreateMetaFileFromTemp未成功\n\n上述情况可以出现在`ApplyResult`丢失、CAS删除并创建`broker.meta`失败，这俩流程中。\n那么重启后，Controller侧是已经认为我们`ApplyBrokerId`流程是成功的了，而且也已经在状态机中修改了BrokerId的分配数据，那么我们这时候重新直接开始步骤3，也就是发送`ApplyBrokerId`请求的这一步。\n\n![fail at create metadata file](../image/controller/persistent_unique_broker_id/fail_create_metadata_file_and_delete_temp.png)\n\n因为我们有`.broker.meta.temp`文件，可以从中拿到我们之前成功在Controller侧应用的`BrokerId`和`RegisterCode`，那么直接发送给Controller，如果Controller中存在该`BrokerId`并且`RegisterCode`和请求中的`RegisterCode`相等，那么视为成功。\n\n### 正确上线后使用BrokerId作为唯一标识\n\n当正确上线之后，之后Broker的请求和状态记录都以`BrokerId`作为唯一标识。心跳等数据的记录都以`BrokerId`为标识。\n同时Controller侧也会记录当前该`BrokerId`的`BrokerAddress`，在主从切换等时候用于通知Broker状态变化。\n\n> 默认持久化ID的文件在~/store/brokerIdentity，也可以设置storePathBrokerIdentity参数来决定存储路径。在自动主备切换模式下，不要随意删除该文件，否则该 Broker 会被当作新 Broker 上线。\n\n## 升级方案\n\n4.x 版本升级遵守 5.0 升级文档流程即可。\n5.0.0 和 5.1.0 非持久化BrokerId版本升级到 5.1.1 及以上持久化BrokerId版本按照如下流程:\n\n### 升级Controller\n\n1. 将旧版本Controller组停机。\n2. 清除Controller数据，即默认在`~/DLedgerController`下的数据文件。\n3. 上线新版Controller组。\n\n> 在上述升级Controller流程中，Broker仍可正常运行，但无法切换。\n\n### 升级Broker\n\n1. 将Broker从节点停机。\n2. 将Broker主节点停机。\n3. 将所有的Broker的Epoch文件删除，即默认为`~/store/epochFileCheckpoint`和`~/store/epochFileCheckpoint.bak`。\n4. 将原先的主Broker先上线，等待该Broker当选为master。(可使用`admin`命令的`getSyncStateSet`来观察)\n5. 将原来的从Broker全部上线。\n\n> 建议停机时先停从再停主，上线时先上原先的主再上原先的从，这样可以保证原来的主备关系。\n若需要改变升级前后主备关系，则需要停机时保证主、备的CommitLog对齐，否则可能导致数据被截断而丢失。\n\n### 兼容性\n\n|                    | 5.1.0 及以下版本 Controller | 5.1.1 及以上版本 Controller             |\n|--------------------|------------------------|------------------------------------|\n| 5.1.0 及以下版本 Broker | 正常运行，可切换               | 若已主备确定则可正常运行，不可切换。若broker重新启动则无法上线 |\n| 5.1.1 及以上版本 Broker | 无法正常上线                 | 正常运行，可切换                           |\n"
  },
  {
    "path": "docs/cn/controller/quick_start.md",
    "content": "# 自动主从切换快速开始\n\n## 前言\n\n![架构图](../image/controller/controller_design_2.png)\n\n该文档主要介绍如何快速构建自动主从切换的 RocketMQ 集群，其架构如上图所示，主要增加支持自动主从切换的Controller组件，其可以独立部署也可以内嵌在NameServer中。\n\n详细设计思路请参考 [设计思想](design.md).\n\n详细的新集群部署和旧集群升级指南请参考 [部署指南](deploy.md)。\n\n## 编译 RocketMQ 源码\n\n```shell\n$ git clone https://github.com/apache/rocketmq.git\n\n$ cd rocketmq\n\n$ mvn -Prelease-all -DskipTests clean install -U\n```\n\n## 快速部署\n\n在构建成功后\n\n```shell\n#{rocketmq-version} replace with rocketmq actual version. example: 5.0.0-SNAPSHOT\n$ cd distribution/target/rocketmq-{rocketmq-version}/rocketmq-{rocketmq-version}/\n\n$ sh bin/controller/fast-try.sh start\n```\n\n如果上面的步骤执行成功，可以通过运维命令查看Controller状态。\n\n```shell\n$ sh bin/mqadmin getControllerMetaData -a localhost:9878\n```\n\n-a代表集群中任意一个Controller的地址\n\n至此，启动成功，现在可以向集群收发消息，并进行切换测试了。\n\n如果需要关闭快速集群，可以执行：\n\n```shell\n$ sh bin/controller/fast-try.sh stop\n```\n\n对于快速部署，默认配置在 conf/controller/quick-start里面，默认的存储路径在 /tmp/rmqstore，且会开启一个 Controller (嵌入在 Namesrv) 和两个 Broker。\n\n### 查看 SyncStateSet\n\n可以通过运维工具查看 SyncStateSet：\n\n```shell\n$ sh bin/mqadmin getSyncStateSet -a localhost:9878 -b broker-a\n```\n\n-a 代表的是任意一个 Controller 的地址\n\n如果顺利的话，可以看到以下内容：\n\n![image-20220605205259913](../image/controller/quick-start/syncstateset.png)\n\n### 查看 BrokerEpoch\n\n可以通过运维工具查看 BrokerEpochEntry：\n\n```shell\n$ sh bin/mqadmin getBrokerEpoch -n localhost:9876 -b broker-a\n```\n\n-n 代表的是任意一个 Namesrv 的地址\n\n如果顺利的话，可以看到以下内容：\n\n![image-20220605205247476](../image/controller/quick-start/epoch.png)\n\n## 切换\n\n部署成功后，现在尝试进行 Master 切换。\n\n首先，kill 掉原 Master 的进程，在上文的例子中，就是使用端口 30911 的进程：\n\n```shell\n#查找端口:\n$ ps -ef|grep java|grep BrokerStartup|grep ./conf/controller/quick-start/broker-n0.conf|grep -v grep|awk '{print $2}'\n#杀掉 master:\n$ kill -9 PID\n```\n\n接着，用 SyncStateSet admin 脚本查看：\n\n```shell\n$ sh bin/mqadmin getSyncStateSet -a localhost:9878 -b broker-a\n```\n\n可以发现 Master 已经发生了切换。\n\n![image-20220605211244128](../image/controller/quick-start/changemaster.png)\n\n\n\n## Controller内嵌Namesvr集群部署\n\nController以插件方式内嵌Namesvr集群(3个Node组成)部署，快速启动：\n\n```shell\n$ sh bin/controller/fast-try-namesrv-plugin.sh start\n```\n\n或者通过命令单独启动：\n\n```shell\n$ nohup bin/mqnamesrv -c ./conf/controller/cluster-3n-namesrv-plugin/namesrv-n0.conf &\n$ nohup bin/mqnamesrv -c ./conf/controller/cluster-3n-namesrv-plugin/namesrv-n1.conf &\n$ nohup bin/mqnamesrv -c ./conf/controller/cluster-3n-namesrv-plugin/namesrv-n2.conf &\n```\n\n如果上面的步骤执行成功，可以通过运维命令查看Controller集群状态。\n\n```shell\n$ sh bin/mqadmin getControllerMetaData -a localhost:9878\n```\n\n-a代表的是任意一个 Controller 的地址\n\n如果controller启动成功可以看到以下内容：\n\n```\n#ControllerGroup        group1\n#ControllerLeaderId     n0\n#ControllerLeaderAddress        127.0.0.1:9878\n#Peer:  n0:127.0.0.1:9878\n#Peer:  n1:127.0.0.1:9868\n#Peer:  n2:127.0.0.1:9858\n```\n\n启动成功后Broker Controller模式部署就能使用Controller集群。\n\n如果需要快速停止集群：\n\n```shell\n$ sh bin/controller/fast-try-namesrv-plugin.sh stop\n```\n\n使用 fast-try-namesrv-plugin.sh 脚本快速部署，默认配置在 conf/controller/cluster-3n-namesrv-plugin里面并且会启动3个Namesvr和3个Controller(内嵌Namesrv)。\n\n## Controller独立集群部署\n\nController独立集群(3个Node组成)部署，快速启动：\n\n```shell\n$ sh bin/controller/fast-try-independent-deployment.sh start\n```\n\n或者通过命令单独启动：\n\n```shell\n$ nohup bin/mqcontroller -c ./conf/controller/cluster-3n-independent/controller-n0.conf &\n$ nohup bin/mqcontroller -c ./conf/controller/cluster-3n-independent/controller-n1.conf &\n$ nohup bin/mqcontroller -c ./conf/controller/cluster-3n-independent/controller-n2.conf &\n```\n\n如果上面的步骤执行成功，可以通过运维命令查看Controller集群状态。\n\n```shell\n$ sh bin/mqadmin getControllerMetaData -a localhost:9878\n```\n\n-a代表的是任意一个 Controller 的地址\n\n如果Controller启动成功可以看到以下内容：\n\n```\n#ControllerGroup        group1\n#ControllerLeaderId     n1\n#ControllerLeaderAddress        127.0.0.1:9868\n#Peer:  n0:127.0.0.1:9878\n#Peer:  n1:127.0.0.1:9868\n#Peer:  n2:127.0.0.1:9858\n```\n\n启动成功后Broker Controller模式部署就能使用Controller集群。\n\n如果需要快速停止集群：\n\n```shell\n$ sh bin/controller/fast-try-independent-deployment.sh stop\n```\n\n使用fast-try-independent-deployment.sh 脚本快速部署，默认配置在 conf/controller/cluster-3n-independent里面并且会启动3个Controller(独立部署)组成一个集群。\n\n\n\n## 注意说明\n\n- 若需要保证Controller具备容错能力，Controller部署需要三副本及以上（遵循Raft的多数派协议）\n- Controller部署配置文件中配置参数`controllerDLegerPeers` 中的IP地址配置成其他节点能够访问的IP,在多机器部署的时候尤为重要。例子仅供参考需要根据实际情况进行修改调整。\n"
  },
  {
    "path": "docs/cn/design.md",
    "content": "\n#  设计(design)\n---\n### 1 消息存储\n\n![](image/rocketmq_design_1.png)\n\n消息存储是RocketMQ中最为复杂和最为重要的一部分，本节将分别从RocketMQ的消息存储整体架构、PageCache与Mmap内存映射以及RocketMQ中两种不同的刷盘方式三方面来分别展开叙述。\n\n#### 1.1 消息存储整体架构\n消息存储架构图中主要有下面三个跟消息存储相关的文件构成。\n\n(1) CommitLog：消息主体以及元数据的存储主体，存储Producer端写入的消息主体内容，消息内容不是定长的。单个文件大小默认1G， 文件名长度为20位，左边补零，剩余为起始偏移量，比如00000000000000000000代表了第一个文件，起始偏移量为0，文件大小为1G=1073741824；当第一个文件写满了，第二个文件为00000000001073741824，起始偏移量为1073741824，以此类推。消息主要是顺序写入日志文件，当文件满了，写入下一个文件；\n\n(2) ConsumeQueue：消息消费索引，引入的目的主要是提高消息消费的性能。由于RocketMQ是基于主题topic的订阅模式，消息消费是针对主题进行的，如果要遍历commitlog文件，根据topic检索消息是非常低效的。Consumer可根据ConsumeQueue来查找待消费的消息。其中，ConsumeQueue作为消费消息的索引，保存了指定Topic下的队列消息在CommitLog中的起始物理偏移量offset，消息大小size和消息Tag的HashCode值。consumequeue文件可以看成是基于topic的commitlog索引文件，故consumequeue文件夹的组织方式如下：topic/queue/file三层组织结构，具体存储路径为：$HOME/store/consumequeue/{topic}/{queueId}/{fileName}。同样consumequeue文件采取定长设计，每一个条目共20个字节，分别为8字节的commitlog物理偏移量、4字节的消息长度、8字节tag hashcode，单个文件由30W个条目组成，可以像数组一样随机访问每一个条目，每个ConsumeQueue文件大小约5.72M；\n\n(3) IndexFile：IndexFile（索引文件）提供了一种可以通过key或时间区间来查询消息的方法。Index文件的存储位置是：$HOME/store/index/{fileName}，文件名fileName是以创建时的时间戳命名的，固定的单个IndexFile文件大小约为400M，一个IndexFile可以保存 2000W个索引，IndexFile的底层存储设计为在文件系统中实现HashMap结构，故RocketMQ的索引文件其底层实现为hash索引。\n\n在上面的RocketMQ的消息存储整体架构图中可以看出，RocketMQ采用的是混合型的存储结构，即为Broker单个实例下所有的队列共用一个日志数据文件（即为CommitLog）来存储。RocketMQ的混合型存储结构(多个Topic的消息实体内容都存储于一个CommitLog中)针对Producer和Consumer分别采用了数据和索引部分相分离的存储结构，Producer发送消息至Broker端，然后Broker端使用同步或者异步的方式对消息刷盘持久化，保存至CommitLog中。只要消息被刷盘持久化至磁盘文件CommitLog中，那么Producer发送的消息就不会丢失。正因为如此，Consumer也就肯定有机会去消费这条消息。当无法拉取到消息后，可以等下一次消息拉取，同时服务端也支持长轮询模式，如果一个消息拉取请求未拉取到消息，Broker允许等待30s的时间，只要这段时间内有新消息到达，将直接返回给消费端。这里，RocketMQ的具体做法是，使用Broker端的后台服务线程—ReputMessageService不停地分发请求并异步构建ConsumeQueue（逻辑消费队列）和IndexFile（索引文件）数据。\n#### 1.2 页缓存与内存映射\n页缓存（PageCache)是OS对文件的缓存，用于加速对文件的读写。一般来说，程序对文件进行顺序读写的速度几乎接近于内存的读写速度，主要原因就是由于OS使用PageCache机制对读写访问操作进行了性能优化，将一部分的内存用作PageCache。对于数据的写入，OS会先写入至Cache内，随后通过异步的方式由pdflush内核线程将Cache内的数据刷盘至物理磁盘上。对于数据的读取，如果一次读取文件时出现未命中PageCache的情况，OS从物理磁盘上访问读取文件的同时，会顺序对其他相邻块的数据文件进行预读取。\n\n在RocketMQ中，ConsumeQueue逻辑消费队列存储的数据较少，并且是顺序读取，在page cache机制的预读取作用下，Consume Queue文件的读性能几乎接近读内存，即使在有消息堆积情况下也不会影响性能。而对于CommitLog消息存储的日志数据文件来说，读取消息内容时候会产生较多的随机访问读取，严重影响性能。如果选择合适的系统IO调度算法，比如设置调度算法为“Deadline”（此时块存储采用SSD的话），随机读的性能也会有所提升。\n\n另外，RocketMQ主要通过MappedByteBuffer对文件进行读写操作。其中，利用了NIO中的FileChannel模型将磁盘上的物理文件直接映射到用户态的内存地址中（这种Mmap的方式减少了传统IO将磁盘文件数据在操作系统内核地址空间的缓冲区和用户应用程序地址空间的缓冲区之间来回进行拷贝的性能开销），将对文件的操作转化为直接对内存地址进行操作，从而极大地提高了文件的读写效率（正因为需要使用内存映射机制，故RocketMQ的文件存储都使用定长结构来存储，方便一次将整个文件映射至内存）。\n#### 1.3 消息刷盘\n\n![](image/rocketmq_design_2.png)\n\n(1) 同步刷盘：如上图所示，只有在消息真正持久化至磁盘后RocketMQ的Broker端才会真正返回给Producer端一个成功的ACK响应。同步刷盘对MQ消息可靠性来说是一种不错的保障，但是性能上会有较大影响，一般适用于金融业务应用该模式较多。\n\n(2) 异步刷盘：能够充分利用OS的PageCache的优势，只要消息写入PageCache即可将成功的ACK返回给Producer端。消息刷盘采用后台异步线程提交的方式进行，降低了读写延迟，提高了MQ的性能和吞吐量。\n\n### 2 通信机制\n\nRocketMQ消息队列集群主要包括NameServer、Broker(Master/Slave)、Producer、Consumer4个角色，基本通讯流程如下：\n\n(1) Broker启动后需要完成一次将自己注册至NameServer的操作；随后每隔30s时间定时向NameServer上报Topic路由信息。\n\n(2) 消息生产者Producer作为客户端发送消息时候，需要根据消息的Topic从本地缓存的TopicPublishInfoTable获取路由信息。如果没有则更新路由信息会从NameServer上重新拉取，同时Producer会默认每隔30s向NameServer拉取一次路由信息。\n\n(3) 消息生产者Producer根据2）中获取的路由信息选择一个队列（MessageQueue）进行消息发送；Broker作为消息的接收者接收消息并落盘存储。\n\n(4) 消息消费者Consumer根据2）中获取的路由信息，并再完成客户端的负载均衡后，选择其中的某一个或者某几个消息队列来拉取消息并进行消费。\n\n从上面1）~3）中可以看出在消息生产者，Broker和NameServer之间都会发生通信（这里只说了MQ的部分通信），因此如何设计一个良好的网络通信模块在MQ中至关重要，它将决定RocketMQ集群整体的消息传输能力与最终的性能。\n\nrocketmq-remoting 模块是 RocketMQ消息队列中负责网络通信的模块，它几乎被其他所有需要网络通信的模块（诸如rocketmq-client、rocketmq-broker、rocketmq-namesrv）所依赖和引用。为了实现客户端与服务器之间高效的数据请求与接收，RocketMQ消息队列自定义了通信协议并在Netty的基础之上扩展了通信模块。\n#### 2.1 Remoting通信类结构\n\n![](image/rocketmq_design_3.png)\n\n#### 2.2 协议设计与编解码\n在Client和Server之间完成一次消息发送时，需要对发送的消息进行一个协议约定，因此就有必要自定义RocketMQ的消息协议。同时，为了高效地在网络中传输消息和对收到的消息读取，就需要对消息进行编解码。在RocketMQ中，RemotingCommand这个类在消息传输过程中对所有数据内容的封装，不但包含了所有的数据结构，还包含了编码解码操作。\n\nHeader字段 | 类型 | Request说明 | Response说明\n--- | --- | --- | --- |\ncode |int | 请求操作码，应答方根据不同的请求码进行不同的业务处理 | 应答响应码。0表示成功，非0则表示各种错误\nlanguage | LanguageCode | 请求方实现的语言 | 应答方实现的语言\nversion | int | 请求方程序的版本 | 应答方程序的版本\nopaque | int |相当于requestId，在同一个连接上的不同请求标识码，与响应消息中的相对应 | 应答不做修改直接返回\nflag | int | 区分是普通RPC还是onewayRPC的标志 | 区分是普通RPC还是onewayRPC的标志\nremark | String | 传输自定义文本信息 | 传输自定义文本信息\nextFields | HashMap<String, String> | 请求自定义扩展信息 | 响应自定义扩展信息\n\n![](image/rocketmq_design_4.png)\n\n可见传输内容主要可以分为以下4部分：\n\n(1) 消息长度：总长度，四个字节存储，占用一个int类型；\n\n(2) 序列化类型&消息头长度：同样占用一个int类型，第一个字节表示序列化类型，后面三个字节表示消息头长度；\n\n(3) 消息头数据：经过序列化后的消息头数据；\n\n(4) 消息主体数据：消息主体的二进制字节数据内容；\n\n#### 2.3 消息的通信方式和流程\n在RocketMQ消息队列中支持通信的方式主要有同步(sync)、异步(async)、单向(oneway)\n三种。其中“单向”通信模式相对简单，一般用在发送心跳包场景下，无需关注其Response。这里，主要介绍RocketMQ的异步通信流程。\n\n![](image/rocketmq_design_5.png)\n\n#### 2.4 Reactor多线程设计\nRocketMQ的RPC通信采用Netty组件作为底层通信库，同样也遵循了Reactor多线程模型，同时又在这之上做了一些扩展和优化。\n\n![](image/rocketmq_design_6.png)\n\n从上面的框图中可以大致了解RocketMQ中NettyRemotingServer的Reactor 多线程模型。一个 Reactor 主线程（eventLoopGroupBoss，即为上面的1）负责监听 TCP网络连接请求，建立好连接，创建SocketChannel，并注册到selector上。RocketMQ的源码中会自动根据OS的类型选择NIO和Epoll，也可以通过参数配置），然后监听真正的网络数据。拿到网络数据后，再丢给Worker线程池（eventLoopGroupSelector，即为上面的“N”，源码中默认设置为3），在真正执行业务逻辑之前需要进行SSL验证、编解码、空闲检查、网络连接管理，这些工作交给defaultEventExecutorGroup（即为上面的“M1”，源码中默认设置为8）去做。而处理业务操作放在业务线程池中执行，根据 RomotingCommand 的业务请求码code去processorTable这个本地缓存变量中找到对应的 processor，然后封装成task任务后，提交给对应的业务processor处理线程池来执行（sendMessageExecutor，以发送消息为例，即为上面的 “M2”）。从入口到业务逻辑的几个步骤中线程池一直再增加，这跟每一步逻辑复杂性相关，越复杂，需要的并发通道越宽。\n\n线程数 | 线程名 | 线程具体说明\n --- | --- | --- \n1 | NettyBoss_%d | Reactor 主线程\nN | NettyServerEPOLLSelector_%d_%d | Reactor 线程池\nM1 | NettyServerCodecThread_%d | Worker线程池\nM2 | RemotingExecutorThread_%d | 业务processor处理线程池\n\n### 3 消息过滤\nRocketMQ分布式消息队列的消息过滤方式有别于其它MQ中间件，是在Consumer端订阅消息时再做消息过滤的。RocketMQ这么做是在于其Producer端写入消息和Consumer端订阅消息采用分离存储的机制来实现的，Consumer端订阅消息是需要通过ConsumeQueue这个消息消费的逻辑队列拿到一个索引，然后再从CommitLog里面读取真正的消息实体内容，所以说到底也是还绕不开其存储结构。其ConsumeQueue的存储结构如下，可以看到其中有8个字节存储的Message Tag的哈希值，基于Tag的消息过滤正是基于这个字段值的。\n\n![](image/rocketmq_design_7.png)\n\n主要支持如下2种的过滤方式\n(1) Tag过滤方式：Consumer端在订阅消息时除了指定Topic还可以指定TAG，如果一个消息有多个TAG，可以用||分隔。其中，Consumer端会将这个订阅请求构建成一个 SubscriptionData，发送一个Pull消息的请求给Broker端。Broker端从RocketMQ的文件存储层—Store读取数据之前，会用这些数据先构建一个MessageFilter，然后传给Store。Store从 ConsumeQueue读取到一条记录后，会用它记录的消息tag hash值去做过滤，由于在服务端只是根据hashcode进行判断，无法精确对tag原始字符串进行过滤，故在消息消费端拉取到消息后，还需要对消息的原始tag字符串进行比对，如果不同，则丢弃该消息，不进行消息消费。\n\n(2) SQL92的过滤方式：这种方式的大致做法和上面的Tag过滤方式一样，只是在Store层的具体过滤过程不太一样，真正的 SQL expression 的构建和执行由rocketmq-filter模块负责的。每次过滤都去执行SQL表达式会影响效率，所以RocketMQ使用了BloomFilter避免了每次都去执行。SQL92的表达式上下文为消息的属性。\n\n### 4 负载均衡\nRocketMQ中的负载均衡都在Client端完成，具体来说的话，主要可以分为Producer端发送消息时候的负载均衡和Consumer端订阅消息的负载均衡。\n#### 4.1 Producer的负载均衡\nProducer端在发送消息的时候，会先根据Topic找到指定的TopicPublishInfo，在获取了TopicPublishInfo路由信息后，RocketMQ的客户端在默认方式下selectOneMessageQueue()方法会从TopicPublishInfo中的messageQueueList中选择一个队列（MessageQueue）进行发送消息。具体的容错策略均在MQFaultStrategy这个类中定义。这里有一个sendLatencyFaultEnable开关变量，如果开启，在随机递增取模的基础上，再过滤掉not available的Broker代理。所谓的\"latencyFaultTolerance\"，是指对之前失败的，按一定的时间做退避。例如，如果上次请求的latency超过550L ms，就退避30000L ms；超过1000L，就退避60000L；如果关闭，采用随机递增取模的方式选择一个队列（MessageQueue）来发送消息，latencyFaultTolerance机制是实现消息发送高可用的核心关键所在。\n#### 4.2 Consumer的负载均衡\n在RocketMQ中，Consumer端的两种消费模式（Push/Pull）都是基于拉模式来获取消息的，而在Push模式只是对pull模式的一种封装，其本质实现为消息拉取线程在从服务器拉取到一批消息后，然后提交到消息消费线程池后，又“马不停蹄”的继续向服务器再次尝试拉取消息。如果未拉取到消息，则延迟一下又继续拉取。在两种基于拉模式的消费方式（Push/Pull）中，均需要Consumer端知道从Broker端的哪一个消息队列中去获取消息。因此，有必要在Consumer端来做负载均衡，即Broker端中多个MessageQueue分配给同一个ConsumerGroup中的哪些Consumer消费。\n\n1、Consumer端的心跳包发送\n\n在Consumer启动后，它就会通过定时任务不断地向RocketMQ集群中的所有Broker实例发送心跳包（其中包含了，消息消费分组名称、订阅关系集合、消息通信模式和客户端id的值等信息）。Broker端在收到Consumer的心跳消息后，会将它维护在ConsumerManager的本地缓存变量—consumerTable，同时并将封装后的客户端网络通道信息保存在本地缓存变量—channelInfoTable中，为之后做Consumer端的负载均衡提供可以依据的元数据信息。\n\n2、Consumer端实现负载均衡的核心类—RebalanceImpl\n\n在Consumer实例的启动流程中的启动MQClientInstance实例部分，会完成负载均衡服务线程—RebalanceService的启动（每隔20s执行一次）。通过查看源码可以发现，RebalanceService线程的run()方法最终调用的是RebalanceImpl类的rebalanceByTopic()方法，该方法是实现Consumer端负载均衡的核心。这里，rebalanceByTopic()方法会根据消费者通信类型为“广播模式”还是“集群模式”做不同的逻辑处理。这里主要来看下集群模式下的主要处理流程：\n\n(1) 从rebalanceImpl实例的本地缓存变量—topicSubscribeInfoTable中，获取该Topic主题下的消息消费队列集合（mqSet）；\n\n(2) 根据topic和consumerGroup为参数调用mQClientFactory.findConsumerIdList()方法向Broker端发送获取该消费组下消费者Id列表的RPC通信请求（Broker端基于前面Consumer端上报的心跳包数据而构建的consumerTable做出响应返回，业务请求码：GET_CONSUMER_LIST_BY_GROUP）；\n\n(3) 先对Topic下的消息消费队列、消费者Id排序，然后用消息队列分配策略算法（默认为：消息队列的平均分配算法），计算出待拉取的消息队列。这里的平均分配算法，类似于分页的算法，将所有MessageQueue排好序类似于记录，将所有消费端Consumer排好序类似页数，并求出每一页需要包含的平均size和每个页面记录的范围range，最后遍历整个range而计算出当前Consumer端应该分配到的记录（这里即为：MessageQueue）。\n\n![](image/rocketmq_design_8.png)\n\n(4) 然后，调用updateProcessQueueTableInRebalance()方法，具体的做法是，先将分配到的消息队列集合（mqSet）与processQueueTable做一个过滤比对。\n\n![](image/rocketmq_design_9.png)\n\n- 上图中processQueueTable标注的红色部分，表示与分配到的消息队列集合mqSet互不包含。将这些队列设置Dropped属性为true，然后查看这些队列是否可以移除出processQueueTable缓存变量，这里具体执行removeUnnecessaryMessageQueue()方法，即每隔1s 查看是否可以获取当前消费处理队列的锁，拿到的话返回true。如果等待1s后，仍然拿不到当前消费处理队列的锁则返回false。如果返回true，则从processQueueTable缓存变量中移除对应的Entry；\n\n- 上图中processQueueTable的绿色部分，表示与分配到的消息队列集合mqSet的交集。判断该ProcessQueue是否已经过期了，在Pull模式的不用管，如果是Push模式的，设置Dropped属性为true，并且调用removeUnnecessaryMessageQueue()方法，像上面一样尝试移除Entry；\n\n最后，为过滤后的消息队列集合（mqSet）中的每个MessageQueue创建一个ProcessQueue对象并存入RebalanceImpl的processQueueTable队列中（其中调用RebalanceImpl实例的computePullFromWhere(MessageQueue mq)方法获取该MessageQueue对象的下一个进度消费值offset，随后填充至接下来要创建的pullRequest对象属性中），并创建拉取请求对象—pullRequest添加到拉取列表—pullRequestList中，最后执行dispatchPullRequest()方法，将Pull消息的请求对象PullRequest依次放入PullMessageService服务线程的阻塞队列pullRequestQueue中，待该服务线程取出后向Broker端发起Pull消息的请求。\n\n消息消费队列在同一消费组不同消费者之间的负载均衡，其核心设计理念是在一个消息消费队列在同一时间只允许被同一消费组内的一个消费者消费，一个消息消费者能同时消费多个消息队列。\n\n### 5 事务消息\nApache RocketMQ在4.3.0版中已经支持分布式事务消息，这里RocketMQ采用了2PC的思想来实现了提交事务消息，同时增加一个补偿逻辑来处理二阶段超时或者失败的消息，如下图所示。\n\n![](image/rocketmq_design_10.png)\n\n#### 5.1 RocketMQ事务消息流程概要\n上图说明了事务消息的大致方案，其中分为两个流程：正常事务消息的发送及提交、事务消息的补偿流程。\n\n1.事务消息发送及提交：\n\n(1) 发送消息（half消息）。\n\n(2) 服务端响应消息写入结果。\n\n(3) 根据发送结果执行本地事务（如果写入失败，此时half消息对业务不可见，本地逻辑不执行）。\n\n(4) 根据本地事务状态执行Commit或者Rollback（Commit操作生成消息索引，消息对消费者可见）\n\n2.补偿流程：\n\n(1) 对没有Commit/Rollback的事务消息（pending状态的消息），从服务端发起一次“回查”\n\n(2) Producer收到回查消息，检查回查消息对应的本地事务的状态\n\n(3) 根据本地事务状态，重新Commit或者Rollback\n\n其中，补偿阶段用于解决消息Commit或者Rollback发生超时或者失败的情况。\n#### 5.2 RocketMQ事务消息设计\n1.事务消息在一阶段对用户不可见\n\n在RocketMQ事务消息的主要流程中，一阶段的消息如何对用户不可见。其中，事务消息相对普通消息最大的特点就是一阶段发送的消息对用户是不可见的。那么，如何做到写入消息但是对用户不可见呢？RocketMQ事务消息的做法是：如果消息是half消息，将备份原消息的主题与消息消费队列，然后改变主题为RMQ_SYS_TRANS_HALF_TOPIC。由于消费组未订阅该主题，故消费端无法消费half类型的消息，然后RocketMQ会开启一个定时任务，从Topic为RMQ_SYS_TRANS_HALF_TOPIC中拉取消息进行消费，根据生产者组获取一个服务提供者发送回查事务状态请求，根据事务状态来决定是提交或回滚消息。\n\n在RocketMQ中，消息在服务端的存储结构如下，每条消息都会有对应的索引信息，Consumer通过ConsumeQueue这个二级索引来读取消息实体内容，其流程如下：\n\n![](image/rocketmq_design_11.png)\n\nRocketMQ的具体实现策略是：写入的如果事务消息，对消息的Topic和Queue等属性进行替换，同时将原来的Topic和Queue信息存储到消息的属性中，正因为消息主题被替换，故消息并不会转发到该原主题的消息消费队列，消费者无法感知消息的存在，不会消费。其实改变消息主题是RocketMQ的常用“套路”，回想一下延时消息的实现机制。\n\n2.Commit和Rollback操作以及Op消息的引入\n\n在完成一阶段写入一条对用户不可见的消息后，二阶段如果是Commit操作，则需要让消息对用户可见；如果是Rollback则需要撤销一阶段的消息。先说Rollback的情况。对于Rollback，本身一阶段的消息对用户是不可见的，其实不需要真正撤销消息（实际上RocketMQ也无法去真正的删除一条消息，因为是顺序写文件的）。但是区别于这条消息没有确定状态（Pending状态，事务悬而未决），需要一个操作来标识这条消息的最终状态。RocketMQ事务消息方案中引入了Op消息的概念，用Op消息标识事务消息已经确定的状态（Commit或者Rollback）。如果一条事务消息没有对应的Op消息，说明这个事务的状态还无法确定（可能是二阶段失败了）。引入Op消息后，事务消息无论是Commit或者Rollback都会记录一个Op操作。Commit相对于Rollback只是在写入Op消息前创建Half消息的索引。\n\n3.Op消息的存储和对应关系\n\nRocketMQ将Op消息写入到全局一个特定的Topic中通过源码中的方法—TransactionalMessageUtil.buildOpTopic()；这个Topic是一个内部的Topic（像Half消息的Topic一样），不会被用户消费。Op消息的内容为对应的Half消息的存储的Offset，这样通过Op消息能索引到Half消息进行后续的回查操作。\n\n![](image/rocketmq_design_12.png)\n\n4.Half消息的索引构建\n\n在执行二阶段Commit操作时，需要构建出Half消息的索引。一阶段的Half消息由于是写到一个特殊的Topic，所以二阶段构建索引时需要读取出Half消息，并将Topic和Queue替换成真正的目标的Topic和Queue，之后通过一次普通消息的写入操作来生成一条对用户可见的消息。所以RocketMQ事务消息二阶段其实是利用了一阶段存储的消息的内容，在二阶段时恢复出一条完整的普通消息，然后走一遍消息写入流程。\n\n5.如何处理二阶段失败的消息？\n\n如果在RocketMQ事务消息的二阶段过程中失败了，例如在做Commit操作时，出现网络问题导致Commit失败，那么需要通过一定的策略使这条消息最终被Commit。RocketMQ采用了一种补偿机制，称为“回查”。Broker端对未确定状态的消息发起回查，将消息发送到对应的Producer端（同一个Group的Producer），由Producer根据消息来检查本地事务的状态，进而执行Commit或者Rollback。Broker端通过对比Half消息和Op消息进行事务消息的回查并且推进CheckPoint（记录那些事务消息的状态是确定的）。\n\n值得注意的是，rocketmq并不会无休止的的信息事务状态回查，默认回查15次，如果15次回查还是无法得知事务状态，rocketmq默认回滚该消息。\n### 6 消息查询\nRocketMQ支持按照下面两种维度（“按照Message Id查询消息”、“按照Message Key查询消息”）进行消息查询。\n#### 6.1   按照MessageId查询消息\nRocketMQ中的MessageId的长度总共有16字节，其中包含了消息存储主机地址（IP地址和端口），消息Commit Log offset。“按照MessageId查询消息”在RocketMQ中具体做法是：Client端从MessageId中解析出Broker的地址（IP地址和端口）和Commit Log的偏移地址后封装成一个RPC请求后通过Remoting通信层发送（业务请求码：VIEW_MESSAGE_BY_ID）。Broker端走的是QueryMessageProcessor，读取消息的过程用其中的 commitLog offset 和 size 去 commitLog 中找到真正的记录并解析成一个完整的消息返回。\n#### 6.2  按照Message Key查询消息\n“按照Message Key查询消息”，主要是基于RocketMQ的IndexFile索引文件来实现的。RocketMQ的索引文件逻辑结构，类似JDK中HashMap的实现。索引文件的具体结构如下：\n\n![](image/rocketmq_design_13.png)\n\nIndexFile索引文件为用户提供通过“按照Message Key查询消息”的消息索引查询服务，IndexFile文件的存储位置是：$HOME\\store\\index\\${fileName}，文件名fileName是以创建时的时间戳命名的，文件大小是固定的，等于40+500W\\*4+2000W\\*20= 420000040个字节大小。如果消息的properties中设置了UNIQ_KEY这个属性，就用 topic + “#” + UNIQ_KEY的value作为 key 来做写入操作。如果消息设置了KEYS属性（多个KEY以空格分隔），也会用 topic + “#” + KEY 来做索引。\n\n其中的索引数据包含了Key Hash/CommitLog Offset/Timestamp/NextIndex offset 这四个字段，一共20 Byte。NextIndex offset 即前面读出来的 slotValue，如果有 hash冲突，就可以用这个字段将所有冲突的索引用链表的方式串起来了。Timestamp记录的是消息storeTimestamp之间的差，并不是一个绝对的时间。整个Index File的结构如图，40 Byte 的Header用于保存一些总的统计信息，4\\*500W的 Slot Table并不保存真正的索引数据，而是保存每个槽位对应的单向链表的头。20\\*2000W 是真正的索引数据，即一个 Index File 可以保存 2000W个索引。\n\n“按照Message Key查询消息”的方式，RocketMQ的具体做法是，主要通过Broker端的QueryMessageProcessor业务处理器来查询，读取消息的过程就是用topic和key找到IndexFile索引文件中的一条记录，根据其中的commitLog offset从CommitLog文件中读取消息的实体内容。\n"
  },
  {
    "path": "docs/cn/dledger/deploy_guide.md",
    "content": "# Dledger集群搭建\n> 该模式为4.x的切换方式，建议采用 5.x [自动主从切换](../controller/design.md)\n---\n## 前言\n该文档主要介绍如何部署自动容灾切换的 RocketMQ-on-DLedger Group。\n\nRocketMQ-on-DLedger Group 是指一组**相同名称**的 Broker，至少需要 3 个节点，通过 Raft 自动选举出一个 Leader，其余节点 作为 Follower，并在 Leader 和 Follower 之间复制数据以保证高可用。  \nRocketMQ-on-DLedger Group 能自动容灾切换，并保证数据一致。  \nRocketMQ-on-DLedger Group 是可以水平扩展的，也即可以部署任意多个 RocketMQ-on-DLedger Group 同时对外提供服务。  \n\n## 1. 新集群部署\n\n#### 1.1 编写配置\n每个 RocketMQ-on-DLedger Group 至少准备三台机器（本文假设为 3）。  \n编写 3 个配置文件，建议参考 conf/dledger 目录下的配置文件样例。  \n关键配置介绍：  \n\n| name | 含义 | 举例 |\n| --- | --- | --- |\n| enableDLegerCommitLog | 是否启动 DLedger  | true |\n| dLegerGroup | DLedger Raft Group的名字，建议和 brokerName 保持一致 | RaftNode00 |\n| dLegerPeers | DLedger Group 内各节点的端口信息，同一个 Group 内的各个节点配置必须要保证一致 | n0-127.0.0.1:40911;n1-127.0.0.1:40912;n2-127.0.0.1:40913 |\n| dLegerSelfId | 节点 id，必须属于 dLegerPeers 中的一个；同 Group 内各个节点要唯一 | n0 |\n| sendMessageThreadPoolNums | 发送线程个数，建议配置成 Cpu 核数 | 16 |\n\n这里贴出 conf/dledger/broker-n0.conf 的配置举例。  \n\n```\nbrokerClusterName = RaftCluster\nbrokerName=RaftNode00\nlistenPort=30911\nnamesrvAddr=127.0.0.1:9876\nstorePathRootDir=/tmp/rmqstore/node00\nstorePathCommitLog=/tmp/rmqstore/node00/commitlog\nenableDLegerCommitLog=true\ndLegerGroup=RaftNode00\ndLegerPeers=n0-127.0.0.1:40911;n1-127.0.0.1:40912;n2-127.0.0.1:40913\n## must be unique\ndLegerSelfId=n0\nsendMessageThreadPoolNums=16\n```\n\n### 1.2 启动 Broker\n\n与老版本的启动方式一致。\n\n`nohup sh bin/mqbroker -c conf/dledger/xxx-n0.conf & `  \n`nohup sh bin/mqbroker -c conf/dledger/xxx-n1.conf & `  \n`nohup sh bin/mqbroker -c conf/dledger/xxx-n2.conf & `  \n\n\n## 2. 旧集群升级\n\n如果旧集群采用 Master 方式部署，则每个 Master 都需要转换成一个 RocketMQ-on-DLedger Group。  \n如果旧集群采用 Master-Slave 方式部署，则每个 Master-Slave 组都需要转换成一个 RocketMQ-on-DLedger Group。\n\n### 2.1 杀掉旧的 Broker\n\n可以通过 kill 命令来完成，也可以调用 `bin/mqshutdown broker`。\n\n### 2.2 检查旧的 Commitlog\n\nRocketMQ-on-DLedger 组中的每个节点，可以兼容旧的 Commitlog ，但其 Raft 复制过程，只能针对新增加的消息。因此，为了避免出现异常，需要保证 旧的 Commitlog 是一致的。  \n如果旧的集群是采用 Master-Slave 方式部署，有可能在shutdown时，其数据并不是一致的，建议通过md5sum 的方式，检查最近的最少 2 个 Commmitlog 文件，如果发现不一致，则通过拷贝的方式进行对齐。  \n\n虽然 RocketMQ-on-DLedger Group 也可以以 2 节点方式部署，但其会丧失容灾切换能力（2n + 1 原则，至少需要3个节点才能容忍其中 1 个宕机）。  \n所以在对齐了 Master 和 Slave 的 Commitlog 之后，还需要准备第 3 台机器，并把旧的 Commitlog 从 Master 拷贝到 第 3 台机器（记得同时拷贝一下 config 文件夹）。  \n   \n在 3 台机器准备好了之后，旧 Commitlog 文件也保证一致之后，就可以开始走下一步修改配置了。\n\n### 2.3 修改配置\n\n参考新集群部署。\n\n### 2.4 重新启动 Broker \n\n参考新集群部署。\n\n\n"
  },
  {
    "path": "docs/cn/dledger/quick_start.md",
    "content": "# Dledger快速搭建\n> 该模式为4.x的切换方式，建议采用 5.x [自动主从切换](../controller/quick_start.md)\n---\n### 前言\n该文档主要介绍如何快速构建和部署基于 DLedger 的可以自动容灾切换的 RocketMQ 集群。\n\n详细的新集群部署和旧集群升级指南请参考 [部署指南](deploy_guide.md)。\n\n### 1. 源码构建\n构建分为两个部分，需要先构建 DLedger，然后 构建 RocketMQ\n\n#### 1.1 构建 DLedger\n\n```shell\n$ git clone https://github.com/openmessaging/dledger.git\n$ cd dledger\n$ mvn clean install -DskipTests\n```\n\n#### 1.2 构建 RocketMQ\n\n```shell\n$ git clone https://github.com/apache/rocketmq.git\n$ cd rocketmq\n$ git checkout -b store_with_dledger origin/store_with_dledger\n$ mvn -Prelease-all -DskipTests clean install -U\n```\n\n### 2. 快速部署\n\n在构建成功后\n\n```shell\n#{rocketmq-version} replace with rocketmq actual version. example: 5.0.0-SNAPSHOT\n$ cd distribution/target/rocketmq-{rocketmq-version}/rocketmq-{rocketmq-version}\n$ sh bin/dledger/fast-try.sh start\n```\n\n如果上面的步骤执行成功，可以通过 mqadmin 运维命令查看集群状态。\n\n```shell\n$ sh bin/mqadmin clusterList -n 127.0.0.1:9876\n```\n\n顺利的话，会看到如下内容：\n\n![ClusterList](https://img.alicdn.com/5476e8b07b923/TB11Z.ZyCzqK1RjSZFLXXcn2XXa)\n\n（BID 为 0 的表示 Master，其余都是 Follower）\n\n启动成功，现在可以向集群收发消息，并进行容灾切换测试了。\n\n关闭快速集群，可以执行：\n\n```shell\n$ sh bin/dledger/fast-try.sh stop\n```\n\n快速部署，默认配置在 conf/dledger 里面，默认的存储路径在 /tmp/rmqstore。\n\n\n### 3. 容灾切换\n\n部署成功，杀掉 Leader 之后（在上面的例子中，杀掉端口 30931 所在的进程），等待约 10s 左右，用 clusterList 命令查看集群，就会发现 Leader 切换到另一个节点了。\n\n\n\n\n\n"
  },
  {
    "path": "docs/cn/features.md",
    "content": "# 特性(features)\n----\n## 1 订阅与发布\n消息的发布是指某个生产者向某个topic发送消息；消息的订阅是指某个消费者关注了某个topic中带有某些tag的消息，进而从该topic消费数据。\n## 2 消息顺序\n消息有序指的是一类消息消费时，能按照发送的顺序来消费。例如：一个订单产生了三条消息分别是订单创建、订单付款、订单完成。消费时要按照这个顺序消费才能有意义，但是同时订单之间是可以并行消费的。RocketMQ可以严格的保证消息有序。\n\n顺序消息分为全局顺序消息与分区顺序消息，全局顺序是指某个Topic下的所有消息都要保证顺序；部分顺序消息只要保证每一组消息被顺序消费即可。\n- 全局顺序\n对于指定的一个 Topic，所有消息按照严格的先入先出（FIFO）的顺序进行发布和消费。\n适用场景：性能要求不高，所有的消息严格按照 FIFO 原则进行消息发布和消费的场景\n- 分区顺序\n对于指定的一个 Topic，所有消息根据 sharding key 进行区块分区。 同一个分区内的消息按照严格的 FIFO 顺序进行发布和消费。 Sharding key 是顺序消息中用来区分不同分区的关键字段，和普通消息的 Key 是完全不同的概念。\n适用场景：性能要求高，以 sharding key 作为分区字段，在同一个区块中严格的按照 FIFO 原则进行消息发布和消费的场景。\n## 3 消息过滤\nRocketMQ的消费者可以根据Tag进行消息过滤，也支持自定义属性过滤。消息过滤目前是在Broker端实现的，优点是减少了对于Consumer无用消息的网络传输，缺点是增加了Broker的负担、而且实现相对复杂。\n## 4 消息可靠性\nRocketMQ支持消息的高可靠，影响消息可靠性的几种情况：\n1) Broker非正常关闭\n2) Broker异常Crash\n3) OS Crash\n4) 机器掉电，但是能立即恢复供电情况\n5) 机器无法开机（可能是cpu、主板、内存等关键设备损坏）\n6) 磁盘设备损坏\n\n1)、2)、3)、4) 四种情况都属于硬件资源可立即恢复情况，RocketMQ在这四种情况下能保证消息不丢，或者丢失少量数据（依赖刷盘方式是同步还是异步）。\n\n5)、6)属于单点故障，且无法恢复，一旦发生，在此单点上的消息全部丢失。RocketMQ在这两种情况下，通过异步复制，可保证99%的消息不丢，但是仍然会有极少量的消息可能丢失。通过同步双写技术可以完全避免单点，同步双写势必会影响性能，适合对消息可靠性要求极高的场合，例如与Money相关的应用。注：RocketMQ从3.0版本开始支持同步双写。\n\n## 5 至少一次\n至少一次(At least Once)指每个消息必须投递一次。Consumer先Pull消息到本地，消费完成后，才向服务器返回ack，如果没有消费一定不会ack消息，所以RocketMQ可以很好的支持此特性。\n\n## 6 回溯消费\n回溯消费是指Consumer已经消费成功的消息，由于业务上需求需要重新消费，要支持此功能，Broker在向Consumer投递成功消息后，消息仍然需要保留。并且重新消费一般是按照时间维度，例如由于Consumer系统故障，恢复后需要重新消费1小时前的数据，那么Broker要提供一种机制，可以按照时间维度来回退消费进度。RocketMQ支持按照时间回溯消费，时间维度精确到毫秒。\n\n## 7 事务消息\nRocketMQ事务消息（Transactional Message）是指应用本地事务和发送消息操作可以被定义到全局事务中，要么同时成功，要么同时失败。RocketMQ的事务消息提供类似 X/Open XA 的分布事务功能，通过事务消息能达到分布式事务的最终一致。\n## 8 定时消息\n定时消息（延迟队列）是指消息发送到broker后，不会立即被消费，等待特定时间投递给真正的topic。\nbroker有配置项messageDelayLevel，默认值为“1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h”，18个level。可以配置自定义messageDelayLevel。注意，messageDelayLevel是broker的属性，不属于某个topic。发消息时，设置delayLevel等级即可：msg.setDelayLevel(level)。level有以下三种情况：\n\n- level == 0，消息为非延迟消息\n- 1<=level<=maxLevel，消息延迟特定时间，例如level==1，延迟1s\n- level > maxLevel，则level== maxLevel，例如level==20，延迟2h\n\n定时消息会暂存在名为SCHEDULE_TOPIC_XXXX的topic中，并根据delayTimeLevel存入特定的queue，queueId = delayTimeLevel – 1，即一个queue只存相同延迟的消息，保证具有相同发送延迟的消息能够顺序消费。broker会调度地消费SCHEDULE_TOPIC_XXXX，将消息写入真实的topic。\n\n需要注意的是，定时消息会在第一次写入和调度写入真实topic时都会计数，因此发送数量、tps都会变高。\n\n## 9 消息重试\nConsumer消费消息失败后，要提供一种重试机制，令消息再消费一次。Consumer消费消息失败通常可以认为有以下几种情况：\n- 由于消息本身的原因，例如反序列化失败，消息数据本身无法处理（例如话费充值，当前消息的手机号被注销，无法充值）等。这种错误通常需要跳过这条消息，再消费其它消息，而这条失败的消息即使立刻重试消费，99%也不成功，所以最好提供一种定时重试机制，即过10秒后再重试。\n- 由于依赖的下游应用服务不可用，例如db连接不可用，外系统网络不可达等。遇到这种错误，即使跳过当前失败的消息，消费其他消息同样也会报错。这种情况建议应用sleep 30s，再消费下一条消息，这样可以减轻Broker重试消息的压力。\n\nRocketMQ会为每个消费组都设置一个Topic名称为“%RETRY%+consumerGroup”的重试队列（这里需要注意的是，这个Topic的重试队列是针对消费组，而不是针对每个Topic设置的），用于暂时保存因为各种异常而导致Consumer端无法消费的消息。考虑到异常恢复起来需要一些时间，会为重试队列设置多个重试级别，每个重试级别都有与之对应的重新投递延时，重试次数越多投递延时就越大。RocketMQ对于重试消息的处理是先保存至Topic名称为“SCHEDULE_TOPIC_XXXX”的延迟队列中，后台定时任务按照对应的时间进行Delay后重新保存至“%RETRY%+consumerGroup”的重试队列中。\n## 10 消息重投\n生产者在发送消息时，同步消息失败会重投，异步消息有重试，oneway没有任何保证。消息重投保证消息尽可能发送成功、不丢失，但可能会造成消息重复，消息重复在RocketMQ中是无法避免的问题。消息重复在一般情况下不会发生，当出现消息量大、网络抖动，消息重复就会是大概率事件。另外，生产者主动重发、consumer负载变化也会导致重复消息。如下方法可以设置消息重试策略：\n\n- retryTimesWhenSendFailed：同步发送失败重投次数，默认为2，因此生产者会最多尝试发送retryTimesWhenSendFailed + 1次。不会选择上次失败的broker，尝试向其他broker发送，最大程度保证消息不丢。超过重投次数，抛出异常，由客户端保证消息不丢。当出现RemotingException、MQClientException和部分MQBrokerException时会重投。\n- retryTimesWhenSendAsyncFailed：异步发送失败重试次数，异步重试不会选择其他broker，仅在同一个broker上做重试，不保证消息不丢。\n- retryAnotherBrokerWhenNotStoreOK：消息刷盘（主或备）超时或slave不可用（返回状态非SEND_OK），是否尝试发送到其他broker，默认false。十分重要消息可以开启。\n\n## 11 流量控制\n生产者流控，因为broker处理能力达到瓶颈；消费者流控，因为消费能力达到瓶颈。\n\n生产者流控：\n- commitLog文件被锁时间超过osPageCacheBusyTimeOutMills时，参数默认为1000ms，返回流控。\n- 如果开启transientStorePoolEnable == true，且broker为异步刷盘的主机，且transientStorePool中资源不足，拒绝当前send请求，返回流控。\n- broker每隔10ms检查send请求队列头部请求的等待时间，如果超过waitTimeMillsInSendQueue，默认200ms，拒绝当前send请求，返回流控。\n- broker通过拒绝send 请求方式实现流量控制。\n\n注意，生产者流控，不会尝试消息重投。\n\n消费者流控：\n- 消费者本地缓存消息数超过pullThresholdForQueue时，默认1000。\n- 消费者本地缓存消息大小超过pullThresholdSizeForQueue时，默认100MB。\n- 消费者本地缓存消息跨度超过consumeConcurrentlyMaxSpan时，默认2000。\n\n消费者流控的结果是降低拉取频率。\n## 12 死信队列\n死信队列用于处理无法被正常消费的消息。当一条消息初次消费失败，消息队列会自动进行消息重试；达到最大重试次数后，若消费依然失败，则表明消费者在正常情况下无法正确地消费该消息，此时，消息队列 不会立刻将消息丢弃，而是将其发送到该消费者对应的特殊队列中。\n\nRocketMQ将这种正常情况下无法被消费的消息称为死信消息（Dead-Letter Message），将存储死信消息的特殊队列称为死信队列（Dead-Letter Queue）。在RocketMQ中，可以通过使用console控制台对死信队列中的消息进行重发来使得消费者实例再次进行消费。\n\n"
  },
  {
    "path": "docs/cn/msg_trace/user_guide.md",
    "content": "# 消息轨迹\n----\n\n## 1. 消息轨迹数据关键属性\n| Producer端| Consumer端 | Broker端 |\n| --- | --- | --- |\n| 生产实例信息 | 消费实例信息 | 消息的Topic |\n| 发送消息时间 | 投递时间，投递轮次  | 消息存储位置 |\n| 消息是否发送成功 | 消息是否消费成功 | 消息的Key值 |\n| 发送耗时 | 消费耗时 | 消息的Tag值 |\n\n## 2. 支持消息轨迹集群部署\n\n### 2.1 Broker端配置文件\n这里贴出Broker端开启消息轨迹特性的properties配置文件内容：\n```\nbrokerClusterName=DefaultCluster\nbrokerName=broker-a\nbrokerId=0\ndeleteWhen=04\nfileReservedTime=48\nbrokerRole=ASYNC_MASTER\nflushDiskType=ASYNC_FLUSH\nstorePathRootDir=/data/rocketmq/rootdir-a-m\nstorePathCommitLog=/data/rocketmq/commitlog-a-m\nautoCreateSubscriptionGroup=true\n## if msg tracing is open,the flag will be true\ntraceTopicEnable=true\nlistenPort=10911\nbrokerIP1=XX.XX.XX.XX1\nnamesrvAddr=XX.XX.XX.XX:9876\n```\n\n### 2.2 普通模式\nRocketMQ集群中每一个Broker节点均用于存储Client端收集并发送过来的消息轨迹数据。因此，对于RocketMQ集群中的Broker节点数量并无要求和限制。\n\n### 2.3 物理IO隔离模式\n对于消息轨迹数据量较大的场景，可以在RocketMQ集群中选择其中一个Broker节点专用于存储消息轨迹，使得用户普通的消息数据与消息轨迹数据的物理IO完全隔离，互不影响。在该模式下，RocketMQ集群中至少有两个Broker节点，其中一个Broker节点定义为存储消息轨迹数据的服务端。\n\n### 2.4 启动开启消息轨迹的Broker\n`nohup sh mqbroker -c ../conf/2m-noslave/broker-a.properties &`\n  \n## 3. 保存消息轨迹的Topic定义\nRocketMQ的消息轨迹特性支持两种存储轨迹数据的方式：\n\n### 3.1 系统级的TraceTopic\n在默认情况下，消息轨迹数据是存储于系统级的TraceTopic中(其名称为：**RMQ_SYS_TRACE_TOPIC**)。该Topic在Broker节点启动时，会自动创建出来（如上所叙，需要在Broker端的配置文件中将**traceTopicEnable**的开关变量设置为**true**）。\n\n### 3.2 用户自定义的TraceTopic \n如果用户不准备将消息轨迹的数据存储于系统级的默认TraceTopic，也可以自己定义并创建用户级的Topic来保存轨迹（即为创建普通的Topic用于保存消息轨迹数据）。下面一节会介绍Client客户端的接口如何支持用户自定义的TraceTopic。\n\n## 4. 支持消息轨迹的Client客户端实践\n为了尽可能地减少用户业务系统使用RocketMQ消息轨迹特性的改造工作量，作者在设计时候采用对原来接口增加一个开关参数(**enableMsgTrace**)来实现消息轨迹是否开启；并新增一个自定义参数(**customizedTraceTopic**)来实现用户存储消息轨迹数据至自己创建的用户级Topic。\n\n### 4.1 发送消息时开启消息轨迹\n```java\n        DefaultMQProducer producer = new DefaultMQProducer(\"ProducerGroupName\",true);\n        producer.setNamesrvAddr(\"XX.XX.XX.XX1\");\n        producer.start();\n            try {\n                {\n                    Message msg = new Message(\"TopicTest\",\n                        \"TagA\",\n                        \"OrderID188\",\n                        \"Hello world\".getBytes(RemotingHelper.DEFAULT_CHARSET));\n                    SendResult sendResult = producer.send(msg);\n                    System.out.printf(\"%s%n\", sendResult);\n                }\n\n            } catch (Exception e) {\n                e.printStackTrace();\n            }\n```\n\n### 4.2 订阅消息时开启消息轨迹\n```java\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"CID_JODIE_1\",true);\n        consumer.subscribe(\"TopicTest\", \"*\");\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        consumer.setConsumeTimestamp(\"20181109221800\");\n        consumer.registerMessageListener(new MessageListenerConcurrently() {\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {\n                System.out.printf(\"%s Receive New Messages: %s %n\", Thread.currentThread().getName(), msgs);\n                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n            }\n        });\n        consumer.start();\n        System.out.printf(\"Consumer Started.%n\");\n```\n\n### 4.3 支持自定义存储消息轨迹Topic\n在上面的发送和订阅消息时候分别将DefaultMQProducer和DefaultMQPushConsumer实例的初始化修改为如下即可支持自定义存储消息轨迹Topic。\n```\n        ##其中Topic_test11111需要用户自己预先创建，来保存消息轨迹；\n        DefaultMQProducer producer = new DefaultMQProducer(\"ProducerGroupName\",true,\"Topic_test11111\");\n        ......\n\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"CID_JODIE_1\",true,\"Topic_test11111\");\n        ......\n```\n\n### 4.4 使用mqadmin命令发送和查看轨迹\n- 发送消息\n```shell\n./mqadmin sendMessage -m true --topic some-topic-name -n 127.0.0.1:9876 -p \"your message content\"\n```\n- 查询轨迹\n```shell\n./mqadmin QueryMsgTraceById -n 127.0.0.1:9876 -i \"some-message-id\"\n```\n- 查询轨迹结果\n```\nRocketMQLog:WARN No appenders could be found for logger (io.netty.util.internal.PlatformDependent0).\nRocketMQLog:WARN Please initialize the logger system properly.\n#Type      #ProducerGroup       #ClientHost          #SendTime            #CostTimes #Status\nPub        1623305799667        xxx.xxx.xxx.xxx       2021-06-10 14:16:40  131ms      success\n```\n"
  },
  {
    "path": "docs/cn/operation.md",
    "content": "\n# 运维管理\n---\n\n### 1   集群搭建\n\n#### 1.1 单Master模式\n\n这种方式风险较大，一旦Broker重启或者宕机时，会导致整个服务不可用。不建议线上环境使用，可以用于本地测试。\n\n##### 1）启动 NameServer\n\n```bash\n### 首先启动Name Server\n$ nohup sh mqnamesrv &\n \n### 验证Name Server 是否启动成功\n$ tail -f ~/logs/rocketmqlogs/namesrv.log\nThe Name Server boot success...\n```\n\n##### 2）启动 Broker\n\n```bash\n### 启动Broker\n$ nohup sh bin/mqbroker -n localhost:9876 &\n\n### 验证Broker是否启动成功，例如Broker的IP为：192.168.1.2，且名称为broker-a\n$ tail -f ~/logs/rocketmqlogs/broker.log \nThe broker[broker-a, 192.169.1.2:10911] boot success...\n```\n\n#### 1.2 多Master模式\n\n一个集群无Slave，全是Master，例如2个Master或者3个Master，这种模式的优缺点如下：\n\n- 优点：配置简单，单个Master宕机或重启维护对应用无影响，在磁盘配置为RAID10时，即使机器宕机不可恢复情况下，由于RAID10磁盘非常可靠，消息也不会丢（异步刷盘丢失少量消息，同步刷盘一条不丢），性能最高；\n\n- 缺点：单台机器宕机期间，这台机器上未被消费的消息在机器恢复之前不可订阅，消息实时性会受到影响。\n\n##### 1）启动NameServer\n\nNameServer需要先于Broker启动，且如果在生产环境使用，为了保证高可用，建议一般规模的集群启动3个NameServer，各节点的启动命令相同，如下：\n\n```bash\n### 首先启动Name Server\n$ nohup sh mqnamesrv &\n \n### 验证Name Server 是否启动成功\n$ tail -f ~/logs/rocketmqlogs/namesrv.log\nThe Name Server boot success...\n```\n\n##### 2）启动Broker集群\n\n```bash\n### 在机器A，启动第一个Master，例如NameServer的IP为：192.168.1.1\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-noslave/broker-a.properties &\n \n### 在机器B，启动第二个Master，例如NameServer的IP为：192.168.1.1\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-noslave/broker-b.properties &\n\n...\n```\n\n如上启动命令是在单个NameServer情况下使用的。对于多个NameServer的集群，Broker启动命令中`-n`后面的地址列表用分号隔开即可，例如 `192.168.1.1:9876;192.161.2:9876`。\n\n#### 1.3 多Master多Slave模式-异步复制\n\n每个Master配置一个Slave，有多对Master-Slave，HA采用异步复制方式，主备有短暂消息延迟（毫秒级），这种模式的优缺点如下：\n\n- 优点：即使磁盘损坏，消息丢失的非常少，且消息实时性不会受影响，同时Master宕机后，消费者仍然可以从Slave消费，而且此过程对应用透明，不需要人工干预，性能同多Master模式几乎一样；\n\n- 缺点：Master宕机，磁盘损坏情况下会丢失少量消息。\n\n##### 1）启动NameServer\n\n```bash\n### 首先启动Name Server\n$ nohup sh mqnamesrv &\n \n### 验证Name Server 是否启动成功\n$ tail -f ~/logs/rocketmqlogs/namesrv.log\nThe Name Server boot success...\n```\n\n##### 2）启动Broker集群\n\n```bash\n### 在机器A，启动第一个Master，例如NameServer的IP为：192.168.1.1\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-a.properties &\n \n### 在机器B，启动第二个Master，例如NameServer的IP为：192.168.1.1\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-b.properties &\n \n### 在机器C，启动第一个Slave，例如NameServer的IP为：192.168.1.1\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-a-s.properties &\n \n### 在机器D，启动第二个Slave，例如NameServer的IP为：192.168.1.1\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-b-s.properties &\n```\n\n#### 1.4 多Master多Slave模式-同步双写\n\n每个Master配置一个Slave，有多对Master-Slave，HA采用同步双写方式，即只有主备都写成功，才向应用返回成功，这种模式的优缺点如下：\n\n- 优点：数据与服务都无单点故障，Master宕机情况下，消息无延迟，服务可用性与数据可用性都非常高；\n\n- 缺点：性能比异步复制模式略低（大约低10%左右），发送单个消息的RT会略高，且目前版本在主节点宕机后，备机不能自动切换为主机。\n\n##### 1）启动NameServer\n\n```bash\n### 首先启动Name Server\n$ nohup sh mqnamesrv &\n \n### 验证Name Server 是否启动成功\n$ tail -f ~/logs/rocketmqlogs/namesrv.log\nThe Name Server boot success...\n```\n\n##### 2）启动Broker集群\n\n```bash\n### 在机器A，启动第一个Master，例如NameServer的IP为：192.168.1.1\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-a.properties &\n \n### 在机器B，启动第二个Master，例如NameServer的IP为：192.168.1.1\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-b.properties &\n \n### 在机器C，启动第一个Slave，例如NameServer的IP为：192.168.1.1\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-a-s.properties &\n \n### 在机器D，启动第二个Slave，例如NameServer的IP为：192.168.1.1\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-b-s.properties &\n```\n\n以上Broker与Slave配对是通过指定相同的BrokerName参数来配对，Master的BrokerId必须是0，Slave的BrokerId必须是大于0的数。另外一个Master下面可以挂载多个Slave，同一Master下的多个Slave通过指定不同的BrokerId来区分。$ROCKETMQ_HOME指的RocketMQ安装目录，需要用户自己设置此环境变量。\n\n#### 1.5  RocketMQ 5.0 自动主从切换\n\nRocketMQ 5.0 开始支持自动主从切换的模式，可参考以下文档\n\n[快速开始](controller/quick_start.md)\n\n[部署文档](controller/deploy.md)\n\n[设计思想](controller/design.md)\n\n### 2 mqadmin管理工具\n\n> 注意：\n>\n> 1. 执行命令方法：`./mqadmin {command} {args}`\n> 2. 几乎所有命令都需要配置-n表示NameServer地址，格式为ip:port\n> 3. 几乎所有命令都可以通过-h获取帮助\n> 4. 如果既有Broker地址（-b）配置项又有clusterName（-c）配置项，则优先以Broker地址执行命令，如果不配置Broker地址，则对集群中所有主机执行命令，只支持一个Broker地址。-b格式为ip:port，port默认是10911\n> 5. 在tools下可以看到很多命令，但并不是所有命令都能使用，只有在MQAdminStartup中初始化的命令才能使用，你也可以修改这个类，增加或自定义命令\n> 6. 由于版本更新问题，少部分命令可能未及时更新，遇到错误请直接阅读相关命令源码\n\n#### 2.1 Topic相关\n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>名称</td>\n  <td class=xl64 width=175 style='width:131pt'>含义</td>\n  <td class=xl64 width=177 style='width:133pt'>命令选项</td>\n  <td class=xl64 width=185 style='width:139pt'>说明</td>\n </tr>\n <tr height=132 style='height:99.0pt'>\n  <td rowspan=8 height=593 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:444.0pt;border-top:none;width:122pt'>updateTopic</td>\n  <td rowspan=8 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>创建更新Topic配置</td>\n  <td class=xl65 width=149 style='width:112pt'>-b</td>\n  <td class=xl66 width=159 style='width:119pt'>Broker 地址，表示 topic 所在\n  Broker，只支持单台Broker，地址为ip:port</td>\n </tr>\n <tr height=132 style='height:99.0pt'>\n  <td height=132 class=xl65 width=149 style='height:99.0pt;width:112pt'>-c</td>\n  <td class=xl66 width=159 style='width:119pt'>cluster 名称，表示 topic 所在集群（集群可通过\n  clusterList 查询）</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-h-</td>\n  <td class=xl66 width=159 style='width:119pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>NameServer服务地址，格式 ip:port</td>\n </tr>\n <tr height=76 style='height:57.0pt'>\n  <td height=76 class=xl65 width=149 style='height:57.0pt;width:112pt'>-p</td>\n  <td class=xl66 width=159 style='width:119pt'>指定新topic的读写权限( W=2|R=4|WR=6 )</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=149 style='height:29.0pt;width:112pt'>-r</td>\n  <td class=xl66 width=159 style='width:119pt'>可读队列数（默认为 8）</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=149 style='height:29.0pt;width:112pt'>-w</td>\n  <td class=xl66 width=159 style='width:119pt'>可写队列数（默认为 8）</td>\n </tr>\n <tr height=95 style='height:71.0pt'>\n  <td height=95 class=xl65 width=149 style='height:71.0pt;width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>topic 名称（名称只能使用字符\n  ^[a-zA-Z0-9_-]+$ ）</td>\n </tr>\n <tr height=132 style='height:99.0pt'>\n  <td rowspan=4 height=307 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:230.0pt;border-top:none;width:122pt'>deleteTopic</td>\n  <td rowspan=4 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>删除Topic</td>\n  <td class=xl65 width=149 style='width:112pt'>-c</td>\n  <td class=xl66 width=159 style='width:119pt'>cluster 名称，表示删除某集群下的某个 topic （集群\n  可通过 clusterList 查询）</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=95 style='height:71.0pt'>\n  <td height=95 class=xl65 width=149 style='height:71.0pt;width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>topic 名称（名称只能使用字符\n  ^[a-zA-Z0-9_-]+$ ）</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=3 height=287 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:215.0pt;border-top:none;width:122pt'>topicList</td>\n  <td rowspan=3 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>查看 Topic 列表信息</td>\n  <td class=xl65 width=149 style='width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>打印帮助</td>\n </tr>\n <tr height=207 style='height:155.0pt'>\n  <td height=207 class=xl65 width=149 style='height:155.0pt;width:112pt'>-c</td>\n  <td class=xl66 width=159 style='width:119pt'>不配置-c只返回topic列表，增加-c返回clusterName,\n  topic, consumerGroup信息，即topic的所属集群和订阅关系，没有参数</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=3 height=103 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:77.0pt;border-top:none;width:122pt'>topicRoute</td>\n  <td rowspan=3 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>查看 Topic 路由信息</td>\n  <td class=xl65 width=149 style='width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>topic 名称</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=3 height=103 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:77.0pt;border-top:none;width:122pt'>topicStatus</td>\n  <td rowspan=3 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>查看 Topic 消息队列offset</td>\n  <td class=xl65 width=149 style='width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>topic 名称</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=3 height=103 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:77.0pt;border-top:none;width:122pt'>topicClusterList</td>\n  <td rowspan=3 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>查看 Topic 所在集群列表</td>\n  <td class=xl65 width=149 style='width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>topic 名称</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=6 height=518 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:380pt;border-top:none;width:122pt'>updateTopicPerm</td>\n  <td rowspan=6 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>更新 Topic 读写权限</td>\n  <td class=xl65 width=149 style='width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>topic 名称</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=132 style='height:99.0pt'>\n  <td height=132 class=xl65 width=149 style='height:99.0pt;width:112pt'>-b</td>\n  <td class=xl66 width=159 style='width:119pt'>Broker 地址，表示 topic 所在\n  Broker，只支持单台Broker，地址为ip:port</td>\n </tr>\n <tr height=76 style='height:57.0pt'>\n  <td height=76 class=xl65 width=149 style='height:57.0pt;width:112pt'>-p</td>\n  <td class=xl66 width=159 style='width:119pt'>指定新 topic 的读写权限( W=2|R=4|WR=6 )</td>\n </tr>\n <tr height=207 style='height:155.0pt'>\n  <td height=207 class=xl65 width=149 style='height:155.0pt;width:112pt'>-c</td>\n  <td class=xl66 width=159 style='width:119pt'>cluster 名称，表示 topic 所在集群（集群可通过\n  clusterList 查询），-b优先，如果没有-b，则对集群中所有Broker执行命令</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=5 height=199 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:149.0pt;border-top:none;width:122pt'>updateOrderConf</td>\n  <td rowspan=5 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>从NameServer上创建、删除、获取特定命名空间的kv配置，目前还未启用</td>\n  <td class=xl65 width=149 style='width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>topic，键</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=149 style='height:29.0pt;width:112pt'>-v</td>\n  <td class=xl66 width=159 style='width:119pt'>orderConf，值</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-m</td>\n  <td class=xl66 width=159 style='width:119pt'>method，可选get、put、delete</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=4 height=198 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:140pt;border-top:none;width:122pt'>allocateMQ</td>\n  <td rowspan=4 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>以平均负载算法计算消费者列表负载消息队列的负载结果</td>\n  <td class=xl65 width=149 style='width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>topic 名称</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=95 style='height:71.0pt'>\n  <td height=95 class=xl65 width=149 style='height:71.0pt;width:112pt'>-i</td>\n  <td class=xl66 width=159 style='width:119pt'>ipList，用逗号分隔，计算这些ip去负载Topic的消息队列</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=4 height=142 class=xl68 width=163 style='border-bottom:1.0pt solid black;\n  height:106.0pt;border-top:1.0pt;width:122pt'>statsAll</td>\n  <td rowspan=4 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>打印Topic订阅关系、TPS、积累量、24h读写总量等信息</td>\n  <td class=xl65 width=149 style='width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=149 style='height:29.0pt;width:112pt'>-a</td>\n  <td class=xl66 width=159 style='width:119pt'>是否只打印活跃topic</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>指定topic</td>\n </tr>\n</table>\n\n\n\n#### 2.2 集群相关\n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>名称</td>\n  <td class=xl64 width=175 style='width:131pt'>含义</td>\n  <td class=xl64 width=177 style='width:133pt'>命令选项</td>\n  <td class=xl64 width=185 style='width:139pt'>说明</td>\n </tr>\n <tr height=207 style='height:155.0pt'>\n  <td rowspan=4 height=326 class=xl67 width=177 style='border-bottom:1.0pt;\n  height:244.0pt;border-top:none;width:133pt'><span\n  style='mso-spacerun:yes'> </span>clusterList</td>\n  <td rowspan=4 class=xl70 width=175 style='border-bottom:1.0pt;\n  border-top:none;width:131pt'>查看集群信息，集群、BrokerName、BrokerId、TPS等信息</td>\n  <td class=xl65 width=177 style='width:133pt'>-m</td>\n  <td class=xl66 width=185 style='width:139pt'>打印更多信息 (增加打印出如下信息 #InTotalYest,\n  #OutTotalYest, #InTotalToday ,#OutTotalToday)</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=177 style='height:17.0pt;width:133pt'>-h</td>\n  <td class=xl66 width=185 style='width:139pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=177 style='height:43.0pt;width:133pt'>-n</td>\n  <td class=xl66 width=185 style='width:139pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=177 style='height:29.0pt;width:133pt'>-i</td>\n  <td class=xl66 width=185 style='width:139pt'>打印间隔，单位秒</td>\n </tr>\n <tr height=95 style='height:71.0pt'>\n  <td rowspan=8 height=391 class=xl67 width=177 style='border-bottom:1.0pt;\n  height:292.0pt;border-top:none;width:133pt'>clusterRT</td>\n  <td rowspan=8 class=xl70 width=175 style='border-bottom:1.0pt;\n  border-top:none;width:131pt'>发送消息检测集群各Broker RT。消息发往${BrokerName} Topic。</td>\n  <td class=xl65 width=177 style='width:133pt'>-a</td>\n  <td class=xl66 width=185 style='width:139pt'>amount，每次探测的总数，RT = 总时间 /\n  amount</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=177 style='height:29.0pt;width:133pt'>-s</td>\n  <td class=xl66 width=185 style='width:139pt'>消息大小，单位B</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=177 style='height:17.0pt;width:133pt'>-c</td>\n  <td class=xl66 width=185 style='width:139pt'>探测哪个集群</td>\n </tr>\n <tr height=76 style='height:57.0pt'>\n  <td height=76 class=xl65 width=177 style='height:57.0pt;width:133pt'>-p</td>\n  <td class=xl66 width=185 style='width:139pt'>是否打印格式化日志，以|分割，默认不打印</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=177 style='height:17.0pt;width:133pt'>-h</td>\n  <td class=xl66 width=185 style='width:139pt'>打印帮助</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=177 style='height:29.0pt;width:133pt'>-m</td>\n  <td class=xl66 width=185 style='width:139pt'>所属机房，打印使用</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=177 style='height:29.0pt;width:133pt'>-i</td>\n  <td class=xl66 width=185 style='width:139pt'>发送间隔，单位秒</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=177 style='height:43.0pt;width:133pt'>-n</td>\n  <td class=xl66 width=185 style='width:139pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n</table>\n\n\n#### 2.3 Broker相关\n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>名称</td>\n  <td class=xl64 width=175 style='width:131pt'>含义</td>\n  <td class=xl64 width=177 style='width:133pt'>命令选项</td>\n  <td class=xl64 width=185 style='width:139pt'>说明</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=6 height=206 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:154.0pt;border-top:none;width:143pt'>updateBrokerConfig</td>\n  <td rowspan=6 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>更新 Broker 配置文件，会修改Broker.conf</td>\n  <td class=xl67 width=87 style='width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Broker 地址，格式为ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>cluster 名称</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-k</td>\n  <td class=xl68 width=87 style='width:65pt'>key 值</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-v</td>\n  <td class=xl68 width=87 style='width:65pt'>value 值</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=3 height=137 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:103.0pt;border-top:none;width:143pt'>brokerStatus</td>\n  <td rowspan=3 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>查看 Broker 统计信息、运行状态（你想要的信息几乎都在里面）</td>\n  <td class=xl67 width=87 style='width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Broker 地址，地址为ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=6 height=256 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:192.0pt;border-top:none;width:143pt'>brokerConsumeStats</td>\n  <td rowspan=6 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>Broker中各个消费者的消费情况，按Message Queue维度返回Consume\n  Offset，Broker Offset，Diff，TImestamp等信息</td>\n  <td class=xl67 width=87 style='width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Broker 地址，地址为ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>请求超时时间</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-l</td>\n  <td class=xl68 width=87 style='width:65pt'>diff阈值，超过阈值才打印</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-o</td>\n  <td class=xl68 width=87 style='width:65pt'>是否为顺序topic，一般为false</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=2 height=114 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:86.0pt;border-top:none;width:143pt'>getBrokerConfig</td>\n  <td rowspan=2 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>获取Broker配置</td>\n  <td class=xl67 width=87 style='width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Broker 地址，地址为ip:port</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=3 height=137 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:103.0pt;border-top:none;width:143pt'>wipeWritePerm</td>\n  <td rowspan=3 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>从NameServer上清除 Broker写权限</td>\n  <td class=xl67 width=87 style='width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>BrokerName</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n   <td rowspan=3 height=137 class=xl69 width=191 style='border-bottom:1.0pt;\n   height:103.0pt;border-top:none;width:143pt'>addWritePerm</td>\n   <td rowspan=3 class=xl72 width=87 style='border-bottom:1.0pt\n   border-top:none;width:65pt'>从NameServer上添加 Broker写权限</td>\n   <td class=xl67 width=87 style='width:65pt'>-b</td>\n   <td class=xl68 width=87 style='width:65pt'>BrokerName</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=4 height=160 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:120.0pt;border-top:none;width:143pt'>cleanExpiredCQ</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>清理Broker上过期的Consume Queue，如果手动减少对列数可能产生过期队列</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Broker 地址，地址为ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>集群名称</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=4 height=160 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:120.0pt;border-top:none;width:143pt'>deleteExpiredCommitLog</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>清理Broker上过期的CommitLog文件，Broker最多会执行20次删除操作，每次最多删除10个文件</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Broker 地址，地址为ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>集群名称</td>\n </tr>\n <tr height=88 style='mso-height-source:userset;height:66.0pt'>\n  <td rowspan=4 height=191 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:143.0pt;border-top:none;width:143pt'>cleanUnusedTopic</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>清理Broker上不使用的Topic，从内存中释放Topic的Consume\n  Queue，如果手动删除Topic会产生不使用的Topic</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Broker 地址，地址为ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>集群名称</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=5 height=199 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:149.0pt;border-top:none;width:143pt'>sendMsgStatus</td>\n  <td rowspan=5 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>向Broker发消息，返回发送状态和RT</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>BrokerName，注意不同于Broker地址</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>消息大小，单位B</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>发送次数</td>\n </tr>\n</table>\n\n\n#### 2.4 消息相关\n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n<tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>名称</td>\n  <td class=xl64 width=175 style='width:131pt'>含义</td>\n  <td class=xl64 width=177 style='width:133pt'>命令选项</td>\n  <td class=xl64 width=185 style='width:139pt'>说明</td>\n </tr>\n <tr height=128 style='height:96.0pt'>\n  <td rowspan=3 height=208 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:156.0pt;border-top:none;width:65pt'>queryMsgById</td>\n  <td rowspan=3 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>根据offsetMsgId查询msg，如果使用开源控制台，应使用offsetMsgId，此命令还有其他参数，具体作用请阅读QueryMsgByIdSubCommand。</td>\n  <td class=xl67 width=87 style='width:65pt'>-i</td>\n  <td class=xl67 width=87 style='width:65pt'>msgId</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=4 height=126 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:94.0pt;border-top:none;width:65pt'>queryMsgByKey</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>根据消息 Key 查询消息</td>\n  <td class=xl67 width=87 style='width:65pt'>-k</td>\n  <td class=xl67 width=87 style='width:65pt'>msgKey</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>Topic 名称</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=225 style='height:169.0pt'>\n  <td rowspan=6 height=390 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:292.0pt;border-top:none;width:65pt'>queryMsgByOffset</td>\n  <td rowspan=6 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>根据 Offset 查询消息</td>\n  <td class=xl67 width=87 style='width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Broker 名称，（这里需要注意\n  填写的是 Broker 的名称，不是 Broker 的地址，Broker 名称可以在 clusterList 查到）</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-i</td>\n  <td class=xl68 width=87 style='width:65pt'>query 队列 id</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-o</td>\n  <td class=xl68 width=87 style='width:65pt'>offset 值</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>topic 名称</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=47>\n  <td rowspan=6 height=209 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:156.0pt;border-top:none;width:65pt'>queryMsgByUniqueKey</td>\n  <td rowspan=6 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>根据msgId查询，msgId不同于offsetMsgId，区别详见常见运维问题。-g，-d配合使用，查到消息后尝试让特定的消费者消费消息并返回消费结果</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-i</td>\n  <td class=xl67 width=87 style='width:65pt'>uniqe msg id</td>\n </tr>\n <tr height=36 style='height:27.0pt'>\n  <td height=36 class=xl67 width=87 style='height:27.0pt;width:65pt'>-g</td>\n  <td class=xl67 width=87 style='width:65pt'>consumerGroup</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-d</td>\n  <td class=xl67 width=87 style='width:65pt'>clientId</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>topic名称</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=5 height=149 class=xl69 width=87 style='border-bottom:1.0pt\n  height:111.0pt;border-top:none;width:65pt'>checkMsgSendRT</td>\n  <td rowspan=5 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>检测向topic发消息的RT，功能类似clusterRT</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>topic名称</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-a</td>\n  <td class=xl68 width=87 style='width:65pt'>探测次数</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>消息大小</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=8 height=218 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:162.0pt;border-top:none;width:65pt'>sendMessage</td>\n  <td rowspan=8 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>发送一条消息，可以根据配置发往特定Message Queue，或普通发送。</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>topic名称</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-p</td>\n  <td class=xl68 width=87 style='width:65pt'>body，消息体</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-k</td>\n  <td class=xl67 width=87 style='width:65pt'>keys</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl67 width=87 style='width:65pt'>tags</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-b</td>\n  <td class=xl67 width=87 style='width:65pt'>BrokerName</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-i</td>\n  <td class=xl67 width=87 style='width:65pt'>queueId</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=10 height=312 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:232.0pt;border-top:none;width:65pt'>consumeMessage</td>\n  <td rowspan=10 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>消费消息。可以根据offset、开始&amp;结束时间戳、消息队列消费消息，配置不同执行不同消费逻辑，详见ConsumeMessageCommand。</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>topic名称</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-b</td>\n  <td class=xl67 width=87 style='width:65pt'>BrokerName</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-o</td>\n  <td class=xl68 width=87 style='width:65pt'>从offset开始消费</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-i</td>\n  <td class=xl67 width=87 style='width:65pt'>queueId</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-g</td>\n  <td class=xl68 width=87 style='width:65pt'>消费者分组</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>开始时间戳，格式详见-h</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-d</td>\n  <td class=xl68 width=87 style='width:65pt'>结束时间戳</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>消费多少条消息</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=8 height=282 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:210.0pt;border-top:none;width:65pt'>printMsg</td>\n  <td rowspan=8 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>从Broker消费消息并打印，可选时间段</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>topic名称</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>字符集，例如UTF-8</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>subExpress，过滤表达式</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>开始时间戳，格式参见-h</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-e</td>\n  <td class=xl68 width=87 style='width:65pt'>结束时间戳</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-d</td>\n  <td class=xl68 width=87 style='width:65pt'>是否打印消息体</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=12 height=390 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:290.0pt;border-top:none;width:65pt'>printMsgByQueue</td>\n  <td rowspan=12 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>类似printMsg，但指定Message Queue</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>topic名称</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-i</td>\n  <td class=xl67 width=87 style='width:65pt'>queueId</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-a</td>\n  <td class=xl67 width=87 style='width:65pt'>BrokerName</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>字符集，例如UTF-8</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>subExpress，过滤表达式</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>开始时间戳，格式参见-h</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-e</td>\n  <td class=xl68 width=87 style='width:65pt'>结束时间戳</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-p</td>\n  <td class=xl68 width=87 style='width:65pt'>是否打印消息</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-d</td>\n  <td class=xl68 width=87 style='width:65pt'>是否打印消息体</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-f</td>\n  <td class=xl68 width=87 style='width:65pt'>是否统计tag数量并打印</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=7 height=410 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:307.0pt;border-top:none;width:65pt'>resetOffsetByTime</td>\n  <td rowspan=7 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>按时间戳重置offset，Broker和consumer都会重置</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-g</td>\n  <td class=xl68 width=87 style='width:65pt'>消费者分组</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>topic名称</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>重置为此时间戳对应的offset</td>\n </tr>\n <tr height=188 style='height:141.0pt'>\n  <td height=188 class=xl67 width=87 style='height:141.0pt;width:65pt'>-f</td>\n  <td class=xl68 width=87 style='width:65pt'>是否强制重置，如果false，只支持回溯offset，如果true，不管时间戳对应offset与consumeOffset关系</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>是否重置c++客户端offset</td>\n </tr>\n</table>\n\n\n#### 2.5 消费者、消费组相关\n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n<tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>名称</td>\n  <td class=xl64 width=175 style='width:131pt'>含义</td>\n  <td class=xl64 width=177 style='width:133pt'>命令选项</td>\n  <td class=xl64 width=185 style='width:139pt'>说明</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td rowspan=4 height=158 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:110pt;border-top:none;width:65pt'>consumerProgress</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>查看订阅组消费状态，可以查看具体的client IP的消息积累量</td>\n  <td class=xl67 width=87 style='width:65pt'>-g</td>\n  <td class=xl68 width=87 style='width:65pt'>消费者所属组名</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>是否打印client IP</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=105 style='mso-height-source:userset;height:79.0pt'>\n  <td rowspan=5 height=260 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:195.0pt;border-top:none;width:65pt'>consumerStatus</td>\n  <td rowspan=5 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>查看消费者状态，包括同一个分组中是否都是相同的订阅，分析Process\n  Queue是否堆积，返回消费者jstack结果，内容较多，使用者参见ConsumerStatusSubCommand</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=36 style='height:27.0pt'>\n  <td height=36 class=xl67 width=87 style='height:27.0pt;width:65pt'>-g</td>\n  <td class=xl67 width=87 style='width:65pt'>consumer group</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-i</td>\n  <td class=xl67 width=87 style='width:65pt'>clientId</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>是否执行jstack</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=13 height=761 class=xl69 width=87 style='border-bottom:1.0pt\n  height:569.0pt;border-top:none;width:65pt'>updateSubGroup</td>\n  <td rowspan=13 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>更新或创建订阅关系</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Broker地址</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>集群名称</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-g</td>\n  <td class=xl68 width=87 style='width:65pt'>消费者分组名称</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>分组是否允许消费</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-m</td>\n  <td class=xl68 width=87 style='width:65pt'>是否从最小offset开始消费</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-d</td>\n  <td class=xl68 width=87 style='width:65pt'>是否是广播模式</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-q</td>\n  <td class=xl68 width=87 style='width:65pt'>重试队列数量</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-r</td>\n  <td class=xl68 width=87 style='width:65pt'>最大重试次数</td>\n </tr>\n <tr height=207 style='height:155.0pt'>\n  <td height=207 class=xl67 width=87 style='height:155.0pt;width:65pt'>-i</td>\n  <td class=xl68 width=87 style='width:65pt'>当slaveReadEnable开启时有效，且还未达到从slave消费时建议从哪个BrokerId消费，可以配置备机id，主动从备机消费</td>\n </tr>\n <tr height=132 style='height:99.0pt'>\n  <td height=132 class=xl67 width=87 style='height:99.0pt;width:65pt'>-w</td>\n  <td class=xl68 width=87 style='width:65pt'>如果Broker建议从slave消费，配置决定从哪个slave消费，配置BrokerId，例如1</td>\n </tr>\n <tr height=76 style='height:57.0pt'>\n  <td height=76 class=xl67 width=87 style='height:57.0pt;width:65pt'>-a</td>\n  <td class=xl68 width=87 style='width:65pt'>当消费者数量变化时是否通知其他消费者负载均衡</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=5 height=165 class=xl69 width=87 style='border-bottom:1.0pt\n  height:123.0pt;border-top:none;width:65pt'>deleteSubGroup</td>\n  <td rowspan=5 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>从Broker删除订阅关系</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Broker地址</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>集群名称</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-g</td>\n  <td class=xl68 width=87 style='width:65pt'>消费者分组名称</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=6 height=172 class=xl69 width=87 style='border-bottom:1.0pt\n  height:120pt;border-top:none;width:65pt'>cloneGroupOffset</td>\n  <td rowspan=6 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>在目标群组中使用源群组的offset</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>源消费者组</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-d</td>\n  <td class=xl68 width=87 style='width:65pt'>目标消费者组</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>topic名称</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-o</td>\n  <td class=xl68 width=87 style='width:65pt'>暂未使用</td>\n </tr>\n</table>\n\n\n\n\n#### 2.6 连接相关\n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n<tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>名称</td>\n  <td class=xl64 width=175 style='width:131pt'>含义</td>\n  <td class=xl64 width=177 style='width:133pt'>命令选项</td>\n  <td class=xl64 width=185 style='width:139pt'>说明</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td rowspan=3 height=119 class=xl69 width=87 style='border-bottom:1.0pt\n  height:89.0pt;border-top:none;width:65pt'>consumerConnection</td>\n  <td rowspan=3 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>查询 Consumer 的网络连接</td>\n  <td class=xl67 width=87 style='width:65pt'>-g</td>\n  <td class=xl68 width=87 style='width:65pt'>消费者所属组名</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td rowspan=4 height=142 class=xl69 width=87 style='border-bottom:1.0pt\n  height:106.0pt;border-top:none;width:65pt'>producerConnection</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>查询 Producer 的网络连接</td>\n  <td class=xl67 width=87 style='width:65pt'>-g</td>\n  <td class=xl68 width=87 style='width:65pt'>生产者所属组名</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>主题名称</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n</table>\n\n\n\n\n#### 2.7 NameServer相关\n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n<tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>名称</td>\n  <td class=xl64 width=175 style='width:131pt'>含义</td>\n  <td class=xl64 width=177 style='width:133pt'>命令选项</td>\n  <td class=xl64 width=185 style='width:139pt'>说明</td>\n </tr>\n <tr height=21 style='height:16.0pt'>\n  <td rowspan=5 height=143 class=xl69 width=87 style='border-bottom:1.0pt\n  height:100pt;border-top:none;width:65pt'>updateKvConfig</td>\n  <td rowspan=5 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>更新NameServer的kv配置，目前还未使用</td>\n  <td class=xl75 width=87 style='width:65pt'>-s</td>\n  <td class=xl76 width=87 style='width:65pt'>命名空间</td>\n </tr>\n <tr height=21 style='height:16.0pt'>\n  <td height=21 class=xl75 width=87 style='height:16.0pt;width:65pt'>-k</td>\n  <td class=xl75 width=87 style='width:65pt'>key</td>\n </tr>\n <tr height=21 style='height:16.0pt'>\n  <td height=21 class=xl75 width=87 style='height:16.0pt;width:65pt'>-v</td>\n  <td class=xl75 width=87 style='width:65pt'>value</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=4 height=126 class=xl69 width=87 style='border-bottom:1.0pt\n  height:94.0pt;border-top:none;width:65pt'>deleteKvConfig</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>删除NameServer的kv配置</td>\n  <td class=xl67 width=87 style='width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>命名空间</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-k</td>\n  <td class=xl67 width=87 style='width:65pt'>key</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=2 height=80 class=xl69 width=87 style='border-bottom:1.0pt\n  height:60.0pt;border-top:none;width:65pt'>getNamesrvConfig</td>\n  <td rowspan=2 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>获取NameServer配置</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=4 height=126 class=xl69 width=87 style='border-bottom:1.0pt\n  height:94.0pt;border-top:none;width:65pt'>updateNamesrvConfig</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>修改NameServer配置</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-k</td>\n  <td class=xl67 width=87 style='width:65pt'>key</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-v</td>\n  <td class=xl67 width=87 style='width:65pt'>value</td>\n </tr>\n</table>\n\n\n\n\n#### 2.8 其他\n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n<tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>名称</td>\n  <td class=xl64 width=175 style='width:131pt'>含义</td>\n  <td class=xl64 width=177 style='width:133pt'>命令选项</td>\n  <td class=xl64 width=185 style='width:139pt'>说明</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=2 height=80 class=xl69 width=87 style='border-bottom:1.0pt\n  height:60.0pt;border-top:none;width:65pt'>startMonitoring</td>\n  <td rowspan=2 class=xl71 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>开启监控进程，监控消息误删、重试队列消息数等</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>打印帮助</td>\n </tr>\n</table>\n\n\n### 3   运维常见问题\n\n#### 3.1 RocketMQ的mqadmin命令报错问题\n\n>  问题描述：有时候在部署完RocketMQ集群后，尝试执行“mqadmin”一些运维命令，会出现下面的异常信息：\n>\n> ```java\n> org.apache.rocketmq.remoting.exception.RemotingConnectException: connect to <null> failed\n> ```\n\n解决方法：可以在部署RocketMQ集群的虚拟机上执行`export NAMESRV_ADDR=ip:9876`（ip指的是集群中部署NameServer组件的机器ip地址）命令之后再使用“mqadmin”的相关命令进行查询，即可得到结果。\n\n#### 3.2 RocketMQ生产端和消费端版本不一致导致不能正常消费的问题\n\n> 问题描述：同一个生产端发出消息，A消费端可消费，B消费端却无法消费，rocketMQ Console中出现：\n>\n> ```java\n> Not found the consumer group consume stats, because return offset table is empty, maybe the consumer not consume any message的异常消息。\n> ```\n\n  解决方案：RocketMQ 的jar包：rocketmq-client等包应该保持生产端，消费端使用相同的version。\n\n#### 3.3  新增一个topic的消费组时，无法消费历史消息的问题\n\n> 问题描述：当同一个topic的新增消费组启动时，消费的消息是当前的offset的消息，并未获取历史消息。    \n\n解决方案：rocketmq默认策略是从消息队列尾部，即跳过历史消息。如果想消费历史消息，则需要设置：`org.apache.rocketmq.client.consumer.DefaultMQPushConsumer#setConsumeFromWhere`。常用的有以下三种配置：\n\n- 默认配置，一个新的订阅组第一次启动从队列的最后位置开始消费，后续再启动接着上次消费的进度开始消费，即跳过历史消息；\n\n```java\nconsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);\n```\n\n- 一个新的订阅组第一次启动从队列的最前位置开始消费，后续再启动接着上次消费的进度开始消费，即消费Broker未过期的历史消息；\n\n```java\nconsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n```\n\n- 一个新的订阅组第一次启动从指定时间点开始消费，后续再启动接着上次消费的进度开始消费，和consumer.setConsumeTimestamp()配合使用，默认是半个小时以前；\n\n```java\nconsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_TIMESTAMP);\n```\n\n#### 3.4 如何开启从Slave读数据功能\n\n在某些情况下，Consumer需要将消费位点重置到1-2天前，这时在内存有限的Master Broker上，CommitLog会承载比较重的IO压力，影响到该Broker的其它消息的读与写。可以开启`slaveReadEnable=true`，当Master Broker发现Consumer的消费位点与CommitLog的最新值的差值的容量超过该机器内存的百分比（`accessMessageInMemoryMaxRatio=40%`），会推荐Consumer从Slave Broker中去读取数据，降低Master Broker的IO。\n\n#### 3.5 性能调优问题\n\n异步刷盘建议使用自旋锁，同步刷盘建议使用重入锁，调整Broker配置项`useReentrantLockWhenPutMessage`，默认为false；异步刷盘建议开启`TransientStorePoolEnable`；建议关闭transferMsgByHeap，提高拉消息效率；同步刷盘建议适当增大`sendMessageThreadPoolNums`，具体配置需要经过压测。\n\n#### 3.6 在RocketMQ中msgId和offsetMsgId的含义与区别\n\n使用RocketMQ完成生产者客户端消息发送后，通常会看到如下日志打印信息：\n\n```java\nSendResult [sendStatus=SEND_OK, msgId=0A42333A0DC818B4AAC246C290FD0000, offsetMsgId=0A42333A00002A9F000000000134F1F5, messageQueue=MessageQueue [topic=topicTest1, BrokerName=mac.local, queueId=3], queueOffset=4]\n```\n\n- msgId，对于客户端来说msgId是由客户端producer实例端生成的，具体来说，调用方法`MessageClientIDSetter.createUniqIDBuffer()`生成唯一的Id；\n- offsetMsgId，offsetMsgId是由Broker服务端在写入消息时生成的（采用”IP地址+Port端口”与“CommitLog的物理偏移量地址”做了一个字符串拼接），其中offsetMsgId就是在RocketMQ控制台直接输入查询的那个messageId。\n"
  },
  {
    "path": "docs/cn/proxy/deploy_guide.md",
    "content": "# RocketMQ Proxy部署指南\n\n## 概述\n\nRocketMQ Proxy 支持两种代理模式: `Local` and `Cluster`。\n\n## 配置\n\n该配置适用于 `Cluster` 和 `Local` 两种模式, 默认路径为 `distribution/conf/rmq-proxy.json`。\n\n## `Cluster` 模式\n\n* 设置 `nameSrvAddr`\n* 设置 `proxyMode` 为 `cluster` (不区分大小写)\n\n运行以下命令:\n\n```shell\nnohup sh mqproxy &\n```\n\n该命令仅会启动 `Proxy` 组件本身。它假设已经在指定的 `nameSrvAddr` 地址上运行着 `Namesrv` 节点，同时也有 broker 节点通过 `nameSrvAddr` 注册自己并运行。\n\n## `Local` 模式\n\n* 设置 `nameSrvAddr`\n* 设置 `proxyMode` 为 `local` (不区分大小写)\n\n运行以下命令:\n\n```shell\nnohup sh mqproxy &\n```\n\n上面的命令将启动`Proxy`，并在同一进程中运行`Broker`。它假设`Namesrv`节点正在按照`nameSrvAddr`指定的地址运行。\n"
  },
  {
    "path": "docs/cn/rpc_request.md",
    "content": "# “Request-Reply”特性\n---\n\n## 1 使用场景\n随着服务规模的扩大，单机服务无法满足性能和容量的要求，此时需要将服务拆分为更小粒度的服务或者部署多个服务实例构成集群来提供服务。在分布式场景下，RPC是最常用的联机调用的方式。\n\n在构建分布式应用时，有些领域，例如金融服务领域，常常使用消息队列来构建服务总线，实现联机调用的目的。消息队列的主要场景是解耦、削峰填谷，在联机调用的场景下，需要将服务的调用抽象成基于消息的交互，并增强同步调用的这种交互逻辑。为了更好地支持消息队列在联机调用场景下的应用，rocketmq-4.6.0推出了“Request-Reply”特性来支持RPC调用。\n\n## 2 设计思路\n在rocketmq中，整个同步调用主要包括两个过程：\n\n（1）请求方生成消息，发送给响应方，并等待响应方回包；\n\n（2）响应方收到请求消息后，消费这条消息，并发出一条响应消息给请求方。\n\n整个过程实质上是两个消息收发过程的组合。所以这里最关键的问题是如何将异步的消息收发过程构建成一个同步的过程。其中主要有两个问题需要解决：\n\n### 2.1 请求方如何同步等待回包\n\n这个问题的解决方案中，一个关键的数据结构是RequestResponseFuture。\n\n```\npublic class RequestResponseFuture {\n    private final String correlationId;\n    private final RequestCallback requestCallback;\n    private final long beginTimestamp = System.currentTimeMillis();\n    private final Message requestMsg = null;\n    private long timeoutMillis;\n    private CountDownLatch countDownLatch = new CountDownLatch(1);\n    private volatile Message responseMsg = null;\n    private volatile boolean sendRequestOk = true;\n    private volatile Throwable cause = null;\n}\n```\nRequestResponseFuture中，利用correlationId来标识一个请求。如下图所示，Producer发送request时创建一个RequestResponseFuture，以correlationId为key，RequestResponseFuture为value存入map，同时请求中带上RequestResponseFuture中的correlationId，收到回包后根据correlationId拿到对应的RequestResponseFuture，并设置回包内容。\n![](image/producer_send_request.png)\n\n### 2.2 consumer消费消息后，如何准确回包\n\n（1）producer在发送消息的时候，会给每条消息生成唯一的标识符，同时还带上了producer的clientId。当consumer收到并消费消息后，从消息中取出消息的标识符correlationId和producer的标识符clientId，放入响应消息，用来确定此响应消息是哪条请求消息的回包，以及此响应消息应该发给哪个producer。同时响应消息中设置了消息的类型以及响应消息的topic，然后consumer将消息发给broker，如下图所示。\n![](image/consumer_reply.png)\n\n（2）broker收到响应消息后，需要将消息发回给指定的producer。Broker如何知道发回给哪个producer？因为消息中包含了producer的标识符clientId，在ProducerManager中，维护了标识符和channel信息的对应关系，通过这个对应关系，就能把回包发给对应的producer。\n\n响应消息发送和一般的消息发送流程区别在于，响应消息不需要producer拉取，而是由broker直接推给producer。同时选择broker的策略也有变化：请求消息从哪个broker发过来，响应消息也发到对应的broker上。\n\nProducer收到响应消息后，根据消息中的唯一标识符，从RequestResponseFuture的map中找到对应的RequestResponseFuture结构，设置响应消息，同时计数器减一，解除等待状态，使请求方收到响应消息。\n\n## 3 使用方法\n\n同步调用的示例在example文件夹的rpc目录下。\n\n### 3.1 Producer\n```\nMessage msg = new Message(topic,\n                \"\",\n                \"Hello world\".getBytes(RemotingHelper.DEFAULT_CHARSET));\n\n            long begin = System.currentTimeMillis();\n            Message retMsg = producer.request(msg, ttl);\n            long cost = System.currentTimeMillis() - begin;\n            System.out.printf(\"request to <%s> cost: %d replyMessage: %s %n\", topic, cost, retMsg);\n```\n调用接口替换为request即可。\n\n### 3.2 Consumer\n需要启动一个producer，同时在覆写consumeMessage方法的时候，自定义响应消息并发送。\n\n```\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {\n                System.out.printf(\"%s Receive New Messages: %s %n\", Thread.currentThread().getName(), msgs);\n                for (MessageExt msg : msgs) {\n                    try {\n                        System.out.printf(\"handle message: %s\", msg.toString());\n                        String replyTo = MessageUtil.getReplyToClient(msg);\n                        byte[] replyContent = \"reply message contents.\".getBytes();\n                        // create reply message with given util, do not create reply message by yourself\n                        Message replyMessage = MessageUtil.createReplyMessage(msg, replyContent);\n\n                        // send reply message with producer\n                        SendResult replyResult = replyProducer.send(replyMessage, 3000);\n                        System.out.printf(\"reply to %s , %s %n\", replyTo, replyResult.toString());\n                    } catch (MQClientException | RemotingException | MQBrokerException | InterruptedException e) {\n                        e.printStackTrace();\n                    }\n                }\n                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n            }\n```\n\n## 4 接口参数\n\n4.1 public Message request(Message msg,long timeout)\n\nmsg：待发送的消息\n\ntimeout：同步调用超时时间\n\n4.2 public void request(Message msg, final RequestCallback requestCallback, long timeout)\n\nmsg：待发送的消息\n\nrequestCallback：回调函数\n\ntimeout：同步调用超时时间\n\n4.3 public Message request(final Message msg, final MessageQueueSelector selector, final Object arg,final long timeout)\n\nmsg：待发送的消息\n\nselector：消息队列选择器\n\narg：消息队列选择器需要的参数\n\ntimeout：同步调用超时时间\n\n4.4 public void request(final Message msg, final MessageQueueSelector selector, final Object arg,final RequestCallback requestCallback, final long timeout)\n\nmsg：待发送的消息\n\nselector：消息队列选择器\n\narg：消息队列选择器需要的参数\n\nrequestCallback：回调函数\n\ntimeout：同步调用超时时间\n\n4.5\tpublic Message request(final Message msg, final MessageQueue mq, final long timeout)\n\nmsg：待发送的消息\n\nmq：目标消息队列\n\ntimeout：同步调用超时时间\n\n4.6\tpublic void request(final Message msg, final MessageQueue mq, final RequestCallback requestCallback, long timeout)\n\nmsg：待发送的消息\n\nmq：目标消息队列\n\nrequestCallback：回调函数\n\ntimeout：同步调用超时时间\n"
  },
  {
    "path": "docs/cn/statictopic/RocketMQ_Static_Topic_Logic_Queue_设计.md",
    "content": "### Version 记录\n| 时间 | 主要内容 | 作者 |\n| --- | --- | --- |\n| 2021-11-01 | 初稿，包括背景、目标、SOT定义与持久化、SOT生命周期、SOT的使用、API逻辑修改、问题与风险 | dongeforever |\n| 2021-11-15 | 修改 LogicQueue 的定义，不要引入新字段，完全复用旧的MessageQueue；RemappingStaticTopic时，不要迁移『位点』『幂等数据』等，而是采用Double-Check-Read 的机制| dongforever |\n| 2021-12-01 | 更新问题与风险，增加关于一致性、OutOfRange、拉取中断的详细说明| dongforever |\n| 2021-12-03 | 增加代码走读的说明| dongforever |\n| 2021-12-10 | 引入Scope概念，保留『多集群动态零耦合』的集群设计模型 | dongforever |\n| 2021-12-23 | 梳理待完成事项；讨论Admin接口的适配方式 | dongforever |\n| 2021-01-05 | Offset存储改成『转换制』，以更好适配原有逻辑 | dongforever |\n\n\n\n\n中文文档在描述特定专业术语时，仍然使用英文。\n\n### 需求背景\nStaticTopic/LogicQueue 本质上是解决『固定队列数量』的需求。\n这个需求是不是必需的呢，如果是做应用集成，则可能不是必需的，但如果是做数据集成，则是必需的。\n\n固定队列数量，首先可以解决『顺序性』的问题。\n在应用集成场景下，应用是无需感知到队列的，只要MQ能保证按顺序投递给应用即可，MQ底层队列数量如何变化，对应用来说是不关心。比如，MQ之前的那套『禁读禁写』就是可以玩转的。\n\n但在数据集成场景中，队列或者叫『分片』，是要暴露给客户端的，客户端所有的数据计算场景，都是基于『分片』来进行的，如果『分片』里的数据，发生了错乱，则计算结果都是错误的。比如，计算WordCount，源数据经过预处理之后，按key写入清洗后的Topic，然后计算侧根据清洗的结果，按照分片来并行计算。如果分片发生变化，则整个清洗逻辑，需要重新处理。\n\n有人可能会反驳，说计算组件清洗后，可以以批的方式写入其它存储组件。这当然是可以的，但如果是这样，MQ的价值就纯粹是一个『源头』价值，而不是『通道』价值。\n\nMQ要想成为一个『数据通道』，则必需要具备可以让计算组件『回写』数据的能力，具备存储『Clean Data』的能力，这样才让MQ有可能在数据集成领域站稳脚跟。\n\n如果是 RocketMQ Streams 这种轻量化的组件，则『回写』会更频繁，更重要。\n\n除此之外，『固定队列数据』对于，RocketMQ 自身后续的发展，也是至关重要的：\n\n- compact topic，如果不能做到严格按key hash，则这个KV系统是有问题的\n- 事务或者其它Coordinator的实现，采用『固定队列数量』，可以选取到准确的Broker来充当协调器\n- Metadata 的存储，按key hash，那么就可以在Broker上，存储千万级的 Topic\n\n『固定队列数量』对于RocketMQ挺进『数据集成』这个领域，有着不可或缺的作用。\nLogicQueue的思路就是为了解决这一问题。\n\n### 设计目标\n#### 总体目标\n提供『Static Topic』的特性。\n引入以下核心概念：\n- physical message queue, physical queue for short, a shard bound to a specified broker.\n- logic message queue, logic queue for short, a shard vertically composed by physical queues.\n- dynamic sharded topic, dynamic topic for short, which has queues increasing with the broker numbers.\n- static sharded topic, static topic for short, which has fixed queues, implemented with logic queues.\n\n『Static Topic』拥有固定的分片数量，每个分片称之为『Logic Queue』。\n每个『Logic Queue』由多个『Physical Queue』进行纵向分段映射组成。\n\n引入以下非核心概念，对用户无感知，但对于讨论问题非常重要：\n- Leader Queue， 某个『Logic Queue』最新映射的『Physical Queue』，也即可写的那个Queue\n- Second Leader Queue，某个『Logic Queue』次新映射的『Physical Queue』，也即最新一次切换之前的『Leader Queue』\n\n#### Scope 目标\n单集群固定 和 全网固定，参考 [The_Scope_Of_Static_Topic](The_Scope_Of_Static_Topic.md)。\n\n\n#### LogicQueue 目标\n在客户端，LogicQueue 与 Physical Queue 使用体感上没有任何区别，使用一样的概念和对象，遵循一样的语义。\n在服务端，针对 LogicQueue 去适配相关的API。\n\n#### 队列语义\nRocketMQ Physical Queue 含有以下语义：\n\n- 队列内的Offset，单调递增且连续\n- 属于同一个 Broker 上的队列，编号单调递增且连续\n\nLogicQueue 需要保障的语义：\n\n- 队列内的offset，单调递增\n\nLogicQueue 可以不保障的语义：\n\n- 队列内的 offset 连续\n- 属于同一个 Broker 上的队列，编号单调递增且连续\n\noffset连续，是一个应该尽量保证的语义，可以允许有少量空洞，但不应该出现大面积不连续的位点。\noffset不连续最直接的问题就是：\n\n- 计算Lag会比较麻烦\n- 不方便客户端进行各种优化计算（比如切批等）\n\n但只要空洞不是大量频繁出现的，那么也是问题不大的。\n\n单机队列编号连续，除了在注册元数据时，可以简约部分字节外，没有其它实际用处，可以不保证。\n当前客户端使用到『单机队列编号连续』这个特点的场景主要有：\n\n- 客户端在获取到TopicRouteData后，转化成MessageQueue时，利用编号进行遍历\n\n\n#### LogicQueue 定义\n当前 MessageQueue 的定义如下\n```\nprivate String topic;\nprivate String brokerName;\nprivate int queueId;\n```\n\n\nLogicQueue需要对客户直接暴露，为了保证使用习惯一致，采用同样的定义，其中 queueId相当于全局Id，而brokerName 固定如下：\n```\nMixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME = \"__logical_queue_broker__\";\n```\n此时，brokerName没有实际含义，但可以用来识别是否是LogicQueue。\n\n采用此种定义，对于客户端内部的实现习惯改变如下：\n\n- **无法直接根据 brokerName 找到addr，而是需要根据 MessageQueue 找到 addr**\n\n具体改法是MQClientInstance中维护一个映射关系\n```\nprivate final ConcurrentMap<String/* Topic */, ConcurrentMap<MessageQueue, String/*brokerName*/>> topicEndPointsTable = new ConcurrentHashMap<>();\n```\n\n\n基本目标与定义清楚了，接下来的设计，从 Source of Truth 开始。\n\n### SOT 定义和持久化\nLogicQueue 的 Source of Truth 就是 LogicQueue 到 Physical Queue 的映射关系。\n只要这个映射关系不丢失，则整个系统的状态都是可以恢复的。\n反之，整个系统可能陷入混乱。\n\n这个SOT，命名为 TopicQueueMapping。\n\n#### Mapping Schema\n```\n{\n\"version\":\"1\",\n\"bname\": \"broker02\" //标记这份数据的原始存储位置，如果发送误拷贝，可以利用这个字段来进行标识\n\"epoch\": 0, //标记修改版本，用来做一致性校验\n\"totalQueues\":\"50\",  //当前Topic 总共有多少 LogicQueues\n\"hostedQueues\": {    //当前Broker 所拥有的 LogicQueues\n\"3\" : [\n       {\n        \"queue\":\"0\",\n        \"bname\":\"broker01\"\n        \"gen\":\"0\",  //标记切换代次\n        \"logicOffset\":\"0\", //logicOffset的起始位置\n        \"startOffset\":\"0\",   // 物理offset的起始位置\n        \"endOffset\":\"1000\" // 可选，物理offset的最大位置，可以根据上下文算出来\n        \"timeOfStart\":\"1561018349243\" //可选，用来优化时间搜索\n        \"timeOfEnd\":\"1561018349243\"   //可选，用来优化时间搜索\n        \"updateTime\":\"1561018349243\" //可选，记录更新的时间\n        },  \n        {\n        \"queue\":\"0\",\n        \"bname\":\"broker02\",\n        \"gen\":\"1\",  //可选，标记切换代次\n        \"logicOffset\":\"1000\", //logicOffset的起始位置\n        \"startOffset\":\"0\",   // 物理offset的起始位置\n        \"endOffset\":\"-1\" // 可选，物理offset的最大位置，可以根据上下文算出来，最新的一个应该是活跃的\n        \"timeOfStart\":\"1561018349243\" //可选，用来优化时间搜索\n        \"timeOfEnd\":\"1561018349243\"   //可选，用来优化时间搜索\n        \"updateTime\":\"1561018349243\" //可选，记录更新的时间\n        }\n      ]\n  }\n}\n```\n上述示例的含义是：\n* broker02 拥有 LogicQueue 3\n* LogicQueue 3 由 2 个 Physical Queue 组成\n* 位点范围『0-1000』映射到 Physical Queue 『broker01-0』上面\n* 位点范围『1000-』映射到 Physical Queue 『broker02-0』上面\n\n『拥有』的定义是指，Leader Queue 在当前Broker。注意，在实现时，也会把Second Leader Queue存储下来作为备份。\n\n注意以下要点：\n\n- 这个数据量会很大，后续需要考虑进行压缩优化（大部分字段可以压缩）\n- 如果将来利用 LogicQueue 去做 Serverless 弹缩，则这个数据会加速膨胀，对这个数据的利用要谨慎\n- 要写上 bname，以具备自我识别能力\n\n#### Leader Completeness\nRocketMQ 没有中心化的元数据存储，那就遵循『Leader Completeness』原则。\n对于每个逻辑队列，把所有映射关系存储在『最新队列所在的Broker上面』，最新队列，其实也是可写队列。\nLeader Completeness，避免了数据的切割，对于后续其它操作有极大的便利。\n\n#### Global Epoch Check\n对于每个Static Topic，在每个Broker都应该拥有一份『TopicQueueMapping』，每份都带有Epoch。\n在创建和更新时，要对已有数据进行完备性校验，如果发现不完备，则说明上次操作失败，或者部分Broker数据丢失，应该先修复再操作。\n\n注意：\n即使当前Broker不拥有任何 LogicQueue 或者 PhysicalQueue，也应该存储一份，以做校验。\n假设某个Static Topic只拥有1个Logic Queue，而对应的Broker整好宕机，则此时可以根据其它Broker的信息判断出该Topic不完备。\n\n\n#### File Isolation\n由于 RocketMQ 很多的运维习惯，都是直接拷贝 Topics.json 到别的机器进行部署的。\n而  TopicQueueMapping 是 Broker 相关的，如果把 TopicQueueMapping 从一个Broker拷贝到另一个Broker，则会造成SOT冲突。\n\n在设计上，TopicQueueMapping 采取独立文件，避免冲突。\n在格式上，queue 里面要写上 bname，以具备自我识别能力，这样即使误拷贝到另一台机器，可以识别并报错，进行忽略即可。\n\n\n### SOT 生命周期\n#### 创建和更新\n映射关系的创建，第一期应该只由 MQAdmin 来进行操作。\n后续，可以考虑引入自动化组件。\n这里的要点是：\n\n- TopicConfig 和 TopicQueueMapping 分开存储，但写入时，需要先写 TopicQueueMapping 再写 TopicConfig（SOT先写）\n- 【加强校验】需要在 TopicConfig 里面加上一个字段来标识『LogicQueue』的 Topic\n- 【加强校验】允许单独更新 TopicConfig，但要带上 TotalQueues 这些基础数据\n- 允许更新单 LogicQueue\n\n\n更多细节在API逻辑修改里面\n#### 存储\n按照 『Leader Completeness』原则进行存储。\n#### 切换\n如果为了保证严格顺序，则应该采取『禁旧再切新』的原则：\n- 从旧 Leader 所在 Broker 获取信息，进行计算\n- 写入旧 Leader，也即禁写旧 Leader\n- 写入新 Leader\n\n如果为了保证最高可用性，则应该采取『切新禁旧再切新』：\n- 从旧 Leader 所在 Broker 获取信息，进行计算\n- 写入新 Leader，保证新 Leader 可写，此时 logicOffset 未定\n- 写入旧 Leader，禁写旧 Leader\n- 更新新 Leader，确定 logicOffset\n\n切换失败处理：\n- 不管哪种方式，数据存储至少成功了1份，后续可以手工恢复\n\n\n#### 清除\n有两部分信息需要清除\n\n- 旧 Broker 上超过2代的映射关系进行清除\n- 对于单个LogicQueue，清除已经过期的 Broker Queue 映射项目\n\n\n### SOT 的使用\n#### Broker 注册数据\nSOT存储在Broker上，所以使用从 Broker开始。\n\n在 RegisterBrokerBody 中，需要带上两个信息：\n\n- 对于每个Topic，带上本机队列编号和逻辑队列编号的映射关系，也即queueMap\n- 对于每个Topic，需要带上 totalQueueNum 这个信息\n\n\n\n异常情况需要考虑，假如本 Broker 不拥有任何 LogicQueue 呢？依然需要带上 totalQueueNum 这个信息。\n注意，不需要带上所有的映射关系，否则Nameserver很快会被打爆。\n\n\n#### Nameserver 组装数据\n原先的 QueueData  增加2个字段：\n\n- totalQueues，标识总逻辑队列数\n- queueMap，也即本机队列编号和逻辑队列编号的映射关系\n\n\n如果 QueueData 里面 totalQueues 的值 > 0 则认为是逻辑队列，在客户端解析时要进行判断。\n\n\n遗留问题：\n是否需要尊重 readQueueNums 和 writeQueueNums ？\n在LogicQueue这里，这个场景是没有意义的，但依然保持尊重。\n\n#### Client 解析数据\n改动两个方法即可：\n\n- topicRouteData2TopicPublishInfo\n- topicRouteData2TopicSubscribeInfo\n\n\n注意，逻辑队列要求队列数是固定，如果发现，解析完之后，存在部分队列空洞，要用虚拟Broker值进行补全。\nProducer 侧如果要对无 key 场景进行优化，可以通过虚拟Broker值来判断，当前队列是不可用的。\n对于key场景，应该让客户端报错。\n\n### API 逻辑修改\n#### Tools\nLogicQueue是为了解决『Static Sharding』的问题。对于客户来说，『LogicQueue』是手段，『Static』才是目的。本着『用户知晓目的，开发者才需要关心手段』的原则，对用户应该只暴露『Static』的概念。所有QueueMapping的生命周期维护，应该都对用户透明。\n\n#### UpdateStaticTopic\n新增UpdateStaticTopic命令，对应RequestCode.UPDATE_AND_CREATE_STATIC_TOPIC=513，主要参数是：\n\n- -t，topic 名字\n- -qn，总队列数量\n- -c cluster列表 或者 -b broker列表\n\n\n\nUpdateStaticTopic 命令会自动计算预期的分布情况，包括但不限于执行以下逻辑：\n\n- 检测Topic的命名冲突\n- 检测旧数据的一致性\n- 创建必要的物理队列\n- 创建必要的映射关系\n\n\n#### RemappingStaticTopic\n迁移动作不引入新命令，计算好之后，执行UPDATE_AND_CREATE_STATIC_TOPIC即可.\n主要参数：\n\n- -t， topic 名字\n- -c cluster列表 或者 -b broker列表\n\n基本操作流程：\n\n- 1 获取旧Leader，计算映射关系\n- 2 迁移『位点』『幂等数据』等\n- 3 映射关系写入新/旧 Leader\n\n其中第二步，数据量可能会很大，导致迁移周期非常长，且不是并发安全的。\n但这些数据，都是覆盖型的，因此可以改造成不迁移的方式，而是在API层做兼容，也即『Double-Read-Check』机制：\n\n- 读取数据时，先从 Leader 读，如果Leader没有，则从Sub-Leader读取。\n- 提交数据，直接在 Leader 层面操作即可，覆盖旧数据。\n\n\n将来实现的幂等逻辑，也是类似。\n\n#### UpdateTopic\n服务端判断『StaticTopic』，禁止该命令进行修改。\n\n#### DeleteTopic\n复用现有逻辑，对于 StaticTopic，执行必要的清除动作。\n\n#### TopicStatus\n复用现有逻辑，同时展现『Logical Queue』和『Physical Queue』。\n\n#### Broker\n#### pullMessage\n分段映射，执行远程读，在返回消息时，不进行offset转换，而是返回 OffsetDelta 变量，由客户端进行转换。\n这里的方式，类似于Batch。\n\n#### getMinOffset\n寻找映射关系，读最早的队列的MinOffset\n\n#### getMaxOffset\n本机读，转换成logicOffset即可。\n\n\n#### getOffsetByTime\n需要分段查找。\n如果要优化查找速度，应该在映射关系里面，插入时间戳。\n\n\n#### consumerOffsets 系列\nOffset的存储，进行转换，存储在对应PhysicalQueue 所在的 Broker上面。  \n读取时，采取『Double-Read-Check』机制，并进行转换。  \n这样可以最大程度与 PhysicalQueue 的相关逻辑进行适配，比如 ConsumerProgress 可以看到『最近拉取时间』。 \n\n#### Client\n\n- MQClientInstance.topicRouteData2TopicXXXInfo，修改解析 TopicRouteData的逻辑\n- Consumer解压消息时，需要加上OffsetDelta逻辑\n\n#### SDK 兼容性分析\n如果要使用StaticTopic，则需要升级Client、Broker、Nameserver。\n\n### 问题与风险\n#### 数据一致性问题\nRocketMQ 没有引入中心化的存储组件，那么如何保证 SOT 的全局一致性呢？\n主要利用两个信息\n* TopicQueueMapping 带上的 epoch\n* TopicQueueMapping 带上 totalQueues  \n\n在更新或者切换时，获取所有Broker上的 TopicQueueMapping，校验 epoch 和 totalQueues，并且根据 TopicQueueMapping 可以完整地构建出对应的Logic Queue，则说明数据是完整一致的。\n\n如果发现数据不一致，可能是以下因素引入的：\n* 集群中有Broker宕机\n* 上次更新没有完全成功\n\n应该要先修复数据，再执行 更新或切换 操作\n\n#### No Target Brokers\nTarget Brokers 是指拥有 LogicQueue 的 Broker。  \n考察1个场景，如果某个Topic 只有1个 LogicQueue，而拥有这个 LogicQueue 的 Broker 正好宕机了。此时去更新 Topic，会不会误认为该 Topic 不存在？  \n解决这个问题的办法是引入 No Target Brokers，也即集群中除去『Target Brokers』之外的 Broker。  \n对于 No Target Broker，依然需要写入一份 TopicQueueMapping，带上 epoch 和 totalQueues，但不拥有任何 LogicQueue。  \n有了这个信息之后，在进行一致性校验时，就可以识别出上述场景。  \n\n尤其要注意，如果 Nameserver 中没有任何信息，则需要主动去所有 Broker 拉取一遍。\n\n#### 切换时最新 LogicQueueMappingItem 的 logicOffset 决策问题\nlogicOffset的决策，依赖于上一个 PhysicalQueue 的最大位点。  \n此时，要么跳跃位点，要么等待上一个 PhysicalQueue 确保已经禁写。  \n当前实现，为了保障高可用，采用『切新禁旧再切新』的方式，同时跳跃位点。  \n\n#### logicOffset 为 -1 时的处理\n此时，可以写，但返回给 客户端的 offset 也是-1。  \n此时，不可以读最新 PhysicalQueue。  \n需要非常小心触发位点被重置：  \n- 忽略logicOffset为 -1 的item\n- 计算staticOffset时，如果发现logicOffset为-1，则报错\n\n目前只允许，SendMessage和GetMin时，返回-1。其余场景，要严格校验并报错。\n  \n\n#### 队列重复映射\n如果允许1个 PhysicalQueue 被重复利用，也即多段映射给多个 LogicQueue，或者从非0开始映射。  \n会带来以下麻烦：\n* 所有位点相关的API，需要考虑 MappingItem endOffset，因为超过了 endOffset 可能已经不属于 当前 LogicQueue 了\n* 新建 MappingItem，需要先获取 旧 MappingItem 的 endOffset  \n\n当前实现，为了保证简洁，禁止 PhysicalQueue 被重复利用，每次更新映射都会让物理层面的 writeQueues++ 和 readQueues++。  \n后续实现，可以考虑复用已经被清除掉的Physical，也即已经没有数据，位点从0开始。\n\n#### 备机更新映射\n当前，admin操作都是要求在Master操作的。因此，没有这个问题。  \nCommand操作时，提前预判Master是否存在，如果不存在，则提前报错，减少中间失败率。  \n\n#### 拉取消息时 OutOfRange 的判断\n以下情况会影响 OutOfRange 的判断\n* 从备机拉取消息（默认不会返回OFFSET_MOVED）\n* 中间 MappingItem 因为Commitlog的提前删除导致 PhysicalQueue Offset Truncation\n\n所以，OutOfRange 的判断，遵循以下原则：\n* 从 Leader Item 拉取，只有requestOffset > maxOffset，尊重 OFFSET_MOVED\n* 从 Earliest Item 拉取，只有 requestOffset < minOffset，尊重 OFFSET_MOVED\n* 其它情况，都忽略 OFFSET_MOVED\n\n如果没有恰当地处理 OFFSET_MOVED，可能造成位点被重置。\n\n需要注意，这个地方，产生了对 PullMessageResponseHeader 中 minOffset 和 maxOffset 的强依赖。\n在次此之前，这两个信息，只对客户端的限流有作用，对业务没有实际的影响。\n\n#### 拉取消息时的 中断问题\n当1个 PhysicalQueue 被拉取干净时，需要修正 nextBeginOffset 到下一个 PhysicalQueue。\n如果没有处理好，则直接会导致拉取中断，无法前进。\n#### pullResult 位点由谁设置的问题\n类似于Batch，由客户端设置，避免服务端解开消息：  \n在PullResultExt中新增字段 offsetDelta。\n#### Admin接口与User接口的适配方式区别\nUser 接口，使用范围广泛如多语言等，应该尽可能简单，把适配逻辑做在服务端，对『客户端』透明。  \n那么 Admin 接口呢，比如 examineTopicStats，适配逻辑是做在『服务端』还是『客户端』？  \n一个 Admin 接口，通常需要访问所有 Broker 的所有队列。\n如果做在服务端，则可能产生交叉访问，在极端情况下，性能会非常差，举个例子：  \n100 个 Broker，相互交叉映射过一遍，则Admin客户端首先要向 100 个 Broker 发请求，然后这 100 个 Broker 为了获取远程信息，各自向其余 Broker 发请求。\n其实际网络请求数就是 100 * 100 = 10000 个网络请求。放大效应十分明显。  \n同时，考虑到 Admin 接口，使用范围是有限的，无需考虑多语言适配等问题，可以把适配逻辑做在 Admin 客户端。\n\n#### 远程读的性能问题\n从实战经验来看，性能损耗几乎不计。\n#### 使用习惯的改变\n利用新的创建命令进行隔离。\n#### 消费SendBack问题\n目前的实现里，消费Send Back，是直接传回Commitlog Pos，这个在LogicQueue里行不通。\n需要修改API，改成传回『Logic Queue Offset』。\n#### 二阶消息的兼容性\n二阶消息，也即『原始消息』存储在『系统Topic』中，需要经过一轮『Read-ReWrite』逻辑才会被用户看见的消息。\n例如，定时消息，事务消息。\n二阶消息需要支持远程读写操作。\n一期的LogicQueue不支持『二阶消息』。\n\n### 待完成事项\n#### 阻止旧客户端的请求\n旧的客户端无法解析逻辑路由，但可以识别物理路由。如果错误使用，则会影响映射关系的准确性。\n#### 阻止Pop模式、事务消息、定时消息使用 LogicQueue\n不兼容 事务消息和定时消息。  \nLogicQueue 当前不支持Pop模式消费。\n#### Nameserver 相关生命周期完善\n目前没有处理Nameserver中Mapping数据的生命周期\n#### ConsumeQueue 的 correctMinOffset 逻辑存在缺陷\n可能导致 LogicQueue 无法清除已经过期的 MappingItem。\n#### getOffsetInQueueByTime 语义有缺陷\n可能导致 LogicQueue 时间搜索不准确。需要专项修复。\n#### Metadata 更新机制\n当前的更新机制太慢。且访问『不存在Broker』时，会频繁访问Nameserver，有打爆Nameserver的风险。\n#### examineConsumeStats 接口获取不到『最近消费时间』\n位点相关的消息可能不在本机，需要远程访问。\n#### resetOffset 需要适配\n当前没有适配。重置位点，可能会得到不符合预期的结果。\n#### MessageQueue 没有被物理清除\n当前只是产生遗留数据，占用一点点存储空间，没有太大影响。  \n如果将来要实现 物理 Queue 复用，则需要先完善相关逻辑。\n\n### 代码走读要点\n#### Admin 入口\n主要看两个类：  \nUpdateStaticTopicSubCommand   \nRemappingStaticTopicSubCommand   \n#### Metadata 入口\n主要看：  \nTopicQueueMappingManager\n#### Client 入口\n重点关注：  \nMQClientInstance.updateTopicRouteInfoFromNameServer\n#### Server 入口\n以 SendMessageProcessor 为例，插桩代码普遍是以下风格：\n```\nTopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader, true);\nRemotingCommand rewriteResult =  rewriteRequestForStaticTopic(requestHeader, mappingContext);\nif (rewriteResult != null) {\n    return CompletableFuture.completedFuture(rewriteResult);\n}\n```\n其它Processor类似\n#### 测试入口\nrocketmq-test模块，statictopic目录。\n\n\n\n\n\n"
  },
  {
    "path": "docs/cn/statictopic/The_Scope_Of_Static_Topic.md",
    "content": "### Version 记录\n| 时间 | 主要内容 | 作者 |\n| --- | --- | --- |\n| 2021-11-01 | 初稿，探讨Static Topic的Scope视线范围 | dongeforever |\n\n\n中文文档在描述特定专业术语时，仍然使用英文。\n\n### 需求背景\nRocketMQ的集群设计，是一个多集群、动态、零耦合的设计，具体体现在以下地方：\n- 一个 Nameserver 可以管理多个 Cluster\n- Broker 与 Cluster 之间是弱关联，Cluster仅仅只是一个标识符，主要在运维时使用来界定Topic的创建范围\n- 开发用户对 Cluster 无感知\n- 不同 Broker 之间没有任何关联\n\n这样的设计，在运维时带来了极大的便利，但也带来了一个问题：\n- Topic 的队列数无法固定\n\n基于 Logic Queue 技术而实现的 Static Topic，就是用来解决『固定队列数量』的问题。\n\n但这个『固定』要到何种范围呢？是一个值得探讨的问题。\n\n从理论上可以分析出来，有以下三种情况：\n- 单集群固定\n- 多集群固定\n- 全网固定\n\n#### 单集群固定\n一个 Static Topic，固定在一个 Cluster 内漂移。\n不同的 Cluster 内，可以拥有相同的 Static Topic。\n对应MessageQueue的Broker 命名规范为：\n```\n__logic__{clusterName}\n```\n#### 多集群固定\n一个 Static Topic，固定在特定的几个 Cluster 内漂移。\n没有交集的Cluster集合之间，可以拥有相同的 Static Topic。\n对应MessageQueue的Broker 命名规范为：\n```\n__logic__{cluster1}_{cluster2}_{xxx}\n```\n#### 全网固定\n全网是指『同一个Nameserver内』。\n一个 Static Topic，不与特定Cluster绑定，同一个Nameserver内，全网漂移。\n同一个Nameserver内，只有一个同名的 Static Topic。\n对应MessageQueue的Broker 命名规范为：\n```\n__logic__global\n```\n#### 为什么要引入Scope\n直接全网固定不就好了吗，为啥还要引入Scope呢？\n主要原因是，不想完全放弃 RocketMQ 『多集群、动态、零耦合』的设计优势。\n而全网固定，则意味着彻底失去了这个优势。\n\n举1个『多活保序』的场景：\n- ClusterA 部署在 SiteA 内，创建 Static Topic 『TopicTest』，有50个队列。\n- ClusterB 部署在 SiteB 内，创建 Static Topic 『TopicTest』，有50个队列。\n\n对Nameserver稍作修改，支持传入标识符(比如为scope或者unitName)，来获取特定范围内的 Topic Route。\n\n正常情况下：\n- SiteA 的Producer和Consumer 只能看见 ClusterA 的 MessageQueue，brokerName为 \"__logic__clusterA\"。\n- SiteB 的Producer和Consumer 只能看见 ClusterB 的 MessageQueue，brokerName为 \"__logic__clusterB\"。\n- 机房内就近访问，且机房内严格保序。\n\n假设 SiteA 宕机，此时对Nameserver发指令允许全网读，也即忽略客户端传入的 Scope或者unitName 标识符：\n- SiteB 的 Producer 仍然看见并写入 ClusterB 的 MessageQueue，brokerName为 \"__logic__clusterB\"\n- SiteB 的 Consumer 可以同时看见并读取 ClusterA 的 MessageQueue 和 ClusterB MessageQueue, brokerName为 \"__logic__clusterB\" 和 \"__logic__clusterA\n- 在这种场景下，Consumer 在消费 ClusterB 数据的同时，同时去消费 ClusterA 未消费完的数据\n\n不同地域的客户端，看见不同Scope的元数据，从而访问不同Scope的节点。\n\n#### 全球容灾集群\nRocketMQ 多个集群的元数据可以无缝在Nameserver处汇聚，同时又可以无缝地根据标识符拆分给不同地域的Producer和Consumer。\n这样一个『元数据可分可合』的设计优势，是其它消息中间件所不具备的，应该值得挖掘一下。  \n引入以下概念：\n- 融合集群，共享同一个Nameserver的集群之和\n- 单元集群，clusterName名字一样的集群，不同单元集群之间，物理隔离\n- namespace，租户，逻辑隔离，只是命名的区别\n\n如果单元集群部署在异地，所形成的『融合集群』，就是全球容灾集群：\n- 客户端引入 scope 或者 unitName 字段，默认情况，不同 scope或者unitName 获取的都是单元集群的元数据\n- 顺序性，关键在于 固定Producer端可见的队列，单元内的队列是固定的，因此可以保证单元内是顺序的\n- Consumer 端按照队列消费，天然是顺序的\n- 正常情况下，单元内封闭，也即单元产生的数据在同单元内消费掉\n- 灾难发生时，改变元数据的可见性，允许读其它 单元集群 未消费完的数据，也即跨单元读\n- 跨单元读，是指读『其它clusterName』的队列，不一定是远程读，如果本单元有相应的Slave节点，则直接本地读\n\n### 设计目标\nStatic Topic 实现 单集群固定 和 全网固定 两种Scope，以适配『全球容灾集群』。\n多集群，暂时没有必要。\n\n一期只实现 全网固定 这个Scope，但在格式上注意兼容\n\n#### SOT 增加 Scope 字段\n```\n{\n\"version\":\"1\",\n\"scope\": \"clusterA\",\n\"bname\": \"broker02\" //标记这份数据的原始存储位置，如果发送误拷贝，可以利用这个字段来进行标识\n\"epoch\": 0, //标记修改版本，用来做一致性校验\n\"totalQueues\":\"50\",  //当前Topic 总共有多少 LogicQueues\n}\n```\n\nscope字段：\n- 单集群固定，则就是 Cluster 名字\n- 全网固定，则为常量『global』\n\n\n\n\n\n\n\n\n"
  },
  {
    "path": "docs/en/CLITools.md",
    "content": "# Instructions on the use of mqadmin Management tools\n\nBefore introducing the mqadmin management tool, the following points need to be declared:\n\n- The way of executing a command is:./mqadmin {command} {args}\n- Almost all commands need to attach the -n option to represent the nameServer address, formatted as ip:port;\n- Almost all commands can get help information with the -h option;\n- If the broker address -b option and clusterName -c option are both configured with specific values, the command execution will select the broker address specified by -b option. The value of the -b option can only be configured with a single address. The format is ip:port. The default port value is 10911. If the value of the -b option is not configured, the command will be applied to all brokers in the entire cluster.\n- You can see many commands under tools, but not all commands can be used, only the commands initialized in MQAdminStartup can be used, you can also modify this class, add or customize commands;\n- Due to the issue of version update, a small number of commands may not be updated in time, please read the related command source code to eliminate and resolve the error.\n\n## 1 Topic related command instructions\n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>Name</td>\n  <td class=xl64 width=175 style='width:131pt'>Meaning</td>\n  <td class=xl64 width=177 style='width:133pt'>Command option</td>\n  <td class=xl64 width=185 style='width:139pt'>Explain</td>\n </tr>\n <tr height=132 style='height:99.0pt'>\n  <td rowspan=8 height=593 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:444.0pt;border-top:none;width:122pt'>updateTopic</td>\n  <td rowspan=8 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>Create or update the configuration of topic</td>\n  <td class=xl65 width=149 style='width:112pt'>-b</td>\n  <td class=xl66 width=159 style='width:119pt'>The -b option declares the specific address of the broker, indicating that the broker, in which the topic is located supports only a single broker and the address format is ip:port.</td>\n </tr>\n <tr height=132 style='height:99.0pt'>\n  <td height=132 class=xl65 width=149 style='height:99.0pt;width:112pt'>-c</td>\n  <td class=xl66 width=159 style='width:119pt'>The -c option declares the name of the cluster, which represents the cluster in which the current topic is located. (clusters are available through clusterList query)</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>Declare the service address of the nameServer, and the option format is ip:port</td>\n </tr>\n <tr height=76 style='height:57.0pt'>\n  <td height=76 class=xl65 width=149 style='height:57.0pt;width:112pt'>-p</td>\n  <td class=xl66 width=159 style='width:119pt'>The -p option is used to specify the read and write permission for the new topic (W=2 | R=4 | WR=6)</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=149 style='height:29.0pt;width:112pt'>-r</td>\n  <td class=xl66 width=159 style='width:119pt'>The -r option declares the number of readable queues (default 8)</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=149 style='height:29.0pt;width:112pt'>-w</td>\n  <td class=xl66 width=159 style='width:119pt'>The -w option declares the number of writable queues (default 8)</td>\n </tr>\n <tr height=95 style='height:71.0pt'>\n  <td height=95 class=xl65 width=149 style='height:71.0pt;width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>The -t option declares the name of the topic (the name can only use characters^ [a-zA-Z0-9s -] + $)</td>\n </tr>\n <tr height=132 style='height:99.0pt'>\n  <td rowspan=4 height=307 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:230.0pt;border-top:none;width:122pt'>deleteTopic</td>\n  <td rowspan=4 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>Delete the topic command</td>\n  <td class=xl65 width=149 style='width:112pt'>-c</td>\n  <td class=xl66 width=159 style='width:119pt'>The -c option specifies the name of the cluster, which means that one of the topic in the specified cluster is deleted (cluster names can be queried via clusterList)</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>Declare the service address of the nameServer, and the option format is ip:port</td>\n </tr>\n <tr height=95 style='height:71.0pt'>\n  <td height=95 class=xl65 width=149 style='height:71.0pt;width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>The -t option declares the name of the topic (the name can only use characters^ [a-zA-Z0-9s -] + $)</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=3 height=287 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:215.0pt;border-top:none;width:122pt'>topicList</td>\n  <td rowspan=3 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>View topic list information</td>\n  <td class=xl65 width=149 style='width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>Print help information</td>\n </tr>\n <tr height=207 style='height:155.0pt'>\n  <td height=207 class=xl65 width=149 style='height:155.0pt;width:112pt'>-c</td>\n  <td class=xl66 width=159 style='width:119pt'>If the -c option is not configured, only the topic list is returned, and the addition of -c option returns additional information about the clusterName, topic, consumerGroup, that is, the cluster and subscription to which the topic belongs, and no other option need to be configured.</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>Declare the service address of the nameServer, and the option format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=3 height=103 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:77.0pt;border-top:none;width:122pt'>topicRoute</td>\n  <td rowspan=3 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>To view topic specific routing information</td>\n  <td class=xl65 width=149 style='width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>Used to specify the name of the topic</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>Declare the service address of the nameServer, and the option format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=3 height=103 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:77.0pt;border-top:none;width:122pt'>topicStatus</td>\n  <td rowspan=3 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>The location of the offset used to view the topic message queue</td>\n  <td class=xl65 width=149 style='width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>Used to specify the name of the topic</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>Declare the service address of the nameServer, and the option format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=3 height=103 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:77.0pt;border-top:none;width:122pt'>topicClusterList</td>\n  <td rowspan=3 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>To view the list of clusters to which topic belongs</td>\n  <td class=xl65 width=149 style='width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>Used to specify the name of the topic</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>Declare the service address of the nameServer, and the option format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=6 height=518 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:380pt;border-top:none;width:122pt'>updateTopicPerm</td>\n  <td rowspan=6 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>This command is used to update read and write permissions for topic</td>\n  <td class=xl65 width=149 style='width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>Used to specify the name of the topic</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>Declare the service address of the nameServer, and the option format is ip:port</td>\n </tr>\n <tr height=132 style='height:99.0pt'>\n  <td height=132 class=xl65 width=149 style='height:99.0pt;width:112pt'>-b</td>\n  <td class=xl66 width=159 style='width:119pt'>The -b option declares the specific address of the broker, indicating that the broker, in which the topic is located supports only a single broker and the address format is ip:port.</td>\n </tr>\n <tr height=76 style='height:57.0pt'>\n  <td height=76 class=xl65 width=149 style='height:57.0pt;width:112pt'>-p</td>\n  <td class=xl66 width=159 style='width:119pt'>The -p option is used to specify the read and write permission for the new topic (W=2 | R=4 | WR=6)</td>\n </tr>\n <tr height=207 style='height:155.0pt'>\n  <td height=207 class=xl65 width=149 style='height:155.0pt;width:112pt'>-c</td>\n  <td class=xl66 width=159 style='width:119pt'>Used to specify the name of the cluster that represents the cluster in which the topic is located, which can be accessed through the clusterList query, but the -b parameter has a higher priority, and if no -b option related configuration is specified, the command is executed on all broker in the cluster</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=5 height=199 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:149.0pt;border-top:none;width:122pt'>updateOrderConf</td>\n  <td rowspan=5 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>The key, value configuration that creates, deletes, and retrieves specific namespaces from nameServer is not yet enabled.</td>\n  <td class=xl65 width=149 style='width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>Declare the service address of the nameServer, and the option format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>topic, key</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=149 style='height:29.0pt;width:112pt'>-v</td>\n  <td class=xl66 width=159 style='width:119pt'>orderConf, value</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-m</td>\n  <td class=xl66 width=159 style='width:119pt'>method, available values include get, put, delete</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=4 height=198 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:140pt;border-top:none;width:122pt'>allocateMQ</td>\n  <td rowspan=4 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>Computing load result of load message queue in consumer list with average load algorithm</td>\n  <td class=xl65 width=149 style='width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>Used to specify the name of the topic</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>Declare the service address of the nameServer, and the option format is ip:port</td>\n </tr>\n <tr height=95 style='height:71.0pt'>\n  <td height=95 class=xl65 width=149 style='height:71.0pt;width:112pt'>-i</td>\n  <td class=xl66 width=159 style='width:119pt'>IpList, is separated by commas to calculate which message queues these ip unload topic</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=4 height=142 class=xl68 width=163 style='border-bottom:1.0pt solid black;\n  height:106.0pt;border-top:1.0pt;width:122pt'>statsAll</td>\n  <td rowspan=4 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>For printing topic subscription, TPS, cumulative amount, 24 hours read and write total, etc.</td>\n  <td class=xl65 width=149 style='width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>Declare the service address of the nameServer, and the option format is ip:port</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=149 style='height:29.0pt;width:112pt'>-a</td>\n  <td class=xl66 width=159 style='width:119pt'>Whether to print only active topic</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>Used to specify the name of the topic</td>\n </tr>\n</table>\n\n\n\n\n\n\n\n## 2 Cluster related command instructions\n\n#### \n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>Name</td>\n  <td class=xl64 width=175 style='width:131pt'>Meaning</td>\n  <td class=xl64 width=177 style='width:133pt'>Command option</td>\n  <td class=xl64 width=185 style='width:139pt'>Explain</td>\n </tr>\n <tr height=207 style='height:155.0pt'>\n  <td rowspan=4 height=326 class=xl67 width=177 style='border-bottom:1.0pt;\n  height:244.0pt;border-top:none;width:133pt'><span\n  style='mso-spacerun:yes'> </span>clusterList</td>\n  <td rowspan=4 class=xl70 width=175 style='border-bottom:1.0pt;\n  border-top:none;width:131pt'>View cluster information, cluster, brokerName, brokerId, TPS, and so on</td>\n  <td class=xl65 width=177 style='width:133pt'>-m</td>\n  <td class=xl66 width=185 style='width:139pt'>Print more information (add print to # InTotalYest,\n  #OutTotalYest, #InTotalToday ,#OutTotalToday)</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=177 style='height:17.0pt;width:133pt'>-h</td>\n  <td class=xl66 width=185 style='width:139pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=177 style='height:43.0pt;width:133pt'>-n</td>\n  <td class=xl66 width=185 style='width:139pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=177 style='height:29.0pt;width:133pt'>-i</td>\n  <td class=xl66 width=185 style='width:139pt'>Print interval, unit basis is seconds</td>\n </tr>\n <tr height=95 style='height:71.0pt'>\n  <td rowspan=8 height=391 class=xl67 width=177 style='border-bottom:1.0pt;\n  height:292.0pt;border-top:none;width:133pt'>clusterRT</td>\n  <td rowspan=8 class=xl70 width=175 style='border-bottom:1.0pt;\n  border-top:none;width:131pt'>Send message to detect each broker RT of the cluster.the message send to ${BrokerName} Topic</td>\n  <td class=xl65 width=177 style='width:133pt'>-a</td>\n  <td class=xl66 width=185 style='width:139pt'>amount, total number per probe, RT = Total time/amount</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=177 style='height:29.0pt;width:133pt'>-s</td>\n  <td class=xl66 width=185 style='width:139pt'>Message size, unit basis is B</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=177 style='height:17.0pt;width:133pt'>-c</td>\n  <td class=xl66 width=185 style='width:139pt'>Which cluster to detect.</td>\n </tr>\n <tr height=76 style='height:57.0pt'>\n  <td height=76 class=xl65 width=177 style='height:57.0pt;width:133pt'>-p</td>\n  <td class=xl66 width=185 style='width:139pt'>Whether to print the formatted log,split with \"|\", not printed by default</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=177 style='height:17.0pt;width:133pt'>-h</td>\n  <td class=xl66 width=185 style='width:139pt'>Print help information</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=177 style='height:29.0pt;width:133pt'>-m</td>\n  <td class=xl66 width=185 style='width:139pt'>Owned computer room for printing</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=177 style='height:29.0pt;width:133pt'>-i</td>\n  <td class=xl66 width=185 style='width:139pt'>The interval, in seconds, at which a message is sent.</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=177 style='height:43.0pt;width:133pt'>-n</td>\n  <td class=xl66 width=185 style='width:139pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n</table>\n\n\n\n\n\n## 3 Broker related command instructions\n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>Name</td>\n  <td class=xl64 width=175 style='width:131pt'>Meaning</td>\n  <td class=xl64 width=177 style='width:133pt'>Command option</td>\n  <td class=xl64 width=185 style='width:139pt'>Explain</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=6 height=206 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:154.0pt;border-top:none;width:143pt'>updateBrokerConfig</td>\n  <td rowspan=6 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>The configuration information used to update the broker and the contents of the Broker.conf file are modified</td>\n  <td class=xl67 width=87 style='width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Declare the address of the broker and format as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>Specify the name of the cluster</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-k</td>\n  <td class=xl68 width=87 style='width:65pt'>the value of k</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-v</td>\n  <td class=xl68 width=87 style='width:65pt'>the value of value</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=3 height=137 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:103.0pt;border-top:none;width:143pt'>brokerStatus</td>\n  <td rowspan=3 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>For viewing broker related statistics and running status (almost all the information you want is inside)</td>\n  <td class=xl67 width=87 style='width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Declare the address of the broker and format as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=6 height=256 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:192.0pt;border-top:none;width:143pt'>brokerConsumeStats</td>\n  <td rowspan=6 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>Get the consumption of each consumer in broker and return information such as consume Offset,broker Offset,diff,timestamp by message queue dimension</td>\n  <td class=xl67 width=87 style='width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Declare the address of the broker and format as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>Configure the timeout of the request</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-l</td>\n  <td class=xl68 width=87 style='width:65pt'>Configure the diff threshold beyond which to print</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-o</td>\n  <td class=xl68 width=87 style='width:65pt'>Specifies whether the order topic, is typically false</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=2 height=114 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:86.0pt;border-top:none;width:143pt'>getBrokerConfig</td>\n  <td rowspan=2 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>Get configuration information for the broker</td>\n  <td class=xl67 width=87 style='width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Declare the address of the broker and format as ip:port</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=1 height=137 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:103.0pt;border-top:none;width:143pt'>wipeWritePerm</td>\n  <td rowspan=1 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>Clear write permissions for broker from nameServer</td>\n  <td class=xl67 width=87 style='width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Declare the BrokerName</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n   <td rowspan=3 height=137 class=xl69 width=191 style='border-bottom:1.0pt;\n   height:103.0pt;border-top:none;width:143pt'>addWritePerm</td>\n   <td rowspan=3 class=xl72 width=87 style='border-bottom:1.0pt\n   border-top:none;width:65pt'>Add write permissions for broker from nameServer</td>\n   <td class=xl67 width=87 style='width:65pt'>-b</td>\n   <td class=xl68 width=87 style='width:65pt'>Declare the BrokerName</td>\n  </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=4 height=160 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:120.0pt;border-top:none;width:143pt'>cleanExpiredCQ</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>Clean up expired consume Queue on broker, An expired queue may be generated if the number of columns is reduced manually</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Declare the address of the broker and format as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>Used to specify the name of the cluster</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=4 height=160 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:120.0pt;border-top:none;width:143pt'>deleteExpiredCommitLog</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>Clean up expired CommitLog files on broker. A maximum of 20 deletion operations can be performed, and a maximum of 10 files can be deleted each time.</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Declare the address of the broker and format as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>Used to specify the name of the cluster</td>\n </tr>\n <tr height=88 style='mso-height-source:userset;height:66.0pt'>\n  <td rowspan=4 height=191 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:143.0pt;border-top:none;width:143pt'>cleanUnusedTopic</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>Clean up unused topic on broker and release topic's consume Queue from memory, If the topic is removed manually, an unused topic will be generated</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Declare the address of the broker and format as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>Used to specify the name of the cluster</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=5 height=199 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:149.0pt;border-top:none;width:143pt'>sendMsgStatus</td>\n  <td rowspan=5 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>Send a message to the broker and then return the send status and RT</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>brokerName, note that this is not broker's address</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>Message size, the unit of account is B</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>Number of messages sent</td>\n </tr>\n</table>\n\n\n\n## 4 Message related command instructions\n\n#### \n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n<tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>Name</td>\n  <td class=xl64 width=175 style='width:131pt'>Meaning</td>\n  <td class=xl64 width=177 style='width:133pt'>Command option</td>\n  <td class=xl64 width=185 style='width:139pt'>Explain</td>\n </tr>\n <tr height=128 style='height:96.0pt'>\n  <td rowspan=3 height=208 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:156.0pt;border-top:none;width:65pt'>queryMsgById</td>\n  <td rowspan=3 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>Query msg according to offsetMsgId. If you use open source console, you should use offsetMsgId. There are other parameters for this command. For details, please read QueryMsgByIdSubCommand.\n</td>\n  <td class=xl67 width=87 style='width:65pt'>-i</td>\n  <td class=xl67 width=87 style='width:65pt'>msgId</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=4 height=126 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:94.0pt;border-top:none;width:65pt'>queryMsgByKey</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>Query messages based on message Key</td>\n  <td class=xl67 width=87 style='width:65pt'>-k</td>\n  <td class=xl67 width=87 style='width:65pt'>msgKey</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>The name of the topic</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=225 style='height:169.0pt'>\n  <td rowspan=6 height=390 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:292.0pt;border-top:none;width:65pt'>queryMsgByOffset</td>\n  <td rowspan=6 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>Query messages based on Offset</td>\n  <td class=xl67 width=87 style='width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>The name of broker,(Note here: the name of broker is filled in, not the address of broker, and the broker name can be found in clusterList)</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-i</td>\n  <td class=xl68 width=87 style='width:65pt'>Queue id of the query</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-o</td>\n  <td class=xl68 width=87 style='width:65pt'>The value of offset</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>The name of the topic</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=47>\n  <td rowspan=6 height=209 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:156.0pt;border-top:none;width:65pt'>queryMsgByUniqueKey</td>\n  <td rowspan=6 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>According to the msgId query, msgId is different from offsetMsgId. The specific differences can be found in common operational and maintenance problems. \"-g\" option and \"-d\" option are to be used together, and when you find the message, try to get a particular consumer to consume the message and return the result of the consumption.</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-i</td>\n  <td class=xl67 width=87 style='width:65pt'>unique msg id</td>\n </tr>\n <tr height=36 style='height:27.0pt'>\n  <td height=36 class=xl67 width=87 style='height:27.0pt;width:65pt'>-g</td>\n  <td class=xl67 width=87 style='width:65pt'>consumerGroup</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-d</td>\n  <td class=xl67 width=87 style='width:65pt'>clientId</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>The name of the topic</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=5 height=149 class=xl69 width=87 style='border-bottom:1.0pt\n  height:111.0pt;border-top:none;width:65pt'>checkMsgSendRT</td>\n  <td rowspan=5 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>Detect RT to send a message to topic, function similar to clusterRT</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>The name of the topic</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-a</td>\n  <td class=xl68 width=87 style='width:65pt'>the number of probes</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>The size of message</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=8 height=218 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:162.0pt;border-top:none;width:65pt'>sendMessage</td>\n  <td rowspan=8 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>Send a message that can be sent, as configured, to a particular message Queue, or to a normal send.</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>The name of the topic</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-p</td>\n  <td class=xl68 width=87 style='width:65pt'>body, message body</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-k</td>\n  <td class=xl67 width=87 style='width:65pt'>keys</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl67 width=87 style='width:65pt'>tags</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-b</td>\n  <td class=xl67 width=87 style='width:65pt'>brokerName</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-i</td>\n  <td class=xl67 width=87 style='width:65pt'>queueId</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=10 height=312 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:232.0pt;border-top:none;width:65pt'>consumeMessage</td>\n  <td rowspan=10 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>Consumer messages. You can consume messages based on offset, start timestamps, end timestamps, message queues, and configure different consumption logic for different execution, as detailed in ConsumeMessageCommand.</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>The name of the topic</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-b</td>\n  <td class=xl67 width=87 style='width:65pt'>brokerName</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-o</td>\n  <td class=xl68 width=87 style='width:65pt'>Start consumption from offset</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-i</td>\n  <td class=xl67 width=87 style='width:65pt'>queueId</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-g</td>\n  <td class=xl68 width=87 style='width:65pt'>Group of consumers</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>Specify a start timestamp in a format see -h</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-d</td>\n  <td class=xl68 width=87 style='width:65pt'>Specify a end timestamp</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>Specify how many messages to consume</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=8 height=282 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:210.0pt;border-top:none;width:65pt'>printMsg</td>\n  <td rowspan=8 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>Consume messages from broker and print them, optional time periods</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>The name of the topic</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>Character set, for example UTF-8</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>subExpress, filter expression</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Specify a start timestamp in a format see -h</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-e</td>\n  <td class=xl68 width=87 style='width:65pt'>Specify the end timestamp</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-d</td>\n  <td class=xl68 width=87 style='width:65pt'>Whether to print the message body</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=12 height=390 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:290.0pt;border-top:none;width:65pt'>printMsgByQueue</td>\n  <td rowspan=12 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>Similar to printMsg, but specifying message queue</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>The name of the topic</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-i</td>\n  <td class=xl67 width=87 style='width:65pt'>queueId</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-a</td>\n  <td class=xl67 width=87 style='width:65pt'>brokerName</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>Character set, for example UTF-8</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>subExpress, filter expression</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Specify a start timestamp in a format see -h</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-e</td>\n  <td class=xl68 width=87 style='width:65pt'>Specify the end timestamp</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-p</td>\n  <td class=xl68 width=87 style='width:65pt'>Whether to print a message</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-d</td>\n  <td class=xl68 width=87 style='width:65pt'>Whether to print the message body</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-f</td>\n  <td class=xl68 width=87 style='width:65pt'>Whether to count the number of tags and print\n</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=7 height=410 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:307.0pt;border-top:none;width:65pt'>resetOffsetByTime</td>\n  <td rowspan=7 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>Reset consumer offset by timestamp(without client restart).</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-g</td>\n  <td class=xl68 width=87 style='width:65pt'>Group of consumers</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>The name of the topic</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>Resets the offset corresponding to this timestamp in a format see -h, if you want to reset to maxOffset, the value is 'now'.</td>\n </tr>\n <tr height=188 style='height:141.0pt'>\n  <td height=188 class=xl67 width=87 style='height:141.0pt;width:65pt'>-f</td>\n  <td class=xl68 width=87 style='width:65pt'>Whether to force a reset, if set to false, only supports backtracking offset, if it is true, regardless of the relationship between offset and consume Offset with the timestamp</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>Whether to reset the C++ client offset</td>\n </tr>\n</table>\n\n\n\n## 5 Consumer and Consumer Group related command instructions\n\n#### \n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n<tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>Name</td>\n  <td class=xl64 width=175 style='width:131pt'>Meaning</td>\n  <td class=xl64 width=177 style='width:133pt'>Command option</td>\n  <td class=xl64 width=185 style='width:139pt'>Explain</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td rowspan=4 height=158 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:110pt;border-top:none;width:65pt'>consumerProgress</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>To view the subscriber consumption status, you can see the amount of message accumulation for a specific client IP</td>\n  <td class=xl67 width=87 style='width:65pt'>-g</td>\n  <td class=xl68 width=87 style='width:65pt'>The group name of consumer</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>Whether to print client IP</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=105 style='mso-height-source:userset;height:79.0pt'>\n  <td rowspan=5 height=260 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:195.0pt;border-top:none;width:65pt'>consumerStatus</td>\n  <td rowspan=5 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>See the consumer status, including whether the same subscription is in the same group, analyze whether the process queue is stacked, return the consumer jstack results, more content, and see ConsumerStatusSubCommand for the user</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=36 style='height:27.0pt'>\n  <td height=36 class=xl67 width=87 style='height:27.0pt;width:65pt'>-g</td>\n  <td class=xl67 width=87 style='width:65pt'>consumer group</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-i</td>\n  <td class=xl67 width=87 style='width:65pt'>clientId</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>Whether to execute jstack</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td rowspan=5 height=181 class=xl69 width=87 style='border-bottom:1.0pt\n  height:135.0pt;border-top:none;width:65pt'>getConsumerStatus</td>\n  <td rowspan=5 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>Get Consumer consumption progress</td>\n  <td class=xl67 width=87 style='width:65pt'>-g</td>\n  <td class=xl68 width=87 style='width:65pt'>the group name of consumer</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>Query topic</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-i</td>\n  <td class=xl68 width=87 style='width:65pt'>Ip address of consumer client</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=13 height=761 class=xl69 width=87 style='border-bottom:1.0pt\n  height:569.0pt;border-top:none;width:65pt'>updateSubGroup</td>\n  <td rowspan=13 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>Update or create a subscription</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>the address of broker</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>The name of cluster</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-g</td>\n  <td class=xl68 width=87 style='width:65pt'>The group name of consumer</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>Whether the group is allowed to consume</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-m</td>\n  <td class=xl68 width=87 style='width:65pt'>Whether to start consumption from the minimum offset</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-d</td>\n  <td class=xl68 width=87 style='width:65pt'>Is it a broadcast mode</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-q</td>\n  <td class=xl68 width=87 style='width:65pt'>The Number of retry queues</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-r</td>\n  <td class=xl68 width=87 style='width:65pt'>Maximum number of retries</td>\n </tr>\n <tr height=207 style='height:155.0pt'>\n  <td height=207 class=xl67 width=87 style='height:155.0pt;width:65pt'>-i</td>\n  <td class=xl68 width=87 style='width:65pt'>When the slaveReadEnable is on and which brokerId consumption is recommended for consumption from slave, the brokerid of slave, can be configured to consume from the slave actively</td>\n </tr>\n <tr height=132 style='height:99.0pt'>\n  <td height=132 class=xl67 width=87 style='height:99.0pt;width:65pt'>-w</td>\n  <td class=xl68 width=87 style='width:65pt'>If broker recommends consumption from slave, configuration determines which slave consumption to consume from, and configure a specific brokerId, such as 1</td>\n </tr>\n <tr height=76 style='height:57.0pt'>\n  <td height=76 class=xl67 width=87 style='height:57.0pt;width:65pt'>-a</td>\n  <td class=xl68 width=87 style='width:65pt'>Whether to notify other consumers of load balancing when the number of consumers changes</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=5 height=165 class=xl69 width=87 style='border-bottom:1.0pt\n  height:123.0pt;border-top:none;width:65pt'>deleteSubGroup</td>\n  <td rowspan=5 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>Remove subscriptions from broker</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>the address of broker</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>The name of cluster</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-g</td>\n  <td class=xl68 width=87 style='width:65pt'>The group name of consumer</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=6 height=172 class=xl69 width=87 style='border-bottom:1.0pt\n  height:120pt;border-top:none;width:65pt'>cloneGroupOffset</td>\n  <td rowspan=6 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>Use the offset of the source group in the target group\n</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>Source consumer group</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-d</td>\n  <td class=xl68 width=87 style='width:65pt'>Target consumer group</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>The name of topic</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-o</td>\n  <td class=xl68 width=87 style='width:65pt'>Not used yet</td>\n </tr>\n</table>\n\n\n\n\n## 6 Connection related command instructions\n\n#### \n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n<tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>Name</td>\n  <td class=xl64 width=175 style='width:131pt'>Meaning</td>\n  <td class=xl64 width=177 style='width:133pt'>Command option</td>\n  <td class=xl64 width=185 style='width:139pt'>Explain</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td rowspan=3 height=119 class=xl69 width=87 style='border-bottom:1.0pt\n  height:89.0pt;border-top:none;width:65pt'>consumerConnection</td>\n  <td rowspan=3 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>Query the network connection of consumer</td>\n  <td class=xl67 width=87 style='width:65pt'>-g</td>\n  <td class=xl68 width=87 style='width:65pt'>The group name of consumer</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td rowspan=4 height=142 class=xl69 width=87 style='border-bottom:1.0pt\n  height:106.0pt;border-top:none;width:65pt'>producerConnection</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>Query the network connection of producer</td>\n  <td class=xl67 width=87 style='width:65pt'>-g</td>\n  <td class=xl68 width=87 style='width:65pt'>the group name of producer</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>The name of topic</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n</table>\n\n\n\n\n## 7 NameServer related command instructions\n\n#### \n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n<tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>Name</td>\n  <td class=xl64 width=175 style='width:131pt'>Meaning</td>\n  <td class=xl64 width=177 style='width:133pt'>Command option</td>\n  <td class=xl64 width=185 style='width:139pt'>Explain</td>\n </tr>\n <tr height=21 style='height:16.0pt'>\n  <td rowspan=5 height=143 class=xl69 width=87 style='border-bottom:1.0pt\n  height:100pt;border-top:none;width:65pt'>updateKvConfig</td>\n  <td rowspan=5 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>Update the kv configuration of nameServer, which is not currently used</td>\n  <td class=xl75 width=87 style='width:65pt'>-s</td>\n  <td class=xl76 width=87 style='width:65pt'>Specify a specific namespace</td>\n </tr>\n <tr height=21 style='height:16.0pt'>\n  <td height=21 class=xl75 width=87 style='height:16.0pt;width:65pt'>-k</td>\n  <td class=xl75 width=87 style='width:65pt'>key</td>\n </tr>\n <tr height=21 style='height:16.0pt'>\n  <td height=21 class=xl75 width=87 style='height:16.0pt;width:65pt'>-v</td>\n  <td class=xl75 width=87 style='width:65pt'>value</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=4 height=126 class=xl69 width=87 style='border-bottom:1.0pt\n  height:94.0pt;border-top:none;width:65pt'>deleteKvConfig</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'> Delete the kv configuration of nameServer</td>\n  <td class=xl67 width=87 style='width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>Specify a specific namespace</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-k</td>\n  <td class=xl67 width=87 style='width:65pt'>key</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=2 height=80 class=xl69 width=87 style='border-bottom:1.0pt\n  height:60.0pt;border-top:none;width:65pt'>getNamesrvConfig</td>\n  <td rowspan=2 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>Get the configuration of the nameServer</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=4 height=126 class=xl69 width=87 style='border-bottom:1.0pt\n  height:94.0pt;border-top:none;width:65pt'>updateNamesrvConfig</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>Modifying the configuration of nameServer</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-k</td>\n  <td class=xl67 width=87 style='width:65pt'>The value of key</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-v</td>\n  <td class=xl67 width=87 style='width:65pt'>The value of value</td>\n </tr>\n</table>\n\n\n\n\n## 8 Other relevant command notes\n\n#### \n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n<tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>Name</td>\n  <td class=xl64 width=175 style='width:131pt'>Meaning</td>\n  <td class=xl64 width=177 style='width:133pt'>Command option</td>\n  <td class=xl64 width=185 style='width:139pt'>Explain</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=2 height=80 class=xl69 width=87 style='border-bottom:1.0pt\n  height:60.0pt;border-top:none;width:65pt'>startMonitoring</td>\n  <td rowspan=2 class=xl71 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>Used to start the monitoring process, monitor message deletion, retry queue messages, etc.</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>Print help information</td>\n </tr>\n</table>\n\n\n"
  },
  {
    "path": "docs/en/Concept.md",
    "content": "# Basic Concept                                \n\n## 1 Message Model\n\nRocketMQ message model is mainly composed of Producer, Broker and Consumer. The producer is responsible for producing messages and the consumer is for consuming messages, while the broker stores messages.     \nThe broker is an independent server during actual deployment, and each broker can store messages from multiple topics. Even messages from the same topic can be stored in the different brokers by sharding strategy.     \nThe message queue is used to store physical offsets of messages, and the message addresses are stored in separate queues. The consumer group consists of multiple consumer instances.    \n##  2 Producer    \nThe Producer is responsible for producing messages, typically by business systems. It sends messages generated by the systems to brokers. RocketMQ provides multiple paradigms of sending: synchronous, asynchronous, sequential and one-way. Both synchronous and asynchronous methods require the confirmation information return from the Broker, but one-way method does not require it.\n## 3 Consumer\nThe Consumer is responsible for consuming messages, typically the background system is responsible for asynchronous consumption. The consumer pulls messages from brokers and feeds them into application. From the perspective of user, two types of consumers are provided: pull consumer and push consumer.\n## 4 Topic\nThe Topic refers to a collection of one kind of message. Each topic contains several messages and one message can only belong to one topic. The topic is the basic unit of RocketMQ for message subscription.\n## 5 Broker Server\nAs the role of the transfer station, the Broker Server stores and forwards messages. In RocketMQ, the broker server is responsible for receiving messages sent from producers, storing them and preparing to handle pull requests. It also stores the related message meta data, including consumer groups, consuming progress, topics, queues info and so on.\n## 6 Name Server\nThe Name Server serves as the provider of routing service. The producer or the consumer can find the list of broker IP addresses for each topic through name server. Multiple name servers can be deployed in one cluster, but they are independent of each other and do not exchange information.\n## 7 Pull Consumer\nA type of Consumer, the application pulls messages from brokers by actively invoking the consumer pull message method, and the application has the advantages of controlling the timing and frequency of pulling messages. Once the batch of messages is pulled, user application will initiate consuming process.                    \n## 8 Push Consumer\nA type of Consumer, the application does not invoke the consumer pull message method to pull messages, instead the client invokes the pull message method itself. At the user level it seems like brokers push to the consumer when new messages arrive.                        \n## 9 Producer Group\nA collection of the same type of Producer, which sends the same type of messages with consistent logic. If a transaction message is sent and the original producer crashes after sending, the broker server will contact other producers in the same producer group to commit or rollback the transactional message.\n## 10 Consumer Group\nA collection of the same type of Consumer, which consume the same type of messages with consistent logic. The consumer group makes load-balance and fault-tolerance super easy in terms of message consuming.\nWarning: consumer instances of one consumer group must have exactly the same topic subscription(s).   \n\nRocketMQ supports two types of consumption mode: Clustering and Broadcasting.\n## 11 Consumption Mode - Clustering\nUnder the Clustering mode, all the messages from one topic will be delivered to all the consumer instances as evenly as possible. That is, one message can be consumed by only one consumer instance.\n## 12 Consumption Mode - Broadcasting\nUnder the Broadcasting mode, each consumer instance of the same consumer group receives every message published to the corresponding topic.\n## 13 Normal Ordered Message\nUnder the Normal Ordered Message mode, the messages received by consumers from the same ConsumeQueue are sequential, but the messages received from the different message queues may be non-sequential.\n## 14 Strictly Ordered Message\nUnder the Strictly Ordered Message mode, all messages received by the consumers from the same topic are sequential as the order they are stored.\n## 15 Message\nThe physical carrier of information transmitted by a messaging system, the smallest unit of production and consumption data, each message must belong to one topic.\nEach Message in RocketMQ has a unique message id and can carry a key used to store business-related value. The system has the function to query messages by its id or key.\n## 16 Tag\nFlags set for messages to distinguish different types of messages under the same topic, functioning as a \"sub-topic\". Messages from the same business unit can set different tags under the same topic for different business purposes. The tag can effectively maintain the clarity and consistency of the code and optimize the query system provided by RocketMQ. The consumer can realize different \"sub-topics\" by using tags in order to achieve better extensibility.\n"
  },
  {
    "path": "docs/en/Configuration_Client.md",
    "content": "## Client Configuration\n\nRelative to RocketMQ's Broker cluster, producers and consumers are clients. This section describes the common behavior configuration of producers and consumers, including updates for **RocketMQ 5.x**.\n\n### 1 Client Addressing Mode\n\n`RocketMQ` allows clients to locate the `Name Server`, which then helps them find the `Broker`. Below are various configurations, prioritized from highest to lowest. Higher priority configurations override lower ones.\n\n- Specified `Name Server` address in the code (multiple addresses separated by semicolons):\n\n```java\nproducer.setNamesrvAddr(\"192.168.0.1:9876;192.168.0.2:9876\");  \nconsumer.setNamesrvAddr(\"192.168.0.1:9876;192.168.0.2:9876\");\n```\n\n- Specified `Name Server` address in Java setup parameters:\n\n```text\n-Drocketmq.namesrv.addr=192.168.0.1:9876;192.168.0.2:9876  \n```\n\n- Specified `Name Server` address in environment variables:\n\n```text\nexport NAMESRV_ADDR=192.168.0.1:9876;192.168.0.2:9876   \n```\n\n- HTTP static server addressing (default):\n\nClients retrieve `Name Server` addresses from a static HTTP server:\n```text\nhttp://jmenv.tbsite.net:8080/rocketmq/nsaddr\n```\n\n- **New in RocketMQ 5.x:**\n  - Improved service discovery mechanism, allowing dynamic Name Server registration.\n  - Introduces `VIP_CHANNEL_ENABLED` for better failover:\n\n  ```java\n  producer.setVipChannelEnabled(false);\n  consumer.setVipChannelEnabled(false);\n  ```\n\n### 2 Client Configuration\n\n`DefaultMQProducer`, `TransactionMQProducer`, `DefaultMQPushConsumer`, and `DefaultMQPullConsumer` all extend the `ClientConfig` class, which provides common client configurations.\n\n#### 2.1 Client Common Configuration\n\n| Parameter Name                | Default Value  | Description  |\n|-------------------------------|---------------|--------------|\n| namesrvAddr                   |               | Name Server address list (semicolon-separated) |\n| clientIP                      | Local IP      | Client's local IP address (useful if automatic detection fails) |\n| instanceName                  | DEFAULT       | Unique name for the client instance |\n| clientCallbackExecutorThreads | 4             | Number of communication layer asynchronous callback threads |\n| pollNameServerInterval        | 30000         | Interval (ms) to poll Name Server |\n| heartbeatBrokerInterval       | 30000         | Interval (ms) for sending heartbeats to Broker |\n| persistConsumerOffsetInterval | 5000          | Interval (ms) for persisting consumer offsets |\n| **autoUpdateNameServer** (5.x) | true          | Automatically update Name Server addresses from registry |\n| **instanceId** (5.x)          |               | Unique identifier for each client instance |\n\n#### 2.2 Producer Configuration\n\n| Parameter Name                 | Default Value         | Description  |\n|--------------------------------|----------------------|--------------|\n| producerGroup                  | DEFAULT_PRODUCER     | Producer group name |\n| sendMsgTimeout                 | 3000                 | Timeout (ms) for sending messages |\n| retryTimesWhenSendFailed       | 2                    | Max retries for failed messages |\n| **enableBatchSend** (5.x)      | true                 | Enables batch message sending |\n| **enableBackPressure** (5.x)   | true                 | Prevents overload in high-traffic scenarios |\n\n#### 2.3 PushConsumer Configuration\n\n| Parameter Name                      | Default Value                      | Description |\n|--------------------------------------|------------------------------------|-------------|\n| consumerGroup                        | DEFAULT_CONSUMER                  | Consumer group name |\n| messageModel                         | CLUSTERING                        | Message consumption mode (CLUSTERING or BROADCAST) |\n| consumeFromWhere                     | CONSUME_FROM_LAST_OFFSET          | Default consumption position |\n| consumeThreadMin                     | 20                                | Min consumption thread count |\n| consumeThreadMax                     | 20                                | Max consumption thread count |\n| **Rebalance Strategies (5.x)**        | AllocateMessageQueueAveragelyByCircle | New rebalance strategy for better distribution |\n\n#### 2.4 PullConsumer Configuration\n\n| Parameter Name                     | Default Value                 | Description |\n|------------------------------------|------------------------------|-------------|\n| consumerGroup                      | DEFAULT_CONSUMER             | Consumer group name |\n| brokerSuspendMaxTimeMillis         | 20000                        | Max suspension time (ms) for long polling |\n| consumerPullTimeoutMillis          | 10000                        | Timeout (ms) for pull requests |\n| **messageGroup** (5.x)             |                              | Enables orderly consumption based on groups |\n\n#### 2.5 Message Data Structure\n\n| Field Name        | Default Value  | Description |\n|-------------------|---------------|-------------|\n| Topic            | null          | Required: Name of the message topic |\n| Body             | null          | Required: Message content |\n| Tags             | null          | Optional: Tag for filtering |\n| Keys             | null          | Optional: Business keys (e.g., order IDs) |\n| Flag             | 0             | Optional: Custom flag |\n| DelayTimeLevel   | 0             | Optional: Message delay level |\n| WaitStoreMsgOK   | TRUE          | Optional: Acknowledgment before storing |\n| **maxReconsumeTimes** (5.x) |   | Max retries before moving to dead-letter queue |\n| **messageGroup** (5.x) |   | Group-based message ordering |\n\n---\n\n"
  },
  {
    "path": "docs/en/Configuration_System.md",
    "content": "# The System Configuration (RocketMQ 5.x)\n\nThis section focuses on the configuration of the system (JVM/OS) for **RocketMQ 5.x**.\n\n## **1 JVM Options** ##\n\nThe latest released version of JDK 1.8 is recommended. Set the same Xms and Xmx value to prevent the JVM from resizing the heap for better performance. A simple JVM configuration is as follows:\n\n    -server -Xms8g -Xmx8g -Xmn4g\n\nDirect ByteBuffer memory size setting. Full GC will be triggered when the Direct ByteBuffer up to the specified size:\n\n    -XX:MaxDirectMemorySize=15g\n\nIf you don’t care about the boot time of RocketMQ broker, pre-touch the Java heap to make sure that every page will be allocated during JVM initialization is a better choice. Those who don’t care about the boot time can enable it:\n    \n    -XX:+AlwaysPreTouch\n\nDisable biased locking to potentially reduce JVM pauses:\n\n    -XX:-UseBiasedLocking\n\nAs for garbage collection, the G1 collector with JDK 1.8 is recommended:\n\n    -XX:+UseG1GC -XX:G1HeapRegionSize=16m\n    -XX:G1ReservePercent=25\n    -XX:InitiatingHeapOccupancyPercent=30\n\nThese GC options may seem a bit aggressive, but they have been proven to provide good performance in our production environment.\n\nDo not set a too small value for -XX:MaxGCPauseMillis, as it may cause the JVM to use a small young generation, leading to frequent minor GCs. Using a rolling GC log file is recommended:\n    \n    -XX:+UseGCLogFileRotation\n    -XX:NumberOfGCLogFiles=5\n    -XX:GCLogFileSize=30m\n    \nIf writing GC logs to disk increases broker latency, consider redirecting the GC log file to a memory file system:\n    \n    -Xloggc:/dev/shm/mq_gc_%p.log123\n\n## **2 Linux Kernel Parameters** ##\n\nThere is an `os.sh` script in the `bin` folder that lists several kernel parameters that can be used for production with minor modifications. Below parameters require attention. For more details, refer to the documentation for `/proc/sys/vm/*`.\n\n- **vm.extra_free_kbytes**: Tells the VM to keep extra free memory between the threshold where background reclaim (kswapd) kicks in and the threshold where direct reclaim (by allocating processes) kicks in. RocketMQ uses this parameter to avoid high latency in memory allocation. (Kernel version-specific)\n\n- **vm.min_free_kbytes**: If set lower than 1024KB, the system may become subtly broken and prone to deadlocks under high loads.\n\n- **vm.max_map_count**: Limits the maximum number of memory map areas a process may have. RocketMQ uses `mmap` to load `CommitLog` and `ConsumeQueue`, so setting a higher value is recommended.\n\n- **vm.swappiness**: Defines how aggressively the kernel swaps memory pages. Higher values increase aggressiveness, while lower values decrease the amount of swap. A value of **10** is recommended to avoid swap latency.\n\n- **File descriptor limits**: RocketMQ requires open file descriptors for files (`CommitLog` and `ConsumeQueue`) and network connections. It is recommended to set this to **655350**.\n\n- **Disk scheduler**: The **deadline I/O scheduler** is recommended for RocketMQ as it attempts to provide a guaranteed latency for requests.\n\n---\n"
  },
  {
    "path": "docs/en/Configuration_TLS.md",
    "content": "# TLS Configuration\nThis section introduces TLS configuration in RocketMQ.\n\n## 1 Generate Certificate Files\nUsers can generate certificate files using OpenSSL. It is suggested to generate files in Linux.\n\n### 1.1 Generate ca.pem\n```shell\nopenssl req -newkey rsa:2048 -keyout ca_rsa_private.pem -x509 -days 365 -out ca.pem\n```\n### 1.2 Generate server.csr\n```shell\nopenssl req -newkey rsa:2048 -keyout server_rsa.key  -out server.csr\n```\n### 1.3 Generate server.pem\n```shell\nopenssl x509 -req -days 365 -in server.csr -CA ca.pem -CAkey ca_rsa_private.pem -CAcreateserial -out server.pem\n```\n### 1.4 Generate client.csr\n```shell\nopenssl req -newkey rsa:2048 -keyout client_rsa.key -out client.csr\n```\n### 1.5 Generate client.pem\n```shell\nopenssl x509 -req -days 365 -in client.csr -CA ca.pem -CAkey ca_rsa_private.pem -CAcreateserial -out client.pem\n```\n### 1.6 Generate server.key\n```shell\nopenssl pkcs8 -topk8 -v1 PBE-SHA1-RC4-128 -in  server_rsa.key -out server.key\n```\n### 1.7 Generate client.key\n```shell\nopenssl pkcs8 -topk8 -v1 PBE-SHA1-RC4-128 -in client_rsa.key -out client.key\n```\n\n## 2 Create tls.properties\nCreate tls.properties, correctly configure the path and password of the generated certificates.\n\n```properties\n# The flag to determine whether use test mode when initialize TLS context. default is true\ntls.test.mode.enable=false                     \n# Indicates how SSL engine respect to client authentication, default is none\ntls.server.need.client.auth=require   \n# The store path of server-side private key\ntls.server.keyPath=/opt/certFiles/server.key\n# The password of the server-side private key\ntls.server.keyPassword=123456\n# The store path of server-side X.509 certificate chain in PEM format\ntls.server.certPath=/opt/certFiles/server.pem\n# To determine whether verify the client endpoint's certificate strictly. default is false\ntls.server.authClient=false\n# The store path of trusted certificates for verifying the client endpoint's certificate\ntls.server.trustCertPath=/opt/certFiles/ca.pem\n```\n\nIf you need to authenticate the client connection, you also need to add the following content to the file.\n\n```properties\n# The store path of client-side private key \ntls.client.keyPath=/opt/certFiles/client.key\n# The password of the client-side private key\ntls.client.keyPassword=123456\n# The store path of client-side X.509 certificate chain in PEM format\ntls.client.certPath=/opt/certFiles/client.pem\n# To determine whether verify the server endpoint's certificate strictly\ntls.client.authServer=false                    \n# The store path of trusted certificates for verifying the server endpoint's certificate\ntls.client.trustCertPath=/opt/certFiles/ca.pem\n```\n\n\n## 3 Update Rocketmq JVM parameters\n\nEdit the configuration file under the rocketmq/bin path to make tls.properties configurations take effect.\n\nThe value of \"tls.config.file\" needs to be replaced by the file path created in step 2.\n\n### 3.1 Edit runserver.sh\nAdd following content in JAVA_OPT:\n```shell\nJAVA_OPT=\"${JAVA_OPT} -Dtls.server.mode=enforcing -Dtls.config.file=/opt/rocketmq-4.9.3/conf/tls.properties\"\n```\n\n### 3.2 Edit runbroker.sh\nAdd following content in JAVA_OPT:\n\n```shell\nJAVA_OPT=\"${JAVA_OPT} -Dorg.apache.rocketmq.remoting.ssl.mode=enforcing -Dtls.config.file=/opt/rocketmq-4.9.3/conf/tls.properties  -Dtls.enable=true\"\n```\n\n# 4 Client connection\n\nCreate tlsclient.properties using by client. Add following content:\n```properties\n# The store path of client-side private key \ntls.client.keyPath=/opt/certFiles/client.key\n# The password of the client-side private key\ntls.client.keyPassword=123456\n# The store path of client-side X.509 certificate chain in PEM format\ntls.client.certPath=/opt/certFiles/client.pem               \n# The store path of trusted certificates for verifying the server endpoint's certificate\ntls.client.trustCertPath=/opt/certFiles/ca.pem\n```\n\nAdd following parameters in JVM. The value of \"tls.config.file\" needs to be replaced by the file path we created:\n```properties\n-Dtls.client.authServer=true -Dtls.enable=true  -Dtls.test.mode.enable=false  -Dtls.config.file=/opt/certs/tlsclient.properties\n```\n\nEnable TLS for client like the following:\n```Java\npublic class ExampleProducer {\n    public static void main(String[] args) throws Exception {\n        DefaultMQProducer producer = new DefaultMQProducer(\"please_rename_unique_group_name\");\n        //setUseTLS should be true\n        producer.setUseTLS(true);\n        producer.start();\n\n        // Send messages as usual.\n        producer.shutdown();\n    }\n}\n```\n\n## 5 Proxy TLS Configuration\n\nRocketMQ Proxy uses `rmq-proxy.json` (not `tls.properties`) for TLS configuration. The proxy supports TLS for both its gRPC and Remoting protocol endpoints.\n\n### 5.1 Configure rmq-proxy.json\n\nAdd TLS-related fields to `distribution/conf/rmq-proxy.json`:\n\n```json\n{\n  \"rocketMQClusterName\": \"DefaultCluster\",\n  \"tlsTestModeEnable\": false,\n  \"tlsKeyPath\": \"/opt/certFiles/server.key\",\n  \"tlsKeyPassword\": \"123456\",\n  \"tlsCertPath\": \"/opt/certFiles/server.pem\"\n}\n```\n\n| Field | Type | Default | Description |\n|-------|------|---------|-------------|\n| `tlsTestModeEnable` | boolean | `true` | Use self-signed certificates for testing. Set to `false` for production. |\n| `tlsKeyPath` | string | `${PROXY_HOME}/conf/tls/rocketmq.key` | Path to the server private key file (PKCS#8 PEM format). |\n| `tlsKeyPassword` | string | `\"\"` | Password for the encrypted private key. Leave empty if the key is not encrypted. |\n| `tlsCertPath` | string | `${PROXY_HOME}/conf/tls/rocketmq.crt` | Path to the server certificate chain file (X.509 PEM format). |\n| `tlsCertWatchIntervalMs` | int | `3600000` | Interval in milliseconds to check for certificate file changes. |\n\n### 5.2 Update Proxy JVM parameters\n\nEdit `runproxy.sh` (or the script that launches the proxy) to enable TLS enforcing mode:\n\n```shell\nJAVA_OPT=\"${JAVA_OPT} -Dtls.server.mode=enforcing\"\n```\n\nThe three available TLS modes are:\n- `disabled` - TLS is not supported; incoming TLS handshakes are rejected.\n- `permissive` - TLS is optional; the proxy accepts both TLS and non-TLS connections.\n- `enforcing` - TLS is required; non-TLS connections are rejected.\n"
  },
  {
    "path": "docs/en/Debug_In_Idea.md",
    "content": "## How to Debug RocketMQ in Idea\n\n### Step0: Resolve dependencies\n1. To download the Maven dependencies required for running RocketMQ, you can use the following command:`mvn clean install -Dmaven.test.skip=true`\n2. Ensure successful local compilation.\n\n### Step1: Start NameServer\n1. The startup class for NameServer is located in `org.apache.rocketmq.namesrv.NamesrvStartup`.\n2. Add environment variable `ROCKETMQ_HOME=<rocketmq repository directory>` in `Idea-Edit Configurations`.\n![Idea_config_nameserver.png](../cn/image/Idea_config_nameserver.png)\n3. Run NameServer and if the following log output is observed, it indicates successful startup.\n```shell\nThe Name Server boot success. serializeType=JSON, address 0.0.0.0:9876\n```\n\n### Step2: Start Broker\n1. The startup class for Broker is located in`org.apache.rocketmq.broker.BrokerStartup`\n2. Create the `/rocketmq/conf/broker.conf` file or simply copy it from the official release package.\n```shell\n# broker.conf\n\nbrokerClusterName = DefaultCluster\nbrokerName = broker-a\nbrokerId = 0\ndeleteWhen = 04\nfileReservedTime = 48\nbrokerRole = ASYNC_MASTER\nflushDiskType = ASYNC_FLUSH\nnamesrvAddr = 127.0.0.1:9876\n```\n3. Add the runtime parameter `-c /Users/xxx/rocketmq/conf/broker.conf` and the environment variable `ROCKETMQ_HOME=<rocketmq repository directory>` in `Idea-Edit Configurations`.\n![Idea_config_broker.png](../cn/image/Idea_config_broker.png)\n4. Run the Broker and if the following log is observed, it indicates successful startup.\n```shell\nThe broker[broker-a,192.169.1.2:10911] boot success...\n```\n\n### Step3: Send or Consume Messages\nRocketMQ startup is now complete. You can use the examples provided in `/example` to send and consume messages.\n\n### Additional: Start the Proxy locally.\n1. RocketMQ 5.x introduced the Proxy mode. Using the `LOCAL` mode eliminates the need for `Step2`. The startup class is located at `org.apache.rocketmq.proxy.ProxyStartup`.\n2. Add the environment variable `ROCKETMQ_HOME=<rocketmq repository directory>` in `Idea-Edit Configurations`.\n3. Create a new configuration file named `rmq-proxy.json` in the `/conf/` directory.\n```json\n{\n  \"rocketMQClusterName\": \"DefaultCluster\",\n  \"nameSrvAddr\": \"127.0.0.1:9876\",\n  \"proxyMode\": \"local\"\n}\n```\n4. Run the Proxy, and if the following log is observed, it indicates successful startup.\n```shell\nSat Aug 26 15:29:33 CST 2023 rocketmq-proxy startup successfully\n```"
  },
  {
    "path": "docs/en/Deployment.md",
    "content": "# Deployment Architectures and Setup Steps\n\n## Cluster Setup\n\n### 1 Single Master mode\n\nThis is the simplest, but also the riskiest mode, that makes the entire service unavailable once the broker restarts or goes down. Production environments are not recommended, but it can be used for local testing and development. Here are the steps to build.\n\n**1）Start NameServer**\n\n```shell\n### Start Name Server first\n$ nohup sh mqnamesrv &\n \n### Then verify that the Name Server starts successfully\n$ tail -f ~/logs/rocketmqlogs/namesrv.log\nThe Name Server boot success...\n```\n\nWe can see 'The Name Server boot success.. ' in namesrv.log that indicates the NameServer has been started successfully.\n\n**2）Start Broker**\n\n```shell\n### Also start broker first\n$ nohup sh bin/mqbroker -n localhost:9876 &\n\n### Then verify that the broker is started successfully, for example, the IP of broker is 192.168.1.2 and the name is broker-a\n$ tail -f ~/logs/rocketmqlogs/Broker.log \nThe broker[broker-a,192.169.1.2:10911] boot success...\n```\n\nWe can see 'The broker[brokerName,ip:port] boot success..' in Broker.log that indicates the broker has been started successfully.\n\n### 2 Multiple Master mode\n\nMultiple master mode means a mode with all master nodes(such as 2 or 3 master nodes) and no slave node. The advantages and disadvantages of this mode are as follows:\n\n- Advantages: \n  1. Simple configuration.\n  2. Outage or restart(for maintenance) of one master node has no impact on the application. \n  3. When the disk is configured as RAID10, messages are not lost because the RAID10 disk is very reliable, even if the machine is not recoverable (In the case of asynchronous flush disk mode of the message, a small number of messages are lost; If the brush mode of a message is synchronous, no message will be lost).\n  4. In this mode, the performance is the highest.\n- Disadvantages:\n  1. During a single machine outage, messages that are not consumed on this machine are not subscribed to until the machine recovers, and message real-time is affected.\n\nThe starting steps for multiple master mode are as follows:\n\n**1）Start NameServer**\n\n```shell\n### Start Name Server first\n$ nohup sh mqnamesrv &\n \n### Then verify that the Name Server starts successfully\n$ tail -f ~/logs/rocketmqlogs/namesrv.log\nThe Name Server boot success...\n```\n\n**2）Start the Broker cluster**\n\n```shell\n### For example, starting the first Master on machine A, assuming that the configured NameServer IP is: 192.168.1.1.\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-noslave/broker-a.properties &\n \n### Then starting the second Master on machine B, assuming that the configured NameServer IP is: 192.168.1.1.\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-noslave/broker-b.properties &\n\n...\n```\n\nThe boot command shown above is used in the case of a single NameServer.For clusters of multiple NameServer, the address list after the -n argument in the broker boot command is separated by semicolons, for example, 192.168.1.1: 9876;192.168.1.2: 9876.\n\n### 3 Multiple Master And Multiple Slave Mode-Asynchronous replication\n\nEach master node configures more than one slave nodes, with multiple pairs of master-slave.HA uses asynchronous replication, with a short message delay (millisecond) between master node and slave node.The advantages and disadvantages of this mode are as follows:\n\n- Advantages: \n  1. Even if the disk is corrupted, very few messages will be lost and the real-time performance of the message will not be affected.\n  2. At the same time, when master node is down, consumers can still consume messages from slave node, and the process is transparent to the application itself and does not require human intervention.\n  3. Performance is almost as high as multiple master mode.\n- Disadvantages:\n  1. A small number of messages will be lost when master node is down and the disk is corrupted.\n\nThe starting steps for multiple master and multiple slave mode are as follows:\n\n**1）Start NameServer**\n\n```shell\n### Start Name Server first\n$ nohup sh mqnamesrv &\n \n### Then verify that the Name Server starts successfully\n$ tail -f ~/logs/rocketmqlogs/namesrv.log\nThe Name Server boot success...\n```\n\n**2）Start the Broker cluster**\n\n```shell\n### For example, starting the first Master on machine A, assuming that the configured NameServer IP is: 192.168.1.1 and port is 9876.\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-a.properties &\n \n### Then starting the second Master on machine B, assuming that the configured NameServer IP is: 192.168.1.1 and port is 9876.\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-b.properties &\n \n### Then starting the first Slave on machine C, assuming that the configured NameServer IP is: 192.168.1.1 and port is 9876.\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-a-s.properties &\n \n### Last starting the second Slave on machine D, assuming that the configured NameServer IP is: 192.168.1.1 and port is 9876.\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-b-s.properties &\n```\n\nThe above shows a startup command for 2M-2S-Async mode, similar to other nM-nS-Async modes.\n\n### 4 Multiple Master And Multiple Slave Mode-Synchronous dual write\n\nIn this mode, multiple slave node are configured for each master node and there are multiple pairs of Master-Slave.HA uses synchronous double-write, that is, the success response will be returned to the application only when the message is successfully written into the master node and replicated to more than one slave node.\n\nThe advantages and disadvantages of this model are as follows:\n\n- Advantages: \n  1. Neither the data nor the service has a single point of failure. \n  2. In the case of master node shutdown, the message is also undelayed. \n  3. Service availability and data availability are very high;\n- Disadvantages:\n  1. The performance in this mode is slightly lower than in asynchronous replication mode (about 10% lower).\n  2. The RT sending a single message is slightly higher, and the current version, the slave node cannot automatically switch to the master after the master node is down.\n\nThe starting steps are as follows:\n\n**1）Start NameServer**\n\n```shell\n### Start Name Server first\n$ nohup sh mqnamesrv &\n \n### Then verify that the Name Server starts successfully\n$ tail -f ~/logs/rocketmqlogs/namesrv.log\nThe Name Server boot success...\n```\n\n**2）Start the Broker cluster**\n\n```shell\n### For example, starting the first Master on machine A, assuming that the configured NameServer IP is: 192.168.1.1 and port is 9876.\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-a.properties &\n \n### Then starting the second Master on machine B, assuming that the configured NameServer IP is: 192.168.1.1 and port is 9876.\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-b.properties &\n \n### Then starting the first Slave on machine C, assuming that the configured NameServer IP is: 192.168.1.1 and port is 9876.\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-a-s.properties &\n \n### Last starting the second Slave on machine D, assuming that the configured NameServer IP is: 192.168.1.1 and port is 9876.\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-b-s.properties &\n```\n\nThe above Master and Slave are paired by specifying the same config named \"brokerName\", the \"brokerId\" of the master node must be 0, and the \"brokerId\" of the slave node must be greater than 0."
  },
  {
    "path": "docs/en/Design_Filter.md",
    "content": "# Message Filter\nRocketMQ - a distributed message queue, is different with all other MQ middleware, on the way of filtering messages. It's do the filter when the messages are subscribed via consumer side.RocketMQ do it lies in the separate storage mechanism that Producer side writing messages and Consumer subscribe messages, Consumer side will get an index from a logical message queue ConsumeQueue when subscribing, then read message entity from CommitLog using the index. So in the end, it is still impossible to get around its storage structure.The storage structure of ConsumeQueue is as follows, and there is a 8-byte Message Tag hashcode, The message filter based on Tag value is just used this Message Tag hash-code.\n\n![](images/rocketmq_design_7.png)\n\nThe RocketMQ has two mainly filter types:\n\n* Tag filtering: Consumer can specify not only the message topic but also the message tag values, when subscribing. Multiple tag values need to be separated by '||'. When consumer subscribing a message, it builds the subscription request into a `SubscriptionData` object and sends a pull message request to the Broker side. Before the Broker side reads data from the RocketMQ file storage layer - Store, it will construct a `MessageFilter` using the `SubscriptionData` object and then pass it to the Store. Store get a record from `ConsumeQueue`, and it will filter the message by the saved tag hashcode, it is unable to filter the messages exactly in the server side because of only the hashcode will be used when filtering, Therefore, after the Consumer pulls the message, it also needs to compare the original tag string of the message. If the original tag string is not same with the expected, the message will be ignored.\n\n* SQL92 filtering: This filter behavior is almost same with the above `Tag filtering` method. The only difference is on the way how Store works. The rocketmq-filter module is responsible for the construction and execution of the real SQL expression. Executing an SQL expression every time a filter is executed affects efficiency, so RocketMQ uses BloomFilter to avoid doing it every time. The expression context of SQL92 is a property of the message.\n"
  },
  {
    "path": "docs/en/Design_LoadBlancing.md",
    "content": "## Load Balancing\nLoad balancing in RocketMQ is accomplished on Client side. Specifically, it can be divided into load balancing at Producer side when sending messages and load balancing at Consumer side when subscribing messages.\n\n### Producer Load Balancing\nWhen the Producer sends a message, it will first find the specified TopicPublishInfo according to Topic. After getting the routing information of TopicPublishInfo, the RocketMQ client will select a queue (MessageQueue) from the messageQueue List in TopicPublishInfo  to send the message by default.Specific fault-tolerant strategies are defined in the MQFaultStrategy class.\nHere is a sendLatencyFaultEnable switch variable, which, if turned on, filters out the Broker agents that are not available on the basis of randomly gradually increasing modular arithmetic selection. The so-called \"latencyFault Tolerance\" refers to a certain period of time to avoid previous failures. For example, if the latency of the last request exceeds 550 Lms, it will evade 30000 Lms; if it exceeds 1000L, it will evade 60000L; if it is closed, it will choose a queue (MessageQueue) to send messages by randomly gradually increasing modular arithmetic, and the latencyFault Tolerance mechanism is the key to achieve high availability of message sending.\n\n### Consumer Load Balancing\nIn RocketMQ, the two consumption modes (Push/Pull) on the Consumer side are both based on the pull mode to get the message, while in the Push mode it is only a kind of encapsulation of the pull mode, which is essentially implemented as the message pulling thread after pulling a batch of messages from the server. After submitting to the message consuming thread pool, it continues to try again to pull the message to the server. If the message is not pulled, the pull is delayed and continues. In both pull mode based consumption patterns (Push/Pull), the Consumer needs to know which message queue - queue from the Broker side to get the message. Therefore, it is necessary to do load balancing on the Consumer side, that is, which Consumer consumption is allocated to the same ConsumerGroup by more than one MessageQueue on the Broker side.\n\n#### 1 Heartbeat Packet Sending on Consumer side\nAfter Consumer is started, it continuously sends heartbeat packets to all Broker instances in the RocketMQ cluster via timing task (which contains the message consumption group name, subscription relationship collection,Message communication mode and the value of the client id,etc). After receiving the heartbeat message from Consumer, Broker side maintains it in Consumer Manager's local caching variable—consumerTable, At the same time, the encapsulated client network channel information is stored in the local caching variable—channelInfoTable, which can provide metadata information for the later load balancing of Consumer.\n#### 2 Core Class for Load Balancing on Consumer side—RebalanceImpl\nStarting the MQClientInstance instance in the startup process of the Consumer instance will complete the start of the load balancing service thread-RebalanceService (executed every 20 s). By looking at the source code, we can find that the run () method of the RebalanceService thread calls the rebalanceByTopic () method of the RebalanceImpl class, which is the core of the Consumer end load balancing. Here, rebalanceByTopic () method will do different logical processing depending on whether the consumer communication type is \"broadcast mode\" or \"cluster mode\". Here we mainly look at the main processing flow in cluster mode:\n##### 1) Get the message consumption queue set (mqSet) under the Topic from the local cache variable—topicSubscribeInfoTable of the rebalanceImpl instance.\n##### 2) Call mQClientFactory. findConsumerIdList () method to send a RPC communication request to Broker side to obtain the consumer Id list under the consumer group based on the parameters of topic and consumer group (consumer table constructed by Broker side based on the heartbeat data reported by the front consumer side responds and returns, business request code: GET_CONSUMER_LIST_BY_GROUP);\n##### 3) First, the message consumption queue and the consumer Id under Topic are sorted, then the message queue to be pulled is calculated by using the message queue allocation strategy algorithm (default: the average allocation algorithm of the message queue). The average allocation algorithm here is similar to the paging algorithm. It ranks all MessageQueues like records. It ranks all consumers like pages. It calculates the average size of each page and the range of each page record. Finally, it traverses the whole range and calculates the records that the current consumer should allocate to (MessageQueue here).\n![Image text](https://github.com/apache/rocketmq/raw/develop/docs/cn/image/rocketmq_design_8.png)\n\n\n##### 4) Then, the updateProcessQueueTableInRebalance () method is invoked, which first compares the allocated message queue set (mqSet) with processQueueTable for filtering.\n![Image text](https://github.com/apache/rocketmq/raw/develop/docs/cn/image/rocketmq_design_9.png)\n\n - The red part of the processQueueTable annotation in the figure above\n   indicates that it is not included with the assigned message queue set\n   mqSet. Set the Dropped attribute to true for these queues, and then\n   check whether these queues can remove the processQueueTable cache\n   variable or not. The removeUnnecessaryMessageQueue () method is\n   executed here, that is, check every 1s to see if the locks of the\n   current consumption processing queue can be retrieved and return true\n   if they are retrieved. If the lock of the current consumer processing\n   queue is still not available after waiting for 1s, it returns false.\n   If true is returned, the corresponding Entry is removed from the\n   processQueueTable cache variable.\n - The green section in processQueueTable above represents the\n   intersection with the assigned message queue set mqSet. Determine\n   whether the ProcessQueue has expired, regardless of Pull mode, if it\n   is Push mode, set the Dropped attribute to true, and call the\n   removeUnnecessaryMessageQueue () method to try to remove Entry as\n   above;\n   \nFinally, a ProcessQueue object is created for each MessageQueue in the filtered message queue set (mqSet) and stored in the processQueueTable queue of RebalanceImpl (where the computePullFromWhere (MessageQueue mq) method of the RebalanceImpl instance is invoked to obtain the next progress consumption value offset of the MessageQueue object, which is then populated into the attribute of pullRequest object to be created next time.), and create pull request object—pullRequest to add to pull list—pullRequestList, and finally execute dispatchPullRequest () method. PullRequest object of Pull message is put into the blocking queue pullRequestQueue of PullMessageService service thread in turn, and the request of Pull message is sent to Broker end after the service thread takes out. \n\nThe core design idea of message consumption queue is that a message consumption queue can only be consumed by one consumer in the same consumer group at the same time, and a message consumer can consume multiple message queues at the same time.\n"
  },
  {
    "path": "docs/en/Design_Query.md",
    "content": "# Message Queries\n\nRocketMQ supports message queries by two dimensions, which are \"Query Message by Message Id\" and \"Query Message by Message Key\".\n\n## 1. Query Message by Message Id\nThe MessageId in RocketMQ has a total length of 16 bytes, including the broker address (IP address and port) and CommitLog offset. In RocketMQ, the specific approach is that the Client resolves the Broker's address (IP address and port) and the CommitLog's offset address from the MessageId. Then both of them are encapsulated into an RPC request, and finally it will be sent through the communication layer (business request code: VIEW_MESSAGE_BY_ID). The Broker reads a message by using the CommitLog offset and size to find the real message in the CommitLog and then return, which is how QueryMessageProcessor works.\n\n## 2. Query Message by Message Key\n\"Query Messages by Message Key\" is mainly based on RocketMQ's IndexFile. The logical structure of the IndexFile is similar to the implementation of HashMap in JDK. The specific structure of the IndexFile is as follows:\n\n![](images/rocketmq_design_message_query.png)\n\nThe IndexFile provides the user with the querying service by “Querying Messages by Message Key”. The IndexFile is stored in $HOME\\store\\index${fileName}, and the file name is named after the timestamp at the time of creation. The file size is fixed, which is 420,000,040 bytes (40+5million\\*4+20million\\*20). If the UNIQ_KEY is set in the properties of the message, then the \"topic + ‘#’ + UNIQ_KEY\" will be used as the index. Likewise, if the KEYS is set in the properties of the message (multiple KEYs should be separated by spaces), then the \"topic + ‘#’ + KEY\" will be used as the index.\n\nThe header of IndexFile contains eight fields, `beginTimestamp`(8 bytes), `endTimestamp`(8 bytes), `beginPhyOffset`(8 bytes), `endPhyOffset`(8 bytes), `hashSlotCount`(4 bytes), and `indexCount`(4 bytes).`beginTimestamp` and `endTimestamp` represents the storeTimestamp of the message corresponding to the first and last index, `beginPhyOffset` and `endPhyOffset` represent the physical offset of the message corresponding to the first and last index. `hashSlotCount` represents the count of hash slot. `indexCount` represents the count of indexes.\n\nThe index data contains four fields, `Key Hash`, `CommitLog offset`, `Timestamp` and `NextIndex offset`, for a total of 20 Bytes. The `NextIndex offset` of the index data will point to the previous index data if the `Key Hash` of the index data is the same as that of the previous index data. If a hash conflict occurs, then the `NextIndex offset` can be used as the field to string all conflicting indexes in a linked list. What the `Timestamp` records is the time difference between the `storeTimestamp` of the message associated with the current key and the `BeginTimestamp` in the `IndexHeader`, instead of a specific time. The structure of the entire IndexFile is shown in the graph. The Header is used to store some general statistics, which needs 40 bytes. The Slot Table of 4\\*5million bytes does not save the real index data, but saves the header of the singly linked list corresponding to each slot. The Index Linked List of 20\\*20million is the real index data, that is, an Index File can hold 20million indexes.\n\nThe specific method of \"Query Message by Message Key\" is that the topic and message key are used to find the record in the IndexFile, and then read the message from the file of CommitLog according to the CommitLog offset in this record."
  },
  {
    "path": "docs/en/Design_Remoting.md",
    "content": "\nRocketMQ message queue cluster mainly includes four roles: NameServer, Broker (Master/Slave), Producer and Consumer. The basic communication process is as follows:\n(1) After Broker start-up, it needs to complete one operation: register itself to NameServer, and then report Topic routing information to NameServer at regular intervals of 30 seconds.\n(2) When message Producer sends a message as a client, it needs to obtain routing information from the local cache TopicPublishInfoTable according to the Topic of the message. If not, it will be retrieved from NameServer and update to local cache, at the same time, Producer will retrieve routing information from NameServer every 30 seconds by default.\n(3) Message Producer chooses a queue to send the message according to the routing information obtained in 2); Broker receives the message and records it in disk as the receiver of the message.\n(4) After message Consumer gets the routing information according to 2) and complete the load balancing of the client, then select one or several message queues to pull messages and consume them.\n\nFrom 1) ~ 3) above, we can see that both Producer, Broker and NameServer communicate with each other(only part of MQ communication is mentioned here), so how to design a good network communication module is very important in MQ. It will determine the overall messaging capability and final performance of the RocketMQ cluster.\n\nrocketmq-remoting module is the module responsible for network communication in RocketMQ message queue. It is relied on and referenced by almost all other modules (such as rocketmq-client,rocketmq-broker,rocketmq-namesrv) that need network communication. In order to realize the efficient data request and reception between the client and the server, the RocketMQ message queue defines the communication protocol and extends the communication module on the basis of Netty.\n\n### 1 Remoting Communication Class Structure\n![](https://github.com/apache/rocketmq/raw/develop/docs/cn/image/rocketmq_design_3.png)\n### 2 Protocol Design and Code\nWhen a message is sent between Client and Server, a protocol convention is needed for the message sent, so it is necessary to customize the message protocol of RocketMQ. At the same time, in order to efficiently transmit messages and read the received messages, it is necessary to encode and decode the messages. In RocketMQ, the RemotingCommand class encapsulates all data content in the process of message transmission, which includes not only all data structures, but also encoding and decoding operations.\n\nHeader field | Type | Request desc | Response desc\n--- | --- | --- | --- |\ncode |int | Request  code. answering business processing is different according to different requests code | Response code. 0 means success, and non-zero means errors.\nlanguage | LanguageCode | Language implemented by the requester | Language implemented by the responder\nversion | int | Version of Request Equation | Version of Response Equation\nopaque | int |Equivalent to requestId, the different request identification codes on the same connection correspond to those in the response message| The response returns directly without modification\nflag | int | Sign, used to distinguish between ordinary RPC or oneway RPC | Sign, used to distinguish between ordinary RPC or oneway RPC\nremark | String | Transfer custom text information | Transfer custom text information \nextFields | HashMap<String, String> | Request custom extension information| Response custom extension information\n![](https://github.com/apache/rocketmq/raw/develop/docs/cn/image/rocketmq_design_4.png)\nFrom the above figure, the transport content can be divided into four parts: \n\n (1) Message length: total length, four bytes of storage, occupying an int type; \n \n(2) Serialization type header length: occupying an int type. The first byte represents the serialization type, and the last three bytes represent the header length;\n\n(3) Header data: serialized header data;\n\n(4) Message body data: binary byte data content of message body;\n### 3 Message Communication Mode and Procedure\nThere are three main ways to support communication in RocketMQ message queue: synchronous (sync), asynchronous (async), one-way (oneway). The \"one-way\" communication mode is relatively simple and is generally used in sending heartbeat packets without paying attention to its Response. Here, mainly introduce the asynchronous communication flow of RocketMQ.\n![](https://github.com/apache/rocketmq/raw/develop/docs/cn/image/rocketmq_design_5.png)\n### 4 Reactor Multithread Design\nThe RPC communication of RocketMQ uses Netty component as the underlying communication library, and also follows the Reactor multithread model. At the same time, some extensions and optimizations are made on it.\n![](https://github.com/apache/rocketmq/raw/develop/docs/cn/image/rocketmq_design_6.png)\nAbove block diagram can roughly understand the Reactor multi-thread model of NettyRemotingServer in RocketMQ. A Reactor main thread (eventLoopGroupBoss, is 1 above) is responsible for listening to TCP network connection requests, establishing connections, creating SocketChannel, and registering on selector. The source code of RocketMQ automatically selects NIO and Epoll according to the type of OS. Then listen to real network data. After you get the network data, you throw it to the Worker thread pool (eventLoopGroupSelector, is the \"N\" above, the default is 3 in the source code). You need to do SSL verification, codec, idle check, network connection management before you really execute the business logic. These tasks to defaultEventExecutorGroup (that is, \"M1\" above, the default set to 8 in the source code) to do. The processing business operations are executed in the business thread pool. According to the RomotingCommand business request code, the corresponding processor is found in the processorTable local cache variable and encapsulated into the task, and then submitted to the corresponding business processor processing thread pool for execution (sendMessageExecutor,). Take sending a message, for example, the \"M2\" above. The thread pool continues to increase in several steps from entry to business logic, which is related to the complexity of each step. The more complex the thread pool is, the wider the concurrent channel is required.\nNumber of thread | Name of thread | Desc of thread\n --- | --- | --- \n1 | NettyBoss_%d | Reactor Main thread\nN | NettyServerEPOLLSelector_%d_%d | Reactor thread pool\nM1 | NettyServerCodecThread_%d | Worker thread pool\nM2 | RemotingExecutorThread_%d | business processor thread pool\n\n\n"
  },
  {
    "path": "docs/en/Design_Store.md",
    "content": "# Message Storage\n\n\n![](images/rocketmq_storage_arch.png)\n\nMessage storage is the most complicated and important part of RocketMQ. This section will describe the three aspects of RocketMQ:\n* Message storage architecture\n* PageCache and memory mapping\n* RocketMQ's two different disk flushing methods.\n\n## 1 Message Storage Architecture\n\n\nThe message storage architecture diagram consists of 3 files related to message storage: `CommitLog` file, `ConsumeQueue` file, and `IndexFile`.\n\n\n* `CommitLog`: The `CommitLog` file stores message body and metadata sent by producer, and the message content is not fixed length. The default size of one `CommitLog` file is 1G, the length of the file name is 20 digits, the left side is zero padded, and the remaining is the starting offset. For example, `00000000000000000000` represents the first file, the starting offset is 0, and the file size is 1G=1073741824, when the first `CommitLog` file is full, the second `CommitLog` file is `00000000001073741824`, the starting offset is 1073741824, and so on. The message is mainly appended to the log file sequentially. When one `CommitLog` file is full, the next will be written.\n* `ConsumeQueue`: The `ConsumeQueue` is used to improve the performance of message consumption. Since RocketMQ uses topic-based subscription mode, message consumption is specific to the topic. Traversing the commitlog file to retrieve messages of one topic is very inefficient. The consumer can find the messages to be consumed according to the `ConsumeQueue`. The `ConsumeQueue`(logic consume queue) as an index of the consuming message stores the starting physical offset `offset` in `CommitLog` of the specified topic, the message size `size` and the hash code of the message tag. The `ConsumeQueue` file can be regarded as a topic-based `CommitLog` index file, so the consumequeue folder is organized as follows: `topic/queue/file` three-layer organization structure, the specific storage path is `$HOME/store/consumequeue/{topic}/{queueId }/{fileName}`. The consumequeue file uses a fixed-length design, each entry occupies 20 bytes, which is an 8-byte commitlog physical offset, a 4-byte message length, and an 8-byte tag hashcode. One consumequeue file consists of 0.3 million entries, each entry can be randomly accessed like an array, each `ConsumeQueue` file's size is about 5.72MB.\n* `IndexFile`: The `IndexFile` provides a way to query messages by key or time interval. The path of the `IndexFile` is `$HOME/store/index/${fileName}`, the file name `fileName` is named after the timestamp when it was created. One IndexFile's size is about 400M, and it can store 2000W indexes. The underlying storage of `IndexFile` is designed to implement the `HashMap` structure in the file system, so RocketMQ's index file is a hash index.\n\n\nFrom the above architecture of the RocketMQ message storage, we can see RocketMQ uses a hybrid storage structure, that is, all the queues in an instance of the broker share a single log file `CommitLog` to store messages. RocketMQ's hybrid storage structure(messages of multiple topics are stored in one CommitLog) uses a separate storage structure for the data and index parts for Producer and Consumer respectively. The Producer sends the message to the Broker, then the Broker persists the message to the CommitLog file synchronously or asynchronously. As long as the message is persisted to the CommitLog on the disk, the message sent by the Producer will not be lost. Because of this, Consumer will definitely have the opportunity to consume this message. When no message can be pulled, the consumer can wait for the next pull. And the server also supports the long polling mode: if a pull request pulls no messages, the Broker can wait for 30 seconds, as long as new message arrives in this interval, it will be returned directly to the consumer. Here, RocketMQ's specific approach is using Broker's background service thread `ReputMessageService` to continuously dispatch requests and asynchronously build ConsumeQueue (Logical Queue) and IndexFile data.\n\n## 2 PageCache and Memory Map\n\nPageCache is a cache of files by the operating system to speed up the reading and writing of files. In general, the speed of sequential read and write files is almost the same as the speed of read and write memory. The main reason is that the OS uses a portion of the memory as PageCache to optimize the performance of the read and write operations. For data writing, the OS will first write to the Cache, and then the `pdflush` kernel thread asynchronously flush the data in the Cache to the physical disk. For data reading, if it can not hit the page cache when reading a file at a time, the OS will read the file from the physical disk and prefetch the data files of other neighboring blocks sequentially.\n\nIn RocketMQ, the logic consumption queue `ConsumeQueue` stores less data and is read sequentially. With the help of prefetch of the page cache mechanism, the read performance of the `ConsumeQueue` file is almost close to the memory read, even in the case of message accumulation, it does not affect performance. But for the log data file `CommitLog`, it will generate many random access reads when reading the message content, which seriously affects the performance. If you choose the appropriate IO scheduling algorithm, such as setting the IO scheduling algorithm to \"Deadline\" (when the block storage uses SSD), the performance of random reads will also be improved.\n\n\nIn addition, RocketMQ mainly reads and writes files through `MappedByteBuffer`. `MappedByteBuffer` uses the `FileChannel` model in NIO to directly map the physical files on the disk to the memory address in user space (`Mmap` method reduces the performance overhead of traditional IO copying disk file data back and forth between the buffer in kernel space and the buffer in user space), it converts the file operation into direct memory address manipulation, which greatly improves the efficiency of reading and writing files (Because of the need to use the memory mapping mechanism, RocketMQ's file storage is fixed-length, making it easy to map the entire file to memory at a time).\n\n## 3 Message Disk Flush\n\n![](images/rocketmq_storage_flush.png)\n\n\n* synchronous flush: As shown above, the RocketMQ's Broker will return a successful `ACK` response to the Producer after the message is truly persisted to disk. Synchronous flushing is a good guarantee for the reliability of MQ messages, but it will have a big impact on performance. Generally, it is suitable for financial business applications.\n* asynchronous flush: Asynchronous flushing can take full advantage of the PageCache of the OS, as long as the message is written to the PageCache, the successful `ACK` can be returned to the Producer. The message flushing is performed by the background asynchronous thread, which reduces the read and write delay and improves the performance and throughput of the MQ.\n"
  },
  {
    "path": "docs/en/Design_Trancation.md",
    "content": "#  Transaction Message\n## 1 Transaction Message\nApache RocketMQ supports distributed transaction message from version 4.3.0. RocketMQ implements transaction message by using the protocol of 2PC(two-phase commit), in addition adding a compensation logic to handle timeout-case or failure-case of commit-phase, as shown below.\n\n![](../cn/image/rocketmq_design_10.png)\n\n### 1.1 The Process of RocketMQ Transaction Message\nThe picture above shows the overall architecture of transaction message, including the sending of message(commit-request phase), the sending of commit/rollback(commit phase) and the compensation process.\n\n1 The sending of message and Commit/Rollback.  \n  (1) Sending the message(named Half message in RocketMQ)  \n  (2) The server responds the writing result(success or failure) of Half message.  \n  (3) Handle local transaction according to the result(local transaction won't be executed when the result is failure).  \n  (4) Sending Commit/Rollback to broker according to the result of local transaction(Commit will generate message index and make the message visible to consumers).\n\n2 Compensation process  \n  (1) For a transaction message without a Commit/Rollback (means the message in the pending status), a \"back-check\" request is initiated from the broker.  \n  (2) The Producer receives the \"back-check\" request and checks the status of the local transaction corresponding to the \"back-check\" message.  \n  (3) Redo Commit or Rollback based on local transaction status.\nThe compensation phase is used to resolve the timeout or failure case of the message Commit or Rollback.\n\n### 1.2 The design of RocketMQ Transaction Message\n1 Transaction message is invisible to users in first phase(commit-request phase)   \n  \n  In the main process of transaction message, the message of first phase is invisible to the user. This is also the biggest difference from normal message. So how do we write the message while making it invisible to the user? And below is the solution of RocketMQ: if the message is a Half message, the topic and queueId of the original message will be backed up, and then changes the topic to RMQ_SYS_TRANS_HALF_TOPIC. Since the consumer group does not subscribe to the topic, the consumer cannot consume the Half message. Then RocketMQ starts a timing task, pulls the message for RMQ_SYS_TRANS_HALF_TOPIC, obtains a channel according to producer group and sends a back-check to query local transaction status, and decide whether to submit or roll back the message according to the status.  \n  \n  In RocketMQ, the storage structure of the message in the broker is as follows. Each message has corresponding index information. The Consumer reads the content of the message through the secondary index of the ConsumeQueue. The flow is as follows:\n\n![](../cn/image/rocketmq_design_11.png)\n\n  The specific implementation strategy of RocketMQ is: if the transaction message is written, topic and queueId of the message are replaced, and the original topic and queueId are stored in the properties of the message. Because the replace of the topic, the message will not be forwarded to the Consumer Queue of the original topic, and the consumer cannot perceive the existence of the message and will not consume it. In fact, changing the topic is the conventional method of RocketMQ(just recall the implementation mechanism of the delay message).\n\n2 Commit/Rollback operation and introduction of Op message  \n  \n  After finishing writing a message that is invisible to the user in the first phase, here comes two cases in the second phase. One is Commit operation, after which the message needs to be visible to the user; the other one is Rollback operation, after which the first phase message(Half message) needs to be revoked. For the case of Rollback, since first-phase message itself is invisible to the user, there is no need to actually revoke the message (in fact, RocketMQ can't actually delete a message because it is a sequential-write file). But still some operation needs to be done to identity the final status of the message, to differ it from pending status message. To do this, the concept of \"Op message\" is introduced, which means the message has a certain status(Commit or Rollback). If a transaction message does not have a corresponding Op message, the status of the transaction is still undetermined (probably the second-phase failed). By introducing the Op message, the RocketMQ records an Op message for every Half message regardless it is Commit or Rollback. The only difference between Commit and Rollback is that when it comes to Commit, the index of the Half message is created before the Op message is written.\n\n3 How Op message stored and the correspondence between Op message and Half message  \n  \n  RocketMQ writes the Op message to a specific system topic(RMQ_SYS_TRANS_OP_HALF_TOPIC) which will be created via the method - TransactionalMessageUtil.buildOpTopic(); this topic is an internal Topic (like the topic of RMQ_SYS_TRANS_HALF_TOPIC) and will not be consumed by the user. The content of the Op message is the physical offset of the corresponding Half message. Through the Op message we can index to the Half message for subsequent check-back operation.\n\n![](../cn/image/rocketmq_design_12.png)\n\n4 Index construction of Half messages  \n  \n  When performing Commit operation of the second phase, the index of the Half message needs to be built. Since the Half message is written to a special topic(RMQ_SYS_TRANS_HALF_TOPIC) in the first phase of 2PC, so it needs to be read out from the special topic when building index, and replace the topic and queueId with the real target topic and queueId, and then write through a normal message that is visible to the user. Therefore, in conclusion, the second phase recovers a complete normal message using the content of the Half message stored in the first phase, and then goes through the message-writing process.\n\n5 How to handle the message failed in the second phase？\n  \n  If commit/rollback phase fails, for example, a network problem causes the Commit to fail when you do Commit. Then certain strategy is required to make sure the message finally commit. RocketMQ uses a compensation mechanism called \"back-check\". The broker initiates a back-check request for the message in pending status, and sends the request to the corresponding producer side (the same producer group as the producer group who sent the Half message). The producer checks the status of local transaction and redo Commit or Rollback. The broker performs the back-check by comparing the RMQ_SYS_TRANS_HALF_TOPIC messages and the RMQ_SYS_TRANS_OP_HALF_TOPIC messages and advances the checkpoint(recording those transaction messages that the status are certain).\n\n  RocketMQ does not back-check the status of transaction messages endlessly. The default time is 15. If the transaction status is still unknown after 15 times, RocketMQ will roll back the message by default.\n"
  },
  {
    "path": "docs/en/Example_Batch.md",
    "content": "# Batch Message Sample\n------\nSending messages in batch improves performance of delivering small messages. Messages of the same batch should have: same topic, same waitStoreMsgOK and no schedule support. You can send messages up to 4MiB at a time, but if you need to send a larger message, it is recommended to divide the larger messages into multiple small messages of no more than 1MiB.\n### 1 Send Batch Messages\nIf you just send messages of no more than 4MiB at a time, it is easy to use batch:\n```java\nString topic = \"BatchTest\";\nList<Message> messages = new ArrayList<>();\nmessages.add(new Message(topic, \"TagA\", \"OrderID001\", \"Hello world 0\".getBytes()));\nmessages.add(new Message(topic, \"TagA\", \"OrderID002\", \"Hello world 1\".getBytes()));\nmessages.add(new Message(topic, \"TagA\", \"OrderID003\", \"Hello world 2\".getBytes()));\ntry {\n    producer.send(messages);\n} catch (Exception e) {\n    e.printStackTrace();\n    //handle the error\n}\n```\n### 2 Split into Lists\nThe complexity only grow when you send large batch and you may not sure if it exceeds the size limit (4MiB). At this time, you’d better split the lists:\n```java\npublic class ListSplitter implements Iterator<List<Message>> { \n    private final int SIZE_LIMIT = 1024 * 1024 * 4;\n    private final List<Message> messages;\n    private int currIndex;\n    public ListSplitter(List<Message> messages) { \n        this.messages = messages;\n    }\n    @Override \n    public boolean hasNext() {\n        return currIndex < messages.size(); \n    }\n    @Override \n    public List<Message> next() { \n        int startIndex = getStartIndex();\n        int nextIndex = startIndex;\n        int totalSize = 0;\n        for (; nextIndex < messages.size(); nextIndex++) {\n            Message message = messages.get(nextIndex); \n            int tmpSize = calcMessageSize(message);\n            if (tmpSize + totalSize > SIZE_LIMIT) {\n                break; \n            } else {\n                totalSize += tmpSize; \n            }\n        }\n        List<Message> subList = messages.subList(startIndex, nextIndex); \n        currIndex = nextIndex;\n        return subList;\n    }\n    private int getStartIndex() {\n        Message currMessage = messages.get(currIndex); \n        int tmpSize = calcMessageSize(currMessage); \n        while(tmpSize > SIZE_LIMIT) {\n            currIndex += 1;\n            Message message = messages.get(curIndex); \n            tmpSize = calcMessageSize(message);\n        }\n        return currIndex; \n    }\n    private int calcMessageSize(Message message) {\n        int tmpSize = message.getTopic().length() + message.getBody().length; \n        Map<String, String> properties = message.getProperties();\n        for (Map.Entry<String, String> entry : properties.entrySet()) {\n            tmpSize += entry.getKey().length() + entry.getValue().length(); \n        }\n        tmpSize = tmpSize + 20; // Increase the log overhead by 20 bytes\n        return tmpSize; \n    }\n}\n\n// then you could split the large list into small ones:\nListSplitter splitter = new ListSplitter(messages);\nwhile (splitter.hasNext()) {\n   try {\n       List<Message>  listItem = splitter.next();\n       producer.send(listItem);\n   } catch (Exception e) {\n       e.printStackTrace();\n       // handle the error\n   }\n}\n```\n"
  },
  {
    "path": "docs/en/Example_Compaction_Topic.md",
    "content": "# Compaction Topic\n\n## use example\n\n### Turn on the opening of support for orderMessages on namesrv\nCompactionTopic relies on orderMessages to ensure consistency\n```shell\n$ bin/mqadmin updateNamesrvConfig -k orderMessageEnable -v true\n```\n\n### create compaction topic\n```shell\n$ bin/mqadmin updateTopic -w 8 -r 8 -a +cleanup.policy=COMPACTION -n localhost:9876 -t ctopic -o true -c DefaultCluster\ncreate topic to 127.0.0.1:10911 success.\nTopicConfig [topicName=ctopic, readQueueNums=8, writeQueueNums=8, perm=RW-, topicFilterType=SINGLE_TAG, topicSysFlag=0, order=false, attributes={+cleanup.policy=COMPACTION}]\n```\n\n### produce message\nthe same with ordinary message\n```java\nDefaultMQProducer producer = new DefaultMQProducer(\"CompactionTestGroup\");\nproducer.setNamesrvAddr(\"localhost:9876\");\nproducer.start();\n\nString topic = \"ctopic\";\nString tag = \"tag1\";\nString key = \"key1\";\nMessage msg = new Message(topic, tag, key, \"bodys\".getBytes(StandardCharsets.UTF_8));\nSendResult sendResult = producer.send(msg, (mqs, message, shardingKey) -> {\n    int select = Math.abs(shardingKey.hashCode());\n    if (select < 0) {\n        select = 0;\n    }\n    return mqs.get(select % mqs.size());\n}, key);\n\nSystem.out.printf(\"%s%n\", sendResult);\n```\n### consume message\nthe message offset remains unchanged after compaction. If the consumer specified offset does not exist, return the most recent message after the offset.\n\nIn the compaction scenario, most consumption was started from the beginning of the queue.\n```java\nDefaultLitePullConsumer consumer = new DefaultLitePullConsumer(\"compactionTestGroup\");\nconsumer.setNamesrvAddr(\"localhost:9876\");\nconsumer.setPullThreadNums(4);\nconsumer.start();\n\nCollection<MessageQueue> messageQueueList = consumer.fetchMessageQueues(\"ctopic\");\nconsumer.assign(messageQueueList);\nmessageQueueList.forEach(mq -> {\n    try {\n        consumer.seekToBegin(mq);\n    } catch (MQClientException e) {\n        e.printStackTrace();\n    }\n});\n\nMap<String, byte[]> kvStore = Maps.newHashMap();\nwhile (true) {\n    List<MessageExt> msgList = consumer.poll(1000);\n    if (CollectionUtils.isNotEmpty(msgList)) {\n        msgList.forEach(msg -> kvStore.put(msg.getKeys(), msg.getBody()));\n    }\n}\n\n//use the kvStore\n```\n"
  },
  {
    "path": "docs/en/Example_CreateTopic.md",
    "content": "# Create Topic\n\n## Background\n\nThe `TopicMessageType` concept is introduced in RocketMQ 5.0, using the existing topic attribute feature to implement it.\n\nThe topic is created by `mqadmin` tool declaring the `message.type` attribute.\n\n## User Example\n\n```shell\n# default\nsh ./mqadmin updateTopic -n <nameserver_address> -t <topic_name> -c DefaultCluster\n\n# normal topic\nsh ./mqadmin updateTopic -n <nameserver_address> -t <topic_name> -c DefaultCluster -a +message.type=NORMAL\n\n# fifo topic\nsh ./mqadmin updateTopic -n <nameserver_address> -t <topic_name> -c DefaultCluster -a +message.type=FIFO\n\n# delay topic\nsh ./mqadmin updateTopic -n <nameserver_address> -t <topic_name> -c DefaultCluster -a +message.type=DELAY\n\n# transaction topic\nsh ./mqadmin updateTopic -n <nameserver_address> -t <topic_name> -c DefaultCluster -a +message.type=TRANSACTION\n```\n"
  },
  {
    "path": "docs/en/Example_Delay.md",
    "content": "# Schedule example\n\n### 1 Start consumer to wait for incoming subscribed messages \n\n```java\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport java.util.List;\n\npublic class ScheduledMessageConsumer {\n\n    public static void main(String[] args) throws Exception {\n        // Instantiate message consumer\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"ExampleConsumer\");\n        // Specify name server addresses\n        consumer.setNamesrvAddr(\"localhost:9876\");\n        // Subscribe topics\n        consumer.subscribe(\"TestTopic\", \"*\");\n        // Register message listener\n        consumer.registerMessageListener(new MessageListenerConcurrently() {\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> messages, ConsumeConcurrentlyContext context) {\n                for (MessageExt message : messages) {\n                    // Print approximate delay time period\n                    System.out.println(\"Receive message[msgId=\" + message.getMsgId() + \"] \"\n                                       + (System.currentTimeMillis() - message.getStoreTimestamp()) + \"ms later\");\n                }\n                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n            }\n        });\n        // Launch consumer\n        consumer.start();\n    }\n}\n```\n\n### 2 Send scheduled messages \n\n```java\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.common.message.Message;\n\npublic class ScheduledMessageProducer {\n\n    public static void main(String[] args) throws Exception {\n        // Instantiate a producer to send scheduled messages\n        DefaultMQProducer producer = new DefaultMQProducer(\"ExampleProducerGroup\");\n        // Specify name server addresses\n        producer.setNamesrvAddr(\"localhost:9876\");\n        // Launch producer\n        producer.start();\n        int totalMessagesToSend = 100;\n        for (int i = 0; i < totalMessagesToSend; i++) {\n            Message message = new Message(\"TestTopic\", (\"Hello scheduled message \" + i).getBytes());\n            // This message will be delivered to consumer 10 seconds later.\n            message.setDelayTimeLevel(3);\n            // Send the message\n            producer.send(message);\n        }\n\n        // Shutdown producer after use.\n        producer.shutdown();\n    }\n\n}\n```\n\n### 3 Verification \n\nYou should see messages are consumed about 10 seconds later than their storing time. \n\n### 4 Use scenarios for scheduled messages\n\nFor example, in e-commerce, if an order is submitted, a delay message can be sent, and the status of the order can be checked after 1 hour. If the order is still unpaid, the order can be cancelled and the inventory released.\n\n### 5 Restrictions on the use of scheduled messages\n\n```java \n// org/apache/rocketmq/store/config/MessageStoreConfig.java\n\nprivate String messageDelayLevel = \"1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h\";\n```\n\nNowadays RocketMq does not support any time delay. It needs to set several fixed delay levels, which correspond to level 1 to 18 from 1s to 2h. Message consumption failure will enter the delay message queue. Message sending time is related to the set delay level and the number of retries.\n\n See `SendMessageProcessor.java` \n"
  },
  {
    "path": "docs/en/Example_Filter.md",
    "content": "# Filter Example\n----------\n\nIn most cases, tag is a simple and useful design to select messages you want. For example:\n\n```java\nDefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"CID_EXAMPLE\");\nconsumer.subscribe(\"TOPIC\", \"TAGA || TAGB || TAGC\");\n```\n\nThe consumer will receive messages that contains TAGA or TAGB or TAGC. But the limitation is that one message only can have one tag, and this may not work for sophisticated scenarios. In this case, you can use SQL expression to filter out messages.\nSQL feature could do some calculation through the properties you put in when sending messages. Under the grammars defined by RocketMQ, you can implement some interesting logic. Here is an example:\n\n```\n------------\n| message  |\n|----------|  a > 5 AND b = 'abc'\n| a = 10   |  --------------------> Gotten\n| b = 'abc'|\n| c = true |\n------------\n------------\n| message  |\n|----------|   a > 5 AND b = 'abc'\n| a = 1    |  --------------------> Missed\n| b = 'abc'|\n| c = true |\n------------\n```\n\n## 1 Grammars\nRocketMQ only defines some basic grammars to support this feature. You could also extend it easily.\n\n- Numeric comparison, like **>**, **>=**, **<**, **<=**, **BETWEEN**, **=**;\n- Character comparison, like **=**, **<>**, **IN**;\n- **IS NULL** or **IS NOT NULL**;\n- Logical **AND**, **OR**, **NOT**;\n\nConstant types are:\n\n- Numeric, like **123, 3.1415**;\n- Character, like **‘abc’**, must be made with single quotes;\n- **NULL**, special constant;\n- Boolean, **TRUE** or **FALSE**;\n\n## 2 Usage constraints\nOnly push consumer could select messages by SQL92. The interface is:\n```\npublic void subscribe(finalString topic, final MessageSelector messageSelector)\n```\n\n## 3 Producer example\nYou can put properties in message through method putUserProperty when sending.\n\n```java\nDefaultMQProducer producer = new DefaultMQProducer(\"please_rename_unique_group_name\");\nproducer.start();\nMessage msg = new Message(\"TopicTest\",\n   tag,\n   (\"Hello RocketMQ \" + i).getBytes(RemotingHelper.DEFAULT_CHARSET)\n);\n// Set some properties.\nmsg.putUserProperty(\"a\", String.valueOf(i));\nSendResult sendResult = producer.send(msg);\n\nproducer.shutdown();\n\n```\n\n## 4 Consumer example\nUse `MessageSelector.bySql` to select messages through SQL when consuming.\n\n\n```java\nDefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"please_rename_unique_group_name_4\");\n// only subscribe messages have property a, also a >=0 and a <= 3\nconsumer.subscribe(\"TopicTest\", MessageSelector.bySql(\"a between 0 and 3\"));\nconsumer.registerMessageListener(new MessageListenerConcurrently() {\n   @Override\n   public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {\n       return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n   }\n});\nconsumer.start();\n\n```\n"
  },
  {
    "path": "docs/en/Example_OpenMessaging.md",
    "content": "# OpenMessaging Example\n[OpenMessaging](https://openmessaging.github.io/), which includes the establishment of industry guidelines and messaging, streaming specifications to provide a common framework for finance, e-commerce, IoT and big-data area. The design principles are the cloud-oriented, simplicity, flexibility, and language independent in distributed heterogeneous environments. Conformance to these specifications will make it possible to develop a heterogeneous messaging applications across all major platforms and operating systems.\n\nRocketMQ provides a partial implementation of OpenMessaging 0.1.0-alpha, the following examples demonstrate how to access RocketMQ based on OpenMessaging.\n\n## OMSProducer\nThe following example shows how to send message to RocketMQ broker in synchronous, asynchronous, or one-way transmissions.\n\n```\npublic class OMSProducer {\n    public static void main(String[] args) {\n        final MessagingAccessPoint messagingAccessPoint = MessagingAccessPointFactory\n            .getMessagingAccessPoint(\"openmessaging:rocketmq://IP1:9876,IP2:9876/namespace\");\n\n        final Producer producer = messagingAccessPoint.createProducer();\n\n        messagingAccessPoint.startup();\n        System.out.printf(\"MessagingAccessPoint startup OK%n\");\n\n        producer.startup();\n        System.out.printf(\"Producer startup OK%n\");\n\n        {\n            Message message = producer.createBytesMessageToTopic(\"OMS_HELLO_TOPIC\", \"OMS_HELLO_BODY\".getBytes(Charset.forName(\"UTF-8\")));\n            SendResult sendResult = producer.send(message);\n            System.out.printf(\"Send sync message OK, msgId: %s%n\", sendResult.messageId());\n        }\n\n        {\n            final Promise<SendResult> result = producer.sendAsync(producer.createBytesMessageToTopic(\"OMS_HELLO_TOPIC\", \"OMS_HELLO_BODY\".getBytes(Charset.forName(\"UTF-8\"))));\n            result.addListener(new PromiseListener<SendResult>() {\n                @Override\n                public void operationCompleted(Promise<SendResult> promise) {\n                    System.out.printf(\"Send async message OK, msgId: %s%n\", promise.get().messageId());\n                }\n\n                @Override\n                public void operationFailed(Promise<SendResult> promise) {\n                    System.out.printf(\"Send async message Failed, error: %s%n\", promise.getThrowable().getMessage());\n                }\n            });\n        }\n\n        {\n            producer.sendOneway(producer.createBytesMessageToTopic(\"OMS_HELLO_TOPIC\", \"OMS_HELLO_BODY\".getBytes(Charset.forName(\"UTF-8\"))));\n            System.out.printf(\"Send oneway message OK%n\");\n        }\n\n        producer.shutdown();\n        messagingAccessPoint.shutdown();\n    }\n}\n```\n## OMSPullConsumer\nUse OMS PullConsumer to poll messages from a specified queue.\n\n```\npublic class OMSPullConsumer {\n    public static void main(String[] args) {\n        final MessagingAccessPoint messagingAccessPoint = MessagingAccessPointFactory\n            .getMessagingAccessPoint(\"openmessaging:rocketmq://IP1:9876,IP2:9876/namespace\");\n\n        final PullConsumer consumer = messagingAccessPoint.createPullConsumer(\"OMS_HELLO_TOPIC\",\n            OMS.newKeyValue().put(NonStandardKeys.CONSUMER_GROUP, \"OMS_CONSUMER\"));\n\n        messagingAccessPoint.startup();\n        System.out.printf(\"MessagingAccessPoint startup OK%n\");\n        \n        consumer.startup();\n        System.out.printf(\"Consumer startup OK%n\");\n\n        Message message = consumer.poll();\n        if (message != null) {\n            String msgId = message.headers().getString(MessageHeader.MESSAGE_ID);\n            System.out.printf(\"Received one message: %s%n\", msgId);\n            consumer.ack(msgId);\n        }\n\n        consumer.shutdown();\n        messagingAccessPoint.shutdown();\n    }\n}\n\n```\n## OMSPushConsumer\nAttaches OMS PushConsumer to a specified queue and consumes messages by MessageListener\n\n```\npublic class OMSPushConsumer {\n    public static void main(String[] args) {\n        final MessagingAccessPoint messagingAccessPoint = MessagingAccessPointFactory\n            .getMessagingAccessPoint(\"openmessaging:rocketmq://IP1:9876,IP2:9876/namespace\");\n\n        final PushConsumer consumer = messagingAccessPoint.\n            createPushConsumer(OMS.newKeyValue().put(NonStandardKeys.CONSUMER_GROUP, \"OMS_CONSUMER\"));\n\n        messagingAccessPoint.startup();\n        System.out.printf(\"MessagingAccessPoint startup OK%n\");\n\n        Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {\n            @Override\n            public void run() {\n                consumer.shutdown();\n                messagingAccessPoint.shutdown();\n            }\n        }));\n        \n        consumer.attachQueue(\"OMS_HELLO_TOPIC\", new MessageListener() {\n            @Override\n            public void onMessage(final Message message, final ReceivedMessageContext context) {\n                System.out.printf(\"Received one message: %s%n\", message.headers().getString(MessageHeader.MESSAGE_ID));\n                context.ack();\n            }\n        });\n        \n    }\n}\n```\n"
  },
  {
    "path": "docs/en/Example_Orderly.md",
    "content": "# Example for Ordered Messages\n\nRocketMQ provides ordered messages using FIFO order. All related messages need to be sent into the same message queue in an orderly manner.\n\nThe following demonstrates ordered messages by ensuring order of create, pay, send and finish steps of sales order process.\n\n## 1 produce ordered messages\n\n```java\npackage org.apache.rocketmq.example.order2\n\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.MessageQueueSelector;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\nimport java.text.SimpleDateFormat;\nimport java.util.ArrayList;\nimport java.util.Date;\nimport java.util.List;\n\n/*\n* ordered messages producer\n*/\npublic class Producer {\n\n    public static void main(String[] args) throws Exception {\n        DefaultMQProducer producer = new DefaultMQProducer(\"please_rename_unique_group_name\");\n        producer.setNamesrvAddr(\"127.0.0.1:9876\");\n        producer.start();\n        String[] tags = new String[]{\"TagA\", \"TagC\", \"TagD\"};\n        // sales orders list\n        List<OrderStep> orderList = new Producer().buildOrders();\n\n        Date date = new Date();\n        SimpleDateFormat sdf = new SimpleDateFormat(\"yyyy-MM-dd HH:mm:ss\");\n        String dateStr = sdf.format(date);\n\n        for (int i = 0; i < 10; i++) {\n            // generate message timestamp\n            String body = dateStr + \" Hello RocketMQ \" + orderList.get(i);\n            Message msg = new Message(\"TopicTest\", tags[i % tags.length], \"KEY\" + i, body.getBytes());\n\n            SendResult sendResult = producer.send(msg, new MessageQueueSelector() {\n                @Override\n                public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {\n                    Long id = (Long) arg;  //message queue is selected by #salesOrderID\n                    long index = id % mqs.size();\n                    return mqs.get((int) index);\n                }\n            }, orderList.get(i).getOrderId());\n\n            System.out.println(String.format(\"SendResult status:%s, queueId:%d, body:%s\",\n                    sendResult.getSendStatus(),\n                    sendResult.getMessageQueue().getQueueId(),\n                    body));\n        }\n\n        producer.shutdown();\n    }\n\n    /**\n     * each sales order step\n     */\n    private static class OrderStep {\n        private long orderId;\n        private String desc;\n\n        public long getOrderId() {\n            return orderId;\n        }\n\n        public void setOrderId(long orderId) {\n            this.orderId = orderId;\n        }\n\n        public String getDesc() {\n            return desc;\n        }\n\n        public void setDesc(String desc) {\n            this.desc = desc;\n        }\n\n        @Override\n        public String toString() {\n            return \"OrderStep{\" +\n                    \"orderId=\" + orderId +\n                    \", desc='\" + desc + '\\'' +\n                    '}';\n        }\n    }\n\n    /**\n     * to generate ten OrderStep objects for three sales orders:\n     *  #SalesOrder \"15103111039L\": create, pay, send, finish;\n     *  #SalesOrder \"15103111065L\": create, pay, finish;\n     *  #SalesOrder \"15103117235L\": create, pay, finish;\n     */    \n\tprivate List<OrderStep> buildOrders() {\n\t\t\n        List<OrderStep> orderList = new ArrayList<OrderStep>();\n\n        //create sales order with orderid=\"15103111039L\"\n        OrderStep orderDemo = new OrderStep();\n        orderDemo.setOrderId(15103111039L);\n        orderDemo.setDesc(\"create\");\n        orderList.add(orderDemo);\n\n        //create sales order with orderid=\"15103111065L\"\n        orderDemo = new OrderStep();\n        orderDemo.setOrderId(15103111065L);\n        orderDemo.setDesc(\"create\");\n        orderList.add(orderDemo);\n\n        //pay sales order #\"15103111039L\"\n        orderDemo = new OrderStep();\n        orderDemo.setOrderId(15103111039L);\n        orderDemo.setDesc(\"pay\");\n        orderList.add(orderDemo);\n\n        //create sales order with orderid=\"15103117235L\"\n        orderDemo = new OrderStep();\n        orderDemo.setOrderId(15103117235L);\n        orderDemo.setDesc(\"create\");\n        orderList.add(orderDemo);\n\n        //pay sales order #\"15103111065L\"\n        orderDemo = new OrderStep();\n        orderDemo.setOrderId(15103111065L);\n        orderDemo.setDesc(\"pay\");\n        orderList.add(orderDemo);\n\n        //pay sales order #\"15103117235L\"\n        orderDemo = new OrderStep();\n        orderDemo.setOrderId(15103117235L);\n        orderDemo.setDesc(\"pay\");\n        orderList.add(orderDemo);\n\n        //mark sales order #\"15103111065L\" as \"finish\"\n        orderDemo = new OrderStep();\n        orderDemo.setOrderId(15103111065L);\n        orderDemo.setDesc(\"finish\");\n        orderList.add(orderDemo);\n\n        //mark mark sales order #\"15103111039L\" as \"send\"\n        orderDemo = new OrderStep();\n        orderDemo.setOrderId(15103111039L);\n        orderDemo.setDesc(\"send\");\n        orderList.add(orderDemo);\n\n        ////mark sales order #\"15103117235L\" as \"finish\"\n        orderDemo = new OrderStep();\n        orderDemo.setOrderId(15103117235L);\n        orderDemo.setDesc(\"finish\");\n        orderList.add(orderDemo);\n\n        //mark sales order #\"15103111039L\" as \"finish\"\n        orderDemo = new OrderStep();\n        orderDemo.setOrderId(15103111039L);\n        orderDemo.setDesc(\"finish\");\n        orderList.add(orderDemo);\n\n        return orderList;\n    }\n}\n\n```\n\n## 2 Consume ordered messages\n\n```java\n\npackage org.apache.rocketmq.example.order2;\n\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.MessageExt;\n\nimport java.util.List;\nimport java.util.Random;\nimport java.util.concurrent.TimeUnit;\n\n/**\n * consume messages in order\n */\npublic class ConsumerInOrder {\n\n    public static void main(String[] args) throws Exception {\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"please_rename_unique_group_name_3\");\n        consumer.setNamesrvAddr(\"127.0.0.1:9876\");\n        /**\n         * when the consumer is first run, the start point of message queue where it can get messages will be set.\n         * or if it is restarted, it will continue from the last place to get messages.\n         */\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n\n        consumer.subscribe(\"TopicTest\", \"TagA || TagC || TagD\");\n\n        consumer.registerMessageListener(new MessageListenerOrderly() {\n\n            Random random = new Random();\n\n            @Override\n            public ConsumeOrderlyStatus consumeMessage(List<MessageExt> msgs, ConsumeOrderlyContext context) {\n                context.setAutoCommit(true);\n                for (MessageExt msg : msgs) {\n                    // one consumer for each message queue, and messages order are kept in a single message queue.\n                    System.out.println(\"consumeThread=\" + Thread.currentThread().getName() + \"queueId=\" + msg.getQueueId() + \", content:\" + new String(msg.getBody()));\n                }\n\n                try {\n                    TimeUnit.SECONDS.sleep(random.nextInt(10));\n                } catch (Exception e) {\n                    e.printStackTrace();\n                }\n                return ConsumeOrderlyStatus.SUCCESS;\n            }\n        });\n\n        consumer.start();\n\n        System.out.println(\"Consumer Started.\");\n    }\n}\n\n```\n\n   \n"
  },
  {
    "path": "docs/en/Example_Simple.md",
    "content": "# Basic Sample \n------\nTwo functions below are provided in the basic sample:\n* The RocketMQ can be utilized to send messages in three ways: reliable synchronous, reliable asynchronous, and one-way transmission.  The first two message types are reliable because there is a response whether they were sent successfully.\n* The RocketMQ can be utilized to consume messages.\n### 1 Add Dependency\nmaven:\n``` java\n<dependency>\n  <groupId>org.apache.rocketmq</groupId>\n  <artifactId>rocketmq-client</artifactId>\n  <version>4.3.0</version>\n</dependency>\n```\ngradle: \n``` java \ncompile 'org.apache.rocketmq:rocketmq-client:4.3.0'\n```\n### 2 Send Messages\n##### 2.1 Use Producer to Send Synchronous Messages\nReliable synchronous transmission is used in extensive scenes, such as important notification messages, SMS notification.\n``` java\npublic class SyncProducer {\n  public static void main(String[] args) throws Exception {\n    // Instantiate with a producer group name\n    DefaultMQProducer producer = new DefaultMQProducer(\"please_rename_unique_group_name\");\n    // Specify name server addresses\n    producer.setNamesrvAddr(\"localhost:9876\");\n    // Launch the producer instance\n    producer.start();\n    for (int i = 0; i < 100; i++) {\n      // Create a message instance with specifying topic, tag and message body\n      Message msg = new Message(\"TopicTest\" /* Topic */,\n        \"TagA\" /* Tag */,\n        (\"Hello RocketMQ \" + i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */\n        );\n      // Send message to one of brokers\n      SendResult sendResult = producer.send(msg);\n      // Check whether the message has been delivered by the callback of sendResult\n      System.out.printf(\"%s%n\", sendResult);\n    }\n    // Shut down once the producer instance is not longer in use\n    producer.shutdown();\n  }\n}\n```\n##### 2.2 Send Asynchronous Messages\nAsynchronous transmission is generally used in response time sensitive business scenarios. It means that it is unable for the sender to wait the response of the Broker too long.\n``` java\npublic class AsyncProducer {\n  public static void main(String[] args) throws Exception {\n    // Instantiate with a producer group name\n    DefaultMQProducer producer = new DefaultMQProducer(\"please_rename_unique_group_name\");\n    // Specify name server addresses\n    producer.setNamesrvAddr(\"localhost:9876\");\n    // Launch the producer instance\n    producer.start();\n    producer.setRetryTimesWhenSendAsyncFailed(0);\n    for (int i = 0; i < 100; i++) {\n      final int index = i;\n      // Create a message instance with specifying topic, tag and message body\n      Message msg = new Message(\"TopicTest\",\n        \"TagA\",\n        \"OrderID188\",\n        \"Hello world\".getBytes(RemotingHelper.DEFAULT_CHARSET));\n      // SendCallback: receive the callback of the asynchronous return result.\n      producer.send(msg, new SendCallback() {\n        @Override\n        public void onSuccess(SendResult sendResult) {\n          System.out.printf(\"%-10d OK %s %n\", index,\n            sendResult.getMsgId());\n        }\n        @Override\n        public void onException(Throwable e) {\n          System.out.printf(\"%-10d Exception %s %n\", index, e);\n          e.printStackTrace();\n        }\n      });\n    }\n    // Shut down once the producer instance is not longer in use\n    producer.shutdown();\n  }\n}\n```\n##### 2.3 Send Messages in One-way Mode\nOne-way transmission is used for cases requiring moderate reliability, such as log collection.\n``` java\npublic class OnewayProducer {\n  public static void main(String[] args) throws Exception{\n    // Instantiate with a producer group name\n    DefaultMQProducer producer = new DefaultMQProducer(\"please_rename_unique_group_name\");\n    // Specify name server addresses\n    producer.setNamesrvAddr(\"localhost:9876\");\n    // Launch the producer instance\n    producer.start();\n    for (int i = 0; i < 100; i++) {\n      // Create a message instance with specifying topic, tag and message body\n      Message msg = new Message(\"TopicTest\" /* Topic */,\n        \"TagA\" /* Tag */,\n        (\"Hello RocketMQ \" + i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */\n      );\n      // Send in one-way mode, no return result\n      producer.sendOneway(msg);\n    }\n    // Shut down once the producer instance is not longer in use\n     producer.shutdown();\n  }\n}\n```\n### 3 Consume Messages\n``` java\npublic class Consumer {\n  public static void main(String[] args) throws InterruptedException, MQClientException {\n    // Instantiate with specified consumer group name\n    DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"please_rename_unique_group_name\");\n    \n    // Specify name server addresses\n    consumer.setNamesrvAddr(\"localhost:9876\");\n\n    // Subscribe one or more topics and tags for finding those messages need to be consumed\n    consumer.subscribe(\"TopicTest\", \"*\");\n    // Register callback to execute on arrival of messages fetched from brokers\n    consumer.registerMessageListener(new MessageListenerConcurrently() {\n      @Override\n      public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {\n        System.out.printf(\"%s Receive New Messages: %s %n\", Thread.currentThread().getName(), msgs);\n        // Mark the message that have been consumed successfully\n        return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n      }\n    });\n    // Launch the consumer instance\n    consumer.start();\n    System.out.printf(\"Consumer Started.%n\");\n  }\n}\n```"
  },
  {
    "path": "docs/en/Example_Transaction.md",
    "content": "# Transaction Message Example \n\n## 1 Transaction message status \nThere are three states for transaction message:  \n- LocalTransactionState.COMMIT_MESSAGE: commit transaction, it means that allow consumers to consume this message.  \n- LocalTransactionState.ROLLBACK_MESSAGE: rollback transaction, it means that the message will be deleted and not allowed to consume.  \n- LocalTransactionState.UNKNOW: intermediate state, it means that MQ is needed to check back to determine the status.\n\n## 2 Send transactional message example\n\n### 2.1 Create the transactional producer \nUse ```TransactionMQProducer```class to create producer client, and specify a unique ```ProducerGroup```, and you can set up a custom thread pool to process check requests. After executing the local transaction, you need to reply to MQ according to the execution result, and the reply status is described in the above section.  \n```java\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport java.util.List;\npublic class TransactionProducer {\n   public static void main(String[] args) throws MQClientException, InterruptedException {\n       TransactionListener transactionListener = new TransactionListenerImpl();\n       TransactionMQProducer producer = new TransactionMQProducer(\"please_rename_unique_group_name\");\n       ExecutorService executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(2000), new ThreadFactory() {\n           @Override\n           public Thread newThread(Runnable r) {\n               Thread thread = new Thread(r);\n               thread.setName(\"client-transaction-msg-check-thread\");\n               return thread;\n           }\n       });\n       producer.setExecutorService(executorService);\n       producer.setTransactionListener(transactionListener);\n       producer.start();\n       String[] tags = new String[] {\"TagA\", \"TagB\", \"TagC\", \"TagD\", \"TagE\"};\n       for (int i = 0; i < 10; i++) {\n           try {\n               Message msg =\n                   new Message(\"TopicTest1234\", tags[i % tags.length], \"KEY\" + i,\n                       (\"Hello RocketMQ \" + i).getBytes(RemotingHelper.DEFAULT_CHARSET));\n               SendResult sendResult = producer.sendMessageInTransaction(msg, null);\n               System.out.printf(\"%s%n\", sendResult);\n               Thread.sleep(10);\n           } catch (MQClientException | UnsupportedEncodingException e) {\n               e.printStackTrace();\n           }\n       }\n       for (int i = 0; i < 100000; i++) {\n           Thread.sleep(1000);\n       }\n       producer.shutdown();\n   }\n}\n```\n\n### 2.2 Implement the TransactionListener interface\nThe ```executeLocalTransaction``` method is used to execute local transaction when send half message succeed. It returns one of three transaction status mentioned in the previous section.\n\nThe ```checkLocalTransaction``` method is used to check the local transaction status and respond to MQ check requests. It also returns one of three transaction status mentioned in the previous section. \n```java\npublic class TransactionListenerImpl implements TransactionListener {\n  private AtomicInteger transactionIndex = new AtomicInteger(0);\n  private ConcurrentHashMap<String, Integer> localTrans = new ConcurrentHashMap<>();\n  @Override\n  public LocalTransactionState executeLocalTransaction(Message msg, Object arg) {\n      int value = transactionIndex.getAndIncrement();\n      int status = value % 3;\n      localTrans.put(msg.getTransactionId(), status);\n      return LocalTransactionState.UNKNOW;\n  }\n  @Override\n  public LocalTransactionState checkLocalTransaction(MessageExt msg) {\n      Integer status = localTrans.get(msg.getTransactionId());\n      if (null != status) {\n          switch (status) {\n              case 0:\n                  return LocalTransactionState.UNKNOW;\n              case 1:\n                  return LocalTransactionState.COMMIT_MESSAGE;\n              case 2:\n                  return LocalTransactionState.ROLLBACK_MESSAGE;\n          }\n      }\n      return LocalTransactionState.COMMIT_MESSAGE;\n  }\n}\n```\n\n## 3 Usage Constraint  \n1. Messages of the transactional have no schedule and batch support.\n2. In order to avoid a single message being checked too many times and lead to half queue message accumulation,  we limited the number of checks for a single message to 15 times by default, but users can change this limit by change the ```transactionCheckMax``` parameter in the configuration of the broker,  if one message has been checked over ```transactionCheckMax``` times,  broker will discard this message and print an error log at the same time by default. Users can change this behavior by override the ```AbstractTransactionalMessageCheckListener``` class.\n3. A transactional message will be checked after a certain period of time that determined by parameter ```transactionTimeout``` in the configuration of the broker. And users also can change this limit by set user property ```CHECK_IMMUNITY_TIME_IN_SECONDS``` when sending transactional message, this parameter takes precedence over the ```transactionTimeout``` parameter. \n4. A transactional message maybe checked or consumed more than once. \n5. Committed message reput to the user’s target topic may fail. Currently, it depends on the log record. High availability is ensured by the high availability mechanism of RocketMQ itself. If you want to ensure that the transactional message isn’t lost and the transaction integrity is guaranteed, it is recommended to use synchronous double write mechanism. \n6. `producerGroup` for producers of transactional messages cannot be shared with `producerGroup` for producers of other types of messages. Unlike other types of message, transactional messages allow backward queries. MQ Server query clients by their `producerGroup` of producers.\n\n"
  },
  {
    "path": "docs/en/FAQ.md",
    "content": "# Frequently Asked Questions\n\nThe following questions are frequently asked with regard to the RocketMQ project in general.\n\n## 1 General\n\n1. Why did we create rocketmq project instead of selecting other products?\n\n   Please refer to [Why RocketMQ](http://rocketmq.apache.org/docs/motivation)\n\n2. Do I have to install other software, such as zookeeper, to use RocketMQ?\n\n   No. RocketMQ can run independently.\n\n## 2 Usage\n\n### 1. Where does the newly created Consumer ID start consuming messages?\n\n&#8195;1) If the topic sends a message within three days, then the consumer start consuming messages from the first message saved in the server.\n  \n&#8195;2) If the topic sends a message three days ago, the consumer starts to consume messages from the latest message in the server, in other words, starting from the tail of message queue.\n  \n&#8195;3) If such consumer is rebooted, then it starts to consume messages from the last consumption location.\n\n### 2. How to reconsume message when consumption fails?\n\n&#8195;1) Cluster consumption pattern, The consumer business logic code returns Action.ReconsumerLater, NULL, or throws an exception, if a message failed to be consumed, it will retry for up to 16 times, after that, the message would be discarded.\n  \n&#8195;2) Broadcast consumption pattern. The broadcast consumption still ensures that a message is consumed at least once, but no resend option is provided.\n\n### 3. How to query the failed message if there is a consumption failure?\n\n&#8195;1) Using topic query by time, you can query messages within a period of time.\n  \n&#8195;2) Using Topic and Message Id to accurately query the message.\n  \n&#8195;3) Using Topic and Message Key accurately query a class of messages with the same Message Key.\n\n### 4. Are messages delivered exactly once?\n\nRocketMQ ensures that all messages are delivered at least once. In most cases, the messages are not repeated.\n\n### 5. How to add a new broker?\n\n&#8195;1) Start up a new broker and register it to the same list of name servers.\n  \n&#8195;2) By default, only internal system topics and consumer groups are created automatically. If you would like to have your business topic and consumer groups on the new node, please replicate them from the existing broker. Admin tool and command lines are provided to handle this.\n\n## 3 Configuration related\n\nThe following answers are all default values and can be modified by configuration.\n\n### 1. How long are the messages saved on the server?\n\nStored messages will be saved for up to 3 days, and messages that are not consumed for more than 3 days will be deleted.\n\n### 2. What is the size limit for message Body?\n\nGenerally 256KB.\n\n### 3. How to set the number of consumer threads?\n\nWhen you start Consumer, set a ConsumeThreadNums property, example is as follows:\n```\nconsumer.setConsumeThreadMin(20);\nconsumer.setConsumeThreadMax(20);\n```\n\n## 4 Errors\n\n### 1. If you start a producer or consumer failed and the error message is producer group or consumer repeat.\n\nReason: Using the same Producer /Consumer Group to launch multiple instances of Producer/Consumer in the same JVM may cause the client fail to start.\n\nSolution: Make sure that a JVM corresponding to one Producer /Consumer Group starts only with one Producer/Consumer instance.\n\n### 2. Consumer failed to start loading json file in broadcast mode.\n\nReason: Fastjson version is too low to allow the broadcast consumer to load local offsets.json, causing the consumer boot failure. Damaged fastjson file can also cause the same problem.\n\nSolution: Fastjson version has to be upgraded to rocketmq client dependent version to ensure that the local offsets.json can be loaded. By default offsets.json file is in /home/{user}/.rocketmq_offsets. Or check the integrity of fastjson.\n\n### 3. What is the impact of a broker crash.\n\n&#8195;1) Master crashes\n\nMessages can no longer be sent to this broker set, but if you have another broker set available, messages can still be sent given the topic is present. Messages can still be consumed from slaves.\n\n&#8195;2) Some slave crash\n\nAs long as there is another working slave, there will be no impact on sending messages. There will also be no impact on consuming messages except when the consumer group is set to consume from this slave preferably. By default, consumer group consumes from master.\n\n&#8195;3) All slaves crash\n\nThere will be no impact on sending messages to master, but, if the master is SYNC_MASTER, producer will get a SLAVE_NOT_AVAILABLE indicating that the message is not sent to any slaves. There will also be no impact on consuming messages except that if the consumer group is set to consume from slave preferably. By default, consumer group consumes from master.\n\n### 4. Producer complains “No Topic Route Info”, how to diagnose?\n\nThis happens when you are trying to send messages to a topic whose routing info is not available to the producer.\n\n&#8195;1) Make sure that the producer can connect to a name server and is capable of fetching routing meta info from it.\n  \n&#8195;2) Make sure that name servers do contain routing meta info of the topic. You may query the routing meta info from name server through topicRoute using admin tools or web console.\n  \n&#8195;3) Make sure that your brokers are sending heartbeats to the same list of name servers your producer is connecting to.\n  \n&#8195;4) Make sure that the topic’s permission is 6(rw-), or at least 2(-w-).\n\nIf you can’t find this topic, create it on a broker via admin tools command updateTopic or web console.\n"
  },
  {
    "path": "docs/en/Feature.md",
    "content": "# Features\n## 1 Subscribe and Publish\nMessage publication refers to that a producer sends messages to a topic; Message subscription means a consumer follows a topic with certain tags and then consumes data from that topic.\n\n## 2 Message Ordering\nMessage ordering refers to that a group of messages can be consumed orderly as they are published. For example, an order generates three messages: order creation, order payment, and order completion. It only makes sense to consume them in their generated order, but orders can be consumed in parallel at the same time. RocketMQ can strictly guarantee these messages are in order.\n\nOrderly message is divided into global orderly message and partitioned orderly message. Global order means that all messages under a certain topic must be in order, partitioned order only requires each group of messages are consumed orderly.\n- Global message ordering:\nFor a given Topic, all messages are published and consumed in strict first-in-first-out (FIFO) order.\nApplicable scenario: the performance requirement is not high, and all messages are published and consumed according to FIFO principle strictly.\n- Partitioned message ordering:\nFor a given Topic, all messages are partitioned according to sharding key. Messages within the same partition are published and consumed in strict FIFO order. Sharding key is the key field to distinguish message's partition, which is a completely different concept from the key of ordinary messages.\nApplicable scenario: high performance requirement, with sharding key as the partition field, messages within the same partition are published and consumed according to FIFO principle strictly.\n\n## 3 Message Filter\nConsumers of RocketMQ can filter messages based on tags as well as supporting for user-defined attribute filtering. Message filter is currently implemented on the Broker side, with the advantage of reducing the network transmission of useless messages for Consumer and the disadvantage of increasing the burden on the Broker and relatively complex implementation.\n\n## 4 Message Reliability\nRocketMQ supports high reliability of messages in several situations:\n1 Broker shutdown normally\n2 Broker abnormal crash\n3 OS Crash\n4 The machine is out of power, but it can be recovered immediately\n5 The machine cannot be started up (the CPU, motherboard, memory and other key equipment may be damaged)\n6 Disk equipment damaged\n\nIn the four cases of 1), 2), 3), and 4) where the hardware resource can be recovered immediately, RocketMQ guarantees that the message will not be lost or a small amount of data will be lost (depending on whether the flush disk type is synchronous or asynchronous).\n\n5 ) and 6) are single point of failure and cannot be recovered. Once it happens, all messages on the single point will be lost. In both cases, RocketMQ ensures that 99% of the messages are not lost through asynchronous replication, but a very few number of messages may still be lost. Synchronous double write mode can completely avoid single point of failure, which will surely affect the performance and suitable for the occasion of high demand for message reliability, such as money related applications. Note: RocketMQ supports synchronous double writes since version 3.0.\n\n## 5 At Least Once\nAt least Once refers to that every message will be delivered at least once. RocketMQ supports this feature because the Consumer pulls the message locally and does not send an ack back to the server until it has consumed it.\n\n## 6 Backtracking Consumption\nBacktracking consumption refers to that the Consumer has consumed the message successfully, but the business needs to consume again. To support this function, the message still needs to be retained after the Broker sends the message to the Consumer successfully. The re-consumption is normally based on time dimension. For example, after the recovery of the Consumer system failures, the data one hour ago needs to be re-consumed, then the Broker needs to provide a mechanism to reverse the consumption progress according to the time dimension. RocketMQ supports backtracking consumption by time trace, with the time dimension down to milliseconds.\n\n## 7 Transactional Message\nRocketMQ transactional message refers to the fact that the application of a local transaction and the sending of a Message operation can be defined in a global transaction which means both succeed or failed simultaneously. RocketMQ transactional message provides distributed transaction functionality similar to X/Open XA, enabling the ultimate consistency of distributed transactions through transactional message.\n\n## 8 Scheduled Message\nScheduled message(delay queue) refers to that messages are not consumed immediately after they are sent to the broker, but waiting to be delivered to the real topic after a specific time. \nThe broker has a configuration item, `messageDelayLevel`, with default values “1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h”, 18 levels. Users can configure a custom `messageDelayLevel`. Note that `messageDelayLevel` is a broker's property rather than a topic's. When sending a message, just set the delayLevel level: msg.setDelayLevel(level). There are three types of levels:\n\n- level == 0, The message is not a delayed message\n- 1<=level<=maxLevel, Message delay specific time, such as level==1, delay for 1s\n- level > maxLevel, than level== maxLevel, such as level==20, delay for 2h\n\nScheduled messages are temporarily saved in a topic named SCHEDULE_TOPIC_XXXX, and saved in a specific queue according to delayTimeLevel, queueId = delayTimeLevel - 1, that is, only messages with the same delay are saved in a queue, ensuring that messages with the same sending delay can be consumed orderly. The broker consumes SCHEDULE_TOPIC_XXXX on schedule and writes messages to the real topic.\n\nNote that Scheduled messages are counted both the first time they are written and the time they are scheduled to be written to the real topic, so both the send number and the TPS are increased.\n\n## 9 Message Retry\nWhen the Consumer fails to consume the message, a retry mechanism is needed to make the message to be consumed again. Consumer's consume failure can usually be classified as follows:\n- Due to the reasons of the message itself, such as deserialization failure, the message data itself cannot be processed (for example, the phone number of the current message is cancelled and cannot be charged), etc. This kind of error usually requires skipping this message and consuming others since immediately retry would be failed 99%, so it is better to provide a timed retry mechanism that retries after 10 seconds.\n- Due to the reasons of dependent downstream application services are not available, such as db connection is not usable, perimeter network is not unreachable, etc. When this kind of error is encountered, consuming other messages will also result in an error even if the current failed message is skipped. In this case, it is recommended to sleep for 30s before consuming the next message, which will reduce the pressure on the broker to retry the message.\n\nRocketMQ will set up a retry queue named “%RETRY%+consumerGroup” for each consumer group(Note that the retry queue for this topic is for consumer groups, not for each topic) to temporarily save messages cannot be consumed by customer due to all kinds of reasons. Considering that it takes some time for the exception to recover, multiple retry levels are set for the retry queue, and each retry level has a corresponding re-deliver delay. The more retries, the greater the deliver delay. RocketMQ first save retry messages to the delay queue which topic is named “SCHEDULE_TOPIC_XXXX”, then background schedule task will save the messages to “%RETRY%+consumerGroup” retry queue according to their corresponding delay.\n\n## 10 Message Resend\nWhen a producer sends a message, the synchronous message will be resent if fails, the asynchronous message will retry and oneway message is without any guarantee. Message resend ensures that messages are sent successfully and without lost as much as possible, but it can lead to message duplication, which is an unavoidable problem in RocketMQ. Under normal circumstances, message duplication will not occur, but when there is a large number of messages and network jitter, message duplication will be a high-probability event. In addition, producer initiative messages resend and the consumer load changes will also result in duplicate messages. The message retry policy can be set as follows:\n\n- `retryTimesWhenSendFailed`: Synchronous message retry times when send failed, default value is 2, so the producer will try to send `retryTimesWhenSendFailed` + 1 times at most. To ensure that the message is not lost, producer will try sending the message to another broker instead of selecting the broker that failed last time. An exception will be thrown if it reaches the retry limit, and the client should guarantee that the message will not be lost. Messages will resend when RemotingException, MQClientException, and partial MQBrokerException occur.\n- `retryTimesWhenSendAsyncFailed`: Asynchronous message retry times when send failed, asynchronous retry sends message to the same broker instead of selecting another one and does not guarantee that the message wont lost.\n- `retryAnotherBrokerWhenNotStoreOK`: Message flush disk (master or slave) timeout or slave not available (return status is not SEND_OK), whether to try to send to another broker, default value is false. Very important messages can set to true.\n\n## 11 Flow Control\nProducer flow control, because broker processing capacity reaches a bottleneck; Consumer flow control, because the consumption capacity reaches a bottleneck.\n\nProducer flow control:\n- When commitLog file locked time exceeds osPageCacheBusyTimeOutMills, default value of `osPageCacheBusyTimeOutMills` is 1000 ms, then return flow control.\n- If `transientStorePoolEnable` == true, and the broker is asynchronous flush disk type, and resources are insufficient in the transientStorePool, reject the current send request and return flow control.\n- The broker checks the head request wait time of the send request queue every 10ms. If the wait time exceeds waitTimeMillsInSendQueue, which default value is 200ms, the current send request is rejected and the flow control is returned.\n- The broker implements flow control by rejecting send requests.\n\nConsumer flow control:\n- When consumer local cache messages number exceeds pullThresholdForQueue, default value is 1000. \n- When consumer local cache messages size exceeds pullThresholdSizeForQueue, default value is 100MB. \n- When consumer local cache messages span exceeds consumeConcurrentlyMaxSpan, default value is 2000.\n\nThe result of consumer flow control is to reduce the pull frequency.\n\n## 12 Dead Letter Queue\nDead letter queue is used to deal messages that cannot be consumed normally. When a message is consumed failed at first time, the message queue will automatically resend the message. If the consumption still fails after the maximum number retry, it indicates that the consumer cannot properly consume the message under normal circumstances. At this time, the message queue will not immediately abandon the message, but send it to the special queue corresponding to the consumer.\n\nRocketMQ defines the messages that could not be consumed under normal circumstances as Dead-Letter Messages, and the special queue in which the Dead-Letter Messages are saved as Dead-Letter Queues. In RocketMQ, the consumer instance can consume again by resending messages in the Dead-Letter Queue using console.\n\n## 13 Pop Consuming\n\nPop consuming refers to that broker fetches messages from queues owned by same broker and returns to clients, which ensures one queue will be consumed by multiple clients. The whole behavior is like a queue\npop process. By invoking `setConsumeMode` sub command of mqadmin, one consumer group can be switch to POP consuming instead of classical PULL consuming without changing a single code line. The new pop consuming will help to mitigate the impact for one queue consuming of an abnormal behaving client."
  },
  {
    "path": "docs/en/Operations_Broker.md",
    "content": "# 3 Broker\n\n## 3.1 Broker Role\nBroker Role is ASYNC_MASTER, SYNC_MASTER or SLAVE. If you cannot tolerate message missing, we suggest you deploy SYNC_MASTER and attach a SLAVE to it. If you feel ok about missing, but you want the Broker to be always available, you may deploy ASYNC_MASTER with SLAVE. If you just want to make it easy, you may only need a ASYNC_MASTER without SLAVE.\n## 3.2 FlushDiskType\nASYNC_FLUSH is recommended, for SYNC_FLUSH is expensive and will cause too much performance loss. If you want reliability, we recommend you use SYNC_MASTER with SLAVE.\n## 3.3 Broker Configuration\n| Parameter name                           | Default                        | Description                                                         |\n| -------------------------------- | ----------------------------- | ------------------------------------------------------------ |\n| listenPort                    | 10911              | listen port for client |\n| namesrvAddr       | null                         | name server address     |\n| brokerIP1 | InetAddress for network interface                         | Should be configured if having multiple addresses |\n| brokerIP2 | InetAddress for network interface                         | If configured for the Master broker in the Master/Slave cluster, slave broker will connect to this port for data synchronization   |\n| brokerName        | null                         | broker name                           |\n| brokerClusterName                     | DefaultCluster                  | this broker belongs to which cluster           |\n| brokerId             | 0                              | broker id, 0 means master, positive integers mean slave                                                 |\n| storePathCommitLog                      | $HOME/store/commitlog/                              | file path for commit log                                                 |\n| storePathConsumerQueue                   | $HOME/store/consumequeue/                              | file path for consume queue                                              |\n| mappedFileSizeCommitLog     | 1024 * 1024 * 1024(1G) | mapped file size for commit log                                        |​ \n| deleteWhen     | 04 | When to delete the commitlog which is out of the reserve time                                        |​ \n| fileReserverdTime     | 72 | The number of hours to keep a commitlog before deleting it                                        |​ \n| brokerRole     | ASYNC_MASTER | SYNC_MASTER/ASYNC_MASTER/SLAVE                                        |​ \n| flushDiskType     | ASYNC_FLUSH | {SYNC_FLUSH/ASYNC_FLUSH}. Broker of SYNC_FLUSH mode flushes each message onto disk before acknowledging producer. Broker of ASYNC_FLUSH mode, on the other hand, takes advantage of group-committing, achieving better performance.                                        |​"
  },
  {
    "path": "docs/en/Operations_Consumer.md",
    "content": "## Consumer\n\n----\n\n### 1 Consumption process idempotent\n\nRocketMQ cannot achieve Exactly-Once, so if the business is very sensitive to consumption repetition, it is important to perform deduplication at the business level. Deduplication can be done with a relational database. First, you need to determine the unique key of the message, which can be either msgId or a unique identifier field in the message content, such as the order Id. Determine if a unique key exists in the relational database before consumption. If it does not exist, insert it and consume it, otherwise skip it. (The actual process should consider the atomic problem, determine whether there is an attempt to insert, if the primary key conflicts, the insertion fails, skip directly)\n\n### 2  Slow message processing\n\n#### 2.1 Increase consumption parallelism\n\nMost messages consumption behaviors are IO-intensive, That is, it may be to operate the database, or call RPC. The consumption speed of such consumption behavior lies in the throughput of the back-end database or the external system. By increasing the consumption parallelism, the total consumption throughput can be increased, but the degree of parallelism is increased to a certain extent. Instead it will fall.Therefore, the application must set a reasonable degree of parallelism. There are several ways to modify the degree of parallelism of consumption as follows:\n\n* Under the same ConsumerGroup, increase the degree of parallelism by increasing the number of Consumer instances (note that the Consumer instance that exceeds the number of subscription queues is invalid). Can be done by adding machines, or by starting multiple processes on an existing machine.\n* Improve the consumption parallel thread of a single Consumer by modifying the parameters consumeThreadMin and consumeThreadMax.\n\n#### 2.2 Batch mode consumption\n\nSome business processes can increase consumption throughput to a large extent if they support batch mode consumption. For example, order deduction application, it takes 1s to process one order at a time, and it takes only 2s to process 10 orders at a time. In this way, the throughput of consumption can be greatly improved. By setting the consumer's consumeMessageBatchMaxSize to return a parameter, the default is 1, that is, only one message is consumed at a time, for example, set to N, then the number of messages consumed each time is less than or equal to N.\n\n#### 2.3 Skip non-critical messages\n\nWhen a message is accumulated, if the consumption speed cannot keep up with the transmission speed, if the service does not require high data, you can choose to discard the unimportant message. For example, when the number of messages in a queue is more than 100,000 , try to discard some or all of the messages, so that you can quickly catch up with the speed of sending messages. The sample code is as follows:\n\n```java\npublic ConsumeConcurrentlyStatus consumeMessage(\n        List<MessageExt> msgs,\n        ConsumeConcurrentlyContext context){\n   long offset = msgs.get(0).getQueueOffset();\n   String maxOffset =    \n               msgs.get(0).getProperty(Message.PROPERTY_MAX_OFFSET);\n   long diff = Long.parseLong(maxOffset) - offset;\n   if(diff > 100000){\n        //TODO Special handling of message accumulation\n       return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n    }\n    //TODO Normal consumption process\n    return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n}\n```\n\n#### 2.4 Optimize each message consumption process\n\nFor example, the consumption process of a message is as follows:\n\n* Query from DB according to the message [data 1]\n* Query from DB according to the message [data 2]\n* Complex business calculations\n* Insert [Data 3] into the DB\n* Insert [Data 4] into the DB\n\nThere are 4 interactions with the DB in the consumption process of this message. If it is calculated by 5ms each time, it takes a total of 20ms. If the business calculation takes 5ms, then the total time is 25ms, So if you can optimize 4 DB interactions to 2 times, the total time can be optimized to 15ms, which means the overall performance is increased by 40%. Therefore, if the application is sensitive to delay, the DB can be deployed on the SSD hard disk. Compared with the SCSI disk, the former RT will be much smaller.\n\n### 3 Print Log\n\nIf the amount of messages is small, it is recommended to print the message in the consumption entry method, consume time, etc., to facilitate subsequent troubleshooting.\n\n```java\npublic ConsumeConcurrentlyStatus consumeMessage(\n          List<MessageExt> msgs,\n    ConsumeConcurrentlyContext context){\n    log.info(\"RECEIVE_MSG_BEGIN: \" + msgs.toString());\n    //TODO Normal consumption process\n    return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n}\n```\n\nIf you can print the time spent on each message, it will be more convenient when troubleshooting online problems such as slow consumption.\n\n### 4 Other consumption suggestions\n\n#### 4.1、Consumer Group and Subscriptions\n\nThe first thing you should be aware of is that different Consumer Group can consume the same topic independently, and each of them will have their own consuming offsets. Please make sure each Consumer within the same Group to subscribe the same topics.\n\n#### 4.2、Orderly\n\nThe Consumer will lock each MessageQueue to make sure it is consumed one by one in order. This will cause a performance loss, but it is useful when you care about the order of the messages. It is not recommended to throw exceptions, you can return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT instead.\n\n#### 4.3、Concurrently\n\nAs the name tells, the Consumer will consume the messages concurrently. It is recommended to use this for good performance. It is not recommended to throw exceptions, you can return ConsumeConcurrentlyStatus.RECONSUME_LATER instead.\n\n#### 4.4、Consume Status\n\nFor MessageListenerConcurrently, you can return RECONSUME_LATER to tell the consumer that you can not consume it right now and want to reconsume it later. Then you can continue to consume other messages. For MessageListenerOrderly, because you care about the order, you can not jump over the message, but you can return SUSPEND_CURRENT_QUEUE_A_MOMENT to tell the consumer to wait for a moment.\n\n#### 4.5、Blocking\n\nIt is not recommend to block the Listener, because it will block the thread pool, and eventually may stop the consuming process.\n\n#### 4.6、Thread Number\n\nThe consumer use a ThreadPoolExecutor to process consuming internally, so you can change it by setting setConsumeThreadMin or setConsumeThreadMax.\n\n#### 4.7、ConsumeFromWhere\n\nWhen a new Consumer Group is established, it will need to decide whether it needs to consume the historical messages which had already existed in the Broker. CONSUME_FROM_LAST_OFFSET will ignore the historical messages, and consume anything produced after that. CONSUME_FROM_FIRST_OFFSET will consume every message existed in the Broker. You can also use CONSUME_FROM_TIMESTAMP to consume messages produced after the specified timestamp.\n\n\n\n\n\n\n\n"
  },
  {
    "path": "docs/en/Operations_Producer.md",
    "content": "### Producer\n----\n##### 1 Message Sending Tips\n###### 1.1 The Use of Tags\nOne application instance should use one topic as much as possible and the subtype of messages can be marked by tags. Tag provides extra flexibility to users. In the consume subscribing process, the messages filtering can only be handled by using tags when the tags are specified in the message sending process: `message.setTags(\"TagA\")`.\n###### 1.2 The Use of Keys\nA business  key can be set in one message and it will be easier to look up the message on a broker server to diagnose issues during development. Each message will be created index(hash index) by server, instance can query the content of this message by topic and key and who consumes the message.Because of the hash index, make sure that the key should be unique in order to avoid potential hash index conflict.\n``` java\n// Order Id\nString orderId = \"20034568923546\";\nmessage.setKeys(orderId);\n```\n###### 1.3 The Log Print\nWhen sending a message,no matter success or fail, a message log must be printed which contains SendResult and Key. It is assumed that we will always get SEND_OK if no exception is thrown. Below is a list of descriptions about each status:\n* SEND_OK\n\nSEND_OK means sending message successfully. SEND_OK does not mean it is reliable. To make sure no message would be lost, you should also enable SYNC_MASTER or SYNC_FLUSH.\n* FLUSH_DISK_TIMEOUT\n\nFLUSH_DISK_TIMEOUT means sending message successfully but the Broker flushing the disk with timeout. In this kind of condition, the Broker has saved this message in memory, this message will be lost only if the Broker was down. The FlushDiskType and SyncFlushTimeout could be specified in MessageStoreConfig. If the Broker set MessageStoreConfig’s FlushDiskType=SYNC_FLUSH(default is ASYNC_FLUSH), and the Broker doesn’t finish flushing the disk within MessageStoreConfig’s syncFlushTimeout(default is 5 secs), you will get this status.\n* FLUSH_SLAVE_TIMEOUT\n\nFLUSH_SLAVE_TIMEOUT means sending messages successfully but the slave Broker does not finish synchronizing with the master. If the Broker’s role is SYNC_MASTER(default is ASYNC_MASTER), and the slave Broker doesn’t finish synchronizing with the master within the MessageStoreConfig’s syncFlushTimeout(default is 5 secs), you will get this status.\n* SLAVE_NOT_AVAILABLE\n\nSLAVE_NOT_AVAILABLE means sending messages successfully but no slave Broker configured. If the Broker’s role is SYNC_MASTER(default is ASYNC_MASTER), but no slave Broker is configured, you will get this status.\n\n##### 2 Operations on Message Sending failed\nThe send method of Producer can be retried, the retry  process is illustrated below:\n* The method will retry at most 2 times(2 times in synchronous mode, 0 times in asynchronous mode).\n* If sending failed, it will turn to the next Broker. This strategy will be executed when the total costing time is less then sendMsgTimeout(default is 10 seconds).\n* The retry method will be terminated if timeout exception was thrown when sending messages to Broker.\n\nThe strategy above could make sure message sending successfully to a certain degree. Some more retry strategies, such as we could try to save the message to database if calling the send synchronous method failed and then retry by background thread's timed tasks, which will make sure the message is sent to Broker,could be improved if asking for high reliability business requirement. \n\nThe reasons why the retry strategy using database have not integrated by the RocketMQ client will be explained below: Firstly, the design mode of the RocketMQ client is stateless mode. It means that the client is designed to be horizontally scalable at each level and the consumption of the client to physical resources is only CPU, memory and network. Then, if a key-value memory module is integrated by the client itself, the Async-Saving strategy will be utilized in consideration of the high resource consumption of the Syn-Saving strategy. However, given that operations staff does not manage the client shutoff, some special commands, such as kill -9, may be used which will lead to the lost of message because of no saving in time. Furthermore, the physical resource running Producer is not appropriate to save some significant data because of low reliability. Above all, the retry process should be controlled by program itself.\n\n##### 3 Send Messages in One-way Mode\nThe message sending is usually a process like below: \n* Client sends request to sever.\n* Sever handles request\n* Sever returns response to client\n\nThe total costing time of sending one message is the sum of costing time of three steps above. Some situations demand that total costing time must be in a quite low level, however, do not take reliable performance into consideration, such as log collection. This kind of application could be called in one-way mode, which means client sends request but not wait for response. In this kind of mode, the cost of sending request is only a call of system operation which means one operation writing data to client socket buffer. Generally, the time cost of this process will be controlled n microseconds level.\n"
  },
  {
    "path": "docs/en/Operations_Trace.md",
    "content": "# Message Trace\n\n## 1 Key Attributes of Message Trace Data\n\n| Producer        | Consumer        | Broker     |\n| ---------------- | ----------------- | ------------ |\n| production instance information     | consumption instance information      | message Topic  |\n| send message time | post time, post round | message storage location |\n| whether the message was sent successfully | Whether the message was successfully consumed  | The Key of the message  |\n| Time spent sending         | Time spent consuming         | Tag of the message  |\n\n## 2 Support for Message Trace Cluster Deployment\n\n### 2.1 Broker Configuration Fille\n\nThe properties profile content of the Broker side enabled message trace feature is pasted here:\n\n```\nbrokerClusterName=DefaultCluster\nbrokerName=broker-a\nbrokerId=0\ndeleteWhen=04\nfileReservedTime=48\nbrokerRole=ASYNC_MASTER\nflushDiskType=ASYNC_FLUSH\nstorePathRootDir=/data/rocketmq/rootdir-a-m\nstorePathCommitLog=/data/rocketmq/commitlog-a-m\nautoCreateSubscriptionGroup=true\n## if msg tracing is open,the flag will be true\ntraceTopicEnable=true\nlistenPort=10911\nbrokerIP1=XX.XX.XX.XX1\nnamesrvAddr=XX.XX.XX.XX:9876\n```\n\n### 2.2 Normal Mode\nEach Broker node in the RocketMQ cluster is used to store message trace data collected and sent from the Client.Therefore, there are no requirements or restrictions on the number of Broker nodes in the RocketMQ cluster.\n\n### 2.3 Physical IO Isolation Mode\nFor scenarios with large amount of trace message data , one of the Broker nodes in the RocketMQ cluster can be selected to store the trace message , so that the common message data of the user and the physical IO of the trace message data are completely isolated from each other.In this mode, there are at least two Broker nodes in the RocketMQ cluster, one of which is defined as the server on which message trace data is stored.\n\n### 2.4 Start the Broker that Starts the MessageTrace\n`nohup sh mqbroker -c ../conf/2m-noslave/broker-a.properties &`\n\n## 3 Save the Topic Definition of Message Trace \nRocketMQ's message trace feature supports two ways to store trace data:\n\n### 3.1 System-level TraceTopic\nBy default, message track data is stored in the system-level TraceTopic(names: **RMQ_SYS_TRACE_TOPIC**).This Topic is automatically created when the Broker node is started(As described above, the switch variable **traceTopicEnable** needs to be set to **true** in the Broker  configuration file）.\n\n### 3.2 Custom TraceTopic \nIf the user is not prepared to store the message track data in the system-level default TraceTopic, you can also define and create a user-level Topic to save the track (that is, to create a regular Topic to save the message track data).The following section introduces how the Client interface supports the user-defined TraceTopic.\n\n## 4 Client Practices that Support Message Trace\nIn order to reduce as much as possible the transformation work of RocketMQ message trace feature used in the user service system, the author added a switch parameter (**enableMsgTrace**) to the original interface in the design to realize whether the message trace is opened or not.\n\n### 4.1 Opening  the Message Trace when Sending  the Message\n```\n        DefaultMQProducer producer = new DefaultMQProducer(\"ProducerGroupName\",true);\n        producer.setNamesrvAddr(\"XX.XX.XX.XX1\");\n        producer.start();\n            try {\n                {\n                    Message msg = new Message(\"TopicTest\",\n                        \"TagA\",\n                        \"OrderID188\",\n                        \"Hello world\".getBytes(RemotingHelper.DEFAULT_CHARSET));\n                    SendResult sendResult = producer.send(msg);\n                    System.out.printf(\"%s%n\", sendResult);\n                }\n\n            } catch (Exception e) {\n                e.printStackTrace();\n            }\n```\n\n### 4.2 Opening Message Trace whenSubscribing to a Message\n```\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"CID_JODIE_1\",true);\n        consumer.subscribe(\"TopicTest\", \"*\");\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        consumer.setConsumeTimestamp(\"20181109221800\");\n        consumer.registerMessageListener(new MessageListenerConcurrently() {\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {\n                System.out.printf(\"%s Receive New Messages: %s %n\", Thread.currentThread().getName(), msgs);\n                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n            }\n        });\n        consumer.start();\n        System.out.printf(\"Consumer Started.%n\");\n```\n\n### 4.3 Support for Custom Storage Message Trace Topic\nThe initialization of `DefaultMQProducer` and `DefaultMQPushConsumer` instances can be changed to support the custom storage message trace Topic as follows when sending and subscribing messages above.\n\n```\n        ##Where Topic_test11111 needs to be pre-created by the user to save the message trace;\n        DefaultMQProducer producer = new DefaultMQProducer(\"ProducerGroupName\",true,\"Topic_test11111\");\n        ......\n\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"CID_JODIE_1\",true,\"Topic_test11111\");\n        ......\n```"
  },
  {
    "path": "docs/en/QuorumACK.md",
    "content": "# Quorum write and automatic downgrade\n\n## Background\n\n![](https://s4.ax1x.com/2022/02/05/HnWo2d.png)\n\nIn RocketMQ, there are two main replication modes between primary and secondary servers: synchronous replication and asynchronous replication. As shown in the above figure, the replication of Slave1 is synchronous, and the Master needs to wait for Slave1 to successfully replicate the message and confirm before reporting success to the Producer. The replication of Slave2 is asynchronous, and the Master does not need to wait for the response from Slave2. In RocketMQ, if everything goes well when sending a message, the Producer client will eventually receive a PUT_OK status. If the Slave synchronization times out, it will return a FLUSH_SLAVE_TIMEOUT status. If the Slave is unavailable or the difference between the CommitLog of the Slave and Master exceeds a certain value (default is 256MB), it will return a SLAVE_NOT_AVAILABLE status. The latter two states will not cause system exceptions and prevent the next message from being written.\n\nSynchronous replication ensures that the data can still be found in the Slave after the Master fails, which is suitable for scenarios with high reliability requirements. Although asynchronous replication may result in message loss, it is more efficient than synchronous replication because it does not need to wait for the Slave's confirmation, and is suitable for scenarios with certain efficiency requirements. However, only two modes are not flexible enough. For example, in scenarios with three or even five copies and high reliability requirements, asynchronous replication cannot meet the requirements, but synchronous replication needs to wait for each copy to confirm before returning, which seriously affects efficiency in the case of many copies. On the other hand, in the synchronous replication mode, if one of the Slaves in the copy group becomes inactive, the entire send will fail until manual processing is performed.\n\nTherefore, RocketMQ 5 introduces quorum write for copy groups. In the synchronous replication mode, the user can specify on the broker side how many copies need to be written before returning after sending, and provides an adaptive downgrade method that can automatically downgrade based on the number of surviving copies and the CommitLog gap.\n\n## Architecture and Parameters\n\n### Quorum Write\n\nquorum write is supported by adding two parameters:\n\n- **totalReplicas**：Total number of brokers in the copy replica. default is 1.\n- **inSyncReplicas**：The number of replica groups that should normally be kept in synchronization. default is 1.\n\nWith these two parameters, you can flexibly specify the number of copies that need ACK in the synchronous replication mode.\n\n![](https://s4.ax1x.com/2022/02/05/HnWHKI.png)\n\nAs shown in the above figure, in the case of two copies, if inSyncReplicas is 2, the message needs to be copied in both the Master and the Slave before it is returned to the client; in the case of three copies, if inSyncReplicas is 2, the message needs to be copied in the Master and any slave before it is returned to the client. In the case of four copies, if inSyncReplicas is 3, the message needs to be copied in the Master and any two slaves before it is returned to the client. By flexibly setting totalReplicas and inSyncReplicas, users can meet the needs of various scenarios.\n\n### Automatic downgrade\n\nThe standards for automatic downgrade are:\n\n- The number of surviving replicas in the current replica group\n- The height difference between the Master Commitlog and the Slave CommitLog\n\n> **NOTE: Automatic downgrade is only effective after the slaveActingMaster mode is enabled**\n\nThe current survival information of the copy group can be obtained through the reverse notification of the Nameserver and the GetBrokerMemberGroup request, and the height difference between the Master and the Slave Commitlog can also be calculated through the position record in the HA service. The following parameters will be added to complete the automatic downgrade:\n\n- **minInSyncReplicas**：The minimum number of copies in the group that must be kept in sync, only effective when enableAutoInSyncReplicas is true, default is 1\n- **enableAutoInSyncReplicas**：The switch for automatic synchronization downgrade, when turned on, if the number of brokers in the current copy group in the synchronization state (including the master itself) does not meet the number specified by inSyncReplicas, it will be synchronized according to minInSyncReplicas. The synchronization state judgment condition is that the slave commitLog lags behind the master length by no more than haSlaveFallBehindMax. The default is false.\n- **haMaxGapNotInSync**：The value for determining whether the slave is in sync with the master. If the slave commitLog lags behind the master length by more than this value, the slave is considered to be out of sync. When enableAutoInSyncReplicas is turned on, the smaller the value, the easier it is to trigger automatic downgrade of the master. When enableAutoInSyncReplicas is turned off and `totalReplicas == inSyncReplicas`, the smaller the value, the more likely it is to cause requests to fail during high traffic. Therefore, in this case, it is appropriate to increase haMaxGapNotInSync. The default is 256K.\n\nNote: In RocketMQ 4.x, there is a haSlaveFallbehindMax parameter, with a default value of 256MB, indicating the CommitLog height difference at which the Slave is considered unavailable. This parameter was cancelled in [RIP-34](https://github.com/apache/rocketmq/wiki/RIP-34-Support-quorum-write-and-adaptive-degradation-in-master-slave-architecture).\n\n```java\n//calculate needAckNums\nint inSyncReplicas = Math.min(this.defaultMessageStore.getAliveReplicaNumInGroup(),\n                              this.defaultMessageStore.getHaService().inSyncSlaveNums(currOffset) + 1);\nneedAckNums = calcNeedAckNums(inSyncReplicas);\nif (needAckNums > inSyncReplicas) {\n    // Tell the producer, don't have enough slaves to handle the send request\n    return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH, null));\n}\n\nprivate int calcNeedAckNums(int inSyncReplicas) {\n    int needAckNums = this.defaultMessageStore.getMessageStoreConfig().getInSyncReplicas();\n    if (this.defaultMessageStore.getMessageStoreConfig().isEnableAutoInSyncReplicas()) {\n        needAckNums = Math.min(needAckNums, inSyncReplicas);\n        needAckNums = Math.max(needAckNums, this.defaultMessageStore.getMessageStoreConfig().getMinInSyncReplicas());\n    }\n    return needAckNums;\n}\n```\n\nWhen enableAutoInSyncReplicas=true, the adaptive downgrade mode is enabled. When the number of surviving replicas in the replica group decreases or the height difference between the Master and the Slave Commitlog is too large, automatic downgrade will be performed, with a minimum of minInSyncReplicas replicas. For example, in two replicas, if totalReplicas=2, InSyncReplicas=2, minInSyncReplicas=1, and enableAutoInSyncReplicas=true are set, under normal circumstances, the two replicas will be in synchronous replication. When the Slave goes offline or hangs, adaptive downgrade will be performed, and the producer only needs to send to the master to succeed.\n\n## Compatibility\n\nTo ensure backward compatibility, users need to set the correct parameters. For example, if the user's original cluster is a two-replica synchronous replication and no parameters are modified, when upgrading to the RocketMQ 5 version, due to the default totalReplicas and inSyncReplicas both being 1, it will downgrade to asynchronous replication. If you want to maintain the same behavior as before, you need to set both totalReplicas and inSyncReplicas to 2.\n\n**references:**\n\n- [RIP-34](https://github.com/apache/rocketmq/wiki/RIP-34-Support-quorum-write-and-adaptive-degradation-in-master-slave-architecture)"
  },
  {
    "path": "docs/en/README.md",
    "content": "Apache RocketMQ Developer Guide\n--------\n\n##### This guide helps developers to understand and use Apache RocketMQ quickly.\n\n### 1. Concepts & Features\n\n- [Concept](Concept.md): introduce basic concepts in RocketMQ.\n\n- [Feature](Feature.md): introduce functional features of RocketMQ's implementations.\n\n\n### 2. Architecture Design\n\n- [Architecture](architecture.md): introduce RocketMQ's deployment and technical architecture.\n\n- [Design](design.md): introduce design concept of RocketMQ's key mechanisms, including message storage, communication mechanisms, message filter, loadbalance, transaction message, etc.\n\n\n### 3. Example\n\n- [Example](RocketMQ_Example.md): introduce RocketMQ's common usage, including basic example, sequence message example, delay message example, batch message example, filter message example, transaction message example, etc.\n\n\n### 4. Best Practice\n- [Best Practice](best_practice.md): introduce RocketMQ's best practice, including producer, consumer, broker, NameServer, configuration of client, and the best parameter configuration of JVM, linux.\n\n- [Message Trace](msg_trace/user_guide.md): introduce how to use RocketMQ's message tracing feature.\n\n- [Auth Management](acl/Operations_ACL.md): introduce how to deployment quickly and how to use RocketMQ cluster enabling auth management feature.\n\n- [Quick Start](dledger/quick_start.md): introduce how to deploy Dledger quickly.\n\n- [Cluster Deployment](dledger/deploy_guide.md): introduce how to deploy Dledger in cluster.\n\n- [Proxy Deployment](proxy/deploy_guide.md)\n  Introduce how to deploy proxy (both `Local` mode and `Cluster` mode).\n\n### 5. Operation and maintenance management\n- [Operation](operation.md): introduce RocketMQ's deployment modes that including single-master mode, multi-master mode, multi-master multi-slave mode and so on, as well as the usage of operation tool mqadmin.\n\n\n### 6. API Reference（TODO）\n\n- [DefaultMQProducer API Reference](client/java/API_Reference_DefaultMQProducer.md)\n\n\n\n\n\n\n\n"
  },
  {
    "path": "docs/en/RocketMQ_Example.md",
    "content": "### Examples List\n\n- [basic example](Example_Simple.md)\n\n- [sequence message example](Example_Orderly.md)\n\n- [delay message example](Example_Delay.md)\n\n- [batch message example](Example_Batch.md)\n\n- [filter message example](Example_Filter.md)\n\n- [transaction message example](Example_Transaction.md)\n\n- [openmessaging example](Example_OpenMessaging.md)\n"
  },
  {
    "path": "docs/en/Troubleshoopting.md",
    "content": "# Operation FAQ\n\n## 1 RocketMQ's mqadmin command error.\n\n>  Problem: Sometimes after deploying the RocketMQ cluster, when you try to execute some commands of \"mqadmin\", the following exception will appear:\n>\n>  ```java\n>  org.apache.rocketmq.remoting.exception.RemotingConnectException: connect to <null> failed\n>  ```\n\nSolution: Execute  `export NAMESRV_ADDR=ip:9876` (ip refers to the address of NameServer deployed in the cluster) on the VM that deploys the RocketMQ cluster.Then you will execute commands of \"mqadmin\" successfully.\n\n## 2 The inconsistent version of RocketMQ between the producer and consumer leads to the problem that messages can't be consumed normally.\n\n> Problem: The same producer sends a message, consumer A can consume, but consumer B can't consume, and the RocketMQ Console appears:\n>\n> ```java\n> Not found the consumer group consume stats, because return offset table is empty, maybe the consumer not consume any message\n> ```\n\nSolution: The jar package of RocketMQ, such as rocketmq-client, should be the same version on the consumer and producer.\n\n## 3 When adding a new topic consumer group, historical messages can't be consumed.\n\n> Problem: When a new consumer group of the same topic is started, the consumed message is the current offset message, and the historical message is not obtained.\n\nSolution: The default policy of rocketmq is to start from the end of the message queue and skip the historical message. If you want to consume historical message, you need to set: \n\n```java\norg.apache.rocketmq.client.consumer.DefaultMQPushConsumer#setConsumeFromWhere\n```\n\nThere are three common configurations: \n\n- By default, a new subscription group starts to consume from the end of the queue for the first time, and then restarts and continue to consume from the last consume position, that is, to skip the historical message.\n\n```java\nconsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);\n```\n\n- A new subscription group starts to consume from the head of the queue for the first time, and then restarts and continue to consume from the last consume position, that is, to consume the historical message  that is not expired on Broker.\n\n```java\nconsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n```\n\n- A new subscription group starts to consume from the specified time point for the first time, and then restarts and continue to consume from the last consume position. It is used together with `consumer.setConsumeTimestamp()`. The default is half an hour ago.\n\n```java\nconsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_TIMESTAMP);\n```\n\n## 4 How to enable reading data from Slave\n\nIn some cases, the Consumer needs to reset the consume position to 1-2 days ago. At this time, on the Master Broker with limited memory, the CommitLog will carry a relatively heavy IO pressure, affecting the reading and writing of other messages on that Broker. You can enable `slaveReadEnable=true`. When Master Broker finds that the difference between the Consumer's consume position and the latest value of CommitLog exceeds the percentage of machine's memory (`accessMessageInMemoryMaxRatio=40%`), it will recommend Consumer  to read from Slave Broker and relieve Master Broker's IO.\n\n## 5 Performance\n\nAsynchronous flush disk is recommended to use spin lock.\n\nSynchronous flush disk is recommended to use reentrant lock. Adjust the Broker configuration item `useReentrantLockWhenPutMessage`, and the default  value is false. \n\nAsynchronous flush disk is recommended to open `TransientStorePoolEnable` and close `transferMsgByHeap` to improve the efficiency of pulling message;\n\nSynchronous flush disk is recommended to increase the `sendMessageThreadPoolNums` appropriately. The specific configuration needs to be tested.\n\n## 6 The meaning and difference between msgId and offsetMsgId in RocketMQ\n\nAfter sending message with RocketMQ, you will usually see the following log:\n\n```java\nSendResult [sendStatus=SEND_OK, msgId=0A42333A0DC818B4AAC246C290FD0000, offsetMsgId=0A42333A00002A9F000000000134F1F5, messageQueue=MessageQueue [topic=topicTest1, BrokerName=mac.local, queueId=3], queueOffset=4]\n```\n\n- msgId, for the client, the msgId is generated by the producer instance. Specifically, the method `MessageClientIDSetter.createUniqIDBuffer()` is called to generate a unique Id.\n- offsetMsgId, offsetMsgId is generated by the Broker server when writing a message ( string consists of \"IP address + port\" and \"CommitLog's physical offset address\"), and offsetMsgId is the messageId used to query in the RocketMQ console.\n"
  },
  {
    "path": "docs/en/acl/Operations_ACL.md",
    "content": "# Access control list\n## Overview\nThis document focuses on how to quickly deploy and use a RocketMQ cluster that supports the privilege control feature.\n\n## 1. Access control features\nAccess Control (ACL) mainly provides Topic resource level user access control for RocketMQ.If you want to enable RocketMQ permission control, you can inject the AccessKey and SecretKey signatures through the RPCHook on the Client side.And then, the corresponding permission control attributes (including Topic access rights, IP whitelist and AccessKey and SecretKey signature) are set in the configuration file of distribution/conf/plain_acl.yml.The Broker side will check the permissions owned by the AccessKey, and if the verification fails, an exception is thrown;\nThe source code about ACL on the Client side can be find in **org.apache.rocketmq.example.simple.AclClient.java**  \n\n## 2. Access control definition and attribute values\n### 2.1 Access control definition\nThe definition of Topic resource access control for RocketMQ is mainly as shown in the following table.\n\n| Permission | explanation |\n| --- | --- |\n| DENY | permission deny |\n| ANY | PUB or SUB permission |\n| PUB | Publishing permission |\n| SUB | Subscription permission |\n\n### 2.2 Main properties\n| key | value | explanation |\n| --- | --- | --- |\n| globalWhiteRemoteAddresses | string |Global IP whitelist,example:<br>\\*; <br>192.168.\\*.\\*; <br>192.168.0.1 |\n| accessKey | string | Access Key |\n| secretKey | string | Secret Key |\n| whiteRemoteAddress | string | User IP whitelist,example:<br>\\*; <br>192.168.\\*.\\*; <br>192.168.0.1 |\n| admin | true;false | Whether an administrator account |\n| defaultTopicPerm | DENY;PUB;SUB;PUB\\|SUB | Default Topic permission |\n| defaultGroupPerm | DENY;PUB;SUB;PUB\\|SUB | Default ConsumerGroup permission |\n| topicPerms | topic=permission | Topic only permission |\n| groupPerms | group=permission | ConsumerGroup only permission |\n\nFor details, please refer to the **distribution/conf/plain_acl.yml** configuration file.\n\n## 3. Cluster deployment with permission control\nAfter defining the permission attribute in the **distribution/conf/plain_acl.yml** configuration file as described above, open the **aclEnable** switch variable to enable the ACL feature of the RocketMQ cluster.The configuration file of the ACL feature enabled on the broker is as follows:\n```properties\nbrokerClusterName=DefaultCluster\nbrokerName=broker-a\nbrokerId=0\ndeleteWhen=04\nfileReservedTime=48\nbrokerRole=ASYNC_MASTER\nflushDiskType=ASYNC_FLUSH\nstorePathRootDir=/data/rocketmq/rootdir-a-m\nstorePathCommitLog=/data/rocketmq/commitlog-a-m\nautoCreateSubscriptionGroup=true\n## if acl is open,the flag will be true\naclEnable=true\nlistenPort=10911\nbrokerIP1=XX.XX.XX.XX1\nnamesrvAddr=XX.XX.XX.XX:9876\n```\n## 4. Main process of access control\nThe main ACL process is divided into two parts, including privilege resolution and privilege check.\n\n### 4.1 Privilege resolution\nThe Broker side parses the client's RequestCommand request and obtains the attribute field that needs to be authenticated.\nmain attributes:  \n (1) AccessKey:Similar to the user name, on behalf of the user entity, the permission data corresponds to it;  \n (2) Signature:The client obtains the string according to the signature of the SecretKey, and the server uses the SecretKey to perform signature verification.\n\n### 4.2 Privilege check\nThe check logic of the right side of the broker is mainly divided into the following steps:  \n (1) Check if the global IP whitelist is hit; if yes, the check passes; otherwise, go to step (2);  \n (2) Check if the user IP whitelist is hit; if yes, the check passes; otherwise, go to step (3);  \n (3) Check the signature, if the verification fails, throw an exception; if the verification passes, go to step (4);  \n (4) Check the permissions required by the user request and the permissions owned by the user; if not, throw an exception;  \n\n\nThe verification of the required permissions of the user requires attention to the following points:  \n (1) Special requests such as UPDATE_AND_CREATE_TOPIC can only be operated by the admin account;  \n (2) For a resource, if there is explicit configuration permission, the configured permission is used; if there is no explicit configuration permission, the default permission is adopted;\n\n## 5. Hot loading modified Access control\nThe default implementation of RocketMQ's permission control store is based on the yml configuration file. Users can dynamically modify the properties defined by the permission control without restarting the Broker service node.\n"
  },
  {
    "path": "docs/en/architecture.md",
    "content": "# Architecture design\n\n## Technology Architecture\n![](image/rocketmq_architecture_1.png)\n\nThe RocketMQ architecture is divided into four parts, as shown in the figure above:\n\n\n- Producer: The role of message publishing supports distributed cluster mode deployment. Producer selects the corresponding Broker cluster queue for message delivery through MQ's load balancing module. The delivery process supports fast failure and low latency.\n\n- Consumer: The role of message consumption supports distributed cluster deployment. Support push, pull two modes to consume messages. It also supports cluster mode and broadcast mode consumption, and it provides a real-time message subscription mechanism to meet the needs of most users.\n\n- NameServer: NameServer is a very simple Topic routing registry with a role similar to ZooKeeper in Dubbo, which supports dynamic registration and discovery of Broker. It mainly includes two functions: Broker management, NameServer accepts the registration information of the Broker cluster and saves it as the basic data of the routing information. Then provide a heartbeat detection mechanism to check whether the broker is still alive; routing information management, each NameServer will save the entire routing information about the Broker cluster and the queue information for the client query. Then the Producer and Consumer can know the routing information of the entire Broker cluster through the NameServer, so as to deliver and consume the message. The NameServer is usually deployed in a cluster mode, and each instance does not communicate with each other. Broker registers its own routing information with each NameServer, so each NameServer instance stores a complete routing information. When a NameServer is offline for some reason, the Broker can still synchronize its routing information with other NameServers. The Producer and Consumer can still dynamically sense the information of the Broker's routing.\n\n- BrokerServer: Broker is responsible for the storage, delivery and query of messages and high availability guarantees. In order to achieve these functions, Broker includes the following important sub-modules.\n1. Remoting Module: The entire broker entity handles requests from the clients side.\n2. Client Manager: Topic subscription information for managing the client (Producer/Consumer) and maintaining the Consumer\n3. Store Service: Provides a convenient and simple API interface for handling message storage to physical hard disks and query functions.\n4. HA Service: Highly available service that provides data synchronization between Master Broker and Slave Broker.\n5. Index Service: The message delivered to the Broker is indexed according to a specific Message key to provide a quick query of the message.\n\n![](image/rocketmq_architecture_2.png)\n\n## Deployment architecture\n\n\n![](image/rocketmq_architecture_3.png)\n\n\n### RocketMQ Network deployment features\n\n- NameServer is an almost stateless node that can be deployed in a cluster without any information synchronization between nodes.\n\n- The broker deployment is relatively complex. The Broker is divided into the Master and the Slave. One Master can correspond to multiple Slaves. However, one Slave can only correspond to one Master. The correspondence between the Master and the Slave is defined by specifying the same BrokerName and different BrokerId. The BrokerId 0 indicates Master, non-zero means Slave. The Master can also deploy multiple. Each broker establishes a long connection with all nodes in the NameServer cluster, and periodically registers Topic information to all NameServers. Note: The current RocketMQ version supports a Master Multi Slave on the deployment architecture, but only the slave server with BrokerId=1 will participate in the read load of the message.\n\n- The Producer establishes a long connection with one of the nodes in the NameServer cluster (randomly selected), periodically obtains Topic routing information from the NameServer, and establishes a long connection to the Master that provides the Topic service, and periodically sends a heartbeat to the Master. Producer is completely stateless and can be deployed in a cluster.\n\n- The Consumer establishes a long connection with one of the nodes in the NameServer cluster (randomly selected), periodically obtains Topic routing information from the NameServer, and establishes a long connection to the Master and Slave that provides the Topic service, and periodically sends heartbeats to the Master and Slave. The Consumer can subscribe to the message from the Master or subscribe to the message from the Slave. When the consumer pulls the message to the Master, the Master server will generate a read according to the distance between the offset and the maximum offset. I/O), and whether the server is readable or not, the next time it is recommended to pull from the Master or Slave.\n\nDescribe the cluster workflow in conjunction with the deployment architecture diagram:\n\n- Start the NameServer, listen to the port after the NameServer, and wait for the Broker, Producer, and Consumer to connect, which is equivalent to a routing control center.\n- The Broker starts, keeps a long connection with all NameServers, and sends heartbeat packets periodically. The heartbeat packet contains the current broker information (IP+ port, etc.) and stores all Topic information. After the registration is successful, there is a mapping relationship between Topic and Broker in the NameServer cluster.\n- Before sending and receiving a message, create a Topic. When creating a Topic, you need to specify which Brokers the Topic should be stored on, or you can automatically create a Topic when sending a message.\n- Producer sends a message. When starting, it first establishes a long connection with one of the NameServer clusters, and obtains from the NameServer which Brokers are currently sent by the Topic. Polling selects a queue from the queue list and then establishes with the broker where the queue is located. Long connection to send a message to the broker.\n- The Consumer is similar to the Producer. It establishes a long connection with one of the NameServers, obtains which Brokers the current Topic exists on, and then directly establishes a connection channel with the Broker to start consuming messages.\n"
  },
  {
    "path": "docs/en/best_practice.md",
    "content": "#  Best practices\r\n\r\n## 1 Producer\r\n### 1.1 Attention of send message\r\n\r\n#### 1  Uses of tags\r\nAn application should use one topic as far as possible, but identify the message's subtype with tags. Tags can be set freely by the application.\r\nOnly when producers set tags while sending messages, can consumers filter messages through broker with tags when subscribing messages: message.setTags(\"TagA\").  \r\n \r\n#### 2 Uses of keys\r\nThe unique identifier for each message at the business level set to the Keys field to help locate message loss problems in the future.\r\nThe server creates an index(hash index) for each message, and the application can query the message content via Topic,key,and who consumed the message.\r\nSince it is a hash index, make sure that the key is as unique as possible to avoid potential hash conflicts.\r\n\r\n```java\r\n   // order id \r\n   String orderId = \"20034568923546\";   \r\n   message.setKeys(orderId);   \r\n```\r\nIf you have multiple keys for a message, please concatenate them with 'KEY_SEPARATOR' char, as shown below:\r\n```java\r\n   // order id \r\n   String orderId = \"20034568923546\";\r\n   String otherKey = \"19101121210831\";\r\n   String keys = new StringBuilder(orderId)\r\n           .append(org.apache.rocketmq.common.message.MessageConst.KEY_SEPARATOR)\r\n           .append(otherKey).toString();\r\n   message.setKeys(keys);\r\n```\r\nAnd if you want to query the message, please use `orderId` and `otherKey` to query respectively instead of `keys`, \r\nbecause the server will unwrap `keys` with `KEY_SEPARATOR` and create corresponding index.\r\nIn the above example, the server will create two indexes, one for `orderId` and one for `otherKey`.\r\n#### 3 Log print\r\nPrint the message log when send success or failed, make sure to print the SendResult and key fields. \r\nSend messages is successful as long as it does not throw exception. Send successful will have multiple states defined in sendResult.\r\nEach state is describing below:     \r\n\r\n- **SEND_OK**\r\n\r\nMessage send successfully. Note that even though message send successfully, it doesn't mean that it is reliable.\r\nTo make sure nothing lost, you should also enable the SYNC_MASTER or SYNC_FLUSH.\r\n\r\n- **FLUSH_DISK_TIMEOUT**\r\n\r\nMessage send successfully, but the server flush messages to disk timeout. At this point, the message has entered the server's memory, and the message will be lost only when the server is down.\r\nFlush mode and sync flush time interval can be set in the configuration parameters. It will return FLUSH_DISK_TIMEOUT when Broker server doesn't finish flush message to disk in timeout(default is 5s\r\n) when sets FlushDiskType=SYNC_FLUSH(default is async flush).\r\n\r\n- **FLUSH_SLAVE_TIMEOUT**\r\n\r\nMessage send successfully, but sync to slave timeout. At this point, the message has entered the server's memory, and the message will be lost only when the server is down.\r\nIt will return FLUSH_SLAVE_TIMEOUT when Broker server role is SYNC_MASTER(default is ASYNC_MASTER),and it doesn't sync message to slave successfully in the timeout(default 5s).\r\n\r\n- **SLAVE_NOT_AVAILABLE**\r\n\r\nMessage send successfully, but slave is not available. It will return SLAVE_NOT_AVAILABLE when Broker role is SYNC_MASTER(default is ASYNC_MASTER), and it doesn't have a slave server available. \r\n\r\n### 1.2 Handling of message send failure\r\nSend method of producer itself supports internal retry. The logic of retry is as follows:\r\n- At most twice.\r\n- Try next broker when sync send mode, try current broker when async mode. The total elapsed time of this method does not exceed the value of sendMsgTimeout(default is 10s).\r\n- It will not be retried when the message is sent to the Broker with a timeout exception.\r\n\r\nThe strategy above ensures the success of message sending to some extent.If the business has a high requirement for message reliability, it is recommended to add the corresponding retry logic:\r\nfor example, if the sync send method fails, try to store the message in DB, and then retry periodically by the bg thread to ensure the message must send to broker successfully. \r\n\r\nWhy the above DB retry strategy is not integrated into the MQ client, but requires the application to complete it by itself is mainly based on the following considerations:\r\nFirst, the MQ client is designed to be stateless mode, convenient for arbitrary horizontal expansion, and only consumes CPU, memory and network resources.\r\nSecond, if the MQ client internal integration a KV storage module, the data can only be relatively reliable when sync flush to disk, but the sync flush will cause performance lose, so it's usually\r\n use async flush.Because the application shutdown is not controlled by the MQ operators, A violent shutdown like kill -9 may often occur, resulting in data not flushed to disk and being lost.\r\nThirdly, the producer is a virtual machine with low reliability, which is not suitable for storing important data.\r\nIn conclusion, it is recommended that the retry process must be controlled by the application.\r\n\r\n### 1.3 Send message by oneway\r\nTypically, this is the process by which messages are sent:\r\n\r\n- Client send request to server\r\n- Server process request\r\n- Server response to client \r\nSo, the time taken to send a message is the sum of the three steps above. Some scenarios require very little time, but not much reliability, such as log collect application.\r\nThis type application can use oneway to send messages. Oneway only send request without waiting for a reply, and send a request at the client implementation level is simply the overhead of an\r\n operating system call that writes data to the client's socket buffer, this process that typically takes microseconds.\r\n\r\n## 2 Consumer\r\n\r\n## 3 Broker\r\n\r\n### 3.1 Broker Role\r\n\r\n### 3.2 FlushDiskType\r\n\r\n### 3.3 Broker Configuration\r\n| Parameter name                           | Default                        | Description                                                         |\r\n| -------------------------------- | ----------------------------- | ------------------------------------------------------------ |\r\n| listenPort                    | 10911              | listen port for client |\r\n| namesrvAddr       | null                         | name server address     |\r\n| brokerIP1 | InetAddress for network interface                         | Should be configured if having multiple addresses |\r\n| brokerIP2 | InetAddress for network interface                         | If configured for the Master broker in the Master/Slave cluster, slave broker will connect to this port for data synchronization   |\r\n| brokerName        | null                         | broker name                           |\r\n| brokerClusterName                     | DefaultCluster                  | this broker belongs to which cluster           |\r\n| brokerId             | 0                              | broker id, 0 means master, positive integers mean slave                                                 |\r\n| storePathRootDir                         | $HOME/store/                   | file path for root store                                            |\r\n| storePathCommitLog                      | $HOME/store/commitlog/                              | file path for commit log                                                 |\r\n| mappedFileSizeCommitLog     | 1024 * 1024 * 1024(1G) | mapped file size for commit log                                        |​ \r\n| deleteWhen     | 04 | When to delete the commitlog which is out of the reserve time                                        |​ \r\n| fileReserverdTime     | 72 | The number of hours to keep a commitlog before deleting it                                        |​ \r\n| brokerRole     | ASYNC_MASTER | SYNC_MASTER/ASYNC_MASTER/SLAVE                                        |​ \r\n| flushDiskType     | ASYNC_FLUSH | {SYNC_FLUSH/ASYNC_FLUSH}. Broker of SYNC_FLUSH mode flushes each message onto disk before acknowledging producer. Broker of ASYNC_FLUSH mode, on the other hand, takes advantage of group-committing, achieving better performance.                                        |​"
  },
  {
    "path": "docs/en/client/java/API_Reference_DefaultMQProducer.md",
    "content": "## DefaultMQProducer\n---\n### Class introduction\n\n`public class DefaultMQProducer \nextends ClientConfig \nimplements MQProducer`\n\n>`DefaultMQProducer` is the entry point for an application to post messages, out of the box, ca  quickly create a producer with a no-argument construction. it is mainly responsible for message sending, support synchronous、asynchronous、one-way send. All of these send methods support batch send.  The parameters of the sender can be adjusted through the getter/setter methods , provided by this class. `DefaultMQProducer` has multi send method and each method is slightly different. Make  sure you know the usage before you use it . Blow is a producer example . [see more examples](https://github.com/apache/rocketmq/blob/master/example/src/main/java/org/apache/rocketmq/example/).\n\n``` java \npublic class Producer {\n    public static void main(String[] args) throws MQClientException {\n        // create a produce with producer_group_name\n        DefaultMQProducer producer = new DefaultMQProducer(\"ProducerGroupName\");\n\n        // start the producer\n        producer.start();\n\n        for (int i = 0; i < 128; i++)\n            try {\n            \t// construct the msg\n                Message msg = new Message(\"TopicTest\",\n                        \"TagA\",\n                        \"OrderID188\",\n                        \"Hello world\".getBytes(RemotingHelper.DEFAULT_CHARSET));\n\n                // send sync\n                SendResult sendResult = producer.send(msg);\n\n                // print the result\n                System.out.printf(\"%s%n\", sendResult);\n            } catch (Exception e) {\n                e.printStackTrace();\n            }\n\n        producer.shutdown();\n    }\n}\n```\n\n**Note** : This class is thread safe. It can be safely shared between multiple threads after configuration and startup is complete.\n\n###  Variable \n\n|Type|Name| description                                                  |\n|------|-------|-------|\n|DefaultMQProducerImpl|defaultMQProducerImpl|The producer's internal default implementation|\n|String|producerGroup|The producer's group|\n|String|createTopicKey| Topics that do not exist on the server are automatically created when the message is sent |\n|int|defaultTopicQueueNums|The default number of queues to create a topic|\n|int|sendMsgTimeout|The timeout for the message to be sent|\n|int|compressMsgBodyOverHowmuch|the threshold of the compress of  message body|\n|int|retryTimesWhenSendFailed|Maximum number of internal attempts to send a message in synchronous mode|\n|int|retryTimesWhenSendAsyncFailed|Maximum number of internal attempts to send a message in asynchronous mode|\n|boolean|retryAnotherBrokerWhenNotStoreOK|Whether to retry another broker if an internal send fails|\n|int|maxMessageSize| Maximum length of message body                                   |\n|TraceDispatcher|traceDispatcher| Message trackers. Use rcpHook to track messages              |\n\n### construction method \n\n|Method name|Method description|\n|-------|------------|\n|DefaultMQProducer()| creates a producer with default parameter values             |\n|DefaultMQProducer(final String producerGroup)| creates a producer with producer group name.                 |\n|DefaultMQProducer(final String producerGroup, boolean enableMsgTrace)|creates a producer with producer group name and set whether to enable message tracking|\n|DefaultMQProducer(final String producerGroup, boolean enableMsgTrace, final String customizedTraceTopic)|creates a producer with producer group name and set whether to enable message tracking、the trace topic.|\n|DefaultMQProducer(RPCHook rpcHook)|creates a producer with  a rpc hook.|\n|DefaultMQProducer(final String producerGroup, RPCHook rpcHook)|creates a producer with  a rpc hook and producer group.|\n|DefaultMQProducer(final String producerGroup, RPCHook rpcHook, boolean enableMsgTrace,final String customizedTraceTopic)|all of above.|\n\n"
  },
  {
    "path": "docs/en/controller/deploy.md",
    "content": "# Deployment and upgrade guidelines\n\n## Controller deployment \n\n If the controller needs to be fault-tolerant, it needs to be deployed in three or more replicas (following the Raft majority protocol).\n\n> Controller can also complete Broker Failover with only one deployment, but if the single point Controller fails, it will affect the switching ability, but will not affect the normal reception and transmission of the existing cluster.\n\nThere are two ways to deploy Controller. One is to embed it in NameServer for deployment, which can be opened through the configuration enableControllerInNamesrv (it can be opened selectively and is not required to be opened on every NameServer). In this mode, the NameServer itself is still stateless, that is, if the NameServer crashes in the embedded mode, it will only affect the switching ability and not affect the original routing acquisition and other functions. The other is independent deployment, which requires separate deployment of the controller.\n\n### Embed NameServer deployment\n\nWhen embedded in NameServer deployment, you only need to set `enableControllerInNamesrv=true` in the NameServer configuration file and fill in the controller configuration.\n\n```\nenableControllerInNamesrv = true\ncontrollerDLegerGroup = group1\ncontrollerDLegerPeers = n0-127.0.0.1:9877;n1-127.0.0.1:9878;n2-127.0.0.1:9879\ncontrollerDLegerSelfId = n0\ncontrollerStorePath = /home/admin/DledgerController\nenableElectUncleanMaster = false\nnotifyBrokerRoleChanged = true\n```\n\nParameter explain：\n\n- enableControllerInNamesrv: Whether to enable controller in Nameserver, default is false.\n- controllerDLegerGroup: The name of the DLedger Raft Group, all nodes in the same DLedger Raft Group should be consistent.\n- controllerDLegerPeers: The port information of the nodes in the DLedger Group, the configuration of each node in the same Group must be consistent.\n- controllerDLegerSelfId: The node id, must belong to one of the controllerDLegerPeers; unique within the Group.\n- controllerStorePath: The location to store controller logs. Controller is stateful and needs to rely on logs to recover data when restarting or crashing, this directory is very important and should not be easily deleted.\n- enableElectUncleanMaster: Whether it is possible to elect Master from outside SyncStateSet, if true, it may select a replica with lagging data as Master and lose messages, default is false.\n- notifyBrokerRoleChanged: Whether to actively notify when the role of the broker replica group changes, default is true.\n\nSome other parameters can be referred to in the ControllerConfig code.\n\nAfter setting the parameters, start the Nameserver by specifying the configuration file.\n\n### Independent deployment\n\nTo deploy independently, execute the following script:\n\n```shell\nsh bin/mqcontroller -c controller.conf\n```\nThe mqcontroller script is located at distribution/bin/mqcontroller, and the configuration parameters are the same as in embedded mode.\n\n## Broker Controller mode deployment\n\nThe Broker start method is the same as before, with the following parameters added:\n\n- enableControllerMode: The overall switch for the Broker controller mode, only when this value is true will the controller mode be opened. Default is false.\n- controllerAddr: The address of the controller, separated by semicolons if there are multiple controllers. For example, `controllerAddr = 127.0.0.1:9877;127.0.0.1:9878;127.0.0.1:9879`\n- syncBrokerMetadataPeriod: The interval for synchronizing Broker replica information with the controller. Default is 5000 (5s).\n- checkSyncStateSetPeriod: The interval for checking SyncStateSet, checking SyncStateSet may shrink SyncState. Default is 5000 (5s).\n- syncControllerMetadataPeriod: The interval for synchronizing controller metadata, mainly to obtain the address of the active controller. Default is 10000 (10s).\n- haMaxTimeSlaveNotCatchup: The maximum interval that a slave has not caught up to the Master, if a slave in SyncStateSet exceeds this interval, it will be removed from SyncStateSet. Default is 15000 (15s).\n- storePathEpochFile: The location to store the epoch file. The epoch file is very important and should not be deleted arbitrarily. Default is in the store directory.\n- allAckInSyncStateSet: If this value is true, a message needs to be replicated to each replica in SyncStateSet before it is returned to the client as successful, ensuring that the message is not lost. Default is false.\n- syncFromLastFile: If the slave is a blank disk start, whether to replicate from the last file. Default is false.\n- asyncLearner: If this value is true, the replica will not enter SyncStateSet, that is, it will not be elected as Master, but will always be a learner replica that performs asynchronous replication. Default is false.\n- inSyncReplicas: The number of replica groups that need to be kept in sync, default is 1, inSyncReplicas is invalid when allAckInSyncStateSet=true.\n- minInSyncReplicas: The minimum number of replica groups that need to be kept in sync, if the number of replicas in SyncStateSet is less than minInSyncReplicas, putMessage will return PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH directly, default is 1.\n\nIn Controller mode, the Broker configuration must set `enableControllerMode=true` and fill in controllerAddr.\n\n### Analysis of important parameters\n\nAmong the parameters such as inSyncReplicas and minInSyncReplicas, there are overlapping and different meanings in normal Master-Slave deployment, SlaveActingMaster mode, and automatic master-slave switching architecture. The specific differences are as follows:\n\n|                      | inSyncReplicas                                                      | minInSyncReplicas                                                        | enableAutoInSyncReplicas                    | allAckInSyncStateSet                                          | haMaxGapNotInSync                     | haMaxTimeSlaveNotCatchup                          |\n|----------------------|---------------------------------------------------------------------|--------------------------------------------------------------------------|---------------------------------------------|---------------------------------------------------------------|---------------------------------------|---------------------------------------------------|\n| Normal Master-Slave deployment | The number of replicas that need to be ACKed in synchronous replication, invalid in asynchronous replication | invalid                                                                | invalid                                   | invalid                                                     | invalid                             | invalid                                         |\n| Enable SlaveActingMaster （slaveActingMaster=true） | The number of replicas that need to be ACKed in synchronous replication in the absence of auto-degradation | The minimum number of replicas that need to be ACKed after auto-degradation | Whether to enable auto-degradation, and the minimum number of replicas that need to be ACKed after auto-degradation is reduced to minInSyncReplicas | invalid                                                     | Basis for degradation determination: the difference in Commitlog heights between Slave and Master, in bytes | invalid                                         |\n| Automatic master-slave switching architecture（enableControllerMode=true） | The number of replicas that need to be ACKed in synchronous replication when allAckInSyncStateSet is not enabled, and this value is invalid when allAckInSyncStateSet is enabled | SyncStateSet can be reduced to the minimum number of replicas, and if the number of replicas in SyncStateSet is less than minInSyncReplicas, it will return directly with insufficient number of replicas | invalid                                   | If this value is true, a message needs to be replicated to every replica in SyncStateSet before it is returned to the client as successful, and this parameter can ensure that the message is not lost | invalid                      | The minimum time difference between Slave and Master when SyncStateSet is contracted, see [RIP-44](https://shimo.im/docs/N2A1Mz9QZltQZoAD) for details. |\n\nTo summarize：\n- In a normal Master-Slave configuration, there is no ability for auto-degradation, and all parameters except for inSyncReplicas are invalid. inSyncReplicas indicates the number of replicas that need to be ACKed in synchronous replication.\n- In slaveActingMaster mode, enabling enableAutoInSyncReplicas enables the ability for degradation, and the minimum number of replicas that can be degraded to is minInSyncReplicas. The basis for degradation is the difference in Commitlog heights (haMaxGapNotInSync) and the survival of the replicas, refer to  [SlaveActingMaster mode adaptive degradation](../QuorumACK.md).\n- Automatic master-slave switching (Controller mode) relies on SyncStateSet contraction for auto-degradation. SyncStateSet replicas can work normally as long as they are contracted to a minimum of minInSyncReplicas. If it is less than minInSyncReplicas, it will return directly with insufficient number of replicas. One of the basis for contraction is the time interval (haMaxTimeSlaveNotCatchup) at which the Slave catches up, rather than the Commitlog height. If allAckInSyncStateSet=true, the inSyncReplicas parameter is invalid.\n\n## Compatibility\n\nThis mode does not make any changes or modifications to any client-level APIs, and there are no compatibility issues with clients.\n\nThe Nameserver itself has not been modified and there are no compatibility issues with the Nameserver. If enableControllerInNamesrv is enabled and the controller parameters are configured correctly, the controller function is enabled.\n\nIf Broker is set to **`enableControllerMode=false`**, it will still operate as before. If **`enableControllerMode=true`**, the Controller must be deployed and the parameters must be configured correctly in order to operate properly.\n\nThe specific behavior is shown in the following table:\n\n|                                    | Old nameserver                  | Old nameserver + Deploy controllers independently | New nameserver enables controller | New nameserver disable controller |\n| ---------------------------------- | ------------------------------- | ------------------------------------------------- | --------------------------------- | --------------------------------- |\n| Old broker                         | Normal running, cannot failover | Normal running, cannot failover                   | Normal running, cannot failover   | Normal running, cannot failover   |\n| New broker enable controller mode  | Unable to go online normally    | Normal running, can failover                      | Normal running, can failover      | Unable to go online normally      |\n| New broker disable controller mode | Normal running, cannot failover | Normal running, cannot failover                   | Normal running, cannot failover   | Normal running, cannot failover   |\n\n## Upgrade Considerations\n\nFrom the compatibility statements above, it can be seen that NameServer can be upgraded normally without compatibility issues. In the case where the Nameserver is not to be upgraded, the controller component can be deployed independently to obtain switching capabilities. For broker upgrades, there are two cases:\n\n1. Master-Slave deployment is upgraded to controller switching architecture\n\n   In-place upgrade with data is possible. For each group of Brokers, stop the primary and secondary Brokers and ensure that the CommitLogs of the primary and secondary are aligned (you can either disable writing to this group of Brokers for a certain period of time before the upgrade or ensure consistency by copying). After upgrading the package, restart it.\n\n   > If the primary and secondary CommitLogs are not aligned, it is necessary to ensure that the primary is online before the secondary is online, otherwise messages may be lost due to data truncation.\n\n2. Upgrade from DLedger mode to Controller switching architecture\n\n   Due to the differences in the format of message data in DLedger mode and Master-Slave mode, there is no in-place upgrade with data. In the case of deploying multiple groups of Brokers, it is possible to disable writing to a group of Brokers for a certain period of time (as long as it is confirmed that all existing messages have been consumed), and then upgrade and deploy the Controller and new Brokers. In this way, the new Brokers will consume messages from the existing Brokers and the existing Brokers will consume messages from the new Brokers until the consumption is balanced, and then the existing Brokers can be decommissioned.\n\n### Upgrade considerations for persistent BrokerID version\n\nThe current version supports a new high-availability architecture with persistent BrokerID version. Upgrading from version 5.x to the current version requires the following considerations:\n\nFor version 4.x, follow the above procedure to upgrade.\n\nFor upgrading from non-persistent BrokerID version in 5.x to persistent BrokerID version, follow the procedure below:\n\n**Upgrade Controller**\n\n1. Stop the old version Controller group.\n2. Clear Controller data, i.e., data files located in `~/DLedgerController` by default.\n3. Bring up the new version Controller group. \n\n> During the Controller upgrade process, Broker can still run normally but cannot failover.\n\n**Upgrade Broker**\n\n1. Stop the secondary Broker.\n2. Stop the primary Broker.\n3. Delete all Epoch files of all Brokers, i.e., `~/store/epochFileCheckpoint` and `~/store/epochFileCheckpoint.bak` by default.\n4. Bring up the original primary Broker and wait for it to be elected as master (you can use the `getSyncStateSet` command of admin to observe).\n5. Bring up all the original secondary Brokers.\n\n> It is recommended to stop the secondary Broker before stopping the primary Broker and bring up the original primary Broker before the original secondary during the online process. This can ensure the original primary-secondary relationship.\n> If you need to change the primary-secondary relationship before and after the upgrade, make sure that the CommitLog of the primary and secondary are aligned when shutting down. Otherwise, data may be truncated and lost."
  },
  {
    "path": "docs/en/controller/design.md",
    "content": "# Background\n\nIn the current RocketMQ Raft mode, the DLedger Commitlog is mainly used to replace the original Commitlog, enabling the Commitlog to have the ability to elect and replicate. However, this also causes some problems:\n\n- In the Raft mode, the number of replicas within the Broker group must be three or more, and the ACK of the replicas must also follow the majority protocol.\n- RocketMQ has two sets of HA replication processes, and the replication in Raft mode cannot utilize RocketMQ's native storage capability.\n\nTherefore, we hope to use DLedger to implement a consistency module (DLedger Controller) based on Raft, and use it as an optional leader election component. It can be deployed independently or embedded in the Nameserver. The Broker completes the election of the Master through interaction with the Controller, thus solving the above problems. We refer to this new mode as the Controller mode.\n\n# Architecture\n\n### Core idea\n\n![架构图](../image/controller/controller_design_1.png)\n\n- The following is a description of the core architecture of the Controller mode, as shown in the figure:\n  - DledgerController: Using DLedger, a DLedger controller that ensures the consistency of metadata is constructed. The Raft election will select an Active DLedger Controller as the main controller. The DLedger Controller can be embedded in the Nameserver or deployed independently. Its main function is to store and manage the SyncStateSet list of Brokers, and actively issue scheduling instructions to switch the Master of the Broker when the Master of the Broker is offline or network isolated.\n  - SyncStateSet: Mainly represents a set of Slave replicas following the Master in a broker replica group, with the main criterion for judgment being the gap between the Master and the Slave. When the Master is offline, we will select a new Master from the SyncStateSet list. The SyncStateSet list is mainly initiated by the Master Broker. The Master completes the Shrink and Expand of the SyncStateSet through a periodic task to determine and synchronize the SyncStateSet, and initiates an Alter SyncStateSet request to the election component Controller.\n  - AutoSwitchHAService: A new HAService that, based on DefaultHAService, supports the switching of BrokerRole and the mutual conversion between Master and Slave (under the control of the Controller). In addition, this HAService unifies the log replication process and truncates the logs during the HA HandShake stage.\n  - ReplicasManager: As an intermediate component, it serves as a link between the upper and lower levels. Upward, it can regularly synchronize control instructions from the Controller, and downward, it can regularly monitor the state of the HAService and modify the SyncStateSet at the appropriate time. The ReplicasManager regularly synchronizes metadata about the Broker from the Controller, and when the Controller elects a new Master, the ReplicasManager can detect the change in metadata and switch the BrokerRole.\n\n## DLedgerController core design\n\n![image-20220605213143645](../image/controller/quick-start/controller.png)\n\n- The following is a description of the core design of the DLedgerController:\n  - DLedgerController can be embedded in Namesrv or deployed independently.\n  - Active DLedgerController is the Leader elected by DLedger. It will accept event requests from clients and initiate consensus through DLedger, and finally apply them to the in-memory metadata state machine.\n  - Not Active DLedgerController, also known as the Follower role, will replicate the event logs from the Active DLedgerController through DLedger and then apply them directly to the state machine.\n\n## Log replication\n\n### Basic concepts and processes\n\nIn order to unify the log replication process, distinguish the log replication boundary of each Master, and facilitate log truncation, the concept of MasterEpoch is introduced, which represents the current Master's term number (similar to the meaning of Raft Term).\n\nFor each Master, it has a MasterEpoch and a StartOffset, which respectively represent the term number and the starting log offset of the Master.\n\nIt should be noted that the MasterEpoch is determined by the Controller and is monotonically increasing.\n\nIn addition, we have introduced the EpochFile, which is used to store the \\<Epoch, StartOffset\\> sequence.\n\n**When a Broker becomes the Master, it will:**\n\n- Truncate the Commitlog to the boundary of the last message.\n- Persist the latest \\<MasterEpoch, startoffset\\> to the EpochFile, where startOffset is the current CommitLog's MaxPhyOffset.\n- Then the HAService listens for connections and creates the HAConnection to interact with the Slave to complete the process.\n\n**When a Broker becomes the Slave, it will:**\n\nReady stage:\n\n- Truncate the Commitlog to the boundary of the last message.\n- Establish a connection with the Master.\n\nHandshake stage:\n\n- Perform log truncation, where the key is for the Slave to compare its local epoch and startOffset with the Master to find the log truncation point and perform log truncation.\n\nTransfer stage:\n\n- Synchronize logs from the Master.\n\n### Truncation algorithm\n\nThe specific log truncation algorithm flow is as follows:\n\n- During the Handshake stage, the Slave obtains the Master's EpochCache from the Master.\n- The Slave compares the obtained Master EpochCache \\<Startoffset, Endoffset\\>, and compares them with the local cache from back to front. If the Epoch and StartOffset of the two are equal, the Epoch is valid, and the truncation point is the smaller Endoffset between them. After truncation, the \\<Epoch, Startoffse\\t> information is corrected and enters the Transfer stage. If they are not equal, the previous epoch of the Slave is compared until the truncation point is found.\n\n```java\nslave：TreeMap<Epoch, Pair<startOffset,endOffset>> epochMap;\nIterator iterator = epochMap.entrySet().iterator();\ntruncateOffset = -1;\n\n//The epochs are sorted from largest to smallest\nwhile (iterator.hasNext()) {\n    Map.Entry<Epoch, Pair<startOffset,endOffset>> curEntry = iterator.next();\n    Pair<startOffset,endOffset> masterOffset=\n    findMasterOffsetByEpoch(curEntry.getKey());\n    \n    if(masterOffset != null && \n            curEntry.getKey().getObejct1() == masterOffset.getObejct1()) {\n        truncateOffset = Math.min(curEntry.getKey().getObejct2(), masterOffset.getObejct2());\n        break;\n   }\n}\n```\n\n### Replication process\n\nSince HA replicates logs based on stream, we cannot distinguish the boundaries of the logs (that is, a batch of transmitted logs may span multiple MasterEpochs), and the Slave cannot detect changes in MasterEpoch and cannot timely modify EpochFile.\n\nTherefore, we have made the following improvements:\n\nWhen the Master transfers logs, it ensures that a batch of logs sent at a time is in the same epoch, but not spanning multiple epochs. We can add two variables in WriteSocketService:\n\n- currentTransferEpoch: represents which epoch WriteSocketService.nextTransferFromWhere belongs to\n- currentTransferEpochEndOffset: corresponds to the end offset of currentTransferEpoch. If currentTransferEpoch == MaxEpoch, then currentTransferEpochEndOffset= -1, indicating no boundary.\n\nWhen WriteSocketService transfers the next batch of logs (assuming the total size of this batch is size), if it finds that nextTransferFromWhere + size > currentTransferEpochEndOffset, it sets selectMappedBufferResult limit to currentTransferEpochEndOffset. Finally, modify currentTransferEpoch and currentTransferEpochEndOffset to the next epoch.\n\nCorrespondingly, when the Slave receives logs, if it finds a change in epoch from the header, it records it in the local epoch file.\n\n### Replication protocol\n\nAccording to the above, we can know the AutoSwitchHaService protocol divides log replication into multiple stages. Below is the protocol for the HaService.\n\n#### Handshake stage\n\n1.AutoSwitchHaClient (Slave) will send a HandShake packet to the Master as follows：\n\n![示意图](../image/controller/controller_design_3.png)\n\n`current state(4byte) + Two flags(4byte) + slaveBrokerId(8byte)`\n\n- `Current state` represents the current HAConnectionState, which is HANDSHAKE.\n\n- Two flags are two status flags, where `isSyncFromLastFile` indicates whether to start copying from the Master's last file, and `isAsyncLearner` indicates whether the Slave is an asynchronous copy and joins the Master as a Learner.\n\n- `slaveBrokerId` represent the brokerId of the Slave, which will be used later to join the SyncStateSet.\n\n2.AutoSwitchHaConnection (Master) will send a HandShake packet back to the Slave as follows:\n\n![示意图](../image/controller/controller_design_4.png)\n\n`current state(4byte) + body size(4byte) + offset(8byte) + epoch(4byte) + body`\n\n- `Current state` represents the current HAConnectionState, which is HANDSHAKE.\n- `Body size` represents the length of the body.\n- `Offset` represents the maximum offset of the log on the Master side.\n- `Epoch` represents the Master's Epoch.\n- The Body contains the EpochEntryList on the Master side.\n\nAfter the Slave receives the packet sent back by the Master, it will perform the log truncation process described above locally.\n\n#### Transfer stage\n\n1.AutoSwitchHaConnection (Master) will continually send log packets to the Slave as follows：\n\n![示意图](../image/controller/controller_design_5.png)\n\n`current state(4byte) + body size(4byte) + offset(8byte) + epoch(4byte)  + epochStartOffset(8byte) + additionalInfo(confirmOffset) (8byte)+ body`\n\n- `Current state`: represents the current HAConnectionState, which is Transfer.\n- `Body size`: represents the length of the body.\n- `Offset`: the starting offset of the current batch of logs.\n- `Epoch`: represents the MasterEpoch to which the current batch of logs belongs.\n- `epochStartOffset`: represents the StartOffset of the MasterEpoch corresponding to the current batch of logs.\n- `confirmOffset`: represents the minimum offset among replicas in SyncStateSet.\n- `Body`: logs.\n\n2.AutoSwitchHaClient (Slave) will send an ACK packet to the Master：\n\n![示意图](../image/controller/controller_design_6.png)\n\n` current state(4byte) + maxOffset(8byte)`\n\n- `Current state`: represents the current HAConnectionState, which is Transfer.\n- `MaxOffset`: represents the current maximum log offset of the Slave.\n\n## Elect Master \n\n### Basic process\n\nELectMaster mainly selects a new Master from the SyncStateSet list when the Master of a Broker replica group is offline or inaccessible. This event is initiated by the Controller itself or through the `electMaster` operation command.\n\nWhether the Controller is deployed independently or embedded in Namesrv, it listens to the connection channels of each Broker. If a Broker channel becomes inactive, it checks whether the Broker is the Master, and if so, it triggers the Master election process.\n\nThe process of electing a Master is relatively simple. We just need to select one from the SyncStateSet list corresponding to the group of Brokers and make it the new Master, and apply the result to the in-memory metadata through the DLedger consensus. Finally, the result is notified to the corresponding Broker replica group.\n\n### SyncStateSet change\n\nSyncStateSet is an important basis for electing a Master. Changes to the SyncStateSet list are mainly initiated by the Master Broker. The Master completes the Shrink and Expand of SyncStateSet through a periodic task and initiates an Alter SyncStateSet request to the election component Controller during the synchronization process.\n\n#### Shrink\n\nShrink SyncStateSet refers to the removal of replicas from the SyncStateSet replica set that are significantly behind the Master, based on the following criteria:\n\n- Increase the haMaxTimeSlaveNotCatchUp parameter.\n- HaConnection records the last time the Slave caught up with the Master's timestamp, lastCaughtUpTimeMs, which means: every time the Master sends data (transferData) to the Slave, it records its current MaxOffset as lastMasterMaxOffset and the current timestamp lastTransferTimeMs.\n- When ReadSocketService receives slaveAckOffset, if slaveAckOffset >= lastMasterMaxOffset, it updates lastCaughtUpTimeMs to lastTransferTimeMs.\n- The Master scans each HaConnection through a periodic task and if (cur_time - connection.lastCaughtUpTimeMs) > haMaxTimeSlaveNotCatchUp, the Slave is Out-of-sync.\n- If a Slave is detected to be out of sync, the master immediately reports SyncStateSet to the Controller, thereby shrinking SyncStateSet.\n\n#### Expand\n\nIf a Slave replica catches up with the Master, the Master needs to timely alter SyncStateSet with the Controller. The condition for adding to SyncStateSet is slaveAckOffset >= ConfirmOffset (the minimum value of MaxOffset among all replicas in the current SyncStateSet).\n\n## Reference\n\n[RIP-44](https://github.com/apache/rocketmq/wiki/RIP-44-Support-DLedger-Controller)\n"
  },
  {
    "path": "docs/en/controller/persistent_unique_broker_id.md",
    "content": "# Persistent unique BrokerId\n\n## Current Issue\n\nCurrently, `BrokerAddress` is used as the unique identifier for the Broker in Controller mode, which causes the following problems：\n\n*   In a container environment, each restart or upgrade of the Broker may result in an IP address change, making it impossible to associate the previous `BrokerAddress` records with the restarted Broker, such as `ReplicaInfo`, `SyncStateSet`, and other data.\n\n## Improvement Plan\n\nIn the Controller side, `BrokerName:BrokerId` is used as the unique identifier instead of `BrokerAddress`. Also, `BrokerId` needs to be persistently stored. Since `ClusterName` and `BrokerName` are both configured in the configuration file when starting up, only the allocation and persistence of `BrokerId` need to be addressed.When the Broker first comes online, only the `ClusterName`, `BrokerName`, and its own `BrokerAddress` configured in the configuration file are available. Therefore, a unique identifier, `BrokerId`, that is determined throughout the lifecycle of the entire cluster needs to be negotiated with the Controller. The `BrokerId` is assigned starting from 1. When the Broker is selected as the Master, it will be re-registered in the Name Server, and at this point, to be compatible with the previous non-HA Master-Slave architecture, the `BrokerId` needs to be temporarily changed to 0 (where id 0 previously represented that the Broker was a Master).\n\n### Online Process\n\n![register process](../image/controller/persistent_unique_broker_id/register_process.png)\n\n#### 1. GetNextBrokerId Request\n\nSend a GetNextBrokerId request to the Controller to obtain the next available BrokerId (allocated starting from 1).\n\n#### 1.1 ReadFromDLedger\n\nUpon receiving the request, the Controller uses DLedger to retrieve the NextBrokerId data from the state machine.\n\n#### 2. GetNextBrokerId Response\n\nThe Controller returns the NextBrokerId to the Broker.\n\n#### 2.1 CreateTempMetaFile\n\nAfter receiving the NextBrokerId, the Broker creates a temporary file .broker.meta.temp, which records the NextBrokerId (the expected BrokerId to be applied) and generates a RegisterCode (used for subsequent identity verification), which is also persisted to the temporary file.\n\n#### 3. ApplyBrokerId Request\n\nThe Broker sends an ApplyBrokerId request to the Controller, carrying its basic data (ClusterName, BrokerName, and BrokerAddress) and the expected BrokerId and RegisterCode.\n\n#### 3.1 CASApplyBrokerId\n\nThe Controller writes this event to DLedger. When the event (log) is applied to the state machine, it checks whether the BrokerId can be applied (if the BrokerId has already been allocated and is not assigned to the Broker, the application fails). It also records the relationship between the BrokerId and RegisterCode.\n\n#### 4. ApplyBrokerId Response\n\nIf the previous step successfully applies the BrokerId, the Controller returns success to the Broker; otherwise, it returns the current NextBrokerId.\n\n#### 4.1 CreateMetaFileFromTemp\n\nIf the BrokerId is successfully applied in the previous step, it can be considered as successfully allocated on the Broker side. At this point, the information of this BrokerId needs to be persisted. This is achieved by atomically deleting the .broker.meta.temp file and creating a .broker.meta file. These two steps need to be atomic operations.\n\n> After the above process, the Broker and Controller that come online for the first time successfully negotiate a BrokerId that both sides agree on and persist it.\n\n#### 5. RegisterBrokerToController Request\n\nThe previous steps have correctly negotiated the BrokerId, but at this point, it is possible that the BrokerAddress saved on the Controller side is the BrokerAddress when the last Broker came online. Therefore, the BrokerAddress needs to be updated now by sending a RegisterBrokerToController request with the current BrokerAddress.\n\n#### 5.1 UpdateBrokerAddress\n\nThe Controller compares the BrokerAddress currently saved in the Controller state machine for this Broker. If it does not match the BrokerAddress carried in the request, it updates it to the BrokerAddress in the request.\n\n#### 6. RegisterBrokerToController Response\n\nAfter updating the BrokerAddress, the Controller can return the master-slave information of the Broker-set where the Broker is located, to notify the Broker to perform the corresponding identity transformation.\n\n### Registration status rotation\n\n![register state transfer](../image/controller/persistent_unique_broker_id/register_state_transfer.png)\n\n### Fault tolerance\n\n> If various crashes occur during the normal online process, the following process ensures the correct allocation of BrokerId.\n\n#### Node online after normal restart\n\nIf it is a normal restart, then a unique BrokerId has already been negotiated by both sides, and the broker.meta already has the data for that BrokerId. Therefore, the registration process is not necessary and the subsequent process can be continued directly. That is, continue to come online from RegisterBrokerToController.\n\n![restart normally](../image/controller/persistent_unique_broker_id/normal_restart.png)\n\n#### CreateTempMetaFile Failure\n\n![fail at creating temp metadata file](../image/controller/persistent_unique_broker_id/fail_create_temp_metadata_file.png)\n\nIf the process shown in the figure fails, then after the Broker restarts, the Controller's state machine has not allocated any BrokerId. The Broker itself has not saved any data. Therefore, just restart the process from the beginning as described above.\n\n#### CreateTempMetaFile success，ApplyBrokerId fail\n\nIf the Controller already considers the ApplyBrokerId request to be incorrect (i.e., requesting to allocate a BrokerId that has already been allocated and the RegisterCode is not equal), and at this time returns the current NextBrokerId to the Broker, then the Broker directly deletes the .broker.meta.temp file and goes back to step 2 to restart the process and subsequent steps.\n\n![fail at applying broker id](../image/controller/persistent_unique_broker_id/fail_apply_broker_id.png)\n\n#### ApplyBrokerId success，CreateMetaFileFromTemp fail\n\nThe above situation can occur in the ApplyResult loss, and in the CAS deletion and creation of broker.meta failure processes. After restart, the Controller side thinks that our ApplyBrokerId process has succeeded and has already modified the BrokerId allocation data in the state machine. So at this point, we can directly start step 3 again, which is to send the ApplyBrokerId request.\n\n![fail at create metadata file](../image/controller/persistent_unique_broker_id/fail_create_metadata_file_and_delete_temp.png)\n\nSince we have the .broker.meta.temp file, we can retrieve the BrokerId and RegisterCode that were successfully applied on the Controller side, and send them directly to the Controller. If the BrokerId exists in the Controller and the RegisterCode is equal to the one in the request, it is considered successful.\n\n### After successful registration, use the BrokerId as the unique identifier.\n\nAfter successful registration, all subsequent requests and state records for the Broker are identified by BrokerId. The recording of heartbeats and other data is also identified by BrokerId. At the same time, the Controller side will also record the BrokerAddress of the current BrokerId, which will be used to notify the Broker of changes in state such as switching between master and slave.\n\n## Upgrade plan\n\nTo upgrade to version 4.x, follow the 5.0 upgrade documentation process.\nFor upgrading from the non-persistent BrokerId version in 5.0.0 or 5.1.0 to the persistent BrokerId version 5.1.1 or above, follow the following steps:\n\n### Upgrade Controller\n\n1.  Shut down the old version of the Controller group.\n2.  Clear the Controller data, i.e., the data files located by default in `~/DLedgerController`.\n3.  Bring up the new version of the Controller group.\n\n> During the above Controller upgrade process, the Broker can still run normally but cannot be switched.\n\n### Upgrade Broker\n\n1.  Shut down the Broker slave node.\n2.  Shut down the Broker master node.\n3.  Delete all the Epoch files for all Brokers, i.e., the ones located at `~/store/epochFileCheckpoint` and `~/store/epochFileCheckpoint.bak` by default.\n4.  Bring up the original master Broker and wait for it to be elected as the new master (you can use the `getSyncStateSet` command in the `admin` tool to check).\n5.  Bring up all the original slave Brokers.\n\n> It is recommended to shut down the slave Brokers before shutting down the master Broker and bring up the original master Broker before bringing up the original slave Brokers. This will ensure that the original master-slave relationship is maintained. If you need to change the master-slave relationship after the upgrade, you need to make sure that the CommitLog of the old master and slave Brokers are aligned before shutting them down, otherwise data may be truncated and lost.\n\n### Compatibility\n\n|                                    | Controller for version 5.1.0 and below              | Controller for version 5.1.1 and above                                               |\n|------------------------------------|--------------------------------| ------------------------------------------------------------ |\n| Broker for version 5.1.0 and below | Normal operation and switch.   | Normal operation and no switch if the master-slave relationship is already determined. The Broker cannot be brought up if it is restarted. |\n| Broker for version 5.1.1 and above | Cannot be brought up normally. | Normal operation and switch.                                 |"
  },
  {
    "path": "docs/en/controller/quick_start.md",
    "content": "# Master-Slave automatic switch Quick start \n\n## Introduction\n\n![架构图](../image/controller/controller_design_2.png)\n\nThis document mainly introduces how to quickly build a RocketMQ cluster that supports automatic master-slave switch, as shown in the above diagram. The main addition is the Controller component, which can be deployed independently or embedded in the NameServer.\n\nFor detailed design ideas, please refer to [Design ideas](design.md).\n\nFor detailed guidelines on new cluster deployment and old cluster upgrades, please refer to [Deployment guide](deploy.md).\n\n## Compile RocketMQ source code\n\n```shell\n$ git clone https://github.com/apache/rocketmq.git\n\n$ cd rocketmq\n\n$ mvn -Prelease-all -DskipTests clean install -U\n```\n\n## Quick deployment\n\nAfter successful build\n\n```shell\n#{rocketmq-version} replace with rocketmq actual version. example: 5.0.0-SNAPSHOT\n$ cd distribution/target/rocketmq-{rocketmq-version}/rocketmq-{rocketmq-version}/\n\n$ sh bin/controller/fast-try.sh start\n```\n\nIf the above steps are successful, you can view the status of the Controller using the operation and maintenance command.\n\n```shell\n$ sh bin/mqadmin getControllerMetaData -a localhost:9878\n```\n\n`-a` represents the address of any controller in the cluster\n\nAt this point, you can send and receive messages in the cluster and perform switch testing.\n\nIf you need to shut down the cluster quickly , you can execute：\n\n```shell\n$ sh bin/controller/fast-try.sh stop\n```\n\nFor quick deployment, the default configuration is in `conf/controller/quick-start`, the default storage path is `/tmp/rmqstore`, and a controller (embedded in Namesrv) and two brokers will be started.\n\n### Query SyncStateSet\n\n Use the operation and maintenance tool to query SyncStateSet：\n\n```shell\n$ sh bin/mqadmin getSyncStateSet -a localhost:9878 -b broker-a\n```\n\n`-a` represents the address of any controller\n\nIf successful, you should see the following content：\n\n![image-20220605205259913](../image/controller/quick-start/syncstateset.png)\n\n### Query BrokerEpoch\n\n Use the operation and maintenance tool to query  BrokerEpochEntry：\n\n```shell\n$ sh bin/mqadmin getBrokerEpoch -n localhost:9876 -b broker-a\n```\n\n`-n` represents the address of any Namesrv\n\nIf successful, you should see the following content：\n\n![image-20220605205247476](../image/controller/quick-start/epoch.png)\n\n## Switch\n\nAfter successful deployment, try to perform a master switch now.\n\nFirst, kill the process of the original master, in the example above, it is the process using port 30911：\n\n```shell\n#query port:\n$ ps -ef|grep java|grep BrokerStartup|grep ./conf/controller/quick-start/broker-n0.conf|grep -v grep|awk '{print $2}'\n#kill master:\n$ kill -9 PID\n```\n\nNext,use `SyncStateSet admin` script to query：\n\n```shell\n$ sh bin/mqadmin getSyncStateSet -a localhost:9878 -b broker-a\n```\n\nThe master has switched.\n\n![image-20220605211244128](../image/controller/quick-start/changemaster.png)\n\n\n\n## Deploying controller embedded in Nameserver cluster\n\nThe Controller component is embedded in the Nameserver cluster (consisting of 3 nodes) and quickly started through the plugin mode：\n\n```shell\n$ sh bin/controller/fast-try-namesrv-plugin.sh start\n```\n\nAlternatively, it can be started separately through a command：\n\n```shell\n$ nohup bin/mqnamesrv -c ./conf/controller/cluster-3n-namesrv-plugin/namesrv-n0.conf &\n$ nohup bin/mqnamesrv -c ./conf/controller/cluster-3n-namesrv-plugin/namesrv-n1.conf &\n$ nohup bin/mqnamesrv -c ./conf/controller/cluster-3n-namesrv-plugin/namesrv-n2.conf &\n```\n\nIf the above steps are successful, you can check the status of the Controller cluster through operational commands:\n\n```shell\n$ sh bin/mqadmin getControllerMetaData -a localhost:9878\n```\n\n`-a` represents the address of any Controller nodes\n\nIf the Controller starts successfully, you can see the following content:\n\n```\n#ControllerGroup        group1\n#ControllerLeaderId     n0\n#ControllerLeaderAddress        127.0.0.1:9878\n#Peer:  n0:127.0.0.1:9878\n#Peer:  n1:127.0.0.1:9868\n#Peer:  n2:127.0.0.1:9858\n```\n\nAfter the successful start, the broker controller mode deployment can use the controller cluster.\n\nIf you need to quickly stop the cluster：\n\n```shell\n$ sh bin/controller/fast-try-namesrv-plugin.sh stop\n```\n\nThe `fast-try-namesrv-plugin.sh` script is used for quick deployment with default configurations in the `conf/controller/cluster-3n-namesrv-plugin` directory, and it will start 3 Nameservers and 3 controllers (embedded in Nameserver).\n\n## Deploying Controller in independent cluster\n\nThe Controller component is deployed in an independent cluster (consisting of 3 nodes) and quickly started.：\n\n```shell\n$ sh bin/controller/fast-try-independent-deployment.sh start\n```\n\nAlternatively, it can be started separately through a command：\n\n```shell\n$ nohup bin/mqcontroller -c ./conf/controller/cluster-3n-independent/controller-n0.conf &\n$ nohup bin/mqcontroller -c ./conf/controller/cluster-3n-independent/controller-n1.conf &\n$ nohup bin/mqcontroller -c ./conf/controller/cluster-3n-independent/controller-n2.conf &\n```\n\nIf the previous steps are successful, you can check the status of the Controller cluster using the operational command.\n\n```shell\n$ sh bin/mqadmin getControllerMetaData -a localhost:9878\n```\n\n`-a` represents the address of any controller.\n\nIf the controller starts successfully, you will see the following content:\n\n```\n#ControllerGroup        group1\n#ControllerLeaderId     n1\n#ControllerLeaderAddress        127.0.0.1:9868\n#Peer:  n0:127.0.0.1:9878\n#Peer:  n1:127.0.0.1:9868\n#Peer:  n2:127.0.0.1:9858\n```\n\nAfter starting successfully, the broker controller mode deployment can use the controller cluster.\n\nIf you need to quickly stop the cluster：\n\n```shell\n$ sh bin/controller/fast-try-independent-deployment.sh stop\n```\n\nUse the `fast-try-independent-deployment.sh` script to quickly deploy, the default configuration is in `conf/controller/cluster-3n-independent` and it will start 3 controllers (independent deployment) to form a cluster.\n\n\n\n## Note\n\n- If you want to ensure that the Controller has fault tolerance, the Controller deployment requires at least three copies (in accordance with the majority protocol of Raft).\n- In the controller deployment configuration file, the IP addresses configured in the `controllerDLegerPeers` parameter should be configured as IPs that can be accessed by other nodes. This is especially important when deploying on multiple machines. The example is for reference only and needs to be modified and adjusted according to the actual situation.\n"
  },
  {
    "path": "docs/en/design.md",
    "content": "\n##  Design\n### 1 Message Store\n\n![](../cn/image/rocketmq_design_1.png)\n\n\n#### 1.1 The Architecture of Message Store\n\n#### 1.2 PageCache and Memory-Map(Mmap)\n\n#### 1.3 Message Flush\n\n![](../cn/image/rocketmq_design_2.png)\n\n\n### 2 Communication Mechanism\n\n#### 2.1 The class diagram of Remoting module\n\n![](../cn/image/rocketmq_design_3.png)\n\n#### 2.2 The design of protocol and encode/decode\n\n![](../cn/image/rocketmq_design_4.png)\n\n\n#### 2.3 The three ways and process of message communication \n\n![](../cn/image/rocketmq_design_5.png)\n\n#### 2.4 The multi-thread design of Reactor\n\n![](../cn/image/rocketmq_design_6.png)\n\n\n### 3 Message Filter\n\n![](../cn/image/rocketmq_design_7.png)\n\n### 4 LoadBalancing\n\n#### 4.1 The loadBalance of Producer\n\n#### 4.2 The loadBalance of Consumer\n\n![](../cn/image/rocketmq_design_8.png)\n\n\n![](../cn/image/rocketmq_design_9.png)\n\n\n\n### 5 Transactional Message\nApache RocketMQ supports distributed transactional message from version 4.3.0. RocketMQ implements transactional message by using the protocol of 2PC(two-phase commit), in addition adding a compensation logic to handle timeout-case or failure-case of commit-phase, as shown below.\n\n![](../cn/image/rocketmq_design_10.png)\n\n#### 5.1 The Process of RocketMQ Transactional Message\nThe picture above shows the overall architecture of transactional message, including the sending of message(commit-request phase), the sending of commit/rollback(commit phase) and the compensation process.\n\n1. The sending of message and Commit/Rollback.  \n  (1) Sending the message(named Half message in RocketMQ)  \n  (2) The server responds the writing result(success or failure) of Half message.  \n  (3) Handle local transaction according to the result(local transaction won't be executed when the result is failure).  \n  (4) Sending Commit/Rollback to broker according to the result of local transaction(Commit will generate message index and make the message visible to consumers).\n\n2. Compensation process  \n  (1) For a transactional message without a Commit/Rollback (means the message in the pending status), a \"back-check\" request is initiated from the broker.  \n  (2) The Producer receives the \"back-check\" request and checks the status of the local transaction corresponding to the \"back-check\" message.  \n  (3) Redo Commit or Rollback based on local transaction status.\nThe compensation phase is used to resolve the timeout or failure case of the message Commit or Rollback.\n\n#### 5.2 The design of RocketMQ Transactional Message\n1. Transactional message is invisible to users in first phase(commit-request phase)   \n  \n  Upon on the main process of transactional message, the message of first phase is invisible to the user. This is also the biggest difference from normal message. So how do we write the message while making it invisible to the user? And below is the solution of RocketMQ: if the message is a Half message, the topic and queueId of the original message will be backed up, and then changes the topic to RMQ_SYS_TRANS_HALF_TOPIC. Since the consumer group does not subscribe to the topic, the consumer cannot consume the Half message. Then RocketMQ starts a timing task, pulls the message for RMQ_SYS_TRANS_HALF_TOPIC, obtains a channel according to producer group and sends a back-check to query local transaction status, and decide whether to submit or roll back the message according to the status.  \n  \n  In RocketMQ, the storage structure of the message in the broker is as follows. Each message has corresponding index information. The Consumer reads the content of the message through the secondary index of the ConsumeQueue. The flow is as follows:\n\n![](../cn/image/rocketmq_design_11.png)\n\n  The specific implementation strategy of RocketMQ is: if the transactional message is written, topic and queueId of the message are replaced, and the original topic and queueId are stored in the properties of the message. Because the replace of the topic, the message will not be forwarded to the Consumer Queue of the original topic, and the consumer cannot perceive the existence of the message and will not consume it. In fact, changing the topic is the conventional method of RocketMQ(just recall the implementation mechanism of the delay message).\n\n2. Commit/Rollback operation and introduction of Op message  \n  \n  After finishing writing a message that is invisible to the user in the first phase, here comes two cases in the second phase. One is Commit operation, after which the message needs to be visible to the user; the other one is Rollback operation, after which the first phase message(Half message) needs to be revoked. For the case of Rollback, since first-phase message itself is invisible to the user, there is no need to actually revoke the message (in fact, RocketMQ can't actually delete a message because it is a sequential-write file). But still some operation needs to be done to identity the final status of the message, to differ it from pending status message. To do this, the concept of \"Op message\" is introduced, which means the message has a certain status(Commit or Rollback). If a transactional message does not have a corresponding Op message, the status of the transaction is still undetermined (probably the second-phase failed). By introducing the Op message, the RocketMQ records an Op message for every Half message regardless it is Commit or Rollback. The only difference between Commit and Rollback is that when it comes to Commit, the index of the Half message is created before the Op message is written.\n\n3. How Op message stored and the correspondence between Op message and Half message  \n  \n  RocketMQ writes the Op message to a specific system topic(RMQ_SYS_TRANS_OP_HALF_TOPIC) which will be created via the method - TransactionalMessageUtil.buildOpTopic(); this topic is an internal Topic (like the topic of RMQ_SYS_TRANS_HALF_TOPIC) and will not be consumed by the user. The content of the Op message is the physical offset of the corresponding Half message. Through the Op message we can index to the Half message for subsequent check-back operation.\n\n![](../cn/image/rocketmq_design_12.png)\n\n4. Index construction of Half messages  \n  \n  When performing Commit operation of the second phase, the index of the Half message needs to be built. Since the Half message is written to a special topic(RMQ_SYS_TRANS_HALF_TOPIC) in the first phase of 2PC, so it needs to be read out from the special topic when building index, and replace the topic and queueId with the real target topic and queueId, and then write through a normal message that is visible to the user. Therefore, in conclusion, the second phase recovers a complete normal message using the content of the Half message stored in the first phase, and then goes through the message-writing process.\n\n5. How to handle the message failed in the second phase？\n  \n  If commit/rollback phase fails, for example, a network problem causes the Commit to fail when you do Commit. Then certain strategy is required to make sure the message finally commit. RocketMQ uses a compensation mechanism called \"back-check\". The broker initiates a back-check request for the message in pending status, and sends the request to the corresponding producer side (the same producer group as the producer group who sent the Half message). The producer checks the status of local transaction and redo Commit or Rollback. The broker performs the back-check by comparing the RMQ_SYS_TRANS_HALF_TOPIC messages and the RMQ_SYS_TRANS_OP_HALF_TOPIC messages and advances the checkpoint(recording those transactional messages that the status are certain).\n\n  RocketMQ does not back-check the status of transactional messages endlessly. The default time is 15. If the transaction status is still unknown after 15 times, RocketMQ will roll back the message by default.\n### 6 Message Query\n\n#### 6.1 Query messages by messageId\n\n#### 6.2 Query messages by message key\n\n![](../cn/image/rocketmq_design_13.png)\n"
  },
  {
    "path": "docs/en/dledger/deploy_guide.md",
    "content": "# Dledger cluster deployment\n---\n## preface\nThis document introduces how to deploy auto failover RocketMQ-on-DLedger Group.\n\nRocketMQ-on-DLedger Group is a broker group with **same name**, needs at least 3 nodes, elect a Leader by Raft algorithm automatically, the others as Follower, replicating data between Leader and Follower for system high available.  \nRocketMQ-on-DLedger Group can failover automatically, and maintains consistent.  \nRocketMQ-on-DLedger Group can scale up horizontal, that is, can deploy any RocketMQ-on-DLedger Groups providing services external.  \n\n## 1. New cluster deployment\n\n#### 1.1 Write the configuration\neach RocketMQ-on-DLedger Group needs at least 3 machines.(assuming 3 in this document)  \nwrite 3 configuration files, advising refer to the directory of conf/dledger 's example configuration file.  \nkey configuration items:  \n\n| name | meaning | example |\n| --- | --- | --- |\n| enableDLegerCommitLog | whether enable DLedger  | true |\n| dLegerGroup | DLedger Raft Group's name, advising maintain consistent to brokerName | RaftNode00 |\n| dLegerPeers | DLedger Group's nodes port infos, each node's configuration stay consistent in the same group. | n0-127.0.0.1:40911;n1-127.0.0.1:40912;n2-127.0.0.1:40913 |\n| dLegerSelfId | node id, must belongs to dLegerPeers; each node is unique in the same group. | n0 |\n| sendMessageThreadPoolNums | the count of sending thread, advising set equal to the cpu cores. | 16 |\n\nthe following presents an example configuration conf/dledger/broker-n0.conf.  \n\n```\nbrokerClusterName = RaftCluster\nbrokerName=RaftNode00\nlistenPort=30911\nnamesrvAddr=127.0.0.1:9876\nstorePathRootDir=/tmp/rmqstore/node00\nstorePathCommitLog=/tmp/rmqstore/node00/commitlog\nenableDLegerCommitLog=true\ndLegerGroup=RaftNode00\ndLegerPeers=n0-127.0.0.1:40911;n1-127.0.0.1:40912;n2-127.0.0.1:40913\n## must be unique\ndLegerSelfId=n0\nsendMessageThreadPoolNums=16\n```\n\n### 1.2 Start Broker\n\nStartup stays consistent with the old version.\n\n`nohup sh bin/mqbroker -c conf/dledger/xxx-n0.conf & `  \n`nohup sh bin/mqbroker -c conf/dledger/xxx-n1.conf & `  \n`nohup sh bin/mqbroker -c conf/dledger/xxx-n2.conf & `  \n\n\n## 2. Upgrade old cluster\n\nIf old cluster deployed in Master mode, then each Master needs to be transformed into a RocketMQ-on-DLedger Group.  \nIf old cluster deployed in Master-Slave mode, then each Master-Slave group needs to be transformed into a RocketMQ-on-DLedger Group.\n\n### 2.1 Kill old Broker\n\nexecute kill command, or call `bin/mqshutdown broker`.\n\n### 2.2 Check old Commitlog\n\nEach node in RocketMQ-on-DLedger group is compatible with old Commitlog, but Raft replicating process works on the adding message only. So, to avoid occurring exceptions, old Commitlog must be consistent.\nIf old cluster deployed in Master-Slave mode, it maybe inconsistent after shutdown. Advising use md5sum to check at least 2 recently Commitlog file, if occur inconsistent, maintain consistent by copy.\n\nAlthough RocketMQ-on-DLedger Group can deployed with 2 nodes, it lacks failover ability(at least 3 nodes can tolerate one node fail).\nMake sure that both Master and Slave's Commitlog is consistent, then prepare 3 machines, copy old Commitlog from Master to this 3 machines(BTW, copy the config directory).\n   \nThen, go ahead to set configurations.\n\n### 2.3 Modify configuration\n\nRefer to New cluster deployment.\n\n### 2.4 Restart Broker \n\nRefer to New cluster deployment.\n\n\n"
  },
  {
    "path": "docs/en/dledger/quick_start.md",
    "content": "# Dledger Quick Deployment\n---\n### preface\nThis document is mainly introduced for how to build and deploy auto failover RocketMQ cluster based on DLedger.\n\nFor detailed new cluster deployment and old cluster upgrade document, please refer to [Deployment Guide](deploy_guide.md).\n\n### 1. Build from source code\nBuild phase contains two parts, first, build DLedger, then build RocketMQ.\n\n#### 1.1 Build DLedger\n\n```shell\n$ git clone https://github.com/openmessaging/dledger.git\n$ cd dledger\n$ mvn clean install -DskipTests\n```\n\n#### 1.2 Build RocketMQ\n\n```shell\n$ git clone https://github.com/apache/rocketmq.git\n$ cd rocketmq\n$ git checkout -b store_with_dledger origin/store_with_dledger\n$ mvn -Prelease-all -DskipTests clean install -U\n```\n\n### 2. Quick Deployment\n\nafter build successful\n\n```shell\n#{rocketmq-version} replace with rocketmq actual version. example: 5.0.0-SNAPSHOT\n$ cd distribution/target/rocketmq-{rocketmq-version}/rocketmq-{rocketmq-version}\n$ sh bin/dledger/fast-try.sh start\n```\n\nif the above commands executed successfully, then check cluster status by using mqadmin operation commands.\n\n```shell\n$ sh bin/mqadmin clusterList -n 127.0.0.1:9876\n```\n\nIf everything goes well, the following content will appear:\n\n![ClusterList](https://img.alicdn.com/5476e8b07b923/TB11Z.ZyCzqK1RjSZFLXXcn2XXa)\n\n（BID is 0 indicate Master, the others are Follower）\n\nAfter startup successful, producer can produce message, and then test failover scenario.\n\nStop cluster fastly, execute the following command:\n\n```shell\n$ sh bin/dledger/fast-try.sh stop\n```\n\nQuick deployment, default configuration is in directory conf/dledger, default storage path is /tmp/rmqstore.\n\n\n### 3. Failover\n\nAfter successful deployment, kill Leader process(as the above example, kill process that binds port 30931), about 10 seconds elapses, use clusterList command check cluster's status, Leader switch to another node.\n\n\n\n\n\n"
  },
  {
    "path": "docs/en/msg_trace/user_guide.md",
    "content": "# message trace\n----\n\n## 1. Message trace data's key properties\n| Producer End| Consumer End| Broker End|\n| --- | --- | --- |\n| produce message | consume message | message's topic |\n| send message time | delivery time, delivery rounds  | message store location |\n| whether the message was sent successfully | whether message was consumed successfully | message's key |\n| send cost-time | consume cost-time | message's tag value |\n\n## 2. Enable message trace in cluster deployment\n\n### 2.1 Broker's configuration file\nfollowing by Broker's properties file configuration that enable message trace:\n```\nbrokerClusterName=DefaultCluster\nbrokerName=broker-a\nbrokerId=0\ndeleteWhen=04\nfileReservedTime=48\nbrokerRole=ASYNC_MASTER\nflushDiskType=ASYNC_FLUSH\nstorePathRootDir=/data/rocketmq/rootdir-a-m\nstorePathCommitLog=/data/rocketmq/commitlog-a-m\nautoCreateSubscriptionGroup=true\n## if msg tracing is open,the flag will be true\ntraceTopicEnable=true\nlistenPort=10911\nbrokerIP1=XX.XX.XX.XX1\nnamesrvAddr=XX.XX.XX.XX:9876\n```\n\n### 2.2 Common mode\nEach Broker node in RocketMQ cluster used for storing message trace data that client collected and sent. So, there is no requirements and limitations to the size of Broker node in RocketMQ cluster.\n\n### 2.3 IO physical isolation mode\nFor huge amounts of message trace data scenario, we can select any one Broker node in RocketMQ cluster used for storing message trace data special, thus, common message data's IO are isolated from message trace data's IO in physical, not impact each other. In this mode, RocketMQ cluster must have at least two Broker nodes, the one that defined as storing message trace data.\n\n### 2.4 Start Broker that enable message trace\n`nohup sh mqbroker -c ../conf/2m-noslave/broker-a.properties &`\n  \n## 3. Save the definition of topic that with support message trace\nRocketMQ's message trace feature supports two types of storage.\n\n### 3.1 System level TraceTopic\nBe default, message trace data is stored in system level TraceTopic(topic name: **RMQ_SYS_TRACE_TOPIC**). That topic will be created at startup of broker(As mentioned above, set **traceTopicEnable** to **true** in Broker's configuration).\n\n### 3.2 User defined TraceTopic\nIf user don't want to store message trace data in system level TraceTopic, he can create user defined TraceTopic used for storing message trace data(that is, create common topic for storing message trace data). The following part will introduce how client SDK support user defined TraceTopic.\n\n## 4. Client SDK demo with message trace feature\nFor business system adapting to use RocketMQ's message trace feature easily, in design phase, the author add a switch parameter(**enableMsgTrace**) for enable message trace; add a custom parameter(**customizedTraceTopic**) for user defined TraceTopic.\n\n### 4.1 Enable message trace when sending messages\n```\n        DefaultMQProducer producer = new DefaultMQProducer(\"ProducerGroupName\",true);\n        producer.setNamesrvAddr(\"XX.XX.XX.XX1\");\n        producer.start();\n            try {\n                {\n                    Message msg = new Message(\"TopicTest\",\n                        \"TagA\",\n                        \"OrderID188\",\n                        \"Hello world\".getBytes(RemotingHelper.DEFAULT_CHARSET));\n                    SendResult sendResult = producer.send(msg);\n                    System.out.printf(\"%s%n\", sendResult);\n                }\n\n            } catch (Exception e) {\n                e.printStackTrace();\n            }\n```\n\n### 4.2 Enable message trace when subscribe messages\n```\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"CID_JODIE_1\",true);\n        consumer.subscribe(\"TopicTest\", \"*\");\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        consumer.setConsumeTimestamp(\"20181109221800\");\n        consumer.registerMessageListener(new MessageListenerConcurrently() {\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {\n                System.out.printf(\"%s Receive New Messages: %s %n\", Thread.currentThread().getName(), msgs);\n                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n            }\n        });\n        consumer.start();\n        System.out.printf(\"Consumer Started.%n\");\n```\n\n### 4.3 Self-defined topic support message trace\nAdjusting instantiation of DefaultMQProducer and DefaultMQPushConsumer as following code to support user defined TraceTopic.\n```\n        ##Topic_test11111 should be created by user, used for storing message trace data.\n        DefaultMQProducer producer = new DefaultMQProducer(\"ProducerGroupName\",true,\"Topic_test11111\");\n        ......\n\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"CID_JODIE_1\",true,\"Topic_test11111\");\n        ......\n```\n\n\n### 4.4 Send and query message trace by mqadmin command\n- send message\n```shell\n./mqadmin sendMessage -m true --topic some-topic-name -n 127.0.0.1:9876 -p \"your message content\"\n```\n- query trace\n```shell\n./mqadmin QueryMsgTraceById -n 127.0.0.1:9876 -i \"some-message-id\"\n```\n- query trace result\n```\nRocketMQLog:WARN No appenders could be found for logger (io.netty.util.internal.PlatformDependent0).\nRocketMQLog:WARN Please initialize the logger system properly.\n#Type      #ProducerGroup       #ClientHost          #SendTime            #CostTimes #Status\nPub        1623305799667        xxx.xxx.xxx.xxx       2021-06-10 14:16:40  131ms      success\n```\n\n\n"
  },
  {
    "path": "docs/en/operation.md",
    "content": "\n# Operation Management\n---\n\n### 1   Deploy cluster\n\n#### 1.1 Single Master mode\n\nThis mode is risky, upon broker restart or broken down, the whole service is unavailable. It's not recommended in production environment, it can be used for local test.\n\n##### 1）Start NameServer\n\n```bash\n### Start Name Server\n$ nohup sh mqnamesrv &\n \n### check whether Name Server is successfully started\n$ tail -f ~/logs/rocketmqlogs/namesrv.log\nThe Name Server boot success...\n```\n\n##### 2）Start Broker\n\n```bash\n### start Broker\n$ nohup sh bin/mqbroker -n localhost:9876 &\n\n### check whether Broker is successfully started, eg: Broker's IP is 192.168.1.2, Broker's name is broker-a\n$ tail -f ~/logs/rocketmqlogs/broker.log \nThe broker[broker-a, 192.169.1.2:10911] boot success...\n```\n\n#### 1.2 Multi Master mode\n\nCluster contains Master node only, no Slave node, eg: 2 Master nodes, 3 Master nodes, advantages and disadvantages of this mode are shown below:\n\n- advantages: simple configuration, single Master node broke down or restart do not impact application. Under RAID10 disk config, even if machine broken down and cannot recover, message do not get lost because of RAID10's high reliable(async flush to disk lost little message, sync to disk do not lost message), this mode get highest performance.\n\n- disadvantages: during the machine's down time, messages have not be consumed on this machine can not be subscribed before recovery. That will impacts message's instantaneity.\n\n##### 1）Start NameServer\n\nNameServer should be started before broker. If under production environment, we recommend start 3 NameServer nodes for high available. Startup command is equal, as shown below:\n\n```bash\n### start Name Server\n$ nohup sh mqnamesrv &\n \n### check whether Name Server is successfully started\n$ tail -f ~/logs/rocketmqlogs/namesrv.log\nThe Name Server boot success...\n```\n\n##### 2）start Broker cluster\n\n```bash\n### start the first Master on machine A, eg:NameServer's IP is 192.168.1.1\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-noslave/broker-a.properties &\n \n### start the second Master on machine B, eg:NameServer's IP is 192.168.1.1\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-noslave/broker-b.properties &\n\n...\n```\n\nThe above commands only used for single NameServer. In multi NameServer cluster, multi addresses concat by semicolon followed by -n in broker start command. \n\n#### 1.3 Multi Master Multi Slave mode - async replication\n\nEach Master node is equipped with one Slave node, this mode has many Master-Slave group, using async replication for HA, slaver has a lag(ms level) behind master, advantages and disadvantages of this mode are shown below:\n\n- advantages: message lost a little, even if disk is broken; message instantaneity do not loss; Consumer can still consume from slave when master is down, this process is transparency to user, no human intervention is required; Performance is almost equal to Multi Master mode.\n\n- disadvantages: message lost a little data, when Master is down and disk broken.\n\n##### 1）Start NameServer\n\n```bash\n### start Name Server\n$ nohup sh mqnamesrv &\n \n### check whether Name Server is successfully started\n$ tail -f ~/logs/rocketmqlogs/namesrv.log\nThe Name Server boot success...\n```\n\n##### 2）Start Broker cluster\n\n```bash\n### start first Master on machine A, eg: NameServer's IP is 192.168.1.1\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-a.properties &\n \n### start second Master on machine B, eg: NameServer's IP is 192.168.1.1\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-b.properties &\n \n### start first Slave on machine C, eg: NameServer's IP is 192.168.1.1\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-a-s.properties &\n \n### start second Slave on machine D, eg: NameServer's IP is 192.168.1.1\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-async/broker-b-s.properties &\n```\n\n#### 1.4 Multi Master Multi Slave mode - synchronous double write\n\nEach Master node is equipped with one Slave node, this mode has many Master-Slave group, using synchronous double write for HA, application's write operation is successful means both master and slave write successful, advantages and disadvantages of this mode are shown below:\n\n- advantages:both data and service have no single point failure, message has no latency even if Master is down, service available and data available is very high;\n\n- disadvantages:this mode's performance is 10% lower than async replication mode, sending latency is a little high,  in the current version, it do not have auto Master-Slave switch when Master is down.\n\n##### 1）Start NameServer\n\n```bash\n### start Name Server\n$ nohup sh mqnamesrv &\n \n### check whether Name Server is successfully started\n$ tail -f ~/logs/rocketmqlogs/namesrv.log\nThe Name Server boot success...\n```\n\n##### 2）Start Broker cluster\n\n```bash\n### start first Master on machine A, eg:NameServer's IP is 192.168.1.1\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-a.properties &\n \n### start second Master on machine B, eg:NameServer's IP is 192.168.1.1\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-b.properties &\n \n### start first Slave on machine C, eg: NameServer's IP is 192.168.1.1\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-a-s.properties &\n \n### start second Slave on machine D, eg: NameServer's IP is 192.168.1.1\n$ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-b-s.properties &\n```\n\nThe above Broker matches Slave by specifying the same BrokerName, Master's BrokerId must be 0, Slave's BrokerId must larger than 0. Besides, a Master can have multi Slaves that each has a distinct BrokerId. $ROCKETMQ_HOME indicates RocketMQ's install directory, user needs to set this environment parameter.\n\n### 2 mqadmin management tool\n\n> Attentions:\n>\n> 1. execute command: `./mqadmin {command} {args}`\n> 2. almost all commands need -n indicates NameSerer address, format is ip:port\n> 3. almost all commands can get help info by -h\n> 4. if command contains both Broker address(-b) and cluster name(-c), it's prior to use broker address. If command do not contains broker address, it will executed on all hosts in this cluster. Support only one broker host. -b format is ip:port, default port is 10911\n> 5. there are many commands under tools, but not all command can be used, only commands that initialized in MQAdminStartup can be used, you can modify this class, add or self-define command.\n> 6. because of version update, little command do not update timely, please refer to source code directly when occur error.\n#### 2.1 Topic\n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>name</td>\n  <td class=xl64 width=175 style='width:131pt'>meaning</td>\n  <td class=xl64 width=177 style='width:133pt'>command items</td>\n  <td class=xl64 width=185 style='width:139pt'>explanation</td>\n </tr>\n <tr height=132 style='height:99.0pt'>\n  <td rowspan=8 height=593 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:444.0pt;border-top:none;width:122pt'>updateTopic</td>\n  <td rowspan=8 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>create or update Topic's config</td>\n  <td class=xl65 width=149 style='width:112pt'>-b</td>\n  <td class=xl66 width=159 style='width:119pt'>Broker address, means which Broker that topic is located, only support single Broker, address format is ip:port</td>\n </tr>\n <tr height=132 style='height:99.0pt'>\n  <td height=132 class=xl65 width=149 style='height:99.0pt;width:112pt'>-c</td>\n  <td class=xl66 width=159 style='width:119pt'>cluster name, which cluster that topic belongs to(query cluster info by clusterList)</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-h-</td>\n  <td class=xl66 width=159 style='width:119pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=76 style='height:57.0pt'>\n  <td height=76 class=xl65 width=149 style='height:57.0pt;width:112pt'>-p</td>\n  <td class=xl66 width=159 style='width:119pt'>assign read write authority to new topic(W=2|R=4|WR=6)</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=149 style='height:29.0pt;width:112pt'>-r</td>\n  <td class=xl66 width=159 style='width:119pt'>the count of queue that can be read(default is 8)</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=149 style='height:29.0pt;width:112pt'>-w</td>\n  <td class=xl66 width=159 style='width:119pt'>the count of queue that can be wrote(default is 8)</td>\n </tr>\n <tr height=95 style='height:71.0pt'>\n  <td height=95 class=xl65 width=149 style='height:71.0pt;width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>topic name(can only use characters ^[a-zA-Z0-9_-]+$ )</td>\n </tr>\n <tr height=132 style='height:99.0pt'>\n  <td rowspan=4 height=307 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:230.0pt;border-top:none;width:122pt'>deleteTopic</td>\n  <td rowspan=4 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>delete Topic</td>\n  <td class=xl65 width=149 style='width:112pt'>-c</td>\n  <td class=xl66 width=159 style='width:119pt'>cluster name, which cluster that topic will be deleted belongs to(query cluster info by clusterList)</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=95 style='height:71.0pt'>\n  <td height=95 class=xl65 width=149 style='height:71.0pt;width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>topic name(can only use characters ^[a-zA-Z0-9_-]+$ )</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=3 height=287 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:215.0pt;border-top:none;width:122pt'>topicList</td>\n  <td rowspan=3 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>query Topic list info</td>\n  <td class=xl65 width=149 style='width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>print help info</td>\n </tr>\n <tr height=207 style='height:155.0pt'>\n  <td height=207 class=xl65 width=149 style='height:155.0pt;width:112pt'>-c</td>\n  <td class=xl66 width=159 style='width:119pt'>return topic list only if do not contains -c, if contains -c, it will return cluster name, topic name, consumer group name</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=3 height=103 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:77.0pt;border-top:none;width:122pt'>topicRoute</td>\n  <td rowspan=3 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>query Topic's route info</td>\n  <td class=xl65 width=149 style='width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>topic name</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=3 height=103 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:77.0pt;border-top:none;width:122pt'>topicStatus</td>\n  <td rowspan=3 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>query Topic's offset</td>\n  <td class=xl65 width=149 style='width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>topic name</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=3 height=103 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:77.0pt;border-top:none;width:122pt'>topicClusterList</td>\n  <td rowspan=3 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>query cluster list where Topic belongs to</td>\n  <td class=xl65 width=149 style='width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>topic name</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=6 height=518 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:380pt;border-top:none;width:122pt'>updateTopicPerm</td>\n  <td rowspan=6 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>update Topic's produce and consume authority</td>\n  <td class=xl65 width=149 style='width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>topic name</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=132 style='height:99.0pt'>\n  <td height=132 class=xl65 width=149 style='height:99.0pt;width:112pt'>-b</td>\n  <td class=xl66 width=159 style='width:119pt'>Broker address which topic belongs to, support single broker only, format is ip:port</td>\n </tr>\n <tr height=76 style='height:57.0pt'>\n  <td height=76 class=xl65 width=149 style='height:57.0pt;width:112pt'>-p</td>\n  <td class=xl66 width=159 style='width:119pt'>assign read and write authority to the new topic(W=2|R=4|WR=6)</td>\n </tr>\n <tr height=207 style='height:155.0pt'>\n  <td height=207 class=xl65 width=149 style='height:155.0pt;width:112pt'>-c</td>\n  <td class=xl66 width=159 style='width:119pt'>cluster name, which topic belongs to(query cluster info by clusterList), if do not have -b, execute command on all brokers.</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=5 height=199 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:149.0pt;border-top:none;width:122pt'>updateOrderConf</td>\n  <td rowspan=5 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>create delete get specified namespace's kv config from NameServer, have not enabled at present</td>\n  <td class=xl65 width=149 style='width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>topic, key</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=149 style='height:29.0pt;width:112pt'>-v</td>\n  <td class=xl66 width=159 style='width:119pt'>orderConf, value</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-m</td>\n  <td class=xl66 width=159 style='width:119pt'>method, including get, put, delete</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=4 height=198 class=xl68 width=163 style='border-bottom:1.0pt;\n  height:140pt;border-top:none;width:122pt'>allocateMQ</td>\n  <td rowspan=4 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>calculate consumer list rebalance result by average rebalance algorithm</td>\n  <td class=xl65 width=149 style='width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>topic name</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=95 style='height:71.0pt'>\n  <td height=95 class=xl65 width=149 style='height:71.0pt;width:112pt'>-i</td>\n  <td class=xl66 width=159 style='width:119pt'>ipList, separate by comma, calculate which topic queue that ips will load.</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=4 height=142 class=xl68 width=163 style='border-bottom:1.0pt solid black;\n  height:106.0pt;border-top:1.0pt;width:122pt'>statsAll</td>\n  <td rowspan=4 class=xl70 width=135 style='border-bottom:1.0pt;\n  border-top:none;width:101pt'>print Topic's subscribe info, TPS, size of message blocked, count of read and write at last 24h, eg.</td>\n  <td class=xl65 width=149 style='width:112pt'>-h</td>\n  <td class=xl66 width=159 style='width:119pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=149 style='height:43.0pt;width:112pt'>-n</td>\n  <td class=xl66 width=159 style='width:119pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=149 style='height:29.0pt;width:112pt'>-a</td>\n  <td class=xl66 width=159 style='width:119pt'>only print active topic or not</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=149 style='height:17.0pt;width:112pt'>-t</td>\n  <td class=xl66 width=159 style='width:119pt'>assign topic</td>\n </tr>\n</table>\n\n\n\n#### 2.2 Cluster\n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>名称</td>\n  <td class=xl64 width=175 style='width:131pt'>meaning</td>\n  <td class=xl64 width=177 style='width:133pt'>command items</td>\n  <td class=xl64 width=185 style='width:139pt'>explanation</td>\n </tr>\n <tr height=207 style='height:155.0pt'>\n  <td rowspan=4 height=326 class=xl67 width=177 style='border-bottom:1.0pt;\n  height:244.0pt;border-top:none;width:133pt'><span\n  style='mso-spacerun:yes'> </span>clusterList</td>\n  <td rowspan=4 class=xl70 width=175 style='border-bottom:1.0pt;\n  border-top:none;width:131pt'>query cluster info, including cluster, BrokerName, BrokerId, TPS, eg.</td>\n  <td class=xl65 width=177 style='width:133pt'>-m</td>\n  <td class=xl66 width=185 style='width:139pt'>print more infos(eg: #InTotalYest, #OutTotalYest, #InTotalToday ,#OutTotalToday)</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=177 style='height:17.0pt;width:133pt'>-h</td>\n  <td class=xl66 width=185 style='width:139pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=177 style='height:43.0pt;width:133pt'>-n</td>\n  <td class=xl66 width=185 style='width:139pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=177 style='height:29.0pt;width:133pt'>-i</td>\n  <td class=xl66 width=185 style='width:139pt'>print interval, unit second</td>\n </tr>\n <tr height=95 style='height:71.0pt'>\n  <td rowspan=8 height=391 class=xl67 width=177 style='border-bottom:1.0pt;\n  height:292.0pt;border-top:none;width:133pt'>clusterRT</td>\n  <td rowspan=8 class=xl70 width=175 style='border-bottom:1.0pt;\n  border-top:none;width:131pt'>send message to detect each cluster's Broker RT. Message will be sent to ${BrokerName} Topic.</td>\n  <td class=xl65 width=177 style='width:133pt'>-a</td>\n  <td class=xl66 width=185 style='width:139pt'>amount, count of detection, RT = sum time /\n  amount</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=177 style='height:29.0pt;width:133pt'>-s</td>\n  <td class=xl66 width=185 style='width:139pt'>size of message, unit B</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=177 style='height:17.0pt;width:133pt'>-c</td>\n  <td class=xl66 width=185 style='width:139pt'>which cluster will be detected</td>\n </tr>\n <tr height=76 style='height:57.0pt'>\n  <td height=76 class=xl65 width=177 style='height:57.0pt;width:133pt'>-p</td>\n  <td class=xl66 width=185 style='width:139pt'>whether print format log, split by |, default is not print</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl65 width=177 style='height:17.0pt;width:133pt'>-h</td>\n  <td class=xl66 width=185 style='width:139pt'>print help info</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=177 style='height:29.0pt;width:133pt'>-m</td>\n  <td class=xl66 width=185 style='width:139pt'>which machine room it belongs to, just for print</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl65 width=177 style='height:29.0pt;width:133pt'>-i</td>\n  <td class=xl66 width=185 style='width:139pt'>send interval, unit second</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl65 width=177 style='height:43.0pt;width:133pt'>-n</td>\n  <td class=xl66 width=185 style='width:139pt'>NameServer Service address, format is ip:port</td>\n </tr>\n</table>\n\n\n#### 2.3 Broker\n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>名称</td>\n  <td class=xl64 width=175 style='width:131pt'>meaning</td>\n  <td class=xl64 width=177 style='width:133pt'>command items</td>\n  <td class=xl64 width=185 style='width:139pt'>explanation</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=6 height=206 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:154.0pt;border-top:none;width:143pt'>updateBrokerConfig</td>\n  <td rowspan=6 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>update Broker's config file, it will modify Broker.conf</td>\n  <td class=xl67 width=87 style='width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Broker address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>cluster name</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-k</td>\n  <td class=xl68 width=87 style='width:65pt'>key</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-v</td>\n  <td class=xl68 width=87 style='width:65pt'>value</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=3 height=137 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:103.0pt;border-top:none;width:143pt'>brokerStatus</td>\n  <td rowspan=3 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>get Broker's statistics info, running status(including whatever you want).</td>\n  <td class=xl67 width=87 style='width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Broker address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=6 height=256 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:192.0pt;border-top:none;width:143pt'>brokerConsumeStats</td>\n  <td rowspan=6 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>Broker's consumer info, including Consume Offset, Broker Offset, Diff, Timestamp that ordered by message Queue</td>\n  <td class=xl67 width=87 style='width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Broker address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>request timeout time</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-l</td>\n  <td class=xl68 width=87 style='width:65pt'>diff threshold, it will print when exceed this threshold.</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-o</td>\n  <td class=xl68 width=87 style='width:65pt'>whether is sequential topic, generally false</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=2 height=114 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:86.0pt;border-top:none;width:143pt'>getBrokerConfig</td>\n  <td rowspan=2 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>get Broker's config</td>\n  <td class=xl67 width=87 style='width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Broker address, format is ip:port</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=3 height=137 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:103.0pt;border-top:none;width:143pt'>wipeWritePerm</td>\n  <td rowspan=3 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>revoke broker's write authority from NameServer.</td>\n  <td class=xl67 width=87 style='width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Broker address, format is ip:port</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=4 height=160 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:120.0pt;border-top:none;width:143pt'>cleanExpiredCQ</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>clean Broker's expired Consume Queue that maybe generated by decrease queue count.</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Broker address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>cluster name</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=4 height=160 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:120.0pt;border-top:none;width:143pt'>deleteExpiredCommitLog</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>delete Broker's expired CommitLog files.</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Broker address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>cluster name</td>\n </tr>\n <tr height=88 style='mso-height-source:userset;height:66.0pt'>\n  <td rowspan=4 height=191 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:143.0pt;border-top:none;width:143pt'>cleanUnusedTopic</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>clean Broker's unused Topic that deleted manually to release memory that Topic's Consume Queue occupied.</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Broker address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>cluster name</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=5 height=199 class=xl69 width=191 style='border-bottom:1.0pt;\n  height:149.0pt;border-top:none;width:143pt'>sendMsgStatus</td>\n  <td rowspan=5 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>send message to Broker, return send status and RT</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>BrokerName, is different from broker address</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>message size, unit B</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>send count</td>\n </tr>\n</table>\n\n\n#### 2.4 Message\n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n<tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>名称</td>\n  <td class=xl64 width=175 style='width:131pt'>meaning</td>\n  <td class=xl64 width=177 style='width:133pt'>command items</td>\n  <td class=xl64 width=185 style='width:139pt'>explanation</td>\n </tr>\n <tr height=128 style='height:96.0pt'>\n  <td rowspan=3 height=208 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:156.0pt;border-top:none;width:65pt'>queryMsgById</td>\n  <td rowspan=3 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>query message by offsetMsgId. If use opensource console, it should use offsetMsgId. Please refer to QueryMsgByIdSubCommand for detail.</td>\n  <td class=xl67 width=87 style='width:65pt'>-i</td>\n  <td class=xl67 width=87 style='width:65pt'>msgId</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=4 height=126 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:94.0pt;border-top:none;width:65pt'>queryMsgByKey</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>query message by Message's Key</td>\n  <td class=xl67 width=87 style='width:65pt'>-k</td>\n  <td class=xl67 width=87 style='width:65pt'>msgKey</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>topic name</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=225 style='height:169.0pt'>\n  <td rowspan=6 height=390 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:292.0pt;border-top:none;width:65pt'>queryMsgByOffset</td>\n  <td rowspan=6 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>query message by Offset</td>\n  <td class=xl67 width=87 style='width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Broker name(it's not broker address, can query Broker name by clusterList).</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-i</td>\n  <td class=xl68 width=87 style='width:65pt'>query queue id</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-o</td>\n  <td class=xl68 width=87 style='width:65pt'>offset value</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>topic name</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=47>\n  <td rowspan=6 height=209 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:156.0pt;border-top:none;width:65pt'>queryMsgByUniqueKey</td>\n  <td rowspan=6 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>query by msgId, msgId is different from offsetMsgId, please refer to Frequently asked questions about operations for details. Use -g and -d to let specified consumer return consume result.</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-i</td>\n  <td class=xl67 width=87 style='width:65pt'>unique msg id</td>\n </tr>\n <tr height=36 style='height:27.0pt'>\n  <td height=36 class=xl67 width=87 style='height:27.0pt;width:65pt'>-g</td>\n  <td class=xl67 width=87 style='width:65pt'>consumerGroup</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-d</td>\n  <td class=xl67 width=87 style='width:65pt'>clientId</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>topic name</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=5 height=149 class=xl69 width=87 style='border-bottom:1.0pt\n  height:111.0pt;border-top:none;width:65pt'>checkMsgSendRT</td>\n  <td rowspan=5 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>detect RT of sending a message to a topic, similar to clusterRT</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>topic name</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-a</td>\n  <td class=xl68 width=87 style='width:65pt'>detection count</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>size of the message</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=8 height=218 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:162.0pt;border-top:none;width:65pt'>sendMessage</td>\n  <td rowspan=8 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>send a message, also can send to a specified Message Queue.</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>topic name</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-p</td>\n  <td class=xl68 width=87 style='width:65pt'>body, message entity</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-k</td>\n  <td class=xl67 width=87 style='width:65pt'>keys</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl67 width=87 style='width:65pt'>tags</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-b</td>\n  <td class=xl67 width=87 style='width:65pt'>BrokerName</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-i</td>\n  <td class=xl67 width=87 style='width:65pt'>queueId</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=10 height=312 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:232.0pt;border-top:none;width:65pt'>consumeMessage</td>\n  <td rowspan=10 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>consume message. Different consume logic depends on offset, start & end timestamp, message queue, please refer to ConsumeMessageCommand for details.</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>topic name</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-b</td>\n  <td class=xl67 width=87 style='width:65pt'>BrokerName</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-o</td>\n  <td class=xl68 width=87 style='width:65pt'>offset that consumer start consume</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-i</td>\n  <td class=xl67 width=87 style='width:65pt'>queueId</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-g</td>\n  <td class=xl68 width=87 style='width:65pt'>consumer group</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>timestamp at start, refer to -h to get format开</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-d</td>\n  <td class=xl68 width=87 style='width:65pt'>timestamp at the end</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>size of message that consumed</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=8 height=282 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:210.0pt;border-top:none;width:65pt'>printMsg</td>\n  <td rowspan=8 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>consume and print messages from broker, support a time range</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>topic name</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>charset, eg: UTF-8</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>subExpress, filter expression</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>timestamp at start, refer to -h to get format</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-e</td>\n  <td class=xl68 width=87 style='width:65pt'>timestamp at the end</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-d</td>\n  <td class=xl68 width=87 style='width:65pt'>whether print message entity or not</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=12 height=390 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:290.0pt;border-top:none;width:65pt'>printMsgByQueue</td>\n  <td rowspan=12 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>similar to printMsg, but it need specified Message Queue</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>topic name</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-i</td>\n  <td class=xl67 width=87 style='width:65pt'>queueId</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-a</td>\n  <td class=xl67 width=87 style='width:65pt'>BrokerName</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>charset, eg: UTF-8</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>subExpress, filter expression</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>timestamp at start, refer to -h to get format</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-e</td>\n  <td class=xl68 width=87 style='width:65pt'>timestamp at the end</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-p</td>\n  <td class=xl68 width=87 style='width:65pt'>whether print message or not</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-d</td>\n  <td class=xl68 width=87 style='width:65pt'>whether print message entity or not</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-f</td>\n  <td class=xl68 width=87 style='width:65pt'>whether count and print tag or not</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=7 height=410 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:307.0pt;border-top:none;width:65pt'>resetOffsetByTime</td>\n  <td rowspan=7 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>reset offset by timestamp, Broker and consumer will all be reset</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-g</td>\n  <td class=xl68 width=87 style='width:65pt'>consumer group</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>topic name</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>reset offset corresponding to this timestamp</td>\n </tr>\n <tr height=188 style='height:141.0pt'>\n  <td height=188 class=xl67 width=87 style='height:141.0pt;width:65pt'>-f</td>\n  <td class=xl68 width=87 style='width:65pt'>whether enforce to reset or not, if set false, only can reset offset, if set true, it omit the relationship between timestamp and consumer offset.</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>whether reset c++ sdk's offset or not</td>\n </tr>\n</table>\n\n\n#### 2.5 Consumer, Consumer Group\n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n<tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>name</td>\n  <td class=xl64 width=175 style='width:131pt'>meaning</td>\n  <td class=xl64 width=177 style='width:133pt'>command items</td>\n  <td class=xl64 width=185 style='width:139pt'>explanation</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td rowspan=4 height=158 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:110pt;border-top:none;width:65pt'>consumerProgress</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt;\n  border-top:none;width:65pt'>query subscribe status, can get blocking counts of a concrete client ip.</td>\n  <td class=xl67 width=87 style='width:65pt'>-g</td>\n  <td class=xl68 width=87 style='width:65pt'>consumer group name</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>whether print client IP or not</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=105 style='mso-height-source:userset;height:79.0pt'>\n  <td rowspan=5 height=260 class=xl69 width=87 style='border-bottom:1.0pt;\n  height:195.0pt;border-top:none;width:65pt'>consumerStatus</td>\n  <td rowspan=5 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>query consumer status, including message blocking, and consumer's jstack result(please refer to ConsumerStatusSubCommand)</td>\n  <td class=xl67 width=87 style='width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=36 style='height:27.0pt'>\n  <td height=36 class=xl67 width=87 style='height:27.0pt;width:65pt'>-g</td>\n  <td class=xl67 width=87 style='width:65pt'>consumer group</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-i</td>\n  <td class=xl67 width=87 style='width:65pt'>clientId</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>whether execute jstack or not</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=13 height=761 class=xl69 width=87 style='border-bottom:1.0pt\n  height:569.0pt;border-top:none;width:65pt'>updateSubGroup</td>\n  <td rowspan=13 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>create or update subscribe info</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Broker address</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>cluster name</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-g</td>\n  <td class=xl68 width=87 style='width:65pt'>consumer group name</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>consumer group is allowed to consume or not</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-m</td>\n  <td class=xl68 width=87 style='width:65pt'>start consume from minimal offset or not</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-d</td>\n  <td class=xl68 width=87 style='width:65pt'>broadcast mode or not</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-q</td>\n  <td class=xl68 width=87 style='width:65pt'>capacity of retry queue</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-r</td>\n  <td class=xl68 width=87 style='width:65pt'>max retry count</td>\n </tr>\n <tr height=207 style='height:155.0pt'>\n  <td height=207 class=xl67 width=87 style='height:155.0pt;width:65pt'>-i</td>\n  <td class=xl68 width=87 style='width:65pt'>It works when slaveReadEnable enabled, and that not consumed from slave. Suggesting that consume from slave node by specify slave id.</td>\n </tr>\n <tr height=132 style='height:99.0pt'>\n  <td height=132 class=xl67 width=87 style='height:99.0pt;width:65pt'>-w</td>\n  <td class=xl68 width=87 style='width:65pt'>If broker consume from slave, which slave node depends on this config that defined by BrokerId, eg: 1.</td>\n </tr>\n <tr height=76 style='height:57.0pt'>\n  <td height=76 class=xl67 width=87 style='height:57.0pt;width:65pt'>-a</td>\n  <td class=xl68 width=87 style='width:65pt'>whether notify other consumers to rebalance or not when the count of consumer changes</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=5 height=165 class=xl69 width=87 style='border-bottom:1.0pt\n  height:123.0pt;border-top:none;width:65pt'>deleteSubGroup</td>\n  <td rowspan=5 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>delete subscribe info from Broker</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-b</td>\n  <td class=xl68 width=87 style='width:65pt'>Broker address</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-c</td>\n  <td class=xl68 width=87 style='width:65pt'>cluster name</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td height=39 class=xl67 width=87 style='height:29.0pt;width:65pt'>-g</td>\n  <td class=xl68 width=87 style='width:65pt'>consumer group name</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=6 height=172 class=xl69 width=87 style='border-bottom:1.0pt\n  height:120pt;border-top:none;width:65pt'>cloneGroupOffset</td>\n  <td rowspan=6 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>use source group's offset at target group</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>source consumer group</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-d</td>\n  <td class=xl68 width=87 style='width:65pt'>target consumer group</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>topic name</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-o</td>\n  <td class=xl68 width=87 style='width:65pt'>not used at present</td>\n </tr>\n</table>\n\n\n\n\n#### 2.6 Connection\n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n<tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>name</td>\n  <td class=xl64 width=175 style='width:131pt'>meaning</td>\n  <td class=xl64 width=177 style='width:133pt'>command items</td>\n  <td class=xl64 width=185 style='width:139pt'>explanation</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td rowspan=3 height=119 class=xl69 width=87 style='border-bottom:1.0pt\n  height:89.0pt;border-top:none;width:65pt'>consumerConnection</td>\n  <td rowspan=3 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>query Consumer's connection</td>\n  <td class=xl67 width=87 style='width:65pt'>-g</td>\n  <td class=xl68 width=87 style='width:65pt'>consumer group name</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=39 style='height:29.0pt'>\n  <td rowspan=4 height=142 class=xl69 width=87 style='border-bottom:1.0pt\n  height:106.0pt;border-top:none;width:65pt'>producerConnection</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>query Producer's connection</td>\n  <td class=xl67 width=87 style='width:65pt'>-g</td>\n  <td class=xl68 width=87 style='width:65pt'>producer group name</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-t</td>\n  <td class=xl68 width=87 style='width:65pt'>topic name</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n</table>\n\n\n\n\n#### 2.7 NameServer\n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n<tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>name</td>\n  <td class=xl64 width=175 style='width:131pt'>meaning</td>\n  <td class=xl64 width=177 style='width:133pt'>command items</td>\n  <td class=xl64 width=185 style='width:139pt'>explanation</td>\n </tr>\n <tr height=21 style='height:16.0pt'>\n  <td rowspan=5 height=143 class=xl69 width=87 style='border-bottom:1.0pt\n  height:100pt;border-top:none;width:65pt'>updateKvConfig</td>\n  <td rowspan=5 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>update NameServer's kv config, not used at present</td>\n  <td class=xl75 width=87 style='width:65pt'>-s</td>\n  <td class=xl76 width=87 style='width:65pt'>namespace</td>\n </tr>\n <tr height=21 style='height:16.0pt'>\n  <td height=21 class=xl75 width=87 style='height:16.0pt;width:65pt'>-k</td>\n  <td class=xl75 width=87 style='width:65pt'>key</td>\n </tr>\n <tr height=21 style='height:16.0pt'>\n  <td height=21 class=xl75 width=87 style='height:16.0pt;width:65pt'>-v</td>\n  <td class=xl75 width=87 style='width:65pt'>value</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td rowspan=4 height=126 class=xl69 width=87 style='border-bottom:1.0pt\n  height:94.0pt;border-top:none;width:65pt'>deleteKvConfig</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>delete NameServer's kv config</td>\n  <td class=xl67 width=87 style='width:65pt'>-s</td>\n  <td class=xl68 width=87 style='width:65pt'>namespace</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-k</td>\n  <td class=xl67 width=87 style='width:65pt'>key</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=2 height=80 class=xl69 width=87 style='border-bottom:1.0pt\n  height:60.0pt;border-top:none;width:65pt'>getNamesrvConfig</td>\n  <td rowspan=2 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>get NameServer's config</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=4 height=126 class=xl69 width=87 style='border-bottom:1.0pt\n  height:94.0pt;border-top:none;width:65pt'>updateNamesrvConfig</td>\n  <td rowspan=4 class=xl72 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>modify NameServer's config</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-k</td>\n  <td class=xl67 width=87 style='width:65pt'>key</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-v</td>\n  <td class=xl67 width=87 style='width:65pt'>value</td>\n </tr>\n</table>\n\n\n\n\n#### 2.8 Other\n\n<table border=0 cellpadding=0 cellspacing=0 width=714>\n <col width=177>\n <col width=175>\n <col width=177>\n <col width=185>\n<tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl63 width=177 style='height:17.0pt;width:133pt'>name</td>\n  <td class=xl64 width=175 style='width:131pt'>meaning</td>\n  <td class=xl64 width=177 style='width:133pt'>command items</td>\n  <td class=xl64 width=185 style='width:139pt'>explanation</td>\n </tr>\n <tr height=57 style='height:43.0pt'>\n  <td rowspan=2 height=80 class=xl69 width=87 style='border-bottom:1.0pt\n  height:60.0pt;border-top:none;width:65pt'>startMonitoring</td>\n  <td rowspan=2 class=xl71 width=87 style='border-bottom:1.0pt\n  border-top:none;width:65pt'>Start the monitoring process, monitor message deletion and the number of retried messages in the queue</td>\n  <td class=xl67 width=87 style='width:65pt'>-n</td>\n  <td class=xl68 width=87 style='width:65pt'>NameServer Service address, format is ip:port</td>\n </tr>\n <tr height=23 style='height:17.0pt'>\n  <td height=23 class=xl67 width=87 style='height:17.0pt;width:65pt'>-h</td>\n  <td class=xl68 width=87 style='width:65pt'>print help info</td>\n </tr>\n</table>\n\n\n### 3   Frequently asked questions about operations\n\n#### 3.1 RocketMQ's mqadmin command error\n\n>  question description: execute mqadmin occur below exception after deploy RocketMQ cluster.\n>\n> ```java\n> org.apache.rocketmq.remoting.exception.RemotingConnectException: connect to <null> failed\n> ```\n\nSolution: execute command `export NAMESRV_ADDR=ip:9876` (ip is NameServer's ip address), then execute mqadmin commands.\n\n#### 3.2 RocketMQ consumer cannot consume, because of different version of producer and consumer.\n\n> question description: one producer produce message, consumer A can consume, consume B cannot consume, RocketMQ console print:\n>\n> ```java\n> Not found the consumer group consume stats, because return offset table is empty, maybe the consumer not consume any message.\n> ```\n\nSolution: make sure that producer and consumer has the same version of rocketmq-client.\n\n#### 3.3  Consumer cannot consume oldest message, when a new consumer group is added.\n\n> question description: when a new consumer group start, it consumes from current offset, do not fetch oldest message.    \n\nSolution: rocketmq's default policy is consume from latest, that is skip oldest message. If you want consume oldest message, you need to set `org.apache.rocketmq.client.consumer.DefaultMQPushConsumer#setConsumeFromWhere`. The following is three common configurations:\n\n- default configuration, a new consumer group consume from latest position at first startup, then consume from last time's offset at next startup, that is skip oldest message;\n\n```java\nconsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);\n```\n\n- a new consumer group consume from oldest position at first startup, then consume from last time's offset at next startup, that is consume the unexpired message;\n\n```java\nconsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n```\n\n- a new consumer group consume from specified timestamp at first startup, then consume from last time's offset at next startup, cooperate with consumer.setConsumeTimestamp(), default is half an hour before;\n\n```java\nconsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_TIMESTAMP);\n```\n\n#### 3.4 How to enable consume from Slave\n\nIn some cases, consumer need reset offset to a day or two before, if Master Broker has limited memory, it's CommitLog will have a high IO load, then it will impact other message's read and write that on this broker. When `slaveReadEnable=true` is set, and consumer's offset exceeds `accessMessageInMemoryMaxRatio=40%`, Master Broker will recommend consumer consume from Slave Broker to lower Master Broker IO.  \n\n#### 3.5 Performance tuning\n\nA spin lock is recommended for asynchronous disk flush, a reentrant lock is recommended for synchronous disk flush, configuration item is `useReentrantLockWhenPutMessage`, default is false; Enable `TransientStorePoolEnable` is recommended when use asynchronous disk flush; Recommend to close `transferMsgByHeap` to improve fetch efficiency; Set a little larger `sendMessageThreadPoolNums`, when use synchronous disk flush.\n\n#### 3.6 The meaning and difference between msgId and offsetMsgId in RocketMQ\n\nYou will usually see the following log print message after sending message by using RocketMQ sdk.\n\n```java\nSendResult [sendStatus=SEND_OK, msgId=0A42333A0DC818B4AAC246C290FD0000, offsetMsgId=0A42333A00002A9F000000000134F1F5, messageQueue=MessageQueue [topic=topicTest1, BrokerName=mac.local, queueId=3], queueOffset=4]\n```\n\n- msgId, is generated by producer sdk. In particular, call method `MessageClientIDSetter.createUniqIDBuffer()` to generate unique Id;\n- offsetMsgId, offsetMsgId is generated by Broker server(format is \"Ip Address + port + CommitLog offset\"). offsetMsgId is messageId that is RocketMQ console's input.\n"
  },
  {
    "path": "docs/en/proxy/deploy_guide.md",
    "content": "# RocketMQ Proxy Deployment Guide\n\n## Overview\n\nRocketMQ Proxy supports two deployment modes: `Local` and `Cluster`.\n\n## Configuration\n\nThe configuration file applies to both `Cluster` and `Local` mode, whose default path is\ndistribution/conf/rmq-proxy.json.\n\n## `Cluster` Mode\n\n* Set configuration field `nameSrvAddr`.\n* Set configuration field `proxyMode` to `cluster` (case insensitive).\n\nRun the command below.\n\n```shell\nnohup sh mqproxy &\n```\n\nThe command will only launch the `Proxy` component itself. It assumes that `Namesrv` nodes are already running at the address specified `nameSrvAddr`, and broker nodes, registering themselves with `nameSrvAddr`, are running too.\n\n## `Local` Mode\n\n* Set configuration field `nameSrvAddr`.\n* Set configuration field `proxyMode` to `local` (case insensitive).\n\nRun the command below.\n\n```shell\nnohup sh mqproxy &\n```\n\nThe previous command will launch the `Proxy`, with `Broker` in the same process. It assumes `Namesrv` nodes are running at the address specified by `nameSrvAddr`.\n"
  },
  {
    "path": "example/pom.xml",
    "content": "<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n    <parent>\n        <artifactId>rocketmq-all</artifactId>\n        <groupId>org.apache.rocketmq</groupId>\n        <version>${revision}</version>\n    </parent>\n\n    <modelVersion>4.0.0</modelVersion>\n    <packaging>jar</packaging>\n    <artifactId>rocketmq-example</artifactId>\n    <name>rocketmq-example ${project.version}</name>\n\n    <properties>\n        <project.root>${basedir}/..</project.root>\n    </properties>\n\n    <dependencies>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-client</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-tools</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-remoting</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-srvutil</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-openmessaging</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-auth</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.javassist</groupId>\n            <artifactId>javassist</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.jaegertracing</groupId>\n            <artifactId>jaeger-core</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.jaegertracing</groupId>\n            <artifactId>jaeger-client</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.jaegertracing</groupId>\n            <artifactId>jaeger-thrift</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>commons-cli</groupId>\n            <artifactId>commons-cli</artifactId>\n        </dependency>\n    </dependencies>\n</project>\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/batch/SimpleBatchProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.example.batch;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.message.Message;\n\npublic class SimpleBatchProducer {\n\n    public static final String PRODUCER_GROUP = \"BatchProducerGroupName\";\n    public static final String DEFAULT_NAMESRVADDR = \"127.0.0.1:9876\";\n    public static final String TOPIC = \"BatchTest\";\n    public static final String TAG = \"Tag\";\n\n    public static void main(String[] args) throws Exception {\n        DefaultMQProducer producer = new DefaultMQProducer(PRODUCER_GROUP);\n        // Uncomment the following line while debugging, namesrvAddr should be set to your local address\n//        producer.setNamesrvAddr(DEFAULT_NAMESRVADDR);\n        producer.start();\n\n        //If you just send messages of no more than 1MiB at a time, it is easy to use batch\n        //Messages of the same batch should have: same topic, same waitStoreMsgOK and no schedule support\n        List<Message> messages = new ArrayList<>();\n        messages.add(new Message(TOPIC, TAG, \"OrderID001\", \"Hello world 0\".getBytes(StandardCharsets.UTF_8)));\n        messages.add(new Message(TOPIC, TAG, \"OrderID002\", \"Hello world 1\".getBytes(StandardCharsets.UTF_8)));\n        messages.add(new Message(TOPIC, TAG, \"OrderID003\", \"Hello world 2\".getBytes(StandardCharsets.UTF_8)));\n\n        SendResult sendResult = producer.send(messages);\n        System.out.printf(\"%s\", sendResult);\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/batch/SplitBatchProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.example.batch;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.message.Message;\n\npublic class SplitBatchProducer {\n\n    public static final String PRODUCER_GROUP = \"BatchProducerGroupName\";\n    public static final String DEFAULT_NAMESRVADDR = \"127.0.0.1:9876\";\n\n    public static final int MESSAGE_COUNT = 100 * 1000;\n    public static final String TOPIC = \"BatchTest\";\n    public static final String TAG = \"Tag\";\n\n    public static void main(String[] args) throws Exception {\n\n        DefaultMQProducer producer = new DefaultMQProducer(PRODUCER_GROUP);\n        // Uncomment the following line while debugging, namesrvAddr should be set to your local address\n//        producer.setNamesrvAddr(DEFAULT_NAMESRVADDR);\n        producer.start();\n\n        //large batch\n        List<Message> messages = new ArrayList<>(MESSAGE_COUNT);\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            messages.add(new Message(TOPIC, TAG, \"OrderID\" + i, (\"Hello world \" + i).getBytes(StandardCharsets.UTF_8)));\n        }\n\n        //split the large batch into small ones:\n        ListSplitter splitter = new ListSplitter(messages);\n        while (splitter.hasNext()) {\n            List<Message> listItem = splitter.next();\n            SendResult sendResult = producer.send(listItem);\n            System.out.printf(\"%s\", sendResult);\n        }\n    }\n\n}\n\nclass ListSplitter implements Iterator<List<Message>> {\n    private static final int SIZE_LIMIT = 1000 * 1000;\n    private final List<Message> messages;\n    private int currIndex;\n\n    public ListSplitter(List<Message> messages) {\n        this.messages = messages;\n    }\n\n    @Override\n    public boolean hasNext() {\n        return currIndex < messages.size();\n    }\n\n    @Override\n    public List<Message> next() {\n        int nextIndex = currIndex;\n        int totalSize = 0;\n        for (; nextIndex < messages.size(); nextIndex++) {\n            Message message = messages.get(nextIndex);\n            int tmpSize = message.getTopic().length() + message.getBody().length;\n            Map<String, String> properties = message.getProperties();\n            for (Map.Entry<String, String> entry : properties.entrySet()) {\n                tmpSize += entry.getKey().length() + entry.getValue().length();\n            }\n            //for log overhead\n            tmpSize = tmpSize + 20;\n            if (tmpSize > SIZE_LIMIT) {\n                //it is unexpected that single message exceeds the sizeLimit\n                //here just let it go, otherwise it will block the splitting process\n                if (nextIndex - currIndex == 0) {\n                    //if the next sublist has no element, add this one and then break, otherwise just break\n                    nextIndex++;\n                }\n                break;\n            }\n            if (tmpSize + totalSize > SIZE_LIMIT) {\n                break;\n            } else {\n                totalSize += tmpSize;\n            }\n\n        }\n        List<Message> subList = messages.subList(currIndex, nextIndex);\n        currIndex = nextIndex;\n        return subList;\n    }\n\n    @Override\n    public void remove() {\n        throw new UnsupportedOperationException(\"Not allowed to remove\");\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/benchmark/AclClient.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.example.benchmark;\n\nimport org.apache.rocketmq.acl.common.AclClientRPCHook;\nimport org.apache.rocketmq.acl.common.SessionCredentials;\nimport org.apache.rocketmq.remoting.RPCHook;\n\npublic class AclClient {\n\n    public static final String ACL_ACCESS_KEY = \"rocketmq2\";\n\n    public static final String ACL_SECRET_KEY = \"12345678\";\n\n    public static RPCHook getAclRPCHook() {\n        return getAclRPCHook(ACL_ACCESS_KEY, ACL_SECRET_KEY);\n    }\n\n    public static RPCHook getAclRPCHook(String ak, String sk) {\n        return new AclClientRPCHook(new SessionCredentials(ak, sk));\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/benchmark/BatchProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.benchmark;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Random;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.concurrent.atomic.LongAdder;\nimport org.apache.commons.cli.CommandLine;\nimport org.apache.commons.cli.DefaultParser;\nimport org.apache.commons.cli.Option;\nimport org.apache.commons.cli.Options;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.RandomStringUtils;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.compression.CompressionType;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.SerializeType;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.srvutil.ServerUtil;\n\npublic class BatchProducer {\n\n    private static byte[] msgBody;\n\n    public static void main(String[] args) throws MQClientException {\n        System.setProperty(RemotingCommand.SERIALIZE_TYPE_PROPERTY, SerializeType.ROCKETMQ.name());\n\n        Options options = ServerUtil.buildCommandlineOptions(new Options());\n        CommandLine commandLine = ServerUtil.parseCmdLine(\"benchmarkBatchProducer\", args, buildCommandlineOptions(options), new DefaultParser());\n        if (null == commandLine) {\n            System.exit(-1);\n        }\n\n        final String namesrv = getOptionValue(commandLine, 'n', \"127.0.0.1:9876\");\n        final String topic = getOptionValue(commandLine, 't', \"BenchmarkTest\");\n        final int threadCount = getOptionValue(commandLine, 'w', 64);\n        final int messageSize = getOptionValue(commandLine, 's', 128);\n        final int batchSize = getOptionValue(commandLine, 'b', 16);\n        final boolean keyEnable = getOptionValue(commandLine, 'k', false);\n        final int propertySize = getOptionValue(commandLine, 'p', 0);\n        final int tagCount = getOptionValue(commandLine, 'l', 0);\n        final boolean msgTraceEnable = getOptionValue(commandLine, 'm', false);\n        final boolean aclEnable = getOptionValue(commandLine, 'a', false);\n        final boolean enableCompress = commandLine.hasOption('c') && Boolean.parseBoolean(commandLine.getOptionValue('c'));\n        final int reportInterval = commandLine.hasOption(\"ri\") ? Integer.parseInt(commandLine.getOptionValue(\"ri\")) : 10000;\n\n        System.out.printf(\"topic: %s, threadCount: %d, messageSize: %d, batchSize: %d, keyEnable: %s, propertySize: %d, tagCount: %d, traceEnable: %s, \" +\n                \"aclEnable: %s%n compressEnable: %s, reportInterval: %d%n\",\n            topic, threadCount, messageSize, batchSize, keyEnable, propertySize, tagCount, msgTraceEnable, aclEnable, enableCompress, reportInterval);\n\n        StringBuilder sb = new StringBuilder(messageSize);\n        for (int i = 0; i < messageSize; i++) {\n            sb.append(RandomStringUtils.randomAlphanumeric(1));\n        }\n        msgBody = sb.toString().getBytes(StandardCharsets.UTF_8);\n\n        final StatsBenchmarkBatchProducer statsBenchmark = new StatsBenchmarkBatchProducer(reportInterval);\n        statsBenchmark.start();\n\n        RPCHook rpcHook = null;\n        if (aclEnable) {\n            String ak = commandLine.hasOption(\"ak\") ? String.valueOf(commandLine.getOptionValue(\"ak\")) : AclClient.ACL_ACCESS_KEY;\n            String sk = commandLine.hasOption(\"sk\") ? String.valueOf(commandLine.getOptionValue(\"sk\")) : AclClient.ACL_SECRET_KEY;\n            rpcHook = AclClient.getAclRPCHook(ak, sk);\n        }\n\n        final DefaultMQProducer producer = initInstance(namesrv, msgTraceEnable, rpcHook);\n\n        if (enableCompress) {\n            String compressType = commandLine.hasOption(\"ct\") ? commandLine.getOptionValue(\"ct\").trim() : \"ZLIB\";\n            int compressLevel = commandLine.hasOption(\"cl\") ? Integer.parseInt(commandLine.getOptionValue(\"cl\")) : 5;\n            int compressOverHowMuch = commandLine.hasOption(\"ch\") ? Integer.parseInt(commandLine.getOptionValue(\"ch\")) : 4096;\n            producer.setCompressType(CompressionType.of(compressType));\n            producer.setCompressLevel(compressLevel);\n            producer.setCompressMsgBodyOverHowmuch(compressOverHowMuch);\n            System.out.printf(\"compressType: %s compressLevel: %s%n\", compressType, compressLevel);\n        } else {\n            producer.setCompressMsgBodyOverHowmuch(Integer.MAX_VALUE);\n        }\n\n        producer.start();\n\n        final Logger logger = LoggerFactory.getLogger(BatchProducer.class);\n        final ExecutorService sendThreadPool = Executors.newFixedThreadPool(threadCount);\n        for (int i = 0; i < threadCount; i++) {\n            sendThreadPool.execute(new Runnable() {\n                @Override\n                public void run() {\n                    while (true) {\n                        List<Message> msgs = buildBathMessage(batchSize, topic);\n\n                        if (CollectionUtils.isEmpty(msgs)) {\n                            return;\n                        }\n\n                        try {\n                            long beginTimestamp = System.currentTimeMillis();\n                            long sendSucCount = statsBenchmark.getSendMessageSuccessCount().longValue();\n\n                            setKeys(keyEnable, msgs, String.valueOf(beginTimestamp / 1000));\n                            setTags(tagCount, msgs, sendSucCount);\n                            setProperties(propertySize, msgs);\n                            SendResult sendResult = producer.send(msgs);\n                            if (sendResult.getSendStatus() == SendStatus.SEND_OK) {\n                                statsBenchmark.getSendRequestSuccessCount().increment();\n                                statsBenchmark.getSendMessageSuccessCount().add(msgs.size());\n                            } else {\n                                statsBenchmark.getSendRequestFailedCount().increment();\n                                statsBenchmark.getSendMessageFailedCount().add(msgs.size());\n                            }\n                            long currentRT = System.currentTimeMillis() - beginTimestamp;\n                            statsBenchmark.getSendMessageSuccessTimeTotal().add(currentRT);\n                            long prevMaxRT = statsBenchmark.getSendMessageMaxRT().longValue();\n                            while (currentRT > prevMaxRT) {\n                                boolean updated = statsBenchmark.getSendMessageMaxRT().compareAndSet(prevMaxRT, currentRT);\n                                if (updated) {\n                                    break;\n                                }\n\n                                prevMaxRT = statsBenchmark.getSendMessageMaxRT().get();\n                            }\n                        } catch (RemotingException e) {\n                            statsBenchmark.getSendRequestFailedCount().increment();\n                            statsBenchmark.getSendMessageFailedCount().add(msgs.size());\n                            logger.error(\"[BENCHMARK_PRODUCER] Send Exception\", e);\n\n                            try {\n                                Thread.sleep(3000);\n                            } catch (InterruptedException ignored) {\n                            }\n                        } catch (InterruptedException e) {\n                            statsBenchmark.getSendRequestFailedCount().increment();\n                            statsBenchmark.getSendMessageFailedCount().add(msgs.size());\n                            try {\n                                Thread.sleep(3000);\n                            } catch (InterruptedException e1) {\n                            }\n                            statsBenchmark.getSendRequestFailedCount().increment();\n                            statsBenchmark.getSendMessageFailedCount().add(msgs.size());\n                            logger.error(\"[BENCHMARK_PRODUCER] Send Exception\", e);\n                        } catch (MQClientException e) {\n                            statsBenchmark.getSendRequestFailedCount().increment();\n                            statsBenchmark.getSendMessageFailedCount().add(msgs.size());\n                            logger.error(\"[BENCHMARK_PRODUCER] Send Exception\", e);\n                        } catch (MQBrokerException e) {\n                            statsBenchmark.getSendRequestFailedCount().increment();\n                            statsBenchmark.getSendMessageFailedCount().add(msgs.size());\n                            logger.error(\"[BENCHMARK_PRODUCER] Send Exception\", e);\n                            try {\n                                Thread.sleep(3000);\n                            } catch (InterruptedException ignored) {\n                            }\n                        }\n                    }\n                }\n            });\n        }\n    }\n\n    public static Options buildCommandlineOptions(final Options options) {\n        Option opt = new Option(\"w\", \"threadCount\", true, \"Thread count, Default: 64\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"s\", \"messageSize\", true, \"Message Size, Default: 128\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"b\", \"batchSize\", true, \"Batch Size, Default: 16\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"k\", \"keyEnable\", true, \"Message Key Enable, Default: false\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"t\", \"topic\", true, \"Topic name, Default: BenchmarkTest\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"l\", \"tagCount\", true, \"Tag count, Default: 0\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"m\", \"msgTraceEnable\", true, \"Message Trace Enable, Default: false\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"a\", \"aclEnable\", true, \"Acl Enable, Default: false\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"ak\", \"accessKey\", true, \"Acl Access Key, Default: rocketmq2\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"sk\", \"secretKey\", true, \"Acl Secret Key, Default: 123456789\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"p\", \"propertySize\", true, \"Property Size, Default: 0\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"n\", \"namesrv\", true, \"name server, Default: 127.0.0.1:9876\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"c\", \"compressEnable\", true, \"Enable compress msg over 4K, Default: false\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"ct\", \"compressType\", true, \"Message compressed type, Default: ZLIB\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"cl\", \"compressLevel\", true, \"Message compressed level, Default: 5\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"ch\", \"compressOverHowMuch\", true, \"Compress message when body over how much(unit Byte), Default: 4096\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"ri\", \"reportInterval\", true, \"The number of ms between reports, Default: 10000\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        return options;\n    }\n\n    private static String getOptionValue(CommandLine commandLine, char key, String defaultValue) {\n        if (commandLine.hasOption(key)) {\n            return commandLine.getOptionValue(key).trim();\n        }\n        return defaultValue;\n    }\n\n    private static int getOptionValue(CommandLine commandLine, char key, int defaultValue) {\n        if (commandLine.hasOption(key)) {\n            return Integer.parseInt(commandLine.getOptionValue(key).trim());\n        }\n        return defaultValue;\n    }\n\n    private static boolean getOptionValue(CommandLine commandLine, char key, boolean defaultValue) {\n        if (commandLine.hasOption(key)) {\n            return Boolean.parseBoolean(commandLine.getOptionValue(key).trim());\n        }\n        return defaultValue;\n    }\n\n    private static List<Message> buildBathMessage(final int batchSize, final String topic) {\n        List<Message> batchMessage = new ArrayList<>(batchSize);\n        for (int i = 0; i < batchSize; i++) {\n            Message msg = new Message(topic, msgBody);\n            batchMessage.add(msg);\n        }\n        return batchMessage;\n    }\n\n    private static void setKeys(boolean keyEnable, List<Message> msgs, String keys) {\n        if (!keyEnable) {\n            return;\n        }\n\n        for (Message msg : msgs) {\n            msg.setKeys(keys);\n        }\n    }\n\n    private static void setTags(int tagCount, List<Message> msgs, long startTagId) {\n        if (tagCount <= 0) {\n            return;\n        }\n\n        long tagId = startTagId % tagCount;\n        for (Message msg : msgs) {\n            msg.setTags(String.format(\"tag%d\", tagId++));\n        }\n    }\n\n    private static void setProperties(int propertySize, List<Message> msgs) {\n        if (propertySize <= 0) {\n            return;\n        }\n\n        for (Message msg : msgs) {\n            if (msg.getProperties() != null) {\n                msg.getProperties().clear();\n            }\n\n            int startValue = (new Random(System.currentTimeMillis())).nextInt(100);\n            int size = 0;\n            for (int i = 0; ; i++) {\n                String prop1 = \"prop\" + i, prop1V = \"hello\" + startValue;\n                msg.putUserProperty(prop1, prop1V);\n                size += prop1.length() + prop1V.length();\n                if (size > propertySize) {\n                    break;\n                }\n                startValue++;\n            }\n        }\n    }\n\n    private static DefaultMQProducer initInstance(String namesrv, boolean traceEnable, RPCHook rpcHook) {\n        final DefaultMQProducer producer = new DefaultMQProducer(\"benchmark_batch_producer\", rpcHook, traceEnable, null);\n        producer.setInstanceName(Long.toString(System.currentTimeMillis()));\n\n        producer.setNamesrvAddr(namesrv);\n        return producer;\n    }\n}\n\nclass StatsBenchmarkBatchProducer {\n\n    private final LongAdder sendRequestSuccessCount = new LongAdder();\n\n    private final LongAdder sendRequestFailedCount = new LongAdder();\n\n    private final LongAdder sendMessageSuccessTimeTotal = new LongAdder();\n\n    private final AtomicLong sendMessageMaxRT = new AtomicLong(0L);\n\n    private final LongAdder sendMessageSuccessCount = new LongAdder();\n\n    private final LongAdder sendMessageFailedCount = new LongAdder();\n\n    private final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(\n        \"BenchmarkTimerThread\", Boolean.TRUE));\n\n    private final LinkedList<Long[]> snapshotList = new LinkedList<>();\n\n    private final int reportInterval;\n\n    public StatsBenchmarkBatchProducer(int reportInterval) {\n        this.reportInterval = reportInterval;\n    }\n\n    public Long[] createSnapshot() {\n        Long[] snap = new Long[] {\n            System.currentTimeMillis(),\n            this.sendRequestSuccessCount.longValue(),\n            this.sendRequestFailedCount.longValue(),\n            this.sendMessageSuccessCount.longValue(),\n            this.sendMessageFailedCount.longValue(),\n            this.sendMessageSuccessTimeTotal.longValue(),\n        };\n\n        return snap;\n    }\n\n    public LongAdder getSendRequestSuccessCount() {\n        return sendRequestSuccessCount;\n    }\n\n    public LongAdder getSendRequestFailedCount() {\n        return sendRequestFailedCount;\n    }\n\n    public LongAdder getSendMessageSuccessTimeTotal() {\n        return sendMessageSuccessTimeTotal;\n    }\n\n    public AtomicLong getSendMessageMaxRT() {\n        return sendMessageMaxRT;\n    }\n\n    public LongAdder getSendMessageSuccessCount() {\n        return sendMessageSuccessCount;\n    }\n\n    public LongAdder getSendMessageFailedCount() {\n        return sendMessageFailedCount;\n    }\n\n    public void start() {\n\n        executorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                snapshotList.addLast(createSnapshot());\n                if (snapshotList.size() > 10) {\n                    snapshotList.removeFirst();\n                }\n            }\n        }, 1000, 1000, TimeUnit.MILLISECONDS);\n\n        executorService.scheduleAtFixedRate(new Runnable() {\n            private void printStats() {\n                if (snapshotList.size() >= 10) {\n                    Long[] begin = snapshotList.getFirst();\n                    Long[] end = snapshotList.getLast();\n\n                    final long sendTps = (long) (((end[1] - begin[1]) / (double) (end[0] - begin[0])) * 1000L);\n                    final long sendMps = (long) (((end[3] - begin[3]) / (double) (end[0] - begin[0])) * 1000L);\n                    final double averageRT = (end[5] - begin[5]) / (double) (end[1] - begin[1]);\n                    final double averageMsgRT = (end[5] - begin[5]) / (double) (end[3] - begin[3]);\n\n                    System.out.printf(\"Current Time: %s | Send TPS: %d | Send MPS: %d | Max RT(ms): %d | Average RT(ms): %7.3f | Average Message RT(ms): %7.3f | Send Failed: %d | Send Message Failed: %d%n\",\n                        UtilAll.timeMillisToHumanString2(System.currentTimeMillis()), sendTps, sendMps, getSendMessageMaxRT().longValue(), averageRT, averageMsgRT, end[2], end[4]);\n                }\n            }\n\n            @Override\n            public void run() {\n                try {\n                    this.printStats();\n                } catch (Exception e) {\n                    e.printStackTrace();\n                }\n            }\n        }, reportInterval, reportInterval, TimeUnit.MILLISECONDS);\n    }\n\n    public void shutdown() {\n        executorService.shutdown();\n    }\n}"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.example.benchmark;\n\nimport java.io.IOException;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.TimerTask;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ScheduledThreadPoolExecutor;\nimport java.util.concurrent.ThreadLocalRandom;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.concurrent.atomic.LongAdder;\nimport org.apache.commons.cli.CommandLine;\nimport org.apache.commons.cli.DefaultParser;\nimport org.apache.commons.cli.Option;\nimport org.apache.commons.cli.Options;\nimport org.apache.commons.lang3.concurrent.BasicThreadFactory;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.MessageSelector;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.SerializeType;\nimport org.apache.rocketmq.srvutil.ServerUtil;\n\npublic class Consumer {\n\n    public static void main(String[] args) throws MQClientException, IOException {\n        System.setProperty(RemotingCommand.SERIALIZE_TYPE_PROPERTY, SerializeType.ROCKETMQ.name());\n        Options options = ServerUtil.buildCommandlineOptions(new Options());\n        CommandLine commandLine = ServerUtil.parseCmdLine(\"benchmarkConsumer\", args, buildCommandlineOptions(options), new DefaultParser());\n        if (null == commandLine) {\n            System.exit(-1);\n        }\n\n        final String topic = commandLine.hasOption('t') ? commandLine.getOptionValue('t').trim() : \"BenchmarkTest\";\n        final int threadCount = commandLine.hasOption('w') ? Integer.parseInt(commandLine.getOptionValue('w')) : 20;\n        final String groupPrefix = commandLine.hasOption('g') ? commandLine.getOptionValue('g').trim() : \"benchmark_consumer\";\n        final String isSuffixEnable = commandLine.hasOption('p') ? commandLine.getOptionValue('p').trim() : \"false\";\n        final String filterType = commandLine.hasOption('f') ? commandLine.getOptionValue('f').trim() : null;\n        final String expression = commandLine.hasOption('e') ? commandLine.getOptionValue('e').trim() : null;\n        final double failRate = commandLine.hasOption('r') ? Double.parseDouble(commandLine.getOptionValue('r').trim()) : 0.0;\n        final boolean msgTraceEnable = commandLine.hasOption('m') && Boolean.parseBoolean(commandLine.getOptionValue('m'));\n        final boolean aclEnable = commandLine.hasOption('a') && Boolean.parseBoolean(commandLine.getOptionValue('a'));\n        final boolean clientRebalanceEnable = commandLine.hasOption('c') ? Boolean.parseBoolean(commandLine.getOptionValue('c')) : true;\n        final int reportInterval = commandLine.hasOption(\"ri\") ? Integer.parseInt(commandLine.getOptionValue(\"ri\")) : 10000;\n\n        String group = groupPrefix;\n        if (Boolean.parseBoolean(isSuffixEnable)) {\n            group = groupPrefix + \"_\" + (System.currentTimeMillis() % 100);\n        }\n\n        System.out.printf(\"topic: %s, threadCount %d, group: %s, suffix: %s, filterType: %s, expression: %s, msgTraceEnable: %s, aclEnable: %s, reportInterval: %d%n\",\n            topic, threadCount, group, isSuffixEnable, filterType, expression, msgTraceEnable, aclEnable, reportInterval);\n\n        final StatsBenchmarkConsumer statsBenchmarkConsumer = new StatsBenchmarkConsumer();\n\n        ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(1,\n            new BasicThreadFactory.Builder().namingPattern(\"BenchmarkTimerThread-%d\").daemon(true).build());\n\n        final LinkedList<Long[]> snapshotList = new LinkedList<>();\n\n        executorService.scheduleAtFixedRate(new TimerTask() {\n            @Override\n            public void run() {\n                snapshotList.addLast(statsBenchmarkConsumer.createSnapshot());\n                if (snapshotList.size() > 10) {\n                    snapshotList.removeFirst();\n                }\n            }\n        }, 1000, 1000, TimeUnit.MILLISECONDS);\n\n        executorService.scheduleAtFixedRate(new TimerTask() {\n            private void printStats() {\n                if (snapshotList.size() >= 10) {\n                    Long[] begin = snapshotList.getFirst();\n                    Long[] end = snapshotList.getLast();\n\n                    final long consumeTps =\n                        (long) (((end[1] - begin[1]) / (double) (end[0] - begin[0])) * 1000L);\n                    final double averageB2CRT = (end[2] - begin[2]) / (double) (end[1] - begin[1]);\n                    final double averageS2CRT = (end[3] - begin[3]) / (double) (end[1] - begin[1]);\n                    final long failCount = end[4] - begin[4];\n                    final long b2cMax = statsBenchmarkConsumer.getBorn2ConsumerMaxRT().get();\n                    final long s2cMax = statsBenchmarkConsumer.getStore2ConsumerMaxRT().get();\n\n                    statsBenchmarkConsumer.getBorn2ConsumerMaxRT().set(0);\n                    statsBenchmarkConsumer.getStore2ConsumerMaxRT().set(0);\n\n                    System.out.printf(\"Current Time: %s | Consume TPS: %d | AVG(B2C) RT(ms): %7.3f | AVG(S2C) RT(ms): %7.3f | MAX(B2C) RT(ms): %d | MAX(S2C) RT(ms): %d | Consume Fail: %d%n\",\n                        UtilAll.timeMillisToHumanString2(System.currentTimeMillis()), consumeTps, averageB2CRT, averageS2CRT, b2cMax, s2cMax, failCount);\n                }\n            }\n\n            @Override\n            public void run() {\n                try {\n                    this.printStats();\n                } catch (Exception e) {\n                    e.printStackTrace();\n                }\n            }\n        }, reportInterval, reportInterval, TimeUnit.MILLISECONDS);\n\n        RPCHook rpcHook = null;\n        if (aclEnable) {\n            String ak = commandLine.hasOption(\"ak\") ? String.valueOf(commandLine.getOptionValue(\"ak\")) : AclClient.ACL_ACCESS_KEY;\n            String sk = commandLine.hasOption(\"sk\") ? String.valueOf(commandLine.getOptionValue(\"sk\")) : AclClient.ACL_SECRET_KEY;\n            rpcHook = AclClient.getAclRPCHook(ak, sk);\n        }\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(group, rpcHook, new AllocateMessageQueueAveragely(), msgTraceEnable, null);\n        if (commandLine.hasOption('n')) {\n            String ns = commandLine.getOptionValue('n');\n            consumer.setNamesrvAddr(ns);\n        }\n        consumer.setConsumeThreadMin(threadCount);\n        consumer.setConsumeThreadMax(threadCount);\n        consumer.setInstanceName(Long.toString(System.currentTimeMillis()));\n        consumer.setClientRebalance(clientRebalanceEnable);\n\n        if (filterType == null || expression == null) {\n            consumer.subscribe(topic, \"*\");\n        } else {\n            if (ExpressionType.TAG.equals(filterType)) {\n                String expr = MixAll.file2String(expression);\n                System.out.printf(\"Expression: %s%n\", expr);\n                consumer.subscribe(topic, MessageSelector.byTag(expr));\n            } else if (ExpressionType.SQL92.equals(filterType)) {\n                String expr = MixAll.file2String(expression);\n                System.out.printf(\"Expression: %s%n\", expr);\n                consumer.subscribe(topic, MessageSelector.bySql(expr));\n            } else {\n                throw new IllegalArgumentException(\"Not support filter type! \" + filterType);\n            }\n        }\n\n        consumer.registerMessageListener(new MessageListenerConcurrently() {\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,\n                ConsumeConcurrentlyContext context) {\n                MessageExt msg = msgs.get(0);\n                long now = System.currentTimeMillis();\n\n                statsBenchmarkConsumer.getReceiveMessageTotalCount().increment();\n\n                long born2ConsumerRT = now - msg.getBornTimestamp();\n                statsBenchmarkConsumer.getBorn2ConsumerTotalRT().add(born2ConsumerRT);\n\n                long store2ConsumerRT = now - msg.getStoreTimestamp();\n                statsBenchmarkConsumer.getStore2ConsumerTotalRT().add(store2ConsumerRT);\n\n                compareAndSetMax(statsBenchmarkConsumer.getBorn2ConsumerMaxRT(), born2ConsumerRT);\n\n                compareAndSetMax(statsBenchmarkConsumer.getStore2ConsumerMaxRT(), store2ConsumerRT);\n\n                if (ThreadLocalRandom.current().nextDouble() < failRate) {\n                    statsBenchmarkConsumer.getFailCount().increment();\n                    return ConsumeConcurrentlyStatus.RECONSUME_LATER;\n                } else {\n                    return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n                }\n            }\n        });\n\n        consumer.start();\n\n        System.out.printf(\"Consumer Started.%n\");\n    }\n\n    public static Options buildCommandlineOptions(final Options options) {\n        Option opt = new Option(\"t\", \"topic\", true, \"Topic name, Default: BenchmarkTest\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"w\", \"threadCount\", true, \"Thread count, Default: 20\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"g\", \"group\", true, \"Consumer group name, Default: benchmark_consumer\");\n        opt.setRequired(false);\n        options.addOption(opt);\n        opt = new Option(\"p\", \"group prefix enable\", true, \"Is group prefix enable, Default: false\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"f\", \"filterType\", true, \"TAG, SQL92\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"e\", \"expression\", true, \"filter expression content file path.ie: ./test/expr\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"r\", \"fail rate\", true, \"consumer fail rate, default 0\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"m\", \"msgTraceEnable\", true, \"Message Trace Enable, Default: false\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"a\", \"aclEnable\", true, \"Acl Enable, Default: false\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"ak\", \"accessKey\", true, \"Acl access key, Default: 12345678\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"sk\", \"secretKey\", true, \"Acl secret key, Default: rocketmq2\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"ri\", \"reportInterval\", true, \"The number of ms between reports, Default: 10000\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        return options;\n    }\n\n    public static void compareAndSetMax(final AtomicLong target, final long value) {\n        long prev = target.get();\n        while (value > prev) {\n            boolean updated = target.compareAndSet(prev, value);\n            if (updated)\n                break;\n\n            prev = target.get();\n        }\n    }\n}\n\nclass StatsBenchmarkConsumer {\n    private final LongAdder receiveMessageTotalCount = new LongAdder();\n\n    private final LongAdder born2ConsumerTotalRT = new LongAdder();\n\n    private final LongAdder store2ConsumerTotalRT = new LongAdder();\n\n    private final AtomicLong born2ConsumerMaxRT = new AtomicLong(0L);\n\n    private final AtomicLong store2ConsumerMaxRT = new AtomicLong(0L);\n\n    private final LongAdder failCount = new LongAdder();\n\n    public Long[] createSnapshot() {\n        Long[] snap = new Long[] {\n            System.currentTimeMillis(),\n            this.receiveMessageTotalCount.longValue(),\n            this.born2ConsumerTotalRT.longValue(),\n            this.store2ConsumerTotalRT.longValue(),\n            this.failCount.longValue()\n        };\n\n        return snap;\n    }\n\n    public LongAdder getReceiveMessageTotalCount() {\n        return receiveMessageTotalCount;\n    }\n\n    public LongAdder getBorn2ConsumerTotalRT() {\n        return born2ConsumerTotalRT;\n    }\n\n    public LongAdder getStore2ConsumerTotalRT() {\n        return store2ConsumerTotalRT;\n    }\n\n    public AtomicLong getBorn2ConsumerMaxRT() {\n        return born2ConsumerMaxRT;\n    }\n\n    public AtomicLong getStore2ConsumerMaxRT() {\n        return store2ConsumerMaxRT;\n    }\n\n    public LongAdder getFailCount() {\n        return failCount;\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.benchmark;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.atomic.LongAdder;\nimport org.apache.commons.cli.CommandLine;\nimport org.apache.commons.cli.DefaultParser;\nimport org.apache.commons.cli.Option;\nimport org.apache.commons.cli.Options;\nimport org.apache.commons.lang3.RandomStringUtils;\nimport org.apache.commons.lang3.concurrent.BasicThreadFactory;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendCallback;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.compression.CompressionType;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.SerializeType;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.srvutil.ServerUtil;\n\nimport java.util.Arrays;\nimport java.util.LinkedList;\nimport java.util.Random;\nimport java.util.TimerTask;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ScheduledThreadPoolExecutor;\nimport java.util.concurrent.atomic.AtomicLong;\n\npublic class Producer {\n\n    private static final Logger log = LoggerFactory.getLogger(Producer.class);\n\n    private static byte[] msgBody;\n    private static final int MAX_LENGTH_ASYNC_QUEUE = 10000;\n    private static final int SLEEP_FOR_A_WHILE = 100;\n\n    public static void main(String[] args) throws MQClientException {\n        System.setProperty(RemotingCommand.SERIALIZE_TYPE_PROPERTY, SerializeType.ROCKETMQ.name());\n\n        Options options = ServerUtil.buildCommandlineOptions(new Options());\n        CommandLine commandLine = ServerUtil.parseCmdLine(\"benchmarkProducer\", args, buildCommandlineOptions(options), new DefaultParser());\n        if (null == commandLine) {\n            System.exit(-1);\n        }\n\n        final String topic = commandLine.hasOption('t') ? commandLine.getOptionValue('t').trim() : \"BenchmarkTest\";\n        final int messageSize = commandLine.hasOption('s') ? Integer.parseInt(commandLine.getOptionValue('s')) : 128;\n        final boolean keyEnable = commandLine.hasOption('k') && Boolean.parseBoolean(commandLine.getOptionValue('k'));\n        final int propertySize = commandLine.hasOption('p') ? Integer.parseInt(commandLine.getOptionValue('p')) : 0;\n        final int tagCount = commandLine.hasOption('l') ? Integer.parseInt(commandLine.getOptionValue('l')) : 0;\n        final boolean msgTraceEnable = commandLine.hasOption('m') && Boolean.parseBoolean(commandLine.getOptionValue('m'));\n        final boolean aclEnable = commandLine.hasOption('a') && Boolean.parseBoolean(commandLine.getOptionValue('a'));\n        final long messageNum = commandLine.hasOption('q') ? Long.parseLong(commandLine.getOptionValue('q')) : 0;\n        final boolean delayEnable = commandLine.hasOption('d') && Boolean.parseBoolean(commandLine.getOptionValue('d'));\n        final int delayLevel = commandLine.hasOption('e') ? Integer.parseInt(commandLine.getOptionValue('e')) : 1;\n        final boolean asyncEnable = commandLine.hasOption('y') && Boolean.parseBoolean(commandLine.getOptionValue('y'));\n        final int threadCount = asyncEnable ? 1 : commandLine.hasOption('w') ? Integer.parseInt(commandLine.getOptionValue('w')) : 64;\n        final boolean enableCompress = commandLine.hasOption('c') && Boolean.parseBoolean(commandLine.getOptionValue('c'));\n        final int reportInterval = commandLine.hasOption(\"ri\") ? Integer.parseInt(commandLine.getOptionValue(\"ri\")) : 10000;\n\n        System.out.printf(\"topic: %s, threadCount: %d, messageSize: %d, keyEnable: %s, propertySize: %d, tagCount: %d, \" +\n                \"traceEnable: %s, aclEnable: %s, messageQuantity: %d, delayEnable: %s, delayLevel: %s, \" +\n                \"asyncEnable: %s%n compressEnable: %s, reportInterval: %d%n\",\n            topic, threadCount, messageSize, keyEnable, propertySize, tagCount, msgTraceEnable, aclEnable, messageNum,\n            delayEnable, delayLevel, asyncEnable, enableCompress, reportInterval);\n\n        StringBuilder sb = new StringBuilder(messageSize);\n        for (int i = 0; i < messageSize; i++) {\n            sb.append(RandomStringUtils.randomAlphanumeric(1));\n        }\n        msgBody = sb.toString().getBytes(StandardCharsets.UTF_8);\n\n        final ExecutorService sendThreadPool = Executors.newFixedThreadPool(threadCount);\n\n        final StatsBenchmarkProducer statsBenchmark = new StatsBenchmarkProducer();\n\n        ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(1,\n                new BasicThreadFactory.Builder().namingPattern(\"BenchmarkTimerThread-%d\").daemon(true).build());\n\n        final LinkedList<Long[]> snapshotList = new LinkedList<>();\n\n        final long[] msgNums = new long[threadCount];\n\n        if (messageNum > 0) {\n            Arrays.fill(msgNums, messageNum / threadCount);\n            long mod = messageNum % threadCount;\n            if (mod > 0) {\n                msgNums[0] += mod;\n            }\n        }\n\n        executorService.scheduleAtFixedRate(new TimerTask() {\n            @Override\n            public void run() {\n                snapshotList.addLast(statsBenchmark.createSnapshot());\n                if (snapshotList.size() > 10) {\n                    snapshotList.removeFirst();\n                }\n            }\n        }, 1000, 1000, TimeUnit.MILLISECONDS);\n\n        executorService.scheduleAtFixedRate(new TimerTask() {\n            private void printStats() {\n                if (snapshotList.size() >= 10) {\n                    doPrintStats(snapshotList,  statsBenchmark, false);\n                }\n            }\n\n            @Override\n            public void run() {\n                try {\n                    this.printStats();\n                } catch (Exception e) {\n                    e.printStackTrace();\n                }\n            }\n        }, reportInterval, reportInterval, TimeUnit.MILLISECONDS);\n\n        RPCHook rpcHook = null;\n        if (aclEnable) {\n            String ak = commandLine.hasOption(\"ak\") ? String.valueOf(commandLine.getOptionValue(\"ak\")) : AclClient.ACL_ACCESS_KEY;\n            String sk = commandLine.hasOption(\"sk\") ? String.valueOf(commandLine.getOptionValue(\"sk\")) : AclClient.ACL_SECRET_KEY;\n            rpcHook = AclClient.getAclRPCHook(ak, sk);\n        }\n        final DefaultMQProducer producer = new DefaultMQProducer(\"benchmark_producer\", rpcHook, msgTraceEnable, null);\n        producer.setInstanceName(Long.toString(System.currentTimeMillis()));\n\n        if (commandLine.hasOption('n')) {\n            String ns = commandLine.getOptionValue('n');\n            producer.setNamesrvAddr(ns);\n        }\n\n        if (enableCompress) {\n            String compressType = commandLine.hasOption(\"ct\") ? commandLine.getOptionValue(\"ct\").trim() : \"ZLIB\";\n            int compressLevel = commandLine.hasOption(\"cl\") ? Integer.parseInt(commandLine.getOptionValue(\"cl\")) : 5;\n            int compressOverHowMuch = commandLine.hasOption(\"ch\") ? Integer.parseInt(commandLine.getOptionValue(\"ch\")) : 4096;\n            producer.setCompressType(CompressionType.of(compressType));\n            producer.setCompressLevel(compressLevel);\n            producer.setCompressMsgBodyOverHowmuch(compressOverHowMuch);\n            System.out.printf(\"compressType: %s compressLevel: %s%n\", compressType, compressLevel);\n        } else {\n            producer.setCompressMsgBodyOverHowmuch(Integer.MAX_VALUE);\n        }\n\n        producer.start();\n\n        for (int i = 0; i < threadCount; i++) {\n            final long msgNumLimit = msgNums[i];\n            if (messageNum > 0 && msgNumLimit == 0) {\n                break;\n            }\n            sendThreadPool.execute(new Runnable() {\n                @Override\n                public void run() {\n                    int num = 0;\n                    while (true) {\n                        try {\n                            final Message msg = buildMessage(topic);\n                            final long beginTimestamp = System.currentTimeMillis();\n                            if (keyEnable) {\n                                msg.setKeys(String.valueOf(beginTimestamp / 1000));\n                            }\n                            if (delayEnable) {\n                                msg.setDelayTimeLevel(delayLevel);\n                            }\n                            if (tagCount > 0) {\n                                msg.setTags(String.format(\"tag%d\", System.currentTimeMillis() % tagCount));\n                            }\n                            if (propertySize > 0) {\n                                if (msg.getProperties() != null) {\n                                    msg.getProperties().clear();\n                                }\n                                int i = 0;\n                                int startValue = (new Random(System.currentTimeMillis())).nextInt(100);\n                                int size = 0;\n                                while (true) {\n                                    String prop1 = \"prop\" + i, prop1V = \"hello\" + startValue;\n                                    String prop2 = \"prop\" + (i + 1), prop2V = String.valueOf(startValue);\n                                    msg.putUserProperty(prop1, prop1V);\n                                    msg.putUserProperty(prop2, prop2V);\n                                    size += prop1.length() + prop2.length() + prop1V.length() + prop2V.length();\n                                    if (size > propertySize) {\n                                        break;\n                                    }\n                                    i += 2;\n                                    startValue += 2;\n                                }\n                            }\n                            if (asyncEnable) {\n                                ThreadPoolExecutor e = (ThreadPoolExecutor) producer.getDefaultMQProducerImpl().getAsyncSenderExecutor();\n                                // Flow control\n                                while (e.getQueue().size() > MAX_LENGTH_ASYNC_QUEUE) {\n                                    Thread.sleep(SLEEP_FOR_A_WHILE);\n                                }\n                                producer.send(msg, new SendCallback() {\n                                    @Override\n                                    public void onSuccess(SendResult sendResult) {\n                                        updateStatsSuccess(statsBenchmark, beginTimestamp);\n                                    }\n\n                                    @Override\n                                    public void onException(Throwable e) {\n                                        statsBenchmark.getSendRequestFailedCount().increment();\n                                    }\n                                });\n                            } else {\n                                producer.send(msg);\n                                updateStatsSuccess(statsBenchmark, beginTimestamp);\n                            }\n                        } catch (RemotingException e) {\n                            statsBenchmark.getSendRequestFailedCount().increment();\n                            log.error(\"[BENCHMARK_PRODUCER] Send Exception\", e);\n\n                            try {\n                                Thread.sleep(3000);\n                            } catch (InterruptedException ignored) {\n                            }\n                        } catch (InterruptedException e) {\n                            statsBenchmark.getSendRequestFailedCount().increment();\n                            try {\n                                Thread.sleep(3000);\n                            } catch (InterruptedException e1) {\n                            }\n                        } catch (MQClientException e) {\n                            statsBenchmark.getSendRequestFailedCount().increment();\n                            log.error(\"[BENCHMARK_PRODUCER] Send Exception\", e);\n                        } catch (MQBrokerException e) {\n                            statsBenchmark.getReceiveResponseFailedCount().increment();\n                            log.error(\"[BENCHMARK_PRODUCER] Send Exception\", e);\n                            try {\n                                Thread.sleep(3000);\n                            } catch (InterruptedException ignored) {\n                            }\n                        }\n                        if (messageNum > 0 && ++num >= msgNumLimit) {\n                            break;\n                        }\n                    }\n                }\n            });\n        }\n        try {\n            sendThreadPool.shutdown();\n            sendThreadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);\n            executorService.shutdown();\n            try {\n                executorService.awaitTermination(5000, TimeUnit.MILLISECONDS);\n            } catch (InterruptedException e) {\n            }\n\n            if (snapshotList.size() > 1) {\n                doPrintStats(snapshotList, statsBenchmark, true);\n            } else {\n                System.out.printf(\"[Complete] Send Total: %d Send Failed: %d Response Failed: %d%n\",\n                    statsBenchmark.getSendRequestSuccessCount().longValue() + statsBenchmark.getSendRequestFailedCount().longValue(),\n                    statsBenchmark.getSendRequestFailedCount().longValue(), statsBenchmark.getReceiveResponseFailedCount().longValue());\n            }\n            producer.shutdown();\n        } catch (InterruptedException e) {\n            log.error(\"[Exit] Thread Interrupted Exception\", e);\n        }\n    }\n\n    private static void updateStatsSuccess(StatsBenchmarkProducer statsBenchmark, long beginTimestamp) {\n        statsBenchmark.getSendRequestSuccessCount().increment();\n        statsBenchmark.getReceiveResponseSuccessCount().increment();\n        final long currentRT = System.currentTimeMillis() - beginTimestamp;\n        statsBenchmark.getSendMessageSuccessTimeTotal().add(currentRT);\n        long prevMaxRT = statsBenchmark.getSendMessageMaxRT().longValue();\n        while (currentRT > prevMaxRT) {\n            boolean updated = statsBenchmark.getSendMessageMaxRT().compareAndSet(prevMaxRT, currentRT);\n            if (updated)\n                break;\n\n            prevMaxRT = statsBenchmark.getSendMessageMaxRT().longValue();\n        }\n    }\n\n    public static Options buildCommandlineOptions(final Options options) {\n        Option opt = new Option(\"w\", \"threadCount\", true, \"Thread count, Default: 64\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"s\", \"messageSize\", true, \"Message Size, Default: 128\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"k\", \"keyEnable\", true, \"Message Key Enable, Default: false\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"t\", \"topic\", true, \"Topic name, Default: BenchmarkTest\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"l\", \"tagCount\", true, \"Tag count, Default: 0\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"m\", \"msgTraceEnable\", true, \"Message Trace Enable, Default: false\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"a\", \"aclEnable\", true, \"Acl Enable, Default: false\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"ak\", \"accessKey\", true, \"Acl access key, Default: 12345678\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"sk\", \"secretKey\", true, \"Acl secret key, Default: rocketmq2\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"q\", \"messageQuantity\", true, \"Send message quantity, Default: 0, running forever\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"d\", \"delayEnable\", true, \"Delay message Enable, Default: false\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"e\", \"delayLevel\", true, \"Delay message level, Default: 1\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"y\", \"asyncEnable\", true, \"Enable async produce, Default: false\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"c\", \"compressEnable\", true, \"Enable compress msg over 4K, Default: false\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"ct\", \"compressType\", true, \"Message compressed type, Default: ZLIB\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"cl\", \"compressLevel\", true, \"Message compressed level, Default: 5\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"ch\", \"compressOverHowMuch\", true, \"Compress message when body over how much(unit Byte), Default: 4096\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"ri\", \"reportInterval\", true, \"The number of ms between reports, Default: 10000\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        return options;\n    }\n\n    private static Message buildMessage(final String topic) {\n        return new Message(topic, msgBody);\n    }\n\n    private static void doPrintStats(final LinkedList<Long[]> snapshotList, final StatsBenchmarkProducer statsBenchmark, boolean done) {\n        Long[] begin = snapshotList.getFirst();\n        Long[] end = snapshotList.getLast();\n\n        final long sendTps = (long) (((end[3] - begin[3]) / (double) (end[0] - begin[0])) * 1000L);\n        final double averageRT = (end[5] - begin[5]) / (double) (end[3] - begin[3]);\n\n        if (done) {\n            System.out.printf(\"[Complete] Send Total: %d | Send TPS: %d | Max RT(ms): %d | Average RT(ms): %7.3f | Send Failed: %d | Response Failed: %d%n\",\n                statsBenchmark.getSendRequestSuccessCount().longValue() + statsBenchmark.getSendRequestFailedCount().longValue(),\n                sendTps, statsBenchmark.getSendMessageMaxRT().longValue(), averageRT, end[2], end[4]);\n        } else {\n            System.out.printf(\"Current Time: %s | Send TPS: %d | Max RT(ms): %d | Average RT(ms): %7.3f | Send Failed: %d | Response Failed: %d%n\",\n                UtilAll.timeMillisToHumanString2(System.currentTimeMillis()), sendTps, statsBenchmark.getSendMessageMaxRT().longValue(), averageRT, end[2], end[4]);\n        }\n    }\n}\n\nclass StatsBenchmarkProducer {\n    private final LongAdder sendRequestSuccessCount = new LongAdder();\n\n    private final LongAdder sendRequestFailedCount = new LongAdder();\n\n    private final LongAdder receiveResponseSuccessCount = new LongAdder();\n\n    private final LongAdder receiveResponseFailedCount = new LongAdder();\n\n    private final LongAdder sendMessageSuccessTimeTotal = new LongAdder();\n\n    private final AtomicLong sendMessageMaxRT = new AtomicLong(0L);\n\n    public Long[] createSnapshot() {\n        Long[] snap = new Long[] {\n            System.currentTimeMillis(),\n            this.sendRequestSuccessCount.longValue(),\n            this.sendRequestFailedCount.longValue(),\n            this.receiveResponseSuccessCount.longValue(),\n            this.receiveResponseFailedCount.longValue(),\n            this.sendMessageSuccessTimeTotal.longValue(),\n        };\n\n        return snap;\n    }\n\n    public LongAdder getSendRequestSuccessCount() {\n        return sendRequestSuccessCount;\n    }\n\n    public LongAdder getSendRequestFailedCount() {\n        return sendRequestFailedCount;\n    }\n\n    public LongAdder getReceiveResponseSuccessCount() {\n        return receiveResponseSuccessCount;\n    }\n\n    public LongAdder getReceiveResponseFailedCount() {\n        return receiveResponseFailedCount;\n    }\n\n    public LongAdder getSendMessageSuccessTimeTotal() {\n        return sendMessageSuccessTimeTotal;\n    }\n\n    public AtomicLong getSendMessageMaxRT() {\n        return sendMessageMaxRT;\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.example.benchmark;\n\nimport org.apache.commons.cli.CommandLine;\nimport org.apache.commons.cli.DefaultParser;\nimport org.apache.commons.cli.Option;\nimport org.apache.commons.cli.Options;\nimport org.apache.commons.lang3.concurrent.BasicThreadFactory;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.LocalTransactionState;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.client.producer.TransactionListener;\nimport org.apache.rocketmq.client.producer.TransactionMQProducer;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.SerializeType;\nimport org.apache.rocketmq.srvutil.ServerUtil;\n\nimport java.io.UnsupportedEncodingException;\nimport java.nio.ByteBuffer;\nimport java.text.SimpleDateFormat;\nimport java.util.ArrayList;\nimport java.util.Date;\nimport java.util.LinkedHashMap;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.TimerTask;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.ThreadLocalRandom;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ScheduledThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.LongAdder;\n\npublic class TransactionProducer {\n    private static final long START_TIME = System.currentTimeMillis();\n    private static final LongAdder MSG_COUNT = new LongAdder();\n\n    //broker max check times should less than this value\n    static final int MAX_CHECK_RESULT_IN_MSG = 20;\n\n    public static void main(String[] args) throws MQClientException, UnsupportedEncodingException {\n        System.setProperty(RemotingCommand.SERIALIZE_TYPE_PROPERTY, SerializeType.ROCKETMQ.name());\n        Options options = ServerUtil.buildCommandlineOptions(new Options());\n        CommandLine commandLine = ServerUtil.parseCmdLine(\"TransactionProducer\", args, buildCommandlineOptions(options), new DefaultParser());\n        TxSendConfig config = new TxSendConfig();\n        config.topic = commandLine.hasOption('t') ? commandLine.getOptionValue('t').trim() : \"BenchmarkTest\";\n        config.threadCount = commandLine.hasOption('w') ? Integer.parseInt(commandLine.getOptionValue('w')) : 32;\n        config.messageSize = commandLine.hasOption('s') ? Integer.parseInt(commandLine.getOptionValue('s')) : 2048;\n        config.sendRollbackRate = commandLine.hasOption(\"sr\") ? Double.parseDouble(commandLine.getOptionValue(\"sr\")) : 0.0;\n        config.sendUnknownRate = commandLine.hasOption(\"su\") ? Double.parseDouble(commandLine.getOptionValue(\"su\")) : 0.0;\n        config.checkRollbackRate = commandLine.hasOption(\"cr\") ? Double.parseDouble(commandLine.getOptionValue(\"cr\")) : 0.0;\n        config.checkUnknownRate = commandLine.hasOption(\"cu\") ? Double.parseDouble(commandLine.getOptionValue(\"cu\")) : 0.0;\n        config.batchId = commandLine.hasOption(\"b\") ? Long.parseLong(commandLine.getOptionValue(\"b\")) : System.currentTimeMillis();\n        config.sendInterval = commandLine.hasOption(\"i\") ? Integer.parseInt(commandLine.getOptionValue(\"i\")) : 0;\n        config.aclEnable = commandLine.hasOption('a') && Boolean.parseBoolean(commandLine.getOptionValue('a'));\n        config.msgTraceEnable = commandLine.hasOption('m') && Boolean.parseBoolean(commandLine.getOptionValue('m'));\n        config.reportInterval = commandLine.hasOption(\"ri\") ? Integer.parseInt(commandLine.getOptionValue(\"ri\")) : 10000;\n\n        final ExecutorService sendThreadPool = Executors.newFixedThreadPool(config.threadCount);\n\n        final StatsBenchmarkTProducer statsBenchmark = new StatsBenchmarkTProducer();\n\n        ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(1,\n            new BasicThreadFactory.Builder().namingPattern(\"BenchmarkTimerThread-%d\").daemon(true).build());\n\n        final LinkedList<Snapshot> snapshotList = new LinkedList<>();\n\n        executorService.scheduleAtFixedRate(new TimerTask() {\n            @Override\n            public void run() {\n                snapshotList.addLast(statsBenchmark.createSnapshot());\n                while (snapshotList.size() > 10) {\n                    snapshotList.removeFirst();\n                }\n            }\n        }, 1000, 1000, TimeUnit.MILLISECONDS);\n\n        executorService.scheduleAtFixedRate(new TimerTask() {\n            private void printStats() {\n                if (snapshotList.size() >= 10) {\n                    Snapshot begin = snapshotList.getFirst();\n                    Snapshot end = snapshotList.getLast();\n\n                    final long sendCount = end.sendRequestSuccessCount - begin.sendRequestSuccessCount;\n                    final long sendTps = (sendCount * 1000L) / (end.endTime - begin.endTime);\n                    final double averageRT = (end.sendMessageTimeTotal - begin.sendMessageTimeTotal) / (double) (end.sendRequestSuccessCount - begin.sendRequestSuccessCount);\n\n                    final long failCount = end.sendRequestFailedCount - begin.sendRequestFailedCount;\n                    final long checkCount = end.checkCount - begin.checkCount;\n                    final long unexpectedCheck = end.unexpectedCheckCount - begin.unexpectedCheckCount;\n                    final long dupCheck = end.duplicatedCheck - begin.duplicatedCheck;\n\n                    System.out.printf(\n                        \"Current Time: %s | Send TPS: %5d | Max RT(ms): %5d | AVG RT(ms): %3.1f | Send Failed: %d | Check: %d | UnexpectedCheck: %d | DuplicatedCheck: %d%n\",\n                        UtilAll.timeMillisToHumanString2(System.currentTimeMillis()), sendTps, statsBenchmark.getSendMessageMaxRT().get(), averageRT, failCount, checkCount,\n                        unexpectedCheck, dupCheck);\n                    statsBenchmark.getSendMessageMaxRT().set(0);\n                }\n            }\n\n            @Override\n            public void run() {\n                try {\n                    this.printStats();\n                } catch (Exception e) {\n                    e.printStackTrace();\n                }\n            }\n        }, config.reportInterval, config.reportInterval, TimeUnit.MILLISECONDS);\n\n        RPCHook rpcHook = null;\n        if (config.aclEnable) {\n            String ak = commandLine.hasOption(\"ak\") ? String.valueOf(commandLine.getOptionValue(\"ak\")) : AclClient.ACL_ACCESS_KEY;\n            String sk = commandLine.hasOption(\"sk\") ? String.valueOf(commandLine.getOptionValue(\"sk\")) : AclClient.ACL_SECRET_KEY;\n            rpcHook = AclClient.getAclRPCHook(ak, sk);\n        }\n        final TransactionListener transactionCheckListener = new TransactionListenerImpl(statsBenchmark, config);\n        final TransactionMQProducer producer = new TransactionMQProducer(\n            \"benchmark_transaction_producer\",\n            rpcHook,\n            config.msgTraceEnable,\n            null);\n        producer.setInstanceName(Long.toString(System.currentTimeMillis()));\n        producer.setTransactionListener(transactionCheckListener);\n        producer.setDefaultTopicQueueNums(1000);\n        if (commandLine.hasOption('n')) {\n            String ns = commandLine.getOptionValue('n');\n            producer.setNamesrvAddr(ns);\n        }\n        producer.start();\n\n        for (int i = 0; i < config.threadCount; i++) {\n            sendThreadPool.execute(new Runnable() {\n                @Override\n                public void run() {\n                    while (true) {\n                        boolean success = false;\n                        final long beginTimestamp = System.currentTimeMillis();\n                        try {\n                            SendResult sendResult =\n                                producer.sendMessageInTransaction(buildMessage(config), null);\n                            success = sendResult != null && sendResult.getSendStatus() == SendStatus.SEND_OK;\n                        } catch (Throwable e) {\n                            success = false;\n                        } finally {\n                            final long currentRT = System.currentTimeMillis() - beginTimestamp;\n                            statsBenchmark.getSendMessageTimeTotal().add(currentRT);\n                            long prevMaxRT = statsBenchmark.getSendMessageMaxRT().get();\n                            while (currentRT > prevMaxRT) {\n                                boolean updated = statsBenchmark.getSendMessageMaxRT()\n                                    .compareAndSet(prevMaxRT, currentRT);\n                                if (updated)\n                                    break;\n\n                                prevMaxRT = statsBenchmark.getSendMessageMaxRT().get();\n                            }\n                            if (success) {\n                                statsBenchmark.getSendRequestSuccessCount().increment();\n                            } else {\n                                statsBenchmark.getSendRequestFailedCount().increment();\n                            }\n                            if (config.sendInterval > 0) {\n                                try {\n                                    Thread.sleep(config.sendInterval);\n                                } catch (InterruptedException e) {\n                                }\n                            }\n                        }\n                    }\n                }\n            });\n        }\n    }\n\n    private static Message buildMessage(TxSendConfig config) {\n        byte[] bs = new byte[config.messageSize];\n        ThreadLocalRandom r = ThreadLocalRandom.current();\n        r.nextBytes(bs);\n\n        ByteBuffer buf = ByteBuffer.wrap(bs);\n        buf.putLong(config.batchId);\n        long sendMachineId = START_TIME << 32;\n        long count = MSG_COUNT.longValue();\n        long msgId = sendMachineId | count;\n        MSG_COUNT.increment();\n        buf.putLong(msgId);\n\n        // save send tx result in message\n        if (r.nextDouble() < config.sendRollbackRate) {\n            buf.put((byte) LocalTransactionState.ROLLBACK_MESSAGE.ordinal());\n        } else if (r.nextDouble() < config.sendUnknownRate) {\n            buf.put((byte) LocalTransactionState.UNKNOW.ordinal());\n        } else {\n            buf.put((byte) LocalTransactionState.COMMIT_MESSAGE.ordinal());\n        }\n\n        // save check tx result in message\n        for (int i = 0; i < MAX_CHECK_RESULT_IN_MSG; i++) {\n            if (r.nextDouble() < config.checkRollbackRate) {\n                buf.put((byte) LocalTransactionState.ROLLBACK_MESSAGE.ordinal());\n            } else if (r.nextDouble() < config.checkUnknownRate) {\n                buf.put((byte) LocalTransactionState.UNKNOW.ordinal());\n            } else {\n                buf.put((byte) LocalTransactionState.COMMIT_MESSAGE.ordinal());\n            }\n        }\n\n        Message msg = new Message();\n        msg.setTopic(config.topic);\n\n        msg.setBody(bs);\n        return msg;\n    }\n\n    public static Options buildCommandlineOptions(final Options options) {\n        Option opt = new Option(\"w\", \"threadCount\", true, \"Thread count, Default: 32\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"s\", \"messageSize\", true, \"Message Size, Default: 2048\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"t\", \"topic\", true, \"Topic name, Default: BenchmarkTest\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"sr\", \"send rollback rate\", true, \"Send rollback rate, Default: 0.0\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"su\", \"send unknown rate\", true, \"Send unknown rate, Default: 0.0\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"cr\", \"check rollback rate\", true, \"Check rollback rate, Default: 0.0\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"cu\", \"check unknown rate\", true, \"Check unknown rate, Default: 0.0\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"b\", \"test batch id\", true, \"test batch id, Default: System.currentMillis()\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"i\", \"send interval\", true, \"sleep interval in millis between messages, Default: 0\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"a\", \"aclEnable\", true, \"Acl Enable, Default: false\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"ak\", \"accessKey\", true, \"Acl access key, Default: 12345678\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"sk\", \"secretKey\", true, \"Acl secret key, Default: rocketmq2\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"m\", \"msgTraceEnable\", true, \"Message Trace Enable, Default: false\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"ri\", \"reportInterval\", true, \"The number of ms between reports, Default: 10000\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        return options;\n    }\n}\n\nclass TransactionListenerImpl implements TransactionListener {\n    private StatsBenchmarkTProducer statBenchmark;\n    private TxSendConfig sendConfig;\n    private final LRUMap<Long, Integer> cache = new LRUMap<>(200000);\n\n    private class MsgMeta {\n        long batchId;\n        long msgId;\n        LocalTransactionState sendResult;\n        List<LocalTransactionState> checkResult;\n    }\n\n    public TransactionListenerImpl(StatsBenchmarkTProducer statsBenchmark, TxSendConfig sendConfig) {\n        this.statBenchmark = statsBenchmark;\n        this.sendConfig = sendConfig;\n    }\n\n    @Override\n    public LocalTransactionState executeLocalTransaction(Message msg, Object arg) {\n        return parseFromMsg(msg).sendResult;\n    }\n\n    private MsgMeta parseFromMsg(Message msg) {\n        byte[] bs = msg.getBody();\n        ByteBuffer buf = ByteBuffer.wrap(bs);\n        MsgMeta msgMeta = new MsgMeta();\n        msgMeta.batchId = buf.getLong();\n        msgMeta.msgId = buf.getLong();\n        msgMeta.sendResult = LocalTransactionState.values()[buf.get()];\n        msgMeta.checkResult = new ArrayList<>();\n        for (int i = 0; i < TransactionProducer.MAX_CHECK_RESULT_IN_MSG; i++) {\n            msgMeta.checkResult.add(LocalTransactionState.values()[buf.get()]);\n        }\n        return msgMeta;\n    }\n\n    @Override\n    public LocalTransactionState checkLocalTransaction(MessageExt msg) {\n        MsgMeta msgMeta = parseFromMsg(msg);\n        if (msgMeta.batchId != sendConfig.batchId) {\n            // message not generated in this test\n            return LocalTransactionState.ROLLBACK_MESSAGE;\n        }\n        statBenchmark.getCheckCount().increment();\n\n        int times = 0;\n        try {\n            String checkTimes = msg.getUserProperty(MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES);\n            times = Integer.parseInt(checkTimes);\n        } catch (Exception e) {\n            return LocalTransactionState.ROLLBACK_MESSAGE;\n        }\n        times = times <= 0 ? 1 : times;\n\n        boolean dup;\n        synchronized (cache) {\n            Integer oldCheckLog = cache.get(msgMeta.msgId);\n            Integer newCheckLog;\n            if (oldCheckLog == null) {\n                newCheckLog = 1 << (times - 1);\n            } else {\n                newCheckLog = oldCheckLog | (1 << (times - 1));\n            }\n            dup = newCheckLog.equals(oldCheckLog);\n        }\n        if (dup) {\n            statBenchmark.getDuplicatedCheckCount().increment();\n        }\n        if (msgMeta.sendResult != LocalTransactionState.UNKNOW) {\n            System.out.printf(\"%s unexpected check: msgId=%s,txId=%s,checkTimes=%s,sendResult=%s\\n\",\n                new SimpleDateFormat(\"HH:mm:ss,SSS\").format(new Date()),\n                msg.getMsgId(), msg.getTransactionId(),\n                msg.getUserProperty(MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES),\n                msgMeta.sendResult.toString());\n            statBenchmark.getUnexpectedCheckCount().increment();\n            return msgMeta.sendResult;\n        }\n\n        for (int i = 0; i < times - 1; i++) {\n            LocalTransactionState s = msgMeta.checkResult.get(i);\n            if (s != LocalTransactionState.UNKNOW) {\n                System.out.printf(\"%s unexpected check: msgId=%s,txId=%s,checkTimes=%s,sendResult,lastCheckResult=%s\\n\",\n                    new SimpleDateFormat(\"HH:mm:ss,SSS\").format(new Date()),\n                    msg.getMsgId(), msg.getTransactionId(),\n                    msg.getUserProperty(MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES), s);\n                statBenchmark.getUnexpectedCheckCount().increment();\n                return s;\n            }\n        }\n        return msgMeta.checkResult.get(times - 1);\n    }\n}\n\nclass Snapshot {\n    long endTime;\n\n    long sendRequestSuccessCount;\n\n    long sendRequestFailedCount;\n\n    long sendMessageTimeTotal;\n\n    long sendMessageMaxRT;\n\n    long checkCount;\n\n    long unexpectedCheckCount;\n\n    long duplicatedCheck;\n}\n\nclass StatsBenchmarkTProducer {\n    private final LongAdder sendRequestSuccessCount = new LongAdder();\n\n    private final LongAdder sendRequestFailedCount = new LongAdder();\n\n    private final LongAdder sendMessageTimeTotal = new LongAdder();\n\n    private final AtomicLong sendMessageMaxRT = new AtomicLong(0L);\n\n    private final LongAdder checkCount = new LongAdder();\n\n    private final LongAdder unexpectedCheckCount = new LongAdder();\n\n    private final LongAdder duplicatedCheckCount = new LongAdder();\n\n    public Snapshot createSnapshot() {\n        Snapshot s = new Snapshot();\n        s.endTime = System.currentTimeMillis();\n        s.sendRequestSuccessCount = sendRequestSuccessCount.longValue();\n        s.sendRequestFailedCount = sendRequestFailedCount.longValue();\n        s.sendMessageTimeTotal = sendMessageTimeTotal.longValue();\n        s.sendMessageMaxRT = sendMessageMaxRT.get();\n        s.checkCount = checkCount.longValue();\n        s.unexpectedCheckCount = unexpectedCheckCount.longValue();\n        s.duplicatedCheck = duplicatedCheckCount.longValue();\n        return s;\n    }\n\n    public LongAdder getSendRequestSuccessCount() {\n        return sendRequestSuccessCount;\n    }\n\n    public LongAdder getSendRequestFailedCount() {\n        return sendRequestFailedCount;\n    }\n\n    public LongAdder getSendMessageTimeTotal() {\n        return sendMessageTimeTotal;\n    }\n\n    public AtomicLong getSendMessageMaxRT() {\n        return sendMessageMaxRT;\n    }\n\n    public LongAdder getCheckCount() {\n        return checkCount;\n    }\n\n    public LongAdder getUnexpectedCheckCount() {\n        return unexpectedCheckCount;\n    }\n\n    public LongAdder getDuplicatedCheckCount() {\n        return duplicatedCheckCount;\n    }\n}\n\nclass TxSendConfig {\n    String topic;\n    int threadCount;\n    int messageSize;\n    double sendRollbackRate;\n    double sendUnknownRate;\n    double checkRollbackRate;\n    double checkUnknownRate;\n    long batchId;\n    int sendInterval;\n    boolean aclEnable;\n    boolean msgTraceEnable;\n    int reportInterval;\n}\n\nclass LRUMap<K, V> extends LinkedHashMap<K, V> {\n\n    private int maxSize;\n\n    public LRUMap(int maxSize) {\n        this.maxSize = maxSize;\n    }\n\n    @Override\n    protected boolean removeEldestEntry(Map.Entry eldest) {\n        return size() > maxSize;\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/benchmark/timer/TimerConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.benchmark.timer;\n\nimport org.apache.commons.cli.CommandLine;\nimport org.apache.commons.cli.DefaultParser;\nimport org.apache.commons.cli.Option;\nimport org.apache.commons.cli.Options;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.srvutil.ServerUtil;\n\nimport java.util.ArrayList;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.TimerTask;\nimport java.util.concurrent.ConcurrentSkipListSet;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ScheduledThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\n\npublic class TimerConsumer {\n    private final String topic;\n\n    private final ScheduledExecutorService scheduledExecutor = new ScheduledThreadPoolExecutor(1, new ThreadFactoryImpl(\"ConsumerScheduleThread_\"));\n\n    private final StatsBenchmarkConsumer statsBenchmark = new StatsBenchmarkConsumer();\n    private final LinkedList<Long[]> snapshotList = new LinkedList<>();\n\n    private final DefaultMQPushConsumer consumer;\n\n    public TimerConsumer(String[] args) {\n        Options options = ServerUtil.buildCommandlineOptions(new Options());\n        final CommandLine commandLine = ServerUtil.parseCmdLine(\"benchmarkTimerConsumer\", args, buildCommandlineOptions(options), new DefaultParser());\n        if (null == commandLine) {\n            System.exit(-1);\n        }\n\n        final String namesrvAddr = commandLine.hasOption('n') ? commandLine.getOptionValue('t').trim() : \"localhost:9876\";\n        topic = commandLine.hasOption('t') ? commandLine.getOptionValue('t').trim() : \"BenchmarkTest\";\n        System.out.printf(\"namesrvAddr: %s, topic: %s%n\", namesrvAddr, topic);\n\n        consumer = new DefaultMQPushConsumer(\"benchmark_consumer\");\n        consumer.setInstanceName(Long.toString(System.currentTimeMillis()));\n        consumer.setNamesrvAddr(namesrvAddr);\n    }\n\n    public void startScheduleTask() {\n        scheduledExecutor.scheduleAtFixedRate(new TimerTask() {\n            @Override\n            public void run() {\n                snapshotList.addLast(statsBenchmark.createSnapshot());\n                if (snapshotList.size() > 10) {\n                    snapshotList.removeFirst();\n                }\n            }\n        }, 1000, 1000, TimeUnit.MILLISECONDS);\n\n        scheduledExecutor.scheduleAtFixedRate(new TimerTask() {\n            private void printStats() {\n                if (snapshotList.size() >= 10) {\n                    Long[] begin = snapshotList.getFirst();\n                    Long[] end = snapshotList.getLast();\n\n                    final long consumeTps =\n                            (long) (((end[1] - begin[1]) / (double) (end[0] - begin[0])) * 1000L);\n                    final double avgDelayedDuration = (end[2] - begin[2]) / (double) (end[1] - begin[1]);\n\n                    List<Long> delayedDurationList = new ArrayList<>(TimerConsumer.this.statsBenchmark.getDelayedDurationMsSet());\n                    if (delayedDurationList.isEmpty()) {\n                        System.out.printf(\"Consume TPS: %d, Avg delayedDuration: %7.3f, Max delayedDuration: 0, %n\",\n                                consumeTps, avgDelayedDuration);\n                    } else {\n                        long delayedDuration25 = delayedDurationList.get((int) (delayedDurationList.size() * 0.25));\n                        long delayedDuration50 = delayedDurationList.get((int) (delayedDurationList.size() * 0.5));\n                        long delayedDuration80 = delayedDurationList.get((int) (delayedDurationList.size() * 0.8));\n                        long delayedDuration90 = delayedDurationList.get((int) (delayedDurationList.size() * 0.9));\n                        long delayedDuration99 = delayedDurationList.get((int) (delayedDurationList.size() * 0.99));\n                        long delayedDuration999 = delayedDurationList.get((int) (delayedDurationList.size() * 0.999));\n\n                        System.out.printf(\"Consume TPS: %d, Avg delayedDuration: %7.3f, Max delayedDuration: %d, \" +\n                                        \"delayDuration %%25: %d, %%50: %d; %%80: %d; %%90: %d; %%99: %d; %%99.9: %d%n\",\n                                consumeTps, avgDelayedDuration, delayedDurationList.get(delayedDurationList.size() - 1),\n                                delayedDuration25, delayedDuration50, delayedDuration80, delayedDuration90, delayedDuration99, delayedDuration999);\n                    }\n                }\n            }\n\n            @Override\n            public void run() {\n                try {\n                    this.printStats();\n                } catch (Exception e) {\n                    e.printStackTrace();\n                }\n            }\n        }, 10000, 10000, TimeUnit.MILLISECONDS);\n    }\n\n    public void start() throws MQClientException {\n        consumer.subscribe(topic, \"*\");\n\n        consumer.registerMessageListener(new MessageListenerConcurrently() {\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {\n                MessageExt msg = msgs.get(0);\n                long now = System.currentTimeMillis();\n\n                statsBenchmark.getReceiveMessageTotalCount().incrementAndGet();\n\n                long deliverTimeMs = Long.parseLong(msg.getProperty(\"MY_RECORD_TIMER_DELIVER_MS\"));\n                long delayedDuration = now - deliverTimeMs;\n\n                statsBenchmark.getDelayedDurationMsSet().add(delayedDuration);\n                statsBenchmark.getDelayedDurationMsTotal().addAndGet(delayedDuration);\n\n                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n            }\n        });\n\n        consumer.start();\n        System.out.printf(\"Start receiving messages%n\");\n    }\n\n    private Options buildCommandlineOptions(Options options) {\n        Option opt = new Option(\"n\", \"namesrvAddr\", true, \"Nameserver address, default: localhost:9876\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"t\", \"topic\", true, \"Send messages to which topic, default: BenchmarkTest\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        return options;\n    }\n\n    public static void main(String[] args) throws MQClientException {\n        TimerConsumer timerConsumer = new TimerConsumer(args);\n        timerConsumer.startScheduleTask();\n        timerConsumer.start();\n    }\n\n\n    public static class StatsBenchmarkConsumer {\n        private final AtomicLong receiveMessageTotalCount = new AtomicLong(0L);\n\n        private final AtomicLong delayedDurationMsTotal = new AtomicLong(0L);\n        private final ConcurrentSkipListSet<Long> delayedDurationMsSet = new ConcurrentSkipListSet<>();\n\n        public Long[] createSnapshot() {\n            return new Long[]{\n                    System.currentTimeMillis(),\n                    this.receiveMessageTotalCount.get(),\n                    this.delayedDurationMsTotal.get(),\n            };\n        }\n\n        public AtomicLong getReceiveMessageTotalCount() {\n            return receiveMessageTotalCount;\n        }\n\n        public AtomicLong getDelayedDurationMsTotal() {\n            return delayedDurationMsTotal;\n        }\n\n        public ConcurrentSkipListSet<Long> getDelayedDurationMsSet() {\n            return delayedDurationMsSet;\n        }\n    }\n\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/benchmark/timer/TimerProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.benchmark.timer;\n\nimport org.apache.commons.cli.CommandLine;\nimport org.apache.commons.cli.DefaultParser;\nimport org.apache.commons.cli.Option;\nimport org.apache.commons.cli.Options;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.srvutil.ServerUtil;\n\nimport java.io.UnsupportedEncodingException;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.TimerTask;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ScheduledThreadPoolExecutor;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\n\npublic class TimerProducer {\n    private static final Logger log = LoggerFactory.getLogger(TimerProducer.class);\n\n    private final String topic;\n    private final int threadCount;\n    private final int messageSize;\n\n    private final int precisionMs;\n    private final int slotsTotal;\n    private final int msgsTotalPerSlotThread;\n    private final int slotDis;\n\n    private final ScheduledExecutorService scheduledExecutor = new ScheduledThreadPoolExecutor(1, new ThreadFactoryImpl(\"ProducerScheduleThread_\"));\n    private final ExecutorService sendThreadPool;\n\n    private final StatsBenchmarkProducer statsBenchmark = new StatsBenchmarkProducer();\n    private final LinkedList<Long[]> snapshotList = new LinkedList<>();\n\n    private final DefaultMQProducer producer;\n\n    public TimerProducer(String[] args) {\n        Options options = ServerUtil.buildCommandlineOptions(new Options());\n        final CommandLine commandLine = ServerUtil.parseCmdLine(\"benchmarkTimerProducer\", args, buildCommandlineOptions(options), new DefaultParser());\n        if (null == commandLine) {\n            System.exit(-1);\n        }\n\n        final String namesrvAddr = commandLine.hasOption('n') ? commandLine.getOptionValue('t').trim() : \"localhost:9876\";\n        topic = commandLine.hasOption('t') ? commandLine.getOptionValue('t').trim() : \"BenchmarkTest\";\n        threadCount = commandLine.hasOption(\"tc\") ? Integer.parseInt(commandLine.getOptionValue(\"tc\")) : 16;\n        messageSize = commandLine.hasOption(\"ms\") ? Integer.parseInt(commandLine.getOptionValue(\"ms\")) : 1024;\n        precisionMs = commandLine.hasOption('p') ? Integer.parseInt(commandLine.getOptionValue(\"p\")) : 1000;\n        slotsTotal = commandLine.hasOption(\"st\") ? Integer.parseInt(commandLine.getOptionValue(\"st\")) : 100;\n        msgsTotalPerSlotThread = commandLine.hasOption(\"mt\") ? Integer.parseInt(commandLine.getOptionValue(\"mt\")) : 5000;\n        slotDis = commandLine.hasOption(\"sd\") ? Integer.parseInt(commandLine.getOptionValue(\"sd\")) : 1000;\n        System.out.printf(\"namesrvAddr: %s, topic: %s, threadCount: %d, messageSize: %d, precisionMs: %d, slotsTotal: %d, msgsTotalPerSlotThread: %d, slotDis: %d%n\",\n                namesrvAddr, topic, threadCount, messageSize, precisionMs, slotsTotal, msgsTotalPerSlotThread, slotDis);\n\n        sendThreadPool = new ThreadPoolExecutor(\n                threadCount,\n                threadCount,\n                0L,\n                TimeUnit.MILLISECONDS,\n            new LinkedBlockingQueue<>(),\n                new ThreadFactoryImpl(\"ProducerSendMessageThread_\"));\n\n        producer = new DefaultMQProducer(\"benchmark_producer\");\n        producer.setInstanceName(Long.toString(System.currentTimeMillis()));\n        producer.setNamesrvAddr(namesrvAddr);\n        producer.setCompressMsgBodyOverHowmuch(Integer.MAX_VALUE);\n    }\n\n    public void startScheduleTask() {\n        scheduledExecutor.scheduleAtFixedRate(new TimerTask() {\n            @Override\n            public void run() {\n                snapshotList.addLast(statsBenchmark.createSnapshot());\n                if (snapshotList.size() > 10) {\n                    snapshotList.removeFirst();\n                }\n            }\n        }, 1000, 1000, TimeUnit.MILLISECONDS);\n\n        scheduledExecutor.scheduleAtFixedRate(new TimerTask() {\n            private void printStats() {\n                if (snapshotList.size() >= 10) {\n                    Long[] begin = snapshotList.getFirst();\n                    Long[] end = snapshotList.getLast();\n\n                    final long sendTps = (long) (((end[3] - begin[3]) / (double) (end[0] - begin[0])) * 1000L);\n                    final double averageRT = (end[5] - begin[5]) / (double) (end[3] - begin[3]);\n\n                    System.out.printf(\"Send TPS: %d, Max RT: %d, Average RT: %7.3f, Send Failed: %d, Response Failed: %d%n\",\n                            sendTps, statsBenchmark.getSendMessageMaxRT().get(), averageRT, end[2], end[4]);\n                }\n            }\n\n            @Override\n            public void run() {\n                try {\n                    this.printStats();\n                } catch (Exception e) {\n                    e.printStackTrace();\n                }\n            }\n        }, 10000, 10000, TimeUnit.MILLISECONDS);\n    }\n\n    public void start() throws MQClientException {\n        producer.start();\n        System.out.printf(\"Start sending messages%n\");\n        List<Long> delayList = new ArrayList<>();\n        final long startDelayTime = System.currentTimeMillis() / precisionMs * precisionMs + 2 * 60 * 1000 + 10;\n        for (int slotCnt = 0; slotCnt < slotsTotal; slotCnt++) {\n            for (int msgCnt = 0; msgCnt < msgsTotalPerSlotThread; msgCnt++) {\n                long delayTime = startDelayTime + slotCnt * slotDis;\n                delayList.add(delayTime);\n            }\n        }\n        Collections.shuffle(delayList);\n        // DelayTime is from 2 minutes later.\n\n        for (int i = 0; i < threadCount; i++) {\n            sendThreadPool.execute(new Runnable() {\n                @Override\n                public void run() {\n                    for (int slotCnt = 0; slotCnt < slotsTotal; slotCnt++) {\n\n                        for (int msgCnt = 0; msgCnt < msgsTotalPerSlotThread; msgCnt++) {\n                            final long beginTimestamp = System.currentTimeMillis();\n\n                            long delayTime = delayList.get(slotCnt * msgsTotalPerSlotThread + msgCnt);\n\n                            final Message msg;\n                            try {\n                                msg = buildMessage(messageSize, topic);\n                            } catch (UnsupportedEncodingException e) {\n                                e.printStackTrace();\n                                return;\n                            }\n                            msg.putUserProperty(\"MY_RECORD_TIMER_DELIVER_MS\", String.valueOf(delayTime));\n                            msg.getProperties().put(MessageConst.PROPERTY_TIMER_DELIVER_MS, String.valueOf(delayTime));\n\n                            try {\n                                producer.send(msg);\n\n                                statsBenchmark.getSendRequestSuccessCount().incrementAndGet();\n                                statsBenchmark.getReceiveResponseSuccessCount().incrementAndGet();\n\n                                final long currentRT = System.currentTimeMillis() - beginTimestamp;\n                                statsBenchmark.getSendMessageSuccessTimeTotal().addAndGet(currentRT);\n\n                                long prevMaxRT = statsBenchmark.getSendMessageMaxRT().get();\n                                while (currentRT > prevMaxRT) {\n                                    if (statsBenchmark.getSendMessageMaxRT().compareAndSet(prevMaxRT, currentRT)) {\n                                        break;\n                                    }\n                                    prevMaxRT = statsBenchmark.getSendMessageMaxRT().get();\n                                }\n                            } catch (RemotingException e) {\n                                statsBenchmark.getSendRequestFailedCount().incrementAndGet();\n                                log.error(\"[BENCHMARK_PRODUCER] Send Exception\", e);\n                                sleep(3000);\n                            } catch (InterruptedException e) {\n                                statsBenchmark.getSendRequestFailedCount().incrementAndGet();\n                                sleep(3000);\n                            } catch (MQClientException e) {\n                                statsBenchmark.getSendRequestFailedCount().incrementAndGet();\n                                log.error(\"[BENCHMARK_PRODUCER] Send Exception\", e);\n                            } catch (MQBrokerException e) {\n                                statsBenchmark.getReceiveResponseFailedCount().incrementAndGet();\n                                log.error(\"[BENCHMARK_PRODUCER] Send Exception\", e);\n                                sleep(3000);\n                            }\n                        }\n\n                    }\n                }\n            });\n        }\n    }\n\n    private Options buildCommandlineOptions(Options options) {\n        Option opt = new Option(\"n\", \"namesrvAddr\", true, \"Nameserver address, default: localhost:9876\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"t\", \"topic\", true, \"Send messages to which topic, default: BenchmarkTest\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"tc\", \"threadCount\", true, \"Thread count, default: 64\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"ms\", \"messageSize\", true, \"Message Size, default: 128\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"p\", \"precisionMs\", true, \"Precision (ms) for TimerMessage, default: 1000\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"st\", \"slotsTotal\", true, \"Send messages to how many slots, default: 100\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"mt\", \"msgsTotalPerSlotThread\", true, \"Messages total for each slot and each thread, default: 100\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"sd\", \"slotDis\", true, \"Time distance between two slots, default: 1000\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        return options;\n    }\n\n    private Message buildMessage(final int messageSize, final String topic) throws UnsupportedEncodingException {\n        Message msg = new Message();\n        msg.setTopic(topic);\n\n        String body = StringUtils.repeat('a', messageSize);\n        msg.setBody(body.getBytes(RemotingHelper.DEFAULT_CHARSET));\n\n        return msg;\n    }\n\n    private void sleep(long timeMs) {\n        try {\n            Thread.sleep(timeMs);\n        } catch (InterruptedException e) {\n            e.printStackTrace();\n        }\n    }\n\n    public static void main(String[] args) throws MQClientException {\n        TimerProducer timerProducer = new TimerProducer(args);\n        timerProducer.startScheduleTask();\n        timerProducer.start();\n    }\n\n\n    public static class StatsBenchmarkProducer {\n        private final AtomicLong sendRequestSuccessCount = new AtomicLong(0L);\n\n        private final AtomicLong sendRequestFailedCount = new AtomicLong(0L);\n\n        private final AtomicLong receiveResponseSuccessCount = new AtomicLong(0L);\n\n        private final AtomicLong receiveResponseFailedCount = new AtomicLong(0L);\n\n        private final AtomicLong sendMessageSuccessTimeTotal = new AtomicLong(0L);\n\n        private final AtomicLong sendMessageMaxRT = new AtomicLong(0L);\n\n        public Long[] createSnapshot() {\n            return new Long[]{\n                    System.currentTimeMillis(),\n                    this.sendRequestSuccessCount.get(),\n                    this.sendRequestFailedCount.get(),\n                    this.receiveResponseSuccessCount.get(),\n                    this.receiveResponseFailedCount.get(),\n                    this.sendMessageSuccessTimeTotal.get(),\n            };\n        }\n\n        public AtomicLong getSendRequestSuccessCount() {\n            return sendRequestSuccessCount;\n        }\n\n        public AtomicLong getSendRequestFailedCount() {\n            return sendRequestFailedCount;\n        }\n\n        public AtomicLong getReceiveResponseSuccessCount() {\n            return receiveResponseSuccessCount;\n        }\n\n        public AtomicLong getReceiveResponseFailedCount() {\n            return receiveResponseFailedCount;\n        }\n\n        public AtomicLong getSendMessageSuccessTimeTotal() {\n            return sendMessageSuccessTimeTotal;\n        }\n\n        public AtomicLong getSendMessageMaxRT() {\n            return sendMessageMaxRT;\n        }\n    }\n\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/broadcast/PushConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.broadcast;\n\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\n\npublic class PushConsumer {\n\n    public static final String CONSUMER_GROUP = \"please_rename_unique_group_name_1\";\n    public static final String DEFAULT_NAMESRVADDR = \"127.0.0.1:9876\";\n    public static final String TOPIC = \"TopicTest\";\n\n    public static final String SUB_EXPRESSION = \"TagA || TagC || TagD\";\n\n    public static void main(String[] args) throws InterruptedException, MQClientException {\n\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(CONSUMER_GROUP);\n\n        // Uncomment the following line while debugging, namesrvAddr should be set to your local address\n//        consumer.setNamesrvAddr(DEFAULT_NAMESRVADDR);\n\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n\n        consumer.setMessageModel(MessageModel.BROADCASTING);\n\n        consumer.subscribe(TOPIC, SUB_EXPRESSION);\n\n        consumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            System.out.printf(\"%s Receive New Messages: %s %n\", Thread.currentThread().getName(), msgs);\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n\n        consumer.start();\n        System.out.printf(\"Broadcast Consumer Started.%n\");\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/filter/SqlFilterConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.example.filter;\n\nimport java.util.List;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.MessageSelector;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic class SqlFilterConsumer {\n\n    public static void main(String[] args) throws Exception {\n\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"please_rename_unique_group_name\");\n\n        // Don't forget to set enablePropertyFilter=true in broker\n        consumer.subscribe(\"SqlFilterTest\",\n            MessageSelector.bySql(\"(TAGS is not null and TAGS in ('TagA', 'TagB'))\" +\n                \"and (a is not null and a between 0 and 3)\"));\n\n        consumer.registerMessageListener(new MessageListenerConcurrently() {\n\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,\n                ConsumeConcurrentlyContext context) {\n                System.out.printf(\"%s Receive New Messages: %s %n\", Thread.currentThread().getName(), msgs);\n                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n            }\n        });\n\n        consumer.start();\n        System.out.printf(\"Consumer Started.%n\");\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/filter/SqlFilterProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.example.filter;\n\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\n\npublic class SqlFilterProducer {\n\n    public static void main(String[] args) throws Exception {\n\n        DefaultMQProducer producer = new DefaultMQProducer(\"please_rename_unique_group_name\");\n\n        producer.start();\n\n        String[] tags = new String[] {\"TagA\", \"TagB\", \"TagC\"};\n\n        for (int i = 0; i < 10; i++) {\n            Message msg = new Message(\"SqlFilterTest\",\n                tags[i % tags.length],\n                (\"Hello RocketMQ \" + i).getBytes(RemotingHelper.DEFAULT_CHARSET)\n            );\n            msg.putUserProperty(\"a\", String.valueOf(i));\n\n            SendResult sendResult = producer.send(msg);\n            System.out.printf(\"%s%n\", sendResult);\n        }\n\n        producer.shutdown();\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/filter/TagFilterConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.filter;\n\nimport java.util.List;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic class TagFilterConsumer {\n\n    public static void main(String[] args) throws MQClientException {\n\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"please_rename_unique_group_name\");\n\n        consumer.subscribe(\"TagFilterTest\", \"TagA || TagC\");\n\n        consumer.registerMessageListener(new MessageListenerConcurrently() {\n\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,\n                ConsumeConcurrentlyContext context) {\n                System.out.printf(\"%s Receive New Messages: %s %n\", Thread.currentThread().getName(), msgs);\n                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n            }\n        });\n\n        consumer.start();\n\n        System.out.printf(\"Consumer Started.%n\");\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/filter/TagFilterProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.filter;\n\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\n\npublic class TagFilterProducer {\n\n    public static void main(String[] args) throws Exception {\n\n        DefaultMQProducer producer = new DefaultMQProducer(\"please_rename_unique_group_name\");\n        producer.start();\n\n        String[] tags = new String[] {\"TagA\", \"TagB\", \"TagC\"};\n\n        for (int i = 0; i < 60; i++) {\n            Message msg = new Message(\"TagFilterTest\",\n                tags[i % tags.length],\n                \"Hello world\".getBytes(RemotingHelper.DEFAULT_CHARSET));\n\n            SendResult sendResult = producer.send(msg);\n            System.out.printf(\"%s%n\", sendResult);\n        }\n\n        producer.shutdown();\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/lmq/LMQProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.lmq;\n\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\n\npublic class LMQProducer {\n    public static final String PRODUCER_GROUP = \"ProducerGroupName\";\n\n    public static final String DEFAULT_NAMESRVADDR = \"127.0.0.1:9876\";\n\n    public static final String TOPIC = \"TopicLMQParent\";\n\n    public static final String TAG = \"TagA\";\n\n    public static final String LMQ_TOPIC_1 = MixAll.LMQ_PREFIX + \"123\";\n\n    public static final String LMQ_TOPIC_2 = MixAll.LMQ_PREFIX + \"456\";\n\n    public static void main(String[] args) throws MQClientException, InterruptedException {\n        DefaultMQProducer producer = new DefaultMQProducer(PRODUCER_GROUP);\n\n        // Uncomment the following line while debugging, namesrvAddr should be set to your local address\n        producer.setNamesrvAddr(DEFAULT_NAMESRVADDR);\n\n        producer.start();\n        for (int i = 0; i < 128; i++) {\n            try {\n                Message msg = new Message(TOPIC, TAG, (\"Hello RocketMQ \" + i).getBytes(RemotingHelper.DEFAULT_CHARSET));\n                msg.setKeys(\"Key\" + i);\n                msg.putUserProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH /* \"INNER_MULTI_DISPATCH\" */,\n                    String.join(MixAll.LMQ_DISPATCH_SEPARATOR, LMQ_TOPIC_1, LMQ_TOPIC_2) /* \"%LMQ%123,%LMQ%456\" */);\n                SendResult sendResult = producer.send(msg);\n                System.out.printf(\"%s%n\", sendResult);\n            } catch (Exception e) {\n                e.printStackTrace();\n            }\n        }\n\n        producer.shutdown();\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/lmq/LMQPullConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.lmq;\n\nimport java.util.Arrays;\nimport java.util.HashSet;\nimport java.util.List;\nimport org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;\nimport org.apache.rocketmq.client.consumer.PullCallback;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\n\n@SuppressWarnings(\"deprecation\")\npublic class LMQPullConsumer {\n    public static final String BROKER_NAME = \"broker-a\";\n\n    public static final String CONSUMER_GROUP = \"CID_LMQ_PULL_1\";\n\n    public static final String TOPIC = \"TopicLMQParent\";\n\n    public static final String LMQ_TOPIC = MixAll.LMQ_PREFIX + \"123\";\n\n    public static final String NAMESRV_ADDR = \"127.0.0.1:9876\";\n\n    public static void main(String[] args) throws MQClientException, RemotingException, InterruptedException {\n\n        DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(CONSUMER_GROUP);\n        consumer.setNamesrvAddr(NAMESRV_ADDR);\n        consumer.setRegisterTopics(new HashSet<>(Arrays.asList(TOPIC)));\n        consumer.start();\n\n        // use parent topic to fill up broker addr table\n        consumer.getDefaultMQPullConsumerImpl().getRebalanceImpl().getmQClientFactory()\n            .updateTopicRouteInfoFromNameServer(TOPIC);\n\n        final MessageQueue lmq = new MessageQueue(LMQ_TOPIC, BROKER_NAME, (int) MixAll.LMQ_QUEUE_ID);\n        long offset = consumer.minOffset(lmq);\n\n        consumer.pullBlockIfNotFound(lmq, \"*\", offset, 32, new PullCallback() {\n            @Override\n            public void onSuccess(PullResult pullResult) {\n                List<MessageExt> list = pullResult.getMsgFoundList();\n                if (list == null || list.isEmpty()) {\n                    return;\n                }\n\n                for (MessageExt msg : list) {\n                    System.out.printf(\"%s Pull New Messages: %s %n\", Thread.currentThread().getName(), msg);\n                }\n            }\n\n            @Override\n            public void onException(Throwable e) {\n\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/lmq/LMQPushConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.lmq;\n\nimport com.google.common.collect.Lists;\n\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\n\npublic class LMQPushConsumer {\n    public static final String CLUSTER_NAME = \"DefaultCluster\";\n\n    public static final String BROKER_NAME = \"broker-a\";\n\n    public static final String TOPIC = \"TopicLMQParent\";\n\n    public static final String LMQ_TOPIC = MixAll.LMQ_PREFIX + \"123\";\n\n    public static final String CONSUMER_GROUP = \"CID_LMQ_1\";\n\n    public static final String NAMESRV_ADDR = \"127.0.0.1:9876\";\n\n    public static final HashMap<Long, String> BROKER_ADDR_MAP = new HashMap<Long, String>() {\n        {\n            put(MixAll.MASTER_ID, \"127.0.0.1:10911\");\n        }\n    };\n\n    public static void main(String[] args) throws InterruptedException, MQClientException {\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(CONSUMER_GROUP);\n        consumer.setNamesrvAddr(NAMESRV_ADDR);\n        consumer.subscribe(LMQ_TOPIC, \"*\");\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        consumer.registerMessageListener(new MessageListenerConcurrently() {\n\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {\n                System.out.printf(\"%s Receive New Messages: %s %n\", Thread.currentThread().getName(), msgs);\n                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n            }\n        });\n        consumer.start();\n\n        // use parent topic to fill up broker addr table\n        consumer.getDefaultMQPushConsumerImpl().getmQClientFactory().updateTopicRouteInfoFromNameServer(TOPIC);\n\n        final TopicRouteData topicRouteData = new TopicRouteData();\n        final BrokerData brokerData = new BrokerData();\n        brokerData.setCluster(CLUSTER_NAME);\n        brokerData.setBrokerName(BROKER_NAME);\n        brokerData.setBrokerAddrs(BROKER_ADDR_MAP);\n        topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData));\n        // compensate LMQ topic route for MQClientInstance#findBrokerAddrByTopic\n        consumer.getDefaultMQPushConsumerImpl().getmQClientFactory().getTopicRouteTable().put(LMQ_TOPIC, topicRouteData);\n        // compensate for RebalanceImpl#topicSubscribeInfoTable\n        consumer.getDefaultMQPushConsumerImpl().updateTopicSubscribeInfo(LMQ_TOPIC,\n            new HashSet<>(Arrays.asList(new MessageQueue(LMQ_TOPIC, BROKER_NAME, (int) MixAll.LMQ_QUEUE_ID))));\n        // re-balance immediately to start pulling messages\n        consumer.getDefaultMQPushConsumerImpl().getmQClientFactory().doRebalance();\n\n        System.out.printf(\"Consumer Started.%n\");\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/lmq/LMQPushPopConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.lmq;\n\nimport com.google.common.collect.Lists;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageRequestMode;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.apache.rocketmq.tools.admin.DefaultMQAdminExt;\n\npublic class LMQPushPopConsumer {\n    public static final String CLUSTER_NAME = \"DefaultCluster\";\n\n    public static final String BROKER_NAME = \"broker-a\";\n\n    public static final String TOPIC = \"TopicLMQParent\";\n\n    public static final String LMQ_TOPIC = MixAll.LMQ_PREFIX + \"456\";\n\n    public static final String NAMESRV_ADDR = \"127.0.0.1:9876\";\n\n    public static final String CONSUMER_GROUP = \"CID_LMQ_POP_1\";\n\n    public static final HashMap<Long, String> BROKER_ADDR_MAP = new HashMap<Long, String>() {\n        {\n            put(MixAll.MASTER_ID, \"127.0.0.1:10911\");\n        }\n    };\n\n    public static void main(String[] args) throws Exception {\n        switchPop();\n\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(CONSUMER_GROUP);\n        consumer.setNamesrvAddr(NAMESRV_ADDR);\n        consumer.subscribe(LMQ_TOPIC, \"*\");\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        consumer.registerMessageListener(new MessageListenerConcurrently() {\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {\n                System.out.printf(\"%s Receive New Messages: %s %n\", Thread.currentThread().getName(), msgs);\n                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n            }\n        });\n        // use server side rebalance\n        consumer.setClientRebalance(false);\n        consumer.start();\n\n        // use parent topic to fill up broker addr table\n        consumer.getDefaultMQPushConsumerImpl().getmQClientFactory().updateTopicRouteInfoFromNameServer(TOPIC);\n\n        final TopicRouteData topicRouteData = new TopicRouteData();\n        final BrokerData brokerData = new BrokerData();\n        brokerData.setCluster(CLUSTER_NAME);\n        brokerData.setBrokerName(BROKER_NAME);\n        brokerData.setBrokerAddrs(BROKER_ADDR_MAP);\n        topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData));\n        // compensate LMQ topic route for MQClientInstance#findBrokerAddrByTopic\n        consumer.getDefaultMQPushConsumerImpl().getmQClientFactory().getTopicRouteTable().put(LMQ_TOPIC, topicRouteData);\n        // re-balance immediately to start pulling messages\n        consumer.getDefaultMQPushConsumerImpl().getmQClientFactory().doRebalance();\n\n        System.out.printf(\"Consumer Started.%n\");\n    }\n\n    private static void switchPop() throws Exception {\n        DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt();\n        mqAdminExt.setNamesrvAddr(NAMESRV_ADDR);\n        mqAdminExt.start();\n        List<BrokerData> brokerDatas = mqAdminExt.examineTopicRouteInfo(TOPIC).getBrokerDatas();\n        for (BrokerData brokerData : brokerDatas) {\n            Set<String> brokerAddrs = new HashSet<>(brokerData.getBrokerAddrs().values());\n            for (String brokerAddr : brokerAddrs) {\n                mqAdminExt.setMessageRequestMode(brokerAddr, LMQ_TOPIC, CONSUMER_GROUP, MessageRequestMode.POP, 8,\n                    3_000);\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/namespace/ProducerWithNamespace.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.namespace;\n\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.message.Message;\n\nimport java.nio.charset.StandardCharsets;\n\npublic class ProducerWithNamespace {\n\n    public static final String NAMESPACE = \"InstanceTest\";\n    public static final String PRODUCER_GROUP = \"pidTest\";\n    public static final String DEFAULT_NAMESRVADDR = \"127.0.0.1:9876\";\n    public static final int MESSAGE_COUNT = 100;\n    public static final String TOPIC = \"NAMESPACE_TOPIC\";\n    public static final String TAG = \"tagTest\";\n\n    public static void main(String[] args) throws Exception {\n\n        DefaultMQProducer producer = new DefaultMQProducer(PRODUCER_GROUP);\n        producer.setNamespaceV2(NAMESPACE);\n\n        producer.setNamesrvAddr(DEFAULT_NAMESRVADDR);\n        producer.start();\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            Message message = new Message(TOPIC, TAG, \"Hello world\".getBytes(StandardCharsets.UTF_8));\n            try {\n                SendResult result = producer.send(message);\n                System.out.printf(\"Topic:%s send success, misId is:%s%n\", message.getTopic(), result.getMsgId());\n            } catch (Exception e) {\n                e.printStackTrace();\n            }\n        }\n        producer.shutdown();\n    }\n}"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/namespace/PullConsumerWithNamespace.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.namespace;\n\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.Set;\n\nimport org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\npublic class PullConsumerWithNamespace {\n\n    public static final String NAMESPACE = \"InstanceTest\";\n    public static final String CONSUMER_GROUP = \"cidTest\";\n    public static final String DEFAULT_NAMESRVADDR = \"127.0.0.1:9876\";\n    public static final String TOPIC = \"NAMESPACE_TOPIC\";\n\n    private static final Map<MessageQueue, Long> OFFSET_TABLE = new HashMap<>();\n\n    public static void main(String[] args) throws Exception {\n        DefaultMQPullConsumer pullConsumer = new DefaultMQPullConsumer(CONSUMER_GROUP);\n        pullConsumer.setNamespaceV2(NAMESPACE);\n        pullConsumer.setNamesrvAddr(DEFAULT_NAMESRVADDR);\n        pullConsumer.start();\n\n        Set<MessageQueue> mqs = pullConsumer.fetchSubscribeMessageQueues(TOPIC);\n        for (MessageQueue mq : mqs) {\n            System.out.printf(\"Consume from the topic: %s, queue: %s%n\", mq.getTopic(), mq);\n            SINGLE_MQ:\n            while (true) {\n                try {\n                    PullResult pullResult =\n                        pullConsumer.pullBlockIfNotFound(mq, null, getMessageQueueOffset(mq), 32);\n                    System.out.printf(\"%s%n\", pullResult);\n\n                    putMessageQueueOffset(mq, pullResult.getNextBeginOffset());\n                    switch (pullResult.getPullStatus()) {\n                        case FOUND:\n                            dealWithPullResult(pullResult);\n                            break;\n                        case NO_MATCHED_MSG:\n                            break;\n                        case NO_NEW_MSG:\n                            break SINGLE_MQ;\n                        case OFFSET_ILLEGAL:\n                            break;\n                        default:\n                            break;\n                    }\n                } catch (Exception e) {\n                    e.printStackTrace();\n                }\n            }\n        }\n\n        pullConsumer.shutdown();\n    }\n\n    private static long getMessageQueueOffset(MessageQueue mq) {\n        Long offset = OFFSET_TABLE.get(mq);\n        if (offset != null) {\n            return offset;\n        }\n\n        return 0;\n    }\n\n    private static void dealWithPullResult(PullResult pullResult) {\n        if (null == pullResult || pullResult.getMsgFoundList().isEmpty()) {\n            return;\n        }\n        pullResult.getMsgFoundList().forEach(\n            msg -> System.out.printf(\"Topic is:%s, msgId is:%s%n\", msg.getTopic(), msg.getMsgId()));\n    }\n\n    private static void putMessageQueueOffset(MessageQueue mq, long offset) {\n        OFFSET_TABLE.put(mq, offset);\n    }\n}"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/namespace/PushConsumerWithNamespace.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.namespace;\n\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\n\npublic class PushConsumerWithNamespace {\n    public static final String NAMESPACE = \"InstanceTest\";\n    public static final String CONSUMER_GROUP = \"cidTest\";\n    public static final String DEFAULT_NAMESRVADDR = \"127.0.0.1:9876\";\n    public static final String TOPIC = \"NAMESPACE_TOPIC\";\n\n    public static void main(String[] args) throws Exception {\n        DefaultMQPushConsumer defaultMQPushConsumer = new DefaultMQPushConsumer(CONSUMER_GROUP);\n        defaultMQPushConsumer.setNamespaceV2(NAMESPACE);\n        defaultMQPushConsumer.setNamesrvAddr(DEFAULT_NAMESRVADDR);\n        defaultMQPushConsumer.subscribe(TOPIC, \"*\");\n        defaultMQPushConsumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            msgs.forEach(msg -> System.out.printf(\"Msg topic is:%s, MsgId is:%s, reconsumeTimes is:%s%n\", msg.getTopic(), msg.getMsgId(), msg.getReconsumeTimes()));\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n\n        defaultMQPushConsumer.start();\n    }\n}"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/openmessaging/SimpleProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.openmessaging;\n\nimport io.openmessaging.Future;\nimport io.openmessaging.Message;\nimport io.openmessaging.MessagingAccessPoint;\nimport io.openmessaging.OMS;\nimport io.openmessaging.producer.Producer;\nimport io.openmessaging.producer.SendResult;\nimport java.nio.charset.StandardCharsets;\nimport java.util.concurrent.CountDownLatch;\n\npublic class SimpleProducer {\n\n    public static final String URL = \"oms:rocketmq://localhost:9876/default:default\";\n    public static final String QUEUE = \"OMS_HELLO_TOPIC\";\n\n    public static void main(String[] args) {\n        // You need to set the environment variable OMS_RMQ_DIRECT_NAME_SRV=true\n        final MessagingAccessPoint messagingAccessPoint =\n            OMS.getMessagingAccessPoint(URL);\n\n        final Producer producer = messagingAccessPoint.createProducer();\n\n        messagingAccessPoint.startup();\n        System.out.printf(\"MessagingAccessPoint startup OK%n\");\n\n        producer.startup();\n        System.out.printf(\"Producer startup OK%n\");\n\n        {\n            Message message = producer.createBytesMessage(QUEUE, \"OMS_HELLO_BODY\".getBytes(StandardCharsets.UTF_8));\n\n            SendResult sendResult = producer.send(message);\n            //final Void aVoid = result.get(3000L);\n            System.out.printf(\"Send async message OK, msgId: %s%n\", sendResult.messageId());\n        }\n\n        final CountDownLatch countDownLatch = new CountDownLatch(1);\n        {\n            final Future<SendResult> result = producer.sendAsync(producer.createBytesMessage(QUEUE,\n                \"OMS_HELLO_BODY\".getBytes(StandardCharsets.UTF_8)));\n            result.addListener(future -> {\n                if (future.getThrowable() != null) {\n                    System.out.printf(\"Send async message Failed, error: %s%n\", future.getThrowable().getMessage());\n                } else {\n                    System.out.printf(\"Send async message OK, msgId: %s%n\", future.get().messageId());\n                }\n                countDownLatch.countDown();\n            });\n        }\n\n        {\n            producer.sendOneway(producer.createBytesMessage(\"OMS_HELLO_TOPIC\", \"OMS_HELLO_BODY\".getBytes(StandardCharsets.UTF_8)));\n            System.out.printf(\"Send oneway message OK%n\");\n        }\n\n        try {\n            countDownLatch.await();\n            // Wait some time for one-way delivery.\n            Thread.sleep(500);\n        } catch (InterruptedException ignore) {\n        }\n\n        producer.shutdown();\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/openmessaging/SimplePullConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.openmessaging;\n\nimport io.openmessaging.Message;\nimport io.openmessaging.MessagingAccessPoint;\nimport io.openmessaging.OMS;\nimport io.openmessaging.OMSBuiltinKeys;\nimport io.openmessaging.consumer.PullConsumer;\nimport io.openmessaging.producer.Producer;\nimport io.openmessaging.producer.SendResult;\nimport java.nio.charset.StandardCharsets;\n\npublic class SimplePullConsumer {\n\n    public static final String URL = \"oms:rocketmq://localhost:9876/default:default\";\n    public static final String QUEUE = \"OMS_CONSUMER\";\n\n    public static void main(String[] args) {\n        // You need to set the environment variable OMS_RMQ_DIRECT_NAME_SRV=true\n\n        final MessagingAccessPoint messagingAccessPoint =\n            OMS.getMessagingAccessPoint(URL);\n\n        messagingAccessPoint.startup();\n\n        final Producer producer = messagingAccessPoint.createProducer();\n\n        final PullConsumer consumer = messagingAccessPoint.createPullConsumer(\n            OMS.newKeyValue().put(OMSBuiltinKeys.CONSUMER_ID, QUEUE));\n\n        messagingAccessPoint.startup();\n        System.out.printf(\"MessagingAccessPoint startup OK%n\");\n\n        final String queueName = \"TopicTest\";\n\n        producer.startup();\n        Message msg = producer.createBytesMessage(queueName, \"Hello Open Messaging\".getBytes(StandardCharsets.UTF_8));\n        SendResult sendResult = producer.send(msg);\n        System.out.printf(\"Send Message OK. MsgId: %s%n\", sendResult.messageId());\n        producer.shutdown();\n\n        consumer.attachQueue(queueName);\n\n        consumer.startup();\n        System.out.printf(\"Consumer startup OK%n\");\n\n        // Keep running until we find the one that has just been sent\n        boolean stop = false;\n        while (!stop) {\n            Message message = consumer.receive();\n            if (message != null) {\n                String msgId = message.sysHeaders().getString(Message.BuiltinKeys.MESSAGE_ID);\n                System.out.printf(\"Received one message: %s%n\", msgId);\n                consumer.ack(msgId);\n\n                if (!stop) {\n                    stop = msgId.equalsIgnoreCase(sendResult.messageId());\n                }\n\n            } else {\n                System.out.printf(\"Return without any message%n\");\n            }\n        }\n\n        consumer.shutdown();\n        messagingAccessPoint.shutdown();\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/openmessaging/SimplePushConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.openmessaging;\n\nimport io.openmessaging.Message;\nimport io.openmessaging.MessagingAccessPoint;\nimport io.openmessaging.OMS;\nimport io.openmessaging.OMSBuiltinKeys;\nimport io.openmessaging.consumer.PushConsumer;\n\npublic class SimplePushConsumer {\n\n    public static final String URL = \"oms:rocketmq://localhost:9876/default:default\";\n    public static final String QUEUE = \"OMS_HELLO_TOPIC\";\n\n    public static void main(String[] args) {\n        // You need to set the environment variable OMS_RMQ_DIRECT_NAME_SRV=true\n\n        final MessagingAccessPoint messagingAccessPoint = OMS\n            .getMessagingAccessPoint(URL);\n\n        final PushConsumer consumer = messagingAccessPoint.\n            createPushConsumer(OMS.newKeyValue().put(OMSBuiltinKeys.CONSUMER_ID, \"OMS_CONSUMER\"));\n\n        messagingAccessPoint.startup();\n        System.out.printf(\"MessagingAccessPoint startup OK%n\");\n\n        Runtime.getRuntime().addShutdownHook(new Thread(() -> {\n            consumer.shutdown();\n            messagingAccessPoint.shutdown();\n        }));\n\n        consumer.attachQueue(QUEUE, (message, context) -> {\n            System.out.printf(\"Received one message: %s%n\", message.sysHeaders().getString(Message.BuiltinKeys.MESSAGE_ID));\n            context.ack();\n        });\n\n        consumer.startup();\n        System.out.printf(\"Consumer startup OK%n\");\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/operation/Consumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.operation;\n\nimport java.util.List;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.commons.cli.CommandLine;\nimport org.apache.commons.cli.DefaultParser;\nimport org.apache.commons.cli.HelpFormatter;\nimport org.apache.commons.cli.Option;\nimport org.apache.commons.cli.Options;\nimport org.apache.commons.cli.ParseException;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic class Consumer {\n\n    public static void main(String[] args) throws MQClientException {\n        CommandLine commandLine = buildCommandline(args);\n        if (commandLine != null) {\n            String subGroup = commandLine.getOptionValue('g');\n            String topic = commandLine.getOptionValue('t');\n            String subExpression = commandLine.getOptionValue('s');\n            final String returnFailedHalf = commandLine.getOptionValue('f');\n\n            DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(subGroup);\n            consumer.setInstanceName(Long.toString(System.currentTimeMillis()));\n\n            consumer.subscribe(topic, subExpression);\n\n            consumer.registerMessageListener(new MessageListenerConcurrently() {\n                AtomicLong consumeTimes = new AtomicLong(0);\n\n                @Override\n                public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,\n                    ConsumeConcurrentlyContext context) {\n                    long currentTimes = this.consumeTimes.incrementAndGet();\n                    System.out.printf(\"%-8d %s%n\", currentTimes, msgs);\n                    if (Boolean.parseBoolean(returnFailedHalf)) {\n                        if ((currentTimes % 2) == 0) {\n                            return ConsumeConcurrentlyStatus.RECONSUME_LATER;\n                        }\n                    }\n                    return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n                }\n            });\n\n            consumer.start();\n\n            System.out.printf(\"Consumer Started.%n\");\n        }\n    }\n\n    public static CommandLine buildCommandline(String[] args) {\n        final Options options = new Options();\n        Option opt = new Option(\"h\", \"help\", false, \"Print help\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"g\", \"consumerGroup\", true, \"Consumer Group Name\");\n        opt.setRequired(true);\n        options.addOption(opt);\n\n        opt = new Option(\"t\", \"topic\", true, \"Topic Name\");\n        opt.setRequired(true);\n        options.addOption(opt);\n\n        opt = new Option(\"s\", \"subscription\", true, \"subscription\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"f\", \"returnFailedHalf\", true, \"return failed result, for half message\");\n        opt.setRequired(true);\n        options.addOption(opt);\n\n        DefaultParser parser = new DefaultParser();\n        HelpFormatter hf = new HelpFormatter();\n        hf.setWidth(110);\n        CommandLine commandLine = null;\n        try {\n            commandLine = parser.parse(options, args);\n            if (commandLine.hasOption('h')) {\n                hf.printHelp(\"producer\", options, true);\n                return null;\n            }\n        } catch (ParseException e) {\n            hf.printHelp(\"producer\", options, true);\n            return null;\n        }\n\n        return commandLine;\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/operation/Producer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.operation;\n\nimport org.apache.commons.cli.CommandLine;\nimport org.apache.commons.cli.DefaultParser;\nimport org.apache.commons.cli.HelpFormatter;\nimport org.apache.commons.cli.Option;\nimport org.apache.commons.cli.Options;\nimport org.apache.commons.cli.ParseException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\n\npublic class Producer {\n\n    public static void main(String[] args) throws MQClientException, InterruptedException {\n        CommandLine commandLine = buildCommandline(args);\n        if (commandLine != null) {\n            String group = commandLine.getOptionValue('g');\n            String topic = commandLine.getOptionValue('t');\n            String tags = commandLine.getOptionValue('a');\n            String keys = commandLine.getOptionValue('k');\n            String msgCount = commandLine.getOptionValue('c');\n\n            DefaultMQProducer producer = new DefaultMQProducer(group);\n            producer.setInstanceName(Long.toString(System.currentTimeMillis()));\n\n            producer.start();\n\n            for (int i = 0; i < Integer.parseInt(msgCount); i++) {\n                try {\n                    Message msg = new Message(\n                        topic,\n                        tags,\n                        keys,\n                        (\"Hello RocketMQ \" + i).getBytes(RemotingHelper.DEFAULT_CHARSET));\n                    SendResult sendResult = producer.send(msg);\n                    System.out.printf(\"%-8d %s%n\", i, sendResult);\n                } catch (Exception e) {\n                    e.printStackTrace();\n                    Thread.sleep(1000);\n                }\n            }\n\n            producer.shutdown();\n        }\n    }\n\n    public static CommandLine buildCommandline(String[] args) {\n        final Options options = new Options();\n        Option opt = new Option(\"h\", \"help\", false, \"Print help\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"g\", \"producerGroup\", true, \"Producer Group Name\");\n        opt.setRequired(true);\n        options.addOption(opt);\n\n        opt = new Option(\"t\", \"topic\", true, \"Topic Name\");\n        opt.setRequired(true);\n        options.addOption(opt);\n\n        opt = new Option(\"a\", \"tags\", true, \"Tags Name\");\n        opt.setRequired(true);\n        options.addOption(opt);\n\n        opt = new Option(\"k\", \"keys\", true, \"Keys Name\");\n        opt.setRequired(true);\n        options.addOption(opt);\n\n        opt = new Option(\"c\", \"msgCount\", true, \"Message Count\");\n        opt.setRequired(true);\n        options.addOption(opt);\n\n        DefaultParser parser = new DefaultParser();\n        HelpFormatter hf = new HelpFormatter();\n        hf.setWidth(110);\n        CommandLine commandLine = null;\n        try {\n            commandLine = parser.parse(options, args);\n            if (commandLine.hasOption('h')) {\n                hf.printHelp(\"producer\", options, true);\n                return null;\n            }\n        } catch (ParseException e) {\n            hf.printHelp(\"producer\", options, true);\n            return null;\n        }\n\n        return commandLine;\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/ordermessage/Consumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.ordermessage;\n\nimport java.util.List;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic class Consumer {\n\n    public static void main(String[] args) throws MQClientException {\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"please_rename_unique_group_name_3\");\n\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n\n        consumer.subscribe(\"TopicTest\", \"TagA || TagC || TagD\");\n\n        consumer.registerMessageListener(new MessageListenerOrderly() {\n            AtomicLong consumeTimes = new AtomicLong(0);\n\n            @Override\n            public ConsumeOrderlyStatus consumeMessage(List<MessageExt> msgs, ConsumeOrderlyContext context) {\n                context.setAutoCommit(true);\n                System.out.printf(\"%s Receive New Messages: %s %n\", Thread.currentThread().getName(), msgs);\n                this.consumeTimes.incrementAndGet();\n                if ((this.consumeTimes.get() % 2) == 0) {\n                    return ConsumeOrderlyStatus.SUCCESS;\n                } else if ((this.consumeTimes.get() % 5) == 0) {\n                    context.setSuspendCurrentQueueTimeMillis(3000);\n                    return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT;\n                }\n\n                return ConsumeOrderlyStatus.SUCCESS;\n            }\n        });\n\n        consumer.start();\n        System.out.printf(\"Consumer Started.%n\");\n    }\n\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/ordermessage/Producer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.ordermessage;\n\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.MessageQueueSelector;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\n\nimport java.util.List;\n\npublic class Producer {\n    public static void main(String[] args) throws MQClientException {\n        try {\n            DefaultMQProducer producer = new DefaultMQProducer(\"please_rename_unique_group_name\");\n            producer.start();\n\n            String[] tags = new String[] {\"TagA\", \"TagB\", \"TagC\", \"TagD\", \"TagE\"};\n            for (int i = 0; i < 100; i++) {\n                int orderId = i % 10;\n                Message msg =\n                    new Message(\"TopicTestjjj\", tags[i % tags.length], \"KEY\" + i,\n                        (\"Hello RocketMQ \" + i).getBytes(RemotingHelper.DEFAULT_CHARSET));\n                SendResult sendResult = producer.send(msg, new MessageQueueSelector() {\n                    @Override\n                    public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {\n                        Integer id = (Integer) arg;\n                        int index = id % mqs.size();\n                        return mqs.get(index);\n                    }\n                }, orderId);\n\n                System.out.printf(\"%s%n\", sendResult);\n            }\n\n            producer.shutdown();\n        } catch (Exception e) {\n            e.printStackTrace();\n            throw new MQClientException(e.getMessage(), null);\n        }\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/quickstart/Consumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.quickstart;\n\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\n\n/**\n * This example shows how to subscribe and consume messages using providing {@link DefaultMQPushConsumer}.\n */\npublic class Consumer {\n\n    public static final String CONSUMER_GROUP = \"please_rename_unique_group_name_4\";\n    public static final String DEFAULT_NAMESRVADDR = \"127.0.0.1:9876\";\n    public static final String TOPIC = \"TopicTest\";\n\n    public static void main(String[] args) throws MQClientException {\n\n        /*\n         * Instantiate with specified consumer group name.\n         */\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(CONSUMER_GROUP);\n\n        /*\n         * Specify name server addresses.\n         * <p/>\n         *\n         * Alternatively, you may specify name server addresses via exporting environmental variable: NAMESRV_ADDR\n         * <pre>\n         * {@code\n         * consumer.setNamesrvAddr(\"name-server1-ip:9876;name-server2-ip:9876\");\n         * }\n         * </pre>\n         */\n        // Uncomment the following line while debugging, namesrvAddr should be set to your local address\n        // consumer.setNamesrvAddr(DEFAULT_NAMESRVADDR);\n\n        /*\n         * Specify where to start in case the specific consumer group is a brand-new one.\n         */\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n\n        /*\n         * Subscribe one more topic to consume.\n         */\n        consumer.subscribe(TOPIC, \"*\");\n\n        /*\n         *  Register callback to execute on arrival of messages fetched from brokers.\n         */\n        consumer.registerMessageListener((MessageListenerConcurrently) (msg, context) -> {\n            System.out.printf(\"%s Receive New Messages: %s %n\", Thread.currentThread().getName(), msg);\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n\n        /*\n         *  Launch the consumer instance.\n         */\n        consumer.start();\n\n        System.out.printf(\"Consumer Started.%n\");\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/quickstart/Producer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.quickstart;\n\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\n\n/**\n * This class demonstrates how to send messages to brokers using provided {@link DefaultMQProducer}.\n */\npublic class Producer {\n\n    /**\n     * The number of produced messages.\n     */\n    public static final int MESSAGE_COUNT = 1000;\n    public static final String PRODUCER_GROUP = \"please_rename_unique_group_name\";\n    public static final String DEFAULT_NAMESRVADDR = \"127.0.0.1:9876\";\n    public static final String TOPIC = \"TopicTest\";\n    public static final String TAG = \"TagA\";\n\n    public static void main(String[] args) throws MQClientException, InterruptedException {\n\n        /*\n         * Instantiate with a producer group name.\n         */\n        DefaultMQProducer producer = new DefaultMQProducer(PRODUCER_GROUP);\n\n        /*\n         * Specify name server addresses.\n         *\n         * Alternatively, you may specify name server addresses via exporting environmental variable: NAMESRV_ADDR\n         * <pre>\n         * {@code\n         *  producer.setNamesrvAddr(\"name-server1-ip:9876;name-server2-ip:9876\");\n         * }\n         * </pre>\n         */\n        // Uncomment the following line while debugging, namesrvAddr should be set to your local address\n        // producer.setNamesrvAddr(DEFAULT_NAMESRVADDR);\n\n        /*\n         * Launch the instance.\n         */\n        producer.start();\n\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            try {\n\n                /*\n                 * Create a message instance, specifying topic, tag and message body.\n                 */\n                Message msg = new Message(TOPIC /* Topic */,\n                    TAG /* Tag */,\n                    (\"Hello RocketMQ \" + i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */\n                );\n\n                /*\n                 * Call send message to deliver message to one of brokers.\n                 */\n                SendResult sendResult = producer.send(msg, 20 * 1000);\n                /*\n                 * There are different ways to send message, if you don't care about the send result,you can use this way\n                 * {@code\n                 * producer.sendOneway(msg);\n                 * }\n                 */\n\n                /*\n                 * if you want to get the send result in a synchronize way, you can use this send method\n                 * {@code\n                 * SendResult sendResult = producer.send(msg);\n                 * System.out.printf(\"%s%n\", sendResult);\n                 * }\n                 */\n\n                /*\n                 * if you want to get the send result in a asynchronize way, you can use this send method\n                 * {@code\n                 *\n                 *  producer.send(msg, new SendCallback() {\n                 *  @Override\n                 *  public void onSuccess(SendResult sendResult) {\n                 *      // do something\n                 *  }\n                 *\n                 *  @Override\n                 *  public void onException(Throwable e) {\n                 *      // do something\n                 *  }\n                 *});\n                 *\n                 *}\n                 */\n\n                System.out.printf(\"%s%n\", sendResult);\n            } catch (Exception e) {\n                e.printStackTrace();\n                Thread.sleep(1000);\n            }\n        }\n\n        /*\n         * Shut down once the producer instance is no longer in use.\n         */\n        producer.shutdown();\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/rpc/AsyncRequestProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.example.rpc;\n\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.RequestCallback;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class AsyncRequestProducer {\n    private static final Logger log = LoggerFactory.getLogger(AsyncRequestProducer.class);\n\n    public static void main(String[] args) throws MQClientException, InterruptedException {\n        String producerGroup = \"please_rename_unique_group_name\";\n        String topic = \"RequestTopic\";\n        long ttl = 3000;\n\n        DefaultMQProducer producer = new DefaultMQProducer(producerGroup);\n        producer.start();\n\n        try {\n            Message msg = new Message(topic,\n                \"\",\n                \"Hello world\".getBytes(RemotingHelper.DEFAULT_CHARSET));\n\n            long begin = System.currentTimeMillis();\n            producer.request(msg, new RequestCallback() {\n                @Override\n                public void onSuccess(Message message) {\n                    long cost = System.currentTimeMillis() - begin;\n                    System.out.printf(\"request to <%s> cost: %d replyMessage: %s %n\", topic, cost, message);\n                }\n\n                @Override\n                public void onException(Throwable e) {\n                    System.err.printf(\"request to <%s> fail.\", topic);\n                }\n            }, ttl);\n        } catch (Exception e) {\n            log.warn(\"\", e);\n        }\n         /* shutdown after your request callback is finished */\n//        producer.shutdown();\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/rpc/RequestProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.example.rpc;\n\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\n\npublic class RequestProducer {\n    public static void main(String[] args) throws MQClientException, InterruptedException {\n        String producerGroup = \"please_rename_unique_group_name\";\n        String topic = \"RequestTopic\";\n        long ttl = 3000;\n\n        DefaultMQProducer producer = new DefaultMQProducer(producerGroup);\n\n        //You need to set namesrvAddr to the address of the local namesrv\n        producer.setNamesrvAddr(\"127.0.0.1:9876\");\n\n        producer.start();\n\n        try {\n            Message msg = new Message(topic,\n                \"\",\n                \"Hello world\".getBytes(RemotingHelper.DEFAULT_CHARSET));\n\n            long begin = System.currentTimeMillis();\n            Message retMsg = producer.request(msg, ttl);\n            long cost = System.currentTimeMillis() - begin;\n            System.out.printf(\"request to <%s> cost: %d replyMessage: %s %n\", topic, cost, retMsg);\n        } catch (Exception e) {\n            e.printStackTrace();\n        }\n        producer.shutdown();\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.example.rpc;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.List;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.utils.MessageUtil;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\n\npublic class ResponseConsumer {\n    public static void main(String[] args) throws InterruptedException, MQClientException {\n        String producerGroup = \"please_rename_unique_group_name\";\n        String consumerGroup = \"please_rename_unique_group_name\";\n        String topic = \"RequestTopic\";\n\n        // create a producer to send reply message\n        DefaultMQProducer replyProducer = new DefaultMQProducer(producerGroup);\n        replyProducer.setNamesrvAddr(\"127.0.0.1:9876\");\n        replyProducer.start();\n\n        // create consumer\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(consumerGroup);\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);\n        consumer.setNamesrvAddr(\"127.0.0.1:9876\");\n\n        // recommend client configs\n        consumer.setPullTimeDelayMillsWhenException(0L);\n\n        consumer.registerMessageListener(new MessageListenerConcurrently() {\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {\n                System.out.printf(\"%s Receive New Messages: %s %n\", Thread.currentThread().getName(), msgs);\n                for (MessageExt msg : msgs) {\n                    try {\n                        System.out.printf(\"handle message: %s %n\", msg.toString());\n                        String replyTo = MessageUtil.getReplyToClient(msg);\n                        byte[] replyContent = \"reply message contents.\".getBytes(StandardCharsets.UTF_8);\n                        // create reply message with given util, do not create reply message by yourself\n                        Message replyMessage = MessageUtil.createReplyMessage(msg, replyContent);\n\n                        // send reply message with producer\n                        SendResult replyResult = replyProducer.send(replyMessage, 3000);\n                        System.out.printf(\"reply to %s , %s %n\", replyTo, replyResult.toString());\n                    } catch (MQClientException | RemotingException | MQBrokerException | InterruptedException e) {\n                        e.printStackTrace();\n                    }\n                }\n                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n            }\n        });\n\n        consumer.subscribe(topic, \"*\");\n        consumer.start();\n        System.out.printf(\"Consumer Started.%n\");\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/schedule/ScheduledMessageConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.schedule;\n\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic class ScheduledMessageConsumer {\n\n    public static final String CONSUMER_GROUP = \"ExampleConsumer\";\n    public static final String DEFAULT_NAMESRVADDR = \"127.0.0.1:9876\";\n    public static final String TOPIC = \"TestTopic\";\n\n    public static void main(String[] args) throws Exception {\n        // Instantiate message consumer\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(CONSUMER_GROUP);\n\n        // Uncomment the following line while debugging, namesrvAddr should be set to your local address\n//        consumer.setNamesrvAddr(DEFAULT_NAMESRVADDR);\n\n        // Subscribe topics\n        consumer.subscribe(TOPIC, \"*\");\n        // Register message listener\n        consumer.registerMessageListener((MessageListenerConcurrently) (messages, context) -> {\n            for (MessageExt message : messages) {\n                // Print approximate delay time period\n                System.out.printf(\"Receive message[msgId=%s %d  ms later]\\n\", message.getMsgId(),\n                    System.currentTimeMillis() - message.getStoreTimestamp());\n            }\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        // Launch consumer\n        consumer.start();\n        //info:to see the time effect, run the consumer first , it will wait for the msg\n        //then start the producer\n    }\n}"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/schedule/ScheduledMessageProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.schedule;\n\nimport java.nio.charset.StandardCharsets;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.message.Message;\n\npublic class ScheduledMessageProducer {\n\n    public static final String PRODUCER_GROUP = \"ExampleProducerGroup\";\n    public static final String DEFAULT_NAMESRVADDR = \"127.0.0.1:9876\";\n    public static final String TOPIC = \"TestTopic\";\n\n    public static void main(String[] args) throws Exception {\n        // Instantiate a producer to send scheduled messages\n        DefaultMQProducer producer = new DefaultMQProducer(PRODUCER_GROUP);\n\n        // Uncomment the following line while debugging, namesrvAddr should be set to your local address\n//        producer.setNamesrvAddr(DEFAULT_NAMESRVADDR);\n\n        // Launch producer\n        producer.start();\n        int totalMessagesToSend = 100;\n        for (int i = 0; i < totalMessagesToSend; i++) {\n            Message message = new Message(TOPIC, (\"Hello scheduled message \" + i).getBytes(StandardCharsets.UTF_8));\n            // This message will be delivered to consumer 10 seconds later.\n            message.setDelayTimeLevel(3);\n            // Send the message\n            SendResult result = producer.send(message);\n            System.out.print(result);\n        }\n\n        // Shutdown producer after use.\n        producer.shutdown();\n    }\n\n}"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/schedule/TimerMessageConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.schedule;\n\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic class TimerMessageConsumer {\n\n    //Note: TimerMessage is a new feature in version 5.0, so be sure to upgrade RocketMQ to version 5.0+ before using it.\n\n    public static final String CONSUMER_GROUP = \"TimerMessageConsumerGroup\";\n    public static final String DEFAULT_NAMESRVADDR = \"127.0.0.1:9876\";\n    public static final String TOPIC = \"TimerTopic\";\n\n    public static void main(String[] args) throws Exception {\n        // Instantiate message consumer\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(CONSUMER_GROUP);\n\n        // Uncomment the following line while debugging, namesrvAddr should be set to your local address\n//        consumer.setNamesrvAddr(DEFAULT_NAMESRVADDR);\n\n        // Subscribe topics\n        consumer.subscribe(TOPIC, \"*\");\n        // Register message listener\n        consumer.registerMessageListener((MessageListenerConcurrently) (messages, context) -> {\n            for (MessageExt message : messages) {\n                // Print approximate delay time period\n                System.out.printf(\"Receive message[msgId=%s %d  ms later]\\n\", message.getMsgId(),\n                    System.currentTimeMillis() - message.getBornTimestamp());\n            }\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        // Launch consumer\n        consumer.start();\n        //info:to see the time effect, run the consumer first , it will wait for the msg\n        //then start the producer\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/schedule/TimerMessageProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.schedule;\n\nimport java.nio.charset.StandardCharsets;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.message.Message;\n\npublic class TimerMessageProducer {\n\n    //Note: TimerMessage is a new feature in version 5.0, so be sure to upgrade RocketMQ to version 5.0+ before using it.\n\n    public static final String PRODUCER_GROUP = \"TimerMessageProducerGroup\";\n    public static final String DEFAULT_NAMESRVADDR = \"127.0.0.1:9876\";\n    public static final String TOPIC = \"TimerTopic\";\n\n    public static void main(String[] args) throws Exception {\n        // Instantiate a producer to send scheduled messages\n        DefaultMQProducer producer = new DefaultMQProducer(PRODUCER_GROUP);\n\n        // Uncomment the following line while debugging, namesrvAddr should be set to your local address\n//        producer.setNamesrvAddr(DEFAULT_NAMESRVADDR);\n\n        // Launch producer\n        producer.start();\n        int totalMessagesToSend = 10;\n        for (int i = 0; i < totalMessagesToSend; i++) {\n            Message message = new Message(TOPIC, (\"Hello scheduled message \" + i).getBytes(StandardCharsets.UTF_8));\n            // This message will be delivered to consumer 10 seconds later.\n            //message.setDelayTimeSec(10);\n            // The effect is the same as the above\n            // message.setDelayTimeMs(10_000L);\n            // Set the specific delivery time, and the effect is the same as the above\n            message.setDeliverTimeMs(System.currentTimeMillis() + 10_000L);\n            // Send the message\n            SendResult result = producer.send(message);\n            System.out.printf(result + \"\\n\");\n        }\n\n        // Shutdown producer after use.\n        producer.shutdown();\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/simple/AclClient.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.simple;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\n\nimport org.apache.rocketmq.acl.common.AclClientRPCHook;\nimport org.apache.rocketmq.acl.common.SessionCredentials;\nimport org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\n\n\npublic class AclClient {\n\n    private static final Map<MessageQueue, Long> OFFSE_TABLE = new HashMap<>();\n\n    private static final String ACL_ACCESS_KEY = \"RocketMQ\";\n\n    private static final String ACL_SECRET_KEY = \"1234567\";\n\n    public static void main(String[] args) throws MQClientException, InterruptedException {\n        producer();\n        pushConsumer();\n        pullConsumer();\n    }\n\n    public static void producer() throws MQClientException {\n        DefaultMQProducer producer = new DefaultMQProducer(\"ProducerGroupName\", getAclRPCHook());\n        producer.setNamesrvAddr(\"127.0.0.1:9876\");\n        producer.start();\n\n        for (int i = 0; i < 128; i++)\n            try {\n                {\n                    Message msg = new Message(\"TopicTest\",\n                        \"TagA\",\n                        \"OrderID188\",\n                        \"Hello world\".getBytes(RemotingHelper.DEFAULT_CHARSET));\n                    SendResult sendResult = producer.send(msg);\n                    System.out.printf(\"%s%n\", sendResult);\n                }\n\n            } catch (Exception e) {\n                e.printStackTrace();\n            }\n\n        producer.shutdown();\n    }\n\n    public static void pushConsumer() throws MQClientException {\n\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(\"please_rename_unique_group_name_5\", getAclRPCHook(), new AllocateMessageQueueAveragely());\n        consumer.setNamesrvAddr(\"127.0.0.1:9876\");\n        consumer.subscribe(\"TopicTest\", \"*\");\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        // Wrong time format 2017_0422_221800\n        consumer.setConsumeTimestamp(\"20180422221800\");\n        consumer.registerMessageListener(new MessageListenerConcurrently() {\n\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {\n                System.out.printf(\"%s Receive New Messages: %s %n\", Thread.currentThread().getName(), msgs);\n                printBody(msgs);\n                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n            }\n        });\n        consumer.start();\n        System.out.printf(\"Consumer Started.%n\");\n    }\n\n    public static void pullConsumer() throws MQClientException {\n        DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(\"please_rename_unique_group_name_6\", getAclRPCHook());\n        consumer.setNamesrvAddr(\"127.0.0.1:9876\");\n        consumer.start();\n\n        Set<MessageQueue> mqs = consumer.fetchSubscribeMessageQueues(\"TopicTest\");\n        for (MessageQueue mq : mqs) {\n            System.out.printf(\"Consume from the queue: %s%n\", mq);\n            SINGLE_MQ:\n            while (true) {\n                try {\n                    PullResult pullResult =\n                        consumer.pullBlockIfNotFound(mq, null, getMessageQueueOffset(mq), 32);\n                    System.out.printf(\"%s%n\", pullResult);\n                    putMessageQueueOffset(mq, pullResult.getNextBeginOffset());\n                    printBody(pullResult);\n                    switch (pullResult.getPullStatus()) {\n                        case FOUND:\n                            break;\n                        case NO_MATCHED_MSG:\n                            break;\n                        case NO_NEW_MSG:\n                            break SINGLE_MQ;\n                        case OFFSET_ILLEGAL:\n                            break;\n                        default:\n                            break;\n                    }\n                } catch (Exception e) {\n                    e.printStackTrace();\n                }\n            }\n        }\n\n        consumer.shutdown();\n    }\n\n    private static void printBody(PullResult pullResult) {\n        printBody(pullResult.getMsgFoundList());\n    }\n\n    private static void printBody(List<MessageExt> msg) {\n        if (msg == null || msg.size() == 0)\n            return;\n        for (MessageExt m : msg) {\n            if (m != null) {\n                System.out.printf(\"msgId : %s  body : %s  \\n\\r\", m.getMsgId(), new String(m.getBody(), StandardCharsets.UTF_8));\n            }\n        }\n    }\n\n    private static long getMessageQueueOffset(MessageQueue mq) {\n        Long offset = OFFSE_TABLE.get(mq);\n        if (offset != null)\n            return offset;\n\n        return 0;\n    }\n\n    private static void putMessageQueueOffset(MessageQueue mq, long offset) {\n        OFFSE_TABLE.put(mq, offset);\n    }\n\n    static RPCHook getAclRPCHook() {\n        return new AclClientRPCHook(new SessionCredentials(ACL_ACCESS_KEY,ACL_SECRET_KEY));\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/simple/AsyncProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.simple;\n\nimport java.io.UnsupportedEncodingException;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendCallback;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\n\npublic class AsyncProducer {\n    public static void main(\n        String[] args) throws MQClientException, InterruptedException, UnsupportedEncodingException {\n\n        DefaultMQProducer producer = new DefaultMQProducer(\"Jodie_Daily_test\");\n        producer.start();\n        // suggest to on enableBackpressureForAsyncMode in heavy traffic, default is false\n        producer.setEnableBackpressureForAsyncMode(true);\n        producer.setRetryTimesWhenSendAsyncFailed(0);\n\n        int messageCount = 100;\n        final CountDownLatch countDownLatch = new CountDownLatch(messageCount);\n        for (int i = 0; i < messageCount; i++) {\n            try {\n                final int index = i;\n                Message msg = new Message(\"Jodie_topic_1023\",\n                    \"TagA\",\n                    \"OrderID188\",\n                    \"Hello world\".getBytes(RemotingHelper.DEFAULT_CHARSET));\n                producer.send(msg, new SendCallback() {\n                    @Override\n                    public void onSuccess(SendResult sendResult) {\n                        countDownLatch.countDown();\n                        System.out.printf(\"%-10d OK %s %n\", index, sendResult.getMsgId());\n                    }\n\n                    @Override\n                    public void onException(Throwable e) {\n                        countDownLatch.countDown();\n                        System.out.printf(\"%-10d Exception %s %n\", index, e);\n                        e.printStackTrace();\n                    }\n                });\n            } catch (Exception e) {\n                e.printStackTrace();\n            }\n        }\n        countDownLatch.await(5, TimeUnit.SECONDS);\n        producer.shutdown();\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/simple/CachedQueue.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.example.simple;\n\nimport java.util.TreeMap;\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic class CachedQueue {\n    private final TreeMap<Long, MessageExt> msgCachedTable = new TreeMap<>();\n\n    public TreeMap<Long, MessageExt> getMsgCachedTable() {\n        return msgCachedTable;\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerAssign.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.simple;\n\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.List;\nimport org.apache.rocketmq.client.consumer.DefaultLitePullConsumer;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\npublic class LitePullConsumerAssign {\n\n    public static volatile boolean running = true;\n\n    public static void main(String[] args) throws Exception {\n        DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(\"please_rename_unique_group_name\");\n        litePullConsumer.setAutoCommit(false);\n        litePullConsumer.start();\n        Collection<MessageQueue> mqSet = litePullConsumer.fetchMessageQueues(\"TopicTest\");\n        List<MessageQueue> list = new ArrayList<>(mqSet);\n        List<MessageQueue> assignList = new ArrayList<>();\n        for (int i = 0; i < list.size() / 2; i++) {\n            assignList.add(list.get(i));\n        }\n        litePullConsumer.assign(assignList);\n        litePullConsumer.seek(assignList.get(0), 10);\n        try {\n            while (running) {\n                List<MessageExt> messageExts = litePullConsumer.poll();\n                System.out.printf(\"%s %n\", messageExts);\n                litePullConsumer.commit();\n            }\n        } finally {\n            litePullConsumer.shutdown();\n        }\n\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerAssignWithSubExpression.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.simple;\n\nimport org.apache.rocketmq.client.consumer.DefaultLitePullConsumer;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.List;\n\npublic class LitePullConsumerAssignWithSubExpression {\n\n    public static volatile boolean running = true;\n\n    public static void main(String[] args) throws Exception {\n        DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(\"please_rename_unique_group_name\");\n        litePullConsumer.setAutoCommit(false);\n        litePullConsumer.setSubExpressionForAssign(\"TopicTest\", \"TagA\");\n        litePullConsumer.start();\n        Collection<MessageQueue> mqSet = litePullConsumer.fetchMessageQueues(\"TopicTest\");\n        List<MessageQueue> list = new ArrayList<>(mqSet);\n        List<MessageQueue> assignList = new ArrayList<>();\n        for (int i = 0; i < list.size(); i++) {\n            assignList.add(list.get(i));\n        }\n        mqSet = litePullConsumer.fetchMessageQueues(\"TopicTest1\");\n        list = new ArrayList<>(mqSet);\n        for (int i = 0; i < list.size(); i++) {\n            assignList.add(list.get(i));\n        }\n        litePullConsumer.assign(assignList);\n        litePullConsumer.seek(assignList.get(0), 10);\n        try {\n            while (running) {\n                List<MessageExt> messageExts = litePullConsumer.poll();\n                System.out.printf(\"%s %n\", messageExts);\n                litePullConsumer.commit();\n            }\n        } finally {\n            litePullConsumer.shutdown();\n        }\n\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerSubscribe.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.simple;\n\nimport java.util.List;\nimport org.apache.rocketmq.client.consumer.DefaultLitePullConsumer;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic class LitePullConsumerSubscribe {\n\n    public static volatile boolean running = true;\n\n    public static void main(String[] args) throws Exception {\n        DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(\"lite_pull_consumer_test\");\n        litePullConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        litePullConsumer.subscribe(\"TopicTest\", \"*\");\n        litePullConsumer.start();\n        try {\n            while (running) {\n                List<MessageExt> messageExts = litePullConsumer.poll();\n                System.out.printf(\"%s%n\", messageExts);\n            }\n        } finally {\n            litePullConsumer.shutdown();\n        }\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/simple/OnewayProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.simple;\n\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.common.message.Message;\n\nimport java.nio.charset.StandardCharsets;\n\npublic class OnewayProducer {\n    public static void main(String[] args) throws Exception {\n        //Instantiate with a producer group name.\n        DefaultMQProducer producer = new DefaultMQProducer(\"please_rename_unique_group_name\");\n        // Specify name server addresses.\n        producer.setNamesrvAddr(\"localhost:9876\");\n        //Launch the instance.\n        producer.start();\n        for (int i = 0; i < 100; i++) {\n            //Create a message instance, specifying topic, tag and message body.\n            Message msg = new Message(\"TopicTest\" /* Topic */,\n                    \"TagA\" /* Tag */,\n                    (\"Hello RocketMQ \" +\n                            i).getBytes(StandardCharsets.UTF_8) /* Message body */\n            );\n            //Call send message to deliver message to one of brokers.\n            producer.sendOneway(msg);\n        }\n        //Wait for sending to complete\n        Thread.sleep(5000);\n        producer.shutdown();\n    }\n}"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/simple/PopConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.simple;\n\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageRequestMode;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.tools.admin.DefaultMQAdminExt;\n\npublic class PopConsumer {\n    public static final String TOPIC = \"TopicTest\";\n    public static final String CONSUMER_GROUP = \"CID_JODIE_1\";\n    public static final String NAMESRV_ADDR = \"127.0.0.1:9876\";\n    public static void main(String[] args) throws Exception {\n        switchPop();\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(CONSUMER_GROUP);\n        consumer.subscribe(TOPIC, \"*\");\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        consumer.registerMessageListener(new MessageListenerConcurrently() {\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {\n                System.out.printf(\"%s Receive New Messages: %s %n\", Thread.currentThread().getName(), msgs);\n                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n            }\n        });\n        // Uncomment the following line while debugging, namesrvAddr should be set to your local address\n        // consumer.setNamesrvAddr(NAMESRV_ADDR);\n        consumer.setClientRebalance(false);\n        consumer.start();\n        System.out.printf(\"Consumer Started.%n\");\n    }\n    private static void switchPop() throws Exception {\n        DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt();\n        // mqAdminExt.setNamesrvAddr(NAMESRV_ADDR);\n        mqAdminExt.start();\n        List<BrokerData> brokerDatas = mqAdminExt.examineTopicRouteInfo(TOPIC).getBrokerDatas();\n        for (BrokerData brokerData : brokerDatas) {\n            Set<String> brokerAddrs = new HashSet<>(brokerData.getBrokerAddrs().values());\n            for (String brokerAddr : brokerAddrs) {\n                mqAdminExt.setMessageRequestMode(brokerAddr, TOPIC, CONSUMER_GROUP, MessageRequestMode.POP, 8, 3_000);\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/simple/Producer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.simple;\n\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.message.Message;\nimport java.nio.charset.StandardCharsets;\n\npublic class Producer {\n\n    public static final String PRODUCER_GROUP = \"ProducerGroupName\";\n    public static final String DEFAULT_NAMESRVADDR = \"127.0.0.1:9876\";\n    public static final String TOPIC = \"TopicTest\";\n    public static final String TAG = \"TagA\";\n\n    public static void main(String[] args) throws MQClientException, InterruptedException {\n\n        DefaultMQProducer producer = new DefaultMQProducer(PRODUCER_GROUP);\n\n        // Uncomment the following line while debugging, namesrvAddr should be set to your local address\n        //producer.setNamesrvAddr(DEFAULT_NAMESRVADDR);\n\n        producer.start();\n        for (int i = 0; i < 128; i++) {\n            try {\n                Message msg = new Message(TOPIC, TAG, \"OrderID188\", \"Hello world\".getBytes(StandardCharsets.UTF_8));\n                SendResult sendResult = producer.send(msg);\n                System.out.printf(\"%s%n\", sendResult);\n            } catch (Exception e) {\n                e.printStackTrace();\n            }\n        }\n\n        producer.shutdown();\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/simple/PullConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.simple;\n\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\n\nimport org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.store.ReadOffsetType;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\n\n@SuppressWarnings(\"deprecation\")\npublic class PullConsumer {\n\n    public static void main(String[] args) throws MQClientException {\n\n        DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(\"please_rename_unique_group_name_5\");\n        consumer.setNamesrvAddr(\"127.0.0.1:9876\");\n        Set<String> topics = new HashSet<>();\n        //You would be better to register topics,It will use in rebalance when starting\n        topics.add(\"TopicTest\");\n        consumer.setRegisterTopics(topics);\n        consumer.start();\n\n        ExecutorService executors = Executors.newFixedThreadPool(topics.size(), new ThreadFactoryImpl(\"PullConsumerThread\"));\n        for (String topic : consumer.getRegisterTopics()) {\n\n            executors.execute(new Runnable() {\n\n                public void doSomething(List<MessageExt> msgs) {\n                    //do your business\n\n                }\n\n                @Override\n                public void run() {\n                    while (true) {\n                        try {\n                            Set<MessageQueue> messageQueues = consumer.fetchMessageQueuesInBalance(topic);\n                            if (messageQueues == null || messageQueues.isEmpty()) {\n                                Thread.sleep(1000);\n                                continue;\n                            }\n                            PullResult pullResult = null;\n                            for (MessageQueue messageQueue : messageQueues) {\n                                try {\n                                    long offset = this.consumeFromOffset(messageQueue);\n                                    pullResult = consumer.pull(messageQueue, \"*\", offset, 32);\n                                    switch (pullResult.getPullStatus()) {\n                                        case FOUND:\n                                            List<MessageExt> msgs = pullResult.getMsgFoundList();\n\n                                            if (msgs != null && !msgs.isEmpty()) {\n                                                this.doSomething(msgs);\n                                                //update offset to local memory, eventually to broker\n                                                consumer.updateConsumeOffset(messageQueue, pullResult.getNextBeginOffset());\n                                                //print pull tps\n                                                this.incPullTPS(topic, pullResult.getMsgFoundList().size());\n                                            }\n                                            break;\n                                        case OFFSET_ILLEGAL:\n                                            consumer.updateConsumeOffset(messageQueue, pullResult.getNextBeginOffset());\n                                            break;\n                                        case NO_NEW_MSG:\n                                            Thread.sleep(1);\n                                            consumer.updateConsumeOffset(messageQueue, pullResult.getNextBeginOffset());\n                                            break;\n                                        case NO_MATCHED_MSG:\n                                            consumer.updateConsumeOffset(messageQueue, pullResult.getNextBeginOffset());\n                                            break;\n                                        default:\n                                    }\n                                } catch (RemotingException e) {\n                                    e.printStackTrace();\n                                } catch (MQBrokerException e) {\n                                    e.printStackTrace();\n                                } catch (Exception e) {\n                                    e.printStackTrace();\n                                }\n                            }\n                        } catch (MQClientException e) {\n                            //reblance error\n                            e.printStackTrace();\n                        } catch (InterruptedException e) {\n                            e.printStackTrace();\n                        } catch (Exception e) {\n                            e.printStackTrace();\n                        }\n                    }\n                }\n\n                public long consumeFromOffset(MessageQueue messageQueue) throws MQClientException {\n                    //-1 when started\n                    long offset = consumer.getOffsetStore().readOffset(messageQueue, ReadOffsetType.READ_FROM_MEMORY);\n                    if (offset < 0) {\n                        //query from broker\n                        offset = consumer.getOffsetStore().readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE);\n                    }\n                    if (offset < 0) {\n                        //first time start from last offset\n                        offset = consumer.maxOffset(messageQueue);\n                    }\n                    //make sure\n                    if (offset < 0) {\n                        offset = 0;\n                    }\n                    return offset;\n                }\n\n                public void incPullTPS(String topic, int pullSize) {\n                    consumer.getDefaultMQPullConsumerImpl().getRebalanceImpl().getmQClientFactory()\n                        .getConsumerStatsManager().incPullTPS(consumer.getConsumerGroup(), topic, pullSize);\n                }\n            });\n\n        }\n//        executors.shutdown();\n//        consumer.shutdown();\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/simple/PullScheduleService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.example.simple;\n\nimport org.apache.rocketmq.client.consumer.MQPullConsumer;\nimport org.apache.rocketmq.client.consumer.MQPullConsumerScheduleService;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullTaskCallback;\nimport org.apache.rocketmq.client.consumer.PullTaskContext;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\n\npublic class PullScheduleService {\n\n    public static void main(String[] args) throws MQClientException {\n        final MQPullConsumerScheduleService scheduleService = new MQPullConsumerScheduleService(\"GroupName1\");\n\n        scheduleService.setMessageModel(MessageModel.CLUSTERING);\n        scheduleService.registerPullTaskCallback(\"TopicTest\", new PullTaskCallback() {\n\n            @Override\n            public void doPullTask(MessageQueue mq, PullTaskContext context) {\n                MQPullConsumer consumer = context.getPullConsumer();\n                try {\n\n                    long offset = consumer.fetchConsumeOffset(mq, false);\n                    if (offset < 0)\n                        offset = 0;\n\n                    PullResult pullResult = consumer.pull(mq, \"*\", offset, 32);\n                    System.out.printf(\"%s%n\", offset + \"\\t\" + mq + \"\\t\" + pullResult);\n                    switch (pullResult.getPullStatus()) {\n                        case FOUND:\n                            break;\n                        case NO_MATCHED_MSG:\n                            break;\n                        case NO_NEW_MSG:\n                        case OFFSET_ILLEGAL:\n                            break;\n                        default:\n                            break;\n                    }\n                    consumer.updateConsumeOffset(mq, pullResult.getNextBeginOffset());\n\n                    context.setPullNextDelayTimeMillis(100);\n                } catch (Exception e) {\n                    e.printStackTrace();\n                }\n            }\n        });\n\n        scheduleService.start();\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/simple/PushConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.simple;\n\nimport java.util.List;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic class PushConsumer {\n    public static final String TOPIC = \"TopicTest\";\n    public static final String CONSUMER_GROUP = \"CID_JODIE_1\";\n    public static final String NAMESRV_ADDR = \"127.0.0.1:9876\";\n    public static void main(String[] args) throws InterruptedException, MQClientException {\n\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(CONSUMER_GROUP);\n\n        // Uncomment the following line while debugging, namesrvAddr should be set to your local address\n//        consumer.setNamesrvAddr(NAMESRV_ADDR);\n\n        consumer.subscribe(TOPIC, \"*\");\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        //wrong time format 2017_0422_221800\n        consumer.setConsumeTimestamp(\"20181109221800\");\n        consumer.registerMessageListener(new MessageListenerConcurrently() {\n\n            @Override\n            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {\n                System.out.printf(\"%s Receive New Messages: %s %n\", Thread.currentThread().getName(), msgs);\n                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n            }\n        });\n        consumer.start();\n        System.out.printf(\"Consumer Started.%n\");\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/simple/RandomAsyncCommit.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.example.simple;\n\nimport java.util.List;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\npublic class RandomAsyncCommit {\n    private final ConcurrentHashMap<MessageQueue, CachedQueue> mqCachedTable =\n        new ConcurrentHashMap<>();\n\n    public void putMessages(final MessageQueue mq, final List<MessageExt> msgs) {\n        CachedQueue cachedQueue = this.mqCachedTable.get(mq);\n        if (null == cachedQueue) {\n            cachedQueue = new CachedQueue();\n            this.mqCachedTable.put(mq, cachedQueue);\n        }\n        for (MessageExt msg : msgs) {\n            cachedQueue.getMsgCachedTable().put(msg.getQueueOffset(), msg);\n        }\n    }\n\n    public void removeMessage(final MessageQueue mq, long offset) {\n        CachedQueue cachedQueue = this.mqCachedTable.get(mq);\n        if (null != cachedQueue) {\n            cachedQueue.getMsgCachedTable().remove(offset);\n        }\n    }\n\n    public long commitableOffset(final MessageQueue mq) {\n        CachedQueue cachedQueue = this.mqCachedTable.get(mq);\n        if (null != cachedQueue) {\n            return cachedQueue.getMsgCachedTable().firstKey();\n        }\n\n        return -1;\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/tracemessage/OpenTracingProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.example.tracemessage;\n\nimport io.jaegertracing.Configuration;\nimport io.jaegertracing.internal.samplers.ConstSampler;\nimport io.opentracing.Tracer;\nimport io.opentracing.util.GlobalTracer;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.trace.hook.SendMessageOpenTracingHookImpl;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\n\npublic class OpenTracingProducer {\n\n    public static final String PRODUCER_GROUP = \"ProducerGroupName\";\n    public static final String DEFAULT_NAMESRVADDR = \"127.0.0.1:9876\";\n    public static final String TOPIC = \"TopicTest\";\n    public static final String TAG = \"TagA\";\n    public static final String KEY = \"OrderID188\";\n\n    public static void main(String[] args) throws MQClientException {\n\n        Tracer tracer = initTracer();\n\n        DefaultMQProducer producer = new DefaultMQProducer(PRODUCER_GROUP);\n\n        // Uncomment the following line while debugging, namesrvAddr should be set to your local address\n//        producer.setNamesrvAddr(DEFAULT_NAMESRVADDR);\n        producer.getDefaultMQProducerImpl().registerSendMessageHook(new SendMessageOpenTracingHookImpl(tracer));\n        producer.start();\n\n        try {\n            Message msg = new Message(TOPIC,\n                TAG,\n                KEY,\n                \"Hello world\".getBytes(RemotingHelper.DEFAULT_CHARSET));\n            SendResult sendResult = producer.send(msg);\n            System.out.printf(\"%s%n\", sendResult);\n\n        } catch (Exception e) {\n            e.printStackTrace();\n        }\n\n        producer.shutdown();\n    }\n\n    private static Tracer initTracer() {\n        Configuration.SamplerConfiguration samplerConfig = Configuration.SamplerConfiguration.fromEnv()\n            .withType(ConstSampler.TYPE)\n            .withParam(1);\n        Configuration.ReporterConfiguration reporterConfig = Configuration.ReporterConfiguration.fromEnv()\n            .withLogSpans(true);\n\n        Configuration config = new Configuration(\"rocketmq\")\n            .withSampler(samplerConfig)\n            .withReporter(reporterConfig);\n        GlobalTracer.registerIfAbsent(config.getTracer());\n        return config.getTracer();\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/tracemessage/OpenTracingPushConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.example.tracemessage;\n\nimport io.jaegertracing.Configuration;\nimport io.jaegertracing.internal.samplers.ConstSampler;\nimport io.opentracing.Tracer;\nimport io.opentracing.util.GlobalTracer;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.trace.hook.ConsumeMessageOpenTracingHookImpl;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\n\npublic class OpenTracingPushConsumer {\n\n    public static final String CONSUMER_GROUP = \"CID_JODIE_1\";\n    public static final String DEFAULT_NAMESRVADDR = \"127.0.0.1:9876\";\n    public static final String TOPIC = \"TopicTest\";\n\n    public static void main(String[] args) throws InterruptedException, MQClientException {\n        Tracer tracer = initTracer();\n\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(CONSUMER_GROUP);\n\n        // Uncomment the following line while debugging, namesrvAddr should be set to your local address\n//        consumer.setNamesrvAddr(DEFAULT_NAMESRVADDR);\n        consumer.registerConsumeMessageHook(new ConsumeMessageOpenTracingHookImpl(tracer));\n\n        consumer.subscribe(TOPIC, \"*\");\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n\n        consumer.setConsumeTimestamp(\"20181109221800\");\n        consumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            System.out.printf(\"%s Receive New Messages: %s %n\", Thread.currentThread().getName(), msgs);\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        consumer.start();\n        System.out.printf(\"Consumer Started.%n\");\n    }\n\n    private static Tracer initTracer() {\n        Configuration.SamplerConfiguration samplerConfig = Configuration.SamplerConfiguration.fromEnv()\n            .withType(ConstSampler.TYPE)\n            .withParam(1);\n        Configuration.ReporterConfiguration reporterConfig = Configuration.ReporterConfiguration.fromEnv()\n            .withLogSpans(true);\n\n        Configuration config = new Configuration(\"rocketmq\")\n            .withSampler(samplerConfig)\n            .withReporter(reporterConfig);\n        GlobalTracer.registerIfAbsent(config.getTracer());\n        return config.getTracer();\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/tracemessage/OpenTracingTransactionProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.example.tracemessage;\n\nimport io.jaegertracing.Configuration;\nimport io.jaegertracing.internal.samplers.ConstSampler;\nimport io.opentracing.Tracer;\nimport io.opentracing.util.GlobalTracer;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.LocalTransactionState;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.TransactionListener;\nimport org.apache.rocketmq.client.producer.TransactionMQProducer;\nimport org.apache.rocketmq.client.trace.hook.EndTransactionOpenTracingHookImpl;\nimport org.apache.rocketmq.client.trace.hook.SendMessageOpenTracingHookImpl;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\n\nimport java.io.UnsupportedEncodingException;\n\npublic class OpenTracingTransactionProducer {\n\n    public static final String PRODUCER_GROUP = \"please_rename_unique_group_name\";\n    public static final String DEFAULT_NAMESRVADDR = \"127.0.0.1:9876\";\n    public static final String TOPIC = \"TopicTest\";\n    public static final String TAG = \"Tag\";\n    public static final String KEY = \"KEY\";\n    public static final int MESSAGE_COUNT = 100000;\n\n    public static void main(String[] args) throws MQClientException, InterruptedException {\n        Tracer tracer = initTracer();\n\n        TransactionMQProducer producer = new TransactionMQProducer(PRODUCER_GROUP);\n\n        // Uncomment the following line while debugging, namesrvAddr should be set to your local address\n//        producer.setNamesrvAddr(DEFAULT_NAMESRVADDR);\n        producer.getDefaultMQProducerImpl().registerSendMessageHook(new SendMessageOpenTracingHookImpl(tracer));\n        producer.getDefaultMQProducerImpl().registerEndTransactionHook(new EndTransactionOpenTracingHookImpl(tracer));\n\n        producer.setTransactionListener(new TransactionListener() {\n            @Override\n            public LocalTransactionState executeLocalTransaction(Message msg, Object arg) {\n                return LocalTransactionState.COMMIT_MESSAGE;\n            }\n\n            @Override\n            public LocalTransactionState checkLocalTransaction(MessageExt msg) {\n                return LocalTransactionState.COMMIT_MESSAGE;\n            }\n        });\n        producer.start();\n\n        try {\n            Message msg = new Message(TOPIC, TAG, KEY,\n                \"Hello RocketMQ\".getBytes(RemotingHelper.DEFAULT_CHARSET));\n            SendResult sendResult = producer.sendMessageInTransaction(msg, null);\n            System.out.printf(\"%s%n\", sendResult);\n        } catch (MQClientException | UnsupportedEncodingException e) {\n            e.printStackTrace();\n        }\n\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            Thread.sleep(1000);\n        }\n        producer.shutdown();\n    }\n\n    private static Tracer initTracer() {\n        Configuration.SamplerConfiguration samplerConfig = Configuration.SamplerConfiguration.fromEnv()\n            .withType(ConstSampler.TYPE)\n            .withParam(1);\n        Configuration.ReporterConfiguration reporterConfig = Configuration.ReporterConfiguration.fromEnv()\n            .withLogSpans(true);\n\n        Configuration config = new Configuration(\"rocketmq\")\n            .withSampler(samplerConfig)\n            .withReporter(reporterConfig);\n        GlobalTracer.registerIfAbsent(config.getTracer());\n        return config.getTracer();\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/tracemessage/TraceProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.example.tracemessage;\n\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\n\npublic class TraceProducer {\n\n    public static final String PRODUCER_GROUP = \"ProducerGroupName\";\n    public static final String DEFAULT_NAMESRVADDR = \"127.0.0.1:9876\";\n    public static final String TOPIC = \"TopicTest\";\n    public static final String TAG = \"TagA\";\n    public static final String KEY = \"OrderID188\";\n    public static final int MESSAGE_COUNT = 128;\n\n    public static void main(String[] args) throws MQClientException, InterruptedException {\n\n        DefaultMQProducer producer = new DefaultMQProducer(PRODUCER_GROUP, true, null);\n\n        // Uncomment the following line while debugging, namesrvAddr should be set to your local address\n//        producer.setNamesrvAddr(DEFAULT_NAMESRVADDR);\n        producer.start();\n\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            try {\n                {\n                    Message msg = new Message(TOPIC,\n                        TAG,\n                        KEY,\n                        \"Hello world\".getBytes(RemotingHelper.DEFAULT_CHARSET));\n                    SendResult sendResult = producer.send(msg);\n                    System.out.printf(\"%s%n\", sendResult);\n                }\n\n            } catch (Exception e) {\n                e.printStackTrace();\n            }\n        }\n\n        producer.shutdown();\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/tracemessage/TracePushConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.example.tracemessage;\n\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\n\npublic class TracePushConsumer {\n\n    public static final String CONSUMER_GROUP = \"ProducerGroupName\";\n    public static final String DEFAULT_NAMESRVADDR = \"127.0.0.1:9876\";\n    public static final String TOPIC = \"TopicTest\";\n\n    public static void main(String[] args) throws InterruptedException, MQClientException {\n        // Here,we use the default message track trace topic name\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(CONSUMER_GROUP, true, null);\n\n        // Uncomment the following line while debugging, namesrvAddr should be set to your local address\n//        consumer.setNamesrvAddr(DEFAULT_NAMESRVADDR);\n        consumer.subscribe(TOPIC, \"*\");\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        // Wrong time format 2017_0422_221800\n        consumer.setConsumeTimestamp(\"20181109221800\");\n        consumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            System.out.printf(\"%s Receive New Messages: %s %n\", Thread.currentThread().getName(), msgs);\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        consumer.start();\n        System.out.printf(\"Consumer Started.%n\");\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/transaction/TransactionListenerImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.transaction;\n\nimport org.apache.rocketmq.client.producer.LocalTransactionState;\nimport org.apache.rocketmq.client.producer.TransactionListener;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageExt;\n\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.atomic.AtomicInteger;\n\npublic class TransactionListenerImpl implements TransactionListener {\n    private AtomicInteger transactionIndex = new AtomicInteger(0);\n\n    private ConcurrentHashMap<String, Integer> localTrans = new ConcurrentHashMap<>();\n\n    @Override\n    public LocalTransactionState executeLocalTransaction(Message msg, Object arg) {\n        int value = transactionIndex.getAndIncrement();\n        int status = value % 3;\n        localTrans.put(msg.getTransactionId(), status);\n        return LocalTransactionState.UNKNOW;\n    }\n\n    @Override\n    public LocalTransactionState checkLocalTransaction(MessageExt msg) {\n        Integer status = localTrans.get(msg.getTransactionId());\n        if (null != status) {\n            switch (status) {\n                case 0:\n                    return LocalTransactionState.UNKNOW;\n                case 1:\n                    return LocalTransactionState.COMMIT_MESSAGE;\n                case 2:\n                    return LocalTransactionState.ROLLBACK_MESSAGE;\n                default:\n                    return LocalTransactionState.COMMIT_MESSAGE;\n            }\n        }\n        return LocalTransactionState.COMMIT_MESSAGE;\n    }\n}\n"
  },
  {
    "path": "example/src/main/java/org/apache/rocketmq/example/transaction/TransactionProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.example.transaction;\n\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.TransactionListener;\nimport org.apache.rocketmq.client.producer.TransactionMQProducer;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\n\nimport java.io.UnsupportedEncodingException;\nimport java.util.Arrays;\nimport java.util.concurrent.ArrayBlockingQueue;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\n\npublic class TransactionProducer {\n\n    public static final String PRODUCER_GROUP = \"please_rename_unique_group_name\";\n    public static final String DEFAULT_NAMESRVADDR = \"127.0.0.1:9876\";\n    public static final String TOPIC = \"TopicTest1234\";\n\n    public static final int MESSAGE_COUNT = 10;\n\n    public static void main(String[] args) throws MQClientException, InterruptedException {\n        TransactionListener transactionListener = new TransactionListenerImpl();\n        TransactionMQProducer producer = new TransactionMQProducer(PRODUCER_GROUP, Arrays.asList(TOPIC));\n\n        // Uncomment the following line while debugging, namesrvAddr should be set to your local address\n//        producer.setNamesrvAddr(DEFAULT_NAMESRVADDR);\n        ExecutorService executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<>(2000), r -> {\n            Thread thread = new Thread(r);\n            thread.setName(\"client-transaction-msg-check-thread\");\n            return thread;\n        });\n\n        producer.setExecutorService(executorService);\n        producer.setTransactionListener(transactionListener);\n        producer.start();\n\n        String[] tags = new String[] {\"TagA\", \"TagB\", \"TagC\", \"TagD\", \"TagE\"};\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            try {\n                Message msg =\n                    new Message(TOPIC, tags[i % tags.length], \"KEY\" + i,\n                        (\"Hello RocketMQ \" + i).getBytes(RemotingHelper.DEFAULT_CHARSET));\n                SendResult sendResult = producer.sendMessageInTransaction(msg, null);\n                System.out.printf(\"%s%n\", sendResult);\n\n                Thread.sleep(10);\n            } catch (MQClientException | UnsupportedEncodingException e) {\n                e.printStackTrace();\n            }\n        }\n\n        for (int i = 0; i < 100000; i++) {\n            Thread.sleep(1000);\n        }\n        producer.shutdown();\n    }\n}\n"
  },
  {
    "path": "filter/BUILD.bazel",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\nload(\"//bazel:GenTestRules.bzl\", \"GenTestRules\")\n\njava_library(\n    name = \"filter\",\n    srcs = glob([\"src/main/java/**/*.java\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \"//common\",\n        \"@maven//:org_apache_commons_commons_lang3\",\n        \"@maven//:commons_validator_commons_validator\",\n        \"@maven//:com_github_luben_zstd_jni\",\n        \"@maven//:org_lz4_lz4_java\",\n        \"@maven//:io_netty_netty_all\",\n        \"@maven//:com_google_guava_guava\",\n    ],\n)\n\njava_library(\n    name = \"tests\",\n    srcs = glob([\"src/test/java/**/*.java\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \":filter\",\n        \"//common\",\n        \"//remoting\",\n        \"//:test_deps\",\n        \"@maven//:org_apache_commons_commons_lang3\", \n        \"@maven//:io_netty_netty_all\",               \n    ],\n    resources = glob([\"src/test/resources/certs/*.pem\"]) + glob([\"src/test/resources/certs/*.key\"])\n)\n\nGenTestRules(\n    name = \"GeneratedTestRules\",\n    test_files = glob([\"src/test/java/**/*Test.java\"]),\n    deps = [\n        \":tests\",\n    ],\n)\n"
  },
  {
    "path": "filter/pom.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one or more\n  ~ contributor license agreements.  See the NOTICE file distributed with\n  ~ this work for additional information regarding copyright ownership.\n  ~ The ASF licenses this file to You under the Apache License, Version 2.0\n  ~ (the \"License\"); you may not use this file except in compliance with\n  ~ the License.  You may obtain a copy of the License at\n  ~\n  ~     http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing, software\n  ~ distributed under the License is distributed on an \"AS IS\" BASIS,\n  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  ~ See the License for the specific language governing permissions and\n  ~ limitations under the License.\n  -->\n\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n    <parent>\n        <artifactId>rocketmq-all</artifactId>\n        <groupId>org.apache.rocketmq</groupId>\n        <version>${revision}</version>\n    </parent>\n\n    <modelVersion>4.0.0</modelVersion>\n    <packaging>jar</packaging>\n    <artifactId>rocketmq-filter</artifactId>\n    <name>rocketmq-filter ${project.version}</name>\n\n    <properties>\n        <project.root>${basedir}/..</project.root>\n    </properties>\n\n    <dependencies>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-common</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-srvutil</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>com.google.guava</groupId>\n            <artifactId>guava</artifactId>\n        </dependency>\n    </dependencies>\n</project>\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/FilterFactory.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter;\n\nimport java.util.HashMap;\nimport java.util.Map;\n\n/**\n * Filter factory: support other filter to register.\n */\npublic class FilterFactory {\n\n    public static final FilterFactory INSTANCE = new FilterFactory();\n\n    protected static final Map<String, FilterSpi> FILTER_SPI_HOLDER = new HashMap<>(4);\n\n    static {\n        FilterFactory.INSTANCE.register(new SqlFilter());\n    }\n\n    /**\n     * Register a filter.\n     * <br>\n     * Note:\n     * <li>1. Filter registered will be used in broker server, so take care of it's reliability and performance.</li>\n     */\n    public void register(FilterSpi filterSpi) {\n        if (FILTER_SPI_HOLDER.containsKey(filterSpi.ofType())) {\n            throw new IllegalArgumentException(String.format(\"Filter spi type(%s) already exist!\", filterSpi.ofType()));\n        }\n\n        FILTER_SPI_HOLDER.put(filterSpi.ofType(), filterSpi);\n    }\n\n    /**\n     * Un register a filter.\n     */\n    public FilterSpi unRegister(String type) {\n        return FILTER_SPI_HOLDER.remove(type);\n    }\n\n    /**\n     * Get a filter registered, null if none exist.\n     */\n    public FilterSpi get(String type) {\n        return FILTER_SPI_HOLDER.get(type);\n    }\n\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/FilterSpi.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter;\n\nimport org.apache.rocketmq.filter.expression.Expression;\nimport org.apache.rocketmq.filter.expression.MQFilterException;\n\n/**\n * Filter spi interface.\n */\npublic interface FilterSpi {\n\n    /**\n     * Compile.\n     */\n    Expression compile(final String expr) throws MQFilterException;\n\n    /**\n     * Which type.\n     */\n    String ofType();\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/SqlFilter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter;\n\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.filter.expression.Expression;\nimport org.apache.rocketmq.filter.expression.MQFilterException;\nimport org.apache.rocketmq.filter.parser.SelectorParser;\n\n/**\n * SQL92 Filter, just a wrapper of {@link org.apache.rocketmq.filter.parser.SelectorParser}.\n * <p/>\n * <p>\n * Do not use this filter directly.Use {@link FilterFactory#get} to select a filter.\n * </p>\n */\npublic class SqlFilter implements FilterSpi {\n\n    @Override\n    public Expression compile(final String expr) throws MQFilterException {\n        return SelectorParser.parse(expr);\n    }\n\n    @Override\n    public String ofType() {\n        return ExpressionType.SQL92;\n    }\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/constant/UnaryType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter.constant;\n\npublic enum UnaryType {\n    NEGATE,\n    IN,\n    NOT,\n    BOOLEANCAST,\n    LIKE\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/expression/BinaryExpression.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter.expression;\n\n/**\n * An expression which performs an operation on two expression values.\n * <p>\n * This class was taken from ActiveMQ org.apache.activemq.filter.BinaryExpression,\n * </p>\n */\npublic abstract class BinaryExpression implements Expression {\n    protected Expression left;\n    protected Expression right;\n\n    public BinaryExpression(Expression left, Expression right) {\n        this.left = left;\n        this.right = right;\n    }\n\n    public Expression getLeft() {\n        return left;\n    }\n\n    public Expression getRight() {\n        return right;\n    }\n\n    /**\n     * @see Object#toString()\n     */\n    public String toString() {\n        return \"(\" + left.toString() + \" \" + getExpressionSymbol() + \" \" + right.toString() + \")\";\n    }\n\n    /**\n     * @see Object#hashCode()\n     */\n    public int hashCode() {\n        return toString().hashCode();\n    }\n\n    /**\n     * @see Object#equals(Object)\n     */\n    public boolean equals(Object o) {\n\n        if (o == null || !this.getClass().equals(o.getClass())) {\n            return false;\n        }\n        return toString().equals(o.toString());\n\n    }\n\n    /**\n     * Returns the symbol that represents this binary expression.  For example, addition is\n     * represented by \"+\"\n     */\n    public abstract String getExpressionSymbol();\n\n    /**\n     * @param expression\n     */\n    public void setRight(Expression expression) {\n        right = expression;\n    }\n\n    /**\n     * @param expression\n     */\n    public void setLeft(Expression expression) {\n        left = expression;\n    }\n\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/expression/BooleanConstantExpression.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter.expression;\n\n/**\n * BooleanConstantExpression\n */\npublic class BooleanConstantExpression extends ConstantExpression implements BooleanExpression {\n\n    public static final BooleanConstantExpression NULL = new BooleanConstantExpression(null);\n    public static final BooleanConstantExpression TRUE = new BooleanConstantExpression(Boolean.TRUE);\n    public static final BooleanConstantExpression FALSE = new BooleanConstantExpression(Boolean.FALSE);\n\n    public BooleanConstantExpression(Object value) {\n        super(value);\n    }\n\n    public boolean matches(EvaluationContext context) throws Exception {\n        Object object = evaluate(context);\n        return object != null && object == Boolean.TRUE;\n    }\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/expression/BooleanExpression.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter.expression;\n\n/**\n * A BooleanExpression is an expression that always\n * produces a Boolean result.\n * <p>\n * This class was taken from ActiveMQ org.apache.activemq.filter.BooleanExpression,\n * but the parameter is changed to an interface.\n * </p>\n *\n * @see org.apache.rocketmq.filter.expression.EvaluationContext\n */\npublic interface BooleanExpression extends Expression {\n\n    /**\n     * @return true if the expression evaluates to Boolean.TRUE.\n     */\n    boolean matches(EvaluationContext context) throws Exception;\n\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/expression/ComparisonExpression.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter.expression;\n\nimport java.util.List;\n\n/**\n * A filter performing a comparison of two objects\n * <p>\n * This class was taken from ActiveMQ org.apache.activemq.filter.ComparisonExpression,\n * but:\n * 1. Remove LIKE expression, and related methods;\n * 2. Extract a new method __compare which has int return value;\n * 3. When create between expression, check whether left value is less or equal than right value;\n * 4. For string type value(can not convert to number), only equal or unequal comparison are supported.\n * </p>\n */\npublic abstract class ComparisonExpression extends BinaryExpression implements BooleanExpression {\n\n    public static final ThreadLocal<Boolean> CONVERT_STRING_EXPRESSIONS = new ThreadLocal<>();\n\n    boolean convertStringExpressions = false;\n\n    /**\n     * @param left\n     * @param right\n     */\n    public ComparisonExpression(Expression left, Expression right) {\n        super(left, right);\n        convertStringExpressions = CONVERT_STRING_EXPRESSIONS.get() != null;\n    }\n\n    public static BooleanExpression createBetween(Expression value, Expression left, Expression right) {\n        // check\n        if (left instanceof ConstantExpression && right instanceof ConstantExpression) {\n            Object lv = ((ConstantExpression) left).getValue();\n            Object rv = ((ConstantExpression) right).getValue();\n            if (lv == null || rv == null) {\n                throw new RuntimeException(\"Illegal values of between, values can not be null!\");\n            }\n            if (lv instanceof Comparable && rv instanceof Comparable) {\n                int ret = __compare((Comparable) rv, (Comparable) lv, true);\n                if (ret < 0)\n                    throw new RuntimeException(\n                        String.format(\"Illegal values of between, left value(%s) must less than or equal to right value(%s)\", lv, rv)\n                    );\n            }\n        }\n\n        return LogicExpression.createAND(createGreaterThanEqual(value, left), createLessThanEqual(value, right));\n    }\n\n    public static BooleanExpression createNotBetween(Expression value, Expression left, Expression right) {\n        return LogicExpression.createOR(createLessThan(value, left), createGreaterThan(value, right));\n    }\n\n    static class ContainsExpression extends UnaryExpression implements BooleanExpression {\n\n        String search;\n\n        public ContainsExpression(Expression right, String search) {\n            super(right);\n            this.search = search;\n        }\n\n        public String getExpressionSymbol() {\n            return \"CONTAINS\";\n        }\n\n        public Object evaluate(EvaluationContext message) throws Exception {\n\n            if (search == null || search.length() == 0) {\n                return Boolean.FALSE;\n            }\n\n            Object rv = this.getRight().evaluate(message);\n\n            if (rv == null) {\n                return Boolean.FALSE;\n            }\n\n            if (!(rv instanceof String)) {\n                return Boolean.FALSE;\n            }\n\n            return ((String)rv).contains(search) ? Boolean.TRUE : Boolean.FALSE;\n        }\n\n        public boolean matches(EvaluationContext message) throws Exception {\n            Object object = evaluate(message);\n            return object != null && object == Boolean.TRUE;\n        }\n    }\n\n    static class NotContainsExpression extends UnaryExpression implements BooleanExpression {\n\n        String search;\n\n        public NotContainsExpression(Expression right, String search) {\n            super(right);\n            this.search = search;\n        }\n\n        public String getExpressionSymbol() {\n            return \"NOT CONTAINS\";\n        }\n\n        public Object evaluate(EvaluationContext message) throws Exception {\n\n            if (search == null || search.length() == 0) {\n                return Boolean.FALSE;\n            }\n\n            Object rv = this.getRight().evaluate(message);\n\n            if (rv == null) {\n                return Boolean.FALSE;\n            }\n\n            if (!(rv instanceof String)) {\n                return Boolean.FALSE;\n            }\n\n            return ((String)rv).contains(search) ? Boolean.FALSE : Boolean.TRUE;\n        }\n\n        public boolean matches(EvaluationContext message) throws Exception {\n            Object object = evaluate(message);\n            return object != null && object == Boolean.TRUE;\n        }\n    }\n\n    public static BooleanExpression createContains(Expression left, String search) {\n        return new ContainsExpression(left, search);\n    }\n\n    public static BooleanExpression createNotContains(Expression left, String search) {\n        return new NotContainsExpression(left, search);\n    }\n\n    static class StartsWithExpression extends UnaryExpression implements BooleanExpression {\n\n        String search;\n\n        public StartsWithExpression(Expression right, String search) {\n            super(right);\n            this.search = search;\n        }\n\n        public String getExpressionSymbol() {\n            return \"STARTSWITH\";\n        }\n\n        public Object evaluate(EvaluationContext message) throws Exception {\n\n            if (search == null || search.length() == 0) {\n                return Boolean.FALSE;\n            }\n\n            Object rv = this.getRight().evaluate(message);\n\n            if (rv == null) {\n                return Boolean.FALSE;\n            }\n\n            if (!(rv instanceof String)) {\n                return Boolean.FALSE;\n            }\n\n            return ((String)rv).startsWith(search) ? Boolean.TRUE : Boolean.FALSE;\n        }\n\n        public boolean matches(EvaluationContext message) throws Exception {\n            Object object = evaluate(message);\n            return object != null && object == Boolean.TRUE;\n        }\n    }\n\n    static class NotStartsWithExpression extends UnaryExpression implements BooleanExpression {\n\n        String search;\n\n        public NotStartsWithExpression(Expression right, String search) {\n            super(right);\n            this.search = search;\n        }\n\n        public String getExpressionSymbol() {\n            return \"NOT STARTSWITH\";\n        }\n\n        public Object evaluate(EvaluationContext message) throws Exception {\n\n            if (search == null || search.length() == 0) {\n                return Boolean.FALSE;\n            }\n\n            Object rv = this.getRight().evaluate(message);\n\n            if (rv == null) {\n                return Boolean.FALSE;\n            }\n\n            if (!(rv instanceof String)) {\n                return Boolean.FALSE;\n            }\n\n            return ((String)rv).startsWith(search) ? Boolean.FALSE : Boolean.TRUE;\n        }\n\n        public boolean matches(EvaluationContext message) throws Exception {\n            Object object = evaluate(message);\n            return object != null && object == Boolean.TRUE;\n        }\n    }\n\n    public static BooleanExpression createStartsWith(Expression left, String search) {\n        return new StartsWithExpression(left, search);\n    }\n\n    public static BooleanExpression createNotStartsWith(Expression left, String search) {\n        return new NotStartsWithExpression(left, search);\n    }\n\n    static class EndsWithExpression extends UnaryExpression implements BooleanExpression {\n\n        String search;\n\n        public EndsWithExpression(Expression right, String search) {\n            super(right);\n            this.search = search;\n        }\n\n        public String getExpressionSymbol() {\n            return \"ENDSWITH\";\n        }\n\n        public Object evaluate(EvaluationContext message) throws Exception {\n\n            if (search == null || search.length() == 0) {\n                return Boolean.FALSE;\n            }\n\n            Object rv = this.getRight().evaluate(message);\n\n            if (rv == null) {\n                return Boolean.FALSE;\n            }\n\n            if (!(rv instanceof String)) {\n                return Boolean.FALSE;\n            }\n\n            return ((String)rv).endsWith(search) ? Boolean.TRUE : Boolean.FALSE;\n        }\n\n        public boolean matches(EvaluationContext message) throws Exception {\n            Object object = evaluate(message);\n            return object != null && object == Boolean.TRUE;\n        }\n    }\n\n    static class NotEndsWithExpression extends UnaryExpression implements BooleanExpression {\n\n        String search;\n\n        public NotEndsWithExpression(Expression right, String search) {\n            super(right);\n            this.search = search;\n        }\n\n        public String getExpressionSymbol() {\n            return \"NOT ENDSWITH\";\n        }\n\n        public Object evaluate(EvaluationContext message) throws Exception {\n\n            if (search == null || search.length() == 0) {\n                return Boolean.FALSE;\n            }\n\n            Object rv = this.getRight().evaluate(message);\n\n            if (rv == null) {\n                return Boolean.FALSE;\n            }\n\n            if (!(rv instanceof String)) {\n                return Boolean.FALSE;\n            }\n\n            return ((String)rv).endsWith(search) ? Boolean.FALSE : Boolean.TRUE;\n        }\n\n        public boolean matches(EvaluationContext message) throws Exception {\n            Object object = evaluate(message);\n            return object != null && object == Boolean.TRUE;\n        }\n    }\n\n    public static BooleanExpression createEndsWith(Expression left, String search) {\n        return new EndsWithExpression(left, search);\n    }\n\n    public static BooleanExpression createNotEndsWith(Expression left, String search) {\n        return new NotEndsWithExpression(left, search);\n    }\n\n    @SuppressWarnings({\"rawtypes\", \"unchecked\"})\n    public static BooleanExpression createInFilter(Expression left, List elements) {\n\n        if (!(left instanceof PropertyExpression)) {\n            throw new RuntimeException(\"Expected a property for In expression, got: \" + left);\n        }\n        return UnaryExpression.createInExpression((PropertyExpression) left, elements, false);\n\n    }\n\n    @SuppressWarnings({\"rawtypes\", \"unchecked\"})\n    public static BooleanExpression createNotInFilter(Expression left, List elements) {\n\n        if (!(left instanceof PropertyExpression)) {\n            throw new RuntimeException(\"Expected a property for In expression, got: \" + left);\n        }\n        return UnaryExpression.createInExpression((PropertyExpression) left, elements, true);\n\n    }\n\n    public static BooleanExpression createIsNull(Expression left) {\n        return doCreateEqual(left, BooleanConstantExpression.NULL);\n    }\n\n    public static BooleanExpression createIsNotNull(Expression left) {\n        return UnaryExpression.createNOT(doCreateEqual(left, BooleanConstantExpression.NULL));\n    }\n\n    public static BooleanExpression createNotEqual(Expression left, Expression right) {\n        return UnaryExpression.createNOT(createEqual(left, right));\n    }\n\n    public static BooleanExpression createEqual(Expression left, Expression right) {\n        checkEqualOperand(left);\n        checkEqualOperand(right);\n        checkEqualOperandCompatability(left, right);\n        return doCreateEqual(left, right);\n    }\n\n    @SuppressWarnings({\"rawtypes\"})\n    private static BooleanExpression doCreateEqual(Expression left, Expression right) {\n        return new ComparisonExpression(left, right) {\n\n            public Object evaluate(EvaluationContext context) throws Exception {\n                Object lv = left.evaluate(context);\n                Object rv = right.evaluate(context);\n\n                // If one of the values is null\n                if (lv == null ^ rv == null) {\n                    if (lv == null) {\n                        return null;\n                    }\n                    return Boolean.FALSE;\n                }\n                if (lv == rv || lv.equals(rv)) {\n                    return Boolean.TRUE;\n                }\n                if (lv instanceof Comparable && rv instanceof Comparable) {\n                    return compare((Comparable) lv, (Comparable) rv);\n                }\n                return Boolean.FALSE;\n            }\n\n            protected boolean asBoolean(int answer) {\n                return answer == 0;\n            }\n\n            public String getExpressionSymbol() {\n                return \"==\";\n            }\n        };\n    }\n\n    public static BooleanExpression createGreaterThan(final Expression left, final Expression right) {\n        checkLessThanOperand(left);\n        checkLessThanOperand(right);\n        return new ComparisonExpression(left, right) {\n            protected boolean asBoolean(int answer) {\n                return answer > 0;\n            }\n\n            public String getExpressionSymbol() {\n                return \">\";\n            }\n        };\n    }\n\n    public static BooleanExpression createGreaterThanEqual(final Expression left, final Expression right) {\n        checkLessThanOperand(left);\n        checkLessThanOperand(right);\n        return new ComparisonExpression(left, right) {\n            protected boolean asBoolean(int answer) {\n                return answer >= 0;\n            }\n\n            public String getExpressionSymbol() {\n                return \">=\";\n            }\n        };\n    }\n\n    public static BooleanExpression createLessThan(final Expression left, final Expression right) {\n        checkLessThanOperand(left);\n        checkLessThanOperand(right);\n        return new ComparisonExpression(left, right) {\n\n            protected boolean asBoolean(int answer) {\n                return answer < 0;\n            }\n\n            public String getExpressionSymbol() {\n                return \"<\";\n            }\n\n        };\n    }\n\n    public static BooleanExpression createLessThanEqual(final Expression left, final Expression right) {\n        checkLessThanOperand(left);\n        checkLessThanOperand(right);\n        return new ComparisonExpression(left, right) {\n\n            protected boolean asBoolean(int answer) {\n                return answer <= 0;\n            }\n\n            public String getExpressionSymbol() {\n                return \"<=\";\n            }\n        };\n    }\n\n    /**\n     * Only Numeric expressions can be used in >, >=, < or <= expressions.s\n     */\n    public static void checkLessThanOperand(Expression expr) {\n        if (expr instanceof ConstantExpression) {\n            Object value = ((ConstantExpression) expr).getValue();\n            if (value instanceof Number) {\n                return;\n            }\n\n            // Else it's boolean or a String..\n            throw new RuntimeException(\"Value '\" + expr + \"' cannot be compared.\");\n        }\n        if (expr instanceof BooleanExpression) {\n            throw new RuntimeException(\"Value '\" + expr + \"' cannot be compared.\");\n        }\n    }\n\n    /**\n     * Validates that the expression can be used in == or <> expression. Cannot\n     * not be NULL TRUE or FALSE litterals.\n     */\n    public static void checkEqualOperand(Expression expr) {\n        if (expr instanceof ConstantExpression) {\n            Object value = ((ConstantExpression) expr).getValue();\n            if (value == null) {\n                throw new RuntimeException(\"'\" + expr + \"' cannot be compared.\");\n            }\n        }\n    }\n\n    /**\n     * @param left\n     * @param right\n     */\n    private static void checkEqualOperandCompatability(Expression left, Expression right) {\n        if (left instanceof ConstantExpression && right instanceof ConstantExpression) {\n            if (left instanceof BooleanExpression && !(right instanceof BooleanExpression)) {\n                throw new RuntimeException(\"'\" + left + \"' cannot be compared with '\" + right + \"'\");\n            }\n        }\n    }\n\n    @SuppressWarnings({\"rawtypes\", \"unchecked\"})\n    public Object evaluate(EvaluationContext context) throws Exception {\n        Comparable<Comparable> lv = (Comparable) left.evaluate(context);\n        if (lv == null) {\n            return null;\n        }\n        Comparable rv = (Comparable) right.evaluate(context);\n        if (rv == null) {\n            return null;\n        }\n        if (getExpressionSymbol().equals(\">=\") || getExpressionSymbol().equals(\">\")\n            || getExpressionSymbol().equals(\"<\") || getExpressionSymbol().equals(\"<=\")) {\n            Class<? extends Comparable> lc = lv.getClass();\n            Class<? extends Comparable> rc = rv.getClass();\n            if (lc == rc && lc == String.class) {\n                // Compare String is illegal\n                // first try to convert to double\n                try {\n                    Comparable lvC = Double.valueOf((String) (Comparable) lv);\n                    Comparable rvC = Double.valueOf((String) rv);\n\n                    return compare(lvC, rvC);\n                } catch (Exception e) {\n                    throw new RuntimeException(\"It's illegal to compare string by '>=', '>', '<', '<='. lv=\" + lv + \", rv=\" + rv, e);\n                }\n            }\n        }\n        return compare(lv, rv);\n    }\n\n    @SuppressWarnings({\"rawtypes\", \"unchecked\"})\n    protected static int __compare(Comparable lv, Comparable rv, boolean convertStringExpressions) {\n        Class<? extends Comparable> lc = lv.getClass();\n        Class<? extends Comparable> rc = rv.getClass();\n        // If the the objects are not of the same type,\n        // try to convert up to allow the comparison.\n        if (lc != rc) {\n            try {\n                if (lc == Boolean.class) {\n                    if (convertStringExpressions && rc == String.class) {\n                        lv = Boolean.valueOf((String) lv).booleanValue();\n                    } else {\n                        return -1;\n                    }\n                } else if (lc == Byte.class) {\n                    if (rc == Short.class) {\n                        lv = Short.valueOf(((Number) lv).shortValue());\n                    } else if (rc == Integer.class) {\n                        lv = Integer.valueOf(((Number) lv).intValue());\n                    } else if (rc == Long.class) {\n                        lv = Long.valueOf(((Number) lv).longValue());\n                    } else if (rc == Float.class) {\n                        lv = new Float(((Number) lv).floatValue());\n                    } else if (rc == Double.class) {\n                        lv = new Double(((Number) lv).doubleValue());\n                    } else if (convertStringExpressions && rc == String.class) {\n                        rv = Byte.valueOf((String) rv);\n                    } else {\n                        return -1;\n                    }\n                } else if (lc == Short.class) {\n                    if (rc == Integer.class) {\n                        lv = Integer.valueOf(((Number) lv).intValue());\n                    } else if (rc == Long.class) {\n                        lv = Long.valueOf(((Number) lv).longValue());\n                    } else if (rc == Float.class) {\n                        lv = new Float(((Number) lv).floatValue());\n                    } else if (rc == Double.class) {\n                        lv = new Double(((Number) lv).doubleValue());\n                    } else if (convertStringExpressions && rc == String.class) {\n                        rv = Short.valueOf((String) rv);\n                    } else {\n                        return -1;\n                    }\n                } else if (lc == Integer.class) {\n                    if (rc == Long.class) {\n                        lv = Long.valueOf(((Number) lv).longValue());\n                    } else if (rc == Float.class) {\n                        lv = new Float(((Number) lv).floatValue());\n                    } else if (rc == Double.class) {\n                        lv = new Double(((Number) lv).doubleValue());\n                    } else if (convertStringExpressions && rc == String.class) {\n                        rv = Integer.valueOf((String) rv);\n                    } else {\n                        return -1;\n                    }\n                } else if (lc == Long.class) {\n                    if (rc == Integer.class) {\n                        rv = Long.valueOf(((Number) rv).longValue());\n                    } else if (rc == Float.class) {\n                        lv = new Float(((Number) lv).floatValue());\n                    } else if (rc == Double.class) {\n                        lv = new Double(((Number) lv).doubleValue());\n                    } else if (convertStringExpressions && rc == String.class) {\n                        rv = Long.valueOf((String) rv);\n                    } else {\n                        return -1;\n                    }\n                } else if (lc == Float.class) {\n                    if (rc == Integer.class) {\n                        rv = new Float(((Number) rv).floatValue());\n                    } else if (rc == Long.class) {\n                        rv = new Float(((Number) rv).floatValue());\n                    } else if (rc == Double.class) {\n                        lv = new Double(((Number) lv).doubleValue());\n                    } else if (convertStringExpressions && rc == String.class) {\n                        rv = Float.valueOf((String) rv);\n                    } else {\n                        return -1;\n                    }\n                } else if (lc == Double.class) {\n                    if (rc == Integer.class) {\n                        rv = new Double(((Number) rv).doubleValue());\n                    } else if (rc == Long.class) {\n                        rv = new Double(((Number) rv).doubleValue());\n                    } else if (rc == Float.class) {\n                        rv = new Float(((Number) rv).doubleValue());\n                    } else if (convertStringExpressions && rc == String.class) {\n                        rv = Double.valueOf((String) rv);\n                    } else {\n                        return -1;\n                    }\n                } else if (convertStringExpressions && lc == String.class) {\n                    if (rc == Boolean.class) {\n                        lv = Boolean.valueOf((String) lv);\n                    } else if (rc == Byte.class) {\n                        lv = Byte.valueOf((String) lv);\n                    } else if (rc == Short.class) {\n                        lv = Short.valueOf((String) lv);\n                    } else if (rc == Integer.class) {\n                        lv = Integer.valueOf((String) lv);\n                    } else if (rc == Long.class) {\n                        lv = Long.valueOf((String) lv);\n                    } else if (rc == Float.class) {\n                        lv = Float.valueOf((String) lv);\n                    } else if (rc == Double.class) {\n                        lv = Double.valueOf((String) lv);\n                    } else {\n                        return -1;\n                    }\n                } else {\n                    return -1;\n                }\n            } catch (NumberFormatException e) {\n                throw new RuntimeException(e);\n            }\n        }\n        return lv.compareTo(rv);\n    }\n\n    @SuppressWarnings({\"rawtypes\", \"unchecked\"})\n    protected Boolean compare(Comparable lv, Comparable rv) {\n        return asBoolean(__compare(lv, rv, convertStringExpressions)) ? Boolean.TRUE : Boolean.FALSE;\n    }\n\n    protected abstract boolean asBoolean(int answer);\n\n    public boolean matches(EvaluationContext context) throws Exception {\n        Object object = evaluate(context);\n        return object != null && object == Boolean.TRUE;\n    }\n\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/expression/ConstantExpression.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter.expression;\n\n/**\n * Represents a constant expression\n * <p>\n * This class was taken from ActiveMQ org.apache.activemq.filter.ConstantExpression,\n * but:\n * 1. For long type constant, the range bound by java Long type;\n * 2. For float type constant, the range bound by java Double type;\n * 3. Remove Hex and Octal expression;\n * 4. Add now expression to support to get current time.\n * </p>\n */\npublic class ConstantExpression implements Expression {\n\n    private Object value;\n\n    public ConstantExpression(Object value) {\n        this.value = value;\n    }\n\n    public static ConstantExpression createFromDecimal(String text) {\n\n        // Strip off the 'l' or 'L' if needed.\n        if (text.endsWith(\"l\") || text.endsWith(\"L\")) {\n            text = text.substring(0, text.length() - 1);\n        }\n\n        // only support Long.MIN_VALUE ~ Long.MAX_VALUE\n        Number value = new Long(text);\n\n        long l = value.longValue();\n        if (Integer.MIN_VALUE <= l && l <= Integer.MAX_VALUE) {\n            value = value.intValue();\n        }\n        return new ConstantExpression(value);\n    }\n\n    public static ConstantExpression createFloat(String text) {\n        Double value = new Double(text);\n        if (value > Double.MAX_VALUE) {\n            throw new RuntimeException(text + \" is greater than \" + Double.MAX_VALUE);\n        }\n        if (value < Double.MIN_VALUE) {\n            throw new RuntimeException(text + \" is less than \" + Double.MIN_VALUE);\n        }\n        return new ConstantExpression(value);\n    }\n\n    public static ConstantExpression createNow() {\n        return new NowExpression();\n    }\n\n    public Object evaluate(EvaluationContext context) throws Exception {\n        return value;\n    }\n\n    public Object getValue() {\n        return value;\n    }\n\n    /**\n     * @see Object#toString()\n     */\n    public String toString() {\n        Object value = getValue();\n        if (value == null) {\n            return \"NULL\";\n        }\n        if (value instanceof Boolean) {\n            return (Boolean) value ? \"TRUE\" : \"FALSE\";\n        }\n        if (value instanceof String) {\n            return encodeString((String) value);\n        }\n        return value.toString();\n    }\n\n    /**\n     * @see Object#hashCode()\n     */\n    public int hashCode() {\n        return toString().hashCode();\n    }\n\n    /**\n     * @see Object#equals(Object)\n     */\n    public boolean equals(Object o) {\n\n        if (o == null || !this.getClass().equals(o.getClass())) {\n            return false;\n        }\n        return toString().equals(o.toString());\n\n    }\n\n    /**\n     * Encodes the value of string so that it looks like it would look like when\n     * it was provided in a selector.\n     */\n    public static String encodeString(String s) {\n\n        StringBuilder builder = new StringBuilder();\n\n        builder.append('\\'');\n        for (int i = 0; i < s.length(); i++) {\n            char c = s.charAt(i);\n            if (c == '\\'') {\n                builder.append(c);\n            }\n            builder.append(c);\n        }\n        builder.append('\\'');\n        return builder.toString();\n    }\n\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/expression/EmptyEvaluationContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter.expression;\n\nimport java.util.Map;\n\n/**\n * Empty context.\n */\npublic class EmptyEvaluationContext implements EvaluationContext {\n    @Override\n    public Object get(String name) {\n        return null;\n    }\n\n    @Override\n    public Map<String, Object> keyValues() {\n        return null;\n    }\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/expression/EvaluationContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter.expression;\n\nimport java.util.Map;\n\n/**\n * Context of evaluate expression.\n *\n * Compare to org.apache.activemq.filter.MessageEvaluationContext, this is just an interface.\n */\npublic interface EvaluationContext {\n\n    /**\n     * Get value by name from context\n     */\n    Object get(String name);\n\n    /**\n     * Context variables.\n     */\n    Map<String, Object> keyValues();\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/expression/Expression.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter.expression;\n\n/**\n * Interface of expression.\n * <p>\n * This class was taken from ActiveMQ org.apache.activemq.filter.Expression,\n * but the parameter is changed to an interface.\n * </p>\n *\n * @see org.apache.rocketmq.filter.expression.EvaluationContext\n */\npublic interface Expression {\n\n    /**\n     * Calculate express result with context.\n     *\n     * @param context context of evaluation\n     * @return the value of this expression\n     */\n    Object evaluate(EvaluationContext context) throws Exception;\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/expression/LogicExpression.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter.expression;\n\n/**\n * A filter performing a comparison of two objects\n * <p>\n * This class was taken from ActiveMQ org.apache.activemq.filter.LogicExpression,\n * </p>\n */\npublic abstract class LogicExpression extends BinaryExpression implements BooleanExpression {\n\n    /**\n     * @param left\n     * @param right\n     */\n    public LogicExpression(BooleanExpression left, BooleanExpression right) {\n        super(left, right);\n    }\n\n    public static BooleanExpression createOR(BooleanExpression lvalue, BooleanExpression rvalue) {\n        return new LogicExpression(lvalue, rvalue) {\n\n            public Object evaluate(EvaluationContext context) throws Exception {\n\n                Boolean lv = (Boolean) left.evaluate(context);\n                if (lv != null && lv.booleanValue()) {\n                    return Boolean.TRUE;\n                }\n                Boolean rv = (Boolean) right.evaluate(context);\n                if (rv != null && rv.booleanValue()) {\n                    return Boolean.TRUE;\n                }\n                if (lv == null || rv == null) {\n                    return null;\n                }\n                return Boolean.FALSE;\n            }\n\n            public String getExpressionSymbol() {\n                return \"||\";\n            }\n        };\n    }\n\n    public static BooleanExpression createAND(BooleanExpression lvalue, BooleanExpression rvalue) {\n        return new LogicExpression(lvalue, rvalue) {\n\n            public Object evaluate(EvaluationContext context) throws Exception {\n\n                Boolean lv = (Boolean) left.evaluate(context);\n\n                if (lv != null && !lv.booleanValue()) {\n                    return Boolean.FALSE;\n                }\n                Boolean rv = (Boolean) right.evaluate(context);\n                if (rv != null && !rv.booleanValue()) {\n                    return Boolean.FALSE;\n                }\n                if (lv == null || rv == null) {\n                    return null;\n                }\n                return Boolean.TRUE;\n            }\n\n            public String getExpressionSymbol() {\n                return \"&&\";\n            }\n        };\n    }\n\n    public abstract Object evaluate(EvaluationContext context) throws Exception;\n\n    public boolean matches(EvaluationContext context) throws Exception {\n        Object object = evaluate(context);\n        return object != null && object == Boolean.TRUE;\n    }\n\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/expression/MQFilterException.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter.expression;\n\n/**\n * Exception.\n */\npublic class MQFilterException extends Exception {\n    private static final long serialVersionUID = 1L;\n    private final int responseCode;\n    private final String errorMessage;\n\n    public MQFilterException(String errorMessage, Throwable cause) {\n        super(cause);\n        this.responseCode = -1;\n        this.errorMessage = errorMessage;\n    }\n\n    public MQFilterException(int responseCode, String errorMessage) {\n        this.responseCode = responseCode;\n        this.errorMessage = errorMessage;\n    }\n\n    public int getResponseCode() {\n        return responseCode;\n    }\n\n    public String getErrorMessage() {\n        return errorMessage;\n    }\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/expression/NowExpression.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter.expression;\n\n/**\n * Current time expression.Just for test.\n */\npublic class NowExpression extends ConstantExpression {\n    public NowExpression() {\n        super(\"now\");\n    }\n\n    @Override\n    public Object evaluate(EvaluationContext context) throws Exception {\n        return new Long(System.currentTimeMillis());\n    }\n\n    public Object getValue() {\n        return new Long(System.currentTimeMillis());\n    }\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/expression/PropertyExpression.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter.expression;\n\n/**\n * Represents a property expression\n * <p>\n * This class was taken from ActiveMQ org.apache.activemq.filter.PropertyExpression,\n * but more simple and no transfer between expression and message property.\n * </p>\n */\npublic class PropertyExpression implements Expression {\n    private final String name;\n\n    public PropertyExpression(String name) {\n        this.name = name;\n    }\n\n    @Override\n    public Object evaluate(EvaluationContext context) throws Exception {\n        return context.get(name);\n    }\n\n    public String getName() {\n        return name;\n    }\n\n    /**\n     * @see Object#toString()\n     */\n    @Override\n    public String toString() {\n        return name;\n    }\n\n    /**\n     * @see Object#hashCode()\n     */\n    @Override\n    public int hashCode() {\n        return name.hashCode();\n    }\n\n    /**\n     * @see Object#equals(Object)\n     */\n    @Override\n    public boolean equals(Object o) {\n\n        if (o == null || !this.getClass().equals(o.getClass())) {\n            return false;\n        }\n        return name.equals(((PropertyExpression) o).name);\n    }\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/expression/UnaryExpression.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter.expression;\n\nimport org.apache.rocketmq.filter.constant.UnaryType;\n\nimport java.math.BigDecimal;\nimport java.util.Collection;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.List;\n\n/**\n * An expression which performs an operation on two expression values\n * <p>\n * This class was taken from ActiveMQ org.apache.activemq.filter.UnaryExpression,\n * but:\n * 1. remove XPath and XQuery expression;\n * 2. Add constant UnaryType to distinguish different unary expression;\n * 3. Extract UnaryInExpression to an independent class.\n * </p>\n */\npublic abstract class UnaryExpression implements Expression {\n\n    private static final BigDecimal BD_LONG_MIN_VALUE = BigDecimal.valueOf(Long.MIN_VALUE);\n    protected Expression right;\n\n    public UnaryType unaryType;\n\n    public UnaryExpression(Expression left) {\n        this.right = left;\n    }\n\n    public UnaryExpression(Expression left, UnaryType unaryType) {\n        this.setUnaryType(unaryType);\n        this.right = left;\n    }\n\n    public static Expression createNegate(Expression left) {\n        return new UnaryExpression(left, UnaryType.NEGATE) {\n            @Override\n            public Object evaluate(EvaluationContext context) throws Exception {\n                Object rvalue = right.evaluate(context);\n                if (rvalue == null) {\n                    return null;\n                }\n                if (rvalue instanceof Number) {\n                    return negate((Number) rvalue);\n                }\n                return null;\n            }\n\n            @Override\n            public String getExpressionSymbol() {\n                return \"-\";\n            }\n        };\n    }\n\n    public static BooleanExpression createInExpression(PropertyExpression right, List<Object> elements,\n        final boolean not) {\n\n        // Use a HashSet if there are many elements.\n        Collection<Object> t;\n        if (elements.size() == 0) {\n            t = null;\n        } else if (elements.size() < 5) {\n            t = elements;\n        } else {\n            t = new HashSet<>(elements);\n        }\n        final Collection inList = t;\n\n        return new UnaryInExpression(right, UnaryType.IN, inList, not) {\n            @Override\n            public Object evaluate(EvaluationContext context) throws Exception {\n\n                Object rvalue = right.evaluate(context);\n                if (rvalue == null) {\n                    return null;\n                }\n                if (rvalue.getClass() != String.class) {\n                    return null;\n                }\n\n                if ((inList != null && inList.contains(rvalue)) ^ not) {\n                    return Boolean.TRUE;\n                } else {\n                    return Boolean.FALSE;\n                }\n\n            }\n\n            @Override\n            public String toString() {\n                StringBuilder answer = new StringBuilder();\n                answer.append(right);\n                answer.append(\" \");\n                answer.append(getExpressionSymbol());\n                answer.append(\" ( \");\n\n                int count = 0;\n                for (Iterator i = inList.iterator(); i.hasNext(); ) {\n                    Object o = (Object) i.next();\n                    if (count != 0) {\n                        answer.append(\", \");\n                    }\n                    answer.append(o);\n                    count++;\n                }\n\n                answer.append(\" )\");\n                return answer.toString();\n            }\n\n            @Override\n            public String getExpressionSymbol() {\n                if (not) {\n                    return \"NOT IN\";\n                } else {\n                    return \"IN\";\n                }\n            }\n        };\n    }\n\n    abstract static class BooleanUnaryExpression extends UnaryExpression implements BooleanExpression {\n        public BooleanUnaryExpression(Expression left, UnaryType unaryType) {\n            super(left, unaryType);\n        }\n\n        @Override\n        public boolean matches(EvaluationContext context) throws Exception {\n            Object object = evaluate(context);\n            return object != null && object == Boolean.TRUE;\n        }\n    }\n\n    public static BooleanExpression createNOT(BooleanExpression left) {\n        return new BooleanUnaryExpression(left, UnaryType.NOT) {\n            @Override\n            public Object evaluate(EvaluationContext context) throws Exception {\n                Boolean lvalue = (Boolean) right.evaluate(context);\n                if (lvalue == null) {\n                    return null;\n                }\n                return lvalue.booleanValue() ? Boolean.FALSE : Boolean.TRUE;\n            }\n\n            @Override\n            public String getExpressionSymbol() {\n                return \"NOT\";\n            }\n        };\n    }\n\n    public static BooleanExpression createBooleanCast(Expression left) {\n        return new BooleanUnaryExpression(left, UnaryType.BOOLEANCAST) {\n            @Override\n            public Object evaluate(EvaluationContext context) throws Exception {\n                Object rvalue = right.evaluate(context);\n                if (rvalue == null) {\n                    return null;\n                }\n                if (!rvalue.getClass().equals(Boolean.class)) {\n                    return Boolean.FALSE;\n                }\n                return ((Boolean) rvalue).booleanValue() ? Boolean.TRUE : Boolean.FALSE;\n            }\n\n            @Override\n            public String toString() {\n                return right.toString();\n            }\n\n            @Override\n            public String getExpressionSymbol() {\n                return \"\";\n            }\n        };\n    }\n\n    private static Number negate(Number left) {\n        Class clazz = left.getClass();\n        if (clazz == Integer.class) {\n            return new Integer(-left.intValue());\n        } else if (clazz == Long.class) {\n            return new Long(-left.longValue());\n        } else if (clazz == Float.class) {\n            return new Float(-left.floatValue());\n        } else if (clazz == Double.class) {\n            return new Double(-left.doubleValue());\n        } else if (clazz == BigDecimal.class) {\n            // We ussually get a big deciamal when we have Long.MIN_VALUE\n            // constant in the\n            // Selector. Long.MIN_VALUE is too big to store in a Long as a\n            // positive so we store it\n            // as a Big decimal. But it gets Negated right away.. to here we try\n            // to covert it back\n            // to a Long.\n            BigDecimal bd = (BigDecimal) left;\n            bd = bd.negate();\n\n            if (BD_LONG_MIN_VALUE.compareTo(bd) == 0) {\n                return Long.valueOf(Long.MIN_VALUE);\n            }\n            return bd;\n        } else {\n            throw new RuntimeException(\"Don't know how to negate: \" + left);\n        }\n    }\n\n    public Expression getRight() {\n        return right;\n    }\n\n    public void setRight(Expression expression) {\n        right = expression;\n    }\n\n    public UnaryType getUnaryType() {\n        return unaryType;\n    }\n\n    public void setUnaryType(UnaryType unaryType) {\n        this.unaryType = unaryType;\n    }\n\n    /**\n     * @see Object#toString()\n     */\n    @Override\n    public String toString() {\n        return \"(\" + getExpressionSymbol() + \" \" + right.toString() + \")\";\n    }\n\n    /**\n     * @see Object#hashCode()\n     */\n    @Override\n    public int hashCode() {\n        return toString().hashCode();\n    }\n\n    /**\n     * @see Object#equals(Object)\n     */\n    @Override\n    public boolean equals(Object o) {\n\n        if (o == null || !this.getClass().equals(o.getClass())) {\n            return false;\n        }\n        return toString().equals(o.toString());\n\n    }\n\n    /**\n     * Returns the symbol that represents this binary expression. For example,\n     * addition is represented by \"+\"\n     */\n    public abstract String getExpressionSymbol();\n\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/expression/UnaryInExpression.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter.expression;\n\nimport org.apache.rocketmq.filter.constant.UnaryType;\n\nimport java.util.Collection;\n\n/**\n * In expression.\n */\nabstract public class UnaryInExpression extends UnaryExpression implements BooleanExpression {\n\n    private boolean not;\n\n    private Collection inList;\n\n    public UnaryInExpression(Expression left, UnaryType unaryType,\n        Collection inList, boolean not) {\n        super(left, unaryType);\n        this.setInList(inList);\n        this.setNot(not);\n\n    }\n\n    public boolean matches(EvaluationContext context) throws Exception {\n        Object object = evaluate(context);\n        return object != null && object == Boolean.TRUE;\n    }\n\n    public boolean isNot() {\n        return not;\n    }\n\n    public void setNot(boolean not) {\n        this.not = not;\n    }\n\n    public Collection getInList() {\n        return inList;\n    }\n\n    public void setInList(Collection inList) {\n        this.inList = inList;\n    }\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/parser/ParseException.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/* Generated By:JavaCC: Do not edit this line. ParseException.java Version 5.0 */\n/* JavaCCOptions:KEEP_LINE_COL=null */\npackage org.apache.rocketmq.filter.parser;\n\n/**\n * This exception is thrown when parse errors are encountered.\n * You can explicitly create objects of this exception type by\n * calling the method generateParseException in the generated\n * parser.\n * <p/>\n * You can modify this class to customize your error reporting\n * mechanisms so long as you retain the public fields.\n */\npublic class ParseException extends Exception {\n\n    /**\n     * The version identifier for this Serializable class.\n     * Increment only if the <i>serialized</i> form of the\n     * class changes.\n     */\n    private static final long serialVersionUID = 1L;\n\n    /**\n     * This constructor is used by the method \"generateParseException\"\n     * in the generated parser.  Calling this constructor generates\n     * a new object of this type with the fields \"currentToken\",\n     * \"expectedTokenSequences\", and \"TOKEN_IMAGE\" set.\n     */\n    public ParseException(Token currentTokenVal,\n        int[][] expectedTokenSequencesVal,\n        String[] tokenImageVal\n    ) {\n        super(initialise(currentTokenVal, expectedTokenSequencesVal, tokenImageVal));\n        currentToken = currentTokenVal;\n        expectedTokenSequences = expectedTokenSequencesVal;\n        tokenImage = tokenImageVal;\n    }\n\n    /**\n     * The following constructors are for use by you for whatever\n     * purpose you can think of.  Constructing the exception in this\n     * manner makes the exception behave in the normal way - i.e., as\n     * documented in the class \"Throwable\".  The fields \"errorToken\",\n     * \"expectedTokenSequences\", and \"TOKEN_IMAGE\" do not contain\n     * relevant information.  The JavaCC generated code does not use\n     * these constructors.\n     */\n\n    public ParseException() {\n        super();\n    }\n\n    /**\n     * Constructor with message.\n     */\n    public ParseException(String message) {\n        super(message);\n    }\n\n    /**\n     * This is the last token that has been consumed successfully.  If\n     * this object has been created due to a parse error, the token\n     * followng this token will (therefore) be the first error token.\n     */\n    public Token currentToken;\n\n    /**\n     * Each entry in this array is an array of integers.  Each array\n     * of integers represents a sequence of tokens (by their ordinal\n     * values) that is expected at this point of the parse.\n     */\n    public int[][] expectedTokenSequences;\n\n    /**\n     * This is a reference to the \"TOKEN_IMAGE\" array of the generated\n     * parser within which the parse error occurred.  This array is\n     * defined in the generated ...Constants interface.\n     */\n    public String[] tokenImage;\n\n    /**\n     * It uses \"currentToken\" and \"expectedTokenSequences\" to generate a parse\n     * error message and returns it.  If this object has been created\n     * due to a parse error, and you do not catch it (it gets thrown\n     * from the parser) the correct error message\n     * gets displayed.\n     */\n    private static String initialise(Token currentToken,\n        int[][] expectedTokenSequences,\n        String[] tokenImage) {\n        String eol = System.getProperty(\"line.separator\", \"\\n\");\n        StringBuilder expected = new StringBuilder();\n        int maxSize = 0;\n        for (int i = 0; i < expectedTokenSequences.length; i++) {\n            if (maxSize < expectedTokenSequences[i].length) {\n                maxSize = expectedTokenSequences[i].length;\n            }\n            for (int j = 0; j < expectedTokenSequences[i].length; j++) {\n                expected.append(tokenImage[expectedTokenSequences[i][j]]).append(' ');\n            }\n            if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) {\n                expected.append(\"...\");\n            }\n            expected.append(eol).append(\"    \");\n        }\n        String retval = \"Encountered \\\"\";\n        Token tok = currentToken.next;\n        for (int i = 0; i < maxSize; i++) {\n            if (i != 0) {\n                retval += \" \";\n            }\n            if (tok.kind == 0) {\n                retval += tokenImage[0];\n                break;\n            }\n            retval += \" \" + tokenImage[tok.kind];\n            retval += \" \\\"\";\n            retval += add_escapes(tok.image);\n            retval += \" \\\"\";\n            tok = tok.next;\n        }\n        retval += \"\\\" at line \" + currentToken.next.beginLine + \", column \" + currentToken.next.beginColumn;\n        retval += \".\" + eol;\n        if (expectedTokenSequences.length == 1) {\n            retval += \"Was expecting:\" + eol + \"    \";\n        } else {\n            retval += \"Was expecting one of:\" + eol + \"    \";\n        }\n        retval += expected.toString();\n        return retval;\n    }\n\n    /**\n     * The end of line string for this machine.\n     */\n    protected String eol = System.getProperty(\"line.separator\", \"\\n\");\n\n    /**\n     * Used to convert raw characters to their escaped version\n     * when these raw version cannot be used as part of an ASCII\n     * string literal.\n     */\n    static String add_escapes(String str) {\n        StringBuilder retval = new StringBuilder();\n        char ch;\n        for (int i = 0; i < str.length(); i++) {\n            switch (str.charAt(i)) {\n                case 0:\n                    continue;\n                case '\\b':\n                    retval.append(\"\\\\b\");\n                    continue;\n                case '\\t':\n                    retval.append(\"\\\\t\");\n                    continue;\n                case '\\n':\n                    retval.append(\"\\\\n\");\n                    continue;\n                case '\\f':\n                    retval.append(\"\\\\f\");\n                    continue;\n                case '\\r':\n                    retval.append(\"\\\\r\");\n                    continue;\n                case '\\\"':\n                    retval.append(\"\\\\\\\"\");\n                    continue;\n                case '\\'':\n                    retval.append(\"\\\\\\'\");\n                    continue;\n                case '\\\\':\n                    retval.append(\"\\\\\\\\\");\n                    continue;\n                default:\n                    if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {\n                        String s = \"0000\" + Integer.toString(ch, 16);\n                        retval.append(\"\\\\u\" + s.substring(s.length() - 4, s.length()));\n                    } else {\n                        retval.append(ch);\n                    }\n                    continue;\n            }\n        }\n        return retval.toString();\n    }\n\n}\n/* JavaCC - OriginalChecksum=60cf9c227a487e4be49599bc903f0a6a (do not edit this line) */\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParser.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/* Generated By:JavaCC: Do not edit this line. SelectorParser.java */\npackage org.apache.rocketmq.filter.parser;\n\nimport com.google.common.cache.Cache;\nimport com.google.common.cache.CacheBuilder;\nimport org.apache.rocketmq.filter.expression.BooleanConstantExpression;\nimport org.apache.rocketmq.filter.expression.BooleanExpression;\nimport org.apache.rocketmq.filter.expression.ComparisonExpression;\nimport org.apache.rocketmq.filter.expression.ConstantExpression;\nimport org.apache.rocketmq.filter.expression.Expression;\nimport org.apache.rocketmq.filter.expression.LogicExpression;\nimport org.apache.rocketmq.filter.expression.MQFilterException;\nimport org.apache.rocketmq.filter.expression.PropertyExpression;\nimport org.apache.rocketmq.filter.expression.UnaryExpression;\n\nimport java.io.StringReader;\nimport java.util.ArrayList;\n\n/**\n * JMS Selector Parser generated by JavaCC\n * <p/>\n * Do not edit this .java file directly - it is autogenerated from SelectorParser.jj\n */\npublic class SelectorParser implements SelectorParserConstants {\n\n    private static final Cache<String, Object> PARSE_CACHE = CacheBuilder.newBuilder().maximumSize(100).build();\n\n    public static BooleanExpression parse(String sql) throws MQFilterException {\n        Object result = PARSE_CACHE.getIfPresent(sql);\n        if (result instanceof MQFilterException) {\n            throw (MQFilterException) result;\n        } else if (result instanceof BooleanExpression) {\n            return (BooleanExpression) result;\n        } else {\n            ComparisonExpression.CONVERT_STRING_EXPRESSIONS.set(true);\n            try {\n\n                BooleanExpression e = new SelectorParser(sql).parse();\n                PARSE_CACHE.put(sql, e);\n                return e;\n            } catch (MQFilterException t) {\n                PARSE_CACHE.put(sql, t);\n                throw t;\n            } finally {\n                ComparisonExpression.CONVERT_STRING_EXPRESSIONS.remove();\n            }\n        }\n    }\n\n    public static void clearCache() {\n        PARSE_CACHE.cleanUp();\n    }\n\n    private String sql;\n\n    protected SelectorParser(String sql) {\n        this(new StringReader(sql));\n        this.sql = sql;\n    }\n\n    protected BooleanExpression parse() throws MQFilterException {\n        try {\n            return this.JmsSelector();\n        } catch (Throwable e) {\n            throw new MQFilterException(\"Invalid MessageSelector. \", e);\n        }\n    }\n\n    private BooleanExpression asBooleanExpression(Expression value) throws ParseException {\n        if (value instanceof BooleanExpression) {\n            return (BooleanExpression) value;\n        }\n        if (value instanceof PropertyExpression) {\n            return UnaryExpression.createBooleanCast(value);\n        }\n        throw new ParseException(\"Expression will not result in a boolean value: \" + value);\n    }\n\n    // ----------------------------------------------------------------------------\n    // Grammar\n    // ----------------------------------------------------------------------------\n    final public BooleanExpression JmsSelector() throws ParseException {\n        Expression left = orExpression();\n        return asBooleanExpression(left);\n    }\n\n    final public Expression orExpression() throws ParseException {\n        Expression left;\n        Expression right;\n        left = andExpression();\n        label_1:\n        while (true) {\n            switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {\n                case OR:\n                    break;\n                default:\n                    jjLa1[0] = jjGen;\n                    break label_1;\n            }\n            jj_consume_token(OR);\n            right = andExpression();\n            left = LogicExpression.createOR(asBooleanExpression(left), asBooleanExpression(right));\n        }\n        return left;\n    }\n\n    final public Expression andExpression() throws ParseException {\n        Expression left;\n        Expression right;\n        left = equalityExpression();\n        label_2:\n        while (true) {\n            switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {\n                case AND:\n                    break;\n                default:\n                    jjLa1[1] = jjGen;\n                    break label_2;\n            }\n            jj_consume_token(AND);\n            right = equalityExpression();\n            left = LogicExpression.createAND(asBooleanExpression(left), asBooleanExpression(right));\n        }\n        return left;\n    }\n\n    final public Expression equalityExpression() throws ParseException {\n        Expression left;\n        Expression right;\n        left = comparisonExpression();\n        label_3:\n        while (true) {\n            switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {\n                case IS:\n                case 25:\n                case 26:\n                    break;\n                default:\n                    jjLa1[2] = jjGen;\n                    break label_3;\n            }\n            switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {\n                case 25:\n                    jj_consume_token(25);\n                    right = comparisonExpression();\n                    left = ComparisonExpression.createEqual(left, right);\n                    break;\n                case 26:\n                    jj_consume_token(26);\n                    right = comparisonExpression();\n                    left = ComparisonExpression.createNotEqual(left, right);\n                    break;\n                default:\n                    jjLa1[3] = jjGen;\n                    if (jj_2_1(2)) {\n                        jj_consume_token(IS);\n                        jj_consume_token(NULL);\n                        left = ComparisonExpression.createIsNull(left);\n                    } else {\n                        switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {\n                            case IS:\n                                jj_consume_token(IS);\n                                jj_consume_token(NOT);\n                                jj_consume_token(NULL);\n                                left = ComparisonExpression.createIsNotNull(left);\n                                break;\n                            default:\n                                jjLa1[4] = jjGen;\n                                jj_consume_token(-1);\n                                throw new ParseException();\n                        }\n                    }\n            }\n        }\n        return left;\n    }\n\n    final public Expression comparisonExpression() throws ParseException {\n        Expression left;\n        Expression right;\n        Expression low;\n        Expression high;\n        String t, u;\n        boolean not;\n        ArrayList list;\n        left = unaryExpr();\n        label_4:\n        while (true) {\n            switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {\n                case NOT:\n                case BETWEEN:\n                case IN:\n                case CONTAINS:\n                case STARTSWITH:\n                case ENDSWITH:\n                case 27:\n                case 28:\n                case 29:\n                case 30:\n                    break;\n                default:\n                    jjLa1[5] = jjGen;\n                    break label_4;\n            }\n            switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {\n                case 27:\n                    jj_consume_token(27);\n                    right = unaryExpr();\n                    left = ComparisonExpression.createGreaterThan(left, right);\n                    break;\n                case 28:\n                    jj_consume_token(28);\n                    right = unaryExpr();\n                    left = ComparisonExpression.createGreaterThanEqual(left, right);\n                    break;\n                case 29:\n                    jj_consume_token(29);\n                    right = unaryExpr();\n                    left = ComparisonExpression.createLessThan(left, right);\n                    break;\n                case 30:\n                    jj_consume_token(30);\n                    right = unaryExpr();\n                    left = ComparisonExpression.createLessThanEqual(left, right);\n                    break;\n                case CONTAINS:\n                    jj_consume_token(CONTAINS);\n                    t = stringLitteral();\n                    left = ComparisonExpression.createContains(left, t);\n                    break;\n                default:\n                    jjLa1[8] = jjGen;\n                    if (jj_2_2(2)) {\n                        jj_consume_token(NOT);\n                        jj_consume_token(CONTAINS);\n                        t = stringLitteral();\n                        left = ComparisonExpression.createNotContains(left, t);\n                    } else {\n                        switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {\n                            case STARTSWITH:\n                                jj_consume_token(STARTSWITH);\n                                t = stringLitteral();\n                                left = ComparisonExpression.createStartsWith(left, t);\n                                break;\n                            default:\n                                jjLa1[9] = jjGen;\n                                if (jj_2_3(2)) {\n                                    jj_consume_token(NOT);\n                                    jj_consume_token(STARTSWITH);\n                                    t = stringLitteral();\n                                    left = ComparisonExpression.createNotStartsWith(left, t);\n                                } else {\n                                    switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {\n                                        case ENDSWITH:\n                                            jj_consume_token(ENDSWITH);\n                                            t = stringLitteral();\n                                            left = ComparisonExpression.createEndsWith(left, t);\n                                            break;\n                                        default:\n                                            jjLa1[10] = jjGen;\n                                            if (jj_2_4(2)) {\n                                                jj_consume_token(NOT);\n                                                jj_consume_token(ENDSWITH);\n                                                t = stringLitteral();\n                                                left = ComparisonExpression.createNotEndsWith(left, t);\n                                            } else {\n                                                switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {\n                                                    case BETWEEN:\n                                                        jj_consume_token(BETWEEN);\n                                                        low = unaryExpr();\n                                                        jj_consume_token(AND);\n                                                        high = unaryExpr();\n                                                        left = ComparisonExpression.createBetween(left, low, high);\n                                                        break;\n                                                    default:\n                                                        jjLa1[11] = jjGen;\n                                                        if (jj_2_5(2)) {\n                                                            jj_consume_token(NOT);\n                                                            jj_consume_token(BETWEEN);\n                                                            low = unaryExpr();\n                                                            jj_consume_token(AND);\n                                                            high = unaryExpr();\n                                                            left = ComparisonExpression.createNotBetween(left, low, high);\n                                                        } else {\n                                                            switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {\n                                                                case IN:\n                                                                    jj_consume_token(IN);\n                                                                    jj_consume_token(31);\n                                                                    t = stringLitteral();\n                                                                    list = new ArrayList();\n                                                                    list.add(t);\n                                                                    label_5:\n                                                                    while (true) {\n                                                                        switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {\n                                                                            case 32:\n                                                                                break;\n                                                                            default:\n                                                                                jjLa1[6] = jjGen;\n                                                                                break label_5;\n                                                                        }\n                                                                        jj_consume_token(32);\n                                                                        t = stringLitteral();\n                                                                        list.add(t);\n                                                                    }\n                                                                    jj_consume_token(33);\n                                                                    left = ComparisonExpression.createInFilter(left, list);\n                                                                    break;\n                                                                default:\n                                                                    jjLa1[12] = jjGen;\n                                                                    if (jj_2_6(2)) {\n                                                                        jj_consume_token(NOT);\n                                                                        jj_consume_token(IN);\n                                                                        jj_consume_token(31);\n                                                                        t = stringLitteral();\n                                                                        list = new ArrayList();\n                                                                        list.add(t);\n                                                                        label_6:\n                                                                        while (true) {\n                                                                            switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {\n                                                                                case 32:\n                                                                                    break;\n                                                                                default:\n                                                                                    jjLa1[7] = jjGen;\n                                                                                    break label_6;\n                                                                            }\n                                                                            jj_consume_token(32);\n                                                                            t = stringLitteral();\n                                                                            list.add(t);\n                                                                        }\n                                                                        jj_consume_token(33);\n                                                                        left = ComparisonExpression.createNotInFilter(left, list);\n                                                                    } else {\n                                                                        jj_consume_token(-1);\n                                                                        throw new ParseException();\n                                                                    }\n                                                            }\n                                                        }\n                                                }\n                                            }\n                                    }\n                                }\n                        }\n                    }\n            }\n        }\n        return left;\n    }\n\n    final public Expression unaryExpr() throws ParseException {\n        String s = null;\n        Expression left = null;\n        if (jj_2_7(2147483647)) {\n            jj_consume_token(34);\n            left = unaryExpr();\n        } else {\n            switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {\n                case 35:\n                    jj_consume_token(35);\n                    left = unaryExpr();\n                    left = UnaryExpression.createNegate(left);\n                    break;\n                case NOT:\n                    jj_consume_token(NOT);\n                    left = unaryExpr();\n                    left = UnaryExpression.createNOT(asBooleanExpression(left));\n                    break;\n                case TRUE:\n                case FALSE:\n                case NULL:\n                case DECIMAL_LITERAL:\n                case FLOATING_POINT_LITERAL:\n                case STRING_LITERAL:\n                case ID:\n                case 31:\n                    left = primaryExpr();\n                    break;\n                default:\n                    jjLa1[13] = jjGen;\n                    jj_consume_token(-1);\n                    throw new ParseException();\n            }\n        }\n        return left;\n    }\n\n    final public Expression primaryExpr() throws ParseException {\n        Expression left = null;\n        switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {\n            case TRUE:\n            case FALSE:\n            case NULL:\n            case DECIMAL_LITERAL:\n            case FLOATING_POINT_LITERAL:\n            case STRING_LITERAL:\n                left = literal();\n                break;\n            case ID:\n                left = variable();\n                break;\n            case 31:\n                jj_consume_token(31);\n                left = orExpression();\n                jj_consume_token(33);\n                break;\n            default:\n                jjLa1[14] = jjGen;\n                jj_consume_token(-1);\n                throw new ParseException();\n        }\n        return left;\n    }\n\n    final public ConstantExpression literal() throws ParseException {\n        Token t;\n        String s;\n        ConstantExpression left = null;\n        switch ((jjNtk == -1) ? jj_ntk() : jjNtk) {\n            case STRING_LITERAL:\n                s = stringLitteral();\n                left = new ConstantExpression(s);\n                break;\n            case DECIMAL_LITERAL:\n                t = jj_consume_token(DECIMAL_LITERAL);\n                left = ConstantExpression.createFromDecimal(t.image);\n                break;\n            case FLOATING_POINT_LITERAL:\n                t = jj_consume_token(FLOATING_POINT_LITERAL);\n                left = ConstantExpression.createFloat(t.image);\n                break;\n            case TRUE:\n                jj_consume_token(TRUE);\n                left = BooleanConstantExpression.TRUE;\n                break;\n            case FALSE:\n                jj_consume_token(FALSE);\n                left = BooleanConstantExpression.FALSE;\n                break;\n            case NULL:\n                jj_consume_token(NULL);\n                left = BooleanConstantExpression.NULL;\n                break;\n            default:\n                jjLa1[15] = jjGen;\n                jj_consume_token(-1);\n                throw new ParseException();\n        }\n        return left;\n    }\n\n    final public String stringLitteral() throws ParseException {\n        Token t;\n        StringBuffer rc = new StringBuffer();\n        boolean first = true;\n        t = jj_consume_token(STRING_LITERAL);\n        // Decode the sting value.\n        String image = t.image;\n        for (int i = 1; i < image.length() - 1; i++) {\n            char c = image.charAt(i);\n            if (c == '\\u005c'')\n                i++;\n            rc.append(c);\n        }\n        return rc.toString();\n    }\n\n    final public PropertyExpression variable() throws ParseException {\n        Token t;\n        PropertyExpression left = null;\n        t = jj_consume_token(ID);\n        left = new PropertyExpression(t.image);\n        return left;\n    }\n\n    private boolean jj_2_1(int xla) {\n        jjLa = xla;\n        jjLastpos = jjScanpos = token;\n        try {\n            return !jj_3_1();\n        } catch (LookaheadSuccess ls) {\n            return true;\n        } finally {\n            jj_save(0, xla);\n        }\n    }\n\n    private boolean jj_2_2(int xla) {\n        jjLa = xla;\n        jjLastpos = jjScanpos = token;\n        try {\n            return !jj_3_2();\n        } catch (LookaheadSuccess ls) {\n            return true;\n        } finally {\n            jj_save(1, xla);\n        }\n    }\n\n    private boolean jj_2_3(int xla) {\n        jjLa = xla;\n        jjLastpos = jjScanpos = token;\n        try {\n            return !jj_3_3();\n        } catch (LookaheadSuccess ls) {\n            return true;\n        } finally {\n            jj_save(2, xla);\n        }\n    }\n\n    private boolean jj_2_4(int xla) {\n        jjLa = xla;\n        jjLastpos = jjScanpos = token;\n        try {\n            return !jj_3_4();\n        } catch (LookaheadSuccess ls) {\n            return true;\n        } finally {\n            jj_save(3, xla);\n        }\n    }\n\n    private boolean jj_2_5(int xla) {\n        jjLa = xla;\n        jjLastpos = jjScanpos = token;\n        try {\n            return !jj_3_5();\n        } catch (LookaheadSuccess ls) {\n            return true;\n        } finally {\n            jj_save(4, xla);\n        }\n    }\n\n    private boolean jj_2_6(int xla) {\n        jjLa = xla;\n        jjLastpos = jjScanpos = token;\n        try {\n            return !jj_3_6();\n        } catch (LookaheadSuccess ls) {\n            return true;\n        } finally {\n            jj_save(5, xla);\n        }\n    }\n\n    private boolean jj_2_7(int xla) {\n        jjLa = xla;\n        jjLastpos = jjScanpos = token;\n        try {\n            return !jj_3_7();\n        } catch (LookaheadSuccess ls) {\n            return true;\n        } finally {\n            jj_save(6, xla);\n        }\n    }\n\n    private boolean jj_3R_34() {\n        if (jj_scan_token(26)) return true;\n        if (jj_3R_30()) return true;\n        return false;\n    }\n\n    private boolean jj_3R_43() {\n        if (jj_scan_token(BETWEEN)) return true;\n        if (jj_3R_7()) return true;\n        if (jj_scan_token(AND)) return true;\n        if (jj_3R_7()) return true;\n        return false;\n    }\n\n    private boolean jj_3R_31() {\n        Token xsp;\n        xsp = jjScanpos;\n        if (jj_3R_33()) {\n            jjScanpos = xsp;\n            if (jj_3R_34()) {\n                jjScanpos = xsp;\n                if (jj_3_1()) {\n                    jjScanpos = xsp;\n                    if (jj_3R_35()) return true;\n                }\n            }\n        }\n        return false;\n    }\n\n    private boolean jj_3R_33() {\n        if (jj_scan_token(25)) return true;\n        if (jj_3R_30()) return true;\n        return false;\n    }\n\n    private boolean jj_3_4() {\n        if (jj_scan_token(NOT)) return true;\n        if (jj_scan_token(ENDSWITH)) return true;\n        if (jj_3R_27()) return true;\n        return false;\n    }\n\n    private boolean jj_3R_15() {\n        if (jj_scan_token(31)) return true;\n        if (jj_3R_18()) return true;\n        if (jj_scan_token(33)) return true;\n        return false;\n    }\n\n    private boolean jj_3R_14() {\n        if (jj_3R_17()) return true;\n        return false;\n    }\n\n    private boolean jj_3R_13() {\n        if (jj_3R_16()) return true;\n        return false;\n    }\n\n    private boolean jj_3R_42() {\n        if (jj_scan_token(ENDSWITH)) return true;\n        if (jj_3R_27()) return true;\n        return false;\n    }\n\n    private boolean jj_3R_17() {\n        if (jj_scan_token(ID)) return true;\n        return false;\n    }\n\n    private boolean jj_3R_12() {\n        Token xsp;\n        xsp = jjScanpos;\n        if (jj_3R_13()) {\n            jjScanpos = xsp;\n            if (jj_3R_14()) {\n                jjScanpos = xsp;\n                if (jj_3R_15()) return true;\n            }\n        }\n        return false;\n    }\n\n    private boolean jj_3R_28() {\n        if (jj_3R_30()) return true;\n        Token xsp;\n        while (true) {\n            xsp = jjScanpos;\n            if (jj_3R_31()) {\n                jjScanpos = xsp;\n                break;\n            }\n        }\n        return false;\n    }\n\n    private boolean jj_3_3() {\n        if (jj_scan_token(NOT)) return true;\n        if (jj_scan_token(STARTSWITH)) return true;\n        if (jj_3R_27()) return true;\n        return false;\n    }\n\n    private boolean jj_3R_41() {\n        if (jj_scan_token(STARTSWITH)) return true;\n        if (jj_3R_27()) return true;\n        return false;\n    }\n\n    private boolean jj_3R_11() {\n        if (jj_3R_12()) return true;\n        return false;\n    }\n\n    private boolean jj_3R_29() {\n        if (jj_scan_token(AND)) return true;\n        if (jj_3R_28()) return true;\n        return false;\n    }\n\n    private boolean jj_3_7() {\n        if (jj_scan_token(34)) return true;\n        if (jj_3R_7()) return true;\n        return false;\n    }\n\n    private boolean jj_3_2() {\n        if (jj_scan_token(NOT)) return true;\n        if (jj_scan_token(CONTAINS)) return true;\n        if (jj_3R_27()) return true;\n        return false;\n    }\n\n    private boolean jj_3R_10() {\n        if (jj_scan_token(NOT)) return true;\n        if (jj_3R_7()) return true;\n        return false;\n    }\n\n    private boolean jj_3R_40() {\n        if (jj_scan_token(CONTAINS)) return true;\n        if (jj_3R_27()) return true;\n        return false;\n    }\n\n    private boolean jj_3R_9() {\n        if (jj_scan_token(35)) return true;\n        if (jj_3R_7()) return true;\n        return false;\n    }\n\n    private boolean jj_3R_27() {\n        if (jj_scan_token(STRING_LITERAL)) return true;\n        return false;\n    }\n\n    private boolean jj_3R_25() {\n        if (jj_3R_28()) return true;\n        Token xsp;\n        while (true) {\n            xsp = jjScanpos;\n            if (jj_3R_29()) {\n                jjScanpos = xsp;\n                break;\n            }\n        }\n        return false;\n    }\n\n    private boolean jj_3R_8() {\n        if (jj_scan_token(34)) return true;\n        if (jj_3R_7()) return true;\n        return false;\n    }\n\n    private boolean jj_3R_39() {\n        if (jj_scan_token(30)) return true;\n        if (jj_3R_7()) return true;\n        return false;\n    }\n\n    private boolean jj_3R_7() {\n        Token xsp;\n        xsp = jjScanpos;\n        if (jj_3R_8()) {\n            jjScanpos = xsp;\n            if (jj_3R_9()) {\n                jjScanpos = xsp;\n                if (jj_3R_10()) {\n                    jjScanpos = xsp;\n                    if (jj_3R_11()) return true;\n                }\n            }\n        }\n        return false;\n    }\n\n    private boolean jj_3R_38() {\n        if (jj_scan_token(29)) return true;\n        if (jj_3R_7()) return true;\n        return false;\n    }\n\n    private boolean jj_3R_46() {\n        if (jj_scan_token(32)) return true;\n        if (jj_3R_27()) return true;\n        return false;\n    }\n\n    private boolean jj_3R_26() {\n        if (jj_scan_token(OR)) return true;\n        if (jj_3R_25()) return true;\n        return false;\n    }\n\n    private boolean jj_3R_37() {\n        if (jj_scan_token(28)) return true;\n        if (jj_3R_7()) return true;\n        return false;\n    }\n\n    private boolean jj_3R_24() {\n        if (jj_scan_token(NULL)) return true;\n        return false;\n    }\n\n    private boolean jj_3R_36() {\n        if (jj_scan_token(27)) return true;\n        if (jj_3R_7()) return true;\n        return false;\n    }\n\n    private boolean jj_3R_32() {\n        Token xsp;\n        xsp = jjScanpos;\n        if (jj_3R_36()) {\n            jjScanpos = xsp;\n            if (jj_3R_37()) {\n                jjScanpos = xsp;\n                if (jj_3R_38()) {\n                    jjScanpos = xsp;\n                    if (jj_3R_39()) {\n                        jjScanpos = xsp;\n                        if (jj_3R_40()) {\n                            jjScanpos = xsp;\n                            if (jj_3_2()) {\n                                jjScanpos = xsp;\n                                if (jj_3R_41()) {\n                                    jjScanpos = xsp;\n                                    if (jj_3_3()) {\n                                        jjScanpos = xsp;\n                                        if (jj_3R_42()) {\n                                            jjScanpos = xsp;\n                                            if (jj_3_4()) {\n                                                jjScanpos = xsp;\n                                                if (jj_3R_43()) {\n                                                    jjScanpos = xsp;\n                                                    if (jj_3_5()) {\n                                                        jjScanpos = xsp;\n                                                        if (jj_3R_44()) {\n                                                            jjScanpos = xsp;\n                                                            if (jj_3_6()) return true;\n                                                        }\n                                                    }\n                                                }\n                                            }\n                                        }\n                                    }\n                                }\n                            }\n                        }\n                    }\n                }\n            }\n        }\n        return false;\n    }\n\n    private boolean jj_3R_23() {\n        if (jj_scan_token(FALSE)) return true;\n        return false;\n    }\n\n    private boolean jj_3R_18() {\n        if (jj_3R_25()) return true;\n        Token xsp;\n        while (true) {\n            xsp = jjScanpos;\n            if (jj_3R_26()) {\n                jjScanpos = xsp;\n                break;\n            }\n        }\n        return false;\n    }\n\n    private boolean jj_3R_22() {\n        if (jj_scan_token(TRUE)) return true;\n        return false;\n    }\n\n    private boolean jj_3_6() {\n        if (jj_scan_token(NOT)) return true;\n        if (jj_scan_token(IN)) return true;\n        if (jj_scan_token(31)) return true;\n        if (jj_3R_27()) return true;\n        Token xsp;\n        while (true) {\n            xsp = jjScanpos;\n            if (jj_3R_46()) {\n                jjScanpos = xsp;\n                break;\n            }\n        }\n        if (jj_scan_token(33)) return true;\n        return false;\n    }\n\n    private boolean jj_3R_45() {\n        if (jj_scan_token(32)) return true;\n        if (jj_3R_27()) return true;\n        return false;\n    }\n\n    private boolean jj_3R_30() {\n        if (jj_3R_7()) return true;\n        Token xsp;\n        while (true) {\n            xsp = jjScanpos;\n            if (jj_3R_32()) {\n                jjScanpos = xsp;\n                break;\n            }\n        }\n        return false;\n    }\n\n    private boolean jj_3R_21() {\n        if (jj_scan_token(FLOATING_POINT_LITERAL)) return true;\n        return false;\n    }\n\n    private boolean jj_3R_20() {\n        if (jj_scan_token(DECIMAL_LITERAL)) return true;\n        return false;\n    }\n\n    private boolean jj_3R_35() {\n        if (jj_scan_token(IS)) return true;\n        if (jj_scan_token(NOT)) return true;\n        if (jj_scan_token(NULL)) return true;\n        return false;\n    }\n\n    private boolean jj_3R_44() {\n        if (jj_scan_token(IN)) return true;\n        if (jj_scan_token(31)) return true;\n        if (jj_3R_27()) return true;\n        Token xsp;\n        while (true) {\n            xsp = jjScanpos;\n            if (jj_3R_45()) {\n                jjScanpos = xsp;\n                break;\n            }\n        }\n        if (jj_scan_token(33)) return true;\n        return false;\n    }\n\n    private boolean jj_3R_19() {\n        if (jj_3R_27()) return true;\n        return false;\n    }\n\n    private boolean jj_3_1() {\n        if (jj_scan_token(IS)) return true;\n        if (jj_scan_token(NULL)) return true;\n        return false;\n    }\n\n    private boolean jj_3R_16() {\n        Token xsp;\n        xsp = jjScanpos;\n        if (jj_3R_19()) {\n            jjScanpos = xsp;\n            if (jj_3R_20()) {\n                jjScanpos = xsp;\n                if (jj_3R_21()) {\n                    jjScanpos = xsp;\n                    if (jj_3R_22()) {\n                        jjScanpos = xsp;\n                        if (jj_3R_23()) {\n                            jjScanpos = xsp;\n                            if (jj_3R_24()) return true;\n                        }\n                    }\n                }\n            }\n        }\n        return false;\n    }\n\n    private boolean jj_3_5() {\n        if (jj_scan_token(NOT)) return true;\n        if (jj_scan_token(BETWEEN)) return true;\n        if (jj_3R_7()) return true;\n        if (jj_scan_token(AND)) return true;\n        if (jj_3R_7()) return true;\n        return false;\n    }\n\n    /**\n     * Generated Token Manager.\n     */\n    public SelectorParserTokenManager tokenSource;\n    SimpleCharStream jjInputStream;\n    /**\n     * Current token.\n     */\n    public Token token;\n    /**\n     * Next token.\n     */\n    public Token jjNt;\n    private int jjNtk;\n    private Token jjScanpos, jjLastpos;\n    private int jjLa;\n    private int jjGen;\n    final private int[] jjLa1 = new int[16];\n    static private int[] jjLa10;\n    static private int[] jjLa11;\n\n    static {\n        jj_la1_init_0();\n        jj_la1_init_1();\n    }\n\n    private static void jj_la1_init_0() {\n        jjLa10 = new int[]{0x400, 0x200, 0x6010000, 0x6000000, 0x10000, 0x780e1900, 0x0, 0x0, 0x78020000, 0x40000, 0x80000, 0x800, 0x1000, 0x81b0e100, 0x81b0e000, 0xb0e000,};\n    }\n\n    private static void jj_la1_init_1() {\n        jjLa11 = new int[]{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8, 0x0, 0x0,};\n    }\n\n    final private JJCalls[] jj2Rtns = new JJCalls[7];\n    private boolean jjRescan = false;\n    private int jjGc = 0;\n\n    /**\n     * Constructor with InputStream.\n     */\n    public SelectorParser(java.io.InputStream stream) {\n        this(stream, null);\n    }\n\n    /**\n     * Constructor with InputStream and supplied encoding\n     */\n    public SelectorParser(java.io.InputStream stream, String encoding) {\n        try {\n            jjInputStream = new SimpleCharStream(stream, encoding, 1, 1);\n        } catch (java.io.UnsupportedEncodingException e) {\n            throw new RuntimeException(e);\n        }\n        tokenSource = new SelectorParserTokenManager(jjInputStream);\n        token = new Token();\n        jjNtk = -1;\n        jjGen = 0;\n        for (int i = 0; i < 16; i++) jjLa1[i] = -1;\n        for (int i = 0; i < jj2Rtns.length; i++) jj2Rtns[i] = new JJCalls();\n    }\n\n    /**\n     * Reinitialise.\n     */\n    public void ReInit(java.io.InputStream stream) {\n        ReInit(stream, null);\n    }\n\n    /**\n     * Reinitialise.\n     */\n    public void ReInit(java.io.InputStream stream, String encoding) {\n        try {\n            jjInputStream.ReInit(stream, encoding, 1, 1);\n        } catch (java.io.UnsupportedEncodingException e) {\n            throw new RuntimeException(e);\n        }\n        tokenSource.ReInit(jjInputStream);\n        token = new Token();\n        jjNtk = -1;\n        jjGen = 0;\n        for (int i = 0; i < 16; i++) jjLa1[i] = -1;\n        for (int i = 0; i < jj2Rtns.length; i++) jj2Rtns[i] = new JJCalls();\n    }\n\n    /**\n     * Constructor.\n     */\n    public SelectorParser(java.io.Reader stream) {\n        jjInputStream = new SimpleCharStream(stream, 1, 1);\n        tokenSource = new SelectorParserTokenManager(jjInputStream);\n        token = new Token();\n        jjNtk = -1;\n        jjGen = 0;\n        for (int i = 0; i < 16; i++) jjLa1[i] = -1;\n        for (int i = 0; i < jj2Rtns.length; i++) jj2Rtns[i] = new JJCalls();\n    }\n\n    /**\n     * Reinitialise.\n     */\n    public void ReInit(java.io.Reader stream) {\n        jjInputStream.ReInit(stream, 1, 1);\n        tokenSource.ReInit(jjInputStream);\n        token = new Token();\n        jjNtk = -1;\n        jjGen = 0;\n        for (int i = 0; i < 16; i++) jjLa1[i] = -1;\n        for (int i = 0; i < jj2Rtns.length; i++) jj2Rtns[i] = new JJCalls();\n    }\n\n    /**\n     * Constructor with generated Token Manager.\n     */\n    public SelectorParser(SelectorParserTokenManager tm) {\n        tokenSource = tm;\n        token = new Token();\n        jjNtk = -1;\n        jjGen = 0;\n        for (int i = 0; i < 16; i++) jjLa1[i] = -1;\n        for (int i = 0; i < jj2Rtns.length; i++) jj2Rtns[i] = new JJCalls();\n    }\n\n    /**\n     * Reinitialise.\n     */\n    public void ReInit(SelectorParserTokenManager tm) {\n        tokenSource = tm;\n        token = new Token();\n        jjNtk = -1;\n        jjGen = 0;\n        for (int i = 0; i < 16; i++) jjLa1[i] = -1;\n        for (int i = 0; i < jj2Rtns.length; i++) jj2Rtns[i] = new JJCalls();\n    }\n\n    private Token jj_consume_token(int kind) throws ParseException {\n        Token oldToken;\n        if ((oldToken = token).next != null) token = token.next;\n        else token = token.next = tokenSource.getNextToken();\n        jjNtk = -1;\n        if (token.kind == kind) {\n            jjGen++;\n            if (++jjGc > 100) {\n                jjGc = 0;\n                for (int i = 0; i < jj2Rtns.length; i++) {\n                    JJCalls c = jj2Rtns[i];\n                    while (c != null) {\n                        if (c.gen < jjGen) c.first = null;\n                        c = c.next;\n                    }\n                }\n            }\n            return token;\n        }\n        token = oldToken;\n        jjKind = kind;\n        throw generateParseException();\n    }\n\n    static private final class LookaheadSuccess extends java.lang.Error {\n    }\n\n    final private LookaheadSuccess jjLs = new LookaheadSuccess();\n\n    private boolean jj_scan_token(int kind) {\n        if (jjScanpos == jjLastpos) {\n            jjLa--;\n            if (jjScanpos.next == null) {\n                jjLastpos = jjScanpos = jjScanpos.next = tokenSource.getNextToken();\n            } else {\n                jjLastpos = jjScanpos = jjScanpos.next;\n            }\n        } else {\n            jjScanpos = jjScanpos.next;\n        }\n        if (jjRescan) {\n            int i = 0;\n            Token tok = token;\n            while (tok != null && tok != jjScanpos) {\n                i++;\n                tok = tok.next;\n            }\n            if (tok != null) jj_add_error_token(kind, i);\n        }\n        if (jjScanpos.kind != kind) return true;\n        if (jjLa == 0 && jjScanpos == jjLastpos) throw jjLs;\n        return false;\n    }\n\n\n    /**\n     * Get the next Token.\n     */\n    final public Token getNextToken() {\n        if (token.next != null) token = token.next;\n        else token = token.next = tokenSource.getNextToken();\n        jjNtk = -1;\n        jjGen++;\n        return token;\n    }\n\n    /**\n     * Get the specific Token.\n     */\n    final public Token getToken(int index) {\n        Token t = token;\n        for (int i = 0; i < index; i++) {\n            if (t.next != null) t = t.next;\n            else t = t.next = tokenSource.getNextToken();\n        }\n        return t;\n    }\n\n    private int jj_ntk() {\n        if ((jjNt = token.next) == null)\n            return jjNtk = (token.next = tokenSource.getNextToken()).kind;\n        else\n            return jjNtk = jjNt.kind;\n    }\n\n    private java.util.List<int[]> jjExpentries = new java.util.ArrayList<>();\n    private int[] jjExpentry;\n    private int jjKind = -1;\n    private int[] jjLasttokens = new int[100];\n    private int jjEndpos;\n\n    private void jj_add_error_token(int kind, int pos) {\n        if (pos >= 100) return;\n        if (pos == jjEndpos + 1) {\n            jjLasttokens[jjEndpos++] = kind;\n        } else if (jjEndpos != 0) {\n            jjExpentry = new int[jjEndpos];\n            for (int i = 0; i < jjEndpos; i++) {\n                jjExpentry[i] = jjLasttokens[i];\n            }\n            boolean exists = false;\n            for (java.util.Iterator<?> it = jjExpentries.iterator(); it.hasNext(); ) {\n                exists = true;\n                int[] oldentry = (int[]) (it.next());\n                if (oldentry.length == jjExpentry.length) {\n                    for (int i = 0; i < jjExpentry.length; i++) {\n                        if (oldentry[i] != jjExpentry[i]) {\n                            exists = false;\n                            break;\n                        }\n                    }\n                    if (exists) break;\n                }\n            }\n            if (!exists) jjExpentries.add(jjExpentry);\n            if (pos != 0) jjLasttokens[(jjEndpos = pos) - 1] = kind;\n        }\n    }\n\n    /**\n     * Generate ParseException.\n     */\n    public ParseException generateParseException() {\n        jjExpentries.clear();\n        boolean[] la1tokens = new boolean[36];\n        if (jjKind >= 0) {\n            la1tokens[jjKind] = true;\n            jjKind = -1;\n        }\n        for (int i = 0; i < 16; i++) {\n            if (jjLa1[i] == jjGen) {\n                for (int j = 0; j < 32; j++) {\n                    if ((jjLa10[i] & (1 << j)) != 0) {\n                        la1tokens[j] = true;\n                    }\n                    if ((jjLa11[i] & (1 << j)) != 0) {\n                        la1tokens[32 + j] = true;\n                    }\n                }\n            }\n        }\n        for (int i = 0; i < 36; i++) {\n            if (la1tokens[i]) {\n                jjExpentry = new int[1];\n                jjExpentry[0] = i;\n                jjExpentries.add(jjExpentry);\n            }\n        }\n        jjEndpos = 0;\n        jj_rescan_token();\n        jj_add_error_token(0, 0);\n        int[][] exptokseq = new int[jjExpentries.size()][];\n        for (int i = 0; i < jjExpentries.size(); i++) {\n            exptokseq[i] = jjExpentries.get(i);\n        }\n        return new ParseException(token, exptokseq, TOKEN_IMAGE);\n    }\n\n    /**\n     * Enable tracing.\n     */\n    final public void enable_tracing() {\n    }\n\n    /**\n     * Disable tracing.\n     */\n    final public void disable_tracing() {\n    }\n\n    private void jj_rescan_token() {\n        jjRescan = true;\n        for (int i = 0; i < 7; i++) {\n            try {\n                JJCalls p = jj2Rtns[i];\n                do {\n                    if (p.gen > jjGen) {\n                        jjLa = p.arg;\n                        jjLastpos = jjScanpos = p.first;\n                        switch (i) {\n                            case 0:\n                                jj_3_1();\n                                break;\n                            case 1:\n                                jj_3_2();\n                                break;\n                            case 2:\n                                jj_3_3();\n                                break;\n                            case 3:\n                                jj_3_4();\n                                break;\n                            case 4:\n                                jj_3_5();\n                                break;\n                            case 5:\n                                jj_3_6();\n                                break;\n                            case 6:\n                                jj_3_7();\n                                break;\n                        }\n                    }\n                    p = p.next;\n                } while (p != null);\n            } catch (LookaheadSuccess ls) {\n            }\n        }\n        jjRescan = false;\n    }\n\n    private void jj_save(int index, int xla) {\n        JJCalls p = jj2Rtns[index];\n        while (p.gen > jjGen) {\n            if (p.next == null) {\n                p = p.next = new JJCalls();\n                break;\n            }\n            p = p.next;\n        }\n        p.gen = jjGen + xla - jjLa;\n        p.first = token;\n        p.arg = xla;\n    }\n\n    static final class JJCalls {\n        int gen;\n        Token first;\n        int arg;\n        JJCalls next;\n    }\n\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParser.jj",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/*\n * This file was taken from ActiveMQ activemq-client/src/main/grammar/SelectorParser.jj.\n *\n * There are some modifications:\n * 1. Convert string expressions default;\n * 2. HEX_LITERAL and OCTAL_LITERAL were removed;\n * 3. LIKE, ESCAPE, XPATH and XQUERY were removed;\n * 4. Computation expressions were removed;\n */\n\n// ----------------------------------------------------------------------------\n// OPTIONS\n// ----------------------------------------------------------------------------\noptions {\n  STATIC = false;\n  UNICODE_INPUT = true;\n\n  //ERROR_REPORTING = false;\n}\n\n// ----------------------------------------------------------------------------\n// PARSER\n// ----------------------------------------------------------------------------\n\nPARSER_BEGIN(SelectorParser)\n\npackage org.apache.rocketmq.filter.parser;\n\nimport com.google.common.cache.Cache;\nimport com.google.common.cache.CacheBuilder;\nimport org.apache.rocketmq.filter.expression.BooleanExpression;\nimport org.apache.rocketmq.filter.expression.ComparisonExpression;\nimport org.apache.rocketmq.filter.expression.ConstantExpression;\nimport org.apache.rocketmq.filter.expression.Expression;\nimport org.apache.rocketmq.filter.expression.LogicExpression;\nimport org.apache.rocketmq.filter.expression.MQFilterException;\nimport org.apache.rocketmq.filter.expression.PropertyExpression;\nimport org.apache.rocketmq.filter.expression.UnaryExpression;\n\nimport java.io.StringReader;\nimport java.util.ArrayList;\n\n/**\n * JMS Selector Parser generated by JavaCC\n *\n * Do not edit this .java file directly - it is autogenerated from SelectorParser.jj\n */\npublic class SelectorParser {\n\n    private static final Cache<String, Object> PARSE_CACHE = CacheBuilder.newBuilder().maximumSize(100).build();\n//    private static final String CONVERT_STRING_EXPRESSIONS_PREFIX = \"convert_string_expressions:\";\n\n    public static BooleanExpression parse(String sql) throws MQFilterException {\n//        sql = \"(\"+sql+\")\";\n        Object result = PARSE_CACHE.getIfPresent(sql);\n        if (result instanceof MQFilterException) {\n            throw (MQFilterException) result;\n        } else if (result instanceof BooleanExpression) {\n            return (BooleanExpression) result;\n        } else {\n\n//            boolean convertStringExpressions = false;\n//            if( sql.startsWith(CONVERT_STRING_EXPRESSIONS_PREFIX)) {\n//                convertStringExpressions = true;\n//                sql = sql.substring(CONVERT_STRING_EXPRESSIONS_PREFIX.length());\n//            }\n//            if( convertStringExpressions ) {\n//                ComparisonExpression.CONVERT_STRING_EXPRESSIONS.set(true);\n//            }\n            ComparisonExpression.CONVERT_STRING_EXPRESSIONS.set(true);\n            try {\n\n                BooleanExpression e = new SelectorParser(sql).parse();\n                PARSE_CACHE.put(sql, e);\n                return e;\n            } catch (MQFilterException t) {\n                PARSE_CACHE.put(sql, t);\n                throw t;\n            } finally {\n                ComparisonExpression.CONVERT_STRING_EXPRESSIONS.remove();\n//                if( convertStringExpressions ) {\n//                    ComparisonExpression.CONVERT_STRING_EXPRESSIONS.remove();\n//                }\n            }\n        }\n    }\n\n    public static void clearCache() {\n        PARSE_CACHE.cleanUp();\n    }\n\n    private String sql;\n\n    protected SelectorParser(String sql) {\n        this(new StringReader(sql));\n        this.sql = sql;\n    }\n\n    protected BooleanExpression parse() throws MQFilterException {\n        try {\n            return this.JmsSelector();\n        }\n        catch (Throwable e) {\n            throw new MQFilterException(\"Invalid MessageSelector. \", e);\n        }\n    }\n\n    private BooleanExpression asBooleanExpression(Expression value) throws ParseException  {\n        if (value instanceof BooleanExpression) {\n            return (BooleanExpression) value;\n        }\n        if (value instanceof PropertyExpression) {\n            return UnaryExpression.createBooleanCast( value );\n        }\n        throw new ParseException(\"Expression will not result in a boolean value: \" + value);\n    }\n\n}\n\nPARSER_END(SelectorParser)\n\n// ----------------------------------------------------------------------------\n// Tokens\n// ----------------------------------------------------------------------------\n\n/* White Space */\nSPECIAL_TOKEN :\n{\n  \" \" | \"\\t\" | \"\\n\" | \"\\r\" | \"\\f\"\n}\n\n/* Comments */\nSKIP:\n{\n  <LINE_COMMENT: \"--\" (~[\"\\n\",\"\\r\"])* (\"\\n\"|\"\\r\"|\"\\r\\n\") >\n}\n\nSKIP:\n{\n  <BLOCK_COMMENT: \"/*\" (~[\"*\"])* \"*\" (\"*\" | (~[\"*\",\"/\"] (~[\"*\"])* \"*\"))* \"/\">\n}\n\n/* Reserved Words */\nTOKEN [IGNORE_CASE] :\n{\n    <  NOT     : \"NOT\">\n  | <  AND     : \"AND\">\n  | <  OR      : \"OR\">\n  | <  BETWEEN : \"BETWEEN\">\n  | <  IN      : \"IN\">\n  | <  TRUE    : \"TRUE\" >\n  | <  FALSE   : \"FALSE\" >\n  | <  NULL    : \"NULL\" >\n  | <  IS      : \"IS\" >\n  | <  CONTAINS    : \"CONTAINS\">\n  | <  STARTSWITH  : \"STARTSWITH\">\n  | <  ENDSWITH    : \"ENDSWITH\">\n}\n\n/* Literals */\nTOKEN [IGNORE_CASE] :\n\n{\n\n    < DECIMAL_LITERAL: \"0\" | [\"1\"-\"9\"] ([\"0\"-\"9\"])* ([\"l\",\"L\"])? >\n  | < FLOATING_POINT_LITERAL:\n          ([\"0\"-\"9\"])+ \".\" ([\"0\"-\"9\"])* (<EXPONENT>)? // matches: 5.5 or 5. or 5.5E10 or 5.E10\n        | \".\" ([\"0\"-\"9\"])+ (<EXPONENT>)?              // matches: .5 or .5E10\n        | ([\"0\"-\"9\"])+ <EXPONENT>                     // matches: 5E10\n    >\n  | < #EXPONENT: \"E\" ([\"+\",\"-\"])? ([\"0\"-\"9\"])+ >\n  | < STRING_LITERAL: \"'\" ( (\"''\") | ~[\"'\"] )*  \"'\" >\n}\n\nTOKEN [IGNORE_CASE] :\n{\n    < ID : [\"a\"-\"z\", \"_\", \"$\"] ([\"a\"-\"z\",\"0\"-\"9\",\"_\", \"$\"])* >\n}\n\n// ----------------------------------------------------------------------------\n// Grammar\n// ----------------------------------------------------------------------------\nBooleanExpression JmsSelector() :\n{\n    Expression left=null;\n}\n{\n    (\n        left = orExpression()\n    )\n    {\n        return asBooleanExpression(left);\n    }\n\n}\n\nExpression orExpression() :\n{\n    Expression left;\n    Expression right;\n}\n{\n    (\n        left = andExpression()\n        (\n            <OR> right = andExpression()\n            {\n                left = LogicExpression.createOR(asBooleanExpression(left), asBooleanExpression(right));\n            }\n        )*\n    )\n    {\n        return left;\n    }\n\n}\n\n\nExpression andExpression() :\n{\n    Expression left;\n    Expression right;\n}\n{\n    (\n        left = equalityExpression()\n        (\n            <AND> right = equalityExpression()\n            {\n                left = LogicExpression.createAND(asBooleanExpression(left), asBooleanExpression(right));\n            }\n        )*\n    )\n    {\n        return left;\n    }\n}\n\nExpression equalityExpression() :\n{\n    Expression left;\n    Expression right;\n}\n{\n    (\n        left = comparisonExpression()\n        (\n\n            \"=\" right = comparisonExpression()\n            {\n                left = ComparisonExpression.createEqual(left, right);\n            }\n            |\n            \"<>\" right = comparisonExpression()\n            {\n                left = ComparisonExpression.createNotEqual(left, right);\n            }\n            |\n            LOOKAHEAD(2)\n            <IS> <NULL>\n            {\n                left = ComparisonExpression.createIsNull(left);\n            }\n            |\n            <IS> <NOT> <NULL>\n            {\n                left = ComparisonExpression.createIsNotNull(left);\n            }\n        )*\n    )\n    {\n        return left;\n    }\n}\n\nExpression comparisonExpression() :\n{\n    Expression left;\n    Expression right;\n    Expression low;\n    Expression high;\n    String t, u;\n    boolean not;\n    ArrayList list;\n}\n{\n    (\n        left = unaryExpr()\n        (\n\n                \">\" right = unaryExpr()\n                {\n                    left = ComparisonExpression.createGreaterThan(left, right);\n                }\n            |\n                \">=\" right = unaryExpr()\n                {\n                    left = ComparisonExpression.createGreaterThanEqual(left, right);\n                }\n            |\n                \"<\" right = unaryExpr()\n                {\n                    left = ComparisonExpression.createLessThan(left, right);\n                }\n            |\n                \"<=\" right = unaryExpr()\n                {\n                    left = ComparisonExpression.createLessThanEqual(left, right);\n                }\n            |\n                <CONTAINS> t = stringLitteral()\n                {\n                    left = ComparisonExpression.createContains(left, t);\n                }\n            |\n                LOOKAHEAD(2)\n                <NOT> <CONTAINS> t = stringLitteral()\n                {\n                    left = ComparisonExpression.createNotContains(left, t);\n                }\n            |\n                <STARTSWITH> t = stringLitteral()\n                {\n                    left = ComparisonExpression.createStartsWith(left, t);\n                }\n            |\n                LOOKAHEAD(2)\n                <NOT> <STARTSWITH> t = stringLitteral()\n                {\n                    left = ComparisonExpression.createNotStartsWith(left, t);\n                }\n            |\n                <ENDSWITH> t = stringLitteral()\n                {\n                    left = ComparisonExpression.createEndsWith(left, t);\n                }\n            |\n                LOOKAHEAD(2)\n                <NOT> <ENDSWITH> t = stringLitteral()\n                {\n                    left = ComparisonExpression.createNotEndsWith(left, t);\n                }\n            |\n                <BETWEEN> low = unaryExpr() <AND> high = unaryExpr()\n                {\n                    left = ComparisonExpression.createBetween(left, low, high);\n                }\n            |\n                LOOKAHEAD(2)\n                <NOT> <BETWEEN> low = unaryExpr() <AND> high = unaryExpr()\n                {\n                    left = ComparisonExpression.createNotBetween(left, low, high);\n                }\n            |\n                <IN>\n                \"(\"\n                    t = stringLitteral()\n                    {\n                        list = new ArrayList();\n                        list.add( t );\n                    }\n                    (\n                        \",\"\n                        t = stringLitteral()\n                        {\n                            list.add( t );\n                        }\n\n                    )*\n                \")\"\n                {\n                   left = ComparisonExpression.createInFilter(left, list);\n                }\n            |\n                LOOKAHEAD(2)\n                <NOT> <IN>\n                \"(\"\n                    t = stringLitteral()\n                    {\n                        list = new ArrayList();\n                        list.add( t );\n                    }\n                    (\n                        \",\"\n                        t = stringLitteral()\n                        {\n                            list.add( t );\n                        }\n\n                    )*\n                \")\"\n                {\n                   left = ComparisonExpression.createNotInFilter(left, list);\n                }\n\n        )*\n    )\n    {\n        return left;\n    }\n}\n\nExpression unaryExpr() :\n{\n    String s=null;\n    Expression left=null;\n}\n{\n    (\n        LOOKAHEAD( \"+\" unaryExpr() )\n        \"+\" left=unaryExpr()\n        |\n        \"-\" left=unaryExpr()\n        {\n            left = UnaryExpression.createNegate(left);\n        }\n        |\n        <NOT> left=unaryExpr()\n        {\n            left = UnaryExpression.createNOT( asBooleanExpression(left) );\n        }\n        |\n        left = primaryExpr()\n    )\n    {\n        return left;\n    }\n\n}\n\nExpression primaryExpr() :\n{\n    Expression left=null;\n}\n{\n    (\n        left = literal()\n        |\n        left = variable()\n        |\n        \"(\" left = orExpression() \")\"\n    )\n    {\n        return left;\n    }\n}\n\n\n\nConstantExpression literal() :\n{\n    Token t;\n    String s;\n    ConstantExpression left=null;\n}\n{\n    (\n        (\n            s = stringLitteral()\n            {\n                left = new ConstantExpression(s);\n            }\n        )\n        |\n        (\n            t = <DECIMAL_LITERAL>\n            {\n                left = ConstantExpression.createFromDecimal(t.image);\n            }\n        )\n        |\n        (\n            t = <FLOATING_POINT_LITERAL>\n            {\n                left = ConstantExpression.createFloat(t.image);\n            }\n        )\n        |\n        (\n            <TRUE>\n            {\n                left = ConstantExpression.TRUE;\n            }\n        )\n        |\n        (\n            <FALSE>\n            {\n                left = ConstantExpression.FALSE;\n            }\n        )\n        |\n        (\n            <NULL>\n            {\n                left = ConstantExpression.NULL;\n            }\n        )\n    )\n    {\n        return left;\n    }\n}\n\nString stringLitteral() :\n{\n    Token t;\n    StringBuffer rc = new StringBuffer();\n    boolean first=true;\n}\n{\n    t = <STRING_LITERAL>\n    {\n        // Decode the sting value.\n        String image = t.image;\n        for( int i=1; i < image.length()-1; i++ ) {\n            char c = image.charAt(i);\n            if( c == '\\'' )\n                i++;\n            rc.append(c);\n        }\n        return rc.toString();\n    }\n}\n\nPropertyExpression variable() :\n{\n    Token t;\n    PropertyExpression left=null;\n}\n{\n    (\n        t = <ID>\n        {\n            left = new PropertyExpression(t.image);\n        }\n    )\n    {\n        return left;\n    }\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParserConstants.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/* Generated By:JavaCC: Do not edit this line. SelectorParserConstants.java */\npackage org.apache.rocketmq.filter.parser;\n\n/**\n * Token literal values and constants.\n * Generated by org.javacc.parser.OtherFilesGen#start()\n */\npublic interface SelectorParserConstants {\n\n    /**\n     * End of File.\n     */\n    int EOF = 0;\n    /**\n     * RegularExpression Id.\n     */\n    int LINE_COMMENT = 6;\n    /**\n     * RegularExpression Id.\n     */\n    int BLOCK_COMMENT = 7;\n    /**\n     * RegularExpression Id.\n     */\n    int NOT = 8;\n    /**\n     * RegularExpression Id.\n     */\n    int AND = 9;\n    /**\n     * RegularExpression Id.\n     */\n    int OR = 10;\n    /**\n     * RegularExpression Id.\n     */\n    int BETWEEN = 11;\n    /**\n     * RegularExpression Id.\n     */\n    int IN = 12;\n    /**\n     * RegularExpression Id.\n     */\n    int TRUE = 13;\n    /**\n     * RegularExpression Id.\n     */\n    int FALSE = 14;\n    /**\n     * RegularExpression Id.\n     */\n    int NULL = 15;\n    /**\n     * RegularExpression Id.\n     */\n    int IS = 16;\n    /**\n     * RegularExpression Id.\n     */\n    int CONTAINS = 17;\n    /**\n     * RegularExpression Id.\n     */\n    int STARTSWITH = 18;\n    /**\n     * RegularExpression Id.\n     */\n    int ENDSWITH = 19;\n    /**\n     * RegularExpression Id.\n     */\n    int DECIMAL_LITERAL = 20;\n    /**\n     * RegularExpression Id.\n     */\n    int FLOATING_POINT_LITERAL = 21;\n    /**\n     * RegularExpression Id.\n     */\n    int EXPONENT = 22;\n    /**\n     * RegularExpression Id.\n     */\n    int STRING_LITERAL = 23;\n    /**\n     * RegularExpression Id.\n     */\n    int ID = 24;\n\n    /**\n     * Lexical state.\n     */\n    int DEFAULT = 0;\n\n    /**\n     * Literal token values.\n     */\n    String[] TOKEN_IMAGE = {\n        \"<EOF>\",\n        \"\\\" \\\"\",\n        \"\\\"\\\\t\\\"\",\n        \"\\\"\\\\n\\\"\",\n        \"\\\"\\\\r\\\"\",\n        \"\\\"\\\\f\\\"\",\n        \"<LINE_COMMENT>\",\n        \"<BLOCK_COMMENT>\",\n        \"\\\"NOT\\\"\",\n        \"\\\"AND\\\"\",\n        \"\\\"OR\\\"\",\n        \"\\\"BETWEEN\\\"\",\n        \"\\\"IN\\\"\",\n        \"\\\"TRUE\\\"\",\n        \"\\\"FALSE\\\"\",\n        \"\\\"NULL\\\"\",\n        \"\\\"IS\\\"\",\n        \"\\\"CONTAINS\\\"\",\n        \"\\\"STARTSWITH\\\"\",\n        \"\\\"ENDSWITH\\\"\",\n        \"<DECIMAL_LITERAL>\",\n        \"<FLOATING_POINT_LITERAL>\",\n        \"<EXPONENT>\",\n        \"<STRING_LITERAL>\",\n        \"<ID>\",\n        \"\\\"=\\\"\",\n        \"\\\"<>\\\"\",\n        \"\\\">\\\"\",\n        \"\\\">=\\\"\",\n        \"\\\"<\\\"\",\n        \"\\\"<=\\\"\",\n        \"\\\"(\\\"\",\n        \"\\\",\\\"\",\n        \"\\\")\\\"\",\n        \"\\\"+\\\"\",\n        \"\\\"-\\\"\",\n    };\n\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParserTokenManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/* Generated By:JavaCC: Do not edit this line. SelectorParserTokenManager.java */\npackage org.apache.rocketmq.filter.parser;\n\n/**\n * Token Manager.\n */\npublic class SelectorParserTokenManager implements SelectorParserConstants {\n\n    /**\n     * Debug output.\n     */\n    public java.io.PrintStream debugStream = System.out;\n\n    /**\n     * Set debug output.\n     */\n    public void setDebugStream(java.io.PrintStream ds) {\n        debugStream = ds;\n    }\n\n    private int jjStopAtPos(int pos, int kind) {\n        jjmatchedKind = kind;\n        jjmatchedPos = pos;\n        return pos + 1;\n    }\n\n    private int jjMoveStringLiteralDfa0_0() {\n        switch (curChar) {\n            case 9:\n                jjmatchedKind = 2;\n                return jjMoveNfa_0(5, 0);\n            case 10:\n                jjmatchedKind = 3;\n                return jjMoveNfa_0(5, 0);\n            case 12:\n                jjmatchedKind = 5;\n                return jjMoveNfa_0(5, 0);\n            case 13:\n                jjmatchedKind = 4;\n                return jjMoveNfa_0(5, 0);\n            case 32:\n                jjmatchedKind = 1;\n                return jjMoveNfa_0(5, 0);\n            case 40:\n                jjmatchedKind = 31;\n                return jjMoveNfa_0(5, 0);\n            case 41:\n                jjmatchedKind = 33;\n                return jjMoveNfa_0(5, 0);\n            case 43:\n                jjmatchedKind = 34;\n                return jjMoveNfa_0(5, 0);\n            case 44:\n                jjmatchedKind = 32;\n                return jjMoveNfa_0(5, 0);\n            case 45:\n                jjmatchedKind = 35;\n                return jjMoveNfa_0(5, 0);\n            case 60:\n                jjmatchedKind = 29;\n                return jjMoveStringLiteralDfa1_0(0x44000000L);\n            case 61:\n                jjmatchedKind = 25;\n                return jjMoveNfa_0(5, 0);\n            case 62:\n                jjmatchedKind = 27;\n                return jjMoveStringLiteralDfa1_0(0x10000000L);\n            case 65:\n                return jjMoveStringLiteralDfa1_0(0x200L);\n            case 66:\n                return jjMoveStringLiteralDfa1_0(0x800L);\n            case 67:\n                return jjMoveStringLiteralDfa1_0(0x20000L);\n            case 69:\n                return jjMoveStringLiteralDfa1_0(0x80000L);\n            case 70:\n                return jjMoveStringLiteralDfa1_0(0x4000L);\n            case 73:\n                return jjMoveStringLiteralDfa1_0(0x11000L);\n            case 78:\n                return jjMoveStringLiteralDfa1_0(0x8100L);\n            case 79:\n                return jjMoveStringLiteralDfa1_0(0x400L);\n            case 83:\n                return jjMoveStringLiteralDfa1_0(0x40000L);\n            case 84:\n                return jjMoveStringLiteralDfa1_0(0x2000L);\n            case 97:\n                return jjMoveStringLiteralDfa1_0(0x200L);\n            case 98:\n                return jjMoveStringLiteralDfa1_0(0x800L);\n            case 99:\n                return jjMoveStringLiteralDfa1_0(0x20000L);\n            case 101:\n                return jjMoveStringLiteralDfa1_0(0x80000L);\n            case 102:\n                return jjMoveStringLiteralDfa1_0(0x4000L);\n            case 105:\n                return jjMoveStringLiteralDfa1_0(0x11000L);\n            case 110:\n                return jjMoveStringLiteralDfa1_0(0x8100L);\n            case 111:\n                return jjMoveStringLiteralDfa1_0(0x400L);\n            case 115:\n                return jjMoveStringLiteralDfa1_0(0x40000L);\n            case 116:\n                return jjMoveStringLiteralDfa1_0(0x2000L);\n            default:\n                return jjMoveNfa_0(5, 0);\n        }\n    }\n\n    private int jjMoveStringLiteralDfa1_0(long active0) {\n        try {\n            curChar = inputStream.readChar();\n        } catch (java.io.IOException e) {\n            return jjMoveNfa_0(5, 0);\n        }\n        switch (curChar) {\n            case 61:\n                if ((active0 & 0x10000000L) != 0L) {\n                    jjmatchedKind = 28;\n                    jjmatchedPos = 1;\n                } else if ((active0 & 0x40000000L) != 0L) {\n                    jjmatchedKind = 30;\n                    jjmatchedPos = 1;\n                }\n                break;\n            case 62:\n                if ((active0 & 0x4000000L) != 0L) {\n                    jjmatchedKind = 26;\n                    jjmatchedPos = 1;\n                }\n                break;\n            case 65:\n                return jjMoveStringLiteralDfa2_0(active0, 0x4000L);\n            case 69:\n                return jjMoveStringLiteralDfa2_0(active0, 0x800L);\n            case 78:\n                if ((active0 & 0x1000L) != 0L) {\n                    jjmatchedKind = 12;\n                    jjmatchedPos = 1;\n                }\n                return jjMoveStringLiteralDfa2_0(active0, 0x80200L);\n            case 79:\n                return jjMoveStringLiteralDfa2_0(active0, 0x20100L);\n            case 82:\n                if ((active0 & 0x400L) != 0L) {\n                    jjmatchedKind = 10;\n                    jjmatchedPos = 1;\n                }\n                return jjMoveStringLiteralDfa2_0(active0, 0x2000L);\n            case 83:\n                if ((active0 & 0x10000L) != 0L) {\n                    jjmatchedKind = 16;\n                    jjmatchedPos = 1;\n                }\n                break;\n            case 84:\n                return jjMoveStringLiteralDfa2_0(active0, 0x40000L);\n            case 85:\n                return jjMoveStringLiteralDfa2_0(active0, 0x8000L);\n            case 97:\n                return jjMoveStringLiteralDfa2_0(active0, 0x4000L);\n            case 101:\n                return jjMoveStringLiteralDfa2_0(active0, 0x800L);\n            case 110:\n                if ((active0 & 0x1000L) != 0L) {\n                    jjmatchedKind = 12;\n                    jjmatchedPos = 1;\n                }\n                return jjMoveStringLiteralDfa2_0(active0, 0x80200L);\n            case 111:\n                return jjMoveStringLiteralDfa2_0(active0, 0x20100L);\n            case 114:\n                if ((active0 & 0x400L) != 0L) {\n                    jjmatchedKind = 10;\n                    jjmatchedPos = 1;\n                }\n                return jjMoveStringLiteralDfa2_0(active0, 0x2000L);\n            case 115:\n                if ((active0 & 0x10000L) != 0L) {\n                    jjmatchedKind = 16;\n                    jjmatchedPos = 1;\n                }\n                break;\n            case 116:\n                return jjMoveStringLiteralDfa2_0(active0, 0x40000L);\n            case 117:\n                return jjMoveStringLiteralDfa2_0(active0, 0x8000L);\n            default:\n                break;\n        }\n        return jjMoveNfa_0(5, 1);\n    }\n\n    private int jjMoveStringLiteralDfa2_0(long old0, long active0) {\n        if (((active0 &= old0)) == 0L)\n            return jjMoveNfa_0(5, 1);\n        try {\n            curChar = inputStream.readChar();\n        } catch (java.io.IOException e) {\n            return jjMoveNfa_0(5, 1);\n        }\n        switch (curChar) {\n            case 65:\n                return jjMoveStringLiteralDfa3_0(active0, 0x40000L);\n            case 68:\n                if ((active0 & 0x200L) != 0L) {\n                    jjmatchedKind = 9;\n                    jjmatchedPos = 2;\n                }\n                return jjMoveStringLiteralDfa3_0(active0, 0x80000L);\n            case 76:\n                return jjMoveStringLiteralDfa3_0(active0, 0xc000L);\n            case 78:\n                return jjMoveStringLiteralDfa3_0(active0, 0x20000L);\n            case 84:\n                if ((active0 & 0x100L) != 0L) {\n                    jjmatchedKind = 8;\n                    jjmatchedPos = 2;\n                }\n                return jjMoveStringLiteralDfa3_0(active0, 0x800L);\n            case 85:\n                return jjMoveStringLiteralDfa3_0(active0, 0x2000L);\n            case 97:\n                return jjMoveStringLiteralDfa3_0(active0, 0x40000L);\n            case 100:\n                if ((active0 & 0x200L) != 0L) {\n                    jjmatchedKind = 9;\n                    jjmatchedPos = 2;\n                }\n                return jjMoveStringLiteralDfa3_0(active0, 0x80000L);\n            case 108:\n                return jjMoveStringLiteralDfa3_0(active0, 0xc000L);\n            case 110:\n                return jjMoveStringLiteralDfa3_0(active0, 0x20000L);\n            case 116:\n                if ((active0 & 0x100L) != 0L) {\n                    jjmatchedKind = 8;\n                    jjmatchedPos = 2;\n                }\n                return jjMoveStringLiteralDfa3_0(active0, 0x800L);\n            case 117:\n                return jjMoveStringLiteralDfa3_0(active0, 0x2000L);\n            default:\n                break;\n        }\n        return jjMoveNfa_0(5, 2);\n    }\n\n    private int jjMoveStringLiteralDfa3_0(long old0, long active0) {\n        if (((active0 &= old0)) == 0L)\n            return jjMoveNfa_0(5, 2);\n        try {\n            curChar = inputStream.readChar();\n        } catch (java.io.IOException e) {\n            return jjMoveNfa_0(5, 2);\n        }\n        switch (curChar) {\n            case 69:\n                if ((active0 & 0x2000L) != 0L) {\n                    jjmatchedKind = 13;\n                    jjmatchedPos = 3;\n                }\n                break;\n            case 76:\n                if ((active0 & 0x8000L) != 0L) {\n                    jjmatchedKind = 15;\n                    jjmatchedPos = 3;\n                }\n                break;\n            case 82:\n                return jjMoveStringLiteralDfa4_0(active0, 0x40000L);\n            case 83:\n                return jjMoveStringLiteralDfa4_0(active0, 0x84000L);\n            case 84:\n                return jjMoveStringLiteralDfa4_0(active0, 0x20000L);\n            case 87:\n                return jjMoveStringLiteralDfa4_0(active0, 0x800L);\n            case 101:\n                if ((active0 & 0x2000L) != 0L) {\n                    jjmatchedKind = 13;\n                    jjmatchedPos = 3;\n                }\n                break;\n            case 108:\n                if ((active0 & 0x8000L) != 0L) {\n                    jjmatchedKind = 15;\n                    jjmatchedPos = 3;\n                }\n                break;\n            case 114:\n                return jjMoveStringLiteralDfa4_0(active0, 0x40000L);\n            case 115:\n                return jjMoveStringLiteralDfa4_0(active0, 0x84000L);\n            case 116:\n                return jjMoveStringLiteralDfa4_0(active0, 0x20000L);\n            case 119:\n                return jjMoveStringLiteralDfa4_0(active0, 0x800L);\n            default:\n                break;\n        }\n        return jjMoveNfa_0(5, 3);\n    }\n\n    private int jjMoveStringLiteralDfa4_0(long old0, long active0) {\n        if (((active0 &= old0)) == 0L)\n            return jjMoveNfa_0(5, 3);\n        try {\n            curChar = inputStream.readChar();\n        } catch (java.io.IOException e) {\n            return jjMoveNfa_0(5, 3);\n        }\n        switch (curChar) {\n            case 65:\n                return jjMoveStringLiteralDfa5_0(active0, 0x20000L);\n            case 69:\n                if ((active0 & 0x4000L) != 0L) {\n                    jjmatchedKind = 14;\n                    jjmatchedPos = 4;\n                }\n                return jjMoveStringLiteralDfa5_0(active0, 0x800L);\n            case 84:\n                return jjMoveStringLiteralDfa5_0(active0, 0x40000L);\n            case 87:\n                return jjMoveStringLiteralDfa5_0(active0, 0x80000L);\n            case 97:\n                return jjMoveStringLiteralDfa5_0(active0, 0x20000L);\n            case 101:\n                if ((active0 & 0x4000L) != 0L) {\n                    jjmatchedKind = 14;\n                    jjmatchedPos = 4;\n                }\n                return jjMoveStringLiteralDfa5_0(active0, 0x800L);\n            case 116:\n                return jjMoveStringLiteralDfa5_0(active0, 0x40000L);\n            case 119:\n                return jjMoveStringLiteralDfa5_0(active0, 0x80000L);\n            default:\n                break;\n        }\n        return jjMoveNfa_0(5, 4);\n    }\n\n    private int jjMoveStringLiteralDfa5_0(long old0, long active0) {\n        if (((active0 &= old0)) == 0L)\n            return jjMoveNfa_0(5, 4);\n        try {\n            curChar = inputStream.readChar();\n        } catch (java.io.IOException e) {\n            return jjMoveNfa_0(5, 4);\n        }\n        switch (curChar) {\n            case 69:\n                return jjMoveStringLiteralDfa6_0(active0, 0x800L);\n            case 73:\n                return jjMoveStringLiteralDfa6_0(active0, 0xa0000L);\n            case 83:\n                return jjMoveStringLiteralDfa6_0(active0, 0x40000L);\n            case 101:\n                return jjMoveStringLiteralDfa6_0(active0, 0x800L);\n            case 105:\n                return jjMoveStringLiteralDfa6_0(active0, 0xa0000L);\n            case 115:\n                return jjMoveStringLiteralDfa6_0(active0, 0x40000L);\n            default:\n                break;\n        }\n        return jjMoveNfa_0(5, 5);\n    }\n\n    private int jjMoveStringLiteralDfa6_0(long old0, long active0) {\n        if (((active0 &= old0)) == 0L)\n            return jjMoveNfa_0(5, 5);\n        try {\n            curChar = inputStream.readChar();\n        } catch (java.io.IOException e) {\n            return jjMoveNfa_0(5, 5);\n        }\n        switch (curChar) {\n            case 78:\n                if ((active0 & 0x800L) != 0L) {\n                    jjmatchedKind = 11;\n                    jjmatchedPos = 6;\n                }\n                return jjMoveStringLiteralDfa7_0(active0, 0x20000L);\n            case 84:\n                return jjMoveStringLiteralDfa7_0(active0, 0x80000L);\n            case 87:\n                return jjMoveStringLiteralDfa7_0(active0, 0x40000L);\n            case 110:\n                if ((active0 & 0x800L) != 0L) {\n                    jjmatchedKind = 11;\n                    jjmatchedPos = 6;\n                }\n                return jjMoveStringLiteralDfa7_0(active0, 0x20000L);\n            case 116:\n                return jjMoveStringLiteralDfa7_0(active0, 0x80000L);\n            case 119:\n                return jjMoveStringLiteralDfa7_0(active0, 0x40000L);\n            default:\n                break;\n        }\n        return jjMoveNfa_0(5, 6);\n    }\n\n    private int jjMoveStringLiteralDfa7_0(long old0, long active0) {\n        if (((active0 &= old0)) == 0L)\n            return jjMoveNfa_0(5, 6);\n        try {\n            curChar = inputStream.readChar();\n        } catch (java.io.IOException e) {\n            return jjMoveNfa_0(5, 6);\n        }\n        switch (curChar) {\n            case 72:\n                if ((active0 & 0x80000L) != 0L) {\n                    jjmatchedKind = 19;\n                    jjmatchedPos = 7;\n                }\n                break;\n            case 73:\n                return jjMoveStringLiteralDfa8_0(active0, 0x40000L);\n            case 83:\n                if ((active0 & 0x20000L) != 0L) {\n                    jjmatchedKind = 17;\n                    jjmatchedPos = 7;\n                }\n                break;\n            case 104:\n                if ((active0 & 0x80000L) != 0L) {\n                    jjmatchedKind = 19;\n                    jjmatchedPos = 7;\n                }\n                break;\n            case 105:\n                return jjMoveStringLiteralDfa8_0(active0, 0x40000L);\n            case 115:\n                if ((active0 & 0x20000L) != 0L) {\n                    jjmatchedKind = 17;\n                    jjmatchedPos = 7;\n                }\n                break;\n            default:\n                break;\n        }\n        return jjMoveNfa_0(5, 7);\n    }\n\n    private int jjMoveStringLiteralDfa8_0(long old0, long active0) {\n        if (((active0 &= old0)) == 0L)\n            return jjMoveNfa_0(5, 7);\n        try {\n            curChar = inputStream.readChar();\n        } catch (java.io.IOException e) {\n            return jjMoveNfa_0(5, 7);\n        }\n        switch (curChar) {\n            case 84:\n                return jjMoveStringLiteralDfa9_0(active0, 0x40000L);\n            case 116:\n                return jjMoveStringLiteralDfa9_0(active0, 0x40000L);\n            default:\n                break;\n        }\n        return jjMoveNfa_0(5, 8);\n    }\n\n    private int jjMoveStringLiteralDfa9_0(long old0, long active0) {\n        if (((active0 &= old0)) == 0L)\n            return jjMoveNfa_0(5, 8);\n        try {\n            curChar = inputStream.readChar();\n        } catch (java.io.IOException e) {\n            return jjMoveNfa_0(5, 8);\n        }\n        switch (curChar) {\n            case 72:\n                if ((active0 & 0x40000L) != 0L) {\n                    jjmatchedKind = 18;\n                    jjmatchedPos = 9;\n                }\n                break;\n            case 104:\n                if ((active0 & 0x40000L) != 0L) {\n                    jjmatchedKind = 18;\n                    jjmatchedPos = 9;\n                }\n                break;\n            default:\n                break;\n        }\n        return jjMoveNfa_0(5, 9);\n    }\n\n    static final long[] JJ_BIT_VEC_0 = {\n        0xfffffffffffffffeL, 0xffffffffffffffffL, 0xffffffffffffffffL, 0xffffffffffffffffL\n    };\n    static final long[] JJ_BIT_VEC_2 = {\n        0x0L, 0x0L, 0xffffffffffffffffL, 0xffffffffffffffffL\n    };\n\n    private int jjMoveNfa_0(int startState, int curPos) {\n        int strKind = jjmatchedKind;\n        int strPos = jjmatchedPos;\n        int seenUpto;\n        inputStream.backup(seenUpto = curPos + 1);\n        try {\n            curChar = inputStream.readChar();\n        } catch (java.io.IOException e) {\n            throw new Error(\"Internal Error\");\n        }\n        curPos = 0;\n        int startsAt = 0;\n        jjnewStateCnt = 40;\n        int i = 1;\n        jjstateSet[0] = startState;\n        int kind = 0x7fffffff;\n        for (; ; ) {\n            if (++jjround == 0x7fffffff)\n                ReInitRounds();\n            if (curChar < 64) {\n                long l = 1L << curChar;\n                do {\n                    switch (jjstateSet[--i]) {\n                        case 5:\n                            if ((0x3ff000000000000L & l) != 0L)\n                                jjCheckNAddStates(0, 3);\n                            else if (curChar == 36) {\n                                if (kind > 24)\n                                    kind = 24;\n                                jjCheckNAdd(28);\n                            } else if (curChar == 39)\n                                jjCheckNAddStates(4, 6);\n                            else if (curChar == 46)\n                                jjCheckNAdd(18);\n                            else if (curChar == 47)\n                                jjstateSet[jjnewStateCnt++] = 6;\n                            else if (curChar == 45)\n                                jjstateSet[jjnewStateCnt++] = 0;\n                            if ((0x3fe000000000000L & l) != 0L) {\n                                if (kind > 20)\n                                    kind = 20;\n                                jjCheckNAddTwoStates(15, 16);\n                            } else if (curChar == 48) {\n                                if (kind > 20)\n                                    kind = 20;\n                            }\n                            break;\n                        case 0:\n                            if (curChar == 45)\n                                jjCheckNAddStates(7, 9);\n                            break;\n                        case 1:\n                            if ((0xffffffffffffdbffL & l) != 0L)\n                                jjCheckNAddStates(7, 9);\n                            break;\n                        case 2:\n                            if ((0x2400L & l) != 0L && kind > 6)\n                                kind = 6;\n                            break;\n                        case 3:\n                            if (curChar == 10 && kind > 6)\n                                kind = 6;\n                            break;\n                        case 4:\n                            if (curChar == 13)\n                                jjstateSet[jjnewStateCnt++] = 3;\n                            break;\n                        case 6:\n                            if (curChar == 42)\n                                jjCheckNAddTwoStates(7, 8);\n                            break;\n                        case 7:\n                            if ((0xfffffbffffffffffL & l) != 0L)\n                                jjCheckNAddTwoStates(7, 8);\n                            break;\n                        case 8:\n                            if (curChar == 42)\n                                jjCheckNAddStates(10, 12);\n                            break;\n                        case 9:\n                            if ((0xffff7bffffffffffL & l) != 0L)\n                                jjCheckNAddTwoStates(10, 8);\n                            break;\n                        case 10:\n                            if ((0xfffffbffffffffffL & l) != 0L)\n                                jjCheckNAddTwoStates(10, 8);\n                            break;\n                        case 11:\n                            if (curChar == 47 && kind > 7)\n                                kind = 7;\n                            break;\n                        case 12:\n                            if (curChar == 47)\n                                jjstateSet[jjnewStateCnt++] = 6;\n                            break;\n                        case 13:\n                            if (curChar == 48 && kind > 20)\n                                kind = 20;\n                            break;\n                        case 14:\n                            if ((0x3fe000000000000L & l) == 0L)\n                                break;\n                            if (kind > 20)\n                                kind = 20;\n                            jjCheckNAddTwoStates(15, 16);\n                            break;\n                        case 15:\n                            if ((0x3ff000000000000L & l) == 0L)\n                                break;\n                            if (kind > 20)\n                                kind = 20;\n                            jjCheckNAddTwoStates(15, 16);\n                            break;\n                        case 17:\n                            if (curChar == 46)\n                                jjCheckNAdd(18);\n                            break;\n                        case 18:\n                            if ((0x3ff000000000000L & l) == 0L)\n                                break;\n                            if (kind > 21)\n                                kind = 21;\n                            jjCheckNAddTwoStates(18, 19);\n                            break;\n                        case 20:\n                            if ((0x280000000000L & l) != 0L)\n                                jjCheckNAdd(21);\n                            break;\n                        case 21:\n                            if ((0x3ff000000000000L & l) == 0L)\n                                break;\n                            if (kind > 21)\n                                kind = 21;\n                            jjCheckNAdd(21);\n                            break;\n                        case 22:\n                        case 23:\n                            if (curChar == 39)\n                                jjCheckNAddStates(4, 6);\n                            break;\n                        case 24:\n                            if (curChar == 39)\n                                jjstateSet[jjnewStateCnt++] = 23;\n                            break;\n                        case 25:\n                            if ((0xffffff7fffffffffL & l) != 0L)\n                                jjCheckNAddStates(4, 6);\n                            break;\n                        case 26:\n                            if (curChar == 39 && kind > 23)\n                                kind = 23;\n                            break;\n                        case 27:\n                            if (curChar != 36)\n                                break;\n                            if (kind > 24)\n                                kind = 24;\n                            jjCheckNAdd(28);\n                            break;\n                        case 28:\n                            if ((0x3ff001000000000L & l) == 0L)\n                                break;\n                            if (kind > 24)\n                                kind = 24;\n                            jjCheckNAdd(28);\n                            break;\n                        case 29:\n                            if ((0x3ff000000000000L & l) != 0L)\n                                jjCheckNAddStates(0, 3);\n                            break;\n                        case 30:\n                            if ((0x3ff000000000000L & l) != 0L)\n                                jjCheckNAddTwoStates(30, 31);\n                            break;\n                        case 31:\n                            if (curChar != 46)\n                                break;\n                            if (kind > 21)\n                                kind = 21;\n                            jjCheckNAddTwoStates(32, 33);\n                            break;\n                        case 32:\n                            if ((0x3ff000000000000L & l) == 0L)\n                                break;\n                            if (kind > 21)\n                                kind = 21;\n                            jjCheckNAddTwoStates(32, 33);\n                            break;\n                        case 34:\n                            if ((0x280000000000L & l) != 0L)\n                                jjCheckNAdd(35);\n                            break;\n                        case 35:\n                            if ((0x3ff000000000000L & l) == 0L)\n                                break;\n                            if (kind > 21)\n                                kind = 21;\n                            jjCheckNAdd(35);\n                            break;\n                        case 36:\n                            if ((0x3ff000000000000L & l) != 0L)\n                                jjCheckNAddTwoStates(36, 37);\n                            break;\n                        case 38:\n                            if ((0x280000000000L & l) != 0L)\n                                jjCheckNAdd(39);\n                            break;\n                        case 39:\n                            if ((0x3ff000000000000L & l) == 0L)\n                                break;\n                            if (kind > 21)\n                                kind = 21;\n                            jjCheckNAdd(39);\n                            break;\n                        default:\n                            break;\n                    }\n                } while (i != startsAt);\n            } else if (curChar < 128) {\n                long l = 1L << (curChar & 077);\n                do {\n                    switch (jjstateSet[--i]) {\n                        case 5:\n                        case 28:\n                            if ((0x7fffffe87fffffeL & l) == 0L)\n                                break;\n                            if (kind > 24)\n                                kind = 24;\n                            jjCheckNAdd(28);\n                            break;\n                        case 1:\n                            jjAddStates(7, 9);\n                            break;\n                        case 7:\n                            jjCheckNAddTwoStates(7, 8);\n                            break;\n                        case 9:\n                        case 10:\n                            jjCheckNAddTwoStates(10, 8);\n                            break;\n                        case 16:\n                            if ((0x100000001000L & l) != 0L && kind > 20)\n                                kind = 20;\n                            break;\n                        case 19:\n                            if ((0x2000000020L & l) != 0L)\n                                jjAddStates(13, 14);\n                            break;\n                        case 25:\n                            jjAddStates(4, 6);\n                            break;\n                        case 33:\n                            if ((0x2000000020L & l) != 0L)\n                                jjAddStates(15, 16);\n                            break;\n                        case 37:\n                            if ((0x2000000020L & l) != 0L)\n                                jjAddStates(17, 18);\n                            break;\n                        default:\n                            break;\n                    }\n                } while (i != startsAt);\n            } else {\n                int hiByte = (int) (curChar >> 8);\n                int i1 = hiByte >> 6;\n                long l1 = 1L << (hiByte & 077);\n                int i2 = (curChar & 0xff) >> 6;\n                long l2 = 1L << (curChar & 077);\n                do {\n                    switch (jjstateSet[--i]) {\n                        case 1:\n                            if (jjCanMove_0(hiByte, i1, i2, l1, l2))\n                                jjAddStates(7, 9);\n                            break;\n                        case 7:\n                            if (jjCanMove_0(hiByte, i1, i2, l1, l2))\n                                jjCheckNAddTwoStates(7, 8);\n                            break;\n                        case 9:\n                        case 10:\n                            if (jjCanMove_0(hiByte, i1, i2, l1, l2))\n                                jjCheckNAddTwoStates(10, 8);\n                            break;\n                        case 25:\n                            if (jjCanMove_0(hiByte, i1, i2, l1, l2))\n                                jjAddStates(4, 6);\n                            break;\n                        default:\n                            break;\n                    }\n                } while (i != startsAt);\n            }\n            if (kind != 0x7fffffff) {\n                jjmatchedKind = kind;\n                jjmatchedPos = curPos;\n                kind = 0x7fffffff;\n            }\n            ++curPos;\n            if ((i = jjnewStateCnt) == (startsAt = 40 - (jjnewStateCnt = startsAt)))\n                break;\n            try {\n                curChar = inputStream.readChar();\n            } catch (java.io.IOException e) {\n                break;\n            }\n        }\n        if (jjmatchedPos > strPos)\n            return curPos;\n\n        int toRet = Math.max(curPos, seenUpto);\n\n        if (curPos < toRet)\n            for (i = toRet - Math.min(curPos, seenUpto); i-- > 0; )\n                try {\n                    curChar = inputStream.readChar();\n                } catch (java.io.IOException e) {\n                    throw new Error(\"Internal Error : Please send a bug report.\");\n                }\n\n        if (jjmatchedPos < strPos) {\n            jjmatchedKind = strKind;\n            jjmatchedPos = strPos;\n        } else if (jjmatchedPos == strPos && jjmatchedKind > strKind)\n            jjmatchedKind = strKind;\n\n        return toRet;\n    }\n\n    static final int[] JJ_NEXT_STATES = {\n        30, 31, 36, 37, 24, 25, 26, 1, 2, 4, 8, 9, 11, 20, 21, 34,\n        35, 38, 39,\n    };\n\n    private static final boolean jjCanMove_0(int hiByte, int i1, int i2, long l1, long l2) {\n        switch (hiByte) {\n            case 0:\n                return (JJ_BIT_VEC_2[i2] & l2) != 0L;\n            default:\n                if ((JJ_BIT_VEC_0[i1] & l1) != 0L)\n                    return true;\n                return false;\n        }\n    }\n\n    /**\n     * Token literal values.\n     */\n    public static final String[] JJ_STR_LITERAL_IMAGES = {\n        \"\", null, null, null, null, null, null, null, null, null, null, null, null,\n        null, null, null, null, null, null, null, null, null, null, null, null, \"\\75\",\n        \"\\74\\76\", \"\\76\", \"\\76\\75\", \"\\74\", \"\\74\\75\", \"\\50\", \"\\54\", \"\\51\", \"\\53\", \"\\55\",};\n\n    /**\n     * Lexer state names.\n     */\n    public static final String[] LEX_STATE_NAMES = {\n        \"DEFAULT\",\n    };\n    static final long[] JJ_TO_TOKEN = {\n        0xfffbfff01L,\n    };\n    static final long[] JJ_TO_SKIP = {\n        0xfeL,\n    };\n    static final long[] JJ_TO_SPECIAL = {\n        0x3eL,\n    };\n    protected SimpleCharStream inputStream;\n    private final int[] jjrounds = new int[40];\n    private final int[] jjstateSet = new int[80];\n    protected char curChar;\n\n    /**\n     * Constructor.\n     */\n    public SelectorParserTokenManager(SimpleCharStream stream) {\n        if (SimpleCharStream.STATIC_FLAG)\n            throw new Error(\"ERROR: Cannot use a static CharStream class with a non-static lexical analyzer.\");\n        inputStream = stream;\n    }\n\n    /**\n     * Constructor.\n     */\n    public SelectorParserTokenManager(SimpleCharStream stream, int lexState) {\n        this(stream);\n        SwitchTo(lexState);\n    }\n\n    /**\n     * Reinitialise parser.\n     */\n    public void ReInit(SimpleCharStream stream) {\n        jjmatchedPos = jjnewStateCnt = 0;\n        curLexState = defaultLexState;\n        inputStream = stream;\n        ReInitRounds();\n    }\n\n    private void ReInitRounds() {\n        int i;\n        jjround = 0x80000001;\n        for (i = 40; i-- > 0; )\n            jjrounds[i] = 0x80000000;\n    }\n\n    /**\n     * Reinitialise parser.\n     */\n    public void ReInit(SimpleCharStream stream, int lexState) {\n        ReInit(stream);\n        SwitchTo(lexState);\n    }\n\n    /**\n     * Switch to specified lex state.\n     */\n    public void SwitchTo(int lexState) {\n        if (lexState >= 1 || lexState < 0)\n            throw new TokenMgrError(\"Error: Ignoring invalid lexical state : \" + lexState + \". State unchanged.\", TokenMgrError.INVALID_LEXICAL_STATE);\n        else\n            curLexState = lexState;\n    }\n\n    protected Token jjFillToken() {\n        final Token t;\n        final String curTokenImage;\n        final int beginLine;\n        final int endLine;\n        final int beginColumn;\n        final int endColumn;\n        String im = JJ_STR_LITERAL_IMAGES[jjmatchedKind];\n        curTokenImage = (im == null) ? inputStream.GetImage() : im;\n        beginLine = inputStream.getBeginLine();\n        beginColumn = inputStream.getBeginColumn();\n        endLine = inputStream.getEndLine();\n        endColumn = inputStream.getEndColumn();\n        t = Token.newToken(jjmatchedKind, curTokenImage);\n\n        t.beginLine = beginLine;\n        t.endLine = endLine;\n        t.beginColumn = beginColumn;\n        t.endColumn = endColumn;\n\n        return t;\n    }\n\n    int curLexState = 0;\n    int defaultLexState = 0;\n    int jjnewStateCnt;\n    int jjround;\n    int jjmatchedPos;\n    int jjmatchedKind;\n\n    /**\n     * Get the next Token.\n     */\n    public Token getNextToken() {\n        Token specialToken = null;\n        Token matchedToken;\n        int curPos = 0;\n\n        EOFLoop:\n        for (; ; ) {\n            try {\n                curChar = inputStream.BeginToken();\n            } catch (java.io.IOException e) {\n                jjmatchedKind = 0;\n                matchedToken = jjFillToken();\n                matchedToken.specialToken = specialToken;\n                return matchedToken;\n            }\n\n            jjmatchedKind = 0x7fffffff;\n            jjmatchedPos = 0;\n            curPos = jjMoveStringLiteralDfa0_0();\n            if (jjmatchedKind != 0x7fffffff) {\n                if (jjmatchedPos + 1 < curPos)\n                    inputStream.backup(curPos - jjmatchedPos - 1);\n                if ((JJ_TO_TOKEN[jjmatchedKind >> 6] & (1L << (jjmatchedKind & 077))) != 0L) {\n                    matchedToken = jjFillToken();\n                    matchedToken.specialToken = specialToken;\n                    return matchedToken;\n                } else {\n                    if ((JJ_TO_SPECIAL[jjmatchedKind >> 6] & (1L << (jjmatchedKind & 077))) != 0L) {\n                        matchedToken = jjFillToken();\n                        if (specialToken == null)\n                            specialToken = matchedToken;\n                        else {\n                            matchedToken.specialToken = specialToken;\n                            specialToken = specialToken.next = matchedToken;\n                        }\n                    }\n                    continue EOFLoop;\n                }\n            }\n            int errorLine = inputStream.getEndLine();\n            int errorColumn = inputStream.getEndColumn();\n            String errorAfter = null;\n            boolean eofSeen = false;\n            try {\n                inputStream.readChar();\n                inputStream.backup(1);\n            } catch (java.io.IOException e1) {\n                eofSeen = true;\n                errorAfter = curPos <= 1 ? \"\" : inputStream.GetImage();\n                if (curChar == '\\n' || curChar == '\\r') {\n                    errorLine++;\n                    errorColumn = 0;\n                } else\n                    errorColumn++;\n            }\n            if (!eofSeen) {\n                inputStream.backup(1);\n                errorAfter = curPos <= 1 ? \"\" : inputStream.GetImage();\n            }\n            throw new TokenMgrError(eofSeen, curLexState, errorLine, errorColumn, errorAfter, curChar, TokenMgrError.LEXICAL_ERROR);\n        }\n    }\n\n    private void jjCheckNAdd(int state) {\n        if (jjrounds[state] != jjround) {\n            jjstateSet[jjnewStateCnt++] = state;\n            jjrounds[state] = jjround;\n        }\n    }\n\n    private void jjAddStates(int start, int end) {\n        do {\n            jjstateSet[jjnewStateCnt++] = JJ_NEXT_STATES[start];\n        } while (start++ != end);\n    }\n\n    private void jjCheckNAddTwoStates(int state1, int state2) {\n        jjCheckNAdd(state1);\n        jjCheckNAdd(state2);\n    }\n\n    private void jjCheckNAddStates(int start, int end) {\n        do {\n            jjCheckNAdd(JJ_NEXT_STATES[start]);\n        } while (start++ != end);\n    }\n\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/parser/SimpleCharStream.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/* Generated By:JavaCC: Do not edit this line. SimpleCharStream.java Version 5.0 */\n/* JavaCCOptions:STATIC=false,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */\npackage org.apache.rocketmq.filter.parser;\n\nimport java.nio.charset.StandardCharsets;\n\n/**\n * An implementation of interface CharStream, where the stream is assumed to\n * contain only ASCII characters (without unicode processing).\n */\n\npublic class SimpleCharStream {\n    /**\n     * Whether parser is static.\n     */\n    public static final boolean STATIC_FLAG = false;\n    int bufsize;\n    int available;\n    int tokenBegin;\n    /**\n     * Position in buffer.\n     */\n    public int bufpos = -1;\n    protected int[] bufline;\n    protected int[] bufcolumn;\n\n    protected int column = 0;\n    protected int line = 1;\n\n    protected boolean prevCharIsCR = false;\n    protected boolean prevCharIsLF = false;\n\n    protected java.io.Reader inputStream;\n\n    protected char[] buffer;\n    protected int maxNextCharInd = 0;\n    protected int inBuf = 0;\n    protected int tabSize = 8;\n\n    protected void setTabSize(int i) {\n        tabSize = i;\n    }\n\n    protected int getTabSize(int i) {\n        return tabSize;\n    }\n\n    protected void ExpandBuff(boolean wrapAround) {\n        char[] newbuffer = new char[bufsize + 2048];\n        int[] newbufline = new int[bufsize + 2048];\n        int[] newbufcolumn = new int[bufsize + 2048];\n\n        try {\n            if (wrapAround) {\n                System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin);\n                System.arraycopy(buffer, 0, newbuffer, bufsize - tokenBegin, bufpos);\n                buffer = newbuffer;\n\n                System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin);\n                System.arraycopy(bufline, 0, newbufline, bufsize - tokenBegin, bufpos);\n                bufline = newbufline;\n\n                System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin);\n                System.arraycopy(bufcolumn, 0, newbufcolumn, bufsize - tokenBegin, bufpos);\n                bufcolumn = newbufcolumn;\n\n                maxNextCharInd = bufpos += bufsize - tokenBegin;\n            } else {\n                System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin);\n                buffer = newbuffer;\n\n                System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin);\n                bufline = newbufline;\n\n                System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin);\n                bufcolumn = newbufcolumn;\n\n                maxNextCharInd = bufpos -= tokenBegin;\n            }\n        } catch (Throwable t) {\n            throw new Error(t.getMessage());\n        }\n\n        bufsize += 2048;\n        available = bufsize;\n        tokenBegin = 0;\n    }\n\n    protected void FillBuff() throws java.io.IOException {\n        if (maxNextCharInd == available) {\n            if (available == bufsize) {\n                if (tokenBegin > 2048) {\n                    bufpos = maxNextCharInd = 0;\n                    available = tokenBegin;\n                } else if (tokenBegin < 0)\n                    bufpos = maxNextCharInd = 0;\n                else\n                    ExpandBuff(false);\n            } else if (available > tokenBegin)\n                available = bufsize;\n            else if ((tokenBegin - available) < 2048)\n                ExpandBuff(true);\n            else\n                available = tokenBegin;\n        }\n\n        int i;\n        try {\n            if ((i = inputStream.read(buffer, maxNextCharInd, available - maxNextCharInd)) == -1) {\n                inputStream.close();\n                throw new java.io.IOException();\n            } else\n                maxNextCharInd += i;\n            return;\n        } catch (java.io.IOException e) {\n            --bufpos;\n            backup(0);\n            if (tokenBegin == -1)\n                tokenBegin = bufpos;\n            throw e;\n        }\n    }\n\n    /**\n     * Start.\n     */\n    public char BeginToken() throws java.io.IOException {\n        tokenBegin = -1;\n        char c = readChar();\n        tokenBegin = bufpos;\n\n        return c;\n    }\n\n    protected void UpdateLineColumn(char c) {\n        column++;\n\n        if (prevCharIsLF) {\n            prevCharIsLF = false;\n            line += column = 1;\n        } else if (prevCharIsCR) {\n            prevCharIsCR = false;\n            if (c == '\\n') {\n                prevCharIsLF = true;\n            } else\n                line += column = 1;\n        }\n\n        switch (c) {\n            case '\\r':\n                prevCharIsCR = true;\n                break;\n            case '\\n':\n                prevCharIsLF = true;\n                break;\n            case '\\t':\n                column--;\n                column += tabSize - (column % tabSize);\n                break;\n            default:\n                break;\n        }\n\n        bufline[bufpos] = line;\n        bufcolumn[bufpos] = column;\n    }\n\n    /**\n     * Read a character.\n     */\n    public char readChar() throws java.io.IOException {\n        if (inBuf > 0) {\n            --inBuf;\n\n            if (++bufpos == bufsize)\n                bufpos = 0;\n\n            return buffer[bufpos];\n        }\n\n        if (++bufpos >= maxNextCharInd)\n            FillBuff();\n\n        char c = buffer[bufpos];\n\n        UpdateLineColumn(c);\n        return c;\n    }\n\n    @Deprecated\n    /**\n     * @deprecated\n     * @see #getEndColumn\n     */\n\n    public int getColumn() {\n        return bufcolumn[bufpos];\n    }\n\n    @Deprecated\n    /**\n     * @deprecated\n     * @see #getEndLine\n     */\n\n    public int getLine() {\n        return bufline[bufpos];\n    }\n\n    /**\n     * Get token end column number.\n     */\n    public int getEndColumn() {\n        return bufcolumn[bufpos];\n    }\n\n    /**\n     * Get token end line number.\n     */\n    public int getEndLine() {\n        return bufline[bufpos];\n    }\n\n    /**\n     * Get token beginning column number.\n     */\n    public int getBeginColumn() {\n        return bufcolumn[tokenBegin];\n    }\n\n    /**\n     * Get token beginning line number.\n     */\n    public int getBeginLine() {\n        return bufline[tokenBegin];\n    }\n\n    /**\n     * Backup a number of characters.\n     */\n    public void backup(int amount) {\n\n        inBuf += amount;\n        if ((bufpos -= amount) < 0)\n            bufpos += bufsize;\n    }\n\n    /**\n     * Constructor.\n     */\n    public SimpleCharStream(java.io.Reader dstream, int startline,\n        int startcolumn, int buffersize) {\n        inputStream = dstream;\n        line = startline;\n        column = startcolumn - 1;\n\n        available = bufsize = buffersize;\n        buffer = new char[buffersize];\n        bufline = new int[buffersize];\n        bufcolumn = new int[buffersize];\n    }\n\n    /**\n     * Constructor.\n     */\n    public SimpleCharStream(java.io.Reader dstream, int startline,\n        int startcolumn) {\n        this(dstream, startline, startcolumn, 4096);\n    }\n\n    /**\n     * Constructor.\n     */\n    public SimpleCharStream(java.io.Reader dstream) {\n        this(dstream, 1, 1, 4096);\n    }\n\n    /**\n     * Reinitialise.\n     */\n    public void ReInit(java.io.Reader dstream, int startline,\n        int startcolumn, int buffersize) {\n        inputStream = dstream;\n        line = startline;\n        column = startcolumn - 1;\n\n        if (buffer == null || buffersize != buffer.length) {\n            available = bufsize = buffersize;\n            buffer = new char[buffersize];\n            bufline = new int[buffersize];\n            bufcolumn = new int[buffersize];\n        }\n        prevCharIsLF = prevCharIsCR = false;\n        tokenBegin = inBuf = maxNextCharInd = 0;\n        bufpos = -1;\n    }\n\n    /**\n     * Reinitialise.\n     */\n    public void ReInit(java.io.Reader dstream, int startline,\n        int startcolumn) {\n        ReInit(dstream, startline, startcolumn, 4096);\n    }\n\n    /**\n     * Reinitialise.\n     */\n    public void ReInit(java.io.Reader dstream) {\n        ReInit(dstream, 1, 1, 4096);\n    }\n\n    /**\n     * Constructor.\n     */\n    public SimpleCharStream(java.io.InputStream dstream, String encoding, int startline,\n        int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException {\n        this(encoding == null ?\n            new java.io.InputStreamReader(dstream, StandardCharsets.UTF_8) :\n            new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize);\n    }\n\n    /**\n     * Constructor.\n     */\n    public SimpleCharStream(java.io.InputStream dstream, int startline,\n        int startcolumn, int buffersize) {\n        this(new java.io.InputStreamReader(dstream, StandardCharsets.UTF_8), startline, startcolumn, buffersize);\n    }\n\n    /**\n     * Constructor.\n     */\n    public SimpleCharStream(java.io.InputStream dstream, String encoding, int startline,\n        int startcolumn) throws java.io.UnsupportedEncodingException {\n        this(dstream, encoding, startline, startcolumn, 4096);\n    }\n\n    /**\n     * Constructor.\n     */\n    public SimpleCharStream(java.io.InputStream dstream, int startline,\n        int startcolumn) {\n        this(dstream, startline, startcolumn, 4096);\n    }\n\n    /**\n     * Constructor.\n     */\n    public SimpleCharStream(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException {\n        this(dstream, encoding, 1, 1, 4096);\n    }\n\n    /**\n     * Constructor.\n     */\n    public SimpleCharStream(java.io.InputStream dstream) {\n        this(dstream, 1, 1, 4096);\n    }\n\n    /**\n     * Reinitialise.\n     */\n    public void ReInit(java.io.InputStream dstream, String encoding, int startline,\n        int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException {\n        ReInit(encoding == null ?\n            new java.io.InputStreamReader(dstream, StandardCharsets.UTF_8) :\n            new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize);\n    }\n\n    /**\n     * Reinitialise.\n     */\n    public void ReInit(java.io.InputStream dstream, int startline,\n        int startcolumn, int buffersize) {\n        ReInit(new java.io.InputStreamReader(dstream, StandardCharsets.UTF_8), startline, startcolumn, buffersize);\n    }\n\n    /**\n     * Reinitialise.\n     */\n    public void ReInit(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException {\n        ReInit(dstream, encoding, 1, 1, 4096);\n    }\n\n    /**\n     * Reinitialise.\n     */\n    public void ReInit(java.io.InputStream dstream) {\n        ReInit(dstream, 1, 1, 4096);\n    }\n\n    /**\n     * Reinitialise.\n     */\n    public void ReInit(java.io.InputStream dstream, String encoding, int startline,\n        int startcolumn) throws java.io.UnsupportedEncodingException {\n        ReInit(dstream, encoding, startline, startcolumn, 4096);\n    }\n\n    /**\n     * Reinitialise.\n     */\n    public void ReInit(java.io.InputStream dstream, int startline,\n        int startcolumn) {\n        ReInit(dstream, startline, startcolumn, 4096);\n    }\n\n    /**\n     * Get token literal value.\n     */\n    public String GetImage() {\n        if (bufpos >= tokenBegin)\n            return new String(buffer, tokenBegin, bufpos - tokenBegin + 1);\n        else\n            return new String(buffer, tokenBegin, bufsize - tokenBegin) +\n                new String(buffer, 0, bufpos + 1);\n    }\n\n    /**\n     * Get the suffix.\n     */\n    public char[] GetSuffix(int len) {\n        char[] ret = new char[len];\n\n        if ((bufpos + 1) >= len)\n            System.arraycopy(buffer, bufpos - len + 1, ret, 0, len);\n        else {\n            System.arraycopy(buffer, bufsize - (len - bufpos - 1), ret, 0,\n                len - bufpos - 1);\n            System.arraycopy(buffer, 0, ret, len - bufpos - 1, bufpos + 1);\n        }\n\n        return ret;\n    }\n\n    /**\n     * Reset buffer when finished.\n     */\n    public void Done() {\n        buffer = null;\n        bufline = null;\n        bufcolumn = null;\n    }\n\n    /**\n     * Method to adjust line and column numbers for the start of a token.\n     */\n    public void adjustBeginLineColumn(int newLine, int newCol) {\n        int start = tokenBegin;\n        int len;\n\n        if (bufpos >= tokenBegin) {\n            len = bufpos - tokenBegin + inBuf + 1;\n        } else {\n            len = bufsize - tokenBegin + bufpos + 1 + inBuf;\n        }\n\n        int i = 0, j = 0, k = 0;\n        int nextColDiff = 0, columnDiff = 0;\n\n        while (i < len && bufline[j = start % bufsize] == bufline[k = ++start % bufsize]) {\n            bufline[j] = newLine;\n            nextColDiff = columnDiff + bufcolumn[k] - bufcolumn[j];\n            bufcolumn[j] = newCol + columnDiff;\n            columnDiff = nextColDiff;\n            i++;\n        }\n\n        if (i < len) {\n            bufline[j] = newLine++;\n            bufcolumn[j] = newCol + columnDiff;\n\n            while (i++ < len) {\n                if (bufline[j = start % bufsize] != bufline[++start % bufsize])\n                    bufline[j] = newLine++;\n                else\n                    bufline[j] = newLine;\n            }\n        }\n\n        line = bufline[j];\n        column = bufcolumn[j];\n    }\n\n}\n/* JavaCC - OriginalChecksum=ea3493f692d4975c1ad70c4a750107d3 (do not edit this line) */\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/parser/Token.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/* Generated By:JavaCC: Do not edit this line. Token.java Version 5.0 */\n/* JavaCCOptions:TOKEN_EXTENDS=,KEEP_LINE_COL=null,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */\npackage org.apache.rocketmq.filter.parser;\n\n/**\n * Describes the input token stream.\n */\n\npublic class Token implements java.io.Serializable {\n\n    /**\n     * The version identifier for this Serializable class.\n     * Increment only if the <i>serialized</i> form of the\n     * class changes.\n     */\n    private static final long serialVersionUID = 1L;\n\n    /**\n     * An integer that describes the kind of this token.  This numbering\n     * system is determined by JavaCCParser, and a table of these numbers is\n     * stored in the file ...Constants.java.\n     */\n    public int kind;\n\n    /**\n     * The line number of the first character of this Token.\n     */\n    public int beginLine;\n    /**\n     * The column number of the first character of this Token.\n     */\n    public int beginColumn;\n    /**\n     * The line number of the last character of this Token.\n     */\n    public int endLine;\n    /**\n     * The column number of the last character of this Token.\n     */\n    public int endColumn;\n\n    /**\n     * The string image of the token.\n     */\n    public String image;\n\n    /**\n     * A reference to the next regular (non-special) token from the input\n     * stream.  If this is the last token from the input stream, or if the\n     * token manager has not read tokens beyond this one, this field is\n     * set to null.  This is true only if this token is also a regular\n     * token.  Otherwise, see below for a description of the contents of\n     * this field.\n     */\n    public Token next;\n\n    /**\n     * This field is used to access special tokens that occur prior to this\n     * token, but after the immediately preceding regular (non-special) token.\n     * If there are no such special tokens, this field is set to null.\n     * When there are more than one such special token, this field refers\n     * to the last of these special tokens, which in turn refers to the next\n     * previous special token through its specialToken field, and so on\n     * until the first special token (whose specialToken field is null).\n     * The next fields of special tokens refer to other special tokens that\n     * immediately follow it (without an intervening regular token).  If there\n     * is no such token, this field is null.\n     */\n    public Token specialToken;\n\n    /**\n     * An optional attribute value of the Token.\n     * Tokens which are not used as syntactic sugar will often contain\n     * meaningful values that will be used later on by the compiler or\n     * interpreter. This attribute value is often different from the image.\n     * Any subclass of Token that actually wants to return a non-null value can\n     * override this method as appropriate.\n     */\n    public Object getValue() {\n        return null;\n    }\n\n    /**\n     * No-argument constructor\n     */\n    public Token() {\n    }\n\n    /**\n     * Constructs a new token for the specified Image.\n     */\n    public Token(int kind) {\n        this(kind, null);\n    }\n\n    /**\n     * Constructs a new token for the specified Image and Kind.\n     */\n    public Token(int kind, String image) {\n        this.kind = kind;\n        this.image = image;\n    }\n\n    /**\n     * Returns the image.\n     */\n    public String toString() {\n        return image;\n    }\n\n    /**\n     * Returns a new Token object, by default. However, if you want, you\n     * can create and return subclass objects based on the value of ofKind.\n     * Simply add the cases to the switch for all those special cases.\n     * For example, if you have a subclass of Token called IDToken that\n     * you want to create if ofKind is ID, simply add something like :\n     * <p/>\n     * case MyParserConstants.ID : return new IDToken(ofKind, image);\n     * <p/>\n     * to the following switch statement. Then you can cast matchedToken\n     * variable to the appropriate type and use sit in your lexical actions.\n     */\n    public static Token newToken(int ofKind, String image) {\n        switch (ofKind) {\n            default:\n                return new Token(ofKind, image);\n        }\n    }\n\n    public static Token newToken(int ofKind) {\n        return newToken(ofKind, null);\n    }\n\n}\n/* JavaCC - OriginalChecksum=20094f1ccfbf423c6d9e770d6a7a0188 (do not edit this line) */\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/parser/TokenMgrError.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 5.0 */\n/* JavaCCOptions: */\npackage org.apache.rocketmq.filter.parser;\n\n/**\n * Token Manager Error.\n */\npublic class TokenMgrError extends Error {\n\n    /**\n     * The version identifier for this Serializable class.\n     * Increment only if the <i>serialized</i> form of the\n     * class changes.\n     */\n    private static final long serialVersionUID = 1L;\n\n  /*\n   * Ordinals for various reasons why an Error of this type can be thrown.\n   */\n\n    /**\n     * Lexical error occurred.\n     */\n    static final int LEXICAL_ERROR = 0;\n\n    /**\n     * An attempt was made to create a second instance of a static token manager.\n     */\n    static final int STATIC_LEXER_ERROR = 1;\n\n    /**\n     * Tried to change to an invalid lexical state.\n     */\n    static final int INVALID_LEXICAL_STATE = 2;\n\n    /**\n     * Detected (and bailed out of) an infinite loop in the token manager.\n     */\n    static final int LOOP_DETECTED = 3;\n\n    /**\n     * Indicates the reason why the exception is thrown. It will have\n     * one of the above 4 values.\n     */\n    int errorCode;\n\n    /**\n     * Replaces unprintable characters by their escaped (or unicode escaped)\n     * equivalents in the given string\n     */\n    protected static final String addEscapes(String str) {\n        StringBuilder retval = new StringBuilder();\n        char ch;\n        for (int i = 0; i < str.length(); i++) {\n            switch (str.charAt(i)) {\n                case 0:\n                    continue;\n                case '\\b':\n                    retval.append(\"\\\\b\");\n                    continue;\n                case '\\t':\n                    retval.append(\"\\\\t\");\n                    continue;\n                case '\\n':\n                    retval.append(\"\\\\n\");\n                    continue;\n                case '\\f':\n                    retval.append(\"\\\\f\");\n                    continue;\n                case '\\r':\n                    retval.append(\"\\\\r\");\n                    continue;\n                case '\\\"':\n                    retval.append(\"\\\\\\\"\");\n                    continue;\n                case '\\'':\n                    retval.append(\"\\\\\\'\");\n                    continue;\n                case '\\\\':\n                    retval.append(\"\\\\\\\\\");\n                    continue;\n                default:\n                    if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {\n                        String s = \"0000\" + Integer.toString(ch, 16);\n                        retval.append(\"\\\\u\" + s.substring(s.length() - 4, s.length()));\n                    } else {\n                        retval.append(ch);\n                    }\n                    continue;\n            }\n        }\n        return retval.toString();\n    }\n\n    /**\n     * Returns a detailed message for the Error when it is thrown by the\n     * token manager to indicate a lexical error.\n     * Parameters :\n     * eofSeen     : indicates if EOF caused the lexical error\n     * curLexState : lexical state in which this error occurred\n     * errorLine   : line number when the error occurred\n     * errorColumn : column number when the error occurred\n     * errorAfter  : prefix that was seen before this error occurred\n     * curchar     : the offending character\n     * Note: You can customize the lexical error message by modifying this method.\n     */\n    protected static String LexicalError(boolean eofSeen, int lexState, int errorLine, int errorColumn,\n        String errorAfter, char curChar) {\n        return \"Lexical error at line \" +\n            errorLine + \", column \" +\n            errorColumn + \".  Encountered: \" +\n            (eofSeen ?\n                \"<EOF> \" :\n                (\"\\\"\" + addEscapes(String.valueOf(curChar)) + \"\\\"\") + \" (\" + (int) curChar + \"), \") +\n            \"after : \\\"\" + addEscapes(errorAfter) + \"\\\"\";\n    }\n\n    /**\n     * You can also modify the body of this method to customize your error messages.\n     * For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not\n     * of end-users concern, so you can return something like :\n     * <p/>\n     * \"Internal Error : Please file a bug report .... \"\n     * <p/>\n     * from this method for such cases in the release version of your parser.\n     */\n    @Override\n    public String getMessage() {\n        return super.getMessage();\n    }\n\n  /*\n   * Constructors of various flavors follow.\n   */\n\n    /**\n     * No arg constructor.\n     */\n    public TokenMgrError() {\n    }\n\n    /**\n     * Constructor with message and reason.\n     */\n    public TokenMgrError(String message, int reason) {\n        super(message);\n        errorCode = reason;\n    }\n\n    /**\n     * Full Constructor.\n     */\n    public TokenMgrError(boolean eofSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar,\n        int reason) {\n        this(LexicalError(eofSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason);\n    }\n}\n/* JavaCC - OriginalChecksum=de79709675790dcbad2e0d728aa630d1 (do not edit this line) */\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/util/BitsArray.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter.util;\n\n/**\n * Wrapper of bytes array, in order to operate single bit easily.\n */\npublic class BitsArray implements Cloneable {\n\n    private byte[] bytes;\n    private int bitLength;\n\n    public static BitsArray create(int bitLength) {\n        return new BitsArray(bitLength);\n    }\n\n    public static BitsArray create(byte[] bytes, int bitLength) {\n        return new BitsArray(bytes, bitLength);\n    }\n\n    public static BitsArray create(byte[] bytes) {\n        return new BitsArray(bytes);\n    }\n\n    private BitsArray(int bitLength) {\n        this.bitLength = bitLength;\n        // init bytes\n        int temp = bitLength / Byte.SIZE;\n        if (bitLength % Byte.SIZE > 0) {\n            temp++;\n        }\n        bytes = new byte[temp];\n        for (int i = 0; i < bytes.length; i++) {\n            bytes[i] = (byte) 0x00;\n        }\n    }\n\n    private BitsArray(byte[] bytes, int bitLength) {\n        if (bytes == null || bytes.length < 1) {\n            throw new IllegalArgumentException(\"Bytes is empty!\");\n        }\n\n        if (bitLength < 1) {\n            throw new IllegalArgumentException(\"Bit is less than 1.\");\n        }\n\n        if (bitLength < bytes.length * Byte.SIZE) {\n            throw new IllegalArgumentException(\"BitLength is less than bytes.length() * \" + Byte.SIZE);\n        }\n\n        this.bytes = new byte[bytes.length];\n        System.arraycopy(bytes, 0, this.bytes, 0, this.bytes.length);\n        this.bitLength = bitLength;\n    }\n\n    private BitsArray(byte[] bytes) {\n        if (bytes == null || bytes.length < 1) {\n            throw new IllegalArgumentException(\"Bytes is empty!\");\n        }\n\n        this.bitLength = bytes.length * Byte.SIZE;\n        this.bytes = new byte[bytes.length];\n        System.arraycopy(bytes, 0, this.bytes, 0, this.bytes.length);\n    }\n\n    public int bitLength() {\n        return this.bitLength;\n    }\n\n    public int byteLength() {\n        return this.bytes.length;\n    }\n\n    public byte[] bytes() {\n        return this.bytes;\n    }\n\n    public void xor(final BitsArray other) {\n        checkInitialized(this);\n        checkInitialized(other);\n\n        int minByteLength = Math.min(this.byteLength(), other.byteLength());\n\n        for (int i = 0; i < minByteLength; i++) {\n            this.bytes[i] = (byte) (this.bytes[i] ^ other.getByte(i));\n        }\n    }\n\n    public void xor(int bitPos, boolean set) {\n        checkBitPosition(bitPos, this);\n\n        boolean value = getBit(bitPos);\n        if (value ^ set) {\n            setBit(bitPos, true);\n        } else {\n            setBit(bitPos, false);\n        }\n    }\n\n    public void or(final BitsArray other) {\n        checkInitialized(this);\n        checkInitialized(other);\n\n        int minByteLength = Math.min(this.byteLength(), other.byteLength());\n\n        for (int i = 0; i < minByteLength; i++) {\n            this.bytes[i] = (byte) (this.bytes[i] | other.getByte(i));\n        }\n    }\n\n    public void or(int bitPos, boolean set) {\n        checkBitPosition(bitPos, this);\n\n        if (set) {\n            setBit(bitPos, true);\n        }\n    }\n\n    public void and(final BitsArray other) {\n        checkInitialized(this);\n        checkInitialized(other);\n\n        int minByteLength = Math.min(this.byteLength(), other.byteLength());\n\n        for (int i = 0; i < minByteLength; i++) {\n            this.bytes[i] = (byte) (this.bytes[i] & other.getByte(i));\n        }\n    }\n\n    public void and(int bitPos, boolean set) {\n        checkBitPosition(bitPos, this);\n\n        if (!set) {\n            setBit(bitPos, false);\n        }\n    }\n\n    public void not(int bitPos) {\n        checkBitPosition(bitPos, this);\n\n        setBit(bitPos, !getBit(bitPos));\n    }\n\n    public void setBit(int bitPos, boolean set) {\n        checkBitPosition(bitPos, this);\n        int sub = subscript(bitPos);\n        int pos = position(bitPos);\n        if (set) {\n            this.bytes[sub] = (byte) (this.bytes[sub] | pos);\n        } else {\n            this.bytes[sub] = (byte) (this.bytes[sub] & ~pos);\n        }\n    }\n\n    public void setByte(int bytePos, byte set) {\n        checkBytePosition(bytePos, this);\n\n        this.bytes[bytePos] = set;\n    }\n\n    public boolean getBit(int bitPos) {\n        checkBitPosition(bitPos, this);\n\n        return (this.bytes[subscript(bitPos)] & position(bitPos)) != 0;\n    }\n\n    public byte getByte(int bytePos) {\n        checkBytePosition(bytePos, this);\n\n        return this.bytes[bytePos];\n    }\n\n    protected int subscript(int bitPos) {\n        return bitPos / Byte.SIZE;\n    }\n\n    protected int position(int bitPos) {\n        return 1 << bitPos % Byte.SIZE;\n    }\n\n    protected void checkBytePosition(int bytePos, BitsArray bitsArray) {\n        checkInitialized(bitsArray);\n        if (bytePos > bitsArray.byteLength()) {\n            throw new IllegalArgumentException(\"BytePos is greater than \" + bytes.length);\n        }\n        if (bytePos < 0) {\n            throw new IllegalArgumentException(\"BytePos is less than 0\");\n        }\n    }\n\n    protected void checkBitPosition(int bitPos, BitsArray bitsArray) {\n        checkInitialized(bitsArray);\n        if (bitPos > bitsArray.bitLength()) {\n            throw new IllegalArgumentException(\"BitPos is greater than \" + bitLength);\n        }\n        if (bitPos < 0) {\n            throw new IllegalArgumentException(\"BitPos is less than 0\");\n        }\n    }\n\n    protected void checkInitialized(BitsArray bitsArray) {\n        if (bitsArray.bytes() == null) {\n            throw new RuntimeException(\"Not initialized!\");\n        }\n    }\n\n    public BitsArray clone() {\n        byte[] clone = new byte[this.byteLength()];\n\n        System.arraycopy(this.bytes, 0, clone, 0, this.byteLength());\n\n        return create(clone, bitLength());\n    }\n\n    @Override\n    public String toString() {\n        if (this.bytes == null) {\n            return \"null\";\n        }\n        StringBuilder stringBuilder = new StringBuilder(this.bytes.length * Byte.SIZE);\n        for (int i = this.bytes.length - 1; i >= 0; i--) {\n\n            int j = Byte.SIZE - 1;\n            if (i == this.bytes.length - 1 && this.bitLength % Byte.SIZE > 0) {\n                // not full byte\n                j = this.bitLength % Byte.SIZE;\n            }\n\n            for (; j >= 0; j--) {\n\n                byte mask = (byte) (1 << j);\n                if ((this.bytes[i] & mask) == mask) {\n                    stringBuilder.append(\"1\");\n                } else {\n                    stringBuilder.append(\"0\");\n                }\n            }\n            if (i % 8 == 0) {\n                stringBuilder.append(\"\\n\");\n            }\n        }\n\n        return stringBuilder.toString();\n    }\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/util/BloomFilter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter.util;\n\nimport com.google.common.hash.Hashing;\n\nimport java.nio.charset.Charset;\nimport java.nio.charset.StandardCharsets;\n\n/**\n * Simple implement of bloom filter.\n */\npublic class BloomFilter {\n\n    public static final Charset UTF_8 = StandardCharsets.UTF_8;\n\n    // as error rate, 10/100 = 0.1\n    private int f = 10;\n    private int n = 128;\n\n    // hash function num, by calculation.\n    private int k;\n    // bit count, by calculation.\n    private int m;\n\n    /**\n     * Create bloom filter by error rate and mapping num.\n     *\n     * @param f error rate\n     * @param n num will mapping to bit\n     */\n    public static BloomFilter createByFn(int f, int n) {\n        return new BloomFilter(f, n);\n    }\n\n    /**\n     * Constructor.\n     *\n     * @param f error rate\n     * @param n num will mapping to bit\n     */\n    private BloomFilter(int f, int n) {\n        if (f < 1 || f >= 100) {\n            throw new IllegalArgumentException(\"f must be greater or equal than 1 and less than 100\");\n        }\n        if (n < 1) {\n            throw new IllegalArgumentException(\"n must be greater than 0\");\n        }\n\n        this.f = f;\n        this.n = n;\n\n        // set p = e^(-kn/m)\n        // f = (1 - p)^k = e^(kln(1-p))\n        // when p = 0.5, k = ln2 * (m/n), f = (1/2)^k = (0.618)^(m/n)\n        double errorRate = f / 100.0;\n        this.k = (int) Math.ceil(logMN(0.5, errorRate));\n\n        if (this.k < 1) {\n            throw new IllegalArgumentException(\"Hash function num is less than 1, maybe you should change the value of error rate or bit num!\");\n        }\n\n        // m >= n*log2(1/f)*log2(e)\n        this.m = (int) Math.ceil(this.n * logMN(2, 1 / errorRate) * logMN(2, Math.E));\n        // m%8 = 0\n        this.m = (int) (Byte.SIZE * Math.ceil(this.m / (Byte.SIZE * 1.0)));\n    }\n\n    /**\n     * Calculate bit positions of {@code str}.\n     * <p>\n     * See \"Less Hashing, Same Performance: Building a Better Bloom Filter\" by Adam Kirsch and Michael\n     * Mitzenmacher.\n     * </p>\n     */\n    public int[] calcBitPositions(String str) {\n        int[] bitPositions = new int[this.k];\n\n        long hash64 = Hashing.murmur3_128().hashString(str, UTF_8).asLong();\n\n        int hash1 = (int) hash64;\n        int hash2 = (int) (hash64 >>> 32);\n\n        for (int i = 1; i <= this.k; i++) {\n            int combinedHash = hash1 + (i * hash2);\n            // Flip all the bits if it's negative (guaranteed positive number)\n            if (combinedHash < 0) {\n                combinedHash = ~combinedHash;\n            }\n            bitPositions[i - 1] = combinedHash % this.m;\n        }\n\n        return bitPositions;\n    }\n\n    /**\n     * Calculate bit positions of {@code str} to construct {@code BloomFilterData}\n     */\n    public BloomFilterData generate(String str) {\n        int[] bitPositions = calcBitPositions(str);\n\n        return new BloomFilterData(bitPositions, this.m);\n    }\n\n    /**\n     * Calculate bit positions of {@code str}, then set the related {@code bits} positions to 1.\n     */\n    public void hashTo(String str, BitsArray bits) {\n        hashTo(calcBitPositions(str), bits);\n    }\n\n    /**\n     * Set the related {@code bits} positions to 1.\n     */\n    public void hashTo(int[] bitPositions, BitsArray bits) {\n        check(bits);\n\n        for (int i : bitPositions) {\n            bits.setBit(i, true);\n        }\n    }\n\n    /**\n     * Extra check:\n     * <li>1. check {@code filterData} belong to this bloom filter.</li>\n     * <p>\n     * Then set the related {@code bits} positions to 1.\n     * </p>\n     */\n    public void hashTo(BloomFilterData filterData, BitsArray bits) {\n        if (!isValid(filterData)) {\n            throw new IllegalArgumentException(\n                String.format(\"Bloom filter data may not belong to this filter! %s, %s\",\n                    filterData, this)\n            );\n        }\n        hashTo(filterData.getBitPos(), bits);\n    }\n\n    /**\n     * Calculate bit positions of {@code str}, then check all the related {@code bits} positions is 1.\n     *\n     * @return true: all the related {@code bits} positions is 1\n     */\n    public boolean isHit(String str, BitsArray bits) {\n        return isHit(calcBitPositions(str), bits);\n    }\n\n    /**\n     * Check all the related {@code bits} positions is 1.\n     *\n     * @return true: all the related {@code bits} positions is 1\n     */\n    public boolean isHit(int[] bitPositions, BitsArray bits) {\n        check(bits);\n        boolean ret = bits.getBit(bitPositions[0]);\n        for (int i = 1; i < bitPositions.length; i++) {\n            ret &= bits.getBit(bitPositions[i]);\n        }\n        return ret;\n    }\n\n    /**\n     * Check all the related {@code bits} positions is 1.\n     *\n     * @return true: all the related {@code bits} positions is 1\n     */\n    public boolean isHit(BloomFilterData filterData, BitsArray bits) {\n        if (!isValid(filterData)) {\n            throw new IllegalArgumentException(\n                String.format(\"Bloom filter data may not belong to this filter! %s, %s\",\n                    filterData, this)\n            );\n        }\n        return isHit(filterData.getBitPos(), bits);\n    }\n\n    /**\n     * Check whether one of {@code bitPositions} has been occupied.\n     *\n     * @return true: if all positions have been occupied.\n     */\n    public boolean checkFalseHit(int[] bitPositions, BitsArray bits) {\n        for (int j = 0; j < bitPositions.length; j++) {\n            int pos = bitPositions[j];\n\n            // check position of bits has been set.\n            // that mean no one occupy the position.\n            if (!bits.getBit(pos)) {\n                return false;\n            }\n        }\n\n        return true;\n    }\n\n    protected void check(BitsArray bits) {\n        if (bits.bitLength() != this.m) {\n            throw new IllegalArgumentException(\n                String.format(\"Length(%d) of bits in BitsArray is not equal to %d!\", bits.bitLength(), this.m)\n            );\n        }\n    }\n\n    /**\n     * Check {@code BloomFilterData} is valid, and belong to this bloom filter.\n     * <li>1. not null</li>\n     * <li>2. {@link org.apache.rocketmq.filter.util.BloomFilterData#getBitNum} must be equal to {@code m} </li>\n     * <li>3. {@link org.apache.rocketmq.filter.util.BloomFilterData#getBitPos} is not null</li>\n     * <li>4. {@link org.apache.rocketmq.filter.util.BloomFilterData#getBitPos}'s length is equal to {@code k}</li>\n     */\n    public boolean isValid(BloomFilterData filterData) {\n        if (filterData == null\n            || filterData.getBitNum() != this.m\n            || filterData.getBitPos() == null\n            || filterData.getBitPos().length != this.k) {\n            return false;\n        }\n\n        return true;\n    }\n\n    /**\n     * error rate.\n     */\n    public int getF() {\n        return f;\n    }\n\n    /**\n     * expect mapping num.\n     */\n    public int getN() {\n        return n;\n    }\n\n    /**\n     * hash function num.\n     */\n    public int getK() {\n        return k;\n    }\n\n    /**\n     * total bit num.\n     */\n    public int getM() {\n        return m;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o)\n            return true;\n        if (!(o instanceof BloomFilter))\n            return false;\n\n        BloomFilter that = (BloomFilter) o;\n\n        if (f != that.f)\n            return false;\n        if (k != that.k)\n            return false;\n        if (m != that.m)\n            return false;\n        if (n != that.n)\n            return false;\n\n        return true;\n    }\n\n    @Override\n    public int hashCode() {\n        int result = f;\n        result = 31 * result + n;\n        result = 31 * result + k;\n        result = 31 * result + m;\n        return result;\n    }\n\n    @Override\n    public String toString() {\n        return String.format(\"f: %d, n: %d, k: %d, m: %d\", f, n, k, m);\n    }\n\n    protected double logMN(double m, double n) {\n        return Math.log(n) / Math.log(m);\n    }\n}\n"
  },
  {
    "path": "filter/src/main/java/org/apache/rocketmq/filter/util/BloomFilterData.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter.util;\n\nimport java.util.Arrays;\n\n/**\n * Data generated by bloom filter, include:\n * <li>1. Bit positions allocated to requester;</li>\n * <li>2. Total bit num when allocating;</li>\n */\npublic class BloomFilterData {\n\n    private int[] bitPos;\n    private int bitNum;\n\n    public BloomFilterData() {\n    }\n\n    public BloomFilterData(int[] bitPos, int bitNum) {\n        this.bitPos = bitPos;\n        this.bitNum = bitNum;\n    }\n\n    public int[] getBitPos() {\n        return bitPos;\n    }\n\n    public int getBitNum() {\n        return bitNum;\n    }\n\n    public void setBitPos(final int[] bitPos) {\n        this.bitPos = bitPos;\n    }\n\n    public void setBitNum(final int bitNum) {\n        this.bitNum = bitNum;\n    }\n\n    @Override\n    public boolean equals(final Object o) {\n        if (this == o)\n            return true;\n        if (!(o instanceof BloomFilterData))\n            return false;\n\n        final BloomFilterData that = (BloomFilterData) o;\n\n        if (bitNum != that.bitNum)\n            return false;\n        if (!Arrays.equals(bitPos, that.bitPos))\n            return false;\n\n        return true;\n    }\n\n    @Override\n    public int hashCode() {\n        int result = bitPos != null ? Arrays.hashCode(bitPos) : 0;\n        result = 31 * result + bitNum;\n        return result;\n    }\n\n    @Override\n    public String toString() {\n        return \"BloomFilterData{\" +\n            \"bitPos=\" + Arrays.toString(bitPos) +\n            \", bitNum=\" + bitNum +\n            '}';\n    }\n}\n"
  },
  {
    "path": "filter/src/test/java/org/apache/rocketmq/filter/BitsArrayTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter;\n\nimport org.apache.rocketmq.filter.util.BitsArray;\nimport org.junit.Test;\n\nimport java.util.Random;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class BitsArrayTest {\n\n    BitsArray gen(int bitCount) {\n        BitsArray bitsArray = BitsArray.create(bitCount);\n\n        for (int i = 0; i < bitCount / Byte.SIZE; i++) {\n            bitsArray.setByte(i, (byte) (new Random(System.currentTimeMillis())).nextInt(0xff));\n            try {\n                Thread.sleep(2);\n            } catch (InterruptedException e) {\n            }\n        }\n\n        return bitsArray;\n    }\n\n    int bitLength = Byte.SIZE;\n\n    @Test\n    public void testConstructor() {\n        BitsArray bitsArray = BitsArray.create(8);\n\n        assertThat(bitsArray.byteLength() == 1 && bitsArray.bitLength() == 8).isTrue();\n\n        bitsArray = BitsArray.create(9);\n\n        assertThat(bitsArray.byteLength() == 2 && bitsArray.bitLength() == 9).isTrue();\n\n        bitsArray = BitsArray.create(7);\n\n        assertThat(bitsArray.byteLength() == 1 && bitsArray.bitLength() == 7).isTrue();\n    }\n\n    @Test\n    public void testSet() {\n        BitsArray bitsArray = gen(bitLength);\n        BitsArray backUp = bitsArray.clone();\n\n        boolean val = bitsArray.getBit(2);\n\n        bitsArray.setBit(2, !val);\n\n        bitsArray.xor(backUp);\n\n        assertThat(bitsArray.getBit(2)).isTrue();\n    }\n\n    @Test\n    public void testAndOr() {\n        BitsArray bitsArray = gen(bitLength);\n\n        boolean val = bitsArray.getBit(2);\n\n        if (val) {\n            bitsArray.and(2, false);\n            assertThat(!bitsArray.getBit(2)).isTrue();\n        } else {\n            bitsArray.or(2, true);\n            assertThat(bitsArray.getBit(2)).isTrue();\n        }\n    }\n\n    @Test\n    public void testXor() {\n        BitsArray bitsArray = gen(bitLength);\n\n        boolean val = bitsArray.getBit(2);\n\n        bitsArray.xor(2, !val);\n\n        assertThat(bitsArray.getBit(2)).isTrue();\n    }\n\n    @Test\n    public void testNot() {\n        BitsArray bitsArray = gen(bitLength);\n        BitsArray backUp = bitsArray.clone();\n\n        bitsArray.not(2);\n\n        bitsArray.xor(backUp);\n\n        assertThat(bitsArray.getBit(2)).isTrue();\n    }\n\n    @Test\n    public void testOr() {\n        BitsArray b1 = BitsArray.create(new byte[] {(byte) 0xff, 0x00});\n        BitsArray b2 = BitsArray.create(new byte[] {0x00, (byte) 0xff});\n\n        b1.or(b2);\n\n        for (int i = 0; i < b1.bitLength(); i++) {\n            assertThat(b1.getBit(i)).isTrue();\n        }\n    }\n}\n"
  },
  {
    "path": "filter/src/test/java/org/apache/rocketmq/filter/BloomFilterTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter;\n\nimport org.apache.rocketmq.filter.util.BitsArray;\nimport org.apache.rocketmq.filter.util.BloomFilter;\nimport org.apache.rocketmq.filter.util.BloomFilterData;\nimport org.junit.Test;\n\nimport java.util.Random;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class BloomFilterTest {\n\n    @Test\n    public void testEquals() {\n        BloomFilter a = BloomFilter.createByFn(10, 20);\n\n        BloomFilter b = BloomFilter.createByFn(10, 20);\n\n        BloomFilter c = BloomFilter.createByFn(12, 20);\n\n        BloomFilter d = BloomFilter.createByFn(10, 30);\n\n        assertThat(a).isEqualTo(b);\n        assertThat(a).isNotEqualTo(c);\n        assertThat(a).isNotEqualTo(d);\n        assertThat(d).isNotEqualTo(c);\n\n        assertThat(a.hashCode()).isEqualTo(b.hashCode());\n        assertThat(a.hashCode()).isNotEqualTo(c.hashCode());\n        assertThat(a.hashCode()).isNotEqualTo(d.hashCode());\n        assertThat(c.hashCode()).isNotEqualTo(d.hashCode());\n    }\n\n    @Test\n    public void testHashTo() {\n        String cid = \"CID_abc_efg\";\n\n        BloomFilter bloomFilter = BloomFilter.createByFn(10, 20);\n\n        BitsArray bits = BitsArray.create(bloomFilter.getM());\n\n        int[] bitPos = bloomFilter.calcBitPositions(cid);\n\n        bloomFilter.hashTo(cid, bits);\n\n        for (int bit : bitPos) {\n            assertThat(bits.getBit(bit)).isTrue();\n        }\n    }\n\n    @Test\n    public void testCalcBitPositions() {\n        String cid = \"CID_abc_efg\";\n\n        BloomFilter bloomFilter = BloomFilter.createByFn(10, 20);\n\n        int[] bitPos = bloomFilter.calcBitPositions(cid);\n\n        assertThat(bitPos).isNotNull();\n        assertThat(bitPos.length).isEqualTo(bloomFilter.getK());\n\n        int[] bitPos2 = bloomFilter.calcBitPositions(cid);\n\n        assertThat(bitPos2).isNotNull();\n        assertThat(bitPos2.length).isEqualTo(bloomFilter.getK());\n\n        assertThat(bitPos).isEqualTo(bitPos2);\n    }\n\n    @Test\n    public void testIsHit() {\n        String cid = \"CID_abc_efg\";\n        String cid2 = \"CID_abc_123\";\n\n        BloomFilter bloomFilter = BloomFilter.createByFn(10, 20);\n\n        BitsArray bits = BitsArray.create(bloomFilter.getM());\n\n        bloomFilter.hashTo(cid, bits);\n\n        assertThat(bloomFilter.isHit(cid, bits)).isTrue();\n        assertThat(!bloomFilter.isHit(cid2, bits)).isTrue();\n\n        bloomFilter.hashTo(cid2, bits);\n\n        assertThat(bloomFilter.isHit(cid, bits)).isTrue();\n        assertThat(bloomFilter.isHit(cid2, bits)).isTrue();\n    }\n\n    @Test\n    public void testBloomFilterData() {\n        BloomFilterData bloomFilterData = new BloomFilterData(new int[] {1, 2, 3}, 128);\n        BloomFilterData bloomFilterData1 = new BloomFilterData(new int[] {1, 2, 3}, 128);\n        BloomFilterData bloomFilterData2 = new BloomFilterData(new int[] {1, 2, 3}, 129);\n\n        assertThat(bloomFilterData).isEqualTo(bloomFilterData1);\n        assertThat(bloomFilterData2).isNotEqualTo(bloomFilterData);\n        assertThat(bloomFilterData2).isNotEqualTo(bloomFilterData1);\n\n        assertThat(bloomFilterData.hashCode()).isEqualTo(bloomFilterData1.hashCode());\n        assertThat(bloomFilterData2.hashCode()).isNotEqualTo(bloomFilterData.hashCode());\n        assertThat(bloomFilterData2.hashCode()).isNotEqualTo(bloomFilterData1.hashCode());\n\n        assertThat(bloomFilterData.getBitPos()).isEqualTo(bloomFilterData2.getBitPos());\n        assertThat(bloomFilterData.getBitNum()).isEqualTo(bloomFilterData1.getBitNum());\n        assertThat(bloomFilterData.getBitNum()).isNotEqualTo(bloomFilterData2.getBitNum());\n\n        bloomFilterData2.setBitNum(128);\n\n        assertThat(bloomFilterData).isEqualTo(bloomFilterData2);\n\n        bloomFilterData2.setBitPos(new int[] {1, 2, 3, 4});\n\n        assertThat(bloomFilterData).isNotEqualTo(bloomFilterData2);\n\n        BloomFilterData nullData = new BloomFilterData();\n\n        assertThat(nullData.getBitNum()).isEqualTo(0);\n        assertThat(nullData.getBitPos()).isNull();\n\n        BloomFilter bloomFilter = BloomFilter.createByFn(1, 300);\n\n        assertThat(bloomFilter).isNotNull();\n        assertThat(bloomFilter.isValid(bloomFilterData)).isFalse();\n    }\n\n    @Test\n    public void testCheckFalseHit() {\n        BloomFilter bloomFilter = BloomFilter.createByFn(1, 300);\n        BitsArray bits = BitsArray.create(bloomFilter.getM());\n        int falseHit = 0;\n        for (int i = 0; i < bloomFilter.getN(); i++) {\n            String str = randomString((new Random(System.nanoTime())).nextInt(127) + 10);\n            int[] bitPos = bloomFilter.calcBitPositions(str);\n\n            if (bloomFilter.checkFalseHit(bitPos, bits)) {\n                falseHit++;\n            }\n\n            bloomFilter.hashTo(bitPos, bits);\n        }\n\n        assertThat(falseHit).isLessThanOrEqualTo(bloomFilter.getF() * bloomFilter.getN() / 100);\n    }\n\n    private String randomString(int length) {\n        StringBuilder stringBuilder = new StringBuilder(length);\n        for (int i = 0; i < length; i++) {\n            stringBuilder.append((char) ((new Random(System.nanoTime())).nextInt(123 - 97) + 97));\n        }\n\n        return stringBuilder.toString();\n    }\n}\n"
  },
  {
    "path": "filter/src/test/java/org/apache/rocketmq/filter/ExpressionTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter;\n\nimport org.apache.rocketmq.filter.expression.ComparisonExpression;\nimport org.apache.rocketmq.filter.expression.ConstantExpression;\nimport org.apache.rocketmq.filter.expression.EvaluationContext;\nimport org.apache.rocketmq.filter.expression.Expression;\nimport org.apache.rocketmq.filter.expression.MQFilterException;\nimport org.apache.rocketmq.filter.expression.PropertyExpression;\nimport org.apache.rocketmq.filter.parser.SelectorParser;\nimport org.junit.Test;\n\nimport java.util.HashMap;\nimport java.util.Map;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class ExpressionTest {\n\n    private static String andExpression = \"a=3 and b<>4 And c>5 AND d<=4\";\n    private static String orExpression = \"a=3 or b<>4 Or c>5 OR d<=4\";\n    private static String inExpression = \"a in ('3', '4', '5')\";\n    private static String notInExpression = \"a not in ('3', '4', '5')\";\n    private static String betweenExpression = \"a between 2 and 10\";\n    private static String notBetweenExpression = \"a not between 2 and 10\";\n    private static String isNullExpression = \"a is null\";\n    private static String isNotNullExpression = \"a is not null\";\n    private static String equalExpression = \"a is not null and a='hello'\";\n    private static String booleanExpression = \"a=TRUE OR b=FALSE\";\n    private static String nullOrExpression = \"a is null OR a='hello'\";\n    private static String stringHasString = \"TAGS is not null and TAGS='''''tag'''''\";\n\n\n    @Test\n    public void testContains_StartsWith_EndsWith_has() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"value\", \"axb\")\n        );\n        eval(genExp(\"value contains 'x'\"), context, Boolean.TRUE);\n        eval(genExp(\"value startswith 'ax'\"), context, Boolean.TRUE);\n        eval(genExp(\"value endswith 'xb'\"), context, Boolean.TRUE);\n    }\n\n    @Test\n    public void test_notContains_notStartsWith_notEndsWith_has() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"value\", \"axb\")\n        );\n        eval(genExp(\"value not contains 'x'\"), context, Boolean.FALSE);\n        eval(genExp(\"value not startswith 'ax'\"), context, Boolean.FALSE);\n        eval(genExp(\"value not endswith 'xb'\"), context, Boolean.FALSE);\n    }\n\n    @Test\n    public void testContains_StartsWith_EndsWith_has_not() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"value\", \"abb\")\n        );\n        eval(genExp(\"value contains 'x'\"), context, Boolean.FALSE);\n        eval(genExp(\"value startswith 'x'\"), context, Boolean.FALSE);\n        eval(genExp(\"value endswith 'x'\"), context, Boolean.FALSE);\n    }\n\n    @Test\n    public void test_notContains_notStartsWith_notEndsWith_has_not() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"value\", \"abb\")\n        );\n        eval(genExp(\"value not contains 'x'\"), context, Boolean.TRUE);\n        eval(genExp(\"value not startswith 'x'\"), context, Boolean.TRUE);\n        eval(genExp(\"value not endswith 'x'\"), context, Boolean.TRUE);\n    }\n\n    @Test\n    public void testContains_StartsWith_EndsWith_hasEmpty() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"value\", \"axb\")\n        );\n        eval(genExp(\"value contains ''\"), context, Boolean.FALSE);\n        eval(genExp(\"value startswith ''\"), context, Boolean.FALSE);\n        eval(genExp(\"value endswith ''\"), context, Boolean.FALSE);\n    }\n\n    @Test\n    public void test_notContains_notStartsWith_notEndsWith_hasEmpty() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"value\", \"axb\")\n        );\n        eval(genExp(\"value not contains ''\"), context, Boolean.FALSE);\n        eval(genExp(\"value not startswith ''\"), context, Boolean.FALSE);\n        eval(genExp(\"value not endswith ''\"), context, Boolean.FALSE);\n    }\n\n    @Test\n    public void testContains_StartsWith_EndsWith_null_has_1() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"value\", null)\n        );\n        eval(genExp(\"value contains 'x'\"), context, Boolean.FALSE);\n        eval(genExp(\"value startswith 'x'\"), context, Boolean.FALSE);\n        eval(genExp(\"value endswith 'x'\"), context, Boolean.FALSE);\n    }\n\n    @Test\n    public void test_notContains_notStartsWith_notEndsWith_null_has_1() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"value\", null)\n        );\n        eval(genExp(\"value not contains 'x'\"), context, Boolean.FALSE);\n        eval(genExp(\"value not startswith 'x'\"), context, Boolean.FALSE);\n        eval(genExp(\"value not endswith 'x'\"), context, Boolean.FALSE);\n    }\n\n    @Test\n    public void testContains_StartsWith_EndsWith_null_has_2() throws Exception {\n        EvaluationContext context = genContext(\n//                KeyValue.c(\"value\", null)\n        );\n        eval(genExp(\"value contains 'x'\"), context, Boolean.FALSE);\n        eval(genExp(\"value startswith 'x'\"), context, Boolean.FALSE);\n        eval(genExp(\"value endswith 'x'\"), context, Boolean.FALSE);\n    }\n\n    @Test\n    public void test_notContains_notStartsWith_notEndsWith_null_has_2() throws Exception {\n        EvaluationContext context = genContext(\n//                KeyValue.c(\"value\", null)\n        );\n        eval(genExp(\"value not contains 'x'\"), context, Boolean.FALSE);\n        eval(genExp(\"value not startswith 'x'\"), context, Boolean.FALSE);\n        eval(genExp(\"value not endswith 'x'\"), context, Boolean.FALSE);\n    }\n\n    @Test\n    public void testContains_StartsWith_EndsWith_number_has() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"value\", 1.23)\n        );\n        eval(genExp(\"value contains 'x'\"), context, Boolean.FALSE);\n        eval(genExp(\"value startswith 'x'\"), context, Boolean.FALSE);\n        eval(genExp(\"value endswith 'x'\"), context, Boolean.FALSE);\n    }\n\n    @Test\n    public void test_notContains_notStartsWith_notEndsWith_number_has() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"value\", 1.23)\n        );\n        eval(genExp(\"value not contains 'x'\"), context, Boolean.FALSE);\n        eval(genExp(\"value not startswith 'x'\"), context, Boolean.FALSE);\n        eval(genExp(\"value not endswith 'x'\"), context, Boolean.FALSE);\n    }\n\n    @Test\n    public void testContains_StartsWith_EndsWith_boolean_has() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"value\", Boolean.TRUE)\n        );\n        eval(genExp(\"value contains 'x'\"), context, Boolean.FALSE);\n        eval(genExp(\"value startswith 'x'\"), context, Boolean.FALSE);\n        eval(genExp(\"value endswith 'x'\"), context, Boolean.FALSE);\n    }\n\n    @Test\n    public void test_notContains_notStartsWith_notEndsWith_boolean_has() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"value\", Boolean.TRUE)\n        );\n        eval(genExp(\"value not contains 'x'\"), context, Boolean.FALSE);\n        eval(genExp(\"value not startswith 'x'\"), context, Boolean.FALSE);\n        eval(genExp(\"value not endswith 'x'\"), context, Boolean.FALSE);\n    }\n\n    @Test\n    public void testContains_StartsWith_EndsWith_object_has() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"value\", new Object())\n        );\n        eval(genExp(\"value contains 'x'\"), context, Boolean.FALSE);\n        eval(genExp(\"value startswith 'x'\"), context, Boolean.FALSE);\n        eval(genExp(\"value endswith 'x'\"), context, Boolean.FALSE);\n    }\n\n    @Test\n    public void testContains_has_not_string_1() throws Exception {\n        try {\n            Expression expr = genExp(\"value contains x\");  // will throw parse exception.\n            EvaluationContext context = genContext(\n                    KeyValue.c(\"value\", \"axb\")\n            );\n            eval(expr, context, Boolean.FALSE);\n        } catch (Throwable e) {\n        }\n    }\n\n    @Test\n    public void test_notContains_has_not_string_1() throws Exception {\n        try {\n            Expression expr = genExp(\"value not contains x\");  // will throw parse exception.\n            EvaluationContext context = genContext(\n                    KeyValue.c(\"value\", \"axb\")\n            );\n            eval(expr, context, Boolean.FALSE);\n        } catch (Throwable e) {\n        }\n    }\n\n    @Test\n    public void testContains_has_not_string_2() throws Exception {\n        try {\n            Expression expr = genExp(\"value contains 123\");  // will throw parse exception.\n            EvaluationContext context = genContext(\n                    KeyValue.c(\"value\", \"axb\")\n            );\n            eval(expr, context, Boolean.FALSE);\n        } catch (Throwable e) {\n        }\n    }\n\n    @Test\n    public void test_notContains_has_not_string_2() throws Exception {\n        try {\n            Expression expr = genExp(\"value not contains 123\");  // will throw parse exception.\n            EvaluationContext context = genContext(\n                    KeyValue.c(\"value\", \"axb\")\n            );\n            eval(expr, context, Boolean.FALSE);\n        } catch (Throwable e) {\n        }\n    }\n\n    @Test\n    public void testContains_StartsWith_EndsWith_string_has_string() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"whatever\", \"whatever\")\n        );\n        eval(genExp(\"'axb' contains 'x'\"), context, Boolean.TRUE);\n        eval(genExp(\"'axb' startswith 'ax'\"), context, Boolean.TRUE);\n        eval(genExp(\"'axb' endswith 'xb'\"), context, Boolean.TRUE);\n    }\n\n    @Test\n    public void test_notContains_notStartsWith_notEndsWith_string_has_string() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"whatever\", \"whatever\")\n        );\n        eval(genExp(\"'axb' not contains 'x'\"), context, Boolean.FALSE);\n        eval(genExp(\"'axb' not startswith 'ax'\"), context, Boolean.FALSE);\n        eval(genExp(\"'axb' not endswith 'xb'\"), context, Boolean.FALSE);\n    }\n\n    @Test\n    public void testContains_startsWith_endsWith_string_has_not_string() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"whatever\", \"whatever\")\n        );\n        eval(genExp(\"'axb' contains 'u'\"), context, Boolean.FALSE);\n        eval(genExp(\"'axb' startswith 'u'\"), context, Boolean.FALSE);\n        eval(genExp(\"'axb' endswith 'u'\"), context, Boolean.FALSE);\n    }\n\n    @Test\n    public void test_notContains_notStartsWith_notEndsWith_string_has_not_string() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"whatever\", \"whatever\")\n        );\n        eval(genExp(\"'axb' not contains 'u'\"), context, Boolean.TRUE);\n        eval(genExp(\"'axb' not startswith 'u'\"), context, Boolean.TRUE);\n        eval(genExp(\"'axb' not endswith 'u'\"), context, Boolean.TRUE);\n    }\n\n    @Test\n    public void testContains_StartsWith_EndsWith_string_has_empty() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"whatever\", \"whatever\")\n        );\n        eval(genExp(\"'axb' contains ''\"), context, Boolean.FALSE);\n        eval(genExp(\"'axb' startswith ''\"), context, Boolean.FALSE);\n        eval(genExp(\"'axb' endswith ''\"), context, Boolean.FALSE);\n    }\n\n    @Test\n    public void test_notContains_notStartsWith_notEndsWith_string_has_empty() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"whatever\", \"whatever\")\n        );\n        eval(genExp(\"'axb' not contains ''\"), context, Boolean.FALSE);\n        eval(genExp(\"'axb' not startswith ''\"), context, Boolean.FALSE);\n        eval(genExp(\"'axb' not endswith ''\"), context, Boolean.FALSE);\n    }\n\n    @Test\n    public void testContains_StartsWith_EndsWith_string_has_space() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"whatever\", \"whatever\")\n        );\n        eval(genExp(\"' ' contains ' '\"), context, Boolean.TRUE);\n        eval(genExp(\"' ' startswith ' '\"), context, Boolean.TRUE);\n        eval(genExp(\"' ' endswith ' '\"), context, Boolean.TRUE);\n    }\n\n    @Test\n    public void test_notContains_notStartsWith_notEndsWith_string_has_space() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"whatever\", \"whatever\")\n        );\n        eval(genExp(\"' ' not contains ' '\"), context, Boolean.FALSE);\n        eval(genExp(\"' ' not startswith ' '\"), context, Boolean.FALSE);\n        eval(genExp(\"' ' not endswith ' '\"), context, Boolean.FALSE);\n    }\n\n    @Test\n    public void testContains_string_has_nothing() throws Exception {\n        try {\n            Expression expr = genExp(\"'axb' contains \");  // will throw parse exception.\n            EvaluationContext context = genContext(\n                    KeyValue.c(\"whatever\", \"whatever\")\n            );\n            eval(expr, context, Boolean.TRUE);\n        } catch (Throwable e) {\n        }\n    }\n\n    @Test\n    public void test_notContains_string_has_nothing() throws Exception {\n        try {\n            Expression expr = genExp(\"'axb' not contains \");  // will throw parse exception.\n            EvaluationContext context = genContext(\n                    KeyValue.c(\"whatever\", \"whatever\")\n            );\n            eval(expr, context, Boolean.TRUE);\n        } catch (Throwable e) {\n        }\n    }\n\n    @Test\n    public void testContains_StartsWith_EndsWith_string_has_special_1() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"whatever\", \"whatever\")\n        );\n        eval(genExp(\"'axb' contains '.'\"), context, Boolean.FALSE);\n        eval(genExp(\"'axb' startswith '.'\"), context, Boolean.FALSE);\n        eval(genExp(\"'axb' endswith '.'\"), context, Boolean.FALSE);\n    }\n\n    @Test\n    public void test_notContains_notStartsWith_notEndsWith_string_has_special_1() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"whatever\", \"whatever\")\n        );\n        eval(genExp(\"'axb' not contains '.'\"), context, Boolean.TRUE);\n        eval(genExp(\"'axb' not startswith '.'\"), context, Boolean.TRUE);\n        eval(genExp(\"'axb' not endswith '.'\"), context, Boolean.TRUE);\n    }\n\n    @Test\n    public void testContains_StartsWith_EndsWith_string_has_special_2() throws Exception {\n        EvaluationContext context = genContext(\n                KeyValue.c(\"whatever\", \"whatever\")\n        );\n        eval(genExp(\"'s' contains '\\\\'\"), context, Boolean.FALSE);\n        eval(genExp(\"'s' startswith '\\\\'\"), context, Boolean.FALSE);\n        eval(genExp(\"'s' endswith '\\\\'\"), context, Boolean.FALSE);\n    }\n\n    @Test\n    public void testContainsAllInOne() throws Exception {\n        Expression expr = genExp(\"a not in ('4', '4', '5') and b between 3 and 10 and c not contains 'axbc'\");\n        EvaluationContext context = genContext(\n                KeyValue.c(\"a\", \"3\"),\n                KeyValue.c(\"b\", 3),\n                KeyValue.c(\"c\", \"axbdc\")\n        );\n        eval(expr, context, Boolean.TRUE);\n    }\n\n    @Test\n    public void testStartsWithAllInOne() throws Exception {\n        Expression expr = genExp(\"a not in ('4', '4', '5') and b between 3 and 10 and c not startswith 'axbc'\");\n        EvaluationContext context = genContext(\n                KeyValue.c(\"a\", \"3\"),\n                KeyValue.c(\"b\", 3),\n                KeyValue.c(\"c\", \"axbdc\")\n        );\n        eval(expr, context, Boolean.TRUE);\n    }\n\n    @Test\n    public void testEndsWithAllInOne() throws Exception {\n        Expression expr = genExp(\"a not in ('4', '4', '5') and b between 3 and 10 and c not endswith 'axbc'\");\n        EvaluationContext context = genContext(\n                KeyValue.c(\"a\", \"3\"),\n                KeyValue.c(\"b\", 3),\n                KeyValue.c(\"c\", \"axbdc\")\n        );\n        eval(expr, context, Boolean.TRUE);\n    }\n\n    @Test\n    public void testEvaluate_stringHasString() throws Exception {\n        Expression expr = genExp(stringHasString);\n\n        EvaluationContext context = genContext(\n            KeyValue.c(\"TAGS\", \"''tag''\")\n        );\n\n        eval(expr, context, Boolean.TRUE);\n    }\n\n    @Test\n    public void testEvaluate_now() throws Exception {\n        EvaluationContext context = genContext(\n            KeyValue.c(\"a\", System.currentTimeMillis())\n        );\n\n        Expression nowExpression = ConstantExpression.createNow();\n        Expression propertyExpression = new PropertyExpression(\"a\");\n\n        Expression expression = ComparisonExpression.createLessThanEqual(propertyExpression,\n            nowExpression);\n\n        eval(expression, context, Boolean.TRUE);\n    }\n\n    @Test(expected = RuntimeException.class)\n    public void testEvaluate_stringCompare() throws Exception {\n        Expression expression = genExp(\"a between up and low\");\n\n        EvaluationContext context = genContext(\n            KeyValue.c(\"a\", \"3.14\")\n        );\n\n        eval(expression, context, Boolean.FALSE);\n\n        {\n            context = genContext(\n                KeyValue.c(\"a\", \"3.14\"),\n                KeyValue.c(\"up\", \"up\"),\n                KeyValue.c(\"low\", \"low\")\n            );\n\n            eval(expression, context, Boolean.FALSE);\n        }\n\n        {\n            expression = genExp(\"key is not null and key between 0 and 100\");\n\n            context = genContext(\n                KeyValue.c(\"key\", \"con\")\n            );\n\n            eval(expression, context, Boolean.FALSE);\n        }\n\n        {\n            expression = genExp(\"a between 0 and 100\");\n\n            context = genContext(\n                KeyValue.c(\"a\", \"abc\")\n            );\n\n            eval(expression, context, Boolean.FALSE);\n        }\n\n        {\n            expression = genExp(\"a=b\");\n\n            context = genContext(\n                KeyValue.c(\"a\", \"3.14\"),\n                KeyValue.c(\"b\", \"3.14\")\n            );\n\n            eval(expression, context, Boolean.TRUE);\n        }\n\n        {\n            expression = genExp(\"a<>b\");\n\n            context = genContext(\n                KeyValue.c(\"a\", \"3.14\"),\n                KeyValue.c(\"b\", \"3.14\")\n            );\n\n            eval(expression, context, Boolean.FALSE);\n        }\n\n        {\n            expression = genExp(\"a<>b\");\n\n            context = genContext(\n                KeyValue.c(\"a\", \"3.14\"),\n                KeyValue.c(\"b\", \"3.141\")\n            );\n\n            eval(expression, context, Boolean.TRUE);\n        }\n    }\n\n    @Test\n    public void testEvaluate_exponent() throws Exception {\n        Expression expression = genExp(\"a > 3.1E10\");\n\n        EvaluationContext context = genContext(\n            KeyValue.c(\"a\", String.valueOf(3.1415 * Math.pow(10, 10)))\n        );\n\n        eval(expression, context, Boolean.TRUE);\n    }\n\n    @Test\n    public void testEvaluate_floatNumber() throws Exception {\n        Expression expression = genExp(\"a > 3.14\");\n\n        EvaluationContext context = genContext(\n            KeyValue.c(\"a\", String.valueOf(3.1415))\n        );\n\n        eval(expression, context, Boolean.TRUE);\n    }\n\n    @Test\n    public void testEvaluate_twoVariable() throws Exception {\n        Expression expression = genExp(\"a > b\");\n\n        EvaluationContext context = genContext(\n            KeyValue.c(\"a\", String.valueOf(10)),\n            KeyValue.c(\"b\", String.valueOf(20))\n        );\n\n        eval(expression, context, Boolean.FALSE);\n    }\n\n    @Test\n    public void testEvaluate_twoVariableGt() throws Exception {\n        Expression expression = genExp(\"a > b\");\n        EvaluationContext context = genContext(\n            KeyValue.c(\"b\", String.valueOf(10)),\n            KeyValue.c(\"a\", String.valueOf(20))\n        );\n\n        eval(expression, context, Boolean.TRUE);\n    }\n\n    @Test\n    public void testEvaluate_nullOr() throws Exception {\n        Expression expression = genExp(nullOrExpression);\n\n        EvaluationContext context = genContext(\n        );\n\n        eval(expression, context, Boolean.TRUE);\n\n        context = genContext(\n            KeyValue.c(\"a\", \"hello\")\n        );\n\n        eval(expression, context, Boolean.TRUE);\n\n        context = genContext(\n            KeyValue.c(\"a\", \"abc\")\n        );\n\n        eval(expression, context, Boolean.FALSE);\n    }\n\n    @Test\n    public void testEvaluate_boolean() throws Exception {\n        Expression expression = genExp(booleanExpression);\n\n        EvaluationContext context = genContext(\n            KeyValue.c(\"a\", \"true\"),\n            KeyValue.c(\"b\", \"false\")\n        );\n\n        eval(expression, context, Boolean.TRUE);\n\n        context = genContext(\n            KeyValue.c(\"a\", \"false\"),\n            KeyValue.c(\"b\", \"true\")\n        );\n\n        eval(expression, context, Boolean.FALSE);\n    }\n\n    @Test\n    public void testEvaluate_equal() throws Exception {\n        Expression expression = genExp(equalExpression);\n\n        EvaluationContext context = genContext(\n            KeyValue.c(\"a\", \"hello\")\n        );\n\n        eval(expression, context, Boolean.TRUE);\n\n        context = genContext(\n        );\n\n        eval(expression, context, Boolean.FALSE);\n    }\n\n    @Test\n    public void testEvaluate_andTrue() throws Exception {\n        Expression expression = genExp(andExpression);\n\n        EvaluationContext context = genContext(\n            KeyValue.c(\"a\", 3),\n            KeyValue.c(\"b\", 5),\n            KeyValue.c(\"c\", 6),\n            KeyValue.c(\"d\", 1)\n        );\n\n        for (int i = 0; i < 500; i++) {\n            eval(expression, context, Boolean.TRUE);\n        }\n\n        long start = System.currentTimeMillis();\n        for (int j = 0; j < 100; j++) {\n            for (int i = 0; i < 1000; i++) {\n                eval(expression, context, Boolean.TRUE);\n            }\n        }\n\n        // use string\n        context = genContext(\n            KeyValue.c(\"a\", \"3\"),\n            KeyValue.c(\"b\", \"5\"),\n            KeyValue.c(\"c\", \"6\"),\n            KeyValue.c(\"d\", \"1\")\n        );\n\n        eval(expression, context, Boolean.TRUE);\n    }\n\n    @Test\n    public void testEvaluate_andFalse() throws Exception {\n        Expression expression = genExp(andExpression);\n\n        EvaluationContext context = genContext(\n            KeyValue.c(\"a\", 4),\n            KeyValue.c(\"b\", 5),\n            KeyValue.c(\"c\", 6),\n            KeyValue.c(\"d\", 1)\n        );\n\n        eval(expression, context, Boolean.FALSE);\n\n        // use string\n        context = genContext(\n            KeyValue.c(\"a\", \"4\"),\n            KeyValue.c(\"b\", \"5\"),\n            KeyValue.c(\"c\", \"6\"),\n            KeyValue.c(\"d\", \"1\")\n        );\n\n        eval(expression, context, Boolean.FALSE);\n    }\n\n    @Test\n    public void testEvaluate_orTrue() throws Exception {\n        Expression expression = genExp(orExpression);\n\n        // first\n        EvaluationContext context = genContext(\n            KeyValue.c(\"a\", 3)\n        );\n        eval(expression, context, Boolean.TRUE);\n\n        // second\n        context = genContext(\n            KeyValue.c(\"a\", 4),\n            KeyValue.c(\"b\", 5)\n        );\n        eval(expression, context, Boolean.TRUE);\n\n        // third\n        context = genContext(\n            KeyValue.c(\"a\", 4),\n            KeyValue.c(\"b\", 4),\n            KeyValue.c(\"c\", 6)\n        );\n        eval(expression, context, Boolean.TRUE);\n\n        // forth\n        context = genContext(\n            KeyValue.c(\"a\", 4),\n            KeyValue.c(\"b\", 4),\n            KeyValue.c(\"c\", 3),\n            KeyValue.c(\"d\", 2)\n        );\n        eval(expression, context, Boolean.TRUE);\n    }\n\n    @Test\n    public void testEvaluate_orFalse() throws Exception {\n        Expression expression = genExp(orExpression);\n        // forth\n        EvaluationContext context = genContext(\n            KeyValue.c(\"a\", 4),\n            KeyValue.c(\"b\", 4),\n            KeyValue.c(\"c\", 3),\n            KeyValue.c(\"d\", 10)\n        );\n        eval(expression, context, Boolean.FALSE);\n    }\n\n    @Test\n    public void testEvaluate_inTrue() throws Exception {\n        Expression expression = genExp(inExpression);\n\n        EvaluationContext context = genContext(\n            KeyValue.c(\"a\", \"3\")\n        );\n        eval(expression, context, Boolean.TRUE);\n\n        context = genContext(\n            KeyValue.c(\"a\", \"4\")\n        );\n        eval(expression, context, Boolean.TRUE);\n\n        context = genContext(\n            KeyValue.c(\"a\", \"5\")\n        );\n        eval(expression, context, Boolean.TRUE);\n    }\n\n    @Test\n    public void testEvaluate_inFalse() throws Exception {\n        Expression expression = genExp(inExpression);\n\n        EvaluationContext context = genContext(\n            KeyValue.c(\"a\", \"8\")\n        );\n        eval(expression, context, Boolean.FALSE);\n    }\n\n    @Test\n    public void testEvaluate_notInTrue() throws Exception {\n        Expression expression = genExp(notInExpression);\n\n        EvaluationContext context = genContext(\n            KeyValue.c(\"a\", \"8\")\n        );\n        eval(expression, context, Boolean.TRUE);\n    }\n\n    @Test\n    public void testEvaluate_notInFalse() throws Exception {\n        Expression expression = genExp(notInExpression);\n\n        EvaluationContext context = genContext(\n            KeyValue.c(\"a\", \"3\")\n        );\n        eval(expression, context, Boolean.FALSE);\n\n        context = genContext(\n            KeyValue.c(\"a\", \"4\")\n        );\n        eval(expression, context, Boolean.FALSE);\n\n        context = genContext(\n            KeyValue.c(\"a\", \"5\")\n        );\n        eval(expression, context, Boolean.FALSE);\n    }\n\n    @Test\n    public void testEvaluate_betweenTrue() throws Exception {\n        Expression expression = genExp(betweenExpression);\n\n        EvaluationContext context = genContext(\n            KeyValue.c(\"a\", \"2\")\n        );\n        eval(expression, context, Boolean.TRUE);\n\n        context = genContext(\n            KeyValue.c(\"a\", \"10\")\n        );\n        eval(expression, context, Boolean.TRUE);\n\n        context = genContext(\n            KeyValue.c(\"a\", \"3\")\n        );\n        eval(expression, context, Boolean.TRUE);\n    }\n\n    @Test\n    public void testEvaluate_betweenFalse() throws Exception {\n        Expression expression = genExp(betweenExpression);\n\n        EvaluationContext context = genContext(\n            KeyValue.c(\"a\", \"1\")\n        );\n        eval(expression, context, Boolean.FALSE);\n\n        context = genContext(\n            KeyValue.c(\"a\", \"11\")\n        );\n        eval(expression, context, Boolean.FALSE);\n    }\n\n    @Test\n    public void testEvaluate_notBetweenTrue() throws Exception {\n        Expression expression = genExp(notBetweenExpression);\n\n        EvaluationContext context = genContext(\n            KeyValue.c(\"a\", \"1\")\n        );\n        eval(expression, context, Boolean.TRUE);\n\n        context = genContext(\n            KeyValue.c(\"a\", \"11\")\n        );\n        eval(expression, context, Boolean.TRUE);\n    }\n\n    @Test\n    public void testEvaluate_notBetweenFalse() throws Exception {\n        Expression expression = genExp(notBetweenExpression);\n\n        EvaluationContext context = genContext(\n            KeyValue.c(\"a\", \"2\")\n        );\n        eval(expression, context, Boolean.FALSE);\n\n        context = genContext(\n            KeyValue.c(\"a\", \"10\")\n        );\n        eval(expression, context, Boolean.FALSE);\n\n        context = genContext(\n            KeyValue.c(\"a\", \"3\")\n        );\n        eval(expression, context, Boolean.FALSE);\n    }\n\n    @Test\n    public void testEvaluate_isNullTrue() throws Exception {\n        Expression expression = genExp(isNullExpression);\n\n        EvaluationContext context = genContext(\n            KeyValue.c(\"abc\", \"2\")\n        );\n        eval(expression, context, Boolean.TRUE);\n    }\n\n    @Test\n    public void testEvaluate_isNullFalse() throws Exception {\n        Expression expression = genExp(isNullExpression);\n\n        EvaluationContext context = genContext(\n            KeyValue.c(\"a\", \"2\")\n        );\n        eval(expression, context, Boolean.FALSE);\n    }\n\n    @Test\n    public void testEvaluate_isNotNullTrue() throws Exception {\n        Expression expression = genExp(isNotNullExpression);\n\n        EvaluationContext context = genContext(\n            KeyValue.c(\"a\", \"2\")\n        );\n        eval(expression, context, Boolean.TRUE);\n    }\n\n    @Test\n    public void testEvaluate_isNotNullFalse() throws Exception {\n        Expression expression = genExp(isNotNullExpression);\n\n        EvaluationContext context = genContext(\n            KeyValue.c(\"abc\", \"2\")\n        );\n        eval(expression, context, Boolean.FALSE);\n    }\n\n    protected void eval(Expression expression, EvaluationContext context, Boolean result) throws Exception {\n        Object ret = expression.evaluate(context);\n        if (ret == null || !(ret instanceof Boolean)) {\n            assertThat(result).isFalse();\n        } else {\n            assertThat(result).isEqualTo(ret);\n        }\n    }\n\n    protected EvaluationContext genContext(KeyValue... keyValues) {\n        if (keyValues == null || keyValues.length < 1) {\n            return new PropertyContext();\n        }\n\n        PropertyContext context = new PropertyContext();\n        for (KeyValue keyValue : keyValues) {\n            context.properties.put(keyValue.key, keyValue.value);\n        }\n\n        return context;\n    }\n\n    protected Expression genExp(String exp) {\n        Expression expression = null;\n\n        try {\n            expression = SelectorParser.parse(exp);\n\n            assertThat(expression).isNotNull();\n        } catch (MQFilterException e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n\n        return expression;\n    }\n\n    static class KeyValue {\n        public static KeyValue c(String key, Object value) {\n            return new KeyValue(key, value);\n        }\n\n        public KeyValue(String key, Object value) {\n            this.key = key;\n            this.value = value;\n        }\n\n        public String key;\n        public Object value;\n    }\n\n    class PropertyContext implements EvaluationContext {\n\n        public Map<String, Object> properties = new HashMap<>(8);\n\n        @Override\n        public Object get(final String name) {\n            return properties.get(name);\n        }\n\n        @Override\n        public Map<String, Object> keyValues() {\n            return properties;\n        }\n\n    }\n}\n"
  },
  {
    "path": "filter/src/test/java/org/apache/rocketmq/filter/FilterSpiTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter;\n\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.filter.expression.EmptyEvaluationContext;\nimport org.apache.rocketmq.filter.expression.EvaluationContext;\nimport org.apache.rocketmq.filter.expression.Expression;\nimport org.apache.rocketmq.filter.expression.MQFilterException;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class FilterSpiTest {\n\n    static class NothingExpression implements Expression {\n\n        @Override\n        public Object evaluate(final EvaluationContext context) throws Exception {\n            return Boolean.TRUE;\n        }\n    }\n\n    static class NothingFilter implements FilterSpi {\n        @Override\n        public Expression compile(final String expr) throws MQFilterException {\n            return new NothingExpression();\n        }\n\n        @Override\n        public String ofType() {\n            return \"Nothing\";\n        }\n    }\n\n    @Test\n    public void testRegister() {\n        FilterFactory.INSTANCE.register(new NothingFilter());\n\n        Expression expr = null;\n        try {\n            expr = FilterFactory.INSTANCE.get(\"Nothing\").compile(\"abc\");\n        } catch (MQFilterException e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n\n        assertThat(expr).isNotNull();\n\n        try {\n            assertThat((Boolean) expr.evaluate(new EmptyEvaluationContext())).isTrue();\n        } catch (Exception e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n        FilterFactory.INSTANCE.unRegister(\"Nothing\");\n    }\n\n    @Test\n    public void testGet() {\n        try {\n            assertThat((Boolean) FilterFactory.INSTANCE.get(ExpressionType.SQL92).compile(\"a is not null and a > 0\")\n                .evaluate(new EmptyEvaluationContext())).isFalse();\n        } catch (Exception e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n    }\n}\n"
  },
  {
    "path": "filter/src/test/java/org/apache/rocketmq/filter/ParserTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.filter;\n\nimport org.apache.rocketmq.filter.expression.Expression;\nimport org.apache.rocketmq.filter.expression.MQFilterException;\nimport org.apache.rocketmq.filter.parser.SelectorParser;\nimport org.junit.Test;\n\nimport java.util.Arrays;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class ParserTest {\n\n    private static String andExpression = \"a=3 and b<>4 And c>5 AND d<=4\";\n    private static String andExpressionHasBlank = \"a=3  and    b<>4 And c>5 AND d<=4\";\n    private static String orExpression = \"a=3 or b<>4 Or c>5 OR d<=4\";\n    private static String inExpression = \"a in ('3', '4', '5')\";\n    private static String notInExpression = \"(a not in ('6', '4', '5')) or (b in ('3', '4', '5'))\";\n    private static String betweenExpression = \"(a between 2 and 10) AND (b not between 6 and 9)\";\n    private static String equalNullExpression = \"a is null\";\n    private static String notEqualNullExpression = \"a is not null\";\n    private static String nowExpression = \"a <= now\";\n    private static String containsExpression = \"a=3 and b contains 'xxx' and c not contains 'xxx'\";\n    private static String invalidExpression = \"a and between 2 and 10\";\n    private static String illegalBetween = \" a between 10 and 0\";\n\n    @Test\n    public void testParse_valid() {\n        for (String expr : Arrays.asList(\n            andExpression, orExpression, inExpression, notInExpression, betweenExpression,\n            equalNullExpression, notEqualNullExpression, nowExpression, containsExpression\n        )) {\n\n            try {\n                Expression expression = SelectorParser.parse(expr);\n                assertThat(expression).isNotNull();\n            } catch (MQFilterException e) {\n                e.printStackTrace();\n                assertThat(Boolean.FALSE).isTrue();\n            }\n\n        }\n    }\n\n    @Test\n    public void testParse_invalid() {\n        try {\n            SelectorParser.parse(invalidExpression);\n\n            assertThat(Boolean.TRUE).isFalse();\n        } catch (MQFilterException e) {\n        }\n    }\n\n    @Test\n    public void testParse_decimalOverFlow() {\n        try {\n            String str = \"100000000000000000000000\";\n\n            SelectorParser.parse(\"a > \" + str);\n\n            assertThat(Boolean.TRUE).isFalse();\n        } catch (Exception e) {\n        }\n    }\n\n    @Test\n    public void testParse_floatOverFlow() {\n        try {\n            StringBuilder sb = new StringBuilder(210000);\n            sb.append(\"1\");\n            for (int i = 0; i < 2048; i ++) {\n                sb.append(\"111111111111111111111111111111111111111111111111111\");\n            }\n            sb.append(\".\");\n            for (int i = 0; i < 2048; i ++) {\n                sb.append(\"111111111111111111111111111111111111111111111111111\");\n            }\n            String str = sb.toString();\n\n\n            SelectorParser.parse(\"a > \" + str);\n\n            assertThat(Boolean.TRUE).isFalse();\n        } catch (Exception e) {\n        }\n    }\n\n    @Test\n    public void testParse_illegalBetween() {\n        try {\n            SelectorParser.parse(illegalBetween);\n\n            assertThat(Boolean.TRUE).isFalse();\n        } catch (Exception e) {\n        }\n    }\n\n    @Test\n    public void testEquals() {\n        try {\n            Expression expr1 = SelectorParser.parse(andExpression);\n\n            Expression expr2 = SelectorParser.parse(andExpressionHasBlank);\n\n            Expression expr3 = SelectorParser.parse(orExpression);\n\n            assertThat(expr1).isEqualTo(expr2);\n            assertThat(expr1).isNotEqualTo(expr3);\n        } catch (MQFilterException e) {\n            e.printStackTrace();\n            assertThat(Boolean.TRUE).isFalse();\n        }\n    }\n}\n"
  },
  {
    "path": "filter/src/test/resources/rmq.logback-test.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n<configuration>\n\n    <appender name=\"CONSOLE\" class=\"ch.qos.logback.core.ConsoleAppender\">\n        <layout class=\"ch.qos.logback.classic.PatternLayout\">\n            <Pattern>\n                %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n\n            </Pattern>\n        </layout>\n    </appender>\n\n    <logger name=\"org.apache.rocketmq\" level=\"error\" additivity=\"false\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </logger>\n\n    <root level=\"error\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </root>\n\n</configuration>"
  },
  {
    "path": "namesrv/BUILD.bazel",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\nload(\"//bazel:GenTestRules.bzl\", \"GenTestRules\")\n\njava_library(\n    name = \"namesrv\",\n    srcs = glob([\"src/main/java/**/*.java\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \"//remoting\",\n        \"//srvutil\",\n        \"//tools\",\n        \"//client\",\n        \"//common\",\n        \"//controller\",\n        \"@maven//:org_apache_commons_commons_lang3\",\n        \"@maven//:commons_validator_commons_validator\",\n        \"@maven//:com_github_luben_zstd_jni\",\n        \"@maven//:org_lz4_lz4_java\",\n        \"@maven//:com_alibaba_fastjson2_fastjson2\",\n        \"@maven//:io_netty_netty_all\",\n        \"@maven//:ch_qos_logback_logback_classic\",\n        \"@maven//:ch_qos_logback_logback_core\",\n        \"@maven//:org_slf4j_slf4j_api\",\n        \"@maven//:org_bouncycastle_bcpkix_jdk15on\",\n        \"@maven//:commons_cli_commons_cli\",\n        \"@maven//:com_google_guava_guava\",\n        \"@maven//:io_github_aliyunmq_rocketmq_slf4j_api\",\n        \"@maven//:io_github_aliyunmq_rocketmq_logback_classic\",\n    ],\n)\n\njava_library(\n    name = \"tests\",\n    srcs = glob([\"src/test/java/**/*.java\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \":namesrv\",\n        \"//remoting\",\n        \"//srvutil\",\n        \"//tools\",\n        \"//client\",\n        \"//common\",\n        \"//:test_deps\",\n        \"@maven//:org_apache_commons_commons_lang3\", \n        \"@maven//:commons_cli_commons_cli\",\n        \"@maven//:io_netty_netty_all\",    \n        \"@maven//:com_google_guava_guava\",   \n        \"@maven//:com_alibaba_fastjson2_fastjson2\",\n        \"@maven//:io_github_aliyunmq_rocketmq_slf4j_api\",\n    ],\n    resources = glob([\"src/test/resources/certs/*.pem\"]) + glob([\"src/test/resources/certs/*.key\"])\n)\n\nGenTestRules(\n    name = \"GeneratedTestRules\",\n    test_files = glob([\"src/test/java/**/*Test.java\"]),\n    deps = [\n        \":tests\",\n    ],\n)\n"
  },
  {
    "path": "namesrv/pom.xml",
    "content": "<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n    <parent>\n        <groupId>org.apache.rocketmq</groupId>\n        <artifactId>rocketmq-all</artifactId>\n        <version>${revision}</version>\n    </parent>\n\n    <modelVersion>4.0.0</modelVersion>\n    <packaging>jar</packaging>\n    <artifactId>rocketmq-namesrv</artifactId>\n    <name>rocketmq-namesrv ${project.version}</name>\n\n    <properties>\n        <project.root>${basedir}/..</project.root>\n    </properties>\n\n    <dependencies>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-controller</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-client</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-tools</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-srvutil</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.openjdk.jmh</groupId>\n            <artifactId>jmh-core</artifactId>\n            <version>1.19</version>\n            <scope>test</scope>\n        </dependency>\n        <dependency>\n            <groupId>org.openjdk.jmh</groupId>\n            <artifactId>jmh-generator-annprocess</artifactId>\n            <version>1.19</version>\n            <scope>test</scope>\n        </dependency>\n        <dependency>\n            <groupId>org.bouncycastle</groupId>\n            <artifactId>bcpkix-jdk18on</artifactId>\n        </dependency>\n    </dependencies>\n</project>\n"
  },
  {
    "path": "namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.namesrv;\n\nimport java.util.Collections;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.lang3.concurrent.BasicThreadFactory;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.future.FutureTaskExt;\nimport org.apache.rocketmq.common.namesrv.NamesrvConfig;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.namesrv.kvconfig.KVConfigManager;\nimport org.apache.rocketmq.namesrv.processor.ClientRequestProcessor;\nimport org.apache.rocketmq.namesrv.processor.ClusterTestRequestProcessor;\nimport org.apache.rocketmq.namesrv.processor.DefaultRequestProcessor;\nimport org.apache.rocketmq.namesrv.route.ZoneRouteRPCHook;\nimport org.apache.rocketmq.namesrv.routeinfo.BrokerHousekeepingService;\nimport org.apache.rocketmq.namesrv.routeinfo.RouteInfoManager;\nimport org.apache.rocketmq.remoting.Configuration;\nimport org.apache.rocketmq.remoting.RemotingClient;\nimport org.apache.rocketmq.remoting.RemotingServer;\nimport org.apache.rocketmq.remoting.common.TlsMode;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingClient;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingServer;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.netty.RequestTask;\nimport org.apache.rocketmq.remoting.netty.TlsSystemConfig;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.srvutil.FileWatchService;\n\npublic class NamesrvController {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME);\n    private static final Logger WATER_MARK_LOG = LoggerFactory.getLogger(LoggerName.NAMESRV_WATER_MARK_LOGGER_NAME);\n\n    private final NamesrvConfig namesrvConfig;\n\n    private final NettyServerConfig nettyServerConfig;\n    private final NettyClientConfig nettyClientConfig;\n\n    private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1,\n            new BasicThreadFactory.Builder().namingPattern(\"NSScheduledThread\").daemon(true).build());\n\n    private final ScheduledExecutorService scanExecutorService = ThreadUtils.newScheduledThreadPool(1,\n            new BasicThreadFactory.Builder().namingPattern(\"NSScanScheduledThread\").daemon(true).build());\n\n    private final KVConfigManager kvConfigManager;\n    private final RouteInfoManager routeInfoManager;\n\n    private RemotingClient remotingClient;\n    private RemotingServer remotingServer;\n\n    private final BrokerHousekeepingService brokerHousekeepingService;\n\n    private ExecutorService defaultExecutor;\n    private ExecutorService clientRequestExecutor;\n\n    private BlockingQueue<Runnable> defaultThreadPoolQueue;\n    private BlockingQueue<Runnable> clientRequestThreadPoolQueue;\n\n    private final Configuration configuration;\n    private FileWatchService fileWatchService;\n\n    public NamesrvController(NamesrvConfig namesrvConfig, NettyServerConfig nettyServerConfig) {\n        this(namesrvConfig, nettyServerConfig, new NettyClientConfig());\n    }\n\n    public NamesrvController(NamesrvConfig namesrvConfig, NettyServerConfig nettyServerConfig, NettyClientConfig nettyClientConfig) {\n        this.namesrvConfig = namesrvConfig;\n        this.nettyServerConfig = nettyServerConfig;\n        this.nettyClientConfig = nettyClientConfig;\n        this.kvConfigManager = new KVConfigManager(this);\n        this.brokerHousekeepingService = new BrokerHousekeepingService(this);\n        this.routeInfoManager = new RouteInfoManager(namesrvConfig, this);\n        this.configuration = new Configuration(LOGGER, this.namesrvConfig, this.nettyServerConfig);\n        this.configuration.setStorePathFromConfig(this.namesrvConfig, \"configStorePath\");\n    }\n\n    public boolean initialize() {\n        loadConfig();\n        initiateNetworkComponents();\n        initiateThreadExecutors();\n        registerProcessor();\n        startScheduleService();\n        initiateSslContext();\n        initiateRpcHooks();\n        return true;\n    }\n\n    private void loadConfig() {\n        this.kvConfigManager.load();\n    }\n\n    private void startScheduleService() {\n        this.scanExecutorService.scheduleAtFixedRate(NamesrvController.this.routeInfoManager::scanNotActiveBroker,\n            5000, this.namesrvConfig.getScanNotActiveBrokerInterval(), TimeUnit.MILLISECONDS);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(NamesrvController.this.kvConfigManager::printAllPeriodically,\n            1, 10, TimeUnit.MINUTES);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(() -> {\n            try {\n                NamesrvController.this.printWaterMark();\n            } catch (Throwable e) {\n                LOGGER.error(\"printWaterMark error.\", e);\n            }\n        }, 10, 1, TimeUnit.SECONDS);\n    }\n\n    private void initiateNetworkComponents() {\n        this.remotingServer = new NettyRemotingServer(this.nettyServerConfig, this.brokerHousekeepingService);\n        this.remotingClient = new NettyRemotingClient(this.nettyClientConfig);\n    }\n\n    private void initiateThreadExecutors() {\n        this.defaultThreadPoolQueue = new LinkedBlockingQueue<>(this.namesrvConfig.getDefaultThreadPoolQueueCapacity());\n        this.defaultExecutor = ThreadUtils.newThreadPoolExecutor(this.namesrvConfig.getDefaultThreadPoolNums(), this.namesrvConfig.getDefaultThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.defaultThreadPoolQueue, new ThreadFactoryImpl(\"RemotingExecutorThread_\"));\n\n        this.clientRequestThreadPoolQueue = new LinkedBlockingQueue<>(this.namesrvConfig.getClientRequestThreadPoolQueueCapacity());\n        this.clientRequestExecutor = ThreadUtils.newThreadPoolExecutor(this.namesrvConfig.getClientRequestThreadPoolNums(), this.namesrvConfig.getClientRequestThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.clientRequestThreadPoolQueue, new ThreadFactoryImpl(\"ClientRequestExecutorThread_\"));\n    }\n\n    private void initiateSslContext() {\n        if (TlsSystemConfig.tlsMode == TlsMode.DISABLED) {\n            return;\n        }\n\n        String[] watchFiles = {TlsSystemConfig.tlsServerCertPath, TlsSystemConfig.tlsServerKeyPath, TlsSystemConfig.tlsServerTrustCertPath};\n\n        FileWatchService.Listener listener = new FileWatchService.Listener() {\n            boolean certChanged, keyChanged = false;\n\n            @Override\n            public void onChanged(String path) {\n                if (path.equals(TlsSystemConfig.tlsServerTrustCertPath)) {\n                    LOGGER.info(\"The trust certificate changed, reload the ssl context\");\n                    ((NettyRemotingServer) remotingServer).loadSslContext();\n                }\n                if (path.equals(TlsSystemConfig.tlsServerCertPath)) {\n                    certChanged = true;\n                }\n                if (path.equals(TlsSystemConfig.tlsServerKeyPath)) {\n                    keyChanged = true;\n                }\n                if (certChanged && keyChanged) {\n                    LOGGER.info(\"The certificate and private key changed, reload the ssl context\");\n                    certChanged = keyChanged = false;\n                    ((NettyRemotingServer) remotingServer).loadSslContext();\n                }\n            }\n        };\n\n        try {\n            fileWatchService = new FileWatchService(watchFiles, listener);\n        } catch (Exception e) {\n            LOGGER.warn(\"FileWatchService created error, can't load the certificate dynamically\");\n        }\n    }\n\n    private void printWaterMark() {\n        WATER_MARK_LOG.info(\"[WATERMARK] ClientQueueSize:{} ClientQueueSlowTime:{} \" + \"DefaultQueueSize:{} DefaultQueueSlowTime:{}\", this.clientRequestThreadPoolQueue.size(), headSlowTimeMills(this.clientRequestThreadPoolQueue), this.defaultThreadPoolQueue.size(), headSlowTimeMills(this.defaultThreadPoolQueue));\n    }\n\n    private long headSlowTimeMills(BlockingQueue<Runnable> q) {\n        long slowTimeMills = 0;\n        final Runnable firstRunnable = q.peek();\n\n        if (firstRunnable instanceof FutureTaskExt) {\n            final Runnable inner = ((FutureTaskExt<?>) firstRunnable).getRunnable();\n            if (inner instanceof RequestTask) {\n                slowTimeMills = System.currentTimeMillis() - ((RequestTask) inner).getCreateTimestamp();\n            }\n        }\n\n        if (slowTimeMills < 0) {\n            slowTimeMills = 0;\n        }\n\n        return slowTimeMills;\n    }\n\n    private void registerProcessor() {\n        if (namesrvConfig.isClusterTest()) {\n\n            this.remotingServer.registerDefaultProcessor(new ClusterTestRequestProcessor(this, namesrvConfig.getProductEnvName()), this.defaultExecutor);\n        } else {\n            // Support get route info only temporarily\n            ClientRequestProcessor clientRequestProcessor = new ClientRequestProcessor(this);\n            this.remotingServer.registerProcessor(RequestCode.GET_ROUTEINFO_BY_TOPIC, clientRequestProcessor, this.clientRequestExecutor);\n\n            this.remotingServer.registerDefaultProcessor(new DefaultRequestProcessor(this), this.defaultExecutor);\n        }\n    }\n\n    private void initiateRpcHooks() {\n        this.remotingServer.registerRPCHook(new ZoneRouteRPCHook());\n    }\n\n    public void start() throws Exception {\n        this.remotingServer.start();\n\n        // In test scenarios where it is up to OS to pick up an available port, set the listening port back to config\n        if (0 == nettyServerConfig.getListenPort()) {\n            nettyServerConfig.setListenPort(this.remotingServer.localListenPort());\n        }\n\n        this.remotingClient.updateNameServerAddressList(Collections.singletonList(NetworkUtil.getLocalAddress()\n            + \":\" + nettyServerConfig.getListenPort()));\n        this.remotingClient.start();\n\n        if (this.fileWatchService != null) {\n            this.fileWatchService.start();\n        }\n\n        this.routeInfoManager.start();\n    }\n\n    public void shutdown() {\n        this.remotingClient.shutdown();\n        this.remotingServer.shutdown();\n        this.defaultExecutor.shutdown();\n        this.clientRequestExecutor.shutdown();\n        this.scheduledExecutorService.shutdown();\n        this.scanExecutorService.shutdown();\n        this.routeInfoManager.shutdown();\n\n        if (this.fileWatchService != null) {\n            this.fileWatchService.shutdown();\n        }\n    }\n\n    public NamesrvConfig getNamesrvConfig() {\n        return namesrvConfig;\n    }\n\n    public NettyServerConfig getNettyServerConfig() {\n        return nettyServerConfig;\n    }\n\n    public KVConfigManager getKvConfigManager() {\n        return kvConfigManager;\n    }\n\n    public RouteInfoManager getRouteInfoManager() {\n        return routeInfoManager;\n    }\n\n    public RemotingServer getRemotingServer() {\n        return remotingServer;\n    }\n\n    public RemotingClient getRemotingClient() {\n        return remotingClient;\n    }\n\n    public void setRemotingServer(RemotingServer remotingServer) {\n        this.remotingServer = remotingServer;\n    }\n\n    public Configuration getConfiguration() {\n        return configuration;\n    }\n}\n"
  },
  {
    "path": "namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.namesrv;\n\nimport java.io.BufferedInputStream;\nimport java.io.InputStream;\nimport java.nio.file.Files;\nimport java.nio.file.Paths;\nimport java.util.Properties;\nimport java.util.concurrent.Callable;\nimport org.apache.commons.cli.CommandLine;\nimport org.apache.commons.cli.DefaultParser;\nimport org.apache.commons.cli.Option;\nimport org.apache.commons.cli.Options;\nimport org.apache.rocketmq.common.ControllerConfig;\nimport org.apache.rocketmq.common.JraftConfig;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.namesrv.NamesrvConfig;\nimport org.apache.rocketmq.controller.ControllerManager;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.srvutil.ServerUtil;\nimport org.apache.rocketmq.srvutil.ShutdownHookThread;\n\npublic class NamesrvStartup {\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME);\n    private static final Logger logConsole = LoggerFactory.getLogger(LoggerName.NAMESRV_CONSOLE_LOGGER_NAME);\n    private static Properties properties = null;\n    private static NamesrvConfig namesrvConfig = null;\n    private static NettyServerConfig nettyServerConfig = null;\n    private static NettyClientConfig nettyClientConfig = null;\n    private static ControllerConfig controllerConfig = null;\n\n    public static void main(String[] args) {\n        main0(args);\n        controllerManagerMain();\n    }\n\n    public static NamesrvController main0(String[] args) {\n        try {\n            parseCommandlineAndConfigFile(args);\n            NamesrvController controller = createAndStartNamesrvController();\n            return controller;\n        } catch (Throwable e) {\n            e.printStackTrace();\n            System.exit(-1);\n        }\n\n        return null;\n    }\n\n    public static ControllerManager controllerManagerMain() {\n        try {\n            if (namesrvConfig.isEnableControllerInNamesrv()) {\n                return createAndStartControllerManager();\n            }\n        } catch (Throwable e) {\n            e.printStackTrace();\n            System.exit(-1);\n        }\n        return null;\n    }\n\n    public static void parseCommandlineAndConfigFile(String[] args) throws Exception {\n        System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION));\n\n        Options options = ServerUtil.buildCommandlineOptions(new Options());\n        CommandLine commandLine = ServerUtil.parseCmdLine(\"mqnamesrv\", args, buildCommandlineOptions(options), new DefaultParser());\n        if (null == commandLine) {\n            System.exit(-1);\n            return;\n        }\n\n        namesrvConfig = new NamesrvConfig();\n        nettyServerConfig = new NettyServerConfig();\n        nettyClientConfig = new NettyClientConfig();\n        nettyServerConfig.setListenPort(9876);\n        if (commandLine.hasOption('c')) {\n            String file = commandLine.getOptionValue('c');\n            if (file != null) {\n                InputStream in = new BufferedInputStream(Files.newInputStream(Paths.get(file)));\n                properties = new Properties();\n                properties.load(in);\n                MixAll.properties2Object(properties, namesrvConfig);\n                MixAll.properties2Object(properties, nettyServerConfig);\n                MixAll.properties2Object(properties, nettyClientConfig);\n                if (namesrvConfig.isEnableControllerInNamesrv()) {\n                    controllerConfig = new ControllerConfig();\n                    JraftConfig jraftConfig = new JraftConfig();\n                    controllerConfig.setJraftConfig(jraftConfig);\n                    MixAll.properties2Object(properties, controllerConfig);\n                    MixAll.properties2Object(properties, jraftConfig);\n                }\n                namesrvConfig.setConfigStorePath(file);\n\n                System.out.printf(\"load config properties file OK, %s%n\", file);\n                in.close();\n            }\n        }\n\n        MixAll.properties2Object(ServerUtil.commandLine2Properties(commandLine), namesrvConfig);\n        if (commandLine.hasOption('p')) {\n            MixAll.printObjectProperties(logConsole, namesrvConfig);\n            MixAll.printObjectProperties(logConsole, nettyServerConfig);\n            MixAll.printObjectProperties(logConsole, nettyClientConfig);\n            if (namesrvConfig.isEnableControllerInNamesrv()) {\n                MixAll.printObjectProperties(logConsole, controllerConfig);\n            }\n            System.exit(0);\n        }\n\n        if (null == namesrvConfig.getRocketmqHome()) {\n            System.out.printf(\"Please set the %s variable in your environment to match the location of the RocketMQ installation%n\", MixAll.ROCKETMQ_HOME_ENV);\n            System.exit(-2);\n        }\n        MixAll.printObjectProperties(log, namesrvConfig);\n        MixAll.printObjectProperties(log, nettyServerConfig);\n\n    }\n\n    public static NamesrvController createAndStartNamesrvController() throws Exception {\n\n        NamesrvController controller = createNamesrvController();\n        start(controller);\n        NettyServerConfig serverConfig = controller.getNettyServerConfig();\n        String tip = String.format(\"The Name Server boot success. serializeType=%s, address %s:%d\", RemotingCommand.getSerializeTypeConfigInThisServer(), serverConfig.getBindAddress(), serverConfig.getListenPort());\n        log.info(tip);\n        System.out.printf(\"%s%n\", tip);\n        return controller;\n    }\n\n    public static NamesrvController createNamesrvController() {\n\n        final NamesrvController controller = new NamesrvController(namesrvConfig, nettyServerConfig, nettyClientConfig);\n        // remember all configs to prevent discard\n        controller.getConfiguration().registerConfig(properties);\n        return controller;\n    }\n\n    public static NamesrvController start(final NamesrvController controller) throws Exception {\n\n        if (null == controller) {\n            throw new IllegalArgumentException(\"NamesrvController is null\");\n        }\n\n        boolean initResult = controller.initialize();\n        if (!initResult) {\n            controller.shutdown();\n            System.exit(-3);\n        }\n\n        Runtime.getRuntime().addShutdownHook(new ShutdownHookThread(log, (Callable<Void>) () -> {\n            controller.shutdown();\n            return null;\n        }));\n\n        controller.start();\n\n        return controller;\n    }\n\n    public static ControllerManager createAndStartControllerManager() throws Exception {\n        ControllerManager controllerManager = createControllerManager();\n        start(controllerManager);\n        String tip = \"The ControllerManager boot success. serializeType=\" + RemotingCommand.getSerializeTypeConfigInThisServer();\n        log.info(tip);\n        System.out.printf(\"%s%n\", tip);\n        return controllerManager;\n    }\n\n    public static ControllerManager createControllerManager() throws Exception {\n        NettyServerConfig controllerNettyServerConfig = (NettyServerConfig) nettyServerConfig.clone();\n        ControllerManager controllerManager = new ControllerManager(controllerConfig, controllerNettyServerConfig, nettyClientConfig);\n        // remember all configs to prevent discard\n        controllerManager.getConfiguration().registerConfig(properties);\n        return controllerManager;\n    }\n\n    public static ControllerManager start(final ControllerManager controllerManager) throws Exception {\n\n        if (null == controllerManager) {\n            throw new IllegalArgumentException(\"ControllerManager is null\");\n        }\n\n        boolean initResult = controllerManager.initialize();\n        if (!initResult) {\n            controllerManager.shutdown();\n            System.exit(-3);\n        }\n\n        Runtime.getRuntime().addShutdownHook(new ShutdownHookThread(log, (Callable<Void>) () -> {\n            controllerManager.shutdown();\n            return null;\n        }));\n\n        controllerManager.start();\n\n        return controllerManager;\n    }\n\n    public static void shutdown(final NamesrvController controller) {\n        controller.shutdown();\n    }\n\n    public static void shutdown(final ControllerManager controllerManager) {\n        controllerManager.shutdown();\n    }\n\n    public static Options buildCommandlineOptions(final Options options) {\n        Option opt = new Option(\"c\", \"configFile\", true, \"Name server config properties file\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"p\", \"printConfigItem\", false, \"Print all config items\");\n        opt.setRequired(false);\n        options.addOption(opt);\n        return options;\n    }\n\n    public static Properties getProperties() {\n        return properties;\n    }\n}\n"
  },
  {
    "path": "namesrv/src/main/java/org/apache/rocketmq/namesrv/kvconfig/KVConfigManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.namesrv.kvconfig;\n\nimport java.io.IOException;\nimport java.util.HashMap;\nimport java.util.Map.Entry;\nimport java.util.concurrent.locks.ReadWriteLock;\nimport java.util.concurrent.locks.ReentrantReadWriteLock;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.namesrv.NamesrvController;\nimport org.apache.rocketmq.remoting.protocol.body.KVTable;\n\npublic class KVConfigManager {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME);\n\n    private final NamesrvController namesrvController;\n\n    private final ReadWriteLock lock = new ReentrantReadWriteLock();\n    private final HashMap<String/* Namespace */, HashMap<String/* Key */, String/* Value */>> configTable =\n        new HashMap<>();\n\n    public KVConfigManager(NamesrvController namesrvController) {\n        this.namesrvController = namesrvController;\n    }\n\n    public void load() {\n        String content = null;\n        try {\n            content = MixAll.file2String(this.namesrvController.getNamesrvConfig().getKvConfigPath());\n        } catch (IOException e) {\n            log.warn(\"Load KV config table exception\", e);\n        }\n        if (content != null) {\n            KVConfigSerializeWrapper kvConfigSerializeWrapper =\n                KVConfigSerializeWrapper.fromJson(content, KVConfigSerializeWrapper.class);\n            if (null != kvConfigSerializeWrapper) {\n                this.configTable.putAll(kvConfigSerializeWrapper.getConfigTable());\n                log.info(\"load KV config table OK\");\n            }\n        }\n    }\n\n    public void putKVConfig(final String namespace, final String key, final String value) {\n        try {\n            this.lock.writeLock().lockInterruptibly();\n            try {\n                HashMap<String, String> kvTable = this.configTable.get(namespace);\n                if (null == kvTable) {\n                    kvTable = new HashMap<>();\n                    this.configTable.put(namespace, kvTable);\n                    log.info(\"putKVConfig create new Namespace {}\", namespace);\n                }\n\n                final String prev = kvTable.put(key, value);\n                if (null != prev) {\n                    log.info(\"putKVConfig update config item, Namespace: {} Key: {} Value: {}\",\n                        namespace, key, value);\n                } else {\n                    log.info(\"putKVConfig create new config item, Namespace: {} Key: {} Value: {}\",\n                        namespace, key, value);\n                }\n            } finally {\n                this.lock.writeLock().unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"putKVConfig InterruptedException\", e);\n        }\n\n        this.persist();\n    }\n\n    public void persist() {\n        try {\n            this.lock.readLock().lockInterruptibly();\n            try {\n                KVConfigSerializeWrapper kvConfigSerializeWrapper = new KVConfigSerializeWrapper();\n                kvConfigSerializeWrapper.setConfigTable(this.configTable);\n\n                String content = kvConfigSerializeWrapper.toJson();\n\n                if (null != content) {\n                    MixAll.string2File(content, this.namesrvController.getNamesrvConfig().getKvConfigPath());\n                }\n            } catch (IOException e) {\n                log.error(\"persist kvconfig Exception, \"\n                    + this.namesrvController.getNamesrvConfig().getKvConfigPath(), e);\n            } finally {\n                this.lock.readLock().unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"persist InterruptedException\", e);\n        }\n\n    }\n\n    public void deleteKVConfig(final String namespace, final String key) {\n        try {\n            this.lock.writeLock().lockInterruptibly();\n            try {\n                HashMap<String, String> kvTable = this.configTable.get(namespace);\n                if (null != kvTable) {\n                    String value = kvTable.remove(key);\n                    log.info(\"deleteKVConfig delete a config item, Namespace: {} Key: {} Value: {}\",\n                        namespace, key, value);\n                }\n            } finally {\n                this.lock.writeLock().unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"deleteKVConfig InterruptedException\", e);\n        }\n\n        this.persist();\n    }\n\n    public byte[] getKVListByNamespace(final String namespace) {\n        try {\n            this.lock.readLock().lockInterruptibly();\n            try {\n                HashMap<String, String> kvTable = this.configTable.get(namespace);\n                if (null != kvTable) {\n                    KVTable table = new KVTable();\n                    table.setTable(kvTable);\n                    return table.encode();\n                }\n            } finally {\n                this.lock.readLock().unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"getKVListByNamespace InterruptedException\", e);\n        }\n\n        return null;\n    }\n\n    public String getKVConfig(final String namespace, final String key) {\n        try {\n            this.lock.readLock().lockInterruptibly();\n            try {\n                HashMap<String, String> kvTable = this.configTable.get(namespace);\n                if (null != kvTable) {\n                    return kvTable.get(key);\n                }\n            } finally {\n                this.lock.readLock().unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"getKVConfig InterruptedException\", e);\n        }\n\n        return null;\n    }\n\n    public void printAllPeriodically() {\n        try {\n            this.lock.readLock().lockInterruptibly();\n            try {\n                log.info(\"--------------------------------------------------------\");\n\n                {\n                    log.info(\"configTable SIZE: {}\", this.configTable.size());\n                    for (Entry<String, HashMap<String, String>> next : this.configTable.entrySet()) {\n                        for (Entry<String, String> nextSub : next.getValue().entrySet()) {\n                            log.info(\"configTable NS: {} Key: {} Value: {}\", next.getKey(), nextSub.getKey(),\n                                    nextSub.getValue());\n                        }\n                    }\n                }\n            } finally {\n                this.lock.readLock().unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"printAllPeriodically InterruptedException\", e);\n        }\n    }\n}\n"
  },
  {
    "path": "namesrv/src/main/java/org/apache/rocketmq/namesrv/kvconfig/KVConfigSerializeWrapper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.namesrv.kvconfig;\n\nimport java.util.HashMap;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class KVConfigSerializeWrapper extends RemotingSerializable {\n    private HashMap<String/* Namespace */, HashMap<String/* Key */, String/* Value */>> configTable;\n\n    public HashMap<String, HashMap<String, String>> getConfigTable() {\n        return configTable;\n    }\n\n    public void setConfigTable(HashMap<String, HashMap<String, String>> configTable) {\n        this.configTable = configTable;\n    }\n}\n"
  },
  {
    "path": "namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClientRequestProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.namesrv.processor;\n\nimport com.alibaba.fastjson2.JSONWriter;\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.help.FAQUrl;\nimport org.apache.rocketmq.common.namesrv.NamesrvUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.namesrv.NamesrvController;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.GetRouteInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\n\nimport java.util.Optional;\nimport java.util.concurrent.TimeUnit;\n\npublic class ClientRequestProcessor implements NettyRequestProcessor {\n\n    private static Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME);\n\n    protected NamesrvController namesrvController;\n    private long startupTimeMillis;\n\n    public ClientRequestProcessor(final NamesrvController namesrvController) {\n        this.namesrvController = namesrvController;\n        this.startupTimeMillis = System.currentTimeMillis();\n    }\n\n    @Override\n    public RemotingCommand processRequest(final ChannelHandlerContext ctx,\n        final RemotingCommand request) throws Exception {\n        return this.getRouteInfoByTopic(ctx, request);\n    }\n\n    public RemotingCommand getRouteInfoByTopic(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final GetRouteInfoRequestHeader requestHeader =\n            (GetRouteInfoRequestHeader) request.decodeCommandCustomHeader(GetRouteInfoRequestHeader.class);\n\n        boolean namesrvReady = System.currentTimeMillis() - startupTimeMillis >= TimeUnit.SECONDS.toMillis(namesrvController.getNamesrvConfig().getWaitSecondsForService());\n\n        if (namesrvController.getNamesrvConfig().isNeedWaitForService() && !namesrvReady) {\n            log.warn(\"name server not ready. request code {} \", request.getCode());\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"name server not ready\");\n            return response;\n        }\n\n        TopicRouteData topicRouteData = this.namesrvController.getRouteInfoManager().pickupTopicRouteData(requestHeader.getTopic());\n\n        if (topicRouteData != null) {\n            if (this.namesrvController.getNamesrvConfig().isOrderMessageEnable()) {\n                String orderTopicConf =\n                    this.namesrvController.getKvConfigManager().getKVConfig(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG,\n                        requestHeader.getTopic());\n                topicRouteData.setOrderTopicConf(orderTopicConf);\n            }\n\n            byte[] content;\n            Boolean standardJsonOnly = Optional.ofNullable(requestHeader.getAcceptStandardJsonOnly()).orElse(false);\n            if (request.getVersion() >= MQVersion.Version.V4_9_4.ordinal() || standardJsonOnly) {\n                content = topicRouteData.encode(JSONWriter.Feature.BrowserCompatible,\n                        JSONWriter.Feature.MapSortField);\n            } else {\n                content = topicRouteData.encode();\n            }\n\n            response.setBody(content);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n            return response;\n        }\n\n        response.setCode(ResponseCode.TOPIC_NOT_EXIST);\n        response.setRemark(\"No topic route info in name server for the topic: \" + requestHeader.getTopic()\n            + FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL));\n        return response;\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n}\n"
  },
  {
    "path": "namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClusterTestRequestProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.namesrv.processor;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.help.FAQUrl;\nimport org.apache.rocketmq.common.namesrv.NamesrvUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.namesrv.NamesrvController;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.GetRouteInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.apache.rocketmq.tools.admin.DefaultMQAdminExt;\n\npublic class ClusterTestRequestProcessor extends ClientRequestProcessor {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME);\n    private final DefaultMQAdminExt adminExt;\n    private final String productEnvName;\n\n    public ClusterTestRequestProcessor(NamesrvController namesrvController, String productEnvName) {\n        super(namesrvController);\n        this.productEnvName = productEnvName;\n        adminExt = new DefaultMQAdminExt();\n        adminExt.setInstanceName(\"CLUSTER_TEST_NS_INS_\" + productEnvName);\n        adminExt.setUnitName(productEnvName);\n        try {\n            adminExt.start();\n        } catch (MQClientException e) {\n            log.error(\"Failed to start processor\", e);\n        }\n    }\n\n    @Override\n    public RemotingCommand getRouteInfoByTopic(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final GetRouteInfoRequestHeader requestHeader =\n            (GetRouteInfoRequestHeader) request.decodeCommandCustomHeader(GetRouteInfoRequestHeader.class);\n\n        TopicRouteData topicRouteData = this.namesrvController.getRouteInfoManager().pickupTopicRouteData(requestHeader.getTopic());\n        if (topicRouteData != null) {\n            String orderTopicConf =\n                this.namesrvController.getKvConfigManager().getKVConfig(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG,\n                    requestHeader.getTopic());\n            topicRouteData.setOrderTopicConf(orderTopicConf);\n        } else {\n            try {\n                topicRouteData = adminExt.examineTopicRouteInfo(requestHeader.getTopic());\n            } catch (Exception e) {\n                log.info(\"get route info by topic from product environment failed. envName={},\", productEnvName);\n            }\n        }\n\n        if (topicRouteData != null) {\n            byte[] content = topicRouteData.encode();\n            response.setBody(content);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n            return response;\n        }\n\n        response.setCode(ResponseCode.TOPIC_NOT_EXIST);\n        response.setRemark(\"No topic route info in name server for the topic: \" + requestHeader.getTopic()\n            + FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL));\n        return response;\n    }\n}\n"
  },
  {
    "path": "namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.namesrv.processor;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport java.io.UnsupportedEncodingException;\nimport java.util.Arrays;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Properties;\nimport java.util.Set;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.MQVersion.Version;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.namesrv.NamesrvUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.namesrv.NamesrvController;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;\nimport org.apache.rocketmq.remoting.protocol.body.GetBrokerMemberGroupResponseBody;\nimport org.apache.rocketmq.remoting.protocol.body.RegisterBrokerBody;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.TopicList;\nimport org.apache.rocketmq.remoting.protocol.header.GetBrokerMemberGroupRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetTopicsByClusterRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.AddWritePermOfBrokerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.AddWritePermOfBrokerResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.BrokerHeartbeatRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.DeleteKVConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.DeleteTopicFromNamesrvRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.GetKVConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.GetKVConfigResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.GetKVListByNamespaceRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.PutKVConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.QueryDataVersionRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.QueryDataVersionResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterTopicRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.UnRegisterBrokerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.WipeWritePermOfBrokerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.WipeWritePermOfBrokerResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.namesrv.RegisterBrokerResult;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\n\npublic class DefaultRequestProcessor implements NettyRequestProcessor {\n    private static Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME);\n\n    protected final NamesrvController namesrvController;\n\n    protected Set<String> configBlackList = new HashSet<>();\n\n    public DefaultRequestProcessor(NamesrvController namesrvController) {\n        this.namesrvController = namesrvController;\n        initConfigBlackList();\n    }\n\n    private void initConfigBlackList() {\n        configBlackList.add(\"configBlackList\");\n        configBlackList.add(\"configStorePath\");\n        configBlackList.add(\"kvConfigPath\");\n        configBlackList.add(\"rocketmqHome\");\n        String[] configArray = namesrvController.getNamesrvConfig().getConfigBlackList().split(\";\");\n        configBlackList.addAll(Arrays.asList(configArray));\n    }\n\n    @Override\n    public RemotingCommand processRequest(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n\n        if (ctx != null) {\n            log.debug(\"receive request, {} {} {}\",\n                request.getCode(),\n                RemotingHelper.parseChannelRemoteAddr(ctx.channel()),\n                request);\n        }\n\n        switch (request.getCode()) {\n            case RequestCode.PUT_KV_CONFIG:\n                return this.putKVConfig(ctx, request);\n            case RequestCode.GET_KV_CONFIG:\n                return this.getKVConfig(ctx, request);\n            case RequestCode.DELETE_KV_CONFIG:\n                return this.deleteKVConfig(ctx, request);\n            case RequestCode.QUERY_DATA_VERSION:\n                return this.queryBrokerTopicConfig(ctx, request);\n            case RequestCode.REGISTER_BROKER:\n                return this.registerBroker(ctx, request);\n            case RequestCode.UNREGISTER_BROKER:\n                return this.unregisterBroker(ctx, request);\n            case RequestCode.BROKER_HEARTBEAT:\n                return this.brokerHeartbeat(ctx, request);\n            case RequestCode.GET_BROKER_MEMBER_GROUP:\n                return this.getBrokerMemberGroup(ctx, request);\n            case RequestCode.GET_BROKER_CLUSTER_INFO:\n                return this.getBrokerClusterInfo(ctx, request);\n            case RequestCode.WIPE_WRITE_PERM_OF_BROKER:\n                return this.wipeWritePermOfBroker(ctx, request);\n            case RequestCode.ADD_WRITE_PERM_OF_BROKER:\n                return this.addWritePermOfBroker(ctx, request);\n            case RequestCode.GET_ALL_TOPIC_LIST_FROM_NAMESERVER:\n                return this.getAllTopicListFromNameserver(ctx, request);\n            case RequestCode.DELETE_TOPIC_IN_NAMESRV:\n                return this.deleteTopicInNamesrv(ctx, request);\n            case RequestCode.REGISTER_TOPIC_IN_NAMESRV:\n                return this.registerTopicToNamesrv(ctx, request);\n            case RequestCode.GET_KVLIST_BY_NAMESPACE:\n                return this.getKVListByNamespace(ctx, request);\n            case RequestCode.GET_TOPICS_BY_CLUSTER:\n                return this.getTopicsByCluster(ctx, request);\n            case RequestCode.GET_SYSTEM_TOPIC_LIST_FROM_NS:\n                return this.getSystemTopicListFromNs(ctx, request);\n            case RequestCode.GET_UNIT_TOPIC_LIST:\n                return this.getUnitTopicList(ctx, request);\n            case RequestCode.GET_HAS_UNIT_SUB_TOPIC_LIST:\n                return this.getHasUnitSubTopicList(ctx, request);\n            case RequestCode.GET_HAS_UNIT_SUB_UNUNIT_TOPIC_LIST:\n                return this.getHasUnitSubUnUnitTopicList(ctx, request);\n            case RequestCode.UPDATE_NAMESRV_CONFIG:\n                return this.updateConfig(ctx, request);\n            case RequestCode.GET_NAMESRV_CONFIG:\n                return this.getConfig(ctx, request);\n            default:\n                String error = \" request type \" + request.getCode() + \" not supported\";\n                return RemotingCommand.createResponseCommand(RemotingSysResponseCode.REQUEST_CODE_NOT_SUPPORTED, error);\n        }\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n\n    public RemotingCommand putKVConfig(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final PutKVConfigRequestHeader requestHeader =\n            (PutKVConfigRequestHeader) request.decodeCommandCustomHeader(PutKVConfigRequestHeader.class);\n\n        if (requestHeader.getNamespace() == null || requestHeader.getKey() == null) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"namespace or key is null\");\n            return response;\n        }\n\n        this.namesrvController.getKvConfigManager().putKVConfig(\n            requestHeader.getNamespace(),\n            requestHeader.getKey(),\n            requestHeader.getValue()\n        );\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    public RemotingCommand getKVConfig(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(GetKVConfigResponseHeader.class);\n        final GetKVConfigResponseHeader responseHeader = (GetKVConfigResponseHeader) response.readCustomHeader();\n        final GetKVConfigRequestHeader requestHeader =\n            (GetKVConfigRequestHeader) request.decodeCommandCustomHeader(GetKVConfigRequestHeader.class);\n\n        String value = this.namesrvController.getKvConfigManager().getKVConfig(\n            requestHeader.getNamespace(),\n            requestHeader.getKey()\n        );\n\n        if (value != null) {\n            responseHeader.setValue(value);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n            return response;\n        }\n\n        response.setCode(ResponseCode.QUERY_NOT_FOUND);\n        response.setRemark(\"No config item, Namespace: \" + requestHeader.getNamespace() + \" Key: \" + requestHeader.getKey());\n        return response;\n    }\n\n    public RemotingCommand deleteKVConfig(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final DeleteKVConfigRequestHeader requestHeader =\n            (DeleteKVConfigRequestHeader) request.decodeCommandCustomHeader(DeleteKVConfigRequestHeader.class);\n\n        this.namesrvController.getKvConfigManager().deleteKVConfig(\n            requestHeader.getNamespace(),\n            requestHeader.getKey()\n        );\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    public RemotingCommand registerBroker(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(RegisterBrokerResponseHeader.class);\n        final RegisterBrokerResponseHeader responseHeader = (RegisterBrokerResponseHeader) response.readCustomHeader();\n        final RegisterBrokerRequestHeader requestHeader =\n            (RegisterBrokerRequestHeader) request.decodeCommandCustomHeader(RegisterBrokerRequestHeader.class);\n\n        if (!checksum(ctx, request, requestHeader)) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"crc32 not match\");\n            return response;\n        }\n\n        TopicConfigSerializeWrapper topicConfigWrapper = null;\n        List<String> filterServerList = null;\n\n        Version brokerVersion = MQVersion.value2Version(request.getVersion());\n        if (brokerVersion.ordinal() >= MQVersion.Version.V3_0_11.ordinal()) {\n            final RegisterBrokerBody registerBrokerBody = extractRegisterBrokerBodyFromRequest(request, requestHeader);\n            topicConfigWrapper = registerBrokerBody.getTopicConfigSerializeWrapper();\n            filterServerList = registerBrokerBody.getFilterServerList();\n        } else {\n            // RegisterBrokerBody of old version only contains TopicConfig.\n            topicConfigWrapper = extractRegisterTopicConfigFromRequest(request);\n        }\n\n        RegisterBrokerResult result = this.namesrvController.getRouteInfoManager().registerBroker(\n            requestHeader.getClusterName(),\n            requestHeader.getBrokerAddr(),\n            requestHeader.getBrokerName(),\n            requestHeader.getBrokerId(),\n            requestHeader.getHaServerAddr(),\n            request.getExtFields().get(MixAll.ZONE_NAME),\n            requestHeader.getHeartbeatTimeoutMillis(),\n            requestHeader.getEnableActingMaster(),\n            topicConfigWrapper,\n            filterServerList,\n            ctx.channel()\n        );\n\n        if (result == null) {\n            // Register single topic route info should be after the broker completes the first registration.\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"register broker failed\");\n            return response;\n        }\n\n        responseHeader.setHaServerAddr(result.getHaServerAddr());\n        responseHeader.setMasterAddr(result.getMasterAddr());\n\n        if (this.namesrvController.getNamesrvConfig().isReturnOrderTopicConfigToBroker()) {\n            byte[] jsonValue = this.namesrvController.getKvConfigManager().getKVListByNamespace(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG);\n            response.setBody(jsonValue);\n        }\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private TopicConfigSerializeWrapper extractRegisterTopicConfigFromRequest(final RemotingCommand request) {\n        TopicConfigSerializeWrapper topicConfigWrapper;\n        if (request.getBody() != null) {\n            topicConfigWrapper = TopicConfigSerializeWrapper.decode(request.getBody(), TopicConfigSerializeWrapper.class);\n        } else {\n            topicConfigWrapper = new TopicConfigSerializeWrapper();\n            topicConfigWrapper.getDataVersion().setCounter(new AtomicLong(0));\n            topicConfigWrapper.getDataVersion().setTimestamp(0L);\n            topicConfigWrapper.getDataVersion().setStateVersion(0L);\n        }\n        return topicConfigWrapper;\n    }\n\n    private RegisterBrokerBody extractRegisterBrokerBodyFromRequest(RemotingCommand request,\n        RegisterBrokerRequestHeader requestHeader) throws RemotingCommandException {\n        RegisterBrokerBody registerBrokerBody = new RegisterBrokerBody();\n\n        if (request.getBody() != null) {\n            try {\n                Version brokerVersion = MQVersion.value2Version(request.getVersion());\n                registerBrokerBody = RegisterBrokerBody.decode(request.getBody(), requestHeader.isCompressed(), brokerVersion);\n            } catch (Exception e) {\n                throw new RemotingCommandException(\"Failed to decode RegisterBrokerBody\", e);\n            }\n        } else {\n            registerBrokerBody.getTopicConfigSerializeWrapper().getDataVersion().setCounter(new AtomicLong(0));\n            registerBrokerBody.getTopicConfigSerializeWrapper().getDataVersion().setTimestamp(0L);\n            registerBrokerBody.getTopicConfigSerializeWrapper().getDataVersion().setStateVersion(0L);\n        }\n        return registerBrokerBody;\n    }\n\n    private RemotingCommand getBrokerMemberGroup(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        GetBrokerMemberGroupRequestHeader requestHeader = (GetBrokerMemberGroupRequestHeader) request.decodeCommandCustomHeader(GetBrokerMemberGroupRequestHeader.class);\n\n        BrokerMemberGroup memberGroup = this.namesrvController.getRouteInfoManager()\n            .getBrokerMemberGroup(requestHeader.getClusterName(), requestHeader.getBrokerName());\n\n        GetBrokerMemberGroupResponseBody responseBody = new GetBrokerMemberGroupResponseBody();\n        responseBody.setBrokerMemberGroup(memberGroup);\n\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setBody(responseBody.encode());\n        return response;\n    }\n\n    private boolean checksum(ChannelHandlerContext ctx, RemotingCommand request,\n        RegisterBrokerRequestHeader requestHeader) {\n        if (requestHeader.getBodyCrc32() != 0) {\n            final int crc32 = UtilAll.crc32(request.getBody());\n            if (crc32 != requestHeader.getBodyCrc32()) {\n                log.warn(String.format(\"receive registerBroker request,crc32 not match,from %s\",\n                    RemotingHelper.parseChannelRemoteAddr(ctx.channel())));\n                return false;\n            }\n        }\n        return true;\n    }\n\n    public RemotingCommand queryBrokerTopicConfig(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(QueryDataVersionResponseHeader.class);\n        final QueryDataVersionResponseHeader responseHeader = (QueryDataVersionResponseHeader) response.readCustomHeader();\n        final QueryDataVersionRequestHeader requestHeader =\n            (QueryDataVersionRequestHeader) request.decodeCommandCustomHeader(QueryDataVersionRequestHeader.class);\n        DataVersion dataVersion = DataVersion.decode(request.getBody(), DataVersion.class);\n        String clusterName = requestHeader.getClusterName();\n        String brokerAddr = requestHeader.getBrokerAddr();\n\n        Boolean changed = this.namesrvController.getRouteInfoManager().isBrokerTopicConfigChanged(clusterName, brokerAddr, dataVersion);\n        this.namesrvController.getRouteInfoManager().updateBrokerInfoUpdateTimestamp(clusterName, brokerAddr);\n\n        DataVersion nameSeverDataVersion = this.namesrvController.getRouteInfoManager().queryBrokerTopicConfig(clusterName, brokerAddr);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n\n        if (nameSeverDataVersion != null) {\n            response.setBody(nameSeverDataVersion.encode());\n        }\n        responseHeader.setChanged(changed);\n        return response;\n    }\n\n    public RemotingCommand unregisterBroker(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final UnRegisterBrokerRequestHeader requestHeader = (UnRegisterBrokerRequestHeader) request.decodeCommandCustomHeader(UnRegisterBrokerRequestHeader.class);\n\n        if (!this.namesrvController.getRouteInfoManager().submitUnRegisterBrokerRequest(requestHeader)) {\n            log.warn(\"Couldn't submit the unregister broker request to handler, broker info: {}\", requestHeader);\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(null);\n            return response;\n        }\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    public RemotingCommand brokerHeartbeat(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final BrokerHeartbeatRequestHeader requestHeader =\n            (BrokerHeartbeatRequestHeader) request.decodeCommandCustomHeader(BrokerHeartbeatRequestHeader.class);\n\n        this.namesrvController.getRouteInfoManager().updateBrokerInfoUpdateTimestamp(requestHeader.getClusterName(), requestHeader.getBrokerAddr());\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand getBrokerClusterInfo(ChannelHandlerContext ctx, RemotingCommand request) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        byte[] content = this.namesrvController.getRouteInfoManager().getAllClusterInfo().encode();\n        response.setBody(content);\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand wipeWritePermOfBroker(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(WipeWritePermOfBrokerResponseHeader.class);\n        final WipeWritePermOfBrokerResponseHeader responseHeader = (WipeWritePermOfBrokerResponseHeader) response.readCustomHeader();\n        final WipeWritePermOfBrokerRequestHeader requestHeader =\n            (WipeWritePermOfBrokerRequestHeader) request.decodeCommandCustomHeader(WipeWritePermOfBrokerRequestHeader.class);\n\n        int wipeTopicCnt = this.namesrvController.getRouteInfoManager().wipeWritePermOfBrokerByLock(requestHeader.getBrokerName());\n\n        if (ctx != null) {\n            log.info(\"wipe write perm of broker[{}], client: {}, {}\",\n                requestHeader.getBrokerName(),\n                RemotingHelper.parseChannelRemoteAddr(ctx.channel()),\n                wipeTopicCnt);\n        }\n\n        responseHeader.setWipeTopicCount(wipeTopicCnt);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand addWritePermOfBroker(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(AddWritePermOfBrokerResponseHeader.class);\n        final AddWritePermOfBrokerResponseHeader responseHeader = (AddWritePermOfBrokerResponseHeader) response.readCustomHeader();\n        final AddWritePermOfBrokerRequestHeader requestHeader = (AddWritePermOfBrokerRequestHeader) request.decodeCommandCustomHeader(AddWritePermOfBrokerRequestHeader.class);\n\n        int addTopicCnt = this.namesrvController.getRouteInfoManager().addWritePermOfBrokerByLock(requestHeader.getBrokerName());\n\n        log.info(\"add write perm of broker[{}], client: {}, {}\",\n            requestHeader.getBrokerName(),\n            RemotingHelper.parseChannelRemoteAddr(ctx.channel()),\n            addTopicCnt);\n\n        responseHeader.setAddTopicCount(addTopicCnt);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand getAllTopicListFromNameserver(ChannelHandlerContext ctx, RemotingCommand request) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        boolean enableAllTopicList = namesrvController.getNamesrvConfig().isEnableAllTopicList();\n        log.warn(\"getAllTopicListFromNameserver {} enable {}\", ctx.channel().remoteAddress(), enableAllTopicList);\n        if (enableAllTopicList) {\n            byte[] body = this.namesrvController.getRouteInfoManager().getAllTopicList().encode();\n            response.setBody(body);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n        } else {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"disable\");\n        }\n\n        return response;\n    }\n\n    private RemotingCommand registerTopicToNamesrv(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        final RegisterTopicRequestHeader requestHeader =\n            (RegisterTopicRequestHeader) request.decodeCommandCustomHeader(RegisterTopicRequestHeader.class);\n\n        TopicRouteData topicRouteData = TopicRouteData.decode(request.getBody(), TopicRouteData.class);\n        if (topicRouteData != null && topicRouteData.getQueueDatas() != null && !topicRouteData.getQueueDatas().isEmpty()) {\n            this.namesrvController.getRouteInfoManager().registerTopic(requestHeader.getTopic(), topicRouteData.getQueueDatas());\n        }\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand deleteTopicInNamesrv(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final DeleteTopicFromNamesrvRequestHeader requestHeader =\n            (DeleteTopicFromNamesrvRequestHeader) request.decodeCommandCustomHeader(DeleteTopicFromNamesrvRequestHeader.class);\n\n        if (requestHeader.getClusterName() != null\n            && !requestHeader.getClusterName().isEmpty()) {\n            this.namesrvController.getRouteInfoManager().deleteTopic(requestHeader.getTopic(), requestHeader.getClusterName());\n        } else {\n            this.namesrvController.getRouteInfoManager().deleteTopic(requestHeader.getTopic());\n        }\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand getKVListByNamespace(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final GetKVListByNamespaceRequestHeader requestHeader =\n            (GetKVListByNamespaceRequestHeader) request.decodeCommandCustomHeader(GetKVListByNamespaceRequestHeader.class);\n\n        byte[] jsonValue = this.namesrvController.getKvConfigManager().getKVListByNamespace(\n            requestHeader.getNamespace());\n        if (null != jsonValue) {\n            response.setBody(jsonValue);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n            return response;\n        }\n\n        response.setCode(ResponseCode.QUERY_NOT_FOUND);\n        response.setRemark(\"No config item, Namespace: \" + requestHeader.getNamespace());\n        return response;\n    }\n\n    private RemotingCommand getTopicsByCluster(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        boolean enableTopicList = namesrvController.getNamesrvConfig().isEnableTopicList();\n        if (!enableTopicList) {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"disable\");\n            return response;\n        }\n        final GetTopicsByClusterRequestHeader requestHeader =\n            (GetTopicsByClusterRequestHeader) request.decodeCommandCustomHeader(GetTopicsByClusterRequestHeader.class);\n\n        TopicList topicsByCluster = this.namesrvController.getRouteInfoManager().getTopicsByCluster(requestHeader.getCluster());\n        byte[] body = topicsByCluster.encode();\n\n        response.setBody(body);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n\n        return response;\n    }\n\n    private RemotingCommand getSystemTopicListFromNs(ChannelHandlerContext ctx,\n        RemotingCommand request) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        TopicList systemTopicList = this.namesrvController.getRouteInfoManager().getSystemTopicList();\n        byte[] body = systemTopicList.encode();\n\n        response.setBody(body);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand getUnitTopicList(ChannelHandlerContext ctx,\n        RemotingCommand request) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        boolean enableTopicList = namesrvController.getNamesrvConfig().isEnableTopicList();\n\n        if (enableTopicList) {\n            TopicList unitTopicList = this.namesrvController.getRouteInfoManager().getUnitTopics();\n            byte[] body = unitTopicList.encode();\n            response.setBody(body);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n        } else {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"disable\");\n        }\n\n        return response;\n    }\n\n    private RemotingCommand getHasUnitSubTopicList(ChannelHandlerContext ctx,\n        RemotingCommand request) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        boolean enableTopicList = namesrvController.getNamesrvConfig().isEnableTopicList();\n\n        if (enableTopicList) {\n            TopicList hasUnitSubTopicList = this.namesrvController.getRouteInfoManager().getHasUnitSubTopicList();\n            byte[] body = hasUnitSubTopicList.encode();\n            response.setBody(body);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n        } else {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"disable\");\n        }\n\n        return response;\n    }\n\n    private RemotingCommand getHasUnitSubUnUnitTopicList(ChannelHandlerContext ctx, RemotingCommand request) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        boolean enableTopicList = namesrvController.getNamesrvConfig().isEnableTopicList();\n\n        if (enableTopicList) {\n            TopicList hasUnitSubUnUnitTopicList = this.namesrvController.getRouteInfoManager().getHasUnitSubUnUnitTopicList();\n            byte[] body = hasUnitSubUnUnitTopicList.encode();\n            response.setBody(body);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n        } else {\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.setRemark(\"disable\");\n        }\n\n        return response;\n    }\n\n    private RemotingCommand updateConfig(ChannelHandlerContext ctx, RemotingCommand request) {\n        if (ctx != null) {\n            log.info(\"updateConfig called by {}\", RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n        }\n\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        byte[] body = request.getBody();\n        if (body != null) {\n            String bodyStr;\n            try {\n                bodyStr = new String(body, MixAll.DEFAULT_CHARSET);\n            } catch (UnsupportedEncodingException e) {\n                log.error(\"updateConfig byte array to string error: \", e);\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"UnsupportedEncodingException \" + e);\n                return response;\n            }\n\n            Properties properties = MixAll.string2Properties(bodyStr);\n            if (properties == null) {\n                log.error(\"updateConfig MixAll.string2Properties error {}\", bodyStr);\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"string2Properties error\");\n                return response;\n            }\n            if (validateBlackListConfigExist(properties)) {\n                response.setCode(ResponseCode.NO_PERMISSION);\n                response.setRemark(\"Can not update config in black list.\");\n                return response;\n            }\n\n            this.namesrvController.getConfiguration().update(properties);\n        }\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private RemotingCommand getConfig(ChannelHandlerContext ctx, RemotingCommand request) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n\n        String content = this.namesrvController.getConfiguration().getAllConfigsFormatString();\n        if (StringUtils.isNotBlank(content)) {\n            try {\n                content = MixAll.adjustConfigForPlatform(content);\n                response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET));\n            } catch (UnsupportedEncodingException e) {\n                log.error(\"getConfig error, \", e);\n                response.setCode(ResponseCode.SYSTEM_ERROR);\n                response.setRemark(\"UnsupportedEncodingException \" + e);\n                return response;\n            }\n        }\n\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n\n    private boolean validateBlackListConfigExist(Properties properties) {\n        for (String blackConfig : configBlackList) {\n            if (properties.containsKey(blackConfig)) {\n                return true;\n            }\n        }\n        return false;\n    }\n\n}\n"
  },
  {
    "path": "namesrv/src/main/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHook.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.namesrv.route;\n\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Map.Entry;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\n\npublic class ZoneRouteRPCHook implements RPCHook {\n\n    @Override\n    public void doBeforeRequest(String remoteAddr, RemotingCommand request) {\n\n    }\n\n    @Override\n    public void doAfterResponse(String remoteAddr, RemotingCommand request, RemotingCommand response) {\n        if (RequestCode.GET_ROUTEINFO_BY_TOPIC != request.getCode()) {\n            return;\n        }\n        if (response == null || response.getBody() == null || ResponseCode.SUCCESS != response.getCode()) {\n            return;\n        }\n        boolean zoneMode = Boolean.parseBoolean(request.getExtFields().get(MixAll.ZONE_MODE));\n        if (!zoneMode) {\n            return;\n        }\n        String zoneName = request.getExtFields().get(MixAll.ZONE_NAME);\n        if (StringUtils.isBlank(zoneName)) {\n            return;\n        }\n        TopicRouteData topicRouteData = RemotingSerializable.decode(response.getBody(), TopicRouteData.class);\n        response.setBody(filterByZoneName(topicRouteData, zoneName).encode());\n    }\n\n    private TopicRouteData filterByZoneName(TopicRouteData topicRouteData, String zoneName) {\n        List<BrokerData> brokerDataReserved = new ArrayList<>();\n        Map<String, BrokerData> brokerDataRemoved = new HashMap<>();\n        for (BrokerData brokerData : topicRouteData.getBrokerDatas()) {\n            if (brokerData.getBrokerAddrs() == null) {\n                continue;\n            }\n            //master down, consume from slave. break nearby route rule.\n            if (brokerData.getBrokerAddrs().get(MixAll.MASTER_ID) == null\n                || StringUtils.equalsIgnoreCase(brokerData.getZoneName(), zoneName)) {\n                brokerDataReserved.add(brokerData);\n            } else {\n                brokerDataRemoved.put(brokerData.getBrokerName(), brokerData);\n            }\n        }\n        topicRouteData.setBrokerDatas(brokerDataReserved);\n\n        List<QueueData> queueDataReserved = new ArrayList<>();\n        for (QueueData queueData : topicRouteData.getQueueDatas()) {\n            if (!brokerDataRemoved.containsKey(queueData.getBrokerName())) {\n                queueDataReserved.add(queueData);\n            }\n        }\n        topicRouteData.setQueueDatas(queueDataReserved);\n        // remove filter server table by broker address\n        if (topicRouteData.getFilterServerTable() != null && !topicRouteData.getFilterServerTable().isEmpty()) {\n            for (Entry<String, BrokerData> entry : brokerDataRemoved.entrySet()) {\n                BrokerData brokerData = entry.getValue();\n                brokerData.getBrokerAddrs().values()\n                    .forEach(brokerAddr -> topicRouteData.getFilterServerTable().remove(brokerAddr));\n            }\n        }\n        return topicRouteData;\n    }\n}\n"
  },
  {
    "path": "namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/BatchUnregistrationService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.namesrv.routeinfo;\n\nimport java.util.HashSet;\nimport java.util.Set;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.namesrv.NamesrvConfig;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.UnRegisterBrokerRequestHeader;\n\n/**\n * BatchUnregistrationService provides a mechanism to unregister brokers in batch manner, which speeds up broker-offline\n * process.\n */\npublic class BatchUnregistrationService extends ServiceThread {\n    private final RouteInfoManager routeInfoManager;\n    private BlockingQueue<UnRegisterBrokerRequestHeader> unregistrationQueue;\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME);\n\n    public BatchUnregistrationService(RouteInfoManager routeInfoManager, NamesrvConfig namesrvConfig) {\n        this.routeInfoManager = routeInfoManager;\n        this.unregistrationQueue = new LinkedBlockingQueue<>(namesrvConfig.getUnRegisterBrokerQueueCapacity());\n    }\n\n    /**\n     * Submits an unregister request to this queue.\n     *\n     * @param unRegisterRequest the request to submit\n     * @return {@code true} if the request was added to this queue, else {@code false}\n     */\n    public boolean submit(UnRegisterBrokerRequestHeader unRegisterRequest) {\n        return unregistrationQueue.offer(unRegisterRequest);\n    }\n\n    @Override\n    public String getServiceName() {\n        return BatchUnregistrationService.class.getName();\n    }\n\n    @Override\n    public void run() {\n        while (!this.isStopped()) {\n            try {\n                final UnRegisterBrokerRequestHeader request = unregistrationQueue.take();\n                Set<UnRegisterBrokerRequestHeader> unregistrationRequests = new HashSet<>();\n                unregistrationQueue.drainTo(unregistrationRequests);\n\n                // Add polled request\n                unregistrationRequests.add(request);\n\n                this.routeInfoManager.unRegisterBroker(unregistrationRequests);\n            } catch (Throwable e) {\n                log.error(\"Handle unregister broker request failed\", e);\n            }\n        }\n    }\n\n    // For test only\n    int queueLength() {\n        return this.unregistrationQueue.size();\n    }\n}\n"
  },
  {
    "path": "namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/BrokerHousekeepingService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.namesrv.routeinfo;\n\nimport io.netty.channel.Channel;\nimport org.apache.rocketmq.namesrv.NamesrvController;\nimport org.apache.rocketmq.remoting.ChannelEventListener;\n\npublic class BrokerHousekeepingService implements ChannelEventListener {\n\n    private final NamesrvController namesrvController;\n\n    public BrokerHousekeepingService(NamesrvController namesrvController) {\n        this.namesrvController = namesrvController;\n    }\n\n    @Override\n    public void onChannelConnect(String remoteAddr, Channel channel) {\n    }\n\n    @Override\n    public void onChannelClose(String remoteAddr, Channel channel) {\n        this.namesrvController.getRouteInfoManager().onChannelDestroy(channel);\n    }\n\n    @Override\n    public void onChannelException(String remoteAddr, Channel channel) {\n        this.namesrvController.getRouteInfoManager().onChannelDestroy(channel);\n    }\n\n    @Override\n    public void onChannelIdle(String remoteAddr, Channel channel) {\n        this.namesrvController.getRouteInfoManager().onChannelDestroy(channel);\n    }\n\n    @Override\n    public void onChannelActive(String remoteAddr, Channel channel) {\n\n    }\n}\n"
  },
  {
    "path": "namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.namesrv.routeinfo;\n\nimport com.google.common.collect.Sets;\nimport io.netty.channel.Channel;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Map.Entry;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.locks.ReadWriteLock;\nimport java.util.concurrent.locks.ReentrantReadWriteLock;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.namesrv.NamesrvConfig;\nimport org.apache.rocketmq.common.sysflag.TopicSysFlag;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.common.utils.ConcurrentHashMapUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.namesrv.NamesrvController;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingConnectException;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;\nimport org.apache.rocketmq.remoting.protocol.body.ClusterInfo;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.TopicList;\nimport org.apache.rocketmq.remoting.protocol.header.NotifyMinBrokerIdChangeRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.UnRegisterBrokerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.namesrv.RegisterBrokerResult;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo;\n\npublic class RouteInfoManager {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME);\n    private static final long DEFAULT_BROKER_CHANNEL_EXPIRED_TIME = 1000 * 60 * 2;\n    private final ReadWriteLock lock = new ReentrantReadWriteLock();\n    private final Map<String/* topic */, Map<String, QueueData>> topicQueueTable;\n    private final Map<String/* brokerName */, BrokerData> brokerAddrTable;\n    private final Map<String/* clusterName */, Set<String/* brokerName */>> clusterAddrTable;\n    private final Map<BrokerAddrInfo/* brokerAddr */, BrokerLiveInfo> brokerLiveTable;\n    private final Map<BrokerAddrInfo/* brokerAddr */, List<String>/* Filter Server */> filterServerTable;\n    private final Map<String/* topic */, Map<String/*brokerName*/, TopicQueueMappingInfo>> topicQueueMappingInfoTable;\n\n    private final BatchUnregistrationService unRegisterService;\n\n    private final NamesrvController namesrvController;\n    private final NamesrvConfig namesrvConfig;\n\n    public RouteInfoManager(final NamesrvConfig namesrvConfig, NamesrvController namesrvController) {\n        this.topicQueueTable = new ConcurrentHashMap<>(1024);\n        this.brokerAddrTable = new ConcurrentHashMap<>(128);\n        this.clusterAddrTable = new ConcurrentHashMap<>(32);\n        this.brokerLiveTable = new ConcurrentHashMap<>(256);\n        this.filterServerTable = new ConcurrentHashMap<>(256);\n        this.topicQueueMappingInfoTable = new ConcurrentHashMap<>(1024);\n        this.unRegisterService = new BatchUnregistrationService(this, namesrvConfig);\n        this.namesrvConfig = namesrvConfig;\n        this.namesrvController = namesrvController;\n    }\n\n    public void start() {\n        this.unRegisterService.start();\n    }\n\n    public void shutdown() {\n        this.unRegisterService.shutdown(true);\n    }\n\n    public boolean submitUnRegisterBrokerRequest(UnRegisterBrokerRequestHeader unRegisterRequest) {\n        return this.unRegisterService.submit(unRegisterRequest);\n    }\n\n    // For test only\n    int blockedUnRegisterRequests() {\n        return this.unRegisterService.queueLength();\n    }\n\n    public ClusterInfo getAllClusterInfo() {\n        ClusterInfo clusterInfoSerializeWrapper = new ClusterInfo();\n        clusterInfoSerializeWrapper.setBrokerAddrTable(this.brokerAddrTable);\n        clusterInfoSerializeWrapper.setClusterAddrTable(this.clusterAddrTable);\n        return clusterInfoSerializeWrapper;\n    }\n\n    public void registerTopic(final String topic, List<QueueData> queueDatas) {\n        if (queueDatas == null || queueDatas.isEmpty()) {\n            return;\n        }\n\n        try {\n            this.lock.writeLock().lockInterruptibly();\n            if (this.topicQueueTable.containsKey(topic)) {\n                Map<String, QueueData> queueDataMap  = this.topicQueueTable.get(topic);\n                for (QueueData queueData : queueDatas) {\n                    if (!this.brokerAddrTable.containsKey(queueData.getBrokerName())) {\n                        log.warn(\"Register topic contains illegal broker, {}, {}\", topic, queueData);\n                        return;\n                    }\n                    queueDataMap.put(queueData.getBrokerName(), queueData);\n                }\n                log.info(\"Topic route already exist.{}, {}\", topic, this.topicQueueTable.get(topic));\n            } else {\n                // check and construct queue data map\n                Map<String, QueueData> queueDataMap = new HashMap<>();\n                for (QueueData queueData : queueDatas) {\n                    if (!this.brokerAddrTable.containsKey(queueData.getBrokerName())) {\n                        log.warn(\"Register topic contains illegal broker, {}, {}\", topic, queueData);\n                        return;\n                    }\n                    queueDataMap.put(queueData.getBrokerName(), queueData);\n                }\n\n                this.topicQueueTable.put(topic, queueDataMap);\n                log.info(\"Register topic route:{}, {}\", topic, queueDatas);\n            }\n        } catch (Exception e) {\n            log.error(\"registerTopic Exception\", e);\n        } finally {\n            this.lock.writeLock().unlock();\n        }\n    }\n\n    public void deleteTopic(final String topic) {\n        try {\n            this.lock.writeLock().lockInterruptibly();\n            this.topicQueueTable.remove(topic);\n        } catch (Exception e) {\n            log.error(\"deleteTopic Exception\", e);\n        } finally {\n            this.lock.writeLock().unlock();\n        }\n    }\n\n    public void deleteTopic(final String topic, final String clusterName) {\n        try {\n            this.lock.writeLock().lockInterruptibly();\n            //get all the brokerNames fot the specified cluster\n            Set<String> brokerNames = this.clusterAddrTable.get(clusterName);\n            if (brokerNames == null || brokerNames.isEmpty()) {\n                return;\n            }\n            //get the store information for single topic\n            Map<String, QueueData> queueDataMap = this.topicQueueTable.get(topic);\n            if (queueDataMap != null) {\n                for (String brokerName : brokerNames) {\n                    final QueueData removedQD = queueDataMap.remove(brokerName);\n                    if (removedQD != null) {\n                        log.info(\"deleteTopic, remove one broker's topic {} {} {}\", brokerName, topic, removedQD);\n                    }\n                }\n                if (queueDataMap.isEmpty()) {\n                    log.info(\"deleteTopic, remove the topic all queue {} {}\", clusterName, topic);\n                    this.topicQueueTable.remove(topic);\n                }\n            }\n        } catch (Exception e) {\n            log.error(\"deleteTopic Exception\", e);\n        } finally {\n            this.lock.writeLock().unlock();\n        }\n    }\n\n    public TopicList getAllTopicList() {\n        TopicList topicList = new TopicList();\n        try {\n            this.lock.readLock().lockInterruptibly();\n            topicList.getTopicList().addAll(this.topicQueueTable.keySet());\n        } catch (Exception e) {\n            log.error(\"getAllTopicList Exception\", e);\n        } finally {\n            this.lock.readLock().unlock();\n        }\n\n        return topicList;\n    }\n\n    public RegisterBrokerResult registerBroker(\n        final String clusterName,\n        final String brokerAddr,\n        final String brokerName,\n        final long brokerId,\n        final String haServerAddr,\n        final String zoneName,\n        final Long timeoutMillis,\n        final TopicConfigSerializeWrapper topicConfigWrapper,\n        final List<String> filterServerList,\n        final Channel channel) {\n        return registerBroker(clusterName, brokerAddr, brokerName, brokerId, haServerAddr, zoneName, timeoutMillis, false, topicConfigWrapper, filterServerList, channel);\n    }\n\n    public RegisterBrokerResult registerBroker(\n        final String clusterName,\n        final String brokerAddr,\n        final String brokerName,\n        final long brokerId,\n        final String haServerAddr,\n        final String zoneName,\n        final Long timeoutMillis,\n        final Boolean enableActingMaster,\n        final TopicConfigSerializeWrapper topicConfigWrapper,\n        final List<String> filterServerList,\n        final Channel channel) {\n        RegisterBrokerResult result = new RegisterBrokerResult();\n        try {\n            this.lock.writeLock().lockInterruptibly();\n\n            //init or update the cluster info\n            Set<String> brokerNames = ConcurrentHashMapUtils.computeIfAbsent((ConcurrentHashMap<String, Set<String>>) this.clusterAddrTable, clusterName, k -> new HashSet<>());\n            brokerNames.add(brokerName);\n\n            boolean registerFirst = false;\n\n            BrokerData brokerData = this.brokerAddrTable.get(brokerName);\n            if (null == brokerData) {\n                registerFirst = true;\n                brokerData = new BrokerData(clusterName, brokerName, new HashMap<>());\n                this.brokerAddrTable.put(brokerName, brokerData);\n            }\n\n            boolean isOldVersionBroker = enableActingMaster == null;\n            brokerData.setEnableActingMaster(!isOldVersionBroker && enableActingMaster);\n            brokerData.setZoneName(zoneName);\n\n            Map<Long, String> brokerAddrsMap = brokerData.getBrokerAddrs();\n\n            boolean isMinBrokerIdChanged = false;\n            long prevMinBrokerId = 0;\n            if (!brokerAddrsMap.isEmpty()) {\n                prevMinBrokerId = Collections.min(brokerAddrsMap.keySet());\n            }\n\n            if (brokerId < prevMinBrokerId) {\n                isMinBrokerIdChanged = true;\n            }\n\n            //Switch slave to master: first remove <1, IP:PORT> in namesrv, then add <0, IP:PORT>\n            //The same IP:PORT must only have one record in brokerAddrTable\n            brokerAddrsMap.entrySet().removeIf(item -> null != brokerAddr && brokerAddr.equals(item.getValue()) && brokerId != item.getKey());\n\n            //If Local brokerId stateVersion bigger than the registering one,\n            String oldBrokerAddr = brokerAddrsMap.get(brokerId);\n            if (null != oldBrokerAddr && !oldBrokerAddr.equals(brokerAddr)) {\n                BrokerLiveInfo oldBrokerInfo = brokerLiveTable.get(new BrokerAddrInfo(clusterName, oldBrokerAddr));\n\n                if (null != oldBrokerInfo) {\n                    long oldStateVersion = oldBrokerInfo.getDataVersion().getStateVersion();\n                    long newStateVersion = topicConfigWrapper.getDataVersion().getStateVersion();\n                    if (oldStateVersion > newStateVersion) {\n                        log.warn(\"Registering Broker conflicts with the existed one, just ignore.: Cluster:{}, BrokerName:{}, BrokerId:{}, \" +\n                                \"Old BrokerAddr:{}, Old Version:{}, New BrokerAddr:{}, New Version:{}.\",\n                            clusterName, brokerName, brokerId, oldBrokerAddr, oldStateVersion, brokerAddr, newStateVersion);\n                        //Remove the rejected brokerAddr from brokerLiveTable.\n                        brokerLiveTable.remove(new BrokerAddrInfo(clusterName, brokerAddr));\n                        return result;\n                    }\n                }\n            }\n\n            if (!brokerAddrsMap.containsKey(brokerId) && topicConfigWrapper.getTopicConfigTable().size() == 1) {\n                log.warn(\"Can't register topicConfigWrapper={} because broker[{}]={} has not registered.\",\n                    topicConfigWrapper.getTopicConfigTable(), brokerId, brokerAddr);\n                return null;\n            }\n\n            String oldAddr = brokerAddrsMap.put(brokerId, brokerAddr);\n            registerFirst = registerFirst || (StringUtils.isEmpty(oldAddr));\n\n            boolean isMaster = MixAll.MASTER_ID == brokerId;\n\n            boolean isPrimeSlave = !isOldVersionBroker && !isMaster\n                && brokerId == Collections.min(brokerAddrsMap.keySet());\n\n            if (null != topicConfigWrapper && (isMaster || isPrimeSlave)) {\n\n                ConcurrentMap<String, TopicConfig> tcTable =\n                    topicConfigWrapper.getTopicConfigTable();\n\n                if (tcTable != null) {\n\n                    TopicConfigAndMappingSerializeWrapper mappingSerializeWrapper = TopicConfigAndMappingSerializeWrapper.from(topicConfigWrapper);\n                    Map<String, TopicQueueMappingInfo> topicQueueMappingInfoMap = mappingSerializeWrapper.getTopicQueueMappingInfoMap();\n\n                    // Delete the topics that don't exist in tcTable from the current broker\n                    // Static topic is not supported currently\n                    if (namesrvConfig.isDeleteTopicWithBrokerRegistration() && topicQueueMappingInfoMap.isEmpty()) {\n                        final Set<String> oldTopicSet = topicSetOfBrokerName(brokerName);\n                        final Set<String> newTopicSet = tcTable.keySet();\n                        final Sets.SetView<String> toDeleteTopics = Sets.difference(oldTopicSet, newTopicSet);\n                        for (final String toDeleteTopic : toDeleteTopics) {\n                            Map<String, QueueData> queueDataMap = topicQueueTable.get(toDeleteTopic);\n                            final QueueData removedQD = queueDataMap.remove(brokerName);\n                            if (removedQD != null) {\n                                log.info(\"deleteTopic, remove one broker's topic {} {} {}\", brokerName, toDeleteTopic, removedQD);\n                            }\n\n                            if (queueDataMap.isEmpty()) {\n                                log.info(\"deleteTopic, remove the topic all queue {}\", toDeleteTopic);\n                                topicQueueTable.remove(toDeleteTopic);\n                            }\n                        }\n                    }\n\n                    for (Map.Entry<String, TopicConfig> entry : tcTable.entrySet()) {\n                        if (registerFirst || this.isTopicConfigChanged(clusterName, brokerAddr,\n                            topicConfigWrapper.getDataVersion(), brokerName,\n                            entry.getValue().getTopicName())) {\n                            final TopicConfig topicConfig = entry.getValue();\n                            // In Slave Acting Master mode, Namesrv will regard the surviving Slave with the smallest brokerId as the \"agent\" Master, and modify the brokerPermission to read-only.\n                            if (isPrimeSlave && brokerData.isEnableActingMaster()) {\n                                // Wipe write perm for prime slave\n                                topicConfig.setPerm(topicConfig.getPerm() & (~PermName.PERM_WRITE));\n                            }\n                            this.createAndUpdateQueueData(brokerName, topicConfig);\n                        }\n                    }\n\n                    if (this.isBrokerTopicConfigChanged(clusterName, brokerAddr, topicConfigWrapper.getDataVersion()) || registerFirst) {\n                        //the topicQueueMappingInfoMap should never be null, but can be empty\n                        for (Map.Entry<String, TopicQueueMappingInfo> entry : topicQueueMappingInfoMap.entrySet()) {\n                            if (!topicQueueMappingInfoTable.containsKey(entry.getKey())) {\n                                topicQueueMappingInfoTable.put(entry.getKey(), new HashMap<>());\n                            }\n                            //Note asset brokerName equal entry.getValue().getBname()\n                            //here use the mappingDetail.bname\n                            topicQueueMappingInfoTable.get(entry.getKey()).put(entry.getValue().getBname(), entry.getValue());\n                        }\n                    }\n                }\n            }\n\n            BrokerAddrInfo brokerAddrInfo = new BrokerAddrInfo(clusterName, brokerAddr);\n            BrokerLiveInfo prevBrokerLiveInfo = this.brokerLiveTable.put(brokerAddrInfo,\n                new BrokerLiveInfo(\n                    System.currentTimeMillis(),\n                    timeoutMillis == null ? DEFAULT_BROKER_CHANNEL_EXPIRED_TIME : timeoutMillis,\n                    topicConfigWrapper == null ? new DataVersion() : topicConfigWrapper.getDataVersion(),\n                    channel,\n                    haServerAddr));\n            if (null == prevBrokerLiveInfo) {\n                log.info(\"new broker registered, {} HAService: {}\", brokerAddrInfo, haServerAddr);\n            }\n\n            if (filterServerList != null) {\n                if (filterServerList.isEmpty()) {\n                    this.filterServerTable.remove(brokerAddrInfo);\n                } else {\n                    this.filterServerTable.put(brokerAddrInfo, filterServerList);\n                }\n            }\n\n            if (MixAll.MASTER_ID != brokerId) {\n                String masterAddr = brokerData.getBrokerAddrs().get(MixAll.MASTER_ID);\n                if (masterAddr != null) {\n                    BrokerAddrInfo masterAddrInfo = new BrokerAddrInfo(clusterName, masterAddr);\n                    BrokerLiveInfo masterLiveInfo = this.brokerLiveTable.get(masterAddrInfo);\n                    if (masterLiveInfo != null) {\n                        result.setHaServerAddr(masterLiveInfo.getHaServerAddr());\n                        result.setMasterAddr(masterAddr);\n                    }\n                }\n            }\n\n            if (isMinBrokerIdChanged && namesrvConfig.isNotifyMinBrokerIdChanged()) {\n                notifyMinBrokerIdChanged(brokerAddrsMap, null,\n                    this.brokerLiveTable.get(brokerAddrInfo).getHaServerAddr());\n            }\n        } catch (Exception e) {\n            log.error(\"registerBroker Exception\", e);\n        } finally {\n            this.lock.writeLock().unlock();\n        }\n\n        return result;\n    }\n\n    private Set<String> topicSetOfBrokerName(final String brokerName) {\n        Set<String> topicOfBroker = new HashSet<>();\n        for (final Entry<String, Map<String, QueueData>> entry : this.topicQueueTable.entrySet()) {\n            if (entry.getValue().containsKey(brokerName)) {\n                topicOfBroker.add(entry.getKey());\n            }\n        }\n        return topicOfBroker;\n    }\n\n    public BrokerMemberGroup getBrokerMemberGroup(String clusterName, String brokerName) {\n        BrokerMemberGroup groupMember = new BrokerMemberGroup(clusterName, brokerName);\n        try {\n            try {\n                this.lock.readLock().lockInterruptibly();\n                final BrokerData brokerData = this.brokerAddrTable.get(brokerName);\n                if (brokerData != null) {\n                    groupMember.getBrokerAddrs().putAll(brokerData.getBrokerAddrs());\n                }\n            } finally {\n                this.lock.readLock().unlock();\n            }\n        } catch (Exception e) {\n            log.error(\"Get broker member group exception\", e);\n        }\n        return groupMember;\n    }\n\n    public boolean isBrokerTopicConfigChanged(final String clusterName, final String brokerAddr,\n        final DataVersion dataVersion) {\n        DataVersion prev = queryBrokerTopicConfig(clusterName, brokerAddr);\n        return null == prev || !prev.equals(dataVersion);\n    }\n\n    public boolean isTopicConfigChanged(final String clusterName, final String brokerAddr,\n        final DataVersion dataVersion, String brokerName, String topic) {\n        boolean isChange = isBrokerTopicConfigChanged(clusterName, brokerAddr, dataVersion);\n        if (isChange) {\n            return true;\n        }\n        final Map<String, QueueData> queueDataMap = this.topicQueueTable.get(topic);\n        if (queueDataMap == null || queueDataMap.isEmpty()) {\n            return true;\n        }\n\n        // The topicQueueTable already contains the broker\n        return !queueDataMap.containsKey(brokerName);\n    }\n\n    public DataVersion queryBrokerTopicConfig(final String clusterName, final String brokerAddr) {\n        BrokerAddrInfo addrInfo = new BrokerAddrInfo(clusterName, brokerAddr);\n        BrokerLiveInfo prev = this.brokerLiveTable.get(addrInfo);\n        if (prev != null) {\n            return prev.getDataVersion();\n        }\n        return null;\n    }\n\n    public void updateBrokerInfoUpdateTimestamp(final String clusterName, final String brokerAddr) {\n        BrokerAddrInfo addrInfo = new BrokerAddrInfo(clusterName, brokerAddr);\n        BrokerLiveInfo prev = this.brokerLiveTable.get(addrInfo);\n        if (prev != null) {\n            prev.setLastUpdateTimestamp(System.currentTimeMillis());\n        }\n    }\n\n    private void createAndUpdateQueueData(final String brokerName, final TopicConfig topicConfig) {\n        QueueData queueData = new QueueData();\n        queueData.setBrokerName(brokerName);\n        queueData.setWriteQueueNums(topicConfig.getWriteQueueNums());\n        queueData.setReadQueueNums(topicConfig.getReadQueueNums());\n        queueData.setPerm(topicConfig.getPerm());\n        queueData.setTopicSysFlag(topicConfig.getTopicSysFlag());\n\n        Map<String, QueueData> queueDataMap = this.topicQueueTable.get(topicConfig.getTopicName());\n        if (null == queueDataMap) {\n            queueDataMap = new HashMap<>();\n            queueDataMap.put(brokerName, queueData);\n            this.topicQueueTable.put(topicConfig.getTopicName(), queueDataMap);\n            log.info(\"new topic registered, {} {}\", topicConfig.getTopicName(), queueData);\n        } else {\n            final QueueData existedQD = queueDataMap.get(brokerName);\n            if (existedQD == null) {\n                queueDataMap.put(brokerName, queueData);\n            } else if (!existedQD.equals(queueData)) {\n                log.info(\"topic changed, {} OLD: {} NEW: {}\", topicConfig.getTopicName(), existedQD,\n                    queueData);\n                queueDataMap.put(brokerName, queueData);\n            }\n        }\n    }\n\n    public int wipeWritePermOfBrokerByLock(final String brokerName) {\n        try {\n            try {\n                this.lock.writeLock().lockInterruptibly();\n                return operateWritePermOfBroker(brokerName, RequestCode.WIPE_WRITE_PERM_OF_BROKER);\n            } finally {\n                this.lock.writeLock().unlock();\n            }\n        } catch (Exception e) {\n            log.error(\"wipeWritePermOfBrokerByLock Exception\", e);\n        }\n\n        return 0;\n    }\n\n    public int addWritePermOfBrokerByLock(final String brokerName) {\n        try {\n            try {\n                this.lock.writeLock().lockInterruptibly();\n                return operateWritePermOfBroker(brokerName, RequestCode.ADD_WRITE_PERM_OF_BROKER);\n            } finally {\n                this.lock.writeLock().unlock();\n            }\n        } catch (Exception e) {\n            log.error(\"addWritePermOfBrokerByLock Exception\", e);\n        }\n        return 0;\n    }\n\n    private int operateWritePermOfBroker(final String brokerName, final int requestCode) {\n        int topicCnt = 0;\n\n        for (Entry<String, Map<String, QueueData>> entry : this.topicQueueTable.entrySet()) {\n            Map<String, QueueData> qdMap = entry.getValue();\n\n            final QueueData qd = qdMap.get(brokerName);\n            if (qd == null) {\n                continue;\n            }\n            int perm = qd.getPerm();\n            switch (requestCode) {\n                case RequestCode.WIPE_WRITE_PERM_OF_BROKER:\n                    perm &= ~PermName.PERM_WRITE;\n                    break;\n                case RequestCode.ADD_WRITE_PERM_OF_BROKER:\n                    perm = PermName.PERM_READ | PermName.PERM_WRITE;\n                    break;\n            }\n            qd.setPerm(perm);\n            topicCnt++;\n        }\n        return topicCnt;\n    }\n\n    public void unregisterBroker(\n        final String clusterName,\n        final String brokerAddr,\n        final String brokerName,\n        final long brokerId) {\n        UnRegisterBrokerRequestHeader unRegisterBrokerRequest = new UnRegisterBrokerRequestHeader();\n        unRegisterBrokerRequest.setClusterName(clusterName);\n        unRegisterBrokerRequest.setBrokerAddr(brokerAddr);\n        unRegisterBrokerRequest.setBrokerName(brokerName);\n        unRegisterBrokerRequest.setBrokerId(brokerId);\n\n        unRegisterBroker(Sets.newHashSet(unRegisterBrokerRequest));\n    }\n\n    public void unRegisterBroker(Set<UnRegisterBrokerRequestHeader> unRegisterRequests) {\n        try {\n            Set<String> removedBroker = new HashSet<>();\n            Set<String> reducedBroker = new HashSet<>();\n            Map<String, BrokerStatusChangeInfo> needNotifyBrokerMap = new HashMap<>();\n\n            this.lock.writeLock().lockInterruptibly();\n            for (final UnRegisterBrokerRequestHeader unRegisterRequest : unRegisterRequests) {\n                final String brokerName = unRegisterRequest.getBrokerName();\n                final String clusterName = unRegisterRequest.getClusterName();\n                final String brokerAddr = unRegisterRequest.getBrokerAddr();\n\n                BrokerAddrInfo brokerAddrInfo = new BrokerAddrInfo(clusterName, brokerAddr);\n\n                BrokerLiveInfo brokerLiveInfo = this.brokerLiveTable.remove(brokerAddrInfo);\n                log.info(\"unregisterBroker, remove from brokerLiveTable {}, {}\",\n                    brokerLiveInfo != null ? \"OK\" : \"Failed\",\n                    brokerAddrInfo\n                );\n\n                this.filterServerTable.remove(brokerAddrInfo);\n\n                boolean removeBrokerName = false;\n                boolean isMinBrokerIdChanged = false;\n                BrokerData brokerData = this.brokerAddrTable.get(brokerName);\n                if (null != brokerData) {\n                    if (!brokerData.getBrokerAddrs().isEmpty() &&\n                        unRegisterRequest.getBrokerId().equals(Collections.min(brokerData.getBrokerAddrs().keySet()))) {\n                        isMinBrokerIdChanged = true;\n                    }\n                    boolean removed = brokerData.getBrokerAddrs().entrySet().removeIf(item -> item.getValue().equals(brokerAddr));\n                    log.info(\"unregisterBroker, remove addr from brokerAddrTable {}, {}\",\n                        removed ? \"OK\" : \"Failed\",\n                        brokerAddrInfo\n                    );\n                    if (brokerData.getBrokerAddrs().isEmpty()) {\n                        this.brokerAddrTable.remove(brokerName);\n                        log.info(\"unregisterBroker, remove name from brokerAddrTable OK, {}\",\n                            brokerName\n                        );\n\n                        removeBrokerName = true;\n                    } else if (isMinBrokerIdChanged) {\n                        needNotifyBrokerMap.put(brokerName, new BrokerStatusChangeInfo(\n                            brokerData.getBrokerAddrs(), brokerAddr, null));\n                    }\n                }\n\n                if (removeBrokerName) {\n                    Set<String> nameSet = this.clusterAddrTable.get(clusterName);\n                    if (nameSet != null) {\n                        boolean removed = nameSet.remove(brokerName);\n                        log.info(\"unregisterBroker, remove name from clusterAddrTable {}, {}\",\n                            removed ? \"OK\" : \"Failed\",\n                            brokerName);\n\n                        if (nameSet.isEmpty()) {\n                            this.clusterAddrTable.remove(clusterName);\n                            log.info(\"unregisterBroker, remove cluster from clusterAddrTable {}\",\n                                clusterName\n                            );\n                        }\n                    }\n                    removedBroker.add(brokerName);\n                } else {\n                    reducedBroker.add(brokerName);\n                }\n            }\n\n            cleanTopicByUnRegisterRequests(removedBroker, reducedBroker);\n\n            if (!needNotifyBrokerMap.isEmpty() && namesrvConfig.isNotifyMinBrokerIdChanged()) {\n                notifyMinBrokerIdChanged(needNotifyBrokerMap);\n            }\n        } catch (Exception e) {\n            log.error(\"unregisterBroker Exception\", e);\n        } finally {\n            this.lock.writeLock().unlock();\n        }\n    }\n\n    private void cleanTopicByUnRegisterRequests(Set<String> removedBroker, Set<String> reducedBroker) {\n        Iterator<Entry<String, Map<String, QueueData>>> itMap = this.topicQueueTable.entrySet().iterator();\n        while (itMap.hasNext()) {\n            Entry<String, Map<String, QueueData>> entry = itMap.next();\n\n            String topic = entry.getKey();\n            Map<String, QueueData> queueDataMap = entry.getValue();\n\n            for (final String brokerName : removedBroker) {\n                final QueueData removedQD = queueDataMap.remove(brokerName);\n                if (removedQD != null) {\n                    log.debug(\"removeTopicByBrokerName, remove one broker's topic {} {}\", topic, removedQD);\n                }\n            }\n\n            if (queueDataMap.isEmpty()) {\n                log.debug(\"removeTopicByBrokerName, remove the topic all queue {}\", topic);\n                itMap.remove();\n            }\n\n            for (final String brokerName : reducedBroker) {\n                final QueueData queueData = queueDataMap.get(brokerName);\n\n                if (queueData != null) {\n                    if (this.brokerAddrTable.get(brokerName).isEnableActingMaster()) {\n                        // Master has been unregistered, wipe the write perm\n                        if (isNoMasterExists(brokerName)) {\n                            queueData.setPerm(queueData.getPerm() & (~PermName.PERM_WRITE));\n                        }\n                    }\n                }\n            }\n        }\n    }\n\n    private boolean isNoMasterExists(String brokerName) {\n        final BrokerData brokerData = this.brokerAddrTable.get(brokerName);\n        if (brokerData == null) {\n            return true;\n        }\n\n        if (brokerData.getBrokerAddrs().size() == 0) {\n            return true;\n        }\n\n        return Collections.min(brokerData.getBrokerAddrs().keySet()) > 0;\n    }\n\n    public TopicRouteData pickupTopicRouteData(final String topic) {\n        TopicRouteData topicRouteData = new TopicRouteData();\n        boolean foundQueueData = false;\n        boolean foundBrokerData = false;\n        List<BrokerData> brokerDataList = new LinkedList<>();\n        topicRouteData.setBrokerDatas(brokerDataList);\n\n        HashMap<String, List<String>> filterServerMap = new HashMap<>();\n        topicRouteData.setFilterServerTable(filterServerMap);\n\n        try {\n            this.lock.readLock().lockInterruptibly();\n            Map<String, QueueData> queueDataMap = this.topicQueueTable.get(topic);\n            if (queueDataMap != null) {\n                topicRouteData.setQueueDatas(new ArrayList<>(queueDataMap.values()));\n                foundQueueData = true;\n\n                Set<String> brokerNameSet = new HashSet<>(queueDataMap.keySet());\n\n                for (String brokerName : brokerNameSet) {\n                    BrokerData brokerData = this.brokerAddrTable.get(brokerName);\n                    if (null == brokerData) {\n                        continue;\n                    }\n                    BrokerData brokerDataClone = new BrokerData(brokerData);\n\n                    brokerDataList.add(brokerDataClone);\n                    foundBrokerData = true;\n                    if (filterServerTable.isEmpty()) {\n                        continue;\n                    }\n                    for (final String brokerAddr : brokerDataClone.getBrokerAddrs().values()) {\n                        BrokerAddrInfo brokerAddrInfo = new BrokerAddrInfo(brokerDataClone.getCluster(), brokerAddr);\n                        List<String> filterServerList = this.filterServerTable.get(brokerAddrInfo);\n                        filterServerMap.put(brokerAddr, filterServerList);\n                    }\n\n                }\n            }\n        } catch (Exception e) {\n            log.error(\"pickupTopicRouteData Exception\", e);\n        } finally {\n            this.lock.readLock().unlock();\n        }\n\n        log.debug(\"pickupTopicRouteData {} {}\", topic, topicRouteData);\n\n        if (foundBrokerData && foundQueueData) {\n\n            topicRouteData.setTopicQueueMappingByBroker(this.topicQueueMappingInfoTable.get(topic));\n\n            if (!namesrvConfig.isSupportActingMaster()) {\n                return topicRouteData;\n            }\n\n            if (topic.startsWith(TopicValidator.SYNC_BROKER_MEMBER_GROUP_PREFIX)) {\n                return topicRouteData;\n            }\n\n            if (topicRouteData.getBrokerDatas().size() == 0 || topicRouteData.getQueueDatas().size() == 0) {\n                return topicRouteData;\n            }\n\n            boolean needActingMaster = false;\n\n            for (final BrokerData brokerData : topicRouteData.getBrokerDatas()) {\n                if (brokerData.getBrokerAddrs().size() != 0\n                    && !brokerData.getBrokerAddrs().containsKey(MixAll.MASTER_ID)) {\n                    needActingMaster = true;\n                    break;\n                }\n            }\n\n            if (!needActingMaster) {\n                return topicRouteData;\n            }\n\n            for (final BrokerData brokerData : topicRouteData.getBrokerDatas()) {\n                final HashMap<Long, String> brokerAddrs = brokerData.getBrokerAddrs();\n                if (brokerAddrs.size() == 0 || brokerAddrs.containsKey(MixAll.MASTER_ID) || !brokerData.isEnableActingMaster()) {\n                    continue;\n                }\n\n                // No master\n                for (final QueueData queueData : topicRouteData.getQueueDatas()) {\n                    if (queueData.getBrokerName().equals(brokerData.getBrokerName())) {\n                        if (!PermName.isWriteable(queueData.getPerm())) {\n                            final Long minBrokerId = Collections.min(brokerAddrs.keySet());\n                            final String actingMasterAddr = brokerAddrs.remove(minBrokerId);\n                            brokerAddrs.put(MixAll.MASTER_ID, actingMasterAddr);\n                        }\n                        break;\n                    }\n                }\n\n            }\n\n            return topicRouteData;\n        }\n\n        return null;\n    }\n\n    public void scanNotActiveBroker() {\n        try {\n            log.info(\"start scanNotActiveBroker\");\n            for (Entry<BrokerAddrInfo, BrokerLiveInfo> next : this.brokerLiveTable.entrySet()) {\n                long last = next.getValue().getLastUpdateTimestamp();\n                long timeoutMillis = next.getValue().getHeartbeatTimeoutMillis();\n                if ((last + timeoutMillis) < System.currentTimeMillis()) {\n                    RemotingHelper.closeChannel(next.getValue().getChannel());\n                    log.warn(\"The broker channel expired, {} {}ms\", next.getKey(), timeoutMillis);\n                    this.onChannelDestroy(next.getKey());\n                }\n            }\n        } catch (Exception e) {\n            log.error(\"scanNotActiveBroker exception\", e);\n        }\n    }\n\n    public void onChannelDestroy(BrokerAddrInfo brokerAddrInfo) {\n        UnRegisterBrokerRequestHeader unRegisterRequest = new UnRegisterBrokerRequestHeader();\n        boolean needUnRegister = false;\n        if (brokerAddrInfo != null) {\n            try {\n                try {\n                    this.lock.readLock().lockInterruptibly();\n                    needUnRegister = setupUnRegisterRequest(unRegisterRequest, brokerAddrInfo);\n                } finally {\n                    this.lock.readLock().unlock();\n                }\n            } catch (Exception e) {\n                log.error(\"onChannelDestroy Exception\", e);\n            }\n        }\n\n        if (needUnRegister) {\n            boolean result = this.submitUnRegisterBrokerRequest(unRegisterRequest);\n            log.info(\"the broker's channel destroyed, submit the unregister request at once, \" +\n                \"broker info: {}, submit result: {}\", unRegisterRequest, result);\n        }\n    }\n\n    public void onChannelDestroy(Channel channel) {\n        UnRegisterBrokerRequestHeader unRegisterRequest = new UnRegisterBrokerRequestHeader();\n        BrokerAddrInfo brokerAddrFound = null;\n        boolean needUnRegister = false;\n        if (channel != null) {\n            try {\n                try {\n                    this.lock.readLock().lockInterruptibly();\n                    for (Entry<BrokerAddrInfo, BrokerLiveInfo> entry : this.brokerLiveTable.entrySet()) {\n                        if (entry.getValue().getChannel() == channel) {\n                            brokerAddrFound = entry.getKey();\n                            break;\n                        }\n                    }\n\n                    if (brokerAddrFound != null) {\n                        needUnRegister = setupUnRegisterRequest(unRegisterRequest, brokerAddrFound);\n                    }\n                } finally {\n                    this.lock.readLock().unlock();\n                }\n            } catch (Exception e) {\n                log.error(\"onChannelDestroy Exception\", e);\n            }\n        }\n\n        if (needUnRegister) {\n            boolean result = this.submitUnRegisterBrokerRequest(unRegisterRequest);\n            log.info(\"the broker's channel destroyed, submit the unregister request at once, \" +\n                \"broker info: {}, submit result: {}\", unRegisterRequest, result);\n        }\n    }\n\n    private boolean setupUnRegisterRequest(UnRegisterBrokerRequestHeader unRegisterRequest,\n        BrokerAddrInfo brokerAddrInfo) {\n        unRegisterRequest.setClusterName(brokerAddrInfo.getClusterName());\n        unRegisterRequest.setBrokerAddr(brokerAddrInfo.getBrokerAddr());\n\n        for (Entry<String, BrokerData> stringBrokerDataEntry : this.brokerAddrTable.entrySet()) {\n            BrokerData brokerData = stringBrokerDataEntry.getValue();\n            if (!brokerAddrInfo.getClusterName().equals(brokerData.getCluster())) {\n                continue;\n            }\n\n            for (Entry<Long, String> entry : brokerData.getBrokerAddrs().entrySet()) {\n                Long brokerId = entry.getKey();\n                String brokerAddr = entry.getValue();\n                if (brokerAddr.equals(brokerAddrInfo.getBrokerAddr())) {\n                    unRegisterRequest.setBrokerName(brokerData.getBrokerName());\n                    unRegisterRequest.setBrokerId(brokerId);\n                    return true;\n                }\n            }\n        }\n\n        return false;\n    }\n\n    private void notifyMinBrokerIdChanged(Map<String, BrokerStatusChangeInfo> needNotifyBrokerMap)\n        throws InterruptedException, RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException,\n        RemotingTooMuchRequestException {\n        for (String brokerName : needNotifyBrokerMap.keySet()) {\n            BrokerStatusChangeInfo brokerStatusChangeInfo = needNotifyBrokerMap.get(brokerName);\n            BrokerData brokerData = brokerAddrTable.get(brokerName);\n            if (brokerData != null && brokerData.isEnableActingMaster()) {\n                notifyMinBrokerIdChanged(brokerStatusChangeInfo.getBrokerAddrs(),\n                    brokerStatusChangeInfo.getOfflineBrokerAddr(), brokerStatusChangeInfo.getHaBrokerAddr());\n            }\n        }\n    }\n\n    private void notifyMinBrokerIdChanged(Map<Long, String> brokerAddrMap, String offlineBrokerAddr,\n        String haBrokerAddr)\n        throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException,\n        RemotingTooMuchRequestException, RemotingConnectException {\n        if (brokerAddrMap == null || brokerAddrMap.isEmpty() || this.namesrvController == null) {\n            return;\n        }\n\n        NotifyMinBrokerIdChangeRequestHeader requestHeader = new NotifyMinBrokerIdChangeRequestHeader();\n        long minBrokerId = Collections.min(brokerAddrMap.keySet());\n        requestHeader.setMinBrokerId(minBrokerId);\n        requestHeader.setMinBrokerAddr(brokerAddrMap.get(minBrokerId));\n        requestHeader.setOfflineBrokerAddr(offlineBrokerAddr);\n        requestHeader.setHaBrokerAddr(haBrokerAddr);\n\n        List<String> brokerAddrsNotify = chooseBrokerAddrsToNotify(brokerAddrMap, offlineBrokerAddr);\n        log.info(\"min broker id changed to {}, notify {}, offline broker addr {}\", minBrokerId, brokerAddrsNotify, offlineBrokerAddr);\n        RemotingCommand request =\n            RemotingCommand.createRequestCommand(RequestCode.NOTIFY_MIN_BROKER_ID_CHANGE, requestHeader);\n        for (String brokerAddr : brokerAddrsNotify) {\n            this.namesrvController.getRemotingClient().invokeOneway(brokerAddr, request, 300);\n        }\n    }\n\n    private List<String> chooseBrokerAddrsToNotify(Map<Long, String> brokerAddrMap, String offlineBrokerAddr) {\n        if (offlineBrokerAddr != null || brokerAddrMap.size() == 1) {\n            // notify the reset brokers.\n            return new ArrayList<>(brokerAddrMap.values());\n        }\n\n        // new broker registered, notify previous brokers.\n        long minBrokerId = Collections.min(brokerAddrMap.keySet());\n        List<String> brokerAddrList = new ArrayList<>();\n        for (Long brokerId : brokerAddrMap.keySet()) {\n            if (brokerId != minBrokerId) {\n                brokerAddrList.add(brokerAddrMap.get(brokerId));\n            }\n        }\n        return brokerAddrList;\n    }\n\n    // For test only\n    public void printAllPeriodically() {\n        try {\n            try {\n                this.lock.readLock().lockInterruptibly();\n                log.info(\"--------------------------------------------------------\");\n                {\n                    log.info(\"topicQueueTable SIZE: {}\", this.topicQueueTable.size());\n                    for (Entry<String, Map<String, QueueData>> next : this.topicQueueTable.entrySet()) {\n                        log.info(\"topicQueueTable Topic: {} {}\", next.getKey(), next.getValue());\n                    }\n                }\n\n                {\n                    log.info(\"brokerAddrTable SIZE: {}\", this.brokerAddrTable.size());\n                    for (Entry<String, BrokerData> next : this.brokerAddrTable.entrySet()) {\n                        log.info(\"brokerAddrTable brokerName: {} {}\", next.getKey(), next.getValue());\n                    }\n                }\n\n                {\n                    log.info(\"brokerLiveTable SIZE: {}\", this.brokerLiveTable.size());\n                    for (Entry<BrokerAddrInfo, BrokerLiveInfo> next : this.brokerLiveTable.entrySet()) {\n                        log.info(\"brokerLiveTable brokerAddr: {} {}\", next.getKey(), next.getValue());\n                    }\n                }\n\n                {\n                    log.info(\"clusterAddrTable SIZE: {}\", this.clusterAddrTable.size());\n                    for (Entry<String, Set<String>> next : this.clusterAddrTable.entrySet()) {\n                        log.info(\"clusterAddrTable clusterName: {} {}\", next.getKey(), next.getValue());\n                    }\n                }\n            } finally {\n                this.lock.readLock().unlock();\n            }\n        } catch (Exception e) {\n            log.error(\"printAllPeriodically Exception\", e);\n        }\n    }\n\n    public TopicList getSystemTopicList() {\n        TopicList topicList = new TopicList();\n        try {\n            this.lock.readLock().lockInterruptibly();\n            for (Map.Entry<String, Set<String>> entry : clusterAddrTable.entrySet()) {\n                topicList.getTopicList().add(entry.getKey());\n                topicList.getTopicList().addAll(entry.getValue());\n            }\n\n            if (!brokerAddrTable.isEmpty()) {\n                for (String s : brokerAddrTable.keySet()) {\n                    BrokerData bd = brokerAddrTable.get(s);\n                    HashMap<Long, String> brokerAddrs = bd.getBrokerAddrs();\n                    if (brokerAddrs != null && !brokerAddrs.isEmpty()) {\n                        Iterator<Long> it2 = brokerAddrs.keySet().iterator();\n                        topicList.setBrokerAddr(brokerAddrs.get(it2.next()));\n                        break;\n                    }\n                }\n            }\n        } catch (Exception e) {\n            log.error(\"getSystemTopicList Exception\", e);\n        } finally {\n            this.lock.readLock().unlock();\n        }\n\n        return topicList;\n    }\n\n    public TopicList getTopicsByCluster(String cluster) {\n        TopicList topicList = new TopicList();\n        try {\n            try {\n                this.lock.readLock().lockInterruptibly();\n                Set<String> brokerNameSet = this.clusterAddrTable.get(cluster);\n                for (String brokerName : brokerNameSet) {\n                    for (Entry<String, Map<String, QueueData>> topicEntry : this.topicQueueTable.entrySet()) {\n                        String topic = topicEntry.getKey();\n                        Map<String, QueueData> queueDataMap = topicEntry.getValue();\n                        final QueueData qd = queueDataMap.get(brokerName);\n                        if (qd != null) {\n                            topicList.getTopicList().add(topic);\n                        }\n                    }\n                }\n            } finally {\n                this.lock.readLock().unlock();\n            }\n        } catch (Exception e) {\n            log.error(\"getTopicsByCluster Exception\", e);\n        }\n\n        return topicList;\n    }\n\n    public TopicList getUnitTopics() {\n        TopicList topicList = new TopicList();\n        try {\n            this.lock.readLock().lockInterruptibly();\n            for (Entry<String, Map<String, QueueData>> topicEntry : this.topicQueueTable.entrySet()) {\n                String topic = topicEntry.getKey();\n                Map<String, QueueData> queueDatas = topicEntry.getValue();\n                if (queueDatas != null && queueDatas.size() > 0\n                    && TopicSysFlag.hasUnitFlag(queueDatas.values().iterator().next().getTopicSysFlag())) {\n                    topicList.getTopicList().add(topic);\n                }\n            }\n        } catch (Exception e) {\n            log.error(\"getUnitTopics Exception\", e);\n        } finally {\n            this.lock.readLock().unlock();\n        }\n\n        return topicList;\n    }\n\n    public TopicList getHasUnitSubTopicList() {\n        TopicList topicList = new TopicList();\n        try {\n            this.lock.readLock().lockInterruptibly();\n            for (Entry<String, Map<String, QueueData>> topicEntry : this.topicQueueTable.entrySet()) {\n                String topic = topicEntry.getKey();\n                Map<String, QueueData> queueDatas = topicEntry.getValue();\n                if (queueDatas != null && queueDatas.size() > 0\n                    && TopicSysFlag.hasUnitSubFlag(queueDatas.values().iterator().next().getTopicSysFlag())) {\n                    topicList.getTopicList().add(topic);\n                }\n            }\n        } catch (Exception e) {\n            log.error(\"getHasUnitSubTopicList Exception\", e);\n        } finally {\n            this.lock.readLock().unlock();\n        }\n\n        return topicList;\n    }\n\n    public TopicList getHasUnitSubUnUnitTopicList() {\n        TopicList topicList = new TopicList();\n        try {\n            this.lock.readLock().lockInterruptibly();\n            for (Entry<String, Map<String, QueueData>> topicEntry : this.topicQueueTable.entrySet()) {\n                String topic = topicEntry.getKey();\n                Map<String, QueueData> queueDatas = topicEntry.getValue();\n                if (queueDatas != null && queueDatas.size() > 0\n                    && !TopicSysFlag.hasUnitFlag(queueDatas.values().iterator().next().getTopicSysFlag())\n                    && TopicSysFlag.hasUnitSubFlag(queueDatas.values().iterator().next().getTopicSysFlag())) {\n                    topicList.getTopicList().add(topic);\n                }\n            }\n        } catch (Exception e) {\n            log.error(\"getHasUnitSubUnUnitTopicList Exception\", e);\n        } finally {\n            this.lock.readLock().unlock();\n        }\n\n        return topicList;\n    }\n}\n\n/**\n * broker address information\n */\nclass BrokerAddrInfo {\n    private String clusterName;\n    private String brokerAddr;\n\n    private int hash;\n\n    public BrokerAddrInfo(String clusterName, String brokerAddr) {\n        this.clusterName = clusterName;\n        this.brokerAddr = brokerAddr;\n    }\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    public String getBrokerAddr() {\n        return brokerAddr;\n    }\n\n    public boolean isEmpty() {\n        return clusterName.isEmpty() && brokerAddr.isEmpty();\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj) {\n            return true;\n        }\n        if (obj == null) {\n            return false;\n        }\n\n        if (obj instanceof BrokerAddrInfo) {\n            BrokerAddrInfo addr = (BrokerAddrInfo) obj;\n            return clusterName.equals(addr.clusterName) && brokerAddr.equals(addr.brokerAddr);\n        }\n        return false;\n    }\n\n    @Override\n    public int hashCode() {\n        int h = hash;\n        if (h == 0 && clusterName.length() + brokerAddr.length() > 0) {\n            for (int i = 0; i < clusterName.length(); i++) {\n                h = 31 * h + clusterName.charAt(i);\n            }\n            h = 31 * h + '_';\n            for (int i = 0; i < brokerAddr.length(); i++) {\n                h = 31 * h + brokerAddr.charAt(i);\n            }\n            hash = h;\n        }\n        return h;\n    }\n\n    @Override\n    public String toString() {\n        return \"BrokerIdentityInfo [clusterName=\" + clusterName + \", brokerAddr=\" + brokerAddr + \"]\";\n    }\n}\n\nclass BrokerLiveInfo {\n    private long lastUpdateTimestamp;\n    private long heartbeatTimeoutMillis;\n    private DataVersion dataVersion;\n    private Channel channel;\n    private String haServerAddr;\n\n    public BrokerLiveInfo(long lastUpdateTimestamp, long heartbeatTimeoutMillis, DataVersion dataVersion,\n        Channel channel,\n        String haServerAddr) {\n        this.lastUpdateTimestamp = lastUpdateTimestamp;\n        this.heartbeatTimeoutMillis = heartbeatTimeoutMillis;\n        this.dataVersion = dataVersion;\n        this.channel = channel;\n        this.haServerAddr = haServerAddr;\n    }\n\n    public long getLastUpdateTimestamp() {\n        return lastUpdateTimestamp;\n    }\n\n    public void setLastUpdateTimestamp(long lastUpdateTimestamp) {\n        this.lastUpdateTimestamp = lastUpdateTimestamp;\n    }\n\n    public long getHeartbeatTimeoutMillis() {\n        return heartbeatTimeoutMillis;\n    }\n\n    public void setHeartbeatTimeoutMillis(long heartbeatTimeoutMillis) {\n        this.heartbeatTimeoutMillis = heartbeatTimeoutMillis;\n    }\n\n    public DataVersion getDataVersion() {\n        return dataVersion;\n    }\n\n    public void setDataVersion(DataVersion dataVersion) {\n        this.dataVersion = dataVersion;\n    }\n\n    public Channel getChannel() {\n        return channel;\n    }\n\n    public void setChannel(Channel channel) {\n        this.channel = channel;\n    }\n\n    public String getHaServerAddr() {\n        return haServerAddr;\n    }\n\n    public void setHaServerAddr(String haServerAddr) {\n        this.haServerAddr = haServerAddr;\n    }\n\n    @Override\n    public String toString() {\n        return \"BrokerLiveInfo [lastUpdateTimestamp=\" + lastUpdateTimestamp + \", dataVersion=\" + dataVersion\n            + \", channel=\" + channel + \", haServerAddr=\" + haServerAddr + \"]\";\n    }\n}\n\nclass BrokerStatusChangeInfo {\n    Map<Long, String> brokerAddrs;\n    String offlineBrokerAddr;\n    String haBrokerAddr;\n\n    public BrokerStatusChangeInfo(Map<Long, String> brokerAddrs, String offlineBrokerAddr, String haBrokerAddr) {\n        this.brokerAddrs = brokerAddrs;\n        this.offlineBrokerAddr = offlineBrokerAddr;\n        this.haBrokerAddr = haBrokerAddr;\n    }\n\n    public Map<Long, String> getBrokerAddrs() {\n        return brokerAddrs;\n    }\n\n    public void setBrokerAddrs(Map<Long, String> brokerAddrs) {\n        this.brokerAddrs = brokerAddrs;\n    }\n\n    public String getOfflineBrokerAddr() {\n        return offlineBrokerAddr;\n    }\n\n    public void setOfflineBrokerAddr(String offlineBrokerAddr) {\n        this.offlineBrokerAddr = offlineBrokerAddr;\n    }\n\n    public String getHaBrokerAddr() {\n        return haBrokerAddr;\n    }\n\n    public void setHaBrokerAddr(String haBrokerAddr) {\n        this.haBrokerAddr = haBrokerAddr;\n    }\n}\n"
  },
  {
    "path": "namesrv/src/main/resources/rmq.namesrv.logback.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n<configuration scan=\"true\" scanPeriod=\"30 seconds\">\n    <appender name=\"DefaultAppender\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}namesrv_default.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}namesrv_default.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>5</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy\n                class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n\n    <appender name=\"RocketmqNamesrvAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}namesrv.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}namesrv.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>5</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy\n                class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqNamesrvAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqNamesrvAppender_inner\"/>\n        <discardingThreshold>0</discardingThreshold>\n    </appender>\n\n    <appender name=\"RocketmqTrafficAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}namesrv_traffic.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}namesrv_traffic.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqTrafficAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqTrafficAppender_inner\"/>\n    </appender>\n\n    <appender name=\"STDOUT\" class=\"ch.qos.logback.core.ConsoleAppender\">\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n\n    <logger name=\"RocketmqNamesrv\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqNamesrvAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqCommon\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqNamesrvAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqRemoting\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqNamesrvAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqNamesrvConsole\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"STDOUT\"/>\n    </logger>\n\n    <logger name=\"RocketmqController\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqNamesrvAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqTraffic\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqTrafficAppender\" />\n    </logger>\n\n    <root level=\"INFO\">\n        <appender-ref ref=\"DefaultAppender\"/>\n    </root>\n</configuration>\n"
  },
  {
    "path": "namesrv/src/test/java/org/apache/rocketmq/namesrv/NameServerInstanceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.namesrv;\n\nimport org.apache.rocketmq.common.namesrv.NamesrvConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class NameServerInstanceTest {\n    protected NamesrvController nameSrvController = null;\n    protected NettyServerConfig nettyServerConfig = new NettyServerConfig();\n    protected NamesrvConfig namesrvConfig = new NamesrvConfig();\n\n    @Before\n    public void startup() throws Exception {\n        nettyServerConfig.setListenPort(9876);\n        nameSrvController = new NamesrvController(namesrvConfig, nettyServerConfig);\n        boolean initResult = nameSrvController.initialize();\n        assertThat(initResult).isTrue();\n        nameSrvController.start();\n    }\n\n    /**\n     * Add a placeholder to make Bazel happy.\n     */\n    @Test\n    public void itWorks() {\n\n    }\n\n    @After\n    public void shutdown() throws Exception {\n        if (nameSrvController != null) {\n            nameSrvController.shutdown();\n        }\n        //maybe need to clean the file store. But we do not suggest deleting anything.\n    }\n}\n"
  },
  {
    "path": "namesrv/src/test/java/org/apache/rocketmq/namesrv/NamesrvControllerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.namesrv;\n\nimport org.apache.rocketmq.common.namesrv.NamesrvConfig;\nimport org.apache.rocketmq.namesrv.kvconfig.KVConfigManager;\nimport org.apache.rocketmq.namesrv.routeinfo.RouteInfoManager;\nimport org.apache.rocketmq.remoting.Configuration;\nimport org.apache.rocketmq.remoting.RemotingServer;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class NamesrvControllerTest {\n\n    @Mock\n    private NettyServerConfig nettyServerConfig;\n    @Mock\n    private RemotingServer remotingServer;\n\n    private NamesrvController namesrvController;\n\n    @Before\n    public void setUp() throws Exception {\n        NamesrvConfig namesrvConfig = new NamesrvConfig();\n        namesrvController = new NamesrvController(namesrvConfig, nettyServerConfig);\n    }\n\n    @Test\n    public void getNamesrvConfig() {\n        NamesrvConfig namesrvConfig = namesrvController.getNamesrvConfig();\n        Assert.assertNotNull(namesrvConfig);\n    }\n\n    @Test\n    public void getNettyServerConfig() {\n        NettyServerConfig nettyServerConfig = namesrvController.getNettyServerConfig();\n        Assert.assertNotNull(nettyServerConfig);\n    }\n\n    @Test\n    public void getKvConfigManager() {\n        KVConfigManager manager = namesrvController.getKvConfigManager();\n        Assert.assertNotNull(manager);\n    }\n\n    @Test\n    public void getRouteInfoManager() {\n        RouteInfoManager manager = namesrvController.getRouteInfoManager();\n        Assert.assertNotNull(manager);\n    }\n\n    @Test\n    public void getRemotingServer() {\n        RemotingServer server = namesrvController.getRemotingServer();\n        Assert.assertNull(server);\n    }\n\n    @Test\n    public void setRemotingServer() {\n        namesrvController.setRemotingServer(remotingServer);\n        RemotingServer server = namesrvController.getRemotingServer();\n        Assert.assertEquals(remotingServer, server);\n    }\n\n    @Test\n    public void getConfiguration() {\n        Configuration configuration = namesrvController.getConfiguration();\n        Assert.assertNotNull(configuration);\n    }\n}\n"
  },
  {
    "path": "namesrv/src/test/java/org/apache/rocketmq/namesrv/NamesrvStartupTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.namesrv;\n\nimport java.util.Properties;\nimport org.apache.commons.cli.Options;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class NamesrvStartupTest {\n\n    @Mock\n    private NamesrvController namesrvController;\n    @Mock\n    private Options options;\n\n    @Before\n    public void setUp() throws Exception {\n        Mockito.when(namesrvController.initialize()).thenReturn(true);\n    }\n\n    @Test\n    public void testStart() throws Exception {\n        NamesrvController controller = NamesrvStartup.start(namesrvController);\n        Assert.assertNotNull(controller);\n    }\n\n    @Test\n    public void testShutdown() {\n        NamesrvStartup.shutdown(namesrvController);\n        Mockito.verify(namesrvController).shutdown();\n    }\n\n    @Test\n    public void testBuildCommandlineOptions() {\n        Options options = NamesrvStartup.buildCommandlineOptions(this.options);\n        Assert.assertNotNull(options);\n    }\n\n    @Test\n    public void testGetProperties() {\n        Properties properties = NamesrvStartup.getProperties();\n        Assert.assertNull(properties);\n    }\n}"
  },
  {
    "path": "namesrv/src/test/java/org/apache/rocketmq/namesrv/kvconfig/KVConfigManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.namesrv.kvconfig;\n\nimport org.apache.rocketmq.common.namesrv.NamesrvUtil;\nimport org.apache.rocketmq.namesrv.NameServerInstanceTest;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class KVConfigManagerTest extends NameServerInstanceTest {\n    private KVConfigManager kvConfigManager;\n\n    @Before\n    public void setup() throws Exception {\n        kvConfigManager = new KVConfigManager(nameSrvController);\n    }\n\n    @Test\n    public void testPutKVConfig() {\n        kvConfigManager.putKVConfig(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, \"UnitTest\", \"test\");\n        byte[] kvConfig = kvConfigManager.getKVListByNamespace(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG);\n        assertThat(kvConfig).isNotNull();\n        String value = kvConfigManager.getKVConfig(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, \"UnitTest\");\n        assertThat(value).isEqualTo(\"test\");\n    }\n\n    @Test\n    public void testDeleteKVConfig() {\n        kvConfigManager.deleteKVConfig(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, \"UnitTest\");\n        byte[] kvConfig = kvConfigManager.getKVListByNamespace(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG);\n        assertThat(kvConfig).isNull();\n        Assert.assertTrue(kvConfig == null);\n        String value = kvConfigManager.getKVConfig(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, \"UnitTest\");\n        assertThat(value).isNull();\n    }\n}"
  },
  {
    "path": "namesrv/src/test/java/org/apache/rocketmq/namesrv/kvconfig/KVConfigSerializeWrapperTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.namesrv.kvconfig;\n\nimport java.util.HashMap;\nimport org.apache.rocketmq.common.namesrv.NamesrvUtil;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class KVConfigSerializeWrapperTest {\n    private KVConfigSerializeWrapper kvConfigSerializeWrapper;\n\n    @Before\n    public void setup() throws Exception {\n        kvConfigSerializeWrapper = new KVConfigSerializeWrapper();\n    }\n\n    @Test\n    public void testEncodeAndDecode() {\n        HashMap<String, HashMap<String, String>> result = new HashMap<>();\n        HashMap<String, String> kvs = new HashMap<>();\n        kvs.put(\"broker-name\", \"default-broker\");\n        kvs.put(\"topic-name\", \"default-topic\");\n        kvs.put(\"cid\", \"default-consumer-name\");\n        result.put(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, kvs);\n        kvConfigSerializeWrapper.setConfigTable(result);\n        byte[] serializeByte = KVConfigSerializeWrapper.encode(kvConfigSerializeWrapper);\n        assertThat(serializeByte).isNotNull();\n\n        KVConfigSerializeWrapper deserializeObject = KVConfigSerializeWrapper.decode(serializeByte, KVConfigSerializeWrapper.class);\n        assertThat(deserializeObject.getConfigTable()).containsKey(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG);\n        assertThat(deserializeObject.getConfigTable().get(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG).get(\"broker-name\")).isEqualTo(\"default-broker\");\n        assertThat(deserializeObject.getConfigTable().get(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG).get(\"topic-name\")).isEqualTo(\"default-topic\");\n        assertThat(deserializeObject.getConfigTable().get(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG).get(\"cid\")).isEqualTo(\"default-consumer-name\");\n    }\n\n}"
  },
  {
    "path": "namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/ClientRequestProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.namesrv.processor;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.namesrv.NamesrvConfig;\nimport org.apache.rocketmq.namesrv.NamesrvController;\nimport org.apache.rocketmq.namesrv.routeinfo.RouteInfoManager;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.GetRouteInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.lang.reflect.Field;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.mockito.Mockito.doReturn;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ClientRequestProcessorTest {\n\n    @Mock\n    private NamesrvController namesrvController;\n\n    @Mock\n    private RouteInfoManager routeInfoManager;\n\n    @Mock\n    private NamesrvConfig namesrvConfig;\n\n    @Mock\n    private ChannelHandlerContext ctx;\n\n    private ClientRequestProcessor clientRequestProcessor;\n\n    @Before\n    public void setup() throws NoSuchFieldException, IllegalAccessException {\n        when(namesrvController.getRouteInfoManager()).thenReturn(routeInfoManager);\n        when(namesrvController.getNamesrvConfig()).thenReturn(namesrvConfig);\n\n        when(namesrvConfig.getWaitSecondsForService()).thenReturn(0);\n        when(namesrvConfig.isNeedWaitForService()).thenReturn(true);\n\n        clientRequestProcessor = new ClientRequestProcessor(namesrvController);\n\n        Field startupTimeMillisField = ClientRequestProcessor.class.getDeclaredField(\"startupTimeMillis\");\n        startupTimeMillisField.setAccessible(true);\n        startupTimeMillisField.set(clientRequestProcessor, System.currentTimeMillis() - 60000);\n    }\n\n    @Test\n    public void testGetRouteInfoByTopicWithHighVersionClient() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC, null);\n        request.setVersion(MQVersion.Version.V4_9_4.ordinal());\n\n        GetRouteInfoRequestHeader requestHeader = new GetRouteInfoRequestHeader();\n        requestHeader.setTopic(\"TestTopic\");\n\n        RemotingCommand spyRequest = spy(request);\n        doReturn(requestHeader).when(spyRequest).decodeCommandCustomHeader(GetRouteInfoRequestHeader.class);\n\n        TopicRouteData topicRouteData = createMockTopicRouteData();\n\n        when(routeInfoManager.pickupTopicRouteData(\"TestTopic\")).thenReturn(topicRouteData);\n\n        RemotingCommand response = clientRequestProcessor.getRouteInfoByTopic(ctx, spyRequest);\n\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n        assertNotNull(response.getBody());\n    }\n\n    @Test\n    public void testGetRouteInfoByTopicWithLowVersionClientAndNoStandardJsonFlag() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC, null);\n        request.setVersion(MQVersion.Version.V4_9_3.ordinal());\n\n        GetRouteInfoRequestHeader requestHeader = new GetRouteInfoRequestHeader();\n        requestHeader.setTopic(\"TestTopic\");\n        requestHeader.setAcceptStandardJsonOnly(false);\n\n        RemotingCommand spyRequest = spy(request);\n        doReturn(requestHeader).when(spyRequest).decodeCommandCustomHeader(GetRouteInfoRequestHeader.class);\n\n        TopicRouteData topicRouteData = createMockTopicRouteData();\n\n        when(routeInfoManager.pickupTopicRouteData(\"TestTopic\")).thenReturn(topicRouteData);\n\n        RemotingCommand response = clientRequestProcessor.getRouteInfoByTopic(ctx, spyRequest);\n\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n        assertNotNull(response.getBody());\n    }\n\n    @Test\n    public void testGetRouteInfoByTopicWithNameServerNotReady() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC, null);\n\n        GetRouteInfoRequestHeader requestHeader = new GetRouteInfoRequestHeader();\n        requestHeader.setTopic(\"TestTopic\");\n\n        RemotingCommand spyRequest = spy(request);\n        doReturn(requestHeader).when(spyRequest).decodeCommandCustomHeader(GetRouteInfoRequestHeader.class);\n\n        when(namesrvConfig.getWaitSecondsForService()).thenReturn(60);\n        when(namesrvConfig.isNeedWaitForService()).thenReturn(true);\n\n        try {\n            Field startupTimeMillisField = ClientRequestProcessor.class.getDeclaredField(\"startupTimeMillis\");\n            startupTimeMillisField.setAccessible(true);\n            startupTimeMillisField.set(clientRequestProcessor, System.currentTimeMillis());\n        } catch (Exception e) {\n            e.printStackTrace();\n        }\n\n        RemotingCommand response = clientRequestProcessor.getRouteInfoByTopic(ctx, spyRequest);\n\n        assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode());\n        assertEquals(\"name server not ready\", response.getRemark());\n    }\n\n    @Test\n    public void testGetRouteInfoByTopicWithTopicNotExist() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC, null);\n\n        GetRouteInfoRequestHeader requestHeader = new GetRouteInfoRequestHeader();\n        requestHeader.setTopic(\"NonExistentTopic\");\n\n        RemotingCommand spyRequest = spy(request);\n        doReturn(requestHeader).when(spyRequest).decodeCommandCustomHeader(GetRouteInfoRequestHeader.class);\n\n        when(routeInfoManager.pickupTopicRouteData(\"NonExistentTopic\")).thenReturn(null);\n\n        RemotingCommand response = clientRequestProcessor.getRouteInfoByTopic(ctx, spyRequest);\n\n        assertEquals(ResponseCode.TOPIC_NOT_EXIST, response.getCode());\n        assertNotNull(response.getRemark());\n    }\n\n    private TopicRouteData createMockTopicRouteData() {\n        TopicRouteData result = new TopicRouteData();\n\n        List<QueueData> queueDataList = new ArrayList<>();\n        QueueData queueData = new QueueData();\n        queueData.setBrokerName(\"broker-a\");\n        queueData.setReadQueueNums(4);\n        queueData.setWriteQueueNums(4);\n        queueData.setPerm(6);\n        queueData.setTopicSysFlag(0);\n        queueDataList.add(queueData);\n        result.setQueueDatas(queueDataList);\n\n        List<BrokerData> brokerDataList = new ArrayList<>();\n        BrokerData brokerData = new BrokerData();\n        brokerData.setBrokerName(\"broker-a\");\n        brokerData.setCluster(\"default-cluster\");\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(0L, \"127.0.0.1:10911\");\n        brokerData.setBrokerAddrs(brokerAddrs);\n        brokerDataList.add(brokerData);\n        result.setBrokerDatas(brokerDataList);\n\n        return result;\n    }\n}"
  },
  {
    "path": "namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/ClusterTestRequestProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.namesrv.processor;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport java.lang.reflect.Field;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.MQClientManager;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.namesrv.NamesrvConfig;\nimport org.apache.rocketmq.namesrv.NamesrvController;\nimport org.apache.rocketmq.namesrv.routeinfo.RouteInfoManager;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.GetRouteInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.apache.rocketmq.tools.admin.DefaultMQAdminExt;\nimport org.apache.rocketmq.tools.admin.DefaultMQAdminExtImpl;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\npublic class ClusterTestRequestProcessorTest {\n    private ClusterTestRequestProcessor clusterTestProcessor;\n    private DefaultMQAdminExtImpl defaultMQAdminExtImpl;\n    private MQClientInstance mqClientInstance = MQClientManager.getInstance()\n            .getOrCreateMQClientInstance(new ClientConfig());\n    private MQClientAPIImpl mQClientAPIImpl;\n    private ChannelHandlerContext ctx;\n\n    @Before\n    public void init() throws NoSuchFieldException, IllegalAccessException, RemotingException, MQClientException,\n            InterruptedException {\n        NamesrvController namesrvController = new NamesrvController(\n                new NamesrvConfig(),\n                new NettyServerConfig());\n\n        clusterTestProcessor = new ClusterTestRequestProcessor(namesrvController, \"default-producer\");\n        mQClientAPIImpl = mock(MQClientAPIImpl.class);\n        DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt();\n        defaultMQAdminExtImpl = new DefaultMQAdminExtImpl(defaultMQAdminExt, 1000);\n        ctx = mock(ChannelHandlerContext.class);\n\n        Field field = DefaultMQAdminExtImpl.class.getDeclaredField(\"mqClientInstance\");\n        field.setAccessible(true);\n        field.set(defaultMQAdminExtImpl, mqClientInstance);\n        field = MQClientInstance.class.getDeclaredField(\"mQClientAPIImpl\");\n        field.setAccessible(true);\n        field.set(mqClientInstance, mQClientAPIImpl);\n        field = ClusterTestRequestProcessor.class.getDeclaredField(\"adminExt\");\n        field.setAccessible(true);\n        field.set(clusterTestProcessor, defaultMQAdminExt);\n\n        TopicRouteData topicRouteData = new TopicRouteData();\n        List<BrokerData> brokerDatas = new ArrayList<>();\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(1234L, \"127.0.0.1:10911\");\n        BrokerData brokerData = new BrokerData();\n        brokerData.setCluster(\"default-cluster\");\n        brokerData.setBrokerName(\"default-broker\");\n        brokerData.setBrokerAddrs(brokerAddrs);\n        brokerDatas.add(brokerData);\n        topicRouteData.setBrokerDatas(brokerDatas);\n        when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(topicRouteData);\n    }\n\n    @After\n    public void terminate() {\n    }\n\n    @Test\n    public void testGetRouteInfoByTopic() throws RemotingCommandException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(12, new CommandCustomHeader() {\n            @Override\n            public void checkFields() throws RemotingCommandException {\n\n            }\n        });\n        RemotingCommand remoting = clusterTestProcessor.getRouteInfoByTopic(ctx, request);\n        assertThat(remoting.getCode()).isEqualTo(ResponseCode.TOPIC_NOT_EXIST);\n        assertThat(remoting.getBody()).isNull();\n        assertThat(remoting.getRemark()).isNotNull();\n    }\n\n    @Test\n    public void testNamesrvReady() throws Exception {\n        String topicName = \"rocketmq-topic-test-ready\";\n        RouteInfoManager routeInfoManager = mockRouteInfoManager();\n        NamesrvController namesrvController = mockNamesrvController(routeInfoManager, true, -1,true);\n        ClientRequestProcessor clientRequestProcessor = new ClientRequestProcessor(namesrvController);\n        GetRouteInfoRequestHeader routeInfoRequestHeader = mockRouteInfoRequestHeader(topicName);\n        RemotingCommand remotingCommand = mockTopicRouteCommand(routeInfoRequestHeader);\n        RemotingCommand response = clientRequestProcessor.processRequest(mock(ChannelHandlerContext.class),\n                remotingCommand);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testNamesrvNoNeedWaitForService() throws Exception {\n        String topicName = \"rocketmq-topic-test-ready\";\n        RouteInfoManager routeInfoManager = mockRouteInfoManager();\n        NamesrvController namesrvController = mockNamesrvController(routeInfoManager, true, 45,false);\n        ClientRequestProcessor clientRequestProcessor = new ClientRequestProcessor(namesrvController);\n        GetRouteInfoRequestHeader routeInfoRequestHeader = mockRouteInfoRequestHeader(topicName);\n        RemotingCommand remotingCommand = mockTopicRouteCommand(routeInfoRequestHeader);\n        RemotingCommand response = clientRequestProcessor.processRequest(mock(ChannelHandlerContext.class),\n            remotingCommand);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testNamesrvNotReady() throws Exception {\n        String topicName = \"rocketmq-topic-test\";\n        RouteInfoManager routeInfoManager = mockRouteInfoManager();\n        NamesrvController namesrvController = mockNamesrvController(routeInfoManager, false, 45,true);\n        GetRouteInfoRequestHeader routeInfoRequestHeader = mockRouteInfoRequestHeader(topicName);\n        RemotingCommand remotingCommand = mockTopicRouteCommand(routeInfoRequestHeader);\n        ClientRequestProcessor clientRequestProcessor = new ClientRequestProcessor(namesrvController);\n        RemotingCommand response = clientRequestProcessor.processRequest(mock(ChannelHandlerContext.class),\n                remotingCommand);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);\n    }\n\n    @Test\n    public void testNamesrv() throws Exception {\n        int waitSecondsForService = 3;\n        String topicName = \"rocketmq-topic-test\";\n        RouteInfoManager routeInfoManager = mockRouteInfoManager();\n        NamesrvController namesrvController = mockNamesrvController(routeInfoManager, false, waitSecondsForService,true);\n        GetRouteInfoRequestHeader routeInfoRequestHeader = mockRouteInfoRequestHeader(topicName);\n        RemotingCommand remotingCommand = mockTopicRouteCommand(routeInfoRequestHeader);\n        ClientRequestProcessor clientRequestProcessor = new ClientRequestProcessor(namesrvController);\n        RemotingCommand response = clientRequestProcessor.processRequest(mock(ChannelHandlerContext.class),\n                remotingCommand);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);\n        TimeUnit.SECONDS.sleep(waitSecondsForService + 1);\n        response = clientRequestProcessor.processRequest(mock(ChannelHandlerContext.class), remotingCommand);\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    private RemotingCommand mockTopicRouteCommand(\n            GetRouteInfoRequestHeader routeInfoRequestHeader) throws RemotingCommandException {\n        RemotingCommand remotingCommand = mock(RemotingCommand.class);\n        when(remotingCommand.decodeCommandCustomHeader(any())).thenReturn(routeInfoRequestHeader);\n        when(remotingCommand.getCode()).thenReturn(RequestCode.GET_ROUTEINFO_BY_TOPIC);\n        return remotingCommand;\n    }\n\n    public NamesrvController mockNamesrvController(RouteInfoManager routeInfoManager, boolean ready,\n            int waitSecondsForService,boolean needWaitForService) {\n        NamesrvConfig namesrvConfig = mock(NamesrvConfig.class);\n        when(namesrvConfig.isNeedWaitForService()).thenReturn(needWaitForService);\n        when(namesrvConfig.getUnRegisterBrokerQueueCapacity()).thenReturn(10);\n        when(namesrvConfig.getWaitSecondsForService()).thenReturn(ready ? 0 : waitSecondsForService);\n        NamesrvController namesrvController = mock(NamesrvController.class);\n        when(namesrvController.getNamesrvConfig()).thenReturn(namesrvConfig);\n        when(namesrvController.getRouteInfoManager()).thenReturn(routeInfoManager);\n\n        return namesrvController;\n    }\n\n    public RouteInfoManager mockRouteInfoManager() {\n        RouteInfoManager routeInfoManager = mock(RouteInfoManager.class);\n        TopicRouteData topicRouteData = mock(TopicRouteData.class);\n        when(routeInfoManager.pickupTopicRouteData(any())).thenReturn(topicRouteData);\n        return routeInfoManager;\n    }\n\n    public GetRouteInfoRequestHeader mockRouteInfoRequestHeader(String topicName) {\n        GetRouteInfoRequestHeader routeInfoRequestHeader = mock(GetRouteInfoRequestHeader.class);\n        when(routeInfoRequestHeader.getTopic()).thenReturn(topicName);\n        return routeInfoRequestHeader;\n    }\n\n}\n"
  },
  {
    "path": "namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/RequestProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.namesrv.processor;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.lang.reflect.Field;\nimport java.lang.reflect.Modifier;\nimport java.net.InetSocketAddress;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.Properties;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.namesrv.NamesrvConfig;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.namesrv.NamesrvController;\nimport org.apache.rocketmq.namesrv.routeinfo.RouteInfoManager;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.RegisterBrokerBody;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.DeleteKVConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.GetKVConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.GetKVConfigResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.PutKVConfigRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.namesrv.RegisterBrokerResult;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.assertj.core.util.Maps;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\npublic class RequestProcessorTest {\n    private DefaultRequestProcessor defaultRequestProcessor;\n\n    private ClientRequestProcessor clientRequestProcessor;\n\n    private NamesrvController namesrvController;\n\n    private NamesrvConfig namesrvConfig;\n\n    private NettyServerConfig nettyServerConfig;\n\n    private RouteInfoManager routeInfoManager;\n\n    private Logger logger;\n\n    @Before\n    public void init() throws Exception {\n        namesrvConfig = new NamesrvConfig();\n        namesrvConfig.setEnableAllTopicList(true);\n        nettyServerConfig = new NettyServerConfig();\n        routeInfoManager = new RouteInfoManager(new NamesrvConfig(), null);\n\n        namesrvController = new NamesrvController(namesrvConfig, nettyServerConfig);\n\n        Field field = NamesrvController.class.getDeclaredField(\"routeInfoManager\");\n        field.setAccessible(true);\n        field.set(namesrvController, routeInfoManager);\n        defaultRequestProcessor = new DefaultRequestProcessor(namesrvController);\n\n        clientRequestProcessor = new ClientRequestProcessor(namesrvController);\n\n        registerRouteInfoManager();\n\n        logger = mock(Logger.class);\n        setFinalStatic(DefaultRequestProcessor.class.getDeclaredField(\"log\"), logger);\n    }\n\n    @Test\n    public void testProcessRequest_PutKVConfig() throws RemotingCommandException {\n        PutKVConfigRequestHeader header = new PutKVConfigRequestHeader();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PUT_KV_CONFIG,\n            header);\n        request.addExtField(\"namespace\", \"namespace\");\n        request.addExtField(\"key\", \"key\");\n        request.addExtField(\"value\", \"value\");\n\n        RemotingCommand response = defaultRequestProcessor.processRequest(null, request);\n\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(response.getRemark()).isNull();\n\n        assertThat(namesrvController.getKvConfigManager().getKVConfig(\"namespace\", \"key\"))\n            .isEqualTo(\"value\");\n    }\n\n    @Test\n    public void testProcessRequest_GetKVConfigReturnNotNull() throws RemotingCommandException {\n        namesrvController.getKvConfigManager().putKVConfig(\"namespace\", \"key\", \"value\");\n\n        GetKVConfigRequestHeader header = new GetKVConfigRequestHeader();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_KV_CONFIG,\n            header);\n        request.addExtField(\"namespace\", \"namespace\");\n        request.addExtField(\"key\", \"key\");\n\n        RemotingCommand response = defaultRequestProcessor.processRequest(null, request);\n\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(response.getRemark()).isNull();\n\n        GetKVConfigResponseHeader responseHeader = (GetKVConfigResponseHeader) response\n            .readCustomHeader();\n\n        assertThat(responseHeader.getValue()).isEqualTo(\"value\");\n    }\n\n    @Test\n    public void testProcessRequest_GetKVConfigReturnNull() throws RemotingCommandException {\n        GetKVConfigRequestHeader header = new GetKVConfigRequestHeader();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_KV_CONFIG,\n            header);\n        request.addExtField(\"namespace\", \"namespace\");\n        request.addExtField(\"key\", \"key\");\n\n        RemotingCommand response = defaultRequestProcessor.processRequest(null, request);\n\n        assertThat(response.getCode()).isEqualTo(ResponseCode.QUERY_NOT_FOUND);\n        assertThat(response.getRemark()).isEqualTo(\"No config item, Namespace: namespace Key: key\");\n\n        GetKVConfigResponseHeader responseHeader = (GetKVConfigResponseHeader) response\n            .readCustomHeader();\n\n        assertThat(responseHeader.getValue()).isNull();\n    }\n\n    @Test\n    public void testProcessRequest_DeleteKVConfig() throws RemotingCommandException {\n        namesrvController.getKvConfigManager().putKVConfig(\"namespace\", \"key\", \"value\");\n\n        DeleteKVConfigRequestHeader header = new DeleteKVConfigRequestHeader();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_KV_CONFIG,\n            header);\n        request.addExtField(\"namespace\", \"namespace\");\n        request.addExtField(\"key\", \"key\");\n\n        RemotingCommand response = defaultRequestProcessor.processRequest(null, request);\n\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(response.getRemark()).isNull();\n\n        assertThat(namesrvController.getKvConfigManager().getKVConfig(\"namespace\", \"key\"))\n            .isNull();\n    }\n\n    @Test\n    public void testProcessRequest_UnSupportedRequest() throws RemotingCommandException {\n        final RemotingCommand unSupportedRequest = RemotingCommand.createRequestCommand(99999, null);\n        final RemotingCommand response = defaultRequestProcessor.processRequest(null, unSupportedRequest);\n\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.REQUEST_CODE_NOT_SUPPORTED);\n    }\n\n    @Test\n    public void testProcessRequest_UpdateConfigPath() throws RemotingCommandException {\n        final RemotingCommand updateConfigRequest = RemotingCommand.createRequestCommand(RequestCode.UPDATE_NAMESRV_CONFIG, null);\n        Properties properties = new Properties();\n\n        // Update allowed value\n        properties.setProperty(\"enableTopicList\", \"true\");\n        updateConfigRequest.setBody(MixAll.properties2String(properties).getBytes(StandardCharsets.UTF_8));\n\n        RemotingCommand response = defaultRequestProcessor.processRequest(null, updateConfigRequest);\n\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n        //update disallowed value\n        properties.clear();\n        properties.setProperty(\"configStorePath\", \"test/path\");\n        updateConfigRequest.setBody(MixAll.properties2String(properties).getBytes(StandardCharsets.UTF_8));\n\n        response = defaultRequestProcessor.processRequest(null, updateConfigRequest);\n\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);\n        assertThat(response.getRemark()).contains(\"Can not update config in black list.\");\n\n        //update disallowed values\n        properties.clear();\n        properties.setProperty(\"kvConfigPath\", \"test/path\");\n        updateConfigRequest.setBody(MixAll.properties2String(properties).getBytes(StandardCharsets.UTF_8));\n\n        response = defaultRequestProcessor.processRequest(null, updateConfigRequest);\n\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);\n        assertThat(response.getRemark()).contains(\"Can not update config in black list\");\n\n        //update disallowed values\n        properties.clear();\n        properties.setProperty(\"configBlackList\", \"test;path\");\n        updateConfigRequest.setBody(MixAll.properties2String(properties).getBytes(StandardCharsets.UTF_8));\n\n        response = defaultRequestProcessor.processRequest(null, updateConfigRequest);\n\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);\n        assertThat(response.getRemark()).contains(\"Can not update config in black list\");\n    }\n\n    @Test\n    public void testProcessRequest_RegisterBroker() throws RemotingCommandException,\n        NoSuchFieldException, IllegalAccessException {\n        RemotingCommand request = genSampleRegisterCmd(true);\n\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        when(ctx.channel()).thenReturn(null);\n\n        RemotingCommand response = defaultRequestProcessor.processRequest(ctx, request);\n\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(response.getRemark()).isNull();\n\n        RouteInfoManager routes = namesrvController.getRouteInfoManager();\n        Field brokerAddrTable = RouteInfoManager.class.getDeclaredField(\"brokerAddrTable\");\n        brokerAddrTable.setAccessible(true);\n\n        BrokerData broker = new BrokerData();\n        broker.setBrokerName(\"broker\");\n        broker.setBrokerAddrs((HashMap) Maps.newHashMap(new Long(2333), \"10.10.1.1\"));\n\n        assertThat((Map) brokerAddrTable.get(routes))\n            .contains(new HashMap.SimpleEntry(\"broker\", broker));\n    }\n\n    /*@Test\n    public void testProcessRequest_RegisterBrokerLogicalQueue() throws Exception {\n        String cluster = \"cluster\";\n        String broker1Name = \"broker1\";\n        String broker1Addr = \"10.10.1.1\";\n        String broker2Name = \"broker2\";\n        String broker2Addr = \"10.10.1.2\";\n        String topic = \"foobar\";\n\n        LogicalQueueRouteData queueRouteData1 = new LogicalQueueRouteData(0, 0, new MessageQueue(topic, broker1Name, 0), MessageQueueRouteState.ReadOnly, 0, 10, 100, 100, broker1Addr);\n        {\n            RegisterBrokerRequestHeader header = new RegisterBrokerRequestHeader();\n            header.setBrokerName(broker1Name);\n            RemotingCommand request = RemotingCommand.createRequestCommand(\n                RequestCode.REGISTER_BROKER, header);\n            request.addExtField(\"brokerName\", broker1Name);\n            request.addExtField(\"brokerAddr\", broker1Addr);\n            request.addExtField(\"clusterName\", cluster);\n            request.addExtField(\"haServerAddr\", \"10.10.2.1\");\n            request.addExtField(\"brokerId\", String.valueOf(MixAll.MASTER_ID));\n            request.setVersion(MQVersion.CURRENT_VERSION);\n            TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();\n            topicConfigSerializeWrapper.setTopicConfigTable(new ConcurrentHashMap<>(Collections.singletonMap(topic, new TopicConfig(topic))));\n            topicConfigSerializeWrapper.setLogicalQueuesInfoMap(Maps.newHashMap(topic, new LogicalQueuesInfo(Collections.singletonMap(0, Lists.newArrayList(\n                queueRouteData1\n            )))));\n            topicConfigSerializeWrapper.setDataVersion(new DataVersion());\n            RegisterBrokerBody requestBody = new RegisterBrokerBody();\n            requestBody.setTopicConfigSerializeWrapper(topicConfigSerializeWrapper);\n            requestBody.setFilterServerList(Lists.<String>newArrayList());\n            request.setBody(requestBody.encode());\n\n            ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n            when(ctx.channel()).thenReturn(null);\n\n            RemotingCommand response = defaultRequestProcessor.processRequest(ctx, request);\n\n            assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n            assertThat(response.getRemark()).isNull();\n        }\n        LogicalQueueRouteData queueRouteData2 = new LogicalQueueRouteData(0, 100, new MessageQueue(topic, broker2Name, 0), MessageQueueRouteState.Normal, 0, -1, -1, -1, broker2Addr);\n        LogicalQueueRouteData queueRouteData3 = new LogicalQueueRouteData(1, 100, new MessageQueue(topic, broker2Name, 0), MessageQueueRouteState.Normal, 0, -1, -1, -1, broker2Addr);\n        {\n            RegisterBrokerRequestHeader header = new RegisterBrokerRequestHeader();\n            header.setBrokerName(broker2Name);\n            RemotingCommand request = RemotingCommand.createRequestCommand(\n                RequestCode.REGISTER_BROKER, header);\n            request.addExtField(\"brokerName\", broker2Name);\n            request.addExtField(\"brokerAddr\", broker2Addr);\n            request.addExtField(\"clusterName\", cluster);\n            request.addExtField(\"haServerAddr\", \"10.10.2.1\");\n            request.addExtField(\"brokerId\", String.valueOf(MixAll.MASTER_ID));\n            request.setVersion(MQVersion.CURRENT_VERSION);\n            TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();\n            topicConfigSerializeWrapper.setTopicConfigTable(new ConcurrentHashMap<>(Collections.singletonMap(topic, new TopicConfig(topic))));\n            topicConfigSerializeWrapper.setLogicalQueuesInfoMap(Maps.newHashMap(topic, new LogicalQueuesInfo(ImmutableMap.of(\n                0, Collections.singletonList(queueRouteData2),\n                1, Collections.singletonList(queueRouteData3)\n            ))));\n            topicConfigSerializeWrapper.setDataVersion(new DataVersion());\n            RegisterBrokerBody requestBody = new RegisterBrokerBody();\n            requestBody.setTopicConfigSerializeWrapper(topicConfigSerializeWrapper);\n            requestBody.setFilterServerList(Lists.<String>newArrayList());\n            request.setBody(requestBody.encode());\n\n            ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n            when(ctx.channel()).thenReturn(null);\n\n            RemotingCommand response = defaultRequestProcessor.processRequest(ctx, request);\n\n            assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n            assertThat(response.getRemark()).isNull();\n        }\n\n        {\n            GetRouteInfoRequestHeader header = new GetRouteInfoRequestHeader();\n            header.setTopic(topic);\n            header.setSysFlag(MessageSysFlag.LOGICAL_QUEUE_FLAG);\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC, header);\n            request.makeCustomHeaderToNet();\n\n            ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n            when(ctx.channel()).thenReturn(null);\n\n            RemotingCommand response = defaultRequestProcessor.processRequest(ctx, request);\n\n            assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n\n            TopicRouteDataNameSrv topicRouteDataNameSrv = JSON.parseObject(response.getBody(), TopicRouteDataNameSrv.class);\n            assertThat(topicRouteDataNameSrv).isNotNull();\n            LogicalQueuesInfoUnordered logicalQueuesInfoUnordered = new LogicalQueuesInfoUnordered();\n            logicalQueuesInfoUnordered.put(0, ImmutableMap.of(\n                new LogicalQueuesInfoUnordered.Key(queueRouteData1.getBrokerName(), queueRouteData1.getQueueId(), queueRouteData1.getOffsetDelta()), queueRouteData1,\n                new LogicalQueuesInfoUnordered.Key(queueRouteData2.getBrokerName(), queueRouteData2.getQueueId(), queueRouteData2.getOffsetDelta()), queueRouteData2\n            ));\n            logicalQueuesInfoUnordered.put(1, ImmutableMap.of(new LogicalQueuesInfoUnordered.Key(queueRouteData3.getBrokerName(), queueRouteData3.getQueueId(), queueRouteData3.getOffsetDelta()), queueRouteData3));\n            assertThat(topicRouteDataNameSrv.getLogicalQueuesInfoUnordered()).isEqualTo(logicalQueuesInfoUnordered);\n        }\n    }\n*/\n    @Test\n    public void testProcessRequest_RegisterBrokerWithFilterServer() throws RemotingCommandException,\n        NoSuchFieldException, IllegalAccessException {\n        RemotingCommand request = genSampleRegisterCmd(true);\n\n        // version >= MQVersion.Version.V3_0_11.ordinal() to register with filter server\n        request.setVersion(100);\n\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        when(ctx.channel()).thenReturn(null);\n\n        RemotingCommand response = defaultRequestProcessor.processRequest(ctx, request);\n\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(response.getRemark()).isNull();\n\n        RouteInfoManager routes = namesrvController.getRouteInfoManager();\n        Field brokerAddrTable = RouteInfoManager.class.getDeclaredField(\"brokerAddrTable\");\n        brokerAddrTable.setAccessible(true);\n\n        BrokerData broker = new BrokerData();\n        broker.setBrokerName(\"broker\");\n        broker.setBrokerAddrs((HashMap) Maps.newHashMap(new Long(2333), \"10.10.1.1\"));\n\n        assertThat((Map) brokerAddrTable.get(routes))\n            .contains(new HashMap.SimpleEntry(\"broker\", broker));\n    }\n\n    @Test\n    public void testProcessRequest_UnregisterBroker() throws RemotingCommandException, NoSuchFieldException, IllegalAccessException {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        when(ctx.channel()).thenReturn(null);\n\n        //Register broker\n        RemotingCommand regRequest = genSampleRegisterCmd(true);\n        defaultRequestProcessor.processRequest(ctx, regRequest);\n\n        //Unregister broker\n        RemotingCommand unregRequest = genSampleRegisterCmd(false);\n        RemotingCommand unregResponse = defaultRequestProcessor.processRequest(ctx, unregRequest);\n\n        assertThat(unregResponse.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(unregResponse.getRemark()).isNull();\n\n        RouteInfoManager routes = namesrvController.getRouteInfoManager();\n        Field brokerAddrTable = RouteInfoManager.class.getDeclaredField(\"brokerAddrTable\");\n        brokerAddrTable.setAccessible(true);\n\n        assertThat((Map) brokerAddrTable.get(routes)).isNotEmpty();\n    }\n\n    @Test\n    public void testGetAllTopicList() throws RemotingCommandException {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        Channel channel = mock(Channel.class);\n        when(channel.remoteAddress()).thenReturn(null);\n        when(ctx.channel()).thenReturn(channel);\n\n        namesrvController.getNamesrvConfig().setEnableAllTopicList(true);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_TOPIC_LIST_FROM_NAMESERVER, null);\n\n        RemotingCommand response = defaultRequestProcessor.processRequest(ctx, request);\n\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(response.getRemark()).isNull();\n\n        namesrvController.getNamesrvConfig().setEnableAllTopicList(false);\n\n        response = defaultRequestProcessor.processRequest(ctx, request);\n\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);\n    }\n\n    @Test\n    public void testGetRouteInfoByTopic() throws Exception {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        when(ctx.channel()).thenReturn(null);\n        RemotingCommand request = getRemotingCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC);\n        RemotingCommand remotingCommandSuccess = clientRequestProcessor.processRequest(ctx, request);\n        assertThat(remotingCommandSuccess.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        request.getExtFields().put(\"topic\", \"test\");\n        RemotingCommand remotingCommandNoTopicRouteInfo = clientRequestProcessor.processRequest(ctx, request);\n        assertThat(remotingCommandNoTopicRouteInfo.getCode()).isEqualTo(ResponseCode.TOPIC_NOT_EXIST);\n    }\n\n    @Test\n    public void testGetBrokerClusterInfo() throws RemotingCommandException {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        when(ctx.channel()).thenReturn(null);\n        RemotingCommand request = getRemotingCommand(RequestCode.GET_BROKER_CLUSTER_INFO);\n        RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request);\n        assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testQueryDataVersion()throws RemotingCommandException {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        when(ctx.channel()).thenReturn(null);\n        RemotingCommand request = getRemotingCommand(RequestCode.QUERY_DATA_VERSION);\n        RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request);\n        assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetBrokerMemberBroker() throws RemotingCommandException {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        when(ctx.channel()).thenReturn(null);\n        RemotingCommand request = getRemotingCommand(RequestCode.GET_BROKER_MEMBER_GROUP);\n        RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request);\n        assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testBrokerHeartBeat() throws RemotingCommandException {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        when(ctx.channel()).thenReturn(null);\n        RemotingCommand request = getRemotingCommand(RequestCode.BROKER_HEARTBEAT);\n        RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request);\n        assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testAddWritePermOfBroker() throws RemotingCommandException {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        when(ctx.channel()).thenReturn(null);\n        RemotingCommand request = getRemotingCommand(RequestCode.ADD_WRITE_PERM_OF_BROKER);\n        RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request);\n        assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testWipeWritePermOfBroker() throws RemotingCommandException {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        when(ctx.channel()).thenReturn(null);\n        RemotingCommand request = getRemotingCommand(RequestCode.WIPE_WRITE_PERM_OF_BROKER);\n        RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request);\n        assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetAllTopicListFromNameserver() throws RemotingCommandException {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        when(ctx.channel()).thenReturn(mock(Channel.class));\n        when(ctx.channel().remoteAddress()).thenReturn(new InetSocketAddress(123));\n        RemotingCommand request = getRemotingCommand(RequestCode.GET_ALL_TOPIC_LIST_FROM_NAMESERVER);\n        RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request);\n        assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testDeleteTopicInNamesrv() throws RemotingCommandException {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        when(ctx.channel()).thenReturn(null);\n        RemotingCommand request = getRemotingCommand(RequestCode.DELETE_TOPIC_IN_NAMESRV);\n        RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request);\n        assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetKVListByNamespace() throws RemotingCommandException {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        when(ctx.channel()).thenReturn(null);\n        RemotingCommand request = getRemotingCommand(RequestCode.GET_KVLIST_BY_NAMESPACE);\n        request.addExtField(\"namespace\", \"default-namespace-1\");\n        RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request);\n        assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.QUERY_NOT_FOUND);\n        namesrvController.getKvConfigManager().putKVConfig(\"default-namespace-1\", \"key\", \"value\");\n        RemotingCommand remotingCommandSuccess = defaultRequestProcessor.processRequest(ctx, request);\n        assertThat(remotingCommandSuccess.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetTopicsByCluster() throws RemotingCommandException {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        when(ctx.channel()).thenReturn(null);\n        RemotingCommand request = getRemotingCommand(RequestCode.GET_TOPICS_BY_CLUSTER);\n        request.addExtField(\"cluster\", \"default-cluster\");\n        RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request);\n        assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetSystemTopicListFromNs() throws RemotingCommandException {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        when(ctx.channel()).thenReturn(null);\n        RemotingCommand request = getRemotingCommand(RequestCode.GET_SYSTEM_TOPIC_LIST_FROM_NS);\n        request.addExtField(\"cluster\", \"default-cluster\");\n        RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request);\n        assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetUnitTopicList() throws RemotingCommandException {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        when(ctx.channel()).thenReturn(null);\n        RemotingCommand request = getRemotingCommand(RequestCode.GET_UNIT_TOPIC_LIST);\n        request.addExtField(\"cluster\", \"default-cluster\");\n        RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request);\n        assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetHasUnitSubTopicList() throws RemotingCommandException {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        when(ctx.channel()).thenReturn(null);\n        RemotingCommand request = getRemotingCommand(RequestCode.GET_HAS_UNIT_SUB_TOPIC_LIST);\n        request.addExtField(\"cluster\", \"default-cluster\");\n        RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request);\n        assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetHasUnitSubUnUnitTopicList() throws RemotingCommandException {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        when(ctx.channel()).thenReturn(null);\n        RemotingCommand request = getRemotingCommand(RequestCode.GET_HAS_UNIT_SUB_UNUNIT_TOPIC_LIST);\n        request.addExtField(\"cluster\", \"default-cluster\");\n        RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request);\n        assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testUpdateConfig() throws RemotingCommandException {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        when(ctx.channel()).thenReturn(null);\n        RemotingCommand request = getRemotingCommand(RequestCode.UPDATE_NAMESRV_CONFIG);\n        request.addExtField(\"cluster\", \"default-cluster\");\n        Map<String, String> propertiesMap = new HashMap<>();\n        propertiesMap.put(\"key\", \"value\");\n        request.setBody(propertiesMap.toString().getBytes());\n        RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request);\n        assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testGetConfig() throws RemotingCommandException {\n        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);\n        when(ctx.channel()).thenReturn(null);\n        RemotingCommand request = getRemotingCommand(RequestCode.GET_NAMESRV_CONFIG);\n        request.addExtField(\"cluster\", \"default-cluster\");\n        RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request);\n        assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    private RemotingCommand getRemotingCommand(int code) {\n        RegisterBrokerRequestHeader header = new RegisterBrokerRequestHeader();\n        header.setBrokerName(\"broker\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(code, header);\n        request.addExtField(\"brokerName\", \"broker\");\n        request.addExtField(\"brokerAddr\", \"10.10.1.1\");\n        request.addExtField(\"clusterName\", \"cluster\");\n        request.addExtField(\"haServerAddr\", \"10.10.2.1\");\n        request.addExtField(\"brokerId\", \"2333\");\n        request.addExtField(\"topic\", \"unit-test0\");\n        return request;\n    }\n\n    private static RemotingCommand genSampleRegisterCmd(boolean reg) {\n        RegisterBrokerRequestHeader header = new RegisterBrokerRequestHeader();\n        byte[] body = null;\n        if (reg) {\n            TopicConfigAndMappingSerializeWrapper topicConfigWrapper = new TopicConfigAndMappingSerializeWrapper();\n            topicConfigWrapper.getTopicConfigTable().put(\"unit-test1\", new TopicConfig());\n            topicConfigWrapper.getTopicConfigTable().put(\"unit-test2\", new TopicConfig());\n            RegisterBrokerBody requestBody = new RegisterBrokerBody();\n            requestBody.setTopicConfigSerializeWrapper(topicConfigWrapper);\n            body = requestBody.encode(false);\n            final int bodyCrc32 = UtilAll.crc32(body);\n            header.setBodyCrc32(bodyCrc32);\n        }\n        header.setBrokerName(\"broker\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(\n            reg ? RequestCode.REGISTER_BROKER : RequestCode.UNREGISTER_BROKER, header);\n        request.addExtField(\"brokerName\", \"broker\");\n        request.addExtField(\"brokerAddr\", \"10.10.1.1\");\n        request.addExtField(\"clusterName\", \"cluster\");\n        request.addExtField(\"haServerAddr\", \"10.10.2.1\");\n        request.addExtField(\"brokerId\", \"2333\");\n        request.setBody(body);\n        return request;\n    }\n\n    private static void setFinalStatic(Field field, Object newValue) throws Exception {\n        field.setAccessible(true);\n        Field modifiersField = Field.class.getDeclaredField(\"modifiers\");\n        modifiersField.setAccessible(true);\n        modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);\n        field.set(null, newValue);\n    }\n\n    private void registerRouteInfoManager() {\n        TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();\n        ConcurrentHashMap<String, TopicConfig> topicConfigConcurrentHashMap = new ConcurrentHashMap<>();\n        for (int i = 0; i < 2; i++) {\n            TopicConfig topicConfig = new TopicConfig();\n            topicConfig.setWriteQueueNums(8);\n            topicConfig.setTopicName(\"unit-test\" + i);\n            topicConfig.setPerm(6);\n            topicConfig.setReadQueueNums(8);\n            topicConfig.setOrder(false);\n            topicConfigConcurrentHashMap.put(topicConfig.getTopicName(), topicConfig);\n        }\n        topicConfigSerializeWrapper.setTopicConfigTable(topicConfigConcurrentHashMap);\n        Channel channel = mock(Channel.class);\n        RegisterBrokerResult registerBrokerResult = routeInfoManager.registerBroker(\"default-cluster\", \"127.0.0.1:10911\", \"default-broker\", 1234, \"127.0.0.1:1001\", \"\",\n            null, topicConfigSerializeWrapper, new ArrayList<>(), channel);\n\n    }\n\n}\n"
  },
  {
    "path": "namesrv/src/test/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHookMoreTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.namesrv.route;\n\n\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNull;\n\npublic class ZoneRouteRPCHookMoreTest {\n\n    private ZoneRouteRPCHook zoneRouteRPCHook;\n\n    @Before\n    public void setUp() {\n        zoneRouteRPCHook = new ZoneRouteRPCHook();\n    }\n\n    @Test\n    public void testFilterByZoneName_ValidInput_ShouldFilterCorrectly() {\n        // Arrange\n        TopicRouteData topicRouteData = new TopicRouteData();\n        topicRouteData.setBrokerDatas(generateBrokerDataList());\n        topicRouteData.setQueueDatas(generateQueueDataList());\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC,null);\n        request.setExtFields(createExtFields(\"true\",\"ZoneA\"));\n\n        RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"remark\");\n\n        // Act\n        zoneRouteRPCHook.doAfterResponse(\"127.0.0.1\", request, response);\n        TopicRouteData decodedResponse = RemotingSerializable.decode(response.getBody(), TopicRouteData.class);\n\n        // Assert\n        assertNull(decodedResponse);\n    }\n\n    @Test\n    public void testFilterByZoneName_NoZoneName_ShouldNotFilter() {\n        // Arrange\n        TopicRouteData topicRouteData = new TopicRouteData();\n        topicRouteData.setBrokerDatas(generateBrokerDataList());\n        topicRouteData.setQueueDatas(generateQueueDataList());\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC,null);\n        HashMap<String, String> extFields = new HashMap<>();\n        extFields.put(MixAll.ZONE_MODE, \"true\");\n        request.setExtFields(extFields);\n\n        RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, null);\n\n        // Act\n        zoneRouteRPCHook.doAfterResponse(\"127.0.0.1\", request, response);\n        TopicRouteData decodedResponse = RemotingSerializable.decode(response.getBody(), TopicRouteData.class);\n\n        // Assert\n        assertEquals(topicRouteData.getBrokerDatas().size(), 2);\n        assertEquals(topicRouteData.getQueueDatas().size(), 2);\n    }\n\n    @Test\n    public void testFilterByZoneName_ZoneModeFalse_ShouldNotFilter() {\n        // Arrange\n        TopicRouteData topicRouteData = new TopicRouteData();\n        topicRouteData.setBrokerDatas(generateBrokerDataList());\n        topicRouteData.setQueueDatas(generateQueueDataList());\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC,null);\n        request.setExtFields(createExtFields(\"false\",\"ZoneA\"));\n\n        RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS ,null);\n\n        // Act\n        zoneRouteRPCHook.doAfterResponse(\"127.0.0.1\", request, response);\n        TopicRouteData decodedResponse = RemotingSerializable.decode(response.getBody(), TopicRouteData.class);\n\n        // Assert\n        assertEquals(topicRouteData.getBrokerDatas().size(), 2);\n        assertEquals(topicRouteData.getQueueDatas().size(), 2);\n    }\n\n    private List<BrokerData> generateBrokerDataList() {\n        List<BrokerData> brokerDataList = new ArrayList<>();\n        BrokerData brokerData1 = new BrokerData();\n        brokerData1.setBrokerName(\"BrokerA\");\n        brokerData1.setZoneName(\"ZoneA\");\n        Map<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(MixAll.MASTER_ID, \"127.0.0.1:10911\");\n        brokerData1.setBrokerAddrs((HashMap<Long, String>) brokerAddrs);\n        brokerDataList.add(brokerData1);\n\n        BrokerData brokerData2 = new BrokerData();\n        brokerData2.setBrokerName(\"BrokerB\");\n        brokerData2.setZoneName(\"ZoneB\");\n        brokerAddrs = new HashMap<>();\n        brokerAddrs.put(MixAll.MASTER_ID, \"127.0.0.1:10912\");\n        brokerData2.setBrokerAddrs((HashMap<Long, String>) brokerAddrs);\n        brokerDataList.add(brokerData2);\n\n        return brokerDataList;\n    }\n\n    private List<QueueData> generateQueueDataList() {\n        List<QueueData> queueDataList = new ArrayList<>();\n        QueueData queueData1 = new QueueData();\n        queueData1.setBrokerName(\"BrokerA\");\n        queueDataList.add(queueData1);\n\n        QueueData queueData2 = new QueueData();\n        queueData2.setBrokerName(\"BrokerB\");\n        queueDataList.add(queueData2);\n\n        return queueDataList;\n    }\n\n    private HashMap<String, String> createExtFields(String zoneMode, String zoneName) {\n        HashMap<String, String> extFields = new HashMap<>();\n        extFields.put(MixAll.ZONE_MODE, zoneMode);\n        extFields.put(MixAll.ZONE_NAME, zoneName);\n        return extFields;\n    }\n\n}"
  },
  {
    "path": "namesrv/src/test/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHookTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.namesrv.route;\n\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\n\n\npublic class ZoneRouteRPCHookTest {\n\n    private ZoneRouteRPCHook zoneRouteRPCHook;\n\n    @Before\n    public void setup() {\n        zoneRouteRPCHook = new ZoneRouteRPCHook();\n    }\n\n    @Test\n    public void testDoAfterResponseWithNoZoneMode() {\n        RemotingCommand request1 = RemotingCommand.createRequestCommand(106,null);\n        zoneRouteRPCHook.doAfterResponse(\"\", request1, null);\n\n        HashMap<String, String> extFields = new HashMap<>();\n        extFields.put(MixAll.ZONE_MODE, \"false\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(105,null);\n        request.setExtFields(extFields);\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setBody(RemotingSerializable.encode(createSampleTopicRouteData()));\n        zoneRouteRPCHook.doAfterResponse(\"\", request, response);\n    }\n\n    @Test\n    public void testDoAfterResponseWithNoZoneName() {\n        HashMap<String, String> extFields = new HashMap<>();\n        extFields.put(MixAll.ZONE_MODE, \"true\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(105,null);\n        request.setExtFields(extFields);\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setBody(RemotingSerializable.encode(createSampleTopicRouteData()));\n        zoneRouteRPCHook.doAfterResponse(\"\", request, response);\n    }\n\n    @Test\n    public void testDoAfterResponseWithNoResponse() {\n        HashMap<String, String> extFields = new HashMap<>();\n        extFields.put(MixAll.ZONE_MODE, \"true\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(105,null);\n        request.setExtFields(extFields);\n        zoneRouteRPCHook.doAfterResponse(\"\", request, null);\n\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        zoneRouteRPCHook.doAfterResponse(\"\", request, response);\n\n        response.setBody(RemotingSerializable.encode(createSampleTopicRouteData()));\n        response.setCode(ResponseCode.NO_PERMISSION);\n        zoneRouteRPCHook.doAfterResponse(\"\", request, response);\n    }\n\n\n    @Test\n    public void testDoAfterResponseWithValidZoneFiltering() throws Exception {\n        HashMap<String, String> extFields = new HashMap<>();\n        extFields.put(MixAll.ZONE_MODE, \"true\");\n        extFields.put(MixAll.ZONE_NAME,\"zone1\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(105,null);\n        request.setExtFields(extFields);\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        TopicRouteData topicRouteData = createSampleTopicRouteData();\n        response.setBody(RemotingSerializable.encode(topicRouteData));\n        zoneRouteRPCHook.doAfterResponse(\"\", request, response);\n\n        HashMap<Long,String> brokeraddrs = new HashMap<>();\n        brokeraddrs.put(MixAll.MASTER_ID,\"127.0.0.1:10911\");\n        topicRouteData.getBrokerDatas().get(0).setBrokerAddrs(brokeraddrs);\n        response.setBody(RemotingSerializable.encode(topicRouteData));\n        zoneRouteRPCHook.doAfterResponse(\"\", request, response);\n\n        topicRouteData.getQueueDatas().add(createQueueData(\"BrokerB\"));\n        HashMap<Long,String> brokeraddrsB = new HashMap<>();\n        brokeraddrsB.put(MixAll.MASTER_ID,\"127.0.0.1:10912\");\n        BrokerData brokerData1 = createBrokerData(\"BrokerB\",\"zone2\",brokeraddrsB);\n        BrokerData brokerData2 = createBrokerData(\"BrokerC\",\"zone1\",null);\n        topicRouteData.getBrokerDatas().add(brokerData1);\n        topicRouteData.getBrokerDatas().add(brokerData2);\n        response.setBody(RemotingSerializable.encode(topicRouteData));\n        zoneRouteRPCHook.doAfterResponse(\"\", request, response);\n\n        topicRouteData.getFilterServerTable().put(\"127.0.0.1:10911\",new ArrayList<>());\n        response.setBody(RemotingSerializable.encode(topicRouteData));\n        zoneRouteRPCHook.doAfterResponse(\"\", request, response);\n        Assert.assertEquals(1,RemotingSerializable\n                .decode(response.getBody(), TopicRouteData.class)\n                .getFilterServerTable()\n                .size());\n\n        topicRouteData.getFilterServerTable().put(\"127.0.0.1:10912\",new ArrayList<>());\n        response.setBody(RemotingSerializable.encode(topicRouteData));\n        zoneRouteRPCHook.doAfterResponse(\"\", request, response);\n        Assert.assertEquals(1,RemotingSerializable\n                .decode(response.getBody(), TopicRouteData.class)\n                .getFilterServerTable()\n                .size());\n    }\n\n    private TopicRouteData createSampleTopicRouteData() {\n        TopicRouteData topicRouteData = new TopicRouteData();\n        List<BrokerData> brokerDatas = new ArrayList<>();\n        BrokerData brokerData = createBrokerData(\"BrokerA\",\"zone1\",new HashMap<>());\n        List<QueueData> queueDatas = new ArrayList<>();\n        QueueData queueData = createQueueData(\"BrokerA\");\n        queueDatas.add(queueData);\n        brokerDatas.add(brokerData);\n        topicRouteData.setBrokerDatas(brokerDatas);\n        topicRouteData.setQueueDatas(queueDatas);\n        return topicRouteData;\n    }\n\n    private BrokerData createBrokerData(String brokerName,String zoneName,HashMap<Long,String> brokerAddrs) {\n        BrokerData brokerData = new BrokerData();\n        brokerData.setBrokerName(brokerName);\n        brokerData.setZoneName(zoneName);\n        brokerData.setBrokerAddrs(brokerAddrs);\n        return  brokerData;\n    }\n\n    private QueueData createQueueData(String brokerName) {\n        QueueData queueData = new QueueData();\n        queueData.setBrokerName(brokerName);\n        queueData.setReadQueueNums(8);\n        queueData.setWriteQueueNums(8);\n        queueData.setPerm(6);\n        return queueData;\n    }\n}\n"
  },
  {
    "path": "namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/BrokerHousekeepingServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.namesrv.routeinfo;\n\nimport org.apache.rocketmq.common.namesrv.NamesrvConfig;\nimport org.apache.rocketmq.namesrv.NamesrvController;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.junit.AfterClass;\nimport org.junit.BeforeClass;\nimport org.junit.Test;\n\npublic class BrokerHousekeepingServiceTest {\n    private static BrokerHousekeepingService brokerHousekeepingService;\n\n    @BeforeClass\n    public static void setup() {\n        NamesrvController namesrvController = new NamesrvController(\n            new NamesrvConfig(),\n            new NettyServerConfig()\n        );\n        brokerHousekeepingService = new BrokerHousekeepingService(namesrvController);\n    }\n\n    @AfterClass\n    public static void terminate() {\n\n    }\n\n    @Test\n    public void testOnChannelClose() {\n        brokerHousekeepingService.onChannelClose(\"127.0.0.1:9876\", null);\n    }\n\n    @Test\n    public void testOnChannelException() {\n        brokerHousekeepingService.onChannelException(\"127.0.0.1:9876\", null);\n    }\n\n    @Test\n    public void testOnChannelIdle() {\n        brokerHousekeepingService.onChannelException(\"127.0.0.1:9876\", null);\n    }\n\n}"
  },
  {
    "path": "namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/GetRouteInfoBenchmark.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.namesrv.routeinfo;\n\nimport io.netty.channel.Channel;\nimport java.util.ArrayList;\nimport java.util.Random;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.commons.lang3.RandomStringUtils;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.namesrv.NamesrvConfig;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;\nimport org.openjdk.jmh.annotations.Benchmark;\nimport org.openjdk.jmh.annotations.BenchmarkMode;\nimport org.openjdk.jmh.annotations.Fork;\nimport org.openjdk.jmh.annotations.Measurement;\nimport org.openjdk.jmh.annotations.Mode;\nimport org.openjdk.jmh.annotations.OutputTimeUnit;\nimport org.openjdk.jmh.annotations.Scope;\nimport org.openjdk.jmh.annotations.Setup;\nimport org.openjdk.jmh.annotations.State;\nimport org.openjdk.jmh.annotations.TearDown;\nimport org.openjdk.jmh.annotations.Threads;\nimport org.openjdk.jmh.annotations.Warmup;\n\nimport static org.mockito.Mockito.mock;\n\n@BenchmarkMode(Mode.SampleTime)\n@OutputTimeUnit(TimeUnit.MILLISECONDS)\n@State(Scope.Benchmark)\npublic class GetRouteInfoBenchmark {\n    private RouteInfoManager routeInfoManager;\n    private String[] topicList = new String[40000];\n    private ExecutorService es = Executors.newCachedThreadPool();\n\n    @Setup\n    public void setup() throws InterruptedException {\n\n        routeInfoManager = new RouteInfoManager(new NamesrvConfig(), null);\n\n        // Init 4 clusters and 8 brokers in each cluster\n        // Each cluster has 10000 topics\n\n        for (int i = 0; i < 40000; i++) {\n            final String topic = RandomStringUtils.randomAlphabetic(32) + i;\n            topicList[i] = topic;\n        }\n\n        for (int i = 0; i < 4; i++) {\n            // Cluster iteration\n            final String clusterName = \"Default-Cluster-\" + i;\n            for (int j = 0; j < 8; j++) {\n                // broker iteration\n                final int startTopicIndex = i * 10000;\n                final String brokerName = \"Default-Broker-\" + j;\n                final String brokerAddr = \"127.0.0.1:500\" + i * j;\n                es.submit(new Runnable() {\n                    @Override\n                    public void run() {\n                        DataVersion dataVersion = new DataVersion();\n                        dataVersion.setCounter(new AtomicLong(10L));\n                        dataVersion.setTimestamp(100L);\n\n                        ConcurrentHashMap<String, TopicConfig> topicConfigConcurrentHashMap = new ConcurrentHashMap<>();\n\n                        for (int k = startTopicIndex; k < startTopicIndex + 10000; k++) {\n                            TopicConfig topicConfig = new TopicConfig();\n                            topicConfig.setWriteQueueNums(8);\n                            topicConfig.setTopicName(topicList[k]);\n                            topicConfig.setPerm(6);\n                            topicConfig.setReadQueueNums(8);\n                            topicConfig.setOrder(false);\n                            topicConfigConcurrentHashMap.put(topicList[k], topicConfig);\n                        }\n\n                        while (true) {\n                            try {\n                                TimeUnit.MILLISECONDS.sleep(new Random().nextInt(100));\n                            } catch (InterruptedException ignored) {\n                            }\n\n                            dataVersion.nextVersion();\n                            TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();\n                            topicConfigSerializeWrapper.setDataVersion(dataVersion);\n                            topicConfigSerializeWrapper.setTopicConfigTable(topicConfigConcurrentHashMap);\n                            Channel channel = mock(Channel.class);\n\n                            routeInfoManager.registerBroker(clusterName, brokerAddr, brokerName, 0, brokerAddr, \"\",\n                                null, topicConfigSerializeWrapper, new ArrayList<>(), channel);\n                        }\n                    }\n                });\n            }\n        }\n\n        // Wait threads startup\n        TimeUnit.SECONDS.sleep(3);\n    }\n\n    @TearDown\n    public void tearDown() {\n        ThreadUtils.shutdownGracefully(es, 3, TimeUnit.SECONDS);\n    }\n\n    @Benchmark\n    @Fork(value = 2)\n    @Measurement(iterations = 10, time = 10)\n    @Warmup(iterations = 10, time = 1)\n    @Threads(4) // Assume we have 128 clients try to pick up route data concurrently\n    public void pickupTopicRouteData() {\n        routeInfoManager.pickupTopicRouteData(topicList[new Random().nextInt(40000)]);\n    }\n\n    public static void main(String[] args) throws Exception {\n        org.openjdk.jmh.Main.main(args);\n    }\n}\n"
  },
  {
    "path": "namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RegisterBrokerBenchmark.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.namesrv.routeinfo;\n\nimport io.netty.channel.Channel;\nimport java.util.ArrayList;\nimport java.util.Random;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.commons.lang3.RandomStringUtils;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.namesrv.NamesrvConfig;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;\nimport org.openjdk.jmh.annotations.Benchmark;\nimport org.openjdk.jmh.annotations.BenchmarkMode;\nimport org.openjdk.jmh.annotations.Fork;\nimport org.openjdk.jmh.annotations.Measurement;\nimport org.openjdk.jmh.annotations.Mode;\nimport org.openjdk.jmh.annotations.OutputTimeUnit;\nimport org.openjdk.jmh.annotations.Scope;\nimport org.openjdk.jmh.annotations.Setup;\nimport org.openjdk.jmh.annotations.State;\nimport org.openjdk.jmh.annotations.TearDown;\nimport org.openjdk.jmh.annotations.Threads;\nimport org.openjdk.jmh.annotations.Warmup;\n\nimport static org.mockito.Mockito.mock;\n\n@BenchmarkMode(Mode.SampleTime)\n@OutputTimeUnit(TimeUnit.MILLISECONDS)\n@State(Scope.Benchmark)\npublic class RegisterBrokerBenchmark {\n    private RouteInfoManager routeInfoManager;\n    private String[] topicList = new String[40000];\n    private ConcurrentHashMap<String, TopicConfig>[] topicConfigMaps = new ConcurrentHashMap[32];\n    private DataVersion[] dataVersions = new DataVersion[32];\n    private ExecutorService es = Executors.newCachedThreadPool();\n    private AtomicLong brokerIndex = new AtomicLong(0);\n\n    @Setup\n    public void setup() throws InterruptedException {\n        routeInfoManager = new RouteInfoManager(new NamesrvConfig(), null);\n\n        // Init 4 clusters and 8 brokers in each cluster\n        // Each cluster has 10000 topics\n\n        for (int i = 0; i < 40000; i++) {\n            final String topic = RandomStringUtils.randomAlphabetic(32) + i;\n            topicList[i] = topic;\n        }\n\n        for (int i = 0; i < 4; i++) {\n            // Cluster iteration\n            final String clusterName = \"Default-Cluster-\" + i;\n            for (int j = 0; j < 8; j++) {\n                // broker iteration\n                final int startTopicIndex = i * 10000;\n                final String brokerName = \"Default-Broker-\" + j;\n                final String brokerAddr = \"127.0.0.1:500\" + (i * 8 + j);\n\n                topicConfigMaps[i * 8 + j] = new ConcurrentHashMap<>();\n                for (int k = startTopicIndex; k < startTopicIndex + 10000; k++) {\n                    TopicConfig topicConfig = new TopicConfig();\n                    topicConfig.setWriteQueueNums(8);\n                    topicConfig.setTopicName(topicList[k]);\n                    topicConfig.setPerm(6);\n                    topicConfig.setReadQueueNums(8);\n                    topicConfig.setOrder(false);\n                    topicConfigMaps[i * 8 + j].put(topicList[k], topicConfig);\n                }\n\n                DataVersion dataVersion = new DataVersion();\n                dataVersion.setCounter(new AtomicLong(10L));\n                dataVersion.setTimestamp(100L);\n\n                dataVersions[i * 8 + j] = dataVersion;\n            }\n        }\n\n        // Init 32 threads to pick up route info\n        for (int i = 0; i < 32; i++) {\n            es.submit(new Runnable() {\n                @Override\n                public void run() {\n                    routeInfoManager.pickupTopicRouteData(topicList[new Random().nextInt(40000)]);\n                    try {\n                        TimeUnit.MILLISECONDS.sleep(new Random().nextInt(10));\n                    } catch (InterruptedException ignored) {\n                    }\n                }\n            });\n        }\n    }\n\n    @TearDown\n    public void tearDown() {\n        ThreadUtils.shutdownGracefully(es, 3, TimeUnit.SECONDS);\n    }\n\n    @Benchmark\n    @Fork(value = 2)\n    @Measurement(iterations = 10, time = 10)\n    @Warmup(iterations = 10, time = 1)\n    @Threads(32) // Assume we have 128 clients try to pick up route data concurrently\n    public void registerBroker() {\n        final long index = Math.abs(brokerIndex.getAndIncrement() % 32);\n        dataVersions[(int) index].nextVersion();\n\n        TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();\n        topicConfigSerializeWrapper.setDataVersion(dataVersions[(int) index]);\n        topicConfigSerializeWrapper.setTopicConfigTable(topicConfigMaps[(int) index]);\n        Channel channel = mock(Channel.class);\n\n        routeInfoManager.registerBroker(\"DefaultCluster\" + index,\n            \"127.0.0.1:500\" + index,\n            \"DefaultBroker\" + index, 0, \"127.0.0.1:400\" + index,\n            \"\",\n            null,\n            topicConfigSerializeWrapper, new ArrayList<>(), channel);\n    }\n\n    @Benchmark\n    @Fork(value = 2)\n    @Measurement(iterations = 10, time = 10)\n    @Warmup(iterations = 10, time = 1)\n    @Threads(32) // Assume we have 128 clients try to pick up route data concurrently\n    @BenchmarkMode(Mode.Throughput)\n    public void registerBroker_Throughput() {\n        final long index = Math.abs(brokerIndex.getAndIncrement() % 32);\n        dataVersions[(int) index].nextVersion();\n\n        TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();\n        topicConfigSerializeWrapper.setDataVersion(dataVersions[(int) index]);\n        topicConfigSerializeWrapper.setTopicConfigTable(topicConfigMaps[(int) index]);\n        Channel channel = mock(Channel.class);\n\n        routeInfoManager.registerBroker(\"DefaultCluster\" + index,\n            \"127.0.0.1:500\" + index,\n            \"DefaultBroker\" + index, 0, \"127.0.0.1:400\" + index,\n            \"\",\n            null,\n            topicConfigSerializeWrapper, new ArrayList<>(), channel);\n    }\n\n    public static void main(String[] args) throws Exception {\n        org.openjdk.jmh.Main.main(args);\n    }\n}\n"
  },
  {
    "path": "namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManagerBrokerPermTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.namesrv.routeinfo;\n\nimport java.lang.reflect.Field;\nimport java.util.HashMap;\nimport java.util.Map;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.namesrv.NamesrvConfig;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class RouteInfoManagerBrokerPermTest extends RouteInfoManagerTestBase {\n    private static RouteInfoManager routeInfoManager;\n    public static String clusterName = \"cluster\";\n    public static String brokerPrefix = \"broker\";\n    public static String topicPrefix = \"topic\";\n\n    public static RouteInfoManagerTestBase.Cluster cluster;\n\n    @Before\n    public void setup() {\n        routeInfoManager = new RouteInfoManager(new NamesrvConfig(), null);\n        cluster = registerCluster(routeInfoManager,\n            clusterName,\n            brokerPrefix,\n            3,\n            3,\n            topicPrefix,\n            10);\n    }\n\n    @After\n    public void terminate() {\n        routeInfoManager.printAllPeriodically();\n\n        for (BrokerData bd : cluster.brokerDataMap.values()) {\n            unregisterBrokerAll(routeInfoManager, bd);\n        }\n    }\n\n    @Test\n    public void testAddWritePermOfBrokerByLock() throws Exception {\n        String brokerName = getBrokerName(brokerPrefix, 0);\n        String topicName = getTopicName(topicPrefix, 0);\n\n        QueueData qd = new QueueData();\n        qd.setPerm(PermName.PERM_READ);\n        qd.setBrokerName(brokerName);\n\n        HashMap<String, Map<String, QueueData>> topicQueueTable = new HashMap<>();\n\n        Map<String, QueueData> queueDataMap = new HashMap<>();\n        queueDataMap.put(brokerName, qd);\n        topicQueueTable.put(topicName, queueDataMap);\n\n        Field filed = RouteInfoManager.class.getDeclaredField(\"topicQueueTable\");\n        filed.setAccessible(true);\n        filed.set(routeInfoManager, topicQueueTable);\n\n        int addTopicCnt = routeInfoManager.addWritePermOfBrokerByLock(brokerName);\n        assertThat(addTopicCnt).isEqualTo(1);\n        assertThat(qd.getPerm()).isEqualTo(PermName.PERM_READ | PermName.PERM_WRITE);\n\n    }\n\n    @Test\n    public void testWipeWritePermOfBrokerByLock() throws Exception {\n        String brokerName = getBrokerName(brokerPrefix, 0);\n        String topicName = getTopicName(topicPrefix, 0);\n\n        QueueData qd = new QueueData();\n        qd.setPerm(PermName.PERM_READ);\n        qd.setBrokerName(brokerName);\n\n        HashMap<String, Map<String, QueueData>> topicQueueTable = new HashMap<>();\n\n        Map<String, QueueData> queueDataMap = new HashMap<>();\n        queueDataMap.put(brokerName, qd);\n        topicQueueTable.put(topicName, queueDataMap);\n\n        Field filed = RouteInfoManager.class.getDeclaredField(\"topicQueueTable\");\n        filed.setAccessible(true);\n        filed.set(routeInfoManager, topicQueueTable);\n\n        int addTopicCnt = routeInfoManager.wipeWritePermOfBrokerByLock(brokerName);\n        assertThat(addTopicCnt).isEqualTo(1);\n        assertThat(qd.getPerm()).isEqualTo(PermName.PERM_READ);\n\n    }\n}\n"
  },
  {
    "path": "namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManagerBrokerRegisterTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.namesrv.routeinfo;\n\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.namesrv.NamesrvConfig;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\n\npublic class RouteInfoManagerBrokerRegisterTest extends RouteInfoManagerTestBase {\n    private static RouteInfoManager routeInfoManager;\n    public static String clusterName = \"cluster\";\n    public static String brokerPrefix = \"broker\";\n    public static String topicPrefix = \"topic\";\n    public static int brokerPerName = 3;\n    public static int brokerNameNumber = 3;\n\n    public static RouteInfoManagerTestBase.Cluster cluster;\n\n    @Before\n    public void setup() {\n        routeInfoManager = new RouteInfoManager(new NamesrvConfig(), null);\n        cluster = registerCluster(routeInfoManager,\n            clusterName,\n            brokerPrefix,\n            brokerNameNumber,\n            brokerPerName,\n            topicPrefix,\n            10);\n    }\n\n    @After\n    public void terminate() {\n        routeInfoManager.printAllPeriodically();\n\n        for (BrokerData bd : cluster.brokerDataMap.values()) {\n            unregisterBrokerAll(routeInfoManager, bd);\n        }\n    }\n\n//    @Test\n//    public void testScanNotActiveBroker() {\n//        for (int j = 0; j < brokerNameNumber; j++) {\n//            String brokerName = getBrokerName(brokerPrefix, j);\n//\n//            for (int i = 0; i < brokerPerName; i++) {\n//                String brokerAddr = getBrokerAddr(clusterName, brokerName, i);\n//\n//                // set not active\n//                routeInfoManager.updateBrokerInfoUpdateTimestamp(brokerAddr, 0);\n//\n//                assertEquals(1, routeInfoManager.scanNotActiveBroker());\n//            }\n//        }\n//\n//    }\n\n    @Test\n    public void testMasterChangeFromSlave() {\n        String topicName = getTopicName(topicPrefix, 0);\n        String brokerName = getBrokerName(brokerPrefix, 0);\n\n        String originMasterAddr = getBrokerAddr(clusterName, brokerName, MixAll.MASTER_ID);\n        TopicRouteData topicRouteData = routeInfoManager.pickupTopicRouteData(topicName);\n        BrokerData brokerDataOrigin = findBrokerDataByBrokerName(topicRouteData.getBrokerDatas(), brokerName);\n\n        // check origin master address\n        Assert.assertEquals(brokerDataOrigin.getBrokerAddrs().get(MixAll.MASTER_ID), originMasterAddr);\n\n        // master changed\n        String newMasterAddr = getBrokerAddr(clusterName, brokerName, 1);\n        registerBrokerWithTopicConfig(routeInfoManager,\n            clusterName,\n            newMasterAddr,\n            brokerName,\n            MixAll.MASTER_ID,\n            newMasterAddr,\n            cluster.topicConfig,\n            new ArrayList<>());\n\n        topicRouteData = routeInfoManager.pickupTopicRouteData(topicName);\n        brokerDataOrigin = findBrokerDataByBrokerName(topicRouteData.getBrokerDatas(), brokerName);\n\n        // check new master address\n        assertEquals(brokerDataOrigin.getBrokerAddrs().get(MixAll.MASTER_ID), newMasterAddr);\n    }\n\n    @Test\n    public void testUnregisterBroker() {\n        String topicName = getTopicName(topicPrefix, 0);\n        String brokerName = getBrokerName(brokerPrefix, 0);\n        long unregisterBrokerId = 2;\n\n        unregisterBroker(routeInfoManager, cluster.brokerDataMap.get(brokerName), unregisterBrokerId);\n\n        TopicRouteData topicRouteData = routeInfoManager.pickupTopicRouteData(topicName);\n        HashMap<Long, String> brokerAddrs = findBrokerDataByBrokerName(topicRouteData.getBrokerDatas(), brokerName).getBrokerAddrs();\n\n        assertFalse(brokerAddrs.containsKey(unregisterBrokerId));\n    }\n}\n"
  },
  {
    "path": "namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManagerNewTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.namesrv.routeinfo;\n\nimport com.google.common.collect.Sets;\nimport io.netty.channel.Channel;\nimport java.time.Duration;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.namesrv.NamesrvConfig;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.body.ClusterInfo;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.body.TopicList;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.UnRegisterBrokerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.namesrv.RegisterBrokerResult;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.Spy;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.awaitility.Awaitility.await;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.spy;\n\npublic class RouteInfoManagerNewTest {\n    private RouteInfoManager routeInfoManager;\n    private static final String DEFAULT_CLUSTER = \"Default_Cluster\";\n    private static final String DEFAULT_BROKER = \"Default_Broker\";\n    private static final String DEFAULT_ADDR_PREFIX = \"127.0.0.1:\";\n    private static final String DEFAULT_ADDR = DEFAULT_ADDR_PREFIX + \"10911\";\n\n    // Synced from RouteInfoManager\n    private static final int BROKER_CHANNEL_EXPIRED_TIME = 1000 * 60 * 2;\n\n    @Spy\n    private static NamesrvConfig config = spy(new NamesrvConfig());\n\n    @Before\n    public void setup() {\n        config.setSupportActingMaster(true);\n        routeInfoManager = new RouteInfoManager(config, null);\n        routeInfoManager.start();\n    }\n\n    @After\n    public void tearDown() throws Exception {\n        routeInfoManager.shutdown();\n    }\n\n    @Test\n    public void getAllClusterInfo() {\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), \"TestTopic\");\n\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker()\n            .cluster(\"AnotherCluster\")\n            .name(\"AnotherBroker\")\n            .addr(DEFAULT_ADDR_PREFIX + 30911), \"TestTopic1\");\n\n        final byte[] content = routeInfoManager.getAllClusterInfo().encode();\n\n        final ClusterInfo clusterInfo = ClusterInfo.decode(content, ClusterInfo.class);\n\n        assertThat(clusterInfo.retrieveAllClusterNames()).contains(DEFAULT_CLUSTER, \"AnotherCluster\");\n        assertThat(clusterInfo.getBrokerAddrTable().keySet()).contains(DEFAULT_BROKER, \"AnotherBroker\");\n\n        final List<String> addrList = Arrays.asList(clusterInfo.getBrokerAddrTable().get(DEFAULT_BROKER).getBrokerAddrs().get(0L),\n            clusterInfo.getBrokerAddrTable().get(\"AnotherBroker\").getBrokerAddrs().get(0L));\n        assertThat(addrList).contains(DEFAULT_ADDR, DEFAULT_ADDR_PREFIX + 30911);\n    }\n\n    @Test\n    public void deleteTopic() {\n        String testTopic = \"TestTopic\";\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), testTopic);\n\n        routeInfoManager.deleteTopic(testTopic);\n\n        assertThat(routeInfoManager.pickupTopicRouteData(testTopic)).isNull();\n\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), testTopic);\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker().cluster(\"AnotherCluster\").name(\"AnotherBroker\"),\n            testTopic);\n\n        assertThat(routeInfoManager.pickupTopicRouteData(testTopic).getBrokerDatas().size()).isEqualTo(2);\n        routeInfoManager.deleteTopic(testTopic, DEFAULT_CLUSTER);\n\n        assertThat(routeInfoManager.pickupTopicRouteData(testTopic).getBrokerDatas().size()).isEqualTo(1);\n        assertThat(routeInfoManager.pickupTopicRouteData(testTopic).getBrokerDatas().get(0).getBrokerName()).isEqualTo(\"AnotherBroker\");\n    }\n\n    @Test\n    public void getAllTopicList() {\n        byte[] content = routeInfoManager.getAllTopicList().encode();\n\n        TopicList topicList = TopicList.decode(content, TopicList.class);\n        assertThat(topicList.getTopicList()).isEmpty();\n\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), \"TestTopic\", \"TestTopic1\", \"TestTopic2\");\n\n        content = routeInfoManager.getAllTopicList().encode();\n\n        topicList = TopicList.decode(content, TopicList.class);\n        assertThat(topicList.getTopicList()).contains(\"TestTopic\", \"TestTopic1\", \"TestTopic2\");\n    }\n    @Test\n    public void registerBroker() {\n        // Register master broker\n        final RegisterBrokerResult masterResult = registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), \"TestTopic\");\n\n        assertThat(masterResult).isNotNull();\n        assertThat(masterResult.getHaServerAddr()).isNull();\n        assertThat(masterResult.getMasterAddr()).isNull();\n\n        // Register slave broker\n\n        final BrokerBasicInfo slaveBroker = BrokerBasicInfo.defaultBroker()\n            .id(1).addr(DEFAULT_ADDR_PREFIX + 30911).haAddr(DEFAULT_ADDR_PREFIX + 40911);\n\n        final RegisterBrokerResult slaveResult = registerBrokerWithNormalTopic(slaveBroker, \"TestTopic\");\n\n        assertThat(slaveResult).isNotNull();\n        assertThat(slaveResult.getHaServerAddr()).isEqualTo(DEFAULT_ADDR_PREFIX + 20911);\n        assertThat(slaveResult.getMasterAddr()).isEqualTo(DEFAULT_ADDR);\n    }\n\n    @Test\n    public void unregisterBroker() {\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), \"TestTopic\", \"TestTopic1\", \"TestTopic2\");\n        routeInfoManager.unregisterBroker(DEFAULT_CLUSTER, DEFAULT_ADDR, DEFAULT_BROKER, 0);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\")).isNull();\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic1\")).isNull();\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic2\")).isNull();\n\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), \"TestTopic\", \"TestTopic1\", \"TestTopic2\");\n\n        routeInfoManager.submitUnRegisterBrokerRequest(BrokerBasicInfo.defaultBroker().unRegisterRequest());\n        await().atMost(Duration.ofSeconds(5)).until(() -> routeInfoManager.blockedUnRegisterRequests() == 0);\n\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\")).isNull();\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic1\")).isNull();\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic2\")).isNull();\n    }\n\n    @Test\n    public void registerSlaveBroker() {\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), \"TestTopic\");\n\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0).getBrokerAddrs()).containsKeys(0L);\n\n        registerBrokerWithNormalTopic(BrokerBasicInfo.slaveBroker(), \"TestTopic\");\n\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0).getBrokerAddrs()).containsKeys(0L, 1L);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0).getBrokerAddrs())\n            .containsValues(BrokerBasicInfo.defaultBroker().brokerAddr, BrokerBasicInfo.slaveBroker().brokerAddr);\n    }\n\n    @Test\n    public void createNewTopic() {\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), \"TestTopic\");\n        registerBrokerWithNormalTopic(BrokerBasicInfo.slaveBroker(), \"TestTopic\");\n\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), \"TestTopic\", \"TestTopic1\");\n\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic1\").getBrokerDatas().get(0).getBrokerAddrs()).containsKeys(0L, 1L);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic1\").getBrokerDatas().get(0).getBrokerAddrs())\n            .containsValues(BrokerBasicInfo.defaultBroker().brokerAddr, BrokerBasicInfo.slaveBroker().brokerAddr);\n    }\n\n    @Test\n    public void switchBrokerRole() {\n        final BrokerBasicInfo masterBroker = BrokerBasicInfo.defaultBroker().enableActingMaster(false);\n        final BrokerBasicInfo slaveBroker = BrokerBasicInfo.slaveBroker().enableActingMaster(false);\n        registerBrokerWithNormalTopic(masterBroker, \"TestTopic\");\n        registerBrokerWithNormalTopic(slaveBroker, \"TestTopic\");\n\n        // Master Down\n        routeInfoManager.unregisterBroker(masterBroker.clusterName, masterBroker.brokerAddr, masterBroker.brokerName, 0);\n\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0).getBrokerAddrs()).containsOnlyKeys(1L);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0).getBrokerAddrs())\n            .containsValues(slaveBroker.brokerAddr);\n\n        // Switch slave to master\n        slaveBroker.id(0).dataVersion.nextVersion();\n        registerBrokerWithNormalTopic(slaveBroker, \"TestTopic\");\n\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0).getBrokerAddrs()).containsOnlyKeys(0L);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0).getBrokerAddrs())\n            .containsValues(slaveBroker.brokerAddr);\n\n        // Old master switch to slave\n        masterBroker.id(1).dataVersion.nextVersion();\n        registerBrokerWithNormalTopic(masterBroker, \"TestTopic\");\n\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0).getBrokerAddrs()).containsKeys(0L, 1L);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0).getBrokerAddrs())\n            .containsValues(BrokerBasicInfo.defaultBroker().brokerAddr, BrokerBasicInfo.slaveBroker().brokerAddr);\n    }\n\n    @Test\n    public void unRegisterSlaveBroker() {\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), \"TestTopic\");\n        final BrokerBasicInfo slaveBroker = BrokerBasicInfo.slaveBroker();\n        registerBrokerWithNormalTopic(slaveBroker, \"TestTopic\");\n\n        routeInfoManager.unregisterBroker(slaveBroker.clusterName, slaveBroker.brokerAddr, slaveBroker.brokerName, 1);\n\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().size()).isEqualTo(1);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0)\n            .getBrokerAddrs().get(0L)).isEqualTo(BrokerBasicInfo.defaultBroker().brokerAddr);\n\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), \"TestTopic\");\n        registerBrokerWithNormalTopic(slaveBroker, \"TestTopic\");\n\n        routeInfoManager.submitUnRegisterBrokerRequest(slaveBroker.unRegisterRequest());\n        await().atMost(Duration.ofSeconds(5)).until(() -> routeInfoManager.blockedUnRegisterRequests() == 0);\n\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().size()).isEqualTo(1);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0)\n            .getBrokerAddrs().get(0L)).isEqualTo(BrokerBasicInfo.defaultBroker().brokerAddr);\n    }\n\n    @Test\n    public void unRegisterMasterBroker() {\n        final BrokerBasicInfo masterBroker = BrokerBasicInfo.defaultBroker();\n        masterBroker.enableActingMaster = true;\n        registerBrokerWithNormalTopic(masterBroker, \"TestTopic\");\n        final BrokerBasicInfo slaveBroker = BrokerBasicInfo.slaveBroker();\n        registerBrokerWithNormalTopic(slaveBroker, \"TestTopic\");\n\n        routeInfoManager.unregisterBroker(masterBroker.clusterName, masterBroker.brokerAddr, masterBroker.brokerName, 0);\n\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().size()).isEqualTo(1);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0)\n            .getBrokerAddrs().get(0L)).isEqualTo(slaveBroker.brokerAddr);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getQueueDatas().size()).isEqualTo(1);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getQueueDatas().get(0).getPerm()).isEqualTo(PermName.PERM_READ);\n    }\n\n    @Test\n    public void unRegisterMasterBrokerOldVersion() {\n        final BrokerBasicInfo masterBroker = BrokerBasicInfo.defaultBroker();\n        masterBroker.enableActingMaster = false;\n        registerBrokerWithNormalTopic(masterBroker, \"TestTopic\");\n        final BrokerBasicInfo slaveBroker = BrokerBasicInfo.slaveBroker();\n        slaveBroker.enableActingMaster = false;\n        registerBrokerWithNormalTopic(slaveBroker, \"TestTopic\");\n\n        routeInfoManager.unregisterBroker(masterBroker.clusterName, masterBroker.brokerAddr, masterBroker.brokerName, 0);\n\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().size()).isEqualTo(1);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0)\n            .getBrokerAddrs().get(0L)).isNull();\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getQueueDatas().size()).isEqualTo(1);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getQueueDatas().get(0).getPerm()).isEqualTo(PermName.PERM_READ | PermName.PERM_WRITE);\n    }\n\n    @Test\n    public void submitMultiUnRegisterRequests() {\n        final BrokerBasicInfo master1 = BrokerBasicInfo.defaultBroker();\n        final BrokerBasicInfo master2 = BrokerBasicInfo.defaultBroker().name(DEFAULT_BROKER + 1).addr(DEFAULT_ADDR + 9);\n        registerBrokerWithNormalTopic(master1, \"TestTopic1\");\n        registerBrokerWithNormalTopic(master2, \"TestTopic2\");\n\n        routeInfoManager.submitUnRegisterBrokerRequest(master1.unRegisterRequest());\n        routeInfoManager.submitUnRegisterBrokerRequest(master2.unRegisterRequest());\n\n        await().atMost(Duration.ofSeconds(5)).until(() -> routeInfoManager.blockedUnRegisterRequests() == 0);\n\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic1\")).isNull();\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic2\")).isNull();\n    }\n\n    @Test\n    public void isBrokerTopicConfigChanged() {\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), \"TestTopic\");\n\n        final DataVersion dataVersion = routeInfoManager.queryBrokerTopicConfig(DEFAULT_CLUSTER, DEFAULT_ADDR);\n\n        assertThat(routeInfoManager.isBrokerTopicConfigChanged(DEFAULT_CLUSTER, DEFAULT_ADDR, dataVersion)).isFalse();\n\n        DataVersion newVersion = new DataVersion();\n        newVersion.setTimestamp(System.currentTimeMillis() + 1000);\n        newVersion.setCounter(new AtomicLong(dataVersion.getCounter().get()));\n\n        assertThat(routeInfoManager.isBrokerTopicConfigChanged(DEFAULT_CLUSTER, DEFAULT_ADDR, newVersion)).isTrue();\n\n        newVersion = new DataVersion();\n        newVersion.setTimestamp(dataVersion.getTimestamp());\n        newVersion.setCounter(new AtomicLong(dataVersion.getCounter().get() + 1));\n\n        assertThat(routeInfoManager.isBrokerTopicConfigChanged(DEFAULT_CLUSTER, DEFAULT_ADDR, newVersion)).isTrue();\n    }\n\n    @Test\n    public void isTopicConfigChanged() {\n        final BrokerBasicInfo brokerInfo = BrokerBasicInfo.defaultBroker();\n        assertThat(routeInfoManager.isTopicConfigChanged(DEFAULT_CLUSTER,\n            DEFAULT_ADDR,\n            brokerInfo.dataVersion,\n            DEFAULT_BROKER, \"TestTopic\")).isTrue();\n\n        registerBrokerWithNormalTopic(brokerInfo, \"TestTopic\");\n\n        assertThat(routeInfoManager.isTopicConfigChanged(DEFAULT_CLUSTER,\n            DEFAULT_ADDR,\n            brokerInfo.dataVersion,\n            DEFAULT_BROKER, \"TestTopic\")).isFalse();\n\n        assertThat(routeInfoManager.isTopicConfigChanged(DEFAULT_CLUSTER,\n            DEFAULT_ADDR,\n            brokerInfo.dataVersion,\n            DEFAULT_BROKER, \"TestTopic1\")).isTrue();\n    }\n\n    @Test\n    public void queryBrokerTopicConfig() {\n        final BrokerBasicInfo basicInfo = BrokerBasicInfo.defaultBroker();\n        registerBrokerWithNormalTopic(basicInfo, \"TestTopic\");\n\n        final DataVersion dataVersion = routeInfoManager.queryBrokerTopicConfig(DEFAULT_CLUSTER, DEFAULT_ADDR);\n\n        assertThat(basicInfo.dataVersion.equals(dataVersion)).isTrue();\n    }\n\n    @Test\n    public void wipeWritePermOfBrokerByLock() {\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), \"TestTopic\");\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getQueueDatas().get(0).getPerm()).isEqualTo(6);\n\n        routeInfoManager.wipeWritePermOfBrokerByLock(DEFAULT_BROKER);\n\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getQueueDatas().get(0).getPerm()).isEqualTo(4);\n    }\n\n    @Test\n    public void pickupTopicRouteData() {\n        String testTopic = \"TestTopic\";\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), testTopic);\n\n        TopicRouteData data = routeInfoManager.pickupTopicRouteData(testTopic);\n        assertThat(data.getBrokerDatas().size()).isEqualTo(1);\n        assertThat(data.getBrokerDatas().get(0).getBrokerName()).isEqualTo(DEFAULT_BROKER);\n        assertThat(data.getBrokerDatas().get(0).getBrokerAddrs().get(0L)).isEqualTo(DEFAULT_ADDR);\n        assertThat(data.getQueueDatas().size()).isEqualTo(1);\n        assertThat(data.getQueueDatas().get(0).getBrokerName()).isEqualTo(DEFAULT_BROKER);\n        assertThat(data.getQueueDatas().get(0).getReadQueueNums()).isEqualTo(8);\n        assertThat(data.getQueueDatas().get(0).getWriteQueueNums()).isEqualTo(8);\n        assertThat(data.getQueueDatas().get(0).getPerm()).isEqualTo(6);\n\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker().name(\"AnotherBroker\"), testTopic);\n        data = routeInfoManager.pickupTopicRouteData(testTopic);\n\n        assertThat(data.getBrokerDatas().size()).isEqualTo(2);\n        assertThat(data.getQueueDatas().size()).isEqualTo(2);\n\n        List<String> brokerList =\n            Arrays.asList(data.getBrokerDatas().get(0).getBrokerName(), data.getBrokerDatas().get(1).getBrokerName());\n        assertThat(brokerList).contains(DEFAULT_BROKER, \"AnotherBroker\");\n\n        brokerList =\n            Arrays.asList(data.getQueueDatas().get(0).getBrokerName(), data.getQueueDatas().get(1).getBrokerName());\n        assertThat(brokerList).contains(DEFAULT_BROKER, \"AnotherBroker\");\n    }\n\n    @Test\n    public void pickupTopicRouteDataWithSlave() {\n        String testTopic = \"TestTopic\";\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), testTopic);\n        registerBrokerWithNormalTopic(BrokerBasicInfo.slaveBroker(), testTopic);\n\n        TopicRouteData routeData = routeInfoManager.pickupTopicRouteData(testTopic);\n\n        assertThat(routeData.getBrokerDatas().get(0).getBrokerAddrs()).hasSize(2);\n        assertThat(PermName.isWriteable(routeData.getQueueDatas().get(0).getPerm())).isTrue();\n\n        routeInfoManager.unRegisterBroker(Sets.newHashSet(BrokerBasicInfo.defaultBroker().unRegisterRequest()));\n        routeData = routeInfoManager.pickupTopicRouteData(testTopic);\n\n        assertThat(routeData.getBrokerDatas().get(0).getBrokerAddrs()).hasSize(1);\n        assertThat(PermName.isWriteable(routeData.getQueueDatas().get(0).getPerm())).isFalse();\n\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), testTopic);\n        routeData = routeInfoManager.pickupTopicRouteData(testTopic);\n\n        assertThat(routeData.getBrokerDatas().get(0).getBrokerAddrs()).hasSize(2);\n        assertThat(PermName.isWriteable(routeData.getQueueDatas().get(0).getPerm())).isTrue();\n    }\n\n    @Test\n    public void scanNotActiveBroker() throws InterruptedException {\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), \"TestTopic\");\n        routeInfoManager.scanNotActiveBroker();\n        registerBrokerWithNormalTopicAndExpire(BrokerBasicInfo.defaultBroker(),\"TestTopic\");\n        Thread.sleep(30000);\n        routeInfoManager.scanNotActiveBroker();\n    }\n\n    @Test\n    public void pickupPartitionOrderTopicRouteData() {\n        String orderTopic = \"PartitionOrderTopicTest\";\n\n        // Case 1: Register global order topic with slave\n        registerBrokerWithOrderTopic(BrokerBasicInfo.slaveBroker(), orderTopic);\n\n        TopicRouteData orderRoute = routeInfoManager.pickupTopicRouteData(orderTopic);\n\n        // Acting master check\n        assertThat(orderRoute.getBrokerDatas().get(0).getBrokerAddrs())\n            .containsOnlyKeys(MixAll.MASTER_ID);\n        assertThat(orderRoute.getBrokerDatas().get(0).getBrokerAddrs())\n            .containsValue(BrokerBasicInfo.slaveBroker().brokerAddr);\n        assertThat(PermName.isWriteable(orderRoute.getQueueDatas().get(0).getPerm())).isFalse();\n\n        routeInfoManager.unRegisterBroker(Sets.newHashSet(BrokerBasicInfo.slaveBroker().unRegisterRequest()));\n\n        // Case 2: Register global order topic with master and slave, then unregister master\n\n        registerBrokerWithOrderTopic(BrokerBasicInfo.slaveBroker(), orderTopic);\n        registerBrokerWithOrderTopic(BrokerBasicInfo.defaultBroker(), orderTopic);\n        routeInfoManager.unRegisterBroker(Sets.newHashSet(BrokerBasicInfo.defaultBroker().unRegisterRequest()));\n\n        orderRoute = routeInfoManager.pickupTopicRouteData(orderTopic);\n\n        // Acting master check\n        assertThat(orderRoute.getBrokerDatas().get(0).getBrokerAddrs())\n            .containsOnlyKeys(MixAll.MASTER_ID);\n        assertThat(orderRoute.getBrokerDatas().get(0).getBrokerAddrs())\n            .containsValue(BrokerBasicInfo.slaveBroker().brokerAddr);\n        assertThat(PermName.isWriteable(orderRoute.getQueueDatas().get(0).getPerm())).isFalse();\n\n        routeInfoManager.unRegisterBroker(Sets.newHashSet(BrokerBasicInfo.slaveBroker().unRegisterRequest()));\n\n        // Case 3: Register two broker groups, only one group enable acting master\n        registerBrokerWithOrderTopic(BrokerBasicInfo.slaveBroker(), orderTopic);\n        registerBrokerWithOrderTopic(BrokerBasicInfo.defaultBroker(), orderTopic);\n\n        final BrokerBasicInfo master1 = BrokerBasicInfo.defaultBroker().name(DEFAULT_BROKER + \"_ANOTHER\");\n        final BrokerBasicInfo slave1 = BrokerBasicInfo.slaveBroker().name(DEFAULT_BROKER + \"_ANOTHER\");\n\n        registerBrokerWithOrderTopic(master1, orderTopic);\n        registerBrokerWithOrderTopic(slave1, orderTopic);\n\n        orderRoute = routeInfoManager.pickupTopicRouteData(orderTopic);\n\n        assertThat(orderRoute.getBrokerDatas()).hasSize(2);\n        assertThat(orderRoute.getQueueDatas()).hasSize(2);\n\n        routeInfoManager.unRegisterBroker(Sets.newHashSet(BrokerBasicInfo.defaultBroker().unRegisterRequest()));\n        orderRoute = routeInfoManager.pickupTopicRouteData(orderTopic);\n\n        assertThat(orderRoute.getBrokerDatas()).hasSize(2);\n        assertThat(orderRoute.getQueueDatas()).hasSize(2);\n\n        for (final BrokerData brokerData : orderRoute.getBrokerDatas()) {\n            if (brokerData.getBrokerAddrs().size() == 1) {\n                assertThat(brokerData.getBrokerAddrs()).containsOnlyKeys(MixAll.MASTER_ID);\n                assertThat(brokerData.getBrokerAddrs()).containsValue(BrokerBasicInfo.slaveBroker().brokerAddr);\n            } else if (brokerData.getBrokerAddrs().size() == 2) {\n                assertThat(brokerData.getBrokerAddrs()).containsKeys(MixAll.MASTER_ID, (long) slave1.brokerId);\n                assertThat(brokerData.getBrokerAddrs()).containsValues(master1.brokerAddr, slave1.brokerAddr);\n            } else {\n                throw new RuntimeException(\"Shouldn't reach here\");\n            }\n        }\n    }\n\n    @Test\n    public void pickupGlobalOrderTopicRouteData() {\n        String orderTopic = \"GlobalOrderTopicTest\";\n\n        // Case 1: Register global order topic with slave\n        registerBrokerWithGlobalOrderTopic(BrokerBasicInfo.slaveBroker(), orderTopic);\n\n        TopicRouteData orderRoute = routeInfoManager.pickupTopicRouteData(orderTopic);\n\n        // Acting master check\n        assertThat(orderRoute.getBrokerDatas().get(0).getBrokerAddrs())\n            .containsOnlyKeys(MixAll.MASTER_ID);\n        assertThat(orderRoute.getBrokerDatas().get(0).getBrokerAddrs())\n            .containsValue(BrokerBasicInfo.slaveBroker().brokerAddr);\n        assertThat(PermName.isWriteable(orderRoute.getQueueDatas().get(0).getPerm())).isFalse();\n\n        routeInfoManager.unRegisterBroker(Sets.newHashSet(BrokerBasicInfo.slaveBroker().unRegisterRequest()));\n\n        // Case 2: Register global order topic with master and slave, then unregister master\n\n        registerBrokerWithGlobalOrderTopic(BrokerBasicInfo.slaveBroker(), orderTopic);\n        registerBrokerWithGlobalOrderTopic(BrokerBasicInfo.defaultBroker(), orderTopic);\n        routeInfoManager.unRegisterBroker(Sets.newHashSet(BrokerBasicInfo.defaultBroker().unRegisterRequest()));\n\n        // Acting master check\n        assertThat(orderRoute.getBrokerDatas().get(0).getBrokerAddrs())\n            .containsOnlyKeys(MixAll.MASTER_ID);\n        assertThat(orderRoute.getBrokerDatas().get(0).getBrokerAddrs())\n            .containsValue(BrokerBasicInfo.slaveBroker().brokerAddr);\n        assertThat(PermName.isWriteable(orderRoute.getQueueDatas().get(0).getPerm())).isFalse();\n    }\n\n    @Test\n    public void registerOnlySlaveBroker() {\n        final String testTopic = \"TestTopic\";\n\n        // Case 1: Only slave broker\n        registerBrokerWithNormalTopic(BrokerBasicInfo.slaveBroker(), testTopic);\n        assertThat(routeInfoManager.pickupTopicRouteData(testTopic)).isNotNull();\n        int topicPerm = routeInfoManager.pickupTopicRouteData(testTopic).getQueueDatas().get(0).getPerm();\n        assertThat(PermName.isWriteable(topicPerm)).isFalse();\n        routeInfoManager.unRegisterBroker(Sets.newHashSet(BrokerBasicInfo.slaveBroker().unRegisterRequest()));\n\n        // Case 2: Register master, and slave, then unregister master, finally recover master\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), testTopic);\n        registerBrokerWithNormalTopic(BrokerBasicInfo.slaveBroker(), testTopic);\n\n        assertThat(routeInfoManager.pickupTopicRouteData(testTopic)).isNotNull();\n        topicPerm = routeInfoManager.pickupTopicRouteData(testTopic).getQueueDatas().get(0).getPerm();\n        assertThat(PermName.isWriteable(topicPerm)).isTrue();\n\n        routeInfoManager.unRegisterBroker(Sets.newHashSet(BrokerBasicInfo.defaultBroker().unRegisterRequest()));\n        assertThat(routeInfoManager.pickupTopicRouteData(testTopic)).isNotNull();\n        topicPerm = routeInfoManager.pickupTopicRouteData(testTopic).getQueueDatas().get(0).getPerm();\n        assertThat(PermName.isWriteable(topicPerm)).isFalse();\n\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), testTopic);\n        assertThat(routeInfoManager.pickupTopicRouteData(testTopic)).isNotNull();\n        topicPerm = routeInfoManager.pickupTopicRouteData(testTopic).getQueueDatas().get(0).getPerm();\n        assertThat(PermName.isWriteable(topicPerm)).isTrue();\n    }\n\n    @Test\n    public void onChannelDestroy() {\n        Channel channel = mock(Channel.class);\n\n        registerBroker(BrokerBasicInfo.defaultBroker(), channel, null, \"TestTopic\", \"TestTopic1\");\n        routeInfoManager.onChannelDestroy(channel);\n        await().atMost(Duration.ofSeconds(5)).until(() -> routeInfoManager.blockedUnRegisterRequests() == 0);\n\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\")).isNull();\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic1\")).isNull();\n\n        final BrokerBasicInfo masterBroker = BrokerBasicInfo.defaultBroker().enableActingMaster(false);\n        final BrokerBasicInfo slaveBroker = BrokerBasicInfo.slaveBroker().enableActingMaster(false);\n\n        Channel masterChannel = mock(Channel.class);\n        Channel slaveChannel = mock(Channel.class);\n\n        registerBroker(masterBroker, masterChannel, null, \"TestTopic\");\n        registerBroker(slaveBroker, slaveChannel, null, \"TestTopic\");\n\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0).getBrokerAddrs()).containsKeys(0L, 1L);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0).getBrokerAddrs())\n            .containsValues(masterBroker.brokerAddr, slaveBroker.brokerAddr);\n\n        routeInfoManager.onChannelDestroy(masterChannel);\n        await().atMost(Duration.ofSeconds(5)).until(() -> routeInfoManager.blockedUnRegisterRequests() == 0);\n\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0).getBrokerAddrs()).containsOnlyKeys(1L);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0).getBrokerAddrs())\n            .containsValues(slaveBroker.brokerAddr);\n\n        routeInfoManager.onChannelDestroy(slaveChannel);\n        await().atMost(Duration.ofSeconds(5)).until(() -> routeInfoManager.blockedUnRegisterRequests() == 0);\n\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\")).isNull();\n    }\n\n    @Test\n    public void onChannelDestroyByBrokerInfo() {\n        registerBroker(BrokerBasicInfo.defaultBroker(), mock(Channel.class), null, \"TestTopic\", \"TestTopic1\");\n        BrokerAddrInfo brokerAddrInfo = new BrokerAddrInfo(DEFAULT_CLUSTER, DEFAULT_ADDR);\n        routeInfoManager.onChannelDestroy(brokerAddrInfo);\n        await().atMost(Duration.ofSeconds(5)).until(() -> routeInfoManager.blockedUnRegisterRequests() == 0);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\")).isNull();\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic1\")).isNull();\n    }\n\n    @Test\n    public void switchBrokerRole_ChannelDestroy() {\n        final BrokerBasicInfo masterBroker = BrokerBasicInfo.defaultBroker().enableActingMaster(false);\n        final BrokerBasicInfo slaveBroker = BrokerBasicInfo.slaveBroker().enableActingMaster(false);\n\n        Channel masterChannel = mock(Channel.class);\n        Channel slaveChannel = mock(Channel.class);\n\n        registerBroker(masterBroker, masterChannel, null, \"TestTopic\");\n        registerBroker(slaveBroker, slaveChannel, null, \"TestTopic\");\n\n        // Master Down\n        routeInfoManager.onChannelDestroy(masterChannel);\n        await().atMost(Duration.ofSeconds(5)).until(() -> routeInfoManager.blockedUnRegisterRequests() == 0);\n\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0).getBrokerAddrs()).containsOnlyKeys(1L);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0).getBrokerAddrs())\n            .containsValues(slaveBroker.brokerAddr);\n\n        // Switch slave to master\n        slaveBroker.id(0).dataVersion.nextVersion();\n        registerBrokerWithNormalTopic(slaveBroker, \"TestTopic\");\n\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0).getBrokerAddrs()).containsOnlyKeys(0L);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0).getBrokerAddrs())\n            .containsValues(slaveBroker.brokerAddr);\n\n        // Old master switch to slave\n        masterBroker.id(1).dataVersion.nextVersion();\n        registerBrokerWithNormalTopic(masterBroker, \"TestTopic\");\n\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0).getBrokerAddrs()).containsKeys(0L, 1L);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0).getBrokerAddrs())\n            .containsValues(BrokerBasicInfo.defaultBroker().brokerAddr, BrokerBasicInfo.slaveBroker().brokerAddr);\n    }\n\n    @Test\n    public void keepTopicWithBrokerRegistration() {\n        RegisterBrokerResult masterResult = registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), \"TestTopic\", \"TestTopic1\");\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\")).isNotNull();\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic1\")).isNotNull();\n\n        masterResult = registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(),  \"TestTopic1\");\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\")).isNotNull();\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic1\")).isNotNull();\n    }\n\n    @Test\n    public void deleteTopicWithBrokerRegistration() {\n        config.setDeleteTopicWithBrokerRegistration(true);\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), \"TestTopic\", \"TestTopic1\");\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\")).isNotNull();\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic1\")).isNotNull();\n\n        registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(),  \"TestTopic1\");\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\")).isNull();\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic1\")).isNotNull();\n    }\n\n    @Test\n    public void deleteTopicWithBrokerRegistration2() {\n        // Register two brokers and delete a specific one by one\n        config.setDeleteTopicWithBrokerRegistration(true);\n        final BrokerBasicInfo master1 = BrokerBasicInfo.defaultBroker();\n        final BrokerBasicInfo master2 = BrokerBasicInfo.defaultBroker().name(DEFAULT_BROKER + 1).addr(DEFAULT_ADDR + 9);\n\n        registerBrokerWithNormalTopic(master1, \"TestTopic\", \"TestTopic1\");\n        registerBrokerWithNormalTopic(master2, \"TestTopic\", \"TestTopic1\");\n\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas()).hasSize(2);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic1\").getBrokerDatas()).hasSize(2);\n\n\n        registerBrokerWithNormalTopic(master1,  \"TestTopic1\");\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas()).hasSize(1);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\").getBrokerDatas().get(0).getBrokerName())\n            .isEqualTo(master2.brokerName);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic1\").getBrokerDatas()).hasSize(2);\n\n        registerBrokerWithNormalTopic(master2,  \"TestTopic1\");\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\")).isNull();\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic1\").getBrokerDatas()).hasSize(2);\n    }\n\n    @Test\n    public void registerSingleTopicWithBrokerRegistration() {\n        config.setDeleteTopicWithBrokerRegistration(true);\n        final BrokerBasicInfo master1 = BrokerBasicInfo.defaultBroker();\n\n        registerSingleTopicWithBrokerName(master1.brokerName, \"TestTopic\");\n\n        // Single topic registration failed because there is no broker connection exists\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\")).isNull();\n\n        // Register broker with TestTopic first and then register single topic TestTopic1\n        registerBrokerWithNormalTopic(master1, \"TestTopic\");\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\")).isNotNull();\n\n        registerSingleTopicWithBrokerName(master1.brokerName, \"TestTopic1\");\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic1\")).isNotNull();\n\n        // Register the two topics to keep the route info\n        registerBrokerWithNormalTopic(master1, \"TestTopic\", \"TestTopic1\");\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\")).isNotNull();\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic1\")).isNotNull();\n\n        // Cancel the TestTopic1 with broker registration\n        registerBrokerWithNormalTopic(master1, \"TestTopic\");\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\")).isNotNull();\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic1\")).isNull();\n\n        // Add TestTopic1 and cancel all the topics with broker un-registration\n        registerSingleTopicWithBrokerName(master1.brokerName, \"TestTopic1\");\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic1\")).isNotNull();\n\n        routeInfoManager.unregisterBroker(master1.clusterName, master1.brokerAddr, master1.brokerName, 0);\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic\")).isNull();\n        assertThat(routeInfoManager.pickupTopicRouteData(\"TestTopic1\")).isNull();\n\n\n    }\n\n    private RegisterBrokerResult registerBrokerWithNormalTopic(BrokerBasicInfo brokerInfo, String... topics) {\n        ConcurrentHashMap<String, TopicConfig> topicConfigConcurrentHashMap = new ConcurrentHashMap<>();\n        TopicConfig baseTopic = new TopicConfig(\"baseTopic\");\n        topicConfigConcurrentHashMap.put(baseTopic.getTopicName(), baseTopic);\n        for (final String topic : topics) {\n            TopicConfig topicConfig = new TopicConfig();\n            topicConfig.setWriteQueueNums(8);\n            topicConfig.setTopicName(topic);\n            topicConfig.setPerm(6);\n            topicConfig.setReadQueueNums(8);\n            topicConfig.setOrder(false);\n            topicConfigConcurrentHashMap.put(topic, topicConfig);\n        }\n\n        return registerBroker(brokerInfo, mock(Channel.class), topicConfigConcurrentHashMap, topics);\n    }\n\n    private RegisterBrokerResult registerBrokerWithNormalTopicAndExpire(BrokerBasicInfo brokerInfo, String... topics) {\n        ConcurrentHashMap<String, TopicConfig> topicConfigConcurrentHashMap = new ConcurrentHashMap<>();\n        TopicConfig baseTopic = new TopicConfig(\"baseTopic\");\n        topicConfigConcurrentHashMap.put(baseTopic.getTopicName(), baseTopic);\n        for (final String topic : topics) {\n            TopicConfig topicConfig = new TopicConfig();\n            topicConfig.setWriteQueueNums(8);\n            topicConfig.setTopicName(topic);\n            topicConfig.setPerm(6);\n            topicConfig.setReadQueueNums(8);\n            topicConfig.setOrder(false);\n            topicConfigConcurrentHashMap.put(topic, topicConfig);\n        }\n\n        return registerBrokerWithExpiredTime(brokerInfo, mock(Channel.class), topicConfigConcurrentHashMap, topics);\n    }\n\n    private RegisterBrokerResult registerBrokerWithOrderTopic(BrokerBasicInfo brokerBasicInfo, String... topics) {\n        ConcurrentHashMap<String, TopicConfig> topicConfigConcurrentHashMap = new ConcurrentHashMap<>();\n\n        TopicConfig baseTopic = new TopicConfig(\"baseTopic\");\n        baseTopic.setOrder(true);\n        topicConfigConcurrentHashMap.put(baseTopic.getTopicName(), baseTopic);\n        for (final String topic : topics) {\n            TopicConfig topicConfig = new TopicConfig();\n            topicConfig.setWriteQueueNums(8);\n            topicConfig.setTopicName(topic);\n            topicConfig.setPerm(6);\n            topicConfig.setReadQueueNums(8);\n            topicConfig.setOrder(true);\n            topicConfigConcurrentHashMap.put(topic, topicConfig);\n        }\n        return registerBroker(brokerBasicInfo, mock(Channel.class), topicConfigConcurrentHashMap, topics);\n    }\n\n    private RegisterBrokerResult registerBrokerWithGlobalOrderTopic(BrokerBasicInfo brokerBasicInfo, String... topics) {\n        ConcurrentHashMap<String, TopicConfig> topicConfigConcurrentHashMap = new ConcurrentHashMap<>();\n        TopicConfig baseTopic = new TopicConfig(\"baseTopic\", 1, 1);\n        baseTopic.setOrder(true);\n        topicConfigConcurrentHashMap.put(baseTopic.getTopicName(), baseTopic);\n        for (final String topic : topics) {\n            TopicConfig topicConfig = new TopicConfig();\n            topicConfig.setWriteQueueNums(1);\n            topicConfig.setTopicName(topic);\n            topicConfig.setPerm(6);\n            topicConfig.setReadQueueNums(1);\n            topicConfig.setOrder(true);\n            topicConfigConcurrentHashMap.put(topic, topicConfig);\n        }\n        return registerBroker(brokerBasicInfo, mock(Channel.class), topicConfigConcurrentHashMap, topics);\n    }\n\n    private RegisterBrokerResult registerBroker(BrokerBasicInfo brokerInfo, Channel channel,\n        ConcurrentMap<String, TopicConfig> topicConfigConcurrentHashMap, String... topics) {\n\n        if (topicConfigConcurrentHashMap == null) {\n            topicConfigConcurrentHashMap = new ConcurrentHashMap<>();\n            TopicConfig baseTopic = new TopicConfig(\"baseTopic\");\n            topicConfigConcurrentHashMap.put(baseTopic.getTopicName(), baseTopic);\n            for (final String topic : topics) {\n                TopicConfig topicConfig = new TopicConfig();\n                topicConfig.setWriteQueueNums(8);\n                topicConfig.setTopicName(topic);\n                topicConfig.setPerm(6);\n                topicConfig.setReadQueueNums(8);\n                topicConfig.setOrder(false);\n                topicConfigConcurrentHashMap.put(topic, topicConfig);\n            }\n        }\n\n        TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();\n        topicConfigSerializeWrapper.setDataVersion(brokerInfo.dataVersion);\n        topicConfigSerializeWrapper.setTopicConfigTable(topicConfigConcurrentHashMap);\n\n        return routeInfoManager.registerBroker(\n            brokerInfo.clusterName,\n            brokerInfo.brokerAddr,\n            brokerInfo.brokerName,\n            brokerInfo.brokerId,\n            brokerInfo.haAddr,\n            \"\",\n            null,\n            brokerInfo.enableActingMaster,\n            topicConfigSerializeWrapper, new ArrayList<>(), channel);\n    }\n\n    private RegisterBrokerResult registerBrokerWithExpiredTime(BrokerBasicInfo brokerInfo, Channel channel,\n                                                               ConcurrentMap<String, TopicConfig> topicConfigConcurrentHashMap, String... topics) {\n\n        if (topicConfigConcurrentHashMap == null) {\n            topicConfigConcurrentHashMap = new ConcurrentHashMap<>();\n            TopicConfig baseTopic = new TopicConfig(\"baseTopic\");\n            topicConfigConcurrentHashMap.put(baseTopic.getTopicName(), baseTopic);\n            for (final String topic : topics) {\n                TopicConfig topicConfig = new TopicConfig();\n                topicConfig.setWriteQueueNums(8);\n                topicConfig.setTopicName(topic);\n                topicConfig.setPerm(6);\n                topicConfig.setReadQueueNums(8);\n                topicConfig.setOrder(false);\n                topicConfigConcurrentHashMap.put(topic, topicConfig);\n            }\n        }\n\n        TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();\n        topicConfigSerializeWrapper.setDataVersion(brokerInfo.dataVersion);\n        topicConfigSerializeWrapper.setTopicConfigTable(topicConfigConcurrentHashMap);\n\n        return routeInfoManager.registerBroker(\n                brokerInfo.clusterName,\n                brokerInfo.brokerAddr,\n                brokerInfo.brokerName,\n                brokerInfo.brokerId,\n                brokerInfo.haAddr,\n                \"\",\n                30000L,\n                brokerInfo.enableActingMaster,\n                topicConfigSerializeWrapper, new ArrayList<>(), channel);\n    }\n\n    private void registerSingleTopicWithBrokerName(String brokerName, String... topics) {\n        for (final String topic : topics) {\n            QueueData queueData = new QueueData();\n            queueData.setBrokerName(brokerName);\n            queueData.setReadQueueNums(8);\n            queueData.setWriteQueueNums(8);\n            queueData.setPerm(6);\n            routeInfoManager.registerTopic(topic, Collections.singletonList(queueData));\n        }\n    }\n\n    static class BrokerBasicInfo {\n        String clusterName;\n        String brokerName;\n        String brokerAddr;\n        String haAddr;\n        int brokerId;\n        boolean enableActingMaster;\n\n        DataVersion dataVersion;\n\n        static BrokerBasicInfo defaultBroker() {\n            BrokerBasicInfo basicInfo = new BrokerBasicInfo();\n            DataVersion dataVersion = new DataVersion();\n            dataVersion.setCounter(new AtomicLong(1));\n            dataVersion.setTimestamp(System.currentTimeMillis());\n            basicInfo.dataVersion = dataVersion;\n\n            return basicInfo.id(0)\n                .name(DEFAULT_BROKER)\n                .cluster(DEFAULT_CLUSTER)\n                .addr(DEFAULT_ADDR)\n                .haAddr(DEFAULT_ADDR_PREFIX + \"20911\")\n                .enableActingMaster(true);\n        }\n\n        UnRegisterBrokerRequestHeader unRegisterRequest() {\n            UnRegisterBrokerRequestHeader unRegisterBrokerRequest = new UnRegisterBrokerRequestHeader();\n            unRegisterBrokerRequest.setBrokerAddr(brokerAddr);\n            unRegisterBrokerRequest.setBrokerName(brokerName);\n            unRegisterBrokerRequest.setClusterName(clusterName);\n            unRegisterBrokerRequest.setBrokerId((long) brokerId);\n            return unRegisterBrokerRequest;\n        }\n\n        static BrokerBasicInfo slaveBroker() {\n            final BrokerBasicInfo slaveBroker = defaultBroker();\n            return slaveBroker\n                .id(1)\n                .addr(DEFAULT_ADDR_PREFIX + \"30911\")\n                .haAddr(DEFAULT_ADDR_PREFIX + \"40911\")\n                .enableActingMaster(true);\n        }\n\n        BrokerBasicInfo name(String name) {\n            this.brokerName = name;\n            return this;\n        }\n\n        BrokerBasicInfo cluster(String name) {\n            this.clusterName = name;\n            return this;\n        }\n\n        BrokerBasicInfo addr(String addr) {\n            this.brokerAddr = addr;\n            return this;\n        }\n\n        BrokerBasicInfo id(int id) {\n            this.brokerId = id;\n            return this;\n        }\n\n        BrokerBasicInfo haAddr(String addr) {\n            this.haAddr = addr;\n            return this;\n        }\n\n        BrokerBasicInfo enableActingMaster(boolean enableActingMaster) {\n            this.enableActingMaster = enableActingMaster;\n            return this;\n        }\n    }\n\n}\n"
  },
  {
    "path": "namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManagerStaticRegisterTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.namesrv.routeinfo;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.namesrv.NamesrvConfig;\nimport org.apache.rocketmq.remoting.protocol.body.ClusterInfo;\nimport org.apache.rocketmq.remoting.protocol.body.TopicList;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNull;\n\npublic class RouteInfoManagerStaticRegisterTest extends RouteInfoManagerTestBase {\n    private static RouteInfoManager routeInfoManager;\n    public static String clusterName = \"cluster\";\n    public static String brokerPrefix = \"broker\";\n    public static String topicPrefix = \"topic\";\n\n    public static RouteInfoManagerTestBase.Cluster cluster;\n\n    @Before\n    public void setup() {\n        routeInfoManager = new RouteInfoManager(new NamesrvConfig(), null);\n        cluster = registerCluster(routeInfoManager,\n            clusterName,\n            brokerPrefix,\n            3,\n            3,\n            topicPrefix,\n            10);\n    }\n\n    @After\n    public void terminate() {\n        routeInfoManager.printAllPeriodically();\n\n        for (BrokerData bd : cluster.brokerDataMap.values()) {\n            unregisterBrokerAll(routeInfoManager, bd);\n        }\n    }\n\n    @Test\n    public void testGetAllClusterInfo() {\n        ClusterInfo clusterInfo = routeInfoManager.getAllClusterInfo();\n        Map<String, Set<String>> clusterAddrTable = clusterInfo.getClusterAddrTable();\n\n        assertEquals(1, clusterAddrTable.size());\n        assertEquals(cluster.getAllBrokerName(), clusterAddrTable.get(clusterName));\n    }\n\n    @Test\n    public void testGetAllTopicList() {\n        TopicList topicInfo = routeInfoManager.getAllTopicList();\n\n        assertEquals(cluster.getAllTopicName(), topicInfo.getTopicList());\n    }\n\n    @Test\n    public void testGetTopicsByCluster() {\n        TopicList topicList = routeInfoManager.getTopicsByCluster(clusterName);\n        assertEquals(cluster.getAllTopicName(), topicList.getTopicList());\n    }\n\n    @Test\n    public void testPickupTopicRouteData() {\n        String topic = getTopicName(topicPrefix, 0);\n\n        TopicRouteData topicRouteData = routeInfoManager.pickupTopicRouteData(topic);\n\n        TopicConfig topicConfig = cluster.topicConfig.get(topic);\n\n        // check broker data\n        Collections.sort(topicRouteData.getBrokerDatas());\n        List<BrokerData> ans = new ArrayList<>(cluster.brokerDataMap.values());\n        Collections.sort(ans);\n\n        assertEquals(topicRouteData.getBrokerDatas(), ans);\n\n        // check queue data\n        HashSet<String> allBrokerNameInQueueData = new HashSet<>();\n\n        for (QueueData queueData : topicRouteData.getQueueDatas()) {\n            allBrokerNameInQueueData.add(queueData.getBrokerName());\n\n            assertEquals(queueData.getWriteQueueNums(), topicConfig.getWriteQueueNums());\n            assertEquals(queueData.getReadQueueNums(), topicConfig.getReadQueueNums());\n            assertEquals(queueData.getPerm(), topicConfig.getPerm());\n            assertEquals(queueData.getTopicSysFlag(), topicConfig.getTopicSysFlag());\n        }\n\n        assertEquals(allBrokerNameInQueueData, new HashSet<>(cluster.getAllBrokerName()));\n    }\n\n    @Test\n    public void testDeleteTopic() {\n        String topic = getTopicName(topicPrefix, 0);\n        routeInfoManager.deleteTopic(topic);\n\n        assertNull(routeInfoManager.pickupTopicRouteData(topic));\n    }\n\n    @Test\n    public void testGetSystemTopicList() {\n        TopicList topicList = routeInfoManager.getSystemTopicList();\n        assertThat(topicList).isNotNull();\n    }\n\n    @Test\n    public void testGetUnitTopics() {\n        TopicList topicList = routeInfoManager.getUnitTopics();\n        assertThat(topicList).isNotNull();\n    }\n\n    @Test\n    public void testGetHasUnitSubTopicList() {\n        TopicList topicList = routeInfoManager.getHasUnitSubTopicList();\n        assertThat(topicList).isNotNull();\n    }\n\n    @Test\n    public void testGetHasUnitSubUnUnitTopicList() {\n        TopicList topicList = routeInfoManager.getHasUnitSubUnUnitTopicList();\n        assertThat(topicList).isNotNull();\n    }\n\n}\n"
  },
  {
    "path": "namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.namesrv.routeinfo;\n\nimport io.netty.channel.Channel;\nimport java.lang.reflect.Field;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.namesrv.NamesrvConfig;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.namesrv.RegisterBrokerResult;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.Mockito.mock;\n\npublic class RouteInfoManagerTest {\n\n    private static RouteInfoManager routeInfoManager;\n\n    @Before\n    public void setup() {\n        routeInfoManager = new RouteInfoManager(new NamesrvConfig(), null);\n        routeInfoManager.start();\n        testRegisterBroker();\n    }\n\n    @After\n    public void terminate() {\n        routeInfoManager.shutdown();\n        routeInfoManager.printAllPeriodically();\n        routeInfoManager.unregisterBroker(\"default-cluster\", \"127.0.0.1:10911\", \"default-broker\", 1234);\n    }\n\n    @Test\n    public void testGetAllClusterInfo() {\n        byte[] clusterInfo = routeInfoManager.getAllClusterInfo().encode();\n        assertThat(clusterInfo).isNotNull();\n    }\n\n    @Test\n    public void testQueryBrokerTopicConfig() {\n        {\n            DataVersion targetVersion = new DataVersion();\n            targetVersion.setCounter(new AtomicLong(10L));\n            targetVersion.setTimestamp(100L);\n\n            DataVersion dataVersion = routeInfoManager.queryBrokerTopicConfig(\"default-cluster\", \"127.0.0.1:10911\");\n            assertThat(dataVersion.equals(targetVersion)).isTrue();\n        }\n\n        {\n            // register broker default-cluster-1 with the same addr, then test\n            DataVersion targetVersion = new DataVersion();\n            targetVersion.setCounter(new AtomicLong(20L));\n            targetVersion.setTimestamp(200L);\n\n            ConcurrentHashMap<String, TopicConfig> topicConfigConcurrentHashMap = new ConcurrentHashMap<>();\n            topicConfigConcurrentHashMap.put(\"unit-test-0\", new TopicConfig(\"unit-test-0\"));\n            topicConfigConcurrentHashMap.put(\"unit-test-1\", new TopicConfig(\"unit-test-1\"));\n\n            TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();\n            topicConfigSerializeWrapper.setDataVersion(targetVersion);\n            topicConfigSerializeWrapper.setTopicConfigTable(topicConfigConcurrentHashMap);\n            Channel channel = mock(Channel.class);\n            RegisterBrokerResult registerBrokerResult = routeInfoManager.registerBroker(\"default-cluster-1\", \"127.0.0.1:10911\", \"default-broker-1\", 1234, \"127.0.0.1:1001\", \"\",\n                    null, topicConfigSerializeWrapper, new ArrayList<>(), channel);\n            assertThat(registerBrokerResult).isNotNull();\n\n            DataVersion dataVersion0 = routeInfoManager.queryBrokerTopicConfig(\"default-cluster\", \"127.0.0.1:10911\");\n            assertThat(targetVersion.equals(dataVersion0)).isFalse();\n\n            DataVersion dataVersion1 = routeInfoManager.queryBrokerTopicConfig(\"default-cluster-1\", \"127.0.0.1:10911\");\n            assertThat(targetVersion.equals(dataVersion1)).isTrue();\n        }\n\n        // unregister broker default-cluster-1, then test\n        {\n            routeInfoManager.unregisterBroker(\"default-cluster-1\", \"127.0.0.1:10911\", \"default-broker-1\", 1234);\n            assertThat(null != routeInfoManager.queryBrokerTopicConfig(\"default-cluster\", \"127.0.0.1:10911\")).isTrue();\n            assertThat(null == routeInfoManager.queryBrokerTopicConfig(\"default-cluster-1\", \"127.0.0.1:10911\")).isTrue();\n        }\n    }\n\n    @Test\n    public void testGetAllTopicList() {\n        byte[] topicInfo = routeInfoManager.getAllTopicList().encode();\n        Assert.assertTrue(topicInfo != null);\n        assertThat(topicInfo).isNotNull();\n    }\n\n    @Test\n    public void testRegisterBroker() {\n        DataVersion dataVersion = new DataVersion();\n        dataVersion.setCounter(new AtomicLong(10L));\n        dataVersion.setTimestamp(100L);\n\n        ConcurrentHashMap<String, TopicConfig> topicConfigConcurrentHashMap = new ConcurrentHashMap<>();\n        topicConfigConcurrentHashMap.put(\"unit-test0\", new TopicConfig(\"unit-test0\"));\n        topicConfigConcurrentHashMap.put(\"unit-test1\", new TopicConfig(\"unit-test1\"));\n\n        TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();\n        topicConfigSerializeWrapper.setDataVersion(dataVersion);\n        topicConfigSerializeWrapper.setTopicConfigTable(topicConfigConcurrentHashMap);\n        Channel channel = mock(Channel.class);\n        RegisterBrokerResult registerBrokerResult = routeInfoManager.registerBroker(\"default-cluster\", \"127.0.0.1:10911\", \"default-broker\", 1234, \"127.0.0.1:1001\", \"\",\n                null, topicConfigSerializeWrapper, new ArrayList<>(), channel);\n        assertThat(registerBrokerResult).isNotNull();\n    }\n\n    @Test\n    public void testWipeWritePermOfBrokerByLock() throws Exception {\n        Map<String, QueueData> qdMap = new HashMap<>();\n\n        QueueData qd = new QueueData();\n        qd.setPerm(PermName.PERM_READ | PermName.PERM_WRITE);\n        qd.setBrokerName(\"broker-a\");\n        qdMap.put(\"broker-a\",qd);\n        HashMap<String, Map<String, QueueData>> topicQueueTable = new HashMap<>();\n        topicQueueTable.put(\"topic-a\", qdMap);\n\n        Field filed = RouteInfoManager.class.getDeclaredField(\"topicQueueTable\");\n        filed.setAccessible(true);\n        filed.set(routeInfoManager, topicQueueTable);\n\n        int addTopicCnt = routeInfoManager.wipeWritePermOfBrokerByLock(\"broker-a\");\n        assertThat(addTopicCnt).isEqualTo(1);\n        assertThat(qd.getPerm()).isEqualTo(PermName.PERM_READ);\n\n    }\n\n    @Test\n    public void testPickupTopicRouteData() {\n        TopicRouteData result = routeInfoManager.pickupTopicRouteData(\"unit_test\");\n        assertThat(result).isNull();\n    }\n\n    @Test\n    public void testGetSystemTopicList() {\n        byte[] topicList = routeInfoManager.getSystemTopicList().encode();\n        assertThat(topicList).isNotNull();\n    }\n\n    @Test\n    public void testGetTopicsByCluster() {\n        byte[] topicList = routeInfoManager.getTopicsByCluster(\"default-cluster\").encode();\n        assertThat(topicList).isNotNull();\n    }\n\n    @Test\n    public void testGetUnitTopics() {\n        byte[] topicList = routeInfoManager.getUnitTopics().encode();\n        assertThat(topicList).isNotNull();\n    }\n\n    @Test\n    public void testGetHasUnitSubTopicList() {\n        byte[] topicList = routeInfoManager.getHasUnitSubTopicList().encode();\n        assertThat(topicList).isNotNull();\n    }\n\n    @Test\n    public void testGetHasUnitSubUnUnitTopicList() {\n        byte[] topicList = routeInfoManager.getHasUnitSubUnUnitTopicList().encode();\n        assertThat(topicList).isNotNull();\n    }\n\n    @Test\n    public void testAddWritePermOfBrokerByLock() throws Exception {\n        Map<String, QueueData> qdMap = new HashMap<>();\n        QueueData qd = new QueueData();\n        qd.setPerm(PermName.PERM_READ);\n        qd.setBrokerName(\"broker-a\");\n        qdMap.put(\"broker-a\",qd);\n        HashMap<String, Map<String, QueueData>> topicQueueTable = new HashMap<>();\n        topicQueueTable.put(\"topic-a\", qdMap);\n\n        Field filed = RouteInfoManager.class.getDeclaredField(\"topicQueueTable\");\n        filed.setAccessible(true);\n        filed.set(routeInfoManager, topicQueueTable);\n\n        int addTopicCnt = routeInfoManager.addWritePermOfBrokerByLock(\"broker-a\");\n        assertThat(addTopicCnt).isEqualTo(1);\n        assertThat(qd.getPerm()).isEqualTo(PermName.PERM_READ | PermName.PERM_WRITE);\n\n    }\n}\n"
  },
  {
    "path": "namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManagerTestBase.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.namesrv.routeinfo;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.embedded.EmbeddedChannel;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;\nimport org.apache.rocketmq.remoting.protocol.namesrv.RegisterBrokerResult;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\n\npublic class RouteInfoManagerTestBase {\n\n    protected static class Cluster {\n        ConcurrentMap<String, TopicConfig> topicConfig;\n        Map<String, BrokerData> brokerDataMap;\n\n        public Cluster(ConcurrentMap<String, TopicConfig> topicConfig, Map<String, BrokerData> brokerData) {\n            this.topicConfig = topicConfig;\n            this.brokerDataMap = brokerData;\n        }\n\n        public Set<String> getAllBrokerName() {\n            return brokerDataMap.keySet();\n        }\n\n        public Set<String> getAllTopicName() {\n            return topicConfig.keySet();\n        }\n    }\n\n    protected Cluster registerCluster(RouteInfoManager routeInfoManager, String cluster,\n                                      String brokerNamePrefix,\n                                      int brokerNameNumber,\n                                      int brokerPerName,\n                                      String topicPrefix,\n                                      int topicNumber) {\n\n        Map<String, BrokerData> brokerDataMap = new HashMap<>();\n\n        // no filterServer address\n        List<String> filterServerAddr = new ArrayList<>();\n\n        ConcurrentMap<String, TopicConfig> topicConfig = genTopicConfig(topicPrefix, topicNumber);\n\n        for (int i = 0; i < brokerNameNumber; i++) {\n            String brokerName = getBrokerName(brokerNamePrefix, i);\n\n            BrokerData brokerData = genBrokerData(cluster, brokerName, brokerPerName, true);\n\n            // avoid object reference copy\n            ConcurrentMap<String, TopicConfig> topicConfigForBroker = genTopicConfig(topicPrefix, topicNumber);\n\n            registerBrokerWithTopicConfig(routeInfoManager, brokerData, topicConfigForBroker, filterServerAddr);\n\n            // avoid object reference copy\n            brokerDataMap.put(brokerData.getBrokerName(), genBrokerData(cluster, brokerName, brokerPerName, true));\n        }\n\n        return new Cluster(topicConfig, brokerDataMap);\n    }\n\n    protected String getBrokerAddr(String cluster, String brokerName, long brokerNumber) {\n        return cluster + \"-\" + brokerName + \":\" + brokerNumber;\n    }\n\n    protected BrokerData genBrokerData(String clusterName, String brokerName, long totalBrokerNumber, boolean hasMaster) {\n        HashMap<Long, String> brokerAddrMap = new HashMap<>();\n\n        long startId = 0;\n        if (hasMaster) {\n            brokerAddrMap.put(MixAll.MASTER_ID, getBrokerAddr(clusterName, brokerName, MixAll.MASTER_ID));\n            startId = 1;\n        }\n\n        for (long i = startId; i < totalBrokerNumber; i++) {\n            brokerAddrMap.put(i, getBrokerAddr(clusterName, brokerName, i));\n        }\n\n        return new BrokerData(clusterName, brokerName, brokerAddrMap);\n    }\n\n    protected void registerBrokerWithTopicConfig(RouteInfoManager routeInfoManager, BrokerData brokerData,\n                                                 ConcurrentMap<String, TopicConfig> topicConfigTable,\n                                                 List<String> filterServerAddr) {\n\n        brokerData.getBrokerAddrs().forEach((brokerId, brokerAddr) -> {\n            registerBrokerWithTopicConfig(routeInfoManager, brokerData.getCluster(),\n                    brokerAddr,\n                    brokerData.getBrokerName(),\n                    brokerId,\n                    brokerAddr, // set ha server address the same as brokerAddr\n                    new ConcurrentHashMap<>(topicConfigTable),\n                    new ArrayList<>(filterServerAddr));\n        });\n    }\n\n    protected void unregisterBrokerAll(RouteInfoManager routeInfoManager, BrokerData brokerData) {\n        for (Map.Entry<Long, String> entry : brokerData.getBrokerAddrs().entrySet()) {\n            routeInfoManager.unregisterBroker(brokerData.getCluster(), entry.getValue(), brokerData.getBrokerName(), entry.getKey());\n        }\n    }\n\n    protected void unregisterBroker(RouteInfoManager routeInfoManager, BrokerData brokerData, long brokerId) {\n        HashMap<Long, String> brokerAddrs = brokerData.getBrokerAddrs();\n        if (brokerAddrs.containsKey(brokerId)) {\n            String address = brokerAddrs.remove(brokerId);\n            routeInfoManager.unregisterBroker(brokerData.getCluster(), address, brokerData.getBrokerName(), brokerId);\n        }\n    }\n\n    protected RegisterBrokerResult registerBrokerWithTopicConfig(RouteInfoManager routeInfoManager, String clusterName,\n                                                 String brokerAddr,\n                                                 String brokerName,\n                                                 long brokerId,\n                                                 String haServerAddr,\n                                                 ConcurrentMap<String, TopicConfig> topicConfigTable,\n                                                 List<String> filterServerAddr) {\n\n        TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();\n        topicConfigSerializeWrapper.setTopicConfigTable(topicConfigTable);\n\n        Channel channel = new EmbeddedChannel();\n        return routeInfoManager.registerBroker(clusterName,\n                brokerAddr,\n                brokerName,\n                brokerId,\n                \"\",\n                haServerAddr,\n                null,\n                topicConfigSerializeWrapper,\n                filterServerAddr,\n                channel);\n    }\n\n\n    protected String getTopicName(String topicPrefix, int topicNumber) {\n        return topicPrefix + \"-\" + topicNumber;\n    }\n\n    protected ConcurrentMap<String, TopicConfig> genTopicConfig(String topicPrefix, int topicNumber) {\n        ConcurrentMap<String, TopicConfig> topicConfigMap = new ConcurrentHashMap<>();\n\n        for (int i = 0; i < topicNumber; i++) {\n            String topicName = getTopicName(topicPrefix, i);\n\n            TopicConfig topicConfig = new TopicConfig();\n            topicConfig.setWriteQueueNums(8);\n            topicConfig.setTopicName(topicName);\n            topicConfig.setPerm(6);\n            topicConfig.setReadQueueNums(8);\n            topicConfig.setOrder(false);\n            topicConfigMap.put(topicName, topicConfig);\n        }\n\n        return topicConfigMap;\n    }\n\n    protected String getBrokerName(String brokerNamePrefix, long brokerNameNumber) {\n        return brokerNamePrefix + \"-\" + brokerNameNumber;\n    }\n\n    protected BrokerData findBrokerDataByBrokerName(List<BrokerData> data, String brokerName) {\n        return data.stream().filter(bd -> bd.getBrokerName().equals(brokerName)).findFirst().orElse(null);\n    }\n\n}\n"
  },
  {
    "path": "namesrv/src/test/resources/rmq.logback-test.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n<configuration>\n\n    <appender name=\"CONSOLE\" class=\"ch.qos.logback.core.ConsoleAppender\">\n        <layout class=\"ch.qos.logback.classic.PatternLayout\">\n            <Pattern>\n                %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n\n            </Pattern>\n        </layout>\n    </appender>\n\n    <logger name=\"org.apache.rocketmq\" level=\"error\" additivity=\"false\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </logger>\n\n    <root level=\"error\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </root>\n\n</configuration>"
  },
  {
    "path": "openmessaging/pom.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n    <parent>\n        <artifactId>rocketmq-all</artifactId>\n        <groupId>org.apache.rocketmq</groupId>\n        <version>${revision}</version>\n    </parent>\n\n    <modelVersion>4.0.0</modelVersion>\n    <artifactId>rocketmq-openmessaging</artifactId>\n    <name>rocketmq-openmessaging ${project.version}</name>\n\n    <properties>\n        <project.root>${basedir}/..</project.root>\n    </properties>\n\n    <dependencies>\n        <dependency>\n            <groupId>io.openmessaging</groupId>\n            <artifactId>openmessaging-api</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-client</artifactId>\n            <version>${project.version}</version>\n        </dependency>\n    </dependencies>\n</project>\n"
  },
  {
    "path": "openmessaging/src/main/java/io/openmessaging/rocketmq/MessagingAccessPointImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.openmessaging.rocketmq;\n\nimport io.openmessaging.KeyValue;\nimport io.openmessaging.MessagingAccessPoint;\nimport io.openmessaging.ResourceManager;\nimport io.openmessaging.consumer.PullConsumer;\nimport io.openmessaging.consumer.PushConsumer;\nimport io.openmessaging.consumer.StreamingConsumer;\nimport io.openmessaging.exception.OMSNotSupportedException;\nimport io.openmessaging.producer.Producer;\nimport io.openmessaging.rocketmq.consumer.PullConsumerImpl;\nimport io.openmessaging.rocketmq.consumer.PushConsumerImpl;\nimport io.openmessaging.rocketmq.producer.ProducerImpl;\nimport io.openmessaging.rocketmq.utils.OMSUtil;\n\npublic class MessagingAccessPointImpl implements MessagingAccessPoint {\n\n    private final KeyValue accessPointProperties;\n\n    public MessagingAccessPointImpl(final KeyValue accessPointProperties) {\n        this.accessPointProperties = accessPointProperties;\n    }\n\n    @Override\n    public KeyValue attributes() {\n        return accessPointProperties;\n    }\n\n    @Override\n    public String implVersion() {\n        return \"0.3.0\";\n    }\n\n    @Override\n    public Producer createProducer() {\n        return new ProducerImpl(this.accessPointProperties);\n    }\n\n    @Override\n    public Producer createProducer(KeyValue properties) {\n        return new ProducerImpl(OMSUtil.buildKeyValue(this.accessPointProperties, properties));\n    }\n\n    @Override\n    public PushConsumer createPushConsumer() {\n        return new PushConsumerImpl(accessPointProperties);\n    }\n\n    @Override\n    public PushConsumer createPushConsumer(KeyValue properties) {\n        return new PushConsumerImpl(OMSUtil.buildKeyValue(this.accessPointProperties, properties));\n    }\n\n    @Override\n    public PullConsumer createPullConsumer() {\n        return new PullConsumerImpl(accessPointProperties);\n    }\n\n    @Override\n    public PullConsumer createPullConsumer(KeyValue attributes) {\n        return new PullConsumerImpl(OMSUtil.buildKeyValue(this.accessPointProperties, attributes));\n    }\n\n    @Override\n    public StreamingConsumer createStreamingConsumer() {\n        return null;\n    }\n\n    @Override\n    public StreamingConsumer createStreamingConsumer(KeyValue attributes) {\n        return null;\n    }\n\n    @Override\n    public ResourceManager resourceManager() {\n        throw new OMSNotSupportedException(\"-1\", \"ResourceManager is not supported in current version.\");\n    }\n\n    @Override\n    public void startup() {\n        //Ignore\n    }\n\n    @Override\n    public void shutdown() {\n        //Ignore\n    }\n}\n"
  },
  {
    "path": "openmessaging/src/main/java/io/openmessaging/rocketmq/config/ClientConfig.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.openmessaging.rocketmq.config;\n\nimport io.openmessaging.OMSBuiltinKeys;\nimport io.openmessaging.rocketmq.domain.NonStandardKeys;\n\npublic class ClientConfig implements OMSBuiltinKeys, NonStandardKeys {\n    private String driverImpl;\n    private String accessPoints;\n    private String namespace;\n    private String producerId;\n    private String consumerId;\n    private int operationTimeout = 5000;\n    private String region;\n    private String routingSource;\n    private String routingDestination;\n    private String routingExpression;\n    private String rmqConsumerGroup;\n    private String rmqProducerGroup = \"__OMS_PRODUCER_DEFAULT_GROUP\";\n    private int rmqMaxRedeliveryTimes = 16;\n    private int rmqMessageConsumeTimeout = 15; //In minutes\n    private int rmqMaxConsumeThreadNums = 64;\n    private int rmqMinConsumeThreadNums = 20;\n    private String rmqMessageDestination;\n    private int rmqPullMessageBatchNums = 32;\n    private int rmqPullMessageCacheCapacity = 1000;\n\n    public String getDriverImpl() {\n        return driverImpl;\n    }\n\n    public void setDriverImpl(final String driverImpl) {\n        this.driverImpl = driverImpl;\n    }\n\n    public String getAccessPoints() {\n        return accessPoints;\n    }\n\n    public void setAccessPoints(final String accessPoints) {\n        this.accessPoints = accessPoints;\n    }\n\n    public String getNamespace() {\n        return namespace;\n    }\n\n    public void setNamespace(final String namespace) {\n        this.namespace = namespace;\n    }\n\n    public String getProducerId() {\n        return producerId;\n    }\n\n    public void setProducerId(final String producerId) {\n        this.producerId = producerId;\n    }\n\n    public String getConsumerId() {\n        return consumerId;\n    }\n\n    public void setConsumerId(final String consumerId) {\n        this.consumerId = consumerId;\n    }\n\n    public int getOperationTimeout() {\n        return operationTimeout;\n    }\n\n    public void setOperationTimeout(final int operationTimeout) {\n        this.operationTimeout = operationTimeout;\n    }\n\n    public String getRoutingSource() {\n        return routingSource;\n    }\n\n    public void setRoutingSource(final String routingSource) {\n        this.routingSource = routingSource;\n    }\n\n    public String getRmqConsumerGroup() {\n        return rmqConsumerGroup;\n    }\n\n    public void setRmqConsumerGroup(final String rmqConsumerGroup) {\n        this.rmqConsumerGroup = rmqConsumerGroup;\n    }\n\n    public String getRmqProducerGroup() {\n        return rmqProducerGroup;\n    }\n\n    public void setRmqProducerGroup(final String rmqProducerGroup) {\n        this.rmqProducerGroup = rmqProducerGroup;\n    }\n\n    public int getRmqMaxRedeliveryTimes() {\n        return rmqMaxRedeliveryTimes;\n    }\n\n    public void setRmqMaxRedeliveryTimes(final int rmqMaxRedeliveryTimes) {\n        this.rmqMaxRedeliveryTimes = rmqMaxRedeliveryTimes;\n    }\n\n    public int getRmqMessageConsumeTimeout() {\n        return rmqMessageConsumeTimeout;\n    }\n\n    public void setRmqMessageConsumeTimeout(final int rmqMessageConsumeTimeout) {\n        this.rmqMessageConsumeTimeout = rmqMessageConsumeTimeout;\n    }\n\n    public int getRmqMaxConsumeThreadNums() {\n        return rmqMaxConsumeThreadNums;\n    }\n\n    public void setRmqMaxConsumeThreadNums(final int rmqMaxConsumeThreadNums) {\n        this.rmqMaxConsumeThreadNums = rmqMaxConsumeThreadNums;\n    }\n\n    public int getRmqMinConsumeThreadNums() {\n        return rmqMinConsumeThreadNums;\n    }\n\n    public void setRmqMinConsumeThreadNums(final int rmqMinConsumeThreadNums) {\n        this.rmqMinConsumeThreadNums = rmqMinConsumeThreadNums;\n    }\n\n    public String getRmqMessageDestination() {\n        return rmqMessageDestination;\n    }\n\n    public void setRmqMessageDestination(final String rmqMessageDestination) {\n        this.rmqMessageDestination = rmqMessageDestination;\n    }\n\n    public int getRmqPullMessageBatchNums() {\n        return rmqPullMessageBatchNums;\n    }\n\n    public void setRmqPullMessageBatchNums(final int rmqPullMessageBatchNums) {\n        this.rmqPullMessageBatchNums = rmqPullMessageBatchNums;\n    }\n\n    public int getRmqPullMessageCacheCapacity() {\n        return rmqPullMessageCacheCapacity;\n    }\n\n    public void setRmqPullMessageCacheCapacity(final int rmqPullMessageCacheCapacity) {\n        this.rmqPullMessageCacheCapacity = rmqPullMessageCacheCapacity;\n    }\n\n    public String getRegion() {\n        return region;\n    }\n\n    public void setRegion(String region) {\n        this.region = region;\n    }\n\n    public String getRoutingDestination() {\n        return routingDestination;\n    }\n\n    public void setRoutingDestination(String routingDestination) {\n        this.routingDestination = routingDestination;\n    }\n\n    public String getRoutingExpression() {\n        return routingExpression;\n    }\n\n    public void setRoutingExpression(String routingExpression) {\n        this.routingExpression = routingExpression;\n    }\n}\n"
  },
  {
    "path": "openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/LocalMessageCache.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.openmessaging.rocketmq.consumer;\n\nimport io.openmessaging.KeyValue;\nimport io.openmessaging.Message;\nimport io.openmessaging.ServiceLifecycle;\nimport io.openmessaging.rocketmq.config.ClientConfig;\nimport io.openmessaging.rocketmq.domain.ConsumeRequest;\nimport java.util.Collections;\nimport java.util.Map;\nimport java.util.TreeMap;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.locks.ReadWriteLock;\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.consumer.ProcessQueue;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\nclass LocalMessageCache implements ServiceLifecycle {\n    private static final Logger log = LoggerFactory.getLogger(LocalMessageCache.class);\n\n    private final BlockingQueue<ConsumeRequest> consumeRequestCache;\n    private final Map<String, ConsumeRequest> consumedRequest;\n    private final ConcurrentHashMap<MessageQueue, Long> pullOffsetTable;\n    private final DefaultMQPullConsumer rocketmqPullConsumer;\n    private final ClientConfig clientConfig;\n    private final ScheduledExecutorService cleanExpireMsgExecutors;\n\n    LocalMessageCache(final DefaultMQPullConsumer rocketmqPullConsumer, final ClientConfig clientConfig) {\n        consumeRequestCache = new LinkedBlockingQueue<>(clientConfig.getRmqPullMessageCacheCapacity());\n        this.consumedRequest = new ConcurrentHashMap<>();\n        this.pullOffsetTable = new ConcurrentHashMap<>();\n        this.rocketmqPullConsumer = rocketmqPullConsumer;\n        this.clientConfig = clientConfig;\n        this.cleanExpireMsgExecutors = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(\n            \"OMS_CleanExpireMsgScheduledThread_\"));\n    }\n\n    int nextPullBatchNums() {\n        return Math.min(clientConfig.getRmqPullMessageBatchNums(), consumeRequestCache.remainingCapacity());\n    }\n\n    long nextPullOffset(MessageQueue remoteQueue) {\n        if (!pullOffsetTable.containsKey(remoteQueue)) {\n            try {\n                pullOffsetTable.putIfAbsent(remoteQueue,\n                    rocketmqPullConsumer.fetchConsumeOffset(remoteQueue, false));\n            } catch (MQClientException e) {\n                log.error(\"An error occurred in fetch consume offset process.\", e);\n            }\n        }\n        return pullOffsetTable.get(remoteQueue);\n    }\n\n    void updatePullOffset(MessageQueue remoteQueue, long nextPullOffset) {\n        pullOffsetTable.put(remoteQueue, nextPullOffset);\n    }\n\n    void submitConsumeRequest(ConsumeRequest consumeRequest) {\n        try {\n            consumeRequestCache.put(consumeRequest);\n        } catch (InterruptedException ignore) {\n        }\n    }\n\n    MessageExt poll() {\n        return poll(clientConfig.getOperationTimeout());\n    }\n\n    MessageExt poll(final KeyValue properties) {\n        int currentPollTimeout = clientConfig.getOperationTimeout();\n        if (properties.containsKey(Message.BuiltinKeys.TIMEOUT)) {\n            currentPollTimeout = properties.getInt(Message.BuiltinKeys.TIMEOUT);\n        }\n        return poll(currentPollTimeout);\n    }\n\n    private MessageExt poll(long timeout) {\n        try {\n            ConsumeRequest consumeRequest = consumeRequestCache.poll(timeout, TimeUnit.MILLISECONDS);\n            if (consumeRequest != null) {\n                MessageExt messageExt = consumeRequest.getMessageExt();\n                consumeRequest.setStartConsumeTimeMillis(System.currentTimeMillis());\n                MessageAccessor.setConsumeStartTimeStamp(messageExt, String.valueOf(consumeRequest.getStartConsumeTimeMillis()));\n                consumedRequest.put(messageExt.getMsgId(), consumeRequest);\n                return messageExt;\n            }\n        } catch (InterruptedException ignore) {\n        }\n        return null;\n    }\n\n    void ack(final String messageId) {\n        ConsumeRequest consumeRequest = consumedRequest.remove(messageId);\n        if (consumeRequest != null) {\n            long offset = consumeRequest.getProcessQueue().removeMessage(Collections.singletonList(consumeRequest.getMessageExt()));\n            try {\n                rocketmqPullConsumer.updateConsumeOffset(consumeRequest.getMessageQueue(), offset);\n            } catch (MQClientException e) {\n                log.error(\"An error occurred in update consume offset process.\", e);\n            }\n        }\n    }\n\n    void ack(final MessageQueue messageQueue, final ProcessQueue processQueue, final MessageExt messageExt) {\n        consumedRequest.remove(messageExt.getMsgId());\n        long offset = processQueue.removeMessage(Collections.singletonList(messageExt));\n        try {\n            rocketmqPullConsumer.updateConsumeOffset(messageQueue, offset);\n        } catch (MQClientException e) {\n            log.error(\"An error occurred in update consume offset process.\", e);\n        }\n    }\n\n    @Override\n    public void startup() {\n        this.cleanExpireMsgExecutors.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                cleanExpireMsg();\n            }\n        }, clientConfig.getRmqMessageConsumeTimeout(), clientConfig.getRmqMessageConsumeTimeout(), TimeUnit.MINUTES);\n    }\n\n    @Override\n    public void shutdown() {\n        ThreadUtils.shutdownGracefully(cleanExpireMsgExecutors, 5000, TimeUnit.MILLISECONDS);\n    }\n\n    private void cleanExpireMsg() {\n        for (final Map.Entry<MessageQueue, ProcessQueue> next : rocketmqPullConsumer.getDefaultMQPullConsumerImpl()\n            .getRebalanceImpl().getProcessQueueTable().entrySet()) {\n            ProcessQueue pq = next.getValue();\n            MessageQueue mq = next.getKey();\n            ReadWriteLock lockTreeMap = getLockInProcessQueue(pq);\n            if (lockTreeMap == null) {\n                log.error(\"Gets tree map lock in process queue error, may be has compatibility issue\");\n                return;\n            }\n\n            TreeMap<Long, MessageExt> msgTreeMap = pq.getMsgTreeMap();\n\n            int loop = msgTreeMap.size();\n            for (int i = 0; i < loop; i++) {\n                MessageExt msg = null;\n                try {\n                    lockTreeMap.readLock().lockInterruptibly();\n                    try {\n                        if (!msgTreeMap.isEmpty()) {\n                            msg = msgTreeMap.firstEntry().getValue();\n                            if (System.currentTimeMillis() - Long.parseLong(MessageAccessor.getConsumeStartTimeStamp(msg))\n                                > clientConfig.getRmqMessageConsumeTimeout() * 60 * 1000) {\n                                //Expired, ack and remove it.\n                            } else {\n                                break;\n                            }\n                        } else {\n                            break;\n                        }\n                    } finally {\n                        lockTreeMap.readLock().unlock();\n                    }\n                } catch (InterruptedException e) {\n                    log.error(\"Gets expired message exception\", e);\n                }\n\n                try {\n                    rocketmqPullConsumer.sendMessageBack(msg, 3);\n                    log.info(\"Send expired msg back. topic={}, msgId={}, storeHost={}, queueId={}, queueOffset={}\",\n                        msg.getTopic(), msg.getMsgId(), msg.getStoreHost(), msg.getQueueId(), msg.getQueueOffset());\n                    ack(mq, pq, msg);\n                } catch (Exception e) {\n                    log.error(\"Send back expired msg exception\", e);\n                }\n            }\n        }\n    }\n\n    private ReadWriteLock getLockInProcessQueue(ProcessQueue pq) {\n        try {\n            return (ReadWriteLock) FieldUtils.readDeclaredField(pq, \"lockTreeMap\", true);\n        } catch (IllegalAccessException e) {\n            return null;\n        }\n    }\n}\n"
  },
  {
    "path": "openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.openmessaging.rocketmq.consumer;\n\nimport io.openmessaging.KeyValue;\nimport io.openmessaging.Message;\nimport io.openmessaging.OMSBuiltinKeys;\nimport io.openmessaging.consumer.PullConsumer;\nimport io.openmessaging.exception.OMSRuntimeException;\nimport io.openmessaging.rocketmq.config.ClientConfig;\nimport io.openmessaging.rocketmq.domain.ConsumeRequest;\nimport io.openmessaging.rocketmq.utils.BeanUtils;\nimport io.openmessaging.rocketmq.utils.OMSUtil;\nimport org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;\nimport org.apache.rocketmq.client.consumer.MQPullConsumer;\nimport org.apache.rocketmq.client.consumer.MQPullConsumerScheduleService;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullTaskCallback;\nimport org.apache.rocketmq.client.consumer.PullTaskContext;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.consumer.ProcessQueue;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class PullConsumerImpl implements PullConsumer {\n    private static final Logger log = LoggerFactory.getLogger(PullConsumerImpl.class);\n\n    private final DefaultMQPullConsumer rocketmqPullConsumer;\n    private final KeyValue properties;\n    private boolean started = false;\n    private final MQPullConsumerScheduleService pullConsumerScheduleService;\n    private final LocalMessageCache localMessageCache;\n    private final ClientConfig clientConfig;\n\n    public PullConsumerImpl(final KeyValue properties) {\n        this.properties = properties;\n        this.clientConfig = BeanUtils.populate(properties, ClientConfig.class);\n\n        String consumerGroup = clientConfig.getConsumerId();\n        if (null == consumerGroup || consumerGroup.isEmpty()) {\n            throw new OMSRuntimeException(\"-1\", \"Consumer Group is necessary for RocketMQ, please set it.\");\n        }\n        pullConsumerScheduleService = new MQPullConsumerScheduleService(consumerGroup);\n\n        this.rocketmqPullConsumer = pullConsumerScheduleService.getDefaultMQPullConsumer();\n\n        if (\"true\".equalsIgnoreCase(System.getenv(\"OMS_RMQ_DIRECT_NAME_SRV\"))) {\n            String accessPoints = clientConfig.getAccessPoints();\n            if (accessPoints == null || accessPoints.isEmpty()) {\n                throw new OMSRuntimeException(\"-1\", \"OMS AccessPoints is null or empty.\");\n            }\n            this.rocketmqPullConsumer.setNamesrvAddr(accessPoints.replace(',', ';'));\n        }\n\n        this.rocketmqPullConsumer.setConsumerGroup(consumerGroup);\n\n        int maxReDeliveryTimes = clientConfig.getRmqMaxRedeliveryTimes();\n        this.rocketmqPullConsumer.setMaxReconsumeTimes(maxReDeliveryTimes);\n\n        String consumerId = OMSUtil.buildInstanceName();\n        this.rocketmqPullConsumer.setInstanceName(consumerId);\n        properties.put(OMSBuiltinKeys.CONSUMER_ID, consumerId);\n\n        this.rocketmqPullConsumer.setLanguage(LanguageCode.OMS);\n\n        this.localMessageCache = new LocalMessageCache(this.rocketmqPullConsumer, clientConfig);\n    }\n\n    @Override\n    public KeyValue attributes() {\n        return properties;\n    }\n\n    @Override\n    public PullConsumer attachQueue(String queueName) {\n        registerPullTaskCallback(queueName);\n        return this;\n    }\n\n    @Override\n    public PullConsumer attachQueue(String queueName, KeyValue attributes) {\n        registerPullTaskCallback(queueName);\n        return this;\n    }\n\n    @Override\n    public PullConsumer detachQueue(String queueName) {\n        this.rocketmqPullConsumer.getRegisterTopics().remove(queueName);\n        return this;\n    }\n\n    @Override\n    public Message receive() {\n        MessageExt rmqMsg = localMessageCache.poll();\n        return rmqMsg == null ? null : OMSUtil.msgConvert(rmqMsg);\n    }\n\n    @Override\n    public Message receive(final KeyValue properties) {\n        MessageExt rmqMsg = localMessageCache.poll(properties);\n        return rmqMsg == null ? null : OMSUtil.msgConvert(rmqMsg);\n    }\n\n    @Override\n    public void ack(final String messageId) {\n        localMessageCache.ack(messageId);\n    }\n\n    @Override\n    public void ack(final String messageId, final KeyValue properties) {\n        localMessageCache.ack(messageId);\n    }\n\n    @Override\n    public synchronized void startup() {\n        if (!started) {\n            try {\n                this.pullConsumerScheduleService.start();\n                this.localMessageCache.startup();\n            } catch (MQClientException e) {\n                throw new OMSRuntimeException(\"-1\", e);\n            }\n        }\n        this.started = true;\n    }\n\n    private void registerPullTaskCallback(final String targetQueueName) {\n        this.pullConsumerScheduleService.registerPullTaskCallback(targetQueueName, new PullTaskCallback() {\n            @Override\n            public void doPullTask(final MessageQueue mq, final PullTaskContext context) {\n                MQPullConsumer consumer = context.getPullConsumer();\n                try {\n                    long offset = localMessageCache.nextPullOffset(mq);\n\n                    PullResult pullResult = consumer.pull(mq, \"*\",\n                        offset, localMessageCache.nextPullBatchNums());\n                    ProcessQueue pq = rocketmqPullConsumer.getDefaultMQPullConsumerImpl().getRebalanceImpl()\n                        .getProcessQueueTable().get(mq);\n                    switch (pullResult.getPullStatus()) {\n                        case FOUND:\n                            if (pq != null) {\n                                pq.putMessage(pullResult.getMsgFoundList());\n                                for (final MessageExt messageExt : pullResult.getMsgFoundList()) {\n                                    localMessageCache.submitConsumeRequest(new ConsumeRequest(messageExt, mq, pq));\n                                }\n                            }\n                            break;\n                        default:\n                            break;\n                    }\n                    localMessageCache.updatePullOffset(mq, pullResult.getNextBeginOffset());\n                } catch (Exception e) {\n                    log.error(\"An error occurred in pull message process.\", e);\n                }\n            }\n        });\n    }\n\n    @Override\n    public synchronized void shutdown() {\n        if (this.started) {\n            this.localMessageCache.shutdown();\n            this.pullConsumerScheduleService.shutdown();\n            this.rocketmqPullConsumer.shutdown();\n        }\n        this.started = false;\n    }\n}\n"
  },
  {
    "path": "openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PushConsumerImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.openmessaging.rocketmq.consumer;\n\nimport io.openmessaging.BytesMessage;\nimport io.openmessaging.KeyValue;\nimport io.openmessaging.OMS;\nimport io.openmessaging.OMSBuiltinKeys;\nimport io.openmessaging.consumer.MessageListener;\nimport io.openmessaging.consumer.PushConsumer;\nimport io.openmessaging.exception.OMSRuntimeException;\nimport io.openmessaging.interceptor.ConsumerInterceptor;\nimport io.openmessaging.rocketmq.config.ClientConfig;\nimport io.openmessaging.rocketmq.domain.NonStandardKeys;\nimport io.openmessaging.rocketmq.utils.BeanUtils;\nimport io.openmessaging.rocketmq.utils.OMSUtil;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\n\npublic class PushConsumerImpl implements PushConsumer {\n    private final DefaultMQPushConsumer rocketmqPushConsumer;\n    private final KeyValue properties;\n    private boolean started = false;\n    private final Map<String, MessageListener> subscribeTable = new ConcurrentHashMap<>();\n    private final ClientConfig clientConfig;\n\n    public PushConsumerImpl(final KeyValue properties) {\n        this.rocketmqPushConsumer = new DefaultMQPushConsumer();\n        this.properties = properties;\n        this.clientConfig = BeanUtils.populate(properties, ClientConfig.class);\n\n        if (\"true\".equalsIgnoreCase(System.getenv(\"OMS_RMQ_DIRECT_NAME_SRV\"))) {\n            String accessPoints = clientConfig.getAccessPoints();\n            if (accessPoints == null || accessPoints.isEmpty()) {\n                throw new OMSRuntimeException(\"-1\", \"OMS AccessPoints is null or empty.\");\n            }\n            this.rocketmqPushConsumer.setNamesrvAddr(accessPoints.replace(',', ';'));\n        }\n\n        String consumerGroup = clientConfig.getConsumerId();\n        if (null == consumerGroup || consumerGroup.isEmpty()) {\n            throw new OMSRuntimeException(\"-1\", \"Consumer Group is necessary for RocketMQ, please set it.\");\n        }\n        this.rocketmqPushConsumer.setConsumerGroup(consumerGroup);\n        this.rocketmqPushConsumer.setMaxReconsumeTimes(clientConfig.getRmqMaxRedeliveryTimes());\n        this.rocketmqPushConsumer.setConsumeTimeout(clientConfig.getRmqMessageConsumeTimeout());\n        this.rocketmqPushConsumer.setConsumeThreadMax(clientConfig.getRmqMaxConsumeThreadNums());\n        this.rocketmqPushConsumer.setConsumeThreadMin(clientConfig.getRmqMinConsumeThreadNums());\n\n        String consumerId = OMSUtil.buildInstanceName();\n        this.rocketmqPushConsumer.setInstanceName(consumerId);\n        properties.put(OMSBuiltinKeys.CONSUMER_ID, consumerId);\n        this.rocketmqPushConsumer.setLanguage(LanguageCode.OMS);\n\n        this.rocketmqPushConsumer.registerMessageListener(new MessageListenerImpl());\n    }\n\n    @Override\n    public KeyValue attributes() {\n        return properties;\n    }\n\n    @Override\n    public void resume() {\n        this.rocketmqPushConsumer.resume();\n    }\n\n    @Override\n    public void suspend() {\n        this.rocketmqPushConsumer.suspend();\n    }\n\n    @Override\n    public void suspend(long timeout) {\n\n    }\n\n    @Override\n    public boolean isSuspended() {\n        return this.rocketmqPushConsumer.isPause();\n    }\n\n    @Override\n    public PushConsumer attachQueue(final String queueName, final MessageListener listener) {\n        this.subscribeTable.put(queueName, listener);\n        try {\n            this.rocketmqPushConsumer.subscribe(queueName, \"*\");\n        } catch (MQClientException e) {\n            throw new OMSRuntimeException(\"-1\", String.format(\"RocketMQ push consumer can't attach to %s.\", queueName));\n        }\n        return this;\n    }\n\n    @Override\n    public PushConsumer attachQueue(String queueName, MessageListener listener, KeyValue attributes) {\n        return this.attachQueue(queueName, listener);\n    }\n\n    @Override\n    public PushConsumer detachQueue(String queueName) {\n        this.subscribeTable.remove(queueName);\n        try {\n            this.rocketmqPushConsumer.unsubscribe(queueName);\n        } catch (Exception e) {\n            throw new OMSRuntimeException(\"-1\", String.format(\"RocketMQ push consumer fails to unsubscribe topic: %s\", queueName));\n        }\n        return null;\n    }\n\n    @Override\n    public void addInterceptor(ConsumerInterceptor interceptor) {\n\n    }\n\n    @Override\n    public void removeInterceptor(ConsumerInterceptor interceptor) {\n\n    }\n\n    @Override\n    public synchronized void startup() {\n        if (!started) {\n            try {\n                this.rocketmqPushConsumer.start();\n            } catch (MQClientException e) {\n                throw new OMSRuntimeException(\"-1\", e);\n            }\n        }\n        this.started = true;\n    }\n\n    @Override\n    public synchronized void shutdown() {\n        if (this.started) {\n            this.rocketmqPushConsumer.shutdown();\n        }\n        this.started = false;\n    }\n\n    class MessageListenerImpl implements MessageListenerConcurrently {\n\n        @Override\n        public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> rmqMsgList,\n            ConsumeConcurrentlyContext contextRMQ) {\n            MessageExt rmqMsg = rmqMsgList.get(0);\n            BytesMessage omsMsg = OMSUtil.msgConvert(rmqMsg);\n\n            MessageListener listener = PushConsumerImpl.this.subscribeTable.get(rmqMsg.getTopic());\n\n            if (listener == null) {\n                throw new OMSRuntimeException(\"-1\",\n                    String.format(\"The topic/queue %s isn't attached to this consumer\", rmqMsg.getTopic()));\n            }\n\n            final KeyValue contextProperties = OMS.newKeyValue();\n            final CountDownLatch sync = new CountDownLatch(1);\n\n            contextProperties.put(NonStandardKeys.MESSAGE_CONSUME_STATUS, ConsumeConcurrentlyStatus.RECONSUME_LATER.name());\n\n            MessageListener.Context context = new MessageListener.Context() {\n                @Override\n                public KeyValue attributes() {\n                    return contextProperties;\n                }\n\n                @Override\n                public void ack() {\n                    sync.countDown();\n                    contextProperties.put(NonStandardKeys.MESSAGE_CONSUME_STATUS,\n                        ConsumeConcurrentlyStatus.CONSUME_SUCCESS.name());\n                }\n            };\n            long begin = System.currentTimeMillis();\n            listener.onReceived(omsMsg, context);\n            long costs = System.currentTimeMillis() - begin;\n            long timeoutMills = clientConfig.getRmqMessageConsumeTimeout() * 60 * 1000;\n            try {\n                sync.await(Math.max(0, timeoutMills - costs), TimeUnit.MILLISECONDS);\n            } catch (InterruptedException ignore) {\n            }\n\n            return ConsumeConcurrentlyStatus.valueOf(contextProperties.getString(NonStandardKeys.MESSAGE_CONSUME_STATUS));\n        }\n    }\n}\n"
  },
  {
    "path": "openmessaging/src/main/java/io/openmessaging/rocketmq/domain/BytesMessageImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.openmessaging.rocketmq.domain;\n\nimport io.openmessaging.BytesMessage;\nimport io.openmessaging.KeyValue;\nimport io.openmessaging.Message;\nimport io.openmessaging.OMS;\nimport io.openmessaging.exception.OMSMessageFormatException;\nimport org.apache.commons.lang3.builder.ToStringBuilder;\n\npublic class BytesMessageImpl implements BytesMessage {\n    private KeyValue sysHeaders;\n    private KeyValue userHeaders;\n    private byte[] body;\n\n    public BytesMessageImpl() {\n        this.sysHeaders = OMS.newKeyValue();\n        this.userHeaders = OMS.newKeyValue();\n    }\n\n    @Override\n    public <T> T getBody(Class<T> type) throws OMSMessageFormatException {\n        if (type == byte[].class) {\n            return (T)body;\n        }\n\n        throw new OMSMessageFormatException(\"\", \"Cannot assign byte[] to \" + type.getName());\n    }\n\n    @Override\n    public BytesMessage setBody(final byte[] body) {\n        this.body = body;\n        return this;\n    }\n\n    @Override\n    public KeyValue sysHeaders() {\n        return sysHeaders;\n    }\n\n    @Override\n    public KeyValue userHeaders() {\n        return userHeaders;\n    }\n\n    @Override\n    public Message putSysHeaders(String key, int value) {\n        sysHeaders.put(key, value);\n        return this;\n    }\n\n    @Override\n    public Message putSysHeaders(String key, long value) {\n        sysHeaders.put(key, value);\n        return this;\n    }\n\n    @Override\n    public Message putSysHeaders(String key, double value) {\n        sysHeaders.put(key, value);\n        return this;\n    }\n\n    @Override\n    public Message putSysHeaders(String key, String value) {\n        sysHeaders.put(key, value);\n        return this;\n    }\n\n    @Override\n    public Message putUserHeaders(String key, int value) {\n        userHeaders.put(key, value);\n        return this;\n    }\n\n    @Override\n    public Message putUserHeaders(String key, long value) {\n        userHeaders.put(key, value);\n        return this;\n    }\n\n    @Override\n    public Message putUserHeaders(String key, double value) {\n        userHeaders.put(key, value);\n        return this;\n    }\n\n    @Override\n    public Message putUserHeaders(String key, String value) {\n        userHeaders.put(key, value);\n        return this;\n    }\n\n    @Override\n    public String toString() {\n        return ToStringBuilder.reflectionToString(this);\n    }\n}\n"
  },
  {
    "path": "openmessaging/src/main/java/io/openmessaging/rocketmq/domain/ConsumeRequest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.openmessaging.rocketmq.domain;\n\nimport org.apache.rocketmq.client.impl.consumer.ProcessQueue;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\npublic class ConsumeRequest {\n    private final MessageExt messageExt;\n    private final MessageQueue messageQueue;\n    private final ProcessQueue processQueue;\n    private long startConsumeTimeMillis;\n\n    public ConsumeRequest(final MessageExt messageExt, final MessageQueue messageQueue,\n        final ProcessQueue processQueue) {\n        this.messageExt = messageExt;\n        this.messageQueue = messageQueue;\n        this.processQueue = processQueue;\n    }\n\n    public MessageExt getMessageExt() {\n        return messageExt;\n    }\n\n    public MessageQueue getMessageQueue() {\n        return messageQueue;\n    }\n\n    public ProcessQueue getProcessQueue() {\n        return processQueue;\n    }\n\n    public long getStartConsumeTimeMillis() {\n        return startConsumeTimeMillis;\n    }\n\n    public void setStartConsumeTimeMillis(final long startConsumeTimeMillis) {\n        this.startConsumeTimeMillis = startConsumeTimeMillis;\n    }\n}\n"
  },
  {
    "path": "openmessaging/src/main/java/io/openmessaging/rocketmq/domain/NonStandardKeys.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.openmessaging.rocketmq.domain;\n\npublic interface NonStandardKeys {\n    String CONSUMER_GROUP = \"rmq.consumer.group\";\n    String PRODUCER_GROUP = \"rmq.producer.group\";\n    String MAX_REDELIVERY_TIMES = \"rmq.max.redelivery.times\";\n    String MESSAGE_CONSUME_TIMEOUT = \"rmq.message.consume.timeout\";\n    String MAX_CONSUME_THREAD_NUMS = \"rmq.max.consume.thread.nums\";\n    String MIN_CONSUME_THREAD_NUMS = \"rmq.min.consume.thread.nums\";\n    String MESSAGE_CONSUME_STATUS = \"rmq.message.consume.status\";\n    String MESSAGE_DESTINATION = \"rmq.message.destination\";\n    String PULL_MESSAGE_BATCH_NUMS = \"rmq.pull.message.batch.nums\";\n    String PULL_MESSAGE_CACHE_CAPACITY = \"rmq.pull.message.cache.capacity\";\n}\n"
  },
  {
    "path": "openmessaging/src/main/java/io/openmessaging/rocketmq/domain/RocketMQConstants.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.openmessaging.rocketmq.domain;\n\npublic interface RocketMQConstants {\n\n    /**\n     * Key of scheduled message delivery time\n     */\n    String START_DELIVER_TIME = \"__STARTDELIVERTIME\";\n\n}\n"
  },
  {
    "path": "openmessaging/src/main/java/io/openmessaging/rocketmq/domain/SendResultImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.openmessaging.rocketmq.domain;\n\nimport io.openmessaging.KeyValue;\nimport io.openmessaging.producer.SendResult;\n\npublic class SendResultImpl implements SendResult {\n    private String messageId;\n    private KeyValue properties;\n\n    public SendResultImpl(final String messageId, final KeyValue properties) {\n        this.messageId = messageId;\n        this.properties = properties;\n    }\n\n    @Override\n    public String messageId() {\n        return messageId;\n    }\n\n    public KeyValue properties() {\n        return properties;\n    }\n}\n"
  },
  {
    "path": "openmessaging/src/main/java/io/openmessaging/rocketmq/producer/AbstractOMSProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.openmessaging.rocketmq.producer;\n\nimport io.openmessaging.BytesMessage;\nimport io.openmessaging.KeyValue;\nimport io.openmessaging.Message;\nimport io.openmessaging.MessageFactory;\nimport io.openmessaging.OMSBuiltinKeys;\nimport io.openmessaging.ServiceLifecycle;\nimport io.openmessaging.exception.OMSMessageFormatException;\nimport io.openmessaging.exception.OMSNotSupportedException;\nimport io.openmessaging.exception.OMSRuntimeException;\nimport io.openmessaging.exception.OMSTimeOutException;\nimport io.openmessaging.rocketmq.config.ClientConfig;\nimport io.openmessaging.rocketmq.domain.BytesMessageImpl;\nimport io.openmessaging.rocketmq.utils.BeanUtils;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.remoting.exception.RemotingConnectException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\n\nimport static io.openmessaging.rocketmq.utils.OMSUtil.buildInstanceName;\n\nabstract class AbstractOMSProducer implements ServiceLifecycle, MessageFactory {\n    final KeyValue properties;\n    final DefaultMQProducer rocketmqProducer;\n    private boolean started = false;\n    private final ClientConfig clientConfig;\n\n    AbstractOMSProducer(final KeyValue properties) {\n        this.properties = properties;\n        this.rocketmqProducer = new DefaultMQProducer();\n        this.clientConfig = BeanUtils.populate(properties, ClientConfig.class);\n\n        if (\"true\".equalsIgnoreCase(System.getenv(\"OMS_RMQ_DIRECT_NAME_SRV\"))) {\n            String accessPoints = clientConfig.getAccessPoints();\n            if (accessPoints == null || accessPoints.isEmpty()) {\n                throw new OMSRuntimeException(\"-1\", \"OMS AccessPoints is null or empty.\");\n            }\n\n            this.rocketmqProducer.setNamesrvAddr(accessPoints.replace(',', ';'));\n        }\n\n        this.rocketmqProducer.setProducerGroup(clientConfig.getRmqProducerGroup());\n\n        String producerId = buildInstanceName();\n        this.rocketmqProducer.setSendMsgTimeout(clientConfig.getOperationTimeout());\n        this.rocketmqProducer.setInstanceName(producerId);\n        this.rocketmqProducer.setMaxMessageSize(1024 * 1024 * 4);\n        this.rocketmqProducer.setLanguage(LanguageCode.OMS);\n        properties.put(OMSBuiltinKeys.PRODUCER_ID, producerId);\n    }\n\n    @Override\n    public synchronized void startup() {\n        if (!started) {\n            try {\n                this.rocketmqProducer.start();\n            } catch (MQClientException e) {\n                throw new OMSRuntimeException(\"-1\", e);\n            }\n        }\n        this.started = true;\n    }\n\n    @Override\n    public synchronized void shutdown() {\n        if (this.started) {\n            this.rocketmqProducer.shutdown();\n        }\n        this.started = false;\n    }\n\n    OMSRuntimeException checkProducerException(String topic, String msgId, Throwable e) {\n        if (e instanceof MQClientException) {\n            if (e.getCause() != null) {\n                if (e.getCause() instanceof RemotingTimeoutException) {\n                    return new OMSTimeOutException(\"-1\", String.format(\"Send message to broker timeout, %dms, Topic=%s, msgId=%s\",\n                        this.rocketmqProducer.getSendMsgTimeout(), topic, msgId), e);\n                } else if (e.getCause() instanceof MQBrokerException || e.getCause() instanceof RemotingConnectException) {\n                    if (e.getCause() instanceof MQBrokerException) {\n                        MQBrokerException brokerException = (MQBrokerException) e.getCause();\n                        return new OMSRuntimeException(\"-1\", String.format(\"Received a broker exception, Topic=%s, msgId=%s, %s\",\n                            topic, msgId, brokerException.getErrorMessage()), e);\n                    }\n\n                    if (e.getCause() instanceof RemotingConnectException) {\n                        RemotingConnectException connectException = (RemotingConnectException)e.getCause();\n                        return new OMSRuntimeException(\"-1\",\n                            String.format(\"Network connection experiences failures. Topic=%s, msgId=%s, %s\",\n                                topic, msgId, connectException.getMessage()),\n                            e);\n                    }\n                }\n            }\n            // Exception thrown by local.\n            else {\n                MQClientException clientException = (MQClientException) e;\n                if (-1 == clientException.getResponseCode()) {\n                    return new OMSRuntimeException(\"-1\", String.format(\"Topic does not exist, Topic=%s, msgId=%s\",\n                        topic, msgId), e);\n                } else if (ResponseCode.MESSAGE_ILLEGAL == clientException.getResponseCode()) {\n                    return new OMSMessageFormatException(\"-1\", String.format(\"A illegal message for RocketMQ, Topic=%s, msgId=%s\",\n                        topic, msgId), e);\n                }\n            }\n        }\n        return new OMSRuntimeException(\"-1\", \"Send message to RocketMQ broker failed.\", e);\n    }\n\n    protected void checkMessageType(Message message) {\n        if (!(message instanceof BytesMessage)) {\n            throw new OMSNotSupportedException(\"-1\", \"Only BytesMessage is supported.\");\n        }\n    }\n\n    @Override\n    public BytesMessage createBytesMessage(String queue, byte[] body) {\n        BytesMessage message = new BytesMessageImpl();\n        message.setBody(body);\n        message.sysHeaders().put(Message.BuiltinKeys.DESTINATION, queue);\n        return message;\n    }\n}\n"
  },
  {
    "path": "openmessaging/src/main/java/io/openmessaging/rocketmq/producer/ProducerImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.openmessaging.rocketmq.producer;\n\nimport io.openmessaging.BytesMessage;\nimport io.openmessaging.KeyValue;\nimport io.openmessaging.Message;\nimport io.openmessaging.Promise;\nimport io.openmessaging.exception.OMSRuntimeException;\nimport io.openmessaging.interceptor.ProducerInterceptor;\nimport io.openmessaging.producer.BatchMessageSender;\nimport io.openmessaging.producer.LocalTransactionExecutor;\nimport io.openmessaging.producer.Producer;\nimport io.openmessaging.producer.SendResult;\nimport io.openmessaging.rocketmq.promise.DefaultPromise;\nimport io.openmessaging.rocketmq.utils.OMSUtil;\nimport org.apache.rocketmq.client.producer.SendCallback;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\nimport static io.openmessaging.rocketmq.utils.OMSUtil.msgConvert;\n\npublic class ProducerImpl extends AbstractOMSProducer implements Producer {\n\n    private static final Logger log = LoggerFactory.getLogger(ProducerImpl.class);\n\n    public ProducerImpl(final KeyValue properties) {\n        super(properties);\n    }\n\n    @Override\n    public KeyValue attributes() {\n        return properties;\n    }\n\n    @Override\n    public SendResult send(final Message message) {\n        return send(message, this.rocketmqProducer.getSendMsgTimeout());\n    }\n\n    @Override\n    public SendResult send(final Message message, final KeyValue properties) {\n        long timeout = properties.containsKey(Message.BuiltinKeys.TIMEOUT)\n            ? properties.getInt(Message.BuiltinKeys.TIMEOUT) : this.rocketmqProducer.getSendMsgTimeout();\n        return send(message, timeout);\n    }\n\n    @Override\n    public SendResult send(Message message, LocalTransactionExecutor branchExecutor, KeyValue attributes) {\n        return null;\n    }\n\n    private SendResult send(final Message message, long timeout) {\n        checkMessageType(message);\n        org.apache.rocketmq.common.message.Message rmqMessage = msgConvert((BytesMessage) message);\n        try {\n            org.apache.rocketmq.client.producer.SendResult rmqResult = this.rocketmqProducer.send(rmqMessage, timeout);\n            if (!rmqResult.getSendStatus().equals(SendStatus.SEND_OK)) {\n                log.error(String.format(\"Send message to RocketMQ failed, %s\", message));\n                throw new OMSRuntimeException(\"-1\", \"Send message to RocketMQ broker failed.\");\n            }\n            message.sysHeaders().put(Message.BuiltinKeys.MESSAGE_ID, rmqResult.getMsgId());\n            return OMSUtil.sendResultConvert(rmqResult);\n        } catch (Exception e) {\n            log.error(String.format(\"Send message to RocketMQ failed, %s\", message), e);\n            throw checkProducerException(rmqMessage.getTopic(), message.sysHeaders().getString(Message.BuiltinKeys.MESSAGE_ID), e);\n        }\n    }\n\n    @Override\n    public Promise<SendResult> sendAsync(final Message message) {\n        return sendAsync(message, this.rocketmqProducer.getSendMsgTimeout());\n    }\n\n    @Override\n    public Promise<SendResult> sendAsync(final Message message, final KeyValue properties) {\n        long timeout = properties.containsKey(Message.BuiltinKeys.TIMEOUT)\n            ? properties.getInt(Message.BuiltinKeys.TIMEOUT) : this.rocketmqProducer.getSendMsgTimeout();\n        return sendAsync(message, timeout);\n    }\n\n    private Promise<SendResult> sendAsync(final Message message, long timeout) {\n        checkMessageType(message);\n        org.apache.rocketmq.common.message.Message rmqMessage = msgConvert((BytesMessage) message);\n        final Promise<SendResult> promise = new DefaultPromise<>();\n        try {\n            this.rocketmqProducer.send(rmqMessage, new SendCallback() {\n                @Override\n                public void onSuccess(final org.apache.rocketmq.client.producer.SendResult rmqResult) {\n                    message.sysHeaders().put(Message.BuiltinKeys.MESSAGE_ID, rmqResult.getMsgId());\n                    promise.set(OMSUtil.sendResultConvert(rmqResult));\n                }\n\n                @Override\n                public void onException(final Throwable e) {\n                    promise.setFailure(e);\n                }\n            }, timeout);\n        } catch (Exception e) {\n            promise.setFailure(e);\n        }\n        return promise;\n    }\n\n    @Override\n    public void sendOneway(final Message message) {\n        checkMessageType(message);\n        org.apache.rocketmq.common.message.Message rmqMessage = msgConvert((BytesMessage) message);\n        try {\n            this.rocketmqProducer.sendOneway(rmqMessage);\n        } catch (Exception ignore) { //Ignore the oneway exception.\n        }\n    }\n\n    @Override\n    public void sendOneway(final Message message, final KeyValue properties) {\n        sendOneway(message);\n    }\n\n    @Override\n    public BatchMessageSender createBatchMessageSender() {\n        return null;\n    }\n\n    @Override\n    public void addInterceptor(ProducerInterceptor interceptor) {\n\n    }\n\n    @Override\n    public void removeInterceptor(ProducerInterceptor interceptor) {\n\n    }\n}\n"
  },
  {
    "path": "openmessaging/src/main/java/io/openmessaging/rocketmq/promise/DefaultPromise.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.openmessaging.rocketmq.promise;\n\nimport io.openmessaging.Promise;\nimport io.openmessaging.FutureListener;\nimport io.openmessaging.exception.OMSRuntimeException;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\npublic class DefaultPromise<V> implements Promise<V> {\n    private static final Logger LOG = LoggerFactory.getLogger(DefaultPromise.class);\n    private final Object lock = new Object();\n    private volatile FutureState state = FutureState.DOING;\n    private V result = null;\n    private long timeout;\n    private long createTime;\n    private Throwable exception = null;\n    private List<FutureListener<V>> promiseListenerList;\n\n    public DefaultPromise() {\n        createTime = System.currentTimeMillis();\n        promiseListenerList = new ArrayList<>();\n        timeout = 5000;\n    }\n\n    @Override\n    public boolean cancel(final boolean mayInterruptIfRunning) {\n        return false;\n    }\n\n    @Override\n    public boolean isCancelled() {\n        return state.isCancelledState();\n    }\n\n    @Override\n    public boolean isDone() {\n        return state.isDoneState();\n    }\n\n    @Override\n    public V get() {\n        return result;\n    }\n\n    @Override\n    public V get(final long timeout) {\n        synchronized (lock) {\n            if (!isDoing()) {\n                return getValueOrThrowable();\n            }\n\n            if (timeout <= 0) {\n                try {\n                    lock.wait();\n                } catch (Exception e) {\n                    cancel(e);\n                }\n                return getValueOrThrowable();\n            } else {\n                long waitTime = timeout - (System.currentTimeMillis() - createTime);\n                if (waitTime > 0) {\n                    for (; ; ) {\n                        try {\n                            lock.wait(waitTime);\n                        } catch (InterruptedException e) {\n                            LOG.error(\"promise get value interrupted,exception:{}\", e.getMessage());\n                        }\n\n                        if (!isDoing()) {\n                            break;\n                        } else {\n                            waitTime = timeout - (System.currentTimeMillis() - createTime);\n                            if (waitTime <= 0) {\n                                break;\n                            }\n                        }\n                    }\n                }\n\n                if (isDoing()) {\n                    timeoutSoCancel();\n                }\n            }\n            return getValueOrThrowable();\n        }\n    }\n\n    @Override\n    public boolean set(final V value) {\n        if (value == null)\n            return false;\n        this.result = value;\n        return done();\n    }\n\n    @Override\n    public boolean setFailure(final Throwable cause) {\n        if (cause == null)\n            return false;\n        this.exception = cause;\n        return done();\n    }\n\n    @Override\n    public void addListener(final FutureListener<V> listener) {\n        if (listener == null) {\n            throw new NullPointerException(\"FutureListener is null\");\n        }\n\n        boolean notifyNow = false;\n        synchronized (lock) {\n            if (!isDoing()) {\n                notifyNow = true;\n            } else {\n                if (promiseListenerList == null) {\n                    promiseListenerList = new ArrayList<>();\n                }\n                promiseListenerList.add(listener);\n            }\n        }\n\n        if (notifyNow) {\n            notifyListener(listener);\n        }\n    }\n\n    @Override\n    public Throwable getThrowable() {\n        return exception;\n    }\n\n    private void notifyListeners() {\n        if (promiseListenerList != null) {\n            for (FutureListener<V> listener : promiseListenerList) {\n                notifyListener(listener);\n            }\n        }\n    }\n\n    private boolean isSuccess() {\n        return isDone() && exception == null;\n    }\n\n    private void timeoutSoCancel() {\n        synchronized (lock) {\n            if (!isDoing()) {\n                return;\n            }\n            state = FutureState.CANCELLED;\n            exception = new RuntimeException(\"Get request result is timeout or interrupted\");\n            lock.notifyAll();\n        }\n        notifyListeners();\n    }\n\n    private V getValueOrThrowable() {\n        if (exception != null) {\n            Throwable e = exception.getCause() != null ? exception.getCause() : exception;\n            throw new OMSRuntimeException(\"-1\", e);\n        }\n        notifyListeners();\n        return result;\n    }\n\n    private boolean isDoing() {\n        return state.isDoingState();\n    }\n\n    private boolean done() {\n        synchronized (lock) {\n            if (!isDoing()) {\n                return false;\n            }\n\n            state = FutureState.DONE;\n            lock.notifyAll();\n        }\n\n        notifyListeners();\n        return true;\n    }\n\n    private void notifyListener(final FutureListener<V> listener) {\n        try {\n            listener.operationComplete(this);\n        } catch (Throwable t) {\n            LOG.error(\"notifyListener {} Error:{}\", listener.getClass().getSimpleName(), t);\n        }\n    }\n\n    private boolean cancel(Exception e) {\n        synchronized (lock) {\n            if (!isDoing()) {\n                return false;\n            }\n\n            state = FutureState.CANCELLED;\n            exception = e;\n            lock.notifyAll();\n        }\n\n        notifyListeners();\n        return true;\n    }\n}\n\n"
  },
  {
    "path": "openmessaging/src/main/java/io/openmessaging/rocketmq/promise/FutureState.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage io.openmessaging.rocketmq.promise;\n\npublic enum FutureState {\n    /**\n     * the task is doing\n     **/\n    DOING(0),\n    /**\n     * the task is done\n     **/\n    DONE(1),\n    /**\n     * ths task is cancelled\n     **/\n    CANCELLED(2);\n\n    public final int value;\n\n    private FutureState(int value) {\n        this.value = value;\n    }\n\n    public boolean isCancelledState() {\n        return this == CANCELLED;\n    }\n\n    public boolean isDoneState() {\n        return this == DONE;\n    }\n\n    public boolean isDoingState() {\n        return this == DOING;\n    }\n}\n"
  },
  {
    "path": "openmessaging/src/main/java/io/openmessaging/rocketmq/utils/BeanUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.openmessaging.rocketmq.utils;\n\nimport io.openmessaging.KeyValue;\nimport java.lang.reflect.InvocationTargetException;\nimport java.lang.reflect.Method;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.Map.Entry;\nimport java.util.Properties;\nimport java.util.Set;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic final class BeanUtils {\n    private static final Logger log = LoggerFactory.getLogger(BeanUtils.class);\n\n    /**\n     * Maps primitive {@code Class}es to their corresponding wrapper {@code Class}.\n     */\n    private static Map<Class<?>, Class<?>> primitiveWrapperMap = new HashMap<>();\n\n    static {\n        primitiveWrapperMap.put(Boolean.TYPE, Boolean.class);\n        primitiveWrapperMap.put(Byte.TYPE, Byte.class);\n        primitiveWrapperMap.put(Character.TYPE, Character.class);\n        primitiveWrapperMap.put(Short.TYPE, Short.class);\n        primitiveWrapperMap.put(Integer.TYPE, Integer.class);\n        primitiveWrapperMap.put(Long.TYPE, Long.class);\n        primitiveWrapperMap.put(Double.TYPE, Double.class);\n        primitiveWrapperMap.put(Float.TYPE, Float.class);\n        primitiveWrapperMap.put(Void.TYPE, Void.TYPE);\n    }\n\n    private static Map<Class<?>, Class<?>> wrapperMap = new HashMap<>();\n\n    static {\n        for (Entry<Class<?>, Class<?>> primitiveClass : primitiveWrapperMap.entrySet()) {\n            final Class<?> wrapperClass = primitiveClass.getValue();\n            if (!primitiveClass.getKey().equals(wrapperClass)) {\n                wrapperMap.put(wrapperClass, primitiveClass.getKey());\n            }\n        }\n        wrapperMap.put(String.class, String.class);\n    }\n\n    /**\n     * <p>Populate the JavaBeans properties of the specified bean, based on\n     * the specified name/value pairs.  This method uses Java reflection APIs\n     * to identify corresponding \"property setter\" method names, and deals\n     * with setter arguments of type <Code>String</Code>, <Code>boolean</Code>,\n     * <Code>int</Code>, <Code>long</Code>, <Code>float</Code>, and\n     * <Code>double</Code>.</p>\n     *\n     * <p>The particular setter method to be called for each property is\n     * determined using the usual JavaBeans introspection mechanisms.  Thus,\n     * you may identify custom setter methods using a BeanInfo class that is\n     * associated with the class of the bean itself.  If no such BeanInfo\n     * class is available, the standard method name conversion (\"set\" plus\n     * the capitalized name of the property in question) is used.</p>\n     *\n     * <p><strong>NOTE</strong>:  It is contrary to the JavaBeans Specification\n     * to have more than one setter method (with different argument\n     * signatures) for the same property.</p>\n     *\n     * @param clazz JavaBean class whose properties are being populated\n     * @param properties Map keyed by property name, with the corresponding (String or String[]) value(s) to be set\n     * @param <T> Class type\n     * @return Class instance\n     */\n    public static <T> T populate(final Properties properties, final Class<T> clazz) {\n        T obj = null;\n        try {\n            obj = clazz.getDeclaredConstructor().newInstance();\n            return populate(properties, obj);\n        } catch (Throwable e) {\n            log.warn(\"Error occurs !\", e);\n        }\n        return obj;\n    }\n\n    public static <T> T populate(final KeyValue properties, final Class<T> clazz) {\n        T obj = null;\n        try {\n            obj = clazz.getDeclaredConstructor().newInstance();\n            return populate(properties, obj);\n        } catch (Throwable e) {\n            log.warn(\"Error occurs !\", e);\n        }\n        return obj;\n    }\n\n    public static Class<?> getMethodClass(Class<?> clazz, String methodName) {\n        Method[] methods = clazz.getMethods();\n        for (Method method : methods) {\n            if (method.getName().equalsIgnoreCase(methodName)) {\n                return method.getParameterTypes()[0];\n            }\n        }\n        return null;\n    }\n\n    public static void setProperties(Class<?> clazz, Object obj, String methodName,\n        Object value) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {\n        Class<?> parameterClass = getMethodClass(clazz, methodName);\n        Method setterMethod = clazz.getMethod(methodName, parameterClass);\n        if (parameterClass == Boolean.TYPE) {\n            setterMethod.invoke(obj, Boolean.valueOf(value.toString()));\n        } else if (parameterClass == Integer.TYPE) {\n            setterMethod.invoke(obj, Integer.valueOf(value.toString()));\n        } else if (parameterClass == Double.TYPE) {\n            setterMethod.invoke(obj, Double.valueOf(value.toString()));\n        } else if (parameterClass == Float.TYPE) {\n            setterMethod.invoke(obj, Float.valueOf(value.toString()));\n        } else if (parameterClass == Long.TYPE) {\n            setterMethod.invoke(obj, Long.valueOf(value.toString()));\n        } else\n            setterMethod.invoke(obj, value);\n    }\n\n    public static <T> T populate(final Properties properties, final T obj) {\n        Class<?> clazz = obj.getClass();\n        try {\n\n            Set<Map.Entry<Object, Object>> entries = properties.entrySet();\n            for (Map.Entry<Object, Object> entry : entries) {\n                String entryKey = entry.getKey().toString();\n                String[] keyGroup = entryKey.split(\"\\\\.\");\n                for (int i = 0; i < keyGroup.length; i++) {\n                    keyGroup[i] = keyGroup[i].toLowerCase();\n                    keyGroup[i] = StringUtils.capitalize(keyGroup[i]);\n                }\n                String beanFieldNameWithCapitalization = StringUtils.join(keyGroup);\n                try {\n                    setProperties(clazz, obj, \"set\" + beanFieldNameWithCapitalization, entry.getValue());\n                } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException ignored) {\n                    //ignored...\n                }\n            }\n        } catch (RuntimeException e) {\n            log.warn(\"Error occurs !\", e);\n        }\n        return obj;\n    }\n\n    public static <T> T populate(final KeyValue properties, final T obj) {\n        Class<?> clazz = obj.getClass();\n        try {\n\n            final Set<String> keySet = properties.keySet();\n            for (String key : keySet) {\n                String[] keyGroup = key.split(\"[\\\\._]\");\n                for (int i = 0; i < keyGroup.length; i++) {\n                    keyGroup[i] = keyGroup[i].toLowerCase();\n                    keyGroup[i] = StringUtils.capitalize(keyGroup[i]);\n                }\n                String beanFieldNameWithCapitalization = StringUtils.join(keyGroup);\n                try {\n                    setProperties(clazz, obj, \"set\" + beanFieldNameWithCapitalization, properties.getString(key));\n                } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException ignored) {\n                    //ignored...\n                }\n            }\n        } catch (RuntimeException e) {\n            log.warn(\"Error occurs !\", e);\n        }\n        return obj;\n    }\n}\n\n"
  },
  {
    "path": "openmessaging/src/main/java/io/openmessaging/rocketmq/utils/OMSUtil.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.openmessaging.rocketmq.utils;\n\nimport io.openmessaging.BytesMessage;\nimport io.openmessaging.KeyValue;\nimport io.openmessaging.Message.BuiltinKeys;\nimport io.openmessaging.OMS;\nimport io.openmessaging.producer.SendResult;\nimport io.openmessaging.rocketmq.domain.BytesMessageImpl;\nimport io.openmessaging.rocketmq.domain.RocketMQConstants;\nimport io.openmessaging.rocketmq.domain.SendResultImpl;\nimport java.lang.reflect.Field;\nimport java.util.Iterator;\nimport java.util.Map;\nimport java.util.NoSuchElementException;\nimport java.util.Set;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.MessageAccessor;\n\npublic class OMSUtil {\n\n    /**\n     * Builds a OMS client instance name.\n     *\n     * @return a unique instance name\n     */\n    public static String buildInstanceName() {\n        return Integer.toString(UtilAll.getPid()) + \"%OpenMessaging\" + \"%\" + System.nanoTime();\n    }\n\n    public static org.apache.rocketmq.common.message.Message msgConvert(BytesMessage omsMessage) {\n        org.apache.rocketmq.common.message.Message rmqMessage = new org.apache.rocketmq.common.message.Message();\n        rmqMessage.setBody(omsMessage.getBody(byte[].class));\n\n        KeyValue sysHeaders = omsMessage.sysHeaders();\n        KeyValue userHeaders = omsMessage.userHeaders();\n\n        //All destinations in RocketMQ use Topic\n        rmqMessage.setTopic(sysHeaders.getString(BuiltinKeys.DESTINATION));\n\n        if (sysHeaders.containsKey(BuiltinKeys.START_TIME)) {\n            long deliverTime = sysHeaders.getLong(BuiltinKeys.START_TIME, 0);\n            if (deliverTime > 0) {\n                rmqMessage.putUserProperty(RocketMQConstants.START_DELIVER_TIME, String.valueOf(deliverTime));\n            }\n        }\n\n        for (String key : userHeaders.keySet()) {\n            MessageAccessor.putProperty(rmqMessage, key, userHeaders.getString(key));\n        }\n\n        //System headers has a high priority\n        for (String key : sysHeaders.keySet()) {\n            MessageAccessor.putProperty(rmqMessage, key, sysHeaders.getString(key));\n        }\n\n        return rmqMessage;\n    }\n\n    public static BytesMessage msgConvert(org.apache.rocketmq.common.message.MessageExt rmqMsg) {\n        BytesMessage omsMsg = new BytesMessageImpl();\n        omsMsg.setBody(rmqMsg.getBody());\n\n        KeyValue headers = omsMsg.sysHeaders();\n        KeyValue properties = omsMsg.userHeaders();\n\n        final Set<Map.Entry<String, String>> entries = rmqMsg.getProperties().entrySet();\n\n        for (final Map.Entry<String, String> entry : entries) {\n            if (isOMSHeader(entry.getKey())) {\n                headers.put(entry.getKey(), entry.getValue());\n            } else {\n                properties.put(entry.getKey(), entry.getValue());\n            }\n        }\n\n        omsMsg.putSysHeaders(BuiltinKeys.MESSAGE_ID, rmqMsg.getMsgId());\n\n        omsMsg.putSysHeaders(BuiltinKeys.DESTINATION, rmqMsg.getTopic());\n\n        omsMsg.putSysHeaders(BuiltinKeys.SEARCH_KEYS, rmqMsg.getKeys());\n        omsMsg.putSysHeaders(BuiltinKeys.BORN_HOST, String.valueOf(rmqMsg.getBornHost()));\n        omsMsg.putSysHeaders(BuiltinKeys.BORN_TIMESTAMP, rmqMsg.getBornTimestamp());\n        omsMsg.putSysHeaders(BuiltinKeys.STORE_HOST, String.valueOf(rmqMsg.getStoreHost()));\n        omsMsg.putSysHeaders(BuiltinKeys.STORE_TIMESTAMP, rmqMsg.getStoreTimestamp());\n        return omsMsg;\n    }\n\n    public static boolean isOMSHeader(String value) {\n        for (Field field : BuiltinKeys.class.getDeclaredFields()) {\n            try {\n                if (field.get(BuiltinKeys.class).equals(value)) {\n                    return true;\n                }\n            } catch (IllegalAccessException e) {\n                return false;\n            }\n        }\n        return false;\n    }\n\n    /**\n     * Convert a RocketMQ SEND_OK SendResult instance to a OMS SendResult.\n     */\n    public static SendResult sendResultConvert(org.apache.rocketmq.client.producer.SendResult rmqResult) {\n        assert rmqResult.getSendStatus().equals(SendStatus.SEND_OK);\n        return new SendResultImpl(rmqResult.getMsgId(), OMS.newKeyValue());\n    }\n\n    public static KeyValue buildKeyValue(KeyValue... keyValues) {\n        KeyValue keyValue = OMS.newKeyValue();\n        for (KeyValue properties : keyValues) {\n            for (String key : properties.keySet()) {\n                keyValue.put(key, properties.getString(key));\n            }\n        }\n        return keyValue;\n    }\n\n    /**\n     * Returns an iterator that cycles indefinitely over the elements of {@code Iterable}.\n     */\n    public static <T> Iterator<T> cycle(final Iterable<T> iterable) {\n        return new Iterator<T>() {\n            Iterator<T> iterator = new Iterator<T>() {\n                @Override\n                public synchronized boolean hasNext() {\n                    return false;\n                }\n\n                @Override\n                public synchronized T next() {\n                    throw new NoSuchElementException();\n                }\n\n                @Override\n                public synchronized void remove() {\n                    //Ignore\n                }\n            };\n\n            @Override\n            public synchronized boolean hasNext() {\n                return iterator.hasNext() || iterable.iterator().hasNext();\n            }\n\n            @Override\n            public synchronized T next() {\n                if (!iterator.hasNext()) {\n                    iterator = iterable.iterator();\n                    if (!iterator.hasNext()) {\n                        throw new NoSuchElementException();\n                    }\n                }\n                return iterator.next();\n            }\n\n            @Override\n            public synchronized void remove() {\n                iterator.remove();\n            }\n        };\n    }\n}\n"
  },
  {
    "path": "openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/LocalMessageCacheTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.openmessaging.rocketmq.consumer;\n\nimport io.openmessaging.rocketmq.config.ClientConfig;\nimport io.openmessaging.rocketmq.domain.ConsumeRequest;\nimport io.openmessaging.rocketmq.domain.NonStandardKeys;\nimport org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyBoolean;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class LocalMessageCacheTest {\n    private LocalMessageCache localMessageCache;\n    @Mock\n    private DefaultMQPullConsumer rocketmqPullConsume;\n    @Mock\n    private ConsumeRequest consumeRequest;\n\n    @Before\n    public void init() {\n        ClientConfig clientConfig = new ClientConfig();\n        clientConfig.setRmqPullMessageBatchNums(512);\n        clientConfig.setRmqPullMessageCacheCapacity(1024);\n        localMessageCache = new LocalMessageCache(rocketmqPullConsume, clientConfig);\n    }\n\n    @Test\n    public void testNextPullBatchNums() throws Exception {\n        assertThat(localMessageCache.nextPullBatchNums()).isEqualTo(512);\n        for (int i = 0; i < 513; i++) {\n            localMessageCache.submitConsumeRequest(consumeRequest);\n        }\n        assertThat(localMessageCache.nextPullBatchNums()).isEqualTo(511);\n    }\n\n    @Test\n    public void testNextPullOffset() throws Exception {\n        MessageQueue messageQueue = new MessageQueue();\n        when(rocketmqPullConsume.fetchConsumeOffset(any(MessageQueue.class), anyBoolean()))\n            .thenReturn(123L);\n        assertThat(localMessageCache.nextPullOffset(new MessageQueue())).isEqualTo(123L);\n    }\n\n    @Test\n    public void testUpdatePullOffset() throws Exception {\n        MessageQueue messageQueue = new MessageQueue();\n        localMessageCache.updatePullOffset(messageQueue, 124L);\n        assertThat(localMessageCache.nextPullOffset(messageQueue)).isEqualTo(124L);\n    }\n\n    @Test\n    public void testSubmitConsumeRequest() throws Exception {\n        byte[] body = new byte[] {'1', '2', '3'};\n        MessageExt consumedMsg = new MessageExt();\n        consumedMsg.setMsgId(\"NewMsgId\");\n        consumedMsg.setBody(body);\n        consumedMsg.putUserProperty(NonStandardKeys.MESSAGE_DESTINATION, \"TOPIC\");\n        consumedMsg.setTopic(\"HELLO_QUEUE\");\n\n        when(consumeRequest.getMessageExt()).thenReturn(consumedMsg);\n        localMessageCache.submitConsumeRequest(consumeRequest);\n        assertThat(localMessageCache.poll()).isEqualTo(consumedMsg);\n    }\n}"
  },
  {
    "path": "openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PullConsumerImplTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.openmessaging.rocketmq.consumer;\n\nimport io.openmessaging.BytesMessage;\nimport io.openmessaging.Message;\nimport io.openmessaging.MessagingAccessPoint;\nimport io.openmessaging.OMS;\nimport io.openmessaging.OMSBuiltinKeys;\nimport io.openmessaging.consumer.PullConsumer;\nimport io.openmessaging.rocketmq.config.ClientConfig;\nimport io.openmessaging.rocketmq.domain.NonStandardKeys;\nimport java.lang.reflect.Field;\nimport org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class PullConsumerImplTest {\n    private PullConsumer consumer;\n    private String queueName = \"HELLO_QUEUE\";\n\n    @Mock\n    private DefaultMQPullConsumer rocketmqPullConsumer;\n    private LocalMessageCache localMessageCache = null;\n\n    @Before\n    public void init() throws NoSuchFieldException, IllegalAccessException {\n        final MessagingAccessPoint messagingAccessPoint = OMS\n            .getMessagingAccessPoint(\"oms:rocketmq://IP1:9876,IP2:9876/namespace\");\n\n        consumer = messagingAccessPoint.createPullConsumer(OMS.newKeyValue().put(OMSBuiltinKeys.CONSUMER_ID, \"TestGroup\"));\n        consumer.attachQueue(queueName);\n\n        Field field = PullConsumerImpl.class.getDeclaredField(\"rocketmqPullConsumer\");\n        field.setAccessible(true);\n        field.set(consumer, rocketmqPullConsumer); //Replace\n\n        ClientConfig clientConfig = new ClientConfig();\n        clientConfig.setOperationTimeout(200);\n        localMessageCache = spy(new LocalMessageCache(rocketmqPullConsumer, clientConfig));\n\n        field = PullConsumerImpl.class.getDeclaredField(\"localMessageCache\");\n        field.setAccessible(true);\n        field.set(consumer, localMessageCache);\n\n        messagingAccessPoint.startup();\n        consumer.startup();\n    }\n\n    @Test\n    public void testPoll() {\n        final byte[] testBody = new byte[] {'a', 'b'};\n        MessageExt consumedMsg = new MessageExt();\n        consumedMsg.setMsgId(\"NewMsgId\");\n        consumedMsg.setBody(testBody);\n        consumedMsg.putUserProperty(NonStandardKeys.MESSAGE_DESTINATION, \"TOPIC\");\n        consumedMsg.setTopic(queueName);\n\n        when(localMessageCache.poll()).thenReturn(consumedMsg);\n\n        Message message = consumer.receive();\n        assertThat(message.sysHeaders().getString(Message.BuiltinKeys.MESSAGE_ID)).isEqualTo(\"NewMsgId\");\n        assertThat(((BytesMessage) message).getBody(byte[].class)).isEqualTo(testBody);\n    }\n\n    @Test\n    public void testPoll_WithTimeout() {\n        //There is a default timeout value, @see ClientConfig#omsOperationTimeout.\n        Message message = consumer.receive();\n        assertThat(message).isNull();\n\n        message = consumer.receive(OMS.newKeyValue().put(Message.BuiltinKeys.TIMEOUT, 100));\n        assertThat(message).isNull();\n    }\n}"
  },
  {
    "path": "openmessaging/src/test/java/io/openmessaging/rocketmq/consumer/PushConsumerImplTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.openmessaging.rocketmq.consumer;\n\nimport io.openmessaging.BytesMessage;\nimport io.openmessaging.Message;\nimport io.openmessaging.OMSBuiltinKeys;\nimport io.openmessaging.consumer.MessageListener;\nimport io.openmessaging.MessagingAccessPoint;\nimport io.openmessaging.OMS;\nimport io.openmessaging.consumer.PushConsumer;\nimport io.openmessaging.rocketmq.domain.NonStandardKeys;\nimport java.lang.reflect.Field;\nimport java.util.Collections;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class PushConsumerImplTest {\n    private PushConsumer consumer;\n\n    @Mock\n    private DefaultMQPushConsumer rocketmqPushConsumer;\n\n    @Before\n    public void init() throws NoSuchFieldException, IllegalAccessException {\n        final MessagingAccessPoint messagingAccessPoint = OMS\n            .getMessagingAccessPoint(\"oms:rocketmq://IP1:9876,IP2:9876/namespace\");\n        consumer = messagingAccessPoint.createPushConsumer(\n            OMS.newKeyValue().put(OMSBuiltinKeys.CONSUMER_ID, \"TestGroup\"));\n\n        Field field = PushConsumerImpl.class.getDeclaredField(\"rocketmqPushConsumer\");\n        field.setAccessible(true);\n        DefaultMQPushConsumer innerConsumer = (DefaultMQPushConsumer) field.get(consumer);\n        field.set(consumer, rocketmqPushConsumer); //Replace\n\n        when(rocketmqPushConsumer.getMessageListener()).thenReturn(innerConsumer.getMessageListener());\n        messagingAccessPoint.startup();\n        consumer.startup();\n    }\n\n    @Test\n    public void testConsumeMessage() {\n        final byte[] testBody = new byte[] {'a', 'b'};\n\n        MessageExt consumedMsg = new MessageExt();\n        consumedMsg.setMsgId(\"NewMsgId\");\n        consumedMsg.setBody(testBody);\n        consumedMsg.putUserProperty(NonStandardKeys.MESSAGE_DESTINATION, \"TOPIC\");\n        consumedMsg.setTopic(\"HELLO_QUEUE\");\n        consumer.attachQueue(\"HELLO_QUEUE\", new MessageListener() {\n            @Override\n            public void onReceived(Message message, Context context) {\n                assertThat(message.sysHeaders().getString(Message.BuiltinKeys.MESSAGE_ID)).isEqualTo(\"NewMsgId\");\n                assertThat(((BytesMessage) message).getBody(byte[].class)).isEqualTo(testBody);\n                context.ack();\n            }\n        });\n        ((MessageListenerConcurrently) rocketmqPushConsumer\n            .getMessageListener()).consumeMessage(Collections.singletonList(consumedMsg), null);\n    }\n}"
  },
  {
    "path": "openmessaging/src/test/java/io/openmessaging/rocketmq/producer/ProducerImplTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.openmessaging.rocketmq.producer;\n\nimport io.openmessaging.MessagingAccessPoint;\nimport io.openmessaging.OMS;\nimport io.openmessaging.exception.OMSRuntimeException;\nimport io.openmessaging.producer.Producer;\nimport java.lang.reflect.Field;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ProducerImplTest {\n    private Producer producer;\n\n    @Mock\n    private DefaultMQProducer rocketmqProducer;\n\n    @Before\n    public void init() throws NoSuchFieldException, IllegalAccessException {\n        final MessagingAccessPoint messagingAccessPoint = OMS\n            .getMessagingAccessPoint(\"oms:rocketmq://IP1:9876,IP2:9876/namespace\");\n        producer = messagingAccessPoint.createProducer();\n\n        Field field = AbstractOMSProducer.class.getDeclaredField(\"rocketmqProducer\");\n        field.setAccessible(true);\n        field.set(producer, rocketmqProducer);\n\n        messagingAccessPoint.startup();\n        producer.startup();\n    }\n\n    @Test\n    public void testSend_OK() throws InterruptedException, RemotingException, MQClientException, MQBrokerException {\n        SendResult sendResult = new SendResult();\n        sendResult.setMsgId(\"TestMsgID\");\n        sendResult.setSendStatus(SendStatus.SEND_OK);\n        when(rocketmqProducer.send(any(Message.class), anyLong())).thenReturn(sendResult);\n        io.openmessaging.producer.SendResult omsResult =\n            producer.send(producer.createBytesMessage(\"HELLO_TOPIC\", new byte[] {'a'}));\n\n        assertThat(omsResult.messageId()).isEqualTo(\"TestMsgID\");\n    }\n\n    @Test\n    public void testSend_Not_OK() throws InterruptedException, RemotingException, MQClientException, MQBrokerException {\n        SendResult sendResult = new SendResult();\n        sendResult.setSendStatus(SendStatus.FLUSH_DISK_TIMEOUT);\n\n        when(rocketmqProducer.send(any(Message.class), anyLong())).thenReturn(sendResult);\n        try {\n            producer.send(producer.createBytesMessage(\"HELLO_TOPIC\", new byte[] {'a'}));\n            failBecauseExceptionWasNotThrown(OMSRuntimeException.class);\n        } catch (Exception e) {\n            assertThat(e).hasMessageContaining(\"Send message to RocketMQ broker failed.\");\n        }\n    }\n\n    @Test\n    public void testSend_WithException() throws InterruptedException, RemotingException, MQClientException, MQBrokerException {\n        when(rocketmqProducer.send(any(Message.class), anyLong())).thenThrow(MQClientException.class);\n        try {\n            producer.send(producer.createBytesMessage(\"HELLO_TOPIC\", new byte[] {'a'}));\n            failBecauseExceptionWasNotThrown(OMSRuntimeException.class);\n        } catch (Exception e) {\n            assertThat(e).hasMessageContaining(\"Send message to RocketMQ broker failed.\");\n        }\n    }\n\n}"
  },
  {
    "path": "openmessaging/src/test/java/io/openmessaging/rocketmq/promise/DefaultPromiseTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.openmessaging.rocketmq.promise;\n\nimport io.openmessaging.Future;\nimport io.openmessaging.FutureListener;\nimport io.openmessaging.Promise;\nimport io.openmessaging.exception.OMSRuntimeException;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown;\n\npublic class DefaultPromiseTest {\n    private Promise<String> promise;\n\n    @Before\n    public void init() {\n        promise = new DefaultPromise<>();\n    }\n\n    @Test\n    public void testIsCancelled() throws Exception {\n        assertThat(promise.isCancelled()).isEqualTo(false);\n    }\n\n    @Test\n    public void testIsDone() throws Exception {\n        assertThat(promise.isDone()).isEqualTo(false);\n        promise.set(\"Done\");\n        assertThat(promise.isDone()).isEqualTo(true);\n    }\n\n    @Test\n    public void testGet() throws Exception {\n        promise.set(\"Done\");\n        assertThat(promise.get()).isEqualTo(\"Done\");\n    }\n\n    @Test\n    public void testGet_WithTimeout() throws Exception {\n        try {\n            promise.get(100);\n            failBecauseExceptionWasNotThrown(OMSRuntimeException.class);\n        } catch (OMSRuntimeException e) {\n            assertThat(e).hasMessageContaining(\"Get request result is timeout or interrupted\");\n        }\n    }\n\n    @Test\n    public void testAddListener() throws Exception {\n        promise.addListener(new FutureListener<String>() {\n            @Override\n            public void operationComplete(Future<String> future) {\n                assertThat(promise.get()).isEqualTo(\"Done\");\n\n            }\n        });\n        promise.set(\"Done\");\n    }\n\n    @Test\n    public void testAddListener_ListenerAfterSet() throws Exception {\n        promise.set(\"Done\");\n        promise.addListener(new FutureListener<String>() {\n            @Override\n            public void operationComplete(Future<String> future) {\n                assertThat(future.get()).isEqualTo(\"Done\");\n            }\n        });\n    }\n\n    @Test\n    public void testAddListener_WithException_ListenerAfterSet() throws Exception {\n        final Throwable exception = new OMSRuntimeException(\"-1\", \"Test Error\");\n        promise.setFailure(exception);\n        promise.addListener(new FutureListener<String>() {\n            @Override\n            public void operationComplete(Future<String> future) {\n                assertThat(promise.getThrowable()).isEqualTo(exception);\n            }\n        });\n    }\n\n    @Test\n    public void testAddListener_WithException() throws Exception {\n        final Throwable exception = new OMSRuntimeException(\"-1\", \"Test Error\");\n        promise.addListener(new FutureListener<String>() {\n            @Override\n            public void operationComplete(Future<String> future) {\n                assertThat(promise.getThrowable()).isEqualTo(exception);\n            }\n        });\n        promise.setFailure(exception);\n    }\n\n    @Test\n    public void getThrowable() throws Exception {\n        assertThat(promise.getThrowable()).isNull();\n        Throwable exception = new OMSRuntimeException(\"-1\", \"Test Error\");\n        promise.setFailure(exception);\n        assertThat(promise.getThrowable()).isEqualTo(exception);\n    }\n\n}"
  },
  {
    "path": "openmessaging/src/test/java/io/openmessaging/rocketmq/utils/BeanUtilsTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage io.openmessaging.rocketmq.utils;\n\nimport io.openmessaging.KeyValue;\nimport io.openmessaging.OMS;\nimport io.openmessaging.rocketmq.config.ClientConfig;\nimport io.openmessaging.rocketmq.domain.NonStandardKeys;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\npublic class BeanUtilsTest {\n    private KeyValue properties = OMS.newKeyValue();\n\n    public static class CustomizedConfig extends ClientConfig {\n        final static String STRING_TEST = \"string.test\";\n        String stringTest = \"foobar\";\n\n        final static String DOUBLE_TEST = \"double.test\";\n        double doubleTest = 123.0;\n\n        final static String LONG_TEST = \"long.test\";\n        long longTest = 123L;\n\n        String getStringTest() {\n            return stringTest;\n        }\n\n        public void setStringTest(String stringTest) {\n            this.stringTest = stringTest;\n        }\n\n        double getDoubleTest() {\n            return doubleTest;\n        }\n\n        public void setDoubleTest(final double doubleTest) {\n            this.doubleTest = doubleTest;\n        }\n\n        long getLongTest() {\n            return longTest;\n        }\n\n        public void setLongTest(final long longTest) {\n            this.longTest = longTest;\n        }\n\n        CustomizedConfig() {\n        }\n    }\n\n    @Before\n    public void init() {\n        properties.put(NonStandardKeys.MAX_REDELIVERY_TIMES, 120);\n        properties.put(CustomizedConfig.STRING_TEST, \"kaka\");\n        properties.put(NonStandardKeys.CONSUMER_GROUP, \"Default_Consumer_Group\");\n        properties.put(NonStandardKeys.MESSAGE_CONSUME_TIMEOUT, 101);\n\n        properties.put(CustomizedConfig.LONG_TEST, 1234567890L);\n        properties.put(CustomizedConfig.DOUBLE_TEST, 10.234);\n    }\n\n    @Test\n    public void testPopulate() {\n        CustomizedConfig config = BeanUtils.populate(properties, CustomizedConfig.class);\n\n        //RemotingConfig config = BeanUtils.populate(properties, RemotingConfig.class);\n        Assert.assertEquals(config.getRmqMaxRedeliveryTimes(), 120);\n        Assert.assertEquals(config.getStringTest(), \"kaka\");\n        Assert.assertEquals(config.getRmqConsumerGroup(), \"Default_Consumer_Group\");\n        Assert.assertEquals(config.getRmqMessageConsumeTimeout(), 101);\n        Assert.assertEquals(config.getLongTest(), 1234567890L);\n        Assert.assertEquals(config.getDoubleTest(), 10.234, 0.000001);\n    }\n\n    @Test\n    public void testPopulate_ExistObj() {\n        CustomizedConfig config = new CustomizedConfig();\n        config.setConsumerId(\"NewConsumerId\");\n\n        Assert.assertEquals(config.getConsumerId(), \"NewConsumerId\");\n\n        config = BeanUtils.populate(properties, config);\n\n        //RemotingConfig config = BeanUtils.populate(properties, RemotingConfig.class);\n        Assert.assertEquals(config.getRmqMaxRedeliveryTimes(), 120);\n        Assert.assertEquals(config.getStringTest(), \"kaka\");\n        Assert.assertEquals(config.getRmqConsumerGroup(), \"Default_Consumer_Group\");\n        Assert.assertEquals(config.getRmqMessageConsumeTimeout(), 101);\n        Assert.assertEquals(config.getLongTest(), 1234567890L);\n        Assert.assertEquals(config.getDoubleTest(), 10.234, 0.000001);\n    }\n\n}"
  },
  {
    "path": "openmessaging/src/test/resources/rmq.logback-test.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n<configuration>\n\n    <appender name=\"CONSOLE\" class=\"ch.qos.logback.core.ConsoleAppender\">\n        <layout class=\"ch.qos.logback.classic.PatternLayout\">\n            <Pattern>\n                %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n\n            </Pattern>\n        </layout>\n    </appender>\n\n    <logger name=\"org.apache.rocketmq\" level=\"error\" additivity=\"false\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </logger>\n\n    <root level=\"error\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </root>\n\n</configuration>"
  },
  {
    "path": "pom.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd\">\n\n    <parent>\n        <groupId>org.apache</groupId>\n        <artifactId>apache</artifactId>\n        <version>18</version>\n    </parent>\n\n    <modelVersion>4.0.0</modelVersion>\n\n    <inceptionYear>2012</inceptionYear>\n    <groupId>org.apache.rocketmq</groupId>\n    <artifactId>rocketmq-all</artifactId>\n    <version>${revision}</version>\n    <packaging>pom</packaging>\n    <name>Apache RocketMQ ${project.version}</name>\n    <url>http://rocketmq.apache.org/</url>\n\n    <scm>\n        <url>git@github.com:apache/rocketmq.git</url>\n        <connection>scm:git:git@github.com:apache/rocketmq.git</connection>\n        <developerConnection>scm:git:git@github.com:apache/rocketmq.git</developerConnection>\n        <tag>HEAD</tag>\n    </scm>\n\n    <mailingLists>\n        <mailingList>\n            <name>Development List</name>\n            <subscribe>dev-subscribe@rocketmq.apache.org</subscribe>\n            <unsubscribe>dev-unsubscribe@rocketmq.apache.org</unsubscribe>\n            <post>dev@rocketmq.apache.org</post>\n        </mailingList>\n        <mailingList>\n            <name>User List</name>\n            <subscribe>users-subscribe@rocketmq.apache.org</subscribe>\n            <unsubscribe>users-unsubscribe@rocketmq.apache.org</unsubscribe>\n            <post>users@rocketmq.apache.org</post>\n        </mailingList>\n        <mailingList>\n            <name>Commits List</name>\n            <subscribe>commits-subscribe@rocketmq.apache.org</subscribe>\n            <unsubscribe>commits-unsubscribe@rocketmq.apache.org</unsubscribe>\n            <post>commits@rocketmq.apache.org</post>\n        </mailingList>\n    </mailingLists>\n\n    <developers>\n        <developer>\n            <id>Apache RocketMQ</id>\n            <name>Apache RocketMQ of ASF</name>\n            <url>https://rocketmq.apache.org/</url>\n        </developer>\n    </developers>\n\n    <licenses>\n        <license>\n            <name>Apache License, Version 2.0</name>\n            <url>http://www.apache.org/licenses/LICENSE-2.0</url>\n            <distribution>repo</distribution>\n        </license>\n    </licenses>\n\n    <organization>\n        <name>Apache Software Foundation</name>\n        <url>http://www.apache.org</url>\n    </organization>\n\n    <issueManagement>\n        <system>jira</system>\n        <url>https://issues.apache.org/jira/browse/RocketMQ</url>\n    </issueManagement>\n\n    <properties>\n        <revision>5.4.0</revision>\n        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>\n        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>\n        <project.root>${basedir}</project.root>\n\n        <!-- Maven properties -->\n        <maven.test.skip>false</maven.test.skip>\n        <maven.javadoc.skip>true</maven.javadoc.skip>\n        <!-- Compiler settings properties -->\n        <maven.compiler.source>1.8</maven.compiler.source>\n        <maven.compiler.target>1.8</maven.compiler.target>\n\n        <commons-cli.version>1.5.0</commons-cli.version>\n        <netty.version>4.1.130.Final</netty.version>\n        <netty.tcnative.version>2.0.53.Final</netty.tcnative.version>\n        <bcpkix-jdk18on.version>1.83</bcpkix-jdk18on.version>\n        <fastjson.version>1.2.83</fastjson.version>\n        <fastjson2.version>2.0.59</fastjson2.version>\n        <javassist.version>3.20.0-GA</javassist.version>\n        <jna.version>4.2.2</jna.version>\n        <commons-lang3.version>3.20.0</commons-lang3.version>\n        <commons-io.version>2.14.0</commons-io.version>\n        <guava.version>32.0.1-jre</guava.version>\n        <gson.version>2.9.0</gson.version>\n        <openmessaging.version>0.3.1-alpha</openmessaging.version>\n        <snakeyaml.version>2.0</snakeyaml.version>\n        <commons-codec.version>1.13</commons-codec.version>\n        <rocketmq-logging.version>1.0.1</rocketmq-logging.version>\n        <slf4j-api.version>2.0.3</slf4j-api.version>\n        <rocketmq-shaded-slf4j-api-bridge.version>1.0.0</rocketmq-shaded-slf4j-api-bridge.version>\n        <commons-validator.version>1.10.0</commons-validator.version>\n        <zstd-jni.version>1.5.2-2</zstd-jni.version>\n        <lz4-java.version>1.10.3</lz4-java.version>\n        <opentracing.version>0.33.0</opentracing.version>\n        <jaeger.version>1.8.1</jaeger.version>\n        <dleger.version>0.3.2</dleger.version>\n        <annotations-api.version>6.0.53</annotations-api.version>\n        <extra-enforcer-rules.version>1.0-beta-4</extra-enforcer-rules.version>\n        <concurrentlinkedhashmap-lru.version>1.4.2</concurrentlinkedhashmap-lru.version>\n        <rocketmq-proto.version>2.1.1</rocketmq-proto.version>\n        <grpc.version>1.53.0</grpc.version>\n        <protobuf.version>3.20.1</protobuf.version>\n        <disruptor.version>1.2.10</disruptor.version>\n        <org.relection.version>0.9.11</org.relection.version>\n        <caffeine.version>2.9.3</caffeine.version>\n        <spring.version>5.3.27</spring.version>\n        <okio-jvm.version>3.4.0</okio-jvm.version>\n        <opentelemetry.version>1.44.1</opentelemetry.version>\n        <opentelemetry-exporter-prometheus.version>1.44.1-alpha</opentelemetry-exporter-prometheus.version>\n        <jul-to-slf4j.version>2.0.6</jul-to-slf4j.version>\n        <s3.version>2.20.29</s3.version>\n        <rocksdb.version>1.0.6</rocksdb.version>\n        <jackson-databind.version>2.13.4.2</jackson-databind.version>\n        <sofa-jraft.version>1.3.14</sofa-jraft.version>\n\n        <!-- Test dependencies -->\n        <junit.version>4.13.2</junit.version>\n        <assertj-core.version>3.22.0</assertj-core.version>\n        <mockito-core.version>3.10.0</mockito-core.version>\n        <mockito-junit-jupiter.version>4.11.0</mockito-junit-jupiter.version>\n        <powermock-version>2.0.9</powermock-version>\n        <awaitility.version>4.1.0</awaitility.version>\n        <truth.version>0.30</truth.version>\n        <s3mock-junit4.version>2.11.0</s3mock-junit4.version>\n        <rocketmq-client-java.version>5.0.5</rocketmq-client-java.version>\n\n        <!-- Build plugin dependencies -->\n        <flatten-maven-plugin.version>1.7.2</flatten-maven-plugin.version>\n        <versions-maven-plugin.version>2.2</versions-maven-plugin.version>\n        <dependency-mediator-maven-plugin.version>1.0.2</dependency-mediator-maven-plugin.version>\n        <clirr-maven-plugin.version>2.7</clirr-maven-plugin.version>\n        <maven-enforcer-plugin.version>1.4.1</maven-enforcer-plugin.version>\n        <maven-compiler-plugin.version>3.5.1</maven-compiler-plugin.version>\n        <maven-source-plugin.version>3.0.1</maven-source-plugin.version>\n        <maven-help-plugin.version>2.2</maven-help-plugin.version>\n        <maven-checkstyle-plugin.version>3.2.0</maven-checkstyle-plugin.version>\n        <apache-rat-plugin.version>0.12</apache-rat-plugin.version>\n        <maven-resources-plugin.version>3.0.2</maven-resources-plugin.version>\n        <coveralls-maven-plugin.version>4.3.0</coveralls-maven-plugin.version>\n        <jacoco-maven-plugin.version>0.8.5</jacoco-maven-plugin.version>\n        <maven-surefire-plugin.version>2.19.1</maven-surefire-plugin.version>\n        <sonar-maven-plugin.version>3.0.2</sonar-maven-plugin.version>\n        <spotbugs-plugin.version>4.2.2</spotbugs-plugin.version>\n        <maven-assembly-plugin.version>3.4.2</maven-assembly-plugin.version>\n        <maven-javadoc-plugin.version>2.10.4</maven-javadoc-plugin.version>\n        <maven-failsafe-plugin.version>2.19.1</maven-failsafe-plugin.version>\n        <maven-shade-plugin.version>3.2.4</maven-shade-plugin.version>\n\n        <sonar.java.coveragePlugin>jacoco</sonar.java.coveragePlugin>\n        <!-- Exclude all generated code -->\n        <sonar.jacoco.itReportPath>${project.basedir}/../test/target/jacoco-it.exec</sonar.jacoco.itReportPath>\n        <sonar.exclusions>file:**/generated-sources/**,**/test/**</sonar.exclusions>\n    </properties>\n\n    <modules>\n        <module>client</module>\n        <module>common</module>\n        <module>broker</module>\n        <module>tools</module>\n        <module>store</module>\n        <module>namesrv</module>\n        <module>remoting</module>\n        <module>srvutil</module>\n        <module>filter</module>\n        <module>test</module>\n        <module>distribution</module>\n        <module>openmessaging</module>\n        <module>auth</module>\n        <module>example</module>\n        <module>container</module>\n        <module>controller</module>\n        <module>proxy</module>\n        <module>tieredstore</module>\n    </modules>\n\n    <build>\n        <plugins>\n            <plugin>\n                <groupId>org.codehaus.mojo</groupId>\n                <artifactId>flatten-maven-plugin</artifactId>\n                <version>${flatten-maven-plugin.version}</version>\n                <inherited>true</inherited>\n                <executions>\n                    <execution>\n                        <id>flatten</id>\n                        <phase>process-resources</phase>\n                        <goals>\n                            <goal>flatten</goal>\n                        </goals>\n                        <configuration>\n                            <updatePomFile>true</updatePomFile>\n                            <flattenMode>oss</flattenMode>\n                            <pomElements>\n                                <distributionManagement>remove</distributionManagement>\n                                <repositories>remove</repositories>\n                            </pomElements>\n                        </configuration>\n                    </execution>\n                    <execution>\n                        <id>flatten-clean</id>\n                        <phase>clean</phase>\n                        <goals>\n                            <goal>clean</goal>\n                        </goals>\n                    </execution>\n                </executions>\n            </plugin>\n            <plugin>\n                <groupId>org.codehaus.mojo</groupId>\n                <artifactId>versions-maven-plugin</artifactId>\n                <version>${versions-maven-plugin.version}</version>\n            </plugin>\n            <plugin>\n                <groupId>com.github.vongosling</groupId>\n                <artifactId>dependency-mediator-maven-plugin</artifactId>\n                <version>${dependency-mediator-maven-plugin.version}</version>\n            </plugin>\n            <plugin>\n                <groupId>org.codehaus.mojo</groupId>\n                <artifactId>clirr-maven-plugin</artifactId>\n                <version>${clirr-maven-plugin.version}</version>\n            </plugin>\n            <plugin>\n                <artifactId>maven-enforcer-plugin</artifactId>\n                <version>${maven-enforcer-plugin.version}</version>\n                <executions>\n                    <execution>\n                        <id>enforce-ban-circular-dependencies</id>\n                        <goals>\n                            <goal>enforce</goal>\n                        </goals>\n                    </execution>\n                </executions>\n                <configuration>\n                    <rules>\n                        <banCircularDependencies />\n                        <dependencyConvergence />\n                    </rules>\n                    <fail>true</fail>\n                </configuration>\n                <dependencies>\n                    <dependency>\n                        <groupId>org.codehaus.mojo</groupId>\n                        <artifactId>extra-enforcer-rules</artifactId>\n                        <version>${extra-enforcer-rules.version}</version>\n                    </dependency>\n                </dependencies>\n            </plugin>\n            <plugin>\n                <artifactId>maven-compiler-plugin</artifactId>\n                <version>${maven-compiler-plugin.version}</version>\n                <configuration>\n                    <source>${maven.compiler.source}</source>\n                    <target>${maven.compiler.target}</target>\n                    <compilerVersion>${maven.compiler.source}</compilerVersion>\n                    <showDeprecation>true</showDeprecation>\n                    <showWarnings>true</showWarnings>\n                </configuration>\n            </plugin>\n            <plugin>\n                <artifactId>maven-source-plugin</artifactId>\n                <version>${maven-source-plugin.version}</version>\n                <executions>\n                    <execution>\n                        <id>attach-sources</id>\n                        <goals>\n                            <goal>jar</goal>\n                        </goals>\n                    </execution>\n                </executions>\n            </plugin>\n            <plugin>\n                <artifactId>maven-help-plugin</artifactId>\n                <version>${maven-help-plugin.version}</version>\n                <executions>\n                    <execution>\n                        <id>generate-effective-dependencies-pom</id>\n                        <phase>generate-resources</phase>\n                        <configuration>\n                            <output>${project.build.directory}/effective-pom/effective-dependencies.xml</output>\n                        </configuration>\n                    </execution>\n                </executions>\n            </plugin>\n            <plugin>\n                <artifactId>maven-checkstyle-plugin</artifactId>\n                <version>${maven-checkstyle-plugin.version}</version>\n                <executions>\n                    <execution>\n                        <id>validate</id>\n                        <phase>validate</phase>\n                        <configuration>\n                            <configLocation>style/rmq_checkstyle.xml</configLocation>\n                            <inputEncoding>UTF-8</inputEncoding>\n                            <consoleOutput>true</consoleOutput>\n                            <failsOnError>true</failsOnError>\n                            <includeTestSourceDirectory>true</includeTestSourceDirectory>\n                            <excludes>**/generated*/**/*</excludes>\n                        </configuration>\n                        <goals>\n                            <goal>check</goal>\n                        </goals>\n                    </execution>\n                </executions>\n            </plugin>\n            <plugin>\n                <groupId>org.apache.rat</groupId>\n                <artifactId>apache-rat-plugin</artifactId>\n                <version>${apache-rat-plugin.version}</version>\n                <configuration>\n                    <excludes>\n                        <exclude>.gitignore</exclude>\n                        <exclude>.travis.yml</exclude>\n                        <exclude>README.md</exclude>\n                        <exclude>CONTRIBUTING.md</exclude>\n                        <exclude>bin/README.md</exclude>\n                        <exclude>.github/**</exclude>\n                        <exclude>src/test/resources/**</exclude>\n                        <exclude>src/test/resources/certs/*</exclude>\n                        <exclude>src/test/**/*.log</exclude>\n                        <exclude>src/test/resources/META-INF/services/*</exclude>\n                        <exclude>src/main/resources/META-INF/services/*</exclude>\n                        <exclude>*/target/**</exclude>\n                        <exclude>*/*.iml</exclude>\n                        <exclude>docs/**</exclude>\n                        <exclude>localbin/**</exclude>\n                        <exclude>conf/rmq-proxy.json</exclude>\n                        <exclude>.bazelversion</exclude>\n                    </excludes>\n                </configuration>\n            </plugin>\n            <plugin>\n                <artifactId>maven-resources-plugin</artifactId>\n                <version>${maven-resources-plugin.version}</version>\n                <configuration>\n                    <!-- We are not suppose to setup the customer resources here-->\n                    <encoding>${project.build.sourceEncoding}</encoding>\n                </configuration>\n            </plugin>\n            <plugin>\n                <groupId>org.eluder.coveralls</groupId>\n                <artifactId>coveralls-maven-plugin</artifactId>\n                <version>${coveralls-maven-plugin.version}</version>\n            </plugin>\n            <plugin>\n                <groupId>org.jacoco</groupId>\n                <artifactId>jacoco-maven-plugin</artifactId>\n                <version>${jacoco-maven-plugin.version}</version>\n                <executions>\n                    <execution>\n                        <id>default-prepare-agent</id>\n                        <goals>\n                            <goal>prepare-agent</goal>\n                        </goals>\n                        <configuration>\n                            <destFile>${project.build.directory}/jacoco.exec</destFile>\n                        </configuration>\n                    </execution>\n                    <execution>\n                        <id>report</id>\n                        <phase>test</phase>\n                        <goals>\n                            <goal>report</goal>\n                        </goals>\n                    </execution>\n                    <execution>\n                        <id>default-prepare-agent-integration</id>\n                        <phase>pre-integration-test</phase>\n                        <goals>\n                            <goal>prepare-agent-integration</goal>\n                        </goals>\n                        <configuration>\n                            <destFile>${project.build.directory}/jacoco-it.exec</destFile>\n                            <propertyName>failsafeArgLine</propertyName>\n                        </configuration>\n                    </execution>\n                    <execution>\n                        <id>default-report</id>\n                        <goals>\n                            <goal>report</goal>\n                        </goals>\n                    </execution>\n                    <execution>\n                        <id>default-report-integration</id>\n                        <goals>\n                            <goal>report-integration</goal>\n                        </goals>\n                    </execution>\n                </executions>\n            </plugin>\n            <plugin>\n                <artifactId>maven-surefire-plugin</artifactId>\n                <version>${maven-surefire-plugin.version}</version>\n                <configuration>\n                    <skipAfterFailureCount>1</skipAfterFailureCount>\n                    <forkCount>1</forkCount>\n                    <reuseForks>true</reuseForks>\n                    <excludes>\n                        <exclude>**/IT*.java</exclude>\n                    </excludes>\n                </configuration>\n            </plugin>\n            <plugin>\n                <groupId>org.sonarsource.scanner.maven</groupId>\n                <artifactId>sonar-maven-plugin</artifactId>\n                <version>${sonar-maven-plugin.version}</version>\n            </plugin>\n            <plugin>\n                <groupId>com.github.spotbugs</groupId>\n                <artifactId>spotbugs-maven-plugin</artifactId>\n                <version>${spotbugs-plugin.version}</version>\n                <executions>\n                    <execution>\n                        <id>check</id>\n                        <phase>compile</phase>\n                        <goals>\n                            <goal>check</goal>\n                        </goals>\n                    </execution>\n                </executions>\n                <configuration>\n                    <failOnError>true</failOnError>\n                    <fork>false</fork>\n                    <spotbugsXmlOutput>true</spotbugsXmlOutput>\n                    <excludeFilterFile>${project.root}/style/spotbugs-suppressions.xml</excludeFilterFile>\n                    <threshold>High</threshold>\n                    <effort>Max</effort>\n                </configuration>\n            </plugin>\n        </plugins>\n\n        <pluginManagement>\n            <plugins>\n                <plugin>\n                    <artifactId>maven-assembly-plugin</artifactId>\n                    <version>${maven-assembly-plugin.version}</version>\n                </plugin>\n            </plugins>\n        </pluginManagement>\n    </build>\n\n    <profiles>\n        <profile>\n            <id>jdk8</id>\n            <activation>\n                <jdk>[1.8,)</jdk>\n            </activation>\n            <!-- Disable doclint under JDK 8 -->\n            <reporting>\n                <plugins>\n                    <plugin>\n                        <artifactId>maven-javadoc-plugin</artifactId>\n                        <version>${maven-javadoc-plugin.version}</version>\n                        <configuration>\n                            <additionalparam>-Xdoclint:none</additionalparam>\n                        </configuration>\n                    </plugin>\n                </plugins>\n            </reporting>\n            <build>\n                <plugins>\n                    <plugin>\n                        <artifactId>maven-javadoc-plugin</artifactId>\n                        <version>${maven-javadoc-plugin.version}</version>\n                        <configuration>\n                            <additionalparam>-Xdoclint:none</additionalparam>\n                        </configuration>\n                    </plugin>\n                </plugins>\n            </build>\n        </profile>\n        <profile>\n            <id>release-sign-artifacts</id>\n            <activation>\n                <property>\n                    <name>performRelease</name>\n                    <value>true</value>\n                </property>\n            </activation>\n            <build>\n                <plugins>\n                    <plugin>\n                        <artifactId>maven-gpg-plugin</artifactId>\n                        <version>1.6</version>\n                        <executions>\n                            <execution>\n                                <id>sign-artifacts</id>\n                                <phase>verify</phase>\n                                <goals>\n                                    <goal>sign</goal>\n                                </goals>\n                            </execution>\n                        </executions>\n                    </plugin>\n                </plugins>\n            </build>\n        </profile>\n        <profile>\n            <id>it-test</id>\n            <build>\n                <plugins>\n                    <plugin>\n                        <artifactId>maven-failsafe-plugin</artifactId>\n                        <version>${maven-failsafe-plugin.version}</version>\n                        <configuration>\n                            <argLine>@{failsafeArgLine}</argLine>\n                            <excludes>\n                                <exclude>**/NormalMsgDelayIT.java</exclude>\n                            </excludes>\n                        </configuration>\n                        <executions>\n                            <execution>\n                                <goals>\n                                    <goal>integration-test</goal>\n                                    <goal>verify</goal>\n                                </goals>\n                            </execution>\n                        </executions>\n                    </plugin>\n                </plugins>\n            </build>\n        </profile>\n        <profile>\n            <id>sonar-apache</id>\n            <properties>\n                <!-- URL of the ASF SonarQube server -->\n                <sonar.host.url>https://builds.apache.org/analysis</sonar.host.url>\n            </properties>\n        </profile>\n        <profile>\n            <id>skip-unit-tests</id>\n            <build>\n                <plugins>\n                    <plugin>\n                        <artifactId>maven-surefire-plugin</artifactId>\n                        <version>${maven-surefire-plugin.version}</version>\n                        <configuration>\n                            <skipTests>true</skipTests>\n                            <skip>true</skip>\n                        </configuration>\n                    </plugin>\n                </plugins>\n            </build>\n        </profile>\n    </profiles>\n\n    <dependencyManagement>\n        <dependencies>\n            <dependency>\n                <groupId>org.apache.rocketmq</groupId>\n                <artifactId>rocketmq-auth</artifactId>\n                <version>${project.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.apache.rocketmq</groupId>\n                <artifactId>rocketmq-broker</artifactId>\n                <version>${project.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.apache.rocketmq</groupId>\n                <artifactId>rocketmq-client</artifactId>\n                <version>${project.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.apache.rocketmq</groupId>\n                <artifactId>rocketmq-common</artifactId>\n                <version>${project.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.apache.rocketmq</groupId>\n                <artifactId>rocketmq-container</artifactId>\n                <version>${project.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.apache.rocketmq</groupId>\n                <artifactId>rocketmq-controller</artifactId>\n                <version>${project.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.apache.rocketmq</groupId>\n                <artifactId>rocketmq-example</artifactId>\n                <version>${project.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.apache.rocketmq</groupId>\n                <artifactId>rocketmq-filter</artifactId>\n                <version>${project.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.apache.rocketmq</groupId>\n                <artifactId>rocketmq-namesrv</artifactId>\n                <version>${project.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.apache.rocketmq</groupId>\n                <artifactId>rocketmq-openmessaging</artifactId>\n                <version>${project.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.apache.rocketmq</groupId>\n                <artifactId>rocketmq-proxy</artifactId>\n                <version>${project.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.apache.rocketmq</groupId>\n                <artifactId>rocketmq-remoting</artifactId>\n                <version>${project.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.apache.rocketmq</groupId>\n                <artifactId>rocketmq-srvutil</artifactId>\n                <version>${project.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.apache.rocketmq</groupId>\n                <artifactId>rocketmq-store</artifactId>\n                <version>${project.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.apache.rocketmq</groupId>\n                <artifactId>rocketmq-tiered-store</artifactId>\n                <version>${project.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.apache.rocketmq</groupId>\n                <artifactId>rocketmq-test</artifactId>\n                <version>${project.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.apache.rocketmq</groupId>\n                <artifactId>rocketmq-tools</artifactId>\n                <version>${project.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>${project.groupId}</groupId>\n                <artifactId>rocketmq-proto</artifactId>\n                <version>${rocketmq-proto.version}</version>\n                <exclusions>\n                    <exclusion>\n                        <groupId>*</groupId>\n                        <artifactId>*</artifactId>\n                    </exclusion>\n                </exclusions>\n            </dependency>\n            <dependency>\n                <groupId>commons-cli</groupId>\n                <artifactId>commons-cli</artifactId>\n                <version>${commons-cli.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>io.netty</groupId>\n                <artifactId>netty-all</artifactId>\n                <version>${netty.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.bouncycastle</groupId>\n                <artifactId>bcpkix-jdk18on</artifactId>\n                <scope>runtime</scope>\n                <type>jar</type>\n                <version>${bcpkix-jdk18on.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>com.alibaba</groupId>\n                <artifactId>fastjson</artifactId>\n                <version>${fastjson.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>com.alibaba.fastjson2</groupId>\n                <artifactId>fastjson2</artifactId>\n                <version>${fastjson2.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.javassist</groupId>\n                <artifactId>javassist</artifactId>\n                <version>${javassist.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>net.java.dev.jna</groupId>\n                <artifactId>jna</artifactId>\n                <version>${jna.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.apache.commons</groupId>\n                <artifactId>commons-lang3</artifactId>\n                <version>${commons-lang3.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>commons-io</groupId>\n                <artifactId>commons-io</artifactId>\n                <version>${commons-io.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>com.google.guava</groupId>\n                <artifactId>guava</artifactId>\n                <version>${guava.version}</version>\n                <exclusions>\n                    <exclusion>\n                        <groupId>com.google.errorprone</groupId>\n                        <artifactId>error_prone_annotations</artifactId>\n                    </exclusion>\n                </exclusions>\n            </dependency>\n            <dependency>\n                <groupId>com.google.code</groupId>\n                <artifactId>gson</artifactId>\n                <version>${gson.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>com.googlecode.concurrentlinkedhashmap</groupId>\n                <artifactId>concurrentlinkedhashmap-lru</artifactId>\n                <version>${concurrentlinkedhashmap-lru.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>io.openmessaging</groupId>\n                <artifactId>openmessaging-api</artifactId>\n                <version>${openmessaging.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.yaml</groupId>\n                <artifactId>snakeyaml</artifactId>\n                <version>${snakeyaml.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>commons-codec</groupId>\n                <artifactId>commons-codec</artifactId>\n                <version>${commons-codec.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.slf4j</groupId>\n                <artifactId>slf4j-api</artifactId>\n                <version>${slf4j-api.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.apache.rocketmq</groupId>\n                <artifactId>rocketmq-rocksdb</artifactId>\n                <version>${rocksdb.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>io.github.aliyunmq</groupId>\n                <artifactId>rocketmq-shaded-slf4j-api-bridge</artifactId>\n                <version>${rocketmq-shaded-slf4j-api-bridge.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>io.github.aliyunmq</groupId>\n                <artifactId>rocketmq-slf4j-api</artifactId>\n                <version>${rocketmq-logging.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>io.github.aliyunmq</groupId>\n                <artifactId>rocketmq-logback-classic</artifactId>\n                <version>${rocketmq-logging.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>commons-validator</groupId>\n                <artifactId>commons-validator</artifactId>\n                <version>${commons-validator.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>com.github.luben</groupId>\n                <artifactId>zstd-jni</artifactId>\n                <version>${zstd-jni.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>at.yawk.lz4</groupId>\n                <artifactId>lz4-java</artifactId>\n                <version>${lz4-java.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>io.opentracing</groupId>\n                <artifactId>opentracing-api</artifactId>\n                <version>${opentracing.version}</version>\n                <scope>provided</scope>\n            </dependency>\n            <dependency>\n                <groupId>io.opentracing</groupId>\n                <artifactId>opentracing-mock</artifactId>\n                <version>${opentracing.version}</version>\n                <scope>test</scope>\n            </dependency>\n            <dependency>\n                <groupId>io.jaegertracing</groupId>\n                <artifactId>jaeger-core</artifactId>\n                <version>${jaeger.version}</version>\n                <exclusions>\n                    <exclusion>\n                        <groupId>com.google.code.gson</groupId>\n                        <artifactId>gson</artifactId>\n                    </exclusion>\n                </exclusions>\n            </dependency>\n            <dependency>\n                <groupId>io.jaegertracing</groupId>\n                <artifactId>jaeger-thrift</artifactId>\n                <version>${jaeger.version}</version>\n                <exclusions>\n                    <exclusion>\n                        <groupId>com.squareup.okhttp3</groupId>\n                        <artifactId>okhttp</artifactId>\n                    </exclusion>\n                </exclusions>\n            </dependency>\n            <dependency>\n                <groupId>io.jaegertracing</groupId>\n                <artifactId>jaeger-client</artifactId>\n                <version>${jaeger.version}</version>\n                <exclusions>\n                    <exclusion>\n                        <groupId>org.jetbrains.kotlin</groupId>\n                        <artifactId>kotlin-stdlib</artifactId>\n                    </exclusion>\n                </exclusions>\n            </dependency>\n            <dependency>\n                <groupId>io.openmessaging.storage</groupId>\n                <artifactId>dledger</artifactId>\n                <version>${dleger.version}</version>\n                <exclusions>\n                    <exclusion>\n                        <groupId>org.slf4j</groupId>\n                        <artifactId>slf4j-api</artifactId>\n                    </exclusion>\n                </exclusions>\n            </dependency>\n            <dependency>\n                <groupId>org.apache.tomcat</groupId>\n                <artifactId>annotations-api</artifactId>\n                <version>${annotations-api.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>junit</groupId>\n                <artifactId>junit</artifactId>\n                <version>${junit.version}</version>\n                <scope>test</scope>\n            </dependency>\n            <dependency>\n                <groupId>org.assertj</groupId>\n                <artifactId>assertj-core</artifactId>\n                <version>${assertj-core.version}</version>\n                <scope>test</scope>\n            </dependency>\n            <dependency>\n                <groupId>org.mockito</groupId>\n                <artifactId>mockito-core</artifactId>\n                <version>${mockito-core.version}</version>\n                <scope>test</scope>\n            </dependency>\n            <dependency>\n                <groupId>org.mockito</groupId>\n                <artifactId>mockito-junit-jupiter</artifactId>\n                <version>${mockito-junit-jupiter.version}</version>\n                <scope>test</scope>\n            </dependency>\n            <dependency>\n                <groupId>org.awaitility</groupId>\n                <artifactId>awaitility</artifactId>\n                <version>${awaitility.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>com.google.truth</groupId>\n                <artifactId>truth</artifactId>\n                <version>${truth.version}</version>\n                <exclusions>\n                    <exclusion>\n                        <groupId>com.google.errorprone</groupId>\n                        <artifactId>error_prone_annotations</artifactId>\n                    </exclusion>\n                </exclusions>\n            </dependency>\n            <!--only used in the test module-->\n            <dependency>\n                <groupId>org.reflections</groupId>\n                <artifactId>reflections</artifactId>\n                <version>${org.relection.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>io.grpc</groupId>\n                <artifactId>grpc-netty-shaded</artifactId>\n                <version>${grpc.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>io.grpc</groupId>\n                <artifactId>grpc-protobuf</artifactId>\n                <version>${grpc.version}</version>\n                <exclusions>\n                    <exclusion>\n                        <groupId>com.google.protobuf</groupId>\n                        <artifactId>protobuf-java</artifactId>\n                    </exclusion>\n                </exclusions>\n            </dependency>\n            <dependency>\n                <groupId>io.grpc</groupId>\n                <artifactId>grpc-stub</artifactId>\n                <version>${grpc.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>io.grpc</groupId>\n                <artifactId>grpc-services</artifactId>\n                <version>${grpc.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>io.grpc</groupId>\n                <artifactId>grpc-testing</artifactId>\n                <version>${grpc.version}</version>\n                <scope>test</scope>\n            </dependency>\n            <dependency>\n                <groupId>com.google.protobuf</groupId>\n                <artifactId>protobuf-java-util</artifactId>\n                <version>${protobuf.version}</version>\n                <exclusions>\n                    <exclusion>\n                        <groupId>com.google.errorprone</groupId>\n                        <artifactId>error_prone_annotations</artifactId>\n                    </exclusion>\n                    <exclusion>\n                        <groupId>com.google.code.gson</groupId>\n                        <artifactId>gson</artifactId>\n                    </exclusion>\n                    <exclusion>\n                        <groupId>com.google.j2objc</groupId>\n                        <artifactId>j2objc-annotations</artifactId>\n                    </exclusion>\n                </exclusions>\n            </dependency>\n            <dependency>\n                <groupId>io.github.aliyunmq</groupId>\n                <artifactId>rocketmq-grpc-netty-codec-haproxy</artifactId>\n                <version>1.0.0</version>\n            </dependency>\n            <dependency>\n                <groupId>com.conversantmedia</groupId>\n                <artifactId>disruptor</artifactId>\n                <version>${disruptor.version}</version>\n                <exclusions>\n                    <exclusion>\n                        <groupId>org.slf4j</groupId>\n                        <artifactId>slf4j-api</artifactId>\n                    </exclusion>\n                </exclusions>\n            </dependency>\n            <dependency>\n                <groupId>com.github.ben-manes.caffeine</groupId>\n                <artifactId>caffeine</artifactId>\n                <version>${caffeine.version}</version>\n                <exclusions>\n                    <exclusion>\n                        <groupId>com.google.errorprone</groupId>\n                        <artifactId>error_prone_annotations</artifactId>\n                    </exclusion>\n                </exclusions>\n            </dependency>\n            <dependency>\n                <groupId>io.netty</groupId>\n                <artifactId>netty-tcnative-boringssl-static</artifactId>\n                <version>${netty.tcnative.version}</version>\n            </dependency>\n\n            <dependency>\n                <groupId>org.springframework</groupId>\n                <artifactId>spring-core</artifactId>\n                <version>${spring.version}</version>\n                <scope>test</scope>\n            </dependency>\n\n            <dependency>\n                <groupId>com.squareup.okio</groupId>\n                <artifactId>okio-jvm</artifactId>\n                <version>${okio-jvm.version}</version>\n                <exclusions>\n                    <exclusion>\n                        <groupId>org.jetbrains.kotlin</groupId>\n                        <artifactId>kotlin-stdlib</artifactId>\n                    </exclusion>\n                    <exclusion>\n                        <groupId>org.jetbrains.kotlin</groupId>\n                        <artifactId>kotlin-stdlib-common</artifactId>\n                    </exclusion>\n                    <exclusion>\n                        <groupId>org.jetbrains.kotlin</groupId>\n                        <artifactId>kotlin-stdlib-jdk8</artifactId>\n                    </exclusion>\n                </exclusions>\n            </dependency>\n            <dependency>\n                <groupId>io.opentelemetry</groupId>\n                <artifactId>opentelemetry-exporter-otlp</artifactId>\n                <version>${opentelemetry.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>io.opentelemetry</groupId>\n                <artifactId>opentelemetry-exporter-prometheus</artifactId>\n                <version>${opentelemetry-exporter-prometheus.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>io.opentelemetry</groupId>\n                <artifactId>opentelemetry-exporter-logging</artifactId>\n                <version>${opentelemetry.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>io.opentelemetry</groupId>\n                <artifactId>opentelemetry-sdk</artifactId>\n                <version>${opentelemetry.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>io.opentelemetry</groupId>\n                <artifactId>opentelemetry-exporter-logging-otlp</artifactId>\n                <version>${opentelemetry.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>org.slf4j</groupId>\n                <artifactId>jul-to-slf4j</artifactId>\n                <version>${jul-to-slf4j.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>software.amazon.awssdk</groupId>\n                <artifactId>s3</artifactId>\n                <version>${s3.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>com.fasterxml.jackson.core</groupId>\n                <artifactId>jackson-databind</artifactId>\n                <version>${jackson-databind.version}</version>\n            </dependency>\n            <dependency>\n                <groupId>com.alipay.sofa</groupId>\n                <artifactId>jraft-core</artifactId>\n                <version>${sofa-jraft.version}</version>\n                <exclusions>\n                    <exclusion>\n                        <groupId>org.ow2.asm</groupId>\n                        <artifactId>asm</artifactId>\n                    </exclusion>\n                    <exclusion>\n                        <groupId>com.google.protobuf</groupId>\n                        <artifactId>protobuf-java</artifactId>\n                    </exclusion>\n                    <exclusion>\n                        <groupId>org.rocksdb</groupId>\n                        <artifactId>rocksdbjni</artifactId>\n                    </exclusion>\n                </exclusions>\n            </dependency>\n            <dependency>\n                <groupId>com.adobe.testing</groupId>\n                <artifactId>s3mock-junit4</artifactId>\n                <version>${s3mock-junit4.version}</version>\n                <scope>test</scope>\n                <exclusions>\n                    <exclusion>\n                        <artifactId>annotations</artifactId>\n                        <groupId>software.amazon.awssdk</groupId>\n                    </exclusion>\n                    <exclusion>\n                        <artifactId>commons-logging</artifactId>\n                        <groupId>commons-logging</groupId>\n                    </exclusion>\n                    <exclusion>\n                        <artifactId>http-client-spi</artifactId>\n                        <groupId>software.amazon.awssdk</groupId>\n                    </exclusion>\n                    <exclusion>\n                        <artifactId>json-utils</artifactId>\n                        <groupId>software.amazon.awssdk</groupId>\n                    </exclusion>\n                    <exclusion>\n                        <artifactId>profiles</artifactId>\n                        <groupId>software.amazon.awssdk</groupId>\n                    </exclusion>\n                    <exclusion>\n                        <artifactId>regions</artifactId>\n                        <groupId>software.amazon.awssdk</groupId>\n                    </exclusion>\n                    <exclusion>\n                        <artifactId>sdk-core</artifactId>\n                        <groupId>software.amazon.awssdk</groupId>\n                    </exclusion>\n                    <exclusion>\n                        <artifactId>utils</artifactId>\n                        <groupId>software.amazon.awssdk</groupId>\n                    </exclusion>\n                    <exclusion>\n                        <artifactId>jackson-dataformat-cbor</artifactId>\n                        <groupId>com.fasterxml.jackson.dataformat</groupId>\n                    </exclusion>\n                </exclusions>\n            </dependency>\n\n            <dependency>\n                <groupId>jakarta.annotation</groupId>\n                <artifactId>jakarta.annotation-api</artifactId>\n                <version>1.3.5</version>\n            </dependency>\n        </dependencies>\n    </dependencyManagement>\n\n    <dependencies>\n        <dependency>\n            <groupId>junit</groupId>\n            <artifactId>junit</artifactId>\n            <version>${junit.version}</version>\n            <scope>test</scope>\n        </dependency>\n        <dependency>\n            <groupId>org.assertj</groupId>\n            <artifactId>assertj-core</artifactId>\n            <version>${assertj-core.version}</version>\n            <scope>test</scope>\n        </dependency>\n        <dependency>\n            <groupId>org.mockito</groupId>\n            <artifactId>mockito-core</artifactId>\n            <version>${mockito-core.version}</version>\n            <scope>test</scope>\n        </dependency>\n        <dependency>\n            <groupId>org.mockito</groupId>\n            <artifactId>mockito-junit-jupiter</artifactId>\n            <version>${mockito-junit-jupiter.version}</version>\n            <scope>test</scope>\n        </dependency>\n        <dependency>\n            <groupId>org.awaitility</groupId>\n            <artifactId>awaitility</artifactId>\n            <version>${awaitility.version}</version>\n        </dependency>\n        <dependency>\n            <groupId>org.powermock</groupId>\n            <artifactId>powermock-module-junit4</artifactId>\n            <version>${powermock-version}</version>\n            <scope>test</scope>\n            <exclusions>\n                <exclusion>\n                    <groupId>org.objenesis</groupId>\n                    <artifactId>objenesis</artifactId>\n                </exclusion>\n                <exclusion>\n                    <groupId>net.bytebuddy</groupId>\n                    <artifactId>byte-buddy</artifactId>\n                </exclusion>\n                <exclusion>\n                    <groupId>net.bytebuddy</groupId>\n                    <artifactId>byte-buddy-agent</artifactId>\n                </exclusion>\n            </exclusions>\n        </dependency>\n        <dependency>\n            <groupId>org.powermock</groupId>\n            <artifactId>powermock-api-mockito2</artifactId>\n            <version>${powermock-version}</version>\n            <scope>test</scope>\n        </dependency>\n    </dependencies>\n</project>\n"
  },
  {
    "path": "proxy/BUILD.bazel",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\nload(\"//bazel:GenTestRules.bzl\", \"GenTestRules\")\n\njava_library(\n    name = \"proxy\",\n    srcs = glob([\"src/main/java/**/*.java\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \"//auth\",\n        \"//broker\",\n        \"//client\",\n        \"//common\",\n        \"//remoting\",\n        \"//srvutil\",\n        \"@maven//:ch_qos_logback_logback_classic\",\n        \"@maven//:ch_qos_logback_logback_core\",\n        \"@maven//:com_alibaba_fastjson2_fastjson2\",\n        \"@maven//:com_github_ben_manes_caffeine_caffeine\",\n        \"@maven//:com_github_luben_zstd_jni\",\n        \"@maven//:com_google_code_findbugs_jsr305\",\n        \"@maven//:com_google_guava_guava\",\n        \"@maven//:com_google_protobuf_protobuf_java\",\n        \"@maven//:com_google_protobuf_protobuf_java_util\",\n        \"@maven//:commons_cli_commons_cli\",\n        \"@maven//:commons_codec_commons_codec\",\n        \"@maven//:commons_collections_commons_collections\",\n        \"@maven//:commons_validator_commons_validator\",\n        \"@maven//:io_grpc_grpc_api\",\n        \"@maven//:io_grpc_grpc_context\",\n        \"@maven//:io_grpc_grpc_netty_shaded\",\n        \"@maven//:io_grpc_grpc_services\",\n        \"@maven//:io_grpc_grpc_stub\",\n        \"@maven//:io_netty_netty_all\",\n        \"@maven//:io_github_aliyunmq_rocketmq_grpc_netty_codec_haproxy\",\n        \"@maven//:io_openmessaging_storage_dledger\",\n        \"@maven//:io_opentelemetry_opentelemetry_api\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_otlp\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_prometheus\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_logging\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk_common\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk_metrics\",\n        \"@maven//:org_apache_commons_commons_lang3\",\n        \"@maven//:org_apache_rocketmq_rocketmq_proto\",\n        \"@maven//:org_checkerframework_checker_qual\",\n        \"@maven//:org_lz4_lz4_java\",\n        \"@maven//:org_slf4j_slf4j_api\",\n        \"@maven//:io_github_aliyunmq_rocketmq_slf4j_api\",\n        \"@maven//:org_slf4j_jul_to_slf4j\",\n        \"@maven//:io_github_aliyunmq_rocketmq_logback_classic\",\n    ],\n)\n\njava_library(\n    name = \"tests\",\n    srcs = glob([\"src/test/java/**/*.java\"]),\n    resources = [\n        \"src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker\",\n        \"src/test/resources/rmq-proxy-home/conf/broker.conf\",\n        \"src/test/resources/rmq-proxy-home/conf/logback_proxy.xml\",\n        \"src/test/resources/rmq-proxy-home/conf/rmq-proxy.json\",\n    ] + glob([\"src/test/resources/certs/*.pem\"]) + glob([\"src/test/resources/certs/*.key\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n\t    \"//auth\",\n        \":proxy\",\n        \"//:test_deps\",\n        \"//broker\",\n        \"//client\",\n        \"//common\",\n        \"//srvutil\",\n        \"//remoting\",\n        \"@maven//:ch_qos_logback_logback_core\",\n        \"@maven//:com_alibaba_fastjson2_fastjson2\",\n        \"@maven//:com_github_ben_manes_caffeine_caffeine\",\n        \"@maven//:com_google_guava_guava\",\n        \"@maven//:com_google_protobuf_protobuf_java\",\n        \"@maven//:com_google_protobuf_protobuf_java_util\",\n        \"@maven//:io_grpc_grpc_api\",\n        \"@maven//:io_grpc_grpc_context\",\n        \"@maven//:io_grpc_grpc_netty_shaded\",\n        \"@maven//:io_grpc_grpc_stub\",\n        \"@maven//:io_netty_netty_all\",\n        \"@maven//:io_github_aliyunmq_rocketmq_grpc_netty_codec_haproxy\",\n        \"@maven//:org_apache_commons_commons_lang3\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_otlp\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_prometheus\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk\",\n        \"@maven//:org_apache_rocketmq_rocketmq_proto\",\n        \"@maven//:org_checkerframework_checker_qual\",\n        \"@maven//:org_slf4j_slf4j_api\",\n        \"@maven//:org_springframework_spring_core\",\n    \t\"@maven//:org_jetbrains_annotations\",\n        \"@maven//:org_slf4j_jul_to_slf4j\",\n        \"@maven//:io_netty_netty_tcnative_boringssl_static\",\n        \"@maven//:commons_codec_commons_codec\",\n    ],\n)\n\nGenTestRules(\n    name = \"GeneratedTestRules\",\n    exclude_tests = [\n        \"src/test/java/org/apache/rocketmq/proxy/config/InitConfigTest\",\n        \"src/test/java/org/apache/rocketmq/proxy/service/cert/TlsCertificateManagerTest\",\n    ],\n    test_files = glob([\"src/test/java/**/*Test.java\"]),\n    deps = [\n        \":tests\",\n    ],\n)\n"
  },
  {
    "path": "proxy/README.md",
    "content": "rocketmq-proxy\n--------\n\n## Introduction\n\n`RocketMQ Proxy` is a stateless component that makes full use of the newly introduced `pop` consumption mechanism to\nachieve stateless consumption behavior. `gRPC` protocol is supported by `Proxy` now and all the message types\nincluding `normal`, `fifo`, `transaction` and `delay` are supported via `pop` consumption mode. `Proxy` will translate\nincoming traffic into customized `Remoting` protocol to access `Broker` and `Namesrv`.\n\n`Proxy` also handles SSL, authorization/authentication and logging/tracing/metrics and is in charge of connection\nmanagement and traffic governance.\n\n### Multi-language support.\n\n`gRPC` combined with `Protocol Buffer` makes it easy to implement clients with both `java` and other programming\nlanguages while the server side doesn't need extra work to support different programming languages.\nSee [rocketmq-clients](https://github.com/apache/rocketmq-clients) for more information.\n\n### Multi-protocol support.\n\nWith `Proxy` served as a traffic interface, it's convenient to implement multiple protocols upon proxy. `gRPC` protocol\nis implemented first and the customized `Remoting` protocol will be implemented later. HTTP/1.1 will also be taken into\nconsideration.\n\n## Architecture\n\n`RocketMQ Proxy` has two deployment modes: `Cluster` mode and `Local` mode. With both modes, `Pop` mode is natively\nsupported in `Proxy`.\n\n### `Cluster` mode\n\nWhile in `Cluster` mode, `Proxy` is an independent cluster that communicates with `Broker` with remote procedure call.\nIn this scenario, `Proxy` acts as a stateless computing component while `Broker` is a stateful component with local\nstorage. This form of deployment introduces the architecture of separation of computing and storage for RocketMQ.\n\nDue to the separation of computing and storage, `RocketMQ Proxy` can be scaled out indefinitely in `Cluster` mode to\nhandle traffic peak while `Broker` can focus on storage engine and high availability.\n\n![](../docs/en/images/rocketmq_proxy_cluster_mode.png)\n\n### `Local` mode\n\n`Proxy` in `Local` mode has more similarity with `RocketMQ` 4.x version, which is easily deployed or upgraded for\ncurrent RocketMQ users. With `Local` mode, `Proxy` deployed with `Broker` in the same process with inter-process\ncommunication so the network overhead is reduced compared to `Cluster` mode.\n\n![](../docs/en/images/rocketmq_proxy_local_mode.png)\n\n## Deploy guide\n\nSee [Proxy Deployment](../docs/en/proxy/deploy_guide.md)\n\n## Related\n\n* [rocketmq-apis](https://github.com/apache/rocketmq-apis): Common communication protocol between server and client.\n* [rocketmq-clients](https://github.com/apache/rocketmq-clients): Collection of Polyglot Clients for Apache RocketMQ.\n* [RIP-37: New and Unified APIs](https://shimo.im/docs/m5kv92OeRRU8olqX): RocketMQ proposal of new and unified APIs\n  crossing different languages.\n* [RIP-39: Support gRPC protocol](https://shimo.im/docs/gXqmeEPYgdUw5bqo): RocketMQ proposal of gRPC protocol support."
  },
  {
    "path": "proxy/pom.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one or more\n  ~ contributor license agreements.  See the NOTICE file distributed with\n  ~ this work for additional information regarding copyright ownership.\n  ~ The ASF licenses this file to You under the Apache License, Version 2.0\n  ~ (the \"License\"); you may not use this file except in compliance with\n  ~ the License.  You may obtain a copy of the License at\n  ~\n  ~     http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing, software\n  ~ distributed under the License is distributed on an \"AS IS\" BASIS,\n  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  ~ See the License for the specific language governing permissions and\n  ~ limitations under the License.\n  -->\n\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n    <parent>\n        <artifactId>rocketmq-all</artifactId>\n        <groupId>org.apache.rocketmq</groupId>\n        <version>${revision}</version>\n    </parent>\n\n    <modelVersion>4.0.0</modelVersion>\n    <packaging>jar</packaging>\n    <artifactId>rocketmq-proxy</artifactId>\n    <name>rocketmq-proxy ${project.version}</name>\n\n    <properties>\n        <maven.compiler.source>8</maven.compiler.source>\n        <maven.compiler.target>8</maven.compiler.target>\n        <project.root>${basedir}/..</project.root>\n    </properties>\n\n    <dependencies>\n        <dependency>\n            <groupId>org.apache.rocketmq</groupId>\n            <artifactId>rocketmq-proto</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.apache.rocketmq</groupId>\n            <artifactId>rocketmq-broker</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.apache.rocketmq</groupId>\n            <artifactId>rocketmq-common</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.apache.rocketmq</groupId>\n            <artifactId>rocketmq-client</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.apache.rocketmq</groupId>\n            <artifactId>rocketmq-auth</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.grpc</groupId>\n            <artifactId>grpc-netty-shaded</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.grpc</groupId>\n            <artifactId>grpc-protobuf</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.grpc</groupId>\n            <artifactId>grpc-stub</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.grpc</groupId>\n            <artifactId>grpc-services</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>com.google.protobuf</groupId>\n            <artifactId>protobuf-java-util</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.github.aliyunmq</groupId>\n            <artifactId>rocketmq-grpc-netty-codec-haproxy</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.apache.commons</groupId>\n            <artifactId>commons-lang3</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.github.aliyunmq</groupId>\n            <artifactId>rocketmq-slf4j-api</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.github.aliyunmq</groupId>\n            <artifactId>rocketmq-logback-classic</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>com.github.ben-manes.caffeine</groupId>\n            <artifactId>caffeine</artifactId>\n            <exclusions>\n                <exclusion>\n                    <groupId>org.checkerframework</groupId>\n                    <artifactId>checker-qual</artifactId>\n                </exclusion>\n            </exclusions>\n        </dependency>\n        <dependency>\n            <groupId>io.netty</groupId>\n            <artifactId>netty-tcnative-boringssl-static</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.springframework</groupId>\n            <artifactId>spring-core</artifactId>\n            <scope>test</scope>\n        </dependency>\n        <dependency>\n            <groupId>org.slf4j</groupId>\n            <artifactId>jul-to-slf4j</artifactId>\n        </dependency>\n    </dependencies>\n</project>\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/CommandLineArgument.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy;\n\npublic class CommandLineArgument {\n    private String namesrvAddr;\n    private String brokerConfigPath;\n    private String proxyConfigPath;\n    private String proxyMode;\n\n    public String getNamesrvAddr() {\n        return namesrvAddr;\n    }\n\n    public void setNamesrvAddr(String namesrvAddr) {\n        this.namesrvAddr = namesrvAddr;\n    }\n\n    public String getBrokerConfigPath() {\n        return brokerConfigPath;\n    }\n\n    public void setBrokerConfigPath(String brokerConfigPath) {\n        this.brokerConfigPath = brokerConfigPath;\n    }\n\n    public String getProxyConfigPath() {\n        return proxyConfigPath;\n    }\n\n    public void setProxyConfigPath(String proxyConfigPath) {\n        this.proxyConfigPath = proxyConfigPath;\n    }\n\n    public String getProxyMode() {\n        return proxyMode;\n    }\n\n    public void setProxyMode(String proxyMode) {\n        this.proxyMode = proxyMode;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/ProxyMode.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy;\n\npublic enum ProxyMode {\n    LOCAL(\"LOCAL\"),\n    CLUSTER(\"CLUSTER\");\n\n    private final String mode;\n\n    ProxyMode(String mode) {\n        this.mode = mode;\n    }\n\n    public static boolean isClusterMode(String mode) {\n        if (mode == null) {\n            return false;\n        }\n        return CLUSTER.mode.equals(mode.toUpperCase());\n    }\n\n    public static boolean isClusterMode(ProxyMode mode) {\n        if (mode == null) {\n            return false;\n        }\n        return CLUSTER.equals(mode);\n    }\n\n    public static boolean isLocalMode(String mode) {\n        if (mode == null) {\n            return false;\n        }\n        return LOCAL.mode.equals(mode.toUpperCase());\n    }\n\n    public static boolean isLocalMode(ProxyMode mode) {\n        if (mode == null) {\n            return false;\n        }\n        return LOCAL.equals(mode);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy;\n\nimport com.google.common.collect.Lists;\nimport io.grpc.protobuf.services.ChannelzService;\nimport io.grpc.protobuf.services.ProtoReflectionService;\nimport org.apache.commons.cli.CommandLine;\nimport org.apache.commons.cli.DefaultParser;\nimport org.apache.commons.cli.Option;\nimport org.apache.commons.cli.Options;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.BrokerStartup;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.thread.ThreadPoolMonitor;\nimport org.apache.rocketmq.common.utils.AbstractStartAndShutdown;\nimport org.apache.rocketmq.common.utils.StartAndShutdown;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.config.Configuration;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.grpc.GrpcServer;\nimport org.apache.rocketmq.proxy.grpc.GrpcServerBuilder;\nimport org.apache.rocketmq.proxy.grpc.v2.GrpcMessagingApplication;\nimport org.apache.rocketmq.proxy.metrics.ProxyMetricsManager;\nimport org.apache.rocketmq.proxy.processor.DefaultMessagingProcessor;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.remoting.RemotingProtocolServer;\nimport org.apache.rocketmq.proxy.service.cert.TlsCertificateManager;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.srvutil.ServerUtil;\n\nimport java.util.Date;\nimport java.util.List;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\n\npublic class ProxyStartup {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    private static final ProxyStartAndShutdown PROXY_START_AND_SHUTDOWN = new ProxyStartAndShutdown();\n\n    private static class ProxyStartAndShutdown extends AbstractStartAndShutdown {\n        @Override\n        public void appendStartAndShutdown(StartAndShutdown startAndShutdown) {\n            super.appendStartAndShutdown(startAndShutdown);\n        }\n    }\n\n    public static void main(String[] args) {\n        try {\n            // parse argument from command line\n            CommandLineArgument commandLineArgument = parseCommandLineArgument(args);\n            initConfiguration(commandLineArgument);\n\n            // init thread pool monitor for proxy.\n            initThreadPoolMonitor();\n\n            ThreadPoolExecutor executor = createServerExecutor();\n\n            MessagingProcessor messagingProcessor = createMessagingProcessor();\n\n            // tls cert update\n            TlsCertificateManager tlsCertificateManager = new TlsCertificateManager();\n            PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(tlsCertificateManager);\n\n            // create grpcServer\n            GrpcServer grpcServer = GrpcServerBuilder.newBuilder(executor,\n                    ConfigurationManager.getProxyConfig().getGrpcServerPort(), tlsCertificateManager)\n                .addService(createServiceProcessor(messagingProcessor))\n                .addService(ChannelzService.newInstance(100))\n                .addService(ProtoReflectionService.newInstance())\n                .configInterceptor()\n                .shutdownTime(ConfigurationManager.getProxyConfig().getGrpcShutdownTimeSeconds(), TimeUnit.SECONDS)\n                .build();\n            PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(grpcServer);\n\n            RemotingProtocolServer remotingServer = new RemotingProtocolServer(messagingProcessor, tlsCertificateManager);\n            PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(remotingServer);\n\n            // start servers one by one.\n            PROXY_START_AND_SHUTDOWN.start();\n\n            Runtime.getRuntime().addShutdownHook(new Thread(() -> {\n                log.info(\"try to shutdown server\");\n                try {\n                    PROXY_START_AND_SHUTDOWN.preShutdown();\n                    PROXY_START_AND_SHUTDOWN.shutdown();\n                } catch (Exception e) {\n                    log.error(\"err when shutdown rocketmq-proxy\", e);\n                }\n            }));\n        } catch (Exception e) {\n            e.printStackTrace();\n            log.error(\"find an unexpect err.\", e);\n            System.exit(1);\n        }\n\n        System.out.printf(\"%s%n\", new Date() + \" rocketmq-proxy startup successfully\");\n        log.info(new Date() + \" rocketmq-proxy startup successfully\");\n    }\n\n    protected static void initConfiguration(CommandLineArgument commandLineArgument) throws Exception {\n        if (StringUtils.isNotBlank(commandLineArgument.getProxyConfigPath())) {\n            System.setProperty(Configuration.CONFIG_PATH_PROPERTY, commandLineArgument.getProxyConfigPath());\n        }\n        ConfigurationManager.initEnv();\n        ConfigurationManager.initConfig();\n        setConfigFromCommandLineArgument(commandLineArgument);\n        log.info(\"Current configuration: \" + ConfigurationManager.formatProxyConfig());\n\n    }\n\n    protected static CommandLineArgument parseCommandLineArgument(String[] args) {\n        CommandLine commandLine = ServerUtil.parseCmdLine(\"mqproxy\", args,\n            buildCommandlineOptions(), new DefaultParser());\n        if (commandLine == null) {\n            throw new RuntimeException(\"parse command line argument failed\");\n        }\n\n        CommandLineArgument commandLineArgument = new CommandLineArgument();\n        MixAll.properties2Object(ServerUtil.commandLine2Properties(commandLine), commandLineArgument);\n        return commandLineArgument;\n    }\n\n    private static Options buildCommandlineOptions() {\n        Options options = ServerUtil.buildCommandlineOptions(new Options());\n\n        Option opt = new Option(\"bc\", \"brokerConfigPath\", true, \"Broker config file path for local mode\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"pc\", \"proxyConfigPath\", true, \"Proxy config file path\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt = new Option(\"pm\", \"proxyMode\", true, \"Proxy run in local or cluster mode\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        return options;\n    }\n\n    private static void setConfigFromCommandLineArgument(CommandLineArgument commandLineArgument) {\n        if (StringUtils.isNotBlank(commandLineArgument.getNamesrvAddr())) {\n            ConfigurationManager.getProxyConfig().setNamesrvAddr(commandLineArgument.getNamesrvAddr());\n        }\n        if (StringUtils.isNotBlank(commandLineArgument.getBrokerConfigPath())) {\n            ConfigurationManager.getProxyConfig().setBrokerConfigPath(commandLineArgument.getBrokerConfigPath());\n        }\n        if (StringUtils.isNotBlank(commandLineArgument.getProxyMode())) {\n            ConfigurationManager.getProxyConfig().setProxyMode(commandLineArgument.getProxyMode());\n        }\n    }\n\n    protected static MessagingProcessor createMessagingProcessor() {\n        String proxyModeStr = ConfigurationManager.getProxyConfig().getProxyMode();\n        MessagingProcessor messagingProcessor;\n\n        if (ProxyMode.isClusterMode(proxyModeStr)) {\n            messagingProcessor = DefaultMessagingProcessor.createForClusterMode();\n            ProxyMetricsManager proxyMetricsManager = ProxyMetricsManager.initClusterMode(ConfigurationManager.getProxyConfig());\n            PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(proxyMetricsManager);\n        } else if (ProxyMode.isLocalMode(proxyModeStr)) {\n            BrokerController brokerController = createBrokerController();\n            ProxyMetricsManager.initLocalMode(brokerController.getBrokerMetricsManager(), ConfigurationManager.getProxyConfig());\n            StartAndShutdown brokerControllerWrapper = new StartAndShutdown() {\n                @Override\n                public void start() throws Exception {\n                    brokerController.start();\n                    String tip = \"The broker[\" + brokerController.getBrokerConfig().getBrokerName() + \", \"\n                        + brokerController.getBrokerAddr() + \"] boot success. serializeType=\" + RemotingCommand.getSerializeTypeConfigInThisServer();\n                    if (null != brokerController.getBrokerConfig().getNamesrvAddr()) {\n                        tip += \" and name server is \" + brokerController.getBrokerConfig().getNamesrvAddr();\n                    }\n                    log.info(tip);\n                }\n\n                @Override\n                public void shutdown() throws Exception {\n                    brokerController.shutdown();\n                }\n            };\n            PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(brokerControllerWrapper);\n            messagingProcessor = DefaultMessagingProcessor.createForLocalMode(brokerController);\n        } else {\n            throw new IllegalArgumentException(\"try to start grpc server with wrong mode, use 'local' or 'cluster'\");\n        }\n        PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(messagingProcessor);\n        return messagingProcessor;\n    }\n\n    private static GrpcMessagingApplication createServiceProcessor(MessagingProcessor messagingProcessor) {\n        GrpcMessagingApplication application = GrpcMessagingApplication.create(messagingProcessor);\n        PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(application);\n        return application;\n    }\n\n    protected static BrokerController createBrokerController() {\n        ProxyConfig config = ConfigurationManager.getProxyConfig();\n        List<String> brokerStartupArgList = Lists.newArrayList(\"-c\", config.getBrokerConfigPath());\n        if (StringUtils.isNotBlank(config.getNamesrvAddr())) {\n            brokerStartupArgList.add(\"-n\");\n            brokerStartupArgList.add(config.getNamesrvAddr());\n        }\n        String[] brokerStartupArgs = brokerStartupArgList.toArray(new String[0]);\n        return BrokerStartup.createBrokerController(brokerStartupArgs);\n    }\n\n    public static ThreadPoolExecutor createServerExecutor() {\n        ProxyConfig config = ConfigurationManager.getProxyConfig();\n        int threadPoolNums = config.getGrpcThreadPoolNums();\n        int threadPoolQueueCapacity = config.getGrpcThreadPoolQueueCapacity();\n        ThreadPoolExecutor executor = ThreadPoolMonitor.createAndMonitor(\n            threadPoolNums,\n            threadPoolNums,\n            1, TimeUnit.MINUTES,\n            \"GrpcRequestExecutorThread\",\n            threadPoolQueueCapacity\n        );\n        PROXY_START_AND_SHUTDOWN.appendShutdown(executor::shutdown);\n        return executor;\n    }\n\n    public static void initThreadPoolMonitor() {\n        ProxyConfig config = ConfigurationManager.getProxyConfig();\n        ThreadPoolMonitor.config(\n            LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME),\n            LoggerFactory.getLogger(LoggerName.PROXY_WATER_MARK_LOGGER_NAME),\n            config.isEnablePrintJstack(), config.getPrintJstackInMillis(),\n            config.getPrintThreadPoolStatusInMillis());\n        ThreadPoolMonitor.init();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/auth/ProxyAuthenticationMetadataProvider.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.auth;\n\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.function.Supplier;\nimport org.apache.rocketmq.auth.authentication.provider.AuthenticationMetadataProvider;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.proxy.service.metadata.MetadataService;\n\npublic class ProxyAuthenticationMetadataProvider implements AuthenticationMetadataProvider {\n\n    protected AuthConfig authConfig;\n    protected MetadataService metadataService;\n\n    @Override\n    public void initialize(AuthConfig authConfig, Supplier<?> metadataService) {\n        this.authConfig = authConfig;\n        if (metadataService != null) {\n            this.metadataService = (MetadataService) metadataService.get();\n        }\n    }\n\n    @Override\n    public void shutdown() {\n\n    }\n\n    @Override\n    public CompletableFuture<Void> createUser(User user) {\n        return null;\n    }\n\n    @Override\n    public CompletableFuture<Void> deleteUser(String username) {\n        return null;\n    }\n\n    @Override\n    public CompletableFuture<Void> updateUser(User user) {\n        return null;\n    }\n\n    @Override\n    public CompletableFuture<User> getUser(String username) {\n        return this.metadataService.getUser(null, username);\n    }\n\n    @Override\n    public CompletableFuture<List<User>> listUser(String filter) {\n        return null;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/auth/ProxyAuthorizationMetadataProvider.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.auth;\n\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.function.Supplier;\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.auth.authorization.provider.AuthorizationMetadataProvider;\nimport org.apache.rocketmq.auth.authorization.model.Acl;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.proxy.service.metadata.MetadataService;\n\npublic class ProxyAuthorizationMetadataProvider implements AuthorizationMetadataProvider {\n\n    protected AuthConfig authConfig;\n\n    protected MetadataService metadataService;\n\n    @Override\n    public void initialize(AuthConfig authConfig, Supplier<?> metadataService) {\n        this.authConfig = authConfig;\n        if (metadataService != null) {\n            this.metadataService = (MetadataService) metadataService.get();\n        }\n    }\n\n    @Override\n    public void shutdown() {\n\n    }\n\n    @Override\n    public CompletableFuture<Void> createAcl(Acl acl) {\n        return null;\n    }\n\n    @Override\n    public CompletableFuture<Void> deleteAcl(Subject subject) {\n        return null;\n    }\n\n    @Override\n    public CompletableFuture<Void> updateAcl(Acl acl) {\n        return null;\n    }\n\n    @Override\n    public CompletableFuture<Acl> getAcl(Subject subject) {\n        return this.metadataService.getAcl(null, subject);\n    }\n\n    @Override\n    public CompletableFuture<List<Acl>> listAcl(String subjectFilter, String resourceFilter) {\n        return null;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/common/AbstractCacheLoader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.common;\n\nimport com.google.common.cache.CacheLoader;\nimport com.google.common.util.concurrent.ListenableFuture;\nimport com.google.common.util.concurrent.ListenableFutureTask;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport javax.annotation.Nonnull;\n\npublic abstract class AbstractCacheLoader<K, V> extends CacheLoader<K, V> {\n    private final ThreadPoolExecutor cacheRefreshExecutor;\n\n    public AbstractCacheLoader(ThreadPoolExecutor cacheRefreshExecutor) {\n        this.cacheRefreshExecutor = cacheRefreshExecutor;\n    }\n\n    @Override\n    public ListenableFuture<V> reload(@Nonnull K key, @Nonnull V oldValue) throws Exception {\n        ListenableFutureTask<V> task = ListenableFutureTask.create(() -> {\n            try {\n                return getDirectly(key);\n            } catch (Exception e) {\n                onErr(key, e);\n                return oldValue;\n            }\n        });\n        cacheRefreshExecutor.execute(task);\n        return task;\n    }\n\n    @Override\n    public V load(@Nonnull K key) throws Exception {\n        return getDirectly(key);\n    }\n\n    protected abstract V getDirectly(K key) throws Exception;\n\n    protected abstract void onErr(K key, Exception e);\n}"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/common/Address.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.common;\n\nimport com.google.common.net.HostAndPort;\nimport java.util.Objects;\nimport org.apache.rocketmq.common.utils.IPAddressUtils;\n\npublic class Address {\n\n    public enum AddressScheme {\n        IPv4,\n        IPv6,\n        DOMAIN_NAME,\n        UNRECOGNIZED\n    }\n\n    private AddressScheme addressScheme;\n    private HostAndPort hostAndPort;\n\n    public Address(HostAndPort hostAndPort) {\n        this.addressScheme = buildScheme(hostAndPort);\n        this.hostAndPort = hostAndPort;\n    }\n\n    public Address(AddressScheme addressScheme, HostAndPort hostAndPort) {\n        this.addressScheme = addressScheme;\n        this.hostAndPort = hostAndPort;\n    }\n\n    public AddressScheme getAddressScheme() {\n        return addressScheme;\n    }\n\n    public void setAddressScheme(AddressScheme addressScheme) {\n        this.addressScheme = addressScheme;\n    }\n\n    public HostAndPort getHostAndPort() {\n        return hostAndPort;\n    }\n\n    public void setHostAndPort(HostAndPort hostAndPort) {\n        this.hostAndPort = hostAndPort;\n    }\n\n    private AddressScheme buildScheme(HostAndPort hostAndPort) {\n        if (hostAndPort == null) {\n            return AddressScheme.UNRECOGNIZED;\n        }\n        String address = hostAndPort.getHost();\n        if (IPAddressUtils.isValidIPv4(address)) {\n            return AddressScheme.IPv4;\n        }\n        if (IPAddressUtils.isValidIPv6(address)) {\n            return AddressScheme.IPv6;\n        }\n        return AddressScheme.DOMAIN_NAME;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) {\n            return true;\n        }\n        if (o == null || getClass() != o.getClass()) {\n            return false;\n        }\n        Address address = (Address) o;\n        return addressScheme == address.addressScheme && Objects.equals(hostAndPort, address.hostAndPort);\n    }\n\n    @Override\n    public int hashCode() {\n        return Objects.hash(addressScheme, hostAndPort);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/common/ContextVariable.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.common;\n\npublic class ContextVariable {\n    public static final String REMOTE_ADDRESS = \"remote-address\";\n    public static final String LOCAL_ADDRESS = \"local-address\";\n    public static final String CLIENT_ID = \"client-id\";\n    public static final String CHANNEL = \"channel\";\n    public static final String LANGUAGE = \"language\";\n    public static final String CLIENT_VERSION = \"client-version\";\n    public static final String REMAINING_MS = \"remaining-ms\";\n    public static final String ACTION = \"action\";\n    public static final String PROTOCOL_TYPE = \"protocol-type\";\n    public static final String NAMESPACE = \"namespace\";\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/common/MessageReceiptHandle.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.common;\n\nimport com.google.common.base.MoreObjects;\nimport com.google.common.base.Objects;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\n\npublic class MessageReceiptHandle {\n    private final String group;\n    private final String topic;\n    private final int queueId;\n    private final String messageId;\n    private final long queueOffset;\n    private final String originalReceiptHandleStr;\n    private final ReceiptHandle originalReceiptHandle;\n    private final int reconsumeTimes;\n\n    private final AtomicInteger renewRetryTimes = new AtomicInteger(0);\n    private final AtomicInteger renewTimes = new AtomicInteger(0);\n    private final long consumeTimestamp;\n    private String liteTopic;\n    private volatile String receiptHandleStr;\n\n    public MessageReceiptHandle(String group, String topic, int queueId, String receiptHandleStr, String messageId,\n        long queueOffset, int reconsumeTimes) {\n        this(group, topic, queueId, receiptHandleStr, messageId, queueOffset, reconsumeTimes, null);\n    }\n\n    public MessageReceiptHandle(String group, String topic, int queueId, String receiptHandleStr, String messageId,\n        long queueOffset, int reconsumeTimes, String liteTopic) {\n        this.originalReceiptHandle = ReceiptHandle.decode(receiptHandleStr);\n        this.group = group;\n        this.topic = topic;\n        this.queueId = queueId;\n        this.receiptHandleStr = receiptHandleStr;\n        this.originalReceiptHandleStr = receiptHandleStr;\n        this.messageId = messageId;\n        this.queueOffset = queueOffset;\n        this.reconsumeTimes = reconsumeTimes;\n        this.consumeTimestamp = originalReceiptHandle.getRetrieveTime();\n        this.liteTopic = liteTopic;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) {\n            return true;\n        }\n        if (o == null || getClass() != o.getClass()) {\n            return false;\n        }\n        MessageReceiptHandle handle = (MessageReceiptHandle) o;\n        return queueId == handle.queueId && queueOffset == handle.queueOffset && consumeTimestamp == handle.consumeTimestamp\n            && reconsumeTimes == handle.reconsumeTimes\n            && Objects.equal(group, handle.group) && Objects.equal(topic, handle.topic)\n            && Objects.equal(messageId, handle.messageId) && Objects.equal(originalReceiptHandleStr, handle.originalReceiptHandleStr)\n            && Objects.equal(receiptHandleStr, handle.receiptHandleStr);\n    }\n\n    @Override\n    public int hashCode() {\n        return Objects.hashCode(group, topic, queueId, messageId, queueOffset, originalReceiptHandleStr, consumeTimestamp,\n            reconsumeTimes, receiptHandleStr);\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"group\", group)\n            .add(\"topic\", topic)\n            .add(\"queueId\", queueId)\n            .add(\"messageId\", messageId)\n            .add(\"queueOffset\", queueOffset)\n            .add(\"originalReceiptHandleStr\", originalReceiptHandleStr)\n            .add(\"reconsumeTimes\", reconsumeTimes)\n            .add(\"renewRetryTimes\", renewRetryTimes)\n            .add(\"firstConsumeTimestamp\", consumeTimestamp)\n            .add(\"receiptHandleStr\", receiptHandleStr)\n            .add(\"liteTopic\", liteTopic)\n            .omitNullValues()\n            .toString();\n    }\n\n    public String getGroup() {\n        return group;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public int getQueueId() {\n        return queueId;\n    }\n\n    public String getReceiptHandleStr() {\n        return receiptHandleStr;\n    }\n\n    public String getOriginalReceiptHandleStr() {\n        return originalReceiptHandleStr;\n    }\n\n    public String getMessageId() {\n        return messageId;\n    }\n\n    public long getQueueOffset() {\n        return queueOffset;\n    }\n\n    public int getReconsumeTimes() {\n        return reconsumeTimes;\n    }\n\n    public long getConsumeTimestamp() {\n        return consumeTimestamp;\n    }\n\n    public void updateReceiptHandle(String receiptHandleStr) {\n        this.receiptHandleStr = receiptHandleStr;\n    }\n\n    public int incrementAndGetRenewRetryTimes() {\n        return this.renewRetryTimes.incrementAndGet();\n    }\n\n    public int incrementRenewTimes() {\n        return this.renewTimes.incrementAndGet();\n    }\n\n    public int getRenewTimes() {\n        return this.renewTimes.get();\n    }\n\n    public void resetRenewRetryTimes() {\n        this.renewRetryTimes.set(0);\n    }\n\n    public int getRenewRetryTimes() {\n        return this.renewRetryTimes.get();\n    }\n\n    public ReceiptHandle getOriginalReceiptHandle() {\n        return originalReceiptHandle;\n    }\n\n    public String getLiteTopic() {\n        return liteTopic;\n    }\n\n    public void setLiteTopic(String liteTopic) {\n        this.liteTopic = liteTopic;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/common/ProxyContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.common;\n\nimport io.netty.channel.Channel;\nimport java.util.HashMap;\nimport java.util.Map;\n\npublic class ProxyContext {\n    public static final String INNER_ACTION_PREFIX = \"Inner\";\n    private final Map<String, Object> value = new HashMap<>();\n\n    public static ProxyContext create() {\n        return new ProxyContext();\n    }\n\n    public static ProxyContext createForInner(String actionName) {\n        return create().setAction(INNER_ACTION_PREFIX + actionName);\n    }\n\n    public static ProxyContext createForInner(Class<?> clazz) {\n        return createForInner(clazz.getSimpleName());\n    }\n\n    public Map<String, Object> getValue() {\n        return this.value;\n    }\n\n    public ProxyContext withVal(String key, Object val) {\n        this.value.put(key, val);\n        return this;\n    }\n\n    public <T> T getVal(String key) {\n        return (T) this.value.get(key);\n    }\n\n    public ProxyContext setLocalAddress(String localAddress) {\n        this.withVal(ContextVariable.LOCAL_ADDRESS, localAddress);\n        return this;\n    }\n\n    public String getLocalAddress() {\n        return this.getVal(ContextVariable.LOCAL_ADDRESS);\n    }\n\n    public ProxyContext setRemoteAddress(String remoteAddress) {\n        this.withVal(ContextVariable.REMOTE_ADDRESS, remoteAddress);\n        return this;\n    }\n\n    public String getRemoteAddress() {\n        return this.getVal(ContextVariable.REMOTE_ADDRESS);\n    }\n\n    public ProxyContext setClientID(String clientID) {\n        this.withVal(ContextVariable.CLIENT_ID, clientID);\n        return this;\n    }\n\n    public String getClientID() {\n        return this.getVal(ContextVariable.CLIENT_ID);\n    }\n\n    public ProxyContext setChannel(Channel channel) {\n        this.withVal(ContextVariable.CHANNEL, channel);\n        return this;\n    }\n\n    public Channel getChannel() {\n        return this.getVal(ContextVariable.CHANNEL);\n    }\n\n    public ProxyContext setLanguage(String language) {\n        this.withVal(ContextVariable.LANGUAGE, language);\n        return this;\n    }\n\n    public String getLanguage() {\n        return this.getVal(ContextVariable.LANGUAGE);\n    }\n\n    public ProxyContext setClientVersion(String clientVersion) {\n        this.withVal(ContextVariable.CLIENT_VERSION, clientVersion);\n        return this;\n    }\n\n    public String getClientVersion() {\n        return this.getVal(ContextVariable.CLIENT_VERSION);\n    }\n\n    public ProxyContext setRemainingMs(Long remainingMs) {\n        this.withVal(ContextVariable.REMAINING_MS, remainingMs);\n        return this;\n    }\n\n    public Long getRemainingMs() {\n        return this.getVal(ContextVariable.REMAINING_MS);\n    }\n\n    public ProxyContext setAction(String action) {\n        this.withVal(ContextVariable.ACTION, action);\n        return this;\n    }\n\n    public String getAction() {\n        return this.getVal(ContextVariable.ACTION);\n    }\n\n    public ProxyContext setProtocolType(String protocol) {\n        this.withVal(ContextVariable.PROTOCOL_TYPE, protocol);\n        return this;\n    }\n\n    public String getProtocolType() {\n        return this.getVal(ContextVariable.PROTOCOL_TYPE);\n    }\n\n    public ProxyContext setNamespace(String namespace) {\n        this.withVal(ContextVariable.NAMESPACE, namespace);\n        return this;\n    }\n\n    public String getNamespace() {\n        return this.getVal(ContextVariable.NAMESPACE);\n    }\n\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/common/ProxyException.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.common;\n\npublic class ProxyException extends RuntimeException {\n\n    private final ProxyExceptionCode code;\n\n    public ProxyException(ProxyExceptionCode code, String message) {\n        super(message);\n        this.code = code;\n    }\n\n    public ProxyException(ProxyExceptionCode code, String message, Throwable cause) {\n        super(message, cause);\n        this.code = code;\n    }\n\n    public ProxyExceptionCode getCode() {\n        return code;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/common/ProxyExceptionCode.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.common;\n\npublic enum ProxyExceptionCode {\n    INVALID_BROKER_NAME,\n    TRANSACTION_DATA_NOT_FOUND,\n    FORBIDDEN,\n    MESSAGE_PROPERTY_CONFLICT_WITH_TYPE,\n    INVALID_RECEIPT_HANDLE,\n    INTERNAL_SERVER_ERROR,\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.common;\n\nimport com.google.common.base.MoreObjects;\nimport com.google.common.base.Objects;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.Semaphore;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.concurrent.atomic.AtomicReference;\nimport java.util.function.Function;\nimport org.apache.commons.lang3.builder.ToStringBuilder;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.common.utils.ConcurrentHashMapUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\n\npublic class ReceiptHandleGroup {\n    protected final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    // The messages having the same messageId will be deduplicated based on the parameters of broker, queueId, and offset\n    protected final Map<String /* msgID */, Map<HandleKey, HandleData>> receiptHandleMap = new ConcurrentHashMap<>();\n\n    public static class HandleKey {\n        private final String originalHandle;\n        private final String broker;\n        private final int queueId;\n        private final long offset;\n\n        public HandleKey(String handle) {\n            this(ReceiptHandle.decode(handle));\n        }\n\n        public HandleKey(ReceiptHandle receiptHandle) {\n            this.originalHandle = receiptHandle.getReceiptHandle();\n            this.broker = receiptHandle.getBrokerName();\n            this.queueId = receiptHandle.getQueueId();\n            this.offset = receiptHandle.getOffset();\n        }\n\n        @Override\n        public boolean equals(Object o) {\n            if (this == o)\n                return true;\n            if (o == null || getClass() != o.getClass())\n                return false;\n            HandleKey key = (HandleKey) o;\n            return queueId == key.queueId && offset == key.offset && Objects.equal(broker, key.broker);\n        }\n\n        @Override\n        public int hashCode() {\n            return Objects.hashCode(broker, queueId, offset);\n        }\n\n        @Override\n        public String toString() {\n            return new ToStringBuilder(this)\n                .append(\"originalHandle\", originalHandle)\n                .append(\"broker\", broker)\n                .append(\"queueId\", queueId)\n                .append(\"offset\", offset)\n                .toString();\n        }\n\n        public String getOriginalHandle() {\n            return originalHandle;\n        }\n\n        public String getBroker() {\n            return broker;\n        }\n\n        public int getQueueId() {\n            return queueId;\n        }\n\n        public long getOffset() {\n            return offset;\n        }\n    }\n\n    public static class HandleData {\n        private final Semaphore semaphore = new Semaphore(1);\n        private final AtomicLong lastLockTimeMs = new AtomicLong(-1L);\n        private volatile boolean needRemove = false;\n        private volatile MessageReceiptHandle messageReceiptHandle;\n\n        public HandleData(MessageReceiptHandle messageReceiptHandle) {\n            this.messageReceiptHandle = messageReceiptHandle;\n        }\n\n        public Long lock(long timeoutMs) {\n            try {\n                boolean result = this.semaphore.tryAcquire(timeoutMs, TimeUnit.MILLISECONDS);\n                long currentTimeMs = System.currentTimeMillis();\n                if (result) {\n                    this.lastLockTimeMs.set(currentTimeMs);\n                    return currentTimeMs;\n                } else {\n                    // if the lock is expired, can be acquired again\n                    long expiredTimeMs = ConfigurationManager.getProxyConfig().getLockTimeoutMsInHandleGroup() * 3;\n                    if (currentTimeMs - this.lastLockTimeMs.get() > expiredTimeMs) {\n                        synchronized (this) {\n                            if (currentTimeMs - this.lastLockTimeMs.get() > expiredTimeMs) {\n                                log.warn(\"HandleData lock expired, acquire lock success and reset lock time. \" +\n                                    \"MessageReceiptHandle={}, lockTime={}\", messageReceiptHandle, currentTimeMs);\n                                this.lastLockTimeMs.set(currentTimeMs);\n                                return currentTimeMs;\n                            }\n                        }\n                    }\n                }\n                return null;\n            } catch (InterruptedException e) {\n                return null;\n            }\n        }\n\n        public void unlock(long lockTimeMs) {\n            // if the lock is expired, we don't need to unlock it\n            if (System.currentTimeMillis() - lockTimeMs > ConfigurationManager.getProxyConfig().getLockTimeoutMsInHandleGroup() * 2) {\n                log.warn(\"HandleData lock expired, unlock fail. MessageReceiptHandle={}, lockTime={}, now={}\",\n                    messageReceiptHandle, lockTimeMs, System.currentTimeMillis());\n                return;\n            }\n            this.semaphore.release();\n        }\n\n        public MessageReceiptHandle getMessageReceiptHandle() {\n            return messageReceiptHandle;\n        }\n\n        @Override\n        public boolean equals(Object o) {\n            return this == o;\n        }\n\n        @Override\n        public int hashCode() {\n            return Objects.hashCode(semaphore, needRemove, messageReceiptHandle);\n        }\n\n        @Override\n        public String toString() {\n            return MoreObjects.toStringHelper(this)\n                .add(\"semaphore\", semaphore)\n                .add(\"needRemove\", needRemove)\n                .add(\"messageReceiptHandle\", messageReceiptHandle)\n                .toString();\n        }\n    }\n\n    public void put(String msgID, MessageReceiptHandle value) {\n        long timeout = ConfigurationManager.getProxyConfig().getLockTimeoutMsInHandleGroup();\n        Map<HandleKey, HandleData> handleMap = ConcurrentHashMapUtils.computeIfAbsent((ConcurrentHashMap<String, Map<HandleKey, HandleData>>) this.receiptHandleMap,\n            msgID, msgIDKey -> new ConcurrentHashMap<>());\n        handleMap.compute(new HandleKey(value.getOriginalReceiptHandle()), (handleKey, handleData) -> {\n            if (handleData == null || handleData.needRemove) {\n                return new HandleData(value);\n            }\n            Long lockTimeMs = handleData.lock(timeout);\n            if (lockTimeMs == null) {\n                throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, \"try to put handle failed\");\n            }\n            try {\n                if (handleData.needRemove) {\n                    return new HandleData(value);\n                }\n                handleData.messageReceiptHandle = value;\n            } finally {\n                handleData.unlock(lockTimeMs);\n            }\n            return handleData;\n        });\n    }\n\n    public boolean isEmpty() {\n        return this.receiptHandleMap.isEmpty();\n    }\n\n    public long getHandleNum() {\n        long handleNum = 0L;\n        for (Map.Entry<String, Map<HandleKey, HandleData>> entry : receiptHandleMap.entrySet()) {\n            handleNum += entry.getValue().size();\n        }\n        return handleNum;\n    }\n\n    public int getMsgCount() {\n        return this.receiptHandleMap.size();\n    }\n\n    public MessageReceiptHandle get(String msgID, String handle) {\n        Map<HandleKey, HandleData> handleMap = this.receiptHandleMap.get(msgID);\n        if (handleMap == null) {\n            return null;\n        }\n        long timeout = ConfigurationManager.getProxyConfig().getLockTimeoutMsInHandleGroup();\n        AtomicReference<MessageReceiptHandle> res = new AtomicReference<>();\n        handleMap.computeIfPresent(new HandleKey(handle), (handleKey, handleData) -> {\n            Long lockTimeMs = handleData.lock(timeout);\n            if (lockTimeMs == null) {\n                throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, \"try to get handle failed\");\n            }\n            try {\n                if (handleData.needRemove) {\n                    return null;\n                }\n                res.set(handleData.messageReceiptHandle);\n            } finally {\n                handleData.unlock(lockTimeMs);\n            }\n            return handleData;\n        });\n        return res.get();\n    }\n\n    public MessageReceiptHandle remove(String msgID, String handle) {\n        Map<HandleKey, HandleData> handleMap = this.receiptHandleMap.get(msgID);\n        if (handleMap == null) {\n            return null;\n        }\n        long timeout = ConfigurationManager.getProxyConfig().getLockTimeoutMsInHandleGroup();\n        AtomicReference<MessageReceiptHandle> res = new AtomicReference<>();\n        handleMap.computeIfPresent(new HandleKey(handle), (handleKey, handleData) -> {\n            Long lockTimeMs = handleData.lock(timeout);\n            if (lockTimeMs == null) {\n                throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, \"try to remove and get handle failed\");\n            }\n            try {\n                if (!handleData.needRemove) {\n                    handleData.needRemove = true;\n                    res.set(handleData.messageReceiptHandle);\n                }\n                return null;\n            } finally {\n                handleData.unlock(lockTimeMs);\n            }\n        });\n        removeHandleMapKeyIfNeed(msgID);\n        return res.get();\n    }\n\n    public MessageReceiptHandle removeOne(String msgID) {\n        Map<HandleKey, HandleData> handleMap = this.receiptHandleMap.get(msgID);\n        if (handleMap == null) {\n            return null;\n        }\n        Set<HandleKey> keys = handleMap.keySet();\n        for (HandleKey key : keys) {\n            MessageReceiptHandle res = this.remove(msgID, key.originalHandle);\n            if (res != null) {\n                return res;\n            }\n        }\n        return null;\n    }\n\n    public void computeIfPresent(String msgID, String handle,\n        Function<MessageReceiptHandle, CompletableFuture<MessageReceiptHandle>> function) {\n        long timeout = ConfigurationManager.getProxyConfig().getLockTimeoutMsInHandleGroup();\n        computeIfPresent(msgID, handle, function, timeout);\n    }\n\n    public void computeIfPresent(String msgID, String handle,\n        Function<MessageReceiptHandle, CompletableFuture<MessageReceiptHandle>> function, long lockTimeout) {\n        Map<HandleKey, HandleData> handleMap = this.receiptHandleMap.get(msgID);\n        if (handleMap == null) {\n            return;\n        }\n        handleMap.computeIfPresent(new HandleKey(handle), (handleKey, handleData) -> {\n            Long lockTimeMs = handleData.lock(lockTimeout);\n            if (lockTimeMs == null) {\n                throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, \"try to compute failed\");\n            }\n            CompletableFuture<MessageReceiptHandle> future = function.apply(handleData.messageReceiptHandle);\n            future.whenComplete((messageReceiptHandle, throwable) -> {\n                try {\n                    if (throwable != null) {\n                        return;\n                    }\n                    if (messageReceiptHandle == null) {\n                        handleData.needRemove = true;\n                    } else {\n                        handleData.messageReceiptHandle = messageReceiptHandle;\n                    }\n                } finally {\n                    handleData.unlock(lockTimeMs);\n                }\n                if (handleData.needRemove) {\n                    handleMap.remove(handleKey, handleData);\n                }\n                removeHandleMapKeyIfNeed(msgID);\n            });\n            return handleData;\n        });\n    }\n\n    protected void removeHandleMapKeyIfNeed(String msgID) {\n        this.receiptHandleMap.computeIfPresent(msgID, (msgIDKey, handleMap) -> {\n            if (handleMap.isEmpty()) {\n                return null;\n            }\n            return handleMap;\n        });\n    }\n\n    public interface DataScanner {\n        void onData(String msgID, String handle, MessageReceiptHandle receiptHandle);\n    }\n\n    public void scan(DataScanner scanner) {\n        this.receiptHandleMap.forEach((msgID, handleMap) -> {\n            handleMap.forEach((handleKey, v) -> {\n                scanner.onData(msgID, handleKey.originalHandle, v.messageReceiptHandle);\n            });\n        });\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"receiptHandleMap\", receiptHandleMap)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroupKey.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.common;\n\nimport com.google.common.base.MoreObjects;\nimport com.google.common.base.Objects;\nimport io.netty.channel.Channel;\n\npublic class ReceiptHandleGroupKey {\n    protected final Channel channel;\n    protected final String group;\n\n    public ReceiptHandleGroupKey(Channel channel, String group) {\n        this.channel = channel;\n        this.group = group;\n    }\n\n    protected String getChannelId() {\n        return channel.id().asLongText();\n    }\n\n    public String getGroup() {\n        return group;\n    }\n\n    public Channel getChannel() {\n        return channel;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) {\n            return true;\n        }\n        if (o == null || getClass() != o.getClass()) {\n            return false;\n        }\n        ReceiptHandleGroupKey key = (ReceiptHandleGroupKey) o;\n        return Objects.equal(getChannelId(), key.getChannelId()) && Objects.equal(group, key.group);\n    }\n\n    @Override\n    public int hashCode() {\n        return Objects.hashCode(getChannelId(), group);\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"channelId\", getChannelId())\n            .add(\"group\", group)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/common/RenewEvent.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.common;\n\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.client.consumer.AckResult;\n\npublic class RenewEvent {\n    protected ReceiptHandleGroupKey key;\n    protected MessageReceiptHandle messageReceiptHandle;\n    protected long renewTime;\n    protected EventType eventType;\n    protected CompletableFuture<AckResult> future;\n\n    public enum EventType {\n        RENEW,\n        STOP_RENEW,\n        CLEAR_GROUP\n    }\n\n    public RenewEvent(ReceiptHandleGroupKey key, MessageReceiptHandle messageReceiptHandle, long renewTime,\n        EventType eventType, CompletableFuture<AckResult> future) {\n        this.key = key;\n        this.messageReceiptHandle = messageReceiptHandle;\n        this.renewTime = renewTime;\n        this.eventType = eventType;\n        this.future = future;\n    }\n\n    public ReceiptHandleGroupKey getKey() {\n        return key;\n    }\n\n    public MessageReceiptHandle getMessageReceiptHandle() {\n        return messageReceiptHandle;\n    }\n\n    public long getRenewTime() {\n        return renewTime;\n    }\n\n    public EventType getEventType() {\n        return eventType;\n    }\n\n    public CompletableFuture<AckResult> getFuture() {\n        return future;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/common/RenewStrategyPolicy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.common;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.remoting.protocol.subscription.RetryPolicy;\n\nimport java.util.concurrent.TimeUnit;\n\n\npublic class RenewStrategyPolicy implements RetryPolicy {\n    // 1m 3m 5m 6m 10m 30m 1h\n    private long[] next = new long[]{\n            TimeUnit.MINUTES.toMillis(1),\n            TimeUnit.MINUTES.toMillis(3),\n            TimeUnit.MINUTES.toMillis(5),\n            TimeUnit.MINUTES.toMillis(10),\n            TimeUnit.MINUTES.toMillis(30),\n            TimeUnit.HOURS.toMillis(1)\n    };\n\n    public RenewStrategyPolicy() {\n    }\n\n    public RenewStrategyPolicy(long[] next) {\n        this.next = next;\n    }\n\n    public long[] getNext() {\n        return next;\n    }\n\n    public void setNext(long[] next) {\n        this.next = next;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n                .add(\"next\", next)\n                .toString();\n    }\n\n    @Override\n    public long nextDelayDuration(int renewTimes) {\n        if (renewTimes < 0) {\n            renewTimes = 0;\n        }\n        int index = renewTimes;\n        if (index >= next.length) {\n            index = next.length - 1;\n        }\n        return next[index];\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/common/channel/ChannelHelper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.common.channel;\n\nimport io.netty.channel.Channel;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcClientChannel;\nimport org.apache.rocketmq.proxy.processor.channel.ChannelProtocolType;\nimport org.apache.rocketmq.proxy.processor.channel.RemoteChannel;\nimport org.apache.rocketmq.proxy.remoting.channel.RemotingChannel;\n\npublic class ChannelHelper {\n\n    /**\n     * judge channel is sync from other proxy or not\n     *\n     * @param channel channel\n     * @return true if is sync from other proxy\n     */\n    public static boolean isRemote(Channel channel) {\n        return channel instanceof RemoteChannel;\n    }\n\n    public static ChannelProtocolType getChannelProtocolType(Channel channel) {\n        if (channel instanceof GrpcClientChannel) {\n            return ChannelProtocolType.GRPC_V2;\n        } else if (channel instanceof RemotingChannel) {\n            return ChannelProtocolType.REMOTING;\n        } else if (channel instanceof RemoteChannel) {\n            RemoteChannel remoteChannel = (RemoteChannel) channel;\n            return remoteChannel.getType();\n        }\n        return ChannelProtocolType.UNKNOWN;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/FilterUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.common.utils;\n\nimport java.util.Set;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\npublic class FilterUtils {\n    /**\n     * Whether the message's tag matches consumerGroup's SubscriptionData\n     *\n     * @param tagsSet, tagSet in {@link SubscriptionData}, tagSet empty means SubscriptionData.SUB_ALL(*)\n     * @param tags,    message's tags, null means not tag attached to the message.\n     */\n    public static boolean isTagMatched(Set<String> tagsSet, String tags) {\n        if (tagsSet.isEmpty()) {\n            return true;\n        }\n\n        if (tags == null) {\n            return false;\n        }\n\n        return tagsSet.contains(tags);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/GrpcUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.common.utils;\n\nimport io.grpc.Attributes;\nimport io.grpc.Metadata;\nimport io.grpc.ServerCall;\n\npublic class GrpcUtils {\n\n    private GrpcUtils() {\n    }\n\n    public static <T> void putHeaderIfNotExist(Metadata headers, Metadata.Key<T> key, T value) {\n        if (headers == null) {\n            return;\n        }\n        if (!headers.containsKey(key) && value != null) {\n            headers.put(key, value);\n        }\n    }\n\n    public static <R, W, T> T getAttribute(ServerCall<R, W> call, Attributes.Key<T> key) {\n        Attributes attributes = call.getAttributes();\n        if (attributes == null) {\n            return null;\n        }\n        return attributes.get(key);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/ProxyUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.common.utils;\n\npublic class ProxyUtils {\n\n    public static final int MAX_MSG_NUMS_FOR_POP_REQUEST = 32;\n\n    public static final String BROKER_ADDR = \"brokerAddr\";\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/config/ConfigFile.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.config;\n\npublic interface ConfigFile {\n\n    void initData();\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/config/Configuration.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.config;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.google.common.base.Charsets;\nimport com.google.common.io.CharStreams;\nimport java.io.File;\nimport java.io.InputStream;\nimport java.io.InputStreamReader;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.util.concurrent.atomic.AtomicReference;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class Configuration {\n    private final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    private final AtomicReference<ProxyConfig> proxyConfigReference = new AtomicReference<>();\n    private final AtomicReference<AuthConfig> authConfigReference = new AtomicReference<>();\n    public static final String CONFIG_PATH_PROPERTY = \"com.rocketmq.proxy.configPath\";\n\n    public void init() throws Exception {\n        String proxyConfigData = loadJsonConfig();\n\n        ProxyConfig proxyConfig = JSON.parseObject(proxyConfigData, ProxyConfig.class);\n        proxyConfig.initData();\n        setProxyConfig(proxyConfig);\n\n        AuthConfig authConfig = JSON.parseObject(proxyConfigData, AuthConfig.class);\n        setAuthConfig(authConfig);\n        authConfig.setConfigName(proxyConfig.getProxyName());\n        authConfig.setClusterName(proxyConfig.getRocketMQClusterName());\n    }\n\n    private String loadJsonConfig() throws Exception {\n        String configFileName = ProxyConfig.DEFAULT_CONFIG_FILE_NAME;\n        String filePath = System.getProperty(CONFIG_PATH_PROPERTY);\n        if (StringUtils.isBlank(filePath)) {\n            final String testResource = \"rmq-proxy-home/conf/\" + configFileName;\n            try (InputStream inputStream = Configuration.class.getClassLoader().getResourceAsStream(testResource)) {\n                if (null != inputStream) {\n                    return CharStreams.toString(new InputStreamReader(inputStream, Charsets.UTF_8));\n                }\n            }\n            filePath = new File(ConfigurationManager.getProxyHome() + File.separator + \"conf\", configFileName).toString();\n        }\n\n        File file = new File(filePath);\n        log.info(\"The current configuration file path is {}\", filePath);\n        if (!file.exists()) {\n            String msg = String.format(\"the config file %s not exist\", filePath);\n            log.warn(msg);\n            throw new RuntimeException(msg);\n        }\n        long fileLength = file.length();\n        if (fileLength <= 0) {\n            String msg = String.format(\"the config file %s length is zero\", filePath);\n            log.warn(msg);\n            throw new RuntimeException(msg);\n        }\n\n        return new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8);\n    }\n\n    public ProxyConfig getProxyConfig() {\n        return proxyConfigReference.get();\n    }\n\n    public void setProxyConfig(ProxyConfig proxyConfig) {\n        proxyConfigReference.set(proxyConfig);\n    }\n\n    public AuthConfig getAuthConfig() {\n        return authConfigReference.get();\n    }\n\n    public void setAuthConfig(AuthConfig authConfig) {\n        authConfigReference.set(authConfig);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/config/ConfigurationManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.config;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.alibaba.fastjson2.JSONWriter.Feature;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.MixAll;\n\npublic class ConfigurationManager {\n    public static final String RMQ_PROXY_HOME = \"RMQ_PROXY_HOME\";\n    protected static final String DEFAULT_RMQ_PROXY_HOME = MixAll.ROCKETMQ_HOME_DIR;\n    protected static String proxyHome;\n    protected static Configuration configuration;\n\n    public static void initEnv() {\n        proxyHome = System.getenv(RMQ_PROXY_HOME);\n        if (StringUtils.isEmpty(proxyHome)) {\n            proxyHome = System.getProperty(RMQ_PROXY_HOME, DEFAULT_RMQ_PROXY_HOME);\n        }\n\n        if (proxyHome == null) {\n            proxyHome = \"./\";\n        }\n    }\n\n    public static void initConfig() throws Exception {\n        configuration = new Configuration();\n        configuration.init();\n    }\n\n    public static String getProxyHome() {\n        return proxyHome;\n    }\n\n    public static ProxyConfig getProxyConfig() {\n        return configuration.getProxyConfig();\n    }\n\n    public static AuthConfig getAuthConfig() {\n        return configuration.getAuthConfig();\n    }\n\n    public static String formatProxyConfig() {\n        return JSON.toJSONString(ConfigurationManager.getProxyConfig(),\n                Feature.PrettyFormat, Feature.WriteMapNullValue, Feature.WriteNullListAsEmpty);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/config/MetricCollectorMode.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.config;\n\npublic enum MetricCollectorMode {\n    /**\n     * Do not collect the metric from clients.\n     */\n    OFF(\"off\"),\n    /**\n     * Collect the metric from clients to the given address.\n     */\n    ON(\"on\"),\n    /**\n     * Collect the metric by the proxy itself.\n     */\n    PROXY(\"proxy\");\n\n    private final String modeString;\n\n    MetricCollectorMode(String modeString) {\n        this.modeString = modeString;\n    }\n\n    public String getModeString() {\n        return modeString;\n    }\n\n    public static MetricCollectorMode getEnumByString(String modeString) {\n        for (MetricCollectorMode mode : MetricCollectorMode.values()) {\n            if (mode.modeString.equals(modeString.toLowerCase())) {\n                return mode;\n            }\n        }\n        return OFF;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.config;\n\nimport java.net.InetAddress;\nimport java.net.UnknownHostException;\nimport java.time.Duration;\nimport java.util.Comparator;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentSkipListMap;\nimport java.util.concurrent.TimeUnit;\nimport java.util.stream.Collectors;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.metrics.MetricsExporterType;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.ProxyMode;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\n\npublic class ProxyConfig implements ConfigFile {\n    private final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    public final static String DEFAULT_CONFIG_FILE_NAME = \"rmq-proxy.json\";\n    private static final int PROCESSOR_NUMBER = Runtime.getRuntime().availableProcessors();\n    private static final String DEFAULT_CLUSTER_NAME = \"DefaultCluster\";\n\n    private static String localHostName;\n\n    static {\n        try {\n            localHostName = InetAddress.getLocalHost().getHostName();\n        } catch (UnknownHostException e) {\n            log.error(\"Failed to obtain the host name\", e);\n        }\n    }\n\n    private String rocketMQClusterName = DEFAULT_CLUSTER_NAME;\n    private String proxyClusterName = DEFAULT_CLUSTER_NAME;\n    private String proxyName = StringUtils.isEmpty(localHostName) ? \"DEFAULT_PROXY\" : localHostName;\n\n    private String localServeAddr = \"\";\n\n    private String heartbeatSyncerTopicClusterName = \"\";\n    private int heartbeatSyncerThreadPoolNums = 4;\n    private int heartbeatSyncerThreadPoolQueueCapacity = 100;\n\n    private String heartbeatSyncerTopicName = \"DefaultHeartBeatSyncerTopic\";\n\n    /**\n     * configuration for ThreadPoolMonitor\n     */\n    private boolean enablePrintJstack = true;\n    private long printJstackInMillis = Duration.ofSeconds(60).toMillis();\n    private long printThreadPoolStatusInMillis = Duration.ofSeconds(3).toMillis();\n\n    private String namesrvAddr = System.getProperty(MixAll.NAMESRV_ADDR_PROPERTY, System.getenv(MixAll.NAMESRV_ADDR_ENV));\n    private String namesrvDomain = \"\";\n    private String namesrvDomainSubgroup = \"\";\n    /**\n     * TLS\n     */\n    private boolean tlsTestModeEnable = true;\n    private String tlsKeyPath = ConfigurationManager.getProxyHome() + \"/conf/tls/rocketmq.key\";\n    private String tlsKeyPassword = \"\";\n    private String tlsCertPath = ConfigurationManager.getProxyHome() + \"/conf/tls/rocketmq.crt\";\n    private int tlsCertWatchIntervalMs = 60 * 60 * 1000; // 1 hour\n    /**\n     * gRPC\n     */\n    private String proxyMode = ProxyMode.CLUSTER.name();\n    private Integer grpcServerPort = 8081;\n    private long grpcShutdownTimeSeconds = 30;\n    private int grpcBossLoopNum = 1;\n    private int grpcWorkerLoopNum = PROCESSOR_NUMBER * 2;\n    private boolean enableGrpcEpoll = false;\n    private int grpcThreadPoolNums = 16 + PROCESSOR_NUMBER * 2;\n    private int grpcThreadPoolQueueCapacity = 100000;\n    private String brokerConfigPath = ConfigurationManager.getProxyHome() + \"/conf/broker.conf\";\n    /**\n     * gRPC max message size\n     * 130M = 4M * 32 messages + 2M attributes\n     */\n    private int grpcMaxInboundMessageSize = 130 * 1024 * 1024;\n    /**\n     * max message body size, 0 or negative number means no limit for proxy\n     */\n    private int maxMessageSize = 4 * 1024 * 1024;\n    /**\n     * if true, proxy will check message body size and reject msg if it's body is empty\n     */\n    private boolean enableMessageBodyEmptyCheck = true;\n    /**\n     * max user property size, 0 or negative number means no limit for proxy\n     */\n    private int maxUserPropertySize = 16 * 1024;\n    private int userPropertyMaxNum = 128;\n\n    /**\n     * max message group size, 0 or negative number means no limit for proxy\n     */\n    private int maxMessageGroupSize = 64;\n    /**\n     * max lite topic size\n     */\n    private int maxLiteTopicSize = 64;\n    private int maxLiteRenewNumPerChannel = 100;\n    // syncLiteSubscription request rate limit per proxy\n    private int maxSyncLiteSubscriptionRate = 5000;\n\n    /**\n     * When a message pops, the message is invisible by default\n     */\n    private long defaultInvisibleTimeMills = Duration.ofSeconds(60).toMillis();\n    private long minInvisibleTimeMillsForRecv = Duration.ofSeconds(10).toMillis();\n    private long maxInvisibleTimeMills = Duration.ofHours(12).toMillis();\n    private long maxDelayTimeMills = Duration.ofDays(1).toMillis();\n    private long maxTransactionRecoverySecond = Duration.ofHours(1).getSeconds();\n    private boolean enableTopicMessageTypeCheck = true;\n\n    private int grpcClientProducerMaxAttempts = 3;\n    private long grpcClientProducerBackoffInitialMillis = 10;\n    private long grpcClientProducerBackoffMaxMillis = 1000;\n    private int grpcClientProducerBackoffMultiplier = 2;\n    private long grpcClientConsumerMinLongPollingTimeoutMillis = Duration.ofSeconds(5).toMillis();\n    private long grpcClientConsumerMaxLongPollingTimeoutMillis = Duration.ofSeconds(20).toMillis();\n    private int grpcClientConsumerLongPollingBatchSize = 32;\n    private long grpcClientIdleTimeMills = Duration.ofSeconds(120).toMillis();\n\n    private int channelExpiredInSeconds = 60;\n    private int contextExpiredInSeconds = 30;\n\n    private int rocketmqMQClientNum = 6;\n\n    private long grpcProxyRelayRequestTimeoutInSeconds = 5;\n    private int grpcProducerThreadPoolNums = PROCESSOR_NUMBER;\n    private int grpcProducerThreadQueueCapacity = 10000;\n    private int grpcConsumerThreadPoolNums = PROCESSOR_NUMBER;\n    private int grpcConsumerThreadQueueCapacity = 10000;\n    private int grpcRouteThreadPoolNums = PROCESSOR_NUMBER;\n    private int grpcRouteThreadQueueCapacity = 10000;\n    private int grpcClientManagerThreadPoolNums = PROCESSOR_NUMBER;\n    private int grpcClientManagerThreadQueueCapacity = 10000;\n    private int grpcTransactionThreadPoolNums = PROCESSOR_NUMBER;\n    private int grpcTransactionThreadQueueCapacity = 10000;\n\n    private int producerProcessorThreadPoolNums = PROCESSOR_NUMBER;\n    private int producerProcessorThreadPoolQueueCapacity = 10000;\n    private int consumerProcessorThreadPoolNums = PROCESSOR_NUMBER;\n    private int consumerProcessorThreadPoolQueueCapacity = 10000;\n\n    private boolean useEndpointPortFromRequest = false;\n\n    private int topicRouteServiceCacheExpiredSeconds = 300;\n    private int topicRouteServiceCacheRefreshSeconds = 20;\n    private int topicRouteServiceCacheMaxNum = 20000;\n    private int topicRouteServiceThreadPoolNums = PROCESSOR_NUMBER;\n    private int topicRouteServiceThreadPoolQueueCapacity = 5000;\n    private int topicConfigCacheExpiredSeconds = 300;\n    private int topicConfigCacheRefreshSeconds = 20;\n    private int topicConfigCacheMaxNum = 20000;\n    private int subscriptionGroupConfigCacheExpiredSeconds = 300;\n    private int subscriptionGroupConfigCacheRefreshSeconds = 20;\n    private int subscriptionGroupConfigCacheMaxNum = 20000;\n    private int userCacheExpiredSeconds = 300;\n    private int userCacheRefreshSeconds = 20;\n    private int userCacheMaxNum = 20000;\n    private int aclCacheExpiredSeconds = 300;\n    private int aclCacheRefreshSeconds = 20;\n    private int aclCacheMaxNum = 20000;\n    private int metadataThreadPoolNums = 3;\n    private int metadataThreadPoolQueueCapacity = 100000;\n\n    private int transactionHeartbeatThreadPoolNums = 20;\n    private int transactionHeartbeatThreadPoolQueueCapacity = 200;\n    private int transactionHeartbeatPeriodSecond = 20;\n    private int transactionHeartbeatBatchNum = 100;\n    private long transactionDataExpireScanPeriodMillis = Duration.ofSeconds(10).toMillis();\n    private long transactionDataMaxWaitClearMillis = Duration.ofSeconds(30).toMillis();\n    private long transactionDataExpireMillis = Duration.ofSeconds(30).toMillis();\n    private int transactionDataMaxNum = 15;\n\n    private long longPollingReserveTimeInMillis = 100;\n\n    private long invisibleTimeMillisWhenClear = 1000L;\n    private boolean enableProxyAutoRenew = true;\n    private int maxRenewRetryTimes = 3;\n    private int renewThreadPoolNums = 2;\n    private int renewMaxThreadPoolNums = 4;\n    private int renewThreadPoolQueueCapacity = 300;\n    private long lockTimeoutMsInHandleGroup = TimeUnit.SECONDS.toMillis(3);\n    private long renewAheadTimeMillis = TimeUnit.SECONDS.toMillis(10);\n    private long renewMaxTimeMillis = TimeUnit.HOURS.toMillis(3);\n    private long renewSchedulePeriodMillis = TimeUnit.SECONDS.toMillis(5);\n    private int returnHandleGroupThreadPoolNums = 2;\n\n    private boolean enableAclRpcHookForClusterMode = false;\n\n    private boolean useDelayLevel = false;\n    private String messageDelayLevel = \"1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h\";\n    private transient ConcurrentSkipListMap<Integer /* level */, Long/* delay timeMillis */> delayLevelTable = new ConcurrentSkipListMap<>();\n\n    private String metricCollectorMode = MetricCollectorMode.OFF.getModeString();\n    // Example address: 127.0.0.1:1234\n    private String metricCollectorAddress = \"\";\n\n    private String regionId = \"\";\n\n    private boolean traceOn = false;\n\n    private MetricsExporterType metricsExporterType = MetricsExporterType.DISABLE;\n\n    private String metricsGrpcExporterTarget = \"\";\n    private String metricsGrpcExporterHeader = \"\";\n    private long metricGrpcExporterTimeOutInMills = 3 * 1000;\n    private long metricGrpcExporterIntervalInMills = 60 * 1000;\n    private long metricLoggingExporterIntervalInMills = 10 * 1000;\n\n    private int metricsPromExporterPort = 5557;\n    private String metricsPromExporterHost = \"\";\n\n    // Label pairs in CSV. Each label follows pattern of Key:Value. eg: instance_id:xxx,uid:xxx\n    private String metricsLabel = \"\";\n\n    private boolean metricsInDelta = false;\n\n    private long channelExpiredTimeout = 1000 * 120;\n\n    // remoting\n    private boolean enableRemotingLocalProxyGrpc = true;\n    private int localProxyConnectTimeoutMs = 3000;\n    private String remotingAccessAddr = \"\";\n    private int remotingListenPort = 8080;\n\n    // related to proxy's send strategy in cluster mode.\n    private boolean sendLatencyEnable = false;\n    private boolean startDetectorEnable = false;\n    private int detectTimeout = 200;\n    private int detectInterval = 2 * 1000;\n\n    private int remotingHeartbeatThreadPoolNums = 2 * PROCESSOR_NUMBER;\n    private int remotingTopicRouteThreadPoolNums = 2 * PROCESSOR_NUMBER;\n    private int remotingSendMessageThreadPoolNums = 4 * PROCESSOR_NUMBER;\n    private int remotingPullMessageThreadPoolNums = 4 * PROCESSOR_NUMBER;\n    private int remotingUpdateOffsetThreadPoolNums = 4 * PROCESSOR_NUMBER;\n    private int remotingDefaultThreadPoolNums = 4 * PROCESSOR_NUMBER;\n\n    private int remotingHeartbeatThreadPoolQueueCapacity = 50000;\n    private int remotingTopicRouteThreadPoolQueueCapacity = 50000;\n    private int remotingSendThreadPoolQueueCapacity = 10000;\n    private int remotingPullThreadPoolQueueCapacity = 50000;\n    private int remotingUpdateOffsetThreadPoolQueueCapacity = 10000;\n    private int remotingDefaultThreadPoolQueueCapacity = 50000;\n\n    private long remotingWaitTimeMillsInSendQueue = 3 * 1000;\n    private long remotingWaitTimeMillsInPullQueue = 5 * 1000;\n    private long remotingWaitTimeMillsInHeartbeatQueue = 31 * 1000;\n    private long remotingWaitTimeMillsInUpdateOffsetQueue = 3 * 1000;\n    private long remotingWaitTimeMillsInTopicRouteQueue = 3 * 1000;\n    private long remotingWaitTimeMillsInDefaultQueue = 3 * 1000;\n\n    private boolean enableBatchAck = false;\n\n    @Override\n    public void initData() {\n        parseDelayLevel();\n        if (StringUtils.isEmpty(localServeAddr)) {\n            this.localServeAddr = NetworkUtil.getLocalAddress();\n        }\n        if (StringUtils.isBlank(localServeAddr)) {\n            throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, \"get local serve ip failed\");\n        }\n        if (StringUtils.isBlank(remotingAccessAddr)) {\n            this.remotingAccessAddr = this.localServeAddr;\n        }\n        if (StringUtils.isBlank(heartbeatSyncerTopicClusterName)) {\n            this.heartbeatSyncerTopicClusterName = this.rocketMQClusterName;\n        }\n    }\n\n    public int computeDelayLevel(long timeMillis) {\n        long intervalMillis = timeMillis - System.currentTimeMillis();\n        List<Map.Entry<Integer, Long>> sortedLevels = delayLevelTable.entrySet().stream().sorted(Comparator.comparingLong(Map.Entry::getValue)).collect(Collectors.toList());\n        for (Map.Entry<Integer, Long> entry : sortedLevels) {\n            if (entry.getValue() > intervalMillis) {\n                return entry.getKey();\n            }\n        }\n        return sortedLevels.get(sortedLevels.size() - 1).getKey();\n    }\n\n    public void parseDelayLevel() {\n        this.delayLevelTable = new ConcurrentSkipListMap<>();\n        Map<String, Long> timeUnitTable = new HashMap<>();\n        timeUnitTable.put(\"s\", 1000L);\n        timeUnitTable.put(\"m\", 1000L * 60);\n        timeUnitTable.put(\"h\", 1000L * 60 * 60);\n        timeUnitTable.put(\"d\", 1000L * 60 * 60 * 24);\n\n        String levelString = this.getMessageDelayLevel();\n        try {\n            String[] levelArray = levelString.split(\" \");\n            for (int i = 0; i < levelArray.length; i++) {\n                String value = levelArray[i];\n                String ch = value.substring(value.length() - 1);\n                Long tu = timeUnitTable.get(ch);\n\n                int level = i + 1;\n                long num = Long.parseLong(value.substring(0, value.length() - 1));\n                long delayTimeMillis = tu * num;\n                this.delayLevelTable.put(level, delayTimeMillis);\n            }\n        } catch (Exception e) {\n            log.error(\"parse delay level failed. messageDelayLevel:{}\", messageDelayLevel, e);\n        }\n    }\n\n    public int getTlsCertWatchIntervalMs() {\n        return tlsCertWatchIntervalMs;\n    }\n\n    public void setTlsCertWatchIntervalMs(int tlsCertWatchIntervalMs) {\n        this.tlsCertWatchIntervalMs = tlsCertWatchIntervalMs;\n    }\n\n    public String getRocketMQClusterName() {\n        return rocketMQClusterName;\n    }\n\n    public void setRocketMQClusterName(String rocketMQClusterName) {\n        this.rocketMQClusterName = rocketMQClusterName;\n    }\n\n    public String getProxyClusterName() {\n        return proxyClusterName;\n    }\n\n    public void setProxyClusterName(String proxyClusterName) {\n        this.proxyClusterName = proxyClusterName;\n    }\n\n    public String getProxyName() {\n        return proxyName;\n    }\n\n    public void setProxyName(String proxyName) {\n        this.proxyName = proxyName;\n    }\n\n    public String getLocalServeAddr() {\n        return localServeAddr;\n    }\n\n    public void setLocalServeAddr(String localServeAddr) {\n        this.localServeAddr = localServeAddr;\n    }\n\n    public String getHeartbeatSyncerTopicClusterName() {\n        return heartbeatSyncerTopicClusterName;\n    }\n\n    public void setHeartbeatSyncerTopicClusterName(String heartbeatSyncerTopicClusterName) {\n        this.heartbeatSyncerTopicClusterName = heartbeatSyncerTopicClusterName;\n    }\n\n    public int getHeartbeatSyncerThreadPoolNums() {\n        return heartbeatSyncerThreadPoolNums;\n    }\n\n    public void setHeartbeatSyncerThreadPoolNums(int heartbeatSyncerThreadPoolNums) {\n        this.heartbeatSyncerThreadPoolNums = heartbeatSyncerThreadPoolNums;\n    }\n\n    public int getHeartbeatSyncerThreadPoolQueueCapacity() {\n        return heartbeatSyncerThreadPoolQueueCapacity;\n    }\n\n    public void setHeartbeatSyncerThreadPoolQueueCapacity(int heartbeatSyncerThreadPoolQueueCapacity) {\n        this.heartbeatSyncerThreadPoolQueueCapacity = heartbeatSyncerThreadPoolQueueCapacity;\n    }\n\n    public String getHeartbeatSyncerTopicName() {\n        return heartbeatSyncerTopicName;\n    }\n\n    public void setHeartbeatSyncerTopicName(String heartbeatSyncerTopicName) {\n        this.heartbeatSyncerTopicName = heartbeatSyncerTopicName;\n    }\n\n    public boolean isEnablePrintJstack() {\n        return enablePrintJstack;\n    }\n\n    public void setEnablePrintJstack(boolean enablePrintJstack) {\n        this.enablePrintJstack = enablePrintJstack;\n    }\n\n    public long getPrintJstackInMillis() {\n        return printJstackInMillis;\n    }\n\n    public void setPrintJstackInMillis(long printJstackInMillis) {\n        this.printJstackInMillis = printJstackInMillis;\n    }\n\n    public long getPrintThreadPoolStatusInMillis() {\n        return printThreadPoolStatusInMillis;\n    }\n\n    public void setPrintThreadPoolStatusInMillis(long printThreadPoolStatusInMillis) {\n        this.printThreadPoolStatusInMillis = printThreadPoolStatusInMillis;\n    }\n\n    public String getNamesrvAddr() {\n        return namesrvAddr;\n    }\n\n    public void setNamesrvAddr(String namesrvAddr) {\n        this.namesrvAddr = namesrvAddr;\n    }\n\n    public String getNamesrvDomain() {\n        return namesrvDomain;\n    }\n\n    public void setNamesrvDomain(String namesrvDomain) {\n        this.namesrvDomain = namesrvDomain;\n    }\n\n    public String getNamesrvDomainSubgroup() {\n        return namesrvDomainSubgroup;\n    }\n\n    public void setNamesrvDomainSubgroup(String namesrvDomainSubgroup) {\n        this.namesrvDomainSubgroup = namesrvDomainSubgroup;\n    }\n\n    public String getProxyMode() {\n        return proxyMode;\n    }\n\n    public void setProxyMode(String proxyMode) {\n        this.proxyMode = proxyMode;\n    }\n\n    public Integer getGrpcServerPort() {\n        return grpcServerPort;\n    }\n\n    public void setGrpcServerPort(Integer grpcServerPort) {\n        this.grpcServerPort = grpcServerPort;\n    }\n\n    public long getGrpcShutdownTimeSeconds() {\n        return grpcShutdownTimeSeconds;\n    }\n\n    public void setGrpcShutdownTimeSeconds(long grpcShutdownTimeSeconds) {\n        this.grpcShutdownTimeSeconds = grpcShutdownTimeSeconds;\n    }\n\n    public boolean isUseEndpointPortFromRequest() {\n        return useEndpointPortFromRequest;\n    }\n\n    public void setUseEndpointPortFromRequest(boolean useEndpointPortFromRequest) {\n        this.useEndpointPortFromRequest = useEndpointPortFromRequest;\n    }\n\n    public boolean isTlsTestModeEnable() {\n        return tlsTestModeEnable;\n    }\n\n    public void setTlsTestModeEnable(boolean tlsTestModeEnable) {\n        this.tlsTestModeEnable = tlsTestModeEnable;\n    }\n\n    public String getTlsKeyPath() {\n        return tlsKeyPath;\n    }\n\n    public void setTlsKeyPath(String tlsKeyPath) {\n        this.tlsKeyPath = tlsKeyPath;\n    }\n\n    public String getTlsKeyPassword() {\n        return tlsKeyPassword;\n    }\n\n    public void setTlsKeyPassword(String tlsKeyPassword) {\n        this.tlsKeyPassword = tlsKeyPassword;\n    }\n\n    public String getTlsCertPath() {\n        return tlsCertPath;\n    }\n\n    public void setTlsCertPath(String tlsCertPath) {\n        this.tlsCertPath = tlsCertPath;\n    }\n\n    public int getGrpcBossLoopNum() {\n        return grpcBossLoopNum;\n    }\n\n    public void setGrpcBossLoopNum(int grpcBossLoopNum) {\n        this.grpcBossLoopNum = grpcBossLoopNum;\n    }\n\n    public int getGrpcWorkerLoopNum() {\n        return grpcWorkerLoopNum;\n    }\n\n    public void setGrpcWorkerLoopNum(int grpcWorkerLoopNum) {\n        this.grpcWorkerLoopNum = grpcWorkerLoopNum;\n    }\n\n    public boolean isEnableGrpcEpoll() {\n        return enableGrpcEpoll;\n    }\n\n    public void setEnableGrpcEpoll(boolean enableGrpcEpoll) {\n        this.enableGrpcEpoll = enableGrpcEpoll;\n    }\n\n    public int getGrpcThreadPoolNums() {\n        return grpcThreadPoolNums;\n    }\n\n    public void setGrpcThreadPoolNums(int grpcThreadPoolNums) {\n        this.grpcThreadPoolNums = grpcThreadPoolNums;\n    }\n\n    public int getGrpcThreadPoolQueueCapacity() {\n        return grpcThreadPoolQueueCapacity;\n    }\n\n    public void setGrpcThreadPoolQueueCapacity(int grpcThreadPoolQueueCapacity) {\n        this.grpcThreadPoolQueueCapacity = grpcThreadPoolQueueCapacity;\n    }\n\n    public String getBrokerConfigPath() {\n        return brokerConfigPath;\n    }\n\n    public void setBrokerConfigPath(String brokerConfigPath) {\n        this.brokerConfigPath = brokerConfigPath;\n    }\n\n    public int getGrpcMaxInboundMessageSize() {\n        return grpcMaxInboundMessageSize;\n    }\n\n    public void setGrpcMaxInboundMessageSize(int grpcMaxInboundMessageSize) {\n        this.grpcMaxInboundMessageSize = grpcMaxInboundMessageSize;\n    }\n\n    public int getMaxMessageSize() {\n        return maxMessageSize;\n    }\n\n    public void setMaxMessageSize(int maxMessageSize) {\n        this.maxMessageSize = maxMessageSize;\n    }\n\n    public int getMaxUserPropertySize() {\n        return maxUserPropertySize;\n    }\n\n    public void setMaxUserPropertySize(int maxUserPropertySize) {\n        this.maxUserPropertySize = maxUserPropertySize;\n    }\n\n    public int getUserPropertyMaxNum() {\n        return userPropertyMaxNum;\n    }\n\n    public void setUserPropertyMaxNum(int userPropertyMaxNum) {\n        this.userPropertyMaxNum = userPropertyMaxNum;\n    }\n\n    public int getMaxMessageGroupSize() {\n        return maxMessageGroupSize;\n    }\n\n    public void setMaxMessageGroupSize(int maxMessageGroupSize) {\n        this.maxMessageGroupSize = maxMessageGroupSize;\n    }\n\n    public long getMinInvisibleTimeMillsForRecv() {\n        return minInvisibleTimeMillsForRecv;\n    }\n\n    public void setMinInvisibleTimeMillsForRecv(long minInvisibleTimeMillsForRecv) {\n        this.minInvisibleTimeMillsForRecv = minInvisibleTimeMillsForRecv;\n    }\n\n    public long getDefaultInvisibleTimeMills() {\n        return defaultInvisibleTimeMills;\n    }\n\n    public void setDefaultInvisibleTimeMills(long defaultInvisibleTimeMills) {\n        this.defaultInvisibleTimeMills = defaultInvisibleTimeMills;\n    }\n\n    public long getMaxInvisibleTimeMills() {\n        return maxInvisibleTimeMills;\n    }\n\n    public void setMaxInvisibleTimeMills(long maxInvisibleTimeMills) {\n        this.maxInvisibleTimeMills = maxInvisibleTimeMills;\n    }\n\n    public long getMaxDelayTimeMills() {\n        return maxDelayTimeMills;\n    }\n\n    public void setMaxDelayTimeMills(long maxDelayTimeMills) {\n        this.maxDelayTimeMills = maxDelayTimeMills;\n    }\n\n    public long getMaxTransactionRecoverySecond() {\n        return maxTransactionRecoverySecond;\n    }\n\n    public void setMaxTransactionRecoverySecond(long maxTransactionRecoverySecond) {\n        this.maxTransactionRecoverySecond = maxTransactionRecoverySecond;\n    }\n\n    public int getGrpcClientProducerMaxAttempts() {\n        return grpcClientProducerMaxAttempts;\n    }\n\n    public void setGrpcClientProducerMaxAttempts(int grpcClientProducerMaxAttempts) {\n        this.grpcClientProducerMaxAttempts = grpcClientProducerMaxAttempts;\n    }\n\n    public long getGrpcClientProducerBackoffInitialMillis() {\n        return grpcClientProducerBackoffInitialMillis;\n    }\n\n    public void setGrpcClientProducerBackoffInitialMillis(long grpcClientProducerBackoffInitialMillis) {\n        this.grpcClientProducerBackoffInitialMillis = grpcClientProducerBackoffInitialMillis;\n    }\n\n    public long getGrpcClientProducerBackoffMaxMillis() {\n        return grpcClientProducerBackoffMaxMillis;\n    }\n\n    public void setGrpcClientProducerBackoffMaxMillis(long grpcClientProducerBackoffMaxMillis) {\n        this.grpcClientProducerBackoffMaxMillis = grpcClientProducerBackoffMaxMillis;\n    }\n\n    public int getGrpcClientProducerBackoffMultiplier() {\n        return grpcClientProducerBackoffMultiplier;\n    }\n\n    public void setGrpcClientProducerBackoffMultiplier(int grpcClientProducerBackoffMultiplier) {\n        this.grpcClientProducerBackoffMultiplier = grpcClientProducerBackoffMultiplier;\n    }\n\n    public long getGrpcClientConsumerMinLongPollingTimeoutMillis() {\n        return grpcClientConsumerMinLongPollingTimeoutMillis;\n    }\n\n    public void setGrpcClientConsumerMinLongPollingTimeoutMillis(long grpcClientConsumerMinLongPollingTimeoutMillis) {\n        this.grpcClientConsumerMinLongPollingTimeoutMillis = grpcClientConsumerMinLongPollingTimeoutMillis;\n    }\n\n    public long getGrpcClientConsumerMaxLongPollingTimeoutMillis() {\n        return grpcClientConsumerMaxLongPollingTimeoutMillis;\n    }\n\n    public void setGrpcClientConsumerMaxLongPollingTimeoutMillis(long grpcClientConsumerMaxLongPollingTimeoutMillis) {\n        this.grpcClientConsumerMaxLongPollingTimeoutMillis = grpcClientConsumerMaxLongPollingTimeoutMillis;\n    }\n\n    public int getGrpcClientConsumerLongPollingBatchSize() {\n        return grpcClientConsumerLongPollingBatchSize;\n    }\n\n    public void setGrpcClientConsumerLongPollingBatchSize(int grpcClientConsumerLongPollingBatchSize) {\n        this.grpcClientConsumerLongPollingBatchSize = grpcClientConsumerLongPollingBatchSize;\n    }\n\n    public int getChannelExpiredInSeconds() {\n        return channelExpiredInSeconds;\n    }\n\n    public void setChannelExpiredInSeconds(int channelExpiredInSeconds) {\n        this.channelExpiredInSeconds = channelExpiredInSeconds;\n    }\n\n    public int getContextExpiredInSeconds() {\n        return contextExpiredInSeconds;\n    }\n\n    public void setContextExpiredInSeconds(int contextExpiredInSeconds) {\n        this.contextExpiredInSeconds = contextExpiredInSeconds;\n    }\n\n    public int getRocketmqMQClientNum() {\n        return rocketmqMQClientNum;\n    }\n\n    public void setRocketmqMQClientNum(int rocketmqMQClientNum) {\n        this.rocketmqMQClientNum = rocketmqMQClientNum;\n    }\n\n    public long getGrpcProxyRelayRequestTimeoutInSeconds() {\n        return grpcProxyRelayRequestTimeoutInSeconds;\n    }\n\n    public void setGrpcProxyRelayRequestTimeoutInSeconds(long grpcProxyRelayRequestTimeoutInSeconds) {\n        this.grpcProxyRelayRequestTimeoutInSeconds = grpcProxyRelayRequestTimeoutInSeconds;\n    }\n\n    public int getGrpcProducerThreadPoolNums() {\n        return grpcProducerThreadPoolNums;\n    }\n\n    public void setGrpcProducerThreadPoolNums(int grpcProducerThreadPoolNums) {\n        this.grpcProducerThreadPoolNums = grpcProducerThreadPoolNums;\n    }\n\n    public int getGrpcProducerThreadQueueCapacity() {\n        return grpcProducerThreadQueueCapacity;\n    }\n\n    public void setGrpcProducerThreadQueueCapacity(int grpcProducerThreadQueueCapacity) {\n        this.grpcProducerThreadQueueCapacity = grpcProducerThreadQueueCapacity;\n    }\n\n    public int getGrpcConsumerThreadPoolNums() {\n        return grpcConsumerThreadPoolNums;\n    }\n\n    public void setGrpcConsumerThreadPoolNums(int grpcConsumerThreadPoolNums) {\n        this.grpcConsumerThreadPoolNums = grpcConsumerThreadPoolNums;\n    }\n\n    public int getGrpcConsumerThreadQueueCapacity() {\n        return grpcConsumerThreadQueueCapacity;\n    }\n\n    public void setGrpcConsumerThreadQueueCapacity(int grpcConsumerThreadQueueCapacity) {\n        this.grpcConsumerThreadQueueCapacity = grpcConsumerThreadQueueCapacity;\n    }\n\n    public int getGrpcRouteThreadPoolNums() {\n        return grpcRouteThreadPoolNums;\n    }\n\n    public void setGrpcRouteThreadPoolNums(int grpcRouteThreadPoolNums) {\n        this.grpcRouteThreadPoolNums = grpcRouteThreadPoolNums;\n    }\n\n    public int getGrpcRouteThreadQueueCapacity() {\n        return grpcRouteThreadQueueCapacity;\n    }\n\n    public void setGrpcRouteThreadQueueCapacity(int grpcRouteThreadQueueCapacity) {\n        this.grpcRouteThreadQueueCapacity = grpcRouteThreadQueueCapacity;\n    }\n\n    public int getGrpcClientManagerThreadPoolNums() {\n        return grpcClientManagerThreadPoolNums;\n    }\n\n    public void setGrpcClientManagerThreadPoolNums(int grpcClientManagerThreadPoolNums) {\n        this.grpcClientManagerThreadPoolNums = grpcClientManagerThreadPoolNums;\n    }\n\n    public int getGrpcClientManagerThreadQueueCapacity() {\n        return grpcClientManagerThreadQueueCapacity;\n    }\n\n    public void setGrpcClientManagerThreadQueueCapacity(int grpcClientManagerThreadQueueCapacity) {\n        this.grpcClientManagerThreadQueueCapacity = grpcClientManagerThreadQueueCapacity;\n    }\n\n    public int getGrpcTransactionThreadPoolNums() {\n        return grpcTransactionThreadPoolNums;\n    }\n\n    public void setGrpcTransactionThreadPoolNums(int grpcTransactionThreadPoolNums) {\n        this.grpcTransactionThreadPoolNums = grpcTransactionThreadPoolNums;\n    }\n\n    public int getGrpcTransactionThreadQueueCapacity() {\n        return grpcTransactionThreadQueueCapacity;\n    }\n\n    public void setGrpcTransactionThreadQueueCapacity(int grpcTransactionThreadQueueCapacity) {\n        this.grpcTransactionThreadQueueCapacity = grpcTransactionThreadQueueCapacity;\n    }\n\n    public int getProducerProcessorThreadPoolNums() {\n        return producerProcessorThreadPoolNums;\n    }\n\n    public void setProducerProcessorThreadPoolNums(int producerProcessorThreadPoolNums) {\n        this.producerProcessorThreadPoolNums = producerProcessorThreadPoolNums;\n    }\n\n    public int getProducerProcessorThreadPoolQueueCapacity() {\n        return producerProcessorThreadPoolQueueCapacity;\n    }\n\n    public void setProducerProcessorThreadPoolQueueCapacity(int producerProcessorThreadPoolQueueCapacity) {\n        this.producerProcessorThreadPoolQueueCapacity = producerProcessorThreadPoolQueueCapacity;\n    }\n\n    public int getConsumerProcessorThreadPoolNums() {\n        return consumerProcessorThreadPoolNums;\n    }\n\n    public void setConsumerProcessorThreadPoolNums(int consumerProcessorThreadPoolNums) {\n        this.consumerProcessorThreadPoolNums = consumerProcessorThreadPoolNums;\n    }\n\n    public int getConsumerProcessorThreadPoolQueueCapacity() {\n        return consumerProcessorThreadPoolQueueCapacity;\n    }\n\n    public void setConsumerProcessorThreadPoolQueueCapacity(int consumerProcessorThreadPoolQueueCapacity) {\n        this.consumerProcessorThreadPoolQueueCapacity = consumerProcessorThreadPoolQueueCapacity;\n    }\n\n    public int getTopicRouteServiceCacheExpiredSeconds() {\n        return topicRouteServiceCacheExpiredSeconds;\n    }\n\n    public void setTopicRouteServiceCacheExpiredSeconds(int topicRouteServiceCacheExpiredSeconds) {\n        this.topicRouteServiceCacheExpiredSeconds = topicRouteServiceCacheExpiredSeconds;\n    }\n\n    public int getTopicRouteServiceCacheRefreshSeconds() {\n        return topicRouteServiceCacheRefreshSeconds;\n    }\n\n    public void setTopicRouteServiceCacheRefreshSeconds(int topicRouteServiceCacheRefreshSeconds) {\n        this.topicRouteServiceCacheRefreshSeconds = topicRouteServiceCacheRefreshSeconds;\n    }\n\n    public int getTopicRouteServiceCacheMaxNum() {\n        return topicRouteServiceCacheMaxNum;\n    }\n\n    public void setTopicRouteServiceCacheMaxNum(int topicRouteServiceCacheMaxNum) {\n        this.topicRouteServiceCacheMaxNum = topicRouteServiceCacheMaxNum;\n    }\n\n    public int getTopicRouteServiceThreadPoolNums() {\n        return topicRouteServiceThreadPoolNums;\n    }\n\n    public void setTopicRouteServiceThreadPoolNums(int topicRouteServiceThreadPoolNums) {\n        this.topicRouteServiceThreadPoolNums = topicRouteServiceThreadPoolNums;\n    }\n\n    public int getTopicRouteServiceThreadPoolQueueCapacity() {\n        return topicRouteServiceThreadPoolQueueCapacity;\n    }\n\n    public void setTopicRouteServiceThreadPoolQueueCapacity(int topicRouteServiceThreadPoolQueueCapacity) {\n        this.topicRouteServiceThreadPoolQueueCapacity = topicRouteServiceThreadPoolQueueCapacity;\n    }\n\n    public int getTopicConfigCacheRefreshSeconds() {\n        return topicConfigCacheRefreshSeconds;\n    }\n\n    public void setTopicConfigCacheRefreshSeconds(int topicConfigCacheRefreshSeconds) {\n        this.topicConfigCacheRefreshSeconds = topicConfigCacheRefreshSeconds;\n    }\n\n    public int getTopicConfigCacheExpiredSeconds() {\n        return topicConfigCacheExpiredSeconds;\n    }\n\n    public void setTopicConfigCacheExpiredSeconds(int topicConfigCacheExpiredSeconds) {\n        this.topicConfigCacheExpiredSeconds = topicConfigCacheExpiredSeconds;\n    }\n\n    public int getTopicConfigCacheMaxNum() {\n        return topicConfigCacheMaxNum;\n    }\n\n    public void setTopicConfigCacheMaxNum(int topicConfigCacheMaxNum) {\n        this.topicConfigCacheMaxNum = topicConfigCacheMaxNum;\n    }\n\n    public int getSubscriptionGroupConfigCacheRefreshSeconds() {\n        return subscriptionGroupConfigCacheRefreshSeconds;\n    }\n\n    public void setSubscriptionGroupConfigCacheRefreshSeconds(int subscriptionGroupConfigCacheRefreshSeconds) {\n        this.subscriptionGroupConfigCacheRefreshSeconds = subscriptionGroupConfigCacheRefreshSeconds;\n    }\n\n    public int getSubscriptionGroupConfigCacheExpiredSeconds() {\n        return subscriptionGroupConfigCacheExpiredSeconds;\n    }\n\n    public void setSubscriptionGroupConfigCacheExpiredSeconds(int subscriptionGroupConfigCacheExpiredSeconds) {\n        this.subscriptionGroupConfigCacheExpiredSeconds = subscriptionGroupConfigCacheExpiredSeconds;\n    }\n\n    public int getSubscriptionGroupConfigCacheMaxNum() {\n        return subscriptionGroupConfigCacheMaxNum;\n    }\n\n    public void setSubscriptionGroupConfigCacheMaxNum(int subscriptionGroupConfigCacheMaxNum) {\n        this.subscriptionGroupConfigCacheMaxNum = subscriptionGroupConfigCacheMaxNum;\n    }\n\n    public int getUserCacheExpiredSeconds() {\n        return userCacheExpiredSeconds;\n    }\n\n    public void setUserCacheExpiredSeconds(int userCacheExpiredSeconds) {\n        this.userCacheExpiredSeconds = userCacheExpiredSeconds;\n    }\n\n    public int getUserCacheRefreshSeconds() {\n        return userCacheRefreshSeconds;\n    }\n\n    public void setUserCacheRefreshSeconds(int userCacheRefreshSeconds) {\n        this.userCacheRefreshSeconds = userCacheRefreshSeconds;\n    }\n\n    public int getUserCacheMaxNum() {\n        return userCacheMaxNum;\n    }\n\n    public void setUserCacheMaxNum(int userCacheMaxNum) {\n        this.userCacheMaxNum = userCacheMaxNum;\n    }\n\n    public int getAclCacheExpiredSeconds() {\n        return aclCacheExpiredSeconds;\n    }\n\n    public void setAclCacheExpiredSeconds(int aclCacheExpiredSeconds) {\n        this.aclCacheExpiredSeconds = aclCacheExpiredSeconds;\n    }\n\n    public int getAclCacheRefreshSeconds() {\n        return aclCacheRefreshSeconds;\n    }\n\n    public void setAclCacheRefreshSeconds(int aclCacheRefreshSeconds) {\n        this.aclCacheRefreshSeconds = aclCacheRefreshSeconds;\n    }\n\n    public int getAclCacheMaxNum() {\n        return aclCacheMaxNum;\n    }\n\n    public void setAclCacheMaxNum(int aclCacheMaxNum) {\n        this.aclCacheMaxNum = aclCacheMaxNum;\n    }\n\n    public int getMetadataThreadPoolNums() {\n        return metadataThreadPoolNums;\n    }\n\n    public void setMetadataThreadPoolNums(int metadataThreadPoolNums) {\n        this.metadataThreadPoolNums = metadataThreadPoolNums;\n    }\n\n    public int getMetadataThreadPoolQueueCapacity() {\n        return metadataThreadPoolQueueCapacity;\n    }\n\n    public void setMetadataThreadPoolQueueCapacity(int metadataThreadPoolQueueCapacity) {\n        this.metadataThreadPoolQueueCapacity = metadataThreadPoolQueueCapacity;\n    }\n\n    public int getTransactionHeartbeatThreadPoolNums() {\n        return transactionHeartbeatThreadPoolNums;\n    }\n\n    public void setTransactionHeartbeatThreadPoolNums(int transactionHeartbeatThreadPoolNums) {\n        this.transactionHeartbeatThreadPoolNums = transactionHeartbeatThreadPoolNums;\n    }\n\n    public int getTransactionHeartbeatThreadPoolQueueCapacity() {\n        return transactionHeartbeatThreadPoolQueueCapacity;\n    }\n\n    public void setTransactionHeartbeatThreadPoolQueueCapacity(int transactionHeartbeatThreadPoolQueueCapacity) {\n        this.transactionHeartbeatThreadPoolQueueCapacity = transactionHeartbeatThreadPoolQueueCapacity;\n    }\n\n    public int getTransactionHeartbeatPeriodSecond() {\n        return transactionHeartbeatPeriodSecond;\n    }\n\n    public void setTransactionHeartbeatPeriodSecond(int transactionHeartbeatPeriodSecond) {\n        this.transactionHeartbeatPeriodSecond = transactionHeartbeatPeriodSecond;\n    }\n\n    public int getTransactionHeartbeatBatchNum() {\n        return transactionHeartbeatBatchNum;\n    }\n\n    public void setTransactionHeartbeatBatchNum(int transactionHeartbeatBatchNum) {\n        this.transactionHeartbeatBatchNum = transactionHeartbeatBatchNum;\n    }\n\n    public long getTransactionDataExpireScanPeriodMillis() {\n        return transactionDataExpireScanPeriodMillis;\n    }\n\n    public void setTransactionDataExpireScanPeriodMillis(long transactionDataExpireScanPeriodMillis) {\n        this.transactionDataExpireScanPeriodMillis = transactionDataExpireScanPeriodMillis;\n    }\n\n    public long getTransactionDataMaxWaitClearMillis() {\n        return transactionDataMaxWaitClearMillis;\n    }\n\n    public void setTransactionDataMaxWaitClearMillis(long transactionDataMaxWaitClearMillis) {\n        this.transactionDataMaxWaitClearMillis = transactionDataMaxWaitClearMillis;\n    }\n\n    public long getTransactionDataExpireMillis() {\n        return transactionDataExpireMillis;\n    }\n\n    public void setTransactionDataExpireMillis(long transactionDataExpireMillis) {\n        this.transactionDataExpireMillis = transactionDataExpireMillis;\n    }\n\n    public int getTransactionDataMaxNum() {\n        return transactionDataMaxNum;\n    }\n\n    public void setTransactionDataMaxNum(int transactionDataMaxNum) {\n        this.transactionDataMaxNum = transactionDataMaxNum;\n    }\n\n    public long getLongPollingReserveTimeInMillis() {\n        return longPollingReserveTimeInMillis;\n    }\n\n    public void setLongPollingReserveTimeInMillis(long longPollingReserveTimeInMillis) {\n        this.longPollingReserveTimeInMillis = longPollingReserveTimeInMillis;\n    }\n\n    public boolean isEnableAclRpcHookForClusterMode() {\n        return enableAclRpcHookForClusterMode;\n    }\n\n    public void setEnableAclRpcHookForClusterMode(boolean enableAclRpcHookForClusterMode) {\n        this.enableAclRpcHookForClusterMode = enableAclRpcHookForClusterMode;\n    }\n\n    public boolean isEnableTopicMessageTypeCheck() {\n        return enableTopicMessageTypeCheck;\n    }\n\n    public void setEnableTopicMessageTypeCheck(boolean enableTopicMessageTypeCheck) {\n        this.enableTopicMessageTypeCheck = enableTopicMessageTypeCheck;\n    }\n\n    public long getInvisibleTimeMillisWhenClear() {\n        return invisibleTimeMillisWhenClear;\n    }\n\n    public void setInvisibleTimeMillisWhenClear(long invisibleTimeMillisWhenClear) {\n        this.invisibleTimeMillisWhenClear = invisibleTimeMillisWhenClear;\n    }\n\n    public boolean isEnableProxyAutoRenew() {\n        return enableProxyAutoRenew;\n    }\n\n    public void setEnableProxyAutoRenew(boolean enableProxyAutoRenew) {\n        this.enableProxyAutoRenew = enableProxyAutoRenew;\n    }\n\n    public int getMaxRenewRetryTimes() {\n        return maxRenewRetryTimes;\n    }\n\n    public void setMaxRenewRetryTimes(int maxRenewRetryTimes) {\n        this.maxRenewRetryTimes = maxRenewRetryTimes;\n    }\n\n    public int getRenewThreadPoolNums() {\n        return renewThreadPoolNums;\n    }\n\n    public void setRenewThreadPoolNums(int renewThreadPoolNums) {\n        this.renewThreadPoolNums = renewThreadPoolNums;\n    }\n\n    public int getRenewMaxThreadPoolNums() {\n        return renewMaxThreadPoolNums;\n    }\n\n    public void setRenewMaxThreadPoolNums(int renewMaxThreadPoolNums) {\n        this.renewMaxThreadPoolNums = renewMaxThreadPoolNums;\n    }\n\n    public int getRenewThreadPoolQueueCapacity() {\n        return renewThreadPoolQueueCapacity;\n    }\n\n    public void setRenewThreadPoolQueueCapacity(int renewThreadPoolQueueCapacity) {\n        this.renewThreadPoolQueueCapacity = renewThreadPoolQueueCapacity;\n    }\n\n    public long getLockTimeoutMsInHandleGroup() {\n        return lockTimeoutMsInHandleGroup;\n    }\n\n    public void setLockTimeoutMsInHandleGroup(long lockTimeoutMsInHandleGroup) {\n        this.lockTimeoutMsInHandleGroup = lockTimeoutMsInHandleGroup;\n    }\n\n    public long getRenewAheadTimeMillis() {\n        return renewAheadTimeMillis;\n    }\n\n    public void setRenewAheadTimeMillis(long renewAheadTimeMillis) {\n        this.renewAheadTimeMillis = renewAheadTimeMillis;\n    }\n\n    public long getRenewMaxTimeMillis() {\n        return renewMaxTimeMillis;\n    }\n\n    public void setRenewMaxTimeMillis(long renewMaxTimeMillis) {\n        this.renewMaxTimeMillis = renewMaxTimeMillis;\n    }\n\n    public long getRenewSchedulePeriodMillis() {\n        return renewSchedulePeriodMillis;\n    }\n\n    public void setRenewSchedulePeriodMillis(long renewSchedulePeriodMillis) {\n        this.renewSchedulePeriodMillis = renewSchedulePeriodMillis;\n    }\n\n    public String getMetricCollectorMode() {\n        return metricCollectorMode;\n    }\n\n    public void setMetricCollectorMode(String metricCollectorMode) {\n        this.metricCollectorMode = metricCollectorMode;\n    }\n\n    public String getMetricCollectorAddress() {\n        return metricCollectorAddress;\n    }\n\n    public void setMetricCollectorAddress(String metricCollectorAddress) {\n        this.metricCollectorAddress = metricCollectorAddress;\n    }\n\n    public boolean isUseDelayLevel() {\n        return useDelayLevel;\n    }\n\n    public void setUseDelayLevel(boolean useDelayLevel) {\n        this.useDelayLevel = useDelayLevel;\n    }\n\n    public String getMessageDelayLevel() {\n        return messageDelayLevel;\n    }\n\n    public void setMessageDelayLevel(String messageDelayLevel) {\n        this.messageDelayLevel = messageDelayLevel;\n    }\n\n    public ConcurrentSkipListMap<Integer, Long> getDelayLevelTable() {\n        return delayLevelTable;\n    }\n\n    public long getGrpcClientIdleTimeMills() {\n        return grpcClientIdleTimeMills;\n    }\n\n    public void setGrpcClientIdleTimeMills(final long grpcClientIdleTimeMills) {\n        this.grpcClientIdleTimeMills = grpcClientIdleTimeMills;\n    }\n\n    public String getRegionId() {\n        return regionId;\n    }\n\n    public void setRegionId(String regionId) {\n        this.regionId = regionId;\n    }\n\n    public boolean isTraceOn() {\n        return traceOn;\n    }\n\n    public void setTraceOn(boolean traceOn) {\n        this.traceOn = traceOn;\n    }\n\n    public String getRemotingAccessAddr() {\n        return remotingAccessAddr;\n    }\n\n    public void setRemotingAccessAddr(String remotingAccessAddr) {\n        this.remotingAccessAddr = remotingAccessAddr;\n    }\n\n    public MetricsExporterType getMetricsExporterType() {\n        return metricsExporterType;\n    }\n\n    public void setMetricsExporterType(MetricsExporterType metricsExporterType) {\n        this.metricsExporterType = metricsExporterType;\n    }\n\n    public void setMetricsExporterType(String metricsExporterType) {\n        this.metricsExporterType = MetricsExporterType.valueOf(metricsExporterType);\n    }\n\n    public String getMetricsGrpcExporterTarget() {\n        return metricsGrpcExporterTarget;\n    }\n\n    public void setMetricsGrpcExporterTarget(String metricsGrpcExporterTarget) {\n        this.metricsGrpcExporterTarget = metricsGrpcExporterTarget;\n    }\n\n    public String getMetricsGrpcExporterHeader() {\n        return metricsGrpcExporterHeader;\n    }\n\n    public void setMetricsGrpcExporterHeader(String metricsGrpcExporterHeader) {\n        this.metricsGrpcExporterHeader = metricsGrpcExporterHeader;\n    }\n\n    public long getMetricGrpcExporterTimeOutInMills() {\n        return metricGrpcExporterTimeOutInMills;\n    }\n\n    public void setMetricGrpcExporterTimeOutInMills(long metricGrpcExporterTimeOutInMills) {\n        this.metricGrpcExporterTimeOutInMills = metricGrpcExporterTimeOutInMills;\n    }\n\n    public long getMetricGrpcExporterIntervalInMills() {\n        return metricGrpcExporterIntervalInMills;\n    }\n\n    public void setMetricGrpcExporterIntervalInMills(long metricGrpcExporterIntervalInMills) {\n        this.metricGrpcExporterIntervalInMills = metricGrpcExporterIntervalInMills;\n    }\n\n    public long getMetricLoggingExporterIntervalInMills() {\n        return metricLoggingExporterIntervalInMills;\n    }\n\n    public void setMetricLoggingExporterIntervalInMills(long metricLoggingExporterIntervalInMills) {\n        this.metricLoggingExporterIntervalInMills = metricLoggingExporterIntervalInMills;\n    }\n\n    public int getMetricsPromExporterPort() {\n        return metricsPromExporterPort;\n    }\n\n    public void setMetricsPromExporterPort(int metricsPromExporterPort) {\n        this.metricsPromExporterPort = metricsPromExporterPort;\n    }\n\n    public String getMetricsPromExporterHost() {\n        return metricsPromExporterHost;\n    }\n\n    public void setMetricsPromExporterHost(String metricsPromExporterHost) {\n        this.metricsPromExporterHost = metricsPromExporterHost;\n    }\n\n    public String getMetricsLabel() {\n        return metricsLabel;\n    }\n\n    public void setMetricsLabel(String metricsLabel) {\n        this.metricsLabel = metricsLabel;\n    }\n\n    public boolean isMetricsInDelta() {\n        return metricsInDelta;\n    }\n\n    public void setMetricsInDelta(boolean metricsInDelta) {\n        this.metricsInDelta = metricsInDelta;\n    }\n\n    public long getChannelExpiredTimeout() {\n        return channelExpiredTimeout;\n    }\n\n    public boolean isEnableRemotingLocalProxyGrpc() {\n        return enableRemotingLocalProxyGrpc;\n    }\n\n    public void setChannelExpiredTimeout(long channelExpiredTimeout) {\n        this.channelExpiredTimeout = channelExpiredTimeout;\n    }\n\n    public void setEnableRemotingLocalProxyGrpc(boolean enableRemotingLocalProxyGrpc) {\n        this.enableRemotingLocalProxyGrpc = enableRemotingLocalProxyGrpc;\n    }\n\n    public int getLocalProxyConnectTimeoutMs() {\n        return localProxyConnectTimeoutMs;\n    }\n\n    public void setLocalProxyConnectTimeoutMs(int localProxyConnectTimeoutMs) {\n        this.localProxyConnectTimeoutMs = localProxyConnectTimeoutMs;\n    }\n\n    public int getRemotingListenPort() {\n        return remotingListenPort;\n    }\n\n    public void setRemotingListenPort(int remotingListenPort) {\n        this.remotingListenPort = remotingListenPort;\n    }\n\n    public int getRemotingHeartbeatThreadPoolNums() {\n        return remotingHeartbeatThreadPoolNums;\n    }\n\n    public void setRemotingHeartbeatThreadPoolNums(int remotingHeartbeatThreadPoolNums) {\n        this.remotingHeartbeatThreadPoolNums = remotingHeartbeatThreadPoolNums;\n    }\n\n    public int getRemotingTopicRouteThreadPoolNums() {\n        return remotingTopicRouteThreadPoolNums;\n    }\n\n    public void setRemotingTopicRouteThreadPoolNums(int remotingTopicRouteThreadPoolNums) {\n        this.remotingTopicRouteThreadPoolNums = remotingTopicRouteThreadPoolNums;\n    }\n\n    public int getRemotingSendMessageThreadPoolNums() {\n        return remotingSendMessageThreadPoolNums;\n    }\n\n    public void setRemotingSendMessageThreadPoolNums(int remotingSendMessageThreadPoolNums) {\n        this.remotingSendMessageThreadPoolNums = remotingSendMessageThreadPoolNums;\n    }\n\n    public int getRemotingPullMessageThreadPoolNums() {\n        return remotingPullMessageThreadPoolNums;\n    }\n\n    public void setRemotingPullMessageThreadPoolNums(int remotingPullMessageThreadPoolNums) {\n        this.remotingPullMessageThreadPoolNums = remotingPullMessageThreadPoolNums;\n    }\n\n    public int getRemotingUpdateOffsetThreadPoolNums() {\n        return remotingUpdateOffsetThreadPoolNums;\n    }\n\n    public void setRemotingUpdateOffsetThreadPoolNums(int remotingUpdateOffsetThreadPoolNums) {\n        this.remotingUpdateOffsetThreadPoolNums = remotingUpdateOffsetThreadPoolNums;\n    }\n\n    public int getRemotingDefaultThreadPoolNums() {\n        return remotingDefaultThreadPoolNums;\n    }\n\n    public void setRemotingDefaultThreadPoolNums(int remotingDefaultThreadPoolNums) {\n        this.remotingDefaultThreadPoolNums = remotingDefaultThreadPoolNums;\n    }\n\n    public int getRemotingHeartbeatThreadPoolQueueCapacity() {\n        return remotingHeartbeatThreadPoolQueueCapacity;\n    }\n\n    public void setRemotingHeartbeatThreadPoolQueueCapacity(int remotingHeartbeatThreadPoolQueueCapacity) {\n        this.remotingHeartbeatThreadPoolQueueCapacity = remotingHeartbeatThreadPoolQueueCapacity;\n    }\n\n    public int getRemotingTopicRouteThreadPoolQueueCapacity() {\n        return remotingTopicRouteThreadPoolQueueCapacity;\n    }\n\n    public void setRemotingTopicRouteThreadPoolQueueCapacity(int remotingTopicRouteThreadPoolQueueCapacity) {\n        this.remotingTopicRouteThreadPoolQueueCapacity = remotingTopicRouteThreadPoolQueueCapacity;\n    }\n\n    public int getRemotingSendThreadPoolQueueCapacity() {\n        return remotingSendThreadPoolQueueCapacity;\n    }\n\n    public void setRemotingSendThreadPoolQueueCapacity(int remotingSendThreadPoolQueueCapacity) {\n        this.remotingSendThreadPoolQueueCapacity = remotingSendThreadPoolQueueCapacity;\n    }\n\n    public int getRemotingPullThreadPoolQueueCapacity() {\n        return remotingPullThreadPoolQueueCapacity;\n    }\n\n    public void setRemotingPullThreadPoolQueueCapacity(int remotingPullThreadPoolQueueCapacity) {\n        this.remotingPullThreadPoolQueueCapacity = remotingPullThreadPoolQueueCapacity;\n    }\n\n    public int getRemotingUpdateOffsetThreadPoolQueueCapacity() {\n        return remotingUpdateOffsetThreadPoolQueueCapacity;\n    }\n\n    public void setRemotingUpdateOffsetThreadPoolQueueCapacity(int remotingUpdateOffsetThreadPoolQueueCapacity) {\n        this.remotingUpdateOffsetThreadPoolQueueCapacity = remotingUpdateOffsetThreadPoolQueueCapacity;\n    }\n\n    public int getRemotingDefaultThreadPoolQueueCapacity() {\n        return remotingDefaultThreadPoolQueueCapacity;\n    }\n\n    public void setRemotingDefaultThreadPoolQueueCapacity(int remotingDefaultThreadPoolQueueCapacity) {\n        this.remotingDefaultThreadPoolQueueCapacity = remotingDefaultThreadPoolQueueCapacity;\n    }\n\n    public long getRemotingWaitTimeMillsInSendQueue() {\n        return remotingWaitTimeMillsInSendQueue;\n    }\n\n    public void setRemotingWaitTimeMillsInSendQueue(long remotingWaitTimeMillsInSendQueue) {\n        this.remotingWaitTimeMillsInSendQueue = remotingWaitTimeMillsInSendQueue;\n    }\n\n    public long getRemotingWaitTimeMillsInPullQueue() {\n        return remotingWaitTimeMillsInPullQueue;\n    }\n\n    public void setRemotingWaitTimeMillsInPullQueue(long remotingWaitTimeMillsInPullQueue) {\n        this.remotingWaitTimeMillsInPullQueue = remotingWaitTimeMillsInPullQueue;\n    }\n\n    public long getRemotingWaitTimeMillsInHeartbeatQueue() {\n        return remotingWaitTimeMillsInHeartbeatQueue;\n    }\n\n    public void setRemotingWaitTimeMillsInHeartbeatQueue(long remotingWaitTimeMillsInHeartbeatQueue) {\n        this.remotingWaitTimeMillsInHeartbeatQueue = remotingWaitTimeMillsInHeartbeatQueue;\n    }\n\n    public long getRemotingWaitTimeMillsInUpdateOffsetQueue() {\n        return remotingWaitTimeMillsInUpdateOffsetQueue;\n    }\n\n    public void setRemotingWaitTimeMillsInUpdateOffsetQueue(long remotingWaitTimeMillsInUpdateOffsetQueue) {\n        this.remotingWaitTimeMillsInUpdateOffsetQueue = remotingWaitTimeMillsInUpdateOffsetQueue;\n    }\n\n    public long getRemotingWaitTimeMillsInTopicRouteQueue() {\n        return remotingWaitTimeMillsInTopicRouteQueue;\n    }\n\n    public void setRemotingWaitTimeMillsInTopicRouteQueue(long remotingWaitTimeMillsInTopicRouteQueue) {\n        this.remotingWaitTimeMillsInTopicRouteQueue = remotingWaitTimeMillsInTopicRouteQueue;\n    }\n\n    public long getRemotingWaitTimeMillsInDefaultQueue() {\n        return remotingWaitTimeMillsInDefaultQueue;\n    }\n\n    public void setRemotingWaitTimeMillsInDefaultQueue(long remotingWaitTimeMillsInDefaultQueue) {\n        this.remotingWaitTimeMillsInDefaultQueue = remotingWaitTimeMillsInDefaultQueue;\n    }\n\n    public boolean isSendLatencyEnable() {\n        return sendLatencyEnable;\n    }\n\n    public boolean isStartDetectorEnable() {\n        return startDetectorEnable;\n    }\n\n    public void setStartDetectorEnable(boolean startDetectorEnable) {\n        this.startDetectorEnable = startDetectorEnable;\n    }\n\n    public void setSendLatencyEnable(boolean sendLatencyEnable) {\n        this.sendLatencyEnable = sendLatencyEnable;\n    }\n\n    public boolean getStartDetectorEnable() {\n        return this.startDetectorEnable;\n    }\n\n    public boolean getSendLatencyEnable() {\n        return this.sendLatencyEnable;\n    }\n\n    public int getDetectTimeout() {\n        return detectTimeout;\n    }\n\n    public void setDetectTimeout(int detectTimeout) {\n        this.detectTimeout = detectTimeout;\n    }\n\n    public int getDetectInterval() {\n        return detectInterval;\n    }\n\n    public void setDetectInterval(int detectInterval) {\n        this.detectInterval = detectInterval;\n    }\n\n    public boolean isEnableBatchAck() {\n        return enableBatchAck;\n    }\n\n    public void setEnableBatchAck(boolean enableBatchAck) {\n        this.enableBatchAck = enableBatchAck;\n    }\n\n    public boolean isEnableMessageBodyEmptyCheck() {\n        return enableMessageBodyEmptyCheck;\n    }\n\n    public void setEnableMessageBodyEmptyCheck(boolean enableMessageBodyEmptyCheck) {\n        this.enableMessageBodyEmptyCheck = enableMessageBodyEmptyCheck;\n    }\n\n    public int getMaxLiteTopicSize() {\n        return maxLiteTopicSize;\n    }\n\n    public void setMaxLiteTopicSize(int maxLiteTopicSize) {\n        this.maxLiteTopicSize = maxLiteTopicSize;\n    }\n\n    public int getMaxLiteRenewNumPerChannel() {\n        return maxLiteRenewNumPerChannel;\n    }\n\n    public void setMaxLiteRenewNumPerChannel(int maxLiteRenewNumPerChannel) {\n        this.maxLiteRenewNumPerChannel = maxLiteRenewNumPerChannel;\n    }\n\n    public int getMaxSyncLiteSubscriptionRate() {\n        return maxSyncLiteSubscriptionRate;\n    }\n\n    public void setMaxSyncLiteSubscriptionRate(int maxSyncLiteSubscriptionRate) {\n        this.maxSyncLiteSubscriptionRate = maxSyncLiteSubscriptionRate;\n    }\n\n    public int getReturnHandleGroupThreadPoolNums() {\n        return returnHandleGroupThreadPoolNums;\n    }\n\n    public void setReturnHandleGroupThreadPoolNums(int returnHandleGroupThreadPoolNums) {\n        this.returnHandleGroupThreadPoolNums = returnHandleGroupThreadPoolNums;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc;\n\nimport com.google.common.annotations.VisibleForTesting;\nimport io.grpc.Server;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.StartAndShutdown;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.service.cert.TlsCertificateManager;\n\nimport java.io.IOException;\nimport java.security.cert.CertificateException;\nimport java.util.concurrent.TimeUnit;\n\npublic class GrpcServer implements StartAndShutdown {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    private final Server server;\n\n    private final long timeout;\n\n    private final TimeUnit unit;\n\n    private final TlsCertificateManager tlsCertificateManager;\n    @VisibleForTesting final GrpcTlsReloadHandler tlsReloadHandler;\n\n    protected GrpcServer(Server server, long timeout, TimeUnit unit,\n        TlsCertificateManager tlsCertificateManager) throws Exception {\n        this.server = server;\n        this.timeout = timeout;\n        this.unit = unit;\n        this.tlsCertificateManager = tlsCertificateManager;\n        this.tlsReloadHandler = new GrpcTlsReloadHandler();\n    }\n\n    public void start() throws Exception {\n        // Register the TLS context reload handler\n        tlsCertificateManager.registerReloadListener(this.tlsReloadHandler);\n\n        this.server.start();\n        log.info(\"grpc server start successfully.\");\n    }\n\n    public void shutdown() {\n        try {\n            // Unregister the TLS context reload handler\n            tlsCertificateManager.unregisterReloadListener(this.tlsReloadHandler);\n\n            this.server.shutdown().awaitTermination(timeout, unit);\n\n            log.info(\"grpc server shutdown successfully.\");\n        } catch (Exception e) {\n            e.printStackTrace();\n            log.error(\"Failed to shutdown grpc server\", e);\n        }\n    }\n\n    @VisibleForTesting\n    class GrpcTlsReloadHandler implements TlsCertificateManager.TlsContextReloadListener {\n        @Override\n        public void onTlsContextReload() {\n            try {\n                ProxyAndTlsProtocolNegotiator.loadSslContext();\n                log.info(\"SslContext reloaded for grpc server\");\n            } catch (CertificateException | IOException e) {\n                log.error(\"Failed to reload SslContext for server\", e);\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.grpc;\n\nimport io.grpc.BindableService;\nimport io.grpc.ServerInterceptor;\nimport io.grpc.ServerServiceDefinition;\nimport io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder;\nimport io.grpc.netty.shaded.io.netty.channel.epoll.EpollEventLoopGroup;\nimport io.grpc.netty.shaded.io.netty.channel.epoll.EpollServerSocketChannel;\nimport io.grpc.netty.shaded.io.netty.channel.nio.NioEventLoopGroup;\nimport io.grpc.netty.shaded.io.netty.channel.socket.nio.NioServerSocketChannel;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.grpc.interceptor.ContextInterceptor;\nimport org.apache.rocketmq.proxy.grpc.interceptor.GlobalExceptionInterceptor;\nimport org.apache.rocketmq.proxy.grpc.interceptor.HeaderInterceptor;\n\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.proxy.service.cert.TlsCertificateManager;\n\npublic class GrpcServerBuilder {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    protected NettyServerBuilder serverBuilder;\n\n    protected long time = 30;\n\n    protected TimeUnit unit = TimeUnit.SECONDS;\n\n    protected TlsCertificateManager tlsCertificateManager;\n\n    public static GrpcServerBuilder newBuilder(ThreadPoolExecutor executor, int port,\n        TlsCertificateManager tlsCertificateManager) {\n        return new GrpcServerBuilder(executor, port, tlsCertificateManager);\n    }\n\n    protected GrpcServerBuilder(ThreadPoolExecutor executor, int port, TlsCertificateManager tlsCertificateManager) {\n        this.tlsCertificateManager = tlsCertificateManager;\n        serverBuilder = NettyServerBuilder.forPort(port);\n\n        serverBuilder.protocolNegotiator(new ProxyAndTlsProtocolNegotiator());\n\n        // build server\n        int bossLoopNum = ConfigurationManager.getProxyConfig().getGrpcBossLoopNum();\n        int workerLoopNum = ConfigurationManager.getProxyConfig().getGrpcWorkerLoopNum();\n        int maxInboundMessageSize = ConfigurationManager.getProxyConfig().getGrpcMaxInboundMessageSize();\n        long idleTimeMills = ConfigurationManager.getProxyConfig().getGrpcClientIdleTimeMills();\n\n        if (ConfigurationManager.getProxyConfig().isEnableGrpcEpoll()) {\n            serverBuilder.bossEventLoopGroup(new EpollEventLoopGroup(bossLoopNum))\n                .workerEventLoopGroup(new EpollEventLoopGroup(workerLoopNum))\n                .channelType(EpollServerSocketChannel.class)\n                .executor(executor);\n        } else {\n            serverBuilder.bossEventLoopGroup(new NioEventLoopGroup(bossLoopNum))\n                .workerEventLoopGroup(new NioEventLoopGroup(workerLoopNum))\n                .channelType(NioServerSocketChannel.class)\n                .executor(executor);\n        }\n\n        serverBuilder.maxInboundMessageSize(maxInboundMessageSize)\n            .maxConnectionIdle(idleTimeMills, TimeUnit.MILLISECONDS);\n\n        log.info(\"grpc server has built. port: {}, bossLoopNum: {}, workerLoopNum: {}, maxInboundMessageSize: {}\",\n            port, bossLoopNum, workerLoopNum, maxInboundMessageSize);\n    }\n\n    public GrpcServerBuilder shutdownTime(long time, TimeUnit unit) {\n        this.time = time;\n        this.unit = unit;\n        return this;\n    }\n\n    public GrpcServerBuilder addService(BindableService service) {\n        this.serverBuilder.addService(service);\n        return this;\n    }\n\n    public GrpcServerBuilder addService(ServerServiceDefinition service) {\n        this.serverBuilder.addService(service);\n        return this;\n    }\n\n    public GrpcServerBuilder appendInterceptor(ServerInterceptor interceptor) {\n        this.serverBuilder.intercept(interceptor);\n        return this;\n    }\n\n    public GrpcServer build() throws Exception {\n        return new GrpcServer(this.serverBuilder.build(), time, unit, tlsCertificateManager);\n    }\n\n    public GrpcServerBuilder configInterceptor() {\n        this.serverBuilder\n            .intercept(new GlobalExceptionInterceptor())\n            .intercept(new ContextInterceptor())\n            .intercept(new HeaderInterceptor());\n        return this;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.grpc;\n\nimport io.grpc.Attributes;\nimport io.grpc.netty.shaded.io.grpc.netty.GrpcHttp2ConnectionHandler;\nimport io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;\nimport io.grpc.netty.shaded.io.grpc.netty.InternalProtocolNegotiationEvent;\nimport io.grpc.netty.shaded.io.grpc.netty.InternalProtocolNegotiator;\nimport io.grpc.netty.shaded.io.grpc.netty.InternalProtocolNegotiators;\nimport io.grpc.netty.shaded.io.grpc.netty.ProtocolNegotiationEvent;\nimport io.grpc.netty.shaded.io.netty.buffer.ByteBuf;\nimport io.grpc.netty.shaded.io.netty.buffer.ByteBufUtil;\nimport io.grpc.netty.shaded.io.netty.channel.ChannelHandler;\nimport io.grpc.netty.shaded.io.netty.channel.ChannelHandlerContext;\nimport io.grpc.netty.shaded.io.netty.channel.ChannelInboundHandlerAdapter;\nimport io.grpc.netty.shaded.io.netty.handler.codec.ByteToMessageDecoder;\nimport io.grpc.netty.shaded.io.netty.handler.codec.ProtocolDetectionResult;\nimport io.grpc.netty.shaded.io.netty.handler.codec.ProtocolDetectionState;\nimport io.grpc.netty.shaded.io.netty.handler.codec.haproxy.HAProxyMessage;\nimport io.grpc.netty.shaded.io.netty.handler.codec.haproxy.HAProxyMessageDecoder;\nimport io.grpc.netty.shaded.io.netty.handler.codec.haproxy.HAProxyProtocolVersion;\nimport io.grpc.netty.shaded.io.netty.handler.codec.haproxy.HAProxyTLV;\nimport io.grpc.netty.shaded.io.netty.handler.ssl.ClientAuth;\nimport io.grpc.netty.shaded.io.netty.handler.ssl.OpenSsl;\nimport io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;\nimport io.grpc.netty.shaded.io.netty.handler.ssl.SslHandler;\nimport io.grpc.netty.shaded.io.netty.handler.ssl.SslProvider;\n\nimport io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory;\nimport io.grpc.netty.shaded.io.netty.handler.ssl.util.SelfSignedCertificate;\nimport io.grpc.netty.shaded.io.netty.util.AsciiString;\nimport io.grpc.netty.shaded.io.netty.util.CharsetUtil;\n\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.nio.file.Files;\nimport java.nio.file.Paths;\nimport java.security.cert.CertificateException;\nimport java.util.List;\n\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.constant.HAProxyConstants;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.BinaryUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.grpc.constant.AttributeKeys;\nimport org.apache.rocketmq.remoting.common.TlsMode;\nimport org.apache.rocketmq.remoting.netty.TlsSystemConfig;\n\npublic class ProxyAndTlsProtocolNegotiator implements InternalProtocolNegotiator.ProtocolNegotiator {\n    protected static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    private static final String HA_PROXY_DECODER = \"HAProxyDecoder\";\n    private static final String HA_PROXY_HANDLER = \"HAProxyHandler\";\n    private static final String TLS_MODE_HANDLER = \"TlsModeHandler\";\n    /**\n     * the length of the ssl record header (in bytes)\n     */\n    private static final int SSL_RECORD_HEADER_LENGTH = 5;\n\n    private static SslContext sslContext;\n\n    public ProxyAndTlsProtocolNegotiator() {\n        try {\n            loadSslContext();\n            log.info(\"SslContext created for proxy server\");\n        } catch (IOException | CertificateException e) {\n            log.error(\"SslContext init error\", e);\n            throw new RuntimeException(e);\n        }\n    }\n\n    @Override\n    public AsciiString scheme() {\n        return AsciiString.of(\"https\");\n    }\n\n    @Override\n    public ChannelHandler newHandler(GrpcHttp2ConnectionHandler grpcHandler) {\n        return new ProxyAndTlsProtocolHandler(grpcHandler);\n    }\n\n    @Override\n    public void close() {\n    }\n\n    public static void loadSslContext() throws CertificateException, IOException {\n        ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();\n        SslProvider provider;\n        if (OpenSsl.isAvailable()) {\n            provider = SslProvider.OPENSSL;\n            log.info(\"Using OpenSSL provider\");\n        } else {\n            provider = SslProvider.JDK;\n            log.info(\"Using JDK SSL provider\");\n        }\n        if (proxyConfig.isTlsTestModeEnable()) {\n            SelfSignedCertificate selfSignedCertificate = new SelfSignedCertificate();\n            sslContext = GrpcSslContexts.forServer(selfSignedCertificate.certificate(), selfSignedCertificate.privateKey())\n                .sslProvider(provider)\n                .trustManager(InsecureTrustManagerFactory.INSTANCE)\n                .clientAuth(ClientAuth.NONE)\n                .build();\n        } else {\n            String tlsCertPath = ConfigurationManager.getProxyConfig().getTlsCertPath();\n            String tlsKeyPath = ConfigurationManager.getProxyConfig().getTlsKeyPath();\n            String tlsKeyPassword = ConfigurationManager.getProxyConfig().getTlsKeyPassword();\n            try (InputStream serverKeyInputStream = Files.newInputStream(\n                Paths.get(tlsKeyPath));\n                 InputStream serverCertificateStream = Files.newInputStream(\n                     Paths.get(tlsCertPath))) {\n                sslContext = GrpcSslContexts.forServer(serverCertificateStream,\n                        serverKeyInputStream,\n                        StringUtils.isNotBlank(tlsKeyPassword) ? tlsKeyPassword : null)\n                    .trustManager(InsecureTrustManagerFactory.INSTANCE)\n                    .clientAuth(ClientAuth.NONE)\n                    .build();\n            }\n        }\n    }\n\n    private class ProxyAndTlsProtocolHandler extends ByteToMessageDecoder {\n\n        private final GrpcHttp2ConnectionHandler grpcHandler;\n\n        private ProtocolNegotiationEvent pne = InternalProtocolNegotiationEvent.getDefault();\n\n        public ProxyAndTlsProtocolHandler(GrpcHttp2ConnectionHandler grpcHandler) {\n            this.grpcHandler = grpcHandler;\n        }\n\n        @Override\n        protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) {\n            try {\n                ProtocolDetectionResult<HAProxyProtocolVersion> ha = HAProxyMessageDecoder.detectProtocol(in);\n                if (ha.state() == ProtocolDetectionState.NEEDS_MORE_DATA) {\n                    return;\n                }\n                if (ha.state() == ProtocolDetectionState.DETECTED) {\n                    ctx.pipeline().addAfter(ctx.name(), HA_PROXY_DECODER, new HAProxyMessageDecoder())\n                        .addAfter(HA_PROXY_DECODER, HA_PROXY_HANDLER, new HAProxyMessageHandler())\n                        .addAfter(HA_PROXY_HANDLER, TLS_MODE_HANDLER, new TlsModeHandler(grpcHandler));\n                } else {\n                    ctx.pipeline().addAfter(ctx.name(), TLS_MODE_HANDLER, new TlsModeHandler(grpcHandler));\n                }\n\n                Attributes.Builder builder = InternalProtocolNegotiationEvent.getAttributes(pne).toBuilder();\n                builder.set(AttributeKeys.CHANNEL_ID, ctx.channel().id().asLongText());\n\n                ctx.fireUserEventTriggered(InternalProtocolNegotiationEvent.withAttributes(pne, builder.build()));\n                ctx.pipeline().remove(this);\n            } catch (Exception e) {\n                log.error(\"process proxy protocol negotiator failed.\", e);\n                throw e;\n            }\n        }\n\n        @Override\n        public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {\n            if (evt instanceof ProtocolNegotiationEvent) {\n                pne = (ProtocolNegotiationEvent) evt;\n            } else {\n                super.userEventTriggered(ctx, evt);\n            }\n        }\n    }\n\n    private class HAProxyMessageHandler extends ChannelInboundHandlerAdapter {\n\n        private ProtocolNegotiationEvent pne = InternalProtocolNegotiationEvent.getDefault();\n\n        @Override\n        public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {\n            if (msg instanceof HAProxyMessage) {\n                handleWithMessage((HAProxyMessage) msg);\n                ctx.fireUserEventTriggered(pne);\n            } else {\n                super.channelRead(ctx, msg);\n            }\n            ctx.pipeline().remove(this);\n        }\n\n        /**\n         * The definition of key refers to the implementation of nginx\n         * <a href=\"https://nginx.org/en/docs/http/ngx_http_core_module.html#var_proxy_protocol_addr\">ngx_http_core_module</a>\n         *\n         * @param msg\n         */\n        private void handleWithMessage(HAProxyMessage msg) {\n            try {\n                Attributes.Builder builder = InternalProtocolNegotiationEvent.getAttributes(pne).toBuilder();\n                if (StringUtils.isNotBlank(msg.sourceAddress())) {\n                    builder.set(AttributeKeys.PROXY_PROTOCOL_ADDR, msg.sourceAddress());\n                }\n                if (msg.sourcePort() > 0) {\n                    builder.set(AttributeKeys.PROXY_PROTOCOL_PORT, String.valueOf(msg.sourcePort()));\n                }\n                if (StringUtils.isNotBlank(msg.destinationAddress())) {\n                    builder.set(AttributeKeys.PROXY_PROTOCOL_SERVER_ADDR, msg.destinationAddress());\n                }\n                if (msg.destinationPort() > 0) {\n                    builder.set(AttributeKeys.PROXY_PROTOCOL_SERVER_PORT, String.valueOf(msg.destinationPort()));\n                }\n                if (CollectionUtils.isNotEmpty(msg.tlvs())) {\n                    msg.tlvs().forEach(tlv -> handleHAProxyTLV(tlv, builder));\n                }\n                pne = InternalProtocolNegotiationEvent\n                    .withAttributes(InternalProtocolNegotiationEvent.getDefault(), builder.build());\n            } finally {\n                msg.release();\n            }\n        }\n\n        @Override\n        public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {\n            if (evt instanceof ProtocolNegotiationEvent) {\n                pne = (ProtocolNegotiationEvent) evt;\n            } else {\n                super.userEventTriggered(ctx, evt);\n            }\n        }\n    }\n\n    protected void handleHAProxyTLV(HAProxyTLV tlv, Attributes.Builder builder) {\n        byte[] valueBytes = ByteBufUtil.getBytes(tlv.content());\n        if (!BinaryUtil.isAscii(valueBytes)) {\n            return;\n        }\n        Attributes.Key<String> key = AttributeKeys.valueOf(\n            HAProxyConstants.PROXY_PROTOCOL_TLV_PREFIX + String.format(\"%02x\", tlv.typeByteValue()));\n        builder.set(key, new String(valueBytes, CharsetUtil.UTF_8));\n    }\n\n    private class TlsModeHandler extends ByteToMessageDecoder {\n\n        private ProtocolNegotiationEvent pne = InternalProtocolNegotiationEvent.getDefault();\n\n        private final ChannelHandler ssl;\n        private final ChannelHandler plaintext;\n\n        public TlsModeHandler(GrpcHttp2ConnectionHandler grpcHandler) {\n            this.ssl = InternalProtocolNegotiators.serverTls(sslContext)\n                .newHandler(grpcHandler);\n            this.plaintext = InternalProtocolNegotiators.serverPlaintext()\n                .newHandler(grpcHandler);\n        }\n\n        @Override\n        protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) {\n            try {\n                TlsMode tlsMode = TlsSystemConfig.tlsMode;\n                if (TlsMode.ENFORCING.equals(tlsMode)) {\n                    ctx.pipeline().addAfter(ctx.name(), null, this.ssl);\n                } else if (TlsMode.DISABLED.equals(tlsMode)) {\n                    ctx.pipeline().addAfter(ctx.name(), null, this.plaintext);\n                } else {\n                    // in SslHandler.isEncrypted, it needs at least 5 bytes to judge is encrypted or not\n                    if (in.readableBytes() < SSL_RECORD_HEADER_LENGTH) {\n                        return;\n                    }\n                    if (SslHandler.isEncrypted(in)) {\n                        ctx.pipeline().addAfter(ctx.name(), null, this.ssl);\n                    } else {\n                        ctx.pipeline().addAfter(ctx.name(), null, this.plaintext);\n                    }\n                }\n                ctx.fireUserEventTriggered(pne);\n                ctx.pipeline().remove(this);\n            } catch (Exception e) {\n                log.error(\"process ssl protocol negotiator failed.\", e);\n                throw e;\n            }\n        }\n\n        @Override\n        public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {\n            if (evt instanceof ProtocolNegotiationEvent) {\n                pne = (ProtocolNegotiationEvent) evt;\n            } else {\n                super.userEventTriggered(ctx, evt);\n            }\n        }\n    }\n}"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/constant/AttributeKeys.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.grpc.constant;\n\nimport io.grpc.Attributes;\nimport org.apache.rocketmq.common.constant.HAProxyConstants;\n\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\n\npublic class AttributeKeys {\n\n    public static final Attributes.Key<String> CHANNEL_ID =\n        Attributes.Key.create(HAProxyConstants.CHANNEL_ID);\n\n    public static final Attributes.Key<String> PROXY_PROTOCOL_ADDR =\n            Attributes.Key.create(HAProxyConstants.PROXY_PROTOCOL_ADDR);\n\n    public static final Attributes.Key<String> PROXY_PROTOCOL_PORT =\n            Attributes.Key.create(HAProxyConstants.PROXY_PROTOCOL_PORT);\n\n    public static final Attributes.Key<String> PROXY_PROTOCOL_SERVER_ADDR =\n            Attributes.Key.create(HAProxyConstants.PROXY_PROTOCOL_SERVER_ADDR);\n\n    public static final Attributes.Key<String> PROXY_PROTOCOL_SERVER_PORT =\n            Attributes.Key.create(HAProxyConstants.PROXY_PROTOCOL_SERVER_PORT);\n\n    private static final Map<String, Attributes.Key<String>> ATTRIBUTES_KEY_MAP = new ConcurrentHashMap<>();\n\n    public static Attributes.Key<String> valueOf(String name) {\n        return ATTRIBUTES_KEY_MAP.computeIfAbsent(name, key -> Attributes.Key.create(name));\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/ContextInterceptor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.interceptor;\n\nimport io.grpc.Context;\nimport io.grpc.Contexts;\nimport io.grpc.Metadata;\nimport io.grpc.ServerCall;\nimport io.grpc.ServerCallHandler;\nimport io.grpc.ServerInterceptor;\nimport org.apache.rocketmq.common.constant.GrpcConstants;\n\npublic class ContextInterceptor implements ServerInterceptor {\n\n    @Override\n    public <R, W> ServerCall.Listener<R> interceptCall(\n        ServerCall<R, W> call,\n        Metadata headers,\n        ServerCallHandler<R, W> next\n    ) {\n        Context context = Context.current().withValue(GrpcConstants.METADATA, headers);\n        return Contexts.interceptCall(context, call, headers, next);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/GlobalExceptionInterceptor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.interceptor;\n\nimport io.grpc.ForwardingServerCall;\nimport io.grpc.ForwardingServerCallListener;\nimport io.grpc.Metadata;\nimport io.grpc.ServerCall;\nimport io.grpc.ServerCallHandler;\nimport io.grpc.ServerInterceptor;\nimport io.grpc.Status;\nimport io.grpc.StatusRuntimeException;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class GlobalExceptionInterceptor implements ServerInterceptor {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    @Override\n    public <R, W> ServerCall.Listener<R> interceptCall(\n        ServerCall<R, W> call,\n        Metadata headers,\n        ServerCallHandler<R, W> next\n    ) {\n        final ServerCall<R, W> serverCall = new ClosableServerCall<>(call);\n        ServerCall.Listener<R> delegate = next.startCall(serverCall, headers);\n        return new ForwardingServerCallListener.SimpleForwardingServerCallListener<R>(delegate) {\n            @Override\n            public void onMessage(R message) {\n                try {\n                    super.onMessage(message);\n                } catch (Throwable e) {\n                    closeWithException(e);\n                }\n            }\n\n            @Override\n            public void onHalfClose() {\n                try {\n                    super.onHalfClose();\n                } catch (Throwable e) {\n                    closeWithException(e);\n                }\n            }\n\n            @Override\n            public void onCancel() {\n                try {\n                    super.onCancel();\n                } catch (Throwable e) {\n                    closeWithException(e);\n                }\n            }\n\n            @Override\n            public void onComplete() {\n                try {\n                    super.onComplete();\n                } catch (Throwable e) {\n                    closeWithException(e);\n                }\n            }\n\n            @Override\n            public void onReady() {\n                try {\n                    super.onReady();\n                } catch (Throwable e) {\n                    closeWithException(e);\n                }\n            }\n\n            private void closeWithException(Throwable t) {\n                Metadata trailers = new Metadata();\n                Status status = Status.INTERNAL.withDescription(t.getMessage());\n                boolean printLog = true;\n\n                if (t instanceof StatusRuntimeException) {\n                    trailers = ((StatusRuntimeException) t).getTrailers();\n                    status = ((StatusRuntimeException) t).getStatus();\n                    // no error stack for permission denied.\n                    if (status.getCode().value() == Status.PERMISSION_DENIED.getCode().value()) {\n                        printLog = false;\n                    }\n                }\n\n                if (printLog) {\n                    log.error(\"grpc server has exception. errorMsg:{}, e:\", t.getMessage(), t);\n                }\n\n                serverCall.close(status, trailers);\n            }\n        };\n    }\n\n    private static class ClosableServerCall<R, W> extends\n        ForwardingServerCall.SimpleForwardingServerCall<R, W> {\n        private boolean closeCalled = false;\n\n        ClosableServerCall(ServerCall<R, W> delegate) {\n            super(delegate);\n        }\n\n        @Override\n        public synchronized void close(final Status status, final Metadata trailers) {\n            if (!closeCalled) {\n                closeCalled = true;\n                ClosableServerCall.super.close(status, trailers);\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/HeaderInterceptor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.interceptor;\n\nimport com.google.common.net.HostAndPort;\nimport io.grpc.Attributes;\nimport io.grpc.Grpc;\nimport io.grpc.Metadata;\nimport io.grpc.ServerCall;\nimport io.grpc.ServerCallHandler;\nimport io.grpc.ServerInterceptor;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.constant.HAProxyConstants;\nimport org.apache.rocketmq.common.constant.GrpcConstants;\nimport org.apache.rocketmq.proxy.common.utils.GrpcUtils;\nimport org.apache.rocketmq.proxy.grpc.constant.AttributeKeys;\n\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\n\npublic class HeaderInterceptor implements ServerInterceptor {\n    @Override\n    public <R, W> ServerCall.Listener<R> interceptCall(\n        ServerCall<R, W> call,\n        Metadata headers,\n        ServerCallHandler<R, W> next\n    ) {\n        String remoteAddress = getProxyProtocolAddress(call.getAttributes());\n        if (StringUtils.isBlank(remoteAddress)) {\n            SocketAddress remoteSocketAddress = call.getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR);\n            remoteAddress = parseSocketAddress(remoteSocketAddress);\n        }\n        GrpcUtils.putHeaderIfNotExist(headers, GrpcConstants.REMOTE_ADDRESS, remoteAddress);\n\n        SocketAddress localSocketAddress = call.getAttributes().get(Grpc.TRANSPORT_ATTR_LOCAL_ADDR);\n        String localAddress = parseSocketAddress(localSocketAddress);\n        GrpcUtils.putHeaderIfNotExist(headers, GrpcConstants.LOCAL_ADDRESS, localAddress);\n\n        for (Attributes.Key<?> key : call.getAttributes().keys()) {\n            if (!StringUtils.startsWith(key.toString(), HAProxyConstants.PROXY_PROTOCOL_PREFIX)) {\n                continue;\n            }\n            Metadata.Key<String> headerKey\n                    = Metadata.Key.of(key.toString(), Metadata.ASCII_STRING_MARSHALLER);\n            String headerValue = String.valueOf(call.getAttributes().get(key));\n            GrpcUtils.putHeaderIfNotExist(headers, headerKey, headerValue);\n        }\n\n        String channelId = call.getAttributes().get(AttributeKeys.CHANNEL_ID);\n        if (StringUtils.isNotBlank(channelId)) {\n            GrpcUtils.putHeaderIfNotExist(headers, GrpcConstants.CHANNEL_ID, channelId);\n        }\n\n        return next.startCall(call, headers);\n    }\n\n    private String parseSocketAddress(SocketAddress socketAddress) {\n        if (socketAddress instanceof InetSocketAddress) {\n            InetSocketAddress inetSocketAddress = (InetSocketAddress) socketAddress;\n            return HostAndPort.fromParts(\n                inetSocketAddress.getAddress()\n                    .getHostAddress(),\n                inetSocketAddress.getPort()\n            ).toString();\n        }\n\n        return \"\";\n    }\n\n    private String getProxyProtocolAddress(Attributes attributes) {\n        String proxyProtocolAddr = attributes.get(AttributeKeys.PROXY_PROTOCOL_ADDR);\n        String proxyProtocolPort = attributes.get(AttributeKeys.PROXY_PROTOCOL_PORT);\n        if (StringUtils.isBlank(proxyProtocolAddr) || StringUtils.isBlank(proxyProtocolPort)) {\n            return null;\n        }\n        return proxyProtocolAddr + \":\" + proxyProtocolPort;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/RequestMapping.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.interceptor;\n\nimport apache.rocketmq.v2.AckMessageRequest;\nimport apache.rocketmq.v2.ChangeInvisibleDurationRequest;\nimport apache.rocketmq.v2.EndTransactionRequest;\nimport apache.rocketmq.v2.ForwardMessageToDeadLetterQueueResponse;\nimport apache.rocketmq.v2.HeartbeatRequest;\nimport apache.rocketmq.v2.NotifyClientTerminationRequest;\nimport apache.rocketmq.v2.QueryAssignmentRequest;\nimport apache.rocketmq.v2.QueryRouteRequest;\nimport apache.rocketmq.v2.RecallMessageRequest;\nimport apache.rocketmq.v2.ReceiveMessageRequest;\nimport apache.rocketmq.v2.SendMessageRequest;\nimport java.util.HashMap;\nimport java.util.Map;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\npublic class RequestMapping {\n    @SuppressWarnings(\"DoubleBraceInitialization\")\n    private final static Map<String, Integer> REQUEST_MAP = new HashMap<String, Integer>() {\n        {\n            // v2\n            put(QueryRouteRequest.getDescriptor().getFullName(), RequestCode.GET_ROUTEINFO_BY_TOPIC);\n            put(HeartbeatRequest.getDescriptor().getFullName(), RequestCode.HEART_BEAT);\n            put(SendMessageRequest.getDescriptor().getFullName(), RequestCode.SEND_MESSAGE_V2);\n            put(RecallMessageRequest.getDescriptor().getFullName(), RequestCode.RECALL_MESSAGE);\n            put(QueryAssignmentRequest.getDescriptor().getFullName(), RequestCode.GET_ROUTEINFO_BY_TOPIC);\n            put(ReceiveMessageRequest.getDescriptor().getFullName(), RequestCode.PULL_MESSAGE);\n            put(AckMessageRequest.getDescriptor().getFullName(), RequestCode.UPDATE_CONSUMER_OFFSET);\n            put(ForwardMessageToDeadLetterQueueResponse.getDescriptor().getFullName(), RequestCode.CONSUMER_SEND_MSG_BACK);\n            put(EndTransactionRequest.getDescriptor().getFullName(), RequestCode.END_TRANSACTION);\n            put(NotifyClientTerminationRequest.getDescriptor().getFullName(), RequestCode.UNREGISTER_CLIENT);\n            put(ChangeInvisibleDurationRequest.getDescriptor().getFullName(), RequestCode.CONSUMER_SEND_MSG_BACK);\n        }\n    };\n\n    public static int map(String rpcFullName) {\n        if (REQUEST_MAP.containsKey(rpcFullName)) {\n            return REQUEST_MAP.get(rpcFullName);\n        }\n        return RequestCode.HEART_BEAT;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/pipeline/AuthenticationPipeline.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.grpc.pipeline;\n\nimport com.google.protobuf.GeneratedMessageV3;\nimport io.grpc.Context;\nimport io.grpc.Metadata;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.auth.authentication.AuthenticationEvaluator;\nimport org.apache.rocketmq.auth.authentication.context.AuthenticationContext;\nimport org.apache.rocketmq.auth.authentication.context.DefaultAuthenticationContext;\nimport org.apache.rocketmq.auth.authentication.exception.AuthenticationException;\nimport org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.constant.GrpcConstants;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.common.utils.GrpcUtils;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\n\npublic class AuthenticationPipeline implements RequestPipeline {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    private final AuthConfig authConfig;\n    private final AuthenticationEvaluator authenticationEvaluator;\n\n    public AuthenticationPipeline(AuthConfig authConfig, MessagingProcessor messagingProcessor) {\n        this.authConfig = authConfig;\n        this.authenticationEvaluator = AuthenticationFactory.getEvaluator(authConfig, messagingProcessor::getMetadataService);\n    }\n\n    @Override\n    public void execute(ProxyContext context, Metadata headers, GeneratedMessageV3 request) {\n        if (!authConfig.isAuthenticationEnabled()) {\n            return;\n        }\n        try {\n            Metadata metadata = GrpcConstants.METADATA.get(Context.current());\n            AuthenticationContext authenticationContext = newContext(context, metadata, request);\n            authenticationEvaluator.evaluate(authenticationContext);\n        } catch (AuthenticationException ex) {\n            throw ex;\n        } catch (Throwable ex) {\n            LOGGER.error(\"authenticate failed, request:{}\", request, ex);\n            throw ex;\n        }\n    }\n\n    /**\n     * Create Context, for extension\n     *\n     * @param context for extension\n     * @param headers gRPC headers\n     * @param request\n     * @return\n     */\n    protected AuthenticationContext newContext(ProxyContext context, Metadata headers, GeneratedMessageV3 request) {\n        AuthenticationContext result = AuthenticationFactory.newContext(authConfig, headers, request);\n        if (result instanceof DefaultAuthenticationContext) {\n            DefaultAuthenticationContext defaultAuthenticationContext = (DefaultAuthenticationContext) result;\n            if (StringUtils.isNotBlank(defaultAuthenticationContext.getUsername())) {\n                GrpcUtils.putHeaderIfNotExist(headers, GrpcConstants.AUTHORIZATION_AK, defaultAuthenticationContext.getUsername());\n            }\n        }\n        return result;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/pipeline/AuthorizationPipeline.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.grpc.pipeline;\n\nimport com.google.protobuf.GeneratedMessageV3;\nimport io.grpc.Metadata;\nimport java.util.List;\nimport org.apache.rocketmq.auth.authentication.exception.AuthenticationException;\nimport org.apache.rocketmq.auth.authorization.AuthorizationEvaluator;\nimport org.apache.rocketmq.auth.authorization.context.AuthorizationContext;\nimport org.apache.rocketmq.auth.authorization.exception.AuthorizationException;\nimport org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\n\npublic class AuthorizationPipeline implements RequestPipeline {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    private final AuthConfig authConfig;\n    private final AuthorizationEvaluator authorizationEvaluator;\n\n    public AuthorizationPipeline(AuthConfig authConfig, MessagingProcessor messagingProcessor) {\n        this.authConfig = authConfig;\n        this.authorizationEvaluator = AuthorizationFactory.getEvaluator(authConfig, messagingProcessor::getMetadataService);\n    }\n\n    @Override\n    public void execute(ProxyContext context, Metadata headers, GeneratedMessageV3 request) {\n        if (!authConfig.isAuthorizationEnabled()) {\n            return;\n        }\n        try {\n            List<AuthorizationContext> contexts = newContexts(context, headers, request);\n            authorizationEvaluator.evaluate(contexts);\n        } catch (AuthorizationException | AuthenticationException ex) {\n            throw ex;\n        }  catch (Throwable ex) {\n            LOGGER.error(\"authorize failed, request:{}\", request, ex);\n            throw ex;\n        }\n    }\n\n    protected List<AuthorizationContext> newContexts(ProxyContext context, Metadata headers, GeneratedMessageV3 request) {\n        return AuthorizationFactory.newContexts(authConfig, headers, request);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/pipeline/ContextInitPipeline.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.grpc.pipeline;\n\nimport com.google.protobuf.GeneratedMessageV3;\nimport io.grpc.Context;\nimport io.grpc.Metadata;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.constant.GrpcConstants;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.processor.channel.ChannelProtocolType;\n\npublic class ContextInitPipeline implements RequestPipeline {\n    @Override\n    public void execute(ProxyContext context, Metadata headers, GeneratedMessageV3 request) {\n        Context ctx = Context.current();\n        context.setLocalAddress(getDefaultStringMetadataInfo(headers, GrpcConstants.LOCAL_ADDRESS))\n            .setRemoteAddress(getDefaultStringMetadataInfo(headers, GrpcConstants.REMOTE_ADDRESS))\n            .setClientID(getDefaultStringMetadataInfo(headers, GrpcConstants.CLIENT_ID))\n            .setProtocolType(ChannelProtocolType.GRPC_V2.getName())\n            .setLanguage(getDefaultStringMetadataInfo(headers, GrpcConstants.LANGUAGE))\n            .setClientVersion(getDefaultStringMetadataInfo(headers, GrpcConstants.CLIENT_VERSION))\n            .setAction(getDefaultStringMetadataInfo(headers, GrpcConstants.SIMPLE_RPC_NAME))\n            .setNamespace(getDefaultStringMetadataInfo(headers, GrpcConstants.NAMESPACE_ID));\n        if (ctx.getDeadline() != null) {\n            context.setRemainingMs(ctx.getDeadline().timeRemaining(TimeUnit.MILLISECONDS));\n        }\n    }\n\n    protected String getDefaultStringMetadataInfo(Metadata headers, Metadata.Key<String> key) {\n        return StringUtils.defaultString(headers.get(key));\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/pipeline/RequestPipeline.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.pipeline;\n\nimport com.google.protobuf.GeneratedMessageV3;\nimport io.grpc.Metadata;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\n\npublic interface RequestPipeline {\n\n    void execute(ProxyContext context, Metadata headers, GeneratedMessageV3 request);\n\n    default RequestPipeline pipe(RequestPipeline source) {\n        return (ctx, headers, request) -> {\n            source.execute(ctx, headers, request);\n            execute(ctx, headers, request);\n        };\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/AbstractMessagingActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.grpc.v2;\n\nimport apache.rocketmq.v2.Resource;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcValidator;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\n\npublic abstract class AbstractMessagingActivity {\n    protected static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    protected final MessagingProcessor messagingProcessor;\n    protected final GrpcClientSettingsManager grpcClientSettingsManager;\n    protected final GrpcChannelManager grpcChannelManager;\n\n    public AbstractMessagingActivity(MessagingProcessor messagingProcessor,\n                                     GrpcClientSettingsManager grpcClientSettingsManager, GrpcChannelManager grpcChannelManager) {\n        this.messagingProcessor = messagingProcessor;\n        this.grpcClientSettingsManager = grpcClientSettingsManager;\n        this.grpcChannelManager = grpcChannelManager;\n    }\n\n    protected void validateTopic(Resource topic) {\n        GrpcValidator.getInstance().validateTopic(topic);\n    }\n\n    protected void validateLiteTopic(String liteTopic) {\n        GrpcValidator.getInstance().validateLiteTopic(liteTopic);\n    }\n\n    protected void validateConsumerGroup(Resource consumerGroup) {\n        GrpcValidator.getInstance().validateConsumerGroup(consumerGroup);\n    }\n\n    protected void validateTopicAndConsumerGroup(Resource topic, Resource consumerGroup) {\n        GrpcValidator.getInstance().validateTopicAndConsumerGroup(topic, consumerGroup);\n    }\n\n    protected void validateInvisibleTime(long invisibleTime) {\n        GrpcValidator.getInstance().validateInvisibleTime(invisibleTime);\n    }\n\n    protected void validateInvisibleTime(long invisibleTime, long minInvisibleTime) {\n        GrpcValidator.getInstance().validateInvisibleTime(invisibleTime, minInvisibleTime);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/ContextStreamObserver.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2;\n\nimport org.apache.rocketmq.proxy.common.ProxyContext;\n\npublic interface ContextStreamObserver<V> {\n\n    void onNext(ProxyContext ctx, V value);\n\n    void onError(Throwable t);\n\n    void onCompleted();\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessagingActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.grpc.v2;\n\nimport apache.rocketmq.v2.AckMessageRequest;\nimport apache.rocketmq.v2.AckMessageResponse;\nimport apache.rocketmq.v2.ChangeInvisibleDurationRequest;\nimport apache.rocketmq.v2.ChangeInvisibleDurationResponse;\nimport apache.rocketmq.v2.EndTransactionRequest;\nimport apache.rocketmq.v2.EndTransactionResponse;\nimport apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest;\nimport apache.rocketmq.v2.ForwardMessageToDeadLetterQueueResponse;\nimport apache.rocketmq.v2.HeartbeatRequest;\nimport apache.rocketmq.v2.HeartbeatResponse;\nimport apache.rocketmq.v2.NotifyClientTerminationRequest;\nimport apache.rocketmq.v2.NotifyClientTerminationResponse;\nimport apache.rocketmq.v2.QueryAssignmentRequest;\nimport apache.rocketmq.v2.QueryAssignmentResponse;\nimport apache.rocketmq.v2.QueryRouteRequest;\nimport apache.rocketmq.v2.QueryRouteResponse;\nimport apache.rocketmq.v2.RecallMessageRequest;\nimport apache.rocketmq.v2.RecallMessageResponse;\nimport apache.rocketmq.v2.ReceiveMessageRequest;\nimport apache.rocketmq.v2.ReceiveMessageResponse;\nimport apache.rocketmq.v2.SendMessageRequest;\nimport apache.rocketmq.v2.SendMessageResponse;\nimport apache.rocketmq.v2.SyncLiteSubscriptionRequest;\nimport apache.rocketmq.v2.SyncLiteSubscriptionResponse;\nimport apache.rocketmq.v2.TelemetryCommand;\nimport io.grpc.stub.StreamObserver;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.common.utils.AbstractStartAndShutdown;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;\nimport org.apache.rocketmq.proxy.grpc.v2.client.ClientActivity;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;\nimport org.apache.rocketmq.proxy.grpc.v2.consumer.AckMessageActivity;\nimport org.apache.rocketmq.proxy.grpc.v2.consumer.ChangeInvisibleDurationActivity;\nimport org.apache.rocketmq.proxy.grpc.v2.consumer.ReceiveMessageActivity;\nimport org.apache.rocketmq.proxy.grpc.v2.producer.ForwardMessageToDLQActivity;\nimport org.apache.rocketmq.proxy.grpc.v2.producer.RecallMessageActivity;\nimport org.apache.rocketmq.proxy.grpc.v2.producer.SendMessageActivity;\nimport org.apache.rocketmq.proxy.grpc.v2.route.RouteActivity;\nimport org.apache.rocketmq.proxy.grpc.v2.transaction.EndTransactionActivity;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\n\npublic class DefaultGrpcMessagingActivity extends AbstractStartAndShutdown implements GrpcMessagingActivity {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    protected GrpcClientSettingsManager grpcClientSettingsManager;\n    protected GrpcChannelManager grpcChannelManager;\n    protected ReceiveMessageActivity receiveMessageActivity;\n    protected AckMessageActivity ackMessageActivity;\n    protected ChangeInvisibleDurationActivity changeInvisibleDurationActivity;\n    protected SendMessageActivity sendMessageActivity;\n    protected RecallMessageActivity recallMessageActivity;\n    protected ForwardMessageToDLQActivity forwardMessageToDLQActivity;\n    protected EndTransactionActivity endTransactionActivity;\n    protected RouteActivity routeActivity;\n    protected ClientActivity clientActivity;\n\n    protected DefaultGrpcMessagingActivity(MessagingProcessor messagingProcessor) {\n        this.init(messagingProcessor);\n    }\n\n    protected void init(MessagingProcessor messagingProcessor) {\n        this.grpcClientSettingsManager = new GrpcClientSettingsManager(messagingProcessor);\n        this.grpcChannelManager = new GrpcChannelManager(messagingProcessor.getProxyRelayService(), this.grpcClientSettingsManager);\n\n        this.receiveMessageActivity = new ReceiveMessageActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n        this.ackMessageActivity = new AckMessageActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n        this.changeInvisibleDurationActivity = new ChangeInvisibleDurationActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n        this.sendMessageActivity = new SendMessageActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n        this.recallMessageActivity = new RecallMessageActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n        this.forwardMessageToDLQActivity = new ForwardMessageToDLQActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n        this.endTransactionActivity = new EndTransactionActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n        this.routeActivity = new RouteActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n        this.clientActivity = new ClientActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n\n        this.appendStartAndShutdown(this.grpcClientSettingsManager);\n    }\n\n    @Override\n    public CompletableFuture<QueryRouteResponse> queryRoute(ProxyContext ctx, QueryRouteRequest request) {\n        return this.routeActivity.queryRoute(ctx, request);\n    }\n\n    @Override\n    public CompletableFuture<HeartbeatResponse> heartbeat(ProxyContext ctx, HeartbeatRequest request) {\n        return this.clientActivity.heartbeat(ctx, request);\n    }\n\n    @Override\n    public CompletableFuture<SendMessageResponse> sendMessage(ProxyContext ctx, SendMessageRequest request) {\n        return this.sendMessageActivity.sendMessage(ctx, request);\n    }\n\n    @Override\n    public CompletableFuture<QueryAssignmentResponse> queryAssignment(ProxyContext ctx,\n        QueryAssignmentRequest request) {\n        return this.routeActivity.queryAssignment(ctx, request);\n    }\n\n    @Override\n    public void receiveMessage(ProxyContext ctx, ReceiveMessageRequest request,\n        StreamObserver<ReceiveMessageResponse> responseObserver) {\n        this.receiveMessageActivity.receiveMessage(ctx, request, responseObserver);\n    }\n\n    @Override\n    public CompletableFuture<AckMessageResponse> ackMessage(ProxyContext ctx, AckMessageRequest request) {\n        return this.ackMessageActivity.ackMessage(ctx, request);\n    }\n\n    @Override\n    public CompletableFuture<ForwardMessageToDeadLetterQueueResponse> forwardMessageToDeadLetterQueue(ProxyContext ctx,\n        ForwardMessageToDeadLetterQueueRequest request) {\n        return this.forwardMessageToDLQActivity.forwardMessageToDeadLetterQueue(ctx, request);\n    }\n\n    @Override\n    public CompletableFuture<EndTransactionResponse> endTransaction(ProxyContext ctx, EndTransactionRequest request) {\n        return this.endTransactionActivity.endTransaction(ctx, request);\n    }\n\n    @Override\n    public CompletableFuture<NotifyClientTerminationResponse> notifyClientTermination(ProxyContext ctx,\n        NotifyClientTerminationRequest request) {\n        return this.clientActivity.notifyClientTermination(ctx, request);\n    }\n\n    @Override\n    public CompletableFuture<ChangeInvisibleDurationResponse> changeInvisibleDuration(ProxyContext ctx,\n        ChangeInvisibleDurationRequest request) {\n        return this.changeInvisibleDurationActivity.changeInvisibleDuration(ctx, request);\n    }\n\n    @Override\n    public CompletableFuture<RecallMessageResponse> recallMessage(ProxyContext ctx,\n        RecallMessageRequest request) {\n        return this.recallMessageActivity.recallMessage(ctx, request);\n    }\n\n    @Override\n    public CompletableFuture<SyncLiteSubscriptionResponse> syncLiteSubscription(ProxyContext ctx,\n        SyncLiteSubscriptionRequest request) {\n        return this.clientActivity.syncLiteSubscription(ctx, request);\n    }\n\n    @Override\n    public ContextStreamObserver<TelemetryCommand> telemetry(StreamObserver<TelemetryCommand> responseObserver) {\n        return this.clientActivity.telemetry(responseObserver);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2;\n\nimport apache.rocketmq.v2.AckMessageRequest;\nimport apache.rocketmq.v2.AckMessageResponse;\nimport apache.rocketmq.v2.ChangeInvisibleDurationRequest;\nimport apache.rocketmq.v2.ChangeInvisibleDurationResponse;\nimport apache.rocketmq.v2.EndTransactionRequest;\nimport apache.rocketmq.v2.EndTransactionResponse;\nimport apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest;\nimport apache.rocketmq.v2.ForwardMessageToDeadLetterQueueResponse;\nimport apache.rocketmq.v2.HeartbeatRequest;\nimport apache.rocketmq.v2.HeartbeatResponse;\nimport apache.rocketmq.v2.NotifyClientTerminationRequest;\nimport apache.rocketmq.v2.NotifyClientTerminationResponse;\nimport apache.rocketmq.v2.QueryAssignmentRequest;\nimport apache.rocketmq.v2.QueryAssignmentResponse;\nimport apache.rocketmq.v2.QueryRouteRequest;\nimport apache.rocketmq.v2.QueryRouteResponse;\nimport apache.rocketmq.v2.RecallMessageRequest;\nimport apache.rocketmq.v2.RecallMessageResponse;\nimport apache.rocketmq.v2.ReceiveMessageRequest;\nimport apache.rocketmq.v2.ReceiveMessageResponse;\nimport apache.rocketmq.v2.SendMessageRequest;\nimport apache.rocketmq.v2.SendMessageResponse;\nimport apache.rocketmq.v2.SyncLiteSubscriptionRequest;\nimport apache.rocketmq.v2.SyncLiteSubscriptionResponse;\nimport apache.rocketmq.v2.TelemetryCommand;\nimport io.grpc.stub.StreamObserver;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.common.utils.StartAndShutdown;\n\npublic interface GrpcMessagingActivity extends StartAndShutdown {\n\n    CompletableFuture<QueryRouteResponse> queryRoute(ProxyContext ctx, QueryRouteRequest request);\n\n    CompletableFuture<HeartbeatResponse> heartbeat(ProxyContext ctx, HeartbeatRequest request);\n\n    CompletableFuture<SendMessageResponse> sendMessage(ProxyContext ctx, SendMessageRequest request);\n\n    CompletableFuture<QueryAssignmentResponse> queryAssignment(ProxyContext ctx, QueryAssignmentRequest request);\n\n    void receiveMessage(ProxyContext ctx, ReceiveMessageRequest request,\n        StreamObserver<ReceiveMessageResponse> responseObserver);\n\n    CompletableFuture<AckMessageResponse> ackMessage(ProxyContext ctx, AckMessageRequest request);\n\n    CompletableFuture<ForwardMessageToDeadLetterQueueResponse> forwardMessageToDeadLetterQueue(ProxyContext ctx,\n        ForwardMessageToDeadLetterQueueRequest request);\n\n    CompletableFuture<EndTransactionResponse> endTransaction(ProxyContext ctx, EndTransactionRequest request);\n\n    CompletableFuture<NotifyClientTerminationResponse> notifyClientTermination(ProxyContext ctx,\n        NotifyClientTerminationRequest request);\n\n    CompletableFuture<ChangeInvisibleDurationResponse> changeInvisibleDuration(ProxyContext ctx,\n        ChangeInvisibleDurationRequest request);\n\n    CompletableFuture<RecallMessageResponse> recallMessage(ProxyContext ctx, RecallMessageRequest request);\n\n    CompletableFuture<SyncLiteSubscriptionResponse> syncLiteSubscription(ProxyContext ctx, SyncLiteSubscriptionRequest request);\n\n    ContextStreamObserver<TelemetryCommand> telemetry(StreamObserver<TelemetryCommand> responseObserver);\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplication.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2;\n\nimport apache.rocketmq.v2.AckMessageRequest;\nimport apache.rocketmq.v2.AckMessageResponse;\nimport apache.rocketmq.v2.ChangeInvisibleDurationRequest;\nimport apache.rocketmq.v2.ChangeInvisibleDurationResponse;\nimport apache.rocketmq.v2.Code;\nimport apache.rocketmq.v2.EndTransactionRequest;\nimport apache.rocketmq.v2.EndTransactionResponse;\nimport apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest;\nimport apache.rocketmq.v2.ForwardMessageToDeadLetterQueueResponse;\nimport apache.rocketmq.v2.HeartbeatRequest;\nimport apache.rocketmq.v2.HeartbeatResponse;\nimport apache.rocketmq.v2.MessagingServiceGrpc;\nimport apache.rocketmq.v2.NotifyClientTerminationRequest;\nimport apache.rocketmq.v2.NotifyClientTerminationResponse;\nimport apache.rocketmq.v2.QueryAssignmentRequest;\nimport apache.rocketmq.v2.QueryAssignmentResponse;\nimport apache.rocketmq.v2.QueryRouteRequest;\nimport apache.rocketmq.v2.QueryRouteResponse;\nimport apache.rocketmq.v2.RecallMessageRequest;\nimport apache.rocketmq.v2.RecallMessageResponse;\nimport apache.rocketmq.v2.ReceiveMessageRequest;\nimport apache.rocketmq.v2.ReceiveMessageResponse;\nimport apache.rocketmq.v2.SendMessageRequest;\nimport apache.rocketmq.v2.SendMessageResponse;\nimport apache.rocketmq.v2.Status;\nimport apache.rocketmq.v2.SyncLiteSubscriptionRequest;\nimport apache.rocketmq.v2.SyncLiteSubscriptionResponse;\nimport apache.rocketmq.v2.TelemetryCommand;\nimport com.google.protobuf.GeneratedMessageV3;\nimport io.grpc.Context;\nimport io.grpc.stub.StreamObserver;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.RejectedExecutionHandler;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport java.util.function.Function;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.constant.GrpcConstants;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.thread.ThreadPoolMonitor;\nimport org.apache.rocketmq.common.utils.StartAndShutdown;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.grpc.pipeline.AuthenticationPipeline;\nimport org.apache.rocketmq.proxy.grpc.pipeline.AuthorizationPipeline;\nimport org.apache.rocketmq.proxy.grpc.pipeline.ContextInitPipeline;\nimport org.apache.rocketmq.proxy.grpc.pipeline.RequestPipeline;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException;\nimport org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder;\nimport org.apache.rocketmq.proxy.grpc.v2.common.ResponseWriter;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\n\npublic class GrpcMessagingApplication extends MessagingServiceGrpc.MessagingServiceImplBase implements StartAndShutdown {\n    private final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    private final GrpcMessagingActivity grpcMessagingActivity;\n\n    protected final RequestPipeline requestPipeline;\n\n    protected ThreadPoolExecutor routeThreadPoolExecutor;\n    protected ThreadPoolExecutor producerThreadPoolExecutor;\n    protected ThreadPoolExecutor consumerThreadPoolExecutor;\n    protected ThreadPoolExecutor clientManagerThreadPoolExecutor;\n    protected ThreadPoolExecutor transactionThreadPoolExecutor;\n\n\n    protected GrpcMessagingApplication(GrpcMessagingActivity grpcMessagingActivity, RequestPipeline requestPipeline) {\n        this.grpcMessagingActivity = grpcMessagingActivity;\n        this.requestPipeline = requestPipeline;\n\n        ProxyConfig config = ConfigurationManager.getProxyConfig();\n        this.routeThreadPoolExecutor = ThreadPoolMonitor.createAndMonitor(\n            config.getGrpcRouteThreadPoolNums(),\n            config.getGrpcRouteThreadPoolNums(),\n            1,\n            TimeUnit.MINUTES,\n            \"GrpcRouteThreadPool\",\n            config.getGrpcRouteThreadQueueCapacity()\n        );\n        this.producerThreadPoolExecutor = ThreadPoolMonitor.createAndMonitor(\n            config.getGrpcProducerThreadPoolNums(),\n            config.getGrpcProducerThreadPoolNums(),\n            1,\n            TimeUnit.MINUTES,\n            \"GrpcProducerThreadPool\",\n            config.getGrpcProducerThreadQueueCapacity()\n        );\n        this.consumerThreadPoolExecutor = ThreadPoolMonitor.createAndMonitor(\n            config.getGrpcConsumerThreadPoolNums(),\n            config.getGrpcConsumerThreadPoolNums(),\n            1,\n            TimeUnit.MINUTES,\n            \"GrpcConsumerThreadPool\",\n            config.getGrpcConsumerThreadQueueCapacity()\n        );\n        this.clientManagerThreadPoolExecutor = ThreadPoolMonitor.createAndMonitor(\n            config.getGrpcClientManagerThreadPoolNums(),\n            config.getGrpcClientManagerThreadPoolNums(),\n            1,\n            TimeUnit.MINUTES,\n            \"GrpcClientManagerThreadPool\",\n            config.getGrpcClientManagerThreadQueueCapacity()\n        );\n        this.transactionThreadPoolExecutor = ThreadPoolMonitor.createAndMonitor(\n            config.getGrpcTransactionThreadPoolNums(),\n            config.getGrpcTransactionThreadPoolNums(),\n            1,\n            TimeUnit.MINUTES,\n            \"GrpcTransactionThreadPool\",\n            config.getGrpcTransactionThreadQueueCapacity()\n        );\n\n        this.init();\n    }\n\n    protected void init() {\n        GrpcTaskRejectedExecutionHandler rejectedExecutionHandler = new GrpcTaskRejectedExecutionHandler();\n        this.routeThreadPoolExecutor.setRejectedExecutionHandler(rejectedExecutionHandler);\n        this.producerThreadPoolExecutor.setRejectedExecutionHandler(rejectedExecutionHandler);\n        this.consumerThreadPoolExecutor.setRejectedExecutionHandler(rejectedExecutionHandler);\n        this.clientManagerThreadPoolExecutor.setRejectedExecutionHandler(rejectedExecutionHandler);\n        this.transactionThreadPoolExecutor.setRejectedExecutionHandler(rejectedExecutionHandler);\n    }\n\n    public static GrpcMessagingApplication create(MessagingProcessor messagingProcessor) {\n        RequestPipeline pipeline = (context, headers, request) -> {\n        };\n        // add pipeline\n        // the last pipe add will execute at the first\n        AuthConfig authConfig = ConfigurationManager.getAuthConfig();\n        if (authConfig != null) {\n            pipeline = pipeline\n                .pipe(new AuthorizationPipeline(authConfig, messagingProcessor))\n                .pipe(new AuthenticationPipeline(authConfig, messagingProcessor));\n        }\n        pipeline = pipeline.pipe(new ContextInitPipeline());\n        return new GrpcMessagingApplication(new DefaultGrpcMessagingActivity(messagingProcessor), pipeline);\n    }\n\n    protected Status flowLimitStatus() {\n        return ResponseBuilder.getInstance().buildStatus(Code.TOO_MANY_REQUESTS, \"flow limit\");\n    }\n\n    protected Status convertExceptionToStatus(Throwable t) {\n        return ResponseBuilder.getInstance().buildStatus(t);\n    }\n\n    protected <V, T> void addExecutor(ExecutorService executor, ProxyContext context, V request, Runnable runnable,\n        StreamObserver<T> responseObserver, Function<Status, T> statusResponseCreator) {\n        if (request instanceof GeneratedMessageV3) {\n            requestPipeline.execute(context, GrpcConstants.METADATA.get(Context.current()), (GeneratedMessageV3) request);\n            validateContext(context);\n        } else {\n            log.error(\"[BUG]grpc request pipe is not been executed\");\n        }\n        executor.submit(new GrpcTask<>(runnable, context, request, responseObserver, statusResponseCreator.apply(flowLimitStatus())));\n    }\n\n    protected <V, T> void writeResponse(ProxyContext context, V request, T response, StreamObserver<T> responseObserver,\n        Throwable t, Function<Status, T> errorResponseCreator) {\n        if (t != null) {\n            ResponseWriter.getInstance().write(\n                responseObserver,\n                errorResponseCreator.apply(convertExceptionToStatus(t))\n            );\n        } else {\n            ResponseWriter.getInstance().write(responseObserver, response);\n        }\n    }\n\n    protected ProxyContext createContext() {\n        return ProxyContext.create();\n    }\n\n    protected void validateContext(ProxyContext context) {\n        if (StringUtils.isBlank(context.getClientID())) {\n            throw new GrpcProxyException(Code.CLIENT_ID_REQUIRED, \"client id cannot be empty\");\n        }\n    }\n\n    @Override\n    public void queryRoute(QueryRouteRequest request, StreamObserver<QueryRouteResponse> responseObserver) {\n        Function<Status, QueryRouteResponse> statusResponseCreator = status -> QueryRouteResponse.newBuilder().setStatus(status).build();\n        ProxyContext context = createContext();\n        try {\n            this.addExecutor(this.routeThreadPoolExecutor,\n                context,\n                request,\n                () -> grpcMessagingActivity.queryRoute(context, request)\n                    .whenComplete((response, throwable) -> writeResponse(context, request, response, responseObserver, throwable, statusResponseCreator)),\n                responseObserver,\n                statusResponseCreator);\n        } catch (Throwable t) {\n            writeResponse(context, request, null, responseObserver, t, statusResponseCreator);\n        }\n    }\n\n    @Override\n    public void heartbeat(HeartbeatRequest request, StreamObserver<HeartbeatResponse> responseObserver) {\n        Function<Status, HeartbeatResponse> statusResponseCreator = status -> HeartbeatResponse.newBuilder().setStatus(status).build();\n        ProxyContext context = createContext();\n        try {\n            this.addExecutor(this.clientManagerThreadPoolExecutor,\n                context,\n                request,\n                () -> grpcMessagingActivity.heartbeat(context, request)\n                    .whenComplete((response, throwable) -> writeResponse(context, request, response, responseObserver, throwable, statusResponseCreator)),\n                responseObserver,\n                statusResponseCreator);\n        } catch (Throwable t) {\n            writeResponse(context, request, null, responseObserver, t, statusResponseCreator);\n        }\n    }\n\n    @Override\n    public void sendMessage(SendMessageRequest request, StreamObserver<SendMessageResponse> responseObserver) {\n        Function<Status, SendMessageResponse> statusResponseCreator = status -> SendMessageResponse.newBuilder().setStatus(status).build();\n        ProxyContext context = createContext();\n        try {\n            this.addExecutor(this.producerThreadPoolExecutor,\n                context,\n                request,\n                () -> grpcMessagingActivity.sendMessage(context, request)\n                    .whenComplete((response, throwable) -> writeResponse(context, request, response, responseObserver, throwable, statusResponseCreator)),\n                responseObserver,\n                statusResponseCreator);\n        } catch (Throwable t) {\n            writeResponse(context, request, null, responseObserver, t, statusResponseCreator);\n        }\n    }\n\n    @Override\n    public void queryAssignment(QueryAssignmentRequest request,\n        StreamObserver<QueryAssignmentResponse> responseObserver) {\n        Function<Status, QueryAssignmentResponse> statusResponseCreator = status -> QueryAssignmentResponse.newBuilder().setStatus(status).build();\n        ProxyContext context = createContext();\n        try {\n            this.addExecutor(this.routeThreadPoolExecutor,\n                context,\n                request,\n                () -> grpcMessagingActivity.queryAssignment(context, request)\n                    .whenComplete((response, throwable) -> writeResponse(context, request, response, responseObserver, throwable, statusResponseCreator)),\n                responseObserver,\n                statusResponseCreator);\n        } catch (Throwable t) {\n            writeResponse(context, request, null, responseObserver, t, statusResponseCreator);\n        }\n    }\n\n    @Override\n    public void receiveMessage(ReceiveMessageRequest request, StreamObserver<ReceiveMessageResponse> responseObserver) {\n        Function<Status, ReceiveMessageResponse> statusResponseCreator = status -> ReceiveMessageResponse.newBuilder().setStatus(status).build();\n        ProxyContext context = createContext();\n        try {\n            this.addExecutor(this.consumerThreadPoolExecutor,\n                context,\n                request,\n                () -> grpcMessagingActivity.receiveMessage(context, request, responseObserver),\n                responseObserver,\n                statusResponseCreator);\n        } catch (Throwable t) {\n            writeResponse(context, request, null, responseObserver, t, statusResponseCreator);\n        }\n    }\n\n    @Override\n    public void ackMessage(AckMessageRequest request, StreamObserver<AckMessageResponse> responseObserver) {\n        Function<Status, AckMessageResponse> statusResponseCreator = status -> AckMessageResponse.newBuilder().setStatus(status).build();\n        ProxyContext context = createContext();\n        try {\n            this.addExecutor(this.consumerThreadPoolExecutor,\n                context,\n                request,\n                () -> grpcMessagingActivity.ackMessage(context, request)\n                    .whenComplete((response, throwable) -> writeResponse(context, request, response, responseObserver, throwable, statusResponseCreator)),\n                responseObserver,\n                statusResponseCreator);\n        } catch (Throwable t) {\n            writeResponse(context, request, null, responseObserver, t, statusResponseCreator);\n        }\n    }\n\n    @Override\n    public void forwardMessageToDeadLetterQueue(ForwardMessageToDeadLetterQueueRequest request,\n        StreamObserver<ForwardMessageToDeadLetterQueueResponse> responseObserver) {\n        Function<Status, ForwardMessageToDeadLetterQueueResponse> statusResponseCreator = status -> ForwardMessageToDeadLetterQueueResponse.newBuilder().setStatus(status).build();\n        ProxyContext context = createContext();\n        try {\n            this.addExecutor(this.producerThreadPoolExecutor,\n                context,\n                request,\n                () -> grpcMessagingActivity.forwardMessageToDeadLetterQueue(context, request)\n                    .whenComplete((response, throwable) -> writeResponse(context, request, response, responseObserver, throwable, statusResponseCreator)),\n                responseObserver,\n                statusResponseCreator);\n        } catch (Throwable t) {\n            writeResponse(context, request, null, responseObserver, t, statusResponseCreator);\n        }\n    }\n\n    @Override\n    public void endTransaction(EndTransactionRequest request, StreamObserver<EndTransactionResponse> responseObserver) {\n        Function<Status, EndTransactionResponse> statusResponseCreator = status -> EndTransactionResponse.newBuilder().setStatus(status).build();\n        ProxyContext context = createContext();\n        try {\n            this.addExecutor(this.transactionThreadPoolExecutor,\n                context,\n                request,\n                () -> grpcMessagingActivity.endTransaction(context, request)\n                    .whenComplete((response, throwable) -> writeResponse(context, request, response, responseObserver, throwable, statusResponseCreator)),\n                responseObserver,\n                statusResponseCreator);\n        } catch (Throwable t) {\n            writeResponse(context, request, null, responseObserver, t, statusResponseCreator);\n        }\n    }\n\n    @Override\n    public void notifyClientTermination(NotifyClientTerminationRequest request,\n        StreamObserver<NotifyClientTerminationResponse> responseObserver) {\n        Function<Status, NotifyClientTerminationResponse> statusResponseCreator = status -> NotifyClientTerminationResponse.newBuilder().setStatus(status).build();\n        ProxyContext context = createContext();\n        try {\n            this.addExecutor(this.clientManagerThreadPoolExecutor,\n                context,\n                request,\n                () -> grpcMessagingActivity.notifyClientTermination(context, request)\n                    .whenComplete((response, throwable) -> writeResponse(context, request, response, responseObserver, throwable, statusResponseCreator)),\n                responseObserver,\n                statusResponseCreator);\n        } catch (Throwable t) {\n            writeResponse(context, request, null, responseObserver, t, statusResponseCreator);\n        }\n    }\n\n    @Override\n    public void changeInvisibleDuration(ChangeInvisibleDurationRequest request,\n        StreamObserver<ChangeInvisibleDurationResponse> responseObserver) {\n        Function<Status, ChangeInvisibleDurationResponse> statusResponseCreator = status -> {\n            ChangeInvisibleDurationResponse.Builder builder =\n                ChangeInvisibleDurationResponse.newBuilder().setStatus(status);\n            if (Code.TOO_MANY_REQUESTS.equals(status.getCode())) {\n                builder.setReceiptHandle(request.getReceiptHandle());\n            }\n            return builder.build();\n        };\n        ProxyContext context = createContext();\n        try {\n            this.addExecutor(this.consumerThreadPoolExecutor,\n                context,\n                request,\n                () -> grpcMessagingActivity.changeInvisibleDuration(context, request)\n                    .whenComplete((response, throwable) -> writeResponse(context, request, response, responseObserver, throwable, statusResponseCreator)),\n                responseObserver,\n                statusResponseCreator);\n        } catch (Throwable t) {\n            writeResponse(context, request, null, responseObserver, t, statusResponseCreator);\n        }\n    }\n\n    @Override\n    public void recallMessage(RecallMessageRequest request, StreamObserver<RecallMessageResponse> responseObserver) {\n        Function<Status, RecallMessageResponse> statusResponseCreator =\n            status -> RecallMessageResponse.newBuilder().setStatus(status).build();\n        ProxyContext context = createContext();\n        try {\n            this.addExecutor(this.producerThreadPoolExecutor, // reuse producer thread pool\n                context,\n                request,\n                () -> grpcMessagingActivity.recallMessage(context, request)\n                    .whenComplete((response, throwable) ->\n                        writeResponse(context, request, response, responseObserver, throwable, statusResponseCreator)),\n                responseObserver,\n                statusResponseCreator);\n        } catch (Throwable t) {\n            writeResponse(context, request, null, responseObserver, t, statusResponseCreator);\n        }\n    }\n\n    @Override\n    public void syncLiteSubscription(SyncLiteSubscriptionRequest request,\n        StreamObserver<SyncLiteSubscriptionResponse> responseObserver) {\n        Function<Status, SyncLiteSubscriptionResponse> statusResponseCreator =\n            status -> SyncLiteSubscriptionResponse.newBuilder().setStatus(status).build();\n        ProxyContext context = createContext();\n        try {\n            this.addExecutor(this.clientManagerThreadPoolExecutor,\n                context,\n                request,\n                () -> grpcMessagingActivity.syncLiteSubscription(context, request)\n                    .whenComplete((response, throwable) ->\n                        writeResponse(context, request, response, responseObserver, throwable, statusResponseCreator)),\n                responseObserver,\n                statusResponseCreator);\n        } catch (Throwable t) {\n            writeResponse(context, request, null, responseObserver, t, statusResponseCreator);\n        }\n    }\n\n    @Override\n    public StreamObserver<TelemetryCommand> telemetry(StreamObserver<TelemetryCommand> responseObserver) {\n        Function<Status, TelemetryCommand> statusResponseCreator = status -> TelemetryCommand.newBuilder().setStatus(status).build();\n        ContextStreamObserver<TelemetryCommand> responseTelemetryCommand = grpcMessagingActivity.telemetry(responseObserver);\n        return new StreamObserver<TelemetryCommand>() {\n            @Override\n            public void onNext(TelemetryCommand value) {\n                ProxyContext context = createContext();\n                try {\n                    addExecutor(clientManagerThreadPoolExecutor,\n                        context,\n                        value,\n                        () -> responseTelemetryCommand.onNext(context, value),\n                        responseObserver,\n                        statusResponseCreator);\n                } catch (Throwable t) {\n                    writeResponse(context, value, null, responseObserver, t, statusResponseCreator);\n                }\n            }\n\n            @Override\n            public void onError(Throwable t) {\n                responseTelemetryCommand.onError(t);\n            }\n\n            @Override\n            public void onCompleted() {\n                responseTelemetryCommand.onCompleted();\n            }\n        };\n    }\n\n    @Override\n    public void shutdown() throws Exception {\n        this.grpcMessagingActivity.shutdown();\n        this.routeThreadPoolExecutor.shutdown();\n        this.producerThreadPoolExecutor.shutdown();\n        this.consumerThreadPoolExecutor.shutdown();\n        this.clientManagerThreadPoolExecutor.shutdown();\n        this.transactionThreadPoolExecutor.shutdown();\n    }\n\n    @Override\n    public void start() throws Exception {\n        this.grpcMessagingActivity.start();\n    }\n\n    protected static class GrpcTask<V, T> implements Runnable {\n\n        protected final Runnable runnable;\n        protected final ProxyContext context;\n        protected final V request;\n        protected final T executeRejectResponse;\n        protected final StreamObserver<T> streamObserver;\n\n        public GrpcTask(Runnable runnable, ProxyContext context, V request, StreamObserver<T> streamObserver,\n            T executeRejectResponse) {\n            this.runnable = runnable;\n            this.context = context;\n            this.streamObserver = streamObserver;\n            this.request = request;\n            this.executeRejectResponse = executeRejectResponse;\n        }\n\n        @Override\n        public void run() {\n            this.runnable.run();\n        }\n    }\n\n    protected class GrpcTaskRejectedExecutionHandler implements RejectedExecutionHandler {\n\n        public GrpcTaskRejectedExecutionHandler() {\n\n        }\n\n        @Override\n        public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {\n            if (r instanceof GrpcTask) {\n                try {\n                    GrpcTask grpcTask = (GrpcTask) r;\n                    writeResponse(grpcTask.context, grpcTask.request, grpcTask.executeRejectResponse, grpcTask.streamObserver, null, null);\n                } catch (Throwable t) {\n                    log.warn(\"write rejected error response failed\", t);\n                }\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2.channel;\n\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.utils.StartAndShutdown;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;\nimport org.apache.rocketmq.proxy.service.relay.ProxyRelayResult;\nimport org.apache.rocketmq.proxy.service.relay.ProxyRelayService;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\n\npublic class GrpcChannelManager implements StartAndShutdown {\n    private final ProxyRelayService proxyRelayService;\n    private final GrpcClientSettingsManager grpcClientSettingsManager;\n    protected final ConcurrentMap<String, GrpcClientChannel> clientIdChannelMap = new ConcurrentHashMap<>();\n\n    protected final AtomicLong nonceIdGenerator = new AtomicLong(0);\n    protected final ConcurrentMap<String /* nonce */, ResultFuture> resultNonceFutureMap = new ConcurrentHashMap<>();\n\n    protected final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor(\n        new ThreadFactoryImpl(\"GrpcChannelManager_\")\n    );\n\n    public GrpcChannelManager(ProxyRelayService proxyRelayService, GrpcClientSettingsManager grpcClientSettingsManager) {\n        this.proxyRelayService = proxyRelayService;\n        this.grpcClientSettingsManager = grpcClientSettingsManager;\n        this.init();\n    }\n\n    protected void init() {\n        this.scheduledExecutorService.scheduleAtFixedRate(\n            this::scanExpireResultFuture,\n            10, 1, TimeUnit.SECONDS\n        );\n    }\n\n    public GrpcClientChannel createChannel(ProxyContext ctx, String clientId) {\n        return this.clientIdChannelMap.computeIfAbsent(clientId,\n            k -> new GrpcClientChannel(proxyRelayService, grpcClientSettingsManager, this, ctx, clientId));\n    }\n\n    public GrpcClientChannel getChannel(String clientId) {\n        return clientIdChannelMap.get(clientId);\n    }\n\n    public GrpcClientChannel removeChannel(String clientId) {\n        return this.clientIdChannelMap.remove(clientId);\n    }\n\n    public <T> String addResponseFuture(CompletableFuture<ProxyRelayResult<T>> responseFuture) {\n        String nonce = this.nextNonce();\n        this.resultNonceFutureMap.put(nonce, new ResultFuture<>(responseFuture));\n        return nonce;\n    }\n\n    public <T> CompletableFuture<ProxyRelayResult<T>> getAndRemoveResponseFuture(String nonce) {\n        ResultFuture<T> resultFuture = this.resultNonceFutureMap.remove(nonce);\n        if (resultFuture != null) {\n            return resultFuture.future;\n        }\n        return null;\n    }\n\n    protected String nextNonce() {\n        return String.valueOf(this.nonceIdGenerator.getAndIncrement());\n    }\n\n    protected void scanExpireResultFuture() {\n        ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();\n        long timeOutMs = TimeUnit.SECONDS.toMillis(proxyConfig.getGrpcProxyRelayRequestTimeoutInSeconds());\n\n        Set<String> nonceSet = this.resultNonceFutureMap.keySet();\n        for (String nonce : nonceSet) {\n            ResultFuture<?> resultFuture = this.resultNonceFutureMap.get(nonce);\n            if (resultFuture == null) {\n                continue;\n            }\n            if (System.currentTimeMillis() - resultFuture.createTime > timeOutMs) {\n                resultFuture = this.resultNonceFutureMap.remove(nonce);\n                if (resultFuture != null) {\n                    resultFuture.future.complete(new ProxyRelayResult<>(ResponseCode.SYSTEM_BUSY, \"call remote timeout\", null));\n                }\n            }\n        }\n    }\n\n    @Override\n    public void shutdown() throws Exception {\n        this.scheduledExecutorService.shutdown();\n    }\n\n    @Override\n    public void start() throws Exception {\n\n    }\n\n    protected static class ResultFuture<T> {\n        public CompletableFuture<ProxyRelayResult<T>> future;\n        public long createTime = System.currentTimeMillis();\n\n        public ResultFuture(CompletableFuture<ProxyRelayResult<T>> future) {\n            this.future = future;\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannel.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.grpc.v2.channel;\n\nimport apache.rocketmq.v2.NotifyUnsubscribeLiteCommand;\nimport apache.rocketmq.v2.PrintThreadStackTraceCommand;\nimport apache.rocketmq.v2.RecoverOrphanedTransactionCommand;\nimport apache.rocketmq.v2.Settings;\nimport apache.rocketmq.v2.TelemetryCommand;\nimport apache.rocketmq.v2.VerifyMessageCommand;\nimport com.google.common.base.MoreObjects;\nimport com.google.common.collect.ComparisonChain;\nimport com.google.protobuf.InvalidProtocolBufferException;\nimport com.google.protobuf.TextFormat;\nimport com.google.protobuf.util.JsonFormat;\nimport io.grpc.StatusRuntimeException;\nimport io.grpc.stub.StreamObserver;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelId;\nimport java.util.Objects;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.atomic.AtomicReference;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.common.channel.ChannelHelper;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter;\nimport org.apache.rocketmq.proxy.processor.channel.ChannelExtendAttributeGetter;\nimport org.apache.rocketmq.proxy.processor.channel.ChannelProtocolType;\nimport org.apache.rocketmq.proxy.processor.channel.RemoteChannel;\nimport org.apache.rocketmq.proxy.processor.channel.RemoteChannelConverter;\nimport org.apache.rocketmq.proxy.service.relay.ProxyChannel;\nimport org.apache.rocketmq.proxy.service.relay.ProxyRelayResult;\nimport org.apache.rocketmq.proxy.service.relay.ProxyRelayService;\nimport org.apache.rocketmq.proxy.service.transaction.TransactionData;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.NotifyUnsubscribeLiteRequestHeader;\n\npublic class GrpcClientChannel extends ProxyChannel implements ChannelExtendAttributeGetter, RemoteChannelConverter {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    private final GrpcChannelManager grpcChannelManager;\n    private final GrpcClientSettingsManager grpcClientSettingsManager;\n\n    private final AtomicReference<StreamObserver<TelemetryCommand>> telemetryCommandRef = new AtomicReference<>();\n    private final Object telemetryWriteLock = new Object();\n    private final String clientId;\n\n    public GrpcClientChannel(ProxyRelayService proxyRelayService, GrpcClientSettingsManager grpcClientSettingsManager,\n        GrpcChannelManager grpcChannelManager, ProxyContext ctx, String clientId) {\n        super(proxyRelayService, null, new GrpcChannelId(clientId),\n            ctx.getRemoteAddress(),\n            ctx.getLocalAddress());\n        this.grpcChannelManager = grpcChannelManager;\n        this.grpcClientSettingsManager = grpcClientSettingsManager;\n        this.clientId = clientId;\n    }\n\n    @Override\n    public String getChannelExtendAttribute() {\n        Settings settings = this.grpcClientSettingsManager.getRawClientSettings(this.clientId);\n        if (settings == null) {\n            return null;\n        }\n        try {\n            return JsonFormat.printer().print(settings);\n        } catch (InvalidProtocolBufferException e) {\n            log.error(\"convert settings to json data failed. settings:{}\", settings, e);\n        }\n        return null;\n    }\n\n    public static Settings parseChannelExtendAttribute(Channel channel) {\n        if (ChannelHelper.getChannelProtocolType(channel).equals(ChannelProtocolType.GRPC_V2) &&\n            channel instanceof ChannelExtendAttributeGetter) {\n            String attr = ((ChannelExtendAttributeGetter) channel).getChannelExtendAttribute();\n            if (attr == null) {\n                return null;\n            }\n\n            Settings.Builder builder = Settings.newBuilder();\n            try {\n                JsonFormat.parser().merge(attr, builder);\n                return builder.build();\n            } catch (InvalidProtocolBufferException e) {\n                log.error(\"convert settings json data to settings failed. data:{}\", attr, e);\n                return null;\n            }\n        }\n        return null;\n    }\n\n    @Override\n    public RemoteChannel toRemoteChannel() {\n        return new RemoteChannel(\n            ConfigurationManager.getProxyConfig().getLocalServeAddr(),\n            this.getRemoteAddress(),\n            this.getLocalAddress(),\n            ChannelProtocolType.GRPC_V2,\n            this.getChannelExtendAttribute());\n    }\n\n    protected static class GrpcChannelId implements ChannelId {\n\n        private final String clientId;\n\n        public GrpcChannelId(String clientId) {\n            this.clientId = clientId;\n        }\n\n        @Override\n        public String asShortText() {\n            return this.clientId;\n        }\n\n        @Override\n        public String asLongText() {\n            return this.clientId;\n        }\n\n        @Override\n        public int compareTo(ChannelId o) {\n            if (this == o) {\n                return 0;\n            }\n            if (o instanceof GrpcChannelId) {\n                GrpcChannelId other = (GrpcChannelId) o;\n                return ComparisonChain.start()\n                    .compare(this.clientId, other.clientId)\n                    .result();\n            }\n\n            return asLongText().compareTo(o.asLongText());\n        }\n    }\n\n    public void setClientObserver(StreamObserver<TelemetryCommand> future) {\n        this.telemetryCommandRef.set(future);\n    }\n\n    protected void clearClientObserver(StreamObserver<TelemetryCommand> future) {\n        this.telemetryCommandRef.compareAndSet(future, null);\n    }\n\n    @Override\n    public boolean isOpen() {\n        return this.telemetryCommandRef.get() != null;\n    }\n\n    @Override\n    public boolean isActive() {\n        return this.telemetryCommandRef.get() != null;\n    }\n\n    @Override\n    public boolean isWritable() {\n        return this.telemetryCommandRef.get() != null;\n    }\n\n    @Override\n    protected CompletableFuture<Void> processOtherMessage(Object msg) {\n        if (msg instanceof TelemetryCommand) {\n            TelemetryCommand response = (TelemetryCommand) msg;\n            this.writeTelemetryCommand(response);\n        }\n        return CompletableFuture.completedFuture(null);\n    }\n\n    @Override\n    protected CompletableFuture<Void> processCheckTransaction(CheckTransactionStateRequestHeader header,\n        MessageExt messageExt, TransactionData transactionData, CompletableFuture<ProxyRelayResult<Void>> responseFuture) {\n        CompletableFuture<Void> writeFuture = new CompletableFuture<>();\n        try {\n            this.writeTelemetryCommand(TelemetryCommand.newBuilder()\n                .setRecoverOrphanedTransactionCommand(RecoverOrphanedTransactionCommand.newBuilder()\n                    .setTransactionId(transactionData.getTransactionId())\n                    .setMessage(GrpcConverter.getInstance().buildMessage(messageExt))\n                    .build())\n                .build());\n            responseFuture.complete(null);\n            writeFuture.complete(null);\n        } catch (Throwable t) {\n            responseFuture.completeExceptionally(t);\n            writeFuture.completeExceptionally(t);\n        }\n        return writeFuture;\n    }\n\n    @Override\n    protected CompletableFuture<Void> processNotifyUnsubscribeLite(NotifyUnsubscribeLiteRequestHeader header) {\n        final String group = header.getConsumerGroup();\n        final String liteTopic = header.getLiteTopic();\n        NotifyUnsubscribeLiteCommand unsubscribeLiteCommand = NotifyUnsubscribeLiteCommand.newBuilder()\n            .setLiteTopic(liteTopic)\n            .build();\n\n        TelemetryCommand telemetryCommand = TelemetryCommand.newBuilder()\n            .setNotifyUnsubscribeLiteCommand(unsubscribeLiteCommand)\n            .build();\n\n        this.writeTelemetryCommand(telemetryCommand);\n\n        log.info(\"notifyUnsubscribeLite liteTopic:{} group:{} clientId:{}\", liteTopic, group, clientId);\n\n        return CompletableFuture.completedFuture(null);\n    }\n\n    @Override\n    protected CompletableFuture<Void> processGetConsumerRunningInfo(RemotingCommand command,\n        GetConsumerRunningInfoRequestHeader header,\n        CompletableFuture<ProxyRelayResult<ConsumerRunningInfo>> responseFuture) {\n        if (Objects.isNull(header) || !header.isJstackEnable()) {\n            return CompletableFuture.completedFuture(null);\n        }\n        this.writeTelemetryCommand(TelemetryCommand.newBuilder()\n            .setPrintThreadStackTraceCommand(PrintThreadStackTraceCommand.newBuilder()\n                .setNonce(this.grpcChannelManager.addResponseFuture(responseFuture))\n                .build())\n            .build());\n        return CompletableFuture.completedFuture(null);\n    }\n\n    @Override\n    protected CompletableFuture<Void> processConsumeMessageDirectly(RemotingCommand command,\n        ConsumeMessageDirectlyResultRequestHeader header,\n        MessageExt messageExt, CompletableFuture<ProxyRelayResult<ConsumeMessageDirectlyResult>> responseFuture) {\n        this.writeTelemetryCommand(TelemetryCommand.newBuilder()\n            .setVerifyMessageCommand(VerifyMessageCommand.newBuilder()\n                .setNonce(this.grpcChannelManager.addResponseFuture(responseFuture))\n                .setMessage(GrpcConverter.getInstance().buildMessage(messageExt))\n                .build())\n            .build());\n        return CompletableFuture.completedFuture(null);\n    }\n\n    public String getClientId() {\n        return clientId;\n    }\n\n    public void writeTelemetryCommand(TelemetryCommand command) {\n        StreamObserver<TelemetryCommand> observer = this.telemetryCommandRef.get();\n        if (observer == null) {\n            log.warn(\"telemetry command observer is null when try to write data. command:{}, channel:{}\", TextFormat.shortDebugString(command), this);\n            return;\n        }\n        synchronized (this.telemetryWriteLock) {\n            observer = this.telemetryCommandRef.get();\n            if (observer == null) {\n                log.warn(\"telemetry command observer is null when try to write data. command:{}, channel:{}\", TextFormat.shortDebugString(command), this);\n                return;\n            }\n            try {\n                observer.onNext(command);\n            } catch (StatusRuntimeException | IllegalStateException exception) {\n                log.warn(\"write telemetry failed. command:{}\", command, exception);\n                this.clearClientObserver(observer);\n            }\n        }\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"clientId\", clientId)\n            .add(\"remoteAddress\", getRemoteAddress())\n            .add(\"localAddress\", getLocalAddress())\n            .toString();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/client/ClientActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.grpc.v2.client;\n\nimport apache.rocketmq.v2.ClientType;\nimport apache.rocketmq.v2.Code;\nimport apache.rocketmq.v2.FilterExpression;\nimport apache.rocketmq.v2.HeartbeatRequest;\nimport apache.rocketmq.v2.HeartbeatResponse;\nimport apache.rocketmq.v2.NotifyClientTerminationRequest;\nimport apache.rocketmq.v2.NotifyClientTerminationResponse;\nimport apache.rocketmq.v2.Resource;\nimport apache.rocketmq.v2.Settings;\nimport apache.rocketmq.v2.Status;\nimport apache.rocketmq.v2.SubscriptionEntry;\nimport apache.rocketmq.v2.SyncLiteSubscriptionRequest;\nimport apache.rocketmq.v2.SyncLiteSubscriptionResponse;\nimport apache.rocketmq.v2.TelemetryCommand;\nimport apache.rocketmq.v2.ThreadStackTrace;\nimport apache.rocketmq.v2.VerifyMessageResult;\nimport com.google.common.collect.ImmutableSet;\nimport io.grpc.StatusRuntimeException;\nimport io.grpc.stub.StreamObserver;\nimport io.netty.channel.Channel;\n\nimport java.time.Duration;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ConsumerGroupEvent;\nimport org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;\nimport org.apache.rocketmq.broker.client.ProducerChangeListener;\nimport org.apache.rocketmq.broker.client.ProducerGroupEvent;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.lite.LiteSubscriptionAction;\nimport org.apache.rocketmq.common.lite.LiteSubscriptionDTO;\nimport org.apache.rocketmq.common.lite.OffsetOption;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.common.channel.ChannelHelper;\nimport org.apache.rocketmq.proxy.grpc.v2.AbstractMessagingActivity;\nimport org.apache.rocketmq.proxy.grpc.v2.ContextStreamObserver;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcClientChannel;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException;\nimport org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.service.relay.ProxyRelayResult;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.CMResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.filter.FilterAPI;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\npublic class ClientActivity extends AbstractMessagingActivity {\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    public ClientActivity(MessagingProcessor messagingProcessor,\n        GrpcClientSettingsManager grpcClientSettingsManager,\n        GrpcChannelManager grpcChannelManager) {\n        super(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n        this.init();\n    }\n\n    protected void init() {\n        this.messagingProcessor.registerConsumerListener(new ConsumerIdsChangeListenerImpl());\n        this.messagingProcessor.registerProducerListener(new ProducerChangeListenerImpl());\n    }\n\n    public CompletableFuture<HeartbeatResponse> heartbeat(ProxyContext ctx, HeartbeatRequest request) {\n        CompletableFuture<HeartbeatResponse> future = new CompletableFuture<>();\n\n        try {\n            Settings clientSettings = grpcClientSettingsManager.getClientSettings(ctx);\n            if (clientSettings == null) {\n                future.complete(HeartbeatResponse.newBuilder()\n                    .setStatus(ResponseBuilder.getInstance().buildStatus(Code.UNRECOGNIZED_CLIENT_TYPE, \"cannot find client settings for this client\"))\n                    .build());\n                return future;\n            }\n            switch (clientSettings.getClientType()) {\n                case PRODUCER: {\n                    for (Resource topic : clientSettings.getPublishing().getTopicsList()) {\n                        String topicName = topic.getName();\n                        this.registerProducer(ctx, topicName);\n                    }\n                    break;\n                }\n                case PUSH_CONSUMER:\n                case LITE_PUSH_CONSUMER:\n                case SIMPLE_CONSUMER: {\n                    validateConsumerGroup(request.getGroup());\n                    String consumerGroup = request.getGroup().getName();\n                    this.registerConsumer(ctx, consumerGroup, clientSettings.getClientType(), clientSettings.getSubscription().getSubscriptionsList(), false);\n                    break;\n                }\n                default: {\n                    future.complete(HeartbeatResponse.newBuilder()\n                        .setStatus(ResponseBuilder.getInstance().buildStatus(Code.UNRECOGNIZED_CLIENT_TYPE, clientSettings.getClientType().name()))\n                        .build());\n                    return future;\n                }\n            }\n            future.complete(HeartbeatResponse.newBuilder()\n                .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()))\n                .build());\n            return future;\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    public CompletableFuture<NotifyClientTerminationResponse> notifyClientTermination(ProxyContext ctx,\n        NotifyClientTerminationRequest request) {\n        CompletableFuture<NotifyClientTerminationResponse> future = new CompletableFuture<>();\n\n        try {\n            String clientId = ctx.getClientID();\n            LanguageCode languageCode = LanguageCode.valueOf(ctx.getLanguage());\n            Settings clientSettings = grpcClientSettingsManager.removeAndGetClientSettings(ctx);\n            if (clientSettings == null) {\n                future.complete(NotifyClientTerminationResponse.newBuilder()\n                    .setStatus(ResponseBuilder.getInstance().buildStatus(Code.UNRECOGNIZED_CLIENT_TYPE, \"cannot find client settings for this client\"))\n                    .build());\n                return future;\n            }\n\n            switch (clientSettings.getClientType()) {\n                case PRODUCER:\n                    for (Resource topic : clientSettings.getPublishing().getTopicsList()) {\n                        String topicName = topic.getName();\n                        GrpcClientChannel channel = this.grpcChannelManager.removeChannel(clientId);\n                        if (channel != null) {\n                            ClientChannelInfo clientChannelInfo = new ClientChannelInfo(channel, clientId, languageCode, MQVersion.Version.V5_0_0.ordinal());\n                            this.messagingProcessor.unRegisterProducer(ctx, topicName, clientChannelInfo);\n                        }\n                    }\n                    break;\n                case PUSH_CONSUMER:\n                case LITE_PUSH_CONSUMER:\n                case SIMPLE_CONSUMER:\n                    validateConsumerGroup(request.getGroup());\n                    String consumerGroup = request.getGroup().getName();\n                    GrpcClientChannel channel = this.grpcChannelManager.removeChannel(clientId);\n                    if (channel != null) {\n                        ClientChannelInfo clientChannelInfo = new ClientChannelInfo(channel, clientId, languageCode, MQVersion.Version.V5_0_0.ordinal());\n                        this.messagingProcessor.unRegisterConsumer(ctx, consumerGroup, clientChannelInfo);\n                        this.grpcClientSettingsManager.offlineClientLiteSubscription(ctx, clientId, clientSettings);\n                    }\n                    break;\n                default:\n                    future.complete(NotifyClientTerminationResponse.newBuilder()\n                        .setStatus(ResponseBuilder.getInstance().buildStatus(Code.UNRECOGNIZED_CLIENT_TYPE, clientSettings.getClientType().name()))\n                        .build());\n                    return future;\n            }\n            future.complete(NotifyClientTerminationResponse.newBuilder()\n                .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()))\n                .build());\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    public CompletableFuture<SyncLiteSubscriptionResponse> syncLiteSubscription(ProxyContext ctx,\n        SyncLiteSubscriptionRequest request) {\n        try {\n            validateTopicAndConsumerGroup(request.getTopic(), request.getGroup());\n\n            final LiteSubscriptionAction action = toLiteAction(request.getAction());\n            final Set<String> liteTopicSet = ImmutableSet.copyOf(request.getLiteTopicSetList());\n            if (LiteSubscriptionAction.PARTIAL_ADD == action) {\n                for (String liteTopic : liteTopicSet) {\n                    validateLiteTopic(liteTopic);\n                }\n            }\n\n            final String group = request.getGroup().getName();\n            final String topic = request.getTopic().getName();\n            LiteSubscriptionDTO liteSubscriptionDTO = new LiteSubscriptionDTO()\n                .setAction(action)\n                .setClientId(ctx.getClientID())\n                .setGroup(group)\n                .setTopic(topic)\n                .setLiteTopicSet(liteTopicSet)\n                .setVersion(request.getVersion());\n\n            if (LiteSubscriptionAction.PARTIAL_ADD == action) {\n                if (request.hasOffsetOption()) {\n                    liteSubscriptionDTO.setOffsetOption(toOffsetOption(request.getOffsetOption()));\n                }\n            }\n\n            return this.messagingProcessor\n                .syncLiteSubscription(ctx, liteSubscriptionDTO, Duration.ofSeconds(2).toMillis())\n                .thenApply(v ->\n                    SyncLiteSubscriptionResponse\n                        .newBuilder()\n                        .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, null))\n                        .build()\n                );\n        } catch (Throwable t) {\n            CompletableFuture<SyncLiteSubscriptionResponse> future = new CompletableFuture<>();\n            future.completeExceptionally(t);\n            return future;\n        }\n    }\n\n    private static OffsetOption toOffsetOption(apache.rocketmq.v2.OffsetOption gRpcOffsetOption) {\n        OffsetOption offsetOption = new OffsetOption();\n        switch (gRpcOffsetOption.getOffsetTypeCase()) {\n            case POLICY:\n                offsetOption.setType(OffsetOption.Type.POLICY);\n                offsetOption.setValue(toOffsetPolicy(gRpcOffsetOption.getPolicy()));\n                break;\n            case OFFSET:\n                offsetOption.setType(OffsetOption.Type.OFFSET);\n                offsetOption.setValue(gRpcOffsetOption.getOffset());\n                break;\n            case TAIL_N:\n                offsetOption.setType(OffsetOption.Type.TAIL_N);\n                offsetOption.setValue(gRpcOffsetOption.getTailN());\n                break;\n            case TIMESTAMP:\n                offsetOption.setType(OffsetOption.Type.TIMESTAMP);\n                offsetOption.setValue(gRpcOffsetOption.getTimestamp());\n                break;\n            default:\n                throw new IllegalArgumentException(\"Unknown OffsetOption type: \" + gRpcOffsetOption.getOffsetTypeCase());\n        }\n        return offsetOption;\n    }\n\n    private static long toOffsetPolicy(apache.rocketmq.v2.OffsetOption.Policy policy) {\n        switch (policy) {\n            case LAST:\n                return OffsetOption.POLICY_LAST_VALUE;\n            case MIN:\n                return OffsetOption.POLICY_MIN_VALUE;\n            case MAX:\n                return OffsetOption.POLICY_MAX_VALUE;\n        }\n        throw new IllegalArgumentException(\"Unknown OffsetOption.Policy value: \" + policy);\n    }\n\n    public ContextStreamObserver<TelemetryCommand> telemetry(StreamObserver<TelemetryCommand> responseObserver) {\n        return new ContextStreamObserver<TelemetryCommand>() {\n            private ProxyContext proxyCtx = null;\n            @Override\n            public void onNext(ProxyContext ctx, TelemetryCommand request) {\n                this.proxyCtx = ctx;\n                try {\n                    switch (request.getCommandCase()) {\n                        case SETTINGS: {\n                            processAndWriteClientSettings(ctx, request, responseObserver);\n                            break;\n                        }\n                        case THREAD_STACK_TRACE: {\n                            reportThreadStackTrace(ctx, request.getStatus(), request.getThreadStackTrace());\n                            break;\n                        }\n                        case VERIFY_MESSAGE_RESULT: {\n                            reportVerifyMessageResult(ctx, request.getStatus(), request.getVerifyMessageResult());\n                            break;\n                        }\n                    }\n                } catch (Throwable t) {\n                    processTelemetryException(request, t, responseObserver);\n                }\n            }\n\n            @Override\n            public void onError(Throwable t) {\n                log.error(\"telemetry on error\", t);\n                handleGrpcCancel(proxyCtx, t);\n            }\n\n            @Override\n            public void onCompleted() {\n                responseObserver.onCompleted();\n            }\n        };\n    }\n\n    private static LiteSubscriptionAction toLiteAction(apache.rocketmq.v2.LiteSubscriptionAction gRpcAction) {\n        switch (gRpcAction) {\n            case PARTIAL_ADD:\n                return LiteSubscriptionAction.PARTIAL_ADD;\n            case PARTIAL_REMOVE:\n                return LiteSubscriptionAction.PARTIAL_REMOVE;\n            case COMPLETE_ADD:\n                return LiteSubscriptionAction.COMPLETE_ADD;\n            case COMPLETE_REMOVE:\n                return LiteSubscriptionAction.COMPLETE_REMOVE;\n        }\n        throw new IllegalArgumentException(\"unknown LiteSubscriptionAction: \" + gRpcAction);\n    }\n\n    private void handleGrpcCancel(ProxyContext ctx, Throwable t) {\n        final String clientId = ctx.getClientID();\n        if (StringUtils.isBlank(clientId)) {\n            return;\n        }\n        if (!(t instanceof StatusRuntimeException)) {\n            return;\n        }\n        log.warn(\"handleGrpcCancel clientId:{}\", clientId);\n        StatusRuntimeException statusException = (StatusRuntimeException) t;\n        if (io.grpc.Status.CANCELLED.getCode() == statusException.getStatus().getCode() ||\n            io.grpc.Status.UNAVAILABLE.getCode() == statusException.getStatus().getCode()) {\n            this.grpcClientSettingsManager.offlineClientLiteSubscription(ctx, clientId, null);\n        }\n    }\n\n    protected void processTelemetryException(TelemetryCommand request, Throwable t,\n        StreamObserver<TelemetryCommand> responseObserver) {\n        StatusRuntimeException exception = io.grpc.Status.INTERNAL\n            .withDescription(\"process client telemetryCommand failed. \" + t.getMessage())\n            .withCause(t)\n            .asRuntimeException();\n        if (t instanceof GrpcProxyException) {\n            GrpcProxyException proxyException = (GrpcProxyException) t;\n            if (proxyException.getCode().getNumber() < Code.INTERNAL_ERROR_VALUE &&\n                proxyException.getCode().getNumber() >= Code.BAD_REQUEST_VALUE) {\n                exception = io.grpc.Status.INVALID_ARGUMENT\n                    .withDescription(\"process client telemetryCommand failed. \" + t.getMessage())\n                    .withCause(t)\n                    .asRuntimeException();\n            }\n        }\n        if (exception.getStatus().getCode().equals(io.grpc.Status.Code.INTERNAL)) {\n            log.warn(\"process client telemetryCommand failed. request:{}\", request, t);\n        }\n        responseObserver.onError(exception);\n    }\n\n    protected void processAndWriteClientSettings(ProxyContext ctx, TelemetryCommand request,\n        StreamObserver<TelemetryCommand> responseObserver) {\n        GrpcClientChannel grpcClientChannel = null;\n        Settings settings = request.getSettings();\n        switch (settings.getPubSubCase()) {\n            case PUBLISHING:\n                for (Resource topic : settings.getPublishing().getTopicsList()) {\n                    validateTopic(topic);\n                    String topicName = topic.getName();\n                    grpcClientChannel = registerProducer(ctx, topicName);\n                    grpcClientChannel.setClientObserver(responseObserver);\n                }\n                break;\n            case SUBSCRIPTION:\n                validateConsumerGroup(settings.getSubscription().getGroup());\n                String groupName = settings.getSubscription().getGroup().getName();\n                grpcClientChannel = registerConsumer(ctx, groupName, settings.getClientType(), settings.getSubscription().getSubscriptionsList(), true);\n                grpcClientChannel.setClientObserver(responseObserver);\n                break;\n            default:\n                break;\n        }\n        if (Settings.PubSubCase.PUBSUB_NOT_SET.equals(settings.getPubSubCase())) {\n            responseObserver.onError(io.grpc.Status.INVALID_ARGUMENT\n                .withDescription(\"there is no publishing or subscription data in settings\")\n                .asRuntimeException());\n            return;\n        }\n        TelemetryCommand command = processClientSettings(ctx, request);\n        if (grpcClientChannel != null) {\n            grpcClientChannel.writeTelemetryCommand(command);\n        } else {\n            responseObserver.onNext(command);\n        }\n    }\n\n    protected TelemetryCommand processClientSettings(ProxyContext ctx, TelemetryCommand request) {\n        String clientId = ctx.getClientID();\n        grpcClientSettingsManager.updateClientSettings(ctx, clientId, request.getSettings());\n        Settings settings = grpcClientSettingsManager.getClientSettings(ctx);\n        return TelemetryCommand.newBuilder()\n            .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()))\n            .setSettings(settings)\n            .build();\n    }\n\n    protected GrpcClientChannel registerProducer(ProxyContext ctx, String topicName) {\n        String clientId = ctx.getClientID();\n        LanguageCode languageCode = LanguageCode.valueOf(ctx.getLanguage());\n\n        GrpcClientChannel channel = this.grpcChannelManager.createChannel(ctx, clientId);\n        // use topic name as producer group\n        ClientChannelInfo clientChannelInfo = new ClientChannelInfo(channel, clientId, languageCode, parseClientVersion(ctx.getClientVersion()));\n        this.messagingProcessor.registerProducer(ctx, topicName, clientChannelInfo);\n        TopicMessageType topicMessageType = this.messagingProcessor.getMetadataService().getTopicMessageType(ctx, topicName);\n        if (TopicMessageType.TRANSACTION.equals(topicMessageType)) {\n            this.messagingProcessor.addTransactionSubscription(ctx, topicName, topicName);\n        }\n        return channel;\n    }\n\n    protected GrpcClientChannel registerConsumer(ProxyContext ctx, String consumerGroup, ClientType clientType,\n        List<SubscriptionEntry> subscriptionEntryList, boolean updateSubscription) {\n        String clientId = ctx.getClientID();\n        LanguageCode languageCode = LanguageCode.valueOf(ctx.getLanguage());\n\n        GrpcClientChannel channel = this.grpcChannelManager.createChannel(ctx, clientId);\n        ClientChannelInfo clientChannelInfo = new ClientChannelInfo(channel, clientId, languageCode, parseClientVersion(ctx.getClientVersion()));\n\n        this.messagingProcessor.registerConsumer(\n            ctx,\n            consumerGroup,\n            clientChannelInfo,\n            this.buildConsumeType(clientType),\n            this.buildMessageModel(clientType),\n            ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET,\n            this.buildSubscriptionDataSet(subscriptionEntryList),\n            updateSubscription\n        );\n        return channel;\n    }\n\n    private int parseClientVersion(String clientVersionStr) {\n        int clientVersion = MQVersion.CURRENT_VERSION;\n        if (!StringUtils.isEmpty(clientVersionStr)) {\n            try {\n                String tmp = StringUtils.upperCase(clientVersionStr);\n                clientVersion = MQVersion.Version.valueOf(tmp).ordinal();\n            } catch (Exception ignored) {\n            }\n        }\n        return clientVersion;\n    }\n\n    protected void reportThreadStackTrace(ProxyContext ctx, Status status, ThreadStackTrace request) {\n        String nonce = request.getNonce();\n        String threadStack = request.getThreadStackTrace();\n        CompletableFuture<ProxyRelayResult<ConsumerRunningInfo>> responseFuture = this.grpcChannelManager.getAndRemoveResponseFuture(nonce);\n        if (responseFuture != null) {\n            try {\n                if (status.getCode().equals(Code.OK)) {\n                    ConsumerRunningInfo runningInfo = new ConsumerRunningInfo();\n                    runningInfo.setJstack(threadStack);\n                    responseFuture.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, \"\", runningInfo));\n                } else if (status.getCode().equals(Code.VERIFY_FIFO_MESSAGE_UNSUPPORTED)) {\n                    responseFuture.complete(new ProxyRelayResult<>(ResponseCode.NO_PERMISSION, \"forbidden to verify message\", null));\n                } else {\n                    responseFuture.complete(new ProxyRelayResult<>(ResponseCode.SYSTEM_ERROR, \"verify message failed\", null));\n                }\n            } catch (Throwable t) {\n                responseFuture.completeExceptionally(t);\n            }\n        }\n    }\n\n    protected void reportVerifyMessageResult(ProxyContext ctx, Status status, VerifyMessageResult request) {\n        String nonce = request.getNonce();\n        CompletableFuture<ProxyRelayResult<ConsumeMessageDirectlyResult>> responseFuture = this.grpcChannelManager.getAndRemoveResponseFuture(nonce);\n        if (responseFuture != null) {\n            try {\n                ConsumeMessageDirectlyResult result = this.buildConsumeMessageDirectlyResult(status, request);\n                responseFuture.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, \"\", result));\n            } catch (Throwable t) {\n                responseFuture.completeExceptionally(t);\n            }\n        }\n    }\n\n    protected ConsumeMessageDirectlyResult buildConsumeMessageDirectlyResult(Status status,\n        VerifyMessageResult request) {\n        ConsumeMessageDirectlyResult consumeMessageDirectlyResult = new ConsumeMessageDirectlyResult();\n        switch (status.getCode().getNumber()) {\n            case Code.OK_VALUE: {\n                consumeMessageDirectlyResult.setConsumeResult(CMResult.CR_SUCCESS);\n                break;\n            }\n            case Code.FAILED_TO_CONSUME_MESSAGE_VALUE: {\n                consumeMessageDirectlyResult.setConsumeResult(CMResult.CR_LATER);\n                break;\n            }\n            case Code.MESSAGE_CORRUPTED_VALUE: {\n                consumeMessageDirectlyResult.setConsumeResult(CMResult.CR_RETURN_NULL);\n                break;\n            }\n        }\n        consumeMessageDirectlyResult.setRemark(\"from gRPC client\");\n        return consumeMessageDirectlyResult;\n    }\n\n    protected ConsumeType buildConsumeType(ClientType clientType) {\n        switch (clientType) {\n            case SIMPLE_CONSUMER:\n                return ConsumeType.CONSUME_ACTIVELY;\n            case PUSH_CONSUMER:\n            case LITE_PUSH_CONSUMER:\n                return ConsumeType.CONSUME_PASSIVELY;\n            default:\n                throw new IllegalArgumentException(\"Client type is not consumer, type: \" + clientType);\n        }\n    }\n\n    protected MessageModel buildMessageModel(ClientType clientType) {\n        if (clientType == ClientType.LITE_PUSH_CONSUMER) {\n            return MessageModel.LITE_SELECTIVE;\n        }\n        return MessageModel.CLUSTERING;\n    }\n\n    protected Set<SubscriptionData> buildSubscriptionDataSet(List<SubscriptionEntry> subscriptionEntryList) {\n        Set<SubscriptionData> subscriptionDataSet = new HashSet<>();\n        for (SubscriptionEntry sub : subscriptionEntryList) {\n            String topicName = sub.getTopic().getName();\n            FilterExpression filterExpression = sub.getExpression();\n            subscriptionDataSet.add(buildSubscriptionData(topicName, filterExpression));\n        }\n        return subscriptionDataSet;\n    }\n\n    protected SubscriptionData buildSubscriptionData(String topicName, FilterExpression filterExpression) {\n        String expression = filterExpression.getExpression();\n        String expressionType = GrpcConverter.getInstance().buildExpressionType(filterExpression.getType());\n        try {\n            return FilterAPI.build(topicName, expression, expressionType);\n        } catch (Exception e) {\n            throw new GrpcProxyException(Code.ILLEGAL_FILTER_EXPRESSION, \"expression format is not correct\", e);\n        }\n    }\n\n    protected class ConsumerIdsChangeListenerImpl implements ConsumerIdsChangeListener {\n\n        @Override\n        public void handle(ConsumerGroupEvent event, String group, Object... args) {\n            switch (event) {\n                case CLIENT_UNREGISTER:\n                    processClientUnregister(group, args);\n                    break;\n                case REGISTER:\n                    processClientRegister(group, args);\n                    break;\n                default:\n                    break;\n            }\n        }\n\n        protected void processClientUnregister(String group, Object... args) {\n            if (args == null || args.length < 1) {\n                return;\n            }\n            if (args[0] instanceof ClientChannelInfo) {\n                ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0];\n                if (ChannelHelper.isRemote(clientChannelInfo.getChannel())) {\n                    return;\n                }\n                GrpcClientChannel removedChannel = grpcChannelManager.removeChannel(clientChannelInfo.getClientId());\n                log.info(\"remove grpc channel when client unregister. group:{}, clientChannelInfo:{}, removed:{}\",\n                    group, clientChannelInfo, removedChannel != null);\n            }\n        }\n\n        protected void processClientRegister(String group, Object... args) {\n            if (args == null || args.length < 2) {\n                return;\n            }\n            if (args[1] instanceof ClientChannelInfo) {\n                ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[1];\n                Channel channel = clientChannelInfo.getChannel();\n                if (ChannelHelper.isRemote(channel)) {\n                    // save settings from channel sync from other proxy\n                    Settings settings = GrpcClientChannel.parseChannelExtendAttribute(channel);\n                    log.debug(\"save client settings sync from other proxy. group:{}, channelInfo:{}, settings:{}\", group, clientChannelInfo, settings);\n                    if (settings == null) {\n                        return;\n                    }\n                    grpcClientSettingsManager.updateClientSettings(\n                        ProxyContext.createForInner(this.getClass()),\n                        clientChannelInfo.getClientId(),\n                        settings\n                    );\n                }\n            }\n        }\n\n        @Override\n        public void shutdown() {\n\n        }\n    }\n\n    protected class ProducerChangeListenerImpl implements ProducerChangeListener {\n\n        @Override\n        public void handle(ProducerGroupEvent event, String group, ClientChannelInfo clientChannelInfo) {\n            if (event == ProducerGroupEvent.CLIENT_UNREGISTER) {\n                grpcChannelManager.removeChannel(clientChannelInfo.getClientId());\n                grpcClientSettingsManager.removeAndGetRawClientSettings(clientChannelInfo.getClientId());\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcClientSettingsManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2.common;\n\nimport apache.rocketmq.v2.Address;\nimport apache.rocketmq.v2.AddressScheme;\nimport apache.rocketmq.v2.ClientType;\nimport apache.rocketmq.v2.CustomizedBackoff;\nimport apache.rocketmq.v2.Endpoints;\nimport apache.rocketmq.v2.ExponentialBackoff;\nimport apache.rocketmq.v2.Metric;\nimport apache.rocketmq.v2.Settings;\nimport com.google.protobuf.Duration;\nimport com.google.protobuf.util.Durations;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.TimeUnit;\nimport java.util.stream.Collectors;\nimport org.apache.rocketmq.broker.client.ConsumerGroupInfo;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.lite.LiteSubscriptionAction;\nimport org.apache.rocketmq.common.lite.LiteSubscriptionDTO;\nimport org.apache.rocketmq.common.utils.StartAndShutdown;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.MetricCollectorMode;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.remoting.protocol.subscription.CustomizedRetryPolicy;\nimport org.apache.rocketmq.remoting.protocol.subscription.ExponentialRetryPolicy;\nimport org.apache.rocketmq.remoting.protocol.subscription.GroupRetryPolicy;\nimport org.apache.rocketmq.remoting.protocol.subscription.GroupRetryPolicyType;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\n\npublic class GrpcClientSettingsManager extends ServiceThread implements StartAndShutdown {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    protected static final Map<String, Settings> CLIENT_SETTINGS_MAP = new ConcurrentHashMap<>();\n\n    private final MessagingProcessor messagingProcessor;\n\n    public GrpcClientSettingsManager(MessagingProcessor messagingProcessor) {\n        this.messagingProcessor = messagingProcessor;\n    }\n\n    public Settings getRawClientSettings(String clientId) {\n        return CLIENT_SETTINGS_MAP.get(clientId);\n    }\n\n    public Settings getClientSettings(ProxyContext ctx) {\n        String clientId = ctx.getClientID();\n        Settings settings = getRawClientSettings(clientId);\n        if (settings == null) {\n            return null;\n        }\n        if (settings.hasPublishing()) {\n            settings = mergeProducerData(settings);\n        } else if (settings.hasSubscription()) {\n            settings = mergeSubscriptionData(ctx, settings, settings.getSubscription().getGroup().getName());\n        }\n        return mergeMetric(settings);\n    }\n\n    protected static Settings mergeProducerData(Settings settings) {\n        ProxyConfig config = ConfigurationManager.getProxyConfig();\n        Settings.Builder builder = settings.toBuilder();\n\n        builder.getBackoffPolicyBuilder()\n            .setMaxAttempts(config.getGrpcClientProducerMaxAttempts())\n            .setExponentialBackoff(ExponentialBackoff.newBuilder()\n                .setInitial(Durations.fromMillis(config.getGrpcClientProducerBackoffInitialMillis()))\n                .setMax(Durations.fromMillis(config.getGrpcClientProducerBackoffMaxMillis()))\n                .setMultiplier(config.getGrpcClientProducerBackoffMultiplier())\n                .build());\n\n        builder.getPublishingBuilder()\n            .setValidateMessageType(config.isEnableTopicMessageTypeCheck())\n            .setMaxBodySize(config.getMaxMessageSize());\n        return builder.build();\n    }\n\n    protected Settings mergeSubscriptionData(ProxyContext ctx, Settings settings, String consumerGroup) {\n        SubscriptionGroupConfig config = this.messagingProcessor.getSubscriptionGroupConfig(ctx, consumerGroup);\n        if (config == null) {\n            return settings;\n        }\n\n        return mergeSubscriptionData(settings, config);\n    }\n\n    protected Settings mergeMetric(Settings settings) {\n        // Construct metric according to the proxy config\n        final ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();\n        final MetricCollectorMode metricCollectorMode =\n            MetricCollectorMode.getEnumByString(proxyConfig.getMetricCollectorMode());\n        final String metricCollectorAddress = proxyConfig.getMetricCollectorAddress();\n        final Metric.Builder metricBuilder = Metric.newBuilder();\n        switch (metricCollectorMode) {\n            case ON:\n                final String[] split = metricCollectorAddress.split(\":\");\n                final String host = split[0];\n                final int port = Integer.parseInt(split[1]);\n                Address address = Address.newBuilder().setHost(host).setPort(port).build();\n                final Endpoints endpoints = Endpoints.newBuilder().setScheme(AddressScheme.IPv4)\n                    .addAddresses(address).build();\n                metricBuilder.setOn(true).setEndpoints(endpoints);\n                break;\n            case PROXY:\n                metricBuilder.setOn(true).setEndpoints(settings.getAccessPoint());\n                break;\n            case OFF:\n            default:\n                metricBuilder.setOn(false);\n                break;\n        }\n        Metric metric = metricBuilder.build();\n        return settings.toBuilder().setMetric(metric).build();\n    }\n\n    protected static Settings mergeSubscriptionData(Settings settings, SubscriptionGroupConfig groupConfig) {\n        Settings.Builder resultSettingsBuilder = settings.toBuilder();\n        ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();\n\n        resultSettingsBuilder.getSubscriptionBuilder()\n            .setReceiveBatchSize(proxyConfig.getGrpcClientConsumerLongPollingBatchSize())\n            .setLongPollingTimeout(Durations.fromMillis(proxyConfig.getGrpcClientConsumerMaxLongPollingTimeoutMillis()))\n            .setFifo(groupConfig.isConsumeMessageOrderly())\n            // client-side lite subscription quota limit\n            .setLiteSubscriptionQuota(groupConfig.getLiteSubClientQuota())\n            .setMaxLiteTopicSize(proxyConfig.getMaxLiteTopicSize());\n\n        resultSettingsBuilder.getBackoffPolicyBuilder().setMaxAttempts(groupConfig.getRetryMaxTimes() + 1);\n\n        GroupRetryPolicy groupRetryPolicy = groupConfig.getGroupRetryPolicy();\n        if (groupRetryPolicy.getType().equals(GroupRetryPolicyType.EXPONENTIAL)) {\n            ExponentialRetryPolicy exponentialRetryPolicy = groupRetryPolicy.getExponentialRetryPolicy();\n            if (exponentialRetryPolicy == null) {\n                exponentialRetryPolicy = new ExponentialRetryPolicy();\n            }\n            resultSettingsBuilder.getBackoffPolicyBuilder().setExponentialBackoff(convertToExponentialBackoff(exponentialRetryPolicy));\n        } else {\n            CustomizedRetryPolicy customizedRetryPolicy = groupRetryPolicy.getCustomizedRetryPolicy();\n            if (customizedRetryPolicy == null) {\n                customizedRetryPolicy = new CustomizedRetryPolicy();\n            }\n            resultSettingsBuilder.getBackoffPolicyBuilder().setCustomizedBackoff(convertToCustomizedRetryPolicy(customizedRetryPolicy));\n        }\n\n        return resultSettingsBuilder.build();\n    }\n\n    protected static ExponentialBackoff convertToExponentialBackoff(ExponentialRetryPolicy retryPolicy) {\n        return ExponentialBackoff.newBuilder()\n            .setInitial(Durations.fromMillis(retryPolicy.getInitial()))\n            .setMax(Durations.fromMillis(retryPolicy.getMax()))\n            .setMultiplier(retryPolicy.getMultiplier())\n            .build();\n    }\n\n    protected static CustomizedBackoff convertToCustomizedRetryPolicy(CustomizedRetryPolicy retryPolicy) {\n        List<Duration> durationList = Arrays.stream(retryPolicy.getNext())\n            .mapToObj(Durations::fromMillis).collect(Collectors.toList());\n        return CustomizedBackoff.newBuilder()\n            .addAllNext(durationList)\n            .build();\n    }\n\n    public void updateClientSettings(ProxyContext ctx, String clientId, Settings settings) {\n        if (settings.hasSubscription()) {\n            settings = createDefaultConsumerSettingsBuilder().mergeFrom(settings).build();\n        }\n        CLIENT_SETTINGS_MAP.put(clientId, settings);\n    }\n\n    protected Settings.Builder createDefaultConsumerSettingsBuilder() {\n        return mergeSubscriptionData(Settings.newBuilder().getDefaultInstanceForType(), new SubscriptionGroupConfig())\n            .toBuilder();\n    }\n\n    public Settings removeAndGetRawClientSettings(String clientId) {\n        return CLIENT_SETTINGS_MAP.remove(clientId);\n    }\n\n    public Settings removeAndGetClientSettings(ProxyContext ctx) {\n        String clientId = ctx.getClientID();\n        Settings settings = this.removeAndGetRawClientSettings(clientId);\n        if (settings == null) {\n            return null;\n        }\n        if (settings.hasSubscription()) {\n            settings = mergeSubscriptionData(ctx, settings, settings.getSubscription().getGroup().getName());\n        }\n        return mergeMetric(settings);\n    }\n\n    @Override\n    public String getServiceName() {\n        return \"GrpcClientSettingsManagerCleaner\";\n    }\n\n    /**\n     * Remove all lite subscriptions when client offline.\n     * \n     * @param ctx       Proxy context\n     * @param clientId  Client identifier\n     * @param settings  Current client settings, if available\n     */\n    public void offlineClientLiteSubscription(ProxyContext ctx, String clientId, Settings settings) {\n        if (settings == null) {\n            settings = getRawClientSettings(clientId);\n        }\n        if (settings == null || ClientType.LITE_PUSH_CONSUMER != settings.getClientType()) {\n            return;\n        }\n        try {\n            String topic = settings.getSubscription().getSubscriptions(0).getTopic().getName();\n            String group = settings.getSubscription().getGroup().getName();\n            log.info(\"offlineClientLiteSubscription, topic:{}, group:{}, clientId:{}\", topic, group, clientId);\n            LiteSubscriptionDTO liteSubscriptionDTO = new LiteSubscriptionDTO()\n                .setAction(LiteSubscriptionAction.COMPLETE_REMOVE)\n                .setClientId(clientId)\n                .setGroup(group)\n                .setTopic(topic);\n            this.messagingProcessor.syncLiteSubscription(ctx, liteSubscriptionDTO, java.time.Duration.ofSeconds(2).toMillis())\n                .whenComplete((result, throwable) -> {\n                    if (throwable != null) {\n                        log.error(\"offlineClientLiteSubscription failed, topic:{}, group:{}, clientId:{}\",\n                            topic, group, clientId, throwable);\n                    }\n                });\n        } catch (Exception e) {\n            log.error(\"offlineClientLiteSubscription error, clientId:{}, settings:{}\", clientId, settings, e);\n        }\n    }\n\n    @Override\n    public void run() {\n        while (!this.isStopped()) {\n            this.waitForRunning(TimeUnit.SECONDS.toMillis(5));\n        }\n    }\n\n    @Override\n    protected void onWaitEnd() {\n        Set<String> clientIdSet = CLIENT_SETTINGS_MAP.keySet();\n        for (String clientId : clientIdSet) {\n            try {\n                CLIENT_SETTINGS_MAP.computeIfPresent(clientId, (clientIdKey, settings) -> {\n                    if (!settings.getClientType().equals(ClientType.PUSH_CONSUMER) &&\n                        !settings.getClientType().equals(ClientType.SIMPLE_CONSUMER) &&\n                        !settings.getClientType().equals(ClientType.LITE_PUSH_CONSUMER)) {\n                        return settings;\n                    }\n                    String consumerGroup = settings.getSubscription().getGroup().getName();\n                    ConsumerGroupInfo consumerGroupInfo = this.messagingProcessor.getConsumerGroupInfo(\n                        ProxyContext.createForInner(this.getClass()),\n                        consumerGroup\n                    );\n                    if (consumerGroupInfo == null || consumerGroupInfo.findChannel(clientId) == null) {\n                        log.info(\"remove unused grpc client settings. group:{}, settings:{}\", consumerGroupInfo, settings);\n                        return null;\n                    }\n                    return settings;\n                });\n            } catch (Throwable t) {\n                log.error(\"check expired grpc client settings failed. clientId:{}\", clientId, t);\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcConverter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2.common;\n\nimport apache.rocketmq.v2.Broker;\nimport apache.rocketmq.v2.DeadLetterQueue;\nimport apache.rocketmq.v2.Digest;\nimport apache.rocketmq.v2.DigestType;\nimport apache.rocketmq.v2.Encoding;\nimport apache.rocketmq.v2.FilterType;\nimport apache.rocketmq.v2.Message;\nimport apache.rocketmq.v2.MessageQueue;\nimport apache.rocketmq.v2.MessageType;\nimport apache.rocketmq.v2.Resource;\nimport apache.rocketmq.v2.SystemProperties;\nimport com.google.protobuf.ByteString;\nimport com.google.protobuf.util.Timestamps;\nimport java.net.SocketAddress;\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.common.utils.BinaryUtil;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\n\npublic class GrpcConverter {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    protected static final Object INSTANCE_CREATE_LOCK = new Object();\n    protected static volatile GrpcConverter instance;\n\n    public static GrpcConverter getInstance() {\n        if (instance == null) {\n            synchronized (INSTANCE_CREATE_LOCK) {\n                if (instance == null) {\n                    instance = new GrpcConverter();\n                }\n            }\n        }\n        return instance;\n    }\n\n    public MessageQueue buildMessageQueue(MessageExt messageExt, String brokerName) {\n        Broker broker = Broker.getDefaultInstance();\n        if (!StringUtils.isEmpty(brokerName)) {\n            broker = Broker.newBuilder()\n                .setName(brokerName)\n                .setId(0)\n                .build();\n        }\n        return MessageQueue.newBuilder()\n            .setId(messageExt.getQueueId())\n            .setTopic(Resource.newBuilder()\n                .setName(NamespaceUtil.withoutNamespace(messageExt.getTopic()))\n                .setResourceNamespace(NamespaceUtil.getNamespaceFromResource(messageExt.getTopic()))\n                .build())\n            .setBroker(broker)\n            .build();\n    }\n\n    public String buildExpressionType(FilterType filterType) {\n        switch (filterType) {\n            case SQL:\n                return ExpressionType.SQL92;\n            case TAG:\n            default:\n                return ExpressionType.TAG;\n        }\n    }\n\n    public Message buildMessage(MessageExt messageExt) {\n        Map<String, String> userProperties = buildUserAttributes(messageExt);\n        SystemProperties systemProperties = buildSystemProperties(messageExt);\n        Resource topic = buildResource(messageExt.getTopic());\n\n        return Message.newBuilder()\n            .setTopic(topic)\n            .putAllUserProperties(userProperties)\n            .setSystemProperties(systemProperties)\n            .setBody(ByteString.copyFrom(messageExt.getBody()))\n            .build();\n    }\n\n    protected Map<String, String> buildUserAttributes(MessageExt messageExt) {\n        Map<String, String> userAttributes = new HashMap<>();\n        Map<String, String> properties = messageExt.getProperties();\n\n        for (Map.Entry<String, String> property : properties.entrySet()) {\n            if (!MessageConst.STRING_HASH_SET.contains(property.getKey())) {\n                userAttributes.put(property.getKey(), property.getValue());\n            }\n        }\n\n        return userAttributes;\n    }\n\n    protected SystemProperties buildSystemProperties(MessageExt messageExt) {\n        SystemProperties.Builder systemPropertiesBuilder = SystemProperties.newBuilder();\n\n        // tag\n        String tag = messageExt.getUserProperty(MessageConst.PROPERTY_TAGS);\n        if (tag != null) {\n            systemPropertiesBuilder.setTag(tag);\n        }\n\n        // keys\n        String keys = messageExt.getKeys();\n        if (keys != null) {\n            String[] keysArray = keys.split(MessageConst.KEY_SEPARATOR);\n            systemPropertiesBuilder.addAllKeys(Arrays.asList(keysArray));\n        }\n\n        // message_id\n        String uniqKey = messageExt.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX);\n\n        if (uniqKey == null) {\n            uniqKey = messageExt.getMsgId();\n        }\n\n        if (uniqKey != null) {\n            systemPropertiesBuilder.setMessageId(uniqKey);\n        }\n\n        // body_digest & body_encoding\n        String md5Result = BinaryUtil.generateMd5(messageExt.getBody());\n        Digest digest = Digest.newBuilder()\n            .setType(DigestType.MD5)\n            .setChecksum(md5Result)\n            .build();\n        systemPropertiesBuilder.setBodyDigest(digest);\n\n        if ((messageExt.getSysFlag() & MessageSysFlag.COMPRESSED_FLAG) == MessageSysFlag.COMPRESSED_FLAG) {\n            systemPropertiesBuilder.setBodyEncoding(Encoding.GZIP);\n        } else {\n            systemPropertiesBuilder.setBodyEncoding(Encoding.IDENTITY);\n        }\n\n        // message_type\n        TopicMessageType topicMessageType = TopicMessageType.parseFromMessageProperty(messageExt.getProperties());\n        systemPropertiesBuilder.setMessageType(convertToGrpcMessageType(topicMessageType));\n\n        // born_timestamp (millis)\n        long bornTimestamp = messageExt.getBornTimestamp();\n        systemPropertiesBuilder.setBornTimestamp(Timestamps.fromMillis(bornTimestamp));\n\n        // born_host\n        String bornHostString = messageExt.getProperty(MessageConst.PROPERTY_BORN_HOST);\n        if (StringUtils.isBlank(bornHostString)) {\n            bornHostString = messageExt.getBornHostString();\n        }\n        if (StringUtils.isNotBlank(bornHostString)) {\n            systemPropertiesBuilder.setBornHost(bornHostString);\n        }\n\n        // store_timestamp (millis)\n        long storeTimestamp = messageExt.getStoreTimestamp();\n        systemPropertiesBuilder.setStoreTimestamp(Timestamps.fromMillis(storeTimestamp));\n\n        // store_host\n        SocketAddress storeHost = messageExt.getStoreHost();\n        if (storeHost != null) {\n            systemPropertiesBuilder.setStoreHost(NetworkUtil.socketAddress2String(storeHost));\n        }\n\n        // delivery_timestamp\n        String deliverMsString;\n        long deliverMs;\n        if (messageExt.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC) != null) {\n            long delayMs = TimeUnit.SECONDS.toMillis(Long.parseLong(messageExt.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC)));\n            deliverMs = System.currentTimeMillis() + delayMs;\n            systemPropertiesBuilder.setDeliveryTimestamp(Timestamps.fromMillis(deliverMs));\n        } else {\n            deliverMsString = messageExt.getProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS);\n            if (deliverMsString != null) {\n                deliverMs = Long.parseLong(deliverMsString);\n                systemPropertiesBuilder.setDeliveryTimestamp(Timestamps.fromMillis(deliverMs));\n            }\n        }\n\n        // priority\n        int priority = messageExt.getPriority();\n        if (priority >= 0) {\n            systemPropertiesBuilder.setPriority(priority);\n        }\n\n        // sharding key\n        String shardingKey = messageExt.getProperty(MessageConst.PROPERTY_SHARDING_KEY);\n        if (shardingKey != null) {\n            systemPropertiesBuilder.setMessageGroup(shardingKey);\n        }\n\n        // lite topic\n        String liteTopic = messageExt.getProperty(MessageConst.PROPERTY_LITE_TOPIC);\n        if (liteTopic != null) {\n            systemPropertiesBuilder.setLiteTopic(liteTopic);\n        }\n\n        // receipt_handle && invisible_period\n        String handle = messageExt.getProperty(MessageConst.PROPERTY_POP_CK);\n        if (handle != null) {\n            systemPropertiesBuilder.setReceiptHandle(handle);\n        }\n\n        // partition_id\n        systemPropertiesBuilder.setQueueId(messageExt.getQueueId());\n\n        // partition_offset\n        systemPropertiesBuilder.setQueueOffset(messageExt.getQueueOffset());\n\n        // delivery_attempt\n        systemPropertiesBuilder.setDeliveryAttempt(messageExt.getReconsumeTimes() + 1);\n\n        // trace context\n        String traceContext = messageExt.getProperty(MessageConst.PROPERTY_TRACE_CONTEXT);\n        if (traceContext != null) {\n            systemPropertiesBuilder.setTraceContext(traceContext);\n        }\n\n        String dlqOriginTopic = messageExt.getProperty(MessageConst.PROPERTY_DLQ_ORIGIN_TOPIC);\n        String dlqOriginMessageId = messageExt.getProperty(MessageConst.PROPERTY_DLQ_ORIGIN_MESSAGE_ID);\n        if (dlqOriginTopic != null && dlqOriginMessageId != null) {\n            DeadLetterQueue dlq = DeadLetterQueue.newBuilder()\n                .setTopic(dlqOriginTopic)\n                .setMessageId(dlqOriginMessageId)\n                .build();\n            systemPropertiesBuilder.setDeadLetterQueue(dlq);\n        }\n        return systemPropertiesBuilder.build();\n    }\n\n    public Resource buildResource(String resourceNameWithNamespace) {\n        return Resource.newBuilder()\n            .setResourceNamespace(NamespaceUtil.getNamespaceFromResource(resourceNameWithNamespace))\n            .setName(NamespaceUtil.withoutNamespace(resourceNameWithNamespace))\n            .build();\n    }\n\n    protected MessageType convertToGrpcMessageType(TopicMessageType topicMessageType) {\n        switch (topicMessageType) {\n            case TRANSACTION:\n                return MessageType.TRANSACTION;\n            case DELAY:\n                return MessageType.DELAY;\n            case FIFO:\n                return MessageType.FIFO;\n            case PRIORITY:\n                return MessageType.PRIORITY;\n            case LITE:\n                return MessageType.LITE;\n            case NORMAL:\n                return MessageType.NORMAL;\n            case UNSPECIFIED:\n            default:\n                return MessageType.NORMAL;\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcProxyException.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.grpc.v2.common;\n\nimport apache.rocketmq.v2.Code;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\n\npublic class GrpcProxyException extends RuntimeException {\n\n    private ProxyException proxyException;\n    private Code code;\n\n    protected static final Map<ProxyExceptionCode, Code> CODE_MAPPING = new ConcurrentHashMap<>();\n\n    static {\n        CODE_MAPPING.put(ProxyExceptionCode.INVALID_BROKER_NAME, Code.BAD_REQUEST);\n        CODE_MAPPING.put(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, Code.INVALID_RECEIPT_HANDLE);\n        CODE_MAPPING.put(ProxyExceptionCode.FORBIDDEN, Code.FORBIDDEN);\n        CODE_MAPPING.put(ProxyExceptionCode.INTERNAL_SERVER_ERROR, Code.INTERNAL_SERVER_ERROR);\n        CODE_MAPPING.put(ProxyExceptionCode.MESSAGE_PROPERTY_CONFLICT_WITH_TYPE, Code.MESSAGE_PROPERTY_CONFLICT_WITH_TYPE);\n    }\n\n    public GrpcProxyException(Code code, String message) {\n        super(message);\n        this.code = code;\n    }\n\n    public GrpcProxyException(Code code, String message, Throwable t) {\n        super(message, t);\n        this.code = code;\n    }\n\n    public GrpcProxyException(ProxyException proxyException) {\n        super(proxyException);\n        this.proxyException = proxyException;\n    }\n\n    public Code getCode() {\n        if (this.code != null) {\n            return this.code;\n        }\n        if (this.proxyException != null) {\n            return CODE_MAPPING.getOrDefault(this.proxyException.getCode(), Code.INTERNAL_SERVER_ERROR);\n        }\n        return Code.INTERNAL_SERVER_ERROR;\n    }\n\n    public ProxyException getProxyException() {\n        return proxyException;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcValidator.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2.common;\n\nimport apache.rocketmq.v2.Code;\nimport apache.rocketmq.v2.Resource;\nimport com.google.common.base.CharMatcher;\nimport java.nio.charset.StandardCharsets;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.Validators;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\n\npublic class GrpcValidator {\n    protected static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    protected static final Object INSTANCE_CREATE_LOCK = new Object();\n    protected static volatile GrpcValidator instance;\n\n    public static GrpcValidator getInstance() {\n        if (instance == null) {\n            synchronized (INSTANCE_CREATE_LOCK) {\n                if (instance == null) {\n                    instance = new GrpcValidator();\n                }\n            }\n        }\n        return instance;\n    }\n\n    public void validateTopic(Resource topic) {\n        validateTopic(topic.getName());\n    }\n\n    public void validateTopic(String topicName) {\n        try {\n            Validators.checkTopic(topicName);\n        } catch (MQClientException mqClientException) {\n            throw new GrpcProxyException(Code.ILLEGAL_TOPIC, mqClientException.getErrorMessage());\n        }\n        if (TopicValidator.isSystemTopic(topicName)) {\n            throw new GrpcProxyException(Code.ILLEGAL_TOPIC, \"cannot access system topic\");\n        }\n    }\n\n    public void validateConsumerGroup(Resource consumerGroup) {\n        validateConsumerGroup(consumerGroup.getName());\n    }\n\n    public void validateConsumerGroup(String consumerGroupName) {\n        try {\n            Validators.checkGroup(consumerGroupName);\n        } catch (MQClientException mqClientException) {\n            throw new GrpcProxyException(Code.ILLEGAL_CONSUMER_GROUP, mqClientException.getErrorMessage());\n        }\n        if (MixAll.isSysConsumerGroup(consumerGroupName)) {\n            throw new GrpcProxyException(Code.ILLEGAL_CONSUMER_GROUP, \"cannot use system consumer group\");\n        }\n    }\n\n    public void validateTopicAndConsumerGroup(Resource topic, Resource consumerGroup) {\n        validateTopic(topic);\n        validateConsumerGroup(consumerGroup);\n    }\n\n    public void validateInvisibleTime(long invisibleTime) {\n        validateInvisibleTime(invisibleTime, 0);\n    }\n\n    public void validateInvisibleTime(long invisibleTime, long minInvisibleTime) {\n        if (invisibleTime < minInvisibleTime) {\n            throw new GrpcProxyException(Code.ILLEGAL_INVISIBLE_TIME, \"the invisibleTime is too small. min is \" + minInvisibleTime);\n        }\n        long maxInvisibleTime = ConfigurationManager.getProxyConfig().getMaxInvisibleTimeMills();\n        if (maxInvisibleTime <= 0) {\n            return;\n        }\n        if (invisibleTime > maxInvisibleTime) {\n            throw new GrpcProxyException(Code.ILLEGAL_INVISIBLE_TIME, \"the invisibleTime is too large. max is \" + maxInvisibleTime);\n        }\n    }\n\n    public void validateTag(String tag) {\n        if (StringUtils.isNotEmpty(tag)) {\n            if (StringUtils.isBlank(tag)) {\n                throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_TAG, \"tag cannot be the char sequence of whitespace\");\n            }\n            if (tag.contains(\"|\")) {\n                throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_TAG, \"tag cannot contain '|'\");\n            }\n            if (containControlCharacter(tag)) {\n                throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_TAG, \"tag cannot contain control character\");\n            }\n        }\n    }\n\n    public boolean containControlCharacter(String data) {\n        for (int i = 0; i < data.length(); i++) {\n            if (CharMatcher.javaIsoControl().matches(data.charAt(i))) {\n                return true;\n            }\n        }\n        return false;\n    }\n\n    public void validateLiteTopic(String liteTopic) {\n        if (StringUtils.isBlank(liteTopic)) {\n            throw new GrpcProxyException(Code.ILLEGAL_LITE_TOPIC, \"lite topic cannot be the char sequence of whitespace\");\n        }\n        int maxSize = ConfigurationManager.getProxyConfig().getMaxLiteTopicSize();\n        if (liteTopic.getBytes(StandardCharsets.UTF_8).length > maxSize) {\n            throw new GrpcProxyException(Code.ILLEGAL_LITE_TOPIC, \"lite topic exceed the max size \" + maxSize);\n        }\n        if (!isValidLiteTopic(liteTopic)) {\n            throw new GrpcProxyException(Code.ILLEGAL_LITE_TOPIC, \"lite topic can only contain alphanumeric characters, hyphens(-), and underscores(_)\");\n        }\n    }\n\n    /**\n     * alternative for regex \"^[a-zA-Z0-9_-]+$\"\n     */\n    private boolean isValidLiteTopic(String liteTopic) {\n        for (int i = 0; i < liteTopic.length(); i++) {\n            char c = liteTopic.charAt(i);\n            if (!(c >= 'a' && c <= 'z') &&\n                !(c >= 'A' && c <= 'Z') &&\n                !(c >= '0' && c <= '9') &&\n                c != '-' && c != '_') {\n                return false;\n            }\n        }\n        return true;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2.common;\n\nimport apache.rocketmq.v2.Code;\nimport apache.rocketmq.v2.Status;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.auth.authentication.exception.AuthenticationException;\nimport org.apache.rocketmq.auth.authorization.exception.AuthorizationException;\nimport org.apache.rocketmq.client.common.ClientErrorCode;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.common.utils.ExceptionUtils;\nimport org.apache.rocketmq.proxy.service.route.TopicRouteHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\n\npublic class ResponseBuilder {\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    protected static final Map<Integer, Code> RESPONSE_CODE_MAPPING = new ConcurrentHashMap<>();\n\n    protected static final Object INSTANCE_CREATE_LOCK = new Object();\n    protected static volatile ResponseBuilder instance;\n\n    static {\n        RESPONSE_CODE_MAPPING.put(ResponseCode.SUCCESS, Code.OK);\n        RESPONSE_CODE_MAPPING.put(ResponseCode.SYSTEM_BUSY, Code.TOO_MANY_REQUESTS);\n        RESPONSE_CODE_MAPPING.put(ResponseCode.REQUEST_CODE_NOT_SUPPORTED, Code.NOT_IMPLEMENTED);\n        RESPONSE_CODE_MAPPING.put(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST, Code.CONSUMER_GROUP_NOT_FOUND);\n        RESPONSE_CODE_MAPPING.put(ResponseCode.LMQ_QUOTA_EXCEEDED, Code.LITE_TOPIC_QUOTA_EXCEEDED);\n        RESPONSE_CODE_MAPPING.put(ResponseCode.LITE_SUBSCRIPTION_QUOTA_EXCEEDED, Code.LITE_SUBSCRIPTION_QUOTA_EXCEEDED);\n        RESPONSE_CODE_MAPPING.put(ClientErrorCode.ACCESS_BROKER_TIMEOUT, Code.PROXY_TIMEOUT);\n    }\n\n    public static ResponseBuilder getInstance() {\n        if (instance == null) {\n            synchronized (INSTANCE_CREATE_LOCK) {\n                if (instance == null) {\n                    instance = new ResponseBuilder();\n                }\n            }\n        }\n        return instance;\n    }\n\n    public Status buildStatus(Throwable t) {\n        t = ExceptionUtils.getRealException(t);\n\n        if (t instanceof ProxyException) {\n            t = new GrpcProxyException((ProxyException) t);\n        }\n        if (t instanceof GrpcProxyException) {\n            GrpcProxyException grpcProxyException = (GrpcProxyException) t;\n            return buildStatus(grpcProxyException.getCode(), grpcProxyException.getMessage());\n        }\n        if (TopicRouteHelper.isTopicNotExistError(t)) {\n            return buildStatus(Code.TOPIC_NOT_FOUND, t.getMessage());\n        }\n        if (t instanceof MQBrokerException) {\n            MQBrokerException mqBrokerException = (MQBrokerException) t;\n            return buildStatus(buildCode(mqBrokerException.getResponseCode()), mqBrokerException.getErrorMessage());\n        }\n        if (t instanceof MQClientException) {\n            MQClientException mqClientException = (MQClientException) t;\n            return buildStatus(buildCode(mqClientException.getResponseCode()), mqClientException.getErrorMessage());\n        }\n        if (t instanceof RemotingTimeoutException) {\n            return buildStatus(Code.PROXY_TIMEOUT, t.getMessage());\n        }\n        if (t instanceof AuthenticationException || t instanceof AuthorizationException) {\n            return buildStatus(Code.UNAUTHORIZED, t.getMessage());\n        }\n\n        log.error(\"internal server error\", t);\n        return buildStatus(Code.INTERNAL_SERVER_ERROR, ExceptionUtils.getErrorDetailMessage(t));\n    }\n\n    public Status buildStatus(Code code, String message) {\n        return Status.newBuilder()\n            .setCode(code)\n            .setMessage(message != null ? message : code.name())\n            .build();\n    }\n\n    public Status buildStatus(int remotingResponseCode, String remark) {\n        String message = remark;\n        if (message == null) {\n            message = String.valueOf(remotingResponseCode);\n        }\n        return Status.newBuilder()\n            .setCode(buildCode(remotingResponseCode))\n            .setMessage(message)\n            .build();\n    }\n\n    public Code buildCode(int remotingResponseCode) {\n        return RESPONSE_CODE_MAPPING.getOrDefault(remotingResponseCode, Code.INTERNAL_SERVER_ERROR);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseWriter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2.common;\n\nimport io.grpc.Status;\nimport io.grpc.StatusRuntimeException;\nimport io.grpc.stub.ServerCallStreamObserver;\nimport io.grpc.stub.StreamObserver;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class ResponseWriter {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    protected static final Object INSTANCE_CREATE_LOCK = new Object();\n    protected static volatile ResponseWriter instance;\n\n    public static ResponseWriter getInstance() {\n        if (instance == null) {\n            synchronized (INSTANCE_CREATE_LOCK) {\n                if (instance == null) {\n                    instance = new ResponseWriter();\n                }\n            }\n        }\n        return instance;\n    }\n\n    public <T> void write(StreamObserver<T> observer, final T response) {\n        if (writeResponse(observer, response)) {\n            observer.onCompleted();\n        }\n    }\n\n    public <T> boolean writeResponse(StreamObserver<T> observer, final T response) {\n        if (null == response) {\n            return false;\n        }\n        log.debug(\"start to write response. response: {}\", response);\n        if (isCancelled(observer)) {\n            log.warn(\"client has cancelled the request. response to write: {}\", response);\n            return false;\n        }\n        try {\n            observer.onNext(response);\n        } catch (StatusRuntimeException statusRuntimeException) {\n            if (Status.CANCELLED.equals(statusRuntimeException.getStatus())) {\n                log.warn(\"client has cancelled the request. response to write: {}\", response);\n                return false;\n            }\n            throw statusRuntimeException;\n        }\n        return true;\n    }\n\n    public <T> boolean isCancelled(StreamObserver<T> observer) {\n        if (observer instanceof ServerCallStreamObserver) {\n            final ServerCallStreamObserver<T> serverCallStreamObserver = (ServerCallStreamObserver<T>) observer;\n            return serverCallStreamObserver.isCancelled();\n        }\n        return false;\n    }\n}\n\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.grpc.v2.consumer;\n\nimport apache.rocketmq.v2.AckMessageEntry;\nimport apache.rocketmq.v2.AckMessageRequest;\nimport apache.rocketmq.v2.AckMessageResponse;\nimport apache.rocketmq.v2.AckMessageResultEntry;\nimport apache.rocketmq.v2.Code;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.AckStatus;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.proxy.common.MessageReceiptHandle;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.grpc.v2.AbstractMessagingActivity;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcClientChannel;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder;\nimport org.apache.rocketmq.proxy.processor.BatchAckResult;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;\n\npublic class AckMessageActivity extends AbstractMessagingActivity {\n\n    public AckMessageActivity(MessagingProcessor messagingProcessor, GrpcClientSettingsManager grpcClientSettingsManager,\n        GrpcChannelManager grpcChannelManager) {\n        super(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n    }\n\n    public CompletableFuture<AckMessageResponse> ackMessage(ProxyContext ctx, AckMessageRequest request) {\n        CompletableFuture<AckMessageResponse> future = new CompletableFuture<>();\n\n        try {\n            validateTopicAndConsumerGroup(request.getTopic(), request.getGroup());\n            String group = request.getGroup().getName();\n            String topic = request.getTopic().getName();\n            boolean isBatchAck = ConfigurationManager.getProxyConfig().isEnableBatchAck()\n                && !request.getEntries(0).hasLiteTopic();\n            if (isBatchAck) {\n                future = ackMessageInBatch(ctx, group, topic, request);\n            } else {\n                future = ackMessageOneByOne(ctx, group, topic, request);\n            }\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    protected CompletableFuture<AckMessageResponse> ackMessageInBatch(ProxyContext ctx, String group, String topic, AckMessageRequest request) {\n        List<ReceiptHandleMessage> handleMessageList = new ArrayList<>(request.getEntriesCount());\n\n        for (AckMessageEntry ackMessageEntry : request.getEntriesList()) {\n            String handleString = getHandleString(ctx, group, request, ackMessageEntry);\n            handleMessageList.add(new ReceiptHandleMessage(ReceiptHandle.decode(handleString), ackMessageEntry.getMessageId()));\n        }\n        return this.messagingProcessor.batchAckMessage(ctx, handleMessageList, group, topic)\n            .thenApply(batchAckResultList -> {\n                AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder();\n                Set<Code> responseCodes = new HashSet<>();\n                for (BatchAckResult batchAckResult : batchAckResultList) {\n                    AckMessageResultEntry entry = convertToAckMessageResultEntry(batchAckResult);\n                    responseBuilder.addEntries(entry);\n                    responseCodes.add(entry.getStatus().getCode());\n                }\n                setAckResponseStatus(responseBuilder, responseCodes);\n                return responseBuilder.build();\n            });\n    }\n\n    protected AckMessageResultEntry convertToAckMessageResultEntry(BatchAckResult batchAckResult) {\n        ReceiptHandleMessage handleMessage = batchAckResult.getReceiptHandleMessage();\n        AckMessageResultEntry.Builder resultBuilder = AckMessageResultEntry.newBuilder()\n            .setMessageId(handleMessage.getMessageId())\n            .setReceiptHandle(handleMessage.getReceiptHandle().getReceiptHandle());\n        if (batchAckResult.getProxyException() != null) {\n            resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(batchAckResult.getProxyException()));\n        } else {\n            AckResult ackResult = batchAckResult.getAckResult();\n            if (AckStatus.OK.equals(ackResult.getStatus())) {\n                resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()));\n            } else {\n                resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, \"ack failed: status is abnormal\"));\n            }\n        }\n        return resultBuilder.build();\n    }\n\n    protected CompletableFuture<AckMessageResponse> ackMessageOneByOne(ProxyContext ctx, String group, String topic, AckMessageRequest request) {\n        CompletableFuture<AckMessageResponse> resultFuture = new CompletableFuture<>();\n        CompletableFuture<AckMessageResultEntry>[] futures = new CompletableFuture[request.getEntriesCount()];\n        for (int i = 0; i < request.getEntriesCount(); i++) {\n            futures[i] = processAckMessage(ctx, group, topic, request, request.getEntries(i));\n        }\n        CompletableFuture.allOf(futures).whenComplete((val, throwable) -> {\n            if (throwable != null) {\n                resultFuture.completeExceptionally(throwable);\n                return;\n            }\n\n            Set<Code> responseCodes = new HashSet<>();\n            List<AckMessageResultEntry> entryList = new ArrayList<>();\n            for (CompletableFuture<AckMessageResultEntry> entryFuture : futures) {\n                AckMessageResultEntry entryResult = entryFuture.join();\n                responseCodes.add(entryResult.getStatus().getCode());\n                entryList.add(entryResult);\n            }\n            AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder()\n                .addAllEntries(entryList);\n            setAckResponseStatus(responseBuilder, responseCodes);\n            resultFuture.complete(responseBuilder.build());\n        });\n        return resultFuture;\n    }\n\n    protected CompletableFuture<AckMessageResultEntry> processAckMessage(ProxyContext ctx, String group, String topic, AckMessageRequest request,\n        AckMessageEntry ackMessageEntry) {\n        CompletableFuture<AckMessageResultEntry> future = new CompletableFuture<>();\n\n        try {\n            String handleString = this.getHandleString(ctx, group, request, ackMessageEntry);\n            CompletableFuture<AckResult> ackResultFuture = this.messagingProcessor.ackMessage(\n                ctx,\n                ReceiptHandle.decode(handleString),\n                ackMessageEntry.getMessageId(),\n                group,\n                topic,\n                ackMessageEntry.hasLiteTopic() ? ackMessageEntry.getLiteTopic() : null\n            );\n            ackResultFuture.thenAccept(result -> {\n                future.complete(convertToAckMessageResultEntry(ctx, ackMessageEntry, result));\n            }).exceptionally(t -> {\n                future.complete(convertToAckMessageResultEntry(ctx, ackMessageEntry, t));\n                return null;\n            });\n        } catch (Throwable t) {\n            future.complete(convertToAckMessageResultEntry(ctx, ackMessageEntry, t));\n        }\n        return future;\n    }\n\n    protected AckMessageResultEntry convertToAckMessageResultEntry(ProxyContext ctx, AckMessageEntry ackMessageEntry, Throwable throwable) {\n        return AckMessageResultEntry.newBuilder()\n            .setStatus(ResponseBuilder.getInstance().buildStatus(throwable))\n            .setMessageId(ackMessageEntry.getMessageId())\n            .setReceiptHandle(ackMessageEntry.getReceiptHandle())\n            .build();\n    }\n\n    protected AckMessageResultEntry convertToAckMessageResultEntry(ProxyContext ctx, AckMessageEntry ackMessageEntry,\n        AckResult ackResult) {\n        if (AckStatus.OK.equals(ackResult.getStatus())) {\n            return AckMessageResultEntry.newBuilder()\n                .setMessageId(ackMessageEntry.getMessageId())\n                .setReceiptHandle(ackMessageEntry.getReceiptHandle())\n                .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()))\n                .build();\n        }\n        return AckMessageResultEntry.newBuilder()\n            .setMessageId(ackMessageEntry.getMessageId())\n            .setReceiptHandle(ackMessageEntry.getReceiptHandle())\n            .setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, \"ack failed: status is abnormal\"))\n            .build();\n    }\n\n    protected void setAckResponseStatus(AckMessageResponse.Builder responseBuilder, Set<Code> responseCodes) {\n        if (responseCodes.size() > 1) {\n            responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.MULTIPLE_RESULTS, Code.MULTIPLE_RESULTS.name()));\n        } else if (responseCodes.size() == 1) {\n            Code code = responseCodes.stream().findAny().get();\n            responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(code, code.name()));\n        } else {\n            responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, \"ack message result is empty\"));\n        }\n    }\n\n    protected String getHandleString(ProxyContext ctx, String group, AckMessageRequest request, AckMessageEntry ackMessageEntry) {\n        String handleString = ackMessageEntry.getReceiptHandle();\n        GrpcClientChannel channel = grpcChannelManager.getChannel(ctx.getClientID());\n        if (channel != null) {\n            MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, channel, group, ackMessageEntry.getMessageId(), ackMessageEntry.getReceiptHandle());\n            if (messageReceiptHandle != null) {\n                handleString = messageReceiptHandle.getReceiptHandleStr();\n            }\n        }\n        return handleString;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ChangeInvisibleDurationActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.grpc.v2.consumer;\n\nimport apache.rocketmq.v2.ChangeInvisibleDurationRequest;\nimport apache.rocketmq.v2.ChangeInvisibleDurationResponse;\nimport apache.rocketmq.v2.Code;\nimport com.google.protobuf.util.Durations;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.AckStatus;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.proxy.common.MessageReceiptHandle;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.grpc.v2.AbstractMessagingActivity;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\n\npublic class ChangeInvisibleDurationActivity extends AbstractMessagingActivity {\n\n    public ChangeInvisibleDurationActivity(MessagingProcessor messagingProcessor,\n        GrpcClientSettingsManager grpcClientSettingsManager, GrpcChannelManager grpcChannelManager) {\n        super(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n    }\n\n    public CompletableFuture<ChangeInvisibleDurationResponse> changeInvisibleDuration(ProxyContext ctx,\n        ChangeInvisibleDurationRequest request) {\n        CompletableFuture<ChangeInvisibleDurationResponse> future = new CompletableFuture<>();\n\n        try {\n            validateTopicAndConsumerGroup(request.getTopic(), request.getGroup());\n            validateInvisibleTime(Durations.toMillis(request.getInvisibleDuration()));\n\n            ReceiptHandle receiptHandle = ReceiptHandle.decode(request.getReceiptHandle());\n            String group = request.getGroup().getName();\n\n            MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, grpcChannelManager.getChannel(ctx.getClientID()), group, request.getMessageId(), receiptHandle.getReceiptHandle());\n            if (messageReceiptHandle != null) {\n                receiptHandle = ReceiptHandle.decode(messageReceiptHandle.getReceiptHandleStr());\n            }\n            return this.messagingProcessor.changeInvisibleTime(\n                ctx,\n                receiptHandle,\n                request.getMessageId(),\n                group,\n                request.getTopic().getName(),\n                Durations.toMillis(request.getInvisibleDuration())\n            ).thenApply(ackResult -> convertToChangeInvisibleDurationResponse(ctx, request, ackResult));\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    protected ChangeInvisibleDurationResponse convertToChangeInvisibleDurationResponse(ProxyContext ctx,\n        ChangeInvisibleDurationRequest request, AckResult ackResult) {\n        if (AckStatus.OK.equals(ackResult.getStatus())) {\n            return ChangeInvisibleDurationResponse.newBuilder()\n                .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()))\n                .setReceiptHandle(ackResult.getExtraInfo())\n                .build();\n        }\n        return ChangeInvisibleDurationResponse.newBuilder()\n            .setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, \"changeInvisibleDuration failed: status is abnormal\"))\n            .build();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/PopMessageResultFilterImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.grpc.v2.consumer;\n\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.common.utils.FilterUtils;\nimport org.apache.rocketmq.proxy.processor.PopMessageResultFilter;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\npublic class PopMessageResultFilterImpl implements PopMessageResultFilter {\n\n    private final int maxAttempts;\n\n    public PopMessageResultFilterImpl(int maxAttempts) {\n        this.maxAttempts = maxAttempts;\n    }\n\n    @Override\n    public FilterResult filterMessage(ProxyContext ctx, String consumerGroup, SubscriptionData subscriptionData,\n        MessageExt messageExt) {\n        if (!FilterUtils.isTagMatched(subscriptionData.getTagsSet(), messageExt.getTags())) {\n            return FilterResult.NO_MATCH;\n        }\n        if (messageExt.getReconsumeTimes() >= maxAttempts) {\n            return FilterResult.TO_DLQ;\n        }\n        return FilterResult.MATCH;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.grpc.v2.consumer;\n\nimport apache.rocketmq.v2.ClientType;\nimport apache.rocketmq.v2.Code;\nimport apache.rocketmq.v2.FilterExpression;\nimport apache.rocketmq.v2.ReceiveMessageRequest;\nimport apache.rocketmq.v2.ReceiveMessageResponse;\nimport apache.rocketmq.v2.Settings;\nimport apache.rocketmq.v2.Subscription;\nimport com.google.protobuf.util.Durations;\nimport io.grpc.stub.StreamObserver;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PopStatus;\nimport org.apache.rocketmq.common.constant.ConsumeInitMode;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.proxy.common.MessageReceiptHandle;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.grpc.v2.AbstractMessagingActivity;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcClientChannel;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.processor.QueueSelector;\nimport org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;\nimport org.apache.rocketmq.proxy.service.route.MessageQueueSelector;\nimport org.apache.rocketmq.proxy.service.route.MessageQueueView;\nimport org.apache.rocketmq.remoting.protocol.filter.FilterAPI;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\npublic class ReceiveMessageActivity extends AbstractMessagingActivity {\n    private static final String ILLEGAL_POLLING_TIME_INTRODUCED_CLIENT_VERSION = \"5.0.3\";\n\n    public ReceiveMessageActivity(MessagingProcessor messagingProcessor,\n        GrpcClientSettingsManager grpcClientSettingsManager, GrpcChannelManager grpcChannelManager) {\n        super(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n    }\n\n    public void receiveMessage(ProxyContext ctx, ReceiveMessageRequest request,\n        StreamObserver<ReceiveMessageResponse> responseObserver) {\n        ReceiveMessageResponseStreamWriter writer = createWriter(ctx, responseObserver);\n\n        try {\n            Settings settings = this.grpcClientSettingsManager.getClientSettings(ctx);\n            final boolean isLite = ClientType.LITE_PUSH_CONSUMER.equals(settings.getClientType());\n\n            Subscription subscription = settings.getSubscription();\n            boolean fifo = subscription.getFifo();\n            int maxAttempts = settings.getBackoffPolicy().getMaxAttempts();\n            ProxyConfig config = ConfigurationManager.getProxyConfig();\n\n            Long timeRemaining = ctx.getRemainingMs();\n            long pollingTime;\n            if (request.hasLongPollingTimeout()) {\n                pollingTime = Durations.toMillis(request.getLongPollingTimeout());\n            } else {\n                pollingTime = timeRemaining - Durations.toMillis(settings.getRequestTimeout()) / 2;\n            }\n            if (pollingTime < config.getGrpcClientConsumerMinLongPollingTimeoutMillis()) {\n                pollingTime = config.getGrpcClientConsumerMinLongPollingTimeoutMillis();\n            }\n            if (pollingTime > config.getGrpcClientConsumerMaxLongPollingTimeoutMillis()) {\n                pollingTime = config.getGrpcClientConsumerMaxLongPollingTimeoutMillis();\n            }\n\n            if (pollingTime > timeRemaining) {\n                if (timeRemaining >= config.getGrpcClientConsumerMinLongPollingTimeoutMillis()) {\n                    pollingTime = timeRemaining;\n                } else {\n                    final String clientVersion = ctx.getClientVersion();\n                    Code code =\n                        null == clientVersion || ILLEGAL_POLLING_TIME_INTRODUCED_CLIENT_VERSION.compareTo(clientVersion) > 0 ?\n                        Code.BAD_REQUEST : Code.ILLEGAL_POLLING_TIME;\n                    writer.writeAndComplete(ctx, code, \"The deadline time remaining is not enough\" +\n                        \" for polling, please check network condition\");\n                    return;\n                }\n            }\n\n            validateTopicAndConsumerGroup(request.getMessageQueue().getTopic(), request.getGroup());\n            String topic = request.getMessageQueue().getTopic().getName();\n            String group = request.getGroup().getName();\n\n            long actualInvisibleTime = Durations.toMillis(request.getInvisibleDuration());\n            ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();\n            if (proxyConfig.isEnableProxyAutoRenew() && request.getAutoRenew()) {\n                actualInvisibleTime = proxyConfig.getDefaultInvisibleTimeMills();\n            } else {\n                validateInvisibleTime(actualInvisibleTime,\n                    ConfigurationManager.getProxyConfig().getMinInvisibleTimeMillsForRecv());\n            }\n\n            FilterExpression filterExpression = request.getFilterExpression();\n            SubscriptionData subscriptionData;\n            try {\n                subscriptionData = FilterAPI.build(topic, filterExpression.getExpression(),\n                    GrpcConverter.getInstance().buildExpressionType(filterExpression.getType()));\n            } catch (Exception e) {\n                writer.writeAndComplete(ctx, Code.ILLEGAL_FILTER_EXPRESSION, e.getMessage());\n                return;\n            }\n\n            CompletableFuture<PopResult> popFuture;\n            if (isLite) {\n\n                GrpcClientChannel clientChannel = grpcChannelManager.getChannel(ctx.getClientID());\n                if (clientChannel == null) {\n                    writer.writeAndComplete(ctx, Code.BAD_REQUEST,\n                        String.format(\"The client [%s] is disconnected.\", ctx.getClientID()));\n                    return;\n                }\n                // check lite consumer max unacked messages\n                int unackedMessageCount = messagingProcessor.getUnackedMessageCount(ctx, clientChannel, group);\n                if (proxyConfig.getMaxLiteRenewNumPerChannel() < unackedMessageCount) {\n                    writer.writeAndComplete(ctx, Code.FORBIDDEN,\n                        String.format(\"The client [%s] has too many unacked messages. Unacked count: %d\",\n                            ctx.getClientID(), unackedMessageCount));\n                    return;\n                }\n\n                popFuture = this.messagingProcessor.popLiteMessage(\n                    ctx,\n                    new ReceiveMessageQueueSelector(\n                        request.getMessageQueue().getBroker().getName()\n                    ),\n                    group,\n                    topic,\n                    request.getBatchSize(),\n                    actualInvisibleTime,\n                    pollingTime,\n                    subscriptionData,\n                    new PopMessageResultFilterImpl(maxAttempts),\n                    request.hasAttemptId() ? request.getAttemptId() : null,\n                    timeRemaining\n                );\n            } else {\n                popFuture = this.messagingProcessor.popMessage(\n                    ctx,\n                    new ReceiveMessageQueueSelector(\n                        request.getMessageQueue().getBroker().getName()\n                    ),\n                    group,\n                    topic,\n                    request.getBatchSize(),\n                    actualInvisibleTime,\n                    pollingTime,\n                    ConsumeInitMode.MAX,\n                    subscriptionData,\n                    fifo,\n                    new PopMessageResultFilterImpl(maxAttempts),\n                    request.hasAttemptId() ? request.getAttemptId() : null,\n                    timeRemaining\n                );\n            }\n\n            final boolean autoRenew = proxyConfig.isEnableProxyAutoRenew() && request.getAutoRenew();\n            popFuture.thenAccept(popResult -> {\n                Runnable doAfterWrite = null;\n                if (autoRenew) {\n                    doAfterWrite = handleAutoRenew(ctx, request, group, topic, popResult, writer);\n                }\n                writer.writeAndComplete(ctx, request, popResult, doAfterWrite);\n            }).exceptionally(t -> {\n                writer.writeAndComplete(ctx, request, t);\n                return null;\n            });\n        } catch (Throwable t) {\n            writer.writeAndComplete(ctx, request, t);\n        }\n    }\n\n    private Runnable handleAutoRenew(ProxyContext ctx, ReceiveMessageRequest request,\n        String group, String topic, PopResult popResult, ReceiveMessageResponseStreamWriter writer\n    ) {\n        if (!PopStatus.FOUND.equals(popResult.getPopStatus())) {\n            return null;\n        }\n\n        GrpcClientChannel clientChannel = grpcChannelManager.getChannel(ctx.getClientID());\n        if (clientChannel == null) {\n            GrpcProxyException e = new GrpcProxyException(Code.MESSAGE_NOT_FOUND,\n                String.format(\"The client [%s] is disconnected.\", ctx.getClientID()));\n            popResult.getMsgFoundList().forEach(messageExt ->\n                writer.processThrowableWhenWriteMessage(e, ctx, request, messageExt));\n            throw e;\n        }\n        return () -> {\n            List<MessageExt> messageExtList = popResult.getMsgFoundList();\n            for (MessageExt messageExt : messageExtList) {\n                String receiptHandle = messageExt.getProperty(MessageConst.PROPERTY_POP_CK);\n                if (receiptHandle != null) {\n                    MessageReceiptHandle messageReceiptHandle =\n                        new MessageReceiptHandle(group, topic, messageExt.getQueueId(), receiptHandle, messageExt.getMsgId(),\n                            messageExt.getQueueOffset(), messageExt.getReconsumeTimes());\n                    messagingProcessor.addReceiptHandle(ctx, clientChannel, group, messageExt.getMsgId(), messageReceiptHandle);\n                }\n            }\n        };\n    }\n\n    protected ReceiveMessageResponseStreamWriter createWriter(ProxyContext ctx,\n        StreamObserver<ReceiveMessageResponse> responseObserver) {\n        return new ReceiveMessageResponseStreamWriter(\n            this.messagingProcessor,\n            responseObserver\n        );\n    }\n\n    protected static class ReceiveMessageQueueSelector implements QueueSelector {\n\n        private final String brokerName;\n\n        public ReceiveMessageQueueSelector(String brokerName) {\n            this.brokerName = brokerName;\n        }\n\n        @Override\n        public AddressableMessageQueue select(ProxyContext ctx, MessageQueueView messageQueueView) {\n            try {\n                AddressableMessageQueue addressableMessageQueue = null;\n                MessageQueueSelector messageQueueSelector = messageQueueView.getReadSelector();\n\n                if (StringUtils.isNotBlank(brokerName)) {\n                    addressableMessageQueue = messageQueueSelector.getQueueByBrokerName(brokerName);\n                }\n\n                if (addressableMessageQueue == null) {\n                    addressableMessageQueue = messageQueueSelector.selectOne(true);\n                }\n                return addressableMessageQueue;\n            } catch (Throwable t) {\n                return null;\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageResponseStreamWriter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.grpc.v2.consumer;\n\nimport apache.rocketmq.v2.Code;\nimport apache.rocketmq.v2.Message;\nimport apache.rocketmq.v2.ReceiveMessageRequest;\nimport apache.rocketmq.v2.ReceiveMessageResponse;\nimport com.google.protobuf.util.Timestamps;\nimport io.grpc.stub.StreamObserver;\nimport java.time.Duration;\nimport java.util.Iterator;\nimport java.util.List;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PopStatus;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter;\nimport org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder;\nimport org.apache.rocketmq.proxy.grpc.v2.common.ResponseWriter;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\n\npublic class ReceiveMessageResponseStreamWriter {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    protected static final long NACK_INVISIBLE_TIME = Duration.ofSeconds(1).toMillis();\n\n    protected final MessagingProcessor messagingProcessor;\n    protected final StreamObserver<ReceiveMessageResponse> streamObserver;\n\n    public ReceiveMessageResponseStreamWriter(\n        MessagingProcessor messagingProcessor,\n        StreamObserver<ReceiveMessageResponse> observer) {\n        this.messagingProcessor = messagingProcessor;\n        this.streamObserver = observer;\n    }\n\n    public void writeAndComplete(ProxyContext ctx, ReceiveMessageRequest request, PopResult popResult) {\n        writeAndComplete(ctx, request, popResult, null);\n    }\n\n    public void writeAndComplete(ProxyContext ctx, ReceiveMessageRequest request, PopResult popResult, Runnable doAfterWrite) {\n        PopStatus status = popResult.getPopStatus();\n        List<MessageExt> messageFoundList = popResult.getMsgFoundList();\n        try {\n            switch (status) {\n                case FOUND:\n                    if (messageFoundList.isEmpty()) {\n                        streamObserver.onNext(ReceiveMessageResponse.newBuilder()\n                            .setStatus(ResponseBuilder.getInstance().buildStatus(Code.MESSAGE_NOT_FOUND, \"no match message\"))\n                            .build());\n                    } else {\n                        try {\n                            streamObserver.onNext(ReceiveMessageResponse.newBuilder()\n                                .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()))\n                                .build());\n                        } catch (Throwable t) {\n                            messageFoundList.forEach(messageExt ->\n                                this.processThrowableWhenWriteMessage(t, ctx, request, messageExt));\n                            throw t;\n                        }\n                        Iterator<MessageExt> messageIterator = messageFoundList.iterator();\n                        while (messageIterator.hasNext()) {\n                            MessageExt curMessageExt = messageIterator.next();\n                            Message curMessage = convertToMessage(curMessageExt);\n                            try {\n                                streamObserver.onNext(ReceiveMessageResponse.newBuilder()\n                                    .setMessage(curMessage)\n                                    .build());\n                            } catch (Throwable t) {\n                                this.processThrowableWhenWriteMessage(t, ctx, request, curMessageExt);\n                                messageIterator.forEachRemaining(messageExt ->\n                                    this.processThrowableWhenWriteMessage(t, ctx, request, messageExt));\n                                return;\n                            }\n                        }\n                    }\n                    break;\n                case POLLING_FULL:\n                    streamObserver.onNext(ReceiveMessageResponse.newBuilder()\n                        .setStatus(ResponseBuilder.getInstance().buildStatus(Code.TOO_MANY_REQUESTS, \"polling full\"))\n                        .build());\n                    break;\n                case NO_NEW_MSG:\n                case POLLING_NOT_FOUND:\n                default:\n                    streamObserver.onNext(ReceiveMessageResponse.newBuilder()\n                        .setStatus(ResponseBuilder.getInstance().buildStatus(Code.MESSAGE_NOT_FOUND, \"no new message\"))\n                        .build());\n                    break;\n            }\n            if (doAfterWrite != null) {\n                doAfterWrite.run();\n            }\n        } catch (Throwable t) {\n            writeResponseWithErrorIgnore(\n                ReceiveMessageResponse.newBuilder().setStatus(ResponseBuilder.getInstance().buildStatus(t)).build());\n        } finally {\n            onComplete();\n        }\n    }\n\n    protected Message convertToMessage(MessageExt messageExt) {\n        return GrpcConverter.getInstance().buildMessage(messageExt);\n    }\n\n    protected void processThrowableWhenWriteMessage(Throwable throwable,\n        ProxyContext ctx, ReceiveMessageRequest request, MessageExt messageExt) {\n\n        String handle = messageExt.getProperty(MessageConst.PROPERTY_POP_CK);\n        if (handle == null) {\n            return;\n        }\n\n        this.messagingProcessor.changeInvisibleTime(\n            ctx,\n            ReceiptHandle.decode(handle),\n            messageExt.getMsgId(),\n            request.getGroup().getName(),\n            request.getMessageQueue().getTopic().getName(),\n            NACK_INVISIBLE_TIME,\n            null,\n            MessagingProcessor.DEFAULT_TIMEOUT_MILLS,\n            true\n        );\n    }\n\n    public void writeAndComplete(ProxyContext ctx, Code code, String message) {\n        writeResponseWithErrorIgnore(\n            ReceiveMessageResponse.newBuilder().setStatus(ResponseBuilder.getInstance().buildStatus(code, message)).build());\n        onComplete();\n    }\n\n    public void writeAndComplete(ProxyContext ctx, ReceiveMessageRequest request, Throwable throwable) {\n        writeResponseWithErrorIgnore(\n            ReceiveMessageResponse.newBuilder().setStatus(ResponseBuilder.getInstance().buildStatus(throwable)).build());\n        onComplete();\n    }\n\n    protected void writeResponseWithErrorIgnore(ReceiveMessageResponse response) {\n        try {\n            ResponseWriter.getInstance().writeResponse(streamObserver, response);\n        } catch (Exception e) {\n            log.error(\"err when write receive message response\", e);\n        }\n    }\n\n    protected void onComplete() {\n        writeResponseWithErrorIgnore(ReceiveMessageResponse.newBuilder()\n            .setDeliveryTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n            .build());\n        try {\n            streamObserver.onCompleted();\n        } catch (Exception e) {\n            log.error(\"err when complete receive message response\", e);\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/ForwardMessageToDLQActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.grpc.v2.producer;\n\nimport apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest;\nimport apache.rocketmq.v2.ForwardMessageToDeadLetterQueueResponse;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.proxy.common.MessageReceiptHandle;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.grpc.v2.AbstractMessagingActivity;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic class ForwardMessageToDLQActivity extends AbstractMessagingActivity {\n\n    public ForwardMessageToDLQActivity(MessagingProcessor messagingProcessor,\n        GrpcClientSettingsManager grpcClientSettingsManager, GrpcChannelManager grpcChannelManager) {\n        super(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n    }\n\n    public CompletableFuture<ForwardMessageToDeadLetterQueueResponse> forwardMessageToDeadLetterQueue(ProxyContext ctx,\n        ForwardMessageToDeadLetterQueueRequest request) {\n        CompletableFuture<ForwardMessageToDeadLetterQueueResponse> future = new CompletableFuture<>();\n        try {\n            validateTopicAndConsumerGroup(request.getTopic(), request.getGroup());\n\n            String group = request.getGroup().getName();\n            String handleString = request.getReceiptHandle();\n            MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, grpcChannelManager.getChannel(ctx.getClientID()), group, request.getMessageId(), request.getReceiptHandle());\n            if (messageReceiptHandle != null) {\n                handleString = messageReceiptHandle.getReceiptHandleStr();\n            }\n            ReceiptHandle receiptHandle = ReceiptHandle.decode(handleString);\n\n            String liteTopic = request.hasLiteTopic() ? request.getLiteTopic() : null;\n\n            return this.messagingProcessor.forwardMessageToDeadLetterQueue(\n                ctx,\n                receiptHandle,\n                request.getMessageId(),\n                request.getGroup().getName(),\n                request.getTopic().getName(),\n                liteTopic\n            ).thenApply(result -> convertToForwardMessageToDeadLetterQueueResponse(ctx, result));\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    protected ForwardMessageToDeadLetterQueueResponse convertToForwardMessageToDeadLetterQueueResponse(ProxyContext ctx,\n        RemotingCommand result) {\n        return ForwardMessageToDeadLetterQueueResponse.newBuilder()\n            .setStatus(ResponseBuilder.getInstance().buildStatus(result.getCode(), result.getRemark()))\n            .build();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/RecallMessageActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2.producer;\n\nimport apache.rocketmq.v2.Code;\nimport apache.rocketmq.v2.RecallMessageRequest;\nimport apache.rocketmq.v2.RecallMessageResponse;\nimport apache.rocketmq.v2.Resource;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.grpc.v2.AbstractMessagingActivity;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\n\nimport java.time.Duration;\nimport java.util.concurrent.CompletableFuture;\n\npublic class RecallMessageActivity extends AbstractMessagingActivity {\n\n    public RecallMessageActivity(MessagingProcessor messagingProcessor,\n                                 GrpcClientSettingsManager grpcClientSettingsManager, GrpcChannelManager grpcChannelManager) {\n        super(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n    }\n\n    public CompletableFuture<RecallMessageResponse> recallMessage(ProxyContext ctx,\n        RecallMessageRequest request) {\n        CompletableFuture<RecallMessageResponse> future = new CompletableFuture<>();\n\n        try {\n            Resource topic = request.getTopic();\n            validateTopic(topic);\n\n            future = this.messagingProcessor.recallMessage(\n                ctx,\n                topic.getName(),\n                request.getRecallHandle(),\n                Duration.ofSeconds(2).toMillis()\n            ).thenApply(result -> RecallMessageResponse.newBuilder()\n                .setMessageId(result)\n                .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()))\n                .build());\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.grpc.v2.producer;\n\nimport apache.rocketmq.v2.Code;\nimport apache.rocketmq.v2.Encoding;\nimport apache.rocketmq.v2.MessageType;\nimport apache.rocketmq.v2.Resource;\nimport apache.rocketmq.v2.SendMessageRequest;\nimport apache.rocketmq.v2.SendMessageResponse;\nimport apache.rocketmq.v2.SendResultEntry;\nimport com.google.common.collect.Maps;\nimport com.google.common.hash.Hashing;\nimport com.google.protobuf.ByteString;\nimport com.google.protobuf.Timestamp;\nimport com.google.protobuf.util.Durations;\nimport com.google.protobuf.util.Timestamps;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.grpc.v2.AbstractMessagingActivity;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcValidator;\nimport org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.processor.QueueSelector;\nimport org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;\nimport org.apache.rocketmq.proxy.service.route.MessageQueueView;\n\npublic class SendMessageActivity extends AbstractMessagingActivity {\n\n    public SendMessageActivity(MessagingProcessor messagingProcessor,\n        GrpcClientSettingsManager grpcClientSettingsManager, GrpcChannelManager grpcChannelManager) {\n        super(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n    }\n\n    public CompletableFuture<SendMessageResponse> sendMessage(ProxyContext ctx, SendMessageRequest request) {\n        CompletableFuture<SendMessageResponse> future = new CompletableFuture<>();\n\n        try {\n            if (request.getMessagesCount() <= 0) {\n                throw new GrpcProxyException(Code.MESSAGE_CORRUPTED, \"no message to send\");\n            }\n\n            List<apache.rocketmq.v2.Message> messageList = request.getMessagesList();\n            apache.rocketmq.v2.Message message = messageList.get(0);\n            Resource topic = message.getTopic();\n            validateTopic(topic);\n\n            future = this.messagingProcessor.sendMessage(\n                ctx,\n                new SendMessageQueueSelector(request),\n                topic.getName(),\n                buildSysFlag(message),\n                buildMessage(ctx, request.getMessagesList(), topic)\n            ).thenApply(result -> convertToSendMessageResponse(ctx, request, result));\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    protected List<Message> buildMessage(ProxyContext context, List<apache.rocketmq.v2.Message> protoMessageList,\n        Resource topic) {\n        String topicName = topic.getName();\n        List<Message> messageExtList = new ArrayList<>();\n        for (apache.rocketmq.v2.Message protoMessage : protoMessageList) {\n            if (!protoMessage.getTopic().equals(topic)) {\n                throw new GrpcProxyException(Code.MESSAGE_CORRUPTED, \"topic in message is not same\");\n            }\n            // here use topicName as producerGroup for transactional checker.\n            messageExtList.add(buildMessage(context, protoMessage, topicName));\n        }\n        return messageExtList;\n    }\n\n    protected Message buildMessage(ProxyContext context, apache.rocketmq.v2.Message protoMessage, String producerGroup) {\n        String topicName = protoMessage.getTopic().getName();\n\n        validateMessageBodySize(protoMessage.getBody());\n        Message messageExt = new Message();\n        messageExt.setTopic(topicName);\n        messageExt.setBody(protoMessage.getBody().toByteArray());\n        Map<String, String> messageProperty = this.buildMessageProperty(context, protoMessage, producerGroup);\n\n        MessageAccessor.setProperties(messageExt, messageProperty);\n        return messageExt;\n    }\n\n    protected int buildSysFlag(apache.rocketmq.v2.Message protoMessage) {\n        // sysFlag (body encoding & message type)\n        int sysFlag = 0;\n        Encoding bodyEncoding = protoMessage.getSystemProperties().getBodyEncoding();\n        if (bodyEncoding.equals(Encoding.GZIP)) {\n            sysFlag |= MessageSysFlag.COMPRESSED_FLAG;\n        }\n        // transaction\n        MessageType messageType = protoMessage.getSystemProperties().getMessageType();\n        if (messageType.equals(MessageType.TRANSACTION)) {\n            sysFlag |= MessageSysFlag.TRANSACTION_PREPARED_TYPE;\n        }\n        return sysFlag;\n    }\n\n    protected void validateMessageBodySize(ByteString body) {\n        if (ConfigurationManager.getProxyConfig().isEnableMessageBodyEmptyCheck()) {\n            if (body.isEmpty()) {\n                throw new GrpcProxyException(Code.MESSAGE_BODY_EMPTY, \"message body cannot be empty\");\n            }\n        }\n        int max = ConfigurationManager.getProxyConfig().getMaxMessageSize();\n        if (max <= 0) {\n            return;\n        }\n        if (body.size() > max) {\n            throw new GrpcProxyException(Code.MESSAGE_BODY_TOO_LARGE, \"message body cannot exceed the max \" + max);\n        }\n    }\n\n    protected void validateMessageKey(String key) {\n        if (StringUtils.isNotEmpty(key)) {\n            if (StringUtils.isBlank(key)) {\n                throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_KEY, \"key cannot be the char sequence of whitespace\");\n            }\n            if (GrpcValidator.getInstance().containControlCharacter(key)) {\n                throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_KEY, \"key cannot contain control character\");\n            }\n        }\n    }\n\n    protected void validateMessageGroup(String messageGroup) {\n        if (StringUtils.isNotEmpty(messageGroup)) {\n            if (StringUtils.isBlank(messageGroup)) {\n                throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_GROUP, \"message group cannot be the char sequence of whitespace\");\n            }\n            int maxSize = ConfigurationManager.getProxyConfig().getMaxMessageGroupSize();\n            if (maxSize <= 0) {\n                return;\n            }\n            if (messageGroup.getBytes(StandardCharsets.UTF_8).length >= maxSize) {\n                throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_GROUP, \"message group exceed the max size \" + maxSize);\n            }\n            if (GrpcValidator.getInstance().containControlCharacter(messageGroup)) {\n                throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_GROUP, \"message group cannot contain control character\");\n            }\n        }\n    }\n\n    protected void validateDelayTime(long deliveryTimestampMs) {\n        long maxDelay = ConfigurationManager.getProxyConfig().getMaxDelayTimeMills();\n        if (maxDelay <= 0) {\n            return;\n        }\n        if (deliveryTimestampMs - System.currentTimeMillis() > maxDelay) {\n            throw new GrpcProxyException(Code.ILLEGAL_DELIVERY_TIME, \"the max delay time of message is too large, max is \" + maxDelay);\n        }\n    }\n\n    protected void validateTransactionRecoverySecond(long transactionRecoverySecond) {\n        long maxTransactionRecoverySecond = ConfigurationManager.getProxyConfig().getMaxTransactionRecoverySecond();\n        if (maxTransactionRecoverySecond <= 0) {\n            return;\n        }\n        if (transactionRecoverySecond > maxTransactionRecoverySecond) {\n            throw new GrpcProxyException(Code.BAD_REQUEST, \"the max transaction recovery time of message is too large, max is \" + maxTransactionRecoverySecond);\n        }\n    }\n\n    protected Map<String, String> buildMessageProperty(ProxyContext context, apache.rocketmq.v2.Message message, String producerGroup) {\n        long userPropertySize = 0;\n        ProxyConfig config = ConfigurationManager.getProxyConfig();\n        org.apache.rocketmq.common.message.Message messageWithHeader = new org.apache.rocketmq.common.message.Message();\n        // set user properties\n        Map<String, String> userProperties = message.getUserPropertiesMap();\n        if (userProperties.size() > config.getUserPropertyMaxNum()) {\n            throw new GrpcProxyException(Code.MESSAGE_PROPERTIES_TOO_LARGE, \"too many user properties, max is \" + config.getUserPropertyMaxNum());\n        }\n        for (Map.Entry<String, String> userPropertiesEntry : userProperties.entrySet()) {\n            if (MessageConst.STRING_HASH_SET.contains(userPropertiesEntry.getKey())) {\n                throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_PROPERTY_KEY, \"property is used by system: \" + userPropertiesEntry.getKey());\n            }\n            if (GrpcValidator.getInstance().containControlCharacter(userPropertiesEntry.getKey())) {\n                throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_PROPERTY_KEY, \"the key of property cannot contain control character\");\n            }\n            if (GrpcValidator.getInstance().containControlCharacter(userPropertiesEntry.getValue())) {\n                throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_PROPERTY_KEY, \"the value of property cannot contain control character\");\n            }\n            userPropertySize += userPropertiesEntry.getKey().getBytes(StandardCharsets.UTF_8).length;\n            userPropertySize += userPropertiesEntry.getValue().getBytes(StandardCharsets.UTF_8).length;\n        }\n        MessageAccessor.setProperties(messageWithHeader, Maps.newHashMap(userProperties));\n\n        // set tag\n        String tag = message.getSystemProperties().getTag();\n        GrpcValidator.getInstance().validateTag(tag);\n        messageWithHeader.setTags(tag);\n        userPropertySize += tag.getBytes(StandardCharsets.UTF_8).length;\n\n        // set keys\n        List<String> keysList = message.getSystemProperties().getKeysList();\n        for (String key : keysList) {\n            validateMessageKey(key);\n            userPropertySize += key.getBytes(StandardCharsets.UTF_8).length;\n        }\n        if (keysList.size() > 0) {\n            messageWithHeader.setKeys(keysList);\n        }\n\n        if (userPropertySize > config.getMaxUserPropertySize()) {\n            throw new GrpcProxyException(Code.MESSAGE_PROPERTIES_TOO_LARGE, \"the total size of user property is too large, max is \" + config.getMaxUserPropertySize());\n        }\n\n        // set message id\n        String messageId = message.getSystemProperties().getMessageId();\n        if (StringUtils.isBlank(messageId)) {\n            throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_ID, \"message id cannot be empty\");\n        }\n        MessageAccessor.putProperty(messageWithHeader, MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, messageId);\n\n        // set transaction property\n        MessageType messageType = message.getSystemProperties().getMessageType();\n        if (messageType.equals(MessageType.TRANSACTION)) {\n            MessageAccessor.putProperty(messageWithHeader, MessageConst.PROPERTY_TRANSACTION_PREPARED, \"true\");\n\n            if (message.getSystemProperties().hasOrphanedTransactionRecoveryDuration()) {\n                long transactionRecoverySecond = Durations.toSeconds(message.getSystemProperties().getOrphanedTransactionRecoveryDuration());\n                validateTransactionRecoverySecond(transactionRecoverySecond);\n                MessageAccessor.putProperty(messageWithHeader, MessageConst.PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS,\n                    String.valueOf(transactionRecoverySecond));\n            }\n        }\n\n        // set delay level or deliver timestamp\n        fillDelayMessageProperty(message, messageWithHeader);\n\n        // set priority\n        if (message.getSystemProperties().hasPriority()) {\n            int priority = message.getSystemProperties().getPriority();\n            messageWithHeader.setPriority(priority);\n        }\n\n        // set reconsume times\n        int reconsumeTimes = message.getSystemProperties().getDeliveryAttempt();\n        MessageAccessor.setReconsumeTime(messageWithHeader, String.valueOf(reconsumeTimes));\n        // set producer group\n        MessageAccessor.putProperty(messageWithHeader, MessageConst.PROPERTY_PRODUCER_GROUP, producerGroup);\n        // set message group\n        String messageGroup = message.getSystemProperties().getMessageGroup();\n        if (StringUtils.isNotEmpty(messageGroup)) {\n            validateMessageGroup(messageGroup);\n            MessageAccessor.putProperty(messageWithHeader, MessageConst.PROPERTY_SHARDING_KEY, messageGroup);\n        }\n        // set lite topic\n        String liteTopic = message.getSystemProperties().getLiteTopic();\n        if (StringUtils.isNotEmpty(liteTopic)) {\n            validateLiteTopic(liteTopic);\n            MessageAccessor.setLiteTopic(messageWithHeader, liteTopic);\n        }\n\n        // set trace context\n        String traceContext = message.getSystemProperties().getTraceContext();\n        if (!traceContext.isEmpty()) {\n            MessageAccessor.putProperty(messageWithHeader, MessageConst.PROPERTY_TRACE_CONTEXT, traceContext);\n        }\n\n        String bornHost = message.getSystemProperties().getBornHost();\n        if (StringUtils.isBlank(bornHost)) {\n            bornHost = context.getRemoteAddress();\n        }\n        if (StringUtils.isNotBlank(bornHost)) {\n            MessageAccessor.putProperty(messageWithHeader, MessageConst.PROPERTY_BORN_HOST, bornHost);\n        }\n\n        Timestamp bornTimestamp = message.getSystemProperties().getBornTimestamp();\n        if (Timestamps.isValid(bornTimestamp)) {\n            MessageAccessor.putProperty(messageWithHeader, MessageConst.PROPERTY_BORN_TIMESTAMP, String.valueOf(Timestamps.toMillis(bornTimestamp)));\n        }\n\n        return messageWithHeader.getProperties();\n    }\n\n    protected void fillDelayMessageProperty(apache.rocketmq.v2.Message message, org.apache.rocketmq.common.message.Message messageWithHeader) {\n        if (message.getSystemProperties().hasDeliveryTimestamp()) {\n            Timestamp deliveryTimestamp = message.getSystemProperties().getDeliveryTimestamp();\n            long deliveryTimestampMs = Timestamps.toMillis(deliveryTimestamp);\n            validateDelayTime(deliveryTimestampMs);\n\n            ProxyConfig config = ConfigurationManager.getProxyConfig();\n            if (config.isUseDelayLevel()) {\n                int delayLevel = config.computeDelayLevel(deliveryTimestampMs);\n                MessageAccessor.putProperty(messageWithHeader, MessageConst.PROPERTY_DELAY_TIME_LEVEL, String.valueOf(delayLevel));\n            }\n\n            String timestampString = String.valueOf(deliveryTimestampMs);\n            MessageAccessor.putProperty(messageWithHeader, MessageConst.PROPERTY_TIMER_DELIVER_MS, timestampString);\n        }\n    }\n\n    protected SendMessageResponse convertToSendMessageResponse(ProxyContext ctx, SendMessageRequest request,\n        List<SendResult> resultList) {\n        SendMessageResponse.Builder builder = SendMessageResponse.newBuilder();\n\n        Set<Code> responseCodes = new HashSet<>();\n        for (SendResult result : resultList) {\n            SendResultEntry resultEntry;\n            switch (result.getSendStatus()) {\n                case FLUSH_DISK_TIMEOUT:\n                    resultEntry = SendResultEntry.newBuilder()\n                        .setStatus(ResponseBuilder.getInstance().buildStatus(Code.MASTER_PERSISTENCE_TIMEOUT, \"send message failed, sendStatus=\" + result.getSendStatus()))\n                        .build();\n                    break;\n                case FLUSH_SLAVE_TIMEOUT:\n                    resultEntry = SendResultEntry.newBuilder()\n                        .setStatus(ResponseBuilder.getInstance().buildStatus(Code.SLAVE_PERSISTENCE_TIMEOUT, \"send message failed, sendStatus=\" + result.getSendStatus()))\n                        .build();\n                    break;\n                case SLAVE_NOT_AVAILABLE:\n                    resultEntry = SendResultEntry.newBuilder()\n                        .setStatus(ResponseBuilder.getInstance().buildStatus(Code.HA_NOT_AVAILABLE, \"send message failed, sendStatus=\" + result.getSendStatus()))\n                        .build();\n                    break;\n                case SEND_OK:\n                    resultEntry = SendResultEntry.newBuilder()\n                        .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()))\n                        .setOffset(result.getQueueOffset())\n                        .setMessageId(StringUtils.defaultString(result.getMsgId()))\n                        .setTransactionId(StringUtils.defaultString(result.getTransactionId()))\n                        .setRecallHandle(StringUtils.defaultString(result.getRecallHandle()))\n                        .build();\n                    break;\n                default:\n                    resultEntry = SendResultEntry.newBuilder()\n                        .setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, \"send message failed, sendStatus=\" + result.getSendStatus()))\n                        .build();\n                    break;\n            }\n            builder.addEntries(resultEntry);\n            responseCodes.add(resultEntry.getStatus().getCode());\n        }\n        if (responseCodes.size() > 1) {\n            builder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.MULTIPLE_RESULTS, Code.MULTIPLE_RESULTS.name()));\n        } else if (responseCodes.size() == 1) {\n            Code code = responseCodes.stream().findAny().get();\n            builder.setStatus(ResponseBuilder.getInstance().buildStatus(code, code.name()));\n        } else {\n            builder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, \"send status is empty\"));\n        }\n        return builder.build();\n    }\n\n    protected static class SendMessageQueueSelector implements QueueSelector {\n\n        private final SendMessageRequest request;\n\n        public SendMessageQueueSelector(SendMessageRequest request) {\n            this.request = request;\n        }\n\n        @Override\n        public AddressableMessageQueue select(ProxyContext ctx, MessageQueueView messageQueueView) {\n            try {\n                apache.rocketmq.v2.Message message = request.getMessages(0);\n                String shardingKey = null;\n                if (request.getMessagesCount() == 1) {\n                    shardingKey = message.getSystemProperties().getMessageGroup();\n                    // lite topic\n                    if (StringUtils.isBlank(shardingKey)) {\n                        shardingKey = message.getSystemProperties().getLiteTopic();\n                    }\n                }\n                AddressableMessageQueue targetMessageQueue;\n                if (StringUtils.isNotEmpty(shardingKey)) {\n                    // With shardingKey\n                    List<AddressableMessageQueue> writeQueues = messageQueueView.getWriteSelector().getQueues();\n                    int bucket = Hashing.consistentHash(shardingKey.hashCode(), writeQueues.size());\n                    targetMessageQueue = writeQueues.get(bucket);\n                } else {\n                    targetMessageQueue = messageQueueView.getWriteSelector().selectOneByPipeline(false);\n                }\n                return targetMessageQueue;\n            } catch (Exception e) {\n                return null;\n            }\n        }\n    }\n\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.grpc.v2.route;\n\nimport apache.rocketmq.v2.Address;\nimport apache.rocketmq.v2.AddressScheme;\nimport apache.rocketmq.v2.Assignment;\nimport apache.rocketmq.v2.Broker;\nimport apache.rocketmq.v2.Code;\nimport apache.rocketmq.v2.Endpoints;\nimport apache.rocketmq.v2.MessageQueue;\nimport apache.rocketmq.v2.MessageType;\nimport apache.rocketmq.v2.Permission;\nimport apache.rocketmq.v2.QueryAssignmentRequest;\nimport apache.rocketmq.v2.QueryAssignmentResponse;\nimport apache.rocketmq.v2.QueryRouteRequest;\nimport apache.rocketmq.v2.QueryRouteResponse;\nimport apache.rocketmq.v2.Resource;\nimport com.google.common.net.HostAndPort;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.grpc.v2.AbstractMessagingActivity;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\n\npublic class RouteActivity extends AbstractMessagingActivity {\n\n    public RouteActivity(MessagingProcessor messagingProcessor,\n        GrpcClientSettingsManager grpcClientSettingsManager, GrpcChannelManager grpcChannelManager) {\n        super(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n    }\n\n    public CompletableFuture<QueryRouteResponse> queryRoute(ProxyContext ctx, QueryRouteRequest request) {\n        CompletableFuture<QueryRouteResponse> future = new CompletableFuture<>();\n        try {\n            validateTopic(request.getTopic());\n            List<org.apache.rocketmq.proxy.common.Address> addressList = this.convertToAddressList(ctx, request.getEndpoints());\n\n            String topicName = request.getTopic().getName();\n            ProxyTopicRouteData proxyTopicRouteData = this.messagingProcessor.getTopicRouteDataForProxy(\n                ctx, addressList, topicName);\n\n            List<MessageQueue> messageQueueList = new ArrayList<>();\n            Map<String, Map<Long, Broker>> brokerMap = buildBrokerMap(proxyTopicRouteData.getBrokerDatas());\n\n            TopicMessageType topicMessageType = messagingProcessor.getMetadataService().getTopicMessageType(ctx, topicName);\n            for (QueueData queueData : proxyTopicRouteData.getQueueDatas()) {\n                String brokerName = queueData.getBrokerName();\n                Map<Long, Broker> brokerIdMap = brokerMap.get(brokerName);\n                if (brokerIdMap == null) {\n                    break;\n                }\n                for (Broker broker : brokerIdMap.values()) {\n                    messageQueueList.addAll(this.genMessageQueueFromQueueData(queueData, request.getTopic(), topicMessageType, broker));\n                }\n            }\n\n            QueryRouteResponse response = QueryRouteResponse.newBuilder()\n                .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()))\n                .addAllMessageQueues(messageQueueList)\n                .build();\n            future.complete(response);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    public CompletableFuture<QueryAssignmentResponse> queryAssignment(ProxyContext ctx,\n        QueryAssignmentRequest request) {\n        CompletableFuture<QueryAssignmentResponse> future = new CompletableFuture<>();\n\n        try {\n            validateTopicAndConsumerGroup(request.getTopic(), request.getGroup());\n            List<org.apache.rocketmq.proxy.common.Address> addressList = this.convertToAddressList(ctx, request.getEndpoints());\n\n            ProxyTopicRouteData proxyTopicRouteData = this.messagingProcessor.getTopicRouteDataForProxy(\n                ctx,\n                addressList,\n                request.getTopic().getName());\n\n            boolean isFifo = false;\n            boolean isLite = false;\n            SubscriptionGroupConfig groupConfig = this.messagingProcessor\n                .getSubscriptionGroupConfig(ctx, request.getGroup().getName());\n            if (groupConfig != null) {\n                isFifo = groupConfig.isConsumeMessageOrderly();\n                isLite = StringUtils.isNotEmpty(groupConfig.getLiteBindTopic());\n            }\n\n            List<Assignment> assignments = new ArrayList<>();\n            Map<String, Map<Long, Broker>> brokerMap = buildBrokerMap(proxyTopicRouteData.getBrokerDatas());\n            for (QueueData queueData : proxyTopicRouteData.getQueueDatas()) {\n                if (PermName.isReadable(queueData.getPerm()) && queueData.getReadQueueNums() > 0) {\n                    Map<Long, Broker> brokerIdMap = brokerMap.get(queueData.getBrokerName());\n                    if (brokerIdMap != null) {\n                        Broker broker = brokerIdMap.get(MixAll.MASTER_ID);\n                        Permission permission = this.convertToPermission(queueData.getPerm());\n                        if (isFifo && !isLite) {\n                            for (int i = 0; i < queueData.getReadQueueNums(); i++) {\n                                MessageQueue defaultMessageQueue = MessageQueue.newBuilder()\n                                    .setTopic(request.getTopic())\n                                    .setId(i)\n                                    .setPermission(permission)\n                                    .setBroker(broker)\n                                    .build();\n                                assignments.add(Assignment.newBuilder()\n                                    .setMessageQueue(defaultMessageQueue)\n                                    .build());\n                            }\n                        } else {\n                            MessageQueue defaultMessageQueue = MessageQueue.newBuilder()\n                                .setTopic(request.getTopic())\n                                .setId(-1)\n                                .setPermission(permission)\n                                .setBroker(broker)\n                                .build();\n                            assignments.add(Assignment.newBuilder()\n                                .setMessageQueue(defaultMessageQueue)\n                                .build());\n                        }\n\n                    }\n                }\n            }\n\n            QueryAssignmentResponse response;\n            if (assignments.isEmpty()) {\n                response = QueryAssignmentResponse.newBuilder()\n                    .setStatus(ResponseBuilder.getInstance().buildStatus(Code.FORBIDDEN, \"no readable queue\"))\n                    .build();\n            } else {\n                response = QueryAssignmentResponse.newBuilder()\n                    .addAllAssignments(assignments)\n                    .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()))\n                    .build();\n            }\n            future.complete(response);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    protected Permission convertToPermission(int perm) {\n        boolean isReadable = PermName.isReadable(perm);\n        boolean isWriteable = PermName.isWriteable(perm);\n        if (isReadable && isWriteable) {\n            return Permission.READ_WRITE;\n        }\n        if (isReadable) {\n            return Permission.READ;\n        }\n        if (isWriteable) {\n            return Permission.WRITE;\n        }\n        return Permission.NONE;\n    }\n\n    protected List<org.apache.rocketmq.proxy.common.Address> convertToAddressList(ProxyContext ctx, Endpoints endpoints) {\n        boolean useEndpointPort = ConfigurationManager.getProxyConfig().isUseEndpointPortFromRequest();\n\n        List<org.apache.rocketmq.proxy.common.Address> addressList = new ArrayList<>();\n        for (Address address : endpoints.getAddressesList()) {\n            int port = ConfigurationManager.getProxyConfig().getGrpcServerPort();\n            if (useEndpointPort) {\n                port = address.getPort();\n            }\n            addressList.add(new org.apache.rocketmq.proxy.common.Address(\n                org.apache.rocketmq.proxy.common.Address.AddressScheme.valueOf(endpoints.getScheme().name()),\n                HostAndPort.fromParts(address.getHost(), port)));\n        }\n        log.debug(\"gRPC build address. clientId={}, addressList={}\", ctx.getClientID(), addressList);\n        return addressList;\n    }\n\n    protected Map<String /*brokerName*/, Map<Long /*brokerID*/, Broker>> buildBrokerMap(\n        List<ProxyTopicRouteData.ProxyBrokerData> brokerDataList) {\n        Map<String, Map<Long, Broker>> brokerMap = new HashMap<>();\n        for (ProxyTopicRouteData.ProxyBrokerData brokerData : brokerDataList) {\n            Map<Long, Broker> brokerIdMap = new HashMap<>();\n            String brokerName = brokerData.getBrokerName();\n            for (Map.Entry<Long, List<org.apache.rocketmq.proxy.common.Address>> entry : brokerData.getBrokerAddrs().entrySet()) {\n                Long brokerId = entry.getKey();\n                List<Address> addressList = new ArrayList<>();\n                AddressScheme addressScheme = AddressScheme.IPv4;\n                for (org.apache.rocketmq.proxy.common.Address address : entry.getValue()) {\n                    addressScheme = AddressScheme.valueOf(address.getAddressScheme().name());\n                    addressList.add(Address.newBuilder()\n                        .setHost(address.getHostAndPort().getHost())\n                        .setPort(address.getHostAndPort().getPort())\n                        .build());\n                }\n\n                Broker broker = Broker.newBuilder()\n                    .setName(brokerName)\n                    .setId(Math.toIntExact(brokerId))\n                    .setEndpoints(Endpoints.newBuilder()\n                        .setScheme(addressScheme)\n                        .addAllAddresses(addressList)\n                        .build())\n                    .build();\n\n                brokerIdMap.put(brokerId, broker);\n            }\n            brokerMap.put(brokerName, brokerIdMap);\n        }\n        return brokerMap;\n    }\n\n    protected List<MessageQueue> genMessageQueueFromQueueData(QueueData queueData, Resource topic,\n        TopicMessageType topicMessageType, Broker broker) {\n        List<MessageQueue> messageQueueList = new ArrayList<>();\n\n        int r = 0;\n        int w = 0;\n        int rw = 0;\n        int n = 0;\n        if (PermName.isWriteable(queueData.getPerm()) && PermName.isReadable(queueData.getPerm())) {\n            rw = Math.min(queueData.getWriteQueueNums(), queueData.getReadQueueNums());\n            r = queueData.getReadQueueNums() - rw;\n            w = queueData.getWriteQueueNums() - rw;\n        } else if (PermName.isWriteable(queueData.getPerm())) {\n            w = queueData.getWriteQueueNums();\n        } else if (PermName.isReadable(queueData.getPerm())) {\n            r = queueData.getReadQueueNums();\n        } else if (!PermName.isAccessible(queueData.getPerm())) {\n            n = Math.max(1, Math.max(queueData.getWriteQueueNums(), queueData.getReadQueueNums()));\n        }\n\n        // r here means readOnly queue nums, w means writeOnly queue nums, while rw means both readable and writable queue nums.\n        int queueIdIndex = 0;\n        for (int i = 0; i < r; i++) {\n            MessageQueue messageQueue = MessageQueue.newBuilder().setBroker(broker).setTopic(topic)\n                .setId(queueIdIndex++)\n                .setPermission(Permission.READ)\n                .addAllAcceptMessageTypes(parseTopicMessageType(topicMessageType))\n                .build();\n            messageQueueList.add(messageQueue);\n        }\n\n        for (int i = 0; i < w; i++) {\n            MessageQueue messageQueue = MessageQueue.newBuilder().setBroker(broker).setTopic(topic)\n                .setId(queueIdIndex++)\n                .setPermission(Permission.WRITE)\n                .addAllAcceptMessageTypes(parseTopicMessageType(topicMessageType))\n                .build();\n            messageQueueList.add(messageQueue);\n        }\n\n        for (int i = 0; i < rw; i++) {\n            MessageQueue messageQueue = MessageQueue.newBuilder().setBroker(broker).setTopic(topic)\n                .setId(queueIdIndex++)\n                .setPermission(Permission.READ_WRITE)\n                .addAllAcceptMessageTypes(parseTopicMessageType(topicMessageType))\n                .build();\n            messageQueueList.add(messageQueue);\n        }\n\n        for (int i = 0; i < n; i++) {\n            MessageQueue messageQueue = MessageQueue.newBuilder().setBroker(broker).setTopic(topic)\n                .setId(queueIdIndex++)\n                .setPermission(Permission.NONE)\n                .addAllAcceptMessageTypes(parseTopicMessageType(topicMessageType))\n                .build();\n            messageQueueList.add(messageQueue);\n        }\n\n        return messageQueueList;\n    }\n\n    private List<MessageType> parseTopicMessageType(TopicMessageType topicMessageType) {\n        switch (topicMessageType) {\n            case NORMAL:\n                return Collections.singletonList(MessageType.NORMAL);\n            case FIFO:\n                return Collections.singletonList(MessageType.FIFO);\n            case LITE:\n                return Collections.singletonList(MessageType.LITE);\n            case TRANSACTION:\n                return Collections.singletonList(MessageType.TRANSACTION);\n            case DELAY:\n                return Collections.singletonList(MessageType.DELAY);\n            case PRIORITY:\n                return Collections.singletonList(MessageType.PRIORITY);\n            case MIXED:\n                return Arrays.asList(MessageType.NORMAL, MessageType.FIFO, MessageType.DELAY, MessageType.TRANSACTION);\n            default:\n                return Collections.singletonList(MessageType.MESSAGE_TYPE_UNSPECIFIED);\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/transaction/EndTransactionActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.grpc.v2.transaction;\n\nimport apache.rocketmq.v2.Code;\nimport apache.rocketmq.v2.EndTransactionRequest;\nimport apache.rocketmq.v2.EndTransactionResponse;\nimport apache.rocketmq.v2.TransactionResolution;\nimport apache.rocketmq.v2.TransactionSource;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.grpc.v2.AbstractMessagingActivity;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException;\nimport org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.processor.TransactionStatus;\n\npublic class EndTransactionActivity extends AbstractMessagingActivity {\n\n    public EndTransactionActivity(MessagingProcessor messagingProcessor,\n        GrpcClientSettingsManager grpcClientSettingsManager, GrpcChannelManager grpcChannelManager) {\n        super(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n    }\n\n    public CompletableFuture<EndTransactionResponse> endTransaction(ProxyContext ctx, EndTransactionRequest request) {\n        CompletableFuture<EndTransactionResponse> future = new CompletableFuture<>();\n        try {\n            validateTopic(request.getTopic());\n            if (StringUtils.isBlank(request.getTransactionId())) {\n                throw new GrpcProxyException(Code.INVALID_TRANSACTION_ID, \"transaction id cannot be empty\");\n            }\n\n            TransactionStatus transactionStatus = TransactionStatus.UNKNOWN;\n            TransactionResolution transactionResolution = request.getResolution();\n            switch (transactionResolution) {\n                case COMMIT:\n                    transactionStatus = TransactionStatus.COMMIT;\n                    break;\n                case ROLLBACK:\n                    transactionStatus = TransactionStatus.ROLLBACK;\n                    break;\n                default:\n                    break;\n            }\n            future = this.messagingProcessor.endTransaction(\n                ctx,\n                request.getTopic().getName(),\n                request.getTransactionId(),\n                request.getMessageId(),\n                request.getTopic().getName(),\n                transactionStatus,\n                request.getSource().equals(TransactionSource.SOURCE_SERVER_CHECK))\n                .thenApply(r -> EndTransactionResponse.newBuilder()\n                    .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()))\n                    .build());\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsConstant.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.metrics;\n\npublic class ProxyMetricsConstant {\n    public static final String GAUGE_PROXY_UP = \"rocketmq_proxy_up\";\n\n    public static final String LABEL_PROXY_MODE = \"proxy_mode\";\n    public static final String NODE_TYPE_PROXY = \"proxy\";\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.metrics;\n\nimport com.google.common.base.Splitter;\nimport io.opentelemetry.api.common.Attributes;\nimport io.opentelemetry.api.common.AttributesBuilder;\nimport io.opentelemetry.api.metrics.Meter;\nimport io.opentelemetry.api.metrics.ObservableLongGauge;\nimport io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter;\nimport io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter;\nimport io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporterBuilder;\nimport io.opentelemetry.exporter.prometheus.PrometheusHttpServer;\nimport io.opentelemetry.sdk.OpenTelemetrySdk;\nimport io.opentelemetry.sdk.metrics.InstrumentType;\nimport io.opentelemetry.sdk.metrics.SdkMeterProvider;\nimport io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder;\nimport io.opentelemetry.sdk.metrics.data.AggregationTemporality;\nimport io.opentelemetry.sdk.metrics.export.MetricExporter;\nimport io.opentelemetry.sdk.metrics.export.PeriodicMetricReader;\nimport io.opentelemetry.sdk.resources.Resource;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.TimeUnit;\nimport java.util.function.Supplier;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.metrics.BrokerMetricsManager;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.metrics.MetricsExporterType;\nimport org.apache.rocketmq.common.utils.StartAndShutdown;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.slf4j.bridge.SLF4JBridgeHandler;\n\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.AGGREGATION_DELTA;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_AGGREGATION;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CLUSTER_NAME;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_NODE_ID;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_NODE_TYPE;\nimport static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.OPEN_TELEMETRY_METER_NAME;\nimport static org.apache.rocketmq.proxy.metrics.ProxyMetricsConstant.GAUGE_PROXY_UP;\nimport static org.apache.rocketmq.proxy.metrics.ProxyMetricsConstant.LABEL_PROXY_MODE;\nimport static org.apache.rocketmq.proxy.metrics.ProxyMetricsConstant.NODE_TYPE_PROXY;\n\npublic class ProxyMetricsManager implements StartAndShutdown {\n    private final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    private static ProxyConfig proxyConfig;\n    private final static Map<String, String> LABEL_MAP = new HashMap<>();\n    public static Supplier<AttributesBuilder> attributesBuilderSupplier;\n\n    private OtlpGrpcMetricExporter metricExporter;\n    private PeriodicMetricReader periodicMetricReader;\n    private PrometheusHttpServer prometheusHttpServer;\n    private MetricExporter loggingMetricExporter;\n\n    public static ObservableLongGauge proxyUp = null;\n\n    public static void initLocalMode(BrokerMetricsManager brokerMetricsManager, ProxyConfig proxyConfig) {\n        if (proxyConfig.getMetricsExporterType() == MetricsExporterType.DISABLE) {\n            return;\n        }\n        ProxyMetricsManager.proxyConfig = proxyConfig;\n        LABEL_MAP.put(LABEL_NODE_TYPE, NODE_TYPE_PROXY);\n        LABEL_MAP.put(LABEL_CLUSTER_NAME, proxyConfig.getProxyClusterName());\n        LABEL_MAP.put(LABEL_NODE_ID, proxyConfig.getProxyName());\n        LABEL_MAP.put(LABEL_PROXY_MODE, proxyConfig.getProxyMode().toLowerCase());\n        initMetrics(brokerMetricsManager.getBrokerMeter(), brokerMetricsManager::newAttributesBuilder);\n    }\n\n    public static ProxyMetricsManager initClusterMode(ProxyConfig proxyConfig) {\n        ProxyMetricsManager.proxyConfig = proxyConfig;\n        return new ProxyMetricsManager();\n    }\n\n    public static AttributesBuilder newAttributesBuilder() {\n        AttributesBuilder attributesBuilder;\n        if (attributesBuilderSupplier == null) {\n            attributesBuilder = Attributes.builder();\n            LABEL_MAP.forEach(attributesBuilder::put);\n            return attributesBuilder;\n        }\n        attributesBuilder = attributesBuilderSupplier.get();\n        LABEL_MAP.forEach(attributesBuilder::put);\n        return attributesBuilder;\n    }\n\n    private static void initMetrics(Meter meter, Supplier<AttributesBuilder> attributesBuilderSupplier) {\n        ProxyMetricsManager.attributesBuilderSupplier = attributesBuilderSupplier;\n\n        proxyUp = meter.gaugeBuilder(GAUGE_PROXY_UP)\n            .setDescription(\"proxy status\")\n            .ofLongs()\n            .buildWithCallback(measurement -> measurement.record(1, newAttributesBuilder().build()));\n    }\n\n    public ProxyMetricsManager() {\n    }\n\n    private boolean checkConfig() {\n        if (proxyConfig == null) {\n            return false;\n        }\n        MetricsExporterType exporterType = proxyConfig.getMetricsExporterType();\n        if (!exporterType.isEnable()) {\n            return false;\n        }\n\n        switch (exporterType) {\n            case OTLP_GRPC:\n                return StringUtils.isNotBlank(proxyConfig.getMetricsGrpcExporterTarget());\n            case PROM:\n                return true;\n            case LOG:\n                return true;\n        }\n        return false;\n    }\n\n    @Override\n    public void start() throws Exception {\n        MetricsExporterType metricsExporterType = proxyConfig.getMetricsExporterType();\n        if (metricsExporterType == MetricsExporterType.DISABLE) {\n            return;\n        }\n        if (!checkConfig()) {\n            log.error(\"check metrics config failed, will not export metrics\");\n            return;\n        }\n\n        String labels = proxyConfig.getMetricsLabel();\n        if (StringUtils.isNotBlank(labels)) {\n            List<String> kvPairs = Splitter.on(',').omitEmptyStrings().splitToList(labels);\n            for (String item : kvPairs) {\n                String[] split = item.split(\":\");\n                if (split.length != 2) {\n                    log.warn(\"metricsLabel is not valid: {}\", labels);\n                    continue;\n                }\n                LABEL_MAP.put(split[0], split[1]);\n            }\n        }\n        if (proxyConfig.isMetricsInDelta()) {\n            LABEL_MAP.put(LABEL_AGGREGATION, AGGREGATION_DELTA);\n        }\n        LABEL_MAP.put(LABEL_NODE_TYPE, NODE_TYPE_PROXY);\n        LABEL_MAP.put(LABEL_CLUSTER_NAME, proxyConfig.getProxyClusterName());\n        LABEL_MAP.put(LABEL_NODE_ID, proxyConfig.getProxyName());\n        LABEL_MAP.put(LABEL_PROXY_MODE, proxyConfig.getProxyMode().toLowerCase());\n\n        SdkMeterProviderBuilder providerBuilder = SdkMeterProvider.builder()\n            .setResource(Resource.empty());\n\n        if (metricsExporterType == MetricsExporterType.OTLP_GRPC) {\n            String endpoint = proxyConfig.getMetricsGrpcExporterTarget();\n            if (!endpoint.startsWith(\"http\")) {\n                endpoint = \"https://\" + endpoint;\n            }\n            OtlpGrpcMetricExporterBuilder metricExporterBuilder = OtlpGrpcMetricExporter.builder()\n                .setEndpoint(endpoint)\n                .setTimeout(proxyConfig.getMetricGrpcExporterTimeOutInMills(), TimeUnit.MILLISECONDS)\n                .setAggregationTemporalitySelector(type -> {\n                    if (proxyConfig.isMetricsInDelta() &&\n                        (type == InstrumentType.COUNTER || type == InstrumentType.OBSERVABLE_COUNTER || type == InstrumentType.HISTOGRAM)) {\n                        return AggregationTemporality.DELTA;\n                    }\n                    return AggregationTemporality.CUMULATIVE;\n                });\n\n            String headers = proxyConfig.getMetricsGrpcExporterHeader();\n            if (StringUtils.isNotBlank(headers)) {\n                Map<String, String> headerMap = new HashMap<>();\n                List<String> kvPairs = Splitter.on(',').omitEmptyStrings().splitToList(headers);\n                for (String item : kvPairs) {\n                    String[] split = item.split(\":\");\n                    if (split.length != 2) {\n                        log.warn(\"metricsGrpcExporterHeader is not valid: {}\", headers);\n                        continue;\n                    }\n                    headerMap.put(split[0], split[1]);\n                }\n                headerMap.forEach(metricExporterBuilder::addHeader);\n            }\n\n            metricExporter = metricExporterBuilder.build();\n\n            periodicMetricReader = PeriodicMetricReader.builder(metricExporter)\n                .setInterval(proxyConfig.getMetricGrpcExporterIntervalInMills(), TimeUnit.MILLISECONDS)\n                .build();\n\n            providerBuilder.registerMetricReader(periodicMetricReader);\n        }\n\n        if (metricsExporterType == MetricsExporterType.PROM) {\n            String promExporterHost = proxyConfig.getMetricsPromExporterHost();\n            if (StringUtils.isBlank(promExporterHost)) {\n                promExporterHost = \"0.0.0.0\";\n            }\n            prometheusHttpServer = PrometheusHttpServer.builder()\n                .setHost(promExporterHost)\n                .setPort(proxyConfig.getMetricsPromExporterPort())\n                .build();\n            providerBuilder.registerMetricReader(prometheusHttpServer);\n        }\n\n        if (metricsExporterType == MetricsExporterType.LOG) {\n            SLF4JBridgeHandler.removeHandlersForRootLogger();\n            SLF4JBridgeHandler.install();\n            loggingMetricExporter = OtlpJsonLoggingMetricExporter.create(proxyConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE);\n            java.util.logging.Logger.getLogger(OtlpJsonLoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST);\n            periodicMetricReader = PeriodicMetricReader.builder(loggingMetricExporter)\n                .setInterval(proxyConfig.getMetricLoggingExporterIntervalInMills(), TimeUnit.MILLISECONDS)\n                .build();\n            providerBuilder.registerMetricReader(periodicMetricReader);\n        }\n\n        Meter proxyMeter = OpenTelemetrySdk.builder()\n            .setMeterProvider(providerBuilder.build())\n            .build()\n            .getMeter(OPEN_TELEMETRY_METER_NAME);\n\n        initMetrics(proxyMeter, null);\n    }\n\n    @Override\n    public void shutdown() throws Exception {\n        if (proxyConfig.getMetricsExporterType() == MetricsExporterType.OTLP_GRPC) {\n            periodicMetricReader.forceFlush();\n            periodicMetricReader.shutdown();\n            metricExporter.shutdown();\n        }\n        if (proxyConfig.getMetricsExporterType() == MetricsExporterType.PROM) {\n            prometheusHttpServer.forceFlush();\n            prometheusHttpServer.shutdown();\n        }\n        if (proxyConfig.getMetricsExporterType() == MetricsExporterType.LOG) {\n            periodicMetricReader.forceFlush();\n            periodicMetricReader.shutdown();\n            loggingMetricExporter.shutdown();\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.processor;\n\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.common.utils.AbstractStartAndShutdown;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\nimport org.apache.rocketmq.proxy.service.ServiceManager;\n\npublic abstract class AbstractProcessor extends AbstractStartAndShutdown {\n\n    protected MessagingProcessor messagingProcessor;\n    protected ServiceManager serviceManager;\n\n    protected static final ProxyException EXPIRED_HANDLE_PROXY_EXCEPTION = new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, \"receipt handle is expired\");\n\n    public AbstractProcessor(MessagingProcessor messagingProcessor,\n        ServiceManager serviceManager) {\n        this.messagingProcessor = messagingProcessor;\n        this.serviceManager = serviceManager;\n    }\n\n    protected void validateReceiptHandle(ReceiptHandle handle) {\n        if (handle.isExpired()) {\n            throw EXPIRED_HANDLE_PROXY_EXCEPTION;\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.processor;\n\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;\n\npublic class BatchAckResult {\n\n    private final ReceiptHandleMessage receiptHandleMessage;\n    private AckResult ackResult;\n    private ProxyException proxyException;\n\n    public BatchAckResult(ReceiptHandleMessage receiptHandleMessage,\n        AckResult ackResult) {\n        this.receiptHandleMessage = receiptHandleMessage;\n        this.ackResult = ackResult;\n    }\n\n    public BatchAckResult(ReceiptHandleMessage receiptHandleMessage,\n        ProxyException proxyException) {\n        this.receiptHandleMessage = receiptHandleMessage;\n        this.proxyException = proxyException;\n    }\n\n    public ReceiptHandleMessage getReceiptHandleMessage() {\n        return receiptHandleMessage;\n    }\n\n    public AckResult getAckResult() {\n        return ackResult;\n    }\n\n    public ProxyException getProxyException() {\n        return proxyException;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/processor/ClientProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.processor;\n\nimport apache.rocketmq.v2.Code;\nimport com.google.common.util.concurrent.RateLimiter;\nimport io.netty.channel.Channel;\nimport java.util.Objects;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ConsumerGroupInfo;\nimport org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;\nimport org.apache.rocketmq.broker.client.ProducerChangeListener;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.lite.LiteSubscriptionAction;\nimport org.apache.rocketmq.common.lite.LiteSubscriptionDTO;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException;\nimport org.apache.rocketmq.proxy.service.ServiceManager;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\n\n@SuppressWarnings(\"UnstableApiUsage\")\npublic class ClientProcessor extends AbstractProcessor {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    private final RateLimiter syncLiteSubscriptionRateLimiter;\n\n    public ClientProcessor(MessagingProcessor messagingProcessor,\n        ServiceManager serviceManager) {\n        super(messagingProcessor, serviceManager);\n\n        ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();\n        this.syncLiteSubscriptionRateLimiter = RateLimiter.create(proxyConfig.getMaxSyncLiteSubscriptionRate());\n    }\n\n    public void registerProducer(\n        ProxyContext ctx,\n        String producerGroup,\n        ClientChannelInfo clientChannelInfo\n    ) {\n        this.serviceManager.getProducerManager().registerProducer(producerGroup, clientChannelInfo);\n    }\n\n    public void unRegisterProducer(\n        ProxyContext ctx,\n        String producerGroup,\n        ClientChannelInfo clientChannelInfo\n    ) {\n        this.serviceManager.getProducerManager().unregisterProducer(producerGroup, clientChannelInfo);\n    }\n\n    public Channel findProducerChannel(\n        ProxyContext ctx,\n        String producerGroup,\n        String clientId\n    ) {\n        return this.serviceManager.getProducerManager().findChannel(clientId);\n    }\n\n    public void registerProducerChangeListener(ProducerChangeListener listener) {\n        this.serviceManager.getProducerManager().appendProducerChangeListener(listener);\n    }\n\n    public void registerConsumer(\n        ProxyContext ctx,\n        String consumerGroup,\n        ClientChannelInfo clientChannelInfo,\n        ConsumeType consumeType,\n        MessageModel messageModel,\n        ConsumeFromWhere consumeFromWhere,\n        Set<SubscriptionData> subList,\n        boolean updateSubscription\n    ) {\n        validateLiteMode(ctx, consumerGroup, messageModel);\n        if (MessageModel.LITE_SELECTIVE == messageModel) {\n            validateLiteSubTopic(ctx, consumerGroup, subList);\n        }\n        this.serviceManager.getConsumerManager().registerConsumer(\n            consumerGroup,\n            clientChannelInfo,\n            consumeType,\n            messageModel,\n            consumeFromWhere,\n            subList,\n            false,\n            updateSubscription);\n    }\n\n    public CompletableFuture<Void> syncLiteSubscription(ProxyContext ctx,\n        LiteSubscriptionDTO liteSubscriptionDTO, long timeoutMillis\n    ) {\n        try {\n            validateLiteBindTopic(ctx, liteSubscriptionDTO.getGroup(), liteSubscriptionDTO.getTopic());\n            if (CollectionUtils.isNotEmpty(liteSubscriptionDTO.getLiteTopicSet())) {\n                validateLiteSubscriptionQuota(ctx, liteSubscriptionDTO.getGroup(), liteSubscriptionDTO.getLiteTopicSet().size());\n            }\n\n            if (LiteSubscriptionAction.PARTIAL_ADD == liteSubscriptionDTO.getAction()) {\n                if (!syncLiteSubscriptionRateLimiter.tryAcquire()) {\n                    String msg = String.format(\"Too many syncLiteSubscription requests, topic=%s, group=%s, clientId=%s\",\n                        liteSubscriptionDTO.getTopic(), liteSubscriptionDTO.getGroup(), ctx.getClientID());\n                    log.warn(msg);\n                    throw new GrpcProxyException(Code.TOO_MANY_REQUESTS, msg);\n                }\n            }\n\n            return this.serviceManager\n                .getLiteSubscriptionService()\n                .syncLiteSubscription(ctx, liteSubscriptionDTO, timeoutMillis);\n        } catch (Throwable t) {\n            CompletableFuture<Void> future = new CompletableFuture<>();\n            future.completeExceptionally(t);\n            return future;\n        }\n    }\n\n    public ClientChannelInfo findConsumerChannel(\n        ProxyContext ctx,\n        String consumerGroup,\n        Channel channel\n    ) {\n        return this.serviceManager.getConsumerManager().findChannel(consumerGroup, channel);\n    }\n\n    public void unRegisterConsumer(\n        ProxyContext ctx,\n        String consumerGroup,\n        ClientChannelInfo clientChannelInfo\n    ) {\n        this.serviceManager.getConsumerManager().unregisterConsumer(consumerGroup, clientChannelInfo, false);\n    }\n\n    public void doChannelCloseEvent(String remoteAddr, Channel channel) {\n        this.serviceManager.getConsumerManager().doChannelCloseEvent(remoteAddr, channel);\n        this.serviceManager.getProducerManager().doChannelCloseEvent(remoteAddr, channel);\n    }\n\n    public void registerConsumerIdsChangeListener(ConsumerIdsChangeListener listener) {\n        this.serviceManager.getConsumerManager().appendConsumerIdsChangeListener(listener);\n    }\n\n    public ConsumerGroupInfo getConsumerGroupInfo(ProxyContext ctx, String consumerGroup) {\n        return this.serviceManager.getConsumerManager().getConsumerGroupInfo(consumerGroup);\n    }\n\n    /**\n     * Validates the message model for a given consumer group.\n     * Ensures that regular groups do not use LITE mode and LITE groups use LITE mode.\n     *\n     * @param ctx          the proxy context\n     * @param group        the consumer group name\n     * @param messageModel the message model to validate\n     */\n    protected void validateLiteMode(ProxyContext ctx, String group, MessageModel messageModel) {\n        String bindTopic = getGroupOrException(ctx, group).getLiteBindTopic();\n        if (StringUtils.isEmpty(bindTopic)) {\n            // regular group\n            if (MessageModel.LITE_SELECTIVE == messageModel) {\n                throw new GrpcProxyException(Code.ILLEGAL_CONSUMER_GROUP,\n                    \"regular group cannot use LITE mode: \" + group);\n            }\n        } else {\n            // lite group\n            if (MessageModel.LITE_SELECTIVE != messageModel) {\n                throw new GrpcProxyException(Code.ILLEGAL_CONSUMER_GROUP,\n                    \"lite group must use LITE mode: \" + group);\n            }\n        }\n    }\n\n    protected void validateLiteSubTopic(ProxyContext ctx, String group, Set<SubscriptionData> subList) {\n        if (CollectionUtils.isEmpty(subList)) {\n            return;\n        }\n        // check bindTopic for sub list\n        validateLiteBindTopic(ctx, group, subList.iterator().next().getTopic());\n    }\n\n    protected void validateLiteBindTopic(ProxyContext ctx, String group, String bindTopic) {\n        String expectedBindTopic = getGroupOrException(ctx, group).getLiteBindTopic();\n        if (!Objects.equals(expectedBindTopic, bindTopic)) {\n            throw new GrpcProxyException(Code.ILLEGAL_TOPIC,\n                String.format(\"lite group %s is expected to bind topic %s, but actual is %s\",\n                    group, expectedBindTopic, bindTopic));\n        }\n    }\n\n    protected void validateLiteSubscriptionQuota(ProxyContext ctx, String group, int actual) {\n        int quota = getGroupOrException(ctx, group).getLiteSubClientQuota();\n        int quotaBuffer = 300;\n        if (actual > quota + quotaBuffer) {\n            throw new GrpcProxyException(Code.LITE_SUBSCRIPTION_QUOTA_EXCEEDED,\n                \"lite subscription quota exceeded: \" + quota);\n        }\n    }\n\n    protected SubscriptionGroupConfig getGroupOrException(ProxyContext ctx, String group) {\n        SubscriptionGroupConfig groupConfig = this.messagingProcessor.getSubscriptionGroupConfig(ctx, group);\n        if (groupConfig == null) {\n            throw new GrpcProxyException(Code.ILLEGAL_CONSUMER_GROUP, \"group not found: \" + group);\n        }\n        return groupConfig;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.processor;\n\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.CopyOnWriteArraySet;\nimport java.util.concurrent.ExecutorService;\nimport java.util.stream.Collectors;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PopStatus;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageClientExt;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.utils.FutureUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\nimport org.apache.rocketmq.proxy.common.utils.ProxyUtils;\nimport org.apache.rocketmq.proxy.service.ServiceManager;\nimport org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;\nimport org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMinOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopLiteMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\npublic class ConsumerProcessor extends AbstractProcessor {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    private final ExecutorService executor;\n\n    public ConsumerProcessor(MessagingProcessor messagingProcessor, ServiceManager serviceManager,\n        ExecutorService executor) {\n        super(messagingProcessor, serviceManager);\n        this.executor = executor;\n    }\n\n    public CompletableFuture<PopResult> popMessage(\n        ProxyContext ctx,\n        QueueSelector queueSelector,\n        String consumerGroup,\n        String topic,\n        int maxMsgNums,\n        long invisibleTime,\n        long pollTime,\n        int initMode,\n        SubscriptionData subscriptionData,\n        boolean fifo,\n        PopMessageResultFilter popMessageResultFilter,\n        String attemptId,\n        long timeoutMillis\n    ) {\n        CompletableFuture<PopResult> future = new CompletableFuture<>();\n        try {\n            AddressableMessageQueue messageQueue = queueSelector.select(ctx, this.serviceManager.getTopicRouteService().getCurrentMessageQueueView(ctx, topic));\n            if (messageQueue == null) {\n                throw new ProxyException(ProxyExceptionCode.FORBIDDEN, \"no readable queue\");\n            }\n            return popMessage(ctx, messageQueue, consumerGroup, topic, maxMsgNums, invisibleTime, pollTime, initMode,\n                subscriptionData, fifo, popMessageResultFilter, attemptId, timeoutMillis);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    public CompletableFuture<PopResult> popMessage(\n        ProxyContext ctx,\n        AddressableMessageQueue messageQueue,\n        String consumerGroup,\n        String topic,\n        int maxMsgNums,\n        long invisibleTime,\n        long pollTime,\n        int initMode,\n        SubscriptionData subscriptionData,\n        boolean fifo,\n        PopMessageResultFilter popMessageResultFilter,\n        String attemptId,\n        long timeoutMillis\n    ) {\n        CompletableFuture<PopResult> future = new CompletableFuture<>();\n        try {\n            if (maxMsgNums > ProxyUtils.MAX_MSG_NUMS_FOR_POP_REQUEST) {\n                log.warn(\"change maxNums from {} to {} for pop request, with info: topic:{}, group:{}\",\n                    maxMsgNums, ProxyUtils.MAX_MSG_NUMS_FOR_POP_REQUEST, topic, consumerGroup);\n                maxMsgNums = ProxyUtils.MAX_MSG_NUMS_FOR_POP_REQUEST;\n            }\n\n            PopMessageRequestHeader requestHeader = new PopMessageRequestHeader();\n            requestHeader.setConsumerGroup(consumerGroup);\n            requestHeader.setTopic(topic);\n            requestHeader.setQueueId(messageQueue.getQueueId());\n            requestHeader.setMaxMsgNums(maxMsgNums);\n            requestHeader.setInvisibleTime(invisibleTime);\n            requestHeader.setPollTime(pollTime);\n            requestHeader.setInitMode(initMode);\n            requestHeader.setExpType(subscriptionData.getExpressionType());\n            requestHeader.setExp(subscriptionData.getSubString());\n            requestHeader.setOrder(fifo);\n            requestHeader.setAttemptId(attemptId);\n\n            future = this.serviceManager.getMessageService().popMessage(\n                    ctx,\n                    messageQueue,\n                    requestHeader,\n                    timeoutMillis)\n                .thenApplyAsync(popResult -> filterPopResult(ctx, popResult,\n                    requestHeader, consumerGroup, topic, subscriptionData, popMessageResultFilter), this.executor);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return FutureUtils.addExecutor(future, this.executor);\n    }\n\n    private PopResult filterPopResult(ProxyContext ctx, PopResult popResult, CommandCustomHeader requestHeader,\n        String consumerGroup, String topic, SubscriptionData subscriptionData,\n        PopMessageResultFilter popMessageResultFilter) {\n        if (PopStatus.FOUND.equals(popResult.getPopStatus()) &&\n            !CollectionUtils.isEmpty(popResult.getMsgFoundList()) &&\n            popMessageResultFilter != null) {\n\n            List<MessageExt> messageExtList = new ArrayList<>();\n            for (MessageExt messageExt : popResult.getMsgFoundList()) {\n                try {\n                    fillUniqIDIfNeed(messageExt);\n                    String handleString = createHandle(messageExt.getProperty(MessageConst.PROPERTY_POP_CK), messageExt.getCommitLogOffset());\n                    if (handleString == null) {\n                        log.error(\"[BUG] pop message from broker but handle is empty. requestHeader:{}, msg:{}\", requestHeader, messageExt);\n                        messageExtList.add(messageExt);\n                        continue;\n                    }\n                    MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_POP_CK, handleString);\n\n                    String liteTopic = messageExt.getProperty(MessageConst.PROPERTY_LITE_TOPIC);\n\n                    PopMessageResultFilter.FilterResult filterResult =\n                        popMessageResultFilter.filterMessage(ctx, consumerGroup, subscriptionData, messageExt);\n                    switch (filterResult) {\n                        case NO_MATCH:\n                            this.messagingProcessor.ackMessage(\n                                ctx,\n                                ReceiptHandle.decode(handleString),\n                                messageExt.getMsgId(),\n                                consumerGroup,\n                                topic,\n                                liteTopic,\n                                MessagingProcessor.DEFAULT_TIMEOUT_MILLS);\n                            break;\n                        case TO_DLQ:\n                            this.messagingProcessor.forwardMessageToDeadLetterQueue(\n                                ctx,\n                                ReceiptHandle.decode(handleString),\n                                messageExt.getMsgId(),\n                                consumerGroup,\n                                topic,\n                                liteTopic,\n                                MessagingProcessor.DEFAULT_TIMEOUT_MILLS);\n                            break;\n                        case TO_RETURN:\n                            this.messagingProcessor.changeInvisibleTime(\n                                    ctx,\n                                    ReceiptHandle.decode(handleString),\n                                    messageExt.getMsgId(),\n                                    consumerGroup,\n                                    topic,\n                                    MessagingProcessor.INVISIBLE_TIME_MS,\n                                    null,\n                                    MessagingProcessor.DEFAULT_TIMEOUT_MILLS,\n                                    true);\n                            break;\n                        case MATCH:\n                        default:\n                            messageExtList.add(messageExt);\n                            break;\n                    }\n                } catch (Throwable t) {\n                    log.error(\"process filterMessage failed. requestHeader:{}, msg:{}\", requestHeader, messageExt, t);\n                    messageExtList.add(messageExt);\n                }\n            }\n            popResult.setMsgFoundList(messageExtList);\n        }\n        return popResult;\n    }\n\n    public CompletableFuture<PopResult> popLiteMessage(\n        ProxyContext ctx,\n        QueueSelector queueSelector,\n        String consumerGroup,\n        String topic,\n        int maxMsgNums,\n        long invisibleTime,\n        long pollTime,\n        SubscriptionData subscriptionData,\n        PopMessageResultFilter popMessageResultFilter,\n        String attemptId,\n        long timeoutMillis\n    ) {\n        CompletableFuture<PopResult> future = new CompletableFuture<>();\n        try {\n            AddressableMessageQueue messageQueue = queueSelector.select(ctx,\n                this.serviceManager.getTopicRouteService().getCurrentMessageQueueView(ctx, topic));\n            if (messageQueue == null) {\n                throw new ProxyException(ProxyExceptionCode.FORBIDDEN, \"no readable queue\");\n            }\n            return doPopLiteMessage(ctx, messageQueue, consumerGroup, topic, maxMsgNums, invisibleTime, pollTime,\n                subscriptionData, popMessageResultFilter, attemptId, timeoutMillis);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    private CompletableFuture<PopResult> doPopLiteMessage(\n        ProxyContext ctx,\n        AddressableMessageQueue messageQueue,\n        String consumerGroup,\n        String topic,\n        int maxMsgNums,\n        long invisibleTime,\n        long pollTime,\n        SubscriptionData subscriptionData,\n        PopMessageResultFilter popMessageResultFilter,\n        String attemptId,\n        long timeoutMillis\n    ) {\n        CompletableFuture<PopResult> future = new CompletableFuture<>();\n        try {\n            if (maxMsgNums > ProxyUtils.MAX_MSG_NUMS_FOR_POP_REQUEST) {\n                log.warn(\"change maxNums from {} to {} for pop request, with info: topic:{}, group:{}\",\n                    maxMsgNums, ProxyUtils.MAX_MSG_NUMS_FOR_POP_REQUEST, topic, consumerGroup);\n                maxMsgNums = ProxyUtils.MAX_MSG_NUMS_FOR_POP_REQUEST;\n            }\n\n            PopLiteMessageRequestHeader requestHeader = new PopLiteMessageRequestHeader();\n            requestHeader.setClientId(ctx.getClientID());\n            requestHeader.setConsumerGroup(consumerGroup);\n            requestHeader.setTopic(topic);\n            requestHeader.setMaxMsgNum(maxMsgNums);\n            requestHeader.setInvisibleTime(invisibleTime);\n            requestHeader.setPollTime(pollTime);\n            requestHeader.setAttemptId(attemptId);\n            requestHeader.setBornTime(System.currentTimeMillis());\n\n            future = this.serviceManager.getMessageService().popLiteMessage(\n                    ctx,\n                    messageQueue,\n                    requestHeader,\n                    timeoutMillis)\n                .thenApplyAsync(popResult -> filterPopResult(ctx, popResult,\n                    requestHeader, consumerGroup, topic, subscriptionData, popMessageResultFilter), this.executor);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n            FutureUtils.addExecutor(future, this.executor);\n        }\n        return future;\n    }\n\n    private void fillUniqIDIfNeed(MessageExt messageExt) {\n        if (StringUtils.isBlank(MessageClientIDSetter.getUniqID(messageExt))) {\n            if (messageExt instanceof MessageClientExt) {\n                MessageClientExt clientExt = (MessageClientExt) messageExt;\n                MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, clientExt.getOffsetMsgId());\n            }\n        }\n    }\n\n    public CompletableFuture<AckResult> ackMessage(\n        ProxyContext ctx,\n        ReceiptHandle handle,\n        String messageId,\n        String consumerGroup,\n        String topic,\n        String liteTopic,\n        long timeoutMillis\n    ) {\n        CompletableFuture<AckResult> future = new CompletableFuture<>();\n        try {\n            this.validateReceiptHandle(handle);\n\n            AckMessageRequestHeader ackMessageRequestHeader = new AckMessageRequestHeader();\n            ackMessageRequestHeader.setConsumerGroup(consumerGroup);\n            ackMessageRequestHeader.setTopic(handle.getRealTopic(topic, consumerGroup));\n            ackMessageRequestHeader.setQueueId(handle.getQueueId());\n            ackMessageRequestHeader.setExtraInfo(handle.getReceiptHandle());\n            ackMessageRequestHeader.setOffset(handle.getOffset());\n            ackMessageRequestHeader.setLiteTopic(liteTopic);\n\n            future = this.serviceManager.getMessageService().ackMessage(\n                ctx,\n                handle,\n                messageId,\n                ackMessageRequestHeader,\n                timeoutMillis);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return FutureUtils.addExecutor(future, this.executor);\n    }\n\n    public CompletableFuture<List<BatchAckResult>> batchAckMessage(\n        ProxyContext ctx,\n        List<ReceiptHandleMessage> handleMessageList,\n        String consumerGroup,\n        String topic,\n        long timeoutMillis\n    ) {\n        CompletableFuture<List<BatchAckResult>> future = new CompletableFuture<>();\n        try {\n            List<BatchAckResult> batchAckResultList = new ArrayList<>(handleMessageList.size());\n            Map<String, List<ReceiptHandleMessage>> brokerHandleListMap = new HashMap<>();\n\n            for (ReceiptHandleMessage handleMessage : handleMessageList) {\n                if (handleMessage.getReceiptHandle().isExpired()) {\n                    batchAckResultList.add(new BatchAckResult(handleMessage, EXPIRED_HANDLE_PROXY_EXCEPTION));\n                    continue;\n                }\n                List<ReceiptHandleMessage> brokerHandleList = brokerHandleListMap.computeIfAbsent(handleMessage.getReceiptHandle().getBrokerName(), key -> new ArrayList<>());\n                brokerHandleList.add(handleMessage);\n            }\n\n            if (brokerHandleListMap.isEmpty()) {\n                return FutureUtils.addExecutor(CompletableFuture.completedFuture(batchAckResultList), this.executor);\n            }\n            Set<Map.Entry<String, List<ReceiptHandleMessage>>> brokerHandleListMapEntrySet = brokerHandleListMap.entrySet();\n            CompletableFuture<List<BatchAckResult>>[] futures = new CompletableFuture[brokerHandleListMapEntrySet.size()];\n            int futureIndex = 0;\n            for (Map.Entry<String, List<ReceiptHandleMessage>> entry : brokerHandleListMapEntrySet) {\n                futures[futureIndex++] = processBrokerHandle(ctx, consumerGroup, topic, entry.getValue(), timeoutMillis);\n            }\n            CompletableFuture.allOf(futures).whenComplete((val, throwable) -> {\n                if (throwable != null) {\n                    future.completeExceptionally(throwable);\n                }\n                for (CompletableFuture<List<BatchAckResult>> resultFuture : futures) {\n                    batchAckResultList.addAll(resultFuture.join());\n                }\n                future.complete(batchAckResultList);\n            });\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return FutureUtils.addExecutor(future, this.executor);\n    }\n\n    protected CompletableFuture<List<BatchAckResult>> processBrokerHandle(ProxyContext ctx, String consumerGroup,\n        String topic, List<ReceiptHandleMessage> handleMessageList, long timeoutMillis) {\n        return this.serviceManager.getMessageService().batchAckMessage(ctx, handleMessageList, consumerGroup, topic, timeoutMillis)\n            .thenApply(result -> {\n                List<BatchAckResult> results = new ArrayList<>();\n                for (ReceiptHandleMessage handleMessage : handleMessageList) {\n                    results.add(new BatchAckResult(handleMessage, result));\n                }\n                return results;\n            })\n            .exceptionally(throwable -> {\n                List<BatchAckResult> results = new ArrayList<>();\n                for (ReceiptHandleMessage handleMessage : handleMessageList) {\n                    results.add(new BatchAckResult(handleMessage, new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, throwable.getMessage(), throwable)));\n                }\n                return results;\n            });\n    }\n\n    public CompletableFuture<AckResult> changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, String messageId,\n        String groupName, String topicName, long invisibleTime, String liteTopic, long timeoutMillis, boolean suspend) {\n        CompletableFuture<AckResult> future = new CompletableFuture<>();\n        try {\n            this.validateReceiptHandle(handle);\n\n            ChangeInvisibleTimeRequestHeader changeInvisibleTimeRequestHeader = new ChangeInvisibleTimeRequestHeader();\n            changeInvisibleTimeRequestHeader.setConsumerGroup(groupName);\n            changeInvisibleTimeRequestHeader.setTopic(handle.getRealTopic(topicName, groupName));\n            changeInvisibleTimeRequestHeader.setQueueId(handle.getQueueId());\n            changeInvisibleTimeRequestHeader.setExtraInfo(handle.getReceiptHandle());\n            changeInvisibleTimeRequestHeader.setOffset(handle.getOffset());\n            changeInvisibleTimeRequestHeader.setInvisibleTime(invisibleTime);\n            changeInvisibleTimeRequestHeader.setLiteTopic(liteTopic);\n            changeInvisibleTimeRequestHeader.setSuspend(suspend);\n            long commitLogOffset = handle.getCommitLogOffset();\n\n            future = this.serviceManager.getMessageService().changeInvisibleTime(\n                    ctx,\n                    handle,\n                    messageId,\n                    changeInvisibleTimeRequestHeader,\n                    timeoutMillis)\n                .thenApplyAsync(ackResult -> {\n                    if (StringUtils.isNotBlank(ackResult.getExtraInfo())) {\n                        AckResult result = new AckResult();\n                        result.setStatus(ackResult.getStatus());\n                        result.setPopTime(result.getPopTime());\n                        result.setExtraInfo(createHandle(ackResult.getExtraInfo(), commitLogOffset));\n                        return result;\n                    } else {\n                        return ackResult;\n                    }\n                }, this.executor);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n            FutureUtils.addExecutor(future, this.executor);\n        }\n        return future;\n    }\n\n    protected String createHandle(String handleString, long commitLogOffset) {\n        if (handleString == null) {\n            return null;\n        }\n        return handleString + MessageConst.KEY_SEPARATOR + commitLogOffset;\n    }\n\n    public CompletableFuture<PullResult> pullMessage(ProxyContext ctx, MessageQueue messageQueue, String consumerGroup,\n        long queueOffset, int maxMsgNums, int sysFlag, long commitOffset,\n        long suspendTimeoutMillis, SubscriptionData subscriptionData, long timeoutMillis) {\n        CompletableFuture<PullResult> future = new CompletableFuture<>();\n        try {\n            AddressableMessageQueue addressableMessageQueue = serviceManager.getTopicRouteService()\n                .buildAddressableMessageQueue(ctx, messageQueue);\n            PullMessageRequestHeader requestHeader = new PullMessageRequestHeader();\n            requestHeader.setConsumerGroup(consumerGroup);\n            requestHeader.setTopic(addressableMessageQueue.getTopic());\n            requestHeader.setQueueId(addressableMessageQueue.getQueueId());\n            requestHeader.setQueueOffset(queueOffset);\n            requestHeader.setMaxMsgNums(maxMsgNums);\n            requestHeader.setSysFlag(sysFlag);\n            requestHeader.setCommitOffset(commitOffset);\n            requestHeader.setSuspendTimeoutMillis(suspendTimeoutMillis);\n            requestHeader.setSubscription(subscriptionData.getSubString());\n            requestHeader.setExpressionType(subscriptionData.getExpressionType());\n            future = serviceManager.getMessageService().pullMessage(ctx, addressableMessageQueue, requestHeader, timeoutMillis);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return FutureUtils.addExecutor(future, this.executor);\n    }\n\n    public CompletableFuture<Void> updateConsumerOffset(ProxyContext ctx, MessageQueue messageQueue,\n        String consumerGroup, long commitOffset, long timeoutMillis) {\n        CompletableFuture<Void> future = new CompletableFuture<>();\n        try {\n            AddressableMessageQueue addressableMessageQueue = serviceManager.getTopicRouteService()\n                .buildAddressableMessageQueue(ctx, messageQueue);\n            UpdateConsumerOffsetRequestHeader requestHeader = new UpdateConsumerOffsetRequestHeader();\n            requestHeader.setConsumerGroup(consumerGroup);\n            requestHeader.setTopic(addressableMessageQueue.getTopic());\n            requestHeader.setQueueId(addressableMessageQueue.getQueueId());\n            requestHeader.setCommitOffset(commitOffset);\n            future = serviceManager.getMessageService().updateConsumerOffset(ctx, addressableMessageQueue, requestHeader, timeoutMillis);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return FutureUtils.addExecutor(future, this.executor);\n    }\n\n    public CompletableFuture<Void> updateConsumerOffsetAsync(ProxyContext ctx, MessageQueue messageQueue,\n        String consumerGroup, long commitOffset, long timeoutMillis) {\n        CompletableFuture<Void> future = new CompletableFuture<>();\n        try {\n            AddressableMessageQueue addressableMessageQueue = serviceManager.getTopicRouteService()\n                .buildAddressableMessageQueue(ctx, messageQueue);\n            UpdateConsumerOffsetRequestHeader requestHeader = new UpdateConsumerOffsetRequestHeader();\n            requestHeader.setConsumerGroup(consumerGroup);\n            requestHeader.setTopic(addressableMessageQueue.getTopic());\n            requestHeader.setQueueId(addressableMessageQueue.getQueueId());\n            requestHeader.setCommitOffset(commitOffset);\n            future = serviceManager.getMessageService().updateConsumerOffsetAsync(ctx, addressableMessageQueue, requestHeader, timeoutMillis);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return FutureUtils.addExecutor(future, this.executor);\n    }\n\n    public CompletableFuture<Long> queryConsumerOffset(ProxyContext ctx, MessageQueue messageQueue,\n        String consumerGroup, long timeoutMillis) {\n        CompletableFuture<Long> future = new CompletableFuture<>();\n        try {\n            AddressableMessageQueue addressableMessageQueue = serviceManager.getTopicRouteService()\n                .buildAddressableMessageQueue(ctx, messageQueue);\n            QueryConsumerOffsetRequestHeader requestHeader = new QueryConsumerOffsetRequestHeader();\n            requestHeader.setConsumerGroup(consumerGroup);\n            requestHeader.setTopic(addressableMessageQueue.getTopic());\n            requestHeader.setQueueId(addressableMessageQueue.getQueueId());\n            future = serviceManager.getMessageService().queryConsumerOffset(ctx, addressableMessageQueue, requestHeader, timeoutMillis);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return FutureUtils.addExecutor(future, this.executor);\n    }\n\n    public CompletableFuture<Set<MessageQueue>> lockBatchMQ(ProxyContext ctx, Set<MessageQueue> mqSet,\n        String consumerGroup, String clientId, long timeoutMillis) {\n        CompletableFuture<Set<MessageQueue>> future = new CompletableFuture<>();\n        try {\n            Set<MessageQueue> successSet = new CopyOnWriteArraySet<>();\n            Set<AddressableMessageQueue> addressableMessageQueueSet = buildAddressableSet(ctx, mqSet);\n            Map<String, List<AddressableMessageQueue>> messageQueueSetMap = buildAddressableMapByBrokerName(addressableMessageQueueSet);\n            List<CompletableFuture<Void>> futureList = new ArrayList<>();\n            messageQueueSetMap.forEach((k, v) -> {\n                LockBatchRequestBody requestBody = new LockBatchRequestBody();\n                requestBody.setConsumerGroup(consumerGroup);\n                requestBody.setClientId(clientId);\n                requestBody.setMqSet(v.stream().map(AddressableMessageQueue::getMessageQueue).collect(Collectors.toSet()));\n                CompletableFuture<Void> future0 = serviceManager.getMessageService()\n                    .lockBatchMQ(ctx, v.get(0), requestBody, timeoutMillis)\n                    .thenAccept(successSet::addAll);\n                futureList.add(FutureUtils.addExecutor(future0, this.executor));\n            });\n            CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).whenComplete((v, t) -> {\n                if (t != null) {\n                    log.error(\"LockBatchMQ failed, group={}\", consumerGroup, t);\n                }\n                future.complete(successSet);\n            });\n        } catch (Throwable t) {\n            log.error(\"LockBatchMQ exception, group={}\", consumerGroup, t);\n            future.completeExceptionally(t);\n        }\n        return FutureUtils.addExecutor(future, this.executor);\n    }\n\n    public CompletableFuture<Void> unlockBatchMQ(ProxyContext ctx, Set<MessageQueue> mqSet,\n        String consumerGroup, String clientId, long timeoutMillis) {\n        CompletableFuture<Void> future = new CompletableFuture<>();\n        try {\n            Set<AddressableMessageQueue> addressableMessageQueueSet = buildAddressableSet(ctx, mqSet);\n            Map<String, List<AddressableMessageQueue>> messageQueueSetMap = buildAddressableMapByBrokerName(addressableMessageQueueSet);\n            List<CompletableFuture<Void>> futureList = new ArrayList<>();\n            messageQueueSetMap.forEach((k, v) -> {\n                UnlockBatchRequestBody requestBody = new UnlockBatchRequestBody();\n                requestBody.setConsumerGroup(consumerGroup);\n                requestBody.setClientId(clientId);\n                requestBody.setMqSet(v.stream().map(AddressableMessageQueue::getMessageQueue).collect(Collectors.toSet()));\n                CompletableFuture<Void> future0 = serviceManager.getMessageService().unlockBatchMQ(ctx, v.get(0), requestBody, timeoutMillis);\n                futureList.add(FutureUtils.addExecutor(future0, this.executor));\n            });\n            CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).whenComplete((v, t) -> {\n                if (t != null) {\n                    log.error(\"UnlockBatchMQ failed, group={}\", consumerGroup, t);\n                }\n                future.complete(null);\n            });\n        } catch (Throwable t) {\n            log.error(\"UnlockBatchMQ exception, group={}\", consumerGroup, t);\n            future.completeExceptionally(t);\n        }\n        return FutureUtils.addExecutor(future, this.executor);\n    }\n\n    public CompletableFuture<Long> getMaxOffset(ProxyContext ctx, MessageQueue messageQueue, long timeoutMillis) {\n        CompletableFuture<Long> future = new CompletableFuture<>();\n        try {\n            AddressableMessageQueue addressableMessageQueue = serviceManager.getTopicRouteService()\n                .buildAddressableMessageQueue(ctx, messageQueue);\n            GetMaxOffsetRequestHeader requestHeader = new GetMaxOffsetRequestHeader();\n            requestHeader.setTopic(addressableMessageQueue.getTopic());\n            requestHeader.setQueueId(addressableMessageQueue.getQueueId());\n            future = serviceManager.getMessageService().getMaxOffset(ctx, addressableMessageQueue, requestHeader, timeoutMillis);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return FutureUtils.addExecutor(future, this.executor);\n    }\n\n    public CompletableFuture<Long> getMinOffset(ProxyContext ctx, MessageQueue messageQueue, long timeoutMillis) {\n        CompletableFuture<Long> future = new CompletableFuture<>();\n        try {\n            AddressableMessageQueue addressableMessageQueue = serviceManager.getTopicRouteService()\n                .buildAddressableMessageQueue(ctx, messageQueue);\n            GetMinOffsetRequestHeader requestHeader = new GetMinOffsetRequestHeader();\n            requestHeader.setTopic(addressableMessageQueue.getTopic());\n            requestHeader.setQueueId(addressableMessageQueue.getQueueId());\n            future = serviceManager.getMessageService().getMinOffset(ctx, addressableMessageQueue, requestHeader, timeoutMillis);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return FutureUtils.addExecutor(future, this.executor);\n    }\n\n    protected Set<AddressableMessageQueue> buildAddressableSet(ProxyContext ctx, Set<MessageQueue> mqSet) {\n        Set<AddressableMessageQueue> addressableMessageQueueSet = new HashSet<>(mqSet.size());\n        for (MessageQueue mq : mqSet) {\n            try {\n                addressableMessageQueueSet.add(serviceManager.getTopicRouteService().buildAddressableMessageQueue(ctx, mq));\n            } catch (Exception e) {\n                log.error(\"build addressable message queue fail, messageQueue = {}\", mq, e);\n            }\n        }\n        return addressableMessageQueueSet;\n    }\n\n    protected HashMap<String, List<AddressableMessageQueue>> buildAddressableMapByBrokerName(\n        final Set<AddressableMessageQueue> mqSet) {\n        HashMap<String, List<AddressableMessageQueue>> result = new HashMap<>();\n        if (mqSet == null) {\n            return result;\n        }\n        for (AddressableMessageQueue mq : mqSet) {\n            if (mq == null) {\n                continue;\n            }\n            List<AddressableMessageQueue> mqs = result.computeIfAbsent(mq.getBrokerName(), k -> new ArrayList<>());\n            mqs.add(mq);\n        }\n        return result;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.processor;\n\nimport com.alibaba.fastjson2.JSON;\nimport io.netty.channel.Channel;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.acl.common.AclClientRPCHook;\nimport org.apache.rocketmq.acl.common.AclUtils;\nimport org.apache.rocketmq.acl.common.SessionCredentials;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ConsumerGroupInfo;\nimport org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;\nimport org.apache.rocketmq.broker.client.ProducerChangeListener;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.common.lite.LiteSubscriptionDTO;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.thread.ThreadPoolMonitor;\nimport org.apache.rocketmq.common.utils.AbstractStartAndShutdown;\nimport org.apache.rocketmq.proxy.common.Address;\nimport org.apache.rocketmq.proxy.common.MessageReceiptHandle;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.service.ServiceManager;\nimport org.apache.rocketmq.proxy.service.ServiceManagerFactory;\nimport org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;\nimport org.apache.rocketmq.proxy.service.metadata.MetadataService;\nimport org.apache.rocketmq.proxy.service.relay.ProxyRelayService;\nimport org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\n\npublic class DefaultMessagingProcessor extends AbstractStartAndShutdown implements MessagingProcessor {\n\n    protected ServiceManager serviceManager;\n    protected ProducerProcessor producerProcessor;\n    protected ConsumerProcessor consumerProcessor;\n    protected TransactionProcessor transactionProcessor;\n    protected ClientProcessor clientProcessor;\n    protected RequestBrokerProcessor requestBrokerProcessor;\n    protected ReceiptHandleProcessor receiptHandleProcessor;\n\n    protected ThreadPoolExecutor producerProcessorExecutor;\n    protected ThreadPoolExecutor consumerProcessorExecutor;\n    protected static final String ROCKETMQ_HOME = MixAll.ROCKETMQ_HOME_DIR;\n\n    protected DefaultMessagingProcessor(ServiceManager serviceManager) {\n        ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();\n        this.producerProcessorExecutor = ThreadPoolMonitor.createAndMonitor(\n            proxyConfig.getProducerProcessorThreadPoolNums(),\n            proxyConfig.getProducerProcessorThreadPoolNums(),\n            1,\n            TimeUnit.MINUTES,\n            \"ProducerProcessorExecutor\",\n            proxyConfig.getProducerProcessorThreadPoolQueueCapacity()\n        );\n        this.consumerProcessorExecutor = ThreadPoolMonitor.createAndMonitor(\n            proxyConfig.getConsumerProcessorThreadPoolNums(),\n            proxyConfig.getConsumerProcessorThreadPoolNums(),\n            1,\n            TimeUnit.MINUTES,\n            \"ConsumerProcessorExecutor\",\n            proxyConfig.getConsumerProcessorThreadPoolQueueCapacity()\n        );\n\n        this.serviceManager = serviceManager;\n        this.producerProcessor = new ProducerProcessor(this, serviceManager, this.producerProcessorExecutor);\n        this.consumerProcessor = new ConsumerProcessor(this, serviceManager, this.consumerProcessorExecutor);\n        this.transactionProcessor = new TransactionProcessor(this, serviceManager);\n        this.clientProcessor = new ClientProcessor(this, serviceManager);\n        this.requestBrokerProcessor = new RequestBrokerProcessor(this, serviceManager);\n        this.receiptHandleProcessor = new ReceiptHandleProcessor(this, serviceManager);\n\n        this.init();\n    }\n\n    public static DefaultMessagingProcessor createForLocalMode(BrokerController brokerController) {\n        return createForLocalMode(brokerController, null);\n    }\n\n    public static DefaultMessagingProcessor createForLocalMode(BrokerController brokerController, RPCHook rpcHook) {\n        return new DefaultMessagingProcessor(ServiceManagerFactory.createForLocalMode(brokerController, rpcHook));\n    }\n\n    public static DefaultMessagingProcessor createForClusterMode() {\n        RPCHook rpcHook = null;\n        if (!ConfigurationManager.getProxyConfig().isEnableAclRpcHookForClusterMode()) {\n            return createForClusterMode(rpcHook);\n        }\n        AuthConfig authConfig = ConfigurationManager.getAuthConfig();\n        if (StringUtils.isNotBlank(authConfig.getInnerClientAuthenticationCredentials())) {\n            SessionCredentials sessionCredentials =\n                JSON.parseObject(authConfig.getInnerClientAuthenticationCredentials(), SessionCredentials.class);\n            if (StringUtils.isNotBlank(sessionCredentials.getAccessKey()) && StringUtils.isNotBlank(sessionCredentials.getSecretKey())) {\n                rpcHook = new AclClientRPCHook(sessionCredentials);\n            }\n        } else {\n            rpcHook = AclUtils.getAclRPCHook(ROCKETMQ_HOME + MixAll.ACL_CONF_TOOLS_FILE);\n        }\n        return createForClusterMode(rpcHook);\n    }\n\n    public static DefaultMessagingProcessor createForClusterMode(RPCHook rpcHook) {\n        return new DefaultMessagingProcessor(ServiceManagerFactory.createForClusterMode(rpcHook));\n    }\n\n    protected void init() {\n        this.appendStartAndShutdown(this.serviceManager);\n        this.appendStartAndShutdown(this.receiptHandleProcessor);\n        this.appendShutdown(this.producerProcessorExecutor::shutdown);\n        this.appendShutdown(this.consumerProcessorExecutor::shutdown);\n    }\n\n    @Override\n    public SubscriptionGroupConfig getSubscriptionGroupConfig(ProxyContext ctx, String consumerGroupName) {\n        return this.serviceManager.getMetadataService().getSubscriptionGroupConfig(ctx, consumerGroupName);\n    }\n\n    @Override\n    public ProxyTopicRouteData getTopicRouteDataForProxy(ProxyContext ctx, List<Address> requestHostAndPortList,\n        String topicName) throws Exception {\n        return this.serviceManager.getTopicRouteService().getTopicRouteForProxy(ctx, requestHostAndPortList, topicName);\n    }\n\n    @Override\n    public CompletableFuture<List<SendResult>> sendMessage(ProxyContext ctx, QueueSelector queueSelector,\n        String producerGroup, int sysFlag, List<Message> msg, long timeoutMillis) {\n        return this.producerProcessor.sendMessage(ctx, queueSelector, producerGroup, sysFlag, msg, timeoutMillis);\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> forwardMessageToDeadLetterQueue(ProxyContext ctx, ReceiptHandle handle,\n        String messageId, String groupName, String topicName, long timeoutMillis) {\n        return this.producerProcessor.forwardMessageToDeadLetterQueue(ctx,\n            handle, messageId, groupName, topicName, null, timeoutMillis);\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> forwardMessageToDeadLetterQueue(ProxyContext ctx, ReceiptHandle handle,\n        String messageId, String groupName, String topicName, String liteTopic, long timeoutMillis) {\n        return this.producerProcessor.forwardMessageToDeadLetterQueue(ctx,\n            handle, messageId, groupName, topicName, liteTopic, timeoutMillis);\n    }\n\n    @Override\n    public CompletableFuture<Void> endTransaction(ProxyContext ctx, String topic, String transactionId,\n        String messageId, String producerGroup,\n        TransactionStatus transactionStatus, boolean fromTransactionCheck,\n        long timeoutMillis) {\n        return this.transactionProcessor.endTransaction(ctx, topic, transactionId, messageId, producerGroup, transactionStatus, fromTransactionCheck, timeoutMillis);\n    }\n\n    @Override\n    public CompletableFuture<PopResult> popMessage(\n        ProxyContext ctx,\n        QueueSelector queueSelector,\n        String consumerGroup,\n        String topic,\n        int maxMsgNums,\n        long invisibleTime,\n        long pollTime,\n        int initMode,\n        SubscriptionData subscriptionData,\n        boolean fifo,\n        PopMessageResultFilter popMessageResultFilter,\n        String attemptId,\n        long timeoutMillis\n    ) {\n        return this.consumerProcessor.popMessage(ctx, queueSelector, consumerGroup, topic, maxMsgNums,\n            invisibleTime, pollTime, initMode, subscriptionData, fifo, popMessageResultFilter, attemptId, timeoutMillis);\n    }\n\n    @Override\n    public CompletableFuture<PopResult> popLiteMessage(ProxyContext ctx, QueueSelector queueSelector,\n        String consumerGroup, String topic, int maxMsgNums, long invisibleTime, long pollTime,\n        SubscriptionData subscriptionData, PopMessageResultFilter popMessageResultFilter,\n        String attemptId, long timeoutMillis) {\n        return this.consumerProcessor.popLiteMessage(ctx, queueSelector,\n            consumerGroup, topic, maxMsgNums, invisibleTime, pollTime,\n            subscriptionData, popMessageResultFilter, attemptId, timeoutMillis);\n    }\n\n    @Override\n    public CompletableFuture<AckResult> ackMessage(ProxyContext ctx, ReceiptHandle handle, String messageId,\n        String consumerGroup, String topic, long timeoutMillis) {\n        return this.consumerProcessor.ackMessage(ctx, handle, messageId, consumerGroup, topic, null, timeoutMillis);\n    }\n\n    @Override\n    public CompletableFuture<AckResult> ackMessage(ProxyContext ctx, ReceiptHandle handle, String messageId,\n        String consumerGroup, String topic, String liteTopic, long timeoutMillis) {\n        return this.consumerProcessor.ackMessage(ctx, handle, messageId, consumerGroup, topic, liteTopic, timeoutMillis);\n    }\n\n    @Override\n    public CompletableFuture<List<BatchAckResult>> batchAckMessage(ProxyContext ctx,\n        List<ReceiptHandleMessage> handleMessageList, String consumerGroup, String topic, long timeoutMillis) {\n        return this.consumerProcessor.batchAckMessage(ctx, handleMessageList, consumerGroup, topic, timeoutMillis);\n    }\n\n    @Override\n    public CompletableFuture<AckResult> changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, String messageId,\n        String groupName, String topicName, long invisibleTime, String liteTopic, long timeoutMillis, boolean suspend) {\n        return this.consumerProcessor.changeInvisibleTime(ctx, handle, messageId, groupName, topicName,\n            invisibleTime, liteTopic, timeoutMillis, suspend);\n    }\n\n    @Override\n    public CompletableFuture<PullResult> pullMessage(ProxyContext ctx, MessageQueue messageQueue, String consumerGroup,\n        long queueOffset, int maxMsgNums, int sysFlag, long commitOffset, long suspendTimeoutMillis,\n        SubscriptionData subscriptionData, long timeoutMillis) {\n        return this.consumerProcessor.pullMessage(ctx, messageQueue, consumerGroup, queueOffset, maxMsgNums,\n            sysFlag, commitOffset, suspendTimeoutMillis, subscriptionData, timeoutMillis);\n    }\n\n    @Override\n    public CompletableFuture<Void> updateConsumerOffset(ProxyContext ctx, MessageQueue messageQueue,\n        String consumerGroup, long commitOffset, long timeoutMillis) {\n        return this.consumerProcessor.updateConsumerOffset(ctx, messageQueue, consumerGroup, commitOffset, timeoutMillis);\n    }\n\n    @Override\n    public CompletableFuture<Void> updateConsumerOffsetAsync(ProxyContext ctx, MessageQueue messageQueue,\n        String consumerGroup, long commitOffset, long timeoutMillis) {\n        return this.consumerProcessor.updateConsumerOffsetAsync(ctx, messageQueue, consumerGroup, commitOffset, timeoutMillis);\n    }\n\n    @Override\n    public CompletableFuture<Long> queryConsumerOffset(ProxyContext ctx, MessageQueue messageQueue,\n        String consumerGroup, long timeoutMillis) {\n        return this.consumerProcessor.queryConsumerOffset(ctx, messageQueue, consumerGroup, timeoutMillis);\n    }\n\n    @Override\n    public CompletableFuture<Set<MessageQueue>> lockBatchMQ(ProxyContext ctx, Set<MessageQueue> mqSet,\n        String consumerGroup, String clientId, long timeoutMillis) {\n        return this.consumerProcessor.lockBatchMQ(ctx, mqSet, consumerGroup, clientId, timeoutMillis);\n    }\n\n    @Override\n    public CompletableFuture<Void> unlockBatchMQ(ProxyContext ctx, Set<MessageQueue> mqSet,\n        String consumerGroup,\n        String clientId, long timeoutMillis) {\n        return this.consumerProcessor.unlockBatchMQ(ctx, mqSet, consumerGroup, clientId, timeoutMillis);\n    }\n\n    @Override\n    public CompletableFuture<Long> getMaxOffset(ProxyContext ctx, MessageQueue messageQueue, long timeoutMillis) {\n        return this.consumerProcessor.getMaxOffset(ctx, messageQueue, timeoutMillis);\n    }\n\n    @Override\n    public CompletableFuture<Long> getMinOffset(ProxyContext ctx, MessageQueue messageQueue, long timeoutMillis) {\n        return this.consumerProcessor.getMinOffset(ctx, messageQueue, timeoutMillis);\n    }\n\n    @Override\n    public CompletableFuture<String> recallMessage(ProxyContext ctx, String topic,\n                                                   String recallHandle, long timeoutMillis) {\n        return this.producerProcessor.recallMessage(ctx, topic, recallHandle, timeoutMillis);\n    }\n\n    @Override\n    public CompletableFuture<Void> syncLiteSubscription(ProxyContext ctx,\n        LiteSubscriptionDTO liteSubscriptionDTO, long timeoutMillis) {\n        return this.clientProcessor.syncLiteSubscription(ctx, liteSubscriptionDTO, timeoutMillis);\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> request(ProxyContext ctx, String brokerName, RemotingCommand request,\n        long timeoutMillis) {\n        int originalRequestOpaque = request.getOpaque();\n        request.setOpaque(RemotingCommand.createNewRequestId());\n        return this.requestBrokerProcessor.request(ctx, brokerName, request, timeoutMillis).thenApply(r -> {\n            request.setOpaque(originalRequestOpaque);\n            return r;\n        });\n    }\n\n    @Override\n    public CompletableFuture<Void> requestOneway(ProxyContext ctx, String brokerName, RemotingCommand request,\n        long timeoutMillis) {\n        int originalRequestOpaque = request.getOpaque();\n        request.setOpaque(RemotingCommand.createNewRequestId());\n        return this.requestBrokerProcessor.requestOneway(ctx, brokerName, request, timeoutMillis).thenApply(r -> {\n            request.setOpaque(originalRequestOpaque);\n            return r;\n        });\n    }\n\n    @Override\n    public void registerProducer(ProxyContext ctx, String producerGroup, ClientChannelInfo clientChannelInfo) {\n        this.clientProcessor.registerProducer(ctx, producerGroup, clientChannelInfo);\n    }\n\n    @Override\n    public void unRegisterProducer(ProxyContext ctx, String producerGroup, ClientChannelInfo clientChannelInfo) {\n        this.clientProcessor.unRegisterProducer(ctx, producerGroup, clientChannelInfo);\n    }\n\n    @Override\n    public Channel findProducerChannel(ProxyContext ctx, String producerGroup, String clientId) {\n        return this.clientProcessor.findProducerChannel(ctx, producerGroup, clientId);\n    }\n\n    @Override\n    public void registerProducerListener(ProducerChangeListener producerChangeListener) {\n        this.clientProcessor.registerProducerChangeListener(producerChangeListener);\n    }\n\n    @Override\n    public void registerConsumer(ProxyContext ctx, String consumerGroup, ClientChannelInfo clientChannelInfo,\n        ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,\n        Set<SubscriptionData> subList, boolean updateSubscription) {\n        this.clientProcessor.registerConsumer(ctx, consumerGroup, clientChannelInfo, consumeType, messageModel, consumeFromWhere, subList, updateSubscription);\n    }\n\n    @Override\n    public ClientChannelInfo findConsumerChannel(ProxyContext ctx, String consumerGroup, Channel channel) {\n        return this.clientProcessor.findConsumerChannel(ctx, consumerGroup, channel);\n    }\n\n    @Override\n    public void unRegisterConsumer(ProxyContext ctx, String consumerGroup, ClientChannelInfo clientChannelInfo) {\n        this.clientProcessor.unRegisterConsumer(ctx, consumerGroup, clientChannelInfo);\n    }\n\n    @Override\n    public void registerConsumerListener(ConsumerIdsChangeListener consumerIdsChangeListener) {\n        this.clientProcessor.registerConsumerIdsChangeListener(consumerIdsChangeListener);\n    }\n\n    @Override\n    public void doChannelCloseEvent(String remoteAddr, Channel channel) {\n        this.clientProcessor.doChannelCloseEvent(remoteAddr, channel);\n    }\n\n    @Override\n    public ConsumerGroupInfo getConsumerGroupInfo(ProxyContext ctx, String consumerGroup) {\n        return this.clientProcessor.getConsumerGroupInfo(ctx, consumerGroup);\n    }\n\n    @Override\n    public void addTransactionSubscription(ProxyContext ctx, String producerGroup, String topic) {\n        this.transactionProcessor.addTransactionSubscription(ctx, producerGroup, topic);\n    }\n\n    @Override\n    public ProxyRelayService getProxyRelayService() {\n        return this.serviceManager.getProxyRelayService();\n    }\n\n    @Override\n    public MetadataService getMetadataService() {\n        return this.serviceManager.getMetadataService();\n    }\n\n    @Override\n    public void addReceiptHandle(ProxyContext ctx, Channel channel, String group, String msgID,\n        MessageReceiptHandle messageReceiptHandle) {\n        receiptHandleProcessor.addReceiptHandle(ctx, channel, group, msgID, messageReceiptHandle);\n    }\n\n    @Override\n    public MessageReceiptHandle removeReceiptHandle(ProxyContext ctx, Channel channel, String group, String msgID,\n        String receiptHandle) {\n        return receiptHandleProcessor.removeReceiptHandle(ctx, channel, group, msgID, receiptHandle);\n    }\n\n    @Override public int getUnackedMessageCount(ProxyContext ctx, Channel channel, String group) {\n        return receiptHandleProcessor.getUnackedMessageCount(ctx, channel, group);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.processor;\n\nimport io.netty.channel.Channel;\nimport java.time.Duration;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ConsumerGroupInfo;\nimport org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;\nimport org.apache.rocketmq.broker.client.ProducerChangeListener;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.common.lite.LiteSubscriptionDTO;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.utils.StartAndShutdown;\nimport org.apache.rocketmq.proxy.common.Address;\nimport org.apache.rocketmq.proxy.common.MessageReceiptHandle;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;\nimport org.apache.rocketmq.proxy.service.metadata.MetadataService;\nimport org.apache.rocketmq.proxy.service.relay.ProxyRelayService;\nimport org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\n\npublic interface MessagingProcessor extends StartAndShutdown {\n\n    long DEFAULT_TIMEOUT_MILLS = Duration.ofSeconds(2).toMillis();\n\n    long INVISIBLE_TIME_MS = Duration.ofSeconds(1).toMillis();\n\n    SubscriptionGroupConfig getSubscriptionGroupConfig(\n        ProxyContext ctx,\n        String consumerGroupName\n    );\n\n    ProxyTopicRouteData getTopicRouteDataForProxy(\n        ProxyContext ctx,\n        List<Address> requestHostAndPortList,\n        String topicName\n    ) throws Exception;\n\n    default CompletableFuture<List<SendResult>> sendMessage(\n        ProxyContext ctx,\n        QueueSelector queueSelector,\n        String producerGroup,\n        int sysFlag,\n        List<Message> msg\n    ) {\n        return sendMessage(ctx, queueSelector, producerGroup, sysFlag, msg, DEFAULT_TIMEOUT_MILLS);\n    }\n\n    CompletableFuture<List<SendResult>> sendMessage(\n        ProxyContext ctx,\n        QueueSelector queueSelector,\n        String producerGroup,\n        int sysFlag,\n        List<Message> msg,\n        long timeoutMillis\n    );\n\n    default CompletableFuture<RemotingCommand> forwardMessageToDeadLetterQueue(\n        ProxyContext ctx,\n        ReceiptHandle handle,\n        String messageId,\n        String groupName,\n        String topicName\n    ) {\n        return forwardMessageToDeadLetterQueue(ctx, handle, messageId, groupName, topicName, DEFAULT_TIMEOUT_MILLS);\n    }\n\n    CompletableFuture<RemotingCommand> forwardMessageToDeadLetterQueue(\n        ProxyContext ctx,\n        ReceiptHandle handle,\n        String messageId,\n        String groupName,\n        String topicName,\n        long timeoutMillis\n    );\n\n    default CompletableFuture<RemotingCommand> forwardMessageToDeadLetterQueue(\n        ProxyContext ctx,\n        ReceiptHandle handle,\n        String messageId,\n        String groupName,\n        String topicName,\n        String liteTopic\n    ) {\n        return forwardMessageToDeadLetterQueue(ctx, handle, messageId, groupName, topicName, liteTopic, DEFAULT_TIMEOUT_MILLS);\n    }\n\n    CompletableFuture<RemotingCommand> forwardMessageToDeadLetterQueue(\n        ProxyContext ctx,\n        ReceiptHandle handle,\n        String messageId,\n        String groupName,\n        String topicName,\n        String liteTopic,\n        long timeoutMillis\n    );\n\n    default CompletableFuture<Void> endTransaction(\n        ProxyContext ctx,\n        String topic,\n        String transactionId,\n        String messageId,\n        String producerGroup,\n        TransactionStatus transactionStatus,\n        boolean fromTransactionCheck\n    ) {\n        return endTransaction(ctx, topic, transactionId, messageId, producerGroup, transactionStatus, fromTransactionCheck, DEFAULT_TIMEOUT_MILLS);\n    }\n\n    CompletableFuture<Void> endTransaction(\n        ProxyContext ctx,\n        String topic,\n        String transactionId,\n        String messageId,\n        String producerGroup,\n        TransactionStatus transactionStatus,\n        boolean fromTransactionCheck,\n        long timeoutMillis\n    );\n\n    CompletableFuture<PopResult> popMessage(\n        ProxyContext ctx,\n        QueueSelector queueSelector,\n        String consumerGroup,\n        String topic,\n        int maxMsgNums,\n        long invisibleTime,\n        long pollTime,\n        int initMode,\n        SubscriptionData subscriptionData,\n        boolean fifo,\n        PopMessageResultFilter popMessageResultFilter,\n        String attemptId,\n        long timeoutMillis\n    );\n\n    CompletableFuture<PopResult> popLiteMessage(\n        ProxyContext ctx,\n        QueueSelector queueSelector,\n        String consumerGroup,\n        String topic,\n        int maxMsgNums,\n        long invisibleTime,\n        long pollTime,\n        SubscriptionData subscriptionData,\n        PopMessageResultFilter popMessageResultFilter,\n        String attemptId,\n        long timeoutMillis\n    );\n\n    default CompletableFuture<AckResult> ackMessage(\n        ProxyContext ctx,\n        ReceiptHandle handle,\n        String messageId,\n        String consumerGroup,\n        String topic\n    ) {\n        return ackMessage(ctx, handle, messageId, consumerGroup, topic, DEFAULT_TIMEOUT_MILLS);\n    }\n\n    CompletableFuture<AckResult> ackMessage(\n        ProxyContext ctx,\n        ReceiptHandle handle,\n        String messageId,\n        String consumerGroup,\n        String topic,\n        long timeoutMillis\n    );\n\n    default CompletableFuture<AckResult> ackMessage(\n        ProxyContext ctx,\n        ReceiptHandle handle,\n        String messageId,\n        String consumerGroup,\n        String topic,\n        String liteTopic\n    ) {\n        return ackMessage(ctx, handle, messageId, consumerGroup, topic, liteTopic, DEFAULT_TIMEOUT_MILLS);\n    }\n\n    CompletableFuture<AckResult> ackMessage(\n        ProxyContext ctx,\n        ReceiptHandle handle,\n        String messageId,\n        String consumerGroup,\n        String topic,\n        String liteTopic,\n        long timeoutMillis\n    );\n\n    default CompletableFuture<List<BatchAckResult>> batchAckMessage(\n        ProxyContext ctx,\n        List<ReceiptHandleMessage> handleMessageList,\n        String consumerGroup,\n        String topic\n    ) {\n        return batchAckMessage(ctx, handleMessageList, consumerGroup, topic, DEFAULT_TIMEOUT_MILLS);\n    }\n\n    CompletableFuture<List<BatchAckResult>> batchAckMessage(\n        ProxyContext ctx,\n        List<ReceiptHandleMessage> handleMessageList,\n        String consumerGroup,\n        String topic,\n        long timeoutMillis\n    );\n\n    default CompletableFuture<AckResult> changeInvisibleTime(\n        ProxyContext ctx,\n        ReceiptHandle handle,\n        String messageId,\n        String groupName,\n        String topicName,\n        long invisibleTime\n    ) {\n        return changeInvisibleTime(ctx, handle, messageId, groupName, topicName, invisibleTime, DEFAULT_TIMEOUT_MILLS);\n    }\n\n    default CompletableFuture<AckResult> changeInvisibleTime(\n        ProxyContext ctx,\n        ReceiptHandle handle,\n        String messageId,\n        String groupName,\n        String topicName,\n        long invisibleTime,\n        long timeoutMillis\n    ) {\n        return changeInvisibleTime(ctx, handle, messageId, groupName, topicName, invisibleTime, null, timeoutMillis, false);\n    }\n\n    default CompletableFuture<AckResult> changeInvisibleTime(\n        ProxyContext ctx,\n        ReceiptHandle handle,\n        String messageId,\n        String groupName,\n        String topicName,\n        long invisibleTime,\n        String liteTopic\n    ) {\n        return changeInvisibleTime(ctx, handle, messageId, groupName, topicName, invisibleTime, liteTopic, DEFAULT_TIMEOUT_MILLS, false);\n    }\n\n    CompletableFuture<AckResult> changeInvisibleTime(\n        ProxyContext ctx,\n        ReceiptHandle handle,\n        String messageId,\n        String groupName,\n        String topicName,\n        long invisibleTime,\n        String liteTopic,\n        long timeoutMillis,\n        boolean suspend\n    );\n\n    CompletableFuture<PullResult> pullMessage(\n        ProxyContext ctx,\n        MessageQueue messageQueue,\n        String consumerGroup,\n        long queueOffset,\n        int maxMsgNums,\n        int sysFlag,\n        long commitOffset,\n        long suspendTimeoutMillis,\n        SubscriptionData subscriptionData,\n        long timeoutMillis\n    );\n\n    CompletableFuture<Void> updateConsumerOffset(\n        ProxyContext ctx,\n        MessageQueue messageQueue,\n        String consumerGroup,\n        long commitOffset,\n        long timeoutMillis\n    );\n\n    CompletableFuture<Void> updateConsumerOffsetAsync(\n        ProxyContext ctx,\n        MessageQueue messageQueue,\n        String consumerGroup,\n        long commitOffset,\n        long timeoutMillis\n    );\n\n    CompletableFuture<Long> queryConsumerOffset(\n        ProxyContext ctx,\n        MessageQueue messageQueue,\n        String consumerGroup,\n        long timeoutMillis\n    );\n\n    CompletableFuture<Set<MessageQueue>> lockBatchMQ(\n        ProxyContext ctx,\n        Set<MessageQueue> mqSet,\n        String consumerGroup,\n        String clientId,\n        long timeoutMillis\n    );\n\n    CompletableFuture<Void> unlockBatchMQ(\n        ProxyContext ctx,\n        Set<MessageQueue> mqSet,\n        String consumerGroup,\n        String clientId,\n        long timeoutMillis\n    );\n\n    CompletableFuture<Long> getMaxOffset(\n        ProxyContext ctx,\n        MessageQueue messageQueue,\n        long timeoutMillis\n    );\n\n    CompletableFuture<Long> getMinOffset(\n        ProxyContext ctx,\n        MessageQueue messageQueue,\n        long timeoutMillis\n    );\n\n    CompletableFuture<String> recallMessage(\n        ProxyContext ctx,\n        String topic,\n        String recallHandle,\n        long timeoutMillis\n    );\n\n    CompletableFuture<Void> syncLiteSubscription(\n        ProxyContext ctx,\n        LiteSubscriptionDTO liteSubscriptionDTO,\n        long timeoutMillis\n    );\n\n    CompletableFuture<RemotingCommand> request(ProxyContext ctx, String brokerName, RemotingCommand request,\n        long timeoutMillis);\n\n    CompletableFuture<Void> requestOneway(ProxyContext ctx, String brokerName, RemotingCommand request,\n        long timeoutMillis);\n\n    void registerProducer(\n        ProxyContext ctx,\n        String producerGroup,\n        ClientChannelInfo clientChannelInfo\n    );\n\n    void unRegisterProducer(\n        ProxyContext ctx,\n        String producerGroup,\n        ClientChannelInfo clientChannelInfo\n    );\n\n    Channel findProducerChannel(\n        ProxyContext ctx,\n        String producerGroup,\n        String clientId\n    );\n\n    void registerProducerListener(\n        ProducerChangeListener producerChangeListener\n    );\n\n    void registerConsumer(\n        ProxyContext ctx,\n        String consumerGroup,\n        ClientChannelInfo clientChannelInfo,\n        ConsumeType consumeType,\n        MessageModel messageModel,\n        ConsumeFromWhere consumeFromWhere,\n        Set<SubscriptionData> subList,\n        boolean updateSubscription\n    );\n\n    ClientChannelInfo findConsumerChannel(\n        ProxyContext ctx,\n        String consumerGroup,\n        Channel channel\n    );\n\n    void unRegisterConsumer(\n        ProxyContext ctx,\n        String consumerGroup,\n        ClientChannelInfo clientChannelInfo\n    );\n\n    void registerConsumerListener(\n        ConsumerIdsChangeListener consumerIdsChangeListener\n    );\n\n    void doChannelCloseEvent(String remoteAddr, Channel channel);\n\n    ConsumerGroupInfo getConsumerGroupInfo(ProxyContext ctx, String consumerGroup);\n\n    void addTransactionSubscription(\n        ProxyContext ctx,\n        String producerGroup,\n        String topic\n    );\n\n    ProxyRelayService getProxyRelayService();\n\n    MetadataService getMetadataService();\n\n    void addReceiptHandle(ProxyContext ctx, Channel channel, String group, String msgID,\n        MessageReceiptHandle messageReceiptHandle);\n\n    MessageReceiptHandle removeReceiptHandle(ProxyContext ctx, Channel channel, String group, String msgID,\n        String receiptHandle);\n\n    int getUnackedMessageCount(ProxyContext ctx, Channel channel, String group);\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/processor/PopMessageResultFilter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.processor;\n\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\npublic interface PopMessageResultFilter {\n\n    enum FilterResult {\n        TO_DLQ,\n        NO_MATCH,\n        MATCH,\n        TO_RETURN\n    }\n\n    FilterResult filterMessage(ProxyContext ctx, String consumerGroup, SubscriptionData subscriptionData,\n        MessageExt messageExt);\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.processor;\n\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ExecutorService;\nimport org.apache.commons.codec.DecoderException;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageId;\nimport org.apache.rocketmq.common.producer.RecallMessageHandle;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.common.utils.FutureUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.processor.validator.DefaultTopicMessageTypeValidator;\nimport org.apache.rocketmq.proxy.processor.validator.TopicMessageTypeValidator;\nimport org.apache.rocketmq.proxy.service.ServiceManager;\nimport org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\n\npublic class ProducerProcessor extends AbstractProcessor {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    private final ExecutorService executor;\n    private final TopicMessageTypeValidator topicMessageTypeValidator;\n\n    public ProducerProcessor(MessagingProcessor messagingProcessor,\n        ServiceManager serviceManager, ExecutorService executor) {\n        super(messagingProcessor, serviceManager);\n        this.executor = executor;\n        this.topicMessageTypeValidator = new DefaultTopicMessageTypeValidator();\n    }\n\n    public CompletableFuture<List<SendResult>> sendMessage(ProxyContext ctx, QueueSelector queueSelector,\n        String producerGroup, int sysFlag, List<Message> messageList, long timeoutMillis) {\n        CompletableFuture<List<SendResult>> future = new CompletableFuture<>();\n        long beginTimestampFirst = System.currentTimeMillis();\n        AddressableMessageQueue messageQueue = null;\n        try {\n            Message message = messageList.get(0);\n            String topic = message.getTopic();\n            if (isNeedCheckTopicMessageType(message)) {\n                if (topicMessageTypeValidator != null) {\n                    // Do not check retry or dlq topic\n                    if (!NamespaceUtil.isRetryTopic(topic) && !NamespaceUtil.isDLQTopic(topic)) {\n                        TopicMessageType topicMessageType = serviceManager.getMetadataService().getTopicMessageType(ctx, topic);\n                        TopicMessageType messageType = TopicMessageType.parseFromMessageProperty(message.getProperties());\n                        topicMessageTypeValidator.validate(topicMessageType, messageType);\n                    }\n                }\n            }\n            messageQueue = queueSelector.select(ctx,\n                this.serviceManager.getTopicRouteService().getCurrentMessageQueueView(ctx, topic));\n            if (messageQueue == null) {\n                throw new ProxyException(ProxyExceptionCode.FORBIDDEN, \"no writable queue\");\n            }\n\n            for (Message msg : messageList) {\n                MessageClientIDSetter.setUniqID(msg);\n            }\n            SendMessageRequestHeader requestHeader = buildSendMessageRequestHeader(messageList, producerGroup, sysFlag, messageQueue.getQueueId());\n\n            AddressableMessageQueue finalMessageQueue = messageQueue;\n            future = this.serviceManager.getMessageService().sendMessage(\n                ctx,\n                messageQueue,\n                messageList,\n                requestHeader,\n                timeoutMillis)\n                .whenCompleteAsync((sendResultList, throwable) -> {\n                    long endTimestamp = System.currentTimeMillis();\n                    if (throwable == null) {\n                        for (SendResult sendResult : sendResultList) {\n                            int tranType = MessageSysFlag.getTransactionValue(requestHeader.getSysFlag());\n                            if (SendStatus.SEND_OK.equals(sendResult.getSendStatus()) &&\n                                tranType == MessageSysFlag.TRANSACTION_PREPARED_TYPE &&\n                                StringUtils.isNotBlank(sendResult.getTransactionId())) {\n                                fillTransactionData(ctx, producerGroup, finalMessageQueue, sendResult, messageList);\n                            }\n                        }\n                        this.serviceManager.getTopicRouteService().updateFaultItem(finalMessageQueue.getBrokerName(), endTimestamp - beginTimestampFirst, false, true);\n                    } else {\n                        this.serviceManager.getTopicRouteService().updateFaultItem(finalMessageQueue.getBrokerName(), endTimestamp - beginTimestampFirst, true, false);\n                    }\n                }, this.executor);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n            FutureUtils.addExecutor(future, this.executor);\n        }\n        return future;\n    }\n\n    public CompletableFuture<String> recallMessage(ProxyContext ctx, String topic,\n                                                   String recallHandle, long timeoutMillis) {\n        CompletableFuture<String> future = new CompletableFuture<>();\n        try {\n            if (ConfigurationManager.getProxyConfig().isEnableTopicMessageTypeCheck()) {\n                TopicMessageType messageType = serviceManager.getMetadataService().getTopicMessageType(ctx, topic);\n                topicMessageTypeValidator.validate(messageType, TopicMessageType.DELAY);\n            }\n\n            RecallMessageHandle.HandleV1 handleEntity;\n            try {\n                handleEntity = (RecallMessageHandle.HandleV1) RecallMessageHandle.decodeHandle(recallHandle);\n            } catch (DecoderException e) {\n                throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, e.getMessage());\n            }\n            String brokerName = handleEntity.getBrokerName();\n            RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader();\n            requestHeader.setTopic(topic);\n            requestHeader.setRecallHandle(recallHandle);\n            requestHeader.setBrokerName(brokerName);\n            future = serviceManager.getMessageService().recallMessage(ctx, brokerName, requestHeader, timeoutMillis);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return FutureUtils.addExecutor(future, this.executor);\n    }\n\n    protected void fillTransactionData(ProxyContext ctx, String producerGroup, AddressableMessageQueue messageQueue, SendResult sendResult, List<Message> messageList) {\n        try {\n            MessageId id;\n            if (sendResult.getOffsetMsgId() != null) {\n                id = MessageDecoder.decodeMessageId(sendResult.getOffsetMsgId());\n            } else {\n                id = MessageDecoder.decodeMessageId(sendResult.getMsgId());\n            }\n            this.serviceManager.getTransactionService().addTransactionDataByBrokerName(\n                ctx,\n                messageQueue.getBrokerName(),\n                messageList.get(0).getTopic(),\n                producerGroup,\n                sendResult.getQueueOffset(),\n                id.getOffset(),\n                sendResult.getTransactionId(),\n                messageList.get(0)\n            );\n        } catch (Throwable t) {\n            log.warn(\"fillTransactionData failed. messageQueue: {}, sendResult: {}\", messageQueue, sendResult, t);\n        }\n    }\n\n    protected SendMessageRequestHeader buildSendMessageRequestHeader(List<Message> messageList,\n        String producerGroup, int sysFlag, int queueId) {\n        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();\n\n        Message message = messageList.get(0);\n\n        requestHeader.setProducerGroup(producerGroup);\n        requestHeader.setTopic(message.getTopic());\n        requestHeader.setDefaultTopic(TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC);\n        requestHeader.setDefaultTopicQueueNums(4);\n        requestHeader.setQueueId(queueId);\n        requestHeader.setSysFlag(sysFlag);\n        /*\n        In RocketMQ 4.0, org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader.bornTimestamp\n        represents the timestamp when the message was born. In RocketMQ 5.0, the bornTimestamp of the message\n        is a message attribute, that is, the timestamp when message was constructed, and there is no\n        bornTimestamp in the SendMessageRequest of RocketMQ 5.0.\n        Note: When using grpc sendMessage to send multiple messages, the bornTimestamp in the requestHeader\n        is set to the bornTimestamp of the first message, which may not be accurate. When a bornTimestamp is\n        required, the bornTimestamp of the message property should be used.\n        * */\n        try {\n            requestHeader.setBornTimestamp(Long.parseLong(message.getProperty(MessageConst.PROPERTY_BORN_TIMESTAMP)));\n        } catch (Exception e) {\n            log.warn(\"parse born time error, with value:{}\", message.getProperty(MessageConst.PROPERTY_BORN_TIMESTAMP));\n            requestHeader.setBornTimestamp(System.currentTimeMillis());\n        }\n        requestHeader.setFlag(message.getFlag());\n        requestHeader.setProperties(MessageDecoder.messageProperties2String(message.getProperties()));\n        requestHeader.setReconsumeTimes(0);\n        if (messageList.size() > 1) {\n            requestHeader.setBatch(true);\n        }\n        if (requestHeader.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n            String reconsumeTimes = MessageAccessor.getReconsumeTime(message);\n            if (reconsumeTimes != null) {\n                requestHeader.setReconsumeTimes(Integer.valueOf(reconsumeTimes));\n                MessageAccessor.clearProperty(message, MessageConst.PROPERTY_RECONSUME_TIME);\n            }\n\n            String maxReconsumeTimes = MessageAccessor.getMaxReconsumeTimes(message);\n            if (maxReconsumeTimes != null) {\n                requestHeader.setMaxReconsumeTimes(Integer.valueOf(maxReconsumeTimes));\n                MessageAccessor.clearProperty(message, MessageConst.PROPERTY_MAX_RECONSUME_TIMES);\n            }\n        }\n\n        return requestHeader;\n    }\n\n    public CompletableFuture<RemotingCommand> forwardMessageToDeadLetterQueue(ProxyContext ctx,\n        ReceiptHandle handle,\n        String messageId,\n        String groupName,\n        String topicName,\n        String liteTopic,\n        long timeoutMillis\n    ) {\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        try {\n            if (handle.getCommitLogOffset() < 0) {\n                throw new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, \"commit log offset is empty\");\n            }\n\n            ConsumerSendMsgBackRequestHeader consumerSendMsgBackRequestHeader = new ConsumerSendMsgBackRequestHeader();\n            consumerSendMsgBackRequestHeader.setOffset(handle.getCommitLogOffset());\n            consumerSendMsgBackRequestHeader.setGroup(groupName);\n            consumerSendMsgBackRequestHeader.setDelayLevel(-1);\n            consumerSendMsgBackRequestHeader.setOriginMsgId(messageId);\n            consumerSendMsgBackRequestHeader.setOriginTopic(handle.getRealTopic(topicName, groupName));\n            consumerSendMsgBackRequestHeader.setMaxReconsumeTimes(0);\n\n            future = this.serviceManager.getMessageService().sendMessageBack(\n                ctx,\n                handle,\n                messageId,\n                consumerSendMsgBackRequestHeader,\n                timeoutMillis\n            ).whenCompleteAsync((remotingCommand, t) -> {\n                if (t == null && remotingCommand.getCode() == ResponseCode.SUCCESS) {\n                    this.messagingProcessor.ackMessage(ctx, handle, messageId,\n                        groupName, topicName, liteTopic, timeoutMillis);\n                }\n            }, this.executor);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return FutureUtils.addExecutor(future, this.executor);\n    }\n\n    private boolean isNeedCheckTopicMessageType(Message message) {\n        return ConfigurationManager.getProxyConfig().isEnableTopicMessageTypeCheck()\n            && !message.hasProperty(MessageConst.PROPERTY_TRANSFER_FLAG);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/processor/QueueSelector.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.processor;\n\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;\nimport org.apache.rocketmq.proxy.service.route.MessageQueueView;\n\npublic interface QueueSelector {\n\n    AddressableMessageQueue select(ProxyContext ctx, MessageQueueView messageQueueView);\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/processor/ReceiptHandleProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.processor;\n\nimport io.netty.channel.Channel;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.common.state.StateEventListener;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.MessageReceiptHandle;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.common.RenewEvent;\nimport org.apache.rocketmq.proxy.service.ServiceManager;\nimport org.apache.rocketmq.proxy.service.receipt.DefaultReceiptHandleManager;\n\npublic class ReceiptHandleProcessor extends AbstractProcessor {\n    protected final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    protected DefaultReceiptHandleManager receiptHandleManager;\n\n    public ReceiptHandleProcessor(MessagingProcessor messagingProcessor, ServiceManager serviceManager) {\n        super(messagingProcessor, serviceManager);\n        StateEventListener<RenewEvent> eventListener = event -> {\n            ProxyContext context = createContext(event.getEventType().name())\n                .setChannel(event.getKey().getChannel());\n            MessageReceiptHandle messageReceiptHandle = event.getMessageReceiptHandle();\n            ReceiptHandle handle = ReceiptHandle.decode(messageReceiptHandle.getReceiptHandleStr());\n            messagingProcessor.changeInvisibleTime(context, handle, messageReceiptHandle.getMessageId(),\n                messageReceiptHandle.getGroup(), messageReceiptHandle.getTopic(), event.getRenewTime(), messageReceiptHandle.getLiteTopic())\n                .whenComplete((v, t) -> {\n                    if (t != null) {\n                        event.getFuture().completeExceptionally(t);\n                        return;\n                    }\n                    event.getFuture().complete(v);\n                });\n        };\n        this.receiptHandleManager = new DefaultReceiptHandleManager(serviceManager.getMetadataService(), serviceManager.getConsumerManager(), eventListener);\n        this.appendStartAndShutdown(receiptHandleManager);\n    }\n\n    protected ProxyContext createContext(String actionName) {\n        return ProxyContext.createForInner(this.getClass().getSimpleName() + actionName);\n    }\n\n    public void addReceiptHandle(ProxyContext ctx, Channel channel, String group, String msgID, MessageReceiptHandle messageReceiptHandle) {\n        receiptHandleManager.addReceiptHandle(ctx, channel, group, msgID, messageReceiptHandle);\n    }\n\n    public MessageReceiptHandle removeReceiptHandle(ProxyContext ctx, Channel channel, String group, String msgID, String receiptHandle) {\n        return receiptHandleManager.removeReceiptHandle(ctx, channel, group, msgID, receiptHandle);\n    }\n\n    public int getUnackedMessageCount(ProxyContext ctx, Channel channel, String group) {\n        return receiptHandleManager.getUnackedMessageCount(ctx, channel, group);\n    }\n\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/processor/RequestBrokerProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.processor;\n\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.service.ServiceManager;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic class RequestBrokerProcessor extends AbstractProcessor {\n\n    public RequestBrokerProcessor(MessagingProcessor messagingProcessor,\n        ServiceManager serviceManager) {\n        super(messagingProcessor, serviceManager);\n    }\n\n    CompletableFuture<RemotingCommand> request(ProxyContext ctx, String brokerName, RemotingCommand request, long timeoutMillis) {\n        return serviceManager.getMessageService().request(ctx, brokerName, request, timeoutMillis);\n    }\n\n    CompletableFuture<Void> requestOneway(ProxyContext ctx, String brokerName, RemotingCommand request, long timeoutMillis) {\n        return serviceManager.getMessageService().requestOneway(ctx, brokerName, request, timeoutMillis);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/processor/TransactionProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.processor;\n\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\nimport org.apache.rocketmq.proxy.service.ServiceManager;\nimport org.apache.rocketmq.proxy.service.transaction.EndTransactionRequestData;\n\npublic class TransactionProcessor extends AbstractProcessor {\n\n    public TransactionProcessor(MessagingProcessor messagingProcessor,\n        ServiceManager serviceManager) {\n        super(messagingProcessor, serviceManager);\n    }\n\n    public CompletableFuture<Void> endTransaction(ProxyContext ctx, String topic, String transactionId, String messageId, String producerGroup,\n        TransactionStatus transactionStatus, boolean fromTransactionCheck, long timeoutMillis) {\n        CompletableFuture<Void> future = new CompletableFuture<>();\n        try {\n            EndTransactionRequestData headerData = serviceManager.getTransactionService().genEndTransactionRequestHeader(\n                ctx,\n                topic,\n                producerGroup,\n                buildCommitOrRollback(transactionStatus),\n                fromTransactionCheck,\n                messageId,\n                transactionId\n            );\n            if (headerData == null) {\n                future.completeExceptionally(new ProxyException(ProxyExceptionCode.TRANSACTION_DATA_NOT_FOUND, \"cannot found transaction data\"));\n                return future;\n            }\n            return this.serviceManager.getMessageService().endTransactionOneway(\n                ctx,\n                headerData.getBrokerName(),\n                headerData.getRequestHeader(),\n                timeoutMillis\n            );\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    protected int buildCommitOrRollback(TransactionStatus transactionStatus) {\n        switch (transactionStatus) {\n            case COMMIT:\n                return MessageSysFlag.TRANSACTION_COMMIT_TYPE;\n            case ROLLBACK:\n                return MessageSysFlag.TRANSACTION_ROLLBACK_TYPE;\n            default:\n                return MessageSysFlag.TRANSACTION_NOT_TYPE;\n        }\n    }\n\n    public void addTransactionSubscription(ProxyContext ctx, String producerGroup, String topic) {\n        this.serviceManager.getTransactionService().addTransactionSubscription(ctx, producerGroup, topic);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/processor/TransactionStatus.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.processor;\n\npublic enum TransactionStatus {\n    UNKNOWN,\n    COMMIT,\n    ROLLBACK\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/ChannelExtendAttributeGetter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.processor.channel;\n\npublic interface ChannelExtendAttributeGetter {\n\n    String getChannelExtendAttribute();\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/ChannelProtocolType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.processor.channel;\n\npublic enum ChannelProtocolType {\n    UNKNOWN(\"unknown\"),\n    GRPC_V2(\"grpc_v2\"),\n    GRPC_V1(\"grpc_v1\"),\n    REMOTING(\"remoting\");\n\n    private final String name;\n\n    ChannelProtocolType(String name) {\n        this.name = name;\n    }\n\n    public String getName() {\n        return name;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannel.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.processor.channel;\n\nimport com.google.common.base.MoreObjects;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelId;\nimport org.apache.rocketmq.proxy.service.channel.SimpleChannel;\n\npublic class RemoteChannel extends SimpleChannel implements ChannelExtendAttributeGetter {\n    protected final ChannelProtocolType type;\n    protected final String remoteProxyIp;\n    protected volatile String extendAttribute;\n\n    public RemoteChannel(String remoteProxyIp, String remoteAddress, String localAddress, ChannelProtocolType type, String extendAttribute) {\n        super(null,\n            new RemoteChannelId(remoteProxyIp, remoteAddress, localAddress, type),\n            remoteAddress, localAddress);\n        this.type = type;\n        this.remoteProxyIp = remoteProxyIp;\n        this.extendAttribute = extendAttribute;\n    }\n\n    public static class RemoteChannelId implements ChannelId {\n\n        private final String id;\n\n        public RemoteChannelId(String remoteProxyIp, String remoteAddress, String localAddress, ChannelProtocolType type) {\n            this.id = remoteProxyIp + \"@\" + remoteAddress + \"@\" + localAddress + \"@\" + type;\n        }\n\n        @Override\n        public String asShortText() {\n            return this.id;\n        }\n\n        @Override\n        public String asLongText() {\n            return this.id;\n        }\n\n        @Override\n        public int compareTo(ChannelId o) {\n            return this.id.compareTo(o.asLongText());\n        }\n\n        @Override\n        public String toString() {\n            return MoreObjects.toStringHelper(this)\n                .add(\"id\", id)\n                .toString();\n        }\n    }\n\n    @Override\n    public boolean isWritable() {\n        return false;\n    }\n\n    public ChannelProtocolType getType() {\n        return type;\n    }\n\n    public String encode() {\n        return RemoteChannelSerializer.toJson(this);\n    }\n\n    public static RemoteChannel decode(String data) {\n        return RemoteChannelSerializer.decodeFromJson(data);\n    }\n\n    public static RemoteChannel create(Channel channel) {\n        if (channel instanceof RemoteChannelConverter) {\n            return ((RemoteChannelConverter) channel).toRemoteChannel();\n        }\n        return null;\n    }\n\n    public String getRemoteProxyIp() {\n        return remoteProxyIp;\n    }\n\n    public void setExtendAttribute(String extendAttribute) {\n        this.extendAttribute = extendAttribute;\n    }\n\n    @Override\n    public String getChannelExtendAttribute() {\n        return this.extendAttribute;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"channelId\", id())\n            .add(\"type\", type)\n            .add(\"remoteProxyIp\", remoteProxyIp)\n            .add(\"extendAttribute\", extendAttribute)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannelConverter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.processor.channel;\n\npublic interface RemoteChannelConverter {\n\n    RemoteChannel toRemoteChannel();\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannelSerializer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.processor.channel;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.alibaba.fastjson2.JSONObject;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\nimport java.util.HashMap;\nimport java.util.Map;\n\npublic class RemoteChannelSerializer {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    private static final String REMOTE_PROXY_IP_KEY = \"remoteProxyIp\";\n    private static final String REMOTE_ADDRESS_KEY = \"remoteAddress\";\n    private static final String LOCAL_ADDRESS_KEY = \"localAddress\";\n    private static final String TYPE_KEY = \"type\";\n    private static final String EXTEND_ATTRIBUTE_KEY = \"extendAttribute\";\n\n    public static String toJson(RemoteChannel remoteChannel) {\n        Map<String, Object> data = new HashMap<>();\n        data.put(REMOTE_PROXY_IP_KEY, remoteChannel.getRemoteProxyIp());\n        data.put(REMOTE_ADDRESS_KEY, remoteChannel.getRemoteAddress());\n        data.put(LOCAL_ADDRESS_KEY, remoteChannel.getLocalAddress());\n        data.put(TYPE_KEY, remoteChannel.getType());\n        data.put(EXTEND_ATTRIBUTE_KEY, remoteChannel.getChannelExtendAttribute());\n        return JSON.toJSONString(data);\n    }\n\n    public static RemoteChannel decodeFromJson(String jsonData) {\n        if (StringUtils.isBlank(jsonData)) {\n            return null;\n        }\n        try {\n            JSONObject jsonObject = JSON.parseObject(jsonData);\n            return new RemoteChannel(\n                jsonObject.getString(REMOTE_PROXY_IP_KEY),\n                jsonObject.getString(REMOTE_ADDRESS_KEY),\n                jsonObject.getString(LOCAL_ADDRESS_KEY),\n                jsonObject.getObject(TYPE_KEY, ChannelProtocolType.class),\n                jsonObject.getObject(EXTEND_ATTRIBUTE_KEY, String.class)\n            );\n        } catch (Throwable t) {\n            log.error(\"decode remote channel data failed. data:{}\", jsonData, t);\n        }\n        return null;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/processor/validator/DefaultTopicMessageTypeValidator.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.processor.validator;\n\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\n\npublic class DefaultTopicMessageTypeValidator implements TopicMessageTypeValidator {\n\n    public void validate(TopicMessageType expectedType, TopicMessageType actualType) {\n        if (actualType.equals(TopicMessageType.UNSPECIFIED)\n                || !actualType.equals(expectedType) && !expectedType.equals(TopicMessageType.MIXED)) {\n            String errorInfo = String.format(\"TopicMessageType validate failed, the expected type is %s, but actual type is %s\", expectedType, actualType);\n            throw new ProxyException(ProxyExceptionCode.MESSAGE_PROPERTY_CONFLICT_WITH_TYPE, errorInfo);\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/processor/validator/TopicMessageTypeValidator.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.processor.validator;\n\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\n\npublic interface TopicMessageTypeValidator {\n    /**\n     * Will throw {@link org.apache.rocketmq.proxy.common.ProxyException} if validate failed.\n     *\n     * @param expectedType Target topic\n     * @param actualType   Message's type\n     */\n    void validate(TopicMessageType expectedType, TopicMessageType actualType);\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/ClientHousekeepingService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting;\n\nimport io.netty.channel.Channel;\nimport org.apache.rocketmq.proxy.remoting.activity.ClientManagerActivity;\nimport org.apache.rocketmq.remoting.ChannelEventListener;\n\npublic class ClientHousekeepingService implements ChannelEventListener {\n\n    private final ClientManagerActivity clientManagerActivity;\n\n    public ClientHousekeepingService(ClientManagerActivity clientManagerActivity) {\n        this.clientManagerActivity = clientManagerActivity;\n    }\n\n    @Override\n    public void onChannelConnect(String remoteAddr, Channel channel) {\n\n    }\n\n    @Override\n    public void onChannelClose(String remoteAddr, Channel channel) {\n        this.clientManagerActivity.doChannelCloseEvent(remoteAddr, channel);\n    }\n\n    @Override\n    public void onChannelException(String remoteAddr, Channel channel) {\n        this.clientManagerActivity.doChannelCloseEvent(remoteAddr, channel);\n    }\n\n    @Override\n    public void onChannelIdle(String remoteAddr, Channel channel) {\n        this.clientManagerActivity.doChannelCloseEvent(remoteAddr, channel);\n    }\n\n    @Override\n    public void onChannelActive(String remoteAddr, Channel channel) {\n\n    }\n}\n\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolRemotingServer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting;\n\nimport io.netty.channel.ChannelPipeline;\nimport io.netty.channel.socket.SocketChannel;\nimport io.netty.handler.timeout.IdleStateHandler;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\nimport org.apache.rocketmq.proxy.remoting.protocol.ProtocolNegotiationHandler;\nimport org.apache.rocketmq.proxy.remoting.protocol.http2proxy.Http2ProtocolProxyHandler;\nimport org.apache.rocketmq.proxy.remoting.protocol.remoting.RemotingProtocolHandler;\nimport org.apache.rocketmq.remoting.ChannelEventListener;\nimport org.apache.rocketmq.remoting.common.TlsMode;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingServer;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.netty.TlsSystemConfig;\n\nimport java.io.IOException;\nimport java.security.cert.CertificateException;\n\n/**\n * support remoting and http2 protocol at one port\n */\npublic class MultiProtocolRemotingServer extends NettyRemotingServer {\n\n    private final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    private final NettyServerConfig nettyServerConfig;\n\n    private final RemotingProtocolHandler remotingProtocolHandler;\n    protected Http2ProtocolProxyHandler http2ProtocolProxyHandler;\n\n    public MultiProtocolRemotingServer(NettyServerConfig nettyServerConfig, ChannelEventListener channelEventListener) {\n        super(nettyServerConfig, channelEventListener);\n        this.nettyServerConfig = nettyServerConfig;\n\n        this.remotingProtocolHandler = new RemotingProtocolHandler(\n            this::getEncoder,\n            this::getDistributionHandler,\n            this::getConnectionManageHandler,\n            this::getServerHandler);\n        this.http2ProtocolProxyHandler = new Http2ProtocolProxyHandler();\n    }\n\n    @Override\n    public void loadSslContext() {\n        TlsMode tlsMode = TlsSystemConfig.tlsMode;\n        log.info(\"Server is running in TLS {} mode\", tlsMode.getName());\n\n        if (tlsMode != TlsMode.DISABLED) {\n            try {\n                sslContext = MultiProtocolTlsHelper.buildSslContext();\n                log.info(\"SslContext created for multi protocol remoting server\");\n            } catch (CertificateException | IOException e) {\n                throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, \"Failed to create SslContext for server\", e);\n            }\n        }\n    }\n\n    @Override\n    protected ChannelPipeline configChannel(SocketChannel ch) {\n        return ch.pipeline()\n            .addLast(this.getDefaultEventExecutorGroup(), HANDSHAKE_HANDLER_NAME, new HandshakeHandler())\n            .addLast(this.getDefaultEventExecutorGroup(),\n                new IdleStateHandler(0, 0, nettyServerConfig.getServerChannelMaxIdleTimeSeconds()),\n                new ProtocolNegotiationHandler(this.remotingProtocolHandler)\n                    .addProtocolHandler(this.http2ProtocolProxyHandler)\n            );\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolTlsHelper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting;\n\nimport io.netty.handler.ssl.ApplicationProtocolConfig;\nimport io.netty.handler.ssl.ApplicationProtocolNames;\nimport io.netty.handler.ssl.ClientAuth;\nimport io.netty.handler.ssl.OpenSsl;\nimport io.netty.handler.ssl.SslContext;\nimport io.netty.handler.ssl.SslContextBuilder;\nimport io.netty.handler.ssl.SslProvider;\nimport io.netty.handler.ssl.util.InsecureTrustManagerFactory;\nimport io.netty.handler.ssl.util.SelfSignedCertificate;\nimport java.io.File;\nimport java.io.FileInputStream;\nimport java.io.IOException;\nimport java.nio.file.Files;\nimport java.nio.file.Paths;\nimport java.security.cert.CertificateException;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.netty.TlsHelper;\n\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerAuthClient;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerCertPath;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerKeyPassword;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerKeyPath;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerNeedClientAuth;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerTrustCertPath;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsTestModeEnable;\n\npublic class MultiProtocolTlsHelper extends TlsHelper {\n    private final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    private static final DecryptionStrategy DECRYPTION_STRATEGY = (privateKeyEncryptPath, forClient) -> new FileInputStream(privateKeyEncryptPath);\n\n    public static SslContext buildSslContext() throws IOException, CertificateException {\n        TlsHelper.buildSslContext(false);\n        SslProvider provider;\n        if (OpenSsl.isAvailable()) {\n            provider = SslProvider.OPENSSL;\n            log.info(\"Using OpenSSL provider\");\n        } else {\n            provider = SslProvider.JDK;\n            log.info(\"Using JDK SSL provider\");\n        }\n\n        SslContextBuilder sslContextBuilder;\n        if (tlsTestModeEnable) {\n            SelfSignedCertificate selfSignedCertificate = new SelfSignedCertificate();\n            sslContextBuilder = SslContextBuilder\n                .forServer(selfSignedCertificate.certificate(), selfSignedCertificate.privateKey())\n                .sslProvider(provider)\n                .clientAuth(ClientAuth.OPTIONAL);\n        } else {\n            sslContextBuilder = SslContextBuilder.forServer(\n                !StringUtils.isBlank(tlsServerCertPath) ? Files.newInputStream(Paths.get(tlsServerCertPath)) : null,\n                !StringUtils.isBlank(tlsServerKeyPath) ? DECRYPTION_STRATEGY.decryptPrivateKey(tlsServerKeyPath, false) : null,\n                !StringUtils.isBlank(tlsServerKeyPassword) ? tlsServerKeyPassword : null)\n                .sslProvider(provider);\n\n            if (!tlsServerAuthClient) {\n                sslContextBuilder.trustManager(InsecureTrustManagerFactory.INSTANCE);\n            } else {\n                if (!StringUtils.isBlank(tlsServerTrustCertPath)) {\n                    sslContextBuilder.trustManager(new File(tlsServerTrustCertPath));\n                }\n            }\n\n            sslContextBuilder.clientAuth(parseClientAuthMode(tlsServerNeedClientAuth));\n        }\n\n        sslContextBuilder.applicationProtocolConfig(new ApplicationProtocolConfig(\n            ApplicationProtocolConfig.Protocol.ALPN,\n            // NO_ADVERTISE is currently the only mode supported by both OpenSsl and JDK providers.\n            ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE,\n            // ACCEPT is currently the only mode supported by both OpenSsl and JDK providers.\n            ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT,\n            ApplicationProtocolNames.HTTP_2));\n\n        moreTlsConfig(sslContextBuilder);\n        return sslContextBuilder.build();\n    }\n\n    private static ClientAuth parseClientAuthMode(String authMode) {\n        if (null == authMode || authMode.trim().isEmpty()) {\n            return ClientAuth.NONE;\n        }\n\n        String authModeUpper = authMode.toUpperCase();\n        for (ClientAuth clientAuth : ClientAuth.values()) {\n            if (clientAuth.name().equals(authModeUpper)) {\n                return clientAuth;\n            }\n        }\n\n        return ClientAuth.NONE;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting;\n\nimport com.google.common.util.concurrent.ThreadFactoryBuilder;\nimport io.netty.channel.Channel;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.future.FutureTaskExt;\nimport org.apache.rocketmq.common.thread.ThreadPoolMonitor;\nimport org.apache.rocketmq.common.thread.ThreadPoolStatusMonitor;\nimport org.apache.rocketmq.common.utils.StartAndShutdown;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.remoting.activity.AckMessageActivity;\nimport org.apache.rocketmq.proxy.remoting.activity.ChangeInvisibleTimeActivity;\nimport org.apache.rocketmq.proxy.remoting.activity.ClientManagerActivity;\nimport org.apache.rocketmq.proxy.remoting.activity.ConsumerManagerActivity;\nimport org.apache.rocketmq.proxy.remoting.activity.GetTopicRouteActivity;\nimport org.apache.rocketmq.proxy.remoting.activity.PopMessageActivity;\nimport org.apache.rocketmq.proxy.remoting.activity.PullMessageActivity;\nimport org.apache.rocketmq.proxy.remoting.activity.RecallMessageActivity;\nimport org.apache.rocketmq.proxy.remoting.activity.SendMessageActivity;\nimport org.apache.rocketmq.proxy.remoting.activity.TransactionActivity;\nimport org.apache.rocketmq.proxy.remoting.channel.RemotingChannelManager;\nimport org.apache.rocketmq.proxy.remoting.pipeline.AuthenticationPipeline;\nimport org.apache.rocketmq.proxy.remoting.pipeline.AuthorizationPipeline;\nimport org.apache.rocketmq.proxy.remoting.pipeline.ContextInitPipeline;\nimport org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;\nimport org.apache.rocketmq.proxy.service.cert.TlsCertificateManager;\nimport org.apache.rocketmq.remoting.ChannelEventListener;\nimport org.apache.rocketmq.remoting.InvokeCallback;\nimport org.apache.rocketmq.remoting.RemotingServer;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingServer;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.netty.RequestTask;\nimport org.apache.rocketmq.remoting.netty.ResponseFuture;\nimport org.apache.rocketmq.remoting.netty.TlsSystemConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\n\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\n\npublic class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOutClient {\n    private final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    protected final MessagingProcessor messagingProcessor;\n    protected final RemotingChannelManager remotingChannelManager;\n    protected final ChannelEventListener clientHousekeepingService;\n    protected final RemotingServer defaultRemotingServer;\n    protected final GetTopicRouteActivity getTopicRouteActivity;\n    protected final ClientManagerActivity clientManagerActivity;\n    protected final ConsumerManagerActivity consumerManagerActivity;\n    protected final SendMessageActivity sendMessageActivity;\n    protected final RecallMessageActivity recallMessageActivity;\n    protected final TransactionActivity transactionActivity;\n    protected final PullMessageActivity pullMessageActivity;\n    protected final PopMessageActivity popMessageActivity;\n    protected final AckMessageActivity ackMessageActivity;\n    protected final ChangeInvisibleTimeActivity changeInvisibleTimeActivity;\n    protected final ThreadPoolExecutor sendMessageExecutor;\n    protected final ThreadPoolExecutor pullMessageExecutor;\n    protected final ThreadPoolExecutor heartbeatExecutor;\n    protected final ThreadPoolExecutor updateOffsetExecutor;\n    protected final ThreadPoolExecutor topicRouteExecutor;\n    protected final ThreadPoolExecutor defaultExecutor;\n    protected final ScheduledExecutorService timerExecutor;\n    protected final TlsCertificateManager tlsCertificateManager;\n    protected final RemotingTlsReloadHandler tlsReloadHandler;\n\n\n    public RemotingProtocolServer(MessagingProcessor messagingProcessor, TlsCertificateManager tlsCertificateManager) throws Exception {\n        this.messagingProcessor = messagingProcessor;\n        this.remotingChannelManager = new RemotingChannelManager(this, messagingProcessor.getProxyRelayService());\n\n        RequestPipeline pipeline = createRequestPipeline(messagingProcessor);\n        this.getTopicRouteActivity = new GetTopicRouteActivity(pipeline, messagingProcessor);\n        this.clientManagerActivity = new ClientManagerActivity(pipeline, messagingProcessor, remotingChannelManager);\n        this.consumerManagerActivity = new ConsumerManagerActivity(pipeline, messagingProcessor);\n        this.sendMessageActivity = new SendMessageActivity(pipeline, messagingProcessor);\n        this.recallMessageActivity = new RecallMessageActivity(pipeline, messagingProcessor);\n        this.transactionActivity = new TransactionActivity(pipeline, messagingProcessor);\n        this.pullMessageActivity = new PullMessageActivity(pipeline, messagingProcessor);\n        this.popMessageActivity = new PopMessageActivity(pipeline, messagingProcessor);\n        this.ackMessageActivity = new AckMessageActivity(pipeline, messagingProcessor);\n        this.changeInvisibleTimeActivity = new ChangeInvisibleTimeActivity(pipeline, messagingProcessor);\n\n        ProxyConfig config = ConfigurationManager.getProxyConfig();\n        NettyServerConfig defaultServerConfig = new NettyServerConfig();\n        defaultServerConfig.setListenPort(config.getRemotingListenPort());\n        TlsSystemConfig.tlsTestModeEnable = config.isTlsTestModeEnable();\n        System.setProperty(TlsSystemConfig.TLS_TEST_MODE_ENABLE, Boolean.toString(config.isTlsTestModeEnable()));\n        TlsSystemConfig.tlsServerCertPath = config.getTlsCertPath();\n        System.setProperty(TlsSystemConfig.TLS_SERVER_CERTPATH, config.getTlsCertPath());\n        TlsSystemConfig.tlsServerKeyPath = config.getTlsKeyPath();\n        System.setProperty(TlsSystemConfig.TLS_SERVER_KEYPATH, config.getTlsKeyPath());\n        TlsSystemConfig.tlsServerKeyPassword = config.getTlsKeyPassword();\n        System.setProperty(TlsSystemConfig.TLS_SERVER_KEYPASSWORD, config.getTlsKeyPassword());\n        this.tlsCertificateManager = tlsCertificateManager;\n        this.tlsReloadHandler = new RemotingTlsReloadHandler();\n\n        this.clientHousekeepingService = new ClientHousekeepingService(this.clientManagerActivity);\n\n        if (config.isEnableRemotingLocalProxyGrpc()) {\n            this.defaultRemotingServer = new MultiProtocolRemotingServer(defaultServerConfig, this.clientHousekeepingService);\n        } else {\n            this.defaultRemotingServer = new NettyRemotingServer(defaultServerConfig, this.clientHousekeepingService);\n        }\n\n        this.sendMessageExecutor = ThreadPoolMonitor.createAndMonitor(\n            config.getRemotingSendMessageThreadPoolNums(),\n            config.getRemotingSendMessageThreadPoolNums(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            \"RemotingSendMessageThread\",\n            config.getRemotingSendThreadPoolQueueCapacity(),\n            new ThreadPoolHeadSlowTimeMillsMonitor(config.getRemotingWaitTimeMillsInSendQueue())\n        );\n\n        this.pullMessageExecutor = ThreadPoolMonitor.createAndMonitor(\n            config.getRemotingPullMessageThreadPoolNums(),\n            config.getRemotingPullMessageThreadPoolNums(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            \"RemotingPullMessageThread\",\n            config.getRemotingPullThreadPoolQueueCapacity(),\n            new ThreadPoolHeadSlowTimeMillsMonitor(config.getRemotingWaitTimeMillsInPullQueue())\n        );\n\n        this.updateOffsetExecutor = ThreadPoolMonitor.createAndMonitor(\n            config.getRemotingUpdateOffsetThreadPoolNums(),\n            config.getRemotingUpdateOffsetThreadPoolNums(),\n            1,\n            TimeUnit.MINUTES,\n            \"RemotingUpdateOffsetThread\",\n            config.getRemotingUpdateOffsetThreadPoolQueueCapacity(),\n            new ThreadPoolHeadSlowTimeMillsMonitor(config.getRemotingWaitTimeMillsInUpdateOffsetQueue())\n        );\n\n        this.heartbeatExecutor = ThreadPoolMonitor.createAndMonitor(\n            config.getRemotingHeartbeatThreadPoolNums(),\n            config.getRemotingHeartbeatThreadPoolNums(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            \"RemotingHeartbeatThread\",\n            config.getRemotingHeartbeatThreadPoolQueueCapacity(),\n            new ThreadPoolHeadSlowTimeMillsMonitor(config.getRemotingWaitTimeMillsInHeartbeatQueue())\n        );\n\n        this.topicRouteExecutor = ThreadPoolMonitor.createAndMonitor(\n            config.getRemotingTopicRouteThreadPoolNums(),\n            config.getRemotingTopicRouteThreadPoolNums(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            \"RemotingTopicRouteThread\",\n            config.getRemotingTopicRouteThreadPoolQueueCapacity(),\n            new ThreadPoolHeadSlowTimeMillsMonitor(config.getRemotingWaitTimeMillsInTopicRouteQueue())\n        );\n\n        this.defaultExecutor = ThreadPoolMonitor.createAndMonitor(\n            config.getRemotingDefaultThreadPoolNums(),\n            config.getRemotingDefaultThreadPoolNums(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            \"RemotingDefaultThread\",\n            config.getRemotingDefaultThreadPoolQueueCapacity(),\n            new ThreadPoolHeadSlowTimeMillsMonitor(config.getRemotingWaitTimeMillsInDefaultQueue())\n        );\n\n        this.timerExecutor = ThreadUtils.newSingleThreadScheduledExecutor(\n            new ThreadFactoryBuilder().setNameFormat(\"RemotingServerScheduler-%d\").build()\n        );\n        this.timerExecutor.scheduleAtFixedRate(this::cleanExpireRequest, 10, 10, TimeUnit.SECONDS);\n\n        this.registerRemotingServer(this.defaultRemotingServer);\n    }\n\n    protected class RemotingTlsReloadHandler implements TlsCertificateManager.TlsContextReloadListener {\n        @Override\n        public void onTlsContextReload() {\n            if (defaultRemotingServer instanceof NettyRemotingServer) {\n                ((NettyRemotingServer) defaultRemotingServer).loadSslContext();\n                log.info(\"SslContext reloaded for remoting server\");\n            }\n        }\n    }\n\n    protected void registerRemotingServer(RemotingServer remotingServer) {\n        remotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendMessageActivity, this.sendMessageExecutor);\n        remotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendMessageActivity, this.sendMessageExecutor);\n        remotingServer.registerProcessor(RequestCode.SEND_BATCH_MESSAGE, sendMessageActivity, this.sendMessageExecutor);\n        remotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendMessageActivity, sendMessageExecutor);\n\n        remotingServer.registerProcessor(RequestCode.END_TRANSACTION, transactionActivity, sendMessageExecutor);\n        remotingServer.registerProcessor(RequestCode.RECALL_MESSAGE, recallMessageActivity, sendMessageExecutor);\n\n        remotingServer.registerProcessor(RequestCode.HEART_BEAT, clientManagerActivity, this.heartbeatExecutor);\n        remotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientManagerActivity, this.defaultExecutor);\n        remotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientManagerActivity, this.defaultExecutor);\n\n        remotingServer.registerProcessor(RequestCode.PULL_MESSAGE, pullMessageActivity, this.pullMessageExecutor);\n        remotingServer.registerProcessor(RequestCode.LITE_PULL_MESSAGE, pullMessageActivity, this.pullMessageExecutor);\n        remotingServer.registerProcessor(RequestCode.POP_MESSAGE, pullMessageActivity, this.pullMessageExecutor);\n\n        remotingServer.registerProcessor(RequestCode.UPDATE_CONSUMER_OFFSET, consumerManagerActivity, this.updateOffsetExecutor);\n        remotingServer.registerProcessor(RequestCode.ACK_MESSAGE, consumerManagerActivity, this.updateOffsetExecutor);\n        remotingServer.registerProcessor(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, consumerManagerActivity, this.updateOffsetExecutor);\n        remotingServer.registerProcessor(RequestCode.GET_CONSUMER_CONNECTION_LIST, consumerManagerActivity, this.updateOffsetExecutor);\n\n        remotingServer.registerProcessor(RequestCode.GET_CONSUMER_LIST_BY_GROUP, consumerManagerActivity, this.defaultExecutor);\n        remotingServer.registerProcessor(RequestCode.GET_MAX_OFFSET, consumerManagerActivity, this.defaultExecutor);\n        remotingServer.registerProcessor(RequestCode.GET_MIN_OFFSET, consumerManagerActivity, this.defaultExecutor);\n        remotingServer.registerProcessor(RequestCode.QUERY_CONSUMER_OFFSET, consumerManagerActivity, this.defaultExecutor);\n        remotingServer.registerProcessor(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, consumerManagerActivity, this.defaultExecutor);\n        remotingServer.registerProcessor(RequestCode.LOCK_BATCH_MQ, consumerManagerActivity, this.defaultExecutor);\n        remotingServer.registerProcessor(RequestCode.UNLOCK_BATCH_MQ, consumerManagerActivity, this.defaultExecutor);\n\n        remotingServer.registerProcessor(RequestCode.GET_ROUTEINFO_BY_TOPIC, getTopicRouteActivity, this.topicRouteExecutor);\n    }\n\n    @Override\n    public void shutdown() throws Exception {\n        // Unregister the TLS context reload handler\n        tlsCertificateManager.unregisterReloadListener(this.tlsReloadHandler);\n\n        this.defaultRemotingServer.shutdown();\n        this.remotingChannelManager.shutdown();\n        this.sendMessageExecutor.shutdown();\n        this.pullMessageExecutor.shutdown();\n        this.heartbeatExecutor.shutdown();\n        this.updateOffsetExecutor.shutdown();\n        this.topicRouteExecutor.shutdown();\n        this.defaultExecutor.shutdown();\n    }\n\n    @Override\n    public void start() throws Exception {\n        // Register the TLS context reload handler\n        tlsCertificateManager.registerReloadListener(this.tlsReloadHandler);\n\n        this.remotingChannelManager.start();\n        this.defaultRemotingServer.start();\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> invokeToClient(Channel channel, RemotingCommand request,\n        long timeoutMillis) {\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        try {\n            this.defaultRemotingServer.invokeAsync(channel, request, timeoutMillis, new InvokeCallback() {\n                @Override\n                public void operationComplete(ResponseFuture responseFuture) {\n\n                }\n\n                @Override\n                public void operationSucceed(RemotingCommand response) {\n                    future.complete(response);\n                }\n\n                @Override\n                public void operationFail(Throwable throwable) {\n                    future.completeExceptionally(throwable);\n                }\n            });\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    protected RequestPipeline createRequestPipeline(MessagingProcessor messagingProcessor) {\n        RequestPipeline pipeline = (ctx, request, context) -> {\n        };\n        // add pipeline\n        // the last pipe add will execute at the first\n        AuthConfig authConfig = ConfigurationManager.getAuthConfig();\n        if (authConfig != null) {\n            pipeline = pipeline.pipe(new AuthorizationPipeline(authConfig, messagingProcessor))\n                .pipe(new AuthenticationPipeline(authConfig, messagingProcessor));\n        }\n        return pipeline.pipe(new ContextInitPipeline());\n    }\n\n    protected class ThreadPoolHeadSlowTimeMillsMonitor implements ThreadPoolStatusMonitor {\n\n        private final long maxWaitTimeMillsInQueue;\n\n        public ThreadPoolHeadSlowTimeMillsMonitor(long maxWaitTimeMillsInQueue) {\n            this.maxWaitTimeMillsInQueue = maxWaitTimeMillsInQueue;\n        }\n\n        @Override\n        public String describe() {\n            return \"headSlow\";\n        }\n\n        @Override\n        public double value(ThreadPoolExecutor executor) {\n            return headSlowTimeMills(executor.getQueue());\n        }\n\n        @Override\n        public boolean needPrintJstack(ThreadPoolExecutor executor, double value) {\n            return value > maxWaitTimeMillsInQueue;\n        }\n    }\n\n    protected long headSlowTimeMills(BlockingQueue<Runnable> q) {\n        try {\n            long slowTimeMills = 0;\n            final Runnable peek = q.peek();\n            if (peek != null) {\n                RequestTask rt = castRunnable(peek);\n                slowTimeMills = rt == null ? 0 : System.currentTimeMillis() - rt.getCreateTimestamp();\n            }\n\n            if (slowTimeMills < 0) {\n                slowTimeMills = 0;\n            }\n\n            return slowTimeMills;\n        } catch (Exception e) {\n            log.error(\"error when headSlowTimeMills.\", e);\n        }\n        return -1;\n    }\n\n    protected void cleanExpireRequest() {\n        ProxyConfig config = ConfigurationManager.getProxyConfig();\n\n        cleanExpiredRequestInQueue(this.sendMessageExecutor, config.getRemotingWaitTimeMillsInSendQueue());\n        cleanExpiredRequestInQueue(this.pullMessageExecutor, config.getRemotingWaitTimeMillsInPullQueue());\n        cleanExpiredRequestInQueue(this.heartbeatExecutor, config.getRemotingWaitTimeMillsInHeartbeatQueue());\n        cleanExpiredRequestInQueue(this.updateOffsetExecutor, config.getRemotingWaitTimeMillsInUpdateOffsetQueue());\n        cleanExpiredRequestInQueue(this.topicRouteExecutor, config.getRemotingWaitTimeMillsInTopicRouteQueue());\n        cleanExpiredRequestInQueue(this.defaultExecutor, config.getRemotingWaitTimeMillsInDefaultQueue());\n    }\n\n    protected void cleanExpiredRequestInQueue(ThreadPoolExecutor threadPoolExecutor, long maxWaitTimeMillsInQueue) {\n        while (true) {\n            try {\n                BlockingQueue<Runnable> blockingQueue = threadPoolExecutor.getQueue();\n                if (!blockingQueue.isEmpty()) {\n                    final Runnable runnable = blockingQueue.peek();\n                    if (null == runnable) {\n                        break;\n                    }\n                    final RequestTask rt = castRunnable(runnable);\n                    if (rt == null || rt.isStopRun()) {\n                        break;\n                    }\n\n                    final long behind = System.currentTimeMillis() - rt.getCreateTimestamp();\n                    if (behind >= maxWaitTimeMillsInQueue) {\n                        if (blockingQueue.remove(runnable)) {\n                            rt.setStopRun(true);\n                            rt.returnResponse(ResponseCode.SYSTEM_BUSY,\n                                String.format(\"[TIMEOUT_CLEAN_QUEUE]broker busy, start flow control for a while, period in queue: %sms, size of queue: %d\", behind, blockingQueue.size()));\n                        }\n                    } else {\n                        break;\n                    }\n                } else {\n                    break;\n                }\n            } catch (Throwable ignored) {\n            }\n        }\n    }\n\n    private RequestTask castRunnable(final Runnable runnable) {\n        try {\n            if (runnable instanceof FutureTaskExt) {\n                FutureTaskExt futureTaskExt = (FutureTaskExt) runnable;\n                return (RequestTask) futureTaskExt.getRunnable();\n            }\n            return null;\n        } catch (Throwable e) {\n            log.error(\"castRunnable exception. class:{}\", runnable.getClass().getName(), e);\n        }\n\n        return null;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProxyOutClient.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting;\n\nimport io.netty.channel.Channel;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic interface RemotingProxyOutClient {\n\n    CompletableFuture<RemotingCommand> invokeToClient(Channel channel, RemotingCommand request, long timeoutMillis);\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.activity;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.HashMap;\nimport java.util.Map;\nimport org.apache.rocketmq.acl.common.AclException;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.utils.ExceptionUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\n\npublic abstract class AbstractRemotingActivity implements NettyRequestProcessor {\n    protected final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    protected final MessagingProcessor messagingProcessor;\n    protected static final String BROKER_NAME_FIELD = \"bname\";\n    protected static final String BROKER_NAME_FIELD_FOR_SEND_MESSAGE_V2 = \"n\";\n    @SuppressWarnings(\"DoubleBraceInitialization\")\n    private static final Map<ProxyExceptionCode, Integer> PROXY_EXCEPTION_RESPONSE_CODE_MAP = new HashMap<ProxyExceptionCode, Integer>() {\n        {\n            put(ProxyExceptionCode.FORBIDDEN, ResponseCode.NO_PERMISSION);\n            put(ProxyExceptionCode.MESSAGE_PROPERTY_CONFLICT_WITH_TYPE, ResponseCode.MESSAGE_ILLEGAL);\n            put(ProxyExceptionCode.INTERNAL_SERVER_ERROR, ResponseCode.SYSTEM_ERROR);\n            put(ProxyExceptionCode.TRANSACTION_DATA_NOT_FOUND, ResponseCode.SUCCESS);\n        }\n    };\n    protected final RequestPipeline requestPipeline;\n\n    public AbstractRemotingActivity(RequestPipeline requestPipeline, MessagingProcessor messagingProcessor) {\n        this.requestPipeline = requestPipeline;\n        this.messagingProcessor = messagingProcessor;\n    }\n\n    protected RemotingCommand request(ChannelHandlerContext ctx, RemotingCommand request,\n        ProxyContext context, long timeoutMillis) throws Exception {\n        String brokerName;\n        if (request.getCode() == RequestCode.SEND_MESSAGE_V2 || request.getCode() == RequestCode.SEND_BATCH_MESSAGE) {\n            if (request.getExtFields().get(BROKER_NAME_FIELD_FOR_SEND_MESSAGE_V2) == null) {\n                return RemotingCommand.buildErrorResponse(ResponseCode.VERSION_NOT_SUPPORTED,\n                    \"Request doesn't have field bname\");\n            }\n            brokerName = request.getExtFields().get(BROKER_NAME_FIELD_FOR_SEND_MESSAGE_V2);\n        } else {\n            if (request.getExtFields().get(BROKER_NAME_FIELD) == null) {\n                return RemotingCommand.buildErrorResponse(ResponseCode.VERSION_NOT_SUPPORTED,\n                    \"Request doesn't have field bname\");\n            }\n            brokerName = request.getExtFields().get(BROKER_NAME_FIELD);\n        }\n        if (request.isOnewayRPC()) {\n            messagingProcessor.requestOneway(context, brokerName, request, timeoutMillis);\n            return null;\n        }\n        messagingProcessor.request(context, brokerName, request, timeoutMillis)\n            .thenAccept(r -> writeResponse(ctx, context, request, r))\n            .exceptionally(t -> {\n                writeErrResponse(ctx, context, request, t);\n                return null;\n            });\n        return null;\n    }\n\n    @Override\n    public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws Exception {\n        ProxyContext context = createContext();\n        try {\n            this.requestPipeline.execute(ctx, request, context);\n            RemotingCommand response = this.processRequest0(ctx, request, context);\n            if (response != null) {\n                writeResponse(ctx, context, request, response);\n            }\n            return null;\n        } catch (Throwable t) {\n            writeErrResponse(ctx, context, request, t);\n            return null;\n        }\n    }\n\n    @Override\n    public boolean rejectRequest() {\n        return false;\n    }\n\n    protected abstract RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,\n        ProxyContext context) throws Exception;\n\n    protected ProxyContext createContext() {\n        return ProxyContext.create();\n    }\n\n    protected void writeErrResponse(ChannelHandlerContext ctx, final ProxyContext context,\n        final RemotingCommand request, Throwable t) {\n        t = ExceptionUtils.getRealException(t);\n        if (t instanceof ProxyException) {\n            ProxyException e = (ProxyException) t;\n            writeResponse(ctx, context, request,\n                RemotingCommand.createResponseCommand(\n                    PROXY_EXCEPTION_RESPONSE_CODE_MAP.getOrDefault(e.getCode(), ResponseCode.SYSTEM_ERROR),\n                    e.getMessage()),\n                t);\n        } else if (t instanceof MQClientException) {\n            MQClientException e = (MQClientException) t;\n            writeResponse(ctx, context, request, RemotingCommand.createResponseCommand(e.getResponseCode(), e.getErrorMessage()), t);\n        } else if (t instanceof MQBrokerException) {\n            MQBrokerException e = (MQBrokerException) t;\n            writeResponse(ctx, context, request, RemotingCommand.createResponseCommand(e.getResponseCode(), e.getErrorMessage()), t);\n        } else if (t instanceof AclException) {\n            writeResponse(ctx, context, request, RemotingCommand.createResponseCommand(ResponseCode.NO_PERMISSION, t.getMessage()), t);\n        } else {\n            writeResponse(ctx, context, request,\n                RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, t.getMessage()), t);\n        }\n    }\n\n    protected void writeResponse(ChannelHandlerContext ctx, final ProxyContext context,\n        final RemotingCommand request, RemotingCommand response) {\n        writeResponse(ctx, context, request, response, null);\n    }\n\n    protected void writeResponse(ChannelHandlerContext ctx, final ProxyContext context,\n        final RemotingCommand request, RemotingCommand response, Throwable t) {\n        if (request.isOnewayRPC()) {\n            return;\n        }\n        if (!ctx.channel().isWritable()) {\n            return;\n        }\n\n        ProxyConfig config = ConfigurationManager.getProxyConfig();\n\n        response.setOpaque(request.getOpaque());\n        response.markResponseType();\n        response.addExtField(MessageConst.PROPERTY_MSG_REGION, config.getRegionId());\n        response.addExtField(MessageConst.PROPERTY_TRACE_SWITCH, String.valueOf(config.isTraceOn()));\n        if (t != null) {\n            response.setRemark(t.getMessage());\n        }\n\n        ctx.writeAndFlush(response);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AckMessageActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.activity;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport java.time.Duration;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic class AckMessageActivity extends AbstractRemotingActivity {\n    public AckMessageActivity(RequestPipeline requestPipeline,\n        MessagingProcessor messagingProcessor) {\n        super(requestPipeline, messagingProcessor);\n    }\n\n    @Override\n    protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,\n        ProxyContext context) throws Exception {\n        return request(ctx, request, context, Duration.ofSeconds(3).toMillis());\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ChangeInvisibleTimeActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.activity;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport java.time.Duration;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic class ChangeInvisibleTimeActivity extends AbstractRemotingActivity {\n    public ChangeInvisibleTimeActivity(RequestPipeline requestPipeline,\n        MessagingProcessor messagingProcessor) {\n        super(requestPipeline, messagingProcessor);\n    }\n\n    @Override\n    protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,\n        ProxyContext context) throws Exception {\n        return request(ctx, request, context, Duration.ofSeconds(3).toMillis());\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ClientManagerActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.activity;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ConsumerGroupEvent;\nimport org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;\nimport org.apache.rocketmq.broker.client.ProducerChangeListener;\nimport org.apache.rocketmq.broker.client.ProducerGroupEvent;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.remoting.channel.RemotingChannel;\nimport org.apache.rocketmq.proxy.remoting.channel.RemotingChannelManager;\nimport org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.netty.AttributeKeys;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.UnregisterClientRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UnregisterClientResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumerData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ProducerData;\n\nimport java.util.Set;\n\npublic class ClientManagerActivity extends AbstractRemotingActivity {\n\n    private final RemotingChannelManager remotingChannelManager;\n\n    public ClientManagerActivity(RequestPipeline requestPipeline, MessagingProcessor messagingProcessor,\n        RemotingChannelManager manager) {\n        super(requestPipeline, messagingProcessor);\n        this.remotingChannelManager = manager;\n        this.init();\n    }\n\n    protected void init() {\n        this.messagingProcessor.registerConsumerListener(new ConsumerIdsChangeListenerImpl());\n        this.messagingProcessor.registerProducerListener(new ProducerChangeListenerImpl());\n    }\n\n    @Override\n    protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,\n        ProxyContext context) throws Exception {\n        switch (request.getCode()) {\n            case RequestCode.HEART_BEAT:\n                return this.heartBeat(ctx, request, context);\n            case RequestCode.UNREGISTER_CLIENT:\n                return this.unregisterClient(ctx, request, context);\n            case RequestCode.CHECK_CLIENT_CONFIG:\n                return this.checkClientConfig(ctx, request, context);\n            default:\n                break;\n        }\n        return null;\n    }\n\n    protected RemotingCommand heartBeat(ChannelHandlerContext ctx, RemotingCommand request,\n        ProxyContext context) {\n        HeartbeatData heartbeatData = HeartbeatData.decode(request.getBody(), HeartbeatData.class);\n        String clientId = heartbeatData.getClientID();\n\n        for (ProducerData data : heartbeatData.getProducerDataSet()) {\n            ClientChannelInfo clientChannelInfo = new ClientChannelInfo(\n                this.remotingChannelManager.createProducerChannel(context, ctx.channel(), data.getGroupName(), clientId),\n                clientId, request.getLanguage(),\n                request.getVersion());\n            setClientPropertiesToChannelAttr(clientChannelInfo);\n            messagingProcessor.registerProducer(context, data.getGroupName(), clientChannelInfo);\n        }\n\n        for (ConsumerData data : heartbeatData.getConsumerDataSet()) {\n            ClientChannelInfo clientChannelInfo = new ClientChannelInfo(\n                this.remotingChannelManager.createConsumerChannel(context, ctx.channel(), data.getGroupName(), clientId, data.getSubscriptionDataSet()),\n                clientId, request.getLanguage(),\n                request.getVersion());\n            setClientPropertiesToChannelAttr(clientChannelInfo);\n            messagingProcessor.registerConsumer(context, data.getGroupName(), clientChannelInfo, data.getConsumeType(),\n                data.getMessageModel(), data.getConsumeFromWhere(), data.getSubscriptionDataSet(), true);\n        }\n\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(\"\");\n        return response;\n    }\n\n    private void setClientPropertiesToChannelAttr(final ClientChannelInfo clientChannelInfo) {\n        Channel channel = clientChannelInfo.getChannel();\n        if (channel instanceof RemotingChannel) {\n            RemotingChannel remotingChannel = (RemotingChannel) channel;\n            Channel parent = remotingChannel.parent();\n            RemotingHelper.setPropertyToAttr(parent, AttributeKeys.CLIENT_ID_KEY, clientChannelInfo.getClientId());\n            RemotingHelper.setPropertyToAttr(parent, AttributeKeys.LANGUAGE_CODE_KEY, clientChannelInfo.getLanguage());\n            RemotingHelper.setPropertyToAttr(parent, AttributeKeys.VERSION_KEY, clientChannelInfo.getVersion());\n        }\n\n    }\n\n    protected RemotingCommand unregisterClient(ChannelHandlerContext ctx, RemotingCommand request,\n        ProxyContext context) throws RemotingCommandException {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(UnregisterClientResponseHeader.class);\n        final UnregisterClientRequestHeader requestHeader =\n            (UnregisterClientRequestHeader) request.decodeCommandCustomHeader(UnregisterClientRequestHeader.class);\n        final String producerGroup = requestHeader.getProducerGroup();\n        if (producerGroup != null) {\n            RemotingChannel channel = this.remotingChannelManager.removeProducerChannel(context, producerGroup, ctx.channel());\n            if (channel != null) {\n                ClientChannelInfo clientChannelInfo = new ClientChannelInfo(\n                    channel,\n                    requestHeader.getClientID(),\n                    request.getLanguage(),\n                    request.getVersion());\n                this.messagingProcessor.unRegisterProducer(context, producerGroup, clientChannelInfo);\n            } else {\n                log.warn(\"unregister producer failed, channel not exist, may has been removed, producerGroup={}, channel={}\", producerGroup, ctx.channel());\n            }\n        }\n        final String consumerGroup = requestHeader.getConsumerGroup();\n        if (consumerGroup != null) {\n            RemotingChannel channel = this.remotingChannelManager.removeConsumerChannel(context, consumerGroup, ctx.channel());\n            if (channel != null) {\n                ClientChannelInfo clientChannelInfo = new ClientChannelInfo(\n                    channel,\n                    requestHeader.getClientID(),\n                    request.getLanguage(),\n                    request.getVersion());\n                this.messagingProcessor.unRegisterConsumer(context, consumerGroup, clientChannelInfo);\n            } else {\n                log.warn(\"unregister consumer failed, channel not exist, may has been removed, consumerGroup={}, channel={}\", consumerGroup, ctx.channel());\n            }\n        }\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(\"\");\n        return response;\n    }\n\n    protected RemotingCommand checkClientConfig(ChannelHandlerContext ctx, RemotingCommand request,\n        ProxyContext context) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(\"\");\n        return response;\n    }\n\n    public void doChannelCloseEvent(String remoteAddr, Channel channel) {\n        Set<RemotingChannel> remotingChannelSet = this.remotingChannelManager.removeChannel(channel);\n        for (RemotingChannel remotingChannel : remotingChannelSet) {\n            this.messagingProcessor.doChannelCloseEvent(remoteAddr, remotingChannel);\n        }\n    }\n\n    protected class ConsumerIdsChangeListenerImpl implements ConsumerIdsChangeListener {\n\n        @Override\n        public void handle(ConsumerGroupEvent event, String group, Object... args) {\n            if (event == ConsumerGroupEvent.CLIENT_UNREGISTER) {\n                if (args == null || args.length < 1) {\n                    return;\n                }\n                if (args[0] instanceof ClientChannelInfo) {\n                    ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0];\n                    remotingChannelManager.removeConsumerChannel(ProxyContext.createForInner(this.getClass()), group, clientChannelInfo.getChannel());\n                    log.info(\"remove remoting channel when client unregister. clientChannelInfo:{}\", clientChannelInfo);\n                }\n            }\n        }\n\n        @Override\n        public void shutdown() {\n\n        }\n    }\n\n    protected class ProducerChangeListenerImpl implements ProducerChangeListener {\n\n        @Override\n        public void handle(ProducerGroupEvent event, String group, ClientChannelInfo clientChannelInfo) {\n            if (event == ProducerGroupEvent.CLIENT_UNREGISTER) {\n                remotingChannelManager.removeProducerChannel(ProxyContext.createForInner(this.getClass()), group, clientChannelInfo.getChannel());\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ConsumerManagerActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.activity;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.time.Duration;\nimport java.util.ArrayList;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ConsumerGroupInfo;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.Connection;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerConnection;\nimport org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerConnectionListRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupResponseBody;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupResponseHeader;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic class ConsumerManagerActivity extends AbstractRemotingActivity {\n    public ConsumerManagerActivity(RequestPipeline requestPipeline, MessagingProcessor messagingProcessor) {\n        super(requestPipeline, messagingProcessor);\n    }\n\n    @Override\n    protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,\n        ProxyContext context) throws Exception {\n        switch (request.getCode()) {\n            case RequestCode.GET_CONSUMER_LIST_BY_GROUP: {\n                return getConsumerListByGroup(ctx, request, context);\n            }\n            case RequestCode.LOCK_BATCH_MQ: {\n                return lockBatchMQ(ctx, request, context);\n            }\n            case RequestCode.UNLOCK_BATCH_MQ: {\n                return unlockBatchMQ(ctx, request, context);\n            }\n            case RequestCode.UPDATE_CONSUMER_OFFSET:\n            case RequestCode.QUERY_CONSUMER_OFFSET:\n            case RequestCode.SEARCH_OFFSET_BY_TIMESTAMP:\n            case RequestCode.GET_MIN_OFFSET:\n            case RequestCode.GET_MAX_OFFSET:\n            case RequestCode.GET_EARLIEST_MSG_STORETIME: {\n                return request(ctx, request, context, Duration.ofSeconds(3).toMillis());\n            }\n            case RequestCode.GET_CONSUMER_CONNECTION_LIST: {\n                return getConsumerConnectionList(ctx, request, context);\n            }\n            default:\n                break;\n        }\n        return null;\n    }\n\n    protected RemotingCommand getConsumerListByGroup(ChannelHandlerContext ctx, RemotingCommand request,\n        ProxyContext context) throws Exception {\n        RemotingCommand response = RemotingCommand.createResponseCommand(GetConsumerListByGroupResponseHeader.class);\n        GetConsumerListByGroupRequestHeader header = (GetConsumerListByGroupRequestHeader) request.decodeCommandCustomHeader(GetConsumerListByGroupRequestHeader.class);\n        ConsumerGroupInfo consumerGroupInfo = messagingProcessor.getConsumerGroupInfo(context, header.getConsumerGroup());\n        List<String> clientIds = consumerGroupInfo.getAllClientId();\n        GetConsumerListByGroupResponseBody body = new GetConsumerListByGroupResponseBody();\n        body.setConsumerIdList(clientIds);\n        response.setBody(body.encode());\n        response.setCode(ResponseCode.SUCCESS);\n        return response;\n    }\n\n    protected RemotingCommand getConsumerConnectionList(ChannelHandlerContext ctx, RemotingCommand request,\n        ProxyContext context) throws Exception {\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        GetConsumerConnectionListRequestHeader header = (GetConsumerConnectionListRequestHeader) request.decodeCommandCustomHeader(GetConsumerConnectionListRequestHeader.class);\n        ConsumerGroupInfo consumerGroupInfo = messagingProcessor.getConsumerGroupInfo(context, header.getConsumerGroup());\n        if (consumerGroupInfo != null) {\n            ConsumerConnection bodydata = new ConsumerConnection();\n            bodydata.setConsumeFromWhere(consumerGroupInfo.getConsumeFromWhere());\n            bodydata.setConsumeType(consumerGroupInfo.getConsumeType());\n            bodydata.setMessageModel(consumerGroupInfo.getMessageModel());\n            bodydata.getSubscriptionTable().putAll(consumerGroupInfo.getSubscriptionTable());\n\n            Iterator<Map.Entry<Channel, ClientChannelInfo>> it = consumerGroupInfo.getChannelInfoTable().entrySet().iterator();\n            while (it.hasNext()) {\n                ClientChannelInfo info = it.next().getValue();\n                Connection connection = new Connection();\n                connection.setClientId(info.getClientId());\n                connection.setLanguage(info.getLanguage());\n                connection.setVersion(info.getVersion());\n                connection.setClientAddr(RemotingHelper.parseChannelRemoteAddr(info.getChannel()));\n\n                bodydata.getConnectionSet().add(connection);\n            }\n\n            byte[] body = bodydata.encode();\n            response.setBody(body);\n            response.setCode(ResponseCode.SUCCESS);\n            response.setRemark(null);\n\n            return response;\n        }\n\n        response.setCode(ResponseCode.CONSUMER_NOT_ONLINE);\n        response.setRemark(\"the consumer group[\" + header.getConsumerGroup() + \"] not online\");\n        return response;\n    }\n\n    protected RemotingCommand lockBatchMQ(ChannelHandlerContext ctx, RemotingCommand request,\n        ProxyContext context) throws Exception {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        LockBatchRequestBody requestBody = LockBatchRequestBody.decode(request.getBody(), LockBatchRequestBody.class);\n        Set<MessageQueue> mqSet = requestBody.getMqSet();\n        if (mqSet.isEmpty()) {\n            response.setBody(requestBody.encode());\n            response.setRemark(\"MessageQueue set is empty\");\n            return response;\n        }\n\n        String brokerName = new ArrayList<>(mqSet).get(0).getBrokerName();\n        messagingProcessor.request(context, brokerName, request, Duration.ofSeconds(3).toMillis())\n            .thenAccept(r -> writeResponse(ctx, context, request, r))\n            .exceptionally(t -> {\n                writeErrResponse(ctx, context, request, t);\n                return null;\n            });\n        return null;\n    }\n\n    protected RemotingCommand unlockBatchMQ(ChannelHandlerContext ctx, RemotingCommand request,\n        ProxyContext context) throws Exception {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        UnlockBatchRequestBody requestBody = UnlockBatchRequestBody.decode(request.getBody(), UnlockBatchRequestBody.class);\n        Set<MessageQueue> mqSet = requestBody.getMqSet();\n        if (mqSet.isEmpty()) {\n            response.setBody(requestBody.encode());\n            response.setRemark(\"MessageQueue set is empty\");\n            return response;\n        }\n\n        String brokerName = new ArrayList<>(mqSet).get(0).getBrokerName();\n        messagingProcessor.request(context, brokerName, request, Duration.ofSeconds(3).toMillis())\n            .thenAccept(r -> writeResponse(ctx, context, request, r))\n            .exceptionally(t -> {\n                writeErrResponse(ctx, context, request, t);\n                return null;\n            });\n        return null;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.activity;\n\nimport com.alibaba.fastjson2.JSONWriter;\nimport com.google.common.net.HostAndPort;\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.proxy.common.Address;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;\nimport org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.GetRouteInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\npublic class GetTopicRouteActivity extends AbstractRemotingActivity {\n    public GetTopicRouteActivity(RequestPipeline requestPipeline,\n        MessagingProcessor messagingProcessor) {\n        super(requestPipeline, messagingProcessor);\n    }\n\n    @Override\n    protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,\n        ProxyContext context) throws Exception {\n        ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();\n        final RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        final GetRouteInfoRequestHeader requestHeader =\n            (GetRouteInfoRequestHeader) request.decodeCommandCustomHeader(GetRouteInfoRequestHeader.class);\n        List<Address> addressList = new ArrayList<>();\n        // AddressScheme is just a placeholder and will not affect topic route result in this case.\n        addressList.add(new Address(HostAndPort.fromParts(proxyConfig.getRemotingAccessAddr(), proxyConfig.getRemotingListenPort())));\n        ProxyTopicRouteData proxyTopicRouteData = messagingProcessor.getTopicRouteDataForProxy(context, addressList, requestHeader.getTopic());\n        TopicRouteData topicRouteData = proxyTopicRouteData.buildTopicRouteData();\n\n        byte[] content;\n        Boolean standardJsonOnly = requestHeader.getAcceptStandardJsonOnly();\n        if (request.getVersion() >= MQVersion.Version.V4_9_4.ordinal() || null != standardJsonOnly && standardJsonOnly) {\n            content = topicRouteData.encode(JSONWriter.Feature.BrowserCompatible, JSONWriter.Feature.MapSortField);\n        } else {\n            content = topicRouteData.encode();\n        }\n\n        response.setBody(content);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n        return response;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PopMessageActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.activity;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport java.time.Duration;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic class PopMessageActivity extends AbstractRemotingActivity {\n    public PopMessageActivity(RequestPipeline requestPipeline,\n        MessagingProcessor messagingProcessor) {\n        super(requestPipeline, messagingProcessor);\n    }\n\n    @Override\n    protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,\n        ProxyContext context) throws Exception {\n        PopMessageRequestHeader popMessageRequestHeader = (PopMessageRequestHeader) request.decodeCommandCustomHeader(PopMessageRequestHeader.class);\n        long timeoutMillis = popMessageRequestHeader.getPollTime();\n        return request(ctx, request, context, timeoutMillis + Duration.ofSeconds(10).toMillis());\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.activity;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport java.time.Duration;\nimport org.apache.rocketmq.broker.client.ConsumerGroupInfo;\nimport org.apache.rocketmq.common.sysflag.PullSysFlag;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\npublic class PullMessageActivity extends AbstractRemotingActivity {\n    public PullMessageActivity(RequestPipeline requestPipeline,\n        MessagingProcessor messagingProcessor) {\n        super(requestPipeline, messagingProcessor);\n    }\n\n    @Override\n    protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,\n        ProxyContext context) throws Exception {\n        PullMessageRequestHeader requestHeader = (PullMessageRequestHeader) request.decodeCommandCustomHeader(PullMessageRequestHeader.class);\n        int sysFlag = requestHeader.getSysFlag();\n        if (!PullSysFlag.hasSubscriptionFlag(sysFlag)) {\n            ConsumerGroupInfo consumerInfo = messagingProcessor.getConsumerGroupInfo(context, requestHeader.getConsumerGroup());\n            if (consumerInfo == null) {\n                return RemotingCommand.buildErrorResponse(ResponseCode.SUBSCRIPTION_NOT_LATEST,\n                    \"the consumer's subscription not latest\");\n            }\n            SubscriptionData subscriptionData = consumerInfo.findSubscriptionData(requestHeader.getTopic());\n            if (subscriptionData == null) {\n                return RemotingCommand.buildErrorResponse(ResponseCode.SUBSCRIPTION_NOT_EXIST,\n                    \"the consumer's subscription not exist\");\n            }\n            requestHeader.setSysFlag(PullSysFlag.buildSysFlagWithSubscription(sysFlag));\n            requestHeader.setSubscription(subscriptionData.getSubString());\n            requestHeader.setExpressionType(subscriptionData.getExpressionType());\n            request.writeCustomHeader(requestHeader);\n            request.makeCustomHeaderToNet();\n        }\n        long timeoutMillis = requestHeader.getSuspendTimeoutMillis() + Duration.ofSeconds(10).toMillis();\n        return request(ctx, request, context, timeoutMillis);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/RecallMessageActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.activity;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.processor.validator.DefaultTopicMessageTypeValidator;\nimport org.apache.rocketmq.proxy.processor.validator.TopicMessageTypeValidator;\nimport org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;\n\nimport java.time.Duration;\n\npublic class RecallMessageActivity extends AbstractRemotingActivity {\n    TopicMessageTypeValidator topicMessageTypeValidator;\n\n    public RecallMessageActivity(RequestPipeline requestPipeline,\n                                 MessagingProcessor messagingProcessor) {\n        super(requestPipeline, messagingProcessor);\n        this.topicMessageTypeValidator = new DefaultTopicMessageTypeValidator();\n    }\n\n    @Override\n    public RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,\n        ProxyContext context) throws Exception {\n        RecallMessageRequestHeader requestHeader = request.decodeCommandCustomHeader(RecallMessageRequestHeader.class);\n        String topic = requestHeader.getTopic();\n        if (ConfigurationManager.getProxyConfig().isEnableTopicMessageTypeCheck()) {\n            TopicMessageType messageType = messagingProcessor.getMetadataService().getTopicMessageType(context, topic);\n            topicMessageTypeValidator.validate(messageType, TopicMessageType.DELAY);\n        }\n        return request(ctx, request, context, Duration.ofSeconds(2).toMillis());\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.activity;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport java.time.Duration;\nimport java.util.Map;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.processor.validator.DefaultTopicMessageTypeValidator;\nimport org.apache.rocketmq.proxy.processor.validator.TopicMessageTypeValidator;\nimport org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;\nimport org.apache.rocketmq.remoting.protocol.NamespaceUtil;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\n\npublic class SendMessageActivity extends AbstractRemotingActivity {\n    TopicMessageTypeValidator topicMessageTypeValidator;\n\n    public SendMessageActivity(RequestPipeline requestPipeline,\n        MessagingProcessor messagingProcessor) {\n        super(requestPipeline, messagingProcessor);\n        this.topicMessageTypeValidator = new DefaultTopicMessageTypeValidator();\n    }\n\n    @Override\n    protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,\n        ProxyContext context) throws Exception {\n        switch (request.getCode()) {\n            case RequestCode.SEND_MESSAGE:\n            case RequestCode.SEND_MESSAGE_V2:\n            case RequestCode.SEND_BATCH_MESSAGE: {\n                return sendMessage(ctx, request, context);\n            }\n            case RequestCode.CONSUMER_SEND_MSG_BACK: {\n                return consumerSendMessage(ctx, request, context);\n            }\n            default:\n                break;\n        }\n        return null;\n    }\n\n    protected RemotingCommand sendMessage(ChannelHandlerContext ctx, RemotingCommand request,\n        ProxyContext context) throws Exception {\n        SendMessageRequestHeader requestHeader = SendMessageRequestHeader.parseRequestHeader(request);\n        String topic = requestHeader.getTopic();\n        Map<String, String> property = MessageDecoder.string2messageProperties(requestHeader.getProperties());\n        TopicMessageType messageType = TopicMessageType.parseFromMessageProperty(property);\n        if (isNeedCheckTopicMessageType(property)) {\n            if (topicMessageTypeValidator != null) {\n                // Do not check retry or dlq topic\n                if (!NamespaceUtil.isRetryTopic(topic) && !NamespaceUtil.isDLQTopic(topic)) {\n                    TopicMessageType topicMessageType = messagingProcessor.getMetadataService().getTopicMessageType(context, topic);\n                    topicMessageTypeValidator.validate(topicMessageType, messageType);\n                }\n            }\n        }\n        if (!NamespaceUtil.isRetryTopic(topic) && !NamespaceUtil.isDLQTopic(topic)) {\n            if (TopicMessageType.TRANSACTION.equals(messageType)) {\n                messagingProcessor.addTransactionSubscription(context, requestHeader.getProducerGroup(), requestHeader.getTopic());\n            }\n        }\n        return request(ctx, request, context, Duration.ofSeconds(3).toMillis());\n    }\n\n    protected RemotingCommand consumerSendMessage(ChannelHandlerContext ctx, RemotingCommand request,\n        ProxyContext context) throws Exception {\n        return request(ctx, request, context, Duration.ofSeconds(3).toMillis());\n    }\n\n    private boolean isNeedCheckTopicMessageType(Map<String, String> property) {\n        return ConfigurationManager.getProxyConfig().isEnableTopicMessageTypeCheck()\n            && !property.containsKey(MessageConst.PROPERTY_TRANSFER_FLAG);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/TransactionActivity.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.activity;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.processor.TransactionStatus;\nimport org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic class TransactionActivity extends AbstractRemotingActivity {\n\n    public TransactionActivity(RequestPipeline requestPipeline,\n        MessagingProcessor messagingProcessor) {\n        super(requestPipeline, messagingProcessor);\n    }\n\n    @Override\n    protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,\n        ProxyContext context) throws Exception {\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        response.setRemark(null);\n\n        final EndTransactionRequestHeader requestHeader = (EndTransactionRequestHeader) request.decodeCommandCustomHeader(EndTransactionRequestHeader.class);\n\n        TransactionStatus transactionStatus = TransactionStatus.UNKNOWN;\n        switch (requestHeader.getCommitOrRollback()) {\n            case MessageSysFlag.TRANSACTION_COMMIT_TYPE:\n                transactionStatus = TransactionStatus.COMMIT;\n                break;\n            case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:\n                transactionStatus = TransactionStatus.ROLLBACK;\n                break;\n            default:\n                break;\n        }\n\n        this.messagingProcessor.endTransaction(\n            context,\n            requestHeader.getTopic(),\n            requestHeader.getTransactionId(),\n            requestHeader.getMsgId(),\n            requestHeader.getProducerGroup(),\n            transactionStatus,\n            requestHeader.getFromTransactionCheck()\n        );\n        return response;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.channel;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.alibaba.fastjson2.TypeReference;\nimport com.google.common.base.MoreObjects;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelConfig;\nimport io.netty.channel.ChannelFuture;\nimport io.netty.channel.ChannelFutureListener;\nimport io.netty.channel.ChannelMetadata;\nimport java.time.Duration;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.commons.lang3.NotImplementedException;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.utils.ExceptionUtils;\nimport org.apache.rocketmq.common.utils.FutureUtils;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.channel.ChannelHelper;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.processor.channel.ChannelExtendAttributeGetter;\nimport org.apache.rocketmq.proxy.processor.channel.ChannelProtocolType;\nimport org.apache.rocketmq.proxy.processor.channel.RemoteChannel;\nimport org.apache.rocketmq.proxy.processor.channel.RemoteChannelConverter;\nimport org.apache.rocketmq.proxy.remoting.RemotingProxyOutClient;\nimport org.apache.rocketmq.proxy.remoting.common.RemotingConverter;\nimport org.apache.rocketmq.proxy.service.relay.ProxyChannel;\nimport org.apache.rocketmq.proxy.service.relay.ProxyRelayResult;\nimport org.apache.rocketmq.proxy.service.relay.ProxyRelayService;\nimport org.apache.rocketmq.proxy.service.transaction.TransactionData;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.NotifyUnsubscribeLiteRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\npublic class RemotingChannel extends ProxyChannel implements RemoteChannelConverter, ChannelExtendAttributeGetter {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    private static final long DEFAULT_MQ_CLIENT_TIMEOUT = Duration.ofSeconds(3).toMillis();\n    private final String clientId;\n    private final String remoteAddress;\n    private final String localAddress;\n    private final RemotingProxyOutClient remotingProxyOutClient;\n    private final Set<SubscriptionData> subscriptionData;\n\n    public RemotingChannel(RemotingProxyOutClient remotingProxyOutClient, ProxyRelayService proxyRelayService,\n        Channel parent,\n        String clientId, Set<SubscriptionData> subscriptionData) {\n        super(proxyRelayService, parent, parent.id(),\n            NetworkUtil.socketAddress2String(parent.remoteAddress()),\n            NetworkUtil.socketAddress2String(parent.localAddress()));\n        this.remotingProxyOutClient = remotingProxyOutClient;\n        this.clientId = clientId;\n        this.remoteAddress = NetworkUtil.socketAddress2String(parent.remoteAddress());\n        this.localAddress = NetworkUtil.socketAddress2String(parent.localAddress());\n        this.subscriptionData = subscriptionData;\n    }\n\n    @Override\n    public boolean isOpen() {\n        return this.parent().isOpen();\n    }\n\n    @Override\n    public boolean isActive() {\n        return this.parent().isActive();\n    }\n\n    @Override\n    public boolean isWritable() {\n        return this.parent().isWritable();\n    }\n\n    @Override\n    public ChannelFuture close() {\n        return this.parent().close();\n    }\n\n    @Override\n    public ChannelConfig config() {\n        return this.parent().config();\n    }\n\n    @Override\n    public ChannelMetadata metadata() {\n        return this.parent().metadata();\n    }\n\n    @Override\n    protected CompletableFuture<Void> processOtherMessage(Object msg) {\n        this.parent().writeAndFlush(msg);\n        return CompletableFuture.completedFuture(null);\n    }\n\n    @Override\n    protected CompletableFuture<Void> processCheckTransaction(CheckTransactionStateRequestHeader header,\n        MessageExt messageExt, TransactionData transactionData,\n        CompletableFuture<ProxyRelayResult<Void>> responseFuture) {\n        CompletableFuture<Void> writeFuture = new CompletableFuture<>();\n        try {\n            CheckTransactionStateRequestHeader requestHeader = new CheckTransactionStateRequestHeader();\n            requestHeader.setTopic(messageExt.getTopic());\n            requestHeader.setCommitLogOffset(transactionData.getCommitLogOffset());\n            requestHeader.setTranStateTableOffset(transactionData.getTranStateTableOffset());\n            requestHeader.setTransactionId(transactionData.getTransactionId());\n            requestHeader.setMsgId(header.getMsgId());\n\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHECK_TRANSACTION_STATE, requestHeader);\n            request.setBody(RemotingConverter.getInstance().convertMsgToBytes(messageExt));\n\n            this.parent().writeAndFlush(request).addListener((ChannelFutureListener) future -> {\n                if (future.isSuccess()) {\n                    responseFuture.complete(null);\n                    writeFuture.complete(null);\n                } else {\n                    Exception e = new RemotingException(\"write and flush data failed\");\n                    responseFuture.completeExceptionally(e);\n                    writeFuture.completeExceptionally(e);\n                }\n            });\n        } catch (Throwable t) {\n            responseFuture.completeExceptionally(t);\n            writeFuture.completeExceptionally(t);\n        }\n        return writeFuture;\n    }\n\n    @Override\n    protected CompletableFuture<Void> processGetConsumerRunningInfo(RemotingCommand command,\n        GetConsumerRunningInfoRequestHeader header,\n        CompletableFuture<ProxyRelayResult<ConsumerRunningInfo>> responseFuture) {\n        try {\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_RUNNING_INFO, header);\n            this.remotingProxyOutClient.invokeToClient(this.parent(), request, DEFAULT_MQ_CLIENT_TIMEOUT)\n                .thenAccept(response -> {\n                    if (response.getCode() == ResponseCode.SUCCESS) {\n                        ConsumerRunningInfo consumerRunningInfo = ConsumerRunningInfo.decode(response.getBody(), ConsumerRunningInfo.class);\n                        responseFuture.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, \"\", consumerRunningInfo));\n                    } else {\n                        String errMsg = String.format(\"get consumer running info failed, code:%s remark:%s\", response.getCode(), response.getRemark());\n                        RuntimeException e = new RuntimeException(errMsg);\n                        responseFuture.completeExceptionally(e);\n                    }\n                })\n                .exceptionally(t -> {\n                    responseFuture.completeExceptionally(ExceptionUtils.getRealException(t));\n                    return null;\n                });\n            return CompletableFuture.completedFuture(null);\n        } catch (Throwable t) {\n            responseFuture.completeExceptionally(t);\n            return FutureUtils.completeExceptionally(t);\n        }\n    }\n\n    @Override\n    protected CompletableFuture<Void> processNotifyUnsubscribeLite(NotifyUnsubscribeLiteRequestHeader header) {\n        throw new NotImplementedException();\n    }\n\n    @Override\n    protected CompletableFuture<Void> processConsumeMessageDirectly(RemotingCommand command,\n        ConsumeMessageDirectlyResultRequestHeader header, MessageExt messageExt,\n        CompletableFuture<ProxyRelayResult<ConsumeMessageDirectlyResult>> responseFuture) {\n        try {\n            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONSUME_MESSAGE_DIRECTLY, header);\n            request.setBody(RemotingConverter.getInstance().convertMsgToBytes(messageExt));\n\n            this.remotingProxyOutClient.invokeToClient(this.parent(), request, DEFAULT_MQ_CLIENT_TIMEOUT)\n                .thenAccept(response -> {\n                    if (response.getCode() == ResponseCode.SUCCESS) {\n                        ConsumeMessageDirectlyResult result = ConsumeMessageDirectlyResult.decode(response.getBody(), ConsumeMessageDirectlyResult.class);\n                        responseFuture.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, \"\", result));\n                    } else {\n                        String errMsg = String.format(\"consume message directly failed, code:%s remark:%s\", response.getCode(), response.getRemark());\n                        RuntimeException e = new RuntimeException(errMsg);\n                        responseFuture.completeExceptionally(e);\n                    }\n                })\n                .exceptionally(t -> {\n                    responseFuture.completeExceptionally(ExceptionUtils.getRealException(t));\n                    return null;\n                });\n            return CompletableFuture.completedFuture(null);\n        } catch (Throwable t) {\n            responseFuture.completeExceptionally(t);\n            return FutureUtils.completeExceptionally(t);\n        }\n    }\n\n    public String getClientId() {\n        return clientId;\n    }\n\n    @Override\n    public String getChannelExtendAttribute() {\n        if (this.subscriptionData == null) {\n            return null;\n        }\n        return JSON.toJSONString(this.subscriptionData);\n    }\n\n    public static Set<SubscriptionData> parseChannelExtendAttribute(Channel channel) {\n        if (ChannelHelper.getChannelProtocolType(channel).equals(ChannelProtocolType.REMOTING) &&\n            channel instanceof ChannelExtendAttributeGetter) {\n            String attr = ((ChannelExtendAttributeGetter) channel).getChannelExtendAttribute();\n            if (attr == null) {\n                return null;\n            }\n\n            try {\n                return JSON.parseObject(attr, new TypeReference<Set<SubscriptionData>>() {\n                });\n            } catch (Exception e) {\n                log.error(\"convert remoting extend attribute to subscriptionDataSet failed. data:{}\", attr, e);\n                return null;\n            }\n        }\n        return null;\n    }\n\n    @Override\n    public RemoteChannel toRemoteChannel() {\n        return new RemoteChannel(\n            ConfigurationManager.getProxyConfig().getLocalServeAddr(),\n            this.getRemoteAddress(),\n            this.getLocalAddress(),\n            ChannelProtocolType.REMOTING,\n            this.getChannelExtendAttribute());\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"parent\", parent())\n            .add(\"clientId\", clientId)\n            .add(\"remoteAddress\", remoteAddress)\n            .add(\"localAddress\", localAddress)\n            .add(\"subscriptionData\", subscriptionData)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.channel;\n\nimport io.netty.channel.Channel;\nimport java.util.Collections;\nimport java.util.HashSet;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.atomic.AtomicReference;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.common.utils.StartAndShutdown;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.remoting.RemotingProxyOutClient;\nimport org.apache.rocketmq.proxy.service.relay.ProxyRelayService;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\npublic class RemotingChannelManager implements StartAndShutdown {\n    protected final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    private final ProxyRelayService proxyRelayService;\n    protected final ConcurrentMap<String /* group */, Map<Channel /* raw channel */, RemotingChannel>> groupChannelMap = new ConcurrentHashMap<>();\n\n    private final RemotingProxyOutClient remotingProxyOutClient;\n\n    public RemotingChannelManager(RemotingProxyOutClient remotingProxyOutClient, ProxyRelayService proxyRelayService) {\n        this.remotingProxyOutClient = remotingProxyOutClient;\n        this.proxyRelayService = proxyRelayService;\n    }\n\n    protected String buildProducerKey(String group) {\n        return buildKey(\"p\", group);\n    }\n\n    protected String buildConsumerKey(String group) {\n        return buildKey(\"c\", group);\n    }\n\n    protected String buildKey(String prefix, String group) {\n        return prefix + group;\n    }\n\n    public RemotingChannel createProducerChannel(ProxyContext ctx, Channel channel, String group, String clientId) {\n        return createChannel(channel, buildProducerKey(group), clientId, Collections.emptySet());\n    }\n\n    public RemotingChannel createConsumerChannel(ProxyContext ctx, Channel channel, String group, String clientId, Set<SubscriptionData> subscriptionData) {\n        return createChannel(channel, buildConsumerKey(group), clientId, subscriptionData);\n    }\n\n    protected RemotingChannel createChannel(Channel channel, String group, String clientId, Set<SubscriptionData> subscriptionData) {\n        this.groupChannelMap.compute(group, (groupKey, clientIdMap) -> {\n            if (clientIdMap == null) {\n                clientIdMap = new ConcurrentHashMap<>();\n            }\n            clientIdMap.computeIfAbsent(channel, clientIdKey -> new RemotingChannel(remotingProxyOutClient, proxyRelayService, channel, clientId, subscriptionData));\n            return clientIdMap;\n        });\n        return getChannel(group, channel);\n    }\n\n    protected RemotingChannel getChannel(String group, Channel channel) {\n        Map<Channel, RemotingChannel> clientIdChannelMap = this.groupChannelMap.get(group);\n        if (clientIdChannelMap == null) {\n            return null;\n        }\n        return clientIdChannelMap.get(channel);\n    }\n\n    public Set<RemotingChannel> removeChannel(Channel channel) {\n        Set<RemotingChannel> removedChannelSet = new HashSet<>();\n        Set<String> groupKeySet = groupChannelMap.keySet();\n        for (String group : groupKeySet) {\n            RemotingChannel remotingChannel = removeChannel(group, channel);\n            if (remotingChannel != null) {\n                removedChannelSet.add(remotingChannel);\n            }\n        }\n        return removedChannelSet;\n    }\n\n    public RemotingChannel removeProducerChannel(ProxyContext ctx, String group, Channel channel) {\n        return removeChannel(buildProducerKey(group), channel);\n    }\n\n    public RemotingChannel removeConsumerChannel(ProxyContext ctx, String group, Channel channel) {\n        return removeChannel(buildConsumerKey(group), channel);\n    }\n\n    protected RemotingChannel removeChannel(String group, Channel channel) {\n        AtomicReference<RemotingChannel> channelRef = new AtomicReference<>();\n\n        this.groupChannelMap.computeIfPresent(group, (groupKey, channelMap) -> {\n            channelRef.set(channelMap.remove(getOrgRawChannel(channel)));\n            if (channelMap.isEmpty()) {\n                return null;\n            }\n            return channelMap;\n        });\n        return channelRef.get();\n    }\n\n    /**\n     * to get the org channel pass by nettyRemotingServer\n     * @param channel\n     * @return\n     */\n    protected Channel getOrgRawChannel(Channel channel) {\n        if (channel instanceof RemotingChannel) {\n            return channel.parent();\n        }\n        return channel;\n    }\n\n    @Override\n    public void shutdown() throws Exception {\n\n    }\n\n    @Override\n    public void start() throws Exception {\n\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/common/RemotingConverter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.common;\n\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class RemotingConverter {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    protected static final Object INSTANCE_CREATE_LOCK = new Object();\n    protected static volatile RemotingConverter instance;\n\n    public static RemotingConverter getInstance() {\n        if (instance == null) {\n            synchronized (INSTANCE_CREATE_LOCK) {\n                if (instance == null) {\n                    instance = new RemotingConverter();\n                }\n            }\n        }\n        return instance;\n    }\n\n    public byte[] convertMsgToBytes(final MessageExt msg) throws Exception {\n        // change to 0 for recalculate storeSize\n        msg.setStoreSize(0);\n        if (msg.getTopic().length() > Byte.MAX_VALUE) {\n            log.warn(\"Topic length is too long, topic: {}\", msg.getTopic());\n        }\n        return MessageDecoder.encode(msg, false);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/pipeline/AuthenticationPipeline.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.pipeline;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.auth.authentication.AuthenticationEvaluator;\nimport org.apache.rocketmq.auth.authentication.context.AuthenticationContext;\nimport org.apache.rocketmq.auth.authentication.exception.AuthenticationException;\nimport org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic class AuthenticationPipeline implements RequestPipeline {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    private final AuthConfig authConfig;\n    private final AuthenticationEvaluator authenticationEvaluator;\n\n    public AuthenticationPipeline(AuthConfig authConfig, MessagingProcessor messagingProcessor) {\n        this.authConfig = authConfig;\n        this.authenticationEvaluator = AuthenticationFactory.getEvaluator(authConfig, messagingProcessor::getMetadataService);\n    }\n\n    @Override\n    public void execute(ChannelHandlerContext ctx, RemotingCommand request, ProxyContext context) throws Exception {\n        if (!authConfig.isAuthenticationEnabled()) {\n            return;\n        }\n        try {\n            AuthenticationContext authenticationContext = newContext(ctx, request, context);\n            authenticationEvaluator.evaluate(authenticationContext);\n        } catch (AuthenticationException ex) {\n            throw ex;\n        } catch (Throwable ex) {\n            LOGGER.error(\"authenticate failed, request:{}\", request, ex);\n            throw ex;\n        }\n    }\n\n    protected AuthenticationContext newContext(ChannelHandlerContext ctx, RemotingCommand request, ProxyContext context) {\n        return AuthenticationFactory.newContext(authConfig, ctx, request);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/pipeline/AuthorizationPipeline.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.pipeline;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.List;\nimport org.apache.rocketmq.auth.authentication.exception.AuthenticationException;\nimport org.apache.rocketmq.auth.authorization.AuthorizationEvaluator;\nimport org.apache.rocketmq.auth.authorization.context.AuthorizationContext;\nimport org.apache.rocketmq.auth.authorization.exception.AuthorizationException;\nimport org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic class AuthorizationPipeline implements RequestPipeline {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    private final AuthConfig authConfig;\n    private final AuthorizationEvaluator authorizationEvaluator;\n\n    public AuthorizationPipeline(AuthConfig authConfig, MessagingProcessor messagingProcessor) {\n        this.authConfig = authConfig;\n        this.authorizationEvaluator = AuthorizationFactory.getEvaluator(authConfig, messagingProcessor::getMetadataService);\n    }\n\n    @Override\n    public void execute(ChannelHandlerContext ctx, RemotingCommand request, ProxyContext context) throws Exception {\n        if (!authConfig.isAuthorizationEnabled()) {\n            return;\n        }\n        try {\n            List<AuthorizationContext> contexts = newContexts(request, ctx, context);\n            authorizationEvaluator.evaluate(contexts);\n        } catch (AuthorizationException | AuthenticationException ex) {\n            throw ex;\n        }  catch (Throwable ex) {\n            LOGGER.error(\"authorize failed, request:{}\", request, ex);\n            throw ex;\n        }\n    }\n\n    protected List<AuthorizationContext> newContexts(RemotingCommand request, ChannelHandlerContext ctx, ProxyContext context) {\n        return AuthorizationFactory.newContexts(authConfig, ctx, request);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/pipeline/ContextInitPipeline.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.remoting.pipeline;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.processor.channel.ChannelProtocolType;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.netty.AttributeKeys;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic class ContextInitPipeline implements RequestPipeline {\n\n    @Override\n    public void execute(ChannelHandlerContext ctx, RemotingCommand request, ProxyContext context) throws Exception {\n        Channel channel = ctx.channel();\n        LanguageCode languageCode = RemotingHelper.getAttributeValue(AttributeKeys.LANGUAGE_CODE_KEY, channel);\n        String clientId = RemotingHelper.getAttributeValue(AttributeKeys.CLIENT_ID_KEY, channel);\n        Integer version = RemotingHelper.getAttributeValue(AttributeKeys.VERSION_KEY, channel);\n        context.setAction(RemotingHelper.getRequestCodeDesc(request.getCode()))\n            .setProtocolType(ChannelProtocolType.REMOTING.getName())\n            .setChannel(channel)\n            .setLocalAddress(NetworkUtil.socketAddress2String(ctx.channel().localAddress()))\n            .setRemoteAddress(RemotingHelper.parseChannelRemoteAddr(ctx.channel()));\n        if (languageCode != null) {\n            context.setLanguage(languageCode.name());\n        }\n        if (clientId != null) {\n            context.setClientID(clientId);\n        }\n        if (version != null) {\n            context.setClientVersion(MQVersion.getVersionDesc(version));\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/pipeline/RequestPipeline.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.pipeline;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic interface RequestPipeline {\n\n    void execute(ChannelHandlerContext ctx, RemotingCommand request, ProxyContext context) throws Exception;\n\n    default RequestPipeline pipe(RequestPipeline source) {\n        return (ctx, request, context) -> {\n            source.execute(ctx, request, context);\n            execute(ctx, request, context);\n        };\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/ProtocolHandler.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.protocol;\n\nimport io.netty.buffer.ByteBuf;\nimport io.netty.channel.ChannelHandlerContext;\n\npublic interface ProtocolHandler {\n\n    boolean match(ByteBuf msg);\n\n    void config(final ChannelHandlerContext ctx, final ByteBuf msg);\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/ProtocolNegotiationHandler.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\npackage org.apache.rocketmq.proxy.remoting.protocol;\n\nimport io.netty.buffer.ByteBuf;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.handler.codec.ByteToMessageDecoder;\nimport java.util.ArrayList;\nimport java.util.List;\n\npublic class ProtocolNegotiationHandler extends ByteToMessageDecoder {\n\n    private final List<ProtocolHandler> protocolHandlerList = new ArrayList<ProtocolHandler>();\n    private final ProtocolHandler fallbackProtocolHandler;\n\n    public ProtocolNegotiationHandler(ProtocolHandler fallbackProtocolHandler) {\n        this.fallbackProtocolHandler = fallbackProtocolHandler;\n    }\n\n    public ProtocolNegotiationHandler addProtocolHandler(ProtocolHandler protocolHandler) {\n        protocolHandlerList.add(protocolHandler);\n        return this;\n    }\n\n    @Override\n    protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {\n        // use 4 bytes to judge protocol\n        if (in.readableBytes() < 4) {\n            return;\n        }\n\n        ProtocolHandler protocolHandler = null;\n        for (ProtocolHandler curProtocolHandler : protocolHandlerList) {\n            if (curProtocolHandler.match(in)) {\n                protocolHandler = curProtocolHandler;\n                break;\n            }\n        }\n\n        if (protocolHandler == null) {\n            protocolHandler = fallbackProtocolHandler;\n        }\n\n        protocolHandler.config(ctx, in);\n        ctx.pipeline().remove(this);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/HAProxyMessageForwarder.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.protocol.http2proxy;\n\nimport io.netty.buffer.ByteBuf;\nimport io.netty.buffer.Unpooled;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.channel.ChannelInboundHandlerAdapter;\nimport io.netty.handler.codec.haproxy.HAProxyCommand;\nimport io.netty.handler.codec.haproxy.HAProxyMessage;\nimport io.netty.handler.codec.haproxy.HAProxyProtocolVersion;\nimport io.netty.handler.codec.haproxy.HAProxyProxiedProtocol;\nimport io.netty.handler.codec.haproxy.HAProxyTLV;\nimport io.netty.util.Attribute;\nimport io.netty.util.DefaultAttributeMap;\nimport java.lang.reflect.Field;\nimport java.nio.charset.Charset;\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.commons.codec.DecoderException;\nimport org.apache.commons.codec.binary.Hex;\nimport org.apache.commons.lang3.ArrayUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.acl.common.AclUtils;\nimport org.apache.rocketmq.common.constant.CommonConstants;\nimport org.apache.rocketmq.common.constant.HAProxyConstants;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.netty.AttributeKeys;\n\npublic class HAProxyMessageForwarder extends ChannelInboundHandlerAdapter {\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME);\n\n    private static final Field FIELD_ATTRIBUTE =\n        FieldUtils.getField(DefaultAttributeMap.class, \"attributes\", true);\n\n    private final Channel outboundChannel;\n\n    public HAProxyMessageForwarder(final Channel outboundChannel) {\n        this.outboundChannel = outboundChannel;\n    }\n\n    @Override\n    public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {\n        try {\n            forwardHAProxyMessage(ctx.channel(), outboundChannel);\n            ctx.fireChannelRead(msg);\n        } catch (Exception e) {\n            log.error(\"Forward HAProxyMessage from Remoting to gRPC server error.\", e);\n            throw e;\n        } finally {\n            ctx.pipeline().remove(this);\n        }\n    }\n\n    private void forwardHAProxyMessage(Channel inboundChannel, Channel outboundChannel) throws Exception {\n        if (!(inboundChannel instanceof DefaultAttributeMap)) {\n            return;\n        }\n\n        HAProxyMessage message = buildHAProxyMessage(inboundChannel);\n        if (message == null) {\n            return;\n        }\n\n        outboundChannel.writeAndFlush(message).sync();\n    }\n\n    protected HAProxyMessage buildHAProxyMessage(Channel inboundChannel) throws IllegalAccessException, DecoderException {\n        String sourceAddress = null, destinationAddress = null;\n        int sourcePort = 0, destinationPort = 0;\n        if (inboundChannel.hasAttr(AttributeKeys.PROXY_PROTOCOL_ADDR)) {\n            Attribute<?>[] attributes = (Attribute<?>[]) FieldUtils.readField(FIELD_ATTRIBUTE, inboundChannel);\n            if (ArrayUtils.isEmpty(attributes)) {\n                return null;\n            }\n            for (Attribute<?> attribute : attributes) {\n                String attributeKey = attribute.key().name();\n                if (!StringUtils.startsWith(attributeKey, HAProxyConstants.PROXY_PROTOCOL_PREFIX)) {\n                    continue;\n                }\n                String attributeValue = (String) attribute.get();\n                if (StringUtils.isEmpty(attributeValue)) {\n                    continue;\n                }\n                if (attribute.key() == AttributeKeys.PROXY_PROTOCOL_ADDR) {\n                    sourceAddress = attributeValue;\n                }\n                if (attribute.key() == AttributeKeys.PROXY_PROTOCOL_PORT) {\n                    sourcePort = Integer.parseInt(attributeValue);\n                }\n                if (attribute.key() == AttributeKeys.PROXY_PROTOCOL_SERVER_ADDR) {\n                    destinationAddress = attributeValue;\n                }\n                if (attribute.key() == AttributeKeys.PROXY_PROTOCOL_SERVER_PORT) {\n                    destinationPort = Integer.parseInt(attributeValue);\n                }\n            }\n        } else {\n            String remoteAddr = RemotingHelper.parseChannelRemoteAddr(inboundChannel);\n            sourceAddress = StringUtils.substringBeforeLast(remoteAddr, CommonConstants.COLON);\n            sourcePort = Integer.parseInt(StringUtils.substringAfterLast(remoteAddr, CommonConstants.COLON));\n\n            String localAddr = RemotingHelper.parseChannelLocalAddr(inboundChannel);\n            destinationAddress = StringUtils.substringBeforeLast(localAddr, CommonConstants.COLON);\n            destinationPort = Integer.parseInt(StringUtils.substringAfterLast(localAddr, CommonConstants.COLON));\n        }\n\n        HAProxyProxiedProtocol proxiedProtocol = AclUtils.isColon(sourceAddress) ? HAProxyProxiedProtocol.TCP6 :\n            HAProxyProxiedProtocol.TCP4;\n\n        List<HAProxyTLV> haProxyTLVs = buildHAProxyTLV(inboundChannel);\n\n        return new HAProxyMessage(HAProxyProtocolVersion.V2, HAProxyCommand.PROXY,\n            proxiedProtocol, sourceAddress, destinationAddress, sourcePort, destinationPort, haProxyTLVs);\n    }\n\n    protected List<HAProxyTLV> buildHAProxyTLV(Channel inboundChannel) throws IllegalAccessException, DecoderException {\n        List<HAProxyTLV> result = new ArrayList<>();\n        if (!inboundChannel.hasAttr(AttributeKeys.PROXY_PROTOCOL_ADDR)) {\n            return result;\n        }\n        Attribute<?>[] attributes = (Attribute<?>[]) FieldUtils.readField(FIELD_ATTRIBUTE, inboundChannel);\n        if (ArrayUtils.isEmpty(attributes)) {\n            return result;\n        }\n        for (Attribute<?> attribute : attributes) {\n            String attributeKey = attribute.key().name();\n            if (!StringUtils.startsWith(attributeKey, HAProxyConstants.PROXY_PROTOCOL_TLV_PREFIX)) {\n                continue;\n            }\n            String attributeValue = (String) attribute.get();\n            HAProxyTLV haProxyTLV = buildHAProxyTLV(attributeKey, attributeValue);\n            if (haProxyTLV != null) {\n                result.add(haProxyTLV);\n            }\n        }\n        return result;\n    }\n\n    protected HAProxyTLV buildHAProxyTLV(String attributeKey, String attributeValue) throws DecoderException {\n        String typeString = StringUtils.substringAfter(attributeKey, HAProxyConstants.PROXY_PROTOCOL_TLV_PREFIX);\n        ByteBuf byteBuf = Unpooled.buffer();\n        byteBuf.writeBytes(attributeValue.getBytes(Charset.defaultCharset()));\n        return new HAProxyTLV(Hex.decodeHex(typeString)[0], byteBuf);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProtocolProxyHandler.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.protocol.http2proxy;\n\nimport io.netty.bootstrap.Bootstrap;\nimport io.netty.buffer.ByteBuf;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelFuture;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.channel.ChannelInitializer;\nimport io.netty.channel.ChannelOption;\nimport io.netty.handler.codec.haproxy.HAProxyMessageEncoder;\nimport io.netty.handler.ssl.ApplicationProtocolConfig;\nimport io.netty.handler.ssl.ApplicationProtocolNames;\nimport io.netty.handler.ssl.SslContext;\nimport io.netty.handler.ssl.SslContextBuilder;\nimport io.netty.handler.ssl.SslHandler;\nimport io.netty.handler.ssl.SslProvider;\nimport io.netty.handler.ssl.util.InsecureTrustManagerFactory;\nimport javax.net.ssl.SSLException;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.remoting.protocol.ProtocolHandler;\nimport org.apache.rocketmq.remoting.common.TlsMode;\nimport org.apache.rocketmq.remoting.netty.TlsSystemConfig;\n\npublic class Http2ProtocolProxyHandler implements ProtocolHandler {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME);\n    private static final String LOCAL_HOST = \"127.0.0.1\";\n    /**\n     * The int value of \"PRI \". Now use 4 bytes to judge protocol, may be has potential risks if there is a new protocol\n     * which start with \"PRI \" in the future\n     * <p>\n     * The full HTTP/2 connection preface is \"PRI * HTTP/2.0\\r\\n\\r\\nSM\\r\\n\\r\\n\"\n     * <p>\n     * ref: https://datatracker.ietf.org/doc/html/rfc7540#section-3.5\n     */\n    private static final int PRI_INT = 0x50524920;\n\n    private final SslContext sslContext;\n\n    public Http2ProtocolProxyHandler() {\n        try {\n            TlsMode tlsMode = TlsSystemConfig.tlsMode;\n            if (TlsMode.DISABLED.equals(tlsMode)) {\n                sslContext = null;\n            } else {\n                sslContext = SslContextBuilder\n                    .forClient()\n                    .sslProvider(SslProvider.OPENSSL)\n                    .trustManager(InsecureTrustManagerFactory.INSTANCE)\n                    .applicationProtocolConfig(new ApplicationProtocolConfig(\n                        ApplicationProtocolConfig.Protocol.ALPN,\n                        ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE,\n                        ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT,\n                        ApplicationProtocolNames.HTTP_2))\n                    .build();\n            }\n        } catch (SSLException e) {\n            log.error(\"Failed to create SslContext for Http2ProtocolProxyHandler\", e);\n            throw new RuntimeException(\"Failed to create SslContext for Http2ProtocolProxyHandler\", e);\n        }\n    }\n\n    @Override\n    public boolean match(ByteBuf in) {\n        if (!ConfigurationManager.getProxyConfig().isEnableRemotingLocalProxyGrpc()) {\n            return false;\n        }\n\n        // If starts with 'PRI '\n        return in.getInt(in.readerIndex()) == PRI_INT;\n    }\n\n    @Override\n    public void config(final ChannelHandlerContext ctx, final ByteBuf msg) {\n        // proxy channel to http2 server\n        final Channel inboundChannel = ctx.channel();\n\n        ProxyConfig config = ConfigurationManager.getProxyConfig();\n        // Start the connection attempt.\n        Bootstrap b = new Bootstrap();\n        b.group(inboundChannel.eventLoop())\n            .channel(ctx.channel().getClass())\n            .handler(new ChannelInitializer<Channel>() {\n                @Override\n                protected void initChannel(Channel ch) throws Exception {\n                    ch.pipeline().addLast(null, Http2ProxyBackendHandler.HANDLER_NAME,\n                            new Http2ProxyBackendHandler(inboundChannel));\n                }\n            })\n            .option(ChannelOption.AUTO_READ, false)\n            .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, config.getLocalProxyConnectTimeoutMs());\n        ChannelFuture f;\n        try {\n            f = b.connect(LOCAL_HOST, config.getGrpcServerPort()).sync();\n        } catch (Exception e) {\n            log.error(\"connect http2 server failed. port:{}\", config.getGrpcServerPort(), e);\n            inboundChannel.close();\n            return;\n        }\n\n        final Channel outboundChannel = f.channel();\n        configPipeline(inboundChannel, outboundChannel);\n\n        SslHandler sslHandler = null;\n        if (sslContext != null) {\n            sslHandler = sslContext.newHandler(outboundChannel.alloc(), LOCAL_HOST, config.getGrpcServerPort());\n        }\n        ctx.pipeline().addLast(new Http2ProxyFrontendHandler(outboundChannel, sslHandler));\n    }\n\n    protected void configPipeline(Channel inboundChannel, Channel outboundChannel) {\n        inboundChannel.pipeline().addLast(new HAProxyMessageForwarder(outboundChannel));\n        outboundChannel.pipeline().addFirst(HAProxyMessageEncoder.INSTANCE);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProxyBackendHandler.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.protocol.http2proxy;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelFuture;\nimport io.netty.channel.ChannelFutureListener;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.channel.ChannelInboundHandlerAdapter;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class Http2ProxyBackendHandler extends ChannelInboundHandlerAdapter {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME);\n\n    public static final String HANDLER_NAME = \"Http2ProxyBackendHandler\";\n\n    private final Channel inboundChannel;\n\n    public Http2ProxyBackendHandler(Channel inboundChannel) {\n        this.inboundChannel = inboundChannel;\n    }\n\n    @Override\n    public void channelActive(ChannelHandlerContext ctx) {\n        ctx.read();\n    }\n\n    @Override\n    public void channelRead(final ChannelHandlerContext ctx, Object msg) {\n        inboundChannel.writeAndFlush(msg).addListener(new ChannelFutureListener() {\n            @Override\n            public void operationComplete(ChannelFuture future) {\n                if (future.isSuccess()) {\n                    ctx.channel().read();\n                } else {\n                    future.channel().close();\n                }\n            }\n        });\n    }\n\n    @Override\n    public void channelInactive(ChannelHandlerContext ctx) {\n        Http2ProxyFrontendHandler.closeOnFlush(inboundChannel);\n    }\n\n    @Override\n    public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {\n        log.error(\"Http2ProxyBackendHandler#exceptionCaught\", cause);\n        Http2ProxyFrontendHandler.closeOnFlush(ctx.channel());\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProxyFrontendHandler.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.protocol.http2proxy;\n\nimport io.netty.buffer.Unpooled;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelFutureListener;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.channel.ChannelInboundHandlerAdapter;\nimport io.netty.handler.ssl.SslHandler;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class Http2ProxyFrontendHandler extends ChannelInboundHandlerAdapter {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME);\n\n    public static final String HANDLER_NAME = \"SslHandler\";\n\n    // As we use inboundChannel.eventLoop() when building the Bootstrap this does not need to be volatile as\n    // the outboundChannel will use the same EventLoop (and therefore Thread) as the inboundChannel.\n    private final Channel outboundChannel;\n    private final SslHandler sslHandler;\n\n    public Http2ProxyFrontendHandler(final Channel outboundChannel, final SslHandler sslHandler) {\n        this.outboundChannel = outboundChannel;\n        this.sslHandler = sslHandler;\n    }\n\n    @Override\n    public void channelRead(final ChannelHandlerContext ctx, Object msg) {\n        if (outboundChannel.isActive()) {\n            if (sslHandler != null && outboundChannel.pipeline().get(HANDLER_NAME) == null) {\n                outboundChannel.pipeline().addBefore(Http2ProxyBackendHandler.HANDLER_NAME, HANDLER_NAME, sslHandler);\n            }\n\n            outboundChannel.writeAndFlush(msg).addListener((ChannelFutureListener) future -> {\n                if (future.isSuccess()) {\n                    // was able to flush out data, start to read the next chunk\n                    ctx.channel().read();\n                } else {\n                    future.channel().close();\n                }\n            });\n        }\n    }\n\n    @Override\n    public void channelInactive(ChannelHandlerContext ctx) {\n        if (outboundChannel != null) {\n            closeOnFlush(outboundChannel);\n        }\n    }\n\n    @Override\n    public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {\n        log.error(\"Http2ProxyFrontendHandler#exceptionCaught\", cause);\n        closeOnFlush(ctx.channel());\n    }\n\n    /**\n     * Closes the specified channel after all queued write requests are flushed.\n     */\n    static void closeOnFlush(Channel ch) {\n        if (ch.isActive()) {\n            ch.writeAndFlush(Unpooled.EMPTY_BUFFER).addListener(ChannelFutureListener.CLOSE);\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/remoting/RemotingProtocolHandler.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.protocol.remoting;\n\nimport io.netty.buffer.ByteBuf;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.function.Supplier;\nimport org.apache.rocketmq.proxy.remoting.protocol.ProtocolHandler;\nimport org.apache.rocketmq.remoting.netty.NettyDecoder;\nimport org.apache.rocketmq.remoting.netty.NettyEncoder;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingServer;\nimport org.apache.rocketmq.remoting.netty.RemotingCodeDistributionHandler;\n\npublic class RemotingProtocolHandler implements ProtocolHandler {\n\n    private final Supplier<NettyEncoder> encoderSupplier;\n    private final Supplier<RemotingCodeDistributionHandler> remotingCodeDistributionHandlerSupplier;\n    private final Supplier<NettyRemotingServer.NettyConnectManageHandler> connectionManageHandlerSupplier;\n    private final Supplier<NettyRemotingServer.NettyServerHandler> serverHandlerSupplier;\n\n    public RemotingProtocolHandler(Supplier<NettyEncoder> encoderSupplier,\n        Supplier<RemotingCodeDistributionHandler> remotingCodeDistributionHandlerSupplier,\n        Supplier<NettyRemotingServer.NettyConnectManageHandler> connectionManageHandlerSupplier,\n        Supplier<NettyRemotingServer.NettyServerHandler> serverHandlerSupplier) {\n        this.encoderSupplier = encoderSupplier;\n        this.remotingCodeDistributionHandlerSupplier = remotingCodeDistributionHandlerSupplier;\n        this.connectionManageHandlerSupplier = connectionManageHandlerSupplier;\n        this.serverHandlerSupplier = serverHandlerSupplier;\n    }\n\n    @Override\n    public boolean match(ByteBuf in) {\n        return true;\n    }\n\n    @Override\n    public void config(ChannelHandlerContext ctx, ByteBuf msg) {\n        ctx.pipeline().addLast(\n            this.encoderSupplier.get(),\n            new NettyDecoder(),\n            this.remotingCodeDistributionHandlerSupplier.get(),\n            this.connectionManageHandlerSupplier.get(),\n            this.serverHandlerSupplier.get()\n        );\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service;\n\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ConsumerGroupEvent;\nimport org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;\nimport org.apache.rocketmq.broker.client.ConsumerManager;\nimport org.apache.rocketmq.broker.client.ProducerChangeListener;\nimport org.apache.rocketmq.broker.client.ProducerGroupEvent;\nimport org.apache.rocketmq.broker.client.ProducerManager;\nimport org.apache.rocketmq.client.common.NameserverAccessConfig;\nimport org.apache.rocketmq.client.impl.mqclient.DoNothingClientRemotingProcessor;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;\nimport org.apache.rocketmq.common.ObjectCreator;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.AbstractStartAndShutdown;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.service.admin.AdminService;\nimport org.apache.rocketmq.proxy.service.admin.DefaultAdminService;\nimport org.apache.rocketmq.proxy.service.client.ClusterConsumerManager;\nimport org.apache.rocketmq.proxy.service.client.ProxyClientRemotingProcessor;\nimport org.apache.rocketmq.proxy.service.lite.LiteSubscriptionService;\nimport org.apache.rocketmq.proxy.service.message.ClusterMessageService;\nimport org.apache.rocketmq.proxy.service.message.MessageService;\nimport org.apache.rocketmq.proxy.service.metadata.ClusterMetadataService;\nimport org.apache.rocketmq.proxy.service.metadata.MetadataService;\nimport org.apache.rocketmq.proxy.service.relay.ClusterProxyRelayService;\nimport org.apache.rocketmq.proxy.service.relay.ProxyRelayService;\nimport org.apache.rocketmq.proxy.service.route.ClusterTopicRouteService;\nimport org.apache.rocketmq.proxy.service.route.TopicRouteService;\nimport org.apache.rocketmq.proxy.service.transaction.ClusterTransactionService;\nimport org.apache.rocketmq.proxy.service.transaction.TransactionService;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.RemotingClient;\n\npublic class ClusterServiceManager extends AbstractStartAndShutdown implements ServiceManager {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    protected ClusterTransactionService clusterTransactionService;\n    protected ProducerManager producerManager;\n    protected ClusterConsumerManager consumerManager;\n    protected TopicRouteService topicRouteService;\n    protected MessageService messageService;\n    protected ProxyRelayService proxyRelayService;\n    protected ClusterMetadataService metadataService;\n    protected AdminService adminService;\n    protected LiteSubscriptionService liteSubscriptionService;\n\n    protected ScheduledExecutorService scheduledExecutorService;\n    protected MQClientAPIFactory messagingClientAPIFactory;\n    protected MQClientAPIFactory operationClientAPIFactory;\n    protected MQClientAPIFactory transactionClientAPIFactory;\n    protected MQClientAPIFactory liteSubscriptionAPIFactory;\n\n    public ClusterServiceManager(RPCHook rpcHook) {\n        this(rpcHook, null);\n    }\n\n    public ClusterServiceManager(RPCHook rpcHook, ObjectCreator<RemotingClient> remotingClientCreator) {\n        ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();\n        NameserverAccessConfig nameserverAccessConfig = new NameserverAccessConfig(proxyConfig.getNamesrvAddr(),\n            proxyConfig.getNamesrvDomain(), proxyConfig.getNamesrvDomainSubgroup());\n        this.scheduledExecutorService = ThreadUtils.newScheduledThreadPool(3);\n\n        this.messagingClientAPIFactory = new MQClientAPIFactory(\n            nameserverAccessConfig,\n            \"ClusterMQClient_\",\n            proxyConfig.getRocketmqMQClientNum(),\n            new DoNothingClientRemotingProcessor(null),\n            rpcHook,\n            scheduledExecutorService,\n            remotingClientCreator\n        );\n\n        this.operationClientAPIFactory = new MQClientAPIFactory(\n            nameserverAccessConfig,\n            \"OperationClient_\",\n            1,\n            new DoNothingClientRemotingProcessor(null),\n            rpcHook,\n            this.scheduledExecutorService,\n            remotingClientCreator\n        );\n\n        this.topicRouteService = new ClusterTopicRouteService(operationClientAPIFactory);\n        this.messageService = new ClusterMessageService(this.topicRouteService, this.messagingClientAPIFactory);\n        this.metadataService = new ClusterMetadataService(topicRouteService, operationClientAPIFactory);\n        this.adminService = new DefaultAdminService(this.operationClientAPIFactory);\n\n        this.producerManager = new ProducerManager();\n        this.consumerManager = new ClusterConsumerManager(this.topicRouteService, this.adminService, this.operationClientAPIFactory, new ConsumerIdsChangeListenerImpl(), proxyConfig.getChannelExpiredTimeout(), rpcHook);\n\n        this.transactionClientAPIFactory = new MQClientAPIFactory(\n            nameserverAccessConfig,\n            \"ClusterTransaction_\",\n            1,\n            new ProxyClientRemotingProcessor(producerManager, consumerManager),\n            rpcHook,\n            scheduledExecutorService,\n            remotingClientCreator\n        );\n\n        this.clusterTransactionService = new ClusterTransactionService(this.topicRouteService, this.producerManager,\n            this.transactionClientAPIFactory);\n        this.proxyRelayService = new ClusterProxyRelayService(this.clusterTransactionService);\n\n        // Lite subscriptions use a separate channel\n        this.liteSubscriptionAPIFactory = new MQClientAPIFactory(\n            nameserverAccessConfig,\n            \"LiteSubscription_\",\n            1,\n            new ProxyClientRemotingProcessor(producerManager, consumerManager),\n            rpcHook,\n            scheduledExecutorService);\n        this.liteSubscriptionService = new LiteSubscriptionService(this.topicRouteService, this.liteSubscriptionAPIFactory);\n\n        this.init();\n    }\n\n    protected void init() {\n        this.producerManager.appendProducerChangeListener(new ProducerChangeListenerImpl());\n\n        this.scheduledExecutorService.scheduleAtFixedRate(() -> {\n            try {\n                producerManager.scanNotActiveChannel();\n                consumerManager.scanNotActiveChannel();\n            } catch (Throwable e) {\n                log.error(\"Error occurred when scan not active client channels.\", e);\n            }\n        }, 1000 * 10, 1000 * 10, TimeUnit.MILLISECONDS);\n\n        this.appendShutdown(scheduledExecutorService::shutdown);\n        this.appendStartAndShutdown(this.messagingClientAPIFactory);\n        this.appendStartAndShutdown(this.operationClientAPIFactory);\n        this.appendStartAndShutdown(this.transactionClientAPIFactory);\n        this.appendStartAndShutdown(this.liteSubscriptionAPIFactory);\n        this.appendStartAndShutdown(this.topicRouteService);\n        this.appendStartAndShutdown(this.clusterTransactionService);\n        this.appendStartAndShutdown(this.metadataService);\n        this.appendStartAndShutdown(this.consumerManager);\n    }\n\n    @Override\n    public MessageService getMessageService() {\n        return this.messageService;\n    }\n\n    @Override\n    public TopicRouteService getTopicRouteService() {\n        return topicRouteService;\n    }\n\n    @Override\n    public ProducerManager getProducerManager() {\n        return this.producerManager;\n    }\n\n    @Override\n    public ConsumerManager getConsumerManager() {\n        return this.consumerManager;\n    }\n\n    @Override\n    public TransactionService getTransactionService() {\n        return this.clusterTransactionService;\n    }\n\n    @Override\n    public ProxyRelayService getProxyRelayService() {\n        return this.proxyRelayService;\n    }\n\n    @Override\n    public MetadataService getMetadataService() {\n        return this.metadataService;\n    }\n\n    @Override\n    public AdminService getAdminService() {\n        return this.adminService;\n    }\n\n    @Override\n    public LiteSubscriptionService getLiteSubscriptionService() {\n        return liteSubscriptionService;\n    }\n\n    protected static class ConsumerIdsChangeListenerImpl implements ConsumerIdsChangeListener {\n\n        @Override\n        public void handle(ConsumerGroupEvent event, String group, Object... args) {\n\n        }\n\n        @Override\n        public void shutdown() {\n\n        }\n    }\n\n    protected class ProducerChangeListenerImpl implements ProducerChangeListener {\n        @Override\n        public void handle(ProducerGroupEvent event, String group, ClientChannelInfo clientChannelInfo) {\n            if (event == ProducerGroupEvent.GROUP_UNREGISTER) {\n                getTransactionService().unSubscribeAllTransactionTopic(ProxyContext.createForInner(this.getClass()), group);\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service;\n\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.client.ConsumerManager;\nimport org.apache.rocketmq.broker.client.ProducerManager;\nimport org.apache.rocketmq.client.common.NameserverAccessConfig;\nimport org.apache.rocketmq.client.impl.mqclient.DoNothingClientRemotingProcessor;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.utils.AbstractStartAndShutdown;\nimport org.apache.rocketmq.common.utils.StartAndShutdown;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.service.admin.AdminService;\nimport org.apache.rocketmq.proxy.service.admin.DefaultAdminService;\nimport org.apache.rocketmq.proxy.service.channel.ChannelManager;\nimport org.apache.rocketmq.proxy.service.lite.LiteSubscriptionService;\nimport org.apache.rocketmq.proxy.service.message.LocalMessageService;\nimport org.apache.rocketmq.proxy.service.message.MessageService;\nimport org.apache.rocketmq.proxy.service.metadata.LocalMetadataService;\nimport org.apache.rocketmq.proxy.service.metadata.MetadataService;\nimport org.apache.rocketmq.proxy.service.relay.LocalProxyRelayService;\nimport org.apache.rocketmq.proxy.service.relay.ProxyRelayService;\nimport org.apache.rocketmq.proxy.service.route.LocalTopicRouteService;\nimport org.apache.rocketmq.proxy.service.route.TopicRouteService;\nimport org.apache.rocketmq.proxy.service.transaction.LocalTransactionService;\nimport org.apache.rocketmq.proxy.service.transaction.TransactionService;\nimport org.apache.rocketmq.remoting.RPCHook;\n\npublic class LocalServiceManager extends AbstractStartAndShutdown implements ServiceManager {\n\n    private final BrokerController brokerController;\n    private final TopicRouteService topicRouteService;\n    private final MessageService messageService;\n    private final TransactionService transactionService;\n    private final ProxyRelayService proxyRelayService;\n    private final MetadataService metadataService;\n    private final AdminService adminService;\n\n    private final MQClientAPIFactory mqClientAPIFactory;\n    private final ChannelManager channelManager;\n\n    private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor(\n        new ThreadFactoryImpl(\"LocalServiceManagerScheduledThread\"));\n\n    public LocalServiceManager(BrokerController brokerController, RPCHook rpcHook) {\n        this.brokerController = brokerController;\n        this.channelManager = new ChannelManager();\n        this.messageService = new LocalMessageService(brokerController, channelManager, rpcHook);\n        ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();\n        NameserverAccessConfig nameserverAccessConfig = new NameserverAccessConfig(proxyConfig.getNamesrvAddr(),\n            proxyConfig.getNamesrvDomain(), proxyConfig.getNamesrvDomainSubgroup());\n        this.mqClientAPIFactory = new MQClientAPIFactory(\n            nameserverAccessConfig,\n            \"LocalMQClient_\",\n            1,\n            new DoNothingClientRemotingProcessor(null),\n            rpcHook,\n            scheduledExecutorService\n        );\n        this.topicRouteService = new LocalTopicRouteService(brokerController, mqClientAPIFactory);\n        this.transactionService = new LocalTransactionService(brokerController.getBrokerConfig());\n        this.proxyRelayService = new LocalProxyRelayService(brokerController, this.transactionService);\n        this.metadataService = new LocalMetadataService(brokerController);\n        this.adminService = new DefaultAdminService(this.mqClientAPIFactory);\n        this.init();\n    }\n\n    protected void init() {\n        this.appendStartAndShutdown(this.mqClientAPIFactory);\n        this.appendStartAndShutdown(this.topicRouteService);\n        this.appendStartAndShutdown(new LocalServiceManagerStartAndShutdown());\n    }\n\n    @Override\n    public MessageService getMessageService() {\n        return this.messageService;\n    }\n\n    @Override\n    public TopicRouteService getTopicRouteService() {\n        return this.topicRouteService;\n    }\n\n    @Override\n    public ProducerManager getProducerManager() {\n        return this.brokerController.getProducerManager();\n    }\n\n    @Override\n    public ConsumerManager getConsumerManager() {\n        return this.brokerController.getConsumerManager();\n    }\n\n    @Override\n    public TransactionService getTransactionService() {\n        return this.transactionService;\n    }\n\n    @Override\n    public ProxyRelayService getProxyRelayService() {\n        return this.proxyRelayService;\n    }\n\n    @Override\n    public MetadataService getMetadataService() {\n        return this.metadataService;\n    }\n\n    @Override\n    public AdminService getAdminService() {\n        return this.adminService;\n    }\n\n    @Override\n    public LiteSubscriptionService getLiteSubscriptionService() {\n        return null;\n    }\n\n    private class LocalServiceManagerStartAndShutdown implements StartAndShutdown {\n        @Override\n        public void start() throws Exception {\n            LocalServiceManager.this.scheduledExecutorService.scheduleWithFixedDelay(channelManager::scanAndCleanChannels, 5, 5, TimeUnit.MINUTES);\n        }\n\n        @Override\n        public void shutdown() throws Exception {\n            LocalServiceManager.this.scheduledExecutorService.shutdown();\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/ServiceManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service;\n\nimport org.apache.rocketmq.broker.client.ConsumerManager;\nimport org.apache.rocketmq.broker.client.ProducerManager;\nimport org.apache.rocketmq.common.utils.StartAndShutdown;\nimport org.apache.rocketmq.proxy.service.admin.AdminService;\nimport org.apache.rocketmq.proxy.service.lite.LiteSubscriptionService;\nimport org.apache.rocketmq.proxy.service.message.MessageService;\nimport org.apache.rocketmq.proxy.service.metadata.MetadataService;\nimport org.apache.rocketmq.proxy.service.relay.ProxyRelayService;\nimport org.apache.rocketmq.proxy.service.route.TopicRouteService;\nimport org.apache.rocketmq.proxy.service.transaction.TransactionService;\n\npublic interface ServiceManager extends StartAndShutdown {\n    MessageService getMessageService();\n\n    TopicRouteService getTopicRouteService();\n\n    ProducerManager getProducerManager();\n\n    ConsumerManager getConsumerManager();\n\n    TransactionService getTransactionService();\n\n    ProxyRelayService getProxyRelayService();\n\n    MetadataService getMetadataService();\n\n    AdminService getAdminService();\n\n    LiteSubscriptionService getLiteSubscriptionService();\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/ServiceManagerFactory.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.ObjectCreator;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.RemotingClient;\n\npublic class ServiceManagerFactory {\n    public static ServiceManager createForLocalMode(BrokerController brokerController) {\n        return createForLocalMode(brokerController, null);\n    }\n\n    public static ServiceManager createForLocalMode(BrokerController brokerController, RPCHook rpcHook) {\n        return new LocalServiceManager(brokerController, rpcHook);\n    }\n\n    public static ServiceManager createForClusterMode() {\n        return createForClusterMode(null, null);\n    }\n\n    public static ServiceManager createForClusterMode(RPCHook rpcHook) {\n        return createForClusterMode(rpcHook, null);\n    }\n\n    public static ServiceManager createForClusterMode(RPCHook rpcHook, ObjectCreator<RemotingClient> remotingClientCreator) {\n        return new ClusterServiceManager(rpcHook, remotingClientCreator);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/AdminService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.admin;\n\nimport java.util.List;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\n\npublic interface AdminService {\n\n    boolean topicExist(String topic);\n\n    boolean createTopicOnTopicBrokerIfNotExist(String createTopic, String sampleTopic, int wQueueNum,\n        int rQueueNum, boolean examineTopic, int retryCheckCount);\n\n    boolean createTopicOnBroker(String topic, int wQueueNum, int rQueueNum, List<BrokerData> curBrokerDataList,\n        List<BrokerData> sampleBrokerDataList, boolean examineTopic, int retryCheckCount) throws Exception;\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.admin;\n\nimport java.time.Duration;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIExt;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;\nimport org.apache.rocketmq.proxy.service.route.TopicRouteHelper;\n\npublic class DefaultAdminService implements AdminService {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    private final MQClientAPIFactory mqClientAPIFactory;\n\n    public DefaultAdminService(MQClientAPIFactory mqClientAPIFactory) {\n        this.mqClientAPIFactory = mqClientAPIFactory;\n    }\n\n    @Override\n    public boolean topicExist(String topic) {\n        boolean topicExist;\n        TopicRouteData topicRouteData;\n        try {\n            topicRouteData = this.getTopicRouteDataDirectlyFromNameServer(topic);\n            topicExist = topicRouteData != null;\n        } catch (Throwable e) {\n            topicExist = false;\n        }\n\n        return topicExist;\n    }\n\n    @Override\n    public boolean createTopicOnTopicBrokerIfNotExist(String createTopic, String sampleTopic, int wQueueNum,\n        int rQueueNum, boolean examineTopic, int retryCheckCount) {\n        TopicRouteData curTopicRouteData = new TopicRouteData();\n        try {\n            curTopicRouteData = this.getTopicRouteDataDirectlyFromNameServer(createTopic);\n        } catch (Exception e) {\n            if (!TopicRouteHelper.isTopicNotExistError(e)) {\n                log.error(\"get cur topic route {} failed.\", createTopic, e);\n                return false;\n            }\n        }\n\n        TopicRouteData sampleTopicRouteData = null;\n        try {\n            sampleTopicRouteData = this.getTopicRouteDataDirectlyFromNameServer(sampleTopic);\n        } catch (Exception e) {\n            log.error(\"create topic {} failed.\", createTopic, e);\n            return false;\n        }\n\n        if (sampleTopicRouteData == null || sampleTopicRouteData.getBrokerDatas().isEmpty()) {\n            return false;\n        }\n\n        try {\n            return this.createTopicOnBroker(createTopic, wQueueNum, rQueueNum, curTopicRouteData.getBrokerDatas(),\n                sampleTopicRouteData.getBrokerDatas(), examineTopic, retryCheckCount);\n        } catch (Exception e) {\n            log.error(\"create topic {} failed.\", createTopic, e);\n        }\n        return false;\n    }\n\n    @Override\n    public boolean createTopicOnBroker(String topic, int wQueueNum, int rQueueNum, List<BrokerData> curBrokerDataList,\n        List<BrokerData> sampleBrokerDataList, boolean examineTopic, int retryCheckCount) throws Exception {\n        Set<String> curBrokerAddr = new HashSet<>();\n        if (curBrokerDataList != null) {\n            for (BrokerData brokerData : curBrokerDataList) {\n                curBrokerAddr.add(brokerData.getBrokerAddrs().get(MixAll.MASTER_ID));\n            }\n        }\n\n        TopicConfig topicConfig = new TopicConfig();\n        topicConfig.setTopicName(topic);\n        topicConfig.setWriteQueueNums(wQueueNum);\n        topicConfig.setReadQueueNums(rQueueNum);\n        topicConfig.setPerm(PermName.PERM_READ | PermName.PERM_WRITE);\n\n        for (BrokerData brokerData : sampleBrokerDataList) {\n            String addr = brokerData.getBrokerAddrs() == null ? null : brokerData.getBrokerAddrs().get(MixAll.MASTER_ID);\n            if (addr == null) {\n                continue;\n            }\n            if (curBrokerAddr.contains(addr)) {\n                continue;\n            }\n\n            try {\n                this.getClient().createTopic(addr, TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC, topicConfig, Duration.ofSeconds(3).toMillis());\n            } catch (Exception e) {\n                log.error(\"create topic on broker failed. topic:{}, broker:{}\", topicConfig, addr, e);\n            }\n        }\n\n        if (examineTopic) {\n            // examine topic exist.\n            int count = retryCheckCount;\n            while (count-- > 0) {\n                if (this.topicExist(topic)) {\n                    return true;\n                }\n            }\n        } else {\n            return true;\n        }\n        return false;\n    }\n\n    protected TopicRouteData getTopicRouteDataDirectlyFromNameServer(String topic) throws Exception {\n        return this.getClient().getTopicRouteInfoFromNameServer(topic, Duration.ofSeconds(3).toMillis());\n    }\n\n    protected MQClientAPIExt getClient() {\n        return this.mqClientAPIFactory.getClient();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/cert/TlsCertificateManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.cert;\n\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.StartAndShutdown;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.remoting.netty.TlsSystemConfig;\nimport org.apache.rocketmq.srvutil.FileWatchService;\nimport java.util.ArrayList;\nimport java.util.List;\n\npublic class TlsCertificateManager implements StartAndShutdown {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    private final FileWatchService fileWatchService;\n    private final List<TlsContextReloadListener> reloadListeners = new ArrayList<>();\n\n    public TlsCertificateManager() {\n        try {\n            this.fileWatchService = new FileWatchService(\n                new String[] {\n                    ConfigurationManager.getProxyConfig().getTlsCertPath(),\n                    ConfigurationManager.getProxyConfig().getTlsKeyPath()\n                },\n                new CertKeyFileWatchListener(),\n                ConfigurationManager.getProxyConfig().getTlsCertWatchIntervalMs()\n            );\n        } catch (Exception e) {\n            log.error(\"Failed to initialize TLS certificate watch service\", e);\n            throw new RuntimeException(\"Failed to initialize TLS certificate manager\", e);\n        }\n    }\n\n    public FileWatchService getFileWatchService() {\n        return this.fileWatchService;\n    }\n\n    public void registerReloadListener(TlsContextReloadListener listener) {\n        if (listener != null) {\n            this.reloadListeners.add(listener);\n        }\n    }\n\n    public void unregisterReloadListener(TlsContextReloadListener listener) {\n        if (listener != null) {\n            this.reloadListeners.remove(listener);\n        }\n    }\n\n    public List<TlsContextReloadListener> getReloadListeners() {\n        return this.reloadListeners;\n    }\n\n    @Override\n    public void start() throws Exception {\n        this.fileWatchService.start();\n        log.info(\"TLS certificate manager started successfully, start watching: {} {}\",\n            ConfigurationManager.getProxyConfig().getTlsCertPath(),\n            ConfigurationManager.getProxyConfig().getTlsKeyPath()\n        );\n    }\n\n    @Override\n    public void shutdown() throws Exception {\n        this.fileWatchService.shutdown();\n        log.info(\"TLS certificate manager shutdown successfully\");\n    }\n\n    private class CertKeyFileWatchListener implements FileWatchService.Listener {\n        private boolean certChanged = false;\n        private boolean keyChanged = false;\n\n        @Override\n        public void onChanged(String path) {\n            log.info(\"File changed: {}\", path);\n            if (path.equals(TlsSystemConfig.tlsServerCertPath)) {\n                certChanged = true;\n            } else if (path.equals(TlsSystemConfig.tlsServerKeyPath)) {\n                keyChanged = true;\n            }\n\n            if (certChanged && keyChanged) {\n                log.info(\"The certificate and private key changed, reload the ssl context\");\n                notifyContextReload();\n                certChanged = false;\n                keyChanged = false;\n            }\n        }\n\n        private void notifyContextReload() {\n            for (TlsContextReloadListener listener : reloadListeners) {\n                try {\n                    listener.onTlsContextReload();\n                } catch (Throwable e) {\n                    log.error(\"Failed to notify TLS context reload to listener: \" + listener, e);\n                }\n            }\n        }\n    }\n\n    // Interface for listeners interested in TLS context reload events\n    public interface TlsContextReloadListener {\n        void onTlsContextReload();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/ChannelManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.channel;\n\nimport com.google.common.base.Strings;\nimport java.util.Iterator;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.ConcurrentHashMapUtils;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class ChannelManager {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    private final ConcurrentMap<String, SimpleChannel> clientIdChannelMap = new ConcurrentHashMap<>();\n\n    public SimpleChannel createChannel(ProxyContext context) {\n        final String clientId = anonymousChannelId(context);\n        if (Strings.isNullOrEmpty(clientId)) {\n            log.warn(\"ClientId is unexpected null or empty\");\n            return createChannelInner(context);\n        }\n        SimpleChannel channel = ConcurrentHashMapUtils.computeIfAbsent(this.clientIdChannelMap,clientId, k -> createChannelInner(context));\n        channel.updateLastAccessTime();\n        return channel;\n    }\n\n    public SimpleChannel createInvocationChannel(ProxyContext context) {\n        final String clientId = anonymousChannelId(InvocationChannel.class.getName(), context);\n        final String clientHost = context.getRemoteAddress();\n        final String localAddress = context.getLocalAddress();\n        if (Strings.isNullOrEmpty(clientId)) {\n            log.warn(\"ClientId is unexpected null or empty\");\n            return new InvocationChannel(clientHost, localAddress);\n        }\n\n        SimpleChannel channel = clientIdChannelMap.computeIfAbsent(clientId, k -> new InvocationChannel(clientHost, localAddress));\n        channel.updateLastAccessTime();\n        return channel;\n    }\n\n    private String anonymousChannelId(ProxyContext context) {\n        final String clientHost = context.getRemoteAddress();\n        final String localAddress = context.getLocalAddress();\n        return clientHost + \"@\" + localAddress;\n    }\n\n    private String anonymousChannelId(String key, ProxyContext context) {\n        final String clientHost = context.getRemoteAddress();\n        final String localAddress = context.getLocalAddress();\n        return key + \"@\" + clientHost + \"@\" + localAddress;\n    }\n\n    private SimpleChannel createChannelInner(ProxyContext context) {\n        return new SimpleChannel(context.getRemoteAddress(), context.getLocalAddress());\n    }\n\n    public void scanAndCleanChannels() {\n        try {\n            Iterator<Map.Entry<String, SimpleChannel>> iterator = clientIdChannelMap.entrySet().iterator();\n            while (iterator.hasNext()) {\n                Map.Entry<String, SimpleChannel> entry = iterator.next();\n                if (!entry.getValue().isActive()) {\n                    iterator.remove();\n                } else {\n                    entry.getValue().clearExpireContext();\n                }\n            }\n        } catch (Throwable e) {\n            log.error(\"Unexpected exception\", e);\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/InvocationChannel.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.channel;\n\nimport io.netty.channel.ChannelFuture;\nimport java.util.Iterator;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic class InvocationChannel extends SimpleChannel {\n    protected final ConcurrentMap<Integer, InvocationContextInterface> inFlightRequestMap;\n\n    public InvocationChannel(String remoteAddress, String localAddress) {\n        super(remoteAddress, localAddress);\n        this.inFlightRequestMap = new ConcurrentHashMap<>();\n    }\n\n    @Override\n    public ChannelFuture writeAndFlush(Object msg) {\n        if (msg instanceof RemotingCommand) {\n            RemotingCommand responseCommand = (RemotingCommand) msg;\n            InvocationContextInterface context = inFlightRequestMap.remove(responseCommand.getOpaque());\n            if (null != context) {\n                context.handle(responseCommand);\n            }\n            inFlightRequestMap.remove(responseCommand.getOpaque());\n        }\n        return super.writeAndFlush(msg);\n    }\n\n    @Override\n    public boolean isWritable() {\n        return inFlightRequestMap.size() > 0;\n    }\n\n    @Override\n    public void registerInvocationContext(int opaque, InvocationContextInterface context) {\n        inFlightRequestMap.put(opaque, context);\n    }\n\n    @Override\n    public void eraseInvocationContext(int opaque) {\n        inFlightRequestMap.remove(opaque);\n    }\n\n    @Override\n    public void clearExpireContext() {\n        Iterator<Map.Entry<Integer, InvocationContextInterface>> iterator = inFlightRequestMap.entrySet().iterator();\n        int count = 0;\n        while (iterator.hasNext()) {\n            Map.Entry<Integer, InvocationContextInterface> entry = iterator.next();\n            if (entry.getValue().expired(ConfigurationManager.getProxyConfig().getChannelExpiredInSeconds())) {\n                iterator.remove();\n                count++;\n                log.debug(\"An expired request is found, request: {}\", entry.getValue());\n            }\n        }\n        if (count > 0) {\n            log.warn(\"[BUG] {} expired in-flight requests is cleaned.\", count);\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/InvocationContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.channel;\n\nimport java.time.Duration;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic class InvocationContext implements InvocationContextInterface {\n    private final CompletableFuture<RemotingCommand> response;\n    private final long timestamp = System.currentTimeMillis();\n\n    public InvocationContext(CompletableFuture<RemotingCommand> resp) {\n        this.response = resp;\n    }\n\n    public boolean expired(long expiredTimeSec) {\n        return System.currentTimeMillis() - timestamp >= Duration.ofSeconds(expiredTimeSec).toMillis();\n    }\n\n    public CompletableFuture<RemotingCommand> getResponse() {\n        return response;\n    }\n\n    public void handle(RemotingCommand remotingCommand) {\n        response.complete(remotingCommand);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/InvocationContextInterface.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.channel;\n\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic interface InvocationContextInterface {\n    void handle(RemotingCommand remotingCommand);\n\n    boolean expired(long expiredTimeSec);\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/SimpleChannel.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.channel;\n\nimport com.google.common.base.Strings;\nimport io.netty.channel.AbstractChannel;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelConfig;\nimport io.netty.channel.ChannelFuture;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.channel.ChannelId;\nimport io.netty.channel.ChannelMetadata;\nimport io.netty.channel.ChannelOutboundBuffer;\nimport io.netty.channel.DefaultChannelPromise;\nimport io.netty.channel.EventLoop;\nimport io.netty.util.concurrent.GlobalEventExecutor;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\n/**\n * SimpleChannel is used to handle writeAndFlush situation in processor\n *\n * @see io.netty.channel.ChannelHandlerContext#writeAndFlush\n * @see io.netty.channel.Channel#writeAndFlush\n */\npublic class SimpleChannel extends AbstractChannel {\n    protected static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    protected final String remoteAddress;\n    protected final String localAddress;\n\n    protected long lastAccessTime;\n    protected ChannelHandlerContext channelHandlerContext;\n\n    /**\n     * Creates a new instance.\n     *\n     * @param parent        the parent of this channel. {@code null} if there's no parent.\n     * @param remoteAddress Remote address\n     * @param localAddress  Local address\n     */\n    public SimpleChannel(Channel parent, String remoteAddress, String localAddress) {\n        this(parent, null, remoteAddress, localAddress);\n    }\n\n    public SimpleChannel(Channel parent, ChannelId id, String remoteAddress, String localAddress) {\n        super(parent, id);\n        lastAccessTime = System.currentTimeMillis();\n        this.remoteAddress = remoteAddress;\n        this.localAddress = localAddress;\n        this.channelHandlerContext = new SimpleChannelHandlerContext(this);\n    }\n\n    public SimpleChannel(String remoteAddress, String localAddress) {\n        this(null, remoteAddress, localAddress);\n    }\n\n    @Override\n    protected AbstractUnsafe newUnsafe() {\n        return null;\n    }\n\n    @Override\n    protected boolean isCompatible(EventLoop loop) {\n        return false;\n    }\n\n    private static SocketAddress parseSocketAddress(String address) {\n        if (Strings.isNullOrEmpty(address)) {\n            return null;\n        }\n\n        String[] segments = address.split(\":\");\n        if (2 == segments.length) {\n            return new InetSocketAddress(segments[0], Integer.parseInt(segments[1]));\n        }\n\n        return null;\n    }\n\n    @Override\n    protected SocketAddress localAddress0() {\n        return parseSocketAddress(localAddress);\n    }\n\n    @Override\n    public SocketAddress localAddress() {\n        return localAddress0();\n    }\n\n    @Override\n    public SocketAddress remoteAddress() {\n        return remoteAddress0();\n    }\n\n    @Override\n    protected SocketAddress remoteAddress0() {\n        return parseSocketAddress(remoteAddress);\n    }\n\n    @Override\n    protected void doBind(SocketAddress localAddress) throws Exception {\n\n    }\n\n    @Override\n    protected void doDisconnect() throws Exception {\n\n    }\n\n    @Override\n    public ChannelFuture close() {\n        DefaultChannelPromise promise = new DefaultChannelPromise(this, GlobalEventExecutor.INSTANCE);\n        promise.setSuccess();\n        return promise;\n    }\n\n    @Override\n    protected void doClose() throws Exception {\n\n    }\n\n    @Override\n    protected void doBeginRead() throws Exception {\n\n    }\n\n    @Override\n    protected void doWrite(ChannelOutboundBuffer in) throws Exception {\n\n    }\n\n    @Override\n    public ChannelConfig config() {\n        return null;\n    }\n\n    @Override\n    public boolean isOpen() {\n        return true;\n    }\n\n    @Override\n    public boolean isActive() {\n        return (System.currentTimeMillis() - lastAccessTime) <= 120L * 1000;\n    }\n\n    @Override\n    public ChannelMetadata metadata() {\n        return null;\n    }\n\n    @Override\n    public ChannelFuture writeAndFlush(Object msg) {\n        DefaultChannelPromise promise = new DefaultChannelPromise(this, GlobalEventExecutor.INSTANCE);\n        promise.setSuccess();\n        return promise;\n    }\n\n    @Override\n    public boolean isWritable() {\n        return true;\n    }\n\n    public void updateLastAccessTime() {\n        this.lastAccessTime = System.currentTimeMillis();\n    }\n\n    public void registerInvocationContext(int opaque, InvocationContextInterface context) {\n\n    }\n\n    public void eraseInvocationContext(int opaque) {\n\n    }\n\n    public void clearExpireContext() {\n\n    }\n\n    public String getRemoteAddress() {\n        return remoteAddress;\n    }\n\n    public String getLocalAddress() {\n        return localAddress;\n    }\n\n    public ChannelHandlerContext getChannelHandlerContext() {\n        return channelHandlerContext;\n    }\n}"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/SimpleChannelHandlerContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.channel;\n\nimport io.netty.buffer.ByteBufAllocator;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelFuture;\nimport io.netty.channel.ChannelHandler;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.channel.ChannelPipeline;\nimport io.netty.channel.ChannelProgressivePromise;\nimport io.netty.channel.ChannelPromise;\nimport io.netty.util.Attribute;\nimport io.netty.util.AttributeKey;\nimport io.netty.util.concurrent.EventExecutor;\nimport java.net.SocketAddress;\nimport org.apache.commons.lang3.NotImplementedException;\n\npublic class SimpleChannelHandlerContext implements ChannelHandlerContext {\n\n    private final Channel channel;\n\n    public SimpleChannelHandlerContext(Channel channel) {\n        this.channel = channel;\n    }\n\n    @Override\n    public Channel channel() {\n        return channel;\n    }\n\n    @Override\n    public EventExecutor executor() {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public String name() {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelHandler handler() {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public boolean isRemoved() {\n        return false;\n    }\n\n    @Override\n    public ChannelHandlerContext fireChannelRegistered() {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelHandlerContext fireChannelUnregistered() {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelHandlerContext fireChannelActive() {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelHandlerContext fireChannelInactive() {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelHandlerContext fireExceptionCaught(Throwable cause) {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelHandlerContext fireUserEventTriggered(Object evt) {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelHandlerContext fireChannelRead(Object msg) {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelHandlerContext fireChannelReadComplete() {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelHandlerContext fireChannelWritabilityChanged() {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelFuture bind(SocketAddress localAddress) {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelFuture connect(SocketAddress remoteAddress) {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelFuture connect(SocketAddress remoteAddress, SocketAddress localAddress) {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelFuture disconnect() {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelFuture close() {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelFuture deregister() {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelFuture bind(SocketAddress localAddress, ChannelPromise promise) {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelFuture connect(SocketAddress remoteAddress, ChannelPromise promise) {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelFuture connect(SocketAddress remoteAddress, SocketAddress localAddress, ChannelPromise promise) {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelFuture disconnect(ChannelPromise promise) {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelFuture close(ChannelPromise promise) {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelFuture deregister(ChannelPromise promise) {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelHandlerContext read() {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelFuture write(Object msg) {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelFuture write(Object msg, ChannelPromise promise) {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelHandlerContext flush() {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelFuture writeAndFlush(Object msg, ChannelPromise promise) {\n        return channel.writeAndFlush(msg, promise);\n    }\n\n    @Override\n    public ChannelFuture writeAndFlush(Object msg) {\n        return channel.writeAndFlush(msg);\n    }\n\n    @Override\n    public ChannelPipeline pipeline() {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ByteBufAllocator alloc() {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelPromise newPromise() {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelProgressivePromise newProgressivePromise() {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelFuture newSucceededFuture() {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelFuture newFailedFuture(Throwable cause) {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public ChannelPromise voidPromise() {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public <T> Attribute<T> attr(AttributeKey<T> key) {\n        throw new NotImplementedException(\"Not implemented\");\n    }\n\n    @Override\n    public <T> boolean hasAttr(AttributeKey<T> attributeKey) {\n        return false;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/client/ClusterConsumerManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.client;\n\nimport java.util.Set;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;\nimport org.apache.rocketmq.broker.client.ConsumerManager;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.utils.StartAndShutdown;\nimport org.apache.rocketmq.proxy.service.admin.AdminService;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;\nimport org.apache.rocketmq.proxy.service.route.TopicRouteService;\nimport org.apache.rocketmq.proxy.service.sysmessage.HeartbeatSyncer;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\npublic class ClusterConsumerManager extends ConsumerManager implements StartAndShutdown {\n\n    protected HeartbeatSyncer heartbeatSyncer;\n\n    public ClusterConsumerManager(TopicRouteService topicRouteService, AdminService adminService,\n                                  MQClientAPIFactory mqClientAPIFactory, ConsumerIdsChangeListener consumerIdsChangeListener, long channelExpiredTimeout, RPCHook rpcHook) {\n        super(consumerIdsChangeListener, channelExpiredTimeout);\n        this.heartbeatSyncer = new HeartbeatSyncer(topicRouteService, adminService, this, mqClientAPIFactory, rpcHook);\n    }\n\n    @Override\n    public boolean registerConsumer(String group, ClientChannelInfo clientChannelInfo,\n        ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,\n        Set<SubscriptionData> subList, boolean isNotifyConsumerIdsChangedEnable, boolean updateSubscription) {\n        this.heartbeatSyncer.onConsumerRegister(group, clientChannelInfo, consumeType, messageModel, consumeFromWhere, subList);\n        return super.registerConsumer(group, clientChannelInfo, consumeType, messageModel, consumeFromWhere, subList,\n            isNotifyConsumerIdsChangedEnable, updateSubscription);\n    }\n\n    @Override\n    public void unregisterConsumer(String group, ClientChannelInfo clientChannelInfo,\n        boolean isNotifyConsumerIdsChangedEnable) {\n        this.heartbeatSyncer.onConsumerUnRegister(group, clientChannelInfo);\n        super.unregisterConsumer(group, clientChannelInfo, isNotifyConsumerIdsChangedEnable);\n    }\n\n    @Override\n    public void shutdown() throws Exception {\n        this.heartbeatSyncer.shutdown();\n    }\n\n    @Override\n    public void start() throws Exception {\n        this.heartbeatSyncer.start();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/client/ProxyClientRemotingProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.client;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport java.nio.ByteBuffer;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ProducerManager;\nimport org.apache.rocketmq.client.impl.ClientRemotingProcessor;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.utils.ProxyUtils;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.NotifyUnsubscribeLiteRequestHeader;\n\npublic class ProxyClientRemotingProcessor extends ClientRemotingProcessor {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    private final ProducerManager producerManager;\n    private final ClusterConsumerManager consumerManager;\n\n    public ProxyClientRemotingProcessor(ProducerManager producerManager, ClusterConsumerManager consumerManager) {\n        super(null);\n        this.producerManager = producerManager;\n        this.consumerManager = consumerManager;\n    }\n\n    @Override\n    public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request)\n        throws RemotingCommandException {\n        if (request.getCode() == RequestCode.CHECK_TRANSACTION_STATE) {\n            return this.checkTransactionState(ctx, request);\n        } else if (request.getCode() == RequestCode.NOTIFY_UNSUBSCRIBE_LITE) {\n            return this.notifyUnsubscribeLite(ctx, request);\n        }\n        return null;\n    }\n\n    @Override\n    public RemotingCommand checkTransactionState(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        final ByteBuffer byteBuffer = ByteBuffer.wrap(request.getBody());\n        final MessageExt messageExt = MessageDecoder.decode(byteBuffer, true, false, false);\n        if (messageExt != null) {\n            final String group = messageExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP);\n            if (group != null) {\n                CheckTransactionStateRequestHeader requestHeader =\n                    (CheckTransactionStateRequestHeader) request.decodeCommandCustomHeader(CheckTransactionStateRequestHeader.class);\n                request.writeCustomHeader(requestHeader);\n                request.addExtField(ProxyUtils.BROKER_ADDR, NetworkUtil.socketAddress2String(ctx.channel().remoteAddress()));\n                Channel channel = this.producerManager.getAvailableChannel(group);\n                if (channel != null) {\n                    channel.writeAndFlush(request);\n                } else {\n                    log.warn(\"check transaction failed, channel is empty. groupId={}, requestHeader:{}\", group, requestHeader);\n                }\n            }\n        }\n        return null;\n    }\n\n    /**\n     * one way, return null response\n     */\n    public RemotingCommand notifyUnsubscribeLite(ChannelHandlerContext ctx,\n        RemotingCommand request) throws RemotingCommandException {\n        NotifyUnsubscribeLiteRequestHeader requestHeader =\n            request.decodeCommandCustomHeader(NotifyUnsubscribeLiteRequestHeader.class);\n        request.writeCustomHeader(requestHeader);\n        final String clientId = requestHeader.getClientId();\n        final String group = requestHeader.getConsumerGroup();\n        if (StringUtils.isBlank(clientId) || StringUtils.isBlank(group)) {\n            log.warn(\"notifyUnsubscribeLite clientId or group is null. {}\", requestHeader);\n            return null;\n        }\n        ClientChannelInfo channelInfo = consumerManager.findChannel(group, clientId);\n        if (channelInfo == null) {\n            log.warn(\"notifyUnsubscribeLite channelInfo is null. {}\", requestHeader);\n            return null;\n        }\n        Channel channel = channelInfo.getChannel();\n        if (channel == null) {\n            log.warn(\"notifyUnsubscribeLite channel is null. {}\", requestHeader);\n            return null;\n        }\n        channel.writeAndFlush(request);\n        return null;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/lite/LiteSubscriptionService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.lite;\n\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.lite.LiteSubscriptionDTO;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;\nimport org.apache.rocketmq.proxy.service.route.MessageQueueView;\nimport org.apache.rocketmq.proxy.service.route.TopicRouteService;\n\npublic class LiteSubscriptionService {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    protected final TopicRouteService topicRouteService;\n    protected final MQClientAPIFactory mqClientAPIFactory;\n\n    public LiteSubscriptionService(TopicRouteService topicRouteService, MQClientAPIFactory mqClientAPIFactory) {\n        this.topicRouteService = topicRouteService;\n        this.mqClientAPIFactory = mqClientAPIFactory;\n    }\n\n    public CompletableFuture<Void> syncLiteSubscription(ProxyContext ctx,\n        LiteSubscriptionDTO liteSubscriptionDTO, long timeoutMillis) {\n        final String topic = liteSubscriptionDTO.getTopic();\n        List<AddressableMessageQueue> readQueues;\n        try {\n            MessageQueueView messageQueueView = topicRouteService.getAllMessageQueueView(ctx, topic);\n            // Send subscriptions to all readable brokers.\n            readQueues = messageQueueView.getReadSelector().getBrokerActingQueues();\n        } catch (Exception e) {\n            CompletableFuture<Void> future = new CompletableFuture<>();\n            future.completeExceptionally(e);\n            return future;\n        }\n\n        return CompletableFuture.allOf(\n            readQueues\n                .stream()\n                .map(writeQ ->\n                    mqClientAPIFactory.getClient().syncLiteSubscriptionAsync(\n                        writeQ.getBrokerAddr(),\n                        liteSubscriptionDTO,\n                        timeoutMillis\n                    ))\n                .toArray(CompletableFuture[]::new)\n        );\n    }\n\n}"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.message;\n\nimport com.google.common.collect.Lists;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.stream.Collectors;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.utils.FutureUtils;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\nimport org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;\nimport org.apache.rocketmq.proxy.service.route.TopicRouteService;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMinOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopLiteMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;\n\npublic class ClusterMessageService implements MessageService {\n    protected final TopicRouteService topicRouteService;\n    protected final MQClientAPIFactory mqClientAPIFactory;\n\n    public ClusterMessageService(TopicRouteService topicRouteService, MQClientAPIFactory mqClientAPIFactory) {\n        this.topicRouteService = topicRouteService;\n        this.mqClientAPIFactory = mqClientAPIFactory;\n    }\n\n    @Override\n    public CompletableFuture<List<SendResult>> sendMessage(ProxyContext ctx, AddressableMessageQueue messageQueue,\n        List<Message> msgList, SendMessageRequestHeader requestHeader, long timeoutMillis) {\n        CompletableFuture<List<SendResult>> future;\n        if (msgList.size() == 1) {\n            future = this.mqClientAPIFactory.getClient().sendMessageAsync(\n                    messageQueue.getBrokerAddr(),\n                    messageQueue.getBrokerName(), msgList.get(0), requestHeader, timeoutMillis)\n                .thenApply(Lists::newArrayList);\n        } else {\n            future = this.mqClientAPIFactory.getClient().sendMessageAsync(\n                    messageQueue.getBrokerAddr(),\n                    messageQueue.getBrokerName(), msgList, requestHeader, timeoutMillis)\n                .thenApply(Lists::newArrayList);\n        }\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> sendMessageBack(ProxyContext ctx, ReceiptHandle handle, String messageId,\n        ConsumerSendMsgBackRequestHeader requestHeader, long timeoutMillis) {\n        return this.mqClientAPIFactory.getClient().sendMessageBackAsync(\n            this.resolveBrokerAddrInReceiptHandle(ctx, handle),\n            requestHeader,\n            timeoutMillis\n        );\n    }\n\n    @Override\n    public CompletableFuture<Void> endTransactionOneway(ProxyContext ctx, String brokerName,\n        EndTransactionRequestHeader requestHeader,\n        long timeoutMillis) {\n        CompletableFuture<Void> future = new CompletableFuture<>();\n        try {\n            this.mqClientAPIFactory.getClient().endTransactionOneway(\n                this.resolveBrokerAddr(ctx, brokerName),\n                requestHeader,\n                \"end transaction from proxy\",\n                timeoutMillis\n            );\n            future.complete(null);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<PopResult> popMessage(ProxyContext ctx, AddressableMessageQueue messageQueue,\n        PopMessageRequestHeader requestHeader, long timeoutMillis) {\n        return this.mqClientAPIFactory.getClient().popMessageAsync(\n            messageQueue.getBrokerAddr(),\n            messageQueue.getBrokerName(),\n            requestHeader,\n            timeoutMillis\n        );\n    }\n\n    @Override\n    public CompletableFuture<PopResult> popLiteMessage(\n        ProxyContext ctx,\n        AddressableMessageQueue messageQueue,\n        PopLiteMessageRequestHeader requestHeader,\n        long timeoutMillis\n    ) {\n        return this.mqClientAPIFactory.getClient().popLiteMessageAsync(\n            messageQueue.getBrokerAddr(),\n            messageQueue.getBrokerName(),\n            requestHeader,\n            timeoutMillis\n        );\n    }\n\n    @Override\n    public CompletableFuture<AckResult> changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, String messageId,\n        ChangeInvisibleTimeRequestHeader requestHeader, long timeoutMillis) {\n        return this.mqClientAPIFactory.getClient().changeInvisibleTimeAsync(\n            this.resolveBrokerAddrInReceiptHandle(ctx, handle),\n            handle.getBrokerName(),\n            requestHeader,\n            timeoutMillis\n        );\n    }\n\n    @Override\n    public CompletableFuture<AckResult> ackMessage(ProxyContext ctx, ReceiptHandle handle, String messageId,\n        AckMessageRequestHeader requestHeader, long timeoutMillis) {\n        return this.mqClientAPIFactory.getClient().ackMessageAsync(\n            this.resolveBrokerAddrInReceiptHandle(ctx, handle),\n            requestHeader,\n            timeoutMillis\n        );\n    }\n\n    @Override\n    public CompletableFuture<AckResult> batchAckMessage(ProxyContext ctx, List<ReceiptHandleMessage> handleList,\n        String consumerGroup,\n        String topic, long timeoutMillis) {\n        List<String> extraInfoList = handleList.stream().map(message -> message.getReceiptHandle().getReceiptHandle()).collect(Collectors.toList());\n        return this.mqClientAPIFactory.getClient().batchAckMessageAsync(\n            this.resolveBrokerAddrInReceiptHandle(ctx, handleList.get(0).getReceiptHandle()),\n            topic,\n            consumerGroup,\n            extraInfoList,\n            timeoutMillis\n        );\n    }\n\n    @Override\n    public CompletableFuture<PullResult> pullMessage(ProxyContext ctx, AddressableMessageQueue messageQueue,\n        PullMessageRequestHeader requestHeader, long timeoutMillis) {\n        return this.mqClientAPIFactory.getClient().pullMessageAsync(\n            messageQueue.getBrokerAddr(),\n            requestHeader,\n            timeoutMillis\n        );\n    }\n\n    @Override\n    public CompletableFuture<Long> queryConsumerOffset(ProxyContext ctx, AddressableMessageQueue messageQueue,\n        QueryConsumerOffsetRequestHeader requestHeader, long timeoutMillis) {\n        return this.mqClientAPIFactory.getClient().queryConsumerOffsetWithFuture(\n            messageQueue.getBrokerAddr(),\n            requestHeader,\n            timeoutMillis\n        );\n    }\n\n    @Override\n    public CompletableFuture<Void> updateConsumerOffset(ProxyContext ctx, AddressableMessageQueue messageQueue,\n        UpdateConsumerOffsetRequestHeader requestHeader, long timeoutMillis) {\n        return this.mqClientAPIFactory.getClient().updateConsumerOffsetOneWay(\n            messageQueue.getBrokerAddr(),\n            requestHeader,\n            timeoutMillis\n        );\n    }\n\n    @Override\n    public CompletableFuture<Void> updateConsumerOffsetAsync(ProxyContext ctx, AddressableMessageQueue messageQueue,\n        UpdateConsumerOffsetRequestHeader requestHeader, long timeoutMillis) {\n        return this.mqClientAPIFactory.getClient().updateConsumerOffsetAsync(\n            messageQueue.getBrokerAddr(),\n            requestHeader,\n            timeoutMillis\n        );\n    }\n\n    @Override\n    public CompletableFuture<Set<MessageQueue>> lockBatchMQ(ProxyContext ctx, AddressableMessageQueue messageQueue,\n        LockBatchRequestBody requestBody, long timeoutMillis) {\n        return this.mqClientAPIFactory.getClient().lockBatchMQWithFuture(\n            messageQueue.getBrokerAddr(),\n            requestBody,\n            timeoutMillis\n        );\n    }\n\n    @Override\n    public CompletableFuture<Void> unlockBatchMQ(ProxyContext ctx, AddressableMessageQueue messageQueue,\n        UnlockBatchRequestBody requestBody, long timeoutMillis) {\n        return this.mqClientAPIFactory.getClient().unlockBatchMQOneway(\n            messageQueue.getBrokerAddr(),\n            requestBody,\n            timeoutMillis\n        );\n    }\n\n    @Override\n    public CompletableFuture<Long> getMaxOffset(ProxyContext ctx, AddressableMessageQueue messageQueue,\n        GetMaxOffsetRequestHeader requestHeader, long timeoutMillis) {\n        return this.mqClientAPIFactory.getClient().getMaxOffset(\n            messageQueue.getBrokerAddr(),\n            requestHeader,\n            timeoutMillis\n        );\n    }\n\n    @Override\n    public CompletableFuture<Long> getMinOffset(ProxyContext ctx, AddressableMessageQueue messageQueue,\n        GetMinOffsetRequestHeader requestHeader, long timeoutMillis) {\n        return this.mqClientAPIFactory.getClient().getMinOffset(\n            messageQueue.getBrokerAddr(),\n            requestHeader,\n            timeoutMillis\n        );\n    }\n\n    @Override\n    public CompletableFuture<String> recallMessage(ProxyContext ctx, String brokerName,\n        RecallMessageRequestHeader requestHeader, long timeoutMillis) {\n        return this.mqClientAPIFactory.getClient().recallMessageAsync(\n            this.resolveBrokerAddr(ctx, brokerName),\n            requestHeader,\n            timeoutMillis\n        );\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> request(ProxyContext ctx, String brokerName, RemotingCommand request,\n        long timeoutMillis) {\n        try {\n            String brokerAddress = topicRouteService.getBrokerAddr(ctx, brokerName);\n            return mqClientAPIFactory.getClient().invoke(brokerAddress, request, timeoutMillis);\n        } catch (Throwable t) {\n            return FutureUtils.completeExceptionally(t);\n        }\n    }\n\n    @Override\n    public CompletableFuture<Void> requestOneway(ProxyContext ctx, String brokerName, RemotingCommand request,\n        long timeoutMillis) {\n        try {\n            String brokerAddress = topicRouteService.getBrokerAddr(ctx, brokerName);\n            return mqClientAPIFactory.getClient().invokeOneway(brokerAddress, request, timeoutMillis);\n        } catch (Throwable t) {\n            return FutureUtils.completeExceptionally(t);\n        }\n    }\n\n    protected String resolveBrokerAddrInReceiptHandle(ProxyContext ctx, ReceiptHandle handle) {\n        try {\n            return this.topicRouteService.getBrokerAddr(ctx, handle.getBrokerName());\n        } catch (Throwable t) {\n            throw new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, \"cannot find broker \" + handle.getBrokerName(), t);\n        }\n    }\n\n    protected String resolveBrokerAddr(ProxyContext ctx, String brokerName) {\n        try {\n            return this.topicRouteService.getBrokerAddr(ctx, brokerName);\n        } catch (Throwable t) {\n            throw new ProxyException(ProxyExceptionCode.INVALID_BROKER_NAME, \"cannot find broker \" + brokerName, t);\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.message;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.BitSet;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.commons.lang3.NotImplementedException;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.AckStatus;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PopStatus;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageBatch;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\nimport org.apache.rocketmq.proxy.service.channel.ChannelManager;\nimport org.apache.rocketmq.proxy.service.channel.InvocationContext;\nimport org.apache.rocketmq.proxy.service.channel.SimpleChannel;\nimport org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.BatchAck;\nimport org.apache.rocketmq.remoting.protocol.body.BatchAckMessageRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;\nimport org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMinOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopLiteMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class LocalMessageService implements MessageService {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    private final BrokerController brokerController;\n    private final ChannelManager channelManager;\n\n    public LocalMessageService(BrokerController brokerController, ChannelManager channelManager, RPCHook rpcHook) {\n        this.brokerController = brokerController;\n        this.channelManager = channelManager;\n    }\n\n    @Override\n    public CompletableFuture<List<SendResult>> sendMessage(ProxyContext ctx, AddressableMessageQueue messageQueue,\n        List<Message> msgList, SendMessageRequestHeader requestHeader, long timeoutMillis) {\n        byte[] body;\n        String messageId;\n        if (msgList.size() > 1) {\n            requestHeader.setBatch(true);\n            MessageBatch msgBatch = MessageBatch.generateFromList(msgList);\n            MessageClientIDSetter.setUniqID(msgBatch);\n            body = msgBatch.encode();\n            msgBatch.setBody(body);\n            messageId = MessageClientIDSetter.getUniqID(msgBatch);\n        } else {\n            Message message = msgList.get(0);\n            body = message.getBody();\n            messageId = MessageClientIDSetter.getUniqID(message);\n        }\n        RemotingCommand request = LocalRemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, requestHeader, ctx.getLanguage());\n        request.setBody(body);\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        SimpleChannel channel = channelManager.createInvocationChannel(ctx);\n        InvocationContext invocationContext = new InvocationContext(future);\n        channel.registerInvocationContext(request.getOpaque(), invocationContext);\n        ChannelHandlerContext simpleChannelHandlerContext = channel.getChannelHandlerContext();\n        try {\n            RemotingCommand response = brokerController.getSendMessageProcessor().processRequest(simpleChannelHandlerContext, request);\n            if (response != null) {\n                invocationContext.handle(response);\n                channel.eraseInvocationContext(request.getOpaque());\n            }\n        } catch (Exception e) {\n            future.completeExceptionally(e);\n            channel.eraseInvocationContext(request.getOpaque());\n            log.error(\"Failed to process sendMessage command\", e);\n        }\n        return future.thenApply(r -> {\n            SendResult sendResult = new SendResult();\n            SendMessageResponseHeader responseHeader = (SendMessageResponseHeader) r.readCustomHeader();\n            SendStatus sendStatus;\n            switch (r.getCode()) {\n                case ResponseCode.FLUSH_DISK_TIMEOUT: {\n                    sendStatus = SendStatus.FLUSH_DISK_TIMEOUT;\n                    break;\n                }\n                case ResponseCode.FLUSH_SLAVE_TIMEOUT: {\n                    sendStatus = SendStatus.FLUSH_SLAVE_TIMEOUT;\n                    break;\n                }\n                case ResponseCode.SLAVE_NOT_AVAILABLE: {\n                    sendStatus = SendStatus.SLAVE_NOT_AVAILABLE;\n                    break;\n                }\n                case ResponseCode.SUCCESS: {\n                    sendStatus = SendStatus.SEND_OK;\n                    break;\n                }\n                default: {\n                    throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, r.getRemark());\n                }\n            }\n            sendResult.setSendStatus(sendStatus);\n            sendResult.setMsgId(messageId);\n            sendResult.setMessageQueue(new MessageQueue(requestHeader.getTopic(), brokerController.getBrokerConfig().getBrokerName(), requestHeader.getQueueId()));\n            sendResult.setQueueOffset(responseHeader.getQueueOffset());\n            sendResult.setTransactionId(responseHeader.getTransactionId());\n            sendResult.setOffsetMsgId(responseHeader.getMsgId());\n            sendResult.setRecallHandle(responseHeader.getRecallHandle());\n            return Collections.singletonList(sendResult);\n        });\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> sendMessageBack(ProxyContext ctx, ReceiptHandle handle, String messageId,\n        ConsumerSendMsgBackRequestHeader requestHeader, long timeoutMillis) {\n        SimpleChannel channel = channelManager.createChannel(ctx);\n        ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext();\n        RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, requestHeader, ctx.getLanguage());\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        try {\n            RemotingCommand response = brokerController.getSendMessageProcessor()\n                .processRequest(channelHandlerContext, command);\n            future.complete(response);\n        } catch (Exception e) {\n            log.error(\"Fail to process sendMessageBack command\", e);\n            future.completeExceptionally(e);\n        }\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<Void> endTransactionOneway(ProxyContext ctx, String brokerName,\n        EndTransactionRequestHeader requestHeader,\n        long timeoutMillis) {\n        CompletableFuture<Void> future = new CompletableFuture<>();\n        SimpleChannel channel = channelManager.createChannel(ctx);\n        ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext();\n        RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.END_TRANSACTION, requestHeader, ctx.getLanguage());\n        try {\n            brokerController.getEndTransactionProcessor()\n                .processRequest(channelHandlerContext, command);\n            future.complete(null);\n        } catch (Exception e) {\n            future.completeExceptionally(e);\n        }\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<PopResult> popLiteMessage(ProxyContext ctx, AddressableMessageQueue messageQueue,\n        PopLiteMessageRequestHeader requestHeader, long timeoutMillis) {\n        throw new NotImplementedException();\n    }\n\n    @Override\n    public CompletableFuture<PopResult> popMessage(ProxyContext ctx, AddressableMessageQueue messageQueue,\n        PopMessageRequestHeader requestHeader, long timeoutMillis) {\n        RemotingCommand request = LocalRemotingCommand.createRequestCommand(RequestCode.POP_MESSAGE, requestHeader, ctx.getLanguage());\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        SimpleChannel channel = channelManager.createInvocationChannel(ctx);\n        InvocationContext invocationContext = new InvocationContext(future);\n        channel.registerInvocationContext(request.getOpaque(), invocationContext);\n        ChannelHandlerContext simpleChannelHandlerContext = channel.getChannelHandlerContext();\n        try {\n            RemotingCommand response = brokerController.getPopMessageProcessor().processRequest(simpleChannelHandlerContext, request);\n            if (response != null) {\n                invocationContext.handle(response);\n                channel.eraseInvocationContext(request.getOpaque());\n            }\n        } catch (Exception e) {\n            future.completeExceptionally(e);\n            channel.eraseInvocationContext(request.getOpaque());\n            log.error(\"Failed to process popMessage command\", e);\n        }\n        return future.thenApply(r -> {\n            PopStatus popStatus;\n            List<MessageExt> messageExtList = new ArrayList<>();\n            switch (r.getCode()) {\n                case ResponseCode.SUCCESS:\n                    popStatus = PopStatus.FOUND;\n                    ByteBuffer byteBuffer = ByteBuffer.wrap(r.getBody());\n                    messageExtList = MessageDecoder.decodesBatch(\n                        byteBuffer,\n                        true,\n                        false,\n                        true\n                    );\n                    break;\n                case ResponseCode.POLLING_FULL:\n                    popStatus = PopStatus.POLLING_FULL;\n                    break;\n                case ResponseCode.POLLING_TIMEOUT:\n                case ResponseCode.PULL_NOT_FOUND:\n                    popStatus = PopStatus.POLLING_NOT_FOUND;\n                    break;\n                default:\n                    throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, r.getRemark());\n            }\n            PopResult popResult = new PopResult(popStatus, messageExtList);\n            PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) r.readCustomHeader();\n\n            if (popStatus == PopStatus.FOUND) {\n                Map<String, Long> startOffsetInfo;\n                Map<String, List<Long>> msgOffsetInfo;\n                Map<String, Integer> orderCountInfo;\n                popResult.setInvisibleTime(responseHeader.getInvisibleTime());\n                popResult.setPopTime(responseHeader.getPopTime());\n                startOffsetInfo = ExtraInfoUtil.parseStartOffsetInfo(responseHeader.getStartOffsetInfo());\n                msgOffsetInfo = ExtraInfoUtil.parseMsgOffsetInfo(responseHeader.getMsgOffsetInfo());\n                orderCountInfo = ExtraInfoUtil.parseOrderCountInfo(responseHeader.getOrderCountInfo());\n                // <topicMark@queueId, msg queueOffset>\n                Map<String, List<Long>> sortMap = new HashMap<>(16);\n                for (MessageExt messageExt : messageExtList) {\n                    // Value of POP_CK is used to determine whether it is a pop retry,\n                    // cause topic could be rewritten by broker.\n                    String key = ExtraInfoUtil.getStartOffsetInfoMapKey(messageExt.getTopic(),\n                        messageExt.getProperty(MessageConst.PROPERTY_POP_CK), messageExt.getQueueId());\n                    if (!sortMap.containsKey(key)) {\n                        sortMap.put(key, new ArrayList<>(4));\n                    }\n                    sortMap.get(key).add(messageExt.getQueueOffset());\n                }\n                Map<String, String> map = new HashMap<>(5);\n                for (MessageExt messageExt : messageExtList) {\n                    if (startOffsetInfo == null) {\n                        // we should set the check point info to extraInfo field , if the command is popMsg\n                        // find pop ck offset\n                        String key = messageExt.getTopic() + messageExt.getQueueId();\n                        if (!map.containsKey(messageExt.getTopic() + messageExt.getQueueId())) {\n                            map.put(key, ExtraInfoUtil.buildExtraInfo(messageExt.getQueueOffset(), responseHeader.getPopTime(), responseHeader.getInvisibleTime(), responseHeader.getReviveQid(),\n                                messageExt.getTopic(), messageQueue.getBrokerName(), messageExt.getQueueId()));\n                        }\n                        messageExt.getProperties().put(MessageConst.PROPERTY_POP_CK, map.get(key) + MessageConst.KEY_SEPARATOR + messageExt.getQueueOffset());\n                    } else {\n                        if (messageExt.getProperty(MessageConst.PROPERTY_POP_CK) == null) {\n                            String key = ExtraInfoUtil.getStartOffsetInfoMapKey(messageExt.getTopic(), messageExt.getQueueId());\n                            int index = sortMap.get(key).indexOf(messageExt.getQueueOffset());\n                            Long msgQueueOffset = msgOffsetInfo.get(key).get(index);\n                            if (msgQueueOffset != messageExt.getQueueOffset()) {\n                                log.warn(\"Queue offset [{}] of msg is strange, not equal to the stored in msg, {}\", msgQueueOffset, messageExt);\n                            }\n\n                            messageExt.getProperties().put(MessageConst.PROPERTY_POP_CK,\n                                ExtraInfoUtil.buildExtraInfo(startOffsetInfo.get(key), responseHeader.getPopTime(), responseHeader.getInvisibleTime(),\n                                    responseHeader.getReviveQid(), messageExt.getTopic(), messageQueue.getBrokerName(), messageExt.getQueueId(), msgQueueOffset)\n                            );\n                            if (requestHeader.isOrder() && orderCountInfo != null) {\n                                Integer count = orderCountInfo.get(key);\n                                if (count != null && count > 0) {\n                                    messageExt.setReconsumeTimes(count);\n                                }\n                            }\n                        }\n                    }\n                    messageExt.getProperties().computeIfAbsent(MessageConst.PROPERTY_FIRST_POP_TIME, k -> String.valueOf(responseHeader.getPopTime()));\n                    messageExt.setBrokerName(messageExt.getBrokerName());\n                    messageExt.setTopic(messageQueue.getTopic());\n                }\n            }\n            return popResult;\n        });\n    }\n\n    @Override\n    public CompletableFuture<AckResult> changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, String messageId,\n        ChangeInvisibleTimeRequestHeader requestHeader, long timeoutMillis) {\n        SimpleChannel channel = channelManager.createChannel(ctx);\n        ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext();\n        RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader, ctx.getLanguage());\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        try {\n            future = brokerController.getChangeInvisibleTimeProcessor()\n                .processRequestAsync(channelHandlerContext.channel(), command, true);\n        } catch (Exception e) {\n            log.error(\"Fail to process changeInvisibleTime command\", e);\n            future.completeExceptionally(e);\n        }\n        return future.thenApply(r -> {\n            ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) r.readCustomHeader();\n            AckResult ackResult = new AckResult();\n            if (ResponseCode.SUCCESS == r.getCode()) {\n                ackResult.setStatus(AckStatus.OK);\n            } else {\n                ackResult.setStatus(AckStatus.NO_EXIST);\n            }\n            ackResult.setPopTime(responseHeader.getPopTime());\n            ackResult.setExtraInfo(ReceiptHandle.builder()\n                .startOffset(handle.getStartOffset())\n                .retrieveTime(responseHeader.getPopTime())\n                .invisibleTime(responseHeader.getInvisibleTime())\n                .reviveQueueId(responseHeader.getReviveQid())\n                .topicType(handle.getTopicType())\n                .brokerName(handle.getBrokerName())\n                .queueId(handle.getQueueId())\n                .offset(handle.getOffset())\n                .build()\n                .encode());\n            return ackResult;\n        });\n    }\n\n    @Override\n    public CompletableFuture<AckResult> ackMessage(ProxyContext ctx, ReceiptHandle handle, String messageId,\n        AckMessageRequestHeader requestHeader, long timeoutMillis) {\n        SimpleChannel channel = channelManager.createChannel(ctx);\n        ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext();\n        RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader, ctx.getLanguage());\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        try {\n            RemotingCommand response = brokerController.getAckMessageProcessor()\n                .processRequest(channelHandlerContext, command);\n            future.complete(response);\n        } catch (Exception e) {\n            log.error(\"Fail to process ackMessage command\", e);\n            future.completeExceptionally(e);\n        }\n        return future.thenApply(r -> {\n            AckResult ackResult = new AckResult();\n            if (ResponseCode.SUCCESS == r.getCode()) {\n                ackResult.setStatus(AckStatus.OK);\n            } else {\n                ackResult.setStatus(AckStatus.NO_EXIST);\n            }\n            return ackResult;\n        });\n    }\n\n    @Override\n    public CompletableFuture<AckResult> batchAckMessage(ProxyContext ctx, List<ReceiptHandleMessage> handleList,\n        String consumerGroup, String topic, long timeoutMillis) {\n        SimpleChannel channel = channelManager.createChannel(ctx);\n        ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext();\n        RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.BATCH_ACK_MESSAGE, null);\n\n        Map<String, BatchAck> batchAckMap = new HashMap<>();\n        for (ReceiptHandleMessage receiptHandleMessage : handleList) {\n            String extraInfo = receiptHandleMessage.getReceiptHandle().getReceiptHandle();\n            String[] extraInfoData = ExtraInfoUtil.split(extraInfo);\n            String mergeKey = ExtraInfoUtil.getRetry(extraInfoData) + \"@\" +\n                ExtraInfoUtil.getQueueId(extraInfoData) + \"@\" +\n                ExtraInfoUtil.getCkQueueOffset(extraInfoData) + \"@\" +\n                ExtraInfoUtil.getPopTime(extraInfoData);\n            BatchAck bAck = batchAckMap.computeIfAbsent(mergeKey, k -> {\n                BatchAck newBatchAck = new BatchAck();\n                newBatchAck.setConsumerGroup(consumerGroup);\n                newBatchAck.setTopic(topic);\n                newBatchAck.setRetry(ExtraInfoUtil.getRetry(extraInfoData));\n                newBatchAck.setStartOffset(ExtraInfoUtil.getCkQueueOffset(extraInfoData));\n                newBatchAck.setQueueId(ExtraInfoUtil.getQueueId(extraInfoData));\n                newBatchAck.setReviveQueueId(ExtraInfoUtil.getReviveQid(extraInfoData));\n                newBatchAck.setPopTime(ExtraInfoUtil.getPopTime(extraInfoData));\n                newBatchAck.setInvisibleTime(ExtraInfoUtil.getInvisibleTime(extraInfoData));\n                newBatchAck.setBitSet(new BitSet());\n                return newBatchAck;\n            });\n            bAck.getBitSet().set((int) (ExtraInfoUtil.getQueueOffset(extraInfoData) - ExtraInfoUtil.getCkQueueOffset(extraInfoData)));\n        }\n        BatchAckMessageRequestBody requestBody = new BatchAckMessageRequestBody();\n        requestBody.setBrokerName(brokerController.getBrokerConfig().getBrokerName());\n        requestBody.setAcks(new ArrayList<>(batchAckMap.values()));\n\n        command.setBody(requestBody.encode());\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        try {\n            RemotingCommand response = brokerController.getAckMessageProcessor()\n                .processRequest(channelHandlerContext, command);\n            future.complete(response);\n        } catch (Exception e) {\n            log.error(\"Fail to process batchAckMessage command\", e);\n            future.completeExceptionally(e);\n        }\n        return future.thenApply(r -> {\n            AckResult ackResult = new AckResult();\n            if (ResponseCode.SUCCESS == r.getCode()) {\n                ackResult.setStatus(AckStatus.OK);\n            } else {\n                ackResult.setStatus(AckStatus.NO_EXIST);\n            }\n            return ackResult;\n        });\n    }\n\n    @Override\n    public CompletableFuture<PullResult> pullMessage(ProxyContext ctx, AddressableMessageQueue messageQueue,\n        PullMessageRequestHeader requestHeader, long timeoutMillis) {\n        throw new NotImplementedException(\"pullMessage is not implemented in LocalMessageService\");\n    }\n\n    @Override\n    public CompletableFuture<Long> queryConsumerOffset(ProxyContext ctx, AddressableMessageQueue messageQueue,\n        QueryConsumerOffsetRequestHeader requestHeader, long timeoutMillis) {\n        throw new NotImplementedException(\"queryConsumerOffset is not implemented in LocalMessageService\");\n    }\n\n    @Override\n    public CompletableFuture<Void> updateConsumerOffset(ProxyContext ctx, AddressableMessageQueue messageQueue,\n        UpdateConsumerOffsetRequestHeader requestHeader, long timeoutMillis) {\n        throw new NotImplementedException(\"updateConsumerOffset is not implemented in LocalMessageService\");\n    }\n\n    @Override\n    public CompletableFuture<Void> updateConsumerOffsetAsync(ProxyContext ctx, AddressableMessageQueue messageQueue,\n        UpdateConsumerOffsetRequestHeader requestHeader, long timeoutMillis) {\n        throw new NotImplementedException(\"updateConsumerOffsetAsync is not implemented in LocalMessageService\");\n    }\n\n    @Override\n    public CompletableFuture<Set<MessageQueue>> lockBatchMQ(ProxyContext ctx, AddressableMessageQueue messageQueue,\n        LockBatchRequestBody requestBody, long timeoutMillis) {\n        throw new NotImplementedException(\"lockBatchMQ is not implemented in LocalMessageService\");\n    }\n\n    @Override\n    public CompletableFuture<Void> unlockBatchMQ(ProxyContext ctx, AddressableMessageQueue messageQueue,\n        UnlockBatchRequestBody requestBody, long timeoutMillis) {\n        throw new NotImplementedException(\"unlockBatchMQ is not implemented in LocalMessageService\");\n    }\n\n    @Override\n    public CompletableFuture<Long> getMaxOffset(ProxyContext ctx, AddressableMessageQueue messageQueue,\n        GetMaxOffsetRequestHeader requestHeader, long timeoutMillis) {\n        throw new NotImplementedException(\"getMaxOffset is not implemented in LocalMessageService\");\n    }\n\n    @Override\n    public CompletableFuture<Long> getMinOffset(ProxyContext ctx, AddressableMessageQueue messageQueue,\n        GetMinOffsetRequestHeader requestHeader, long timeoutMillis) {\n        throw new NotImplementedException(\"getMinOffset is not implemented in LocalMessageService\");\n    }\n\n    @Override\n    public CompletableFuture<String> recallMessage(ProxyContext ctx, String brokerName,\n        RecallMessageRequestHeader requestHeader, long timeoutMillis) {\n        SimpleChannel channel = channelManager.createChannel(ctx);\n        ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext();\n        RemotingCommand command =\n            LocalRemotingCommand.createRequestCommand(RequestCode.RECALL_MESSAGE, requestHeader, ctx.getLanguage());\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        try {\n            RemotingCommand response = brokerController.getRecallMessageProcessor()\n                .processRequest(channelHandlerContext, command);\n            future.complete(response);\n        } catch (Exception e) {\n            log.error(\"Fail to process recallMessage command\", e);\n            future.completeExceptionally(e);\n        }\n        return future.thenApply(r -> {\n            switch (r.getCode()) {\n                case ResponseCode.SUCCESS:\n                    return ((RecallMessageResponseHeader) r.readCustomHeader()).getMsgId();\n                default:\n                    throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, r.getRemark());\n            }\n        });\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> request(ProxyContext ctx, String brokerName, RemotingCommand request,\n        long timeoutMillis) {\n        throw new NotImplementedException(\"request is not implemented in LocalMessageService\");\n    }\n\n    @Override\n    public CompletableFuture<Void> requestOneway(ProxyContext ctx, String brokerName, RemotingCommand request,\n        long timeoutMillis) {\n        throw new NotImplementedException(\"requestOneway is not implemented in LocalMessageService\");\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalRemotingCommand.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.message;\n\nimport java.util.HashMap;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic class LocalRemotingCommand extends RemotingCommand {\n\n    public static LocalRemotingCommand createRequestCommand(int code, CommandCustomHeader customHeader, String language) {\n        LocalRemotingCommand cmd = new LocalRemotingCommand();\n        cmd.setCode(code);\n        cmd.setLanguage(LanguageCode.getCode(language));\n        cmd.writeCustomHeader(customHeader);\n        cmd.setExtFields(new HashMap<>());\n        setCmdVersion(cmd);\n        cmd.makeCustomHeaderToNet();\n        return cmd;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.message;\n\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;\nimport org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMinOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopLiteMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;\n\npublic interface MessageService {\n\n    CompletableFuture<List<SendResult>> sendMessage(\n        ProxyContext ctx,\n        AddressableMessageQueue messageQueue,\n        List<Message> msgList,\n        SendMessageRequestHeader requestHeader,\n        long timeoutMillis\n    );\n\n    CompletableFuture<RemotingCommand> sendMessageBack(\n        ProxyContext ctx,\n        ReceiptHandle handle,\n        String messageId,\n        ConsumerSendMsgBackRequestHeader requestHeader,\n        long timeoutMillis\n    );\n\n    CompletableFuture<Void> endTransactionOneway(\n        ProxyContext ctx,\n        String brokerName,\n        EndTransactionRequestHeader requestHeader,\n        long timeoutMillis\n    );\n\n    CompletableFuture<PopResult> popMessage(\n        ProxyContext ctx,\n        AddressableMessageQueue messageQueue,\n        PopMessageRequestHeader requestHeader,\n        long timeoutMillis\n    );\n\n    CompletableFuture<PopResult> popLiteMessage(\n        ProxyContext ctx,\n        AddressableMessageQueue messageQueue,\n        PopLiteMessageRequestHeader requestHeader,\n        long timeoutMillis\n    );\n\n    CompletableFuture<AckResult> changeInvisibleTime(\n        ProxyContext ctx,\n        ReceiptHandle handle,\n        String messageId,\n        ChangeInvisibleTimeRequestHeader requestHeader,\n        long timeoutMillis\n    );\n\n    CompletableFuture<AckResult> ackMessage(\n        ProxyContext ctx,\n        ReceiptHandle handle,\n        String messageId,\n        AckMessageRequestHeader requestHeader,\n        long timeoutMillis\n    );\n\n    CompletableFuture<AckResult> batchAckMessage(\n        ProxyContext ctx,\n        List<ReceiptHandleMessage> handleList,\n        String consumerGroup,\n        String topic,\n        long timeoutMillis\n    );\n\n    CompletableFuture<PullResult> pullMessage(\n        ProxyContext ctx,\n        AddressableMessageQueue messageQueue,\n        PullMessageRequestHeader requestHeader,\n        long timeoutMillis\n    );\n\n    CompletableFuture<Long> queryConsumerOffset(\n        ProxyContext ctx,\n        AddressableMessageQueue messageQueue,\n        QueryConsumerOffsetRequestHeader requestHeader,\n        long timeoutMillis\n    );\n\n    CompletableFuture<Void> updateConsumerOffset(\n        ProxyContext ctx,\n        AddressableMessageQueue messageQueue,\n        UpdateConsumerOffsetRequestHeader requestHeader,\n        long timeoutMillis\n    );\n\n    CompletableFuture<Void> updateConsumerOffsetAsync(\n        ProxyContext ctx,\n        AddressableMessageQueue messageQueue,\n        UpdateConsumerOffsetRequestHeader requestHeader,\n        long timeoutMillis\n    );\n\n    CompletableFuture<Set<MessageQueue>> lockBatchMQ(\n        ProxyContext ctx,\n        AddressableMessageQueue messageQueue,\n        LockBatchRequestBody requestBody,\n        long timeoutMillis\n    );\n\n    CompletableFuture<Void> unlockBatchMQ(\n        ProxyContext ctx,\n        AddressableMessageQueue messageQueue,\n        UnlockBatchRequestBody requestBody,\n        long timeoutMillis\n    );\n\n    CompletableFuture<Long> getMaxOffset(\n        ProxyContext ctx,\n        AddressableMessageQueue messageQueue,\n        GetMaxOffsetRequestHeader requestHeader,\n        long timeoutMillis\n    );\n\n    CompletableFuture<Long> getMinOffset(\n        ProxyContext ctx,\n        AddressableMessageQueue messageQueue,\n        GetMinOffsetRequestHeader requestHeader,\n        long timeoutMillis\n    );\n\n    CompletableFuture<String> recallMessage(\n        ProxyContext ctx,\n        String brokerName,\n        RecallMessageRequestHeader requestHeader,\n        long timeoutMillis\n    );\n\n    CompletableFuture<RemotingCommand> request(ProxyContext ctx, String brokerName, RemotingCommand request,\n        long timeoutMillis);\n\n    CompletableFuture<Void> requestOneway(ProxyContext ctx, String brokerName, RemotingCommand request,\n        long timeoutMillis);\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.message;\n\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\n\npublic class ReceiptHandleMessage {\n\n    private final ReceiptHandle receiptHandle;\n    private final String messageId;\n\n    public ReceiptHandleMessage(ReceiptHandle receiptHandle, String messageId) {\n        this.receiptHandle = receiptHandle;\n        this.messageId = messageId;\n    }\n\n    public ReceiptHandle getReceiptHandle() {\n        return receiptHandle;\n    }\n\n    public String getMessageId() {\n        return messageId;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.metadata;\n\nimport com.google.common.cache.CacheBuilder;\nimport com.google.common.cache.LoadingCache;\nimport java.util.List;\nimport java.util.Optional;\nimport java.util.Random;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.auth.authorization.model.Acl;\nimport org.apache.rocketmq.broker.auth.converter.AclConverter;\nimport org.apache.rocketmq.broker.auth.converter.UserConverter;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.thread.ThreadPoolMonitor;\nimport org.apache.rocketmq.common.utils.AbstractStartAndShutdown;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.AbstractCacheLoader;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.service.route.TopicRouteHelper;\nimport org.apache.rocketmq.proxy.service.route.TopicRouteService;\nimport org.apache.rocketmq.remoting.protocol.body.AclInfo;\nimport org.apache.rocketmq.remoting.protocol.body.UserInfo;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicConfigAndQueueMapping;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\n\npublic class ClusterMetadataService extends AbstractStartAndShutdown implements MetadataService {\n    protected static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    private static final long DEFAULT_TIMEOUT = 3000;\n\n    private final TopicRouteService topicRouteService;\n    private final MQClientAPIFactory mqClientAPIFactory;\n\n    protected final ThreadPoolExecutor cacheRefreshExecutor;\n\n    protected final LoadingCache<String, TopicConfigAndQueueMapping> topicConfigCache;\n    protected final static TopicConfigAndQueueMapping EMPTY_TOPIC_CONFIG = new TopicConfigAndQueueMapping();\n\n    protected final LoadingCache<String, SubscriptionGroupConfig> subscriptionGroupConfigCache;\n    protected final static SubscriptionGroupConfig EMPTY_SUBSCRIPTION_GROUP_CONFIG = new SubscriptionGroupConfig();\n\n    protected final LoadingCache<String, User> userCache;\n\n    protected final static User EMPTY_USER = new User();\n\n    protected final LoadingCache<String, Acl> aclCache;\n\n    protected final static Acl EMPTY_ACL = new Acl();\n\n    protected final Random random = new Random();\n\n\n    public ClusterMetadataService(TopicRouteService topicRouteService, MQClientAPIFactory mqClientAPIFactory) {\n        this.topicRouteService = topicRouteService;\n        this.mqClientAPIFactory = mqClientAPIFactory;\n\n        ProxyConfig config = ConfigurationManager.getProxyConfig();\n        this.cacheRefreshExecutor = ThreadPoolMonitor.createAndMonitor(\n            config.getMetadataThreadPoolNums(),\n            config.getMetadataThreadPoolNums(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            \"MetadataCacheRefresh\",\n            config.getMetadataThreadPoolQueueCapacity()\n        );\n        this.topicConfigCache = CacheBuilder.newBuilder()\n            .maximumSize(config.getTopicConfigCacheMaxNum())\n            .expireAfterAccess(config.getTopicConfigCacheExpiredSeconds(), TimeUnit.SECONDS)\n            .refreshAfterWrite(config.getTopicConfigCacheRefreshSeconds(), TimeUnit.SECONDS)\n            .build(new ClusterTopicConfigCacheLoader());\n        this.subscriptionGroupConfigCache = CacheBuilder.newBuilder()\n            .maximumSize(config.getSubscriptionGroupConfigCacheMaxNum())\n            .expireAfterAccess(config.getSubscriptionGroupConfigCacheExpiredSeconds(), TimeUnit.SECONDS)\n            .refreshAfterWrite(config.getSubscriptionGroupConfigCacheRefreshSeconds(), TimeUnit.SECONDS)\n            .build(new ClusterSubscriptionGroupConfigCacheLoader());\n        this.userCache = CacheBuilder.newBuilder()\n            .maximumSize(config.getUserCacheMaxNum())\n            .expireAfterAccess(config.getUserCacheExpiredSeconds(), TimeUnit.SECONDS)\n            .refreshAfterWrite(config.getUserCacheRefreshSeconds(), TimeUnit.SECONDS)\n            .build(new ClusterUserCacheLoader());\n        this.aclCache = CacheBuilder.newBuilder()\n            .maximumSize(config.getAclCacheMaxNum())\n            .expireAfterAccess(config.getAclCacheExpiredSeconds(), TimeUnit.SECONDS)\n            .refreshAfterWrite(config.getAclCacheRefreshSeconds(), TimeUnit.SECONDS)\n            .build(new ClusterAclCacheLoader());\n\n        this.init();\n    }\n\n    protected void init() {\n        this.appendShutdown(this.cacheRefreshExecutor::shutdown);\n    }\n\n    @Override\n    public TopicMessageType getTopicMessageType(ProxyContext ctx, String topic) {\n        TopicConfigAndQueueMapping topicConfigAndQueueMapping;\n        try {\n            topicConfigAndQueueMapping = topicConfigCache.get(topic);\n        } catch (Exception e) {\n            return TopicMessageType.UNSPECIFIED;\n        }\n        if (topicConfigAndQueueMapping.equals(EMPTY_TOPIC_CONFIG)) {\n            return TopicMessageType.UNSPECIFIED;\n        }\n        return topicConfigAndQueueMapping.getTopicMessageType();\n    }\n\n    @Override\n    public SubscriptionGroupConfig getSubscriptionGroupConfig(ProxyContext ctx, String group) {\n        SubscriptionGroupConfig config;\n        try {\n            config = this.subscriptionGroupConfigCache.get(group);\n        } catch (Exception e) {\n            return null;\n        }\n        if (config == EMPTY_SUBSCRIPTION_GROUP_CONFIG) {\n            return null;\n        }\n        return config;\n    }\n\n    @Override\n    public CompletableFuture<User> getUser(ProxyContext ctx, String username) {\n        CompletableFuture<User> result = new CompletableFuture<>();\n        try {\n            User user = this.userCache.get(username);\n            if (user == EMPTY_USER) {\n                user = null;\n            }\n            result.complete(user);\n        } catch (Exception e) {\n            result.completeExceptionally(e);\n        }\n        return result;\n    }\n\n    @Override\n    public CompletableFuture<Acl> getAcl(ProxyContext ctx, Subject subject) {\n        CompletableFuture<Acl> result = new CompletableFuture<>();\n        try {\n            Acl acl = this.aclCache.get(subject.getSubjectKey());\n            if (acl == EMPTY_ACL) {\n                acl = null;\n            }\n            result.complete(acl);\n        } catch (Exception e) {\n            result.completeExceptionally(e);\n        }\n        return result;\n    }\n\n    protected class ClusterSubscriptionGroupConfigCacheLoader extends AbstractCacheLoader<String, SubscriptionGroupConfig> {\n\n        public ClusterSubscriptionGroupConfigCacheLoader() {\n            super(cacheRefreshExecutor);\n        }\n\n        @Override\n        protected SubscriptionGroupConfig getDirectly(String consumerGroup) throws Exception {\n            ProxyConfig config = ConfigurationManager.getProxyConfig();\n            String clusterName = config.getRocketMQClusterName();\n            Optional<BrokerData> brokerDataOptional = findOneBroker(clusterName);\n            if (brokerDataOptional.isPresent()) {\n                String brokerAddress = brokerDataOptional.get().selectBrokerAddr();\n                return mqClientAPIFactory.getClient().getSubscriptionGroupConfig(brokerAddress, consumerGroup, DEFAULT_TIMEOUT);\n            }\n            return EMPTY_SUBSCRIPTION_GROUP_CONFIG;\n        }\n\n        @Override\n        protected void onErr(String consumerGroup, Exception e) {\n            log.error(\"load subscription config failed. consumerGroup:{}\", consumerGroup, e);\n        }\n    }\n\n    protected class ClusterTopicConfigCacheLoader extends AbstractCacheLoader<String, TopicConfigAndQueueMapping> {\n\n        public ClusterTopicConfigCacheLoader() {\n            super(cacheRefreshExecutor);\n        }\n\n        @Override\n        protected TopicConfigAndQueueMapping getDirectly(String topic) throws Exception {\n            Optional<BrokerData> brokerDataOptional = findOneBroker(topic);\n            if (brokerDataOptional.isPresent()) {\n                String brokerAddress = brokerDataOptional.get().selectBrokerAddr();\n                return mqClientAPIFactory.getClient().getTopicConfig(brokerAddress, topic, DEFAULT_TIMEOUT);\n            }\n            return EMPTY_TOPIC_CONFIG;\n        }\n\n        @Override\n        protected void onErr(String key, Exception e) {\n            log.error(\"load topic config failed. topic:{}\", key, e);\n        }\n    }\n\n    protected class ClusterUserCacheLoader extends AbstractCacheLoader<String, User> {\n\n        public ClusterUserCacheLoader() {\n            super(cacheRefreshExecutor);\n        }\n\n        @Override\n        protected User getDirectly(String username) throws Exception {\n            ProxyConfig config = ConfigurationManager.getProxyConfig();\n            String clusterName = config.getRocketMQClusterName();\n            Optional<BrokerData> brokerDataOptional = findOneBroker(clusterName);\n            if (brokerDataOptional.isPresent()) {\n                String brokerAddress = brokerDataOptional.get().selectBrokerAddr();\n                UserInfo userInfo = mqClientAPIFactory.getClient().getUser(brokerAddress, username, DEFAULT_TIMEOUT);\n                if (userInfo == null) {\n                    return EMPTY_USER;\n                }\n                return UserConverter.convertUser(userInfo);\n            }\n            return EMPTY_USER;\n        }\n\n        @Override\n        protected void onErr(String key, Exception e) {\n            log.error(\"load user failed. username:{}\", key, e);\n        }\n    }\n\n    protected class ClusterAclCacheLoader extends AbstractCacheLoader<String, Acl> {\n\n        public ClusterAclCacheLoader() {\n            super(cacheRefreshExecutor);\n        }\n\n        @Override\n        protected Acl getDirectly(String subject) throws Exception {\n            ProxyConfig config = ConfigurationManager.getProxyConfig();\n            String clusterName = config.getRocketMQClusterName();\n            Optional<BrokerData> brokerDataOptional = findOneBroker(clusterName);\n            if (brokerDataOptional.isPresent()) {\n                String brokerAddress = brokerDataOptional.get().selectBrokerAddr();\n                AclInfo aclInfo = mqClientAPIFactory.getClient().getAcl(brokerAddress, subject, DEFAULT_TIMEOUT);\n                if (aclInfo == null) {\n                    return EMPTY_ACL;\n                }\n                return AclConverter.convertAcl(aclInfo);\n            }\n            return EMPTY_ACL;\n        }\n\n        @Override\n        protected void onErr(String key, Exception e) {\n            log.error(\"load acl failed. subject:{}\", key, e);\n        }\n    }\n\n    protected Optional<BrokerData> findOneBroker(String topic) throws Exception {\n        try {\n            List<BrokerData> brokerDatas = topicRouteService.getAllMessageQueueView(ProxyContext.createForInner(this.getClass()), topic).getTopicRouteData().getBrokerDatas();\n            int skipNum = random.nextInt(brokerDatas.size());\n            return brokerDatas.stream().skip(skipNum).findFirst();\n        } catch (Exception e) {\n            if (TopicRouteHelper.isTopicNotExistError(e)) {\n                return Optional.empty();\n            }\n            throw e;\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/LocalMetadataService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.metadata;\n\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.auth.authorization.model.Acl;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\n\npublic class LocalMetadataService implements MetadataService {\n    private final BrokerController brokerController;\n\n    public LocalMetadataService(BrokerController brokerController) {\n        this.brokerController = brokerController;\n    }\n\n    @Override\n    public TopicMessageType getTopicMessageType(ProxyContext ctx, String topic) {\n        TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(topic);\n        if (topicConfig == null) {\n            return TopicMessageType.UNSPECIFIED;\n        }\n        return topicConfig.getTopicMessageType();\n    }\n\n    @Override\n    public SubscriptionGroupConfig getSubscriptionGroupConfig(ProxyContext ctx, String group) {\n        return this.brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().get(group);\n    }\n\n    @Override\n    public CompletableFuture<User> getUser(ProxyContext ctx, String username) {\n        return this.brokerController.getAuthenticationMetadataManager().getUser(username);\n    }\n\n    @Override\n    public CompletableFuture<Acl> getAcl(ProxyContext ctx, Subject subject) {\n        return this.brokerController.getAuthorizationMetadataManager().getAcl(subject);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/MetadataService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.metadata;\n\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.auth.authentication.model.Subject;\nimport org.apache.rocketmq.auth.authentication.model.User;\nimport org.apache.rocketmq.auth.authorization.model.Acl;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\n\npublic interface MetadataService {\n\n    TopicMessageType getTopicMessageType(ProxyContext ctx, String topic);\n\n    SubscriptionGroupConfig getSubscriptionGroupConfig(ProxyContext ctx, String group);\n\n    CompletableFuture<User> getUser(ProxyContext ctx, String username);\n\n    CompletableFuture<Acl> getAcl(ProxyContext ctx, Subject subject);\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.receipt;\n\nimport com.google.common.base.Stopwatch;\nimport io.netty.channel.Channel;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ConsumerGroupEvent;\nimport org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;\nimport org.apache.rocketmq.broker.client.ConsumerManager;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.AckStatus;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.common.state.StateEventListener;\nimport org.apache.rocketmq.common.thread.ThreadPoolMonitor;\nimport org.apache.rocketmq.common.utils.AbstractStartAndShutdown;\nimport org.apache.rocketmq.common.utils.ConcurrentHashMapUtils;\nimport org.apache.rocketmq.common.utils.ExceptionUtils;\nimport org.apache.rocketmq.common.utils.StartAndShutdown;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.MessageReceiptHandle;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\nimport org.apache.rocketmq.proxy.common.ReceiptHandleGroup;\nimport org.apache.rocketmq.proxy.common.ReceiptHandleGroupKey;\nimport org.apache.rocketmq.proxy.common.RenewEvent;\nimport org.apache.rocketmq.proxy.common.RenewStrategyPolicy;\nimport org.apache.rocketmq.proxy.common.channel.ChannelHelper;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.service.metadata.MetadataService;\nimport org.apache.rocketmq.remoting.protocol.subscription.RetryPolicy;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\n\npublic class DefaultReceiptHandleManager extends AbstractStartAndShutdown implements ReceiptHandleManager {\n    protected final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    protected final MetadataService metadataService;\n    protected final ConsumerManager consumerManager;\n    protected final ConcurrentMap<ReceiptHandleGroupKey, ReceiptHandleGroup> receiptHandleGroupMap;\n    protected final StateEventListener<RenewEvent> eventListener;\n    protected final static RetryPolicy RENEW_POLICY = new RenewStrategyPolicy();\n    protected final ScheduledExecutorService scheduledExecutorService =\n        ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(\"RenewalScheduledThread_\"));\n    protected final ThreadPoolExecutor renewalWorkerService;\n    protected final ThreadPoolExecutor returnHandleGroupWorkerService;\n\n    public DefaultReceiptHandleManager(MetadataService metadataService, ConsumerManager consumerManager, StateEventListener<RenewEvent> eventListener) {\n        this.metadataService = metadataService;\n        this.consumerManager = consumerManager;\n        this.eventListener = eventListener;\n        ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();\n        this.renewalWorkerService = ThreadPoolMonitor.createAndMonitor(\n            proxyConfig.getRenewThreadPoolNums(),\n            proxyConfig.getRenewMaxThreadPoolNums(),\n            1, TimeUnit.MINUTES,\n            \"RenewalWorkerThread\",\n            proxyConfig.getRenewThreadPoolQueueCapacity()\n        );\n        this.returnHandleGroupWorkerService = ThreadPoolMonitor.createAndMonitor(\n            proxyConfig.getReturnHandleGroupThreadPoolNums(),\n            proxyConfig.getReturnHandleGroupThreadPoolNums() * 2,\n            1, TimeUnit.MINUTES,\n            \"ReturnHandleGroupWorkerThread\",\n            proxyConfig.getRenewThreadPoolQueueCapacity()\n        );\n        consumerManager.appendConsumerIdsChangeListener(new ConsumerIdsChangeListener() {\n            @Override\n            public void handle(ConsumerGroupEvent event, String group, Object... args) {\n                if (ConsumerGroupEvent.CLIENT_UNREGISTER.equals(event)) {\n                    if (args == null || args.length < 1) {\n                        return;\n                    }\n                    if (args[0] instanceof ClientChannelInfo) {\n                        ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0];\n                        if (ChannelHelper.isRemote(clientChannelInfo.getChannel())) {\n                            // if the channel sync from other proxy is expired, not to clear data of connect to current proxy\n                            return;\n                        }\n                        clearGroup(new ReceiptHandleGroupKey(clientChannelInfo.getChannel(), group));\n                        log.info(\"clear handle of this client when client unregister. group:{}, clientChannelInfo:{}\", group, clientChannelInfo);\n                    }\n                }\n            }\n\n            @Override\n            public void shutdown() {\n\n            }\n        });\n        this.receiptHandleGroupMap = new ConcurrentHashMap<>();\n        this.renewalWorkerService.setRejectedExecutionHandler((r, executor) -> log.warn(\"add renew task failed. queueSize:{}\", executor.getQueue().size()));\n        this.appendStartAndShutdown(new StartAndShutdown() {\n            @Override\n            public void start() throws Exception {\n                scheduledExecutorService.scheduleWithFixedDelay(() -> scheduleRenewTask(), 0,\n                    ConfigurationManager.getProxyConfig().getRenewSchedulePeriodMillis(), TimeUnit.MILLISECONDS);\n            }\n\n            @Override\n            public void shutdown() throws Exception {\n                scheduledExecutorService.shutdown();\n                clearAllHandle();\n            }\n        });\n    }\n\n    public void addReceiptHandle(ProxyContext context, Channel channel, String group, String msgID, MessageReceiptHandle messageReceiptHandle) {\n        ConcurrentHashMapUtils.computeIfAbsent(this.receiptHandleGroupMap, new ReceiptHandleGroupKey(channel, group),\n            k -> new ReceiptHandleGroup()).put(msgID, messageReceiptHandle);\n    }\n\n    public MessageReceiptHandle removeReceiptHandle(ProxyContext context, Channel channel, String group, String msgID, String receiptHandle) {\n        ReceiptHandleGroup handleGroup = receiptHandleGroupMap.get(new ReceiptHandleGroupKey(channel, group));\n        if (handleGroup == null) {\n            return null;\n        }\n        return handleGroup.remove(msgID, receiptHandle);\n    }\n\n    public int getUnackedMessageCount(ProxyContext context, Channel channel, String group) {\n        ReceiptHandleGroup handleGroup = receiptHandleGroupMap.get(new ReceiptHandleGroupKey(channel, group));\n        return handleGroup == null ? 0 : handleGroup.getMsgCount();\n    }\n\n    protected boolean clientIsOffline(ReceiptHandleGroupKey groupKey) {\n        return this.consumerManager.findChannel(groupKey.getGroup(), groupKey.getChannel()) == null;\n    }\n\n    protected void scheduleRenewTask() {\n        Stopwatch stopwatch = Stopwatch.createStarted();\n        try {\n            ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();\n            for (Map.Entry<ReceiptHandleGroupKey, ReceiptHandleGroup> entry : receiptHandleGroupMap.entrySet()) {\n                ReceiptHandleGroupKey key = entry.getKey();\n                if (clientIsOffline(key)) {\n                    clearGroup(key);\n                    continue;\n                }\n\n                ReceiptHandleGroup group = entry.getValue();\n                group.scan((msgID, handleStr, v) -> {\n                    long current = System.currentTimeMillis();\n                    ReceiptHandle handle = ReceiptHandle.decode(v.getReceiptHandleStr());\n                    if (handle.getNextVisibleTime() - current > proxyConfig.getRenewAheadTimeMillis()) {\n                        return;\n                    }\n                    renewalWorkerService.submit(() -> renewMessage(createContext(\"RenewMessage\"), key, group,\n                        msgID, handleStr));\n                });\n            }\n        } catch (Exception e) {\n            log.error(\"unexpect error when schedule renew task\", e);\n        }\n\n        log.debug(\"scan for renewal done. cost:{}ms\", stopwatch.elapsed().toMillis());\n    }\n\n    protected void renewMessage(ProxyContext context, ReceiptHandleGroupKey key, ReceiptHandleGroup group, String msgID, String handleStr) {\n        try {\n            group.computeIfPresent(msgID, handleStr, messageReceiptHandle -> startRenewMessage(context, key, messageReceiptHandle), 0);\n        } catch (Exception e) {\n            log.error(\"error when renew message. msgID:{}, handleStr:{}\", msgID, handleStr, e);\n        }\n    }\n\n    protected CompletableFuture<MessageReceiptHandle> startRenewMessage(ProxyContext context, ReceiptHandleGroupKey key, MessageReceiptHandle messageReceiptHandle) {\n        CompletableFuture<MessageReceiptHandle> resFuture = new CompletableFuture<>();\n        ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();\n        long current = System.currentTimeMillis();\n        try {\n            if (messageReceiptHandle.getRenewRetryTimes() >= proxyConfig.getMaxRenewRetryTimes()) {\n                log.warn(\"handle has exceed max renewRetryTimes. handle:{}\", messageReceiptHandle);\n                return CompletableFuture.completedFuture(null);\n            }\n            if (current - messageReceiptHandle.getConsumeTimestamp() < proxyConfig.getRenewMaxTimeMillis()) {\n                CompletableFuture<AckResult> future = new CompletableFuture<>();\n                eventListener.fireEvent(new RenewEvent(key, messageReceiptHandle, RENEW_POLICY.nextDelayDuration(messageReceiptHandle.getRenewTimes()), RenewEvent.EventType.RENEW, future));\n                future.whenComplete((ackResult, throwable) -> {\n                    if (throwable != null) {\n                        log.error(\"error when renew. handle:{}\", messageReceiptHandle, throwable);\n                        if (renewExceptionNeedRetry(throwable)) {\n                            messageReceiptHandle.incrementAndGetRenewRetryTimes();\n                            resFuture.complete(messageReceiptHandle);\n                        } else {\n                            resFuture.complete(null);\n                        }\n                    } else if (AckStatus.OK.equals(ackResult.getStatus())) {\n                        messageReceiptHandle.updateReceiptHandle(ackResult.getExtraInfo());\n                        messageReceiptHandle.resetRenewRetryTimes();\n                        messageReceiptHandle.incrementRenewTimes();\n                        resFuture.complete(messageReceiptHandle);\n                    } else {\n                        log.error(\"renew response is not ok. result:{}, handle:{}\", ackResult, messageReceiptHandle);\n                        resFuture.complete(null);\n                    }\n                });\n            } else {\n                SubscriptionGroupConfig subscriptionGroupConfig =\n                    metadataService.getSubscriptionGroupConfig(context, messageReceiptHandle.getGroup());\n                if (subscriptionGroupConfig == null) {\n                    log.error(\"group's subscriptionGroupConfig is null when renew. handle: {}\", messageReceiptHandle);\n                    return CompletableFuture.completedFuture(null);\n                }\n                RetryPolicy retryPolicy = subscriptionGroupConfig.getGroupRetryPolicy().getRetryPolicy();\n                CompletableFuture<AckResult> future = new CompletableFuture<>();\n                eventListener.fireEvent(new RenewEvent(key, messageReceiptHandle, retryPolicy.nextDelayDuration(messageReceiptHandle.getReconsumeTimes()), RenewEvent.EventType.STOP_RENEW, future));\n                future.whenComplete((ackResult, throwable) -> {\n                    if (throwable != null) {\n                        log.error(\"error when nack in renew. handle:{}\", messageReceiptHandle, throwable);\n                    }\n                    resFuture.complete(null);\n                });\n            }\n        } catch (Throwable t) {\n            log.error(\"unexpect error when renew message, stop to renew it. handle:{}\", messageReceiptHandle, t);\n            resFuture.complete(null);\n        }\n        return resFuture;\n    }\n\n    protected void clearGroup(ReceiptHandleGroupKey key) {\n        if (key == null) {\n            return;\n        }\n        ReceiptHandleGroup handleGroup = receiptHandleGroupMap.remove(key);\n        returnHandleGroupWorkerService.submit(() -> returnHandleGroup(key, handleGroup));\n    }\n\n    // There is no longer any waiting for lock, and only the locked handles will be processed immediately,\n    // while the handles that cannot be acquired will be kept waiting for the next scheduling.\n    private void returnHandleGroup(ReceiptHandleGroupKey key, ReceiptHandleGroup handleGroup) {\n        if (handleGroup == null || handleGroup.isEmpty()) {\n            return;\n        }\n        ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();\n        handleGroup.scan((msgID, handle, v) -> {\n            try {\n                handleGroup.computeIfPresent(msgID, handle, messageReceiptHandle -> {\n                    CompletableFuture<AckResult> future = new CompletableFuture<>();\n                    eventListener.fireEvent(new RenewEvent(key, messageReceiptHandle, proxyConfig.getInvisibleTimeMillisWhenClear(), RenewEvent.EventType.CLEAR_GROUP, future));\n                    return CompletableFuture.completedFuture(null);\n                }, 0);\n            } catch (Exception e) {\n                log.error(\"error when clear handle for group. key:{}\", key, e);\n            }\n        });\n        // scheduleRenewTask will trigger cleanup again\n        if (!handleGroup.isEmpty()) {\n            log.warn(\"The handle cannot be completely cleared, the remaining quantity is {}, key:{}\", handleGroup.getHandleNum(), key);\n            receiptHandleGroupMap.putIfAbsent(key, handleGroup);\n        }\n    }\n\n    protected void clearAllHandle() {\n        log.info(\"start clear all handle in receiptHandleProcessor\");\n        Set<ReceiptHandleGroupKey> keySet = receiptHandleGroupMap.keySet();\n        for (ReceiptHandleGroupKey key : keySet) {\n            clearGroup(key);\n        }\n        log.info(\"clear all handle in receiptHandleProcessor done\");\n    }\n\n    protected boolean renewExceptionNeedRetry(Throwable t) {\n        t = ExceptionUtils.getRealException(t);\n        if (t instanceof ProxyException) {\n            ProxyException proxyException = (ProxyException) t;\n            if (ProxyExceptionCode.INVALID_BROKER_NAME.equals(proxyException.getCode()) ||\n                ProxyExceptionCode.INVALID_RECEIPT_HANDLE.equals(proxyException.getCode())) {\n                return false;\n            }\n        }\n        return true;\n    }\n\n    protected ProxyContext createContext(String actionName) {\n        return ProxyContext.createForInner(this.getClass().getSimpleName() + actionName);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/ReceiptHandleManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.receipt;\n\nimport io.netty.channel.Channel;\nimport org.apache.rocketmq.proxy.common.MessageReceiptHandle;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\n\npublic interface ReceiptHandleManager {\n    void addReceiptHandle(ProxyContext context, Channel channel, String group, String msgID, MessageReceiptHandle messageReceiptHandle);\n\n    MessageReceiptHandle removeReceiptHandle(ProxyContext context, Channel channel, String group, String msgID, String receiptHandle);\n\n    int getUnackedMessageCount(ProxyContext context, Channel channel, String group);\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/AbstractProxyRelayService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.relay;\n\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\nimport org.apache.rocketmq.proxy.common.utils.ProxyUtils;\nimport org.apache.rocketmq.proxy.service.transaction.TransactionData;\nimport org.apache.rocketmq.proxy.service.transaction.TransactionService;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;\n\npublic abstract class AbstractProxyRelayService implements ProxyRelayService {\n\n    protected final TransactionService transactionService;\n\n    public AbstractProxyRelayService(TransactionService transactionService) {\n        this.transactionService = transactionService;\n    }\n\n    @Override\n    public RelayData<TransactionData, Void> processCheckTransactionState(ProxyContext context,\n        RemotingCommand command, CheckTransactionStateRequestHeader header, MessageExt messageExt) {\n        CompletableFuture<ProxyRelayResult<Void>> future = new CompletableFuture<>();\n        String group = messageExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP);\n        TransactionData transactionData = transactionService.addTransactionDataByBrokerAddr(\n            context,\n            command.getExtFields().get(ProxyUtils.BROKER_ADDR),\n            messageExt.getTopic(),\n            group,\n            header.getTranStateTableOffset(),\n            header.getCommitLogOffset(),\n            header.getTransactionId(),\n            messageExt);\n        if (transactionData == null) {\n            throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR,\n                String.format(\"add transaction data failed. request:%s, message:%s\", command, messageExt));\n        }\n        future.exceptionally(throwable -> {\n            this.transactionService.onSendCheckTransactionStateFailed(context, group, transactionData);\n            return null;\n        });\n        return new RelayData<>(transactionData, future);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ClusterProxyRelayService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.relay;\n\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.service.transaction.TransactionService;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader;\n\n/**\n * not implement yet\n */\npublic class ClusterProxyRelayService extends AbstractProxyRelayService {\n\n    public ClusterProxyRelayService(TransactionService transactionService) {\n        super(transactionService);\n    }\n\n    @Override\n    public CompletableFuture<ProxyRelayResult<ConsumerRunningInfo>> processGetConsumerRunningInfo(\n        ProxyContext context, RemotingCommand command,\n        GetConsumerRunningInfoRequestHeader header) {\n        return null;\n    }\n\n    @Override\n    public CompletableFuture<ProxyRelayResult<ConsumeMessageDirectlyResult>> processConsumeMessageDirectly(\n        ProxyContext context, RemotingCommand command,\n        ConsumeMessageDirectlyResultRequestHeader header) {\n        return null;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/LocalProxyRelayService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.relay;\n\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.service.channel.SimpleChannel;\nimport org.apache.rocketmq.proxy.service.transaction.TransactionService;\nimport org.apache.rocketmq.remoting.RemotingServer;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingAbstract;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader;\n\npublic class LocalProxyRelayService extends AbstractProxyRelayService {\n\n    private final BrokerController brokerController;\n\n    public LocalProxyRelayService(BrokerController brokerController, TransactionService transactionService) {\n        super(transactionService);\n        this.brokerController = brokerController;\n    }\n\n    @Override\n    public CompletableFuture<ProxyRelayResult<ConsumerRunningInfo>> processGetConsumerRunningInfo(\n        ProxyContext context, RemotingCommand command, GetConsumerRunningInfoRequestHeader header) {\n        CompletableFuture<ProxyRelayResult<ConsumerRunningInfo>> future = new CompletableFuture<>();\n        future.thenAccept(proxyOutResult -> {\n            RemotingServer remotingServer = this.brokerController.getRemotingServer();\n            if (remotingServer instanceof NettyRemotingAbstract) {\n                NettyRemotingAbstract nettyRemotingAbstract = (NettyRemotingAbstract) remotingServer;\n                RemotingCommand remotingCommand = RemotingCommand.createResponseCommand(null);\n                remotingCommand.setOpaque(command.getOpaque());\n                remotingCommand.setCode(proxyOutResult.getCode());\n                remotingCommand.setRemark(proxyOutResult.getRemark());\n                if (proxyOutResult.getCode() == ResponseCode.SUCCESS && proxyOutResult.getResult() != null) {\n                    ConsumerRunningInfo consumerRunningInfo = proxyOutResult.getResult();\n                    remotingCommand.setBody(consumerRunningInfo.encode());\n                }\n                SimpleChannel simpleChannel = new SimpleChannel(context.getRemoteAddress(), context.getLocalAddress());\n                nettyRemotingAbstract.processResponseCommand(simpleChannel.getChannelHandlerContext(), remotingCommand);\n            }\n        });\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<ProxyRelayResult<ConsumeMessageDirectlyResult>> processConsumeMessageDirectly(\n        ProxyContext context, RemotingCommand command,\n        ConsumeMessageDirectlyResultRequestHeader header) {\n        CompletableFuture<ProxyRelayResult<ConsumeMessageDirectlyResult>> future = new CompletableFuture<>();\n        future.thenAccept(proxyOutResult -> {\n            RemotingServer remotingServer = this.brokerController.getRemotingServer();\n            if (remotingServer instanceof NettyRemotingAbstract) {\n                NettyRemotingAbstract nettyRemotingAbstract = (NettyRemotingAbstract) remotingServer;\n                RemotingCommand remotingCommand = RemotingCommand.createResponseCommand(null);\n                remotingCommand.setOpaque(command.getOpaque());\n                remotingCommand.setCode(proxyOutResult.getCode());\n                remotingCommand.setRemark(proxyOutResult.getRemark());\n                if (proxyOutResult.getCode() == ResponseCode.SUCCESS && proxyOutResult.getResult() != null) {\n                    ConsumeMessageDirectlyResult consumeMessageDirectlyResult = proxyOutResult.getResult();\n                    remotingCommand.setBody(consumeMessageDirectlyResult.encode());\n                }\n                SimpleChannel simpleChannel = new SimpleChannel(context.getRemoteAddress(), context.getLocalAddress());\n                nettyRemotingAbstract.processResponseCommand(simpleChannel.getChannelHandlerContext(), remotingCommand);\n            }\n        });\n        return future;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ProxyChannel.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.relay;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelConfig;\nimport io.netty.channel.ChannelFuture;\nimport io.netty.channel.ChannelId;\nimport io.netty.channel.ChannelMetadata;\nimport io.netty.channel.ChannelOutboundBuffer;\nimport io.netty.channel.DefaultChannelPromise;\nimport io.netty.channel.EventLoop;\nimport io.netty.util.concurrent.GlobalEventExecutor;\nimport java.net.SocketAddress;\nimport java.nio.ByteBuffer;\nimport java.util.HashMap;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.service.channel.SimpleChannel;\nimport org.apache.rocketmq.proxy.service.transaction.TransactionData;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.NotifyUnsubscribeLiteRequestHeader;\n\npublic abstract class ProxyChannel extends SimpleChannel {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    protected final SocketAddress remoteSocketAddress;\n    protected final SocketAddress localSocketAddress;\n\n    protected final ProxyRelayService proxyRelayService;\n\n    protected ProxyChannel(ProxyRelayService proxyRelayService, Channel parent, String remoteAddress,\n        String localAddress) {\n        super(parent, remoteAddress, localAddress);\n        this.proxyRelayService = proxyRelayService;\n        this.remoteSocketAddress = NetworkUtil.string2SocketAddress(remoteAddress);\n        this.localSocketAddress = NetworkUtil.string2SocketAddress(localAddress);\n    }\n\n    protected ProxyChannel(ProxyRelayService proxyRelayService, Channel parent, ChannelId id, String remoteAddress,\n        String localAddress) {\n        super(parent, id, remoteAddress, localAddress);\n        this.proxyRelayService = proxyRelayService;\n        this.remoteSocketAddress = NetworkUtil.string2SocketAddress(remoteAddress);\n        this.localSocketAddress = NetworkUtil.string2SocketAddress(localAddress);\n    }\n\n    @Override\n    public ChannelFuture writeAndFlush(Object msg) {\n        CompletableFuture<Void> processFuture = new CompletableFuture<>();\n\n        try {\n            if (msg instanceof RemotingCommand) {\n                ProxyContext context = ProxyContext.createForInner(this.getClass())\n                    .setRemoteAddress(remoteAddress)\n                    .setLocalAddress(localAddress);\n                RemotingCommand command = (RemotingCommand) msg;\n                if (command.getExtFields() == null) {\n                    command.setExtFields(new HashMap<>());\n                }\n                switch (command.getCode()) {\n                    case RequestCode.CHECK_TRANSACTION_STATE: {\n                        CheckTransactionStateRequestHeader header = (CheckTransactionStateRequestHeader) command.readCustomHeader();\n                        MessageExt messageExt = MessageDecoder.decode(ByteBuffer.wrap(command.getBody()), true, false, false);\n                        RelayData<TransactionData, Void> relayData = this.proxyRelayService.processCheckTransactionState(context, command, header, messageExt);\n                        processFuture = this.processCheckTransaction(header, messageExt, relayData.getProcessResult(), relayData.getRelayFuture());\n                        break;\n                    }\n                    case RequestCode.GET_CONSUMER_RUNNING_INFO: {\n                        GetConsumerRunningInfoRequestHeader header = (GetConsumerRunningInfoRequestHeader) command.readCustomHeader();\n                        CompletableFuture<ProxyRelayResult<ConsumerRunningInfo>> relayFuture = this.proxyRelayService.processGetConsumerRunningInfo(context, command, header);\n                        processFuture = this.processGetConsumerRunningInfo(command, header, relayFuture);\n                        break;\n                    }\n                    case RequestCode.CONSUME_MESSAGE_DIRECTLY: {\n                        ConsumeMessageDirectlyResultRequestHeader header = (ConsumeMessageDirectlyResultRequestHeader) command.readCustomHeader();\n                        MessageExt messageExt = MessageDecoder.decode(ByteBuffer.wrap(command.getBody()), true, false, false);\n                        processFuture = this.processConsumeMessageDirectly(command, header, messageExt,\n                            this.proxyRelayService.processConsumeMessageDirectly(context, command, header));\n                        break;\n                    }\n                    case RequestCode.NOTIFY_UNSUBSCRIBE_LITE: {\n                        NotifyUnsubscribeLiteRequestHeader header = (NotifyUnsubscribeLiteRequestHeader) command.readCustomHeader();\n                        processFuture = this.processNotifyUnsubscribeLite(header);\n                        break;\n                    }\n                    default:\n                        break;\n                }\n            } else {\n                processFuture = processOtherMessage(msg);\n            }\n        } catch (Throwable t) {\n            log.error(\"process failed. msg:{}\", msg, t);\n            processFuture.completeExceptionally(t);\n        }\n\n        DefaultChannelPromise promise = new DefaultChannelPromise(this, GlobalEventExecutor.INSTANCE);\n        processFuture.thenAccept(ignore -> promise.setSuccess())\n            .exceptionally(t -> {\n                promise.setFailure(t);\n                return null;\n            });\n        return promise;\n    }\n\n    protected abstract CompletableFuture<Void> processOtherMessage(Object msg);\n\n    protected abstract CompletableFuture<Void> processCheckTransaction(\n        CheckTransactionStateRequestHeader header,\n        MessageExt messageExt,\n        TransactionData transactionData,\n        CompletableFuture<ProxyRelayResult<Void>> responseFuture);\n\n    protected abstract CompletableFuture<Void> processNotifyUnsubscribeLite(NotifyUnsubscribeLiteRequestHeader header);\n\n    protected abstract CompletableFuture<Void> processGetConsumerRunningInfo(\n        RemotingCommand command,\n        GetConsumerRunningInfoRequestHeader header,\n        CompletableFuture<ProxyRelayResult<ConsumerRunningInfo>> responseFuture);\n\n    protected abstract CompletableFuture<Void> processConsumeMessageDirectly(\n        RemotingCommand command,\n        ConsumeMessageDirectlyResultRequestHeader header,\n        MessageExt messageExt,\n        CompletableFuture<ProxyRelayResult<ConsumeMessageDirectlyResult>> responseFuture);\n\n    @Override\n    public ChannelConfig config() {\n        return null;\n    }\n\n    @Override\n    public ChannelMetadata metadata() {\n        return null;\n    }\n\n    @Override\n    protected AbstractUnsafe newUnsafe() {\n        return null;\n    }\n\n    @Override\n    protected boolean isCompatible(EventLoop loop) {\n        return false;\n    }\n\n    @Override\n    protected void doBind(SocketAddress localAddress) throws Exception {\n\n    }\n\n    @Override\n    protected void doDisconnect() throws Exception {\n\n    }\n\n    @Override\n    protected void doClose() throws Exception {\n\n    }\n\n    @Override\n    protected void doBeginRead() throws Exception {\n\n    }\n\n    @Override\n    protected void doWrite(ChannelOutboundBuffer in) throws Exception {\n\n    }\n\n    @Override\n    protected SocketAddress localAddress0() {\n        return this.localSocketAddress;\n    }\n\n    @Override\n    protected SocketAddress remoteAddress0() {\n        return this.remoteSocketAddress;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ProxyRelayResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.relay;\n\npublic class ProxyRelayResult<T> {\n    private int code;\n    private String remark;\n    private T result;\n\n    public ProxyRelayResult(int code, String remark, T result) {\n        this.code = code;\n        this.remark = remark;\n        this.result = result;\n    }\n\n    public int getCode() {\n        return code;\n    }\n\n    public void setCode(int code) {\n        this.code = code;\n    }\n\n    public String getRemark() {\n        return remark;\n    }\n\n    public void setRemark(String remark) {\n        this.remark = remark;\n    }\n\n    public T getResult() {\n        return result;\n    }\n\n    public void setResult(T result) {\n        this.result = result;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ProxyRelayService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.relay;\n\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.service.transaction.TransactionData;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader;\n\npublic interface ProxyRelayService {\n\n    CompletableFuture<ProxyRelayResult<ConsumerRunningInfo>> processGetConsumerRunningInfo(\n        ProxyContext context,\n        RemotingCommand command,\n        GetConsumerRunningInfoRequestHeader header\n    );\n\n    CompletableFuture<ProxyRelayResult<ConsumeMessageDirectlyResult>> processConsumeMessageDirectly(\n        ProxyContext context,\n        RemotingCommand command,\n        ConsumeMessageDirectlyResultRequestHeader header\n    );\n\n    RelayData<TransactionData, Void> processCheckTransactionState(\n        ProxyContext context,\n        RemotingCommand command,\n        CheckTransactionStateRequestHeader header,\n        MessageExt messageExt\n    );\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/RelayData.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.relay;\n\nimport java.util.concurrent.CompletableFuture;\n\npublic class RelayData<T, R> {\n    private T processResult;\n    private CompletableFuture<ProxyRelayResult<R>> relayFuture;\n\n    public RelayData(T processResult, CompletableFuture<ProxyRelayResult<R>> relayFuture) {\n        this.processResult = processResult;\n        this.relayFuture = relayFuture;\n    }\n\n    public CompletableFuture<ProxyRelayResult<R>> getRelayFuture() {\n        return relayFuture;\n    }\n\n    public void setRelayFuture(\n        CompletableFuture<ProxyRelayResult<R>> relayFuture) {\n        this.relayFuture = relayFuture;\n    }\n\n    public T getProcessResult() {\n        return processResult;\n    }\n\n    public void setProcessResult(T processResult) {\n        this.processResult = processResult;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/route/AddressableMessageQueue.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.route;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\npublic class AddressableMessageQueue extends MessageQueue {\n    private final String brokerAddr;\n\n    public AddressableMessageQueue(MessageQueue messageQueue, String brokerAddr) {\n        super(messageQueue);\n        this.brokerAddr = brokerAddr;\n    }\n\n    public String getBrokerAddr() {\n        return brokerAddr;\n    }\n\n    public MessageQueue getMessageQueue() {\n        return new MessageQueue(getTopic(), getBrokerName(), getQueueId());\n    }\n\n    @Override\n    public int hashCode() {\n        return super.hashCode();\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) {\n            return true;\n        }\n        if (!(o instanceof AddressableMessageQueue)) {\n            return false;\n        }\n        return super.equals(o);\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"messageQueue\", super.toString())\n            .add(\"brokerAddr\", brokerAddr)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ClusterTopicRouteService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.route;\n\nimport java.util.List;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.proxy.common.Address;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\n\npublic class ClusterTopicRouteService extends TopicRouteService {\n\n    public ClusterTopicRouteService(MQClientAPIFactory mqClientAPIFactory) {\n        super(mqClientAPIFactory);\n    }\n\n    @Override\n    public MessageQueueView getCurrentMessageQueueView(ProxyContext ctx, String topicName) throws Exception {\n        return getAllMessageQueueView(ctx, topicName);\n    }\n\n    @Override\n    public ProxyTopicRouteData getTopicRouteForProxy(ProxyContext ctx, List<Address> requestHostAndPortList,\n        String topicName) throws Exception {\n        TopicRouteData topicRouteData = getAllMessageQueueView(ctx, topicName).getTopicRouteData();\n        return new ProxyTopicRouteData(topicRouteData, requestHostAndPortList);\n    }\n\n    @Override\n    public String getBrokerAddr(ProxyContext ctx, String brokerName) throws Exception {\n        TopicRouteWrapper topicRouteWrapper = getAllMessageQueueView(ctx, brokerName).getTopicRouteWrapper();\n        return topicRouteWrapper.getMasterAddr(brokerName);\n    }\n\n    @Override\n    public AddressableMessageQueue buildAddressableMessageQueue(ProxyContext ctx, MessageQueue messageQueue) throws Exception {\n        String brokerAddress = getBrokerAddr(ctx, messageQueue.getBrokerName());\n        return new AddressableMessageQueue(messageQueue, brokerAddress);\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/route/DefaultMessageQueuePriorityProvider.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.route;\n\npublic class DefaultMessageQueuePriorityProvider implements MessageQueuePriorityProvider<AddressableMessageQueue> {\n    @Override\n    public int priorityOf(AddressableMessageQueue queue) {\n        return 0;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.route;\n\nimport com.google.common.collect.Lists;\nimport java.util.HashMap;\nimport java.util.List;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.proxy.common.Address;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\n\npublic class LocalTopicRouteService extends TopicRouteService {\n\n    private final BrokerController brokerController;\n    private final List<BrokerData> brokerDataList;\n    private final int grpcPort;\n\n    public LocalTopicRouteService(BrokerController brokerController, MQClientAPIFactory mqClientAPIFactory) {\n        super(mqClientAPIFactory);\n        this.brokerController = brokerController;\n        BrokerConfig brokerConfig = this.brokerController.getBrokerConfig();\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(MixAll.MASTER_ID, this.brokerController.getBrokerAddr());\n        this.brokerDataList = Lists.newArrayList(\n            new BrokerData(brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName(), brokerAddrs)\n        );\n        this.grpcPort = ConfigurationManager.getProxyConfig().getGrpcServerPort();\n    }\n\n    @Override\n    public MessageQueueView getCurrentMessageQueueView(ProxyContext ctx, String topic) throws Exception {\n        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().getTopicConfigTable().get(topic);\n        return new MessageQueueView(topic, toTopicRouteData(topicConfig), null);\n    }\n\n    @Override\n    public ProxyTopicRouteData getTopicRouteForProxy(ProxyContext ctx, List<Address> requestHostAndPortList,\n        String topicName) throws Exception {\n        MessageQueueView messageQueueView = getAllMessageQueueView(ctx, topicName);\n        TopicRouteData topicRouteData = messageQueueView.getTopicRouteData();\n        return new ProxyTopicRouteData(topicRouteData, grpcPort);\n    }\n\n    @Override\n    public String getBrokerAddr(ProxyContext ctx, String brokerName) throws Exception {\n        return this.brokerController.getBrokerAddr();\n    }\n\n    @Override\n    public AddressableMessageQueue buildAddressableMessageQueue(ProxyContext ctx, MessageQueue messageQueue) throws Exception {\n        String brokerAddress = getBrokerAddr(ctx, messageQueue.getBrokerName());\n        return new AddressableMessageQueue(messageQueue, brokerAddress);\n    }\n\n    protected TopicRouteData toTopicRouteData(TopicConfig topicConfig) {\n        TopicRouteData topicRouteData = new TopicRouteData();\n        topicRouteData.setBrokerDatas(brokerDataList);\n\n        QueueData queueData = new QueueData();\n        queueData.setPerm(topicConfig.getPerm());\n        queueData.setReadQueueNums(topicConfig.getReadQueueNums());\n        queueData.setWriteQueueNums(topicConfig.getWriteQueueNums());\n        queueData.setTopicSysFlag(topicConfig.getTopicSysFlag());\n        queueData.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());\n        topicRouteData.setQueueDatas(Lists.newArrayList(queueData));\n\n        return topicRouteData;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueuePenalizer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.route;\n\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.commons.lang3.tuple.Pair;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\n@FunctionalInterface\npublic interface MessageQueuePenalizer<Q extends MessageQueue> {\n\n    /**\n     * Returns the penalty value for the given MessageQueue; lower is better.\n     */\n    int penaltyOf(Q messageQueue);\n\n    /**\n     * Aggregates penalties from multiple penalizers for the same MessageQueue (by summing them up).\n     */\n    static <Q extends MessageQueue> int evaluatePenalty(Q messageQueue, List<MessageQueuePenalizer<Q>> penalizers) {\n        Objects.requireNonNull(messageQueue, \"messageQueue\");\n        if (penalizers == null || penalizers.isEmpty()) {\n            return 0;\n        }\n        int sum = 0;\n        for (MessageQueuePenalizer<Q> p : penalizers) {\n            sum += p.penaltyOf(messageQueue);\n        }\n        return sum;\n    }\n\n    /**\n     * Selects the queue with the lowest evaluated penalty from the given queue list.\n     *\n     * <p>The method iterates through all queues exactly once, but starts from a rotating index\n     * derived from {@code startIndex} (round-robin) to avoid always scanning from position 0 .</p>\n     *\n     * <p>For each queue, it computes a penalty via {@link #evaluatePenalty} using\n     * the provided {@code penalizers}. The queue with the smallest penalty is selected.</p>\n     *\n     * <p>Short-circuit rule: if any queue has a {@code penalty<= 0}, it is returned immediately,\n     * since no better result than 0 is expected.</p>\n     *\n     * @param queues candidate queues to select from\n     * @param penalizers penalty evaluators applied to each queue\n     * @param startIndex atomic counter used to determine the rotating start position (round-robin)\n     * @param <Q> queue type\n     * @return a {@code Pair} of (selected queue, penalty), or {@code null} if {@code queues} is null/empty\n     */\n    static <Q extends MessageQueue> Pair<Q, Integer> selectLeastPenalty(List<Q> queues,\n        List<MessageQueuePenalizer<Q>> penalizers, AtomicInteger startIndex) {\n        if (queues == null || queues.isEmpty()) {\n            return null;\n        }\n        Q bestQueue = null;\n        int bestPenalty = Integer.MAX_VALUE;\n\n        for (int i = 0; i < queues.size(); i++) {\n            int index = Math.floorMod(startIndex.getAndIncrement(), queues.size());\n            Q messageQueue = queues.get(index);\n            int penalty = evaluatePenalty(messageQueue, penalizers);\n\n            // Short-circuit: cannot do better than 0\n            if (penalty <= 0) {\n                return Pair.of(messageQueue, penalty);\n            }\n\n            if (penalty < bestPenalty) {\n                bestPenalty = penalty;\n                bestQueue = messageQueue;\n            }\n        }\n        return Pair.of(bestQueue,  bestPenalty);\n    }\n\n    /**\n     * Selects a queue with the lowest computed penalty from multiple priority groups.\n     *\n     * <p>The input {@code queuesWithPriority} is a list of queue groups ordered by priority.\n     * For each priority group, this method delegates to {@link #selectLeastPenalty} to pick the best queue\n     * within that group and obtain its penalty.</p>\n     *\n     * <p>Short-circuit rule: if any priority group yields a queue whose {@code penalty <= 0},\n     * that result is returned immediately.</p>\n     *\n     * <p>Otherwise, it returns the queue with the smallest positive penalty among all groups.\n     * If multiple groups produce the same minimum penalty, the first encountered one wins.</p>\n     *\n     * @param queuesWithPriority priority-ordered groups of queues; each inner list represents one priority level\n     * @param penalizers penalty calculators used by {@code selectLeastPenalty} to score queues\n     * @param startIndex round-robin start index forwarded to {@code selectLeastPenalty} to reduce contention/hotspots\n     * @param <Q> queue type\n     * @return a {@code Pair} of (selected queue, penalty), or {@code null} if {@code queuesWithPriority} is null/empty\n     */\n    static <Q extends MessageQueue> Pair<Q, Integer> selectLeastPenaltyWithPriority(List<List<Q>> queuesWithPriority,\n        List<MessageQueuePenalizer<Q>> penalizers, AtomicInteger startIndex) {\n        if (queuesWithPriority == null || queuesWithPriority.isEmpty()) {\n            return null;\n        }\n        if (queuesWithPriority.size() == 1) {\n            return selectLeastPenalty(queuesWithPriority.get(0), penalizers, startIndex);\n        }\n        Q bestQueue = null;\n        int bestPenalty = Integer.MAX_VALUE;\n        for (List<Q> queues : queuesWithPriority) {\n            Pair<Q, Integer> queueAndPenalty = selectLeastPenalty(queues, penalizers, startIndex);\n            int penalty =  queueAndPenalty.getRight();\n            if (queueAndPenalty.getRight() <= 0) {\n                return queueAndPenalty;\n            }\n            if (penalty < bestPenalty) {\n                bestPenalty = penalty;\n                bestQueue = queueAndPenalty.getLeft();\n            }\n        }\n        return Pair.of(bestQueue,  bestPenalty);\n    }\n}"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueuePriorityProvider.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.route;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.TreeMap;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\n/**\n * A functional interface for providing priority values for message queues.\n * This interface allows custom priority determination logic to be applied to message queues,\n * enabling queue selection and routing based on priority levels.\n * <p>\n * The priority value follows the convention that smaller numeric values indicate higher priority.\n * For example, priority 0 is higher than priority 1.\n * </p>\n *\n * @param <Q> the type of message queue, must extend {@link MessageQueue}\n */\n@FunctionalInterface\npublic interface MessageQueuePriorityProvider<Q extends MessageQueue> {\n\n    /**\n     * Determines the priority value of the given message queue.\n     * <p>\n     * Smaller values indicate higher priority. For example:\n     * <ul>\n     *   <li>Priority 0: Highest priority</li>\n     *   <li>Priority 1: Medium priority</li>\n     *   <li>Priority 2: Lower priority</li>\n     * </ul>\n     * </p>\n     *\n     * @param q the message queue to evaluate\n     * @return the priority value, where smaller values indicate higher priority\n     */\n    int priorityOf(Q q);\n\n    /**\n     * Groups message queues by their priority levels and returns them in priority order.\n     * <p>\n     * This static utility method takes a list of message queues and a priority provider,\n     * then organizes the queues into groups based on their priority values.\n     * The returned list is ordered from highest priority to lowest priority.\n     * </p>\n     *\n     * @param <Q>      the type of message queue, must extend {@link MessageQueue}\n     * @param queues   the list of message queues to group by priority, can be null or empty\n     * @param provider the priority provider to determine the priority of each queue\n     * @return a list of lists, where each inner list contains queues of the same priority level,\n     *         ordered from highest priority (smallest value) to lowest priority (largest value).\n     *         Returns an empty list if the input queues are null or empty.\n     */\n    static <Q extends MessageQueue> List<List<Q>> buildPriorityGroups(List<Q> queues, MessageQueuePriorityProvider<Q> provider) {\n        if (queues == null || queues.isEmpty()) {\n            return Collections.emptyList();\n        }\n\n        Map<Integer, List<Q>> buckets = new TreeMap<>();\n        for (Q q : queues) {\n            int p = provider.priorityOf(q);\n            buckets.computeIfAbsent(p, k -> new ArrayList<>()).add(q);\n        }\n        return new ArrayList<>(buckets.values());\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.route;\n\nimport com.google.common.base.MoreObjects;\nimport com.google.common.math.IntMath;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Random;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.stream.Collectors;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.commons.lang3.tuple.Pair;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\n\nimport static org.apache.rocketmq.proxy.service.route.MessageQueuePenalizer.selectLeastPenaltyWithPriority;\nimport static org.apache.rocketmq.proxy.service.route.MessageQueuePriorityProvider.buildPriorityGroups;\n\npublic class MessageQueueSelector {\n    private static final int BROKER_ACTING_QUEUE_ID = -1;\n\n    // multiple queues for brokers with queueId : normal\n    private final List<AddressableMessageQueue> queues = new ArrayList<>();\n    // one queue for brokers with queueId : -1\n    private final List<AddressableMessageQueue> brokerActingQueues = new ArrayList<>();\n    private final Map<String, AddressableMessageQueue> brokerNameQueueMap = new ConcurrentHashMap<>();\n    private final AtomicInteger queueIndex;\n    private final AtomicInteger brokerIndex;\n    private final List<MessageQueuePenalizer<AddressableMessageQueue>> penalizers = new ArrayList<>();\n\n    // ordered by priority asc (smaller => higher priority)\n    private final List<List<AddressableMessageQueue>> queuesWithPriority;\n    private final List<List<AddressableMessageQueue>> brokerActingQueuesWithPriority;\n\n    public MessageQueueSelector(TopicRouteWrapper topicRouteWrapper, boolean read) {\n        this(topicRouteWrapper, read, null);\n    }\n\n    public MessageQueueSelector(TopicRouteWrapper topicRouteWrapper, boolean read,\n        MessageQueuePriorityProvider<AddressableMessageQueue> priorityProvider) {\n        if (read) {\n            this.queues.addAll(buildRead(topicRouteWrapper));\n        } else {\n            this.queues.addAll(buildWrite(topicRouteWrapper));\n        }\n        buildBrokerActingQueues(topicRouteWrapper.getTopicName(), this.queues);\n        Random random = new Random();\n        this.queueIndex = new AtomicInteger(random.nextInt());\n        this.brokerIndex = new AtomicInteger(random.nextInt());\n\n        if (priorityProvider == null) {\n            priorityProvider = new DefaultMessageQueuePriorityProvider();\n        }\n        this.queuesWithPriority = buildPriorityGroups(queues, priorityProvider);\n        this.brokerActingQueuesWithPriority = buildPriorityGroups(brokerActingQueues, priorityProvider);\n    }\n\n    private static List<AddressableMessageQueue> buildRead(TopicRouteWrapper topicRoute) {\n        Set<AddressableMessageQueue> queueSet = new HashSet<>();\n        List<QueueData> qds = topicRoute.getQueueDatas();\n        if (qds == null) {\n            return new ArrayList<>();\n        }\n\n        for (QueueData qd : qds) {\n            if (PermName.isReadable(qd.getPerm())) {\n                String brokerAddr = topicRoute.getMasterAddrPrefer(qd.getBrokerName());\n                if (brokerAddr == null) {\n                    continue;\n                }\n\n                for (int i = 0; i < qd.getReadQueueNums(); i++) {\n                    AddressableMessageQueue mq = new AddressableMessageQueue(\n                        new MessageQueue(topicRoute.getTopicName(), qd.getBrokerName(), i),\n                        brokerAddr);\n                    queueSet.add(mq);\n                }\n            }\n        }\n\n        return queueSet.stream().sorted().collect(Collectors.toList());\n    }\n\n    private static List<AddressableMessageQueue> buildWrite(TopicRouteWrapper topicRoute) {\n        Set<AddressableMessageQueue> queueSet = new HashSet<>();\n        // order topic route.\n        if (StringUtils.isNotBlank(topicRoute.getOrderTopicConf())) {\n            String[] brokers = topicRoute.getOrderTopicConf().split(\";\");\n            for (String broker : brokers) {\n                String[] item = broker.split(\":\");\n                String brokerName = item[0];\n                String brokerAddr = topicRoute.getMasterAddr(brokerName);\n                if (brokerAddr == null) {\n                    continue;\n                }\n\n                int nums = Integer.parseInt(item[1]);\n                for (int i = 0; i < nums; i++) {\n                    AddressableMessageQueue mq = new AddressableMessageQueue(\n                        new MessageQueue(topicRoute.getTopicName(), brokerName, i),\n                        brokerAddr);\n                    queueSet.add(mq);\n                }\n            }\n        } else {\n            List<QueueData> qds = topicRoute.getQueueDatas();\n            if (qds == null) {\n                return new ArrayList<>();\n            }\n\n            for (QueueData qd : qds) {\n                if (PermName.isWriteable(qd.getPerm())) {\n                    String brokerAddr = topicRoute.getMasterAddr(qd.getBrokerName());\n                    if (brokerAddr == null) {\n                        continue;\n                    }\n\n                    for (int i = 0; i < qd.getWriteQueueNums(); i++) {\n                        AddressableMessageQueue mq = new AddressableMessageQueue(\n                            new MessageQueue(topicRoute.getTopicName(), qd.getBrokerName(), i),\n                            brokerAddr);\n                        queueSet.add(mq);\n                    }\n                }\n            }\n        }\n\n        return queueSet.stream().sorted().collect(Collectors.toList());\n    }\n\n    private void buildBrokerActingQueues(String topic, List<AddressableMessageQueue> normalQueues) {\n        for (AddressableMessageQueue mq : normalQueues) {\n            AddressableMessageQueue brokerActingQueue = new AddressableMessageQueue(\n                new MessageQueue(topic, mq.getBrokerName(), BROKER_ACTING_QUEUE_ID),\n                mq.getBrokerAddr());\n\n            if (!brokerActingQueues.contains(brokerActingQueue)) {\n                brokerActingQueues.add(brokerActingQueue);\n                brokerNameQueueMap.put(brokerActingQueue.getBrokerName(), brokerActingQueue);\n            }\n        }\n\n        Collections.sort(brokerActingQueues);\n    }\n\n    public AddressableMessageQueue getQueueByBrokerName(String brokerName) {\n        return this.brokerNameQueueMap.get(brokerName);\n    }\n\n    public AddressableMessageQueue selectOne(boolean onlyBroker) {\n        int nextIndex = onlyBroker ? brokerIndex.getAndIncrement() : queueIndex.getAndIncrement();\n        return selectOneByIndex(nextIndex, onlyBroker);\n    }\n\n    public AddressableMessageQueue selectOneByPipeline(boolean onlyBroker) {\n        if (CollectionUtils.isNotEmpty(penalizers)) {\n            Pair<AddressableMessageQueue, Integer> queueAndPenalty;\n            if (onlyBroker) {\n                queueAndPenalty = selectLeastPenaltyWithPriority(brokerActingQueuesWithPriority, penalizers, brokerIndex);\n            } else {\n                queueAndPenalty = selectLeastPenaltyWithPriority(queuesWithPriority, penalizers, queueIndex);\n            }\n            if (queueAndPenalty != null && queueAndPenalty.getLeft() != null) {\n                return queueAndPenalty.getLeft();\n            }\n        }\n\n        // SendLatency is not enabled, or no queue is selected, then select by index.\n        return selectOne(onlyBroker);\n    }\n\n    public AddressableMessageQueue selectNextOne(AddressableMessageQueue last) {\n        boolean onlyBroker = last.getQueueId() < 0;\n        AddressableMessageQueue newOne = last;\n        int count = onlyBroker ? brokerActingQueues.size() : queues.size();\n\n        for (int i = 0; i < count; i++) {\n            newOne = selectOne(onlyBroker);\n            if (!newOne.getBrokerName().equals(last.getBrokerName()) || newOne.getQueueId() != last.getQueueId()) {\n                break;\n            }\n        }\n        return newOne;\n    }\n\n    public AddressableMessageQueue selectOneByIndex(int index, boolean onlyBroker) {\n        if (onlyBroker) {\n            if (brokerActingQueues.isEmpty()) {\n                return null;\n            }\n            return brokerActingQueues.get(IntMath.mod(index, brokerActingQueues.size()));\n        }\n\n        if (queues.isEmpty()) {\n            return null;\n        }\n        return queues.get(IntMath.mod(index, queues.size()));\n    }\n\n    public List<AddressableMessageQueue> getQueues() {\n        return queues;\n    }\n\n    public List<AddressableMessageQueue> getBrokerActingQueues() {\n        return brokerActingQueues;\n    }\n\n    public void addPenalizer(MessageQueuePenalizer<AddressableMessageQueue> penalizer) {\n        if (penalizer != null) {\n            this.penalizers.add(penalizer);\n        }\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) {\n            return true;\n        }\n        if (!(o instanceof MessageQueueSelector)) {\n            return false;\n        }\n        MessageQueueSelector queue = (MessageQueueSelector) o;\n        return Objects.equals(queues, queue.queues) &&\n            Objects.equals(brokerActingQueues, queue.brokerActingQueues);\n    }\n\n    @Override\n    public int hashCode() {\n        return Objects.hash(queues, brokerActingQueues);\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"queues\", queues)\n            .add(\"brokerActingQueues\", brokerActingQueues)\n            .add(\"brokerNameQueueMap\", brokerNameQueueMap)\n            .add(\"queueIndex\", queueIndex)\n            .add(\"brokerIndex\", brokerIndex)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.route;\n\nimport com.google.common.base.MoreObjects;\nimport java.util.List;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\n\npublic class MessageQueueView {\n    public static final MessageQueueView WRAPPED_EMPTY_QUEUE = new MessageQueueView(\"\", new TopicRouteData(), null);\n\n    private final MessageQueueSelector readSelector;\n    private final MessageQueueSelector writeSelector;\n    private final TopicRouteWrapper topicRouteWrapper;\n\n\n    public MessageQueueView(String topic, TopicRouteData topicRouteData, List<MessageQueuePenalizer<AddressableMessageQueue>> penalizer) {\n        this(topic, topicRouteData, penalizer, null);\n    }\n\n    public MessageQueueView(String topic, TopicRouteData topicRouteData, List<MessageQueuePenalizer<AddressableMessageQueue>> penalizer,\n        MessageQueuePriorityProvider<AddressableMessageQueue> priorityProvider) {\n        this.topicRouteWrapper = new TopicRouteWrapper(topicRouteData, topic);\n\n        this.readSelector = new MessageQueueSelector(topicRouteWrapper, true, priorityProvider);\n        this.writeSelector = new MessageQueueSelector(topicRouteWrapper, false, priorityProvider);\n\n        if (CollectionUtils.isNotEmpty(penalizer)) {\n            for (MessageQueuePenalizer<AddressableMessageQueue> p : penalizer) {\n                this.readSelector.addPenalizer(p);\n                this.writeSelector.addPenalizer(p);\n            }\n        }\n    }\n\n    public TopicRouteData getTopicRouteData() {\n        return topicRouteWrapper.getTopicRouteData();\n    }\n\n    public TopicRouteWrapper getTopicRouteWrapper() {\n        return topicRouteWrapper;\n    }\n\n    public String getTopicName() {\n        return topicRouteWrapper.getTopicName();\n    }\n\n    public boolean isEmptyCachedQueue() {\n        return this == WRAPPED_EMPTY_QUEUE;\n    }\n\n    public MessageQueueSelector getReadSelector() {\n        return readSelector;\n    }\n\n    public MessageQueueSelector getWriteSelector() {\n        return writeSelector;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"readSelector\", readSelector)\n            .add(\"writeSelector\", writeSelector)\n            .add(\"topicRouteWrapper\", topicRouteWrapper)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ProxyTopicRouteData.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.route;\n\nimport com.google.common.collect.Lists;\nimport com.google.common.net.HostAndPort;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.stream.Collectors;\nimport org.apache.rocketmq.proxy.common.Address;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\n\npublic class ProxyTopicRouteData {\n    public ProxyTopicRouteData() {\n    }\n\n    public ProxyTopicRouteData(TopicRouteData topicRouteData) {\n        this.queueDatas = topicRouteData.getQueueDatas();\n        this.brokerDatas = new ArrayList<>();\n\n        for (BrokerData brokerData : topicRouteData.getBrokerDatas()) {\n            ProxyTopicRouteData.ProxyBrokerData proxyBrokerData = new ProxyTopicRouteData.ProxyBrokerData();\n            proxyBrokerData.setCluster(brokerData.getCluster());\n            proxyBrokerData.setBrokerName(brokerData.getBrokerName());\n            brokerData.getBrokerAddrs().forEach((brokerId, brokerAddr) -> {\n                HostAndPort brokerHostAndPort = HostAndPort.fromString(brokerAddr);\n\n                proxyBrokerData.getBrokerAddrs().put(brokerId, Lists.newArrayList(new Address(brokerHostAndPort)));\n            });\n            this.brokerDatas.add(proxyBrokerData);\n        }\n    }\n\n    public ProxyTopicRouteData(TopicRouteData topicRouteData, int port) {\n        this.queueDatas = topicRouteData.getQueueDatas();\n        this.brokerDatas = new ArrayList<>();\n\n        for (BrokerData brokerData : topicRouteData.getBrokerDatas()) {\n            ProxyTopicRouteData.ProxyBrokerData proxyBrokerData = new ProxyTopicRouteData.ProxyBrokerData();\n            proxyBrokerData.setCluster(brokerData.getCluster());\n            proxyBrokerData.setBrokerName(brokerData.getBrokerName());\n            brokerData.getBrokerAddrs().forEach((brokerId, brokerAddr) -> {\n                HostAndPort brokerHostAndPort = HostAndPort.fromString(brokerAddr);\n                HostAndPort proxyHostAndPort = HostAndPort.fromParts(brokerHostAndPort.getHost(), port);\n\n                proxyBrokerData.getBrokerAddrs().put(brokerId, Lists.newArrayList(new Address(proxyHostAndPort)));\n            });\n            this.brokerDatas.add(proxyBrokerData);\n        }\n    }\n\n    public ProxyTopicRouteData(TopicRouteData topicRouteData, List<Address> requestHostAndPortList) {\n        this.queueDatas = topicRouteData.getQueueDatas();\n        this.brokerDatas = new ArrayList<>();\n\n        for (BrokerData brokerData : topicRouteData.getBrokerDatas()) {\n            ProxyTopicRouteData.ProxyBrokerData proxyBrokerData = new ProxyTopicRouteData.ProxyBrokerData();\n            proxyBrokerData.setCluster(brokerData.getCluster());\n            proxyBrokerData.setBrokerName(brokerData.getBrokerName());\n            for (Long brokerId : brokerData.getBrokerAddrs().keySet()) {\n                proxyBrokerData.getBrokerAddrs().put(brokerId, requestHostAndPortList);\n            }\n            this.brokerDatas.add(proxyBrokerData);\n        }\n    }\n\n    public static class ProxyBrokerData {\n        private String cluster;\n        private String brokerName;\n        private Map<Long/* brokerId */, List<Address>/* broker address */> brokerAddrs = new HashMap<>();\n\n        public String getCluster() {\n            return cluster;\n        }\n\n        public void setCluster(String cluster) {\n            this.cluster = cluster;\n        }\n\n        public String getBrokerName() {\n            return brokerName;\n        }\n\n        public void setBrokerName(String brokerName) {\n            this.brokerName = brokerName;\n        }\n\n        public Map<Long, List<Address>> getBrokerAddrs() {\n            return brokerAddrs;\n        }\n\n        public void setBrokerAddrs(Map<Long, List<Address>> brokerAddrs) {\n            this.brokerAddrs = brokerAddrs;\n        }\n\n        public BrokerData buildBrokerData() {\n            BrokerData brokerData = new BrokerData();\n            brokerData.setCluster(cluster);\n            brokerData.setBrokerName(brokerName);\n            HashMap<Long, String> buildBrokerAddress = new HashMap<>();\n            brokerAddrs.forEach((k, v) -> {\n                if (!v.isEmpty()) {\n                    buildBrokerAddress.put(k, v.get(0).getHostAndPort().toString());\n                }\n            });\n            brokerData.setBrokerAddrs(buildBrokerAddress);\n            return brokerData;\n        }\n    }\n\n    private List<QueueData> queueDatas = new ArrayList<>();\n    private List<ProxyBrokerData> brokerDatas = new ArrayList<>();\n\n    public List<QueueData> getQueueDatas() {\n        return queueDatas;\n    }\n\n    public void setQueueDatas(List<QueueData> queueDatas) {\n        this.queueDatas = queueDatas;\n    }\n\n    public List<ProxyBrokerData> getBrokerDatas() {\n        return brokerDatas;\n    }\n\n    public void setBrokerDatas(List<ProxyBrokerData> brokerDatas) {\n        this.brokerDatas = brokerDatas;\n    }\n\n    public TopicRouteData buildTopicRouteData() {\n        TopicRouteData topicRouteData = new TopicRouteData();\n        topicRouteData.setQueueDatas(queueDatas);\n        topicRouteData.setBrokerDatas(brokerDatas.stream()\n            .map(ProxyBrokerData::buildBrokerData)\n            .collect(Collectors.toList()));\n        return topicRouteData;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteHelper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.route;\n\nimport org.apache.rocketmq.client.common.ClientErrorCode;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\n\npublic class TopicRouteHelper {\n\n    public static boolean isTopicNotExistError(Throwable e) {\n        if (e instanceof MQBrokerException) {\n            if (((MQBrokerException) e).getResponseCode() == ResponseCode.TOPIC_NOT_EXIST) {\n                return true;\n            }\n        }\n\n        if (e instanceof MQClientException) {\n            int code = ((MQClientException) e).getResponseCode();\n            if (code == ResponseCode.TOPIC_NOT_EXIST || code == ClientErrorCode.NOT_FOUND_TOPIC_EXCEPTION) {\n                return true;\n            }\n\n            Throwable cause = e.getCause();\n            if (cause instanceof MQClientException) {\n                int causeCode = ((MQClientException) cause).getResponseCode();\n                return causeCode == ResponseCode.TOPIC_NOT_EXIST || causeCode == ClientErrorCode.NOT_FOUND_TOPIC_EXCEPTION;\n            }\n        }\n\n        return false;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.route;\n\nimport com.github.benmanes.caffeine.cache.CacheLoader;\nimport com.github.benmanes.caffeine.cache.Caffeine;\nimport com.github.benmanes.caffeine.cache.LoadingCache;\nimport com.google.common.annotations.VisibleForTesting;\nimport java.time.Duration;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Optional;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;\nimport org.apache.rocketmq.client.latency.MQFaultStrategy;\nimport org.apache.rocketmq.client.latency.Resolver;\nimport org.apache.rocketmq.client.latency.ServiceDetector;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.thread.ThreadPoolMonitor;\nimport org.apache.rocketmq.common.utils.AbstractStartAndShutdown;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.Address;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.checkerframework.checker.nullness.qual.NonNull;\nimport org.checkerframework.checker.nullness.qual.Nullable;\n\npublic abstract class TopicRouteService extends AbstractStartAndShutdown {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    private final MQFaultStrategy mqFaultStrategy;\n    protected final LoadingCache<String /* topicName */, MessageQueueView> topicCache;\n    protected final ThreadPoolExecutor cacheRefreshExecutor;\n    protected final List<MessageQueuePenalizer<AddressableMessageQueue>> penalizers = new ArrayList<>();\n    protected MessageQueuePriorityProvider<AddressableMessageQueue> priorityProvider = new DefaultMessageQueuePriorityProvider();\n\n    public TopicRouteService(MQClientAPIFactory mqClientAPIFactory) {\n        ProxyConfig config = ConfigurationManager.getProxyConfig();\n\n        this.cacheRefreshExecutor = ThreadPoolMonitor.createAndMonitor(\n            config.getTopicRouteServiceThreadPoolNums(),\n            config.getTopicRouteServiceThreadPoolNums(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            \"TopicRouteCacheRefresh\",\n            config.getTopicRouteServiceThreadPoolQueueCapacity()\n        );\n\n        this.topicCache = Caffeine.newBuilder().maximumSize(config.getTopicRouteServiceCacheMaxNum())\n            .expireAfterAccess(config.getTopicRouteServiceCacheExpiredSeconds(), TimeUnit.SECONDS)\n            .refreshAfterWrite(config.getTopicRouteServiceCacheRefreshSeconds(), TimeUnit.SECONDS)\n            .executor(cacheRefreshExecutor)\n            .build(new CacheLoader<String, MessageQueueView>() {\n                @Override\n                public @Nullable MessageQueueView load(String topic) throws Exception {\n                    try {\n                        TopicRouteData topicRouteData = mqClientAPIFactory.getClient().getTopicRouteInfoFromNameServer(topic, Duration.ofSeconds(3).toMillis());\n                        return buildMessageQueueView(topic, topicRouteData);\n                    } catch (Exception e) {\n                        if (TopicRouteHelper.isTopicNotExistError(e)) {\n                            return MessageQueueView.WRAPPED_EMPTY_QUEUE;\n                        }\n                        throw e;\n                    }\n                }\n\n                @Override\n                public @Nullable MessageQueueView reload(@NonNull String key,\n                    @NonNull MessageQueueView oldValue) throws Exception {\n                    try {\n                        return load(key);\n                    } catch (Exception e) {\n                        log.warn(String.format(\"reload topic route from namesrv. topic: %s\", key), e);\n                        return oldValue;\n                    }\n                }\n            });\n        ServiceDetector serviceDetector = new ServiceDetector() {\n            @Override\n            public boolean detect(String endpoint, long timeoutMillis) {\n                Optional<String> candidateTopic = pickTopic();\n                if (!candidateTopic.isPresent()) {\n                    return false;\n                }\n                try {\n                    GetMaxOffsetRequestHeader requestHeader = new GetMaxOffsetRequestHeader();\n                    requestHeader.setTopic(candidateTopic.get());\n                    requestHeader.setQueueId(0);\n                    Long maxOffset = mqClientAPIFactory.getClient().getMaxOffset(endpoint, requestHeader, timeoutMillis).get();\n                    return true;\n                } catch (Exception e) {\n                    return false;\n                }\n            }\n        };\n        mqFaultStrategy = new MQFaultStrategy(extractClientConfigFromProxyConfig(config), new Resolver() {\n            @Override\n            public String resolve(String name) {\n                try {\n                    String brokerAddr = getBrokerAddr(ProxyContext.createForInner(\"MQFaultStrategy\"), name);\n                    return brokerAddr;\n                } catch (Exception e) {\n                    return null;\n                }\n            }\n        }, serviceDetector);\n\n        this.penalizers.addAll(buildPenalizerByMQFaultStrategy(mqFaultStrategy));\n        this.init();\n    }\n\n    // pickup one topic in the topic cache\n    private Optional<String> pickTopic() {\n        if (topicCache.asMap().isEmpty()) {\n            return Optional.empty();\n        }\n        return Optional.of(topicCache.asMap().keySet().iterator().next());\n    }\n\n    protected void init() {\n        this.appendStartAndShutdown(this.mqFaultStrategy);\n    }\n\n    public ClientConfig extractClientConfigFromProxyConfig(ProxyConfig proxyConfig) {\n        ClientConfig tempClientConfig = new ClientConfig();\n        tempClientConfig.setSendLatencyEnable(proxyConfig.getSendLatencyEnable());\n        tempClientConfig.setStartDetectorEnable(proxyConfig.getStartDetectorEnable());\n        tempClientConfig.setDetectTimeout(proxyConfig.getDetectTimeout());\n        tempClientConfig.setDetectInterval(proxyConfig.getDetectInterval());\n        return tempClientConfig;\n    }\n\n    public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation,\n                                boolean reachable) {\n        checkSendFaultToleranceEnable();\n        this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation, reachable);\n    }\n\n    public void checkSendFaultToleranceEnable() {\n        boolean hotLatencySwitch = ConfigurationManager.getProxyConfig().isSendLatencyEnable();\n        boolean hotDetectorSwitch = ConfigurationManager.getProxyConfig().isStartDetectorEnable();\n        this.mqFaultStrategy.setSendLatencyFaultEnable(hotLatencySwitch);\n        this.mqFaultStrategy.setStartDetectorEnable(hotDetectorSwitch);\n    }\n\n    public MQFaultStrategy getMqFaultStrategy() {\n        return this.mqFaultStrategy;\n    }\n\n    public MessageQueueView getAllMessageQueueView(ProxyContext ctx, String topicName) throws Exception {\n        return getCacheMessageQueueWrapper(this.topicCache, topicName);\n    }\n\n    public abstract MessageQueueView getCurrentMessageQueueView(ProxyContext ctx, String topicName) throws Exception;\n\n    public abstract ProxyTopicRouteData getTopicRouteForProxy(ProxyContext ctx, List<Address> requestHostAndPortList,\n        String topicName) throws Exception;\n\n    public abstract String getBrokerAddr(ProxyContext ctx, String brokerName) throws Exception;\n\n    public abstract AddressableMessageQueue buildAddressableMessageQueue(ProxyContext ctx, MessageQueue messageQueue) throws Exception;\n\n    protected static MessageQueueView getCacheMessageQueueWrapper(LoadingCache<String, MessageQueueView> topicCache,\n        String key) throws Exception {\n        MessageQueueView res = topicCache.get(key);\n        if (res != null && res.isEmptyCachedQueue()) {\n            throw new MQClientException(ResponseCode.TOPIC_NOT_EXIST,\n                \"No topic route info in name server for the topic: \" + key);\n        }\n        return res;\n    }\n\n    protected static boolean isTopicRouteValid(TopicRouteData routeData) {\n        return routeData != null && routeData.getQueueDatas() != null && !routeData.getQueueDatas().isEmpty()\n            && routeData.getBrokerDatas() != null && !routeData.getBrokerDatas().isEmpty();\n    }\n\n    protected MessageQueueView buildMessageQueueView(String topic, TopicRouteData topicRouteData) {\n        if (isTopicRouteValid(topicRouteData)) {\n            MessageQueueView tmp = new MessageQueueView(topic, topicRouteData, this.penalizers, this.priorityProvider);\n            log.debug(\"load topic route from namesrv. topic: {}, queue: {}\", topic, tmp);\n            return tmp;\n        }\n        return MessageQueueView.WRAPPED_EMPTY_QUEUE;\n    }\n\n    public void setPriorityProvider(MessageQueuePriorityProvider<AddressableMessageQueue> priorityProvider) {\n        this.priorityProvider = priorityProvider;\n    }\n\n    public void addPenalizer(MessageQueuePenalizer<AddressableMessageQueue> penalizer) {\n        this.penalizers.add(penalizer);\n    }\n\n    @VisibleForTesting\n    public static List<MessageQueuePenalizer<AddressableMessageQueue>> buildPenalizerByMQFaultStrategy(MQFaultStrategy mqFaultStrategy) {\n        List<MessageQueuePenalizer<AddressableMessageQueue>> penalizers = new ArrayList<>();\n        penalizers.add(messageQueue -> {\n            if (!mqFaultStrategy.isSendLatencyFaultEnable() || mqFaultStrategy.getAvailableFilter().filter(messageQueue)) {\n                return 0;\n            }\n            return 10;\n        });\n        penalizers.add(messageQueue -> {\n            if (!mqFaultStrategy.isSendLatencyFaultEnable() || mqFaultStrategy.getReachableFilter().filter(messageQueue)) {\n                return 0;\n            }\n            return 100;\n        });\n        return penalizers;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteWrapper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.route;\n\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Optional;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\n\npublic class TopicRouteWrapper {\n\n    private final TopicRouteData topicRouteData;\n    private final String topicName;\n    private final Map<String/* brokerName */, BrokerData> brokerNameRouteData = new HashMap<>();\n\n    public TopicRouteWrapper(TopicRouteData topicRouteData, String topicName) {\n        this.topicRouteData = topicRouteData;\n        this.topicName = topicName;\n\n        if (this.topicRouteData.getBrokerDatas() != null) {\n            for (BrokerData brokerData : this.topicRouteData.getBrokerDatas()) {\n                this.brokerNameRouteData.put(brokerData.getBrokerName(), brokerData);\n            }\n        }\n    }\n\n    public String getMasterAddr(String brokerName) {\n        return this.brokerNameRouteData.get(brokerName).getBrokerAddrs().get(MixAll.MASTER_ID);\n    }\n\n    public String getMasterAddrPrefer(String brokerName) {\n        HashMap<Long, String> brokerAddr = brokerNameRouteData.get(brokerName).getBrokerAddrs();\n        String addr = brokerAddr.get(MixAll.MASTER_ID);\n        if (addr == null) {\n            Optional<Long> optional = brokerAddr.keySet().stream().findFirst();\n            return optional.map(brokerAddr::get).orElse(null);\n        }\n        return addr;\n    }\n\n    public String getTopicName() {\n        return topicName;\n    }\n\n    public TopicRouteData getTopicRouteData() {\n        return topicRouteData;\n    }\n\n    public List<QueueData> getQueueDatas() {\n        return this.topicRouteData.getQueueDatas();\n    }\n\n    public String getOrderTopicConf() {\n        return this.topicRouteData.getOrderTopicConf();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/AbstractSystemMessageSyncer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.sysmessage;\n\nimport com.alibaba.fastjson2.JSON;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.common.utils.StartAndShutdown;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.service.admin.AdminService;\nimport org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;\nimport org.apache.rocketmq.proxy.service.route.TopicRouteService;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\n\nimport java.nio.charset.StandardCharsets;\nimport java.time.Duration;\n\npublic abstract class AbstractSystemMessageSyncer implements StartAndShutdown, MessageListenerConcurrently {\n    protected static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n    protected final TopicRouteService topicRouteService;\n    protected final AdminService adminService;\n    protected final MQClientAPIFactory mqClientAPIFactory;\n    protected final RPCHook rpcHook;\n    protected DefaultMQPushConsumer defaultMQPushConsumer;\n\n    public AbstractSystemMessageSyncer(TopicRouteService topicRouteService, AdminService adminService, MQClientAPIFactory mqClientAPIFactory, RPCHook rpcHook) {\n        this.topicRouteService = topicRouteService;\n        this.adminService = adminService;\n        this.mqClientAPIFactory = mqClientAPIFactory;\n        this.rpcHook = rpcHook;\n    }\n\n    protected String getSystemMessageProducerId() {\n        return \"PID_\" + getBroadcastTopicName();\n    }\n\n    protected String getSystemMessageConsumerId() {\n        return \"CID_\" + getBroadcastTopicName();\n    }\n\n    protected String getBroadcastTopicName() {\n        return ConfigurationManager.getProxyConfig().getHeartbeatSyncerTopicName();\n    }\n\n    protected String getSubTag() {\n        return \"*\";\n    }\n\n    protected String getBroadcastTopicClusterName() {\n        ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();\n        return proxyConfig.getHeartbeatSyncerTopicClusterName();\n    }\n\n    protected int getBroadcastTopicQueueNum() {\n        return 1;\n    }\n\n    public RPCHook getRpcHook() {\n        return rpcHook;\n    }\n\n    protected void sendSystemMessage(Object data) {\n        String targetTopic = this.getBroadcastTopicName();\n        try {\n            Message message = new Message(\n                targetTopic,\n                JSON.toJSONString(data).getBytes(StandardCharsets.UTF_8)\n            );\n\n            AddressableMessageQueue messageQueue = this.topicRouteService.getAllMessageQueueView(ProxyContext.createForInner(this.getClass()), targetTopic)\n                .getWriteSelector().selectOne(true);\n            this.mqClientAPIFactory.getClient().sendMessageAsync(\n                messageQueue.getBrokerAddr(),\n                messageQueue.getBrokerName(),\n                message,\n                buildSendMessageRequestHeader(message, this.getSystemMessageProducerId(), messageQueue.getQueueId()),\n                Duration.ofSeconds(3).toMillis()\n            ).whenCompleteAsync((result, throwable) -> {\n                if (throwable != null) {\n                    log.error(\"send system message failed. data: {}, topic: {}\", data, getBroadcastTopicName(), throwable);\n                    return;\n                }\n                if (SendStatus.SEND_OK != result.getSendStatus()) {\n                    log.error(\"send system message failed. data: {}, topic: {}, sendResult:{}\", data, getBroadcastTopicName(), result);\n                }\n            });\n        } catch (Throwable t) {\n            log.error(\"send system message failed. data: {}, topic: {}\", data, targetTopic, t);\n        }\n    }\n\n    protected SendMessageRequestHeader buildSendMessageRequestHeader(Message message,\n        String producerGroup, int queueId) {\n        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();\n\n        requestHeader.setProducerGroup(producerGroup);\n        requestHeader.setTopic(message.getTopic());\n        requestHeader.setDefaultTopic(TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC);\n        requestHeader.setDefaultTopicQueueNums(0);\n        requestHeader.setQueueId(queueId);\n        requestHeader.setSysFlag(0);\n        requestHeader.setBornTimestamp(System.currentTimeMillis());\n        requestHeader.setFlag(message.getFlag());\n        requestHeader.setProperties(MessageDecoder.messageProperties2String(message.getProperties()));\n        requestHeader.setReconsumeTimes(0);\n        requestHeader.setBatch(false);\n        return requestHeader;\n    }\n\n    @Override\n    public void start() throws Exception {\n        this.createSysTopic();\n        RPCHook rpcHook = this.getRpcHook();\n        this.defaultMQPushConsumer = new DefaultMQPushConsumer(this.getSystemMessageConsumerId(), rpcHook);\n\n        this.defaultMQPushConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);\n        this.defaultMQPushConsumer.setMessageModel(MessageModel.BROADCASTING);\n        try {\n            this.defaultMQPushConsumer.subscribe(this.getBroadcastTopicName(), this.getSubTag());\n        } catch (MQClientException e) {\n            throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, \"subscribe to broadcast topic \" + this.getBroadcastTopicName() + \" failed. \" + e.getMessage());\n        }\n        this.defaultMQPushConsumer.registerMessageListener(this);\n        this.defaultMQPushConsumer.start();\n    }\n\n    protected void createSysTopic() {\n        String clusterName = this.getBroadcastTopicClusterName();\n        if (StringUtils.isEmpty(clusterName)) {\n            throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, \"system topic cluster cannot be empty\");\n        }\n\n        boolean createSuccess = this.adminService.createTopicOnTopicBrokerIfNotExist(\n            this.getBroadcastTopicName(),\n            clusterName,\n            this.getBroadcastTopicQueueNum(),\n            this.getBroadcastTopicQueueNum(),\n            true,\n            3\n        );\n        if (!createSuccess) {\n            throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, \"create system broadcast topic \" + this.getBroadcastTopicName() + \" failed on cluster \" + clusterName);\n        }\n    }\n\n    @Override\n    public void shutdown() throws Exception {\n        this.defaultMQPushConsumer.shutdown();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.sysmessage;\n\nimport com.alibaba.fastjson2.JSON;\nimport io.netty.channel.Channel;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ConsumerGroupEvent;\nimport org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;\nimport org.apache.rocketmq.broker.client.ConsumerManager;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.thread.ThreadPoolMonitor;\nimport org.apache.rocketmq.proxy.common.channel.ChannelHelper;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.processor.channel.RemoteChannel;\nimport org.apache.rocketmq.proxy.service.admin.AdminService;\nimport org.apache.rocketmq.proxy.service.route.TopicRouteService;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\n\npublic class HeartbeatSyncer extends AbstractSystemMessageSyncer {\n\n    protected ThreadPoolExecutor threadPoolExecutor;\n    protected ConsumerManager consumerManager;\n    protected final Map<String /* group @ channelId as longText */, RemoteChannel> remoteChannelMap = new ConcurrentHashMap<>();\n    protected String localProxyId;\n\n    public HeartbeatSyncer(TopicRouteService topicRouteService, AdminService adminService,\n                           ConsumerManager consumerManager, MQClientAPIFactory mqClientAPIFactory, RPCHook rpcHook) {\n        super(topicRouteService, adminService, mqClientAPIFactory, rpcHook);\n        this.consumerManager = consumerManager;\n        this.localProxyId = buildLocalProxyId();\n        this.init();\n    }\n\n    protected void init() {\n        ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();\n        this.threadPoolExecutor = ThreadPoolMonitor.createAndMonitor(\n            proxyConfig.getHeartbeatSyncerThreadPoolNums(),\n            proxyConfig.getHeartbeatSyncerThreadPoolNums(),\n            1,\n            TimeUnit.MINUTES,\n            \"HeartbeatSyncer\",\n            proxyConfig.getHeartbeatSyncerThreadPoolQueueCapacity()\n        );\n        this.consumerManager.appendConsumerIdsChangeListener(new ConsumerIdsChangeListener() {\n            @Override\n            public void handle(ConsumerGroupEvent event, String group, Object... args) {\n                processConsumerGroupEvent(event, group, args);\n            }\n\n            @Override\n            public void shutdown() {\n\n            }\n        });\n    }\n\n    @Override\n    public void shutdown() throws Exception {\n        this.threadPoolExecutor.shutdown();\n        super.shutdown();\n    }\n\n    protected void processConsumerGroupEvent(ConsumerGroupEvent event, String group, Object... args) {\n        if (event == ConsumerGroupEvent.CLIENT_UNREGISTER) {\n            if (args == null || args.length < 1) {\n                return;\n            }\n            if (args[0] instanceof ClientChannelInfo) {\n                ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0];\n                remoteChannelMap.remove(buildKey(group, clientChannelInfo.getChannel()));\n            }\n        }\n    }\n\n    public void onConsumerRegister(String consumerGroup, ClientChannelInfo clientChannelInfo,\n        ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,\n        Set<SubscriptionData> subList) {\n        if (clientChannelInfo == null || ChannelHelper.isRemote(clientChannelInfo.getChannel())) {\n            return;\n        }\n        try {\n            this.threadPoolExecutor.submit(() -> {\n                try {\n                    RemoteChannel remoteChannel = RemoteChannel.create(clientChannelInfo.getChannel());\n                    if (remoteChannel == null) {\n                        return;\n                    }\n                    HeartbeatSyncerData data = new HeartbeatSyncerData(\n                        HeartbeatType.REGISTER,\n                        clientChannelInfo.getClientId(),\n                        clientChannelInfo.getLanguage(),\n                        clientChannelInfo.getVersion(),\n                        consumerGroup,\n                        consumeType,\n                        messageModel,\n                        consumeFromWhere,\n                        localProxyId,\n                        remoteChannel.encode()\n                    );\n                    data.setSubscriptionDataSet(subList);\n\n                    log.debug(\"sync register heart beat. topic:{}, data:{}\", this.getBroadcastTopicName(), data);\n                    this.sendSystemMessage(data);\n                } catch (Throwable t) {\n                    log.error(\"heartbeat register broadcast failed. group:{}, clientChannelInfo:{}, consumeType:{}, messageModel:{}, consumeFromWhere:{}, subList:{}\",\n                        consumerGroup, clientChannelInfo, consumeType, messageModel, consumeFromWhere, subList, t);\n                }\n            });\n        } catch (Throwable t) {\n            log.error(\"heartbeat submit register broadcast failed. group:{}, clientChannelInfo:{}, consumeType:{}, messageModel:{}, consumeFromWhere:{}, subList:{}\",\n                consumerGroup, clientChannelInfo, consumeType, messageModel, consumeFromWhere, subList, t);\n        }\n    }\n\n    public void onConsumerUnRegister(String consumerGroup, ClientChannelInfo clientChannelInfo) {\n        if (clientChannelInfo == null || ChannelHelper.isRemote(clientChannelInfo.getChannel())) {\n            return;\n        }\n        try {\n            this.threadPoolExecutor.submit(() -> {\n                try {\n                    RemoteChannel remoteChannel = RemoteChannel.create(clientChannelInfo.getChannel());\n                    if (remoteChannel == null) {\n                        return;\n                    }\n                    HeartbeatSyncerData data = new HeartbeatSyncerData(\n                        HeartbeatType.UNREGISTER,\n                        clientChannelInfo.getClientId(),\n                        clientChannelInfo.getLanguage(),\n                        clientChannelInfo.getVersion(),\n                        consumerGroup,\n                        null,\n                        null,\n                        null,\n                        localProxyId,\n                        remoteChannel.encode()\n                    );\n\n                    log.debug(\"sync unregister heart beat. topic:{}, data:{}\", this.getBroadcastTopicName(), data);\n                    this.sendSystemMessage(data);\n                } catch (Throwable t) {\n                    log.error(\"heartbeat unregister broadcast failed. group:{}, clientChannelInfo:{}, consumeType:{}\",\n                        consumerGroup, clientChannelInfo, t);\n                }\n            });\n        } catch (Throwable t) {\n            log.error(\"heartbeat submit unregister broadcast failed. group:{}, clientChannelInfo:{}, consumeType:{}\",\n                consumerGroup, clientChannelInfo, t);\n        }\n    }\n\n    @Override\n    public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {\n        if (msgs == null || msgs.isEmpty()) {\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        }\n\n        for (MessageExt msg : msgs) {\n            try {\n                HeartbeatSyncerData data = JSON.parseObject(new String(msg.getBody(), StandardCharsets.UTF_8), HeartbeatSyncerData.class);\n                if (data.getLocalProxyId().equals(localProxyId)) {\n                    continue;\n                }\n\n                RemoteChannel decodedChannel = RemoteChannel.decode(data.getChannelData());\n                RemoteChannel channel = remoteChannelMap.computeIfAbsent(buildKey(data.getGroup(), decodedChannel), key -> decodedChannel);\n                channel.setExtendAttribute(decodedChannel.getChannelExtendAttribute());\n                ClientChannelInfo clientChannelInfo = new ClientChannelInfo(\n                    channel,\n                    data.getClientId(),\n                    data.getLanguage(),\n                    data.getVersion()\n                );\n                log.debug(\"start process remote channel. data:{}, clientChannelInfo:{}\", data, clientChannelInfo);\n                if (data.getHeartbeatType().equals(HeartbeatType.REGISTER)) {\n                    this.consumerManager.registerConsumer(\n                        data.getGroup(),\n                        clientChannelInfo,\n                        data.getConsumeType(),\n                        data.getMessageModel(),\n                        data.getConsumeFromWhere(),\n                        data.getSubscriptionDataSet(),\n                        false\n                    );\n                } else {\n                    this.consumerManager.unregisterConsumer(\n                        data.getGroup(),\n                        clientChannelInfo,\n                        false\n                    );\n                }\n            } catch (Throwable t) {\n                log.error(\"heartbeat consume message failed. msg:{}, data:{}\", msg, new String(msg.getBody(), StandardCharsets.UTF_8), t);\n            }\n        }\n\n        return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n    }\n\n    private String buildLocalProxyId() {\n        ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();\n        // use local address, remoting port and grpc port to build unique local proxy Id\n        return proxyConfig.getLocalServeAddr() + \"%\" + proxyConfig.getRemotingListenPort() + \"%\" + proxyConfig.getGrpcServerPort();\n    }\n\n    private static String buildKey(String group, Channel channel) {\n        return group + \"@\" + channel.id().asLongText();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerData.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.sysmessage;\n\nimport com.google.common.base.MoreObjects;\nimport java.util.Set;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\n\npublic class HeartbeatSyncerData {\n    private HeartbeatType heartbeatType;\n    private String clientId;\n    private LanguageCode language;\n    private int version;\n    private long lastUpdateTimestamp = System.currentTimeMillis();\n    private Set<SubscriptionData> subscriptionDataSet;\n    private String group;\n    private ConsumeType consumeType;\n    private MessageModel messageModel;\n    private ConsumeFromWhere consumeFromWhere;\n    private String localProxyId;\n    private String channelData;\n\n    public HeartbeatSyncerData() {\n    }\n\n    public HeartbeatSyncerData(HeartbeatType heartbeatType, String clientId,\n        LanguageCode language, int version, String group,\n        ConsumeType consumeType, MessageModel messageModel,\n        ConsumeFromWhere consumeFromWhere, String localProxyId,\n        String channelData) {\n        this.heartbeatType = heartbeatType;\n        this.clientId = clientId;\n        this.language = language;\n        this.version = version;\n        this.group = group;\n        this.consumeType = consumeType;\n        this.messageModel = messageModel;\n        this.consumeFromWhere = consumeFromWhere;\n        this.localProxyId = localProxyId;\n        this.channelData = channelData;\n    }\n\n    public HeartbeatType getHeartbeatType() {\n        return heartbeatType;\n    }\n\n    public void setHeartbeatType(HeartbeatType heartbeatType) {\n        this.heartbeatType = heartbeatType;\n    }\n\n    public String getClientId() {\n        return clientId;\n    }\n\n    public void setClientId(String clientId) {\n        this.clientId = clientId;\n    }\n\n    public LanguageCode getLanguage() {\n        return language;\n    }\n\n    public void setLanguage(LanguageCode language) {\n        this.language = language;\n    }\n\n    public int getVersion() {\n        return version;\n    }\n\n    public void setVersion(int version) {\n        this.version = version;\n    }\n\n    public long getLastUpdateTimestamp() {\n        return lastUpdateTimestamp;\n    }\n\n    public void setLastUpdateTimestamp(long lastUpdateTimestamp) {\n        this.lastUpdateTimestamp = lastUpdateTimestamp;\n    }\n\n    public Set<SubscriptionData> getSubscriptionDataSet() {\n        return subscriptionDataSet;\n    }\n\n    public void setSubscriptionDataSet(\n        Set<SubscriptionData> subscriptionDataSet) {\n        this.subscriptionDataSet = subscriptionDataSet;\n    }\n\n    public String getGroup() {\n        return group;\n    }\n\n    public void setGroup(String group) {\n        this.group = group;\n    }\n\n    public ConsumeType getConsumeType() {\n        return consumeType;\n    }\n\n    public void setConsumeType(ConsumeType consumeType) {\n        this.consumeType = consumeType;\n    }\n\n    public MessageModel getMessageModel() {\n        return messageModel;\n    }\n\n    public void setMessageModel(MessageModel messageModel) {\n        this.messageModel = messageModel;\n    }\n\n    public ConsumeFromWhere getConsumeFromWhere() {\n        return consumeFromWhere;\n    }\n\n    public void setConsumeFromWhere(ConsumeFromWhere consumeFromWhere) {\n        this.consumeFromWhere = consumeFromWhere;\n    }\n\n    public String getLocalProxyId() {\n        return localProxyId;\n    }\n\n    public void setLocalProxyId(String localProxyId) {\n        this.localProxyId = localProxyId;\n    }\n\n    public String getChannelData() {\n        return channelData;\n    }\n\n    public void setChannelData(String channelData) {\n        this.channelData = channelData;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"heartbeatType\", heartbeatType)\n            .add(\"clientId\", clientId)\n            .add(\"language\", language)\n            .add(\"version\", version)\n            .add(\"lastUpdateTimestamp\", lastUpdateTimestamp)\n            .add(\"subscriptionDataSet\", subscriptionDataSet)\n            .add(\"group\", group)\n            .add(\"consumeType\", consumeType)\n            .add(\"messageModel\", messageModel)\n            .add(\"consumeFromWhere\", consumeFromWhere)\n            .add(\"connectProxyIp\", localProxyId)\n            .add(\"channelData\", channelData)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.sysmessage;\n\npublic enum HeartbeatType {\n    REGISTER,\n    UNREGISTER;\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/AbstractTransactionService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.transaction;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.common.utils.StartAndShutdown;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;\n\npublic abstract class AbstractTransactionService implements TransactionService, StartAndShutdown {\n\n    protected TransactionDataManager transactionDataManager = new TransactionDataManager();\n\n    @Override\n    public TransactionData addTransactionDataByBrokerAddr(ProxyContext ctx, String brokerAddr, String topic, String producerGroup, long tranStateTableOffset, long commitLogOffset, String transactionId,\n        Message message) {\n        return this.addTransactionDataByBrokerName(ctx, this.getBrokerNameByAddr(brokerAddr), topic, producerGroup, tranStateTableOffset, commitLogOffset, transactionId, message);\n    }\n\n    @Override\n    public TransactionData addTransactionDataByBrokerName(ProxyContext ctx, String brokerName, String topic, String producerGroup, long tranStateTableOffset, long commitLogOffset, String transactionId,\n        Message message) {\n        if (StringUtils.isBlank(brokerName)) {\n            return null;\n        }\n        TransactionData transactionData = new TransactionData(\n            brokerName,\n            topic,\n            tranStateTableOffset, commitLogOffset, transactionId,\n            System.currentTimeMillis(),\n            ConfigurationManager.getProxyConfig().getTransactionDataExpireMillis());\n\n        this.transactionDataManager.addTransactionData(\n            producerGroup,\n            transactionId,\n            transactionData\n        );\n        return transactionData;\n    }\n\n    @Override\n    public EndTransactionRequestData genEndTransactionRequestHeader(ProxyContext ctx, String topic, String producerGroup, Integer commitOrRollback,\n        boolean fromTransactionCheck, String msgId, String transactionId) {\n        TransactionData transactionData = this.transactionDataManager.pollNoExpireTransactionData(producerGroup, transactionId);\n        if (transactionData == null) {\n            return null;\n        }\n        EndTransactionRequestHeader header = new EndTransactionRequestHeader();\n        header.setTopic(topic);\n        header.setProducerGroup(producerGroup);\n        header.setCommitOrRollback(commitOrRollback);\n        header.setFromTransactionCheck(fromTransactionCheck);\n        header.setMsgId(msgId);\n        header.setTransactionId(transactionId);\n        header.setTranStateTableOffset(transactionData.getTranStateTableOffset());\n        header.setCommitLogOffset(transactionData.getCommitLogOffset());\n        return new EndTransactionRequestData(transactionData.getBrokerName(), header);\n    }\n\n    @Override\n    public void onSendCheckTransactionStateFailed(ProxyContext context, String producerGroup, TransactionData transactionData) {\n        this.transactionDataManager.removeTransactionData(producerGroup, transactionData.getTransactionId(), transactionData);\n    }\n\n    protected abstract String getBrokerNameByAddr(String brokerAddr);\n\n    @Override\n    public void shutdown() throws Exception {\n        this.transactionDataManager.shutdown();\n    }\n\n    @Override\n    public void start() throws Exception {\n        this.transactionDataManager.start();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.transaction;\n\nimport com.google.common.collect.Sets;\nimport java.time.Duration;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicReference;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.broker.client.ProducerManager;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.thread.ThreadPoolMonitor;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;\nimport org.apache.rocketmq.proxy.service.route.MessageQueueView;\nimport org.apache.rocketmq.proxy.service.route.TopicRouteService;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ProducerData;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\n\npublic class ClusterTransactionService extends AbstractTransactionService {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    private static final String TRANS_HEARTBEAT_CLIENT_ID = \"rmq-proxy-producer-client\";\n\n    private final MQClientAPIFactory mqClientAPIFactory;\n    private final TopicRouteService topicRouteService;\n    private final ProducerManager producerManager;\n\n    private ThreadPoolExecutor heartbeatExecutors;\n    private final Map<String /* group */, Set<ClusterData>/* cluster list */> groupClusterData = new ConcurrentHashMap<>();\n    private final AtomicReference<Map<String /* brokerAddr */, String /* brokerName */>> brokerAddrNameMapRef = new AtomicReference<>();\n    private TxHeartbeatServiceThread txHeartbeatServiceThread;\n\n    public ClusterTransactionService(TopicRouteService topicRouteService, ProducerManager producerManager,\n        MQClientAPIFactory mqClientAPIFactory) {\n        this.topicRouteService = topicRouteService;\n        this.producerManager = producerManager;\n        this.mqClientAPIFactory = mqClientAPIFactory;\n    }\n\n    @Override\n    public void addTransactionSubscription(ProxyContext ctx, String group, List<String> topicList) {\n        for (String topic : topicList) {\n            addTransactionSubscription(ctx, group, topic);\n        }\n    }\n\n    @Override\n    public void addTransactionSubscription(ProxyContext ctx, String group, String topic) {\n        try {\n            groupClusterData.compute(group, (groupName, clusterDataSet) -> {\n                if (clusterDataSet == null) {\n                    clusterDataSet = Sets.newHashSet();\n                }\n                clusterDataSet.addAll(getClusterDataFromTopic(ctx, topic));\n                return clusterDataSet;\n            });\n        } catch (Exception e) {\n            log.error(\"add producer group err in txHeartBeat. groupId: {}, err: {}\", group, e);\n        }\n    }\n\n    @Override\n    public void replaceTransactionSubscription(ProxyContext ctx, String group, List<String> topicList) {\n        Set<ClusterData> clusterDataSet = new HashSet<>();\n        for (String topic : topicList) {\n            clusterDataSet.addAll(getClusterDataFromTopic(ctx, topic));\n        }\n        groupClusterData.put(group, clusterDataSet);\n    }\n\n    private Set<ClusterData> getClusterDataFromTopic(ProxyContext ctx, String topic) {\n        try {\n            MessageQueueView messageQueue = this.topicRouteService.getAllMessageQueueView(ctx, topic);\n            List<BrokerData> brokerDataList = messageQueue.getTopicRouteData().getBrokerDatas();\n\n            if (brokerDataList == null) {\n                return Collections.emptySet();\n            }\n            Set<ClusterData> res = Sets.newHashSet();\n            for (BrokerData brokerData : brokerDataList) {\n                res.add(new ClusterData(brokerData.getCluster()));\n            }\n            return res;\n        } catch (Throwable t) {\n            log.error(\"get cluster data failed in txHeartBeat. topic: {}, err: {}\", topic, t);\n        }\n        return Collections.emptySet();\n    }\n\n    @Override\n    public void unSubscribeAllTransactionTopic(ProxyContext ctx, String group) {\n        groupClusterData.remove(group);\n    }\n\n    public void scanProducerHeartBeat() {\n        Set<String> groupSet = groupClusterData.keySet();\n\n        Map<String /* cluster */, List<HeartbeatData>> clusterHeartbeatData = new HashMap<>();\n        for (String group : groupSet) {\n            groupClusterData.computeIfPresent(group, (groupName, clusterDataSet) -> {\n                if (clusterDataSet.isEmpty()) {\n                    return null;\n                }\n                if (!this.producerManager.groupOnline(groupName)) {\n                    return null;\n                }\n\n                ProducerData producerData = new ProducerData();\n                producerData.setGroupName(groupName);\n\n                for (ClusterData clusterData : clusterDataSet) {\n                    List<HeartbeatData> heartbeatDataList = clusterHeartbeatData.get(clusterData.cluster);\n                    if (heartbeatDataList == null) {\n                        heartbeatDataList = new ArrayList<>();\n                    }\n\n                    HeartbeatData heartbeatData;\n                    if (heartbeatDataList.isEmpty()) {\n                        heartbeatData = new HeartbeatData();\n                        heartbeatData.setClientID(TRANS_HEARTBEAT_CLIENT_ID);\n                        heartbeatDataList.add(heartbeatData);\n                    } else {\n                        heartbeatData = heartbeatDataList.get(heartbeatDataList.size() - 1);\n                        if (heartbeatData.getProducerDataSet().size() >= ConfigurationManager.getProxyConfig().getTransactionHeartbeatBatchNum()) {\n                            heartbeatData = new HeartbeatData();\n                            heartbeatData.setClientID(TRANS_HEARTBEAT_CLIENT_ID);\n                            heartbeatDataList.add(heartbeatData);\n                        }\n                    }\n\n                    heartbeatData.getProducerDataSet().add(producerData);\n                    clusterHeartbeatData.put(clusterData.cluster, heartbeatDataList);\n                }\n\n                if (clusterDataSet.isEmpty()) {\n                    return null;\n                }\n                return clusterDataSet;\n            });\n        }\n\n        if (clusterHeartbeatData.isEmpty()) {\n            return;\n        }\n        Map<String, String> brokerAddrNameMap = new ConcurrentHashMap<>();\n        Set<Map.Entry<String, List<HeartbeatData>>> clusterEntry = clusterHeartbeatData.entrySet();\n        for (Map.Entry<String, List<HeartbeatData>> entry : clusterEntry) {\n            sendHeartBeatToCluster(entry.getKey(), entry.getValue(), brokerAddrNameMap);\n        }\n        this.brokerAddrNameMapRef.set(brokerAddrNameMap);\n    }\n\n    public Map<String, Set<ClusterData>> getGroupClusterData() {\n        return groupClusterData;\n    }\n\n    protected void sendHeartBeatToCluster(String clusterName, List<HeartbeatData> heartbeatDataList, Map<String, String> brokerAddrNameMap) {\n        if (heartbeatDataList == null) {\n            return;\n        }\n        for (HeartbeatData heartbeatData : heartbeatDataList) {\n            sendHeartBeatToCluster(clusterName, heartbeatData, brokerAddrNameMap);\n        }\n        this.brokerAddrNameMapRef.set(brokerAddrNameMap);\n    }\n\n    protected void sendHeartBeatToCluster(String clusterName, HeartbeatData heartbeatData, Map<String, String> brokerAddrNameMap) {\n        try {\n            MessageQueueView messageQueue = this.topicRouteService.getAllMessageQueueView(ProxyContext.createForInner(this.getClass()), clusterName);\n            List<BrokerData> brokerDataList = messageQueue.getTopicRouteData().getBrokerDatas();\n            if (brokerDataList == null) {\n                return;\n            }\n            for (BrokerData brokerData : brokerDataList) {\n                brokerAddrNameMap.put(brokerData.selectBrokerAddr(), brokerData.getBrokerName());\n                heartbeatExecutors.submit(() -> {\n                    String brokerAddr = brokerData.selectBrokerAddr();\n                    this.mqClientAPIFactory.getClient()\n                        .sendHeartbeatOneway(brokerAddr, heartbeatData, Duration.ofSeconds(3).toMillis())\n                        .exceptionally(t -> {\n                            log.error(\"Send transactionHeartbeat to broker err. brokerAddr: {}\", brokerAddr, t);\n                            return null;\n                        });\n                });\n            }\n        } catch (Exception e) {\n            log.error(\"get broker add in cluster failed in tx. clusterName: {}\", clusterName, e);\n        }\n    }\n\n    @Override\n    protected String getBrokerNameByAddr(String brokerAddr) {\n        if (StringUtils.isBlank(brokerAddr)) {\n            return null;\n        }\n        return brokerAddrNameMapRef.get().get(brokerAddr);\n    }\n\n    static class ClusterData {\n        private final String cluster;\n\n        public ClusterData(String cluster) {\n            this.cluster = cluster;\n        }\n\n        public String getCluster() {\n            return cluster;\n        }\n\n        @Override\n        public boolean equals(Object obj) {\n            if (obj == this) {\n                return true;\n            }\n            if (!(obj instanceof ClusterData)) {\n                return super.equals(obj);\n            }\n\n            ClusterData other = (ClusterData) obj;\n            return cluster.equals(other.cluster);\n        }\n\n        @Override\n        public int hashCode() {\n            return cluster.hashCode();\n        }\n    }\n\n    class TxHeartbeatServiceThread extends ServiceThread {\n\n        @Override\n        public String getServiceName() {\n            return TxHeartbeatServiceThread.class.getName();\n        }\n\n        @Override\n        public void run() {\n            while (!this.isStopped()) {\n                this.waitForRunning(TimeUnit.SECONDS.toMillis(ConfigurationManager.getProxyConfig().getTransactionHeartbeatPeriodSecond()));\n            }\n        }\n\n        @Override\n        protected void onWaitEnd() {\n            scanProducerHeartBeat();\n        }\n    }\n\n    @Override\n    public void start() throws Exception {\n        ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();\n        txHeartbeatServiceThread = new TxHeartbeatServiceThread();\n\n        super.start();\n        txHeartbeatServiceThread.start();\n        heartbeatExecutors = ThreadPoolMonitor.createAndMonitor(\n            proxyConfig.getTransactionHeartbeatThreadPoolNums(),\n            proxyConfig.getTransactionHeartbeatThreadPoolNums(),\n            0L, TimeUnit.MILLISECONDS,\n            \"TransactionHeartbeatRegisterThread\",\n            proxyConfig.getTransactionHeartbeatThreadPoolQueueCapacity()\n        );\n    }\n\n    @Override\n    public void shutdown() throws Exception {\n        txHeartbeatServiceThread.shutdown();\n        heartbeatExecutors.shutdown();\n        super.shutdown();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/EndTransactionRequestData.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.transaction;\n\nimport org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;\n\npublic class EndTransactionRequestData {\n    private String brokerName;\n    private EndTransactionRequestHeader requestHeader;\n\n    public EndTransactionRequestData(String brokerName, EndTransactionRequestHeader requestHeader) {\n        this.brokerName = brokerName;\n        this.requestHeader = requestHeader;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public EndTransactionRequestHeader getRequestHeader() {\n        return requestHeader;\n    }\n\n    public void setRequestHeader(EndTransactionRequestHeader requestHeader) {\n        this.requestHeader = requestHeader;\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/LocalTransactionService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.transaction;\n\nimport java.util.List;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\n\n/**\n * no need to implements, because the channel of producer will put into the broker's producerManager\n */\npublic class LocalTransactionService extends AbstractTransactionService {\n\n    protected final BrokerConfig brokerConfig;\n\n    public LocalTransactionService(BrokerConfig brokerConfig) {\n        this.brokerConfig = brokerConfig;\n    }\n\n    @Override\n    public void addTransactionSubscription(ProxyContext ctx, String group, List<String> topicList) {\n\n    }\n\n    @Override\n    public void addTransactionSubscription(ProxyContext ctx, String group, String topic) {\n\n    }\n\n    @Override\n    public void replaceTransactionSubscription(ProxyContext ctx, String group, List<String> topicList) {\n\n    }\n\n    @Override\n    public void unSubscribeAllTransactionTopic(ProxyContext ctx, String group) {\n\n    }\n\n    @Override\n    protected String getBrokerNameByAddr(String brokerAddr) {\n        return this.brokerConfig.getBrokerName();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/TransactionData.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.transaction;\n\nimport com.google.common.base.MoreObjects;\nimport com.google.common.base.Objects;\nimport com.google.common.collect.ComparisonChain;\n\npublic class TransactionData implements Comparable<TransactionData> {\n    private final String brokerName;\n    private final String topic;\n    private final long tranStateTableOffset;\n    private final long commitLogOffset;\n    private final String transactionId;\n    private final long checkTimestamp;\n    private final long expireMs;\n\n    public TransactionData(String brokerName, String topic, long tranStateTableOffset, long commitLogOffset, String transactionId,\n        long checkTimestamp, long expireMs) {\n        this.brokerName = brokerName;\n        this.topic = topic;\n        this.tranStateTableOffset = tranStateTableOffset;\n        this.commitLogOffset = commitLogOffset;\n        this.transactionId = transactionId;\n        this.checkTimestamp = checkTimestamp;\n        this.expireMs = expireMs;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public long getTranStateTableOffset() {\n        return tranStateTableOffset;\n    }\n\n    public long getCommitLogOffset() {\n        return commitLogOffset;\n    }\n\n    public String getTransactionId() {\n        return transactionId;\n    }\n\n    public long getCheckTimestamp() {\n        return checkTimestamp;\n    }\n\n    public long getExpireMs() {\n        return expireMs;\n    }\n\n    public long getExpireTime() {\n        return checkTimestamp + expireMs;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) {\n            return true;\n        }\n        if (o == null || getClass() != o.getClass()) {\n            return false;\n        }\n        TransactionData data = (TransactionData) o;\n        return tranStateTableOffset == data.tranStateTableOffset && commitLogOffset == data.commitLogOffset &&\n            getExpireTime() == data.getExpireTime() && Objects.equal(brokerName, data.brokerName) &&\n            Objects.equal(transactionId, data.transactionId);\n    }\n\n    @Override\n    public int hashCode() {\n        return Objects.hashCode(brokerName, transactionId, tranStateTableOffset, commitLogOffset, getExpireTime());\n    }\n\n    @Override\n    public int compareTo(TransactionData o) {\n        return ComparisonChain.start()\n            .compare(getExpireTime(), o.getExpireTime())\n            .compare(brokerName, o.brokerName)\n            .compare(commitLogOffset, o.commitLogOffset)\n            .compare(tranStateTableOffset, o.tranStateTableOffset)\n            .compare(transactionId, o.transactionId)\n            .result();\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"brokerName\", brokerName)\n            .add(\"tranStateTableOffset\", tranStateTableOffset)\n            .add(\"commitLogOffset\", commitLogOffset)\n            .add(\"transactionId\", transactionId)\n            .add(\"checkTimestamp\", checkTimestamp)\n            .add(\"expireMs\", expireMs)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/TransactionDataManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.transaction;\n\nimport java.util.Iterator;\nimport java.util.Map;\nimport java.util.NavigableSet;\nimport java.util.NoSuchElementException;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentSkipListSet;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.concurrent.atomic.AtomicReference;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.common.utils.StartAndShutdown;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\n\npublic class TransactionDataManager implements StartAndShutdown {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);\n\n    protected final AtomicLong maxTransactionDataExpireTime = new AtomicLong(System.currentTimeMillis());\n    protected final Map<String /* producerGroup@transactionId */, NavigableSet<TransactionData>> transactionIdDataMap = new ConcurrentHashMap<>();\n    protected final TransactionDataCleaner transactionDataCleaner = new TransactionDataCleaner();\n\n    protected String buildKey(String producerGroup, String transactionId) {\n        return producerGroup + \"@\" + transactionId;\n    }\n\n    public void addTransactionData(String producerGroup, String transactionId, TransactionData transactionData) {\n        this.transactionIdDataMap.compute(buildKey(producerGroup, transactionId), (key, dataSet) -> {\n            if (dataSet == null) {\n                dataSet = new ConcurrentSkipListSet<>();\n            }\n            dataSet.add(transactionData);\n            if (dataSet.size() > ConfigurationManager.getProxyConfig().getTransactionDataMaxNum()) {\n                dataSet.pollFirst();\n            }\n            return dataSet;\n        });\n    }\n\n    public TransactionData pollNoExpireTransactionData(String producerGroup, String transactionId) {\n        AtomicReference<TransactionData> res = new AtomicReference<>();\n        long currTimestamp = System.currentTimeMillis();\n        this.transactionIdDataMap.computeIfPresent(buildKey(producerGroup, transactionId), (key, dataSet) -> {\n            TransactionData data = dataSet.pollLast();\n            while (data != null && data.getExpireTime() < currTimestamp) {\n                data = dataSet.pollLast();\n            }\n            if (data != null) {\n                res.set(data);\n            }\n            if (dataSet.isEmpty()) {\n                return null;\n            }\n            return dataSet;\n        });\n        return res.get();\n    }\n\n    public void removeTransactionData(String producerGroup, String transactionId, TransactionData transactionData) {\n        this.transactionIdDataMap.computeIfPresent(buildKey(producerGroup, transactionId), (key, dataSet) -> {\n            dataSet.remove(transactionData);\n            if (dataSet.isEmpty()) {\n                return null;\n            }\n            return dataSet;\n        });\n    }\n\n    protected void cleanExpireTransactionData() {\n        long currTimestamp = System.currentTimeMillis();\n        Set<String> transactionIdSet = this.transactionIdDataMap.keySet();\n        for (String transactionId : transactionIdSet) {\n            this.transactionIdDataMap.computeIfPresent(transactionId, (transactionIdKey, dataSet) -> {\n                Iterator<TransactionData> iterator = dataSet.iterator();\n                while (iterator.hasNext()) {\n                    try {\n                        TransactionData data = iterator.next();\n                        if (data.getExpireTime() < currTimestamp) {\n                            iterator.remove();\n                        } else {\n                            break;\n                        }\n                    } catch (NoSuchElementException ignore) {\n                        break;\n                    }\n                }\n                if (dataSet.isEmpty()) {\n                    return null;\n                }\n                try {\n                    TransactionData maxData = dataSet.last();\n                    maxTransactionDataExpireTime.set(Math.max(maxTransactionDataExpireTime.get(), maxData.getExpireTime()));\n                } catch (NoSuchElementException ignore) {\n                }\n                return dataSet;\n            });\n        }\n    }\n\n    protected class TransactionDataCleaner extends ServiceThread {\n\n        @Override\n        public String getServiceName() {\n            return \"TransactionDataCleaner\";\n        }\n\n        @Override\n        public void run() {\n            log.info(this.getServiceName() + \" service started\");\n            while (!this.isStopped()) {\n                this.waitForRunning(ConfigurationManager.getProxyConfig().getTransactionDataExpireScanPeriodMillis());\n            }\n            log.info(this.getServiceName() + \" service stopped\");\n        }\n\n        @Override\n        protected void onWaitEnd() {\n            cleanExpireTransactionData();\n        }\n    }\n\n    protected void waitTransactionDataClear() throws InterruptedException {\n        this.cleanExpireTransactionData();\n        long waitMs = Math.max(this.maxTransactionDataExpireTime.get() - System.currentTimeMillis(), 0);\n        waitMs = Math.min(waitMs, ConfigurationManager.getProxyConfig().getTransactionDataMaxWaitClearMillis());\n\n        if (waitMs > 0) {\n            TimeUnit.MILLISECONDS.sleep(waitMs);\n        }\n    }\n\n    @Override\n    public void shutdown() throws Exception {\n        this.transactionDataCleaner.shutdown();\n        this.waitTransactionDataClear();\n    }\n\n    @Override\n    public void start() throws Exception {\n        this.transactionDataCleaner.start();\n    }\n}\n"
  },
  {
    "path": "proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/TransactionService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.transaction;\n\nimport java.util.List;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\n\npublic interface TransactionService {\n\n    void addTransactionSubscription(ProxyContext ctx, String group, List<String> topicList);\n\n    void addTransactionSubscription(ProxyContext ctx, String group, String topic);\n\n    void replaceTransactionSubscription(ProxyContext ctx, String group, List<String> topicList);\n\n    void unSubscribeAllTransactionTopic(ProxyContext ctx, String group);\n\n    TransactionData addTransactionDataByBrokerAddr(ProxyContext ctx, String brokerAddr, String topic, String producerGroup, long tranStateTableOffset, long commitLogOffset, String transactionId,\n        Message message);\n\n    TransactionData addTransactionDataByBrokerName(ProxyContext ctx, String brokerName, String topic, String producerGroup, long tranStateTableOffset, long commitLogOffset, String transactionId,\n        Message message);\n\n    EndTransactionRequestData genEndTransactionRequestHeader(ProxyContext ctx, String topic, String producerGroup, Integer commitOrRollback,\n        boolean fromTransactionCheck, String msgId, String transactionId);\n\n    void onSendCheckTransactionStateFailed(ProxyContext context, String producerGroup, TransactionData transactionData);\n}\n"
  },
  {
    "path": "proxy/src/main/resources/rmq.proxy.logback.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n<configuration scan=\"true\" scanPeriod=\"30 seconds\">\n\n    <appender name=\"RocketmqProxyAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}proxy.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}proxy.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>128MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqProxyAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqProxyAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqProxyWatermarkAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}proxy_watermark.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}proxy_watermark.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>128MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqProxyWatermarkAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqProxyWatermarkAppender_inner\"/>\n    </appender>\n\n    <!-- Below is the logger configuration for broker-->\n    <appender name=\"DefaultAppender\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}broker_default.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}broker_default.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n\n    <appender name=\"RocketmqBrokerAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}broker.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}broker.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>20</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>128MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqBrokerAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqBrokerAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqProtectionAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}protection.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}protection.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqProtectionAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqProtectionAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqWaterMarkAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}watermark.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}watermark.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqWaterMarkAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqWaterMarkAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqStoreAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}store.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}store.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>128MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqStoreAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqStoreAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqRemotingAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}remoting.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}remoting.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqRemotingAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqRemotingAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqStoreErrorAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}storeerror.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}storeerror.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqStoreErrorAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqStoreErrorAppender_inner\"/>\n    </appender>\n\n\n    <appender name=\"RocketmqTransactionAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}transaction.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}transaction.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqTransactionAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqTransactionAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqRebalanceLockAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}lock.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}lock.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>5</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqRebalanceLockAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqRebalanceLockAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqFilterAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}filter.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}filter.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqFilterAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqFilterAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqStatsAppender\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}stats.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}stats.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>5</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n\n    <appender name=\"RocketmqCommercialAppender\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}commercial.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}commercial.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>500MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n\n    <appender name=\"RocketmqPopAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}pop.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}pop.%i.log\n            </fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>20</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy\n            class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>128MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n\n    <appender name=\"RocketmqProxyMetricsAppender\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}proxy_metric.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}proxy_metric.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>500MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n\n    <appender name=\"RocketmqAuthAuditAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>\n            ${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}auth_audit.log\n        </file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>\n                ${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}auth_audit.%i.log.gz\n            </fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqAuthAuditAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqAuthAuditAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqTrafficAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>\n            ${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}proxy_traffic.log\n        </file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>\n                ${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}proxy_traffic.%i.log.gz\n            </fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqTrafficAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqTrafficAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqPopAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqPopAppender_inner\"/>\n    </appender>\n\n    <appender name=\"STDOUT\" class=\"ch.qos.logback.core.ConsoleAppender\">\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n\n    <logger name=\"RocketmqBroker\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqBrokerAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqProtection\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqProtectionAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqWaterMark\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqWaterMarkAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqCommon\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqBrokerAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqStore\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqStoreAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqStoreError\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqStoreErrorAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqTransaction\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqTransactionAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqRebalanceLock\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqRebalanceLockAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqRemoting\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqRemotingAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqStats\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqStatsAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqCommercial\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqCommercialAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqFilter\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqFilterAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqConsole\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"STDOUT\"/>\n    </logger>\n\n    <logger name=\"RocketmqPop\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqPopAppender\" />\n    </logger>\n\n    <logger name=\"RocketmqProxy\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqProxyAppender\" />\n    </logger>\n\n    <logger name=\"RocketmqProxyWatermark\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqProxyWatermarkAppender\" />\n    </logger>\n\n    <!-- Use json formatter to log metrics -->\n    <logger name=\"io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqProxyMetricsAppender\"/>\n    </logger>\n\n    <logger name=\"io.opentelemetry.exporter.logging.LoggingMetricExporter\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqProxyMetricsAppender\" />\n    </logger>\n\n    <logger name=\"RocketmqAuthAudit\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqAuthAuditAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqTraffic\" additivity=\"false\" level=\"INFO\">\n        <appender-ref ref=\"RocketmqTrafficAppender\"/>\n    </logger>\n\n    <root level=\"INFO\">\n        <appender-ref ref=\"DefaultAppender\"/>\n    </root>\n</configuration>\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/ProxyStartupTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy;\n\nimport com.google.common.base.Preconditions;\nimport com.google.common.base.Splitter;\nimport io.opentelemetry.sdk.OpenTelemetrySdk;\nimport java.io.BufferedInputStream;\nimport java.io.BufferedOutputStream;\nimport java.io.File;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.util.Iterator;\nimport java.util.UUID;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.BrokerStartup;\nimport org.apache.rocketmq.broker.metrics.BrokerMetricsManager;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.proxy.config.Configuration;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.processor.DefaultMessagingProcessor;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.ArgumentCaptor;\nimport org.mockito.MockedStatic;\nimport org.mockito.Mockito;\nimport org.springframework.core.io.Resource;\nimport org.springframework.core.io.support.PathMatchingResourcePatternResolver;\n\nimport static org.apache.rocketmq.proxy.config.ConfigurationManager.RMQ_PROXY_HOME;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertSame;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.mockStatic;\n\npublic class ProxyStartupTest {\n\n    private File proxyHome;\n\n    @Before\n    public void before() throws Throwable {\n        proxyHome = new File(System.getProperty(\"java.io.tmpdir\"), UUID.randomUUID().toString().replace('-', '_'));\n        if (!proxyHome.exists()) {\n            proxyHome.mkdirs();\n        }\n        String folder = \"rmq-proxy-home\";\n        PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(getClass().getClassLoader());\n        Resource[] resources = resolver.getResources(String.format(\"classpath:%s/**/*\", folder));\n        for (Resource resource : resources) {\n            if (!resource.isReadable()) {\n                continue;\n            }\n            String description = resource.getDescription();\n            int start = description.indexOf('[');\n            int end = description.lastIndexOf(']');\n            String path = description.substring(start + 1, end);\n            try (InputStream inputStream = resource.getInputStream()) {\n                copyTo(path, inputStream, proxyHome, folder);\n            }\n        }\n        System.setProperty(RMQ_PROXY_HOME, proxyHome.getAbsolutePath());\n    }\n\n    private void copyTo(String path, InputStream src, File dstDir, String flag) throws IOException {\n        Preconditions.checkNotNull(flag);\n        Iterator<String> iterator = Splitter.on(File.separatorChar).split(path).iterator();\n        boolean found = false;\n        File dir = dstDir;\n        while (iterator.hasNext()) {\n            String current = iterator.next();\n            if (!found && flag.equals(current)) {\n                found = true;\n                continue;\n            }\n            if (found) {\n                if (!iterator.hasNext()) {\n                    dir = new File(dir, current);\n                } else {\n                    dir = new File(dir, current);\n                    if (!dir.exists()) {\n                        Assert.assertTrue(dir.mkdir());\n                    }\n                }\n            }\n        }\n\n        Assert.assertTrue(dir.createNewFile());\n        byte[] buffer = new byte[4096];\n        BufferedInputStream bis = new BufferedInputStream(src);\n        int len = 0;\n        try (BufferedOutputStream bos = new BufferedOutputStream(Files.newOutputStream(dir.toPath()))) {\n            while ((len = bis.read(buffer)) > 0) {\n                bos.write(buffer, 0, len);\n            }\n        }\n    }\n\n    private void recursiveDelete(File file) {\n        if (file.isFile()) {\n            file.delete();\n            return;\n        }\n\n        File[] files = file.listFiles();\n        for (File f : files) {\n            recursiveDelete(f);\n        }\n        file.delete();\n    }\n\n    @After\n    public void after() {\n        System.clearProperty(RMQ_PROXY_HOME);\n        System.clearProperty(MixAll.NAMESRV_ADDR_PROPERTY);\n        System.clearProperty(Configuration.CONFIG_PATH_PROPERTY);\n        recursiveDelete(proxyHome);\n    }\n\n    @Test\n    public void testParseAndInitCommandLineArgument() throws Exception {\n        Path configFilePath = Files.createTempFile(\"testParseAndInitCommandLineArgument\", \".json\");\n        String configData = \"{}\";\n        Files.write(configFilePath, configData.getBytes(StandardCharsets.UTF_8));\n\n        String brokerConfigPath = \"brokerConfigPath\";\n        String proxyConfigPath = configFilePath.toAbsolutePath().toString();\n        String proxyMode = \"LOCAL\";\n        String namesrvAddr = \"namesrvAddr\";\n        CommandLineArgument commandLineArgument = ProxyStartup.parseCommandLineArgument(new String[] {\n            \"-bc\", brokerConfigPath,\n            \"-pc\", proxyConfigPath,\n            \"-pm\", proxyMode,\n            \"-n\", namesrvAddr\n        });\n\n        assertEquals(brokerConfigPath, commandLineArgument.getBrokerConfigPath());\n        assertEquals(proxyConfigPath, commandLineArgument.getProxyConfigPath());\n        assertEquals(proxyMode, commandLineArgument.getProxyMode());\n        assertEquals(namesrvAddr, commandLineArgument.getNamesrvAddr());\n\n        ProxyStartup.initConfiguration(commandLineArgument);\n\n        ProxyConfig config = ConfigurationManager.getProxyConfig();\n        assertEquals(brokerConfigPath, config.getBrokerConfigPath());\n        assertEquals(proxyMode, config.getProxyMode());\n        assertEquals(namesrvAddr, config.getNamesrvAddr());\n    }\n\n    @Test\n    public void testLocalModeWithNameSrvAddrByProperty() throws Exception {\n        String namesrvAddr = \"namesrvAddr\";\n        System.setProperty(MixAll.NAMESRV_ADDR_PROPERTY, namesrvAddr);\n        CommandLineArgument commandLineArgument = ProxyStartup.parseCommandLineArgument(new String[] {\n            \"-pm\", \"local\"\n        });\n        ProxyStartup.initConfiguration(commandLineArgument);\n\n        ProxyConfig config = ConfigurationManager.getProxyConfig();\n        assertEquals(namesrvAddr, config.getNamesrvAddr());\n\n        validateBrokerCreateArgsWithNamsrvAddr(config, namesrvAddr);\n    }\n\n    private void validateBrokerCreateArgsWithNamsrvAddr(ProxyConfig config, String namesrvAddr) {\n        try (MockedStatic<BrokerStartup> brokerStartupMocked = mockStatic(BrokerStartup.class);\n             MockedStatic<DefaultMessagingProcessor> messagingProcessorMocked = mockStatic(DefaultMessagingProcessor.class)) {\n            ArgumentCaptor<Object> args = ArgumentCaptor.forClass(Object.class);\n            BrokerController brokerControllerMocked = mock(BrokerController.class);\n            BrokerMetricsManager brokerMetricsManagerMocked = mock(BrokerMetricsManager.class);\n            Mockito.when(brokerMetricsManagerMocked.getBrokerMeter()).thenReturn(OpenTelemetrySdk.builder().build().getMeter(\"test\"));\n            Mockito.when(brokerControllerMocked.getBrokerMetricsManager()).thenReturn(brokerMetricsManagerMocked);\n            brokerStartupMocked.when(() -> BrokerStartup.createBrokerController((String[]) args.capture()))\n                .thenReturn(brokerControllerMocked);\n            messagingProcessorMocked.when(() -> DefaultMessagingProcessor.createForLocalMode(any(), any()))\n                .thenReturn(mock(DefaultMessagingProcessor.class));\n\n            ProxyStartup.createMessagingProcessor();\n            String[] passArgs = (String[]) args.getValue();\n            assertEquals(\"-c\", passArgs[0]);\n            assertEquals(config.getBrokerConfigPath(), passArgs[1]);\n            assertEquals(\"-n\", passArgs[2]);\n            assertEquals(namesrvAddr, passArgs[3]);\n            assertEquals(4, passArgs.length);\n        }\n    }\n\n    @Test\n    public void testLocalModeWithNameSrvAddrByConfigFile() throws Exception {\n        String namesrvAddr = \"namesrvAddr\";\n        System.setProperty(MixAll.NAMESRV_ADDR_PROPERTY, \"foo\");\n        Path configFilePath = Files.createTempFile(\"testLocalModeWithNameSrvAddrByConfigFile\", \".json\");\n        String configData = \"{\\n\" +\n            \"  \\\"namesrvAddr\\\": \\\"namesrvAddr\\\"\\n\" +\n            \"}\";\n        Files.write(configFilePath, configData.getBytes(StandardCharsets.UTF_8));\n\n        CommandLineArgument commandLineArgument = ProxyStartup.parseCommandLineArgument(new String[] {\n            \"-pm\", \"local\",\n            \"-pc\", configFilePath.toAbsolutePath().toString()\n        });\n        ProxyStartup.initConfiguration(commandLineArgument);\n\n        ProxyConfig config = ConfigurationManager.getProxyConfig();\n        assertEquals(namesrvAddr, config.getNamesrvAddr());\n\n        validateBrokerCreateArgsWithNamsrvAddr(config, namesrvAddr);\n    }\n\n    @Test\n    public void testLocalModeWithNameSrvAddrByCommandLine() throws Exception {\n        String namesrvAddr = \"namesrvAddr\";\n        System.setProperty(MixAll.NAMESRV_ADDR_PROPERTY, \"foo\");\n        Path configFilePath = Files.createTempFile(\"testLocalModeWithNameSrvAddrByCommandLine\", \".json\");\n        String configData = \"{\\n\" +\n            \"  \\\"namesrvAddr\\\": \\\"foo\\\"\\n\" +\n            \"}\";\n        Files.write(configFilePath, configData.getBytes(StandardCharsets.UTF_8));\n\n        CommandLineArgument commandLineArgument = ProxyStartup.parseCommandLineArgument(new String[] {\n            \"-pm\", \"local\",\n            \"-pc\", configFilePath.toAbsolutePath().toString(),\n            \"-n\", namesrvAddr\n        });\n        ProxyStartup.initConfiguration(commandLineArgument);\n\n        ProxyConfig config = ConfigurationManager.getProxyConfig();\n        assertEquals(namesrvAddr, config.getNamesrvAddr());\n\n        validateBrokerCreateArgsWithNamsrvAddr(config, namesrvAddr);\n    }\n\n    @Test\n    public void testLocalModeWithAllArgs() throws Exception {\n        String namesrvAddr = \"namesrvAddr\";\n        System.setProperty(MixAll.NAMESRV_ADDR_PROPERTY, \"foo\");\n        Path configFilePath = Files.createTempFile(\"testLocalMode\", \".json\");\n        String configData = \"{\\n\" +\n            \"  \\\"namesrvAddr\\\": \\\"foo\\\"\\n\" +\n            \"}\";\n        Files.write(configFilePath, configData.getBytes(StandardCharsets.UTF_8));\n        Path brokerConfigFilePath = Files.createTempFile(\"testLocalModeBrokerConfig\", \".json\");\n\n        CommandLineArgument commandLineArgument = ProxyStartup.parseCommandLineArgument(new String[] {\n            \"-pm\", \"local\",\n            \"-pc\", configFilePath.toAbsolutePath().toString(),\n            \"-n\", namesrvAddr,\n            \"-bc\", brokerConfigFilePath.toAbsolutePath().toString()\n        });\n        ProxyStartup.initConfiguration(commandLineArgument);\n\n        ProxyConfig config = ConfigurationManager.getProxyConfig();\n        assertEquals(namesrvAddr, config.getNamesrvAddr());\n        assertEquals(brokerConfigFilePath.toAbsolutePath().toString(), config.getBrokerConfigPath());\n\n        validateBrokerCreateArgsWithNamsrvAddr(config, namesrvAddr);\n    }\n\n    @Test\n    public void testClusterMode() throws Exception {\n        CommandLineArgument commandLineArgument = ProxyStartup.parseCommandLineArgument(new String[] {\n            \"-pm\", \"cluster\"\n        });\n        ProxyStartup.initConfiguration(commandLineArgument);\n\n        try (MockedStatic<DefaultMessagingProcessor> messagingProcessorMocked = mockStatic(DefaultMessagingProcessor.class)) {\n            DefaultMessagingProcessor processor = mock(DefaultMessagingProcessor.class);\n            messagingProcessorMocked.when(DefaultMessagingProcessor::createForClusterMode)\n                .thenReturn(processor);\n\n            assertSame(processor, ProxyStartup.createMessagingProcessor());\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/common/AddressTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.common;\n\nimport com.google.common.net.HostAndPort;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNull;\npublic class AddressTest {\n\n    @Test\n    public void testConstructorWithIPv4() {\n        HostAndPort hostAndPort = HostAndPort.fromString(\"192.168.1.1:8080\");\n        Address address = new Address(hostAndPort);\n\n        assertEquals(Address.AddressScheme.IPv4, address.getAddressScheme());\n        assertEquals(hostAndPort, address.getHostAndPort());\n    }\n\n    @Test\n    public void testConstructorWithIPv6() {\n        HostAndPort hostAndPort = HostAndPort.fromString(\"[2001:db8::1]:8080\");\n        Address address = new Address(hostAndPort);\n\n        assertEquals(Address.AddressScheme.IPv6, address.getAddressScheme());\n        assertEquals(hostAndPort, address.getHostAndPort());\n    }\n\n    @Test\n    public void testConstructorWithDomainName() {\n        HostAndPort hostAndPort = HostAndPort.fromString(\"example.com:8080\");\n        Address address = new Address(hostAndPort);\n\n        assertEquals(Address.AddressScheme.DOMAIN_NAME, address.getAddressScheme());\n        assertEquals(hostAndPort, address.getHostAndPort());\n    }\n\n    @Test\n    public void testConstructorWithNullHostAndPort() {\n        Address address = new Address(null);\n\n        assertEquals(Address.AddressScheme.UNRECOGNIZED, address.getAddressScheme());\n        assertNull(address.getHostAndPort());\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroupTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.common;\n\nimport java.time.Duration;\nimport java.util.Random;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicReference;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.utils.FutureUtils;\nimport org.apache.rocketmq.proxy.config.InitConfigTest;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertTrue;\n\npublic class ReceiptHandleGroupTest extends InitConfigTest {\n\n    private static final String TOPIC = \"topic\";\n    private static final String GROUP = \"group\";\n    private ReceiptHandleGroup receiptHandleGroup;\n    private String msgID;\n    private final Random random = new Random();\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        receiptHandleGroup = new ReceiptHandleGroup();\n        msgID = MessageClientIDSetter.createUniqID();\n    }\n\n    protected String createHandle() {\n        return ReceiptHandle.builder()\n            .startOffset(0L)\n            .retrieveTime(System.currentTimeMillis())\n            .invisibleTime(3000)\n            .reviveQueueId(1)\n            .topicType(ReceiptHandle.NORMAL_TOPIC)\n            .brokerName(\"brokerName\")\n            .queueId(random.nextInt(10))\n            .offset(random.nextInt(10))\n            .commitLogOffset(0L)\n            .build().encode();\n    }\n\n    @Test\n    public void testAddDuplicationHandle() {\n        String handle1 = ReceiptHandle.builder()\n            .startOffset(0L)\n            .retrieveTime(System.currentTimeMillis())\n            .invisibleTime(3000)\n            .reviveQueueId(1)\n            .topicType(ReceiptHandle.NORMAL_TOPIC)\n            .brokerName(\"brokerName\")\n            .queueId(1)\n            .offset(123)\n            .commitLogOffset(0L)\n            .build().encode();\n        String handle2 = ReceiptHandle.builder()\n            .startOffset(0L)\n            .retrieveTime(System.currentTimeMillis() + 1000)\n            .invisibleTime(3000)\n            .reviveQueueId(1)\n            .topicType(ReceiptHandle.NORMAL_TOPIC)\n            .brokerName(\"brokerName\")\n            .queueId(1)\n            .offset(123)\n            .commitLogOffset(0L)\n            .build().encode();\n\n        receiptHandleGroup.put(msgID, createMessageReceiptHandle(handle1, msgID));\n        receiptHandleGroup.put(msgID, createMessageReceiptHandle(handle2, msgID));\n\n        assertEquals(1, receiptHandleGroup.receiptHandleMap.get(msgID).size());\n    }\n\n    @Test\n    public void testGetWhenComputeIfPresent() {\n        String handle1 = createHandle();\n        String handle2 = createHandle();\n        AtomicReference<MessageReceiptHandle> getHandleRef = new AtomicReference<>();\n\n        receiptHandleGroup.put(msgID, createMessageReceiptHandle(handle1, msgID));\n        CountDownLatch latch = new CountDownLatch(2);\n        Thread getThread = new Thread(() -> {\n            try {\n                latch.countDown();\n                latch.await();\n                getHandleRef.set(receiptHandleGroup.get(msgID, handle1));\n            } catch (Exception ignored) {\n            }\n        }, \"getThread\");\n        Thread computeThread = new Thread(() -> {\n            try {\n                receiptHandleGroup.computeIfPresent(msgID, handle1, messageReceiptHandle -> {\n                    try {\n                        latch.countDown();\n                        latch.await();\n                    } catch (Exception ignored) {\n                    }\n                    messageReceiptHandle.updateReceiptHandle(handle2);\n                    return FutureUtils.addExecutor(CompletableFuture.completedFuture(messageReceiptHandle), Executors.newCachedThreadPool());\n                });\n            } catch (Exception ignored) {\n            }\n        }, \"computeThread\");\n        getThread.start();\n        computeThread.start();\n\n        await().atMost(Duration.ofSeconds(1)).until(() -> getHandleRef.get() != null);\n        assertEquals(handle2, getHandleRef.get().getReceiptHandleStr());\n        assertFalse(receiptHandleGroup.isEmpty());\n    }\n\n    @Test\n    public void testGetWhenComputeIfPresentReturnNull() {\n        String handle1 = createHandle();\n        AtomicBoolean getCalled = new AtomicBoolean(false);\n        AtomicReference<MessageReceiptHandle> getHandleRef = new AtomicReference<>();\n\n        receiptHandleGroup.put(msgID, createMessageReceiptHandle(handle1, msgID));\n        CountDownLatch latch = new CountDownLatch(2);\n        Thread getThread = new Thread(() -> {\n            try {\n                latch.countDown();\n                latch.await();\n                getHandleRef.set(receiptHandleGroup.get(msgID, handle1));\n                getCalled.set(true);\n            } catch (Exception ignored) {\n            }\n        }, \"getThread\");\n        Thread computeThread = new Thread(() -> {\n            try {\n                receiptHandleGroup.computeIfPresent(msgID, handle1, messageReceiptHandle -> {\n                    try {\n                        latch.countDown();\n                        latch.await();\n                    } catch (Exception ignored) {\n                    }\n                    return FutureUtils.addExecutor(CompletableFuture.completedFuture(null), Executors.newCachedThreadPool());\n                });\n            } catch (Exception ignored) {\n            }\n        }, \"computeThread\");\n        getThread.start();\n        computeThread.start();\n\n        await().atMost(Duration.ofSeconds(1)).until(getCalled::get);\n        assertNull(getHandleRef.get());\n        assertTrue(receiptHandleGroup.isEmpty());\n    }\n\n    @Test\n    public void testRemoveWhenComputeIfPresent() {\n        String handle1 = createHandle();\n        String handle2 = createHandle();\n        AtomicReference<MessageReceiptHandle> removeHandleRef = new AtomicReference<>();\n\n        receiptHandleGroup.put(msgID, createMessageReceiptHandle(handle1, msgID));\n        CountDownLatch latch = new CountDownLatch(2);\n        Thread removeThread = new Thread(() -> {\n            try {\n                latch.countDown();\n                latch.await();\n                removeHandleRef.set(receiptHandleGroup.remove(msgID, handle1));\n            } catch (Exception ignored) {\n            }\n        }, \"removeThread\");\n        Thread computeThread = new Thread(() -> {\n            try {\n                receiptHandleGroup.computeIfPresent(msgID, handle1, messageReceiptHandle -> {\n                    try {\n                        latch.countDown();\n                        latch.await();\n                    } catch (Exception ignored) {\n                    }\n                    messageReceiptHandle.updateReceiptHandle(handle2);\n                    return FutureUtils.addExecutor(CompletableFuture.completedFuture(messageReceiptHandle), Executors.newCachedThreadPool());\n                });\n            } catch (Exception ignored) {\n            }\n        }, \"computeThread\");\n        removeThread.start();\n        computeThread.start();\n\n        await().atMost(Duration.ofSeconds(1)).until(() -> removeHandleRef.get() != null);\n        assertEquals(handle2, removeHandleRef.get().getReceiptHandleStr());\n        assertTrue(receiptHandleGroup.isEmpty());\n    }\n\n    @Test\n    public void testRemoveWhenComputeIfPresentReturnNull() {\n        String handle1 = createHandle();\n        AtomicBoolean removeCalled = new AtomicBoolean(false);\n        AtomicReference<MessageReceiptHandle> removeHandleRef = new AtomicReference<>();\n\n        receiptHandleGroup.put(msgID, createMessageReceiptHandle(handle1, msgID));\n        CountDownLatch latch = new CountDownLatch(2);\n        Thread removeThread = new Thread(() -> {\n            try {\n                latch.countDown();\n                latch.await();\n                removeHandleRef.set(receiptHandleGroup.remove(msgID, handle1));\n                removeCalled.set(true);\n            } catch (Exception ignored) {\n            }\n        }, \"removeThread\");\n        Thread computeThread = new Thread(() -> {\n            try {\n                receiptHandleGroup.computeIfPresent(msgID, handle1, messageReceiptHandle -> {\n                    try {\n                        latch.countDown();\n                        latch.await();\n                    } catch (Exception ignored) {\n                    }\n                    return FutureUtils.addExecutor(CompletableFuture.completedFuture(null), Executors.newCachedThreadPool());\n                });\n            } catch (Exception ignored) {\n            }\n        }, \"computeThread\");\n        removeThread.start();\n        computeThread.start();\n\n        await().atMost(Duration.ofSeconds(1)).until(removeCalled::get);\n        assertNull(removeHandleRef.get());\n        assertTrue(receiptHandleGroup.isEmpty());\n    }\n\n    @Test\n    public void testRemoveMultiThread() {\n        String handle1 = createHandle();\n        AtomicReference<MessageReceiptHandle> removeHandleRef = new AtomicReference<>();\n        AtomicInteger count = new AtomicInteger();\n\n        receiptHandleGroup.put(msgID, createMessageReceiptHandle(handle1, msgID));\n        int threadNum = Math.max(Runtime.getRuntime().availableProcessors(), 3);\n        CountDownLatch latch = new CountDownLatch(threadNum);\n        for (int i = 0; i < threadNum; i++) {\n            Thread thread = new Thread(() -> {\n                try {\n                    latch.countDown();\n                    latch.await();\n                    MessageReceiptHandle handle = receiptHandleGroup.remove(msgID, handle1);\n                    if (handle != null) {\n                        removeHandleRef.set(handle);\n                        count.incrementAndGet();\n                    }\n                } catch (Exception ignored) {\n                }\n            });\n            thread.start();\n        }\n\n        await().atMost(Duration.ofSeconds(1)).untilAsserted(() -> assertEquals(1, count.get()));\n        assertEquals(handle1, removeHandleRef.get().getReceiptHandleStr());\n        assertTrue(receiptHandleGroup.isEmpty());\n    }\n\n    @Test\n    public void testRemoveOne() {\n        String handle1 = createHandle();\n        AtomicReference<MessageReceiptHandle> removeHandleRef = new AtomicReference<>();\n        AtomicInteger count = new AtomicInteger();\n\n        receiptHandleGroup.put(msgID, createMessageReceiptHandle(handle1, msgID));\n        int threadNum = Math.max(Runtime.getRuntime().availableProcessors(), 3);\n        CountDownLatch latch = new CountDownLatch(threadNum);\n        for (int i = 0; i < threadNum; i++) {\n            Thread thread = new Thread(() -> {\n                try {\n                    latch.countDown();\n                    latch.await();\n                    MessageReceiptHandle handle = receiptHandleGroup.removeOne(msgID);\n                    if (handle != null) {\n                        removeHandleRef.set(handle);\n                        count.incrementAndGet();\n                    }\n                } catch (Exception ignored) {\n                }\n            });\n            thread.start();\n        }\n\n        await().atMost(Duration.ofSeconds(1)).untilAsserted(() -> assertEquals(1, count.get()));\n        assertEquals(handle1, removeHandleRef.get().getReceiptHandleStr());\n        assertTrue(receiptHandleGroup.isEmpty());\n    }\n\n    private MessageReceiptHandle createMessageReceiptHandle(String handle, String msgID) {\n        return new MessageReceiptHandle(GROUP, TOPIC, 0, handle, msgID, 0, 0);\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/common/RenewStrategyPolicyTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.common;\n\nimport org.apache.rocketmq.remoting.protocol.subscription.RetryPolicy;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicInteger;\n\nimport static org.junit.Assert.assertEquals;\n\npublic class RenewStrategyPolicyTest {\n\n    private RetryPolicy retryPolicy;\n    private final AtomicInteger times = new AtomicInteger(0);\n\n    @Before\n    public void before() throws Throwable {\n        this.retryPolicy = new RenewStrategyPolicy();\n    }\n\n    @Test\n    public void testNextDelayDuration() {\n        long value = this.retryPolicy.nextDelayDuration(times.getAndIncrement());\n        assertEquals(value, TimeUnit.MINUTES.toMillis(1));\n\n        value = this.retryPolicy.nextDelayDuration(times.getAndIncrement());\n        assertEquals(value, TimeUnit.MINUTES.toMillis(3));\n\n        value = this.retryPolicy.nextDelayDuration(times.getAndIncrement());\n        assertEquals(value, TimeUnit.MINUTES.toMillis(5));\n\n        value = this.retryPolicy.nextDelayDuration(times.getAndIncrement());\n        assertEquals(value, TimeUnit.MINUTES.toMillis(10));\n\n        value = this.retryPolicy.nextDelayDuration(times.getAndIncrement());\n        assertEquals(value, TimeUnit.MINUTES.toMillis(30));\n\n        value = this.retryPolicy.nextDelayDuration(times.getAndIncrement());\n        assertEquals(value, TimeUnit.HOURS.toMillis(1));\n    }\n\n\n    @After\n    public void after() {\n    }\n\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/common/utils/FilterUtilTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.common.utils;\n\nimport org.apache.rocketmq.remoting.protocol.filter.FilterAPI;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class FilterUtilTest {\n    @Test\n    public void testTagMatched() throws Exception {\n        SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(\"\", \"tagA\");\n        assertThat(FilterUtils.isTagMatched(subscriptionData.getTagsSet(), \"tagA\")).isTrue();\n    }\n\n    @Test\n    public void testTagNotMatched() throws Exception {\n        SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(\"\", \"tagA\");\n        assertThat(FilterUtils.isTagMatched(subscriptionData.getTagsSet(), \"tagB\")).isFalse();\n    }\n\n    @Test\n    public void testTagMatchedStar() throws Exception {\n        SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(\"\", \"*\");\n        assertThat(FilterUtils.isTagMatched(subscriptionData.getTagsSet(), \"tagA\")).isTrue();\n    }\n\n    @Test\n    public void testTagNotMatchedNull() throws Exception {\n        SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(\"\", \"tagA\");\n        assertThat(FilterUtils.isTagMatched(subscriptionData.getTagsSet(), null)).isFalse();\n    }\n\n    @Test\n    public void testBuildSubscriptionData() throws Exception {\n        // Test case 1: expressionType is null, will use TAG as default.\n        String topic = \"topic\";\n        String subString = \"substring\";\n        String expressionType = null;\n        SubscriptionData result = FilterAPI.buildSubscriptionData(topic, subString, expressionType);\n        assertThat(result).isNotNull();\n        assertThat(topic).isEqualTo(result.getTopic());\n        assertThat(subString).isEqualTo(result.getSubString());\n        assertThat(result.getExpressionType()).isEqualTo(\"TAG\");\n        assertThat(result.getCodeSet().size()).isEqualTo(1);\n\n        // Test case 2: expressionType is not null\n        topic = \"topic\";\n        subString = \"substring1||substring2\";\n        expressionType = \"SQL92\";\n        result = FilterAPI.buildSubscriptionData(topic, subString, expressionType);\n        assertThat(result).isNotNull();\n        assertThat(topic).isEqualTo(result.getTopic());\n        assertThat(subString).isEqualTo(result.getSubString());\n        assertThat(result.getExpressionType()).isEqualTo(expressionType);\n        assertThat(result.getCodeSet().size()).isEqualTo(2);\n    }\n\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/config/ConfigurationManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.config;\n\nimport org.apache.rocketmq.proxy.ProxyMode;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertTrue;\n\npublic class ConfigurationManagerTest extends InitConfigTest {\n\n    @Test\n    public void testInitConfig() {\n        assertThat(ConfigurationManager.getProxyConfig()).isNotNull();\n        assertThat(ConfigurationManager.getProxyConfig().getProxyMode()).isEqualToIgnoringCase(ProxyMode.CLUSTER.toString());\n\n        String brokerConfig = ConfigurationManager.getProxyConfig().getBrokerConfigPath();\n        assertThat(brokerConfig).isEqualTo(ConfigurationManager.getProxyHome() + \"/conf/broker.conf\");\n    }\n\n    @Test\n    public void testGetProxyHome() {\n        // test configured proxy home\n        assertThat(ConfigurationManager.getProxyHome()).isIn(mockProxyHome, \"./\");\n    }\n\n    @Test\n    public void testGetProxyConfig() {\n        assertThat(ConfigurationManager.getProxyConfig()).isNotNull();\n    }\n\n    @Test\n    public void testFormatProxyConfig() {\n        String actual = ConfigurationManager.formatProxyConfig();\n        assertNotNull(actual);\n        ProxyConfig expected = ConfigurationManager.getProxyConfig();\n        assertTrue(actual.contains(expected.getProxyMode()));\n        assertTrue(actual.contains(expected.getProxyName()));\n    }\n} \n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/config/ConfigurationTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.config;\n\nimport org.apache.rocketmq.auth.config.AuthConfig;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertNotNull;\nimport static org.mockito.Mockito.spy;\n\npublic class ConfigurationTest {\n\n    private Configuration configuration;\n\n    @Before\n    public void init() {\n        configuration = spy(new Configuration());\n    }\n\n    @Test\n    public void testInit() throws Exception {\n        configuration.init();\n\n        ProxyConfig loadedProxyConfig = configuration.getProxyConfig();\n        assertNotNull(loadedProxyConfig);\n\n        AuthConfig loadedAuthConfig = configuration.getAuthConfig();\n        assertNotNull(loadedAuthConfig);\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/config/InitConfigTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.config;\n\nimport java.net.URL;\nimport org.assertj.core.util.Strings;\n\nimport org.junit.After;\nimport org.junit.Before;\n\nimport static org.apache.rocketmq.proxy.config.ConfigurationManager.RMQ_PROXY_HOME;\n\npublic class InitConfigTest {\n    public static String mockProxyHome = \"/mock/rmq/proxy/home\";\n\n    @Before\n    public void before() throws Throwable {\n        URL mockProxyHomeURL = getClass().getClassLoader().getResource(\"rmq-proxy-home\");\n        if (mockProxyHomeURL != null) {\n            mockProxyHome = mockProxyHomeURL.toURI().getPath();\n        }\n\n        if (!Strings.isNullOrEmpty(mockProxyHome)) {\n            System.setProperty(RMQ_PROXY_HOME, mockProxyHome);\n        }\n\n        ConfigurationManager.initEnv();\n        ConfigurationManager.initConfig();\n    }\n\n    @After\n    public void after() {\n        System.clearProperty(RMQ_PROXY_HOME);\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/config/MetricCollectorModeTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.config;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class MetricCollectorModeTest {\n\n    @Test\n    public void testGetEnumByOrdinal() {\n        Assert.assertEquals(MetricCollectorMode.OFF, MetricCollectorMode.getEnumByString(\"off\"));\n        Assert.assertEquals(MetricCollectorMode.ON, MetricCollectorMode.getEnumByString(\"on\"));\n        Assert.assertEquals(MetricCollectorMode.PROXY, MetricCollectorMode.getEnumByString(\"proxy\"));\n\n        Assert.assertEquals(MetricCollectorMode.OFF, MetricCollectorMode.getEnumByString(\"OFF\"));\n        Assert.assertEquals(MetricCollectorMode.ON, MetricCollectorMode.getEnumByString(\"ON\"));\n        Assert.assertEquals(MetricCollectorMode.PROXY, MetricCollectorMode.getEnumByString(\"PROXY\"));\n    }\n\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiatorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\npackage org.apache.rocketmq.proxy.grpc;\n\nimport io.grpc.Attributes;\nimport io.grpc.netty.shaded.io.netty.buffer.ByteBuf;\nimport io.grpc.netty.shaded.io.netty.buffer.Unpooled;\nimport io.grpc.netty.shaded.io.netty.handler.codec.haproxy.HAProxyTLV;\nimport java.io.File;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.nio.file.StandardCopyOption;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.junit.MockitoJUnitRunner;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ProxyAndTlsProtocolNegotiatorTest {\n\n    private ProxyAndTlsProtocolNegotiator negotiator;\n\n    @Before\n    public void setUp() throws Exception {\n        ConfigurationManager.initConfig();\n        ConfigurationManager.getProxyConfig().setTlsTestModeEnable(true);\n        negotiator = new ProxyAndTlsProtocolNegotiator();\n    }\n\n    @After\n    public void tearDown() {\n        ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();\n        proxyConfig.setTlsTestModeEnable(true);\n        proxyConfig.setTlsKeyPath(\"\");\n        proxyConfig.setTlsCertPath(\"\");\n        proxyConfig.setTlsKeyPassword(\"\");\n    }\n\n    @Test\n    public void handleHAProxyTLV() {\n        ByteBuf content = Unpooled.buffer();\n        content.writeBytes(\"xxxx\".getBytes(StandardCharsets.UTF_8));\n        HAProxyTLV haProxyTLV = new HAProxyTLV((byte) 0xE1, content);\n        negotiator.handleHAProxyTLV(haProxyTLV, Attributes.newBuilder());\n    }\n\n    @Test\n    public void testLoadSslContextWithUnencryptedKey() throws Exception {\n        configureTls(\"server.key\", \"server.pem\", \"\");\n        ProxyAndTlsProtocolNegotiator.loadSslContext();\n    }\n\n    @Test\n    public void testLoadSslContextWithEncryptedKey() throws Exception {\n        // \"1234\" is the password of certs/client.key, inherited from remoting module test resources\n        configureTls(\"client.key\", \"client.pem\", \"1234\");\n        ProxyAndTlsProtocolNegotiator.loadSslContext();\n    }\n\n    @Test(expected = IllegalArgumentException.class)\n    public void testLoadSslContextWithWrongPassword() throws Exception {\n        configureTls(\"client.key\", \"client.pem\", \"wrong_password\");\n        ProxyAndTlsProtocolNegotiator.loadSslContext();\n    }\n\n    private void configureTls(String keyFile, String certFile, String password) throws IOException {\n        ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();\n        proxyConfig.setTlsTestModeEnable(false);\n        proxyConfig.setTlsKeyPath(getCertsPath(keyFile));\n        proxyConfig.setTlsCertPath(getCertsPath(certFile));\n        proxyConfig.setTlsKeyPassword(password);\n    }\n\n    private static String getCertsPath(String fileName) throws IOException {\n        File tempFile = File.createTempFile(fileName, null);\n        tempFile.deleteOnExit();\n        try (InputStream is = ProxyAndTlsProtocolNegotiatorTest.class\n            .getClassLoader().getResourceAsStream(\"certs/\" + fileName)) {\n            Files.copy(is, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING);\n        }\n        return tempFile.getAbsolutePath();\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/AbstractMessagingActivityTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2;\n\nimport apache.rocketmq.v2.Resource;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.proxy.config.InitConfigTest;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertThrows;\n\npublic class AbstractMessagingActivityTest extends InitConfigTest {\n\n    public static class MockMessagingActivity extends AbstractMessagingActivity {\n\n        public MockMessagingActivity(MessagingProcessor messagingProcessor,\n                                     GrpcClientSettingsManager grpcClientSettingsManager,\n                                     GrpcChannelManager grpcChannelManager) {\n            super(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n        }\n    }\n\n    private AbstractMessagingActivity messagingActivity;\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        this.messagingActivity = new MockMessagingActivity(null, null, null);\n    }\n\n    @Test\n    public void testValidateTopic() {\n        assertThrows(GrpcProxyException.class, () -> messagingActivity.validateTopic(Resource.newBuilder().build()));\n        assertThrows(GrpcProxyException.class, () -> messagingActivity.validateTopic(Resource.newBuilder().setName(TopicValidator.RMQ_SYS_TRACE_TOPIC).build()));\n        assertThrows(GrpcProxyException.class, () -> messagingActivity.validateTopic(Resource.newBuilder().setName(\"@\").build()));\n        assertThrows(GrpcProxyException.class, () -> messagingActivity.validateTopic(Resource.newBuilder().setName(createString(128)).build()));\n        messagingActivity.validateTopic(Resource.newBuilder().setName(createString(127)).build());\n    }\n\n    @Test\n    public void testValidateConsumer() {\n        assertThrows(GrpcProxyException.class, () -> messagingActivity.validateConsumerGroup(Resource.newBuilder().build()));\n        assertThrows(GrpcProxyException.class, () -> messagingActivity.validateConsumerGroup(Resource.newBuilder().setName(MixAll.CID_SYS_RMQ_TRANS).build()));\n        assertThrows(GrpcProxyException.class, () -> messagingActivity.validateConsumerGroup(Resource.newBuilder().setName(\"@\").build()));\n        assertThrows(GrpcProxyException.class, () -> messagingActivity.validateConsumerGroup(Resource.newBuilder().setName(createString(256)).build()));\n        messagingActivity.validateConsumerGroup(Resource.newBuilder().setName(createString(120)).build());\n    }\n\n    private static String createString(int len) {\n        StringBuilder sb = new StringBuilder();\n        for (int i = 0; i < len; i++) {\n            sb.append('a');\n        }\n        return sb.toString();\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/BaseActivityTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2;\n\nimport io.grpc.Metadata;\nimport java.time.Duration;\nimport java.util.Random;\nimport java.util.UUID;\nimport org.apache.rocketmq.proxy.common.ContextVariable;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.InitConfigTest;\nimport org.apache.rocketmq.common.constant.GrpcConstants;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.processor.ReceiptHandleProcessor;\nimport org.apache.rocketmq.proxy.service.metadata.MetadataService;\nimport org.apache.rocketmq.proxy.service.relay.ProxyRelayService;\nimport org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;\nimport org.junit.Ignore;\nimport org.junit.runner.RunWith;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@Ignore\n@RunWith(MockitoJUnitRunner.Silent.class)\npublic class BaseActivityTest extends InitConfigTest {\n    protected static final Random RANDOM = new Random();\n    protected MessagingProcessor messagingProcessor;\n    protected GrpcClientSettingsManager grpcClientSettingsManager;\n    protected GrpcChannelManager grpcChannelManager;\n    protected ProxyRelayService proxyRelayService;\n    protected ReceiptHandleProcessor receiptHandleProcessor;\n    protected MetadataService metadataService;\n\n    protected static final String REMOTE_ADDR = \"192.168.0.1:8080\";\n    protected static final String LOCAL_ADDR = \"127.0.0.1:8080\";\n    protected Metadata metadata = new Metadata();\n\n    protected static final String CLIENT_ID = \"client-id\" + UUID.randomUUID();\n    protected static final String JAVA = \"JAVA\";\n\n    public void before() throws Throwable {\n        super.before();\n        messagingProcessor = mock(MessagingProcessor.class);\n        grpcClientSettingsManager = mock(GrpcClientSettingsManager.class);\n        proxyRelayService = mock(ProxyRelayService.class);\n        receiptHandleProcessor = mock(ReceiptHandleProcessor.class);\n        metadataService = mock(MetadataService.class);\n\n        metadata.put(GrpcConstants.CLIENT_ID, CLIENT_ID);\n        metadata.put(GrpcConstants.LANGUAGE, JAVA);\n        metadata.put(GrpcConstants.REMOTE_ADDRESS, REMOTE_ADDR);\n        metadata.put(GrpcConstants.LOCAL_ADDRESS, LOCAL_ADDR);\n        when(messagingProcessor.getProxyRelayService()).thenReturn(proxyRelayService);\n        when(messagingProcessor.getMetadataService()).thenReturn(metadataService);\n        grpcChannelManager = new GrpcChannelManager(messagingProcessor.getProxyRelayService(), grpcClientSettingsManager);\n    }\n\n    protected ProxyContext createContext() {\n        return ProxyContext.create()\n            .withVal(ContextVariable.CLIENT_ID, CLIENT_ID)\n            .withVal(ContextVariable.LANGUAGE, JAVA)\n            .withVal(ContextVariable.REMOTE_ADDRESS, REMOTE_ADDR)\n            .withVal(ContextVariable.LOCAL_ADDRESS, LOCAL_ADDR)\n            .withVal(ContextVariable.REMAINING_MS, Duration.ofSeconds(10).toMillis());\n    }\n\n    protected static String buildReceiptHandle(String topic, long popTime, long invisibleTime) {\n        return ExtraInfoUtil.buildExtraInfo(\n            RANDOM.nextInt(Integer.MAX_VALUE),\n            popTime,\n            invisibleTime,\n            0,\n            topic,\n            \"brokerName\",\n            RANDOM.nextInt(8),\n            RANDOM.nextInt(Integer.MAX_VALUE)\n        );\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplicationTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2;\n\nimport apache.rocketmq.v2.Address;\nimport apache.rocketmq.v2.AddressScheme;\nimport apache.rocketmq.v2.Code;\nimport apache.rocketmq.v2.Endpoints;\nimport apache.rocketmq.v2.MessageQueue;\nimport apache.rocketmq.v2.QueryRouteRequest;\nimport apache.rocketmq.v2.QueryRouteResponse;\nimport apache.rocketmq.v2.Resource;\nimport io.grpc.Context;\nimport io.grpc.Metadata;\nimport io.grpc.stub.StreamObserver;\nimport java.util.UUID;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.common.constant.GrpcConstants;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.InitConfigTest;\nimport org.apache.rocketmq.proxy.grpc.pipeline.ContextInitPipeline;\nimport org.apache.rocketmq.proxy.grpc.pipeline.RequestPipeline;\nimport org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.ArgumentCaptor;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertEquals;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class GrpcMessagingApplicationTest extends InitConfigTest {\n    protected static final String REMOTE_ADDR = \"192.168.0.1:8080\";\n    protected static final String LOCAL_ADDR = \"127.0.0.1:8080\";\n    protected static final String CLIENT_ID = \"client-id\" + UUID.randomUUID();\n    protected static final String JAVA = \"JAVA\";\n    @Mock\n    StreamObserver<QueryRouteResponse> queryRouteResponseStreamObserver;\n    @Mock\n    GrpcMessagingActivity grpcMessagingActivity;\n    GrpcMessagingApplication grpcMessagingApplication;\n\n    private static final String TOPIC = \"topic\";\n    private static Endpoints grpcEndpoints = Endpoints.newBuilder()\n        .setScheme(AddressScheme.IPv4)\n        .addAddresses(Address.newBuilder().setHost(\"127.0.0.1\").setPort(8080).build())\n        .addAddresses(Address.newBuilder().setHost(\"127.0.0.2\").setPort(8080).build())\n        .build();\n\n    @Before\n    public void setUp() throws Throwable {\n        super.before();\n        RequestPipeline pipeline = (context, headers, request) -> {\n        };\n        pipeline = pipeline.pipe(new ContextInitPipeline());\n        grpcMessagingApplication = new GrpcMessagingApplication(grpcMessagingActivity, pipeline);\n    }\n\n    @Test\n    public void testQueryRoute() {\n        Metadata metadata = new Metadata();\n        metadata.put(GrpcConstants.CLIENT_ID, CLIENT_ID);\n        metadata.put(GrpcConstants.LANGUAGE, JAVA);\n        metadata.put(GrpcConstants.REMOTE_ADDRESS, REMOTE_ADDR);\n        metadata.put(GrpcConstants.LOCAL_ADDRESS, LOCAL_ADDR);\n        \n        Assert.assertNotNull(Context.current()\n            .withValue(GrpcConstants.METADATA, metadata)\n            .attach());\n\n        CompletableFuture<QueryRouteResponse> future = new CompletableFuture<>();\n        QueryRouteRequest request = QueryRouteRequest.newBuilder()\n            .setEndpoints(grpcEndpoints)\n            .setTopic(Resource.newBuilder().setName(TOPIC).build())\n            .build();\n        Mockito.when(grpcMessagingActivity.queryRoute(Mockito.any(ProxyContext.class), Mockito.eq(request)))\n            .thenReturn(future);\n        QueryRouteResponse response = QueryRouteResponse.newBuilder()\n            .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()))\n            .addMessageQueues(MessageQueue.getDefaultInstance())\n            .build();\n        grpcMessagingApplication.queryRoute(request, queryRouteResponseStreamObserver);\n        future.complete(response);\n        await().untilAsserted(() -> {\n            Mockito.verify(queryRouteResponseStreamObserver, Mockito.times(1)).onNext(Mockito.same(response));\n        });\n    }\n\n    @Test\n    public void testQueryRouteWithBadClientID() {\n        Metadata metadata = new Metadata();\n        metadata.put(GrpcConstants.LANGUAGE, JAVA);\n        metadata.put(GrpcConstants.REMOTE_ADDRESS, REMOTE_ADDR);\n        metadata.put(GrpcConstants.LOCAL_ADDRESS, LOCAL_ADDR);\n\n        Assert.assertNotNull(Context.current()\n            .withValue(GrpcConstants.METADATA, metadata)\n            .attach());\n\n        QueryRouteRequest request = QueryRouteRequest.newBuilder()\n            .setEndpoints(grpcEndpoints)\n            .setTopic(Resource.newBuilder().setName(TOPIC).build())\n            .build();\n        grpcMessagingApplication.queryRoute(request, queryRouteResponseStreamObserver);\n\n        ArgumentCaptor<QueryRouteResponse> responseArgumentCaptor = ArgumentCaptor.forClass(QueryRouteResponse.class);\n        await().untilAsserted(() -> {\n            Mockito.verify(queryRouteResponseStreamObserver, Mockito.times(1)).onNext(responseArgumentCaptor.capture());\n        });\n\n        assertEquals(Code.CLIENT_ID_REQUIRED, responseArgumentCaptor.getValue().getStatus().getCode());\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannelTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2.channel;\n\nimport apache.rocketmq.v2.Publishing;\nimport apache.rocketmq.v2.Resource;\nimport apache.rocketmq.v2.Settings;\nimport org.apache.commons.lang3.RandomStringUtils;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.InitConfigTest;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;\nimport org.apache.rocketmq.proxy.processor.channel.ChannelProtocolType;\nimport org.apache.rocketmq.proxy.processor.channel.RemoteChannel;\nimport org.apache.rocketmq.proxy.remoting.channel.RemotingChannel;\nimport org.apache.rocketmq.proxy.service.relay.ProxyRelayService;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNull;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class GrpcClientChannelTest extends InitConfigTest {\n\n    @Mock\n    private ProxyRelayService proxyRelayService;\n    @Mock\n    private GrpcClientSettingsManager grpcClientSettingsManager;\n    @Mock\n    private GrpcChannelManager grpcChannelManager;\n\n    private String clientId;\n    private GrpcClientChannel grpcClientChannel;\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        this.clientId = RandomStringUtils.randomAlphabetic(10);\n        this.grpcClientChannel = new GrpcClientChannel(proxyRelayService, grpcClientSettingsManager, grpcChannelManager,\n            ProxyContext.create().setRemoteAddress(\"10.152.39.53:9768\").setLocalAddress(\"11.193.0.1:1210\"),\n            this.clientId);\n    }\n\n    @Test\n    public void testChannelExtendAttributeParse() {\n        Settings clientSettings = Settings.newBuilder()\n            .setPublishing(Publishing.newBuilder()\n                .addTopics(Resource.newBuilder()\n                    .setName(\"topic\")\n                    .build())\n                .build())\n            .build();\n        when(grpcClientSettingsManager.getRawClientSettings(eq(clientId))).thenReturn(clientSettings);\n\n        RemoteChannel remoteChannel = this.grpcClientChannel.toRemoteChannel();\n        assertEquals(ChannelProtocolType.GRPC_V2, remoteChannel.getType());\n        assertEquals(clientSettings, GrpcClientChannel.parseChannelExtendAttribute(remoteChannel));\n        assertEquals(clientSettings, GrpcClientChannel.parseChannelExtendAttribute(this.grpcClientChannel));\n        assertNull(GrpcClientChannel.parseChannelExtendAttribute(mock(RemotingChannel.class)));\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/client/ClientActivityTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2.client;\n\nimport apache.rocketmq.v2.ClientType;\nimport apache.rocketmq.v2.Code;\nimport apache.rocketmq.v2.FilterExpression;\nimport apache.rocketmq.v2.FilterType;\nimport apache.rocketmq.v2.HeartbeatRequest;\nimport apache.rocketmq.v2.HeartbeatResponse;\nimport apache.rocketmq.v2.LiteSubscriptionAction;\nimport apache.rocketmq.v2.NotifyClientTerminationRequest;\nimport apache.rocketmq.v2.NotifyClientTerminationResponse;\nimport apache.rocketmq.v2.Publishing;\nimport apache.rocketmq.v2.Resource;\nimport apache.rocketmq.v2.Settings;\nimport apache.rocketmq.v2.Subscription;\nimport apache.rocketmq.v2.SubscriptionEntry;\nimport apache.rocketmq.v2.SyncLiteSubscriptionRequest;\nimport apache.rocketmq.v2.SyncLiteSubscriptionResponse;\nimport apache.rocketmq.v2.TelemetryCommand;\nimport apache.rocketmq.v2.ThreadStackTrace;\nimport apache.rocketmq.v2.VerifyMessageResult;\nimport io.grpc.Status;\nimport io.grpc.StatusRuntimeException;\nimport io.grpc.stub.StreamObserver;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ExecutionException;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.lite.LiteSubscriptionDTO;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest;\nimport org.apache.rocketmq.proxy.grpc.v2.ContextStreamObserver;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcClientChannel;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcValidator;\nimport org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder;\nimport org.apache.rocketmq.proxy.service.relay.ProxyRelayResult;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.CMResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.assertj.core.util.Lists;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.ArgumentCaptor;\nimport org.mockito.Captor;\nimport org.mockito.Mock;\nimport org.mockito.MockedStatic;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\nimport static org.junit.Assert.fail;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyBoolean;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.doNothing;\nimport static org.mockito.Mockito.doThrow;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.mockStatic;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ClientActivityTest extends BaseActivityTest {\n\n    private static final String TOPIC = \"topic\";\n    private static final String CONSUMER_GROUP = \"consumerGroup\";\n\n    private ClientActivity clientActivity;\n    @Mock\n    private GrpcChannelManager grpcChannelManagerMock;\n    @Mock\n    private CompletableFuture<ProxyRelayResult<ConsumerRunningInfo>> runningInfoFutureMock;\n    @Captor\n    ArgumentCaptor<ProxyRelayResult<ConsumerRunningInfo>> runningInfoArgumentCaptor;\n    @Mock\n    private CompletableFuture<ProxyRelayResult<ConsumeMessageDirectlyResult>> resultFutureMock;\n    @Captor\n    ArgumentCaptor<ProxyRelayResult<ConsumeMessageDirectlyResult>> resultArgumentCaptor;\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        this.clientActivity = new ClientActivity(this.messagingProcessor, this.grpcClientSettingsManager, grpcChannelManager);\n    }\n\n    protected TelemetryCommand sendProducerTelemetry(ProxyContext context) throws Throwable {\n        return this.sendClientTelemetry(\n            context,\n            Settings.newBuilder()\n                .setClientType(ClientType.PRODUCER)\n                .setPublishing(Publishing.newBuilder()\n                    .addTopics(Resource.newBuilder().setName(TOPIC).build())\n                    .build())\n                .build()).get();\n    }\n\n    protected HeartbeatResponse sendProducerHeartbeat(ProxyContext context) throws Throwable {\n        return this.clientActivity.heartbeat(context, HeartbeatRequest.newBuilder()\n            .setClientType(ClientType.PRODUCER)\n            .build()).get();\n    }\n\n    @Test\n    public void testProducerHeartbeat() throws Throwable {\n        ProxyContext context = createContext();\n\n        this.sendProducerTelemetry(context);\n\n        ArgumentCaptor<String> registerProducerGroupArgumentCaptor = ArgumentCaptor.forClass(String.class);\n        ArgumentCaptor<ClientChannelInfo> channelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class);\n        doNothing().when(this.messagingProcessor).registerProducer(any(),\n            registerProducerGroupArgumentCaptor.capture(),\n            channelInfoArgumentCaptor.capture());\n\n        ArgumentCaptor<String> txProducerGroupArgumentCaptor = ArgumentCaptor.forClass(String.class);\n        ArgumentCaptor<String> txProducerTopicArgumentCaptor = ArgumentCaptor.forClass(String.class);\n        doNothing().when(this.messagingProcessor).addTransactionSubscription(any(),\n            txProducerGroupArgumentCaptor.capture(),\n            txProducerTopicArgumentCaptor.capture()\n        );\n\n        when(this.metadataService.getTopicMessageType(any(), anyString())).thenReturn(TopicMessageType.TRANSACTION);\n\n        HeartbeatResponse response = this.sendProducerHeartbeat(context);\n\n        assertEquals(Code.OK, response.getStatus().getCode());\n\n        assertEquals(Lists.newArrayList(TOPIC), registerProducerGroupArgumentCaptor.getAllValues());\n        ClientChannelInfo clientChannelInfo = channelInfoArgumentCaptor.getValue();\n        assertClientChannelInfo(clientChannelInfo, TOPIC);\n\n        assertEquals(Lists.newArrayList(TOPIC), txProducerGroupArgumentCaptor.getAllValues());\n        assertEquals(Lists.newArrayList(TOPIC), txProducerTopicArgumentCaptor.getAllValues());\n    }\n\n    protected TelemetryCommand sendConsumerTelemetry(ProxyContext context) throws Throwable {\n        return this.sendClientTelemetry(\n            context,\n            Settings.newBuilder()\n                .setClientType(ClientType.PUSH_CONSUMER)\n                .setSubscription(Subscription.newBuilder()\n                    .setGroup(Resource.newBuilder().setName(\"Group\").build())\n                    .addSubscriptions(SubscriptionEntry.newBuilder()\n                        .setExpression(FilterExpression.newBuilder()\n                            .setExpression(\"tag\")\n                            .setType(FilterType.TAG)\n                            .build())\n                        .setTopic(Resource.newBuilder().setName(TOPIC).build())\n                        .build())\n                    .build())\n                .build()).get();\n    }\n\n    protected HeartbeatResponse sendConsumerHeartbeat(ProxyContext context) throws Throwable {\n        return this.clientActivity.heartbeat(context, HeartbeatRequest.newBuilder()\n            .setClientType(ClientType.PUSH_CONSUMER)\n            .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build())\n            .build()).get();\n    }\n\n    @Test\n    public void testConsumerHeartbeat() throws Throwable {\n        ProxyContext context = createContext();\n        this.sendConsumerTelemetry(context);\n\n        ArgumentCaptor<Set<SubscriptionData>> subscriptionDatasArgumentCaptor = ArgumentCaptor.forClass(Set.class);\n        ArgumentCaptor<ClientChannelInfo> channelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class);\n        doNothing().when(this.messagingProcessor).registerConsumer(any(),\n            anyString(),\n            channelInfoArgumentCaptor.capture(),\n            any(),\n            any(),\n            any(),\n            subscriptionDatasArgumentCaptor.capture(),\n            anyBoolean()\n        );\n\n        HeartbeatResponse response = this.sendConsumerHeartbeat(context);\n        assertEquals(Code.OK, response.getStatus().getCode());\n\n        ClientChannelInfo clientChannelInfo = channelInfoArgumentCaptor.getValue();\n        assertClientChannelInfo(clientChannelInfo, CONSUMER_GROUP);\n\n        SubscriptionData data = subscriptionDatasArgumentCaptor.getValue().stream().findAny().get();\n        assertEquals(\"TAG\", data.getExpressionType());\n        assertEquals(\"tag\", data.getSubString());\n    }\n\n    protected void assertClientChannelInfo(ClientChannelInfo clientChannelInfo, String group) {\n        assertEquals(LanguageCode.JAVA, clientChannelInfo.getLanguage());\n        assertEquals(CLIENT_ID, clientChannelInfo.getClientId());\n        assertTrue(clientChannelInfo.getChannel() instanceof GrpcClientChannel);\n        GrpcClientChannel channel = (GrpcClientChannel) clientChannelInfo.getChannel();\n        assertEquals(REMOTE_ADDR, channel.getRemoteAddress());\n        assertEquals(LOCAL_ADDR, channel.getLocalAddress());\n    }\n\n    @Test\n    public void testProducerNotifyClientTermination() throws Throwable {\n        ProxyContext context = createContext();\n\n        when(this.grpcClientSettingsManager.removeAndGetClientSettings(any())).thenReturn(Settings.newBuilder()\n            .setClientType(ClientType.PRODUCER)\n            .setPublishing(Publishing.newBuilder()\n                .addTopics(Resource.newBuilder().setName(TOPIC).build())\n                .build())\n            .build());\n        ArgumentCaptor<ClientChannelInfo> channelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class);\n        doNothing().when(this.messagingProcessor).unRegisterProducer(any(), anyString(), channelInfoArgumentCaptor.capture());\n        when(this.metadataService.getTopicMessageType(any(), anyString())).thenReturn(TopicMessageType.NORMAL);\n\n        this.sendProducerTelemetry(context);\n        this.sendProducerHeartbeat(context);\n\n        NotifyClientTerminationResponse response = this.clientActivity.notifyClientTermination(\n            context,\n            NotifyClientTerminationRequest.newBuilder()\n                .build()\n        ).get();\n\n        assertEquals(Code.OK, response.getStatus().getCode());\n        ClientChannelInfo clientChannelInfo = channelInfoArgumentCaptor.getValue();\n        assertClientChannelInfo(clientChannelInfo, TOPIC);\n    }\n\n    @Test\n    public void testConsumerNotifyClientTermination() throws Throwable {\n        ProxyContext context = createContext();\n\n        when(this.grpcClientSettingsManager.removeAndGetClientSettings(any())).thenReturn(Settings.newBuilder()\n            .setClientType(ClientType.PUSH_CONSUMER)\n            .build());\n        ArgumentCaptor<ClientChannelInfo> channelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class);\n        doNothing().when(this.messagingProcessor).unRegisterConsumer(any(), anyString(), channelInfoArgumentCaptor.capture());\n\n        this.sendConsumerTelemetry(context);\n        this.sendConsumerHeartbeat(context);\n\n        NotifyClientTerminationResponse response = this.clientActivity.notifyClientTermination(\n            context,\n            NotifyClientTerminationRequest.newBuilder()\n                .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build())\n                .build()\n        ).get();\n\n        assertEquals(Code.OK, response.getStatus().getCode());\n        ClientChannelInfo clientChannelInfo = channelInfoArgumentCaptor.getValue();\n        assertClientChannelInfo(clientChannelInfo, CONSUMER_GROUP);\n    }\n\n    @Test\n    public void testErrorConsumerGroupName() throws Throwable {\n        ProxyContext context = createContext();\n        try {\n            this.sendClientTelemetry(\n                context,\n                Settings.newBuilder()\n                    .setClientType(ClientType.PUSH_CONSUMER)\n                    .setSubscription(Subscription.newBuilder()\n                        .addSubscriptions(SubscriptionEntry.newBuilder()\n                            .setExpression(FilterExpression.newBuilder()\n                                .setExpression(\"tag\")\n                                .setType(FilterType.TAG)\n                                .build())\n                            .setTopic(Resource.newBuilder().setName(TOPIC).build())\n                            .build())\n                        .build())\n                    .build()).get();\n            fail();\n        } catch (ExecutionException e) {\n            StatusRuntimeException exception = (StatusRuntimeException) e.getCause();\n            assertEquals(Status.Code.INVALID_ARGUMENT, exception.getStatus().getCode());\n        }\n    }\n\n    @Test\n    public void testErrorProducerConfig() throws Throwable {\n        ProxyContext context = createContext();\n        try {\n            this.sendClientTelemetry(\n                context,\n                Settings.newBuilder()\n                    .setClientType(ClientType.PRODUCER)\n                    .setPublishing(Publishing.newBuilder()\n                        .addTopics(Resource.newBuilder().setName(\"()\").build())\n                        .build())\n                    .build()).get();\n            fail();\n        } catch (ExecutionException e) {\n            StatusRuntimeException exception = (StatusRuntimeException) e.getCause();\n            assertEquals(Status.Code.INVALID_ARGUMENT, exception.getStatus().getCode());\n        }\n    }\n\n    @Test\n    public void testEmptySettings() throws Throwable {\n        ProxyContext context = createContext();\n        try {\n            this.sendClientTelemetry(\n                context,\n                Settings.getDefaultInstance()).get();\n            fail();\n        } catch (ExecutionException e) {\n            StatusRuntimeException exception = (StatusRuntimeException) e.getCause();\n            assertEquals(Status.Code.INVALID_ARGUMENT, exception.getStatus().getCode());\n        }\n    }\n\n    @Test\n    public void testEmptyProducerSettings() throws Throwable {\n        ProxyContext context = createContext();\n        TelemetryCommand command = this.sendClientTelemetry(\n            context,\n            Settings.newBuilder()\n                .setClientType(ClientType.PRODUCER)\n                .setPublishing(Publishing.getDefaultInstance())\n                .build()).get();\n        assertTrue(command.hasSettings());\n        assertTrue(command.getSettings().hasPublishing());\n    }\n\n    @Test\n    public void testReportThreadStackTrace() {\n        this.clientActivity = new ClientActivity(this.messagingProcessor, this.grpcClientSettingsManager, grpcChannelManagerMock);\n        String jstack = \"jstack\";\n        String nonce = \"123\";\n        when(grpcChannelManagerMock.getAndRemoveResponseFuture(anyString())).thenReturn((CompletableFuture) runningInfoFutureMock);\n        ProxyContext context = createContext();\n        ContextStreamObserver<TelemetryCommand> streamObserver = clientActivity.telemetry(new StreamObserver<TelemetryCommand>() {\n            @Override\n            public void onNext(TelemetryCommand value) {\n            }\n\n            @Override\n            public void onError(Throwable t) {\n            }\n\n            @Override\n            public void onCompleted() {\n            }\n        });\n        streamObserver.onNext(context, TelemetryCommand.newBuilder()\n            .setThreadStackTrace(ThreadStackTrace.newBuilder()\n                .setThreadStackTrace(jstack)\n                .setNonce(nonce)\n                .build())\n            .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()))\n            .build());\n        verify(runningInfoFutureMock, times(1)).complete(runningInfoArgumentCaptor.capture());\n        ProxyRelayResult<ConsumerRunningInfo> result = runningInfoArgumentCaptor.getValue();\n        assertThat(result.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(result.getResult().getJstack()).isEqualTo(jstack);\n    }\n\n    @Test\n    public void testReportVerifyMessageResult() {\n        this.clientActivity = new ClientActivity(this.messagingProcessor, this.grpcClientSettingsManager, grpcChannelManagerMock);\n        String nonce = \"123\";\n        when(grpcChannelManagerMock.getAndRemoveResponseFuture(anyString())).thenReturn((CompletableFuture) resultFutureMock);\n        ProxyContext context = createContext();\n        ContextStreamObserver<TelemetryCommand> streamObserver = clientActivity.telemetry(new StreamObserver<TelemetryCommand>() {\n            @Override\n            public void onNext(TelemetryCommand value) {\n            }\n\n            @Override\n            public void onError(Throwable t) {\n            }\n\n            @Override\n            public void onCompleted() {\n            }\n        });\n        streamObserver.onNext(context, TelemetryCommand.newBuilder()\n            .setVerifyMessageResult(VerifyMessageResult.newBuilder()\n                .setNonce(nonce)\n                .build())\n            .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()))\n            .build());\n        verify(resultFutureMock, times(1)).complete(resultArgumentCaptor.capture());\n        ProxyRelayResult<ConsumeMessageDirectlyResult> result = resultArgumentCaptor.getValue();\n        assertThat(result.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(result.getResult().getConsumeResult()).isEqualTo(CMResult.CR_SUCCESS);\n    }\n\n    protected CompletableFuture<TelemetryCommand> sendClientTelemetry(ProxyContext ctx, Settings settings) {\n        when(grpcClientSettingsManager.getClientSettings(any())).thenReturn(settings);\n\n        CompletableFuture<TelemetryCommand> future = new CompletableFuture<>();\n        StreamObserver<TelemetryCommand> responseObserver = new StreamObserver<TelemetryCommand>() {\n            @Override\n            public void onNext(TelemetryCommand value) {\n                future.complete(value);\n            }\n\n            @Override\n            public void onError(Throwable t) {\n                future.completeExceptionally(t);\n            }\n\n            @Override\n            public void onCompleted() {\n\n            }\n        };\n        ContextStreamObserver<TelemetryCommand> requestObserver = this.clientActivity.telemetry(responseObserver);\n        requestObserver.onNext(ctx, TelemetryCommand.newBuilder()\n            .setSettings(settings)\n            .build());\n        return future;\n    }\n\n    @Test\n    public void testSyncLiteSubscription_Success() {\n        ProxyContext proxyContext = createContext();\n        proxyContext.setClientID(\"client-id\");\n        Resource topic = Resource.newBuilder().setName(\"test-topic\").build();\n        Resource group = Resource.newBuilder().setName(\"test-group\").build();\n        SyncLiteSubscriptionRequest request = SyncLiteSubscriptionRequest.newBuilder()\n            .setTopic(topic)\n            .setGroup(group)\n            .setAction(LiteSubscriptionAction.PARTIAL_ADD)\n            .addAllLiteTopicSet(java.util.Collections.emptyList())\n            .setVersion(1L)\n            .build();\n\n        when(messagingProcessor.syncLiteSubscription(any(), any(LiteSubscriptionDTO.class), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(null));\n\n        CompletableFuture<SyncLiteSubscriptionResponse> future = clientActivity.syncLiteSubscription(proxyContext, request);\n\n        SyncLiteSubscriptionResponse response = future.join();\n        assertEquals(Code.OK, response.getStatus().getCode());\n    }\n\n    @Test\n    public void testSyncLiteSubscription_ValidationFailure() {\n        ProxyContext proxyContext = createContext();\n        Resource topic = Resource.newBuilder().setName(\"test-topic\").build();\n        Resource group = Resource.newBuilder().setName(\"test-group\").build();\n        SyncLiteSubscriptionRequest request = SyncLiteSubscriptionRequest.newBuilder()\n            .setTopic(topic)\n            .setGroup(group)\n            .build();\n\n        // Mock the GrpcValidator singleton\n        GrpcValidator mockValidator = mock(GrpcValidator.class);\n        try (MockedStatic<GrpcValidator> mocked = mockStatic(GrpcValidator.class)) {\n            mocked.when(GrpcValidator::getInstance).thenReturn(mockValidator);\n\n            doThrow(new IllegalArgumentException(\"Invalid topic\"))\n                .when(mockValidator).validateTopicAndConsumerGroup(topic, group);\n\n            CompletableFuture<SyncLiteSubscriptionResponse> future = clientActivity.syncLiteSubscription(proxyContext, request);\n\n            assertTrue(future.isCompletedExceptionally());\n        }\n    }\n\n    @Test\n    public void testSyncLiteSubscription_ProcessingFailure() {\n        ProxyContext proxyContext = createContext();\n        proxyContext.setClientID(\"client-id\");\n        Resource topic = Resource.newBuilder().setName(\"test-topic\").build();\n        Resource group = Resource.newBuilder().setName(\"test-group\").build();\n        SyncLiteSubscriptionRequest request = SyncLiteSubscriptionRequest.newBuilder()\n            .setTopic(topic)\n            .setGroup(group)\n            .setAction(LiteSubscriptionAction.PARTIAL_ADD)\n            .addAllLiteTopicSet(java.util.Collections.emptyList())\n            .setVersion(1L)\n            .build();\n\n        CompletableFuture<Void> failedFuture = new CompletableFuture<>();\n        failedFuture.completeExceptionally(new RuntimeException(\"Processing failed\"));\n        when(messagingProcessor.syncLiteSubscription(any(), any(LiteSubscriptionDTO.class), anyLong()))\n            .thenReturn(failedFuture);\n\n        CompletableFuture<SyncLiteSubscriptionResponse> future = clientActivity.syncLiteSubscription(proxyContext, request);\n\n        assertTrue(future.isCompletedExceptionally());\n    }\n\n    @Test\n    public void testSyncLiteSubscription_NullContext() {\n        Resource topic = Resource.newBuilder().setName(\"test-topic\").build();\n        Resource group = Resource.newBuilder().setName(\"test-group\").build();\n        SyncLiteSubscriptionRequest request = SyncLiteSubscriptionRequest.newBuilder()\n            .setTopic(topic)\n            .setGroup(group)\n            .build();\n\n        CompletableFuture<SyncLiteSubscriptionResponse> future = clientActivity.syncLiteSubscription(null, request);\n\n        assertTrue(future.isCompletedExceptionally());\n    }\n\n    @Test\n    public void testSyncLiteSubscription_NullRequest() {\n        ProxyContext proxyContext = createContext();\n\n        CompletableFuture<SyncLiteSubscriptionResponse> future = clientActivity.syncLiteSubscription(proxyContext, null);\n\n        assertTrue(future.isCompletedExceptionally());\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcClientSettingsManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2.common;\n\nimport apache.rocketmq.v2.ClientType;\nimport apache.rocketmq.v2.CustomizedBackoff;\nimport apache.rocketmq.v2.ExponentialBackoff;\nimport apache.rocketmq.v2.Publishing;\nimport apache.rocketmq.v2.Resource;\nimport apache.rocketmq.v2.RetryPolicy;\nimport apache.rocketmq.v2.Settings;\nimport apache.rocketmq.v2.Subscription;\nimport apache.rocketmq.v2.SubscriptionEntry;\nimport com.google.protobuf.util.Durations;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.common.lite.LiteSubscriptionDTO;\nimport org.apache.rocketmq.proxy.common.ContextVariable;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest;\nimport org.apache.rocketmq.remoting.protocol.subscription.CustomizedRetryPolicy;\nimport org.apache.rocketmq.remoting.protocol.subscription.ExponentialRetryPolicy;\nimport org.apache.rocketmq.remoting.protocol.subscription.GroupRetryPolicyType;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotEquals;\nimport static org.junit.Assert.assertNull;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.doReturn;\nimport static org.mockito.Mockito.never;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\npublic class GrpcClientSettingsManagerTest extends BaseActivityTest {\n\n    private final ProxyContext ctx = ProxyContext.create();\n    private final String clientId = \"testClientId\";\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        grpcClientSettingsManager = spy(new GrpcClientSettingsManager(messagingProcessor));\n    }\n\n    @Test\n    public void testGetProducerData() {\n        ProxyContext context = ProxyContext.create().withVal(ContextVariable.CLIENT_ID, CLIENT_ID);\n\n        this.grpcClientSettingsManager.updateClientSettings(context, CLIENT_ID, Settings.newBuilder()\n            .setBackoffPolicy(RetryPolicy.getDefaultInstance())\n            .setPublishing(Publishing.getDefaultInstance())\n            .build());\n        Settings settings = this.grpcClientSettingsManager.getClientSettings(context);\n        assertNotEquals(settings.getBackoffPolicy(), settings.getBackoffPolicy().getDefaultInstanceForType());\n        assertNotEquals(settings.getPublishing(), settings.getPublishing().getDefaultInstanceForType());\n    }\n\n    @Test\n    public void testGetSubscriptionData() {\n        ProxyContext context = ProxyContext.create().withVal(ContextVariable.CLIENT_ID, CLIENT_ID);\n\n        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n        when(this.messagingProcessor.getSubscriptionGroupConfig(any(), any()))\n            .thenReturn(subscriptionGroupConfig);\n\n        this.grpcClientSettingsManager.updateClientSettings(context, CLIENT_ID, Settings.newBuilder()\n            .setSubscription(Subscription.newBuilder()\n                .setGroup(Resource.newBuilder().setName(\"group\").build())\n                .build())\n            .build());\n\n        Settings settings = this.grpcClientSettingsManager.getClientSettings(context);\n        assertEquals(settings.getBackoffPolicy(), this.grpcClientSettingsManager.createDefaultConsumerSettingsBuilder().build().getBackoffPolicy());\n\n        subscriptionGroupConfig.setRetryMaxTimes(3);\n        subscriptionGroupConfig.getGroupRetryPolicy().setType(GroupRetryPolicyType.CUSTOMIZED);\n        subscriptionGroupConfig.getGroupRetryPolicy().setCustomizedRetryPolicy(new CustomizedRetryPolicy(new long[] {1000}));\n        settings = this.grpcClientSettingsManager.getClientSettings(context);\n        assertEquals(RetryPolicy.newBuilder()\n            .setMaxAttempts(4)\n            .setCustomizedBackoff(CustomizedBackoff.newBuilder()\n                .addNext(Durations.fromSeconds(1))\n                .build())\n            .build(), settings.getBackoffPolicy());\n\n        subscriptionGroupConfig.setRetryMaxTimes(10);\n        subscriptionGroupConfig.getGroupRetryPolicy().setType(GroupRetryPolicyType.EXPONENTIAL);\n        subscriptionGroupConfig.getGroupRetryPolicy().setExponentialRetryPolicy(new ExponentialRetryPolicy(1000, 2000, 3));\n        settings = this.grpcClientSettingsManager.getClientSettings(context);\n        assertEquals(RetryPolicy.newBuilder()\n            .setMaxAttempts(11)\n            .setExponentialBackoff(ExponentialBackoff.newBuilder()\n                .setMax(Durations.fromSeconds(2))\n                .setInitial(Durations.fromSeconds(1))\n                .setMultiplier(3)\n                .build())\n            .build(), settings.getBackoffPolicy());\n\n        Settings settings1 = this.grpcClientSettingsManager.removeAndGetClientSettings(context);\n        assertEquals(settings, settings1);\n\n        assertNull(this.grpcClientSettingsManager.getClientSettings(context));\n        assertNull(this.grpcClientSettingsManager.removeAndGetClientSettings(context));\n    }\n\n    @Test\n    public void testOfflineClientLiteSubscription_SettingsNullAndNoCachedSettings() {\n        doReturn(null).when(grpcClientSettingsManager).getRawClientSettings(anyString());\n\n        grpcClientSettingsManager.offlineClientLiteSubscription(ctx, clientId, null);\n\n        verify(messagingProcessor, never()).syncLiteSubscription(any(), any(), anyLong());\n    }\n\n    @Test\n    public void testOfflineClientLiteSubscription_SettingsNull_CachedSettingsNotLite() {\n        Settings cachedSettings = Settings.newBuilder()\n            .setClientType(ClientType.PRODUCER)\n            .build();\n        doReturn(cachedSettings).when(grpcClientSettingsManager).getRawClientSettings(anyString());\n\n        grpcClientSettingsManager.offlineClientLiteSubscription(ctx, clientId, null);\n\n        verify(messagingProcessor, never()).syncLiteSubscription(any(), any(), anyLong());\n    }\n\n    @Test\n    public void testOfflineClientLiteSubscription_SettingsNotNull_NotLiteConsumer() {\n        Settings settings = Settings.newBuilder()\n            .setClientType(ClientType.PUSH_CONSUMER)\n            .build();\n\n        grpcClientSettingsManager.offlineClientLiteSubscription(ctx, clientId, settings);\n\n        verify(messagingProcessor, never()).syncLiteSubscription(any(), any(), anyLong());\n    }\n\n    @Test\n    public void testOfflineClientLiteSubscription_ValidLiteConsumer_Success() {\n        Subscription subscription = Subscription.newBuilder()\n            .setGroup(Resource.newBuilder().setName(\"testGroup\").build())\n            .addSubscriptions(SubscriptionEntry.newBuilder()\n                .setTopic(Resource.newBuilder().setName(\"testTopic\").build())\n                .build())\n            .build();\n\n        Settings settings = Settings.newBuilder()\n            .setClientType(ClientType.LITE_PUSH_CONSUMER)\n            .setSubscription(subscription)\n            .build();\n\n        when(messagingProcessor.syncLiteSubscription(any(), any(LiteSubscriptionDTO.class), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(null));\n\n        grpcClientSettingsManager.offlineClientLiteSubscription(ctx, clientId, settings);\n\n        verify(messagingProcessor, times(1)).syncLiteSubscription(any(), any(LiteSubscriptionDTO.class), anyLong());\n    }\n\n    @Test\n    public void testOfflineClientLiteSubscription_ValidLiteConsumer_SyncThrowsException() {\n        Subscription subscription = Subscription.newBuilder()\n            .setGroup(Resource.newBuilder().setName(\"testGroup\").build())\n            .addSubscriptions(SubscriptionEntry.newBuilder()\n                .setTopic(Resource.newBuilder().setName(\"testTopic\").build())\n                .build())\n            .build();\n\n        Settings settings = Settings.newBuilder()\n            .setClientType(ClientType.LITE_PUSH_CONSUMER)\n            .setSubscription(subscription)\n            .build();\n\n        CompletableFuture<Void> future = new CompletableFuture<>();\n        future.completeExceptionally(new RuntimeException(\"Simulated error\"));\n        when(messagingProcessor.syncLiteSubscription(any(), any(LiteSubscriptionDTO.class), anyLong()))\n            .thenReturn(future);\n\n        grpcClientSettingsManager.offlineClientLiteSubscription(ctx, clientId, settings);\n\n        verify(messagingProcessor, times(1)).syncLiteSubscription(any(), any(LiteSubscriptionDTO.class), anyLong());\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcConverterTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2.common;\n\nimport apache.rocketmq.v2.MessageQueue;\nimport apache.rocketmq.v2.MessageType;\nimport java.net.InetSocketAddress;\nimport java.nio.charset.StandardCharsets;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertTrue;\n\npublic class GrpcConverterTest {\n    @Test\n    public void testBuildMessageQueue() {\n        String topic = \"topic\";\n        String brokerName = \"brokerName\";\n        int queueId = 1;\n        MessageExt messageExt = new MessageExt();\n        messageExt.setQueueId(queueId);\n        messageExt.setTopic(topic);\n\n        MessageQueue messageQueue = GrpcConverter.getInstance().buildMessageQueue(messageExt, brokerName);\n        assertThat(messageQueue.getTopic().getName()).isEqualTo(topic);\n        assertThat(messageQueue.getBroker().getName()).isEqualTo(brokerName);\n        assertThat(messageQueue.getId()).isEqualTo(queueId);\n    }\n\n    @Test\n    public void testBuildMessageWithLiteTopic() {\n        final String topic = \"test-topic\";\n        final String liteTopic = \"test-lite-topic\";\n        // Build a message with lite topic properties\n        MessageExt messageExt = new MessageExt();\n        messageExt.setTopic(topic);\n        messageExt.setBody(\"test-body\".getBytes(StandardCharsets.UTF_8));\n        messageExt.setQueueId(1);\n        messageExt.setQueueOffset(100L);\n        messageExt.setBornTimestamp(System.currentTimeMillis());\n        messageExt.setStoreTimestamp(System.currentTimeMillis());\n        messageExt.setBornHost(new InetSocketAddress(\"127.0.0.1\", 1234));\n        messageExt.setStoreHost(new InetSocketAddress(\"127.0.0.1\", 5678));\n        messageExt.setReconsumeTimes(0);\n        messageExt.setMsgId(\"test-msg-id\");\n\n        // Set lite topic property\n        MessageAccessor.setLiteTopic(messageExt, liteTopic);\n\n        // Convert message\n        GrpcConverter grpcConverter = GrpcConverter.getInstance();\n        apache.rocketmq.v2.Message grpcMessage = grpcConverter.buildMessage(messageExt);\n\n        // Verify basic properties\n        assertNotNull(grpcMessage);\n        assertEquals(topic, grpcMessage.getTopic().getName());\n        assertEquals(\"test-body\", grpcMessage.getBody().toString(StandardCharsets.UTF_8));\n\n        // Verify lite topic in system properties\n        assertNotNull(grpcMessage.getSystemProperties());\n        assertTrue(grpcMessage.getSystemProperties().hasLiteTopic());\n        assertEquals(liteTopic, grpcMessage.getSystemProperties().getLiteTopic());\n\n        // Verify message type is LITE\n        assertEquals(MessageType.LITE, grpcMessage.getSystemProperties().getMessageType());\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcValidatorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2.common;\n\nimport org.apache.commons.lang3.RandomStringUtils;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.MockedStatic;\n\nimport static org.junit.Assert.assertThrows;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.mockStatic;\nimport static org.mockito.Mockito.when;\n\npublic class GrpcValidatorTest {\n\n    private GrpcValidator grpcValidator;\n\n    @Before\n    public void before() {\n        this.grpcValidator = GrpcValidator.getInstance();\n    }\n\n    @Test\n    public void testValidateTopic() {\n        assertThrows(GrpcProxyException.class, () -> grpcValidator.validateTopic(\"\"));\n        assertThrows(GrpcProxyException.class, () -> grpcValidator.validateTopic(\"rmq_sys_xxxx\"));\n        grpcValidator.validateTopic(\"topicName\");\n    }\n\n    @Test\n    public void testValidateConsumerGroup() {\n        assertThrows(GrpcProxyException.class, () -> grpcValidator.validateConsumerGroup(\"\"));\n        assertThrows(GrpcProxyException.class, () -> grpcValidator.validateConsumerGroup(\"CID_RMQ_SYS_xxxx\"));\n        grpcValidator.validateConsumerGroup(\"consumerGroupName\");\n    }\n\n\n    @Test\n    public void testValidateLiteTopic_Null() {\n        assertThrows(GrpcProxyException.class, () -> grpcValidator.validateLiteTopic(null));\n    }\n\n    @Test\n    public void testValidateLiteTopic_Blank() {\n        assertThrows(GrpcProxyException.class, () -> grpcValidator.validateLiteTopic(\"   \"));\n    }\n\n    @Test\n    public void testValidateLiteTopic_TooLong() {\n        try (MockedStatic<ConfigurationManager> mockedConfig = mockStatic(ConfigurationManager.class)) {\n            ProxyConfig proxyConfig = mock(ProxyConfig.class);\n            when(proxyConfig.getMaxLiteTopicSize()).thenReturn(5);\n            mockedConfig.when(ConfigurationManager::getProxyConfig).thenReturn(proxyConfig);\n\n            assertThrows(GrpcProxyException.class, () -> grpcValidator.validateLiteTopic(\"toolongtopic\"));\n        }\n    }\n\n    @Test\n    public void testValidateLiteTopic_IllegalCharacter() {\n        try (MockedStatic<ConfigurationManager> mockedConfig = mockStatic(ConfigurationManager.class)) {\n            ProxyConfig proxyConfig = mock(ProxyConfig.class);\n            when(proxyConfig.getMaxLiteTopicSize()).thenReturn(100);\n            mockedConfig.when(ConfigurationManager::getProxyConfig).thenReturn(proxyConfig);\n\n            assertThrows(GrpcProxyException.class, () -> grpcValidator.validateLiteTopic(\"invalid@topic\"));\n\n            assertThrows(GrpcProxyException.class, () -> grpcValidator.validateLiteTopic(\"invalid$topic\"));\n\n            assertThrows(GrpcProxyException.class, () -> grpcValidator.validateLiteTopic(\"invalid%topic\"));\n\n            assertThrows(GrpcProxyException.class, () -> grpcValidator.validateLiteTopic(\"invalid\\ttopic\"));\n\n            assertThrows(GrpcProxyException.class, () -> grpcValidator.validateLiteTopic(\"invalid\\ntopic\"));\n\n            assertThrows(GrpcProxyException.class, () -> grpcValidator.validateLiteTopic(\"invalid\\0topic\"));\n        }\n    }\n\n    @Test\n    public void testValidateLiteTopic_Valid() {\n        try (MockedStatic<ConfigurationManager> mockedConfig = mockStatic(ConfigurationManager.class)) {\n            ProxyConfig proxyConfig = mock(ProxyConfig.class);\n            when(proxyConfig.getMaxLiteTopicSize()).thenReturn(64);\n            mockedConfig.when(ConfigurationManager::getProxyConfig).thenReturn(proxyConfig);\n\n            grpcValidator.validateLiteTopic(\"Valid_Topic-123\");\n\n            grpcValidator.validateLiteTopic(RandomStringUtils.randomAlphanumeric(64));\n\n            grpcValidator.validateLiteTopic(RandomStringUtils.randomAlphanumeric(63));\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2.consumer;\n\nimport apache.rocketmq.v2.AckMessageEntry;\nimport apache.rocketmq.v2.AckMessageRequest;\nimport apache.rocketmq.v2.AckMessageResponse;\nimport apache.rocketmq.v2.AckMessageResultEntry;\nimport apache.rocketmq.v2.Code;\nimport apache.rocketmq.v2.Resource;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.AckStatus;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest;\nimport org.apache.rocketmq.proxy.processor.BatchAckResult;\nimport org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.stubbing.Answer;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyList;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.doAnswer;\nimport static org.mockito.Mockito.when;\n\npublic class AckMessageActivityTest extends BaseActivityTest {\n\n    private AckMessageActivity ackMessageActivity;\n\n    private static final String TOPIC = \"topic\";\n    private static final String GROUP = \"group\";\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        this.ackMessageActivity = new AckMessageActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n    }\n\n    @Test\n    public void testAckMessage() throws Throwable {\n        ConfigurationManager.getProxyConfig().setEnableBatchAck(false);\n\n        String msg1 = \"msg1\";\n        String msg2 = \"msg2\";\n        String msg3 = \"msg3\";\n\n        when(this.messagingProcessor.ackMessage(any(), any(), eq(msg1), anyString(), anyString(), any()))\n            .thenThrow(new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, \"receipt handle is expired\"));\n\n        AckResult msg2AckResult = new AckResult();\n        msg2AckResult.setStatus(AckStatus.OK);\n        when(this.messagingProcessor.ackMessage(any(), any(), eq(msg2), anyString(), anyString(), any()))\n            .thenReturn(CompletableFuture.completedFuture(msg2AckResult));\n\n        AckResult msg3AckResult = new AckResult();\n        msg3AckResult.setStatus(AckStatus.NO_EXIST);\n        when(this.messagingProcessor.ackMessage(any(), any(), eq(msg3), anyString(), anyString(), any()))\n            .thenReturn(CompletableFuture.completedFuture(msg3AckResult));\n\n        {\n            AckMessageResponse response = this.ackMessageActivity.ackMessage(\n                createContext(),\n                AckMessageRequest.newBuilder()\n                    .setTopic(Resource.newBuilder().setName(TOPIC).build())\n                    .setGroup(Resource.newBuilder().setName(GROUP).build())\n                    .addEntries(AckMessageEntry.newBuilder()\n                        .setMessageId(msg1)\n                        .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000))\n                        .build())\n                    .build()\n            ).get();\n            assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getStatus().getCode());\n        }\n        {\n            AckMessageResponse response = this.ackMessageActivity.ackMessage(\n                createContext(),\n                AckMessageRequest.newBuilder()\n                    .setTopic(Resource.newBuilder().setName(TOPIC).build())\n                    .setGroup(Resource.newBuilder().setName(GROUP).build())\n                    .addEntries(AckMessageEntry.newBuilder()\n                        .setMessageId(msg2)\n                        .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000))\n                        .build())\n                    .build()\n            ).get();\n            assertEquals(Code.OK, response.getStatus().getCode());\n        }\n        {\n            AckMessageResponse response = this.ackMessageActivity.ackMessage(\n                createContext(),\n                AckMessageRequest.newBuilder()\n                    .setTopic(Resource.newBuilder().setName(TOPIC).build())\n                    .setGroup(Resource.newBuilder().setName(GROUP).build())\n                    .addEntries(AckMessageEntry.newBuilder()\n                        .setMessageId(msg3)\n                        .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000))\n                        .build())\n                    .build()\n            ).get();\n            assertEquals(Code.INTERNAL_SERVER_ERROR, response.getStatus().getCode());\n        }\n        {\n            AckMessageResponse response = this.ackMessageActivity.ackMessage(\n                createContext(),\n                AckMessageRequest.newBuilder()\n                    .setTopic(Resource.newBuilder().setName(TOPIC).build())\n                    .setGroup(Resource.newBuilder().setName(GROUP).build())\n                    .addEntries(AckMessageEntry.newBuilder()\n                        .setMessageId(msg1)\n                        .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000))\n                        .build())\n                    .addEntries(AckMessageEntry.newBuilder()\n                        .setMessageId(msg2)\n                        .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))\n                        .build())\n                    .addEntries(AckMessageEntry.newBuilder()\n                        .setMessageId(msg3)\n                        .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))\n                        .build())\n                    .build()\n            ).get();\n\n            assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode());\n            assertEquals(3, response.getEntriesCount());\n            assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getEntries(0).getStatus().getCode());\n            assertEquals(Code.OK, response.getEntries(1).getStatus().getCode());\n            assertEquals(Code.INTERNAL_SERVER_ERROR, response.getEntries(2).getStatus().getCode());\n        }\n    }\n\n    @Test\n    public void testAckMessageInBatch() throws Throwable {\n        ConfigurationManager.getProxyConfig().setEnableBatchAck(true);\n\n        String successMessageId = \"msg1\";\n        String notOkMessageId = \"msg2\";\n        String exceptionMessageId = \"msg3\";\n\n        doAnswer((Answer<CompletableFuture<List<BatchAckResult>>>) invocation -> {\n            List<ReceiptHandleMessage> receiptHandleMessageList = invocation.getArgument(1, List.class);\n            List<BatchAckResult> batchAckResultList = new ArrayList<>();\n            for (ReceiptHandleMessage receiptHandleMessage : receiptHandleMessageList) {\n                BatchAckResult batchAckResult;\n                if (receiptHandleMessage.getMessageId().equals(successMessageId)) {\n                    AckResult ackResult = new AckResult();\n                    ackResult.setStatus(AckStatus.OK);\n                    batchAckResult = new BatchAckResult(receiptHandleMessage, ackResult);\n                } else if (receiptHandleMessage.getMessageId().equals(notOkMessageId)) {\n                    AckResult ackResult = new AckResult();\n                    ackResult.setStatus(AckStatus.NO_EXIST);\n                    batchAckResult = new BatchAckResult(receiptHandleMessage, ackResult);\n                } else {\n                    batchAckResult = new BatchAckResult(receiptHandleMessage, new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, \"\"));\n                }\n                batchAckResultList.add(batchAckResult);\n            }\n            return CompletableFuture.completedFuture(batchAckResultList);\n        }).when(this.messagingProcessor).batchAckMessage(any(), anyList(), anyString(), anyString());\n\n        {\n            AckMessageResponse response = this.ackMessageActivity.ackMessage(\n                createContext(),\n                AckMessageRequest.newBuilder()\n                    .setTopic(Resource.newBuilder().setName(TOPIC).build())\n                    .setGroup(Resource.newBuilder().setName(GROUP).build())\n                    .addEntries(AckMessageEntry.newBuilder()\n                        .setMessageId(successMessageId)\n                        .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))\n                        .build())\n                    .build()\n            ).get();\n            assertEquals(Code.OK, response.getStatus().getCode());\n        }\n        {\n            AckMessageResponse response = this.ackMessageActivity.ackMessage(\n                createContext(),\n                AckMessageRequest.newBuilder()\n                    .setTopic(Resource.newBuilder().setName(TOPIC).build())\n                    .setGroup(Resource.newBuilder().setName(GROUP).build())\n                    .addEntries(AckMessageEntry.newBuilder()\n                        .setMessageId(notOkMessageId)\n                        .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))\n                        .build())\n                    .build()\n            ).get();\n            assertEquals(Code.INTERNAL_SERVER_ERROR, response.getStatus().getCode());\n        }\n        {\n            AckMessageResponse response = this.ackMessageActivity.ackMessage(\n                createContext(),\n                AckMessageRequest.newBuilder()\n                    .setTopic(Resource.newBuilder().setName(TOPIC).build())\n                    .setGroup(Resource.newBuilder().setName(GROUP).build())\n                    .addEntries(AckMessageEntry.newBuilder()\n                        .setMessageId(exceptionMessageId)\n                        .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))\n                        .build())\n                    .build()\n            ).get();\n            assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getStatus().getCode());\n        }\n        {\n            AckMessageResponse response = this.ackMessageActivity.ackMessage(\n                createContext(),\n                AckMessageRequest.newBuilder()\n                    .setTopic(Resource.newBuilder().setName(TOPIC).build())\n                    .setGroup(Resource.newBuilder().setName(GROUP).build())\n                    .addEntries(AckMessageEntry.newBuilder()\n                        .setMessageId(successMessageId)\n                        .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))\n                        .build())\n                    .addEntries(AckMessageEntry.newBuilder()\n                        .setMessageId(notOkMessageId)\n                        .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))\n                        .build())\n                    .addEntries(AckMessageEntry.newBuilder()\n                        .setMessageId(exceptionMessageId)\n                        .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))\n                        .build())\n                    .build()\n            ).get();\n\n            assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode());\n            assertEquals(3, response.getEntriesCount());\n            Map<String, Code> msgCode = new HashMap<>();\n            for (AckMessageResultEntry entry : response.getEntriesList()) {\n                msgCode.put(entry.getMessageId(), entry.getStatus().getCode());\n            }\n            assertEquals(Code.OK, msgCode.get(successMessageId));\n            assertEquals(Code.INTERNAL_SERVER_ERROR, msgCode.get(notOkMessageId));\n            assertEquals(Code.INVALID_RECEIPT_HANDLE, msgCode.get(exceptionMessageId));\n        }\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ChangeInvisibleDurationActivityTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2.consumer;\n\nimport apache.rocketmq.v2.ChangeInvisibleDurationRequest;\nimport apache.rocketmq.v2.ChangeInvisibleDurationResponse;\nimport apache.rocketmq.v2.Code;\nimport apache.rocketmq.v2.Resource;\nimport com.google.protobuf.util.Durations;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.AckStatus;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.proxy.common.MessageReceiptHandle;\nimport org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.ArgumentCaptor;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.when;\n\npublic class ChangeInvisibleDurationActivityTest extends BaseActivityTest {\n\n    private static final String TOPIC = \"topic\";\n    private static final String CONSUMER_GROUP = \"consumerGroup\";\n    private ChangeInvisibleDurationActivity changeInvisibleDurationActivity;\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        this.changeInvisibleDurationActivity = new ChangeInvisibleDurationActivity(messagingProcessor,\n            grpcClientSettingsManager, grpcChannelManager);\n    }\n\n    @Test\n    public void testChangeInvisibleDurationActivity() throws Throwable {\n        String newHandle = \"newHandle\";\n        ArgumentCaptor<Long> invisibleTimeArgumentCaptor = ArgumentCaptor.forClass(Long.class);\n        AckResult ackResult = new AckResult();\n        ackResult.setExtraInfo(newHandle);\n        ackResult.setStatus(AckStatus.OK);\n        when(this.messagingProcessor.changeInvisibleTime(\n            any(), any(), anyString(), anyString(), anyString(), invisibleTimeArgumentCaptor.capture()\n        )).thenReturn(CompletableFuture.completedFuture(ackResult));\n\n        ChangeInvisibleDurationResponse response = this.changeInvisibleDurationActivity.changeInvisibleDuration(\n            createContext(),\n            ChangeInvisibleDurationRequest.newBuilder()\n                .setInvisibleDuration(Durations.fromSeconds(3))\n                .setTopic(Resource.newBuilder().setName(TOPIC).build())\n                .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build())\n                .setMessageId(\"msgId\")\n                .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))\n                .build()\n        ).get();\n\n        assertEquals(Code.OK, response.getStatus().getCode());\n        assertEquals(TimeUnit.SECONDS.toMillis(3), invisibleTimeArgumentCaptor.getValue().longValue());\n        assertEquals(newHandle, response.getReceiptHandle());\n    }\n\n    @Test\n    public void testChangeInvisibleDurationActivityWhenHasMappingHandle() throws Throwable {\n        String newHandle = \"newHandle\";\n        ArgumentCaptor<Long> invisibleTimeArgumentCaptor = ArgumentCaptor.forClass(Long.class);\n        AckResult ackResult = new AckResult();\n        ackResult.setExtraInfo(newHandle);\n        ackResult.setStatus(AckStatus.OK);\n        String savedHandleStr = buildReceiptHandle(\"topic\", System.currentTimeMillis(),3000);\n        ArgumentCaptor<ReceiptHandle> receiptHandleCaptor = ArgumentCaptor.forClass(ReceiptHandle.class);\n        when(this.messagingProcessor.changeInvisibleTime(\n            any(), receiptHandleCaptor.capture(), anyString(), anyString(), anyString(), invisibleTimeArgumentCaptor.capture()\n        )).thenReturn(CompletableFuture.completedFuture(ackResult));\n        when(messagingProcessor.removeReceiptHandle(any(), any(), anyString(), anyString(), anyString()))\n            .thenReturn(new MessageReceiptHandle(\"group\", \"topic\", 0, savedHandleStr, \"msgId\", 0, 0));\n\n        ChangeInvisibleDurationResponse response = this.changeInvisibleDurationActivity.changeInvisibleDuration(\n            createContext(),\n            ChangeInvisibleDurationRequest.newBuilder()\n                .setInvisibleDuration(Durations.fromSeconds(3))\n                .setTopic(Resource.newBuilder().setName(TOPIC).build())\n                .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build())\n                .setMessageId(\"msgId\")\n                .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))\n                .build()\n        ).get();\n\n        assertEquals(Code.OK, response.getStatus().getCode());\n        assertEquals(TimeUnit.SECONDS.toMillis(3), invisibleTimeArgumentCaptor.getValue().longValue());\n        assertEquals(savedHandleStr, receiptHandleCaptor.getValue().getReceiptHandle());\n        assertEquals(newHandle, response.getReceiptHandle());\n    }\n\n\n    @Test\n    public void testChangeInvisibleDurationActivityFailed() throws Throwable {\n        ArgumentCaptor<Long> invisibleTimeArgumentCaptor = ArgumentCaptor.forClass(Long.class);\n        AckResult ackResult = new AckResult();\n        ackResult.setStatus(AckStatus.NO_EXIST);\n        when(this.messagingProcessor.changeInvisibleTime(\n            any(), any(), anyString(), anyString(), anyString(), invisibleTimeArgumentCaptor.capture()\n        )).thenReturn(CompletableFuture.completedFuture(ackResult));\n\n        ChangeInvisibleDurationResponse response = this.changeInvisibleDurationActivity.changeInvisibleDuration(\n            createContext(),\n            ChangeInvisibleDurationRequest.newBuilder()\n                .setInvisibleDuration(Durations.fromSeconds(3))\n                .setTopic(Resource.newBuilder().setName(TOPIC).build())\n                .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build())\n                .setMessageId(\"msgId\")\n                .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))\n                .build()\n        ).get();\n\n        assertEquals(Code.INTERNAL_SERVER_ERROR, response.getStatus().getCode());\n        assertEquals(TimeUnit.SECONDS.toMillis(3), invisibleTimeArgumentCaptor.getValue().longValue());\n    }\n\n    @Test\n    public void testChangeInvisibleDurationInvisibleTimeTooSmall() throws Throwable {\n        try {\n            this.changeInvisibleDurationActivity.changeInvisibleDuration(\n                createContext(),\n                ChangeInvisibleDurationRequest.newBuilder()\n                    .setInvisibleDuration(Durations.fromSeconds(-1))\n                    .setTopic(Resource.newBuilder().setName(TOPIC).build())\n                    .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build())\n                    .setMessageId(\"msgId\")\n                    .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))\n                    .build()\n            ).get();\n        } catch (ExecutionException executionException) {\n            GrpcProxyException exception = (GrpcProxyException) executionException.getCause();\n            assertEquals(Code.ILLEGAL_INVISIBLE_TIME, exception.getCode());\n        }\n    }\n\n    @Test\n    public void testChangeInvisibleDurationInvisibleTimeTooLarge() throws Throwable {\n        try {\n            this.changeInvisibleDurationActivity.changeInvisibleDuration(\n                createContext(),\n                ChangeInvisibleDurationRequest.newBuilder()\n                    .setInvisibleDuration(Durations.fromDays(7))\n                    .setTopic(Resource.newBuilder().setName(TOPIC).build())\n                    .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build())\n                    .setMessageId(\"msgId\")\n                    .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000))\n                    .build()\n            ).get();\n        } catch (ExecutionException executionException) {\n            GrpcProxyException exception = (GrpcProxyException) executionException.getCause();\n            assertEquals(Code.ILLEGAL_INVISIBLE_TIME, exception.getCode());\n        }\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2.consumer;\n\nimport apache.rocketmq.v2.Code;\nimport apache.rocketmq.v2.FilterExpression;\nimport apache.rocketmq.v2.FilterType;\nimport apache.rocketmq.v2.MessageQueue;\nimport apache.rocketmq.v2.ReceiveMessageRequest;\nimport apache.rocketmq.v2.ReceiveMessageResponse;\nimport apache.rocketmq.v2.Resource;\nimport apache.rocketmq.v2.Settings;\nimport com.google.protobuf.Duration;\nimport com.google.protobuf.util.Durations;\nimport io.grpc.stub.ServerCallStreamObserver;\nimport io.grpc.stub.StreamObserver;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.stream.Collectors;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PopStatus;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest;\nimport org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;\nimport org.apache.rocketmq.proxy.service.route.MessageQueueView;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.ArgumentCaptor;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotEquals;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyBoolean;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.isNull;\nimport static org.mockito.Mockito.doNothing;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\n\npublic class ReceiveMessageActivityTest extends BaseActivityTest {\n\n    protected static final String BROKER_NAME = \"broker\";\n    protected static final String CLUSTER_NAME = \"cluster\";\n    protected static final String BROKER_ADDR = \"127.0.0.1:10911\";\n    private static final String TOPIC = \"topic\";\n    private static final String CONSUMER_GROUP = \"consumerGroup\";\n    private ReceiveMessageActivity receiveMessageActivity;\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        ConfigurationManager.getProxyConfig().setGrpcClientConsumerMinLongPollingTimeoutMillis(0);\n        this.receiveMessageActivity = new ReceiveMessageActivity(messagingProcessor,\n            grpcClientSettingsManager, grpcChannelManager);\n    }\n\n    @Test\n    public void testReceiveMessagePollingTime() {\n        StreamObserver<ReceiveMessageResponse> receiveStreamObserver = mock(ServerCallStreamObserver.class);\n        ArgumentCaptor<ReceiveMessageResponse> responseArgumentCaptor = ArgumentCaptor.forClass(ReceiveMessageResponse.class);\n        doNothing().when(receiveStreamObserver).onNext(responseArgumentCaptor.capture());\n\n        ArgumentCaptor<Long> pollTimeCaptor = ArgumentCaptor.forClass(Long.class);\n        when(this.grpcClientSettingsManager.getClientSettings(any())).thenReturn(Settings.newBuilder()\n            .setRequestTimeout(Durations.fromSeconds(3))\n            .build());\n        when(this.messagingProcessor.popMessage(any(), any(), anyString(), anyString(), anyInt(), anyLong(),\n            pollTimeCaptor.capture(), anyInt(), any(), anyBoolean(), any(), isNull(), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(new PopResult(PopStatus.NO_NEW_MSG, Collections.emptyList())));\n\n        ProxyContext context = createContext();\n        context.setRemainingMs(1L);\n        this.receiveMessageActivity.receiveMessage(\n            context,\n            ReceiveMessageRequest.newBuilder()\n                .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build())\n                .setMessageQueue(MessageQueue.newBuilder().setTopic(Resource.newBuilder().setName(TOPIC).build()).build())\n                .setAutoRenew(true)\n                .setFilterExpression(FilterExpression.newBuilder()\n                    .setType(FilterType.TAG)\n                    .setExpression(\"*\")\n                    .build())\n                .build(),\n            receiveStreamObserver\n        );\n\n        assertEquals(Code.MESSAGE_NOT_FOUND, getResponseCodeFromReceiveMessageResponseList(responseArgumentCaptor.getAllValues()));\n        assertEquals(0L, pollTimeCaptor.getValue().longValue());\n    }\n\n    @Test\n    public void testReceiveMessageWithIllegalPollingTime() {\n        StreamObserver<ReceiveMessageResponse> receiveStreamObserver = mock(ServerCallStreamObserver.class);\n        ArgumentCaptor<ReceiveMessageResponse> responseArgumentCaptor0 = ArgumentCaptor.forClass(ReceiveMessageResponse.class);\n        doNothing().when(receiveStreamObserver).onNext(responseArgumentCaptor0.capture());\n\n        when(this.grpcClientSettingsManager.getClientSettings(any())).thenReturn(Settings.newBuilder().getDefaultInstanceForType());\n\n        final ProxyContext context = createContext();\n        context.setClientVersion(\"5.0.2\");\n        context.setRemainingMs(-1L);\n        final ReceiveMessageRequest request = ReceiveMessageRequest.newBuilder()\n            .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build())\n            .setMessageQueue(MessageQueue.newBuilder().setTopic(Resource.newBuilder().setName(TOPIC).build()).build())\n            .setAutoRenew(false)\n            .setLongPollingTimeout(Duration.newBuilder().setSeconds(20).build())\n            .setFilterExpression(FilterExpression.newBuilder()\n                .setType(FilterType.TAG)\n                .setExpression(\"*\")\n                .build())\n            .build();\n        this.receiveMessageActivity.receiveMessage(\n            context,\n            request,\n            receiveStreamObserver\n        );\n        assertEquals(Code.BAD_REQUEST, getResponseCodeFromReceiveMessageResponseList(responseArgumentCaptor0.getAllValues()));\n\n        ArgumentCaptor<ReceiveMessageResponse> responseArgumentCaptor1 =\n            ArgumentCaptor.forClass(ReceiveMessageResponse.class);\n        doNothing().when(receiveStreamObserver).onNext(responseArgumentCaptor1.capture());\n        context.setClientVersion(\"5.0.3\");\n        this.receiveMessageActivity.receiveMessage(\n            context,\n            request,\n            receiveStreamObserver\n        );\n        assertEquals(Code.ILLEGAL_POLLING_TIME,\n            getResponseCodeFromReceiveMessageResponseList(responseArgumentCaptor1.getAllValues()));\n    }\n\n    @Test\n    public void testReceiveMessageIllegalFilter() {\n        StreamObserver<ReceiveMessageResponse> receiveStreamObserver = mock(ServerCallStreamObserver.class);\n        ArgumentCaptor<ReceiveMessageResponse> responseArgumentCaptor = ArgumentCaptor.forClass(ReceiveMessageResponse.class);\n        doNothing().when(receiveStreamObserver).onNext(responseArgumentCaptor.capture());\n\n        when(this.grpcClientSettingsManager.getClientSettings(any())).thenReturn(Settings.newBuilder().getDefaultInstanceForType());\n\n        this.receiveMessageActivity.receiveMessage(\n            createContext(),\n            ReceiveMessageRequest.newBuilder()\n                .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build())\n                .setMessageQueue(MessageQueue.newBuilder().setTopic(Resource.newBuilder().setName(TOPIC).build()).build())\n                .setAutoRenew(true)\n                .setFilterExpression(FilterExpression.newBuilder()\n                    .setType(FilterType.SQL)\n                    .setExpression(\"\")\n                    .build())\n                .build(),\n            receiveStreamObserver\n        );\n\n        assertEquals(Code.ILLEGAL_FILTER_EXPRESSION, getResponseCodeFromReceiveMessageResponseList(responseArgumentCaptor.getAllValues()));\n    }\n\n    @Test\n    public void testReceiveMessageIllegalInvisibleTimeTooSmall() {\n        StreamObserver<ReceiveMessageResponse> receiveStreamObserver = mock(ServerCallStreamObserver.class);\n        ArgumentCaptor<ReceiveMessageResponse> responseArgumentCaptor = ArgumentCaptor.forClass(ReceiveMessageResponse.class);\n        doNothing().when(receiveStreamObserver).onNext(responseArgumentCaptor.capture());\n\n        when(this.grpcClientSettingsManager.getClientSettings(any())).thenReturn(Settings.newBuilder().getDefaultInstanceForType());\n\n        this.receiveMessageActivity.receiveMessage(\n            createContext(),\n            ReceiveMessageRequest.newBuilder()\n                .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build())\n                .setMessageQueue(MessageQueue.newBuilder().setTopic(Resource.newBuilder().setName(TOPIC).build()).build())\n                .setAutoRenew(false)\n                .setInvisibleDuration(Durations.fromSeconds(0))\n                .build(),\n            receiveStreamObserver\n        );\n\n        assertEquals(Code.ILLEGAL_INVISIBLE_TIME, getResponseCodeFromReceiveMessageResponseList(responseArgumentCaptor.getAllValues()));\n    }\n\n    @Test\n    public void testReceiveMessageIllegalInvisibleTimeTooLarge() {\n        StreamObserver<ReceiveMessageResponse> receiveStreamObserver = mock(ServerCallStreamObserver.class);\n        ArgumentCaptor<ReceiveMessageResponse> responseArgumentCaptor = ArgumentCaptor.forClass(ReceiveMessageResponse.class);\n        doNothing().when(receiveStreamObserver).onNext(responseArgumentCaptor.capture());\n\n        when(this.grpcClientSettingsManager.getClientSettings(any())).thenReturn(Settings.newBuilder().getDefaultInstanceForType());\n\n        this.receiveMessageActivity.receiveMessage(\n            createContext(),\n            ReceiveMessageRequest.newBuilder()\n                .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build())\n                .setMessageQueue(MessageQueue.newBuilder().setTopic(Resource.newBuilder().setName(TOPIC).build()).build())\n                .setAutoRenew(false)\n                .setInvisibleDuration(Durations.fromDays(7))\n                .build(),\n            receiveStreamObserver\n        );\n\n        assertEquals(Code.ILLEGAL_INVISIBLE_TIME, getResponseCodeFromReceiveMessageResponseList(responseArgumentCaptor.getAllValues()));\n    }\n\n    @Test\n    public void testReceiveMessageAddReceiptHandle() {\n        ConfigurationManager.getProxyConfig().setEnableProxyAutoRenew(true);\n        StreamObserver<ReceiveMessageResponse> receiveStreamObserver = mock(ServerCallStreamObserver.class);\n        doNothing().when(receiveStreamObserver).onNext(any());\n        when(this.grpcClientSettingsManager.getClientSettings(any())).thenReturn(Settings.newBuilder().getDefaultInstanceForType());\n\n        MessageExt messageExt1 = new MessageExt();\n        String msgId1 = \"msgId1\";\n        String popCk1 = \"0 0 60000 0 0 broker 0 0 0\";\n        messageExt1.setTopic(TOPIC);\n        messageExt1.setMsgId(msgId1);\n        MessageAccessor.putProperty(messageExt1, MessageConst.PROPERTY_POP_CK, popCk1);\n        messageExt1.setBody(\"body1\".getBytes());\n        MessageExt messageExt2 = new MessageExt();\n        String msgId2 = \"msgId2\";\n        String popCk2 = \"0 0 60000 0 0 broker 0 1 1000\";\n        messageExt2.setTopic(TOPIC);\n        messageExt2.setMsgId(msgId2);\n        MessageAccessor.putProperty(messageExt2, MessageConst.PROPERTY_POP_CK, popCk2);\n        messageExt2.setBody(\"body2\".getBytes());\n        PopResult popResult = new PopResult(PopStatus.FOUND, Arrays.asList(messageExt1, messageExt2));\n        when(this.messagingProcessor.popMessage(\n            any(),\n            any(),\n            anyString(),\n            anyString(),\n            anyInt(),\n            anyLong(),\n            anyLong(),\n            anyInt(),\n            any(),\n            anyBoolean(),\n            any(),\n            isNull(),\n            anyLong())).thenReturn(CompletableFuture.completedFuture(popResult));\n        ArgumentCaptor<String> msgIdCaptor = ArgumentCaptor.forClass(String.class);\n        ArgumentCaptor<ReceiptHandle> receiptHandleCaptor = ArgumentCaptor.forClass(ReceiptHandle.class);\n        when(this.messagingProcessor.changeInvisibleTime(\n            any(),\n            receiptHandleCaptor.capture(),\n            msgIdCaptor.capture(),\n            anyString(),\n            anyString(),\n            anyLong(),\n            any(),\n            anyLong(),\n            anyBoolean())).thenReturn(CompletableFuture.completedFuture(new AckResult()));\n\n        // normal\n        ProxyContext ctx = createContext();\n        this.grpcChannelManager.createChannel(ctx, ctx.getClientID());\n        ReceiveMessageRequest receiveMessageRequest = ReceiveMessageRequest.newBuilder()\n            .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build())\n            .setMessageQueue(MessageQueue.newBuilder().setTopic(Resource.newBuilder().setName(TOPIC).build()).build())\n            .setAutoRenew(true)\n            .setFilterExpression(FilterExpression.newBuilder()\n                .setType(FilterType.TAG)\n                .setExpression(\"*\")\n                .build())\n            .build();\n        this.receiveMessageActivity.receiveMessage(ctx, receiveMessageRequest, receiveStreamObserver);\n        verify(this.messagingProcessor, times(0)).changeInvisibleTime(\n            any(),\n            any(),\n            anyString(),\n            anyString(),\n            anyString(),\n            anyLong());\n\n        // abnormal\n        this.grpcChannelManager.removeChannel(ctx.getClientID());\n        this.receiveMessageActivity.receiveMessage(ctx, receiveMessageRequest, receiveStreamObserver);\n        verify(this.messagingProcessor, times(2)).changeInvisibleTime(\n            any(),\n            any(),\n            anyString(),\n            anyString(),\n            anyString(),\n            anyLong(),\n            any(),\n            anyLong(),\n            anyBoolean());\n        assertEquals(Arrays.asList(msgId1, msgId2), msgIdCaptor.getAllValues());\n        assertEquals(Arrays.asList(popCk1, popCk2), receiptHandleCaptor.getAllValues().stream().map(ReceiptHandle::encode).collect(Collectors.toList()));\n    }\n\n    @Test\n    public void testReceiveMessage() {\n        StreamObserver<ReceiveMessageResponse> receiveStreamObserver = mock(ServerCallStreamObserver.class);\n        ArgumentCaptor<ReceiveMessageResponse> responseArgumentCaptor = ArgumentCaptor.forClass(ReceiveMessageResponse.class);\n        doNothing().when(receiveStreamObserver).onNext(responseArgumentCaptor.capture());\n\n        when(this.grpcClientSettingsManager.getClientSettings(any())).thenReturn(Settings.newBuilder().getDefaultInstanceForType());\n\n        PopResult popResult = new PopResult(PopStatus.NO_NEW_MSG, new ArrayList<>());\n        when(this.messagingProcessor.popMessage(\n            any(),\n            any(),\n            anyString(),\n            anyString(),\n            anyInt(),\n            anyLong(),\n            anyLong(),\n            anyInt(),\n            any(),\n            anyBoolean(),\n            any(),\n            isNull(),\n            anyLong())).thenReturn(CompletableFuture.completedFuture(popResult));\n\n        this.receiveMessageActivity.receiveMessage(\n            createContext(),\n            ReceiveMessageRequest.newBuilder()\n                .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build())\n                .setMessageQueue(MessageQueue.newBuilder().setTopic(Resource.newBuilder().setName(TOPIC).build()).build())\n                .setAutoRenew(true)\n                .setFilterExpression(FilterExpression.newBuilder()\n                    .setType(FilterType.TAG)\n                    .setExpression(\"*\")\n                    .build())\n                .build(),\n            receiveStreamObserver\n        );\n        assertEquals(Code.MESSAGE_NOT_FOUND, getResponseCodeFromReceiveMessageResponseList(responseArgumentCaptor.getAllValues()));\n    }\n\n    private Code getResponseCodeFromReceiveMessageResponseList(List<ReceiveMessageResponse> responseList) {\n        for (ReceiveMessageResponse response : responseList) {\n            if (response.hasStatus()) {\n                return response.getStatus().getCode();\n            }\n        }\n        return null;\n    }\n\n    @Test\n    public void testReceiveMessageQueueSelector() throws Exception {\n        TopicRouteData topicRouteData = new TopicRouteData();\n        List<QueueData> queueDatas = new ArrayList<>();\n        for (int i = 0; i < 2; i++) {\n            QueueData queueData = new QueueData();\n            queueData.setBrokerName(BROKER_NAME + i);\n            queueData.setReadQueueNums(1);\n            queueData.setPerm(PermName.PERM_READ);\n            queueDatas.add(queueData);\n        }\n        topicRouteData.setQueueDatas(queueDatas);\n\n        List<BrokerData> brokerDatas = new ArrayList<>();\n        for (int i = 0; i < 2; i++) {\n            BrokerData brokerData = new BrokerData();\n            brokerData.setCluster(CLUSTER_NAME);\n            brokerData.setBrokerName(BROKER_NAME + i);\n            HashMap<Long, String> brokerAddrs = new HashMap<>();\n            brokerAddrs.put(MixAll.MASTER_ID, BROKER_ADDR);\n            brokerData.setBrokerAddrs(brokerAddrs);\n            brokerDatas.add(brokerData);\n        }\n        topicRouteData.setBrokerDatas(brokerDatas);\n\n        MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, null);\n        ReceiveMessageActivity.ReceiveMessageQueueSelector selector = new ReceiveMessageActivity.ReceiveMessageQueueSelector(\"\");\n\n        AddressableMessageQueue firstSelect = selector.select(ProxyContext.create(), messageQueueView);\n        AddressableMessageQueue secondSelect = selector.select(ProxyContext.create(), messageQueueView);\n        AddressableMessageQueue thirdSelect = selector.select(ProxyContext.create(), messageQueueView);\n\n        assertEquals(firstSelect, thirdSelect);\n        assertNotEquals(firstSelect, secondSelect);\n\n        for (int i = 0; i < 2; i++) {\n            ReceiveMessageActivity.ReceiveMessageQueueSelector selectorBrokerName =\n                new ReceiveMessageActivity.ReceiveMessageQueueSelector(BROKER_NAME + i);\n            assertEquals(BROKER_NAME + i, selectorBrokerName.select(ProxyContext.create(), messageQueueView).getBrokerName());\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageResponseStreamWriterTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2.consumer;\n\nimport apache.rocketmq.v2.Code;\nimport apache.rocketmq.v2.FilterExpression;\nimport apache.rocketmq.v2.FilterType;\nimport apache.rocketmq.v2.MessageQueue;\nimport apache.rocketmq.v2.ReceiveMessageRequest;\nimport apache.rocketmq.v2.ReceiveMessageResponse;\nimport apache.rocketmq.v2.Resource;\nimport io.grpc.stub.StreamObserver;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.UUID;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PopStatus;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport java.lang.reflect.Method;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest;\nimport org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.ArgumentCaptor;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyBoolean;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.doAnswer;\nimport static org.mockito.Mockito.doNothing;\nimport static org.mockito.Mockito.doReturn;\nimport static org.mockito.Mockito.doThrow;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\n\npublic class ReceiveMessageResponseStreamWriterTest extends BaseActivityTest {\n\n    private static final String TOPIC = \"topic\";\n    private static final String CONSUMER_GROUP = \"consumerGroup\";\n    private ReceiveMessageResponseStreamWriter writer;\n    private StreamObserver<ReceiveMessageResponse> streamObserver;\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        this.streamObserver = mock(StreamObserver.class);\n        this.writer = new ReceiveMessageResponseStreamWriter(this.messagingProcessor, this.streamObserver);\n    }\n\n    @Test\n    public void testWriteMessage() {\n        ArgumentCaptor<String> changeInvisibleTimeMsgIdCaptor = ArgumentCaptor.forClass(String.class);\n        doReturn(CompletableFuture.completedFuture(mock(AckResult.class))).when(this.messagingProcessor)\n            .changeInvisibleTime(any(), any(), changeInvisibleTimeMsgIdCaptor.capture(), anyString(), anyString(), anyLong(), any(), anyLong(), anyBoolean());\n\n        ArgumentCaptor<ReceiveMessageResponse> responseArgumentCaptor = ArgumentCaptor.forClass(ReceiveMessageResponse.class);\n        AtomicInteger onNextCallNum = new AtomicInteger(0);\n        doAnswer(mock -> {\n            if (onNextCallNum.incrementAndGet() > 2) {\n                throw new RuntimeException();\n            }\n            return null;\n        }).when(streamObserver).onNext(responseArgumentCaptor.capture());\n\n        List<MessageExt> messageExtList = new ArrayList<>();\n        messageExtList.add(createMessageExt(TOPIC, \"tag\"));\n        messageExtList.add(createMessageExt(TOPIC, \"tag\"));\n        PopResult popResult = new PopResult(PopStatus.FOUND, messageExtList);\n        ReceiveMessageRequest receiveMessageRequest = ReceiveMessageRequest.newBuilder()\n            .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build())\n            .setMessageQueue(MessageQueue.newBuilder().setTopic(Resource.newBuilder().setName(TOPIC).build()).build())\n            .setFilterExpression(FilterExpression.newBuilder()\n                .setType(FilterType.TAG)\n                .setExpression(\"*\")\n                .build())\n            .build();\n        writer.writeAndComplete(\n            ProxyContext.create(),\n            receiveMessageRequest,\n            popResult\n        );\n\n        verify(streamObserver, times(1)).onCompleted();\n        verify(streamObserver, times(4)).onNext(any());\n        verify(this.messagingProcessor, times(1))\n            .changeInvisibleTime(any(), any(), anyString(), anyString(), anyString(), anyLong(), any(), anyLong(), eq(true));\n\n        assertTrue(responseArgumentCaptor.getAllValues().get(0).hasStatus());\n        assertEquals(Code.OK, responseArgumentCaptor.getAllValues().get(0).getStatus().getCode());\n        assertTrue(responseArgumentCaptor.getAllValues().get(1).hasMessage());\n        assertEquals(messageExtList.get(0).getMsgId(), responseArgumentCaptor.getAllValues().get(1).getMessage().getSystemProperties().getMessageId());\n\n        assertEquals(messageExtList.get(1).getMsgId(), changeInvisibleTimeMsgIdCaptor.getValue());\n\n        // case: fail to write response status at first step\n        doThrow(new RuntimeException()).when(streamObserver).onNext(any());\n        writer.writeAndComplete(\n            ProxyContext.create(),\n            receiveMessageRequest,\n            popResult\n        );\n        verify(this.messagingProcessor, times(3))\n            .changeInvisibleTime(any(), any(), anyString(), anyString(), anyString(), anyLong(), any(), anyLong(), eq(true));\n    }\n\n    @Test\n    public void testPollingFull() {\n        ArgumentCaptor<ReceiveMessageResponse> responseArgumentCaptor = ArgumentCaptor.forClass(ReceiveMessageResponse.class);\n        doNothing().when(streamObserver).onNext(responseArgumentCaptor.capture());\n\n        PopResult popResult = new PopResult(PopStatus.POLLING_FULL, new ArrayList<>());\n        writer.writeAndComplete(\n            ProxyContext.create(),\n            ReceiveMessageRequest.newBuilder()\n                .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build())\n                .setMessageQueue(MessageQueue.newBuilder().setTopic(Resource.newBuilder().setName(TOPIC).build()).build())\n                .setFilterExpression(FilterExpression.newBuilder()\n                    .setType(FilterType.TAG)\n                    .setExpression(\"*\")\n                    .build())\n                .build(),\n            popResult\n        );\n\n        ReceiveMessageResponse response = responseArgumentCaptor.getAllValues().stream().filter(ReceiveMessageResponse::hasStatus)\n            .findFirst().get();\n        assertEquals(Code.TOO_MANY_REQUESTS, response.getStatus().getCode());\n    }\n\n    @Test\n    public void testNackMessageWithSuspendTrue() {\n        ArgumentCaptor<String> changeInvisibleTimeMsgIdCaptor = ArgumentCaptor.forClass(String.class);\n        ArgumentCaptor<String> changeInvisibleTimeGroupCaptor = ArgumentCaptor.forClass(String.class);\n        ArgumentCaptor<String> changeInvisibleTimeTopicCaptor = ArgumentCaptor.forClass(String.class);\n        ArgumentCaptor<Long> changeInvisibleTimeInvisibleTimeCaptor = ArgumentCaptor.forClass(Long.class);\n        ArgumentCaptor<Boolean> changeInvisibleTimeSuspendCaptor = ArgumentCaptor.forClass(Boolean.class);\n\n        doReturn(CompletableFuture.completedFuture(mock(AckResult.class))).when(this.messagingProcessor)\n            .changeInvisibleTime(any(), any(), changeInvisibleTimeMsgIdCaptor.capture(),\n                changeInvisibleTimeGroupCaptor.capture(), changeInvisibleTimeTopicCaptor.capture(),\n                changeInvisibleTimeInvisibleTimeCaptor.capture(), any(), anyLong(),\n                changeInvisibleTimeSuspendCaptor.capture());\n\n        MessageExt messageExt = createMessageExt(TOPIC, \"tag\");\n        ReceiveMessageRequest receiveMessageRequest = ReceiveMessageRequest.newBuilder()\n            .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build())\n            .setMessageQueue(MessageQueue.newBuilder().setTopic(Resource.newBuilder().setName(TOPIC).build()).build())\n            .build();\n\n        // Simulate nack by calling processThrowableWhenWriteMessage using reflection\n        // This is called when an exception occurs during message processing\n        try {\n            Method method = ReceiveMessageResponseStreamWriter.class.getDeclaredMethod(\n                \"processThrowableWhenWriteMessage\",\n                Throwable.class, ProxyContext.class, ReceiveMessageRequest.class, MessageExt.class);\n            method.setAccessible(true);\n            method.invoke(writer,\n                new RuntimeException(\"Test exception\"),\n                ProxyContext.create(),\n                receiveMessageRequest,\n                messageExt);\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n\n        // Verify that changeInvisibleTime was called with suspend=true\n        verify(this.messagingProcessor, times(1))\n            .changeInvisibleTime(any(), any(), eq(messageExt.getMsgId()),\n                eq(CONSUMER_GROUP), eq(TOPIC), eq(ReceiveMessageResponseStreamWriter.NACK_INVISIBLE_TIME),\n                eq(null), eq(org.apache.rocketmq.proxy.processor.MessagingProcessor.DEFAULT_TIMEOUT_MILLS),\n                eq(true));\n\n        assertEquals(messageExt.getMsgId(), changeInvisibleTimeMsgIdCaptor.getValue());\n        assertEquals(CONSUMER_GROUP, changeInvisibleTimeGroupCaptor.getValue());\n        assertEquals(TOPIC, changeInvisibleTimeTopicCaptor.getValue());\n        assertEquals(ReceiveMessageResponseStreamWriter.NACK_INVISIBLE_TIME,\n            changeInvisibleTimeInvisibleTimeCaptor.getValue().longValue());\n        assertTrue(\"Suspend should be true for nack\", changeInvisibleTimeSuspendCaptor.getValue());\n    }\n\n\n    private static MessageExt createMessageExt(String topic, String tags) {\n        String msgId = MessageClientIDSetter.createUniqID();\n\n        MessageExt messageExt = new MessageExt();\n        messageExt.setTopic(topic);\n        messageExt.setTags(tags);\n        messageExt.setBody(UUID.randomUUID().toString().getBytes(StandardCharsets.UTF_8));\n        messageExt.setMsgId(msgId);\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, msgId);\n        messageExt.setCommitLogOffset(RANDOM.nextInt(Integer.MAX_VALUE));\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_POP_CK,\n            ExtraInfoUtil.buildExtraInfo(RANDOM.nextInt(Integer.MAX_VALUE), System.currentTimeMillis(), 3000,\n                RANDOM.nextInt(Integer.MAX_VALUE), topic, \"mockBroker\", RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE)));\n        return messageExt;\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/ForwardMessageToDLQActivityTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2.producer;\n\nimport apache.rocketmq.v2.Code;\nimport apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest;\nimport apache.rocketmq.v2.ForwardMessageToDeadLetterQueueResponse;\nimport apache.rocketmq.v2.Resource;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.proxy.common.MessageReceiptHandle;\nimport org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.ArgumentCaptor;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.when;\n\npublic class ForwardMessageToDLQActivityTest extends BaseActivityTest {\n\n    private ForwardMessageToDLQActivity forwardMessageToDLQActivity;\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        this.forwardMessageToDLQActivity = new ForwardMessageToDLQActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n    }\n\n    @Test\n    public void testForwardMessageToDeadLetterQueue() throws Throwable {\n        ArgumentCaptor<ReceiptHandle> receiptHandleCaptor = ArgumentCaptor.forClass(ReceiptHandle.class);\n        when(this.messagingProcessor.forwardMessageToDeadLetterQueue(any(), receiptHandleCaptor.capture(), anyString(), anyString(), anyString(), any()))\n            .thenReturn(CompletableFuture.completedFuture(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"\")));\n\n        String handleStr = buildReceiptHandle(\"topic\", System.currentTimeMillis(), 3000);\n        ForwardMessageToDeadLetterQueueResponse response = this.forwardMessageToDLQActivity.forwardMessageToDeadLetterQueue(\n            createContext(),\n            ForwardMessageToDeadLetterQueueRequest.newBuilder()\n                .setTopic(Resource.newBuilder().setName(\"topic\").build())\n                .setGroup(Resource.newBuilder().setName(\"group\").build())\n                .setMessageId(MessageClientIDSetter.createUniqID())\n                .setReceiptHandle(handleStr)\n                .build()\n        ).get();\n\n        assertEquals(Code.OK, response.getStatus().getCode());\n        assertEquals(handleStr, receiptHandleCaptor.getValue().getReceiptHandle());\n    }\n\n    @Test\n    public void testForwardMessageToDeadLetterQueueWhenHasMappingHandle() throws Throwable {\n        ArgumentCaptor<ReceiptHandle> receiptHandleCaptor = ArgumentCaptor.forClass(ReceiptHandle.class);\n        when(this.messagingProcessor.forwardMessageToDeadLetterQueue(any(), receiptHandleCaptor.capture(), anyString(), anyString(), anyString(), any()))\n            .thenReturn(CompletableFuture.completedFuture(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"\")));\n\n        String savedHandleStr = buildReceiptHandle(\"topic\", System.currentTimeMillis(),3000);\n        when(messagingProcessor.removeReceiptHandle(any(), any(), anyString(), anyString(), anyString()))\n            .thenReturn(new MessageReceiptHandle(\"group\", \"topic\", 0, savedHandleStr, \"msgId\", 0, 0));\n\n        ForwardMessageToDeadLetterQueueResponse response = this.forwardMessageToDLQActivity.forwardMessageToDeadLetterQueue(\n            createContext(),\n            ForwardMessageToDeadLetterQueueRequest.newBuilder()\n                .setTopic(Resource.newBuilder().setName(\"topic\").build())\n                .setGroup(Resource.newBuilder().setName(\"group\").build())\n                .setMessageId(MessageClientIDSetter.createUniqID())\n                .setReceiptHandle(buildReceiptHandle(\"topic\", System.currentTimeMillis(), 3000))\n                .build()\n        ).get();\n\n        assertEquals(Code.OK, response.getStatus().getCode());\n        assertEquals(savedHandleStr, receiptHandleCaptor.getValue().getReceiptHandle());\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/RecallMessageActivityTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2.producer;\n\nimport apache.rocketmq.v2.Code;\nimport apache.rocketmq.v2.RecallMessageRequest;\nimport apache.rocketmq.v2.RecallMessageResponse;\nimport apache.rocketmq.v2.Resource;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\nimport org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.CompletionException;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.Mockito.when;\n\npublic class RecallMessageActivityTest extends BaseActivityTest {\n    private RecallMessageActivity recallMessageActivity;\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        this.recallMessageActivity =\n            new RecallMessageActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n    }\n\n    @Test\n    public void testRecallMessage_success() {\n        when(this.messagingProcessor.recallMessage(any(), any(), any(), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(\"msgId\"));\n\n        RecallMessageResponse response = this.recallMessageActivity.recallMessage(\n            createContext(),\n            RecallMessageRequest.newBuilder()\n                .setRecallHandle(\"handle\")\n                .setTopic(Resource.newBuilder().setResourceNamespace(\"ns\").setName(\"topic\"))\n                .build()\n        ).join();\n\n        assertEquals(Code.OK, response.getStatus().getCode());\n        assertEquals(\"msgId\", response.getMessageId());\n    }\n\n    @Test\n    public void testRecallMessage_fail() {\n        CompletableFuture<String> exceptionFuture = new CompletableFuture();\n        when(this.messagingProcessor.recallMessage(any(), any(), any(), anyLong())).thenReturn(exceptionFuture);\n        exceptionFuture.completeExceptionally(\n            new ProxyException(ProxyExceptionCode.MESSAGE_PROPERTY_CONFLICT_WITH_TYPE, \"info\"));\n\n        CompletionException exception = Assert.assertThrows(CompletionException.class, () -> {\n            this.recallMessageActivity.recallMessage(\n                createContext(),\n                RecallMessageRequest.newBuilder()\n                    .setRecallHandle(\"handle\")\n                    .setTopic(Resource.newBuilder().setResourceNamespace(\"ns\").setName(\"topic\"))\n                    .build()\n            ).join();\n        });\n        Assert.assertTrue(exception.getCause() instanceof ProxyException);\n        ProxyException cause = (ProxyException) exception.getCause();\n        Assert.assertEquals(ProxyExceptionCode.MESSAGE_PROPERTY_CONFLICT_WITH_TYPE, cause.getCode());\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2.producer;\n\nimport apache.rocketmq.v2.Code;\nimport apache.rocketmq.v2.Encoding;\nimport apache.rocketmq.v2.Message;\nimport apache.rocketmq.v2.MessageType;\nimport apache.rocketmq.v2.Resource;\nimport apache.rocketmq.v2.SendMessageRequest;\nimport apache.rocketmq.v2.SendMessageResponse;\nimport apache.rocketmq.v2.SystemProperties;\nimport com.google.protobuf.ByteString;\nimport com.google.protobuf.util.Durations;\nimport com.google.protobuf.util.Timestamps;\nimport java.time.Duration;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.latency.MQFaultStrategy;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException;\nimport org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;\nimport org.apache.rocketmq.proxy.service.route.MessageQueueView;\nimport org.apache.rocketmq.proxy.service.route.TopicRouteService;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.assertj.core.util.Lists;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.apache.rocketmq.proxy.service.route.TopicRouteService.buildPenalizerByMQFaultStrategy;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotEquals;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertThrows;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\npublic class SendMessageActivityTest extends BaseActivityTest {\n\n    protected static final String BROKER_NAME = \"broker\";\n    protected static final String BROKER_NAME2 = \"broker2\";\n    protected static final String CLUSTER_NAME = \"cluster\";\n    protected static final String BROKER_ADDR = \"127.0.0.1:10911\";\n    protected static final String BROKER_ADDR2 = \"127.0.0.1:10912\";\n    private static final String TOPIC = \"topic\";\n    private static final String CONSUMER_GROUP = \"consumerGroup\";\n    MQFaultStrategy mqFaultStrategy;\n\n    private SendMessageActivity sendMessageActivity;\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        this.sendMessageActivity = new SendMessageActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n    }\n\n    @Test\n    public void sendMessage() throws Exception {\n        String msgId = MessageClientIDSetter.createUniqID();\n\n        SendResult sendResult = new SendResult();\n        sendResult.setSendStatus(SendStatus.SEND_OK);\n        sendResult.setMsgId(msgId);\n        when(this.messagingProcessor.sendMessage(any(), any(), anyString(), anyInt(), any()))\n            .thenReturn(CompletableFuture.completedFuture(Lists.newArrayList(sendResult)));\n\n        SendMessageResponse response = this.sendMessageActivity.sendMessage(\n            createContext(),\n            SendMessageRequest.newBuilder()\n                .addMessages(Message.newBuilder()\n                    .setTopic(Resource.newBuilder()\n                        .setName(TOPIC)\n                        .build())\n                    .setSystemProperties(SystemProperties.newBuilder()\n                        .setMessageId(msgId)\n                        .setQueueId(0)\n                        .setMessageType(MessageType.NORMAL)\n                        .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                        .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                        .build())\n                    .setBody(ByteString.copyFromUtf8(\"123\"))\n                    .build())\n                .build()\n        ).get();\n\n        assertEquals(Code.OK, response.getStatus().getCode());\n        assertEquals(msgId, response.getEntries(0).getMessageId());\n    }\n\n    @Test\n    public void testConvertToSendMessageResponse() {\n        {\n            SendMessageResponse response = this.sendMessageActivity.convertToSendMessageResponse(\n                ProxyContext.create(),\n                SendMessageRequest.newBuilder().build(),\n                Lists.newArrayList(new SendResult(SendStatus.FLUSH_DISK_TIMEOUT, null, null, null, 0))\n            );\n            assertEquals(Code.MASTER_PERSISTENCE_TIMEOUT, response.getStatus().getCode());\n            assertEquals(Code.MASTER_PERSISTENCE_TIMEOUT, response.getEntries(0).getStatus().getCode());\n        }\n\n        {\n            SendMessageResponse response = this.sendMessageActivity.convertToSendMessageResponse(\n                ProxyContext.create(),\n                SendMessageRequest.newBuilder().build(),\n                Lists.newArrayList(new SendResult(SendStatus.FLUSH_SLAVE_TIMEOUT, null, null, null, 0))\n            );\n            assertEquals(Code.SLAVE_PERSISTENCE_TIMEOUT, response.getStatus().getCode());\n            assertEquals(Code.SLAVE_PERSISTENCE_TIMEOUT, response.getEntries(0).getStatus().getCode());\n        }\n\n        {\n            SendMessageResponse response = this.sendMessageActivity.convertToSendMessageResponse(\n                ProxyContext.create(),\n                SendMessageRequest.newBuilder().build(),\n                Lists.newArrayList(new SendResult(SendStatus.SLAVE_NOT_AVAILABLE, null, null, null, 0))\n            );\n            assertEquals(Code.HA_NOT_AVAILABLE, response.getStatus().getCode());\n            assertEquals(Code.HA_NOT_AVAILABLE, response.getEntries(0).getStatus().getCode());\n        }\n\n        {\n            SendMessageResponse response = this.sendMessageActivity.convertToSendMessageResponse(\n                ProxyContext.create(),\n                SendMessageRequest.newBuilder().build(),\n                Lists.newArrayList(new SendResult(SendStatus.SEND_OK, null, null, null, 0))\n            );\n            assertEquals(Code.OK, response.getStatus().getCode());\n            assertEquals(Code.OK, response.getEntries(0).getStatus().getCode());\n        }\n\n        {\n            SendMessageResponse response = this.sendMessageActivity.convertToSendMessageResponse(\n                ProxyContext.create(),\n                SendMessageRequest.newBuilder().build(),\n                Lists.newArrayList(\n                    new SendResult(SendStatus.SEND_OK, null, null, null, 0),\n                    new SendResult(SendStatus.SLAVE_NOT_AVAILABLE, null, null, null, 0)\n                )\n            );\n            assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode());\n        }\n    }\n\n    @Test(expected = GrpcProxyException.class)\n    public void testBuildErrorMessage() {\n        this.sendMessageActivity.buildMessage(null,\n            Lists.newArrayList(\n                Message.newBuilder()\n                    .setTopic(Resource.newBuilder()\n                        .setName(TOPIC)\n                        .build())\n                    .setSystemProperties(SystemProperties.newBuilder()\n                        .setMessageId(MessageClientIDSetter.createUniqID())\n                        .setQueueId(0)\n                        .setMessageType(MessageType.NORMAL)\n                        .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                        .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                        .build())\n                    .setBody(ByteString.copyFromUtf8(\"123\"))\n                    .build(),\n                Message.newBuilder()\n                    .setTopic(Resource.newBuilder()\n                        .setName(TOPIC + 2)\n                        .build())\n                    .setSystemProperties(SystemProperties.newBuilder()\n                        .setMessageId(MessageClientIDSetter.createUniqID())\n                        .setQueueId(0)\n                        .setMessageType(MessageType.NORMAL)\n                        .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                        .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                        .build())\n                    .setBody(ByteString.copyFromUtf8(\"123\"))\n                    .build()\n            ),\n            Resource.newBuilder().setName(TOPIC).build());\n    }\n\n    @Test\n    public void testBuildMessage() {\n        long deliveryTime = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(5);\n        ConfigurationManager.getProxyConfig().setMessageDelayLevel(\"1s 5s\");\n        ConfigurationManager.getProxyConfig().initData();\n        String msgId = MessageClientIDSetter.createUniqID();\n\n        org.apache.rocketmq.common.message.Message messageExt = this.sendMessageActivity.buildMessage(null,\n            Lists.newArrayList(\n                Message.newBuilder()\n                    .setTopic(Resource.newBuilder()\n                        .setName(TOPIC)\n                        .build())\n                    .setSystemProperties(SystemProperties.newBuilder()\n                        .setMessageId(msgId)\n                        .setQueueId(0)\n                        .setMessageType(MessageType.DELAY)\n                        .setDeliveryTimestamp(Timestamps.fromMillis(deliveryTime))\n                        .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                        .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                        .build())\n                    .setBody(ByteString.copyFromUtf8(\"123\"))\n                    .build()\n            ),\n            Resource.newBuilder().setName(TOPIC).build()).get(0);\n\n        assertEquals(MessageClientIDSetter.getUniqID(messageExt), msgId);\n        assertEquals(deliveryTime, Long.parseLong(messageExt.getProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS)));\n    }\n\n    @Test\n    public void testBuildMessageWithLiteTopic() {\n        String msgId = MessageClientIDSetter.createUniqID();\n        String liteTopic = \"build-test-lite-topic\";\n        String topic = \"build-test-topic\";\n\n        org.apache.rocketmq.common.message.Message messageExt = this.sendMessageActivity.buildMessage(\n            ProxyContext.create(),\n            Message.newBuilder()\n                .setTopic(Resource.newBuilder()\n                    .setName(topic)\n                    .build())\n                .setSystemProperties(SystemProperties.newBuilder()\n                    .setMessageId(msgId)\n                    .setQueueId(0)\n                    .setMessageType(MessageType.LITE)\n                    .setLiteTopic(liteTopic)\n                    .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                    .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                    .build())\n                .setBody(ByteString.copyFromUtf8(\"test body\"))\n                .build(),\n            \"test-producer-group\"\n        );\n\n        assertEquals(liteTopic, messageExt.getProperty(MessageConst.PROPERTY_LITE_TOPIC));\n        assertNull(messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH));\n    }\n\n    @Test\n    public void testTxMessage() {\n        String msgId = MessageClientIDSetter.createUniqID();\n\n        Message message = Message.newBuilder()\n            .setTopic(Resource.newBuilder()\n                .setName(TOPIC)\n                .build())\n            .setSystemProperties(SystemProperties.newBuilder()\n                .setMessageId(msgId)\n                .setQueueId(0)\n                .setMessageType(MessageType.TRANSACTION)\n                .setOrphanedTransactionRecoveryDuration(Durations.fromSeconds(30))\n                .setBodyEncoding(Encoding.GZIP)\n                .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                .build())\n            .setBody(ByteString.copyFromUtf8(\"123\"))\n            .build();\n        org.apache.rocketmq.common.message.Message messageExt = this.sendMessageActivity.buildMessage(null,\n            Lists.newArrayList(\n                message\n            ),\n            Resource.newBuilder().setName(TOPIC).build()).get(0);\n\n        assertEquals(MessageClientIDSetter.getUniqID(messageExt), msgId);\n        assertEquals(MessageSysFlag.TRANSACTION_PREPARED_TYPE | MessageSysFlag.COMPRESSED_FLAG, sendMessageActivity.buildSysFlag(message));\n    }\n\n    @Test\n    public void testPriorityMessage() {\n        String msgId = MessageClientIDSetter.createUniqID();\n        Message message = Message.newBuilder()\n            .setTopic(Resource.newBuilder()\n                .setName(TOPIC)\n                .build())\n            .setSystemProperties(SystemProperties.newBuilder()\n                .setMessageId(msgId)\n                .setQueueId(0)\n                .setMessageType(MessageType.PRIORITY)\n                .setPriority(5)\n                .setBodyEncoding(Encoding.GZIP)\n                .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                .build())\n            .setBody(ByteString.copyFromUtf8(\"123\"))\n            .build();\n        org.apache.rocketmq.common.message.Message messageExt = this.sendMessageActivity.buildMessage(null,\n            Lists.newArrayList(\n                message\n            ),\n            Resource.newBuilder().setName(TOPIC).build()).get(0);\n\n        assertEquals(MessageClientIDSetter.getUniqID(messageExt), msgId);\n        assertEquals(5, messageExt.getPriority());\n    }\n\n    @Test\n    public void testSendOrderMessageQueueSelector() throws Exception {\n        TopicRouteData topicRouteData = new TopicRouteData();\n        QueueData queueData = new QueueData();\n        BrokerData brokerData = new BrokerData();\n        queueData.setBrokerName(BROKER_NAME);\n        queueData.setWriteQueueNums(8);\n        queueData.setPerm(PermName.PERM_WRITE);\n        topicRouteData.setQueueDatas(Lists.newArrayList(queueData));\n        brokerData.setCluster(CLUSTER_NAME);\n        brokerData.setBrokerName(BROKER_NAME);\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(MixAll.MASTER_ID, BROKER_ADDR);\n        brokerData.setBrokerAddrs(brokerAddrs);\n        topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData));\n\n        MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, null);\n        SendMessageActivity.SendMessageQueueSelector selector1 = new SendMessageActivity.SendMessageQueueSelector(\n            SendMessageRequest.newBuilder()\n                .addMessages(Message.newBuilder()\n                    .setSystemProperties(SystemProperties.newBuilder()\n                        .setMessageGroup(String.valueOf(1))\n                        .build())\n                    .build())\n                .build()\n        );\n\n        TopicRouteService topicRouteService = mock(TopicRouteService.class);\n        MQFaultStrategy mqFaultStrategy = mock(MQFaultStrategy.class);\n        when(topicRouteService.getAllMessageQueueView(any(), any())).thenReturn(messageQueueView);\n        when(topicRouteService.getMqFaultStrategy()).thenReturn(mqFaultStrategy);\n        when(mqFaultStrategy.isSendLatencyFaultEnable()).thenReturn(false);\n\n        SendMessageActivity.SendMessageQueueSelector selector2 = new SendMessageActivity.SendMessageQueueSelector(\n            SendMessageRequest.newBuilder()\n                .addMessages(Message.newBuilder()\n                    .setSystemProperties(SystemProperties.newBuilder()\n                        .setMessageGroup(String.valueOf(1))\n                        .build())\n                    .build())\n                .build()\n        );\n\n        SendMessageActivity.SendMessageQueueSelector selector3 = new SendMessageActivity.SendMessageQueueSelector(\n            SendMessageRequest.newBuilder()\n                .addMessages(Message.newBuilder()\n                    .setSystemProperties(SystemProperties.newBuilder()\n                        .setMessageGroup(String.valueOf(2))\n                        .build())\n                    .build())\n                .build()\n        );\n\n        assertEquals(selector1.select(ProxyContext.create(), messageQueueView), selector2.select(ProxyContext.create(), messageQueueView));\n        assertNotEquals(selector1.select(ProxyContext.create(), messageQueueView), selector3.select(ProxyContext.create(), messageQueueView));\n    }\n\n    @Test\n    public void testSendNormalMessageQueueSelector() {\n        TopicRouteData topicRouteData = new TopicRouteData();\n        QueueData queueData = new QueueData();\n        BrokerData brokerData = new BrokerData();\n        queueData.setBrokerName(BROKER_NAME);\n        queueData.setWriteQueueNums(2);\n        queueData.setPerm(PermName.PERM_WRITE);\n        topicRouteData.setQueueDatas(Lists.newArrayList(queueData));\n        brokerData.setCluster(CLUSTER_NAME);\n        brokerData.setBrokerName(BROKER_NAME);\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(MixAll.MASTER_ID, BROKER_ADDR);\n        brokerData.setBrokerAddrs(brokerAddrs);\n        topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData));\n\n\n        SendMessageActivity.SendMessageQueueSelector selector = new SendMessageActivity.SendMessageQueueSelector(\n            SendMessageRequest.newBuilder()\n                .addMessages(Message.newBuilder().build())\n                .build()\n        );\n        TopicRouteService topicRouteService = mock(TopicRouteService.class);\n        MQFaultStrategy mqFaultStrategy = mock(MQFaultStrategy.class);\n        when(topicRouteService.getMqFaultStrategy()).thenReturn(mqFaultStrategy);\n        when(mqFaultStrategy.isSendLatencyFaultEnable()).thenReturn(false);\n        MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, null);\n\n        AddressableMessageQueue firstSelect = selector.select(ProxyContext.create(), messageQueueView);\n        AddressableMessageQueue secondSelect = selector.select(ProxyContext.create(), messageQueueView);\n        AddressableMessageQueue thirdSelect = selector.select(ProxyContext.create(), messageQueueView);\n\n        assertEquals(firstSelect, thirdSelect);\n        assertNotEquals(firstSelect, secondSelect);\n    }\n\n    @Test\n    public void testSendNormalMessageQueueSelectorPipeLine() throws Exception {\n        TopicRouteData topicRouteData = new TopicRouteData();\n        int queueNums = 2;\n\n        QueueData queueData = createQueueData(BROKER_NAME, queueNums);\n        QueueData queueData2 = createQueueData(BROKER_NAME2, queueNums);\n        topicRouteData.setQueueDatas(Lists.newArrayList(queueData,queueData2));\n\n\n        BrokerData brokerData = createBrokerData(CLUSTER_NAME, BROKER_NAME, BROKER_ADDR);\n        BrokerData brokerData2 = createBrokerData(CLUSTER_NAME, BROKER_NAME2, BROKER_ADDR2);\n        topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData, brokerData2));\n\n        SendMessageActivity.SendMessageQueueSelector selector = new SendMessageActivity.SendMessageQueueSelector(\n                SendMessageRequest.newBuilder()\n                        .addMessages(Message.newBuilder().build())\n                        .build()\n        );\n\n        ClientConfig cc = new ClientConfig();\n        this.mqFaultStrategy = new MQFaultStrategy(cc, null, null);\n        mqFaultStrategy.setSendLatencyFaultEnable(true);\n        mqFaultStrategy.updateFaultItem(BROKER_NAME2, 1000, true, true);\n        mqFaultStrategy.updateFaultItem(BROKER_NAME, 1000, true, false);\n\n        MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, buildPenalizerByMQFaultStrategy(mqFaultStrategy));\n\n        AddressableMessageQueue firstSelect = selector.select(ProxyContext.create(), messageQueueView);\n        assertEquals(firstSelect.getBrokerName(), BROKER_NAME2);\n\n        mqFaultStrategy.updateFaultItem(BROKER_NAME2, 1000, true, false);\n        mqFaultStrategy.updateFaultItem(BROKER_NAME, 1000, true, true);\n        AddressableMessageQueue secondSelect = selector.select(ProxyContext.create(), messageQueueView);\n        assertEquals(secondSelect.getBrokerName(), BROKER_NAME);\n    }\n    @Test\n    public void testParameterValidate() {\n        // too large message body\n        assertThrows(GrpcProxyException.class, () -> {\n            try {\n                this.sendMessageActivity.sendMessage(\n                    createContext(),\n                    SendMessageRequest.newBuilder()\n                        .addMessages(Message.newBuilder()\n                            .setTopic(Resource.newBuilder()\n                                .setName(TOPIC)\n                                .build())\n                            .setSystemProperties(SystemProperties.newBuilder()\n                                .setMessageId(\"msgId\")\n                                .setQueueId(0)\n                                .setMessageType(MessageType.NORMAL)\n                                .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                                .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                                .build())\n                            .setBody(ByteString.copyFrom(new byte[4 * 1024 * 1024 + 1]))\n                            .build())\n                        .build()\n                ).get();\n            } catch (ExecutionException t) {\n                GrpcProxyException e = (GrpcProxyException) t.getCause();\n                assertEquals(Code.MESSAGE_BODY_TOO_LARGE, e.getCode());\n                throw e;\n            }\n        });\n\n        // black tag\n        assertThrows(GrpcProxyException.class, () -> {\n            try {\n                this.sendMessageActivity.sendMessage(\n                    createContext(),\n                    SendMessageRequest.newBuilder()\n                        .addMessages(Message.newBuilder()\n                            .setTopic(Resource.newBuilder()\n                                .setName(TOPIC)\n                                .build())\n                            .setSystemProperties(SystemProperties.newBuilder()\n                                .setMessageId(\"msgId\")\n                                .setQueueId(0)\n                                .setTag(\"   \")\n                                .setMessageType(MessageType.NORMAL)\n                                .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                                .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                                .build())\n                            .setBody(ByteString.copyFrom(new byte[3]))\n                            .build())\n                        .build()\n                ).get();\n            } catch (ExecutionException t) {\n                GrpcProxyException e = (GrpcProxyException) t.getCause();\n                assertEquals(Code.ILLEGAL_MESSAGE_TAG, e.getCode());\n                throw e;\n            }\n        });\n\n        // tag with '|'\n        assertThrows(GrpcProxyException.class, () -> {\n            try {\n                this.sendMessageActivity.sendMessage(\n                    createContext(),\n                    SendMessageRequest.newBuilder()\n                        .addMessages(Message.newBuilder()\n                            .setTopic(Resource.newBuilder()\n                                .setName(TOPIC)\n                                .build())\n                            .setSystemProperties(SystemProperties.newBuilder()\n                                .setMessageId(\"msgId\")\n                                .setQueueId(0)\n                                .setTag(\"|\")\n                                .setMessageType(MessageType.NORMAL)\n                                .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                                .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                                .build())\n                            .setBody(ByteString.copyFrom(new byte[3]))\n                            .build())\n                        .build()\n                ).get();\n            } catch (ExecutionException t) {\n                GrpcProxyException e = (GrpcProxyException) t.getCause();\n                assertEquals(Code.ILLEGAL_MESSAGE_TAG, e.getCode());\n                throw e;\n            }\n        });\n\n        // tag with \\t\n        assertThrows(GrpcProxyException.class, () -> {\n            try {\n                this.sendMessageActivity.sendMessage(\n                    createContext(),\n                    SendMessageRequest.newBuilder()\n                        .addMessages(Message.newBuilder()\n                            .setTopic(Resource.newBuilder()\n                                .setName(TOPIC)\n                                .build())\n                            .setSystemProperties(SystemProperties.newBuilder()\n                                .setMessageId(\"msgId\")\n                                .setQueueId(0)\n                                .setTag(\"\\t\")\n                                .setMessageType(MessageType.NORMAL)\n                                .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                                .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                                .build())\n                            .setBody(ByteString.copyFrom(new byte[3]))\n                            .build())\n                        .build()\n                ).get();\n            } catch (ExecutionException t) {\n                GrpcProxyException e = (GrpcProxyException) t.getCause();\n                assertEquals(Code.ILLEGAL_MESSAGE_TAG, e.getCode());\n                throw e;\n            }\n        });\n\n        // blank message key\n        assertThrows(GrpcProxyException.class, () -> {\n            try {\n                this.sendMessageActivity.sendMessage(\n                    createContext(),\n                    SendMessageRequest.newBuilder()\n                        .addMessages(Message.newBuilder()\n                            .setTopic(Resource.newBuilder()\n                                .setName(TOPIC)\n                                .build())\n                            .setSystemProperties(SystemProperties.newBuilder()\n                                .setMessageId(\"msgId\")\n                                .setQueueId(0)\n                                .addKeys(\"  \")\n                                .setMessageType(MessageType.NORMAL)\n                                .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                                .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                                .build())\n                            .setBody(ByteString.copyFrom(new byte[3]))\n                            .build())\n                        .build()\n                ).get();\n            } catch (ExecutionException t) {\n                GrpcProxyException e = (GrpcProxyException) t.getCause();\n                assertEquals(Code.ILLEGAL_MESSAGE_KEY, e.getCode());\n                throw e;\n            }\n        });\n\n        // blank message with \\t\n        assertThrows(GrpcProxyException.class, () -> {\n            try {\n                this.sendMessageActivity.sendMessage(\n                    createContext(),\n                    SendMessageRequest.newBuilder()\n                        .addMessages(Message.newBuilder()\n                            .setTopic(Resource.newBuilder()\n                                .setName(TOPIC)\n                                .build())\n                            .setSystemProperties(SystemProperties.newBuilder()\n                                .setMessageId(\"msgId\")\n                                .setQueueId(0)\n                                .addKeys(\"\\t\")\n                                .setMessageType(MessageType.NORMAL)\n                                .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                                .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                                .build())\n                            .setBody(ByteString.copyFrom(new byte[3]))\n                            .build())\n                        .build()\n                ).get();\n            } catch (ExecutionException t) {\n                GrpcProxyException e = (GrpcProxyException) t.getCause();\n                assertEquals(Code.ILLEGAL_MESSAGE_KEY, e.getCode());\n                throw e;\n            }\n        });\n\n        // blank message group\n        assertThrows(GrpcProxyException.class, () -> {\n            try {\n                this.sendMessageActivity.sendMessage(\n                    createContext(),\n                    SendMessageRequest.newBuilder()\n                        .addMessages(Message.newBuilder()\n                            .setTopic(Resource.newBuilder()\n                                .setName(TOPIC)\n                                .build())\n                            .setSystemProperties(SystemProperties.newBuilder()\n                                .setMessageId(\"msgId\")\n                                .setQueueId(0)\n                                .setMessageGroup(\"  \")\n                                .setMessageType(MessageType.NORMAL)\n                                .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                                .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                                .build())\n                            .setBody(ByteString.copyFrom(new byte[3]))\n                            .build())\n                        .build()\n                ).get();\n            } catch (ExecutionException t) {\n                GrpcProxyException e = (GrpcProxyException) t.getCause();\n                assertEquals(Code.ILLEGAL_MESSAGE_GROUP, e.getCode());\n                throw e;\n            }\n        });\n\n        // long message group\n        assertThrows(GrpcProxyException.class, () -> {\n            try {\n                this.sendMessageActivity.sendMessage(\n                    createContext(),\n                    SendMessageRequest.newBuilder()\n                        .addMessages(Message.newBuilder()\n                            .setTopic(Resource.newBuilder()\n                                .setName(TOPIC)\n                                .build())\n                            .setSystemProperties(SystemProperties.newBuilder()\n                                .setMessageId(\"msgId\")\n                                .setQueueId(0)\n                                .setMessageGroup(createStr(ConfigurationManager.getProxyConfig().getMaxMessageGroupSize() + 1))\n                                .setMessageType(MessageType.NORMAL)\n                                .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                                .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                                .build())\n                            .setBody(ByteString.copyFrom(new byte[3]))\n                            .build())\n                        .build()\n                ).get();\n            } catch (ExecutionException t) {\n                GrpcProxyException e = (GrpcProxyException) t.getCause();\n                assertEquals(Code.ILLEGAL_MESSAGE_GROUP, e.getCode());\n                throw e;\n            }\n        });\n\n        // message group with \\t\n        assertThrows(GrpcProxyException.class, () -> {\n            try {\n                this.sendMessageActivity.sendMessage(\n                    createContext(),\n                    SendMessageRequest.newBuilder()\n                        .addMessages(Message.newBuilder()\n                            .setTopic(Resource.newBuilder()\n                                .setName(TOPIC)\n                                .build())\n                            .setSystemProperties(SystemProperties.newBuilder()\n                                .setMessageId(\"msgId\")\n                                .setQueueId(0)\n                                .setMessageGroup(\"\\t\")\n                                .setMessageType(MessageType.NORMAL)\n                                .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                                .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                                .build())\n                            .setBody(ByteString.copyFrom(new byte[3]))\n                            .build())\n                        .build()\n                ).get();\n            } catch (ExecutionException t) {\n                GrpcProxyException e = (GrpcProxyException) t.getCause();\n                assertEquals(Code.ILLEGAL_MESSAGE_GROUP, e.getCode());\n                throw e;\n            }\n        });\n\n        // too large message property\n        assertThrows(GrpcProxyException.class, () -> {\n            try {\n                this.sendMessageActivity.sendMessage(\n                    createContext(),\n                    SendMessageRequest.newBuilder()\n                        .addMessages(Message.newBuilder()\n                            .setTopic(Resource.newBuilder()\n                                .setName(TOPIC)\n                                .build())\n                            .setSystemProperties(SystemProperties.newBuilder()\n                                .setMessageId(\"msgId\")\n                                .setQueueId(0)\n                                .setMessageType(MessageType.NORMAL)\n                                .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                                .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                                .build())\n                            .putUserProperties(\"key\", createStr(16 * 1024 + 1))\n                            .setBody(ByteString.copyFrom(new byte[3]))\n                            .build())\n                        .build()\n                ).get();\n            } catch (ExecutionException t) {\n                GrpcProxyException e = (GrpcProxyException) t.getCause();\n                assertEquals(Code.MESSAGE_PROPERTIES_TOO_LARGE, e.getCode());\n                throw e;\n            }\n        });\n\n        // too large message property\n        assertThrows(GrpcProxyException.class, () -> {\n            Map<String, String> p = new HashMap<>();\n            for (int i = 0; i <= ConfigurationManager.getProxyConfig().getUserPropertyMaxNum(); i++) {\n                p.put(String.valueOf(i), String.valueOf(i));\n            }\n            try {\n                this.sendMessageActivity.sendMessage(\n                    createContext(),\n                    SendMessageRequest.newBuilder()\n                        .addMessages(Message.newBuilder()\n                            .setTopic(Resource.newBuilder()\n                                .setName(TOPIC)\n                                .build())\n                            .setSystemProperties(SystemProperties.newBuilder()\n                                .setMessageId(\"msgId\")\n                                .setQueueId(0)\n                                .setMessageType(MessageType.NORMAL)\n                                .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                                .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                                .build())\n                            .putAllUserProperties(p)\n                            .setBody(ByteString.copyFrom(new byte[3]))\n                            .build())\n                        .build()\n                ).get();\n            } catch (ExecutionException t) {\n                GrpcProxyException e = (GrpcProxyException) t.getCause();\n                assertEquals(Code.MESSAGE_PROPERTIES_TOO_LARGE, e.getCode());\n                throw e;\n            }\n        });\n\n        // set system properties\n        assertThrows(GrpcProxyException.class, () -> {\n            try {\n                this.sendMessageActivity.sendMessage(\n                    createContext(),\n                    SendMessageRequest.newBuilder()\n                        .addMessages(Message.newBuilder()\n                            .setTopic(Resource.newBuilder()\n                                .setName(TOPIC)\n                                .build())\n                            .setSystemProperties(SystemProperties.newBuilder()\n                                .setMessageId(\"msgId\")\n                                .setQueueId(0)\n                                .setMessageType(MessageType.NORMAL)\n                                .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                                .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                                .build())\n                            .putUserProperties(MessageConst.PROPERTY_TRACE_SWITCH, \"false\")\n                            .setBody(ByteString.copyFrom(new byte[3]))\n                            .build())\n                        .build()\n                ).get();\n            } catch (ExecutionException t) {\n                GrpcProxyException e = (GrpcProxyException) t.getCause();\n                assertEquals(Code.ILLEGAL_MESSAGE_PROPERTY_KEY, e.getCode());\n                throw e;\n            }\n        });\n\n        // set the key of user property with control character\n        assertThrows(GrpcProxyException.class, () -> {\n            try {\n                this.sendMessageActivity.sendMessage(\n                    createContext(),\n                    SendMessageRequest.newBuilder()\n                        .addMessages(Message.newBuilder()\n                            .setTopic(Resource.newBuilder()\n                                .setName(TOPIC)\n                                .build())\n                            .setSystemProperties(SystemProperties.newBuilder()\n                                .setMessageId(\"msgId\")\n                                .setQueueId(0)\n                                .setMessageType(MessageType.NORMAL)\n                                .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                                .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                                .build())\n                            .putUserProperties(\"\\u0000\", \"hello\")\n                            .setBody(ByteString.copyFrom(new byte[3]))\n                            .build())\n                        .build()\n                ).get();\n            } catch (ExecutionException t) {\n                GrpcProxyException e = (GrpcProxyException) t.getCause();\n                assertEquals(Code.ILLEGAL_MESSAGE_PROPERTY_KEY, e.getCode());\n                throw e;\n            }\n        });\n\n        // set the value of user property with control character\n        assertThrows(GrpcProxyException.class, () -> {\n            try {\n                this.sendMessageActivity.sendMessage(\n                    createContext(),\n                    SendMessageRequest.newBuilder()\n                        .addMessages(Message.newBuilder()\n                            .setTopic(Resource.newBuilder()\n                                .setName(TOPIC)\n                                .build())\n                            .setSystemProperties(SystemProperties.newBuilder()\n                                .setMessageId(\"msgId\")\n                                .setQueueId(0)\n                                .setMessageType(MessageType.NORMAL)\n                                .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                                .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                                .build())\n                            .putUserProperties(\"p\", \"\\u0000\")\n                            .setBody(ByteString.copyFrom(new byte[3]))\n                            .build())\n                        .build()\n                ).get();\n            } catch (ExecutionException t) {\n                GrpcProxyException e = (GrpcProxyException) t.getCause();\n                assertEquals(Code.ILLEGAL_MESSAGE_PROPERTY_KEY, e.getCode());\n                throw e;\n            }\n        });\n\n        // empty message id\n        assertThrows(GrpcProxyException.class, () -> {\n            try {\n                this.sendMessageActivity.sendMessage(\n                    createContext(),\n                    SendMessageRequest.newBuilder()\n                        .addMessages(Message.newBuilder()\n                            .setTopic(Resource.newBuilder()\n                                .setName(TOPIC)\n                                .build())\n                            .setSystemProperties(SystemProperties.newBuilder()\n                                .setMessageId(\" \")\n                                .setQueueId(0)\n                                .setMessageType(MessageType.NORMAL)\n                                .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                                .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                                .build())\n                            .setBody(ByteString.copyFrom(new byte[3]))\n                            .build())\n                        .build()\n                ).get();\n            } catch (ExecutionException t) {\n                GrpcProxyException e = (GrpcProxyException) t.getCause();\n                assertEquals(Code.ILLEGAL_MESSAGE_ID, e.getCode());\n                throw e;\n            }\n        });\n\n        // delay time\n        assertThrows(GrpcProxyException.class, () -> {\n            try {\n                this.sendMessageActivity.sendMessage(\n                    createContext(),\n                    SendMessageRequest.newBuilder()\n                        .addMessages(Message.newBuilder()\n                            .setTopic(Resource.newBuilder()\n                                .setName(TOPIC)\n                                .build())\n                            .setSystemProperties(SystemProperties.newBuilder()\n                                .setMessageId(\"id\")\n                                .setDeliveryTimestamp(\n                                    Timestamps.fromMillis(System.currentTimeMillis() + Duration.ofDays(1).toMillis() + Duration.ofSeconds(10).toMillis()))\n                                .setQueueId(0)\n                                .setMessageType(MessageType.NORMAL)\n                                .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                                .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                                .build())\n                            .setBody(ByteString.copyFrom(new byte[3]))\n                            .build())\n                        .build()\n                ).get();\n            } catch (ExecutionException t) {\n                GrpcProxyException e = (GrpcProxyException) t.getCause();\n                assertEquals(Code.ILLEGAL_DELIVERY_TIME, e.getCode());\n                throw e;\n            }\n        });\n\n        // transactionRecoverySecond\n        assertThrows(GrpcProxyException.class, () -> {\n            try {\n                this.sendMessageActivity.sendMessage(\n                    createContext(),\n                    SendMessageRequest.newBuilder()\n                        .addMessages(Message.newBuilder()\n                            .setTopic(Resource.newBuilder()\n                                .setName(TOPIC)\n                                .build())\n                            .setSystemProperties(SystemProperties.newBuilder()\n                                .setMessageId(\"id\")\n                                .setQueueId(0)\n                                .setMessageType(MessageType.NORMAL)\n                                .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                                .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                                .setOrphanedTransactionRecoveryDuration(Durations.fromHours(2))\n                                .setMessageType(MessageType.TRANSACTION)\n                                .build())\n                            .setBody(ByteString.copyFrom(new byte[3]))\n                            .build())\n                        .build()\n                ).get();\n            } catch (ExecutionException t) {\n                GrpcProxyException e = (GrpcProxyException) t.getCause();\n                assertEquals(Code.BAD_REQUEST, e.getCode());\n                throw e;\n            }\n        });\n    }\n\n    private static String createStr(int len) {\n        StringBuilder sb = new StringBuilder();\n        for (int i = 0; i < len; i++) {\n            sb.append(\"a\");\n        }\n        return sb.toString();\n    }\n\n    private static QueueData createQueueData(String brokerName, int writeQueueNums) {\n        QueueData queueData = new QueueData();\n        queueData.setBrokerName(brokerName);\n        queueData.setWriteQueueNums(writeQueueNums);\n        queueData.setPerm(PermName.PERM_WRITE);\n        return queueData;\n    }\n\n    private static BrokerData createBrokerData(String clusterName, String brokerName, String brokerAddrs) {\n        BrokerData brokerData = new BrokerData();\n        brokerData.setCluster(clusterName);\n        brokerData.setBrokerName(brokerName);\n        HashMap<Long, String> brokerAddrsMap = new HashMap<>();\n        brokerAddrsMap.put(MixAll.MASTER_ID, brokerAddrs);\n        brokerData.setBrokerAddrs(brokerAddrsMap);\n\n        return brokerData;\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivityTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2.route;\n\nimport apache.rocketmq.v2.Address;\nimport apache.rocketmq.v2.AddressScheme;\nimport apache.rocketmq.v2.Broker;\nimport apache.rocketmq.v2.Code;\nimport apache.rocketmq.v2.Endpoints;\nimport apache.rocketmq.v2.MessageQueue;\nimport apache.rocketmq.v2.MessageType;\nimport apache.rocketmq.v2.Permission;\nimport apache.rocketmq.v2.QueryAssignmentRequest;\nimport apache.rocketmq.v2.QueryAssignmentResponse;\nimport apache.rocketmq.v2.QueryRouteRequest;\nimport apache.rocketmq.v2.QueryRouteResponse;\nimport apache.rocketmq.v2.Resource;\nimport com.google.common.net.HostAndPort;\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest;\nimport org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder;\nimport org.apache.rocketmq.proxy.service.metadata.LocalMetadataService;\nimport org.apache.rocketmq.proxy.service.metadata.MetadataService;\nimport org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.ArgumentCaptor;\nimport org.mockito.Mockito;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.fail;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.when;\n\npublic class RouteActivityTest extends BaseActivityTest {\n\n    private RouteActivity routeActivity;\n\n    private static final String CLUSTER = \"cluster\";\n    private static final String TOPIC = \"topic\";\n    private static final String GROUP = \"group\";\n    private static final String BROKER_NAME = \"brokerName\";\n    private static final Broker GRPC_BROKER = Broker.newBuilder().setName(BROKER_NAME).build();\n    private static final Resource GRPC_TOPIC = Resource.newBuilder()\n        .setName(TOPIC)\n        .build();\n    private static final Resource GRPC_GROUP = Resource.newBuilder()\n        .setName(GROUP)\n        .build();\n    private static Endpoints grpcEndpoints = Endpoints.newBuilder()\n        .setScheme(AddressScheme.IPv4)\n        .addAddresses(Address.newBuilder().setHost(\"127.0.0.1\").setPort(8080).build())\n        .addAddresses(Address.newBuilder().setHost(\"127.0.0.2\").setPort(8080).build())\n        .build();\n    private static List<org.apache.rocketmq.proxy.common.Address> addressArrayList = new ArrayList<>();\n\n    static {\n        addressArrayList.add(new org.apache.rocketmq.proxy.common.Address(\n            org.apache.rocketmq.proxy.common.Address.AddressScheme.IPv4,\n            HostAndPort.fromParts(\"127.0.0.1\", 8080)));\n        addressArrayList.add(new org.apache.rocketmq.proxy.common.Address(\n            org.apache.rocketmq.proxy.common.Address.AddressScheme.IPv4,\n            HostAndPort.fromParts(\"127.0.0.2\", 8080)));\n    }\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        this.routeActivity = new RouteActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n    }\n\n    @Test\n    public void testQueryRoute() throws Throwable {\n        ConfigurationManager.getProxyConfig().setGrpcServerPort(8080);\n        ArgumentCaptor<List<org.apache.rocketmq.proxy.common.Address>> addressListCaptor = ArgumentCaptor.forClass(List.class);\n        when(this.messagingProcessor.getTopicRouteDataForProxy(any(), addressListCaptor.capture(), anyString()))\n            .thenReturn(createProxyTopicRouteData(2, 2, 6));\n        MetadataService metadataService = Mockito.mock(LocalMetadataService.class);\n        when(this.messagingProcessor.getMetadataService()).thenReturn(metadataService);\n        when(metadataService.getTopicMessageType(any(), anyString())).thenReturn(TopicMessageType.NORMAL);\n\n        QueryRouteResponse response = this.routeActivity.queryRoute(\n            createContext(),\n            QueryRouteRequest.newBuilder()\n                .setEndpoints(grpcEndpoints)\n                .setTopic(Resource.newBuilder().setName(TOPIC).build())\n                .build()\n        ).get();\n\n        assertEquals(Code.OK, response.getStatus().getCode());\n        assertEquals(4, response.getMessageQueuesCount());\n        for (MessageQueue messageQueue : response.getMessageQueuesList()) {\n            assertEquals(grpcEndpoints, messageQueue.getBroker().getEndpoints());\n            assertEquals(Permission.READ_WRITE, messageQueue.getPermission());\n        }\n    }\n\n    @Test\n    public void testQueryRouteTopicExist() throws Throwable {\n        when(this.messagingProcessor.getTopicRouteDataForProxy(any(), any(), anyString()))\n            .thenThrow(new MQBrokerException(ResponseCode.TOPIC_NOT_EXIST, \"\"));\n\n        try {\n            this.routeActivity.queryRoute(\n                createContext(),\n                QueryRouteRequest.newBuilder()\n                    .setEndpoints(grpcEndpoints)\n                    .setTopic(GRPC_TOPIC)\n                    .build()\n            ).get();\n        } catch (Throwable t) {\n            assertEquals(Code.TOPIC_NOT_FOUND, ResponseBuilder.getInstance().buildStatus(t).getCode());\n            return;\n        }\n        fail();\n    }\n\n    @Test\n    public void testQueryAssignmentWithNoReadPerm() throws Throwable {\n        when(this.messagingProcessor.getTopicRouteDataForProxy(any(), any(), anyString()))\n            .thenReturn(createProxyTopicRouteData(2, 2, PermName.PERM_WRITE));\n\n        QueryAssignmentResponse response = this.routeActivity.queryAssignment(\n            createContext(),\n            QueryAssignmentRequest.newBuilder()\n                .setEndpoints(grpcEndpoints)\n                .setTopic(GRPC_TOPIC)\n                .setGroup(GRPC_GROUP)\n                .build()\n        ).get();\n\n        assertEquals(Code.FORBIDDEN, response.getStatus().getCode());\n    }\n\n    @Test\n    public void testQueryAssignmentWithNoReadQueue() throws Throwable {\n        when(this.messagingProcessor.getTopicRouteDataForProxy(any(), any(), anyString()))\n            .thenReturn(createProxyTopicRouteData(0, 2, 6));\n\n        QueryAssignmentResponse response = this.routeActivity.queryAssignment(\n            createContext(),\n            QueryAssignmentRequest.newBuilder()\n                .setEndpoints(grpcEndpoints)\n                .setTopic(GRPC_TOPIC)\n                .setGroup(GRPC_GROUP)\n                .build()\n        ).get();\n\n        assertEquals(Code.FORBIDDEN, response.getStatus().getCode());\n    }\n\n    @Test\n    public void testQueryAssignment() throws Throwable {\n        when(this.messagingProcessor.getTopicRouteDataForProxy(any(), any(), anyString()))\n            .thenReturn(createProxyTopicRouteData(2, 2, 6));\n\n        QueryAssignmentResponse response = this.routeActivity.queryAssignment(\n            createContext(),\n            QueryAssignmentRequest.newBuilder()\n                .setEndpoints(grpcEndpoints)\n                .setTopic(GRPC_TOPIC)\n                .setGroup(GRPC_GROUP)\n                .build()\n        ).get();\n\n        assertEquals(Code.OK, response.getStatus().getCode());\n        assertEquals(1, response.getAssignmentsCount());\n        assertEquals(grpcEndpoints, response.getAssignments(0).getMessageQueue().getBroker().getEndpoints());\n    }\n\n    @Test\n    public void testQueryFifoAssignment() throws Throwable {\n        when(this.messagingProcessor.getTopicRouteDataForProxy(any(), any(), anyString()))\n            .thenReturn(createProxyTopicRouteData(2, 2, 6));\n        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n        subscriptionGroupConfig.setConsumeMessageOrderly(true);\n        when(this.messagingProcessor.getSubscriptionGroupConfig(any(), anyString())).thenReturn(subscriptionGroupConfig);\n\n        QueryAssignmentResponse response = this.routeActivity.queryAssignment(\n            createContext(),\n            QueryAssignmentRequest.newBuilder()\n                .setEndpoints(grpcEndpoints)\n                .setTopic(GRPC_TOPIC)\n                .setGroup(GRPC_GROUP)\n                .build()\n        ).get();\n\n        assertEquals(Code.OK, response.getStatus().getCode());\n        assertEquals(2, response.getAssignmentsCount());\n        assertEquals(grpcEndpoints, response.getAssignments(0).getMessageQueue().getBroker().getEndpoints());\n    }\n\n    private static ProxyTopicRouteData createProxyTopicRouteData(int r, int w, int p) {\n        ProxyTopicRouteData proxyTopicRouteData = new ProxyTopicRouteData();\n        proxyTopicRouteData.getQueueDatas().add(createQueueData(r, w, p));\n        ProxyTopicRouteData.ProxyBrokerData proxyBrokerData = new ProxyTopicRouteData.ProxyBrokerData();\n        proxyBrokerData.setCluster(CLUSTER);\n        proxyBrokerData.setBrokerName(BROKER_NAME);\n        proxyBrokerData.getBrokerAddrs().put(0L, addressArrayList);\n        proxyBrokerData.getBrokerAddrs().put(1L, addressArrayList);\n        proxyTopicRouteData.getBrokerDatas().add(proxyBrokerData);\n        return proxyTopicRouteData;\n    }\n\n    @Test\n    public void testGenPartitionFromQueueData() throws Exception {\n        // test queueData with 8 read queues, 8 write queues, and rw permission, expect 8 rw queues.\n        QueueData queueDataWith8R8WPermRW = createQueueData(8, 8, PermName.PERM_READ | PermName.PERM_WRITE);\n        List<MessageQueue> partitionWith8R8WPermRW = this.routeActivity.genMessageQueueFromQueueData(queueDataWith8R8WPermRW, GRPC_TOPIC, TopicMessageType.NORMAL, GRPC_BROKER);\n        assertEquals(8, partitionWith8R8WPermRW.size());\n        assertEquals(8, partitionWith8R8WPermRW.stream().filter(a -> a.getAcceptMessageTypesValue(0) == MessageType.NORMAL.getNumber()).count());\n        assertEquals(8, partitionWith8R8WPermRW.stream().filter(a -> a.getPermission() == Permission.READ_WRITE).count());\n        assertEquals(0, partitionWith8R8WPermRW.stream().filter(a -> a.getPermission() == Permission.READ).count());\n        assertEquals(0, partitionWith8R8WPermRW.stream().filter(a -> a.getPermission() == Permission.WRITE).count());\n\n        // test queueData with 8 read queues, 8 write queues, and read only permission, expect 8 read only queues.\n        QueueData queueDataWith8R8WPermR = createQueueData(8, 8, PermName.PERM_READ);\n        List<MessageQueue> partitionWith8R8WPermR = this.routeActivity.genMessageQueueFromQueueData(queueDataWith8R8WPermR, GRPC_TOPIC, TopicMessageType.FIFO, GRPC_BROKER);\n        assertEquals(8, partitionWith8R8WPermR.size());\n        assertEquals(8, partitionWith8R8WPermR.stream().filter(a -> a.getAcceptMessageTypesValue(0) == MessageType.FIFO.getNumber()).count());\n        assertEquals(8, partitionWith8R8WPermR.stream().filter(a -> a.getPermission() == Permission.READ).count());\n        assertEquals(0, partitionWith8R8WPermR.stream().filter(a -> a.getPermission() == Permission.READ_WRITE).count());\n        assertEquals(0, partitionWith8R8WPermR.stream().filter(a -> a.getPermission() == Permission.WRITE).count());\n\n        // test queueData with 8 read queues, 8 write queues, and write only permission, expect 8 write only queues.\n        QueueData queueDataWith8R8WPermW = createQueueData(8, 8, PermName.PERM_WRITE);\n        List<MessageQueue> partitionWith8R8WPermW = this.routeActivity.genMessageQueueFromQueueData(queueDataWith8R8WPermW, GRPC_TOPIC, TopicMessageType.TRANSACTION, GRPC_BROKER);\n        assertEquals(8, partitionWith8R8WPermW.size());\n        assertEquals(8, partitionWith8R8WPermW.stream().filter(a -> a.getAcceptMessageTypesValue(0) == MessageType.TRANSACTION.getNumber()).count());\n        assertEquals(8, partitionWith8R8WPermW.stream().filter(a -> a.getPermission() == Permission.WRITE).count());\n        assertEquals(0, partitionWith8R8WPermW.stream().filter(a -> a.getPermission() == Permission.READ_WRITE).count());\n        assertEquals(0, partitionWith8R8WPermW.stream().filter(a -> a.getPermission() == Permission.READ).count());\n\n        // test queueData with 8 read queues, 0 write queues, and rw permission, expect 8 read only queues.\n        QueueData queueDataWith8R0WPermRW = createQueueData(8, 0, PermName.PERM_READ | PermName.PERM_WRITE);\n        List<MessageQueue> partitionWith8R0WPermRW = this.routeActivity.genMessageQueueFromQueueData(queueDataWith8R0WPermRW, GRPC_TOPIC, TopicMessageType.DELAY, GRPC_BROKER);\n        assertEquals(8, partitionWith8R0WPermRW.size());\n        assertEquals(8, partitionWith8R0WPermRW.stream().filter(a -> a.getAcceptMessageTypesValue(0) == MessageType.DELAY.getNumber()).count());\n        assertEquals(8, partitionWith8R0WPermRW.stream().filter(a -> a.getPermission() == Permission.READ).count());\n        assertEquals(0, partitionWith8R0WPermRW.stream().filter(a -> a.getPermission() == Permission.READ_WRITE).count());\n        assertEquals(0, partitionWith8R0WPermRW.stream().filter(a -> a.getPermission() == Permission.WRITE).count());\n\n        // test queueData with 4 read queues, 8 write queues, and rw permission, expect 4 rw queues and  4 write only queues.\n        QueueData queueDataWith4R8WPermRW = createQueueData(4, 8, PermName.PERM_READ | PermName.PERM_WRITE);\n        List<MessageQueue> partitionWith4R8WPermRW = this.routeActivity.genMessageQueueFromQueueData(queueDataWith4R8WPermRW, GRPC_TOPIC, TopicMessageType.UNSPECIFIED, GRPC_BROKER);\n        assertEquals(8, partitionWith4R8WPermRW.size());\n        assertEquals(8, partitionWith4R8WPermRW.stream().filter(a -> a.getAcceptMessageTypesValue(0) == MessageType.MESSAGE_TYPE_UNSPECIFIED.getNumber()).count());\n        assertEquals(4, partitionWith4R8WPermRW.stream().filter(a -> a.getPermission() == Permission.WRITE).count());\n        assertEquals(4, partitionWith4R8WPermRW.stream().filter(a -> a.getPermission() == Permission.READ_WRITE).count());\n        assertEquals(0, partitionWith4R8WPermRW.stream().filter(a -> a.getPermission() == Permission.READ).count());\n\n        // test queueData with 2 read queues, 2 write queues, and no permission, expect 2 no permission queues.\n        QueueData queueDataWith2R2WNoPerm = createQueueData(2, 2, 0);\n        List<MessageQueue> partitionWith2R2WNoPerm = this.routeActivity.genMessageQueueFromQueueData(queueDataWith2R2WNoPerm, GRPC_TOPIC, TopicMessageType.UNSPECIFIED, GRPC_BROKER);\n        assertEquals(2, partitionWith2R2WNoPerm.size());\n        assertEquals(2, partitionWith2R2WNoPerm.stream().filter(a -> a.getAcceptMessageTypesValue(0) == MessageType.MESSAGE_TYPE_UNSPECIFIED.getNumber()).count());\n        assertEquals(2, partitionWith2R2WNoPerm.stream().filter(a -> a.getPermission() == Permission.NONE).count());\n        assertEquals(0, partitionWith2R2WNoPerm.stream().filter(a -> a.getPermission() == Permission.WRITE).count());\n        assertEquals(0, partitionWith2R2WNoPerm.stream().filter(a -> a.getPermission() == Permission.READ_WRITE).count());\n        assertEquals(0, partitionWith2R2WNoPerm.stream().filter(a -> a.getPermission() == Permission.READ).count());\n\n        // test queueData with 0 read queues, 0 write queues, and no permission, expect 1 no permission queue.\n        QueueData queueDataWith0R0WNoPerm = createQueueData(0, 0, 0);\n        List<MessageQueue> partitionWith0R0WNoPerm = this.routeActivity.genMessageQueueFromQueueData(queueDataWith0R0WNoPerm, GRPC_TOPIC, TopicMessageType.UNSPECIFIED, GRPC_BROKER);\n        assertEquals(1, partitionWith0R0WNoPerm.size());\n        assertEquals(1, partitionWith0R0WNoPerm.stream().filter(a -> a.getAcceptMessageTypesValue(0) == MessageType.MESSAGE_TYPE_UNSPECIFIED.getNumber()).count());\n        assertEquals(1, partitionWith0R0WNoPerm.stream().filter(a -> a.getPermission() == Permission.NONE).count());\n        assertEquals(0, partitionWith0R0WNoPerm.stream().filter(a -> a.getPermission() == Permission.WRITE).count());\n        assertEquals(0, partitionWith0R0WNoPerm.stream().filter(a -> a.getPermission() == Permission.READ_WRITE).count());\n        assertEquals(0, partitionWith0R0WNoPerm.stream().filter(a -> a.getPermission() == Permission.READ).count());\n    }\n\n    private static QueueData createQueueData(int r, int w, int perm) {\n        QueueData queueData = new QueueData();\n        queueData.setBrokerName(BROKER_NAME);\n        queueData.setReadQueueNums(r);\n        queueData.setWriteQueueNums(w);\n        queueData.setPerm(perm);\n        return queueData;\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/transaction/EndTransactionActivityTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.grpc.v2.transaction;\n\nimport apache.rocketmq.v2.Code;\nimport apache.rocketmq.v2.EndTransactionRequest;\nimport apache.rocketmq.v2.EndTransactionResponse;\nimport apache.rocketmq.v2.Resource;\nimport apache.rocketmq.v2.TransactionResolution;\nimport apache.rocketmq.v2.TransactionSource;\nimport java.util.Arrays;\nimport java.util.Collection;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest;\nimport org.apache.rocketmq.proxy.processor.TransactionStatus;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.junit.runners.Parameterized;\nimport org.mockito.ArgumentCaptor;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.when;\n\n@RunWith(Parameterized.class)\npublic class EndTransactionActivityTest extends BaseActivityTest {\n\n    private EndTransactionActivity endTransactionActivity;\n    private TransactionResolution resolution;\n    private TransactionSource source;\n    private TransactionStatus transactionStatus;\n    private Boolean fromTransactionCheck;\n\n    public EndTransactionActivityTest(TransactionResolution resolution, TransactionSource source,\n        TransactionStatus transactionStatus, Boolean fromTransactionCheck) {\n        this.resolution = resolution;\n        this.source = source;\n        this.transactionStatus = transactionStatus;\n        this.fromTransactionCheck = fromTransactionCheck;\n    }\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        this.endTransactionActivity = new EndTransactionActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);\n    }\n\n    @Test\n    public void testEndTransaction() throws Throwable {\n        ArgumentCaptor<TransactionStatus> transactionStatusCaptor = ArgumentCaptor.forClass(TransactionStatus.class);\n        ArgumentCaptor<Boolean> fromTransactionCheckCaptor = ArgumentCaptor.forClass(Boolean.class);\n        when(this.messagingProcessor.endTransaction(any(), any(), anyString(), anyString(), anyString(),\n            transactionStatusCaptor.capture(),\n            fromTransactionCheckCaptor.capture())).thenReturn(CompletableFuture.completedFuture(null));\n\n        EndTransactionResponse response = this.endTransactionActivity.endTransaction(\n            createContext(),\n            EndTransactionRequest.newBuilder()\n                .setResolution(resolution)\n                .setTopic(Resource.newBuilder().setName(\"topic\").build())\n                .setMessageId(MessageClientIDSetter.createUniqID())\n                .setTransactionId(MessageClientIDSetter.createUniqID())\n                .setSource(source)\n                .build()\n        ).get();\n\n        assertEquals(Code.OK, response.getStatus().getCode());\n        assertEquals(transactionStatus, transactionStatusCaptor.getValue());\n        assertEquals(fromTransactionCheck, fromTransactionCheckCaptor.getValue());\n    }\n\n    @Parameterized.Parameters\n    public static Collection<Object[]> parameters() {\n        Object[][] p = new Object[][] {\n            {TransactionResolution.COMMIT, TransactionSource.SOURCE_CLIENT, TransactionStatus.COMMIT, false},\n            {TransactionResolution.ROLLBACK, TransactionSource.SOURCE_SERVER_CHECK, TransactionStatus.ROLLBACK, true},\n            {TransactionResolution.TRANSACTION_RESOLUTION_UNSPECIFIED, TransactionSource.SOURCE_SERVER_CHECK, TransactionStatus.UNKNOWN, true},\n        };\n        return Arrays.asList(p);\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.processor;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.Random;\nimport java.util.UUID;\nimport org.apache.rocketmq.broker.client.ConsumerManager;\nimport org.apache.rocketmq.broker.client.ProducerManager;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.InitConfigTest;\nimport org.apache.rocketmq.proxy.service.ServiceManager;\nimport org.apache.rocketmq.proxy.service.message.MessageService;\nimport org.apache.rocketmq.proxy.service.metadata.MetadataService;\nimport org.apache.rocketmq.proxy.service.relay.ProxyRelayService;\nimport org.apache.rocketmq.proxy.service.route.TopicRouteService;\nimport org.apache.rocketmq.proxy.service.transaction.TransactionService;\nimport org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;\nimport org.junit.Ignore;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.mockito.Mockito.when;\n\n@Ignore\n@RunWith(MockitoJUnitRunner.Silent.class)\npublic class BaseProcessorTest extends InitConfigTest {\n    protected static final Random RANDOM = new Random();\n\n    @Mock\n    protected MessagingProcessor messagingProcessor;\n    @Mock\n    protected ServiceManager serviceManager;\n    @Mock\n    protected MessageService messageService;\n    @Mock\n    protected TopicRouteService topicRouteService;\n    @Mock\n    protected ProducerManager producerManager;\n    @Mock\n    protected ConsumerManager consumerManager;\n    @Mock\n    protected TransactionService transactionService;\n    @Mock\n    protected ProxyRelayService proxyRelayService;\n    @Mock\n    protected MetadataService metadataService;\n\n    public void before() throws Throwable {\n        super.before();\n        when(serviceManager.getMessageService()).thenReturn(messageService);\n        when(serviceManager.getTopicRouteService()).thenReturn(topicRouteService);\n        when(serviceManager.getProducerManager()).thenReturn(producerManager);\n        when(serviceManager.getConsumerManager()).thenReturn(consumerManager);\n        when(serviceManager.getTransactionService()).thenReturn(transactionService);\n        when(serviceManager.getProxyRelayService()).thenReturn(proxyRelayService);\n        when(serviceManager.getMetadataService()).thenReturn(metadataService);\n        when(messagingProcessor.getMetadataService()).thenReturn(metadataService);\n    }\n\n    protected static ProxyContext createContext() {\n        return ProxyContext.create();\n    }\n\n    protected static MessageExt createMessageExt(String topic, String tags, int reconsumeTimes, long invisibleTime) {\n        return createMessageExt(topic, tags, reconsumeTimes, invisibleTime, System.currentTimeMillis(),\n            RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE),\n            RANDOM.nextInt(Integer.MAX_VALUE), \"mockBroker\");\n    }\n\n    protected static MessageExt createMessageExt(String topic, String tags, int reconsumeTimes, long invisibleTime, long popTime,\n        long startOffset, int reviveQid, int queueId, long queueOffset, String brokerName) {\n        MessageExt messageExt = new MessageExt();\n        messageExt.setTopic(topic);\n        messageExt.setTags(tags);\n        messageExt.setReconsumeTimes(reconsumeTimes);\n        messageExt.setBody(UUID.randomUUID().toString().getBytes(StandardCharsets.UTF_8));\n        messageExt.setMsgId(MessageClientIDSetter.createUniqID());\n        messageExt.setCommitLogOffset(RANDOM.nextInt(Integer.MAX_VALUE));\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_POP_CK,\n            ExtraInfoUtil.buildExtraInfo(startOffset, popTime, invisibleTime, reviveQid, topic, brokerName, queueId, queueOffset));\n        return messageExt;\n    }\n\n    protected static ReceiptHandle create(MessageExt messageExt) {\n        String ckInfo = messageExt.getProperty(MessageConst.PROPERTY_POP_CK);\n        if (ckInfo == null) {\n            return null;\n        }\n        return ReceiptHandle.decode(ckInfo + MessageConst.KEY_SEPARATOR + messageExt.getCommitLogOffset());\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/processor/ClientProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.processor;\n\nimport java.util.HashSet;\nimport java.util.Set;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException;\nimport org.apache.rocketmq.proxy.service.ServiceManager;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.junit.jupiter.api.Assertions.assertDoesNotThrow;\nimport static org.junit.jupiter.api.Assertions.assertEquals;\nimport static org.junit.jupiter.api.Assertions.assertThrows;\nimport static org.junit.jupiter.api.Assertions.assertTrue;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ClientProcessorTest {\n\n    @Mock\n    private MessagingProcessor messagingProcessor;\n\n    @Mock\n    private ServiceManager serviceManager;\n\n    @Mock\n    private ProxyContext ctx;\n\n    @Mock\n    private SubscriptionGroupConfig groupConfig;\n\n    private ClientProcessor clientProcessor;\n\n    @Before\n    public void setUp() throws Exception {\n        ConfigurationManager.initConfig();\n        clientProcessor = new ClientProcessor(messagingProcessor, serviceManager);\n    }\n\n    @Test\n    public void testValidateLiteMode_regularGroupWithLiteMode_throwsException() {\n        String group = \"regularGroup\";\n        when(groupConfig.getLiteBindTopic()).thenReturn(\"\");\n        when(messagingProcessor.getSubscriptionGroupConfig(ctx, group)).thenReturn(groupConfig);\n\n        GrpcProxyException exception = assertThrows(GrpcProxyException.class, () -> {\n            clientProcessor.validateLiteMode(ctx, group, MessageModel.LITE_SELECTIVE);\n        });\n\n        assertEquals(\"regular group cannot use LITE mode: \" + group, exception.getMessage());\n    }\n\n    @Test\n    public void testValidateLiteMode_liteGroupWithoutLiteMode_throwsException() {\n        String group = \"liteGroup\";\n        when(groupConfig.getLiteBindTopic()).thenReturn(\"topic1\");\n        when(messagingProcessor.getSubscriptionGroupConfig(ctx, group)).thenReturn(groupConfig);\n\n        GrpcProxyException exception = assertThrows(GrpcProxyException.class, () -> {\n            clientProcessor.validateLiteMode(ctx, group, MessageModel.CLUSTERING);\n        });\n\n        assertEquals(\"lite group must use LITE mode: \" + group, exception.getMessage());\n    }\n\n    @Test\n    public void testValidateLiteMode_regularGroupWithoutLiteMode_noException() {\n        String group = \"regularGroup\";\n        when(groupConfig.getLiteBindTopic()).thenReturn(\"\");\n        when(messagingProcessor.getSubscriptionGroupConfig(ctx, group)).thenReturn(groupConfig);\n\n        assertDoesNotThrow(() -> {\n            clientProcessor.validateLiteMode(ctx, group, MessageModel.CLUSTERING);\n        });\n    }\n\n    @Test\n    public void testValidateLiteMode_liteGroupWithLiteMode_noException() {\n        String group = \"liteGroup\";\n        when(groupConfig.getLiteBindTopic()).thenReturn(\"topic1\");\n        when(messagingProcessor.getSubscriptionGroupConfig(ctx, group)).thenReturn(groupConfig);\n\n        assertDoesNotThrow(() -> {\n            clientProcessor.validateLiteMode(ctx, group, MessageModel.LITE_SELECTIVE);\n        });\n    }\n\n    @Test\n    public void testValidateLiteSubTopic_emptySubList_noException() {\n        String group = \"group\";\n        Set<SubscriptionData> subList = new HashSet<>();\n\n        assertDoesNotThrow(() -> {\n            clientProcessor.validateLiteSubTopic(ctx, group, subList);\n        });\n    }\n\n    @Test\n    public void testValidateLiteSubTopic_validSubList_noException() {\n        String group = \"group\";\n        String topic = \"topic1\";\n        SubscriptionData subscriptionData = new SubscriptionData();\n        subscriptionData.setTopic(topic);\n        Set<SubscriptionData> subList = new HashSet<>();\n        subList.add(subscriptionData);\n\n        when(groupConfig.getLiteBindTopic()).thenReturn(topic);\n        when(messagingProcessor.getSubscriptionGroupConfig(ctx, group)).thenReturn(groupConfig);\n\n        assertDoesNotThrow(() -> {\n            clientProcessor.validateLiteSubTopic(ctx, group, subList);\n        });\n    }\n\n    @Test\n    public void testValidateLiteBindTopic_matchingTopics_noException() {\n        String group = \"group\";\n        String bindTopic = \"topic1\";\n\n        when(groupConfig.getLiteBindTopic()).thenReturn(bindTopic);\n        when(messagingProcessor.getSubscriptionGroupConfig(ctx, group)).thenReturn(groupConfig);\n\n        assertDoesNotThrow(() -> {\n            clientProcessor.validateLiteBindTopic(ctx, group, bindTopic);\n        });\n    }\n\n    @Test\n    public void testValidateLiteBindTopic_mismatchedTopics_throwsException() {\n        String group = \"group\";\n        String expectedTopic = \"expectedTopic\";\n        String actualTopic = \"actualTopic\";\n\n        when(groupConfig.getLiteBindTopic()).thenReturn(expectedTopic);\n        when(messagingProcessor.getSubscriptionGroupConfig(ctx, group)).thenReturn(groupConfig);\n\n        GrpcProxyException exception = assertThrows(GrpcProxyException.class, () -> {\n            clientProcessor.validateLiteBindTopic(ctx, group, actualTopic);\n        });\n\n        assertTrue(exception.getMessage().contains(\"expected to bind topic\"));\n    }\n\n    @Test\n    public void testValidateLiteSubscriptionQuota_withinQuota_noException() {\n        String group = \"group\";\n        int quota = 10;\n        int actual = 5;\n\n        when(groupConfig.getLiteSubClientQuota()).thenReturn(quota);\n        when(messagingProcessor.getSubscriptionGroupConfig(ctx, group)).thenReturn(groupConfig);\n\n        assertDoesNotThrow(() -> {\n            clientProcessor.validateLiteSubscriptionQuota(ctx, group, actual);\n        });\n    }\n\n    @Test\n    public void testValidateLiteSubscriptionQuota_exceedsQuota_throwsException() {\n        String group = \"group\";\n        int quota = 10;\n        int actual = 15 + 300 /*quota buffer*/;\n\n        when(groupConfig.getLiteSubClientQuota()).thenReturn(quota);\n        when(messagingProcessor.getSubscriptionGroupConfig(ctx, group)).thenReturn(groupConfig);\n\n        GrpcProxyException exception = assertThrows(GrpcProxyException.class, () -> {\n            clientProcessor.validateLiteSubscriptionQuota(ctx, group, actual);\n        });\n\n        assertTrue(exception.getMessage().contains(\"lite subscription quota exceeded\"));\n    }\n\n    @Test\n    public void testGetGroupOrException_groupExists_returnsConfig() {\n        String group = \"group\";\n        when(messagingProcessor.getSubscriptionGroupConfig(ctx, group)).thenReturn(groupConfig);\n\n        SubscriptionGroupConfig result = clientProcessor.getGroupOrException(ctx, group);\n        assertEquals(groupConfig, result);\n    }\n\n    @Test\n    public void testGetGroupOrException_groupNotExists_throwsException() {\n        String group = \"nonExistentGroup\";\n        when(messagingProcessor.getSubscriptionGroupConfig(ctx, group)).thenReturn(null);\n\n        GrpcProxyException exception = assertThrows(GrpcProxyException.class, () -> {\n            clientProcessor.getGroupOrException(ctx, group);\n        });\n\n        assertEquals(\"group not found: \" + group, exception.getMessage());\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.processor;\n\nimport com.google.common.collect.Sets;\nimport java.time.Duration;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.Executors;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.AckStatus;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PopStatus;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.ConsumeInitMode;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\nimport org.apache.rocketmq.common.utils.FutureUtils;\nimport org.apache.rocketmq.proxy.common.utils.ProxyUtils;\nimport org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;\nimport org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;\nimport org.apache.rocketmq.proxy.service.route.MessageQueueView;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.filter.FilterAPI;\nimport org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.ArgumentCaptor;\nimport org.mockito.stubbing.Answer;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertSame;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyList;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.doAnswer;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.never;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\npublic class ConsumerProcessorTest extends BaseProcessorTest {\n\n    private static final String CONSUMER_GROUP = \"consumerGroup\";\n    private static final String TOPIC = \"topic\";\n    private static final String CLIENT_ID = \"clientId\";\n\n    private ConsumerProcessor consumerProcessor;\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        this.consumerProcessor = new ConsumerProcessor(messagingProcessor, serviceManager, Executors.newCachedThreadPool());\n    }\n\n    @Test\n    public void testPopMessage() throws Throwable {\n        final String tag = \"tag\";\n        final long invisibleTime = Duration.ofSeconds(15).toMillis();\n        ArgumentCaptor<AddressableMessageQueue> messageQueueArgumentCaptor = ArgumentCaptor.forClass(AddressableMessageQueue.class);\n        ArgumentCaptor<PopMessageRequestHeader> requestHeaderArgumentCaptor = ArgumentCaptor.forClass(PopMessageRequestHeader.class);\n\n        List<MessageExt> messageExtList = new ArrayList<>();\n        messageExtList.add(createMessageExt(TOPIC, \"noMatch\", 0, invisibleTime));\n        messageExtList.add(createMessageExt(TOPIC, tag, 0, invisibleTime));\n        messageExtList.add(createMessageExt(TOPIC, tag, 1, invisibleTime));\n        PopResult innerPopResult = new PopResult(PopStatus.FOUND, messageExtList);\n        when(this.messageService.popMessage(any(), messageQueueArgumentCaptor.capture(), requestHeaderArgumentCaptor.capture(), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(innerPopResult));\n\n        when(this.topicRouteService.getCurrentMessageQueueView(any(), anyString()))\n            .thenReturn(mock(MessageQueueView.class));\n\n        ArgumentCaptor<String> ackMessageIdArgumentCaptor = ArgumentCaptor.forClass(String.class);\n        when(this.messagingProcessor.ackMessage(any(), any(), ackMessageIdArgumentCaptor.capture(), anyString(), anyString(), any(), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(mock(AckResult.class)));\n\n        ArgumentCaptor<String> toDLQMessageIdArgumentCaptor = ArgumentCaptor.forClass(String.class);\n        when(this.messagingProcessor.forwardMessageToDeadLetterQueue(any(), any(), toDLQMessageIdArgumentCaptor.capture(), anyString(), anyString(), any(), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(mock(RemotingCommand.class)));\n\n        AddressableMessageQueue messageQueue = mock(AddressableMessageQueue.class);\n        PopResult popResult = this.consumerProcessor.popMessage(\n            createContext(),\n            (ctx, messageQueueView) -> messageQueue,\n            CONSUMER_GROUP,\n            TOPIC,\n            60,\n            invisibleTime,\n            Duration.ofSeconds(3).toMillis(),\n            ConsumeInitMode.MAX,\n            FilterAPI.build(TOPIC, tag, ExpressionType.TAG),\n            false,\n            (ctx, consumerGroup, subscriptionData, messageExt) -> {\n                if (!messageExt.getTags().equals(tag)) {\n                    return PopMessageResultFilter.FilterResult.NO_MATCH;\n                }\n                if (messageExt.getReconsumeTimes() > 0) {\n                    return PopMessageResultFilter.FilterResult.TO_DLQ;\n                }\n                return PopMessageResultFilter.FilterResult.MATCH;\n            },\n            null,\n            Duration.ofSeconds(3).toMillis()\n        ).get();\n\n        assertSame(messageQueue, messageQueueArgumentCaptor.getValue());\n        assertEquals(CONSUMER_GROUP, requestHeaderArgumentCaptor.getValue().getConsumerGroup());\n        assertEquals(TOPIC, requestHeaderArgumentCaptor.getValue().getTopic());\n        assertEquals(ProxyUtils.MAX_MSG_NUMS_FOR_POP_REQUEST, requestHeaderArgumentCaptor.getValue().getMaxMsgNums());\n        assertEquals(tag, requestHeaderArgumentCaptor.getValue().getExp());\n        assertEquals(ExpressionType.TAG, requestHeaderArgumentCaptor.getValue().getExpType());\n\n        assertEquals(PopStatus.FOUND, popResult.getPopStatus());\n        assertEquals(1, popResult.getMsgFoundList().size());\n        assertEquals(messageExtList.get(1), popResult.getMsgFoundList().get(0));\n\n        assertEquals(messageExtList.get(0).getMsgId(), ackMessageIdArgumentCaptor.getValue());\n        assertEquals(messageExtList.get(2).getMsgId(), toDLQMessageIdArgumentCaptor.getValue());\n    }\n\n    @Test\n    public void testAckMessage() throws Throwable {\n        ReceiptHandle handle = create(createMessageExt(MixAll.RETRY_GROUP_TOPIC_PREFIX + TOPIC, \"\", 0, 3000));\n        assertNotNull(handle);\n\n        ArgumentCaptor<AckMessageRequestHeader> requestHeaderArgumentCaptor = ArgumentCaptor.forClass(AckMessageRequestHeader.class);\n        AckResult innerAckResult = new AckResult();\n        innerAckResult.setStatus(AckStatus.OK);\n        when(this.messageService.ackMessage(any(), any(), anyString(), requestHeaderArgumentCaptor.capture(), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(innerAckResult));\n\n        AckResult ackResult = this.consumerProcessor.ackMessage(createContext(), handle, MessageClientIDSetter.createUniqID(),\n            CONSUMER_GROUP, TOPIC, null, 3000).get();\n\n        assertEquals(AckStatus.OK, ackResult.getStatus());\n        assertEquals(KeyBuilder.buildPopRetryTopic(TOPIC, CONSUMER_GROUP, new BrokerConfig().isEnableRetryTopicV2()), requestHeaderArgumentCaptor.getValue().getTopic());\n        assertEquals(CONSUMER_GROUP, requestHeaderArgumentCaptor.getValue().getConsumerGroup());\n        assertEquals(handle.getReceiptHandle(), requestHeaderArgumentCaptor.getValue().getExtraInfo());\n    }\n\n    @Test\n    public void testBatchAckExpireMessage() throws Throwable {\n        String brokerName1 = \"brokerName1\";\n\n        List<ReceiptHandleMessage> receiptHandleMessageList = new ArrayList<>();\n        for (int i = 0; i < 3; i++) {\n            MessageExt expireMessage = createMessageExt(TOPIC, \"\", 0, 3000, System.currentTimeMillis() - 10000,\n                0, 0, 0, i, brokerName1);\n            ReceiptHandle expireHandle = create(expireMessage);\n            receiptHandleMessageList.add(new ReceiptHandleMessage(expireHandle, expireMessage.getMsgId()));\n        }\n\n        List<BatchAckResult> batchAckResultList = this.consumerProcessor.batchAckMessage(createContext(), receiptHandleMessageList, CONSUMER_GROUP, TOPIC, 3000).get();\n\n        verify(this.messageService, never()).batchAckMessage(any(), anyList(), anyString(), anyString(), anyLong());\n        assertEquals(receiptHandleMessageList.size(), batchAckResultList.size());\n        for (BatchAckResult batchAckResult : batchAckResultList) {\n            assertNull(batchAckResult.getAckResult());\n            assertNotNull(batchAckResult.getProxyException());\n            assertNotNull(batchAckResult.getReceiptHandleMessage());\n        }\n\n    }\n\n    @Test\n    public void testBatchAckMessage() throws Throwable {\n        String brokerName1 = \"brokerName1\";\n        String brokerName2 = \"brokerName2\";\n        String errThrowBrokerName = \"errThrowBrokerName\";\n        MessageExt expireMessage = createMessageExt(TOPIC, \"\", 0, 3000, System.currentTimeMillis() - 10000,\n            0, 0, 0, 0, brokerName1);\n        ReceiptHandle expireHandle = create(expireMessage);\n\n        List<ReceiptHandleMessage> receiptHandleMessageList = new ArrayList<>();\n        receiptHandleMessageList.add(new ReceiptHandleMessage(expireHandle, expireMessage.getMsgId()));\n        List<String> broker1Msg = new ArrayList<>();\n        List<String> broker2Msg = new ArrayList<>();\n\n        long now = System.currentTimeMillis();\n        int msgNum = 3;\n        for (int i = 0; i < msgNum; i++) {\n            MessageExt brokerMessage = createMessageExt(TOPIC, \"\", 0, 3000, now,\n                0, 0, 0, i + 1, brokerName1);\n            ReceiptHandle brokerHandle = create(brokerMessage);\n            receiptHandleMessageList.add(new ReceiptHandleMessage(brokerHandle, brokerMessage.getMsgId()));\n            broker1Msg.add(brokerMessage.getMsgId());\n        }\n        for (int i = 0; i < msgNum; i++) {\n            MessageExt brokerMessage = createMessageExt(TOPIC, \"\", 0, 3000, now,\n                0, 0, 0, i + 1, brokerName2);\n            ReceiptHandle brokerHandle = create(brokerMessage);\n            receiptHandleMessageList.add(new ReceiptHandleMessage(brokerHandle, brokerMessage.getMsgId()));\n            broker2Msg.add(brokerMessage.getMsgId());\n        }\n\n        // for this message, will throw exception in batchAckMessage\n        MessageExt errThrowMessage = createMessageExt(TOPIC, \"\", 0, 3000, now,\n            0, 0, 0, 0, errThrowBrokerName);\n        ReceiptHandle errThrowHandle = create(errThrowMessage);\n        receiptHandleMessageList.add(new ReceiptHandleMessage(errThrowHandle, errThrowMessage.getMsgId()));\n\n        Collections.shuffle(receiptHandleMessageList);\n\n        doAnswer((Answer<CompletableFuture<AckResult>>) invocation -> {\n            List<ReceiptHandleMessage> handleMessageList = invocation.getArgument(1, List.class);\n            AckResult ackResult = new AckResult();\n            String brokerName = handleMessageList.get(0).getReceiptHandle().getBrokerName();\n            if (brokerName.equals(brokerName1)) {\n                ackResult.setStatus(AckStatus.OK);\n            } else if (brokerName.equals(brokerName2)) {\n                ackResult.setStatus(AckStatus.NO_EXIST);\n            } else {\n                return FutureUtils.completeExceptionally(new RuntimeException());\n            }\n\n            return CompletableFuture.completedFuture(ackResult);\n        }).when(this.messageService).batchAckMessage(any(), anyList(), anyString(), anyString(), anyLong());\n\n        List<BatchAckResult> batchAckResultList = this.consumerProcessor.batchAckMessage(createContext(), receiptHandleMessageList, CONSUMER_GROUP, TOPIC, 3000).get();\n        assertEquals(receiptHandleMessageList.size(), batchAckResultList.size());\n\n        // check ackResult for each msg\n        Map<String, BatchAckResult> msgBatchAckResult = new HashMap<>();\n        for (BatchAckResult batchAckResult : batchAckResultList) {\n            msgBatchAckResult.put(batchAckResult.getReceiptHandleMessage().getMessageId(), batchAckResult);\n        }\n        for (String msgId : broker1Msg) {\n            assertEquals(AckStatus.OK, msgBatchAckResult.get(msgId).getAckResult().getStatus());\n            assertNull(msgBatchAckResult.get(msgId).getProxyException());\n        }\n        for (String msgId : broker2Msg) {\n            assertEquals(AckStatus.NO_EXIST, msgBatchAckResult.get(msgId).getAckResult().getStatus());\n            assertNull(msgBatchAckResult.get(msgId).getProxyException());\n        }\n        assertNotNull(msgBatchAckResult.get(expireMessage.getMsgId()).getProxyException());\n        assertEquals(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, msgBatchAckResult.get(expireMessage.getMsgId()).getProxyException().getCode());\n        assertNull(msgBatchAckResult.get(expireMessage.getMsgId()).getAckResult());\n\n        assertNotNull(msgBatchAckResult.get(errThrowMessage.getMsgId()).getProxyException());\n        assertEquals(ProxyExceptionCode.INTERNAL_SERVER_ERROR, msgBatchAckResult.get(errThrowMessage.getMsgId()).getProxyException().getCode());\n        assertNull(msgBatchAckResult.get(errThrowMessage.getMsgId()).getAckResult());\n    }\n\n    @Test\n    public void testChangeInvisibleTime() throws Throwable {\n        ReceiptHandle handle = create(createMessageExt(MixAll.RETRY_GROUP_TOPIC_PREFIX + TOPIC, \"\", 0, 3000));\n        assertNotNull(handle);\n\n        ArgumentCaptor<ChangeInvisibleTimeRequestHeader> requestHeaderArgumentCaptor = ArgumentCaptor.forClass(ChangeInvisibleTimeRequestHeader.class);\n        AckResult innerAckResult = new AckResult();\n        innerAckResult.setStatus(AckStatus.OK);\n        when(this.messageService.changeInvisibleTime(any(), any(), anyString(), requestHeaderArgumentCaptor.capture(), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(innerAckResult));\n\n        AckResult ackResult = this.consumerProcessor.changeInvisibleTime(createContext(), handle, MessageClientIDSetter.createUniqID(),\n            CONSUMER_GROUP, TOPIC, 1000, null, 3000, true).get();\n\n        assertEquals(AckStatus.OK, ackResult.getStatus());\n        assertEquals(KeyBuilder.buildPopRetryTopic(TOPIC, CONSUMER_GROUP, new BrokerConfig().isEnableRetryTopicV2()), requestHeaderArgumentCaptor.getValue().getTopic());\n        assertEquals(CONSUMER_GROUP, requestHeaderArgumentCaptor.getValue().getConsumerGroup());\n        assertEquals(1000, requestHeaderArgumentCaptor.getValue().getInvisibleTime().longValue());\n        assertEquals(handle.getReceiptHandle(), requestHeaderArgumentCaptor.getValue().getExtraInfo());\n    }\n\n    @Test\n    public void testLockBatch() throws Throwable {\n        Set<MessageQueue> mqSet = new HashSet<>();\n        MessageQueue mq1 = new MessageQueue(TOPIC, \"broker1\", 0);\n        AddressableMessageQueue addressableMessageQueue1 = new AddressableMessageQueue(mq1, \"127.0.0.1\");\n        MessageQueue mq2 = new MessageQueue(TOPIC, \"broker2\", 0);\n        AddressableMessageQueue addressableMessageQueue2 = new AddressableMessageQueue(mq2, \"127.0.0.1\");\n        mqSet.add(mq1);\n        mqSet.add(mq2);\n        when(this.topicRouteService.buildAddressableMessageQueue(any(), any())).thenAnswer(i -> new AddressableMessageQueue((MessageQueue) i.getArguments()[1], \"127.0.0.1\"));\n        when(this.messageService.lockBatchMQ(any(), eq(addressableMessageQueue1), any(), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(Sets.newHashSet(mq1)));\n        when(this.messageService.lockBatchMQ(any(), eq(addressableMessageQueue2), any(), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(Sets.newHashSet(mq2)));\n        Set<MessageQueue> result = this.consumerProcessor.lockBatchMQ(ProxyContext.create(), mqSet, CONSUMER_GROUP, CLIENT_ID, 1000)\n            .get();\n        assertThat(result).isEqualTo(mqSet);\n    }\n\n    @Test\n    public void testLockBatchPartialSuccess() throws Throwable {\n        Set<MessageQueue> mqSet = new HashSet<>();\n        MessageQueue mq1 = new MessageQueue(TOPIC, \"broker1\", 0);\n        AddressableMessageQueue addressableMessageQueue1 = new AddressableMessageQueue(mq1, \"127.0.0.1\");\n        MessageQueue mq2 = new MessageQueue(TOPIC, \"broker2\", 0);\n        AddressableMessageQueue addressableMessageQueue2 = new AddressableMessageQueue(mq2, \"127.0.0.1\");\n        mqSet.add(mq1);\n        mqSet.add(mq2);\n        when(this.topicRouteService.buildAddressableMessageQueue(any(), any())).thenAnswer(i -> new AddressableMessageQueue((MessageQueue) i.getArguments()[1], \"127.0.0.1\"));\n        when(this.messageService.lockBatchMQ(any(), eq(addressableMessageQueue1), any(), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(Sets.newHashSet(mq1)));\n        when(this.messageService.lockBatchMQ(any(), eq(addressableMessageQueue2), any(), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(Sets.newHashSet()));\n        Set<MessageQueue> result = this.consumerProcessor.lockBatchMQ(ProxyContext.create(), mqSet, CONSUMER_GROUP, CLIENT_ID, 1000)\n            .get();\n        assertThat(result).isEqualTo(Sets.newHashSet(mq1));\n    }\n\n    @Test\n    public void testLockBatchPartialSuccessWithException() throws Throwable {\n        Set<MessageQueue> mqSet = new HashSet<>();\n        MessageQueue mq1 = new MessageQueue(TOPIC, \"broker1\", 0);\n        AddressableMessageQueue addressableMessageQueue1 = new AddressableMessageQueue(mq1, \"127.0.0.1\");\n        MessageQueue mq2 = new MessageQueue(TOPIC, \"broker2\", 0);\n        AddressableMessageQueue addressableMessageQueue2 = new AddressableMessageQueue(mq2, \"127.0.0.1\");\n        mqSet.add(mq1);\n        mqSet.add(mq2);\n        when(this.topicRouteService.buildAddressableMessageQueue(any(), any())).thenAnswer(i -> new AddressableMessageQueue((MessageQueue) i.getArguments()[1], \"127.0.0.1\"));\n        when(this.messageService.lockBatchMQ(any(), eq(addressableMessageQueue1), any(), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(Sets.newHashSet(mq1)));\n        CompletableFuture<Set<MessageQueue>> future = new CompletableFuture<>();\n        future.completeExceptionally(new MQBrokerException(1, \"err\"));\n        when(this.messageService.lockBatchMQ(any(), eq(addressableMessageQueue2), any(), anyLong()))\n            .thenReturn(future);\n        Set<MessageQueue> result = this.consumerProcessor.lockBatchMQ(ProxyContext.create(), mqSet, CONSUMER_GROUP, CLIENT_ID, 1000)\n            .get();\n        assertThat(result).isEqualTo(Sets.newHashSet(mq1));\n    }\n\n    @Test\n    public void testPopMessageWithToReturnFilter() throws Throwable {\n        final String tag = \"tag\";\n        final long invisibleTime = Duration.ofSeconds(15).toMillis();\n        ArgumentCaptor<AddressableMessageQueue> messageQueueArgumentCaptor = ArgumentCaptor.forClass(AddressableMessageQueue.class);\n        ArgumentCaptor<PopMessageRequestHeader> requestHeaderArgumentCaptor = ArgumentCaptor.forClass(PopMessageRequestHeader.class);\n\n        List<MessageExt> messageExtList = new ArrayList<>();\n        messageExtList.add(createMessageExt(TOPIC, tag, 0, invisibleTime));\n        PopResult innerPopResult = new PopResult(PopStatus.FOUND, messageExtList);\n        when(this.messageService.popMessage(any(), messageQueueArgumentCaptor.capture(), requestHeaderArgumentCaptor.capture(), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(innerPopResult));\n\n        when(this.topicRouteService.getCurrentMessageQueueView(any(), anyString()))\n            .thenReturn(mock(MessageQueueView.class));\n\n        ArgumentCaptor<String> ackMessageIdArgumentCaptor = ArgumentCaptor.forClass(String.class);\n        when(this.messagingProcessor.ackMessage(any(), any(), ackMessageIdArgumentCaptor.capture(), anyString(), anyString(), any(), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(mock(AckResult.class)));\n\n        ArgumentCaptor<String> changeInvisibleTimeMessageIdArgumentCaptor = ArgumentCaptor.forClass(String.class);\n        ArgumentCaptor<Long> changeInvisibleTimeInvisibleTimeArgumentCaptor = ArgumentCaptor.forClass(Long.class);\n        ArgumentCaptor<Boolean> changeInvisibleTimeSuspendArgumentCaptor = ArgumentCaptor.forClass(Boolean.class);\n        when(this.messagingProcessor.changeInvisibleTime(any(), any(), changeInvisibleTimeMessageIdArgumentCaptor.capture(),\n            anyString(), anyString(), changeInvisibleTimeInvisibleTimeArgumentCaptor.capture(), any(), anyLong(),\n            changeInvisibleTimeSuspendArgumentCaptor.capture()))\n            .thenReturn(CompletableFuture.completedFuture(mock(AckResult.class)));\n\n        AddressableMessageQueue messageQueue = mock(AddressableMessageQueue.class);\n        PopResult popResult = this.consumerProcessor.popMessage(\n            createContext(),\n            (ctx, messageQueueView) -> messageQueue,\n            CONSUMER_GROUP,\n            TOPIC,\n            60,\n            invisibleTime,\n            Duration.ofSeconds(3).toMillis(),\n            ConsumeInitMode.MAX,\n            FilterAPI.build(TOPIC, tag, ExpressionType.TAG),\n            false,\n            (ctx, consumerGroup, subscriptionData, messageExt) -> {\n                // Return TO_RETURN for the message\n                return PopMessageResultFilter.FilterResult.TO_RETURN;\n            },\n            null,\n            Duration.ofSeconds(3).toMillis()\n        ).get();\n\n        // Verify that changeInvisibleTime was called with suspend=true\n        verify(this.messagingProcessor).changeInvisibleTime(any(), any(), eq(messageExtList.get(0).getMsgId()),\n            eq(CONSUMER_GROUP), eq(TOPIC), eq(Duration.ofSeconds(1).toMillis()), eq(null),\n            eq(MessagingProcessor.DEFAULT_TIMEOUT_MILLS), eq(true));\n\n        // Verify that the message was NOT added to the result list\n        assertEquals(PopStatus.FOUND, popResult.getPopStatus());\n        assertEquals(0, popResult.getMsgFoundList().size());\n    }\n\n    @Test\n    public void testChangeInvisibleTimeWithSuspendFalse() throws Throwable {\n        ReceiptHandle handle = create(createMessageExt(MixAll.RETRY_GROUP_TOPIC_PREFIX + TOPIC, \"\", 0, 3000));\n        assertNotNull(handle);\n\n        ArgumentCaptor<ChangeInvisibleTimeRequestHeader> requestHeaderArgumentCaptor = ArgumentCaptor.forClass(ChangeInvisibleTimeRequestHeader.class);\n        AckResult innerAckResult = new AckResult();\n        innerAckResult.setStatus(AckStatus.OK);\n        when(this.messageService.changeInvisibleTime(any(), any(), anyString(), requestHeaderArgumentCaptor.capture(), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(innerAckResult));\n\n        AckResult ackResult = this.consumerProcessor.changeInvisibleTime(createContext(), handle, MessageClientIDSetter.createUniqID(),\n            CONSUMER_GROUP, TOPIC, 1000, null, 3000, false).get();\n\n        assertEquals(AckStatus.OK, ackResult.getStatus());\n        assertEquals(KeyBuilder.buildPopRetryTopic(TOPIC, CONSUMER_GROUP, new BrokerConfig().isEnableRetryTopicV2()), requestHeaderArgumentCaptor.getValue().getTopic());\n        assertEquals(CONSUMER_GROUP, requestHeaderArgumentCaptor.getValue().getConsumerGroup());\n        assertEquals(1000, requestHeaderArgumentCaptor.getValue().getInvisibleTime().longValue());\n        assertEquals(handle.getReceiptHandle(), requestHeaderArgumentCaptor.getValue().getExtraInfo());\n        assertFalse(\"Suspend should be false\", requestHeaderArgumentCaptor.getValue().isSuspend());\n    }\n\n    @Test\n    public void testChangeInvisibleTimeWithSuspendTrue() throws Throwable {\n        ReceiptHandle handle = create(createMessageExt(MixAll.RETRY_GROUP_TOPIC_PREFIX + TOPIC, \"\", 0, 3000));\n        assertNotNull(handle);\n\n        ArgumentCaptor<ChangeInvisibleTimeRequestHeader> requestHeaderArgumentCaptor = ArgumentCaptor.forClass(ChangeInvisibleTimeRequestHeader.class);\n        AckResult innerAckResult = new AckResult();\n        innerAckResult.setStatus(AckStatus.OK);\n        when(this.messageService.changeInvisibleTime(any(), any(), anyString(), requestHeaderArgumentCaptor.capture(), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(innerAckResult));\n\n        AckResult ackResult = this.consumerProcessor.changeInvisibleTime(createContext(), handle, MessageClientIDSetter.createUniqID(),\n            CONSUMER_GROUP, TOPIC, 1000, null, 3000, true).get();\n\n        assertEquals(AckStatus.OK, ackResult.getStatus());\n        assertEquals(KeyBuilder.buildPopRetryTopic(TOPIC, CONSUMER_GROUP, new BrokerConfig().isEnableRetryTopicV2()), requestHeaderArgumentCaptor.getValue().getTopic());\n        assertEquals(CONSUMER_GROUP, requestHeaderArgumentCaptor.getValue().getConsumerGroup());\n        assertEquals(1000, requestHeaderArgumentCaptor.getValue().getInvisibleTime().longValue());\n        assertEquals(handle.getReceiptHandle(), requestHeaderArgumentCaptor.getValue().getExtraInfo());\n        assertTrue(\"Suspend should be true\", requestHeaderArgumentCaptor.getValue().isSuspend());\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/processor/ProducerProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.processor;\n\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.CompletionException;\nimport java.util.concurrent.Executors;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.producer.RecallMessageHandle;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\nimport org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;\nimport org.apache.rocketmq.proxy.service.transaction.TransactionData;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.assertj.core.util.Lists;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.ArgumentCaptor;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\npublic class ProducerProcessorTest extends BaseProcessorTest {\n\n    private static final String PRODUCER_GROUP = \"producerGroup\";\n    private static final String CONSUMER_GROUP = \"consumerGroup\";\n    private static final String TOPIC = \"topic\";\n\n    private ProducerProcessor producerProcessor;\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        this.producerProcessor = new ProducerProcessor(this.messagingProcessor, this.serviceManager, Executors.newCachedThreadPool());\n    }\n\n    @Test\n    public void testSendMessage() throws Throwable {\n        when(metadataService.getTopicMessageType(any(), eq(TOPIC))).thenReturn(TopicMessageType.NORMAL);\n        String txId = MessageClientIDSetter.createUniqID();\n        String msgId = MessageClientIDSetter.createUniqID();\n        long commitLogOffset = 1000L;\n        long queueOffset = 100L;\n\n        SendResult sendResult = new SendResult();\n        sendResult.setSendStatus(SendStatus.SEND_OK);\n        sendResult.setTransactionId(txId);\n        sendResult.setMsgId(msgId);\n        sendResult.setOffsetMsgId(createOffsetMsgId(commitLogOffset));\n        sendResult.setQueueOffset(queueOffset);\n        ArgumentCaptor<SendMessageRequestHeader> requestHeaderArgumentCaptor = ArgumentCaptor.forClass(SendMessageRequestHeader.class);\n        when(this.messageService.sendMessage(any(), any(), any(), requestHeaderArgumentCaptor.capture(), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(Lists.newArrayList(sendResult)));\n\n        List<Message> messageList = new ArrayList<>();\n        Message messageExt = createMessageExt(TOPIC, \"tag\", 0, 0);\n        messageList.add(messageExt);\n        AddressableMessageQueue messageQueue = mock(AddressableMessageQueue.class);\n        when(messageQueue.getBrokerName()).thenReturn(\"mockBroker\");\n\n        ArgumentCaptor<String> brokerNameCaptor = ArgumentCaptor.forClass(String.class);\n        ArgumentCaptor<Long> tranStateTableOffsetCaptor = ArgumentCaptor.forClass(Long.class);\n        ArgumentCaptor<Long> commitLogOffsetCaptor = ArgumentCaptor.forClass(Long.class);\n        when(transactionService.addTransactionDataByBrokerName(\n            any(),\n            brokerNameCaptor.capture(),\n            anyString(),\n            anyString(),\n            tranStateTableOffsetCaptor.capture(),\n            commitLogOffsetCaptor.capture(),\n            anyString(), any())).thenReturn(mock(TransactionData.class));\n\n        List<SendResult> sendResultList = this.producerProcessor.sendMessage(\n            createContext(),\n            (ctx, messageQueueView) -> messageQueue,\n            PRODUCER_GROUP,\n            MessageSysFlag.TRANSACTION_PREPARED_TYPE,\n            messageList,\n            3000\n        ).get();\n\n        assertNotNull(sendResultList);\n        assertEquals(\"mockBroker\", brokerNameCaptor.getValue());\n        assertEquals(queueOffset, tranStateTableOffsetCaptor.getValue().longValue());\n        assertEquals(commitLogOffset, commitLogOffsetCaptor.getValue().longValue());\n\n        SendMessageRequestHeader requestHeader = requestHeaderArgumentCaptor.getValue();\n        assertEquals(PRODUCER_GROUP, requestHeader.getProducerGroup());\n        assertEquals(TOPIC, requestHeader.getTopic());\n    }\n\n    @Test\n    public void testSendRetryMessage() throws Throwable {\n        String txId = MessageClientIDSetter.createUniqID();\n        String msgId = MessageClientIDSetter.createUniqID();\n        long commitLogOffset = 1000L;\n        long queueOffset = 100L;\n\n        SendResult sendResult = new SendResult();\n        sendResult.setSendStatus(SendStatus.SEND_OK);\n        sendResult.setTransactionId(txId);\n        sendResult.setMsgId(msgId);\n        sendResult.setOffsetMsgId(createOffsetMsgId(commitLogOffset));\n        sendResult.setQueueOffset(queueOffset);\n        ArgumentCaptor<SendMessageRequestHeader> requestHeaderArgumentCaptor = ArgumentCaptor.forClass(SendMessageRequestHeader.class);\n        when(this.messageService.sendMessage(any(), any(), any(), requestHeaderArgumentCaptor.capture(), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(Lists.newArrayList(sendResult)));\n\n        List<Message> messageExtList = new ArrayList<>();\n        Message messageExt = createMessageExt(MixAll.getRetryTopic(CONSUMER_GROUP), \"tag\", 0, 0);\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_RECONSUME_TIME, \"1\");\n        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_MAX_RECONSUME_TIMES, \"16\");\n        messageExtList.add(messageExt);\n        AddressableMessageQueue messageQueue = mock(AddressableMessageQueue.class);\n        when(messageQueue.getBrokerName()).thenReturn(\"mockBroker\");\n\n        ArgumentCaptor<String> brokerNameCaptor = ArgumentCaptor.forClass(String.class);\n        ArgumentCaptor<Long> tranStateTableOffsetCaptor = ArgumentCaptor.forClass(Long.class);\n        ArgumentCaptor<Long> commitLogOffsetCaptor = ArgumentCaptor.forClass(Long.class);\n        when(transactionService.addTransactionDataByBrokerName(\n            any(),\n            brokerNameCaptor.capture(),\n            anyString(),\n            anyString(),\n            tranStateTableOffsetCaptor.capture(),\n            commitLogOffsetCaptor.capture(),\n            anyString(), any())).thenReturn(mock(TransactionData.class));\n\n        List<SendResult> sendResultList = this.producerProcessor.sendMessage(\n            createContext(),\n            (ctx, messageQueueView) -> messageQueue,\n            PRODUCER_GROUP,\n            MessageSysFlag.TRANSACTION_PREPARED_TYPE,\n            messageExtList,\n            3000\n        ).get();\n\n        assertNotNull(sendResultList);\n        assertEquals(\"mockBroker\", brokerNameCaptor.getValue());\n        assertEquals(queueOffset, tranStateTableOffsetCaptor.getValue().longValue());\n        assertEquals(commitLogOffset, commitLogOffsetCaptor.getValue().longValue());\n\n        SendMessageRequestHeader requestHeader = requestHeaderArgumentCaptor.getValue();\n        assertEquals(PRODUCER_GROUP, requestHeader.getProducerGroup());\n        assertEquals(MixAll.getRetryTopic(CONSUMER_GROUP), requestHeader.getTopic());\n        assertEquals(1, requestHeader.getReconsumeTimes().intValue());\n        assertEquals(16, requestHeader.getMaxReconsumeTimes().intValue());\n    }\n\n    @Test\n    public void testForwardMessageToDeadLetterQueue() throws Throwable {\n        ArgumentCaptor<ConsumerSendMsgBackRequestHeader> requestHeaderArgumentCaptor = ArgumentCaptor.forClass(ConsumerSendMsgBackRequestHeader.class);\n        when(this.messageService.sendMessageBack(any(), any(), anyString(), requestHeaderArgumentCaptor.capture(), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(mock(RemotingCommand.class)));\n\n        MessageExt messageExt = createMessageExt(KeyBuilder.buildPopRetryTopic(TOPIC, CONSUMER_GROUP, new BrokerConfig().isEnableRetryTopicV2()), \"\", 16, 3000);\n        RemotingCommand remotingCommand = this.producerProcessor.forwardMessageToDeadLetterQueue(\n            createContext(),\n            create(messageExt),\n            messageExt.getMsgId(),\n            CONSUMER_GROUP,\n            TOPIC,\n            null,\n            3000\n        ).get();\n\n        assertNotNull(remotingCommand);\n        ConsumerSendMsgBackRequestHeader requestHeader = requestHeaderArgumentCaptor.getValue();\n        assertEquals(messageExt.getTopic(), requestHeader.getOriginTopic());\n        assertEquals(messageExt.getMsgId(), requestHeader.getOriginMsgId());\n        assertEquals(CONSUMER_GROUP, requestHeader.getGroup());\n    }\n\n    @Test\n    public void testRecallMessage_notDelayMessage() {\n        when(metadataService.getTopicMessageType(any(), any())).thenReturn(TopicMessageType.NORMAL);\n        CompletionException exception = Assert.assertThrows(CompletionException.class, () -> {\n            producerProcessor.recallMessage(createContext(), TOPIC, \"handle\", 3000).join();\n        });\n        assertTrue(exception.getCause() instanceof ProxyException);\n        ProxyException cause = (ProxyException) exception.getCause();\n        assertEquals(ProxyExceptionCode.MESSAGE_PROPERTY_CONFLICT_WITH_TYPE, cause.getCode());\n    }\n\n    @Test\n    public void testRecallMessage_invalidRecallHandle() {\n        when(metadataService.getTopicMessageType(any(), any())).thenReturn(TopicMessageType.DELAY);\n        CompletionException exception = Assert.assertThrows(CompletionException.class, () -> {\n            producerProcessor.recallMessage(createContext(), TOPIC, \"handle\", 3000).join();\n        });\n        assertTrue(exception.getCause() instanceof ProxyException);\n        ProxyException cause = (ProxyException) exception.getCause();\n        assertEquals(\"recall handle is invalid\", cause.getMessage());\n    }\n\n    @Test\n    public void testRecallMessage_success() {\n        when(metadataService.getTopicMessageType(any(), any())).thenReturn(TopicMessageType.DELAY);\n        when(this.messageService.recallMessage(any(), any(), any(), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(\"msgId\"));\n\n        String handle = RecallMessageHandle.HandleV1.buildHandle(TOPIC, \"brokerName\", \"timestampStr\", \"whateverId\");\n        String msgId = producerProcessor.recallMessage(createContext(), TOPIC, handle, 3000).join();\n        assertEquals(\"msgId\", msgId);\n    }\n\n    private static String createOffsetMsgId(long commitLogOffset) {\n        int msgIDLength = 4 + 4 + 8;\n        ByteBuffer byteBufferMsgId = ByteBuffer.allocate(msgIDLength);\n        return MessageDecoder.createMessageId(byteBufferMsgId,\n            MessageExt.socketAddress2ByteBuffer(NetworkUtil.string2SocketAddress(\"127.0.0.1:10911\")),\n            commitLogOffset);\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/processor/ReceiptHandleProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.processor;\n\nimport io.netty.channel.local.LocalChannel;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ConsumerManager;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.proxy.common.ContextVariable;\nimport org.apache.rocketmq.proxy.common.MessageReceiptHandle;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.InitConfigTest;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.service.ServiceManager;\nimport org.apache.rocketmq.proxy.service.metadata.MetadataService;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ReceiptHandleProcessorTest extends InitConfigTest {\n\n    @Mock\n    protected MessagingProcessor messagingProcessor;\n    @Mock\n    protected ServiceManager serviceManager;\n    @Mock\n    protected ConsumerManager consumerManager;\n    @Mock\n    protected MetadataService metadataService;\n\n    private static final ProxyContext PROXY_CONTEXT = ProxyContext.create();\n    private static final String CONSUMER_GROUP = \"consumerGroup\";\n    private static final String TOPIC = \"topic\";\n    private static final String BROKER_NAME = \"broker\";\n    private static final int QUEUE_ID = 1;\n    private static final String MESSAGE_ID = \"messageId\";\n    private static final long OFFSET = 123L;\n    private static final long INVISIBLE_TIME = 60000L;\n    private static final int RECONSUME_TIMES = 1;\n    private static final String MSG_ID = MessageClientIDSetter.createUniqID();\n    private MessageReceiptHandle messageReceiptHandle;\n\n    private ReceiptHandleProcessor receiptHandleProcessor;\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        when(serviceManager.getConsumerManager()).thenReturn(consumerManager);\n        when(serviceManager.getMetadataService()).thenReturn(metadataService);\n        this.receiptHandleProcessor = new ReceiptHandleProcessor(this.messagingProcessor, this.serviceManager);\n        ProxyConfig config = ConfigurationManager.getProxyConfig();\n        String receiptHandle = ReceiptHandle.builder()\n            .startOffset(0L)\n            .retrieveTime(System.currentTimeMillis() - INVISIBLE_TIME + config.getRenewAheadTimeMillis() - 5)\n            .invisibleTime(INVISIBLE_TIME)\n            .reviveQueueId(1)\n            .topicType(ReceiptHandle.NORMAL_TOPIC)\n            .brokerName(BROKER_NAME)\n            .queueId(QUEUE_ID)\n            .offset(OFFSET)\n            .commitLogOffset(0L)\n            .build().encode();\n        PROXY_CONTEXT.withVal(ContextVariable.CLIENT_ID, \"channel-id\");\n        PROXY_CONTEXT.withVal(ContextVariable.CHANNEL, new LocalChannel());\n        messageReceiptHandle = new MessageReceiptHandle(CONSUMER_GROUP, TOPIC, QUEUE_ID, receiptHandle, MESSAGE_ID, OFFSET,\n            RECONSUME_TIMES);\n    }\n\n    @Test\n    public void testStart() throws Exception {\n        receiptHandleProcessor.start();\n        receiptHandleProcessor.addReceiptHandle(PROXY_CONTEXT, PROXY_CONTEXT.getChannel(), CONSUMER_GROUP, MSG_ID, messageReceiptHandle);\n        Mockito.when(consumerManager.findChannel(Mockito.eq(CONSUMER_GROUP), Mockito.eq(PROXY_CONTEXT.getChannel()))).thenReturn(Mockito.mock(ClientChannelInfo.class));\n        Mockito.verify(messagingProcessor, Mockito.timeout(10000).times(1))\n            .changeInvisibleTime(Mockito.any(ProxyContext.class), Mockito.any(ReceiptHandle.class), Mockito.eq(MESSAGE_ID),\n                Mockito.eq(CONSUMER_GROUP), Mockito.eq(TOPIC), Mockito.eq(ConfigurationManager.getProxyConfig().getDefaultInvisibleTimeMills()), Mockito.eq(null));\n    }\n\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/processor/TransactionProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.processor;\n\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.proxy.service.transaction.EndTransactionRequestData;\nimport org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.ArgumentCaptor;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyBoolean;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.reset;\nimport static org.mockito.Mockito.when;\n\npublic class TransactionProcessorTest extends BaseProcessorTest {\n\n    private static final String PRODUCER_GROUP = \"producerGroup\";\n    private TransactionProcessor transactionProcessor;\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        this.transactionProcessor = new TransactionProcessor(this.messagingProcessor, this.serviceManager);\n    }\n\n    @Test\n    public void testEndTransaction() throws Throwable {\n        testEndTransaction(MessageSysFlag.TRANSACTION_COMMIT_TYPE, TransactionStatus.COMMIT);\n        testEndTransaction(MessageSysFlag.TRANSACTION_NOT_TYPE, TransactionStatus.UNKNOWN);\n        testEndTransaction(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE, TransactionStatus.ROLLBACK);\n    }\n\n    protected void testEndTransaction(int sysFlag, TransactionStatus transactionStatus) throws Throwable {\n        when(this.messageService.endTransactionOneway(any(), any(), any(), anyLong())).thenReturn(CompletableFuture.completedFuture(null));\n        ArgumentCaptor<Integer> commitOrRollbackCaptor = ArgumentCaptor.forClass(Integer.class);\n        when(transactionService.genEndTransactionRequestHeader(any(), anyString(), anyString(), commitOrRollbackCaptor.capture(), anyBoolean(), anyString(), anyString()))\n            .thenReturn(new EndTransactionRequestData(\"brokerName\", new EndTransactionRequestHeader()));\n\n        this.transactionProcessor.endTransaction(\n            createContext(),\n            \"topic\",\n            \"transactionId\",\n            \"msgId\",\n            PRODUCER_GROUP,\n            transactionStatus,\n            true,\n            3000\n        );\n\n        assertEquals(sysFlag, commitOrRollbackCaptor.getValue().intValue());\n\n        reset(this.messageService);\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannelTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.processor.channel;\n\nimport org.apache.commons.lang3.RandomStringUtils;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\n\npublic class RemoteChannelTest {\n\n    @Test\n    public void testEncodeAndDecode() {\n        String remoteProxyIp = \"11.193.0.1\";\n        String remoteAddress = \"10.152.39.53:9768\";\n        String localAddress = \"11.193.0.1:1210\";\n        ChannelProtocolType type = ChannelProtocolType.REMOTING;\n        String extendAttribute = RandomStringUtils.randomAlphabetic(10);\n        RemoteChannel remoteChannel = new RemoteChannel(remoteProxyIp, remoteAddress, localAddress, type, extendAttribute);\n\n        String encodedData = remoteChannel.encode();\n        assertNotNull(encodedData);\n\n        RemoteChannel decodedChannel = RemoteChannel.decode(encodedData);\n        assertEquals(remoteProxyIp, decodedChannel.remoteProxyIp);\n        assertEquals(remoteAddress, decodedChannel.getRemoteAddress());\n        assertEquals(localAddress, decodedChannel.getLocalAddress());\n        assertEquals(type, decodedChannel.type);\n        assertEquals(extendAttribute, decodedChannel.extendAttribute);\n\n        assertNull(RemoteChannel.decode(\"\"));\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivityTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.activity;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelFuture;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.channel.ChannelPromise;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.acl.common.AclException;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\nimport org.apache.rocketmq.proxy.config.InitConfigTest;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.service.channel.SimpleChannel;\nimport org.apache.rocketmq.proxy.service.channel.SimpleChannelHandlerContext;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.netty.AttributeKeys;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.ArgumentCaptor;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.never;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class AbstractRemotingActivityTest extends InitConfigTest {\n\n    private static final String CLIENT_ID = \"test@clientId\";\n    AbstractRemotingActivity remotingActivity;\n    @Mock\n    MessagingProcessor messagingProcessorMock;\n    @Spy\n    ChannelHandlerContext ctx = new SimpleChannelHandlerContext(new SimpleChannel(null, \"0.0.0.0:0\", \"1.1.1.1:1\")) {\n        @Override\n        public ChannelFuture writeAndFlush(Object msg, ChannelPromise promise) {\n            return null;\n        }\n    };\n\n    @Before\n    public void setup() {\n        remotingActivity = new AbstractRemotingActivity(null, messagingProcessorMock) {\n            @Override\n            protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,\n                ProxyContext context) throws Exception {\n                return null;\n            }\n        };\n        Channel channel = ctx.channel();\n        RemotingHelper.setPropertyToAttr(channel, AttributeKeys.CLIENT_ID_KEY, CLIENT_ID);\n        RemotingHelper.setPropertyToAttr(channel, AttributeKeys.LANGUAGE_CODE_KEY, LanguageCode.JAVA);\n        RemotingHelper.setPropertyToAttr(channel, AttributeKeys.VERSION_KEY, MQVersion.CURRENT_VERSION);\n    }\n\n    @Test\n    public void testRequest() throws Exception {\n        String brokerName = \"broker\";\n        RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"remark\");\n        when(messagingProcessorMock.request(any(), eq(brokerName), any(), anyLong())).thenReturn(CompletableFuture.completedFuture(\n            response\n        ));\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);\n        request.addExtField(AbstractRemotingActivity.BROKER_NAME_FIELD, brokerName);\n        RemotingCommand remotingCommand = remotingActivity.request(ctx, request, null, 10000);\n        assertThat(remotingCommand).isNull();\n        verify(ctx, times(1)).writeAndFlush(response);\n    }\n\n    @Test\n    public void testRequestOneway() throws Exception {\n        String brokerName = \"broker\";\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);\n        request.markOnewayRPC();\n        request.addExtField(AbstractRemotingActivity.BROKER_NAME_FIELD, brokerName);\n        RemotingCommand remotingCommand = remotingActivity.request(ctx, request, null, 10000);\n        assertThat(remotingCommand).isNull();\n        verify(messagingProcessorMock, times(1)).requestOneway(any(), eq(brokerName), any(), anyLong());\n    }\n\n    @Test\n    public void testRequestInvalid() throws Exception {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);\n        request.addExtField(\"test\", \"test\");\n        RemotingCommand remotingCommand = remotingActivity.request(ctx, request, null, 10000);\n        assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.VERSION_NOT_SUPPORTED);\n        verify(ctx, never()).writeAndFlush(any());\n    }\n\n    @Test\n    public void testRequestProxyException() throws Exception {\n        ArgumentCaptor<RemotingCommand> captor = ArgumentCaptor.forClass(RemotingCommand.class);\n        String brokerName = \"broker\";\n        String remark = \"exception\";\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        future.completeExceptionally(new ProxyException(ProxyExceptionCode.FORBIDDEN, remark));\n        when(messagingProcessorMock.request(any(), eq(brokerName), any(), anyLong())).thenReturn(future);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);\n        request.addExtField(AbstractRemotingActivity.BROKER_NAME_FIELD, brokerName);\n        RemotingCommand remotingCommand = remotingActivity.request(ctx, request, null, 10000);\n        assertThat(remotingCommand).isNull();\n        verify(ctx, times(1)).writeAndFlush(captor.capture());\n        assertThat(captor.getValue().getCode()).isEqualTo(ResponseCode.NO_PERMISSION);\n    }\n\n    @Test\n    public void testRequestClientException() throws Exception {\n        ArgumentCaptor<RemotingCommand> captor = ArgumentCaptor.forClass(RemotingCommand.class);\n        String brokerName = \"broker\";\n        String remark = \"exception\";\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        future.completeExceptionally(new MQClientException(remark, null));\n        when(messagingProcessorMock.request(any(), eq(brokerName), any(), anyLong())).thenReturn(future);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);\n        request.addExtField(AbstractRemotingActivity.BROKER_NAME_FIELD, brokerName);\n        RemotingCommand remotingCommand = remotingActivity.request(ctx, request, null, 10000);\n        assertThat(remotingCommand).isNull();\n        verify(ctx, times(1)).writeAndFlush(captor.capture());\n        assertThat(captor.getValue().getCode()).isEqualTo(-1);\n    }\n\n    @Test\n    public void testRequestBrokerException() throws Exception {\n        ArgumentCaptor<RemotingCommand> captor = ArgumentCaptor.forClass(RemotingCommand.class);\n        String brokerName = \"broker\";\n        String remark = \"exception\";\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        future.completeExceptionally(new MQBrokerException(ResponseCode.FLUSH_DISK_TIMEOUT, remark));\n        when(messagingProcessorMock.request(any(), eq(brokerName), any(), anyLong())).thenReturn(future);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);\n        request.addExtField(AbstractRemotingActivity.BROKER_NAME_FIELD, brokerName);\n        RemotingCommand remotingCommand = remotingActivity.request(ctx, request, null, 10000);\n        assertThat(remotingCommand).isNull();\n        verify(ctx, times(1)).writeAndFlush(captor.capture());\n        assertThat(captor.getValue().getCode()).isEqualTo(ResponseCode.FLUSH_DISK_TIMEOUT);\n    }\n\n    @Test\n    public void testRequestAclException() throws Exception {\n        ArgumentCaptor<RemotingCommand> captor = ArgumentCaptor.forClass(RemotingCommand.class);\n        String brokerName = \"broker\";\n        String remark = \"exception\";\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        future.completeExceptionally(new AclException(remark, ResponseCode.MESSAGE_ILLEGAL));\n        when(messagingProcessorMock.request(any(), eq(brokerName), any(), anyLong())).thenReturn(future);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);\n        request.addExtField(AbstractRemotingActivity.BROKER_NAME_FIELD, brokerName);\n        RemotingCommand remotingCommand = remotingActivity.request(ctx, request, null, 10000);\n        assertThat(remotingCommand).isNull();\n        verify(ctx, times(1)).writeAndFlush(captor.capture());\n        assertThat(captor.getValue().getCode()).isEqualTo(ResponseCode.NO_PERMISSION);\n    }\n\n    @Test\n    public void testRequestDefaultException() throws Exception {\n        ArgumentCaptor<RemotingCommand> captor = ArgumentCaptor.forClass(RemotingCommand.class);\n        String brokerName = \"broker\";\n        String remark = \"exception\";\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        future.completeExceptionally(new Exception(remark));\n        when(messagingProcessorMock.request(any(), eq(brokerName), any(), anyLong())).thenReturn(future);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);\n        request.addExtField(AbstractRemotingActivity.BROKER_NAME_FIELD, brokerName);\n        RemotingCommand remotingCommand = remotingActivity.request(ctx, request, null, 10000);\n        assertThat(remotingCommand).isNull();\n        verify(ctx, times(1)).writeAndFlush(captor.capture());\n        assertThat(captor.getValue().getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivityTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.activity;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.alibaba.fastjson2.JSONWriter;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;\nimport org.apache.rocketmq.proxy.service.channel.SimpleChannel;\nimport org.apache.rocketmq.proxy.service.channel.SimpleChannelHandlerContext;\nimport org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.GetRouteInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyList;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class GetTopicRouteActivityTest {\n\n    @Mock\n    private RequestPipeline requestPipeline;\n\n    @Mock\n    private MessagingProcessor messagingProcessor;\n\n    private GetTopicRouteActivity getTopicRouteActivity;\n\n    private ChannelHandlerContext ctx;\n\n    private ProxyContext context;\n\n    @Before\n    public void setup() throws Exception {\n        getTopicRouteActivity = new GetTopicRouteActivity(requestPipeline, messagingProcessor);\n\n        ConfigurationManager.initEnv();\n        ConfigurationManager.initConfig();\n\n        Channel channel = new SimpleChannel(null, \"0.0.0.0:0\", \"1.1.1.1:1\");\n        ctx = new SimpleChannelHandlerContext(channel);\n\n        context = ProxyContext.create();\n    }\n\n    @Test\n    public void testProcessRequest0_HighVersion_SerializeWithFeatures() throws Exception {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC, null);\n        request.setVersion(MQVersion.Version.V4_9_4.ordinal());\n\n        GetRouteInfoRequestHeader header = new GetRouteInfoRequestHeader();\n        header.setTopic(\"TestTopic\");\n        header.setAcceptStandardJsonOnly(false);\n        request.writeCustomHeader(header);\n\n        TopicRouteData topicRouteData = prepareTopicRouteData();\n\n        TopicRouteData spyTopicRouteData = Mockito.spy(topicRouteData);\n\n        ProxyTopicRouteData proxyTopicRouteData = mock(ProxyTopicRouteData.class);\n        when(proxyTopicRouteData.buildTopicRouteData()).thenReturn(spyTopicRouteData);\n        when(messagingProcessor.getTopicRouteDataForProxy(any(ProxyContext.class), anyList(), any()))\n                .thenReturn(proxyTopicRouteData);\n\n        RemotingCommand response = getTopicRouteActivity.processRequest0(ctx, request, context);\n\n        assertNotNull(response);\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n\n        verify(spyTopicRouteData).encode(\n                JSONWriter.Feature.BrowserCompatible,\n                JSONWriter.Feature.MapSortField\n        );\n\n        TopicRouteData deserializedData = JSON.parseObject(response.getBody(), TopicRouteData.class);\n        assertEquals(topicRouteData.getOrderTopicConf(), deserializedData.getOrderTopicConf());\n        assertEquals(topicRouteData.getQueueDatas().size(), deserializedData.getQueueDatas().size());\n    }\n\n    @Test\n    public void testProcessRequest0_LowVersion_StandardJsonOnly_SerializeWithFeatures() throws Exception {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC, null);\n        request.setVersion(MQVersion.Version.V4_9_3.ordinal());\n\n        GetRouteInfoRequestHeader header = new GetRouteInfoRequestHeader();\n        header.setTopic(\"TestTopic\");\n        header.setAcceptStandardJsonOnly(true);\n        request.writeCustomHeader(header);\n\n        TopicRouteData topicRouteData = prepareTopicRouteData();\n\n        TopicRouteData spyTopicRouteData = Mockito.spy(topicRouteData);\n\n        ProxyTopicRouteData proxyTopicRouteData = mock(ProxyTopicRouteData.class);\n        when(proxyTopicRouteData.buildTopicRouteData()).thenReturn(spyTopicRouteData);\n        when(messagingProcessor.getTopicRouteDataForProxy(any(ProxyContext.class), anyList(), any()))\n                .thenReturn(proxyTopicRouteData);\n\n        RemotingCommand response = getTopicRouteActivity.processRequest0(ctx, request, context);\n\n        assertNotNull(response);\n        assertEquals(ResponseCode.SUCCESS, response.getCode());\n\n        verify(spyTopicRouteData).encode();\n    }\n\n    private TopicRouteData prepareTopicRouteData() {\n        TopicRouteData result = new TopicRouteData();\n        result.setOrderTopicConf(\"orderTopicConf\");\n\n        List<QueueData> queueDatas = new ArrayList<>();\n        QueueData queueData = new QueueData();\n        queueData.setBrokerName(\"broker-a\");\n        queueData.setPerm(6);\n        queueData.setReadQueueNums(4);\n        queueData.setWriteQueueNums(4);\n        queueData.setTopicSysFlag(0);\n        queueDatas.add(queueData);\n        result.setQueueDatas(queueDatas);\n\n        List<BrokerData> brokerDatas = new ArrayList<>();\n        BrokerData brokerData = new BrokerData();\n        brokerData.setBrokerName(\"broker-a\");\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(0L, \"127.0.0.1:10911\");\n        brokerData.setBrokerAddrs(brokerAddrs);\n        brokerDatas.add(brokerData);\n        result.setBrokerDatas(brokerDatas);\n        return result;\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivityTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.activity;\n\nimport io.netty.channel.ChannelFuture;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.channel.ChannelPromise;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.broker.client.ConsumerGroupInfo;\nimport org.apache.rocketmq.common.sysflag.PullSysFlag;\nimport org.apache.rocketmq.proxy.config.InitConfigTest;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.service.channel.SimpleChannel;\nimport org.apache.rocketmq.proxy.service.channel.SimpleChannelHandlerContext;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.ArgumentCaptor;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class PullMessageActivityTest extends InitConfigTest {\n    PullMessageActivity pullMessageActivity;\n\n    @Mock\n    MessagingProcessor messagingProcessorMock;\n    @Mock\n    ConsumerGroupInfo consumerGroupInfoMock;\n\n    String topic = \"topic\";\n    String group = \"group\";\n    String brokerName = \"brokerName\";\n    String subString = \"sub\";\n    String type = \"type\";\n    @Spy\n    ChannelHandlerContext ctx = new SimpleChannelHandlerContext(new SimpleChannel(null, \"1\", \"2\")) {\n        @Override\n        public ChannelFuture writeAndFlush(Object msg, ChannelPromise promise) {\n            return null;\n        }\n    };\n\n    @Before\n    public void setup() throws Exception {\n        pullMessageActivity = new PullMessageActivity(null, messagingProcessorMock);\n    }\n\n    @Test\n    public void testPullMessageWithoutSub() throws Exception {\n        when(messagingProcessorMock.getConsumerGroupInfo(any(), eq(group)))\n            .thenReturn(consumerGroupInfoMock);\n        SubscriptionData subscriptionData = new SubscriptionData();\n        subscriptionData.setSubString(subString);\n        subscriptionData.setExpressionType(type);\n        when(consumerGroupInfoMock.findSubscriptionData(eq(topic)))\n            .thenReturn(subscriptionData);\n\n        PullMessageRequestHeader header = new PullMessageRequestHeader();\n        header.setTopic(topic);\n        header.setConsumerGroup(group);\n        header.setQueueId(0);\n        header.setQueueOffset(0L);\n        header.setMaxMsgNums(16);\n        header.setSysFlag(PullSysFlag.buildSysFlag(true, false, false, false));\n        header.setCommitOffset(0L);\n        header.setSuspendTimeoutMillis(1000L);\n        header.setSubVersion(0L);\n        header.setBrokerName(brokerName);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, header);\n        request.makeCustomHeaderToNet();\n        RemotingCommand expectResponse = RemotingCommand.createResponseCommand(ResponseCode.NO_MESSAGE, \"success\");\n        PullMessageRequestHeader newHeader = new PullMessageRequestHeader();\n        newHeader.setTopic(topic);\n        newHeader.setConsumerGroup(group);\n        newHeader.setQueueId(0);\n        newHeader.setQueueOffset(0L);\n        newHeader.setMaxMsgNums(16);\n        newHeader.setSysFlag(PullSysFlag.buildSysFlag(true, false, true, false));\n        newHeader.setCommitOffset(0L);\n        newHeader.setSuspendTimeoutMillis(1000L);\n        newHeader.setSubVersion(0L);\n        newHeader.setBrokerName(brokerName);\n        newHeader.setSubscription(subString);\n        newHeader.setExpressionType(type);\n        RemotingCommand matchRequest = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, newHeader);\n        matchRequest.setOpaque(request.getOpaque());\n        matchRequest.makeCustomHeaderToNet();\n\n        ArgumentCaptor<RemotingCommand> captor = ArgumentCaptor.forClass(RemotingCommand.class);\n        when(messagingProcessorMock.request(any(), eq(brokerName), captor.capture(), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(expectResponse));\n        RemotingCommand response = pullMessageActivity.processRequest0(ctx, request, null);\n        assertThat(captor.getValue().getExtFields()).isEqualTo(matchRequest.getExtFields());\n        assertThat(response).isNull();\n        verify(ctx, times(1)).writeAndFlush(eq(expectResponse));\n    }\n\n    @Test\n    public void testPullMessageWithSub() throws Exception {\n        when(messagingProcessorMock.getConsumerGroupInfo(any(), eq(group)))\n            .thenReturn(consumerGroupInfoMock);\n        SubscriptionData subscriptionData = new SubscriptionData();\n        subscriptionData.setSubString(subString);\n        subscriptionData.setExpressionType(type);\n        when(consumerGroupInfoMock.findSubscriptionData(eq(topic)))\n            .thenReturn(subscriptionData);\n\n        PullMessageRequestHeader header = new PullMessageRequestHeader();\n        header.setTopic(topic);\n        header.setConsumerGroup(group);\n        header.setQueueId(0);\n        header.setQueueOffset(0L);\n        header.setMaxMsgNums(16);\n        header.setSysFlag(PullSysFlag.buildSysFlag(true, true, false, false));\n        header.setCommitOffset(0L);\n        header.setSuspendTimeoutMillis(1000L);\n        header.setSubVersion(0L);\n        header.setBrokerName(brokerName);\n        header.setSubscription(subString);\n        header.setExpressionType(type);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, header);\n        request.makeCustomHeaderToNet();\n        RemotingCommand expectResponse = RemotingCommand.createResponseCommand(ResponseCode.NO_MESSAGE, \"success\");\n\n        ArgumentCaptor<RemotingCommand> captor = ArgumentCaptor.forClass(RemotingCommand.class);\n        when(messagingProcessorMock.request(any(), eq(brokerName), captor.capture(), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(expectResponse));\n        RemotingCommand response = pullMessageActivity.processRequest0(ctx, request, null);\n        assertThat(captor.getValue().getExtFields()).isEqualTo(request.getExtFields());\n        assertThat(response).isNull();\n        verify(ctx, times(1)).writeAndFlush(eq(expectResponse));\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/RecallMessageActivityTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.activity;\n\nimport io.netty.channel.ChannelFuture;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.channel.ChannelPromise;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\nimport org.apache.rocketmq.proxy.config.InitConfigTest;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.service.channel.SimpleChannel;\nimport org.apache.rocketmq.proxy.service.channel.SimpleChannelHandlerContext;\nimport org.apache.rocketmq.proxy.service.metadata.MetadataService;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.util.concurrent.CompletableFuture;\n\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class RecallMessageActivityTest extends InitConfigTest {\n    private static final String TOPIC = \"topic\";\n    private static final String GROUP = \"group\";\n    private static final String BROKER_NAME = \"brokerName\";\n\n    private RecallMessageActivity recallMessageActivity;\n    @Mock\n    private MessagingProcessor messagingProcessor;\n    @Mock\n    private MetadataService metadataService;\n\n    @Spy\n    private ChannelHandlerContext ctx = new SimpleChannelHandlerContext(new SimpleChannel(null, \"1\", \"2\")) {\n        @Override\n        public ChannelFuture writeAndFlush(Object msg, ChannelPromise promise) {\n            return null;\n        }\n    };\n\n    @Before\n    public void init() {\n        recallMessageActivity = new RecallMessageActivity(null, messagingProcessor);\n        when(messagingProcessor.getMetadataService()).thenReturn(metadataService);\n    }\n\n    @Test\n    public void testRecallMessage_notDelayMessage() {\n        when(metadataService.getTopicMessageType(any(), eq(TOPIC))).thenReturn(TopicMessageType.NORMAL);\n        ProxyException exception = Assert.assertThrows(ProxyException.class, () -> {\n            recallMessageActivity.processRequest0(ctx, mockRequest(), null);\n        });\n        Assert.assertEquals(ProxyExceptionCode.MESSAGE_PROPERTY_CONFLICT_WITH_TYPE, exception.getCode());\n    }\n\n    @Test\n    public void testRecallMessage_success() throws Exception {\n        when(metadataService.getTopicMessageType(any(), eq(TOPIC))).thenReturn(TopicMessageType.DELAY);\n        RemotingCommand request = mockRequest();\n        RemotingCommand expectResponse = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"\");\n        when(messagingProcessor.request(any(), eq(BROKER_NAME), eq(request), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(expectResponse));\n        RemotingCommand response = recallMessageActivity.processRequest0(ctx, request, null);\n        Assert.assertNull(response);\n        verify(ctx, times(1)).writeAndFlush(eq(expectResponse));\n    }\n\n    private RemotingCommand mockRequest() {\n        RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader();\n        requestHeader.setProducerGroup(GROUP);\n        requestHeader.setTopic(TOPIC);\n        requestHeader.setRecallHandle(\"handle\");\n        requestHeader.setBrokerName(BROKER_NAME);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RECALL_MESSAGE, requestHeader);\n        request.makeCustomHeaderToNet();\n        return request;\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivityTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.activity;\n\nimport io.netty.channel.ChannelFuture;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.channel.ChannelPromise;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.proxy.config.InitConfigTest;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.service.channel.SimpleChannel;\nimport org.apache.rocketmq.proxy.service.channel.SimpleChannelHandlerContext;\nimport org.apache.rocketmq.proxy.service.metadata.MetadataService;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class SendMessageActivityTest extends InitConfigTest {\n    SendMessageActivity sendMessageActivity;\n\n    @Mock\n    MessagingProcessor messagingProcessorMock;\n    @Mock\n    MetadataService metadataServiceMock;\n\n    String topic = \"topic\";\n    String producerGroup = \"group\";\n    String brokerName = \"brokerName\";\n    @Spy\n    ChannelHandlerContext ctx = new SimpleChannelHandlerContext(new SimpleChannel(null, \"1\", \"2\")) {\n        @Override\n        public ChannelFuture writeAndFlush(Object msg, ChannelPromise promise) {\n            return null;\n        }\n    };\n\n    @Before\n    public void setup() {\n        sendMessageActivity = new SendMessageActivity(null, messagingProcessorMock);\n        when(messagingProcessorMock.getMetadataService()).thenReturn(metadataServiceMock);\n    }\n\n    @Test\n    public void testSendMessage() throws Exception {\n        when(metadataServiceMock.getTopicMessageType(any(), eq(topic))).thenReturn(TopicMessageType.NORMAL);\n        Message message = new Message(topic, \"123\".getBytes());\n        message.putUserProperty(\"a\", \"b\");\n        SendMessageRequestHeader sendMessageRequestHeader = new SendMessageRequestHeader();\n        sendMessageRequestHeader.setTopic(topic);\n        sendMessageRequestHeader.setProducerGroup(producerGroup);\n        sendMessageRequestHeader.setDefaultTopic(\"\");\n        sendMessageRequestHeader.setDefaultTopicQueueNums(0);\n        sendMessageRequestHeader.setQueueId(0);\n        sendMessageRequestHeader.setSysFlag(0);\n        sendMessageRequestHeader.setBrokerName(brokerName);\n        sendMessageRequestHeader.setProperties(MessageDecoder.messageProperties2String(message.getProperties()));\n        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, sendMessageRequestHeader);\n        remotingCommand.setBody(message.getBody());\n        remotingCommand.makeCustomHeaderToNet();\n\n        RemotingCommand expectResponse = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"success\");\n        when(messagingProcessorMock.request(any(), eq(brokerName), eq(remotingCommand), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(expectResponse));\n        RemotingCommand response = sendMessageActivity.processRequest0(ctx, remotingCommand, null);\n        assertThat(response).isNull();\n        verify(ctx, times(1)).writeAndFlush(eq(expectResponse));\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.channel;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelId;\nimport java.util.HashSet;\nimport org.apache.commons.lang3.RandomStringUtils;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.remoting.RemotingProxyOutClient;\nimport org.apache.rocketmq.proxy.service.channel.SimpleChannel;\nimport org.apache.rocketmq.proxy.service.relay.ProxyRelayService;\nimport org.jetbrains.annotations.NotNull;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNotSame;\nimport static org.junit.Assert.assertSame;\nimport static org.junit.Assert.assertTrue;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class RemotingChannelManagerTest {\n    @Mock\n    private RemotingProxyOutClient remotingProxyOutClient;\n    @Mock\n    private ProxyRelayService proxyRelayService;\n\n    private final String remoteAddress = \"10.152.39.53:9768\";\n    private final String localAddress = \"11.193.0.1:1210\";\n    private RemotingChannelManager remotingChannelManager;\n    private final ProxyContext ctx = ProxyContext.createForInner(this.getClass());\n\n    @Before\n    public void before() {\n        this.remotingChannelManager = new RemotingChannelManager(this.remotingProxyOutClient, this.proxyRelayService);\n    }\n\n    @Test\n    public void testCreateChannel() {\n        String group = \"group\";\n        String clientId = RandomStringUtils.randomAlphabetic(10);\n\n        Channel producerChannel = createMockChannel();\n        RemotingChannel producerRemotingChannel = this.remotingChannelManager.createProducerChannel(ctx, producerChannel, group, clientId);\n        assertNotNull(producerRemotingChannel);\n        assertSame(producerRemotingChannel, this.remotingChannelManager.createProducerChannel(ctx, producerChannel, group, clientId));\n\n        Channel consumerChannel = createMockChannel();\n        RemotingChannel consumerRemotingChannel = this.remotingChannelManager.createConsumerChannel(ctx, consumerChannel, group, clientId, new HashSet<>());\n        assertSame(consumerRemotingChannel, this.remotingChannelManager.createConsumerChannel(ctx, consumerChannel, group, clientId, new HashSet<>()));\n        assertNotNull(consumerRemotingChannel);\n\n        assertNotSame(producerRemotingChannel, consumerRemotingChannel);\n    }\n\n    @Test\n    public void testRemoveProducerChannel() {\n        String group = \"group\";\n        String clientId = RandomStringUtils.randomAlphabetic(10);\n\n        {\n            Channel producerChannel = createMockChannel();\n            RemotingChannel producerRemotingChannel = this.remotingChannelManager.createProducerChannel(ctx, producerChannel, group, clientId);\n            assertSame(producerRemotingChannel, this.remotingChannelManager.removeProducerChannel(ctx, group, producerRemotingChannel));\n            assertTrue(this.remotingChannelManager.groupChannelMap.isEmpty());\n        }\n        {\n            Channel producerChannel = createMockChannel();\n            RemotingChannel producerRemotingChannel = this.remotingChannelManager.createProducerChannel(ctx, producerChannel, group, clientId);\n            assertSame(producerRemotingChannel, this.remotingChannelManager.removeProducerChannel(ctx, group, producerChannel));\n            assertTrue(this.remotingChannelManager.groupChannelMap.isEmpty());\n        }\n    }\n\n    @Test\n    public void testRemoveConsumerChannel() {\n        String group = \"group\";\n        String clientId = RandomStringUtils.randomAlphabetic(10);\n\n        {\n            Channel consumerChannel = createMockChannel();\n            RemotingChannel consumerRemotingChannel = this.remotingChannelManager.createConsumerChannel(ctx, consumerChannel, group, clientId, new HashSet<>());\n            assertSame(consumerRemotingChannel, this.remotingChannelManager.removeConsumerChannel(ctx, group, consumerRemotingChannel));\n            assertTrue(this.remotingChannelManager.groupChannelMap.isEmpty());\n        }\n        {\n            Channel consumerChannel = createMockChannel();\n            RemotingChannel consumerRemotingChannel = this.remotingChannelManager.createConsumerChannel(ctx, consumerChannel, group, clientId, new HashSet<>());\n            assertSame(consumerRemotingChannel, this.remotingChannelManager.removeConsumerChannel(ctx, group, consumerChannel));\n            assertTrue(this.remotingChannelManager.groupChannelMap.isEmpty());\n        }\n    }\n\n    @Test\n    public void testRemoveChannel() {\n        String consumerGroup = \"consumerGroup\";\n        String producerGroup = \"producerGroup\";\n        String clientId = RandomStringUtils.randomAlphabetic(10);\n\n        Channel consumerChannel = createMockChannel();\n        RemotingChannel consumerRemotingChannel = this.remotingChannelManager.createConsumerChannel(ctx, consumerChannel, consumerGroup, clientId, new HashSet<>());\n        Channel producerChannel = createMockChannel();\n        RemotingChannel producerRemotingChannel = this.remotingChannelManager.createProducerChannel(ctx, producerChannel, producerGroup, clientId);\n\n        assertSame(consumerRemotingChannel, this.remotingChannelManager.removeChannel(consumerChannel).stream().findFirst().get());\n        assertSame(producerRemotingChannel, this.remotingChannelManager.removeChannel(producerChannel).stream().findFirst().get());\n\n        assertTrue(this.remotingChannelManager.groupChannelMap.isEmpty());\n    }\n\n    private Channel createMockChannel() {\n        return new MockChannel(RandomStringUtils.randomAlphabetic(10));\n    }\n\n    private class MockChannel extends SimpleChannel {\n\n        public MockChannel(String channelId) {\n            super(null, new MockChannelId(channelId), RemotingChannelManagerTest.this.remoteAddress, RemotingChannelManagerTest.this.localAddress);\n        }\n    }\n\n    private static class MockChannelId implements ChannelId {\n\n        private final String channelId;\n\n        public MockChannelId(String channelId) {\n            this.channelId = channelId;\n        }\n\n        @Override\n        public String asShortText() {\n            return channelId;\n        }\n\n        @Override\n        public String asLongText() {\n            return channelId;\n        }\n\n        @Override\n        public int compareTo(@NotNull ChannelId o) {\n            return this.channelId.compareTo(o.asLongText());\n        }\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.channel;\n\nimport io.netty.channel.Channel;\nimport java.util.HashSet;\nimport java.util.Set;\nimport org.apache.commons.lang3.RandomStringUtils;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.proxy.config.InitConfigTest;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcClientChannel;\nimport org.apache.rocketmq.proxy.processor.channel.ChannelProtocolType;\nimport org.apache.rocketmq.proxy.processor.channel.RemoteChannel;\nimport org.apache.rocketmq.proxy.remoting.RemotingProxyOutClient;\nimport org.apache.rocketmq.proxy.service.relay.ProxyRelayService;\nimport org.apache.rocketmq.remoting.protocol.filter.FilterAPI;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNull;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class RemotingChannelTest extends InitConfigTest {\n    @Mock\n    private RemotingProxyOutClient remotingProxyOutClient;\n    @Mock\n    private ProxyRelayService proxyRelayService;\n    @Mock\n    private Channel parent;\n\n    private String clientId;\n    private Set<SubscriptionData> subscriptionData;\n    private RemotingChannel remotingChannel;\n\n    private final String remoteAddress = \"10.152.39.53:9768\";\n    private final String localAddress = \"11.193.0.1:1210\";\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        this.clientId = RandomStringUtils.randomAlphabetic(10);\n        when(parent.remoteAddress()).thenReturn(NetworkUtil.string2SocketAddress(remoteAddress));\n        when(parent.localAddress()).thenReturn(NetworkUtil.string2SocketAddress(localAddress));\n        this.subscriptionData = new HashSet<>();\n        this.subscriptionData.add(FilterAPI.buildSubscriptionData(\"topic\", \"subTag\"));\n        this.remotingChannel = new RemotingChannel(remotingProxyOutClient, proxyRelayService,\n            parent, clientId, subscriptionData);\n    }\n\n    @Test\n    public void testChannelExtendAttributeParse() {\n        RemoteChannel remoteChannel = this.remotingChannel.toRemoteChannel();\n        assertEquals(ChannelProtocolType.REMOTING, remoteChannel.getType());\n        assertEquals(subscriptionData, RemotingChannel.parseChannelExtendAttribute(remoteChannel));\n        assertEquals(subscriptionData, RemotingChannel.parseChannelExtendAttribute(this.remotingChannel));\n        assertNull(RemotingChannel.parseChannelExtendAttribute(mock(GrpcClientChannel.class)));\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/HAProxyMessageForwarderTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\npackage org.apache.rocketmq.proxy.remoting.protocol.http2proxy;\n\nimport io.netty.channel.Channel;\nimport io.netty.handler.codec.haproxy.HAProxyTLV;\nimport org.apache.commons.codec.DecoderException;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class HAProxyMessageForwarderTest {\n\n    private HAProxyMessageForwarder haProxyMessageForwarder;\n\n    @Mock\n    private Channel outboundChannel;\n\n    @Before\n    public void setUp() throws Exception {\n        haProxyMessageForwarder = new HAProxyMessageForwarder(outboundChannel);\n    }\n\n    @Test\n    public void buildHAProxyTLV() throws DecoderException {\n        HAProxyTLV haProxyTLV = haProxyMessageForwarder.buildHAProxyTLV(\"proxy_protocol_tlv_0xe1\", \"xxxx\");\n        assert haProxyTLV != null;\n        assert haProxyTLV.typeByteValue() == (byte) 0xe1;\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProtocolProxyHandlerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.remoting.protocol.http2proxy;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelPipeline;\nimport io.netty.handler.codec.haproxy.HAProxyMessageEncoder;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class Http2ProtocolProxyHandlerTest {\n\n    private Http2ProtocolProxyHandler http2ProtocolProxyHandler;\n    @Mock\n    private Channel inboundChannel;\n    @Mock\n    private ChannelPipeline inboundPipeline;\n    @Mock\n    private Channel outboundChannel;\n    @Mock\n    private ChannelPipeline outboundPipeline;\n\n    @Before\n    public void setUp() throws Exception {\n        http2ProtocolProxyHandler = new Http2ProtocolProxyHandler();\n    }\n\n    @Test\n    public void configPipeline() {\n        when(inboundChannel.pipeline()).thenReturn(inboundPipeline);\n        when(inboundPipeline.addLast(any(HAProxyMessageForwarder.class))).thenReturn(inboundPipeline);\n        when(outboundChannel.pipeline()).thenReturn(outboundPipeline);\n        when(outboundPipeline.addFirst(any(HAProxyMessageEncoder.class))).thenReturn(outboundPipeline);\n        http2ProtocolProxyHandler.configPipeline(inboundChannel, outboundChannel);\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service;\n\nimport java.util.HashMap;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.proxy.config.InitConfigTest;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIExt;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;\nimport org.apache.rocketmq.proxy.service.route.MessageQueueView;\nimport org.apache.rocketmq.proxy.service.route.TopicRouteService;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.assertj.core.util.Lists;\nimport org.junit.Before;\nimport org.junit.Ignore;\nimport org.junit.runner.RunWith;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@Ignore\n@RunWith(MockitoJUnitRunner.Silent.class)\npublic class BaseServiceTest extends InitConfigTest {\n\n    protected TopicRouteService topicRouteService;\n    protected MQClientAPIFactory mqClientAPIFactory;\n    protected MQClientAPIExt mqClientAPIExt;\n\n    protected static final String ERR_TOPIC = \"errTopic\";\n    protected static final String TOPIC = \"topic\";\n    protected static final String GROUP = \"group\";\n    protected static final String BROKER_NAME = \"broker\";\n    protected static final String CLUSTER_NAME = \"cluster\";\n    protected static final String BROKER_ADDR = \"127.0.0.1:10911\";\n\n    protected final TopicRouteData topicRouteData = new TopicRouteData();\n    protected final QueueData queueData = new QueueData();\n    protected final BrokerData brokerData = new BrokerData();\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n\n        topicRouteService = mock(TopicRouteService.class);\n        mqClientAPIFactory = mock(MQClientAPIFactory.class);\n        mqClientAPIExt = mock(MQClientAPIExt.class);\n        when(mqClientAPIFactory.getClient()).thenReturn(mqClientAPIExt);\n\n        queueData.setBrokerName(BROKER_NAME);\n        topicRouteData.setQueueDatas(Lists.newArrayList(queueData));\n        brokerData.setCluster(CLUSTER_NAME);\n        brokerData.setBrokerName(BROKER_NAME);\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(MixAll.MASTER_ID, BROKER_ADDR);\n        brokerData.setBrokerAddrs(brokerAddrs);\n        topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData));\n\n        when(this.topicRouteService.getAllMessageQueueView(any(), eq(ERR_TOPIC))).thenThrow(new MQClientException(ResponseCode.TOPIC_NOT_EXIST, \"\"));\n        when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData, null));\n        when(this.topicRouteService.getAllMessageQueueView(any(), eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, topicRouteData, null));\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.admin;\n\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Set;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIExt;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.ArgumentCaptor;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.doNothing;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class DefaultAdminServiceTest {\n    @Mock\n    private MQClientAPIFactory mqClientAPIFactory;\n    @Mock\n    private MQClientAPIExt mqClientAPIExt;\n\n    private DefaultAdminService defaultAdminService;\n\n    @Before\n    public void before() {\n        when(mqClientAPIFactory.getClient()).thenReturn(mqClientAPIExt);\n        defaultAdminService = new DefaultAdminService(mqClientAPIFactory);\n    }\n\n    @Test\n    public void testCreateTopic() throws Exception {\n        when(mqClientAPIExt.getTopicRouteInfoFromNameServer(eq(\"createTopic\"), anyLong()))\n            .thenThrow(new MQClientException(ResponseCode.TOPIC_NOT_EXIST, \"\"))\n            .thenReturn(createTopicRouteData(1));\n        when(mqClientAPIExt.getTopicRouteInfoFromNameServer(eq(\"sampleTopic\"), anyLong()))\n            .thenReturn(createTopicRouteData(2));\n\n        ArgumentCaptor<String> addrArgumentCaptor = ArgumentCaptor.forClass(String.class);\n        ArgumentCaptor<TopicConfig> topicConfigArgumentCaptor = ArgumentCaptor.forClass(TopicConfig.class);\n        doNothing().when(mqClientAPIExt).createTopic(addrArgumentCaptor.capture(), anyString(), topicConfigArgumentCaptor.capture(), anyLong());\n\n        assertTrue(defaultAdminService.createTopicOnTopicBrokerIfNotExist(\n            \"createTopic\",\n            \"sampleTopic\",\n            7,\n            8,\n            true,\n            1\n        ));\n\n        assertEquals(2, addrArgumentCaptor.getAllValues().size());\n        Set<String> createAddr = new HashSet<>(addrArgumentCaptor.getAllValues());\n        assertTrue(createAddr.contains(\"127.0.0.1:10911\"));\n        assertTrue(createAddr.contains(\"127.0.0.2:10911\"));\n        assertEquals(\"createTopic\", topicConfigArgumentCaptor.getValue().getTopicName());\n        assertEquals(7, topicConfigArgumentCaptor.getValue().getWriteQueueNums());\n        assertEquals(8, topicConfigArgumentCaptor.getValue().getReadQueueNums());\n    }\n\n    private TopicRouteData createTopicRouteData(int brokerNum) {\n        TopicRouteData topicRouteData = new TopicRouteData();\n        for (int i = 0; i < brokerNum; i++) {\n            BrokerData brokerData = new BrokerData();\n            HashMap<Long, String> addrMap = new HashMap<>();\n            addrMap.put(0L, \"127.0.0.\" + (i + 1) + \":10911\");\n            brokerData.setBrokerAddrs(addrMap);\n            brokerData.setBrokerName(\"broker-\" + i);\n            brokerData.setCluster(\"cluster\");\n            topicRouteData.getBrokerDatas().add(brokerData);\n        }\n        return topicRouteData;\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/service/cert/TlsCertificateManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.cert;\n\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.remoting.netty.TlsSystemConfig;\nimport org.apache.rocketmq.srvutil.FileWatchService;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Rule;\nimport org.junit.Test;\nimport org.junit.rules.TemporaryFolder;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.io.File;\nimport java.io.FileWriter;\nimport java.lang.reflect.Constructor;\nimport java.lang.reflect.Field;\nimport java.lang.reflect.Method;\nimport java.util.List;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\nimport static org.junit.jupiter.api.Assertions.assertFalse;\nimport static org.junit.jupiter.api.Assertions.assertNotNull;\nimport static org.junit.jupiter.api.Assertions.assertTrue;\nimport static org.mockito.Mockito.doThrow;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.never;\nimport static org.mockito.Mockito.reset;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class TlsCertificateManagerTest {\n\n    @Rule\n    public TemporaryFolder tempDir = new TemporaryFolder();\n\n    private TlsCertificateManager manager;\n\n    @Mock\n    private TlsCertificateManager.TlsContextReloadListener listener1;\n\n    @Mock\n    private TlsCertificateManager.TlsContextReloadListener listener2;\n\n    private File certFile;\n    private File keyFile;\n    private FileWatchService.Listener fileWatchListener;\n    private Field configField;\n    private ProxyConfig originalConfig;\n\n    @Before\n    public void setUp() throws Exception {\n        ConfigurationManager.initEnv();\n        ConfigurationManager.initConfig();\n        // Create temporary certificate and key files\n        certFile = tempDir.newFile(\"server.crt\");\n        keyFile = tempDir.newFile(\"server.key\");\n        try (FileWriter certWriter = new FileWriter(certFile);\n             FileWriter keyWriter = new FileWriter(keyFile)) {\n            certWriter.write(\"test certificate content\");\n            keyWriter.write(\"test key content\");\n        }\n\n        // Set TlsSystemConfig paths\n        TlsSystemConfig.tlsServerCertPath = certFile.getAbsolutePath();\n        TlsSystemConfig.tlsServerKeyPath = keyFile.getAbsolutePath();\n\n        // Create the TlsCertificateManager\n        manager = new TlsCertificateManager();\n\n        // Extract the file watch listener using reflection\n        fileWatchListener = extractFileWatchListener(manager);\n    }\n\n    @After\n    public void tearDown() throws Exception {\n        // Restore the original config\n        if (configField != null && originalConfig != null) {\n            configField.set(null, originalConfig);\n        }\n    }\n\n    private FileWatchService.Listener extractFileWatchListener(TlsCertificateManager manager) throws Exception {\n        Field fileWatchServiceField = TlsCertificateManager.class.getDeclaredField(\"fileWatchService\");\n        fileWatchServiceField.setAccessible(true);\n        FileWatchService fileWatchService = (FileWatchService) fileWatchServiceField.get(manager);\n\n        Field listenerField = FileWatchService.class.getDeclaredField(\"listener\");\n        listenerField.setAccessible(true);\n        return (FileWatchService.Listener) listenerField.get(fileWatchService);\n    }\n\n    @Test\n    public void testConstructor() {\n        // The constructor should initialize the FileWatchService with the correct paths\n        assertNotNull(manager);\n    }\n\n    @Test\n    public void testStartAndShutdown() throws Exception {\n        TlsCertificateManager managerSpy = spy(manager);\n\n        Field watchServiceField = TlsCertificateManager.class.getDeclaredField(\"fileWatchService\");\n        watchServiceField.setAccessible(true);\n        FileWatchService watchService = (FileWatchService) watchServiceField.get(managerSpy);\n        FileWatchService watchServiceSpy = spy(watchService);\n        watchServiceField.set(managerSpy, watchServiceSpy);\n\n        managerSpy.start();\n        verify(watchServiceSpy).start();\n\n        managerSpy.shutdown();\n        verify(watchServiceSpy).shutdown();\n    }\n\n    @Test\n    public void testRegisterAndUnregisterListener() {\n        manager.registerReloadListener(listener1);\n\n        List<TlsCertificateManager.TlsContextReloadListener> listeners = manager.getReloadListeners();\n        assertEquals(1, listeners.size());\n        assertTrue(listeners.contains(listener1));\n\n        manager.registerReloadListener(listener2);\n        assertEquals(2, listeners.size());\n        assertTrue(listeners.contains(listener2));\n\n        manager.unregisterReloadListener(listener1);\n        assertEquals(1, listeners.size());\n        assertFalse(listeners.contains(listener1));\n        assertTrue(listeners.contains(listener2));\n\n        manager.registerReloadListener(null);\n        assertEquals(1, listeners.size()); // Should remain unchanged\n\n        manager.unregisterReloadListener(null);\n        assertEquals(1, listeners.size()); // Should remain unchanged\n    }\n\n    @Test\n    public void testFileChangeNotification_CertOnly() throws Exception {\n        manager.registerReloadListener(listener1);\n\n        fileWatchListener.onChanged(certFile.getAbsolutePath());\n\n        verify(listener1, never()).onTlsContextReload();\n    }\n\n    @Test\n    public void testFileChangeNotification_KeyOnly() throws Exception {\n        manager.registerReloadListener(listener1);\n\n        fileWatchListener.onChanged(keyFile.getAbsolutePath());\n\n        verify(listener1, never()).onTlsContextReload();\n    }\n\n    @Test\n    public void testFileChangeNotification_BothFiles() throws Exception {\n        manager.registerReloadListener(listener1);\n\n        fileWatchListener.onChanged(certFile.getAbsolutePath());\n        fileWatchListener.onChanged(keyFile.getAbsolutePath());\n\n        verify(listener1, times(1)).onTlsContextReload();\n    }\n\n    @Test\n    public void testFileChangeNotification_MultipleListeners() throws Exception {\n        manager.registerReloadListener(listener1);\n        manager.registerReloadListener(listener2);\n\n        fileWatchListener.onChanged(certFile.getAbsolutePath());\n        fileWatchListener.onChanged(keyFile.getAbsolutePath());\n\n        verify(listener1, times(1)).onTlsContextReload();\n        verify(listener2, times(1)).onTlsContextReload();\n    }\n\n    @Test\n    public void testFileChangeNotification_BothFilesReverseOrder() throws Exception {\n        manager.registerReloadListener(listener1);\n\n        fileWatchListener.onChanged(keyFile.getAbsolutePath());\n        fileWatchListener.onChanged(certFile.getAbsolutePath());\n\n        verify(listener1, times(1)).onTlsContextReload();\n    }\n\n    @Test\n    public void testFileChangeNotification_RepeatedChanges() throws Exception {\n        manager.registerReloadListener(listener1);\n\n        fileWatchListener.onChanged(certFile.getAbsolutePath());\n        fileWatchListener.onChanged(keyFile.getAbsolutePath());\n\n        verify(listener1, times(1)).onTlsContextReload();\n\n        fileWatchListener.onChanged(certFile.getAbsolutePath());\n        fileWatchListener.onChanged(keyFile.getAbsolutePath());\n\n        verify(listener1, times(2)).onTlsContextReload();\n    }\n\n    @Test\n    public void testFileChangeNotification_UnknownFile() throws Exception {\n        manager.registerReloadListener(listener1);\n\n        fileWatchListener.onChanged(\"/unknown/file/path\");\n\n        verify(listener1, never()).onTlsContextReload();\n    }\n\n    @Test\n    public void testFileChangeNotification_ListenerThrowsException() throws Exception {\n        TlsCertificateManager.TlsContextReloadListener exceptionListener = mock(TlsCertificateManager.TlsContextReloadListener.class);\n        doThrow(new RuntimeException(\"Test exception\")).when(exceptionListener).onTlsContextReload();\n\n        manager.registerReloadListener(exceptionListener);\n        manager.registerReloadListener(listener1);\n\n        fileWatchListener.onChanged(certFile.getAbsolutePath());\n        fileWatchListener.onChanged(keyFile.getAbsolutePath());\n\n        verify(exceptionListener, times(1)).onTlsContextReload();\n        verify(listener1, times(1)).onTlsContextReload();\n    }\n\n    @Test\n    public void testInnerCertKeyFileWatchListener() throws Exception {\n        Class<?> innerClass = null;\n        for (Class<?> clazz : TlsCertificateManager.class.getDeclaredClasses()) {\n            if (clazz.getSimpleName().equals(\"CertKeyFileWatchListener\")) {\n                innerClass = clazz;\n                break;\n            }\n        }\n\n        assertNotNull(innerClass, \"CertKeyFileWatchListener class not found\");\n\n        Constructor<?> constructor = innerClass.getDeclaredConstructor(TlsCertificateManager.class);\n        constructor.setAccessible(true);\n        Object innerListener = constructor.newInstance(manager);\n\n        manager.registerReloadListener(listener1);\n\n        Method onChangedMethod = innerClass.getDeclaredMethod(\"onChanged\", String.class);\n        onChangedMethod.setAccessible(true);\n\n        onChangedMethod.invoke(innerListener, certFile.getAbsolutePath());\n        verify(listener1, never()).onTlsContextReload();\n\n        onChangedMethod.invoke(innerListener, keyFile.getAbsolutePath());\n        verify(listener1, times(1)).onTlsContextReload();\n\n        reset(listener1);\n\n        onChangedMethod.invoke(innerListener, certFile.getAbsolutePath());\n        verify(listener1, never()).onTlsContextReload();\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/service/lite/LiteSubscriptionServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.lite;\n\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIExt;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;\nimport org.apache.rocketmq.common.lite.LiteSubscriptionDTO;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;\nimport org.apache.rocketmq.proxy.service.route.MessageQueueSelector;\nimport org.apache.rocketmq.proxy.service.route.MessageQueueView;\nimport org.apache.rocketmq.proxy.service.route.TopicRouteService;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.junit.jupiter.api.Assertions.assertDoesNotThrow;\nimport static org.junit.jupiter.api.Assertions.assertTrue;\nimport static org.mockito.Mockito.any;\nimport static org.mockito.Mockito.anyLong;\nimport static org.mockito.Mockito.anyString;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.never;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class LiteSubscriptionServiceTest {\n\n    @Mock\n    private TopicRouteService topicRouteService;\n\n    @Mock\n    private MQClientAPIFactory mqClientAPIFactory;\n\n    @Mock\n    private MQClientAPIExt mqClientAPIExt;\n\n    private LiteSubscriptionService liteSubscriptionService;\n\n    @Before\n    public void setUp() {\n        liteSubscriptionService = new LiteSubscriptionService(topicRouteService, mqClientAPIFactory);\n    }\n\n    /**\n     * Test successful case: all brokers sync successfully\n     */\n    @Test\n    public void testSyncLiteSubscription_Success() throws Exception {\n        ProxyContext ctx = ProxyContext.create();\n        LiteSubscriptionDTO liteSubscriptionDTO = new LiteSubscriptionDTO();\n        liteSubscriptionDTO.setTopic(\"testTopic\");\n        long timeoutMillis = 3000L;\n\n        MessageQueueView messageQueueView = mock(MessageQueueView.class);\n        MessageQueueSelector readSelector = mock(MessageQueueSelector.class);\n        when(messageQueueView.getReadSelector()).thenReturn(readSelector);\n\n        AddressableMessageQueue queue1 = mock(AddressableMessageQueue.class);\n        AddressableMessageQueue queue2 = mock(AddressableMessageQueue.class);\n        when(queue1.getBrokerAddr()).thenReturn(\"broker1:10911\");\n        when(queue2.getBrokerAddr()).thenReturn(\"broker2:10911\");\n        List<AddressableMessageQueue> readQueues = Arrays.asList(queue1, queue2);\n        when(readSelector.getBrokerActingQueues()).thenReturn(readQueues);\n\n        when(topicRouteService.getAllMessageQueueView(ctx, \"testTopic\")).thenReturn(messageQueueView);\n\n        when(mqClientAPIFactory.getClient()).thenReturn(mqClientAPIExt);\n\n        when(mqClientAPIExt.syncLiteSubscriptionAsync(anyString(), any(LiteSubscriptionDTO.class), anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(null))\n            .thenReturn(CompletableFuture.completedFuture(null));\n\n        CompletableFuture<Void> future = liteSubscriptionService.syncLiteSubscription(ctx, liteSubscriptionDTO, timeoutMillis);\n\n        assertDoesNotThrow(() -> future.get());\n        verify(mqClientAPIExt, times(2)).syncLiteSubscriptionAsync(anyString(), any(LiteSubscriptionDTO.class), anyLong());\n    }\n\n    /**\n     * Test exception case: topicRouteService throws exception\n     */\n    @Test\n    public void testSyncLiteSubscription_TopicRouteServiceException() throws Exception {\n        ProxyContext ctx = ProxyContext.create();\n        LiteSubscriptionDTO liteSubscriptionDTO = new LiteSubscriptionDTO();\n        liteSubscriptionDTO.setTopic(\"testTopic\");\n        long timeoutMillis = 3000L;\n\n        when(topicRouteService.getAllMessageQueueView(ctx, \"testTopic\"))\n            .thenThrow(new RuntimeException(\"Topic route error\"));\n\n        CompletableFuture<Void> future = liteSubscriptionService.syncLiteSubscription(ctx, liteSubscriptionDTO, timeoutMillis);\n\n        assertTrue(future.isCompletedExceptionally());\n        verify(mqClientAPIFactory, never()).getClient();\n    }\n\n    /**\n     * Test exception case: some broker sync fails\n     */\n    @Test\n    public void testSyncLiteSubscription_SomeBrokerFail() throws Exception {\n        ProxyContext ctx = ProxyContext.create();\n        LiteSubscriptionDTO liteSubscriptionDTO = new LiteSubscriptionDTO();\n        liteSubscriptionDTO.setTopic(\"testTopic\");\n        long timeoutMillis = 3000L;\n\n        MessageQueueView messageQueueView = mock(MessageQueueView.class);\n        MessageQueueSelector readSelector = mock(MessageQueueSelector.class);\n        when(messageQueueView.getReadSelector()).thenReturn(readSelector);\n\n        AddressableMessageQueue queue1 = mock(AddressableMessageQueue.class);\n        AddressableMessageQueue queue2 = mock(AddressableMessageQueue.class);\n        when(queue1.getBrokerAddr()).thenReturn(\"broker1:10911\");\n        when(queue2.getBrokerAddr()).thenReturn(\"broker2:10911\");\n        List<AddressableMessageQueue> readQueues = Arrays.asList(queue1, queue2);\n        when(readSelector.getBrokerActingQueues()).thenReturn(readQueues);\n\n        when(topicRouteService.getAllMessageQueueView(ctx, \"testTopic\")).thenReturn(messageQueueView);\n\n        when(mqClientAPIFactory.getClient()).thenReturn(mqClientAPIExt);\n\n        CompletableFuture<Void> failedFuture = new CompletableFuture<>();\n        failedFuture.completeExceptionally(new RuntimeException(\"Broker sync failed\"));\n\n        when(mqClientAPIExt.syncLiteSubscriptionAsync(anyString(), any(LiteSubscriptionDTO.class), anyLong()))\n            .thenReturn(failedFuture);\n\n        CompletableFuture<Void> future = liteSubscriptionService.syncLiteSubscription(ctx, liteSubscriptionDTO, timeoutMillis);\n\n        assertTrue(future.isCompletedExceptionally());\n        verify(mqClientAPIExt, times(2)).syncLiteSubscriptionAsync(anyString(), any(LiteSubscriptionDTO.class), anyLong());\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/service/message/ClusterMessageServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.proxy.service.message;\n\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;\nimport org.apache.rocketmq.proxy.service.route.TopicRouteService;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\nimport static org.junit.Assert.fail;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\npublic class ClusterMessageServiceTest {\n\n    private TopicRouteService topicRouteService;\n    private ClusterMessageService clusterMessageService;\n\n    @Before\n    public void before() {\n        this.topicRouteService = mock(TopicRouteService.class);\n        MQClientAPIFactory mqClientAPIFactory = mock(MQClientAPIFactory.class);\n        this.clusterMessageService = new ClusterMessageService(this.topicRouteService, mqClientAPIFactory);\n    }\n\n    @Test\n    public void testAckMessageByInvalidBrokerNameHandle() throws Exception {\n        when(topicRouteService.getBrokerAddr(any(), anyString())).thenThrow(new MQClientException(ResponseCode.TOPIC_NOT_EXIST, \"\"));\n        try {\n            this.clusterMessageService.ackMessage(\n                ProxyContext.create(),\n                ReceiptHandle.builder()\n                    .startOffset(0L)\n                    .retrieveTime(System.currentTimeMillis())\n                    .invisibleTime(3000)\n                    .reviveQueueId(1)\n                    .topicType(ReceiptHandle.NORMAL_TOPIC)\n                    .brokerName(\"notExistBroker\")\n                    .queueId(0)\n                    .offset(123)\n                    .commitLogOffset(0L)\n                    .build(),\n                MessageClientIDSetter.createUniqID(),\n                new AckMessageRequestHeader(),\n                3000);\n            fail();\n        } catch (Exception e) {\n            assertTrue(e instanceof ProxyException);\n            ProxyException proxyException = (ProxyException) e;\n            assertEquals(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, proxyException.getCode());\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/service/message/LocalMessageServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.message;\n\nimport io.netty.channel.Channel;\nimport java.net.InetSocketAddress;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.CompletionException;\nimport java.util.concurrent.ExecutionException;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.processor.AckMessageProcessor;\nimport org.apache.rocketmq.broker.processor.ChangeInvisibleTimeProcessor;\nimport org.apache.rocketmq.broker.processor.EndTransactionProcessor;\nimport org.apache.rocketmq.broker.processor.PopMessageProcessor;\nimport org.apache.rocketmq.broker.processor.RecallMessageProcessor;\nimport org.apache.rocketmq.broker.processor.SendMessageProcessor;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.AckStatus;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PopStatus;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageBatch;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.proxy.common.ContextVariable;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.InitConfigTest;\nimport org.apache.rocketmq.proxy.service.channel.ChannelManager;\nimport org.apache.rocketmq.proxy.service.channel.SimpleChannelHandlerContext;\nimport org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.assertj.core.api.Assertions.catchThrowableOfType;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class LocalMessageServiceTest extends InitConfigTest {\n    private LocalMessageService localMessageService;\n    @Mock\n    private SendMessageProcessor sendMessageProcessorMock;\n    @Mock\n    private EndTransactionProcessor endTransactionProcessorMock;\n    @Mock\n    private PopMessageProcessor popMessageProcessorMock;\n    @Mock\n    private ChangeInvisibleTimeProcessor changeInvisibleTimeProcessorMock;\n    @Mock\n    private AckMessageProcessor ackMessageProcessorMock;\n    @Mock\n    private RecallMessageProcessor recallMessageProcessorMock;\n    @Mock\n    private BrokerController brokerControllerMock;\n\n    private ProxyContext proxyContext;\n\n    private ChannelManager channelManager;\n\n    private String topic = \"topic\";\n\n    private String brokerName = \"brokerName\";\n\n    private int queueId = 0;\n\n    private long queueOffset = 0L;\n\n    private String transactionId = \"transactionId\";\n\n    private String offsetMessageId = \"offsetMessageId\";\n\n    @Before\n    public void setUp() throws Throwable {\n        super.before();\n        ConfigurationManager.getProxyConfig().setNamesrvAddr(\"1.1.1.1\");\n        channelManager = new ChannelManager();\n        Mockito.when(brokerControllerMock.getSendMessageProcessor()).thenReturn(sendMessageProcessorMock);\n        Mockito.when(brokerControllerMock.getPopMessageProcessor()).thenReturn(popMessageProcessorMock);\n        Mockito.when(brokerControllerMock.getChangeInvisibleTimeProcessor()).thenReturn(changeInvisibleTimeProcessorMock);\n        Mockito.when(brokerControllerMock.getAckMessageProcessor()).thenReturn(ackMessageProcessorMock);\n        Mockito.when(brokerControllerMock.getEndTransactionProcessor()).thenReturn(endTransactionProcessorMock);\n        Mockito.when(brokerControllerMock.getRecallMessageProcessor()).thenReturn(recallMessageProcessorMock);\n        Mockito.when(brokerControllerMock.getBrokerConfig()).thenReturn(new BrokerConfig());\n        localMessageService = new LocalMessageService(brokerControllerMock, channelManager, null);\n        proxyContext = ProxyContext.create().withVal(ContextVariable.REMOTE_ADDRESS, \"0.0.0.1\")\n            .withVal(ContextVariable.LOCAL_ADDRESS, \"0.0.0.2\");\n    }\n\n    @Test\n    public void testSendMessageWriteAndFlush() throws Exception {\n        Message message = new Message(topic, \"body\".getBytes(StandardCharsets.UTF_8));\n        MessageClientIDSetter.setUniqID(message);\n        List<Message> messagesList = Collections.singletonList(message);\n        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setQueueId(queueId);\n        Mockito.when(sendMessageProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class), Mockito.argThat(argument -> {\n            boolean first = argument.getCode() == RequestCode.SEND_MESSAGE;\n            boolean second = Arrays.equals(argument.getBody(), message.getBody());\n            return first & second;\n        }))).thenAnswer(invocation -> {\n            SimpleChannelHandlerContext simpleChannelHandlerContext = invocation.getArgument(0);\n            RemotingCommand request = invocation.getArgument(1);\n            RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);\n            response.setOpaque(request.getOpaque());\n            response.setCode(ResponseCode.SUCCESS);\n            response.setBody(message.getBody());\n            SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader();\n            sendMessageResponseHeader.setQueueId(queueId);\n            sendMessageResponseHeader.setQueueOffset(queueOffset);\n            sendMessageResponseHeader.setMsgId(offsetMessageId);\n            sendMessageResponseHeader.setTransactionId(transactionId);\n            simpleChannelHandlerContext.writeAndFlush(response);\n            return null;\n        });\n\n        CompletableFuture<List<SendResult>> future = localMessageService.sendMessage(proxyContext, null, messagesList, requestHeader, 1000L);\n        SendResult sendResult = future.get().get(0);\n        assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n        assertThat(sendResult.getMsgId()).isEqualTo(MessageClientIDSetter.getUniqID(message));\n        assertThat(sendResult.getMessageQueue())\n            .isEqualTo(new MessageQueue(topic, brokerControllerMock.getBrokerConfig().getBrokerName(), queueId));\n        assertThat(sendResult.getQueueOffset()).isEqualTo(queueOffset);\n        assertThat(sendResult.getTransactionId()).isEqualTo(transactionId);\n        assertThat(sendResult.getOffsetMsgId()).isEqualTo(offsetMessageId);\n    }\n\n    @Test\n    public void testSendBatchMessageWriteAndFlush() throws Exception {\n        Message message1 = new Message(topic, \"body1\".getBytes(StandardCharsets.UTF_8));\n        Message message2 = new Message(topic, \"body2\".getBytes(StandardCharsets.UTF_8));\n        MessageClientIDSetter.setUniqID(message1);\n        MessageClientIDSetter.setUniqID(message2);\n        List<Message> messagesList = Arrays.asList(message1, message2);\n        MessageBatch msgBatch = MessageBatch.generateFromList(messagesList);\n        MessageClientIDSetter.setUniqID(msgBatch);\n        byte[] body = msgBatch.encode();\n        msgBatch.setBody(body);\n        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setQueueId(queueId);\n        Mockito.when(sendMessageProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class), Mockito.argThat(argument -> {\n            boolean first = argument.getCode() == RequestCode.SEND_MESSAGE;\n            boolean second = Arrays.equals(argument.getBody(), body);\n            return first & second;\n        }))).thenAnswer(invocation -> {\n            SimpleChannelHandlerContext simpleChannelHandlerContext = invocation.getArgument(0);\n            RemotingCommand request = invocation.getArgument(1);\n            RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);\n            response.setOpaque(request.getOpaque());\n            response.setCode(ResponseCode.SUCCESS);\n            response.setBody(body);\n            SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader();\n            sendMessageResponseHeader.setQueueId(queueId);\n            sendMessageResponseHeader.setQueueOffset(queueOffset);\n            sendMessageResponseHeader.setMsgId(offsetMessageId);\n            sendMessageResponseHeader.setTransactionId(transactionId);\n            simpleChannelHandlerContext.writeAndFlush(response);\n            return null;\n        });\n\n        CompletableFuture<List<SendResult>> future = localMessageService.sendMessage(proxyContext, null, messagesList, requestHeader, 1000L);\n        SendResult sendResult = future.get().get(0);\n        assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n        assertThat(sendResult.getMessageQueue())\n            .isEqualTo(new MessageQueue(topic, brokerControllerMock.getBrokerConfig().getBrokerName(), queueId));\n        assertThat(sendResult.getQueueOffset()).isEqualTo(queueOffset);\n        assertThat(sendResult.getTransactionId()).isEqualTo(transactionId);\n        assertThat(sendResult.getOffsetMsgId()).isEqualTo(offsetMessageId);\n    }\n\n    @Test\n    public void testSendMessageError() throws Exception {\n        RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);\n        response.setCode(ResponseCode.SYSTEM_ERROR);\n        Message message = new Message(\"topic\", \"body\".getBytes(StandardCharsets.UTF_8));\n        MessageClientIDSetter.setUniqID(message);\n        List<Message> messagesList = Collections.singletonList(message);\n        SendMessageRequestHeader sendMessageRequestHeader = new SendMessageRequestHeader();\n        sendMessageRequestHeader.setTopic(topic);\n        sendMessageRequestHeader.setQueueId(queueId);\n\n        Mockito.when(sendMessageProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class), Mockito.any(RemotingCommand.class)))\n            .thenReturn(response);\n\n        CompletableFuture<List<SendResult>> future = localMessageService.sendMessage(proxyContext, null, messagesList, sendMessageRequestHeader, 1000L);\n        ExecutionException exception = catchThrowableOfType(future::get, ExecutionException.class);\n        assertThat(exception.getCause()).isInstanceOf(ProxyException.class);\n        assertThat(((ProxyException) exception.getCause()).getCode()).isEqualTo(ProxyExceptionCode.INTERNAL_SERVER_ERROR);\n    }\n\n    @Test\n    public void testSendMessageWithException() throws Exception {\n        Mockito.when(sendMessageProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class), Mockito.any(RemotingCommand.class)))\n            .thenThrow(new RemotingCommandException(\"test\"));\n        Message message = new Message(\"topic\", \"body\".getBytes(StandardCharsets.UTF_8));\n        MessageClientIDSetter.setUniqID(message);\n        List<Message> messagesList = Collections.singletonList(message);\n        SendMessageRequestHeader sendMessageRequestHeader = new SendMessageRequestHeader();\n        CompletableFuture<List<SendResult>> future = localMessageService.sendMessage(proxyContext, null, messagesList, sendMessageRequestHeader, 1000L);\n        ExecutionException exception = catchThrowableOfType(future::get, ExecutionException.class);\n        assertThat(exception.getCause()).isInstanceOf(RemotingCommandException.class);\n    }\n\n    @Test\n    public void testSendMessageBack() throws Exception {\n        RemotingCommand remotingCommand = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"\");\n        Mockito.when(sendMessageProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class), Mockito.argThat(argument -> {\n            boolean first = argument.getCode() == RequestCode.CONSUMER_SEND_MSG_BACK;\n            boolean second = argument.readCustomHeader() instanceof ConsumerSendMsgBackRequestHeader;\n            return first && second;\n        }))).thenReturn(remotingCommand);\n        ConsumerSendMsgBackRequestHeader requestHeader = new ConsumerSendMsgBackRequestHeader();\n        CompletableFuture<RemotingCommand> future = localMessageService.sendMessageBack(proxyContext, null, null, requestHeader, 1000L);\n        RemotingCommand response = future.get();\n        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);\n    }\n\n    @Test\n    public void testEndTransaction() throws Exception {\n        EndTransactionRequestHeader requestHeader = new EndTransactionRequestHeader();\n        localMessageService.endTransactionOneway(proxyContext, null, requestHeader, 1000L);\n        Mockito.verify(endTransactionProcessorMock, Mockito.times(1)).processRequest(Mockito.any(SimpleChannelHandlerContext.class), Mockito.argThat(argument -> {\n            boolean first = argument.getCode() == RequestCode.END_TRANSACTION;\n            boolean second = argument.readCustomHeader() instanceof EndTransactionRequestHeader;\n            return first && second;\n        }));\n    }\n\n    @Test\n    public void testPopMessageWriteAndFlush() throws Exception {\n        int reviveQueueId = 1;\n        long popTime = System.currentTimeMillis();\n        long invisibleTime = 3000L;\n        long startOffset = 100L;\n        long restNum = 0L;\n        StringBuilder startOffsetStringBuilder = new StringBuilder();\n        StringBuilder messageOffsetStringBuilder = new StringBuilder();\n        List<MessageExt> messageExtList = new ArrayList<>();\n        List<Long> messageOffsetList = new ArrayList<>();\n        MessageExt message1 = buildMessageExt(topic, 0, startOffset);\n        messageExtList.add(message1);\n        messageOffsetList.add(startOffset);\n        byte[] body1 = MessageDecoder.encode(message1, false);\n        MessageExt message2 = buildMessageExt(topic, 0, startOffset + 1);\n        messageExtList.add(message2);\n        messageOffsetList.add(startOffset + 1);\n        ExtraInfoUtil.buildStartOffsetInfo(startOffsetStringBuilder, topic, queueId, startOffset);\n        ExtraInfoUtil.buildMsgOffsetInfo(messageOffsetStringBuilder, topic, queueId, messageOffsetList);\n        byte[] body2 = MessageDecoder.encode(message2, false);\n        ByteBuffer byteBuffer1 = ByteBuffer.wrap(body1);\n        ByteBuffer byteBuffer2 = ByteBuffer.wrap(body2);\n        ByteBuffer b3 = ByteBuffer.allocate(byteBuffer1.limit() + byteBuffer2.limit());\n        b3.put(byteBuffer1);\n        b3.put(byteBuffer2);\n        PopMessageRequestHeader requestHeader = new PopMessageRequestHeader();\n        requestHeader.setInvisibleTime(invisibleTime);\n        Mockito.when(popMessageProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class), Mockito.argThat(argument -> {\n            boolean first = argument.getCode() == RequestCode.POP_MESSAGE;\n            boolean second = argument.readCustomHeader() instanceof PopMessageRequestHeader;\n            return first && second;\n        }))).thenAnswer(invocation -> {\n            SimpleChannelHandlerContext simpleChannelHandlerContext = invocation.getArgument(0);\n            RemotingCommand request = invocation.getArgument(1);\n            RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class);\n            response.setOpaque(request.getOpaque());\n            response.setCode(ResponseCode.SUCCESS);\n            response.setBody(b3.array());\n            PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader();\n            responseHeader.setStartOffsetInfo(startOffsetStringBuilder.toString());\n            responseHeader.setMsgOffsetInfo(messageOffsetStringBuilder.toString());\n            responseHeader.setInvisibleTime(requestHeader.getInvisibleTime());\n            responseHeader.setPopTime(popTime);\n            responseHeader.setRestNum(restNum);\n            responseHeader.setReviveQid(reviveQueueId);\n            simpleChannelHandlerContext.writeAndFlush(response);\n            return null;\n        });\n        MessageQueue messageQueue = new MessageQueue(topic, brokerName, queueId);\n        CompletableFuture<PopResult> future = localMessageService.popMessage(proxyContext, new AddressableMessageQueue(messageQueue, \"\"), requestHeader, 1000L);\n        PopResult popResult = future.get();\n        assertThat(popResult.getPopTime()).isEqualTo(popTime);\n        assertThat(popResult.getInvisibleTime()).isEqualTo(invisibleTime);\n        assertThat(popResult.getPopStatus()).isEqualTo(PopStatus.FOUND);\n        assertThat(popResult.getRestNum()).isEqualTo(restNum);\n        assertThat(popResult.getMsgFoundList().size()).isEqualTo(messageExtList.size());\n        for (int i = 0; i < popResult.getMsgFoundList().size(); i++) {\n            assertMessageExt(popResult.getMsgFoundList().get(i), messageExtList.get(i));\n        }\n    }\n\n    @Test\n    public void testPopMessagePollingTimeout() throws Exception {\n        RemotingCommand remotingCommand = RemotingCommand.createResponseCommand(ResponseCode.POLLING_TIMEOUT, \"\");\n        Mockito.when(popMessageProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class), Mockito.argThat(argument -> {\n            boolean first = argument.getCode() == RequestCode.POP_MESSAGE;\n            boolean second = argument.readCustomHeader() instanceof PopMessageRequestHeader;\n            return first && second;\n        }))).thenReturn(remotingCommand);\n        PopMessageRequestHeader requestHeader = new PopMessageRequestHeader();\n        CompletableFuture<PopResult> future = localMessageService.popMessage(proxyContext, null, requestHeader, 1000L);\n        PopResult popResult = future.get();\n        assertThat(popResult.getPopStatus()).isEqualTo(PopStatus.POLLING_NOT_FOUND);\n    }\n\n    @Test\n    public void testChangeInvisibleTime() throws Exception {\n        String messageId = \"messageId\";\n        long popTime = System.currentTimeMillis();\n        long invisibleTime = 3000L;\n        int reviveQueueId = 1;\n        ReceiptHandle handle = ReceiptHandle.builder()\n            .startOffset(0L)\n            .retrieveTime(popTime)\n            .invisibleTime(invisibleTime)\n            .reviveQueueId(reviveQueueId)\n            .topicType(ReceiptHandle.NORMAL_TOPIC)\n            .brokerName(brokerName)\n            .queueId(queueId)\n            .offset(queueOffset)\n            .build();\n        RemotingCommand remotingCommand = RemotingCommand.createResponseCommand(ChangeInvisibleTimeResponseHeader.class);\n        remotingCommand.setCode(ResponseCode.SUCCESS);\n        remotingCommand.setRemark(\"\");\n        long newPopTime = System.currentTimeMillis();\n        long newInvisibleTime = 5000L;\n        int newReviveQueueId = 2;\n        ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) remotingCommand.readCustomHeader();\n        responseHeader.setReviveQid(newReviveQueueId);\n        responseHeader.setInvisibleTime(newInvisibleTime);\n        responseHeader.setPopTime(newPopTime);\n        Mockito.when(changeInvisibleTimeProcessorMock.processRequestAsync(Mockito.any(Channel.class), Mockito.argThat(argument -> {\n            boolean first = argument.getCode() == RequestCode.CHANGE_MESSAGE_INVISIBLETIME;\n            boolean second = argument.readCustomHeader() instanceof ChangeInvisibleTimeRequestHeader;\n            return first && second;\n        }), Mockito.any(Boolean.class))).thenReturn(CompletableFuture.completedFuture(remotingCommand));\n        ChangeInvisibleTimeRequestHeader requestHeader = new ChangeInvisibleTimeRequestHeader();\n        CompletableFuture<AckResult> future = localMessageService.changeInvisibleTime(proxyContext, handle, messageId,\n            requestHeader, 1000L);\n        AckResult ackResult = future.get();\n        assertThat(ackResult.getStatus()).isEqualTo(AckStatus.OK);\n        assertThat(ackResult.getPopTime()).isEqualTo(newPopTime);\n        assertThat(ackResult.getExtraInfo()).isEqualTo(ReceiptHandle.builder()\n            .startOffset(0L)\n            .retrieveTime(newPopTime)\n            .invisibleTime(newInvisibleTime)\n            .reviveQueueId(newReviveQueueId)\n            .topicType(ReceiptHandle.NORMAL_TOPIC)\n            .brokerName(brokerName)\n            .queueId(queueId)\n            .offset(queueOffset)\n            .build()\n            .encode());\n    }\n\n    @Test\n    public void testAckMessage() throws Exception {\n        String messageId = \"messageId\";\n        long popTime = System.currentTimeMillis();\n        long invisibleTime = 3000L;\n        int reviveQueueId = 1;\n        ReceiptHandle handle = ReceiptHandle.builder()\n            .startOffset(0L)\n            .retrieveTime(popTime)\n            .invisibleTime(invisibleTime)\n            .reviveQueueId(reviveQueueId)\n            .topicType(ReceiptHandle.NORMAL_TOPIC)\n            .brokerName(brokerName)\n            .queueId(queueId)\n            .offset(queueOffset)\n            .build();\n        RemotingCommand remotingCommand = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, null);\n        Mockito.when(ackMessageProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class), Mockito.argThat(argument -> {\n            boolean first = argument.getCode() == RequestCode.ACK_MESSAGE;\n            boolean second = argument.readCustomHeader() instanceof AckMessageRequestHeader;\n            return first && second;\n        }))).thenReturn(remotingCommand);\n        AckMessageRequestHeader requestHeader = new AckMessageRequestHeader();\n        CompletableFuture<AckResult> future = localMessageService.ackMessage(proxyContext, handle, messageId,\n            requestHeader, 1000L);\n        AckResult ackResult = future.get();\n        assertThat(ackResult.getStatus()).isEqualTo(AckStatus.OK);\n    }\n\n    @Test\n    public void testRecallMessage_success() throws Exception {\n        RecallMessageResponseHeader responseHeader = new RecallMessageResponseHeader();\n        responseHeader.setMsgId(\"msgId\");\n        RemotingCommand response = RemotingCommand.createResponseCommandWithHeader(ResponseCode.SUCCESS, responseHeader);\n        Mockito.when(recallMessageProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class),\n            Mockito.any())).thenReturn(response);\n        RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader();\n        String msgId = localMessageService.recallMessage(proxyContext, \"brokerName\", requestHeader, 1000L).join();\n        assertThat(msgId).isEqualTo(\"msgId\");\n    }\n\n    @Test\n    public void testRecallMessage_fail() throws Exception {\n        RecallMessageResponseHeader responseHeader = new RecallMessageResponseHeader();\n        RemotingCommand response = RemotingCommand.createResponseCommandWithHeader(ResponseCode.SLAVE_NOT_AVAILABLE, responseHeader);\n        Mockito.when(recallMessageProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class),\n            Mockito.any())).thenReturn(response);\n        RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader();\n        CompletionException exception = Assert.assertThrows(CompletionException.class, () -> {\n            localMessageService.recallMessage(proxyContext, \"brokerName\", requestHeader, 1000L).join();\n        });\n        Assert.assertTrue(exception.getCause() instanceof ProxyException);\n    }\n\n    private MessageExt buildMessageExt(String topic, int queueId, long queueOffset) {\n        MessageExt message1 = new MessageExt();\n        message1.setTopic(topic);\n        message1.setBody(\"body\".getBytes(StandardCharsets.UTF_8));\n        message1.setFlag(0);\n        message1.setQueueId(queueId);\n        message1.setQueueOffset(queueOffset);\n        message1.setCommitLogOffset(1000L);\n        message1.setSysFlag(0);\n        message1.setBornTimestamp(0L);\n        InetSocketAddress inetSocketAddress = new InetSocketAddress(\"127.0.0.1\", 80);\n        message1.setBornHost(inetSocketAddress);\n        message1.setStoreHost(inetSocketAddress);\n        message1.setReconsumeTimes(0);\n        message1.setPreparedTransactionOffset(0L);\n        message1.putUserProperty(\"K\", \"V\");\n        return message1;\n    }\n\n    private void assertMessageExt(MessageExt messageExt1, MessageExt messageExt2) {\n        assertThat(messageExt1.getBody()).isEqualTo(messageExt2.getBody());\n        assertThat(messageExt1.getTopic()).isEqualTo(messageExt2.getTopic());\n        assertThat(messageExt1.getQueueId()).isEqualTo(messageExt2.getQueueId());\n        assertThat(messageExt1.getQueueOffset()).isEqualTo(messageExt2.getQueueOffset());\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.metadata;\n\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Optional;\nimport java.util.Set;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.service.BaseServiceTest;\nimport org.apache.rocketmq.proxy.service.route.MessageQueueView;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicConfigAndQueueMapping;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.when;\n\npublic class ClusterMetadataServiceTest extends BaseServiceTest {\n\n    private ClusterMetadataService clusterMetadataService;\n\n    protected static final String BROKER2_ADDR = \"127.0.0.2:10911\";\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        ConfigurationManager.getProxyConfig().setRocketMQClusterName(CLUSTER_NAME);\n\n        TopicConfigAndQueueMapping topicConfigAndQueueMapping = new TopicConfigAndQueueMapping();\n        topicConfigAndQueueMapping.setAttributes(new HashMap<>());\n        topicConfigAndQueueMapping.setTopicMessageType(TopicMessageType.NORMAL);\n        when(this.mqClientAPIExt.getTopicConfig(anyString(), eq(TOPIC), anyLong())).thenReturn(topicConfigAndQueueMapping);\n\n        when(this.mqClientAPIExt.getSubscriptionGroupConfig(anyString(), eq(GROUP), anyLong())).thenReturn(new SubscriptionGroupConfig());\n\n        this.clusterMetadataService = new ClusterMetadataService(this.topicRouteService, this.mqClientAPIFactory);\n\n        BrokerData brokerData2 = new BrokerData();\n        brokerData2.setBrokerName(\"brokerName2\");\n        HashMap<Long, String> addrs = new HashMap<>();\n        addrs.put(MixAll.MASTER_ID, BROKER2_ADDR);\n        brokerData2.setBrokerAddrs(addrs);\n        brokerData2.setCluster(CLUSTER_NAME);\n        topicRouteData.getBrokerDatas().add(brokerData2);\n        when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(CLUSTER_NAME, topicRouteData, null));\n\n    }\n\n    @Test\n    public void testGetTopicMessageType() {\n        ProxyContext ctx = ProxyContext.create();\n        assertEquals(TopicMessageType.UNSPECIFIED, this.clusterMetadataService.getTopicMessageType(ctx, ERR_TOPIC));\n        assertEquals(1, this.clusterMetadataService.topicConfigCache.asMap().size());\n        assertEquals(TopicMessageType.UNSPECIFIED, this.clusterMetadataService.getTopicMessageType(ctx, ERR_TOPIC));\n\n        assertEquals(TopicMessageType.NORMAL, this.clusterMetadataService.getTopicMessageType(ctx, TOPIC));\n        assertEquals(2, this.clusterMetadataService.topicConfigCache.asMap().size());\n    }\n\n    @Test\n    public void testGetSubscriptionGroupConfig() {\n        ProxyContext ctx = ProxyContext.create();\n        assertNotNull(this.clusterMetadataService.getSubscriptionGroupConfig(ctx, GROUP));\n        assertEquals(1, this.clusterMetadataService.subscriptionGroupConfigCache.asMap().size());\n    }\n\n    @Test\n    public void findOneBroker() {\n\n        Set<String> resultBrokerNames = new HashSet<>();\n        // run 1000 times to test the random\n        for (int i = 0; i < 1000; i++) {\n            Optional<BrokerData> brokerData = null;\n            try {\n                brokerData = this.clusterMetadataService.findOneBroker(TOPIC);\n                resultBrokerNames.add(brokerData.get().getBrokerName());\n            } catch (Exception e) {\n                throw new RuntimeException(e);\n            }\n        }\n        // we should choose two brokers\n        assertEquals(2, resultBrokerNames.size());\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.mqclient;\n\nimport java.lang.reflect.Field;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.UUID;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ThreadLocalRandom;\nimport java.util.concurrent.atomic.AtomicReference;\nimport java.util.stream.Collectors;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.consumer.AckCallback;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.PopCallback;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PopStatus;\nimport org.apache.rocketmq.client.consumer.PullCallback;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.impl.CommunicationMode;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.consumer.PullResultExt;\nimport org.apache.rocketmq.client.impl.mqclient.DoNothingClientRemotingProcessor;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIExt;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.lite.LiteSubscriptionDTO;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.remoting.InvokeCallback;\nimport org.apache.rocketmq.remoting.RemotingClient;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.ResponseFuture;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.GetLiteTopicInfoResponseBody;\nimport org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupResponseBody;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopLiteMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SearchOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SearchOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;\nimport org.assertj.core.util.Lists;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\nimport org.mockito.stubbing.Answer;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertSame;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.doAnswer;\nimport static org.mockito.Mockito.doReturn;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class MQClientAPIExtTest {\n\n    private static final String BROKER_ADDR = \"127.0.0.1:10911\";\n    private static final String BROKER_NAME = \"brokerName\";\n    private static final long TIMEOUT = 3000;\n    private static final String CONSUMER_GROUP = \"group\";\n    private static final String TOPIC = \"topic\";\n\n    @Spy\n    private final MQClientAPIExt mqClientAPI = new MQClientAPIExt(new ClientConfig(), new NettyClientConfig(), new DoNothingClientRemotingProcessor(null), null);\n    @Mock\n    private RemotingClient remotingClient;\n\n    @Before\n    public void init() throws Exception {\n        Field field = MQClientAPIImpl.class.getDeclaredField(\"remotingClient\");\n        field.setAccessible(true);\n        field.set(mqClientAPI, remotingClient);\n    }\n\n    @Test\n    public void testSendHeartbeatAsync() throws Exception {\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        future.complete(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"\"));\n        doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());\n\n        assertNotNull(mqClientAPI.sendHeartbeatAsync(BROKER_ADDR, new HeartbeatData(), TIMEOUT).get());\n    }\n\n    @Test\n    public void testSendMessageAsync() throws Exception {\n        AtomicReference<String> msgIdRef = new AtomicReference<>();\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);\n        SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader();\n        sendMessageResponseHeader.setMsgId(msgIdRef.get());\n        sendMessageResponseHeader.setQueueId(0);\n        sendMessageResponseHeader.setQueueOffset(1L);\n        response.setCode(ResponseCode.SUCCESS);\n        response.makeCustomHeaderToNet();\n        future.complete(response);\n        doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());\n\n        MessageExt messageExt = createMessage();\n        msgIdRef.set(MessageClientIDSetter.getUniqID(messageExt));\n\n        SendResult sendResult = mqClientAPI.sendMessageAsync(BROKER_ADDR, BROKER_NAME, messageExt, new SendMessageRequestHeader(), TIMEOUT)\n            .get();\n        assertNotNull(sendResult);\n        assertEquals(msgIdRef.get(), sendResult.getMsgId());\n        assertEquals(SendStatus.SEND_OK, sendResult.getSendStatus());\n    }\n\n    @Test\n    public void testSendMessageListAsync() throws Exception {\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);\n        SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader();\n        sendMessageResponseHeader.setMsgId(\"\");\n        sendMessageResponseHeader.setQueueId(0);\n        sendMessageResponseHeader.setQueueOffset(1L);\n        response.setCode(ResponseCode.SUCCESS);\n        response.makeCustomHeaderToNet();\n        future.complete(response);\n        doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());\n\n        List<MessageExt> messageExtList = new ArrayList<>();\n        StringBuilder sb = new StringBuilder();\n        for (int i = 0; i < 3; i++) {\n            MessageExt messageExt = createMessage();\n            sb.append(sb.length() == 0 ? \"\" : \",\").append(MessageClientIDSetter.getUniqID(messageExt));\n            messageExtList.add(messageExt);\n        }\n\n        SendResult sendResult = mqClientAPI.sendMessageAsync(BROKER_ADDR, BROKER_NAME, messageExtList, new SendMessageRequestHeader(), TIMEOUT)\n            .get();\n        assertNotNull(sendResult);\n        assertEquals(sb.toString(), sendResult.getMsgId());\n        assertEquals(SendStatus.SEND_OK, sendResult.getSendStatus());\n    }\n\n    @Test\n    public void testSendMessageBackAsync() throws Exception {\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        future.complete(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"\"));\n        doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());\n\n        RemotingCommand remotingCommand = mqClientAPI.sendMessageBackAsync(BROKER_ADDR, new ConsumerSendMsgBackRequestHeader(), TIMEOUT)\n            .get();\n        assertNotNull(remotingCommand);\n        assertEquals(ResponseCode.SUCCESS, remotingCommand.getCode());\n    }\n\n    @Test\n    public void testPopMessageAsync() throws Exception {\n        PopResult popResult = new PopResult(PopStatus.POLLING_NOT_FOUND, null);\n        doAnswer((Answer<Void>) mock -> {\n            PopCallback popCallback = mock.getArgument(4);\n            popCallback.onSuccess(popResult);\n            return null;\n        }).when(mqClientAPI).popMessageAsync(anyString(), anyString(), any(), anyLong(), any());\n\n        assertSame(popResult, mqClientAPI.popMessageAsync(BROKER_ADDR, BROKER_NAME, new PopMessageRequestHeader(), TIMEOUT).get());\n    }\n\n    @Test\n    public void testPopLiteMessageAsync() throws Exception {\n        PopResult popResult = new PopResult(PopStatus.FOUND, new ArrayList<>());\n        doAnswer((Answer<Void>) mock -> {\n            PopCallback popCallback = mock.getArgument(4);\n            popCallback.onSuccess(popResult);\n            return null;\n        }).when(mqClientAPI).popLiteMessageAsync(anyString(), anyString(), any(), anyLong(), any());\n\n        assertSame(popResult, mqClientAPI.popLiteMessageAsync(BROKER_ADDR, BROKER_NAME, new PopLiteMessageRequestHeader(), TIMEOUT).get());\n    }\n\n    @Test\n    public void testPopLiteMessageAsync_Exception() throws Exception {\n        Throwable throwable = new RuntimeException(\"test exception\");\n        doAnswer((Answer<Void>) mock -> {\n            PopCallback popCallback = mock.getArgument(4);\n            popCallback.onException(throwable);\n            return null;\n        }).when(mqClientAPI).popLiteMessageAsync(anyString(), anyString(), any(), anyLong(), any());\n\n        CompletableFuture<PopResult> future = mqClientAPI.popLiteMessageAsync(BROKER_ADDR, BROKER_NAME, new PopLiteMessageRequestHeader(), TIMEOUT);\n        assertTrue(future.isCompletedExceptionally());\n    }\n\n    @Test\n    public void testAckMessageAsync() throws Exception {\n        AckResult ackResult = new AckResult();\n        doAnswer((Answer<Void>) mock -> {\n            AckCallback ackCallback = mock.getArgument(2);\n            ackCallback.onSuccess(ackResult);\n            return null;\n        }).when(mqClientAPI).ackMessageAsync(anyString(), anyLong(), any(AckCallback.class), any());\n\n        assertSame(ackResult, mqClientAPI.ackMessageAsync(BROKER_ADDR, new AckMessageRequestHeader(), TIMEOUT).get());\n    }\n\n    @Test\n    public void testBatchAckMessageAsync() throws Exception {\n        AckResult ackResult = new AckResult();\n        doAnswer((Answer<Void>) mock -> {\n            AckCallback ackCallback = mock.getArgument(2);\n            ackCallback.onSuccess(ackResult);\n            return null;\n        }).when(mqClientAPI).batchAckMessageAsync(anyString(), anyLong(), any(AckCallback.class), any());\n\n        assertSame(ackResult, mqClientAPI.batchAckMessageAsync(BROKER_ADDR, TOPIC, CONSUMER_GROUP, new ArrayList<>(), TIMEOUT).get());\n    }\n\n    @Test\n    public void testChangeInvisibleTimeAsync() throws Exception {\n        AckResult ackResult = new AckResult();\n        doAnswer((Answer<Void>) mock -> {\n            AckCallback ackCallback = mock.getArgument(4);\n            ackCallback.onSuccess(ackResult);\n            return null;\n        }).when(mqClientAPI).changeInvisibleTimeAsync(anyString(), anyString(), any(), anyLong(), any(AckCallback.class));\n\n        assertSame(ackResult, mqClientAPI.changeInvisibleTimeAsync(BROKER_ADDR, BROKER_NAME, new ChangeInvisibleTimeRequestHeader(), TIMEOUT).get());\n    }\n\n    @Test\n    public void testPullMessageAsync() throws Exception {\n        MessageExt msg1 = createMessage();\n        byte[] msg1Byte = MessageDecoder.encode(msg1, false);\n        MessageExt msg2 = createMessage();\n        byte[] msg2Byte = MessageDecoder.encode(msg2, false);\n\n        ByteBuffer byteBuffer = ByteBuffer.allocate(msg1Byte.length + msg2Byte.length);\n        byteBuffer.put(msg1Byte);\n        byteBuffer.put(msg2Byte);\n\n        PullResultExt pullResultExt = new PullResultExt(PullStatus.FOUND, 0, 0, 1, null, 0,\n            byteBuffer.array());\n        doAnswer((Answer<Void>) mock -> {\n            PullCallback pullCallback = mock.getArgument(4);\n            pullCallback.onSuccess(pullResultExt);\n            return null;\n        }).when(mqClientAPI).pullMessage(anyString(), any(), anyLong(), any(CommunicationMode.class), any(PullCallback.class));\n\n        PullResult pullResult = mqClientAPI.pullMessageAsync(BROKER_ADDR, new PullMessageRequestHeader(), TIMEOUT).get();\n        assertNotNull(pullResult);\n        assertEquals(2, pullResult.getMsgFoundList().size());\n\n        Set<String> msgIdSet = pullResult.getMsgFoundList().stream().map(MessageClientIDSetter::getUniqID).collect(Collectors.toSet());\n        assertTrue(msgIdSet.contains(MessageClientIDSetter.getUniqID(msg1)));\n        assertTrue(msgIdSet.contains(MessageClientIDSetter.getUniqID(msg2)));\n    }\n\n    @Test\n    public void testGetConsumerListByGroupAsync() throws Exception {\n        List<String> clientIds = Lists.newArrayList(\"clientIds\");\n        doAnswer((Answer<Void>) mock -> {\n            InvokeCallback invokeCallback = mock.getArgument(3);\n            ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null);\n            RemotingCommand response = RemotingCommand.createResponseCommand(GetConsumerListByGroupResponseHeader.class);\n            response.setCode(ResponseCode.SUCCESS);\n            response.makeCustomHeaderToNet();\n            GetConsumerListByGroupResponseBody body = new GetConsumerListByGroupResponseBody();\n            body.setConsumerIdList(clientIds);\n            response.setBody(body.encode());\n            responseFuture.putResponse(response);\n            invokeCallback.operationSucceed(responseFuture.getResponseCommand());\n            return null;\n        }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any());\n\n        List<String> res = mqClientAPI.getConsumerListByGroupAsync(BROKER_ADDR, new GetConsumerListByGroupRequestHeader(), TIMEOUT).get();\n        assertEquals(clientIds, res);\n    }\n\n    @Test\n    public void testGetEmptyConsumerListByGroupAsync() throws Exception {\n        doAnswer((Answer<Void>) mock -> {\n            InvokeCallback invokeCallback = mock.getArgument(3);\n            ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null);\n            RemotingCommand response = RemotingCommand.createResponseCommand(GetConsumerListByGroupRequestHeader.class);\n            response.setCode(ResponseCode.SYSTEM_ERROR);\n            response.makeCustomHeaderToNet();\n            responseFuture.putResponse(response);\n            invokeCallback.operationSucceed(responseFuture.getResponseCommand());\n            return null;\n        }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any());\n\n        List<String> res = mqClientAPI.getConsumerListByGroupAsync(BROKER_ADDR, new GetConsumerListByGroupRequestHeader(), TIMEOUT).get();\n        assertTrue(res.isEmpty());\n    }\n\n    @Test\n    public void testGetMaxOffsetAsync() throws Exception {\n        long offset = ThreadLocalRandom.current().nextLong();\n        doAnswer((Answer<Void>) mock -> {\n            InvokeCallback invokeCallback = mock.getArgument(3);\n            ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null);\n            RemotingCommand response = RemotingCommand.createResponseCommand(GetMaxOffsetResponseHeader.class);\n            GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.readCustomHeader();\n            responseHeader.setOffset(offset);\n            response.setCode(ResponseCode.SUCCESS);\n            response.makeCustomHeaderToNet();\n            responseFuture.putResponse(response);\n            invokeCallback.operationSucceed(responseFuture.getResponseCommand());\n            return null;\n        }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any());\n\n        GetMaxOffsetRequestHeader requestHeader = new GetMaxOffsetRequestHeader();\n        requestHeader.setTopic(TOPIC);\n        requestHeader.setQueueId(0);\n        assertEquals(offset, mqClientAPI.getMaxOffset(BROKER_ADDR, requestHeader, TIMEOUT).get().longValue());\n    }\n\n    @Test\n    public void testSearchOffsetAsync() throws Exception {\n        long offset = ThreadLocalRandom.current().nextLong();\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class);\n        SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader();\n        responseHeader.setOffset(offset);\n        response.setCode(ResponseCode.SUCCESS);\n        response.makeCustomHeaderToNet();\n        future.complete(response);\n\n        doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());\n\n        SearchOffsetRequestHeader requestHeader = new SearchOffsetRequestHeader();\n        requestHeader.setTopic(TOPIC);\n        requestHeader.setQueueId(0);\n        requestHeader.setTimestamp(System.currentTimeMillis());\n        assertEquals(offset, mqClientAPI.searchOffset(BROKER_ADDR, requestHeader, TIMEOUT).get().longValue());\n    }\n\n    protected MessageExt createMessage() {\n        MessageExt messageExt = new MessageExt();\n        messageExt.setTopic(\"topic\");\n        messageExt.setBornHost(NetworkUtil.string2SocketAddress(\"127.0.0.2:8888\"));\n        messageExt.setStoreHost(NetworkUtil.string2SocketAddress(\"127.0.0.1:10911\"));\n        messageExt.setBody(UUID.randomUUID().toString().getBytes(StandardCharsets.UTF_8));\n        MessageClientIDSetter.setUniqID(messageExt);\n        return messageExt;\n    }\n\n    @Test\n    public void testSyncLiteSubscriptionAsync_Success() throws Exception {\n        LiteSubscriptionDTO liteSubscriptionDTO = new LiteSubscriptionDTO();\n        liteSubscriptionDTO.setTopic(\"test-topic\");\n        liteSubscriptionDTO.setGroup(\"test-group\");\n\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"\");\n        future.complete(response);\n\n        doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());\n\n        CompletableFuture<Void> result = mqClientAPI.syncLiteSubscriptionAsync(BROKER_ADDR, liteSubscriptionDTO, TIMEOUT);\n\n        assertNotNull(result);\n        result.get();\n    }\n\n    @Test\n    public void testSyncLiteSubscriptionAsync_Failure() throws Exception {\n        LiteSubscriptionDTO liteSubscriptionDTO = new LiteSubscriptionDTO();\n        liteSubscriptionDTO.setTopic(\"test-topic\");\n        liteSubscriptionDTO.setGroup(\"test-group\");\n\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, \"System error\");\n        future.complete(response);\n\n        doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());\n\n        CompletableFuture<Void> result = mqClientAPI.syncLiteSubscriptionAsync(BROKER_ADDR, liteSubscriptionDTO, TIMEOUT);\n\n        assertNotNull(result);\n        assertTrue(result.isCompletedExceptionally());\n\n        try {\n            result.get();\n        } catch (Exception e) {\n            assertTrue(e.getCause() instanceof MQBrokerException);\n            MQBrokerException brokerException = (MQBrokerException) e.getCause();\n            assertEquals(ResponseCode.SYSTEM_ERROR, brokerException.getResponseCode());\n        }\n    }\n\n    @Test\n    public void testSyncLiteSubscriptionAsync_Exception() throws Exception {\n        LiteSubscriptionDTO liteSubscriptionDTO = new LiteSubscriptionDTO();\n        liteSubscriptionDTO.setTopic(\"test-topic\");\n        liteSubscriptionDTO.setGroup(\"test-group\");\n\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        future.completeExceptionally(new RuntimeException(\"Network error\"));\n\n        doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());\n\n        CompletableFuture<Void> result = mqClientAPI.syncLiteSubscriptionAsync(BROKER_ADDR, liteSubscriptionDTO, TIMEOUT);\n\n        assertNotNull(result);\n        assertTrue(result.isCompletedExceptionally());\n\n        try {\n            result.get();\n        } catch (Exception e) {\n            assertTrue(e.getCause() instanceof RuntimeException);\n            assertEquals(\"Network error\", e.getCause().getMessage());\n        }\n    }\n\n    @Test\n    public void testSyncLiteSubscriptionAsync_EmptySubscription() throws Exception {\n        LiteSubscriptionDTO liteSubscriptionDTO = new LiteSubscriptionDTO();\n\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"\");\n        future.complete(response);\n\n        doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());\n\n        CompletableFuture<Void> result = mqClientAPI.syncLiteSubscriptionAsync(BROKER_ADDR, liteSubscriptionDTO, TIMEOUT);\n\n        assertNotNull(result);\n        result.get();\n    }\n\n    @Test\n    public void testGetLiteTopicInfoAsync_Success() throws Exception {\n        String parentTopic = \"parentTopic\";\n        String liteTopic = \"liteTopic\";\n\n        GetLiteTopicInfoResponseBody responseBody = new GetLiteTopicInfoResponseBody();\n        responseBody.setLiteTopic(liteTopic);\n        responseBody.setParentTopic(parentTopic);\n\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, \"\");\n        response.setBody(responseBody.encode());\n        future.complete(response);\n\n        doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());\n\n        CompletableFuture<GetLiteTopicInfoResponseBody> result =\n            mqClientAPI.getLiteTopicInfoAsync(BROKER_ADDR, parentTopic, liteTopic, TIMEOUT);\n\n        assertNotNull(result);\n        GetLiteTopicInfoResponseBody actualBody = result.get();\n        assertNotNull(actualBody);\n        assertEquals(liteTopic, actualBody.getLiteTopic());\n        assertEquals(parentTopic, actualBody.getParentTopic());\n    }\n\n    @Test\n    public void testGetLiteTopicInfoAsync_Failure() throws Exception {\n        String parentTopic = \"parentTopic\";\n        String liteTopic = \"liteTopic\";\n\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, \"System error\");\n        future.complete(response);\n\n        doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());\n\n        CompletableFuture<GetLiteTopicInfoResponseBody> result =\n            mqClientAPI.getLiteTopicInfoAsync(BROKER_ADDR, parentTopic, liteTopic, TIMEOUT);\n\n        assertNotNull(result);\n        assertTrue(result.isCompletedExceptionally());\n\n        try {\n            result.get();\n        } catch (Exception e) {\n            assertTrue(e.getCause() instanceof MQBrokerException);\n        }\n    }\n\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/ProxyClientRemotingProcessorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.mqclient;\n\nimport apache.rocketmq.v2.TelemetryCommand;\nimport io.grpc.Status;\nimport io.grpc.StatusRuntimeException;\nimport io.grpc.stub.ServerCallStreamObserver;\nimport io.netty.channel.Channel;\nimport java.net.InetSocketAddress;\nimport java.time.Duration;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.rocketmq.broker.client.ProducerManager;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.proxy.service.client.ProxyClientRemotingProcessor;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcClientChannel;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;\nimport org.apache.rocketmq.proxy.service.channel.SimpleChannelHandlerContext;\nimport org.apache.rocketmq.proxy.service.relay.ProxyRelayResult;\nimport org.apache.rocketmq.proxy.service.relay.ProxyRelayService;\nimport org.apache.rocketmq.proxy.service.relay.RelayData;\nimport org.apache.rocketmq.proxy.service.transaction.TransactionData;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.awaitility.Awaitility.await;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.doThrow;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ProxyClientRemotingProcessorTest {\n    @Mock\n    private ProducerManager producerManager;\n    @Mock\n    private GrpcClientSettingsManager grpcClientSettingsManager;\n    @Mock\n    private ProxyRelayService proxyRelayService;\n\n    @Test\n    public void testTransactionCheck() throws Exception {\n        // Temporarily skip this test on the Mac system as it is flaky\n        if (MixAll.isMac()) {\n            return;\n        }\n        CompletableFuture<ProxyRelayResult<Void>> proxyRelayResultFuture = new CompletableFuture<>();\n        when(proxyRelayService.processCheckTransactionState(any(), any(), any(), any()))\n            .thenReturn(new RelayData<>(\n                new TransactionData(\"brokerName\", \"topic\", 0, 0, \"id\", System.currentTimeMillis(), 3000),\n                proxyRelayResultFuture));\n\n        GrpcClientChannel grpcClientChannel = new GrpcClientChannel(proxyRelayService, grpcClientSettingsManager, null,\n            ProxyContext.create().setRemoteAddress(\"127.0.0.1:8888\").setLocalAddress(\"127.0.0.1:10911\"), \"clientId\");\n        when(producerManager.getAvailableChannel(anyString()))\n            .thenReturn(grpcClientChannel);\n\n        ProxyClientRemotingProcessor processor = new ProxyClientRemotingProcessor(producerManager, null);\n        CheckTransactionStateRequestHeader requestHeader = new CheckTransactionStateRequestHeader();\n        RemotingCommand command = RemotingCommand.createRequestCommand(RequestCode.CHECK_TRANSACTION_STATE, requestHeader);\n        MessageExt message = new MessageExt();\n        message.setQueueId(0);\n        message.setFlag(12);\n        message.setQueueOffset(0L);\n        message.setCommitLogOffset(100L);\n        message.setSysFlag(0);\n        message.setBornTimestamp(System.currentTimeMillis());\n        message.setBornHost(new InetSocketAddress(\"127.0.0.1\", 10));\n        message.setStoreTimestamp(System.currentTimeMillis());\n        message.setStoreHost(new InetSocketAddress(\"127.0.0.1\", 11));\n        message.setBody(\"body\".getBytes());\n        message.setTopic(\"topic\");\n        MessageAccessor.putProperty(message, MessageConst.PROPERTY_PRODUCER_GROUP, \"group\");\n        command.setBody(MessageDecoder.encode(message, false));\n\n        processor.processRequest(new MockChannelHandlerContext(null), command);\n\n        ServerCallStreamObserver<TelemetryCommand> observer = mock(ServerCallStreamObserver.class);\n        grpcClientChannel.setClientObserver(observer);\n\n        processor.processRequest(new MockChannelHandlerContext(null), command);\n        verify(observer, times(1)).onNext(any());\n\n        // throw exception to test clear observer\n        doThrow(new StatusRuntimeException(Status.CANCELLED)).when(observer).onNext(any());\n\n        ExecutorService executorService = Executors.newCachedThreadPool();\n        AtomicInteger count = new AtomicInteger();\n        for (int i = 0; i < 100; i++) {\n            executorService.submit(() -> {\n                try {\n                    processor.processRequest(new MockChannelHandlerContext(null), command);\n                    count.incrementAndGet();\n                } catch (RemotingCommandException ignored) {\n                }\n            });\n        }\n        await().atMost(Duration.ofSeconds(3)).until(() -> count.get() == 100);\n        verify(observer, times(2)).onNext(any());\n    }\n\n    protected static class MockChannelHandlerContext extends SimpleChannelHandlerContext {\n\n        public MockChannelHandlerContext(Channel channel) {\n            super(channel);\n        }\n\n        @Override\n        public Channel channel() {\n            Channel channel = mock(Channel.class);\n            when(channel.remoteAddress()).thenReturn(NetworkUtil.string2SocketAddress(\"127.0.0.1:10911\"));\n            return channel;\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.receipt;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.local.LocalChannel;\nimport java.time.Duration;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ConsumerGroupEvent;\nimport org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;\nimport org.apache.rocketmq.broker.client.ConsumerManager;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.AckStatus;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.consumer.ReceiptHandle;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.state.StateEventListener;\nimport org.apache.rocketmq.proxy.common.RenewEvent;\nimport org.apache.rocketmq.proxy.common.ContextVariable;\nimport org.apache.rocketmq.proxy.common.MessageReceiptHandle;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.common.ProxyException;\nimport org.apache.rocketmq.proxy.common.ProxyExceptionCode;\nimport org.apache.rocketmq.proxy.common.ReceiptHandleGroup;\nimport org.apache.rocketmq.proxy.common.RenewStrategyPolicy;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.common.ReceiptHandleGroupKey;\nimport org.apache.rocketmq.proxy.service.BaseServiceTest;\nimport org.apache.rocketmq.proxy.service.metadata.MetadataService;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.subscription.RetryPolicy;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.ArgumentCaptor;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.stubbing.Answer;\n\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\n\npublic class DefaultReceiptHandleManagerTest extends BaseServiceTest {\n    private DefaultReceiptHandleManager receiptHandleManager;\n    @Mock\n    protected MessagingProcessor messagingProcessor;\n    @Mock\n    protected MetadataService metadataService;\n    @Mock\n    protected ConsumerManager consumerManager;\n\n    private static final ProxyContext PROXY_CONTEXT = ProxyContext.create();\n    private static final String GROUP = \"group\";\n    private static final String TOPIC = \"topic\";\n    private static final String BROKER_NAME = \"broker\";\n    private static final int QUEUE_ID = 1;\n    private static final String MESSAGE_ID = \"messageId\";\n    private static final long OFFSET = 123L;\n    private static final long INVISIBLE_TIME = 60000L;\n    private static final int RECONSUME_TIMES = 1;\n    private static final String MSG_ID = MessageClientIDSetter.createUniqID();\n    private MessageReceiptHandle messageReceiptHandle;\n\n    private String receiptHandle;\n\n    @Before\n    public void setup() {\n        receiptHandleManager = new DefaultReceiptHandleManager(metadataService, consumerManager, new StateEventListener<RenewEvent>() {\n            @Override\n            public void fireEvent(RenewEvent event) {\n                MessageReceiptHandle messageReceiptHandle = event.getMessageReceiptHandle();\n                ReceiptHandle handle = ReceiptHandle.decode(messageReceiptHandle.getReceiptHandleStr());\n                messagingProcessor.changeInvisibleTime(PROXY_CONTEXT, handle, messageReceiptHandle.getMessageId(),\n                        messageReceiptHandle.getGroup(), messageReceiptHandle.getTopic(), event.getRenewTime())\n                    .whenComplete((v, t) -> {\n                        if (t != null) {\n                            event.getFuture().completeExceptionally(t);\n                            return;\n                        }\n                        event.getFuture().complete(v);\n                    });\n            }\n        });\n        ProxyConfig config = ConfigurationManager.getProxyConfig();\n        receiptHandle = ReceiptHandle.builder()\n            .startOffset(0L)\n            .retrieveTime(System.currentTimeMillis() - INVISIBLE_TIME + config.getRenewAheadTimeMillis() - 5)\n            .invisibleTime(INVISIBLE_TIME)\n            .reviveQueueId(1)\n            .topicType(ReceiptHandle.NORMAL_TOPIC)\n            .brokerName(BROKER_NAME)\n            .queueId(QUEUE_ID)\n            .offset(OFFSET)\n            .commitLogOffset(0L)\n            .build().encode();\n        PROXY_CONTEXT.withVal(ContextVariable.CLIENT_ID, \"channel-id\");\n        PROXY_CONTEXT.withVal(ContextVariable.CHANNEL, new LocalChannel());\n        Mockito.doNothing().when(consumerManager).appendConsumerIdsChangeListener(Mockito.any(ConsumerIdsChangeListener.class));\n        messageReceiptHandle = new MessageReceiptHandle(GROUP, TOPIC, QUEUE_ID, receiptHandle, MESSAGE_ID, OFFSET,\n            RECONSUME_TIMES);\n    }\n\n    @Test\n    public void testAddReceiptHandle() {\n        Channel channel = new LocalChannel();\n        receiptHandleManager.addReceiptHandle(PROXY_CONTEXT, channel, GROUP, MSG_ID, messageReceiptHandle);\n        Mockito.when(metadataService.getSubscriptionGroupConfig(Mockito.any(), Mockito.eq(GROUP))).thenReturn(new SubscriptionGroupConfig());\n        Mockito.when(consumerManager.findChannel(Mockito.eq(GROUP), Mockito.eq(channel))).thenReturn(Mockito.mock(ClientChannelInfo.class));\n        receiptHandleManager.scheduleRenewTask();\n        Mockito.verify(messagingProcessor, Mockito.timeout(1000).times(1))\n            .changeInvisibleTime(Mockito.any(ProxyContext.class), Mockito.any(ReceiptHandle.class), Mockito.eq(MESSAGE_ID),\n                Mockito.eq(GROUP), Mockito.eq(TOPIC), Mockito.eq(ConfigurationManager.getProxyConfig().getDefaultInvisibleTimeMills()));\n    }\n\n    @Test\n    public void testAddDuplicationMessage() {\n        ProxyConfig config = ConfigurationManager.getProxyConfig();\n        Channel channel = PROXY_CONTEXT.getVal(ContextVariable.CHANNEL);\n        {\n            String receiptHandle = ReceiptHandle.builder()\n                .startOffset(0L)\n                .retrieveTime(System.currentTimeMillis() - INVISIBLE_TIME + config.getRenewAheadTimeMillis() - 1000)\n                .invisibleTime(INVISIBLE_TIME)\n                .reviveQueueId(1)\n                .topicType(ReceiptHandle.NORMAL_TOPIC)\n                .brokerName(BROKER_NAME)\n                .queueId(QUEUE_ID)\n                .offset(OFFSET)\n                .commitLogOffset(0L)\n                .build().encode();\n            MessageReceiptHandle messageReceiptHandle = new MessageReceiptHandle(GROUP, TOPIC, QUEUE_ID, receiptHandle, MESSAGE_ID, OFFSET,\n                RECONSUME_TIMES);\n            receiptHandleManager.addReceiptHandle(PROXY_CONTEXT, channel, GROUP, MSG_ID, messageReceiptHandle);\n        }\n        receiptHandleManager.addReceiptHandle(PROXY_CONTEXT, channel, GROUP, MSG_ID, messageReceiptHandle);\n        Mockito.when(metadataService.getSubscriptionGroupConfig(Mockito.any(), Mockito.eq(GROUP))).thenReturn(new SubscriptionGroupConfig());\n        Mockito.when(consumerManager.findChannel(Mockito.eq(GROUP), Mockito.eq(channel))).thenReturn(Mockito.mock(ClientChannelInfo.class));\n        receiptHandleManager.scheduleRenewTask();\n        ArgumentCaptor<ReceiptHandle> handleArgumentCaptor = ArgumentCaptor.forClass(ReceiptHandle.class);\n        Mockito.verify(messagingProcessor, Mockito.timeout(1000).times(1))\n            .changeInvisibleTime(Mockito.any(ProxyContext.class), handleArgumentCaptor.capture(), Mockito.eq(MESSAGE_ID),\n                Mockito.eq(GROUP), Mockito.eq(TOPIC), Mockito.eq(ConfigurationManager.getProxyConfig().getDefaultInvisibleTimeMills()));\n\n        assertEquals(receiptHandle, handleArgumentCaptor.getValue().encode());\n    }\n\n    @Test\n    public void testRenewReceiptHandle() {\n        ProxyConfig config = ConfigurationManager.getProxyConfig();\n        Channel channel = PROXY_CONTEXT.getVal(ContextVariable.CHANNEL);\n        receiptHandleManager.addReceiptHandle(PROXY_CONTEXT, channel, GROUP, MSG_ID, messageReceiptHandle);\n        SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig();\n        Mockito.when(metadataService.getSubscriptionGroupConfig(Mockito.any(), Mockito.eq(GROUP))).thenReturn(groupConfig);\n        Mockito.when(consumerManager.findChannel(Mockito.eq(GROUP), Mockito.eq(channel))).thenReturn(Mockito.mock(ClientChannelInfo.class));\n        long newInvisibleTime = 18000L;\n\n        ReceiptHandle newReceiptHandleClass = ReceiptHandle.builder()\n            .startOffset(0L)\n            .retrieveTime(System.currentTimeMillis() - newInvisibleTime + config.getRenewAheadTimeMillis() - 5)\n            .invisibleTime(newInvisibleTime)\n            .reviveQueueId(1)\n            .topicType(ReceiptHandle.NORMAL_TOPIC)\n            .brokerName(BROKER_NAME)\n            .queueId(QUEUE_ID)\n            .offset(OFFSET)\n            .commitLogOffset(0L)\n            .build();\n        String newReceiptHandle = newReceiptHandleClass.encode();\n\n        RetryPolicy retryPolicy = new RenewStrategyPolicy();\n        AtomicInteger times = new AtomicInteger(0);\n\n        AckResult ackResult = new AckResult();\n        ackResult.setStatus(AckStatus.OK);\n        ackResult.setExtraInfo(newReceiptHandle);\n\n        Mockito.when(messagingProcessor.changeInvisibleTime(Mockito.any(ProxyContext.class), Mockito.any(ReceiptHandle.class), Mockito.eq(MESSAGE_ID),\n                Mockito.eq(GROUP), Mockito.eq(TOPIC), Mockito.eq(retryPolicy.nextDelayDuration(times.get()))))\n            .thenReturn(CompletableFuture.completedFuture(ackResult));\n        receiptHandleManager.scheduleRenewTask();\n\n        Mockito.verify(messagingProcessor, Mockito.timeout(1000).times(1))\n            .changeInvisibleTime(Mockito.any(ProxyContext.class), Mockito.argThat(r -> r.getInvisibleTime() == INVISIBLE_TIME), Mockito.eq(MESSAGE_ID),\n                Mockito.eq(GROUP), Mockito.eq(TOPIC), Mockito.eq(retryPolicy.nextDelayDuration(times.get())));\n        receiptHandleManager.scheduleRenewTask();\n\n        Mockito.verify(messagingProcessor, Mockito.timeout(1000).times(1))\n            .changeInvisibleTime(Mockito.any(ProxyContext.class), Mockito.argThat(r -> r.getInvisibleTime() == newInvisibleTime), Mockito.eq(MESSAGE_ID),\n                Mockito.eq(GROUP), Mockito.eq(TOPIC), Mockito.eq(retryPolicy.nextDelayDuration(times.incrementAndGet())));\n        receiptHandleManager.scheduleRenewTask();\n    }\n\n    @Test\n    public void testRenewExceedMaxRenewTimes() {\n        Channel channel = PROXY_CONTEXT.getVal(ContextVariable.CHANNEL);\n        Mockito.when(consumerManager.findChannel(Mockito.eq(GROUP), Mockito.eq(channel))).thenReturn(Mockito.mock(ClientChannelInfo.class));\n        receiptHandleManager.addReceiptHandle(PROXY_CONTEXT, channel, GROUP, MSG_ID, messageReceiptHandle);\n\n        CompletableFuture<AckResult> ackResultFuture = new CompletableFuture<>();\n        ackResultFuture.completeExceptionally(new MQClientException(0, \"error\"));\n\n        RetryPolicy retryPolicy = new RenewStrategyPolicy();\n\n        Mockito.when(messagingProcessor.changeInvisibleTime(Mockito.any(ProxyContext.class), Mockito.any(ReceiptHandle.class), Mockito.eq(MESSAGE_ID),\n                Mockito.eq(GROUP), Mockito.eq(TOPIC), Mockito.eq(retryPolicy.nextDelayDuration(messageReceiptHandle.getRenewTimes()))))\n            .thenReturn(ackResultFuture);\n\n        await().atMost(Duration.ofSeconds(3)).until(() -> {\n            receiptHandleManager.scheduleRenewTask();\n            try {\n                ReceiptHandleGroup receiptHandleGroup = receiptHandleManager.receiptHandleGroupMap.values().stream().findFirst().get();\n                return receiptHandleGroup.isEmpty();\n            } catch (Exception e) {\n                return false;\n            }\n        });\n\n        Mockito.verify(messagingProcessor, Mockito.times(3))\n            .changeInvisibleTime(Mockito.any(ProxyContext.class), Mockito.any(ReceiptHandle.class), Mockito.eq(MESSAGE_ID),\n                Mockito.eq(GROUP), Mockito.eq(TOPIC), Mockito.eq(retryPolicy.nextDelayDuration(messageReceiptHandle.getRenewTimes())));\n    }\n\n    @Test\n    public void testRenewWithInvalidHandle() {\n        Channel channel = PROXY_CONTEXT.getVal(ContextVariable.CHANNEL);\n        Mockito.when(consumerManager.findChannel(Mockito.eq(GROUP), Mockito.eq(channel))).thenReturn(Mockito.mock(ClientChannelInfo.class));\n        receiptHandleManager.addReceiptHandle(PROXY_CONTEXT, channel, GROUP, MSG_ID, messageReceiptHandle);\n\n        CompletableFuture<AckResult> ackResultFuture = new CompletableFuture<>();\n        ackResultFuture.completeExceptionally(new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, \"error\"));\n        Mockito.when(messagingProcessor.changeInvisibleTime(Mockito.any(ProxyContext.class), Mockito.any(ReceiptHandle.class), Mockito.eq(MESSAGE_ID),\n                Mockito.eq(GROUP), Mockito.eq(TOPIC), Mockito.eq(ConfigurationManager.getProxyConfig().getDefaultInvisibleTimeMills())))\n            .thenReturn(ackResultFuture);\n\n        await().atMost(Duration.ofSeconds(1)).until(() -> {\n            receiptHandleManager.scheduleRenewTask();\n            try {\n                ReceiptHandleGroup receiptHandleGroup = receiptHandleManager.receiptHandleGroupMap.values().stream().findFirst().get();\n                return receiptHandleGroup.isEmpty();\n            } catch (Exception e) {\n                return false;\n            }\n        });\n    }\n\n    @Test\n    public void testRenewWithErrorThenOK() {\n        ProxyConfig config = ConfigurationManager.getProxyConfig();\n        Channel channel = PROXY_CONTEXT.getVal(ContextVariable.CHANNEL);\n        Mockito.when(consumerManager.findChannel(Mockito.eq(GROUP), Mockito.eq(channel))).thenReturn(Mockito.mock(ClientChannelInfo.class));\n        receiptHandleManager.addReceiptHandle(PROXY_CONTEXT, channel, GROUP, MSG_ID, messageReceiptHandle);\n\n        AtomicInteger count = new AtomicInteger(0);\n        List<CompletableFuture<AckResult>> futureList = new ArrayList<>();\n        {\n            CompletableFuture<AckResult> ackResultFuture = new CompletableFuture<>();\n            ackResultFuture.completeExceptionally(new MQClientException(0, \"error\"));\n            futureList.add(ackResultFuture);\n            futureList.add(ackResultFuture);\n        }\n        {\n            long newInvisibleTime = 2000L;\n            ReceiptHandle newReceiptHandleClass = ReceiptHandle.builder()\n                .startOffset(0L)\n                .retrieveTime(System.currentTimeMillis() - newInvisibleTime + config.getRenewAheadTimeMillis() - 5)\n                .invisibleTime(newInvisibleTime)\n                .reviveQueueId(1)\n                .topicType(ReceiptHandle.NORMAL_TOPIC)\n                .brokerName(BROKER_NAME)\n                .queueId(QUEUE_ID)\n                .offset(OFFSET)\n                .commitLogOffset(0L)\n                .build();\n            String newReceiptHandle = newReceiptHandleClass.encode();\n            AckResult ackResult = new AckResult();\n            ackResult.setStatus(AckStatus.OK);\n            ackResult.setExtraInfo(newReceiptHandle);\n            futureList.add(CompletableFuture.completedFuture(ackResult));\n        }\n        {\n            CompletableFuture<AckResult> ackResultFuture = new CompletableFuture<>();\n            ackResultFuture.completeExceptionally(new MQClientException(0, \"error\"));\n            futureList.add(ackResultFuture);\n            futureList.add(ackResultFuture);\n            futureList.add(ackResultFuture);\n            futureList.add(ackResultFuture);\n        }\n\n        RetryPolicy retryPolicy = new RenewStrategyPolicy();\n        AtomicInteger times = new AtomicInteger(0);\n        for (int i = 0; i < 6; i++) {\n            Mockito.doAnswer((Answer<CompletableFuture<AckResult>>) mock -> {\n                return futureList.get(count.getAndIncrement());\n            }).when(messagingProcessor).changeInvisibleTime(Mockito.any(ProxyContext.class), Mockito.any(ReceiptHandle.class), Mockito.eq(MESSAGE_ID),\n                Mockito.eq(GROUP), Mockito.eq(TOPIC), Mockito.eq(retryPolicy.nextDelayDuration(times.getAndIncrement())));\n        }\n\n        await().pollDelay(Duration.ZERO).pollInterval(Duration.ofMillis(10)).atMost(Duration.ofSeconds(10)).until(() -> {\n            receiptHandleManager.scheduleRenewTask();\n            try {\n                ReceiptHandleGroup receiptHandleGroup = receiptHandleManager.receiptHandleGroupMap.values().stream().findFirst().get();\n                return receiptHandleGroup.isEmpty();\n            } catch (Exception e) {\n                return false;\n            }\n        });\n\n        assertEquals(6, count.get());\n    }\n\n    @Test\n    public void testRenewReceiptHandleWhenTimeout() {\n        long newInvisibleTime = 200L;\n        long maxRenewMs = ConfigurationManager.getProxyConfig().getRenewMaxTimeMillis();\n        String newReceiptHandle = ReceiptHandle.builder()\n            .startOffset(0L)\n            .retrieveTime(System.currentTimeMillis() - maxRenewMs)\n            .invisibleTime(newInvisibleTime)\n            .reviveQueueId(1)\n            .topicType(ReceiptHandle.NORMAL_TOPIC)\n            .brokerName(BROKER_NAME)\n            .queueId(QUEUE_ID)\n            .offset(OFFSET)\n            .commitLogOffset(0L)\n            .build().encode();\n        messageReceiptHandle = new MessageReceiptHandle(GROUP, TOPIC, QUEUE_ID, newReceiptHandle, MESSAGE_ID, OFFSET,\n            RECONSUME_TIMES);\n        Channel channel = PROXY_CONTEXT.getVal(ContextVariable.CHANNEL);\n        receiptHandleManager.addReceiptHandle(PROXY_CONTEXT, channel, GROUP, MSG_ID, messageReceiptHandle);\n        Mockito.when(consumerManager.findChannel(Mockito.eq(GROUP), Mockito.eq(channel))).thenReturn(Mockito.mock(ClientChannelInfo.class));\n        SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig();\n        Mockito.when(metadataService.getSubscriptionGroupConfig(Mockito.any(), Mockito.eq(GROUP))).thenReturn(groupConfig);\n        Mockito.when(messagingProcessor.changeInvisibleTime(Mockito.any(), Mockito.any(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(new AckResult()));\n        receiptHandleManager.scheduleRenewTask();\n        Mockito.verify(messagingProcessor, Mockito.timeout(1000).times(1))\n            .changeInvisibleTime(Mockito.any(ProxyContext.class), Mockito.any(ReceiptHandle.class), Mockito.eq(MESSAGE_ID),\n                Mockito.eq(GROUP), Mockito.eq(TOPIC), Mockito.eq(groupConfig.getGroupRetryPolicy().getRetryPolicy().nextDelayDuration(RECONSUME_TIMES)));\n\n        await().atMost(Duration.ofSeconds(1)).untilAsserted(() -> {\n            ReceiptHandleGroup receiptHandleGroup = receiptHandleManager.receiptHandleGroupMap.values().stream().findFirst().get();\n            assertTrue(receiptHandleGroup.isEmpty());\n        });\n    }\n\n    @Test\n    public void testRenewReceiptHandleWhenTimeoutWithNoSubscription() {\n        long newInvisibleTime = 0L;\n        String newReceiptHandle = ReceiptHandle.builder()\n            .startOffset(0L)\n            .retrieveTime(0)\n            .invisibleTime(newInvisibleTime)\n            .reviveQueueId(1)\n            .topicType(ReceiptHandle.NORMAL_TOPIC)\n            .brokerName(BROKER_NAME)\n            .queueId(QUEUE_ID)\n            .offset(OFFSET)\n            .commitLogOffset(0L)\n            .build().encode();\n        messageReceiptHandle = new MessageReceiptHandle(GROUP, TOPIC, QUEUE_ID, newReceiptHandle, MESSAGE_ID, OFFSET,\n            RECONSUME_TIMES);\n        Channel channel = PROXY_CONTEXT.getVal(ContextVariable.CHANNEL);\n        receiptHandleManager.addReceiptHandle(PROXY_CONTEXT, channel, GROUP, MSG_ID, messageReceiptHandle);\n        Mockito.when(consumerManager.findChannel(Mockito.eq(GROUP), Mockito.eq(channel))).thenReturn(Mockito.mock(ClientChannelInfo.class));\n        Mockito.when(metadataService.getSubscriptionGroupConfig(Mockito.any(), Mockito.eq(GROUP))).thenReturn(null);\n        Mockito.when(messagingProcessor.changeInvisibleTime(Mockito.any(), Mockito.any(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyLong()))\n            .thenReturn(CompletableFuture.completedFuture(new AckResult()));\n        receiptHandleManager.scheduleRenewTask();\n        await().atMost(Duration.ofSeconds(1)).until(() -> {\n            try {\n                ReceiptHandleGroup receiptHandleGroup = receiptHandleManager.receiptHandleGroupMap.values().stream().findFirst().get();\n                return receiptHandleGroup.isEmpty();\n            } catch (Exception e) {\n                return false;\n            }\n        });\n\n        Mockito.verify(messagingProcessor, Mockito.timeout(1000).times(0))\n            .changeInvisibleTime(Mockito.any(ProxyContext.class), Mockito.any(ReceiptHandle.class), Mockito.anyString(),\n                Mockito.anyString(), Mockito.anyString(), Mockito.anyLong());\n    }\n\n    @Test\n    public void testRenewReceiptHandleWhenNotArrivingTime() {\n        String newReceiptHandle = ReceiptHandle.builder()\n            .startOffset(0L)\n            .retrieveTime(System.currentTimeMillis())\n            .invisibleTime(INVISIBLE_TIME)\n            .reviveQueueId(1)\n            .topicType(ReceiptHandle.NORMAL_TOPIC)\n            .brokerName(BROKER_NAME)\n            .queueId(QUEUE_ID)\n            .offset(OFFSET)\n            .commitLogOffset(0L)\n            .build().encode();\n        messageReceiptHandle = new MessageReceiptHandle(GROUP, TOPIC, QUEUE_ID, newReceiptHandle, MESSAGE_ID, OFFSET,\n            RECONSUME_TIMES);\n        Channel channel = PROXY_CONTEXT.getVal(ContextVariable.CHANNEL);\n        receiptHandleManager.addReceiptHandle(PROXY_CONTEXT, channel, GROUP, MSG_ID, messageReceiptHandle);\n        SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig();\n        Mockito.when(metadataService.getSubscriptionGroupConfig(Mockito.any(), Mockito.eq(GROUP))).thenReturn(groupConfig);\n        Mockito.when(consumerManager.findChannel(Mockito.eq(GROUP), Mockito.eq(channel))).thenReturn(Mockito.mock(ClientChannelInfo.class));\n        receiptHandleManager.scheduleRenewTask();\n        Mockito.verify(messagingProcessor, Mockito.timeout(1000).times(0))\n            .changeInvisibleTime(Mockito.any(ProxyContext.class), Mockito.any(ReceiptHandle.class), Mockito.anyString(),\n                Mockito.anyString(), Mockito.anyString(), Mockito.anyLong());\n    }\n\n    @Test\n    public void testRemoveReceiptHandle() {\n        Channel channel = PROXY_CONTEXT.getVal(ContextVariable.CHANNEL);\n        receiptHandleManager.addReceiptHandle(PROXY_CONTEXT, channel, GROUP, MSG_ID, messageReceiptHandle);\n        receiptHandleManager.removeReceiptHandle(PROXY_CONTEXT, channel, GROUP, MSG_ID, receiptHandle);\n        SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig();\n        Mockito.when(metadataService.getSubscriptionGroupConfig(Mockito.any(), Mockito.eq(GROUP))).thenReturn(groupConfig);\n        receiptHandleManager.scheduleRenewTask();\n        Mockito.verify(messagingProcessor, Mockito.timeout(1000).times(0))\n            .changeInvisibleTime(Mockito.any(ProxyContext.class), Mockito.any(ReceiptHandle.class), Mockito.anyString(),\n                Mockito.anyString(), Mockito.anyString(), Mockito.anyLong());\n    }\n\n    @Test\n    public void testClearGroup() {\n        Channel channel = PROXY_CONTEXT.getVal(ContextVariable.CHANNEL);\n        receiptHandleManager.addReceiptHandle(PROXY_CONTEXT, channel, GROUP, MSG_ID, messageReceiptHandle);\n        receiptHandleManager.clearGroup(new ReceiptHandleGroupKey(channel, GROUP));\n        SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig();\n        Mockito.when(metadataService.getSubscriptionGroupConfig(Mockito.any(), Mockito.eq(GROUP))).thenReturn(groupConfig);\n        receiptHandleManager.scheduleRenewTask();\n        Mockito.verify(messagingProcessor, Mockito.timeout(1000).times(1))\n            .changeInvisibleTime(Mockito.any(ProxyContext.class), Mockito.any(ReceiptHandle.class), Mockito.eq(MESSAGE_ID),\n                Mockito.eq(GROUP), Mockito.eq(TOPIC), Mockito.eq(ConfigurationManager.getProxyConfig().getInvisibleTimeMillisWhenClear()));\n    }\n\n    @Test\n    public void testClientOffline() {\n        ArgumentCaptor<ConsumerIdsChangeListener> listenerArgumentCaptor = ArgumentCaptor.forClass(ConsumerIdsChangeListener.class);\n        Mockito.verify(consumerManager, Mockito.times(1)).appendConsumerIdsChangeListener(listenerArgumentCaptor.capture());\n        Channel channel = PROXY_CONTEXT.getVal(ContextVariable.CHANNEL);\n        receiptHandleManager.addReceiptHandle(PROXY_CONTEXT, channel, GROUP, MSG_ID, messageReceiptHandle);\n        listenerArgumentCaptor.getValue().handle(ConsumerGroupEvent.CLIENT_UNREGISTER, GROUP, new ClientChannelInfo(channel, \"\", LanguageCode.JAVA, 0));\n        assertTrue(receiptHandleManager.receiptHandleGroupMap.isEmpty());\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/service/relay/LocalProxyRelayServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.relay;\n\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.service.channel.SimpleChannelHandlerContext;\nimport org.apache.rocketmq.proxy.service.transaction.TransactionService;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingServer;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.body.CMResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.ArgumentCaptor;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class LocalProxyRelayServiceTest {\n    private LocalProxyRelayService localProxyRelayService;\n    @Mock\n    private BrokerController brokerControllerMock;\n    @Mock\n    private TransactionService transactionService;\n    @Mock\n    private NettyRemotingServer nettyRemotingServerMock;\n\n    @Before\n    public void setUp() {\n        localProxyRelayService = new LocalProxyRelayService(brokerControllerMock, transactionService);\n        Mockito.when(brokerControllerMock.getRemotingServer()).thenReturn(nettyRemotingServerMock);\n    }\n\n    @Test\n    public void testProcessGetConsumerRunningInfo() {\n        ConsumerRunningInfo runningInfo = new ConsumerRunningInfo();\n        runningInfo.setJstack(\"jstack\");\n        String remark = \"ok\";\n        int opaque = 123;\n        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_RUNNING_INFO, null);\n        remotingCommand.setOpaque(opaque);\n        GetConsumerRunningInfoRequestHeader requestHeader = new GetConsumerRunningInfoRequestHeader();\n        requestHeader.setJstackEnable(true);\n        ArgumentCaptor<RemotingCommand> argumentCaptor = ArgumentCaptor.forClass(RemotingCommand.class);\n        CompletableFuture<ProxyRelayResult<ConsumerRunningInfo>> future =\n            localProxyRelayService.processGetConsumerRunningInfo(ProxyContext.create(), remotingCommand, requestHeader);\n        future.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, remark, runningInfo));\n        Mockito.verify(nettyRemotingServerMock, Mockito.times(1))\n            .processResponseCommand(Mockito.any(SimpleChannelHandlerContext.class), argumentCaptor.capture());\n        RemotingCommand remotingCommand1 = argumentCaptor.getValue();\n        assertThat(remotingCommand1.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(remotingCommand1.getRemark()).isEqualTo(remark);\n        assertThat(remotingCommand1.getBody()).isEqualTo(runningInfo.encode());\n    }\n\n    @Test\n    public void testProcessConsumeMessageDirectly() {\n        ConsumeMessageDirectlyResultRequestHeader requestHeader = new ConsumeMessageDirectlyResultRequestHeader();\n        String remark = \"ok\";\n        int opaque = 123;\n        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.CONSUME_MESSAGE_DIRECTLY, null);\n        remotingCommand.setOpaque(opaque);\n        ConsumeMessageDirectlyResult result = new ConsumeMessageDirectlyResult();\n        result.setConsumeResult(CMResult.CR_SUCCESS);\n        ArgumentCaptor<RemotingCommand> argumentCaptor = ArgumentCaptor.forClass(RemotingCommand.class);\n        CompletableFuture<ProxyRelayResult<ConsumeMessageDirectlyResult>> future =\n            localProxyRelayService.processConsumeMessageDirectly(ProxyContext.create(), remotingCommand, requestHeader);\n        future.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, remark, result));\n        Mockito.verify(nettyRemotingServerMock, Mockito.times(1))\n            .processResponseCommand(Mockito.any(SimpleChannelHandlerContext.class), argumentCaptor.capture());\n        RemotingCommand remotingCommand1 = argumentCaptor.getValue();\n        assertThat(remotingCommand1.getCode()).isEqualTo(ResponseCode.SUCCESS);\n        assertThat(remotingCommand1.getRemark()).isEqualTo(remark);\n        assertThat(remotingCommand1.getBody()).isEqualTo(result.encode());\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/service/relay/ProxyChannelTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.relay;\n\nimport io.netty.channel.Channel;\nimport java.nio.charset.StandardCharsets;\nimport java.util.UUID;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.commons.lang3.NotImplementedException;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.proxy.service.transaction.TransactionData;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;\nimport org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;\nimport org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.NotifyUnsubscribeLiteRequestHeader;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.ArgumentCaptor;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.junit.Assert.assertArrayEquals;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ProxyChannelTest {\n\n    @Mock\n    private ProxyRelayService proxyRelayService;\n\n    protected abstract static class MockProxyChannel extends ProxyChannel {\n\n        protected MockProxyChannel(ProxyRelayService proxyRelayService, Channel parent,\n            String remoteAddress, String localAddress) {\n            super(proxyRelayService, parent, remoteAddress, localAddress);\n        }\n\n        @Override\n        public boolean isOpen() {\n            return false;\n        }\n\n        @Override\n        public boolean isActive() {\n            return false;\n        }\n    }\n\n    @Test\n    public void testWriteAndFlush() throws Exception {\n        when(this.proxyRelayService.processCheckTransactionState(any(), any(), any(), any()))\n            .thenReturn(new RelayData<>(mock(TransactionData.class), new CompletableFuture<>()));\n\n        ArgumentCaptor<ConsumeMessageDirectlyResultRequestHeader> consumeMessageDirectlyArgumentCaptor =\n            ArgumentCaptor.forClass(ConsumeMessageDirectlyResultRequestHeader.class);\n        when(this.proxyRelayService.processConsumeMessageDirectly(any(), any(), consumeMessageDirectlyArgumentCaptor.capture()))\n            .thenReturn(new CompletableFuture<>());\n\n        ArgumentCaptor<GetConsumerRunningInfoRequestHeader> getConsumerRunningInfoArgumentCaptor =\n            ArgumentCaptor.forClass(GetConsumerRunningInfoRequestHeader.class);\n        when(this.proxyRelayService.processGetConsumerRunningInfo(any(), any(), getConsumerRunningInfoArgumentCaptor.capture()))\n            .thenReturn(new CompletableFuture<>());\n\n        CheckTransactionStateRequestHeader checkTransactionStateRequestHeader = new CheckTransactionStateRequestHeader();\n        checkTransactionStateRequestHeader.setTransactionId(MessageClientIDSetter.createUniqID());\n        RemotingCommand checkTransactionRequest = RemotingCommand.createRequestCommand(RequestCode.CHECK_TRANSACTION_STATE, checkTransactionStateRequestHeader);\n        MessageExt transactionMessageExt = new MessageExt();\n        transactionMessageExt.setTopic(\"topic\");\n        transactionMessageExt.setTags(\"tags\");\n        transactionMessageExt.setBornHost(NetworkUtil.string2SocketAddress(\"127.0.0.2:8888\"));\n        transactionMessageExt.setStoreHost(NetworkUtil.string2SocketAddress(\"127.0.0.1:10911\"));\n        transactionMessageExt.setBody(UUID.randomUUID().toString().getBytes(StandardCharsets.UTF_8));\n        transactionMessageExt.setMsgId(MessageClientIDSetter.createUniqID());\n        checkTransactionRequest.setBody(MessageDecoder.encode(transactionMessageExt, false));\n\n        GetConsumerRunningInfoRequestHeader consumerRunningInfoRequestHeader = new GetConsumerRunningInfoRequestHeader();\n        consumerRunningInfoRequestHeader.setConsumerGroup(\"group\");\n        consumerRunningInfoRequestHeader.setClientId(\"clientId\");\n        RemotingCommand consumerRunningInfoRequest = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_RUNNING_INFO, consumerRunningInfoRequestHeader);\n\n        ConsumeMessageDirectlyResultRequestHeader consumeMessageDirectlyResultRequestHeader = new ConsumeMessageDirectlyResultRequestHeader();\n        consumeMessageDirectlyResultRequestHeader.setConsumerGroup(\"group\");\n        consumeMessageDirectlyResultRequestHeader.setClientId(\"clientId\");\n        MessageExt consumeMessageDirectlyMessageExt = new MessageExt();\n        consumeMessageDirectlyMessageExt.setTopic(\"topic\");\n        consumeMessageDirectlyMessageExt.setTags(\"tags\");\n        consumeMessageDirectlyMessageExt.setBornHost(NetworkUtil.string2SocketAddress(\"127.0.0.2:8888\"));\n        consumeMessageDirectlyMessageExt.setStoreHost(NetworkUtil.string2SocketAddress(\"127.0.0.1:10911\"));\n        consumeMessageDirectlyMessageExt.setBody(UUID.randomUUID().toString().getBytes(StandardCharsets.UTF_8));\n        consumeMessageDirectlyMessageExt.setMsgId(MessageClientIDSetter.createUniqID());\n        RemotingCommand consumeMessageDirectlyResult = RemotingCommand.createRequestCommand(RequestCode.CONSUME_MESSAGE_DIRECTLY, consumeMessageDirectlyResultRequestHeader);\n        consumeMessageDirectlyResult.setBody(MessageDecoder.encode(consumeMessageDirectlyMessageExt, false));\n\n        MockProxyChannel channel = new MockProxyChannel(this.proxyRelayService, null, \"127.0.0.2:8888\", \"127.0.0.1:10911\") {\n            @Override\n            protected CompletableFuture<Void> processOtherMessage(Object msg) {\n                return CompletableFuture.completedFuture(null);\n            }\n\n            @Override\n            protected CompletableFuture<Void> processCheckTransaction(CheckTransactionStateRequestHeader header,\n                MessageExt messageExt, TransactionData transactionData, CompletableFuture<ProxyRelayResult<Void>> responseFuture) {\n                assertEquals(checkTransactionStateRequestHeader, header);\n                assertArrayEquals(transactionMessageExt.getBody(), messageExt.getBody());\n                return CompletableFuture.completedFuture(null);\n            }\n\n            @Override\n            protected CompletableFuture<Void> processGetConsumerRunningInfo(RemotingCommand command,\n                GetConsumerRunningInfoRequestHeader header,\n                CompletableFuture<ProxyRelayResult<ConsumerRunningInfo>> responseFuture) {\n                assertEquals(consumerRunningInfoRequestHeader, getConsumerRunningInfoArgumentCaptor.getValue());\n                assertEquals(consumerRunningInfoRequestHeader, header);\n                return CompletableFuture.completedFuture(null);\n            }\n\n            @Override\n            protected CompletableFuture<Void> processConsumeMessageDirectly(RemotingCommand command,\n                ConsumeMessageDirectlyResultRequestHeader header, MessageExt messageExt,\n                CompletableFuture<ProxyRelayResult<ConsumeMessageDirectlyResult>> responseFuture) {\n                assertEquals(consumeMessageDirectlyResultRequestHeader, consumeMessageDirectlyArgumentCaptor.getValue());\n                assertEquals(consumeMessageDirectlyResultRequestHeader, header);\n                assertArrayEquals(consumeMessageDirectlyMessageExt.getBody(), messageExt.getBody());\n                return CompletableFuture.completedFuture(null);\n            }\n\n            @Override\n            protected CompletableFuture<Void> processNotifyUnsubscribeLite(NotifyUnsubscribeLiteRequestHeader header) {\n                throw new NotImplementedException();\n            }\n        };\n\n        assertTrue(channel.writeAndFlush(checkTransactionRequest).isSuccess());\n        assertTrue(channel.writeAndFlush(consumerRunningInfoRequest).isSuccess());\n        assertTrue(channel.writeAndFlush(consumeMessageDirectlyResult).isSuccess());\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/service/route/ClusterTopicRouteServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.route;\n\nimport com.github.benmanes.caffeine.cache.CacheLoader;\nimport com.github.benmanes.caffeine.cache.Caffeine;\nimport com.github.benmanes.caffeine.cache.LoadingCache;\nimport com.google.common.net.HostAndPort;\n\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.thread.ThreadPoolMonitor;\nimport org.apache.rocketmq.proxy.common.Address;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.service.BaseServiceTest;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.assertj.core.util.Lists;\nimport org.checkerframework.checker.nullness.qual.NonNull;\nimport org.checkerframework.checker.nullness.qual.Nullable;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.assertj.core.api.Assertions.catchThrowableOfType;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.when;\n\npublic class ClusterTopicRouteServiceTest extends BaseServiceTest {\n\n    private ClusterTopicRouteService topicRouteService;\n\n    protected static final String BROKER2_NAME = \"broker2\";\n    protected static final String BROKER2_ADDR = \"127.0.0.2:10911\";\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        this.topicRouteService = new ClusterTopicRouteService(this.mqClientAPIFactory);\n\n        when(this.mqClientAPIExt.getTopicRouteInfoFromNameServer(eq(TOPIC), anyLong())).thenReturn(topicRouteData);\n        when(this.mqClientAPIExt.getTopicRouteInfoFromNameServer(eq(ERR_TOPIC), anyLong())).thenThrow(new MQClientException(ResponseCode.TOPIC_NOT_EXIST, \"\"));\n\n        // build broker\n        BrokerData brokerData = new BrokerData();\n        brokerData.setCluster(CLUSTER_NAME);\n        brokerData.setBrokerName(BROKER_NAME);\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(MixAll.MASTER_ID, BROKER_ADDR);\n        brokerData.setBrokerAddrs(brokerAddrs);\n\n        // build broker2\n        BrokerData broke2Data = new BrokerData();\n        broke2Data.setCluster(CLUSTER_NAME);\n        broke2Data.setBrokerName(BROKER2_NAME);\n        HashMap<Long, String> broker2Addrs = new HashMap<>();\n        broker2Addrs.put(MixAll.MASTER_ID, BROKER2_ADDR);\n        broke2Data.setBrokerAddrs(broker2Addrs);\n\n        // add brokers\n        TopicRouteData brokerTopicRouteData = new TopicRouteData();\n        brokerTopicRouteData.setBrokerDatas(Lists.newArrayList(brokerData, broke2Data));\n\n        // add queue data\n        QueueData queueData = new QueueData();\n        queueData.setBrokerName(BROKER_NAME);\n\n        QueueData queue2Data = new QueueData();\n        queue2Data.setBrokerName(BROKER2_NAME);\n        brokerTopicRouteData.setQueueDatas(Lists.newArrayList(queueData, queue2Data));\n        when(this.mqClientAPIExt.getTopicRouteInfoFromNameServer(eq(BROKER_NAME), anyLong())).thenReturn(brokerTopicRouteData);\n        when(this.mqClientAPIExt.getTopicRouteInfoFromNameServer(eq(BROKER2_NAME), anyLong())).thenReturn(brokerTopicRouteData);\n    }\n\n    @Test\n    public void testGetCurrentMessageQueueView() throws Throwable {\n        ProxyContext ctx = ProxyContext.create();\n        MQClientException exception = catchThrowableOfType(() -> this.topicRouteService.getCurrentMessageQueueView(ctx, ERR_TOPIC), MQClientException.class);\n        assertTrue(TopicRouteHelper.isTopicNotExistError(exception));\n        assertEquals(1, this.topicRouteService.topicCache.asMap().size());\n\n        assertNotNull(this.topicRouteService.getCurrentMessageQueueView(ctx, TOPIC));\n        assertEquals(2, this.topicRouteService.topicCache.asMap().size());\n    }\n\n    @Test\n    public void testGetBrokerAddr() throws Throwable {\n        ProxyContext ctx = ProxyContext.create();\n        assertEquals(BROKER_ADDR, topicRouteService.getBrokerAddr(ctx, BROKER_NAME));\n        assertEquals(BROKER2_ADDR, topicRouteService.getBrokerAddr(ctx, BROKER2_NAME));\n    }\n\n    @Test\n    public void testGetTopicRouteForProxy() throws Throwable {\n        ProxyContext ctx = ProxyContext.create();\n        List<Address> addressList = Lists.newArrayList(new Address(Address.AddressScheme.IPv4, HostAndPort.fromParts(\"127.0.0.1\", 8888)));\n        ProxyTopicRouteData proxyTopicRouteData = this.topicRouteService.getTopicRouteForProxy(ctx, addressList, TOPIC);\n\n        assertEquals(1, proxyTopicRouteData.getBrokerDatas().size());\n        assertEquals(addressList, proxyTopicRouteData.getBrokerDatas().get(0).getBrokerAddrs().get(MixAll.MASTER_ID));\n    }\n\n    @Test\n    public void testTopicRouteCaffeineCache() throws InterruptedException {\n        String key = \"abc\";\n        String value = key;\n        final AtomicBoolean throwException = new AtomicBoolean();\n        ThreadPoolExecutor cacheRefreshExecutor = ThreadPoolMonitor.createAndMonitor(\n            10, 10, 30L, TimeUnit.SECONDS, \"test\", 10);\n        LoadingCache<String /* topicName */, String> topicCache = Caffeine.newBuilder().maximumSize(30).\n            refreshAfterWrite(2, TimeUnit.SECONDS).executor(cacheRefreshExecutor).build(new CacheLoader<String, String>() {\n                @Override public @Nullable String load(@NonNull String key) throws Exception {\n                    try {\n                        if (throwException.get()) {\n                            throw new RuntimeException();\n                        } else {\n                            throwException.set(true);\n                            return value;\n                        }\n                    } catch (Exception e) {\n                        if (TopicRouteHelper.isTopicNotExistError(e)) {\n                            return \"\";\n                        }\n                        throw e;\n                    }\n                }\n\n                @Override\n                public @Nullable String reload(@NonNull String key, @NonNull String oldValue) throws Exception {\n                    try {\n                        return load(key);\n                    } catch (Exception e) {\n                        return oldValue;\n                    }\n                }\n            });\n        assertThat(value).isEqualTo(topicCache.get(key));\n        TimeUnit.SECONDS.sleep(5);\n        assertThat(value).isEqualTo(topicCache.get(key));\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.route;\n\nimport com.google.common.net.HostAndPort;\nimport java.util.ArrayList;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.topic.TopicConfigManager;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.proxy.common.Address;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.service.BaseServiceTest;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.assertj.core.util.Lists;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.Mock;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.when;\n\npublic class LocalTopicRouteServiceTest extends BaseServiceTest {\n\n    private static final String LOCAL_BROKER_NAME = \"localBroker\";\n    private static final String LOCAL_CLUSTER_NAME = \"localCluster\";\n    private static final String LOCAL_HOST = \"127.0.0.2\";\n    private static final int LOCAL_PORT = 10911;\n    private static final String LOCAL_ADDR = LOCAL_HOST + \":\" + LOCAL_PORT;\n    @Mock\n    private BrokerController brokerController;\n    @Mock\n    private TopicConfigManager topicConfigManager;\n    private ConcurrentMap<String, TopicConfig> topicConfigTable = new ConcurrentHashMap<>();\n    private BrokerConfig brokerConfig = new BrokerConfig();\n    private LocalTopicRouteService topicRouteService;\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        this.brokerConfig.setBrokerName(LOCAL_BROKER_NAME);\n        this.brokerConfig.setBrokerClusterName(LOCAL_CLUSTER_NAME);\n\n        when(this.brokerController.getBrokerAddr()).thenReturn(LOCAL_ADDR);\n        when(this.brokerController.getBrokerConfig()).thenReturn(this.brokerConfig);\n        when(this.brokerController.getTopicConfigManager()).thenReturn(this.topicConfigManager);\n        when(this.topicConfigManager.getTopicConfigTable()).thenReturn(this.topicConfigTable);\n\n        this.topicRouteService = new LocalTopicRouteService(this.brokerController, this.mqClientAPIFactory);\n\n        when(this.mqClientAPIExt.getTopicRouteInfoFromNameServer(eq(TOPIC), anyLong())).thenReturn(topicRouteData);\n        when(this.mqClientAPIExt.getTopicRouteInfoFromNameServer(eq(ERR_TOPIC), anyLong())).thenThrow(new MQClientException(ResponseCode.TOPIC_NOT_EXIST, \"\"));\n    }\n\n    @Test\n    public void testGetCurrentMessageQueueView() throws Throwable {\n        ProxyContext ctx = ProxyContext.create();\n        this.topicConfigTable.put(TOPIC, new TopicConfig(TOPIC, 3, 2, PermName.PERM_WRITE | PermName.PERM_READ));\n        MessageQueueView messageQueueView = this.topicRouteService.getCurrentMessageQueueView(ctx, TOPIC);\n        assertEquals(3, messageQueueView.getReadSelector().getQueues().size());\n        assertEquals(2, messageQueueView.getWriteSelector().getQueues().size());\n        assertEquals(1, messageQueueView.getReadSelector().getBrokerActingQueues().size());\n        assertEquals(1, messageQueueView.getWriteSelector().getBrokerActingQueues().size());\n\n        assertEquals(LOCAL_ADDR, messageQueueView.getReadSelector().selectOne(true).getBrokerAddr());\n        assertEquals(LOCAL_BROKER_NAME, messageQueueView.getReadSelector().selectOne(true).getBrokerName());\n        assertEquals(messageQueueView.getReadSelector().selectOne(true), messageQueueView.getWriteSelector().selectOne(true));\n    }\n\n    @Test\n    public void testGetTopicRouteForProxy() throws Throwable {\n        ProxyContext ctx = ProxyContext.create();\n        ProxyTopicRouteData proxyTopicRouteData = this.topicRouteService.getTopicRouteForProxy(ctx, new ArrayList<>(), TOPIC);\n\n        assertEquals(1, proxyTopicRouteData.getBrokerDatas().size());\n        assertEquals(\n            Lists.newArrayList(new Address(Address.AddressScheme.IPv4, HostAndPort.fromParts(\n                HostAndPort.fromString(BROKER_ADDR).getHost(),\n                ConfigurationManager.getProxyConfig().getGrpcServerPort()))),\n            proxyTopicRouteData.getBrokerDatas().get(0).getBrokerAddrs().get(MixAll.MASTER_ID));\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueuePenalizerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.route;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.commons.lang3.tuple.Pair;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertTrue;\n\npublic class MessageQueuePenalizerTest {\n\n    /**\n     * Test evaluatePenalty with null messageQueue should throw NullPointerException\n     */\n    @Test(expected = NullPointerException.class)\n    public void testEvaluatePenalty_NullMessageQueue() {\n        List<MessageQueuePenalizer<MessageQueue>> penalizers = new ArrayList<>();\n        penalizers.add(mq -> 10);\n        MessageQueuePenalizer.evaluatePenalty(null, penalizers);\n    }\n\n    /**\n     * Test evaluatePenalty with null penalizers should return 0\n     */\n    @Test\n    public void testEvaluatePenalty_NullPenalizers() {\n        MessageQueue mq = new MessageQueue(\"topic\", \"broker\", 0);\n        int penalty = MessageQueuePenalizer.evaluatePenalty(mq, null);\n        assertEquals(0, penalty);\n    }\n\n    /**\n     * Test evaluatePenalty with empty penalizers should return 0\n     */\n    @Test\n    public void testEvaluatePenalty_EmptyPenalizers() {\n        MessageQueue mq = new MessageQueue(\"topic\", \"broker\", 0);\n        int penalty = MessageQueuePenalizer.evaluatePenalty(mq, Collections.emptyList());\n        assertEquals(0, penalty);\n    }\n\n    /**\n     * Test evaluatePenalty aggregates penalties from multiple penalizers by summing them up\n     */\n    @Test\n    public void testEvaluatePenalty_MultiplePenalizers() {\n        MessageQueue mq = new MessageQueue(\"topic\", \"broker\", 0);\n        List<MessageQueuePenalizer<MessageQueue>> penalizers = Arrays.asList(\n            q -> 10,\n            q -> 20,\n            q -> 5\n        );\n        int penalty = MessageQueuePenalizer.evaluatePenalty(mq, penalizers);\n        assertEquals(35, penalty);\n    }\n\n    /**\n     * Test evaluatePenalty with negative penalties (sum should still work)\n     */\n    @Test\n    public void testEvaluatePenalty_NegativePenalties() {\n        MessageQueue mq = new MessageQueue(\"topic\", \"broker\", 0);\n        List<MessageQueuePenalizer<MessageQueue>> penalizers = Arrays.asList(\n            q -> -5,\n            q -> 10,\n            q -> -3\n        );\n        int penalty = MessageQueuePenalizer.evaluatePenalty(mq, penalizers);\n        assertEquals(2, penalty);\n    }\n\n    /**\n     * Test selectLeastPenalty with null queues should return null\n     */\n    @Test\n    public void testSelectLeastPenalty_NullQueues() {\n        List<MessageQueuePenalizer<MessageQueue>> penalizers = Collections.singletonList(mq -> 10);\n        AtomicInteger startIndex = new AtomicInteger(0);\n        Pair<MessageQueue, Integer> result = MessageQueuePenalizer.selectLeastPenalty(null, penalizers, startIndex);\n        assertNull(result);\n    }\n\n    /**\n     * Test selectLeastPenalty with empty queues should return null\n     */\n    @Test\n    public void testSelectLeastPenalty_EmptyQueues() {\n        List<MessageQueuePenalizer<MessageQueue>> penalizers = Collections.singletonList(mq -> 10);\n        AtomicInteger startIndex = new AtomicInteger(0);\n        Pair<MessageQueue, Integer> result = MessageQueuePenalizer.selectLeastPenalty(\n            Collections.emptyList(), penalizers, startIndex);\n        assertNull(result);\n    }\n\n    /**\n     * Test selectLeastPenalty selects the queue with the lowest penalty\n     */\n    @Test\n    public void testSelectLeastPenalty_LowestPenalty() {\n        MessageQueue mq0 = new MessageQueue(\"topic\", \"broker\", 0);\n        MessageQueue mq1 = new MessageQueue(\"topic\", \"broker\", 1);\n        MessageQueue mq2 = new MessageQueue(\"topic\", \"broker\", 2);\n        List<MessageQueue> queues = Arrays.asList(mq0, mq1, mq2);\n\n        // Penalizer that assigns different penalties based on queue id\n        List<MessageQueuePenalizer<MessageQueue>> penalizers = Collections.singletonList(\n            mq -> mq.getQueueId() == 0 ? 50 : (mq.getQueueId() == 1 ? 10 : 30)\n        );\n\n        AtomicInteger startIndex = new AtomicInteger(0);\n        Pair<MessageQueue, Integer> result = MessageQueuePenalizer.selectLeastPenalty(queues, penalizers, startIndex);\n\n        assertNotNull(result);\n        assertEquals(mq1, result.getLeft());\n        assertEquals(10, result.getRight().intValue());\n    }\n\n    /**\n     * Test selectLeastPenalty short-circuits when penalty <= 0\n     */\n    @Test\n    public void testSelectLeastPenalty_ShortCircuitZeroPenalty() {\n        MessageQueue mq0 = new MessageQueue(\"topic\", \"broker\", 0);\n        MessageQueue mq1 = new MessageQueue(\"topic\", \"broker\", 1);\n        MessageQueue mq2 = new MessageQueue(\"topic\", \"broker\", 2);\n        List<MessageQueue> queues = Arrays.asList(mq0, mq1, mq2);\n\n        // mq1 has penalty 0, should short-circuit\n        List<MessageQueuePenalizer<MessageQueue>> penalizers = Collections.singletonList(\n            mq -> mq.getQueueId() == 0 ? 50 : (mq.getQueueId() == 1 ? 0 : 30)\n        );\n\n        AtomicInteger startIndex = new AtomicInteger(0);\n        Pair<MessageQueue, Integer> result = MessageQueuePenalizer.selectLeastPenalty(queues, penalizers, startIndex);\n\n        assertNotNull(result);\n        assertEquals(mq1, result.getLeft());\n        assertEquals(0, result.getRight().intValue());\n    }\n\n    /**\n     * Test selectLeastPenalty short-circuits when penalty is negative\n     */\n    @Test\n    public void testSelectLeastPenalty_ShortCircuitNegativePenalty() {\n        MessageQueue mq0 = new MessageQueue(\"topic\", \"broker\", 0);\n        MessageQueue mq1 = new MessageQueue(\"topic\", \"broker\", 1);\n        MessageQueue mq2 = new MessageQueue(\"topic\", \"broker\", 2);\n        List<MessageQueue> queues = Arrays.asList(mq0, mq1, mq2);\n\n        // mq1 has penalty -5, should short-circuit\n        List<MessageQueuePenalizer<MessageQueue>> penalizers = Collections.singletonList(\n            mq -> mq.getQueueId() == 0 ? 50 : (mq.getQueueId() == 1 ? -5 : 30)\n        );\n\n        AtomicInteger startIndex = new AtomicInteger(0);\n        Pair<MessageQueue, Integer> result = MessageQueuePenalizer.selectLeastPenalty(queues, penalizers, startIndex);\n\n        assertNotNull(result);\n        assertEquals(mq1, result.getLeft());\n        assertEquals(-5, result.getRight().intValue());\n    }\n\n    /**\n     * Test selectLeastPenalty with round-robin behavior (rotating start index)\n     * Verifies that startIndex affects the iteration order\n     */\n    @Test\n    public void testSelectLeastPenalty_RoundRobinStartIndex() {\n        MessageQueue mq0 = new MessageQueue(\"topic\", \"broker\", 0);\n        MessageQueue mq1 = new MessageQueue(\"topic\", \"broker\", 1);\n        MessageQueue mq2 = new MessageQueue(\"topic\", \"broker\", 2);\n        List<MessageQueue> queues = Arrays.asList(mq0, mq1, mq2);\n\n        // All queues have penalty 0, so whichever is encountered first will be returned\n        List<MessageQueuePenalizer<MessageQueue>> penalizers = Collections.singletonList(mq -> 0);\n\n        // Starting from index 0\n        AtomicInteger startIndex1 = new AtomicInteger(0);\n        Pair<MessageQueue, Integer> result1 = MessageQueuePenalizer.selectLeastPenalty(queues, penalizers, startIndex1);\n        assertNotNull(result1);\n        assertEquals(mq0, result1.getLeft());\n\n        // Starting from index 1\n        AtomicInteger startIndex2 = new AtomicInteger(1);\n        Pair<MessageQueue, Integer> result2 = MessageQueuePenalizer.selectLeastPenalty(queues, penalizers, startIndex2);\n        assertNotNull(result2);\n        assertEquals(mq1, result2.getLeft());\n\n        // Starting from index 2\n        AtomicInteger startIndex3 = new AtomicInteger(2);\n        Pair<MessageQueue, Integer> result3 = MessageQueuePenalizer.selectLeastPenalty(queues, penalizers, startIndex3);\n        assertNotNull(result3);\n        assertEquals(mq2, result3.getLeft());\n    }\n\n    /**\n     * Test selectLeastPenalty increments startIndex for each iteration\n     */\n    @Test\n    public void testSelectLeastPenalty_IncrementStartIndex() {\n        MessageQueue mq0 = new MessageQueue(\"topic\", \"broker\", 0);\n        MessageQueue mq1 = new MessageQueue(\"topic\", \"broker\", 1);\n        MessageQueue mq2 = new MessageQueue(\"topic\", \"broker\", 2);\n        List<MessageQueue> queues = Arrays.asList(mq0, mq1, mq2);\n\n        List<MessageQueuePenalizer<MessageQueue>> penalizers = Collections.singletonList(mq -> 10);\n\n        AtomicInteger startIndex = new AtomicInteger(0);\n        MessageQueuePenalizer.selectLeastPenalty(queues, penalizers, startIndex);\n\n        // After iterating through 3 queues, startIndex should be incremented 3 times\n        assertEquals(3, startIndex.get());\n    }\n\n    /**\n     * Test selectLeastPenalty handles startIndex wrapping with Math.floorMod\n     */\n    @Test\n    public void testSelectLeastPenalty_StartIndexWrapping() {\n        MessageQueue mq0 = new MessageQueue(\"topic\", \"broker\", 0);\n        MessageQueue mq1 = new MessageQueue(\"topic\", \"broker\", 1);\n        MessageQueue mq2 = new MessageQueue(\"topic\", \"broker\", 2);\n        List<MessageQueue> queues = Arrays.asList(mq0, mq1, mq2);\n\n        List<MessageQueuePenalizer<MessageQueue>> penalizers = Collections.singletonList(mq -> 0);\n\n        // Start with large index to test wrapping\n        AtomicInteger startIndex = new AtomicInteger(100);\n        Pair<MessageQueue, Integer> result = MessageQueuePenalizer.selectLeastPenalty(queues, penalizers, startIndex);\n\n        assertNotNull(result);\n        // 100 % 3 = 1, so should start from mq1\n        assertEquals(mq1, result.getLeft());\n    }\n\n    /**\n     * Test selectLeastPenaltyWithPriority with null queuesWithPriority should return null\n     */\n    @Test\n    public void testSelectLeastPenaltyWithPriority_NullQueues() {\n        List<MessageQueuePenalizer<MessageQueue>> penalizers = Collections.singletonList(mq -> 10);\n        AtomicInteger startIndex = new AtomicInteger(0);\n        Pair<MessageQueue, Integer> result = MessageQueuePenalizer.selectLeastPenaltyWithPriority(\n            null, penalizers, startIndex);\n        assertNull(result);\n    }\n\n    /**\n     * Test selectLeastPenaltyWithPriority with empty queuesWithPriority should return null\n     */\n    @Test\n    public void testSelectLeastPenaltyWithPriority_EmptyQueues() {\n        List<MessageQueuePenalizer<MessageQueue>> penalizers = Collections.singletonList(mq -> 10);\n        AtomicInteger startIndex = new AtomicInteger(0);\n        Pair<MessageQueue, Integer> result = MessageQueuePenalizer.selectLeastPenaltyWithPriority(\n            Collections.emptyList(), penalizers, startIndex);\n        assertNull(result);\n    }\n\n    /**\n     * Test selectLeastPenaltyWithPriority with single priority group delegates to selectLeastPenalty\n     */\n    @Test\n    public void testSelectLeastPenaltyWithPriority_SinglePriorityGroup() {\n        MessageQueue mq0 = new MessageQueue(\"topic\", \"broker\", 0);\n        MessageQueue mq1 = new MessageQueue(\"topic\", \"broker\", 1);\n        List<MessageQueue> queues = Arrays.asList(mq0, mq1);\n\n        List<MessageQueuePenalizer<MessageQueue>> penalizers = Collections.singletonList(\n            mq -> mq.getQueueId() == 0 ? 20 : 10\n        );\n\n        AtomicInteger startIndex = new AtomicInteger(0);\n        Pair<MessageQueue, Integer> result = MessageQueuePenalizer.selectLeastPenaltyWithPriority(\n            Collections.singletonList(queues), penalizers, startIndex);\n\n        assertNotNull(result);\n        assertEquals(mq1, result.getLeft());\n        assertEquals(10, result.getRight().intValue());\n    }\n\n    /**\n     * Test selectLeastPenaltyWithPriority selects queue with lowest penalty across multiple priority groups\n     */\n    @Test\n    public void testSelectLeastPenaltyWithPriority_MultiplePriorityGroups() {\n        // Priority group 1 (higher priority)\n        MessageQueue mq0 = new MessageQueue(\"topic\", \"broker-high\", 0);\n        MessageQueue mq1 = new MessageQueue(\"topic\", \"broker-high\", 1);\n        List<MessageQueue> highPriorityQueues = Arrays.asList(mq0, mq1);\n\n        // Priority group 2 (lower priority)\n        MessageQueue mq2 = new MessageQueue(\"topic\", \"broker-low\", 0);\n        MessageQueue mq3 = new MessageQueue(\"topic\", \"broker-low\", 1);\n        List<MessageQueue> lowPriorityQueues = Arrays.asList(mq2, mq3);\n\n        List<List<MessageQueue>> queuesWithPriority = Arrays.asList(highPriorityQueues, lowPriorityQueues);\n\n        // Assign penalties: high-priority queues have higher penalties, low-priority have lower\n        List<MessageQueuePenalizer<MessageQueue>> penalizers = Collections.singletonList(\n            mq -> mq.getBrokerName().equals(\"broker-high\") ? 50 : 10\n        );\n\n        AtomicInteger startIndex = new AtomicInteger(0);\n        Pair<MessageQueue, Integer> result = MessageQueuePenalizer.selectLeastPenaltyWithPriority(\n            queuesWithPriority, penalizers, startIndex);\n\n        assertNotNull(result);\n        // Should select from low-priority group because it has lower penalty\n        assertTrue(result.getLeft().getBrokerName().equals(\"broker-low\"));\n        assertEquals(10, result.getRight().intValue());\n    }\n\n    /**\n     * Test selectLeastPenaltyWithPriority short-circuits when a priority group yields penalty <= 0\n     */\n    @Test\n    public void testSelectLeastPenaltyWithPriority_ShortCircuitZeroPenalty() {\n        // Priority group 1\n        MessageQueue mq0 = new MessageQueue(\"topic\", \"broker-high\", 0);\n        List<MessageQueue> highPriorityQueues = Collections.singletonList(mq0);\n\n        // Priority group 2\n        MessageQueue mq1 = new MessageQueue(\"topic\", \"broker-low\", 0);\n        List<MessageQueue> lowPriorityQueues = Collections.singletonList(mq1);\n\n        List<List<MessageQueue>> queuesWithPriority = Arrays.asList(highPriorityQueues, lowPriorityQueues);\n\n        // First group has penalty 0, should short-circuit\n        List<MessageQueuePenalizer<MessageQueue>> penalizers = Collections.singletonList(\n            mq -> mq.getBrokerName().equals(\"broker-high\") ? 0 : 100\n        );\n\n        AtomicInteger startIndex = new AtomicInteger(0);\n        Pair<MessageQueue, Integer> result = MessageQueuePenalizer.selectLeastPenaltyWithPriority(\n            queuesWithPriority, penalizers, startIndex);\n\n        assertNotNull(result);\n        assertEquals(mq0, result.getLeft());\n        assertEquals(0, result.getRight().intValue());\n    }\n\n    /**\n     * Test selectLeastPenaltyWithPriority when first group encounters zero penalty during iteration\n     */\n    @Test\n    public void testSelectLeastPenaltyWithPriority_FirstGroupHasZeroPenalty() {\n        // Priority group 1\n        MessageQueue mq0 = new MessageQueue(\"topic\", \"broker1\", 0);\n        MessageQueue mq1 = new MessageQueue(\"topic\", \"broker1\", 1);\n        List<MessageQueue> group1 = Arrays.asList(mq0, mq1);\n\n        // Priority group 2\n        MessageQueue mq2 = new MessageQueue(\"topic\", \"broker2\", 0);\n        List<MessageQueue> group2 = Collections.singletonList(mq2);\n\n        List<List<MessageQueue>> queuesWithPriority = Arrays.asList(group1, group2);\n\n        // mq1 in first group has penalty 0\n        List<MessageQueuePenalizer<MessageQueue>> penalizers = Collections.singletonList(\n            mq -> mq.getQueueId() == 1 && mq.getBrokerName().equals(\"broker1\") ? 0 : 50\n        );\n\n        AtomicInteger startIndex = new AtomicInteger(0);\n        Pair<MessageQueue, Integer> result = MessageQueuePenalizer.selectLeastPenaltyWithPriority(\n            queuesWithPriority, penalizers, startIndex);\n\n        assertNotNull(result);\n        assertEquals(mq1, result.getLeft());\n        assertEquals(0, result.getRight().intValue());\n    }\n\n    /**\n     * Test selectLeastPenaltyWithPriority returns first encountered minimum when multiple groups have same minimum penalty\n     */\n    @Test\n    public void testSelectLeastPenaltyWithPriority_SameMinimumPenalty() {\n        // Priority group 1\n        MessageQueue mq0 = new MessageQueue(\"topic\", \"broker1\", 0);\n        List<MessageQueue> group1 = Collections.singletonList(mq0);\n\n        // Priority group 2\n        MessageQueue mq1 = new MessageQueue(\"topic\", \"broker2\", 0);\n        List<MessageQueue> group2 = Collections.singletonList(mq1);\n\n        // Priority group 3\n        MessageQueue mq2 = new MessageQueue(\"topic\", \"broker3\", 0);\n        List<MessageQueue> group3 = Collections.singletonList(mq2);\n\n        List<List<MessageQueue>> queuesWithPriority = Arrays.asList(group1, group2, group3);\n\n        // All have same penalty\n        List<MessageQueuePenalizer<MessageQueue>> penalizers = Collections.singletonList(mq -> 10);\n\n        AtomicInteger startIndex = new AtomicInteger(0);\n        Pair<MessageQueue, Integer> result = MessageQueuePenalizer.selectLeastPenaltyWithPriority(\n            queuesWithPriority, penalizers, startIndex);\n\n        assertNotNull(result);\n        // Should return first encountered (from group1)\n        assertEquals(mq0, result.getLeft());\n        assertEquals(10, result.getRight().intValue());\n    }\n\n    /**\n     * Test selectLeastPenaltyWithPriority with complex scenario:\n     * Multiple priority groups with varying penalties\n     */\n    @Test\n    public void testSelectLeastPenaltyWithPriority_ComplexScenario() {\n        // Priority group 1: penalties 100, 90\n        MessageQueue mq0 = new MessageQueue(\"topic\", \"broker1\", 0);\n        MessageQueue mq1 = new MessageQueue(\"topic\", \"broker1\", 1);\n        List<MessageQueue> group1 = Arrays.asList(mq0, mq1);\n\n        // Priority group 2: penalties 50, 30\n        MessageQueue mq2 = new MessageQueue(\"topic\", \"broker2\", 0);\n        MessageQueue mq3 = new MessageQueue(\"topic\", \"broker2\", 1);\n        List<MessageQueue> group2 = Arrays.asList(mq2, mq3);\n\n        // Priority group 3: penalties 80, 20\n        MessageQueue mq4 = new MessageQueue(\"topic\", \"broker3\", 0);\n        MessageQueue mq5 = new MessageQueue(\"topic\", \"broker3\", 1);\n        List<MessageQueue> group3 = Arrays.asList(mq4, mq5);\n\n        List<List<MessageQueue>> queuesWithPriority = Arrays.asList(group1, group2, group3);\n\n        List<MessageQueuePenalizer<MessageQueue>> penalizers = Collections.singletonList(mq -> {\n            if (mq.getBrokerName().equals(\"broker1\")) {\n                return mq.getQueueId() == 0 ? 100 : 90;\n            } else if (mq.getBrokerName().equals(\"broker2\")) {\n                return mq.getQueueId() == 0 ? 50 : 30;\n            } else {\n                return mq.getQueueId() == 0 ? 80 : 20;\n            }\n        });\n\n        AtomicInteger startIndex = new AtomicInteger(0);\n        Pair<MessageQueue, Integer> result = MessageQueuePenalizer.selectLeastPenaltyWithPriority(\n            queuesWithPriority, penalizers, startIndex);\n\n        assertNotNull(result);\n        // Should select mq5 from group3 with penalty 20 (the global minimum)\n        assertEquals(mq5, result.getLeft());\n        assertEquals(20, result.getRight().intValue());\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueuePriorityProviderTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.route;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.List;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertTrue;\n\npublic class MessageQueuePriorityProviderTest {\n\n    @Test\n    public void testPriorityOfWithLambda() {\n        // Test functional interface implementation using lambda\n        MessageQueuePriorityProvider<MessageQueue> provider = mq -> mq.getQueueId();\n        \n        MessageQueue queue1 = new MessageQueue(\"topic\", \"broker\", 0);\n        MessageQueue queue2 = new MessageQueue(\"topic\", \"broker\", 5);\n        MessageQueue queue3 = new MessageQueue(\"topic\", \"broker\", 10);\n        \n        assertEquals(0, provider.priorityOf(queue1));\n        assertEquals(5, provider.priorityOf(queue2));\n        assertEquals(10, provider.priorityOf(queue3));\n    }\n\n    @Test\n    public void testPriorityOfWithConstantValue() {\n        // Test with constant priority\n        MessageQueuePriorityProvider<MessageQueue> constantProvider = mq -> 1;\n        \n        MessageQueue queue1 = new MessageQueue(\"topic1\", \"broker1\", 0);\n        MessageQueue queue2 = new MessageQueue(\"topic2\", \"broker2\", 5);\n        \n        assertEquals(1, constantProvider.priorityOf(queue1));\n        assertEquals(1, constantProvider.priorityOf(queue2));\n    }\n\n    @Test\n    public void testPriorityOfBasedOnBrokerName() {\n        // Test priority based on broker name hash\n        MessageQueuePriorityProvider<MessageQueue> brokerProvider = \n            mq -> mq.getBrokerName().hashCode() % 10;\n        \n        MessageQueue queue1 = new MessageQueue(\"topic\", \"broker-a\", 0);\n        MessageQueue queue2 = new MessageQueue(\"topic\", \"broker-b\", 0);\n        \n        int priority1 = brokerProvider.priorityOf(queue1);\n        int priority2 = brokerProvider.priorityOf(queue2);\n        \n        // Priorities should be deterministic for the same broker\n        assertEquals(priority1, brokerProvider.priorityOf(queue1));\n        assertEquals(priority2, brokerProvider.priorityOf(queue2));\n    }\n\n    @Test\n    public void testBuildPriorityGroupsWithNullList() {\n        MessageQueuePriorityProvider<MessageQueue> provider = mq -> 0;\n        List<List<MessageQueue>> result = MessageQueuePriorityProvider.buildPriorityGroups(null, provider);\n        \n        assertNotNull(result);\n        assertTrue(result.isEmpty());\n    }\n\n    @Test\n    public void testBuildPriorityGroupsWithEmptyList() {\n        MessageQueuePriorityProvider<MessageQueue> provider = mq -> 0;\n        List<List<MessageQueue>> result = MessageQueuePriorityProvider.buildPriorityGroups(\n            Collections.emptyList(), provider);\n        \n        assertNotNull(result);\n        assertTrue(result.isEmpty());\n    }\n\n    @Test\n    public void testBuildPriorityGroupsWithSinglePriority() {\n        MessageQueuePriorityProvider<MessageQueue> provider = mq -> 0;\n        \n        List<MessageQueue> queues = Arrays.asList(\n            new MessageQueue(\"topic\", \"broker1\", 0),\n            new MessageQueue(\"topic\", \"broker1\", 1),\n            new MessageQueue(\"topic\", \"broker1\", 2)\n        );\n        \n        List<List<MessageQueue>> result = MessageQueuePriorityProvider.buildPriorityGroups(queues, provider);\n        \n        assertNotNull(result);\n        assertEquals(1, result.size());\n        assertEquals(3, result.get(0).size());\n    }\n\n    @Test\n    public void testBuildPriorityGroupsWithMultiplePriorities() {\n        // Priority based on queue ID: 0->high, 1->medium, 2->low\n        MessageQueuePriorityProvider<MessageQueue> provider = mq -> {\n            if (mq.getQueueId() < 2) return 0; // High priority\n            if (mq.getQueueId() < 4) return 1; // Medium priority\n            return 2; // Low priority\n        };\n        \n        List<MessageQueue> queues = Arrays.asList(\n            new MessageQueue(\"topic\", \"broker\", 0), // priority 0\n            new MessageQueue(\"topic\", \"broker\", 1), // priority 0\n            new MessageQueue(\"topic\", \"broker\", 2), // priority 1\n            new MessageQueue(\"topic\", \"broker\", 3), // priority 1\n            new MessageQueue(\"topic\", \"broker\", 4), // priority 2\n            new MessageQueue(\"topic\", \"broker\", 5)  // priority 2\n        );\n        \n        List<List<MessageQueue>> result = MessageQueuePriorityProvider.buildPriorityGroups(queues, provider);\n        \n        assertNotNull(result);\n        assertEquals(3, result.size());\n        \n        // First group (highest priority 0)\n        assertEquals(2, result.get(0).size());\n        assertEquals(0, result.get(0).get(0).getQueueId());\n        assertEquals(1, result.get(0).get(1).getQueueId());\n        \n        // Second group (medium priority 1)\n        assertEquals(2, result.get(1).size());\n        assertEquals(2, result.get(1).get(0).getQueueId());\n        assertEquals(3, result.get(1).get(1).getQueueId());\n        \n        // Third group (low priority 2)\n        assertEquals(2, result.get(2).size());\n        assertEquals(4, result.get(2).get(0).getQueueId());\n        assertEquals(5, result.get(2).get(1).getQueueId());\n    }\n\n    @Test\n    public void testBuildPriorityGroupsOrderedByPriority() {\n        // Test that groups are ordered from high to low priority (ascending numeric value)\n        MessageQueuePriorityProvider<MessageQueue> provider = mq -> mq.getQueueId();\n        \n        List<MessageQueue> queues = Arrays.asList(\n            new MessageQueue(\"topic\", \"broker\", 5),\n            new MessageQueue(\"topic\", \"broker\", 0),\n            new MessageQueue(\"topic\", \"broker\", 3),\n            new MessageQueue(\"topic\", \"broker\", 1)\n        );\n        \n        List<List<MessageQueue>> result = MessageQueuePriorityProvider.buildPriorityGroups(queues, provider);\n        \n        assertNotNull(result);\n        assertEquals(4, result.size());\n        \n        // Verify order: 0, 1, 3, 5 (ascending)\n        assertEquals(0, result.get(0).get(0).getQueueId());\n        assertEquals(1, result.get(1).get(0).getQueueId());\n        assertEquals(3, result.get(2).get(0).getQueueId());\n        assertEquals(5, result.get(3).get(0).getQueueId());\n    }\n\n    @Test\n    public void testBuildPriorityGroupsWithNegativePriorities() {\n        // Test with negative priority values\n        MessageQueuePriorityProvider<MessageQueue> provider = mq -> mq.getQueueId() - 5;\n        \n        List<MessageQueue> queues = Arrays.asList(\n            new MessageQueue(\"topic\", \"broker\", 0), // priority -5\n            new MessageQueue(\"topic\", \"broker\", 5), // priority 0\n            new MessageQueue(\"topic\", \"broker\", 10) // priority 5\n        );\n        \n        List<List<MessageQueue>> result = MessageQueuePriorityProvider.buildPriorityGroups(queues, provider);\n        \n        assertNotNull(result);\n        assertEquals(3, result.size());\n        \n        // Verify order: -5, 0, 5 (ascending)\n        assertEquals(0, result.get(0).get(0).getQueueId());\n        assertEquals(5, result.get(1).get(0).getQueueId());\n        assertEquals(10, result.get(2).get(0).getQueueId());\n    }\n\n    @Test\n    public void testBuildPriorityGroupsWithMixedBrokers() {\n        // Priority based on broker name\n        MessageQueuePriorityProvider<MessageQueue> provider = mq -> {\n            if (mq.getBrokerName().equals(\"broker-high\")) return 0;\n            if (mq.getBrokerName().equals(\"broker-medium\")) return 1;\n            return 2;\n        };\n        \n        List<MessageQueue> queues = Arrays.asList(\n            new MessageQueue(\"topic\", \"broker-high\", 0),\n            new MessageQueue(\"topic\", \"broker-low\", 0),\n            new MessageQueue(\"topic\", \"broker-medium\", 0),\n            new MessageQueue(\"topic\", \"broker-high\", 1),\n            new MessageQueue(\"topic\", \"broker-medium\", 1)\n        );\n        \n        List<List<MessageQueue>> result = MessageQueuePriorityProvider.buildPriorityGroups(queues, provider);\n        \n        assertNotNull(result);\n        assertEquals(3, result.size());\n        \n        // High priority group\n        assertEquals(2, result.get(0).size());\n        assertEquals(\"broker-high\", result.get(0).get(0).getBrokerName());\n        assertEquals(\"broker-high\", result.get(0).get(1).getBrokerName());\n        \n        // Medium priority group\n        assertEquals(2, result.get(1).size());\n        assertEquals(\"broker-medium\", result.get(1).get(0).getBrokerName());\n        \n        // Low priority group\n        assertEquals(1, result.get(2).size());\n        assertEquals(\"broker-low\", result.get(2).get(0).getBrokerName());\n    }\n\n    @Test\n    public void testBuildPriorityGroupsPreservesQueueOrder() {\n        // Test that queues with same priority maintain their relative order\n        MessageQueuePriorityProvider<MessageQueue> provider = mq -> 0;\n        \n        List<MessageQueue> queues = new ArrayList<>();\n        for (int i = 0; i < 10; i++) {\n            queues.add(new MessageQueue(\"topic\", \"broker\", i));\n        }\n        \n        List<List<MessageQueue>> result = MessageQueuePriorityProvider.buildPriorityGroups(queues, provider);\n        \n        assertNotNull(result);\n        assertEquals(1, result.size());\n        assertEquals(10, result.get(0).size());\n        \n        // Verify order is maintained\n        for (int i = 0; i < 10; i++) {\n            assertEquals(i, result.get(0).get(i).getQueueId());\n        }\n    }\n\n    @Test\n    public void testBuildPriorityGroupsWithCustomMessageQueue() {\n        // Test with extended MessageQueue type\n        class CustomMessageQueue extends MessageQueue {\n            private int customPriority;\n            \n            public CustomMessageQueue(String topic, String brokerName, int queueId, int customPriority) {\n                super(topic, brokerName, queueId);\n                this.customPriority = customPriority;\n            }\n            \n            public int getCustomPriority() {\n                return customPriority;\n            }\n        }\n        \n        MessageQueuePriorityProvider<CustomMessageQueue> provider = \n            CustomMessageQueue::getCustomPriority;\n        \n        List<CustomMessageQueue> queues = Arrays.asList(\n            new CustomMessageQueue(\"topic\", \"broker\", 0, 2),\n            new CustomMessageQueue(\"topic\", \"broker\", 1, 0),\n            new CustomMessageQueue(\"topic\", \"broker\", 2, 1)\n        );\n        \n        List<List<CustomMessageQueue>> result = MessageQueuePriorityProvider.buildPriorityGroups(queues, provider);\n        \n        assertNotNull(result);\n        assertEquals(3, result.size());\n        \n        // Verify order by custom priority: 0, 1, 2\n        assertEquals(0, result.get(0).get(0).getCustomPriority());\n        assertEquals(1, result.get(1).get(0).getCustomPriority());\n        assertEquals(2, result.get(2).get(0).getCustomPriority());\n    }\n\n    @Test\n    public void testBuildPriorityGroupsWithLargeNumberOfQueues() {\n        // Test with large number of queues\n        MessageQueuePriorityProvider<MessageQueue> provider = mq -> mq.getQueueId() % 5;\n        \n        List<MessageQueue> queues = new ArrayList<>();\n        for (int i = 0; i < 100; i++) {\n            queues.add(new MessageQueue(\"topic\", \"broker\", i));\n        }\n        \n        List<List<MessageQueue>> result = MessageQueuePriorityProvider.buildPriorityGroups(queues, provider);\n        \n        assertNotNull(result);\n        assertEquals(5, result.size()); // 5 different priorities (0-4)\n        \n        // Each group should have 20 queues (100 / 5)\n        for (List<MessageQueue> group : result) {\n            assertEquals(20, group.size());\n        }\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.route;\n\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.proxy.service.BaseServiceTest;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\n\npublic class MessageQueueSelectorTest extends BaseServiceTest {\n\n    @Test\n    public void testReadMessageQueue() {\n        queueData.setPerm(PermName.PERM_READ);\n        queueData.setReadQueueNums(0);\n        MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), true);\n        assertTrue(messageQueueSelector.getQueues().isEmpty());\n\n        queueData.setPerm(PermName.PERM_READ);\n        queueData.setReadQueueNums(3);\n        messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), true);\n        assertEquals(3, messageQueueSelector.getQueues().size());\n        assertEquals(1, messageQueueSelector.getBrokerActingQueues().size());\n        for (int i = 0; i < messageQueueSelector.getQueues().size(); i++) {\n            AddressableMessageQueue messageQueue = messageQueueSelector.getQueues().get(i);\n            assertEquals(i, messageQueue.getQueueId());\n        }\n\n        AddressableMessageQueue brokerQueue = messageQueueSelector.getQueueByBrokerName(BROKER_NAME);\n        assertEquals(brokerQueue, messageQueueSelector.getBrokerActingQueues().get(0));\n        assertEquals(brokerQueue, messageQueueSelector.selectOne(true));\n        assertEquals(brokerQueue, messageQueueSelector.selectOneByIndex(3, true));\n\n        AddressableMessageQueue queue = messageQueueSelector.selectOne(false);\n        messageQueueSelector.selectOne(false);\n        messageQueueSelector.selectOne(false);\n        assertEquals(queue, messageQueueSelector.selectOne(false));\n    }\n\n    @Test\n    public void testWriteMessageQueue() {\n        queueData.setPerm(PermName.PERM_WRITE);\n        queueData.setReadQueueNums(0);\n        MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), false);\n        assertTrue(messageQueueSelector.getQueues().isEmpty());\n\n        queueData.setPerm(PermName.PERM_WRITE);\n        queueData.setWriteQueueNums(3);\n        messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), false);\n        assertEquals(3, messageQueueSelector.getQueues().size());\n        assertEquals(1, messageQueueSelector.getBrokerActingQueues().size());\n        for (int i = 0; i < messageQueueSelector.getQueues().size(); i++) {\n            AddressableMessageQueue messageQueue = messageQueueSelector.getQueues().get(i);\n            assertEquals(i, messageQueue.getQueueId());\n        }\n\n        AddressableMessageQueue brokerQueue = messageQueueSelector.getQueueByBrokerName(BROKER_NAME);\n        assertEquals(brokerQueue, messageQueueSelector.getBrokerActingQueues().get(0));\n        assertEquals(brokerQueue, messageQueueSelector.selectOne(true));\n        assertEquals(brokerQueue, messageQueueSelector.selectOneByIndex(3, true));\n\n        AddressableMessageQueue queue = messageQueueSelector.selectOne(false);\n        messageQueueSelector.selectOne(false);\n        messageQueueSelector.selectOne(false);\n        assertEquals(queue, messageQueueSelector.selectOne(false));\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.sysmessage;\n\nimport apache.rocketmq.v2.FilterExpression;\nimport apache.rocketmq.v2.FilterType;\nimport apache.rocketmq.v2.Resource;\nimport apache.rocketmq.v2.Settings;\nimport apache.rocketmq.v2.Subscription;\nimport apache.rocketmq.v2.SubscriptionEntry;\nimport com.google.common.collect.Sets;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelId;\nimport java.time.Duration;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.stream.Collectors;\nimport org.apache.commons.lang3.RandomStringUtils;\nimport org.apache.rocketmq.broker.client.ClientChannelInfo;\nimport org.apache.rocketmq.broker.client.ConsumerGroupEvent;\nimport org.apache.rocketmq.broker.client.ConsumerManager;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIExt;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.InitConfigTest;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;\nimport org.apache.rocketmq.proxy.grpc.v2.channel.GrpcClientChannel;\nimport org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;\nimport org.apache.rocketmq.proxy.remoting.RemotingProxyOutClient;\nimport org.apache.rocketmq.proxy.remoting.channel.RemotingChannel;\nimport org.apache.rocketmq.proxy.service.admin.AdminService;\nimport org.apache.rocketmq.proxy.service.channel.SimpleChannel;\nimport org.apache.rocketmq.proxy.service.relay.ProxyRelayService;\nimport org.apache.rocketmq.proxy.service.route.MessageQueueView;\nimport org.apache.rocketmq.proxy.service.route.TopicRouteService;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.filter.FilterAPI;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.assertj.core.util.Lists;\nimport org.jetbrains.annotations.NotNull;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.ArgumentCaptor;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotSame;\nimport static org.junit.Assert.assertSame;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyBoolean;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.doNothing;\nimport static org.mockito.Mockito.doReturn;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.never;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class HeartbeatSyncerTest extends InitConfigTest {\n    @Mock\n    private TopicRouteService topicRouteService;\n    @Mock\n    private AdminService adminService;\n    @Mock\n    private ConsumerManager consumerManager;\n    @Mock\n    private MQClientAPIFactory mqClientAPIFactory;\n    @Mock\n    private MQClientAPIExt mqClientAPIExt;\n    @Mock\n    private ProxyRelayService proxyRelayService;\n\n    private String clientId;\n    private final String remoteAddress = \"10.152.39.53:9768\";\n    private final String localAddress = \"11.193.0.1:1210\";\n    private final String clusterName = \"cluster\";\n    private final String brokerName = \"broker-01\";\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        this.clientId = RandomStringUtils.randomAlphabetic(10);\n        when(mqClientAPIFactory.getClient()).thenReturn(mqClientAPIExt);\n\n        {\n            TopicRouteData topicRouteData = new TopicRouteData();\n            QueueData queueData = new QueueData();\n            queueData.setReadQueueNums(8);\n            queueData.setWriteQueueNums(8);\n            queueData.setPerm(6);\n            queueData.setBrokerName(brokerName);\n            topicRouteData.getQueueDatas().add(queueData);\n            BrokerData brokerData = new BrokerData();\n            brokerData.setCluster(clusterName);\n            brokerData.setBrokerName(brokerName);\n            HashMap<Long, String> brokerAddr = new HashMap<>();\n            brokerAddr.put(0L, \"127.0.0.1:10911\");\n            brokerData.setBrokerAddrs(brokerAddr);\n            topicRouteData.getBrokerDatas().add(brokerData);\n            MessageQueueView messageQueueView = new MessageQueueView(\"foo\", topicRouteData, null);\n            when(this.topicRouteService.getAllMessageQueueView(any(), anyString())).thenReturn(messageQueueView);\n        }\n    }\n\n    @Test\n    public void testSyncGrpcV2Channel() throws Exception {\n        String consumerGroup = \"consumerGroup\";\n        GrpcClientSettingsManager grpcClientSettingsManager = mock(GrpcClientSettingsManager.class);\n        GrpcChannelManager grpcChannelManager = mock(GrpcChannelManager.class);\n        GrpcClientChannel grpcClientChannel = new GrpcClientChannel(\n            proxyRelayService, grpcClientSettingsManager, grpcChannelManager,\n            ProxyContext.create().setRemoteAddress(remoteAddress).setLocalAddress(localAddress),\n            clientId);\n        ClientChannelInfo clientChannelInfo = new ClientChannelInfo(\n            grpcClientChannel,\n            clientId,\n            LanguageCode.JAVA,\n            5\n        );\n\n        ArgumentCaptor<Message> messageArgumentCaptor = ArgumentCaptor.forClass(Message.class);\n        SendResult sendResult = new SendResult();\n        sendResult.setSendStatus(SendStatus.SEND_OK);\n        doReturn(CompletableFuture.completedFuture(sendResult)).when(this.mqClientAPIExt)\n            .sendMessageAsync(anyString(), anyString(), messageArgumentCaptor.capture(), any(), anyLong());\n\n        Settings settings = Settings.newBuilder()\n            .setSubscription(Subscription.newBuilder()\n                .addSubscriptions(SubscriptionEntry.newBuilder()\n                    .setTopic(Resource.newBuilder().setName(\"topic\").build())\n                    .setExpression(FilterExpression.newBuilder()\n                        .setType(FilterType.TAG)\n                        .setExpression(\"tag\")\n                        .build())\n                    .build())\n                .build())\n            .build();\n        when(grpcClientSettingsManager.getRawClientSettings(eq(clientId))).thenReturn(settings);\n\n        HeartbeatSyncer heartbeatSyncer = new HeartbeatSyncer(topicRouteService, adminService, consumerManager, mqClientAPIFactory, null);\n        heartbeatSyncer.onConsumerRegister(\n            consumerGroup,\n            clientChannelInfo,\n            ConsumeType.CONSUME_PASSIVELY,\n            MessageModel.CLUSTERING,\n            ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET,\n            Sets.newHashSet(FilterAPI.buildSubscriptionData(\"topic\", \"tag\"))\n        );\n\n        await().atMost(Duration.ofSeconds(3)).until(() -> !messageArgumentCaptor.getAllValues().isEmpty());\n        heartbeatSyncer.consumeMessage(Lists.newArrayList(convertFromMessage(messageArgumentCaptor.getValue())), null);\n        verify(consumerManager, never()).registerConsumer(anyString(), any(), any(), any(), any(), any(), anyBoolean());\n\n        String localServeAddr = ConfigurationManager.getProxyConfig().getLocalServeAddr();\n        // change local serve addr, to simulate other proxy receive messages\n        heartbeatSyncer.localProxyId = RandomStringUtils.randomAlphabetic(10);\n        ArgumentCaptor<ClientChannelInfo> syncChannelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class);\n        doReturn(true).when(consumerManager).registerConsumer(anyString(), syncChannelInfoArgumentCaptor.capture(), any(), any(), any(), any(), anyBoolean());\n\n        heartbeatSyncer.consumeMessage(Lists.newArrayList(convertFromMessage(messageArgumentCaptor.getValue())), null);\n        heartbeatSyncer.consumeMessage(Lists.newArrayList(convertFromMessage(messageArgumentCaptor.getValue())), null);\n        assertEquals(2, syncChannelInfoArgumentCaptor.getAllValues().size());\n        List<ClientChannelInfo> channelInfoList = syncChannelInfoArgumentCaptor.getAllValues();\n        assertSame(channelInfoList.get(0).getChannel(), channelInfoList.get(1).getChannel());\n        assertEquals(settings, GrpcClientChannel.parseChannelExtendAttribute(channelInfoList.get(0).getChannel()));\n        assertEquals(settings, GrpcClientChannel.parseChannelExtendAttribute(channelInfoList.get(1).getChannel()));\n\n        // start test sync client unregister\n        // reset localServeAddr\n        ConfigurationManager.getProxyConfig().setLocalServeAddr(localServeAddr);\n        heartbeatSyncer.onConsumerUnRegister(consumerGroup, clientChannelInfo);\n        await().atMost(Duration.ofSeconds(3)).until(() -> messageArgumentCaptor.getAllValues().size() == 2);\n\n        ArgumentCaptor<ClientChannelInfo> syncUnRegisterChannelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class);\n        doNothing().when(consumerManager).unregisterConsumer(anyString(), syncUnRegisterChannelInfoArgumentCaptor.capture(), anyBoolean());\n\n        // change local serve addr, to simulate other proxy receive messages\n        heartbeatSyncer.localProxyId = RandomStringUtils.randomAlphabetic(10);\n        heartbeatSyncer.consumeMessage(Lists.newArrayList(convertFromMessage(messageArgumentCaptor.getAllValues().get(1))), null);\n        assertSame(channelInfoList.get(0).getChannel(), syncUnRegisterChannelInfoArgumentCaptor.getValue().getChannel());\n    }\n\n    @Test\n    public void testSyncRemotingChannel() throws Exception {\n        String consumerGroup = \"consumerGroup\";\n        String consumerGroup2 = \"consumerGroup2\";\n        Channel channel = createMockChannel();\n        Set<SubscriptionData> subscriptionDataSet = new HashSet<>();\n        subscriptionDataSet.add(FilterAPI.buildSubscriptionData(\"topic\", \"tagSub\"));\n        Set<SubscriptionData> subscriptionDataSet2 = new HashSet<>();\n        subscriptionDataSet2.add(FilterAPI.buildSubscriptionData(\"topic2\", \"tagSub2\"));\n        RemotingProxyOutClient remotingProxyOutClient = mock(RemotingProxyOutClient.class);\n        RemotingChannel remotingChannel = new RemotingChannel(remotingProxyOutClient, proxyRelayService, channel, clientId, subscriptionDataSet);\n        ClientChannelInfo clientChannelInfo = new ClientChannelInfo(\n            remotingChannel,\n            clientId,\n            LanguageCode.JAVA,\n            4\n        );\n        RemotingChannel remotingChannel2 = new RemotingChannel(remotingProxyOutClient, proxyRelayService, channel, clientId, subscriptionDataSet2);\n        ClientChannelInfo clientChannelInfo2 = new ClientChannelInfo(\n            remotingChannel2,\n            clientId,\n            LanguageCode.JAVA,\n            4\n        );\n\n        HeartbeatSyncer heartbeatSyncer = new HeartbeatSyncer(topicRouteService, adminService, consumerManager, mqClientAPIFactory, null);\n        SendResult okSendResult = new SendResult();\n        okSendResult.setSendStatus(SendStatus.SEND_OK);\n        {\n            ArgumentCaptor<Message> messageArgumentCaptor = ArgumentCaptor.forClass(Message.class);\n            doReturn(CompletableFuture.completedFuture(okSendResult)).when(this.mqClientAPIExt)\n                .sendMessageAsync(anyString(), anyString(), messageArgumentCaptor.capture(), any(), anyLong());\n\n            heartbeatSyncer.onConsumerRegister(\n                consumerGroup,\n                clientChannelInfo,\n                ConsumeType.CONSUME_PASSIVELY,\n                MessageModel.CLUSTERING,\n                ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET,\n                subscriptionDataSet\n            );\n            heartbeatSyncer.onConsumerRegister(\n                consumerGroup2,\n                clientChannelInfo2,\n                ConsumeType.CONSUME_PASSIVELY,\n                MessageModel.CLUSTERING,\n                ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET,\n                subscriptionDataSet2\n            );\n\n            await().atMost(Duration.ofSeconds(3)).until(() -> messageArgumentCaptor.getAllValues().size() == 2);\n            heartbeatSyncer.consumeMessage(convertFromMessage(messageArgumentCaptor.getAllValues()), null);\n            verify(consumerManager, never()).registerConsumer(anyString(), any(), any(), any(), any(), any(), anyBoolean());\n\n            // change local serve addr, to simulate other proxy receive messages\n            heartbeatSyncer.localProxyId = RandomStringUtils.randomAlphabetic(10);\n            ArgumentCaptor<ClientChannelInfo> syncChannelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class);\n            doReturn(true).when(consumerManager).registerConsumer(anyString(), syncChannelInfoArgumentCaptor.capture(), any(), any(), any(), any(), anyBoolean());\n\n            heartbeatSyncer.consumeMessage(convertFromMessage(messageArgumentCaptor.getAllValues()), null);\n            heartbeatSyncer.consumeMessage(convertFromMessage(messageArgumentCaptor.getAllValues()), null);\n            /*\n            data in syncChannelInfoArgumentCaptor will be like:\n            1st, data of group1\n            2nd, data of group2\n            3rd, data of group1\n            4th, data of group2\n             */\n            assertEquals(4, syncChannelInfoArgumentCaptor.getAllValues().size());\n            List<ClientChannelInfo> channelInfoList = syncChannelInfoArgumentCaptor.getAllValues();\n            assertSame(channelInfoList.get(0).getChannel(), channelInfoList.get(2).getChannel());\n            assertNotSame(channelInfoList.get(0).getChannel(), channelInfoList.get(1).getChannel());\n            Set<Set<SubscriptionData>> checkSubscriptionDatas = new HashSet<>();\n            checkSubscriptionDatas.add(RemotingChannel.parseChannelExtendAttribute(channelInfoList.get(0).getChannel()));\n            checkSubscriptionDatas.add(RemotingChannel.parseChannelExtendAttribute(channelInfoList.get(1).getChannel()));\n            assertTrue(checkSubscriptionDatas.contains(subscriptionDataSet));\n            assertTrue(checkSubscriptionDatas.contains(subscriptionDataSet2));\n        }\n\n        {\n            // start test sync client unregister\n            // reset localServeAddr\n            ArgumentCaptor<Message> messageArgumentCaptor = ArgumentCaptor.forClass(Message.class);\n            doReturn(CompletableFuture.completedFuture(okSendResult)).when(this.mqClientAPIExt)\n                .sendMessageAsync(anyString(), anyString(), messageArgumentCaptor.capture(), any(), anyLong());\n            heartbeatSyncer.onConsumerUnRegister(consumerGroup, clientChannelInfo);\n            heartbeatSyncer.onConsumerUnRegister(consumerGroup2, clientChannelInfo2);\n            await().atMost(Duration.ofSeconds(3)).until(() -> messageArgumentCaptor.getAllValues().size() == 2);\n\n            ArgumentCaptor<ClientChannelInfo> syncUnRegisterChannelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class);\n            doNothing().when(consumerManager).unregisterConsumer(anyString(), syncUnRegisterChannelInfoArgumentCaptor.capture(), anyBoolean());\n\n            // change local serve addr, to simulate other proxy receive messages\n            heartbeatSyncer.localProxyId = RandomStringUtils.randomAlphabetic(10);\n            heartbeatSyncer.consumeMessage(convertFromMessage(messageArgumentCaptor.getAllValues()), null);\n            List<ClientChannelInfo> channelInfoList = syncUnRegisterChannelInfoArgumentCaptor.getAllValues();\n            assertNotSame(channelInfoList.get(0).getChannel(), channelInfoList.get(1).getChannel());\n            Set<Set<SubscriptionData>> checkSubscriptionDatas = new HashSet<>();\n            checkSubscriptionDatas.add(RemotingChannel.parseChannelExtendAttribute(channelInfoList.get(0).getChannel()));\n            checkSubscriptionDatas.add(RemotingChannel.parseChannelExtendAttribute(channelInfoList.get(1).getChannel()));\n            assertTrue(checkSubscriptionDatas.contains(subscriptionDataSet));\n            assertTrue(checkSubscriptionDatas.contains(subscriptionDataSet2));\n        }\n    }\n\n    @Test\n    public void testProcessConsumerGroupEventForRemoting() {\n        String consumerGroup = \"consumerGroup\";\n        Channel channel = createMockChannel();\n        RemotingProxyOutClient remotingProxyOutClient = mock(RemotingProxyOutClient.class);\n        RemotingChannel remotingChannel = new RemotingChannel(remotingProxyOutClient, proxyRelayService, channel, clientId, Collections.emptySet());\n        ClientChannelInfo clientChannelInfo = new ClientChannelInfo(\n            remotingChannel,\n            clientId,\n            LanguageCode.JAVA,\n            4\n        );\n\n        testProcessConsumerGroupEvent(consumerGroup, clientChannelInfo);\n    }\n\n    @Test\n    public void testProcessConsumerGroupEventForGrpcV2() {\n        String consumerGroup = \"consumerGroup\";\n        GrpcClientSettingsManager grpcClientSettingsManager = mock(GrpcClientSettingsManager.class);\n        GrpcChannelManager grpcChannelManager = mock(GrpcChannelManager.class);\n        GrpcClientChannel grpcClientChannel = new GrpcClientChannel(\n            proxyRelayService, grpcClientSettingsManager, grpcChannelManager,\n            ProxyContext.create().setRemoteAddress(remoteAddress).setLocalAddress(localAddress),\n            clientId);\n        ClientChannelInfo clientChannelInfo = new ClientChannelInfo(\n            grpcClientChannel,\n            clientId,\n            LanguageCode.JAVA,\n            5\n        );\n\n        testProcessConsumerGroupEvent(consumerGroup, clientChannelInfo);\n    }\n\n    private void testProcessConsumerGroupEvent(String consumerGroup, ClientChannelInfo clientChannelInfo) {\n        HeartbeatSyncer heartbeatSyncer = new HeartbeatSyncer(topicRouteService, adminService, consumerManager, mqClientAPIFactory, null);\n        SendResult okSendResult = new SendResult();\n        okSendResult.setSendStatus(SendStatus.SEND_OK);\n\n        ArgumentCaptor<Message> messageArgumentCaptor = ArgumentCaptor.forClass(Message.class);\n        doReturn(CompletableFuture.completedFuture(okSendResult)).when(this.mqClientAPIExt)\n            .sendMessageAsync(anyString(), anyString(), messageArgumentCaptor.capture(), any(), anyLong());\n\n        heartbeatSyncer.onConsumerRegister(\n            consumerGroup,\n            clientChannelInfo,\n            ConsumeType.CONSUME_PASSIVELY,\n            MessageModel.CLUSTERING,\n            ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET,\n            Collections.emptySet()\n        );\n        await().atMost(Duration.ofSeconds(3)).until(() -> messageArgumentCaptor.getAllValues().size() == 1);\n\n        // change local serve addr, to simulate other proxy receive messages\n        heartbeatSyncer.localProxyId = RandomStringUtils.randomAlphabetic(10);\n        ArgumentCaptor<ClientChannelInfo> channelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class);\n        doReturn(true).when(consumerManager).registerConsumer(anyString(), channelInfoArgumentCaptor.capture(), any(), any(), any(), any(), anyBoolean());\n\n        heartbeatSyncer.consumeMessage(convertFromMessage(messageArgumentCaptor.getAllValues()), null);\n        assertEquals(1, heartbeatSyncer.remoteChannelMap.size());\n\n        heartbeatSyncer.processConsumerGroupEvent(ConsumerGroupEvent.CLIENT_UNREGISTER, consumerGroup, channelInfoArgumentCaptor.getValue());\n        assertTrue(heartbeatSyncer.remoteChannelMap.isEmpty());\n    }\n\n    private MessageExt convertFromMessage(Message message) {\n        MessageExt messageExt = new MessageExt();\n        messageExt.setTopic(message.getTopic());\n        messageExt.setBody(message.getBody());\n        return messageExt;\n    }\n\n    private List<MessageExt> convertFromMessage(List<Message> message) {\n        return message.stream().map(this::convertFromMessage).collect(Collectors.toList());\n    }\n\n    private Channel createMockChannel() {\n        return new MockChannel(RandomStringUtils.randomAlphabetic(10));\n    }\n\n    private class MockChannel extends SimpleChannel {\n\n        public MockChannel(String channelId) {\n            super(null, new MockChannelId(channelId), HeartbeatSyncerTest.this.remoteAddress, HeartbeatSyncerTest.this.localAddress);\n        }\n    }\n\n    private static class MockChannelId implements ChannelId {\n\n        private final String channelId;\n\n        public MockChannelId(String channelId) {\n            this.channelId = channelId;\n        }\n\n        @Override\n        public String asShortText() {\n            return channelId;\n        }\n\n        @Override\n        public String asLongText() {\n            return channelId;\n        }\n\n        @Override\n        public int compareTo(@NotNull ChannelId o) {\n            return this.channelId.compareTo(o.asLongText());\n        }\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/AbstractTransactionServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.transaction;\n\nimport java.util.List;\nimport java.util.Random;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.InitConfigTest;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\n\npublic class AbstractTransactionServiceTest extends InitConfigTest {\n\n    private static final String BROKER_NAME = \"mockBroker\";\n    private static final String PRODUCER_GROUP = \"producerGroup\";\n    private static final Random RANDOM = new Random();\n    private final ProxyContext ctx = ProxyContext.createForInner(this.getClass());\n\n    public static class MockAbstractTransactionServiceTest extends AbstractTransactionService {\n\n        @Override\n        protected String getBrokerNameByAddr(String brokerAddr) {\n            return BROKER_NAME;\n        }\n\n        @Override\n        public void addTransactionSubscription(ProxyContext ctx, String group, List<String> topicList) {\n\n        }\n\n        @Override\n        public void addTransactionSubscription(ProxyContext ctx, String group, String topic) {\n\n        }\n\n        @Override\n        public void replaceTransactionSubscription(ProxyContext ctx, String group, List<String> topicList) {\n\n        }\n\n        @Override\n        public void unSubscribeAllTransactionTopic(ProxyContext ctx, String group) {\n\n        }\n    }\n\n    private TransactionService transactionService;\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        this.transactionService = new MockAbstractTransactionServiceTest();\n    }\n\n    @Test\n    public void testAddAndGenEndHeader() {\n        Message message = new Message();\n        message.putUserProperty(MessageConst.PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS, \"30\");\n        String txId = MessageClientIDSetter.createUniqID();\n\n        TransactionData transactionData = transactionService.addTransactionDataByBrokerName(\n            ctx,\n            BROKER_NAME,\n            \"Topic\",\n            PRODUCER_GROUP,\n            RANDOM.nextLong(),\n            RANDOM.nextLong(),\n            txId,\n            message\n        );\n        assertNotNull(transactionData);\n\n        EndTransactionRequestData requestData = transactionService.genEndTransactionRequestHeader(\n            ctx,\n            \"topic\",\n            PRODUCER_GROUP,\n            MessageSysFlag.TRANSACTION_COMMIT_TYPE,\n            true,\n            txId,\n            txId\n        );\n\n        assertEquals(BROKER_NAME, requestData.getBrokerName());\n        assertEquals(BROKER_NAME, transactionData.getBrokerName());\n        assertEquals(transactionData.getCommitLogOffset(), requestData.getRequestHeader().getCommitLogOffset().longValue());\n        assertEquals(transactionData.getTranStateTableOffset(), requestData.getRequestHeader().getTranStateTableOffset().longValue());\n\n        assertNull(transactionService.genEndTransactionRequestHeader(\n            ctx,\n            \"topic\",\n            \"group\",\n            MessageSysFlag.TRANSACTION_COMMIT_TYPE,\n            true,\n            txId,\n            txId\n        ));\n    }\n\n    @Test\n    public void testOnSendCheckTransactionStateFailedFailed() {\n        Message message = new Message();\n        message.putUserProperty(MessageConst.PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS, \"30\");\n        String txId = MessageClientIDSetter.createUniqID();\n\n        TransactionData transactionData = transactionService.addTransactionDataByBrokerName(\n            ctx,\n            BROKER_NAME,\n            \"Topic\",\n            PRODUCER_GROUP,\n            RANDOM.nextLong(),\n            RANDOM.nextLong(),\n            txId,\n            message\n        );\n        transactionService.onSendCheckTransactionStateFailed(ProxyContext.createForInner(this.getClass()), PRODUCER_GROUP, transactionData);\n        assertNull(transactionService.genEndTransactionRequestHeader(\n            ctx,\n            \"topic\",\n            PRODUCER_GROUP,\n            MessageSysFlag.TRANSACTION_COMMIT_TYPE,\n            true,\n            txId,\n            txId\n        ));\n    }\n}"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.transaction;\n\nimport java.time.Duration;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.stream.Collectors;\nimport org.apache.rocketmq.broker.client.ProducerManager;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.proxy.common.ProxyContext;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.service.BaseServiceTest;\nimport org.apache.rocketmq.proxy.service.route.MessageQueueView;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ProducerData;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.QueueData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.assertj.core.util.Lists;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.ArgumentCaptor;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\n\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.when;\n\npublic class ClusterTransactionServiceTest extends BaseServiceTest {\n\n    @Mock\n    private ProducerManager producerManager;\n    private ProxyContext ctx = ProxyContext.create();\n    private ClusterTransactionService clusterTransactionService;\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        this.clusterTransactionService = new ClusterTransactionService(this.topicRouteService, this.producerManager,\n            this.mqClientAPIFactory);\n\n        MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, null);\n        when(this.topicRouteService.getAllMessageQueueView(any(), anyString()))\n            .thenReturn(messageQueueView);\n\n        when(mqClientAPIFactory.getClient()).thenReturn(mqClientAPIExt);\n    }\n\n    @Test\n    public void testAddTransactionSubscription() {\n        this.clusterTransactionService.addTransactionSubscription(ctx, GROUP, TOPIC);\n\n        assertEquals(1, this.clusterTransactionService.getGroupClusterData().size());\n        assertEquals(CLUSTER_NAME, this.clusterTransactionService.getGroupClusterData().get(GROUP).stream().findAny().get().getCluster());\n    }\n\n    @Test\n    public void testAddTransactionSubscriptionTopicList() {\n        this.clusterTransactionService.addTransactionSubscription(ctx, GROUP, Lists.newArrayList(TOPIC + 1, TOPIC + 2));\n\n        assertEquals(1, this.clusterTransactionService.getGroupClusterData().size());\n        assertEquals(CLUSTER_NAME, this.clusterTransactionService.getGroupClusterData().get(GROUP).stream().findAny().get().getCluster());\n    }\n\n    @Test\n    public void testReplaceTransactionSubscription() {\n        this.clusterTransactionService.addTransactionSubscription(ctx, GROUP, TOPIC);\n\n        assertEquals(1, this.clusterTransactionService.getGroupClusterData().size());\n        assertEquals(CLUSTER_NAME, this.clusterTransactionService.getGroupClusterData().get(GROUP).stream().findAny().get().getCluster());\n\n        this.brokerData.setCluster(CLUSTER_NAME + 1);\n        this.clusterTransactionService.replaceTransactionSubscription(ctx, GROUP, Lists.newArrayList(TOPIC + 1));\n        assertEquals(1, this.clusterTransactionService.getGroupClusterData().size());\n        assertEquals(CLUSTER_NAME + 1, this.clusterTransactionService.getGroupClusterData().get(GROUP).stream().findAny().get().getCluster());\n    }\n\n    @Test\n    public void testUnSubscribeAllTransactionTopic() {\n        this.clusterTransactionService.addTransactionSubscription(ctx, GROUP, TOPIC);\n        this.clusterTransactionService.unSubscribeAllTransactionTopic(ctx, GROUP);\n\n        assertEquals(0, this.clusterTransactionService.getGroupClusterData().size());\n    }\n\n    @Test\n    public void testScanProducerHeartBeat() throws Exception {\n        when(this.producerManager.groupOnline(anyString())).thenReturn(true);\n\n        Mockito.reset(this.topicRouteService);\n        String brokerName2 = \"broker-2-01\";\n        String clusterName2 = \"broker-2\";\n        String brokerAddr2 = \"127.0.0.2:10911\";\n\n        BrokerData brokerData = new BrokerData();\n        QueueData queueData = new QueueData();\n        queueData.setBrokerName(brokerName2);\n        brokerData.setCluster(clusterName2);\n        brokerData.setBrokerName(brokerName2);\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(MixAll.MASTER_ID, brokerName2);\n        brokerData.setBrokerAddrs(brokerAddrs);\n        topicRouteData.getQueueDatas().add(queueData);\n        topicRouteData.getBrokerDatas().add(brokerData);\n        when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData, null));\n\n        TopicRouteData clusterTopicRouteData = new TopicRouteData();\n        QueueData clusterQueueData = new QueueData();\n        BrokerData clusterBrokerData = new BrokerData();\n\n        clusterQueueData.setBrokerName(BROKER_NAME);\n        clusterTopicRouteData.setQueueDatas(Lists.newArrayList(clusterQueueData));\n        clusterBrokerData.setCluster(CLUSTER_NAME);\n        clusterBrokerData.setBrokerName(BROKER_NAME);\n        brokerAddrs = new HashMap<>();\n        brokerAddrs.put(MixAll.MASTER_ID, BROKER_ADDR);\n        clusterBrokerData.setBrokerAddrs(brokerAddrs);\n        clusterTopicRouteData.setBrokerDatas(Lists.newArrayList(clusterBrokerData));\n        when(this.topicRouteService.getAllMessageQueueView(any(), eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, clusterTopicRouteData, null));\n\n        TopicRouteData clusterTopicRouteData2 = new TopicRouteData();\n        QueueData clusterQueueData2 = new QueueData();\n        BrokerData clusterBrokerData2 = new BrokerData();\n\n        clusterQueueData2.setBrokerName(brokerName2);\n        clusterTopicRouteData2.setQueueDatas(Lists.newArrayList(clusterQueueData2));\n        clusterBrokerData2.setCluster(clusterName2);\n        clusterBrokerData2.setBrokerName(brokerName2);\n        brokerAddrs = new HashMap<>();\n        brokerAddrs.put(MixAll.MASTER_ID, brokerAddr2);\n        clusterBrokerData2.setBrokerAddrs(brokerAddrs);\n        clusterTopicRouteData2.setBrokerDatas(Lists.newArrayList(clusterBrokerData2));\n        when(this.topicRouteService.getAllMessageQueueView(any(), eq(clusterName2))).thenReturn(new MessageQueueView(clusterName2, clusterTopicRouteData2, null));\n\n        ConfigurationManager.getProxyConfig().setTransactionHeartbeatBatchNum(2);\n        this.clusterTransactionService.start();\n        Set<String> groupSet = new HashSet<>();\n\n        for (int i = 0; i < 3; i++) {\n            groupSet.add(GROUP + i);\n            this.clusterTransactionService.addTransactionSubscription(ctx, GROUP + i, TOPIC);\n        }\n\n        ArgumentCaptor<String> brokerAddrArgumentCaptor = ArgumentCaptor.forClass(String.class);\n        ArgumentCaptor<HeartbeatData> heartbeatDataArgumentCaptor = ArgumentCaptor.forClass(HeartbeatData.class);\n        when(mqClientAPIExt.sendHeartbeatOneway(\n            brokerAddrArgumentCaptor.capture(),\n            heartbeatDataArgumentCaptor.capture(),\n            anyLong()\n        )).thenReturn(CompletableFuture.completedFuture(null));\n\n        this.clusterTransactionService.scanProducerHeartBeat();\n\n        await().atMost(Duration.ofSeconds(1)).until(() -> brokerAddrArgumentCaptor.getAllValues().size() == 4);\n\n        assertEquals(Lists.newArrayList(BROKER_ADDR, BROKER_ADDR, brokerAddr2, brokerAddr2),\n            brokerAddrArgumentCaptor.getAllValues().stream().sorted().collect(Collectors.toList()));\n\n        List<HeartbeatData> heartbeatDataList = heartbeatDataArgumentCaptor.getAllValues();\n\n        for (final HeartbeatData heartbeatData : heartbeatDataList) {\n            for (ProducerData producerData : heartbeatData.getProducerDataSet()) {\n                groupSet.remove(producerData.getGroupName());\n            }\n        }\n\n        assertTrue(groupSet.isEmpty());\n        assertEquals(brokerName2, this.clusterTransactionService.getBrokerNameByAddr(brokerAddr2));\n        assertEquals(BROKER_NAME, this.clusterTransactionService.getBrokerNameByAddr(BROKER_ADDR));\n    }\n}\n"
  },
  {
    "path": "proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/TransactionDataManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.proxy.service.transaction;\n\nimport java.time.Duration;\nimport java.util.Random;\nimport org.apache.commons.lang3.time.StopWatch;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.proxy.config.InitConfigTest;\nimport org.junit.After;\nimport org.junit.Assume;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertSame;\nimport static org.junit.Assert.assertTrue;\n\npublic class TransactionDataManagerTest extends InitConfigTest {\n    private static final String PRODUCER_GROUP = \"producerGroup\";\n    private static final Random RANDOM = new Random();\n    private TransactionDataManager transactionDataManager;\n\n    @Before\n    public void before() throws Throwable {\n        super.before();\n        this.transactionDataManager = new TransactionDataManager();\n    }\n\n    @After\n    public void after() {\n        super.after();\n    }\n\n    @Test\n    public void testAddAndRemove() {\n        TransactionData transactionData1 = createTransactionData();\n        TransactionData transactionData2 = createTransactionData(transactionData1.getTransactionId());\n        this.transactionDataManager.addTransactionData(PRODUCER_GROUP, transactionData1.getTransactionId(), transactionData1);\n        this.transactionDataManager.addTransactionData(PRODUCER_GROUP, transactionData1.getTransactionId(), transactionData2);\n\n        assertEquals(1, this.transactionDataManager.transactionIdDataMap.size());\n        assertEquals(2, this.transactionDataManager.transactionIdDataMap.get(\n            transactionDataManager.buildKey(PRODUCER_GROUP, transactionData1.getTransactionId())).size());\n\n        this.transactionDataManager.removeTransactionData(PRODUCER_GROUP, transactionData1.getTransactionId(), transactionData1);\n        assertEquals(1, this.transactionDataManager.transactionIdDataMap.size());\n        this.transactionDataManager.removeTransactionData(PRODUCER_GROUP, transactionData1.getTransactionId(), transactionData2);\n        assertEquals(0, this.transactionDataManager.transactionIdDataMap.size());\n    }\n\n    @Test\n    public void testPoll() {\n        String txId = MessageClientIDSetter.createUniqID();\n        TransactionData transactionData1 = createTransactionData(txId, System.currentTimeMillis() - Duration.ofMinutes(2).toMillis());\n        TransactionData transactionData2 = createTransactionData(txId);\n\n        this.transactionDataManager.addTransactionData(PRODUCER_GROUP, txId, transactionData1);\n        this.transactionDataManager.addTransactionData(PRODUCER_GROUP, txId, transactionData2);\n\n        TransactionData resTransactionData = this.transactionDataManager.pollNoExpireTransactionData(PRODUCER_GROUP, txId);\n        assertSame(transactionData2, resTransactionData);\n        assertNull(this.transactionDataManager.pollNoExpireTransactionData(PRODUCER_GROUP, txId));\n    }\n\n    @Test\n    public void testCleanExpire() {\n        String txId = MessageClientIDSetter.createUniqID();\n        this.transactionDataManager.addTransactionData(PRODUCER_GROUP, txId,\n            createTransactionData(txId, System.currentTimeMillis(), Duration.ofMillis(100).toMillis()));\n        this.transactionDataManager.addTransactionData(PRODUCER_GROUP, txId,\n            createTransactionData(txId, System.currentTimeMillis(), Duration.ofMillis(500).toMillis()));\n\n        this.transactionDataManager.addTransactionData(PRODUCER_GROUP, MessageClientIDSetter.createUniqID(),\n            createTransactionData(txId, System.currentTimeMillis(), Duration.ofMillis(1000).toMillis()));\n\n        await().atMost(Duration.ofSeconds(2)).until(() -> {\n            this.transactionDataManager.cleanExpireTransactionData();\n            return this.transactionDataManager.transactionIdDataMap.isEmpty();\n        });\n    }\n\n    @Test\n    public void testWaitTransactionDataClear() throws InterruptedException {\n        // Skip this test case on Mac as it's not stable enough.\n        Assume.assumeFalse(MixAll.isMac());\n        String txId = MessageClientIDSetter.createUniqID();\n        this.transactionDataManager.addTransactionData(PRODUCER_GROUP, txId,\n            createTransactionData(txId, System.currentTimeMillis(), Duration.ofMillis(100).toMillis()));\n        this.transactionDataManager.addTransactionData(PRODUCER_GROUP, txId,\n            createTransactionData(txId, System.currentTimeMillis(), Duration.ofMillis(500).toMillis()));\n\n        this.transactionDataManager.addTransactionData(PRODUCER_GROUP, MessageClientIDSetter.createUniqID(),\n            createTransactionData(txId, System.currentTimeMillis(), Duration.ofMillis(1000).toMillis()));\n\n        StopWatch stopWatch = new StopWatch();\n        stopWatch.start();\n        this.transactionDataManager.waitTransactionDataClear();\n        stopWatch.stop();\n        assertTrue(Math.abs(stopWatch.getTime() - 1000) <= 50);\n    }\n\n    private static TransactionData createTransactionData() {\n        return createTransactionData(MessageClientIDSetter.createUniqID());\n    }\n\n    private static TransactionData createTransactionData(String txId) {\n        return createTransactionData(txId, System.currentTimeMillis());\n    }\n\n    private static TransactionData createTransactionData(String txId, long checkTimestamp) {\n        return createTransactionData(txId, checkTimestamp, Duration.ofMinutes(1).toMillis());\n    }\n\n    private static TransactionData createTransactionData(String txId, long checkTimestamp, long checkImmunityTime) {\n        return new TransactionData(\n            \"brokerName\",\n            \"topicName\",\n            RANDOM.nextLong(),\n            RANDOM.nextLong(),\n            txId,\n            checkTimestamp,\n            checkImmunityTime\n        );\n    }\n}"
  },
  {
    "path": "proxy/src/test/resources/certs/client.key",
    "content": "-----BEGIN ENCRYPTED PRIVATE KEY-----\nMIICoTAbBgkqhkiG9w0BBQMwDgQI1vtPpDhOYRcCAggABIICgMHwgw0p9fx95R/+\ncWnNdEq8I3ZOOy2wDjammFvPrYXcCJzS3Xg/0GDJ8pdJRKrI7253e4u3mxf5oMuY\nRrvpB3KfdelU1k/5QKqOxL/N0gQafQLViN53f6JelyBEAmO1UxQtKZtkTrdZg8ZP\n0u1cPPWxmgNdn1Xx3taMw+Wo05ysHjnHJhOEDQ2WT3VXigiRmFSX3H567yjYMRD+\nzmvBq+qqR9JPbH9Cn7X1oRXX6c8VsZHWF/Ds0I4i+5zJxsSIuNZxjZw9XXNgXtFv\n7FEFC0HDgDQQUY/FNPUbmjQUp1y0YxoOBjlyIqBIx5FWxu95p2xITS0OimQPFT0o\nIngaSb+EKRDhqpLxxIVEbDdkQrdRqcmmLGJioAysExTBDsDwkaEJGOp44bLDM4QW\nSIA9SB01omuCXgn7RjUyVXb5g0Lz+Nvsfp1YXUkPDO9hILfz3eMHDSW7/FzbB81M\nr8URaTagQxBZnvIoCoWszLDXn3JwEjpZEA6y55Naptps3mMRf7+XMt42lX0e4y9a\nogNu5Zw/RZD9YcaTjC2z5XeKiMCs1Ymhy9iuzbo+eRGESqzvUE4VirtsiEwxJRci\nJHAvuAl3X4XnpTty4ahOU+DihM9lALxdU68CN9++7mx581pYuvjzrV+Z5+PuptZX\nAjCZmkZLDh8TCHSzWRqvP/Hcvo9BjW8l1Lq6tOa222PefSNCc6gs6Hq+jUghbabZ\n/ux4WuFc0Zd6bfQWAZohSvd78/ixsdJPNGm2OP+LUIrEDKIkLuH1PM0uq4wzJZNu\nBo7oJ5iFWF67u3MC8oq+BqOVKDNWaCMi7iiSrN2XW8FBo/rpx4Lf/VYREL+Y0mP6\nvzJrZqw=\n-----END ENCRYPTED PRIVATE KEY-----\n"
  },
  {
    "path": "proxy/src/test/resources/certs/client.pem",
    "content": "-----BEGIN CERTIFICATE-----\nMIIDATCCAekCAQIwDQYJKoZIhvcNAQEFBQAwfDELMAkGA1UEBhMCemgxCzAJBgNV\nBAgMAnpqMQswCQYDVQQHDAJoejEPMA0GA1UECgwGYXBhY2hlMREwDwYDVQQLDAhy\nb2NrZXRtcTEOMAwGA1UEAwwFeXVrb24xHzAdBgkqhkiG9w0BCQEWEHl1a29uQGFw\nYWNoZS5vcmcwIBcNMTgwMTE2MDYxNjQ0WhgPMjExNzEyMjMwNjE2NDRaMIGSMQsw\nCQYDVQQGEwJDTjERMA8GA1UECAwIWmhlamlhbmcxETAPBgNVBAcMCEhhbmd6aG91\nMQ8wDQYDVQQKDAZhcGFjaGUxETAPBgNVBAsMCHJvY2tldG1xMRgwFgYDVQQDDA9h\ncGFjaGUgcm9ja2V0bXExHzAdBgkqhkiG9w0BCQEWEHl1a29uQGFwYWNoZS5vcmcw\ngZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAOjPlSjZk37XLBJBc5G/qQNsNdVD\nvZnEGntrqW0UuHjF2T/LPtsGOavLP5wCHvn2zwMR2eCXZwKdKIzSvk0L3XOjH/XY\nOLgRa3cg90lV7Wzn9UMGq3nOjFtjIODPjtz3lwYAuAt1MH+K0E+ChuCFBgFqdY9U\nE0suW3DX0Mt/WB3pAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAFGPaZKyCZzQihKj\nn/7I1J0wKl1HrU7N4sOie8E+ntcpKeX9zKYAou/4Iy0qwgxgRsnucB1rDous560a\n+8DFDU8+FnikK9cQtKfQqu4F266IkkXolviZMSfkmB+NIsByIl95eMJlQHVlAvnX\nvnpGdhD/Jhs+acE1VHhO6K+8omKLA6Og8MmYGRwmnBLcxIvqoSNDlEShfQyjaECg\nI4bEi4ZhH3lSHE46FybJdoxDbj9IjHWqpOnjM23EOyfd1zcwOZJA7a54kfOpiTjz\nwrtes5yoQznun5WtGcLM8ZmyaQ+Jr3j6NyZhOwULzK1+A8YUsW6Ww39xTxQoIHEQ\n7eirb54=\n-----END CERTIFICATE-----\n"
  },
  {
    "path": "proxy/src/test/resources/certs/server.key",
    "content": "-----BEGIN PRIVATE KEY-----\nMIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBAOsmp4YtrIRsBdBQ\nLyPImafCRynTJls3NNF4g6nZr9e0efBY830gw9kBebcm603sdZNl95fzRr2+srXi\n5FJbG7Fmq1+F0xLNK/kKWirGtNMT2DubmhVdKyXYJSvInoGRkrQzbOG0MdAyzE6Q\nO6OjjNN+xGkmadWyCyNF6S8YqMJTAgMBAAECgYEAj0OlnOIG0Ube4+N2VN7KfqKm\nqJy0Ka6gx14dGUY/E7Qo9n27GujzaSq09RkJExiVKZBeIH1fBAtC5f2uDV7kpy0l\nuNpTpQkbw0g2EQLxDsVwaUEYbu+t9qVeXoDd1vFeoXHBuRwvI9UW1BrxVtvKODia\n5StU8Lw4yjcm2lQalwECQQD/sKj56thIsIY7D9qBHk7fnFLd8aYzhnP2GsbZX4V/\nT1KHRxr/8MqdNQX53DE5qcyM/Mqu95FIpTAniUtvcBujAkEA62+fAMYFTAEWj4Z4\nvCmcoPqfVPWhBKFR/wo3L8uUARiIzlbYNU3LIqC2s16QO50+bLUd41oVHNw9Y+uM\nfxQpkQJACg/WpncSadHghmR6UchyjCQnsqo2wyJQX+fv2VAD/d2OPtqSem3sW0Fh\n6dI7cax36zhrdXUyl2xAt92URV9hBwJALX93sdWSxnpbWsc449wCydVFH00MnfFz\nAB+ARLtJ0eBk58M+qyZqgDmgtQ8sPmkH3EgwC3SoKdiiAIJPt2s1EQJBAKnISZZr\nqB2F2PfAW2JJbQlrPyVzkxhv9XYdiVNOErmuxLFae3AI7nECgGuFBtvmeqzm2yRj\n7RBMCmzyWG7MF3o=\n-----END PRIVATE KEY-----\n"
  },
  {
    "path": "proxy/src/test/resources/certs/server.pem",
    "content": "-----BEGIN CERTIFICATE-----\nMIIDATCCAekCAQIwDQYJKoZIhvcNAQEFBQAwfDELMAkGA1UEBhMCemgxCzAJBgNV\nBAgMAnpqMQswCQYDVQQHDAJoejEPMA0GA1UECgwGYXBhY2hlMREwDwYDVQQLDAhy\nb2NrZXRtcTEOMAwGA1UEAwwFeXVrb24xHzAdBgkqhkiG9w0BCQEWEHl1a29uQGFw\nYWNoZS5vcmcwIBcNMTgwMTE2MDYxMzQ5WhgPMjExNzEyMjMwNjEzNDlaMIGSMQsw\nCQYDVQQGEwJDTjERMA8GA1UECAwIWmhlamlhbmcxETAPBgNVBAcMCEhhbmd6aG91\nMQ8wDQYDVQQKDAZhcGFjaGUxETAPBgNVBAsMCHJvY2tldG1xMRgwFgYDVQQDDA9h\ncGFjaGUgcm9ja2V0bXExHzAdBgkqhkiG9w0BCQEWEHl1a29uQGFwYWNoZS5vcmcw\ngZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAOsmp4YtrIRsBdBQLyPImafCRynT\nJls3NNF4g6nZr9e0efBY830gw9kBebcm603sdZNl95fzRr2+srXi5FJbG7Fmq1+F\n0xLNK/kKWirGtNMT2DubmhVdKyXYJSvInoGRkrQzbOG0MdAyzE6QO6OjjNN+xGkm\nadWyCyNF6S8YqMJTAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAAzbwXyAULmXitiU\n+8/2vbUZQlzB/nXY52OIq7qu3F55hE5qlHkcVxG2JZjO3p5UETwOyNUpU4dpu3uT\n7WSdygH4Iagl87ILpGsob9pAf0joAbaXAY4sGDhg+WjR5JInAxbmT+QWZ+4NTuLQ\nfSudUSJrv+HmUlmcVOvLiNStgt9rbtcgJAvpVwY+iCv0HQziFuQxmOkDv09ZLzu/\nlxCMqnbgkEFYkwdntN6MVk38K3MovszedGO/n19hNOFss7nn5XDEeEnc6BqKGdck\nYDoy6amohY0Ds0o0gJ2rq0Y8Gjl9spQ3oeXpoNUoz84OF4KIBRTzSMv8CrmqPdFY\nZd2MGjw=\n-----END CERTIFICATE-----\n"
  },
  {
    "path": "proxy/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker",
    "content": "mock-maker-inline"
  },
  {
    "path": "proxy/src/test/resources/rmq-proxy-home/conf/broker.conf",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\nbrokerClusterName = DefaultCluster\nbrokerName = broker-a\nbrokerId = 0\ndeleteWhen = 04\nfileReservedTime = 48\nbrokerRole = ASYNC_MASTER\nflushDiskType = ASYNC_FLUSH\n"
  },
  {
    "path": "proxy/src/test/resources/rmq-proxy-home/conf/logback_proxy.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n<configuration>\n\n    <appender name=\"RocketmqProxyAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}proxy.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}proxy.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>128MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqProxyAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqProxyAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqProxyWatermarkAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}proxy_watermark.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}proxy_watermark.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>128MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8}%m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqProxyWatermarkAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqProxyWatermarkAppender_inner\"/>\n    </appender>\n\n    <!-- Below is the logger configuration for broker-->\n    <appender name=\"DefaultAppender\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}broker_default.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}broker_default.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n\n    <appender name=\"RocketmqBrokerAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}broker.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}broker.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>20</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>128MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqBrokerAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqBrokerAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqProtectionAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}protection.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}protection.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqProtectionAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqProtectionAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqWaterMarkAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}watermark.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}watermark.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqWaterMarkAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqWaterMarkAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqStoreAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}store.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}store.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>128MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqStoreAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqStoreAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqRemotingAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}remoting.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}remoting.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqRemotingAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqRemotingAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqStoreErrorAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}storeerror.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}storeerror.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqStoreErrorAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqStoreErrorAppender_inner\"/>\n    </appender>\n\n\n    <appender name=\"RocketmqTransactionAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}transaction.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}transaction.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqTransactionAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqTransactionAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqRebalanceLockAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}lock.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}lock.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>5</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqRebalanceLockAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqRebalanceLockAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqFilterAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}filter.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}filter.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqFilterAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqFilterAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqStatsAppender\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}stats.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}stats.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>5</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n\n    <appender name=\"RocketmqCommercialAppender\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}commercial.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}commercial.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>500MB</maxFileSize>\n        </triggeringPolicy>\n    </appender>\n\n    <appender name=\"RocketmqPopAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}pop.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}pop.%i.log\n            </fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>20</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy\n                class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>128MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n\n    <appender name=\"RocketmqPopAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqPopAppender_inner\"/>\n    </appender>\n\n    <appender name=\"STDOUT\" class=\"ch.qos.logback.core.ConsoleAppender\">\n        <append>true</append>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n\n    <logger name=\"RocketmqBroker\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqBrokerAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqProtection\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqProtectionAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqWaterMark\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqWaterMarkAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqCommon\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqBrokerAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqStore\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqStoreAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqStoreError\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqStoreErrorAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqTransaction\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqTransactionAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqRebalanceLock\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqRebalanceLockAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqRemoting\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqRemotingAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqStats\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqStatsAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqCommercial\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqCommercialAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqFilter\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqFilterAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqConsole\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"STDOUT\"/>\n    </logger>\n\n    <logger name=\"RocketmqPop\" additivity=\"false\">\n        <level value=\"INFO\" />\n        <appender-ref ref=\"RocketmqPopAppender\" />\n    </logger>\n\n    <logger name=\"RocketmqProxy\" additivity=\"false\">\n        <level value=\"INFO\" />\n        <appender-ref ref=\"RocketmqProxyAppender\" />\n    </logger>\n\n    <logger name=\"RocketmqProxyWatermark\" additivity=\"false\">\n        <level value=\"INFO\" />\n        <appender-ref ref=\"RocketmqProxyWatermarkAppender\" />\n    </logger>\n\n    <root>\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"DefaultAppender\"/>\n    </root>\n</configuration>\n"
  },
  {
    "path": "proxy/src/test/resources/rmq-proxy-home/conf/rmq-proxy.json",
    "content": "{\n  \"proxyMode\": \"cluster\"\n}"
  },
  {
    "path": "proxy/src/test/resources/rmq.logback-test.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n<configuration>\n\n    <appender name=\"CONSOLE\" class=\"ch.qos.logback.core.ConsoleAppender\">\n        <layout class=\"ch.qos.logback.classic.PatternLayout\">\n            <Pattern>\n                %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n\n            </Pattern>\n        </layout>\n    </appender>\n\n    <logger name=\"org.apache.rocketmq\" level=\"error\" additivity=\"false\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </logger>\n\n    <root level=\"error\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </root>\n\n</configuration>"
  },
  {
    "path": "remoting/BUILD.bazel",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\nload(\"//bazel:GenTestRules.bzl\", \"GenTestRules\")\n\njava_library(\n    name = \"remoting\",\n    srcs = glob([\"src/main/java/**/*.java\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \"//common\",\n        \"@maven//:com_alibaba_fastjson2_fastjson2\",\n        \"@maven//:com_google_guava_guava\",\n        \"@maven//:com_google_code_findbugs_jsr305\",\n        \"@maven//:com_squareup_okio_okio_jvm\",\n        \"@maven//:io_netty_netty_all\",\n        \"@maven//:io_opentelemetry_opentelemetry_api\",\n        \"@maven//:io_opentelemetry_opentelemetry_context\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_otlp\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_prometheus\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk_common\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk_metrics\",\n        \"@maven//:org_apache_tomcat_annotations_api\",\n        \"@maven//:org_apache_commons_commons_lang3\",\n        \"@maven//:io_github_aliyunmq_rocketmq_slf4j_api\",\n        \"@maven//:io_github_aliyunmq_rocketmq_logback_classic\",\n        \"@maven//:commons_collections_commons_collections\",\n        \"@maven//:org_reflections_reflections\",\n    ],\n)\n\njava_library(\n    name = \"tests\",\n    srcs = glob([\"src/test/java/**/*.java\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \":remoting\",\n        \"//common\",\n        \"//:test_deps\",\n        \"@maven//:org_objenesis_objenesis\",\n        \"@maven//:com_alibaba_fastjson\",\n        \"@maven//:com_alibaba_fastjson2_fastjson2\",\n        \"@maven//:com_google_code_gson_gson\",\n        \"@maven//:com_google_guava_guava\",\n        \"@maven//:com_google_code_findbugs_jsr305\",\n        \"@maven//:com_squareup_okio_okio_jvm\",\n        \"@maven//:io_netty_netty_all\",\n        \"@maven//:io_opentelemetry_opentelemetry_api\",\n        \"@maven//:io_opentelemetry_opentelemetry_context\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_otlp\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_prometheus\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk_common\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk_metrics\",\n        \"@maven//:org_apache_tomcat_annotations_api\",\n        \"@maven//:org_apache_commons_commons_lang3\",\n        \"@maven//:org_jetbrains_annotations\",\n        \"@maven//:org_reflections_reflections\",\n    ],\n    resources = glob([\"src/test/resources/certs/*.pem\"]) + glob([\"src/test/resources/certs/*.key\"])\n)\n\nGenTestRules(\n    name = \"GeneratedTestRules\",\n    test_files = glob([\"src/test/java/**/*Test.java\"]),\n    deps = [\n        \":tests\",\n    ],\n)\n"
  },
  {
    "path": "remoting/pom.xml",
    "content": "<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n    <parent>\n        <groupId>org.apache.rocketmq</groupId>\n        <artifactId>rocketmq-all</artifactId>\n        <version>${revision}</version>\n    </parent>\n\n    <modelVersion>4.0.0</modelVersion>\n    <packaging>jar</packaging>\n    <artifactId>rocketmq-remoting</artifactId>\n    <name>rocketmq-remoting ${project.version}</name>\n\n    <properties>\n        <project.root>${basedir}/..</project.root>\n    </properties>\n\n    <dependencies>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-common</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.apache.commons</groupId>\n            <artifactId>commons-lang3</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.reflections</groupId>\n            <artifactId>reflections</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>com.google.code.gson</groupId>\n            <artifactId>gson</artifactId>\n            <version>2.9.0</version>\n            <scope>test</scope>\n        </dependency>\n    </dependencies>\n</project>\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/ChannelEventListener.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting;\n\nimport io.netty.channel.Channel;\n\npublic interface ChannelEventListener {\n    void onChannelConnect(final String remoteAddr, final Channel channel);\n\n    void onChannelClose(final String remoteAddr, final Channel channel);\n\n    void onChannelException(final String remoteAddr, final Channel channel);\n\n    void onChannelIdle(final String remoteAddr, final Channel channel);\n\n    void onChannelActive(final String remoteAddr, final Channel channel);\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/CommandCallback.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting;\n\npublic interface CommandCallback {\n\n    void accept();\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/CommandCustomHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting;\n\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic interface CommandCustomHeader {\n    void checkFields() throws RemotingCommandException;\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/Configuration.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting;\n\nimport java.io.IOException;\nimport java.lang.reflect.Field;\nimport java.lang.reflect.Modifier;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map.Entry;\nimport java.util.Properties;\nimport java.util.concurrent.locks.ReadWriteLock;\nimport java.util.concurrent.locks.ReentrantReadWriteLock;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\n\npublic class Configuration {\n\n    private final Logger log;\n\n    private List<Object> configObjectList = new ArrayList<>(4);\n    private String storePath;\n    private boolean storePathFromConfig = false;\n    private Object storePathObject;\n    private Field storePathField;\n    private DataVersion dataVersion = new DataVersion();\n    private ReadWriteLock readWriteLock = new ReentrantReadWriteLock();\n    /**\n     * All properties include configs in object and extend properties.\n     */\n    private Properties allConfigs = new Properties();\n\n    public Configuration(Logger log) {\n        this.log = log;\n    }\n\n    public Configuration(Logger log, Object... configObjects) {\n        this.log = log;\n        if (configObjects == null || configObjects.length == 0) {\n            return;\n        }\n        for (Object configObject : configObjects) {\n            if (configObject == null) {\n                continue;\n            }\n            registerConfig(configObject);\n        }\n    }\n\n    public Configuration(Logger log, String storePath, Object... configObjects) {\n        this(log, configObjects);\n        this.storePath = storePath;\n    }\n\n    /**\n     * register config object\n     *\n     * @return the current Configuration object\n     */\n    public Configuration registerConfig(Object configObject) {\n        try {\n            readWriteLock.writeLock().lockInterruptibly();\n\n            try {\n\n                Properties registerProps = MixAll.object2Properties(configObject);\n\n                merge(registerProps, this.allConfigs);\n\n                configObjectList.add(configObject);\n            } finally {\n                readWriteLock.writeLock().unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"registerConfig lock error\");\n        }\n        return this;\n    }\n\n    /**\n     * register config properties\n     *\n     * @return the current Configuration object\n     */\n    public Configuration registerConfig(Properties extProperties) {\n        if (extProperties == null) {\n            return this;\n        }\n\n        try {\n            readWriteLock.writeLock().lockInterruptibly();\n\n            try {\n                merge(extProperties, this.allConfigs);\n            } finally {\n                readWriteLock.writeLock().unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"register lock error. {}\" + extProperties);\n        }\n\n        return this;\n    }\n\n    /**\n     * The store path will be gotten from the field of object.\n     *\n     * @throws java.lang.RuntimeException if the field of object is not exist.\n     */\n    public void setStorePathFromConfig(Object object, String fieldName) {\n        assert object != null;\n\n        try {\n            readWriteLock.writeLock().lockInterruptibly();\n\n            try {\n                this.storePathFromConfig = true;\n                this.storePathObject = object;\n                // check\n                this.storePathField = object.getClass().getDeclaredField(fieldName);\n                assert this.storePathField != null\n                    && !Modifier.isStatic(this.storePathField.getModifiers());\n                this.storePathField.setAccessible(true);\n            } catch (NoSuchFieldException e) {\n                throw new RuntimeException(e);\n            } finally {\n                readWriteLock.writeLock().unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"setStorePathFromConfig lock error\");\n        }\n    }\n\n    private String getStorePath() {\n        String realStorePath = null;\n        try {\n            readWriteLock.readLock().lockInterruptibly();\n\n            try {\n                realStorePath = this.storePath;\n\n                if (this.storePathFromConfig) {\n                    try {\n                        realStorePath = (String) storePathField.get(this.storePathObject);\n                    } catch (IllegalAccessException e) {\n                        log.error(\"getStorePath error, \", e);\n                    }\n                }\n            } finally {\n                readWriteLock.readLock().unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"getStorePath lock error\");\n        }\n\n        return realStorePath;\n    }\n\n    public void setStorePath(final String storePath) {\n        this.storePath = storePath;\n    }\n\n    public void update(Properties properties) {\n        try {\n            readWriteLock.writeLock().lockInterruptibly();\n\n            try {\n                // the property must be exist when update\n                mergeIfExist(properties, this.allConfigs);\n\n                for (Object configObject : configObjectList) {\n                    // not allConfigs to update...\n                    MixAll.properties2Object(properties, configObject);\n                }\n\n                this.dataVersion.nextVersion();\n\n            } finally {\n                readWriteLock.writeLock().unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"update lock error, {}\", properties);\n            return;\n        }\n\n        persist();\n    }\n\n    public void persist() {\n        try {\n            readWriteLock.readLock().lockInterruptibly();\n\n            try {\n                String allConfigs = getAllConfigsInternal();\n\n                MixAll.string2File(allConfigs, getStorePath());\n            } catch (IOException e) {\n                log.error(\"persist string2File error, \", e);\n            } finally {\n                readWriteLock.readLock().unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"persist lock error\");\n        }\n    }\n\n    public String getAllConfigsFormatString() {\n        try {\n            readWriteLock.readLock().lockInterruptibly();\n\n            try {\n\n                return getAllConfigsInternal();\n\n            } finally {\n                readWriteLock.readLock().unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"getAllConfigsFormatString lock error\");\n        }\n\n        return null;\n    }\n\n    public String getClientConfigsFormatString(List<String> clientKeys) {\n        try {\n            readWriteLock.readLock().lockInterruptibly();\n\n            try {\n\n                return getClientConfigsInternal(clientKeys);\n\n            } finally {\n                readWriteLock.readLock().unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"getAllConfigsFormatString lock error\");\n        }\n\n        return null;\n    }\n\n    public String getDataVersionJson() {\n        return this.dataVersion.toJson();\n    }\n\n    public Properties getAllConfigs() {\n        try {\n            readWriteLock.readLock().lockInterruptibly();\n\n            try {\n\n                return this.allConfigs;\n\n            } finally {\n                readWriteLock.readLock().unlock();\n            }\n        } catch (InterruptedException e) {\n            log.error(\"getAllConfigs lock error\");\n        }\n\n        return null;\n    }\n\n    private String getAllConfigsInternal() {\n        StringBuilder stringBuilder = new StringBuilder();\n\n        // reload from config object ?\n        for (Object configObject : this.configObjectList) {\n            Properties properties = MixAll.object2Properties(configObject);\n            if (properties != null) {\n                merge(properties, this.allConfigs);\n            } else {\n                log.warn(\"getAllConfigsInternal object2Properties is null, {}\", configObject.getClass());\n            }\n        }\n\n        {\n            stringBuilder.append(MixAll.properties2String(this.allConfigs, true));\n        }\n\n        return stringBuilder.toString();\n    }\n\n    private String getClientConfigsInternal(List<String> clientConigKeys) {\n        StringBuilder stringBuilder = new StringBuilder();\n        Properties clientProperties = new Properties();\n\n        // reload from config object ?\n        for (Object configObject : this.configObjectList) {\n            Properties properties = MixAll.object2Properties(configObject);\n\n            for (String nameNow : clientConigKeys) {\n                if (properties.containsKey(nameNow)) {\n                    clientProperties.put(nameNow, properties.get(nameNow));\n                }\n            }\n\n        }\n        stringBuilder.append(MixAll.properties2String(clientProperties));\n\n        return stringBuilder.toString();\n    }\n\n    private void merge(Properties from, Properties to) {\n        for (Entry<Object, Object> next : from.entrySet()) {\n            Object fromObj = next.getValue(), toObj = to.get(next.getKey());\n            if (toObj != null && !toObj.equals(fromObj)) {\n                log.info(\"Replace, key: {}, value: {} -> {}\", next.getKey(), toObj, fromObj);\n            }\n            to.put(next.getKey(), fromObj);\n        }\n    }\n\n    private void mergeIfExist(Properties from, Properties to) {\n        for (Entry<Object, Object> next : from.entrySet()) {\n            if (!to.containsKey(next.getKey())) {\n                continue;\n            }\n\n            Object fromObj = next.getValue(), toObj = to.get(next.getKey());\n            if (toObj != null && !toObj.equals(fromObj)) {\n                log.info(\"Replace, key: {}, value: {} -> {}\", next.getKey(), toObj, fromObj);\n            }\n            to.put(next.getKey(), fromObj);\n        }\n    }\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting;\n\nimport org.apache.rocketmq.remoting.netty.ResponseFuture;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic interface InvokeCallback {\n    /**\n     * This method is expected to be invoked after {@link #operationSucceed(RemotingCommand)}\n     * or {@link #operationFail(Throwable)}\n     *\n     * @param responseFuture the returned object contains response or exception\n     */\n    void operationComplete(final ResponseFuture responseFuture);\n\n    default void operationSucceed(final RemotingCommand response) {\n\n    }\n\n    default void operationFail(final Throwable throwable) {\n\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/RPCHook.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting;\n\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic interface RPCHook {\n    void doBeforeRequest(final String remoteAddr, final RemotingCommand request);\n\n    void doAfterResponse(final String remoteAddr, final RemotingCommand request,\n                         final RemotingCommand response);\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting;\n\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ExecutorService;\nimport org.apache.rocketmq.remoting.exception.RemotingConnectException;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.netty.ResponseFuture;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic interface RemotingClient extends RemotingService {\n\n    void updateNameServerAddressList(final List<String> addrs);\n\n    List<String> getNameServerAddressList();\n\n    List<String> getAvailableNameSrvList();\n\n    RemotingCommand invokeSync(final String addr, final RemotingCommand request,\n        final long timeoutMillis) throws InterruptedException, RemotingConnectException,\n        RemotingSendRequestException, RemotingTimeoutException;\n\n    void invokeAsync(final String addr, final RemotingCommand request, final long timeoutMillis,\n        final InvokeCallback invokeCallback) throws InterruptedException, RemotingConnectException,\n        RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException;\n\n    void invokeOneway(final String addr, final RemotingCommand request, final long timeoutMillis)\n        throws InterruptedException, RemotingConnectException, RemotingTooMuchRequestException,\n        RemotingTimeoutException, RemotingSendRequestException;\n\n    default CompletableFuture<RemotingCommand> invoke(final String addr, final RemotingCommand request,\n        final long timeoutMillis) {\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        try {\n            invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {\n\n                @Override\n                public void operationComplete(ResponseFuture responseFuture) {\n\n                }\n\n                @Override\n                public void operationSucceed(RemotingCommand response) {\n                    future.complete(response);\n                }\n\n                @Override\n                public void operationFail(Throwable throwable) {\n                    future.completeExceptionally(throwable);\n                }\n            });\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    void registerProcessor(final int requestCode, final NettyRequestProcessor processor,\n        final ExecutorService executor);\n\n    void setCallbackExecutor(final ExecutorService callbackExecutor);\n\n    boolean isChannelWritable(final String addr);\n\n    boolean isAddressReachable(final String addr);\n\n    void closeChannels(final List<String> addrList);\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/RemotingServer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting;\n\nimport io.netty.channel.Channel;\nimport java.util.concurrent.ExecutorService;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic interface RemotingServer extends RemotingService {\n\n    void registerProcessor(final int requestCode, final NettyRequestProcessor processor,\n        final ExecutorService executor);\n\n    void registerDefaultProcessor(final NettyRequestProcessor processor, final ExecutorService executor);\n\n    int localListenPort();\n\n    Pair<NettyRequestProcessor, ExecutorService> getProcessorPair(final int requestCode);\n\n    Pair<NettyRequestProcessor, ExecutorService> getDefaultProcessorPair();\n\n    RemotingServer newRemotingServer(int port);\n\n    void removeRemotingServer(int port);\n\n    RemotingCommand invokeSync(final Channel channel, final RemotingCommand request,\n        final long timeoutMillis) throws InterruptedException, RemotingSendRequestException,\n        RemotingTimeoutException;\n\n    void invokeAsync(final Channel channel, final RemotingCommand request, final long timeoutMillis,\n        final InvokeCallback invokeCallback) throws InterruptedException,\n        RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException;\n\n    void invokeOneway(final Channel channel, final RemotingCommand request, final long timeoutMillis)\n        throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException,\n        RemotingSendRequestException;\n\n    void writeResponse(final Channel channel, final RemotingCommand request, \n        final RemotingCommand response, final java.util.function.Consumer<io.netty.util.concurrent.Future<?>> callback);\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/RemotingService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting;\n\nimport org.apache.rocketmq.remoting.pipeline.RequestPipeline;\n\npublic interface RemotingService {\n    void start();\n\n    void shutdown();\n\n    void registerRPCHook(RPCHook rpcHook);\n\n    void setRequestPipeline(RequestPipeline pipeline);\n\n    /**\n     * Remove all rpc hooks.\n     */\n    void clearRPCHook();\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/annotation/CFNotNull.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.annotation;\n\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.ElementType;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.RetentionPolicy;\nimport java.lang.annotation.Target;\n\n@Documented\n@Retention(RetentionPolicy.RUNTIME)\n@Target({ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER, ElementType.LOCAL_VARIABLE})\npublic @interface CFNotNull {\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/annotation/CFNullable.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.annotation;\n\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.ElementType;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.RetentionPolicy;\nimport java.lang.annotation.Target;\n\n@Documented\n@Retention(RetentionPolicy.RUNTIME)\n@Target({ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER, ElementType.LOCAL_VARIABLE})\npublic @interface CFNullable {\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/common/HeartbeatV2Result.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.common;\n\npublic class HeartbeatV2Result {\n    private int version = 0;\n    private boolean isSubChange = false;\n    private boolean isSupportV2 = false;\n\n    public HeartbeatV2Result(int version, boolean isSubChange, boolean isSupportV2) {\n        this.version = version;\n        this.isSubChange = isSubChange;\n        this.isSupportV2 = isSupportV2;\n    }\n\n    public int getVersion() {\n        return version;\n    }\n\n    public void setVersion(int version) {\n        this.version = version;\n    }\n\n    public boolean isSubChange() {\n        return isSubChange;\n    }\n\n    public void setSubChange(boolean subChange) {\n        isSubChange = subChange;\n    }\n\n    public boolean isSupportV2() {\n        return isSupportV2;\n    }\n\n    public void setSupportV2(boolean supportV2) {\n        isSupportV2 = supportV2;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingHelper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.common;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelFuture;\nimport io.netty.channel.ChannelFutureListener;\nimport io.netty.util.Attribute;\nimport io.netty.util.AttributeKey;\nimport java.io.IOException;\nimport java.lang.reflect.Field;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.nio.ByteBuffer;\nimport java.nio.channels.SocketChannel;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.exception.RemotingConnectException;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.netty.AttributeKeys;\nimport org.apache.rocketmq.remoting.netty.NettySystemConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\n\n@SuppressWarnings(\"DoubleBraceInitialization\")\npublic class RemotingHelper {\n    public static final String DEFAULT_CHARSET = \"UTF-8\";\n    public static final String DEFAULT_CIDR_ALL = \"0.0.0.0/0\";\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME);\n\n    public static final Map<Integer, String> REQUEST_CODE_MAP = new HashMap<Integer, String>() {\n        {\n            try {\n                Field[] f = RequestCode.class.getFields();\n                for (Field field : f) {\n                    if (field.getType() == int.class) {\n                        put((int) field.get(null), field.getName().toLowerCase());\n                    }\n                }\n            } catch (IllegalAccessException ignore) {\n            }\n        }\n    };\n\n    public static final Map<Integer, String> RESPONSE_CODE_MAP = new HashMap<Integer, String>() {\n        {\n            try {\n                Field[] f = ResponseCode.class.getFields();\n                for (Field field : f) {\n                    if (field.getType() == int.class) {\n                        put((int) field.get(null), field.getName().toLowerCase());\n                    }\n                }\n            } catch (IllegalAccessException ignore) {\n            }\n        }\n    };\n\n    public static <T> T getAttributeValue(AttributeKey<T> key, final Channel channel) {\n        if (channel.hasAttr(key)) {\n            Attribute<T> attribute = channel.attr(key);\n            return attribute.get();\n        }\n        return null;\n    }\n\n    public static <T> void setPropertyToAttr(final Channel channel, AttributeKey<T> attributeKey, T value) {\n        if (channel == null) {\n            return;\n        }\n        channel.attr(attributeKey).set(value);\n    }\n\n    public static SocketAddress string2SocketAddress(final String addr) {\n        int split = addr.lastIndexOf(\":\");\n        String host = addr.substring(0, split);\n        String port = addr.substring(split + 1);\n        InetSocketAddress isa = new InetSocketAddress(host, Integer.parseInt(port));\n        return isa;\n    }\n\n    public static RemotingCommand invokeSync(final String addr, final RemotingCommand request,\n        final long timeoutMillis) throws InterruptedException, RemotingConnectException,\n        RemotingSendRequestException, RemotingTimeoutException, RemotingCommandException {\n        long beginTime = System.currentTimeMillis();\n        SocketAddress socketAddress = NetworkUtil.string2SocketAddress(addr);\n        SocketChannel socketChannel = connect(socketAddress);\n        if (socketChannel != null) {\n            boolean sendRequestOK = false;\n\n            try {\n\n                socketChannel.configureBlocking(true);\n\n                //bugfix  http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4614802\n                socketChannel.socket().setSoTimeout((int) timeoutMillis);\n\n                ByteBuffer byteBufferRequest = request.encode();\n                while (byteBufferRequest.hasRemaining()) {\n                    int length = socketChannel.write(byteBufferRequest);\n                    if (length > 0) {\n                        if (byteBufferRequest.hasRemaining()) {\n                            if ((System.currentTimeMillis() - beginTime) > timeoutMillis) {\n\n                                throw new RemotingSendRequestException(addr);\n                            }\n                        }\n                    } else {\n                        throw new RemotingSendRequestException(addr);\n                    }\n\n                    Thread.sleep(1);\n                }\n\n                sendRequestOK = true;\n\n                ByteBuffer byteBufferSize = ByteBuffer.allocate(4);\n                while (byteBufferSize.hasRemaining()) {\n                    int length = socketChannel.read(byteBufferSize);\n                    if (length > 0) {\n                        if (byteBufferSize.hasRemaining()) {\n                            if ((System.currentTimeMillis() - beginTime) > timeoutMillis) {\n\n                                throw new RemotingTimeoutException(addr, timeoutMillis);\n                            }\n                        }\n                    } else {\n                        throw new RemotingTimeoutException(addr, timeoutMillis);\n                    }\n\n                    Thread.sleep(1);\n                }\n\n                int size = byteBufferSize.getInt(0);\n                ByteBuffer byteBufferBody = ByteBuffer.allocate(size);\n                while (byteBufferBody.hasRemaining()) {\n                    int length = socketChannel.read(byteBufferBody);\n                    if (length > 0) {\n                        if (byteBufferBody.hasRemaining()) {\n                            if ((System.currentTimeMillis() - beginTime) > timeoutMillis) {\n\n                                throw new RemotingTimeoutException(addr, timeoutMillis);\n                            }\n                        }\n                    } else {\n                        throw new RemotingTimeoutException(addr, timeoutMillis);\n                    }\n\n                    Thread.sleep(1);\n                }\n\n                byteBufferBody.flip();\n                return RemotingCommand.decode(byteBufferBody);\n            } catch (IOException e) {\n                log.error(\"invokeSync failure\", e);\n\n                if (sendRequestOK) {\n                    throw new RemotingTimeoutException(addr, timeoutMillis);\n                } else {\n                    throw new RemotingSendRequestException(addr);\n                }\n            } finally {\n                try {\n                    socketChannel.close();\n                } catch (IOException e) {\n                    e.printStackTrace();\n                }\n            }\n        } else {\n            throw new RemotingConnectException(addr);\n        }\n    }\n\n    public static String parseChannelRemoteAddr(final Channel channel) {\n        if (null == channel) {\n            return \"\";\n        }\n        String addr = getProxyProtocolAddress(channel);\n        if (StringUtils.isNotBlank(addr)) {\n            return addr;\n        }\n        Attribute<String> att = channel.attr(AttributeKeys.REMOTE_ADDR_KEY);\n        if (att == null) {\n            // mocked in unit test\n            return parseChannelRemoteAddr0(channel);\n        }\n        addr = att.get();\n        if (addr == null) {\n            addr = parseChannelRemoteAddr0(channel);\n            att.set(addr);\n        }\n        return addr;\n    }\n\n    private static String getProxyProtocolAddress(Channel channel) {\n        if (!channel.hasAttr(AttributeKeys.PROXY_PROTOCOL_ADDR)) {\n            return null;\n        }\n        String proxyProtocolAddr = getAttributeValue(AttributeKeys.PROXY_PROTOCOL_ADDR, channel);\n        String proxyProtocolPort = getAttributeValue(AttributeKeys.PROXY_PROTOCOL_PORT, channel);\n        if (StringUtils.isBlank(proxyProtocolAddr) || proxyProtocolPort == null) {\n            return null;\n        }\n        return proxyProtocolAddr + \":\" + proxyProtocolPort;\n    }\n\n    private static String parseChannelRemoteAddr0(final Channel channel) {\n        SocketAddress remote = channel.remoteAddress();\n        final String addr = remote != null ? remote.toString() : \"\";\n\n        if (addr.length() > 0) {\n            int index = addr.lastIndexOf(\"/\");\n            if (index >= 0) {\n                return addr.substring(index + 1);\n            }\n\n            return addr;\n        }\n\n        return \"\";\n    }\n\n    public static String parseChannelLocalAddr(final Channel channel) {\n        SocketAddress remote = channel.localAddress();\n        final String addr = remote != null ? remote.toString() : \"\";\n\n        if (addr.length() > 0) {\n            int index = addr.lastIndexOf(\"/\");\n            if (index >= 0) {\n                return addr.substring(index + 1);\n            }\n\n            return addr;\n        }\n\n        return \"\";\n    }\n\n    public static String parseHostFromAddress(String address) {\n        if (address == null) {\n            return \"\";\n        }\n\n        String[] addressSplits = address.split(\":\");\n        if (addressSplits.length < 1) {\n            return \"\";\n        }\n\n        return addressSplits[0];\n    }\n\n    public static String parseSocketAddressAddr(SocketAddress socketAddress) {\n        if (socketAddress != null) {\n            // Default toString of InetSocketAddress is \"hostName/IP:port\"\n            final String addr = socketAddress.toString();\n            int index = addr.lastIndexOf(\"/\");\n            return (index != -1) ? addr.substring(index + 1) : addr;\n        }\n        return \"\";\n    }\n\n    public static Integer parseSocketAddressPort(SocketAddress socketAddress) {\n        if (socketAddress instanceof InetSocketAddress) {\n            return ((InetSocketAddress) socketAddress).getPort();\n        }\n        return -1;\n    }\n\n    public static int ipToInt(String ip) {\n        String[] ips = ip.split(\"\\\\.\");\n        return (Integer.parseInt(ips[0]) << 24)\n            | (Integer.parseInt(ips[1]) << 16)\n            | (Integer.parseInt(ips[2]) << 8)\n            | Integer.parseInt(ips[3]);\n    }\n\n    public static boolean ipInCIDR(String ip, String cidr) {\n        int ipAddr = ipToInt(ip);\n        String[] cidrArr = cidr.split(\"/\");\n        int netId = Integer.parseInt(cidrArr[1]);\n        int mask = 0xFFFFFFFF << (32 - netId);\n        int cidrIpAddr = ipToInt(cidrArr[0]);\n\n        return (ipAddr & mask) == (cidrIpAddr & mask);\n    }\n\n    public static SocketChannel connect(SocketAddress remote) {\n        return connect(remote, 1000 * 5);\n    }\n\n    public static SocketChannel connect(SocketAddress remote, final int timeoutMillis) {\n        SocketChannel sc = null;\n        try {\n            sc = SocketChannel.open();\n            sc.configureBlocking(true);\n            sc.socket().setSoLinger(false, -1);\n            sc.socket().setTcpNoDelay(true);\n            if (NettySystemConfig.socketSndbufSize > 0) {\n                sc.socket().setReceiveBufferSize(NettySystemConfig.socketSndbufSize);\n            }\n            if (NettySystemConfig.socketRcvbufSize > 0) {\n                sc.socket().setSendBufferSize(NettySystemConfig.socketRcvbufSize);\n            }\n            sc.socket().connect(remote, timeoutMillis);\n            sc.configureBlocking(false);\n            return sc;\n        } catch (Exception e) {\n            if (sc != null) {\n                try {\n                    sc.close();\n                } catch (IOException e1) {\n                    e1.printStackTrace();\n                }\n            }\n        }\n\n        return null;\n    }\n\n    public static void closeChannel(Channel channel) {\n        final String addrRemote = RemotingHelper.parseChannelRemoteAddr(channel);\n        if (\"\".equals(addrRemote)) {\n            channel.close();\n        } else {\n            channel.close().addListener(new ChannelFutureListener() {\n                @Override\n                public void operationComplete(ChannelFuture future) throws Exception {\n                    log.info(\"closeChannel: close the connection to remote address[{}] result: {}\", addrRemote,\n                        future.isSuccess());\n                }\n            });\n        }\n    }\n\n    public static CompletableFuture<Void> convertChannelFutureToCompletableFuture(ChannelFuture channelFuture) {\n        CompletableFuture<Void> completableFuture = new CompletableFuture<>();\n        channelFuture.addListener((ChannelFutureListener) future -> {\n            if (future.isSuccess()) {\n                completableFuture.complete(null);\n            } else {\n                completableFuture.completeExceptionally(new RemotingConnectException(channelFuture.channel().remoteAddress().toString(), future.cause()));\n            }\n        });\n        return completableFuture;\n    }\n\n    public static String getRequestCodeDesc(int code) {\n        return REQUEST_CODE_MAP.getOrDefault(code, String.valueOf(code));\n    }\n\n    public static String getResponseCodeDesc(int code) {\n        return RESPONSE_CODE_MAP.getOrDefault(code, String.valueOf(code));\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/common/SemaphoreReleaseOnlyOnce.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.common;\n\nimport java.util.concurrent.Semaphore;\nimport java.util.concurrent.atomic.AtomicBoolean;\n\npublic class SemaphoreReleaseOnlyOnce {\n    private final AtomicBoolean released = new AtomicBoolean(false);\n    private final Semaphore semaphore;\n\n    public SemaphoreReleaseOnlyOnce(Semaphore semaphore) {\n        this.semaphore = semaphore;\n    }\n\n    public void release() {\n        if (this.semaphore != null) {\n            if (this.released.compareAndSet(false, true)) {\n                this.semaphore.release();\n            }\n        }\n    }\n\n    public Semaphore getSemaphore() {\n        return semaphore;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/common/ServiceThread.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.common;\n\n\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\n/**\n * Base class for background thread\n */\npublic abstract class ServiceThread implements Runnable {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME);\n\n    private static final long JOIN_TIME = 90 * 1000;\n    protected final Thread thread;\n    protected volatile boolean hasNotified = false;\n    protected volatile boolean stopped = false;\n\n    public ServiceThread() {\n        this.thread = new Thread(this, this.getServiceName());\n    }\n\n    public abstract String getServiceName();\n\n    public void start() {\n        this.thread.start();\n    }\n\n    public void shutdown() {\n        this.shutdown(false);\n    }\n\n    public void shutdown(final boolean interrupt) {\n        this.stopped = true;\n        log.info(\"shutdown thread \" + this.getServiceName() + \" interrupt \" + interrupt);\n        synchronized (this) {\n            if (!this.hasNotified) {\n                this.hasNotified = true;\n                this.notify();\n            }\n        }\n\n        try {\n            if (interrupt) {\n                this.thread.interrupt();\n            }\n\n            long beginTime = System.currentTimeMillis();\n            this.thread.join(this.getJointime());\n            long elapsedTime = System.currentTimeMillis() - beginTime;\n            log.info(\"join thread \" + this.getServiceName() + \" elapsed time(ms) \" + elapsedTime + \" \"\n                + this.getJointime());\n        } catch (InterruptedException e) {\n            log.error(\"Interrupted\", e);\n        }\n    }\n\n    public long getJointime() {\n        return JOIN_TIME;\n    }\n\n    public boolean isStopped() {\n        return stopped;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/common/TlsMode.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.common;\n\n/**\n * For server, three SSL modes are supported: disabled, permissive and enforcing.\n * <ol>\n *     <li><strong>disabled:</strong> SSL is not supported; any incoming SSL handshake will be rejected, causing connection closed.</li>\n *     <li><strong>permissive:</strong> SSL is optional, aka, server in this mode can serve client connections with or without SSL;</li>\n *     <li><strong>enforcing:</strong> SSL is required, aka, non SSL connection will be rejected.</li>\n * </ol>\n */\npublic enum TlsMode {\n\n    DISABLED(\"disabled\"),\n    PERMISSIVE(\"permissive\"),\n    ENFORCING(\"enforcing\");\n\n    private String name;\n\n    TlsMode(String name) {\n        this.name = name;\n    }\n\n    public static TlsMode parse(String mode) {\n        for (TlsMode tlsMode : TlsMode.values()) {\n            if (tlsMode.name.equals(mode)) {\n                return tlsMode;\n            }\n        }\n\n        return PERMISSIVE;\n    }\n\n    public String getName() {\n        return name;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/exception/RemotingCommandException.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.exception;\n\npublic class RemotingCommandException extends RemotingException {\n    private static final long serialVersionUID = -6061365915274953096L;\n\n    public RemotingCommandException(String message) {\n        super(message, null);\n    }\n\n    public RemotingCommandException(String message, Throwable cause) {\n        super(message, cause);\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/exception/RemotingConnectException.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.exception;\n\npublic class RemotingConnectException extends RemotingException {\n    private static final long serialVersionUID = -5565366231695911316L;\n\n    public RemotingConnectException(String addr) {\n        this(addr, null);\n    }\n\n    public RemotingConnectException(String addr, Throwable cause) {\n        super(\"connect to \" + addr + \" failed\", cause);\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/exception/RemotingException.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.exception;\n\npublic class RemotingException extends Exception {\n    private static final long serialVersionUID = -5690687334570505110L;\n\n    public RemotingException(String message) {\n        super(message);\n    }\n\n    public RemotingException(String message, Throwable cause) {\n        super(message, cause);\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/exception/RemotingSendRequestException.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.exception;\n\npublic class RemotingSendRequestException extends RemotingException {\n    private static final long serialVersionUID = 5391285827332471674L;\n\n    public RemotingSendRequestException(String addr) {\n        this(addr, null);\n    }\n\n    public RemotingSendRequestException(String addr, Throwable cause) {\n        super(\"send request to <\" + addr + \"> failed\", cause);\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/exception/RemotingTimeoutException.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.exception;\n\npublic class RemotingTimeoutException extends RemotingException {\n\n    private static final long serialVersionUID = 4106899185095245979L;\n\n    public RemotingTimeoutException(String message) {\n        super(message);\n    }\n\n    public RemotingTimeoutException(String addr, long timeoutMillis) {\n        this(addr, timeoutMillis, null);\n    }\n\n    public RemotingTimeoutException(String addr, long timeoutMillis, Throwable cause) {\n        super(\"wait response on the channel <\" + addr + \"> timeout, \" + timeoutMillis + \"(ms)\", cause);\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/exception/RemotingTooMuchRequestException.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.exception;\n\npublic class RemotingTooMuchRequestException extends RemotingException {\n    private static final long serialVersionUID = 4326919581254519654L;\n\n    public RemotingTooMuchRequestException(String message) {\n        super(message);\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/metrics/RemotingMetricsConstant.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.metrics;\n\npublic class RemotingMetricsConstant {\n    public static final String HISTOGRAM_RPC_LATENCY = \"rocketmq_rpc_latency\";\n    public static final String LABEL_PROTOCOL_TYPE = \"protocol_type\";\n    public static final String LABEL_REQUEST_CODE = \"request_code\";\n    public static final String LABEL_RESPONSE_CODE = \"response_code\";\n    public static final String LABEL_IS_LONG_POLLING = \"is_long_polling\";\n    public static final String LABEL_RESULT = \"result\";\n\n    public static final String PROTOCOL_TYPE_REMOTING = \"remoting\";\n\n    public static final String RESULT_ONEWAY = \"oneway\";\n    public static final String RESULT_SUCCESS = \"success\";\n    public static final String RESULT_CANCELED = \"cancelled\";\n    public static final String RESULT_PROCESS_REQUEST_FAILED = \"process_request_failed\";\n    public static final String RESULT_WRITE_CHANNEL_FAILED = \"write_channel_failed\";\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/metrics/RemotingMetricsManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.metrics;\n\nimport com.google.common.collect.Lists;\nimport io.netty.util.concurrent.Future;\nimport io.opentelemetry.api.common.Attributes;\nimport io.opentelemetry.api.common.AttributesBuilder;\nimport io.opentelemetry.api.metrics.LongHistogram;\nimport io.opentelemetry.api.metrics.Meter;\nimport io.opentelemetry.sdk.metrics.Aggregation;\nimport io.opentelemetry.sdk.metrics.InstrumentSelector;\nimport io.opentelemetry.sdk.metrics.InstrumentType;\nimport io.opentelemetry.sdk.metrics.View;\nimport io.opentelemetry.sdk.metrics.ViewBuilder;\nimport java.time.Duration;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.function.Supplier;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.metrics.NopLongHistogram;\n\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.HISTOGRAM_RPC_LATENCY;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_PROTOCOL_TYPE;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.PROTOCOL_TYPE_REMOTING;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.RESULT_CANCELED;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.RESULT_SUCCESS;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.RESULT_WRITE_CHANNEL_FAILED;\n\npublic class RemotingMetricsManager {\n    private LongHistogram rpcLatency = new NopLongHistogram();\n    private Supplier<AttributesBuilder> attributesBuilderSupplier;\n\n    public RemotingMetricsManager() {\n    }\n\n    public AttributesBuilder newAttributesBuilder() {\n        if (this.attributesBuilderSupplier == null) {\n            return Attributes.builder();\n        }\n        return this.attributesBuilderSupplier.get()\n            .put(LABEL_PROTOCOL_TYPE, PROTOCOL_TYPE_REMOTING);\n    }\n\n    public void initMetrics(Meter meter, Supplier<AttributesBuilder> attributesBuilderSupplier) {\n        this.attributesBuilderSupplier = attributesBuilderSupplier;\n        this.rpcLatency = meter.histogramBuilder(HISTOGRAM_RPC_LATENCY)\n            .setDescription(\"Rpc latency\")\n            .setUnit(\"milliseconds\")\n            .ofLongs()\n            .build();\n    }\n\n    public List<Pair<InstrumentSelector, ViewBuilder>> getMetricsView() {\n        List<Double> rpcCostTimeBuckets = Arrays.asList(\n            (double) Duration.ofMillis(1).toMillis(),\n            (double) Duration.ofMillis(3).toMillis(),\n            (double) Duration.ofMillis(5).toMillis(),\n            (double) Duration.ofMillis(7).toMillis(),\n            (double) Duration.ofMillis(10).toMillis(),\n            (double) Duration.ofMillis(100).toMillis(),\n            (double) Duration.ofSeconds(1).toMillis(),\n            (double) Duration.ofSeconds(2).toMillis(),\n            (double) Duration.ofSeconds(3).toMillis()\n        );\n        InstrumentSelector selector = InstrumentSelector.builder()\n            .setType(InstrumentType.HISTOGRAM)\n            .setName(HISTOGRAM_RPC_LATENCY)\n            .build();\n        ViewBuilder viewBuilder = View.builder()\n            .setAggregation(Aggregation.explicitBucketHistogram(rpcCostTimeBuckets));\n        return Lists.newArrayList(new Pair<>(selector, viewBuilder));\n    }\n\n    public String getWriteAndFlushResult(Future<?> future) {\n        String result = RESULT_SUCCESS;\n        if (future.isCancelled()) {\n            result = RESULT_CANCELED;\n        } else if (!future.isSuccess()) {\n            result = RESULT_WRITE_CHANNEL_FAILED;\n        }\n        return result;\n    }\n\n    // Getter methods for external access\n    public LongHistogram getRpcLatency() {\n        return rpcLatency;\n    }\n\n    public Supplier<AttributesBuilder> getAttributesBuilderSupplier() {\n        return attributesBuilderSupplier;\n    }\n\n    // Setter methods for testing\n    public void setAttributesBuilderSupplier(Supplier<AttributesBuilder> attributesBuilderSupplier) {\n        this.attributesBuilderSupplier = attributesBuilderSupplier;\n    }\n\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/netty/AttributeKeys.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.netty;\n\n\nimport io.netty.util.AttributeKey;\nimport org.apache.rocketmq.common.constant.HAProxyConstants;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\n\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\n\npublic class AttributeKeys {\n\n    public static final AttributeKey<String> REMOTE_ADDR_KEY = AttributeKey.valueOf(\"RemoteAddr\");\n\n    public static final AttributeKey<String> CLIENT_ID_KEY = AttributeKey.valueOf(\"ClientId\");\n\n    public static final AttributeKey<Integer> VERSION_KEY = AttributeKey.valueOf(\"Version\");\n\n    public static final AttributeKey<LanguageCode> LANGUAGE_CODE_KEY = AttributeKey.valueOf(\"LanguageCode\");\n\n    public static final AttributeKey<String> PROXY_PROTOCOL_ADDR =\n            AttributeKey.valueOf(HAProxyConstants.PROXY_PROTOCOL_ADDR);\n\n    public static final AttributeKey<String> PROXY_PROTOCOL_PORT =\n            AttributeKey.valueOf(HAProxyConstants.PROXY_PROTOCOL_PORT);\n\n    public static final AttributeKey<String> PROXY_PROTOCOL_SERVER_ADDR =\n            AttributeKey.valueOf(HAProxyConstants.PROXY_PROTOCOL_SERVER_ADDR);\n\n    public static final AttributeKey<String> PROXY_PROTOCOL_SERVER_PORT =\n            AttributeKey.valueOf(HAProxyConstants.PROXY_PROTOCOL_SERVER_PORT);\n\n    private static final Map<String, AttributeKey<String>> ATTRIBUTE_KEY_MAP = new ConcurrentHashMap<>();\n\n    public static AttributeKey<String> valueOf(String name) {\n        return ATTRIBUTE_KEY_MAP.computeIfAbsent(name, AttributeKey::valueOf);\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/netty/FileRegionEncoder.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.netty;\n\nimport io.netty.buffer.ByteBuf;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.channel.FileRegion;\nimport io.netty.handler.codec.MessageToByteEncoder;\n\nimport io.netty.handler.ssl.SslHandler;\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport java.nio.channels.WritableByteChannel;\n\n/**\n * <p>\n *     By default, file region are directly transferred to socket channel which is known as zero copy. In case we need\n *     to encrypt transmission, data being sent should go through the {@link SslHandler}. This encoder ensures this\n *     process.\n * </p>\n */\npublic class FileRegionEncoder extends MessageToByteEncoder<FileRegion> {\n\n    /**\n     * Encode a message into a {@link io.netty.buffer.ByteBuf}. This method will be called for each written message that\n     * can be handled by this encoder.\n     *\n     * @param ctx the {@link io.netty.channel.ChannelHandlerContext} which this {@link\n     * io.netty.handler.codec.MessageToByteEncoder} belongs to\n     * @param msg the message to encode\n     * @param out the {@link io.netty.buffer.ByteBuf} into which the encoded message will be written\n     * @throws Exception is thrown if an error occurs\n     */\n    @Override\n    protected void encode(ChannelHandlerContext ctx, FileRegion msg, final ByteBuf out) throws Exception {\n        WritableByteChannel writableByteChannel = new WritableByteChannel() {\n            @Override\n            public int write(ByteBuffer src) {\n                int prev = out.writerIndex();\n                out.writeBytes(src);\n                return out.writerIndex() - prev;\n            }\n\n            @Override\n            public boolean isOpen() {\n                return true;\n            }\n\n            @Override\n            public void close() throws IOException {\n            }\n        };\n\n        long toTransfer = msg.count();\n\n        while (true) {\n            long transferred = msg.transferred();\n            if (toTransfer - transferred <= 0) {\n                break;\n            }\n            msg.transferTo(writableByteChannel, transferred);\n        }\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.netty;\n\nimport org.apache.rocketmq.remoting.common.TlsMode;\n\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_ENABLE;\n\npublic class NettyClientConfig {\n    /**\n     * Worker thread number\n     */\n    private int clientWorkerThreads = NettySystemConfig.clientWorkerSize;\n    private int clientCallbackExecutorThreads = Runtime.getRuntime().availableProcessors();\n    private int clientOnewaySemaphoreValue = NettySystemConfig.CLIENT_ONEWAY_SEMAPHORE_VALUE;\n    private int clientAsyncSemaphoreValue = NettySystemConfig.CLIENT_ASYNC_SEMAPHORE_VALUE;\n    private int connectTimeoutMillis = NettySystemConfig.connectTimeoutMillis;\n    private long channelNotActiveInterval = 1000 * 60;\n\n    private boolean isScanAvailableNameSrv = true;\n\n    /**\n     * IdleStateEvent will be triggered when neither read nor write was performed for\n     * the specified period of this time. Specify {@code 0} to disable\n     */\n    private int clientChannelMaxIdleTimeSeconds = NettySystemConfig.clientChannelMaxIdleTimeSeconds;\n\n    private int clientSocketSndBufSize = NettySystemConfig.socketSndbufSize;\n    private int clientSocketRcvBufSize = NettySystemConfig.socketRcvbufSize;\n    private boolean clientPooledByteBufAllocatorEnable = false;\n    private boolean clientCloseSocketIfTimeout = NettySystemConfig.clientCloseSocketIfTimeout;\n\n    private boolean useTLS = Boolean.parseBoolean(System.getProperty(TLS_ENABLE,\n        String.valueOf(TlsSystemConfig.tlsMode == TlsMode.ENFORCING)));\n\n    private String socksProxyConfig = \"{}\";\n\n    private int writeBufferHighWaterMark = NettySystemConfig.writeBufferHighWaterMark;\n    private int writeBufferLowWaterMark = NettySystemConfig.writeBufferLowWaterMark;\n\n    private boolean disableCallbackExecutor = false;\n    private boolean disableNettyWorkerGroup = false;\n\n    private long maxReconnectIntervalTimeSeconds = 60;\n\n    private boolean enableReconnectForGoAway = true;\n\n    public boolean isClientCloseSocketIfTimeout() {\n        return clientCloseSocketIfTimeout;\n    }\n\n    public void setClientCloseSocketIfTimeout(final boolean clientCloseSocketIfTimeout) {\n        this.clientCloseSocketIfTimeout = clientCloseSocketIfTimeout;\n    }\n\n    public int getClientWorkerThreads() {\n        return clientWorkerThreads;\n    }\n\n    public void setClientWorkerThreads(int clientWorkerThreads) {\n        this.clientWorkerThreads = clientWorkerThreads;\n    }\n\n    public int getClientOnewaySemaphoreValue() {\n        return clientOnewaySemaphoreValue;\n    }\n\n    public void setClientOnewaySemaphoreValue(int clientOnewaySemaphoreValue) {\n        this.clientOnewaySemaphoreValue = clientOnewaySemaphoreValue;\n    }\n\n    public int getConnectTimeoutMillis() {\n        return connectTimeoutMillis;\n    }\n\n    public void setConnectTimeoutMillis(int connectTimeoutMillis) {\n        this.connectTimeoutMillis = connectTimeoutMillis;\n    }\n\n    public int getClientCallbackExecutorThreads() {\n        return clientCallbackExecutorThreads;\n    }\n\n    public void setClientCallbackExecutorThreads(int clientCallbackExecutorThreads) {\n        this.clientCallbackExecutorThreads = clientCallbackExecutorThreads;\n    }\n\n    public long getChannelNotActiveInterval() {\n        return channelNotActiveInterval;\n    }\n\n    public void setChannelNotActiveInterval(long channelNotActiveInterval) {\n        this.channelNotActiveInterval = channelNotActiveInterval;\n    }\n\n    public int getClientAsyncSemaphoreValue() {\n        return clientAsyncSemaphoreValue;\n    }\n\n    public void setClientAsyncSemaphoreValue(int clientAsyncSemaphoreValue) {\n        this.clientAsyncSemaphoreValue = clientAsyncSemaphoreValue;\n    }\n\n    public int getClientChannelMaxIdleTimeSeconds() {\n        return clientChannelMaxIdleTimeSeconds;\n    }\n\n    public void setClientChannelMaxIdleTimeSeconds(int clientChannelMaxIdleTimeSeconds) {\n        this.clientChannelMaxIdleTimeSeconds = clientChannelMaxIdleTimeSeconds;\n    }\n\n    public int getClientSocketSndBufSize() {\n        return clientSocketSndBufSize;\n    }\n\n    public void setClientSocketSndBufSize(int clientSocketSndBufSize) {\n        this.clientSocketSndBufSize = clientSocketSndBufSize;\n    }\n\n    public int getClientSocketRcvBufSize() {\n        return clientSocketRcvBufSize;\n    }\n\n    public void setClientSocketRcvBufSize(int clientSocketRcvBufSize) {\n        this.clientSocketRcvBufSize = clientSocketRcvBufSize;\n    }\n\n    public boolean isClientPooledByteBufAllocatorEnable() {\n        return clientPooledByteBufAllocatorEnable;\n    }\n\n    public void setClientPooledByteBufAllocatorEnable(boolean clientPooledByteBufAllocatorEnable) {\n        this.clientPooledByteBufAllocatorEnable = clientPooledByteBufAllocatorEnable;\n    }\n\n    public boolean isUseTLS() {\n        return useTLS;\n    }\n\n    public void setUseTLS(boolean useTLS) {\n        this.useTLS = useTLS;\n    }\n\n    public int getWriteBufferLowWaterMark() {\n        return writeBufferLowWaterMark;\n    }\n\n    public void setWriteBufferLowWaterMark(int writeBufferLowWaterMark) {\n        this.writeBufferLowWaterMark = writeBufferLowWaterMark;\n    }\n\n    public int getWriteBufferHighWaterMark() {\n        return writeBufferHighWaterMark;\n    }\n\n    public void setWriteBufferHighWaterMark(int writeBufferHighWaterMark) {\n        this.writeBufferHighWaterMark = writeBufferHighWaterMark;\n    }\n\n    public boolean isDisableCallbackExecutor() {\n        return disableCallbackExecutor;\n    }\n\n    public void setDisableCallbackExecutor(boolean disableCallbackExecutor) {\n        this.disableCallbackExecutor = disableCallbackExecutor;\n    }\n\n    public boolean isDisableNettyWorkerGroup() {\n        return disableNettyWorkerGroup;\n    }\n\n    public void setDisableNettyWorkerGroup(boolean disableNettyWorkerGroup) {\n        this.disableNettyWorkerGroup = disableNettyWorkerGroup;\n    }\n\n    public long getMaxReconnectIntervalTimeSeconds() {\n        return maxReconnectIntervalTimeSeconds;\n    }\n\n    public void setMaxReconnectIntervalTimeSeconds(long maxReconnectIntervalTimeSeconds) {\n        this.maxReconnectIntervalTimeSeconds = maxReconnectIntervalTimeSeconds;\n    }\n\n    public boolean isEnableReconnectForGoAway() {\n        return enableReconnectForGoAway;\n    }\n\n    public void setEnableReconnectForGoAway(boolean enableReconnectForGoAway) {\n        this.enableReconnectForGoAway = enableReconnectForGoAway;\n    }\n\n    public String getSocksProxyConfig() {\n        return socksProxyConfig;\n    }\n\n    public void setSocksProxyConfig(String socksProxyConfig) {\n        this.socksProxyConfig = socksProxyConfig;\n    }\n\n    public boolean isScanAvailableNameSrv() {\n        return isScanAvailableNameSrv;\n    }\n\n    public void setScanAvailableNameSrv(boolean scanAvailableNameSrv) {\n        this.isScanAvailableNameSrv = scanAvailableNameSrv;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyDecoder.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.netty;\n\nimport com.google.common.base.Stopwatch;\nimport io.netty.buffer.ByteBuf;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.handler.codec.LengthFieldBasedFrameDecoder;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic class NettyDecoder extends LengthFieldBasedFrameDecoder {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME);\n\n    private static final int FRAME_MAX_LENGTH =\n        Integer.parseInt(System.getProperty(\"com.rocketmq.remoting.frameMaxLength\", \"16777216\"));\n\n    public NettyDecoder() {\n        super(FRAME_MAX_LENGTH, 0, 4, 0, 4);\n    }\n\n    @Override\n    public Object decode(ChannelHandlerContext ctx, ByteBuf in) throws Exception {\n        ByteBuf frame = null;\n        Stopwatch timer = Stopwatch.createStarted();\n        try {\n            frame = (ByteBuf) super.decode(ctx, in);\n            if (null == frame) {\n                return null;\n            }\n            RemotingCommand cmd = RemotingCommand.decode(frame);\n            cmd.setProcessTimer(timer);\n            return cmd;\n        } catch (Exception e) {\n            log.error(\"decode exception, \" + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), e);\n            RemotingHelper.closeChannel(ctx.channel());\n        } finally {\n            if (null != frame) {\n                frame.release();\n            }\n        }\n\n        return null;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyEncoder.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.netty;\n\nimport io.netty.buffer.ByteBuf;\nimport io.netty.channel.ChannelHandler;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.handler.codec.MessageToByteEncoder;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\n@ChannelHandler.Sharable\npublic class NettyEncoder extends MessageToByteEncoder<RemotingCommand> {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME);\n\n    @Override\n    public void encode(ChannelHandlerContext ctx, RemotingCommand remotingCommand, ByteBuf out)\n        throws Exception {\n        try {\n            remotingCommand.fastEncodeHeader(out);\n            byte[] body = remotingCommand.getBody();\n            if (body != null) {\n                out.writeBytes(body);\n            }\n        } catch (Exception e) {\n            log.error(\"encode exception, \" + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), e);\n            if (remotingCommand != null) {\n                log.error(remotingCommand.toString());\n            }\n            RemotingHelper.closeChannel(ctx.channel());\n        }\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyEvent.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.netty;\n\nimport io.netty.channel.Channel;\n\npublic class NettyEvent {\n    private final NettyEventType type;\n    private final String remoteAddr;\n    private final Channel channel;\n\n    public NettyEvent(NettyEventType type, String remoteAddr, Channel channel) {\n        this.type = type;\n        this.remoteAddr = remoteAddr;\n        this.channel = channel;\n    }\n\n    public NettyEventType getType() {\n        return type;\n    }\n\n    public String getRemoteAddr() {\n        return remoteAddr;\n    }\n\n    public Channel getChannel() {\n        return channel;\n    }\n\n    @Override\n    public String toString() {\n        return \"NettyEvent [type=\" + type + \", remoteAddr=\" + remoteAddr + \", channel=\" + channel + \"]\";\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyEventType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.netty;\n\npublic enum NettyEventType {\n    CONNECT,\n    CLOSE,\n    IDLE,\n    EXCEPTION,\n    ACTIVE\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyLogger.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.netty;\n\n\nimport io.netty.util.internal.logging.InternalLogLevel;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\nimport java.util.concurrent.atomic.AtomicBoolean;\n\npublic class NettyLogger {\n\n    private static AtomicBoolean nettyLoggerSeted = new AtomicBoolean(false);\n    \n    private static InternalLogLevel nettyLogLevel = InternalLogLevel.ERROR;\n\n    public static void initNettyLogger() {\n        if (!nettyLoggerSeted.get()) {\n            try {\n                io.netty.util.internal.logging.InternalLoggerFactory.setDefaultFactory(new NettyBridgeLoggerFactory());\n            } catch (Throwable e) {\n                //ignore\n            }\n            nettyLoggerSeted.set(true);\n        }\n    }\n\n    private static class NettyBridgeLoggerFactory extends io.netty.util.internal.logging.InternalLoggerFactory {\n        @Override\n        protected io.netty.util.internal.logging.InternalLogger newInstance(String s) {\n            return new NettyBridgeLogger(s);\n        }\n    }\n\n    private static class NettyBridgeLogger implements io.netty.util.internal.logging.InternalLogger {\n\n        private Logger logger = null;\n\n        private static final String EXCEPTION_MESSAGE = \"Unexpected exception:\";\n\n        public NettyBridgeLogger(String name) {\n            logger = LoggerFactory.getLogger(name);\n        }\n\n        @Override\n        public String name() {\n            return logger.getName();\n        }\n\n        @Override\n        public boolean isEnabled(InternalLogLevel internalLogLevel) {\n            return nettyLogLevel.ordinal() <= internalLogLevel.ordinal();\n        }\n\n        @Override\n        public void log(InternalLogLevel internalLogLevel, String s) {\n            if (internalLogLevel.equals(InternalLogLevel.DEBUG)) {\n                logger.debug(s);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.TRACE)) {\n                logger.trace(s);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.INFO)) {\n                logger.info(s);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.WARN)) {\n                logger.warn(s);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.ERROR)) {\n                logger.error(s);\n            }\n        }\n\n        @Override\n        public void log(InternalLogLevel internalLogLevel, String s, Object o) {\n            if (internalLogLevel.equals(InternalLogLevel.DEBUG)) {\n                logger.debug(s, o);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.TRACE)) {\n                logger.trace(s, o);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.INFO)) {\n                logger.info(s, o);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.WARN)) {\n                logger.warn(s, o);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.ERROR)) {\n                logger.error(s, o);\n            }\n        }\n\n        @Override\n        public void log(InternalLogLevel internalLogLevel, String s, Object o, Object o1) {\n            if (internalLogLevel.equals(InternalLogLevel.DEBUG)) {\n                logger.debug(s, o, o1);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.TRACE)) {\n                logger.trace(s, o, o1);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.INFO)) {\n                logger.info(s, o, o1);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.WARN)) {\n                logger.warn(s, o, o1);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.ERROR)) {\n                logger.error(s, o, o1);\n            }\n        }\n\n        @Override\n        public void log(InternalLogLevel internalLogLevel, String s, Object... objects) {\n            if (internalLogLevel.equals(InternalLogLevel.DEBUG)) {\n                logger.debug(s, objects);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.TRACE)) {\n                logger.trace(s, objects);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.INFO)) {\n                logger.info(s, objects);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.WARN)) {\n                logger.warn(s, objects);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.ERROR)) {\n                logger.error(s, objects);\n            }\n        }\n\n        @Override\n        public void log(InternalLogLevel internalLogLevel, String s, Throwable throwable) {\n            if (internalLogLevel.equals(InternalLogLevel.DEBUG)) {\n                logger.debug(s, throwable);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.TRACE)) {\n                logger.trace(s, throwable);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.INFO)) {\n                logger.info(s, throwable);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.WARN)) {\n                logger.warn(s, throwable);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.ERROR)) {\n                logger.error(s, throwable);\n            }\n        }\n\n        @Override\n        public void log(InternalLogLevel internalLogLevel, Throwable throwable) {\n            if (internalLogLevel.equals(InternalLogLevel.DEBUG)) {\n                logger.debug(EXCEPTION_MESSAGE, throwable);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.TRACE)) {\n                logger.trace(EXCEPTION_MESSAGE, throwable);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.INFO)) {\n                logger.info(EXCEPTION_MESSAGE, throwable);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.WARN)) {\n                logger.warn(EXCEPTION_MESSAGE, throwable);\n            }\n            if (internalLogLevel.equals(InternalLogLevel.ERROR)) {\n                logger.error(EXCEPTION_MESSAGE, throwable);\n            }\n        }\n\n        @Override\n        public boolean isTraceEnabled() {\n            return isEnabled(InternalLogLevel.TRACE);\n        }\n\n        @Override\n        public void trace(String var1) {\n            logger.trace(var1);\n        }\n\n        @Override\n        public void trace(String var1, Object var2) {\n            logger.trace(var1, var2);\n        }\n\n        @Override\n        public void trace(String var1, Object var2, Object var3) {\n            logger.trace(var1, var2, var3);\n        }\n\n        @Override\n        public void trace(String var1, Object... var2) {\n            logger.trace(var1, var2);\n        }\n\n        @Override\n        public void trace(String var1, Throwable var2) {\n            logger.trace(var1, var2);\n        }\n\n        @Override\n        public void trace(Throwable var1) {\n            logger.trace(EXCEPTION_MESSAGE, var1);\n        }\n\n        @Override\n        public boolean isDebugEnabled() {\n            return isEnabled(InternalLogLevel.DEBUG);\n        }\n\n        @Override\n        public void debug(String var1) {\n            logger.debug(var1);\n        }\n\n        @Override\n        public void debug(String var1, Object var2) {\n            logger.debug(var1, var2);\n        }\n\n        @Override\n        public void debug(String var1, Object var2, Object var3) {\n            logger.debug(var1, var2, var3);\n        }\n\n        @Override\n        public void debug(String var1, Object... var2) {\n            logger.debug(var1, var2);\n        }\n\n        @Override\n        public void debug(String var1, Throwable var2) {\n            logger.debug(var1, var2);\n        }\n\n        @Override\n        public void debug(Throwable var1) {\n            logger.debug(EXCEPTION_MESSAGE, var1);\n        }\n\n        @Override\n        public boolean isInfoEnabled() {\n            return isEnabled(InternalLogLevel.INFO);\n        }\n\n        @Override\n        public void info(String var1) {\n            logger.info(var1);\n        }\n\n        @Override\n        public void info(String var1, Object var2) {\n            logger.info(var1, var2);\n        }\n\n        @Override\n        public void info(String var1, Object var2, Object var3) {\n            logger.info(var1, var2, var3);\n        }\n\n        @Override\n        public void info(String var1, Object... var2) {\n            logger.info(var1, var2);\n        }\n\n        @Override\n        public void info(String var1, Throwable var2) {\n            logger.info(var1, var2);\n        }\n\n        @Override\n        public void info(Throwable var1) {\n            logger.info(EXCEPTION_MESSAGE, var1);\n        }\n\n        @Override\n        public boolean isWarnEnabled() {\n            return isEnabled(InternalLogLevel.WARN);\n        }\n\n        @Override\n        public void warn(String var1) {\n            logger.warn(var1);\n        }\n\n        @Override\n        public void warn(String var1, Object var2) {\n            logger.warn(var1, var2);\n        }\n\n        @Override\n        public void warn(String var1, Object... var2) {\n            logger.warn(var1, var2);\n        }\n\n        @Override\n        public void warn(String var1, Object var2, Object var3) {\n            logger.warn(var1, var2, var3);\n        }\n\n        @Override\n        public void warn(String var1, Throwable var2) {\n            logger.warn(var1, var2);\n        }\n\n        @Override\n        public void warn(Throwable var1) {\n            logger.warn(EXCEPTION_MESSAGE, var1);\n        }\n\n        @Override\n        public boolean isErrorEnabled() {\n            return isEnabled(InternalLogLevel.ERROR);\n        }\n\n        @Override\n        public void error(String var1) {\n            logger.error(var1);\n        }\n\n        @Override\n        public void error(String var1, Object var2) {\n            logger.error(var1, var2);\n        }\n\n        @Override\n        public void error(String var1, Object var2, Object var3) {\n            logger.error(var1, var2, var3);\n        }\n\n        @Override\n        public void error(String var1, Object... var2) {\n            logger.error(var1, var2);\n        }\n\n        @Override\n        public void error(String var1, Throwable var2) {\n            logger.error(var1, var2);\n        }\n\n        @Override\n        public void error(Throwable var1) {\n            logger.error(EXCEPTION_MESSAGE, var1);\n        }\n    }\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.netty;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelFutureListener;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.handler.ssl.SslContext;\nimport io.netty.handler.ssl.SslHandler;\nimport io.netty.util.concurrent.Future;\nimport io.opentelemetry.api.common.AttributesBuilder;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.Iterator;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Map.Entry;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.RejectedExecutionException;\nimport java.util.concurrent.Semaphore;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.TimeoutException;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.concurrent.atomic.AtomicReference;\nimport java.util.function.Consumer;\nimport javax.annotation.Nullable;\nimport org.apache.rocketmq.common.AbortProcessException;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.ExceptionUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.ChannelEventListener;\nimport org.apache.rocketmq.remoting.InvokeCallback;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;\nimport org.apache.rocketmq.remoting.metrics.RemotingMetricsManager;\nimport org.apache.rocketmq.remoting.pipeline.RequestPipeline;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\n\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_IS_LONG_POLLING;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_REQUEST_CODE;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_RESPONSE_CODE;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_RESULT;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.RESULT_ONEWAY;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.RESULT_PROCESS_REQUEST_FAILED;\nimport static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.RESULT_WRITE_CHANNEL_FAILED;\n\npublic abstract class NettyRemotingAbstract {\n\n    /**\n     * Remoting logger instance.\n     */\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME);\n\n    /**\n     * Semaphore to limit maximum number of on-going one-way requests, which protects system memory footprint.\n     */\n    protected final Semaphore semaphoreOneway;\n\n    /**\n     * Semaphore to limit maximum number of on-going asynchronous requests, which protects system memory footprint.\n     */\n    protected final Semaphore semaphoreAsync;\n\n    /**\n     * This map caches all on-going requests.\n     */\n    protected final ConcurrentMap<Integer /* opaque */, ResponseFuture> responseTable =\n        new ConcurrentHashMap<>(256);\n\n    /**\n     * This container holds all processors per request code, aka, for each incoming request, we may look up the\n     * responding processor in this map to handle the request.\n     */\n    protected final HashMap<Integer/* request code */, Pair<NettyRequestProcessor, ExecutorService>> processorTable =\n        new HashMap<>(64);\n\n    /**\n     * Executor to feed netty events to user defined {@link ChannelEventListener}.\n     */\n    protected final NettyEventExecutor nettyEventExecutor = new NettyEventExecutor();\n\n    /**\n     * The default request processor to use in case there is no exact match in {@link #processorTable} per request\n     * code.\n     */\n    protected Pair<NettyRequestProcessor, ExecutorService> defaultRequestProcessorPair;\n\n    /**\n     * SSL context via which to create {@link SslHandler}.\n     */\n    protected volatile SslContext sslContext;\n\n    /**\n     * custom rpc hooks\n     */\n    protected List<RPCHook> rpcHooks = new ArrayList<>();\n\n    protected RequestPipeline requestPipeline;\n\n    protected AtomicBoolean isShuttingDown = new AtomicBoolean(false);\n\n    /**\n     * Remoting metrics manager instance for this remoting server.\n     */\n    protected RemotingMetricsManager remotingMetricsManager;\n\n    static {\n        NettyLogger.initNettyLogger();\n    }\n\n    /**\n     * Constructor, specifying capacity of one-way and asynchronous semaphores.\n     *\n     * @param permitsOneway Number of permits for one-way requests.\n     * @param permitsAsync  Number of permits for asynchronous requests.\n     */\n    public NettyRemotingAbstract(final int permitsOneway, final int permitsAsync) {\n        this.semaphoreOneway = new Semaphore(permitsOneway, true);\n        this.semaphoreAsync = new Semaphore(permitsAsync, true);\n    }\n\n    /**\n     * Custom channel event listener.\n     *\n     * @return custom channel event listener if defined; null otherwise.\n     */\n    public abstract ChannelEventListener getChannelEventListener();\n\n    /**\n     * Set the remoting metrics manager for this remoting server.\n     *\n     * @param remotingMetricsManager the remoting metrics manager instance\n     */\n    public void setRemotingMetricsManager(RemotingMetricsManager remotingMetricsManager) {\n        this.remotingMetricsManager = remotingMetricsManager;\n    }\n\n    /**\n     * Get the remoting metrics manager for this remoting server.\n     *\n     * @return the remoting metrics manager instance\n     */\n    public RemotingMetricsManager getRemotingMetricsManager() {\n        return remotingMetricsManager;\n    }\n\n\n    /**\n     * Put a netty event to the executor.\n     *\n     * @param event Netty event instance.\n     */\n    public void putNettyEvent(final NettyEvent event) {\n        this.nettyEventExecutor.putNettyEvent(event);\n    }\n\n    /**\n     * Entry of incoming command processing.\n     *\n     * <p>\n     * <strong>Note:</strong>\n     * The incoming remoting command may be\n     * <ul>\n     * <li>An inquiry request from a remote peer component;</li>\n     * <li>A response to a previous request issued by this very participant.</li>\n     * </ul>\n     * </p>\n     *\n     * @param ctx Channel handler context.\n     * @param msg incoming remoting command.\n     */\n    public void processMessageReceived(ChannelHandlerContext ctx, RemotingCommand msg) {\n        if (msg != null) {\n            switch (msg.getType()) {\n                case REQUEST_COMMAND:\n                    processRequestCommand(ctx, msg);\n                    break;\n                case RESPONSE_COMMAND:\n                    processResponseCommand(ctx, msg);\n                    break;\n                default:\n                    break;\n            }\n        }\n    }\n\n    protected void doBeforeRpcHooks(String addr, RemotingCommand request) {\n        if (rpcHooks.size() > 0) {\n            for (RPCHook rpcHook : rpcHooks) {\n                rpcHook.doBeforeRequest(addr, request);\n            }\n        }\n    }\n\n    public void doAfterRpcHooks(String addr, RemotingCommand request, RemotingCommand response) {\n        if (rpcHooks.size() > 0) {\n            for (RPCHook rpcHook : rpcHooks) {\n                rpcHook.doAfterResponse(addr, request, response);\n            }\n        }\n    }\n\n    public static void writeResponse(Channel channel, RemotingCommand request, @Nullable RemotingCommand response,\n        Consumer<Future<?>> callback, RemotingMetricsManager remotingMetricsManager) {\n        if (response == null) {\n            return;\n        }\n        final AttributesBuilder attributesBuilder;\n        if (remotingMetricsManager != null) {\n            attributesBuilder = remotingMetricsManager.newAttributesBuilder();\n            attributesBuilder.put(LABEL_IS_LONG_POLLING, request.isSuspended())\n                .put(LABEL_REQUEST_CODE, RemotingHelper.getRequestCodeDesc(request.getCode()))\n                .put(LABEL_RESPONSE_CODE, RemotingHelper.getResponseCodeDesc(response.getCode()));\n        } else {\n            attributesBuilder = null;\n        }\n        if (request.isOnewayRPC()) {\n            if (attributesBuilder != null) {\n                attributesBuilder.put(LABEL_RESULT, RESULT_ONEWAY);\n                remotingMetricsManager.getRpcLatency().record(request.getProcessTimer().elapsed(TimeUnit.MILLISECONDS), attributesBuilder.build());\n            }\n            return;\n        }\n        response.setOpaque(request.getOpaque());\n        response.markResponseType();\n        try {\n            channel.writeAndFlush(response).addListener((ChannelFutureListener) future -> {\n                if (future.isSuccess()) {\n                    log.debug(\"Response[request code: {}, response code: {}, opaque: {}] is written to channel{}\",\n                        request.getCode(), response.getCode(), response.getOpaque(), channel);\n                } else {\n                    log.error(\"Failed to write response[request code: {}, response code: {}, opaque: {}] to channel{}\",\n                        request.getCode(), response.getCode(), response.getOpaque(), channel, future.cause());\n                }\n                if (remotingMetricsManager != null) {\n                    attributesBuilder.put(LABEL_RESULT, remotingMetricsManager.getWriteAndFlushResult(future));\n                    remotingMetricsManager.getRpcLatency().record(request.getProcessTimer().elapsed(TimeUnit.MILLISECONDS), attributesBuilder.build());\n                }\n                if (callback != null) {\n                    callback.accept(future);\n                }\n            });\n        } catch (Throwable e) {\n            log.error(\"process request over, but response failed\", e);\n            log.error(request.toString());\n            log.error(response.toString());\n            if (remotingMetricsManager != null) {\n                attributesBuilder.put(LABEL_RESULT, RESULT_WRITE_CHANNEL_FAILED);\n                remotingMetricsManager.getRpcLatency().record(request.getProcessTimer().elapsed(TimeUnit.MILLISECONDS), attributesBuilder.build());\n            }\n        }\n\n    }\n\n    public void writeResponse(Channel channel, RemotingCommand request, @Nullable RemotingCommand response,\n        Consumer<Future<?>> callback) {\n        if (response == null) {\n            return;\n        }\n        final AttributesBuilder attributesBuilder;\n        if (this.remotingMetricsManager != null) {\n            attributesBuilder = this.remotingMetricsManager.newAttributesBuilder();\n            attributesBuilder.put(LABEL_IS_LONG_POLLING, request.isSuspended())\n                .put(LABEL_REQUEST_CODE, RemotingHelper.getRequestCodeDesc(request.getCode()))\n                .put(LABEL_RESPONSE_CODE, RemotingHelper.getResponseCodeDesc(response.getCode()));\n        } else {\n            attributesBuilder = null;\n        }\n        if (request.isOnewayRPC()) {\n            if (attributesBuilder != null) {\n                attributesBuilder.put(LABEL_RESULT, RESULT_ONEWAY);\n                this.remotingMetricsManager.getRpcLatency().record(request.getProcessTimer().elapsed(TimeUnit.MILLISECONDS), attributesBuilder.build());\n            }\n            return;\n        }\n        response.setOpaque(request.getOpaque());\n        response.markResponseType();\n        try {\n            channel.writeAndFlush(response).addListener((ChannelFutureListener) future -> {\n                if (future.isSuccess()) {\n                    log.debug(\"Response[request code: {}, response code: {}, opaque: {}] is written to channel{}\",\n                        request.getCode(), response.getCode(), response.getOpaque(), channel);\n                } else {\n                    log.error(\"Failed to write response[request code: {}, response code: {}, opaque: {}] to channel{}\",\n                        request.getCode(), response.getCode(), response.getOpaque(), channel, future.cause());\n                }\n                if (this.remotingMetricsManager != null && attributesBuilder != null) {\n                    attributesBuilder.put(LABEL_RESULT, this.remotingMetricsManager.getWriteAndFlushResult(future));\n                    this.remotingMetricsManager.getRpcLatency().record(request.getProcessTimer().elapsed(TimeUnit.MILLISECONDS), attributesBuilder.build());\n                }\n                if (callback != null) {\n                    callback.accept(future);\n                }\n            });\n        } catch (Throwable e) {\n            log.error(\"process request over, but response failed\", e);\n            log.error(request.toString());\n            log.error(response.toString());\n            if (this.remotingMetricsManager != null && attributesBuilder != null) {\n                attributesBuilder.put(LABEL_RESULT, RESULT_WRITE_CHANNEL_FAILED);\n                this.remotingMetricsManager.getRpcLatency().record(request.getProcessTimer().elapsed(TimeUnit.MILLISECONDS), attributesBuilder.build());\n            }\n        }\n    }\n\n    /**\n     * Process incoming request command issued by remote peer.\n     *\n     * @param ctx channel handler context.\n     * @param cmd request command.\n     */\n    public void processRequestCommand(final ChannelHandlerContext ctx, final RemotingCommand cmd) {\n        final Pair<NettyRequestProcessor, ExecutorService> matched = this.processorTable.get(cmd.getCode());\n        final Pair<NettyRequestProcessor, ExecutorService> pair = null == matched ? this.defaultRequestProcessorPair : matched;\n        final int opaque = cmd.getOpaque();\n\n        if (pair == null) {\n            String error = \" request type \" + cmd.getCode() + \" not supported\";\n            final RemotingCommand response =\n                RemotingCommand.createResponseCommand(RemotingSysResponseCode.REQUEST_CODE_NOT_SUPPORTED, error);\n            response.setOpaque(opaque);\n            this.writeResponse(ctx.channel(), cmd, response, null);\n            log.error(RemotingHelper.parseChannelRemoteAddr(ctx.channel()) + error);\n            return;\n        }\n\n        Runnable run = buildProcessRequestHandler(ctx, cmd, pair, opaque);\n\n        if (isShuttingDown.get()) {\n            if (cmd.getVersion() > MQVersion.Version.V5_3_1.ordinal()) {\n                final RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.GO_AWAY,\n                    \"please go away\");\n                response.setOpaque(opaque);\n                this.writeResponse(ctx.channel(), cmd, response, null);\n                log.info(\"proxy is shutting down, write response GO_AWAY. channel={}, requestCode={}, opaque={}\", ctx.channel(), cmd.getCode(), opaque);\n                return;\n            }\n        }\n\n        if (pair.getObject1().rejectRequest()) {\n            final RemotingCommand response = RemotingCommand.createResponseCommand(RemotingSysResponseCode.SYSTEM_BUSY,\n                \"[REJECTREQUEST]system busy, start flow control for a while\");\n            response.setOpaque(opaque);\n            this.writeResponse(ctx.channel(), cmd, response, null);\n            return;\n        }\n\n        try {\n            final RequestTask requestTask = new RequestTask(run, ctx.channel(), cmd);\n            //async execute task, current thread return directly\n            pair.getObject2().submit(requestTask);\n        } catch (RejectedExecutionException e) {\n            if ((System.currentTimeMillis() % 10000) == 0) {\n                log.warn(RemotingHelper.parseChannelRemoteAddr(ctx.channel())\n                    + \", too many requests and system thread pool busy, RejectedExecutionException \"\n                    + pair.getObject2().toString()\n                    + \" request code: \" + cmd.getCode());\n            }\n\n            final RemotingCommand response = RemotingCommand.createResponseCommand(RemotingSysResponseCode.SYSTEM_BUSY,\n                \"[OVERLOAD]system busy, start flow control for a while\");\n            response.setOpaque(opaque);\n            this.writeResponse(ctx.channel(), cmd, response, null);\n        } catch (Throwable e) {\n            if (remotingMetricsManager != null) {\n                AttributesBuilder attributesBuilder = remotingMetricsManager.newAttributesBuilder()\n                    .put(LABEL_REQUEST_CODE, RemotingHelper.getRequestCodeDesc(cmd.getCode()))\n                    .put(LABEL_RESULT, RESULT_PROCESS_REQUEST_FAILED);\n                remotingMetricsManager.getRpcLatency().record(cmd.getProcessTimer().elapsed(TimeUnit.MILLISECONDS), attributesBuilder.build());\n            }\n        }\n    }\n\n    private Runnable buildProcessRequestHandler(ChannelHandlerContext ctx, RemotingCommand cmd,\n        Pair<NettyRequestProcessor, ExecutorService> pair, int opaque) {\n        return () -> {\n            Exception exception = null;\n            RemotingCommand response;\n            String remoteAddr = null;\n\n            try {\n                remoteAddr = RemotingHelper.parseChannelRemoteAddr(ctx.channel());\n                try {\n                    doBeforeRpcHooks(remoteAddr, cmd);\n                } catch (AbortProcessException e) {\n                    throw e;\n                } catch (Exception e) {\n                    exception = e;\n                }\n\n                if (this.requestPipeline != null) {\n                    this.requestPipeline.execute(ctx, cmd);\n                }\n\n                if (exception == null) {\n                    response = pair.getObject1().processRequest(ctx, cmd);\n                } else {\n                    response = RemotingCommand.createResponseCommand(RemotingSysResponseCode.SYSTEM_ERROR, null);\n                }\n\n                try {\n                    doAfterRpcHooks(remoteAddr, cmd, response);\n                } catch (AbortProcessException e) {\n                    throw e;\n                } catch (Exception e) {\n                    exception = e;\n                }\n\n                if (exception != null) {\n                    throw exception;\n                }\n\n                this.writeResponse(ctx.channel(), cmd, response, null);\n            } catch (AbortProcessException e) {\n                response = RemotingCommand.createResponseCommand(e.getResponseCode(), e.getErrorMessage());\n                response.setOpaque(opaque);\n                this.writeResponse(ctx.channel(), cmd, response, null);\n            } catch (Throwable e) {\n                log.error(\"process request exception, remoteAddr: {}\", remoteAddr, e);\n                log.error(cmd.toString());\n\n                if (!cmd.isOnewayRPC()) {\n                    response = RemotingCommand.createResponseCommand(RemotingSysResponseCode.SYSTEM_ERROR,\n                        UtilAll.exceptionSimpleDesc(e));\n                    response.setOpaque(opaque);\n                    this.writeResponse(ctx.channel(), cmd, response, null);\n                }\n            }\n        };\n    }\n\n    /**\n     * Process response from remote peer to the previous issued requests.\n     *\n     * @param ctx channel handler context.\n     * @param cmd response command instance.\n     */\n    public void processResponseCommand(ChannelHandlerContext ctx, RemotingCommand cmd) {\n        final int opaque = cmd.getOpaque();\n        final ResponseFuture responseFuture = responseTable.get(opaque);\n        if (responseFuture != null) {\n            responseFuture.setResponseCommand(cmd);\n\n            responseTable.remove(opaque);\n\n            if (responseFuture.getInvokeCallback() != null) {\n                executeInvokeCallback(responseFuture);\n            } else {\n                responseFuture.putResponse(cmd);\n                responseFuture.release();\n            }\n        } else {\n            log.warn(\"receive response, cmd={}, but not matched any request, address={}, channelId={}\", cmd, RemotingHelper.parseChannelRemoteAddr(ctx.channel()), ctx.channel().id());\n        }\n    }\n\n    /**\n     * Execute callback in callback executor. If callback executor is null, run directly in current thread\n     */\n    private void executeInvokeCallback(final ResponseFuture responseFuture) {\n        boolean runInThisThread = false;\n        ExecutorService executor = this.getCallbackExecutor();\n        if (executor != null && !executor.isShutdown()) {\n            try {\n                executor.submit(() -> {\n                    try {\n                        responseFuture.executeInvokeCallback();\n                    } catch (Throwable e) {\n                        log.warn(\"execute callback in executor exception, and callback throw\", e);\n                    } finally {\n                        responseFuture.release();\n                    }\n                });\n            } catch (Exception e) {\n                runInThisThread = true;\n                log.warn(\"execute callback in executor exception, maybe executor busy\", e);\n            }\n        } else {\n            runInThisThread = true;\n        }\n\n        if (runInThisThread) {\n            try {\n                responseFuture.executeInvokeCallback();\n            } catch (Throwable e) {\n                log.warn(\"executeInvokeCallback Exception\", e);\n            } finally {\n                responseFuture.release();\n            }\n        }\n    }\n\n    /**\n     * Custom RPC hooks.\n     *\n     * @return RPC hooks if specified; null otherwise.\n     */\n    public List<RPCHook> getRPCHook() {\n        return rpcHooks;\n    }\n\n    public void registerRPCHook(RPCHook rpcHook) {\n        if (rpcHook != null && !rpcHooks.contains(rpcHook)) {\n            rpcHooks.add(rpcHook);\n        }\n    }\n\n    public void setRequestPipeline(RequestPipeline pipeline) {\n        this.requestPipeline = pipeline;\n    }\n\n    public void clearRPCHook() {\n        rpcHooks.clear();\n    }\n\n    /**\n     * This method specifies thread pool to use while invoking callback methods.\n     *\n     * @return Dedicated thread pool instance if specified; or null if the callback is supposed to be executed in the\n     * netty event-loop thread.\n     */\n    public abstract ExecutorService getCallbackExecutor();\n\n    /**\n     * <p>\n     * This method is periodically invoked to scan and expire deprecated request.\n     * </p>\n     */\n    public void scanResponseTable() {\n        final List<ResponseFuture> rfList = new LinkedList<>();\n        Iterator<Entry<Integer, ResponseFuture>> it = this.responseTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Entry<Integer, ResponseFuture> next = it.next();\n            ResponseFuture rep = next.getValue();\n\n            if ((rep.getBeginTimestamp() + rep.getTimeoutMillis() + 1000) <= System.currentTimeMillis()) {\n                rep.release();\n                it.remove();\n                rfList.add(rep);\n                log.warn(\"remove timeout request, \" + rep);\n            }\n        }\n\n        for (ResponseFuture rf : rfList) {\n            try {\n                executeInvokeCallback(rf);\n            } catch (Throwable e) {\n                log.warn(\"scanResponseTable, operationComplete Exception\", e);\n            }\n        }\n    }\n\n    public RemotingCommand invokeSyncImpl(final Channel channel, final RemotingCommand request,\n        final long timeoutMillis)\n        throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException {\n        try {\n            return invokeImpl(channel, request, timeoutMillis).thenApply(ResponseFuture::getResponseCommand)\n                .get(timeoutMillis, TimeUnit.MILLISECONDS);\n        } catch (ExecutionException e) {\n            throw new RemotingSendRequestException(channel.remoteAddress().toString(), e.getCause());\n        } catch (TimeoutException e) {\n            throw new RemotingTimeoutException(channel.remoteAddress().toString(), timeoutMillis, e.getCause());\n        }\n    }\n\n    public CompletableFuture<ResponseFuture> invokeImpl(final Channel channel, final RemotingCommand request,\n        final long timeoutMillis) {\n        return invoke0(channel, request, timeoutMillis);\n    }\n\n    protected CompletableFuture<ResponseFuture> invoke0(final Channel channel, final RemotingCommand request,\n        final long timeoutMillis) {\n        CompletableFuture<ResponseFuture> future = new CompletableFuture<>();\n        long beginStartTime = System.currentTimeMillis();\n        final int opaque = request.getOpaque();\n\n        boolean acquired;\n        try {\n            acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n            return future;\n        }\n        if (acquired) {\n            final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreAsync);\n            long costTime = System.currentTimeMillis() - beginStartTime;\n            if (timeoutMillis < costTime) {\n                once.release();\n                future.completeExceptionally(new RemotingTimeoutException(\"invokeAsyncImpl call timeout\"));\n                return future;\n            }\n\n            AtomicReference<ResponseFuture> responseFutureReference = new AtomicReference<>();\n            final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, request, timeoutMillis - costTime,\n                new InvokeCallback() {\n                    @Override\n                    public void operationComplete(ResponseFuture responseFuture) {\n\n                    }\n\n                    @Override\n                    public void operationSucceed(RemotingCommand response) {\n                        future.complete(responseFutureReference.get());\n                    }\n\n                    @Override\n                    public void operationFail(Throwable throwable) {\n                        future.completeExceptionally(throwable);\n                    }\n                }, once);\n            responseFutureReference.set(responseFuture);\n            this.responseTable.put(opaque, responseFuture);\n            try {\n                channel.writeAndFlush(request).addListener((ChannelFutureListener) f -> {\n                    if (f.isSuccess()) {\n                        responseFuture.setSendRequestOK(true);\n                        return;\n                    }\n                    requestFail(opaque);\n                    log.warn(\"send a request command to channel <{}>, channelId={}, failed.\", RemotingHelper.parseChannelRemoteAddr(channel), channel.id());\n                });\n                return future;\n            } catch (Exception e) {\n                responseTable.remove(opaque);\n                responseFuture.release();\n                log.warn(\"send a request command to channel <{}> channelId={} Exception\", RemotingHelper.parseChannelRemoteAddr(channel), channel.id(), e);\n                future.completeExceptionally(new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e));\n                return future;\n            }\n        } else {\n            if (timeoutMillis <= 0) {\n                future.completeExceptionally(new RemotingTooMuchRequestException(\"invokeAsyncImpl invoke too fast\"));\n            } else {\n                String info =\n                    String.format(\"invokeAsyncImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreAsyncValue: %d\",\n                        timeoutMillis,\n                        this.semaphoreAsync.getQueueLength(),\n                        this.semaphoreAsync.availablePermits()\n                    );\n                log.warn(info);\n                future.completeExceptionally(new RemotingTimeoutException(info));\n            }\n            return future;\n        }\n    }\n\n    public void invokeAsyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis,\n        final InvokeCallback invokeCallback) {\n        invokeImpl(channel, request, timeoutMillis)\n            .whenComplete((v, t) -> {\n                if (t == null) {\n                    invokeCallback.operationComplete(v);\n                } else {\n                    ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, timeoutMillis, null, null);\n                    responseFuture.setCause(t);\n                    invokeCallback.operationComplete(responseFuture);\n                }\n            })\n            .thenAccept(responseFuture -> invokeCallback.operationSucceed(responseFuture.getResponseCommand()))\n            .exceptionally(t -> {\n                invokeCallback.operationFail(ExceptionUtils.getRealException(t));\n                return null;\n            });\n    }\n\n    private void requestFail(final int opaque) {\n        ResponseFuture responseFuture = responseTable.remove(opaque);\n        if (responseFuture != null) {\n            responseFuture.setSendRequestOK(false);\n            responseFuture.putResponse(null);\n            try {\n                executeInvokeCallback(responseFuture);\n            } catch (Throwable e) {\n                log.warn(\"execute callback in requestFail, and callback throw\", e);\n            } finally {\n                responseFuture.release();\n            }\n        }\n    }\n\n    /**\n     * mark the request of the specified channel as fail and to invoke fail callback immediately\n     *\n     * @param channel the channel which is close already\n     */\n    protected void failFast(final Channel channel) {\n        for (Entry<Integer, ResponseFuture> entry : responseTable.entrySet()) {\n            if (entry.getValue().getChannel() == channel) {\n                Integer opaque = entry.getKey();\n                if (opaque != null) {\n                    requestFail(opaque);\n                }\n            }\n        }\n    }\n\n    public void invokeOnewayImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis)\n        throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException {\n        request.markOnewayRPC();\n        boolean acquired = this.semaphoreOneway.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS);\n        if (acquired) {\n            final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreOneway);\n            try {\n                channel.writeAndFlush(request).addListener((ChannelFutureListener) f -> {\n                    once.release();\n                    if (!f.isSuccess()) {\n                        log.warn(\"send a request command to channel <\" + channel.remoteAddress() + \"> failed.\");\n                    }\n                });\n            } catch (Exception e) {\n                once.release();\n                log.warn(\"write send a request command to channel <\" + channel.remoteAddress() + \"> failed.\");\n                throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e);\n            }\n        } else {\n            if (timeoutMillis <= 0) {\n                throw new RemotingTooMuchRequestException(\"invokeOnewayImpl invoke too fast\");\n            } else {\n                String info = String.format(\n                    \"invokeOnewayImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreOnewayValue: %d\",\n                    timeoutMillis,\n                    this.semaphoreOneway.getQueueLength(),\n                    this.semaphoreOneway.availablePermits()\n                );\n                log.warn(info);\n                throw new RemotingTimeoutException(info);\n            }\n        }\n    }\n\n    public HashMap<Integer, Pair<NettyRequestProcessor, ExecutorService>> getProcessorTable() {\n        return processorTable;\n    }\n\n    class NettyEventExecutor extends ServiceThread {\n        private final LinkedBlockingQueue<NettyEvent> eventQueue = new LinkedBlockingQueue<>();\n\n        public void putNettyEvent(final NettyEvent event) {\n            int currentSize = this.eventQueue.size();\n            int maxSize = 10000;\n            if (currentSize <= maxSize) {\n                this.eventQueue.add(event);\n            } else {\n                log.warn(\"event queue size [{}] over the limit [{}], so drop this event {}\", currentSize, maxSize, event.toString());\n            }\n        }\n\n        @Override\n        public void run() {\n            log.info(this.getServiceName() + \" service started\");\n\n            final ChannelEventListener listener = NettyRemotingAbstract.this.getChannelEventListener();\n\n            while (!this.isStopped()) {\n                try {\n                    NettyEvent event = this.eventQueue.poll(3000, TimeUnit.MILLISECONDS);\n                    if (event != null && listener != null) {\n                        switch (event.getType()) {\n                            case IDLE:\n                                listener.onChannelIdle(event.getRemoteAddr(), event.getChannel());\n                                break;\n                            case CLOSE:\n                                listener.onChannelClose(event.getRemoteAddr(), event.getChannel());\n                                break;\n                            case CONNECT:\n                                listener.onChannelConnect(event.getRemoteAddr(), event.getChannel());\n                                break;\n                            case EXCEPTION:\n                                listener.onChannelException(event.getRemoteAddr(), event.getChannel());\n                                break;\n                            case ACTIVE:\n                                listener.onChannelActive(event.getRemoteAddr(), event.getChannel());\n                                break;\n                            default:\n                                break;\n\n                        }\n                    }\n                } catch (Exception e) {\n                    log.warn(this.getServiceName() + \" service has exception. \", e);\n                }\n            }\n\n            log.info(this.getServiceName() + \" service end\");\n        }\n\n        @Override\n        public String getServiceName() {\n            return NettyEventExecutor.class.getSimpleName();\n        }\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.netty;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.alibaba.fastjson2.TypeReference;\nimport com.google.common.base.Stopwatch;\nimport io.netty.bootstrap.Bootstrap;\nimport io.netty.buffer.PooledByteBufAllocator;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelDuplexHandler;\nimport io.netty.channel.ChannelFuture;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.channel.ChannelInitializer;\nimport io.netty.channel.ChannelOption;\nimport io.netty.channel.ChannelPipeline;\nimport io.netty.channel.ChannelPromise;\nimport io.netty.channel.EventLoopGroup;\nimport io.netty.channel.SimpleChannelInboundHandler;\nimport io.netty.channel.WriteBufferWaterMark;\nimport io.netty.channel.nio.NioEventLoopGroup;\nimport io.netty.channel.socket.SocketChannel;\nimport io.netty.channel.socket.nio.NioSocketChannel;\nimport io.netty.handler.proxy.Socks5ProxyHandler;\nimport io.netty.handler.timeout.IdleState;\nimport io.netty.handler.timeout.IdleStateEvent;\nimport io.netty.handler.timeout.IdleStateHandler;\nimport io.netty.resolver.NoopAddressResolverGroup;\nimport io.netty.util.AttributeKey;\nimport io.netty.util.HashedWheelTimer;\nimport io.netty.util.Timeout;\nimport io.netty.util.TimerTask;\nimport io.netty.util.concurrent.DefaultEventExecutorGroup;\nimport io.netty.util.concurrent.EventExecutorGroup;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.FutureUtils;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.ChannelEventListener;\nimport org.apache.rocketmq.remoting.InvokeCallback;\nimport org.apache.rocketmq.remoting.RemotingClient;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingConnectException;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.proxy.SocksProxyConfig;\n\nimport java.io.IOException;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.security.cert.CertificateException;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Random;\nimport java.util.Set;\nimport java.util.concurrent.ArrayBlockingQueue;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicReference;\nimport java.util.concurrent.locks.Lock;\nimport java.util.concurrent.locks.ReentrantLock;\nimport java.util.concurrent.locks.ReentrantReadWriteLock;\n\nimport static org.apache.rocketmq.remoting.common.RemotingHelper.convertChannelFutureToCompletableFuture;\n\npublic class NettyRemotingClient extends NettyRemotingAbstract implements RemotingClient {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME);\n\n    private static final AttributeKey<ChannelWrapper> CHANNEL_WRAPPER_ATTRIBUTE_KEY = AttributeKey.valueOf(\n        \"channelWrapper\");\n\n    private static final long LOCK_TIMEOUT_MILLIS = 3000;\n    private static final long MIN_CLOSE_TIMEOUT_MILLIS = 100;\n\n    protected final NettyClientConfig nettyClientConfig;\n    private final Bootstrap bootstrap = new Bootstrap();\n    private final EventLoopGroup eventLoopGroupWorker;\n    private final Lock lockChannelTables = new ReentrantLock();\n    private final Map<String /* cidr */, SocksProxyConfig /* proxy */> proxyMap = new HashMap<>();\n    private final ConcurrentHashMap<String /* cidr */, Bootstrap> bootstrapMap = new ConcurrentHashMap<>();\n    private final ConcurrentMap<String /* addr */, ChannelWrapper> channelTables = new ConcurrentHashMap<>();\n\n    private final HashedWheelTimer timer = new HashedWheelTimer(r -> new Thread(r, \"ClientHouseKeepingService\"));\n\n    private final AtomicReference<List<String>> namesrvAddrList = new AtomicReference<>();\n    private final ConcurrentMap<String, Boolean> availableNamesrvAddrMap = new ConcurrentHashMap<>();\n    private final AtomicReference<String> namesrvAddrChoosed = new AtomicReference<>();\n    private final AtomicInteger namesrvIndex = new AtomicInteger(initValueIndex());\n    private final Lock namesrvChannelLock = new ReentrantLock();\n\n    private final ExecutorService publicExecutor;\n    private final ExecutorService scanExecutor;\n\n    /**\n     * Invoke the callback methods in this executor when process response.\n     */\n    private ExecutorService callbackExecutor;\n    private final ChannelEventListener channelEventListener;\n    private EventExecutorGroup defaultEventExecutorGroup;\n\n    public NettyRemotingClient(final NettyClientConfig nettyClientConfig) {\n        this(nettyClientConfig, null);\n    }\n\n    public NettyRemotingClient(final NettyClientConfig nettyClientConfig,\n        final ChannelEventListener channelEventListener) {\n        this(nettyClientConfig, channelEventListener, null, null);\n    }\n\n    public NettyRemotingClient(final NettyClientConfig nettyClientConfig,\n        final ChannelEventListener channelEventListener,\n        final EventLoopGroup eventLoopGroup,\n        final EventExecutorGroup eventExecutorGroup) {\n        super(nettyClientConfig.getClientOnewaySemaphoreValue(), nettyClientConfig.getClientAsyncSemaphoreValue());\n        this.nettyClientConfig = nettyClientConfig;\n        this.channelEventListener = channelEventListener;\n\n        this.loadSocksProxyJson();\n\n        int publicThreadNums = nettyClientConfig.getClientCallbackExecutorThreads();\n        if (publicThreadNums <= 0) {\n            publicThreadNums = 4;\n        }\n\n        this.publicExecutor = Executors.newFixedThreadPool(publicThreadNums, new ThreadFactoryImpl(\"NettyClientPublicExecutor_\"));\n\n        this.scanExecutor = ThreadUtils.newThreadPoolExecutor(4, 10, 60, TimeUnit.SECONDS,\n            new ArrayBlockingQueue<>(32), new ThreadFactoryImpl(\"NettyClientScan_thread_\"));\n\n        if (eventLoopGroup != null) {\n            this.eventLoopGroupWorker = eventLoopGroup;\n        } else {\n            this.eventLoopGroupWorker = new NioEventLoopGroup(1, new ThreadFactoryImpl(\"NettyClientSelector_\"));\n        }\n        this.defaultEventExecutorGroup = eventExecutorGroup;\n\n        if (nettyClientConfig.isUseTLS()) {\n            try {\n                sslContext = TlsHelper.buildSslContext(true);\n                LOGGER.info(\"SSL enabled for client\");\n            } catch (IOException e) {\n                LOGGER.error(\"Failed to create SslContext\", e);\n            } catch (CertificateException e) {\n                LOGGER.error(\"Failed to create SslContext\", e);\n                throw new RuntimeException(\"Failed to create SslContext\", e);\n            }\n        }\n    }\n\n    private static int initValueIndex() {\n        Random r = new Random();\n        return r.nextInt(999);\n    }\n\n    private void loadSocksProxyJson() {\n        Map<String, SocksProxyConfig> sockProxyMap = JSON.parseObject(\n            nettyClientConfig.getSocksProxyConfig(), new TypeReference<Map<String, SocksProxyConfig>>() {\n            });\n        if (sockProxyMap != null) {\n            proxyMap.putAll(sockProxyMap);\n        }\n    }\n\n    @Override\n    public void start() {\n        if (this.defaultEventExecutorGroup == null) {\n            this.defaultEventExecutorGroup = new DefaultEventExecutorGroup(\n                nettyClientConfig.getClientWorkerThreads(),\n                new ThreadFactoryImpl(\"NettyClientWorkerThread_\"));\n        }\n        Bootstrap handler = this.bootstrap.group(this.eventLoopGroupWorker).channel(NioSocketChannel.class)\n            .option(ChannelOption.TCP_NODELAY, true)\n            .option(ChannelOption.SO_KEEPALIVE, false)\n            .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, nettyClientConfig.getConnectTimeoutMillis())\n            .handler(new ChannelInitializer<SocketChannel>() {\n                @Override\n                public void initChannel(SocketChannel ch) throws Exception {\n                    ChannelPipeline pipeline = ch.pipeline();\n                    if (nettyClientConfig.isUseTLS()) {\n                        if (null != sslContext) {\n                            pipeline.addFirst(defaultEventExecutorGroup, \"sslHandler\", sslContext.newHandler(ch.alloc()));\n                            LOGGER.info(\"Prepend SSL handler\");\n                        } else {\n                            LOGGER.warn(\"Connections are insecure as SslContext is null!\");\n                        }\n                    }\n                    ch.pipeline().addLast(\n                        nettyClientConfig.isDisableNettyWorkerGroup() ? null : defaultEventExecutorGroup,\n                        new NettyEncoder(),\n                        new NettyDecoder(),\n                        new IdleStateHandler(0, 0, nettyClientConfig.getClientChannelMaxIdleTimeSeconds()),\n                        new NettyConnectManageHandler(),\n                        new NettyClientHandler());\n                }\n            });\n        if (nettyClientConfig.getClientSocketSndBufSize() > 0) {\n            LOGGER.info(\"client set SO_SNDBUF to {}\", nettyClientConfig.getClientSocketSndBufSize());\n            handler.option(ChannelOption.SO_SNDBUF, nettyClientConfig.getClientSocketSndBufSize());\n        }\n        if (nettyClientConfig.getClientSocketRcvBufSize() > 0) {\n            LOGGER.info(\"client set SO_RCVBUF to {}\", nettyClientConfig.getClientSocketRcvBufSize());\n            handler.option(ChannelOption.SO_RCVBUF, nettyClientConfig.getClientSocketRcvBufSize());\n        }\n        if (nettyClientConfig.getWriteBufferLowWaterMark() > 0 && nettyClientConfig.getWriteBufferHighWaterMark() > 0) {\n            LOGGER.info(\"client set netty WRITE_BUFFER_WATER_MARK to {},{}\",\n                nettyClientConfig.getWriteBufferLowWaterMark(), nettyClientConfig.getWriteBufferHighWaterMark());\n            handler.option(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark(\n                nettyClientConfig.getWriteBufferLowWaterMark(), nettyClientConfig.getWriteBufferHighWaterMark()));\n        }\n        if (nettyClientConfig.isClientPooledByteBufAllocatorEnable()) {\n            handler.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);\n        }\n\n        nettyEventExecutor.start();\n\n        TimerTask timerTaskScanResponseTable = new TimerTask() {\n            @Override\n            public void run(Timeout timeout) {\n                try {\n                    NettyRemotingClient.this.scanResponseTable();\n                } catch (Throwable e) {\n                    LOGGER.error(\"scanResponseTable exception\", e);\n                } finally {\n                    timer.newTimeout(this, 1000, TimeUnit.MILLISECONDS);\n                }\n            }\n        };\n        this.timer.newTimeout(timerTaskScanResponseTable, 1000 * 3, TimeUnit.MILLISECONDS);\n\n        if (nettyClientConfig.isScanAvailableNameSrv()) {\n            int connectTimeoutMillis = this.nettyClientConfig.getConnectTimeoutMillis();\n            TimerTask timerTaskScanAvailableNameSrv = new TimerTask() {\n                @Override\n                public void run(Timeout timeout) {\n                    try {\n                        NettyRemotingClient.this.scanAvailableNameSrv();\n                    } catch (Exception e) {\n                        LOGGER.error(\"scanAvailableNameSrv exception\", e);\n                    } finally {\n                        timer.newTimeout(this, connectTimeoutMillis, TimeUnit.MILLISECONDS);\n                    }\n                }\n            };\n            this.timer.newTimeout(timerTaskScanAvailableNameSrv, 0, TimeUnit.MILLISECONDS);\n        }\n        \n        \n    }\n\n    private Map.Entry<String, SocksProxyConfig> getProxy(String addr) {\n        if (StringUtils.isBlank(addr) || !addr.contains(\":\")) {\n            return null;\n        }\n        String[] hostAndPort = this.getHostAndPort(addr);\n        for (Map.Entry<String, SocksProxyConfig> entry : proxyMap.entrySet()) {\n            String cidr = entry.getKey();\n            if (RemotingHelper.DEFAULT_CIDR_ALL.equals(cidr) || RemotingHelper.ipInCIDR(hostAndPort[0], cidr)) {\n                return entry;\n            }\n        }\n        return null;\n    }\n\n    protected ChannelFuture doConnect(String addr) {\n        String[] hostAndPort = getHostAndPort(addr);\n        String host = hostAndPort[0];\n        int port = Integer.parseInt(hostAndPort[1]);\n        return fetchBootstrap(addr).connect(host, port);\n    }\n\n    private Bootstrap fetchBootstrap(String addr) {\n        Map.Entry<String, SocksProxyConfig> proxyEntry = getProxy(addr);\n        if (proxyEntry == null) {\n            return bootstrap;\n        }\n\n        String cidr = proxyEntry.getKey();\n        SocksProxyConfig socksProxyConfig = proxyEntry.getValue();\n\n        LOGGER.info(\"Netty fetch bootstrap, addr: {}, cidr: {}, proxy: {}\",\n            addr, cidr, socksProxyConfig != null ? socksProxyConfig.getAddr() : \"\");\n\n        Bootstrap bootstrapWithProxy = bootstrapMap.get(cidr);\n        if (bootstrapWithProxy == null) {\n            bootstrapWithProxy = createBootstrap(socksProxyConfig);\n            Bootstrap old = bootstrapMap.putIfAbsent(cidr, bootstrapWithProxy);\n            if (old != null) {\n                bootstrapWithProxy = old;\n            }\n        }\n        return bootstrapWithProxy;\n    }\n\n    private Bootstrap createBootstrap(final SocksProxyConfig proxy) {\n        Bootstrap bootstrap = new Bootstrap();\n        bootstrap.group(this.eventLoopGroupWorker).channel(NioSocketChannel.class)\n            .option(ChannelOption.TCP_NODELAY, true)\n            .option(ChannelOption.SO_KEEPALIVE, false)\n            .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, nettyClientConfig.getConnectTimeoutMillis())\n            .option(ChannelOption.SO_SNDBUF, nettyClientConfig.getClientSocketSndBufSize())\n            .option(ChannelOption.SO_RCVBUF, nettyClientConfig.getClientSocketRcvBufSize())\n            .handler(new ChannelInitializer<SocketChannel>() {\n                @Override\n                public void initChannel(SocketChannel ch) {\n                    ChannelPipeline pipeline = ch.pipeline();\n                    if (nettyClientConfig.isUseTLS()) {\n                        if (null != sslContext) {\n                            pipeline.addFirst(defaultEventExecutorGroup,\n                                \"sslHandler\", sslContext.newHandler(ch.alloc()));\n                            LOGGER.info(\"Prepend SSL handler\");\n                        } else {\n                            LOGGER.warn(\"Connections are insecure as SslContext is null!\");\n                        }\n                    }\n\n                    // Netty Socks5 Proxy\n                    if (proxy != null) {\n                        String[] hostAndPort = getHostAndPort(proxy.getAddr());\n                        pipeline.addFirst(new Socks5ProxyHandler(\n                            new InetSocketAddress(hostAndPort[0], Integer.parseInt(hostAndPort[1])),\n                            proxy.getUsername(), proxy.getPassword()));\n                    }\n\n                    pipeline.addLast(\n                        nettyClientConfig.isDisableNettyWorkerGroup() ? null : defaultEventExecutorGroup,\n                        new NettyEncoder(),\n                        new NettyDecoder(),\n                        new IdleStateHandler(0, 0, nettyClientConfig.getClientChannelMaxIdleTimeSeconds()),\n                        new NettyConnectManageHandler(),\n                        new NettyClientHandler());\n                }\n            });\n\n        // Support Netty Socks5 Proxy\n        if (proxy != null) {\n            bootstrap.resolver(NoopAddressResolverGroup.INSTANCE);\n        }\n        return bootstrap;\n    }\n\n    // Do not use RemotingHelper.string2SocketAddress(), it will directly resolve the domain\n    protected String[] getHostAndPort(String address) {\n        int split = address.lastIndexOf(\":\");\n        return split < 0 ? new String[]{address} : new String[]{address.substring(0, split), address.substring(split + 1)};\n    }\n\n    @Override\n    public void shutdown() {\n        try {\n            this.timer.stop();\n\n            for (Map.Entry<String, ChannelWrapper> channel : this.channelTables.entrySet()) {\n                channel.getValue().close();\n            }\n\n            this.channelTables.clear();\n\n            this.eventLoopGroupWorker.shutdownGracefully();\n\n            if (this.nettyEventExecutor != null) {\n                this.nettyEventExecutor.shutdown();\n            }\n\n            if (this.defaultEventExecutorGroup != null) {\n                this.defaultEventExecutorGroup.shutdownGracefully();\n            }\n        } catch (Exception e) {\n            LOGGER.error(\"NettyRemotingClient shutdown exception, \", e);\n        }\n\n        if (this.publicExecutor != null) {\n            try {\n                this.publicExecutor.shutdown();\n            } catch (Exception e) {\n                LOGGER.error(\"NettyRemotingServer shutdown exception, \", e);\n            }\n        }\n\n        if (this.scanExecutor != null) {\n            try {\n                this.scanExecutor.shutdown();\n            } catch (Exception e) {\n                LOGGER.error(\"NettyRemotingServer shutdown exception, \", e);\n            }\n        }\n    }\n\n    public void closeChannel(final String addr, final Channel channel) {\n        if (null == channel) {\n            return;\n        }\n\n        final String addrRemote = null == addr ? RemotingHelper.parseChannelRemoteAddr(channel) : addr;\n\n        try {\n            if (this.lockChannelTables.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {\n                try {\n                    boolean removeItemFromTable = true;\n                    final ChannelWrapper prevCW = this.channelTables.get(addrRemote);\n\n                    LOGGER.info(\"closeChannel: begin close the channel[addr={}, id={}] Found: {}\", addrRemote, channel.id(), prevCW != null);\n\n                    if (null == prevCW) {\n                        LOGGER.info(\"closeChannel: the channel[addr={}, id={}] has been removed from the channel table before\", addrRemote, channel.id());\n                        removeItemFromTable = false;\n                    } else if (prevCW.isWrapperOf(channel)) {\n                        LOGGER.info(\"closeChannel: the channel[addr={}, id={}] has been closed before, and has been created again, nothing to do.\",\n                            addrRemote, channel.id());\n                        removeItemFromTable = false;\n                    }\n\n                    if (removeItemFromTable) {\n                        ChannelWrapper channelWrapper =\n                            RemotingHelper.getAttributeValue(CHANNEL_WRAPPER_ATTRIBUTE_KEY, channel);\n                        if (channelWrapper != null && channelWrapper.tryClose(channel)) {\n                            this.channelTables.remove(addrRemote);\n                        }\n                        LOGGER.info(\"closeChannel: the channel[addr={}, id={}] was removed from channel table\", addrRemote, channel.id());\n                    }\n\n                    RemotingHelper.closeChannel(channel);\n                } catch (Exception e) {\n                    LOGGER.error(\"closeChannel: close the channel exception\", e);\n                } finally {\n                    this.lockChannelTables.unlock();\n                }\n            } else {\n                LOGGER.warn(\"closeChannel: try to lock channel table, but timeout, {}ms\", LOCK_TIMEOUT_MILLIS);\n            }\n        } catch (InterruptedException e) {\n            LOGGER.error(\"closeChannel exception\", e);\n        }\n    }\n\n    public void closeChannel(final Channel channel) {\n        if (null == channel) {\n            return;\n        }\n\n        try {\n            if (this.lockChannelTables.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {\n                try {\n                    boolean removeItemFromTable = true;\n                    ChannelWrapper prevCW = null;\n                    String addrRemote = null;\n                    for (Map.Entry<String, ChannelWrapper> entry : channelTables.entrySet()) {\n                        String key = entry.getKey();\n                        ChannelWrapper prev = entry.getValue();\n                        if (prev.isWrapperOf(channel)) {\n                            prevCW = prev;\n                            addrRemote = key;\n                            break;\n                        }\n                    }\n\n                    if (null == prevCW) {\n                        LOGGER.info(\"eventCloseChannel: the channel[addr={}, id={}] has been removed from the channel table before\", RemotingHelper.parseChannelRemoteAddr(channel), channel.id());\n                        removeItemFromTable = false;\n                    }\n\n                    if (removeItemFromTable) {\n                        ChannelWrapper channelWrapper =\n                            RemotingHelper.getAttributeValue(CHANNEL_WRAPPER_ATTRIBUTE_KEY, channel);\n                        if (channelWrapper != null && channelWrapper.tryClose(channel)) {\n                            this.channelTables.remove(addrRemote);\n                        }\n                        LOGGER.info(\"closeChannel: the channel[addr={}, id={}] was removed from channel table\", addrRemote, channel.id());\n                        RemotingHelper.closeChannel(channel);\n                    }\n                } catch (Exception e) {\n                    LOGGER.error(\"closeChannel: close the channel[id={}] exception\", channel.id(), e);\n                } finally {\n                    this.lockChannelTables.unlock();\n                }\n            } else {\n                LOGGER.warn(\"closeChannel: try to lock channel table, but timeout, {}ms\", LOCK_TIMEOUT_MILLIS);\n            }\n        } catch (InterruptedException e) {\n            LOGGER.error(\"closeChannel exception\", e);\n        }\n    }\n\n    @Override\n    public void updateNameServerAddressList(List<String> addrs) {\n        List<String> old = this.namesrvAddrList.get();\n        boolean update = false;\n\n        if (!addrs.isEmpty()) {\n            if (null == old) {\n                update = true;\n            } else if (addrs.size() != old.size()) {\n                update = true;\n            } else {\n                for (String addr : addrs) {\n                    if (!old.contains(addr)) {\n                        update = true;\n                        break;\n                    }\n                }\n            }\n\n            if (update) {\n                Collections.shuffle(addrs);\n                LOGGER.info(\"name server address updated. NEW : {} , OLD: {}\", addrs, old);\n                this.namesrvAddrList.set(addrs);\n\n                // should close the channel if choosed addr is not exist.\n                String chosenNameServerAddr = this.namesrvAddrChoosed.get();\n                if (chosenNameServerAddr != null && !addrs.contains(chosenNameServerAddr)) {\n                    namesrvAddrChoosed.compareAndSet(chosenNameServerAddr, null);\n                    for (String addr : this.channelTables.keySet()) {\n                        if (addr.contains(chosenNameServerAddr)) {\n                            ChannelWrapper channelWrapper = this.channelTables.get(addr);\n                            if (channelWrapper != null) {\n                                channelWrapper.close();\n                            }\n                        }\n                    }\n                }\n            }\n        }\n    }\n\n    @Override\n    public RemotingCommand invokeSync(String addr, final RemotingCommand request, long timeoutMillis)\n        throws InterruptedException, RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException {\n        long beginStartTime = System.currentTimeMillis();\n        final Channel channel = this.getAndCreateChannel(addr);\n        String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel);\n        if (channel != null && channel.isActive()) {\n            long left = timeoutMillis;\n            try {\n                long costTime = System.currentTimeMillis() - beginStartTime;\n                left -= costTime;\n                if (left <= 0) {\n                    throw new RemotingTimeoutException(\"invokeSync call the addr[\" + channelRemoteAddr + \"] timeout\");\n                }\n                RemotingCommand response = this.invokeSyncImpl(channel, request, left);\n                updateChannelLastResponseTime(addr);\n                return response;\n            } catch (RemotingSendRequestException e) {\n                LOGGER.warn(\"invokeSync: send request exception, so close the channel[addr={}, id={}]\", channelRemoteAddr, channel.id());\n                this.closeChannel(addr, channel);\n                throw e;\n            } catch (RemotingTimeoutException e) {\n                // avoid close the success channel if left timeout is small, since it may cost too much time in get the success channel, the left timeout for read is small\n                boolean shouldClose = left > MIN_CLOSE_TIMEOUT_MILLIS || left > timeoutMillis / 4;\n                if (nettyClientConfig.isClientCloseSocketIfTimeout() && shouldClose) {\n                    this.closeChannel(addr, channel);\n                    LOGGER.warn(\"invokeSync: close socket because of timeout, {}ms, channel[addr={}, id={}]\", timeoutMillis, channelRemoteAddr, channel.id());\n                }\n                LOGGER.warn(\"invokeSync: wait response timeout exception, the channel[addr={}, id={}]\", channelRemoteAddr, channel.id());\n                throw e;\n            }\n        } else {\n            this.closeChannel(addr, channel);\n            throw new RemotingConnectException(channelRemoteAddr);\n        }\n    }\n\n    @Override\n    public void closeChannels(List<String> addrList) {\n        for (String addr : addrList) {\n            ChannelWrapper cw = this.channelTables.get(addr);\n            if (cw == null) {\n                continue;\n            }\n            this.closeChannel(addr, cw.getChannel());\n        }\n        interruptPullRequests(new HashSet<>(addrList));\n    }\n\n    private void interruptPullRequests(Set<String> brokerAddrSet) {\n        for (ResponseFuture responseFuture : responseTable.values()) {\n            RemotingCommand cmd = responseFuture.getRequestCommand();\n            if (cmd == null) {\n                continue;\n            }\n            String remoteAddr = RemotingHelper.parseChannelRemoteAddr(responseFuture.getChannel());\n            // interrupt only pull message request\n            if (brokerAddrSet.contains(remoteAddr) && (cmd.getCode() == RequestCode.PULL_MESSAGE || cmd.getCode() == RequestCode.LITE_PULL_MESSAGE)) {\n                LOGGER.info(\"interrupt {}\", cmd);\n                responseFuture.interrupt();\n            }\n        }\n    }\n\n    private void updateChannelLastResponseTime(final String addr) {\n        String address = addr;\n        if (address == null) {\n            address = this.namesrvAddrChoosed.get();\n        }\n        if (address == null) {\n            LOGGER.warn(\"[updateChannelLastResponseTime] could not find address!!\");\n            return;\n        }\n        ChannelWrapper channelWrapper = this.channelTables.get(address);\n        if (channelWrapper != null && channelWrapper.isOK()) {\n            channelWrapper.updateLastResponseTime();\n        }\n    }\n\n    private ChannelFuture getAndCreateChannelAsync(final String addr) throws InterruptedException {\n        if (null == addr) {\n            return getAndCreateNameserverChannelAsync();\n        }\n\n        ChannelWrapper cw = this.channelTables.get(addr);\n        if (cw != null && cw.isOK()) {\n            return cw.getChannelFuture();\n        }\n\n        return this.createChannelAsync(addr);\n    }\n\n    private Channel getAndCreateChannel(final String addr) throws InterruptedException {\n        ChannelFuture channelFuture = getAndCreateChannelAsync(addr);\n        if (channelFuture == null) {\n            return null;\n        }\n        return channelFuture.awaitUninterruptibly().channel();\n    }\n\n    private ChannelFuture getAndCreateNameserverChannelAsync() throws InterruptedException {\n        String addr = this.namesrvAddrChoosed.get();\n        if (addr != null) {\n            ChannelWrapper cw = this.channelTables.get(addr);\n            if (cw != null && cw.isOK()) {\n                return cw.getChannelFuture();\n            }\n        }\n\n        final List<String> addrList = this.namesrvAddrList.get();\n        if (this.namesrvChannelLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {\n            try {\n                addr = this.namesrvAddrChoosed.get();\n                if (addr != null) {\n                    ChannelWrapper cw = this.channelTables.get(addr);\n                    if (cw != null && cw.isOK()) {\n                        return cw.getChannelFuture();\n                    }\n                }\n\n                if (addrList != null && !addrList.isEmpty()) {\n                    int index = this.namesrvIndex.incrementAndGet();\n                    index = Math.abs(index);\n                    index = index % addrList.size();\n                    String newAddr = addrList.get(index);\n\n                    this.namesrvAddrChoosed.set(newAddr);\n                    LOGGER.info(\"new name server is chosen. OLD: {} , NEW: {}. namesrvIndex = {}\", addr, newAddr, namesrvIndex);\n                    return this.createChannelAsync(newAddr);\n                }\n            } catch (Exception e) {\n                LOGGER.error(\"getAndCreateNameserverChannel: create name server channel exception\", e);\n            } finally {\n                this.namesrvChannelLock.unlock();\n            }\n        } else {\n            LOGGER.warn(\"getAndCreateNameserverChannel: try to lock name server, but timeout, {}ms\", LOCK_TIMEOUT_MILLIS);\n        }\n\n        return null;\n    }\n\n    private ChannelFuture createChannelAsync(final String addr) throws InterruptedException {\n        ChannelWrapper cw = this.channelTables.get(addr);\n        if (cw != null && cw.isOK()) {\n            return cw.getChannelFuture();\n        }\n\n        if (this.lockChannelTables.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {\n            try {\n                cw = this.channelTables.get(addr);\n                if (cw != null) {\n                    if (cw.isOK() || !cw.getChannelFuture().isDone()) {\n                        return cw.getChannelFuture();\n                    } else {\n                        this.channelTables.remove(addr);\n                    }\n                }\n                return createChannel(addr).getChannelFuture();\n            } catch (Exception e) {\n                LOGGER.error(\"createChannel: create channel exception\", e);\n            } finally {\n                this.lockChannelTables.unlock();\n            }\n        } else {\n            LOGGER.warn(\"createChannel: try to lock channel table, but timeout, {}ms\", LOCK_TIMEOUT_MILLIS);\n        }\n\n        return null;\n    }\n\n    private ChannelWrapper createChannel(String addr) {\n        ChannelFuture channelFuture = doConnect(addr);\n        LOGGER.info(\"createChannel: begin to connect remote host[{}] asynchronously\", addr);\n        ChannelWrapper cw = new ChannelWrapper(addr, channelFuture);\n        this.channelTables.put(addr, cw);\n        return cw;\n    }\n\n    @Override\n    public void invokeAsync(String addr, RemotingCommand request, long timeoutMillis, InvokeCallback invokeCallback)\n        throws InterruptedException, RemotingConnectException, RemotingTooMuchRequestException, RemotingTimeoutException,\n        RemotingSendRequestException {\n        long beginStartTime = System.currentTimeMillis();\n        final ChannelFuture channelFuture = this.getAndCreateChannelAsync(addr);\n        if (channelFuture == null) {\n            invokeCallback.operationFail(new RemotingConnectException(addr));\n            return;\n        }\n        channelFuture.addListener(future -> {\n            if (future.isSuccess()) {\n                Channel channel = channelFuture.channel();\n                String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel);\n                if (channel != null && channel.isActive()) {\n                    long costTime = System.currentTimeMillis() - beginStartTime;\n                    if (timeoutMillis < costTime) {\n                        invokeCallback.operationFail(new RemotingTooMuchRequestException(\"invokeAsync call the addr[\" + channelRemoteAddr + \"] timeout\"));\n                        return;\n                    }\n                    this.invokeAsyncImpl(channel, request, timeoutMillis - costTime, new InvokeCallbackWrapper(invokeCallback, addr));\n                } else {\n                    this.closeChannel(addr, channel);\n                    invokeCallback.operationFail(new RemotingConnectException(addr));\n                }\n            } else {\n                invokeCallback.operationFail(new RemotingConnectException(addr));\n            }\n        });\n    }\n\n    @Override\n    public void invokeOneway(String addr, RemotingCommand request, long timeoutMillis) throws InterruptedException,\n        RemotingConnectException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException {\n        final ChannelFuture channelFuture = this.getAndCreateChannelAsync(addr);\n        if (channelFuture == null) {\n            throw new RemotingConnectException(addr);\n        }\n        channelFuture.addListener(future -> {\n            if (future.isSuccess()) {\n                Channel channel = channelFuture.channel();\n                String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel);\n                if (channel != null && channel.isActive()) {\n                    doBeforeRpcHooks(channelRemoteAddr, request);\n                    this.invokeOnewayImpl(channel, request, timeoutMillis);\n                } else {\n                    this.closeChannel(addr, channel);\n                }\n            }\n        });\n    }\n\n    @Override\n    public CompletableFuture<RemotingCommand> invoke(String addr, RemotingCommand request,\n        long timeoutMillis) {\n        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();\n        try {\n            final ChannelFuture channelFuture = this.getAndCreateChannelAsync(addr);\n            if (channelFuture == null) {\n                future.completeExceptionally(new RemotingConnectException(addr));\n                return future;\n            }\n            channelFuture.addListener(f -> {\n                if (f.isSuccess()) {\n                    Channel channel = channelFuture.channel();\n                    if (channel != null && channel.isActive()) {\n                        invokeImpl(channel, request, timeoutMillis).whenComplete((v, t) -> {\n                            if (t == null) {\n                                updateChannelLastResponseTime(addr);\n                            }\n                        }).thenApply(ResponseFuture::getResponseCommand).whenComplete((v, t) -> {\n                            if (t != null) {\n                                future.completeExceptionally(t);\n                            } else {\n                                future.complete(v);\n                            }\n                        });\n                    } else {\n                        this.closeChannel(addr, channel);\n                        future.completeExceptionally(new RemotingConnectException(addr));\n                    }\n                } else {\n                    future.completeExceptionally(new RemotingConnectException(addr));\n                }\n            });\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    @Override\n    public CompletableFuture<ResponseFuture> invokeImpl(final Channel channel, final RemotingCommand request,\n        final long timeoutMillis) {\n        Stopwatch stopwatch = Stopwatch.createStarted();\n        String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel);\n        doBeforeRpcHooks(channelRemoteAddr, request);\n\n        return super.invokeImpl(channel, request, timeoutMillis).thenCompose(responseFuture -> {\n            RemotingCommand response = responseFuture.getResponseCommand();\n            if (response.getCode() == ResponseCode.GO_AWAY) {\n                if (nettyClientConfig.isEnableReconnectForGoAway()) {\n                    LOGGER.info(\"Receive go away from channelId={}, channel={}\", channel.id(), channel);\n                    ChannelWrapper channelWrapper = RemotingHelper.getAttributeValue(CHANNEL_WRAPPER_ATTRIBUTE_KEY,\n                        channel);\n                    if (channelWrapper != null && channelWrapper.reconnect(channel)) {\n                        LOGGER.info(\"Receive go away from channelId={}, channel={}, recreate the channelId={}\",\n                            channel.id(), channel, channelWrapper.getChannel().id());\n                    }\n                    if (channelWrapper != null && !channelWrapper.isWrapperOf(channel)) {\n                        RemotingCommand retryRequest = RemotingCommand.createRequestCommand(request.getCode(), request.readCustomHeader());\n                        retryRequest.setBody(request.getBody());\n                        retryRequest.setExtFields(request.getExtFields());\n                        CompletableFuture<Void> future = convertChannelFutureToCompletableFuture(channelWrapper.getChannelFuture());\n                        return future.thenCompose(v -> {\n                            long duration = stopwatch.elapsed(TimeUnit.MILLISECONDS);\n                            stopwatch.stop();\n                            return super.invokeImpl(channelWrapper.getChannel(), retryRequest, timeoutMillis - duration)\n                                .thenCompose(r -> {\n                                    if (r.getResponseCommand().getCode() == ResponseCode.GO_AWAY) {\n                                        return FutureUtils.completeExceptionally(new RemotingSendRequestException(channelRemoteAddr,\n                                            new Throwable(\"Receive GO_AWAY twice in request from channelId=\" + channel.id())));\n                                    }\n                                    return CompletableFuture.completedFuture(r);\n                                });\n                        });\n                    } else {\n                        LOGGER.warn(\"invokeImpl receive GO_AWAY, channelWrapper is null or channel is the same in wrapper, channelId={}\", channel.id());\n                    }\n                }\n                return FutureUtils.completeExceptionally(new RemotingSendRequestException(channelRemoteAddr, new Throwable(\"Receive GO_AWAY from channelId=\" + channel.id())));\n            }\n            return CompletableFuture.completedFuture(responseFuture);\n        }).whenComplete((v, t) -> {\n            if (t == null) {\n                doAfterRpcHooks(channelRemoteAddr, request, v.getResponseCommand());\n            }\n        });\n    }\n\n    @Override\n    public void registerProcessor(int requestCode, NettyRequestProcessor processor, ExecutorService executor) {\n        ExecutorService executorThis = executor;\n        if (null == executor) {\n            executorThis = this.publicExecutor;\n        }\n\n        Pair<NettyRequestProcessor, ExecutorService> pair = new Pair<>(processor, executorThis);\n        this.processorTable.put(requestCode, pair);\n    }\n\n    @Override\n    public boolean isChannelWritable(String addr) {\n        ChannelWrapper cw = this.channelTables.get(addr);\n        if (cw != null && cw.isOK()) {\n            return cw.isWritable();\n        }\n        return true;\n    }\n\n    @Override\n    public boolean isAddressReachable(String addr) {\n        if (addr == null || addr.isEmpty()) {\n            return false;\n        }\n        try {\n            Channel channel = getAndCreateChannel(addr);\n            return channel != null && channel.isActive();\n        } catch (Exception e) {\n            LOGGER.warn(\"Get and create channel of {} failed\", addr, e);\n            return false;\n        }\n    }\n\n    @Override\n    public List<String> getNameServerAddressList() {\n        return this.namesrvAddrList.get();\n    }\n\n    @Override\n    public List<String> getAvailableNameSrvList() {\n        return new ArrayList<>(this.availableNamesrvAddrMap.keySet());\n    }\n\n    @Override\n    public ChannelEventListener getChannelEventListener() {\n        return channelEventListener;\n    }\n\n    @Override\n    public ExecutorService getCallbackExecutor() {\n        if (nettyClientConfig.isDisableCallbackExecutor()) {\n            return null;\n        }\n        return callbackExecutor != null ? callbackExecutor : publicExecutor;\n    }\n\n    @Override\n    public void setCallbackExecutor(final ExecutorService callbackExecutor) {\n        this.callbackExecutor = callbackExecutor;\n    }\n\n    protected void scanChannelTablesOfNameServer() {\n        List<String> nameServerList = this.namesrvAddrList.get();\n        if (nameServerList == null) {\n            LOGGER.warn(\"[SCAN] Addresses of name server is empty!\");\n            return;\n        }\n\n        for (Map.Entry<String, ChannelWrapper> entry : this.channelTables.entrySet()) {\n            String addr = entry.getKey();\n            ChannelWrapper channelWrapper = entry.getValue();\n            if (channelWrapper == null) {\n                continue;\n            }\n\n            if ((System.currentTimeMillis() - channelWrapper.getLastResponseTime()) > this.nettyClientConfig.getChannelNotActiveInterval()) {\n                LOGGER.warn(\"[SCAN] No response after {} from name server {}, so close it!\", channelWrapper.getLastResponseTime(),\n                    addr);\n                closeChannel(addr, channelWrapper.getChannel());\n            }\n        }\n    }\n\n    private void scanAvailableNameSrv() {\n        List<String> nameServerList = this.namesrvAddrList.get();\n        if (nameServerList == null) {\n            LOGGER.debug(\"scanAvailableNameSrv addresses of name server is null!\");\n            return;\n        }\n\n        for (String address : NettyRemotingClient.this.availableNamesrvAddrMap.keySet()) {\n            if (!nameServerList.contains(address)) {\n                LOGGER.warn(\"scanAvailableNameSrv remove invalid address {}\", address);\n                NettyRemotingClient.this.availableNamesrvAddrMap.remove(address);\n            }\n        }\n\n        for (final String namesrvAddr : nameServerList) {\n            scanExecutor.execute(new Runnable() {\n                @Override\n                public void run() {\n                    try {\n                        Channel channel = NettyRemotingClient.this.getAndCreateChannel(namesrvAddr);\n                        if (channel != null) {\n                            NettyRemotingClient.this.availableNamesrvAddrMap.putIfAbsent(namesrvAddr, true);\n                        } else {\n                            Boolean value = NettyRemotingClient.this.availableNamesrvAddrMap.remove(namesrvAddr);\n                            if (value != null) {\n                                LOGGER.warn(\"scanAvailableNameSrv remove unconnected address {}\", namesrvAddr);\n                            }\n                        }\n                    } catch (Exception e) {\n                        LOGGER.error(\"scanAvailableNameSrv get channel of {} failed, \", namesrvAddr, e);\n                    }\n                }\n            });\n        }\n    }\n\n    class ChannelWrapper {\n        private final ReentrantReadWriteLock lock;\n        private ChannelFuture channelFuture;\n        // only affected by sync or async request, oneway is not included.\n        private ChannelFuture channelToClose;\n        private long lastResponseTime;\n        private final String channelAddress;\n\n        public ChannelWrapper(String address, ChannelFuture channelFuture) {\n            this.lock = new ReentrantReadWriteLock();\n            this.channelFuture = channelFuture;\n            this.lastResponseTime = System.currentTimeMillis();\n            this.channelAddress = address;\n            RemotingHelper.setPropertyToAttr(channelFuture.channel(), CHANNEL_WRAPPER_ATTRIBUTE_KEY, this);\n        }\n\n        public boolean isOK() {\n            return getChannel() != null && getChannel().isActive();\n        }\n\n        public boolean isWritable() {\n            return getChannel().isWritable();\n        }\n\n        public boolean isWrapperOf(Channel channel) {\n            return this.channelFuture.channel() != null && this.channelFuture.channel() == channel;\n        }\n\n        private Channel getChannel() {\n            return getChannelFuture().channel();\n        }\n\n        public ChannelFuture getChannelFuture() {\n            lock.readLock().lock();\n            try {\n                return this.channelFuture;\n            } finally {\n                lock.readLock().unlock();\n            }\n        }\n\n        public long getLastResponseTime() {\n            return this.lastResponseTime;\n        }\n\n        public void updateLastResponseTime() {\n            this.lastResponseTime = System.currentTimeMillis();\n        }\n\n        public String getChannelAddress() {\n            return channelAddress;\n        }\n\n        public boolean reconnect(Channel channel) {\n            if (!isWrapperOf(channel)) {\n                LOGGER.warn(\"channelWrapper has reconnect, so do nothing, now channelId={}, input channelId={}\",getChannel().id(), channel.id());\n                return false;\n            }\n            if (lock.writeLock().tryLock()) {\n                try {\n                    if (isWrapperOf(channel)) {\n                        channelToClose = channelFuture;\n                        channelFuture = doConnect(channelAddress);\n                        RemotingHelper.setPropertyToAttr(channelFuture.channel(), CHANNEL_WRAPPER_ATTRIBUTE_KEY, this);\n                        return true;\n                    } else {\n                        LOGGER.warn(\"channelWrapper has reconnect, so do nothing, now channelId={}, input channelId={}\",getChannel().id(), channel.id());\n                    }\n                } catch (Throwable t) {\n                    LOGGER.error(\"ChannelWrapper {} reconnect error\", this, t);\n                } finally {\n                    lock.writeLock().unlock();\n                }\n            } else {\n                LOGGER.warn(\"channelWrapper reconnect try lock fail, now channelId={}\", getChannel().id());\n            }\n            return false;\n        }\n\n        public boolean tryClose(Channel channel) {\n            try {\n                lock.readLock().lock();\n                if (channelFuture != null) {\n                    if (channelFuture.channel().equals(channel)) {\n                        return true;\n                    }\n                }\n            } finally {\n                lock.readLock().unlock();\n            }\n            return false;\n        }\n\n        public void close() {\n            try {\n                lock.writeLock().lock();\n                if (channelFuture != null) {\n                    closeChannel(channelFuture.channel());\n                }\n                if (channelToClose != null) {\n                    closeChannel(channelToClose.channel());\n                }\n            } finally {\n                lock.writeLock().unlock();\n            }\n        }\n    }\n\n    class InvokeCallbackWrapper implements InvokeCallback {\n\n        private final InvokeCallback invokeCallback;\n        private final String addr;\n\n        public InvokeCallbackWrapper(InvokeCallback invokeCallback, String addr) {\n            this.invokeCallback = invokeCallback;\n            this.addr = addr;\n        }\n\n        @Override\n        public void operationComplete(ResponseFuture responseFuture) {\n            this.invokeCallback.operationComplete(responseFuture);\n        }\n\n        @Override\n        public void operationSucceed(RemotingCommand response) {\n            updateChannelLastResponseTime(addr);\n            this.invokeCallback.operationSucceed(response);\n        }\n\n        @Override\n        public void operationFail(final Throwable throwable) {\n            this.invokeCallback.operationFail(throwable);\n        }\n    }\n\n    public class NettyClientHandler extends SimpleChannelInboundHandler<RemotingCommand> {\n        @Override\n        protected void channelRead0(ChannelHandlerContext ctx, RemotingCommand msg) throws Exception {\n            processMessageReceived(ctx, msg);\n        }\n    }\n\n    public class NettyConnectManageHandler extends ChannelDuplexHandler {\n        @Override\n        public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, SocketAddress localAddress,\n            ChannelPromise promise) throws Exception {\n            final String local = localAddress == null ? NetworkUtil.getLocalAddress() : RemotingHelper.parseSocketAddressAddr(localAddress);\n            final String remote = remoteAddress == null ? \"UNKNOWN\" : RemotingHelper.parseSocketAddressAddr(remoteAddress);\n            LOGGER.info(\"NETTY CLIENT PIPELINE: CONNECT  {} => {}\", local, remote);\n\n            super.connect(ctx, remoteAddress, localAddress, promise);\n\n            if (NettyRemotingClient.this.channelEventListener != null) {\n                NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.CONNECT, remote, ctx.channel()));\n            }\n        }\n\n        @Override\n        public void channelActive(ChannelHandlerContext ctx) throws Exception {\n            final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());\n            LOGGER.info(\"NETTY CLIENT PIPELINE: ACTIVE, {}, channelId={}\", remoteAddress, ctx.channel().id());\n            super.channelActive(ctx);\n\n            if (NettyRemotingClient.this.channelEventListener != null) {\n                NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.ACTIVE, remoteAddress, ctx.channel()));\n            }\n        }\n\n        @Override\n        public void disconnect(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception {\n            final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());\n            LOGGER.info(\"NETTY CLIENT PIPELINE: DISCONNECT {}\", remoteAddress);\n            closeChannel(ctx.channel());\n            super.disconnect(ctx, promise);\n\n            if (NettyRemotingClient.this.channelEventListener != null) {\n                NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.CLOSE, remoteAddress, ctx.channel()));\n            }\n        }\n\n        @Override\n        public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception {\n            final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());\n            LOGGER.info(\"NETTY CLIENT PIPELINE: CLOSE channel[addr={}, id={}]\", remoteAddress, ctx.channel().id());\n            closeChannel(ctx.channel());\n            super.close(ctx, promise);\n            NettyRemotingClient.this.failFast(ctx.channel());\n            if (NettyRemotingClient.this.channelEventListener != null) {\n                NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.CLOSE, remoteAddress, ctx.channel()));\n            }\n        }\n\n        @Override\n        public void channelInactive(ChannelHandlerContext ctx) throws Exception {\n            final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());\n            LOGGER.info(\"NETTY CLIENT PIPELINE: channelInactive, the channel[addr={}, id={}]\", remoteAddress, ctx.channel().id());\n            closeChannel(ctx.channel());\n            super.channelInactive(ctx);\n        }\n\n        @Override\n        public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {\n            if (evt instanceof IdleStateEvent) {\n                IdleStateEvent event = (IdleStateEvent) evt;\n                if (event.state().equals(IdleState.ALL_IDLE)) {\n                    final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());\n                    LOGGER.warn(\"NETTY CLIENT PIPELINE: IDLE exception channel[addr={}, id={}]\", remoteAddress, ctx.channel().id());\n                    closeChannel(ctx.channel());\n                    if (NettyRemotingClient.this.channelEventListener != null) {\n                        NettyRemotingClient.this\n                            .putNettyEvent(new NettyEvent(NettyEventType.IDLE, remoteAddress, ctx.channel()));\n                    }\n                }\n            }\n\n            ctx.fireUserEventTriggered(evt);\n        }\n\n        @Override\n        public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {\n            final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());\n            LOGGER.warn(\"NETTY CLIENT PIPELINE: exceptionCaught channel[addr={}, id={}]\", remoteAddress, ctx.channel().id(), cause);\n            closeChannel(ctx.channel());\n            if (NettyRemotingClient.this.channelEventListener != null) {\n                NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.EXCEPTION, remoteAddress, ctx.channel()));\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.netty;\n\nimport io.netty.bootstrap.ServerBootstrap;\nimport io.netty.buffer.ByteBuf;\nimport io.netty.buffer.ByteBufUtil;\nimport io.netty.buffer.PooledByteBufAllocator;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelDuplexHandler;\nimport io.netty.channel.ChannelFuture;\nimport io.netty.channel.ChannelHandler;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.channel.ChannelInboundHandlerAdapter;\nimport io.netty.channel.ChannelInitializer;\nimport io.netty.channel.ChannelOption;\nimport io.netty.channel.ChannelPipeline;\nimport io.netty.channel.EventLoopGroup;\nimport io.netty.channel.SimpleChannelInboundHandler;\nimport io.netty.channel.WriteBufferWaterMark;\nimport io.netty.channel.epoll.Epoll;\nimport io.netty.channel.epoll.EpollEventLoopGroup;\nimport io.netty.channel.epoll.EpollServerSocketChannel;\nimport io.netty.channel.nio.NioEventLoopGroup;\nimport io.netty.channel.socket.SocketChannel;\nimport io.netty.channel.socket.nio.NioServerSocketChannel;\nimport io.netty.handler.codec.ByteToMessageDecoder;\nimport io.netty.handler.codec.ProtocolDetectionResult;\nimport io.netty.handler.codec.ProtocolDetectionState;\nimport io.netty.handler.codec.haproxy.HAProxyMessage;\nimport io.netty.handler.codec.haproxy.HAProxyMessageDecoder;\nimport io.netty.handler.codec.haproxy.HAProxyProtocolVersion;\nimport io.netty.handler.codec.haproxy.HAProxyTLV;\nimport io.netty.handler.timeout.IdleState;\nimport io.netty.handler.timeout.IdleStateEvent;\nimport io.netty.handler.timeout.IdleStateHandler;\nimport io.netty.util.AttributeKey;\nimport io.netty.util.CharsetUtil;\nimport io.netty.util.HashedWheelTimer;\nimport io.netty.util.Timeout;\nimport io.netty.util.TimerTask;\nimport io.netty.util.concurrent.DefaultEventExecutorGroup;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.constant.HAProxyConstants;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.BinaryUtil;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.ChannelEventListener;\nimport org.apache.rocketmq.remoting.InvokeCallback;\nimport org.apache.rocketmq.remoting.RemotingServer;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.common.TlsMode;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\nimport java.io.IOException;\nimport java.net.InetSocketAddress;\nimport java.security.cert.CertificateException;\nimport java.time.Duration;\nimport java.util.List;\nimport java.util.NoSuchElementException;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\n\npublic class NettyRemotingServer extends NettyRemotingAbstract implements RemotingServer {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME);\n    private static final Logger TRAFFIC_LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_TRAFFIC_NAME);\n\n    private final ServerBootstrap serverBootstrap;\n    protected final EventLoopGroup eventLoopGroupSelector;\n    protected final EventLoopGroup eventLoopGroupBoss;\n    protected final NettyServerConfig nettyServerConfig;\n\n    private final ExecutorService publicExecutor;\n    private final ScheduledExecutorService scheduledExecutorService;\n    private final ChannelEventListener channelEventListener;\n\n    private final HashedWheelTimer timer = new HashedWheelTimer(r -> new Thread(r, \"ServerHouseKeepingService\"));\n\n    private DefaultEventExecutorGroup defaultEventExecutorGroup;\n\n    /**\n     * NettyRemotingServer may hold multiple SubRemotingServer, each server will be stored in this container with a\n     * ListenPort key.\n     */\n    private final ConcurrentMap<Integer/*Port*/, NettyRemotingAbstract> remotingServerTable = new ConcurrentHashMap<>();\n\n    public static final String HANDSHAKE_HANDLER_NAME = \"handshakeHandler\";\n    public static final String HA_PROXY_DECODER = \"HAProxyDecoder\";\n    public static final String HA_PROXY_HANDLER = \"HAProxyHandler\";\n    public static final String TLS_MODE_HANDLER = \"TlsModeHandler\";\n    public static final String TLS_HANDLER_NAME = \"sslHandler\";\n    public static final String FILE_REGION_ENCODER_NAME = \"fileRegionEncoder\";\n\n    // sharable handlers\n    protected final TlsModeHandler tlsModeHandler = new TlsModeHandler(TlsSystemConfig.tlsMode);\n    protected final NettyEncoder encoder = new NettyEncoder();\n    protected final NettyConnectManageHandler connectionManageHandler = new NettyConnectManageHandler();\n    protected final NettyServerHandler serverHandler = new NettyServerHandler();\n    protected final RemotingCodeDistributionHandler distributionHandler = new RemotingCodeDistributionHandler();\n\n    public NettyRemotingServer(final NettyServerConfig nettyServerConfig) {\n        this(nettyServerConfig, null);\n    }\n\n    public NettyRemotingServer(final NettyServerConfig nettyServerConfig,\n                               final ChannelEventListener channelEventListener) {\n        super(nettyServerConfig.getServerOnewaySemaphoreValue(), nettyServerConfig.getServerAsyncSemaphoreValue());\n        this.serverBootstrap = new ServerBootstrap();\n        this.nettyServerConfig = nettyServerConfig;\n        this.channelEventListener = channelEventListener;\n\n        this.publicExecutor = buildPublicExecutor(nettyServerConfig);\n        this.scheduledExecutorService = buildScheduleExecutor();\n\n        this.eventLoopGroupBoss = buildEventLoopGroupBoss();\n        this.eventLoopGroupSelector = buildEventLoopGroupSelector();\n\n        loadSslContext();\n    }\n\n    protected EventLoopGroup buildEventLoopGroupSelector() {\n        if (useEpoll()) {\n            return new EpollEventLoopGroup(nettyServerConfig.getServerSelectorThreads(), new ThreadFactoryImpl(\"NettyServerEPOLLSelector_\"));\n        } else {\n            return new NioEventLoopGroup(nettyServerConfig.getServerSelectorThreads(), new ThreadFactoryImpl(\"NettyServerNIOSelector_\"));\n        }\n    }\n\n    protected EventLoopGroup buildEventLoopGroupBoss() {\n        if (useEpoll()) {\n            return new EpollEventLoopGroup(1, new ThreadFactoryImpl(\"NettyEPOLLBoss_\"));\n        } else {\n            return new NioEventLoopGroup(1, new ThreadFactoryImpl(\"NettyNIOBoss_\"));\n        }\n    }\n\n    private ExecutorService buildPublicExecutor(NettyServerConfig nettyServerConfig) {\n        int publicThreadNums = nettyServerConfig.getServerCallbackExecutorThreads();\n        if (publicThreadNums <= 0) {\n            publicThreadNums = 4;\n        }\n\n        return Executors.newFixedThreadPool(publicThreadNums, new ThreadFactoryImpl(\"NettyServerPublicExecutor_\"));\n    }\n\n    private ScheduledExecutorService buildScheduleExecutor() {\n        return ThreadUtils.newScheduledThreadPool(1,\n            new ThreadFactoryImpl(\"NettyServerScheduler_\", true),\n            new ThreadPoolExecutor.DiscardOldestPolicy());\n    }\n\n    public void loadSslContext() {\n        TlsMode tlsMode = TlsSystemConfig.tlsMode;\n        log.info(\"Server is running in TLS {} mode\", tlsMode.getName());\n\n        if (tlsMode != TlsMode.DISABLED) {\n            try {\n                sslContext = TlsHelper.buildSslContext(false);\n                log.info(\"SslContext created for server\");\n            } catch (CertificateException | IOException e) {\n                log.error(\"Failed to create SslContext for server\", e);\n            }\n        }\n    }\n\n    private boolean useEpoll() {\n        return NetworkUtil.isLinuxPlatform()\n            && nettyServerConfig.isUseEpollNativeSelector()\n            && Epoll.isAvailable();\n    }\n\n    protected void initServerBootstrap(ServerBootstrap serverBootstrap) {\n        serverBootstrap.group(this.eventLoopGroupBoss, this.eventLoopGroupSelector)\n            .channel(useEpoll() ? EpollServerSocketChannel.class : NioServerSocketChannel.class)\n            .option(ChannelOption.SO_BACKLOG, 1024)\n            .option(ChannelOption.SO_REUSEADDR, true)\n            .childOption(ChannelOption.SO_KEEPALIVE, false)\n            .childOption(ChannelOption.TCP_NODELAY, true)\n            .localAddress(new InetSocketAddress(this.nettyServerConfig.getBindAddress(),\n                this.nettyServerConfig.getListenPort()))\n            .childHandler(new ChannelInitializer<SocketChannel>() {\n                @Override\n                public void initChannel(SocketChannel ch) {\n                    configChannel(ch);\n                }\n            });\n\n        addCustomConfig(serverBootstrap);\n    }\n\n    @Override\n    public void start() {\n        this.defaultEventExecutorGroup = new DefaultEventExecutorGroup(nettyServerConfig.getServerWorkerThreads(),\n            new ThreadFactoryImpl(\"NettyServerCodecThread_\"));\n\n        initServerBootstrap(serverBootstrap);\n\n        try {\n            ChannelFuture sync = serverBootstrap.bind().sync();\n            InetSocketAddress addr = (InetSocketAddress) sync.channel().localAddress();\n            if (0 == nettyServerConfig.getListenPort()) {\n                this.nettyServerConfig.setListenPort(addr.getPort());\n            }\n            log.info(\"RemotingServer started, listening {}:{}\", this.nettyServerConfig.getBindAddress(),\n                this.nettyServerConfig.getListenPort());\n            this.remotingServerTable.put(this.nettyServerConfig.getListenPort(), this);\n        } catch (Exception e) {\n            throw new IllegalStateException(String.format(\"Failed to bind to %s:%d\", nettyServerConfig.getBindAddress(),\n                nettyServerConfig.getListenPort()), e);\n        }\n\n        if (this.channelEventListener != null) {\n            this.nettyEventExecutor.start();\n        }\n\n        TimerTask timerScanResponseTable = new TimerTask() {\n            @Override\n            public void run(Timeout timeout) {\n                try {\n                    NettyRemotingServer.this.scanResponseTable();\n                } catch (Throwable e) {\n                    log.error(\"scanResponseTable exception\", e);\n                } finally {\n                    timer.newTimeout(this, 1000, TimeUnit.MILLISECONDS);\n                }\n            }\n        };\n        this.timer.newTimeout(timerScanResponseTable, 1000 * 3, TimeUnit.MILLISECONDS);\n\n        scheduledExecutorService.scheduleWithFixedDelay(() -> {\n            try {\n                NettyRemotingServer.this.printRemotingCodeDistribution();\n            } catch (Throwable e) {\n                TRAFFIC_LOGGER.error(\"NettyRemotingServer print remoting code distribution exception\", e);\n            }\n        }, 1, 1, TimeUnit.SECONDS);\n    }\n\n    /**\n     * config channel in ChannelInitializer\n     *\n     * @param ch the SocketChannel needed to init\n     * @return the initialized ChannelPipeline, sub class can use it to extent in the future\n     */\n    protected ChannelPipeline configChannel(SocketChannel ch) {\n        return ch.pipeline()\n            .addLast(getDefaultEventExecutorGroup(),\n                HANDSHAKE_HANDLER_NAME, new HandshakeHandler())\n            .addLast(getDefaultEventExecutorGroup(),\n                encoder,\n                new NettyDecoder(),\n                distributionHandler,\n                new IdleStateHandler(0, 0,\n                    nettyServerConfig.getServerChannelMaxIdleTimeSeconds()),\n                connectionManageHandler,\n                serverHandler\n            );\n    }\n\n    private void addCustomConfig(ServerBootstrap childHandler) {\n        if (nettyServerConfig.getServerSocketSndBufSize() > 0) {\n            log.info(\"server set SO_SNDBUF to {}\", nettyServerConfig.getServerSocketSndBufSize());\n            childHandler.childOption(ChannelOption.SO_SNDBUF, nettyServerConfig.getServerSocketSndBufSize());\n        }\n        if (nettyServerConfig.getServerSocketRcvBufSize() > 0) {\n            log.info(\"server set SO_RCVBUF to {}\", nettyServerConfig.getServerSocketRcvBufSize());\n            childHandler.childOption(ChannelOption.SO_RCVBUF, nettyServerConfig.getServerSocketRcvBufSize());\n        }\n        if (nettyServerConfig.getWriteBufferLowWaterMark() > 0 && nettyServerConfig.getWriteBufferHighWaterMark() > 0) {\n            log.info(\"server set netty WRITE_BUFFER_WATER_MARK to {},{}\",\n                nettyServerConfig.getWriteBufferLowWaterMark(), nettyServerConfig.getWriteBufferHighWaterMark());\n            childHandler.childOption(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark(\n                nettyServerConfig.getWriteBufferLowWaterMark(), nettyServerConfig.getWriteBufferHighWaterMark()));\n        }\n\n        if (nettyServerConfig.isServerPooledByteBufAllocatorEnable()) {\n            childHandler.childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);\n        }\n    }\n\n    @Override\n    public void shutdown() {\n        try {\n            if (nettyServerConfig.isEnableShutdownGracefully() && isShuttingDown.compareAndSet(false, true)) {\n                Thread.sleep(Duration.ofSeconds(nettyServerConfig.getShutdownWaitTimeSeconds()).toMillis());\n            }\n\n            this.timer.stop();\n\n            this.eventLoopGroupBoss.shutdownGracefully();\n\n            this.eventLoopGroupSelector.shutdownGracefully();\n\n            this.nettyEventExecutor.shutdown();\n\n            if (this.defaultEventExecutorGroup != null) {\n                this.defaultEventExecutorGroup.shutdownGracefully();\n            }\n        } catch (Exception e) {\n            log.error(\"NettyRemotingServer shutdown exception, \", e);\n        }\n\n        if (this.publicExecutor != null) {\n            try {\n                this.publicExecutor.shutdown();\n            } catch (Exception e) {\n                log.error(\"NettyRemotingServer shutdown exception, \", e);\n            }\n        }\n    }\n\n    @Override\n    public void registerProcessor(int requestCode, NettyRequestProcessor processor, ExecutorService executor) {\n        ExecutorService executorThis = executor;\n        if (null == executor) {\n            executorThis = this.publicExecutor;\n        }\n\n        Pair<NettyRequestProcessor, ExecutorService> pair = new Pair<>(processor, executorThis);\n        this.processorTable.put(requestCode, pair);\n    }\n\n    @Override\n    public void registerDefaultProcessor(NettyRequestProcessor processor, ExecutorService executor) {\n        this.defaultRequestProcessorPair = new Pair<>(processor, executor);\n    }\n\n    @Override\n    public int localListenPort() {\n        return this.nettyServerConfig.getListenPort();\n    }\n\n    @Override\n    public Pair<NettyRequestProcessor, ExecutorService> getProcessorPair(int requestCode) {\n        return processorTable.get(requestCode);\n    }\n\n    @Override\n    public Pair<NettyRequestProcessor, ExecutorService> getDefaultProcessorPair() {\n        return defaultRequestProcessorPair;\n    }\n\n    @Override\n    public RemotingServer newRemotingServer(final int port) {\n        SubRemotingServer remotingServer = new SubRemotingServer(port,\n            this.nettyServerConfig.getServerOnewaySemaphoreValue(),\n            this.nettyServerConfig.getServerAsyncSemaphoreValue());\n        NettyRemotingAbstract existingServer = this.remotingServerTable.putIfAbsent(port, remotingServer);\n        if (existingServer != null) {\n            throw new RuntimeException(\"The port \" + port + \" already in use by another RemotingServer\");\n        }\n        return remotingServer;\n    }\n\n    @Override\n    public void removeRemotingServer(final int port) {\n        this.remotingServerTable.remove(port);\n    }\n\n    @Override\n    public RemotingCommand invokeSync(final Channel channel, final RemotingCommand request, final long timeoutMillis)\n        throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException {\n        return this.invokeSyncImpl(channel, request, timeoutMillis);\n    }\n\n    @Override\n    public void invokeAsync(Channel channel, RemotingCommand request, long timeoutMillis, InvokeCallback invokeCallback)\n        throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException {\n        this.invokeAsyncImpl(channel, request, timeoutMillis, invokeCallback);\n    }\n\n    @Override\n    public void invokeOneway(Channel channel, RemotingCommand request, long timeoutMillis) throws InterruptedException,\n        RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException {\n        this.invokeOnewayImpl(channel, request, timeoutMillis);\n    }\n\n    @Override\n    public ChannelEventListener getChannelEventListener() {\n        return channelEventListener;\n    }\n\n    @Override\n    public ExecutorService getCallbackExecutor() {\n        return this.publicExecutor;\n    }\n\n    private void printRemotingCodeDistribution() {\n        if (distributionHandler != null) {\n            String inBoundSnapshotString = distributionHandler.getInBoundSnapshotString();\n            if (inBoundSnapshotString != null) {\n                TRAFFIC_LOGGER.info(\"Port: {}, RequestCode Distribution: {}\",\n                    nettyServerConfig.getListenPort(), inBoundSnapshotString);\n            }\n\n            String outBoundSnapshotString = distributionHandler.getOutBoundSnapshotString();\n            if (outBoundSnapshotString != null) {\n                TRAFFIC_LOGGER.info(\"Port: {}, ResponseCode Distribution: {}\",\n                    nettyServerConfig.getListenPort(), outBoundSnapshotString);\n            }\n        }\n    }\n\n    public DefaultEventExecutorGroup getDefaultEventExecutorGroup() {\n        return nettyServerConfig.isServerNettyWorkerGroupEnable() ? defaultEventExecutorGroup : null;\n    }\n\n    public NettyEncoder getEncoder() {\n        return encoder;\n    }\n\n    public NettyConnectManageHandler getConnectionManageHandler() {\n        return connectionManageHandler;\n    }\n\n    public NettyServerHandler getServerHandler() {\n        return serverHandler;\n    }\n\n    public RemotingCodeDistributionHandler getDistributionHandler() {\n        return distributionHandler;\n    }\n\n    public class HandshakeHandler extends ByteToMessageDecoder {\n\n        public HandshakeHandler() {\n        }\n\n        @Override\n        protected void decode(ChannelHandlerContext ctx, ByteBuf byteBuf, List<Object> out) throws Exception {\n            try {\n                ProtocolDetectionResult<HAProxyProtocolVersion> detectionResult = HAProxyMessageDecoder.detectProtocol(byteBuf);\n                if (detectionResult.state() == ProtocolDetectionState.NEEDS_MORE_DATA) {\n                    return;\n                }\n                if (detectionResult.state() == ProtocolDetectionState.DETECTED) {\n                    ctx.pipeline().addAfter(getDefaultEventExecutorGroup(), ctx.name(), HA_PROXY_DECODER, new HAProxyMessageDecoder())\n                        .addAfter(getDefaultEventExecutorGroup(), HA_PROXY_DECODER, HA_PROXY_HANDLER, new HAProxyMessageHandler())\n                        .addAfter(getDefaultEventExecutorGroup(), HA_PROXY_HANDLER, TLS_MODE_HANDLER, tlsModeHandler);\n                } else {\n                    ctx.pipeline().addAfter(getDefaultEventExecutorGroup(), ctx.name(), TLS_MODE_HANDLER, tlsModeHandler);\n                }\n\n                try {\n                    // Remove this handler\n                    ctx.pipeline().remove(this);\n                } catch (NoSuchElementException e) {\n                    log.error(\"Error while removing HandshakeHandler\", e);\n                }\n            } catch (Exception e) {\n                log.error(\"process proxy protocol negotiator failed.\", e);\n                throw e;\n            }\n        }\n    }\n\n    @ChannelHandler.Sharable\n    public class TlsModeHandler extends SimpleChannelInboundHandler<ByteBuf> {\n\n        private final TlsMode tlsMode;\n\n        private static final byte HANDSHAKE_MAGIC_CODE = 0x16;\n\n        TlsModeHandler(TlsMode tlsMode) {\n            this.tlsMode = tlsMode;\n        }\n\n        @Override\n        protected void channelRead0(ChannelHandlerContext ctx, ByteBuf msg) {\n\n            // Peek the current read index byte to determine if the content is starting with TLS handshake\n            byte b = msg.getByte(msg.readerIndex());\n\n            if (b == HANDSHAKE_MAGIC_CODE) {\n                switch (tlsMode) {\n                    case DISABLED:\n                        ctx.close();\n                        log.warn(\"Clients intend to establish an SSL connection while this server is running in SSL disabled mode\");\n                        throw new UnsupportedOperationException(\"The NettyRemotingServer in SSL disabled mode doesn't support ssl client\");\n                    case PERMISSIVE:\n                    case ENFORCING:\n                        if (null != sslContext) {\n                            ctx.pipeline()\n                                .addAfter(getDefaultEventExecutorGroup(), TLS_MODE_HANDLER, TLS_HANDLER_NAME, sslContext.newHandler(ctx.channel().alloc()))\n                                .addAfter(getDefaultEventExecutorGroup(), TLS_HANDLER_NAME, FILE_REGION_ENCODER_NAME, new FileRegionEncoder());\n                            log.info(\"Handlers prepended to channel pipeline to establish SSL connection\");\n                        } else {\n                            ctx.close();\n                            log.error(\"Trying to establish an SSL connection but SslContext is null\");\n                        }\n                        break;\n\n                    default:\n                        log.warn(\"Unknown TLS mode\");\n                        break;\n                }\n            } else if (tlsMode == TlsMode.ENFORCING) {\n                ctx.close();\n                log.warn(\"Clients intend to establish an insecure connection while this server is running in SSL enforcing mode\");\n                throw new UnsupportedOperationException(\"The NettyRemotingServer in SSL enforcing mode doesn't support plain client\");\n            }\n\n            try {\n                // Remove this handler\n                ctx.pipeline().remove(this);\n            } catch (NoSuchElementException e) {\n                log.error(\"Error while removing TlsModeHandler\", e);\n            }\n\n            // Hand over this message to the next .\n            ctx.fireChannelRead(msg.retain());\n        }\n    }\n\n    @ChannelHandler.Sharable\n    public class NettyServerHandler extends SimpleChannelInboundHandler<RemotingCommand> {\n\n        @Override\n        protected void channelRead0(ChannelHandlerContext ctx, RemotingCommand msg) {\n            int localPort = RemotingHelper.parseSocketAddressPort(ctx.channel().localAddress());\n            NettyRemotingAbstract remotingAbstract = NettyRemotingServer.this.remotingServerTable.get(localPort);\n            if (localPort != -1 && remotingAbstract != null) {\n                remotingAbstract.processMessageReceived(ctx, msg);\n                return;\n            }\n            // The related remoting server has been shutdown, so close the connected channel\n            RemotingHelper.closeChannel(ctx.channel());\n        }\n\n        @Override\n        public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exception {\n            Channel channel = ctx.channel();\n            if (channel.isWritable()) {\n                if (!channel.config().isAutoRead()) {\n                    channel.config().setAutoRead(true);\n                    log.info(\"Channel[{}] turns writable, bytes to buffer before changing channel to un-writable: {}\",\n                        RemotingHelper.parseChannelRemoteAddr(channel), channel.bytesBeforeUnwritable());\n                }\n            } else {\n                channel.config().setAutoRead(false);\n                log.warn(\"Channel[{}] auto-read is disabled, bytes to drain before it turns writable: {}\",\n                    RemotingHelper.parseChannelRemoteAddr(channel), channel.bytesBeforeWritable());\n            }\n            super.channelWritabilityChanged(ctx);\n        }\n    }\n\n    @ChannelHandler.Sharable\n    public class NettyConnectManageHandler extends ChannelDuplexHandler {\n        @Override\n        public void channelRegistered(ChannelHandlerContext ctx) throws Exception {\n            final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());\n            log.info(\"NETTY SERVER PIPELINE: channelRegistered {}\", remoteAddress);\n            super.channelRegistered(ctx);\n        }\n\n        @Override\n        public void channelUnregistered(ChannelHandlerContext ctx) throws Exception {\n            final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());\n            log.info(\"NETTY SERVER PIPELINE: channelUnregistered, the channel[{}]\", remoteAddress);\n            super.channelUnregistered(ctx);\n        }\n\n        @Override\n        public void channelActive(ChannelHandlerContext ctx) throws Exception {\n            final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());\n            log.info(\"NETTY SERVER PIPELINE: channelActive, the channel[{}]\", remoteAddress);\n            super.channelActive(ctx);\n\n            if (NettyRemotingServer.this.channelEventListener != null) {\n                NettyRemotingServer.this.putNettyEvent(new NettyEvent(NettyEventType.CONNECT, remoteAddress, ctx.channel()));\n            }\n        }\n\n        @Override\n        public void channelInactive(ChannelHandlerContext ctx) throws Exception {\n            final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());\n            log.info(\"NETTY SERVER PIPELINE: channelInactive, the channel[{}]\", remoteAddress);\n            super.channelInactive(ctx);\n\n            if (NettyRemotingServer.this.channelEventListener != null) {\n                NettyRemotingServer.this.putNettyEvent(new NettyEvent(NettyEventType.CLOSE, remoteAddress, ctx.channel()));\n            }\n        }\n\n        @Override\n        public void userEventTriggered(ChannelHandlerContext ctx, Object evt) {\n            if (evt instanceof IdleStateEvent) {\n                IdleStateEvent event = (IdleStateEvent) evt;\n                if (event.state().equals(IdleState.ALL_IDLE)) {\n                    final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());\n                    log.warn(\"NETTY SERVER PIPELINE: IDLE exception [{}]\", remoteAddress);\n                    RemotingHelper.closeChannel(ctx.channel());\n                    if (NettyRemotingServer.this.channelEventListener != null) {\n                        NettyRemotingServer.this\n                            .putNettyEvent(new NettyEvent(NettyEventType.IDLE, remoteAddress, ctx.channel()));\n                    }\n                }\n            }\n\n            ctx.fireUserEventTriggered(evt);\n        }\n\n        @Override\n        public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {\n            final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());\n            log.warn(\"NETTY SERVER PIPELINE: exceptionCaught {}\", remoteAddress);\n            log.warn(\"NETTY SERVER PIPELINE: exceptionCaught exception.\", cause);\n\n            if (NettyRemotingServer.this.channelEventListener != null) {\n                NettyRemotingServer.this.putNettyEvent(new NettyEvent(NettyEventType.EXCEPTION, remoteAddress, ctx.channel()));\n            }\n\n            RemotingHelper.closeChannel(ctx.channel());\n        }\n    }\n\n    /**\n     * The NettyRemotingServer supports bind multiple ports, each port bound by a SubRemotingServer. The\n     * SubRemotingServer will delegate all the functions to NettyRemotingServer, so the sub server can share all the\n     * resources from its parent server.\n     */\n    class SubRemotingServer extends NettyRemotingAbstract implements RemotingServer {\n        private volatile int listenPort;\n        private volatile Channel serverChannel;\n\n        SubRemotingServer(final int port, final int permitsOnway, final int permitsAsync) {\n            super(permitsOnway, permitsAsync);\n            listenPort = port;\n        }\n\n        @Override\n        public void registerProcessor(final int requestCode, final NettyRequestProcessor processor,\n                                      final ExecutorService executor) {\n            ExecutorService executorThis = executor;\n            if (null == executor) {\n                executorThis = NettyRemotingServer.this.publicExecutor;\n            }\n\n            Pair<NettyRequestProcessor, ExecutorService> pair = new Pair<>(processor, executorThis);\n            this.processorTable.put(requestCode, pair);\n        }\n\n        @Override\n        public void registerDefaultProcessor(final NettyRequestProcessor processor, final ExecutorService executor) {\n            this.defaultRequestProcessorPair = new Pair<>(processor, executor);\n        }\n\n        @Override\n        public int localListenPort() {\n            return listenPort;\n        }\n\n        @Override\n        public Pair<NettyRequestProcessor, ExecutorService> getProcessorPair(final int requestCode) {\n            return this.processorTable.get(requestCode);\n        }\n\n        @Override\n        public Pair<NettyRequestProcessor, ExecutorService> getDefaultProcessorPair() {\n            return this.defaultRequestProcessorPair;\n        }\n\n        @Override\n        public RemotingServer newRemotingServer(final int port) {\n            throw new UnsupportedOperationException(\"The SubRemotingServer of NettyRemotingServer \" +\n                \"doesn't support new nested RemotingServer\");\n        }\n\n        @Override\n        public void removeRemotingServer(final int port) {\n            throw new UnsupportedOperationException(\"The SubRemotingServer of NettyRemotingServer \" +\n                \"doesn't support remove nested RemotingServer\");\n        }\n\n        @Override\n        public RemotingCommand invokeSync(final Channel channel, final RemotingCommand request,\n                                          final long timeoutMillis) throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException {\n            return this.invokeSyncImpl(channel, request, timeoutMillis);\n        }\n\n        @Override\n        public void invokeAsync(final Channel channel, final RemotingCommand request, final long timeoutMillis,\n                                final InvokeCallback invokeCallback) throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException {\n            this.invokeAsyncImpl(channel, request, timeoutMillis, invokeCallback);\n        }\n\n        @Override\n        public void invokeOneway(final Channel channel, final RemotingCommand request,\n                                 final long timeoutMillis) throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException {\n            this.invokeOnewayImpl(channel, request, timeoutMillis);\n        }\n\n        @Override\n        public void start() {\n            try {\n                if (listenPort < 0) {\n                    listenPort = 0;\n                }\n                this.serverChannel = NettyRemotingServer.this.serverBootstrap.bind(listenPort).sync().channel();\n                if (0 == listenPort) {\n                    InetSocketAddress addr = (InetSocketAddress) this.serverChannel.localAddress();\n                    this.listenPort = addr.getPort();\n                }\n            } catch (InterruptedException e) {\n                throw new RuntimeException(\"this.subRemotingServer.serverBootstrap.bind().sync() InterruptedException\", e);\n            }\n        }\n\n        @Override\n        public void shutdown() {\n            isShuttingDown.set(true);\n            if (this.serverChannel != null) {\n                try {\n                    this.serverChannel.close().await(5, TimeUnit.SECONDS);\n                } catch (InterruptedException ignored) {\n                }\n            }\n        }\n\n        @Override\n        public ChannelEventListener getChannelEventListener() {\n            return NettyRemotingServer.this.getChannelEventListener();\n        }\n\n        @Override\n        public ExecutorService getCallbackExecutor() {\n            return NettyRemotingServer.this.getCallbackExecutor();\n        }\n    }\n\n    public class HAProxyMessageHandler extends ChannelInboundHandlerAdapter {\n\n        @Override\n        public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {\n            if (msg instanceof HAProxyMessage) {\n                handleWithMessage((HAProxyMessage) msg, ctx.channel());\n            } else {\n                super.channelRead(ctx, msg);\n            }\n            ctx.pipeline().remove(this);\n        }\n\n        /**\n         * The definition of key refers to the implementation of nginx\n         * <a href=\"https://nginx.org/en/docs/http/ngx_http_core_module.html#var_proxy_protocol_addr\">ngx_http_core_module</a>\n         * @param msg\n         * @param channel\n         */\n        private void handleWithMessage(HAProxyMessage msg, Channel channel) {\n            try {\n                if (StringUtils.isNotBlank(msg.sourceAddress())) {\n                    RemotingHelper.setPropertyToAttr(channel, AttributeKeys.PROXY_PROTOCOL_ADDR, msg.sourceAddress());\n                }\n                if (msg.sourcePort() > 0) {\n                    RemotingHelper.setPropertyToAttr(channel, AttributeKeys.PROXY_PROTOCOL_PORT, String.valueOf(msg.sourcePort()));\n                }\n                if (StringUtils.isNotBlank(msg.destinationAddress())) {\n                    RemotingHelper.setPropertyToAttr(channel, AttributeKeys.PROXY_PROTOCOL_SERVER_ADDR, msg.destinationAddress());\n                }\n                if (msg.destinationPort() > 0) {\n                    RemotingHelper.setPropertyToAttr(channel, AttributeKeys.PROXY_PROTOCOL_SERVER_PORT, String.valueOf(msg.destinationPort()));\n                }\n                if (CollectionUtils.isNotEmpty(msg.tlvs())) {\n                    msg.tlvs().forEach(tlv -> {\n                        handleHAProxyTLV(tlv, channel);\n                    });\n                }\n            } finally {\n                msg.release();\n            }\n        }\n    }\n\n    protected void handleHAProxyTLV(HAProxyTLV tlv, Channel channel) {\n        byte[] valueBytes = ByteBufUtil.getBytes(tlv.content());\n        if (!BinaryUtil.isAscii(valueBytes)) {\n            return;\n        }\n        AttributeKey<String> key = AttributeKeys.valueOf(\n            HAProxyConstants.PROXY_PROTOCOL_TLV_PREFIX + String.format(\"%02x\", tlv.typeByteValue()));\n        RemotingHelper.setPropertyToAttr(channel, key, new String(valueBytes, CharsetUtil.UTF_8));\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRequestProcessor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.netty;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\n/**\n * Common remoting command processor\n */\npublic interface NettyRequestProcessor {\n    RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request)\n        throws Exception;\n\n    boolean rejectRequest();\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.netty;\n\npublic class NettyServerConfig implements Cloneable {\n\n    /**\n     * Bind address may be hostname, IPv4 or IPv6.\n     * By default, it's wildcard address, listening all network interfaces.\n     */\n    private String bindAddress = \"0.0.0.0\";\n    private int listenPort = 0;\n    private int serverWorkerThreads = 8;\n    private int serverCallbackExecutorThreads = 0;\n    private int serverSelectorThreads = 3;\n    private int serverOnewaySemaphoreValue = 256;\n    private int serverAsyncSemaphoreValue = 64;\n    private int serverChannelMaxIdleTimeSeconds = 120;\n\n    private int serverSocketSndBufSize = NettySystemConfig.socketSndbufSize;\n    private int serverSocketRcvBufSize = NettySystemConfig.socketRcvbufSize;\n    private int writeBufferHighWaterMark = NettySystemConfig.writeBufferHighWaterMark;\n    private int writeBufferLowWaterMark = NettySystemConfig.writeBufferLowWaterMark;\n    private int serverSocketBacklog = NettySystemConfig.socketBacklog;\n    private boolean serverNettyWorkerGroupEnable = true;\n    private boolean serverPooledByteBufAllocatorEnable = true;\n\n    private boolean enableShutdownGracefully = false;\n    private int shutdownWaitTimeSeconds = 30;\n\n    /**\n     * make install\n     *\n     *\n     * ../glibc-2.10.1/configure \\ --prefix=/usr \\ --with-headers=/usr/include \\\n     * --host=x86_64-linux-gnu \\ --build=x86_64-pc-linux-gnu \\ --without-gd\n     */\n    private boolean useEpollNativeSelector = false;\n\n    public String getBindAddress() {\n        return bindAddress;\n    }\n\n    public void setBindAddress(String bindAddress) {\n        this.bindAddress = bindAddress;\n    }\n\n    public int getListenPort() {\n        return listenPort;\n    }\n\n    public void setListenPort(int listenPort) {\n        this.listenPort = listenPort;\n    }\n\n    public int getServerWorkerThreads() {\n        return serverWorkerThreads;\n    }\n\n    public void setServerWorkerThreads(int serverWorkerThreads) {\n        this.serverWorkerThreads = serverWorkerThreads;\n    }\n\n    public int getServerSelectorThreads() {\n        return serverSelectorThreads;\n    }\n\n    public void setServerSelectorThreads(int serverSelectorThreads) {\n        this.serverSelectorThreads = serverSelectorThreads;\n    }\n\n    public int getServerOnewaySemaphoreValue() {\n        return serverOnewaySemaphoreValue;\n    }\n\n    public void setServerOnewaySemaphoreValue(int serverOnewaySemaphoreValue) {\n        this.serverOnewaySemaphoreValue = serverOnewaySemaphoreValue;\n    }\n\n    public int getServerCallbackExecutorThreads() {\n        return serverCallbackExecutorThreads;\n    }\n\n    public void setServerCallbackExecutorThreads(int serverCallbackExecutorThreads) {\n        this.serverCallbackExecutorThreads = serverCallbackExecutorThreads;\n    }\n\n    public int getServerAsyncSemaphoreValue() {\n        return serverAsyncSemaphoreValue;\n    }\n\n    public void setServerAsyncSemaphoreValue(int serverAsyncSemaphoreValue) {\n        this.serverAsyncSemaphoreValue = serverAsyncSemaphoreValue;\n    }\n\n    public int getServerChannelMaxIdleTimeSeconds() {\n        return serverChannelMaxIdleTimeSeconds;\n    }\n\n    public void setServerChannelMaxIdleTimeSeconds(int serverChannelMaxIdleTimeSeconds) {\n        this.serverChannelMaxIdleTimeSeconds = serverChannelMaxIdleTimeSeconds;\n    }\n\n    public int getServerSocketSndBufSize() {\n        return serverSocketSndBufSize;\n    }\n\n    public void setServerSocketSndBufSize(int serverSocketSndBufSize) {\n        this.serverSocketSndBufSize = serverSocketSndBufSize;\n    }\n\n    public int getServerSocketRcvBufSize() {\n        return serverSocketRcvBufSize;\n    }\n\n    public void setServerSocketRcvBufSize(int serverSocketRcvBufSize) {\n        this.serverSocketRcvBufSize = serverSocketRcvBufSize;\n    }\n\n    public int getServerSocketBacklog() {\n        return serverSocketBacklog;\n    }\n\n    public void setServerSocketBacklog(int serverSocketBacklog) {\n        this.serverSocketBacklog = serverSocketBacklog;\n    }\n\n    public boolean isServerPooledByteBufAllocatorEnable() {\n        return serverPooledByteBufAllocatorEnable;\n    }\n\n    public void setServerPooledByteBufAllocatorEnable(boolean serverPooledByteBufAllocatorEnable) {\n        this.serverPooledByteBufAllocatorEnable = serverPooledByteBufAllocatorEnable;\n    }\n\n    public boolean isUseEpollNativeSelector() {\n        return useEpollNativeSelector;\n    }\n\n    public void setUseEpollNativeSelector(boolean useEpollNativeSelector) {\n        this.useEpollNativeSelector = useEpollNativeSelector;\n    }\n\n    @Override\n    public Object clone() throws CloneNotSupportedException {\n        return super.clone();\n    }\n\n    public int getWriteBufferLowWaterMark() {\n        return writeBufferLowWaterMark;\n    }\n\n    public void setWriteBufferLowWaterMark(int writeBufferLowWaterMark) {\n        this.writeBufferLowWaterMark = writeBufferLowWaterMark;\n    }\n\n    public int getWriteBufferHighWaterMark() {\n        return writeBufferHighWaterMark;\n    }\n\n    public void setWriteBufferHighWaterMark(int writeBufferHighWaterMark) {\n        this.writeBufferHighWaterMark = writeBufferHighWaterMark;\n    }\n\n    public boolean isServerNettyWorkerGroupEnable() {\n        return serverNettyWorkerGroupEnable;\n    }\n\n    public void setServerNettyWorkerGroupEnable(boolean serverNettyWorkerGroupEnable) {\n        this.serverNettyWorkerGroupEnable = serverNettyWorkerGroupEnable;\n    }\n\n    public boolean isEnableShutdownGracefully() {\n        return enableShutdownGracefully;\n    }\n\n    public void setEnableShutdownGracefully(boolean enableShutdownGracefully) {\n        this.enableShutdownGracefully = enableShutdownGracefully;\n    }\n\n    public int getShutdownWaitTimeSeconds() {\n        return shutdownWaitTimeSeconds;\n    }\n\n    public void setShutdownWaitTimeSeconds(int shutdownWaitTimeSeconds) {\n        this.shutdownWaitTimeSeconds = shutdownWaitTimeSeconds;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettySystemConfig.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.netty;\n\npublic class NettySystemConfig {\n    public static final String COM_ROCKETMQ_REMOTING_NETTY_POOLED_BYTE_BUF_ALLOCATOR_ENABLE =\n        \"com.rocketmq.remoting.nettyPooledByteBufAllocatorEnable\";\n    public static final String COM_ROCKETMQ_REMOTING_SOCKET_SNDBUF_SIZE =\n        \"com.rocketmq.remoting.socket.sndbuf.size\";\n    public static final String COM_ROCKETMQ_REMOTING_SOCKET_RCVBUF_SIZE =\n        \"com.rocketmq.remoting.socket.rcvbuf.size\";\n    public static final String COM_ROCKETMQ_REMOTING_SOCKET_BACKLOG =\n        \"com.rocketmq.remoting.socket.backlog\";\n    public static final String COM_ROCKETMQ_REMOTING_CLIENT_ASYNC_SEMAPHORE_VALUE =\n        \"com.rocketmq.remoting.clientAsyncSemaphoreValue\";\n    public static final String COM_ROCKETMQ_REMOTING_CLIENT_ONEWAY_SEMAPHORE_VALUE =\n        \"com.rocketmq.remoting.clientOnewaySemaphoreValue\";\n    public static final String COM_ROCKETMQ_REMOTING_CLIENT_WORKER_SIZE =\n        \"com.rocketmq.remoting.client.worker.size\";\n    public static final String COM_ROCKETMQ_REMOTING_CLIENT_CONNECT_TIMEOUT =\n        \"com.rocketmq.remoting.client.connect.timeout\";\n    public static final String COM_ROCKETMQ_REMOTING_CLIENT_CHANNEL_MAX_IDLE_SECONDS =\n        \"com.rocketmq.remoting.client.channel.maxIdleTimeSeconds\";\n    public static final String COM_ROCKETMQ_REMOTING_CLIENT_CLOSE_SOCKET_IF_TIMEOUT =\n        \"com.rocketmq.remoting.client.closeSocketIfTimeout\";\n    public static final String COM_ROCKETMQ_REMOTING_WRITE_BUFFER_HIGH_WATER_MARK_VALUE =\n        \"com.rocketmq.remoting.write.buffer.high.water.mark\";\n    public static final String COM_ROCKETMQ_REMOTING_WRITE_BUFFER_LOW_WATER_MARK =\n        \"com.rocketmq.remoting.write.buffer.low.water.mark\";\n\n    public static final boolean NETTY_POOLED_BYTE_BUF_ALLOCATOR_ENABLE = //\n        Boolean.parseBoolean(System.getProperty(COM_ROCKETMQ_REMOTING_NETTY_POOLED_BYTE_BUF_ALLOCATOR_ENABLE, \"false\"));\n    public static final int CLIENT_ASYNC_SEMAPHORE_VALUE = //\n        Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_CLIENT_ASYNC_SEMAPHORE_VALUE, \"65535\"));\n    public static final int CLIENT_ONEWAY_SEMAPHORE_VALUE =\n        Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_CLIENT_ONEWAY_SEMAPHORE_VALUE, \"65535\"));\n    public static int socketSndbufSize =\n        Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_SOCKET_SNDBUF_SIZE, \"0\"));\n    public static int socketRcvbufSize =\n        Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_SOCKET_RCVBUF_SIZE, \"0\"));\n    public static int socketBacklog =\n        Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_SOCKET_BACKLOG, \"1024\"));\n    public static int clientWorkerSize =\n        Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_CLIENT_WORKER_SIZE, \"4\"));\n    public static int connectTimeoutMillis =\n        Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_CLIENT_CONNECT_TIMEOUT, \"3000\"));\n    public static int clientChannelMaxIdleTimeSeconds =\n        Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_CLIENT_CHANNEL_MAX_IDLE_SECONDS, \"120\"));\n    public static boolean clientCloseSocketIfTimeout =\n        Boolean.parseBoolean(System.getProperty(COM_ROCKETMQ_REMOTING_CLIENT_CLOSE_SOCKET_IF_TIMEOUT, \"true\"));\n    public static int writeBufferHighWaterMark =\n        Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_WRITE_BUFFER_HIGH_WATER_MARK_VALUE, \"0\"));\n    public static int writeBufferLowWaterMark =\n        Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_WRITE_BUFFER_LOW_WATER_MARK, \"0\"));\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/netty/RemotingCodeDistributionHandler.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.netty;\n\nimport io.netty.channel.ChannelDuplexHandler;\nimport io.netty.channel.ChannelHandler;\nimport io.netty.channel.ChannelHandlerContext;\nimport io.netty.channel.ChannelPromise;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.atomic.LongAdder;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\n@ChannelHandler.Sharable\npublic class RemotingCodeDistributionHandler extends ChannelDuplexHandler {\n\n    private final ConcurrentMap<Integer, LongAdder> inboundDistribution;\n    private final ConcurrentMap<Integer, LongAdder> outboundDistribution;\n\n    public RemotingCodeDistributionHandler() {\n        inboundDistribution = new ConcurrentHashMap<>();\n        outboundDistribution = new ConcurrentHashMap<>();\n    }\n\n    private void countInbound(int requestCode) {\n        LongAdder item = inboundDistribution.computeIfAbsent(requestCode, k -> new LongAdder());\n        item.increment();\n    }\n\n    private void countOutbound(int responseCode) {\n        LongAdder item = outboundDistribution.computeIfAbsent(responseCode, k -> new LongAdder());\n        item.increment();\n    }\n\n    @Override\n    public void channelRead(ChannelHandlerContext ctx, Object msg) {\n        if (msg instanceof RemotingCommand) {\n            RemotingCommand cmd = (RemotingCommand) msg;\n            countInbound(cmd.getCode());\n        }\n        ctx.fireChannelRead(msg);\n    }\n\n    @Override\n    public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception {\n        if (msg instanceof RemotingCommand) {\n            RemotingCommand cmd = (RemotingCommand) msg;\n            countOutbound(cmd.getCode());\n        }\n        ctx.write(msg, promise);\n    }\n\n    private Map<Integer, Long> getDistributionSnapshot(Map<Integer, LongAdder> countMap) {\n        Map<Integer, Long> map = new HashMap<>(countMap.size());\n        for (Map.Entry<Integer, LongAdder> entry : countMap.entrySet()) {\n            map.put(entry.getKey(), entry.getValue().sumThenReset());\n        }\n        return map;\n    }\n\n    private String snapshotToString(Map<Integer, Long> distribution) {\n        if (null != distribution && !distribution.isEmpty()) {\n            StringBuilder sb = new StringBuilder(\"{\");\n            boolean first = true;\n            for (Map.Entry<Integer, Long> entry : distribution.entrySet()) {\n                if (0L == entry.getValue()) {\n                    continue;\n                }\n                sb.append(first ? \"\" : \", \").append(entry.getKey()).append(\":\").append(entry.getValue());\n                first = false;\n            }\n            if (first) {\n                return null;\n            }\n            sb.append(\"}\");\n            return sb.toString();\n        }\n        return null;\n    }\n\n    public String getInBoundSnapshotString() {\n        return this.snapshotToString(this.getDistributionSnapshot(this.inboundDistribution));\n    }\n\n    public String getOutBoundSnapshotString() {\n        return this.snapshotToString(this.getDistributionSnapshot(this.outboundDistribution));\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/netty/RemotingResponseCallback.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.netty;\n\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic interface RemotingResponseCallback {\n    void callback(RemotingCommand response);\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/netty/RequestTask.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.netty;\n\nimport io.netty.channel.Channel;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic class RequestTask implements Runnable {\n    private final Runnable runnable;\n    private final long createTimestamp = System.currentTimeMillis();\n    private final Channel channel;\n    private final RemotingCommand request;\n    private volatile boolean stopRun = false;\n\n    public RequestTask(final Runnable runnable, final Channel channel, final RemotingCommand request) {\n        this.runnable = runnable;\n        this.channel = channel;\n        this.request = request;\n    }\n\n    @Override\n    public int hashCode() {\n        int result = runnable != null ? runnable.hashCode() : 0;\n        result = 31 * result + (int) (getCreateTimestamp() ^ (getCreateTimestamp() >>> 32));\n        result = 31 * result + (channel != null ? channel.hashCode() : 0);\n        result = 31 * result + (request != null ? request.hashCode() : 0);\n        result = 31 * result + (isStopRun() ? 1 : 0);\n        return result;\n    }\n\n    @Override\n    public boolean equals(final Object o) {\n        if (this == o)\n            return true;\n        if (!(o instanceof RequestTask))\n            return false;\n\n        final RequestTask that = (RequestTask) o;\n\n        if (getCreateTimestamp() != that.getCreateTimestamp())\n            return false;\n        if (isStopRun() != that.isStopRun())\n            return false;\n        if (channel != null ? !channel.equals(that.channel) : that.channel != null)\n            return false;\n        return request != null ? request.getOpaque() == that.request.getOpaque() : that.request == null;\n\n    }\n\n    public long getCreateTimestamp() {\n        return createTimestamp;\n    }\n\n    public boolean isStopRun() {\n        return stopRun;\n    }\n\n    public void setStopRun(final boolean stopRun) {\n        this.stopRun = stopRun;\n    }\n\n    @Override\n    public void run() {\n        if (!this.stopRun)\n            this.runnable.run();\n    }\n\n    public void returnResponse(int code, String remark) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(code, remark);\n        response.setOpaque(request.getOpaque());\n        this.channel.writeAndFlush(response);\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.netty;\n\nimport io.netty.channel.Channel;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport org.apache.rocketmq.remoting.InvokeCallback;\nimport org.apache.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic class ResponseFuture {\n    private final Channel channel;\n    private final int opaque;\n    private final RemotingCommand request;\n    private final long timeoutMillis;\n    private final InvokeCallback invokeCallback;\n    private final long beginTimestamp = System.currentTimeMillis();\n    private final CountDownLatch countDownLatch = new CountDownLatch(1);\n\n    private final SemaphoreReleaseOnlyOnce once;\n\n    private final AtomicBoolean executeCallbackOnlyOnce = new AtomicBoolean(false);\n    private volatile RemotingCommand responseCommand;\n    private volatile boolean sendRequestOK = true;\n    private volatile Throwable cause;\n    private volatile boolean interrupted = false;\n\n    public ResponseFuture(Channel channel, int opaque, long timeoutMillis, InvokeCallback invokeCallback,\n                          SemaphoreReleaseOnlyOnce once) {\n        this(channel, opaque, null, timeoutMillis, invokeCallback, once);\n    }\n\n    public ResponseFuture(Channel channel, int opaque, RemotingCommand request, long timeoutMillis, InvokeCallback invokeCallback,\n                          SemaphoreReleaseOnlyOnce once) {\n        this.channel = channel;\n        this.opaque = opaque;\n        this.request = request;\n        this.timeoutMillis = timeoutMillis;\n        this.invokeCallback = invokeCallback;\n        this.once = once;\n    }\n\n    public void executeInvokeCallback() {\n        if (invokeCallback != null) {\n            if (this.executeCallbackOnlyOnce.compareAndSet(false, true)) {\n                RemotingCommand response = getResponseCommand();\n                if (response != null) {\n                    invokeCallback.operationSucceed(response);\n                } else {\n                    if (!isSendRequestOK()) {\n                        invokeCallback.operationFail(new RemotingSendRequestException(channel.remoteAddress().toString(), getCause()));\n                    } else if (isTimeout()) {\n                        invokeCallback.operationFail(new RemotingTimeoutException(channel.remoteAddress().toString(), getTimeoutMillis(), getCause()));\n                    } else {\n                        invokeCallback.operationFail(new RemotingException(getRequestCommand().toString(), getCause()));\n                    }\n                }\n                invokeCallback.operationComplete(this);\n            }\n        }\n    }\n\n    public void interrupt() {\n        interrupted = true;\n        executeInvokeCallback();\n    }\n\n    public void release() {\n        if (this.once != null) {\n            this.once.release();\n        }\n    }\n\n    public boolean isTimeout() {\n        long diff = System.currentTimeMillis() - this.beginTimestamp;\n        return diff > this.timeoutMillis;\n    }\n\n    public RemotingCommand waitResponse(final long timeoutMillis) throws InterruptedException {\n        this.countDownLatch.await(timeoutMillis, TimeUnit.MILLISECONDS);\n        return this.responseCommand;\n    }\n\n    public void putResponse(final RemotingCommand responseCommand) {\n        this.responseCommand = responseCommand;\n        this.countDownLatch.countDown();\n    }\n\n    public long getBeginTimestamp() {\n        return beginTimestamp;\n    }\n\n    public boolean isSendRequestOK() {\n        return sendRequestOK;\n    }\n\n    public void setSendRequestOK(boolean sendRequestOK) {\n        this.sendRequestOK = sendRequestOK;\n    }\n\n    public long getTimeoutMillis() {\n        return timeoutMillis;\n    }\n\n    public InvokeCallback getInvokeCallback() {\n        return invokeCallback;\n    }\n\n    public Throwable getCause() {\n        return cause;\n    }\n\n    public void setCause(Throwable cause) {\n        this.cause = cause;\n    }\n\n    public RemotingCommand getResponseCommand() {\n        return responseCommand;\n    }\n\n    public void setResponseCommand(RemotingCommand responseCommand) {\n        this.responseCommand = responseCommand;\n    }\n\n    public int getOpaque() {\n        return opaque;\n    }\n\n    public RemotingCommand getRequestCommand() {\n        return request;\n    }\n\n    public Channel getChannel() {\n        return channel;\n    }\n\n    public boolean isInterrupted() {\n        return interrupted;\n    }\n\n    @Override\n    public String toString() {\n        return \"ResponseFuture [responseCommand=\" + responseCommand + \", sendRequestOK=\" + sendRequestOK\n            + \", cause=\" + cause + \", opaque=\" + opaque + \", timeoutMillis=\" + timeoutMillis\n            + \", invokeCallback=\" + invokeCallback + \", beginTimestamp=\" + beginTimestamp\n            + \", countDownLatch=\" + countDownLatch + \"]\";\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/netty/TlsHelper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.netty;\n\nimport io.netty.handler.ssl.ClientAuth;\nimport io.netty.handler.ssl.OpenSsl;\nimport io.netty.handler.ssl.SslContext;\nimport io.netty.handler.ssl.SslContextBuilder;\nimport io.netty.handler.ssl.SslProvider;\nimport io.netty.handler.ssl.util.InsecureTrustManagerFactory;\nimport io.netty.handler.ssl.util.SelfSignedCertificate;\nimport java.io.File;\nimport java.io.FileInputStream;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.security.cert.CertificateException;\nimport java.util.Arrays;\nimport java.util.Properties;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_CIPHERS;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_CLIENT_AUTHSERVER;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_CLIENT_CERTPATH;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_CLIENT_KEYPASSWORD;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_CLIENT_KEYPATH;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_CLIENT_TRUSTCERTPATH;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_PROTOCOLS;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_SERVER_AUTHCLIENT;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_SERVER_CERTPATH;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_SERVER_KEYPASSWORD;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_SERVER_KEYPATH;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_SERVER_NEED_CLIENT_AUTH;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_SERVER_TRUSTCERTPATH;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_TEST_MODE_ENABLE;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsCiphers;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsClientAuthServer;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsClientCertPath;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsClientKeyPassword;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsClientKeyPath;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsClientTrustCertPath;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsProtocols;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerAuthClient;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerCertPath;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerKeyPassword;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerKeyPath;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerNeedClientAuth;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerTrustCertPath;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsTestModeEnable;\n\npublic class TlsHelper {\n\n    public interface DecryptionStrategy {\n        /**\n         * Decrypt the target encrpted private key file.\n         *\n         * @param privateKeyEncryptPath A pathname string\n         * @param forClient tells whether it's a client-side key file\n         * @return An input stream for a decrypted key file\n         * @throws IOException if an I/O error has occurred\n         */\n        InputStream decryptPrivateKey(String privateKeyEncryptPath, boolean forClient) throws IOException;\n    }\n\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME);\n\n    private static DecryptionStrategy decryptionStrategy = new DecryptionStrategy() {\n        @Override\n        public InputStream decryptPrivateKey(final String privateKeyEncryptPath,\n            final boolean forClient) throws IOException {\n            return new FileInputStream(privateKeyEncryptPath);\n        }\n    };\n\n\n    public static void registerDecryptionStrategy(final DecryptionStrategy decryptionStrategy) {\n        TlsHelper.decryptionStrategy = decryptionStrategy;\n    }\n\n    public static SslContext buildSslContext(boolean forClient) throws IOException, CertificateException {\n        File configFile = new File(TlsSystemConfig.tlsConfigFile);\n        extractTlsConfigFromFile(configFile);\n        logTheFinalUsedTlsConfig();\n\n        SslProvider provider;\n        if (OpenSsl.isAvailable()) {\n            provider = SslProvider.OPENSSL;\n            LOGGER.info(\"Using OpenSSL provider\");\n        } else {\n            provider = SslProvider.JDK;\n            LOGGER.info(\"Using JDK SSL provider\");\n        }\n\n        SslContextBuilder sslContextBuilder = null;\n        if (forClient) {\n            if (tlsTestModeEnable) {\n                sslContextBuilder = SslContextBuilder\n                    .forClient()\n                    .sslProvider(SslProvider.JDK)\n                    .trustManager(InsecureTrustManagerFactory.INSTANCE);\n            } else {\n                sslContextBuilder = SslContextBuilder.forClient().sslProvider(SslProvider.JDK);\n\n\n                if (!tlsClientAuthServer) {\n                    sslContextBuilder.trustManager(InsecureTrustManagerFactory.INSTANCE);\n                } else {\n                    if (!isNullOrEmpty(tlsClientTrustCertPath)) {\n                        sslContextBuilder.trustManager(new File(tlsClientTrustCertPath));\n                    }\n                }\n\n                sslContextBuilder = sslContextBuilder.keyManager(\n                    !isNullOrEmpty(tlsClientCertPath) ? new FileInputStream(tlsClientCertPath) : null,\n                    !isNullOrEmpty(tlsClientKeyPath) ? decryptionStrategy.decryptPrivateKey(tlsClientKeyPath, true) : null,\n                    !isNullOrEmpty(tlsClientKeyPassword) ? tlsClientKeyPassword : null);\n            }\n        } else {\n\n            if (tlsTestModeEnable) {\n                SelfSignedCertificate selfSignedCertificate = new SelfSignedCertificate();\n                sslContextBuilder = SslContextBuilder\n                    .forServer(selfSignedCertificate.certificate(), selfSignedCertificate.privateKey())\n                    .sslProvider(provider)\n                    .clientAuth(ClientAuth.OPTIONAL);\n            } else {\n                sslContextBuilder = SslContextBuilder.forServer(\n                    !isNullOrEmpty(tlsServerCertPath) ? new FileInputStream(tlsServerCertPath) : null,\n                    !isNullOrEmpty(tlsServerKeyPath) ? decryptionStrategy.decryptPrivateKey(tlsServerKeyPath, false) : null,\n                    !isNullOrEmpty(tlsServerKeyPassword) ? tlsServerKeyPassword : null)\n                    .sslProvider(provider);\n\n                if (!tlsServerAuthClient) {\n                    sslContextBuilder.trustManager(InsecureTrustManagerFactory.INSTANCE);\n                } else {\n                    if (!isNullOrEmpty(tlsServerTrustCertPath)) {\n                        sslContextBuilder.trustManager(new File(tlsServerTrustCertPath));\n                    }\n                }\n\n                sslContextBuilder.clientAuth(parseClientAuthMode(tlsServerNeedClientAuth));\n            }\n        }\n        moreTlsConfig(sslContextBuilder);\n        return sslContextBuilder.build();\n    }\n\n    protected static void moreTlsConfig(SslContextBuilder sslContextBuilder) {\n        if (tlsCiphers != null) {\n            sslContextBuilder.ciphers(Arrays.asList(tlsCiphers.split(\",\")));\n        }\n        if (tlsProtocols != null) {\n            sslContextBuilder.protocols(tlsProtocols.split(\",\"));\n        }\n    }\n    private static void extractTlsConfigFromFile(final File configFile) {\n        if (!(configFile.exists() && configFile.isFile() && configFile.canRead())) {\n            LOGGER.info(\"Tls config file doesn't exist, skip it\");\n            return;\n        }\n\n        Properties properties;\n        properties = new Properties();\n        InputStream inputStream = null;\n        try {\n            inputStream = new FileInputStream(configFile);\n            properties.load(inputStream);\n        } catch (IOException ignore) {\n        } finally {\n            if (null != inputStream) {\n                try {\n                    inputStream.close();\n                } catch (IOException ignore) {\n                }\n            }\n        }\n\n        tlsTestModeEnable = Boolean.parseBoolean(properties.getProperty(TLS_TEST_MODE_ENABLE, String.valueOf(tlsTestModeEnable)));\n        tlsServerNeedClientAuth = properties.getProperty(TLS_SERVER_NEED_CLIENT_AUTH, tlsServerNeedClientAuth);\n        tlsServerKeyPath = properties.getProperty(TLS_SERVER_KEYPATH, tlsServerKeyPath);\n        tlsServerKeyPassword = properties.getProperty(TLS_SERVER_KEYPASSWORD, tlsServerKeyPassword);\n        tlsServerCertPath = properties.getProperty(TLS_SERVER_CERTPATH, tlsServerCertPath);\n        tlsServerAuthClient = Boolean.parseBoolean(properties.getProperty(TLS_SERVER_AUTHCLIENT, String.valueOf(tlsServerAuthClient)));\n        tlsServerTrustCertPath = properties.getProperty(TLS_SERVER_TRUSTCERTPATH, tlsServerTrustCertPath);\n\n        tlsClientKeyPath = properties.getProperty(TLS_CLIENT_KEYPATH, tlsClientKeyPath);\n        tlsClientKeyPassword = properties.getProperty(TLS_CLIENT_KEYPASSWORD, tlsClientKeyPassword);\n        tlsClientCertPath = properties.getProperty(TLS_CLIENT_CERTPATH, tlsClientCertPath);\n        tlsClientAuthServer = Boolean.parseBoolean(properties.getProperty(TLS_CLIENT_AUTHSERVER, String.valueOf(tlsClientAuthServer)));\n        tlsClientTrustCertPath = properties.getProperty(TLS_CLIENT_TRUSTCERTPATH, tlsClientTrustCertPath);\n\n        tlsCiphers = properties.getProperty(TLS_CIPHERS, tlsCiphers);\n        tlsProtocols = properties.getProperty(TLS_PROTOCOLS, tlsProtocols);\n    }\n\n    private static void logTheFinalUsedTlsConfig() {\n        LOGGER.info(\"Log the final used tls related configuration\");\n        LOGGER.info(\"{} = {}\", TLS_TEST_MODE_ENABLE, tlsTestModeEnable);\n        LOGGER.debug(\"{} = {}\", TLS_SERVER_NEED_CLIENT_AUTH, tlsServerNeedClientAuth);\n        LOGGER.debug(\"{} = {}\", TLS_SERVER_KEYPATH, tlsServerKeyPath);\n        LOGGER.debug(\"{} = {}\", TLS_SERVER_CERTPATH, tlsServerCertPath);\n        LOGGER.debug(\"{} = {}\", TLS_SERVER_AUTHCLIENT, tlsServerAuthClient);\n        LOGGER.debug(\"{} = {}\", TLS_SERVER_TRUSTCERTPATH, tlsServerTrustCertPath);\n\n        LOGGER.debug(\"{} = {}\", TLS_CLIENT_KEYPATH, tlsClientKeyPath);\n        LOGGER.debug(\"{} = {}\", TLS_CLIENT_CERTPATH, tlsClientCertPath);\n        LOGGER.debug(\"{} = {}\", TLS_CLIENT_AUTHSERVER, tlsClientAuthServer);\n        LOGGER.debug(\"{} = {}\", TLS_CLIENT_TRUSTCERTPATH, tlsClientTrustCertPath);\n\n        LOGGER.debug(\"{} = {}\", TLS_CIPHERS, tlsCiphers);\n        LOGGER.debug(\"{} = {}\", TLS_PROTOCOLS, tlsProtocols);\n    }\n\n    private static ClientAuth parseClientAuthMode(String authMode) {\n        if (null == authMode || authMode.trim().isEmpty()) {\n            return ClientAuth.NONE;\n        }\n\n        String authModeUpper = authMode.toUpperCase();\n        for (ClientAuth clientAuth : ClientAuth.values()) {\n            if (clientAuth.name().equals(authModeUpper)) {\n                return clientAuth;\n            }\n        }\n\n        return ClientAuth.NONE;\n    }\n\n    /**\n     * Determine if a string is {@code null} or {@link String#isEmpty()} returns {@code true}.\n     */\n    private static boolean isNullOrEmpty(String s) {\n        return s == null || s.isEmpty();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/netty/TlsSystemConfig.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.netty;\n\nimport io.netty.handler.ssl.SslContext;\nimport org.apache.rocketmq.remoting.common.TlsMode;\n\npublic class TlsSystemConfig {\n    public static final String TLS_SERVER_MODE = \"tls.server.mode\";\n    public static final String TLS_ENABLE = \"tls.enable\";\n    public static final String TLS_CONFIG_FILE = \"tls.config.file\";\n    public static final String TLS_TEST_MODE_ENABLE = \"tls.test.mode.enable\";\n\n    public static final String TLS_SERVER_NEED_CLIENT_AUTH = \"tls.server.need.client.auth\";\n    public static final String TLS_SERVER_KEYPATH = \"tls.server.keyPath\";\n    public static final String TLS_SERVER_KEYPASSWORD = \"tls.server.keyPassword\";\n    public static final String TLS_SERVER_CERTPATH = \"tls.server.certPath\";\n    public static final String TLS_SERVER_AUTHCLIENT = \"tls.server.authClient\";\n    public static final String TLS_SERVER_TRUSTCERTPATH = \"tls.server.trustCertPath\";\n\n    public static final String TLS_CLIENT_KEYPATH = \"tls.client.keyPath\";\n    public static final String TLS_CLIENT_KEYPASSWORD = \"tls.client.keyPassword\";\n    public static final String TLS_CLIENT_CERTPATH = \"tls.client.certPath\";\n    public static final String TLS_CLIENT_AUTHSERVER = \"tls.client.authServer\";\n    public static final String TLS_CLIENT_TRUSTCERTPATH = \"tls.client.trustCertPath\";\n\n    public static final String TLS_CIPHERS = \"tls.ciphers\";\n    public static final String TLS_PROTOCOLS = \"tls.protocols\";\n\n\n    /**\n     * To determine whether use SSL in client-side, include SDK client and BrokerOuterAPI\n     */\n    public static boolean tlsEnable = Boolean.parseBoolean(System.getProperty(TLS_ENABLE, \"false\"));\n\n    /**\n     * To determine whether use test mode when initialize TLS context\n     */\n    public static boolean tlsTestModeEnable = Boolean.parseBoolean(System.getProperty(TLS_TEST_MODE_ENABLE, \"true\"));\n\n    /**\n     * Indicates the state of the {@link javax.net.ssl.SSLEngine} with respect to client authentication.\n     * This configuration item really only applies when building the server-side {@link SslContext},\n     * and can be set to none, require or optional.\n     */\n    public static String tlsServerNeedClientAuth = System.getProperty(TLS_SERVER_NEED_CLIENT_AUTH, \"none\");\n    /**\n     * The store path of server-side private key\n     */\n    public static String tlsServerKeyPath = System.getProperty(TLS_SERVER_KEYPATH, null);\n\n    /**\n     * The  password of the server-side private key\n     */\n    public static String tlsServerKeyPassword = System.getProperty(TLS_SERVER_KEYPASSWORD, null);\n\n    /**\n     * The store path of server-side X.509 certificate chain in PEM format\n     */\n    public static String tlsServerCertPath = System.getProperty(TLS_SERVER_CERTPATH, null);\n\n    /**\n     * To determine whether verify the client endpoint's certificate strictly\n     */\n    public static boolean tlsServerAuthClient = Boolean.parseBoolean(System.getProperty(TLS_SERVER_AUTHCLIENT, \"false\"));\n\n    /**\n     * The store path of trusted certificates for verifying the client endpoint's certificate\n     */\n    public static String tlsServerTrustCertPath = System.getProperty(TLS_SERVER_TRUSTCERTPATH, null);\n\n    /**\n     * The store path of client-side private key\n     */\n    public static String tlsClientKeyPath = System.getProperty(TLS_CLIENT_KEYPATH, null);\n\n    /**\n     * The  password of the client-side private key\n     */\n    public static String tlsClientKeyPassword = System.getProperty(TLS_CLIENT_KEYPASSWORD, null);\n\n    /**\n     * The store path of client-side X.509 certificate chain in PEM format\n     */\n    public static String tlsClientCertPath = System.getProperty(TLS_CLIENT_CERTPATH, null);\n\n    /**\n     * To determine whether verify the server endpoint's certificate strictly\n     */\n    public static boolean tlsClientAuthServer = Boolean.parseBoolean(System.getProperty(TLS_CLIENT_AUTHSERVER, \"false\"));\n\n    /**\n     * The store path of trusted certificates for verifying the server endpoint's certificate\n     */\n    public static String tlsClientTrustCertPath = System.getProperty(TLS_CLIENT_TRUSTCERTPATH, null);\n\n    /**\n     * For server, three SSL modes are supported: disabled, permissive and enforcing.\n     * For client, use {@link TlsSystemConfig#tlsEnable} to determine whether use SSL.\n     * <ol>\n     *     <li><strong>disabled:</strong> SSL is not supported; any incoming SSL handshake will be rejected, causing connection closed.</li>\n     *     <li><strong>permissive:</strong> SSL is optional, aka, server in this mode can serve client connections with or without SSL;</li>\n     *     <li><strong>enforcing:</strong> SSL is required, aka, non SSL connection will be rejected.</li>\n     * </ol>\n     */\n    public static TlsMode tlsMode = TlsMode.parse(System.getProperty(TLS_SERVER_MODE, \"permissive\"));\n\n    /**\n     * A config file to store the above TLS related configurations,\n     * except {@link TlsSystemConfig#tlsMode} and {@link TlsSystemConfig#tlsEnable}\n     */\n    public static String tlsConfigFile = System.getProperty(TLS_CONFIG_FILE, \"/etc/rocketmq/tls.properties\");\n\n    /**\n     * The ciphers to be used in TLS\n     * <ol>\n     *     <li>If null, use the default ciphers</li>\n     *     <li>Otherwise, use the ciphers specified in this string, eg: -Dtls.ciphers=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256</li>\n     * </ol>\n     */\n    public static String tlsCiphers = System.getProperty(TLS_CIPHERS, null);\n\n    /**\n     * The protocols to be used in TLS\n     * <ol>\n     *     <li>If null, use the default protocols</li>\n     *     <li>Otherwise, use the protocols specified in this string, eg: -Dtls.protocols=TLSv1.2,TLSv1.3</li>\n     * </ol>\n     */\n    public static String tlsProtocols = System.getProperty(TLS_PROTOCOLS, null);\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/pipeline/RequestPipeline.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.pipeline;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic interface RequestPipeline {\n\n    void execute(ChannelHandlerContext ctx, RemotingCommand request) throws Exception;\n\n    default RequestPipeline pipe(RequestPipeline source) {\n        return (ctx, request) -> {\n            source.execute(ctx, request);\n            execute(ctx, request);\n        };\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/BitSetSerializerDeserializer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol;\n\nimport com.alibaba.fastjson2.JSONReader;\nimport com.alibaba.fastjson2.JSONWriter;\nimport com.alibaba.fastjson2.reader.ObjectReader;\nimport com.alibaba.fastjson2.writer.ObjectWriter;\n\nimport java.lang.reflect.Type;\nimport java.util.Base64;\nimport java.util.BitSet;\n\npublic class BitSetSerializerDeserializer implements ObjectReader<BitSet>, ObjectWriter<BitSet> {\n\n    @Override\n    public void write(JSONWriter writer, Object object, Object fieldName, Type fieldType, long features) {\n        if (object == null) {\n            writer.writeBase64(null);\n        } else {\n            writer.writeBase64(((BitSet) object).toByteArray());\n        }\n    }\n\n    @Override\n    public BitSet readObject(JSONReader reader, Type fieldType, Object fieldName, long features) {\n        if (reader.nextIfNull()) {\n            return null;\n        }\n        String base64 = reader.readString();\n        if (base64 == null || base64.isEmpty()) {\n            return null;\n        }\n        byte[] bytes = Base64.getDecoder().decode(base64);\n        return BitSet.valueOf(bytes);\n    }\n\n    @Override\n    public long getFeatures() {\n        return 0L;\n    }\n\n    @Override\n    public Class<BitSet> getObjectClass() {\n        return ObjectReader.super.getObjectClass();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/BrokerSyncInfo.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol;\n\npublic class BrokerSyncInfo extends RemotingSerializable {\n    /**\n     * For slave online sync, retrieve HA address before register\n     */\n    private String masterHaAddress;\n\n    private long masterFlushOffset;\n\n    private String masterAddress;\n\n    public BrokerSyncInfo(String masterHaAddress, long masterFlushOffset, String masterAddress) {\n        this.masterHaAddress = masterHaAddress;\n        this.masterFlushOffset = masterFlushOffset;\n        this.masterAddress = masterAddress;\n    }\n\n    public String getMasterHaAddress() {\n        return masterHaAddress;\n    }\n\n    public void setMasterHaAddress(String masterHaAddress) {\n        this.masterHaAddress = masterHaAddress;\n    }\n\n    public long getMasterFlushOffset() {\n        return masterFlushOffset;\n    }\n\n    public void setMasterFlushOffset(long masterFlushOffset) {\n        this.masterFlushOffset = masterFlushOffset;\n    }\n\n    public String getMasterAddress() {\n        return masterAddress;\n    }\n\n    public void setMasterAddress(String masterAddress) {\n        this.masterAddress = masterAddress;\n    }\n\n    @Override\n    public String toString() {\n        return \"BrokerSyncInfo{\" +\n            \"masterHaAddress='\" + masterHaAddress + '\\'' +\n            \", masterFlushOffset=\" + masterFlushOffset +\n            \", masterAddress=\" + masterAddress +\n            '}';\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/DataVersion.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol;\n\nimport java.util.concurrent.atomic.AtomicLong;\n\npublic class DataVersion extends RemotingSerializable {\n    private long stateVersion = 0L;\n    private long timestamp = System.currentTimeMillis();\n    private AtomicLong counter = new AtomicLong(0);\n\n    public void assignNewOne(final DataVersion dataVersion) {\n        this.timestamp = dataVersion.timestamp;\n        this.stateVersion = dataVersion.stateVersion;\n        this.counter.set(dataVersion.counter.get());\n    }\n\n    public void nextVersion() {\n        this.nextVersion(0L);\n    }\n\n    public void nextVersion(long stateVersion) {\n        this.timestamp = System.currentTimeMillis();\n        this.stateVersion = stateVersion;\n        this.counter.incrementAndGet();\n    }\n\n    public long getStateVersion() {\n        return stateVersion;\n    }\n\n    public void setStateVersion(long stateVersion) {\n        this.stateVersion = stateVersion;\n    }\n\n    public long getTimestamp() {\n        return timestamp;\n    }\n\n    public void setTimestamp(long timestamp) {\n        this.timestamp = timestamp;\n    }\n\n    public AtomicLong getCounter() {\n        return counter;\n    }\n\n    public void setCounter(AtomicLong counter) {\n        this.counter = counter;\n    }\n\n    @Override\n    public boolean equals(final Object o) {\n        if (this == o)\n            return true;\n        if (o == null || getClass() != o.getClass())\n            return false;\n\n        DataVersion version = (DataVersion) o;\n\n        if (getStateVersion() != version.getStateVersion())\n            return false;\n        if (getTimestamp() != version.getTimestamp())\n            return false;\n\n        if (counter != null && version.counter != null) {\n            return counter.longValue() == version.counter.longValue();\n        }\n\n        return null == counter && null == version.counter;\n\n    }\n\n    @Override\n    public int hashCode() {\n        int result = (int) (getStateVersion() ^ (getStateVersion() >>> 32));\n        result = 31 * result + (int) (getTimestamp() ^ (getTimestamp() >>> 32));\n        if (null != counter) {\n            long l = counter.get();\n            result = 31 * result + (int) (l ^ (l >>> 32));\n        }\n        return result;\n    }\n\n    @Override\n    public String toString() {\n        final StringBuilder sb = new StringBuilder(\"DataVersion[\");\n        sb.append(\"timestamp=\").append(timestamp);\n        sb.append(\", counter=\").append(counter);\n        sb.append(']');\n        return sb.toString();\n    }\n\n    public int compare(DataVersion dataVersion) {\n        if (this.getStateVersion() > dataVersion.getStateVersion()) {\n            return 1;\n        } else if (this.getStateVersion() < dataVersion.getStateVersion()) {\n            return -1;\n        } else if (this.getCounter().get() > dataVersion.getCounter().get()) {\n            return 1;\n        } else if (this.getCounter().get() < dataVersion.getCounter().get()) {\n            return -1;\n        } else if (this.getTimestamp() > dataVersion.getTimestamp()) {\n            return 1;\n        } else if (this.getTimestamp() < dataVersion.getTimestamp()) {\n            return -1;\n        }\n        return 0;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/EpochEntry.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol;\n\nimport java.util.Objects;\n\npublic class EpochEntry extends RemotingSerializable {\n\n    public static final long LAST_EPOCH_END_OFFSET = Long.MAX_VALUE;\n    private int epoch;\n    private long startOffset;\n    private long endOffset = LAST_EPOCH_END_OFFSET;\n\n    public EpochEntry(EpochEntry entry) {\n        this.epoch = entry.getEpoch();\n        this.startOffset = entry.getStartOffset();\n        this.endOffset = entry.getEndOffset();\n    }\n\n    public EpochEntry(int epoch, long startOffset) {\n        this.epoch = epoch;\n        this.startOffset = startOffset;\n    }\n\n    public EpochEntry(int epoch, long startOffset, long endOffset) {\n        this.epoch = epoch;\n        this.startOffset = startOffset;\n        this.endOffset = endOffset;\n    }\n\n    public int getEpoch() {\n        return epoch;\n    }\n\n    public void setEpoch(int epoch) {\n        this.epoch = epoch;\n    }\n\n    public long getStartOffset() {\n        return startOffset;\n    }\n\n    public void setStartOffset(long startOffset) {\n        this.startOffset = startOffset;\n    }\n\n    public long getEndOffset() {\n        return endOffset;\n    }\n\n    public void setEndOffset(long endOffset) {\n        this.endOffset = endOffset;\n    }\n\n    @Override\n    public String toString() {\n        return \"EpochEntry{\" +\n            \"epoch=\" + epoch +\n            \", startOffset=\" + startOffset +\n            \", endOffset=\" + endOffset +\n            '}';\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) {\n            return true;\n        }\n\n        if (o == null || getClass() != o.getClass()) {\n            return false;\n        }\n\n        EpochEntry entry = (EpochEntry) o;\n        return epoch == entry.epoch && startOffset == entry.startOffset && endOffset == entry.endOffset;\n    }\n\n    @Override\n    public int hashCode() {\n        return Objects.hash(epoch, startOffset, endOffset);\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/FastCodesHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol;\n\nimport java.util.HashMap;\n\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\nimport io.netty.buffer.ByteBuf;\n\npublic interface FastCodesHeader {\n\n    default String getAndCheckNotNull(HashMap<String, String> fields, String field) {\n        String value = fields.get(field);\n        if (value == null) {\n            String headerClass = this.getClass().getSimpleName();\n            RemotingCommand.log.error(\"the custom field {}.{} is null\", headerClass, field);\n            // no exception throws, keep compatible with RemotingCommand.decodeCommandCustomHeader\n        }\n        return value;\n    }\n\n    default void writeIfNotNull(ByteBuf out, String key, Object value) {\n        if (value != null) {\n            RocketMQSerializable.writeStr(out, true, key);\n            RocketMQSerializable.writeStr(out, false, value.toString());\n        }\n    }\n\n    void encode(ByteBuf out);\n\n    void decode(HashMap<String, String> fields) throws RemotingCommandException;\n\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ForbiddenType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol;\n\n/**\n *\n * gives the reason for a no permission messaging pulling.\n *\n */\npublic interface ForbiddenType {\n\n    /**\n     * 1=forbidden by broker\n     */\n    int BROKER_FORBIDDEN               = 1;\n    /**\n     * 2=forbidden by groupId\n     */\n    int GROUP_FORBIDDEN                = 2;\n    /**\n     * 3=forbidden by topic\n     */\n    int TOPIC_FORBIDDEN                = 3;\n    /**\n     * 4=forbidden by broadcasting mode\n     */\n    int BROADCASTING_DISABLE_FORBIDDEN = 4;\n    /**\n     * 5=forbidden for a subscription(group with a topic)\n     */\n    int SUBSCRIPTION_FORBIDDEN         = 5;\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol;\n\nimport java.util.Arrays;\nimport java.util.Map;\nimport java.util.function.Function;\nimport java.util.stream.Collectors;\n\npublic enum LanguageCode {\n    JAVA((byte) 0),\n    CPP((byte) 1),\n    DOTNET((byte) 2),\n    PYTHON((byte) 3),\n    DELPHI((byte) 4),\n    ERLANG((byte) 5),\n    RUBY((byte) 6),\n    OTHER((byte) 7),\n    HTTP((byte) 8),\n    GO((byte) 9),\n    PHP((byte) 10),\n    OMS((byte) 11),\n    RUST((byte) 12),\n    NODE_JS((byte) 13);\n\n    private byte code;\n\n    LanguageCode(byte code) {\n        this.code = code;\n    }\n\n    public static LanguageCode valueOf(byte code) {\n        for (LanguageCode languageCode : LanguageCode.values()) {\n            if (languageCode.getCode() == code) {\n                return languageCode;\n            }\n        }\n        return null;\n    }\n\n    public byte getCode() {\n        return code;\n    }\n    \n    private static final Map<String, LanguageCode> MAP = Arrays.stream(LanguageCode.values()).collect(Collectors.toMap(LanguageCode::name, Function.identity()));\n\n    public static LanguageCode getCode(String language) {\n        return MAP.get(language);\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/MQProtosHelper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol;\n\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerRequestHeader;\n\npublic class MQProtosHelper {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);\n\n    public static boolean registerBrokerToNameServer(final String nsaddr, final String brokerAddr,\n        final long timeoutMillis) {\n        RegisterBrokerRequestHeader requestHeader = new RegisterBrokerRequestHeader();\n        requestHeader.setBrokerAddr(brokerAddr);\n\n        RemotingCommand request =\n            RemotingCommand.createRequestCommand(RequestCode.REGISTER_BROKER, requestHeader);\n\n        try {\n            RemotingCommand response = RemotingHelper.invokeSync(nsaddr, request, timeoutMillis);\n            if (response != null) {\n                return ResponseCode.SUCCESS == response.getCode();\n            }\n        } catch (Exception e) {\n            log.error(\"Failed to register broker\", e);\n        }\n\n        return false;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/NamespaceUtil.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.topic.TopicValidator;\n\npublic class NamespaceUtil {\n    public static final char NAMESPACE_SEPARATOR = '%';\n    public static final String STRING_BLANK = \"\";\n    public static final int RETRY_PREFIX_LENGTH = MixAll.RETRY_GROUP_TOPIC_PREFIX.length();\n    public static final int DLQ_PREFIX_LENGTH = MixAll.DLQ_GROUP_TOPIC_PREFIX.length();\n\n    /**\n     * Unpack namespace from resource, just like:\n     * (1) MQ_INST_XX%Topic_XXX --> Topic_XXX\n     * (2) %RETRY%MQ_INST_XX%GID_XXX --> %RETRY%GID_XXX\n     *\n     * @param resourceWithNamespace, topic/groupId with namespace.\n     * @return topic/groupId without namespace.\n     */\n    public static String withoutNamespace(String resourceWithNamespace) {\n        if (StringUtils.isEmpty(resourceWithNamespace) || isSystemResource(resourceWithNamespace)) {\n            return resourceWithNamespace;\n        }\n\n        StringBuilder stringBuilder = new StringBuilder();\n        if (isRetryTopic(resourceWithNamespace)) {\n            stringBuilder.append(MixAll.RETRY_GROUP_TOPIC_PREFIX);\n        }\n        if (isDLQTopic(resourceWithNamespace)) {\n            stringBuilder.append(MixAll.DLQ_GROUP_TOPIC_PREFIX);\n        }\n\n        String resourceWithoutRetryAndDLQ = withOutRetryAndDLQ(resourceWithNamespace);\n        int index = resourceWithoutRetryAndDLQ.indexOf(NAMESPACE_SEPARATOR);\n        if (index > 0) {\n            String resourceWithoutNamespace = resourceWithoutRetryAndDLQ.substring(index + 1);\n            return stringBuilder.append(resourceWithoutNamespace).toString();\n        }\n\n        return resourceWithNamespace;\n    }\n\n    /**\n     * If resource contains the namespace, unpack namespace from resource, just like:\n     * (1) (MQ_INST_XX1%Topic_XXX1, MQ_INST_XX1) --> Topic_XXX1\n     * (2) (MQ_INST_XX2%Topic_XXX2, NULL) --> MQ_INST_XX2%Topic_XXX2\n     * (3) (%RETRY%MQ_INST_XX1%GID_XXX1, MQ_INST_XX1) --> %RETRY%GID_XXX1\n     * (4) (%RETRY%MQ_INST_XX2%GID_XXX2, MQ_INST_XX3) --> %RETRY%MQ_INST_XX2%GID_XXX2\n     *\n     * @param resourceWithNamespace, topic/groupId with namespace.\n     * @param namespace, namespace to be unpacked.\n     * @return topic/groupId without namespace.\n     */\n    public static String withoutNamespace(String resourceWithNamespace, String namespace) {\n        if (StringUtils.isEmpty(resourceWithNamespace) || StringUtils.isEmpty(namespace)) {\n            return resourceWithNamespace;\n        }\n\n        String resourceWithoutRetryAndDLQ = withOutRetryAndDLQ(resourceWithNamespace);\n        if (resourceWithoutRetryAndDLQ.startsWith(namespace + NAMESPACE_SEPARATOR)) {\n            return withoutNamespace(resourceWithNamespace);\n        }\n\n        return resourceWithNamespace;\n    }\n\n    public static String wrapNamespace(String namespace, String resourceWithOutNamespace) {\n        if (StringUtils.isEmpty(namespace) || StringUtils.isEmpty(resourceWithOutNamespace)) {\n            return resourceWithOutNamespace;\n        }\n\n        if (isSystemResource(resourceWithOutNamespace) || isAlreadyWithNamespace(resourceWithOutNamespace, namespace)) {\n            return resourceWithOutNamespace;\n        }\n\n        String resourceWithoutRetryAndDLQ = withOutRetryAndDLQ(resourceWithOutNamespace);\n        StringBuilder stringBuilder = new StringBuilder();\n\n        if (isRetryTopic(resourceWithOutNamespace)) {\n            stringBuilder.append(MixAll.RETRY_GROUP_TOPIC_PREFIX);\n        }\n\n        if (isDLQTopic(resourceWithOutNamespace)) {\n            stringBuilder.append(MixAll.DLQ_GROUP_TOPIC_PREFIX);\n        }\n\n        return stringBuilder.append(namespace).append(NAMESPACE_SEPARATOR).append(resourceWithoutRetryAndDLQ).toString();\n\n    }\n\n    public static boolean isAlreadyWithNamespace(String resource, String namespace) {\n        if (StringUtils.isEmpty(namespace) || StringUtils.isEmpty(resource) || isSystemResource(resource)) {\n            return false;\n        }\n\n        String resourceWithoutRetryAndDLQ = withOutRetryAndDLQ(resource);\n\n        return resourceWithoutRetryAndDLQ.startsWith(namespace + NAMESPACE_SEPARATOR);\n    }\n\n    public static String wrapNamespaceAndRetry(String namespace, String consumerGroup) {\n        if (StringUtils.isEmpty(consumerGroup)) {\n            return null;\n        }\n\n        return new StringBuilder()\n            .append(MixAll.RETRY_GROUP_TOPIC_PREFIX)\n            .append(wrapNamespace(namespace, consumerGroup))\n            .toString();\n    }\n\n    public static String getNamespaceFromResource(String resource) {\n        if (StringUtils.isEmpty(resource) || isSystemResource(resource)) {\n            return STRING_BLANK;\n        }\n        String resourceWithoutRetryAndDLQ = withOutRetryAndDLQ(resource);\n        int index = resourceWithoutRetryAndDLQ.indexOf(NAMESPACE_SEPARATOR);\n\n        return index > 0 ? resourceWithoutRetryAndDLQ.substring(0, index) : STRING_BLANK;\n    }\n\n    public static String withOutRetryAndDLQ(String originalResource) {\n        if (StringUtils.isEmpty(originalResource)) {\n            return STRING_BLANK;\n        }\n        if (isRetryTopic(originalResource)) {\n            return originalResource.substring(RETRY_PREFIX_LENGTH);\n        }\n\n        if (isDLQTopic(originalResource)) {\n            return originalResource.substring(DLQ_PREFIX_LENGTH);\n        }\n\n        return originalResource;\n    }\n\n    private static boolean isSystemResource(String resource) {\n        if (StringUtils.isEmpty(resource)) {\n            return false;\n        }\n\n        if (TopicValidator.isSystemTopic(resource) || MixAll.isSysConsumerGroup(resource)) {\n            return true;\n        }\n\n        return false;\n    }\n\n    public static boolean isRetryTopic(String resource) {\n        return StringUtils.isNotBlank(resource) && resource.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX);\n    }\n\n    public static boolean isDLQTopic(String resource) {\n        return StringUtils.isNotBlank(resource) && resource.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX);\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol;\n\nimport com.alibaba.fastjson2.annotation.JSONField;\nimport com.google.common.base.Stopwatch;\nimport io.netty.buffer.ByteBuf;\nimport io.netty.buffer.Unpooled;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.CommandCallback;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.Field;\nimport java.lang.reflect.InvocationTargetException;\nimport java.lang.reflect.Modifier;\nimport java.nio.Buffer;\nimport java.nio.ByteBuffer;\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.atomic.AtomicInteger;\n\npublic class RemotingCommand {\n    public static final String SERIALIZE_TYPE_PROPERTY = \"rocketmq.serialize.type\";\n    public static final String SERIALIZE_TYPE_ENV = \"ROCKETMQ_SERIALIZE_TYPE\";\n    public static final String REMOTING_VERSION_KEY = \"rocketmq.remoting.version\";\n    static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME);\n    private static final int RPC_TYPE = 0; // 0, REQUEST_COMMAND\n    private static final int RPC_ONEWAY = 1; // 0, RPC\n    private static final Map<Class<? extends CommandCustomHeader>, Field[]> CLASS_HASH_MAP =\n        new HashMap<>();\n    private static final Map<Class, String> CANONICAL_NAME_CACHE = new HashMap<>();\n    // 1, Oneway\n    // 1, RESPONSE_COMMAND\n    private static final Map<Field, Boolean> NULLABLE_FIELD_CACHE = new HashMap<>();\n    private static final String STRING_CANONICAL_NAME = String.class.getCanonicalName();\n    private static final String DOUBLE_CANONICAL_NAME_1 = Double.class.getCanonicalName();\n    private static final String DOUBLE_CANONICAL_NAME_2 = double.class.getCanonicalName();\n    private static final String INTEGER_CANONICAL_NAME_1 = Integer.class.getCanonicalName();\n    private static final String INTEGER_CANONICAL_NAME_2 = int.class.getCanonicalName();\n    private static final String LONG_CANONICAL_NAME_1 = Long.class.getCanonicalName();\n    private static final String LONG_CANONICAL_NAME_2 = long.class.getCanonicalName();\n    private static final String BOOLEAN_CANONICAL_NAME_1 = Boolean.class.getCanonicalName();\n    private static final String BOOLEAN_CANONICAL_NAME_2 = boolean.class.getCanonicalName();\n    private static final String BOUNDARY_TYPE_CANONICAL_NAME = BoundaryType.class.getCanonicalName();\n    private static volatile int configVersion = -1;\n    private static AtomicInteger requestId = new AtomicInteger(0);\n\n    private static SerializeType serializeTypeConfigInThisServer = SerializeType.JSON;\n\n    static {\n        final String protocol = System.getProperty(SERIALIZE_TYPE_PROPERTY, System.getenv(SERIALIZE_TYPE_ENV));\n        if (!StringUtils.isBlank(protocol)) {\n            try {\n                serializeTypeConfigInThisServer = SerializeType.valueOf(protocol);\n            } catch (IllegalArgumentException e) {\n                throw new RuntimeException(\"parser specified protocol error. protocol=\" + protocol, e);\n            }\n        }\n    }\n\n    private int code;\n    private LanguageCode language = LanguageCode.JAVA;\n    private int version = 0;\n    private int opaque = requestId.getAndIncrement();\n    private int flag = 0;\n    private String remark;\n    private HashMap<String, String> extFields;\n    private transient CommandCustomHeader customHeader;\n    private transient CommandCustomHeader cachedHeader;\n\n    private SerializeType serializeTypeCurrentRPC = serializeTypeConfigInThisServer;\n\n    private transient byte[] body;\n    private boolean suspended;\n    private transient Stopwatch processTimer;\n    private transient List<CommandCallback> callbackList;\n\n    protected RemotingCommand() {\n    }\n\n    public static RemotingCommand createRequestCommand(int code, CommandCustomHeader customHeader) {\n        RemotingCommand cmd = new RemotingCommand();\n        cmd.setCode(code);\n        cmd.customHeader = customHeader;\n        setCmdVersion(cmd);\n        return cmd;\n    }\n\n    public static RemotingCommand createResponseCommandWithHeader(int code, CommandCustomHeader customHeader) {\n        RemotingCommand cmd = new RemotingCommand();\n        cmd.setCode(code);\n        cmd.markResponseType();\n        cmd.customHeader = customHeader;\n        setCmdVersion(cmd);\n        return cmd;\n    }\n\n    protected static void setCmdVersion(RemotingCommand cmd) {\n        if (configVersion >= 0) {\n            cmd.setVersion(configVersion);\n        } else {\n            String v = System.getProperty(REMOTING_VERSION_KEY);\n            if (v != null) {\n                int value = Integer.parseInt(v);\n                cmd.setVersion(value);\n                configVersion = value;\n            }\n        }\n    }\n\n    public static RemotingCommand createResponseCommand(Class<? extends CommandCustomHeader> classHeader) {\n        return createResponseCommand(RemotingSysResponseCode.SYSTEM_ERROR, \"not set any response code\", classHeader);\n    }\n\n    public static RemotingCommand buildErrorResponse(int code, String remark,\n        Class<? extends CommandCustomHeader> classHeader) {\n        final RemotingCommand response = RemotingCommand.createResponseCommand(classHeader);\n        response.setCode(code);\n        response.setRemark(remark);\n        return response;\n    }\n\n    public static RemotingCommand buildErrorResponse(int code, String remark) {\n        return buildErrorResponse(code, remark, null);\n    }\n\n    public static RemotingCommand createResponseCommand(int code, String remark,\n        Class<? extends CommandCustomHeader> classHeader) {\n        RemotingCommand cmd = new RemotingCommand();\n        cmd.markResponseType();\n        cmd.setCode(code);\n        cmd.setRemark(remark);\n        setCmdVersion(cmd);\n\n        if (classHeader != null) {\n            try {\n                CommandCustomHeader objectHeader = classHeader.getDeclaredConstructor().newInstance();\n                cmd.customHeader = objectHeader;\n            } catch (InstantiationException e) {\n                return null;\n            } catch (IllegalAccessException e) {\n                return null;\n            } catch (InvocationTargetException e) {\n                return null;\n            } catch (NoSuchMethodException e) {\n                return null;\n            }\n        }\n\n        return cmd;\n    }\n\n    public static RemotingCommand createResponseCommand(int code, String remark) {\n        return createResponseCommand(code, remark, null);\n    }\n\n    public static RemotingCommand decode(final byte[] array) throws RemotingCommandException {\n        ByteBuffer byteBuffer = ByteBuffer.wrap(array);\n        return decode(byteBuffer);\n    }\n\n    public static RemotingCommand decode(final ByteBuffer byteBuffer) throws RemotingCommandException {\n        return decode(Unpooled.wrappedBuffer(byteBuffer));\n    }\n\n    public static RemotingCommand decode(final ByteBuf byteBuffer) throws RemotingCommandException {\n        int length = byteBuffer.readableBytes();\n        int oriHeaderLen = byteBuffer.readInt();\n        int headerLength = getHeaderLength(oriHeaderLen);\n        if (headerLength > length - 4) {\n            throw new RemotingCommandException(\"decode error, bad header length: \" + headerLength);\n        }\n\n        RemotingCommand cmd = headerDecode(byteBuffer, headerLength, getProtocolType(oriHeaderLen));\n\n        int bodyLength = length - 4 - headerLength;\n        byte[] bodyData = null;\n        if (bodyLength > 0) {\n            bodyData = new byte[bodyLength];\n            byteBuffer.readBytes(bodyData);\n        }\n        cmd.body = bodyData;\n\n        return cmd;\n    }\n\n    public static int getHeaderLength(int length) {\n        return length & 0xFFFFFF;\n    }\n\n    private static RemotingCommand headerDecode(ByteBuf byteBuffer, int len,\n        SerializeType type) throws RemotingCommandException {\n        switch (type) {\n            case JSON:\n                byte[] headerData = new byte[len];\n                byteBuffer.readBytes(headerData);\n                RemotingCommand resultJson = RemotingSerializable.decode(headerData, RemotingCommand.class);\n                resultJson.setSerializeTypeCurrentRPC(type);\n                return resultJson;\n            case ROCKETMQ:\n                RemotingCommand resultRMQ = RocketMQSerializable.rocketMQProtocolDecode(byteBuffer, len);\n                resultRMQ.setSerializeTypeCurrentRPC(type);\n                return resultRMQ;\n            default:\n                break;\n        }\n\n        return null;\n    }\n\n    public static SerializeType getProtocolType(int source) {\n        return SerializeType.valueOf((byte) ((source >> 24) & 0xFF));\n    }\n\n    public static int createNewRequestId() {\n        return requestId.getAndIncrement();\n    }\n\n    public static SerializeType getSerializeTypeConfigInThisServer() {\n        return serializeTypeConfigInThisServer;\n    }\n\n    public static int markProtocolType(int source, SerializeType type) {\n        return (type.getCode() << 24) | (source & 0x00FFFFFF);\n    }\n\n    public void markResponseType() {\n        int bits = 1 << RPC_TYPE;\n        this.flag |= bits;\n    }\n\n    public CommandCustomHeader readCustomHeader() {\n        return customHeader;\n    }\n\n    public void writeCustomHeader(CommandCustomHeader customHeader) {\n        this.customHeader = customHeader;\n    }\n\n    public <T extends CommandCustomHeader> T decodeCommandCustomHeader(\n        Class<T> classHeader) throws RemotingCommandException {\n        return decodeCommandCustomHeader(classHeader, false);\n    }\n\n    public <T extends CommandCustomHeader> T decodeCommandCustomHeader(\n        Class<T> classHeader, boolean isCached) throws RemotingCommandException {\n        if (isCached && cachedHeader != null) {\n            return classHeader.cast(cachedHeader);\n        }\n        cachedHeader = decodeCommandCustomHeaderDirectly(classHeader, true);\n        if (cachedHeader == null) {\n            return null;\n        }\n        return classHeader.cast(cachedHeader);\n    }\n\n    public <T extends CommandCustomHeader> T decodeCommandCustomHeaderDirectly(Class<T> classHeader,\n        boolean useFastEncode) throws RemotingCommandException {\n        T objectHeader;\n        try {\n            objectHeader = classHeader.getDeclaredConstructor().newInstance();\n        } catch (Exception e) {\n            return null;\n        }\n\n        if (this.extFields != null) {\n            if (objectHeader instanceof FastCodesHeader && useFastEncode) {\n                ((FastCodesHeader) objectHeader).decode(this.extFields);\n                objectHeader.checkFields();\n                return objectHeader;\n            }\n\n            Field[] fields = getClazzFields(classHeader);\n            for (Field field : fields) {\n                if (!Modifier.isStatic(field.getModifiers())) {\n                    String fieldName = field.getName();\n                    if (!fieldName.startsWith(\"this\")) {\n                        try {\n                            String value = this.extFields.get(fieldName);\n                            if (null == value) {\n                                if (!isFieldNullable(field)) {\n                                    throw new RemotingCommandException(\"the custom field <\" + fieldName + \"> is null\");\n                                }\n                                continue;\n                            }\n\n                            field.setAccessible(true);\n                            String type = getCanonicalName(field.getType());\n                            Object valueParsed;\n\n                            if (type.equals(STRING_CANONICAL_NAME)) {\n                                valueParsed = value;\n                            } else if (type.equals(INTEGER_CANONICAL_NAME_1) || type.equals(INTEGER_CANONICAL_NAME_2)) {\n                                valueParsed = Integer.parseInt(value);\n                            } else if (type.equals(LONG_CANONICAL_NAME_1) || type.equals(LONG_CANONICAL_NAME_2)) {\n                                valueParsed = Long.parseLong(value);\n                            } else if (type.equals(BOOLEAN_CANONICAL_NAME_1) || type.equals(BOOLEAN_CANONICAL_NAME_2)) {\n                                valueParsed = Boolean.parseBoolean(value);\n                            } else if (type.equals(DOUBLE_CANONICAL_NAME_1) || type.equals(DOUBLE_CANONICAL_NAME_2)) {\n                                valueParsed = Double.parseDouble(value);\n                            } else if (type.equals(BOUNDARY_TYPE_CANONICAL_NAME)) {\n                                valueParsed = BoundaryType.getType(value);\n                            } else {\n                                throw new RemotingCommandException(\"the custom field <\" + fieldName + \"> type is not supported\");\n                            }\n\n                            field.set(objectHeader, valueParsed);\n\n                        } catch (Throwable e) {\n                            log.error(\"Failed field [{}] decoding\", fieldName, e);\n                        }\n                    }\n                }\n            }\n\n            objectHeader.checkFields();\n        }\n\n        return objectHeader;\n    }\n\n    //make it able to test\n    Field[] getClazzFields(Class<? extends CommandCustomHeader> classHeader) {\n        Field[] field = CLASS_HASH_MAP.get(classHeader);\n\n        if (field == null) {\n            Set<Field> fieldList = new HashSet<>();\n            for (Class className = classHeader; className != Object.class; className = className.getSuperclass()) {\n                Field[] fields = className.getDeclaredFields();\n                fieldList.addAll(Arrays.asList(fields));\n            }\n            field = fieldList.toArray(new Field[0]);\n            synchronized (CLASS_HASH_MAP) {\n                CLASS_HASH_MAP.put(classHeader, field);\n            }\n        }\n        return field;\n    }\n\n    private boolean isFieldNullable(Field field) {\n        if (!NULLABLE_FIELD_CACHE.containsKey(field)) {\n            Annotation annotation = field.getAnnotation(CFNotNull.class);\n            synchronized (NULLABLE_FIELD_CACHE) {\n                NULLABLE_FIELD_CACHE.put(field, annotation == null);\n            }\n        }\n        return NULLABLE_FIELD_CACHE.get(field);\n    }\n\n    private String getCanonicalName(Class clazz) {\n        String name = CANONICAL_NAME_CACHE.get(clazz);\n\n        if (name == null) {\n            name = clazz.getCanonicalName();\n            synchronized (CANONICAL_NAME_CACHE) {\n                CANONICAL_NAME_CACHE.put(clazz, name);\n            }\n        }\n        return name;\n    }\n\n    public ByteBuffer encode() {\n        // 1> header length size\n        int length = 4;\n\n        // 2> header data length\n        byte[] headerData = this.headerEncode();\n        length += headerData.length;\n\n        // 3> body data length\n        if (this.body != null) {\n            length += body.length;\n        }\n\n        ByteBuffer result = ByteBuffer.allocate(4 + length);\n\n        // length\n        result.putInt(length);\n\n        // header length\n        result.putInt(markProtocolType(headerData.length, serializeTypeCurrentRPC));\n\n        // header data\n        result.put(headerData);\n\n        // body data;\n        if (this.body != null) {\n            result.put(this.body);\n        }\n\n        result.flip();\n\n        return result;\n    }\n\n    private byte[] headerEncode() {\n        this.makeCustomHeaderToNet();\n        if (SerializeType.ROCKETMQ == serializeTypeCurrentRPC) {\n            return RocketMQSerializable.rocketMQProtocolEncode(this);\n        } else {\n            return RemotingSerializable.encode(this);\n        }\n    }\n\n    public void makeCustomHeaderToNet() {\n        if (this.customHeader != null) {\n            Field[] fields = getClazzFields(customHeader.getClass());\n            if (null == this.extFields) {\n                this.extFields = new HashMap<>();\n            }\n\n            for (Field field : fields) {\n                if (!Modifier.isStatic(field.getModifiers())) {\n                    String name = field.getName();\n                    if (!name.startsWith(\"this\")) {\n                        Object value = null;\n                        try {\n                            field.setAccessible(true);\n                            value = field.get(this.customHeader);\n                        } catch (Exception e) {\n                            log.error(\"Failed to access field [{}]\", name, e);\n                        }\n\n                        if (value != null) {\n                            this.extFields.put(name, value.toString());\n                        }\n                    }\n                }\n            }\n        }\n    }\n\n    public void fastEncodeHeader(ByteBuf out) {\n        int bodySize = this.body != null ? this.body.length : 0;\n        int beginIndex = out.writerIndex();\n        // skip 8 bytes\n        out.writeLong(0);\n        int headerSize;\n        if (SerializeType.ROCKETMQ == serializeTypeCurrentRPC) {\n            if (customHeader != null && !(customHeader instanceof FastCodesHeader)) {\n                this.makeCustomHeaderToNet();\n            }\n            headerSize = RocketMQSerializable.rocketMQProtocolEncode(this, out);\n        } else {\n            this.makeCustomHeaderToNet();\n            byte[] header = RemotingSerializable.encode(this);\n            headerSize = header.length;\n            out.writeBytes(header);\n        }\n        out.setInt(beginIndex, 4 + headerSize + bodySize);\n        out.setInt(beginIndex + 4, markProtocolType(headerSize, serializeTypeCurrentRPC));\n    }\n\n    public ByteBuffer encodeHeader() {\n        return encodeHeader(this.body != null ? this.body.length : 0);\n    }\n\n    public ByteBuffer encodeHeader(final int bodyLength) {\n        // 1> header length size\n        int length = 4;\n\n        // 2> header data length\n        byte[] headerData;\n        headerData = this.headerEncode();\n\n        length += headerData.length;\n\n        // 3> body data length\n        length += bodyLength;\n\n        ByteBuffer result = ByteBuffer.allocate(4 + length - bodyLength);\n\n        // length\n        result.putInt(length);\n\n        // header length\n        result.putInt(markProtocolType(headerData.length, serializeTypeCurrentRPC));\n\n        // header data\n        result.put(headerData);\n\n        ((Buffer) result).flip();\n\n        return result;\n    }\n\n    public void markOnewayRPC() {\n        int bits = 1 << RPC_ONEWAY;\n        this.flag |= bits;\n    }\n\n    @JSONField(serialize = false)\n    public boolean isOnewayRPC() {\n        int bits = 1 << RPC_ONEWAY;\n        return (this.flag & bits) == bits;\n    }\n\n    public int getCode() {\n        return code;\n    }\n\n    public void setCode(int code) {\n        this.code = code;\n    }\n\n    @JSONField(serialize = false)\n    public RemotingCommandType getType() {\n        if (this.isResponseType()) {\n            return RemotingCommandType.RESPONSE_COMMAND;\n        }\n\n        return RemotingCommandType.REQUEST_COMMAND;\n    }\n\n    @JSONField(serialize = false)\n    public boolean isResponseType() {\n        int bits = 1 << RPC_TYPE;\n        return (this.flag & bits) == bits;\n    }\n\n    public LanguageCode getLanguage() {\n        return language;\n    }\n\n    public void setLanguage(LanguageCode language) {\n        this.language = language;\n    }\n\n    public int getVersion() {\n        return version;\n    }\n\n    public void setVersion(int version) {\n        this.version = version;\n    }\n\n    public int getOpaque() {\n        return opaque;\n    }\n\n    public void setOpaque(int opaque) {\n        this.opaque = opaque;\n    }\n\n    public int getFlag() {\n        return flag;\n    }\n\n    public void setFlag(int flag) {\n        this.flag = flag;\n    }\n\n    public String getRemark() {\n        return remark;\n    }\n\n    public void setRemark(String remark) {\n        this.remark = remark;\n    }\n\n    public byte[] getBody() {\n        return body;\n    }\n\n    public void setBody(byte[] body) {\n        this.body = body;\n    }\n\n    @JSONField(serialize = false)\n    public boolean isSuspended() {\n        return suspended;\n    }\n\n    @JSONField(serialize = false)\n    public void setSuspended(boolean suspended) {\n        this.suspended = suspended;\n    }\n\n    public HashMap<String, String> getExtFields() {\n        return extFields;\n    }\n\n    public void setExtFields(HashMap<String, String> extFields) {\n        this.extFields = extFields;\n    }\n\n    public void addExtField(String key, String value) {\n        if (null == extFields) {\n            extFields = new HashMap<>(256);\n        }\n        extFields.put(key, value);\n    }\n\n    public void addExtFieldIfNotExist(String key, String value) {\n        extFields.putIfAbsent(key, value);\n    }\n\n    @Override\n    public String toString() {\n        return \"RemotingCommand [code=\" + code + \", language=\" + language + \", version=\" + version + \", opaque=\" + opaque + \", flag(B)=\"\n            + Integer.toBinaryString(flag) + \", remark=\" + remark + \", extFields=\" + extFields + \", serializeTypeCurrentRPC=\"\n            + serializeTypeCurrentRPC + \"]\";\n    }\n\n    public SerializeType getSerializeTypeCurrentRPC() {\n        return serializeTypeCurrentRPC;\n    }\n\n    public void setSerializeTypeCurrentRPC(SerializeType serializeTypeCurrentRPC) {\n        this.serializeTypeCurrentRPC = serializeTypeCurrentRPC;\n    }\n\n    public Stopwatch getProcessTimer() {\n        return processTimer;\n    }\n\n    public void setProcessTimer(Stopwatch processTimer) {\n        this.processTimer = processTimer;\n    }\n\n    public List<CommandCallback> getCallbackList() {\n        return callbackList;\n    }\n\n    public void setCallbackList(List<CommandCallback> callbackList) {\n        this.callbackList = callbackList;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommandType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol;\n\npublic enum RemotingCommandType {\n    REQUEST_COMMAND,\n    RESPONSE_COMMAND;\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingSerializable.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.alibaba.fastjson2.JSONWriter;\n\nimport java.nio.charset.Charset;\nimport java.nio.charset.StandardCharsets;\nimport java.util.List;\n\npublic abstract class RemotingSerializable {\n    private final static Charset CHARSET_UTF8 = StandardCharsets.UTF_8;\n\n    public static byte[] encode(final Object obj) {\n        if (obj == null) {\n            return null;\n        }\n        return JSON.toJSONBytes(obj, CHARSET_UTF8);\n    }\n\n    public static String toJson(final Object obj, boolean prettyFormat) {\n        if (prettyFormat) {\n            return JSON.toJSONString(obj, JSONWriter.Feature.PrettyFormat);\n        }\n        return JSON.toJSONString(obj);\n    }\n\n    public static <T> T decode(final byte[] data, Class<T> classOfT) {\n        if (data == null) {\n            return null;\n        }\n        return JSON.parseObject(data, classOfT);\n    }\n\n    public static <T> List<T> decodeList(final byte[] data, Class<T> classOfT) {\n        if (data == null) {\n            return null;\n        }\n        return JSON.parseArray(data, 0, data.length, CHARSET_UTF8, classOfT);\n    }\n\n    public static <T> T fromJson(String json, Class<T> classOfT) {\n        return JSON.parseObject(json, classOfT);\n    }\n\n    public byte[] encode() {\n        return JSON.toJSONBytes(this, CHARSET_UTF8);\n    }\n\n    /**\n     * Allow call-site to apply specific features according to their requirements.\n     *\n     * @param features Features to apply\n     * @return serialized data.\n     */\n    public byte[] encode(JSONWriter.Feature... features) {\n        return JSON.toJSONBytes(this, CHARSET_UTF8, features);\n    }\n\n    public String toJson() {\n        return toJson(false);\n    }\n\n    public String toJson(final boolean prettyFormat) {\n        return toJson(this, prettyFormat);\n    }\n}"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingSysResponseCode.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol;\n\npublic class RemotingSysResponseCode {\n\n    public static final int SUCCESS = 0;\n\n    public static final int SYSTEM_ERROR = 1;\n\n    public static final int SYSTEM_BUSY = 2;\n\n    public static final int REQUEST_CODE_NOT_SUPPORTED = 3;\n\n    public static final int TRANSACTION_FAILED = 4;\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol;\n\npublic class RequestCode {\n\n    public static final int SEND_MESSAGE = 10;\n\n    public static final int PULL_MESSAGE = 11;\n\n    public static final int QUERY_MESSAGE = 12;\n    public static final int QUERY_BROKER_OFFSET = 13;\n    public static final int QUERY_CONSUMER_OFFSET = 14;\n    public static final int UPDATE_CONSUMER_OFFSET = 15;\n    public static final int UPDATE_AND_CREATE_TOPIC = 17;\n    public static final int UPDATE_AND_CREATE_TOPIC_LIST = 18;\n    public static final int GET_ALL_TOPIC_CONFIG = 21;\n    public static final int GET_TOPIC_CONFIG_LIST = 22;\n\n    public static final int GET_TOPIC_NAME_LIST = 23;\n\n    public static final int UPDATE_BROKER_CONFIG = 25;\n\n    public static final int GET_BROKER_CONFIG = 26;\n\n    public static final int TRIGGER_DELETE_FILES = 27;\n\n    public static final int GET_BROKER_RUNTIME_INFO = 28;\n    public static final int SEARCH_OFFSET_BY_TIMESTAMP = 29;\n    public static final int GET_MAX_OFFSET = 30;\n    public static final int GET_MIN_OFFSET = 31;\n\n    public static final int GET_EARLIEST_MSG_STORETIME = 32;\n\n    public static final int VIEW_MESSAGE_BY_ID = 33;\n\n    public static final int HEART_BEAT = 34;\n\n    public static final int UNREGISTER_CLIENT = 35;\n\n    public static final int CONSUMER_SEND_MSG_BACK = 36;\n\n    public static final int END_TRANSACTION = 37;\n    public static final int GET_CONSUMER_LIST_BY_GROUP = 38;\n\n    public static final int CHECK_TRANSACTION_STATE = 39;\n\n    public static final int NOTIFY_CONSUMER_IDS_CHANGED = 40;\n\n    public static final int LOCK_BATCH_MQ = 41;\n\n    public static final int UNLOCK_BATCH_MQ = 42;\n    public static final int GET_ALL_CONSUMER_OFFSET = 43;\n\n    public static final int GET_ALL_DELAY_OFFSET = 45;\n\n    public static final int CHECK_CLIENT_CONFIG = 46;\n\n    public static final int GET_CLIENT_CONFIG = 47;\n\n    public static final int GET_TIMER_CHECK_POINT = 60;\n\n    public static final int GET_TIMER_METRICS = 61;\n\n    public static final int POP_MESSAGE = 200050;\n    public static final int ACK_MESSAGE = 200051;\n    public static final int BATCH_ACK_MESSAGE = 200151;\n    public static final int PEEK_MESSAGE = 200052;\n    public static final int CHANGE_MESSAGE_INVISIBLETIME = 200053;\n    public static final int NOTIFICATION = 200054;\n    public static final int POLLING_INFO = 200055;\n    public static final int POP_ROLLBACK = 200056;\n\n    public static final int POP_LITE_MESSAGE = 200070;\n    public static final int LITE_SUBSCRIPTION_CTL = 200071;\n    public static final int ACK_LITE_MESSAGE = 200072;\n    public static final int NOTIFY_UNSUBSCRIBE_LITE = 200073;\n    // lite admin api\n    public static final int GET_BROKER_LITE_INFO = 200074;\n    public static final int GET_PARENT_TOPIC_INFO = 200075;\n    public static final int GET_LITE_TOPIC_INFO = 200076;\n    public static final int GET_LITE_CLIENT_INFO = 200077;\n    public static final int GET_LITE_GROUP_INFO = 200078;\n    public static final int TRIGGER_LITE_DISPATCH = 200079;\n\n    public static final int PUT_KV_CONFIG = 100;\n\n    public static final int GET_KV_CONFIG = 101;\n\n    public static final int DELETE_KV_CONFIG = 102;\n\n    public static final int REGISTER_BROKER = 103;\n\n    public static final int UNREGISTER_BROKER = 104;\n    public static final int GET_ROUTEINFO_BY_TOPIC = 105;\n\n    public static final int GET_BROKER_CLUSTER_INFO = 106;\n    public static final int UPDATE_AND_CREATE_SUBSCRIPTIONGROUP = 200;\n    public static final int GET_ALL_SUBSCRIPTIONGROUP_CONFIG = 201;\n    public static final int GET_TOPIC_STATS_INFO = 202;\n    public static final int GET_CONSUMER_CONNECTION_LIST = 203;\n    public static final int GET_PRODUCER_CONNECTION_LIST = 204;\n    public static final int WIPE_WRITE_PERM_OF_BROKER = 205;\n\n    public static final int GET_ALL_TOPIC_LIST_FROM_NAMESERVER = 206;\n\n    public static final int DELETE_SUBSCRIPTIONGROUP = 207;\n    public static final int GET_CONSUME_STATS = 208;\n\n    public static final int SUSPEND_CONSUMER = 209;\n\n    public static final int RESUME_CONSUMER = 210;\n    public static final int RESET_CONSUMER_OFFSET_IN_CONSUMER = 211;\n    public static final int RESET_CONSUMER_OFFSET_IN_BROKER = 212;\n\n    public static final int ADJUST_CONSUMER_THREAD_POOL = 213;\n\n    public static final int WHO_CONSUME_THE_MESSAGE = 214;\n\n    public static final int DELETE_TOPIC_IN_BROKER = 215;\n\n    public static final int DELETE_TOPIC_IN_NAMESRV = 216;\n    public static final int REGISTER_TOPIC_IN_NAMESRV = 217;\n    public static final int GET_KVLIST_BY_NAMESPACE = 219;\n\n    public static final int RESET_CONSUMER_CLIENT_OFFSET = 220;\n\n    public static final int GET_CONSUMER_STATUS_FROM_CLIENT = 221;\n\n    public static final int INVOKE_BROKER_TO_RESET_OFFSET = 222;\n\n    public static final int INVOKE_BROKER_TO_GET_CONSUMER_STATUS = 223;\n\n    public static final int QUERY_TOPIC_CONSUME_BY_WHO = 300;\n\n    public static final int GET_TOPICS_BY_CLUSTER = 224;\n\n    public static final int UPDATE_AND_CREATE_SUBSCRIPTIONGROUP_LIST = 225;\n\n    public static final int QUERY_TOPICS_BY_CONSUMER = 343;\n    public static final int QUERY_SUBSCRIPTION_BY_CONSUMER = 345;\n\n    public static final int REGISTER_FILTER_SERVER = 301;\n    public static final int REGISTER_MESSAGE_FILTER_CLASS = 302;\n\n    public static final int QUERY_CONSUME_TIME_SPAN = 303;\n\n    public static final int GET_SYSTEM_TOPIC_LIST_FROM_NS = 304;\n    public static final int GET_SYSTEM_TOPIC_LIST_FROM_BROKER = 305;\n\n    public static final int CLEAN_EXPIRED_CONSUMEQUEUE = 306;\n\n    public static final int GET_CONSUMER_RUNNING_INFO = 307;\n\n    public static final int QUERY_CORRECTION_OFFSET = 308;\n    public static final int CONSUME_MESSAGE_DIRECTLY = 309;\n\n    public static final int SEND_MESSAGE_V2 = 310;\n\n    public static final int GET_UNIT_TOPIC_LIST = 311;\n\n    public static final int GET_HAS_UNIT_SUB_TOPIC_LIST = 312;\n\n    public static final int GET_HAS_UNIT_SUB_UNUNIT_TOPIC_LIST = 313;\n\n    public static final int CLONE_GROUP_OFFSET = 314;\n\n    public static final int VIEW_BROKER_STATS_DATA = 315;\n\n    public static final int CLEAN_UNUSED_TOPIC = 316;\n\n    public static final int GET_BROKER_CONSUME_STATS = 317;\n\n    /**\n     * update the config of name server\n     */\n    public static final int UPDATE_NAMESRV_CONFIG = 318;\n\n    /**\n     * get config from name server\n     */\n    public static final int GET_NAMESRV_CONFIG = 319;\n\n    public static final int SEND_BATCH_MESSAGE = 320;\n\n    public static final int QUERY_CONSUME_QUEUE = 321;\n\n    public static final int QUERY_DATA_VERSION = 322;\n\n    /**\n     * resume logic of checking half messages that have been put in TRANS_CHECK_MAXTIME_TOPIC before\n     */\n    public static final int RESUME_CHECK_HALF_MESSAGE = 323;\n\n    public static final int SEND_REPLY_MESSAGE = 324;\n\n    public static final int SEND_REPLY_MESSAGE_V2 = 325;\n\n    public static final int PUSH_REPLY_MESSAGE_TO_CLIENT = 326;\n\n    public static final int ADD_WRITE_PERM_OF_BROKER = 327;\n\n    public static final int GET_ALL_PRODUCER_INFO = 328;\n\n    public static final int DELETE_EXPIRED_COMMITLOG = 329;\n\n    public static final int GET_TOPIC_CONFIG = 351;\n\n    public static final int GET_SUBSCRIPTIONGROUP_CONFIG = 352;\n    public static final int UPDATE_AND_GET_GROUP_FORBIDDEN = 353;\n    public static final int CHECK_ROCKSDB_CQ_WRITE_PROGRESS = 354;\n    public static final int EXPORT_ROCKSDB_CONFIG_TO_JSON = 355;\n\n    public static final int LITE_PULL_MESSAGE = 361;\n    public static final int RECALL_MESSAGE = 370;\n\n    public static final int QUERY_ASSIGNMENT = 400;\n    public static final int SET_MESSAGE_REQUEST_MODE = 401;\n    public static final int GET_ALL_MESSAGE_REQUEST_MODE = 402;\n\n    public static final int UPDATE_AND_CREATE_STATIC_TOPIC = 513;\n\n    public static final int GET_BROKER_MEMBER_GROUP = 901;\n\n    public static final int ADD_BROKER = 902;\n\n    public static final int REMOVE_BROKER = 903;\n\n    public static final int BROKER_HEARTBEAT = 904;\n\n    public static final int NOTIFY_MIN_BROKER_ID_CHANGE = 905;\n\n    public static final int EXCHANGE_BROKER_HA_INFO = 906;\n\n    public static final int GET_BROKER_HA_STATUS = 907;\n\n    public static final int RESET_MASTER_FLUSH_OFFSET = 908;\n\n    /**\n     * Controller code\n     */\n    public static final int CONTROLLER_ALTER_SYNC_STATE_SET = 1001;\n\n    public static final int CONTROLLER_ELECT_MASTER = 1002;\n\n    public static final int CONTROLLER_REGISTER_BROKER = 1003;\n\n    public static final int CONTROLLER_GET_REPLICA_INFO = 1004;\n\n    public static final int CONTROLLER_GET_METADATA_INFO = 1005;\n\n    public static final int CONTROLLER_GET_SYNC_STATE_DATA = 1006;\n\n    public static final int GET_BROKER_EPOCH_CACHE = 1007;\n\n    public static final int NOTIFY_BROKER_ROLE_CHANGED = 1008;\n\n    /**\n     * update the config of controller\n     */\n    public static final int UPDATE_CONTROLLER_CONFIG = 1009;\n\n    /**\n     * get config from controller\n     */\n    public static final int GET_CONTROLLER_CONFIG = 1010;\n\n    /**\n     * clean broker data\n     */\n    public static final int CLEAN_BROKER_DATA = 1011;\n    public static final int CONTROLLER_GET_NEXT_BROKER_ID = 1012;\n\n    public static final int CONTROLLER_APPLY_BROKER_ID = 1013;\n    public static final short BROKER_CLOSE_CHANNEL_REQUEST = 1014;\n    public static final short CHECK_NOT_ACTIVE_BROKER_REQUEST = 1015;\n    public static final short GET_BROKER_LIVE_INFO_REQUEST = 1016;\n    public static final short GET_SYNC_STATE_DATA_REQUEST = 1017;\n    public static final short RAFT_BROKER_HEART_BEAT_EVENT_REQUEST = 1018;\n\n    public static final int UPDATE_COLD_DATA_FLOW_CTR_CONFIG = 2001;\n    public static final int REMOVE_COLD_DATA_FLOW_CTR_CONFIG = 2002;\n    public static final int GET_COLD_DATA_FLOW_CTR_INFO = 2003;\n    public static final int SET_COMMITLOG_READ_MODE = 2004;\n\n    public static final int AUTH_CREATE_USER = 3001;\n    public static final int AUTH_UPDATE_USER = 3002;\n    public static final int AUTH_DELETE_USER = 3003;\n    public static final int AUTH_GET_USER = 3004;\n    public static final int AUTH_LIST_USER = 3005;\n\n    public static final int AUTH_CREATE_ACL = 3006;\n    public static final int AUTH_UPDATE_ACL = 3007;\n    public static final int AUTH_DELETE_ACL = 3008;\n    public static final int AUTH_GET_ACL = 3009;\n    public static final int AUTH_LIST_ACL = 3010;\n\n    public static final int SWITCH_TIMER_ENGINE = 5001;\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestHeaderRegistry.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol;\n\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.Set;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.reflections.Reflections;\nimport org.reflections.scanners.SubTypesScanner;\nimport org.reflections.util.ClasspathHelper;\nimport org.reflections.util.ConfigurationBuilder;\n\npublic class RequestHeaderRegistry {\n\n    private static final String PACKAGE_NAME = \"org.apache.rocketmq.remoting.protocol.header\";\n\n    private final Map<Integer, Class<? extends CommandCustomHeader>> requestHeaderMap = new HashMap<>();\n\n    public static RequestHeaderRegistry getInstance() {\n        return RequestHeaderRegistryHolder.INSTANCE;\n    }\n\n    public void initialize() {\n        Reflections reflections = new Reflections(new ConfigurationBuilder()\n            .setUrls(ClasspathHelper.forPackage(PACKAGE_NAME))\n            .setScanners(new SubTypesScanner(false)));\n\n        Set<Class<? extends CommandCustomHeader>> classes = reflections.getSubTypesOf(CommandCustomHeader.class);\n\n        classes.forEach(this::registerHeader);\n    }\n\n    public Class<? extends CommandCustomHeader> getRequestHeader(int requestCode) {\n        return this.requestHeaderMap.get(requestCode);\n    }\n\n    private void registerHeader(Class<? extends CommandCustomHeader> clazz) {\n        if (!clazz.isAnnotationPresent(RocketMQAction.class)) {\n            return;\n        }\n        RocketMQAction action = clazz.getAnnotation(RocketMQAction.class);\n        this.requestHeaderMap.putIfAbsent(action.value(), clazz);\n    }\n\n    private static class RequestHeaderRegistryHolder {\n        private static final RequestHeaderRegistry INSTANCE = new RequestHeaderRegistry();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestSource.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol;\n\npublic enum RequestSource {\n\n    SDK(-1),\n    PROXY_FOR_ORDER(0),\n    PROXY_FOR_BROADCAST(1),\n    PROXY_FOR_STREAM(2);\n\n    public static final String SYSTEM_PROPERTY_KEY = \"rocketmq.requestSource\";\n    private final int value;\n\n    RequestSource(int value) {\n        this.value = value;\n    }\n\n    public int getValue() {\n        return value;\n    }\n\n    public static boolean isValid(Integer value) {\n        return null != value && value >= -1 && value < RequestSource.values().length - 1;\n    }\n\n    public static RequestSource parseInteger(Integer value) {\n        if (isValid(value)) {\n            return RequestSource.values()[value + 1];\n        }\n        return SDK;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol;\n\npublic enum RequestType {\n    STREAM((byte) 0);\n\n    private final byte code;\n\n    RequestType(byte code) {\n        this.code = code;\n    }\n\n    public static RequestType valueOf(byte code) {\n        for (RequestType requestType : RequestType.values()) {\n            if (requestType.getCode() == code) {\n                return requestType;\n            }\n        }\n        return null;\n    }\n\n    public byte getCode() {\n        return code;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol;\n\npublic class ResponseCode extends RemotingSysResponseCode {\n\n    public static final int FLUSH_DISK_TIMEOUT = 10;\n\n    public static final int SLAVE_NOT_AVAILABLE = 11;\n\n    public static final int FLUSH_SLAVE_TIMEOUT = 12;\n\n    public static final int MESSAGE_ILLEGAL = 13;\n\n    public static final int SERVICE_NOT_AVAILABLE = 14;\n\n    public static final int VERSION_NOT_SUPPORTED = 15;\n\n    public static final int NO_PERMISSION = 16;\n\n    public static final int TOPIC_NOT_EXIST = 17;\n    public static final int TOPIC_EXIST_ALREADY = 18;\n    public static final int PULL_NOT_FOUND = 19;\n\n    public static final int PULL_RETRY_IMMEDIATELY = 20;\n\n    public static final int PULL_OFFSET_MOVED = 21;\n\n    public static final int QUERY_NOT_FOUND = 22;\n\n    public static final int SUBSCRIPTION_PARSE_FAILED = 23;\n\n    public static final int SUBSCRIPTION_NOT_EXIST = 24;\n\n    public static final int SUBSCRIPTION_NOT_LATEST = 25;\n\n    public static final int SUBSCRIPTION_GROUP_NOT_EXIST = 26;\n\n    public static final int FILTER_DATA_NOT_EXIST = 27;\n\n    public static final int FILTER_DATA_NOT_LATEST = 28;\n\n    public static final int INVALID_PARAMETER = 29;\n\n    public static final int TRANSACTION_SHOULD_COMMIT = 200;\n\n    public static final int TRANSACTION_SHOULD_ROLLBACK = 201;\n\n    public static final int TRANSACTION_STATE_UNKNOW = 202;\n\n    public static final int TRANSACTION_STATE_GROUP_WRONG = 203;\n    public static final int NO_BUYER_ID = 204;\n\n    public static final int NOT_IN_CURRENT_UNIT = 205;\n\n    public static final int CONSUMER_NOT_ONLINE = 206;\n\n    public static final int CONSUME_MSG_TIMEOUT = 207;\n\n    public static final int NO_MESSAGE = 208;\n\n    public static final int POLLING_FULL = 209;\n\n    public static final int POLLING_TIMEOUT = 210;\n\n    public static final int BROKER_NOT_EXIST = 211;\n\n    public static final int BROKER_DISPATCH_NOT_COMPLETE = 212;\n\n    public static final int BROADCAST_CONSUMPTION = 213;\n\n    public static final int FLOW_CONTROL = 215;\n\n    public static final int NOT_LEADER_FOR_QUEUE = 501;\n\n    public static final int ILLEGAL_OPERATION = 604;\n\n    public static final int RPC_UNKNOWN = -1000;\n    public static final int RPC_ADDR_IS_NULL = -1002;\n    public static final int RPC_SEND_TO_CHANNEL_FAILED = -1004;\n    public static final int RPC_TIME_OUT = -1006;\n\n    public static final int GO_AWAY = 1500;\n\n    /**\n     * Controller response code\n     */\n    public static final int CONTROLLER_FENCED_MASTER_EPOCH = 2000;\n    public static final int CONTROLLER_FENCED_SYNC_STATE_SET_EPOCH = 2001;\n    public static final int CONTROLLER_INVALID_MASTER = 2002;\n    public static final int CONTROLLER_INVALID_REPLICAS = 2003;\n    public static final int CONTROLLER_MASTER_NOT_AVAILABLE = 2004;\n    public static final int CONTROLLER_INVALID_REQUEST = 2005;\n    public static final int CONTROLLER_BROKER_NOT_ALIVE = 2006;\n    public static final int CONTROLLER_NOT_LEADER = 2007;\n\n    public static final int CONTROLLER_BROKER_METADATA_NOT_EXIST = 2008;\n\n    public static final int CONTROLLER_INVALID_CLEAN_BROKER_METADATA = 2009;\n\n    public static final int CONTROLLER_BROKER_NEED_TO_BE_REGISTERED = 2010;\n\n    public static final int CONTROLLER_MASTER_STILL_EXIST = 2011;\n\n    public static final int CONTROLLER_ELECT_MASTER_FAILED = 2012;\n    \n    public static final int CONTROLLER_ALTER_SYNC_STATE_SET_FAILED = 2013;\n\n    public static final int CONTROLLER_BROKER_ID_INVALID = 2014;\n\n    public static final int CONTROLLER_JRAFT_INTERNAL_ERROR = 2015;\n\n    public static final int CONTROLLER_BROKER_LIVE_INFO_NOT_EXISTS = 2016;\n\n    public static final int LMQ_QUOTA_EXCEEDED = 2017;\n\n    public static final int LITE_SUBSCRIPTION_QUOTA_EXCEEDED = 2018;\n\n    public static final int USER_NOT_EXIST = 3001;\n\n    public static final int POLICY_NOT_EXIST = 3002;\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RocketMQSerializable.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol;\n\nimport java.nio.ByteBuffer;\nimport java.nio.charset.Charset;\nimport java.nio.charset.StandardCharsets;\nimport java.util.HashMap;\nimport java.util.Iterator;\nimport java.util.Map;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\nimport io.netty.buffer.ByteBuf;\n\npublic class RocketMQSerializable {\n    private static final Charset CHARSET_UTF8 = StandardCharsets.UTF_8;\n\n    public static void writeStr(ByteBuf buf, boolean useShortLength, String str) {\n        int lenIndex = buf.writerIndex();\n        if (useShortLength) {\n            buf.writeShort(0);\n        } else {\n            buf.writeInt(0);\n        }\n        int len = buf.writeCharSequence(str, StandardCharsets.UTF_8);\n        if (useShortLength) {\n            buf.setShort(lenIndex, len);\n        } else {\n            buf.setInt(lenIndex, len);\n        }\n    }\n\n    private static String readStr(ByteBuf buf, boolean useShortLength, int limit) throws RemotingCommandException {\n        int len = useShortLength ? buf.readShort() : buf.readInt();\n        if (len == 0) {\n            return null;\n        }\n        if (len > limit) {\n            throw new RemotingCommandException(\"string length exceed limit:\" + limit);\n        }\n        CharSequence cs = buf.readCharSequence(len, StandardCharsets.UTF_8);\n        return cs == null ? null : cs.toString();\n    }\n\n    public static int rocketMQProtocolEncode(RemotingCommand cmd, ByteBuf out) {\n        int beginIndex = out.writerIndex();\n        // int code(~32767)\n        out.writeShort(cmd.getCode());\n        // LanguageCode language\n        out.writeByte(cmd.getLanguage().getCode());\n        // int version(~32767)\n        out.writeShort(cmd.getVersion());\n        // int opaque\n        out.writeInt(cmd.getOpaque());\n        // int flag\n        out.writeInt(cmd.getFlag());\n        // String remark\n        String remark = cmd.getRemark();\n        if (remark != null && !remark.isEmpty()) {\n            writeStr(out, false, remark);\n        } else {\n            out.writeInt(0);\n        }\n\n        int mapLenIndex = out.writerIndex();\n        out.writeInt(0);\n        if (cmd.readCustomHeader() instanceof FastCodesHeader) {\n            ((FastCodesHeader) cmd.readCustomHeader()).encode(out);\n        }\n        HashMap<String, String> map = cmd.getExtFields();\n        if (map != null && !map.isEmpty()) {\n            map.forEach((k, v) -> {\n                if (k != null && v != null) {\n                    writeStr(out, true, k);\n                    writeStr(out, false, v);\n                }\n            });\n        }\n        out.setInt(mapLenIndex, out.writerIndex() - mapLenIndex - 4);\n        return out.writerIndex() - beginIndex;\n    }\n\n    public static byte[] rocketMQProtocolEncode(RemotingCommand cmd) {\n        // String remark\n        byte[] remarkBytes = null;\n        int remarkLen = 0;\n        if (cmd.getRemark() != null && cmd.getRemark().length() > 0) {\n            remarkBytes = cmd.getRemark().getBytes(CHARSET_UTF8);\n            remarkLen = remarkBytes.length;\n        }\n\n        // HashMap<String, String> extFields\n        byte[] extFieldsBytes = null;\n        int extLen = 0;\n        if (cmd.getExtFields() != null && !cmd.getExtFields().isEmpty()) {\n            extFieldsBytes = mapSerialize(cmd.getExtFields());\n            extLen = extFieldsBytes.length;\n        }\n\n        int totalLen = calTotalLen(remarkLen, extLen);\n\n        ByteBuffer headerBuffer = ByteBuffer.allocate(totalLen);\n        // int code(~32767)\n        headerBuffer.putShort((short) cmd.getCode());\n        // LanguageCode language\n        headerBuffer.put(cmd.getLanguage().getCode());\n        // int version(~32767)\n        headerBuffer.putShort((short) cmd.getVersion());\n        // int opaque\n        headerBuffer.putInt(cmd.getOpaque());\n        // int flag\n        headerBuffer.putInt(cmd.getFlag());\n        // String remark\n        if (remarkBytes != null) {\n            headerBuffer.putInt(remarkBytes.length);\n            headerBuffer.put(remarkBytes);\n        } else {\n            headerBuffer.putInt(0);\n        }\n        // HashMap<String, String> extFields;\n        if (extFieldsBytes != null) {\n            headerBuffer.putInt(extFieldsBytes.length);\n            headerBuffer.put(extFieldsBytes);\n        } else {\n            headerBuffer.putInt(0);\n        }\n\n        return headerBuffer.array();\n    }\n\n    public static byte[] mapSerialize(HashMap<String, String> map) {\n        // keySize+key+valSize+val\n        if (null == map || map.isEmpty())\n            return null;\n\n        int totalLength = 0;\n        int kvLength;\n        Iterator<Map.Entry<String, String>> it = map.entrySet().iterator();\n        while (it.hasNext()) {\n            Map.Entry<String, String> entry = it.next();\n            if (entry.getKey() != null && entry.getValue() != null) {\n                kvLength =\n                    // keySize + Key\n                    2 + entry.getKey().getBytes(CHARSET_UTF8).length\n                        // valSize + val\n                        + 4 + entry.getValue().getBytes(CHARSET_UTF8).length;\n                totalLength += kvLength;\n            }\n        }\n\n        ByteBuffer content = ByteBuffer.allocate(totalLength);\n        byte[] key;\n        byte[] val;\n        it = map.entrySet().iterator();\n        while (it.hasNext()) {\n            Map.Entry<String, String> entry = it.next();\n            if (entry.getKey() != null && entry.getValue() != null) {\n                key = entry.getKey().getBytes(CHARSET_UTF8);\n                val = entry.getValue().getBytes(CHARSET_UTF8);\n\n                content.putShort((short) key.length);\n                content.put(key);\n\n                content.putInt(val.length);\n                content.put(val);\n            }\n        }\n\n        return content.array();\n    }\n\n    private static int calTotalLen(int remark, int ext) {\n        // int code(~32767)\n        int length = 2\n            // LanguageCode language\n            + 1\n            // int version(~32767)\n            + 2\n            // int opaque\n            + 4\n            // int flag\n            + 4\n            // String remark\n            + 4 + remark\n            // HashMap<String, String> extFields\n            + 4 + ext;\n\n        return length;\n    }\n\n    public static RemotingCommand rocketMQProtocolDecode(final ByteBuf headerBuffer,\n        int headerLen) throws RemotingCommandException {\n        RemotingCommand cmd = new RemotingCommand();\n        // int code(~32767)\n        cmd.setCode(headerBuffer.readShort());\n        // LanguageCode language\n        cmd.setLanguage(LanguageCode.valueOf(headerBuffer.readByte()));\n        // int version(~32767)\n        cmd.setVersion(headerBuffer.readShort());\n        // int opaque\n        cmd.setOpaque(headerBuffer.readInt());\n        // int flag\n        cmd.setFlag(headerBuffer.readInt());\n        // String remark\n        cmd.setRemark(readStr(headerBuffer, false, headerLen));\n\n        // HashMap<String, String> extFields\n        int extFieldsLength = headerBuffer.readInt();\n        if (extFieldsLength > 0) {\n            if (extFieldsLength > headerLen) {\n                throw new RemotingCommandException(\"RocketMQ protocol decoding failed, extFields length: \" + extFieldsLength + \", but header length: \" + headerLen);\n            }\n            cmd.setExtFields(mapDeserialize(headerBuffer, extFieldsLength));\n        }\n        return cmd;\n    }\n\n    public static HashMap<String, String> mapDeserialize(ByteBuf byteBuffer, int len) throws RemotingCommandException {\n\n        HashMap<String, String> map = new HashMap<>(128);\n        int endIndex = byteBuffer.readerIndex() + len;\n\n        while (byteBuffer.readerIndex() < endIndex) {\n            String k = readStr(byteBuffer, true, len);\n            String v = readStr(byteBuffer, false, len);\n            map.put(k, v);\n        }\n        return map;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/SerializeType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol;\n\npublic enum SerializeType {\n    JSON((byte) 0),\n    ROCKETMQ((byte) 1);\n\n    private byte code;\n\n    SerializeType(byte code) {\n        this.code = code;\n    }\n\n    public static SerializeType valueOf(byte code) {\n        for (SerializeType serializeType : SerializeType.values()) {\n            if (serializeType.getCode() == code) {\n                return serializeType;\n            }\n        }\n        return null;\n    }\n\n    public byte getCode() {\n        return code;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/admin/ConsumeStats.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.admin;\n\nimport java.util.Map;\nimport java.util.Map.Entry;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class ConsumeStats extends RemotingSerializable {\n    private Map<MessageQueue, OffsetWrapper> offsetTable = new ConcurrentHashMap<>();\n    private double consumeTps = 0;\n\n    public long computeTotalDiff() {\n        long diffTotal = 0L;\n        for (Entry<MessageQueue, OffsetWrapper> entry : this.offsetTable.entrySet()) {\n            diffTotal += entry.getValue().getBrokerOffset() - entry.getValue().getConsumerOffset();\n        }\n        return diffTotal;\n    }\n\n    public long computeInflightTotalDiff() {\n        long diffTotal = 0L;\n        for (Entry<MessageQueue, OffsetWrapper> entry : this.offsetTable.entrySet()) {\n            diffTotal += entry.getValue().getPullOffset() - entry.getValue().getConsumerOffset();\n        }\n        return diffTotal;\n    }\n\n    public Map<MessageQueue, OffsetWrapper> getOffsetTable() {\n        return offsetTable;\n    }\n\n    public void setOffsetTable(Map<MessageQueue, OffsetWrapper> offsetTable) {\n        this.offsetTable = offsetTable;\n    }\n\n    public double getConsumeTps() {\n        return consumeTps;\n    }\n\n    public void setConsumeTps(double consumeTps) {\n        this.consumeTps = consumeTps;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/admin/OffsetWrapper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.admin;\n\npublic class OffsetWrapper {\n    private long brokerOffset;\n    private long consumerOffset;\n    private long pullOffset;\n    private long lastTimestamp;\n\n    public long getBrokerOffset() {\n        return brokerOffset;\n    }\n\n    public void setBrokerOffset(long brokerOffset) {\n        this.brokerOffset = brokerOffset;\n    }\n\n    public long getConsumerOffset() {\n        return consumerOffset;\n    }\n\n    public void setConsumerOffset(long consumerOffset) {\n        this.consumerOffset = consumerOffset;\n    }\n\n    public long getPullOffset() {\n        return pullOffset;\n    }\n\n    public void setPullOffset(long pullOffset) {\n        this.pullOffset = pullOffset;\n    }\n\n    public long getLastTimestamp() {\n        return lastTimestamp;\n    }\n\n    public void setLastTimestamp(long lastTimestamp) {\n        this.lastTimestamp = lastTimestamp;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/admin/RollbackStats.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.admin;\n\npublic class RollbackStats {\n    private String brokerName;\n    private long queueId;\n    private long brokerOffset;\n    private long consumerOffset;\n    private long timestampOffset;\n    private long rollbackOffset;\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public long getQueueId() {\n        return queueId;\n    }\n\n    public void setQueueId(long queueId) {\n        this.queueId = queueId;\n    }\n\n    public long getBrokerOffset() {\n        return brokerOffset;\n    }\n\n    public void setBrokerOffset(long brokerOffset) {\n        this.brokerOffset = brokerOffset;\n    }\n\n    public long getConsumerOffset() {\n        return consumerOffset;\n    }\n\n    public void setConsumerOffset(long consumerOffset) {\n        this.consumerOffset = consumerOffset;\n    }\n\n    public long getTimestampOffset() {\n        return timestampOffset;\n    }\n\n    public void setTimestampOffset(long timestampOffset) {\n        this.timestampOffset = timestampOffset;\n    }\n\n    public long getRollbackOffset() {\n        return rollbackOffset;\n    }\n\n    public void setRollbackOffset(long rollbackOffset) {\n        this.rollbackOffset = rollbackOffset;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/admin/TopicOffset.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.admin;\n\npublic class TopicOffset {\n    private long minOffset;\n    private long maxOffset;\n    private long lastUpdateTimestamp;\n\n    public long getMinOffset() {\n        return minOffset;\n    }\n\n    public void setMinOffset(long minOffset) {\n        this.minOffset = minOffset;\n    }\n\n    public long getMaxOffset() {\n        return maxOffset;\n    }\n\n    public void setMaxOffset(long maxOffset) {\n        this.maxOffset = maxOffset;\n    }\n\n    public long getLastUpdateTimestamp() {\n        return lastUpdateTimestamp;\n    }\n\n    public void setLastUpdateTimestamp(long lastUpdateTimestamp) {\n        this.lastUpdateTimestamp = lastUpdateTimestamp;\n    }\n\n    @Override\n    public String toString() {\n        return \"TopicOffset{\" +\n                \"minOffset=\" + minOffset +\n                \", maxOffset=\" + maxOffset +\n                \", lastUpdateTimestamp=\" + lastUpdateTimestamp +\n                '}';\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/admin/TopicStatsTable.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.admin;\n\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class TopicStatsTable extends RemotingSerializable {\n    private double topicPutTps;\n\n    private Map<MessageQueue, TopicOffset> offsetTable = new ConcurrentHashMap<>();\n\n    public Map<MessageQueue, TopicOffset> getOffsetTable() {\n        return offsetTable;\n    }\n\n    public void setOffsetTable(Map<MessageQueue, TopicOffset> offsetTable) {\n        this.offsetTable = offsetTable;\n    }\n\n    public double getTopicPutTps() {\n        return topicPutTps;\n    }\n\n    public void setTopicPutTps(double topicPutTps) {\n        this.topicPutTps = topicPutTps;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/AclInfo.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.stream.Collectors;\n\npublic class AclInfo {\n\n    private String subject;\n\n    private List<PolicyInfo> policies;\n\n    public static AclInfo of(String subject, List<String> resources, List<String> actions,\n        List<String> sourceIps,\n        String decision) {\n        AclInfo aclInfo = new AclInfo();\n        aclInfo.setSubject(subject);\n        PolicyInfo policyInfo = PolicyInfo.of(resources, actions, sourceIps, decision);\n        aclInfo.setPolicies(Collections.singletonList(policyInfo));\n        return aclInfo;\n    }\n\n    public static class PolicyInfo {\n\n        private String policyType;\n\n        private List<PolicyEntryInfo> entries;\n\n        public static PolicyInfo of(List<String> resources, List<String> actions,\n            List<String> sourceIps, String decision) {\n            PolicyInfo policyInfo = new PolicyInfo();\n            List<PolicyEntryInfo> entries = resources.stream()\n                .map(resource -> PolicyEntryInfo.of(resource, actions, sourceIps, decision))\n                .collect(Collectors.toList());\n            policyInfo.setEntries(entries);\n            return policyInfo;\n        }\n\n        public String getPolicyType() {\n            return policyType;\n        }\n\n        public void setPolicyType(String policyType) {\n            this.policyType = policyType;\n        }\n\n        public List<PolicyEntryInfo> getEntries() {\n            return entries;\n        }\n\n        public void setEntries(List<PolicyEntryInfo> entries) {\n            this.entries = entries;\n        }\n    }\n\n    public static class PolicyEntryInfo {\n        private String resource;\n\n        private List<String> actions;\n\n        private List<String> sourceIps;\n\n        private String decision;\n\n        public static PolicyEntryInfo of(String resource, List<String> actions, List<String> sourceIps,\n            String decision) {\n            PolicyEntryInfo policyEntryInfo = new PolicyEntryInfo();\n            policyEntryInfo.setResource(resource);\n            policyEntryInfo.setActions(actions);\n            policyEntryInfo.setSourceIps(sourceIps);\n            policyEntryInfo.setDecision(decision);\n            return policyEntryInfo;\n        }\n\n        public String getResource() {\n            return resource;\n        }\n\n        public void setResource(String resource) {\n            this.resource = resource;\n        }\n\n        public List<String> getActions() {\n            return actions;\n        }\n\n        public void setActions(List<String> actions) {\n            this.actions = actions;\n        }\n\n        public List<String> getSourceIps() {\n            return sourceIps;\n        }\n\n        public void setSourceIps(List<String> sourceIps) {\n            this.sourceIps = sourceIps;\n        }\n\n        public String getDecision() {\n            return decision;\n        }\n\n        public void setDecision(String decision) {\n            this.decision = decision;\n        }\n    }\n\n    public String getSubject() {\n        return subject;\n    }\n\n    public void setSubject(String subject) {\n        this.subject = subject;\n    }\n\n    public List<PolicyInfo> getPolicies() {\n        return policies;\n    }\n\n    public void setPolicies(List<PolicyInfo> policies) {\n        this.policies = policies;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/BatchAck.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport com.alibaba.fastjson2.annotation.JSONField;\nimport org.apache.rocketmq.remoting.protocol.BitSetSerializerDeserializer;\n\nimport java.io.Serializable;\nimport java.util.BitSet;\n\npublic class BatchAck implements Serializable {\n    @JSONField(name = \"c\", alternateNames = {\"consumerGroup\"})\n    private String consumerGroup;\n    @JSONField(name = \"t\", alternateNames = {\"topic\"})\n    private String topic;\n    @JSONField(name = \"r\", alternateNames = {\"retry\"})\n    private String retry; // \"1\" if is retry topic\n    @JSONField(name = \"so\", alternateNames = {\"startOffset\"})\n    private long startOffset;\n    @JSONField(name = \"q\", alternateNames = {\"queueId\"})\n    private int queueId;\n    @JSONField(name = \"rq\", alternateNames = {\"reviveQueueId\"})\n    private int reviveQueueId;\n    @JSONField(name = \"pt\", alternateNames = {\"popTime\"})\n    private long popTime;\n    @JSONField(name = \"it\", alternateNames = {\"invisibleTime\"})\n    private long invisibleTime;\n    @JSONField(name = \"b\", alternateNames = {\"bitSet\"}, serializeUsing = BitSetSerializerDeserializer.class, deserializeUsing = BitSetSerializerDeserializer.class)\n    private BitSet bitSet; // ack offsets bitSet\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getRetry() {\n        return retry;\n    }\n\n    public void setRetry(String retry) {\n        this.retry = retry;\n    }\n\n    public long getStartOffset() {\n        return startOffset;\n    }\n\n    public void setStartOffset(long startOffset) {\n        this.startOffset = startOffset;\n    }\n\n    public int getQueueId() {\n        return queueId;\n    }\n\n    public void setQueueId(int queueId) {\n        this.queueId = queueId;\n    }\n\n    public int getReviveQueueId() {\n        return reviveQueueId;\n    }\n\n    public void setReviveQueueId(int reviveQueueId) {\n        this.reviveQueueId = reviveQueueId;\n    }\n\n    public long getPopTime() {\n        return popTime;\n    }\n\n    public void setPopTime(long popTime) {\n        this.popTime = popTime;\n    }\n\n    public long getInvisibleTime() {\n        return invisibleTime;\n    }\n\n    public void setInvisibleTime(long invisibleTime) {\n        this.invisibleTime = invisibleTime;\n    }\n\n    public BitSet getBitSet() {\n        return bitSet;\n    }\n\n    public void setBitSet(BitSet bitSet) {\n        this.bitSet = bitSet;\n    }\n\n    @Override\n    public String toString() {\n        return \"BatchAck{\" +\n                \"consumerGroup='\" + consumerGroup + '\\'' +\n                \", topic='\" + topic + '\\'' +\n                \", retry='\" + retry + '\\'' +\n                \", startOffset=\" + startOffset +\n                \", queueId=\" + queueId +\n                \", reviveQueueId=\" + reviveQueueId +\n                \", popTime=\" + popTime +\n                \", invisibleTime=\" + invisibleTime +\n                \", bitSet=\" + bitSet +\n                '}';\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/BatchAckMessageRequestBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\nimport java.util.List;\n\npublic class BatchAckMessageRequestBody extends RemotingSerializable {\n    private String brokerName;\n    private List<BatchAck> acks;\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public List<BatchAck> getAcks() {\n        return acks;\n    }\n\n    public void setAcks(List<BatchAck> acks) {\n        this.acks = acks;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/BrokerMemberGroup.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport com.google.common.base.Objects;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.Map;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class BrokerMemberGroup extends RemotingSerializable {\n    private String cluster;\n    private String brokerName;\n    private Map<Long/* brokerId */, String/* broker address */> brokerAddrs;\n\n    // Provide default constructor for serializer\n    public BrokerMemberGroup() {\n        this.brokerAddrs = new HashMap<>();\n    }\n\n    public BrokerMemberGroup(final String cluster, final String brokerName) {\n        this.cluster = cluster;\n        this.brokerName = brokerName;\n        this.brokerAddrs = new HashMap<>();\n    }\n\n    public long minimumBrokerId() {\n        if (this.brokerAddrs.isEmpty()) {\n            return 0;\n        }\n        return Collections.min(brokerAddrs.keySet());\n    }\n\n    public String getCluster() {\n        return cluster;\n    }\n\n    public void setCluster(final String cluster) {\n        this.cluster = cluster;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(final String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public Map<Long, String> getBrokerAddrs() {\n        return brokerAddrs;\n    }\n\n    public void setBrokerAddrs(final Map<Long, String> brokerAddrs) {\n        this.brokerAddrs = brokerAddrs;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) {\n            return true;\n        }\n        if (o == null || getClass() != o.getClass()) {\n            return false;\n        }\n        BrokerMemberGroup that = (BrokerMemberGroup) o;\n        return Objects.equal(cluster, that.cluster) &&\n            Objects.equal(brokerName, that.brokerName) &&\n            Objects.equal(brokerAddrs, that.brokerAddrs);\n    }\n\n    @Override\n    public int hashCode() {\n        return Objects.hashCode(cluster, brokerName, brokerAddrs);\n    }\n\n    @Override\n    public String toString() {\n        return \"BrokerMemberGroup{\" +\n            \"cluster='\" + cluster + '\\'' +\n            \", brokerName='\" + brokerName + '\\'' +\n            \", brokerAddrs=\" + brokerAddrs +\n            '}';\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/BrokerReplicasInfo.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\n\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class BrokerReplicasInfo extends RemotingSerializable  {\n    private Map<String/*brokerName*/, ReplicasInfo> replicasInfoTable;\n\n    public BrokerReplicasInfo() {\n        this.replicasInfoTable = new HashMap<>();\n    }\n\n    public void addReplicaInfo(final String brokerName, final ReplicasInfo replicasInfo) {\n        this.replicasInfoTable.put(brokerName, replicasInfo);\n    }\n\n    public Map<String, ReplicasInfo> getReplicasInfoTable() {\n        return replicasInfoTable;\n    }\n\n    public void setReplicasInfoTable(\n            Map<String, ReplicasInfo> replicasInfoTable) {\n        this.replicasInfoTable = replicasInfoTable;\n    }\n\n    public static class ReplicasInfo extends RemotingSerializable {\n\n        private Long masterBrokerId;\n\n        private String masterAddress;\n        private Integer masterEpoch;\n        private Integer syncStateSetEpoch;\n        private List<ReplicaIdentity> inSyncReplicas;\n        private List<ReplicaIdentity> notInSyncReplicas;\n\n        public ReplicasInfo(Long masterBrokerId, String masterAddress, int masterEpoch, int syncStateSetEpoch,\n                            List<ReplicaIdentity> inSyncReplicas, List<ReplicaIdentity> notInSyncReplicas) {\n            this.masterBrokerId = masterBrokerId;\n            this.masterAddress = masterAddress;\n            this.masterEpoch = masterEpoch;\n            this.syncStateSetEpoch = syncStateSetEpoch;\n            this.inSyncReplicas = inSyncReplicas;\n            this.notInSyncReplicas = notInSyncReplicas;\n        }\n\n        public String getMasterAddress() {\n            return masterAddress;\n        }\n\n        public void setMasterAddress(String masterAddress) {\n            this.masterAddress = masterAddress;\n        }\n\n        public int getMasterEpoch() {\n            return masterEpoch;\n        }\n\n        public void setMasterEpoch(int masterEpoch) {\n            this.masterEpoch = masterEpoch;\n        }\n\n        public int getSyncStateSetEpoch() {\n            return syncStateSetEpoch;\n        }\n\n        public void setSyncStateSetEpoch(int syncStateSetEpoch) {\n            this.syncStateSetEpoch = syncStateSetEpoch;\n        }\n\n        public List<ReplicaIdentity> getInSyncReplicas() {\n            return inSyncReplicas;\n        }\n\n        public void setInSyncReplicas(\n                List<ReplicaIdentity> inSyncReplicas) {\n            this.inSyncReplicas = inSyncReplicas;\n        }\n\n        public List<ReplicaIdentity> getNotInSyncReplicas() {\n            return notInSyncReplicas;\n        }\n\n        public void setNotInSyncReplicas(\n                List<ReplicaIdentity> notInSyncReplicas) {\n            this.notInSyncReplicas = notInSyncReplicas;\n        }\n\n        public void setMasterBrokerId(Long masterBrokerId) {\n            this.masterBrokerId = masterBrokerId;\n        }\n\n        public Long getMasterBrokerId() {\n            return masterBrokerId;\n        }\n\n        public boolean isExistInSync(String brokerName, Long brokerId, String brokerAddress) {\n            return this.getInSyncReplicas().contains(new ReplicaIdentity(brokerName, brokerId, brokerAddress));\n        }\n\n        public boolean isExistInNotSync(String brokerName, Long brokerId, String brokerAddress) {\n            return this.getNotInSyncReplicas().contains(new ReplicaIdentity(brokerName, brokerId, brokerAddress));\n        }\n\n        public boolean isExistInAllReplicas(String brokerName, Long brokerId, String brokerAddress) {\n            return this.isExistInSync(brokerName, brokerId, brokerAddress) || this.isExistInNotSync(brokerName, brokerId, brokerAddress);\n        }\n    }\n\n    public static class ReplicaIdentity extends RemotingSerializable {\n        private String brokerName;\n        private Long brokerId;\n\n        private String brokerAddress;\n        private Boolean alive;\n\n        public ReplicaIdentity(String brokerName, Long brokerId, String brokerAddress) {\n            this.brokerName = brokerName;\n            this.brokerId = brokerId;\n            this.brokerAddress = brokerAddress;\n            this.alive = false;\n        }\n\n        public ReplicaIdentity(String brokerName, Long brokerId, String brokerAddress, Boolean alive) {\n            this.brokerName = brokerName;\n            this.brokerId = brokerId;\n            this.brokerAddress = brokerAddress;\n            this.alive = alive;\n        }\n\n        public String getBrokerName() {\n            return brokerName;\n        }\n\n        public void setBrokerName(String brokerName) {\n            this.brokerName = brokerName;\n        }\n\n        public String getBrokerAddress() {\n            return brokerAddress;\n        }\n\n        public void setBrokerAddress(String brokerAddress) {\n            this.brokerAddress = brokerAddress;\n        }\n\n        public Long getBrokerId() {\n            return brokerId;\n        }\n\n        public void setBrokerId(Long brokerId) {\n            this.brokerId = brokerId;\n        }\n\n        public Boolean getAlive() {\n            return alive;\n        }\n\n        public void setAlive(Boolean alive) {\n            this.alive = alive;\n        }\n\n        @Override\n        public String toString() {\n            return \"ReplicaIdentity{\" +\n                    \"brokerName='\" + brokerName + '\\'' +\n                    \", brokerId=\" + brokerId +\n                    \", brokerAddress='\" + brokerAddress + '\\'' +\n                    \", alive=\" + alive +\n                    '}';\n        }\n\n        @Override\n        public boolean equals(Object o) {\n            if (this == o) return true;\n            if (o == null || getClass() != o.getClass()) return false;\n            ReplicaIdentity that = (ReplicaIdentity) o;\n            return brokerName.equals(that.brokerName) && brokerId.equals(that.brokerId) && brokerAddress.equals(that.brokerAddress);\n        }\n\n        @Override\n        public int hashCode() {\n            return Objects.hash(brokerName, brokerId, brokerAddress);\n        }\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/BrokerStatsData.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class BrokerStatsData extends RemotingSerializable {\n\n    private BrokerStatsItem statsMinute;\n\n    private BrokerStatsItem statsHour;\n\n    private BrokerStatsItem statsDay;\n\n    public BrokerStatsItem getStatsMinute() {\n        return statsMinute;\n    }\n\n    public void setStatsMinute(BrokerStatsItem statsMinute) {\n        this.statsMinute = statsMinute;\n    }\n\n    public BrokerStatsItem getStatsHour() {\n        return statsHour;\n    }\n\n    public void setStatsHour(BrokerStatsItem statsHour) {\n        this.statsHour = statsHour;\n    }\n\n    public BrokerStatsItem getStatsDay() {\n        return statsDay;\n    }\n\n    public void setStatsDay(BrokerStatsItem statsDay) {\n        this.statsDay = statsDay;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/BrokerStatsItem.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\npublic class BrokerStatsItem {\n    private long sum;\n    private double tps;\n    private double avgpt;\n\n    public long getSum() {\n        return sum;\n    }\n\n    public void setSum(long sum) {\n        this.sum = sum;\n    }\n\n    public double getTps() {\n        return tps;\n    }\n\n    public void setTps(double tps) {\n        this.tps = tps;\n    }\n\n    public double getAvgpt() {\n        return avgpt;\n    }\n\n    public void setAvgpt(double avgpt) {\n        this.avgpt = avgpt;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/CMResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\npublic enum CMResult {\n    CR_SUCCESS,\n    CR_LATER,\n    CR_ROLLBACK,\n    CR_COMMIT,\n    CR_THROW_EXCEPTION,\n    CR_RETURN_NULL,\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/CheckClientRequestBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\npublic class CheckClientRequestBody extends RemotingSerializable {\n\n    private String clientId;\n    private String group;\n    private SubscriptionData subscriptionData;\n    private String namespace;\n\n    public String getClientId() {\n        return clientId;\n    }\n\n    public void setClientId(String clientId) {\n        this.clientId = clientId;\n    }\n\n    public String getGroup() {\n        return group;\n    }\n\n    public void setGroup(String group) {\n        this.group = group;\n    }\n\n    public SubscriptionData getSubscriptionData() {\n        return subscriptionData;\n    }\n\n    public void setSubscriptionData(SubscriptionData subscriptionData) {\n        this.subscriptionData = subscriptionData;\n    }\n\n    public String getNamespace() {\n        return namespace;\n    }\n\n    public void setNamespace(String namespace) {\n        this.namespace = namespace;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/ClusterInfo.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport com.google.common.base.Objects;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\n\npublic class ClusterInfo extends RemotingSerializable {\n    private Map<String/* brokerName */, BrokerData> brokerAddrTable;\n    private Map<String/* clusterName */, Set<String/* brokerName */>> clusterAddrTable;\n\n    public Map<String, BrokerData> getBrokerAddrTable() {\n        return brokerAddrTable;\n    }\n\n    public void setBrokerAddrTable(Map<String, BrokerData> brokerAddrTable) {\n        this.brokerAddrTable = brokerAddrTable;\n    }\n\n    public Map<String, Set<String>> getClusterAddrTable() {\n        return clusterAddrTable;\n    }\n\n    public void setClusterAddrTable(Map<String, Set<String>> clusterAddrTable) {\n        this.clusterAddrTable = clusterAddrTable;\n    }\n\n    public String[] retrieveAllAddrByCluster(String cluster) {\n        List<String> addrs = new ArrayList<>();\n        if (clusterAddrTable.containsKey(cluster)) {\n            Set<String> brokerNames = clusterAddrTable.get(cluster);\n            for (String brokerName : brokerNames) {\n                BrokerData brokerData = brokerAddrTable.get(brokerName);\n                if (null != brokerData) {\n                    addrs.addAll(brokerData.getBrokerAddrs().values());\n                }\n            }\n        }\n\n        return addrs.toArray(new String[] {});\n    }\n\n    public String[] retrieveAllClusterNames() {\n        return clusterAddrTable.keySet().toArray(new String[] {});\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o)\n            return true;\n        if (o == null || getClass() != o.getClass())\n            return false;\n        ClusterInfo info = (ClusterInfo) o;\n        return Objects.equal(brokerAddrTable, info.brokerAddrTable) && Objects.equal(clusterAddrTable, info.clusterAddrTable);\n    }\n\n    @Override\n    public int hashCode() {\n        return Objects.hashCode(brokerAddrTable, clusterAddrTable);\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/Connection.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\n\npublic class Connection {\n    private String clientId;\n    private String clientAddr;\n    private LanguageCode language;\n    private int version;\n\n    public String getClientId() {\n        return clientId;\n    }\n\n    public void setClientId(String clientId) {\n        this.clientId = clientId;\n    }\n\n    public String getClientAddr() {\n        return clientAddr;\n    }\n\n    public void setClientAddr(String clientAddr) {\n        this.clientAddr = clientAddr;\n    }\n\n    public LanguageCode getLanguage() {\n        return language;\n    }\n\n    public void setLanguage(LanguageCode language) {\n        this.language = language;\n    }\n\n    public int getVersion() {\n        return version;\n    }\n\n    public void setVersion(int version) {\n        this.version = version;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/ConsumeByWho.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.HashSet;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class ConsumeByWho extends RemotingSerializable {\n    private HashSet<String> consumedGroup = new HashSet<>();\n    private HashSet<String> notConsumedGroup = new HashSet<>();\n    private String topic;\n    private int queueId;\n    private long offset;\n\n    public HashSet<String> getConsumedGroup() {\n        return consumedGroup;\n    }\n\n    public void setConsumedGroup(HashSet<String> consumedGroup) {\n        this.consumedGroup = consumedGroup;\n    }\n\n    public HashSet<String> getNotConsumedGroup() {\n        return notConsumedGroup;\n    }\n\n    public void setNotConsumedGroup(HashSet<String> notConsumedGroup) {\n        this.notConsumedGroup = notConsumedGroup;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public int getQueueId() {\n        return queueId;\n    }\n\n    public void setQueueId(int queueId) {\n        this.queueId = queueId;\n    }\n\n    public long getOffset() {\n        return offset;\n    }\n\n    public void setOffset(long offset) {\n        this.offset = offset;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/ConsumeMessageDirectlyResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class ConsumeMessageDirectlyResult extends RemotingSerializable {\n    private boolean order = false;\n    private boolean autoCommit = true;\n    private CMResult consumeResult;\n    private String remark;\n    private long spentTimeMills;\n\n    public boolean isOrder() {\n        return order;\n    }\n\n    public void setOrder(boolean order) {\n        this.order = order;\n    }\n\n    public boolean isAutoCommit() {\n        return autoCommit;\n    }\n\n    public void setAutoCommit(boolean autoCommit) {\n        this.autoCommit = autoCommit;\n    }\n\n    public String getRemark() {\n        return remark;\n    }\n\n    public void setRemark(String remark) {\n        this.remark = remark;\n    }\n\n    public CMResult getConsumeResult() {\n        return consumeResult;\n    }\n\n    public void setConsumeResult(CMResult consumeResult) {\n        this.consumeResult = consumeResult;\n    }\n\n    public long getSpentTimeMills() {\n        return spentTimeMills;\n    }\n\n    public void setSpentTimeMills(long spentTimeMills) {\n        this.spentTimeMills = spentTimeMills;\n    }\n\n    @Override\n    public String toString() {\n        return \"ConsumeMessageDirectlyResult [order=\" + order + \", autoCommit=\" + autoCommit\n            + \", consumeResult=\" + consumeResult + \", remark=\" + remark + \", spentTimeMills=\"\n            + spentTimeMills + \"]\";\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/ConsumeQueueData.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\npublic class ConsumeQueueData {\n\n    private long physicOffset;\n    private int physicSize;\n    private long tagsCode;\n    private String extendDataJson;\n    private String bitMap;\n    private boolean eval;\n    private String msg;\n\n    public long getPhysicOffset() {\n        return physicOffset;\n    }\n\n    public void setPhysicOffset(long physicOffset) {\n        this.physicOffset = physicOffset;\n    }\n\n    public int getPhysicSize() {\n        return physicSize;\n    }\n\n    public void setPhysicSize(int physicSize) {\n        this.physicSize = physicSize;\n    }\n\n    public long getTagsCode() {\n        return tagsCode;\n    }\n\n    public void setTagsCode(long tagsCode) {\n        this.tagsCode = tagsCode;\n    }\n\n    public String getExtendDataJson() {\n        return extendDataJson;\n    }\n\n    public void setExtendDataJson(String extendDataJson) {\n        this.extendDataJson = extendDataJson;\n    }\n\n    public String getBitMap() {\n        return bitMap;\n    }\n\n    public void setBitMap(String bitMap) {\n        this.bitMap = bitMap;\n    }\n\n    public boolean isEval() {\n        return eval;\n    }\n\n    public void setEval(boolean eval) {\n        this.eval = eval;\n    }\n\n    public String getMsg() {\n        return msg;\n    }\n\n    public void setMsg(String msg) {\n        this.msg = msg;\n    }\n\n    @Override\n    public String toString() {\n        return \"ConsumeQueueData{\" +\n            \"physicOffset=\" + physicOffset +\n            \", physicSize=\" + physicSize +\n            \", tagsCode=\" + tagsCode +\n            \", extendDataJson='\" + extendDataJson + '\\'' +\n            \", bitMap='\" + bitMap + '\\'' +\n            \", eval=\" + eval +\n            \", msg='\" + msg + '\\'' +\n            '}';\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/ConsumeStatsList.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.admin.ConsumeStats;\n\npublic class ConsumeStatsList extends RemotingSerializable {\n    private List<Map<String/*subscriptionGroupName*/, List<ConsumeStats>>> consumeStatsList = new ArrayList<>();\n    private String brokerAddr;\n    private long totalDiff;\n    private long totalInflightDiff;\n\n    public List<Map<String, List<ConsumeStats>>> getConsumeStatsList() {\n        return consumeStatsList;\n    }\n\n    public void setConsumeStatsList(List<Map<String, List<ConsumeStats>>> consumeStatsList) {\n        this.consumeStatsList = consumeStatsList;\n    }\n\n    public String getBrokerAddr() {\n        return brokerAddr;\n    }\n\n    public void setBrokerAddr(String brokerAddr) {\n        this.brokerAddr = brokerAddr;\n    }\n\n    public long getTotalDiff() {\n        return totalDiff;\n    }\n\n    public void setTotalDiff(long totalDiff) {\n        this.totalDiff = totalDiff;\n    }\n\n    public long getTotalInflightDiff() {\n        return totalInflightDiff;\n    }\n\n    public void setTotalInflightDiff(long totalInflightDiff) {\n        this.totalInflightDiff = totalInflightDiff;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/ConsumeStatus.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\npublic class ConsumeStatus {\n    private double pullRT;\n    private double pullTPS;\n    private double consumeRT;\n    private double consumeOKTPS;\n    private double consumeFailedTPS;\n\n    private long consumeFailedMsgs;\n\n    public double getPullRT() {\n        return pullRT;\n    }\n\n    public void setPullRT(double pullRT) {\n        this.pullRT = pullRT;\n    }\n\n    public double getPullTPS() {\n        return pullTPS;\n    }\n\n    public void setPullTPS(double pullTPS) {\n        this.pullTPS = pullTPS;\n    }\n\n    public double getConsumeRT() {\n        return consumeRT;\n    }\n\n    public void setConsumeRT(double consumeRT) {\n        this.consumeRT = consumeRT;\n    }\n\n    public double getConsumeOKTPS() {\n        return consumeOKTPS;\n    }\n\n    public void setConsumeOKTPS(double consumeOKTPS) {\n        this.consumeOKTPS = consumeOKTPS;\n    }\n\n    public double getConsumeFailedTPS() {\n        return consumeFailedTPS;\n    }\n\n    public void setConsumeFailedTPS(double consumeFailedTPS) {\n        this.consumeFailedTPS = consumeFailedTPS;\n    }\n\n    public long getConsumeFailedMsgs() {\n        return consumeFailedMsgs;\n    }\n\n    public void setConsumeFailedMsgs(long consumeFailedMsgs) {\n        this.consumeFailedMsgs = consumeFailedMsgs;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/ConsumerConnection.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.HashSet;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\npublic class ConsumerConnection extends RemotingSerializable {\n    private HashSet<Connection> connectionSet = new HashSet<>();\n    private ConcurrentMap<String/* Topic */, SubscriptionData> subscriptionTable =\n        new ConcurrentHashMap<>();\n    private ConsumeType consumeType;\n    private MessageModel messageModel;\n    private ConsumeFromWhere consumeFromWhere;\n\n    public int computeMinVersion() {\n        int minVersion = Integer.MAX_VALUE;\n        for (Connection c : this.connectionSet) {\n            if (c.getVersion() < minVersion) {\n                minVersion = c.getVersion();\n            }\n        }\n\n        return minVersion;\n    }\n\n    public HashSet<Connection> getConnectionSet() {\n        return connectionSet;\n    }\n\n    public void setConnectionSet(HashSet<Connection> connectionSet) {\n        this.connectionSet = connectionSet;\n    }\n\n    public ConcurrentMap<String, SubscriptionData> getSubscriptionTable() {\n        return subscriptionTable;\n    }\n\n    public void setSubscriptionTable(ConcurrentHashMap<String, SubscriptionData> subscriptionTable) {\n        this.subscriptionTable = subscriptionTable;\n    }\n\n    public ConsumeType getConsumeType() {\n        return consumeType;\n    }\n\n    public void setConsumeType(ConsumeType consumeType) {\n        this.consumeType = consumeType;\n    }\n\n    public MessageModel getMessageModel() {\n        return messageModel;\n    }\n\n    public void setMessageModel(MessageModel messageModel) {\n        this.messageModel = messageModel;\n    }\n\n    public ConsumeFromWhere getConsumeFromWhere() {\n        return consumeFromWhere;\n    }\n\n    public void setConsumeFromWhere(ConsumeFromWhere consumeFromWhere) {\n        this.consumeFromWhere = consumeFromWhere;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/ConsumerOffsetSerializeWrapper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class ConsumerOffsetSerializeWrapper extends RemotingSerializable {\n    private ConcurrentMap<String/* topic@group */, ConcurrentMap<Integer, Long>> offsetTable =\n        new ConcurrentHashMap<>(512);\n    private DataVersion dataVersion;\n\n    public ConcurrentMap<String, ConcurrentMap<Integer, Long>> getOffsetTable() {\n        return offsetTable;\n    }\n\n    public void setOffsetTable(ConcurrentMap<String, ConcurrentMap<Integer, Long>> offsetTable) {\n        this.offsetTable = offsetTable;\n    }\n\n    public DataVersion getDataVersion() {\n        return dataVersion;\n    }\n\n    public void setDataVersion(DataVersion dataVersion) {\n        this.dataVersion = dataVersion;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/ConsumerRunningInfo.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.Iterator;\nimport java.util.Map.Entry;\nimport java.util.Properties;\nimport java.util.TreeMap;\nimport java.util.TreeSet;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\npublic class ConsumerRunningInfo extends RemotingSerializable {\n    public static final String PROP_NAMESERVER_ADDR = \"PROP_NAMESERVER_ADDR\";\n    public static final String PROP_THREADPOOL_CORE_SIZE = \"PROP_THREADPOOL_CORE_SIZE\";\n    public static final String PROP_CONSUME_ORDERLY = \"PROP_CONSUMEORDERLY\";\n    public static final String PROP_CONSUME_TYPE = \"PROP_CONSUME_TYPE\";\n    public static final String PROP_CLIENT_VERSION = \"PROP_CLIENT_VERSION\";\n    public static final String PROP_CONSUMER_START_TIMESTAMP = \"PROP_CONSUMER_START_TIMESTAMP\";\n\n    private Properties properties = new Properties();\n\n    private TreeSet<SubscriptionData> subscriptionSet = new TreeSet<>();\n\n    private TreeMap<MessageQueue, ProcessQueueInfo> mqTable = new TreeMap<>();\n\n    private TreeMap<MessageQueue, PopProcessQueueInfo> mqPopTable = new TreeMap<>();\n\n    private TreeMap<String/* Topic */, ConsumeStatus> statusTable = new TreeMap<>();\n\n    private TreeMap<String, String> userConsumerInfo = new TreeMap<>();\n\n    private String jstack;\n\n    public static boolean analyzeSubscription(final TreeMap<String/* clientId */, ConsumerRunningInfo> criTable) {\n        ConsumerRunningInfo prev = criTable.firstEntry().getValue();\n\n        boolean push = isPushType(prev);\n\n        boolean startForAWhile = false;\n        {\n\n            String property = prev.getProperties().getProperty(ConsumerRunningInfo.PROP_CONSUMER_START_TIMESTAMP);\n            if (property == null) {\n                property = String.valueOf(prev.getProperties().get(ConsumerRunningInfo.PROP_CONSUMER_START_TIMESTAMP));\n            }\n            startForAWhile = (System.currentTimeMillis() - Long.parseLong(property)) > (1000 * 60 * 2);\n        }\n\n        if (push && startForAWhile) {\n\n            {\n                Iterator<Entry<String, ConsumerRunningInfo>> it = criTable.entrySet().iterator();\n                while (it.hasNext()) {\n                    Entry<String, ConsumerRunningInfo> next = it.next();\n                    ConsumerRunningInfo current = next.getValue();\n                    boolean equals = current.getSubscriptionSet().equals(prev.getSubscriptionSet());\n\n                    if (!equals) {\n                        // Different subscription in the same group of consumer\n                        return false;\n                    }\n\n                    prev = next.getValue();\n                }\n\n                // after consumer.unsubscribe , SubscriptionSet is Empty\n                //if (prev != null) {\n                //\n                //    if (prev.getSubscriptionSet().isEmpty()) {\n                //        // Subscription empty!\n                //        return false;\n                //    }\n                //}\n            }\n        }\n\n        return true;\n    }\n\n    public static boolean isPushType(ConsumerRunningInfo consumerRunningInfo) {\n        String property = consumerRunningInfo.getProperties().getProperty(ConsumerRunningInfo.PROP_CONSUME_TYPE);\n\n        if (property == null) {\n            property = ((ConsumeType) consumerRunningInfo.getProperties().get(ConsumerRunningInfo.PROP_CONSUME_TYPE)).name();\n        }\n        return ConsumeType.valueOf(property) == ConsumeType.CONSUME_PASSIVELY;\n    }\n\n    public static boolean analyzeRebalance(final TreeMap<String/* clientId */, ConsumerRunningInfo> criTable) {\n        return true;\n    }\n\n    public static String analyzeProcessQueue(final String clientId, ConsumerRunningInfo info) {\n        StringBuilder sb = new StringBuilder();\n        boolean push = false;\n        {\n            String property = info.getProperties().getProperty(ConsumerRunningInfo.PROP_CONSUME_TYPE);\n\n            if (property == null) {\n                property = ((ConsumeType) info.getProperties().get(ConsumerRunningInfo.PROP_CONSUME_TYPE)).name();\n            }\n            push = ConsumeType.valueOf(property) == ConsumeType.CONSUME_PASSIVELY;\n        }\n\n        boolean orderMsg = false;\n        {\n            String property = info.getProperties().getProperty(ConsumerRunningInfo.PROP_CONSUME_ORDERLY);\n            orderMsg = Boolean.parseBoolean(property);\n        }\n\n        if (push) {\n            Iterator<Entry<MessageQueue, ProcessQueueInfo>> it = info.getMqTable().entrySet().iterator();\n            while (it.hasNext()) {\n                Entry<MessageQueue, ProcessQueueInfo> next = it.next();\n                MessageQueue mq = next.getKey();\n                ProcessQueueInfo pq = next.getValue();\n\n                if (orderMsg) {\n\n                    if (!pq.isLocked()) {\n                        sb.append(String.format(\"%s %s can't lock for a while, %dms%n\",\n                            clientId,\n                            mq,\n                            System.currentTimeMillis() - pq.getLastLockTimestamp()));\n                    } else {\n                        if (pq.isDroped() && pq.getTryUnlockTimes() > 0) {\n                            sb.append(String.format(\"%s %s unlock %d times, still failed%n\",\n                                clientId,\n                                mq,\n                                pq.getTryUnlockTimes()));\n                        }\n                    }\n\n                } else {\n                    long diff = System.currentTimeMillis() - pq.getLastConsumeTimestamp();\n\n                    if (diff > (1000 * 60) && pq.getCachedMsgCount() > 0) {\n                        sb.append(String.format(\"%s %s can't consume for a while, maybe blocked, %dms%n\",\n                            clientId,\n                            mq,\n                            diff));\n                    }\n                }\n            }\n        }\n\n        return sb.toString();\n    }\n\n    public Properties getProperties() {\n        return properties;\n    }\n\n    public void setProperties(Properties properties) {\n        this.properties = properties;\n    }\n\n    public TreeSet<SubscriptionData> getSubscriptionSet() {\n        return subscriptionSet;\n    }\n\n    public void setSubscriptionSet(TreeSet<SubscriptionData> subscriptionSet) {\n        this.subscriptionSet = subscriptionSet;\n    }\n\n    public TreeMap<MessageQueue, ProcessQueueInfo> getMqTable() {\n        return mqTable;\n    }\n\n    public void setMqTable(TreeMap<MessageQueue, ProcessQueueInfo> mqTable) {\n        this.mqTable = mqTable;\n    }\n\n    public TreeMap<String, ConsumeStatus> getStatusTable() {\n        return statusTable;\n    }\n\n    public void setStatusTable(TreeMap<String, ConsumeStatus> statusTable) {\n        this.statusTable = statusTable;\n    }\n\n    public TreeMap<String, String> getUserConsumerInfo() {\n        return userConsumerInfo;\n    }\n\n    public String formatString() {\n        StringBuilder sb = new StringBuilder();\n\n        {\n            sb.append(\"#Consumer Properties#\\n\");\n            Iterator<Entry<Object, Object>> it = this.properties.entrySet().iterator();\n            while (it.hasNext()) {\n                Entry<Object, Object> next = it.next();\n                String item = String.format(\"%-40s: %s%n\", next.getKey().toString(), next.getValue().toString());\n                sb.append(item);\n            }\n        }\n\n        {\n            sb.append(\"\\n\\n#Consumer Subscription#\\n\");\n\n            Iterator<SubscriptionData> it = this.subscriptionSet.iterator();\n            int i = 0;\n            while (it.hasNext()) {\n                SubscriptionData next = it.next();\n                String item = String.format(\"%03d Topic: %-40s ClassFilter: %-8s SubExpression: %s%n\",\n                    ++i,\n                    next.getTopic(),\n                    next.isClassFilterMode(),\n                    next.getSubString());\n\n                sb.append(item);\n            }\n        }\n\n        {\n            sb.append(\"\\n\\n#Consumer Offset#\\n\");\n            sb.append(String.format(\"%-64s  %-32s  %-4s  %-20s%n\",\n                \"#Topic\",\n                \"#Broker Name\",\n                \"#QID\",\n                \"#Consumer Offset\"\n            ));\n\n            Iterator<Entry<MessageQueue, ProcessQueueInfo>> it = this.mqTable.entrySet().iterator();\n            while (it.hasNext()) {\n                Entry<MessageQueue, ProcessQueueInfo> next = it.next();\n                String item = String.format(\"%-32s  %-32s  %-4d  %-20d%n\",\n                    next.getKey().getTopic(),\n                    next.getKey().getBrokerName(),\n                    next.getKey().getQueueId(),\n                    next.getValue().getCommitOffset());\n\n                sb.append(item);\n            }\n        }\n\n        {\n            sb.append(\"\\n\\n#Consumer MQ Detail#\\n\");\n            sb.append(String.format(\"%-64s  %-32s  %-4s  %-20s%n\",\n                \"#Topic\",\n                \"#Broker Name\",\n                \"#QID\",\n                \"#ProcessQueueInfo\"\n            ));\n\n            Iterator<Entry<MessageQueue, ProcessQueueInfo>> it = this.mqTable.entrySet().iterator();\n            while (it.hasNext()) {\n                Entry<MessageQueue, ProcessQueueInfo> next = it.next();\n                String item = String.format(\"%-64s  %-32s  %-4d  %s%n\",\n                    next.getKey().getTopic(),\n                    next.getKey().getBrokerName(),\n                    next.getKey().getQueueId(),\n                    next.getValue().toString());\n\n                sb.append(item);\n            }\n        }\n\n        {\n            sb.append(\"\\n\\n#Consumer Pop Detail#\\n\");\n            sb.append(String.format(\"%-32s  %-32s  %-4s  %-20s%n\",\n                \"#Topic\",\n                \"#Broker Name\",\n                \"#QID\",\n                \"#ProcessQueueInfo\"\n            ));\n\n            Iterator<Entry<MessageQueue, PopProcessQueueInfo>> it = this.mqPopTable.entrySet().iterator();\n            while (it.hasNext()) {\n                Entry<MessageQueue, PopProcessQueueInfo> next = it.next();\n                String item = String.format(\"%-32s  %-32s  %-4d  %s%n\",\n                    next.getKey().getTopic(),\n                    next.getKey().getBrokerName(),\n                    next.getKey().getQueueId(),\n                    next.getValue().toString());\n\n                sb.append(item);\n            }\n        }\n\n        {\n            sb.append(\"\\n\\n#Consumer RT&TPS#\\n\");\n            sb.append(String.format(\"%-64s  %14s %14s %14s %14s %18s %25s%n\",\n                \"#Topic\",\n                \"#Pull RT\",\n                \"#Pull TPS\",\n                \"#Consume RT\",\n                \"#ConsumeOK TPS\",\n                \"#ConsumeFailed TPS\",\n                \"#ConsumeFailedMsgsInHour\"\n            ));\n\n            Iterator<Entry<String, ConsumeStatus>> it = this.statusTable.entrySet().iterator();\n            while (it.hasNext()) {\n                Entry<String, ConsumeStatus> next = it.next();\n                String item = String.format(\"%-32s  %14.2f %14.2f %14.2f %14.2f %18.2f %25d%n\",\n                    next.getKey(),\n                    next.getValue().getPullRT(),\n                    next.getValue().getPullTPS(),\n                    next.getValue().getConsumeRT(),\n                    next.getValue().getConsumeOKTPS(),\n                    next.getValue().getConsumeFailedTPS(),\n                    next.getValue().getConsumeFailedMsgs()\n                );\n\n                sb.append(item);\n            }\n        }\n\n        if (this.userConsumerInfo != null) {\n            sb.append(\"\\n\\n#User Consume Info#\\n\");\n            Iterator<Entry<String, String>> it = this.userConsumerInfo.entrySet().iterator();\n            while (it.hasNext()) {\n                Entry<String, String> next = it.next();\n                String item = String.format(\"%-40s: %s%n\", next.getKey(), next.getValue());\n                sb.append(item);\n            }\n        }\n\n        if (this.jstack != null) {\n            sb.append(\"\\n\\n#Consumer jstack#\\n\");\n            sb.append(this.jstack);\n        }\n\n        return sb.toString();\n    }\n\n    public String getJstack() {\n        return jstack;\n    }\n\n    public void setJstack(String jstack) {\n        this.jstack = jstack;\n    }\n\n    public TreeMap<MessageQueue, PopProcessQueueInfo> getMqPopTable() {\n        return mqPopTable;\n    }\n\n    public void setMqPopTable(\n        TreeMap<MessageQueue, PopProcessQueueInfo> mqPopTable) {\n        this.mqPopTable = mqPopTable;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/CreateTopicListRequestBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.List;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class CreateTopicListRequestBody extends RemotingSerializable {\n    @CFNotNull\n    private List<TopicConfig> topicConfigList;\n\n    public CreateTopicListRequestBody() {}\n\n    public CreateTopicListRequestBody(List<TopicConfig> topicConfigList) {\n        this.topicConfigList = topicConfigList;\n    }\n\n    public List<TopicConfig> getTopicConfigList() {\n        return topicConfigList;\n    }\n\n    public void setTopicConfigList(List<TopicConfig> topicConfigList) {\n        this.topicConfigList = topicConfigList;\n    }\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/ElectMasterResponseBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport com.google.common.base.Objects;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport java.util.HashSet;\nimport java.util.Set;\n\npublic class ElectMasterResponseBody extends RemotingSerializable {\n    private BrokerMemberGroup brokerMemberGroup;\n    private Set<Long> syncStateSet;\n\n    // Provide default constructor for serializer\n    public ElectMasterResponseBody() {\n        this.syncStateSet = new HashSet<Long>();\n        this.brokerMemberGroup = null;\n    }\n\n    public ElectMasterResponseBody(final Set<Long> syncStateSet) {\n        this.syncStateSet = syncStateSet;\n        this.brokerMemberGroup = null;\n    }\n\n    public ElectMasterResponseBody(final BrokerMemberGroup brokerMemberGroup, final Set<Long> syncStateSet) {\n        this.brokerMemberGroup = brokerMemberGroup;\n        this.syncStateSet = syncStateSet;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) {\n            return true;\n        }\n        if (o == null || getClass() != o.getClass()) {\n            return false;\n        }\n        ElectMasterResponseBody that = (ElectMasterResponseBody) o;\n        return Objects.equal(brokerMemberGroup, that.brokerMemberGroup) &&\n            Objects.equal(syncStateSet, that.syncStateSet);\n    }\n\n    @Override\n    public int hashCode() {\n        return Objects.hashCode(brokerMemberGroup, syncStateSet);\n    }\n\n    @Override\n    public String toString() {\n        return \"BrokerMemberGroup{\" +\n            \"brokerMemberGroup='\" + brokerMemberGroup.toString() + '\\'' +\n            \", syncStateSet='\" + syncStateSet.toString() +\n            '}';\n    }\n\n    public void setBrokerMemberGroup(BrokerMemberGroup brokerMemberGroup) {\n        this.brokerMemberGroup = brokerMemberGroup;\n    }\n\n    public BrokerMemberGroup getBrokerMemberGroup() {\n        return brokerMemberGroup;\n    }\n\n    public void setSyncStateSet(Set<Long> syncStateSet) {\n        this.syncStateSet = syncStateSet;\n    }\n\n    public Set<Long> getSyncStateSet() {\n        return syncStateSet;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/EpochEntryCache.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.List;\nimport org.apache.rocketmq.remoting.protocol.EpochEntry;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class EpochEntryCache extends RemotingSerializable {\n    private String clusterName;\n    private String brokerName;\n    private long brokerId;\n    private List<EpochEntry> epochList;\n    private long maxOffset;\n\n    public EpochEntryCache(String clusterName, String brokerName, long brokerId, List<EpochEntry> epochList, long maxOffset) {\n        this.clusterName = clusterName;\n        this.brokerName = brokerName;\n        this.brokerId = brokerId;\n        this.epochList = epochList;\n        this.maxOffset = maxOffset;\n    }\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    public void setClusterName(String clusterName) {\n        this.clusterName = clusterName;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public long getBrokerId() {\n        return brokerId;\n    }\n\n    public void setBrokerId(long brokerId) {\n        this.brokerId = brokerId;\n    }\n\n    public List<EpochEntry> getEpochList() {\n        return this.epochList;\n    }\n\n    public void setEpochList(List<EpochEntry> epochList) {\n        this.epochList = epochList;\n    }\n\n    public long getMaxOffset() {\n        return maxOffset;\n    }\n\n    public void setMaxOffset(long maxOffset) {\n        this.maxOffset = maxOffset;\n    }\n\n    @Override\n    public String toString() {\n        return \"EpochEntryCache{\" +\n            \"clusterName='\" + clusterName + '\\'' +\n            \", brokerName='\" + brokerName + '\\'' +\n            \", brokerId=\" + brokerId +\n            \", epochList=\" + epochList +\n            \", maxOffset=\" + maxOffset +\n            '}';\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetBrokerLiteInfoResponseBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\nimport java.util.Set;\nimport java.util.Map;\n\npublic class GetBrokerLiteInfoResponseBody extends RemotingSerializable {\n\n    private String storeType;\n    private int maxLmqNum;\n    private int currentLmqNum;\n    private int liteSubscriptionCount;\n    private int orderInfoCount;\n    private int cqTableSize;\n    private int offsetTableSize;\n    private int eventMapSize;\n    private Map<String, Integer> topicMeta;\n    private Map<String, Set<String>> groupMeta;\n\n    public String getStoreType() {\n        return storeType;\n    }\n\n    public void setStoreType(String storeType) {\n        this.storeType = storeType;\n    }\n\n    public int getMaxLmqNum() {\n        return maxLmqNum;\n    }\n\n    public void setMaxLmqNum(int maxLmqNum) {\n        this.maxLmqNum = maxLmqNum;\n    }\n\n    public int getCurrentLmqNum() {\n        return currentLmqNum;\n    }\n\n    public void setCurrentLmqNum(int currentLmqNum) {\n        this.currentLmqNum = currentLmqNum;\n    }\n\n    public int getLiteSubscriptionCount() {\n        return liteSubscriptionCount;\n    }\n\n    public void setLiteSubscriptionCount(int liteSubscriptionCount) {\n        this.liteSubscriptionCount = liteSubscriptionCount;\n    }\n\n    public int getOrderInfoCount() {\n        return orderInfoCount;\n    }\n\n    public void setOrderInfoCount(int orderInfoCount) {\n        this.orderInfoCount = orderInfoCount;\n    }\n\n    public int getCqTableSize() {\n        return cqTableSize;\n    }\n\n    public void setCqTableSize(int cqTableSize) {\n        this.cqTableSize = cqTableSize;\n    }\n\n    public int getOffsetTableSize() {\n        return offsetTableSize;\n    }\n\n    public void setOffsetTableSize(int offsetTableSize) {\n        this.offsetTableSize = offsetTableSize;\n    }\n\n    public int getEventMapSize() {\n        return eventMapSize;\n    }\n\n    public void setEventMapSize(int eventMapSize) {\n        this.eventMapSize = eventMapSize;\n    }\n\n    public Map<String, Integer> getTopicMeta() {\n        return topicMeta;\n    }\n\n    public void setTopicMeta(Map<String, Integer> topicMeta) {\n        this.topicMeta = topicMeta;\n    }\n\n    public Map<String, Set<String>> getGroupMeta() {\n        return groupMeta;\n    }\n\n    public void setGroupMeta(Map<String, Set<String>> groupMeta) {\n        this.groupMeta = groupMeta;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetBrokerMemberGroupResponseBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class GetBrokerMemberGroupResponseBody extends RemotingSerializable {\n    // Contains the broker member info of the same broker group\n    private BrokerMemberGroup brokerMemberGroup;\n\n    public BrokerMemberGroup getBrokerMemberGroup() {\n        return brokerMemberGroup;\n    }\n\n    public void setBrokerMemberGroup(final BrokerMemberGroup brokerMemberGroup) {\n        this.brokerMemberGroup = brokerMemberGroup;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetConsumerStatusBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.HashMap;\nimport java.util.Map;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\n@Deprecated\npublic class GetConsumerStatusBody extends RemotingSerializable {\n    private Map<MessageQueue, Long> messageQueueTable = new HashMap<>();\n    private Map<String, Map<MessageQueue, Long>> consumerTable =\n        new HashMap<>();\n\n    public Map<MessageQueue, Long> getMessageQueueTable() {\n        return messageQueueTable;\n    }\n\n    public void setMessageQueueTable(Map<MessageQueue, Long> messageQueueTable) {\n        this.messageQueueTable = messageQueueTable;\n    }\n\n    public Map<String, Map<MessageQueue, Long>> getConsumerTable() {\n        return consumerTable;\n    }\n\n    public void setConsumerTable(Map<String, Map<MessageQueue, Long>> consumerTable) {\n        this.consumerTable = consumerTable;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetLiteClientInfoResponseBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\nimport java.util.Set;\n\npublic class GetLiteClientInfoResponseBody extends RemotingSerializable {\n\n    private String parentTopic;\n    private String group;\n    private String clientId;\n    private long lastAccessTime;\n    private long lastConsumeTime;\n    private int liteTopicCount;\n    private Set<String> liteTopicSet;\n\n    public String getParentTopic() {\n        return parentTopic;\n    }\n\n    public void setParentTopic(String parentTopic) {\n        this.parentTopic = parentTopic;\n    }\n\n    public String getGroup() {\n        return group;\n    }\n\n    public void setGroup(String group) {\n        this.group = group;\n    }\n\n    public String getClientId() {\n        return clientId;\n    }\n\n    public void setClientId(String clientId) {\n        this.clientId = clientId;\n    }\n\n    public long getLastAccessTime() {\n        return lastAccessTime;\n    }\n\n    public void setLastAccessTime(long lastAccessTime) {\n        this.lastAccessTime = lastAccessTime;\n    }\n\n    public long getLastConsumeTime() {\n        return lastConsumeTime;\n    }\n\n    public void setLastConsumeTime(long lastConsumeTime) {\n        this.lastConsumeTime = lastConsumeTime;\n    }\n\n    public int getLiteTopicCount() {\n        return liteTopicCount;\n    }\n\n    public void setLiteTopicCount(int liteTopicCount) {\n        this.liteTopicCount = liteTopicCount;\n    }\n\n    public Set<String> getLiteTopicSet() {\n        return liteTopicSet;\n    }\n\n    public void setLiteTopicSet(Set<String> liteTopicSet) {\n        this.liteTopicSet = liteTopicSet;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetLiteGroupInfoResponseBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.List;\nimport org.apache.rocketmq.common.lite.LiteLagInfo;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.admin.OffsetWrapper;\n\npublic class GetLiteGroupInfoResponseBody extends RemotingSerializable {\n    private String group;\n    private String parentTopic;\n    private String liteTopic;\n    // total log info\n    private long earliestUnconsumedTimestamp = -1;\n    private long totalLagCount;\n    // lite topic detail info\n    private OffsetWrapper liteTopicOffsetWrapper; // if lite topic specified\n    // topK info\n    private List<LiteLagInfo> lagCountTopK;\n    private List<LiteLagInfo> lagTimestampTopK;\n\n    public String getGroup() {\n        return group;\n    }\n\n    public void setGroup(String group) {\n        this.group = group;\n    }\n\n    public String getParentTopic() {\n        return parentTopic;\n    }\n\n    public void setParentTopic(String parentTopic) {\n        this.parentTopic = parentTopic;\n    }\n\n    public String getLiteTopic() {\n        return liteTopic;\n    }\n\n    public void setLiteTopic(String liteTopic) {\n        this.liteTopic = liteTopic;\n    }\n\n    public long getEarliestUnconsumedTimestamp() {\n        return earliestUnconsumedTimestamp;\n    }\n\n    public void setEarliestUnconsumedTimestamp(long earliestUnconsumedTimestamp) {\n        this.earliestUnconsumedTimestamp = earliestUnconsumedTimestamp;\n    }\n\n    public long getTotalLagCount() {\n        return totalLagCount;\n    }\n\n    public void setTotalLagCount(long totalLagCount) {\n        this.totalLagCount = totalLagCount;\n    }\n\n    public OffsetWrapper getLiteTopicOffsetWrapper() {\n        return liteTopicOffsetWrapper;\n    }\n\n    public void setLiteTopicOffsetWrapper(OffsetWrapper liteTopicOffsetWrapper) {\n        this.liteTopicOffsetWrapper = liteTopicOffsetWrapper;\n    }\n\n    public List<LiteLagInfo> getLagCountTopK() {\n        return lagCountTopK;\n    }\n\n    public void setLagCountTopK(List<LiteLagInfo> lagCountTopK) {\n        this.lagCountTopK = lagCountTopK;\n    }\n\n    public List<LiteLagInfo> getLagTimestampTopK() {\n        return lagTimestampTopK;\n    }\n\n    public void setLagTimestampTopK(List<LiteLagInfo> lagTimestampTopK) {\n        this.lagTimestampTopK = lagTimestampTopK;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetLiteTopicInfoResponseBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport org.apache.rocketmq.common.entity.ClientGroup;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.admin.TopicOffset;\n\nimport java.util.Set;\n\npublic class GetLiteTopicInfoResponseBody extends RemotingSerializable {\n\n    private String parentTopic;\n    private String liteTopic;\n    private Set<ClientGroup> subscriber;\n    private TopicOffset topicOffset;\n    private boolean shardingToBroker;\n\n    public String getParentTopic() {\n        return parentTopic;\n    }\n\n    public void setParentTopic(String parentTopic) {\n        this.parentTopic = parentTopic;\n    }\n\n    public String getLiteTopic() {\n        return liteTopic;\n    }\n\n    public void setLiteTopic(String liteTopic) {\n        this.liteTopic = liteTopic;\n    }\n\n    public Set<ClientGroup> getSubscriber() {\n        return subscriber;\n    }\n\n    public void setSubscriber(Set<ClientGroup> subscriber) {\n        this.subscriber = subscriber;\n    }\n\n    public TopicOffset getTopicOffset() {\n        return topicOffset;\n    }\n\n    public void setTopicOffset(TopicOffset topicOffset) {\n        this.topicOffset = topicOffset;\n    }\n\n    public boolean isShardingToBroker() {\n        return shardingToBroker;\n    }\n\n    public void setShardingToBroker(boolean shardingToBroker) {\n        this.shardingToBroker = shardingToBroker;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetParentTopicInfoResponseBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\nimport java.util.Set;\n\npublic class GetParentTopicInfoResponseBody extends RemotingSerializable {\n\n    private String topic;\n    private int ttl;\n    private Set<String> groups;\n    private int lmqNum;\n    private int liteTopicCount;\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public int getTtl() {\n        return ttl;\n    }\n\n    public void setTtl(int ttl) {\n        this.ttl = ttl;\n    }\n\n    public Set<String> getGroups() {\n        return groups;\n    }\n\n    public void setGroups(Set<String> groups) {\n        this.groups = groups;\n    }\n\n    public int getLmqNum() {\n        return lmqNum;\n    }\n\n    public void setLmqNum(int lmqNum) {\n        this.lmqNum = lmqNum;\n    }\n\n    public int getLiteTopicCount() {\n        return liteTopicCount;\n    }\n\n    public void setLiteTopicCount(int liteTopicCount) {\n        this.liteTopicCount = liteTopicCount;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GroupList.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.HashSet;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class GroupList extends RemotingSerializable {\n    private HashSet<String> groupList = new HashSet<>();\n\n    public HashSet<String> getGroupList() {\n        return groupList;\n    }\n\n    public void setGroupList(HashSet<String> groupList) {\n        this.groupList = groupList;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/HARuntimeInfo.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class HARuntimeInfo extends RemotingSerializable {\n\n    private boolean master;\n    private long masterCommitLogMaxOffset;\n    private int inSyncSlaveNums;\n    private List<HAConnectionRuntimeInfo> haConnectionInfo = new ArrayList<>();\n    private HAClientRuntimeInfo haClientRuntimeInfo = new HAClientRuntimeInfo();\n\n    public boolean isMaster() {\n        return this.master;\n    }\n\n    public void setMaster(boolean master) {\n        this.master = master;\n    }\n\n    public long getMasterCommitLogMaxOffset() {\n        return this.masterCommitLogMaxOffset;\n    }\n\n    public void setMasterCommitLogMaxOffset(long masterCommitLogMaxOffset) {\n        this.masterCommitLogMaxOffset = masterCommitLogMaxOffset;\n    }\n\n    public int getInSyncSlaveNums() {\n        return this.inSyncSlaveNums;\n    }\n\n    public void setInSyncSlaveNums(int inSyncSlaveNums) {\n        this.inSyncSlaveNums = inSyncSlaveNums;\n    }\n\n    public List<HAConnectionRuntimeInfo> getHaConnectionInfo() {\n        return this.haConnectionInfo;\n    }\n\n    public void setHaConnectionInfo(List<HAConnectionRuntimeInfo> haConnectionInfo) {\n        this.haConnectionInfo = haConnectionInfo;\n    }\n\n    public HAClientRuntimeInfo getHaClientRuntimeInfo() {\n        return this.haClientRuntimeInfo;\n    }\n\n    public void setHaClientRuntimeInfo(HAClientRuntimeInfo haClientRuntimeInfo) {\n        this.haClientRuntimeInfo = haClientRuntimeInfo;\n    }\n\n    public static class HAConnectionRuntimeInfo extends RemotingSerializable {\n        private String addr;\n        private long slaveAckOffset;\n        private long diff;\n        private boolean inSync;\n        private long transferredByteInSecond;\n        private long transferFromWhere;\n\n        public String getAddr() {\n            return this.addr;\n        }\n\n        public void setAddr(String addr) {\n            this.addr = addr;\n        }\n\n        public long getSlaveAckOffset() {\n            return this.slaveAckOffset;\n        }\n\n        public void setSlaveAckOffset(long slaveAckOffset) {\n            this.slaveAckOffset = slaveAckOffset;\n        }\n\n        public long getDiff() {\n            return this.diff;\n        }\n\n        public void setDiff(long diff) {\n            this.diff = diff;\n        }\n\n        public boolean isInSync() {\n            return this.inSync;\n        }\n\n        public void setInSync(boolean inSync) {\n            this.inSync = inSync;\n        }\n\n        public long getTransferredByteInSecond() {\n            return this.transferredByteInSecond;\n        }\n\n        public void setTransferredByteInSecond(long transferredByteInSecond) {\n            this.transferredByteInSecond = transferredByteInSecond;\n        }\n\n        public long getTransferFromWhere() {\n            return transferFromWhere;\n        }\n\n        public void setTransferFromWhere(long transferFromWhere) {\n            this.transferFromWhere = transferFromWhere;\n        }\n    }\n\n    public static class HAClientRuntimeInfo extends RemotingSerializable {\n        private String masterAddr;\n        private long transferredByteInSecond;\n        private long maxOffset;\n        private long lastReadTimestamp;\n        private long lastWriteTimestamp;\n        private long masterFlushOffset;\n        private boolean isActivated = false;\n\n        public String getMasterAddr() {\n            return this.masterAddr;\n        }\n\n        public void setMasterAddr(String masterAddr) {\n            this.masterAddr = masterAddr;\n        }\n\n        public long getTransferredByteInSecond() {\n            return this.transferredByteInSecond;\n        }\n\n        public void setTransferredByteInSecond(long transferredByteInSecond) {\n            this.transferredByteInSecond = transferredByteInSecond;\n        }\n\n        public long getMaxOffset() {\n            return this.maxOffset;\n        }\n\n        public void setMaxOffset(long maxOffset) {\n            this.maxOffset = maxOffset;\n        }\n\n        public long getLastReadTimestamp() {\n            return this.lastReadTimestamp;\n        }\n\n        public void setLastReadTimestamp(long lastReadTimestamp) {\n            this.lastReadTimestamp = lastReadTimestamp;\n        }\n\n        public long getLastWriteTimestamp() {\n            return this.lastWriteTimestamp;\n        }\n\n        public void setLastWriteTimestamp(long lastWriteTimestamp) {\n            this.lastWriteTimestamp = lastWriteTimestamp;\n        }\n\n        public long getMasterFlushOffset() {\n            return masterFlushOffset;\n        }\n\n        public void setMasterFlushOffset(long masterFlushOffset) {\n            this.masterFlushOffset = masterFlushOffset;\n        }\n    }\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/KVTable.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.HashMap;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class KVTable extends RemotingSerializable {\n    private HashMap<String, String> table = new HashMap<>();\n\n    public HashMap<String, String> getTable() {\n        return table;\n    }\n\n    public void setTable(HashMap<String, String> table) {\n        this.table = table;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/LiteSubscriptionCtlRequestBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.Set;\nimport org.apache.rocketmq.common.lite.LiteSubscriptionDTO;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class LiteSubscriptionCtlRequestBody extends RemotingSerializable {\n\n    private Set<LiteSubscriptionDTO> subscriptionSet;\n\n    public void setSubscriptionSet(Set<LiteSubscriptionDTO> subscriptionSet) {\n        this.subscriptionSet = subscriptionSet;\n    }\n\n    public Set<LiteSubscriptionDTO> getSubscriptionSet() {\n        return subscriptionSet;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/LockBatchRequestBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport com.google.common.base.MoreObjects;\nimport java.util.HashSet;\nimport java.util.Set;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class LockBatchRequestBody extends RemotingSerializable {\n    private String consumerGroup;\n    private String clientId;\n    private boolean onlyThisBroker = false;\n    private Set<MessageQueue> mqSet = new HashSet<>();\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public String getClientId() {\n        return clientId;\n    }\n\n    public void setClientId(String clientId) {\n        this.clientId = clientId;\n    }\n\n    public boolean isOnlyThisBroker() {\n        return onlyThisBroker;\n    }\n\n    public void setOnlyThisBroker(boolean onlyThisBroker) {\n        this.onlyThisBroker = onlyThisBroker;\n    }\n\n    public Set<MessageQueue> getMqSet() {\n        return mqSet;\n    }\n\n    public void setMqSet(Set<MessageQueue> mqSet) {\n        this.mqSet = mqSet;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"consumerGroup\", consumerGroup)\n            .add(\"clientId\", clientId)\n            .add(\"onlyThisBroker\", onlyThisBroker)\n            .add(\"mqSet\", mqSet)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/LockBatchResponseBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.HashSet;\nimport java.util.Set;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class LockBatchResponseBody extends RemotingSerializable {\n\n    private Set<MessageQueue> lockOKMQSet = new HashSet<>();\n\n    public Set<MessageQueue> getLockOKMQSet() {\n        return lockOKMQSet;\n    }\n\n    public void setLockOKMQSet(Set<MessageQueue> lockOKMQSet) {\n        this.lockOKMQSet = lockOKMQSet;\n    }\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/MessageRequestModeSerializeWrapper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class MessageRequestModeSerializeWrapper extends RemotingSerializable {\n\n    private ConcurrentHashMap<String/* Topic */, ConcurrentHashMap<String/* Group */, SetMessageRequestModeRequestBody>>\n            messageRequestModeMap = new ConcurrentHashMap<>();\n\n    public ConcurrentHashMap<String, ConcurrentHashMap<String, SetMessageRequestModeRequestBody>> getMessageRequestModeMap() {\n        return messageRequestModeMap;\n    }\n\n    public void setMessageRequestModeMap(ConcurrentHashMap<String, ConcurrentHashMap<String, SetMessageRequestModeRequestBody>> messageRequestModeMap) {\n        this.messageRequestModeMap = messageRequestModeMap;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/PopProcessQueueInfo.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.body;\n\npublic class PopProcessQueueInfo {\n    private int waitAckCount;\n    private boolean droped;\n    private long lastPopTimestamp;\n\n\n    public int getWaitAckCount() {\n        return waitAckCount;\n    }\n\n\n    public void setWaitAckCount(int waitAckCount) {\n        this.waitAckCount = waitAckCount;\n    }\n\n\n    public boolean isDroped() {\n        return droped;\n    }\n\n\n    public void setDroped(boolean droped) {\n        this.droped = droped;\n    }\n\n\n    public long getLastPopTimestamp() {\n        return lastPopTimestamp;\n    }\n\n\n    public void setLastPopTimestamp(long lastPopTimestamp) {\n        this.lastPopTimestamp = lastPopTimestamp;\n    }\n\n    @Override\n    public String toString() {\n        return \"PopProcessQueueInfo [waitAckCount:\" + waitAckCount +\n                \", droped:\" + droped + \", lastPopTimestamp:\" + lastPopTimestamp + \"]\";\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/ProcessQueueInfo.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport org.apache.rocketmq.common.UtilAll;\n\npublic class ProcessQueueInfo {\n    private long commitOffset;\n\n    private long cachedMsgMinOffset;\n    private long cachedMsgMaxOffset;\n    private int cachedMsgCount;\n    private int cachedMsgSizeInMiB;\n\n    private long transactionMsgMinOffset;\n    private long transactionMsgMaxOffset;\n    private int transactionMsgCount;\n\n    private boolean locked;\n    private long tryUnlockTimes;\n    private long lastLockTimestamp;\n\n    private boolean droped;\n    private long lastPullTimestamp;\n    private long lastConsumeTimestamp;\n\n    public long getCommitOffset() {\n        return commitOffset;\n    }\n\n    public void setCommitOffset(long commitOffset) {\n        this.commitOffset = commitOffset;\n    }\n\n    public long getCachedMsgMinOffset() {\n        return cachedMsgMinOffset;\n    }\n\n    public void setCachedMsgMinOffset(long cachedMsgMinOffset) {\n        this.cachedMsgMinOffset = cachedMsgMinOffset;\n    }\n\n    public long getCachedMsgMaxOffset() {\n        return cachedMsgMaxOffset;\n    }\n\n    public void setCachedMsgMaxOffset(long cachedMsgMaxOffset) {\n        this.cachedMsgMaxOffset = cachedMsgMaxOffset;\n    }\n\n    public int getCachedMsgCount() {\n        return cachedMsgCount;\n    }\n\n    public void setCachedMsgCount(int cachedMsgCount) {\n        this.cachedMsgCount = cachedMsgCount;\n    }\n\n    public long getTransactionMsgMinOffset() {\n        return transactionMsgMinOffset;\n    }\n\n    public void setTransactionMsgMinOffset(long transactionMsgMinOffset) {\n        this.transactionMsgMinOffset = transactionMsgMinOffset;\n    }\n\n    public long getTransactionMsgMaxOffset() {\n        return transactionMsgMaxOffset;\n    }\n\n    public void setTransactionMsgMaxOffset(long transactionMsgMaxOffset) {\n        this.transactionMsgMaxOffset = transactionMsgMaxOffset;\n    }\n\n    public int getTransactionMsgCount() {\n        return transactionMsgCount;\n    }\n\n    public void setTransactionMsgCount(int transactionMsgCount) {\n        this.transactionMsgCount = transactionMsgCount;\n    }\n\n    public boolean isLocked() {\n        return locked;\n    }\n\n    public void setLocked(boolean locked) {\n        this.locked = locked;\n    }\n\n    public long getTryUnlockTimes() {\n        return tryUnlockTimes;\n    }\n\n    public void setTryUnlockTimes(long tryUnlockTimes) {\n        this.tryUnlockTimes = tryUnlockTimes;\n    }\n\n    public long getLastLockTimestamp() {\n        return lastLockTimestamp;\n    }\n\n    public void setLastLockTimestamp(long lastLockTimestamp) {\n        this.lastLockTimestamp = lastLockTimestamp;\n    }\n\n    public boolean isDroped() {\n        return droped;\n    }\n\n    public void setDroped(boolean droped) {\n        this.droped = droped;\n    }\n\n    public long getLastPullTimestamp() {\n        return lastPullTimestamp;\n    }\n\n    public void setLastPullTimestamp(long lastPullTimestamp) {\n        this.lastPullTimestamp = lastPullTimestamp;\n    }\n\n    public long getLastConsumeTimestamp() {\n        return lastConsumeTimestamp;\n    }\n\n    public void setLastConsumeTimestamp(long lastConsumeTimestamp) {\n        this.lastConsumeTimestamp = lastConsumeTimestamp;\n    }\n\n    public int getCachedMsgSizeInMiB() {\n        return cachedMsgSizeInMiB;\n    }\n\n    public void setCachedMsgSizeInMiB(final int cachedMsgSizeInMiB) {\n        this.cachedMsgSizeInMiB = cachedMsgSizeInMiB;\n    }\n\n    @Override\n    public String toString() {\n        return \"ProcessQueueInfo [commitOffset=\" + commitOffset + \", cachedMsgMinOffset=\"\n            + cachedMsgMinOffset + \", cachedMsgMaxOffset=\" + cachedMsgMaxOffset\n            + \", cachedMsgCount=\" + cachedMsgCount + \", cachedMsgSizeInMiB=\" + cachedMsgSizeInMiB\n            + \", transactionMsgMinOffset=\" + transactionMsgMinOffset\n            + \", transactionMsgMaxOffset=\" + transactionMsgMaxOffset + \", transactionMsgCount=\"\n            + transactionMsgCount + \", locked=\" + locked + \", tryUnlockTimes=\" + tryUnlockTimes\n            + \", lastLockTimestamp=\" + UtilAll.timeMillisToHumanString(lastLockTimestamp) + \", droped=\"\n            + droped + \", lastPullTimestamp=\" + UtilAll.timeMillisToHumanString(lastPullTimestamp)\n            + \", lastConsumeTimestamp=\" + UtilAll.timeMillisToHumanString(lastConsumeTimestamp) + \"]\";\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/ProducerConnection.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.HashSet;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class ProducerConnection extends RemotingSerializable {\n    private HashSet<Connection> connectionSet = new HashSet<>();\n\n    public HashSet<Connection> getConnectionSet() {\n        return connectionSet;\n    }\n\n    public void setConnectionSet(HashSet<Connection> connectionSet) {\n        this.connectionSet = connectionSet;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/ProducerInfo.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\n\npublic class ProducerInfo extends RemotingSerializable {\n    private String clientId;\n    private String remoteIP;\n    private LanguageCode language;\n    private int version;\n    private long lastUpdateTimestamp;\n\n    public ProducerInfo(String clientId, String remoteIP, LanguageCode language, int version, long lastUpdateTimestamp) {\n        this.clientId = clientId;\n        this.remoteIP = remoteIP;\n        this.language = language;\n        this.version = version;\n        this.lastUpdateTimestamp = lastUpdateTimestamp;\n    }\n\n    public String getClientId() {\n        return clientId;\n    }\n\n    public void setClientId(String clientId) {\n        this.clientId = clientId;\n    }\n\n    public String getRemoteIP() {\n        return remoteIP;\n    }\n\n    public void setRemoteIP(String remoteIP) {\n        this.remoteIP = remoteIP;\n    }\n\n    public LanguageCode getLanguage() {\n        return language;\n    }\n\n    public void setLanguage(LanguageCode language) {\n        this.language = language;\n    }\n\n    public int getVersion() {\n        return version;\n    }\n\n    public void setVersion(int version) {\n        this.version = version;\n    }\n\n    public long getLastUpdateTimestamp() {\n        return lastUpdateTimestamp;\n    }\n\n    public void setLastUpdateTimestamp(long lastUpdateTimestamp) {\n        this.lastUpdateTimestamp = lastUpdateTimestamp;\n    }\n\n    @Override\n    public String toString() {\n        return String.format(\"clientId=%s,remoteIP=%s, language=%s, version=%d, lastUpdateTimestamp=%d\",\n                clientId, remoteIP, language.name(), version, lastUpdateTimestamp);\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/ProducerTableInfo.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.List;\nimport java.util.Map;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class ProducerTableInfo extends RemotingSerializable {\n    public ProducerTableInfo(Map<String, List<ProducerInfo>> data) {\n        this.data = data;\n    }\n\n    private Map<String, List<ProducerInfo>> data;\n\n    public Map<String, List<ProducerInfo>> getData() {\n        return data;\n    }\n\n    public void setData(Map<String, List<ProducerInfo>> data) {\n        this.data = data;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/QueryAssignmentRequestBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\n\npublic class QueryAssignmentRequestBody extends RemotingSerializable {\n\n    private String topic;\n\n    private String consumerGroup;\n\n    private String clientId;\n\n    private String strategyName;\n\n    private MessageModel messageModel;\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public String getClientId() {\n        return clientId;\n    }\n\n    public void setClientId(String clientId) {\n        this.clientId = clientId;\n    }\n\n    public String getStrategyName() {\n        return strategyName;\n    }\n\n    public void setStrategyName(String strategyName) {\n        this.strategyName = strategyName;\n    }\n\n    public MessageModel getMessageModel() {\n        return messageModel;\n    }\n\n    public void setMessageModel(MessageModel messageModel) {\n        this.messageModel = messageModel;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/QueryAssignmentResponseBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.Set;\nimport org.apache.rocketmq.common.message.MessageQueueAssignment;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class QueryAssignmentResponseBody extends RemotingSerializable {\n\n    private Set<MessageQueueAssignment> messageQueueAssignments;\n\n    public Set<MessageQueueAssignment> getMessageQueueAssignments() {\n        return messageQueueAssignments;\n    }\n\n    public void setMessageQueueAssignments(\n        Set<MessageQueueAssignment> messageQueueAssignments) {\n        this.messageQueueAssignments = messageQueueAssignments;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/QueryConsumeQueueResponseBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.List;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\npublic class QueryConsumeQueueResponseBody extends RemotingSerializable {\n\n    private SubscriptionData subscriptionData;\n    private String filterData;\n    private List<ConsumeQueueData> queueData;\n    private long maxQueueIndex;\n    private long minQueueIndex;\n\n    public SubscriptionData getSubscriptionData() {\n        return subscriptionData;\n    }\n\n    public void setSubscriptionData(SubscriptionData subscriptionData) {\n        this.subscriptionData = subscriptionData;\n    }\n\n    public String getFilterData() {\n        return filterData;\n    }\n\n    public void setFilterData(String filterData) {\n        this.filterData = filterData;\n    }\n\n    public List<ConsumeQueueData> getQueueData() {\n        return queueData;\n    }\n\n    public void setQueueData(List<ConsumeQueueData> queueData) {\n        this.queueData = queueData;\n    }\n\n    public long getMaxQueueIndex() {\n        return maxQueueIndex;\n    }\n\n    public void setMaxQueueIndex(long maxQueueIndex) {\n        this.maxQueueIndex = maxQueueIndex;\n    }\n\n    public long getMinQueueIndex() {\n        return minQueueIndex;\n    }\n\n    public void setMinQueueIndex(long minQueueIndex) {\n        this.minQueueIndex = minQueueIndex;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/QueryConsumeTimeSpanBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class QueryConsumeTimeSpanBody extends RemotingSerializable {\n    List<QueueTimeSpan> consumeTimeSpanSet = new ArrayList<>();\n\n    public List<QueueTimeSpan> getConsumeTimeSpanSet() {\n        return consumeTimeSpanSet;\n    }\n\n    public void setConsumeTimeSpanSet(List<QueueTimeSpan> consumeTimeSpanSet) {\n        this.consumeTimeSpanSet = consumeTimeSpanSet;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/QueryCorrectionOffsetBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.HashMap;\nimport java.util.Map;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class QueryCorrectionOffsetBody extends RemotingSerializable {\n    private Map<Integer, Long> correctionOffsets = new HashMap<>();\n\n    public Map<Integer, Long> getCorrectionOffsets() {\n        return correctionOffsets;\n    }\n\n    public void setCorrectionOffsets(Map<Integer, Long> correctionOffsets) {\n        this.correctionOffsets = correctionOffsets;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/QuerySubscriptionResponseBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\npublic class QuerySubscriptionResponseBody extends RemotingSerializable {\n\n    private SubscriptionData subscriptionData;\n    private String group;\n    private String topic;\n\n    public SubscriptionData getSubscriptionData() {\n        return subscriptionData;\n    }\n\n    public void setSubscriptionData(SubscriptionData subscriptionData) {\n        this.subscriptionData = subscriptionData;\n    }\n\n    public String getGroup() {\n        return group;\n    }\n\n    public void setGroup(String group) {\n        this.group = group;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/QueueTimeSpan.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.Date;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\npublic class QueueTimeSpan {\n    private MessageQueue messageQueue;\n    private long minTimeStamp;\n    private long maxTimeStamp;\n    private long consumeTimeStamp;\n    private long delayTime;\n\n    public MessageQueue getMessageQueue() {\n        return messageQueue;\n    }\n\n    public void setMessageQueue(MessageQueue messageQueue) {\n        this.messageQueue = messageQueue;\n    }\n\n    public long getMinTimeStamp() {\n        return minTimeStamp;\n    }\n\n    public void setMinTimeStamp(long minTimeStamp) {\n        this.minTimeStamp = minTimeStamp;\n    }\n\n    public long getMaxTimeStamp() {\n        return maxTimeStamp;\n    }\n\n    public void setMaxTimeStamp(long maxTimeStamp) {\n        this.maxTimeStamp = maxTimeStamp;\n    }\n\n    public long getConsumeTimeStamp() {\n        return consumeTimeStamp;\n    }\n\n    public void setConsumeTimeStamp(long consumeTimeStamp) {\n        this.consumeTimeStamp = consumeTimeStamp;\n    }\n\n    public String getMinTimeStampStr() {\n        return UtilAll.formatDate(new Date(minTimeStamp), UtilAll.YYYY_MM_DD_HH_MM_SS_SSS);\n    }\n\n    public String getMaxTimeStampStr() {\n        return UtilAll.formatDate(new Date(maxTimeStamp), UtilAll.YYYY_MM_DD_HH_MM_SS_SSS);\n    }\n\n    public String getConsumeTimeStampStr() {\n        return UtilAll.formatDate(new Date(consumeTimeStamp), UtilAll.YYYY_MM_DD_HH_MM_SS_SSS);\n    }\n\n    public long getDelayTime() {\n        return delayTime;\n    }\n\n    public void setDelayTime(long delayTime) {\n        this.delayTime = delayTime;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/RegisterBrokerBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport com.alibaba.fastjson2.JSON;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo;\n\nimport java.io.ByteArrayInputStream;\nimport java.io.ByteArrayOutputStream;\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.zip.Deflater;\nimport java.util.zip.DeflaterOutputStream;\nimport java.util.zip.InflaterInputStream;\n\npublic class RegisterBrokerBody extends RemotingSerializable {\n\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);\n    private TopicConfigAndMappingSerializeWrapper topicConfigSerializeWrapper = new TopicConfigAndMappingSerializeWrapper();\n    private List<String> filterServerList = new ArrayList<>();\n    private static final long MINIMUM_TAKE_TIME_MILLISECOND = 50;\n\n    public byte[] encode(boolean compress) {\n\n        if (!compress) {\n            return super.encode();\n        }\n        long start = System.currentTimeMillis();\n        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();\n        try (DeflaterOutputStream outputStream = new DeflaterOutputStream(byteArrayOutputStream, new Deflater(Deflater.BEST_COMPRESSION))) {\n            DataVersion dataVersion = topicConfigSerializeWrapper.getDataVersion();\n            ConcurrentMap<String, TopicConfig> topicConfigTable = cloneTopicConfigTable(topicConfigSerializeWrapper.getTopicConfigTable());\n            assert topicConfigTable != null;\n            byte[] buffer = dataVersion.encode();\n\n            // write data version\n            outputStream.write(convertIntToByteArray(buffer.length));\n            outputStream.write(buffer);\n\n            int topicNumber = topicConfigTable.size();\n\n            // write number of topic configs\n            outputStream.write(convertIntToByteArray(topicNumber));\n\n            // write topic config entry one by one.\n            for (ConcurrentMap.Entry<String, TopicConfig> next : topicConfigTable.entrySet()) {\n                buffer = next.getValue().encode().getBytes(MixAll.DEFAULT_CHARSET);\n                outputStream.write(convertIntToByteArray(buffer.length));\n                outputStream.write(buffer);\n            }\n\n            buffer = JSON.toJSONString(filterServerList).getBytes(MixAll.DEFAULT_CHARSET);\n\n            // write filter server list json length\n            outputStream.write(convertIntToByteArray(buffer.length));\n\n            // write filter server list json\n            outputStream.write(buffer);\n\n            //write the topic queue mapping\n            Map<String, TopicQueueMappingInfo> topicQueueMappingInfoMap = topicConfigSerializeWrapper.getTopicQueueMappingInfoMap();\n            if (topicQueueMappingInfoMap == null) {\n                //as the placeholder\n                topicQueueMappingInfoMap = new ConcurrentHashMap<>();\n            }\n            outputStream.write(convertIntToByteArray(topicQueueMappingInfoMap.size()));\n            for (TopicQueueMappingInfo info: topicQueueMappingInfoMap.values()) {\n                buffer = JSON.toJSONString(info).getBytes(MixAll.DEFAULT_CHARSET);\n                outputStream.write(convertIntToByteArray(buffer.length));\n                // write filter server list json\n                outputStream.write(buffer);\n            }\n\n            outputStream.finish();\n            long takeTime = System.currentTimeMillis() - start;\n            if (takeTime > MINIMUM_TAKE_TIME_MILLISECOND) {\n                LOGGER.info(\"Compressing takes {}ms\", takeTime);\n            }\n            return byteArrayOutputStream.toByteArray();\n        } catch (IOException e) {\n            LOGGER.error(\"Failed to compress RegisterBrokerBody object\", e);\n        }\n\n        return null;\n    }\n\n    public static RegisterBrokerBody decode(byte[] data, boolean compressed, MQVersion.Version brokerVersion) throws IOException {\n        if (!compressed) {\n            return RegisterBrokerBody.decode(data, RegisterBrokerBody.class);\n        }\n        long start = System.currentTimeMillis();\n        try (InflaterInputStream inflaterInputStream = new InflaterInputStream(new ByteArrayInputStream(data))) {\n            int dataVersionLength = readInt(inflaterInputStream);\n            byte[] dataVersionBytes = readBytes(inflaterInputStream, dataVersionLength);\n            DataVersion dataVersion = DataVersion.decode(dataVersionBytes, DataVersion.class);\n\n            RegisterBrokerBody registerBrokerBody = new RegisterBrokerBody();\n            registerBrokerBody.getTopicConfigSerializeWrapper().setDataVersion(dataVersion);\n            ConcurrentMap<String, TopicConfig> topicConfigTable = registerBrokerBody.getTopicConfigSerializeWrapper().getTopicConfigTable();\n\n            int topicConfigNumber = readInt(inflaterInputStream);\n            LOGGER.debug(\"{} topic configs to extract\", topicConfigNumber);\n\n            for (int i = 0; i < topicConfigNumber; i++) {\n                int topicConfigJsonLength = readInt(inflaterInputStream);\n                byte[] buffer = readBytes(inflaterInputStream, topicConfigJsonLength);\n                TopicConfig topicConfig = new TopicConfig();\n                String topicConfigJson = new String(buffer, MixAll.DEFAULT_CHARSET);\n                topicConfig.decode(topicConfigJson);\n                topicConfigTable.put(topicConfig.getTopicName(), topicConfig);\n            }\n\n            int filterServerListJsonLength = readInt(inflaterInputStream);\n            byte[] filterServerListBuffer = readBytes(inflaterInputStream, filterServerListJsonLength);\n            String filterServerListJson = new String(filterServerListBuffer, MixAll.DEFAULT_CHARSET);\n            List<String> filterServerList = new ArrayList<>();\n            try {\n                filterServerList = JSON.parseArray(filterServerListJson, String.class);\n            } catch (Exception e) {\n                LOGGER.error(\"Decompressing occur Exception {}\", filterServerListJson, e);\n            }\n            registerBrokerBody.setFilterServerList(filterServerList);\n\n            if (brokerVersion.ordinal() >= MQVersion.Version.V5_0_0.ordinal()) {\n                int topicQueueMappingNum = readInt(inflaterInputStream);\n                Map<String, TopicQueueMappingInfo> topicQueueMappingInfoMap = new ConcurrentHashMap<>();\n                for (int i = 0; i < topicQueueMappingNum; i++) {\n                    int mappingJsonLen = readInt(inflaterInputStream);\n                    byte[] buffer = readBytes(inflaterInputStream, mappingJsonLen);\n                    TopicQueueMappingInfo info = TopicQueueMappingInfo.decode(buffer, TopicQueueMappingInfo.class);\n                    topicQueueMappingInfoMap.put(info.getTopic(), info);\n                }\n                registerBrokerBody.getTopicConfigSerializeWrapper().setTopicQueueMappingInfoMap(topicQueueMappingInfoMap);\n            }\n\n            long takeTime = System.currentTimeMillis() - start;\n            if (takeTime > MINIMUM_TAKE_TIME_MILLISECOND) {\n                LOGGER.info(\"Decompressing takes {}ms\", takeTime);\n            }\n            return registerBrokerBody;\n        }\n    }\n\n    private static byte[] convertIntToByteArray(int n) {\n        ByteBuffer byteBuffer = ByteBuffer.allocate(4);\n        byteBuffer.putInt(n);\n        return byteBuffer.array();\n    }\n\n    private static byte[] readBytes(InflaterInputStream inflaterInputStream, int length) throws IOException {\n        byte[] buffer = new byte[length];\n        int bytesRead = 0;\n        while (bytesRead < length) {\n            int len = inflaterInputStream.read(buffer, bytesRead, length - bytesRead);\n            if (len == -1) {\n                throw new IOException(\"End of compressed data has reached\");\n            } else {\n                bytesRead += len;\n            }\n        }\n        return buffer;\n    }\n\n    private static int readInt(InflaterInputStream inflaterInputStream) throws IOException {\n        byte[] buffer = readBytes(inflaterInputStream, 4);\n        ByteBuffer byteBuffer = ByteBuffer.wrap(buffer);\n        return byteBuffer.getInt();\n    }\n\n    public TopicConfigAndMappingSerializeWrapper getTopicConfigSerializeWrapper() {\n        return topicConfigSerializeWrapper;\n    }\n\n    public void setTopicConfigSerializeWrapper(TopicConfigAndMappingSerializeWrapper topicConfigSerializeWrapper) {\n        this.topicConfigSerializeWrapper = topicConfigSerializeWrapper;\n    }\n\n    public List<String> getFilterServerList() {\n        return filterServerList;\n    }\n\n    public void setFilterServerList(List<String> filterServerList) {\n        this.filterServerList = filterServerList;\n    }\n\n    private ConcurrentMap<String, TopicConfig> cloneTopicConfigTable(\n        ConcurrentMap<String, TopicConfig> topicConfigConcurrentMap) {\n        if (topicConfigConcurrentMap == null) {\n            return null;\n        }\n        ConcurrentHashMap<String, TopicConfig> result = new ConcurrentHashMap<>(topicConfigConcurrentMap.size());\n        result.putAll(topicConfigConcurrentMap);\n        return result;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/ResetOffsetBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.HashMap;\nimport java.util.Map;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class ResetOffsetBody extends RemotingSerializable {\n\n    private Map<MessageQueue, Long> offsetTable;\n\n    public ResetOffsetBody() {\n        offsetTable = new HashMap<>();\n    }\n\n    public Map<MessageQueue, Long> getOffsetTable() {\n        return offsetTable;\n    }\n\n    public void setOffsetTable(Map<MessageQueue, Long> offsetTable) {\n        this.offsetTable = offsetTable;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/ResetOffsetBodyForC.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.List;\nimport org.apache.rocketmq.common.message.MessageQueueForC;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class ResetOffsetBodyForC extends RemotingSerializable {\n\n    private List<MessageQueueForC> offsetTable;\n\n    public List<MessageQueueForC> getOffsetTable() {\n        return offsetTable;\n    }\n\n    public void setOffsetTable(List<MessageQueueForC> offsetTable) {\n        this.offsetTable = offsetTable;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/RoleChangeNotifyEntry.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\n\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;\n\nimport java.util.Set;\n\npublic class RoleChangeNotifyEntry {\n\n    private final BrokerMemberGroup brokerMemberGroup;\n\n    private final String masterAddress;\n\n    private final Long masterBrokerId;\n\n    private final int masterEpoch;\n\n    private final int syncStateSetEpoch;\n\n    private final Set<Long> syncStateSet;\n\n    public RoleChangeNotifyEntry(BrokerMemberGroup brokerMemberGroup, String masterAddress, Long masterBrokerId, int masterEpoch, int syncStateSetEpoch, Set<Long> syncStateSet) {\n        this.brokerMemberGroup = brokerMemberGroup;\n        this.masterAddress = masterAddress;\n        this.masterEpoch = masterEpoch;\n        this.syncStateSetEpoch = syncStateSetEpoch;\n        this.masterBrokerId = masterBrokerId;\n        this.syncStateSet = syncStateSet;\n    }\n\n    public static RoleChangeNotifyEntry convert(RemotingCommand electMasterResponse) {\n        final ElectMasterResponseHeader header = (ElectMasterResponseHeader) electMasterResponse.readCustomHeader();\n        BrokerMemberGroup brokerMemberGroup = null;\n        Set<Long> syncStateSet = null;\n\n        if (electMasterResponse.getBody() != null && electMasterResponse.getBody().length > 0) {\n            ElectMasterResponseBody body = RemotingSerializable.decode(electMasterResponse.getBody(), ElectMasterResponseBody.class);\n            brokerMemberGroup = body.getBrokerMemberGroup();\n            syncStateSet = body.getSyncStateSet();\n        }\n\n        return new RoleChangeNotifyEntry(brokerMemberGroup, header.getMasterAddress(), header.getMasterBrokerId(), header.getMasterEpoch(), header.getSyncStateSetEpoch(), syncStateSet);\n    }\n\n\n    public BrokerMemberGroup getBrokerMemberGroup() {\n        return brokerMemberGroup;\n    }\n\n    public String getMasterAddress() {\n        return masterAddress;\n    }\n\n    public int getMasterEpoch() {\n        return masterEpoch;\n    }\n\n    public int getSyncStateSetEpoch() {\n        return syncStateSetEpoch;\n    }\n\n    public Long getMasterBrokerId() {\n        return masterBrokerId;\n    }\n\n    public Set<Long> getSyncStateSet() {\n        return syncStateSet;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/SetMessageRequestModeRequestBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport org.apache.rocketmq.common.message.MessageRequestMode;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class SetMessageRequestModeRequestBody extends RemotingSerializable {\n\n    private String topic;\n\n    private String consumerGroup;\n\n    private MessageRequestMode mode = MessageRequestMode.PULL;\n\n    /*\n    consumer working in pop mode could share the MessageQueues assigned to the N (N = popShareQueueNum) consumers following it in the cid list\n     */\n    private int popShareQueueNum = 0;\n\n    public SetMessageRequestModeRequestBody() {\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public MessageRequestMode getMode() {\n        return mode;\n    }\n\n    public void setMode(MessageRequestMode mode) {\n        this.mode = mode;\n    }\n\n    public int getPopShareQueueNum() {\n        return popShareQueueNum;\n    }\n\n    public void setPopShareQueueNum(int popShareQueueNum) {\n        this.popShareQueueNum = popShareQueueNum;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/SubscriptionGroupList.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.List;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\n\npublic class SubscriptionGroupList extends RemotingSerializable {\n    @CFNotNull\n    private List<SubscriptionGroupConfig> groupConfigList;\n\n    public SubscriptionGroupList() {}\n\n    public SubscriptionGroupList(List<SubscriptionGroupConfig> groupConfigList) {\n        this.groupConfigList = groupConfigList;\n    }\n\n    public List<SubscriptionGroupConfig> getGroupConfigList() {\n        return groupConfigList;\n    }\n\n    public void setGroupConfigList(List<SubscriptionGroupConfig> groupConfigList) {\n        this.groupConfigList = groupConfigList;\n    }\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/SubscriptionGroupWrapper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\n\npublic class SubscriptionGroupWrapper extends RemotingSerializable {\n    private ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroupTable =\n        new ConcurrentHashMap<>(1024);\n    private ConcurrentMap<String, ConcurrentMap<String, Integer>> forbiddenTable =\n        new ConcurrentHashMap<>(1024);\n    private DataVersion dataVersion = new DataVersion();\n\n    public ConcurrentMap<String, SubscriptionGroupConfig> getSubscriptionGroupTable() {\n        return subscriptionGroupTable;\n    }\n\n    public void setSubscriptionGroupTable(\n        ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroupTable) {\n        this.subscriptionGroupTable = subscriptionGroupTable;\n    }\n\n    public ConcurrentMap<String, ConcurrentMap<String, Integer>> getForbiddenTable() {\n        return forbiddenTable;\n    }\n\n    public void setForbiddenTable(ConcurrentMap<String, ConcurrentMap<String, Integer>> forbiddenTable) {\n        this.forbiddenTable = forbiddenTable;\n    }\n\n    public DataVersion getDataVersion() {\n        return dataVersion;\n    }\n\n    public void setDataVersion(DataVersion dataVersion) {\n        this.dataVersion = dataVersion;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/SyncStateSet.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.HashSet;\nimport java.util.Set;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class SyncStateSet extends RemotingSerializable {\n    private Set<Long> syncStateSet;\n    private int syncStateSetEpoch;\n\n    public SyncStateSet(Set<Long> syncStateSet, int syncStateSetEpoch) {\n        this.syncStateSet = new HashSet<>(syncStateSet);\n        this.syncStateSetEpoch = syncStateSetEpoch;\n    }\n\n    public Set<Long> getSyncStateSet() {\n        return new HashSet<>(syncStateSet);\n    }\n\n    public void setSyncStateSet(Set<Long> syncStateSet) {\n        this.syncStateSet = new HashSet<>(syncStateSet);\n    }\n\n    public int getSyncStateSetEpoch() {\n        return syncStateSetEpoch;\n    }\n\n    public void setSyncStateSetEpoch(int syncStateSetEpoch) {\n        this.syncStateSetEpoch = syncStateSetEpoch;\n    }\n\n    @Override\n    public String toString() {\n        return \"SyncStateSet{\" +\n            \"syncStateSet=\" + syncStateSet +\n            \", syncStateSetEpoch=\" + syncStateSetEpoch +\n            '}';\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/TopicConfigAndMappingSerializeWrapper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingDetail;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo;\n\npublic class TopicConfigAndMappingSerializeWrapper extends TopicConfigSerializeWrapper {\n    private Map<String/* topic */, TopicQueueMappingInfo> topicQueueMappingInfoMap = new ConcurrentHashMap<>();\n\n    private Map<String/* topic */, TopicQueueMappingDetail> topicQueueMappingDetailMap = new ConcurrentHashMap<>();\n\n    private DataVersion mappingDataVersion = new DataVersion();\n\n\n    public Map<String, TopicQueueMappingInfo> getTopicQueueMappingInfoMap() {\n        return topicQueueMappingInfoMap;\n    }\n\n    public void setTopicQueueMappingInfoMap(Map<String, TopicQueueMappingInfo> topicQueueMappingInfoMap) {\n        this.topicQueueMappingInfoMap = topicQueueMappingInfoMap;\n    }\n\n    public Map<String, TopicQueueMappingDetail> getTopicQueueMappingDetailMap() {\n        return topicQueueMappingDetailMap;\n    }\n\n    public void setTopicQueueMappingDetailMap(Map<String, TopicQueueMappingDetail> topicQueueMappingDetailMap) {\n        this.topicQueueMappingDetailMap = topicQueueMappingDetailMap;\n    }\n\n    public DataVersion getMappingDataVersion() {\n        return mappingDataVersion;\n    }\n\n    public void setMappingDataVersion(DataVersion mappingDataVersion) {\n        this.mappingDataVersion = mappingDataVersion;\n    }\n\n    public static TopicConfigAndMappingSerializeWrapper from(TopicConfigSerializeWrapper wrapper) {\n        if (wrapper instanceof  TopicConfigAndMappingSerializeWrapper) {\n            return (TopicConfigAndMappingSerializeWrapper) wrapper;\n        }\n        TopicConfigAndMappingSerializeWrapper mappingSerializeWrapper =  new TopicConfigAndMappingSerializeWrapper();\n        mappingSerializeWrapper.setDataVersion(wrapper.getDataVersion());\n        mappingSerializeWrapper.setTopicConfigTable(wrapper.getTopicConfigTable());\n        return mappingSerializeWrapper;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/TopicConfigSerializeWrapper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class TopicConfigSerializeWrapper extends RemotingSerializable {\n    private ConcurrentMap<String, TopicConfig> topicConfigTable =\n        new ConcurrentHashMap<>();\n    private DataVersion dataVersion = new DataVersion();\n\n    public ConcurrentMap<String, TopicConfig> getTopicConfigTable() {\n        return topicConfigTable;\n    }\n\n    public void setTopicConfigTable(ConcurrentMap<String, TopicConfig> topicConfigTable) {\n        this.topicConfigTable = topicConfigTable;\n    }\n\n    public DataVersion getDataVersion() {\n        return dataVersion;\n    }\n\n    public void setDataVersion(DataVersion dataVersion) {\n        this.dataVersion = dataVersion;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/TopicList.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class TopicList extends RemotingSerializable {\n    private Set<String> topicList = ConcurrentHashMap.newKeySet();\n    private String brokerAddr;\n\n    public Set<String> getTopicList() {\n        return topicList;\n    }\n\n    public void setTopicList(Set<String> topicList) {\n        this.topicList = topicList;\n    }\n\n    public String getBrokerAddr() {\n        return brokerAddr;\n    }\n\n    public void setBrokerAddr(String brokerAddr) {\n        this.brokerAddr = brokerAddr;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/TopicQueueMappingSerializeWrapper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.Map;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingDetail;\n\npublic class TopicQueueMappingSerializeWrapper extends RemotingSerializable {\n    private Map<String/* topic */, TopicQueueMappingDetail> topicQueueMappingInfoMap;\n    private DataVersion dataVersion = new DataVersion();\n\n    public Map<String, TopicQueueMappingDetail> getTopicQueueMappingInfoMap() {\n        return topicQueueMappingInfoMap;\n    }\n\n    public void setTopicQueueMappingInfoMap(Map<String, TopicQueueMappingDetail> topicQueueMappingInfoMap) {\n        this.topicQueueMappingInfoMap = topicQueueMappingInfoMap;\n    }\n\n    public DataVersion getDataVersion() {\n        return dataVersion;\n    }\n\n    public void setDataVersion(DataVersion dataVersion) {\n        this.dataVersion = dataVersion;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/UnlockBatchRequestBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport com.google.common.base.MoreObjects;\nimport java.util.HashSet;\nimport java.util.Set;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class UnlockBatchRequestBody extends RemotingSerializable {\n    private String consumerGroup;\n    private String clientId;\n    private boolean onlyThisBroker = false;\n    private Set<MessageQueue> mqSet = new HashSet<>();\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public String getClientId() {\n        return clientId;\n    }\n\n    public void setClientId(String clientId) {\n        this.clientId = clientId;\n    }\n\n    public boolean isOnlyThisBroker() {\n        return onlyThisBroker;\n    }\n\n    public void setOnlyThisBroker(boolean onlyThisBroker) {\n        this.onlyThisBroker = onlyThisBroker;\n    }\n\n    public Set<MessageQueue> getMqSet() {\n        return mqSet;\n    }\n\n    public void setMqSet(Set<MessageQueue> mqSet) {\n        this.mqSet = mqSet;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"consumerGroup\", consumerGroup)\n            .add(\"clientId\", clientId)\n            .add(\"onlyThisBroker\", onlyThisBroker)\n            .add(\"mqSet\", mqSet)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/UserInfo.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.body;\n\npublic class UserInfo {\n\n    private String username;\n\n    private String password;\n\n    private String userType;\n\n    private String userStatus;\n\n    public static UserInfo of(String username, String password, String userType) {\n        UserInfo userInfo = new UserInfo();\n        userInfo.setUsername(username);\n        userInfo.setPassword(password);\n        userInfo.setUserType(userType);\n        return userInfo;\n    }\n\n    public static UserInfo of(String username, String password, String userType, String userStatus) {\n        UserInfo userInfo = new UserInfo();\n        userInfo.setUsername(username);\n        userInfo.setPassword(password);\n        userInfo.setUserType(userType);\n        userInfo.setUserStatus(userStatus);\n        return userInfo;\n    }\n\n    public String getUsername() {\n        return username;\n    }\n\n    public void setUsername(String username) {\n        this.username = username;\n    }\n\n    public String getPassword() {\n        return password;\n    }\n\n    public void setPassword(String password) {\n        this.password = password;\n    }\n\n    public String getUserType() {\n        return userType;\n    }\n\n    public void setUserType(String userType) {\n        this.userType = userType;\n    }\n\n    public String getUserStatus() {\n        return userStatus;\n    }\n\n    public void setUserStatus(String userStatus) {\n        this.userStatus = userStatus;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/filter/FilterAPI.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.filter;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\nimport java.util.Arrays;\n\npublic class FilterAPI {\n\n    public static SubscriptionData buildSubscriptionData(String topic, String subString) throws Exception {\n        final SubscriptionData subscriptionData = new SubscriptionData();\n        subscriptionData.setTopic(topic);\n        subscriptionData.setSubString(subString);\n\n        if (StringUtils.isEmpty(subString) || subString.equals(SubscriptionData.SUB_ALL)) {\n            subscriptionData.setSubString(SubscriptionData.SUB_ALL);\n            return subscriptionData;\n        }\n        String[] tags = subString.split(\"\\\\|\\\\|\");\n        if (tags.length > 0) {\n            Arrays.stream(tags).map(String::trim).filter(tag -> !tag.isEmpty()).forEach(tag -> {\n                subscriptionData.getTagsSet().add(tag);\n                subscriptionData.getCodeSet().add(tag.hashCode());\n            });\n        } else {\n            throw new Exception(\"subString split error\");\n        }\n\n        return subscriptionData;\n    }\n\n    public static SubscriptionData buildSubscriptionData(String topic, String subString, String expressionType) throws Exception {\n        final SubscriptionData subscriptionData = buildSubscriptionData(topic, subString);\n        if (StringUtils.isNotBlank(expressionType)) {\n            subscriptionData.setExpressionType(expressionType);\n        }\n        return subscriptionData;\n    }\n\n    public static SubscriptionData build(final String topic, final String subString,\n        final String type) throws Exception {\n        if (ExpressionType.TAG.equals(type) || type == null) {\n            return buildSubscriptionData(topic, subString);\n        }\n\n        if (StringUtils.isEmpty(subString)) {\n            throw new IllegalArgumentException(\"Expression can't be null! \" + type);\n        }\n\n        SubscriptionData subscriptionData = new SubscriptionData();\n        subscriptionData.setTopic(topic);\n        subscriptionData.setSubString(subString);\n        subscriptionData.setExpressionType(type);\n\n        return subscriptionData;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/AckMessageRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;\n\n@RocketMQAction(value = RequestCode.ACK_MESSAGE, action = Action.SUB)\npublic class AckMessageRequestHeader extends TopicQueueRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String consumerGroup;\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNotNull\n    private Integer queueId;\n    @CFNotNull\n    private String extraInfo;\n\n    @CFNotNull\n    private Long offset;\n\n    private String liteTopic;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public void setOffset(Long offset) {\n        this.offset = offset;\n    }\n\n    public Long getOffset() {\n        return offset;\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setExtraInfo(String extraInfo) {\n        this.extraInfo = extraInfo;\n    }\n\n    public String getExtraInfo() {\n        return extraInfo;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public Integer getQueueId() {\n        return queueId;\n    }\n\n    public void setQueueId(Integer queueId) {\n        this.queueId = queueId;\n    }\n\n    public String getLiteTopic() {\n        return liteTopic;\n    }\n\n    public void setLiteTopic(String liteTopic) {\n        this.liteTopic = liteTopic;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"consumerGroup\", consumerGroup)\n            .add(\"topic\", topic)\n            .add(\"queueId\", queueId)\n            .add(\"extraInfo\", extraInfo)\n            .add(\"offset\", offset)\n            .add(\"liteTopic\", liteTopic)\n            .omitNullValues()\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/AddBrokerRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.ADD_BROKER, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class AddBrokerRequestHeader implements CommandCustomHeader {\n    @CFNullable\n    private String configPath;\n\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getConfigPath() {\n        return configPath;\n    }\n\n    public void setConfigPath(String configPath) {\n        this.configPath = configPath;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ChangeInvisibleTimeRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;\n\n@RocketMQAction(value = RequestCode.CHANGE_MESSAGE_INVISIBLETIME, action = Action.SUB)\npublic class ChangeInvisibleTimeRequestHeader extends TopicQueueRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String consumerGroup;\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNotNull\n    private Integer queueId;\n    /**\n     * startOffset popTime invisibleTime queueId\n     */\n    @CFNotNull\n    private String extraInfo;\n\n    @CFNotNull\n    private Long offset;\n\n    @CFNotNull\n    private Long invisibleTime;\n\n    private String liteTopic;\n\n    private boolean suspend = false;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public void setOffset(Long offset) {\n        this.offset = offset;\n    }\n\n    public Long getOffset() {\n        return offset;\n    }\n\n    public Long getInvisibleTime() {\n        return invisibleTime;\n    }\n\n    public void setInvisibleTime(Long invisibleTime) {\n        this.invisibleTime = invisibleTime;\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setExtraInfo(String extraInfo) {\n        this.extraInfo = extraInfo;\n    }\n\n    /**\n     * startOffset popTime invisibleTime queueId\n     */\n    public String getExtraInfo() {\n        return extraInfo;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public Integer getQueueId() {\n        return queueId;\n    }\n\n    public void setQueueId(Integer queueId) {\n        this.queueId = queueId;\n    }\n\n    public String getLiteTopic() {\n        return liteTopic;\n    }\n\n    public void setLiteTopic(String liteTopic) {\n        this.liteTopic = liteTopic;\n    }\n\n    public boolean isSuspend() {\n        return suspend;\n    }\n\n    public void setSuspend(boolean suspend) {\n        this.suspend = suspend;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"consumerGroup\", consumerGroup)\n            .add(\"topic\", topic)\n            .add(\"queueId\", queueId)\n            .add(\"extraInfo\", extraInfo)\n            .add(\"offset\", offset)\n            .add(\"invisibleTime\", invisibleTime)\n            .add(\"liteTopic\", liteTopic)\n            .add(\"suspend\", suspend)\n            .omitNullValues()\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ChangeInvisibleTimeResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class ChangeInvisibleTimeResponseHeader implements CommandCustomHeader {\n\n\n    @CFNotNull\n    private long popTime;\n    @CFNotNull\n    private long invisibleTime;\n\n    @CFNotNull\n    private int reviveQid;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public long getPopTime() {\n        return popTime;\n    }\n\n    public void setPopTime(long popTime) {\n        this.popTime = popTime;\n    }\n\n    public long getInvisibleTime() {\n        return invisibleTime;\n    }\n\n    public void setInvisibleTime(long invisibleTime) {\n        this.invisibleTime = invisibleTime;\n    }\n\n    public int getReviveQid() {\n        return reviveQid;\n    }\n\n    public void setReviveQid(int reviveQid) {\n        this.reviveQid = reviveQid;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CheckRocksdbCqWriteProgressRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.CHECK_ROCKSDB_CQ_WRITE_PROGRESS, action = Action.GET)\npublic class CheckRocksdbCqWriteProgressRequestHeader implements CommandCustomHeader {\n\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n\n    private long checkStoreTime;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public long getCheckStoreTime() {\n        return checkStoreTime;\n    }\n\n    public void setCheckStoreTime(long checkStoreTime) {\n        this.checkStoreTime = checkStoreTime;\n    }\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CheckTransactionStateRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: EndTransactionRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.RpcRequestHeader;\n\n@RocketMQAction(value = RequestCode.CHECK_TRANSACTION_STATE, action = Action.PUB)\npublic class CheckTransactionStateRequestHeader extends RpcRequestHeader {\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNotNull\n    private Long tranStateTableOffset;\n    @CFNotNull\n    private Long commitLogOffset;\n    private String msgId;\n    private String transactionId;\n    private String offsetMsgId;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public Long getTranStateTableOffset() {\n        return tranStateTableOffset;\n    }\n\n    public void setTranStateTableOffset(Long tranStateTableOffset) {\n        this.tranStateTableOffset = tranStateTableOffset;\n    }\n\n    public Long getCommitLogOffset() {\n        return commitLogOffset;\n    }\n\n    public void setCommitLogOffset(Long commitLogOffset) {\n        this.commitLogOffset = commitLogOffset;\n    }\n\n    public String getMsgId() {\n        return msgId;\n    }\n\n    public void setMsgId(String msgId) {\n        this.msgId = msgId;\n    }\n\n    public String getTransactionId() {\n        return transactionId;\n    }\n\n    public void setTransactionId(String transactionId) {\n        this.transactionId = transactionId;\n    }\n\n    public String getOffsetMsgId() {\n        return offsetMsgId;\n    }\n\n    public void setOffsetMsgId(String offsetMsgId) {\n        this.offsetMsgId = offsetMsgId;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"tranStateTableOffset\", tranStateTableOffset)\n            .add(\"commitLogOffset\", commitLogOffset)\n            .add(\"msgId\", msgId)\n            .add(\"transactionId\", transactionId)\n            .add(\"offsetMsgId\", offsetMsgId)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CheckTransactionStateResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: EndTransactionResponseHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class CheckTransactionStateResponseHeader implements CommandCustomHeader {\n    @CFNotNull\n    private String producerGroup;\n    @CFNotNull\n    private Long tranStateTableOffset;\n    @CFNotNull\n    private Long commitLogOffset;\n    @CFNotNull\n    private Integer commitOrRollback; // TRANSACTION_COMMIT_TYPE\n\n    // TRANSACTION_ROLLBACK_TYPE\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n        if (MessageSysFlag.TRANSACTION_COMMIT_TYPE == this.commitOrRollback) {\n            return;\n        }\n\n        if (MessageSysFlag.TRANSACTION_ROLLBACK_TYPE == this.commitOrRollback) {\n            return;\n        }\n\n        throw new RemotingCommandException(\"commitOrRollback field wrong\");\n    }\n\n    public String getProducerGroup() {\n        return producerGroup;\n    }\n\n    public void setProducerGroup(String producerGroup) {\n        this.producerGroup = producerGroup;\n    }\n\n    public Long getTranStateTableOffset() {\n        return tranStateTableOffset;\n    }\n\n    public void setTranStateTableOffset(Long tranStateTableOffset) {\n        this.tranStateTableOffset = tranStateTableOffset;\n    }\n\n    public Long getCommitLogOffset() {\n        return commitLogOffset;\n    }\n\n    public void setCommitLogOffset(Long commitLogOffset) {\n        this.commitLogOffset = commitLogOffset;\n    }\n\n    public Integer getCommitOrRollback() {\n        return commitOrRollback;\n    }\n\n    public void setCommitOrRollback(Integer commitOrRollback) {\n        this.commitOrRollback = commitOrRollback;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CloneGroupOffsetRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: DeleteTopicRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.RpcRequestHeader;\n\n@RocketMQAction(value = RequestCode.CLONE_GROUP_OFFSET, action = Action.UPDATE)\npublic class CloneGroupOffsetRequestHeader extends RpcRequestHeader {\n    @CFNotNull\n    private String srcGroup;\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String destGroup;\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    private boolean offline;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getDestGroup() {\n        return destGroup;\n    }\n\n    public void setDestGroup(String destGroup) {\n        this.destGroup = destGroup;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getSrcGroup() {\n\n        return srcGroup;\n    }\n\n    public void setSrcGroup(String srcGroup) {\n        this.srcGroup = srcGroup;\n    }\n\n    public boolean isOffline() {\n        return offline;\n    }\n\n    public void setOffline(boolean offline) {\n        this.offline = offline;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"srcGroup\", srcGroup)\n            .add(\"destGroup\", destGroup)\n            .add(\"topic\", topic)\n            .add(\"offline\", offline)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ConsumeMessageDirectlyResultRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicRequestHeader;\n\n@RocketMQAction(value = RequestCode.CONSUME_MESSAGE_DIRECTLY, action = Action.SUB)\npublic class ConsumeMessageDirectlyResultRequestHeader extends TopicRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String consumerGroup;\n    @CFNullable\n    private String clientId;\n    @CFNullable\n    private String msgId;\n    @CFNullable\n    private String brokerName;\n    @CFNullable\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNullable\n    private Integer topicSysFlag;\n    @CFNullable\n    private Integer groupSysFlag;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public String getClientId() {\n        return clientId;\n    }\n\n    public void setClientId(String clientId) {\n        this.clientId = clientId;\n    }\n\n    public String getMsgId() {\n        return msgId;\n    }\n\n    public void setMsgId(String msgId) {\n        this.msgId = msgId;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public Integer getTopicSysFlag() {\n        return topicSysFlag;\n    }\n\n    public void setTopicSysFlag(Integer topicSysFlag) {\n        this.topicSysFlag = topicSysFlag;\n    }\n\n    public Integer getGroupSysFlag() {\n        return groupSysFlag;\n    }\n\n    public void setGroupSysFlag(Integer groupSysFlag) {\n        this.groupSysFlag = groupSysFlag;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"consumerGroup\", consumerGroup)\n            .add(\"clientId\", clientId)\n            .add(\"msgId\", msgId)\n            .add(\"brokerName\", brokerName)\n            .add(\"topic\", topic)\n            .add(\"topicSysFlag\", topicSysFlag)\n            .add(\"groupSysFlag\", groupSysFlag)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ConsumerSendMsgBackRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.RpcRequestHeader;\n\n@RocketMQAction(value = RequestCode.CONSUMER_SEND_MSG_BACK, action = Action.SUB)\npublic class ConsumerSendMsgBackRequestHeader extends RpcRequestHeader {\n    @CFNotNull\n    private Long offset;\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String group;\n    @CFNotNull\n    private Integer delayLevel;\n    private String originMsgId;\n    @RocketMQResource(ResourceType.TOPIC)\n    private String originTopic;\n    @CFNullable\n    private boolean unitMode = false;\n    private Integer maxReconsumeTimes;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public Long getOffset() {\n        return offset;\n    }\n\n    public void setOffset(Long offset) {\n        this.offset = offset;\n    }\n\n    public String getGroup() {\n        return group;\n    }\n\n    public void setGroup(String group) {\n        this.group = group;\n    }\n\n    public Integer getDelayLevel() {\n        return delayLevel;\n    }\n\n    public void setDelayLevel(Integer delayLevel) {\n        this.delayLevel = delayLevel;\n    }\n\n    public String getOriginMsgId() {\n        return originMsgId;\n    }\n\n    public void setOriginMsgId(String originMsgId) {\n        this.originMsgId = originMsgId;\n    }\n\n    public String getOriginTopic() {\n        return originTopic;\n    }\n\n    public void setOriginTopic(String originTopic) {\n        this.originTopic = originTopic;\n    }\n\n    public boolean isUnitMode() {\n        return unitMode;\n    }\n\n    public void setUnitMode(boolean unitMode) {\n        this.unitMode = unitMode;\n    }\n\n    public Integer getMaxReconsumeTimes() {\n        return maxReconsumeTimes;\n    }\n\n    public void setMaxReconsumeTimes(final Integer maxReconsumeTimes) {\n        this.maxReconsumeTimes = maxReconsumeTimes;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"offset\", offset)\n            .add(\"group\", group)\n            .add(\"delayLevel\", delayLevel)\n            .add(\"originMsgId\", originMsgId)\n            .add(\"originTopic\", originTopic)\n            .add(\"unitMode\", unitMode)\n            .add(\"maxReconsumeTimes\", maxReconsumeTimes)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateAclRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.AUTH_CREATE_ACL, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class CreateAclRequestHeader implements CommandCustomHeader {\n\n    private String subject;\n\n    public CreateAclRequestHeader() {\n    }\n\n    public CreateAclRequestHeader(String subject) {\n        this.subject = subject;\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getSubject() {\n        return subject;\n    }\n\n    public void setSubject(String subject) {\n        this.subject = subject;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateTopicListRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.RpcRequestHeader;\n\n@RocketMQAction(value = RequestCode.UPDATE_AND_CREATE_TOPIC_LIST, action = Action.CREATE)\npublic class CreateTopicListRequestHeader extends RpcRequestHeader {\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateTopicRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: CreateTopicRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.common.TopicFilterType;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicRequestHeader;\n\n@RocketMQAction(value = RequestCode.UPDATE_AND_CREATE_TOPIC, action = Action.CREATE)\npublic class CreateTopicRequestHeader extends TopicRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNotNull\n    private String defaultTopic;\n    @CFNotNull\n    private Integer readQueueNums;\n    @CFNotNull\n    private Integer writeQueueNums;\n    @CFNotNull\n    private Integer perm;\n    @CFNotNull\n    private String topicFilterType;\n    private Integer topicSysFlag;\n    @CFNotNull\n    private Boolean order = false;\n    private String attributes;\n\n    @CFNullable\n    private Boolean force = false;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n        try {\n            TopicFilterType.valueOf(this.topicFilterType);\n        } catch (Exception e) {\n            throw new RemotingCommandException(\"topicFilterType = [\" + topicFilterType + \"] value invalid\", e);\n        }\n    }\n\n    public TopicFilterType getTopicFilterTypeEnum() {\n        return TopicFilterType.valueOf(this.topicFilterType);\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getDefaultTopic() {\n        return defaultTopic;\n    }\n\n    public void setDefaultTopic(String defaultTopic) {\n        this.defaultTopic = defaultTopic;\n    }\n\n    public Integer getReadQueueNums() {\n        return readQueueNums;\n    }\n\n    public void setReadQueueNums(Integer readQueueNums) {\n        this.readQueueNums = readQueueNums;\n    }\n\n    public Integer getWriteQueueNums() {\n        return writeQueueNums;\n    }\n\n    public void setWriteQueueNums(Integer writeQueueNums) {\n        this.writeQueueNums = writeQueueNums;\n    }\n\n    public Integer getPerm() {\n        return perm;\n    }\n\n    public void setPerm(Integer perm) {\n        this.perm = perm;\n    }\n\n    public String getTopicFilterType() {\n        return topicFilterType;\n    }\n\n    public void setTopicFilterType(String topicFilterType) {\n        this.topicFilterType = topicFilterType;\n    }\n\n    public Integer getTopicSysFlag() {\n        return topicSysFlag;\n    }\n\n    public void setTopicSysFlag(Integer topicSysFlag) {\n        this.topicSysFlag = topicSysFlag;\n    }\n\n    public Boolean getOrder() {\n        return order;\n    }\n\n    public void setOrder(Boolean order) {\n        this.order = order;\n    }\n\n    public Boolean getForce() {\n        return force;\n    }\n\n    public void setForce(Boolean force) {\n        this.force = force;\n    }\n\n    public String getAttributes() {\n        return attributes;\n    }\n\n    public void setAttributes(String attributes) {\n        this.attributes = attributes;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"topic\", topic)\n            .add(\"defaultTopic\", defaultTopic)\n            .add(\"readQueueNums\", readQueueNums)\n            .add(\"writeQueueNums\", writeQueueNums)\n            .add(\"perm\", perm)\n            .add(\"topicFilterType\", topicFilterType)\n            .add(\"topicSysFlag\", topicSysFlag)\n            .add(\"order\", order)\n            .add(\"attributes\", attributes)\n            .add(\"force\", force)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateUserRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.AUTH_CREATE_USER, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class CreateUserRequestHeader implements CommandCustomHeader {\n\n    private String username;\n\n    public CreateUserRequestHeader() {\n    }\n\n    public CreateUserRequestHeader(String username) {\n        this.username = username;\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getUsername() {\n        return username;\n    }\n\n    public void setUsername(String username) {\n        this.username = username;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/DeleteAclRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.AUTH_DELETE_ACL, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class DeleteAclRequestHeader implements CommandCustomHeader {\n\n    private String subject;\n\n    private String policyType;\n\n    private String resource;\n\n    public DeleteAclRequestHeader() {\n    }\n\n    public DeleteAclRequestHeader(String subject, String resource) {\n        this.subject = subject;\n        this.resource = resource;\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getSubject() {\n        return subject;\n    }\n\n    public void setSubject(String subject) {\n        this.subject = subject;\n    }\n\n    public String getPolicyType() {\n        return policyType;\n    }\n\n    public void setPolicyType(String policyType) {\n        this.policyType = policyType;\n    }\n\n    public String getResource() {\n        return resource;\n    }\n\n    public void setResource(String resource) {\n        this.resource = resource;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/DeleteSubscriptionGroupRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.RpcRequestHeader;\n\n@RocketMQAction(value = RequestCode.DELETE_SUBSCRIPTIONGROUP, action = Action.DELETE)\npublic class DeleteSubscriptionGroupRequestHeader extends RpcRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String groupName;\n\n    private boolean cleanOffset = false;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getGroupName() {\n        return groupName;\n    }\n\n    public void setGroupName(String groupName) {\n        this.groupName = groupName;\n    }\n\n    public boolean isCleanOffset() {\n        return cleanOffset;\n    }\n\n    public void setCleanOffset(boolean cleanOffset) {\n        this.cleanOffset = cleanOffset;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/DeleteTopicRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: DeleteTopicRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicRequestHeader;\n\n@RocketMQAction(value = RequestCode.DELETE_TOPIC_IN_BROKER, action = Action.DELETE)\npublic class DeleteTopicRequestHeader extends TopicRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/DeleteUserRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.AUTH_DELETE_USER, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class DeleteUserRequestHeader implements CommandCustomHeader {\n\n    private String username;\n\n    public DeleteUserRequestHeader() {\n    }\n\n    public DeleteUserRequestHeader(String username) {\n        this.username = username;\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getUsername() {\n        return username;\n    }\n\n    public void setUsername(String username) {\n        this.username = username;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/EndTransactionRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.RpcRequestHeader;\n\n@RocketMQAction(value = RequestCode.END_TRANSACTION, action = Action.PUB)\npublic class EndTransactionRequestHeader extends RpcRequestHeader {\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNotNull\n    private String producerGroup;\n    @CFNotNull\n    private Long tranStateTableOffset;\n    @CFNotNull\n    private Long commitLogOffset;\n    @CFNotNull\n    private Integer commitOrRollback; // TRANSACTION_COMMIT_TYPE\n    // TRANSACTION_ROLLBACK_TYPE\n    // TRANSACTION_NOT_TYPE\n\n    @CFNullable\n    private Boolean fromTransactionCheck = false;\n\n    @CFNotNull\n    private String msgId;\n\n    private String transactionId;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n        if (MessageSysFlag.TRANSACTION_NOT_TYPE == this.commitOrRollback) {\n            return;\n        }\n\n        if (MessageSysFlag.TRANSACTION_COMMIT_TYPE == this.commitOrRollback) {\n            return;\n        }\n\n        if (MessageSysFlag.TRANSACTION_ROLLBACK_TYPE == this.commitOrRollback) {\n            return;\n        }\n\n        throw new RemotingCommandException(\"commitOrRollback field wrong\");\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getProducerGroup() {\n        return producerGroup;\n    }\n\n    public void setProducerGroup(String producerGroup) {\n        this.producerGroup = producerGroup;\n    }\n\n    public Long getTranStateTableOffset() {\n        return tranStateTableOffset;\n    }\n\n    public void setTranStateTableOffset(Long tranStateTableOffset) {\n        this.tranStateTableOffset = tranStateTableOffset;\n    }\n\n    public Long getCommitLogOffset() {\n        return commitLogOffset;\n    }\n\n    public void setCommitLogOffset(Long commitLogOffset) {\n        this.commitLogOffset = commitLogOffset;\n    }\n\n    public Integer getCommitOrRollback() {\n        return commitOrRollback;\n    }\n\n    public void setCommitOrRollback(Integer commitOrRollback) {\n        this.commitOrRollback = commitOrRollback;\n    }\n\n    public Boolean getFromTransactionCheck() {\n        return fromTransactionCheck;\n    }\n\n    public void setFromTransactionCheck(Boolean fromTransactionCheck) {\n        this.fromTransactionCheck = fromTransactionCheck;\n    }\n\n    public String getMsgId() {\n        return msgId;\n    }\n\n    public void setMsgId(String msgId) {\n        this.msgId = msgId;\n    }\n\n    public String getTransactionId() {\n        return transactionId;\n    }\n\n    public void setTransactionId(String transactionId) {\n        this.transactionId = transactionId;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"producerGroup\", producerGroup)\n            .add(\"tranStateTableOffset\", tranStateTableOffset)\n            .add(\"commitLogOffset\", commitLogOffset)\n            .add(\"commitOrRollback\", commitOrRollback)\n            .add(\"fromTransactionCheck\", fromTransactionCheck)\n            .add(\"msgId\", msgId)\n            .add(\"transactionId\", transactionId)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/EndTransactionResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: EndTransactionResponseHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class EndTransactionResponseHeader implements CommandCustomHeader {\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExchangeHAInfoRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.EXCHANGE_BROKER_HA_INFO,resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class ExchangeHAInfoRequestHeader implements CommandCustomHeader {\n    @CFNullable\n    public String masterHaAddress;\n\n    @CFNullable\n    public Long masterFlushOffset;\n\n    @CFNullable\n    public String masterAddress;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getMasterHaAddress() {\n        return masterHaAddress;\n    }\n\n    public void setMasterHaAddress(String masterHaAddress) {\n        this.masterHaAddress = masterHaAddress;\n    }\n\n    public Long getMasterFlushOffset() {\n        return masterFlushOffset;\n    }\n\n    public void setMasterFlushOffset(Long masterFlushOffset) {\n        this.masterFlushOffset = masterFlushOffset;\n    }\n\n    public String getMasterAddress() {\n        return masterAddress;\n    }\n\n    public void setMasterAddress(String masterAddress) {\n        this.masterAddress = masterAddress;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExchangeHAInfoResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class ExchangeHAInfoResponseHeader implements CommandCustomHeader {\n    @CFNullable\n    public String masterHaAddress;\n\n    @CFNullable\n    public Long masterFlushOffset;\n\n    @CFNullable\n    public String masterAddress;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getMasterHaAddress() {\n        return masterHaAddress;\n    }\n\n    public void setMasterHaAddress(String masterHaAddress) {\n        this.masterHaAddress = masterHaAddress;\n    }\n\n    public Long getMasterFlushOffset() {\n        return masterFlushOffset;\n    }\n\n    public void setMasterFlushOffset(Long masterFlushOffset) {\n        this.masterFlushOffset = masterFlushOffset;\n    }\n\n    public String getMasterAddress() {\n        return masterAddress;\n    }\n\n    public void setMasterAddress(String masterAddress) {\n        this.masterAddress = masterAddress;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExportRocksDBConfigToJsonRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.EXPORT_ROCKSDB_CONFIG_TO_JSON, resource = ResourceType.CLUSTER, action = Action.GET)\npublic class ExportRocksDBConfigToJsonRequestHeader implements CommandCustomHeader {\n    private static final String CONFIG_TYPE_SEPARATOR = \";\";\n\n    public enum ConfigType {\n        TOPICS(\"topics\"),\n        SUBSCRIPTION_GROUPS(\"subscriptionGroups\"),\n        CONSUMER_OFFSETS(\"consumerOffsets\");\n\n        private final String typeName;\n\n        ConfigType(String typeName) {\n            this.typeName = typeName;\n        }\n\n        public static ConfigType getConfigTypeByName(String typeName) {\n            for (ConfigType configType : ConfigType.values()) {\n                if (configType.getTypeName().equalsIgnoreCase(typeName.trim())) {\n                    return configType;\n                }\n            }\n            throw new IllegalArgumentException(\"Unknown config type: \" + typeName);\n        }\n\n        public static List<ConfigType> fromString(String ordinal) {\n            String[] configTypeNames = StringUtils.split(ordinal, CONFIG_TYPE_SEPARATOR);\n            List<ConfigType> configTypes = new ArrayList<>();\n            for (String configTypeName : configTypeNames) {\n                if (StringUtils.isNotEmpty(configTypeName)) {\n                    configTypes.add(getConfigTypeByName(configTypeName));\n                }\n            }\n            return configTypes;\n        }\n\n        public static String toString(List<ConfigType> configTypes) {\n            StringBuilder sb = new StringBuilder();\n            for (ConfigType configType : configTypes) {\n                sb.append(configType.getTypeName()).append(CONFIG_TYPE_SEPARATOR);\n            }\n            return sb.toString();\n        }\n\n        public String getTypeName() {\n            return typeName;\n        }\n    }\n\n    @CFNotNull\n    private String configType;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public List<ConfigType> fetchConfigType() {\n        return ConfigType.fromString(configType);\n    }\n\n    public void updateConfigType(List<ConfigType> configType) {\n        this.configType = ConfigType.toString(configType);\n    }\n\n    public String getConfigType() {\n        return configType;\n    }\n\n    public void setConfigType(String configType) {\n        this.configType = configType;\n    }\n}"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExtraInfoUtil.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.stream.Collectors;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.commons.lang3.math.NumberUtils;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageConst;\n\npublic class ExtraInfoUtil {\n    private static final String NORMAL_TOPIC = \"0\";\n    private static final String RETRY_TOPIC = \"1\";\n    private static final String RETRY_TOPIC_V2 = \"2\";\n    private static final String QUEUE_OFFSET = \"qo\";\n\n    public static String[] split(String extraInfo) {\n        if (extraInfo == null) {\n            throw new IllegalArgumentException(\"split extraInfo is null\");\n        }\n        return extraInfo.split(MessageConst.KEY_SEPARATOR);\n    }\n\n    public static Long getCkQueueOffset(String[] extraInfoStrs) {\n        if (extraInfoStrs == null || extraInfoStrs.length < 1) {\n            throw new IllegalArgumentException(\"getCkQueueOffset fail, extraInfoStrs length \" + (extraInfoStrs == null ? 0 : extraInfoStrs.length));\n        }\n        return Long.valueOf(extraInfoStrs[0]);\n    }\n\n    public static Long getPopTime(String[] extraInfoStrs) {\n        if (extraInfoStrs == null || extraInfoStrs.length < 2) {\n            throw new IllegalArgumentException(\"getPopTime fail, extraInfoStrs length \" + (extraInfoStrs == null ? 0 : extraInfoStrs.length));\n        }\n        return Long.valueOf(extraInfoStrs[1]);\n    }\n\n    public static Long getInvisibleTime(String[] extraInfoStrs) {\n        if (extraInfoStrs == null || extraInfoStrs.length < 3) {\n            throw new IllegalArgumentException(\"getInvisibleTime fail, extraInfoStrs length \" + (extraInfoStrs == null ? 0 : extraInfoStrs.length));\n        }\n        return Long.valueOf(extraInfoStrs[2]);\n    }\n\n    public static int getReviveQid(String[] extraInfoStrs) {\n        if (extraInfoStrs == null || extraInfoStrs.length < 4) {\n            throw new IllegalArgumentException(\"getReviveQid fail, extraInfoStrs length \" + (extraInfoStrs == null ? 0 : extraInfoStrs.length));\n        }\n        return Integer.parseInt(extraInfoStrs[3]);\n    }\n\n    public static String getRealTopic(String[] extraInfoStrs, String topic, String cid) {\n        if (extraInfoStrs == null || extraInfoStrs.length < 5) {\n            throw new IllegalArgumentException(\"getRealTopic fail, extraInfoStrs length \" + (extraInfoStrs == null ? 0 : extraInfoStrs.length));\n        }\n        if (RETRY_TOPIC.equals(extraInfoStrs[4])) {\n            return KeyBuilder.buildPopRetryTopicV1(topic, cid);\n        } else if (RETRY_TOPIC_V2.equals(extraInfoStrs[4])) {\n            return KeyBuilder.buildPopRetryTopicV2(topic, cid);\n        } else {\n            return topic;\n        }\n    }\n\n    public static String getRealTopic(String topic, String cid, String retry) {\n        if (retry.equals(NORMAL_TOPIC)) {\n            return topic;\n        } else if (retry.equals(RETRY_TOPIC)) {\n            return KeyBuilder.buildPopRetryTopicV1(topic, cid);\n        } else if (retry.equals(RETRY_TOPIC_V2)) {\n            return KeyBuilder.buildPopRetryTopicV2(topic, cid);\n        } else {\n            throw new IllegalArgumentException(\"getRetry fail, format is wrong\");\n        }\n    }\n\n    public static String getRetry(String[] extraInfoStrs) {\n        if (extraInfoStrs == null || extraInfoStrs.length < 5) {\n            throw new IllegalArgumentException(\"getRetry fail, extraInfoStrs length \" + (extraInfoStrs == null ? 0 : extraInfoStrs.length));\n        }\n        return extraInfoStrs[4];\n    }\n\n    public static String getBrokerName(String[] extraInfoStrs) {\n        if (extraInfoStrs == null || extraInfoStrs.length < 6) {\n            throw new IllegalArgumentException(\"getBrokerName fail, extraInfoStrs length \" + (extraInfoStrs == null ? 0 : extraInfoStrs.length));\n        }\n        return extraInfoStrs[5];\n    }\n\n    public static int getQueueId(String[] extraInfoStrs) {\n        if (extraInfoStrs == null || extraInfoStrs.length < 7) {\n            throw new IllegalArgumentException(\"getQueueId fail, extraInfoStrs length \" + (extraInfoStrs == null ? 0 : extraInfoStrs.length));\n        }\n        return Integer.parseInt(extraInfoStrs[6]);\n    }\n\n    public static long getQueueOffset(String[] extraInfoStrs) {\n        if (extraInfoStrs == null || extraInfoStrs.length < 8) {\n            throw new IllegalArgumentException(\"getQueueOffset fail, extraInfoStrs length \" + (extraInfoStrs == null ? 0 : extraInfoStrs.length));\n        }\n        return Long.parseLong(extraInfoStrs[7]);\n    }\n\n    public static String buildExtraInfo(long ckQueueOffset, long popTime, long invisibleTime, int reviveQid, String topic, String brokerName, int queueId) {\n        String t = getRetry(topic);\n        return ckQueueOffset + MessageConst.KEY_SEPARATOR + popTime + MessageConst.KEY_SEPARATOR + invisibleTime + MessageConst.KEY_SEPARATOR + reviveQid + MessageConst.KEY_SEPARATOR + t\n            + MessageConst.KEY_SEPARATOR + brokerName + MessageConst.KEY_SEPARATOR + queueId;\n    }\n\n    public static String buildExtraInfo(long ckQueueOffset, long popTime, long invisibleTime, int reviveQid, String topic, String brokerName, int queueId,\n                                        long msgQueueOffset) {\n        String t = getRetry(topic);\n        return ckQueueOffset\n            + MessageConst.KEY_SEPARATOR + popTime + MessageConst.KEY_SEPARATOR + invisibleTime\n            + MessageConst.KEY_SEPARATOR + reviveQid + MessageConst.KEY_SEPARATOR + t\n            + MessageConst.KEY_SEPARATOR + brokerName + MessageConst.KEY_SEPARATOR + queueId\n            + MessageConst.KEY_SEPARATOR + msgQueueOffset;\n    }\n\n    public static void buildStartOffsetInfo(StringBuilder stringBuilder, String topic, int queueId, long startOffset) {\n        if (stringBuilder == null) {\n            stringBuilder = new StringBuilder(64);\n        }\n\n        if (stringBuilder.length() > 0) {\n            stringBuilder.append(\";\");\n        }\n\n        stringBuilder.append(getRetry(topic))\n            .append(MessageConst.KEY_SEPARATOR).append(queueId)\n            .append(MessageConst.KEY_SEPARATOR).append(startOffset);\n    }\n\n    public static void buildQueueIdOrderCountInfo(StringBuilder stringBuilder, String topic, int queueId, int orderCount) {\n        if (stringBuilder == null) {\n            stringBuilder = new StringBuilder(64);\n        }\n\n        if (stringBuilder.length() > 0) {\n            stringBuilder.append(\";\");\n        }\n\n        stringBuilder.append(getRetry(topic))\n                .append(MessageConst.KEY_SEPARATOR).append(queueId)\n                .append(MessageConst.KEY_SEPARATOR).append(orderCount);\n    }\n\n    public static void buildQueueOffsetOrderCountInfo(StringBuilder stringBuilder, String topic, long queueId, long queueOffset, int orderCount) {\n        if (stringBuilder == null) {\n            stringBuilder = new StringBuilder(64);\n        }\n\n        if (stringBuilder.length() > 0) {\n            stringBuilder.append(\";\");\n        }\n\n        stringBuilder.append(getRetry(topic))\n            .append(MessageConst.KEY_SEPARATOR).append(getQueueOffsetKeyValueKey(queueId, queueOffset))\n            .append(MessageConst.KEY_SEPARATOR).append(orderCount);\n    }\n\n    public static void buildMsgOffsetInfo(StringBuilder stringBuilder, String topic, int queueId, List<Long> msgOffsets) {\n        if (stringBuilder == null) {\n            stringBuilder = new StringBuilder(64);\n        }\n\n        if (stringBuilder.length() > 0) {\n            stringBuilder.append(\";\");\n        }\n\n        stringBuilder.append(getRetry(topic))\n            .append(MessageConst.KEY_SEPARATOR).append(queueId)\n            .append(MessageConst.KEY_SEPARATOR);\n\n        for (int i = 0; i < msgOffsets.size(); i++) {\n            stringBuilder.append(msgOffsets.get(i));\n            if (i < msgOffsets.size() - 1) {\n                stringBuilder.append(\",\");\n            }\n        }\n    }\n\n    public static Map<String, List<Long>> parseMsgOffsetInfo(String msgOffsetInfo) {\n        if (msgOffsetInfo == null || msgOffsetInfo.length() == 0) {\n            return null;\n        }\n\n        Map<String, List<Long>> msgOffsetMap = new HashMap<>(4);\n        String[] array;\n        if (msgOffsetInfo.indexOf(\";\") < 0) {\n            array = new String[]{msgOffsetInfo};\n        } else {\n            array = msgOffsetInfo.split(\";\");\n        }\n\n        for (String one : array) {\n            String[] split = one.split(MessageConst.KEY_SEPARATOR);\n            if (split.length != 3) {\n                throw new IllegalArgumentException(\"parse msgOffsetMap error, \" + msgOffsetMap);\n            }\n            String key = split[0] + \"@\" + split[1];\n            if (msgOffsetMap.containsKey(key)) {\n                throw new IllegalArgumentException(\"parse msgOffsetMap error, duplicate, \" + msgOffsetMap);\n            }\n            msgOffsetMap.put(key, new ArrayList<>(8));\n            String[] msgOffsets = split[2].split(\",\");\n            for (String msgOffset : msgOffsets) {\n                msgOffsetMap.get(key).add(Long.valueOf(msgOffset));\n            }\n        }\n\n        return msgOffsetMap;\n    }\n\n    public static Map<String, Long> parseStartOffsetInfo(String startOffsetInfo) {\n        if (startOffsetInfo == null || startOffsetInfo.length() == 0) {\n            return null;\n        }\n        Map<String, Long> startOffsetMap = new HashMap<>(4);\n        String[] array;\n        if (startOffsetInfo.indexOf(\";\") < 0) {\n            array = new String[]{startOffsetInfo};\n        } else {\n            array = startOffsetInfo.split(\";\");\n        }\n\n        for (String one : array) {\n            String[] split = one.split(MessageConst.KEY_SEPARATOR);\n            if (split.length != 3) {\n                throw new IllegalArgumentException(\"parse startOffsetInfo error, \" + startOffsetInfo);\n            }\n            String key = split[0] + \"@\" + split[1];\n            if (startOffsetMap.containsKey(key)) {\n                throw new IllegalArgumentException(\"parse startOffsetInfo error, duplicate, \" + startOffsetInfo);\n            }\n            startOffsetMap.put(key, Long.valueOf(split[2]));\n        }\n\n        return startOffsetMap;\n    }\n\n    public static Map<String, Integer> parseOrderCountInfo(String orderCountInfo) {\n        if (orderCountInfo == null || orderCountInfo.length() == 0) {\n            return null;\n        }\n        Map<String, Integer> startOffsetMap = new HashMap<>(4);\n        String[] array;\n        if (orderCountInfo.indexOf(\";\") < 0) {\n            array = new String[]{orderCountInfo};\n        } else {\n            array = orderCountInfo.split(\";\");\n        }\n\n        for (String one : array) {\n            String[] split = one.split(MessageConst.KEY_SEPARATOR);\n            if (split.length != 3) {\n                throw new IllegalArgumentException(\"parse orderCountInfo error, \" + orderCountInfo);\n            }\n            String key = split[0] + \"@\" + split[1];\n            if (startOffsetMap.containsKey(key)) {\n                throw new IllegalArgumentException(\"parse orderCountInfo error, duplicate, \" + orderCountInfo);\n            }\n            startOffsetMap.put(key, Integer.valueOf(split[2]));\n        }\n\n        return startOffsetMap;\n    }\n\n    public static List<Integer> parseLiteOrderCountInfo(String orderCountInfo, int msgCount) {\n        if (StringUtils.isEmpty(orderCountInfo)) {\n            return null;\n        }\n        String[] infos = orderCountInfo.split(\";\");\n        if (infos.length != msgCount) {\n            return null;\n        }\n        return Arrays.stream(infos).map(ExtraInfoUtil::parseLiteOrderCount).collect(Collectors.toList());\n    }\n\n    private static int parseLiteOrderCount(String info) {\n        if (StringUtils.isBlank(info)) {\n            return 0;\n        }\n        if (!info.contains(QUEUE_OFFSET)) {\n            return NumberUtils.toInt(info, 0);\n        }\n        String[] split = info.split(MessageConst.KEY_SEPARATOR);\n        return split.length != 3 ? 0 : NumberUtils.toInt(split[2], 0);\n    }\n\n    public static String getStartOffsetInfoMapKey(String topic, long key) {\n        return getRetry(topic) + \"@\" + key;\n    }\n\n    public static String getStartOffsetInfoMapKey(String topic, String popCk, long key) {\n        return getRetry(topic, popCk) + \"@\" + key;\n    }\n\n    public static String getQueueOffsetKeyValueKey(long queueId, long queueOffset) {\n        return QUEUE_OFFSET + queueId + \"%\" + queueOffset;\n    }\n\n    public static String getQueueOffsetMapKey(String topic, long queueId, long queueOffset) {\n        return getRetry(topic) + \"@\" + getQueueOffsetKeyValueKey(queueId, queueOffset);\n    }\n\n    public static boolean isOrder(String[] extraInfo) {\n        return ExtraInfoUtil.getReviveQid(extraInfo) == KeyBuilder.POP_ORDER_REVIVE_QUEUE;\n    }\n\n    private static String getRetry(String topic) {\n        String t = NORMAL_TOPIC;\n        if (KeyBuilder.isPopRetryTopicV2(topic)) {\n            t = RETRY_TOPIC_V2;\n        } else if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n            t = RETRY_TOPIC;\n        }\n        return t;\n    }\n\n    private static String getRetry(String topic, String popCk) {\n        if (popCk != null) {\n            return getRetry(split(popCk));\n        }\n        return getRetry(topic);\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetAclRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.AUTH_GET_ACL, resource = ResourceType.CLUSTER, action = Action.GET)\npublic class GetAclRequestHeader implements CommandCustomHeader {\n\n    private String subject;\n\n    public GetAclRequestHeader() {\n    }\n\n    public GetAclRequestHeader(String subject) {\n        this.subject = subject;\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getSubject() {\n        return subject;\n    }\n\n    public void setSubject(String subject) {\n        this.subject = subject;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetAllProducerInfoRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.GET_ALL_PRODUCER_INFO, resource = ResourceType.CLUSTER, action = Action.GET)\npublic class GetAllProducerInfoRequestHeader implements CommandCustomHeader {\n    @Override\n    public void checkFields() throws RemotingCommandException {\n        // To change body of implemented methods use File | Settings | File\n        // Templates.\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetAllSubscriptionGroupRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.GET_ALL_SUBSCRIPTIONGROUP_CONFIG, resource = ResourceType.GROUP, action = Action.GET)\npublic class GetAllSubscriptionGroupRequestHeader implements CommandCustomHeader {\n    @Override\n    public void checkFields() throws RemotingCommandException {\n        // nothing\n    }\n\n    @CFNotNull\n    private Integer groupSeq;\n\n    private String dataVersion;\n\n    private Integer maxGroupNum;\n\n    public Integer getGroupSeq() {\n        return groupSeq;\n    }\n\n    public void setGroupSeq(Integer groupSeq) {\n        this.groupSeq = groupSeq;\n    }\n\n    public String getDataVersion() {\n        return dataVersion;\n    }\n\n    public void setDataVersion(String dataVersion) {\n        this.dataVersion = dataVersion;\n    }\n\n    public Integer getMaxGroupNum() {\n        return maxGroupNum;\n    }\n\n    public void setMaxGroupNum(Integer maxGroupNum) {\n        this.maxGroupNum = maxGroupNum;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetAllSubscriptionGroupResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.GET_ALL_SUBSCRIPTIONGROUP_CONFIG, resource = ResourceType.GROUP, action = Action.LIST)\npublic class GetAllSubscriptionGroupResponseHeader implements CommandCustomHeader {\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    @CFNotNull\n    private Integer totalGroupNum;\n\n    public Integer getTotalGroupNum() {\n        return totalGroupNum;\n    }\n\n    public void setTotalGroupNum(Integer totalGroupNum) {\n        this.totalGroupNum = totalGroupNum;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetAllTopicConfigRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.GET_ALL_TOPIC_CONFIG, resource = ResourceType.TOPIC, action = Action.GET)\npublic class GetAllTopicConfigRequestHeader implements CommandCustomHeader {\n    @Override\n    public void checkFields() throws RemotingCommandException {\n        // nothing\n    }\n\n    @CFNotNull\n    private Integer topicSeq;\n\n    private String dataVersion;\n\n    private Integer maxTopicNum;\n\n    public Integer getTopicSeq() {\n        return topicSeq;\n    }\n\n    public void setTopicSeq(Integer topicSeq) {\n        this.topicSeq = topicSeq;\n    }\n\n    public String getDataVersion() {\n        return dataVersion;\n    }\n\n    public void setDataVersion(String dataVersion) {\n        this.dataVersion = dataVersion;\n    }\n\n    public Integer getMaxTopicNum() {\n        return maxTopicNum;\n    }\n\n    public void setMaxTopicNum(Integer maxTopicNum) {\n        this.maxTopicNum = maxTopicNum;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetAllTopicConfigResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: GetAllTopicConfigResponseHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.GET_ALL_TOPIC_CONFIG, resource = ResourceType.TOPIC, action = Action.LIST)\npublic class GetAllTopicConfigResponseHeader implements CommandCustomHeader {\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    private Integer totalTopicNum;\n\n    public Integer getTotalTopicNum() {\n        return totalTopicNum;\n    }\n\n    public void setTotalTopicNum(Integer totalTopicNum) {\n        this.totalTopicNum = totalTopicNum;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetBrokerConfigResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: GetBrokerConfigResponseHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class GetBrokerConfigResponseHeader implements CommandCustomHeader {\n    @CFNotNull\n    private String version;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getVersion() {\n        return version;\n    }\n\n    public void setVersion(String version) {\n        this.version = version;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetBrokerMemberGroupRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.GET_BROKER_MEMBER_GROUP, action = Action.GET)\npublic class GetBrokerMemberGroupRequestHeader implements CommandCustomHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.CLUSTER)\n    private String clusterName;\n\n    @CFNotNull\n    private String brokerName;\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    public void setClusterName(final String clusterName) {\n        this.clusterName = clusterName;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(final String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumeStatsInBrokerHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.GET_BROKER_CONSUME_STATS, resource = ResourceType.CLUSTER, action = Action.GET)\npublic class GetConsumeStatsInBrokerHeader implements CommandCustomHeader {\n    @CFNotNull\n    private boolean isOrder;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public boolean isOrder() {\n        return isOrder;\n    }\n\n    public void setIsOrder(boolean isOrder) {\n        this.isOrder = isOrder;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumeStatsRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.List;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicRequestHeader;\n\n@RocketMQAction(value = RequestCode.GET_CONSUME_STATS, action = Action.GET)\npublic class GetConsumeStatsRequestHeader extends TopicRequestHeader {\n    private static final String TOPIC_NAME_SEPARATOR = \";\";\n\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String consumerGroup;\n\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n\n    // if topicList is provided, topic will be ignored\n    @RocketMQResource(value = ResourceType.TOPIC, splitter = TOPIC_NAME_SEPARATOR)\n    private String topicList;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public List<String> fetchTopicList() {\n        if (StringUtils.isBlank(topicList)) {\n            return Collections.emptyList();\n        }\n        return Arrays.asList(StringUtils.split(topicList, TOPIC_NAME_SEPARATOR));\n    }\n\n    public void updateTopicList(List<String> topicList) {\n        if (topicList == null || topicList.isEmpty()) {\n            return;\n        }\n        StringBuilder sb = new StringBuilder();\n        topicList.forEach(topic -> sb.append(topic).append(TOPIC_NAME_SEPARATOR));\n        this.setTopicList(sb.toString());\n    }\n\n    public String getTopicList() {\n        return topicList;\n    }\n\n    public void setTopicList(String topicList) {\n        this.topicList = topicList;\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"consumerGroup\", consumerGroup)\n            .add(\"topic\", topic)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumerConnectionListRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.RpcRequestHeader;\n\n@RocketMQAction(value = RequestCode.GET_CONSUMER_CONNECTION_LIST, action = Action.GET)\npublic class GetConsumerConnectionListRequestHeader extends RpcRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String consumerGroup;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n        // To change body of implemented methods use File | Settings | File\n        // Templates.\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumerListByGroupRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.rpc.RpcRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.GET_CONSUMER_LIST_BY_GROUP, action = Action.SUB)\npublic class GetConsumerListByGroupRequestHeader extends RpcRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String consumerGroup;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"consumerGroup\", consumerGroup)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumerListByGroupResponseBody.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport java.util.List;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class GetConsumerListByGroupResponseBody extends RemotingSerializable {\n    private List<String> consumerIdList;\n\n    public List<String> getConsumerIdList() {\n        return consumerIdList;\n    }\n\n    public void setConsumerIdList(List<String> consumerIdList) {\n        this.consumerIdList = consumerIdList;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumerListByGroupResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class GetConsumerListByGroupResponseHeader implements CommandCustomHeader {\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumerRunningInfoRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.rpc.RpcRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.GET_CONSUMER_RUNNING_INFO, action = Action.GET)\npublic class GetConsumerRunningInfoRequestHeader extends RpcRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String consumerGroup;\n    @CFNotNull\n    private String clientId;\n    @CFNullable\n    private boolean jstackEnable;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public String getClientId() {\n        return clientId;\n    }\n\n    public void setClientId(String clientId) {\n        this.clientId = clientId;\n    }\n\n    public boolean isJstackEnable() {\n        return jstackEnable;\n    }\n\n    public void setJstackEnable(boolean jstackEnable) {\n        this.jstackEnable = jstackEnable;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"consumerGroup\", consumerGroup)\n            .add(\"clientId\", clientId)\n            .add(\"jstackEnable\", jstackEnable)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumerStatusRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.rpc.TopicRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.INVOKE_BROKER_TO_GET_CONSUMER_STATUS, action = Action.GET)\npublic class GetConsumerStatusRequestHeader extends TopicRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String group;\n    @CFNullable\n    private String clientAddr;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getGroup() {\n        return group;\n    }\n\n    public void setGroup(String group) {\n        this.group = group;\n    }\n\n    public String getClientAddr() {\n        return clientAddr;\n    }\n\n    public void setClientAddr(String clientAddr) {\n        this.clientAddr = clientAddr;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"topic\", topic)\n            .add(\"group\", group)\n            .add(\"clientAddr\", clientAddr)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetEarliestMsgStoretimeRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: GetEarliestMsgStoretimeRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;\n\n@RocketMQAction(value = RequestCode.GET_EARLIEST_MSG_STORETIME, action = Action.GET)\npublic class GetEarliestMsgStoretimeRequestHeader extends TopicQueueRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNotNull\n    private Integer queueId;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    @Override\n    public String getTopic() {\n        return topic;\n    }\n\n    @Override\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    @Override\n    public Integer getQueueId() {\n        return queueId;\n    }\n\n    @Override\n    public void setQueueId(Integer queueId) {\n        this.queueId = queueId;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetEarliestMsgStoretimeResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: GetEarliestMsgStoretimeResponseHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class GetEarliestMsgStoretimeResponseHeader implements CommandCustomHeader {\n    @CFNotNull\n    private Long timestamp;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public Long getTimestamp() {\n        return timestamp;\n    }\n\n    public void setTimestamp(Long timestamp) {\n        this.timestamp = timestamp;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetLiteClientInfoRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class GetLiteClientInfoRequestHeader implements CommandCustomHeader {\n\n    private String parentTopic;\n    private String group;\n    private String clientId;\n    private int maxCount = 1000;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n        if (maxCount <= 0) {\n            throw new RemotingCommandException(\"[maxCount] field invalid\");\n        }\n    }\n\n    public String getParentTopic() {\n        return parentTopic;\n    }\n\n    public void setParentTopic(String parentTopic) {\n        this.parentTopic = parentTopic;\n    }\n\n    public String getGroup() {\n        return group;\n    }\n\n    public void setGroup(String group) {\n        this.group = group;\n    }\n\n    public String getClientId() {\n        return clientId;\n    }\n\n    public void setClientId(String clientId) {\n        this.clientId = clientId;\n    }\n\n    public int getMaxCount() {\n        return maxCount;\n    }\n\n    public void setMaxCount(int maxCount) {\n        this.maxCount = maxCount;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetLiteGroupInfoRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class GetLiteGroupInfoRequestHeader implements CommandCustomHeader {\n\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String group;\n\n    private String liteTopic;\n\n    private int topK;\n\n    public String getGroup() {\n        return group;\n    }\n\n    public void setGroup(String group) {\n        this.group = group;\n    }\n\n    public String getLiteTopic() {\n        return liteTopic;\n    }\n\n    public void setLiteTopic(String liteTopic) {\n        this.liteTopic = liteTopic;\n    }\n\n    public int getTopK() {\n        return topK;\n    }\n\n    public void setTopK(int topK) {\n        this.topK = topK;\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetLiteTopicInfoRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class GetLiteTopicInfoRequestHeader implements CommandCustomHeader {\n\n    private String parentTopic;\n    private String liteTopic;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getParentTopic() {\n        return parentTopic;\n    }\n\n    public void setParentTopic(String parentTopic) {\n        this.parentTopic = parentTopic;\n    }\n\n    public String getLiteTopic() {\n        return liteTopic;\n    }\n\n    public void setLiteTopic(String liteTopic) {\n        this.liteTopic = liteTopic;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetMaxOffsetRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: GetMaxOffsetRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;\n\n@RocketMQAction(value = RequestCode.GET_MAX_OFFSET, action = Action.GET)\npublic class GetMaxOffsetRequestHeader extends TopicQueueRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNotNull\n    private Integer queueId;\n\n    /**\n     * A message at committed offset has been dispatched from Topic to MessageQueue, so it can be consumed immediately,\n     * while a message at inflight offset is not visible for a consumer temporarily.\n     * Set this flag true if the max committed offset is needed, or false if the max inflight offset is preferred.\n     * The default value is true.\n     */\n    @CFNullable\n    private boolean committed = true;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    @Override\n    public String getTopic() {\n        return topic;\n    }\n\n    @Override\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    @Override\n    public Integer getQueueId() {\n        return queueId;\n    }\n\n    @Override\n    public void setQueueId(Integer queueId) {\n        this.queueId = queueId;\n    }\n\n    public boolean isCommitted() {\n        return committed;\n    }\n\n    public void setCommitted(final boolean committed) {\n        this.committed = committed;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"topic\", topic)\n            .add(\"queueId\", queueId)\n            .add(\"committed\", committed)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetMaxOffsetResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: GetMaxOffsetResponseHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class GetMaxOffsetResponseHeader implements CommandCustomHeader {\n    @CFNotNull\n    private Long offset;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public Long getOffset() {\n        return offset;\n    }\n\n    public void setOffset(Long offset) {\n        this.offset = offset;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetMinOffsetRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: GetMinOffsetRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;\n\n@RocketMQAction(value = RequestCode.GET_MIN_OFFSET, action = Action.GET)\npublic class GetMinOffsetRequestHeader extends TopicQueueRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNotNull\n    private Integer queueId;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    @Override\n    public String getTopic() {\n        return topic;\n    }\n\n    @Override\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    @Override\n    public Integer getQueueId() {\n        return queueId;\n    }\n\n    @Override\n    public void setQueueId(Integer queueId) {\n        this.queueId = queueId;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"topic\", topic)\n            .add(\"queueId\", queueId)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetMinOffsetResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: GetMinOffsetResponseHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class GetMinOffsetResponseHeader implements CommandCustomHeader {\n    @CFNotNull\n    private Long offset;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public Long getOffset() {\n        return offset;\n    }\n\n    public void setOffset(Long offset) {\n        this.offset = offset;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetParentTopicInfoRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class GetParentTopicInfoRequestHeader implements CommandCustomHeader {\n\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetProducerConnectionListRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.RpcRequestHeader;\n\n@RocketMQAction(value = RequestCode.GET_PRODUCER_CONNECTION_LIST, resource = ResourceType.CLUSTER, action = Action.GET)\npublic class GetProducerConnectionListRequestHeader extends RpcRequestHeader {\n    @CFNotNull\n    private String producerGroup;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n        // To change body of implemented methods use File | Settings | File\n        // Templates.\n    }\n\n    public String getProducerGroup() {\n        return producerGroup;\n    }\n\n    public void setProducerGroup(String producerGroup) {\n        this.producerGroup = producerGroup;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetSubscriptionGroupConfigRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: GetAllTopicConfigResponseHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.RpcRequestHeader;\n\n@RocketMQAction(value = RequestCode.GET_SUBSCRIPTIONGROUP_CONFIG, action = Action.GET)\npublic class GetSubscriptionGroupConfigRequestHeader extends RpcRequestHeader {\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String group;\n\n    /**\n     * @return the group\n     */\n    public String getGroup() {\n        return group;\n    }\n\n    /**\n     * @param group the group to set\n     */\n    public void setGroup(String group) {\n        this.group = group;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetTopicConfigRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicRequestHeader;\n\n@RocketMQAction(value = RequestCode.GET_TOPIC_CONFIG, action = Action.GET)\npublic class GetTopicConfigRequestHeader extends TopicRequestHeader {\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n\n\n    /**\n     * @return the topic\n     */\n    public String getTopic() {\n        return topic;\n    }\n\n    /**\n     * @param topic the topic to set\n     */\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetTopicStatsInfoRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicRequestHeader;\n\n@RocketMQAction(value = RequestCode.GET_TOPIC_STATS_INFO, action = Action.GET)\npublic class GetTopicStatsInfoRequestHeader extends TopicRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetTopicsByClusterRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.GET_TOPICS_BY_CLUSTER, resource = ResourceType.TOPIC, action = Action.LIST)\npublic class GetTopicsByClusterRequestHeader implements CommandCustomHeader {\n    @CFNotNull\n    private String cluster;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getCluster() {\n        return cluster;\n    }\n\n    public void setCluster(String cluster) {\n        this.cluster = cluster;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetUserRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.AUTH_GET_USER, resource = ResourceType.CLUSTER, action = Action.GET)\npublic class GetUserRequestHeader implements CommandCustomHeader {\n\n    private String username;\n\n    public GetUserRequestHeader() {\n    }\n\n    public GetUserRequestHeader(String username) {\n        this.username = username;\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getUsername() {\n        return username;\n    }\n\n    public void setUsername(String username) {\n        this.username = username;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/HeartbeatRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.RpcRequestHeader;\n\n@RocketMQAction(value = RequestCode.HEART_BEAT, resource = ResourceType.GROUP, action = {Action.PUB, Action.SUB})\npublic class HeartbeatRequestHeader extends RpcRequestHeader {\n    // for namespace\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/InitConsumerOffsetRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.rpc.TopicRequestHeader;\n\npublic class InitConsumerOffsetRequestHeader extends TopicRequestHeader {\n\n    private String topic;\n    // @see ConsumeInitMode\n    private int initMode;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public int getInitMode() {\n        return initMode;\n    }\n\n    public void setInitMode(int initMode) {\n        this.initMode = initMode;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ListAclsRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.AUTH_LIST_ACL, resource = ResourceType.CLUSTER, action = Action.GET)\npublic class ListAclsRequestHeader implements CommandCustomHeader {\n\n    private String subjectFilter;\n\n    private String resourceFilter;\n\n    public ListAclsRequestHeader() {\n    }\n\n    public ListAclsRequestHeader(String subjectFilter, String resourceFilter) {\n        this.subjectFilter = subjectFilter;\n        this.resourceFilter = resourceFilter;\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getSubjectFilter() {\n        return subjectFilter;\n    }\n\n    public void setSubjectFilter(String subjectFilter) {\n        this.subjectFilter = subjectFilter;\n    }\n\n    public String getResourceFilter() {\n        return resourceFilter;\n    }\n\n    public void setResourceFilter(String resourceFilter) {\n        this.resourceFilter = resourceFilter;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ListUsersRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.AUTH_LIST_USER, resource = ResourceType.CLUSTER, action = Action.GET)\npublic class ListUsersRequestHeader implements CommandCustomHeader {\n\n    private String filter;\n\n    public ListUsersRequestHeader() {\n    }\n\n    public ListUsersRequestHeader(String filter) {\n        this.filter = filter;\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getFilter() {\n        return filter;\n    }\n\n    public void setFilter(String filter) {\n        this.filter = filter;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/LiteSubscriptionCtlRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.rpc.RpcRequestHeader;\n\npublic class LiteSubscriptionCtlRequestHeader extends RpcRequestHeader {\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/LockBatchMqRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.RpcRequestHeader;\n\n@RocketMQAction(value = RequestCode.LOCK_BATCH_MQ, action = Action.SUB)\npublic class LockBatchMqRequestHeader extends RpcRequestHeader {\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotificationRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;\n\n@RocketMQAction(value = RequestCode.NOTIFICATION, action = Action.SUB)\npublic class NotificationRequestHeader extends TopicQueueRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String consumerGroup;\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNotNull\n    private int queueId;\n    @CFNotNull\n    private long pollTime;\n    @CFNotNull\n    private long bornTime;\n\n    private Boolean order = Boolean.FALSE;\n    private String attemptId;\n\n    private String expType;\n    private String exp;\n\n    @CFNotNull\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public long getPollTime() {\n        return pollTime;\n    }\n\n    public void setPollTime(long pollTime) {\n        this.pollTime = pollTime;\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public long getBornTime() {\n        return bornTime;\n    }\n\n    public void setBornTime(long bornTime) {\n        this.bornTime = bornTime;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public Integer getQueueId() {\n        if (queueId < 0) {\n            return -1;\n        }\n        return queueId;\n    }\n\n    public void setQueueId(Integer queueId) {\n        this.queueId = queueId;\n    }\n\n    public Boolean getOrder() {\n        return order;\n    }\n\n    public void setOrder(Boolean order) {\n        this.order = order;\n    }\n\n    public String getAttemptId() {\n        return attemptId;\n    }\n\n    public void setAttemptId(String attemptId) {\n        this.attemptId = attemptId;\n    }\n\n    public String getExpType() {\n        return expType;\n    }\n\n    public void setExpType(String expType) {\n        this.expType = expType;\n    }\n\n    public String getExp() {\n        return exp;\n    }\n\n    public void setExp(String exp) {\n        this.exp = exp;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"consumerGroup\", consumerGroup)\n            .add(\"topic\", topic)\n            .add(\"queueId\", queueId)\n            .add(\"pollTime\", pollTime)\n            .add(\"bornTime\", bornTime)\n            .add(\"order\", order)\n            .add(\"attemptId\", attemptId)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotificationResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class NotificationResponseHeader implements CommandCustomHeader {\n\n\n    @CFNotNull\n    private boolean hasMsg = false;\n\n    private boolean pollingFull = false;\n\n    public boolean isHasMsg() {\n        return hasMsg;\n    }\n\n    public boolean isPollingFull() {\n        return pollingFull;\n    }\n\n    public void setPollingFull(boolean pollingFull) {\n        this.pollingFull = pollingFull;\n    }\n\n    public void setHasMsg(boolean hasMsg) {\n        this.hasMsg = hasMsg;\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotifyBrokerRoleChangedRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.NOTIFY_BROKER_ROLE_CHANGED, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class NotifyBrokerRoleChangedRequestHeader implements CommandCustomHeader {\n    private String masterAddress;\n    private Integer masterEpoch;\n    private Integer syncStateSetEpoch;\n    private Long masterBrokerId;\n\n    public NotifyBrokerRoleChangedRequestHeader() {\n    }\n\n    public NotifyBrokerRoleChangedRequestHeader(String masterAddress, Long masterBrokerId, Integer masterEpoch, Integer syncStateSetEpoch) {\n        this.masterAddress = masterAddress;\n        this.masterEpoch = masterEpoch;\n        this.syncStateSetEpoch = syncStateSetEpoch;\n        this.masterBrokerId = masterBrokerId;\n    }\n\n    public String getMasterAddress() {\n        return masterAddress;\n    }\n\n    public void setMasterAddress(String masterAddress) {\n        this.masterAddress = masterAddress;\n    }\n\n    public Integer getMasterEpoch() {\n        return masterEpoch;\n    }\n\n    public void setMasterEpoch(Integer masterEpoch) {\n        this.masterEpoch = masterEpoch;\n    }\n\n    public Integer getSyncStateSetEpoch() {\n        return syncStateSetEpoch;\n    }\n\n    public void setSyncStateSetEpoch(Integer syncStateSetEpoch) {\n        this.syncStateSetEpoch = syncStateSetEpoch;\n    }\n\n    public Long getMasterBrokerId() {\n        return masterBrokerId;\n    }\n\n    public void setMasterBrokerId(Long masterBrokerId) {\n        this.masterBrokerId = masterBrokerId;\n    }\n\n    @Override\n    public String toString() {\n        return \"NotifyBrokerRoleChangedRequestHeader{\" +\n                \"masterAddress='\" + masterAddress + '\\'' +\n                \", masterEpoch=\" + masterEpoch +\n                \", syncStateSetEpoch=\" + syncStateSetEpoch +\n                \", masterBrokerId=\" + masterBrokerId +\n                '}';\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotifyConsumerIdsChangedRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.rpc.RpcRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.NOTIFY_CONSUMER_IDS_CHANGED, action = Action.SUB)\npublic class NotifyConsumerIdsChangedRequestHeader extends RpcRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String consumerGroup;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotifyMinBrokerIdChangeRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.NOTIFY_MIN_BROKER_ID_CHANGE, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class NotifyMinBrokerIdChangeRequestHeader implements CommandCustomHeader {\n    @CFNullable\n    private Long minBrokerId;\n\n    @CFNullable\n    private String brokerName;\n\n    @CFNullable\n    private String minBrokerAddr;\n\n    @CFNullable\n    private String offlineBrokerAddr;\n\n    @CFNullable\n    private String haBrokerAddr;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public Long getMinBrokerId() {\n        return minBrokerId;\n    }\n\n    public void setMinBrokerId(Long minBrokerId) {\n        this.minBrokerId = minBrokerId;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public String getMinBrokerAddr() {\n        return minBrokerAddr;\n    }\n\n    public void setMinBrokerAddr(String minBrokerAddr) {\n        this.minBrokerAddr = minBrokerAddr;\n    }\n\n    public String getOfflineBrokerAddr() {\n        return offlineBrokerAddr;\n    }\n\n    public void setOfflineBrokerAddr(String offlineBrokerAddr) {\n        this.offlineBrokerAddr = offlineBrokerAddr;\n    }\n\n    public String getHaBrokerAddr() {\n        return haBrokerAddr;\n    }\n\n    public void setHaBrokerAddr(String haBrokerAddr) {\n        this.haBrokerAddr = haBrokerAddr;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotifyUnsubscribeLiteRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.RpcRequestHeader;\n\n@RocketMQAction(value = RequestCode.NOTIFY_UNSUBSCRIBE_LITE, action = Action.SUB)\npublic class NotifyUnsubscribeLiteRequestHeader extends RpcRequestHeader {\n\n    @CFNotNull\n    private String liteTopic;\n\n    @RocketMQResource(ResourceType.GROUP)\n    @CFNotNull\n    private String consumerGroup;\n\n    @CFNotNull\n    private String clientId;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getLiteTopic() {\n        return liteTopic;\n    }\n\n    public void setLiteTopic(String liteTopic) {\n        this.liteTopic = liteTopic;\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public String getClientId() {\n        return clientId;\n    }\n\n    public void setClientId(String clientId) {\n        this.clientId = clientId;\n    }\n\n    @Override\n    public String toString() {\n        return \"NotifyUnsubscribeLiteRequestHeader{\" +\n            \"liteTopic='\" + liteTopic + '\\'' +\n            \", consumerGroup='\" + consumerGroup + '\\'' +\n            \", clientId='\" + clientId + '\\'' +\n            '}';\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PeekMessageRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;\n\n@RocketMQAction(value = RequestCode.PEEK_MESSAGE, action = Action.SUB)\npublic class PeekMessageRequestHeader extends TopicQueueRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNotNull\n    private int queueId;\n    @CFNotNull\n    private int maxMsgNums;\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String consumerGroup;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public Integer getQueueId() {\n        return queueId;\n    }\n\n    public void setQueueId(Integer queueId) {\n        this.queueId = queueId;\n    }\n\n\n    public int getMaxMsgNums() {\n        return maxMsgNums;\n    }\n\n    public void setMaxMsgNums(int maxMsgNums) {\n        this.maxMsgNums = maxMsgNums;\n    }\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PollingInfoRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;\n\n@RocketMQAction(value = RequestCode.POLLING_INFO, action = Action.GET)\npublic class PollingInfoRequestHeader extends TopicQueueRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String consumerGroup;\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNotNull\n    private int queueId;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public Integer getQueueId() {\n        if (queueId < 0) {\n            return -1;\n        }\n        return queueId;\n    }\n\n    public void setQueueId(Integer queueId) {\n        this.queueId = queueId;\n    }\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PollingInfoResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class PollingInfoResponseHeader implements CommandCustomHeader {\n\n\n    @CFNotNull\n    private int pollingNum;\n\n    public int getPollingNum() {\n        return pollingNum;\n    }\n\n    public void setPollingNum(int pollingNum) {\n        this.pollingNum = pollingNum;\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PopLiteMessageRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.rpc.RpcRequestHeader;\n\npublic class PopLiteMessageRequestHeader extends RpcRequestHeader {\n\n    @CFNotNull\n    private String clientId;\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String consumerGroup;\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNotNull\n    private int maxMsgNum;\n    @CFNotNull\n    private long invisibleTime;\n    @CFNotNull\n    private long pollTime;\n    @CFNotNull\n    private long bornTime;\n\n    private String attemptId;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getClientId() {\n        return clientId;\n    }\n\n    public void setClientId(String clientId) {\n        this.clientId = clientId;\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public int getMaxMsgNum() {\n        return maxMsgNum;\n    }\n\n    public void setMaxMsgNum(int maxMsgNum) {\n        this.maxMsgNum = maxMsgNum;\n    }\n\n    public long getInvisibleTime() {\n        return invisibleTime;\n    }\n\n    public void setInvisibleTime(long invisibleTime) {\n        this.invisibleTime = invisibleTime;\n    }\n\n    public long getPollTime() {\n        return pollTime;\n    }\n\n    public void setPollTime(long pollTime) {\n        this.pollTime = pollTime;\n    }\n\n    public long getBornTime() {\n        return bornTime;\n    }\n\n    public void setBornTime(long bornTime) {\n        this.bornTime = bornTime;\n    }\n\n    public String getAttemptId() {\n        return attemptId;\n    }\n\n    public void setAttemptId(String attemptId) {\n        this.attemptId = attemptId;\n    }\n\n    public boolean isTimeoutTooMuch() {\n        return System.currentTimeMillis() - bornTime - pollTime > 500;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"consumerGroup\", consumerGroup)\n            .add(\"topic\", topic)\n            .add(\"maxMsgNum\", maxMsgNum)\n            .add(\"invisibleTime\", invisibleTime)\n            .add(\"pollTime\", pollTime)\n            .add(\"bornTime\", bornTime)\n            .add(\"attemptId\", attemptId)\n            .add(\"clientId\", clientId)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PopLiteMessageResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class PopLiteMessageResponseHeader implements CommandCustomHeader {\n\n    @CFNotNull\n    private long popTime;\n    @CFNotNull\n    private long invisibleTime;\n    @CFNotNull\n    private int reviveQid; // reuse current ack implementation\n\n    private String startOffsetInfo;\n    private String msgOffsetInfo;\n    private String orderCountInfo;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public long getPopTime() {\n        return popTime;\n    }\n\n    public void setPopTime(long popTime) {\n        this.popTime = popTime;\n    }\n\n    public long getInvisibleTime() {\n        return invisibleTime;\n    }\n\n    public void setInvisibleTime(long invisibleTime) {\n        this.invisibleTime = invisibleTime;\n    }\n\n    public int getReviveQid() {\n        return reviveQid;\n    }\n\n    public void setReviveQid(int reviveQid) {\n        this.reviveQid = reviveQid;\n    }\n\n    public String getStartOffsetInfo() {\n        return startOffsetInfo;\n    }\n\n    public void setStartOffsetInfo(String startOffsetInfo) {\n        this.startOffsetInfo = startOffsetInfo;\n    }\n\n    public String getMsgOffsetInfo() {\n        return msgOffsetInfo;\n    }\n\n    public void setMsgOffsetInfo(String msgOffsetInfo) {\n        this.msgOffsetInfo = msgOffsetInfo;\n    }\n\n    public String getOrderCountInfo() {\n        return orderCountInfo;\n    }\n\n    public void setOrderCountInfo(String orderCountInfo) {\n        this.orderCountInfo = orderCountInfo;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PopMessageRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;\n\n@RocketMQAction(value = RequestCode.POP_MESSAGE, action = Action.SUB)\npublic class PopMessageRequestHeader extends TopicQueueRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String consumerGroup;\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNotNull\n    private int queueId;\n    @CFNotNull\n    private int maxMsgNums;\n    @CFNotNull\n    private long invisibleTime;\n    @CFNotNull\n    private long pollTime;\n    @CFNotNull\n    private long bornTime;\n    @CFNotNull\n    private int initMode;\n\n    private String expType;\n    private String exp;\n\n    /**\n     * marked as order consume, if true\n     * 1. not commit offset\n     * 2. not pop retry, because no retry\n     * 3. not append check point, because no retry\n     */\n    private Boolean order = Boolean.FALSE;\n\n    private String attemptId;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public void setInitMode(int initMode) {\n        this.initMode = initMode;\n    }\n\n    public int getInitMode() {\n        return initMode;\n    }\n\n    public long getInvisibleTime() {\n        return invisibleTime;\n    }\n\n    public void setInvisibleTime(long invisibleTime) {\n        this.invisibleTime = invisibleTime;\n    }\n\n    public long getPollTime() {\n        return pollTime;\n    }\n\n    public void setPollTime(long pollTime) {\n        this.pollTime = pollTime;\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public long getBornTime() {\n        return bornTime;\n    }\n\n    public void setBornTime(long bornTime) {\n        this.bornTime = bornTime;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public Integer getQueueId() {\n        if (queueId < 0) {\n            return -1;\n        }\n        return queueId;\n    }\n\n    @Override\n    public void setQueueId(Integer queueId) {\n        this.queueId = queueId;\n    }\n\n    public int getMaxMsgNums() {\n        return maxMsgNums;\n    }\n\n    public void setMaxMsgNums(int maxMsgNums) {\n        this.maxMsgNums = maxMsgNums;\n    }\n\n    public boolean isTimeoutTooMuch() {\n        return System.currentTimeMillis() - bornTime - pollTime > 500;\n    }\n\n    public String getExpType() {\n        return expType;\n    }\n\n    public void setExpType(String expType) {\n        this.expType = expType;\n    }\n\n    public String getExp() {\n        return exp;\n    }\n\n    public void setExp(String exp) {\n        this.exp = exp;\n    }\n\n    public Boolean getOrder() {\n        return order;\n    }\n\n    public void setOrder(Boolean order) {\n        this.order = order;\n    }\n\n    public boolean isOrder() {\n        return this.order != null && this.order.booleanValue();\n    }\n\n    public String getAttemptId() {\n        return attemptId;\n    }\n\n    public void setAttemptId(String attemptId) {\n        this.attemptId = attemptId;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"consumerGroup\", consumerGroup)\n            .add(\"topic\", topic)\n            .add(\"queueId\", queueId)\n            .add(\"maxMsgNums\", maxMsgNums)\n            .add(\"invisibleTime\", invisibleTime)\n            .add(\"pollTime\", pollTime)\n            .add(\"bornTime\", bornTime)\n            .add(\"initMode\", initMode)\n            .add(\"expType\", expType)\n            .add(\"exp\", exp)\n            .add(\"order\", order)\n            .add(\"attemptId\", attemptId)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PopMessageResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class PopMessageResponseHeader implements CommandCustomHeader {\n\n\n    @CFNotNull\n    private long popTime;\n    @CFNotNull\n    private long invisibleTime;\n\n    @CFNotNull\n    private int reviveQid;\n    /**\n     * the rest num in queue\n     */\n    @CFNotNull\n    private long restNum;\n\n    private String startOffsetInfo;\n    private String msgOffsetInfo;\n    private String orderCountInfo;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public long getPopTime() {\n        return popTime;\n    }\n\n    public void setPopTime(long popTime) {\n        this.popTime = popTime;\n    }\n\n    public long getInvisibleTime() {\n        return invisibleTime;\n    }\n\n    public long getRestNum() {\n        return restNum;\n    }\n\n    public void setRestNum(long restNum) {\n        this.restNum = restNum;\n    }\n\n    public void setInvisibleTime(long invisibleTime) {\n        this.invisibleTime = invisibleTime;\n    }\n\n    public int getReviveQid() {\n        return reviveQid;\n    }\n\n    public void setReviveQid(int reviveQid) {\n        this.reviveQid = reviveQid;\n    }\n\n    public String getStartOffsetInfo() {\n        return startOffsetInfo;\n    }\n\n    public void setStartOffsetInfo(String startOffsetInfo) {\n        this.startOffsetInfo = startOffsetInfo;\n    }\n\n    public String getMsgOffsetInfo() {\n        return msgOffsetInfo;\n    }\n\n    public void setMsgOffsetInfo(String msgOffsetInfo) {\n        this.msgOffsetInfo = msgOffsetInfo;\n    }\n\n    public String getOrderCountInfo() {\n        return orderCountInfo;\n    }\n\n    public void setOrderCountInfo(String orderCountInfo) {\n        this.orderCountInfo = orderCountInfo;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PullMessageRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: PullMessageRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport io.netty.buffer.ByteBuf;\nimport java.util.HashMap;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.FastCodesHeader;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;\n\n@RocketMQAction(value = RequestCode.PULL_MESSAGE, action = Action.SUB)\npublic class PullMessageRequestHeader extends TopicQueueRequestHeader implements FastCodesHeader {\n\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String consumerGroup;\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    private String liteTopic;\n    @CFNotNull\n    private Integer queueId;\n    @CFNotNull\n    private Long queueOffset;\n    @CFNotNull\n    private Integer maxMsgNums;\n    @CFNotNull\n    private Integer sysFlag;\n    @CFNotNull\n    private Long commitOffset;\n    @CFNotNull\n    private Long suspendTimeoutMillis;\n    @CFNullable\n    private String subscription;\n    @CFNotNull\n    private Long subVersion;\n    private String expressionType;\n\n    @CFNullable\n    private Integer maxMsgBytes;\n\n    /**\n     * mark the source of this pull request\n     */\n    private Integer requestSource;\n\n    /**\n     * the real clientId when request from proxy\n     */\n    private String proxyFrowardClientId;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    @Override\n    public void encode(ByteBuf out) {\n        writeIfNotNull(out, \"consumerGroup\", consumerGroup);\n        writeIfNotNull(out, \"topic\", topic);\n        writeIfNotNull(out, \"liteTopic\", liteTopic);\n        writeIfNotNull(out, \"queueId\", queueId);\n        writeIfNotNull(out, \"queueOffset\", queueOffset);\n        writeIfNotNull(out, \"maxMsgNums\", maxMsgNums);\n        writeIfNotNull(out, \"sysFlag\", sysFlag);\n        writeIfNotNull(out, \"commitOffset\", commitOffset);\n        writeIfNotNull(out, \"suspendTimeoutMillis\", suspendTimeoutMillis);\n        writeIfNotNull(out, \"subscription\", subscription);\n        writeIfNotNull(out, \"subVersion\", subVersion);\n        writeIfNotNull(out, \"expressionType\", expressionType);\n        writeIfNotNull(out, \"maxMsgBytes\", maxMsgBytes);\n        writeIfNotNull(out, \"requestSource\", requestSource);\n        writeIfNotNull(out, \"proxyFrowardClientId\", proxyFrowardClientId);\n        writeIfNotNull(out, \"lo\", lo);\n        writeIfNotNull(out, \"ns\", ns);\n        writeIfNotNull(out, \"nsd\", nsd);\n        writeIfNotNull(out, \"bname\", bname);\n        writeIfNotNull(out, \"oway\", oway);\n    }\n\n    @Override\n    public void decode(HashMap<String, String> fields) throws RemotingCommandException {\n        String str = getAndCheckNotNull(fields, \"consumerGroup\");\n        if (str != null) {\n            this.consumerGroup = str;\n        }\n\n        str = getAndCheckNotNull(fields, \"topic\");\n        if (str != null) {\n            this.topic = str;\n        }\n\n        str = fields.get(\"liteTopic\");\n        if (str != null) {\n            this.liteTopic = str;\n        }\n\n        str = getAndCheckNotNull(fields, \"queueId\");\n        if (str != null) {\n            this.queueId = Integer.parseInt(str);\n        }\n\n        str = getAndCheckNotNull(fields, \"queueOffset\");\n        if (str != null) {\n            this.queueOffset = Long.parseLong(str);\n        }\n\n        str = getAndCheckNotNull(fields, \"maxMsgNums\");\n        if (str != null) {\n            this.maxMsgNums = Integer.parseInt(str);\n        }\n\n        str = getAndCheckNotNull(fields, \"sysFlag\");\n        if (str != null) {\n            this.sysFlag = Integer.parseInt(str);\n        }\n\n        str = getAndCheckNotNull(fields, \"commitOffset\");\n        if (str != null) {\n            this.commitOffset = Long.parseLong(str);\n        }\n\n        str = getAndCheckNotNull(fields, \"suspendTimeoutMillis\");\n        if (str != null) {\n            this.suspendTimeoutMillis = Long.parseLong(str);\n        }\n\n        str = fields.get(\"subscription\");\n        if (str != null) {\n            this.subscription = str;\n        }\n\n        str = getAndCheckNotNull(fields, \"subVersion\");\n        if (str != null) {\n            this.subVersion = Long.parseLong(str);\n        }\n\n        str = fields.get(\"expressionType\");\n        if (str != null) {\n            this.expressionType = str;\n        }\n\n        str = fields.get(\"maxMsgBytes\");\n        if (str != null) {\n            this.maxMsgBytes = Integer.parseInt(str);\n        }\n\n        str = fields.get(\"requestSource\");\n        if (str != null) {\n            this.requestSource = Integer.parseInt(str);\n        }\n\n        str = fields.get(\"proxyFrowardClientId\");\n        if (str != null) {\n            this.proxyFrowardClientId = str;\n        }\n\n        str = fields.get(\"lo\");\n        if (str != null) {\n            this.lo = Boolean.parseBoolean(str);\n        }\n\n        str = fields.get(\"ns\");\n        if (str != null) {\n            this.ns = str;\n        }\n\n        str = fields.get(\"nsd\");\n        if (str != null) {\n            this.nsd = Boolean.parseBoolean(str);\n        }\n\n        str = fields.get(\"bname\");\n        if (str != null) {\n            this.bname = str;\n        }\n\n        str = fields.get(\"oway\");\n        if (str != null) {\n            this.oway = Boolean.parseBoolean(str);\n        }\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    @Override\n    public String getTopic() {\n        return topic;\n    }\n\n    @Override\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getLiteTopic() {\n        return liteTopic;\n    }\n\n    public void setLiteTopic(String liteTopic) {\n        this.liteTopic = liteTopic;\n    }\n\n    @Override\n    public Integer getQueueId() {\n        return queueId;\n    }\n\n    @Override\n    public void setQueueId(Integer queueId) {\n        this.queueId = queueId;\n    }\n\n    public Long getQueueOffset() {\n        return queueOffset;\n    }\n\n    public void setQueueOffset(Long queueOffset) {\n        this.queueOffset = queueOffset;\n    }\n\n    public Integer getMaxMsgNums() {\n        return maxMsgNums;\n    }\n\n    public void setMaxMsgNums(Integer maxMsgNums) {\n        this.maxMsgNums = maxMsgNums;\n    }\n\n    public Integer getSysFlag() {\n        return sysFlag;\n    }\n\n    public void setSysFlag(Integer sysFlag) {\n        this.sysFlag = sysFlag;\n    }\n\n    public Long getCommitOffset() {\n        return commitOffset;\n    }\n\n    public void setCommitOffset(Long commitOffset) {\n        this.commitOffset = commitOffset;\n    }\n\n    public Long getSuspendTimeoutMillis() {\n        return suspendTimeoutMillis;\n    }\n\n    public void setSuspendTimeoutMillis(Long suspendTimeoutMillis) {\n        this.suspendTimeoutMillis = suspendTimeoutMillis;\n    }\n\n    public String getSubscription() {\n        return subscription;\n    }\n\n    public void setSubscription(String subscription) {\n        this.subscription = subscription;\n    }\n\n    public Long getSubVersion() {\n        return subVersion;\n    }\n\n    public void setSubVersion(Long subVersion) {\n        this.subVersion = subVersion;\n    }\n\n    public String getExpressionType() {\n        return expressionType;\n    }\n\n    public void setExpressionType(String expressionType) {\n        this.expressionType = expressionType;\n    }\n\n    public Integer getMaxMsgBytes() {\n        return maxMsgBytes;\n    }\n\n    public void setMaxMsgBytes(Integer maxMsgBytes) {\n        this.maxMsgBytes = maxMsgBytes;\n    }\n\n    public Integer getRequestSource() {\n        return requestSource;\n    }\n\n    public void setRequestSource(Integer requestSource) {\n        this.requestSource = requestSource;\n    }\n\n    public String getProxyFrowardClientId() {\n        return proxyFrowardClientId;\n    }\n\n    public void setProxyFrowardClientId(String proxyFrowardClientId) {\n        this.proxyFrowardClientId = proxyFrowardClientId;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"consumerGroup\", consumerGroup)\n            .add(\"topic\", topic)\n            .add(\"liteTopic\", liteTopic)\n            .add(\"queueId\", queueId)\n            .add(\"queueOffset\", queueOffset)\n            .add(\"maxMsgBytes\", maxMsgBytes)\n            .add(\"maxMsgNums\", maxMsgNums)\n            .add(\"sysFlag\", sysFlag)\n            .add(\"commitOffset\", commitOffset)\n            .add(\"suspendTimeoutMillis\", suspendTimeoutMillis)\n            .add(\"subscription\", subscription)\n            .add(\"subVersion\", subVersion)\n            .add(\"expressionType\", expressionType)\n            .add(\"requestSource\", requestSource)\n            .add(\"proxyFrowardClientId\", proxyFrowardClientId)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PullMessageResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: PullMessageResponseHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport io.netty.buffer.ByteBuf;\nimport java.util.HashMap;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.FastCodesHeader;\n\npublic class PullMessageResponseHeader implements CommandCustomHeader, FastCodesHeader {\n    @CFNotNull\n    private Long suggestWhichBrokerId;\n    @CFNotNull\n    private Long nextBeginOffset;\n    @CFNotNull\n    private Long minOffset;\n    @CFNotNull\n    private Long maxOffset;\n    @CFNullable\n    private Long offsetDelta;\n    @CFNullable\n    private Integer topicSysFlag;\n    @CFNullable\n    private Integer groupSysFlag;\n    @CFNullable\n    private Integer forbiddenType;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    @Override\n    public void encode(ByteBuf out) {\n        writeIfNotNull(out, \"suggestWhichBrokerId\", suggestWhichBrokerId);\n        writeIfNotNull(out, \"nextBeginOffset\", nextBeginOffset);\n        writeIfNotNull(out, \"minOffset\", minOffset);\n        writeIfNotNull(out, \"maxOffset\", maxOffset);\n        writeIfNotNull(out, \"offsetDelta\", offsetDelta);\n        writeIfNotNull(out, \"topicSysFlag\", topicSysFlag);\n        writeIfNotNull(out, \"groupSysFlag\", groupSysFlag);\n        writeIfNotNull(out, \"forbiddenType\", forbiddenType);\n    }\n\n    @Override\n    public void decode(HashMap<String, String> fields) throws RemotingCommandException {\n        String str = getAndCheckNotNull(fields, \"suggestWhichBrokerId\");\n        if (str != null) {\n            this.suggestWhichBrokerId = Long.parseLong(str);\n        }\n\n        str = getAndCheckNotNull(fields, \"nextBeginOffset\");\n        if (str != null) {\n            this.nextBeginOffset = Long.parseLong(str);\n        }\n\n        str = getAndCheckNotNull(fields, \"minOffset\");\n        if (str != null) {\n            this.minOffset = Long.parseLong(str);\n        }\n\n        str = getAndCheckNotNull(fields, \"maxOffset\");\n        if (str != null) {\n            this.maxOffset = Long.parseLong(str);\n        }\n\n        str = fields.get(\"offsetDelta\");\n        if (str != null) {\n            this.offsetDelta = Long.parseLong(str);\n        }\n\n        str = fields.get(\"topicSysFlag\");\n        if (str != null) {\n            this.topicSysFlag = Integer.parseInt(str);\n        }\n\n        str = fields.get(\"groupSysFlag\");\n        if (str != null) {\n            this.groupSysFlag = Integer.parseInt(str);\n        }\n\n        str = fields.get(\"forbiddenType\");\n        if (str != null) {\n            this.forbiddenType = Integer.parseInt(str);\n        }\n\n    }\n\n    public Long getNextBeginOffset() {\n        return nextBeginOffset;\n    }\n\n    public void setNextBeginOffset(Long nextBeginOffset) {\n        this.nextBeginOffset = nextBeginOffset;\n    }\n\n    public Long getMinOffset() {\n        return minOffset;\n    }\n\n    public void setMinOffset(Long minOffset) {\n        this.minOffset = minOffset;\n    }\n\n    public Long getMaxOffset() {\n        return maxOffset;\n    }\n\n    public void setMaxOffset(Long maxOffset) {\n        this.maxOffset = maxOffset;\n    }\n\n    public Long getSuggestWhichBrokerId() {\n        return suggestWhichBrokerId;\n    }\n\n    public void setSuggestWhichBrokerId(Long suggestWhichBrokerId) {\n        this.suggestWhichBrokerId = suggestWhichBrokerId;\n    }\n\n    public Integer getTopicSysFlag() {\n        return topicSysFlag;\n    }\n\n    public void setTopicSysFlag(Integer topicSysFlag) {\n        this.topicSysFlag = topicSysFlag;\n    }\n\n    public Integer getGroupSysFlag() {\n        return groupSysFlag;\n    }\n\n    public void setGroupSysFlag(Integer groupSysFlag) {\n        this.groupSysFlag = groupSysFlag;\n    }\n\n    public Integer getForbiddenType() {\n        return forbiddenType;\n    }\n\n    public void setForbiddenType(Integer forbiddenType) {\n        this.forbiddenType = forbiddenType;\n    }\n\n    public Long getOffsetDelta() {\n        return offsetDelta;\n    }\n\n    public void setOffsetDelta(Long offsetDelta) {\n        this.offsetDelta = offsetDelta;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryConsumeQueueRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.QUERY_CONSUME_QUEUE, action = Action.GET)\npublic class QueryConsumeQueueRequestHeader extends TopicQueueRequestHeader {\n\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    private int queueId;\n    private long index;\n    private int count;\n    @RocketMQResource(ResourceType.GROUP)\n    private String consumerGroup;\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public Integer getQueueId() {\n        return queueId;\n    }\n\n    public void setQueueId(Integer queueId) {\n        this.queueId = queueId;\n    }\n\n    public long getIndex() {\n        return index;\n    }\n\n    public void setIndex(long index) {\n        this.index = index;\n    }\n\n    public int getCount() {\n        return count;\n    }\n\n    public void setCount(int count) {\n        this.count = count;\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryConsumeTimeSpanRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicRequestHeader;\n\n@RocketMQAction(value = RequestCode.QUERY_CONSUME_TIME_SPAN, action = Action.GET)\npublic class QueryConsumeTimeSpanRequestHeader extends TopicRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String group;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getGroup() {\n        return group;\n    }\n\n    public void setGroup(String group) {\n        this.group = group;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryConsumerOffsetRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: QueryConsumerOffsetRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;\n\n@RocketMQAction(value = RequestCode.QUERY_CONSUMER_OFFSET, action = Action.GET)\npublic class QueryConsumerOffsetRequestHeader extends TopicQueueRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String consumerGroup;\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNotNull\n    private Integer queueId;\n\n    private Boolean setZeroIfNotFound;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    @Override\n    public String getTopic() {\n        return topic;\n    }\n\n    @Override\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    @Override\n    public Integer getQueueId() {\n        return queueId;\n    }\n\n    @Override\n    public void setQueueId(Integer queueId) {\n        this.queueId = queueId;\n    }\n\n    public Boolean getSetZeroIfNotFound() {\n        return setZeroIfNotFound;\n    }\n\n    public void setSetZeroIfNotFound(Boolean setZeroIfNotFound) {\n        this.setZeroIfNotFound = setZeroIfNotFound;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"consumerGroup\", consumerGroup)\n            .add(\"topic\", topic)\n            .add(\"queueId\", queueId)\n            .add(\"setZeroIfNotFound\", setZeroIfNotFound)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryConsumerOffsetResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: QueryConsumerOffsetResponseHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class QueryConsumerOffsetResponseHeader implements CommandCustomHeader {\n    @CFNotNull\n    private Long offset;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public Long getOffset() {\n        return offset;\n    }\n\n    public void setOffset(Long offset) {\n        this.offset = offset;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryCorrectionOffsetHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: GetMinOffsetRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.rpc.TopicRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.QUERY_CORRECTION_OFFSET, action = Action.GET)\npublic class QueryCorrectionOffsetHeader extends TopicRequestHeader {\n    @RocketMQResource(value = ResourceType.GROUP, splitter = \",\")\n    private String filterGroups;\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String compareGroup;\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getFilterGroups() {\n        return filterGroups;\n    }\n\n    public void setFilterGroups(String filterGroups) {\n        this.filterGroups = filterGroups;\n    }\n\n    public String getCompareGroup() {\n        return compareGroup;\n    }\n\n    public void setCompareGroup(String compareGroup) {\n        this.compareGroup = compareGroup;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryMessageRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: QueryMessageRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.rpc.TopicRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.QUERY_MESSAGE, action = {Action.SUB, Action.GET})\npublic class QueryMessageRequestHeader extends TopicRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNotNull\n    private String key;\n    @CFNotNull\n    private Integer maxNum;\n    @CFNotNull\n    private Long beginTimestamp;\n    @CFNotNull\n    private Long endTimestamp;\n    private String indexType;\n    private String lastKey;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getKey() {\n        return key;\n    }\n\n    public void setKey(String key) {\n        this.key = key;\n    }\n\n    public Integer getMaxNum() {\n        return maxNum;\n    }\n\n    public void setMaxNum(Integer maxNum) {\n        this.maxNum = maxNum;\n    }\n\n    public Long getBeginTimestamp() {\n        return beginTimestamp;\n    }\n\n    public void setBeginTimestamp(Long beginTimestamp) {\n        this.beginTimestamp = beginTimestamp;\n    }\n\n    public Long getEndTimestamp() {\n        return endTimestamp;\n    }\n\n    public void setEndTimestamp(Long endTimestamp) {\n        this.endTimestamp = endTimestamp;\n    }\n\n    public String getIndexType() {\n        return indexType;\n    }\n\n    public void setIndexType(String indexType) {\n        this.indexType = indexType;\n    }\n\n    public String getLastKey() {\n        return lastKey;\n    }\n\n    public void setLastKey(String lastKey) {\n        this.lastKey = lastKey;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryMessageResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: QueryMessageResponseHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class QueryMessageResponseHeader implements CommandCustomHeader {\n    @CFNotNull\n    private Long indexLastUpdateTimestamp;\n    @CFNotNull\n    private Long indexLastUpdatePhyoffset;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public Long getIndexLastUpdateTimestamp() {\n        return indexLastUpdateTimestamp;\n    }\n\n    public void setIndexLastUpdateTimestamp(Long indexLastUpdateTimestamp) {\n        this.indexLastUpdateTimestamp = indexLastUpdateTimestamp;\n    }\n\n    public Long getIndexLastUpdatePhyoffset() {\n        return indexLastUpdatePhyoffset;\n    }\n\n    public void setIndexLastUpdatePhyoffset(Long indexLastUpdatePhyoffset) {\n        this.indexLastUpdatePhyoffset = indexLastUpdatePhyoffset;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QuerySubscriptionByConsumerRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: QueryMessageRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.rpc.TopicRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.QUERY_SUBSCRIPTION_BY_CONSUMER, action = Action.GET)\npublic class QuerySubscriptionByConsumerRequestHeader extends TopicRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String group;\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getGroup() {\n        return group;\n    }\n\n    public void setGroup(String group) {\n        this.group = group;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryTopicConsumeByWhoRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: QueryMessageRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicRequestHeader;\n\n@RocketMQAction(value = RequestCode.QUERY_TOPIC_CONSUME_BY_WHO, action = Action.GET)\npublic class QueryTopicConsumeByWhoRequestHeader extends TopicRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryTopicsByConsumerRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: QueryMessageRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.RpcRequestHeader;\n\n@RocketMQAction(value = RequestCode.QUERY_TOPICS_BY_CONSUMER, action = Action.GET)\npublic class QueryTopicsByConsumerRequestHeader extends RpcRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String group;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getGroup() {\n        return group;\n    }\n\n    public void setGroup(String group) {\n        this.group = group;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/RecallMessageRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicRequestHeader;\n\n@RocketMQAction(value = RequestCode.RECALL_MESSAGE, action = Action.PUB)\npublic class RecallMessageRequestHeader extends TopicRequestHeader {\n    @CFNullable\n    private String producerGroup;\n\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n\n    @CFNotNull\n    private String recallHandle;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getProducerGroup() {\n        return producerGroup;\n    }\n\n    public void setProducerGroup(String producerGroup) {\n        this.producerGroup = producerGroup;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getRecallHandle() {\n        return recallHandle;\n    }\n\n    public void setRecallHandle(String recallHandle) {\n        this.recallHandle = recallHandle;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n                .add(\"producerGroup\", producerGroup)\n                .add(\"topic\", topic)\n                .add(\"recallHandle\", recallHandle)\n                .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/RecallMessageResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class RecallMessageResponseHeader implements CommandCustomHeader {\n    @CFNotNull\n    private String msgId;\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getMsgId() {\n        return msgId;\n    }\n\n    public void setMsgId(String msgId) {\n        this.msgId = msgId;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/RemoveBrokerRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.REMOVE_BROKER, resource = ResourceType.CLUSTER,action = Action.UPDATE)\npublic class RemoveBrokerRequestHeader implements CommandCustomHeader {\n    @CFNotNull\n    private String brokerName;\n    @CFNotNull\n    @RocketMQResource(ResourceType.CLUSTER)\n    private String brokerClusterName;\n    @CFNotNull\n    private Long brokerId;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public String getBrokerClusterName() {\n        return brokerClusterName;\n    }\n\n    public void setBrokerClusterName(String brokerClusterName) {\n        this.brokerClusterName = brokerClusterName;\n    }\n\n    public Long getBrokerId() {\n        return brokerId;\n    }\n\n    public void setBrokerId(Long brokerId) {\n        this.brokerId = brokerId;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ReplyMessageRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.PUSH_REPLY_MESSAGE_TO_CLIENT, action = Action.SUB)\npublic class ReplyMessageRequestHeader extends TopicQueueRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String producerGroup;\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNotNull\n    private String defaultTopic;\n    @CFNotNull\n    private Integer defaultTopicQueueNums;\n    @CFNotNull\n    private Integer queueId;\n    @CFNotNull\n    private Integer sysFlag;\n    @CFNotNull\n    private Long bornTimestamp;\n    @CFNotNull\n    private Integer flag;\n    @CFNullable\n    private String properties;\n    @CFNullable\n    private Integer reconsumeTimes;\n    @CFNullable\n    private boolean unitMode = false;\n\n    @CFNotNull\n    private String bornHost;\n    @CFNotNull\n    private String storeHost;\n    @CFNotNull\n    private long storeTimestamp;\n\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getProducerGroup() {\n        return producerGroup;\n    }\n\n    public void setProducerGroup(String producerGroup) {\n        this.producerGroup = producerGroup;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getDefaultTopic() {\n        return defaultTopic;\n    }\n\n    public void setDefaultTopic(String defaultTopic) {\n        this.defaultTopic = defaultTopic;\n    }\n\n    public Integer getDefaultTopicQueueNums() {\n        return defaultTopicQueueNums;\n    }\n\n    public void setDefaultTopicQueueNums(Integer defaultTopicQueueNums) {\n        this.defaultTopicQueueNums = defaultTopicQueueNums;\n    }\n\n    public Integer getQueueId() {\n        return queueId;\n    }\n\n    public void setQueueId(Integer queueId) {\n        this.queueId = queueId;\n    }\n\n    public Integer getSysFlag() {\n        return sysFlag;\n    }\n\n    public void setSysFlag(Integer sysFlag) {\n        this.sysFlag = sysFlag;\n    }\n\n    public Long getBornTimestamp() {\n        return bornTimestamp;\n    }\n\n    public void setBornTimestamp(Long bornTimestamp) {\n        this.bornTimestamp = bornTimestamp;\n    }\n\n    public Integer getFlag() {\n        return flag;\n    }\n\n    public void setFlag(Integer flag) {\n        this.flag = flag;\n    }\n\n    public String getProperties() {\n        return properties;\n    }\n\n    public void setProperties(String properties) {\n        this.properties = properties;\n    }\n\n    public Integer getReconsumeTimes() {\n        return reconsumeTimes;\n    }\n\n    public void setReconsumeTimes(Integer reconsumeTimes) {\n        this.reconsumeTimes = reconsumeTimes;\n    }\n\n    public boolean isUnitMode() {\n        return unitMode;\n    }\n\n    public void setUnitMode(boolean unitMode) {\n        this.unitMode = unitMode;\n    }\n\n    public String getBornHost() {\n        return bornHost;\n    }\n\n    public void setBornHost(String bornHost) {\n        this.bornHost = bornHost;\n    }\n\n    public String getStoreHost() {\n        return storeHost;\n    }\n\n    public void setStoreHost(String storeHost) {\n        this.storeHost = storeHost;\n    }\n\n    public long getStoreTimestamp() {\n        return storeTimestamp;\n    }\n\n    public void setStoreTimestamp(long storeTimestamp) {\n        this.storeTimestamp = storeTimestamp;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ResetMasterFlushOffsetHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.RESET_MASTER_FLUSH_OFFSET, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class ResetMasterFlushOffsetHeader implements CommandCustomHeader {\n    @CFNotNull\n    private Long masterFlushOffset;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public Long getMasterFlushOffset() {\n        return masterFlushOffset;\n    }\n\n    public void setMasterFlushOffset(Long masterFlushOffset) {\n        this.masterFlushOffset = masterFlushOffset;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ResetOffsetRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.INVOKE_BROKER_TO_RESET_OFFSET, action = Action.UPDATE)\npublic class ResetOffsetRequestHeader extends TopicQueueRequestHeader {\n\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String group;\n\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n\n    private int queueId = -1;\n\n    private Long offset;\n\n    @CFNotNull\n    private long timestamp;\n\n    @CFNotNull\n    private boolean isForce;\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getGroup() {\n        return group;\n    }\n\n    public void setGroup(String group) {\n        this.group = group;\n    }\n\n    public long getTimestamp() {\n        return timestamp;\n    }\n\n    public void setTimestamp(long timestamp) {\n        this.timestamp = timestamp;\n    }\n\n    public boolean isForce() {\n        return isForce;\n    }\n\n    public void setForce(boolean isForce) {\n        this.isForce = isForce;\n    }\n\n    public Integer getQueueId() {\n        return queueId;\n    }\n\n    public void setQueueId(Integer queueId) {\n        this.queueId = queueId;\n    }\n\n    public Long getOffset() {\n        return offset;\n    }\n\n    public void setOffset(Long offset) {\n        this.offset = offset;\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ResumeCheckHalfMessageRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.RESUME_CHECK_HALF_MESSAGE, action = Action.UPDATE)\npublic class ResumeCheckHalfMessageRequestHeader implements CommandCustomHeader {\n\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNullable\n    private String msgId;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getMsgId() {\n        return msgId;\n    }\n\n    public void setMsgId(String msgId) {\n        this.msgId = msgId;\n    }\n\n    @Override\n    public String toString() {\n        return \"ResumeCheckHalfMessageRequestHeader [msgId=\" + msgId + \"]\";\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SearchOffsetRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: SearchOffsetRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;\n\n@RocketMQAction(value = RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, action = Action.GET)\npublic class SearchOffsetRequestHeader extends TopicQueueRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    private String liteTopic;\n    @CFNotNull\n    private Integer queueId;\n    @CFNotNull\n    private Long timestamp;\n\n    private BoundaryType boundaryType;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    @Override\n    public String getTopic() {\n        return topic;\n    }\n\n    @Override\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getLiteTopic() {\n        return liteTopic;\n    }\n\n    public void setLiteTopic(String liteTopic) {\n        this.liteTopic = liteTopic;\n    }\n\n    @Override\n    public Integer getQueueId() {\n        return queueId;\n    }\n\n    @Override\n    public void setQueueId(Integer queueId) {\n        this.queueId = queueId;\n    }\n\n    public Long getTimestamp() {\n        return timestamp;\n    }\n\n    public void setTimestamp(Long timestamp) {\n        this.timestamp = timestamp;\n    }\n\n    public BoundaryType getBoundaryType() {\n        // default return LOWER\n        return boundaryType == null ? BoundaryType.LOWER : boundaryType;\n    }\n\n    public void setBoundaryType(BoundaryType boundaryType) {\n        this.boundaryType = boundaryType;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"topic\", topic)\n            .add(\"liteTopic\", liteTopic)\n            .add(\"queueId\", queueId)\n            .add(\"timestamp\", timestamp)\n            .add(\"boundaryType\", boundaryType.getName())\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SearchOffsetResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: SearchOffsetResponseHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class SearchOffsetResponseHeader implements CommandCustomHeader {\n    @CFNotNull\n    private Long offset;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public Long getOffset() {\n        return offset;\n    }\n\n    public void setOffset(Long offset) {\n        this.offset = offset;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: SendMessageRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;\n\n@RocketMQAction(value = RequestCode.SEND_MESSAGE, action = Action.PUB)\npublic class SendMessageRequestHeader extends TopicQueueRequestHeader {\n    @CFNotNull\n    private String producerGroup;\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNotNull\n    private String defaultTopic;\n    @CFNotNull\n    private Integer defaultTopicQueueNums;\n    @CFNotNull\n    private Integer queueId;\n    @CFNotNull\n    private Integer sysFlag;\n    @CFNotNull\n    private Long bornTimestamp;\n    @CFNotNull\n    private Integer flag;\n    @CFNullable\n    private String properties;\n    @CFNullable\n    private Integer reconsumeTimes;\n    @CFNullable\n    private Boolean unitMode;\n    @CFNullable\n    private Boolean batch;\n    private Integer maxReconsumeTimes;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getProducerGroup() {\n        return producerGroup;\n    }\n\n    public void setProducerGroup(String producerGroup) {\n        this.producerGroup = producerGroup;\n    }\n\n    @Override\n    public String getTopic() {\n        return topic;\n    }\n\n    @Override\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getDefaultTopic() {\n        return defaultTopic;\n    }\n\n    public void setDefaultTopic(String defaultTopic) {\n        this.defaultTopic = defaultTopic;\n    }\n\n    public Integer getDefaultTopicQueueNums() {\n        return defaultTopicQueueNums;\n    }\n\n    public void setDefaultTopicQueueNums(Integer defaultTopicQueueNums) {\n        this.defaultTopicQueueNums = defaultTopicQueueNums;\n    }\n\n    @Override\n    public Integer getQueueId() {\n        return queueId;\n    }\n\n    @Override\n    public void setQueueId(Integer queueId) {\n        this.queueId = queueId;\n    }\n\n    public Integer getSysFlag() {\n        return sysFlag;\n    }\n\n    public void setSysFlag(Integer sysFlag) {\n        this.sysFlag = sysFlag;\n    }\n\n    public Long getBornTimestamp() {\n        return bornTimestamp;\n    }\n\n    public void setBornTimestamp(Long bornTimestamp) {\n        this.bornTimestamp = bornTimestamp;\n    }\n\n    public Integer getFlag() {\n        return flag;\n    }\n\n    public void setFlag(Integer flag) {\n        this.flag = flag;\n    }\n\n    public String getProperties() {\n        return properties;\n    }\n\n    public void setProperties(String properties) {\n        this.properties = properties;\n    }\n\n    public Integer getReconsumeTimes() {\n        if (null == reconsumeTimes) {\n            return 0;\n        }\n        return reconsumeTimes;\n    }\n\n    public void setReconsumeTimes(Integer reconsumeTimes) {\n        this.reconsumeTimes = reconsumeTimes;\n    }\n\n    public boolean isUnitMode() {\n        if (null == unitMode) {\n            return false;\n        }\n        return unitMode;\n    }\n\n    public void setUnitMode(Boolean isUnitMode) {\n        this.unitMode = isUnitMode;\n    }\n\n    public Integer getMaxReconsumeTimes() {\n        return maxReconsumeTimes;\n    }\n\n    public void setMaxReconsumeTimes(final Integer maxReconsumeTimes) {\n        this.maxReconsumeTimes = maxReconsumeTimes;\n    }\n\n    public boolean isBatch() {\n        if (null == batch) {\n            return false;\n        }\n        return batch;\n    }\n\n    public void setBatch(Boolean batch) {\n        this.batch = batch;\n    }\n\n    public static SendMessageRequestHeader parseRequestHeader(RemotingCommand request) throws RemotingCommandException {\n        SendMessageRequestHeaderV2 requestHeaderV2 = null;\n        SendMessageRequestHeader requestHeader = null;\n        switch (request.getCode()) {\n            case RequestCode.SEND_BATCH_MESSAGE:\n            case RequestCode.SEND_MESSAGE_V2:\n                requestHeaderV2 = request.decodeCommandCustomHeader(SendMessageRequestHeaderV2.class);\n            case RequestCode.SEND_MESSAGE:\n                if (null == requestHeaderV2) {\n                    requestHeader = request.decodeCommandCustomHeader(SendMessageRequestHeader.class);\n                } else {\n                    requestHeader = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV1(requestHeaderV2);\n                }\n            default:\n                break;\n        }\n        return requestHeader;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"producerGroup\", producerGroup)\n            .add(\"topic\", topic)\n            .add(\"defaultTopic\", defaultTopic)\n            .add(\"defaultTopicQueueNums\", defaultTopicQueueNums)\n            .add(\"queueId\", queueId)\n            .add(\"sysFlag\", sysFlag)\n            .add(\"bornTimestamp\", bornTimestamp)\n            .add(\"flag\", flag)\n            .add(\"properties\", properties)\n            .add(\"reconsumeTimes\", reconsumeTimes)\n            .add(\"unitMode\", unitMode)\n            .add(\"batch\", batch)\n            .add(\"maxReconsumeTimes\", maxReconsumeTimes)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageRequestHeaderV2.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport io.netty.buffer.ByteBuf;\nimport java.util.HashMap;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.FastCodesHeader;\nimport org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n/**\n * Use short variable name to speed up FastJson deserialization process.\n */\n@RocketMQAction(value = RequestCode.SEND_MESSAGE_V2, action = Action.PUB)\npublic class SendMessageRequestHeaderV2 extends TopicQueueRequestHeader implements CommandCustomHeader, FastCodesHeader {\n    @CFNotNull\n    private String a; // producerGroup;\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String b; // topic;\n    @CFNotNull\n    private String c; // defaultTopic;\n    @CFNotNull\n    private Integer d; // defaultTopicQueueNums;\n    @CFNotNull\n    private Integer e; // queueId;\n    @CFNotNull\n    private Integer f; // sysFlag;\n    @CFNotNull\n    private Long g; // bornTimestamp;\n    @CFNotNull\n    private Integer h; // flag;\n    @CFNullable\n    private String i; // properties;\n    @CFNullable\n    private Integer j; // reconsumeTimes;\n    @CFNullable\n    private Boolean k; // unitMode;\n\n    private Integer l; // consumeRetryTimes\n\n    @CFNullable\n    private Boolean m; //batch\n    @CFNullable\n    private String n; // brokerName\n\n    public static SendMessageRequestHeader createSendMessageRequestHeaderV1(final SendMessageRequestHeaderV2 v2) {\n        SendMessageRequestHeader v1 = new SendMessageRequestHeader();\n        v1.setProducerGroup(v2.a);\n        v1.setTopic(v2.b);\n        v1.setDefaultTopic(v2.c);\n        v1.setDefaultTopicQueueNums(v2.d);\n        v1.setQueueId(v2.e);\n        v1.setSysFlag(v2.f);\n        v1.setBornTimestamp(v2.g);\n        v1.setFlag(v2.h);\n        v1.setProperties(v2.i);\n        v1.setReconsumeTimes(v2.j);\n        v1.setUnitMode(v2.k);\n        v1.setMaxReconsumeTimes(v2.l);\n        v1.setBatch(v2.m);\n        v1.setBrokerName(v2.n);\n        return v1;\n    }\n\n    public static SendMessageRequestHeaderV2 createSendMessageRequestHeaderV2(final SendMessageRequestHeader v1) {\n        SendMessageRequestHeaderV2 v2 = new SendMessageRequestHeaderV2();\n        v2.a = v1.getProducerGroup();\n        v2.b = v1.getTopic();\n        v2.c = v1.getDefaultTopic();\n        v2.d = v1.getDefaultTopicQueueNums();\n        v2.e = v1.getQueueId();\n        v2.f = v1.getSysFlag();\n        v2.g = v1.getBornTimestamp();\n        v2.h = v1.getFlag();\n        v2.i = v1.getProperties();\n        v2.j = v1.getReconsumeTimes();\n        v2.k = v1.isUnitMode();\n        v2.l = v1.getMaxReconsumeTimes();\n        v2.m = v1.isBatch();\n        v2.n = v1.getBrokerName();\n        return v2;\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    @Override\n    public void encode(ByteBuf out) {\n        writeIfNotNull(out, \"a\", a);\n        writeIfNotNull(out, \"b\", b);\n        writeIfNotNull(out, \"c\", c);\n        writeIfNotNull(out, \"d\", d);\n        writeIfNotNull(out, \"e\", e);\n        writeIfNotNull(out, \"f\", f);\n        writeIfNotNull(out, \"g\", g);\n        writeIfNotNull(out, \"h\", h);\n        writeIfNotNull(out, \"i\", i);\n        writeIfNotNull(out, \"j\", j);\n        writeIfNotNull(out, \"k\", k);\n        writeIfNotNull(out, \"l\", l);\n        writeIfNotNull(out, \"m\", m);\n        writeIfNotNull(out, \"n\", n);\n    }\n\n    @Override\n    public void decode(HashMap<String, String> fields) throws RemotingCommandException {\n\n        String str = getAndCheckNotNull(fields, \"a\");\n        if (str != null) {\n            a = str;\n        }\n\n        str = getAndCheckNotNull(fields, \"b\");\n        if (str != null) {\n            b = str;\n        }\n\n        str = getAndCheckNotNull(fields, \"c\");\n        if (str != null) {\n            c = str;\n        }\n\n        str = getAndCheckNotNull(fields, \"d\");\n        if (str != null) {\n            d = Integer.parseInt(str);\n        }\n\n        str = getAndCheckNotNull(fields, \"e\");\n        if (str != null) {\n            e = Integer.parseInt(str);\n        }\n\n        str = getAndCheckNotNull(fields, \"f\");\n        if (str != null) {\n            f = Integer.parseInt(str);\n        }\n\n        str = getAndCheckNotNull(fields, \"g\");\n        if (str != null) {\n            g = Long.parseLong(str);\n        }\n\n        str = getAndCheckNotNull(fields, \"h\");\n        if (str != null) {\n            h = Integer.parseInt(str);\n        }\n\n        str = fields.get(\"i\");\n        if (str != null) {\n            i = str;\n        }\n\n        str = fields.get(\"j\");\n        if (str != null) {\n            j = Integer.parseInt(str);\n        }\n\n        str = fields.get(\"k\");\n        if (str != null) {\n            k = Boolean.parseBoolean(str);\n        }\n\n        str = fields.get(\"l\");\n        if (str != null) {\n            l = Integer.parseInt(str);\n        }\n\n        str = fields.get(\"m\");\n        if (str != null) {\n            m = Boolean.parseBoolean(str);\n        }\n\n        str = fields.get(\"n\");\n        if (str != null) {\n            n = str;\n        }\n    }\n\n    public String getA() {\n        return a;\n    }\n\n    public void setA(String a) {\n        this.a = a;\n    }\n\n    public String getB() {\n        return b;\n    }\n\n    public void setB(String b) {\n        this.b = b;\n    }\n\n    public String getC() {\n        return c;\n    }\n\n    public void setC(String c) {\n        this.c = c;\n    }\n\n    public Integer getD() {\n        return d;\n    }\n\n    public void setD(Integer d) {\n        this.d = d;\n    }\n\n    public Integer getE() {\n        return e;\n    }\n\n    public void setE(Integer e) {\n        this.e = e;\n    }\n\n    public Integer getF() {\n        return f;\n    }\n\n    public void setF(Integer f) {\n        this.f = f;\n    }\n\n    public Long getG() {\n        return g;\n    }\n\n    public void setG(Long g) {\n        this.g = g;\n    }\n\n    public Integer getH() {\n        return h;\n    }\n\n    public void setH(Integer h) {\n        this.h = h;\n    }\n\n    public String getI() {\n        return i;\n    }\n\n    public void setI(String i) {\n        this.i = i;\n    }\n\n    public Integer getJ() {\n        return j;\n    }\n\n    public void setJ(Integer j) {\n        this.j = j;\n    }\n\n    public Boolean isK() {\n        return k;\n    }\n\n    public void setK(Boolean k) {\n        this.k = k;\n    }\n\n    public Integer getL() {\n        return l;\n    }\n\n    public void setL(final Integer l) {\n        this.l = l;\n    }\n\n    public Boolean isM() {\n        return m;\n    }\n\n    public void setM(Boolean m) {\n        this.m = m;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"a\", a)\n            .add(\"b\", b)\n            .add(\"c\", c)\n            .add(\"d\", d)\n            .add(\"e\", e)\n            .add(\"f\", f)\n            .add(\"g\", g)\n            .add(\"h\", h)\n            .add(\"i\", i)\n            .add(\"j\", j)\n            .add(\"k\", k)\n            .add(\"l\", l)\n            .add(\"m\", m)\n            .add(\"n\", n)\n            .toString();\n    }\n\n    @Override\n    public Integer getQueueId() {\n        return e;\n    }\n\n    @Override\n    public void setQueueId(Integer queueId) {\n        this.e = queueId;\n    }\n\n    @Override\n    public String getTopic() {\n        return b;\n    }\n\n    @Override\n    public void setTopic(String topic) {\n        this.b = topic;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: SendMessageResponseHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport io.netty.buffer.ByteBuf;\nimport java.util.HashMap;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.FastCodesHeader;\n\npublic class SendMessageResponseHeader implements CommandCustomHeader, FastCodesHeader {\n    @CFNotNull\n    private String msgId;\n    @CFNotNull\n    private Integer queueId;\n    @CFNotNull\n    private Long queueOffset;\n    private String transactionId;\n    private String batchUniqId;\n    private String recallHandle;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    @Override\n    public void encode(ByteBuf out) {\n        writeIfNotNull(out, \"msgId\", msgId);\n        writeIfNotNull(out, \"queueId\", queueId);\n        writeIfNotNull(out, \"queueOffset\", queueOffset);\n        writeIfNotNull(out, \"transactionId\", transactionId);\n        writeIfNotNull(out, \"batchUniqId\", batchUniqId);\n        writeIfNotNull(out, \"recallHandle\", recallHandle);\n    }\n\n    @Override\n    public void decode(HashMap<String, String> fields) throws RemotingCommandException {\n        String str = getAndCheckNotNull(fields, \"msgId\");\n        if (str != null) {\n            this.msgId = str;\n        }\n\n        str = getAndCheckNotNull(fields, \"queueId\");\n        if (str != null) {\n            this.queueId = Integer.parseInt(str);\n        }\n\n        str = getAndCheckNotNull(fields, \"queueOffset\");\n        if (str != null) {\n            this.queueOffset = Long.parseLong(str);\n        }\n\n        str = fields.get(\"transactionId\");\n        if (str != null) {\n            this.transactionId = str;\n        }\n\n        str = fields.get(\"batchUniqId\");\n        if (str != null) {\n            this.batchUniqId = str;\n        }\n\n        str = fields.get(\"recallHandle\");\n        if (str != null) {\n            this.recallHandle = str;\n        }\n    }\n\n    public String getMsgId() {\n        return msgId;\n    }\n\n    public void setMsgId(String msgId) {\n        this.msgId = msgId;\n    }\n\n    public Integer getQueueId() {\n        return queueId;\n    }\n\n    public void setQueueId(Integer queueId) {\n        this.queueId = queueId;\n    }\n\n    public Long getQueueOffset() {\n        return queueOffset;\n    }\n\n    public void setQueueOffset(Long queueOffset) {\n        this.queueOffset = queueOffset;\n    }\n\n    public String getTransactionId() {\n        return transactionId;\n    }\n\n    public void setTransactionId(String transactionId) {\n        this.transactionId = transactionId;\n    }\n\n    public String getBatchUniqId() {\n        return batchUniqId;\n    }\n\n    public void setBatchUniqId(String batchUniqId) {\n        this.batchUniqId = batchUniqId;\n    }\n\n    public String getRecallHandle() {\n        return recallHandle;\n    }\n\n    public void setRecallHandle(String recallHandle) {\n        this.recallHandle = recallHandle;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/StatisticsMessagesRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;\n\npublic class StatisticsMessagesRequestHeader extends TopicQueueRequestHeader {\n    @CFNotNull\n    private String consumerGroup;\n    @CFNotNull\n    private String topic;\n    @CFNotNull\n    private int queueId;\n\n    private long fromTime;\n    private long toTime;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public Integer getQueueId() {\n        if (queueId < 0) {\n            return -1;\n        }\n        return queueId;\n    }\n\n    public void setQueueId(Integer queueId) {\n        this.queueId = queueId;\n    }\n\n    public long getFromTime() {\n        return fromTime;\n    }\n\n    public void setFromTime(long fromTime) {\n        this.fromTime = fromTime;\n    }\n\n    public long getToTime() {\n        return toTime;\n    }\n\n    public void setToTime(long toTime) {\n        this.toTime = toTime;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/TriggerLiteDispatchRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class TriggerLiteDispatchRequestHeader implements CommandCustomHeader {\n\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String group;\n\n    private String clientId;\n\n    public String getGroup() {\n        return group;\n    }\n\n    public void setGroup(String group) {\n        this.group = group;\n    }\n\n    public String getClientId() {\n        return clientId;\n    }\n\n    public void setClientId(String clientId) {\n        this.clientId = clientId;\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UnlockBatchMqRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.RpcRequestHeader;\n\n@RocketMQAction(value = RequestCode.UNLOCK_BATCH_MQ, action = Action.SUB)\npublic class UnlockBatchMqRequestHeader extends RpcRequestHeader {\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UnregisterClientRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.rpc.RpcRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.UNREGISTER_CLIENT, action = {Action.PUB, Action.SUB})\npublic class UnregisterClientRequestHeader extends RpcRequestHeader {\n    @CFNotNull\n    private String clientID;\n\n    @CFNullable\n    private String producerGroup;\n    @CFNullable\n    @RocketMQResource(ResourceType.GROUP)\n    private String consumerGroup;\n\n    public String getClientID() {\n        return clientID;\n    }\n\n    public void setClientID(String clientID) {\n        this.clientID = clientID;\n    }\n\n    public String getProducerGroup() {\n        return producerGroup;\n    }\n\n    public void setProducerGroup(String producerGroup) {\n        this.producerGroup = producerGroup;\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UnregisterClientResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class UnregisterClientResponseHeader implements CommandCustomHeader {\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UpdateAclRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.AUTH_UPDATE_ACL, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class UpdateAclRequestHeader implements CommandCustomHeader {\n\n    private String subject;\n\n    public UpdateAclRequestHeader() {\n    }\n\n    public UpdateAclRequestHeader(String subject) {\n        this.subject = subject;\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getSubject() {\n        return subject;\n    }\n\n    public void setSubject(String subject) {\n        this.subject = subject;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UpdateConsumerOffsetRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: UpdateConsumerOffsetRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport com.google.common.base.MoreObjects;\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;\n\n@RocketMQAction(value = RequestCode.UPDATE_CONSUMER_OFFSET, action = Action.SUB)\npublic class UpdateConsumerOffsetRequestHeader extends TopicQueueRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String consumerGroup;\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNotNull\n    private Integer queueId;\n    @CFNotNull\n    private Long commitOffset;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    @Override\n    public String getTopic() {\n        return topic;\n    }\n\n    @Override\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    @Override\n    public Integer getQueueId() {\n        return queueId;\n    }\n\n    @Override\n    public void setQueueId(Integer queueId) {\n        this.queueId = queueId;\n    }\n\n    public Long getCommitOffset() {\n        return commitOffset;\n    }\n\n    public void setCommitOffset(Long commitOffset) {\n        this.commitOffset = commitOffset;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"consumerGroup\", consumerGroup)\n            .add(\"topic\", topic)\n            .add(\"queueId\", queueId)\n            .add(\"commitOffset\", commitOffset)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UpdateConsumerOffsetResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: UpdateConsumerOffsetResponseHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class UpdateConsumerOffsetResponseHeader implements CommandCustomHeader {\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UpdateGroupForbiddenRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: CreateTopicRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.rpc.TopicRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.UPDATE_AND_GET_GROUP_FORBIDDEN, action = Action.UPDATE)\npublic class UpdateGroupForbiddenRequestHeader extends TopicRequestHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.GROUP)\n    private String  group;\n    @CFNotNull\n    @RocketMQResource(ResourceType.TOPIC)\n    private String  topic;\n\n    private Boolean readable;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getGroup() {\n        return group;\n    }\n\n    public void setGroup(String group) {\n        this.group = group;\n    }\n\n    public Boolean getReadable() {\n        return readable;\n    }\n\n    public void setReadable(Boolean readable) {\n        this.readable = readable;\n    }\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UpdateUserRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.AUTH_UPDATE_USER, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class UpdateUserRequestHeader implements CommandCustomHeader {\n\n    private String username;\n\n    public UpdateUserRequestHeader() {\n    }\n\n    public UpdateUserRequestHeader(String username) {\n        this.username = username;\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getUsername() {\n        return username;\n    }\n\n    public void setUsername(String username) {\n        this.username = username;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ViewBrokerStatsDataRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.VIEW_BROKER_STATS_DATA, resource = ResourceType.CLUSTER, action = Action.GET)\npublic class ViewBrokerStatsDataRequestHeader implements CommandCustomHeader {\n    @CFNotNull\n    private String statsName;\n    @CFNotNull\n    private String statsKey;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getStatsName() {\n        return statsName;\n    }\n\n    public void setStatsName(String statsName) {\n        this.statsName = statsName;\n    }\n\n    public String getStatsKey() {\n        return statsKey;\n    }\n\n    public void setStatsKey(String statsKey) {\n        this.statsKey = statsKey;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ViewMessageRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: ViewMessageRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.VIEW_MESSAGE_BY_ID, action = Action.GET)\npublic class ViewMessageRequestHeader implements CommandCustomHeader {\n    @RocketMQResource(ResourceType.TOPIC)\n    private String topic;\n    @CFNotNull\n    private Long offset;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public Long getOffset() {\n        return offset;\n    }\n\n    public void setOffset(Long offset) {\n        this.offset = offset;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ViewMessageResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: ViewMessageResponseHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class ViewMessageResponseHeader implements CommandCustomHeader {\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/AlterSyncStateSetRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header.controller;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.CONTROLLER_ALTER_SYNC_STATE_SET, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class AlterSyncStateSetRequestHeader implements CommandCustomHeader {\n    private String brokerName;\n    private Long masterBrokerId;\n    private Integer masterEpoch;\n    private long invokeTime = System.currentTimeMillis();\n\n    public AlterSyncStateSetRequestHeader() {\n    }\n\n    public AlterSyncStateSetRequestHeader(String brokerName, Long masterBrokerId, Integer masterEpoch) {\n        this.brokerName = brokerName;\n        this.masterBrokerId = masterBrokerId;\n        this.masterEpoch = masterEpoch;\n    }\n\n    public long getInvokeTime() {\n        return invokeTime;\n    }\n\n    public void setInvokeTime(long invokeTime) {\n        this.invokeTime = invokeTime;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public Long getMasterBrokerId() {\n        return masterBrokerId;\n    }\n\n    public void setMasterBrokerId(Long masterBrokerId) {\n        this.masterBrokerId = masterBrokerId;\n    }\n\n    public Integer getMasterEpoch() {\n        return masterEpoch;\n    }\n\n    public void setMasterEpoch(Integer masterEpoch) {\n        this.masterEpoch = masterEpoch;\n    }\n\n    @Override\n    public String toString() {\n        return \"AlterSyncStateSetRequestHeader{\" +\n                \"brokerName='\" + brokerName + '\\'' +\n                \", masterBrokerId=\" + masterBrokerId +\n                \", masterEpoch=\" + masterEpoch +\n                '}';\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/AlterSyncStateSetResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header.controller;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class AlterSyncStateSetResponseHeader implements CommandCustomHeader {\n    private int newSyncStateSetEpoch;\n\n    public AlterSyncStateSetResponseHeader() {\n    }\n\n    public int getNewSyncStateSetEpoch() {\n        return newSyncStateSetEpoch;\n    }\n\n    public void setNewSyncStateSetEpoch(int newSyncStateSetEpoch) {\n        this.newSyncStateSetEpoch = newSyncStateSetEpoch;\n    }\n\n    @Override\n    public String toString() {\n        return \"AlterSyncStateSetResponseHeader{\" +\n            \"newSyncStateSetEpoch=\" + newSyncStateSetEpoch +\n            '}';\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/ElectMasterRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header.controller;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.CONTROLLER_ELECT_MASTER, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class ElectMasterRequestHeader implements CommandCustomHeader {\n\n    @CFNotNull\n    @RocketMQResource(ResourceType.CLUSTER)\n    private String clusterName = \"\";\n\n    @CFNotNull\n    private String brokerName = \"\";\n\n    /**\n     * brokerId\n     * for brokerTrigger electMaster: this brokerId will be elected as a master when it is the first time to elect\n     * in this broker-set\n     * for adminTrigger electMaster: this brokerId is also named assignedBrokerId, which means we must prefer to elect\n     * it as a new master when this broker is valid.\n     */\n    @CFNotNull\n    private Long brokerId = -1L;\n\n    @CFNotNull\n    private Boolean designateElect = false;\n\n    private Long invokeTime = System.currentTimeMillis();\n\n    public ElectMasterRequestHeader() {\n    }\n\n    public ElectMasterRequestHeader(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public ElectMasterRequestHeader(String clusterName, String brokerName, Long brokerId) {\n        this.clusterName = clusterName;\n        this.brokerName = brokerName;\n        this.brokerId = brokerId;\n    }\n\n    public ElectMasterRequestHeader(String clusterName, String brokerName, Long brokerId, boolean designateElect) {\n        this.clusterName = clusterName;\n        this.brokerName = brokerName;\n        this.brokerId = brokerId;\n        this.designateElect = designateElect;\n    }\n\n    public static ElectMasterRequestHeader ofBrokerTrigger(String clusterName, String brokerName,\n        Long brokerId) {\n        return new ElectMasterRequestHeader(clusterName, brokerName, brokerId);\n    }\n\n    public static ElectMasterRequestHeader ofControllerTrigger(String brokerName) {\n        return new ElectMasterRequestHeader(brokerName);\n    }\n\n    public static ElectMasterRequestHeader ofAdminTrigger(String clusterName, String brokerName, Long brokerId) {\n        return new ElectMasterRequestHeader(clusterName, brokerName, brokerId, true);\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public Long getBrokerId() {\n        return brokerId;\n    }\n\n    public void setBrokerId(Long brokerId) {\n        this.brokerId = brokerId;\n    }\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    public void setClusterName(String clusterName) {\n        this.clusterName = clusterName;\n    }\n\n    public boolean getDesignateElect() {\n        return this.designateElect;\n    }\n\n    public Long getInvokeTime() {\n        return invokeTime;\n    }\n\n    public void setInvokeTime(Long invokeTime) {\n        this.invokeTime = invokeTime;\n    }\n\n    @Override\n    public String toString() {\n        return \"ElectMasterRequestHeader{\" +\n                \"clusterName='\" + clusterName + '\\'' +\n                \", brokerName='\" + brokerName + '\\'' +\n                \", brokerId=\" + brokerId +\n                \", designateElect=\" + designateElect +\n                '}';\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/ElectMasterResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header.controller;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\n\npublic class ElectMasterResponseHeader implements CommandCustomHeader {\n\n    private Long masterBrokerId;\n    private String masterAddress;\n    private Integer masterEpoch;\n    private Integer syncStateSetEpoch;\n\n    public ElectMasterResponseHeader() {\n    }\n\n    public ElectMasterResponseHeader(Long masterBrokerId, String masterAddress, Integer masterEpoch, Integer syncStateSetEpoch) {\n        this.masterBrokerId = masterBrokerId;\n        this.masterAddress = masterAddress;\n        this.masterEpoch = masterEpoch;\n        this.syncStateSetEpoch = syncStateSetEpoch;\n    }\n\n    public String getMasterAddress() {\n        return masterAddress;\n    }\n\n    public void setMasterAddress(String masterAddress) {\n        this.masterAddress = masterAddress;\n    }\n\n    public Integer getMasterEpoch() {\n        return masterEpoch;\n    }\n\n    public void setMasterEpoch(Integer masterEpoch) {\n        this.masterEpoch = masterEpoch;\n    }\n\n    public Integer getSyncStateSetEpoch() {\n        return syncStateSetEpoch;\n    }\n\n    public void setSyncStateSetEpoch(Integer syncStateSetEpoch) {\n        this.syncStateSetEpoch = syncStateSetEpoch;\n    }\n\n    public void setMasterBrokerId(Long masterBrokerId) {\n        this.masterBrokerId = masterBrokerId;\n    }\n\n    public Long getMasterBrokerId() {\n        return masterBrokerId;\n    }\n\n    @Override\n    public String toString() {\n        return \"ElectMasterResponseHeader{\" +\n                \"masterBrokerId=\" + masterBrokerId +\n                \", masterAddress='\" + masterAddress + '\\'' +\n                \", masterEpoch=\" + masterEpoch +\n                \", syncStateSetEpoch=\" + syncStateSetEpoch +\n                '}';\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/GetMetaDataResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header.controller;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class GetMetaDataResponseHeader implements CommandCustomHeader {\n    private String group;\n    private String controllerLeaderId;\n    private String controllerLeaderAddress;\n    private boolean isLeader;\n    private String peers;\n\n    public GetMetaDataResponseHeader() {\n    }\n\n    public GetMetaDataResponseHeader(String group, String controllerLeaderId, String controllerLeaderAddress, boolean isLeader, String peers) {\n        this.group = group;\n        this.controllerLeaderId = controllerLeaderId;\n        this.controllerLeaderAddress = controllerLeaderAddress;\n        this.isLeader = isLeader;\n        this.peers = peers;\n    }\n\n    public String getGroup() {\n        return group;\n    }\n\n    public void setGroup(String group) {\n        this.group = group;\n    }\n\n    public String getControllerLeaderId() {\n        return controllerLeaderId;\n    }\n\n    public void setControllerLeaderId(String controllerLeaderId) {\n        this.controllerLeaderId = controllerLeaderId;\n    }\n\n    public String getControllerLeaderAddress() {\n        return controllerLeaderAddress;\n    }\n\n    public void setControllerLeaderAddress(String controllerLeaderAddress) {\n        this.controllerLeaderAddress = controllerLeaderAddress;\n    }\n\n    public boolean isLeader() {\n        return isLeader;\n    }\n\n    public void setIsLeader(boolean leader) {\n        isLeader = leader;\n    }\n\n    public String getPeers() {\n        return peers;\n    }\n\n    public void setPeers(String peers) {\n        this.peers = peers;\n    }\n\n    @Override\n    public String toString() {\n        return \"GetMetaDataResponseHeader{\" +\n            \"group='\" + group + '\\'' +\n            \", controllerLeaderId='\" + controllerLeaderId + '\\'' +\n            \", controllerLeaderAddress='\" + controllerLeaderAddress + '\\'' +\n            \", isLeader=\" + isLeader +\n            \", peers='\" + peers + '\\'' +\n            '}';\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/GetReplicaInfoRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header.controller;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.CONTROLLER_GET_REPLICA_INFO, resource = ResourceType.CLUSTER, action = Action.GET)\npublic class GetReplicaInfoRequestHeader implements CommandCustomHeader {\n    private String brokerName;\n\n    public GetReplicaInfoRequestHeader() {\n    }\n\n    public GetReplicaInfoRequestHeader(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    @Override\n    public String toString() {\n        return \"GetReplicaInfoRequestHeader{\" +\n                \"brokerName='\" + brokerName + '\\'' +\n                '}';\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/GetReplicaInfoResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header.controller;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class GetReplicaInfoResponseHeader implements CommandCustomHeader {\n\n    private Long masterBrokerId;\n    private String masterAddress;\n    private Integer masterEpoch;\n\n    public GetReplicaInfoResponseHeader() {\n    }\n\n    public String getMasterAddress() {\n        return masterAddress;\n    }\n\n    public void setMasterAddress(String masterAddress) {\n        this.masterAddress = masterAddress;\n    }\n\n    public Integer getMasterEpoch() {\n        return masterEpoch;\n    }\n\n    public void setMasterEpoch(Integer masterEpoch) {\n        this.masterEpoch = masterEpoch;\n    }\n\n    public Long getMasterBrokerId() {\n        return masterBrokerId;\n    }\n\n    public void setMasterBrokerId(Long masterBrokerId) {\n        this.masterBrokerId = masterBrokerId;\n    }\n\n    @Override\n    public String toString() {\n        return \"GetReplicaInfoResponseHeader{\" +\n                \"masterBrokerId=\" + masterBrokerId +\n                \", masterAddress='\" + masterAddress + '\\'' +\n                \", masterEpoch=\" + masterEpoch +\n                '}';\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/admin/CleanControllerBrokerDataRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header.controller.admin;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.CLEAN_BROKER_DATA, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class CleanControllerBrokerDataRequestHeader implements CommandCustomHeader {\n\n    @CFNullable\n    @RocketMQResource(ResourceType.CLUSTER)\n    private String clusterName;\n\n    @CFNotNull\n    private String brokerName;\n\n    @CFNullable\n    private String brokerControllerIdsToClean;\n\n    private boolean isCleanLivingBroker = false;\n    private long invokeTime = System.currentTimeMillis();\n\n    public CleanControllerBrokerDataRequestHeader() {\n    }\n\n    public CleanControllerBrokerDataRequestHeader(String clusterName, String brokerName, String brokerIdSetToClean,\n        boolean isCleanLivingBroker) {\n        this.clusterName = clusterName;\n        this.brokerName = brokerName;\n        this.brokerControllerIdsToClean = brokerIdSetToClean;\n        this.isCleanLivingBroker = isCleanLivingBroker;\n    }\n\n    public CleanControllerBrokerDataRequestHeader(String clusterName, String brokerName, String brokerIdSetToClean) {\n        this(clusterName, brokerName, brokerIdSetToClean, false);\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public long getInvokeTime() {\n        return invokeTime;\n    }\n\n    public void setInvokeTime(long invokeTime) {\n        this.invokeTime = invokeTime;\n    }\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    public void setClusterName(String clusterName) {\n        this.clusterName = clusterName;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public String getBrokerControllerIdsToClean() {\n        return brokerControllerIdsToClean;\n    }\n\n    public void setBrokerControllerIdsToClean(String brokerIdSetToClean) {\n        this.brokerControllerIdsToClean = brokerIdSetToClean;\n    }\n\n    public boolean isCleanLivingBroker() {\n        return isCleanLivingBroker;\n    }\n\n    public void setCleanLivingBroker(boolean cleanLivingBroker) {\n        isCleanLivingBroker = cleanLivingBroker;\n    }\n\n    @Override\n    public String toString() {\n        return \"CleanControllerBrokerDataRequestHeader{\" +\n                \"clusterName='\" + clusterName + '\\'' +\n                \", brokerName='\" + brokerName + '\\'' +\n                \", brokerIdSetToClean='\" + brokerControllerIdsToClean + '\\'' +\n                \", isCleanLivingBroker=\" + isCleanLivingBroker +\n                '}';\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/register/ApplyBrokerIdRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header.controller.register;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.CONTROLLER_APPLY_BROKER_ID, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class ApplyBrokerIdRequestHeader implements CommandCustomHeader {\n\n    @RocketMQResource(ResourceType.CLUSTER)\n    private String clusterName;\n\n    private String brokerName;\n\n    private Long appliedBrokerId;\n\n    private String registerCheckCode;\n\n    public ApplyBrokerIdRequestHeader() {\n\n    }\n\n    public ApplyBrokerIdRequestHeader(String clusterName, String brokerName, Long appliedBrokerId, String registerCheckCode) {\n        this.clusterName = clusterName;\n        this.brokerName = brokerName;\n        this.appliedBrokerId = appliedBrokerId;\n        this.registerCheckCode = registerCheckCode;\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public Long getAppliedBrokerId() {\n        return appliedBrokerId;\n    }\n\n    public String getRegisterCheckCode() {\n        return registerCheckCode;\n    }\n\n    public void setClusterName(String clusterName) {\n        this.clusterName = clusterName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public void setAppliedBrokerId(Long appliedBrokerId) {\n        this.appliedBrokerId = appliedBrokerId;\n    }\n\n    public void setRegisterCheckCode(String registerCheckCode) {\n        this.registerCheckCode = registerCheckCode;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/register/ApplyBrokerIdResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header.controller.register;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class ApplyBrokerIdResponseHeader implements CommandCustomHeader {\n\n    private String clusterName;\n\n    private String brokerName;\n\n    public ApplyBrokerIdResponseHeader() {\n    }\n\n    public ApplyBrokerIdResponseHeader(String clusterName, String brokerName) {\n        this.clusterName = clusterName;\n        this.brokerName = brokerName;\n    }\n\n\n    @Override\n    public String toString() {\n        return \"ApplyBrokerIdResponseHeader{\" +\n                \"clusterName='\" + clusterName + '\\'' +\n                \", brokerName='\" + brokerName + '\\'' +\n                '}';\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    public void setClusterName(String clusterName) {\n        this.clusterName = clusterName;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/register/GetNextBrokerIdRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header.controller.register;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.CONTROLLER_GET_NEXT_BROKER_ID, resource = ResourceType.CLUSTER, action = Action.GET)\npublic class GetNextBrokerIdRequestHeader implements CommandCustomHeader {\n\n    @RocketMQResource(ResourceType.CLUSTER)\n    private String clusterName;\n\n    private String brokerName;\n\n    public GetNextBrokerIdRequestHeader() {\n\n    }\n\n    public GetNextBrokerIdRequestHeader(String clusterName, String brokerName) {\n        this.clusterName = clusterName;\n        this.brokerName = brokerName;\n    }\n\n    @Override\n    public String toString() {\n        return \"GetNextBrokerIdRequestHeader{\" +\n                \"clusterName='\" + clusterName + '\\'' +\n                \", brokerName='\" + brokerName + '\\'' +\n                '}';\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public void setClusterName(String clusterName) {\n        this.clusterName = clusterName;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/register/GetNextBrokerIdResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header.controller.register;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class GetNextBrokerIdResponseHeader implements CommandCustomHeader {\n\n    private String clusterName;\n\n    private String brokerName;\n\n    private Long nextBrokerId;\n\n    public GetNextBrokerIdResponseHeader() {\n    }\n\n    public GetNextBrokerIdResponseHeader(String clusterName, String brokerName) {\n        this(clusterName, brokerName, null);\n    }\n\n    public GetNextBrokerIdResponseHeader(String clusterName, String brokerName, Long nextBrokerId) {\n        this.clusterName = clusterName;\n        this.brokerName = brokerName;\n        this.nextBrokerId = nextBrokerId;\n    }\n\n    @Override\n    public String toString() {\n        return \"GetNextBrokerIdResponseHeader{\" +\n                \"clusterName='\" + clusterName + '\\'' +\n                \", brokerName='\" + brokerName + '\\'' +\n                \", nextBrokerId=\" + nextBrokerId +\n                '}';\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public void setNextBrokerId(Long nextBrokerId) {\n        this.nextBrokerId = nextBrokerId;\n    }\n\n    public Long getNextBrokerId() {\n        return nextBrokerId;\n    }\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    public void setClusterName(String clusterName) {\n        this.clusterName = clusterName;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/register/RegisterBrokerToControllerRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header.controller.register;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.CONTROLLER_REGISTER_BROKER, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class RegisterBrokerToControllerRequestHeader implements CommandCustomHeader {\n\n    @RocketMQResource(ResourceType.CLUSTER)\n    private String clusterName;\n\n    private String brokerName;\n\n    private Long brokerId;\n\n    private String brokerAddress;\n\n    private long invokeTime;\n\n    public RegisterBrokerToControllerRequestHeader() {\n    }\n\n    public RegisterBrokerToControllerRequestHeader(String clusterName, String brokerName, Long brokerId, String brokerAddress) {\n        this.clusterName = clusterName;\n        this.brokerName = brokerName;\n        this.brokerId = brokerId;\n        this.brokerAddress = brokerAddress;\n        this.invokeTime = System.currentTimeMillis();\n    }\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public long getInvokeTime() {\n        return invokeTime;\n    }\n\n    public void setInvokeTime(long invokeTime) {\n        this.invokeTime = invokeTime;\n    }\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public Long getBrokerId() {\n        return brokerId;\n    }\n\n    public String getBrokerAddress() {\n        return brokerAddress;\n    }\n\n    public void setClusterName(String clusterName) {\n        this.clusterName = clusterName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public void setBrokerId(Long brokerId) {\n        this.brokerId = brokerId;\n    }\n\n    public void setBrokerAddress(String brokerAddress) {\n        this.brokerAddress = brokerAddress;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/register/RegisterBrokerToControllerResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header.controller.register;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class RegisterBrokerToControllerResponseHeader implements CommandCustomHeader {\n\n    private String clusterName;\n\n    private String brokerName;\n\n    private Long masterBrokerId;\n\n    private String masterAddress;\n\n    private Integer masterEpoch;\n\n    private Integer syncStateSetEpoch;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public RegisterBrokerToControllerResponseHeader() {\n    }\n\n    public RegisterBrokerToControllerResponseHeader(String clusterName, String brokerName) {\n        this.clusterName = clusterName;\n        this.brokerName = brokerName;\n    }\n\n    public void setMasterBrokerId(Long masterBrokerId) {\n        this.masterBrokerId = masterBrokerId;\n    }\n\n    public void setMasterAddress(String masterAddress) {\n        this.masterAddress = masterAddress;\n    }\n\n    public void setMasterEpoch(Integer masterEpoch) {\n        this.masterEpoch = masterEpoch;\n    }\n\n    public void setSyncStateSetEpoch(Integer syncStateSetEpoch) {\n        this.syncStateSetEpoch = syncStateSetEpoch;\n    }\n\n    public Integer getMasterEpoch() {\n        return masterEpoch;\n    }\n\n    public Integer getSyncStateSetEpoch() {\n        return syncStateSetEpoch;\n    }\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public Long getMasterBrokerId() {\n        return masterBrokerId;\n    }\n\n    public String getMasterAddress() {\n        return masterAddress;\n    }\n\n    public void setClusterName(String clusterName) {\n        this.clusterName = clusterName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/AddWritePermOfBrokerRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header.namesrv;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.ADD_WRITE_PERM_OF_BROKER, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class AddWritePermOfBrokerRequestHeader implements CommandCustomHeader {\n    @CFNotNull\n    private String brokerName;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/AddWritePermOfBrokerResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header.namesrv;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class AddWritePermOfBrokerResponseHeader implements CommandCustomHeader {\n    @CFNotNull\n    private Integer addTopicCount;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public Integer getAddTopicCount() {\n        return addTopicCount;\n    }\n\n    public void setAddTopicCount(Integer addTopicCount) {\n        this.addTopicCount = addTopicCount;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/BrokerHeartbeatRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header.namesrv;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.BROKER_HEARTBEAT, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class BrokerHeartbeatRequestHeader implements CommandCustomHeader {\n    @CFNotNull\n    @RocketMQResource(ResourceType.CLUSTER)\n    private String clusterName;\n    @CFNotNull\n    private String brokerAddr;\n    @CFNotNull\n    private String brokerName;\n    @CFNullable\n    private Long brokerId;\n    @CFNullable\n    private Integer epoch;\n    @CFNullable\n    private Long maxOffset;\n    @CFNullable\n    private Long confirmOffset;\n    @CFNullable\n    private Long heartbeatTimeoutMills;\n    @CFNullable\n    private Integer electionPriority;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getBrokerAddr() {\n        return brokerAddr;\n    }\n\n    public void setBrokerAddr(String brokerAddr) {\n        this.brokerAddr = brokerAddr;\n    }\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    public void setClusterName(String clusterName) {\n        this.clusterName = clusterName;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public Integer getEpoch() {\n        return epoch;\n    }\n\n    public void setEpoch(Integer epoch) {\n        this.epoch = epoch;\n    }\n\n    public Long getMaxOffset() {\n        return maxOffset;\n    }\n\n    public void setMaxOffset(Long maxOffset) {\n        this.maxOffset = maxOffset;\n    }\n\n    public Long getConfirmOffset() {\n        return confirmOffset;\n    }\n\n    public void setConfirmOffset(Long confirmOffset) {\n        this.confirmOffset = confirmOffset;\n    }\n\n    public Long getBrokerId() {\n        return brokerId;\n    }\n\n    public void setBrokerId(Long brokerId) {\n        this.brokerId = brokerId;\n    }\n\n    public Long getHeartbeatTimeoutMills() {\n        return heartbeatTimeoutMills;\n    }\n\n    public void setHeartbeatTimeoutMills(Long heartbeatTimeoutMills) {\n        this.heartbeatTimeoutMills = heartbeatTimeoutMills;\n    }\n\n    public Integer getElectionPriority() {\n        return electionPriority;\n    }\n\n    public void setElectionPriority(Integer electionPriority) {\n        this.electionPriority = electionPriority;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/DeleteKVConfigRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header.namesrv;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.DELETE_KV_CONFIG, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class DeleteKVConfigRequestHeader implements CommandCustomHeader {\n    @CFNotNull\n    private String namespace;\n    @CFNotNull\n    private String key;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getNamespace() {\n        return namespace;\n    }\n\n    public void setNamespace(String namespace) {\n        this.namespace = namespace;\n    }\n\n    public String getKey() {\n        return key;\n    }\n\n    public void setKey(String key) {\n        this.key = key;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/DeleteTopicFromNamesrvRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header.namesrv;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicRequestHeader;\n\n@RocketMQAction(value = RequestCode.DELETE_TOPIC_IN_NAMESRV, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class DeleteTopicFromNamesrvRequestHeader extends TopicRequestHeader {\n    @CFNotNull\n    private String topic;\n\n    @RocketMQResource(ResourceType.CLUSTER)\n    private String clusterName;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    public void setClusterName(String clusterName) {\n        this.clusterName = clusterName;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/GetKVConfigRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header.namesrv;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.GET_KV_CONFIG, resource = ResourceType.CLUSTER, action = Action.GET)\npublic class GetKVConfigRequestHeader implements CommandCustomHeader {\n    @CFNotNull\n    private String namespace;\n    @CFNotNull\n    private String key;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getNamespace() {\n        return namespace;\n    }\n\n    public void setNamespace(String namespace) {\n        this.namespace = namespace;\n    }\n\n    public String getKey() {\n        return key;\n    }\n\n    public void setKey(String key) {\n        this.key = key;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/GetKVConfigResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header.namesrv;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class GetKVConfigResponseHeader implements CommandCustomHeader {\n    @CFNullable\n    private String value;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getValue() {\n        return value;\n    }\n\n    public void setValue(String value) {\n        this.value = value;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/GetKVListByNamespaceRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header.namesrv;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.GET_KVLIST_BY_NAMESPACE, resource = ResourceType.CLUSTER, action = Action.GET)\npublic class GetKVListByNamespaceRequestHeader implements CommandCustomHeader {\n    @CFNotNull\n    private String namespace;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getNamespace() {\n        return namespace;\n    }\n\n    public void setNamespace(String namespace) {\n        this.namespace = namespace;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/GetRouteInfoRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: GetRouteInfoRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header.namesrv;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.rpc.TopicRequestHeader;\n\n@RocketMQAction(value = RequestCode.GET_ROUTEINFO_BY_TOPIC, resource = ResourceType.CLUSTER, action = Action.GET)\npublic class GetRouteInfoRequestHeader extends TopicRequestHeader {\n\n    @CFNotNull\n    private String topic;\n\n    @CFNullable\n    private Boolean acceptStandardJsonOnly;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public Boolean getAcceptStandardJsonOnly() {\n        return acceptStandardJsonOnly;\n    }\n\n    public void setAcceptStandardJsonOnly(Boolean acceptStandardJsonOnly) {\n        this.acceptStandardJsonOnly = acceptStandardJsonOnly;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/PutKVConfigRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header.namesrv;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.PUT_KV_CONFIG, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class PutKVConfigRequestHeader implements CommandCustomHeader {\n    @CFNotNull\n    private String namespace;\n    @CFNotNull\n    private String key;\n    @CFNotNull\n    private String value;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getNamespace() {\n        return namespace;\n    }\n\n    public void setNamespace(String namespace) {\n        this.namespace = namespace;\n    }\n\n    public String getKey() {\n        return key;\n    }\n\n    public void setKey(String key) {\n        this.key = key;\n    }\n\n    public String getValue() {\n        return value;\n    }\n\n    public void setValue(String value) {\n        this.value = value;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/QueryDataVersionRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header.namesrv;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.QUERY_DATA_VERSION, resource = ResourceType.CLUSTER, action = Action.GET)\npublic class QueryDataVersionRequestHeader implements CommandCustomHeader {\n    @CFNotNull\n    private String brokerName;\n    @CFNotNull\n    private String brokerAddr;\n    @CFNotNull\n    @RocketMQResource(ResourceType.CLUSTER)\n    private String clusterName;\n    @CFNotNull\n    private Long brokerId;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public String getBrokerAddr() {\n        return brokerAddr;\n    }\n\n    public void setBrokerAddr(String brokerAddr) {\n        this.brokerAddr = brokerAddr;\n    }\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    public void setClusterName(String clusterName) {\n        this.clusterName = clusterName;\n    }\n\n    public Long getBrokerId() {\n        return brokerId;\n    }\n\n    public void setBrokerId(Long brokerId) {\n        this.brokerId = brokerId;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/QueryDataVersionResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header.namesrv;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class QueryDataVersionResponseHeader implements CommandCustomHeader {\n    @CFNotNull\n    private Boolean changed;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public Boolean getChanged() {\n        return changed;\n    }\n\n    public void setChanged(Boolean changed) {\n        this.changed = changed;\n    }\n\n    @Override\n    public String toString() {\n        final StringBuilder sb = new StringBuilder(\"QueryDataVersionResponseHeader{\");\n        sb.append(\"changed=\").append(changed);\n        sb.append('}');\n        return sb.toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/RegisterBrokerRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: RegisterBrokerRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header.namesrv;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.REGISTER_BROKER, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class RegisterBrokerRequestHeader implements CommandCustomHeader {\n    @CFNotNull\n    private String brokerName;\n    @CFNotNull\n    private String brokerAddr;\n    @CFNotNull\n    @RocketMQResource(ResourceType.CLUSTER)\n    private String clusterName;\n    @CFNotNull\n    private String haServerAddr;\n    @CFNotNull\n    private Long brokerId;\n    @CFNullable\n    private Long heartbeatTimeoutMillis;\n    @CFNullable\n    private Boolean enableActingMaster;\n\n    private boolean compressed;\n\n    private Integer bodyCrc32 = 0;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public String getBrokerAddr() {\n        return brokerAddr;\n    }\n\n    public void setBrokerAddr(String brokerAddr) {\n        this.brokerAddr = brokerAddr;\n    }\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    public void setClusterName(String clusterName) {\n        this.clusterName = clusterName;\n    }\n\n    public String getHaServerAddr() {\n        return haServerAddr;\n    }\n\n    public void setHaServerAddr(String haServerAddr) {\n        this.haServerAddr = haServerAddr;\n    }\n\n    public Long getBrokerId() {\n        return brokerId;\n    }\n\n    public void setBrokerId(Long brokerId) {\n        this.brokerId = brokerId;\n    }\n\n    public Long getHeartbeatTimeoutMillis() {\n        return heartbeatTimeoutMillis;\n    }\n\n    public void setHeartbeatTimeoutMillis(Long heartbeatTimeoutMillis) {\n        this.heartbeatTimeoutMillis = heartbeatTimeoutMillis;\n    }\n\n    public boolean isCompressed() {\n        return compressed;\n    }\n\n    public void setCompressed(boolean compressed) {\n        this.compressed = compressed;\n    }\n\n    public Integer getBodyCrc32() {\n        return bodyCrc32;\n    }\n\n    public void setBodyCrc32(Integer bodyCrc32) {\n        this.bodyCrc32 = bodyCrc32;\n    }\n\n    public Boolean getEnableActingMaster() {\n        return enableActingMaster;\n    }\n\n    public void setEnableActingMaster(Boolean enableActingMaster) {\n        this.enableActingMaster = enableActingMaster;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/RegisterBrokerResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header.namesrv;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class RegisterBrokerResponseHeader implements CommandCustomHeader {\n    @CFNullable\n    private String haServerAddr;\n    @CFNullable\n    private String masterAddr;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getHaServerAddr() {\n        return haServerAddr;\n    }\n\n    public void setHaServerAddr(String haServerAddr) {\n        this.haServerAddr = haServerAddr;\n    }\n\n    public String getMasterAddr() {\n        return masterAddr;\n    }\n\n    public void setMasterAddr(String masterAddr) {\n        this.masterAddr = masterAddr;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/RegisterOrderTopicRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: RegisterOrderTopicRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header.namesrv;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class RegisterOrderTopicRequestHeader implements CommandCustomHeader {\n    @CFNotNull\n    private String topic;\n    @CFNotNull\n    private String orderTopicString;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getOrderTopicString() {\n        return orderTopicString;\n    }\n\n    public void setOrderTopicString(String orderTopicString) {\n        this.orderTopicString = orderTopicString;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/RegisterTopicRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header.namesrv;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.rpc.TopicRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.REGISTER_TOPIC_IN_NAMESRV, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class RegisterTopicRequestHeader extends TopicRequestHeader {\n    @CFNotNull\n    private String topic;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/UnRegisterBrokerRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: UnRegisterBrokerRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.header.namesrv;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.common.resource.RocketMQResource;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.UNREGISTER_BROKER, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class UnRegisterBrokerRequestHeader implements CommandCustomHeader {\n    @CFNotNull\n    private String brokerName;\n    @CFNotNull\n    private String brokerAddr;\n    @CFNotNull\n    @RocketMQResource(ResourceType.CLUSTER)\n    private String clusterName;\n    @CFNotNull\n    private Long brokerId;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public String getBrokerAddr() {\n        return brokerAddr;\n    }\n\n    public void setBrokerAddr(String brokerAddr) {\n        this.brokerAddr = brokerAddr;\n    }\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    public void setClusterName(String clusterName) {\n        this.clusterName = clusterName;\n    }\n\n    public Long getBrokerId() {\n        return brokerId;\n    }\n\n    public void setBrokerId(Long brokerId) {\n        this.brokerId = brokerId;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/WipeWritePermOfBrokerRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header.namesrv;\n\nimport org.apache.rocketmq.common.action.Action;\nimport org.apache.rocketmq.common.action.RocketMQAction;\nimport org.apache.rocketmq.common.resource.ResourceType;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\n\n@RocketMQAction(value = RequestCode.WIPE_WRITE_PERM_OF_BROKER, resource = ResourceType.CLUSTER, action = Action.UPDATE)\npublic class WipeWritePermOfBrokerRequestHeader implements CommandCustomHeader {\n    @CFNotNull\n    private String brokerName;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/WipeWritePermOfBrokerResponseHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header.namesrv;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\n\npublic class WipeWritePermOfBrokerResponseHeader implements CommandCustomHeader {\n    @CFNotNull\n    private Integer wipeTopicCount;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public Integer getWipeTopicCount() {\n        return wipeTopicCount;\n    }\n\n    public void setWipeTopicCount(Integer wipeTopicCount) {\n        this.wipeTopicCount = wipeTopicCount;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/heartbeat/ConsumeType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: ConsumeType.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.heartbeat;\n\npublic enum ConsumeType {\n\n    CONSUME_ACTIVELY(\"PULL\"),\n\n    CONSUME_PASSIVELY(\"PUSH\"),\n\n    CONSUME_POP(\"POP\");\n\n    private String typeCN;\n\n    ConsumeType(String typeCN) {\n        this.typeCN = typeCN;\n    }\n\n    public String getTypeCN() {\n        return typeCN;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/heartbeat/ConsumerData.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: ConsumerData.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.heartbeat;\n\nimport java.util.HashSet;\nimport java.util.Set;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\n\npublic class ConsumerData {\n    private String groupName;\n    private ConsumeType consumeType;\n    private MessageModel messageModel;\n    private ConsumeFromWhere consumeFromWhere;\n    private Set<SubscriptionData> subscriptionDataSet = new HashSet<>();\n    private boolean unitMode;\n\n    public String getGroupName() {\n        return groupName;\n    }\n\n    public void setGroupName(String groupName) {\n        this.groupName = groupName;\n    }\n\n    public ConsumeType getConsumeType() {\n        return consumeType;\n    }\n\n    public void setConsumeType(ConsumeType consumeType) {\n        this.consumeType = consumeType;\n    }\n\n    public MessageModel getMessageModel() {\n        return messageModel;\n    }\n\n    public void setMessageModel(MessageModel messageModel) {\n        this.messageModel = messageModel;\n    }\n\n    public ConsumeFromWhere getConsumeFromWhere() {\n        return consumeFromWhere;\n    }\n\n    public void setConsumeFromWhere(ConsumeFromWhere consumeFromWhere) {\n        this.consumeFromWhere = consumeFromWhere;\n    }\n\n    public Set<SubscriptionData> getSubscriptionDataSet() {\n        return subscriptionDataSet;\n    }\n\n    public void setSubscriptionDataSet(Set<SubscriptionData> subscriptionDataSet) {\n        this.subscriptionDataSet = subscriptionDataSet;\n    }\n\n    public boolean isUnitMode() {\n        return unitMode;\n    }\n\n    public void setUnitMode(boolean isUnitMode) {\n        this.unitMode = isUnitMode;\n    }\n\n    @Override\n    public String toString() {\n        return \"ConsumerData [groupName=\" + groupName + \", consumeType=\" + consumeType + \", messageModel=\"\n            + messageModel + \", consumeFromWhere=\" + consumeFromWhere + \", unitMode=\" + unitMode\n            + \", subscriptionDataSet=\" + subscriptionDataSet + \"]\";\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/heartbeat/HeartbeatData.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: HeartbeatData.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.heartbeat;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.alibaba.fastjson2.JSONWriter;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\nimport java.util.HashSet;\nimport java.util.Set;\n\npublic class HeartbeatData extends RemotingSerializable {\n    private String clientID;\n    private Set<ProducerData> producerDataSet = new HashSet<>();\n    private Set<ConsumerData> consumerDataSet = new HashSet<>();\n    private int heartbeatFingerprint = 0;\n    private boolean isWithoutSub = false;\n\n    public String getClientID() {\n        return clientID;\n    }\n\n    public void setClientID(String clientID) {\n        this.clientID = clientID;\n    }\n\n    public Set<ProducerData> getProducerDataSet() {\n        return producerDataSet;\n    }\n\n    public void setProducerDataSet(Set<ProducerData> producerDataSet) {\n        this.producerDataSet = producerDataSet;\n    }\n\n    public Set<ConsumerData> getConsumerDataSet() {\n        return consumerDataSet;\n    }\n\n    public void setConsumerDataSet(Set<ConsumerData> consumerDataSet) {\n        this.consumerDataSet = consumerDataSet;\n    }\n\n    public int getHeartbeatFingerprint() {\n        return heartbeatFingerprint;\n    }\n\n    public void setHeartbeatFingerprint(int heartbeatFingerprint) {\n        this.heartbeatFingerprint = heartbeatFingerprint;\n    }\n\n    public boolean isWithoutSub() {\n        return isWithoutSub;\n    }\n\n    public void setWithoutSub(boolean withoutSub) {\n        isWithoutSub = withoutSub;\n    }\n\n    @Override\n    public String toString() {\n        return \"HeartbeatData [clientID=\" + clientID + \", producerDataSet=\" + producerDataSet\n            + \", consumerDataSet=\" + consumerDataSet + \"]\";\n    }\n\n    public int computeHeartbeatFingerprint() {\n        HeartbeatData heartbeatDataCopy = JSON.parseObject(JSON.toJSONString(this, JSONWriter.Feature.ReferenceDetection), HeartbeatData.class);\n        for (ConsumerData consumerData : heartbeatDataCopy.getConsumerDataSet()) {\n            for (SubscriptionData subscriptionData : consumerData.getSubscriptionDataSet()) {\n                subscriptionData.setSubVersion(0L);\n            }\n        }\n        heartbeatDataCopy.setWithoutSub(false);\n        heartbeatDataCopy.setHeartbeatFingerprint(0);\n        heartbeatDataCopy.setClientID(\"\");\n        return JSON.toJSONString(heartbeatDataCopy, JSONWriter.Feature.ReferenceDetection).hashCode();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/heartbeat/MessageModel.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: MessageModel.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.heartbeat;\n\n/**\n * Message model\n */\npublic enum MessageModel {\n    /**\n     * broadcast\n     */\n    BROADCASTING(\"BROADCASTING\"),\n    /**\n     * clustering\n     */\n    CLUSTERING(\"CLUSTERING\"),\n    /**\n     * for lite consumer\n     */\n    LITE_SELECTIVE(\"LITE_SELECTIVE\");\n\n    private String modeCN;\n\n    MessageModel(String modeCN) {\n        this.modeCN = modeCN;\n    }\n\n    public String getModeCN() {\n        return modeCN;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/heartbeat/ProducerData.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: ProducerData.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.heartbeat;\n\npublic class ProducerData {\n    private String groupName;\n\n    public String getGroupName() {\n        return groupName;\n    }\n\n    public void setGroupName(String groupName) {\n        this.groupName = groupName;\n    }\n\n    @Override\n    public String toString() {\n        return \"ProducerData [groupName=\" + groupName + \"]\";\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/heartbeat/SubscriptionData.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: SubscriptionData.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.heartbeat;\n\nimport com.alibaba.fastjson2.annotation.JSONField;\nimport org.apache.rocketmq.common.filter.ExpressionType;\n\nimport java.util.HashSet;\nimport java.util.Set;\n\npublic class SubscriptionData implements Comparable<SubscriptionData> {\n    public final static String SUB_ALL = \"*\";\n    private boolean classFilterMode = false;\n    private String topic;\n    private String subString;\n    private Set<String> tagsSet = new HashSet<>();\n    private Set<Integer> codeSet = new HashSet<>();\n    private long subVersion = System.currentTimeMillis();\n    private String expressionType = ExpressionType.TAG;\n\n    @JSONField(serialize = false)\n    private String filterClassSource;\n\n    public SubscriptionData() {\n\n    }\n\n    public SubscriptionData(String topic, String subString) {\n        super();\n        this.topic = topic;\n        this.subString = subString;\n    }\n\n    public String getFilterClassSource() {\n        return filterClassSource;\n    }\n\n    public void setFilterClassSource(String filterClassSource) {\n        this.filterClassSource = filterClassSource;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getSubString() {\n        return subString;\n    }\n\n    public void setSubString(String subString) {\n        this.subString = subString;\n    }\n\n    public Set<String> getTagsSet() {\n        return tagsSet;\n    }\n\n    public void setTagsSet(Set<String> tagsSet) {\n        this.tagsSet = tagsSet;\n    }\n\n    public long getSubVersion() {\n        return subVersion;\n    }\n\n    public void setSubVersion(long subVersion) {\n        this.subVersion = subVersion;\n    }\n\n    public Set<Integer> getCodeSet() {\n        return codeSet;\n    }\n\n    public void setCodeSet(Set<Integer> codeSet) {\n        this.codeSet = codeSet;\n    }\n\n    public boolean isClassFilterMode() {\n        return classFilterMode;\n    }\n\n    public void setClassFilterMode(boolean classFilterMode) {\n        this.classFilterMode = classFilterMode;\n    }\n\n    public String getExpressionType() {\n        return expressionType;\n    }\n\n    public void setExpressionType(String expressionType) {\n        this.expressionType = expressionType;\n    }\n\n    @Override\n    public int hashCode() {\n        final int prime = 31;\n        int result = 1;\n        result = prime * result + (classFilterMode ? 1231 : 1237);\n        result = prime * result + ((codeSet == null) ? 0 : codeSet.hashCode());\n        result = prime * result + ((subString == null) ? 0 : subString.hashCode());\n        result = prime * result + ((tagsSet == null) ? 0 : tagsSet.hashCode());\n        result = prime * result + ((topic == null) ? 0 : topic.hashCode());\n        result = prime * result + ((expressionType == null) ? 0 : expressionType.hashCode());\n        return result;\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj)\n            return true;\n        if (obj == null)\n            return false;\n        if (getClass() != obj.getClass())\n            return false;\n        SubscriptionData other = (SubscriptionData) obj;\n        if (classFilterMode != other.classFilterMode)\n            return false;\n        if (codeSet == null) {\n            if (other.codeSet != null)\n                return false;\n        } else if (!codeSet.equals(other.codeSet))\n            return false;\n        if (subString == null) {\n            if (other.subString != null)\n                return false;\n        } else if (!subString.equals(other.subString))\n            return false;\n        if (subVersion != other.subVersion)\n            return false;\n        if (tagsSet == null) {\n            if (other.tagsSet != null)\n                return false;\n        } else if (!tagsSet.equals(other.tagsSet))\n            return false;\n        if (topic == null) {\n            if (other.topic != null)\n                return false;\n        } else if (!topic.equals(other.topic))\n            return false;\n        if (expressionType == null) {\n            if (other.expressionType != null)\n                return false;\n        } else if (!expressionType.equals(other.expressionType))\n            return false;\n        return true;\n    }\n\n    @Override\n    public String toString() {\n        return \"SubscriptionData [classFilterMode=\" + classFilterMode + \", topic=\" + topic + \", subString=\"\n            + subString + \", tagsSet=\" + tagsSet + \", codeSet=\" + codeSet + \", subVersion=\" + subVersion\n            + \", expressionType=\" + expressionType + \"]\";\n    }\n\n    @Override\n    public int compareTo(SubscriptionData other) {\n        String thisValue = this.topic + \"@\" + this.subString;\n        String otherValue = other.topic + \"@\" + other.subString;\n        return thisValue.compareTo(otherValue);\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/namesrv/RegisterBrokerResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.namesrv;\n\nimport org.apache.rocketmq.remoting.protocol.body.KVTable;\n\npublic class RegisterBrokerResult {\n    private String haServerAddr;\n    private String masterAddr;\n    private KVTable kvTable;\n\n    public String getHaServerAddr() {\n        return haServerAddr;\n    }\n\n    public void setHaServerAddr(String haServerAddr) {\n        this.haServerAddr = haServerAddr;\n    }\n\n    public String getMasterAddr() {\n        return masterAddr;\n    }\n\n    public void setMasterAddr(String masterAddr) {\n        this.masterAddr = masterAddr;\n    }\n\n    public KVTable getKvTable() {\n        return kvTable;\n    }\n\n    public void setKvTable(KVTable kvTable) {\n        this.kvTable = kvTable;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/route/BrokerData.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.route;\n\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Random;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.MixAll;\n\n/**\n * The class describes that a typical broker cluster's (in replication) details: the cluster (in sharding) name\n * that it belongs to, and all the single instance information for this cluster.\n */\npublic class BrokerData implements Comparable<BrokerData> {\n    private String cluster;\n    private String brokerName;\n\n    /**\n     * The container that store the all single instances for the current broker replication cluster.\n     * The key is the brokerId, and the value is the address of the single broker instance.\n     */\n    private HashMap<Long, String> brokerAddrs;\n    private String zoneName;\n    private final Random random = new Random();\n\n    /**\n     * Enable acting master or not, used for old version HA adaption,\n     */\n    private boolean enableActingMaster = false;\n\n    public BrokerData() {\n\n    }\n\n    public BrokerData(BrokerData brokerData) {\n        this.cluster = brokerData.cluster;\n        this.brokerName = brokerData.brokerName;\n        if (brokerData.brokerAddrs != null) {\n            this.brokerAddrs = new HashMap<>(brokerData.brokerAddrs);\n        }\n        this.zoneName = brokerData.zoneName;\n        this.enableActingMaster = brokerData.enableActingMaster;\n    }\n\n    public BrokerData(String cluster, String brokerName, HashMap<Long, String> brokerAddrs) {\n        this.cluster = cluster;\n        this.brokerName = brokerName;\n        this.brokerAddrs = brokerAddrs;\n    }\n\n    public BrokerData(String cluster, String brokerName, HashMap<Long, String> brokerAddrs,\n        boolean enableActingMaster) {\n        this.cluster = cluster;\n        this.brokerName = brokerName;\n        this.brokerAddrs = brokerAddrs;\n        this.enableActingMaster = enableActingMaster;\n    }\n\n    public BrokerData(String cluster, String brokerName, HashMap<Long, String> brokerAddrs, boolean enableActingMaster,\n        String zoneName) {\n        this.cluster = cluster;\n        this.brokerName = brokerName;\n        this.brokerAddrs = brokerAddrs;\n        this.enableActingMaster = enableActingMaster;\n        this.zoneName = zoneName;\n    }\n\n    /**\n     * Selects a (preferably master) broker address from the registered list. If the master's address cannot be found, a\n     * slave broker address is selected in a random manner.\n     *\n     * @return Broker address.\n     */\n    public String selectBrokerAddr() {\n        String masterAddress = this.brokerAddrs.get(MixAll.MASTER_ID);\n\n        if (masterAddress == null) {\n            List<String> addrs = new ArrayList<>(brokerAddrs.values());\n            return addrs.get(random.nextInt(addrs.size()));\n        }\n\n        return masterAddress;\n    }\n\n    public HashMap<Long, String> getBrokerAddrs() {\n        return brokerAddrs;\n    }\n\n    public void setBrokerAddrs(HashMap<Long, String> brokerAddrs) {\n        this.brokerAddrs = brokerAddrs;\n    }\n\n    public String getCluster() {\n        return cluster;\n    }\n\n    public void setCluster(String cluster) {\n        this.cluster = cluster;\n    }\n\n    public boolean isEnableActingMaster() {\n        return enableActingMaster;\n    }\n\n    public void setEnableActingMaster(boolean enableActingMaster) {\n        this.enableActingMaster = enableActingMaster;\n    }\n\n    public String getZoneName() {\n        return zoneName;\n    }\n\n    public void setZoneName(String zoneName) {\n        this.zoneName = zoneName;\n    }\n\n    @Override\n    public int hashCode() {\n        final int prime = 31;\n        int result = 1;\n        result = prime * result + ((brokerAddrs == null) ? 0 : brokerAddrs.hashCode());\n        result = prime * result + ((brokerName == null) ? 0 : brokerName.hashCode());\n        return result;\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj) {\n            return true;\n        }\n        if (obj == null) {\n            return false;\n        }\n        if (getClass() != obj.getClass()) {\n            return false;\n        }\n        BrokerData other = (BrokerData) obj;\n        if (brokerAddrs == null) {\n            if (other.brokerAddrs != null) {\n                return false;\n            }\n        } else if (!brokerAddrs.equals(other.brokerAddrs)) {\n            return false;\n        }\n        return StringUtils.equals(brokerName, other.brokerName);\n    }\n\n    @Override\n    public String toString() {\n        return \"BrokerData [brokerName=\" + brokerName + \", brokerAddrs=\" + brokerAddrs + \", enableActingMaster=\" + enableActingMaster + \"]\";\n    }\n\n    @Override\n    public int compareTo(BrokerData o) {\n        return this.brokerName.compareTo(o.getBrokerName());\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/route/MessageQueueRouteState.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.route;\n\npublic enum MessageQueueRouteState {\n    // do not change below order, since ordinal() is used\n    Expired,\n    ReadOnly,\n    Normal,\n    WriteOnly,\n    ;\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/route/QueueData.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/*\n  $Id: QueueData.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.route;\n\npublic class QueueData implements Comparable<QueueData> {\n    private String brokerName;\n    private int readQueueNums;\n    private int writeQueueNums;\n    private int perm;\n    private int topicSysFlag;\n\n    public QueueData() {\n\n    }\n\n    // Deep copy QueueData\n    public QueueData(QueueData queueData) {\n        this.brokerName = queueData.brokerName;\n        this.readQueueNums = queueData.readQueueNums;\n        this.writeQueueNums = queueData.writeQueueNums;\n        this.perm = queueData.perm;\n        this.topicSysFlag = queueData.topicSysFlag;\n    }\n\n    public int getReadQueueNums() {\n        return readQueueNums;\n    }\n\n    public void setReadQueueNums(int readQueueNums) {\n        this.readQueueNums = readQueueNums;\n    }\n\n    public int getWriteQueueNums() {\n        return writeQueueNums;\n    }\n\n    public void setWriteQueueNums(int writeQueueNums) {\n        this.writeQueueNums = writeQueueNums;\n    }\n\n    public int getPerm() {\n        return perm;\n    }\n\n    public void setPerm(int perm) {\n        this.perm = perm;\n    }\n\n    public int getTopicSysFlag() {\n        return topicSysFlag;\n    }\n\n    public void setTopicSysFlag(int topicSysFlag) {\n        this.topicSysFlag = topicSysFlag;\n    }\n\n    @Override\n    public int hashCode() {\n        final int prime = 31;\n        int result = 1;\n        result = prime * result + ((brokerName == null) ? 0 : brokerName.hashCode());\n        result = prime * result + perm;\n        result = prime * result + readQueueNums;\n        result = prime * result + writeQueueNums;\n        result = prime * result + topicSysFlag;\n        return result;\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj)\n            return true;\n        if (obj == null)\n            return false;\n        if (getClass() != obj.getClass())\n            return false;\n        QueueData other = (QueueData) obj;\n        if (brokerName == null) {\n            if (other.brokerName != null)\n                return false;\n        } else if (!brokerName.equals(other.brokerName))\n            return false;\n        if (perm != other.perm)\n            return false;\n        if (readQueueNums != other.readQueueNums)\n            return false;\n        if (writeQueueNums != other.writeQueueNums)\n            return false;\n        return topicSysFlag == other.topicSysFlag;\n    }\n\n    @Override\n    public String toString() {\n        return \"QueueData [brokerName=\" + brokerName + \", readQueueNums=\" + readQueueNums\n            + \", writeQueueNums=\" + writeQueueNums + \", perm=\" + perm + \", topicSysFlag=\" + topicSysFlag\n            + \"]\";\n    }\n\n    @Override\n    public int compareTo(QueueData o) {\n        return this.brokerName.compareTo(o.getBrokerName());\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/route/TopicRouteData.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: TopicRouteData.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.remoting.protocol.route;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo;\n\npublic class TopicRouteData extends RemotingSerializable {\n    private String orderTopicConf;\n    private List<QueueData> queueDatas;\n    private List<BrokerData> brokerDatas;\n    private HashMap<String/* brokerAddr */, List<String>/* Filter Server */> filterServerTable;\n    //It could be null or empty\n    private Map<String/*brokerName*/, TopicQueueMappingInfo> topicQueueMappingByBroker;\n\n    public TopicRouteData() {\n        queueDatas = new ArrayList<>();\n        brokerDatas = new ArrayList<>();\n        filterServerTable = new HashMap<>();\n    }\n\n    public TopicRouteData(TopicRouteData topicRouteData) {\n        this.queueDatas = new ArrayList<>();\n        this.brokerDatas = new ArrayList<>();\n        this.filterServerTable = new HashMap<>();\n        this.orderTopicConf = topicRouteData.orderTopicConf;\n\n        if (topicRouteData.queueDatas != null) {\n            this.queueDatas.addAll(topicRouteData.queueDatas);\n        }\n\n        if (topicRouteData.brokerDatas != null) {\n            this.brokerDatas.addAll(topicRouteData.brokerDatas);\n        }\n\n        if (topicRouteData.filterServerTable != null) {\n            this.filterServerTable.putAll(topicRouteData.filterServerTable);\n        }\n\n        if (topicRouteData.topicQueueMappingByBroker != null) {\n            this.topicQueueMappingByBroker = new HashMap<>(topicRouteData.topicQueueMappingByBroker);\n        }\n    }\n\n    public TopicRouteData cloneTopicRouteData() {\n        TopicRouteData topicRouteData = new TopicRouteData();\n        topicRouteData.setQueueDatas(new ArrayList<>());\n        topicRouteData.setBrokerDatas(new ArrayList<>());\n        topicRouteData.setFilterServerTable(new HashMap<>());\n        topicRouteData.setOrderTopicConf(this.orderTopicConf);\n\n        topicRouteData.getQueueDatas().addAll(this.queueDatas);\n        topicRouteData.getBrokerDatas().addAll(this.brokerDatas);\n        topicRouteData.getFilterServerTable().putAll(this.filterServerTable);\n        if (this.topicQueueMappingByBroker != null) {\n            Map<String, TopicQueueMappingInfo> cloneMap = new HashMap<>(this.topicQueueMappingByBroker);\n            topicRouteData.setTopicQueueMappingByBroker(cloneMap);\n        }\n        return topicRouteData;\n    }\n\n    public TopicRouteData deepCloneTopicRouteData() {\n        TopicRouteData topicRouteData = new TopicRouteData();\n\n        topicRouteData.setOrderTopicConf(this.orderTopicConf);\n\n        for (final QueueData queueData : this.queueDatas) {\n            topicRouteData.getQueueDatas().add(new QueueData(queueData));\n        }\n\n        for (final BrokerData brokerData : this.brokerDatas) {\n            topicRouteData.getBrokerDatas().add(new BrokerData(brokerData));\n        }\n\n        for (final Map.Entry<String, List<String>> listEntry : this.filterServerTable.entrySet()) {\n            topicRouteData.getFilterServerTable().put(listEntry.getKey(),\n                new ArrayList<>(listEntry.getValue()));\n        }\n        if (this.topicQueueMappingByBroker != null) {\n            Map<String, TopicQueueMappingInfo> cloneMap = new HashMap<>(this.topicQueueMappingByBroker.size());\n            for (final Map.Entry<String, TopicQueueMappingInfo> entry : this.getTopicQueueMappingByBroker().entrySet()) {\n                TopicQueueMappingInfo topicQueueMappingInfo = new TopicQueueMappingInfo(entry.getValue().getTopic(), entry.getValue().getTotalQueues(), entry.getValue().getBname(), entry.getValue().getEpoch());\n                topicQueueMappingInfo.setDirty(entry.getValue().isDirty());\n                topicQueueMappingInfo.setScope(entry.getValue().getScope());\n                ConcurrentMap<Integer, Integer> concurrentMap = new ConcurrentHashMap<>(entry.getValue().getCurrIdMap());\n                topicQueueMappingInfo.setCurrIdMap(concurrentMap);\n                cloneMap.put(entry.getKey(), topicQueueMappingInfo);\n            }\n            topicRouteData.setTopicQueueMappingByBroker(cloneMap);\n        }\n\n        return topicRouteData;\n    }\n\n    public boolean topicRouteDataChanged(TopicRouteData oldData) {\n        if (oldData == null)\n            return true;\n        TopicRouteData old = new TopicRouteData(oldData);\n        TopicRouteData now = new TopicRouteData(this);\n        Collections.sort(old.getQueueDatas());\n        Collections.sort(old.getBrokerDatas());\n        Collections.sort(now.getQueueDatas());\n        Collections.sort(now.getBrokerDatas());\n        return !old.equals(now);\n    }\n\n    public List<QueueData> getQueueDatas() {\n        return queueDatas;\n    }\n\n    public void setQueueDatas(List<QueueData> queueDatas) {\n        this.queueDatas = queueDatas;\n    }\n\n    public List<BrokerData> getBrokerDatas() {\n        return brokerDatas;\n    }\n\n    public void setBrokerDatas(List<BrokerData> brokerDatas) {\n        this.brokerDatas = brokerDatas;\n    }\n\n    public HashMap<String, List<String>> getFilterServerTable() {\n        return filterServerTable;\n    }\n\n    public void setFilterServerTable(HashMap<String, List<String>> filterServerTable) {\n        this.filterServerTable = filterServerTable;\n    }\n\n    public String getOrderTopicConf() {\n        return orderTopicConf;\n    }\n\n    public void setOrderTopicConf(String orderTopicConf) {\n        this.orderTopicConf = orderTopicConf;\n    }\n\n    public Map<String, TopicQueueMappingInfo> getTopicQueueMappingByBroker() {\n        return topicQueueMappingByBroker;\n    }\n\n    public void setTopicQueueMappingByBroker(Map<String, TopicQueueMappingInfo> topicQueueMappingByBroker) {\n        this.topicQueueMappingByBroker = topicQueueMappingByBroker;\n    }\n\n    @Override\n    public int hashCode() {\n        final int prime = 31;\n        int result = 1;\n        result = prime * result + ((brokerDatas == null) ? 0 : brokerDatas.hashCode());\n        result = prime * result + ((orderTopicConf == null) ? 0 : orderTopicConf.hashCode());\n        result = prime * result + ((queueDatas == null) ? 0 : queueDatas.hashCode());\n        result = prime * result + ((filterServerTable == null) ? 0 : filterServerTable.hashCode());\n        result = prime * result + ((topicQueueMappingByBroker == null) ? 0 : topicQueueMappingByBroker.hashCode());\n        return result;\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj)\n            return true;\n        if (obj == null)\n            return false;\n        if (getClass() != obj.getClass())\n            return false;\n        TopicRouteData other = (TopicRouteData) obj;\n        if (brokerDatas == null) {\n            if (other.brokerDatas != null)\n                return false;\n        } else if (!brokerDatas.equals(other.brokerDatas))\n            return false;\n        if (orderTopicConf == null) {\n            if (other.orderTopicConf != null)\n                return false;\n        } else if (!orderTopicConf.equals(other.orderTopicConf))\n            return false;\n        if (queueDatas == null) {\n            if (other.queueDatas != null)\n                return false;\n        } else if (!queueDatas.equals(other.queueDatas))\n            return false;\n        if (filterServerTable == null) {\n            if (other.filterServerTable != null)\n                return false;\n        } else if (!filterServerTable.equals(other.filterServerTable))\n            return false;\n        if (topicQueueMappingByBroker == null) {\n            if (other.topicQueueMappingByBroker != null)\n                return false;\n        } else if (!topicQueueMappingByBroker.equals(other.topicQueueMappingByBroker))\n            return false;\n        return true;\n    }\n\n    @Override\n    public String toString() {\n        return \"TopicRouteData [orderTopicConf=\" + orderTopicConf + \", queueDatas=\" + queueDatas\n            + \", brokerDatas=\" + brokerDatas + \", filterServerTable=\" + filterServerTable + \", topicQueueMappingInfoTable=\" + topicQueueMappingByBroker + \"]\";\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/LogicQueueMappingItem.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.statictopic;\n\nimport org.apache.commons.lang3.builder.EqualsBuilder;\nimport org.apache.commons.lang3.builder.HashCodeBuilder;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class LogicQueueMappingItem extends RemotingSerializable {\n\n    private int gen; // immutable\n    private int queueId; //, immutable\n    private String bname; //important, immutable\n    private long logicOffset; // the start of the logic offset, important, can be changed by command only once\n    private long startOffset; // the start of the physical offset, should always be 0, immutable\n    private long endOffset = -1; // the end of the physical offset, excluded, revered -1, mutable\n    private long timeOfStart = -1; // mutable, reserved\n    private long timeOfEnd = -1; // mutable, reserved\n\n    //make sure it has a default constructor\n    public LogicQueueMappingItem() {\n\n    }\n\n    public LogicQueueMappingItem(int gen, int queueId, String bname, long logicOffset, long startOffset, long endOffset, long timeOfStart, long timeOfEnd) {\n        this.gen = gen;\n        this.queueId = queueId;\n        this.bname = bname;\n        this.logicOffset = logicOffset;\n        this.startOffset = startOffset;\n        this.endOffset = endOffset;\n        this.timeOfStart = timeOfStart;\n        this.timeOfEnd = timeOfEnd;\n    }\n\n\n    //should only be user in sendMessage and getMinOffset\n    public long computeStaticQueueOffsetLoosely(long physicalQueueOffset) {\n        //consider the newly mapped item\n        if (logicOffset < 0) {\n            return -1;\n        }\n        if (physicalQueueOffset < startOffset) {\n            return logicOffset;\n        }\n        if (endOffset >= startOffset\n            && endOffset < physicalQueueOffset) {\n            return logicOffset + (endOffset - startOffset);\n        }\n        return  logicOffset + (physicalQueueOffset - startOffset);\n    }\n\n    public long computeStaticQueueOffsetStrictly(long physicalQueueOffset) {\n        assert logicOffset >= 0;\n\n        if (physicalQueueOffset < startOffset) {\n            return logicOffset;\n        }\n        return  logicOffset + (physicalQueueOffset - startOffset);\n    }\n\n    public long computePhysicalQueueOffset(long staticQueueOffset) {\n        return  (staticQueueOffset - logicOffset) + startOffset;\n    }\n\n    public long computeMaxStaticQueueOffset() {\n        if (endOffset >= startOffset) {\n            return logicOffset + endOffset - startOffset;\n        } else {\n            return logicOffset;\n        }\n    }\n    public boolean checkIfEndOffsetDecided() {\n        //if the endOffset == startOffset, then the item should be deleted\n        return endOffset > startOffset;\n    }\n\n    public boolean checkIfLogicoffsetDecided() {\n        return logicOffset >= 0;\n    }\n\n    public long computeOffsetDelta() {\n        return logicOffset - startOffset;\n    }\n\n    public int getGen() {\n        return gen;\n    }\n\n    public int getQueueId() {\n        return queueId;\n    }\n\n    public String getBname() {\n        return bname;\n    }\n\n    public long getLogicOffset() {\n        return logicOffset;\n    }\n\n    public long getStartOffset() {\n        return startOffset;\n    }\n\n    public long getEndOffset() {\n        return endOffset;\n    }\n\n    public long getTimeOfStart() {\n        return timeOfStart;\n    }\n\n    public long getTimeOfEnd() {\n        return timeOfEnd;\n    }\n\n    public void setLogicOffset(long logicOffset) {\n        this.logicOffset = logicOffset;\n    }\n\n    public void setEndOffset(long endOffset) {\n        this.endOffset = endOffset;\n    }\n\n    public void setTimeOfStart(long timeOfStart) {\n        this.timeOfStart = timeOfStart;\n    }\n\n    public void setTimeOfEnd(long timeOfEnd) {\n        this.timeOfEnd = timeOfEnd;\n    }\n\n    public void setGen(int gen) {\n        this.gen = gen;\n    }\n\n    public void setQueueId(int queueId) {\n        this.queueId = queueId;\n    }\n\n    public void setBname(String bname) {\n        this.bname = bname;\n    }\n\n    public void setStartOffset(long startOffset) {\n        this.startOffset = startOffset;\n    }\n\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) return true;\n\n        if (!(o instanceof LogicQueueMappingItem)) return false;\n\n        LogicQueueMappingItem item = (LogicQueueMappingItem) o;\n\n        return new EqualsBuilder()\n                .append(gen, item.gen)\n                .append(queueId, item.queueId)\n                .append(logicOffset, item.logicOffset)\n                .append(startOffset, item.startOffset)\n                .append(endOffset, item.endOffset)\n                .append(timeOfStart, item.timeOfStart)\n                .append(timeOfEnd, item.timeOfEnd)\n                .append(bname, item.bname)\n                .isEquals();\n    }\n\n    @Override\n    public int hashCode() {\n        return new HashCodeBuilder(17, 37)\n                .append(gen)\n                .append(queueId)\n                .append(bname)\n                .append(logicOffset)\n                .append(startOffset)\n                .append(endOffset)\n                .append(timeOfStart)\n                .append(timeOfEnd)\n                .toHashCode();\n    }\n\n    @Override\n    public String toString() {\n        return \"LogicQueueMappingItem{\" +\n                \"gen=\" + gen +\n                \", queueId=\" + queueId +\n                \", bname='\" + bname + '\\'' +\n                \", logicOffset=\" + logicOffset +\n                \", startOffset=\" + startOffset +\n                \", endOffset=\" + endOffset +\n                \", timeOfStart=\" + timeOfStart +\n                \", timeOfEnd=\" + timeOfEnd +\n                '}';\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicConfigAndQueueMapping.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.statictopic;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.commons.lang3.builder.EqualsBuilder;\nimport org.apache.commons.lang3.builder.HashCodeBuilder;\nimport org.apache.rocketmq.common.TopicConfig;\n\npublic class TopicConfigAndQueueMapping extends TopicConfig {\n    private TopicQueueMappingDetail mappingDetail;\n\n    public TopicConfigAndQueueMapping() {\n    }\n\n    public TopicConfigAndQueueMapping(TopicConfig topicConfig, TopicQueueMappingDetail mappingDetail) {\n        super(topicConfig);\n        this.mappingDetail = mappingDetail;\n    }\n\n    public TopicQueueMappingDetail getMappingDetail() {\n        return mappingDetail;\n    }\n\n    public void setMappingDetail(TopicQueueMappingDetail mappingDetail) {\n        this.mappingDetail = mappingDetail;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) return true;\n\n        if (!(o instanceof TopicConfigAndQueueMapping)) return false;\n\n        TopicConfigAndQueueMapping that = (TopicConfigAndQueueMapping) o;\n\n        return new EqualsBuilder()\n                .appendSuper(super.equals(o))\n                .append(mappingDetail, that.mappingDetail)\n                .isEquals();\n    }\n\n    @Override\n    public int hashCode() {\n        return new HashCodeBuilder(17, 37)\n                .appendSuper(super.hashCode())\n                .append(mappingDetail)\n                .toHashCode();\n    }\n\n    @Override\n    public String toString() {\n        String string = super.toString();\n        if (StringUtils.isNotBlank(string)) {\n            string = string.substring(0, string.length() - 1) + \", mappingDetail=\" + mappingDetail + \"]\";\n        }\n        return string;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicQueueMappingContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.statictopic;\n\nimport com.google.common.collect.ImmutableList;\nimport java.util.List;\n\npublic class TopicQueueMappingContext  {\n    private String topic;\n    private Integer globalId;\n    private TopicQueueMappingDetail mappingDetail;\n    private List<LogicQueueMappingItem> mappingItemList;\n    private LogicQueueMappingItem leaderItem;\n\n    private LogicQueueMappingItem currentItem;\n\n    public TopicQueueMappingContext(String topic, Integer globalId, TopicQueueMappingDetail mappingDetail, List<LogicQueueMappingItem> mappingItemList, LogicQueueMappingItem leaderItem) {\n        this.topic = topic;\n        this.globalId = globalId;\n        this.mappingDetail = mappingDetail;\n        this.mappingItemList = mappingItemList;\n        this.leaderItem = leaderItem;\n\n    }\n\n\n    public boolean isLeader() {\n        return leaderItem != null && leaderItem.getBname().equals(mappingDetail.getBname());\n    }\n\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public Integer getGlobalId() {\n        return globalId;\n    }\n\n    public void setGlobalId(Integer globalId) {\n        this.globalId = globalId;\n    }\n\n\n    public TopicQueueMappingDetail getMappingDetail() {\n        return mappingDetail;\n    }\n\n    public void setMappingDetail(TopicQueueMappingDetail mappingDetail) {\n        this.mappingDetail = mappingDetail;\n    }\n\n    public List<LogicQueueMappingItem> getMappingItemList() {\n        return mappingItemList;\n    }\n\n    public void setMappingItemList(ImmutableList<LogicQueueMappingItem> mappingItemList) {\n        this.mappingItemList = mappingItemList;\n    }\n\n    public LogicQueueMappingItem getLeaderItem() {\n        return leaderItem;\n    }\n\n    public void setLeaderItem(LogicQueueMappingItem leaderItem) {\n        this.leaderItem = leaderItem;\n    }\n\n    public LogicQueueMappingItem getCurrentItem() {\n        return currentItem;\n    }\n\n    public void setCurrentItem(LogicQueueMappingItem currentItem) {\n        this.currentItem = currentItem;\n    }\n\n    public void setMappingItemList(List<LogicQueueMappingItem> mappingItemList) {\n        this.mappingItemList = mappingItemList;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicQueueMappingDetail.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.statictopic;\n\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.commons.lang3.builder.EqualsBuilder;\nimport org.apache.commons.lang3.builder.HashCodeBuilder;\n\npublic class TopicQueueMappingDetail extends TopicQueueMappingInfo {\n\n    // the mapping info in current broker, do not register to nameserver\n    // make sure this value is not null\n    private ConcurrentMap<Integer/*global id*/, List<LogicQueueMappingItem>> hostedQueues = new ConcurrentHashMap<>();\n\n    //make sure there is a default constructor\n    public TopicQueueMappingDetail() {\n\n    }\n\n    public TopicQueueMappingDetail(String topic, int totalQueues, String bname, long epoch) {\n        super(topic, totalQueues, bname, epoch);\n    }\n\n\n\n    public static boolean putMappingInfo(TopicQueueMappingDetail mappingDetail, Integer globalId, List<LogicQueueMappingItem> mappingInfo) {\n        if (mappingInfo.isEmpty()) {\n            return true;\n        }\n        mappingDetail.hostedQueues.put(globalId, mappingInfo);\n        return true;\n    }\n\n    public static List<LogicQueueMappingItem> getMappingInfo(TopicQueueMappingDetail mappingDetail, Integer globalId) {\n        return mappingDetail.hostedQueues.get(globalId);\n    }\n\n    public static ConcurrentMap<Integer, Integer> buildIdMap(TopicQueueMappingDetail mappingDetail, int level) {\n        //level 0 means current leader in this broker\n        //level 1 means previous leader in this broker, reserved for\n        assert level == LEVEL_0 ;\n\n        if (mappingDetail.hostedQueues == null || mappingDetail.hostedQueues.isEmpty()) {\n            return new ConcurrentHashMap<>();\n        }\n        ConcurrentMap<Integer, Integer> tmpIdMap = new ConcurrentHashMap<>();\n        for (Map.Entry<Integer, List<LogicQueueMappingItem>> entry: mappingDetail.hostedQueues.entrySet()) {\n            Integer globalId =  entry.getKey();\n            List<LogicQueueMappingItem> items = entry.getValue();\n            if (level == LEVEL_0\n                    && items.size() >= 1) {\n                LogicQueueMappingItem curr = items.get(items.size() - 1);\n                if (mappingDetail.bname.equals(curr.getBname())) {\n                    tmpIdMap.put(globalId, curr.getQueueId());\n                }\n            }\n        }\n        return tmpIdMap;\n    }\n\n\n    public static long computeMaxOffsetFromMapping(TopicQueueMappingDetail mappingDetail, Integer globalId) {\n        List<LogicQueueMappingItem> mappingItems = getMappingInfo(mappingDetail, globalId);\n        if (mappingItems == null\n                || mappingItems.isEmpty()) {\n            return -1;\n        }\n        LogicQueueMappingItem item =  mappingItems.get(mappingItems.size() - 1);\n        return item.computeMaxStaticQueueOffset();\n    }\n\n\n    public static TopicQueueMappingInfo cloneAsMappingInfo(TopicQueueMappingDetail mappingDetail) {\n        TopicQueueMappingInfo topicQueueMappingInfo = new TopicQueueMappingInfo(mappingDetail.topic, mappingDetail.totalQueues, mappingDetail.bname, mappingDetail.epoch);\n        topicQueueMappingInfo.currIdMap = TopicQueueMappingDetail.buildIdMap(mappingDetail, LEVEL_0);\n        return topicQueueMappingInfo;\n    }\n\n    public static boolean checkIfAsPhysical(TopicQueueMappingDetail mappingDetail, Integer globalId) {\n        List<LogicQueueMappingItem> mappingItems = getMappingInfo(mappingDetail, globalId);\n        return mappingItems == null\n                || mappingItems.size() == 1\n                &&  mappingItems.get(0).getLogicOffset() == 0;\n    }\n\n    public ConcurrentMap<Integer, List<LogicQueueMappingItem>> getHostedQueues() {\n        return hostedQueues;\n    }\n\n    public void setHostedQueues(ConcurrentMap<Integer, List<LogicQueueMappingItem>> hostedQueues) {\n        this.hostedQueues = hostedQueues;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) return true;\n\n        if (!(o instanceof TopicQueueMappingDetail)) return false;\n\n        TopicQueueMappingDetail that = (TopicQueueMappingDetail) o;\n\n        return new EqualsBuilder()\n                .append(hostedQueues, that.hostedQueues)\n                .isEquals();\n    }\n\n    @Override\n    public int hashCode() {\n        return new HashCodeBuilder(17, 37)\n                .append(hostedQueues)\n                .toHashCode();\n    }\n\n    @Override\n    public String toString() {\n        return \"TopicQueueMappingDetail{\" +\n                \"hostedQueues=\" + hostedQueues +\n                \", topic='\" + topic + '\\'' +\n                \", totalQueues=\" + totalQueues +\n                \", bname='\" + bname + '\\'' +\n                \", epoch=\" + epoch +\n                \", dirty=\" + dirty +\n                \", currIdMap=\" + currIdMap +\n                '}';\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicQueueMappingInfo.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.statictopic;\n\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class TopicQueueMappingInfo extends RemotingSerializable {\n    public static final int LEVEL_0 = 0;\n\n    String topic; // redundant field\n    String scope = MixAll.METADATA_SCOPE_GLOBAL;\n    int totalQueues;\n    String bname;  //identify the hosted broker name\n    long epoch; //important to fence the old dirty data\n    boolean dirty; //indicate if the data is dirty\n    //register to broker to construct the route\n    protected ConcurrentMap<Integer/*logicId*/, Integer/*physicalId*/> currIdMap = new ConcurrentHashMap<>();\n\n    public TopicQueueMappingInfo() {\n\n    }\n\n    public TopicQueueMappingInfo(String topic, int totalQueues, String bname, long epoch) {\n        this.topic = topic;\n        this.totalQueues = totalQueues;\n        this.bname = bname;\n        this.epoch = epoch;\n        this.dirty = false;\n    }\n\n    public boolean isDirty() {\n        return dirty;\n    }\n\n    public void setDirty(boolean dirty) {\n        this.dirty = dirty;\n    }\n\n    public int getTotalQueues() {\n        return totalQueues;\n    }\n\n\n    public String getBname() {\n        return bname;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public long getEpoch() {\n        return epoch;\n    }\n\n    public void setEpoch(long epoch) {\n        this.epoch = epoch;\n    }\n\n    public void setTotalQueues(int totalQueues) {\n        this.totalQueues = totalQueues;\n    }\n\n    public ConcurrentMap<Integer, Integer> getCurrIdMap() {\n        return currIdMap;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public void setBname(String bname) {\n        this.bname = bname;\n    }\n\n    public void setCurrIdMap(ConcurrentMap<Integer, Integer> currIdMap) {\n        this.currIdMap = currIdMap;\n    }\n\n    public String getScope() {\n        return scope;\n    }\n\n    public void setScope(String scope) {\n        this.scope = scope;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) return true;\n        if (!(o instanceof TopicQueueMappingInfo)) return false;\n\n        TopicQueueMappingInfo info = (TopicQueueMappingInfo) o;\n\n        if (totalQueues != info.totalQueues) return false;\n        if (epoch != info.epoch) return false;\n        if (dirty != info.dirty) return false;\n        if (topic != null ? !topic.equals(info.topic) : info.topic != null) return false;\n        if (scope != null ? !scope.equals(info.scope) : info.scope != null) return false;\n        if (bname != null ? !bname.equals(info.bname) : info.bname != null) return false;\n        return currIdMap != null ? currIdMap.equals(info.currIdMap) : info.currIdMap == null;\n    }\n\n    @Override\n    public int hashCode() {\n        int result = topic != null ? topic.hashCode() : 0;\n        result = 31 * result + (scope != null ? scope.hashCode() : 0);\n        result = 31 * result + totalQueues;\n        result = 31 * result + (bname != null ? bname.hashCode() : 0);\n        result = 31 * result + (int) (epoch ^ (epoch >>> 32));\n        result = 31 * result + (dirty ? 1 : 0);\n        result = 31 * result + (currIdMap != null ? currIdMap.hashCode() : 0);\n        return result;\n    }\n\n    @Override\n    public String toString() {\n        return \"TopicQueueMappingInfo{\" +\n                \"topic='\" + topic + '\\'' +\n                \", scope='\" + scope + '\\'' +\n                \", totalQueues=\" + totalQueues +\n                \", bname='\" + bname + '\\'' +\n                \", epoch=\" + epoch +\n                \", dirty=\" + dirty +\n                \", currIdMap=\" + currIdMap +\n                '}';\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicQueueMappingOne.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.statictopic;\n\nimport java.util.List;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class TopicQueueMappingOne extends RemotingSerializable {\n\n    String topic; // redundant field\n    String bname;  //identify the hosted broker name\n    Integer globalId;\n    List<LogicQueueMappingItem> items;\n    TopicQueueMappingDetail mappingDetail;\n\n    public TopicQueueMappingOne(TopicQueueMappingDetail mappingDetail, String topic, String bname, Integer globalId, List<LogicQueueMappingItem> items) {\n        this.mappingDetail =  mappingDetail;\n        this.topic = topic;\n        this.bname = bname;\n        this.globalId = globalId;\n        this.items = items;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public String getBname() {\n        return bname;\n    }\n\n    public Integer getGlobalId() {\n        return globalId;\n    }\n\n    public List<LogicQueueMappingItem> getItems() {\n        return items;\n    }\n\n    public TopicQueueMappingDetail getMappingDetail() {\n        return mappingDetail;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o)\n            return true;\n        if (!(o instanceof TopicQueueMappingOne))\n            return false;\n\n        TopicQueueMappingOne that = (TopicQueueMappingOne) o;\n\n        if (topic != null ? !topic.equals(that.topic) : that.topic != null)\n            return false;\n        if (bname != null ? !bname.equals(that.bname) : that.bname != null)\n            return false;\n        if (globalId != null ? !globalId.equals(that.globalId) : that.globalId != null)\n            return false;\n        if (items != null ? !items.equals(that.items) : that.items != null)\n            return false;\n        return mappingDetail != null ? mappingDetail.equals(that.mappingDetail) : that.mappingDetail == null;\n    }\n\n    @Override\n    public int hashCode() {\n        int result = topic != null ? topic.hashCode() : 0;\n        result = 31 * result + (bname != null ? bname.hashCode() : 0);\n        result = 31 * result + (globalId != null ? globalId.hashCode() : 0);\n        result = 31 * result + (items != null ? items.hashCode() : 0);\n        result = 31 * result + (mappingDetail != null ? mappingDetail.hashCode() : 0);\n        return result;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicQueueMappingUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.statictopic;\n\nimport java.io.File;\nimport java.util.AbstractMap;\nimport java.util.ArrayDeque;\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Queue;\nimport java.util.Set;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\n\npublic class TopicQueueMappingUtils {\n\n    public static final int DEFAULT_BLOCK_SEQ_SIZE = 10000;\n\n    public static class MappingAllocator {\n        Map<String, Integer> brokerNumMap = new HashMap<>();\n        Map<Integer, String> idToBroker = new HashMap<>();\n        //used for remapping\n        Map<String, Integer> brokerNumMapBeforeRemapping;\n        int currentIndex = 0;\n        List<String> leastBrokers = new ArrayList<>();\n        private MappingAllocator(Map<Integer, String> idToBroker, Map<String, Integer> brokerNumMap, Map<String, Integer> brokerNumMapBeforeRemapping) {\n            this.idToBroker.putAll(idToBroker);\n            this.brokerNumMap.putAll(brokerNumMap);\n            this.brokerNumMapBeforeRemapping = brokerNumMapBeforeRemapping;\n        }\n\n        private void freshState() {\n            int minNum = Integer.MAX_VALUE;\n            for (Map.Entry<String, Integer> entry : brokerNumMap.entrySet()) {\n                if (entry.getValue() < minNum) {\n                    leastBrokers.clear();\n                    leastBrokers.add(entry.getKey());\n                    minNum = entry.getValue();\n                } else if (entry.getValue() == minNum) {\n                    leastBrokers.add(entry.getKey());\n                }\n            }\n            //reduce the remapping\n            if (brokerNumMapBeforeRemapping != null\n                    && !brokerNumMapBeforeRemapping.isEmpty()) {\n                leastBrokers.sort((o1, o2) -> {\n                    int i1 = 0, i2 = 0;\n                    if (brokerNumMapBeforeRemapping.containsKey(o1)) {\n                        i1 = brokerNumMapBeforeRemapping.get(o1);\n                    }\n                    if (brokerNumMapBeforeRemapping.containsKey(o2)) {\n                        i2 = brokerNumMapBeforeRemapping.get(o2);\n                    }\n                    return i1 - i2;\n                });\n            } else {\n                //reduce the imbalance\n                Collections.shuffle(leastBrokers);\n            }\n            currentIndex = leastBrokers.size() - 1;\n        }\n        private String nextBroker() {\n            if (leastBrokers.isEmpty()) {\n                freshState();\n            }\n            int tmpIndex = currentIndex % leastBrokers.size();\n            return leastBrokers.remove(tmpIndex);\n        }\n\n        public Map<String, Integer> getBrokerNumMap() {\n            return brokerNumMap;\n        }\n\n        public void upToNum(int maxQueueNum) {\n            int currSize = idToBroker.size();\n            if (maxQueueNum <= currSize) {\n                return;\n            }\n            for (int i = currSize; i < maxQueueNum; i++) {\n                String nextBroker = nextBroker();\n                if (brokerNumMap.containsKey(nextBroker)) {\n                    brokerNumMap.put(nextBroker, brokerNumMap.get(nextBroker) + 1);\n                } else {\n                    brokerNumMap.put(nextBroker, 1);\n                }\n                idToBroker.put(i, nextBroker);\n            }\n        }\n\n        public Map<Integer, String> getIdToBroker() {\n            return idToBroker;\n        }\n    }\n\n\n    public static MappingAllocator buildMappingAllocator(Map<Integer, String> idToBroker, Map<String, Integer> brokerNumMap, Map<String, Integer> brokerNumMapBeforeRemapping) {\n        return new MappingAllocator(idToBroker, brokerNumMap, brokerNumMapBeforeRemapping);\n    }\n\n    public static Map.Entry<Long, Integer> findMaxEpochAndQueueNum(List<TopicQueueMappingDetail> mappingDetailList) {\n        long epoch = -1;\n        int queueNum = 0;\n        for (TopicQueueMappingDetail mappingDetail : mappingDetailList) {\n            if (mappingDetail.getEpoch() > epoch) {\n                epoch = mappingDetail.getEpoch();\n            }\n            if (mappingDetail.getTotalQueues() > queueNum) {\n                queueNum = mappingDetail.getTotalQueues();\n            }\n        }\n        return new AbstractMap.SimpleImmutableEntry<>(epoch, queueNum);\n    }\n\n    public static List<TopicQueueMappingDetail> getMappingDetailFromConfig(Collection<TopicConfigAndQueueMapping> configs) {\n        List<TopicQueueMappingDetail> detailList = new ArrayList<>();\n        for (TopicConfigAndQueueMapping configMapping : configs) {\n            if (configMapping.getMappingDetail() != null) {\n                detailList.add(configMapping.getMappingDetail());\n            }\n        }\n        return detailList;\n    }\n\n    public static Map.Entry<Long, Integer> checkNameEpochNumConsistence(String topic, Map<String, TopicConfigAndQueueMapping> brokerConfigMap) {\n        if (brokerConfigMap == null\n            || brokerConfigMap.isEmpty()) {\n            return null;\n        }\n        //make sure it is not null\n        long maxEpoch = -1;\n        int maxNum = -1;\n        String scope = null;\n        for (Map.Entry<String, TopicConfigAndQueueMapping> entry : brokerConfigMap.entrySet()) {\n            String broker = entry.getKey();\n            TopicConfigAndQueueMapping configMapping = entry.getValue();\n            if (configMapping.getMappingDetail() == null) {\n                throw new RuntimeException(\"Mapping info should not be null in broker \" + broker);\n            }\n            TopicQueueMappingDetail mappingDetail = configMapping.getMappingDetail();\n            if (!broker.equals(mappingDetail.getBname())) {\n                throw new RuntimeException(String.format(\"The broker name is not equal %s != %s \", broker, mappingDetail.getBname()));\n            }\n            if (mappingDetail.isDirty()) {\n                throw new RuntimeException(\"The mapping info is dirty in broker  \" + broker);\n            }\n            if (!configMapping.getTopicName().equals(mappingDetail.getTopic())) {\n                throw new RuntimeException(\"The topic name is inconsistent in broker  \" + broker);\n            }\n            if (topic != null\n                && !topic.equals(mappingDetail.getTopic())) {\n                throw new RuntimeException(\"The topic name is not match for broker  \" + broker);\n            }\n\n            if (scope != null\n                && !scope.equals(mappingDetail.getScope())) {\n                throw new RuntimeException(String.format(\"scope does not match %s != %s in %s\", mappingDetail.getScope(), scope, broker));\n            } else {\n                scope = mappingDetail.getScope();\n            }\n\n            if (maxEpoch != -1\n                && maxEpoch != mappingDetail.getEpoch()) {\n                throw new RuntimeException(String.format(\"epoch does not match %d != %d in %s\", maxEpoch, mappingDetail.getEpoch(), mappingDetail.getBname()));\n            } else {\n                maxEpoch = mappingDetail.getEpoch();\n            }\n\n            if (maxNum != -1\n                && maxNum != mappingDetail.getTotalQueues()) {\n                throw new RuntimeException(String.format(\"total queue number does not match %d != %d in %s\", maxNum, mappingDetail.getTotalQueues(), mappingDetail.getBname()));\n            } else {\n                maxNum = mappingDetail.getTotalQueues();\n            }\n        }\n        return new AbstractMap.SimpleEntry<>(maxEpoch, maxNum);\n    }\n\n    public static String getMockBrokerName(String scope) {\n        assert scope != null;\n        if (scope.equals(MixAll.METADATA_SCOPE_GLOBAL)) {\n            return MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX + scope.substring(2);\n        } else {\n            return MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX + scope;\n        }\n    }\n\n    public static void makeSureLogicQueueMappingItemImmutable(List<LogicQueueMappingItem> oldItems, List<LogicQueueMappingItem> newItems, boolean epochEqual, boolean isCLean) {\n        if (oldItems == null || oldItems.isEmpty()) {\n            return;\n        }\n        if (newItems == null || newItems.isEmpty()) {\n            throw new RuntimeException(\"The new item list is null or empty\");\n        }\n        int iold = 0, inew = 0;\n        while (iold < oldItems.size() && inew < newItems.size()) {\n            LogicQueueMappingItem newItem = newItems.get(inew);\n            LogicQueueMappingItem oldItem = oldItems.get(iold);\n            if (newItem.getGen() < oldItem.getGen()) {\n                //the earliest item may have been deleted concurrently\n                inew++;\n            } else if (oldItem.getGen() < newItem.getGen()) {\n                //in the following cases, the new item-list has fewer items than old item-list\n                //1. the queue is mapped back to a broker which hold the logic queue before\n                //2. The earliest item is deleted by  TopicQueueMappingCleanService\n                iold++;\n            } else {\n                assert oldItem.getBname().equals(newItem.getBname());\n                assert oldItem.getQueueId() == newItem.getQueueId();\n                assert oldItem.getStartOffset() == newItem.getStartOffset();\n                if (oldItem.getLogicOffset() != -1) {\n                    assert oldItem.getLogicOffset() == newItem.getLogicOffset();\n                }\n                iold++;\n                inew++;\n            }\n        }\n        if (epochEqual) {\n            LogicQueueMappingItem oldLeader = oldItems.get(oldItems.size() - 1);\n            LogicQueueMappingItem newLeader = newItems.get(newItems.size() - 1);\n            if (newLeader.getGen() != oldLeader.getGen()\n                || !newLeader.getBname().equals(oldLeader.getBname())\n                || newLeader.getQueueId() != oldLeader.getQueueId()\n                || newLeader.getStartOffset() != oldLeader.getStartOffset()) {\n                throw new RuntimeException(\"The new leader is different but epoch equal\");\n            }\n        }\n    }\n\n\n    public static void checkLogicQueueMappingItemOffset(List<LogicQueueMappingItem> items) {\n        if (items == null\n            || items.isEmpty()) {\n            return;\n        }\n        int lastGen = -1;\n        long lastOffset = -1;\n        for (int i = items.size() - 1; i >= 0 ; i--) {\n            LogicQueueMappingItem item = items.get(i);\n            if (item.getStartOffset() < 0\n                    || item.getGen() < 0\n                    || item.getQueueId() < 0) {\n                throw new RuntimeException(\"The field is illegal, should not be negative\");\n            }\n            if (items.size() >= 2\n                    && i <= items.size() - 2\n                    && items.get(i).getLogicOffset() < 0) {\n                throw new RuntimeException(\"The non-latest item has negative logic offset\");\n            }\n            if (lastGen != -1 && item.getGen() >= lastGen) {\n                throw new RuntimeException(\"The gen does not increase monotonically\");\n            }\n\n            if (item.getEndOffset() != -1\n                && item.getEndOffset() < item.getStartOffset()) {\n                throw new RuntimeException(\"The endOffset is smaller than the start offset\");\n            }\n\n            if (lastOffset != -1 && item.getLogicOffset() != -1) {\n                if (item.getLogicOffset() >= lastOffset) {\n                    throw new RuntimeException(\"The base logic offset does not increase monotonically\");\n                }\n                if (item.computeMaxStaticQueueOffset() >= lastOffset) {\n                    throw new RuntimeException(\"The max logic offset does not increase monotonically\");\n                }\n            }\n            lastGen = item.getGen();\n            lastOffset = item.getLogicOffset();\n        }\n    }\n\n    public static void  checkIfReusePhysicalQueue(Collection<TopicQueueMappingOne> mappingOnes) {\n        Map<String, TopicQueueMappingOne>  physicalQueueIdMap = new HashMap<>();\n        for (TopicQueueMappingOne mappingOne : mappingOnes) {\n            for (LogicQueueMappingItem item: mappingOne.items) {\n                String physicalQueueId = item.getBname() + \"-\" + item.getQueueId();\n                if (physicalQueueIdMap.containsKey(physicalQueueId)) {\n                    throw new RuntimeException(String.format(\"Topic %s global queue id %d and %d shared the same physical queue %s\",\n                            mappingOne.topic, mappingOne.globalId, physicalQueueIdMap.get(physicalQueueId).globalId, physicalQueueId));\n                } else {\n                    physicalQueueIdMap.put(physicalQueueId, mappingOne);\n                }\n            }\n        }\n    }\n\n    public static void  checkLeaderInTargetBrokers(Collection<TopicQueueMappingOne> mappingOnes, Set<String> targetBrokers) {\n        for (TopicQueueMappingOne mappingOne : mappingOnes) {\n            if (!targetBrokers.contains(mappingOne.bname)) {\n                throw new RuntimeException(\"The leader broker does not in target broker\");\n            }\n        }\n    }\n\n    public static void  checkPhysicalQueueConsistence(Map<String, TopicConfigAndQueueMapping> brokerConfigMap) {\n        for (Map.Entry<String, TopicConfigAndQueueMapping> entry : brokerConfigMap.entrySet()) {\n            TopicConfigAndQueueMapping configMapping = entry.getValue();\n            assert configMapping != null;\n            assert configMapping.getMappingDetail() != null;\n            if (configMapping.getReadQueueNums() < configMapping.getWriteQueueNums()) {\n                throw new RuntimeException(\"Read queues is smaller than write queues\");\n            }\n            for (List<LogicQueueMappingItem> items: configMapping.getMappingDetail().getHostedQueues().values()) {\n                for (LogicQueueMappingItem item: items) {\n                    if (item.getStartOffset() != 0) {\n                        throw new RuntimeException(\"The start offset does not begin from 0\");\n                    }\n                    TopicConfig topicConfig = brokerConfigMap.get(item.getBname());\n                    if (topicConfig == null) {\n                        throw new RuntimeException(\"The broker of item does not exist\");\n                    }\n                    if (item.getQueueId() >= topicConfig.getWriteQueueNums()) {\n                        throw new RuntimeException(\"The physical queue id is overflow the write queues\");\n                    }\n                }\n            }\n        }\n    }\n\n\n\n    public static Map<Integer, TopicQueueMappingOne> checkAndBuildMappingItems(List<TopicQueueMappingDetail> mappingDetailList, boolean replace, boolean checkConsistence) {\n        mappingDetailList.sort((o1, o2) -> (int) (o2.getEpoch() - o1.getEpoch()));\n\n        int maxNum = 0;\n        Map<Integer, TopicQueueMappingOne> globalIdMap = new HashMap<>();\n        for (TopicQueueMappingDetail mappingDetail : mappingDetailList) {\n            if (mappingDetail.totalQueues > maxNum) {\n                maxNum = mappingDetail.totalQueues;\n            }\n            for (Map.Entry<Integer, List<LogicQueueMappingItem>>  entry : mappingDetail.getHostedQueues().entrySet()) {\n                Integer globalid = entry.getKey();\n                checkLogicQueueMappingItemOffset(entry.getValue());\n                String leaderBrokerName  = getLeaderBroker(entry.getValue());\n                if (!leaderBrokerName.equals(mappingDetail.getBname())) {\n                    //not the leader\n                    continue;\n                }\n                if (globalIdMap.containsKey(globalid)) {\n                    if (!replace) {\n                        throw new RuntimeException(String.format(\"The queue id is duplicated in broker %s %s\", leaderBrokerName, mappingDetail.getBname()));\n                    }\n                } else {\n                    globalIdMap.put(globalid, new TopicQueueMappingOne(mappingDetail, mappingDetail.topic, mappingDetail.bname, globalid, entry.getValue()));\n                }\n            }\n        }\n        if (checkConsistence) {\n            if (maxNum != globalIdMap.size()) {\n                throw new RuntimeException(String.format(\"The total queue number in config does not match the real hosted queues %d != %d\", maxNum, globalIdMap.size()));\n            }\n            for (int i = 0; i < maxNum; i++) {\n                if (!globalIdMap.containsKey(i)) {\n                    throw new RuntimeException(String.format(\"The queue number %s is not in globalIdMap\", i));\n                }\n            }\n        }\n        checkIfReusePhysicalQueue(globalIdMap.values());\n        return globalIdMap;\n    }\n\n    public static String getLeaderBroker(List<LogicQueueMappingItem> items) {\n        return getLeaderItem(items).getBname();\n    }\n    public static LogicQueueMappingItem getLeaderItem(List<LogicQueueMappingItem> items) {\n        assert items.size() > 0;\n        return items.get(items.size() - 1);\n    }\n\n    public static String writeToTemp(TopicRemappingDetailWrapper wrapper, boolean after) {\n        String topic = wrapper.getTopic();\n        String data = wrapper.toJson();\n        String suffix = TopicRemappingDetailWrapper.SUFFIX_BEFORE;\n        if (after) {\n            suffix = TopicRemappingDetailWrapper.SUFFIX_AFTER;\n        }\n        String fileName = System.getProperty(\"java.io.tmpdir\") + File.separator + topic + \"-\" + wrapper.getEpoch() + suffix;\n        try {\n            MixAll.string2File(data, fileName);\n            return fileName;\n        } catch (Exception e) {\n            throw new RuntimeException(\"write file failed \" + fileName,e);\n        }\n    }\n\n    public static long blockSeqRoundUp(long offset, long blockSeqSize) {\n        long num = offset / blockSeqSize;\n        long left = offset % blockSeqSize;\n        if (left < blockSeqSize / 2) {\n            return (num + 1) * blockSeqSize;\n        } else {\n            return (num + 2) * blockSeqSize;\n        }\n    }\n\n    public static void checkTargetBrokersComplete(Set<String> targetBrokers, Map<String, TopicConfigAndQueueMapping> brokerConfigMap) {\n        for (String broker : brokerConfigMap.keySet()) {\n            if (brokerConfigMap.get(broker).getMappingDetail().getHostedQueues().isEmpty()) {\n                continue;\n            }\n            if (!targetBrokers.contains(broker)) {\n                throw new RuntimeException(\"The existed broker \" + broker + \" does not in target brokers \");\n            }\n        }\n    }\n\n    public static void checkNonTargetBrokers(Set<String> targetBrokers, Set<String> nonTargetBrokers) {\n        for (String broker : nonTargetBrokers) {\n            if (targetBrokers.contains(broker)) {\n                throw new RuntimeException(\"The non-target broker exist in target broker\");\n            }\n        }\n    }\n\n    public static TopicRemappingDetailWrapper createTopicConfigMapping(String topic, int queueNum, Set<String> targetBrokers, Map<String, TopicConfigAndQueueMapping> brokerConfigMap) {\n        checkTargetBrokersComplete(targetBrokers, brokerConfigMap);\n        Map<Integer, TopicQueueMappingOne> globalIdMap = new HashMap<>();\n        Map.Entry<Long, Integer> maxEpochAndNum = new AbstractMap.SimpleImmutableEntry<>(System.currentTimeMillis(), queueNum);\n        if (!brokerConfigMap.isEmpty()) {\n            maxEpochAndNum = TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap);\n            globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(TopicQueueMappingUtils.getMappingDetailFromConfig(brokerConfigMap.values())), false, true);\n            checkIfReusePhysicalQueue(globalIdMap.values());\n            checkPhysicalQueueConsistence(brokerConfigMap);\n        }\n        if (queueNum < globalIdMap.size()) {\n            throw new RuntimeException(String.format(\"Cannot decrease the queue num for static topic %d < %d\", queueNum, globalIdMap.size()));\n        }\n        //check the queue number\n        if (queueNum == globalIdMap.size()) {\n            throw new RuntimeException(\"The topic queue num is equal the existed queue num, do nothing\");\n        }\n\n        //the check is ok, now do the mapping allocation\n        Map<String, Integer> brokerNumMap = new HashMap<>();\n        for (String broker: targetBrokers) {\n            brokerNumMap.put(broker, 0);\n        }\n        final Map<Integer, String> oldIdToBroker = new HashMap<>();\n        for (Map.Entry<Integer, TopicQueueMappingOne> entry : globalIdMap.entrySet()) {\n            String leaderbroker = entry.getValue().getBname();\n            oldIdToBroker.put(entry.getKey(), leaderbroker);\n            if (!brokerNumMap.containsKey(leaderbroker)) {\n                brokerNumMap.put(leaderbroker, 1);\n            } else {\n                brokerNumMap.put(leaderbroker, brokerNumMap.get(leaderbroker) + 1);\n            }\n        }\n        TopicQueueMappingUtils.MappingAllocator allocator = TopicQueueMappingUtils.buildMappingAllocator(oldIdToBroker, brokerNumMap, null);\n        allocator.upToNum(queueNum);\n        Map<Integer, String> newIdToBroker = allocator.getIdToBroker();\n\n        //construct the topic configAndMapping\n        long newEpoch = Math.max(maxEpochAndNum.getKey() + 1000, System.currentTimeMillis());\n        for (Map.Entry<Integer, String> e : newIdToBroker.entrySet()) {\n            Integer queueId = e.getKey();\n            String broker = e.getValue();\n            if (globalIdMap.containsKey(queueId)) {\n                //ignore the exited\n                continue;\n            }\n            TopicConfigAndQueueMapping configMapping;\n            if (!brokerConfigMap.containsKey(broker)) {\n                configMapping = new TopicConfigAndQueueMapping(new TopicConfig(topic), new TopicQueueMappingDetail(topic, 0, broker, System.currentTimeMillis()));\n                configMapping.setWriteQueueNums(1);\n                configMapping.setReadQueueNums(1);\n                brokerConfigMap.put(broker, configMapping);\n            } else {\n                configMapping = brokerConfigMap.get(broker);\n                configMapping.setWriteQueueNums(configMapping.getWriteQueueNums() + 1);\n                configMapping.setReadQueueNums(configMapping.getReadQueueNums() + 1);\n            }\n            LogicQueueMappingItem mappingItem = new LogicQueueMappingItem(0, configMapping.getWriteQueueNums() - 1, broker, 0, 0, -1, -1, -1);\n            TopicQueueMappingDetail.putMappingInfo(configMapping.getMappingDetail(), queueId, new ArrayList<>(Collections.singletonList(mappingItem)));\n        }\n\n        // set the topic config\n        for (Map.Entry<String, TopicConfigAndQueueMapping> entry : brokerConfigMap.entrySet()) {\n            TopicConfigAndQueueMapping configMapping = entry.getValue();\n            configMapping.getMappingDetail().setEpoch(newEpoch);\n            configMapping.getMappingDetail().setTotalQueues(queueNum);\n        }\n        //double check the config\n        {\n            TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap);\n            globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(getMappingDetailFromConfig(brokerConfigMap.values()), false, true);\n            checkIfReusePhysicalQueue(globalIdMap.values());\n            checkPhysicalQueueConsistence(brokerConfigMap);\n        }\n        return new TopicRemappingDetailWrapper(topic, TopicRemappingDetailWrapper.TYPE_CREATE_OR_UPDATE, newEpoch, brokerConfigMap, new HashSet<>(), new HashSet<>());\n    }\n\n\n    public static TopicRemappingDetailWrapper remappingStaticTopic(String topic, Map<String, TopicConfigAndQueueMapping> brokerConfigMap, Set<String> targetBrokers) {\n        Map.Entry<Long, Integer> maxEpochAndNum = TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap);\n        Map<Integer, TopicQueueMappingOne> globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(getMappingDetailFromConfig(brokerConfigMap.values()), false, true);\n        TopicQueueMappingUtils.checkPhysicalQueueConsistence(brokerConfigMap);\n        TopicQueueMappingUtils.checkIfReusePhysicalQueue(globalIdMap.values());\n\n        //the check is ok, now do the mapping allocation\n        int maxNum = maxEpochAndNum.getValue();\n\n        Map<String, Integer> brokerNumMap = new HashMap<>();\n        for (String broker: targetBrokers) {\n            brokerNumMap.put(broker, 0);\n        }\n        Map<String, Integer> brokerNumMapBeforeRemapping = new HashMap<>();\n        for (TopicQueueMappingOne mappingOne: globalIdMap.values()) {\n            if (brokerNumMapBeforeRemapping.containsKey(mappingOne.bname)) {\n                brokerNumMapBeforeRemapping.put(mappingOne.bname, brokerNumMapBeforeRemapping.get(mappingOne.bname) + 1);\n            } else {\n                brokerNumMapBeforeRemapping.put(mappingOne.bname, 1);\n            }\n        }\n\n        TopicQueueMappingUtils.MappingAllocator allocator = TopicQueueMappingUtils.buildMappingAllocator(new HashMap<>(), brokerNumMap, brokerNumMapBeforeRemapping);\n        allocator.upToNum(maxNum);\n        Map<String, Integer> expectedBrokerNumMap = allocator.getBrokerNumMap();\n        Queue<Integer> waitAssignQueues = new ArrayDeque<>();\n        //cannot directly use the idBrokerMap from allocator, for the number of globalId maybe not in the natural order\n        Map<Integer, String> expectedIdToBroker = new HashMap<>();\n        //the following logic will make sure that, for one broker, either \"map in\" or \"map out\"\n        //It can't both,  map in some queues but also map out some queues.\n        for (Map.Entry<Integer, TopicQueueMappingOne> entry : globalIdMap.entrySet()) {\n            Integer queueId = entry.getKey();\n            TopicQueueMappingOne mappingOne = entry.getValue();\n            String leaderBroker = mappingOne.getBname();\n            if (expectedBrokerNumMap.containsKey(leaderBroker)) {\n                if (expectedBrokerNumMap.get(leaderBroker) > 0) {\n                    expectedIdToBroker.put(queueId, leaderBroker);\n                    expectedBrokerNumMap.put(leaderBroker, expectedBrokerNumMap.get(leaderBroker) - 1);\n                } else {\n                    waitAssignQueues.add(queueId);\n                    expectedBrokerNumMap.remove(leaderBroker);\n                }\n            } else {\n                waitAssignQueues.add(queueId);\n            }\n        }\n\n        for (Map.Entry<String, Integer> entry: expectedBrokerNumMap.entrySet()) {\n            String broker = entry.getKey();\n            Integer queueNum = entry.getValue();\n            for (int i = 0; i < queueNum; i++) {\n                Integer queueId = waitAssignQueues.poll();\n                assert queueId != null;\n                expectedIdToBroker.put(queueId, broker);\n            }\n        }\n        long newEpoch = Math.max(maxEpochAndNum.getKey() + 1000, System.currentTimeMillis());\n\n        //Now construct the remapping info\n        Set<String> brokersToMapOut = new HashSet<>();\n        Set<String> brokersToMapIn = new HashSet<>();\n        for (Map.Entry<Integer, String> mapEntry : expectedIdToBroker.entrySet()) {\n            Integer queueId = mapEntry.getKey();\n            String broker = mapEntry.getValue();\n            TopicQueueMappingOne topicQueueMappingOne = globalIdMap.get(queueId);\n            assert topicQueueMappingOne != null;\n            if (topicQueueMappingOne.getBname().equals(broker)) {\n                continue;\n            }\n            //remapping\n            final String mapInBroker = broker;\n            final String mapOutBroker = topicQueueMappingOne.getBname();\n            brokersToMapIn.add(mapInBroker);\n            brokersToMapOut.add(mapOutBroker);\n            TopicConfigAndQueueMapping mapInConfig = brokerConfigMap.get(mapInBroker);\n            TopicConfigAndQueueMapping mapOutConfig = brokerConfigMap.get(mapOutBroker);\n\n            if (mapInConfig == null) {\n                mapInConfig = new TopicConfigAndQueueMapping(new TopicConfig(topic, 0, 0), new TopicQueueMappingDetail(topic, maxNum, mapInBroker, newEpoch));\n                brokerConfigMap.put(mapInBroker, mapInConfig);\n            }\n\n            mapInConfig.setWriteQueueNums(mapInConfig.getWriteQueueNums() + 1);\n            mapInConfig.setReadQueueNums(mapInConfig.getReadQueueNums() + 1);\n\n            List<LogicQueueMappingItem> items = new ArrayList<>(topicQueueMappingOne.getItems());\n            LogicQueueMappingItem last = items.get(items.size() - 1);\n            items.add(new LogicQueueMappingItem(last.getGen() + 1, mapInConfig.getWriteQueueNums() - 1, mapInBroker, -1, 0, -1, -1, -1));\n\n            //Use the same object\n            TopicQueueMappingDetail.putMappingInfo(mapInConfig.getMappingDetail(), queueId, items);\n            TopicQueueMappingDetail.putMappingInfo(mapOutConfig.getMappingDetail(), queueId, items);\n        }\n\n        for (Map.Entry<String, TopicConfigAndQueueMapping> entry : brokerConfigMap.entrySet()) {\n            TopicConfigAndQueueMapping configMapping = entry.getValue();\n            configMapping.getMappingDetail().setEpoch(newEpoch);\n            configMapping.getMappingDetail().setTotalQueues(maxNum);\n        }\n\n        //double check\n        {\n            TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap);\n            globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(getMappingDetailFromConfig(brokerConfigMap.values()), false, true);\n            TopicQueueMappingUtils.checkPhysicalQueueConsistence(brokerConfigMap);\n            TopicQueueMappingUtils.checkIfReusePhysicalQueue(globalIdMap.values());\n            TopicQueueMappingUtils.checkLeaderInTargetBrokers(globalIdMap.values(), targetBrokers);\n        }\n        return new TopicRemappingDetailWrapper(topic, TopicRemappingDetailWrapper.TYPE_REMAPPING, newEpoch, brokerConfigMap, brokersToMapIn, brokersToMapOut);\n    }\n\n    public static LogicQueueMappingItem findLogicQueueMappingItem(List<LogicQueueMappingItem> mappingItems, long logicOffset, boolean ignoreNegative) {\n        if (mappingItems == null\n                || mappingItems.isEmpty()) {\n            return null;\n        }\n        //Could use bi-search to polish performance\n        for (int i = mappingItems.size() - 1; i >= 0; i--) {\n            LogicQueueMappingItem item =  mappingItems.get(i);\n            if (ignoreNegative && item.getLogicOffset() < 0) {\n                continue;\n            }\n            if (logicOffset >= item.getLogicOffset()) {\n                return item;\n            }\n        }\n        //if not found, maybe out of range, return the first one\n        for (int i = 0; i < mappingItems.size(); i++) {\n            LogicQueueMappingItem item =  mappingItems.get(i);\n            if (ignoreNegative && item.getLogicOffset() < 0) {\n                continue;\n            } else {\n                return item;\n            }\n        }\n        return null;\n    }\n\n    public static LogicQueueMappingItem findNext(List<LogicQueueMappingItem> items, LogicQueueMappingItem currentItem, boolean ignoreNegative) {\n        if (items == null\n            || currentItem == null) {\n            return null;\n        }\n        for (int i = 0; i < items.size(); i++) {\n            LogicQueueMappingItem item = items.get(i);\n            if (ignoreNegative && item.getLogicOffset() < 0) {\n                continue;\n            }\n            if (item.getGen() == currentItem.getGen()) {\n                if (i < items.size() - 1) {\n                    item = items.get(i  + 1);\n                    if (ignoreNegative && item.getLogicOffset() < 0) {\n                        return null;\n                    } else {\n                        return item;\n                    }\n                } else {\n                    return null;\n                }\n            }\n        }\n        return null;\n    }\n\n\n    public static boolean checkIfLeader(List<LogicQueueMappingItem> items, TopicQueueMappingDetail mappingDetail) {\n        if (items == null\n            || mappingDetail == null\n            || items.isEmpty()) {\n            return false;\n        }\n        return items.get(items.size() - 1).getBname().equals(mappingDetail.getBname());\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicRemappingDetailWrapper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.statictopic;\n\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Map;\nimport java.util.Set;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class TopicRemappingDetailWrapper extends RemotingSerializable {\n    public static final String TYPE_CREATE_OR_UPDATE = \"CREATE_OR_UPDATE\";\n    public static final String TYPE_REMAPPING = \"REMAPPING\";\n\n    public static final String SUFFIX_BEFORE = \".before\";\n    public static final String SUFFIX_AFTER = \".after\";\n\n\n    private String topic;\n    private String type;\n    private long epoch;\n\n    private Map<String, TopicConfigAndQueueMapping> brokerConfigMap = new HashMap<>();\n\n    private Set<String> brokerToMapIn = new HashSet<>();\n\n    private Set<String> brokerToMapOut = new HashSet<>();\n\n    public TopicRemappingDetailWrapper() {\n\n    }\n\n    public TopicRemappingDetailWrapper(String topic, String type, long epoch, Map<String, TopicConfigAndQueueMapping> brokerConfigMap, Set<String> brokerToMapIn, Set<String> brokerToMapOut) {\n        this.topic = topic;\n        this.type = type;\n        this.epoch = epoch;\n        this.brokerConfigMap = brokerConfigMap;\n        this.brokerToMapIn = brokerToMapIn;\n        this.brokerToMapOut = brokerToMapOut;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public String getType() {\n        return type;\n    }\n\n    public long getEpoch() {\n        return epoch;\n    }\n\n    public Map<String, TopicConfigAndQueueMapping> getBrokerConfigMap() {\n        return brokerConfigMap;\n    }\n\n    public Set<String> getBrokerToMapIn() {\n        return brokerToMapIn;\n    }\n\n    public Set<String> getBrokerToMapOut() {\n        return brokerToMapOut;\n    }\n\n    public void setBrokerConfigMap(Map<String, TopicConfigAndQueueMapping> brokerConfigMap) {\n        this.brokerConfigMap = brokerConfigMap;\n    }\n\n    public void setBrokerToMapIn(Set<String> brokerToMapIn) {\n        this.brokerToMapIn = brokerToMapIn;\n    }\n\n    public void setBrokerToMapOut(Set<String> brokerToMapOut) {\n        this.brokerToMapOut = brokerToMapOut;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public void setType(String type) {\n        this.type = type;\n    }\n\n    public void setEpoch(long epoch) {\n        this.epoch = epoch;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/subscription/CustomizedRetryPolicy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.subscription;\n\nimport com.google.common.base.MoreObjects;\nimport java.util.concurrent.TimeUnit;\n\n/**\n * CustomizedRetryPolicy is aim to make group's behavior compatible with messageDelayLevel\n *\n * @see <a href=\"https://github.com/apache/rocketmq/blob/3bd4b2b2f61a824196f19b03146e2c929c62777b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java#L137\">org.apache.rocketmq.store.config.MessageStoreConfig</a>\n */\npublic class CustomizedRetryPolicy implements RetryPolicy {\n    // 1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h\n    private long[] next = new long[] {\n        TimeUnit.SECONDS.toMillis(1),\n        TimeUnit.SECONDS.toMillis(5),\n        TimeUnit.SECONDS.toMillis(10),\n        TimeUnit.SECONDS.toMillis(30),\n        TimeUnit.MINUTES.toMillis(1),\n        TimeUnit.MINUTES.toMillis(2),\n        TimeUnit.MINUTES.toMillis(3),\n        TimeUnit.MINUTES.toMillis(4),\n        TimeUnit.MINUTES.toMillis(5),\n        TimeUnit.MINUTES.toMillis(6),\n        TimeUnit.MINUTES.toMillis(7),\n        TimeUnit.MINUTES.toMillis(8),\n        TimeUnit.MINUTES.toMillis(9),\n        TimeUnit.MINUTES.toMillis(10),\n        TimeUnit.MINUTES.toMillis(20),\n        TimeUnit.MINUTES.toMillis(30),\n        TimeUnit.HOURS.toMillis(1),\n        TimeUnit.HOURS.toMillis(2)\n    };\n\n    public CustomizedRetryPolicy() {\n    }\n\n    public CustomizedRetryPolicy(long[] next) {\n        this.next = next;\n    }\n\n    public long[] getNext() {\n        return next;\n    }\n\n    public void setNext(long[] next) {\n        this.next = next;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"next\", next)\n            .toString();\n    }\n\n    /**\n     * Index = reconsumeTimes + 2 is compatible logic, cause old delayLevelTable starts from index 1,\n     * and old index is reconsumeTime + 3\n     *\n     * @param reconsumeTimes Message reconsumeTimes {@link org.apache.rocketmq.common.message.MessageExt#getReconsumeTimes}\n     * @see <a href=\"https://github.com/apache/rocketmq/blob/3bddd514646826253a239f95959c14840a87034a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java#L210\">org.apache.rocketmq.broker.processor.AbstractSendMessageProcessor</a>\n     * @see <a href=\"https://github.com/apache/rocketmq/blob/3bddd514646826253a239f95959c14840a87034a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java#L242\">org.apache.rocketmq.store.DefaultMessageStore</a>\n     */\n    @Override\n    public long nextDelayDuration(int reconsumeTimes) {\n        if (reconsumeTimes < 0) {\n            reconsumeTimes = 0;\n        }\n        int index = reconsumeTimes + 2;\n        if (index >= next.length) {\n            index = next.length - 1;\n        }\n        return next[index];\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/subscription/ExponentialRetryPolicy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.subscription;\n\nimport com.google.common.base.MoreObjects;\nimport java.util.concurrent.TimeUnit;\n\npublic class ExponentialRetryPolicy implements RetryPolicy {\n    private long initial = TimeUnit.SECONDS.toMillis(5);\n    private long max = TimeUnit.HOURS.toMillis(2);\n    private long multiplier = 2;\n\n    public ExponentialRetryPolicy() {\n    }\n\n    public ExponentialRetryPolicy(long initial, long max, long multiplier) {\n        this.initial = initial;\n        this.max = max;\n        this.multiplier = multiplier;\n    }\n\n    public long getInitial() {\n        return initial;\n    }\n\n    public void setInitial(long initial) {\n        this.initial = initial;\n    }\n\n    public long getMax() {\n        return max;\n    }\n\n    public void setMax(long max) {\n        this.max = max;\n    }\n\n    public long getMultiplier() {\n        return multiplier;\n    }\n\n    public void setMultiplier(long multiplier) {\n        this.multiplier = multiplier;\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"initial\", initial)\n            .add(\"max\", max)\n            .add(\"multiplier\", multiplier)\n            .toString();\n    }\n\n    @Override\n    public long nextDelayDuration(int reconsumeTimes) {\n        if (reconsumeTimes < 0) {\n            reconsumeTimes = 0;\n        }\n        if (reconsumeTimes > 32) {\n            reconsumeTimes = 32;\n        }\n        return Math.min(max, initial * (long) Math.pow(multiplier, reconsumeTimes));\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/subscription/GroupForbidden.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.subscription;\n\nimport org.apache.commons.lang3.builder.EqualsBuilder;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\n/**\n *\n */\npublic class GroupForbidden extends RemotingSerializable {\n\n    private String  topic;\n    private String  group;\n    private Boolean readable;\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getGroup() {\n        return group;\n    }\n\n    public void setGroup(String group) {\n        this.group = group;\n    }\n\n    public Boolean getReadable() {\n        return readable;\n    }\n\n    public void setReadable(Boolean readable) {\n        this.readable = readable;\n    }\n\n    @Override\n    public int hashCode() {\n        final int prime = 31;\n        int result = 1;\n        result = prime * result + ((group == null) ? 0 : group.hashCode());\n        result = prime * result + ((readable == null) ? 0 : readable.hashCode());\n        result = prime * result + ((topic == null) ? 0 : topic.hashCode());\n        return result;\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj)\n            return true;\n        if (obj == null)\n            return false;\n        if (getClass() != obj.getClass())\n            return false;\n        GroupForbidden other = (GroupForbidden) obj;\n        return new EqualsBuilder()\n                .append(topic, other.topic)\n                .append(group, other.group)\n                .append(readable, other.readable)\n                .isEquals();\n    }\n\n    @Override\n    public String toString() {\n        return \"GroupForbidden [topic=\" + topic + \", group=\" + group + \", readable=\" + readable + \"]\";\n    }\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/subscription/GroupRetryPolicy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.subscription;\n\nimport com.alibaba.fastjson2.annotation.JSONField;\nimport com.google.common.base.MoreObjects;\n\npublic class GroupRetryPolicy {\n    private final static RetryPolicy DEFAULT_RETRY_POLICY = new CustomizedRetryPolicy();\n    private GroupRetryPolicyType type = GroupRetryPolicyType.CUSTOMIZED;\n    private ExponentialRetryPolicy exponentialRetryPolicy;\n    private CustomizedRetryPolicy customizedRetryPolicy;\n\n    public GroupRetryPolicyType getType() {\n        return type;\n    }\n\n    public void setType(GroupRetryPolicyType type) {\n        this.type = type;\n    }\n\n    public ExponentialRetryPolicy getExponentialRetryPolicy() {\n        return exponentialRetryPolicy;\n    }\n\n    public void setExponentialRetryPolicy(ExponentialRetryPolicy exponentialRetryPolicy) {\n        this.exponentialRetryPolicy = exponentialRetryPolicy;\n    }\n\n    public CustomizedRetryPolicy getCustomizedRetryPolicy() {\n        return customizedRetryPolicy;\n    }\n\n    public void setCustomizedRetryPolicy(CustomizedRetryPolicy customizedRetryPolicy) {\n        this.customizedRetryPolicy = customizedRetryPolicy;\n    }\n\n    @JSONField(serialize = false, deserialize = false)\n    public RetryPolicy getRetryPolicy() {\n        if (GroupRetryPolicyType.EXPONENTIAL.equals(type)) {\n            if (exponentialRetryPolicy == null) {\n                return DEFAULT_RETRY_POLICY;\n            }\n            return exponentialRetryPolicy;\n        } else if (GroupRetryPolicyType.CUSTOMIZED.equals(type)) {\n            if (customizedRetryPolicy == null) {\n                return DEFAULT_RETRY_POLICY;\n            }\n            return customizedRetryPolicy;\n        } else {\n            return DEFAULT_RETRY_POLICY;\n        }\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"type\", type)\n            .add(\"exponentialRetryPolicy\", exponentialRetryPolicy)\n            .add(\"customizedRetryPolicy\", customizedRetryPolicy)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/subscription/GroupRetryPolicyType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.subscription;\n\npublic enum GroupRetryPolicyType {\n    EXPONENTIAL,\n    CUSTOMIZED\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/subscription/RetryPolicy.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.subscription;\n\npublic interface RetryPolicy {\n    /**\n     * Compute message's next delay duration by specify reconsumeTimes\n     *\n     * @param reconsumeTimes Message reconsumeTimes\n     * @return Message's nextDelayDuration in milliseconds\n     */\n    long nextDelayDuration(int reconsumeTimes);\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/subscription/SimpleSubscriptionData.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.subscription;\n\nimport com.google.common.base.MoreObjects;\nimport java.util.Objects;\n\npublic class SimpleSubscriptionData {\n    private String topic;\n    private String expressionType;\n    private String expression;\n    private long version;\n\n    public SimpleSubscriptionData(String topic, String expressionType, String expression, long version) {\n        this.topic = topic;\n        this.expressionType = expressionType;\n        this.expression = expression;\n        this.version = version;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getExpressionType() {\n        return expressionType;\n    }\n\n    public void setExpressionType(String expressionType) {\n        this.expressionType = expressionType;\n    }\n\n    public String getExpression() {\n        return expression;\n    }\n\n    public void setExpression(String expression) {\n        this.expression = expression;\n    }\n\n    public long getVersion() {\n        return version;\n    }\n\n    public void setVersion(long version) {\n        this.version = version;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) {\n            return true;\n        }\n        if (o == null || getClass() != o.getClass()) {\n            return false;\n        }\n        SimpleSubscriptionData that = (SimpleSubscriptionData) o;\n        return Objects.equals(topic, that.topic) && Objects.equals(expressionType, that.expressionType) && Objects.equals(expression, that.expression);\n    }\n\n    @Override\n    public int hashCode() {\n        return Objects.hash(topic, expressionType, expression);\n    }\n\n    @Override public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"topic\", topic)\n            .add(\"expressionType\", expressionType)\n            .add(\"expression\", expression)\n            .add(\"version\", version)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/subscription/SubscriptionGroupConfig.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.subscription;\n\nimport com.alibaba.fastjson2.annotation.JSONField;\nimport com.google.common.base.MoreObjects;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Set;\nimport org.apache.commons.lang3.builder.EqualsBuilder;\nimport org.apache.commons.lang3.math.NumberUtils;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.attribute.LiteSubModel;\n\nimport static org.apache.rocketmq.common.SubscriptionGroupAttributes.LITE_SUB_CLIENT_MAX_EVENT_COUNT;\nimport static org.apache.rocketmq.common.SubscriptionGroupAttributes.LITE_SUB_CLIENT_QUOTA_ATTRIBUTE;\nimport static org.apache.rocketmq.common.SubscriptionGroupAttributes.LITE_SUB_MODEL_ATTRIBUTE;\nimport static org.apache.rocketmq.common.SubscriptionGroupAttributes.LITE_SUB_RESET_OFFSET_EXCLUSIVE_ATTRIBUTE;\nimport static org.apache.rocketmq.common.SubscriptionGroupAttributes.LITE_BIND_TOPIC_ATTRIBUTE;\nimport static org.apache.rocketmq.common.SubscriptionGroupAttributes.LITE_SUB_RESET_OFFSET_UNSUBSCRIBE_ATTRIBUTE;\n\nimport static org.apache.rocketmq.common.SubscriptionGroupAttributes.PRIORITY_FACTOR_ATTRIBUTE;\n\npublic class SubscriptionGroupConfig {\n\n    private String groupName;\n\n    private boolean consumeEnable = true;\n    private boolean consumeFromMinEnable = true;\n    private boolean consumeBroadcastEnable = true;\n    private boolean consumeMessageOrderly = false;\n\n    private int retryQueueNums = 1;\n\n    private int retryMaxTimes = 16;\n    private GroupRetryPolicy groupRetryPolicy = new GroupRetryPolicy();\n\n    private long brokerId = MixAll.MASTER_ID;\n\n    private long whichBrokerWhenConsumeSlowly = 1;\n\n    private boolean notifyConsumerIdsChangedEnable = true;\n\n    private int groupSysFlag = 0;\n\n    // Only valid for push consumer\n    private int consumeTimeoutMinute = 15;\n\n    private Set<SimpleSubscriptionData> subscriptionDataSet;\n\n    private Map<String, String> attributes = new HashMap<>();\n\n    public String getGroupName() {\n        return groupName;\n    }\n\n    public void setGroupName(String groupName) {\n        this.groupName = groupName;\n    }\n\n    public boolean isConsumeEnable() {\n        return consumeEnable;\n    }\n\n    public void setConsumeEnable(boolean consumeEnable) {\n        this.consumeEnable = consumeEnable;\n    }\n\n    public boolean isConsumeFromMinEnable() {\n        return consumeFromMinEnable;\n    }\n\n    public void setConsumeFromMinEnable(boolean consumeFromMinEnable) {\n        this.consumeFromMinEnable = consumeFromMinEnable;\n    }\n\n    public boolean isConsumeBroadcastEnable() {\n        return consumeBroadcastEnable;\n    }\n\n    public void setConsumeBroadcastEnable(boolean consumeBroadcastEnable) {\n        this.consumeBroadcastEnable = consumeBroadcastEnable;\n    }\n\n    public boolean isConsumeMessageOrderly() {\n        return consumeMessageOrderly;\n    }\n\n    public void setConsumeMessageOrderly(boolean consumeMessageOrderly) {\n        this.consumeMessageOrderly = consumeMessageOrderly;\n    }\n\n    public int getRetryQueueNums() {\n        return retryQueueNums;\n    }\n\n    public void setRetryQueueNums(int retryQueueNums) {\n        this.retryQueueNums = retryQueueNums;\n    }\n\n    public int getRetryMaxTimes() {\n        return retryMaxTimes;\n    }\n\n    public void setRetryMaxTimes(int retryMaxTimes) {\n        this.retryMaxTimes = retryMaxTimes;\n    }\n\n    public GroupRetryPolicy getGroupRetryPolicy() {\n        return groupRetryPolicy;\n    }\n\n    public void setGroupRetryPolicy(GroupRetryPolicy groupRetryPolicy) {\n        this.groupRetryPolicy = groupRetryPolicy;\n    }\n\n    public long getBrokerId() {\n        return brokerId;\n    }\n\n    public void setBrokerId(long brokerId) {\n        this.brokerId = brokerId;\n    }\n\n    public long getWhichBrokerWhenConsumeSlowly() {\n        return whichBrokerWhenConsumeSlowly;\n    }\n\n    public void setWhichBrokerWhenConsumeSlowly(long whichBrokerWhenConsumeSlowly) {\n        this.whichBrokerWhenConsumeSlowly = whichBrokerWhenConsumeSlowly;\n    }\n\n    public boolean isNotifyConsumerIdsChangedEnable() {\n        return notifyConsumerIdsChangedEnable;\n    }\n\n    public void setNotifyConsumerIdsChangedEnable(final boolean notifyConsumerIdsChangedEnable) {\n        this.notifyConsumerIdsChangedEnable = notifyConsumerIdsChangedEnable;\n    }\n\n    public int getGroupSysFlag() {\n        return groupSysFlag;\n    }\n\n    public void setGroupSysFlag(int groupSysFlag) {\n        this.groupSysFlag = groupSysFlag;\n    }\n\n    public int getConsumeTimeoutMinute() {\n        return consumeTimeoutMinute;\n    }\n\n    public void setConsumeTimeoutMinute(int consumeTimeoutMinute) {\n        this.consumeTimeoutMinute = consumeTimeoutMinute;\n    }\n\n    public Set<SimpleSubscriptionData> getSubscriptionDataSet() {\n        return subscriptionDataSet;\n    }\n\n    public void setSubscriptionDataSet(Set<SimpleSubscriptionData> subscriptionDataSet) {\n        this.subscriptionDataSet = subscriptionDataSet;\n    }\n\n    public Map<String, String> getAttributes() {\n        return attributes;\n    }\n\n    public void setAttributes(Map<String, String> attributes) {\n        this.attributes = attributes;\n    }\n\n    @JSONField(serialize = false, deserialize = false)\n    public long getPriorityFactor() {\n        String factorStr = null == attributes ? null : attributes.get(PRIORITY_FACTOR_ATTRIBUTE.getName());\n        return NumberUtils.toLong(factorStr, PRIORITY_FACTOR_ATTRIBUTE.getDefaultValue());\n    }\n\n    @JSONField(serialize = false, deserialize = false)\n    public void setLiteBindTopic(String liteBindTopic) {\n        if (liteBindTopic != null) {\n            attributes.put(LITE_BIND_TOPIC_ATTRIBUTE.getName(), liteBindTopic);\n        }\n    }\n\n    @JSONField(serialize = false, deserialize = false)\n    public String getLiteBindTopic() {\n        return attributes.get(LITE_BIND_TOPIC_ATTRIBUTE.getName());\n    }\n\n    @JSONField(serialize = false, deserialize = false)\n    public int getLiteSubClientQuota() {\n        long quota = LITE_SUB_CLIENT_QUOTA_ATTRIBUTE.getDefaultValue();\n        String quotaStr = attributes.get(LITE_SUB_CLIENT_QUOTA_ATTRIBUTE.getName());\n        if (quotaStr != null) {\n            quota = Long.parseLong(quotaStr);\n        }\n        return Math.toIntExact(quota);\n    }\n\n    @JSONField(serialize = false, deserialize = false)\n    public boolean isLiteSubExclusive() {\n        String subLiteModel = attributes.get(LITE_SUB_MODEL_ATTRIBUTE.getName());\n        return Objects.equals(LiteSubModel.Exclusive.name(), subLiteModel);\n    }\n\n    /**\n     * Whether to reset offset in exclusive mode\n     */\n    @JSONField(serialize = false, deserialize = false)\n    public boolean isResetOffsetInExclusiveMode() {\n        String boolStr = attributes.get(LITE_SUB_RESET_OFFSET_EXCLUSIVE_ATTRIBUTE.getName());\n        return Boolean.parseBoolean(boolStr);\n    }\n\n    @JSONField(serialize = false, deserialize = false)\n    public boolean isResetOffsetOnUnsubscribe() {\n        String boolStr = attributes.get(LITE_SUB_RESET_OFFSET_UNSUBSCRIBE_ATTRIBUTE.getName());\n        return Boolean.parseBoolean(boolStr);\n    }\n\n    @JSONField(serialize = false, deserialize = false)\n    public int getMaxClientEventCount() {\n        String content = attributes.get(LITE_SUB_CLIENT_MAX_EVENT_COUNT.getName());\n        if (content == null) {\n            return -1;\n        }\n        return NumberUtils.toInt(content, -1);\n    }\n\n    @Override\n    public int hashCode() {\n        final int prime = 31;\n        int result = 1;\n        result = prime * result + (int) (brokerId ^ (brokerId >>> 32));\n        result = prime * result + (consumeBroadcastEnable ? 1231 : 1237);\n        result = prime * result + (consumeEnable ? 1231 : 1237);\n        result = prime * result + (consumeFromMinEnable ? 1231 : 1237);\n        result = prime * result + (notifyConsumerIdsChangedEnable ? 1231 : 1237);\n        result = prime * result + (consumeMessageOrderly ? 1231 : 1237);\n        result = prime * result + ((groupName == null) ? 0 : groupName.hashCode());\n        result = prime * result + retryMaxTimes;\n        result = prime * result + retryQueueNums;\n        result =\n            prime * result + (int) (whichBrokerWhenConsumeSlowly ^ (whichBrokerWhenConsumeSlowly >>> 32));\n        result = prime * result + groupSysFlag;\n        result = prime * result + consumeTimeoutMinute;\n        result = prime * result + ((subscriptionDataSet == null) ? 0 : subscriptionDataSet.hashCode());\n        result = prime * result + attributes.hashCode();\n        return result;\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj)\n            return true;\n        if (obj == null)\n            return false;\n        if (getClass() != obj.getClass())\n            return false;\n        SubscriptionGroupConfig other = (SubscriptionGroupConfig) obj;\n        return new EqualsBuilder()\n            .append(groupName, other.groupName)\n            .append(consumeEnable, other.consumeEnable)\n            .append(consumeFromMinEnable, other.consumeFromMinEnable)\n            .append(consumeBroadcastEnable, other.consumeBroadcastEnable)\n            .append(consumeMessageOrderly, other.consumeMessageOrderly)\n            .append(retryQueueNums, other.retryQueueNums)\n            .append(retryMaxTimes, other.retryMaxTimes)\n            .append(whichBrokerWhenConsumeSlowly, other.whichBrokerWhenConsumeSlowly)\n            .append(notifyConsumerIdsChangedEnable, other.notifyConsumerIdsChangedEnable)\n            .append(groupSysFlag, other.groupSysFlag)\n            .append(consumeTimeoutMinute, other.consumeTimeoutMinute)\n            .append(subscriptionDataSet, other.subscriptionDataSet)\n            .append(attributes, other.attributes)\n            .isEquals();\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"groupName\", groupName)\n            .add(\"consumeEnable\", consumeEnable)\n            .add(\"consumeFromMinEnable\", consumeFromMinEnable)\n            .add(\"consumeBroadcastEnable\", consumeBroadcastEnable)\n            .add(\"consumeMessageOrderly\", consumeMessageOrderly)\n            .add(\"retryQueueNums\", retryQueueNums)\n            .add(\"retryMaxTimes\", retryMaxTimes)\n            .add(\"groupRetryPolicy\", groupRetryPolicy)\n            .add(\"brokerId\", brokerId)\n            .add(\"whichBrokerWhenConsumeSlowly\", whichBrokerWhenConsumeSlowly)\n            .add(\"notifyConsumerIdsChangedEnable\", notifyConsumerIdsChangedEnable)\n            .add(\"groupSysFlag\", groupSysFlag)\n            .add(\"consumeTimeoutMinute\", consumeTimeoutMinute)\n            .add(\"subscriptionDataSet\", subscriptionDataSet)\n            .add(\"attributes\", attributes)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/protocol/topic/OffsetMovedEvent.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.topic;\n\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class OffsetMovedEvent extends RemotingSerializable {\n    private String consumerGroup;\n    private MessageQueue messageQueue;\n    private long offsetRequest;\n    private long offsetNew;\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public MessageQueue getMessageQueue() {\n        return messageQueue;\n    }\n\n    public void setMessageQueue(MessageQueue messageQueue) {\n        this.messageQueue = messageQueue;\n    }\n\n    public long getOffsetRequest() {\n        return offsetRequest;\n    }\n\n    public void setOffsetRequest(long offsetRequest) {\n        this.offsetRequest = offsetRequest;\n    }\n\n    public long getOffsetNew() {\n        return offsetNew;\n    }\n\n    public void setOffsetNew(long offsetNew) {\n        this.offsetNew = offsetNew;\n    }\n\n    @Override\n    public String toString() {\n        return \"OffsetMovedEvent [consumerGroup=\" + consumerGroup + \", messageQueue=\" + messageQueue\n            + \", offsetRequest=\" + offsetRequest + \", offsetNew=\" + offsetNew + \"]\";\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/proxy/SocksProxyConfig.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.proxy;\n\npublic class SocksProxyConfig {\n    private String addr;\n    private String username;\n    private String password;\n\n    public SocksProxyConfig() {\n    }\n\n    public SocksProxyConfig(String addr) {\n        this.addr = addr;\n    }\n\n    public SocksProxyConfig(String addr, String username, String password) {\n        this.addr = addr;\n        this.username = username;\n        this.password = password;\n    }\n\n    public String getAddr() {\n        return addr;\n    }\n\n    public void setAddr(String addr) {\n        this.addr = addr;\n    }\n\n    public String getUsername() {\n        return username;\n    }\n\n    public void setUsername(String username) {\n        this.username = username;\n    }\n\n    public String getPassword() {\n        return password;\n    }\n\n    public void setPassword(String password) {\n        this.password = password;\n    }\n\n    @Override\n    public String toString() {\n        return String.format(\"SocksProxy address: %s, username: %s, password: %s\", addr, username, password);\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/rpc/ClientMetadata.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.rpc;\n\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.body.ClusterInfo;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingUtils;\n\npublic class ClientMetadata {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);\n\n    private final ConcurrentMap<String/* Topic */, TopicRouteData> topicRouteTable = new ConcurrentHashMap<>();\n    private final ConcurrentMap<String/* Topic */, ConcurrentMap<MessageQueue, String/*brokerName*/>> topicEndPointsTable = new ConcurrentHashMap<>();\n    private final ConcurrentMap<String/* Broker Name */, HashMap<Long/* brokerId */, String/* address */>> brokerAddrTable =\n        new ConcurrentHashMap<>();\n    private final ConcurrentMap<String/* Broker Name */, HashMap<String/* address */, Integer>> brokerVersionTable =\n        new ConcurrentHashMap<>();\n\n    public void freshTopicRoute(String topic, TopicRouteData topicRouteData) {\n        if (topic == null\n            || topicRouteData == null) {\n            return;\n        }\n        TopicRouteData old = this.topicRouteTable.get(topic);\n        if (!topicRouteData.topicRouteDataChanged(old)) {\n            return ;\n        }\n        {\n            for (BrokerData bd : topicRouteData.getBrokerDatas()) {\n                this.brokerAddrTable.put(bd.getBrokerName(), bd.getBrokerAddrs());\n            }\n        }\n        {\n            ConcurrentMap<MessageQueue, String> mqEndPoints = topicRouteData2EndpointsForStaticTopic(topic, topicRouteData);\n            if (mqEndPoints != null\n                    && !mqEndPoints.isEmpty()) {\n                topicEndPointsTable.put(topic, mqEndPoints);\n            }\n        }\n    }\n\n    public String getBrokerNameFromMessageQueue(final MessageQueue mq) {\n        if (topicEndPointsTable.get(mq.getTopic()) != null\n                && !topicEndPointsTable.get(mq.getTopic()).isEmpty()) {\n            return topicEndPointsTable.get(mq.getTopic()).get(mq);\n        }\n        return mq.getBrokerName();\n    }\n\n    public void refreshClusterInfo(ClusterInfo clusterInfo) {\n        if (clusterInfo == null\n            || clusterInfo.getBrokerAddrTable() == null) {\n            return;\n        }\n        for (Map.Entry<String, BrokerData> entry : clusterInfo.getBrokerAddrTable().entrySet()) {\n            brokerAddrTable.put(entry.getKey(), entry.getValue().getBrokerAddrs());\n        }\n    }\n\n    public String findMasterBrokerAddr(String brokerName) {\n        if (!brokerAddrTable.containsKey(brokerName)) {\n            return null;\n        }\n        return brokerAddrTable.get(brokerName).get(MixAll.MASTER_ID);\n    }\n\n    public ConcurrentMap<String, HashMap<Long, String>> getBrokerAddrTable() {\n        return brokerAddrTable;\n    }\n\n    public static ConcurrentMap<MessageQueue, String> topicRouteData2EndpointsForStaticTopic(final String topic, final TopicRouteData route) {\n        if (route.getTopicQueueMappingByBroker() == null\n                || route.getTopicQueueMappingByBroker().isEmpty()) {\n            return new ConcurrentHashMap<>();\n        }\n        ConcurrentMap<MessageQueue, String> mqEndPointsOfBroker = new ConcurrentHashMap<>();\n\n        Map<String, Map<String, TopicQueueMappingInfo>> mappingInfosByScope = new HashMap<>();\n        for (Map.Entry<String, TopicQueueMappingInfo> entry : route.getTopicQueueMappingByBroker().entrySet()) {\n            TopicQueueMappingInfo info = entry.getValue();\n            String scope = info.getScope();\n            if (scope != null) {\n                if (!mappingInfosByScope.containsKey(scope)) {\n                    mappingInfosByScope.put(scope, new HashMap<>());\n                }\n                mappingInfosByScope.get(scope).put(entry.getKey(), entry.getValue());\n            }\n        }\n\n        for (Map.Entry<String, Map<String, TopicQueueMappingInfo>> mapEntry : mappingInfosByScope.entrySet()) {\n            String scope = mapEntry.getKey();\n            Map<String, TopicQueueMappingInfo> topicQueueMappingInfoMap =  mapEntry.getValue();\n            ConcurrentMap<MessageQueue, TopicQueueMappingInfo> mqEndPoints = new ConcurrentHashMap<>();\n            List<Map.Entry<String, TopicQueueMappingInfo>> mappingInfos = new ArrayList<>(topicQueueMappingInfoMap.entrySet());\n            mappingInfos.sort((o1, o2) -> (int) (o2.getValue().getEpoch() - o1.getValue().getEpoch()));\n            int maxTotalNums = 0;\n            long maxTotalNumOfEpoch = -1;\n            for (Map.Entry<String, TopicQueueMappingInfo> entry : mappingInfos) {\n                TopicQueueMappingInfo info = entry.getValue();\n                if (info.getEpoch() >= maxTotalNumOfEpoch && info.getTotalQueues() > maxTotalNums) {\n                    maxTotalNums = info.getTotalQueues();\n                }\n                for (Map.Entry<Integer, Integer> idEntry : entry.getValue().getCurrIdMap().entrySet()) {\n                    int globalId = idEntry.getKey();\n                    MessageQueue mq = new MessageQueue(topic, TopicQueueMappingUtils.getMockBrokerName(info.getScope()), globalId);\n                    TopicQueueMappingInfo oldInfo = mqEndPoints.get(mq);\n                    if (oldInfo == null ||  oldInfo.getEpoch() <= info.getEpoch()) {\n                        mqEndPoints.put(mq, info);\n                    }\n                }\n            }\n\n\n            //accomplish the static logic queues\n            for (int i = 0; i < maxTotalNums; i++) {\n                MessageQueue mq = new MessageQueue(topic, TopicQueueMappingUtils.getMockBrokerName(scope), i);\n                if (!mqEndPoints.containsKey(mq)) {\n                    mqEndPointsOfBroker.put(mq, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME_NOT_EXIST);\n                } else {\n                    mqEndPointsOfBroker.put(mq, mqEndPoints.get(mq).getBname());\n                }\n            }\n        }\n        return mqEndPointsOfBroker;\n    }\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RequestBuilder.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.rpc;\n\nimport java.util.HashMap;\nimport java.util.Map;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\n\npublic class RequestBuilder {\n\n    private static Map<Integer, Class> requestCodeMap = new HashMap<>();\n    static {\n        requestCodeMap.put(RequestCode.PULL_MESSAGE, PullMessageRequestHeader.class);\n    }\n\n    public static RpcRequestHeader buildCommonRpcHeader(int requestCode, String destBrokerName) {\n        return buildCommonRpcHeader(requestCode, null, destBrokerName);\n    }\n\n    public static RpcRequestHeader buildCommonRpcHeader(int requestCode, Boolean oneway, String destBrokerName) {\n        Class requestHeaderClass = requestCodeMap.get(requestCode);\n        if (requestHeaderClass == null) {\n            throw new UnsupportedOperationException(\"unknown \" + requestCode);\n        }\n        try {\n            RpcRequestHeader requestHeader = (RpcRequestHeader) requestHeaderClass.newInstance();\n            requestHeader.setOneway(oneway);\n            requestHeader.setBrokerName(destBrokerName);\n            return requestHeader;\n        } catch (Throwable t) {\n            throw new RuntimeException(t);\n        }\n    }\n\n    public static TopicQueueRequestHeader buildTopicQueueRequestHeader(int requestCode, MessageQueue mq) {\n        return buildTopicQueueRequestHeader(requestCode, null, mq.getBrokerName(), mq.getTopic(), mq.getQueueId(), null);\n    }\n\n    public static TopicQueueRequestHeader buildTopicQueueRequestHeader(int requestCode, MessageQueue mq, Boolean logic) {\n        return buildTopicQueueRequestHeader(requestCode, null, mq.getBrokerName(), mq.getTopic(), mq.getQueueId(), logic);\n    }\n\n    public static TopicQueueRequestHeader buildTopicQueueRequestHeader(int requestCode, Boolean oneway, MessageQueue mq, Boolean logic) {\n        return buildTopicQueueRequestHeader(requestCode, oneway, mq.getBrokerName(), mq.getTopic(), mq.getQueueId(), logic);\n    }\n\n    public static TopicQueueRequestHeader buildTopicQueueRequestHeader(int requestCode,  Boolean oneway, String destBrokerName, String topic, int queueId, Boolean logic) {\n        Class requestHeaderClass = requestCodeMap.get(requestCode);\n        if (requestHeaderClass == null) {\n            throw new UnsupportedOperationException(\"unknown \" + requestCode);\n        }\n        try {\n            TopicQueueRequestHeader requestHeader = (TopicQueueRequestHeader) requestHeaderClass.newInstance();\n            requestHeader.setOneway(oneway);\n            requestHeader.setBrokerName(destBrokerName);\n            requestHeader.setTopic(topic);\n            requestHeader.setQueueId(queueId);\n            requestHeader.setLo(logic);\n            return requestHeader;\n        } catch (Throwable t) {\n            throw new RuntimeException(t);\n        }\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClient.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.rpc;\n\nimport java.util.concurrent.Future;\nimport org.apache.rocketmq.common.message.MessageQueue;\n\npublic interface RpcClient {\n\n\n    //common invoke paradigm, the logic remote addr is defined in \"bname\" field of request\n    //For oneway request, the sign is labeled in request, and do not need an another method named \"invokeOneway\"\n    //For one\n    Future<RpcResponse>  invoke(RpcRequest request, long timeoutMs) throws RpcException;\n\n    //For rocketmq, most requests are corresponded to MessageQueue\n    //And for LogicQueue, the broker name is mocked, the physical addr could only be defined by MessageQueue\n    Future<RpcResponse>  invoke(MessageQueue mq, RpcRequest request, long timeoutMs) throws RpcException;\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientHook.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.rpc;\n\npublic abstract class RpcClientHook {\n\n    //if the return is not null, return it\n    public abstract RpcResponse beforeRequest(RpcRequest rpcRequest) throws RpcException;\n\n    //if the return is not null, return it\n    public abstract RpcResponse afterResponse(RpcResponse rpcResponse) throws RpcException;\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.rpc;\n\nimport io.netty.util.concurrent.ImmediateEventExecutor;\nimport io.netty.util.concurrent.Promise;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.Future;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.InvokeCallback;\nimport org.apache.rocketmq.remoting.RemotingClient;\nimport org.apache.rocketmq.remoting.netty.ResponseFuture;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;\nimport org.apache.rocketmq.remoting.protocol.header.GetEarliestMsgStoretimeResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMinOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SearchOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicConfigAndQueueMapping;\n\npublic class RpcClientImpl implements RpcClient {\n\n    private ClientMetadata clientMetadata;\n\n    private RemotingClient remotingClient;\n\n    private List<RpcClientHook> clientHookList = new ArrayList<>();\n\n    public RpcClientImpl(ClientMetadata clientMetadata, RemotingClient remotingClient) {\n        this.clientMetadata = clientMetadata;\n        this.remotingClient = remotingClient;\n    }\n\n    public void registerHook(RpcClientHook hook) {\n        clientHookList.add(hook);\n    }\n\n    @Override\n    public Future<RpcResponse>  invoke(MessageQueue mq, RpcRequest request, long timeoutMs) throws RpcException {\n        String bname =  clientMetadata.getBrokerNameFromMessageQueue(mq);\n        request.getHeader().setBrokerName(bname);\n        return invoke(request, timeoutMs);\n    }\n\n\n    public Promise<RpcResponse> createResponseFuture()  {\n        return ImmediateEventExecutor.INSTANCE.newPromise();\n    }\n\n    @Override\n    public Future<RpcResponse>  invoke(RpcRequest request, long timeoutMs) throws RpcException {\n        if (clientHookList.size() > 0) {\n            for (RpcClientHook rpcClientHook: clientHookList) {\n                RpcResponse response = rpcClientHook.beforeRequest(request);\n                if (response != null) {\n                    //For 1.6, there is not easy-to-use future impl\n                    return createResponseFuture().setSuccess(response);\n                }\n            }\n        }\n        String addr = getBrokerAddrByNameOrException(request.getHeader().bname);\n        Promise<RpcResponse> rpcResponsePromise = null;\n        try {\n            switch (request.getCode()) {\n                case RequestCode.PULL_MESSAGE:\n                    rpcResponsePromise = handlePullMessage(addr, request, timeoutMs);\n                    break;\n                case RequestCode.GET_MIN_OFFSET:\n                    rpcResponsePromise = handleGetMinOffset(addr, request, timeoutMs);\n                    break;\n                case RequestCode.GET_MAX_OFFSET:\n                    rpcResponsePromise = handleGetMaxOffset(addr, request, timeoutMs);\n                    break;\n                case RequestCode.SEARCH_OFFSET_BY_TIMESTAMP:\n                    rpcResponsePromise = handleSearchOffset(addr, request, timeoutMs);\n                    break;\n                case RequestCode.GET_EARLIEST_MSG_STORETIME:\n                    rpcResponsePromise = handleGetEarliestMsgStoretime(addr, request, timeoutMs);\n                    break;\n                case RequestCode.QUERY_CONSUMER_OFFSET:\n                    rpcResponsePromise = handleQueryConsumerOffset(addr, request, timeoutMs);\n                    break;\n                case RequestCode.UPDATE_CONSUMER_OFFSET:\n                    rpcResponsePromise = handleUpdateConsumerOffset(addr, request, timeoutMs);\n                    break;\n                case RequestCode.GET_TOPIC_STATS_INFO:\n                    rpcResponsePromise = handleCommonBodyRequest(addr, request, timeoutMs, TopicStatsTable.class);\n                    break;\n                case RequestCode.GET_TOPIC_CONFIG:\n                    rpcResponsePromise = handleCommonBodyRequest(addr, request, timeoutMs, TopicConfigAndQueueMapping.class);\n                    break;\n                default:\n                    throw new RpcException(ResponseCode.REQUEST_CODE_NOT_SUPPORTED, \"Unknown request code \" + request.getCode());\n            }\n        } catch (RpcException rpcException) {\n            throw rpcException;\n        } catch (Exception e) {\n            throw new RpcException(ResponseCode.RPC_UNKNOWN, \"error from remoting layer\", e);\n        }\n        return rpcResponsePromise;\n    }\n\n\n    private String getBrokerAddrByNameOrException(String bname) throws RpcException {\n        String addr = this.clientMetadata.findMasterBrokerAddr(bname);\n        if (addr == null) {\n            throw new RpcException(ResponseCode.SYSTEM_ERROR, \"cannot find addr for broker \" + bname);\n        }\n        return addr;\n    }\n\n\n    private void processFailedResponse(String addr, RemotingCommand requestCommand,  ResponseFuture responseFuture, Promise<RpcResponse> rpcResponsePromise) {\n        RemotingCommand responseCommand = responseFuture.getResponseCommand();\n        if (responseCommand != null) {\n            //this should not happen\n            return;\n        }\n        int errorCode = ResponseCode.RPC_UNKNOWN;\n        String errorMessage = null;\n        if (!responseFuture.isSendRequestOK()) {\n            errorCode = ResponseCode.RPC_SEND_TO_CHANNEL_FAILED;\n            errorMessage = \"send request failed to \" + addr + \". Request: \" + requestCommand;\n        } else if (responseFuture.isTimeout()) {\n            errorCode = ResponseCode.RPC_TIME_OUT;\n            errorMessage = \"wait response from \" + addr + \" timeout :\" + responseFuture.getTimeoutMillis() + \"ms\" + \". Request: \" + requestCommand;\n        } else {\n            errorMessage = \"unknown reason. addr: \" + addr + \", timeoutMillis: \" + responseFuture.getTimeoutMillis() + \". Request: \" + requestCommand;\n        }\n        rpcResponsePromise.setSuccess(new RpcResponse(new RpcException(errorCode, errorMessage)));\n    }\n\n\n    public Promise<RpcResponse> handlePullMessage(final String addr, RpcRequest rpcRequest, long timeoutMillis)  throws Exception {\n        final RemotingCommand requestCommand = RpcClientUtils.createCommandForRpcRequest(rpcRequest);\n\n        final Promise<RpcResponse> rpcResponsePromise = createResponseFuture();\n\n        InvokeCallback callback = new InvokeCallback() {\n            @Override\n            public void operationComplete(ResponseFuture responseFuture) {\n\n            }\n\n            @Override\n            public void operationSucceed(RemotingCommand response) {\n                try {\n                    switch (response.getCode()) {\n                        case ResponseCode.SUCCESS:\n                        case ResponseCode.PULL_NOT_FOUND:\n                        case ResponseCode.PULL_RETRY_IMMEDIATELY:\n                        case ResponseCode.PULL_OFFSET_MOVED:\n                            PullMessageResponseHeader responseHeader =\n                                (PullMessageResponseHeader) response.decodeCommandCustomHeader(PullMessageResponseHeader.class);\n                            rpcResponsePromise.setSuccess(new RpcResponse(response.getCode(), responseHeader, response.getBody()));\n                            break;\n                        default:\n                            RpcResponse rpcResponse = new RpcResponse(new RpcException(response.getCode(), \"unexpected remote response code\"));\n                            rpcResponsePromise.setSuccess(rpcResponse);\n\n                    }\n                } catch (Exception e) {\n                    String errorMessage = \"process failed. addr: \" + addr + \", timeoutMillis: \" + timeoutMillis + \". Request: \" + requestCommand;\n                    RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, e));\n                    rpcResponsePromise.setSuccess(rpcResponse);\n                }\n            }\n\n            @Override\n            public void operationFail(Throwable throwable) {\n                String errorMessage = \"process failed. addr: \" + addr + \". Request: \" + requestCommand;\n                RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, throwable));\n                rpcResponsePromise.setSuccess(rpcResponse);\n            }\n        };\n\n        this.remotingClient.invokeAsync(addr, requestCommand, timeoutMillis, callback);\n        return rpcResponsePromise;\n    }\n\n    public Promise<RpcResponse> handleSearchOffset(String addr, RpcRequest rpcRequest, long timeoutMillis) throws Exception {\n        final Promise<RpcResponse> rpcResponsePromise = createResponseFuture();\n\n        RemotingCommand requestCommand = RpcClientUtils.createCommandForRpcRequest(rpcRequest);\n        RemotingCommand responseCommand = this.remotingClient.invokeSync(addr, requestCommand, timeoutMillis);\n        assert responseCommand != null;\n        switch (responseCommand.getCode()) {\n            case ResponseCode.SUCCESS: {\n                SearchOffsetResponseHeader responseHeader =\n                        (SearchOffsetResponseHeader) responseCommand.decodeCommandCustomHeader(SearchOffsetResponseHeader.class);\n                rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody()));\n                break;\n            }\n            default: {\n                rpcResponsePromise.setSuccess(new RpcResponse(new RpcException(responseCommand.getCode(), \"unknown remote error\")));\n            }\n        }\n        return rpcResponsePromise;\n    }\n\n\n\n    public Promise<RpcResponse> handleQueryConsumerOffset(String addr, RpcRequest rpcRequest, long timeoutMillis) throws Exception {\n        final Promise<RpcResponse> rpcResponsePromise = createResponseFuture();\n\n        RemotingCommand requestCommand = RpcClientUtils.createCommandForRpcRequest(rpcRequest);\n        RemotingCommand responseCommand = this.remotingClient.invokeSync(addr, requestCommand, timeoutMillis);\n        assert responseCommand != null;\n        switch (responseCommand.getCode()) {\n            case ResponseCode.SUCCESS: {\n                QueryConsumerOffsetResponseHeader responseHeader =\n                        (QueryConsumerOffsetResponseHeader) responseCommand.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class);\n                rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody()));\n                break;\n            }\n            case ResponseCode.QUERY_NOT_FOUND: {\n                rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), null, null));\n                break;\n            }\n            default: {\n                rpcResponsePromise.setSuccess(new RpcResponse(new RpcException(responseCommand.getCode(), \"unknown remote error\")));\n            }\n        }\n        return rpcResponsePromise;\n    }\n\n    public Promise<RpcResponse> handleUpdateConsumerOffset(String addr, RpcRequest rpcRequest, long timeoutMillis) throws Exception {\n        final Promise<RpcResponse> rpcResponsePromise = createResponseFuture();\n\n        RemotingCommand requestCommand = RpcClientUtils.createCommandForRpcRequest(rpcRequest);\n        RemotingCommand responseCommand = this.remotingClient.invokeSync(addr, requestCommand, timeoutMillis);\n        assert responseCommand != null;\n        switch (responseCommand.getCode()) {\n            case ResponseCode.SUCCESS: {\n                UpdateConsumerOffsetResponseHeader responseHeader =\n                    (UpdateConsumerOffsetResponseHeader) responseCommand.decodeCommandCustomHeader(UpdateConsumerOffsetResponseHeader.class);\n                rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody()));\n                break;\n            }\n            default: {\n                rpcResponsePromise.setSuccess(new RpcResponse(new RpcException(responseCommand.getCode(), \"unknown remote error\")));\n            }\n        }\n        return rpcResponsePromise;\n    }\n\n    public Promise<RpcResponse> handleCommonBodyRequest(final String addr, RpcRequest rpcRequest, long timeoutMillis, Class bodyClass) throws Exception {\n        final Promise<RpcResponse> rpcResponsePromise = createResponseFuture();\n        RemotingCommand requestCommand = RpcClientUtils.createCommandForRpcRequest(rpcRequest);\n        RemotingCommand responseCommand = this.remotingClient.invokeSync(addr, requestCommand, timeoutMillis);\n        assert responseCommand != null;\n        switch (responseCommand.getCode()) {\n            case ResponseCode.SUCCESS: {\n                rpcResponsePromise.setSuccess(new RpcResponse(ResponseCode.SUCCESS, null, RemotingSerializable.decode(responseCommand.getBody(), bodyClass)));\n                break;\n            }\n            default: {\n                rpcResponsePromise.setSuccess(new RpcResponse(new RpcException(responseCommand.getCode(), \"unknown remote error\")));\n            }\n        }\n        return rpcResponsePromise;\n    }\n\n    public Promise<RpcResponse> handleGetMinOffset(String addr, RpcRequest rpcRequest, long timeoutMillis) throws Exception {\n        final Promise<RpcResponse> rpcResponsePromise = createResponseFuture();\n\n        RemotingCommand requestCommand = RpcClientUtils.createCommandForRpcRequest(rpcRequest);\n\n        RemotingCommand responseCommand = this.remotingClient.invokeSync(addr, requestCommand, timeoutMillis);\n        assert responseCommand != null;\n        switch (responseCommand.getCode()) {\n            case ResponseCode.SUCCESS: {\n                GetMinOffsetResponseHeader responseHeader =\n                        (GetMinOffsetResponseHeader) responseCommand.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class);\n                rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody()));\n                break;\n            }\n            default: {\n                rpcResponsePromise.setSuccess(new RpcResponse(new RpcException(responseCommand.getCode(), \"unknown remote error\")));\n            }\n        }\n        return rpcResponsePromise;\n    }\n\n    public Promise<RpcResponse> handleGetMaxOffset(String addr, RpcRequest rpcRequest, long timeoutMillis) throws Exception {\n        final Promise<RpcResponse> rpcResponsePromise = createResponseFuture();\n\n        RemotingCommand requestCommand = RpcClientUtils.createCommandForRpcRequest(rpcRequest);\n\n        RemotingCommand responseCommand = this.remotingClient.invokeSync(addr, requestCommand, timeoutMillis);\n        assert responseCommand != null;\n        switch (responseCommand.getCode()) {\n            case ResponseCode.SUCCESS: {\n                GetMaxOffsetResponseHeader responseHeader =\n                        (GetMaxOffsetResponseHeader) responseCommand.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class);\n                rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody()));\n                break;\n            }\n            default: {\n                rpcResponsePromise.setSuccess(new RpcResponse(new RpcException(responseCommand.getCode(), \"unknown remote error\")));\n            }\n        }\n        return rpcResponsePromise;\n    }\n\n    public Promise<RpcResponse> handleGetEarliestMsgStoretime(String addr, RpcRequest rpcRequest, long timeoutMillis) throws Exception {\n        final Promise<RpcResponse> rpcResponsePromise = createResponseFuture();\n\n        RemotingCommand requestCommand = RpcClientUtils.createCommandForRpcRequest(rpcRequest);\n\n        RemotingCommand responseCommand = this.remotingClient.invokeSync(addr, requestCommand, timeoutMillis);\n        assert responseCommand != null;\n        switch (responseCommand.getCode()) {\n            case ResponseCode.SUCCESS: {\n                GetEarliestMsgStoretimeResponseHeader responseHeader =\n                        (GetEarliestMsgStoretimeResponseHeader) responseCommand.decodeCommandCustomHeader(GetEarliestMsgStoretimeResponseHeader.class);\n                rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody()));\n                break;\n            }\n            default: {\n                rpcResponsePromise.setSuccess(new RpcResponse(new RpcException(responseCommand.getCode(), \"unknown remote error\")));\n            }\n        }\n        return rpcResponsePromise;\n    }\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.rpc;\n\nimport java.nio.ByteBuffer;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\npublic class RpcClientUtils {\n\n    public static RemotingCommand createCommandForRpcRequest(RpcRequest rpcRequest) {\n        RemotingCommand cmd = RemotingCommand.createRequestCommand(rpcRequest.getCode(), rpcRequest.getHeader());\n        cmd.setBody(encodeBody(rpcRequest.getBody()));\n        return cmd;\n    }\n\n    public static RemotingCommand createCommandForRpcResponse(RpcResponse rpcResponse) {\n        RemotingCommand cmd = RemotingCommand.createResponseCommandWithHeader(rpcResponse.getCode(), rpcResponse.getHeader());\n        cmd.setRemark(rpcResponse.getException() == null ? \"\" : rpcResponse.getException().getMessage());\n        cmd.setBody(encodeBody(rpcResponse.getBody()));\n        return cmd;\n    }\n\n    public static byte[] encodeBody(Object body) {\n        if (body == null) {\n            return null;\n        }\n        if (body instanceof byte[]) {\n            return (byte[])body;\n        } else if (body instanceof RemotingSerializable) {\n            return ((RemotingSerializable) body).encode();\n        } else if (body instanceof ByteBuffer) {\n            ByteBuffer buffer = (ByteBuffer)body;\n            buffer.mark();\n            byte[] data = new byte[buffer.remaining()];\n            buffer.get(data);\n            buffer.reset();\n            return data;\n        } else {\n            throw new RuntimeException(\"Unsupported body type \" + body.getClass());\n        }\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcException.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.rpc;\n\nimport org.apache.rocketmq.remoting.exception.RemotingException;\n\npublic class RpcException extends RemotingException {\n    private int errorCode;\n    public RpcException(int errorCode, String message) {\n        super(message);\n        this.errorCode = errorCode;\n    }\n\n    public RpcException(int errorCode, String message, Throwable cause) {\n        super(message, cause);\n        this.errorCode = errorCode;\n    }\n\n    public int getErrorCode() {\n        return errorCode;\n    }\n\n    public void setErrorCode(int errorCode) {\n        this.errorCode = errorCode;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcRequest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.rpc;\n\npublic class RpcRequest {\n    int code;\n    private RpcRequestHeader header;\n    private Object body;\n\n    public RpcRequest(int code, RpcRequestHeader header, Object body) {\n        this.code = code;\n        this.header = header;\n        this.body = body;\n    }\n\n    public RpcRequestHeader getHeader() {\n        return header;\n    }\n\n    public Object getBody() {\n        return body;\n    }\n\n    public int getCode() {\n        return code;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.rpc;\n\nimport com.google.common.base.MoreObjects;\nimport java.util.Objects;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\n\npublic abstract class RpcRequestHeader implements CommandCustomHeader {\n    //the namespace name\n    protected String ns;\n    //if the data has been namespaced\n    protected Boolean nsd;\n    //the abstract remote addr name, usually the physical broker name\n    protected String bname;\n    //oneway\n    protected Boolean oway;\n\n    @Deprecated\n    public String getBname() {\n        return bname;\n    }\n\n    @Deprecated\n    public void setBname(String brokerName) {\n        this.bname = brokerName;\n    }\n\n    public String getBrokerName() {\n        return bname;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.bname = brokerName;\n    }\n\n    public String getNamespace() {\n        return ns;\n    }\n\n    public void setNamespace(String namespace) {\n        this.ns = namespace;\n    }\n\n    public Boolean getNamespaced() {\n        return nsd;\n    }\n\n    public void setNamespaced(Boolean namespaced) {\n        this.nsd = namespaced;\n    }\n\n    public Boolean getOneway() {\n        return oway;\n    }\n\n    public void setOneway(Boolean oneway) {\n        this.oway = oneway;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) {\n            return true;\n        }\n        if (o == null || getClass() != o.getClass()) {\n            return false;\n        }\n        RpcRequestHeader header = (RpcRequestHeader) o;\n        return Objects.equals(ns, header.ns) && Objects.equals(nsd, header.nsd) && Objects.equals(bname, header.bname) && Objects.equals(oway, header.oway);\n    }\n\n    @Override\n    public int hashCode() {\n        return Objects.hash(ns, nsd, bname, oway);\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n            .add(\"namespace\", ns)\n            .add(\"namespaced\", nsd)\n            .add(\"brokerName\", bname)\n            .add(\"oneway\", oway)\n            .toString();\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcResponse.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.rpc;\n\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\n\npublic class RpcResponse   {\n    private int code;\n    private CommandCustomHeader header;\n    private Object body;\n    public RpcException exception;\n\n    public RpcResponse() {\n\n    }\n\n    public RpcResponse(int code, CommandCustomHeader header, Object body) {\n        this.code = code;\n        this.header = header;\n        this.body = body;\n    }\n\n    public RpcResponse(RpcException rpcException) {\n        this.code = rpcException.getErrorCode();\n        this.exception = rpcException;\n    }\n\n    public int getCode() {\n        return code;\n    }\n\n    public CommandCustomHeader getHeader() {\n        return header;\n    }\n\n    public void setHeader(CommandCustomHeader header) {\n        this.header = header;\n    }\n\n    public Object getBody() {\n        return body;\n    }\n\n    public void setBody(Object body) {\n        this.body = body;\n    }\n\n    public RpcException getException() {\n        return exception;\n    }\n\n    public void setException(RpcException exception) {\n        this.exception = exception;\n    }\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/rpc/TopicQueueRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.rpc;\n\npublic abstract class TopicQueueRequestHeader extends TopicRequestHeader {\n\n    public abstract Integer getQueueId();\n    public abstract void setQueueId(Integer queueId);\n\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/rpc/TopicRequestHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.rpc;\n\npublic abstract class TopicRequestHeader extends RpcRequestHeader {\n    //logical\n    protected Boolean lo;\n\n    public abstract String getTopic();\n    public abstract void setTopic(String topic);\n\n    public Boolean getLo() {\n        return lo;\n    }\n    public void setLo(Boolean lo) {\n        this.lo = lo;\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/rpchook/DynamicalExtFieldRPCHook.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.rpchook;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\n\npublic class DynamicalExtFieldRPCHook implements RPCHook {\n\n    @Override\n    public void doBeforeRequest(String remoteAddr, RemotingCommand request) {\n        String zoneName = System.getProperty(MixAll.ROCKETMQ_ZONE_PROPERTY, System.getenv(MixAll.ROCKETMQ_ZONE_ENV));\n        if (StringUtils.isNotBlank(zoneName)) {\n            request.addExtField(MixAll.ZONE_NAME, zoneName);\n        }\n        String zoneMode = System.getProperty(MixAll.ROCKETMQ_ZONE_MODE_PROPERTY, System.getenv(MixAll.ROCKETMQ_ZONE_MODE_ENV));\n        if (StringUtils.isNotBlank(zoneMode)) {\n            request.addExtField(MixAll.ZONE_MODE, zoneMode);\n        }\n    }\n\n    @Override\n    public void doAfterResponse(String remoteAddr, RemotingCommand request, RemotingCommand response) {\n\n    }\n}\n"
  },
  {
    "path": "remoting/src/main/java/org/apache/rocketmq/remoting/rpchook/StreamTypeRPCHook.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.rpchook;\n\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestType;\n\npublic class StreamTypeRPCHook implements RPCHook {\n    @Override\n    public void doBeforeRequest(String remoteAddr, RemotingCommand request) {\n        request.addExtField(MixAll.REQ_T, String.valueOf(RequestType.STREAM.getCode()));\n    }\n\n    @Override\n    public void doAfterResponse(String remoteAddr, RemotingCommand request,\n        RemotingCommand response) {\n\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/ProxyProtocolTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting;\n\nimport io.netty.buffer.ByteBuf;\nimport io.netty.buffer.Unpooled;\nimport io.netty.channel.Channel;\nimport io.netty.handler.codec.haproxy.HAProxyCommand;\nimport io.netty.handler.codec.haproxy.HAProxyMessage;\nimport io.netty.handler.codec.haproxy.HAProxyMessageEncoder;\nimport io.netty.handler.codec.haproxy.HAProxyProtocolVersion;\nimport io.netty.handler.codec.haproxy.HAProxyProxiedProtocol;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingClient;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.io.IOException;\nimport java.lang.reflect.Method;\nimport java.net.Socket;\nimport java.time.Duration;\nimport java.util.concurrent.TimeUnit;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertNotNull;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ProxyProtocolTest {\n\n    private RemotingServer remotingServer;\n    private RemotingClient remotingClient;\n\n    @Before\n    public void setUp() throws Exception {\n        NettyClientConfig clientConfig = new NettyClientConfig();\n        clientConfig.setUseTLS(false);\n\n        remotingServer = RemotingServerTest.createRemotingServer();\n        remotingClient = RemotingServerTest.createRemotingClient(clientConfig);\n\n        await().pollDelay(Duration.ofMillis(10))\n                .pollInterval(Duration.ofMillis(10))\n                .atMost(20, TimeUnit.SECONDS).until(() -> isHostConnectable(getServerAddress()));\n    }\n\n    @Test\n    public void testProxyProtocol() throws Exception {\n        sendHAProxyMessage(remotingClient);\n        requestThenAssertResponse(remotingClient);\n    }\n\n    private void requestThenAssertResponse(RemotingClient remotingClient) throws Exception {\n        RemotingCommand response = remotingClient.invokeSync(getServerAddress(), createRequest(), 10000 * 3);\n        assertNotNull(response);\n        assertThat(response.getLanguage()).isEqualTo(LanguageCode.JAVA);\n        assertThat(response.getExtFields()).hasSize(2);\n        assertThat(response.getExtFields().get(\"messageTitle\")).isEqualTo(\"Welcome\");\n    }\n\n    private void sendHAProxyMessage(RemotingClient remotingClient) throws Exception {\n        Method getAndCreateChannel = NettyRemotingClient.class.getDeclaredMethod(\"getAndCreateChannel\", String.class);\n        getAndCreateChannel.setAccessible(true);\n        NettyRemotingClient nettyRemotingClient = (NettyRemotingClient) remotingClient;\n        Channel channel = (Channel) getAndCreateChannel.invoke(nettyRemotingClient, getServerAddress());\n        HAProxyMessage message = new HAProxyMessage(HAProxyProtocolVersion.V2, HAProxyCommand.PROXY,\n                HAProxyProxiedProtocol.TCP4, \"127.0.0.1\", \"127.0.0.2\", 8000, 9000);\n\n        ByteBuf byteBuf = Unpooled.directBuffer();\n        Method encode = HAProxyMessageEncoder.class.getDeclaredMethod(\"encodeV2\", HAProxyMessage.class, ByteBuf.class);\n        encode.setAccessible(true);\n        encode.invoke(HAProxyMessageEncoder.INSTANCE, message, byteBuf);\n        channel.writeAndFlush(byteBuf).sync();\n    }\n\n    private static RemotingCommand createRequest() {\n        RequestHeader requestHeader = new RequestHeader();\n        requestHeader.setCount(1);\n        requestHeader.setMessageTitle(\"Welcome\");\n        return RemotingCommand.createRequestCommand(0, requestHeader);\n    }\n\n\n    private String getServerAddress() {\n        return \"localhost:\" + remotingServer.localListenPort();\n    }\n\n    private boolean isHostConnectable(String addr) {\n        try (Socket socket = new Socket()) {\n            socket.connect(NetworkUtil.string2SocketAddress(addr));\n            return true;\n        } catch (IOException ignored) {\n        }\n        return false;\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.Executors;\nimport org.apache.rocketmq.remoting.annotation.CFNullable;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.exception.RemotingConnectException;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingClient;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingServer;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.netty.ResponseFuture;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.junit.AfterClass;\nimport org.junit.BeforeClass;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.junit.Assert.assertNotNull;\n\npublic class RemotingServerTest {\n    private static RemotingServer remotingServer;\n    private static RemotingClient remotingClient;\n\n    public static RemotingServer createRemotingServer() throws InterruptedException {\n        NettyServerConfig config = new NettyServerConfig();\n        RemotingServer remotingServer = new NettyRemotingServer(config);\n        remotingServer.registerProcessor(0, new NettyRequestProcessor() {\n            @Override\n            public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) {\n                request.setRemark(\"Hi \" + ctx.channel().remoteAddress());\n                return request;\n            }\n\n            @Override\n            public boolean rejectRequest() {\n                return false;\n            }\n        }, Executors.newCachedThreadPool());\n\n        remotingServer.start();\n\n        return remotingServer;\n    }\n\n    public static RemotingClient createRemotingClient() {\n        return createRemotingClient(new NettyClientConfig());\n    }\n\n    public static RemotingClient createRemotingClient(NettyClientConfig nettyClientConfig) {\n        RemotingClient client = new NettyRemotingClient(nettyClientConfig);\n        client.start();\n        return client;\n    }\n\n    @BeforeClass\n    public static void setup() throws InterruptedException {\n        remotingServer = createRemotingServer();\n        remotingClient = createRemotingClient();\n    }\n\n    @AfterClass\n    public static void destroy() {\n        remotingClient.shutdown();\n        remotingServer.shutdown();\n    }\n\n    @Test\n    public void testInvokeSync() throws InterruptedException, RemotingConnectException,\n        RemotingSendRequestException, RemotingTimeoutException {\n        RequestHeader requestHeader = new RequestHeader();\n        requestHeader.setCount(1);\n        requestHeader.setMessageTitle(\"Welcome\");\n        RemotingCommand request = RemotingCommand.createRequestCommand(0, requestHeader);\n        RemotingCommand response = remotingClient.invokeSync(\"localhost:\" + remotingServer.localListenPort(), request, 1000 * 3);\n        assertNotNull(response);\n        assertThat(response.getLanguage()).isEqualTo(LanguageCode.JAVA);\n        assertThat(response.getExtFields()).hasSize(2);\n\n    }\n\n    @Test\n    public void testInvokeOneway() throws InterruptedException, RemotingConnectException,\n        RemotingTimeoutException, RemotingTooMuchRequestException, RemotingSendRequestException {\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(0, null);\n        request.setRemark(\"messi\");\n        remotingClient.invokeOneway(\"localhost:\" + remotingServer.localListenPort(), request, 1000 * 3);\n    }\n\n    @Test\n    public void testInvokeAsync() throws InterruptedException, RemotingConnectException,\n        RemotingTimeoutException, RemotingTooMuchRequestException, RemotingSendRequestException {\n\n        final CountDownLatch latch = new CountDownLatch(1);\n        RemotingCommand request = RemotingCommand.createRequestCommand(0, null);\n        request.setRemark(\"messi\");\n        remotingClient.invokeAsync(\"localhost:\" + remotingServer.localListenPort(), request, 1000 * 3, new InvokeCallback() {\n            @Override\n            public void operationComplete(ResponseFuture responseFuture) {\n\n            }\n\n            @Override\n            public void operationSucceed(RemotingCommand response) {\n                latch.countDown();\n                assertThat(response.getLanguage()).isEqualTo(LanguageCode.JAVA);\n                assertThat(response.getExtFields()).hasSize(2);\n            }\n\n            @Override\n            public void operationFail(Throwable throwable) {\n\n            }\n        });\n        latch.await();\n    }\n}\n\nclass RequestHeader implements CommandCustomHeader {\n    @CFNullable\n    private Integer count;\n\n    @CFNullable\n    private String messageTitle;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public Integer getCount() {\n        return count;\n    }\n\n    public void setCount(Integer count) {\n        this.count = count;\n    }\n\n    public String getMessageTitle() {\n        return messageTitle;\n    }\n\n    public void setMessageTitle(String messageTitle) {\n        this.messageTitle = messageTitle;\n    }\n}\n\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/SubRemotingServerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingConnectException;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode;\nimport org.junit.AfterClass;\nimport org.junit.BeforeClass;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown;\n\npublic class SubRemotingServerTest {\n    private static final int SUB_SERVER_PORT = 1234;\n\n    private static RemotingServer remotingServer;\n    private static RemotingClient remotingClient;\n    private static RemotingServer subServer;\n\n    @BeforeClass\n    public static void setup() throws InterruptedException {\n        remotingServer = RemotingServerTest.createRemotingServer();\n        remotingClient = RemotingServerTest.createRemotingClient();\n        subServer = createSubRemotingServer(remotingServer);\n    }\n\n    @AfterClass\n    public static void destroy() {\n        remotingClient.shutdown();\n        remotingServer.shutdown();\n    }\n\n    public static RemotingServer createSubRemotingServer(RemotingServer parentServer) {\n        RemotingServer subServer = parentServer.newRemotingServer(SUB_SERVER_PORT);\n        subServer.registerProcessor(1, new NettyRequestProcessor() {\n            @Override\n            public RemotingCommand processRequest(final ChannelHandlerContext ctx,\n                    final RemotingCommand request) throws Exception {\n                request.setRemark(String.valueOf(RemotingHelper.parseSocketAddressPort(ctx.channel().localAddress())));\n                return request;\n            }\n\n            @Override\n            public boolean rejectRequest() {\n                return false;\n            }\n        }, null);\n        subServer.start();\n        return subServer;\n    }\n\n    @Test\n    public void testInvokeSubRemotingServer() throws InterruptedException, RemotingTimeoutException,\n            RemotingConnectException, RemotingSendRequestException {\n        RequestHeader requestHeader = new RequestHeader();\n        requestHeader.setCount(1);\n        requestHeader.setMessageTitle(\"Welcome\");\n\n        // Parent remoting server doesn't support RequestCode 1\n        RemotingCommand request = RemotingCommand.createRequestCommand(1, requestHeader);\n        RemotingCommand response = remotingClient.invokeSync(\"localhost:\" + remotingServer.localListenPort(), request,\n                1000 * 3);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(RemotingSysResponseCode.REQUEST_CODE_NOT_SUPPORTED);\n\n        // Issue request to SubRemotingServer\n        response = remotingClient.invokeSync(\"localhost:1234\", request, 1000 * 3);\n        assertThat(response).isNotNull();\n        assertThat(response.getExtFields()).hasSize(2);\n        assertThat(response.getRemark()).isEqualTo(String.valueOf(SUB_SERVER_PORT));\n\n        // Issue unsupported request to SubRemotingServer\n        request.setCode(0);\n        response = remotingClient.invokeSync(\"localhost:1234\", request, 1000 * 3);\n        assertThat(response).isNotNull();\n        assertThat(response.getCode()).isEqualTo(RemotingSysResponseCode.REQUEST_CODE_NOT_SUPPORTED);\n\n        // Issue request to a closed SubRemotingServer\n        request.setCode(1);\n        remotingServer.removeRemotingServer(SUB_SERVER_PORT);\n        subServer.shutdown();\n        try {\n            remotingClient.invokeSync(\"localhost:1234\", request, 1000 * 3);\n            failBecauseExceptionWasNotThrown(RemotingTimeoutException.class);\n        } catch (Exception e) {\n            assertThat(e).isInstanceOfAny(RemotingTimeoutException.class, RemotingSendRequestException.class);\n        }\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting;\n\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.remoting.common.TlsMode;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingServer;\nimport org.apache.rocketmq.remoting.netty.TlsHelper;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Rule;\nimport org.junit.Test;\nimport org.junit.rules.TemporaryFolder;\nimport org.junit.rules.TestName;\nimport org.junit.runner.RunWith;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.io.BufferedInputStream;\nimport java.io.BufferedOutputStream;\nimport java.io.BufferedWriter;\nimport java.io.File;\nimport java.io.FileOutputStream;\nimport java.io.FileWriter;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.PrintWriter;\nimport java.net.Socket;\nimport java.time.Duration;\nimport java.util.UUID;\nimport java.util.concurrent.TimeUnit;\n\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_CLIENT_AUTHSERVER;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_CLIENT_CERTPATH;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_CLIENT_KEYPASSWORD;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_CLIENT_KEYPATH;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_CLIENT_TRUSTCERTPATH;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_SERVER_AUTHCLIENT;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_SERVER_CERTPATH;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_SERVER_KEYPASSWORD;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_SERVER_KEYPATH;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_SERVER_NEED_CLIENT_AUTH;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_SERVER_TRUSTCERTPATH;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsClientAuthServer;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsClientCertPath;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsClientKeyPassword;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsClientKeyPath;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsClientTrustCertPath;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsConfigFile;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsMode;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerAuthClient;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerCertPath;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerKeyPassword;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerKeyPath;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerNeedClientAuth;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerTrustCertPath;\nimport static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsTestModeEnable;\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown;\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertNotNull;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class TlsTest {\n    private RemotingServer remotingServer;\n    private RemotingClient remotingClient;\n\n    @Rule\n    public TestName name = new TestName();\n\n    @Rule\n    public TemporaryFolder tempFolder = new TemporaryFolder();\n\n    @Before\n    public void setUp() throws InterruptedException {\n        tlsMode = TlsMode.ENFORCING;\n        tlsTestModeEnable = false;\n        tlsServerNeedClientAuth = \"require\";\n        tlsServerKeyPath = getCertsPath(\"server.key\");\n        tlsServerCertPath = getCertsPath(\"server.pem\");\n        tlsServerAuthClient = true;\n        tlsServerTrustCertPath = getCertsPath(\"ca.pem\");\n        tlsClientKeyPath = getCertsPath(\"client.key\");\n        tlsClientCertPath = getCertsPath(\"client.pem\");\n        tlsClientAuthServer = true;\n        tlsClientTrustCertPath = getCertsPath(\"ca.pem\");\n        tlsClientKeyPassword = \"1234\";\n        tlsServerKeyPassword = \"\";\n\n        NettyClientConfig clientConfig = new NettyClientConfig();\n        clientConfig.setUseTLS(true);\n\n        if (\"serverRejectsUntrustedClientCert\".equals(name.getMethodName())) {\n            // Create a client. Its credentials come from a CA that the server does not trust. The client\n            // trusts both test CAs to ensure the handshake failure is due to the server rejecting the client's cert.\n            tlsClientKeyPath = getCertsPath(\"badClient.key\");\n            tlsClientCertPath = getCertsPath(\"badClient.pem\");\n        } else if (\"serverAcceptsUntrustedClientCert\".equals(name.getMethodName())) {\n            tlsClientKeyPath = getCertsPath(\"badClient.key\");\n            tlsClientCertPath = getCertsPath(\"badClient.pem\");\n            tlsServerAuthClient = false;\n        }\n        else if (\"noClientAuthFailure\".equals(name.getMethodName())) {\n            //Clear the client cert config to ensure produce the handshake error\n            tlsClientKeyPath = \"\";\n            tlsClientCertPath = \"\";\n        } else if (\"clientRejectsUntrustedServerCert\".equals(name.getMethodName())) {\n            tlsServerKeyPath = getCertsPath(\"badServer.key\");\n            tlsServerCertPath = getCertsPath(\"badServer.pem\");\n        } else if (\"clientAcceptsUntrustedServerCert\".equals(name.getMethodName())) {\n            tlsServerKeyPath = getCertsPath(\"badServer.key\");\n            tlsServerCertPath = getCertsPath(\"badServer.pem\");\n            tlsClientAuthServer = false;\n        } else if (\"serverNotNeedClientAuth\".equals(name.getMethodName())) {\n            tlsServerNeedClientAuth = \"none\";\n            tlsClientKeyPath = \"\";\n            tlsClientCertPath = \"\";\n        } else if (\"serverWantClientAuth\".equals(name.getMethodName())) {\n            tlsServerNeedClientAuth = \"optional\";\n        } else if (\"serverWantClientAuth_ButClientNoCert\".equals(name.getMethodName())) {\n            tlsServerNeedClientAuth = \"optional\";\n            tlsClientKeyPath = \"\";\n            tlsClientCertPath = \"\";\n        } else if (\"serverAcceptsUnAuthClient\".equals(name.getMethodName())) {\n            tlsMode = TlsMode.PERMISSIVE;\n            tlsClientKeyPath = \"\";\n            tlsClientCertPath = \"\";\n            clientConfig.setUseTLS(false);\n        } else if (\"disabledServerRejectsSSLClient\".equals(name.getMethodName())) {\n            tlsMode = TlsMode.DISABLED;\n        } else if (\"disabledServerAcceptUnAuthClient\".equals(name.getMethodName())) {\n            tlsMode = TlsMode.DISABLED;\n            tlsClientKeyPath = \"\";\n            tlsClientCertPath = \"\";\n            clientConfig.setUseTLS(false);\n        } else if (\"reloadSslContextForServer\".equals(name.getMethodName())) {\n            tlsClientAuthServer = false;\n            tlsServerNeedClientAuth = \"none\";\n        }\n\n        remotingServer = RemotingServerTest.createRemotingServer();\n        remotingClient = RemotingServerTest.createRemotingClient(clientConfig);\n\n        await().pollDelay(Duration.ofMillis(10))\n                .pollInterval(Duration.ofMillis(10))\n                .atMost(20, TimeUnit.SECONDS).until(() -> isHostConnectable(getServerAddress()));\n    }\n\n    @After\n    public void tearDown() {\n        remotingClient.shutdown();\n        remotingServer.shutdown();\n        tlsMode = TlsMode.PERMISSIVE;\n    }\n\n    /**\n     * Tests that a client and a server configured using two-way SSL auth can successfully\n     * communicate with each other.\n     */\n    @Test\n    public void basicClientServerIntegrationTest() throws Exception {\n        requestThenAssertResponse();\n    }\n\n    @Test\n    public void reloadSslContextForServer() throws Exception {\n        requestThenAssertResponse();\n\n        //Use new cert and private key\n        tlsClientKeyPath = getCertsPath(\"badClient.key\");\n        tlsClientCertPath = getCertsPath(\"badClient.pem\");\n\n        ((NettyRemotingServer) remotingServer).loadSslContext();\n\n        //Request Again\n        requestThenAssertResponse();\n\n        //Start another client\n        NettyClientConfig clientConfig = new NettyClientConfig();\n        clientConfig.setUseTLS(true);\n        RemotingClient remotingClient = RemotingServerTest.createRemotingClient(clientConfig);\n        requestThenAssertResponse(remotingClient);\n    }\n\n    @Test\n    public void serverNotNeedClientAuth() throws Exception {\n        requestThenAssertResponse();\n    }\n\n    @Test\n    public void serverWantClientAuth_ButClientNoCert() throws Exception {\n        requestThenAssertResponse();\n    }\n\n    @Test\n    public void serverAcceptsUnAuthClient() throws Exception {\n        requestThenAssertResponse();\n    }\n\n    @Test\n    public void disabledServerRejectsSSLClient() throws Exception {\n        try {\n            RemotingCommand response = remotingClient.invokeSync(getServerAddress(), createRequest(), 1000 * 5);\n            failBecauseExceptionWasNotThrown(RemotingSendRequestException.class);\n        } catch (RemotingSendRequestException ignore) {\n        }\n    }\n\n    @Test\n    public void disabledServerAcceptUnAuthClient() throws Exception {\n        requestThenAssertResponse();\n    }\n\n    /**\n     * Tests that a server configured to require client authentication refuses to accept connections\n     * from a client that has an untrusted certificate.\n     */\n    @Test\n    public void serverRejectsUntrustedClientCert() throws Exception {\n        try {\n            RemotingCommand response = remotingClient.invokeSync(getServerAddress(), createRequest(), 1000 * 5);\n            failBecauseExceptionWasNotThrown(RemotingSendRequestException.class);\n        } catch (RemotingSendRequestException ignore) {\n        }\n    }\n\n    @Test\n    public void serverAcceptsUntrustedClientCert() throws Exception {\n        requestThenAssertResponse();\n//        Thread.sleep(1000000L);\n    }\n\n    /**\n     * Tests that a server configured to require client authentication actually does require client\n     * authentication.\n     */\n    @Test\n    public void noClientAuthFailure() throws Exception {\n        try {\n            RemotingCommand response = remotingClient.invokeSync(getServerAddress(), createRequest(), 1000 * 3);\n            failBecauseExceptionWasNotThrown(RemotingSendRequestException.class);\n        } catch (RemotingSendRequestException ignore) {\n        }\n    }\n\n    /**\n     * Tests that a client configured using GrpcSslContexts refuses to talk to a server that has an\n     * an untrusted certificate.\n     */\n    @Test\n    public void clientRejectsUntrustedServerCert() throws Exception {\n        try {\n            RemotingCommand response = remotingClient.invokeSync(getServerAddress(), createRequest(), 1000 * 3);\n            failBecauseExceptionWasNotThrown(RemotingSendRequestException.class);\n        } catch (RemotingSendRequestException ignore) {\n        }\n    }\n\n    @Test\n    public void clientAcceptsUntrustedServerCert() throws Exception {\n        requestThenAssertResponse();\n    }\n\n    @Test\n    public void testTlsConfigThroughFile() throws Exception {\n        File file = tempFolder.newFile(\"tls.config\");\n        tlsTestModeEnable = true;\n\n        tlsConfigFile = file.getAbsolutePath();\n\n        StringBuilder sb = new StringBuilder();\n        sb.append(TLS_SERVER_NEED_CLIENT_AUTH + \"=require\\n\");\n        sb.append(TLS_SERVER_KEYPATH + \"=/server.key\\n\");\n        sb.append(TLS_SERVER_CERTPATH + \"=/server.pem\\n\");\n        sb.append(TLS_SERVER_KEYPASSWORD + \"=2345\\n\");\n        sb.append(TLS_SERVER_AUTHCLIENT + \"=true\\n\");\n        sb.append(TLS_SERVER_TRUSTCERTPATH + \"=/ca.pem\\n\");\n        sb.append(TLS_CLIENT_KEYPATH + \"=/client.key\\n\");\n        sb.append(TLS_CLIENT_KEYPASSWORD + \"=1234\\n\");\n        sb.append(TLS_CLIENT_CERTPATH + \"=/client.pem\\n\");\n        sb.append(TLS_CLIENT_AUTHSERVER + \"=false\\n\");\n        sb.append(TLS_CLIENT_TRUSTCERTPATH + \"=/ca.pem\\n\");\n\n        writeStringToFile(file.getAbsolutePath(), sb.toString());\n        TlsHelper.buildSslContext(false);\n\n        assertThat(tlsServerNeedClientAuth).isEqualTo(\"require\");\n        assertThat(tlsServerKeyPath).isEqualTo(\"/server.key\");\n        assertThat(tlsServerCertPath).isEqualTo(\"/server.pem\");\n        assertThat(tlsServerKeyPassword).isEqualTo(\"2345\");\n        assertThat(tlsServerAuthClient).isEqualTo(true);\n        assertThat(tlsServerTrustCertPath).isEqualTo(\"/ca.pem\");\n        assertThat(tlsClientKeyPath).isEqualTo(\"/client.key\");\n        assertThat(tlsClientKeyPassword).isEqualTo(\"1234\");\n        assertThat(tlsClientCertPath).isEqualTo(\"/client.pem\");\n        assertThat(tlsClientAuthServer).isEqualTo(false);\n        assertThat(tlsClientTrustCertPath).isEqualTo(\"/ca.pem\");\n\n        tlsConfigFile = \"/notFound\";\n    }\n\n    private static void writeStringToFile(String path, String content) {\n        try {\n            PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(path, true)));\n            out.println(content);\n            out.close();\n        } catch (IOException ignore) {\n        }\n    }\n\n    private static String getCertsPath(String fileName) {\n        ClassLoader loader = TlsTest.class.getClassLoader();\n        InputStream stream = loader.getResourceAsStream(\"certs/\" + fileName);\n        if (null == stream) {\n            throw new RuntimeException(\"File: \" + fileName + \" is not found\");\n        }\n\n        try {\n            String[] segments = fileName.split(\"\\\\.\");\n            File f = File.createTempFile(UUID.randomUUID().toString(), segments[1]);\n            f.deleteOnExit();\n\n            try (BufferedInputStream bis = new BufferedInputStream(stream);\n                 BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(f))) {\n                byte[] buffer = new byte[1024];\n                int len;\n                while ((len = bis.read(buffer)) > 0) {\n                    bos.write(buffer, 0, len);\n                }\n            } catch (IOException e) {\n                throw new RuntimeException(e);\n            }\n            return f.getAbsolutePath();\n        } catch (IOException e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    private String getServerAddress() {\n        return \"localhost:\" + remotingServer.localListenPort();\n    }\n\n    private static RemotingCommand createRequest() {\n        RequestHeader requestHeader = new RequestHeader();\n        requestHeader.setCount(1);\n        requestHeader.setMessageTitle(\"Welcome\");\n        return RemotingCommand.createRequestCommand(0, requestHeader);\n    }\n\n    private void requestThenAssertResponse() throws Exception {\n        requestThenAssertResponse(remotingClient);\n    }\n\n    private void requestThenAssertResponse(RemotingClient remotingClient) throws Exception {\n        RemotingCommand response = remotingClient.invokeSync(getServerAddress(), createRequest(), 1000 * 3);\n        assertNotNull(response);\n        assertThat(response.getLanguage()).isEqualTo(LanguageCode.JAVA);\n        assertThat(response.getExtFields()).hasSize(2);\n        assertThat(response.getExtFields().get(\"messageTitle\")).isEqualTo(\"Welcome\");\n    }\n\n    private boolean isHostConnectable(String addr) {\n        try (Socket socket = new Socket()) {\n            socket.connect(NetworkUtil.string2SocketAddress(addr));\n            return true;\n        } catch (IOException ignored) {\n        }\n        return false;\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/netty/FileRegionEncoderTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.netty;\n\nimport io.netty.buffer.ByteBuf;\nimport io.netty.channel.DefaultFileRegion;\nimport io.netty.channel.FileRegion;\nimport io.netty.channel.embedded.EmbeddedChannel;\nimport java.io.BufferedOutputStream;\nimport java.io.File;\nimport java.io.FileOutputStream;\nimport java.io.IOException;\nimport java.util.Random;\nimport java.util.UUID;\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class FileRegionEncoderTest {\n\n    /**\n     * This unit test case ensures that {@link FileRegionEncoder} indeed wraps {@link FileRegion} to\n     * {@link ByteBuf}.\n     * @throws IOException if there is an error.\n     */\n    @Test\n    public void testEncode() throws IOException {\n        FileRegionEncoder fileRegionEncoder = new FileRegionEncoder();\n        EmbeddedChannel channel = new EmbeddedChannel(fileRegionEncoder);\n        File file = File.createTempFile(UUID.randomUUID().toString(), \".data\");\n        file.deleteOnExit();\n        Random random = new Random(System.currentTimeMillis());\n        int dataLength = 1 << 10;\n        byte[] data = new byte[dataLength];\n        random.nextBytes(data);\n        write(file, data);\n        FileRegion fileRegion = new DefaultFileRegion(file, 0, dataLength);\n        Assert.assertEquals(0, fileRegion.transferred());\n        Assert.assertEquals(dataLength, fileRegion.count());\n        Assert.assertTrue(channel.writeOutbound(fileRegion));\n        ByteBuf out = (ByteBuf) channel.readOutbound();\n        byte[] arr = new byte[out.readableBytes()];\n        out.getBytes(0, arr);\n        Assert.assertArrayEquals(\"Data should be identical\", data, arr);\n    }\n\n    /**\n     * Write byte array to the specified file.\n     *\n     * @param file File to write to.\n     * @param data byte array to write.\n     * @throws IOException in case there is an exception.\n     */\n    private static void write(File file, byte[] data) throws IOException {\n        BufferedOutputStream bufferedOutputStream = null;\n        try {\n            bufferedOutputStream = new BufferedOutputStream(new FileOutputStream(file, false));\n            bufferedOutputStream.write(data);\n            bufferedOutputStream.flush();\n        } finally {\n            if (null != bufferedOutputStream) {\n                bufferedOutputStream.close();\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.netty;\n\nimport io.netty.channel.ChannelFuture;\nimport io.netty.channel.local.LocalChannel;\n\npublic class MockChannel extends LocalChannel {\n    @Override\n    public ChannelFuture writeAndFlush(Object msg) {\n        return new MockChannelPromise(MockChannel.this);\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.netty;\n\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelPromise;\nimport io.netty.util.concurrent.Future;\nimport io.netty.util.concurrent.GenericFutureListener;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.TimeoutException;\nimport org.jetbrains.annotations.NotNull;\n\npublic class MockChannelPromise implements ChannelPromise {\n    protected Channel channel;\n\n    public MockChannelPromise(Channel channel) {\n        this.channel = channel;\n    }\n\n    @Override\n    public Channel channel() {\n        return channel;\n    }\n\n    @Override\n    public ChannelPromise setSuccess(Void result) {\n        return this;\n    }\n\n    @Override\n    public ChannelPromise setSuccess() {\n        return this;\n    }\n\n    @Override\n    public boolean trySuccess() {\n        return false;\n    }\n\n    @Override\n    public ChannelPromise setFailure(Throwable cause) {\n        return this;\n    }\n\n    @Override\n    public ChannelPromise addListener(GenericFutureListener<? extends Future<? super Void>> listener) {\n        return this;\n    }\n\n    @Override\n    public ChannelPromise addListeners(GenericFutureListener<? extends Future<? super Void>>... listeners) {\n        return this;\n    }\n\n    @Override\n    public ChannelPromise removeListener(GenericFutureListener<? extends Future<? super Void>> listener) {\n        return this;\n    }\n\n    @Override\n    public ChannelPromise removeListeners(GenericFutureListener<? extends Future<? super Void>>... listeners) {\n        return this;\n    }\n\n    @Override\n    public ChannelPromise sync() throws InterruptedException {\n        return this;\n    }\n\n    @Override\n    public ChannelPromise syncUninterruptibly() {\n        return this;\n    }\n\n    @Override\n    public ChannelPromise await() throws InterruptedException {\n        return this;\n    }\n\n    @Override\n    public ChannelPromise awaitUninterruptibly() {\n        return this;\n    }\n\n    @Override\n    public ChannelPromise unvoid() {\n        return this;\n    }\n\n    @Override\n    public boolean isVoid() {\n        return false;\n    }\n\n    @Override\n    public boolean trySuccess(Void result) {\n        return false;\n    }\n\n    @Override\n    public boolean tryFailure(Throwable cause) {\n        return false;\n    }\n\n    @Override\n    public boolean setUncancellable() {\n        return false;\n    }\n\n    @Override\n    public boolean isSuccess() {\n        return false;\n    }\n\n    @Override\n    public boolean isCancellable() {\n        return false;\n    }\n\n    @Override\n    public Throwable cause() {\n        return null;\n    }\n\n    @Override\n    public boolean await(long timeout, TimeUnit unit) throws InterruptedException {\n        return false;\n    }\n\n    @Override\n    public boolean await(long timeoutMillis) throws InterruptedException {\n        return false;\n    }\n\n    @Override\n    public boolean awaitUninterruptibly(long timeout, TimeUnit unit) {\n        return false;\n    }\n\n    @Override\n    public boolean awaitUninterruptibly(long timeoutMillis) {\n        return false;\n    }\n\n    @Override\n    public Void getNow() {\n        return null;\n    }\n\n    @Override\n    public boolean cancel(boolean mayInterruptIfRunning) {\n        return false;\n    }\n\n    @Override\n    public boolean isCancelled() {\n        return false;\n    }\n\n    @Override\n    public boolean isDone() {\n        return false;\n    }\n\n    @Override\n    public Void get() throws InterruptedException, ExecutionException {\n        return null;\n    }\n\n    @Override\n    public Void get(long timeout,\n        @NotNull java.util.concurrent.TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {\n        return null;\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyClientConfigTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.netty;\n\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class NettyClientConfigTest {\n\n    @Test\n    public void testChangeConfigBySystemProperty() {\n\n\n        System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_WORKER_SIZE, \"1\");\n        System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_CONNECT_TIMEOUT, \"2000\");\n        System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_CHANNEL_MAX_IDLE_SECONDS, \"60\");\n        System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_SNDBUF_SIZE, \"16383\");\n        System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_RCVBUF_SIZE, \"16384\");\n        System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_CLOSE_SOCKET_IF_TIMEOUT, \"false\");\n        System.setProperty(TlsSystemConfig.TLS_ENABLE, \"true\");\n\n\n        NettySystemConfig.socketSndbufSize =\n            Integer.parseInt(System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_SNDBUF_SIZE, \"65535\"));\n        NettySystemConfig.socketRcvbufSize =\n            Integer.parseInt(System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_RCVBUF_SIZE, \"65535\"));\n        NettySystemConfig.clientWorkerSize =\n            Integer.parseInt(System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_WORKER_SIZE, \"4\"));\n        NettySystemConfig.connectTimeoutMillis =\n            Integer.parseInt(System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_CONNECT_TIMEOUT, \"3000\"));\n        NettySystemConfig.clientChannelMaxIdleTimeSeconds =\n            Integer.parseInt(System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_CHANNEL_MAX_IDLE_SECONDS, \"120\"));\n        NettySystemConfig.clientCloseSocketIfTimeout =\n            Boolean.parseBoolean(System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_CLIENT_CLOSE_SOCKET_IF_TIMEOUT, \"true\"));\n\n        NettyClientConfig changedConfig = new NettyClientConfig();\n        assertThat(changedConfig.getClientWorkerThreads()).isEqualTo(1);\n        assertThat(changedConfig.getClientOnewaySemaphoreValue()).isEqualTo(65535);\n        assertThat(changedConfig.getClientAsyncSemaphoreValue()).isEqualTo(65535);\n        assertThat(changedConfig.getConnectTimeoutMillis()).isEqualTo(2000);\n        assertThat(changedConfig.getClientChannelMaxIdleTimeSeconds()).isEqualTo(60);\n        assertThat(changedConfig.getClientSocketSndBufSize()).isEqualTo(16383);\n        assertThat(changedConfig.getClientSocketRcvBufSize()).isEqualTo(16384);\n        assertThat(changedConfig.isClientCloseSocketIfTimeout()).isEqualTo(false);\n        assertThat(changedConfig.isUseTLS()).isEqualTo(true);\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.netty;\n\nimport java.util.concurrent.Semaphore;\nimport org.apache.rocketmq.remoting.InvokeCallback;\nimport org.apache.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.junit.Assert.assertNull;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class NettyRemotingAbstractTest {\n    @Spy\n    private NettyRemotingAbstract remotingAbstract = new NettyRemotingClient(new NettyClientConfig());\n\n    @Test\n    public void testProcessResponseCommand() throws InterruptedException {\n        final Semaphore semaphore = new Semaphore(0);\n        ResponseFuture responseFuture = new ResponseFuture(null, 1, 3000, new InvokeCallback() {\n            @Override\n            public void operationComplete(ResponseFuture responseFuture) {\n\n            }\n\n            @Override\n            public void operationSucceed(RemotingCommand response) {\n                assertThat(semaphore.availablePermits()).isEqualTo(0);\n            }\n\n            @Override\n            public void operationFail(Throwable throwable) {\n\n            }\n        }, new SemaphoreReleaseOnlyOnce(semaphore));\n\n        remotingAbstract.responseTable.putIfAbsent(1, responseFuture);\n\n        RemotingCommand response = RemotingCommand.createResponseCommand(0, \"Foo\");\n        response.setOpaque(1);\n        remotingAbstract.processResponseCommand(null, response);\n\n        // Acquire the release permit after call back\n        semaphore.acquire(1);\n        assertThat(semaphore.availablePermits()).isEqualTo(0);\n    }\n\n    @Test\n    public void testProcessResponseCommand_NullCallBack() throws InterruptedException {\n        final Semaphore semaphore = new Semaphore(0);\n        ResponseFuture responseFuture = new ResponseFuture(null, 1, 3000, null,\n            new SemaphoreReleaseOnlyOnce(semaphore));\n\n        remotingAbstract.responseTable.putIfAbsent(1, responseFuture);\n\n        RemotingCommand response = RemotingCommand.createResponseCommand(0, \"Foo\");\n        response.setOpaque(1);\n        remotingAbstract.processResponseCommand(null, response);\n\n        assertThat(semaphore.availablePermits()).isEqualTo(1);\n    }\n\n    @Test\n    public void testProcessResponseCommand_RunCallBackInCurrentThread() throws InterruptedException {\n        final Semaphore semaphore = new Semaphore(0);\n        ResponseFuture responseFuture = new ResponseFuture(null, 1, 3000, new InvokeCallback() {\n            @Override\n            public void operationComplete(ResponseFuture responseFuture) {\n\n            }\n\n            @Override\n            public void operationSucceed(RemotingCommand response) {\n                assertThat(semaphore.availablePermits()).isEqualTo(0);\n            }\n\n            @Override\n            public void operationFail(Throwable throwable) {\n\n            }\n        }, new SemaphoreReleaseOnlyOnce(semaphore));\n\n        remotingAbstract.responseTable.putIfAbsent(1, responseFuture);\n        when(remotingAbstract.getCallbackExecutor()).thenReturn(null);\n\n        RemotingCommand response = RemotingCommand.createResponseCommand(0, \"Foo\");\n        response.setOpaque(1);\n        remotingAbstract.processResponseCommand(null, response);\n\n        // Acquire the release permit after call back finished in current thread\n        semaphore.acquire(1);\n        assertThat(semaphore.availablePermits()).isEqualTo(0);\n    }\n\n    @Test\n    public void testScanResponseTable() {\n        int dummyId = 1;\n        // mock timeout\n        ResponseFuture responseFuture = new ResponseFuture(null, dummyId, -1000, new InvokeCallback() {\n            @Override\n            public void operationComplete(ResponseFuture responseFuture) {\n\n            }\n\n            @Override\n            public void operationSucceed(RemotingCommand response) {\n\n            }\n\n            @Override\n            public void operationFail(Throwable throwable) {\n\n            }\n        }, null);\n        remotingAbstract.responseTable.putIfAbsent(dummyId, responseFuture);\n        remotingAbstract.scanResponseTable();\n        assertNull(remotingAbstract.responseTable.get(dummyId));\n    }\n\n    @Test\n    public void testProcessRequestCommand() throws InterruptedException {\n        final Semaphore semaphore = new Semaphore(0);\n        RemotingCommand request = RemotingCommand.createRequestCommand(1, null);\n        ResponseFuture responseFuture = new ResponseFuture(null, 1, request, 3000,\n            new InvokeCallback() {\n                @Override\n                public void operationComplete(ResponseFuture responseFuture) {\n\n                }\n\n                @Override\n                public void operationSucceed(RemotingCommand response) {\n                    assertThat(semaphore.availablePermits()).isEqualTo(0);\n                }\n\n                @Override\n                public void operationFail(Throwable throwable) {\n\n                }\n            }, new SemaphoreReleaseOnlyOnce(semaphore));\n\n        remotingAbstract.responseTable.putIfAbsent(1, responseFuture);\n        RemotingCommand response = RemotingCommand.createResponseCommand(0, \"Foo\");\n        response.setOpaque(1);\n        remotingAbstract.processResponseCommand(null, response);\n\n        // Acquire the release permit after call back\n        semaphore.acquire(1);\n        assertThat(semaphore.availablePermits()).isEqualTo(0);\n    }\n}"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.netty;\n\nimport io.netty.bootstrap.Bootstrap;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelFuture;\nimport io.netty.channel.local.LocalChannel;\n\nimport java.lang.reflect.Field;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.Semaphore;\nimport org.apache.rocketmq.remoting.InvokeCallback;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce;\nimport org.apache.rocketmq.remoting.exception.RemotingConnectException;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Spy;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.assertj.core.api.Assertions.assertThatThrownBy;\nimport static org.assertj.core.api.AssertionsForClassTypes.catchThrowable;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.doReturn;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.never;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.times;\nimport static org.mockito.Mockito.verify;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class NettyRemotingClientTest {\n    @Spy\n    private NettyRemotingClient remotingClient = new NettyRemotingClient(new NettyClientConfig());\n    @Mock\n    private RPCHook rpcHookMock;\n\n    @Test\n    public void testSetCallbackExecutor() {\n        ExecutorService customized = Executors.newCachedThreadPool();\n        remotingClient.setCallbackExecutor(customized);\n        assertThat(remotingClient.getCallbackExecutor()).isEqualTo(customized);\n    }\n\n    @Test\n    public void testInvokeResponse() throws Exception {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);\n\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);\n        responseFuture.setResponseCommand(response);\n        CompletableFuture<RemotingCommand> future0 = new CompletableFuture<>();\n        future0.complete(responseFuture.getResponseCommand());\n        doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());\n\n        CompletableFuture<RemotingCommand> future = remotingClient.invoke(\"0.0.0.0\", request, 1000);\n        RemotingCommand actual = future.get();\n        assertThat(actual).isEqualTo(response);\n    }\n\n    @Test\n    public void testRemotingSendRequestException() throws Exception {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);\n\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        CompletableFuture<RemotingCommand> future0 = new CompletableFuture<>();\n        future0.completeExceptionally(new RemotingSendRequestException(null));\n        doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());\n\n        CompletableFuture<RemotingCommand> future = remotingClient.invoke(\"0.0.0.0\", request, 1000);\n        Throwable thrown = catchThrowable(future::get);\n        assertThat(thrown.getCause()).isInstanceOf(RemotingSendRequestException.class);\n    }\n\n    @Test\n    public void testRemotingTimeoutException() throws Exception {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);\n\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        CompletableFuture<RemotingCommand> future0 = new CompletableFuture<>();\n        future0.completeExceptionally(new RemotingTimeoutException(\"\"));\n        doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());\n\n        CompletableFuture<RemotingCommand> future = remotingClient.invoke(\"0.0.0.0\", request, 1000);\n        Throwable thrown = catchThrowable(future::get);\n        assertThat(thrown.getCause()).isInstanceOf(RemotingTimeoutException.class);\n    }\n\n    @Test\n    public void testRemotingException() throws Exception {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);\n\n        CompletableFuture<RemotingCommand> future0 = new CompletableFuture<>();\n        future0.completeExceptionally(new RemotingException(\"\"));\n        doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());\n\n        CompletableFuture<RemotingCommand> future = remotingClient.invoke(\"0.0.0.0\", request, 1000);\n        Throwable thrown = catchThrowable(future::get);\n        assertThat(thrown.getCause()).isInstanceOf(RemotingException.class);\n    }\n\n    @Test\n    public void testInvokeOnewayException() throws Exception {\n        String addr = \"0.0.0.0\";\n        try {\n            remotingClient.invokeOneway(addr, null, 1000);\n        } catch (RemotingConnectException e) {\n            assertThat(e.getMessage()).contains(addr);\n        }\n    }\n\n    @Test\n    public void testInvoke0() throws ExecutionException, InterruptedException {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        Channel channel = new MockChannel() {\n            @Override\n            public ChannelFuture writeAndFlush(Object msg) {\n                ResponseFuture responseFuture = remotingClient.responseTable.get(request.getOpaque());\n                responseFuture.setResponseCommand(response);\n                responseFuture.executeInvokeCallback();\n                return super.writeAndFlush(msg);\n            }\n        };\n        CompletableFuture<ResponseFuture> future = remotingClient.invoke0(channel, request, 1000L);\n        assertThat(future.get().getResponseCommand()).isEqualTo(response);\n    }\n\n    @Test\n    public void testInvoke0WithException() {\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        Channel channel = new MockChannel() {\n            @Override\n            public ChannelFuture writeAndFlush(Object msg) {\n                ResponseFuture responseFuture = remotingClient.responseTable.get(request.getOpaque());\n                responseFuture.executeInvokeCallback();\n                return super.writeAndFlush(msg);\n            }\n        };\n        CompletableFuture<ResponseFuture> future = remotingClient.invoke0(channel, request, 1000L);\n        assertThatThrownBy(future::get).getCause().isInstanceOf(RemotingException.class);\n    }\n\n    @Test\n    public void testInvokeSync() throws RemotingSendRequestException, RemotingTimeoutException, InterruptedException {\n        remotingClient.registerRPCHook(rpcHookMock);\n\n        Channel channel = new LocalChannel();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() {\n            @Override\n            public void operationComplete(ResponseFuture responseFuture) {\n\n            }\n        }, new SemaphoreReleaseOnlyOnce(new Semaphore(1)));\n        responseFuture.setResponseCommand(response);\n        CompletableFuture<ResponseFuture> future = new CompletableFuture<>();\n        future.complete(responseFuture);\n\n        doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong());\n        RemotingCommand actual = remotingClient.invokeSyncImpl(channel, request, 1000);\n        assertThat(actual).isEqualTo(response);\n\n        verify(rpcHookMock).doBeforeRequest(anyString(), eq(request));\n        verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response));\n    }\n\n    @Test\n    public void testInvokeAsync() {\n        remotingClient.registerRPCHook(rpcHookMock);\n        Channel channel = new LocalChannel();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() {\n            @Override\n            public void operationComplete(ResponseFuture responseFuture) {\n\n            }\n        }, new SemaphoreReleaseOnlyOnce(new Semaphore(1)));\n        responseFuture.setResponseCommand(response);\n        CompletableFuture<ResponseFuture> future = new CompletableFuture<>();\n        future.complete(responseFuture);\n\n        doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong());\n\n        InvokeCallback callback = mock(InvokeCallback.class);\n        remotingClient.invokeAsyncImpl(channel, request, 1000, callback);\n        verify(callback, times(1)).operationSucceed(eq(response));\n        verify(callback, times(1)).operationComplete(eq(responseFuture));\n        verify(callback, never()).operationFail(any());\n\n        verify(rpcHookMock).doBeforeRequest(anyString(), eq(request));\n        verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response));\n    }\n\n    @Test\n    public void testInvokeAsyncFail() {\n        remotingClient.registerRPCHook(rpcHookMock);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);\n\n        Channel channel = new LocalChannel();\n        CompletableFuture<ResponseFuture> future = new CompletableFuture<>();\n        future.completeExceptionally(new RemotingException(null));\n\n        doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong());\n\n        InvokeCallback callback = mock(InvokeCallback.class);\n        remotingClient.invokeAsyncImpl(channel, request, 1000, callback);\n        verify(callback, never()).operationSucceed(any());\n        verify(callback, times(1)).operationComplete(any());\n        verify(callback, times(1)).operationFail(any());\n\n        verify(rpcHookMock).doBeforeRequest(anyString(), eq(request));\n        verify(rpcHookMock, never()).doAfterResponse(anyString(), eq(request), any());\n    }\n\n    @Test\n    public void testInvokeImpl() throws ExecutionException, InterruptedException {\n        remotingClient.registerRPCHook(rpcHookMock);\n        Channel channel = new LocalChannel();\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);\n        RemotingCommand response = RemotingCommand.createResponseCommand(null);\n        response.setCode(ResponseCode.SUCCESS);\n        ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() {\n            @Override\n            public void operationComplete(ResponseFuture responseFuture) {\n\n            }\n        }, new SemaphoreReleaseOnlyOnce(new Semaphore(1)));\n        responseFuture.setResponseCommand(response);\n        CompletableFuture<ResponseFuture> future = new CompletableFuture<>();\n        future.complete(responseFuture);\n\n        doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong());\n\n        CompletableFuture<ResponseFuture> future0 = remotingClient.invokeImpl(channel, request, 1000);\n        assertThat(future0.get()).isEqualTo(responseFuture);\n\n        verify(rpcHookMock).doBeforeRequest(anyString(), eq(request));\n        verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response));\n    }\n\n    @Test\n    public void testInvokeImplFail() {\n        remotingClient.registerRPCHook(rpcHookMock);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);\n\n        Channel channel = new LocalChannel();\n        CompletableFuture<ResponseFuture> future = new CompletableFuture<>();\n        future.completeExceptionally(new RemotingException(null));\n\n        doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong());\n\n        assertThatThrownBy(() -> remotingClient.invokeImpl(channel, request, 1000).get()).getCause().isInstanceOf(RemotingException.class);\n\n        verify(rpcHookMock).doBeforeRequest(anyString(), eq(request));\n        verify(rpcHookMock, never()).doAfterResponse(anyString(), eq(request), any());\n    }\n\n    @Test\n    public void testIsAddressReachableFail() throws NoSuchFieldException, IllegalAccessException {\n        Bootstrap bootstrap = spy(Bootstrap.class);\n        Field field = NettyRemotingClient.class.getDeclaredField(\"bootstrap\");\n        field.setAccessible(true);\n        field.set(remotingClient, bootstrap);\n        assertThat(remotingClient.isAddressReachable(\"0.0.0.0:8080\")).isFalse();\n        verify(bootstrap).connect(eq(\"0.0.0.0\"), eq(8080));\n        assertThat(remotingClient.isAddressReachable(\"[fe80::]:8080\")).isFalse();\n        verify(bootstrap).connect(eq(\"[fe80::]\"), eq(8080));\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingServerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\npackage org.apache.rocketmq.remoting.netty;\n\nimport io.netty.buffer.ByteBuf;\nimport io.netty.buffer.Unpooled;\nimport io.netty.channel.Channel;\nimport io.netty.handler.codec.haproxy.HAProxyTLV;\nimport io.netty.util.Attribute;\nimport io.netty.util.AttributeKey;\nimport java.nio.charset.StandardCharsets;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.Mockito.doNothing;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class NettyRemotingServerTest {\n\n    private NettyRemotingServer nettyRemotingServer;\n\n    @Mock\n    private Channel channel;\n\n    @Mock\n    private Attribute attribute;\n\n    @Before\n    public void setUp() throws Exception {\n        NettyServerConfig nettyServerConfig = new NettyServerConfig();\n        nettyRemotingServer = new NettyRemotingServer(nettyServerConfig);\n    }\n\n    @Test\n    public void handleHAProxyTLV() {\n        when(channel.attr(any(AttributeKey.class))).thenReturn(attribute);\n        doNothing().when(attribute).set(any());\n\n        ByteBuf content = Unpooled.buffer();\n        content.writeBytes(\"xxxx\".getBytes(StandardCharsets.UTF_8));\n        HAProxyTLV haProxyTLV = new HAProxyTLV((byte) 0xE1, content);\n        nettyRemotingServer.handleHAProxyTLV(haProxyTLV, channel);\n    }\n}"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyServerConfigTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.netty;\n\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class NettyServerConfigTest {\n\n    @Test\n    public void testChangeConfigBySystemProperty() {\n        System.setProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_BACKLOG, \"65535\");\n        NettySystemConfig.socketBacklog =\n                Integer.parseInt(System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_BACKLOG, \"1024\"));\n        NettyServerConfig changedConfig = new NettyServerConfig();\n        assertThat(changedConfig.getServerSocketBacklog()).isEqualTo(65535);\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/netty/RemotingCodeDistributionHandlerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.netty;\n\nimport java.lang.reflect.Method;\nimport java.time.Duration;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport static org.awaitility.Awaitility.await;\n\npublic class RemotingCodeDistributionHandlerTest {\n\n    private final RemotingCodeDistributionHandler distributionHandler = new RemotingCodeDistributionHandler();\n\n    @Test\n    public void remotingCodeCountTest() throws Exception {\n        Class<RemotingCodeDistributionHandler> clazz = RemotingCodeDistributionHandler.class;\n        Method methodIn = clazz.getDeclaredMethod(\"countInbound\", int.class);\n        Method methodOut = clazz.getDeclaredMethod(\"countOutbound\", int.class);\n        methodIn.setAccessible(true);\n        methodOut.setAccessible(true);\n\n        int threadCount = 4;\n        int count = 1000 * 1000;\n        CountDownLatch latch = new CountDownLatch(threadCount);\n        AtomicBoolean result = new AtomicBoolean(true);\n        ExecutorService executorService = Executors.newFixedThreadPool(threadCount, new ThreadFactoryImpl(\"RemotingCodeTest_\"));\n\n        for (int i = 0; i < threadCount; i++) {\n            executorService.submit(() -> {\n                try {\n                    for (int j = 0; j < count; j++) {\n                        methodIn.invoke(distributionHandler, 1);\n                        methodOut.invoke(distributionHandler, 2);\n                    }\n                } catch (Exception e) {\n                    result.set(false);\n                } finally {\n                    latch.countDown();\n                }\n            });\n        }\n\n        latch.await();\n        Assert.assertTrue(result.get());\n        await().pollInterval(Duration.ofMillis(100)).atMost(Duration.ofSeconds(10)).until(() -> {\n            boolean f1 = (\"{1:\" + count * threadCount + \"}\").equals(distributionHandler.getInBoundSnapshotString());\n            boolean f2 = (\"{2:\" + count * threadCount + \"}\").equals(distributionHandler.getOutBoundSnapshotString());\n            return f1 && f2;\n        });\n    }\n}"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/CheckpointFileTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.nio.file.Paths;\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.utils.CheckpointFile;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\npublic class CheckpointFileTest {\n\n    private static final String FILE_PATH =\n        Paths.get(System.getProperty(\"java.io.tmpdir\"), \"store-test\", \"epoch.ckpt\").toString();\n\n    private List<EpochEntry> entryList;\n    private CheckpointFile<EpochEntry> checkpoint;\n\n    static class EpochEntrySerializer implements CheckpointFile.CheckpointSerializer<EpochEntry> {\n\n        @Override\n        public String toLine(EpochEntry entry) {\n            if (entry != null) {\n                return String.format(\"%d-%d\", entry.getEpoch(), entry.getStartOffset());\n            } else {\n                return null;\n            }\n        }\n\n        @Override\n        public EpochEntry fromLine(String line) {\n            final String[] arr = line.split(\"-\");\n            if (arr.length == 2) {\n                final int epoch = Integer.parseInt(arr[0]);\n                final long startOffset = Long.parseLong(arr[1]);\n                return new EpochEntry(epoch, startOffset);\n            }\n            return null;\n        }\n    }\n\n    @Before\n    public void init() throws IOException {\n        this.entryList = new ArrayList<>();\n        entryList.add(new EpochEntry(7, 7000));\n        entryList.add(new EpochEntry(8, 8000));\n        this.checkpoint = new CheckpointFile<>(FILE_PATH, new EpochEntrySerializer());\n        this.checkpoint.write(entryList);\n    }\n\n    @After\n    public void destroy() {\n        UtilAll.deleteFile(new File(FILE_PATH));\n        UtilAll.deleteFile(new File(FILE_PATH + \".bak\"));\n    }\n\n    @Test\n    public void testNormalWriteAndRead() throws IOException {\n        List<EpochEntry> listFromFile = checkpoint.read();\n        Assert.assertEquals(entryList, listFromFile);\n        checkpoint.write(entryList);\n        listFromFile = checkpoint.read();\n        Assert.assertEquals(entryList, listFromFile);\n    }\n\n    @Test\n    public void testAbNormalWriteAndRead() throws IOException {\n        this.checkpoint.write(entryList);\n        UtilAll.deleteFile(new File(FILE_PATH));\n        List<EpochEntry> listFromFile = checkpoint.read();\n        Assert.assertEquals(entryList, listFromFile);\n        checkpoint.write(entryList);\n        listFromFile = checkpoint.read();\n        Assert.assertEquals(entryList, listFromFile);\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/ClusterInfoTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol;\n\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Set;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.remoting.protocol.body.ClusterInfo;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertArrayEquals;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertTrue;\n\npublic class ClusterInfoTest {\n\n    @Test\n    public void testFormJson() throws Exception {\n        ClusterInfo clusterInfo = buildClusterInfo();\n        byte[] data = clusterInfo.encode();\n        ClusterInfo json = RemotingSerializable.decode(data, ClusterInfo.class);\n\n        assertNotNull(json);\n        assertNotNull(json.getClusterAddrTable());\n        assertTrue(json.getClusterAddrTable().containsKey(\"DEFAULT_CLUSTER\"));\n        assertTrue(json.getClusterAddrTable().get(\"DEFAULT_CLUSTER\").contains(\"master\"));\n        assertNotNull(json.getBrokerAddrTable());\n        assertTrue(json.getBrokerAddrTable().containsKey(\"master\"));\n        assertEquals(json.getBrokerAddrTable().get(\"master\").getBrokerName(), \"master\");\n        assertEquals(json.getBrokerAddrTable().get(\"master\").getCluster(), \"DEFAULT_CLUSTER\");\n        assertEquals(json.getBrokerAddrTable().get(\"master\").getBrokerAddrs().get(MixAll.MASTER_ID), MixAll.getLocalhostByNetworkInterface());\n    }\n\n    @Test\n    public void testRetrieveAllClusterNames() throws Exception {\n        ClusterInfo clusterInfo = buildClusterInfo();\n        byte[] data = clusterInfo.encode();\n        ClusterInfo json = RemotingSerializable.decode(data, ClusterInfo.class);\n\n        assertArrayEquals(new String[]{\"DEFAULT_CLUSTER\"}, json.retrieveAllClusterNames());\n    }\n\n\n    @Test\n    public void testRetrieveAllAddrByCluster() throws Exception {\n        ClusterInfo clusterInfo = buildClusterInfo();\n        byte[] data = clusterInfo.encode();\n        ClusterInfo json = RemotingSerializable.decode(data, ClusterInfo.class);\n\n        assertArrayEquals(new String[]{MixAll.getLocalhostByNetworkInterface()}, json.retrieveAllAddrByCluster(\"DEFAULT_CLUSTER\"));\n    }\n\n\n    private ClusterInfo buildClusterInfo() throws Exception {\n        ClusterInfo clusterInfo = new ClusterInfo();\n        HashMap<String, BrokerData> brokerAddrTable = new HashMap<>();\n        HashMap<String, Set<String>> clusterAddrTable = new HashMap<>();\n\n        //build brokerData\n        BrokerData brokerData = new BrokerData();\n        brokerData.setBrokerName(\"master\");\n        brokerData.setCluster(\"DEFAULT_CLUSTER\");\n\n        //build brokerAddrs\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(MixAll.MASTER_ID, MixAll.getLocalhostByNetworkInterface());\n\n        brokerData.setBrokerAddrs(brokerAddrs);\n        brokerAddrTable.put(\"master\", brokerData);\n\n        Set<String> brokerNames = new HashSet<>();\n        brokerNames.add(\"master\");\n\n        clusterAddrTable.put(\"DEFAULT_CLUSTER\", brokerNames);\n\n        clusterInfo.setBrokerAddrTable(brokerAddrTable);\n        clusterInfo.setClusterAddrTable(clusterAddrTable);\n        return clusterInfo;\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/ConsumeStatusTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol;\n\nimport org.apache.rocketmq.remoting.protocol.body.ConsumeStatus;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.assertj.core.api.Assertions.within;\n\npublic class ConsumeStatusTest {\n\n    @Test\n    public void testFromJson() throws Exception {\n        ConsumeStatus cs = new ConsumeStatus();\n        cs.setConsumeFailedTPS(10);\n        cs.setPullRT(100);\n        cs.setPullTPS(1000);\n        String json = RemotingSerializable.toJson(cs, true);\n        ConsumeStatus fromJson = RemotingSerializable.fromJson(json, ConsumeStatus.class);\n        assertThat(fromJson.getPullRT()).isCloseTo(cs.getPullRT(), within(0.0001));\n        assertThat(fromJson.getPullTPS()).isCloseTo(cs.getPullTPS(), within(0.0001));\n        assertThat(fromJson.getConsumeFailedTPS()).isCloseTo(cs.getConsumeFailedTPS(), within(0.0001));\n    }\n\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/DataVersionTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol;\n\nimport com.alibaba.fastjson2.JSON;\nimport org.junit.Test;\n\nimport java.util.concurrent.atomic.AtomicLong;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertTrue;\n\npublic class DataVersionTest {\n\n    @Test\n    public void testEquals() {\n        DataVersion dataVersion = new DataVersion();\n        DataVersion other = new DataVersion();\n        other.setTimestamp(dataVersion.getTimestamp());\n        assertEquals(dataVersion, other);\n    }\n\n    @Test\n    public void testEquals_falseWhenCounterDifferent() {\n        DataVersion dataVersion = new DataVersion();\n        DataVersion other = new DataVersion();\n        other.setCounter(new AtomicLong(1L));\n        other.setTimestamp(dataVersion.getTimestamp());\n        assertNotEquals(dataVersion, other);\n    }\n\n    @Test\n    public void testEquals_falseWhenCounterDifferent2() {\n        DataVersion dataVersion = new DataVersion();\n        DataVersion other = new DataVersion();\n        other.setCounter(null);\n        other.setTimestamp(dataVersion.getTimestamp());\n        assertNotEquals(dataVersion, other);\n    }\n\n    @Test\n    public void testEquals_falseWhenCounterDifferent3() {\n        DataVersion dataVersion = new DataVersion();\n        dataVersion.setCounter(null);\n        DataVersion other = new DataVersion();\n        other.setTimestamp(dataVersion.getTimestamp());\n        assertNotEquals(dataVersion, other);\n    }\n\n    @Test\n    public void testEquals_trueWhenCountersBothNull() {\n        DataVersion dataVersion = new DataVersion();\n        dataVersion.setCounter(null);\n        DataVersion other = new DataVersion();\n        other.setCounter(null);\n        other.setTimestamp(dataVersion.getTimestamp());\n        assertEquals(dataVersion, other);\n    }\n\n    @Test\n    public void testEncode() {\n        DataVersion dataVersion = new DataVersion();\n        assertTrue(dataVersion.encode().length > 0);\n        assertNotNull(dataVersion.toJson());\n    }\n\n    @Test\n    public void testJsonSerializationAndDeserialization() {\n        DataVersion expected = new DataVersion();\n        expected.setCounter(new AtomicLong(Long.MAX_VALUE));\n        expected.setTimestamp(expected.getTimestamp());\n        String jsonStr = expected.toJson();\n        assertNotNull(jsonStr);\n        DataVersion actual = JSON.parseObject(jsonStr, DataVersion.class);\n        assertNotNull(actual);\n        assertEquals(expected.getTimestamp(), actual.getTimestamp());\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/GroupListTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol;\n\nimport java.util.HashSet;\nimport java.util.UUID;\nimport org.apache.rocketmq.remoting.protocol.body.GroupList;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\n/**\n * Created by guoyao on 2019/2/18.\n */\npublic class GroupListTest {\n\n    @Test\n    public void testSetGet() throws Exception {\n        HashSet<String> fisrtUniqueSet = createUniqueNewSet();\n        HashSet<String> secondUniqueSet = createUniqueNewSet();\n        assertThat(fisrtUniqueSet).isNotEqualTo(secondUniqueSet);\n        GroupList gl = new GroupList();\n        gl.setGroupList(fisrtUniqueSet);\n        assertThat(gl.getGroupList()).isEqualTo(fisrtUniqueSet);\n        assertThat(gl.getGroupList()).isNotEqualTo(secondUniqueSet);\n        gl.setGroupList(secondUniqueSet);\n        assertThat(gl.getGroupList()).isNotEqualTo(fisrtUniqueSet);\n        assertThat(gl.getGroupList()).isEqualTo(secondUniqueSet);\n    }\n\n    private HashSet<String> createUniqueNewSet() {\n        HashSet<String> groups = new HashSet<>();\n        groups.add(UUID.randomUUID().toString());\n        return groups;\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/LanguageCodeTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol;\n\nimport org.junit.Test;\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class LanguageCodeTest {\n\n    @Test\n    public void testLanguageCodeRust() {\n        LanguageCode code = LanguageCode.valueOf((byte) 12);\n        assertThat(code).isEqualTo(LanguageCode.RUST);\n\n        code = LanguageCode.valueOf(\"RUST\");\n        assertThat(code).isEqualTo(LanguageCode.RUST);\n    }\n    \n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/NamespaceUtilTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol;\n\nimport org.apache.rocketmq.common.MixAll;\nimport org.junit.Assert;\nimport org.junit.Test;\n\n/**\n * MQDevelopers\n */\npublic class NamespaceUtilTest {\n\n    private static final String INSTANCE_ID = \"MQ_INST_XXX\";\n    private static final String INSTANCE_ID_WRONG = \"MQ_INST_XXX1\";\n    private static final String TOPIC = \"TOPIC_XXX\";\n    private static final String GROUP_ID = \"GID_XXX\";\n    private static final String SYSTEM_TOPIC = \"rmq_sys_topic\";\n    private static final String GROUP_ID_WITH_NAMESPACE = INSTANCE_ID + NamespaceUtil.NAMESPACE_SEPARATOR + GROUP_ID;\n    private static final String TOPIC_WITH_NAMESPACE = INSTANCE_ID + NamespaceUtil.NAMESPACE_SEPARATOR + TOPIC;\n    private static final String RETRY_TOPIC = MixAll.RETRY_GROUP_TOPIC_PREFIX + GROUP_ID;\n    private static final String RETRY_TOPIC_WITH_NAMESPACE =\n        MixAll.RETRY_GROUP_TOPIC_PREFIX + INSTANCE_ID + NamespaceUtil.NAMESPACE_SEPARATOR + GROUP_ID;\n    private static final String DLQ_TOPIC = MixAll.DLQ_GROUP_TOPIC_PREFIX + GROUP_ID;\n    private static final String DLQ_TOPIC_WITH_NAMESPACE =\n        MixAll.DLQ_GROUP_TOPIC_PREFIX + INSTANCE_ID + NamespaceUtil.NAMESPACE_SEPARATOR + GROUP_ID;\n\n    @Test\n    public void testWithoutNamespace() {\n        String topic = NamespaceUtil.withoutNamespace(TOPIC_WITH_NAMESPACE, INSTANCE_ID);\n        Assert.assertEquals(topic, TOPIC);\n        String topic1 = NamespaceUtil.withoutNamespace(TOPIC_WITH_NAMESPACE);\n        Assert.assertEquals(topic1, TOPIC);\n        String groupId = NamespaceUtil.withoutNamespace(GROUP_ID_WITH_NAMESPACE, INSTANCE_ID);\n        Assert.assertEquals(groupId, GROUP_ID);\n        String groupId1 = NamespaceUtil.withoutNamespace(GROUP_ID_WITH_NAMESPACE);\n        Assert.assertEquals(groupId1, GROUP_ID);\n        String consumerId = NamespaceUtil.withoutNamespace(RETRY_TOPIC_WITH_NAMESPACE, INSTANCE_ID);\n        Assert.assertEquals(consumerId, RETRY_TOPIC);\n        String consumerId1 = NamespaceUtil.withoutNamespace(RETRY_TOPIC_WITH_NAMESPACE);\n        Assert.assertEquals(consumerId1, RETRY_TOPIC);\n        String consumerId2 = NamespaceUtil.withoutNamespace(RETRY_TOPIC_WITH_NAMESPACE, INSTANCE_ID_WRONG);\n        Assert.assertEquals(consumerId2, RETRY_TOPIC_WITH_NAMESPACE);\n        Assert.assertNotEquals(consumerId2, RETRY_TOPIC);\n    }\n\n    @Test\n    public void testWrapNamespace() {\n        String topic1 = NamespaceUtil.wrapNamespace(INSTANCE_ID, TOPIC);\n        Assert.assertEquals(topic1, TOPIC_WITH_NAMESPACE);\n        String topicWithNamespaceAgain = NamespaceUtil.wrapNamespace(INSTANCE_ID, topic1);\n        Assert.assertEquals(topicWithNamespaceAgain, TOPIC_WITH_NAMESPACE);\n        //Wrap retry topic\n        String retryTopicWithNamespace = NamespaceUtil.wrapNamespace(INSTANCE_ID, RETRY_TOPIC);\n        Assert.assertEquals(retryTopicWithNamespace, RETRY_TOPIC_WITH_NAMESPACE);\n        String retryTopicWithNamespaceAgain = NamespaceUtil.wrapNamespace(INSTANCE_ID, retryTopicWithNamespace);\n        Assert.assertEquals(retryTopicWithNamespaceAgain, retryTopicWithNamespace);\n        //Wrap DLQ topic\n        String dlqTopicWithNamespace = NamespaceUtil.wrapNamespace(INSTANCE_ID, DLQ_TOPIC);\n        Assert.assertEquals(dlqTopicWithNamespace, DLQ_TOPIC_WITH_NAMESPACE);\n        String dlqTopicWithNamespaceAgain = NamespaceUtil.wrapNamespace(INSTANCE_ID, dlqTopicWithNamespace);\n        Assert.assertEquals(dlqTopicWithNamespaceAgain, dlqTopicWithNamespace);\n        Assert.assertEquals(dlqTopicWithNamespaceAgain, DLQ_TOPIC_WITH_NAMESPACE);\n        //test system topic\n        String systemTopic = NamespaceUtil.wrapNamespace(INSTANCE_ID, SYSTEM_TOPIC);\n        Assert.assertEquals(systemTopic, SYSTEM_TOPIC);\n    }\n\n    @Test\n    public void testGetNamespaceFromResource() {\n        String namespaceExpectBlank = NamespaceUtil.getNamespaceFromResource(TOPIC);\n        Assert.assertEquals(namespaceExpectBlank, NamespaceUtil.STRING_BLANK);\n        String namespace =  NamespaceUtil.getNamespaceFromResource(TOPIC_WITH_NAMESPACE);\n        Assert.assertEquals(namespace, INSTANCE_ID);\n        String namespaceFromRetryTopic = NamespaceUtil.getNamespaceFromResource(RETRY_TOPIC_WITH_NAMESPACE);\n        Assert.assertEquals(namespaceFromRetryTopic, INSTANCE_ID);\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/QueryConsumeTimeSpanBodyTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Random;\nimport java.util.UUID;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.body.QueryConsumeTimeSpanBody;\nimport org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class QueryConsumeTimeSpanBodyTest {\n\n    @Test\n    public void testSetGet() throws Exception {\n        QueryConsumeTimeSpanBody queryConsumeTimeSpanBody = new QueryConsumeTimeSpanBody();\n        List<QueueTimeSpan> firstQueueTimeSpans = newUniqueConsumeTimeSpanSet();\n        List<QueueTimeSpan> secondQueueTimeSpans = newUniqueConsumeTimeSpanSet();\n        queryConsumeTimeSpanBody.setConsumeTimeSpanSet(firstQueueTimeSpans);\n        assertThat(queryConsumeTimeSpanBody.getConsumeTimeSpanSet()).isEqualTo(firstQueueTimeSpans);\n        assertThat(queryConsumeTimeSpanBody.getConsumeTimeSpanSet()).isNotEqualTo(secondQueueTimeSpans);\n        queryConsumeTimeSpanBody.setConsumeTimeSpanSet(secondQueueTimeSpans);\n        assertThat(queryConsumeTimeSpanBody.getConsumeTimeSpanSet()).isEqualTo(secondQueueTimeSpans);\n        assertThat(queryConsumeTimeSpanBody.getConsumeTimeSpanSet()).isNotEqualTo(firstQueueTimeSpans);\n    }\n\n    @Test\n    public void testFromJson() throws Exception {\n        QueryConsumeTimeSpanBody qctsb = new QueryConsumeTimeSpanBody();\n        List<QueueTimeSpan> queueTimeSpans = new ArrayList<>();\n        QueueTimeSpan queueTimeSpan = new QueueTimeSpan();\n        queueTimeSpan.setMinTimeStamp(1550825710000L);\n        queueTimeSpan.setMaxTimeStamp(1550825790000L);\n        queueTimeSpan.setConsumeTimeStamp(1550825760000L);\n        queueTimeSpan.setDelayTime(5000L);\n        MessageQueue messageQueue = new MessageQueue(\"topicName\", \"brokerName\", 1);\n        queueTimeSpan.setMessageQueue(messageQueue);\n        queueTimeSpans.add(queueTimeSpan);\n        qctsb.setConsumeTimeSpanSet(queueTimeSpans);\n        String json = RemotingSerializable.toJson(qctsb, true);\n        QueryConsumeTimeSpanBody fromJson = RemotingSerializable.fromJson(json, QueryConsumeTimeSpanBody.class);\n        assertThat(fromJson.getConsumeTimeSpanSet().get(0).getMaxTimeStamp()).isEqualTo(1550825790000L);\n        assertThat(fromJson.getConsumeTimeSpanSet().get(0).getMinTimeStamp()).isEqualTo(1550825710000L);\n        assertThat(fromJson.getConsumeTimeSpanSet().get(0).getDelayTime()).isEqualTo(5000L);\n        assertThat(fromJson.getConsumeTimeSpanSet().get(0).getMessageQueue()).isEqualTo(messageQueue);\n    }\n\n    @Test\n    public void testFromJsonRandom() throws Exception {\n        QueryConsumeTimeSpanBody origin = new QueryConsumeTimeSpanBody();\n        List<QueueTimeSpan> queueTimeSpans = newUniqueConsumeTimeSpanSet();\n        origin.setConsumeTimeSpanSet(queueTimeSpans);\n        String json = origin.toJson(true);\n        QueryConsumeTimeSpanBody fromJson = RemotingSerializable.fromJson(json, QueryConsumeTimeSpanBody.class);\n        assertThat(fromJson.getConsumeTimeSpanSet().get(0).getMinTimeStamp()).isEqualTo(origin.getConsumeTimeSpanSet().get(0).getMinTimeStamp());\n        assertThat(fromJson.getConsumeTimeSpanSet().get(0).getMaxTimeStamp()).isEqualTo(origin.getConsumeTimeSpanSet().get(0).getMaxTimeStamp());\n        assertThat(fromJson.getConsumeTimeSpanSet().get(0).getConsumeTimeStamp()).isEqualTo(origin.getConsumeTimeSpanSet().get(0).getConsumeTimeStamp());\n        assertThat(fromJson.getConsumeTimeSpanSet().get(0).getDelayTime()).isEqualTo(origin.getConsumeTimeSpanSet().get(0).getDelayTime());\n        assertThat(fromJson.getConsumeTimeSpanSet().get(0).getMessageQueue().getBrokerName()).isEqualTo(origin.getConsumeTimeSpanSet().get(0).getMessageQueue().getBrokerName());\n        assertThat(fromJson.getConsumeTimeSpanSet().get(0).getMessageQueue().getTopic()).isEqualTo(origin.getConsumeTimeSpanSet().get(0).getMessageQueue().getTopic());\n        assertThat(fromJson.getConsumeTimeSpanSet().get(0).getMessageQueue().getQueueId()).isEqualTo(origin.getConsumeTimeSpanSet().get(0).getMessageQueue().getQueueId());\n    }\n\n    @Test\n    public void testEncode() throws Exception {\n        QueryConsumeTimeSpanBody origin = new QueryConsumeTimeSpanBody();\n        List<QueueTimeSpan> queueTimeSpans = newUniqueConsumeTimeSpanSet();\n        origin.setConsumeTimeSpanSet(queueTimeSpans);\n        byte[] data = origin.encode();\n        QueryConsumeTimeSpanBody fromData = RemotingSerializable.decode(data, QueryConsumeTimeSpanBody.class);\n        assertThat(fromData.getConsumeTimeSpanSet().get(0).getMinTimeStamp()).isEqualTo(origin.getConsumeTimeSpanSet().get(0).getMinTimeStamp());\n        assertThat(fromData.getConsumeTimeSpanSet().get(0).getMaxTimeStamp()).isEqualTo(origin.getConsumeTimeSpanSet().get(0).getMaxTimeStamp());\n        assertThat(fromData.getConsumeTimeSpanSet().get(0).getConsumeTimeStamp()).isEqualTo(origin.getConsumeTimeSpanSet().get(0).getConsumeTimeStamp());\n        assertThat(fromData.getConsumeTimeSpanSet().get(0).getDelayTime()).isEqualTo(origin.getConsumeTimeSpanSet().get(0).getDelayTime());\n        assertThat(fromData.getConsumeTimeSpanSet().get(0).getMessageQueue().getBrokerName()).isEqualTo(origin.getConsumeTimeSpanSet().get(0).getMessageQueue().getBrokerName());\n        assertThat(fromData.getConsumeTimeSpanSet().get(0).getMessageQueue().getTopic()).isEqualTo(origin.getConsumeTimeSpanSet().get(0).getMessageQueue().getTopic());\n        assertThat(fromData.getConsumeTimeSpanSet().get(0).getMessageQueue().getQueueId()).isEqualTo(origin.getConsumeTimeSpanSet().get(0).getMessageQueue().getQueueId());\n    }\n\n    private List<QueueTimeSpan> newUniqueConsumeTimeSpanSet() {\n        List<QueueTimeSpan> queueTimeSpans = new ArrayList<>();\n        QueueTimeSpan queueTimeSpan = new QueueTimeSpan();\n        queueTimeSpan.setMinTimeStamp(System.currentTimeMillis());\n        queueTimeSpan.setMaxTimeStamp(UtilAll.computeNextHourTimeMillis());\n        queueTimeSpan.setConsumeTimeStamp(UtilAll.computeNextMinutesTimeMillis());\n        queueTimeSpan.setDelayTime(5000L);\n        MessageQueue messageQueue = new MessageQueue(UUID.randomUUID().toString(), UUID.randomUUID().toString(), new Random().nextInt());\n        queueTimeSpan.setMessageQueue(messageQueue);\n        queueTimeSpans.add(queueTimeSpan);\n        return queueTimeSpans;\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RegisterBrokerBodyTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol;\n\nimport java.io.IOException;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\n\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.remoting.protocol.body.RegisterBrokerBody;\nimport org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\n\npublic class RegisterBrokerBodyTest {\n    @Test\n    public void test_encode_decode() throws IOException {\n        RegisterBrokerBody registerBrokerBody = new RegisterBrokerBody();\n        TopicConfigAndMappingSerializeWrapper topicConfigSerializeWrapper = new TopicConfigAndMappingSerializeWrapper();\n        registerBrokerBody.setTopicConfigSerializeWrapper(topicConfigSerializeWrapper);\n\n        ConcurrentMap<String, TopicConfig> topicConfigTable = new ConcurrentHashMap<>();\n        for (int i = 0; i < 10000; i++) {\n            topicConfigTable.put(String.valueOf(i), new TopicConfig(String.valueOf(i)));\n        }\n\n        topicConfigSerializeWrapper.setTopicConfigTable(topicConfigTable);\n\n        byte[] compareEncode = registerBrokerBody.encode(true);\n        byte[] encode2 = registerBrokerBody.encode(false);\n        RegisterBrokerBody decodeRegisterBrokerBody = RegisterBrokerBody.decode(compareEncode, true, MQVersion.Version.V5_0_0);\n\n        assertEquals(registerBrokerBody.getTopicConfigSerializeWrapper().getTopicConfigTable().size(), decodeRegisterBrokerBody.getTopicConfigSerializeWrapper().getTopicConfigTable().size());\n\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RemotingCommandTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol;\n\nimport java.lang.reflect.Field;\nimport java.lang.reflect.Method;\nimport java.nio.ByteBuffer;\nimport java.util.HashSet;\nimport java.util.Set;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.annotation.CFNotNull;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class RemotingCommandTest {\n    @Test\n    public void testMarkProtocolType_JSONProtocolType() {\n        int source = 261;\n        SerializeType type = SerializeType.JSON;\n\n        byte[] result = new byte[4];\n        int x = RemotingCommand.markProtocolType(source, type);\n        result[0] = (byte) (x >> 24);\n        result[1] = (byte) (x >> 16);\n        result[2] = (byte) (x >> 8);\n        result[3] = (byte) x;\n        assertThat(result).isEqualTo(new byte[] {0, 0, 1, 5});\n    }\n\n    @Test\n    public void testMarkProtocolType_ROCKETMQProtocolType() {\n        int source = 16777215;\n        SerializeType type = SerializeType.ROCKETMQ;\n        byte[] result = new byte[4];\n        int x = RemotingCommand.markProtocolType(source, type);\n        result[0] = (byte) (x >> 24);\n        result[1] = (byte) (x >> 16);\n        result[2] = (byte) (x >> 8);\n        result[3] = (byte) x;\n        assertThat(result).isEqualTo(new byte[] {1, -1, -1, -1});\n    }\n\n    @Test\n    public void testCreateRequestCommand_RegisterBroker() {\n        System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, \"2333\");\n\n        int code = 103; //org.apache.rocketmq.remoting.protocol.RequestCode.REGISTER_BROKER\n        CommandCustomHeader header = new SampleCommandCustomHeader();\n        RemotingCommand cmd = RemotingCommand.createRequestCommand(code, header);\n        assertThat(cmd.getCode()).isEqualTo(code);\n        assertThat(cmd.getVersion()).isEqualTo(2333);\n        assertThat(cmd.getFlag() & 0x01).isEqualTo(0); //flag bit 0: 0 presents request\n    }\n\n    @Test\n    public void testCreateResponseCommand_SuccessWithHeader() {\n        System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, \"2333\");\n\n        int code = RemotingSysResponseCode.SUCCESS;\n        String remark = \"Sample remark\";\n        RemotingCommand cmd = RemotingCommand.createResponseCommand(code, remark, SampleCommandCustomHeader.class);\n        assertThat(cmd.getCode()).isEqualTo(code);\n        assertThat(cmd.getVersion()).isEqualTo(2333);\n        assertThat(cmd.getRemark()).isEqualTo(remark);\n        assertThat(cmd.getFlag() & 0x01).isEqualTo(1); //flag bit 0: 1 presents response\n    }\n\n    @Test\n    public void testCreateResponseCommand_SuccessWithoutHeader() {\n        System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, \"2333\");\n\n        int code = RemotingSysResponseCode.SUCCESS;\n        String remark = \"Sample remark\";\n        RemotingCommand cmd = RemotingCommand.createResponseCommand(code, remark);\n        assertThat(cmd.getCode()).isEqualTo(code);\n        assertThat(cmd.getVersion()).isEqualTo(2333);\n        assertThat(cmd.getRemark()).isEqualTo(remark);\n        assertThat(cmd.getFlag() & 0x01).isEqualTo(1); //flag bit 0: 1 presents response\n    }\n\n    @Test\n    public void testCreateResponseCommand_FailToCreateCommand() {\n        System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, \"2333\");\n\n        int code = RemotingSysResponseCode.SUCCESS;\n        String remark = \"Sample remark\";\n        RemotingCommand cmd = RemotingCommand.createResponseCommand(code, remark, CommandCustomHeader.class);\n        assertThat(cmd).isNull();\n    }\n\n    @Test\n    public void testCreateResponseCommand_SystemError() {\n        System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, \"2333\");\n\n        RemotingCommand cmd = RemotingCommand.createResponseCommand(SampleCommandCustomHeader.class);\n        assertThat(cmd.getCode()).isEqualTo(RemotingSysResponseCode.SYSTEM_ERROR);\n        assertThat(cmd.getVersion()).isEqualTo(2333);\n        assertThat(cmd.getRemark()).contains(\"not set any response code\");\n        assertThat(cmd.getFlag() & 0x01).isEqualTo(1); //flag bit 0: 1 presents response\n    }\n\n    @Test\n    public void testEncodeAndDecode_EmptyBody() {\n        System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, \"2333\");\n\n        int code = 103; //org.apache.rocketmq.remoting.protocol.RequestCode.REGISTER_BROKER\n        CommandCustomHeader header = new SampleCommandCustomHeader();\n        RemotingCommand cmd = RemotingCommand.createRequestCommand(code, header);\n\n        ByteBuffer buffer = cmd.encode();\n\n        //Simulate buffer being read in NettyDecoder\n        buffer.getInt();\n        byte[] bytes = new byte[buffer.limit() - 4];\n        buffer.get(bytes, 0, buffer.limit() - 4);\n        buffer = ByteBuffer.wrap(bytes);\n\n        RemotingCommand decodedCommand = null;\n        try {\n            decodedCommand = RemotingCommand.decode(buffer);\n\n            assertThat(decodedCommand.getSerializeTypeCurrentRPC()).isEqualTo(SerializeType.JSON);\n            assertThat(decodedCommand.getBody()).isNull();\n        } catch (RemotingCommandException e) {\n            e.printStackTrace();\n            Assert.fail(\"Should not throw IOException\");\n        }\n\n    }\n\n    @Test\n    public void testEncodeAndDecode_FilledBody() {\n        System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, \"2333\");\n\n        int code = 103; //org.apache.rocketmq.remoting.protocol.RequestCode.REGISTER_BROKER\n        CommandCustomHeader header = new SampleCommandCustomHeader();\n        RemotingCommand cmd = RemotingCommand.createRequestCommand(code, header);\n        cmd.setBody(new byte[] {0, 1, 2, 3, 4});\n\n        ByteBuffer buffer = cmd.encode();\n\n        //Simulate buffer being read in NettyDecoder\n        buffer.getInt();\n        byte[] bytes = new byte[buffer.limit() - 4];\n        buffer.get(bytes, 0, buffer.limit() - 4);\n        buffer = ByteBuffer.wrap(bytes);\n\n        RemotingCommand decodedCommand = null;\n        try {\n            decodedCommand = RemotingCommand.decode(buffer);\n\n            assertThat(decodedCommand.getSerializeTypeCurrentRPC()).isEqualTo(SerializeType.JSON);\n            assertThat(decodedCommand.getBody()).isEqualTo(new byte[] {0, 1, 2, 3, 4});\n        } catch (RemotingCommandException e) {\n            e.printStackTrace();\n            Assert.fail(\"Should not throw IOException\");\n        }\n    }\n\n    @Test\n    public void testEncodeAndDecode_FilledBodyWithExtFields() throws RemotingCommandException {\n        System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, \"2333\");\n\n        int code = 103; //org.apache.rocketmq.remoting.protocol.RequestCode.REGISTER_BROKER\n        CommandCustomHeader header = new ExtFieldsHeader();\n        RemotingCommand cmd = RemotingCommand.createRequestCommand(code, header);\n\n        cmd.addExtField(\"key\", \"value\");\n\n        ByteBuffer buffer = cmd.encode();\n\n        //Simulate buffer being read in NettyDecoder\n        buffer.getInt();\n        byte[] bytes = new byte[buffer.limit() - 4];\n        buffer.get(bytes, 0, buffer.limit() - 4);\n        buffer = ByteBuffer.wrap(bytes);\n\n        RemotingCommand decodedCommand = null;\n        try {\n            decodedCommand = RemotingCommand.decode(buffer);\n\n            assertThat(decodedCommand.getExtFields().get(\"stringValue\")).isEqualTo(\"bilibili\");\n            assertThat(decodedCommand.getExtFields().get(\"intValue\")).isEqualTo(\"2333\");\n            assertThat(decodedCommand.getExtFields().get(\"longValue\")).isEqualTo(\"23333333\");\n            assertThat(decodedCommand.getExtFields().get(\"booleanValue\")).isEqualTo(\"true\");\n            assertThat(decodedCommand.getExtFields().get(\"doubleValue\")).isEqualTo(\"0.618\");\n\n            assertThat(decodedCommand.getExtFields().get(\"key\")).isEqualTo(\"value\");\n\n            CommandCustomHeader decodedHeader = decodedCommand.decodeCommandCustomHeader(ExtFieldsHeader.class);\n            assertThat(((ExtFieldsHeader) decodedHeader).getStringValue()).isEqualTo(\"bilibili\");\n            assertThat(((ExtFieldsHeader) decodedHeader).getIntValue()).isEqualTo(2333);\n            assertThat(((ExtFieldsHeader) decodedHeader).getLongValue()).isEqualTo(23333333L);\n            assertThat(((ExtFieldsHeader) decodedHeader).isBooleanValue()).isEqualTo(true);\n            assertThat(((ExtFieldsHeader) decodedHeader).getDoubleValue()).isBetween(0.617, 0.619);\n        } catch (RemotingCommandException e) {\n            e.printStackTrace();\n\n            Assert.fail(\"Should not throw IOException\");\n        }\n\n    }\n\n    @Test\n    public void testNotNullField() throws Exception {\n        RemotingCommand remotingCommand = new RemotingCommand();\n        Method method = RemotingCommand.class.getDeclaredMethod(\"isFieldNullable\", Field.class);\n        method.setAccessible(true);\n\n        Field nullString = FieldTestClass.class.getDeclaredField(\"nullString\");\n        assertThat(method.invoke(remotingCommand, nullString)).isEqualTo(false);\n\n        Field nullableString = FieldTestClass.class.getDeclaredField(\"nullable\");\n        assertThat(method.invoke(remotingCommand, nullableString)).isEqualTo(true);\n\n        Field value = FieldTestClass.class.getDeclaredField(\"value\");\n        assertThat(method.invoke(remotingCommand, value)).isEqualTo(false);\n    }\n\n    @Test\n    public void testParentField() throws Exception {\n        SubExtFieldsHeader subExtFieldsHeader = new SubExtFieldsHeader();\n        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(1, subExtFieldsHeader);\n        Field[] fields  = remotingCommand.getClazzFields(subExtFieldsHeader.getClass());\n        Set<String> fieldNames = new HashSet<>();\n        for (Field field: fields) {\n            fieldNames.add(field.getName());\n        }\n        Assert.assertTrue(fields.length >= 7);\n        Set<String> names = new HashSet<>();\n        names.add(\"stringValue\");\n        names.add(\"intValue\");\n        names.add(\"longValue\");\n        names.add(\"booleanValue\");\n        names.add(\"doubleValue\");\n        names.add(\"name\");\n        names.add(\"value\");\n        for (String name: names) {\n            Assert.assertTrue(fieldNames.contains(name));\n        }\n        remotingCommand.makeCustomHeaderToNet();\n        SubExtFieldsHeader other = (SubExtFieldsHeader) remotingCommand.decodeCommandCustomHeader(subExtFieldsHeader.getClass());\n        Assert.assertEquals(other, subExtFieldsHeader);\n    }\n}\n\nclass FieldTestClass {\n    @CFNotNull\n    String nullString = null;\n\n    String nullable = null;\n\n    @CFNotNull\n    String value = \"NotNull\";\n}\n\nclass SampleCommandCustomHeader implements CommandCustomHeader {\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n}\n\nclass ExtFieldsHeader implements CommandCustomHeader {\n    private String stringValue = \"bilibili\";\n    private int intValue = 2333;\n    private long longValue = 23333333L;\n    private boolean booleanValue = true;\n    private double doubleValue = 0.618;\n\n    @Override\n    public void checkFields() throws RemotingCommandException {\n    }\n\n    public String getStringValue() {\n        return stringValue;\n    }\n\n    public int getIntValue() {\n        return intValue;\n    }\n\n    public long getLongValue() {\n        return longValue;\n    }\n\n    public boolean isBooleanValue() {\n        return booleanValue;\n    }\n\n    public double getDoubleValue() {\n        return doubleValue;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) return true;\n        if (!(o instanceof ExtFieldsHeader)) return false;\n\n        ExtFieldsHeader that = (ExtFieldsHeader) o;\n\n        if (intValue != that.intValue) return false;\n        if (longValue != that.longValue) return false;\n        if (booleanValue != that.booleanValue) return false;\n        if (Double.compare(that.doubleValue, doubleValue) != 0) return false;\n        return stringValue != null ? stringValue.equals(that.stringValue) : that.stringValue == null;\n    }\n\n    @Override\n    public int hashCode() {\n        int result;\n        long temp;\n        result = stringValue != null ? stringValue.hashCode() : 0;\n        result = 31 * result + intValue;\n        result = 31 * result + (int) (longValue ^ (longValue >>> 32));\n        result = 31 * result + (booleanValue ? 1 : 0);\n        temp = Double.doubleToLongBits(doubleValue);\n        result = 31 * result + (int) (temp ^ (temp >>> 32));\n        return result;\n    }\n}\n\n\nclass SubExtFieldsHeader extends ExtFieldsHeader {\n    private String name = \"12321\";\n    private int value = 111;\n    public String getName() {\n        return name;\n    }\n\n    public void setName(String name) {\n        this.name = name;\n    }\n\n    public int getValue() {\n        return value;\n    }\n\n    public void setValue(int value) {\n        this.value = value;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) return true;\n        if (!(o instanceof SubExtFieldsHeader)) return false;\n        if (!super.equals(o)) return false;\n\n        SubExtFieldsHeader that = (SubExtFieldsHeader) o;\n\n        if (value != that.value) return false;\n        return name != null ? name.equals(that.name) : that.name == null;\n    }\n\n    @Override\n    public int hashCode() {\n        int result = super.hashCode();\n        result = 31 * result + (name != null ? name.hashCode() : 0);\n        result = 31 * result + value;\n        return result;\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RemotingSerializableCompatTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol;\n\nimport com.alibaba.fastjson.annotation.JSONField;\nimport com.alibaba.fastjson2.JSON;\nimport org.apache.rocketmq.remoting.protocol.body.BatchAck;\nimport org.junit.Test;\nimport org.objenesis.ObjenesisStd;\nimport org.reflections.Reflections;\n\nimport java.lang.reflect.Array;\nimport java.lang.reflect.Field;\nimport java.lang.reflect.Modifier;\nimport java.util.ArrayList;\nimport java.util.BitSet;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Properties;\nimport java.util.Random;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.atomic.AtomicLong;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\n\npublic class RemotingSerializableCompatTest {\n    \n    @Test\n    public void testCompatibilityCheck() {\n        Reflections reflections = new Reflections(\"org.apache.rocketmq.remoting.protocol\");\n        Set<Class<? extends RemotingSerializable>> subTypes = reflections.getSubTypesOf(RemotingSerializable.class);\n        \n        for (Class<? extends RemotingSerializable> clazz : subTypes) {\n            if (clazz.isInterface() || Modifier.isAbstract(clazz.getModifiers()) || clazz.getSimpleName().endsWith(\"Test\")\n                    || clazz.isAnonymousClass() || clazz.getName().contains(\"$\")) {\n                continue;\n            }\n            try {\n                RemotingSerializable instance;\n                try {\n                    instance = clazz.getDeclaredConstructor().newInstance();\n                } catch (NoSuchMethodException e) {\n                    instance = allocateInstance(clazz);\n                }\n                fillDefaultFields(instance, clazz);\n                assertTrue(checkCompatible(instance, clazz));\n            } catch (Exception e) {\n                System.err.printf(\"Class %s: incompatible, error: %s\\n\", clazz.getName(), e.getMessage());\n            }\n        }\n    }\n\n    @Test\n    public void testCompatibilityCheckWithBitSet() {\n        BitSet bitSet = new BitSet();\n        bitSet.set(1);\n        bitSet.set(3);\n        bitSet.set(5);\n        String fastjson1Str = \"{\\\"b\\\":\\\"Kg==\\\",\\\"c\\\":\\\"DEFAULT_CONSUMER\\\",\\\"it\\\":5000,\\\"pt\\\":1760694281326,\\\"q\\\":1,\\\"r\\\":\\\"0\\\",\\\"rq\\\":2,\\\"so\\\":100,\\\"t\\\":\\\"myTopic\\\"}\";\n        BatchAck batchAck = JSON.parseObject(fastjson1Str, BatchAck.class);\n        assertEquals(bitSet, batchAck.getBitSet());\n        assertEquals(\"DEFAULT_CONSUMER\", batchAck.getConsumerGroup());\n        assertEquals(5000, batchAck.getInvisibleTime());\n        assertEquals(1760694281326L, batchAck.getPopTime());\n        assertEquals(1, batchAck.getQueueId());\n        assertEquals(\"0\", batchAck.getRetry());\n        assertEquals(2, batchAck.getReviveQueueId());\n        assertEquals(100, batchAck.getStartOffset());\n        assertEquals(\"myTopic\", batchAck.getTopic());\n    }\n    \n    private void fillDefaultFields(final Object obj, final Class<?> clazz) throws Exception {\n        if (null == clazz || clazz == Object.class) {\n            return;\n        }\n        for (Field field : clazz.getDeclaredFields()) {\n            if (Modifier.isStatic(field.getModifiers())) {\n                continue;\n            }\n            field.setAccessible(true);\n            Class<?> type = field.getType();\n            \n            if (type.isArray()) {\n                Class<?> componentType = type.getComponentType();\n                Object arr = Array.newInstance(componentType, 1);\n                Object element = createElementOrDefault(componentType);\n                if (element != null) {\n                    Array.set(arr, 0, element);\n                }\n                field.set(obj, arr);\n            } else if (Properties.class.isAssignableFrom(type)) {\n                field.set(obj, new Properties());\n            } else if (type.isEnum()) {\n                Object[] enumConstants = type.getEnumConstants();\n                if (enumConstants != null && enumConstants.length > 0) {\n                    field.set(obj, enumConstants[0]);\n                }\n            } else if (ConcurrentHashMap.KeySetView.class.isAssignableFrom(type)) {\n                field.set(obj, ConcurrentHashMap.newKeySet());\n            } else if (ConcurrentHashMap.class.isAssignableFrom(type) || ConcurrentMap.class.isAssignableFrom(type)) {\n                field.set(obj, new ConcurrentHashMap<>());\n            } else if (Set.class.isAssignableFrom(type)) {\n                Set<Object> set = type.isInterface() ? new HashSet<>() : (Set<Object>) type.getDeclaredConstructor().newInstance();\n                Class<?> genericType = getFirstGenericType(field);\n                Object element = createElementOrDefault(genericType);\n                if (element != null)\n                    set.add(element);\n                field.set(obj, set);\n            } else if (List.class.isAssignableFrom(type)) {\n                List<Object> list = new ArrayList<>();\n                Class<?> genericType = getFirstGenericType(field);\n                Object element = createElementOrDefault(genericType);\n                if (null != element) {\n                    list.add(element);\n                }\n                field.set(obj, list);\n            } else if (Map.class.isAssignableFrom(type)) {\n                Map<Object, Object> map = type.isInterface() ? new HashMap<>() : (Map<Object, Object>) type.getDeclaredConstructor().newInstance();\n                Class<?> keyType = getGenericType(field, 0);\n                Class<?> valueType = getGenericType(field, 1);\n                Object key = createElementOrDefault(keyType);\n                Object value = createElementOrDefault(valueType);\n                if (null != key && null != value) {\n                    map.put(key, value);\n                }\n                field.set(obj, map);\n            } else if (type == AtomicLong.class) {\n                field.set(obj, new AtomicLong(1));\n            } else {\n                Object value = getDefaultValue(type);\n                if (null != value) {\n                    field.set(obj, value);\n                } else if (!type.isPrimitive() && !type.getName().startsWith(\"java.\")) {\n                    Object subObj;\n                    try {\n                        subObj = type.getDeclaredConstructor().newInstance();\n                    } catch (NoSuchMethodException e) {\n                        subObj = allocateInstance(type);\n                    }\n                    fillDefaultFields(subObj, type);\n                    field.set(obj, subObj);\n                }\n            }\n        }\n        fillDefaultFields(obj, clazz.getSuperclass());\n    }\n    \n    private Object createElementOrDefault(final Class<?> type) throws Exception {\n        if (null == type) {\n            return null;\n        }\n        Object value = getDefaultValue(type);\n        if (null != value) {\n            return value;\n        }\n        if (type.isEnum()) {\n            Object[] enumConstants = type.getEnumConstants();\n            if (null != enumConstants && enumConstants.length > 0) {\n                return enumConstants[0];\n            }\n            return null;\n        }\n        if (type.isArray()) {\n            Class<?> componentType = type.getComponentType();\n            Object arr = Array.newInstance(componentType, 1);\n            Object element = createElementOrDefault(componentType);\n            if (null != element) {\n                Array.set(arr, 0, element);\n            }\n            return arr;\n        }\n        if (!type.isPrimitive()) {\n            Object obj;\n            try {\n                obj = type.getDeclaredConstructor().newInstance();\n            } catch (NoSuchMethodException e) {\n                obj = allocateInstance(type);\n            }\n            fillDefaultFields(obj, type);\n            return obj;\n        }\n        return null;\n    }\n    \n    private Class<?> getFirstGenericType(final Field field) {\n        return getGenericType(field, 0);\n    }\n    \n    private Class<?> getGenericType(final Field field, final int index) {\n        try {\n            java.lang.reflect.Type genericType = field.getGenericType();\n            if (genericType instanceof java.lang.reflect.ParameterizedType) {\n                java.lang.reflect.Type[] types = ((java.lang.reflect.ParameterizedType) genericType).getActualTypeArguments();\n                if (types.length > index && types[index] instanceof Class) {\n                    return (Class<?>) types[index];\n                }\n            }\n        } catch (Exception ignored) {\n        }\n        return null;\n    }\n    \n    private Object getDefaultValue(final Class<?> type) {\n        if (null == type) {\n            return null;\n        }\n        if (type == boolean.class || type == Boolean.class) {\n            return false;\n        }\n        if (type == byte.class || type == Byte.class) {\n            return (byte) 1;\n        }\n        if (type == short.class || type == Short.class) {\n            return (short) 1;\n        }\n        if (type == int.class || type == Integer.class) {\n            return 1;\n        }\n        if (type == long.class || type == Long.class) {\n            return 1L;\n        }\n        if (type == float.class || type == Float.class) {\n            return 1f;\n        }\n        if (type == double.class || type == Double.class) {\n            return 1d;\n        }\n        if (type == char.class || type == Character.class) {\n            return '\\0';\n        }\n        if (type == String.class) {\n            return \"test\";\n        }\n        return null;\n    }\n    \n    private boolean checkCompatible(final Object original, final Object deserialized, final String path, final Map<Object, Object> visited) {\n        if (null == original && null == deserialized) {\n            return true;\n        }\n        if (null == original || null == deserialized) {\n            System.err.printf(\"Objects at %s incompatible: one is null\\n\", path);\n            return false;\n        }\n        \n        if (!isPrimitiveOrWrapper(original.getClass())) {\n            if (visited.containsKey(original)) {\n                return true;\n            }\n            visited.put(original, deserialized);\n        }\n        \n        Class<?> clazz = original.getClass();\n        boolean result = true;\n        for (Field field : clazz.getDeclaredFields()) {\n            if (Modifier.isStatic(field.getModifiers())) {\n                continue;\n            }\n            JSONField jsonField = field.getAnnotation(JSONField.class);\n            if (null != jsonField && !jsonField.serialize()) {\n                continue;\n            }\n            if (\"hash\".equals(field.getName()) || \"serialVersionUID\".equals(field.getName())) {\n                continue;\n            }\n            \n            field.setAccessible(true);\n            try {\n                Object v1 = field.get(original);\n                Object v2 = field.get(deserialized);\n                String fieldPath = path + \".\" + field.getName();\n                \n                if (null == v1 && null == v2) {\n                    continue;\n                }\n                if (v1 instanceof Random && v2 instanceof Random) {\n                    continue;\n                }\n                if (v1 instanceof AtomicLong && v2 instanceof AtomicLong) {\n                    if (((AtomicLong) v1).get() != ((AtomicLong) v2).get()) {\n                        result = false;\n                        System.err.printf(\"Field %s incompatible: original=%s, deserialized=%s\\n\", fieldPath, v1, v2);\n                    }\n                    continue;\n                }\n                if (v1 instanceof Set && v2 instanceof Set) {\n                    Set<?> s1 = (Set<?>) v1, s2 = (Set<?>) v2;\n                    if (s1.size() != s2.size()) {\n                        result = false;\n                        System.err.printf(\"Field %s incompatible: set size original=%d, deserialized=%d\\n\", fieldPath, s1.size(), s2.size());\n                    } else if (!s1.isEmpty()) {\n                        List<?> list1 = new ArrayList<>(s1);\n                        List<?> list2 = new ArrayList<>(s2);\n                        if (new HashSet<>(list1).equals(new HashSet<>(list2))) {\n                            continue;\n                        }\n                        boolean elementsCompatible = true;\n                        for (Object e1 : list1) {\n                            boolean foundMatch = false;\n                            for (Object e2 : list2) {\n                                if (checkCompatible(e1, e2, fieldPath + \".element\", new HashMap<>(visited))) {\n                                    foundMatch = true;\n                                    break;\n                                }\n                            }\n                            if (!foundMatch) {\n                                elementsCompatible = false;\n                                break;\n                            }\n                        }\n                        if (!elementsCompatible) {\n                            result = false;\n                            System.err.printf(\"Field %s incompatible: sets have different elements\\n\", fieldPath);\n                        }\n                    }\n                    continue;\n                }\n                if (v1 instanceof List && v2 instanceof List) {\n                    List<?> l1 = (List<?>) v1, l2 = (List<?>) v2;\n                    if (l1.size() != l2.size()) {\n                        result = false;\n                        System.err.printf(\"Field %s incompatible: list size original=%d, deserialized=%d\\n\", fieldPath, l1.size(), l2.size());\n                    } else {\n                        for (int i = 0; i < l1.size(); i++) {\n                            Object e1 = l1.get(i);\n                            Object e2 = l2.get(i);\n                            if (!checkCompatible(e1, e2, fieldPath + \"[\" + i + \"]\", new HashMap<>(visited))) {\n                                result = false;\n                            }\n                        }\n                    }\n                    continue;\n                }\n                if (v1 instanceof Map && v2 instanceof Map) {\n                    Map<?, ?> m1 = (Map<?, ?>) v1, m2 = (Map<?, ?>) v2;\n                    if (!m1.keySet().equals(m2.keySet())) {\n                        result = false;\n                        System.err.printf(\"Field %s incompatible: map keys original=%s, deserialized=%s\\n\", fieldPath, m1.keySet(), m2.keySet());\n                    } else {\n                        for (Object key : m1.keySet()) {\n                            Object val1 = m1.get(key), val2 = m2.get(key);\n                            if (val1 != null && val2 != null) {\n                                if (!checkCompatible(val1, val2, fieldPath + \"[\" + key + \"]\", new HashMap<>(visited))) {\n                                    result = false;\n                                }\n                            } else if (val1 != val2) {\n                                result = false;\n                                System.err.printf(\"Field %s key %s incompatible: original=%s, deserialized=%s\\n\",\n                                        fieldPath, key, val1, val2);\n                            }\n                        }\n                    }\n                    continue;\n                }\n                Class<?> type = field.getType();\n                if (null != v1 && null != v2 && !type.isPrimitive() && !type.getName().startsWith(\"java.\")) {\n                    if (!checkCompatible(v1, v2, fieldPath, new HashMap<>(visited))) {\n                        result = false;\n                    }\n                    continue;\n                }\n                if (null == v1 || null == v2 || !v1.equals(v2)) {\n                    result = false;\n                    System.err.printf(\"Field %s incompatible: original=%s, deserialized=%s\\n\", fieldPath, v1, v2);\n                }\n            } catch (Exception e) {\n                result = false;\n                System.err.printf(\"Field %s error: %s\\n\", path + \".\" + field.getName(), e.getMessage());\n            }\n        }\n        if (result) {\n            System.out.printf(\"Class %s compatible\\n\", path);\n        }\n        return result;\n    }\n    \n    private boolean isPrimitiveOrWrapper(final Class<?> clazz) {\n        return clazz.isPrimitive() ||\n                clazz == String.class ||\n                clazz == Boolean.class ||\n                clazz == Character.class ||\n                clazz == Byte.class ||\n                clazz == Short.class ||\n                clazz == Integer.class ||\n                clazz == Long.class ||\n                clazz == Float.class ||\n                clazz == Double.class;\n    }\n    \n    private boolean checkCompatible(final Object original, final Class<?> clazz) {\n        String json = com.alibaba.fastjson.JSON.toJSONString(original);\n        Object deserialized;\n        try {\n            deserialized = com.alibaba.fastjson2.JSON.parseObject(json, clazz);\n        } catch (Exception e) {\n            System.err.printf(\"Deserialization failed for %s: %s\\n\", clazz.getName(), e.getMessage());\n            return false;\n        }\n        return checkCompatible(original, deserialized, clazz.getSimpleName(), new HashMap<>());\n    }\n    \n    private <T> T allocateInstance(final Class<T> clazz) {\n        return new ObjenesisStd().newInstance(clazz);\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RemotingSerializableTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol;\n\nimport com.alibaba.fastjson2.JSONWriter;\nimport com.google.gson.Gson;\nimport com.google.gson.JsonElement;\nimport com.google.gson.TypeAdapter;\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport java.io.IOException;\nimport java.nio.charset.Charset;\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class RemotingSerializableTest {\n    @Test\n    public void testEncodeAndDecode_HeterogeneousClass() {\n        Sample sample = new Sample();\n\n        byte[] bytes = RemotingSerializable.encode(sample);\n        Sample decodedSample = RemotingSerializable.decode(bytes, Sample.class);\n\n        assertThat(decodedSample).isEqualTo(sample);\n    }\n\n    @Test\n    public void testToJson_normalString() {\n        RemotingSerializable serializable = new RemotingSerializable() {\n            private List<String> stringList = Arrays.asList(\"a\", \"o\", \"e\", \"i\", \"u\", \"v\");\n\n            public List<String> getStringList() {\n                return stringList;\n            }\n\n            public void setStringList(List<String> stringList) {\n                this.stringList = stringList;\n            }\n        };\n\n        String string = serializable.toJson();\n\n        assertThat(string).isEqualTo(\"{\\\"stringList\\\":[\\\"a\\\",\\\"o\\\",\\\"e\\\",\\\"i\\\",\\\"u\\\",\\\"v\\\"]}\");\n    }\n\n    @Test\n    public void testToJson_prettyString() {\n        RemotingSerializable serializable = new RemotingSerializable() {\n            private List<String> stringList = Arrays.asList(\"a\", \"o\", \"e\", \"i\", \"u\", \"v\");\n\n            public List<String> getStringList() {\n                return stringList;\n            }\n\n            public void setStringList(List<String> stringList) {\n                this.stringList = stringList;\n            }\n        };\n\n        String prettyString = serializable.toJson(true);\n\n        assertThat(prettyString).isEqualTo(\"{\\n\" +\n            \"\\t\\\"stringList\\\":[\\n\" +\n            \"\\t\\t\\\"a\\\",\\n\" +\n            \"\\t\\t\\\"o\\\",\\n\" +\n            \"\\t\\t\\\"e\\\",\\n\" +\n            \"\\t\\t\\\"i\\\",\\n\" +\n            \"\\t\\t\\\"u\\\",\\n\" +\n            \"\\t\\t\\\"v\\\"\\n\" +\n            \"\\t]\\n\" +\n            \"}\");\n    }\n\n    @Test\n    public void testEncode() {\n        class Foo extends RemotingSerializable {\n            Map<Long, String> map = new HashMap<>();\n\n            Foo() {\n                map.put(0L, \"Test\");\n            }\n\n            public Map<Long, String> getMap() {\n                return map;\n            }\n        }\n        Foo foo = new Foo();\n        String invalid = new String(foo.encode(), Charset.defaultCharset());\n        String valid = new String(foo.encode(JSONWriter.Feature.BrowserCompatible, JSONWriter.Feature.MapSortField), Charset.defaultCharset());\n\n        Gson gson = new Gson();\n        final TypeAdapter<JsonElement> strictAdapter = gson.getAdapter(JsonElement.class);\n        try {\n            strictAdapter.fromJson(invalid);\n            Assert.fail(\"Should have thrown\");\n        } catch (IOException ignore) {\n        }\n\n        try {\n            strictAdapter.fromJson(valid);\n        } catch (IOException ignore) {\n            Assert.fail(\"Should not throw\");\n        }\n    }\n}\n\nclass Sample {\n    private String stringValue = \"string\";\n    private int intValue = 2333;\n    private Integer integerValue = 666;\n    private double[] doubleArray = new double[] {0.618, 1.618};\n    private List<String> stringList = Arrays.asList(\"a\", \"o\", \"e\", \"i\", \"u\", \"v\");\n\n    public String getStringValue() {\n        return stringValue;\n    }\n\n    public void setStringValue(String stringValue) {\n        this.stringValue = stringValue;\n    }\n\n    public int getIntValue() {\n        return intValue;\n    }\n\n    public void setIntValue(int intValue) {\n        this.intValue = intValue;\n    }\n\n    public Integer getIntegerValue() {\n        return integerValue;\n    }\n\n    public void setIntegerValue(Integer integerValue) {\n        this.integerValue = integerValue;\n    }\n\n    public double[] getDoubleArray() {\n        return doubleArray;\n    }\n\n    public void setDoubleArray(double[] doubleArray) {\n        this.doubleArray = doubleArray;\n    }\n\n    public List<String> getStringList() {\n        return stringList;\n    }\n\n    public void setStringList(List<String> stringList) {\n        this.stringList = stringList;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o)\n            return true;\n        if (o == null || getClass() != o.getClass())\n            return false;\n\n        Sample sample = (Sample) o;\n\n        if (intValue != sample.intValue)\n            return false;\n        if (stringValue != null ? !stringValue.equals(sample.stringValue) : sample.stringValue != null)\n            return false;\n        if (integerValue != null ? !integerValue.equals(sample.integerValue) : sample.integerValue != null)\n            return false;\n        if (!Arrays.equals(doubleArray, sample.doubleArray))\n            return false;\n        return stringList != null ? stringList.equals(sample.stringList) : sample.stringList == null;\n\n    }\n\n    @Override\n    public int hashCode() {\n        int result = stringValue != null ? stringValue.hashCode() : 0;\n        result = 31 * result + intValue;\n        result = 31 * result + (integerValue != null ? integerValue.hashCode() : 0);\n        result = 31 * result + Arrays.hashCode(doubleArray);\n        result = 31 * result + (stringList != null ? stringList.hashCode() : 0);\n        return result;\n    }\n}"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RequestSourceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol;\n\nimport junit.framework.TestCase;\n\npublic class RequestSourceTest extends TestCase {\n\n    public void testIsValid() {\n        assertEquals(4, RequestSource.values().length);\n\n        assertTrue(RequestSource.isValid(-1));\n        assertTrue(RequestSource.isValid(0));\n        assertTrue(RequestSource.isValid(1));\n        assertTrue(RequestSource.isValid(2));\n\n        assertFalse(RequestSource.isValid(-2));\n        assertFalse(RequestSource.isValid(3));\n    }\n\n    public void testParseInteger() {\n        assertEquals(RequestSource.SDK, RequestSource.parseInteger(-1));\n        assertEquals(RequestSource.PROXY_FOR_ORDER, RequestSource.parseInteger(0));\n        assertEquals(RequestSource.PROXY_FOR_BROADCAST, RequestSource.parseInteger(1));\n        assertEquals(RequestSource.PROXY_FOR_STREAM, RequestSource.parseInteger(2));\n\n        assertEquals(RequestSource.SDK, RequestSource.parseInteger(-10));\n        assertEquals(RequestSource.SDK, RequestSource.parseInteger(10));\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RequestTypeTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol;\n\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class RequestTypeTest {\n    @Test\n    public void testValueOf() {\n        RequestType requestType = RequestType.valueOf(RequestType.STREAM.getCode());\n        assertThat(requestType).isEqualTo(RequestType.STREAM);\n\n        requestType = RequestType.valueOf((byte) 1);\n        assertThat(requestType).isNull();\n    }\n}"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RocketMQSerializableTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol;\n\nimport io.netty.buffer.ByteBuf;\nimport io.netty.buffer.ByteBufAllocator;\nimport io.netty.buffer.Unpooled;\nimport java.util.HashMap;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class RocketMQSerializableTest {\n    @Test\n    public void testRocketMQProtocolEncodeAndDecode_WithoutRemarkWithoutExtFields() {\n        System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, \"2333\");\n\n        //org.apache.rocketmq.remoting.protocol.RequestCode.REGISTER_BROKER\n        int code = 103;\n        RemotingCommand cmd = RemotingCommand.createRequestCommand(code, new SampleCommandCustomHeader());\n        cmd.setSerializeTypeCurrentRPC(SerializeType.ROCKETMQ);\n\n        byte[] result = RocketMQSerializable.rocketMQProtocolEncode(cmd);\n        int opaque = cmd.getOpaque();\n\n        assertThat(result).hasSize(21);\n        assertThat(parseToShort(result, 0)).isEqualTo((short) code); //code\n        assertThat(result[2]).isEqualTo(LanguageCode.JAVA.getCode()); //language\n        assertThat(parseToShort(result, 3)).isEqualTo((short) 2333); //version\n        assertThat(parseToInt(result, 9)).isEqualTo(0); //flag\n        assertThat(parseToInt(result, 13)).isEqualTo(0); //empty remark\n        assertThat(parseToInt(result, 17)).isEqualTo(0); //empty extFields\n\n        RemotingCommand decodedCommand = null;\n        try {\n            decodedCommand = RocketMQSerializable.rocketMQProtocolDecode(Unpooled.wrappedBuffer(result), result.length);\n\n            assertThat(decodedCommand.getCode()).isEqualTo(code);\n            assertThat(decodedCommand.getLanguage()).isEqualTo(LanguageCode.JAVA);\n            assertThat(decodedCommand.getVersion()).isEqualTo(2333);\n            assertThat(decodedCommand.getOpaque()).isEqualTo(opaque);\n            assertThat(decodedCommand.getFlag()).isEqualTo(0);\n            assertThat(decodedCommand.getRemark()).isNull();\n            assertThat(decodedCommand.getExtFields()).isNull();\n        } catch (RemotingCommandException e) {\n            e.printStackTrace();\n\n            Assert.fail(\"Should not throw IOException\");\n        }\n    }\n\n    @Test\n    public void testRocketMQProtocolEncodeAndDecode_WithRemarkWithoutExtFields() {\n        System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, \"2333\");\n\n        //org.apache.rocketmq.remoting.protocol.RequestCode.REGISTER_BROKER\n        int code = 103;\n        RemotingCommand cmd = RemotingCommand.createRequestCommand(code,\n            new SampleCommandCustomHeader());\n        cmd.setSerializeTypeCurrentRPC(SerializeType.ROCKETMQ);\n        cmd.setRemark(\"Sample Remark\");\n\n        byte[] result = RocketMQSerializable.rocketMQProtocolEncode(cmd);\n        int opaque = cmd.getOpaque();\n\n        assertThat(result).hasSize(34);\n        assertThat(parseToShort(result, 0)).isEqualTo((short) code); //code\n        assertThat(result[2]).isEqualTo(LanguageCode.JAVA.getCode()); //language\n        assertThat(parseToShort(result, 3)).isEqualTo((short) 2333); //version\n        assertThat(parseToInt(result, 9)).isEqualTo(0); //flag\n        assertThat(parseToInt(result, 13)).isEqualTo(13); //remark length\n\n        byte[] remarkArray = new byte[13];\n        System.arraycopy(result, 17, remarkArray, 0, 13);\n        assertThat(new String(remarkArray)).isEqualTo(\"Sample Remark\");\n\n        assertThat(parseToInt(result, 30)).isEqualTo(0); //empty extFields\n\n        try {\n            RemotingCommand decodedCommand = RocketMQSerializable.rocketMQProtocolDecode(Unpooled.wrappedBuffer(result), result.length);\n\n            assertThat(decodedCommand.getCode()).isEqualTo(code);\n            assertThat(decodedCommand.getLanguage()).isEqualTo(LanguageCode.JAVA);\n            assertThat(decodedCommand.getVersion()).isEqualTo(2333);\n            assertThat(decodedCommand.getOpaque()).isEqualTo(opaque);\n            assertThat(decodedCommand.getFlag()).isEqualTo(0);\n            assertThat(decodedCommand.getRemark()).contains(\"Sample Remark\");\n            assertThat(decodedCommand.getExtFields()).isNull();\n        } catch (RemotingCommandException e) {\n            e.printStackTrace();\n\n            Assert.fail(\"Should not throw IOException\");\n        }\n    }\n\n    @Test\n    public void testRocketMQProtocolEncodeAndDecode_WithoutRemarkWithExtFields() throws Exception {\n        System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, \"2333\");\n\n        //org.apache.rocketmq.remoting.protocol.RequestCode.REGISTER_BROKER\n        int code = 103;\n        RemotingCommand cmd = RemotingCommand.createRequestCommand(code,\n            new SampleCommandCustomHeader());\n        cmd.setSerializeTypeCurrentRPC(SerializeType.ROCKETMQ);\n        cmd.addExtField(\"key\", \"value\");\n\n        byte[] result = RocketMQSerializable.rocketMQProtocolEncode(cmd);\n        int opaque = cmd.getOpaque();\n\n        assertThat(result).hasSize(35);\n        assertThat(parseToShort(result, 0)).isEqualTo((short) code); //code\n        assertThat(result[2]).isEqualTo(LanguageCode.JAVA.getCode()); //language\n        assertThat(parseToShort(result, 3)).isEqualTo((short) 2333); //version\n        assertThat(parseToInt(result, 9)).isEqualTo(0); //flag\n        assertThat(parseToInt(result, 13)).isEqualTo(0); //empty remark\n        assertThat(parseToInt(result, 17)).isEqualTo(14); //extFields length\n\n        byte[] extFieldsArray = new byte[14];\n        System.arraycopy(result, 21, extFieldsArray, 0, 14);\n        HashMap<String, String> extFields =\n                RocketMQSerializable.mapDeserialize(Unpooled.wrappedBuffer(extFieldsArray), extFieldsArray.length);\n        assertThat(extFields).contains(new HashMap.SimpleEntry(\"key\", \"value\"));\n\n        try {\n            RemotingCommand decodedCommand = RocketMQSerializable.rocketMQProtocolDecode(Unpooled.wrappedBuffer(result), result.length);\n            assertThat(decodedCommand.getCode()).isEqualTo(code);\n            assertThat(decodedCommand.getLanguage()).isEqualTo(LanguageCode.JAVA);\n            assertThat(decodedCommand.getVersion()).isEqualTo(2333);\n            assertThat(decodedCommand.getOpaque()).isEqualTo(opaque);\n            assertThat(decodedCommand.getFlag()).isEqualTo(0);\n            assertThat(decodedCommand.getRemark()).isNull();\n            assertThat(decodedCommand.getExtFields()).contains(new HashMap.SimpleEntry(\"key\", \"value\"));\n        } catch (RemotingCommandException e) {\n            e.printStackTrace();\n\n            Assert.fail(\"Should not throw IOException\");\n        }\n    }\n\n    private short parseToShort(byte[] array, int index) {\n        return (short) (array[index] * 256 + array[++index]);\n    }\n\n    private int parseToInt(byte[] array, int index) {\n        return array[index] * 16777216 + array[++index] * 65536 + array[++index] * 256\n            + array[++index];\n    }\n\n    public static class MyHeader1 implements CommandCustomHeader {\n        private String str;\n        private int num;\n\n        @Override\n        public void checkFields() throws RemotingCommandException {\n        }\n\n        public String getStr() {\n            return str;\n        }\n\n        public void setStr(String str) {\n            this.str = str;\n        }\n\n        public int getNum() {\n            return num;\n        }\n\n        public void setNum(int num) {\n            this.num = num;\n        }\n    }\n\n    @Test\n    public void testFastEncode() throws Exception {\n        ByteBuf buf = ByteBufAllocator.DEFAULT.buffer(16);\n        MyHeader1 header1 = new MyHeader1();\n        header1.setStr(\"s1\");\n        header1.setNum(100);\n        RemotingCommand cmd = RemotingCommand.createRequestCommand(1, header1);\n        cmd.setRemark(\"remark\");\n        cmd.setOpaque(1001);\n        cmd.setVersion(99);\n        cmd.setLanguage(LanguageCode.JAVA);\n        cmd.setFlag(3);\n        cmd.makeCustomHeaderToNet();\n        RocketMQSerializable.rocketMQProtocolEncode(cmd, buf);\n        RemotingCommand cmd2 = RocketMQSerializable.rocketMQProtocolDecode(buf, buf.readableBytes());\n        assertThat(cmd2.getRemark()).isEqualTo(\"remark\");\n        assertThat(cmd2.getCode()).isEqualTo(1);\n        assertThat(cmd2.getOpaque()).isEqualTo(1001);\n        assertThat(cmd2.getVersion()).isEqualTo(99);\n        assertThat(cmd2.getLanguage()).isEqualTo(LanguageCode.JAVA);\n        assertThat(cmd2.getFlag()).isEqualTo(3);\n\n        MyHeader1 h2 = (MyHeader1) cmd2.decodeCommandCustomHeader(MyHeader1.class);\n        assertThat(h2.getStr()).isEqualTo(\"s1\");\n        assertThat(h2.getNum()).isEqualTo(100);\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/admin/ConsumeStatsTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.admin;\n\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.junit.Assert;\nimport org.junit.Test;\nimport org.mockito.Mockito;\n\npublic class ConsumeStatsTest {\n\n    @Test\n    public void testComputeTotalDiff() {\n        ConsumeStats stats = new ConsumeStats();\n        MessageQueue messageQueue = Mockito.mock(MessageQueue.class);\n        OffsetWrapper offsetWrapper = Mockito.mock(OffsetWrapper.class);\n        Mockito.when(offsetWrapper.getConsumerOffset()).thenReturn(1L);\n        Mockito.when(offsetWrapper.getBrokerOffset()).thenReturn(2L);\n        stats.getOffsetTable().put(messageQueue, offsetWrapper);\n\n        MessageQueue messageQueue2 = Mockito.mock(MessageQueue.class);\n        OffsetWrapper offsetWrapper2 = Mockito.mock(OffsetWrapper.class);\n        Mockito.when(offsetWrapper2.getConsumerOffset()).thenReturn(2L);\n        Mockito.when(offsetWrapper2.getBrokerOffset()).thenReturn(3L);\n        stats.getOffsetTable().put(messageQueue2, offsetWrapper2);\n        Assert.assertEquals(2L, stats.computeTotalDiff());\n    }\n\n    @Test\n    public void testComputeInflightTotalDiff() {\n        ConsumeStats stats = new ConsumeStats();\n        MessageQueue messageQueue = Mockito.mock(MessageQueue.class);\n        OffsetWrapper offsetWrapper = Mockito.mock(OffsetWrapper.class);\n        Mockito.when(offsetWrapper.getBrokerOffset()).thenReturn(3L);\n        Mockito.when(offsetWrapper.getPullOffset()).thenReturn(2L);\n        stats.getOffsetTable().put(messageQueue, offsetWrapper);\n\n        MessageQueue messageQueue2 = Mockito.mock(MessageQueue.class);\n        OffsetWrapper offsetWrapper2 = Mockito.mock(OffsetWrapper.class);\n        Mockito.when(offsetWrapper.getBrokerOffset()).thenReturn(3L);\n        Mockito.when(offsetWrapper.getPullOffset()).thenReturn(2L);\n        stats.getOffsetTable().put(messageQueue2, offsetWrapper2);\n        Assert.assertEquals(2L, stats.computeInflightTotalDiff());\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/admin/TopicStatsTableTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.admin;\n\nimport java.util.HashMap;\nimport java.util.Map;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\n\npublic class TopicStatsTableTest {\n\n    private volatile TopicStatsTable topicStatsTable;\n\n    private static final String TEST_TOPIC = \"test_topic\";\n\n    private static final String TEST_BROKER = \"test_broker\";\n\n    private static final int QUEUE_ID = 1;\n\n    private static final long CURRENT_TIME_MILLIS = System.currentTimeMillis();\n\n    private static final long MAX_OFFSET = CURRENT_TIME_MILLIS + 100;\n\n    private static final long MIN_OFFSET = CURRENT_TIME_MILLIS - 100;\n\n    @Before\n    public void buildTopicStatsTable() {\n        HashMap<MessageQueue, TopicOffset> offsetTableMap = new HashMap<>();\n\n        MessageQueue messageQueue = new MessageQueue(TEST_TOPIC, TEST_BROKER, QUEUE_ID);\n\n        TopicOffset topicOffset = new TopicOffset();\n        topicOffset.setLastUpdateTimestamp(CURRENT_TIME_MILLIS);\n        topicOffset.setMinOffset(MIN_OFFSET);\n        topicOffset.setMaxOffset(MAX_OFFSET);\n\n        offsetTableMap.put(messageQueue, topicOffset);\n\n        topicStatsTable = new TopicStatsTable();\n        topicStatsTable.setOffsetTable(offsetTableMap);\n    }\n\n    @Test\n    public void testGetOffsetTable() throws Exception {\n        validateTopicStatsTable(topicStatsTable);\n    }\n\n    @Test\n    public void testFromJson() throws Exception {\n        String json = RemotingSerializable.toJson(topicStatsTable, true);\n        TopicStatsTable fromJson = RemotingSerializable.fromJson(json, TopicStatsTable.class);\n\n        validateTopicStatsTable(fromJson);\n    }\n\n    private static void validateTopicStatsTable(TopicStatsTable topicStatsTable) throws Exception {\n        Map.Entry<MessageQueue, TopicOffset> savedTopicStatsTableMap = topicStatsTable.getOffsetTable().entrySet().iterator().next();\n        MessageQueue savedMessageQueue = savedTopicStatsTableMap.getKey();\n        TopicOffset savedTopicOffset = savedTopicStatsTableMap.getValue();\n\n        Assert.assertTrue(savedMessageQueue.getTopic().equals(TEST_TOPIC));\n        Assert.assertTrue(savedMessageQueue.getBrokerName().equals(TEST_BROKER));\n        Assert.assertTrue(savedMessageQueue.getQueueId() == QUEUE_ID);\n\n        Assert.assertTrue(savedTopicOffset.getLastUpdateTimestamp() == CURRENT_TIME_MILLIS);\n        Assert.assertTrue(savedTopicOffset.getMaxOffset() == MAX_OFFSET);\n        Assert.assertTrue(savedTopicOffset.getMinOffset() == MIN_OFFSET);\n    }\n\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/body/BatchAckTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport com.alibaba.fastjson2.JSON;\nimport org.apache.rocketmq.common.MixAll;\nimport org.junit.Test;\n\nimport java.util.Arrays;\nimport java.util.BitSet;\nimport java.util.List;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class BatchAckTest {\n    private static String topic = \"myTopic\";\n    private static String cid = MixAll.DEFAULT_CONSUMER_GROUP;\n    private static long startOffset = 100;\n    private static int qId = 1;\n    private static int rqId = 2;\n    private static long popTime = System.currentTimeMillis();\n    private static long invisibleTime = 5000;\n\n    @Test\n    public void testBatchAckSerializerDeserializer() {\n        List<Long> ackOffsetList = Arrays.asList(startOffset + 1, startOffset + 3, startOffset + 5);\n        BatchAck batchAck = new BatchAck();\n        batchAck.setConsumerGroup(cid);\n        batchAck.setTopic(topic);\n        batchAck.setRetry(\"0\");\n        batchAck.setStartOffset(startOffset);\n        batchAck.setQueueId(qId);\n        batchAck.setReviveQueueId(rqId);\n        batchAck.setPopTime(popTime);\n        batchAck.setInvisibleTime(invisibleTime);\n        batchAck.setBitSet(new BitSet());\n        for (Long offset : ackOffsetList) {\n            batchAck.getBitSet().set((int) (offset - startOffset));\n        }\n        String jsonStr = JSON.toJSONString(batchAck);\n\n        BatchAck bAck = JSON.parseObject(jsonStr, BatchAck.class);\n        assertThat(bAck.getConsumerGroup()).isEqualTo(cid);\n        assertThat(bAck.getTopic()).isEqualTo(topic);\n        assertThat(bAck.getStartOffset()).isEqualTo(startOffset);\n        assertThat(bAck.getQueueId()).isEqualTo(qId);\n        assertThat(bAck.getReviveQueueId()).isEqualTo(rqId);\n        assertThat(bAck.getPopTime()).isEqualTo(popTime);\n        assertThat(bAck.getInvisibleTime()).isEqualTo(invisibleTime);\n        for (int i = 0; i < bAck.getBitSet().length(); i++) {\n            long ackOffset = startOffset + i;\n            if (ackOffsetList.contains(ackOffset)) {\n                assertThat(bAck.getBitSet().get(i)).isTrue();\n            } else {\n                assertThat(bAck.getBitSet().get(i)).isFalse();\n            }\n        }\n    }\n\n    @Test\n    public void testWithBatchAckMessageRequestBody() {\n        List<Long> ackOffsetList = Arrays.asList(startOffset + 1, startOffset + 3, startOffset + 5);\n        BatchAck batchAck = new BatchAck();\n        batchAck.setConsumerGroup(cid);\n        batchAck.setTopic(topic);\n        batchAck.setRetry(\"0\");\n        batchAck.setStartOffset(startOffset);\n        batchAck.setQueueId(qId);\n        batchAck.setReviveQueueId(rqId);\n        batchAck.setPopTime(popTime);\n        batchAck.setInvisibleTime(invisibleTime);\n        batchAck.setBitSet(new BitSet());\n        for (Long offset : ackOffsetList) {\n            batchAck.getBitSet().set((int) (offset - startOffset));\n        }\n\n        BatchAckMessageRequestBody batchAckMessageRequestBody = new BatchAckMessageRequestBody();\n        batchAckMessageRequestBody.setAcks(Arrays.asList(batchAck));\n        byte[] bytes = batchAckMessageRequestBody.encode();\n        BatchAckMessageRequestBody reqBody = BatchAckMessageRequestBody.decode(bytes, BatchAckMessageRequestBody.class);\n        BatchAck bAck = reqBody.getAcks().get(0);\n        assertThat(bAck.getConsumerGroup()).isEqualTo(cid);\n        assertThat(bAck.getTopic()).isEqualTo(topic);\n        assertThat(bAck.getStartOffset()).isEqualTo(startOffset);\n        assertThat(bAck.getQueueId()).isEqualTo(qId);\n        assertThat(bAck.getReviveQueueId()).isEqualTo(rqId);\n        assertThat(bAck.getPopTime()).isEqualTo(popTime);\n        assertThat(bAck.getInvisibleTime()).isEqualTo(invisibleTime);\n        for (int i = 0; i < bAck.getBitSet().length(); i++) {\n            long ackOffset = startOffset + i;\n            if (ackOffsetList.contains(ackOffset)) {\n                assertThat(bAck.getBitSet().get(i)).isTrue();\n            } else {\n                assertThat(bAck.getBitSet().get(i)).isFalse();\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/body/BrokerStatsDataTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.assertj.core.api.Assertions.within;\n\npublic class BrokerStatsDataTest {\n\n    @Test\n    public void testFromJson() throws Exception {\n        BrokerStatsData brokerStatsData = new BrokerStatsData();\n\n        {\n            BrokerStatsItem brokerStatsItem = new BrokerStatsItem();\n            brokerStatsItem.setAvgpt(10.0);\n            brokerStatsItem.setSum(100L);\n            brokerStatsItem.setTps(100.0);\n            brokerStatsData.setStatsDay(brokerStatsItem);\n        }\n\n        {\n            BrokerStatsItem brokerStatsItem = new BrokerStatsItem();\n            brokerStatsItem.setAvgpt(10.0);\n            brokerStatsItem.setSum(100L);\n            brokerStatsItem.setTps(100.0);\n            brokerStatsData.setStatsHour(brokerStatsItem);\n        }\n\n        {\n            BrokerStatsItem brokerStatsItem = new BrokerStatsItem();\n            brokerStatsItem.setAvgpt(10.0);\n            brokerStatsItem.setSum(100L);\n            brokerStatsItem.setTps(100.0);\n            brokerStatsData.setStatsMinute(brokerStatsItem);\n        }\n\n        String json = RemotingSerializable.toJson(brokerStatsData, true);\n        BrokerStatsData brokerStatsDataResult = RemotingSerializable.fromJson(json, BrokerStatsData.class);\n\n        assertThat(brokerStatsDataResult.getStatsMinute().getAvgpt()).isCloseTo(brokerStatsData.getStatsMinute().getAvgpt(), within(0.0001));\n        assertThat(brokerStatsDataResult.getStatsMinute().getTps()).isCloseTo(brokerStatsData.getStatsMinute().getTps(), within(0.0001));\n        assertThat(brokerStatsDataResult.getStatsMinute().getSum()).isEqualTo(brokerStatsData.getStatsMinute().getSum());\n\n        assertThat(brokerStatsDataResult.getStatsHour().getAvgpt()).isCloseTo(brokerStatsData.getStatsHour().getAvgpt(), within(0.0001));\n        assertThat(brokerStatsDataResult.getStatsHour().getTps()).isCloseTo(brokerStatsData.getStatsHour().getTps(), within(0.0001));\n        assertThat(brokerStatsDataResult.getStatsHour().getSum()).isEqualTo(brokerStatsData.getStatsHour().getSum());\n\n        assertThat(brokerStatsDataResult.getStatsDay().getAvgpt()).isCloseTo(brokerStatsData.getStatsDay().getAvgpt(), within(0.0001));\n        assertThat(brokerStatsDataResult.getStatsDay().getTps()).isCloseTo(brokerStatsData.getStatsDay().getTps(), within(0.0001));\n        assertThat(brokerStatsDataResult.getStatsDay().getSum()).isEqualTo(brokerStatsData.getStatsDay().getSum());\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/body/CheckClientRequestBodyTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class CheckClientRequestBodyTest {\n\n    @Test\n    public void testFromJson() {\n        SubscriptionData subscriptionData = new SubscriptionData();\n        String expectedClientId = \"defalutId\";\n        String expectedGroup = \"defaultGroup\";\n        CheckClientRequestBody checkClientRequestBody = new CheckClientRequestBody();\n        checkClientRequestBody.setClientId(expectedClientId);\n        checkClientRequestBody.setGroup(expectedGroup);\n        checkClientRequestBody.setSubscriptionData(subscriptionData);\n        String json = RemotingSerializable.toJson(checkClientRequestBody, true);\n        CheckClientRequestBody fromJson = RemotingSerializable.fromJson(json, CheckClientRequestBody.class);\n        assertThat(fromJson.getClientId()).isEqualTo(expectedClientId);\n        assertThat(fromJson.getGroup()).isEqualTo(expectedGroup);\n        assertThat(fromJson.getSubscriptionData()).isEqualTo(subscriptionData);\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/body/ConsumeMessageDirectlyResultTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\n\npublic class ConsumeMessageDirectlyResultTest {\n    @Test\n    public void testFromJson() throws Exception {\n        ConsumeMessageDirectlyResult result = new ConsumeMessageDirectlyResult();\n        boolean defaultAutoCommit = true;\n        boolean defaultOrder = false;\n        long defaultSpentTimeMills = 1234567L;\n        String defaultRemark = \"defaultMark\";\n        CMResult defaultCMResult = CMResult.CR_COMMIT;\n\n        result.setAutoCommit(defaultAutoCommit);\n        result.setOrder(defaultOrder);\n        result.setRemark(defaultRemark);\n        result.setSpentTimeMills(defaultSpentTimeMills);\n        result.setConsumeResult(defaultCMResult);\n\n        String json = RemotingSerializable.toJson(result, true);\n        ConsumeMessageDirectlyResult fromJson = RemotingSerializable.fromJson(json, ConsumeMessageDirectlyResult.class);\n        assertThat(fromJson).isNotNull();\n\n        assertThat(fromJson.getRemark()).isEqualTo(defaultRemark);\n        assertThat(fromJson.getSpentTimeMills()).isEqualTo(defaultSpentTimeMills);\n        assertThat(fromJson.getConsumeResult()).isEqualTo(defaultCMResult);\n        assertThat(fromJson.isOrder()).isEqualTo(defaultOrder);\n\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/body/ConsumeStatsListTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.admin.ConsumeStats;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class ConsumeStatsListTest {\n\n    @Test\n    public void testFromJson() {\n        ConsumeStats consumeStats = new ConsumeStats();\n        ArrayList<ConsumeStats> consumeStatsListValue = new ArrayList<>();\n        consumeStatsListValue.add(consumeStats);\n        HashMap<String, List<ConsumeStats>> map = new HashMap<>();\n        map.put(\"subscriptionGroupName\", consumeStatsListValue);\n        List<Map<String/*subscriptionGroupName*/, List<ConsumeStats>>> consumeStatsListValue2 = new ArrayList<>();\n        consumeStatsListValue2.add(map);\n\n        String brokerAddr = \"brokerAddr\";\n        long totalDiff = 12352L;\n        ConsumeStatsList consumeStatsList = new ConsumeStatsList();\n        consumeStatsList.setBrokerAddr(brokerAddr);\n        consumeStatsList.setTotalDiff(totalDiff);\n        consumeStatsList.setConsumeStatsList(consumeStatsListValue2);\n\n        String toJson = RemotingSerializable.toJson(consumeStatsList, true);\n        ConsumeStatsList fromJson = RemotingSerializable.fromJson(toJson, ConsumeStatsList.class);\n\n        assertThat(fromJson.getBrokerAddr()).isEqualTo(brokerAddr);\n        assertThat(fromJson.getTotalDiff()).isEqualTo(totalDiff);\n\n        List<Map<String, List<ConsumeStats>>> fromJsonConsumeStatsList = fromJson.getConsumeStatsList();\n        assertThat(fromJsonConsumeStatsList).isInstanceOf(List.class);\n\n        ConsumeStats fromJsonConsumeStats = fromJsonConsumeStatsList.get(0).get(\"subscriptionGroupName\").get(0);\n        assertThat(fromJsonConsumeStats).isExactlyInstanceOf(ConsumeStats.class);\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/body/ConsumerConnectionTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.HashSet;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.assertj.core.api.Assertions;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class ConsumerConnectionTest {\n\n    @Test\n    public void testFromJson() {\n        ConsumerConnection consumerConnection = new ConsumerConnection();\n        HashSet<Connection> connections = new HashSet<>();\n        Connection conn = new Connection();\n        connections.add(conn);\n\n        ConcurrentHashMap<String/* Topic */, SubscriptionData> subscriptionTable = new ConcurrentHashMap<>();\n        SubscriptionData subscriptionData = new SubscriptionData();\n        subscriptionTable.put(\"topicA\", subscriptionData);\n\n        ConsumeType consumeType = ConsumeType.CONSUME_ACTIVELY;\n        MessageModel messageModel = MessageModel.CLUSTERING;\n        ConsumeFromWhere consumeFromWhere = ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET;\n\n        consumerConnection.setConnectionSet(connections);\n        consumerConnection.setSubscriptionTable(subscriptionTable);\n        consumerConnection.setConsumeType(consumeType);\n        consumerConnection.setMessageModel(messageModel);\n        consumerConnection.setConsumeFromWhere(consumeFromWhere);\n\n        String json = RemotingSerializable.toJson(consumerConnection, true);\n        ConsumerConnection fromJson = RemotingSerializable.fromJson(json, ConsumerConnection.class);\n        assertThat(fromJson.getConsumeType()).isEqualTo(ConsumeType.CONSUME_ACTIVELY);\n        assertThat(fromJson.getMessageModel()).isEqualTo(MessageModel.CLUSTERING);\n\n        HashSet<Connection> connectionSet = fromJson.getConnectionSet();\n        Assertions.assertThat(connectionSet).isInstanceOf(Set.class);\n\n        SubscriptionData data = fromJson.getSubscriptionTable().get(\"topicA\");\n        assertThat(data).isExactlyInstanceOf(SubscriptionData.class);\n    }\n\n    @Test\n    public void testComputeMinVersion() {\n        ConsumerConnection consumerConnection = new ConsumerConnection();\n        HashSet<Connection> connections = new HashSet<>();\n        Connection conn1 = new Connection();\n        conn1.setVersion(1);\n        connections.add(conn1);\n        Connection conn2 = new Connection();\n        conn2.setVersion(10);\n        connections.add(conn2);\n        consumerConnection.setConnectionSet(connections);\n\n        int version = consumerConnection.computeMinVersion();\n        assertThat(version).isEqualTo(1);\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/body/ConsumerRunningInfoTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.Properties;\nimport java.util.TreeMap;\nimport java.util.TreeSet;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType.CONSUME_ACTIVELY;\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class ConsumerRunningInfoTest {\n\n    private ConsumerRunningInfo consumerRunningInfo;\n\n    private TreeMap<String, ConsumerRunningInfo> criTable;\n\n    private MessageQueue messageQueue;\n\n    @Before\n    public void init() {\n        consumerRunningInfo = new ConsumerRunningInfo();\n        consumerRunningInfo.setJstack(\"test\");\n\n        TreeMap<MessageQueue, ProcessQueueInfo> mqTable = new TreeMap<>();\n        messageQueue = new MessageQueue(\"topicA\",\"broker\", 1);\n        mqTable.put(messageQueue, new ProcessQueueInfo());\n        consumerRunningInfo.setMqTable(mqTable);\n\n        TreeMap<String, ConsumeStatus> statusTable = new TreeMap<>();\n        statusTable.put(\"topicA\", new ConsumeStatus());\n        consumerRunningInfo.setStatusTable(statusTable);\n\n        TreeSet<SubscriptionData> subscriptionSet = new TreeSet<>();\n        subscriptionSet.add(new SubscriptionData());\n        consumerRunningInfo.setSubscriptionSet(subscriptionSet);\n\n        Properties properties = new Properties();\n        properties.put(ConsumerRunningInfo.PROP_CONSUME_TYPE, CONSUME_ACTIVELY);\n        properties.put(ConsumerRunningInfo.PROP_CONSUMER_START_TIMESTAMP, System.currentTimeMillis());\n        consumerRunningInfo.setProperties(properties);\n\n        criTable = new TreeMap<>();\n        criTable.put(\"client_id\", consumerRunningInfo);\n    }\n\n    @Test\n    public void testFromJson() {\n        String toJson = RemotingSerializable.toJson(consumerRunningInfo, true);\n        ConsumerRunningInfo fromJson = RemotingSerializable.fromJson(toJson, ConsumerRunningInfo.class);\n\n        assertThat(fromJson.getJstack()).isEqualTo(\"test\");\n        assertThat(fromJson.getProperties().get(ConsumerRunningInfo.PROP_CONSUME_TYPE)).isEqualTo(ConsumeType.CONSUME_ACTIVELY.name());\n\n        ConsumeStatus consumeStatus = fromJson.getStatusTable().get(\"topicA\");\n        assertThat(consumeStatus).isExactlyInstanceOf(ConsumeStatus.class);\n\n        SubscriptionData subscription = fromJson.getSubscriptionSet().first();\n        assertThat(subscription).isExactlyInstanceOf(SubscriptionData.class);\n\n        ProcessQueueInfo processQueueInfo = fromJson.getMqTable().get(messageQueue);\n        assertThat(processQueueInfo).isExactlyInstanceOf(ProcessQueueInfo.class);\n    }\n\n    @Test\n    public void testAnalyzeRebalance() {\n        boolean result = ConsumerRunningInfo.analyzeRebalance(criTable);\n        assertThat(result).isTrue();\n    }\n\n    @Test\n    public void testAnalyzeProcessQueue() {\n        String result = ConsumerRunningInfo.analyzeProcessQueue(\"client_id\", consumerRunningInfo);\n        assertThat(result).isEmpty();\n\n    }\n\n    @Test\n    public void testAnalyzeSubscription() {\n        boolean result = ConsumerRunningInfo.analyzeSubscription(criTable);\n        assertThat(result).isTrue();\n    }\n\n\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/body/KVTableTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.HashMap;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class KVTableTest {\n\n    @Test\n    public void testFromJson() throws Exception {\n        HashMap<String, String> table = new HashMap<>();\n        table.put(\"key1\", \"value1\");\n        table.put(\"key2\", \"value2\");\n\n        KVTable kvTable = new KVTable();\n        kvTable.setTable(table);\n\n        String json = RemotingSerializable.toJson(kvTable, true);\n        KVTable fromJson = RemotingSerializable.fromJson(json, KVTable.class);\n\n        assertThat(fromJson).isNotEqualTo(kvTable);\n        assertThat(fromJson.getTable().get(\"key1\")).isEqualTo(kvTable.getTable().get(\"key1\"));\n        assertThat(fromJson.getTable().get(\"key2\")).isEqualTo(kvTable.getTable().get(\"key2\"));\n    }\n\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/body/MessageRequestModeSerializeWrapperTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.common.message.MessageRequestMode;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class MessageRequestModeSerializeWrapperTest {\n\n    @Test\n    public void testFromJson() {\n        MessageRequestModeSerializeWrapper  messageRequestModeSerializeWrapper = new MessageRequestModeSerializeWrapper();\n        ConcurrentHashMap<String, ConcurrentHashMap<String, SetMessageRequestModeRequestBody>>\n                messageRequestModeMap = new ConcurrentHashMap<>();\n        String topic = \"TopicTest\";\n        String group = \"Consumer\";\n        MessageRequestMode requestMode = MessageRequestMode.POP;\n        int popShareQueueNum = -1;\n        SetMessageRequestModeRequestBody requestBody = new SetMessageRequestModeRequestBody();\n        requestBody.setTopic(topic);\n        requestBody.setConsumerGroup(group);\n        requestBody.setMode(requestMode);\n        requestBody.setPopShareQueueNum(popShareQueueNum);\n        ConcurrentHashMap<String, SetMessageRequestModeRequestBody> map = new ConcurrentHashMap<>();\n        map.put(group, requestBody);\n        messageRequestModeMap.put(topic, map);\n\n        messageRequestModeSerializeWrapper.setMessageRequestModeMap(messageRequestModeMap);\n\n        String json = RemotingSerializable.toJson(messageRequestModeSerializeWrapper, true);\n        MessageRequestModeSerializeWrapper fromJson = RemotingSerializable.fromJson(json, MessageRequestModeSerializeWrapper.class);\n        assertThat(fromJson.getMessageRequestModeMap()).containsKey(topic);\n        assertThat(fromJson.getMessageRequestModeMap().get(topic)).containsKey(group);\n        assertThat(fromJson.getMessageRequestModeMap().get(topic).get(group).getTopic()).isEqualTo(topic);\n        assertThat(fromJson.getMessageRequestModeMap().get(topic).get(group).getConsumerGroup()).isEqualTo(group);\n        assertThat(fromJson.getMessageRequestModeMap().get(topic).get(group).getMode()).isEqualTo(requestMode);\n        assertThat(fromJson.getMessageRequestModeMap().get(topic).get(group).getPopShareQueueNum()).isEqualTo(popShareQueueNum);\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/body/QueryConsumeQueueResponseBodyTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class QueryConsumeQueueResponseBodyTest {\n\n    @Test\n    public void test() {\n        QueryConsumeQueueResponseBody body = new QueryConsumeQueueResponseBody();\n\n        SubscriptionData subscriptionData = new SubscriptionData();\n        ConsumeQueueData data = new ConsumeQueueData();\n        data.setBitMap(\"defaultBitMap\");\n        data.setEval(false);\n        data.setMsg(\"this is default msg\");\n        data.setPhysicOffset(10L);\n        data.setPhysicSize(1);\n        data.setTagsCode(1L);\n        List<ConsumeQueueData> list = new ArrayList<>();\n        list.add(data);\n\n        body.setQueueData(list);\n        body.setFilterData(\"default filter data\");\n        body.setMaxQueueIndex(100L);\n        body.setMinQueueIndex(1L);\n        body.setSubscriptionData(subscriptionData);\n\n        String json = RemotingSerializable.toJson(body, true);\n        QueryConsumeQueueResponseBody fromJson = RemotingSerializable.fromJson(json, QueryConsumeQueueResponseBody.class);\n        //test ConsumeQueue\n        ConsumeQueueData jsonData = fromJson.getQueueData().get(0);\n        assertThat(jsonData.getMsg()).isEqualTo(\"this is default msg\");\n        assertThat(jsonData.getPhysicSize()).isEqualTo(1);\n        assertThat(jsonData.getBitMap()).isEqualTo(\"defaultBitMap\");\n        assertThat(jsonData.getTagsCode()).isEqualTo(1L);\n        assertThat(jsonData.getPhysicSize()).isEqualTo(1);\n\n        //test QueryConsumeQueueResponseBody\n        assertThat(fromJson.getFilterData()).isEqualTo(\"default filter data\");\n        assertThat(fromJson.getMaxQueueIndex()).isEqualTo(100L);\n        assertThat(fromJson.getMinQueueIndex()).isEqualTo(1L);\n        assertThat(fromJson.getSubscriptionData()).isEqualTo(subscriptionData);\n\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/body/QueryCorrectionOffsetBodyTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.HashMap;\nimport java.util.Map;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class QueryCorrectionOffsetBodyTest {\n\n    @Test\n    public void testFromJson() throws Exception {\n        QueryCorrectionOffsetBody qcob = new QueryCorrectionOffsetBody();\n        Map<Integer, Long> offsetMap = new HashMap<>();\n        offsetMap.put(1, 100L);\n        offsetMap.put(2, 200L);\n        qcob.setCorrectionOffsets(offsetMap);\n        String json = RemotingSerializable.toJson(qcob, true);\n        QueryCorrectionOffsetBody fromJson = RemotingSerializable.fromJson(json, QueryCorrectionOffsetBody.class);\n        assertThat(fromJson.getCorrectionOffsets().get(1)).isEqualTo(100L);\n        assertThat(fromJson.getCorrectionOffsets().get(2)).isEqualTo(200L);\n        assertThat(fromJson.getCorrectionOffsets().size()).isEqualTo(2);\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/body/ResetOffsetBodyTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.HashMap;\nimport java.util.Map;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class ResetOffsetBodyTest {\n\n    @Test\n    public void testFromJson() throws Exception {\n        ResetOffsetBody rob = new ResetOffsetBody();\n        Map<MessageQueue, Long> offsetMap = new HashMap<>();\n        MessageQueue queue = new MessageQueue();\n        queue.setQueueId(1);\n        queue.setBrokerName(\"brokerName\");\n        queue.setTopic(\"topic\");\n        offsetMap.put(queue, 100L);\n        rob.setOffsetTable(offsetMap);\n        String json = RemotingSerializable.toJson(rob, true);\n        ResetOffsetBody fromJson = RemotingSerializable.fromJson(json, ResetOffsetBody.class);\n        assertThat(fromJson.getOffsetTable().get(queue)).isEqualTo(100L);\n        assertThat(fromJson.getOffsetTable().size()).isEqualTo(1);\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/body/SubscriptionGroupWrapperTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.body;\n\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class SubscriptionGroupWrapperTest {\n\n    @Test\n    public void testFromJson() {\n        SubscriptionGroupWrapper subscriptionGroupWrapper = new SubscriptionGroupWrapper();\n        ConcurrentHashMap<String, SubscriptionGroupConfig> subscriptions = new ConcurrentHashMap<>();\n        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();\n        subscriptionGroupConfig.setConsumeBroadcastEnable(true);\n        subscriptionGroupConfig.setBrokerId(1234);\n        subscriptionGroupConfig.setGroupName(\"Consumer-group-one\");\n        subscriptions.put(\"Consumer-group-one\", subscriptionGroupConfig);\n        subscriptionGroupWrapper.setSubscriptionGroupTable(subscriptions);\n        DataVersion dataVersion = new DataVersion();\n        dataVersion.nextVersion();\n        subscriptionGroupWrapper.setDataVersion(dataVersion);\n        String json = RemotingSerializable.toJson(subscriptionGroupWrapper, true);\n        SubscriptionGroupWrapper fromJson = RemotingSerializable.fromJson(json, SubscriptionGroupWrapper.class);\n        assertThat(fromJson.getSubscriptionGroupTable()).containsKey(\"Consumer-group-one\");\n        assertThat(fromJson.getSubscriptionGroupTable().get(\"Consumer-group-one\").getGroupName()).isEqualTo(\"Consumer-group-one\");\n        assertThat(fromJson.getSubscriptionGroupTable().get(\"Consumer-group-one\").getBrokerId()).isEqualTo(1234);\n    }\n\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/filter/FilterAPITest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.filter;\n\nimport java.util.HashSet;\nimport java.util.Set;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class FilterAPITest {\n    private String topic = \"FooBar\";\n    private String group = \"FooBarGroup\";\n    private String subString = \"TAG1 || Tag2 || tag3\";\n\n    @Test\n    public void testBuildSubscriptionData() throws Exception {\n        SubscriptionData subscriptionData =\n                FilterAPI.buildSubscriptionData(topic, subString);\n        assertThat(subscriptionData.getTopic()).isEqualTo(topic);\n        assertThat(subscriptionData.getSubString()).isEqualTo(subString);\n        String[] tags = subString.split(\"\\\\|\\\\|\");\n        Set<String> tagSet = new HashSet<>();\n        for (String tag : tags) {\n            tagSet.add(tag.trim());\n        }\n        assertThat(subscriptionData.getTagsSet()).isEqualTo(tagSet);\n    }\n\n    @Test\n    public void testBuildTagSome() {\n        try {\n            SubscriptionData subscriptionData = FilterAPI.build(\n                    \"TOPIC\", \"A || B\", ExpressionType.TAG\n            );\n\n            assertThat(subscriptionData).isNotNull();\n            assertThat(subscriptionData.getTopic()).isEqualTo(\"TOPIC\");\n            assertThat(subscriptionData.getSubString()).isEqualTo(\"A || B\");\n            assertThat(ExpressionType.isTagType(subscriptionData.getExpressionType())).isTrue();\n\n            assertThat(subscriptionData.getTagsSet()).isNotNull();\n            assertThat(subscriptionData.getTagsSet()).containsExactlyInAnyOrder(\"A\", \"B\");\n        } catch (Exception e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n    }\n\n    @Test\n    public void testBuildSQL() {\n        try {\n            SubscriptionData subscriptionData = FilterAPI.build(\n                    \"TOPIC\", \"a is not null\", ExpressionType.SQL92\n            );\n\n            assertThat(subscriptionData).isNotNull();\n            assertThat(subscriptionData.getTopic()).isEqualTo(\"TOPIC\");\n            assertThat(subscriptionData.getExpressionType()).isEqualTo(ExpressionType.SQL92);\n        } catch (Exception e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n    }\n\n    @Test(expected = IllegalArgumentException.class)\n    public void testBuildSQLWithNullSubString() throws Exception {\n        FilterAPI.build(\"TOPIC\", null, ExpressionType.SQL92);\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/header/ExportRocksDBConfigToJsonRequestHeaderTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class ExportRocksDBConfigToJsonRequestHeaderTest {\n    @Test\n    public void configTypeTest() {\n        List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> configTypes = new ArrayList<>();\n        configTypes.add(ExportRocksDBConfigToJsonRequestHeader.ConfigType.TOPICS);\n        configTypes.add(ExportRocksDBConfigToJsonRequestHeader.ConfigType.SUBSCRIPTION_GROUPS);\n\n        String string = ExportRocksDBConfigToJsonRequestHeader.ConfigType.toString(configTypes);\n\n        List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> newConfigTypes = ExportRocksDBConfigToJsonRequestHeader.ConfigType.fromString(string);\n        assert newConfigTypes.size() == 2;\n        assert configTypes.equals(newConfigTypes);\n\n        List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> topics = ExportRocksDBConfigToJsonRequestHeader.ConfigType.fromString(\"topics\");\n        assert topics.size() == 1;\n        assert topics.get(0).equals(ExportRocksDBConfigToJsonRequestHeader.ConfigType.TOPICS);\n\n        List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> mix = ExportRocksDBConfigToJsonRequestHeader.ConfigType.fromString(\"toPics; subScriptiongroups\");\n        assert mix.size() == 2;\n        assert mix.get(0).equals(ExportRocksDBConfigToJsonRequestHeader.ConfigType.TOPICS);\n        assert mix.get(1).equals(ExportRocksDBConfigToJsonRequestHeader.ConfigType.SUBSCRIPTION_GROUPS);\n\n        Assert.assertThrows(IllegalArgumentException.class, () -> {\n            ExportRocksDBConfigToJsonRequestHeader.ConfigType.fromString(\"topics; subscription\");\n        });\n\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/header/ExtraInfoUtilTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport java.util.Map;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\n\npublic class ExtraInfoUtilTest {\n\n    @Test\n    public void testOrderCountInfo() {\n        String topic = \"TOPIC\";\n        int queueId = 0;\n        long queueOffset = 1234;\n\n        Integer queueIdCount = 1;\n        Integer queueOffsetCount = 2;\n\n        String queueIdKey = ExtraInfoUtil.getStartOffsetInfoMapKey(topic, queueId);\n        String queueOffsetKey = ExtraInfoUtil.getQueueOffsetMapKey(topic, queueId, queueOffset);\n\n        StringBuilder sb = new StringBuilder();\n        ExtraInfoUtil.buildQueueIdOrderCountInfo(sb, topic, queueId, queueIdCount);\n        ExtraInfoUtil.buildQueueOffsetOrderCountInfo(sb, topic, queueId, queueOffset, queueOffsetCount);\n        Map<String, Integer> orderCountInfo = ExtraInfoUtil.parseOrderCountInfo(sb.toString());\n\n        assertEquals(queueIdCount, orderCountInfo.get(queueIdKey));\n        assertEquals(queueOffsetCount, orderCountInfo.get(queueOffsetKey));\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/header/FastCodesHeaderTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport java.lang.reflect.Field;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.protocol.FastCodesHeader;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class FastCodesHeaderTest {\n\n    @Test\n    public void testFastDecode() throws Exception {\n        testFastDecode(SendMessageRequestHeaderV2.class);\n        testFastDecode(SendMessageResponseHeader.class);\n        testFastDecode(PullMessageRequestHeader.class);\n        testFastDecode(PullMessageResponseHeader.class);\n    }\n\n    private void testFastDecode(Class<? extends CommandCustomHeader> classHeader) throws Exception {\n        Field[] declaredFields = classHeader.getDeclaredFields();\n        List<Field> declaredFieldsList = new ArrayList<>();\n        for (Field f : declaredFields) {\n            if (f.getName().startsWith(\"$\")) {\n                continue;\n            }\n            f.setAccessible(true);\n            declaredFieldsList.add(f);\n        }\n        RemotingCommand command = RemotingCommand.createRequestCommand(0, null);\n        HashMap<String, String> m = buildExtFields(declaredFieldsList);\n        command.setExtFields(m);\n        check(command, declaredFieldsList, classHeader);\n    }\n\n    private HashMap<String, String> buildExtFields(List<Field> fields) {\n        HashMap<String, String> extFields = new HashMap<>();\n        for (Field f: fields) {\n            Class<?> c = f.getType();\n            if (c.equals(String.class)) {\n                extFields.put(f.getName(), \"str\");\n            } else if (c.equals(Integer.class) || c.equals(int.class)) {\n                extFields.put(f.getName(), \"123\");\n            } else if (c.equals(Long.class) || c.equals(long.class)) {\n                extFields.put(f.getName(), \"1234\");\n            } else if (c.equals(Boolean.class) || c.equals(boolean.class)) {\n                extFields.put(f.getName(), \"true\");\n            } else {\n                throw new RuntimeException(f.getName() + \":\" + f.getType().getName());\n            }\n        }\n        return extFields;\n    }\n\n    private void check(RemotingCommand command, List<Field> fields,\n            Class<? extends CommandCustomHeader> classHeader) throws Exception {\n        CommandCustomHeader o1 = command.decodeCommandCustomHeaderDirectly(classHeader, false);\n        CommandCustomHeader o2 = classHeader.getDeclaredConstructor().newInstance();\n        ((FastCodesHeader)o2).decode(command.getExtFields());\n        for (Field f : fields) {\n            Object value1 = f.get(o1);\n            Object value2 = f.get(o2);\n            if (value1 == null) {\n                Assert.assertNull(value2);\n            } else {\n                Assert.assertEquals(value1, value2);\n            }\n        }\n    }\n\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/header/GetConsumeStatsRequestHeaderTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.List;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNull;\n\npublic class GetConsumeStatsRequestHeaderTest {\n\n    private GetConsumeStatsRequestHeader header;\n\n    @Before\n    public void setUp() {\n        header = new GetConsumeStatsRequestHeader();\n    }\n\n    @Test\n    public void updateTopicList_NullTopicList_DoesNotUpdate() {\n        header.updateTopicList(null);\n        assertNull(header.getTopicList());\n    }\n\n    @Test\n    public void updateTopicList_EmptyTopicList_SetsEmptyString() {\n        header.updateTopicList(Collections.emptyList());\n        assertNull(header.getTopicList());\n    }\n\n    @Test\n    public void updateTopicList_SingleTopic_SetsSingleTopicString() {\n        List<String> topicList = Collections.singletonList(\"TopicA\");\n        header.updateTopicList(topicList);\n        assertEquals(\"TopicA;\", header.getTopicList());\n    }\n\n    @Test\n    public void updateTopicList_MultipleTopics_SetsMultipleTopicsString() {\n        List<String> topicList = Arrays.asList(\"TopicA\", \"TopicB\", \"TopicC\");\n        header.updateTopicList(topicList);\n        assertEquals(\"TopicA;TopicB;TopicC;\", header.getTopicList());\n    }\n\n    @Test\n    public void updateTopicList_RepeatedTopics_SetsRepeatedTopicsString() {\n        List<String> topicList = Arrays.asList(\"TopicA\", \"TopicA\", \"TopicB\");\n        header.updateTopicList(topicList);\n        assertEquals(\"TopicA;TopicA;TopicB;\", header.getTopicList());\n    }\n\n    @Test\n    public void fetchTopicList_NullTopicList_ReturnsEmptyList() {\n        header.setTopicList(null);\n        List<String> topicList = header.fetchTopicList();\n        assertEquals(Collections.emptyList(), topicList);\n\n        header.updateTopicList(new ArrayList<>());\n        topicList = header.fetchTopicList();\n        assertEquals(Collections.emptyList(), topicList);\n    }\n\n    @Test\n    public void fetchTopicList_EmptyTopicList_ReturnsEmptyList() {\n        header.setTopicList(\"\");\n        List<String> topicList = header.fetchTopicList();\n        assertEquals(Collections.emptyList(), topicList);\n    }\n\n    @Test\n    public void fetchTopicList_BlankTopicList_ReturnsEmptyList() {\n        header.setTopicList(\"   \");\n        List<String> topicList = header.fetchTopicList();\n        assertEquals(Collections.emptyList(), topicList);\n    }\n\n    @Test\n    public void fetchTopicList_SingleTopic_ReturnsSingleTopicList() {\n        header.setTopicList(\"TopicA\");\n        List<String> topicList = header.fetchTopicList();\n        assertEquals(Collections.singletonList(\"TopicA\"), topicList);\n    }\n\n    @Test\n    public void fetchTopicList_MultipleTopics_ReturnsTopicList() {\n        header.setTopicList(\"TopicA;TopicB;TopicC\");\n        List<String> topicList = header.fetchTopicList();\n        assertEquals(Arrays.asList(\"TopicA\", \"TopicB\", \"TopicC\"), topicList);\n    }\n\n    @Test\n    public void fetchTopicList_TopicListEndsWithSeparator_ReturnsTopicList() {\n        header.setTopicList(\"TopicA;TopicB;\");\n        List<String> topicList = header.fetchTopicList();\n        assertEquals(Arrays.asList(\"TopicA\", \"TopicB\"), topicList);\n    }\n\n    @Test\n    public void fetchTopicList_TopicListStartsWithSeparator_ReturnsTopicList() {\n        header.setTopicList(\";TopicA;TopicB\");\n        List<String> topicList = header.fetchTopicList();\n        assertEquals(Arrays.asList(\"TopicA\", \"TopicB\"), topicList);\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/header/SendMessageRequestHeaderV2Test.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.header;\n\nimport java.nio.ByteBuffer;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class SendMessageRequestHeaderV2Test {\n    SendMessageRequestHeaderV2 header = new SendMessageRequestHeaderV2();\n    String topic = \"test\";\n    int queueId = 5;\n\n    @Test\n    public void testEncodeDecode() throws RemotingCommandException {\n        header.setQueueId(queueId);\n        header.setTopic(topic);\n\n        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, header);\n        ByteBuffer buffer = remotingCommand.encode();\n\n        //Simulate buffer being read in NettyDecoder\n        buffer.getInt();\n        byte[] bytes = new byte[buffer.limit() - 4];\n        buffer.get(bytes, 0, buffer.limit() - 4);\n        buffer = ByteBuffer.wrap(bytes);\n\n        RemotingCommand decodeRequest = RemotingCommand.decode(buffer);\n        assertThat(decodeRequest.getExtFields().get(\"e\")).isEqualTo(String.valueOf(queueId));\n        assertThat(decodeRequest.getExtFields().get(\"b\")).isEqualTo(topic);\n    }\n}"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/heartbeat/SubscriptionDataTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.heartbeat;\n\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.assertj.core.util.Sets;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class SubscriptionDataTest {\n\n    @Test\n    public void testConstructor1() {\n        SubscriptionData subscriptionData = new SubscriptionData();\n        assertThat(subscriptionData.getTopic()).isNull();\n        assertThat(subscriptionData.getSubString()).isNull();\n        assertThat(subscriptionData.getSubVersion()).isLessThanOrEqualTo(System.currentTimeMillis());\n        assertThat(subscriptionData.getExpressionType()).isEqualTo(ExpressionType.TAG);\n        assertThat(subscriptionData.getFilterClassSource()).isNull();\n        assertThat(subscriptionData.getCodeSet()).isEmpty();\n        assertThat(subscriptionData.getTagsSet()).isEmpty();\n        assertThat(subscriptionData.isClassFilterMode()).isFalse();\n    }\n\n    @Test\n    public void testConstructor2() {\n        SubscriptionData subscriptionData = new SubscriptionData(\"TOPICA\", \"*\");\n        assertThat(subscriptionData.getTopic()).isEqualTo(\"TOPICA\");\n        assertThat(subscriptionData.getSubString()).isEqualTo(\"*\");\n        assertThat(subscriptionData.getSubVersion()).isLessThanOrEqualTo(System.currentTimeMillis());\n        assertThat(subscriptionData.getExpressionType()).isEqualTo(ExpressionType.TAG);\n        assertThat(subscriptionData.getFilterClassSource()).isNull();\n        assertThat(subscriptionData.getCodeSet()).isEmpty();\n        assertThat(subscriptionData.getTagsSet()).isEmpty();\n        assertThat(subscriptionData.isClassFilterMode()).isFalse();\n    }\n\n\n    @Test\n    public void testHashCodeNotEquals() {\n        SubscriptionData subscriptionData = new SubscriptionData(\"TOPICA\", \"*\");\n        subscriptionData.setCodeSet(Sets.newLinkedHashSet(1, 2, 3));\n        subscriptionData.setTagsSet(Sets.newLinkedHashSet(\"TAGA\", \"TAGB\", \"TAG3\"));\n        assertThat(subscriptionData.hashCode()).isNotEqualTo(System.identityHashCode(subscriptionData));\n    }\n\n    @Test\n    public void testFromJson() throws Exception {\n        SubscriptionData subscriptionData = new SubscriptionData(\"TOPICA\", \"*\");\n        subscriptionData.setFilterClassSource(\"TestFilterClassSource\");\n        subscriptionData.setCodeSet(Sets.newLinkedHashSet(1, 2, 3));\n        subscriptionData.setTagsSet(Sets.newLinkedHashSet(\"TAGA\", \"TAGB\", \"TAG3\"));\n        String json = RemotingSerializable.toJson(subscriptionData, true);\n        SubscriptionData fromJson = RemotingSerializable.fromJson(json, SubscriptionData.class);\n        assertThat(subscriptionData).isEqualTo(fromJson);\n        assertThat(subscriptionData).isEqualByComparingTo(fromJson);\n        assertThat(subscriptionData.getFilterClassSource()).isEqualTo(\"TestFilterClassSource\");\n        assertThat(fromJson.getFilterClassSource()).isNull();\n    }\n\n\n    @Test\n    public void testCompareTo() {\n        SubscriptionData subscriptionData = new SubscriptionData(\"TOPICA\", \"*\");\n        SubscriptionData subscriptionData1 = new SubscriptionData(\"TOPICBA\", \"*\");\n        assertThat(subscriptionData.compareTo(subscriptionData1)).isEqualTo(\"TOPICA@*\".compareTo(\"TOPICB@*\"));\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/route/TopicRouteDataTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.route;\n\n\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\n\npublic class TopicRouteDataTest {\n    @Test\n    public void testTopicRouteDataClone() throws Exception {\n\n        TopicRouteData topicRouteData = new TopicRouteData();\n\n        QueueData queueData = new QueueData();\n        queueData.setBrokerName(\"broker-a\");\n        queueData.setPerm(6);\n        queueData.setReadQueueNums(8);\n        queueData.setWriteQueueNums(8);\n        queueData.setTopicSysFlag(0);\n\n        List<QueueData> queueDataList = new ArrayList<>();\n        queueDataList.add(queueData);\n\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(0L, \"192.168.0.47:10911\");\n        brokerAddrs.put(1L, \"192.168.0.47:10921\");\n\n        BrokerData brokerData = new BrokerData();\n        brokerData.setBrokerAddrs(brokerAddrs);\n        brokerData.setBrokerName(\"broker-a\");\n        brokerData.setCluster(\"TestCluster\");\n\n        List<BrokerData> brokerDataList = new ArrayList<>();\n        brokerDataList.add(brokerData);\n\n        topicRouteData.setBrokerDatas(brokerDataList);\n        topicRouteData.setFilterServerTable(new HashMap<>());\n        topicRouteData.setQueueDatas(queueDataList);\n\n        assertThat(new TopicRouteData(topicRouteData)).isEqualTo(topicRouteData);\n\n    }\n\n    @Test\n    public void testTopicRouteDataJsonSerialize() throws Exception {\n\n        TopicRouteData topicRouteData = new TopicRouteData();\n\n        QueueData queueData = new QueueData();\n        queueData.setBrokerName(\"broker-a\");\n        queueData.setPerm(6);\n        queueData.setReadQueueNums(8);\n        queueData.setWriteQueueNums(8);\n        queueData.setTopicSysFlag(0);\n\n        List<QueueData> queueDataList = new ArrayList<>();\n        queueDataList.add(queueData);\n\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(0L, \"192.168.0.47:10911\");\n        brokerAddrs.put(1L, \"192.168.0.47:10921\");\n\n        BrokerData brokerData = new BrokerData();\n        brokerData.setBrokerAddrs(brokerAddrs);\n        brokerData.setBrokerName(\"broker-a\");\n        brokerData.setCluster(\"TestCluster\");\n\n        List<BrokerData> brokerDataList = new ArrayList<>();\n        brokerDataList.add(brokerData);\n\n        topicRouteData.setBrokerDatas(brokerDataList);\n        topicRouteData.setFilterServerTable(new HashMap<>());\n        topicRouteData.setQueueDatas(queueDataList);\n\n        String topicRouteDataJsonStr = RemotingSerializable.toJson(topicRouteData, true);\n        TopicRouteData topicRouteDataFromJson = RemotingSerializable.fromJson(topicRouteDataJsonStr, TopicRouteData.class);\n\n        assertThat(topicRouteDataJsonStr).isNotEqualTo(topicRouteDataFromJson);\n        assertThat(topicRouteDataFromJson.getBrokerDatas()).isEqualTo(topicRouteData.getBrokerDatas());\n        assertThat(topicRouteDataFromJson.getFilterServerTable()).isEqualTo(topicRouteData.getFilterServerTable());\n        assertThat(topicRouteDataFromJson.getQueueDatas()).isEqualTo(topicRouteData.getQueueDatas());\n\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicQueueMappingTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.statictopic;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.alibaba.fastjson2.JSONObject;\nimport com.google.common.collect.ImmutableList;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport java.util.Map;\n\npublic class TopicQueueMappingTest {\n\n    @Test\n    public void testJsonSerialize() {\n        LogicQueueMappingItem mappingItem = new LogicQueueMappingItem(1, 2, \"broker01\", 33333333333333333L, 44444444444444444L, 555555555555555555L, 6666666666666666L, 77777777777777777L);\n        String mappingItemJson = JSON.toJSONString(mappingItem) ;\n        {\n            Map<String, Object> mappingItemMap = JSON.parseObject(mappingItemJson, Map.class);\n            Assert.assertEquals(8, mappingItemMap.size());\n            Assert.assertEquals(mappingItemMap.get(\"bname\"), mappingItem.getBname());\n            Assert.assertEquals(mappingItemMap.get(\"gen\"), mappingItem.getGen());\n            Assert.assertEquals(mappingItemMap.get(\"logicOffset\"), mappingItem.getLogicOffset());\n            Assert.assertEquals(mappingItemMap.get(\"startOffset\"), mappingItem.getStartOffset());\n            Assert.assertEquals(mappingItemMap.get(\"endOffset\"), mappingItem.getEndOffset());\n            Assert.assertEquals(mappingItemMap.get(\"timeOfStart\"), mappingItem.getTimeOfStart());\n            Assert.assertEquals(mappingItemMap.get(\"timeOfEnd\"), mappingItem.getTimeOfEnd());\n\n        }\n        //test the decode encode\n        {\n            LogicQueueMappingItem mappingItemFromJson = RemotingSerializable.fromJson(mappingItemJson, LogicQueueMappingItem.class);\n            Assert.assertEquals(mappingItem, mappingItemFromJson);\n            Assert.assertEquals(mappingItemJson, RemotingSerializable.toJson(mappingItemFromJson, false));\n        }\n        TopicQueueMappingDetail mappingDetail = new TopicQueueMappingDetail(\"test\", 1, \"broker01\", System.currentTimeMillis());\n        TopicQueueMappingDetail.putMappingInfo(mappingDetail, 0, ImmutableList.of(mappingItem));\n\n        String mappingDetailJson = JSON.toJSONString(mappingDetail);\n        {\n            Map  mappingDetailMap = JSON.parseObject(mappingDetailJson);\n            Assert.assertTrue(mappingDetailMap.containsKey(\"currIdMap\"));\n            Assert.assertEquals(8, mappingDetailMap.size());\n            Assert.assertEquals(1, ((JSONObject) mappingDetailMap.get(\"hostedQueues\")).size());\n        }\n        {\n            TopicQueueMappingDetail mappingDetailFromJson = RemotingSerializable.decode(mappingDetailJson.getBytes(), TopicQueueMappingDetail.class);\n            Assert.assertEquals(1, mappingDetailFromJson.getHostedQueues().size());\n            Assert.assertEquals(1, mappingDetailFromJson.getHostedQueues().get(0).size());\n            Assert.assertEquals(mappingItem, mappingDetailFromJson.getHostedQueues().get(0).get(0));\n            Assert.assertEquals(mappingDetailJson, RemotingSerializable.toJson(mappingDetailFromJson, false));\n        }\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicQueueMappingUtilsTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.statictopic;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Random;\nimport java.util.Set;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class TopicQueueMappingUtilsTest {\n\n\n    private Set<String> buildTargetBrokers(int num) {\n        return buildTargetBrokers(num, \"\");\n    }\n\n    private Set<String> buildTargetBrokers(int num, String suffix) {\n        Set<String> brokers = new HashSet<>();\n        for (int i = 0; i < num; i++) {\n            brokers.add(\"broker\" + suffix + i);\n        }\n        return brokers;\n    }\n\n    private Map<String, Integer> buildBrokerNumMap(int num) {\n        Map<String, Integer> map = new HashMap<>();\n        for (int i = 0; i < num; i++) {\n            map.put(\"broker\" + i, 0);\n        }\n        return map;\n    }\n\n    private Map<String, Integer> buildBrokerNumMap(int num, int queues) {\n        Map<String, Integer> map = new HashMap<>();\n        int random = new Random().nextInt(num);\n        for (int i = 0; i < num; i++) {\n            map.put(\"broker\" + i, queues);\n            if (i == random) {\n                map.put(\"broker\" + i, queues + 1);\n            }\n        }\n        return map;\n    }\n\n    private void testIdToBroker(Map<Integer, String> idToBroker, Map<String, Integer> brokerNumMap) {\n        Map<String, Integer> brokerNumOther = new HashMap<>();\n        for (int i = 0; i < idToBroker.size(); i++) {\n            Assert.assertTrue(idToBroker.containsKey(i));\n            String broker = idToBroker.get(i);\n            if (brokerNumOther.containsKey(broker)) {\n                brokerNumOther.put(broker, brokerNumOther.get(broker) + 1);\n            } else {\n                brokerNumOther.put(broker, 1);\n            }\n        }\n        Assert.assertEquals(brokerNumMap.size(), brokerNumOther.size());\n        for (Map.Entry<String, Integer> entry: brokerNumOther.entrySet()) {\n            Assert.assertEquals(entry.getValue(), brokerNumMap.get(entry.getKey()));\n        }\n    }\n\n    @Test\n    public void testAllocator() {\n        //stability\n        for (int i = 0; i < 10; i++) {\n            int num = 3;\n            Map<String, Integer> brokerNumMap = buildBrokerNumMap(num);\n            TopicQueueMappingUtils.MappingAllocator  allocator = TopicQueueMappingUtils.buildMappingAllocator(new HashMap<>(), brokerNumMap,  null);\n            allocator.upToNum(num * 2);\n            for (Map.Entry<String, Integer> entry: allocator.getBrokerNumMap().entrySet()) {\n                Assert.assertEquals(2L, entry.getValue().longValue());\n            }\n            Assert.assertEquals(num * 2, allocator.getIdToBroker().size());\n            testIdToBroker(allocator.idToBroker, allocator.getBrokerNumMap());\n\n            allocator.upToNum(num * 3 - 1);\n\n            for (Map.Entry<String, Integer> entry: allocator.getBrokerNumMap().entrySet()) {\n                Assert.assertTrue(entry.getValue() >= 2);\n                Assert.assertTrue(entry.getValue() <= 3);\n            }\n            Assert.assertEquals(num * 3 - 1, allocator.getIdToBroker().size());\n            testIdToBroker(allocator.idToBroker, allocator.getBrokerNumMap());\n        }\n    }\n\n    @Test\n    public void testRemappingAllocator() {\n        for (int i = 0; i < 10; i++) {\n            int num = (i + 2) * 2;\n            Map<String, Integer> brokerNumMap = buildBrokerNumMap(num);\n            Map<String, Integer> brokerNumMapBeforeRemapping = buildBrokerNumMap(num, num);\n            TopicQueueMappingUtils.MappingAllocator  allocator = TopicQueueMappingUtils.buildMappingAllocator(new HashMap<>(), brokerNumMap, brokerNumMapBeforeRemapping);\n            allocator.upToNum(num * num + 1);\n            Assert.assertEquals(brokerNumMapBeforeRemapping, allocator.getBrokerNumMap());\n        }\n    }\n\n\n    @Test(expected = RuntimeException.class)\n    public void testTargetBrokersComplete() {\n        String topic = \"static\";\n        String broker1 = \"broker1\";\n        String broker2 = \"broker2\";\n        Set<String> targetBrokers = new HashSet<>();\n        targetBrokers.add(broker1);\n        Map<String, TopicConfigAndQueueMapping> brokerConfigMap = new HashMap<>();\n        TopicQueueMappingDetail mappingDetail = new TopicQueueMappingDetail(topic, 0, broker2, 0);\n        mappingDetail.getHostedQueues().put(1, new ArrayList<>());\n        brokerConfigMap.put(broker2, new TopicConfigAndQueueMapping(new TopicConfig(topic, 0, 0), mappingDetail));\n        TopicQueueMappingUtils.checkTargetBrokersComplete(targetBrokers, brokerConfigMap);\n    }\n\n\n\n    @Test\n    public void testCreateStaticTopic() {\n        String topic = \"static\";\n        int queueNum;\n        Map<String, TopicConfigAndQueueMapping> brokerConfigMap = new HashMap<>();\n        for (int i = 1; i < 10; i++) {\n            Set<String> targetBrokers = buildTargetBrokers(2 * i);\n            Set<String> nonTargetBrokers = buildTargetBrokers(2 * i, \"test\");\n            queueNum = 10 * i;\n            TopicRemappingDetailWrapper wrapper  = TopicQueueMappingUtils.createTopicConfigMapping(topic, queueNum, targetBrokers, brokerConfigMap);\n            Assert.assertEquals(wrapper.getBrokerConfigMap(), brokerConfigMap);\n            Assert.assertEquals(2 * i, brokerConfigMap.size());\n\n            //do the check manually\n            Map.Entry<Long, Integer> maxEpochAndNum = TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap);\n            Assert.assertEquals(queueNum, maxEpochAndNum.getValue().longValue());\n            Map<Integer, TopicQueueMappingOne> globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(TopicQueueMappingUtils.getMappingDetailFromConfig(brokerConfigMap.values())), false, true);\n            TopicQueueMappingUtils.checkIfReusePhysicalQueue(globalIdMap.values());\n            TopicQueueMappingUtils.checkPhysicalQueueConsistence(brokerConfigMap);\n\n            for (Map.Entry<String, TopicConfigAndQueueMapping> entry : brokerConfigMap.entrySet()) {\n                TopicConfigAndQueueMapping configMapping = entry.getValue();\n                if (nonTargetBrokers.contains(configMapping.getMappingDetail().bname)) {\n                    Assert.assertEquals(0, configMapping.getReadQueueNums());\n                    Assert.assertEquals(0, configMapping.getWriteQueueNums());\n                    Assert.assertEquals(0, configMapping.getMappingDetail().getHostedQueues().size());\n                } else {\n                    Assert.assertEquals(5, configMapping.getReadQueueNums());\n                    Assert.assertEquals(5, configMapping.getWriteQueueNums());\n                    Assert.assertTrue(configMapping.getMappingDetail().epoch > System.currentTimeMillis());\n                    for (List<LogicQueueMappingItem> items: configMapping.getMappingDetail().getHostedQueues().values()) {\n                        for (LogicQueueMappingItem item: items) {\n                            Assert.assertEquals(0, item.getStartOffset());\n                            Assert.assertEquals(0, item.getLogicOffset());\n                            TopicConfig topicConfig = brokerConfigMap.get(item.getBname());\n                            Assert.assertTrue(item.getQueueId() < topicConfig.getWriteQueueNums());\n                        }\n                    }\n                }\n            }\n        }\n    }\n\n    @Test\n    public void testRemappingStaticTopic() {\n        String topic = \"static\";\n        int queueNum = 7;\n        Map<String, TopicConfigAndQueueMapping> brokerConfigMap = new HashMap<>();\n        Set<String>  originalBrokers = buildTargetBrokers(2);\n        TopicRemappingDetailWrapper wrapper  = TopicQueueMappingUtils.createTopicConfigMapping(topic, queueNum, originalBrokers, brokerConfigMap);\n        Assert.assertEquals(wrapper.getBrokerConfigMap(), brokerConfigMap);\n        Assert.assertEquals(2, brokerConfigMap.size());\n\n        {\n            //do the check manually\n            TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap);\n            TopicQueueMappingUtils.checkPhysicalQueueConsistence(brokerConfigMap);\n            Map<Integer, TopicQueueMappingOne> globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(TopicQueueMappingUtils.getMappingDetailFromConfig(brokerConfigMap.values())), false, true);\n            TopicQueueMappingUtils.checkIfReusePhysicalQueue(globalIdMap.values());\n        }\n\n        for (int i = 0; i < 10; i++) {\n            Set<String> targetBrokers = buildTargetBrokers(2, \"test\" + i);\n            TopicQueueMappingUtils.remappingStaticTopic(topic, brokerConfigMap, targetBrokers);\n            //do the check manually\n            TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap);\n            TopicQueueMappingUtils.checkPhysicalQueueConsistence(brokerConfigMap);\n            Map<Integer, TopicQueueMappingOne> globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(TopicQueueMappingUtils.getMappingDetailFromConfig(brokerConfigMap.values())), false, true);\n            TopicQueueMappingUtils.checkIfReusePhysicalQueue(globalIdMap.values());\n            TopicQueueMappingUtils.checkLeaderInTargetBrokers(globalIdMap.values(), targetBrokers);\n\n            Assert.assertEquals((i + 2) * 2, brokerConfigMap.size());\n\n            //check and complete the logicOffset\n            for (Map.Entry<String, TopicConfigAndQueueMapping> entry : brokerConfigMap.entrySet()) {\n                TopicConfigAndQueueMapping configMapping = entry.getValue();\n                if (!targetBrokers.contains(configMapping.getMappingDetail().bname)) {\n                    continue;\n                }\n                for (List<LogicQueueMappingItem> items: configMapping.getMappingDetail().getHostedQueues().values()) {\n                    Assert.assertEquals(i + 2, items.size());\n                    items.get(items.size() - 1).setLogicOffset(i + 1);\n                }\n            }\n        }\n    }\n\n    @Test\n    public void testRemappingStaticTopicStability() {\n        String topic = \"static\";\n        int queueNum = 7;\n        Map<String, TopicConfigAndQueueMapping> brokerConfigMap = new HashMap<>();\n        Set<String>  originalBrokers = buildTargetBrokers(2);\n        {\n            TopicRemappingDetailWrapper wrapper  = TopicQueueMappingUtils.createTopicConfigMapping(topic, queueNum, originalBrokers,  brokerConfigMap);\n            Assert.assertEquals(wrapper.getBrokerConfigMap(), brokerConfigMap);\n            Assert.assertEquals(2, brokerConfigMap.size());\n        }\n        for (int i = 0; i < 10; i++) {\n            TopicRemappingDetailWrapper wrapper = TopicQueueMappingUtils.remappingStaticTopic(topic, brokerConfigMap, originalBrokers);\n            Assert.assertEquals(wrapper.getBrokerConfigMap(), brokerConfigMap);\n            Assert.assertEquals(2, brokerConfigMap.size());\n            Assert.assertTrue(wrapper.getBrokerToMapIn().isEmpty());\n            Assert.assertTrue(wrapper.getBrokerToMapOut().isEmpty());\n        }\n    }\n\n\n\n    @Test\n    public void testUtilsCheck() {\n        String topic = \"static\";\n        int queueNum = 10;\n        Map<String, TopicConfigAndQueueMapping> brokerConfigMap = new HashMap<>();\n        Set<String> targetBrokers = buildTargetBrokers(2);\n        TopicRemappingDetailWrapper wrapper  = TopicQueueMappingUtils.createTopicConfigMapping(topic, queueNum, targetBrokers, brokerConfigMap);\n        Assert.assertEquals(wrapper.getBrokerConfigMap(), brokerConfigMap);\n        Assert.assertEquals(2, brokerConfigMap.size());\n        TopicConfigAndQueueMapping configMapping = brokerConfigMap.values().iterator().next();\n        List<LogicQueueMappingItem> items = configMapping.getMappingDetail().getHostedQueues().values().iterator().next();\n        Map.Entry<Long, Integer> maxEpochNum = TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap);\n        int exceptionNum = 0;\n        try {\n            configMapping.getMappingDetail().setTopic(\"xxxx\");\n            TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap);\n        } catch (RuntimeException ignore) {\n            exceptionNum++;\n            configMapping.getMappingDetail().setTopic(topic);\n            TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap);\n        }\n\n        try {\n            configMapping.getMappingDetail().setTotalQueues(1);\n            TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap);\n        } catch (RuntimeException ignore) {\n            exceptionNum++;\n            configMapping.getMappingDetail().setTotalQueues(10);\n            TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap);\n        }\n\n        try {\n            configMapping.getMappingDetail().setEpoch(0);\n            TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap);\n        } catch (RuntimeException ignore) {\n            exceptionNum++;\n            configMapping.getMappingDetail().setEpoch(maxEpochNum.getKey());\n            TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap);\n        }\n\n\n        try {\n            configMapping.getMappingDetail().getHostedQueues().put(10000, new ArrayList<>(Collections.singletonList(new LogicQueueMappingItem(1, 1, targetBrokers.iterator().next(), 0, 0, -1, -1, -1))));\n            TopicQueueMappingUtils.checkAndBuildMappingItems(TopicQueueMappingUtils.getMappingDetailFromConfig(brokerConfigMap.values()), false, true);\n        } catch (RuntimeException ignore) {\n            exceptionNum++;\n            configMapping.getMappingDetail().getHostedQueues().remove(10000);\n            TopicQueueMappingUtils.checkAndBuildMappingItems(TopicQueueMappingUtils.getMappingDetailFromConfig(brokerConfigMap.values()), false, true);\n        }\n\n        try {\n            configMapping.setWriteQueueNums(1);\n            TopicQueueMappingUtils.checkPhysicalQueueConsistence(brokerConfigMap);\n        } catch (RuntimeException ignore) {\n            exceptionNum++;\n            configMapping.setWriteQueueNums(5);\n            TopicQueueMappingUtils.checkPhysicalQueueConsistence(brokerConfigMap);\n        }\n\n        try {\n            items.add(new LogicQueueMappingItem(1, 1, targetBrokers.iterator().next(), 0, 0, -1, -1, -1));\n            Map<Integer, TopicQueueMappingOne> map = TopicQueueMappingUtils.checkAndBuildMappingItems(TopicQueueMappingUtils.getMappingDetailFromConfig(brokerConfigMap.values()), false, true);\n            TopicQueueMappingUtils.checkIfReusePhysicalQueue(map.values());\n        } catch (RuntimeException ignore) {\n            exceptionNum++;\n            items.remove(items.size() - 1);\n            Map<Integer, TopicQueueMappingOne> map = TopicQueueMappingUtils.checkAndBuildMappingItems(TopicQueueMappingUtils.getMappingDetailFromConfig(brokerConfigMap.values()), false, true);\n            TopicQueueMappingUtils.checkIfReusePhysicalQueue(map.values());\n        }\n        Assert.assertEquals(6, exceptionNum);\n\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/subscription/CustomizedRetryPolicyTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.subscription;\n\nimport java.util.concurrent.TimeUnit;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class CustomizedRetryPolicyTest {\n\n    @Test\n    public void testNextDelayDuration() {\n        CustomizedRetryPolicy customizedRetryPolicy = new CustomizedRetryPolicy();\n        long actual = customizedRetryPolicy.nextDelayDuration(0);\n        assertThat(actual).isEqualTo(TimeUnit.SECONDS.toMillis(10));\n        actual = customizedRetryPolicy.nextDelayDuration(10);\n        assertThat(actual).isEqualTo(TimeUnit.MINUTES.toMillis(9));\n    }\n\n    @Test\n    public void testNextDelayDurationOutOfRange() {\n        CustomizedRetryPolicy customizedRetryPolicy = new CustomizedRetryPolicy();\n        long actual = customizedRetryPolicy.nextDelayDuration(-1);\n        assertThat(actual).isEqualTo(TimeUnit.SECONDS.toMillis(10));\n        actual = customizedRetryPolicy.nextDelayDuration(100);\n        assertThat(actual).isEqualTo(TimeUnit.HOURS.toMillis(2));\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/subscription/ExponentialRetryPolicyTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.subscription;\n\nimport java.util.concurrent.TimeUnit;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class ExponentialRetryPolicyTest {\n\n    @Test\n    public void testNextDelayDuration() {\n        ExponentialRetryPolicy exponentialRetryPolicy = new ExponentialRetryPolicy();\n        long actual = exponentialRetryPolicy.nextDelayDuration(0);\n        assertThat(actual).isEqualTo(TimeUnit.SECONDS.toMillis(5));\n        actual = exponentialRetryPolicy.nextDelayDuration(10);\n        assertThat(actual).isEqualTo(TimeUnit.SECONDS.toMillis(1024 * 5));\n    }\n\n    @Test\n    public void testNextDelayDurationOutOfRange() {\n        ExponentialRetryPolicy exponentialRetryPolicy = new ExponentialRetryPolicy();\n        long actual = exponentialRetryPolicy.nextDelayDuration(-1);\n        assertThat(actual).isEqualTo(TimeUnit.SECONDS.toMillis(5));\n        actual = exponentialRetryPolicy.nextDelayDuration(100);\n        assertThat(actual).isEqualTo(TimeUnit.HOURS.toMillis(2));\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/subscription/GroupRetryPolicyTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.subscription;\n\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class GroupRetryPolicyTest {\n\n    @Test\n    public void testGetRetryPolicy() {\n        GroupRetryPolicy groupRetryPolicy = new GroupRetryPolicy();\n        RetryPolicy retryPolicy = groupRetryPolicy.getRetryPolicy();\n        assertThat(retryPolicy).isInstanceOf(CustomizedRetryPolicy.class);\n        groupRetryPolicy.setType(GroupRetryPolicyType.EXPONENTIAL);\n        retryPolicy = groupRetryPolicy.getRetryPolicy();\n        assertThat(retryPolicy).isInstanceOf(CustomizedRetryPolicy.class);\n\n        groupRetryPolicy.setType(GroupRetryPolicyType.CUSTOMIZED);\n        groupRetryPolicy.setCustomizedRetryPolicy(new CustomizedRetryPolicy());\n        retryPolicy = groupRetryPolicy.getRetryPolicy();\n        assertThat(retryPolicy).isInstanceOf(CustomizedRetryPolicy.class);\n\n        groupRetryPolicy.setType(GroupRetryPolicyType.EXPONENTIAL);\n        groupRetryPolicy.setExponentialRetryPolicy(new ExponentialRetryPolicy());\n        retryPolicy = groupRetryPolicy.getRetryPolicy();\n        assertThat(retryPolicy).isInstanceOf(ExponentialRetryPolicy.class);\n\n        groupRetryPolicy.setType(null);\n        retryPolicy = groupRetryPolicy.getRetryPolicy();\n        assertThat(retryPolicy).isInstanceOf(CustomizedRetryPolicy.class);\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/subscription/SimpleSubscriptionDataTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.subscription;\n\nimport com.google.common.collect.Sets;\nimport java.util.Set;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class SimpleSubscriptionDataTest {\n    @Test\n    public void testNotEqual() {\n        String topic = \"test-topic\";\n        String expressionType = \"TAG\";\n        String expression1 = \"test-expression-1\";\n        String expression2 = \"test-expression-2\";\n        SimpleSubscriptionData simpleSubscriptionData1 = new SimpleSubscriptionData(topic, expressionType, expression1, 1);\n        SimpleSubscriptionData simpleSubscriptionData2 = new SimpleSubscriptionData(topic, expressionType, expression2, 1);\n        assertThat(simpleSubscriptionData1.equals(simpleSubscriptionData2)).isFalse();\n    }\n\n    @Test\n    public void testEqual() {\n        String topic = \"test-topic\";\n        String expressionType = \"TAG\";\n        String expression1 = \"test-expression-1\";\n        String expression2 = \"test-expression-1\";\n        SimpleSubscriptionData simpleSubscriptionData1 = new SimpleSubscriptionData(topic, expressionType, expression1, 1);\n        SimpleSubscriptionData simpleSubscriptionData2 = new SimpleSubscriptionData(topic, expressionType, expression2, 1);\n        assertThat(simpleSubscriptionData1.equals(simpleSubscriptionData2)).isTrue();\n    }\n\n    @Test\n    public void testSetNotEqual() {\n        String topic = \"test-topic\";\n        String expressionType = \"TAG\";\n        String expression1 = \"test-expression-1\";\n        String expression2 = \"test-expression-2\";\n        Set<SimpleSubscriptionData> set1 = Sets.newHashSet(new SimpleSubscriptionData(topic, expressionType, expression1, 1));\n        Set<SimpleSubscriptionData> set2 = Sets.newHashSet(new SimpleSubscriptionData(topic, expressionType, expression2, 1));\n        assertThat(set1.equals(set2)).isFalse();\n    }\n\n    @Test\n    public void testSetEqual() {\n        String topic = \"test-topic\";\n        String expressionType = \"TAG\";\n        String expression1 = \"test-expression-1\";\n        String expression2 = \"test-expression-1\";\n        Set<SimpleSubscriptionData> set1 = Sets.newHashSet(new SimpleSubscriptionData(topic, expressionType, expression1, 1));\n        Set<SimpleSubscriptionData> set2 = Sets.newHashSet(new SimpleSubscriptionData(topic, expressionType, expression2, 1));\n        assertThat(set1.equals(set2)).isTrue();\n    }\n}"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/protocol/topic/OffsetMovedEventTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.protocol.topic;\n\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class OffsetMovedEventTest {\n\n    @Test\n    public void testFromJson() throws Exception {\n        OffsetMovedEvent event = mockOffsetMovedEvent();\n\n        String json = event.toJson();\n        OffsetMovedEvent fromJson = RemotingSerializable.fromJson(json, OffsetMovedEvent.class);\n\n        assertEquals(event, fromJson);\n    }\n\n    @Test\n    public void testFromBytes() throws Exception {\n        OffsetMovedEvent event = mockOffsetMovedEvent();\n\n        byte[] encodeData = event.encode();\n        OffsetMovedEvent decodeData = RemotingSerializable.decode(encodeData, OffsetMovedEvent.class);\n\n        assertEquals(event, decodeData);\n    }\n\n    private void assertEquals(OffsetMovedEvent srcData, OffsetMovedEvent decodeData) {\n        assertThat(decodeData.getConsumerGroup()).isEqualTo(srcData.getConsumerGroup());\n        assertThat(decodeData.getMessageQueue().getTopic())\n            .isEqualTo(srcData.getMessageQueue().getTopic());\n        assertThat(decodeData.getMessageQueue().getBrokerName())\n            .isEqualTo(srcData.getMessageQueue().getBrokerName());\n        assertThat(decodeData.getMessageQueue().getQueueId())\n            .isEqualTo(srcData.getMessageQueue().getQueueId());\n        assertThat(decodeData.getOffsetRequest()).isEqualTo(srcData.getOffsetRequest());\n        assertThat(decodeData.getOffsetNew()).isEqualTo(srcData.getOffsetNew());\n    }\n\n    private OffsetMovedEvent mockOffsetMovedEvent() {\n        OffsetMovedEvent event = new OffsetMovedEvent();\n        event.setConsumerGroup(\"test-group\");\n        event.setMessageQueue(new MessageQueue(\"test-topic\", \"test-broker\", 0));\n        event.setOffsetRequest(3000L);\n        event.setOffsetNew(1000L);\n        return event;\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/rpc/ClientMetadataTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.rpc;\n\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertTrue;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class ClientMetadataTest {\n\n    private ClientMetadata clientMetadata;\n\n    private final ConcurrentMap<String, TopicRouteData> topicRouteTable = new ConcurrentHashMap<>();\n\n    private final ConcurrentMap<String, ConcurrentMap<MessageQueue, String>> topicEndPointsTable = new ConcurrentHashMap<>();\n\n    private final ConcurrentMap<String, HashMap<Long, String>> brokerAddrTable = new ConcurrentHashMap<>();\n\n    private final String defaultTopic = \"defaultTopic\";\n\n    private final String defaultBroker = \"defaultBroker\";\n\n    @Before\n    public void init() throws IllegalAccessException {\n        clientMetadata = new ClientMetadata();\n\n        FieldUtils.writeDeclaredField(clientMetadata, \"topicRouteTable\", topicRouteTable, true);\n        FieldUtils.writeDeclaredField(clientMetadata, \"topicEndPointsTable\", topicEndPointsTable, true);\n        FieldUtils.writeDeclaredField(clientMetadata, \"brokerAddrTable\", brokerAddrTable, true);\n    }\n\n    @Test\n    public void testGetBrokerNameFromMessageQueue() {\n        MessageQueue mq1 = new MessageQueue(defaultTopic, \"broker0\", 0);\n        MessageQueue mq2 = new MessageQueue(defaultTopic, \"broker1\", 0);\n        ConcurrentMap<MessageQueue, String> messageQueueMap = new ConcurrentHashMap<>();\n        messageQueueMap.put(mq1, \"broker0\");\n        messageQueueMap.put(mq2, \"broker1\");\n        topicEndPointsTable.put(defaultTopic, messageQueueMap);\n\n        String actual = clientMetadata.getBrokerNameFromMessageQueue(mq1);\n        assertEquals(\"broker0\", actual);\n    }\n\n    @Test\n    public void testGetBrokerNameFromMessageQueueNotFound() {\n        MessageQueue mq = new MessageQueue(\"topic1\", \"broker0\", 0);\n        topicEndPointsTable.put(defaultTopic, new ConcurrentHashMap<>());\n\n        String actual = clientMetadata.getBrokerNameFromMessageQueue(mq);\n        assertEquals(\"broker0\", actual);\n    }\n\n    @Test\n    public void testFindMasterBrokerAddrNotFound() {\n        assertNull(clientMetadata.findMasterBrokerAddr(defaultBroker));\n    }\n\n    @Test\n    public void testFindMasterBrokerAddr() {\n        String defaultBrokerAddr = \"127.0.0.1:10911\";\n        brokerAddrTable.put(defaultBroker, new HashMap<>());\n        brokerAddrTable.get(defaultBroker).put(0L, defaultBrokerAddr);\n\n        String actual = clientMetadata.findMasterBrokerAddr(defaultBroker);\n        assertEquals(defaultBrokerAddr, actual);\n    }\n\n    @Test\n    public void testTopicRouteData2EndpointsForStaticTopicNotFound() {\n        TopicRouteData topicRouteData = new TopicRouteData();\n        topicRouteData.setTopicQueueMappingByBroker(null);\n\n        ConcurrentMap<MessageQueue, String> actual = ClientMetadata.topicRouteData2EndpointsForStaticTopic(defaultTopic, topicRouteData);\n        assertTrue(actual.isEmpty());\n    }\n\n    @Test\n    public void testTopicRouteData2EndpointsForStaticTopic() {\n        TopicRouteData topicRouteData = new TopicRouteData();\n        Map<String, TopicQueueMappingInfo> mappingInfos = new HashMap<>();\n        TopicQueueMappingInfo info = new TopicQueueMappingInfo();\n        info.setScope(\"scope\");\n        info.setCurrIdMap(new ConcurrentHashMap<>());\n        info.getCurrIdMap().put(0, 0);\n        info.setTotalQueues(1);\n        info.setBname(\"bname\");\n        mappingInfos.put(defaultBroker, info);\n        topicRouteData.setTopicQueueMappingByBroker(mappingInfos);\n\n        ConcurrentMap<MessageQueue, String> actual = ClientMetadata.topicRouteData2EndpointsForStaticTopic(defaultTopic, topicRouteData);\n        assertEquals(1, actual.size());\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/rpc/RpcClientImplTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.remoting.rpc;\n\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.InvokeCallback;\nimport org.apache.rocketmq.remoting.RemotingClient;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;\nimport org.apache.rocketmq.remoting.protocol.header.GetEarliestMsgStoretimeResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.GetMinOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.SearchOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicConfigAndQueueMapping;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport java.util.concurrent.Future;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.Mockito.doAnswer;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class RpcClientImplTest {\n\n    @Mock\n    private RemotingClient remotingClient;\n\n    @Mock\n    private ClientMetadata clientMetadata;\n\n    private RpcClientImpl rpcClient;\n\n    private MessageQueue mq;\n\n    @Mock\n    private RpcRequest request;\n\n    private final long defaultTimeout = 3000L;\n\n    @Before\n    public void init() throws IllegalAccessException {\n        rpcClient = new RpcClientImpl(clientMetadata, remotingClient);\n\n        String defaultBroker = \"brokerName\";\n        mq = new MessageQueue(\"defaultTopic\", defaultBroker, 0);\n        RpcRequestHeader header = mock(RpcRequestHeader.class);\n        when(request.getHeader()).thenReturn(header);\n        when(clientMetadata.getBrokerNameFromMessageQueue(mq)).thenReturn(defaultBroker);\n        when(clientMetadata.findMasterBrokerAddr(any())).thenReturn(\"127.0.0.1:10911\");\n    }\n\n    @Test\n    public void testInvoke_PULL_MESSAGE() throws Exception {\n        when(request.getCode()).thenReturn(RequestCode.PULL_MESSAGE);\n\n        doAnswer(invocation -> {\n            InvokeCallback callback = invocation.getArgument(3);\n            RemotingCommand response = mock(RemotingCommand.class);\n            when(response.getBody()).thenReturn(\"success\".getBytes());\n            PullMessageResponseHeader responseHeader = mock(PullMessageResponseHeader.class);\n            when(response.decodeCommandCustomHeader(PullMessageResponseHeader.class)).thenReturn(responseHeader);\n            callback.operationSucceed(response);\n            return null;\n        }).when(remotingClient).invokeAsync(\n                any(),\n                any(RemotingCommand.class),\n                anyLong(),\n                any(InvokeCallback.class));\n\n        Future<RpcResponse> future = rpcClient.invoke(mq, request, defaultTimeout);\n        RpcResponse actual = future.get();\n\n        assertEquals(ResponseCode.SUCCESS, actual.getCode());\n        assertEquals(\"success\", new String((byte[]) actual.getBody()));\n    }\n\n    @Test\n    public void testInvoke_GET_MIN_OFFSET() throws Exception {\n        when(request.getCode()).thenReturn(RequestCode.GET_MIN_OFFSET);\n\n        RemotingCommand responseCommand = mock(RemotingCommand.class);\n        when(responseCommand.getBody()).thenReturn(\"1\".getBytes());\n        GetMinOffsetResponseHeader responseHeader = mock(GetMinOffsetResponseHeader.class);\n        when(responseCommand.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class)).thenReturn(responseHeader);\n        when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand);\n\n        Future<RpcResponse> future = rpcClient.invoke(mq, request, defaultTimeout);\n        RpcResponse actual = future.get();\n\n        assertEquals(ResponseCode.SUCCESS, actual.getCode());\n        assertEquals(\"1\", new String((byte[]) actual.getBody()));\n    }\n\n    @Test\n    public void testInvoke_GET_MAX_OFFSET() throws Exception {\n        when(request.getCode()).thenReturn(RequestCode.GET_MAX_OFFSET);\n\n        RemotingCommand responseCommand = mock(RemotingCommand.class);\n        when(responseCommand.getBody()).thenReturn(\"1000\".getBytes());\n        GetMaxOffsetResponseHeader responseHeader = mock(GetMaxOffsetResponseHeader.class);\n        when(responseCommand.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class)).thenReturn(responseHeader);\n        when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand);\n\n        Future<RpcResponse> future = rpcClient.invoke(mq, request, defaultTimeout);\n        RpcResponse actual = future.get();\n\n        assertEquals(ResponseCode.SUCCESS, actual.getCode());\n        assertEquals(\"1000\", new String((byte[]) actual.getBody()));\n    }\n\n    @Test\n    public void testInvoke_SEARCH_OFFSET_BY_TIMESTAMP() throws Exception {\n        when(request.getCode()).thenReturn(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP);\n\n        RemotingCommand responseCommand = mock(RemotingCommand.class);\n        when(responseCommand.getBody()).thenReturn(\"1000\".getBytes());\n        SearchOffsetResponseHeader responseHeader = mock(SearchOffsetResponseHeader.class);\n        when(responseCommand.decodeCommandCustomHeader(SearchOffsetResponseHeader.class)).thenReturn(responseHeader);\n        when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand);\n\n        Future<RpcResponse> future = rpcClient.invoke(mq, request, defaultTimeout);\n        RpcResponse actual = future.get();\n\n        assertEquals(ResponseCode.SUCCESS, actual.getCode());\n        assertEquals(\"1000\", new String((byte[]) actual.getBody()));\n    }\n\n    @Test\n    public void testInvoke_GET_EARLIEST_MSG_STORETIME() throws Exception {\n        when(request.getCode()).thenReturn(RequestCode.GET_EARLIEST_MSG_STORETIME);\n\n        RemotingCommand responseCommand = mock(RemotingCommand.class);\n        when(responseCommand.getBody()).thenReturn(\"10000\".getBytes());\n        GetEarliestMsgStoretimeResponseHeader responseHeader = mock(GetEarliestMsgStoretimeResponseHeader.class);\n        when(responseCommand.decodeCommandCustomHeader(GetEarliestMsgStoretimeResponseHeader.class)).thenReturn(responseHeader);\n        when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand);\n\n        Future<RpcResponse> future = rpcClient.invoke(mq, request, defaultTimeout);\n        RpcResponse actual = future.get();\n\n        assertEquals(ResponseCode.SUCCESS, actual.getCode());\n        assertEquals(\"10000\", new String((byte[]) actual.getBody()));\n    }\n\n    @Test\n    public void testInvoke_QUERY_CONSUMER_OFFSET() throws Exception {\n        when(request.getCode()).thenReturn(RequestCode.QUERY_CONSUMER_OFFSET);\n\n        RemotingCommand responseCommand = mock(RemotingCommand.class);\n        when(responseCommand.getBody()).thenReturn(\"1000\".getBytes());\n        QueryConsumerOffsetResponseHeader responseHeader = mock(QueryConsumerOffsetResponseHeader.class);\n        when(responseCommand.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class)).thenReturn(responseHeader);\n        when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand);\n\n        Future<RpcResponse> future = rpcClient.invoke(mq, request, defaultTimeout);\n        RpcResponse actual = future.get();\n\n        assertEquals(ResponseCode.SUCCESS, actual.getCode());\n        assertEquals(\"1000\", new String((byte[]) actual.getBody()));\n    }\n\n    @Test\n    public void testInvoke_UPDATE_CONSUMER_OFFSET() throws Exception {\n        when(request.getCode()).thenReturn(RequestCode.UPDATE_CONSUMER_OFFSET);\n\n        RemotingCommand responseCommand = mock(RemotingCommand.class);\n        when(responseCommand.getBody()).thenReturn(\"success\".getBytes());\n        UpdateConsumerOffsetResponseHeader responseHeader = mock(UpdateConsumerOffsetResponseHeader.class);\n        when(responseCommand.decodeCommandCustomHeader(UpdateConsumerOffsetResponseHeader.class)).thenReturn(responseHeader);\n        when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand);\n\n        Future<RpcResponse> future = rpcClient.invoke(mq, request, defaultTimeout);\n        RpcResponse actual = future.get();\n\n        assertEquals(ResponseCode.SUCCESS, actual.getCode());\n        assertEquals(\"success\", new String((byte[]) actual.getBody()));\n    }\n\n    @Test\n    public void testInvoke_GET_TOPIC_STATS_INFO() throws Exception {\n        when(request.getCode()).thenReturn(RequestCode.GET_TOPIC_STATS_INFO);\n\n        RemotingCommand responseCommand = mock(RemotingCommand.class);\n        TopicStatsTable topicStatsTable = new TopicStatsTable();\n        when(responseCommand.getBody()).thenReturn(topicStatsTable.encode());\n        when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand);\n\n        Future<RpcResponse> future = rpcClient.invoke(mq, request, defaultTimeout);\n        RpcResponse actual = future.get();\n\n        assertEquals(ResponseCode.SUCCESS, actual.getCode());\n        assertTrue(actual.getBody() instanceof TopicStatsTable);\n    }\n\n    @Test\n    public void testInvoke_GET_TOPIC_CONFIG() throws Exception {\n        when(request.getCode()).thenReturn(RequestCode.GET_TOPIC_CONFIG);\n\n        RemotingCommand responseCommand = mock(RemotingCommand.class);\n        TopicConfigAndQueueMapping topicConfigAndQueueMapping = new TopicConfigAndQueueMapping();\n        when(responseCommand.getBody()).thenReturn(RemotingSerializable.encode(topicConfigAndQueueMapping));\n        when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand);\n\n        Future<RpcResponse> future = rpcClient.invoke(mq, request, defaultTimeout);\n        RpcResponse actual = future.get();\n\n        assertEquals(ResponseCode.SUCCESS, actual.getCode());\n        assertTrue(actual.getBody() instanceof TopicConfigAndQueueMapping);\n    }\n}\n"
  },
  {
    "path": "remoting/src/test/java/org/apache/rocketmq/remoting/rpc/RpcRequestHeaderTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.remoting.rpc;\n\nimport java.nio.ByteBuffer;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class RpcRequestHeaderTest {\n    String brokerName = \"brokerName1\";\n    String namespace = \"namespace1\";\n    boolean namespaced = true;\n    boolean oneway = false;\n    static class TestRequestHeader extends RpcRequestHeader {\n\n        @Override\n        public void checkFields() throws RemotingCommandException {\n\n        }\n    }\n\n    @Test\n    public void testEncodeDecode() throws RemotingCommandException {\n        TestRequestHeader requestHeader = new TestRequestHeader();\n        requestHeader.setBrokerName(brokerName);\n        requestHeader.setNamespace(namespace);\n        requestHeader.setNamespaced(namespaced);\n        requestHeader.setOneway(oneway);\n\n        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, requestHeader);\n        ByteBuffer buffer = remotingCommand.encode();\n\n        //Simulate buffer being read in NettyDecoder\n        buffer.getInt();\n        byte[] bytes = new byte[buffer.limit() - 4];\n        buffer.get(bytes, 0, buffer.limit() - 4);\n        buffer = ByteBuffer.wrap(bytes);\n\n        RemotingCommand decodeRequest = RemotingCommand.decode(buffer);\n        assertThat(decodeRequest.getExtFields().get(\"bname\")).isEqualTo(brokerName);\n        assertThat(decodeRequest.getExtFields().get(\"nsd\")).isEqualTo(String.valueOf(namespaced));\n        assertThat(decodeRequest.getExtFields().get(\"ns\")).isEqualTo(namespace);\n        assertThat(decodeRequest.getExtFields().get(\"oway\")).isEqualTo(String.valueOf(oneway));\n    }\n}"
  },
  {
    "path": "remoting/src/test/resources/certs/badClient.key",
    "content": "-----BEGIN ENCRYPTED PRIVATE KEY-----\nMIICoTAbBgkqhkiG9w0BBQMwDgQIc2h7vaLYK6gCAggABIICgNrkvD1Xxez79Jgk\nWhRJg06CG8UthncfeuymR4hgp9HIneUzUHOoaf64mpxUbDWe3YOzA29REcBQsjF0\nRpv+Uyg3cyDG14TmeRoSufOxB3MWLcIenoPPyNNtxe3XXmdkJTXX2YR0j7EOzH2v\nqlmuxmN4A7UonV5RdGxCz0sm7bU7EyZKdLO/DwBNxlX7ukcVLxAAqsc7ondclYj0\nSFJKk1nzfysCsk/Pq+q3PAVVpG6x5RFaLVS7Zt+gU6IEp+0S0eeYukkTjGh9PMPl\nwjCOcRiR3O+g4b3DevmW8TcoBqAZ2cFaf4lGhYlNBfa9PaQ3spJLL8l8xBbRIs8T\n3UnaFIa49r9DO/ZpCwpDeUE+URCx/SpcO6lchWQhdEuFt+DnFKOPYDSCHtHJSWHf\n9Z2bltjcYYPy/8nkPeqsO9vn4/r6jo+l7MYWKyWolLCW+7RYbpx5R2s4SBGtBP6w\nbwQOtOASbpG+mqTf7+ARpffHaZm9cKoKwobXigjDojPeaBCg5DgRuLIS1tO46Pjg\nUSJ8sZilXifUwc6qRZ/2KiTSiJYCPMJD2ZTvK2Inkv2qzg6X3kw7CYCaW+iDL9zN\ne3ES7bps1wZ6D8cGq80WUQgrtpaGAXLzIv4FvM5yDoqrre/dh/XDO9l2hYfUmRVv\nrynKdSxjhhyHaK2ei8cX4LGEIlRNiu9ZIxSYeUAy37IJ0rVC7vtBWTh30JTeMRop\niIPmygBMX2FEhQ2l/eS2lRhiybR0QXA4kCeJkVQas3aMMBGp2ThPNahLpzP82B7V\nf9137okQC95/KXRz/ZLYFsJtY/53206mG7gU/+dYsYI4slLAlnSe8k2sS0D9qkWJ\nVV9F7PM=\n-----END ENCRYPTED PRIVATE KEY-----\n"
  },
  {
    "path": "remoting/src/test/resources/certs/badClient.pem",
    "content": "-----BEGIN CERTIFICATE-----\nMIIC8zCCAdsCAQIwDQYJKoZIhvcNAQEFBQAwfDELMAkGA1UEBhMCemgxCzAJBgNV\nBAgMAnpqMQswCQYDVQQHDAJoejEPMA0GA1UECgwGYXBhY2hlMREwDwYDVQQLDAhy\nb2NrZXRtcTEOMAwGA1UEAwwFeXVrb24xHzAdBgkqhkiG9w0BCQEWEHl1a29uQGFw\nYWNoZS5vcmcwHhcNMTcxMjExMDk0NDExWhcNMTgwMTEwMDk0NDExWjCBhjELMAkG\nA1UEBhMCemgxCzAJBgNVBAgMAnpqMQswCQYDVQQHDAJoejEPMA0GA1UECgwGYXBh\nY2hlMREwDwYDVQQLDAhyb2NrZXRtcTEWMBQGA1UEAwwNZm9vYmFyLmNsaWVudDEh\nMB8GCSqGSIb3DQEJARYSZm9vQGJhci5jbGllbnQuY29tMIGfMA0GCSqGSIb3DQEB\nAQUAA4GNADCBiQKBgQC+3bvrKGF1Y9/kN5UBtf8bXRtxn6L1W6mCRrX6aHBb+vQp\nBEYk3Pwu/OLd7TkOC5zwjCIPIlwV4FaYnWh0KooqpmvXuKJLAQBFa8yGWERYys73\n9a/U31cu6lndnG2lZfb47NTy+KdzDYsqB4GfnASqA7PbxJHDU4Fu7wp7gN3HRQID\nAQABMA0GCSqGSIb3DQEBBQUAA4IBAQBsFroSKr3MbCq1HjWpCLDEz2uS4LQV6L1G\nsmNfGNY17ELOcY9uweBBXOsfKVOEizYJJqatbJlz6FmPkIbfsGW2Wospkp1gvYMy\nNGL27vX3rB5vOo5vdFITaaV9/dEu53A0iWdsn3wH/FJnMsqBmynb+/3FY+Lff9d1\nXBaXLr+DeBx4lrE8rWTvhWh8gqDkuNLBTygdH0+g8/xkqhQhLqjIlMCSnrG2cTfj\nLewizVcX/VZ6DNC2M2vjEFbCShclZHocG80N7udl5KNsLEU2jyO1F61Q0yo+VYGS\n7n8dRYgbOKyCjMdu69fAfZvp4aoy1SXqtjMphDh5R7y7mhP60e0A\n-----END CERTIFICATE-----\n"
  },
  {
    "path": "remoting/src/test/resources/certs/badServer.key",
    "content": "-----BEGIN PRIVATE KEY-----\nMIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBALBvxESq2VvSpJl1\nskv8SzyPYKgU8bZx37hEOCmoeYvd9gWNfeYZuITng2/5mpWX+zuAgKsgPU66YG0v\n++dT5GBQPr0Imb25IMl3xOY2eEiLeMokYiWbnA1C+pw1a27zMqk6pgbcRaMfLdh5\nnpusWtqBzZIxqo1TpaOGEmyQTNRlAgMBAAECgYBSigbUZOTIWxObov7lI0MDMsPx\n/dJSGpWhe3CWtHUgJJdKY7XpJlE3A6Nuh+N0ZiQm4ufOpodnxDMGAXOj9ZAZY16Y\ni7I0ayXepcpTqYqo0o0+ze2x7SECAXe26bqvLRuKG2hpUyM59vAmll9gmQM5n8z4\nZzoAzqRqkRHdo5bTxQJBAOF6SwSSfb8KEtTjWpJ48W1PO/NmKbW3QsNCWuk/w5p7\nE8L2g3nwakJiFmVNCga74rUbcgbCkw7y/lLeM8yC74MCQQDIUgCN/vuHm+eT85xk\nQoVKhDljXzLoog6wTUf5SMtrmUFTbQqyvw5xjHYdp3TWJM/Px8IyLxOr97sSnnft\nl7/3AkEAukYLv6U+GRs7X4DMDIG6AjIZNwXJo4PYtfMVo+i3seHH+6MoDw8c2eaq\n1dmFVPbXXgNkek04rHr2vIMxi90H/QJAAMOfUOtaFkhX986EGDXQwFoExgZE8XI8\n0BtbXO4UKJLrFuBhnBDygyhgAvjyjyaQzGAcs4hOcOd/BTEpj/R2PQJBANUKa9T9\nqWYhDhWN1Uj7qXhC1j2z/vTAzcYuwhpPRjt3RaVl27itI7cqiGquFhwfKZZFaOh5\npnnWHv63YbGQ2Qc=\n-----END PRIVATE KEY-----\n"
  },
  {
    "path": "remoting/src/test/resources/certs/badServer.pem",
    "content": "-----BEGIN CERTIFICATE-----\nMIIC5DCCAcwCAQEwDQYJKoZIhvcNAQEFBQAwfDELMAkGA1UEBhMCemgxCzAJBgNV\nBAgMAnpqMQswCQYDVQQHDAJoejEPMA0GA1UECgwGYXBhY2hlMREwDwYDVQQLDAhy\nb2NrZXRtcTEOMAwGA1UEAwwFeXVrb24xHzAdBgkqhkiG9w0BCQEWEHl1a29uQGFw\nYWNoZS5vcmcwHhcNMTcxMjExMDk0MzE3WhcNMTgwMTEwMDk0MzE3WjB4MQswCQYD\nVQQGEwJ6aDELMAkGA1UECAwCemoxCzAJBgNVBAcMAmh6MQ8wDQYDVQQKDAZhcGFj\naGUxETAPBgNVBAsMCHJvY2tldG1xMQ8wDQYDVQQDDAZmb29iYXIxGjAYBgkqhkiG\n9w0BCQEWC2Zvb0BiYXIuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCw\nb8REqtlb0qSZdbJL/Es8j2CoFPG2cd+4RDgpqHmL3fYFjX3mGbiE54Nv+ZqVl/s7\ngICrID1OumBtL/vnU+RgUD69CJm9uSDJd8TmNnhIi3jKJGIlm5wNQvqcNWtu8zKp\nOqYG3EWjHy3YeZ6brFragc2SMaqNU6WjhhJskEzUZQIDAQABMA0GCSqGSIb3DQEB\nBQUAA4IBAQAx+0Se3yIvUOe23oQp6UecaHtfXJCZmi1p5WbwJi7jUcYz78JB8oBj\ntVsa+1jftJG+cJJxqgxo2IeIAVbcEteO19xm7dc8tgfH/Bl0rxQz4WEYKb2oF/EQ\neRgcvj4uZ0d9WuprAvJgA4r0Slu2ZZ0cVkzi06NevTweTBYIKFzHaPShqUWEw8ki\n42V5jAtRve7sT0c4TH/01dd2fs3V4Ul3E2U3LOP6VizIfKckdht0Bh6B6/5L8wvH\n4l1f4ni7w34vXGANpmTP2FGjQQ3kYjKL7GzgMphh3Kozhil6g1GLMhxvp6ccCA9W\nm5g0cPa3RZnjI/FoD0lZ5S1Q5s9qXbLm\n-----END CERTIFICATE-----\n"
  },
  {
    "path": "remoting/src/test/resources/certs/ca.pem",
    "content": "-----BEGIN CERTIFICATE-----\nMIIDyzCCArOgAwIBAgIJAKzXC2VLdPclMA0GCSqGSIb3DQEBBQUAMHwxCzAJBgNV\nBAYTAnpoMQswCQYDVQQIDAJ6ajELMAkGA1UEBwwCaHoxDzANBgNVBAoMBmFwYWNo\nZTERMA8GA1UECwwIcm9ja2V0bXExDjAMBgNVBAMMBXl1a29uMR8wHQYJKoZIhvcN\nAQkBFhB5dWtvbkBhcGFjaGUub3JnMB4XDTE3MTIxMTA5MjUxNFoXDTE4MDExMDA5\nMjUxNFowfDELMAkGA1UEBhMCemgxCzAJBgNVBAgMAnpqMQswCQYDVQQHDAJoejEP\nMA0GA1UECgwGYXBhY2hlMREwDwYDVQQLDAhyb2NrZXRtcTEOMAwGA1UEAwwFeXVr\nb24xHzAdBgkqhkiG9w0BCQEWEHl1a29uQGFwYWNoZS5vcmcwggEiMA0GCSqGSIb3\nDQEBAQUAA4IBDwAwggEKAoIBAQDLUZi/zPj+7sYbfTng/gJeHpvvrWZkiudNwh1t\n5kxAusrJyGBkGm+xmRPJeQPZzbhfwfrz/UiQSbjlyV4K+SEZuNIHBSU80aTnXFWg\nwIgIAKvu3ZwYkcTjSDBvZv1DgbRkuqAB5ExsJ4vovoNqZcsLFLKsqT1G7lTAwRKU\n/FTKgD4g/zvhEoolonzKuk7CPivfKWFzcTpe8zRQlI0O9+j9Pq38F+5yxP7atK/b\nuYw36Efgt8nbkjusWIyXibpDMbAUroJNNYlFnunb+XKLpslkrIrfLGiMUq2Ru940\nooQaANYWzogRQeIofsMN6H9CCRXtVIzcgJJU3wWXGXPRuNr7AgMBAAGjUDBOMB0G\nA1UdDgQWBBTd3bmAcazOY2/TI/h4zaGhni+nJzAfBgNVHSMEGDAWgBTd3bmAcazO\nY2/TI/h4zaGhni+nJzAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4IBAQBp\nKRcnYsVtFZJejyt02+7SaMudTNRgh5SexFWsl1O7qWUc+fMVgMHHDzGRbkdevcdZ\n9uKDwUoa6M1wlOeosTTfQlH9b/AwW6QT7KqdpcpMXlmoV/PNCAVt2QeVclmplvqo\nRx8qUHNckvvzNZt1W6AkBG93P0BLK/3FMJDyYmxkstwnpBPf/3A+t5k2INUI7yQf\nB3Tqzs/4iQ3idCLqz2WhTNUYpZOREtpJMcFaOdMsGNnIF+LvkKGij0MPVd/mwJtL\nUvQXwbOWpCS7A73sWFqPnrSzpi4VwcvAsi8lUYXsc0H064oagb58zvYz3kXqybcb\nKQntj5dP4C3lLHUTTcAV\n-----END CERTIFICATE-----\n"
  },
  {
    "path": "remoting/src/test/resources/certs/client.key",
    "content": "-----BEGIN ENCRYPTED PRIVATE KEY-----\nMIICoTAbBgkqhkiG9w0BBQMwDgQI1vtPpDhOYRcCAggABIICgMHwgw0p9fx95R/+\ncWnNdEq8I3ZOOy2wDjammFvPrYXcCJzS3Xg/0GDJ8pdJRKrI7253e4u3mxf5oMuY\nRrvpB3KfdelU1k/5QKqOxL/N0gQafQLViN53f6JelyBEAmO1UxQtKZtkTrdZg8ZP\n0u1cPPWxmgNdn1Xx3taMw+Wo05ysHjnHJhOEDQ2WT3VXigiRmFSX3H567yjYMRD+\nzmvBq+qqR9JPbH9Cn7X1oRXX6c8VsZHWF/Ds0I4i+5zJxsSIuNZxjZw9XXNgXtFv\n7FEFC0HDgDQQUY/FNPUbmjQUp1y0YxoOBjlyIqBIx5FWxu95p2xITS0OimQPFT0o\nIngaSb+EKRDhqpLxxIVEbDdkQrdRqcmmLGJioAysExTBDsDwkaEJGOp44bLDM4QW\nSIA9SB01omuCXgn7RjUyVXb5g0Lz+Nvsfp1YXUkPDO9hILfz3eMHDSW7/FzbB81M\nr8URaTagQxBZnvIoCoWszLDXn3JwEjpZEA6y55Naptps3mMRf7+XMt42lX0e4y9a\nogNu5Zw/RZD9YcaTjC2z5XeKiMCs1Ymhy9iuzbo+eRGESqzvUE4VirtsiEwxJRci\nJHAvuAl3X4XnpTty4ahOU+DihM9lALxdU68CN9++7mx581pYuvjzrV+Z5+PuptZX\nAjCZmkZLDh8TCHSzWRqvP/Hcvo9BjW8l1Lq6tOa222PefSNCc6gs6Hq+jUghbabZ\n/ux4WuFc0Zd6bfQWAZohSvd78/ixsdJPNGm2OP+LUIrEDKIkLuH1PM0uq4wzJZNu\nBo7oJ5iFWF67u3MC8oq+BqOVKDNWaCMi7iiSrN2XW8FBo/rpx4Lf/VYREL+Y0mP6\nvzJrZqw=\n-----END ENCRYPTED PRIVATE KEY-----\n"
  },
  {
    "path": "remoting/src/test/resources/certs/client.pem",
    "content": "-----BEGIN CERTIFICATE-----\nMIIDATCCAekCAQIwDQYJKoZIhvcNAQEFBQAwfDELMAkGA1UEBhMCemgxCzAJBgNV\nBAgMAnpqMQswCQYDVQQHDAJoejEPMA0GA1UECgwGYXBhY2hlMREwDwYDVQQLDAhy\nb2NrZXRtcTEOMAwGA1UEAwwFeXVrb24xHzAdBgkqhkiG9w0BCQEWEHl1a29uQGFw\nYWNoZS5vcmcwIBcNMTgwMTE2MDYxNjQ0WhgPMjExNzEyMjMwNjE2NDRaMIGSMQsw\nCQYDVQQGEwJDTjERMA8GA1UECAwIWmhlamlhbmcxETAPBgNVBAcMCEhhbmd6aG91\nMQ8wDQYDVQQKDAZhcGFjaGUxETAPBgNVBAsMCHJvY2tldG1xMRgwFgYDVQQDDA9h\ncGFjaGUgcm9ja2V0bXExHzAdBgkqhkiG9w0BCQEWEHl1a29uQGFwYWNoZS5vcmcw\ngZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAOjPlSjZk37XLBJBc5G/qQNsNdVD\nvZnEGntrqW0UuHjF2T/LPtsGOavLP5wCHvn2zwMR2eCXZwKdKIzSvk0L3XOjH/XY\nOLgRa3cg90lV7Wzn9UMGq3nOjFtjIODPjtz3lwYAuAt1MH+K0E+ChuCFBgFqdY9U\nE0suW3DX0Mt/WB3pAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAFGPaZKyCZzQihKj\nn/7I1J0wKl1HrU7N4sOie8E+ntcpKeX9zKYAou/4Iy0qwgxgRsnucB1rDous560a\n+8DFDU8+FnikK9cQtKfQqu4F266IkkXolviZMSfkmB+NIsByIl95eMJlQHVlAvnX\nvnpGdhD/Jhs+acE1VHhO6K+8omKLA6Og8MmYGRwmnBLcxIvqoSNDlEShfQyjaECg\nI4bEi4ZhH3lSHE46FybJdoxDbj9IjHWqpOnjM23EOyfd1zcwOZJA7a54kfOpiTjz\nwrtes5yoQznun5WtGcLM8ZmyaQ+Jr3j6NyZhOwULzK1+A8YUsW6Ww39xTxQoIHEQ\n7eirb54=\n-----END CERTIFICATE-----\n"
  },
  {
    "path": "remoting/src/test/resources/certs/privkey.pem",
    "content": "-----BEGIN ENCRYPTED PRIVATE KEY-----\nMIIFDjBABgkqhkiG9w0BBQ0wMzAbBgkqhkiG9w0BBQwwDgQIBTqUKpwFlcUCAggA\nMBQGCCqGSIb3DQMHBAii5M3Oni0WEwSCBMj5amhdPBva0QxgsWWrSBkfvmc3QQbl\nYQE5gHXzL5oaG0lbXCLQe19pr9jFckdDH8uPIi+aRie3WpZXzYLdihjsV0CgE7iy\n90ac9fgzIZbJIk/WQIDgwUZm5dEYo2v+B0WwKiD5IHzlTlXM6HVv+DRdPwKjHjAt\nTCcCSPUgjKVxHWFtQzY7mqo8P8wcNQHGkEfoQQub6tEsUDeesjS0FoK5Z2oYsmhW\nd0PNuGXw3UIMbG109DmC2ILFuTf5WSc7mxI11qL9Z5wTmcFqN7KKb8+MIQEoteni\nHICOFfKQWn8er14lmYw9anQAyaeyF/JnYkmVB8vaHBFYs/5EFZtvznpEIIhLKCuG\nlve8PJQmfWuBlPdwwJhCXHrLvjfwku4jUF8febU0BHZ5HETaB195g8r9RWfdZcrG\nf3fMO4Kq/YoP6oSxKhMP4L2pwj57EMV9N5P87ZyDFNp9BwgIjCawDRUc1Gi9YKak\nrpDNabTCr0I3NW27VGGF9m/mby7BLragc01LgTH7SFWS+1D/61/V2YBDFmWn2yV4\n4eKGeBkR3w0m/nWWfNXko8UzM/hjJ4P7Njq8HXdvEpnbDRZWzwdTGWTEvn/TAI3h\nj7vmWUHdpOQgb0WGlvEUx3V9wi2Fc1rCseHtYZgLf3KdKYHauPAMSON7KBtKaFuU\n6685sUoJbhahN7ILfP3sDxM3VYjSvlPL00lgOdqDT/iO6pNXvnNnQROCCE4kcOQT\nuSnEu+wmFHj+QlC60ftRl6zGVqjBxf1+TGmzTEByAOfZtEQ8V/clzRI4BCxYbCAG\nmJSa+q1RSju8yClBkXGT2zfhUeNqJnXEIaD/uXCPVGg7hfLyCcVVSmL97aw9QAIe\nlBJN/4bdxXLnJaHFKyztRe9N97JAKY9HAPMKKhKtqprWB7LedTIPHtXnpoSjyTrG\nSEtlOTQ38s3v9bUPXqF7TYZb+ytj5bIQpl6+WqF9ZNj3gRyx7rcsILhBBg08olVQ\nWZwr7LlIxUcDlrbYmrwd9lsMz2nOW2CLCD7mVqJQa2Wm35l6vJHQAI0WiQlHnopC\nM2Y49JruWWim2lC8ZzHgTyiU54bIfkXKQxua8G5WGxpW4dDRrM1d0uYe9M1TOqvP\njFxq+XEIj/LntJpY7XIZs+33wuNLIVvkee9zsap++zYNH+KIGmbXz/HUO6gYeJYw\nEeaBTLfXtNlgHV9TpMjj3Js6p8hMoVDx27kzPOXa3nrFbHiHmUYY39ZSBCE53Wew\nSKIr/FlKtthzJwJkoMNxxsZ1QcI6WgmRANSC4oP9OyM+hPnWlISJZsy9UlXRZNEJ\n1lI8P/FUbdk/hsRN98j4pb/hzI0yyG1tKaj0TBipjdEsxfthKdwS2sE1wG+F2hrg\nA1jG+UOYmreQjCbrzEiq0J0H4wSL6/s/zN7SyXIWBG0UFalWFiehG5242mEOyX03\n0Yi94mPhF/kcqYsWZ/JJo6cq/EqgIeIqEzkbKx+TOsXk13K2y6apgvCxeHDDv3yT\nDqQueRgFicl0319yEK8ARnREFBm8D5oqwMHjJzVzjrqhFGLW1jfQG9HEW5A+s8WF\nd+2OtH1o/jVdAPXoP1DnxdF+G7fNXDI4cyjejC7uhLuxHCOx648UpRE9+mCiI2IO\nLDM=\n-----END ENCRYPTED PRIVATE KEY-----\n"
  },
  {
    "path": "remoting/src/test/resources/certs/server.key",
    "content": "-----BEGIN PRIVATE KEY-----\nMIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBAOsmp4YtrIRsBdBQ\nLyPImafCRynTJls3NNF4g6nZr9e0efBY830gw9kBebcm603sdZNl95fzRr2+srXi\n5FJbG7Fmq1+F0xLNK/kKWirGtNMT2DubmhVdKyXYJSvInoGRkrQzbOG0MdAyzE6Q\nO6OjjNN+xGkmadWyCyNF6S8YqMJTAgMBAAECgYEAj0OlnOIG0Ube4+N2VN7KfqKm\nqJy0Ka6gx14dGUY/E7Qo9n27GujzaSq09RkJExiVKZBeIH1fBAtC5f2uDV7kpy0l\nuNpTpQkbw0g2EQLxDsVwaUEYbu+t9qVeXoDd1vFeoXHBuRwvI9UW1BrxVtvKODia\n5StU8Lw4yjcm2lQalwECQQD/sKj56thIsIY7D9qBHk7fnFLd8aYzhnP2GsbZX4V/\nT1KHRxr/8MqdNQX53DE5qcyM/Mqu95FIpTAniUtvcBujAkEA62+fAMYFTAEWj4Z4\nvCmcoPqfVPWhBKFR/wo3L8uUARiIzlbYNU3LIqC2s16QO50+bLUd41oVHNw9Y+uM\nfxQpkQJACg/WpncSadHghmR6UchyjCQnsqo2wyJQX+fv2VAD/d2OPtqSem3sW0Fh\n6dI7cax36zhrdXUyl2xAt92URV9hBwJALX93sdWSxnpbWsc449wCydVFH00MnfFz\nAB+ARLtJ0eBk58M+qyZqgDmgtQ8sPmkH3EgwC3SoKdiiAIJPt2s1EQJBAKnISZZr\nqB2F2PfAW2JJbQlrPyVzkxhv9XYdiVNOErmuxLFae3AI7nECgGuFBtvmeqzm2yRj\n7RBMCmzyWG7MF3o=\n-----END PRIVATE KEY-----\n"
  },
  {
    "path": "remoting/src/test/resources/certs/server.pem",
    "content": "-----BEGIN CERTIFICATE-----\nMIIDATCCAekCAQIwDQYJKoZIhvcNAQEFBQAwfDELMAkGA1UEBhMCemgxCzAJBgNV\nBAgMAnpqMQswCQYDVQQHDAJoejEPMA0GA1UECgwGYXBhY2hlMREwDwYDVQQLDAhy\nb2NrZXRtcTEOMAwGA1UEAwwFeXVrb24xHzAdBgkqhkiG9w0BCQEWEHl1a29uQGFw\nYWNoZS5vcmcwIBcNMTgwMTE2MDYxMzQ5WhgPMjExNzEyMjMwNjEzNDlaMIGSMQsw\nCQYDVQQGEwJDTjERMA8GA1UECAwIWmhlamlhbmcxETAPBgNVBAcMCEhhbmd6aG91\nMQ8wDQYDVQQKDAZhcGFjaGUxETAPBgNVBAsMCHJvY2tldG1xMRgwFgYDVQQDDA9h\ncGFjaGUgcm9ja2V0bXExHzAdBgkqhkiG9w0BCQEWEHl1a29uQGFwYWNoZS5vcmcw\ngZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAOsmp4YtrIRsBdBQLyPImafCRynT\nJls3NNF4g6nZr9e0efBY830gw9kBebcm603sdZNl95fzRr2+srXi5FJbG7Fmq1+F\n0xLNK/kKWirGtNMT2DubmhVdKyXYJSvInoGRkrQzbOG0MdAyzE6QO6OjjNN+xGkm\nadWyCyNF6S8YqMJTAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAAzbwXyAULmXitiU\n+8/2vbUZQlzB/nXY52OIq7qu3F55hE5qlHkcVxG2JZjO3p5UETwOyNUpU4dpu3uT\n7WSdygH4Iagl87ILpGsob9pAf0joAbaXAY4sGDhg+WjR5JInAxbmT+QWZ+4NTuLQ\nfSudUSJrv+HmUlmcVOvLiNStgt9rbtcgJAvpVwY+iCv0HQziFuQxmOkDv09ZLzu/\nlxCMqnbgkEFYkwdntN6MVk38K3MovszedGO/n19hNOFss7nn5XDEeEnc6BqKGdck\nYDoy6amohY0Ds0o0gJ2rq0Y8Gjl9spQ3oeXpoNUoz84OF4KIBRTzSMv8CrmqPdFY\nZd2MGjw=\n-----END CERTIFICATE-----\n"
  },
  {
    "path": "remoting/src/test/resources/rmq.logback-test.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n<configuration>\n\n    <appender name=\"CONSOLE\" class=\"ch.qos.logback.core.ConsoleAppender\">\n        <layout class=\"ch.qos.logback.classic.PatternLayout\">\n            <Pattern>\n                %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n\n            </Pattern>\n        </layout>\n    </appender>\n\n    <logger name=\"org.apache.rocketmq\" level=\"error\" additivity=\"false\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </logger>\n\n    <root level=\"error\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </root>\n\n</configuration>"
  },
  {
    "path": "srvutil/BUILD.bazel",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\nload(\"//bazel:GenTestRules.bzl\", \"GenTestRules\")\n\njava_library(\n    name = \"srvutil\",\n    srcs = glob([\"src/main/java/**/*.java\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \"//common\",\n        \"@maven//:org_apache_commons_commons_lang3\",\n        \"@maven//:commons_validator_commons_validator\",\n        \"@maven//:com_github_luben_zstd_jni\",\n        \"@maven//:org_lz4_lz4_java\",\n        \"@maven//:io_netty_netty_all\",\n        \"@maven//:commons_cli_commons_cli\",\n        \"@maven//:com_googlecode_concurrentlinkedhashmap_concurrentlinkedhashmap_lru\",\n        \"@maven//:com_google_guava_guava\",\n        \"@maven//:io_github_aliyunmq_rocketmq_slf4j_api\",\n        \"@maven//:io_github_aliyunmq_rocketmq_logback_classic\",\n    ],\n)\n\njava_library(\n    name = \"tests\",\n    srcs = glob([\"src/test/java/**/*.java\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \":srvutil\",\n        \"//common\",\n        \"//:test_deps\",\n        \"@maven//:org_apache_commons_commons_lang3\", \n        \"@maven//:io_netty_netty_all\",               \n    ],\n)\n\nGenTestRules(\n    name = \"GeneratedTestRules\",\n    test_files = glob([\"src/test/java/**/*Test.java\"]),\n    deps = [\n        \":tests\",\n    ],\n)\n"
  },
  {
    "path": "srvutil/pom.xml",
    "content": "<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n    <parent>\n        <groupId>org.apache.rocketmq</groupId>\n        <artifactId>rocketmq-all</artifactId>\n        <version>${revision}</version>\n    </parent>\n\n    <modelVersion>4.0.0</modelVersion>\n    <packaging>jar</packaging>\n    <artifactId>rocketmq-srvutil</artifactId>\n    <name>rocketmq-srvutil ${project.version}</name>\n\n    <properties>\n        <project.root>${basedir}/..</project.root>\n    </properties>\n\n    <dependencies>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-remoting</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-common</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>commons-cli</groupId>\n            <artifactId>commons-cli</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>com.google.guava</groupId>\n            <artifactId>guava</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>com.googlecode.concurrentlinkedhashmap</groupId>\n            <artifactId>concurrentlinkedhashmap-lru</artifactId>\n        </dependency>\n    </dependencies>\n</project>\n"
  },
  {
    "path": "srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.srvutil;\n\nimport com.google.common.base.Strings;\nimport java.io.File;\nimport java.io.IOException;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.security.MessageDigest;\nimport java.util.HashMap;\nimport java.util.Map;\nimport org.apache.rocketmq.common.LifecycleAwareServiceThread;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class FileWatchService extends LifecycleAwareServiceThread {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);\n    private static final int DEFAULT_WATCH_INTERVAL = 500;\n\n    private final Map<String, String> currentHash = new HashMap<>();\n    private final Listener listener;\n    private final int watchInterval;\n    private final MessageDigest md = MessageDigest.getInstance(\"MD5\");\n\n    public FileWatchService(final String[] watchFiles, final Listener listener) throws Exception {\n        this(watchFiles, listener, DEFAULT_WATCH_INTERVAL);\n    }\n\n    public FileWatchService(final String[] watchFiles, final Listener listener, int watchInterval) throws Exception {\n        this.listener = listener;\n        this.watchInterval = watchInterval;\n        for (String file : watchFiles) {\n            if (!Strings.isNullOrEmpty(file) && new File(file).exists()) {\n                currentHash.put(file, md5Digest(file));\n            }\n        }\n    }\n\n    @Override\n    public String getServiceName() {\n        return \"FileWatchService\";\n    }\n\n    @Override\n    public void run0() {\n        log.info(this.getServiceName() + \" service started\");\n\n        while (!this.isStopped()) {\n            try {\n                this.waitForRunning(watchInterval);\n                for (Map.Entry<String, String> entry : currentHash.entrySet()) {\n                    String newHash = md5Digest(entry.getKey());\n                    if (!newHash.equals(entry.getValue())) {\n                        entry.setValue(newHash);\n                        listener.onChanged(entry.getKey());\n                    }\n                }\n            } catch (Exception e) {\n                log.warn(this.getServiceName() + \" service raised an unexpected exception.\", e);\n            }\n        }\n        log.info(this.getServiceName() + \" service end\");\n    }\n\n    /**\n     * Note: we ignore DELETE event on purpose. This is useful when application renew CA file.\n     * When the operator delete/rename the old CA file and copy a new one, this ensures the old CA file is used during\n     * the operation.\n     * <p>\n     * As we know exactly what to do when file does not exist or when IO exception is raised, there is no need to\n     * propagate the exception up.\n     *\n     * @param filePath Absolute path of the file to calculate its MD5 digest.\n     * @return Hash of the file content if exists; empty string otherwise.\n     */\n    private String md5Digest(String filePath) {\n        Path path = Paths.get(filePath);\n        if (!path.toFile().exists()) {\n            // Reuse previous hash result\n            return currentHash.getOrDefault(filePath, \"\");\n        }\n        byte[] raw;\n        try {\n            raw = Files.readAllBytes(path);\n        } catch (IOException e) {\n            log.info(\"Failed to read content of {}\", filePath);\n            // Reuse previous hash result\n            return currentHash.getOrDefault(filePath, \"\");\n        }\n        md.update(raw);\n        byte[] hash = md.digest();\n        return UtilAll.bytes2string(hash);\n    }\n\n    public interface Listener {\n        /**\n         * Will be called when the target files are changed\n         *\n         * @param path the changed file path\n         */\n        void onChanged(String path);\n    }\n}\n"
  },
  {
    "path": "srvutil/src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.srvutil;\n\nimport java.util.Properties;\nimport org.apache.commons.cli.CommandLine;\nimport org.apache.commons.cli.CommandLineParser;\nimport org.apache.commons.cli.HelpFormatter;\nimport org.apache.commons.cli.Option;\nimport org.apache.commons.cli.Options;\nimport org.apache.commons.cli.ParseException;\n\npublic class ServerUtil {\n\n    public static Options buildCommandlineOptions(final Options options) {\n        Option opt = new Option(\"h\", \"help\", false, \"Print help\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        opt =\n            new Option(\"n\", \"namesrvAddr\", true,\n                \"Name server address list, eg: '192.168.0.1:9876;192.168.0.2:9876'\");\n        opt.setRequired(false);\n        options.addOption(opt);\n\n        return options;\n    }\n\n    public static CommandLine parseCmdLine(final String appName, String[] args, Options options,\n        CommandLineParser parser) {\n        HelpFormatter hf = new HelpFormatter();\n        hf.setWidth(110);\n        CommandLine commandLine = null;\n        try {\n            commandLine = parser.parse(options, args);\n            if (commandLine.hasOption('h')) {\n                hf.printHelp(appName, options, true);\n                System.exit(0);\n            }\n        } catch (ParseException e) {\n            System.err.println(e.getMessage());\n            hf.printHelp(appName, options, true);\n            System.exit(1);\n        }\n\n        return commandLine;\n    }\n\n    public static void printCommandLineHelp(final String appName, final Options options) {\n        HelpFormatter hf = new HelpFormatter();\n        hf.setWidth(110);\n        hf.printHelp(appName, options, true);\n    }\n\n    public static Properties commandLine2Properties(final CommandLine commandLine) {\n        Properties properties = new Properties();\n        Option[] opts = commandLine.getOptions();\n\n        if (opts != null) {\n            for (Option opt : opts) {\n                String name = opt.getLongOpt();\n                String value = commandLine.getOptionValue(name);\n                if (value != null) {\n                    properties.setProperty(name, value);\n                }\n            }\n        }\n\n        return properties;\n    }\n\n}\n"
  },
  {
    "path": "srvutil/src/main/java/org/apache/rocketmq/srvutil/ShutdownHookThread.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.srvutil;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\n\nimport java.util.concurrent.Callable;\nimport java.util.concurrent.atomic.AtomicInteger;\n\n/**\n * {@link ShutdownHookThread} is the standard hook for filtersrv and namesrv modules.\n * Through {@link Callable} interface, this hook can customization operations in anywhere.\n */\npublic class ShutdownHookThread extends Thread {\n    private volatile boolean hasShutdown = false;\n    private AtomicInteger shutdownTimes = new AtomicInteger(0);\n    private final Logger log;\n    private final Callable callback;\n\n    /**\n     * Create the standard hook thread, with a call back, by using {@link Callable} interface.\n     *\n     * @param log The log instance is used in hook thread.\n     * @param callback The call back function.\n     */\n    public ShutdownHookThread(Logger log, Callable callback) {\n        super(\"ShutdownHook\");\n        this.log = log;\n        this.callback = callback;\n    }\n\n    /**\n     * Thread run method.\n     * Invoke when the jvm shutdown.\n     * 1. count the invocation times.\n     * 2. execute the {@link ShutdownHookThread#callback}, and time it.\n     */\n    @Override\n    public void run() {\n        synchronized (this) {\n            log.info(\"shutdown hook was invoked, \" + this.shutdownTimes.incrementAndGet() + \" times.\");\n            if (!this.hasShutdown) {\n                this.hasShutdown = true;\n                long beginTime = System.currentTimeMillis();\n                try {\n                    this.callback.call();\n                } catch (Exception e) {\n                    log.error(\"shutdown hook callback invoked failure.\", e);\n                }\n                long consumingTimeTotal = System.currentTimeMillis() - beginTime;\n                log.info(\"shutdown hook done, consuming time total(ms): \" + consumingTimeTotal);\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "srvutil/src/test/java/org/apache/rocketmq/srvutil/FileWatchServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.srvutil;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.io.PrintWriter;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.util.concurrent.Semaphore;\nimport java.util.concurrent.TimeUnit;\nimport org.junit.Rule;\nimport org.junit.Test;\nimport org.junit.rules.TemporaryFolder;\nimport org.junit.runner.RunWith;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class FileWatchServiceTest {\n    @Rule\n    public TemporaryFolder tempFolder = new TemporaryFolder();\n\n    @Test\n    public void watchSingleFile() throws Exception {\n        final File file = tempFolder.newFile();\n        final Semaphore waitSemaphore = new Semaphore(0);\n        FileWatchService fileWatchService = new FileWatchService(new String[] {file.getAbsolutePath()}, path -> {\n            assertThat(file.getAbsolutePath()).isEqualTo(path);\n            waitSemaphore.release();\n        });\n        fileWatchService.start();\n        fileWatchService.awaitStarted(1000);\n        modifyFile(file);\n        boolean result = waitSemaphore.tryAcquire(1, 1000, TimeUnit.MILLISECONDS);\n        assertThat(result).isTrue();\n        fileWatchService.shutdown();\n    }\n\n    @Test\n    public void watchSingleFile_FileDeleted() throws Exception {\n        File file = tempFolder.newFile();\n        final Semaphore waitSemaphore = new Semaphore(0);\n        FileWatchService fileWatchService = new FileWatchService(new String[] {file.getAbsolutePath()},\n            path -> waitSemaphore.release());\n        fileWatchService.start();\n        fileWatchService.awaitStarted(1000);\n        assertThat(file.delete()).isTrue();\n        boolean result = waitSemaphore.tryAcquire(1, 1000, TimeUnit.MILLISECONDS);\n        assertThat(result).isFalse();\n        assertThat(file.createNewFile()).isTrue();\n        modifyFile(file);\n        result = waitSemaphore.tryAcquire(1, 2000, TimeUnit.MILLISECONDS);\n        assertThat(result).isTrue();\n        fileWatchService.shutdown();\n    }\n\n    @Test\n    public void watchTwoFile_FileDeleted() throws Exception {\n        File fileA = tempFolder.newFile();\n        File fileB = tempFolder.newFile();\n        Files.write(fileA.toPath(), \"Hello, World!\".getBytes(StandardCharsets.UTF_8));\n        Files.write(fileB.toPath(), \"Hello, World!\".getBytes(StandardCharsets.UTF_8));\n        final Semaphore waitSemaphore = new Semaphore(0);\n        FileWatchService fileWatchService = new FileWatchService(\n            new String[] {fileA.getAbsolutePath(), fileB.getAbsolutePath()},\n            path -> waitSemaphore.release());\n        fileWatchService.start();\n        fileWatchService.awaitStarted(1000);\n        assertThat(fileA.delete()).isTrue();\n        boolean result = waitSemaphore.tryAcquire(1, 1000, TimeUnit.MILLISECONDS);\n        assertThat(result).isFalse();\n        modifyFile(fileB);\n        result = waitSemaphore.tryAcquire(1, 1000, TimeUnit.MILLISECONDS);\n        assertThat(result).isTrue();\n        assertThat(fileA.createNewFile()).isTrue();\n        modifyFile(fileA);\n        result = waitSemaphore.tryAcquire(1, 1000, TimeUnit.MILLISECONDS);\n        assertThat(result).isTrue();\n        fileWatchService.shutdown();\n    }\n\n    @Test\n    public void watchTwoFiles_ModifyOne() throws Exception {\n        final File fileA = tempFolder.newFile();\n        File fileB = tempFolder.newFile();\n        final Semaphore waitSemaphore = new Semaphore(0);\n        FileWatchService fileWatchService = new FileWatchService(\n            new String[] {fileA.getAbsolutePath(), fileB.getAbsolutePath()},\n            path -> {\n                assertThat(path).isEqualTo(fileA.getAbsolutePath());\n                waitSemaphore.release();\n            });\n        fileWatchService.start();\n        fileWatchService.awaitStarted(1000);\n        modifyFile(fileA);\n        boolean result = waitSemaphore.tryAcquire(1, 2000, TimeUnit.MILLISECONDS);\n        assertThat(result).isTrue();\n        fileWatchService.shutdown();\n    }\n\n    @Test\n    public void watchTwoFiles() throws Exception {\n        File fileA = tempFolder.newFile();\n        File fileB = tempFolder.newFile();\n        final Semaphore waitSemaphore = new Semaphore(0);\n        FileWatchService fileWatchService = new FileWatchService(\n            new String[] {fileA.getAbsolutePath(), fileB.getAbsolutePath()},\n            path -> waitSemaphore.release());\n        fileWatchService.start();\n        fileWatchService.awaitStarted(1000);\n        modifyFile(fileA);\n        modifyFile(fileB);\n        boolean result = waitSemaphore.tryAcquire(2, 1000, TimeUnit.MILLISECONDS);\n        assertThat(result).isTrue();\n    }\n\n    private static void modifyFile(File file) {\n        try {\n            PrintWriter out = new PrintWriter(file);\n            out.println(System.nanoTime());\n            out.flush();\n            out.close();\n        } catch (IOException ignore) {\n        }\n    }\n}"
  },
  {
    "path": "srvutil/src/test/resources/rmq.logback-test.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n<configuration>\n\n    <appender name=\"CONSOLE\" class=\"ch.qos.logback.core.ConsoleAppender\">\n        <layout class=\"ch.qos.logback.classic.PatternLayout\">\n            <Pattern>\n                %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n\n            </Pattern>\n        </layout>\n    </appender>\n\n    <logger name=\"org.apache.rocketmq\" level=\"error\" additivity=\"false\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </logger>\n\n    <root level=\"error\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </root>\n\n</configuration>"
  },
  {
    "path": "store/BUILD.bazel",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\nload(\"//bazel:GenTestRules.bzl\", \"GenTestRules\")\n\njava_library(\n    name = \"store\",\n    srcs = glob([\"src/main/java/**/*.java\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \"//common\",\n        \"//remoting\",\n        \"@maven//:com_alibaba_fastjson2_fastjson2\",\n        \"@maven//:com_conversantmedia_disruptor\",\n        \"@maven//:com_google_guava_guava\",\n        \"@maven//:commons_collections_commons_collections\",\n        \"@maven//:commons_io_commons_io\",\n        \"@maven//:io_netty_netty_all\",\n        \"@maven//:io_openmessaging_storage_dledger\",\n        \"@maven//:io_opentelemetry_opentelemetry_api\",\n        \"@maven//:io_opentelemetry_opentelemetry_context\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_otlp\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_prometheus\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk_common\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk_metrics\",\n        \"@maven//:net_java_dev_jna_jna\",\n        \"@maven//:org_apache_commons_commons_lang3\",\n        \"@maven//:io_github_aliyunmq_rocketmq_slf4j_api\",\n        \"@maven//:io_github_aliyunmq_rocketmq_logback_classic\",\n        \"@maven//:org_apache_rocketmq_rocketmq_rocksdb\",\n        \"@maven//:com_google_code_findbugs_jsr305\",\n        \"@maven//:commons_validator_commons_validator\",\n    ],\n)\n\njava_library(\n    name = \"tests\",\n    srcs = glob([\"src/test/java/**/*.java\"]),\n    resources = glob([\"src/test/resources/certs/*.pem\"]) + glob([\"src/test/resources/certs/*.key\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \":store\",\n        \"//:test_deps\",\n        \"//common\",\n        \"//remoting\",\n        \"@maven//:com_alibaba_fastjson2_fastjson2\",\n        \"@maven//:com_conversantmedia_disruptor\",\n        \"@maven//:io_openmessaging_storage_dledger\",\n        \"@maven//:org_apache_commons_commons_lang3\",\n        \"@maven//:com_google_guava_guava\",\n        \"@maven//:io_github_aliyunmq_rocketmq_slf4j_api\",\n        \"@maven//:io_github_aliyunmq_rocketmq_logback_classic\",\n        \"@maven//:org_apache_rocketmq_rocketmq_rocksdb\",\n        \"@maven//:org_junit_jupiter_junit_jupiter_api\",\n    ],\n)\n\nGenTestRules(\n    name = \"GeneratedTestRules\",\n    exclude_tests = [\n        # These tests are extremely slow and flaky, exclude them before they are properly fixed.\n        \"src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest\",\n        \"src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest\",\n        \"src/test/java/org/apache/rocketmq/store/dledger/DLedgerMultiPathTest\",\n    ],\n    medium_tests = [\n        \"src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest\",\n        \"src/test/java/org/apache/rocketmq/store/HATest\",\n        \"src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest\",\n        \"src/test/java/org/apache/rocketmq/store/MappedFileQueueTest\",\n        \"src/test/java/org/apache/rocketmq/store/queue/BatchConsumeMessageTest\",\n        \"src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest\",\n        \"src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest\",\n        \"src/test/java/org/apache/rocketmq/store/timer/TimerMessageStoreTest\",\n    ],\n    test_files = glob([\"src/test/java/**/*Test.java\"]),\n    deps = [\n        \":tests\",\n    ],\n)\n"
  },
  {
    "path": "store/pom.xml",
    "content": "<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n    <parent>\n        <groupId>org.apache.rocketmq</groupId>\n        <artifactId>rocketmq-all</artifactId>\n        <version>${revision}</version>\n    </parent>\n\n    <modelVersion>4.0.0</modelVersion>\n    <packaging>jar</packaging>\n    <artifactId>rocketmq-store</artifactId>\n    <name>rocketmq-store ${project.version}</name>\n\n    <properties>\n        <project.root>${basedir}/..</project.root>\n    </properties>\n\n    <dependencies>\n        <dependency>\n            <groupId>io.openmessaging.storage</groupId>\n            <artifactId>dledger</artifactId>\n            <exclusions>\n                <exclusion>\n                    <groupId>org.apache.rocketmq</groupId>\n                    <artifactId>rocketmq-remoting</artifactId>\n                </exclusion>\n            </exclusions>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-remoting</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>net.java.dev.jna</groupId>\n            <artifactId>jna</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>com.conversantmedia</groupId>\n            <artifactId>disruptor</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>com.google.guava</groupId>\n            <artifactId>guava</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>commons-io</groupId>\n            <artifactId>commons-io</artifactId>\n        </dependency>\n        <!-- Required by DLedger -->\n        <dependency>\n            <groupId>org.slf4j</groupId>\n            <artifactId>slf4j-api</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.github.aliyunmq</groupId>\n            <artifactId>rocketmq-shaded-slf4j-api-bridge</artifactId>\n        </dependency>\n    </dependencies>\n</project>\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.util.ServiceLoader;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.PriorityBlockingQueue;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.logfile.DefaultMappedFile;\nimport org.apache.rocketmq.store.logfile.MappedFile;\n\n/**\n * Create MappedFile in advance\n */\npublic class AllocateMappedFileService extends ServiceThread {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private static int waitTimeOut = 1000 * 5;\n    private ConcurrentMap<String, AllocateRequest> requestTable =\n        new ConcurrentHashMap<>();\n    private PriorityBlockingQueue<AllocateRequest> requestQueue =\n        new PriorityBlockingQueue<>();\n    private volatile boolean hasException = false;\n    private DefaultMessageStore messageStore;\n    private PreprocessHandler preprocessHandler;\n\n    public AllocateMappedFileService(DefaultMessageStore messageStore) {\n        this.messageStore = messageStore;\n    }\n\n    /**\n     * Set preprocess handler for external extension\n     *\n     * @param preprocessHandler the preprocess handler\n     */\n    public void setPreprocessHandler(PreprocessHandler preprocessHandler) {\n        this.preprocessHandler = preprocessHandler;\n    }\n\n    public MappedFile putRequestAndReturnMappedFile(String nextFilePath, String nextNextFilePath, int fileSize) {\n        // Execute preprocess logic if handler is set\n        final PreprocessHandler finalPreprocessHandler = this.preprocessHandler;\n        if (finalPreprocessHandler != null) {\n            try {\n                finalPreprocessHandler.preprocess(nextFilePath, nextNextFilePath, fileSize);\n            } catch (Throwable t) {\n                log.warn(\"Preprocess handler in AllocateMappedFileService execution failed\", t);\n            }\n        }\n        int canSubmitRequests = 2;\n        if (this.messageStore.isTransientStorePoolEnable()) {\n            if (this.messageStore.getMessageStoreConfig().isFastFailIfNoBufferInStorePool()\n                && BrokerRole.SLAVE != this.messageStore.getMessageStoreConfig().getBrokerRole()) { //if broker is slave, don't fast fail even no buffer in pool\n                canSubmitRequests = this.messageStore.remainTransientStoreBufferNumbs() - this.requestQueue.size();\n            }\n        }\n\n        AllocateRequest nextReq = new AllocateRequest(nextFilePath, fileSize);\n        boolean nextPutOK = this.requestTable.putIfAbsent(nextFilePath, nextReq) == null;\n\n        if (nextPutOK) {\n            if (canSubmitRequests <= 0) {\n                log.warn(\"[NOTIFYME]TransientStorePool is not enough, so create mapped file error, \" +\n                    \"RequestQueueSize : {}, StorePoolSize: {}\", this.requestQueue.size(), this.messageStore.remainTransientStoreBufferNumbs());\n                this.requestTable.remove(nextFilePath);\n                return null;\n            }\n            boolean offerOK = this.requestQueue.offer(nextReq);\n            if (!offerOK) {\n                log.warn(\"never expected here, add a request to preallocate queue failed\");\n            }\n            canSubmitRequests--;\n        }\n\n        AllocateRequest nextNextReq = new AllocateRequest(nextNextFilePath, fileSize);\n        boolean nextNextPutOK = this.requestTable.putIfAbsent(nextNextFilePath, nextNextReq) == null;\n        if (nextNextPutOK) {\n            if (canSubmitRequests <= 0) {\n                log.warn(\"[NOTIFYME]TransientStorePool is not enough, so skip preallocate mapped file, \" +\n                    \"RequestQueueSize : {}, StorePoolSize: {}\", this.requestQueue.size(), this.messageStore.remainTransientStoreBufferNumbs());\n                this.requestTable.remove(nextNextFilePath);\n            } else {\n                boolean offerOK = this.requestQueue.offer(nextNextReq);\n                if (!offerOK) {\n                    log.warn(\"never expected here, add a request to preallocate queue failed\");\n                }\n            }\n        }\n\n        if (hasException) {\n            log.warn(this.getServiceName() + \" service has exception. so return null\");\n            return null;\n        }\n\n        AllocateRequest result = this.requestTable.get(nextFilePath);\n        try {\n            if (result != null) {\n                messageStore.getPerfCounter().startTick(\"WAIT_MAPFILE_TIME_MS\");\n                boolean waitOK = result.getCountDownLatch().await(waitTimeOut, TimeUnit.MILLISECONDS);\n                messageStore.getPerfCounter().endTick(\"WAIT_MAPFILE_TIME_MS\");\n                if (!waitOK) {\n                    log.warn(\"create mmap timeout \" + result.getFilePath() + \" \" + result.getFileSize());\n                    return null;\n                } else {\n                    this.requestTable.remove(nextFilePath);\n                    return result.getMappedFile();\n                }\n            } else {\n                log.error(\"find preallocate mmap failed, this never happen\");\n            }\n        } catch (InterruptedException e) {\n            log.warn(this.getServiceName() + \" service has exception. \", e);\n        }\n\n        return null;\n    }\n\n    @Override\n    public String getServiceName() {\n        if (messageStore != null && messageStore.getBrokerConfig().isInBrokerContainer()) {\n            return messageStore.getBrokerIdentity().getIdentifier() + AllocateMappedFileService.class.getSimpleName();\n        }\n        return AllocateMappedFileService.class.getSimpleName();\n    }\n\n    @Override\n    public void shutdown() {\n        super.shutdown(true);\n        for (AllocateRequest req : this.requestTable.values()) {\n            if (req.mappedFile != null) {\n                log.info(\"delete pre allocated mapped file, {}\", req.mappedFile.getFileName());\n                req.mappedFile.destroy(1000);\n            }\n        }\n    }\n\n    @Override\n    public void run() {\n        log.info(this.getServiceName() + \" service started\");\n\n        while (!this.isStopped() && this.mmapOperation()) {\n\n        }\n        log.info(this.getServiceName() + \" service end\");\n    }\n\n    /**\n     * Only interrupted by the external thread, will return false\n     */\n    private boolean mmapOperation() {\n        boolean isSuccess = false;\n        AllocateRequest req = null;\n        try {\n            req = this.requestQueue.take();\n            AllocateRequest expectedRequest = this.requestTable.get(req.getFilePath());\n            if (null == expectedRequest) {\n                log.warn(\"this mmap request expired, maybe cause timeout \" + req.getFilePath() + \" \"\n                    + req.getFileSize());\n                return true;\n            }\n            if (expectedRequest != req) {\n                log.warn(\"never expected here,  maybe cause timeout \" + req.getFilePath() + \" \"\n                    + req.getFileSize() + \", req:\" + req + \", expectedRequest:\" + expectedRequest);\n                return true;\n            }\n\n            if (req.getMappedFile() == null) {\n                long beginTime = System.currentTimeMillis();\n\n                MappedFile mappedFile;\n                boolean writeWithoutMmap = messageStore.getMessageStoreConfig().isWriteWithoutMmap();\n                RunningFlags runningFlags = messageStore.getMessageStoreConfig().isEnableRunningFlagsInFlush() \n                    ? messageStore.getRunningFlags() : null;\n                if (messageStore.isTransientStorePoolEnable()) {\n                    try {\n                        mappedFile = ServiceLoader.load(MappedFile.class).iterator().next();\n                        mappedFile.init(req.getFilePath(), req.getFileSize(), runningFlags, messageStore.getTransientStorePool());\n                    } catch (RuntimeException e) {\n                        log.warn(\"Use default implementation.\");\n                        mappedFile = new DefaultMappedFile(req.getFilePath(), req.getFileSize(), runningFlags, messageStore.getTransientStorePool(), writeWithoutMmap);\n                    }\n                } else {\n                    mappedFile = new DefaultMappedFile(req.getFilePath(), req.getFileSize(), runningFlags, writeWithoutMmap);\n                }\n\n                long elapsedTime = UtilAll.computeElapsedTimeMilliseconds(beginTime);\n                if (elapsedTime > 10) {\n                    int queueSize = this.requestQueue.size();\n                    log.warn(\"create mappedFile spent time(ms) \" + elapsedTime + \" queue size \" + queueSize\n                        + \" \" + req.getFilePath() + \" \" + req.getFileSize());\n                }\n\n                // pre write mappedFile\n                if (mappedFile.getFileSize() >= this.messageStore.getMessageStoreConfig()\n                    .getMappedFileSizeCommitLog()\n                    &&\n                    this.messageStore.getMessageStoreConfig().isWarmMapedFileEnable()\n                    &&\n                    !this.messageStore.getMessageStoreConfig().isWriteWithoutMmap()) {\n                    mappedFile.warmMappedFile(this.messageStore.getMessageStoreConfig().getFlushDiskType(),\n                        this.messageStore.getMessageStoreConfig().getFlushLeastPagesWhenWarmMapedFile());\n                }\n\n                req.setMappedFile(mappedFile);\n                this.hasException = false;\n                isSuccess = true;\n            }\n        } catch (InterruptedException e) {\n            log.warn(this.getServiceName() + \" interrupted, possibly by shutdown.\");\n            this.hasException = true;\n            return false;\n        } catch (IOException e) {\n            log.warn(this.getServiceName() + \" service has exception. \", e);\n            this.hasException = true;\n            if (null != req) {\n                requestQueue.offer(req);\n                try {\n                    Thread.sleep(1);\n                } catch (InterruptedException ignored) {\n                }\n            }\n        } finally {\n            if (req != null && isSuccess)\n                req.getCountDownLatch().countDown();\n        }\n        return true;\n    }\n\n    /**\n     * Preprocess handler interface for external extension\n     */\n    @FunctionalInterface\n    public interface PreprocessHandler {\n        /**\n         * Preprocess before allocating mapped file\n         *\n         * @param nextFilePath the next file path\n         * @param nextNextFilePath the next next file path\n         * @param fileSize the file size\n         */\n        void preprocess(String nextFilePath, String nextNextFilePath, int fileSize);\n    }\n\n    static class AllocateRequest implements Comparable<AllocateRequest> {\n        // Full file path\n        private String filePath;\n        private int fileSize;\n        private CountDownLatch countDownLatch = new CountDownLatch(1);\n        private volatile MappedFile mappedFile = null;\n\n        public AllocateRequest(String filePath, int fileSize) {\n            this.filePath = filePath;\n            this.fileSize = fileSize;\n        }\n\n        public String getFilePath() {\n            return filePath;\n        }\n\n        public void setFilePath(String filePath) {\n            this.filePath = filePath;\n        }\n\n        public int getFileSize() {\n            return fileSize;\n        }\n\n        public void setFileSize(int fileSize) {\n            this.fileSize = fileSize;\n        }\n\n        public CountDownLatch getCountDownLatch() {\n            return countDownLatch;\n        }\n\n        public void setCountDownLatch(CountDownLatch countDownLatch) {\n            this.countDownLatch = countDownLatch;\n        }\n\n        public MappedFile getMappedFile() {\n            return mappedFile;\n        }\n\n        public void setMappedFile(MappedFile mappedFile) {\n            this.mappedFile = mappedFile;\n        }\n\n        public int compareTo(AllocateRequest other) {\n            if (this.fileSize < other.fileSize)\n                return 1;\n            else if (this.fileSize > other.fileSize) {\n                return -1;\n            } else {\n                int mIndex = this.filePath.lastIndexOf(File.separator);\n                long mName = Long.parseLong(this.filePath.substring(mIndex + 1));\n                int oIndex = other.filePath.lastIndexOf(File.separator);\n                long oName = Long.parseLong(other.filePath.substring(oIndex + 1));\n                if (mName < oName) {\n                    return -1;\n                } else if (mName > oName) {\n                    return 1;\n                } else {\n                    return 0;\n                }\n            }\n            // return this.fileSize < other.fileSize ? 1 : this.fileSize >\n            // other.fileSize ? -1 : 0;\n        }\n\n        @Override\n        public int hashCode() {\n            final int prime = 31;\n            int result = 1;\n            result = prime * result + ((filePath == null) ? 0 : filePath.hashCode());\n            result = prime * result + fileSize;\n            return result;\n        }\n\n        @Override\n        public boolean equals(Object obj) {\n            if (this == obj)\n                return true;\n            if (obj == null)\n                return false;\n            if (getClass() != obj.getClass())\n                return false;\n            AllocateRequest other = (AllocateRequest) obj;\n            if (filePath == null) {\n                if (other.filePath != null)\n                    return false;\n            } else if (!filePath.equals(other.filePath))\n                return false;\n            if (fileSize != other.fileSize)\n                return false;\n            return true;\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/AppendMessageCallback.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport java.nio.ByteBuffer;\nimport org.apache.rocketmq.common.message.MessageExtBatch;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\n\n/**\n * Write messages callback interface\n */\npublic interface AppendMessageCallback {\n\n    /**\n     * After message serialization, write MappedByteBuffer\n     *\n     * @return How many bytes to write\n     */\n    AppendMessageResult doAppend(final long fileFromOffset, final ByteBuffer byteBuffer,\n        final int maxBlank, final MessageExtBrokerInner msg, PutMessageContext putMessageContext);\n\n    /**\n     * After batched message serialization, write MappedByteBuffer\n     *\n     * @param messageExtBatch, backed up by a byte array\n     * @return How many bytes to write\n     */\n    AppendMessageResult doAppend(final long fileFromOffset, final ByteBuffer byteBuffer,\n        final int maxBlank, final MessageExtBatch messageExtBatch, PutMessageContext putMessageContext);\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/AppendMessageResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport java.util.function.Supplier;\n\n/**\n * When write a message to the commit log, returns results\n */\npublic class AppendMessageResult {\n    // Return code\n    private AppendMessageStatus status;\n    // Where to start writing\n    private long wroteOffset;\n    // Write Bytes\n    private int wroteBytes;\n    // Message ID\n    private String msgId;\n    private Supplier<String> msgIdSupplier;\n    // Message storage timestamp\n    private long storeTimestamp;\n    // Consume queue's offset(step by one)\n    private long logicsOffset;\n    private long pagecacheRT = 0;\n\n    private int msgNum = 1;\n\n    public AppendMessageResult(AppendMessageStatus status) {\n        this(status, 0, 0, \"\", 0, 0, 0);\n    }\n\n    public AppendMessageResult(AppendMessageStatus status, long wroteOffset, int wroteBytes, String msgId,\n        long storeTimestamp, long logicsOffset, long pagecacheRT) {\n        this.status = status;\n        this.wroteOffset = wroteOffset;\n        this.wroteBytes = wroteBytes;\n        this.msgId = msgId;\n        this.storeTimestamp = storeTimestamp;\n        this.logicsOffset = logicsOffset;\n        this.pagecacheRT = pagecacheRT;\n    }\n\n    public AppendMessageResult(AppendMessageStatus status, long wroteOffset, int wroteBytes, long storeTimestamp) {\n        this.status = status;\n        this.wroteOffset = wroteOffset;\n        this.wroteBytes = wroteBytes;\n        this.storeTimestamp = storeTimestamp;\n    }\n\n    public AppendMessageResult(AppendMessageStatus status, long wroteOffset, int wroteBytes, Supplier<String> msgIdSupplier,\n            long storeTimestamp, long logicsOffset, long pagecacheRT) {\n        this.status = status;\n        this.wroteOffset = wroteOffset;\n        this.wroteBytes = wroteBytes;\n        this.msgIdSupplier = msgIdSupplier;\n        this.storeTimestamp = storeTimestamp;\n        this.logicsOffset = logicsOffset;\n        this.pagecacheRT = pagecacheRT;\n    }\n\n    public AppendMessageResult(AppendMessageStatus status, long wroteOffset, int wroteBytes, Supplier<String> msgIdSupplier,\n            long storeTimestamp, long logicsOffset, long pagecacheRT, int msgNum) {\n        this.status = status;\n        this.wroteOffset = wroteOffset;\n        this.wroteBytes = wroteBytes;\n        this.msgIdSupplier = msgIdSupplier;\n        this.storeTimestamp = storeTimestamp;\n        this.logicsOffset = logicsOffset;\n        this.pagecacheRT = pagecacheRT;\n        this.msgNum = msgNum;\n    }\n\n    public long getPagecacheRT() {\n        return pagecacheRT;\n    }\n\n    public void setPagecacheRT(final long pagecacheRT) {\n        this.pagecacheRT = pagecacheRT;\n    }\n\n    public boolean isOk() {\n        return this.status == AppendMessageStatus.PUT_OK;\n    }\n\n    public AppendMessageStatus getStatus() {\n        return status;\n    }\n\n    public void setStatus(AppendMessageStatus status) {\n        this.status = status;\n    }\n\n    public long getWroteOffset() {\n        return wroteOffset;\n    }\n\n    public void setWroteOffset(long wroteOffset) {\n        this.wroteOffset = wroteOffset;\n    }\n\n    public int getWroteBytes() {\n        return wroteBytes;\n    }\n\n    public void setWroteBytes(int wroteBytes) {\n        this.wroteBytes = wroteBytes;\n    }\n\n    public String getMsgId() {\n        if (msgId == null && msgIdSupplier != null) {\n            msgId = msgIdSupplier.get();\n        }\n        return msgId;\n    }\n\n    public void setMsgId(String msgId) {\n        this.msgId = msgId;\n    }\n\n    public long getStoreTimestamp() {\n        return storeTimestamp;\n    }\n\n    public void setStoreTimestamp(long storeTimestamp) {\n        this.storeTimestamp = storeTimestamp;\n    }\n\n    public long getLogicsOffset() {\n        return logicsOffset;\n    }\n\n    public void setLogicsOffset(long logicsOffset) {\n        this.logicsOffset = logicsOffset;\n    }\n\n    public int getMsgNum() {\n        return msgNum;\n    }\n\n    public void setMsgNum(int msgNum) {\n        this.msgNum = msgNum;\n    }\n\n    @Override\n    public String toString() {\n        return \"AppendMessageResult{\" +\n            \"status=\" + status +\n            \", wroteOffset=\" + wroteOffset +\n            \", wroteBytes=\" + wroteBytes +\n            \", msgId='\" + msgId + '\\'' +\n            \", storeTimestamp=\" + storeTimestamp +\n            \", logicsOffset=\" + logicsOffset +\n            \", pagecacheRT=\" + pagecacheRT +\n            \", msgNum=\" + msgNum +\n            '}';\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/AppendMessageStatus.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\n/**\n * When write a message to the commit log, returns code\n */\npublic enum AppendMessageStatus {\n    PUT_OK,\n    END_OF_FILE,\n    MESSAGE_SIZE_EXCEEDED,\n    PROPERTIES_SIZE_EXCEEDED,\n    UNKNOWN_ERROR,\n    ROCKSDB_ERROR,\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/CommitLog.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport com.google.common.base.Strings;\nimport com.sun.jna.NativeLong;\nimport com.sun.jna.Pointer;\nimport java.net.Inet6Address;\nimport java.net.InetSocketAddress;\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Optional;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.TimeoutException;\nimport java.util.function.Supplier;\nimport java.util.stream.Collectors;\n\nimport io.netty.util.internal.PlatformDependent;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.SystemClock;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBatch;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.message.MessageVersion;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.common.utils.QueueTypeUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.MessageExtEncoder.PutMessageThreadLocal;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.config.FlushDiskType;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.exception.StoreException;\nimport org.apache.rocketmq.store.ha.HAService;\nimport org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService;\nimport org.apache.rocketmq.store.lock.AdaptiveBackOffSpinLockImpl;\nimport org.apache.rocketmq.store.logfile.MappedFile;\nimport org.apache.rocketmq.store.queue.ConsumeQueueInterface;\nimport org.apache.rocketmq.store.queue.CqUnit;\nimport org.apache.rocketmq.store.util.LibC;\nimport org.rocksdb.RocksDBException;\n\n/**\n * Store all metadata downtime for recovery, data protection reliability\n */\npublic class CommitLog implements Swappable {\n    // Message's MAGIC CODE daa320a7\n    public final static int MESSAGE_MAGIC_CODE = -626843481;\n    protected static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    // End of file empty MAGIC CODE cbd43194\n    public final static int BLANK_MAGIC_CODE = -875286124;\n    /**\n     * CRC32 Format: [PROPERTY_CRC32 + NAME_VALUE_SEPARATOR + 10-digit fixed-length string + PROPERTY_SEPARATOR]\n     */\n    public static final int CRC32_RESERVED_LEN = MessageConst.PROPERTY_CRC32.length() + 1 + 10 + 1;\n    protected final MappedFileQueue mappedFileQueue;\n    protected final DefaultMessageStore defaultMessageStore;\n\n    private final FlushManager flushManager;\n    private final ColdDataCheckService coldDataCheckService;\n\n    private final AppendMessageCallback appendMessageCallback;\n    private final ThreadLocal<PutMessageThreadLocal> putMessageThreadLocal;\n\n    protected volatile long confirmOffset = -1L;\n\n    private volatile long beginTimeInLock = 0;\n\n    protected final PutMessageLock putMessageLock;\n\n    protected final TopicQueueLock topicQueueLock;\n\n    private volatile Set<String> fullStorePaths = Collections.emptySet();\n\n    private final FlushDiskWatcher flushDiskWatcher;\n\n    protected int commitLogSize;\n\n    private final boolean enabledAppendPropCRC;\n\n    public CommitLog(final DefaultMessageStore messageStore) {\n        String storePath = messageStore.getMessageStoreConfig().getStorePathCommitLog();\n        RunningFlags runningFlags = messageStore.getMessageStoreConfig().isEnableRunningFlagsInFlush()\n            ? messageStore.getRunningFlags() : null;\n\n        if (storePath.contains(MixAll.MULTI_PATH_SPLITTER)) {\n            this.mappedFileQueue = new MultiPathMappedFileQueue(messageStore.getMessageStoreConfig(),\n                messageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(),\n                messageStore.getAllocateMappedFileService(), this::getFullStorePaths, runningFlags);\n        } else {\n            this.mappedFileQueue = new MappedFileQueue(storePath,\n                messageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(),\n                messageStore.getAllocateMappedFileService(),\n                runningFlags,\n                messageStore.getMessageStoreConfig().isWriteWithoutMmap());\n        }\n\n        this.defaultMessageStore = messageStore;\n\n        this.flushManager = new DefaultFlushManager();\n        this.coldDataCheckService = new ColdDataCheckService();\n\n        this.appendMessageCallback = new DefaultAppendMessageCallback(defaultMessageStore.getMessageStoreConfig());\n        putMessageThreadLocal = new ThreadLocal<PutMessageThreadLocal>() {\n            @Override\n            protected PutMessageThreadLocal initialValue() {\n                return new PutMessageThreadLocal(defaultMessageStore.getMessageStoreConfig());\n            }\n        };\n\n        PutMessageLock adaptiveBackOffSpinLock = new AdaptiveBackOffSpinLockImpl();\n\n        this.putMessageLock = messageStore.getMessageStoreConfig().getUseABSLock() ? adaptiveBackOffSpinLock :\n            messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock();\n\n        this.flushDiskWatcher = new FlushDiskWatcher();\n\n        this.topicQueueLock = new TopicQueueLock(messageStore.getMessageStoreConfig().getTopicQueueLockNum());\n\n        this.commitLogSize = messageStore.getMessageStoreConfig().getMappedFileSizeCommitLog();\n\n        this.enabledAppendPropCRC = messageStore.getMessageStoreConfig().isEnabledAppendPropCRC();\n    }\n\n    public void setFullStorePaths(Set<String> fullStorePaths) {\n        this.fullStorePaths = fullStorePaths;\n    }\n\n    public Set<String> getFullStorePaths() {\n        return fullStorePaths;\n    }\n\n    public long getTotalSize() {\n        return this.mappedFileQueue.getTotalFileSize();\n    }\n\n    public ThreadLocal<PutMessageThreadLocal> getPutMessageThreadLocal() {\n        return putMessageThreadLocal;\n    }\n\n    public boolean load() {\n        boolean result = this.mappedFileQueue.load();\n        if (result && !defaultMessageStore.getMessageStoreConfig().isDataReadAheadEnable()) {\n            scanFileAndSetReadMode(LibC.MADV_RANDOM);\n        }\n        this.mappedFileQueue.checkSelf();\n        log.info(\"load commit log {}\", result ? \"OK\" : \"Failed\");\n        return result;\n    }\n\n    public void cleanResourceAll() {\n        mappedFileQueue.cleanResourcesAll();\n    }\n\n    public void start() {\n        this.flushManager.start();\n        log.info(\"start commitLog successfully. storeRoot: {}\", this.defaultMessageStore.getMessageStoreConfig().getStorePathRootDir());\n        flushDiskWatcher.setDaemon(true);\n        flushDiskWatcher.start();\n        if (this.coldDataCheckService != null) {\n            this.coldDataCheckService.start();\n        }\n    }\n\n    public void shutdown() {\n        if (this.flushManager != null) {\n            this.flushManager.shutdown();\n        }\n        if (flushDiskWatcher != null) {\n            flushDiskWatcher.shutdown(true);\n        }\n        if (this.coldDataCheckService != null) {\n            this.coldDataCheckService.shutdown();\n        }\n        putMessageThreadLocal.remove();\n        log.info(\"shutdown commitLog successfully. storeRoot: {}\", this.defaultMessageStore.getMessageStoreConfig().getStorePathRootDir());\n    }\n\n    public long flush() {\n        this.mappedFileQueue.commit(0);\n        this.mappedFileQueue.flush(0);\n        return this.mappedFileQueue.getFlushedWhere();\n    }\n\n    public long getFlushedWhere() {\n        return this.mappedFileQueue.getFlushedWhere();\n    }\n\n    public long getMaxOffset() {\n        return this.mappedFileQueue.getMaxOffset();\n    }\n\n    public long remainHowManyDataToCommit() {\n        return this.mappedFileQueue.remainHowManyDataToCommit();\n    }\n\n    public long remainHowManyDataToFlush() {\n        return this.mappedFileQueue.remainHowManyDataToFlush();\n    }\n\n    public int deleteExpiredFile(\n        final long expiredTime,\n        final int deleteFilesInterval,\n        final long intervalForcibly,\n        final boolean cleanImmediately\n    ) {\n        return deleteExpiredFile(expiredTime, deleteFilesInterval, intervalForcibly, cleanImmediately, 0);\n    }\n\n    public int deleteExpiredFile(\n        final long expiredTime,\n        final int deleteFilesInterval,\n        final long intervalForcibly,\n        final boolean cleanImmediately,\n        final int deleteFileBatchMax\n    ) {\n        return this.mappedFileQueue.deleteExpiredFileByTime(expiredTime, deleteFilesInterval, intervalForcibly, cleanImmediately, deleteFileBatchMax);\n    }\n\n    /**\n     * Read CommitLog data, use data replication\n     */\n    public SelectMappedBufferResult getData(final long offset) {\n        return this.getData(offset, offset == 0);\n    }\n\n    public SelectMappedBufferResult getData(final long offset, final boolean returnFirstOnNotFound) {\n        int mappedFileSize = this.defaultMessageStore.getMessageStoreConfig().getMappedFileSizeCommitLog();\n        MappedFile mappedFile = this.mappedFileQueue.findMappedFileByOffset(offset, returnFirstOnNotFound);\n        if (mappedFile != null) {\n            int pos = (int) (offset % mappedFileSize);\n            SelectMappedBufferResult result = mappedFile.selectMappedBuffer(pos);\n            return result;\n        }\n\n        return null;\n    }\n\n    public boolean getData(final long offset, final int size, final ByteBuffer byteBuffer) {\n        int mappedFileSize = this.defaultMessageStore.getMessageStoreConfig().getMappedFileSizeCommitLog();\n        MappedFile mappedFile = this.mappedFileQueue.findMappedFileByOffset(offset, offset == 0);\n        if (mappedFile != null) {\n            int pos = (int) (offset % mappedFileSize);\n            return mappedFile.getData(pos, size, byteBuffer);\n        }\n        return false;\n    }\n\n    public List<SelectMappedBufferResult> getBulkData(final long offset, final int size) {\n        List<SelectMappedBufferResult> bufferResultList = new ArrayList<>();\n\n        int mappedFileSize = this.defaultMessageStore.getMessageStoreConfig().getMappedFileSizeCommitLog();\n        int remainSize = size;\n        long startOffset = offset;\n        long maxOffset = this.getMaxOffset();\n        if (offset + size > maxOffset) {\n            remainSize = (int) (maxOffset - offset);\n            log.warn(\"get bulk data size out of range, correct to max offset. offset: {}, size: {}, max: {}\", offset, remainSize, maxOffset);\n        }\n\n        while (remainSize > 0) {\n            MappedFile mappedFile = this.mappedFileQueue.findMappedFileByOffset(startOffset, startOffset == 0);\n            if (mappedFile != null) {\n                int pos = (int) (startOffset % mappedFileSize);\n                int readableSize = mappedFile.getReadPosition() - pos;\n                int readSize = Math.min(remainSize, readableSize);\n\n                SelectMappedBufferResult bufferResult = mappedFile.selectMappedBuffer(pos, readSize);\n                if (bufferResult == null) {\n                    break;\n                }\n                bufferResultList.add(bufferResult);\n                remainSize -= readSize;\n                startOffset += readSize;\n            }\n        }\n\n        return bufferResultList;\n    }\n\n    public SelectMappedFileResult getFile(final long offset) {\n        int mappedFileSize = this.defaultMessageStore.getMessageStoreConfig().getMappedFileSizeCommitLog();\n        MappedFile mappedFile = this.mappedFileQueue.findMappedFileByOffset(offset, offset == 0);\n        if (mappedFile != null) {\n            int size = (int) (mappedFile.getReadPosition() - offset % mappedFileSize);\n            if (size > 0) {\n                return new SelectMappedFileResult(size, mappedFile);\n            }\n        }\n        return null;\n    }\n\n    //Create new mappedFile if not exits.\n    public boolean getLastMappedFile(final long startOffset) {\n        MappedFile lastMappedFile = this.mappedFileQueue.getLastMappedFile(startOffset);\n        if (null == lastMappedFile) {\n            log.error(\"getLastMappedFile error. offset:{}\", startOffset);\n            return false;\n        }\n\n        return true;\n    }\n\n    /**\n     * When the normal exit, data recovery, all memory data have been flush\n     *\n     * @throws RocksDBException only in rocksdb mode\n     */\n    public void recoverNormally(long dispatchFromPhyOffset) throws RocksDBException {\n        boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover();\n        boolean checkDupInfo = this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable();\n        int maxRecoverNum = this.defaultMessageStore.getMessageStoreConfig().getCommitLogRecoverMaxNum();\n        if (maxRecoverNum <= 0) {\n            maxRecoverNum = 10;\n        }\n        log.info(\"recoverNormally maxRecoverNum: {}\", maxRecoverNum);\n        final List<MappedFile> mappedFiles = this.mappedFileQueue.getMappedFiles();\n        if (!mappedFiles.isEmpty()) {\n            int index = mappedFiles.size() - 1;\n            while (index > 0) {\n                MappedFile mappedFile = mappedFiles.get(index);\n                maxRecoverNum--;\n                if (isMappedFileMatchedRecover(mappedFile, true) || maxRecoverNum <= 0) {\n                    // It's safe to recover from this mapped file\n                    break;\n                }\n                index--;\n            }\n            // TODO: Discuss if we need to load more commit-log mapped files into memory.\n\n            MappedFile mappedFile = mappedFiles.get(index);\n            ByteBuffer byteBuffer = mappedFile.sliceByteBuffer();\n            long processOffset = mappedFile.getFileFromOffset();\n            long mappedFileOffset = 0;\n            long lastValidMsgPhyOffset = this.getConfirmOffset();\n\n            while (true) {\n                DispatchRequest dispatchRequest = this.checkMessageAndReturnSize(byteBuffer, checkCRCOnRecover, checkDupInfo);\n                int size = dispatchRequest.getMsgSize();\n                boolean doDispatch = dispatchRequest.getCommitLogOffset() > dispatchFromPhyOffset;\n                // Normal data\n                if (dispatchRequest.isSuccess() && size > 0) {\n                    lastValidMsgPhyOffset = processOffset + mappedFileOffset;\n                    mappedFileOffset += size;\n                    this.getMessageStore().onCommitLogDispatch(dispatchRequest, doDispatch, mappedFile, true, false);\n                }\n                // Come the end of the file, switch to the next file Since the\n                // return 0 representatives met last hole,\n                // this can not be included in truncate offset\n                else if (dispatchRequest.isSuccess() && size == 0) {\n                    this.getMessageStore().onCommitLogDispatch(dispatchRequest, doDispatch, mappedFile, true, true);\n                    index++;\n                    if (index >= mappedFiles.size()) {\n                        // Current branch can not happen\n                        log.info(\"recover last 3 physics file over, last mapped file {}\", mappedFile.getFileName());\n                        break;\n                    } else {\n                        mappedFile = mappedFiles.get(index);\n                        byteBuffer = mappedFile.sliceByteBuffer();\n                        processOffset = mappedFile.getFileFromOffset();\n                        mappedFileOffset = 0;\n                        log.info(\"recover next physics file, {}\", mappedFile.getFileName());\n                    }\n                }\n                // Intermediate file read error\n                else if (!dispatchRequest.isSuccess()) {\n                    if (size > 0) {\n                        log.warn(\"found a half message at {}, it will be truncated.\", processOffset + mappedFileOffset);\n                    }\n                    log.info(\"recover physics file end, {}\", mappedFile.getFileName());\n                    break;\n                }\n            }\n\n            processOffset += mappedFileOffset;\n\n            if (this.defaultMessageStore.getBrokerConfig().isEnableControllerMode()) {\n                if (this.defaultMessageStore.getConfirmOffset() < this.defaultMessageStore.getMinPhyOffset()) {\n                    log.error(\"confirmOffset {} is less than minPhyOffset {}, correct confirmOffset to minPhyOffset\", this.defaultMessageStore.getConfirmOffset(), this.defaultMessageStore.getMinPhyOffset());\n                    this.defaultMessageStore.setConfirmOffset(this.defaultMessageStore.getMinPhyOffset());\n                } else if (this.defaultMessageStore.getConfirmOffset() > processOffset) {\n                    log.error(\"confirmOffset {} is larger than processOffset {}, correct confirmOffset to processOffset\", this.defaultMessageStore.getConfirmOffset(), processOffset);\n                    this.defaultMessageStore.setConfirmOffset(processOffset);\n                }\n            } else {\n                this.setConfirmOffset(lastValidMsgPhyOffset);\n            }\n\n            // Clear ConsumeQueue redundant data\n            this.defaultMessageStore.truncateDirtyLogicFiles(processOffset);\n\n            this.mappedFileQueue.setFlushedWhere(processOffset);\n            this.mappedFileQueue.setCommittedWhere(processOffset);\n            this.mappedFileQueue.truncateDirtyFiles(processOffset);\n        } else {\n            // Commitlog case files are deleted\n            log.warn(\"The commitlog files are deleted, and delete the consume queue files\");\n            this.mappedFileQueue.setFlushedWhere(0);\n            this.mappedFileQueue.setCommittedWhere(0);\n            this.defaultMessageStore.destroyConsumeQueueStore(true);\n        }\n    }\n\n    public DispatchRequest checkMessageAndReturnSize(java.nio.ByteBuffer byteBuffer, final boolean checkCRC,\n        final boolean checkDupInfo) {\n        return this.checkMessageAndReturnSize(byteBuffer, checkCRC, checkDupInfo, true);\n    }\n\n    private void doNothingForDeadCode(final Object obj) {\n        if (obj != null) {\n            log.debug(String.valueOf(obj.hashCode()));\n        }\n    }\n\n    /**\n     * check the message and returns the message size\n     *\n     * @return 0 Come the end of the file // >0 Normal messages // -1 Message checksum failure\n     */\n    public DispatchRequest checkMessageAndReturnSize(java.nio.ByteBuffer byteBuffer, final boolean checkCRC,\n        final boolean checkDupInfo, final boolean readBody) {\n        try {\n            if (byteBuffer.remaining() <= 4) {\n                return new DispatchRequest(-1, false /* fail */);\n            }\n            // 1 TOTAL SIZE\n            int totalSize = byteBuffer.getInt();\n            if (byteBuffer.remaining() < totalSize - 4) {\n                return new DispatchRequest(-1, false /* fail */);\n            }\n\n            // 2 MAGIC CODE\n            int magicCode = byteBuffer.getInt();\n            switch (magicCode) {\n                case MessageDecoder.MESSAGE_MAGIC_CODE:\n                case MessageDecoder.MESSAGE_MAGIC_CODE_V2:\n                    break;\n                case BLANK_MAGIC_CODE:\n                    return new DispatchRequest(0, true /* success */);\n                default:\n                    log.warn(\"found a illegal magic code 0x{}\", Integer.toHexString(magicCode));\n                    return new DispatchRequest(-1, false /* success */);\n            }\n\n            MessageVersion messageVersion = MessageVersion.valueOfMagicCode(magicCode);\n\n            byte[] bytesContent = new byte[totalSize];\n\n            int bodyCRC = byteBuffer.getInt();\n\n            int queueId = byteBuffer.getInt();\n\n            int flag = byteBuffer.getInt();\n\n            long queueOffset = byteBuffer.getLong();\n\n            long physicOffset = byteBuffer.getLong();\n\n            int sysFlag = byteBuffer.getInt();\n\n            long bornTimeStamp = byteBuffer.getLong();\n\n            ByteBuffer byteBuffer1;\n            if ((sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0) {\n                byteBuffer1 = byteBuffer.get(bytesContent, 0, 4 + 4);\n            } else {\n                byteBuffer1 = byteBuffer.get(bytesContent, 0, 16 + 4);\n            }\n\n            long storeTimestamp = byteBuffer.getLong();\n\n            ByteBuffer byteBuffer2;\n            if ((sysFlag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0) {\n                byteBuffer2 = byteBuffer.get(bytesContent, 0, 4 + 4);\n            } else {\n                byteBuffer2 = byteBuffer.get(bytesContent, 0, 16 + 4);\n            }\n\n            int reconsumeTimes = byteBuffer.getInt();\n\n            long preparedTransactionOffset = byteBuffer.getLong();\n\n            int bodyLen = byteBuffer.getInt();\n            if (bodyLen > 0) {\n                if (readBody) {\n                    byteBuffer.get(bytesContent, 0, bodyLen);\n\n                    if (checkCRC) {\n                        /**\n                         * When the forceVerifyPropCRC = false,\n                         * use original bodyCrc validation.\n                         */\n                        if (!this.defaultMessageStore.getMessageStoreConfig().isForceVerifyPropCRC()) {\n                            int crc = UtilAll.crc32(bytesContent, 0, bodyLen);\n                            if (crc != bodyCRC) {\n                                log.warn(\"CRC check failed. bodyCRC={}, currentCRC={}\", crc, bodyCRC);\n                                return new DispatchRequest(-1, false/* success */);\n                            }\n                        }\n                    }\n                } else {\n                    byteBuffer.position(byteBuffer.position() + bodyLen);\n                }\n            }\n\n            int topicLen = messageVersion.getTopicLength(byteBuffer);\n            byteBuffer.get(bytesContent, 0, topicLen);\n            String topic = new String(bytesContent, 0, topicLen, MessageDecoder.CHARSET_UTF8);\n\n            long tagsCode = 0;\n            String keys = \"\";\n            String uniqKey = null;\n\n            short propertiesLength = byteBuffer.getShort();\n            Map<String, String> propertiesMap = null;\n            if (propertiesLength > 0) {\n                byteBuffer.get(bytesContent, 0, propertiesLength);\n                String properties = new String(bytesContent, 0, propertiesLength, MessageDecoder.CHARSET_UTF8);\n                propertiesMap = MessageDecoder.string2messageProperties(properties);\n\n                keys = propertiesMap.get(MessageConst.PROPERTY_KEYS);\n\n                uniqKey = propertiesMap.get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX);\n\n                if (checkDupInfo) {\n                    String dupInfo = propertiesMap.get(MessageConst.DUP_INFO);\n                    if (null == dupInfo || dupInfo.split(\"_\").length != 2) {\n                        log.warn(\"DupInfo in properties check failed. dupInfo={}\", dupInfo);\n                        return new DispatchRequest(-1, false);\n                    }\n                }\n\n                String tags = propertiesMap.get(MessageConst.PROPERTY_TAGS);\n                if (!Strings.isNullOrEmpty(tags)) {\n                    tagsCode = MessageExtBrokerInner.tagsString2tagsCode(MessageExt.parseTopicFilterType(sysFlag), tags);\n                }\n\n                // Timing message processing\n                {\n                    String t = propertiesMap.get(MessageConst.PROPERTY_DELAY_TIME_LEVEL);\n                    if (TopicValidator.RMQ_SYS_SCHEDULE_TOPIC.equals(topic) && t != null) {\n                        int delayLevel = Integer.parseInt(t);\n\n                        if (delayLevel > this.defaultMessageStore.getMaxDelayLevel()) {\n                            delayLevel = this.defaultMessageStore.getMaxDelayLevel();\n                        }\n\n                        if (delayLevel > 0) {\n                            tagsCode = this.defaultMessageStore.computeDeliverTimestamp(delayLevel,\n                                storeTimestamp);\n                        }\n                    }\n                }\n            }\n\n            if (checkCRC) {\n                /**\n                 * When the forceVerifyPropCRC = true,\n                 * Crc verification needs to be performed on the entire message data (excluding the length reserved at the tail)\n                 */\n                if (this.defaultMessageStore.getMessageStoreConfig().isForceVerifyPropCRC()) {\n                    int expectedCRC = -1;\n                    if (propertiesMap != null) {\n                        String crc32Str = propertiesMap.get(MessageConst.PROPERTY_CRC32);\n                        if (crc32Str != null) {\n                            expectedCRC = 0;\n                            for (int i = crc32Str.length() - 1; i >= 0; i--) {\n                                int num = crc32Str.charAt(i) - '0';\n                                expectedCRC *= 10;\n                                expectedCRC += num;\n                            }\n                        }\n                    }\n                    if (expectedCRC >= 0) {\n                        ByteBuffer tmpBuffer = byteBuffer.duplicate();\n                        tmpBuffer.position(tmpBuffer.position() - totalSize);\n                        tmpBuffer.limit(tmpBuffer.position() + totalSize - CommitLog.CRC32_RESERVED_LEN);\n                        int crc = UtilAll.crc32(tmpBuffer);\n                        if (crc != expectedCRC) {\n                            log.warn(\n                                \"CommitLog#checkAndDispatchMessage: failed to check message CRC, expected \"\n                                    + \"CRC={}, actual CRC={}\", bodyCRC, crc);\n                            return new DispatchRequest(-1, false/* success */);\n                        }\n                    } else {\n                        // Read full message for logging when error occurs\n                        ByteBuffer fullMessageBuffer = byteBuffer.duplicate();\n                        int messageStartPos = fullMessageBuffer.position() - totalSize;\n                        fullMessageBuffer.position(messageStartPos);\n                        fullMessageBuffer.limit(messageStartPos + totalSize);\n                        byte[] fullMessageBytes = new byte[totalSize];\n                        fullMessageBuffer.get(fullMessageBytes, 0, totalSize);\n                        \n                        // Print full message and especially properties\n                        log.warn(\n                            \"CommitLog#checkAndDispatchMessage: failed to check message CRC, not found CRC in properties. topic={}, properties={}, propertiesLength={}, fullMessageHex={}\",\n                            topic, propertiesMap != null ? propertiesMap.toString() : \"null\", propertiesLength, UtilAll.bytes2string(fullMessageBytes));\n                        return new DispatchRequest(-1, false/* success */);\n                    }\n                }\n            }\n\n            int readLength = MessageExtEncoder.calMsgLength(messageVersion, sysFlag, bodyLen, topicLen, propertiesLength);\n            if (totalSize != readLength) {\n                doNothingForDeadCode(reconsumeTimes);\n                doNothingForDeadCode(flag);\n                doNothingForDeadCode(bornTimeStamp);\n                doNothingForDeadCode(byteBuffer1);\n                doNothingForDeadCode(byteBuffer2);\n                log.error(\n                    \"[BUG]read total count not equals msg total size. totalSize={}, readTotalCount={}, bodyLen={}, topicLen={}, propertiesLength={}\",\n                    totalSize, readLength, bodyLen, topicLen, propertiesLength);\n                return new DispatchRequest(totalSize, false/* success */);\n            }\n\n            DispatchRequest dispatchRequest = new DispatchRequest(\n                topic,\n                queueId,\n                physicOffset,\n                totalSize,\n                tagsCode,\n                storeTimestamp,\n                queueOffset,\n                keys,\n                uniqKey,\n                sysFlag,\n                preparedTransactionOffset,\n                propertiesMap\n            );\n\n            setBatchSizeIfNeeded(propertiesMap, dispatchRequest);\n\n            return dispatchRequest;\n        } catch (Exception e) {\n            log.error(\"checkMessageAndReturnSize failed, may can not dispatch\", e);\n        }\n\n        return new DispatchRequest(-1, false /* success */);\n    }\n\n    private void setBatchSizeIfNeeded(Map<String, String> propertiesMap, DispatchRequest dispatchRequest) {\n        if (null != propertiesMap && propertiesMap.containsKey(MessageConst.PROPERTY_INNER_NUM) && propertiesMap.containsKey(MessageConst.PROPERTY_INNER_BASE)) {\n            dispatchRequest.setMsgBaseOffset(Long.parseLong(propertiesMap.get(MessageConst.PROPERTY_INNER_BASE)));\n            dispatchRequest.setBatchSize(Short.parseShort(propertiesMap.get(MessageConst.PROPERTY_INNER_NUM)));\n        }\n    }\n\n    // Fetch and compute the newest confirmOffset.\n    // Even if it is just inited.\n    public long getConfirmOffset() {\n        if (this.defaultMessageStore.getBrokerConfig().isEnableControllerMode()) {\n            if (this.defaultMessageStore.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE && !this.defaultMessageStore.getRunningFlags().isFenced()) {\n                if (((AutoSwitchHAService) this.defaultMessageStore.getHaService()).getLocalSyncStateSet().size() == 1\n                    || !this.defaultMessageStore.getMessageStoreConfig().isAllAckInSyncStateSet()) {\n                    return this.defaultMessageStore.getMaxPhyOffset();\n                }\n                // First time it will compute the confirmOffset.\n                if (this.confirmOffset < 0) {\n                    setConfirmOffset(((AutoSwitchHAService) this.defaultMessageStore.getHaService()).computeConfirmOffset());\n                    log.info(\"Init the confirmOffset to {}.\", this.confirmOffset);\n                }\n            }\n            return this.confirmOffset;\n        } else if (this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable()) {\n            return this.confirmOffset;\n        } else {\n            return this.defaultMessageStore.isSyncDiskFlush() ? getFlushedWhere() : getMaxOffset();\n        }\n    }\n\n    // Fetch the original confirmOffset's value.\n    // Without checking and re-computing.\n    public long getConfirmOffsetDirectly() {\n        if (this.defaultMessageStore.getBrokerConfig().isEnableControllerMode()) {\n            if (this.defaultMessageStore.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE && !this.defaultMessageStore.getRunningFlags().isFenced()) {\n                if (((AutoSwitchHAService) this.defaultMessageStore.getHaService()).getLocalSyncStateSet().size() == 1) {\n                    return this.defaultMessageStore.getMaxPhyOffset();\n                }\n            }\n            return this.confirmOffset;\n        } else if (this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable()) {\n            return this.confirmOffset;\n        } else {\n            return getMaxOffset();\n        }\n    }\n\n    public void setConfirmOffset(long phyOffset) {\n        this.confirmOffset = phyOffset;\n        this.defaultMessageStore.getStoreCheckpoint().setConfirmPhyOffset(confirmOffset);\n    }\n\n    public long getLastFileFromOffset() {\n        MappedFile lastMappedFile = this.mappedFileQueue.getLastMappedFile();\n        if (lastMappedFile != null) {\n            if (lastMappedFile.isAvailable()) {\n                return lastMappedFile.getFileFromOffset();\n            }\n        }\n\n        return -1;\n    }\n\n    /**\n     * @throws RocksDBException only in rocksdb mode\n     */\n    public void recoverAbnormally(long dispatchFromPhyOffset) throws RocksDBException {\n        // recover by the minimum time stamp\n        boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover();\n        boolean checkDupInfo = this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable();\n        boolean checkCommitLogOffsetOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCommitLogOffsetOnRecover();\n        int maxRecoverNum = this.defaultMessageStore.getMessageStoreConfig().getCommitLogRecoverMaxNum();\n        if (maxRecoverNum <= 0) {\n            maxRecoverNum = 10;\n        }\n        log.info(\"recoverAbnormally maxRecoverNum: {}\", maxRecoverNum);\n        final List<MappedFile> mappedFiles = this.mappedFileQueue.getMappedFiles();\n        if (!mappedFiles.isEmpty()) {\n            // Looking beginning to recover from which file\n            int index = mappedFiles.size() - 1;\n            MappedFile mappedFile = null;\n            for (; index >= 0; index--) {\n                mappedFile = mappedFiles.get(index);\n                maxRecoverNum--;\n                if (this.isMappedFileMatchedRecover(mappedFile, false) || maxRecoverNum <= 0) {\n                    log.info(\"recover from this mapped file \" + mappedFile.getFileName());\n                    break;\n                }\n            }\n\n            if (index < 0) {\n                index = 0;\n                mappedFile = mappedFiles.get(index);\n            }\n\n            ByteBuffer byteBuffer = mappedFile.sliceByteBuffer();\n            long processOffset = mappedFile.getFileFromOffset();\n            long mappedFileOffset;\n            long lastValidMsgPhyOffset;\n            long lastConfirmValidMsgPhyOffset;\n\n            if (defaultMessageStore.getMessageStoreConfig().isEnableAcceleratedRecovery()) {\n                mappedFileOffset = dispatchFromPhyOffset - mappedFile.getFileFromOffset();\n                // Protective measures, falling back to non-accelerated mode, which is extremely unlikely to occur\n                if (mappedFileOffset < 0) {\n                    mappedFileOffset = 0;\n                    lastValidMsgPhyOffset = processOffset;\n                    lastConfirmValidMsgPhyOffset = processOffset;\n                } else {\n                    log.info(\"recover using acceleration, recovery offset is {}\", dispatchFromPhyOffset);\n                    lastValidMsgPhyOffset = dispatchFromPhyOffset;\n                    lastConfirmValidMsgPhyOffset = dispatchFromPhyOffset;\n                    byteBuffer.position((int) mappedFileOffset);\n                }\n            } else {\n                mappedFileOffset = 0;\n                lastValidMsgPhyOffset = processOffset;\n                lastConfirmValidMsgPhyOffset = processOffset;\n            }\n            // abnormal recover require dispatching\n            boolean doDispatch = true;\n            while (true) {\n                DispatchRequest dispatchRequest = this.checkMessageAndReturnSize(byteBuffer, checkCRCOnRecover, checkDupInfo);\n                int size = dispatchRequest.getMsgSize();\n                if (dispatchRequest.isSuccess()) {\n                    // Check commitlog offset validity if enabled\n                    if (size > 0 && checkCommitLogOffsetOnRecover) {\n                        if (dispatchRequest.getCommitLogOffset() < mappedFile.getFileFromOffset()\n                            || dispatchRequest.getCommitLogOffset() > mappedFile.getFileFromOffset() + mappedFile.getFileSize()) {\n                            log.warn(\"found illegal commitlog offset {} in {}, current pos={}, it will be truncated.\",\n                                dispatchRequest.getCommitLogOffset(), mappedFile.getFileName(), processOffset + mappedFileOffset);\n                            log.info(\"recover physics file end, {} pos={}\", mappedFile.getFileName(), byteBuffer.position());\n\n                            break;\n                        }\n                    }\n                    // Normal data\n                    if (size > 0) {\n                        lastValidMsgPhyOffset = processOffset + mappedFileOffset;\n                        mappedFileOffset += size;\n\n                        if (this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable() || this.defaultMessageStore.getBrokerConfig().isEnableControllerMode()) {\n                            if (dispatchRequest.getCommitLogOffset() + size <= this.defaultMessageStore.getCommitLog().getConfirmOffset()) {\n                                this.getMessageStore().onCommitLogDispatch(dispatchRequest, doDispatch, mappedFile, true, false);\n                                lastConfirmValidMsgPhyOffset = dispatchRequest.getCommitLogOffset() + size;\n                            }\n                        } else {\n                            this.getMessageStore().onCommitLogDispatch(dispatchRequest, doDispatch, mappedFile, true, false);\n                        }\n                    }\n                    // Come the end of the file, switch to the next file\n                    // Since the return 0 representatives met last hole, this can\n                    // not be included in truncate offset\n                    else if (size == 0) {\n                        this.getMessageStore().onCommitLogDispatch(dispatchRequest, doDispatch, mappedFile, true, true);\n                        index++;\n                        if (index >= mappedFiles.size()) {\n                            // The current branch under normal circumstances should\n                            // not happen\n                            log.info(\"recover physics file over, last mapped file {}\", mappedFile.getFileName());\n                            break;\n                        } else {\n                            mappedFile = mappedFiles.get(index);\n                            byteBuffer = mappedFile.sliceByteBuffer();\n                            processOffset = mappedFile.getFileFromOffset();\n                            mappedFileOffset = 0;\n                            log.info(\"recover next physics file, {}\", mappedFile.getFileName());\n                        }\n                    }\n                } else {\n\n                    if (size > 0) {\n                        log.warn(\"found a half message at {}, it will be truncated.\", processOffset + mappedFileOffset);\n                    }\n\n                    log.info(\"recover physics file end, {} pos={}\", mappedFile.getFileName(), byteBuffer.position());\n                    break;\n                }\n            }\n\n            try {\n                this.getMessageStore().getQueueStore().flush();\n            } catch (StoreException e) {\n                log.error(\"Failed to flush ConsumeQueueStore\", e);\n            }\n\n            processOffset += mappedFileOffset;\n            if (this.defaultMessageStore.getBrokerConfig().isEnableControllerMode()) {\n                if (this.defaultMessageStore.getConfirmOffset() < this.defaultMessageStore.getMinPhyOffset()) {\n                    log.error(\"confirmOffset {} is less than minPhyOffset {}, correct confirmOffset to minPhyOffset\", this.defaultMessageStore.getConfirmOffset(), this.defaultMessageStore.getMinPhyOffset());\n                    this.defaultMessageStore.setConfirmOffset(this.defaultMessageStore.getMinPhyOffset());\n                } else if (this.defaultMessageStore.getConfirmOffset() > lastConfirmValidMsgPhyOffset) {\n                    log.error(\"confirmOffset {} is larger than lastConfirmValidMsgPhyOffset {}, correct confirmOffset to lastConfirmValidMsgPhyOffset\", this.defaultMessageStore.getConfirmOffset(), lastConfirmValidMsgPhyOffset);\n                    this.defaultMessageStore.setConfirmOffset(lastConfirmValidMsgPhyOffset);\n                }\n            } else {\n                this.setConfirmOffset(lastValidMsgPhyOffset);\n            }\n\n            // Clear ConsumeQueue redundant data\n            this.defaultMessageStore.truncateDirtyLogicFiles(processOffset);\n\n            this.mappedFileQueue.setFlushedWhere(processOffset);\n            this.mappedFileQueue.setCommittedWhere(processOffset);\n            this.mappedFileQueue.truncateDirtyFiles(processOffset);\n        }\n        // Commitlog case files are deleted\n        else {\n            log.warn(\"The commitlog files are deleted, and delete the consume queue files\");\n            this.mappedFileQueue.setFlushedWhere(0);\n            this.mappedFileQueue.setCommittedWhere(0);\n            this.defaultMessageStore.destroyConsumeQueueStore(true);\n        }\n    }\n\n    public void truncateDirtyFiles(long phyOffset) {\n        if (phyOffset <= this.getFlushedWhere()) {\n            this.mappedFileQueue.setFlushedWhere(phyOffset);\n        }\n\n        if (phyOffset <= this.mappedFileQueue.getCommittedWhere()) {\n            this.mappedFileQueue.setCommittedWhere(phyOffset);\n        }\n\n        this.mappedFileQueue.truncateDirtyFiles(phyOffset);\n        if (this.confirmOffset > phyOffset) {\n            this.setConfirmOffset(phyOffset);\n        }\n    }\n\n    protected void onCommitLogAppend(MessageExtBrokerInner msg, AppendMessageResult result, MappedFile commitLogFile) {\n        this.getMessageStore().onCommitLogAppend(msg, result, commitLogFile);\n    }\n\n    private boolean isMappedFileMatchedRecover(final MappedFile mappedFile,\n        boolean recoverNormally) throws RocksDBException {\n        ByteBuffer byteBuffer = mappedFile.sliceByteBuffer();\n\n        boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover();\n        boolean checkDupInfo = this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable();\n\n        DispatchRequest dispatchRequest = this.checkMessageAndReturnSize(byteBuffer, checkCRCOnRecover, checkDupInfo);\n\n        if (!dispatchRequest.isSuccess()) {\n            return false;\n        }\n\n        long storeTimestamp = dispatchRequest.getStoreTimestamp();\n        long phyOffset = dispatchRequest.getCommitLogOffset();\n\n        if (0 == dispatchRequest.getStoreTimestamp()) {\n            return false;\n        }\n\n        return isMappedFileMatchedRecover(phyOffset, storeTimestamp, recoverNormally);\n    }\n\n    private boolean isMappedFileMatchedRecover(long phyOffset, long storeTimestamp,\n        boolean recoverNormally) throws RocksDBException {\n        boolean result = this.defaultMessageStore.getQueueStore().isMappedFileMatchedRecover(phyOffset, storeTimestamp, recoverNormally);\n        // Check all registered CommitLogDispatchStore instances\n        for (CommitLogDispatchStore store : defaultMessageStore.getCommitLogDispatchStores()) {\n            result = result && store.isMappedFileMatchedRecover(phyOffset, storeTimestamp, recoverNormally);\n        }\n        return result;\n    }\n\n    public boolean resetOffset(long offset) {\n        return this.mappedFileQueue.resetOffset(offset);\n    }\n\n    public long getBeginTimeInLock() {\n        return beginTimeInLock;\n    }\n\n    public String generateKey(StringBuilder keyBuilder, MessageExt messageExt) {\n        keyBuilder.setLength(0);\n        keyBuilder.append(messageExt.getTopic());\n        keyBuilder.append('-');\n        keyBuilder.append(messageExt.getQueueId());\n        return keyBuilder.toString();\n    }\n\n    public void setMappedFileQueueOffset(final long phyOffset) {\n        this.mappedFileQueue.setFlushedWhere(phyOffset);\n        this.mappedFileQueue.setCommittedWhere(phyOffset);\n    }\n\n    public void updateMaxMessageSize(PutMessageThreadLocal putMessageThreadLocal) {\n        // dynamically adjust maxMessageSize, but not support DLedger mode temporarily\n        int newMaxMessageSize = this.defaultMessageStore.getMessageStoreConfig().getMaxMessageSize();\n        if (newMaxMessageSize >= 10 &&\n            putMessageThreadLocal.getEncoder().getMaxMessageBodySize() != newMaxMessageSize) {\n            putMessageThreadLocal.getEncoder().updateEncoderBufferCapacity(newMaxMessageSize);\n        }\n    }\n\n    public CompletableFuture<PutMessageResult> asyncPutMessage(final MessageExtBrokerInner msg) {\n        // Set the storage time\n        if (!defaultMessageStore.getMessageStoreConfig().isDuplicationEnable()) {\n            msg.setStoreTimestamp(System.currentTimeMillis());\n        }\n        // Set the message body CRC (consider the most appropriate setting on the client)\n        msg.setBodyCRC(UtilAll.crc32(msg.getBody()));\n        if (enabledAppendPropCRC) {\n            // delete crc32 properties if exist\n            msg.deleteProperty(MessageConst.PROPERTY_CRC32);\n        }\n        // Back to Results\n        AppendMessageResult result = null;\n\n        StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService();\n\n        String topic = msg.getTopic();\n        msg.setVersion(MessageVersion.MESSAGE_VERSION_V1);\n        boolean autoMessageVersionOnTopicLen =\n            this.defaultMessageStore.getMessageStoreConfig().isAutoMessageVersionOnTopicLen();\n        if (autoMessageVersionOnTopicLen && topic.length() > Byte.MAX_VALUE) {\n            msg.setVersion(MessageVersion.MESSAGE_VERSION_V2);\n        }\n\n        InetSocketAddress bornSocketAddress = (InetSocketAddress) msg.getBornHost();\n        if (bornSocketAddress.getAddress() instanceof Inet6Address) {\n            msg.setBornHostV6Flag();\n        }\n\n        InetSocketAddress storeSocketAddress = (InetSocketAddress) msg.getStoreHost();\n        if (storeSocketAddress.getAddress() instanceof Inet6Address) {\n            msg.setStoreHostAddressV6Flag();\n        }\n\n        PutMessageThreadLocal putMessageThreadLocal = this.putMessageThreadLocal.get();\n        updateMaxMessageSize(putMessageThreadLocal);\n        String topicQueueKey = generateKey(putMessageThreadLocal.getKeyBuilder(), msg);\n        long elapsedTimeInLock = 0;\n        MappedFile unlockMappedFile = null;\n        MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();\n\n        long currOffset;\n        if (mappedFile == null) {\n            currOffset = 0;\n        } else {\n            currOffset = mappedFile.getFileFromOffset() + mappedFile.getWrotePosition();\n        }\n\n        int needAckNums = this.defaultMessageStore.getMessageStoreConfig().getInSyncReplicas();\n        boolean needHandleHA = needHandleHA(msg);\n\n        if (needHandleHA && this.defaultMessageStore.getBrokerConfig().isEnableControllerMode()) {\n            if (this.defaultMessageStore.getHaService().inSyncReplicasNums(currOffset) < this.defaultMessageStore.getMessageStoreConfig().getMinInSyncReplicas()) {\n                return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH, null));\n            }\n            if (this.defaultMessageStore.getMessageStoreConfig().isAllAckInSyncStateSet()) {\n                // -1 means all ack in SyncStateSet\n                needAckNums = MixAll.ALL_ACK_IN_SYNC_STATE_SET;\n            }\n        } else if (needHandleHA && this.defaultMessageStore.getBrokerConfig().isEnableSlaveActingMaster()) {\n            int inSyncReplicas = Math.min(this.defaultMessageStore.getAliveReplicaNumInGroup(),\n                this.defaultMessageStore.getHaService().inSyncReplicasNums(currOffset));\n            needAckNums = calcNeedAckNums(inSyncReplicas);\n            if (needAckNums > inSyncReplicas) {\n                // Tell the producer, don't have enough slaves to handle the send request\n                return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH, null));\n            }\n        }\n\n        topicQueueLock.lock(topicQueueKey);\n        try {\n\n            boolean needAssignOffset = true;\n            if (defaultMessageStore.getMessageStoreConfig().isDuplicationEnable()\n                && defaultMessageStore.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE) {\n                needAssignOffset = false;\n            }\n            if (needAssignOffset) {\n                defaultMessageStore.assignOffset(msg);\n            }\n\n            PutMessageResult encodeResult = putMessageThreadLocal.getEncoder().encode(msg);\n            if (encodeResult != null) {\n                return CompletableFuture.completedFuture(encodeResult);\n            }\n            msg.setEncodedBuff(putMessageThreadLocal.getEncoder().getEncoderBuffer());\n            PutMessageContext putMessageContext = new PutMessageContext(topicQueueKey);\n\n            putMessageLock.lock(); //spin or ReentrantLock, depending on store config\n            try {\n                long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now();\n                this.beginTimeInLock = beginLockTimestamp;\n\n                // Here settings are stored timestamp, in order to ensure an orderly\n                // global\n                if (!defaultMessageStore.getMessageStoreConfig().isDuplicationEnable()) {\n                    msg.setStoreTimestamp(beginLockTimestamp);\n                }\n\n                if (null == mappedFile || mappedFile.isFull()) {\n                    mappedFile = this.mappedFileQueue.getLastMappedFile(0); // Mark: NewFile may be cause noise\n                    if (isCloseReadAhead()) {\n                        setFileReadMode(mappedFile, LibC.MADV_RANDOM);\n                    }\n                }\n                if (null == mappedFile) {\n                    log.error(\"create mapped file1 error, topic: {} clientAddr: {}\", msg.getTopic(), msg.getBornHostString());\n                    return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, null));\n                }\n\n                result = mappedFile.appendMessage(msg, this.appendMessageCallback, putMessageContext);\n                switch (result.getStatus()) {\n                    case PUT_OK:\n                        onCommitLogAppend(msg, result, mappedFile);\n                        break;\n                    case END_OF_FILE:\n                        onCommitLogAppend(msg, result, mappedFile);\n                        unlockMappedFile = mappedFile;\n                        // Create a new file, re-write the message\n                        mappedFile = this.mappedFileQueue.getLastMappedFile(0);\n                        if (null == mappedFile) {\n                            // XXX: warn and notify me\n                            log.error(\"create mapped file2 error, topic: {} clientAddr: {}\", msg.getTopic(), msg.getBornHostString());\n                            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, result));\n                        }\n                        if (isCloseReadAhead()) {\n                            setFileReadMode(mappedFile, LibC.MADV_RANDOM);\n                        }\n                        result = mappedFile.appendMessage(msg, this.appendMessageCallback, putMessageContext);\n                        if (AppendMessageStatus.PUT_OK.equals(result.getStatus())) {\n                            onCommitLogAppend(msg, result, mappedFile);\n                        }\n                        break;\n                    case MESSAGE_SIZE_EXCEEDED:\n                    case PROPERTIES_SIZE_EXCEEDED:\n                        return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result));\n                    case UNKNOWN_ERROR:\n                    default:\n                        return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result));\n                }\n\n                elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginLockTimestamp;\n            } finally {\n                beginTimeInLock = 0;\n                putMessageLock.unlock();\n            }\n            // Increase queue offset when messages are successfully written\n            if (AppendMessageStatus.PUT_OK.equals(result.getStatus())) {\n                this.defaultMessageStore.increaseOffset(msg, getMessageNum(msg));\n            }\n        } catch (RocksDBException e) {\n            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result));\n        } finally {\n            topicQueueLock.unlock(topicQueueKey);\n        }\n\n        if (elapsedTimeInLock > 500) {\n            log.warn(\"[NOTIFYME]putMessage in lock cost time(ms)={}, bodyLength={} AppendMessageResult={}\", elapsedTimeInLock, msg.getBody().length, result);\n        }\n\n        if (null != unlockMappedFile && this.defaultMessageStore.getMessageStoreConfig().isWarmMapedFileEnable()) {\n            this.defaultMessageStore.unlockMappedFile(unlockMappedFile);\n        }\n\n        PutMessageResult putMessageResult = new PutMessageResult(PutMessageStatus.PUT_OK, result);\n\n        // Statistics\n        storeStatsService.getSinglePutMessageTopicTimesTotal(msg.getTopic()).add(result.getMsgNum());\n        storeStatsService.getSinglePutMessageTopicSizeTotal(topic).add(result.getWroteBytes());\n\n        return handleDiskFlushAndHA(putMessageResult, msg, needAckNums, needHandleHA);\n    }\n\n    public CompletableFuture<PutMessageResult> asyncPutMessages(final MessageExtBatch messageExtBatch) {\n        messageExtBatch.setStoreTimestamp(System.currentTimeMillis());\n        AppendMessageResult result = null;\n\n        StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService();\n\n        final int tranType = MessageSysFlag.getTransactionValue(messageExtBatch.getSysFlag());\n\n        if (tranType != MessageSysFlag.TRANSACTION_NOT_TYPE) {\n            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null));\n        }\n        if (messageExtBatch.getDelayTimeLevel() > 0) {\n            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null));\n        }\n\n        InetSocketAddress bornSocketAddress = (InetSocketAddress) messageExtBatch.getBornHost();\n        if (bornSocketAddress.getAddress() instanceof Inet6Address) {\n            messageExtBatch.setBornHostV6Flag();\n        }\n\n        InetSocketAddress storeSocketAddress = (InetSocketAddress) messageExtBatch.getStoreHost();\n        if (storeSocketAddress.getAddress() instanceof Inet6Address) {\n            messageExtBatch.setStoreHostAddressV6Flag();\n        }\n\n        long elapsedTimeInLock = 0;\n        MappedFile unlockMappedFile = null;\n        MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();\n\n        long currOffset;\n        if (mappedFile == null) {\n            currOffset = 0;\n        } else {\n            currOffset = mappedFile.getFileFromOffset() + mappedFile.getWrotePosition();\n        }\n\n        int needAckNums = this.defaultMessageStore.getMessageStoreConfig().getInSyncReplicas();\n        boolean needHandleHA = needHandleHA(messageExtBatch);\n\n        if (needHandleHA && this.defaultMessageStore.getBrokerConfig().isEnableControllerMode()) {\n            if (this.defaultMessageStore.getHaService().inSyncReplicasNums(currOffset) < this.defaultMessageStore.getMessageStoreConfig().getMinInSyncReplicas()) {\n                return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH, null));\n            }\n            if (this.defaultMessageStore.getMessageStoreConfig().isAllAckInSyncStateSet()) {\n                // -1 means all ack in SyncStateSet\n                needAckNums = MixAll.ALL_ACK_IN_SYNC_STATE_SET;\n            }\n        } else if (needHandleHA && this.defaultMessageStore.getBrokerConfig().isEnableSlaveActingMaster()) {\n            int inSyncReplicas = Math.min(this.defaultMessageStore.getAliveReplicaNumInGroup(),\n                this.defaultMessageStore.getHaService().inSyncReplicasNums(currOffset));\n            needAckNums = calcNeedAckNums(inSyncReplicas);\n            if (needAckNums > inSyncReplicas) {\n                // Tell the producer, don't have enough slaves to handle the send request\n                return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH, null));\n            }\n        }\n\n        messageExtBatch.setVersion(MessageVersion.MESSAGE_VERSION_V1);\n        boolean autoMessageVersionOnTopicLen =\n            this.defaultMessageStore.getMessageStoreConfig().isAutoMessageVersionOnTopicLen();\n        if (autoMessageVersionOnTopicLen && messageExtBatch.getTopic().length() > Byte.MAX_VALUE) {\n            messageExtBatch.setVersion(MessageVersion.MESSAGE_VERSION_V2);\n        }\n\n        //fine-grained lock instead of the coarse-grained\n        PutMessageThreadLocal pmThreadLocal = this.putMessageThreadLocal.get();\n        updateMaxMessageSize(pmThreadLocal);\n        MessageExtEncoder batchEncoder = pmThreadLocal.getEncoder();\n\n        String topicQueueKey = generateKey(pmThreadLocal.getKeyBuilder(), messageExtBatch);\n\n        PutMessageContext putMessageContext = new PutMessageContext(topicQueueKey);\n        messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch, putMessageContext));\n\n        topicQueueLock.lock(topicQueueKey);\n        try {\n            defaultMessageStore.assignOffset(messageExtBatch);\n\n            putMessageLock.lock();\n            try {\n                long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now();\n                this.beginTimeInLock = beginLockTimestamp;\n\n                // Here settings are stored timestamp, in order to ensure an orderly\n                // global\n                messageExtBatch.setStoreTimestamp(beginLockTimestamp);\n\n                if (null == mappedFile || mappedFile.isFull()) {\n                    mappedFile = this.mappedFileQueue.getLastMappedFile(0); // Mark: NewFile may be cause noise\n                    if (isCloseReadAhead()) {\n                        setFileReadMode(mappedFile, LibC.MADV_RANDOM);\n                    }\n                }\n                if (null == mappedFile) {\n                    log.error(\"Create mapped file1 error, topic: {} clientAddr: {}\", messageExtBatch.getTopic(), messageExtBatch.getBornHostString());\n                    return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, null));\n                }\n\n                result = mappedFile.appendMessages(messageExtBatch, this.appendMessageCallback, putMessageContext);\n                switch (result.getStatus()) {\n                    case PUT_OK:\n                        break;\n                    case END_OF_FILE:\n                        unlockMappedFile = mappedFile;\n                        // Create a new file, re-write the message\n                        mappedFile = this.mappedFileQueue.getLastMappedFile(0);\n                        if (null == mappedFile) {\n                            // XXX: warn and notify me\n                            log.error(\"Create mapped file2 error, topic: {} clientAddr: {}\", messageExtBatch.getTopic(), messageExtBatch.getBornHostString());\n                            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, result));\n                        }\n                        if (isCloseReadAhead()) {\n                            setFileReadMode(mappedFile, LibC.MADV_RANDOM);\n                        }\n                        result = mappedFile.appendMessages(messageExtBatch, this.appendMessageCallback, putMessageContext);\n                        break;\n                    case MESSAGE_SIZE_EXCEEDED:\n                    case PROPERTIES_SIZE_EXCEEDED:\n                        return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result));\n                    case UNKNOWN_ERROR:\n                    default:\n                        return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result));\n                }\n\n                elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginLockTimestamp;\n            } finally {\n                beginTimeInLock = 0;\n                putMessageLock.unlock();\n            }\n\n            // Increase queue offset when messages are successfully written\n            if (AppendMessageStatus.PUT_OK.equals(result.getStatus())) {\n                this.defaultMessageStore.increaseOffset(messageExtBatch, (short) putMessageContext.getBatchSize());\n            }\n        } catch (RocksDBException e) {\n            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result));\n        } finally {\n            topicQueueLock.unlock(topicQueueKey);\n        }\n\n        if (elapsedTimeInLock > 500) {\n            log.warn(\"[NOTIFYME]putMessages in lock cost time(ms)={}, bodyLength={} AppendMessageResult={}\", elapsedTimeInLock, messageExtBatch.getBody().length, result);\n        }\n\n        if (null != unlockMappedFile && this.defaultMessageStore.getMessageStoreConfig().isWarmMapedFileEnable()) {\n            this.defaultMessageStore.unlockMappedFile(unlockMappedFile);\n        }\n\n        PutMessageResult putMessageResult = new PutMessageResult(PutMessageStatus.PUT_OK, result);\n\n        // Statistics\n        storeStatsService.getSinglePutMessageTopicTimesTotal(messageExtBatch.getTopic()).add(result.getMsgNum());\n        storeStatsService.getSinglePutMessageTopicSizeTotal(messageExtBatch.getTopic()).add(result.getWroteBytes());\n\n        return handleDiskFlushAndHA(putMessageResult, messageExtBatch, needAckNums, needHandleHA);\n    }\n\n    private int calcNeedAckNums(int inSyncReplicas) {\n        int needAckNums = this.defaultMessageStore.getMessageStoreConfig().getInSyncReplicas();\n        if (this.defaultMessageStore.getMessageStoreConfig().isEnableAutoInSyncReplicas()) {\n            needAckNums = Math.min(needAckNums, inSyncReplicas);\n            needAckNums = Math.max(needAckNums, this.defaultMessageStore.getMessageStoreConfig().getMinInSyncReplicas());\n        }\n        return needAckNums;\n    }\n\n    private boolean needHandleHA(MessageExt messageExt) {\n\n        if (!messageExt.isWaitStoreMsgOK()) {\n            /*\n              No need to sync messages that special config to extra broker slaves.\n              @see MessageConst.PROPERTY_WAIT_STORE_MSG_OK\n             */\n            return false;\n        }\n\n        if (this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable()) {\n            return false;\n        }\n\n        if (BrokerRole.SYNC_MASTER != this.defaultMessageStore.getMessageStoreConfig().getBrokerRole()) {\n            // No need to check ha in async or slave broker\n            return false;\n        }\n\n        return true;\n    }\n\n    private CompletableFuture<PutMessageResult> handleDiskFlushAndHA(PutMessageResult putMessageResult,\n        MessageExt messageExt, int needAckNums, boolean needHandleHA) {\n        CompletableFuture<PutMessageStatus> flushResultFuture = handleDiskFlush(putMessageResult.getAppendMessageResult(), messageExt);\n        CompletableFuture<PutMessageStatus> replicaResultFuture;\n        if (!needHandleHA) {\n            replicaResultFuture = CompletableFuture.completedFuture(PutMessageStatus.PUT_OK);\n        } else {\n            replicaResultFuture = handleHA(putMessageResult.getAppendMessageResult(), putMessageResult, needAckNums);\n        }\n\n        return flushResultFuture.thenCombine(replicaResultFuture, (flushStatus, replicaStatus) -> {\n            if (flushStatus != PutMessageStatus.PUT_OK) {\n                putMessageResult.setPutMessageStatus(flushStatus);\n            }\n            if (replicaStatus != PutMessageStatus.PUT_OK) {\n                putMessageResult.setPutMessageStatus(replicaStatus);\n            }\n            return putMessageResult;\n        });\n    }\n\n    private CompletableFuture<PutMessageStatus> handleDiskFlush(AppendMessageResult result, MessageExt messageExt) {\n        return this.flushManager.handleDiskFlush(result, messageExt);\n    }\n\n    private CompletableFuture<PutMessageStatus> handleHA(AppendMessageResult result, PutMessageResult putMessageResult,\n        int needAckNums) {\n        if (needAckNums >= 0 && needAckNums <= 1) {\n            return CompletableFuture.completedFuture(PutMessageStatus.PUT_OK);\n        }\n\n        HAService haService = this.defaultMessageStore.getHaService();\n\n        long nextOffset = result.getWroteOffset() + result.getWroteBytes();\n\n        // Wait enough acks from different slaves\n        GroupCommitRequest request = new GroupCommitRequest(nextOffset, this.defaultMessageStore.getMessageStoreConfig().getSlaveTimeout(), needAckNums);\n        haService.putRequest(request);\n        haService.getWaitNotifyObject().wakeupAll();\n        return request.future();\n    }\n\n    /**\n     * According to receive certain message or offset storage time if an error occurs, it returns -1\n     */\n    public long pickupStoreTimestamp(final long offset, final int size) {\n        if (defaultMessageStore.isShutdown()) {\n            throw new RuntimeException(\"message store has shutdown\");\n        }\n        if (offset >= this.getMinOffset() && offset + size <= this.getMaxOffset()) {\n            SelectMappedBufferResult result = this.getMessage(offset, size);\n            if (null != result) {\n                try {\n                    int sysFlag = result.getByteBuffer().getInt(MessageDecoder.SYSFLAG_POSITION);\n                    int bornhostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20;\n                    int msgStoreTimePos = 4 + 4 + 4 + 4 + 4 + 8 + 8 + 4 + 8 + bornhostLength;\n                    return result.getByteBuffer().getLong(msgStoreTimePos);\n                } finally {\n                    result.release();\n                }\n            }\n        }\n\n        return -1;\n    }\n\n    public long getMinOffset() {\n        MappedFile mappedFile = this.mappedFileQueue.getFirstMappedFile();\n        if (mappedFile != null) {\n            if (mappedFile.isAvailable()) {\n                return mappedFile.getFileFromOffset();\n            } else {\n                return this.rollNextFile(mappedFile.getFileFromOffset());\n            }\n        }\n\n        return -1;\n    }\n\n    public SelectMappedBufferResult getMessage(final long offset, final int size) {\n        int mappedFileSize = this.defaultMessageStore.getMessageStoreConfig().getMappedFileSizeCommitLog();\n        MappedFile mappedFile = this.mappedFileQueue.findMappedFileByOffset(offset, offset == 0);\n        if (mappedFile != null) {\n            int pos = (int) (offset % mappedFileSize);\n            SelectMappedBufferResult selectMappedBufferResult = mappedFile.selectMappedBuffer(pos, size);\n            if (null != selectMappedBufferResult) {\n                selectMappedBufferResult.setInCache(coldDataCheckService.isDataInPageCache(offset));\n                return selectMappedBufferResult;\n            }\n        }\n        return null;\n    }\n\n    public long rollNextFile(final long offset) {\n        int mappedFileSize = this.defaultMessageStore.getMessageStoreConfig().getMappedFileSizeCommitLog();\n        return offset + mappedFileSize - offset % mappedFileSize;\n    }\n\n    public void destroy() {\n        this.mappedFileQueue.destroy();\n    }\n\n    public boolean appendData(long startOffset, byte[] data, int dataStart, int dataLength) {\n        putMessageLock.lock();\n        try {\n            MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(startOffset);\n            if (null == mappedFile) {\n                log.error(\"appendData getLastMappedFile error {}\", startOffset);\n                return false;\n            }\n\n            return mappedFile.appendMessage(data, dataStart, dataLength);\n        } finally {\n            putMessageLock.unlock();\n        }\n    }\n\n    public boolean retryDeleteFirstFile(final long intervalForcibly) {\n        return this.mappedFileQueue.retryDeleteFirstFile(intervalForcibly);\n    }\n\n    public void checkSelf() {\n        mappedFileQueue.checkSelf();\n    }\n\n    public long lockTimeMills() {\n        long diff = 0;\n        long begin = this.beginTimeInLock;\n        if (begin > 0) {\n            diff = this.defaultMessageStore.now() - begin;\n        }\n\n        if (diff < 0) {\n            diff = 0;\n        }\n\n        return diff;\n    }\n\n    protected short getMessageNum(MessageExtBrokerInner msgInner) {\n        short messageNum = 1;\n        // IF inner batch, build batchQueueOffset and batchNum property.\n        CQType cqType = getCqType(msgInner);\n\n        if (MessageSysFlag.check(msgInner.getSysFlag(), MessageSysFlag.INNER_BATCH_FLAG) || CQType.BatchCQ.equals(cqType)) {\n            if (msgInner.getProperty(MessageConst.PROPERTY_INNER_NUM) != null) {\n                messageNum = Short.parseShort(msgInner.getProperty(MessageConst.PROPERTY_INNER_NUM));\n                messageNum = messageNum >= 1 ? messageNum : 1;\n            }\n        }\n\n        return messageNum;\n    }\n\n    private CQType getCqType(MessageExtBrokerInner msgInner) {\n        Optional<TopicConfig> topicConfig = this.defaultMessageStore.getTopicConfig(msgInner.getTopic());\n        return QueueTypeUtils.getCQType(topicConfig);\n    }\n\n    abstract class FlushCommitLogService extends ServiceThread {\n        protected static final int RETRY_TIMES_OVER = 10;\n    }\n\n    class CommitRealTimeService extends FlushCommitLogService {\n\n        private long lastCommitTimestamp = 0;\n\n        @Override\n        public String getServiceName() {\n            if (CommitLog.this.defaultMessageStore.getBrokerConfig().isInBrokerContainer()) {\n                return CommitLog.this.defaultMessageStore.getBrokerIdentity().getIdentifier() + CommitRealTimeService.class.getSimpleName();\n            }\n            return CommitRealTimeService.class.getSimpleName();\n        }\n\n        @Override\n        public void run() {\n            CommitLog.log.info(\"{} service started\", this.getServiceName());\n            while (!this.isStopped()) {\n                int interval = CommitLog.this.defaultMessageStore.getMessageStoreConfig().getCommitIntervalCommitLog();\n\n                int commitDataLeastPages = CommitLog.this.defaultMessageStore.getMessageStoreConfig().getCommitCommitLogLeastPages();\n\n                int commitDataThoroughInterval =\n                    CommitLog.this.defaultMessageStore.getMessageStoreConfig().getCommitCommitLogThoroughInterval();\n\n                long begin = System.currentTimeMillis();\n                if (begin >= (this.lastCommitTimestamp + commitDataThoroughInterval)) {\n                    this.lastCommitTimestamp = begin;\n                    commitDataLeastPages = 0;\n                }\n\n                try {\n                    boolean result = CommitLog.this.mappedFileQueue.commit(commitDataLeastPages);\n                    long end = System.currentTimeMillis();\n                    if (!result) {\n                        this.lastCommitTimestamp = end; // result = false means some data committed.\n                        CommitLog.this.flushManager.wakeUpFlush();\n                    }\n                    CommitLog.this.getMessageStore().getPerfCounter().flowOnce(\"COMMIT_DATA_TIME_MS\", (int) (end - begin));\n                    if (end - begin > 500) {\n                        log.info(\"Commit data to file costs {} ms\", end - begin);\n                    }\n                    this.waitForRunning(interval);\n                } catch (Throwable e) {\n                    CommitLog.log.error(\"{} service has exception. \", this.getServiceName(), e);\n                }\n            }\n\n            boolean result = false;\n            for (int i = 0; i < RETRY_TIMES_OVER && !result; i++) {\n                result = CommitLog.this.mappedFileQueue.commit(0);\n                CommitLog.log.info(\"{} service shutdown, retry {} times {}\", this.getServiceName(), i + 1, result ? \"OK\" : \"Not OK\");\n            }\n            CommitLog.log.info(\"{} service end\", this.getServiceName());\n        }\n    }\n\n    class FlushRealTimeService extends FlushCommitLogService {\n        private long lastFlushTimestamp = 0;\n        private long printTimes = 0;\n\n        @Override\n        public void run() {\n            CommitLog.log.info(\"{} service started\", this.getServiceName());\n\n            while (!this.isStopped()) {\n                boolean flushCommitLogTimed = CommitLog.this.defaultMessageStore.getMessageStoreConfig().isFlushCommitLogTimed();\n\n                int interval = CommitLog.this.defaultMessageStore.getMessageStoreConfig().getFlushIntervalCommitLog();\n                int flushPhysicQueueLeastPages = CommitLog.this.defaultMessageStore.getMessageStoreConfig().getFlushCommitLogLeastPages();\n\n                int flushPhysicQueueThoroughInterval =\n                    CommitLog.this.defaultMessageStore.getMessageStoreConfig().getFlushCommitLogThoroughInterval();\n\n                boolean printFlushProgress = false;\n\n                // Print flush progress\n                long currentTimeMillis = System.currentTimeMillis();\n                if (currentTimeMillis >= (this.lastFlushTimestamp + flushPhysicQueueThoroughInterval)) {\n                    this.lastFlushTimestamp = currentTimeMillis;\n                    flushPhysicQueueLeastPages = 0;\n                    printFlushProgress = (printTimes++ % 10) == 0;\n                }\n\n                try {\n                    if (flushCommitLogTimed) {\n                        Thread.sleep(interval);\n                    } else {\n                        this.waitForRunning(interval);\n                    }\n\n                    if (printFlushProgress) {\n                        this.printFlushProgress();\n                    }\n\n                    long begin = System.currentTimeMillis();\n                    CommitLog.this.mappedFileQueue.flush(flushPhysicQueueLeastPages);\n                    long storeTimestamp = CommitLog.this.mappedFileQueue.getStoreTimestamp();\n                    if (storeTimestamp > 0) {\n                        CommitLog.this.defaultMessageStore.getStoreCheckpoint().setPhysicMsgTimestamp(storeTimestamp);\n                    }\n                    long past = System.currentTimeMillis() - begin;\n                    CommitLog.this.getMessageStore().getPerfCounter().flowOnce(\"FLUSH_DATA_TIME_MS\", (int) past);\n                    if (past > 500) {\n                        log.info(\"Flush data to disk costs {} ms\", past);\n                    }\n                } catch (Throwable e) {\n                    CommitLog.log.warn(\"{} service has exception. \", this.getServiceName(), e);\n                    this.printFlushProgress();\n                }\n            }\n\n            // Normal shutdown, to ensure that all the flush before exit\n            boolean result = false;\n            for (int i = 0; i < RETRY_TIMES_OVER && !result; i++) {\n                result = CommitLog.this.mappedFileQueue.flush(0);\n                CommitLog.log.info(\"{} service shutdown, retry {} times {}\", this.getServiceName(), i + 1, result ? \"OK\" : \"Not OK\");\n            }\n\n            this.printFlushProgress();\n\n            CommitLog.log.info(\"{} service end\", this.getServiceName());\n        }\n\n        @Override\n        public String getServiceName() {\n            if (CommitLog.this.defaultMessageStore.getBrokerConfig().isInBrokerContainer()) {\n                return CommitLog.this.defaultMessageStore.getBrokerConfig().getIdentifier() + FlushRealTimeService.class.getSimpleName();\n            }\n            return FlushRealTimeService.class.getSimpleName();\n        }\n\n        private void printFlushProgress() {\n            // CommitLog.log.info(\"how much disk fall behind memory, \"\n            // + CommitLog.this.mappedFileQueue.howMuchFallBehind());\n        }\n\n        @Override\n        public long getJoinTime() {\n            return 1000 * 60 * 5;\n        }\n    }\n\n    public static class GroupCommitRequest {\n        private final long nextOffset;\n        // Indicate the GroupCommitRequest result: true or false\n        private final CompletableFuture<PutMessageStatus> flushOKFuture = new CompletableFuture<>();\n        private volatile int ackNums = 1;\n        private final long deadLine;\n\n        public GroupCommitRequest(long nextOffset, long timeoutMillis) {\n            this.nextOffset = nextOffset;\n            this.deadLine = System.nanoTime() + (timeoutMillis * 1_000_000);\n        }\n\n        public GroupCommitRequest(long nextOffset, long timeoutMillis, int ackNums) {\n            this(nextOffset, timeoutMillis);\n            this.ackNums = ackNums;\n        }\n\n        public long getNextOffset() {\n            return nextOffset;\n        }\n\n        public int getAckNums() {\n            return ackNums;\n        }\n\n        public long getDeadLine() {\n            return deadLine;\n        }\n\n        public void wakeupCustomer(final PutMessageStatus status) {\n            this.flushOKFuture.complete(status);\n        }\n\n        public CompletableFuture<PutMessageStatus> future() {\n            return flushOKFuture;\n        }\n    }\n\n    /**\n     * GroupCommit Service\n     */\n    class GroupCommitService extends FlushCommitLogService {\n        private LinkedList<GroupCommitRequest> requestsWrite = new LinkedList<>();\n        private LinkedList<GroupCommitRequest> requestsRead = new LinkedList<>();\n        private final PutMessageSpinLock lock = new PutMessageSpinLock();\n\n        public void putRequest(final GroupCommitRequest request) {\n            lock.lock();\n            try {\n                this.requestsWrite.add(request);\n            } finally {\n                lock.unlock();\n            }\n            this.wakeup();\n        }\n\n        private void swapRequests() {\n            lock.lock();\n            try {\n                LinkedList<GroupCommitRequest> tmp = this.requestsWrite;\n                this.requestsWrite = this.requestsRead;\n                this.requestsRead = tmp;\n            } finally {\n                lock.unlock();\n            }\n        }\n\n        private void doCommit() {\n            if (!this.requestsRead.isEmpty()) {\n                for (GroupCommitRequest req : this.requestsRead) {\n                    boolean flushOK = CommitLog.this.mappedFileQueue.getFlushedWhere() >= req.getNextOffset();\n                    for (int i = 0; i < 1000 && !flushOK; i++) {\n                        CommitLog.this.mappedFileQueue.flush(0);\n                        flushOK = CommitLog.this.mappedFileQueue.getFlushedWhere() >= req.getNextOffset();\n                        if (flushOK) {\n                            break;\n                        } else {\n                            // When transientStorePoolEnable is true, the messages in writeBuffer may not be committed\n                            // to pageCache very quickly, and flushOk here may almost be false, so we can sleep 1ms to\n                            // wait for the messages to be committed to pageCache.\n                            try {\n                                Thread.sleep(1);\n                            } catch (InterruptedException ignored) {\n                            }\n                        }\n                    }\n\n                    req.wakeupCustomer(flushOK ? PutMessageStatus.PUT_OK : PutMessageStatus.FLUSH_DISK_TIMEOUT);\n                }\n\n                long storeTimestamp = CommitLog.this.mappedFileQueue.getStoreTimestamp();\n                if (storeTimestamp > 0) {\n                    CommitLog.this.defaultMessageStore.getStoreCheckpoint().setPhysicMsgTimestamp(storeTimestamp);\n                }\n\n                this.requestsRead = new LinkedList<>();\n            } else {\n                // Because of individual messages is set to not sync flush, it\n                // will come to this process\n                CommitLog.this.mappedFileQueue.flush(0);\n            }\n        }\n\n        @Override\n        public void run() {\n            CommitLog.log.info(\"{} service started\", this.getServiceName());\n\n            while (!this.isStopped()) {\n                try {\n                    this.waitForRunning(10);\n                    this.doCommit();\n                } catch (Exception e) {\n                    CommitLog.log.warn(\"{} service has exception. \", this.getServiceName(), e);\n                }\n            }\n\n            // Under normal circumstances shutdown, wait for the arrival of the\n            // request, and then flush\n            try {\n                Thread.sleep(10);\n            } catch (InterruptedException e) {\n                CommitLog.log.warn(\"GroupCommitService Exception, \", e);\n            }\n\n            this.swapRequests();\n            this.doCommit();\n\n            CommitLog.log.info(\"{} service end\", this.getServiceName());\n        }\n\n        @Override\n        protected void onWaitEnd() {\n            this.swapRequests();\n        }\n\n        @Override\n        public String getServiceName() {\n            if (CommitLog.this.defaultMessageStore.getBrokerConfig().isInBrokerContainer()) {\n                return CommitLog.this.defaultMessageStore.getBrokerConfig().getIdentifier() + GroupCommitService.class.getSimpleName();\n            }\n            return GroupCommitService.class.getSimpleName();\n        }\n\n        @Override\n        public long getJoinTime() {\n            return 1000 * 60 * 5;\n        }\n    }\n\n    class GroupCheckService extends FlushCommitLogService {\n        private volatile List<GroupCommitRequest> requestsWrite = new ArrayList<>();\n        private volatile List<GroupCommitRequest> requestsRead = new ArrayList<>();\n\n        public boolean isAsyncRequestsFull() {\n            return requestsWrite.size() > CommitLog.this.defaultMessageStore.getMessageStoreConfig().getMaxAsyncPutMessageRequests() * 2;\n        }\n\n        public synchronized boolean putRequest(final GroupCommitRequest request) {\n            synchronized (this.requestsWrite) {\n                this.requestsWrite.add(request);\n            }\n            if (hasNotified.compareAndSet(false, true)) {\n                waitPoint.countDown(); // notify\n            }\n            boolean flag = this.requestsWrite.size() >\n                CommitLog.this.defaultMessageStore.getMessageStoreConfig().getMaxAsyncPutMessageRequests();\n            if (flag) {\n                log.info(\"Async requests {} exceeded the threshold {}\", requestsWrite.size(),\n                    CommitLog.this.defaultMessageStore.getMessageStoreConfig().getMaxAsyncPutMessageRequests());\n            }\n\n            return flag;\n        }\n\n        private void swapRequests() {\n            List<GroupCommitRequest> tmp = this.requestsWrite;\n            this.requestsWrite = this.requestsRead;\n            this.requestsRead = tmp;\n        }\n\n        private void doCommit() {\n            synchronized (this.requestsRead) {\n                if (!this.requestsRead.isEmpty()) {\n                    for (GroupCommitRequest req : this.requestsRead) {\n                        // There may be a message in the next file, so a maximum of\n                        // two times the flush\n                        boolean flushOK = false;\n                        for (int i = 0; i < 1000; i++) {\n                            flushOK = CommitLog.this.mappedFileQueue.getFlushedWhere() >= req.getNextOffset();\n                            if (flushOK) {\n                                break;\n                            } else {\n                                try {\n                                    Thread.sleep(1);\n                                } catch (Throwable ignored) {\n\n                                }\n                            }\n                        }\n                        req.wakeupCustomer(flushOK ? PutMessageStatus.PUT_OK : PutMessageStatus.FLUSH_DISK_TIMEOUT);\n                    }\n\n                    long storeTimestamp = CommitLog.this.mappedFileQueue.getStoreTimestamp();\n                    if (storeTimestamp > 0) {\n                        CommitLog.this.defaultMessageStore.getStoreCheckpoint().setPhysicMsgTimestamp(storeTimestamp);\n                    }\n\n                    this.requestsRead.clear();\n                }\n            }\n        }\n\n        public void run() {\n            CommitLog.log.info(\"{} service started\", this.getServiceName());\n\n            while (!this.isStopped()) {\n                try {\n                    this.waitForRunning(1);\n                    this.doCommit();\n                } catch (Exception e) {\n                    CommitLog.log.warn(\"{} service has exception. \", this.getServiceName(), e);\n                }\n            }\n\n            // Under normal circumstances shutdown, wait for the arrival of the\n            // request, and then flush\n            try {\n                Thread.sleep(10);\n            } catch (InterruptedException e) {\n                CommitLog.log.warn(\"GroupCommitService Exception, \", e);\n            }\n\n            synchronized (this) {\n                this.swapRequests();\n            }\n\n            this.doCommit();\n\n            CommitLog.log.info(\"{} service end\", this.getServiceName());\n        }\n\n        @Override\n        protected void onWaitEnd() {\n            this.swapRequests();\n        }\n\n        @Override\n        public String getServiceName() {\n            if (CommitLog.this.defaultMessageStore.getBrokerConfig().isInBrokerContainer()) {\n                return CommitLog.this.defaultMessageStore.getBrokerConfig().getIdentifier() + GroupCheckService.class.getSimpleName();\n            }\n            return GroupCheckService.class.getSimpleName();\n        }\n\n        @Override\n        public long getJoinTime() {\n            return 1000 * 60 * 5;\n        }\n    }\n\n    class DefaultAppendMessageCallback implements AppendMessageCallback {\n        // File at the end of the minimum fixed length empty\n        private static final int END_FILE_MIN_BLANK_LENGTH = 4 + 4;\n        // Store the message content\n        private final ByteBuffer msgStoreItemMemory;\n        private final int crc32ReservedLength;\n        private final MessageStoreConfig messageStoreConfig;\n\n        DefaultAppendMessageCallback(MessageStoreConfig messageStoreConfig) {\n            this.msgStoreItemMemory = ByteBuffer.allocate(END_FILE_MIN_BLANK_LENGTH);\n            this.messageStoreConfig = messageStoreConfig;\n            this.crc32ReservedLength = messageStoreConfig.isEnabledAppendPropCRC() ? CommitLog.CRC32_RESERVED_LEN : 0;\n        }\n\n        public AppendMessageResult handlePropertiesForLmqMsg(ByteBuffer preEncodeBuffer,\n            final MessageExtBrokerInner msgInner) {\n            if (msgInner.isEncodeCompleted()) {\n                return null;\n            }\n\n            try {\n                LmqDispatch.wrapLmqDispatch(defaultMessageStore, msgInner);\n            } catch (ConsumeQueueException e) {\n                if (e.getCause() instanceof RocksDBException) {\n                    log.error(\"Failed to wrap multi-dispatch\", e);\n                    return new AppendMessageResult(AppendMessageStatus.ROCKSDB_ERROR);\n                }\n                log.error(\"Failed to wrap multi-dispatch\", e);\n                return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);\n            }\n\n            msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));\n\n            final byte[] propertiesData =\n                msgInner.getPropertiesString() == null ? null : msgInner.getPropertiesString().getBytes(MessageDecoder.CHARSET_UTF8);\n\n            boolean needAppendLastPropertySeparator = enabledAppendPropCRC && propertiesData != null && propertiesData.length > 0\n                && propertiesData[propertiesData.length - 1] != MessageDecoder.PROPERTY_SEPARATOR;\n\n            final int propertiesLength = (propertiesData == null ? 0 : propertiesData.length) + (needAppendLastPropertySeparator ? 1 : 0) + crc32ReservedLength;\n\n            if (propertiesLength > Short.MAX_VALUE) {\n                log.warn(\"putMessage message properties length too long. length={}\", propertiesData.length);\n                return new AppendMessageResult(AppendMessageStatus.PROPERTIES_SIZE_EXCEEDED);\n            }\n\n            int msgLenWithoutProperties = preEncodeBuffer.getInt(0);\n\n            int msgLen = msgLenWithoutProperties + 2 + propertiesLength;\n\n            // Exceeds the maximum message\n            if (msgLen > this.messageStoreConfig.getMaxMessageSize()) {\n                log.warn(\"message size exceeded, msg total size: {}, maxMessageSize: {}\", msgLen, this.messageStoreConfig.getMaxMessageSize());\n                return new AppendMessageResult(AppendMessageStatus.MESSAGE_SIZE_EXCEEDED);\n            }\n\n            // Back filling total message length\n            preEncodeBuffer.putInt(0, msgLen);\n            // Modify position to msgLenWithoutProperties\n            preEncodeBuffer.position(msgLenWithoutProperties);\n\n            preEncodeBuffer.putShort((short) propertiesLength);\n\n            if (propertiesLength > crc32ReservedLength) {\n                preEncodeBuffer.put(propertiesData);\n            }\n\n            if (needAppendLastPropertySeparator) {\n                preEncodeBuffer.put((byte) MessageDecoder.PROPERTY_SEPARATOR);\n            }\n            // 18 CRC32\n            preEncodeBuffer.position(preEncodeBuffer.position() + crc32ReservedLength);\n\n            msgInner.setEncodeCompleted(true);\n\n            return null;\n        }\n\n        public AppendMessageResult doAppend(final long fileFromOffset, final ByteBuffer byteBuffer, final int maxBlank,\n            final MessageExtBrokerInner msgInner, PutMessageContext putMessageContext) {\n            // STORETIMESTAMP + STOREHOSTADDRESS + OFFSET <br>\n\n            ByteBuffer preEncodeBuffer = msgInner.getEncodedBuff();\n            boolean isMultiDispatchMsg = messageStoreConfig.isEnableLmq() && msgInner.needDispatchLMQ();\n            if (isMultiDispatchMsg) {\n                AppendMessageResult appendMessageResult = handlePropertiesForLmqMsg(preEncodeBuffer, msgInner);\n                if (appendMessageResult != null) {\n                    return appendMessageResult;\n                }\n            }\n\n            final int msgLen = preEncodeBuffer.getInt(0);\n            preEncodeBuffer.position(0);\n            preEncodeBuffer.limit(msgLen);\n\n            // PHY OFFSET\n            long wroteOffset = fileFromOffset + byteBuffer.position();\n\n            Supplier<String> msgIdSupplier = () -> {\n                int sysflag = msgInner.getSysFlag();\n                int msgIdLen = (sysflag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 + 4 + 8 : 16 + 4 + 8;\n                ByteBuffer msgIdBuffer = ByteBuffer.allocate(msgIdLen);\n                MessageExt.socketAddress2ByteBuffer(msgInner.getStoreHost(), msgIdBuffer);\n                msgIdBuffer.clear();//because socketAddress2ByteBuffer flip the buffer\n                msgIdBuffer.putLong(msgIdLen - 8, wroteOffset);\n                return UtilAll.bytes2string(msgIdBuffer.array());\n            };\n\n            // Record ConsumeQueue information\n            Long queueOffset = msgInner.getQueueOffset();\n\n            // this msg maybe an inner-batch msg.\n            short messageNum = getMessageNum(msgInner);\n\n            // Transaction messages that require special handling\n            final int tranType = MessageSysFlag.getTransactionValue(msgInner.getSysFlag());\n            switch (tranType) {\n                // Prepared and Rollback message is not consumed, will not enter the consume queue\n                case MessageSysFlag.TRANSACTION_PREPARED_TYPE:\n                case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:\n                    queueOffset = 0L;\n                    break;\n                case MessageSysFlag.TRANSACTION_NOT_TYPE:\n                case MessageSysFlag.TRANSACTION_COMMIT_TYPE:\n                default:\n                    break;\n            }\n\n            // Determines whether there is sufficient free space\n            if ((msgLen + END_FILE_MIN_BLANK_LENGTH) > maxBlank) {\n                this.msgStoreItemMemory.clear();\n                // 1 TOTALSIZE\n                this.msgStoreItemMemory.putInt(maxBlank);\n                // 2 MAGICCODE\n                this.msgStoreItemMemory.putInt(CommitLog.BLANK_MAGIC_CODE);\n                // 3 The remaining space may be any value\n                // Here the length of the specially set maxBlank\n                final long beginTimeMills = CommitLog.this.defaultMessageStore.now();\n                byteBuffer.put(this.msgStoreItemMemory.array(), 0, 8);\n                return new AppendMessageResult(AppendMessageStatus.END_OF_FILE, wroteOffset,\n                    maxBlank, /* only wrote 8 bytes, but declare wrote maxBlank for compute write position */\n                    msgIdSupplier, msgInner.getStoreTimestamp(),\n                    queueOffset, CommitLog.this.defaultMessageStore.now() - beginTimeMills);\n            }\n\n            int pos = 4     // 1 TOTALSIZE\n                + 4     // 2 MAGICCODE\n                + 4     // 3 BODYCRC\n                + 4     // 4 QUEUEID\n                + 4;    // 5 FLAG\n            // 6 QUEUEOFFSET\n            preEncodeBuffer.putLong(pos, queueOffset);\n            pos += 8;\n            // 7 PHYSICALOFFSET\n            preEncodeBuffer.putLong(pos, fileFromOffset + byteBuffer.position());\n            pos += 8;\n            int ipLen = (msgInner.getSysFlag() & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 4 + 4 : 16 + 4;\n            // 8 SYSFLAG, 9 BORNTIMESTAMP, 10 BORNHOST\n            pos += 4 + 8 + ipLen;\n            // 11 STORETIMESTAMP refresh store time stamp in lock\n            preEncodeBuffer.putLong(pos, msgInner.getStoreTimestamp());\n            if (enabledAppendPropCRC) {\n                // 18 CRC32\n                int checkSize = msgLen - crc32ReservedLength;\n                ByteBuffer tmpBuffer = preEncodeBuffer.duplicate();\n                tmpBuffer.limit(tmpBuffer.position() + checkSize);\n                int crc32 = UtilAll.crc32(tmpBuffer);   // UtilAll.crc32 function will change the position to limit of the buffer\n                tmpBuffer.limit(tmpBuffer.position() + crc32ReservedLength);\n                MessageDecoder.createCrc32(tmpBuffer, crc32);\n            }\n\n            final long beginTimeMills = CommitLog.this.defaultMessageStore.now();\n            CommitLog.this.getMessageStore().getPerfCounter().startTick(\"WRITE_MEMORY_TIME_MS\");\n            // Write messages to the queue buffer\n            byteBuffer.put(preEncodeBuffer);\n            CommitLog.this.getMessageStore().getPerfCounter().endTick(\"WRITE_MEMORY_TIME_MS\");\n            msgInner.setEncodedBuff(null);\n\n            if (isMultiDispatchMsg) {\n                try {\n                    LmqDispatch.updateLmqOffsets(defaultMessageStore, msgInner);\n                } catch (ConsumeQueueException e) {\n                    // Increase in-memory max offset of the queue should not fail.\n                    return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);\n                }\n            }\n\n            return new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, msgLen, msgIdSupplier,\n                msgInner.getStoreTimestamp(), queueOffset, CommitLog.this.defaultMessageStore.now() - beginTimeMills, messageNum);\n        }\n\n        public AppendMessageResult doAppend(final long fileFromOffset, final ByteBuffer byteBuffer, final int maxBlank,\n            final MessageExtBatch messageExtBatch, PutMessageContext putMessageContext) {\n            byteBuffer.mark();\n            //physical offset\n            long wroteOffset = fileFromOffset + byteBuffer.position();\n            // Record ConsumeQueue information\n            Long queueOffset = messageExtBatch.getQueueOffset();\n            long beginQueueOffset = queueOffset;\n            int totalMsgLen = 0;\n            int msgNum = 0;\n\n            final long beginTimeMills = CommitLog.this.defaultMessageStore.now();\n            ByteBuffer messagesByteBuff = messageExtBatch.getEncodedBuff();\n\n            int sysFlag = messageExtBatch.getSysFlag();\n            int bornHostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 4 + 4 : 16 + 4;\n            int storeHostLength = (sysFlag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 + 4 : 16 + 4;\n            Supplier<String> msgIdSupplier = () -> {\n                int msgIdLen = storeHostLength + 8;\n                int batchCount = putMessageContext.getBatchSize();\n                long[] phyPosArray = putMessageContext.getPhyPos();\n                ByteBuffer msgIdBuffer = ByteBuffer.allocate(msgIdLen);\n                MessageExt.socketAddress2ByteBuffer(messageExtBatch.getStoreHost(), msgIdBuffer);\n                msgIdBuffer.clear();//because socketAddress2ByteBuffer flip the buffer\n\n                StringBuilder buffer = new StringBuilder(batchCount * msgIdLen * 2 + batchCount - 1);\n                for (int i = 0; i < phyPosArray.length; i++) {\n                    msgIdBuffer.putLong(msgIdLen - 8, phyPosArray[i]);\n                    String msgId = UtilAll.bytes2string(msgIdBuffer.array());\n                    if (i != 0) {\n                        buffer.append(',');\n                    }\n                    buffer.append(msgId);\n                }\n                return buffer.toString();\n            };\n\n            messagesByteBuff.mark();\n            int index = 0;\n            while (messagesByteBuff.hasRemaining()) {\n                // 1 TOTALSIZE\n                final int msgPos = messagesByteBuff.position();\n                final int msgLen = messagesByteBuff.getInt();\n\n                totalMsgLen += msgLen;\n                // Determines whether there is sufficient free space\n                if ((totalMsgLen + END_FILE_MIN_BLANK_LENGTH) > maxBlank) {\n                    this.msgStoreItemMemory.clear();\n                    // 1 TOTALSIZE\n                    this.msgStoreItemMemory.putInt(maxBlank);\n                    // 2 MAGICCODE\n                    this.msgStoreItemMemory.putInt(CommitLog.BLANK_MAGIC_CODE);\n                    // 3 The remaining space may be any value\n                    //ignore previous read\n                    messagesByteBuff.reset();\n                    // Here the length of the specially set maxBlank\n                    byteBuffer.reset(); //ignore the previous appended messages\n                    byteBuffer.put(this.msgStoreItemMemory.array(), 0, 8);\n                    return new AppendMessageResult(AppendMessageStatus.END_OF_FILE, wroteOffset, maxBlank, msgIdSupplier, messageExtBatch.getStoreTimestamp(),\n                        beginQueueOffset, CommitLog.this.defaultMessageStore.now() - beginTimeMills);\n                }\n                //move to add queue offset and commitlog offset\n                int pos = msgPos + 20;\n                messagesByteBuff.putLong(pos, queueOffset);\n                pos += 8;\n                messagesByteBuff.putLong(pos, wroteOffset + totalMsgLen - msgLen);\n                // 8 SYSFLAG, 9 BORNTIMESTAMP, 10 BORNHOST, 11 STORETIMESTAMP\n                pos += 8 + 4 + 8 + bornHostLength;\n                // refresh store time stamp in lock\n                messagesByteBuff.putLong(pos, messageExtBatch.getStoreTimestamp());\n                if (enabledAppendPropCRC) {\n                    //append crc32\n                    int checkSize = msgLen - crc32ReservedLength;\n                    ByteBuffer tmpBuffer = messagesByteBuff.duplicate();\n                    tmpBuffer.position(msgPos).limit(msgPos + checkSize);\n                    int crc32 = UtilAll.crc32(tmpBuffer);\n                    messagesByteBuff.position(msgPos + checkSize);\n                    MessageDecoder.createCrc32(messagesByteBuff, crc32);\n                }\n\n                putMessageContext.getPhyPos()[index++] = wroteOffset + totalMsgLen - msgLen;\n                queueOffset++;\n                msgNum++;\n                messagesByteBuff.position(msgPos + msgLen);\n            }\n\n            messagesByteBuff.position(0);\n            messagesByteBuff.limit(totalMsgLen);\n            byteBuffer.put(messagesByteBuff);\n            messageExtBatch.setEncodedBuff(null);\n            AppendMessageResult result = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, totalMsgLen, msgIdSupplier,\n                messageExtBatch.getStoreTimestamp(), beginQueueOffset, CommitLog.this.defaultMessageStore.now() - beginTimeMills);\n            result.setMsgNum(msgNum);\n\n            return result;\n        }\n\n    }\n\n    class DefaultFlushManager implements FlushManager {\n\n        private final FlushCommitLogService flushCommitLogService;\n\n        //If TransientStorePool enabled, we must flush message to FileChannel at fixed periods\n        private final FlushCommitLogService commitRealTimeService;\n\n        public DefaultFlushManager() {\n            if (FlushDiskType.SYNC_FLUSH == CommitLog.this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {\n                this.flushCommitLogService = new CommitLog.GroupCommitService();\n            } else {\n                this.flushCommitLogService = new CommitLog.FlushRealTimeService();\n            }\n\n            this.commitRealTimeService = new CommitLog.CommitRealTimeService();\n        }\n\n        @Override\n        public void start() {\n            this.flushCommitLogService.start();\n\n            if (defaultMessageStore.isTransientStorePoolEnable()) {\n                this.commitRealTimeService.start();\n            }\n        }\n\n        @Override\n        public void handleDiskFlush(AppendMessageResult result, PutMessageResult putMessageResult,\n            MessageExt messageExt) {\n            // Synchronization flush\n            if (FlushDiskType.SYNC_FLUSH == CommitLog.this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {\n                final GroupCommitService service = (GroupCommitService) this.flushCommitLogService;\n                if (messageExt.isWaitStoreMsgOK()) {\n                    GroupCommitRequest request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes(), CommitLog.this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());\n                    service.putRequest(request);\n                    CompletableFuture<PutMessageStatus> flushOkFuture = request.future();\n                    PutMessageStatus flushStatus = null;\n                    try {\n                        flushStatus = flushOkFuture.get(CommitLog.this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout(), TimeUnit.MILLISECONDS);\n                    } catch (InterruptedException | ExecutionException | TimeoutException e) {\n                        //flushOK=false;\n                    }\n                    if (flushStatus != PutMessageStatus.PUT_OK) {\n                        log.error(\"do groupcommit, wait for flush failed, topic: {} tags: {} client address: {}\", messageExt.getTopic(), messageExt.getTags(), messageExt.getBornHostString());\n                        putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_DISK_TIMEOUT);\n                    }\n                } else {\n                    service.wakeup();\n                }\n            }\n            // Asynchronous flush\n            else {\n                if (!CommitLog.this.defaultMessageStore.isTransientStorePoolEnable()) {\n                    if (defaultMessageStore.getMessageStoreConfig().isWakeFlushWhenPutMessage()) {\n                        flushCommitLogService.wakeup();\n                    }\n                } else {\n                    if (defaultMessageStore.getMessageStoreConfig().isWakeCommitWhenPutMessage()) {\n                        commitRealTimeService.wakeup();\n                    }\n                }\n            }\n        }\n\n        @Override\n        public CompletableFuture<PutMessageStatus> handleDiskFlush(AppendMessageResult result, MessageExt messageExt) {\n            // Synchronization flush\n            if (FlushDiskType.SYNC_FLUSH == CommitLog.this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {\n                final GroupCommitService service = (GroupCommitService) this.flushCommitLogService;\n                if (messageExt.isWaitStoreMsgOK()) {\n                    GroupCommitRequest request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes(), CommitLog.this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());\n                    flushDiskWatcher.add(request);\n                    service.putRequest(request);\n                    return request.future();\n                } else {\n                    service.wakeup();\n                    return CompletableFuture.completedFuture(PutMessageStatus.PUT_OK);\n                }\n            }\n            // Asynchronous flush\n            else {\n                if (!CommitLog.this.defaultMessageStore.isTransientStorePoolEnable()) {\n                    if (defaultMessageStore.getMessageStoreConfig().isWakeFlushWhenPutMessage()) {\n                        flushCommitLogService.wakeup();\n                    }\n                } else {\n                    if (defaultMessageStore.getMessageStoreConfig().isWakeCommitWhenPutMessage()) {\n                        commitRealTimeService.wakeup();\n                    }\n                }\n                return CompletableFuture.completedFuture(PutMessageStatus.PUT_OK);\n            }\n        }\n\n        @Override\n        public void wakeUpFlush() {\n            // now wake up flush thread.\n            flushCommitLogService.wakeup();\n        }\n\n        @Override\n        public void wakeUpCommit() {\n            // now wake up commit log thread.\n            commitRealTimeService.wakeup();\n        }\n\n        @Override\n        public void shutdown() {\n            if (defaultMessageStore.isTransientStorePoolEnable()) {\n                this.commitRealTimeService.shutdown();\n            }\n\n            this.flushCommitLogService.shutdown();\n        }\n\n    }\n\n    public int getCommitLogSize() {\n        return commitLogSize;\n    }\n\n    public MappedFileQueue getMappedFileQueue() {\n        return mappedFileQueue;\n    }\n\n    public MessageStore getMessageStore() {\n        return defaultMessageStore;\n    }\n\n    public MappedFile getEarliestMappedFile() {\n        return mappedFileQueue.getEarliestMappedFile();\n    }\n\n    @Override\n    public void swapMap(int reserveNum, long forceSwapIntervalMs, long normalSwapIntervalMs) {\n        this.getMappedFileQueue().swapMap(reserveNum, forceSwapIntervalMs, normalSwapIntervalMs);\n    }\n\n    public boolean isMappedFilesEmpty() {\n        return this.mappedFileQueue.isMappedFilesEmpty();\n    }\n\n    @Override\n    public void cleanSwappedMap(long forceCleanSwapIntervalMs) {\n        this.getMappedFileQueue().cleanSwappedMap(forceCleanSwapIntervalMs);\n    }\n\n    public FlushManager getFlushManager() {\n        return flushManager;\n    }\n\n    private boolean isCloseReadAhead() {\n        return !MixAll.isWindows() && !defaultMessageStore.getMessageStoreConfig().isDataReadAheadEnable();\n    }\n\n    public class ColdDataCheckService extends ServiceThread {\n        private final SystemClock systemClock = new SystemClock();\n        private final ConcurrentHashMap<String, byte[]> pageCacheMap = new ConcurrentHashMap<>();\n        private int pageSize = -1;\n        private int sampleSteps = 32;\n\n        public ColdDataCheckService() {\n            sampleSteps = defaultMessageStore.getMessageStoreConfig().getSampleSteps();\n            if (sampleSteps <= 0) {\n                sampleSteps = 32;\n            }\n            initPageSize();\n            scanFilesInPageCache();\n        }\n\n        @Override\n        public String getServiceName() {\n            return ColdDataCheckService.class.getSimpleName();\n        }\n\n        @Override\n        public void run() {\n            log.info(\"{} service started\", this.getServiceName());\n            while (!this.isStopped()) {\n                try {\n                    if (MixAll.isWindows() || !defaultMessageStore.getMessageStoreConfig().isColdDataFlowControlEnable() || !defaultMessageStore.getMessageStoreConfig().isColdDataScanEnable()) {\n                        pageCacheMap.clear();\n                        this.waitForRunning(180 * 1000);\n                        continue;\n                    } else {\n                        this.waitForRunning(defaultMessageStore.getMessageStoreConfig().getTimerColdDataCheckIntervalMs());\n                    }\n\n                    if (pageSize < 0) {\n                        initPageSize();\n                    }\n\n                    long beginClockTimestamp = this.systemClock.now();\n                    scanFilesInPageCache();\n                    long costTime = this.systemClock.now() - beginClockTimestamp;\n                    log.info(\"[{}] scanFilesInPageCache-cost {} ms.\", costTime > 30 * 1000 ? \"NOTIFYME\" : \"OK\", costTime);\n                } catch (Throwable e) {\n                    log.warn(\"{} service has e: \", this.getServiceName(), e);\n                }\n            }\n            log.info(\"{} service end\", this.getServiceName());\n        }\n\n        public boolean isDataInPageCache(final long offset) {\n            if (!defaultMessageStore.getMessageStoreConfig().isColdDataFlowControlEnable()) {\n                return true;\n            }\n            if (pageSize <= 0 || sampleSteps <= 0) {\n                return true;\n            }\n            if (!defaultMessageStore.checkInColdAreaByCommitOffset(offset, getMaxOffset())) {\n                return true;\n            }\n            if (!defaultMessageStore.getMessageStoreConfig().isColdDataScanEnable()) {\n                return false;\n            }\n\n            MappedFile mappedFile = mappedFileQueue.findMappedFileByOffset(offset, offset == 0);\n            if (null == mappedFile) {\n                return true;\n            }\n            byte[] bytes = pageCacheMap.get(mappedFile.getFileName());\n            if (null == bytes) {\n                return true;\n            }\n\n            int pos = (int) (offset % defaultMessageStore.getMessageStoreConfig().getMappedFileSizeCommitLog());\n            int realIndex = pos / pageSize / sampleSteps;\n            return bytes.length - 1 >= realIndex && bytes[realIndex] != 0;\n        }\n\n        private void scanFilesInPageCache() {\n            if (MixAll.isWindows() || !defaultMessageStore.getMessageStoreConfig().isColdDataFlowControlEnable() || !defaultMessageStore.getMessageStoreConfig().isColdDataScanEnable() || pageSize <= 0) {\n                return;\n            }\n            try {\n                log.info(\"pageCacheMap key size: {}\", pageCacheMap.size());\n                clearExpireMappedFile();\n                mappedFileQueue.getMappedFiles().forEach(mappedFile -> {\n                    byte[] pageCacheTable = checkFileInPageCache(mappedFile);\n                    if (sampleSteps > 1) {\n                        pageCacheTable = sampling(pageCacheTable, sampleSteps);\n                    }\n                    pageCacheMap.put(mappedFile.getFileName(), pageCacheTable);\n                });\n            } catch (Exception e) {\n                log.error(\"scanFilesInPageCache exception\", e);\n            }\n        }\n\n        private void clearExpireMappedFile() {\n            Set<String> currentFileSet = mappedFileQueue.getMappedFiles().stream().map(MappedFile::getFileName).collect(Collectors.toSet());\n            pageCacheMap.forEach((key, value) -> {\n                if (!currentFileSet.contains(key)) {\n                    pageCacheMap.remove(key);\n                    log.info(\"clearExpireMappedFile fileName: {}, has been clear\", key);\n                }\n            });\n        }\n\n        private byte[] sampling(byte[] pageCacheTable, int sampleStep) {\n            byte[] sample = new byte[(pageCacheTable.length + sampleStep - 1) / sampleStep];\n            for (int i = 0, j = 0; i < pageCacheTable.length && j < sample.length; i += sampleStep) {\n                sample[j++] = pageCacheTable[i];\n            }\n            return sample;\n        }\n\n        private byte[] checkFileInPageCache(MappedFile mappedFile) {\n            long fileSize = mappedFile.getFileSize();\n            final long address = PlatformDependent.directBufferAddress(mappedFile.getMappedByteBuffer());\n            int pageNums = (int) (fileSize + this.pageSize - 1) / this.pageSize;\n            byte[] pageCacheRst = new byte[pageNums];\n            int mincore = LibC.INSTANCE.mincore(new Pointer(address), new NativeLong(fileSize), pageCacheRst);\n            if (mincore != 0) {\n                log.error(\"checkFileInPageCache call the LibC.INSTANCE.mincore error, fileName: {}, fileSize: {}\",\n                    mappedFile.getFileName(), fileSize);\n                for (int i = 0; i < pageNums; i++) {\n                    pageCacheRst[i] = 1;\n                }\n            }\n            return pageCacheRst;\n        }\n\n        private void initPageSize() {\n            if (pageSize < 0 && defaultMessageStore.getMessageStoreConfig().isColdDataFlowControlEnable()) {\n                try {\n                    if (!MixAll.isWindows()) {\n                        pageSize = LibC.INSTANCE.getpagesize();\n                    } else {\n                        defaultMessageStore.getMessageStoreConfig().setColdDataFlowControlEnable(false);\n                        log.info(\"windows os, coldDataCheckEnable force setting to be false\");\n                    }\n                    log.info(\"initPageSize pageSize: {}\", pageSize);\n                } catch (Exception e) {\n                    defaultMessageStore.getMessageStoreConfig().setColdDataFlowControlEnable(false);\n                    log.error(\"initPageSize error, coldDataCheckEnable force setting to be false \", e);\n                }\n            }\n        }\n\n        /**\n         * this result is not high accurate.\n         */\n        public boolean isMsgInColdArea(String group, String topic, int queueId, long offset) {\n            if (!defaultMessageStore.getMessageStoreConfig().isColdDataFlowControlEnable()) {\n                return false;\n            }\n            try {\n                ConsumeQueueInterface consumeQueue = defaultMessageStore.findConsumeQueue(topic, queueId);\n                if (null == consumeQueue) {\n                    return false;\n                }\n                CqUnit cqUnit = consumeQueue.get(offset);\n                if (null == cqUnit) {\n                    return false;\n                }\n                long offsetPy = cqUnit.getPos();\n                if (offsetPy < 0L) {\n                    return false;\n                }\n                return defaultMessageStore.checkInColdAreaByCommitOffset(offsetPy, getMaxOffset());\n            } catch (Exception e) {\n                log.error(\"isMsgInColdArea group: {}, topic: {}, queueId: {}, offset: {}\", group, topic, queueId, offset, e);\n            }\n            return false;\n        }\n    }\n\n    public void scanFileAndSetReadMode(int mode) {\n        if (MixAll.isWindows()) {\n            log.info(\"windows os stop scanFileAndSetReadMode\");\n            return;\n        }\n        try {\n            log.info(\"scanFileAndSetReadMode mode: {}\", mode);\n            mappedFileQueue.getMappedFiles().forEach(mappedFile -> {\n                setFileReadMode(mappedFile, mode);\n            });\n        } catch (Exception e) {\n            log.error(\"scanFileAndSetReadMode exception\", e);\n        }\n    }\n\n    private int setFileReadMode(MappedFile mappedFile, int mode) {\n        if (null == mappedFile) {\n            log.error(\"setFileReadMode mappedFile is null\");\n            return -1;\n        }\n        final long address = PlatformDependent.directBufferAddress(mappedFile.getMappedByteBuffer());\n        int madvise = LibC.INSTANCE.madvise(new Pointer(address), new NativeLong(mappedFile.getFileSize()), mode);\n        if (madvise != 0) {\n            log.error(\"setFileReadMode error fileName: {}, madvise: {}, mode:{}\", mappedFile.getFileName(), madvise, mode);\n        }\n        return madvise;\n    }\n\n    public ColdDataCheckService getColdDataCheckService() {\n        return coldDataCheckService;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/CommitLogDispatchStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport org.rocksdb.RocksDBException;\n\n/**\n * Interface for stores that require commitlog dispatch and recovery. Each store implementing this interface should\n * register itself in the commitlog when loading. This abstraction allows the commitlog recovery process to\n * automatically consider all registered stores without needing to modify the recovery logic when adding a new store.\n */\npublic interface CommitLogDispatchStore {\n\n    /**\n     * Get the dispatch offset in the store. Messages whose phyOffset larger than this offset need to be dispatched. The\n     * dispatch offset is only used during recovery.\n     *\n     * @param recoverNormally true if broker exited normally last time (normal recovery), false for abnormal recovery\n     * @return the dispatch phyOffset, or null if the store is not enabled or has no valid offset\n     * @throws RocksDBException if there is an error accessing RocksDB storage\n     */\n    Long getDispatchFromPhyOffset(boolean recoverNormally) throws RocksDBException;\n\n    /**\n     * Used to determine whether to start doDispatch from this commitLog mappedFile.\n     *\n     * @param phyOffset the offset of the first message in this commitlog mappedFile\n     * @param storeTimestamp the timestamp of the first message in this commitlog mappedFile\n     * @param recoverNormally whether this is a normal recovery\n     * @return whether to start recovering from this MappedFile\n     * @throws RocksDBException if there is an error accessing RocksDB storage\n     */\n    boolean isMappedFileMatchedRecover(long phyOffset, long storeTimestamp,\n        boolean recoverNormally) throws RocksDBException;\n}\n\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport org.rocksdb.RocksDBException;\n\n/**\n * Dispatcher of commit log.\n */\npublic interface CommitLogDispatcher {\n\n    /**\n     *  Dispatch messages from store to build consume queues, indexes, and filter data\n     * @param request dispatch message request\n     * @throws RocksDBException only in rocksdb mode\n     */\n    void dispatch(final DispatchRequest request) throws RocksDBException;\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/CompactionAppendMsgCallback.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport java.nio.ByteBuffer;\n\npublic interface CompactionAppendMsgCallback {\n    AppendMessageResult doAppend(ByteBuffer bbDest, long fileFromOffset, int maxBlank, ByteBuffer bbSrc);\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport java.io.File;\nimport java.nio.ByteBuffer;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Map;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.config.StorePathConfigHelper;\nimport org.apache.rocketmq.store.logfile.MappedFile;\nimport org.apache.rocketmq.store.queue.ConsumeQueueInterface;\nimport org.apache.rocketmq.store.queue.ConsumeQueueStore;\nimport org.apache.rocketmq.store.queue.CqUnit;\nimport org.apache.rocketmq.store.queue.MultiDispatchUtils;\nimport org.apache.rocketmq.store.queue.QueueOffsetOperator;\nimport org.apache.rocketmq.store.queue.ReferredIterator;\n\npublic class ConsumeQueue implements ConsumeQueueInterface {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n\n    /**\n     * ConsumeQueue's store unit. Format:\n     * <pre>\n     * ┌───────────────────────────────┬───────────────────┬───────────────────────────────┐\n     * │    CommitLog Physical Offset  │      Body Size    │            Tag HashCode       │\n     * │          (8 Bytes)            │      (4 Bytes)    │             (8 Bytes)         │\n     * ├───────────────────────────────┴───────────────────┴───────────────────────────────┤\n     * │                                     Store Unit                                    │\n     * │                                                                                   │\n     * </pre>\n     * ConsumeQueue's store unit. Size: CommitLog Physical Offset(8) + Body Size(4) + Tag HashCode(8) = 20 Bytes\n     */\n    public static final int CQ_STORE_UNIT_SIZE = 20;\n    public static final int MSG_TAG_OFFSET_INDEX = 12;\n    private static final Logger LOG_ERROR = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);\n\n    private final MessageStore messageStore;\n    private final ConsumeQueueStore consumeQueueStore;\n\n    private final MappedFileQueue mappedFileQueue;\n    private final String topic;\n    private final int queueId;\n    private final ByteBuffer byteBufferIndex;\n\n    private final String storePath;\n    private final int mappedFileSize;\n    private long maxPhysicOffset = -1;\n\n    /**\n     * Minimum offset of the consume file queue that points to valid commit log record.\n     */\n    private volatile long minLogicOffset = 0;\n    private ConsumeQueueExt consumeQueueExt = null;\n\n    public ConsumeQueue(final String topic, final int queueId, final String storePath, final int mappedFileSize,\n        final MessageStore messageStore) {\n        this(topic, queueId, storePath, mappedFileSize, messageStore, (ConsumeQueueStore) messageStore.getQueueStore());\n    }\n\n    public ConsumeQueue(final String topic, final int queueId, final String storePath, final int mappedFileSize,\n        final MessageStore messageStore, final ConsumeQueueStore consumeQueueStore) {\n        this.storePath = storePath;\n        this.mappedFileSize = mappedFileSize;\n        this.messageStore = messageStore;\n        this.consumeQueueStore = consumeQueueStore;\n\n        this.topic = topic;\n        this.queueId = queueId;\n\n        String queueDir = this.storePath\n            + File.separator + topic\n            + File.separator + queueId;\n\n        boolean writeWithoutMmap = false;\n        if (messageStore.getMessageStoreConfig() != null) {\n            writeWithoutMmap = messageStore.getMessageStoreConfig().isWriteWithoutMmap();\n        }\n\n        this.mappedFileQueue = new MappedFileQueue(queueDir, mappedFileSize, null, writeWithoutMmap);\n\n        this.byteBufferIndex = ByteBuffer.allocate(CQ_STORE_UNIT_SIZE);\n\n        if (messageStore.getMessageStoreConfig().isEnableConsumeQueueExt()) {\n            this.consumeQueueExt = new ConsumeQueueExt(\n                topic,\n                queueId,\n                StorePathConfigHelper.getStorePathConsumeQueueExt(messageStore.getMessageStoreConfig().getStorePathRootDir()),\n                messageStore.getMessageStoreConfig().getMappedFileSizeConsumeQueueExt(),\n                messageStore.getMessageStoreConfig().getBitMapLengthConsumeQueueExt(),\n                writeWithoutMmap\n            );\n        }\n    }\n\n    @Override\n    public boolean load() {\n        boolean result = this.mappedFileQueue.load();\n        log.info(\"load consume queue \" + this.topic + \"-\" + this.queueId + \" \" + (result ? \"OK\" : \"Failed\"));\n        if (isExtReadEnable()) {\n            result &= this.consumeQueueExt.load();\n        }\n        return result;\n    }\n\n    @Override\n    public void recover() {\n        final List<MappedFile> mappedFiles = this.mappedFileQueue.getMappedFiles();\n        if (!mappedFiles.isEmpty()) {\n\n            int index = mappedFiles.size() - 3;\n            if (index < 0) {\n                index = 0;\n            }\n\n            int mappedFileSizeLogics = this.mappedFileSize;\n            MappedFile mappedFile = mappedFiles.get(index);\n            ByteBuffer byteBuffer = mappedFile.sliceByteBuffer();\n            long processOffset = mappedFile.getFileFromOffset();\n            long mappedFileOffset = 0;\n            long maxExtAddr = 1;\n            while (true) {\n                for (int i = 0; i < mappedFileSizeLogics; i += CQ_STORE_UNIT_SIZE) {\n                    long offset = byteBuffer.getLong();\n                    int size = byteBuffer.getInt();\n                    long tagsCode = byteBuffer.getLong();\n\n                    if (offset >= 0 && size > 0) {\n                        mappedFileOffset = i + CQ_STORE_UNIT_SIZE;\n                        this.setMaxPhysicOffset(offset + size);\n                        if (isExtAddr(tagsCode)) {\n                            maxExtAddr = tagsCode;\n                        }\n                    } else {\n                        log.info(\"recover current consume queue file over,  \" + mappedFile.getFileName() + \" \"\n                            + offset + \" \" + size + \" \" + tagsCode);\n                        break;\n                    }\n                }\n\n                if (mappedFileOffset == mappedFileSizeLogics) {\n                    index++;\n                    if (index >= mappedFiles.size()) {\n\n                        log.info(\"recover last consume queue file over, last mapped file \"\n                            + mappedFile.getFileName());\n                        break;\n                    } else {\n                        mappedFile = mappedFiles.get(index);\n                        byteBuffer = mappedFile.sliceByteBuffer();\n                        processOffset = mappedFile.getFileFromOffset();\n                        mappedFileOffset = 0;\n                        log.info(\"recover next consume queue file, \" + mappedFile.getFileName());\n                    }\n                } else {\n                    log.info(\"recover current consume queue over \" + mappedFile.getFileName() + \" \"\n                        + (processOffset + mappedFileOffset));\n                    break;\n                }\n            }\n\n            processOffset += mappedFileOffset;\n            this.mappedFileQueue.setFlushedWhere(processOffset);\n            this.mappedFileQueue.setCommittedWhere(processOffset);\n            this.mappedFileQueue.truncateDirtyFiles(processOffset);\n\n            if (isExtReadEnable()) {\n                this.consumeQueueExt.recover();\n                log.info(\"Truncate consume queue extend file by max {}\", maxExtAddr);\n                this.consumeQueueExt.truncateByMaxAddress(maxExtAddr);\n            }\n        }\n    }\n\n    @Override\n    public long getTotalSize() {\n        long totalSize = this.mappedFileQueue.getTotalFileSize();\n        if (isExtReadEnable()) {\n            totalSize += this.consumeQueueExt.getTotalSize();\n        }\n        return totalSize;\n    }\n\n    @Override\n    public int getUnitSize() {\n        return CQ_STORE_UNIT_SIZE;\n    }\n\n    @Deprecated\n    @Override\n    public long getOffsetInQueueByTime(final long timestamp) {\n        MappedFile mappedFile = this.mappedFileQueue.getConsumeQueueMappedFileByTime(timestamp,\n            messageStore.getCommitLog(), BoundaryType.LOWER);\n        return binarySearchInQueueByTime(mappedFile, timestamp, BoundaryType.LOWER);\n    }\n\n    @Override\n    public long getOffsetInQueueByTime(final long timestamp, final BoundaryType boundaryType) {\n        MappedFile mappedFile = this.mappedFileQueue.getConsumeQueueMappedFileByTime(timestamp,\n            messageStore.getCommitLog(), boundaryType);\n        return binarySearchInQueueByTime(mappedFile, timestamp, boundaryType);\n    }\n\n    private long binarySearchInQueueByTime(final MappedFile mappedFile, final long timestamp,\n        BoundaryType boundaryType) {\n        if (mappedFile != null) {\n            long offset = 0;\n            int low = minLogicOffset > mappedFile.getFileFromOffset() ? (int) (minLogicOffset - mappedFile.getFileFromOffset()) : 0;\n            int high = 0;\n            int midOffset = -1, targetOffset = -1, leftOffset = -1, rightOffset = -1;\n            long minPhysicOffset = this.messageStore.getMinPhyOffset();\n            int range = mappedFile.getFileSize();\n            if (mappedFile.getWrotePosition() != 0 && mappedFile.getWrotePosition() != mappedFile.getFileSize()) {\n                // mappedFile is the last one and is currently being written.\n                range = mappedFile.getWrotePosition();\n            }\n            SelectMappedBufferResult sbr = mappedFile.selectMappedBuffer(0, range);\n            if (null != sbr) {\n                ByteBuffer byteBuffer = sbr.getByteBuffer();\n                int ceiling = byteBuffer.limit() - CQ_STORE_UNIT_SIZE;\n                int floor = low;\n                high = ceiling;\n                try {\n                    // Handle the following corner cases first:\n                    // 1. store time of (high) < timestamp\n                    // 2. store time of (low) > timestamp\n                    long storeTime;\n                    long phyOffset;\n                    int size;\n                    // Handle case 1\n                    byteBuffer.position(ceiling);\n                    phyOffset = byteBuffer.getLong();\n                    size = byteBuffer.getInt();\n                    storeTime = messageStore.getCommitLog().pickupStoreTimestamp(phyOffset, size);\n                    if (storeTime < timestamp) {\n                        switch (boundaryType) {\n                            case LOWER:\n                                return (mappedFile.getFileFromOffset() + ceiling + CQ_STORE_UNIT_SIZE) / CQ_STORE_UNIT_SIZE;\n                            case UPPER:\n                                return (mappedFile.getFileFromOffset() + ceiling) / CQ_STORE_UNIT_SIZE;\n                            default:\n                                log.warn(\"Unknown boundary type\");\n                                break;\n                        }\n                    }\n\n                    // Handle case 2\n                    byteBuffer.position(floor);\n                    phyOffset = byteBuffer.getLong();\n                    size = byteBuffer.getInt();\n                    storeTime = messageStore.getCommitLog().pickupStoreTimestamp(phyOffset, size);\n                    if (storeTime > timestamp) {\n                        switch (boundaryType) {\n                            case LOWER:\n                                return mappedFile.getFileFromOffset() / CQ_STORE_UNIT_SIZE;\n                            case UPPER:\n                                return 0;\n                            default:\n                                log.warn(\"Unknown boundary type\");\n                                break;\n                        }\n                    }\n\n                    // Perform binary search\n                    while (high >= low) {\n                        midOffset = (low + high) / (2 * CQ_STORE_UNIT_SIZE) * CQ_STORE_UNIT_SIZE;\n                        byteBuffer.position(midOffset);\n                        phyOffset = byteBuffer.getLong();\n                        size = byteBuffer.getInt();\n                        if (phyOffset < minPhysicOffset) {\n                            low = midOffset + CQ_STORE_UNIT_SIZE;\n                            leftOffset = midOffset;\n                            continue;\n                        }\n\n                        storeTime = this.messageStore.getCommitLog().pickupStoreTimestamp(phyOffset, size);\n                        if (storeTime < 0) {\n                            log.warn(\"Failed to query store timestamp for commit log offset: {}\", phyOffset);\n                            return 0;\n                        } else if (storeTime == timestamp) {\n                            targetOffset = midOffset;\n                            break;\n                        } else if (storeTime > timestamp) {\n                            high = midOffset - CQ_STORE_UNIT_SIZE;\n                            rightOffset = midOffset;\n                        } else {\n                            low = midOffset + CQ_STORE_UNIT_SIZE;\n                            leftOffset = midOffset;\n                        }\n                    }\n\n                    if (targetOffset != -1) {\n                        // We just found ONE matched record. These next to it might also share the same store-timestamp.\n                        offset = targetOffset;\n                        switch (boundaryType) {\n                            case LOWER: {\n                                int previousAttempt = targetOffset;\n                                while (true) {\n                                    int attempt = previousAttempt - CQ_STORE_UNIT_SIZE;\n                                    if (attempt < floor) {\n                                        break;\n                                    }\n                                    byteBuffer.position(attempt);\n                                    long physicalOffset = byteBuffer.getLong();\n                                    int messageSize = byteBuffer.getInt();\n                                    long messageStoreTimestamp = messageStore.getCommitLog()\n                                        .pickupStoreTimestamp(physicalOffset, messageSize);\n                                    if (messageStoreTimestamp == timestamp) {\n                                        previousAttempt = attempt;\n                                        continue;\n                                    }\n                                    break;\n                                }\n                                offset = previousAttempt;\n                                break;\n                            }\n                            case UPPER: {\n                                int previousAttempt = targetOffset;\n                                while (true) {\n                                    int attempt = previousAttempt + CQ_STORE_UNIT_SIZE;\n                                    if (attempt > ceiling) {\n                                        break;\n                                    }\n                                    byteBuffer.position(attempt);\n                                    long physicalOffset = byteBuffer.getLong();\n                                    int messageSize = byteBuffer.getInt();\n                                    long messageStoreTimestamp = messageStore.getCommitLog()\n                                        .pickupStoreTimestamp(physicalOffset, messageSize);\n                                    if (messageStoreTimestamp == timestamp) {\n                                        previousAttempt = attempt;\n                                        continue;\n                                    }\n                                    break;\n                                }\n                                offset = previousAttempt;\n                                break;\n                            }\n                            default: {\n                                log.warn(\"Unknown boundary type\");\n                                break;\n                            }\n                        }\n                    } else {\n                        // Given timestamp does not have any message records. But we have a range enclosing the\n                        // timestamp.\n                        /*\n                         * Consider the follow case: t2 has no consume queue entry and we are searching offset of\n                         * t2 for lower and upper boundaries.\n                         *  --------------------------\n                         *   timestamp   Consume Queue\n                         *       t1          1\n                         *       t1          2\n                         *       t1          3\n                         *       t3          4\n                         *       t3          5\n                         *   --------------------------\n                         * Now, we return 3 as upper boundary of t2 and 4 as its lower boundary. It looks\n                         * contradictory at first sight, but it does make sense when performing range queries.\n                         */\n                        switch (boundaryType) {\n                            case LOWER: {\n                                offset = rightOffset;\n                                break;\n                            }\n\n                            case UPPER: {\n                                offset = leftOffset;\n                                break;\n                            }\n                            default: {\n                                log.warn(\"Unknown boundary type\");\n                                break;\n                            }\n                        }\n                    }\n                    return (mappedFile.getFileFromOffset() + offset) / CQ_STORE_UNIT_SIZE;\n                } finally {\n                    sbr.release();\n                }\n            }\n        }\n        return 0;\n    }\n\n    @Override\n    public void truncateDirtyLogicFiles(long phyOffset) {\n        truncateDirtyLogicFiles(phyOffset, true);\n    }\n\n    public void truncateDirtyLogicFiles(long phyOffset, boolean deleteFile) {\n\n        int logicFileSize = this.mappedFileSize;\n\n        this.setMaxPhysicOffset(phyOffset);\n        long maxExtAddr = 1;\n        boolean shouldDeleteFile = false;\n        while (true) {\n            MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();\n            if (mappedFile != null) {\n                ByteBuffer byteBuffer = mappedFile.sliceByteBuffer();\n\n                mappedFile.setWrotePosition(0);\n                mappedFile.setCommittedPosition(0);\n                mappedFile.setFlushedPosition(0);\n\n                for (int i = 0; i < logicFileSize; i += CQ_STORE_UNIT_SIZE) {\n                    long offset = byteBuffer.getLong();\n                    int size = byteBuffer.getInt();\n                    long tagsCode = byteBuffer.getLong();\n\n                    if (0 == i) {\n                        if (offset >= phyOffset) {\n                            shouldDeleteFile = true;\n                            break;\n                        } else {\n                            int pos = i + CQ_STORE_UNIT_SIZE;\n                            mappedFile.setWrotePosition(pos);\n                            mappedFile.setCommittedPosition(pos);\n                            mappedFile.setFlushedPosition(pos);\n                            this.setMaxPhysicOffset(offset + size);\n                            // This maybe not take effect, when not every consume queue has extend file.\n                            if (isExtAddr(tagsCode)) {\n                                maxExtAddr = tagsCode;\n                            }\n                        }\n                    } else {\n\n                        if (offset >= 0 && size > 0) {\n\n                            if (offset >= phyOffset) {\n                                return;\n                            }\n\n                            int pos = i + CQ_STORE_UNIT_SIZE;\n                            mappedFile.setWrotePosition(pos);\n                            mappedFile.setCommittedPosition(pos);\n                            mappedFile.setFlushedPosition(pos);\n                            this.setMaxPhysicOffset(offset + size);\n                            if (isExtAddr(tagsCode)) {\n                                maxExtAddr = tagsCode;\n                            }\n\n                            if (pos == logicFileSize) {\n                                return;\n                            }\n                        } else {\n                            return;\n                        }\n                    }\n                }\n\n                if (shouldDeleteFile) {\n                    if (deleteFile) {\n                        this.mappedFileQueue.deleteLastMappedFile();\n                    } else {\n                        this.mappedFileQueue.deleteExpiredFile(Collections.singletonList(this.mappedFileQueue.getLastMappedFile()));\n                    }\n                }\n\n            } else {\n                break;\n            }\n        }\n\n        if (isExtReadEnable()) {\n            this.consumeQueueExt.truncateByMaxAddress(maxExtAddr);\n        }\n    }\n\n    @Override\n    public long getLastOffset() {\n        long lastOffset = -1;\n\n        int logicFileSize = this.mappedFileSize;\n\n        MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();\n        if (mappedFile != null) {\n\n            int position = mappedFile.getWrotePosition() - CQ_STORE_UNIT_SIZE;\n            if (position < 0)\n                position = 0;\n\n            ByteBuffer byteBuffer = mappedFile.sliceByteBuffer();\n            byteBuffer.position(position);\n            for (int i = 0; i < logicFileSize; i += CQ_STORE_UNIT_SIZE) {\n                long offset = byteBuffer.getLong();\n                int size = byteBuffer.getInt();\n                byteBuffer.getLong();\n\n                if (offset >= 0 && size > 0) {\n                    lastOffset = offset + size;\n                } else {\n                    break;\n                }\n            }\n        }\n\n        return lastOffset;\n    }\n\n    @Override\n    public boolean flush(final int flushLeastPages) {\n        boolean result = this.mappedFileQueue.flush(flushLeastPages);\n        if (isExtReadEnable()) {\n            result = result & this.consumeQueueExt.flush(flushLeastPages);\n        }\n\n        return result;\n    }\n\n    @Override\n    public int deleteExpiredFile(long offset) {\n        int cnt = this.mappedFileQueue.deleteExpiredFileByOffset(offset, CQ_STORE_UNIT_SIZE);\n        this.correctMinOffset(offset);\n        return cnt;\n    }\n\n    /**\n     * Update minLogicOffset such that entries after it would point to valid commit log address.\n     *\n     * @param minCommitLogOffset Minimum commit log offset\n     */\n    @Override\n    public void correctMinOffset(long minCommitLogOffset) {\n        // Check if the consume queue is the state of deprecation.\n        if (minLogicOffset >= mappedFileQueue.getMaxOffset()) {\n            log.info(\"ConsumeQueue[Topic={}, queue-id={}] contains no valid entries\", topic, queueId);\n            return;\n        }\n\n        // Check whether the consume queue maps no valid data at all. This check may cost 1 IO operation.\n        // The rationale is that consume queue always preserves the last file. In case there are many deprecated topics,\n        // This check would save a lot of efforts.\n        MappedFile lastMappedFile = this.mappedFileQueue.getLastMappedFile();\n        if (null == lastMappedFile) {\n            return;\n        }\n\n        SelectMappedBufferResult lastRecord = null;\n        try {\n            int maxReadablePosition = lastMappedFile.getReadPosition();\n            if (maxReadablePosition >= ConsumeQueue.CQ_STORE_UNIT_SIZE) {\n                lastRecord = lastMappedFile.selectMappedBuffer(maxReadablePosition - ConsumeQueue.CQ_STORE_UNIT_SIZE,\n                    ConsumeQueue.CQ_STORE_UNIT_SIZE);\n            }\n            if (null != lastRecord) {\n                ByteBuffer buffer = lastRecord.getByteBuffer();\n                long commitLogOffset = buffer.getLong();\n                if (commitLogOffset < minCommitLogOffset) {\n                    // Keep the largest known consume offset, even if this consume-queue contains no valid entries at\n                    // all. Let minLogicOffset point to a future slot.\n                    this.minLogicOffset = lastMappedFile.getFileFromOffset() + maxReadablePosition;\n                    log.info(\"ConsumeQueue[topic={}, queue-id={}] contains no valid entries. Min-offset is assigned as: {}.\",\n                        topic, queueId, getMinOffsetInQueue());\n                    return;\n                }\n            }\n        } finally {\n            if (null != lastRecord) {\n                lastRecord.release();\n            }\n        }\n\n        MappedFile mappedFile = this.mappedFileQueue.getFirstMappedFile();\n        long minExtAddr = 1;\n        if (mappedFile != null) {\n            // Search from previous min logical offset. Typically, a consume queue file segment contains 300,000 entries\n            // searching from previous position saves significant amount of comparisons and IOs\n            boolean intact = true; // Assume previous value is still valid\n            long start = this.minLogicOffset - mappedFile.getFileFromOffset();\n            if (start < 0) {\n                intact = false;\n                start = 0;\n            }\n\n            if (start > mappedFile.getReadPosition()) {\n                log.error(\"[Bug][InconsistentState] ConsumeQueue file {} should have been deleted\",\n                    mappedFile.getFileName());\n                return;\n            }\n\n            SelectMappedBufferResult result = mappedFile.selectMappedBuffer((int) start);\n            if (result == null) {\n                log.warn(\"[Bug] Failed to scan consume queue entries from file on correcting min offset: {}\",\n                    mappedFile.getFileName());\n                return;\n            }\n\n            try {\n                // No valid consume entries\n                if (result.getSize() == 0) {\n                    log.debug(\"ConsumeQueue[topic={}, queue-id={}] contains no valid entries\", topic, queueId);\n                    return;\n                }\n\n                ByteBuffer buffer = result.getByteBuffer().slice();\n                // Verify whether the previous value is still valid or not before conducting binary search\n                long commitLogOffset = buffer.getLong();\n                if (intact && commitLogOffset >= minCommitLogOffset) {\n                    log.info(\"Abort correction as previous min-offset points to {}, which is greater than {}\",\n                        commitLogOffset, minCommitLogOffset);\n                    return;\n                }\n\n                // Binary search between range [previous_min_logic_offset, first_file_from_offset + file_size)\n                // Note the consume-queue deletion procedure ensures the last entry points to somewhere valid.\n                int low = 0;\n                int high = result.getSize() - ConsumeQueue.CQ_STORE_UNIT_SIZE;\n                while (true) {\n                    if (high - low <= ConsumeQueue.CQ_STORE_UNIT_SIZE) {\n                        break;\n                    }\n                    int mid = (low + high) / 2 / ConsumeQueue.CQ_STORE_UNIT_SIZE * ConsumeQueue.CQ_STORE_UNIT_SIZE;\n                    buffer.position(mid);\n                    commitLogOffset = buffer.getLong();\n                    if (commitLogOffset > minCommitLogOffset) {\n                        high = mid;\n                    } else if (commitLogOffset == minCommitLogOffset) {\n                        low = mid;\n                        high = mid;\n                        break;\n                    } else {\n                        low = mid;\n                    }\n                }\n\n                // Examine the last one or two entries\n                for (int i = low; i <= high; i += ConsumeQueue.CQ_STORE_UNIT_SIZE) {\n                    buffer.position(i);\n                    long offsetPy = buffer.getLong();\n                    buffer.position(i + 12);\n                    long tagsCode = buffer.getLong();\n\n                    if (offsetPy >= minCommitLogOffset) {\n                        this.minLogicOffset = mappedFile.getFileFromOffset() + start + i;\n                        log.info(\"Compute logical min offset: {}, topic: {}, queueId: {}\",\n                            this.getMinOffsetInQueue(), this.topic, this.queueId);\n                        // This maybe not take effect, when not every consume queue has an extended file.\n                        if (isExtAddr(tagsCode)) {\n                            minExtAddr = tagsCode;\n                        }\n                        break;\n                    }\n                }\n            } catch (Exception e) {\n                log.error(\"Exception thrown when correctMinOffset\", e);\n            } finally {\n                result.release();\n            }\n        }\n\n        if (isExtReadEnable()) {\n            this.consumeQueueExt.truncateByMinAddress(minExtAddr);\n        }\n    }\n\n    @Override\n    public long getMinOffsetInQueue() {\n        return this.minLogicOffset / CQ_STORE_UNIT_SIZE;\n    }\n\n    @Override\n    public void putMessagePositionInfoWrapper(DispatchRequest request) {\n        final int maxRetries = 30;\n        boolean canWrite = this.messageStore.getRunningFlags().isCQWriteable();\n        for (int i = 0; i < maxRetries && canWrite; i++) {\n            long tagsCode = request.getTagsCode();\n            if (isExtWriteEnable()) {\n                ConsumeQueueExt.CqExtUnit cqExtUnit = new ConsumeQueueExt.CqExtUnit();\n                cqExtUnit.setFilterBitMap(request.getBitMap());\n                cqExtUnit.setMsgStoreTime(request.getStoreTimestamp());\n                cqExtUnit.setTagsCode(request.getTagsCode());\n\n                long extAddr = this.consumeQueueExt.put(cqExtUnit);\n                if (isExtAddr(extAddr)) {\n                    tagsCode = extAddr;\n                } else {\n                    log.warn(\"Save consume queue extend fail, So just save tagsCode! {}, topic:{}, queueId:{}, offset:{}\", cqExtUnit,\n                        topic, queueId, request.getCommitLogOffset());\n                }\n            }\n            boolean result = this.putMessagePositionInfo(request.getCommitLogOffset(),\n                request.getMsgSize(), tagsCode, request.getConsumeQueueOffset());\n            if (result) {\n                if (this.messageStore.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE ||\n                    this.messageStore.getMessageStoreConfig().isEnableDLegerCommitLog()) {\n                    this.messageStore.getStoreCheckpoint().setPhysicMsgTimestamp(request.getStoreTimestamp());\n                }\n                this.messageStore.getStoreCheckpoint().setTmpLogicsMsgTimestamp(request.getStoreTimestamp());\n                this.messageStore.getStoreCheckpoint().setTmpLogicsPhysicalOffset(request.getCommitLogOffset());\n                if (MultiDispatchUtils.checkMultiDispatchQueue(this.messageStore.getMessageStoreConfig(), request)) {\n                    multiDispatchLmqQueue(request, maxRetries);\n                }\n                return;\n            } else {\n                // XXX: warn and notify me\n                log.warn(\"[BUG]put commit log position info to \" + topic + \":\" + queueId + \" \" + request.getCommitLogOffset()\n                    + \" failed, retry \" + i + \" times\");\n\n                try {\n                    Thread.sleep(1000);\n                } catch (InterruptedException e) {\n                    log.warn(\"\", e);\n                }\n            }\n        }\n\n        // XXX: warn and notify me\n        log.error(\"[BUG]consume queue can not write, {} {}\", this.topic, this.queueId);\n        this.messageStore.getRunningFlags().makeLogicsQueueError();\n    }\n\n    private void multiDispatchLmqQueue(DispatchRequest request, int maxRetries) {\n        Map<String, String> prop = request.getPropertiesMap();\n        String multiDispatchQueue = prop.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);\n        String multiQueueOffset = prop.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET);\n        String[] queues = multiDispatchQueue.split(MixAll.LMQ_DISPATCH_SEPARATOR);\n        String[] queueOffsets = multiQueueOffset.split(MixAll.LMQ_DISPATCH_SEPARATOR);\n        if (queues.length != queueOffsets.length) {\n            log.error(\"[bug] queues.length!=queueOffsets.length \", request.getTopic());\n            return;\n        }\n        for (int i = 0; i < queues.length; i++) {\n            String queueName = queues[i];\n            if (StringUtils.contains(queueName, File.separator)) {\n                continue;\n            }\n            long queueOffset = Long.parseLong(queueOffsets[i]);\n            int queueId = request.getQueueId();\n            if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) {\n                queueId = 0;\n            }\n            doDispatchLmqQueue(request, maxRetries, queueName, queueOffset, queueId);\n        }\n    }\n\n    private void doDispatchLmqQueue(DispatchRequest request, int maxRetries, String queueName, long queueOffset,\n        int queueId) {\n        ConsumeQueueInterface cq = this.messageStore.findConsumeQueue(queueName, queueId);\n        boolean canWrite = this.messageStore.getRunningFlags().isCQWriteable();\n        for (int i = 0; i < maxRetries && canWrite; i++) {\n            boolean result = ((ConsumeQueue) cq).putMessagePositionInfo(request.getCommitLogOffset(), request.getMsgSize(),\n                request.getTagsCode(),\n                queueOffset);\n            if (result) {\n                break;\n            } else {\n                log.warn(\"[BUG]put commit log position info to \" + queueName + \":\" + queueId + \" \" + request.getCommitLogOffset()\n                    + \" failed, retry \" + i + \" times\");\n\n                try {\n                    Thread.sleep(1000);\n                } catch (InterruptedException e) {\n                    log.warn(\"\", e);\n                }\n            }\n        }\n    }\n\n    @Override\n    public void assignQueueOffset(QueueOffsetOperator queueOffsetOperator, MessageExtBrokerInner msg) {\n        String topicQueueKey = getTopic() + \"-\" + getQueueId();\n        long queueOffset = queueOffsetOperator.getQueueOffset(topicQueueKey);\n        msg.setQueueOffset(queueOffset);\n    }\n\n    @Override\n    public void increaseQueueOffset(QueueOffsetOperator queueOffsetOperator, MessageExtBrokerInner msg,\n        short messageNum) {\n        String topicQueueKey = getTopic() + \"-\" + getQueueId();\n        queueOffsetOperator.increaseQueueOffset(topicQueueKey, messageNum);\n    }\n\n    private boolean putMessagePositionInfo(final long offset, final int size, final long tagsCode,\n        final long cqOffset) {\n\n        if (offset + size <= this.getMaxPhysicOffset()) {\n            // During the recovery process after broker crashes, this logs will cause the scrolling of valid logs.\n            if (messageStore.getStateMachine().getCurrentState().isAfter(MessageStoreStateMachine.MessageStoreState.RECOVER_COMMITLOG_OK) ||\n                messageStore.getMessageStoreConfig().isEnableLogConsumeQueueRepeatedlyBuildWhenRecover()) {\n                log.warn(\"Maybe try to build consume queue repeatedly maxPhysicOffset={} phyOffset={}\",\n                    this.getMaxPhysicOffset(), offset);\n            }\n            return true;\n        }\n\n        this.byteBufferIndex.flip();\n        this.byteBufferIndex.limit(CQ_STORE_UNIT_SIZE);\n        this.byteBufferIndex.putLong(offset);\n        this.byteBufferIndex.putInt(size);\n        this.byteBufferIndex.putLong(tagsCode);\n\n        final long expectLogicOffset = cqOffset * CQ_STORE_UNIT_SIZE;\n\n        MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(expectLogicOffset);\n        if (mappedFile != null) {\n\n            if (mappedFile.isFirstCreateInQueue() && cqOffset != 0 && mappedFile.getWrotePosition() == 0) {\n                this.minLogicOffset = expectLogicOffset;\n                this.mappedFileQueue.setFlushedWhere(expectLogicOffset);\n                this.mappedFileQueue.setCommittedWhere(expectLogicOffset);\n                this.fillPreBlank(mappedFile, expectLogicOffset);\n                log.info(\"fill pre blank space \" + mappedFile.getFileName() + \" \" + expectLogicOffset + \" \"\n                    + mappedFile.getWrotePosition());\n            }\n\n            if (cqOffset != 0) {\n                long currentLogicOffset = mappedFile.getWrotePosition() + mappedFile.getFileFromOffset();\n\n                if (expectLogicOffset < currentLogicOffset) {\n                    log.warn(\"Build consume queue repeatedly, expectLogicOffset: {} currentLogicOffset: {} Topic: {} QID: {} Diff: {}\",\n                        expectLogicOffset, currentLogicOffset, this.topic, this.queueId, expectLogicOffset - currentLogicOffset);\n                    return true;\n                }\n\n                if (expectLogicOffset != currentLogicOffset) {\n                    LOG_ERROR.warn(\n                        \"[BUG]logic queue order maybe wrong, expectLogicOffset: {} currentLogicOffset: {} Topic: {} QID: {} Diff: {}\",\n                        expectLogicOffset,\n                        currentLogicOffset,\n                        this.topic,\n                        this.queueId,\n                        expectLogicOffset - currentLogicOffset\n                    );\n                }\n            }\n            this.setMaxPhysicOffset(offset + size);\n            boolean appendResult;\n            if (messageStore.getMessageStoreConfig().isPutConsumeQueueDataByFileChannel()) {\n                appendResult = mappedFile.appendMessageUsingFileChannel(this.byteBufferIndex.array());\n            } else {\n                appendResult = mappedFile.appendMessage(this.byteBufferIndex.array());\n            }\n            return appendResult;\n        }\n        return false;\n    }\n\n    private void fillPreBlank(final MappedFile mappedFile, final long untilWhere) {\n        ByteBuffer byteBuffer = ByteBuffer.allocate(CQ_STORE_UNIT_SIZE);\n        byteBuffer.putLong(0L);\n        byteBuffer.putInt(Integer.MAX_VALUE);\n        byteBuffer.putLong(0L);\n\n        int until = (int) (untilWhere % this.mappedFileQueue.getMappedFileSize());\n        for (int i = 0; i < until; i += CQ_STORE_UNIT_SIZE) {\n            if (messageStore.getMessageStoreConfig().isPutConsumeQueueDataByFileChannel()) {\n                mappedFile.appendMessageUsingFileChannel(byteBuffer.array());\n            } else {\n                mappedFile.appendMessage(byteBuffer.array());\n            }\n\n        }\n    }\n\n    public SelectMappedBufferResult getIndexBuffer(final long startIndex) {\n        int mappedFileSize = this.mappedFileSize;\n        long offset = startIndex * CQ_STORE_UNIT_SIZE;\n        if (offset >= this.getMinLogicOffset()) {\n            MappedFile mappedFile = this.mappedFileQueue.findMappedFileByOffset(offset);\n            if (mappedFile != null) {\n                return mappedFile.selectMappedBuffer((int) (offset % mappedFileSize));\n            }\n        }\n        return null;\n    }\n\n    @Override\n    public ReferredIterator<CqUnit> iterateFrom(long startOffset) {\n        SelectMappedBufferResult sbr = getIndexBuffer(startOffset);\n        if (sbr == null) {\n            return null;\n        }\n        return new ConsumeQueueIterator(sbr);\n    }\n\n    @Override\n    public ReferredIterator<CqUnit> iterateFrom(long startIndex, int count) {\n        return iterateFrom(startIndex);\n    }\n\n    @Override\n    public CqUnit get(long offset) {\n        ReferredIterator<CqUnit> it = iterateFrom(offset);\n        if (it == null) {\n            return null;\n        }\n        return it.nextAndRelease();\n    }\n\n    @Override\n    public Pair<CqUnit, Long> getCqUnitAndStoreTime(long index) {\n        CqUnit cqUnit = get(index);\n        Long messageStoreTime = this.consumeQueueStore.getStoreTime(cqUnit);\n        return new Pair<>(cqUnit, messageStoreTime);\n    }\n\n    @Override\n    public Pair<CqUnit, Long> getEarliestUnitAndStoreTime() {\n        CqUnit cqUnit = getEarliestUnit();\n        Long messageStoreTime = this.consumeQueueStore.getStoreTime(cqUnit);\n        return new Pair<>(cqUnit, messageStoreTime);\n    }\n\n    @Override\n    public CqUnit getEarliestUnit() {\n        /**\n         * here maybe should not return null\n         */\n        ReferredIterator<CqUnit> it = iterateFrom(minLogicOffset / CQ_STORE_UNIT_SIZE);\n        if (it == null) {\n            return null;\n        }\n        return it.nextAndRelease();\n    }\n\n    @Override\n    public CqUnit getLatestUnit() {\n        ReferredIterator<CqUnit> it = iterateFrom((mappedFileQueue.getMaxOffset() / CQ_STORE_UNIT_SIZE) - 1);\n        if (it == null) {\n            return null;\n        }\n        return it.nextAndRelease();\n    }\n\n    @Override\n    public boolean isFirstFileAvailable() {\n        return false;\n    }\n\n    @Override\n    public boolean isFirstFileExist() {\n        return false;\n    }\n\n    private class ConsumeQueueIterator implements ReferredIterator<CqUnit> {\n        private SelectMappedBufferResult sbr;\n        private int relativePos = 0;\n\n        public ConsumeQueueIterator(SelectMappedBufferResult sbr) {\n            this.sbr = sbr;\n            if (sbr != null && sbr.getByteBuffer() != null) {\n                relativePos = sbr.getByteBuffer().position();\n            }\n        }\n\n        @Override\n        public boolean hasNext() {\n            if (sbr == null || sbr.getByteBuffer() == null) {\n                return false;\n            }\n\n            return sbr.getByteBuffer().hasRemaining();\n        }\n\n        @Override\n        public CqUnit next() {\n            if (!hasNext()) {\n                return null;\n            }\n            long queueOffset = (sbr.getStartOffset() + sbr.getByteBuffer().position() - relativePos) / CQ_STORE_UNIT_SIZE;\n            CqUnit cqUnit = new CqUnit(queueOffset,\n                sbr.getByteBuffer().getLong(),\n                sbr.getByteBuffer().getInt(),\n                sbr.getByteBuffer().getLong());\n\n            if (isExtAddr(cqUnit.getTagsCode())) {\n                ConsumeQueueExt.CqExtUnit cqExtUnit = new ConsumeQueueExt.CqExtUnit();\n                boolean extRet = getExt(cqUnit.getTagsCode(), cqExtUnit);\n                if (extRet) {\n                    cqUnit.setTagsCode(cqExtUnit.getTagsCode());\n                    cqUnit.setCqExtUnit(cqExtUnit);\n                } else {\n                    // can't find ext content.Client will filter messages by tag also.\n                    log.error(\"[BUG] can't find consume queue extend file content! addr={}, offsetPy={}, sizePy={}, topic={}\",\n                        cqUnit.getTagsCode(), cqUnit.getPos(), cqUnit.getPos(), getTopic());\n                }\n            }\n            return cqUnit;\n        }\n\n        @Override\n        public void remove() {\n            throw new UnsupportedOperationException(\"remove\");\n        }\n\n        @Override\n        public void release() {\n            if (sbr != null) {\n                sbr.release();\n                sbr = null;\n            }\n        }\n\n        @Override\n        public CqUnit nextAndRelease() {\n            try {\n                return next();\n            } finally {\n                release();\n            }\n        }\n    }\n\n    public ConsumeQueueExt.CqExtUnit getExt(final long offset) {\n        if (isExtReadEnable()) {\n            return this.consumeQueueExt.get(offset);\n        }\n        return null;\n    }\n\n    public boolean getExt(final long offset, ConsumeQueueExt.CqExtUnit cqExtUnit) {\n        if (isExtReadEnable()) {\n            return this.consumeQueueExt.get(offset, cqExtUnit);\n        }\n        return false;\n    }\n\n    @Override\n    public long getMinLogicOffset() {\n        return minLogicOffset;\n    }\n\n    public void setMinLogicOffset(long minLogicOffset) {\n        this.minLogicOffset = minLogicOffset;\n    }\n\n    @Override\n    public long rollNextFile(final long nextBeginOffset) {\n        int mappedFileSize = this.mappedFileSize;\n        int totalUnitsInFile = mappedFileSize / CQ_STORE_UNIT_SIZE;\n        return nextBeginOffset + totalUnitsInFile - nextBeginOffset % totalUnitsInFile;\n    }\n\n    @Override\n    public String getTopic() {\n        return topic;\n    }\n\n    @Override\n    public int getQueueId() {\n        return queueId;\n    }\n\n    @Override\n    public CQType getCQType() {\n        return CQType.SimpleCQ;\n    }\n\n    @Override\n    public long getMaxPhysicOffset() {\n        return maxPhysicOffset;\n    }\n\n    public void setMaxPhysicOffset(long maxPhysicOffset) {\n        this.maxPhysicOffset = maxPhysicOffset;\n    }\n\n    @Override\n    public void destroy() {\n        this.setMaxPhysicOffset(-1);\n        this.minLogicOffset = 0;\n        this.mappedFileQueue.destroy();\n        if (isExtReadEnable()) {\n            this.consumeQueueExt.destroy();\n        }\n    }\n\n    @Override\n    public long getMessageTotalInQueue() {\n        return this.getMaxOffsetInQueue() - this.getMinOffsetInQueue();\n    }\n\n    @Override\n    public long getMaxOffsetInQueue() {\n        return this.mappedFileQueue.getMaxOffset() / CQ_STORE_UNIT_SIZE;\n    }\n\n    @Override\n    public void checkSelf() {\n        mappedFileQueue.checkSelf();\n        if (isExtReadEnable()) {\n            this.consumeQueueExt.checkSelf();\n        }\n    }\n\n    protected boolean isExtReadEnable() {\n        return this.consumeQueueExt != null;\n    }\n\n    protected boolean isExtWriteEnable() {\n        return this.consumeQueueExt != null\n            && this.messageStore.getMessageStoreConfig().isEnableConsumeQueueExt();\n    }\n\n    /**\n     * Check {@code tagsCode} is address of extend file or tags code.\n     */\n    public boolean isExtAddr(long tagsCode) {\n        return ConsumeQueueExt.isExtAddr(tagsCode);\n    }\n\n    @Override\n    public void swapMap(int reserveNum, long forceSwapIntervalMs, long normalSwapIntervalMs) {\n        mappedFileQueue.swapMap(reserveNum, forceSwapIntervalMs, normalSwapIntervalMs);\n    }\n\n    @Override\n    public void cleanSwappedMap(long forceCleanSwapIntervalMs) {\n        mappedFileQueue.cleanSwappedMap(forceCleanSwapIntervalMs);\n    }\n\n    @Override\n    public long estimateMessageCount(long from, long to, MessageFilter filter) {\n        long physicalOffsetFrom = from * CQ_STORE_UNIT_SIZE;\n        long physicalOffsetTo = to * CQ_STORE_UNIT_SIZE;\n        List<MappedFile> mappedFiles = mappedFileQueue.range(physicalOffsetFrom, physicalOffsetTo);\n        if (mappedFiles.isEmpty()) {\n            return -1;\n        }\n\n        boolean sample = false;\n        long match = 0;\n        long raw = 0;\n\n        for (MappedFile mappedFile : mappedFiles) {\n            int start = 0;\n            int len = mappedFile.getFileSize();\n\n            // calculate start and len for first segment and last segment to reduce scanning\n            // first file segment\n            if (mappedFile.getFileFromOffset() <= physicalOffsetFrom) {\n                start = (int) (physicalOffsetFrom - mappedFile.getFileFromOffset());\n                if (mappedFile.getFileFromOffset() + mappedFile.getFileSize() >= physicalOffsetTo) {\n                    // current mapped file covers search range completely.\n                    len = (int) (physicalOffsetTo - physicalOffsetFrom);\n                } else {\n                    len = mappedFile.getFileSize() - start;\n                }\n            }\n\n            // last file segment\n            if (0 == start && mappedFile.getFileFromOffset() + mappedFile.getFileSize() > physicalOffsetTo) {\n                len = (int) (physicalOffsetTo - mappedFile.getFileFromOffset());\n            }\n\n            // select partial data to scan\n            SelectMappedBufferResult slice = mappedFile.selectMappedBuffer(start, len);\n            if (null != slice) {\n                try {\n                    ByteBuffer buffer = slice.getByteBuffer();\n                    int current = 0;\n                    while (current < len) {\n                        // skip physicalOffset and message length fields.\n                        buffer.position(current + MSG_TAG_OFFSET_INDEX);\n                        long tagCode = buffer.getLong();\n                        ConsumeQueueExt.CqExtUnit ext = null;\n                        if (isExtWriteEnable()) {\n                            ext = consumeQueueExt.get(tagCode);\n                            tagCode = ext.getTagsCode();\n                        }\n                        if (filter.isMatchedByConsumeQueue(tagCode, ext)) {\n                            match++;\n                        }\n                        raw++;\n                        current += CQ_STORE_UNIT_SIZE;\n\n                        if (raw >= messageStore.getMessageStoreConfig().getMaxConsumeQueueScan()) {\n                            sample = true;\n                            break;\n                        }\n\n                        if (match > messageStore.getMessageStoreConfig().getSampleCountThreshold()) {\n                            sample = true;\n                            break;\n                        }\n                    }\n                } finally {\n                    slice.release();\n                }\n            }\n            // we have scanned enough entries, now is the time to return an educated guess.\n            if (sample) {\n                break;\n            }\n        }\n\n        long result = match;\n        if (sample) {\n            if (0 == raw) {\n                log.error(\"[BUG]. Raw should NOT be 0\");\n                return 0;\n            }\n            result = (long) (match * (to - from) * 1.0 / raw);\n        }\n        log.debug(\"Result={}, raw={}, match={}, sample={}\", result, raw, match, sample);\n        return result;\n    }\n\n    @Override\n    public void initializeWithOffset(long offset, long minPhyOffset) {\n        // Because the file version cq requires that files are continuous,\n        // If existing cq not be completely deleted, new cq can not initialize with given offset.\n        destroy();\n\n        // correct min offset\n        // TODO: when min commitLog offset is 0 and restart store, min offset of cq may be set to 0 incorrectly\n        setMinLogicOffset(offset * ConsumeQueue.CQ_STORE_UNIT_SIZE);\n\n        // transientStorePool is null, only need set wrote position here\n        MappedFile mappedFile = mappedFileQueue.getLastMappedFile(offset * ConsumeQueue.CQ_STORE_UNIT_SIZE, true);\n        fillPreBlank(mappedFile, offset * ConsumeQueue.CQ_STORE_UNIT_SIZE);\n\n        flush(0);\n    }\n\n    @Override\n    public boolean shutdown() {\n        this.mappedFileQueue.cleanResourcesAll();\n        return true;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ConsumeQueueExt.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\nimport java.io.File;\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.List;\nimport org.apache.rocketmq.store.logfile.MappedFile;\n\n/**\n * Extend of consume queue, to store something not important,\n * such as message store time, filter bit map and etc.\n * <p/>\n * <li>1. This class is used only by {@link ConsumeQueue}</li>\n * <li>2. And is weakly reliable.</li>\n * <li>3. Be careful, address returned is always less than 0.</li>\n * <li>4. Pls keep this file small.</li>\n */\npublic class ConsumeQueueExt {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n\n    private final MappedFileQueue mappedFileQueue;\n    private final String topic;\n    private final int queueId;\n\n    private final String storePath;\n    private final int mappedFileSize;\n    private ByteBuffer tempContainer;\n\n    public static final int END_BLANK_DATA_LENGTH = 4;\n\n    /**\n     * Addr can not exceed this value.For compatible.\n     */\n    public static final long MAX_ADDR = Integer.MIN_VALUE - 1L;\n    public static final long MAX_REAL_OFFSET = MAX_ADDR - Long.MIN_VALUE;\n\n    /**\n     * Constructor.\n     *\n     * @param topic topic\n     * @param queueId id of queue\n     * @param storePath root dir of files to store.\n     * @param mappedFileSize file size\n     * @param bitMapLength bit map length.\n     */\n    public ConsumeQueueExt(final String topic,\n        final int queueId,\n        final String storePath,\n        final int mappedFileSize,\n        final int bitMapLength) {\n\n        this.storePath = storePath;\n        this.mappedFileSize = mappedFileSize;\n\n        this.topic = topic;\n        this.queueId = queueId;\n\n        String queueDir = this.storePath\n            + File.separator + topic\n            + File.separator + queueId;\n\n        this.mappedFileQueue = new MappedFileQueue(queueDir, mappedFileSize, null);\n\n        if (bitMapLength > 0) {\n            this.tempContainer = ByteBuffer.allocate(\n                bitMapLength / Byte.SIZE\n            );\n        }\n    }\n\n    /**\n     * Constructor with writeWithoutMmap support.\n     *\n     * @param topic topic\n     * @param queueId id of queue\n     * @param storePath root dir of files to store.\n     * @param mappedFileSize file size\n     * @param bitMapLength bit map length.\n     * @param writeWithoutMmap whether to use RandomAccessFile instead of MappedByteBuffer\n     */\n    public ConsumeQueueExt(final String topic,\n        final int queueId,\n        final String storePath,\n        final int mappedFileSize,\n        final int bitMapLength,\n        final boolean writeWithoutMmap) {\n\n        this.storePath = storePath;\n        this.mappedFileSize = mappedFileSize;\n\n        this.topic = topic;\n        this.queueId = queueId;\n\n        String queueDir = this.storePath\n            + File.separator + topic\n            + File.separator + queueId;\n\n        this.mappedFileQueue = new MappedFileQueue(queueDir, mappedFileSize, null, writeWithoutMmap);\n\n        if (bitMapLength > 0) {\n            this.tempContainer = ByteBuffer.allocate(\n                bitMapLength / Byte.SIZE\n            );\n        }\n    }\n\n    public long getTotalSize() {\n        return this.mappedFileQueue.getTotalFileSize();\n    }\n\n    /**\n     * Check whether {@code address} point to extend file.\n     * <p>\n     * Just test {@code address} is less than 0.\n     * </p>\n     */\n    public static boolean isExtAddr(final long address) {\n        return address <= MAX_ADDR;\n    }\n\n    /**\n     * Transform {@code address}(decorated by {@link #decorate}) to offset in mapped file.\n     * <p>\n     * if {@code address} is less than 0, return {@code address} - {@link java.lang.Long#MIN_VALUE};\n     * else, just return {@code address}\n     * </p>\n     */\n    public long unDecorate(final long address) {\n        if (isExtAddr(address)) {\n            return address - Long.MIN_VALUE;\n        }\n        return address;\n    }\n\n    /**\n     * Decorate {@code offset} from mapped file, in order to distinguish with tagsCode(saved in cq originally).\n     * <p>\n     * if {@code offset} is greater than or equal to 0, then return {@code offset} + {@link java.lang.Long#MIN_VALUE};\n     * else, just return {@code offset}\n     * </p>\n     *\n     * @return ext address(value is less than 0)\n     */\n    public long decorate(final long offset) {\n        if (!isExtAddr(offset)) {\n            return offset + Long.MIN_VALUE;\n        }\n        return offset;\n    }\n\n    /**\n     * Get data from buffer.\n     *\n     * @param address less than 0\n     */\n    public CqExtUnit get(final long address) {\n        CqExtUnit cqExtUnit = new CqExtUnit();\n        if (get(address, cqExtUnit)) {\n            return cqExtUnit;\n        }\n\n        return null;\n    }\n\n    /**\n     * Get data from buffer, and set to {@code cqExtUnit}\n     *\n     * @param address less than 0\n     */\n    public boolean get(final long address, final CqExtUnit cqExtUnit) {\n        if (!isExtAddr(address)) {\n            return false;\n        }\n\n        final int mappedFileSize = this.mappedFileSize;\n        final long realOffset = unDecorate(address);\n\n        MappedFile mappedFile = this.mappedFileQueue.findMappedFileByOffset(realOffset, realOffset == 0);\n        if (mappedFile == null) {\n            return false;\n        }\n\n        int pos = (int) (realOffset % mappedFileSize);\n\n        SelectMappedBufferResult bufferResult = mappedFile.selectMappedBuffer(pos);\n        if (bufferResult == null) {\n            log.warn(\"[BUG] Consume queue extend unit({}) is not found!\", realOffset);\n            return false;\n        }\n        boolean ret = false;\n        try {\n            ret = cqExtUnit.read(bufferResult.getByteBuffer());\n        } finally {\n            bufferResult.release();\n        }\n\n        return ret;\n    }\n\n    /**\n     * Save to mapped buffer of file and return address.\n     * <p>\n     * Be careful, this method is not thread safe.\n     * </p>\n     *\n     * @return success: < 0: fail: >=0\n     */\n    public long put(final CqExtUnit cqExtUnit) {\n        final int retryTimes = 3;\n        try {\n            int size = cqExtUnit.calcUnitSize();\n            if (size > CqExtUnit.MAX_EXT_UNIT_SIZE) {\n                log.error(\"Size of cq ext unit is greater than {}, {}\", CqExtUnit.MAX_EXT_UNIT_SIZE, cqExtUnit);\n                return 1;\n            }\n            if (this.mappedFileQueue.getMaxOffset() + size > MAX_REAL_OFFSET) {\n                log.warn(\"Capacity of ext is maximum!{}, {}\", this.mappedFileQueue.getMaxOffset(), size);\n                return 1;\n            }\n            // unit size maybe change.but, the same most of the time.\n            if (this.tempContainer == null || this.tempContainer.capacity() < size) {\n                this.tempContainer = ByteBuffer.allocate(size);\n            }\n\n            for (int i = 0; i < retryTimes; i++) {\n                MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();\n\n                if (mappedFile == null || mappedFile.isFull()) {\n                    mappedFile = this.mappedFileQueue.getLastMappedFile(0);\n                }\n\n                if (mappedFile == null) {\n                    log.error(\"Create mapped file when save consume queue extend, {}\", cqExtUnit);\n                    continue;\n                }\n                final int wrotePosition = mappedFile.getWrotePosition();\n                final int blankSize = this.mappedFileSize - wrotePosition - END_BLANK_DATA_LENGTH;\n\n                // check whether has enough space.\n                if (size > blankSize) {\n                    fullFillToEnd(mappedFile, wrotePosition);\n                    log.info(\"No enough space(need:{}, has:{}) of file {}, so fill to end\",\n                        size, blankSize, mappedFile.getFileName());\n                    continue;\n                }\n\n                if (mappedFile.appendMessage(cqExtUnit.write(this.tempContainer), 0, size)) {\n                    return decorate(wrotePosition + mappedFile.getFileFromOffset());\n                }\n            }\n        } catch (Throwable e) {\n            log.error(\"Save consume queue extend error, \" + cqExtUnit, e);\n        }\n\n        return 1;\n    }\n\n    protected void fullFillToEnd(final MappedFile mappedFile, final int wrotePosition) {\n        ByteBuffer mappedFileBuffer = mappedFile.sliceByteBuffer();\n        mappedFileBuffer.position(wrotePosition);\n\n        // ending.\n        mappedFileBuffer.putShort((short) -1);\n\n        mappedFile.setWrotePosition(this.mappedFileSize);\n    }\n\n    /**\n     * Load data from file when startup.\n     */\n    public boolean load() {\n        boolean result = this.mappedFileQueue.load();\n        log.info(\"load consume queue extend\" + this.topic + \"-\" + this.queueId + \" \" + (result ? \"OK\" : \"Failed\"));\n        return result;\n    }\n\n    /**\n     * Check whether the step size in mapped file queue is correct.\n     */\n    public void checkSelf() {\n        this.mappedFileQueue.checkSelf();\n    }\n\n    /**\n     * Recover.\n     */\n    public void recover() {\n        final List<MappedFile> mappedFiles = this.mappedFileQueue.getMappedFiles();\n        if (mappedFiles == null || mappedFiles.isEmpty()) {\n            return;\n        }\n\n        // load all files, consume queue will truncate extend files.\n        int index = 0;\n\n        MappedFile mappedFile = mappedFiles.get(index);\n        ByteBuffer byteBuffer = mappedFile.sliceByteBuffer();\n        long processOffset = mappedFile.getFileFromOffset();\n        long mappedFileOffset = 0;\n        CqExtUnit extUnit = new CqExtUnit();\n        while (true) {\n            extUnit.readBySkip(byteBuffer);\n\n            // check whether write sth.\n            if (extUnit.getSize() > 0) {\n                mappedFileOffset += extUnit.getSize();\n                continue;\n            }\n\n            index++;\n            if (index < mappedFiles.size()) {\n                mappedFile = mappedFiles.get(index);\n                byteBuffer = mappedFile.sliceByteBuffer();\n                processOffset = mappedFile.getFileFromOffset();\n                mappedFileOffset = 0;\n                log.info(\"Recover next consume queue extend file, \" + mappedFile.getFileName());\n                continue;\n            }\n\n            log.info(\"All files of consume queue extend has been recovered over, last mapped file \"\n                + mappedFile.getFileName());\n            break;\n        }\n\n        processOffset += mappedFileOffset;\n        this.mappedFileQueue.setFlushedWhere(processOffset);\n        this.mappedFileQueue.setCommittedWhere(processOffset);\n        this.mappedFileQueue.truncateDirtyFiles(processOffset);\n    }\n\n    /**\n     * Delete files before {@code minAddress}.\n     *\n     * @param minAddress less than 0\n     */\n    public void truncateByMinAddress(final long minAddress) {\n        if (!isExtAddr(minAddress)) {\n            return;\n        }\n\n        log.info(\"Truncate consume queue ext by min {}.\", minAddress);\n\n        List<MappedFile> willRemoveFiles = new ArrayList<>();\n\n        List<MappedFile> mappedFiles = this.mappedFileQueue.getMappedFiles();\n        final long realOffset = unDecorate(minAddress);\n\n        for (MappedFile file : mappedFiles) {\n            long fileTailOffset = file.getFileFromOffset() + this.mappedFileSize;\n\n            if (fileTailOffset < realOffset) {\n                log.info(\"Destroy consume queue ext by min: file={}, fileTailOffset={}, minOffset={}\", file.getFileName(),\n                    fileTailOffset, realOffset);\n                if (file.destroy(1000)) {\n                    willRemoveFiles.add(file);\n                }\n            }\n        }\n\n        this.mappedFileQueue.deleteExpiredFile(willRemoveFiles);\n    }\n\n    /**\n     * Delete files after {@code maxAddress}, and reset wrote/commit/flush position to last file.\n     *\n     * @param maxAddress less than 0\n     */\n    public void truncateByMaxAddress(final long maxAddress) {\n        if (!isExtAddr(maxAddress)) {\n            return;\n        }\n\n        log.info(\"Truncate consume queue ext by max {}.\", maxAddress);\n\n        CqExtUnit cqExtUnit = get(maxAddress);\n        if (cqExtUnit == null) {\n            log.error(\"[BUG] address {} of consume queue extend not found!\", maxAddress);\n            return;\n        }\n\n        final long realOffset = unDecorate(maxAddress);\n\n        this.mappedFileQueue.truncateDirtyFiles(realOffset + cqExtUnit.getSize());\n    }\n\n    /**\n     * flush buffer to file.\n     */\n    public boolean flush(final int flushLeastPages) {\n        return this.mappedFileQueue.flush(flushLeastPages);\n    }\n\n    /**\n     * delete files and directory.\n     */\n    public void destroy() {\n        this.mappedFileQueue.destroy();\n    }\n\n    /**\n     * Max address(value is less than 0).\n     * <p/>\n     * <p>\n     * Be careful: it's an address just when invoking this method.\n     * </p>\n     */\n    public long getMaxAddress() {\n        MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();\n        if (mappedFile == null) {\n            return decorate(0);\n        }\n        return decorate(mappedFile.getFileFromOffset() + mappedFile.getWrotePosition());\n    }\n\n    /**\n     * Minus address saved in file.\n     */\n    public long getMinAddress() {\n        MappedFile firstFile = this.mappedFileQueue.getFirstMappedFile();\n        if (firstFile == null) {\n            return decorate(0);\n        }\n        return decorate(firstFile.getFileFromOffset());\n    }\n\n    /**\n     * Store unit.\n     */\n    public static class CqExtUnit {\n        public static final short MIN_EXT_UNIT_SIZE\n            = 2 * 1 // size, 32k max\n            + 8 * 2 // msg time + tagCode\n            + 2; // bitMapSize\n\n        public static final int MAX_EXT_UNIT_SIZE = Short.MAX_VALUE;\n\n        public CqExtUnit() {\n        }\n\n        public CqExtUnit(Long tagsCode, long msgStoreTime, byte[] filterBitMap) {\n            this.tagsCode = tagsCode == null ? 0 : tagsCode;\n            this.msgStoreTime = msgStoreTime;\n            this.filterBitMap = filterBitMap;\n            this.bitMapSize = (short) (filterBitMap == null ? 0 : filterBitMap.length);\n            this.size = (short) (MIN_EXT_UNIT_SIZE + this.bitMapSize);\n        }\n\n        /**\n         * unit size\n         */\n        private short size;\n        /**\n         * has code of tags\n         */\n        private long tagsCode;\n        /**\n         * the time to store into commit log of message\n         */\n        private long msgStoreTime;\n        /**\n         * size of bit map\n         */\n        private short bitMapSize;\n        /**\n         * filter bit map\n         */\n        private byte[] filterBitMap;\n\n        /**\n         * build unit from buffer from current position.\n         */\n        private boolean read(final ByteBuffer buffer) {\n            if (buffer.position() + 2 > buffer.limit()) {\n                return false;\n            }\n\n            this.size = buffer.getShort();\n\n            if (this.size < 1) {\n                return false;\n            }\n\n            this.tagsCode = buffer.getLong();\n            this.msgStoreTime = buffer.getLong();\n            this.bitMapSize = buffer.getShort();\n\n            if (this.bitMapSize < 1) {\n                return true;\n            }\n\n            if (this.filterBitMap == null || this.filterBitMap.length != this.bitMapSize) {\n                this.filterBitMap = new byte[bitMapSize];\n            }\n\n            buffer.get(this.filterBitMap);\n            return true;\n        }\n\n        /**\n         * Only read first 2 byte to get unit size.\n         * <p>\n         * if size > 0, then skip buffer position with size.\n         * </p>\n         * <p>\n         * if size <= 0, nothing to do.\n         * </p>\n         */\n        private void readBySkip(final ByteBuffer buffer) {\n            ByteBuffer temp = buffer.slice();\n\n            short tempSize = temp.getShort();\n            this.size = tempSize;\n\n            if (tempSize > 0) {\n                buffer.position(buffer.position() + this.size);\n            }\n        }\n\n        /**\n         * Transform unit data to byte array.\n         * <p/>\n         * <li>1. @{code container} can be null, it will be created if null.</li>\n         * <li>2. if capacity of @{code container} is less than unit size, it will be created also.</li>\n         * <li>3. Pls be sure that size of unit is not greater than {@link #MAX_EXT_UNIT_SIZE}</li>\n         */\n        private byte[] write(final ByteBuffer container) {\n            this.bitMapSize = (short) (filterBitMap == null ? 0 : filterBitMap.length);\n            this.size = (short) (MIN_EXT_UNIT_SIZE + this.bitMapSize);\n\n            ByteBuffer temp = container;\n\n            if (temp == null || temp.capacity() < this.size) {\n                temp = ByteBuffer.allocate(this.size);\n            }\n\n            temp.flip();\n            temp.limit(this.size);\n\n            temp.putShort(this.size);\n            temp.putLong(this.tagsCode);\n            temp.putLong(this.msgStoreTime);\n            temp.putShort(this.bitMapSize);\n            if (this.bitMapSize > 0) {\n                temp.put(this.filterBitMap);\n            }\n\n            return temp.array();\n        }\n\n        /**\n         * Calculate unit size by current data.\n         */\n        private int calcUnitSize() {\n            int sizeTemp = MIN_EXT_UNIT_SIZE + (filterBitMap == null ? 0 : filterBitMap.length);\n            return sizeTemp;\n        }\n\n        public long getTagsCode() {\n            return tagsCode;\n        }\n\n        public void setTagsCode(final long tagsCode) {\n            this.tagsCode = tagsCode;\n        }\n\n        public long getMsgStoreTime() {\n            return msgStoreTime;\n        }\n\n        public void setMsgStoreTime(final long msgStoreTime) {\n            this.msgStoreTime = msgStoreTime;\n        }\n\n        public byte[] getFilterBitMap() {\n            if (this.bitMapSize < 1) {\n                return null;\n            }\n            return filterBitMap;\n        }\n\n        public void setFilterBitMap(final byte[] filterBitMap) {\n            this.filterBitMap = filterBitMap;\n            // not safe transform, but size will be calculate by #calcUnitSize\n            this.bitMapSize = (short) (filterBitMap == null ? 0 : filterBitMap.length);\n        }\n\n        public short getSize() {\n            return size;\n        }\n\n        @Override\n        public boolean equals(Object o) {\n            if (this == o)\n                return true;\n            if (!(o instanceof CqExtUnit))\n                return false;\n\n            CqExtUnit cqExtUnit = (CqExtUnit) o;\n\n            if (bitMapSize != cqExtUnit.bitMapSize)\n                return false;\n            if (msgStoreTime != cqExtUnit.msgStoreTime)\n                return false;\n            if (size != cqExtUnit.size)\n                return false;\n            if (tagsCode != cqExtUnit.tagsCode)\n                return false;\n            if (!Arrays.equals(filterBitMap, cqExtUnit.filterBitMap))\n                return false;\n\n            return true;\n        }\n\n        @Override\n        public int hashCode() {\n            int result = (int) size;\n            result = 31 * result + (int) (tagsCode ^ (tagsCode >>> 32));\n            result = 31 * result + (int) (msgStoreTime ^ (msgStoreTime >>> 32));\n            result = 31 * result + (int) bitMapSize;\n            result = 31 * result + (filterBitMap != null ? Arrays.hashCode(filterBitMap) : 0);\n            return result;\n        }\n\n        @Override\n        public String toString() {\n            return \"CqExtUnit{\" +\n                \"size=\" + size +\n                \", tagsCode=\" + tagsCode +\n                \", msgStoreTime=\" + msgStoreTime +\n                \", bitMapSize=\" + bitMapSize +\n                \", filterBitMap=\" + Arrays.toString(filterBitMap) +\n                '}';\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/DefaultMessageFilter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport java.nio.ByteBuffer;\nimport java.util.Map;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\npublic class DefaultMessageFilter implements MessageFilter {\n\n    private SubscriptionData subscriptionData;\n\n    public DefaultMessageFilter(final SubscriptionData subscriptionData) {\n        this.subscriptionData = subscriptionData;\n    }\n\n    @Override\n    public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {\n        if (null == tagsCode || null == subscriptionData) {\n            return true;\n        }\n\n        if (subscriptionData.isClassFilterMode()) {\n            return true;\n        }\n\n        return subscriptionData.getSubString().equals(SubscriptionData.SUB_ALL)\n            || subscriptionData.getCodeSet().contains(tagsCode.intValue());\n    }\n\n    @Override\n    public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map<String, String> properties) {\n        return true;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport com.google.common.collect.Sets;\nimport com.google.common.hash.Hashing;\nimport io.openmessaging.storage.dledger.entry.DLedgerEntry;\nimport io.opentelemetry.api.common.AttributesBuilder;\nimport io.opentelemetry.api.metrics.Meter;\nimport io.opentelemetry.sdk.metrics.InstrumentSelector;\nimport io.opentelemetry.sdk.metrics.ViewBuilder;\nimport java.io.File;\nimport java.io.IOException;\nimport java.io.RandomAccessFile;\nimport java.net.Inet6Address;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.nio.ByteBuffer;\nimport java.nio.channels.FileLock;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Optional;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentLinkedQueue;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.ConcurrentSkipListMap;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.TimeoutException;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.function.Supplier;\nimport com.alibaba.fastjson2.JSON;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.BrokerIdentity;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.SystemClock;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.attribute.CleanupPolicy;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBatch;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.running.RunningStats;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.common.utils.CleanupPolicyUtils;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.common.utils.QueueTypeUtils;\nimport org.apache.rocketmq.common.utils.ServiceProvider;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.config.FlushDiskType;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.config.StorePathConfigHelper;\nimport org.apache.rocketmq.store.dledger.DLedgerCommitLog;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.ha.DefaultHAService;\nimport org.apache.rocketmq.store.ha.HAService;\nimport org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService;\nimport org.apache.rocketmq.store.hook.PutMessageHook;\nimport org.apache.rocketmq.store.hook.SendMessageBackHook;\nimport org.apache.rocketmq.store.index.IndexService;\nimport org.apache.rocketmq.store.index.QueryOffsetResult;\nimport org.apache.rocketmq.store.index.rocksdb.IndexRocksDBStore;\nimport org.apache.rocketmq.store.kv.CommitLogDispatcherCompaction;\nimport org.apache.rocketmq.store.kv.CompactionService;\nimport org.apache.rocketmq.store.kv.CompactionStore;\nimport org.apache.rocketmq.store.logfile.MappedFile;\nimport org.apache.rocketmq.store.logfile.SharedByteBufferManager;\nimport org.apache.rocketmq.store.metrics.DefaultStoreMetricsManager;\nimport org.apache.rocketmq.store.queue.CombineConsumeQueueStore;\nimport org.apache.rocketmq.store.queue.ConsumeQueueInterface;\nimport org.apache.rocketmq.store.queue.ConsumeQueueStore;\nimport org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface;\nimport org.apache.rocketmq.store.queue.CqUnit;\nimport org.apache.rocketmq.store.queue.ReferredIterator;\nimport org.apache.rocketmq.store.rocksdb.MessageRocksDBStorage;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.apache.rocketmq.store.timer.TimerMessageStore;\nimport org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStore;\nimport org.apache.rocketmq.store.transaction.TransMessageRocksDBStore;\nimport org.apache.rocketmq.store.util.PerfCounter;\nimport org.apache.rocketmq.store.metrics.StoreMetricsManager;\nimport org.rocksdb.RocksDBException;\n\npublic class DefaultMessageStore implements MessageStore {\n    protected static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    protected static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);\n\n    public final PerfCounter.Ticks perfs = new PerfCounter.Ticks(LOGGER);\n\n    private final MessageStoreConfig messageStoreConfig;\n    // CommitLog\n    protected final CommitLog commitLog;\n\n    protected final ConsumeQueueStoreInterface consumeQueueStore;\n\n    protected final CleanCommitLogService cleanCommitLogService;\n\n    protected final IndexService indexService;\n    protected final IndexRocksDBStore indexRocksDBStore;\n\n    private final AllocateMappedFileService allocateMappedFileService;\n\n    private ReputMessageService reputMessageService;\n\n    private HAService haService;\n\n    // CompactionLog\n    private CompactionStore compactionStore;\n\n    private CompactionService compactionService;\n\n    private final StoreStatsService storeStatsService;\n\n    private final TransientStorePool transientStorePool;\n\n    protected final RunningFlags runningFlags = new RunningFlags();\n    private final SystemClock systemClock = new SystemClock();\n\n    private final ScheduledExecutorService scheduledExecutorService;\n    private final BrokerStatsManager brokerStatsManager;\n    private final MessageArrivingListener messageArrivingListener;\n    private final BrokerConfig brokerConfig;\n\n    private volatile boolean shutdown = true;\n\n    private boolean notifyMessageArriveInBatch = false;\n\n    protected StoreCheckpoint storeCheckpoint;\n    private MessageRocksDBStorage messageRocksDBStorage;\n    private TimerMessageStore timerMessageStore;\n    private final DefaultStoreMetricsManager defaultStoreMetricsManager;\n    private TimerMessageRocksDBStore timerMessageRocksDBStore;\n    private TransMessageRocksDBStore transMessageRocksDBStore;\n\n    private final LinkedList<CommitLogDispatcher> dispatcherList = new LinkedList<>();\n\n    /**\n     * List of stores that require commitlog dispatch and recovery. Each store registers itself when loading.\n     */\n    private final List<CommitLogDispatchStore> commitLogDispatchStores = new ArrayList<>();\n\n    private final RandomAccessFile lockFile;\n\n    private FileLock lock;\n\n    boolean shutDownNormal = false;\n    // Max pull msg size\n    private final static int MAX_PULL_MSG_SIZE = 128 * 1024 * 1024;\n\n    private volatile int aliveReplicasNum = 1;\n\n    // Refer the MessageStore of MasterBroker in the same process.\n    // If current broker is master, this reference point to null or itself.\n    // If current broker is slave, this reference point to the store of master broker, and the two stores belong to\n    // different broker groups.\n    private MessageStore masterStoreInProcess = null;\n\n    private volatile long masterFlushedOffset = -1L;\n\n    private volatile long brokerInitMaxOffset = -1L;\n\n    private final List<PutMessageHook> putMessageHookList = new ArrayList<>();\n\n    private SendMessageBackHook sendMessageBackHook;\n\n    private final ConcurrentSkipListMap<Integer /* level */, Long/* delay timeMillis */> delayLevelTable =\n        new ConcurrentSkipListMap<>();\n\n    private int maxDelayLevel;\n\n    private final AtomicInteger mappedPageHoldCount = new AtomicInteger(0);\n\n    private final ConcurrentLinkedQueue<BatchDispatchRequest> batchDispatchRequestQueue = new ConcurrentLinkedQueue<>();\n\n    private final int dispatchRequestOrderlyQueueSize = 16;\n\n    private final DispatchRequestOrderlyQueue dispatchRequestOrderlyQueue = new DispatchRequestOrderlyQueue(dispatchRequestOrderlyQueueSize);\n\n    private long stateMachineVersion = 0L;\n\n    // this is a unmodifiableMap\n    private final ConcurrentMap<String, TopicConfig> topicConfigTable;\n\n    private final MessageStoreStateMachine stateMachine;\n\n    private final ScheduledExecutorService scheduledCleanQueueExecutorService =\n        ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(\"StoreCleanQueueScheduledThread\"));\n\n    public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager,\n        final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig,\n        final ConcurrentMap<String, TopicConfig> topicConfigTable) throws IOException {\n        stateMachine = new MessageStoreStateMachine(LOGGER);\n        this.messageArrivingListener = messageArrivingListener;\n        this.brokerConfig = brokerConfig;\n        this.messageStoreConfig = messageStoreConfig;\n        this.aliveReplicasNum = messageStoreConfig.getTotalReplicas();\n        this.brokerStatsManager = brokerStatsManager;\n        this.topicConfigTable = topicConfigTable;\n        this.allocateMappedFileService = new AllocateMappedFileService(this);\n\n        this.commitLog = messageStoreConfig.isEnableDLegerCommitLog() ?\n            new DLedgerCommitLog(this) : new CommitLog(this);\n        this.consumeQueueStore = createConsumeQueueStore();\n        this.cleanCommitLogService = new CleanCommitLogService();\n        this.storeStatsService = new StoreStatsService(getBrokerIdentity());\n        this.messageRocksDBStorage = new MessageRocksDBStorage(getMessageStoreConfig());\n        this.indexService = new IndexService(this);\n        this.indexRocksDBStore = new IndexRocksDBStore(this);\n        this.dispatcherList.addLast(new CommitLogDispatcherBuildConsumeQueue());\n        this.dispatcherList.addLast(new CommitLogDispatcherBuildIndex());\n        this.dispatcherList.addLast(new CommitLogDispatcherBuildTransIndex());\n\n        initializeHAService();\n\n        this.reputMessageService = messageStoreConfig.isEnableBuildConsumeQueueConcurrently() ?\n            new ConcurrentReputMessageService() : new ReputMessageService();\n\n        this.transientStorePool = new TransientStorePool(messageStoreConfig.getTransientStorePoolSize(), messageStoreConfig.getMappedFileSizeCommitLog());\n\n        if (messageStoreConfig.isWriteWithoutMmap()) {\n            SharedByteBufferManager.getInstance().init(messageStoreConfig.getMaxMessageSize(), messageStoreConfig.getSharedByteBufferNum());\n        }\n\n        this.defaultStoreMetricsManager = new DefaultStoreMetricsManager();\n\n        this.scheduledExecutorService =\n            ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(\"StoreScheduledThread\", getBrokerIdentity()));\n\n        if (messageStoreConfig.isEnableCompaction()) {\n            this.compactionStore = new CompactionStore(this);\n            this.compactionService = new CompactionService(commitLog, this, compactionStore);\n            this.dispatcherList.addLast(new CommitLogDispatcherCompaction(compactionService));\n        }\n\n        File file = new File(StorePathConfigHelper.getLockFile(messageStoreConfig.getStorePathRootDir()));\n        UtilAll.ensureDirOK(file.getParent());\n        UtilAll.ensureDirOK(getStorePathPhysic());\n        UtilAll.ensureDirOK(getStorePathLogic());\n        lockFile = new RandomAccessFile(file, \"rw\");\n\n        parseDelayLevel();\n    }\n\n    public ConsumeQueueStoreInterface createConsumeQueueStore() {\n        if (messageStoreConfig.isRocksdbCQDoubleWriteEnable()) {\n            return new CombineConsumeQueueStore(this);\n        }\n        return new ConsumeQueueStore(this);\n    }\n\n    public boolean parseDelayLevel() {\n        HashMap<String, Long> timeUnitTable = new HashMap<>();\n        timeUnitTable.put(\"s\", 1000L);\n        timeUnitTable.put(\"m\", 1000L * 60);\n        timeUnitTable.put(\"h\", 1000L * 60 * 60);\n        timeUnitTable.put(\"d\", 1000L * 60 * 60 * 24);\n\n        String levelString = messageStoreConfig.getMessageDelayLevel();\n        try {\n            String[] levelArray = levelString.split(\" \");\n            for (int i = 0; i < levelArray.length; i++) {\n                String value = levelArray[i];\n                String ch = value.substring(value.length() - 1);\n                Long tu = timeUnitTable.get(ch);\n\n                int level = i + 1;\n                if (level > this.maxDelayLevel) {\n                    this.maxDelayLevel = level;\n                }\n                long num = Long.parseLong(value.substring(0, value.length() - 1));\n                long delayTimeMillis = tu * num;\n                this.delayLevelTable.put(level, delayTimeMillis);\n            }\n        } catch (Exception e) {\n            LOGGER.error(\"parse message delay level failed. messageDelayLevel = {}\", levelString, e);\n            return false;\n        }\n\n        return true;\n    }\n\n    /**\n     * @throws IOException\n     */\n    @Override\n    public boolean load() {\n        boolean result = true;\n        stateMachine.transitTo(MessageStoreStateMachine.MessageStoreState.LOAD_BEGIN);\n        try {\n            boolean lastExitOK = !this.isTempFileExist();\n            LOGGER.info(\"last shutdown {}, store path root dir: {}\",\n                lastExitOK ? \"normally\" : \"abnormally\", messageStoreConfig.getStorePathRootDir());\n\n            // load Commit Log\n            result = this.commitLog.load();\n            stateMachine.transitTo(MessageStoreStateMachine.MessageStoreState.LOAD_COMMITLOG_OK, result);\n            // load Consume Queue\n            result = result && this.consumeQueueStore.load();\n            stateMachine.transitTo(MessageStoreStateMachine.MessageStoreState.LOAD_CONSUME_QUEUE_OK, result);\n            // Register consume queue store for commitlog dispatch\n            // AbstractConsumeQueueStore implements CommitLogDispatchStore, so we can register it directly\n            if (this.consumeQueueStore != null) {\n                registerCommitLogDispatchStore(this.consumeQueueStore);\n            }\n\n            if (messageStoreConfig.isEnableCompaction()) {\n                result = result && this.compactionService.load(lastExitOK);\n                stateMachine.transitTo(MessageStoreStateMachine.MessageStoreState.LOAD_COMPACTION_OK, result);\n            }\n\n            if (result) {\n                loadCheckPoint();\n                result = this.indexService.load(lastExitOK);\n                registerCommitLogDispatchStore(this.indexService);\n                stateMachine.transitTo(MessageStoreStateMachine.MessageStoreState.LOAD_INDEX_OK, result);\n                // Register IndexRocksDBStore and TransMessageRocksDBStore for commit-log dispatch\n                if (messageStoreConfig.isIndexRocksDBEnable()) {\n                    registerCommitLogDispatchStore(this.indexRocksDBStore);\n                }\n                if (messageStoreConfig.isTransRocksDBEnable() && transMessageRocksDBStore != null) {\n                    registerCommitLogDispatchStore(this.transMessageRocksDBStore);\n                }\n                this.recover(lastExitOK);\n                LOGGER.info(\"message store recover end, and the max phy offset = {}\", this.getMaxPhyOffset());\n            }\n\n            long maxOffset = this.getMaxPhyOffset();\n            this.setBrokerInitMaxOffset(maxOffset);\n            LOGGER.info(\"load over, and the max phy offset = {}\", maxOffset);\n        } catch (Exception e) {\n            LOGGER.error(\"load exception\", e);\n            result = false;\n        }\n\n        if (!result) {\n            this.allocateMappedFileService.shutdown();\n        }\n\n        return result;\n    }\n\n    public void loadCheckPoint() throws IOException {\n        this.storeCheckpoint =\n            new StoreCheckpoint(\n                StorePathConfigHelper.getStoreCheckpoint(this.messageStoreConfig.getStorePathRootDir()));\n        this.masterFlushedOffset = this.storeCheckpoint.getMasterFlushedOffset();\n        setConfirmOffset(this.storeCheckpoint.getConfirmPhyOffset());\n    }\n\n    private void recover(final boolean lastExitOK) throws RocksDBException {\n        this.stateMachine.transitTo(MessageStoreStateMachine.MessageStoreState.RECOVER_BEGIN);\n        // recover consume queue\n        this.consumeQueueStore.recover(this.brokerConfig.isRecoverConcurrently());\n        this.stateMachine.transitTo(MessageStoreStateMachine.MessageStoreState.RECOVER_CONSUME_QUEUE_OK);\n\n        // recover commitlog\n        // Calculate the minimum dispatch offset from all registered stores\n        Long dispatchFromPhyOffset = this.consumeQueueStore.getDispatchFromPhyOffset(lastExitOK);\n\n        for (CommitLogDispatchStore store : commitLogDispatchStores) {\n            Long storeOffset = store.getDispatchFromPhyOffset(lastExitOK);\n            if (storeOffset != null && storeOffset > 0) {\n                dispatchFromPhyOffset = Math.min(dispatchFromPhyOffset, storeOffset);\n            }\n        }\n\n        if (lastExitOK) {\n            this.commitLog.recoverNormally(dispatchFromPhyOffset);\n        } else {\n            this.commitLog.recoverAbnormally(dispatchFromPhyOffset);\n        }\n        this.stateMachine.transitTo(MessageStoreStateMachine.MessageStoreState.RECOVER_COMMITLOG_OK);\n\n        // recover consume offset table\n        this.recoverTopicQueueTable();\n        this.stateMachine.transitTo(MessageStoreStateMachine.MessageStoreState.RECOVER_TOPIC_QUEUE_TABLE_OK);\n    }\n\n    /**\n     * @throws Exception\n     */\n    @Override\n    public void start() throws Exception {\n        if (!messageStoreConfig.isEnableDLegerCommitLog() && !this.messageStoreConfig.isDuplicationEnable()) {\n            this.haService.init(this);\n        }\n\n        if (this.isTransientStorePoolEnable()) {\n            this.transientStorePool.init();\n        }\n        this.allocateMappedFileService.start();\n\n        this.indexService.start();\n\n        lock = lockFile.getChannel().tryLock(0, 1, false);\n        if (lock == null || lock.isShared() || !lock.isValid()) {\n            throw new RuntimeException(\"Lock failed, MQ already started, lock status: \" + lock);\n        }\n\n        lockFile.getChannel().write(ByteBuffer.wrap(\"lock\".getBytes(StandardCharsets.UTF_8)));\n        lockFile.getChannel().force(true);\n\n        this.reputMessageService.setReputFromOffset(this.commitLog.getConfirmOffset());\n        this.reputMessageService.start();\n\n        // Checking is not necessary, as long as the dLedger's implementation exactly follows the definition of Recover,\n        // which is eliminating the dispatch inconsistency between the commitLog and consumeQueue at the end of recovery.\n        this.doRecheckReputOffsetFromCq();\n\n        this.commitLog.start();\n        this.consumeQueueStore.start();\n        this.storeStatsService.start();\n\n        if (this.haService != null) {\n            this.haService.start();\n        }\n\n        this.createTempFile();\n        this.addScheduleTask();\n        this.perfs.start();\n        this.shutdown = false;\n\n        this.stateMachine.transitTo(MessageStoreStateMachine.MessageStoreState.RUNNING);\n    }\n\n    private void doRecheckReputOffsetFromCq() throws InterruptedException {\n        if (!messageStoreConfig.isRecheckReputOffsetFromCq()) {\n            return;\n        }\n\n        /*\n         * 1. Make sure the fast-forward messages to be truncated during the recovering according to the max physical offset of the commitlog;\n         * 2. DLedger committedPos may be missing, so the maxPhysicalPosInLogicQueue maybe bigger that maxOffset returned by DLedgerCommitLog, just let it go;\n         * 3. Calculate the reput offset according to the consume queue;\n         * 4. Make sure the fall-behind messages to be dispatched before starting the commitlog, especially when the broker role are automatically changed.\n         */\n        long maxPhysicalPosInLogicQueue = commitLog.getMinOffset();\n        for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : this.getConsumeQueueTable().values()) {\n            for (ConsumeQueueInterface logic : maps.values()) {\n                if (logic.getMaxPhysicOffset() > maxPhysicalPosInLogicQueue) {\n                    maxPhysicalPosInLogicQueue = logic.getMaxPhysicOffset();\n                }\n            }\n        }\n        // If maxPhyPos(CQs) < minPhyPos(CommitLog), some newly deleted topics may be re-dispatched into cqs mistakenly.\n        if (maxPhysicalPosInLogicQueue < 0) {\n            maxPhysicalPosInLogicQueue = 0;\n        }\n        if (maxPhysicalPosInLogicQueue < this.commitLog.getMinOffset()) {\n            maxPhysicalPosInLogicQueue = this.commitLog.getMinOffset();\n            /*\n             * This happens in following conditions:\n             * 1. If someone removes all the consumequeue files or the disk get damaged.\n             * 2. Launch a new broker, and copy the commitlog from other brokers.\n             *\n             * All the conditions has the same in common that the maxPhysicalPosInLogicQueue should be 0.\n             * If the maxPhysicalPosInLogicQueue is gt 0, there maybe something wrong.\n             */\n            LOGGER.warn(\"[TooSmallCqOffset] maxPhysicalPosInLogicQueue={} clMinOffset={}\", maxPhysicalPosInLogicQueue, this.commitLog.getMinOffset());\n        }\n        LOGGER.info(\"[SetReputOffset] maxPhysicalPosInLogicQueue={} clMinOffset={} clMaxOffset={} clConfirmedOffset={}\",\n            maxPhysicalPosInLogicQueue, this.commitLog.getMinOffset(), this.commitLog.getMaxOffset(), this.commitLog.getConfirmOffset());\n        this.reputMessageService.setReputFromOffset(maxPhysicalPosInLogicQueue);\n\n        /**\n         *  1. Finish dispatching the messages fall behind, then to start other services.\n         *  2. DLedger committedPos may be missing, so here just require dispatchBehindBytes <= 0\n         */\n        while (true) {\n            if (dispatchBehindBytes() <= 0) {\n                break;\n            }\n            Thread.sleep(1000);\n            LOGGER.info(\"Try to finish doing reput the messages fall behind during the starting, reputOffset={} maxOffset={} behind={}\", this.reputMessageService.getReputFromOffset(), this.getMaxPhyOffset(), this.dispatchBehindBytes());\n        }\n        this.recoverTopicQueueTable();\n    }\n\n    @Override\n    public void shutdown() {\n        if (!this.stateMachine.getCurrentState().equals(MessageStoreStateMachine.MessageStoreState.SHUTDOWN_OK)) {\n            this.shutdown = true;\n            this.stateMachine.transitTo(MessageStoreStateMachine.MessageStoreState.SHUTDOWN_BEGIN);\n\n            if (this.scheduledExecutorService != null) {\n                this.scheduledExecutorService.shutdown();\n            }\n\n            this.scheduledCleanQueueExecutorService.shutdown();\n\n            try {\n                this.scheduledExecutorService.awaitTermination(3, TimeUnit.SECONDS);\n                this.scheduledCleanQueueExecutorService.awaitTermination(3, TimeUnit.SECONDS);\n                Thread.sleep(1000 * 3);\n            } catch (Exception e) {\n                LOGGER.error(\"shutdown Exception, \", e);\n            }\n\n            if (this.haService != null) {\n                this.haService.shutdown();\n            }\n\n            if (this.storeStatsService != null) {\n                this.storeStatsService.shutdown();\n            }\n\n            if (this.commitLog != null) {\n                this.commitLog.shutdown();\n            }\n\n            if (this.reputMessageService != null) {\n                this.reputMessageService.shutdown();\n            }\n\n            if (this.consumeQueueStore != null) {\n                this.consumeQueueStore.shutdown();\n            }\n\n            // dispatch-related services must be shut down after reputMessageService\n            if (this.indexService != null) {\n                this.indexService.shutdown();\n            }\n\n            if (this.indexRocksDBStore != null) {\n                this.indexRocksDBStore.shutdown();\n            }\n\n            if (this.compactionService != null) {\n                this.compactionService.shutdown();\n            }\n\n            if (this.allocateMappedFileService != null) {\n                this.allocateMappedFileService.shutdown();\n            }\n\n            if (this.storeCheckpoint != null) {\n                this.storeCheckpoint.shutdown();\n            }\n\n            this.perfs.shutdown();\n\n            if (this.runningFlags.isWriteable() && dispatchBehindBytes() == 0) {\n                this.deleteFile(StorePathConfigHelper.getAbortFile(this.messageStoreConfig.getStorePathRootDir()));\n                shutDownNormal = true;\n                this.stateMachine.transitTo(MessageStoreStateMachine.MessageStoreState.SHUTDOWN_OK);\n            } else {\n                LOGGER.warn(\"the store may be wrong, so shutdown abnormally, and keep abort file.\");\n            }\n        }\n\n        if (this.transientStorePool != null) {\n            this.transientStorePool.destroy();\n        }\n\n        if (this.messageRocksDBStorage != null) {\n            this.messageRocksDBStorage.shutdown();\n        }\n\n        if (lock != null) {\n            try {\n                lock.release();\n            } catch (IOException e) {\n                LOGGER.error(\"release file lock error\", e);\n            }\n        }\n\n        if (lockFile != null) {\n            try {\n                lockFile.close();\n            } catch (Throwable e) {\n                LOGGER.error(\"lock file close error\", e);\n            }\n        }\n    }\n\n    @Override\n    public void destroy() {\n        this.consumeQueueStore.destroy(false);\n        this.commitLog.destroy();\n        this.indexService.destroy();\n        this.indexRocksDBStore.destroy();\n        this.deleteFile(StorePathConfigHelper.getAbortFile(this.messageStoreConfig.getStorePathRootDir()));\n        this.deleteFile(StorePathConfigHelper.getStoreCheckpoint(this.messageStoreConfig.getStorePathRootDir()));\n    }\n\n    public long getMajorFileSize() {\n        long commitLogSize = 0;\n        if (this.commitLog != null) {\n            commitLogSize = this.commitLog.getTotalSize();\n        }\n\n        long consumeQueueSize = 0;\n        if (this.consumeQueueStore != null) {\n            consumeQueueSize = this.consumeQueueStore.getTotalSize();\n        }\n\n        long indexFileSize = 0;\n        if (this.indexService != null) {\n            indexFileSize = this.indexService.getTotalSize();\n        }\n\n        return commitLogSize + consumeQueueSize + indexFileSize;\n    }\n\n    @Override\n    public CompletableFuture<PutMessageResult> asyncPutMessage(MessageExtBrokerInner msg) {\n\n        for (PutMessageHook putMessageHook : putMessageHookList) {\n            PutMessageResult handleResult = putMessageHook.executeBeforePutMessage(msg);\n            if (handleResult != null) {\n                return CompletableFuture.completedFuture(handleResult);\n            }\n        }\n\n        if (msg.getProperties().containsKey(MessageConst.PROPERTY_INNER_NUM)\n            && !MessageSysFlag.check(msg.getSysFlag(), MessageSysFlag.INNER_BATCH_FLAG)) {\n            LOGGER.warn(\"[BUG]The message had property {} but is not an inner batch\", MessageConst.PROPERTY_INNER_NUM);\n            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null));\n        }\n\n        if (MessageSysFlag.check(msg.getSysFlag(), MessageSysFlag.INNER_BATCH_FLAG)) {\n            Optional<TopicConfig> topicConfig = this.getTopicConfig(msg.getTopic());\n            if (!QueueTypeUtils.isBatchCq(topicConfig)) {\n                LOGGER.error(\"[BUG]The message is an inner batch but cq type is not batch cq\");\n                return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null));\n            }\n        }\n\n        long beginTime = this.getSystemClock().now();\n        CompletableFuture<PutMessageResult> putResultFuture = this.commitLog.asyncPutMessage(msg);\n\n        putResultFuture.thenAccept(result -> {\n            long elapsedTime = this.getSystemClock().now() - beginTime;\n            if (elapsedTime > 500) {\n                LOGGER.warn(\"DefaultMessageStore#putMessage: CommitLog#putMessage cost {}ms, topic={}, bodyLength={}\",\n                    elapsedTime, msg.getTopic(), msg.getBody().length);\n            }\n            this.storeStatsService.setPutMessageEntireTimeMax(elapsedTime);\n\n            if (null == result || !result.isOk()) {\n                this.storeStatsService.getPutMessageFailedTimes().add(1);\n            }\n        });\n\n        return putResultFuture;\n    }\n\n    @Override\n    public CompletableFuture<PutMessageResult> asyncPutMessages(MessageExtBatch messageExtBatch) {\n\n        for (PutMessageHook putMessageHook : putMessageHookList) {\n            PutMessageResult handleResult = putMessageHook.executeBeforePutMessage(messageExtBatch);\n            if (handleResult != null) {\n                return CompletableFuture.completedFuture(handleResult);\n            }\n        }\n\n        long beginTime = this.getSystemClock().now();\n        CompletableFuture<PutMessageResult> putResultFuture = this.commitLog.asyncPutMessages(messageExtBatch);\n\n        putResultFuture.thenAccept(result -> {\n            long eclipseTime = this.getSystemClock().now() - beginTime;\n            if (eclipseTime > 500) {\n                LOGGER.warn(\"not in lock eclipse time(ms)={}, bodyLength={}\", eclipseTime, messageExtBatch.getBody().length);\n            }\n            this.storeStatsService.setPutMessageEntireTimeMax(eclipseTime);\n\n            if (null == result || !result.isOk()) {\n                this.storeStatsService.getPutMessageFailedTimes().add(1);\n            }\n        });\n\n        return putResultFuture;\n    }\n\n    @Override\n    public PutMessageResult putMessage(MessageExtBrokerInner msg) {\n        return waitForPutResult(asyncPutMessage(msg));\n    }\n\n    @Override\n    public PutMessageResult putMessages(MessageExtBatch messageExtBatch) {\n        return waitForPutResult(asyncPutMessages(messageExtBatch));\n    }\n\n    private PutMessageResult waitForPutResult(CompletableFuture<PutMessageResult> putMessageResultFuture) {\n        try {\n            int putMessageTimeout =\n                Math.max(this.messageStoreConfig.getSyncFlushTimeout(),\n                    this.messageStoreConfig.getSlaveTimeout()) + 5000;\n            return putMessageResultFuture.get(putMessageTimeout, TimeUnit.MILLISECONDS);\n        } catch (ExecutionException | InterruptedException e) {\n            return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, null);\n        } catch (TimeoutException e) {\n            LOGGER.error(\"usually it will never timeout, putMessageTimeout is much bigger than slaveTimeout and \"\n                + \"flushTimeout so the result can be got anyway, but in some situations timeout will happen like full gc \"\n                + \"process hangs or other unexpected situations.\");\n            return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, null);\n        }\n    }\n\n    @Override\n    public boolean isOSPageCacheBusy() {\n        long begin = this.getCommitLog().getBeginTimeInLock();\n        long diff = this.systemClock.now() - begin;\n\n        return diff < 10000000\n            && diff > this.messageStoreConfig.getOsPageCacheBusyTimeOutMills();\n    }\n\n    @Override\n    public long lockTimeMills() {\n        return this.commitLog.lockTimeMills();\n    }\n\n    @Override\n    public long getMasterFlushedOffset() {\n        return this.masterFlushedOffset;\n    }\n\n    @Override\n    public void setMasterFlushedOffset(long masterFlushedOffset) {\n        this.masterFlushedOffset = masterFlushedOffset;\n        this.storeCheckpoint.setMasterFlushedOffset(masterFlushedOffset);\n    }\n\n    @Override\n    public long getBrokerInitMaxOffset() {\n        return this.brokerInitMaxOffset;\n    }\n\n    @Override\n    public void setBrokerInitMaxOffset(long brokerInitMaxOffset) {\n        this.brokerInitMaxOffset = brokerInitMaxOffset;\n    }\n\n    public SystemClock getSystemClock() {\n        return systemClock;\n    }\n\n    @Override\n    public CommitLog getCommitLog() {\n        return commitLog;\n    }\n\n    public void truncateDirtyFiles(long offsetToTruncate) throws RocksDBException {\n\n        LOGGER.info(\"truncate dirty files to {}\", offsetToTruncate);\n\n        if (offsetToTruncate >= this.getMaxPhyOffset()) {\n            LOGGER.info(\"no need to truncate files, truncate offset is {}, max physical offset is {}\", offsetToTruncate, this.getMaxPhyOffset());\n            return;\n        }\n\n        this.reputMessageService.shutdown();\n\n        long oldReputFromOffset = this.reputMessageService.getReputFromOffset();\n\n        // truncate consume queue\n        this.truncateDirtyLogicFiles(offsetToTruncate);\n\n        // truncate commitLog\n        this.commitLog.truncateDirtyFiles(offsetToTruncate);\n\n        this.recoverTopicQueueTable();\n\n        if (!messageStoreConfig.isEnableBuildConsumeQueueConcurrently()) {\n            this.reputMessageService = new ReputMessageService();\n        } else {\n            this.reputMessageService = new ConcurrentReputMessageService();\n        }\n\n        long resetReputOffset = Math.min(oldReputFromOffset, offsetToTruncate);\n\n        LOGGER.info(\"oldReputFromOffset is {}, reset reput from offset to {}\", oldReputFromOffset, resetReputOffset);\n\n        this.reputMessageService.setReputFromOffset(resetReputOffset);\n        this.reputMessageService.start();\n    }\n\n    @Override\n    public void truncateDirtyLogicFiles(long phyOffset) throws RocksDBException {\n        this.consumeQueueStore.truncateDirty(phyOffset);\n    }\n\n    @Override\n    public boolean truncateFiles(long offsetToTruncate) throws RocksDBException {\n        if (offsetToTruncate >= this.getMaxPhyOffset()) {\n            LOGGER.info(\"no need to truncate files, truncate offset is {}, max physical offset is {}\", offsetToTruncate, this.getMaxPhyOffset());\n            return true;\n        }\n\n        if (!isOffsetAligned(offsetToTruncate)) {\n            LOGGER.error(\"offset {} is not align, truncate failed, need manual fix\", offsetToTruncate);\n            return false;\n        }\n        truncateDirtyFiles(offsetToTruncate);\n        return true;\n    }\n\n    @Override\n    public boolean isOffsetAligned(long offset) {\n        SelectMappedBufferResult mappedBufferResult = this.getCommitLogData(offset);\n\n        if (mappedBufferResult == null) {\n            return true;\n        }\n\n        DispatchRequest dispatchRequest = this.commitLog.checkMessageAndReturnSize(mappedBufferResult.getByteBuffer(), true, false);\n        return dispatchRequest.isSuccess();\n    }\n\n    @Override\n    public GetMessageResult getMessage(final String group, final String topic, final int queueId, final long offset,\n        final int maxMsgNums, final MessageFilter messageFilter) {\n        return getMessage(group, topic, queueId, offset, maxMsgNums, MAX_PULL_MSG_SIZE, messageFilter);\n    }\n\n    @Override\n    public CompletableFuture<GetMessageResult> getMessageAsync(String group, String topic,\n        int queueId, long offset, int maxMsgNums, MessageFilter messageFilter) {\n        return CompletableFuture.completedFuture(getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter));\n    }\n\n    @Override\n    public GetMessageResult getMessage(final String group, final String topic, final int queueId, final long offset,\n        final int maxMsgNums, final int maxTotalMsgSize, final MessageFilter messageFilter) {\n        if (this.shutdown) {\n            LOGGER.warn(\"message store has shutdown, so getMessage is forbidden\");\n            return null;\n        }\n\n        if (!this.runningFlags.isReadable()) {\n            LOGGER.warn(\"message store is not readable, so getMessage is forbidden \" + this.runningFlags.getFlagBits());\n            return null;\n        }\n\n        Optional<TopicConfig> topicConfig = getTopicConfig(topic);\n        CleanupPolicy policy = CleanupPolicyUtils.getDeletePolicy(topicConfig);\n        //check request topic flag\n        if (Objects.equals(policy, CleanupPolicy.COMPACTION) && messageStoreConfig.isEnableCompaction()) {\n            return compactionStore.getMessage(group, topic, queueId, offset, maxMsgNums, maxTotalMsgSize);\n        } // else skip\n\n        long beginTime = this.getSystemClock().now();\n\n        GetMessageStatus status = GetMessageStatus.NO_MESSAGE_IN_QUEUE;\n        long nextBeginOffset = offset;\n        long minOffset = 0;\n        long maxOffset = 0;\n\n        GetMessageResult getResult = new GetMessageResult();\n        int filterMessageCount = 0;\n\n        final long maxOffsetPy = this.commitLog.getMaxOffset();\n\n        ConsumeQueueInterface consumeQueue = findConsumeQueue(topic, queueId);\n        if (consumeQueue != null) {\n            minOffset = consumeQueue.getMinOffsetInQueue();\n            maxOffset = consumeQueue.getMaxOffsetInQueue();\n\n            if (maxOffset == 0) {\n                status = GetMessageStatus.NO_MESSAGE_IN_QUEUE;\n                nextBeginOffset = nextOffsetCorrection(offset, 0);\n            } else if (offset < minOffset) {\n                status = GetMessageStatus.OFFSET_TOO_SMALL;\n                nextBeginOffset = nextOffsetCorrection(offset, minOffset);\n            } else if (offset == maxOffset) {\n                status = GetMessageStatus.OFFSET_OVERFLOW_ONE;\n                nextBeginOffset = nextOffsetCorrection(offset, offset);\n            } else if (offset > maxOffset) {\n                status = GetMessageStatus.OFFSET_OVERFLOW_BADLY;\n                nextBeginOffset = nextOffsetCorrection(offset, maxOffset);\n            } else {\n                final int maxFilterMessageSize = Math.max(this.messageStoreConfig.getMaxFilterMessageSize(), maxMsgNums * consumeQueue.getUnitSize());\n                final boolean diskFallRecorded = this.messageStoreConfig.isDiskFallRecorded();\n\n                long maxPullSize = Math.max(maxTotalMsgSize, 100);\n                if (maxPullSize > MAX_PULL_MSG_SIZE) {\n                    LOGGER.warn(\"The max pull size is too large maxPullSize={} topic={} queueId={}\", maxPullSize, topic, queueId);\n                    maxPullSize = MAX_PULL_MSG_SIZE;\n                }\n                status = GetMessageStatus.NO_MATCHED_MESSAGE;\n                long maxPhyOffsetPulling = 0;\n                int cqFileNum = 0;\n\n                while (getResult.getBufferTotalSize() <= 0\n                    && nextBeginOffset < maxOffset\n                    && cqFileNum++ < this.messageStoreConfig.getTravelCqFileNumWhenGetMessage()) {\n                    ReferredIterator<CqUnit> bufferConsumeQueue = null;\n\n                    try {\n                        bufferConsumeQueue = consumeQueue.iterateFrom(nextBeginOffset, maxMsgNums);\n\n                        if (bufferConsumeQueue == null) {\n                            status = GetMessageStatus.OFFSET_FOUND_NULL;\n                            nextBeginOffset = nextOffsetCorrection(nextBeginOffset, consumeQueue.rollNextFile(nextBeginOffset));\n                            LOGGER.warn(\"consumer request topic: \" + topic + \", offset: \" + offset + \", minOffset: \" + minOffset + \", maxOffset: \"\n                                + maxOffset + \", but access logic queue failed. Correct nextBeginOffset to \" + nextBeginOffset);\n                            break;\n                        }\n\n                        long nextPhyFileStartOffset = Long.MIN_VALUE;\n                        while (bufferConsumeQueue.hasNext()\n                            && nextBeginOffset < maxOffset) {\n                            CqUnit cqUnit = bufferConsumeQueue.next();\n                            long offsetPy = cqUnit.getPos();\n                            int sizePy = cqUnit.getSize();\n\n                            boolean isInMem = estimateInMemByCommitOffset(offsetPy, maxOffsetPy);\n\n                            if ((cqUnit.getQueueOffset() - offset) * consumeQueue.getUnitSize() >= maxFilterMessageSize) {\n                                break;\n                            }\n\n                            if (this.isTheBatchFull(sizePy, cqUnit.getBatchNum(), maxMsgNums, maxPullSize, getResult.getBufferTotalSize(), getResult.getMessageCount(), isInMem)) {\n                                break;\n                            }\n\n                            if (getResult.getBufferTotalSize() >= maxPullSize) {\n                                break;\n                            }\n\n                            maxPhyOffsetPulling = offsetPy;\n\n                            //Be careful, here should before the isTheBatchFull\n                            nextBeginOffset = cqUnit.getQueueOffset() + cqUnit.getBatchNum();\n\n                            if (nextPhyFileStartOffset != Long.MIN_VALUE) {\n                                if (offsetPy < nextPhyFileStartOffset) {\n                                    continue;\n                                }\n                            }\n\n                            if (messageFilter != null\n                                && !messageFilter.isMatchedByConsumeQueue(cqUnit.getValidTagsCodeAsLong(), cqUnit.getCqExtUnit())) {\n                                if (getResult.getBufferTotalSize() == 0) {\n                                    status = GetMessageStatus.NO_MATCHED_MESSAGE;\n                                }\n\n                                continue;\n                            }\n\n                            SelectMappedBufferResult selectResult = this.commitLog.getMessage(offsetPy, sizePy);\n                            if (null == selectResult) {\n                                if (getResult.getBufferTotalSize() == 0) {\n                                    status = GetMessageStatus.MESSAGE_WAS_REMOVING;\n                                }\n\n                                nextPhyFileStartOffset = this.commitLog.rollNextFile(offsetPy);\n                                continue;\n                            }\n\n                            if (messageStoreConfig.isColdDataFlowControlEnable() && !MixAll.isSysConsumerGroupPullMessage(group) && !selectResult.isInCache()) {\n                                getResult.setColdDataSum(getResult.getColdDataSum() + sizePy);\n                            }\n\n                            if (messageFilter != null\n                                && !messageFilter.isMatchedByCommitLog(selectResult.getByteBuffer().slice(), null)) {\n                                if (getResult.getBufferTotalSize() == 0) {\n                                    status = GetMessageStatus.NO_MATCHED_MESSAGE;\n                                }\n                                // release...\n                                selectResult.release();\n                                filterMessageCount++;\n                                continue;\n                            }\n                            this.storeStatsService.getGetMessageTransferredMsgCount().add(cqUnit.getBatchNum());\n                            getResult.addMessage(selectResult, cqUnit.getQueueOffset(), cqUnit.getBatchNum());\n                            status = GetMessageStatus.FOUND;\n                            nextPhyFileStartOffset = Long.MIN_VALUE;\n                        }\n                    } catch (RocksDBException e) {\n                        ERROR_LOG.error(\"getMessage Failed. cid: {}, topic: {}, queueId: {}, offset: {}, minOffset: {}, maxOffset: {}, {}\",\n                            group, topic, queueId, offset, minOffset, maxOffset, e.getMessage());\n                    } finally {\n                        if (bufferConsumeQueue != null) {\n                            bufferConsumeQueue.release();\n                        }\n                    }\n                }\n\n                if (diskFallRecorded) {\n                    long fallBehind = maxOffsetPy - maxPhyOffsetPulling;\n                    brokerStatsManager.recordDiskFallBehindSize(group, topic, queueId, fallBehind);\n                }\n\n                long diff = maxOffsetPy - maxPhyOffsetPulling;\n                long memory = (long) (StoreUtil.TOTAL_PHYSICAL_MEMORY_SIZE\n                    * (this.messageStoreConfig.getAccessMessageInMemoryMaxRatio() / 100.0));\n                getResult.setSuggestPullingFromSlave(diff > memory);\n            }\n        } else {\n            status = GetMessageStatus.NO_MATCHED_LOGIC_QUEUE;\n            nextBeginOffset = nextOffsetCorrection(offset, 0);\n        }\n\n        if (GetMessageStatus.FOUND == status) {\n            this.storeStatsService.getGetMessageTimesTotalFound().add(1);\n        } else {\n            this.storeStatsService.getGetMessageTimesTotalMiss().add(1);\n        }\n\n        if (this.messageStoreConfig.isDiskFallRecorded() && GetMessageStatus.OFFSET_OVERFLOW_ONE == status) {\n            brokerStatsManager.recordDiskFallBehindSize(group, topic, queueId, 0);\n            brokerStatsManager.recordDiskFallBehindTime(group, topic, queueId, 0);\n        }\n\n        long elapsedTime = this.getSystemClock().now() - beginTime;\n        this.storeStatsService.setGetMessageEntireTimeMax(elapsedTime);\n\n        // lazy init no data found.\n        if (getResult == null) {\n            getResult = new GetMessageResult(0);\n        }\n\n        getResult.setStatus(status);\n        getResult.setNextBeginOffset(nextBeginOffset);\n        getResult.setMaxOffset(maxOffset);\n        getResult.setMinOffset(minOffset);\n        getResult.setFilterMessageCount(filterMessageCount);\n        return getResult;\n    }\n\n    @Override\n    public CompletableFuture<GetMessageResult> getMessageAsync(String group, String topic,\n        int queueId, long offset, int maxMsgNums, int maxTotalMsgSize, MessageFilter messageFilter) {\n        return CompletableFuture.completedFuture(getMessage(group, topic, queueId, offset, maxMsgNums, maxTotalMsgSize, messageFilter));\n    }\n\n    @Override\n    public long getMaxOffsetInQueue(String topic, int queueId) throws ConsumeQueueException {\n        return getMaxOffsetInQueue(topic, queueId, true);\n    }\n\n    @Override\n    public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) throws ConsumeQueueException {\n        if (committed) {\n            ConsumeQueueInterface logic = this.getConsumeQueue(topic, queueId);\n            if (logic != null) {\n                return logic.getMaxOffsetInQueue();\n            }\n        } else {\n            Long offset = this.consumeQueueStore.getMaxOffset(topic, queueId);\n            if (offset != null) {\n                return offset;\n            }\n        }\n\n        return 0;\n    }\n\n    @Override\n    public long getMinOffsetInQueue(String topic, int queueId) {\n        try {\n            return this.consumeQueueStore.getMinOffsetInQueue(topic, queueId);\n        } catch (RocksDBException e) {\n            ERROR_LOG.error(\"getMinOffsetInQueue Failed. topic: {}, queueId: {}\", topic, queueId, e);\n            return -1;\n        }\n    }\n\n    @Override\n    public TimerMessageStore getTimerMessageStore() {\n        return this.timerMessageStore;\n    }\n\n    @Override\n    public TimerMessageRocksDBStore getTimerMessageRocksDBStore() {\n        return this.timerMessageRocksDBStore;\n    }\n\n    @Override\n    public TransMessageRocksDBStore getTransMessageRocksDBStore() {\n        return this.transMessageRocksDBStore;\n    }\n\n    @Override\n    public void setTimerMessageStore(TimerMessageStore timerMessageStore) {\n        this.timerMessageStore = timerMessageStore;\n    }\n\n    @Override\n    public void setTimerMessageRocksDBStore(TimerMessageRocksDBStore timerMessageRocksDBStore) {\n        this.timerMessageRocksDBStore = timerMessageRocksDBStore;\n    }\n\n    @Override\n    public void setTransMessageRocksDBStore(TransMessageRocksDBStore transMessageRocksDBStore) {\n        this.transMessageRocksDBStore = transMessageRocksDBStore;\n        // Register TransMessageRocksDBStore for commitlog dispatch if enabled\n        if (transMessageRocksDBStore != null && messageStoreConfig.isTransRocksDBEnable()) {\n            registerCommitLogDispatchStore(this.transMessageRocksDBStore);\n        }\n    }\n\n    /**\n     * Register a store that requires commitlog dispatch and recovery. Each store should register itself when loading.\n     *\n     * @param store the store to register\n     */\n    public void registerCommitLogDispatchStore(CommitLogDispatchStore store) {\n        if (store != null) {\n            commitLogDispatchStores.add(store);\n            LOGGER.info(\"Registered CommitLogDispatchStore: {}\", store.getClass().getSimpleName());\n        }\n    }\n\n    /**\n     * Get all registered CommitLogDispatchStore instances.\n     *\n     * @return list of registered stores\n     */\n    public List<CommitLogDispatchStore> getCommitLogDispatchStores() {\n        return commitLogDispatchStores;\n    }\n\n    @Override\n    public long getCommitLogOffsetInQueue(String topic, int queueId, long consumeQueueOffset) {\n        ConsumeQueueInterface consumeQueue = getConsumeQueue(topic, queueId);\n        if (consumeQueue != null) {\n            CqUnit cqUnit = consumeQueue.get(consumeQueueOffset);\n            if (cqUnit != null) {\n                return cqUnit.getPos();\n            }\n        }\n        return 0;\n    }\n\n    @Override\n    public long getOffsetInQueueByTime(String topic, int queueId, long timestamp) {\n        return this.getOffsetInQueueByTime(topic, queueId, timestamp, BoundaryType.LOWER);\n    }\n\n    @Override\n    public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) {\n        try {\n            return this.consumeQueueStore.getOffsetInQueueByTime(topic, queueId, timestamp, boundaryType);\n        } catch (RocksDBException e) {\n            ERROR_LOG.error(\"getOffsetInQueueByTime Failed. topic: {}, queueId: {}, timestamp: {} boundaryType: {}, {}\",\n                topic, queueId, timestamp, boundaryType, e.getMessage());\n        }\n        return 0;\n    }\n\n    @Override\n    public MessageExt lookMessageByOffset(long commitLogOffset) {\n        SelectMappedBufferResult sbr = this.commitLog.getMessage(commitLogOffset, 4);\n        if (null != sbr) {\n            try {\n                // 1 TOTALSIZE\n                int size = sbr.getByteBuffer().getInt();\n                return lookMessageByOffset(commitLogOffset, size);\n            } finally {\n                sbr.release();\n            }\n        }\n\n        return null;\n    }\n\n    @Override\n    public SelectMappedBufferResult selectOneMessageByOffset(long commitLogOffset) {\n        SelectMappedBufferResult sbr = this.commitLog.getMessage(commitLogOffset, 4);\n        if (null != sbr) {\n            try {\n                // 1 TOTALSIZE\n                int size = sbr.getByteBuffer().getInt();\n                return this.commitLog.getMessage(commitLogOffset, size);\n            } finally {\n                sbr.release();\n            }\n        }\n\n        return null;\n    }\n\n    @Override\n    public SelectMappedBufferResult selectOneMessageByOffset(long commitLogOffset, int msgSize) {\n        return this.commitLog.getMessage(commitLogOffset, msgSize);\n    }\n\n    @Override\n    public String getRunningDataInfo() {\n        return this.storeStatsService.toString();\n    }\n\n    public String getStorePathPhysic() {\n        String storePathPhysic;\n        if (DefaultMessageStore.this.getMessageStoreConfig().isEnableDLegerCommitLog()) {\n            storePathPhysic = ((DLedgerCommitLog) DefaultMessageStore.this.getCommitLog()).getdLedgerServer().getdLedgerConfig().getDataStorePath();\n        } else {\n            storePathPhysic = DefaultMessageStore.this.getMessageStoreConfig().getStorePathCommitLog();\n        }\n        return storePathPhysic;\n    }\n\n    public String getStorePathLogic() {\n        return StorePathConfigHelper.getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir());\n    }\n\n    public MessageArrivingListener getMessageArrivingListener() {\n        return messageArrivingListener;\n    }\n\n    @Override\n    public HashMap<String, String> getRuntimeInfo() {\n        HashMap<String, String> result = this.storeStatsService.getRuntimeInfo();\n\n        {\n            double minPhysicsUsedRatio = Double.MAX_VALUE;\n            String commitLogStorePath = getStorePathPhysic();\n            String[] paths = commitLogStorePath.trim().split(MixAll.MULTI_PATH_SPLITTER);\n            for (String clPath : paths) {\n                double physicRatio = UtilAll.isPathExists(clPath) ?\n                    UtilAll.getDiskPartitionSpaceUsedPercent(clPath) : -1;\n                result.put(RunningStats.commitLogDiskRatio.name() + \"_\" + clPath, String.valueOf(physicRatio));\n                minPhysicsUsedRatio = Math.min(minPhysicsUsedRatio, physicRatio);\n            }\n            result.put(RunningStats.commitLogDiskRatio.name(), String.valueOf(minPhysicsUsedRatio));\n        }\n\n        {\n            double logicsRatio = UtilAll.getDiskPartitionSpaceUsedPercent(getStorePathLogic());\n            result.put(RunningStats.consumeQueueDiskRatio.name(), String.valueOf(logicsRatio));\n        }\n\n        result.put(RunningStats.commitLogMinOffset.name(), String.valueOf(DefaultMessageStore.this.getMinPhyOffset()));\n        result.put(RunningStats.commitLogMaxOffset.name(), String.valueOf(DefaultMessageStore.this.getMaxPhyOffset()));\n\n        return result;\n    }\n\n    @Override\n    public long getMaxPhyOffset() {\n        return this.commitLog.getMaxOffset();\n    }\n\n    @Override\n    public long getMinPhyOffset() {\n        return this.commitLog.getMinOffset();\n    }\n\n    @Override\n    public long getLastFileFromOffset() {\n        return this.commitLog.getLastFileFromOffset();\n    }\n\n    @Override\n    public boolean getLastMappedFile(long startOffset) {\n        return this.commitLog.getLastMappedFile(startOffset);\n    }\n\n    @Override\n    public long getEarliestMessageTime(String topic, int queueId) {\n        ConsumeQueueInterface logicQueue = this.getConsumeQueue(topic, queueId);\n        if (logicQueue != null) {\n            Pair<CqUnit, Long> pair = logicQueue.getEarliestUnitAndStoreTime();\n            if (pair != null && pair.getObject2() != null) {\n                return pair.getObject2();\n            }\n        }\n\n        return -1;\n    }\n\n    @Override\n    public CompletableFuture<Long> getEarliestMessageTimeAsync(String topic, int queueId) {\n        return CompletableFuture.completedFuture(getEarliestMessageTime(topic, queueId));\n    }\n\n    @Override\n    public long getEarliestMessageTime() {\n        long minPhyOffset = this.getMinPhyOffset();\n        if (this.getCommitLog() instanceof DLedgerCommitLog) {\n            minPhyOffset += DLedgerEntry.BODY_OFFSET;\n        }\n        int size = MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION + 8;\n        if (NetworkUtil.validCommonInet6Address(this.brokerConfig.getBrokerIP1())) {\n            size = MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION + 20;\n        }\n        return this.getCommitLog().pickupStoreTimestamp(minPhyOffset, size);\n    }\n\n    @Override\n    public long getMessageStoreTimeStamp(String topic, int queueId, long consumeQueueOffset) {\n        ConsumeQueueInterface logicQueue = this.getConsumeQueue(topic, queueId);\n        if (logicQueue != null) {\n            Pair<CqUnit, Long> pair = logicQueue.getCqUnitAndStoreTime(consumeQueueOffset);\n            if (pair != null && pair.getObject2() != null) {\n                return pair.getObject2();\n            }\n        }\n        return -1;\n    }\n\n    @Override\n    public CompletableFuture<Long> getMessageStoreTimeStampAsync(String topic, int queueId,\n        long consumeQueueOffset) {\n        return CompletableFuture.completedFuture(getMessageStoreTimeStamp(topic, queueId, consumeQueueOffset));\n    }\n\n    @Override\n    public long getMessageTotalInQueue(String topic, int queueId) {\n        ConsumeQueueInterface logicQueue = this.getConsumeQueue(topic, queueId);\n        if (logicQueue != null) {\n            return logicQueue.getMessageTotalInQueue();\n        }\n\n        return 0;\n    }\n\n    @Override\n    public SelectMappedBufferResult getCommitLogData(final long offset) {\n        if (this.shutdown) {\n            LOGGER.warn(\"message store has shutdown, so getPhyQueueData is forbidden\");\n            return null;\n        }\n\n        return this.commitLog.getData(offset);\n    }\n\n    @Override\n    public List<SelectMappedBufferResult> getBulkCommitLogData(final long offset, final int size) {\n        if (this.shutdown) {\n            LOGGER.warn(\"message store has shutdown, so getBulkCommitLogData is forbidden\");\n            return null;\n        }\n\n        return this.commitLog.getBulkData(offset, size);\n    }\n\n    @Override\n    public boolean appendToCommitLog(long startOffset, byte[] data, int dataStart, int dataLength) {\n        if (this.shutdown) {\n            LOGGER.warn(\"message store has shutdown, so appendToCommitLog is forbidden\");\n            return false;\n        }\n\n        boolean result = this.commitLog.appendData(startOffset, data, dataStart, dataLength);\n        if (result) {\n            this.reputMessageService.wakeup();\n        } else {\n            LOGGER.error(\n                \"DefaultMessageStore#appendToCommitLog: failed to append data to commitLog, physical offset={}, data \"\n                    + \"length={}\", startOffset, data.length);\n        }\n\n        return result;\n    }\n\n    @Override\n    public void executeDeleteFilesManually() {\n        this.cleanCommitLogService.executeDeleteFilesManually();\n    }\n\n    @Override\n    public QueryMessageResult queryMessage(String topic, String key, int maxNum, long begin, long end) {\n        QueryMessageResult queryMessageResult = new QueryMessageResult();\n\n        long lastQueryMsgTime = end;\n\n        for (int i = 0; i < 3; i++) {\n            QueryOffsetResult queryOffsetResult = null;\n            if (messageStoreConfig.isIndexFileReadEnable()) {\n                queryOffsetResult = this.indexService.queryOffset(topic, key, maxNum, begin, lastQueryMsgTime, null);\n                LOGGER.debug(\"indexService query Message queryOffsetResult : {}\", JSON.toJSONString(queryOffsetResult));\n            } else if (messageStoreConfig.isIndexRocksDBEnable()) {\n                queryOffsetResult = this.indexRocksDBStore.queryOffset(topic, key, maxNum, begin, lastQueryMsgTime, null, null);\n                LOGGER.debug(\"indexRocksDBStore query Message queryOffsetResult : {}\", JSON.toJSONString(queryOffsetResult));\n            }\n            if (null == queryOffsetResult || CollectionUtils.isEmpty(queryOffsetResult.getPhyOffsets())) {\n                break;\n            }\n\n            Collections.sort(queryOffsetResult.getPhyOffsets());\n\n            queryMessageResult.setIndexLastUpdatePhyoffset(queryOffsetResult.getIndexLastUpdatePhyoffset());\n            queryMessageResult.setIndexLastUpdateTimestamp(queryOffsetResult.getIndexLastUpdateTimestamp());\n\n            for (int m = 0; m < queryOffsetResult.getPhyOffsets().size(); m++) {\n                long offset = queryOffsetResult.getPhyOffsets().get(m);\n\n                try {\n                    MessageExt msg = this.lookMessageByOffset(offset);\n                    if (0 == m) {\n                        lastQueryMsgTime = msg.getStoreTimestamp();\n                    }\n\n                    SelectMappedBufferResult result = this.commitLog.getData(offset, false);\n                    if (result != null) {\n                        int size = result.getByteBuffer().getInt(0);\n                        result.getByteBuffer().limit(size);\n                        result.setSize(size);\n                        queryMessageResult.addMessage(result);\n                    }\n                } catch (Exception e) {\n                    LOGGER.error(\"queryMessage exception\", e);\n                }\n            }\n\n            if (queryMessageResult.getBufferTotalSize() > 0) {\n                break;\n            }\n\n            if (lastQueryMsgTime < begin) {\n                break;\n            }\n        }\n        return queryMessageResult;\n    }\n\n    @Override\n    public QueryMessageResult queryMessage(String topic, String key, int maxNum, long begin, long end, String indexType,\n        String lastKey) {\n        QueryMessageResult queryMessageResult = new QueryMessageResult();\n        long lastQueryMsgTime = end;\n        for (int i = 0; i < 3; i++) {\n            QueryOffsetResult queryOffsetResult = null;\n            if (messageStoreConfig.isIndexFileReadEnable()) {\n                queryOffsetResult = this.indexService.queryOffset(topic, key, maxNum, begin, lastQueryMsgTime, indexType);\n                LOGGER.debug(\"indexService query Message queryOffsetResult : {}\", JSON.toJSONString(queryOffsetResult));\n            } else if (messageStoreConfig.isIndexRocksDBEnable()) {\n                queryOffsetResult = this.indexRocksDBStore.queryOffset(topic, key, maxNum, begin, lastQueryMsgTime, indexType, lastKey);\n                LOGGER.debug(\"indexRocksDBStore query Message queryOffsetResult : {}\", JSON.toJSONString(queryOffsetResult));\n            }\n            if (null == queryOffsetResult || CollectionUtils.isEmpty(queryOffsetResult.getPhyOffsets())) {\n                break;\n            }\n            Collections.sort(queryOffsetResult.getPhyOffsets());\n            queryMessageResult.setIndexLastUpdatePhyoffset(queryOffsetResult.getIndexLastUpdatePhyoffset());\n            queryMessageResult.setIndexLastUpdateTimestamp(queryOffsetResult.getIndexLastUpdateTimestamp());\n            for (int m = 0; m < queryOffsetResult.getPhyOffsets().size(); m++) {\n                long offset = queryOffsetResult.getPhyOffsets().get(m);\n                try {\n                    MessageExt msg = this.lookMessageByOffset(offset);\n                    if (0 == m && null != msg) {\n                        lastQueryMsgTime = msg.getStoreTimestamp();\n                    }\n                    SelectMappedBufferResult result = this.commitLog.getData(offset, false);\n                    if (result != null) {\n                        int size = result.getByteBuffer().getInt(0);\n                        result.getByteBuffer().limit(size);\n                        result.setSize(size);\n                        queryMessageResult.addMessage(result);\n                    }\n                } catch (Exception e) {\n                    LOGGER.error(\"queryMessage exception\", e);\n                }\n            }\n\n            if (queryMessageResult.getBufferTotalSize() > 0) {\n                break;\n            }\n\n            if (lastQueryMsgTime < begin) {\n                break;\n            }\n        }\n        return queryMessageResult;\n    }\n\n    @Override public CompletableFuture<QueryMessageResult> queryMessageAsync(String topic, String key,\n        int maxNum, long begin, long end) {\n        return CompletableFuture.completedFuture(queryMessage(topic, key, maxNum, begin, end));\n    }\n\n    @Override\n    public CompletableFuture<QueryMessageResult> queryMessageAsync(String topic, String key,\n        int maxNum, long begin, long end, String indexType, String lastKey) {\n        return CompletableFuture.completedFuture(queryMessage(topic, key, maxNum, begin, end, indexType, lastKey));\n    }\n\n    @Override\n    public void updateHaMasterAddress(String newAddr) {\n        if (this.haService != null) {\n            this.haService.updateHaMasterAddress(newAddr);\n        }\n    }\n\n    @Override\n    public void updateMasterAddress(String newAddr) {\n        if (this.haService != null) {\n            this.haService.updateMasterAddress(newAddr);\n        }\n        if (this.compactionService != null) {\n            this.compactionService.updateMasterAddress(newAddr);\n        }\n    }\n\n    @Override\n    public void setAliveReplicaNumInGroup(int aliveReplicaNums) {\n        this.aliveReplicasNum = aliveReplicaNums;\n    }\n\n    @Override\n    public void wakeupHAClient() {\n        if (this.haService != null) {\n            this.haService.getHAClient().wakeup();\n        }\n    }\n\n    @Override\n    public int getAliveReplicaNumInGroup() {\n        return this.aliveReplicasNum;\n    }\n\n    @Override\n    public long slaveFallBehindMuch() {\n        if (this.haService == null || this.messageStoreConfig.isDuplicationEnable() || this.messageStoreConfig.isEnableDLegerCommitLog()) {\n            LOGGER.warn(\"haServer is null or duplication is enable or enableDLegerCommitLog is true\");\n            return -1;\n        } else {\n            return this.commitLog.getMaxOffset() - this.haService.getPush2SlaveMaxOffset().get();\n        }\n\n    }\n\n    @Override\n    public long now() {\n        return this.systemClock.now();\n    }\n\n    /**\n     * Lazy clean queue offset table. If offset table is cleaned, and old messages are dispatching after the old consume\n     * queue is cleaned, consume queue will be created with old offset, then later message with new offset table can not\n     * be dispatched to consume queue.\n     */\n    @Override\n    public int deleteTopics(final Set<String> deleteTopics) {\n        if (deleteTopics == null || deleteTopics.isEmpty()) {\n            return 0;\n        }\n\n        int deleteCount = 0;\n        for (String topic : deleteTopics) {\n            if (!consumeQueueStore.deleteTopic(topic)) {\n                continue;\n            }\n\n            if (this.brokerConfig.isAutoDeleteUnusedStats()) {\n                if (!MixAll.isLmq(topic)) {\n                    this.brokerStatsManager.onTopicDeleted(topic);\n                }\n            }\n\n            // destroy consume queue dir\n            String consumeQueueDir = StorePathConfigHelper.getStorePathConsumeQueue(\n                this.messageStoreConfig.getStorePathRootDir()) + File.separator + topic;\n            String consumeQueueExtDir = StorePathConfigHelper.getStorePathConsumeQueueExt(\n                this.messageStoreConfig.getStorePathRootDir()) + File.separator + topic;\n            String batchConsumeQueueDir = StorePathConfigHelper.getStorePathBatchConsumeQueue(\n                this.messageStoreConfig.getStorePathRootDir()) + File.separator + topic;\n\n            UtilAll.deleteEmptyDirectory(new File(consumeQueueDir));\n            UtilAll.deleteEmptyDirectory(new File(consumeQueueExtDir));\n            UtilAll.deleteEmptyDirectory(new File(batchConsumeQueueDir));\n\n            LOGGER.info(\"DeleteTopic: Topic has been destroyed, topic={}\", topic);\n            deleteCount++;\n        }\n        return deleteCount;\n    }\n\n    @Override\n    public int cleanUnusedTopic(final Set<String> retainTopics) {\n        Set<String> consumeQueueTopicSet = this.getConsumeQueueTable().keySet();\n        int deleteCount = 0;\n        for (String topicName : Sets.difference(consumeQueueTopicSet, retainTopics)) {\n            if (retainTopics.contains(topicName) ||\n                TopicValidator.isSystemTopic(topicName) ||\n                MixAll.isLmq(topicName)) {\n                continue;\n            }\n            deleteCount += this.deleteTopics(Sets.newHashSet(topicName));\n        }\n        return deleteCount;\n    }\n\n    @Override\n    public void cleanExpiredConsumerQueue() {\n        long minCommitLogOffset = this.commitLog.getMinOffset();\n\n        this.consumeQueueStore.cleanExpired(minCommitLogOffset);\n    }\n\n    public Map<String, Long> getMessageIds(final String topic, final int queueId, long minOffset, long maxOffset,\n        SocketAddress storeHost) {\n        Map<String, Long> messageIds = new HashMap<>();\n        if (this.shutdown) {\n            return messageIds;\n        }\n\n        ConsumeQueueInterface consumeQueue = findConsumeQueue(topic, queueId);\n        if (consumeQueue != null) {\n            minOffset = Math.max(minOffset, consumeQueue.getMinOffsetInQueue());\n            maxOffset = Math.min(maxOffset, consumeQueue.getMaxOffsetInQueue());\n\n            if (maxOffset == 0) {\n                return messageIds;\n            }\n\n            long nextOffset = minOffset;\n            while (nextOffset < maxOffset) {\n                ReferredIterator<CqUnit> bufferConsumeQueue = consumeQueue.iterateFrom(nextOffset);\n                try {\n                    if (bufferConsumeQueue != null && bufferConsumeQueue.hasNext()) {\n                        while (bufferConsumeQueue.hasNext()) {\n                            CqUnit cqUnit = bufferConsumeQueue.next();\n                            long offsetPy = cqUnit.getPos();\n                            InetSocketAddress inetSocketAddress = (InetSocketAddress) storeHost;\n                            int msgIdLength = (inetSocketAddress.getAddress() instanceof Inet6Address) ? 16 + 4 + 8 : 4 + 4 + 8;\n                            final ByteBuffer msgIdMemory = ByteBuffer.allocate(msgIdLength);\n                            String msgId =\n                                MessageDecoder.createMessageId(msgIdMemory, MessageExt.socketAddress2ByteBuffer(storeHost), offsetPy);\n                            messageIds.put(msgId, cqUnit.getQueueOffset());\n                            nextOffset = cqUnit.getQueueOffset() + cqUnit.getBatchNum();\n                            if (nextOffset >= maxOffset) {\n                                return messageIds;\n                            }\n                        }\n                    } else {\n                        return messageIds;\n                    }\n                } finally {\n                    if (bufferConsumeQueue != null) {\n                        bufferConsumeQueue.release();\n                    }\n                }\n            }\n        }\n        return messageIds;\n    }\n\n    @Override\n    @Deprecated\n    public boolean checkInDiskByConsumeOffset(final String topic, final int queueId, long consumeOffset) {\n\n        final long maxOffsetPy = this.commitLog.getMaxOffset();\n\n        ConsumeQueueInterface consumeQueue = getConsumeQueue(topic, queueId);\n        if (consumeQueue != null) {\n            CqUnit cqUnit = consumeQueue.get(consumeOffset);\n\n            if (cqUnit != null) {\n                long offsetPy = cqUnit.getPos();\n                return !estimateInMemByCommitOffset(offsetPy, maxOffsetPy);\n            } else {\n                return false;\n            }\n        }\n        return false;\n    }\n\n    @Override\n    public boolean checkInMemByConsumeOffset(final String topic, final int queueId, long consumeOffset, int batchSize) {\n        ConsumeQueueInterface consumeQueue = getConsumeQueue(topic, queueId);\n        if (consumeQueue != null) {\n            CqUnit firstCQItem = consumeQueue.get(consumeOffset);\n            if (firstCQItem == null) {\n                return false;\n            }\n            long startOffsetPy = firstCQItem.getPos();\n            if (batchSize <= 1) {\n                int size = firstCQItem.getSize();\n                return checkInMemByCommitOffset(startOffsetPy, size);\n            }\n\n            CqUnit lastCQItem = consumeQueue.get(consumeOffset + batchSize);\n            if (lastCQItem == null) {\n                int size = firstCQItem.getSize();\n                return checkInMemByCommitOffset(startOffsetPy, size);\n            }\n            long endOffsetPy = lastCQItem.getPos();\n            int size = (int) (endOffsetPy - startOffsetPy) + lastCQItem.getSize();\n            return checkInMemByCommitOffset(startOffsetPy, size);\n        }\n        return false;\n    }\n\n    @Override\n    public boolean checkInStoreByConsumeOffset(String topic, int queueId, long consumeOffset) {\n        long commitLogOffset = getCommitLogOffsetInQueue(topic, queueId, consumeOffset);\n        return checkInDiskByCommitOffset(commitLogOffset);\n    }\n\n    @Override\n    public long dispatchBehindBytes() {\n        return this.reputMessageService.behind();\n    }\n\n    @Override\n    public long dispatchBehindMilliseconds() {\n        return this.reputMessageService.behindMs();\n    }\n\n    @Override\n    public long flushBehindBytes() {\n        if (this.messageStoreConfig.isTransientStorePoolEnable()) {\n            return this.commitLog.remainHowManyDataToCommit() + this.commitLog.remainHowManyDataToFlush();\n        } else {\n            return this.commitLog.remainHowManyDataToFlush();\n        }\n    }\n\n    @Override\n    public long flush() {\n        return this.commitLog.flush();\n    }\n\n    @Override\n    public long getFlushedWhere() {\n        return this.commitLog.getFlushedWhere();\n    }\n\n    // Fetch and compute the newest confirmOffset.\n    // Even if it is just inited.\n    @Override\n    public long getConfirmOffset() {\n        return this.commitLog.getConfirmOffset();\n    }\n\n    // Fetch the original confirmOffset's value.\n    // Without checking and re-computing.\n    public long getConfirmOffsetDirectly() {\n        return this.commitLog.getConfirmOffsetDirectly();\n    }\n\n    @Override\n    public void setConfirmOffset(long phyOffset) {\n        this.commitLog.setConfirmOffset(phyOffset);\n    }\n\n    @Override\n    public byte[] calcDeltaChecksum(long from, long to) {\n        if (from < 0 || to <= from) {\n            return new byte[0];\n        }\n\n        int size = (int) (to - from);\n\n        if (size > this.messageStoreConfig.getMaxChecksumRange()) {\n            LOGGER.error(\"Checksum range from {}, size {} exceeds threshold {}\", from, size, this.messageStoreConfig.getMaxChecksumRange());\n            return null;\n        }\n\n        List<MessageExt> msgList = new ArrayList<>();\n        List<SelectMappedBufferResult> bufferResultList = this.getBulkCommitLogData(from, size);\n        if (bufferResultList.isEmpty()) {\n            return new byte[0];\n        }\n\n        for (SelectMappedBufferResult bufferResult : bufferResultList) {\n            msgList.addAll(MessageDecoder.decodesBatch(bufferResult.getByteBuffer(), true, false, false));\n            bufferResult.release();\n        }\n\n        if (msgList.isEmpty()) {\n            return new byte[0];\n        }\n\n        ByteBuffer byteBuffer = ByteBuffer.allocate(size);\n        for (MessageExt msg : msgList) {\n            try {\n                byteBuffer.put(MessageDecoder.encodeUniquely(msg, false));\n            } catch (IOException ignore) {\n            }\n        }\n\n        return Hashing.murmur3_128().hashBytes(byteBuffer.array()).asBytes();\n    }\n\n    @Override\n    public void setPhysicalOffset(long phyOffset) {\n        this.commitLog.setMappedFileQueueOffset(phyOffset);\n    }\n\n    @Override\n    public boolean isMappedFilesEmpty() {\n        return this.commitLog.isMappedFilesEmpty();\n    }\n\n    @Override\n    public MessageExt lookMessageByOffset(long commitLogOffset, int size) {\n        SelectMappedBufferResult sbr = this.commitLog.getMessage(commitLogOffset, size);\n        if (null != sbr) {\n            try {\n                return MessageDecoder.decode(sbr.getByteBuffer(), true, false);\n            } finally {\n                sbr.release();\n            }\n        }\n\n        return null;\n    }\n\n    @Override\n    public ConsumeQueueInterface findConsumeQueue(String topic, int queueId) {\n        return this.consumeQueueStore.findOrCreateConsumeQueue(topic, queueId);\n    }\n\n    private long nextOffsetCorrection(long oldOffset, long newOffset) {\n        long nextOffset = oldOffset;\n        if (this.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE ||\n            this.getMessageStoreConfig().isOffsetCheckInSlave()) {\n            nextOffset = newOffset;\n        }\n        return nextOffset;\n    }\n\n    private boolean estimateInMemByCommitOffset(long offsetPy, long maxOffsetPy) {\n        long memory = (long) (StoreUtil.TOTAL_PHYSICAL_MEMORY_SIZE * (this.messageStoreConfig.getAccessMessageInMemoryMaxRatio() / 100.0));\n        return (maxOffsetPy - offsetPy) <= memory;\n    }\n\n    private boolean checkInMemByCommitOffset(long offsetPy, int size) {\n        SelectMappedBufferResult message = this.commitLog.getMessage(offsetPy, size);\n        if (message != null) {\n            try {\n                return message.isInMem();\n            } finally {\n                message.release();\n            }\n        }\n        return false;\n    }\n\n    public boolean checkInDiskByCommitOffset(long offsetPy) {\n        return offsetPy >= commitLog.getMinOffset();\n    }\n\n    /**\n     * The ratio val is estimated by the experiment and experience so that the result is not high accurate for different\n     * business\n     *\n     * @return\n     */\n    public boolean checkInColdAreaByCommitOffset(long offsetPy, long maxOffsetPy) {\n        long memory = (long) (StoreUtil.TOTAL_PHYSICAL_MEMORY_SIZE * (this.messageStoreConfig.getAccessMessageInMemoryHotRatio() / 100.0));\n        return (maxOffsetPy - offsetPy) > memory;\n    }\n\n    private boolean isTheBatchFull(int sizePy, int unitBatchNum, int maxMsgNums, long maxMsgSize, int bufferTotal,\n        int messageTotal, boolean isInMem) {\n\n        if (0 == bufferTotal || 0 == messageTotal) {\n            return false;\n        }\n\n        if (messageTotal + unitBatchNum > maxMsgNums) {\n            return true;\n        }\n\n        if (bufferTotal + sizePy > maxMsgSize) {\n            return true;\n        }\n\n        if (isInMem) {\n            if ((bufferTotal + sizePy) > this.messageStoreConfig.getMaxTransferBytesOnMessageInMemory()) {\n                return true;\n            }\n\n            return messageTotal > this.messageStoreConfig.getMaxTransferCountOnMessageInMemory() - 1;\n        } else {\n            if ((bufferTotal + sizePy) > this.messageStoreConfig.getMaxTransferBytesOnMessageInDisk()) {\n                return true;\n            }\n\n            return messageTotal > this.messageStoreConfig.getMaxTransferCountOnMessageInDisk() - 1;\n        }\n    }\n\n    private void deleteFile(final String fileName) {\n        File file = new File(fileName);\n        boolean result = file.delete();\n        LOGGER.info(fileName + (result ? \" delete OK\" : \" delete Failed\"));\n    }\n\n    /**\n     * @throws IOException\n     */\n    private void createTempFile() throws IOException {\n        String fileName = StorePathConfigHelper.getAbortFile(this.messageStoreConfig.getStorePathRootDir());\n        File file = new File(fileName);\n        UtilAll.ensureDirOK(file.getParent());\n        boolean result = file.createNewFile();\n        LOGGER.info(fileName + (result ? \" create OK\" : \" already exists\"));\n        MixAll.string2File(Long.toString(MixAll.getPID()), file.getAbsolutePath());\n    }\n\n    private void addScheduleTask() {\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                DefaultMessageStore.this.cleanFilesPeriodically();\n            }\n        }, 1000 * 60, this.messageStoreConfig.getCleanResourceInterval(), TimeUnit.MILLISECONDS);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                DefaultMessageStore.this.checkSelf();\n            }\n        }, 1, 10, TimeUnit.MINUTES);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                if (DefaultMessageStore.this.getMessageStoreConfig().isDebugLockEnable()) {\n                    try {\n                        if (DefaultMessageStore.this.commitLog.getBeginTimeInLock() != 0) {\n                            long lockTime = System.currentTimeMillis() - DefaultMessageStore.this.commitLog.getBeginTimeInLock();\n                            if (lockTime > 1000 && lockTime < 10000000) {\n\n                                String stack = UtilAll.jstack();\n                                final String fileName = System.getProperty(\"user.home\") + File.separator + \"debug/lock/stack-\"\n                                    + DefaultMessageStore.this.commitLog.getBeginTimeInLock() + \"-\" + lockTime;\n                                MixAll.string2FileNotSafe(stack, fileName);\n                            }\n                        }\n                    } catch (Exception e) {\n                    }\n                }\n            }\n        }, 1, 1, TimeUnit.SECONDS);\n\n        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                DefaultMessageStore.this.storeCheckpoint.flush();\n            }\n        }, 1, 1, TimeUnit.SECONDS);\n    }\n\n    private void initializeHAService() {\n        if (!this.messageStoreConfig.isEnableDLegerCommitLog() && !this.messageStoreConfig.isDuplicationEnable()) {\n            if (brokerConfig.isEnableControllerMode()) {\n                this.haService = new AutoSwitchHAService();\n                LOGGER.warn(\"Load AutoSwitch HA Service: {}\", AutoSwitchHAService.class.getSimpleName());\n            } else {\n                this.haService = ServiceProvider.loadClass(HAService.class);\n                if (null == this.haService) {\n                    this.haService = new DefaultHAService();\n                    LOGGER.warn(\"Load default HA Service: {}\", DefaultHAService.class.getSimpleName());\n                }\n            }\n        }\n    }\n\n    private void cleanFilesPeriodically() {\n        this.cleanCommitLogService.run();\n    }\n\n    private void checkSelf() {\n        this.commitLog.checkSelf();\n        this.consumeQueueStore.checkSelf();\n    }\n\n    private boolean isTempFileExist() {\n        String fileName = StorePathConfigHelper.getAbortFile(this.messageStoreConfig.getStorePathRootDir());\n        File file = new File(fileName);\n        return file.exists();\n    }\n\n    @Override\n    public long getTimingMessageCount(String topic) {\n        if (null == timerMessageStore) {\n            return 0L;\n        } else {\n            return timerMessageStore.getTimerMetrics().getTimingCount(topic);\n        }\n    }\n\n    @Override\n    public MessageStoreConfig getMessageStoreConfig() {\n        return messageStoreConfig;\n    }\n\n    @Override\n    public TransientStorePool getTransientStorePool() {\n        return transientStorePool;\n    }\n\n    @Override\n    public void recoverTopicQueueTable() {\n        long minPhyOffset = this.commitLog.getMinOffset();\n        this.consumeQueueStore.recoverOffsetTable(minPhyOffset);\n    }\n\n    @Override\n    public AllocateMappedFileService getAllocateMappedFileService() {\n        return allocateMappedFileService;\n    }\n\n    @Override\n    public StoreStatsService getStoreStatsService() {\n        return storeStatsService;\n    }\n\n    public RunningFlags getAccessRights() {\n        return runningFlags;\n    }\n\n    public ConcurrentMap<String, ConcurrentMap<Integer, ConsumeQueueInterface>> getConsumeQueueTable() {\n        return consumeQueueStore.getConsumeQueueTable();\n    }\n\n    @Override\n    public StoreCheckpoint getStoreCheckpoint() {\n        return storeCheckpoint;\n    }\n\n    @Override\n    public HAService getHaService() {\n        return haService;\n    }\n\n    @Override\n    public RunningFlags getRunningFlags() {\n        return runningFlags;\n    }\n\n    public void doDispatch(DispatchRequest req) throws RocksDBException {\n        for (CommitLogDispatcher dispatcher : this.dispatcherList) {\n            dispatcher.dispatch(req);\n        }\n    }\n\n    /**\n     * @param dispatchRequest\n     * @throws RocksDBException only in rocksdb mode\n     */\n    protected void putMessagePositionInfo(DispatchRequest dispatchRequest) throws RocksDBException {\n        this.consumeQueueStore.putMessagePositionInfoWrapper(dispatchRequest);\n    }\n\n    @Override\n    public DispatchRequest checkMessageAndReturnSize(final ByteBuffer byteBuffer, final boolean checkCRC,\n        final boolean checkDupInfo, final boolean readBody) {\n        return this.commitLog.checkMessageAndReturnSize(byteBuffer, checkCRC, checkDupInfo, readBody);\n    }\n\n    @Override\n    public long getStateMachineVersion() {\n        return stateMachineVersion;\n    }\n\n    public void setStateMachineVersion(long stateMachineVersion) {\n        this.stateMachineVersion = stateMachineVersion;\n    }\n\n    public BrokerStatsManager getBrokerStatsManager() {\n        return brokerStatsManager;\n    }\n\n    public BrokerConfig getBrokerConfig() {\n        return brokerConfig;\n    }\n\n    public int remainTransientStoreBufferNumbs() {\n        if (this.isTransientStorePoolEnable()) {\n            return this.transientStorePool.availableBufferNums();\n        }\n        return Integer.MAX_VALUE;\n    }\n\n    @Override\n    public boolean isTransientStorePoolDeficient() {\n        return remainTransientStoreBufferNumbs() == 0;\n    }\n\n    @Override\n    public long remainHowManyDataToCommit() {\n        return this.commitLog.remainHowManyDataToCommit();\n    }\n\n    @Override\n    public long remainHowManyDataToFlush() {\n        return this.commitLog.remainHowManyDataToFlush();\n    }\n\n    @Override\n    public LinkedList<CommitLogDispatcher> getDispatcherList() {\n        return this.dispatcherList;\n    }\n\n    @Override\n    public void addDispatcher(CommitLogDispatcher dispatcher) {\n        this.dispatcherList.add(dispatcher);\n    }\n\n    @Override\n    public void setMasterStoreInProcess(MessageStore masterStoreInProcess) {\n        this.masterStoreInProcess = masterStoreInProcess;\n    }\n\n    @Override\n    public MessageStore getMasterStoreInProcess() {\n        return this.masterStoreInProcess;\n    }\n\n    @Override\n    public boolean getData(long offset, int size, ByteBuffer byteBuffer) {\n        return this.commitLog.getData(offset, size, byteBuffer);\n    }\n\n    @Override\n    public ConsumeQueueInterface getConsumeQueue(String topic, int queueId) {\n        return this.consumeQueueStore.getConsumeQueue(topic, queueId);\n    }\n\n    @Override\n    public void unlockMappedFile(final MappedFile mappedFile) {\n        this.scheduledExecutorService.schedule(new Runnable() {\n            @Override\n            public void run() {\n                mappedFile.munlock();\n            }\n        }, 6, TimeUnit.SECONDS);\n    }\n\n    @Override\n    public PerfCounter.Ticks getPerfCounter() {\n        return perfs;\n    }\n\n    @Override\n    public ConsumeQueueStoreInterface getQueueStore() {\n        return consumeQueueStore;\n    }\n\n    @Override\n    public void onCommitLogAppend(MessageExtBrokerInner msg, AppendMessageResult result, MappedFile commitLogFile) {\n        // empty\n    }\n\n    @Override\n    public void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile,\n        boolean isRecover, boolean isFileEnd) throws RocksDBException {\n        if (doDispatch && !isFileEnd) {\n            this.doDispatch(dispatchRequest);\n        }\n    }\n\n    @Override\n    public boolean isSyncDiskFlush() {\n        return FlushDiskType.SYNC_FLUSH == this.getMessageStoreConfig().getFlushDiskType();\n    }\n\n    @Override\n    public boolean isSyncMaster() {\n        return BrokerRole.SYNC_MASTER == this.getMessageStoreConfig().getBrokerRole();\n    }\n\n    @Override\n    public void assignOffset(MessageExtBrokerInner msg) throws RocksDBException {\n        final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag());\n\n        if (tranType == MessageSysFlag.TRANSACTION_NOT_TYPE || tranType == MessageSysFlag.TRANSACTION_COMMIT_TYPE) {\n            this.consumeQueueStore.assignQueueOffset(msg);\n        }\n    }\n\n    @Override\n    public void increaseOffset(MessageExtBrokerInner msg, short messageNum) {\n        final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag());\n\n        if (tranType == MessageSysFlag.TRANSACTION_NOT_TYPE || tranType == MessageSysFlag.TRANSACTION_COMMIT_TYPE) {\n            this.consumeQueueStore.increaseQueueOffset(msg, messageNum);\n        }\n    }\n\n    public ConcurrentMap<String, TopicConfig> getTopicConfigs() {\n        return this.topicConfigTable;\n    }\n\n    public Optional<TopicConfig> getTopicConfig(String topic) {\n        if (this.topicConfigTable == null) {\n            return Optional.empty();\n        }\n\n        return Optional.ofNullable(this.topicConfigTable.get(topic));\n    }\n\n    public BrokerIdentity getBrokerIdentity() {\n        if (messageStoreConfig.isEnableDLegerCommitLog()) {\n            return new BrokerIdentity(\n                brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName(),\n                Integer.parseInt(messageStoreConfig.getdLegerSelfId().substring(1)), brokerConfig.isInBrokerContainer());\n        } else {\n            return new BrokerIdentity(\n                brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName(),\n                brokerConfig.getBrokerId(), brokerConfig.isInBrokerContainer());\n        }\n    }\n\n    class CommitLogDispatcherBuildConsumeQueue implements CommitLogDispatcher {\n\n        @Override\n        public void dispatch(DispatchRequest request) throws RocksDBException {\n            final int tranType = MessageSysFlag.getTransactionValue(request.getSysFlag());\n            switch (tranType) {\n                case MessageSysFlag.TRANSACTION_NOT_TYPE:\n                case MessageSysFlag.TRANSACTION_COMMIT_TYPE:\n                    putMessagePositionInfo(request);\n                    break;\n                case MessageSysFlag.TRANSACTION_PREPARED_TYPE:\n                case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:\n                    break;\n            }\n        }\n    }\n\n    class CommitLogDispatcherBuildIndex implements CommitLogDispatcher {\n\n        @Override\n        public void dispatch(DispatchRequest request) {\n            if (DefaultMessageStore.this.messageStoreConfig.isMessageIndexEnable()) {\n                if (DefaultMessageStore.this.messageStoreConfig.isIndexFileWriteEnable()) {\n                    DefaultMessageStore.this.indexService.buildIndex(request);\n                }\n                if (DefaultMessageStore.this.messageStoreConfig.isIndexRocksDBEnable()) {\n                    DefaultMessageStore.this.indexRocksDBStore.buildIndex(request);\n                }\n            }\n        }\n    }\n\n    class CommitLogDispatcherBuildTransIndex implements CommitLogDispatcher {\n\n        @Override\n        public void dispatch(DispatchRequest request) {\n            if (DefaultMessageStore.this.messageStoreConfig.isTransRocksDBEnable()) {\n                if (null == request || StringUtils.isEmpty(request.getTopic())) {\n                    return;\n                }\n                if (!request.getTopic().equals(TopicValidator.RMQ_SYS_ROCKSDB_TRANS_HALF_TOPIC) && !request.getTopic().equals(TopicValidator.RMQ_SYS_ROCKSDB_TRANS_OP_HALF_TOPIC)) {\n                    return;\n                }\n                if (null == DefaultMessageStore.this.transMessageRocksDBStore) {\n                    if (System.currentTimeMillis() % 1000 == 0) {\n                        LOGGER.error(\"CommitLogDispatcherBuildTransIndex dispatch error, transMessageRocksDBStore is null\");\n                    }\n                    return;\n                }\n                DefaultMessageStore.this.transMessageRocksDBStore.buildTransIndex(request);\n            }\n        }\n    }\n\n    public boolean isTimeToDelete() {\n        String when = messageStoreConfig.getDeleteWhen();\n        if (UtilAll.isItTimeToDo(when)) {\n            LOGGER.info(\"it's time to reclaim disk space, \" + when);\n            return true;\n        }\n        return false;\n    }\n\n    class CleanCommitLogService {\n\n        private final static int MAX_MANUAL_DELETE_FILE_TIMES = 20;\n        private final String diskSpaceWarningLevelRatio =\n            System.getProperty(\"rocketmq.broker.diskSpaceWarningLevelRatio\", \"\");\n\n        private final String diskSpaceCleanForciblyRatio =\n            System.getProperty(\"rocketmq.broker.diskSpaceCleanForciblyRatio\", \"\");\n        private long lastRedeleteTimestamp = 0;\n\n        private final AtomicInteger manualDeleteFileSeveralTimes = new AtomicInteger();\n\n        private volatile boolean cleanImmediately = false;\n\n        private int forceCleanFailedTimes = 0;\n\n        double getDiskSpaceWarningLevelRatio() {\n            double finalDiskSpaceWarningLevelRatio;\n            if (\"\".equals(diskSpaceWarningLevelRatio)) {\n                finalDiskSpaceWarningLevelRatio = DefaultMessageStore.this.getMessageStoreConfig().getDiskSpaceWarningLevelRatio() / 100.0;\n            } else {\n                finalDiskSpaceWarningLevelRatio = Double.parseDouble(diskSpaceWarningLevelRatio);\n            }\n\n            if (finalDiskSpaceWarningLevelRatio > 0.90) {\n                finalDiskSpaceWarningLevelRatio = 0.90;\n            }\n            if (finalDiskSpaceWarningLevelRatio < 0.35) {\n                finalDiskSpaceWarningLevelRatio = 0.35;\n            }\n\n            return finalDiskSpaceWarningLevelRatio;\n        }\n\n        double getDiskSpaceCleanForciblyRatio() {\n            double finalDiskSpaceCleanForciblyRatio;\n            if (\"\".equals(diskSpaceCleanForciblyRatio)) {\n                finalDiskSpaceCleanForciblyRatio = DefaultMessageStore.this.getMessageStoreConfig().getDiskSpaceCleanForciblyRatio() / 100.0;\n            } else {\n                finalDiskSpaceCleanForciblyRatio = Double.parseDouble(diskSpaceCleanForciblyRatio);\n            }\n\n            if (finalDiskSpaceCleanForciblyRatio > 0.85) {\n                finalDiskSpaceCleanForciblyRatio = 0.85;\n            }\n            if (finalDiskSpaceCleanForciblyRatio < 0.30) {\n                finalDiskSpaceCleanForciblyRatio = 0.30;\n            }\n\n            return finalDiskSpaceCleanForciblyRatio;\n        }\n\n        public void executeDeleteFilesManually() {\n            this.manualDeleteFileSeveralTimes.set(MAX_MANUAL_DELETE_FILE_TIMES);\n            DefaultMessageStore.LOGGER.info(\"executeDeleteFilesManually was invoked\");\n        }\n\n        public void run() {\n            try {\n                this.deleteExpiredFiles();\n                this.reDeleteHangedFile();\n            } catch (Throwable e) {\n                DefaultMessageStore.LOGGER.warn(this.getServiceName() + \" service has exception. \", e);\n            }\n        }\n\n        private void deleteExpiredFiles() {\n            int deleteCount = 0;\n            long fileReservedTime = DefaultMessageStore.this.getMessageStoreConfig().getFileReservedTime();\n            int deletePhysicFilesInterval = DefaultMessageStore.this.getMessageStoreConfig().getDeleteCommitLogFilesInterval();\n            int destroyMappedFileIntervalForcibly = DefaultMessageStore.this.getMessageStoreConfig().getDestroyMapedFileIntervalForcibly();\n            int deleteFileBatchMax = DefaultMessageStore.this.getMessageStoreConfig().getDeleteFileBatchMax();\n\n            boolean isTimeUp = DefaultMessageStore.this.isTimeToDelete();\n            boolean isUsageExceedsThreshold = this.isSpaceToDelete();\n            boolean isManualDelete = this.manualDeleteFileSeveralTimes.get() > 0;\n\n            if (isTimeUp || isUsageExceedsThreshold || isManualDelete) {\n\n                if (isManualDelete) {\n                    this.manualDeleteFileSeveralTimes.decrementAndGet();\n                }\n\n                boolean cleanAtOnce = DefaultMessageStore.this.getMessageStoreConfig().isCleanFileForciblyEnable() && this.cleanImmediately;\n\n                LOGGER.info(\"begin to delete before {} hours file. isTimeUp: {} isUsageExceedsThreshold: {} manualDeleteFileSeveralTimes: {} cleanAtOnce: {} deleteFileBatchMax: {}\",\n                    fileReservedTime,\n                    isTimeUp,\n                    isUsageExceedsThreshold,\n                    manualDeleteFileSeveralTimes.get(),\n                    cleanAtOnce,\n                    deleteFileBatchMax);\n\n                fileReservedTime *= 60 * 60 * 1000;\n\n                deleteCount = DefaultMessageStore.this.commitLog.deleteExpiredFile(fileReservedTime, deletePhysicFilesInterval,\n                    destroyMappedFileIntervalForcibly, cleanAtOnce, deleteFileBatchMax);\n                if (deleteCount > 0) {\n                    // If in the controller mode, we should notify the AutoSwitchHaService to truncateEpochFile\n                    if (DefaultMessageStore.this.brokerConfig.isEnableControllerMode()) {\n                        if (DefaultMessageStore.this.haService instanceof AutoSwitchHAService) {\n                            final long minPhyOffset = getMinPhyOffset();\n                            ((AutoSwitchHAService) DefaultMessageStore.this.haService).truncateEpochFilePrefix(minPhyOffset - 1);\n                        }\n                    }\n                } else if (isUsageExceedsThreshold) {\n                    LOGGER.warn(\"disk space will be full soon, but delete file failed.\");\n                }\n            }\n        }\n\n        private void reDeleteHangedFile() {\n            int interval = DefaultMessageStore.this.getMessageStoreConfig().getRedeleteHangedFileInterval();\n            long currentTimestamp = System.currentTimeMillis();\n            if ((currentTimestamp - this.lastRedeleteTimestamp) > interval) {\n                this.lastRedeleteTimestamp = currentTimestamp;\n                int destroyMappedFileIntervalForcibly =\n                    DefaultMessageStore.this.getMessageStoreConfig().getDestroyMapedFileIntervalForcibly();\n                if (DefaultMessageStore.this.commitLog.retryDeleteFirstFile(destroyMappedFileIntervalForcibly)) {\n                }\n            }\n        }\n\n        public String getServiceName() {\n            return DefaultMessageStore.this.brokerConfig.getIdentifier() + CleanCommitLogService.class.getSimpleName();\n        }\n\n        private boolean isSpaceToDelete() {\n            cleanImmediately = false;\n\n            String commitLogStorePath = DefaultMessageStore.this.getStorePathPhysic();\n            String[] storePaths = commitLogStorePath.trim().split(MixAll.MULTI_PATH_SPLITTER);\n            Set<String> fullStorePath = new HashSet<>();\n            double minPhysicRatio = 100;\n            String minStorePath = null;\n            for (String storePathPhysic : storePaths) {\n                double physicRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathPhysic);\n                if (minPhysicRatio > physicRatio) {\n                    minPhysicRatio = physicRatio;\n                    minStorePath = storePathPhysic;\n                }\n                if (physicRatio > getDiskSpaceCleanForciblyRatio()) {\n                    fullStorePath.add(storePathPhysic);\n                }\n            }\n            DefaultMessageStore.this.commitLog.setFullStorePaths(fullStorePath);\n            if (minPhysicRatio > getDiskSpaceWarningLevelRatio()) {\n                boolean diskFull = DefaultMessageStore.this.runningFlags.getAndMakeDiskFull();\n                if (diskFull) {\n                    DefaultMessageStore.LOGGER.error(\"physic disk maybe full soon \" + minPhysicRatio +\n                        \", so mark disk full, storePathPhysic=\" + minStorePath);\n                }\n\n                cleanImmediately = true;\n                return true;\n            } else if (minPhysicRatio > getDiskSpaceCleanForciblyRatio()) {\n                cleanImmediately = true;\n                return true;\n            } else {\n                boolean diskOK = DefaultMessageStore.this.runningFlags.getAndMakeDiskOK();\n                if (!diskOK) {\n                    DefaultMessageStore.LOGGER.info(\"physic disk space OK \" + minPhysicRatio +\n                        \", so mark disk ok, storePathPhysic=\" + minStorePath);\n                }\n            }\n\n            String storePathLogics = StorePathConfigHelper\n                .getStorePathConsumeQueue(DefaultMessageStore.this.getMessageStoreConfig().getStorePathRootDir());\n            double logicsRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathLogics);\n            if (logicsRatio > getDiskSpaceWarningLevelRatio()) {\n                boolean diskOK = DefaultMessageStore.this.runningFlags.getAndMakeDiskFull();\n                if (diskOK) {\n                    DefaultMessageStore.LOGGER.error(\"logics disk maybe full soon \" + logicsRatio + \", so mark disk full\");\n                }\n\n                cleanImmediately = true;\n                return true;\n            } else if (logicsRatio > getDiskSpaceCleanForciblyRatio()) {\n                cleanImmediately = true;\n                return true;\n            } else {\n                boolean diskOK = DefaultMessageStore.this.runningFlags.getAndMakeDiskOK();\n                if (!diskOK) {\n                    DefaultMessageStore.LOGGER.info(\"logics disk space OK \" + logicsRatio + \", so mark disk ok\");\n                }\n            }\n\n            double ratio = DefaultMessageStore.this.getMessageStoreConfig().getDiskMaxUsedSpaceRatio() / 100.0;\n            int replicasPerPartition = DefaultMessageStore.this.getMessageStoreConfig().getReplicasPerDiskPartition();\n            // Only one commitLog in node\n            if (replicasPerPartition <= 1) {\n                if (minPhysicRatio < 0 || minPhysicRatio > ratio) {\n                    DefaultMessageStore.LOGGER.info(\"commitLog disk maybe full soon, so reclaim space, \" + minPhysicRatio);\n                    return true;\n                }\n\n                if (logicsRatio < 0 || logicsRatio > ratio) {\n                    DefaultMessageStore.LOGGER.info(\"consumeQueue disk maybe full soon, so reclaim space, \" + logicsRatio);\n                    return true;\n                }\n                return false;\n            } else {\n                long majorFileSize = DefaultMessageStore.this.getMajorFileSize();\n                long partitionLogicalSize = UtilAll.getDiskPartitionTotalSpace(minStorePath) / replicasPerPartition;\n                double logicalRatio = 1.0 * majorFileSize / partitionLogicalSize;\n\n                if (logicalRatio > DefaultMessageStore.this.getMessageStoreConfig().getLogicalDiskSpaceCleanForciblyThreshold()) {\n                    // if logical ratio exceeds 0.80, then clean immediately\n                    DefaultMessageStore.LOGGER.info(\"Logical disk usage {} exceeds logical disk space clean forcibly threshold {}, forcibly: {}\",\n                        logicalRatio, minPhysicRatio, cleanImmediately);\n                    cleanImmediately = true;\n                    return true;\n                }\n\n                boolean isUsageExceedsThreshold = logicalRatio > ratio;\n                if (isUsageExceedsThreshold) {\n                    DefaultMessageStore.LOGGER.info(\"Logical disk usage {} exceeds clean threshold {}, forcibly: {}\",\n                        logicalRatio, ratio, cleanImmediately);\n                }\n                return isUsageExceedsThreshold;\n            }\n        }\n\n        public int getManualDeleteFileSeveralTimes() {\n            return manualDeleteFileSeveralTimes.get();\n        }\n\n        public void setManualDeleteFileSeveralTimes(int manualDeleteFileSeveralTimes) {\n            this.manualDeleteFileSeveralTimes.set(manualDeleteFileSeveralTimes);\n        }\n\n        public double calcStorePathPhysicRatio() {\n            Set<String> fullStorePath = new HashSet<>();\n            String storePath = getStorePathPhysic();\n            String[] paths = storePath.trim().split(MixAll.MULTI_PATH_SPLITTER);\n            double minPhysicRatio = 100;\n            for (String path : paths) {\n                double physicRatio = UtilAll.isPathExists(path) ?\n                    UtilAll.getDiskPartitionSpaceUsedPercent(path) : -1;\n                minPhysicRatio = Math.min(minPhysicRatio, physicRatio);\n                if (physicRatio > getDiskSpaceCleanForciblyRatio()) {\n                    fullStorePath.add(path);\n                }\n            }\n            DefaultMessageStore.this.commitLog.setFullStorePaths(fullStorePath);\n            return minPhysicRatio;\n\n        }\n\n        public boolean isSpaceFull() {\n            double physicRatio = calcStorePathPhysicRatio();\n            double ratio = DefaultMessageStore.this.getMessageStoreConfig().getDiskMaxUsedSpaceRatio() / 100.0;\n            if (physicRatio > ratio) {\n                DefaultMessageStore.LOGGER.info(\"physic disk of commitLog used: \" + physicRatio);\n            }\n            if (physicRatio > this.getDiskSpaceWarningLevelRatio()) {\n                boolean diskok = DefaultMessageStore.this.runningFlags.getAndMakeDiskFull();\n                if (diskok) {\n                    DefaultMessageStore.LOGGER.error(\"physic disk of commitLog maybe full soon, used \" + physicRatio + \", so mark disk full\");\n                }\n\n                return true;\n            } else {\n                boolean diskok = DefaultMessageStore.this.runningFlags.getAndMakeDiskOK();\n\n                if (!diskok) {\n                    DefaultMessageStore.LOGGER.info(\"physic disk space of commitLog OK \" + physicRatio + \", so mark disk ok\");\n                }\n\n                return false;\n            }\n        }\n    }\n\n    static class BatchDispatchRequest {\n\n        private final ByteBuffer byteBuffer;\n\n        private final int position;\n\n        private final int size;\n\n        private final long id;\n\n        public BatchDispatchRequest(ByteBuffer byteBuffer, int position, int size, long id) {\n            this.byteBuffer = byteBuffer;\n            this.position = position;\n            this.size = size;\n            this.id = id;\n        }\n    }\n\n    static class DispatchRequestOrderlyQueue {\n\n        DispatchRequest[][] buffer;\n\n        long ptr = 0;\n\n        AtomicLong maxPtr = new AtomicLong();\n\n        public DispatchRequestOrderlyQueue(int bufferNum) {\n            this.buffer = new DispatchRequest[bufferNum][];\n        }\n\n        public void put(long index, DispatchRequest[] dispatchRequests) {\n            while (ptr + this.buffer.length <= index) {\n                synchronized (this) {\n                    try {\n                        this.wait();\n                    } catch (InterruptedException e) {\n                        throw new RuntimeException(e);\n                    }\n                }\n            }\n            int mod = (int) (index % this.buffer.length);\n            this.buffer[mod] = dispatchRequests;\n            maxPtr.incrementAndGet();\n        }\n\n        public DispatchRequest[] get(List<DispatchRequest[]> dispatchRequestsList) {\n            synchronized (this) {\n                for (int i = 0; i < this.buffer.length; i++) {\n                    int mod = (int) (ptr % this.buffer.length);\n                    DispatchRequest[] ret = this.buffer[mod];\n                    if (ret == null) {\n                        this.notifyAll();\n                        return null;\n                    }\n                    dispatchRequestsList.add(ret);\n                    this.buffer[mod] = null;\n                    ptr++;\n                }\n            }\n            return null;\n        }\n\n        public synchronized boolean isEmpty() {\n            return maxPtr.get() == ptr;\n        }\n\n    }\n\n    @Override\n    public void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest) {\n        if (DefaultMessageStore.this.brokerConfig.isLongPollingEnable()\n            && DefaultMessageStore.this.messageArrivingListener != null) {\n            DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(),\n                dispatchRequest.getQueueId(), dispatchRequest.getConsumeQueueOffset() + 1,\n                dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(),\n                dispatchRequest.getBitMap(), dispatchRequest.getPropertiesMap());\n            DefaultMessageStore.this.reputMessageService.notifyMessageArrive4MultiQueue(dispatchRequest);\n        }\n    }\n\n    class ReputMessageService extends ServiceThread {\n\n        protected volatile long reputFromOffset = 0;\n        protected volatile long currentReputTimestamp = System.currentTimeMillis();\n\n        public long getReputFromOffset() {\n            return reputFromOffset;\n        }\n\n        public void setReputFromOffset(long reputFromOffset) {\n            this.reputFromOffset = reputFromOffset;\n        }\n\n        public long getCurrentReputTimestamp() {\n            return currentReputTimestamp;\n        }\n\n        @Override\n        public void shutdown() {\n            for (int i = 0; i < 50 && this.isCommitLogAvailable(); i++) {\n                try {\n                    Thread.sleep(100);\n                } catch (InterruptedException ignored) {\n                }\n            }\n\n            if (this.isCommitLogAvailable()) {\n                LOGGER.warn(\"shutdown ReputMessageService, but CommitLog have not finish to be dispatched, CommitLog max\" +\n                        \" offset={}, reputFromOffset={}\", DefaultMessageStore.this.commitLog.getMaxOffset(),\n                    this.reputFromOffset);\n            }\n\n            super.shutdown();\n        }\n\n        public long behind() {\n            return DefaultMessageStore.this.getConfirmOffset() - this.reputFromOffset;\n        }\n\n        public long behindMs() {\n            long lastCommitLogFileTimeStamp = System.currentTimeMillis();\n            MappedFile lastMappedFile = DefaultMessageStore.this.commitLog.getMappedFileQueue().getLastMappedFile();\n            if (lastMappedFile != null) {\n                lastCommitLogFileTimeStamp = lastMappedFile.getStoreTimestamp();\n            }\n            return Math.max(0, lastCommitLogFileTimeStamp - this.currentReputTimestamp);\n        }\n\n        public boolean isCommitLogAvailable() {\n            return this.reputFromOffset < getReputEndOffset();\n        }\n\n        protected long getReputEndOffset() {\n            return DefaultMessageStore.this.getMessageStoreConfig().isReadUnCommitted() ? DefaultMessageStore.this.commitLog.getMaxOffset() : DefaultMessageStore.this.commitLog.getConfirmOffset();\n        }\n\n        public void doReput() {\n            if (this.reputFromOffset < DefaultMessageStore.this.commitLog.getMinOffset()) {\n                LOGGER.warn(\"The reputFromOffset={} is smaller than minPyOffset={}, this usually indicate that the dispatch behind too much and the commitlog has expired.\",\n                    this.reputFromOffset, DefaultMessageStore.this.commitLog.getMinOffset());\n                this.reputFromOffset = DefaultMessageStore.this.commitLog.getMinOffset();\n            }\n            boolean isCommitLogAvailable = isCommitLogAvailable();\n            if (!isCommitLogAvailable) {\n                currentReputTimestamp = System.currentTimeMillis();\n            }\n            for (boolean doNext = true; isCommitLogAvailable() && doNext; ) {\n\n                SelectMappedBufferResult result = DefaultMessageStore.this.commitLog.getData(reputFromOffset);\n\n                if (result == null) {\n                    break;\n                }\n\n                try {\n                    this.reputFromOffset = result.getStartOffset();\n\n                    for (int readSize = 0; readSize < result.getSize() && reputFromOffset < getReputEndOffset() && doNext; ) {\n                        DispatchRequest dispatchRequest =\n                            DefaultMessageStore.this.commitLog.checkMessageAndReturnSize(result.getByteBuffer(), false, false, false);\n                        int size = dispatchRequest.getBufferSize() == -1 ? dispatchRequest.getMsgSize() : dispatchRequest.getBufferSize();\n\n                        if (reputFromOffset + size > getReputEndOffset()) {\n                            doNext = false;\n                            break;\n                        }\n\n                        if (dispatchRequest.isSuccess()) {\n                            if (size > 0) {\n                                currentReputTimestamp = dispatchRequest.getStoreTimestamp();\n                                DefaultMessageStore.this.doDispatch(dispatchRequest);\n\n                                if (!notifyMessageArriveInBatch) {\n                                    notifyMessageArriveIfNecessary(dispatchRequest);\n                                }\n\n                                this.reputFromOffset += size;\n                                readSize += size;\n                                if (!DefaultMessageStore.this.getMessageStoreConfig().isDuplicationEnable() &&\n                                    DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) {\n                                    DefaultMessageStore.this.storeStatsService\n                                        .getSinglePutMessageTopicTimesTotal(dispatchRequest.getTopic()).add(dispatchRequest.getBatchSize());\n                                    DefaultMessageStore.this.storeStatsService\n                                        .getSinglePutMessageTopicSizeTotal(dispatchRequest.getTopic())\n                                        .add(dispatchRequest.getMsgSize());\n                                }\n                            } else if (size == 0) {\n                                this.reputFromOffset = DefaultMessageStore.this.commitLog.rollNextFile(this.reputFromOffset);\n                                readSize = result.getSize();\n                            }\n                        } else {\n                            if (size > 0) {\n                                LOGGER.error(\"[BUG]read total count not equals msg total size. reputFromOffset={}\", reputFromOffset);\n                                this.reputFromOffset += size;\n                            } else {\n                                doNext = false;\n                                // If user open the dledger pattern or the broker is master node,\n                                // it will not ignore the exception and fix the reputFromOffset variable\n                                if (DefaultMessageStore.this.getMessageStoreConfig().isEnableDLegerCommitLog() ||\n                                    DefaultMessageStore.this.brokerConfig.getBrokerId() == MixAll.MASTER_ID) {\n                                    LOGGER.error(\"[BUG]dispatch message to consume queue error, COMMITLOG OFFSET: {}\",\n                                        this.reputFromOffset);\n                                    this.reputFromOffset += result.getSize() - readSize;\n                                }\n                            }\n                        }\n                    }\n                } catch (RocksDBException e) {\n                    ERROR_LOG.info(\"dispatch message to cq exception. reputFromOffset: {}\", this.reputFromOffset, e);\n                    return;\n                } finally {\n                    result.release();\n                }\n            }\n        }\n\n        private void notifyMessageArrive4MultiQueue(DispatchRequest dispatchRequest) {\n            Map<String, String> prop = dispatchRequest.getPropertiesMap();\n            if (prop == null || dispatchRequest.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {\n                return;\n            }\n            String multiDispatchQueue = prop.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);\n            String multiQueueOffset = prop.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET);\n            if (StringUtils.isBlank(multiDispatchQueue) || StringUtils.isBlank(multiQueueOffset)) {\n                return;\n            }\n            String[] queues = multiDispatchQueue.split(MixAll.LMQ_DISPATCH_SEPARATOR);\n            String[] queueOffsets = multiQueueOffset.split(MixAll.LMQ_DISPATCH_SEPARATOR);\n            if (queues.length != queueOffsets.length) {\n                return;\n            }\n            for (int i = 0; i < queues.length; i++) {\n                String queueName = queues[i];\n                long queueOffset = Long.parseLong(queueOffsets[i]);\n                int queueId = dispatchRequest.getQueueId();\n                if (DefaultMessageStore.this.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) {\n                    queueId = MixAll.LMQ_QUEUE_ID;\n                }\n                DefaultMessageStore.this.messageArrivingListener.arriving(\n                    queueName, queueId, queueOffset + 1, dispatchRequest.getTagsCode(),\n                    dispatchRequest.getStoreTimestamp(), dispatchRequest.getBitMap(), dispatchRequest.getPropertiesMap());\n            }\n        }\n\n        @Override\n        public void run() {\n            DefaultMessageStore.LOGGER.info(this.getServiceName() + \" service started\");\n\n            while (!this.isStopped()) {\n                try {\n                    TimeUnit.MILLISECONDS.sleep(1);\n                    this.doReput();\n                } catch (Throwable e) {\n                    DefaultMessageStore.LOGGER.warn(this.getServiceName() + \" service has exception. \", e);\n                }\n            }\n\n            DefaultMessageStore.LOGGER.info(this.getServiceName() + \" service end\");\n        }\n\n        @Override\n        public String getServiceName() {\n            if (DefaultMessageStore.this.getBrokerConfig().isInBrokerContainer()) {\n                return DefaultMessageStore.this.getBrokerIdentity().getIdentifier() + ReputMessageService.class.getSimpleName();\n            }\n            return ReputMessageService.class.getSimpleName();\n        }\n\n    }\n\n    class MainBatchDispatchRequestService extends ServiceThread {\n\n        private final ExecutorService batchDispatchRequestExecutor;\n\n        public MainBatchDispatchRequestService() {\n            batchDispatchRequestExecutor = ThreadUtils.newThreadPoolExecutor(\n                DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(),\n                DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(),\n                1000 * 60,\n                TimeUnit.MICROSECONDS,\n                new LinkedBlockingQueue<>(4096),\n                new ThreadFactoryImpl(\"BatchDispatchRequestServiceThread_\"),\n                new ThreadPoolExecutor.AbortPolicy());\n        }\n\n        private void pollBatchDispatchRequest() {\n            try {\n                if (!batchDispatchRequestQueue.isEmpty()) {\n                    BatchDispatchRequest task = batchDispatchRequestQueue.peek();\n                    batchDispatchRequestExecutor.execute(() -> {\n                        try {\n                            ByteBuffer tmpByteBuffer = task.byteBuffer;\n                            tmpByteBuffer.position(task.position);\n                            tmpByteBuffer.limit(task.position + task.size);\n                            List<DispatchRequest> dispatchRequestList = new ArrayList<>();\n                            while (tmpByteBuffer.hasRemaining()) {\n                                DispatchRequest dispatchRequest = DefaultMessageStore.this.commitLog.checkMessageAndReturnSize(tmpByteBuffer, false, false, false);\n                                if (dispatchRequest.isSuccess()) {\n                                    dispatchRequestList.add(dispatchRequest);\n                                } else {\n                                    LOGGER.error(\"[BUG]read total count not equals msg total size.\");\n                                }\n                            }\n                            dispatchRequestOrderlyQueue.put(task.id, dispatchRequestList.toArray(new DispatchRequest[dispatchRequestList.size()]));\n                            mappedPageHoldCount.getAndDecrement();\n                        } catch (Exception e) {\n                            LOGGER.error(\"There is an exception in task execution.\", e);\n                        }\n                    });\n                    batchDispatchRequestQueue.poll();\n                }\n            } catch (Exception e) {\n                DefaultMessageStore.LOGGER.warn(this.getServiceName() + \" service has exception. \", e);\n            }\n        }\n\n        @Override\n        public void run() {\n            DefaultMessageStore.LOGGER.info(this.getServiceName() + \" service started\");\n\n            while (!this.isStopped()) {\n                try {\n                    TimeUnit.MILLISECONDS.sleep(1);\n                    pollBatchDispatchRequest();\n                } catch (Exception e) {\n                    DefaultMessageStore.LOGGER.warn(this.getServiceName() + \" service has exception. \", e);\n                }\n            }\n\n            DefaultMessageStore.LOGGER.info(this.getServiceName() + \" service end\");\n        }\n\n        @Override\n        public String getServiceName() {\n            if (DefaultMessageStore.this.getBrokerConfig().isInBrokerContainer()) {\n                return DefaultMessageStore.this.getBrokerIdentity().getIdentifier() + MainBatchDispatchRequestService.class.getSimpleName();\n            }\n            return MainBatchDispatchRequestService.class.getSimpleName();\n        }\n\n    }\n\n    class DispatchService extends ServiceThread {\n\n        private final List<DispatchRequest[]> dispatchRequestsList = new ArrayList<>();\n\n        // dispatchRequestsList:[\n        //      {dispatchRequests:[{dispatchRequest}, {dispatchRequest}]},\n        //      {dispatchRequests:[{dispatchRequest}, {dispatchRequest}]}]\n        private void dispatch() throws Exception {\n            dispatchRequestsList.clear();\n            dispatchRequestOrderlyQueue.get(dispatchRequestsList);\n            if (!dispatchRequestsList.isEmpty()) {\n                for (DispatchRequest[] dispatchRequests : dispatchRequestsList) {\n                    for (DispatchRequest dispatchRequest : dispatchRequests) {\n                        DefaultMessageStore.this.doDispatch(dispatchRequest);\n                        // wake up long-polling\n                        DefaultMessageStore.this.notifyMessageArriveIfNecessary(dispatchRequest);\n\n                        if (!DefaultMessageStore.this.getMessageStoreConfig().isDuplicationEnable() &&\n                            DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) {\n                            DefaultMessageStore.this.storeStatsService\n                                .getSinglePutMessageTopicTimesTotal(dispatchRequest.getTopic()).add(1);\n                            DefaultMessageStore.this.storeStatsService\n                                .getSinglePutMessageTopicSizeTotal(dispatchRequest.getTopic())\n                                .add(dispatchRequest.getMsgSize());\n                        }\n                    }\n                }\n            }\n        }\n\n        @Override\n        public void run() {\n            DefaultMessageStore.LOGGER.info(this.getServiceName() + \" service started\");\n\n            while (!this.isStopped()) {\n                try {\n                    TimeUnit.MILLISECONDS.sleep(1);\n                    dispatch();\n                } catch (Exception e) {\n                    DefaultMessageStore.LOGGER.warn(this.getServiceName() + \" service has exception. \", e);\n                }\n            }\n\n            DefaultMessageStore.LOGGER.info(this.getServiceName() + \" service end\");\n        }\n\n        @Override\n        public String getServiceName() {\n            if (DefaultMessageStore.this.getBrokerConfig().isInBrokerContainer()) {\n                return DefaultMessageStore.this.getBrokerIdentity().getIdentifier() + DispatchService.class.getSimpleName();\n            }\n            return DispatchService.class.getSimpleName();\n        }\n    }\n\n    class ConcurrentReputMessageService extends ReputMessageService {\n\n        private static final int BATCH_SIZE = 1024 * 1024 * 4;\n\n        private long batchId = 0;\n\n        private MainBatchDispatchRequestService mainBatchDispatchRequestService;\n\n        private DispatchService dispatchService;\n\n        public ConcurrentReputMessageService() {\n            super();\n            this.mainBatchDispatchRequestService = new MainBatchDispatchRequestService();\n            this.dispatchService = new DispatchService();\n        }\n\n        public void createBatchDispatchRequest(ByteBuffer byteBuffer, int position, int size) {\n            if (position < 0) {\n                return;\n            }\n            mappedPageHoldCount.getAndIncrement();\n            BatchDispatchRequest task = new BatchDispatchRequest(byteBuffer.duplicate(), position, size, batchId++);\n            batchDispatchRequestQueue.offer(task);\n        }\n\n        @Override\n        public void start() {\n            super.start();\n            this.mainBatchDispatchRequestService.start();\n            this.dispatchService.start();\n        }\n\n        @Override\n        public void doReput() {\n            if (this.reputFromOffset < DefaultMessageStore.this.commitLog.getMinOffset()) {\n                LOGGER.warn(\"The reputFromOffset={} is smaller than minPyOffset={}, this usually indicate that the dispatch behind too much and the commitlog has expired.\",\n                    this.reputFromOffset, DefaultMessageStore.this.commitLog.getMinOffset());\n                this.reputFromOffset = DefaultMessageStore.this.commitLog.getMinOffset();\n            }\n            for (boolean doNext = true; this.isCommitLogAvailable() && doNext; ) {\n\n                SelectMappedBufferResult result = DefaultMessageStore.this.commitLog.getData(reputFromOffset);\n\n                if (result == null) {\n                    break;\n                }\n\n                int batchDispatchRequestStart = -1;\n                int batchDispatchRequestSize = -1;\n                try {\n                    this.reputFromOffset = result.getStartOffset();\n\n                    for (int readSize = 0; readSize < result.getSize() && reputFromOffset < getReputEndOffset() && doNext; ) {\n                        ByteBuffer byteBuffer = result.getByteBuffer();\n\n                        int totalSize = preCheckMessageAndReturnSize(byteBuffer);\n\n                        if (totalSize > 0) {\n                            if (batchDispatchRequestStart == -1) {\n                                batchDispatchRequestStart = byteBuffer.position();\n                                batchDispatchRequestSize = 0;\n                            }\n                            batchDispatchRequestSize += totalSize;\n                            if (batchDispatchRequestSize > BATCH_SIZE) {\n                                this.createBatchDispatchRequest(byteBuffer, batchDispatchRequestStart, batchDispatchRequestSize);\n                                batchDispatchRequestStart = -1;\n                                batchDispatchRequestSize = -1;\n                            }\n                            byteBuffer.position(byteBuffer.position() + totalSize);\n                            this.reputFromOffset += totalSize;\n                            readSize += totalSize;\n                        } else {\n                            doNext = false;\n                            if (totalSize == 0) {\n                                this.reputFromOffset = DefaultMessageStore.this.commitLog.rollNextFile(this.reputFromOffset);\n                            }\n                            this.createBatchDispatchRequest(byteBuffer, batchDispatchRequestStart, batchDispatchRequestSize);\n                            batchDispatchRequestStart = -1;\n                            batchDispatchRequestSize = -1;\n                        }\n                    }\n                } finally {\n                    this.createBatchDispatchRequest(result.getByteBuffer(), batchDispatchRequestStart, batchDispatchRequestSize);\n                    boolean over = mappedPageHoldCount.get() == 0;\n                    while (!over) {\n                        try {\n                            TimeUnit.MILLISECONDS.sleep(1);\n                        } catch (Exception e) {\n                            e.printStackTrace();\n                        }\n                        over = mappedPageHoldCount.get() == 0;\n                    }\n                    result.release();\n                }\n            }\n        }\n\n        /**\n         * pre-check the message and returns the message size\n         *\n         * @return 0 Come to the end of file // >0 Normal messages // -1 Message checksum failure\n         */\n        public int preCheckMessageAndReturnSize(ByteBuffer byteBuffer) {\n            byteBuffer.mark();\n\n            int totalSize = byteBuffer.getInt();\n            if (reputFromOffset + totalSize > DefaultMessageStore.this.getConfirmOffset()) {\n                return -1;\n            }\n\n            int magicCode = byteBuffer.getInt();\n            switch (magicCode) {\n                case MessageDecoder.MESSAGE_MAGIC_CODE:\n                case MessageDecoder.MESSAGE_MAGIC_CODE_V2:\n                    break;\n                case MessageDecoder.BLANK_MAGIC_CODE:\n                    return 0;\n                default:\n                    return -1;\n            }\n\n            byteBuffer.reset();\n\n            return totalSize;\n        }\n\n        @Override\n        public void shutdown() {\n            for (int i = 0; i < 50 && this.isCommitLogAvailable(); i++) {\n                try {\n                    TimeUnit.MILLISECONDS.sleep(100);\n                } catch (InterruptedException ignored) {\n                }\n            }\n\n            if (this.isCommitLogAvailable()) {\n                LOGGER.warn(\"shutdown concurrentReputMessageService, but CommitLog have not finish to be dispatched, CommitLog max\" +\n                        \" offset={}, reputFromOffset={}\", DefaultMessageStore.this.commitLog.getMaxOffset(),\n                    this.reputFromOffset);\n            }\n\n            this.mainBatchDispatchRequestService.shutdown();\n            this.dispatchService.shutdown();\n            super.shutdown();\n        }\n\n        @Override\n        public String getServiceName() {\n            if (DefaultMessageStore.this.getBrokerConfig().isInBrokerContainer()) {\n                return DefaultMessageStore.this.getBrokerIdentity().getIdentifier() + ConcurrentReputMessageService.class.getSimpleName();\n            }\n            return ConcurrentReputMessageService.class.getSimpleName();\n        }\n    }\n\n    @Override\n    public HARuntimeInfo getHARuntimeInfo() {\n        if (haService != null) {\n            return this.haService.getRuntimeInfo(this.commitLog.getMaxOffset());\n        } else {\n            return null;\n        }\n    }\n\n    public int getMaxDelayLevel() {\n        return maxDelayLevel;\n    }\n\n    public long computeDeliverTimestamp(final int delayLevel, final long storeTimestamp) {\n        Long time = this.delayLevelTable.get(delayLevel);\n        if (time != null) {\n            return time + storeTimestamp;\n        }\n\n        return storeTimestamp + 1000;\n    }\n\n    public List<PutMessageHook> getPutMessageHookList() {\n        return putMessageHookList;\n    }\n\n    @Override\n    public void setSendMessageBackHook(SendMessageBackHook sendMessageBackHook) {\n        this.sendMessageBackHook = sendMessageBackHook;\n    }\n\n    @Override\n    public SendMessageBackHook getSendMessageBackHook() {\n        return sendMessageBackHook;\n    }\n\n    @Override\n    public boolean isShutdown() {\n        return shutdown;\n    }\n\n    @Override\n    public long estimateMessageCount(String topic, int queueId, long from, long to, MessageFilter filter) {\n        if (from < 0) {\n            from = 0;\n        }\n\n        if (from >= to) {\n            return 0;\n        }\n\n        if (null == filter) {\n            return to - from;\n        }\n\n        ConsumeQueueInterface consumeQueue = findConsumeQueue(topic, queueId);\n        if (null == consumeQueue) {\n            return 0;\n        }\n\n        // correct the \"from\" argument to min offset in queue if it is too small\n        long minOffset = consumeQueue.getMinOffsetInQueue();\n        if (from < minOffset) {\n            long diff = to - from;\n            from = minOffset;\n            to = from + diff;\n        }\n\n        long msgCount = consumeQueue.estimateMessageCount(from, to, filter);\n        return msgCount == -1 ? to - from : msgCount;\n    }\n\n    @Override\n    public List<Pair<InstrumentSelector, ViewBuilder>> getMetricsView() {\n        return this.defaultStoreMetricsManager.getMetricsView();\n    }\n\n    @Override\n    public void initMetrics(Meter meter, Supplier<AttributesBuilder> attributesBuilderSupplier) {\n        this.defaultStoreMetricsManager.init(meter, attributesBuilderSupplier, this);\n    }\n\n    /**\n     * Enable transient commitLog store pool only if transientStorePoolEnable is true and broker role is not SLAVE or\n     * enableControllerMode is true\n     *\n     * @return <tt>true</tt> or <tt>false</tt>\n     */\n    public boolean isTransientStorePoolEnable() {\n        return this.messageStoreConfig.isTransientStorePoolEnable() &&\n            (this.brokerConfig.isEnableControllerMode() || this.messageStoreConfig.getBrokerRole() != BrokerRole.SLAVE)\n            && !messageStoreConfig.isWriteWithoutMmap();\n    }\n\n    public long getReputFromOffset() {\n        return this.reputMessageService.getReputFromOffset();\n    }\n\n    public CompactionStore getCompactionStore() {\n        return compactionStore;\n    }\n\n    public IndexService getIndexService() {\n        return indexService;\n    }\n\n    public ScheduledExecutorService getScheduledCleanQueueExecutorService() {\n        return scheduledCleanQueueExecutorService;\n    }\n\n    public void destroyConsumeQueueStore(boolean loadAfterDestroy) {\n        consumeQueueStore.destroy(loadAfterDestroy);\n    }\n\n    public MessageStoreStateMachine getStateMachine() {\n        return stateMachine;\n    }\n\n    @Override\n    public MessageRocksDBStorage getMessageRocksDBStorage() {\n        return this.messageRocksDBStorage;\n    }\n\n    public boolean isNotifyMessageArriveInBatch() {\n        return notifyMessageArriveInBatch;\n    }\n\n    public void setNotifyMessageArriveInBatch(boolean notifyMessageArriveInBatch) {\n        this.notifyMessageArriveInBatch = notifyMessageArriveInBatch;\n    }\n\n    public DefaultStoreMetricsManager getDefaultStoreMetricsManager() {\n        return defaultStoreMetricsManager;\n    }\n\n    @Override\n    public StoreMetricsManager getStoreMetricsManager() {\n        return defaultStoreMetricsManager;\n    }\n\n    public IndexRocksDBStore getIndexRocksDBStore() {\n        return indexRocksDBStore;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport java.util.Map;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageConst;\n\npublic class DispatchRequest {\n    private final String topic;\n    private final int queueId;\n    private final long commitLogOffset;\n    private int msgSize;\n    private final long tagsCode;\n    private final long storeTimestamp;\n    private final long consumeQueueOffset;\n    private final String keys;\n    private final boolean success;\n    private final String uniqKey;\n\n    private final int sysFlag;\n    private final long preparedTransactionOffset;\n    private final Map<String, String> propertiesMap;\n    private byte[] bitMap;\n\n    private int bufferSize = -1;//the buffer size maybe larger than the msg size if the message is wrapped by something\n\n    // for batch consume queue\n    private long  msgBaseOffset = -1;\n    private short batchSize = 1;\n\n    private long nextReputFromOffset = -1;\n\n    private String offsetId;\n\n    public DispatchRequest(\n        final String topic,\n        final int queueId,\n        final long commitLogOffset,\n        final int msgSize,\n        final long tagsCode,\n        final long storeTimestamp,\n        final long consumeQueueOffset,\n        final String keys,\n        final String uniqKey,\n        final int sysFlag,\n        final long preparedTransactionOffset,\n        final Map<String, String> propertiesMap\n    ) {\n        this.topic = topic;\n        this.queueId = queueId;\n        this.commitLogOffset = commitLogOffset;\n        this.msgSize = msgSize;\n        this.tagsCode = tagsCode;\n        this.storeTimestamp = storeTimestamp;\n        this.consumeQueueOffset = consumeQueueOffset;\n        this.msgBaseOffset = consumeQueueOffset;\n        this.keys = keys;\n        this.uniqKey = uniqKey;\n\n        this.sysFlag = sysFlag;\n        this.preparedTransactionOffset = preparedTransactionOffset;\n        this.success = true;\n        this.propertiesMap = propertiesMap;\n    }\n\n    public DispatchRequest(String topic, int queueId, long consumeQueueOffset, long commitLogOffset, int size, long tagsCode) {\n        this.topic = topic;\n        this.queueId = queueId;\n        this.commitLogOffset = commitLogOffset;\n        this.msgSize = size;\n        this.tagsCode = tagsCode;\n        this.storeTimestamp = 0;\n        this.consumeQueueOffset = consumeQueueOffset;\n        this.keys = \"\";\n        this.uniqKey = null;\n        this.sysFlag = 0;\n        this.preparedTransactionOffset = 0;\n        this.success = false;\n        this.propertiesMap = null;\n    }\n\n    public DispatchRequest(int size) {\n        this.topic = \"\";\n        this.queueId = 0;\n        this.commitLogOffset = 0;\n        this.msgSize = size;\n        this.tagsCode = 0;\n        this.storeTimestamp = 0;\n        this.consumeQueueOffset = 0;\n        this.keys = \"\";\n        this.uniqKey = null;\n        this.sysFlag = 0;\n        this.preparedTransactionOffset = 0;\n        this.success = false;\n        this.propertiesMap = null;\n    }\n\n    public DispatchRequest(int size, boolean success) {\n        this.topic = \"\";\n        this.queueId = 0;\n        this.commitLogOffset = 0;\n        this.msgSize = size;\n        this.tagsCode = 0;\n        this.storeTimestamp = 0;\n        this.consumeQueueOffset = 0;\n        this.keys = \"\";\n        this.uniqKey = null;\n        this.sysFlag = 0;\n        this.preparedTransactionOffset = 0;\n        this.success = success;\n        this.propertiesMap = null;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public int getQueueId() {\n        return queueId;\n    }\n\n    public long getCommitLogOffset() {\n        return commitLogOffset;\n    }\n\n    public int getMsgSize() {\n        return msgSize;\n    }\n\n    public long getStoreTimestamp() {\n        return storeTimestamp;\n    }\n\n    public long getConsumeQueueOffset() {\n        return consumeQueueOffset;\n    }\n\n    public String getKeys() {\n        return keys;\n    }\n\n    public long getTagsCode() {\n        return tagsCode;\n    }\n\n    public int getSysFlag() {\n        return sysFlag;\n    }\n\n    public long getPreparedTransactionOffset() {\n        return preparedTransactionOffset;\n    }\n\n    public boolean isSuccess() {\n        return success;\n    }\n\n    public String getUniqKey() {\n        return uniqKey;\n    }\n\n    public Map<String, String> getPropertiesMap() {\n        return propertiesMap;\n    }\n\n    public byte[] getBitMap() {\n        return bitMap;\n    }\n\n    public void setBitMap(byte[] bitMap) {\n        this.bitMap = bitMap;\n    }\n\n    public short getBatchSize() {\n        return batchSize;\n    }\n\n    public void setBatchSize(short batchSize) {\n        this.batchSize = batchSize;\n    }\n\n    public void setMsgSize(int msgSize) {\n        this.msgSize = msgSize;\n    }\n\n    public long getMsgBaseOffset() {\n        return msgBaseOffset;\n    }\n\n    public void setMsgBaseOffset(long msgBaseOffset) {\n        this.msgBaseOffset = msgBaseOffset;\n    }\n\n    public int getBufferSize() {\n        return bufferSize;\n    }\n\n    public void setBufferSize(int bufferSize) {\n        this.bufferSize = bufferSize;\n    }\n\n    public long getNextReputFromOffset() {\n        return nextReputFromOffset;\n    }\n\n    public void setNextReputFromOffset(long nextReputFromOffset) {\n        this.nextReputFromOffset = nextReputFromOffset;\n    }\n\n    public String getOffsetId() {\n        return offsetId;\n    }\n\n    public void setOffsetId(String offsetId) {\n        this.offsetId = offsetId;\n    }\n\n    public boolean containsLMQ() {\n        if (!MixAll.topicAllowsLMQ(topic)) {\n            return false;\n        }\n        if (null == propertiesMap || propertiesMap.isEmpty()) {\n            return false;\n        }\n        String lmqNames = propertiesMap.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);\n        String lmqOffsets = propertiesMap.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET);\n        return !StringUtils.isBlank(lmqNames) && !StringUtils.isBlank(lmqOffsets);\n    }\n\n    @Override\n    public String toString() {\n        return \"DispatchRequest{\" +\n                \"topic='\" + topic + '\\'' +\n                \", queueId=\" + queueId +\n                \", commitLogOffset=\" + commitLogOffset +\n                \", msgSize=\" + msgSize +\n                \", success=\" + success +\n                \", msgBaseOffset=\" + msgBaseOffset +\n                \", batchSize=\" + batchSize +\n                \", nextReputFromOffset=\" + nextReputFromOffset +\n            '}';\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/FileQueueSnapshot.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport org.apache.rocketmq.store.logfile.MappedFile;\n\npublic class FileQueueSnapshot {\n    private MappedFile firstFile;\n    private long firstFileIndex;\n    private MappedFile lastFile;\n    private long lastFileIndex;\n    private long currentFile;\n    private long currentFileIndex;\n    private long behindCount;\n    private boolean exist;\n\n    public FileQueueSnapshot() {\n    }\n\n    public FileQueueSnapshot(MappedFile firstFile, long firstFileIndex, MappedFile lastFile, long lastFileIndex, long currentFile, long currentFileIndex, long behindCount, boolean exist) {\n        this.firstFile = firstFile;\n        this.firstFileIndex = firstFileIndex;\n        this.lastFile = lastFile;\n        this.lastFileIndex = lastFileIndex;\n        this.currentFile = currentFile;\n        this.currentFileIndex = currentFileIndex;\n        this.behindCount = behindCount;\n        this.exist = exist;\n    }\n\n    public MappedFile getFirstFile() {\n        return firstFile;\n    }\n\n    public long getFirstFileIndex() {\n        return firstFileIndex;\n    }\n\n    public MappedFile getLastFile() {\n        return lastFile;\n    }\n\n    public long getLastFileIndex() {\n        return lastFileIndex;\n    }\n\n    public long getCurrentFile() {\n        return currentFile;\n    }\n\n    public long getCurrentFileIndex() {\n        return currentFileIndex;\n    }\n\n    public long getBehindCount() {\n        return behindCount;\n    }\n\n    public boolean isExist() {\n        return exist;\n    }\n\n    @Override\n    public String toString() {\n        return \"FileQueueSnapshot{\" +\n                \"firstFile=\" + firstFile +\n                \", firstFileIndex=\" + firstFileIndex +\n                \", lastFile=\" + lastFile +\n                \", lastFileIndex=\" + lastFileIndex +\n                \", currentFile=\" + currentFile +\n                \", currentFileIndex=\" + currentFileIndex +\n                \", behindCount=\" + behindCount +\n                \", exist=\" + exist +\n                '}';\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/FlushDiskWatcher.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\n\nimport java.util.concurrent.LinkedBlockingQueue;\n\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.CommitLog.GroupCommitRequest;\n\npublic class FlushDiskWatcher extends ServiceThread {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private final LinkedBlockingQueue<GroupCommitRequest> commitRequests = new LinkedBlockingQueue<>();\n\n    @Override\n    public String getServiceName() {\n        return FlushDiskWatcher.class.getSimpleName();\n    }\n\n    @Override\n    public void run() {\n        while (!isStopped()) {\n            GroupCommitRequest request = null;\n            try {\n                request = commitRequests.take();\n            } catch (InterruptedException e) {\n                log.warn(\"take flush disk commit request, but interrupted, this may caused by shutdown\");\n                continue;\n            }\n            while (!request.future().isDone()) {\n                long now = System.nanoTime();\n                if (now - request.getDeadLine() >= 0) {\n                    request.wakeupCustomer(PutMessageStatus.FLUSH_DISK_TIMEOUT);\n                    break;\n                }\n                // To avoid frequent thread switching, replace future.get with sleep here,\n                long sleepTime = (request.getDeadLine() - now) / 1_000_000;\n                sleepTime = Math.min(10, sleepTime);\n                if (sleepTime == 0) {\n                    request.wakeupCustomer(PutMessageStatus.FLUSH_DISK_TIMEOUT);\n                    break;\n                }\n                try {\n                    Thread.sleep(sleepTime);\n                } catch (InterruptedException e) {\n                    log.warn(\n                            \"An exception occurred while waiting for flushing disk to complete. this may caused by shutdown\");\n                    break;\n                }\n            }\n        }\n    }\n\n    public void add(GroupCommitRequest request) {\n        commitRequests.add(request);\n    }\n\n    public int queueSize() {\n        return commitRequests.size();\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/FlushManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic interface FlushManager {\n\n    void start();\n\n    void shutdown();\n\n    void wakeUpFlush();\n\n    void wakeUpCommit();\n\n    void handleDiskFlush(AppendMessageResult result, PutMessageResult putMessageResult, MessageExt messageExt);\n\n    CompletableFuture<PutMessageStatus> handleDiskFlush(AppendMessageResult result, MessageExt messageExt);\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/GetMessageResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\n\npublic class GetMessageResult {\n\n    private final List<SelectMappedBufferResult> messageMapedList;\n    private final List<ByteBuffer> messageBufferList;\n    private final List<Long> messageQueueOffset;\n\n    private GetMessageStatus status;\n    private long nextBeginOffset;\n    private long minOffset;\n    private long maxOffset;\n\n    private int bufferTotalSize = 0;\n\n    private int messageCount = 0;\n\n    private boolean suggestPullingFromSlave = false;\n\n    private int msgCount4Commercial = 0;\n    private int commercialSizePerMsg = 4 * 1024;\n\n    private long coldDataSum = 0L;\n\n    private int filterMessageCount;\n\n    public static final GetMessageResult NO_MATCH_LOGIC_QUEUE =\n        new GetMessageResult(GetMessageStatus.NO_MATCHED_LOGIC_QUEUE, 0, 0, 0, Collections.emptyList(),\n            Collections.emptyList(), Collections.emptyList());\n\n    public GetMessageResult() {\n        messageMapedList = new ArrayList<>(100);\n        messageBufferList = new ArrayList<>(100);\n        messageQueueOffset = new ArrayList<>(100);\n    }\n\n    public GetMessageResult(int resultSize) {\n        messageMapedList = new ArrayList<>(resultSize);\n        messageBufferList = new ArrayList<>(resultSize);\n        messageQueueOffset = new ArrayList<>(resultSize);\n    }\n\n    private GetMessageResult(GetMessageStatus status, long nextBeginOffset, long minOffset, long maxOffset,\n        List<SelectMappedBufferResult> messageMapedList, List<ByteBuffer> messageBufferList, List<Long> messageQueueOffset) {\n        this.status = status;\n        this.nextBeginOffset = nextBeginOffset;\n        this.minOffset = minOffset;\n        this.maxOffset = maxOffset;\n        this.messageMapedList = messageMapedList;\n        this.messageBufferList = messageBufferList;\n        this.messageQueueOffset = messageQueueOffset;\n    }\n\n    public GetMessageStatus getStatus() {\n        return status;\n    }\n\n    public void setStatus(GetMessageStatus status) {\n        this.status = status;\n    }\n\n    public long getNextBeginOffset() {\n        return nextBeginOffset;\n    }\n\n    public void setNextBeginOffset(long nextBeginOffset) {\n        this.nextBeginOffset = nextBeginOffset;\n    }\n\n    public long getMinOffset() {\n        return minOffset;\n    }\n\n    public void setMinOffset(long minOffset) {\n        this.minOffset = minOffset;\n    }\n\n    public long getMaxOffset() {\n        return maxOffset;\n    }\n\n    public void setMaxOffset(long maxOffset) {\n        this.maxOffset = maxOffset;\n    }\n\n    public List<SelectMappedBufferResult> getMessageMapedList() {\n        return messageMapedList;\n    }\n\n    public List<ByteBuffer> getMessageBufferList() {\n        return messageBufferList;\n    }\n\n    public void addMessage(final SelectMappedBufferResult mapedBuffer) {\n        this.messageMapedList.add(mapedBuffer);\n        this.messageBufferList.add(mapedBuffer.getByteBuffer());\n        this.bufferTotalSize += mapedBuffer.getSize();\n        this.msgCount4Commercial += (int) Math.ceil(\n            mapedBuffer.getSize() /  (double)commercialSizePerMsg);\n        this.messageCount++;\n    }\n\n    public void addMessage(final SelectMappedBufferResult mapedBuffer, final long queueOffset) {\n        this.messageMapedList.add(mapedBuffer);\n        this.messageBufferList.add(mapedBuffer.getByteBuffer());\n        this.bufferTotalSize += mapedBuffer.getSize();\n        this.msgCount4Commercial += (int) Math.ceil(\n            mapedBuffer.getSize() /  (double)commercialSizePerMsg);\n        this.messageCount++;\n        this.messageQueueOffset.add(queueOffset);\n    }\n\n\n    public void addMessage(final SelectMappedBufferResult mapedBuffer, final long queueOffset, final int batchNum) {\n        addMessage(mapedBuffer, queueOffset);\n        messageCount += batchNum - 1;\n    }\n\n    public void release() {\n        for (SelectMappedBufferResult select : this.messageMapedList) {\n            select.release();\n        }\n    }\n\n    public int getBufferTotalSize() {\n        return bufferTotalSize;\n    }\n\n    public int getMessageCount() {\n        return messageCount;\n    }\n\n    public boolean isSuggestPullingFromSlave() {\n        return suggestPullingFromSlave;\n    }\n\n    public void setSuggestPullingFromSlave(boolean suggestPullingFromSlave) {\n        this.suggestPullingFromSlave = suggestPullingFromSlave;\n    }\n\n    public int getMsgCount4Commercial() {\n        return msgCount4Commercial;\n    }\n\n    public void setMsgCount4Commercial(int msgCount4Commercial) {\n        this.msgCount4Commercial = msgCount4Commercial;\n    }\n\n    public List<Long> getMessageQueueOffset() {\n        return messageQueueOffset;\n    }\n\n    public long getColdDataSum() {\n        return coldDataSum;\n    }\n\n    public void setColdDataSum(long coldDataSum) {\n        this.coldDataSum = coldDataSum;\n    }\n\n    public int getFilterMessageCount() {\n        return filterMessageCount;\n    }\n\n    public void setFilterMessageCount(int filterMessageCount) {\n        this.filterMessageCount = filterMessageCount;\n    }\n\n    @Override\n    public String toString() {\n        return \"GetMessageResult [status=\" + status + \", nextBeginOffset=\" + nextBeginOffset + \", minOffset=\"\n            + minOffset + \", maxOffset=\" + maxOffset + \", bufferTotalSize=\" + bufferTotalSize + \", messageCount=\" + messageCount\n            + \", filterMessageCount=\" + filterMessageCount + \", suggestPullingFromSlave=\" + suggestPullingFromSlave + \"]\";\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/GetMessageStatus.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\npublic enum GetMessageStatus {\n\n    FOUND,\n\n    NO_MATCHED_MESSAGE,\n\n    MESSAGE_WAS_REMOVING,\n\n    OFFSET_FOUND_NULL,\n\n    OFFSET_OVERFLOW_BADLY,\n\n    OFFSET_OVERFLOW_ONE,\n\n    OFFSET_TOO_SMALL,\n\n    NO_MATCHED_LOGIC_QUEUE,\n\n    NO_MESSAGE_IN_QUEUE,\n\n    OFFSET_RESET\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/LmqDispatch.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\n\npublic class LmqDispatch {\n    private static final short VALUE_OF_EACH_INCREMENT = 1;\n\n    public static void wrapLmqDispatch(MessageStore messageStore, final MessageExtBrokerInner msg)\n        throws ConsumeQueueException {\n        String lmqNames = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);\n        String[] queueNames = lmqNames.split(MixAll.LMQ_DISPATCH_SEPARATOR);\n        Long[] queueOffsets = new Long[queueNames.length];\n        if (messageStore.getMessageStoreConfig().isEnableLmq()) {\n            for (int i = 0; i < queueNames.length; i++) {\n                if (MixAll.isLmq(queueNames[i])) {\n                    queueOffsets[i] = messageStore.getQueueStore().getLmqQueueOffset(queueNames[i], MixAll.LMQ_QUEUE_ID);\n                }\n            }\n        }\n        MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET,\n            StringUtils.join(queueOffsets, MixAll.LMQ_DISPATCH_SEPARATOR));\n        msg.removeWaitStorePropertyString();\n    }\n\n    public static void updateLmqOffsets(MessageStore messageStore, final MessageExtBrokerInner msgInner)\n        throws ConsumeQueueException {\n        String lmqNames = msgInner.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);\n        String[] queueNames = lmqNames.split(MixAll.LMQ_DISPATCH_SEPARATOR);\n        for (String queueName : queueNames) {\n            if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) {\n                messageStore.getQueueStore().increaseLmqOffset(queueName, MixAll.LMQ_QUEUE_ID, VALUE_OF_EACH_INCREMENT);\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport com.google.common.collect.Lists;\nimport java.io.File;\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Comparator;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.ListIterator;\nimport java.util.concurrent.CopyOnWriteArrayList;\nimport java.util.stream.Stream;\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.logfile.DefaultMappedFile;\nimport org.apache.rocketmq.store.logfile.MappedFile;\n\npublic class MappedFileQueue implements Swappable {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private static final Logger LOG_ERROR = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);\n\n    protected final String storePath;\n\n    protected final int mappedFileSize;\n\n    protected final CopyOnWriteArrayList<MappedFile> mappedFiles = new CopyOnWriteArrayList<>();\n\n    protected final AllocateMappedFileService allocateMappedFileService;\n\n    protected long flushedWhere = 0;\n    protected long committedWhere = 0;\n\n    protected volatile long storeTimestamp = 0;\n\n    protected RunningFlags runningFlags;\n\n    /**\n     * Configuration flag to use RandomAccessFile instead of MappedByteBuffer for writing\n     */\n    protected boolean writeWithoutMmap = false;\n\n    public MappedFileQueue(final String storePath, int mappedFileSize,\n        AllocateMappedFileService allocateMappedFileService) {\n        this(storePath, mappedFileSize, allocateMappedFileService, null, false);\n    }\n\n    public MappedFileQueue(final String storePath, int mappedFileSize,\n        AllocateMappedFileService allocateMappedFileService, RunningFlags runningFlags) {\n        this(storePath, mappedFileSize, allocateMappedFileService, runningFlags, false);\n    }\n\n    public MappedFileQueue(final String storePath, int mappedFileSize,\n        AllocateMappedFileService allocateMappedFileService, boolean writeWithoutMmap) {\n        this(storePath, mappedFileSize, allocateMappedFileService, null, writeWithoutMmap);\n    }\n\n    public MappedFileQueue(final String storePath, int mappedFileSize,\n        AllocateMappedFileService allocateMappedFileService, RunningFlags runningFlags, boolean writeWithoutMmap) {\n        this.storePath = storePath;\n        this.mappedFileSize = mappedFileSize;\n        this.allocateMappedFileService = allocateMappedFileService;\n        this.runningFlags = runningFlags;\n        this.writeWithoutMmap = writeWithoutMmap;\n    }\n\n    public void checkSelf() {\n        List<MappedFile> mappedFiles = new ArrayList<>(this.mappedFiles);\n        if (!mappedFiles.isEmpty()) {\n            Iterator<MappedFile> iterator = mappedFiles.iterator();\n            MappedFile pre = null;\n            while (iterator.hasNext()) {\n                MappedFile cur = iterator.next();\n\n                if (pre != null) {\n                    if (cur.getFileFromOffset() - pre.getFileFromOffset() != this.mappedFileSize) {\n                        LOG_ERROR.error(\"[BUG]The mappedFile queue's data is damaged, the adjacent mappedFile's offset don't match. pre file {}, cur file {}\",\n                            pre.getFileName(), cur.getFileName());\n                    }\n                }\n                pre = cur;\n            }\n        }\n    }\n\n    public MappedFile getConsumeQueueMappedFileByTime(final long timestamp, CommitLog commitLog,\n        BoundaryType boundaryType) {\n        Object[] mfs = copyMappedFiles(0);\n        if (null == mfs) {\n            return null;\n        }\n\n        /*\n         * Make sure each mapped file in consume queue has accurate start and stop time in accordance with commit log\n         * mapped files. Note last modified time from file system is not reliable.\n         */\n        for (int i = mfs.length - 1; i >= 0; i--) {\n            DefaultMappedFile mappedFile = (DefaultMappedFile) mfs[i];\n            // Figure out the earliest message store time in the consume queue mapped file.\n            if (mappedFile.getStartTimestamp() < 0) {\n                SelectMappedBufferResult selectMappedBufferResult = mappedFile.selectMappedBuffer(0, ConsumeQueue.CQ_STORE_UNIT_SIZE);\n                if (null != selectMappedBufferResult) {\n                    try {\n                        ByteBuffer buffer = selectMappedBufferResult.getByteBuffer();\n                        long physicalOffset = buffer.getLong();\n                        int messageSize = buffer.getInt();\n                        long messageStoreTime = commitLog.pickupStoreTimestamp(physicalOffset, messageSize);\n                        if (messageStoreTime > 0) {\n                            mappedFile.setStartTimestamp(messageStoreTime);\n                        }\n                    } finally {\n                        selectMappedBufferResult.release();\n                    }\n                }\n            }\n            // Figure out the latest message store time in the consume queue mapped file.\n            if (i < mfs.length - 1 && mappedFile.getStopTimestamp() < 0) {\n                SelectMappedBufferResult selectMappedBufferResult = mappedFile.selectMappedBuffer(mappedFileSize - ConsumeQueue.CQ_STORE_UNIT_SIZE, ConsumeQueue.CQ_STORE_UNIT_SIZE);\n                if (null != selectMappedBufferResult) {\n                    try {\n                        ByteBuffer buffer = selectMappedBufferResult.getByteBuffer();\n                        long physicalOffset = buffer.getLong();\n                        int messageSize = buffer.getInt();\n                        long messageStoreTime = commitLog.pickupStoreTimestamp(physicalOffset, messageSize);\n                        if (messageStoreTime > 0) {\n                            mappedFile.setStopTimestamp(messageStoreTime);\n                        }\n                    } finally {\n                        selectMappedBufferResult.release();\n                    }\n                }\n            }\n        }\n\n        switch (boundaryType) {\n            case LOWER: {\n                for (int i = 0; i < mfs.length; i++) {\n                    DefaultMappedFile mappedFile = (DefaultMappedFile) mfs[i];\n                    if (i < mfs.length - 1) {\n                        long stopTimestamp = mappedFile.getStopTimestamp();\n                        if (stopTimestamp >= timestamp) {\n                            return mappedFile;\n                        }\n                    }\n\n                    // Just return the latest one.\n                    if (i == mfs.length - 1) {\n                        return mappedFile;\n                    }\n                }\n            }\n            case UPPER: {\n                for (int i = mfs.length - 1; i >= 0; i--) {\n                    DefaultMappedFile mappedFile = (DefaultMappedFile) mfs[i];\n                    if (mappedFile.getStartTimestamp() <= timestamp) {\n                        return mappedFile;\n                    }\n                }\n            }\n\n            default: {\n                log.warn(\"Unknown boundary type\");\n                break;\n            }\n        }\n        return null;\n    }\n\n    public MappedFile getMappedFileByTime(final long timestamp) {\n        Object[] mfs = this.copyMappedFiles(0);\n\n        if (null == mfs)\n            return null;\n\n        for (int i = 0; i < mfs.length; i++) {\n            MappedFile mappedFile = (MappedFile) mfs[i];\n            if (mappedFile.getLastModifiedTimestamp() >= timestamp) {\n                return mappedFile;\n            }\n        }\n\n        return (MappedFile) mfs[mfs.length - 1];\n    }\n\n    protected Object[] copyMappedFiles(final int reservedMappedFiles) {\n        Object[] mfs;\n\n        if (this.mappedFiles.size() <= reservedMappedFiles) {\n            return null;\n        }\n\n        mfs = this.mappedFiles.toArray();\n        return mfs;\n    }\n\n    public void truncateDirtyFiles(long offset) {\n        List<MappedFile> willRemoveFiles = new ArrayList<>();\n\n        for (MappedFile file : this.mappedFiles) {\n            long fileTailOffset = file.getFileFromOffset() + this.mappedFileSize;\n            if (fileTailOffset > offset) {\n                if (offset >= file.getFileFromOffset()) {\n                    file.setWrotePosition((int) (offset % this.mappedFileSize));\n                    file.setCommittedPosition((int) (offset % this.mappedFileSize));\n                    file.setFlushedPosition((int) (offset % this.mappedFileSize));\n                } else {\n                    file.destroy(1000);\n                    willRemoveFiles.add(file);\n                }\n            }\n        }\n\n        this.deleteExpiredFile(willRemoveFiles);\n    }\n\n    void deleteExpiredFile(List<MappedFile> files) {\n\n        if (!files.isEmpty()) {\n\n            Iterator<MappedFile> iterator = files.iterator();\n            while (iterator.hasNext()) {\n                MappedFile cur = iterator.next();\n                if (!this.mappedFiles.contains(cur)) {\n                    iterator.remove();\n                    log.info(\"This mappedFile {} is not contained by mappedFiles, so skip it.\", cur.getFileName());\n                }\n            }\n\n            try {\n                if (!this.mappedFiles.removeAll(files)) {\n                    log.error(\"deleteExpiredFile remove failed.\");\n                }\n            } catch (Exception e) {\n                log.error(\"deleteExpiredFile has exception.\", e);\n            }\n        }\n    }\n\n\n    public boolean load() {\n        File dir = new File(this.storePath);\n        File[] ls = dir.listFiles();\n        if (ls != null) {\n            return doLoad(Arrays.asList(ls));\n        }\n        return true;\n    }\n\n    public boolean doLoad(List<File> files) {\n        // ascending order\n        files.sort(Comparator.comparing(File::getName));\n\n        for (int i = 0; i < files.size(); i++) {\n            File file = files.get(i);\n            if (file.isDirectory()) {\n                continue;\n            }\n\n            if (file.length() == 0 && i == files.size() - 1) {\n                boolean ok = file.delete();\n                log.warn(\"{} size is 0, auto delete. is_ok: {}\", file, ok);\n                continue;\n            }\n\n            if (file.length() != this.mappedFileSize) {\n                log.warn(file + \"\\t\" + file.length()\n                        + \" length not matched message store config value, please check it manually\");\n                return false;\n            }\n\n            try {\n                MappedFile mappedFile = new DefaultMappedFile(file.getPath(), mappedFileSize, runningFlags, writeWithoutMmap);\n\n                mappedFile.setWrotePosition(this.mappedFileSize);\n                mappedFile.setFlushedPosition(this.mappedFileSize);\n                mappedFile.setCommittedPosition(this.mappedFileSize);\n                this.mappedFiles.add(mappedFile);\n                log.info(\"load \" + file.getPath() + \" OK\");\n            } catch (IOException e) {\n                log.error(\"load file \" + file + \" error\", e);\n                return false;\n            }\n        }\n        return true;\n    }\n\n    public long howMuchFallBehind() {\n        if (this.mappedFiles.isEmpty())\n            return 0;\n\n        long committed = this.getFlushedWhere();\n        if (committed != 0) {\n            MappedFile mappedFile = this.getLastMappedFile(0, false);\n            if (mappedFile != null) {\n                return (mappedFile.getFileFromOffset() + mappedFile.getWrotePosition()) - committed;\n            }\n        }\n\n        return 0;\n    }\n\n    public MappedFile getLastMappedFile(final long startOffset, boolean needCreate) {\n        long createOffset = -1;\n        MappedFile mappedFileLast = getLastMappedFile();\n\n        if (mappedFileLast == null) {\n            createOffset = startOffset - (startOffset % this.mappedFileSize);\n        }\n\n        if (mappedFileLast != null && mappedFileLast.isFull()) {\n            createOffset = mappedFileLast.getFileFromOffset() + this.mappedFileSize;\n        }\n\n        if (createOffset != -1 && needCreate) {\n            return tryCreateMappedFile(createOffset);\n        }\n\n        return mappedFileLast;\n    }\n\n    public boolean isMappedFilesEmpty() {\n        return this.mappedFiles.isEmpty();\n    }\n\n    public boolean isEmptyOrCurrentFileFull() {\n        MappedFile mappedFileLast = getLastMappedFile();\n        if (mappedFileLast == null) {\n            return true;\n        }\n        if (mappedFileLast.isFull()) {\n            return true;\n        }\n        return false;\n    }\n\n    public boolean shouldRoll(final int msgSize) {\n        if (isEmptyOrCurrentFileFull()) {\n            return true;\n        }\n        MappedFile mappedFileLast = getLastMappedFile();\n        if (mappedFileLast.getWrotePosition() + msgSize > mappedFileLast.getFileSize()) {\n            return true;\n        }\n        return false;\n    }\n\n    public MappedFile tryCreateMappedFile(long createOffset) {\n        String nextFilePath = this.storePath + File.separator + UtilAll.offset2FileName(createOffset);\n        String nextNextFilePath = this.storePath + File.separator + UtilAll.offset2FileName(createOffset\n                + this.mappedFileSize);\n        return doCreateMappedFile(nextFilePath, nextNextFilePath);\n    }\n\n    protected MappedFile doCreateMappedFile(String nextFilePath, String nextNextFilePath) {\n        MappedFile mappedFile = null;\n\n        if (this.allocateMappedFileService != null) {\n            mappedFile = this.allocateMappedFileService.putRequestAndReturnMappedFile(nextFilePath,\n                    nextNextFilePath, this.mappedFileSize);\n        } else {\n            try {\n                mappedFile = new DefaultMappedFile(nextFilePath, this.mappedFileSize, runningFlags, this.writeWithoutMmap);\n            } catch (IOException e) {\n                log.error(\"create mappedFile exception\", e);\n            }\n        }\n\n        if (mappedFile != null) {\n            if (this.mappedFiles.isEmpty()) {\n                mappedFile.setFirstCreateInQueue(true);\n            }\n            this.mappedFiles.add(mappedFile);\n        }\n\n        return mappedFile;\n    }\n\n    public MappedFile getLastMappedFile(final long startOffset) {\n        return getLastMappedFile(startOffset, true);\n    }\n\n    public MappedFile getLastMappedFile() {\n        MappedFile mappedFileLast = null;\n        while (!this.mappedFiles.isEmpty()) {\n            try {\n                mappedFileLast = this.mappedFiles.get(this.mappedFiles.size() - 1);\n                break;\n            } catch (IndexOutOfBoundsException e) {\n                //continue;\n            } catch (Exception e) {\n                log.error(\"getLastMappedFile has exception.\", e);\n                break;\n            }\n        }\n        return mappedFileLast;\n    }\n\n    public boolean resetOffset(long offset) {\n        MappedFile mappedFileLast = getLastMappedFile();\n\n        if (mappedFileLast != null) {\n            long lastOffset = mappedFileLast.getFileFromOffset() +\n                mappedFileLast.getWrotePosition();\n            long diff = lastOffset - offset;\n\n            final int maxDiff = this.mappedFileSize * 2;\n            if (diff > maxDiff)\n                return false;\n        }\n\n        ListIterator<MappedFile> iterator = this.mappedFiles.listIterator(mappedFiles.size());\n        List<MappedFile> toRemoves = new ArrayList<>();\n\n        while (iterator.hasPrevious()) {\n            mappedFileLast = iterator.previous();\n            if (offset >= mappedFileLast.getFileFromOffset()) {\n                int where = (int) (offset % mappedFileLast.getFileSize());\n                mappedFileLast.setFlushedPosition(where);\n                mappedFileLast.setWrotePosition(where);\n                mappedFileLast.setCommittedPosition(where);\n                break;\n            } else {\n                toRemoves.add(mappedFileLast);\n            }\n        }\n\n        if (!toRemoves.isEmpty()) {\n            this.mappedFiles.removeAll(toRemoves);\n        }\n\n        return true;\n    }\n\n    public long getMinOffset() {\n\n        if (!this.mappedFiles.isEmpty()) {\n            try {\n                return this.mappedFiles.get(0).getFileFromOffset();\n            } catch (IndexOutOfBoundsException e) {\n                //continue;\n            } catch (Exception e) {\n                log.error(\"getMinOffset has exception.\", e);\n            }\n        }\n        return -1;\n    }\n\n    public long getMaxOffset() {\n        MappedFile mappedFile = getLastMappedFile();\n        if (mappedFile != null) {\n            return mappedFile.getFileFromOffset() + mappedFile.getReadPosition();\n        }\n        return 0;\n    }\n\n    public long getMaxWrotePosition() {\n        MappedFile mappedFile = getLastMappedFile();\n        if (mappedFile != null) {\n            return mappedFile.getFileFromOffset() + mappedFile.getWrotePosition();\n        }\n        return 0;\n    }\n\n    public long remainHowManyDataToCommit() {\n        return getMaxWrotePosition() - getCommittedWhere();\n    }\n\n    public long remainHowManyDataToFlush() {\n        return getMaxOffset() - this.getFlushedWhere();\n    }\n\n    public void deleteLastMappedFile() {\n        MappedFile lastMappedFile = getLastMappedFile();\n        if (lastMappedFile != null) {\n            lastMappedFile.destroy(1000);\n            this.mappedFiles.remove(lastMappedFile);\n            log.info(\"on recover, destroy a logic mapped file \" + lastMappedFile.getFileName());\n\n        }\n    }\n\n    public int deleteExpiredFileByTime(final long expiredTime,\n        final int deleteFilesInterval,\n        final long intervalForcibly,\n        final boolean cleanImmediately,\n        final int deleteFileBatchMax) {\n        Object[] mfs = this.copyMappedFiles(0);\n\n        if (null == mfs)\n            return 0;\n\n        int mfsLength = mfs.length - 1;\n        int deleteCount = 0;\n        List<MappedFile> files = new ArrayList<>();\n        int skipFileNum = 0;\n        if (null != mfs) {\n            //do check before deleting\n            checkSelf();\n            for (int i = 0; i < mfsLength; i++) {\n                MappedFile mappedFile = (MappedFile) mfs[i];\n                long liveMaxTimestamp = mappedFile.getLastModifiedTimestamp() + expiredTime;\n                if (System.currentTimeMillis() >= liveMaxTimestamp || cleanImmediately) {\n                    if (skipFileNum > 0) {\n                        log.info(\"Delete CommitLog {} but skip {} files\", mappedFile.getFileName(), skipFileNum);\n                    }\n                    if (mappedFile.destroy(intervalForcibly)) {\n                        files.add(mappedFile);\n                        deleteCount++;\n\n                        if (files.size() >= deleteFileBatchMax) {\n                            break;\n                        }\n\n                        if (deleteFilesInterval > 0 && (i + 1) < mfsLength) {\n                            try {\n                                Thread.sleep(deleteFilesInterval);\n                            } catch (InterruptedException e) {\n                            }\n                        }\n                    } else {\n                        break;\n                    }\n                } else {\n                    skipFileNum++;\n                    //avoid deleting files in the middle\n                    break;\n                }\n            }\n        }\n\n        deleteExpiredFile(files);\n\n        return deleteCount;\n    }\n\n    public int deleteExpiredFileByOffset(long offset, int unitSize) {\n        Object[] mfs = this.copyMappedFiles(0);\n\n        List<MappedFile> files = new ArrayList<>();\n        int deleteCount = 0;\n        if (null != mfs) {\n\n            int mfsLength = mfs.length - 1;\n\n            for (int i = 0; i < mfsLength; i++) {\n                boolean destroy;\n                MappedFile mappedFile = (MappedFile) mfs[i];\n                SelectMappedBufferResult result = mappedFile.selectMappedBuffer(this.mappedFileSize - unitSize);\n                if (result != null) {\n                    long maxOffsetInLogicQueue = result.getByteBuffer().getLong();\n                    result.release();\n                    destroy = maxOffsetInLogicQueue < offset;\n                    if (destroy) {\n                        log.info(\"physic min offset \" + offset + \", logics in current mappedFile max offset \"\n                            + maxOffsetInLogicQueue + \", delete it\");\n                    }\n                } else if (!mappedFile.isAvailable()) { // Handle hanged file.\n                    log.warn(\"Found a hanged consume queue file, attempting to delete it.\");\n                    destroy = true;\n                } else {\n                    log.warn(\"this being not executed forever.\");\n                    break;\n                }\n\n                if (destroy && mappedFile.destroy(1000 * 60)) {\n                    files.add(mappedFile);\n                    deleteCount++;\n                } else {\n                    break;\n                }\n            }\n        }\n\n        deleteExpiredFile(files);\n\n        return deleteCount;\n    }\n\n    public int deleteExpiredFileByOffsetForTimerLog(long offset, int checkOffset, int unitSize) {\n        Object[] mfs = this.copyMappedFiles(0);\n\n        List<MappedFile> files = new ArrayList<>();\n        int deleteCount = 0;\n        if (null != mfs) {\n\n            int mfsLength = mfs.length - 1;\n\n            for (int i = 0; i < mfsLength; i++) {\n                boolean destroy = false;\n                MappedFile mappedFile = (MappedFile) mfs[i];\n                SelectMappedBufferResult result = mappedFile.selectMappedBuffer(checkOffset);\n                try {\n                    if (result != null) {\n                        int position = result.getByteBuffer().position();\n                        int size = result.getByteBuffer().getInt();//size\n                        result.getByteBuffer().getLong(); //prev pos\n                        int magic = result.getByteBuffer().getInt();\n                        if (size == unitSize && (magic | 0xF) == 0xF) {\n                            result.getByteBuffer().position(position + MixAll.UNIT_PRE_SIZE_FOR_MSG);\n                            long maxOffsetPy = result.getByteBuffer().getLong();\n                            destroy = maxOffsetPy < offset;\n                            if (destroy) {\n                                log.info(\"physic min commitlog offset \" + offset + \", current mappedFile's max offset \"\n                                    + maxOffsetPy + \", delete it\");\n                            }\n                        } else {\n                            log.warn(\"Found error data in [{}] checkOffset:{} unitSize:{}\", mappedFile.getFileName(),\n                                checkOffset, unitSize);\n                        }\n                    } else if (!mappedFile.isAvailable()) { // Handle hanged file.\n                        log.warn(\"Found a hanged consume queue file, attempting to delete it.\");\n                        destroy = true;\n                    } else {\n                        log.warn(\"this being not executed forever.\");\n                        break;\n                    }\n                } finally {\n                    if (null != result) {\n                        result.release();\n                    }\n                }\n\n                if (destroy && mappedFile.destroy(1000 * 60)) {\n                    files.add(mappedFile);\n                    deleteCount++;\n                } else {\n                    break;\n                }\n            }\n        }\n\n        deleteExpiredFile(files);\n\n        return deleteCount;\n    }\n\n    public boolean flush(final int flushLeastPages) {\n        boolean result = true;\n        MappedFile mappedFile = this.findMappedFileByOffset(this.getFlushedWhere(), this.getFlushedWhere() == 0);\n        if (mappedFile != null) {\n            long tmpTimeStamp = mappedFile.getStoreTimestamp();\n            int offset = mappedFile.flush(flushLeastPages);\n            long where = mappedFile.getFileFromOffset() + offset;\n            result = where == this.getFlushedWhere();\n            this.setFlushedWhere(where);\n            if (0 == flushLeastPages) {\n                this.setStoreTimestamp(tmpTimeStamp);\n            }\n        }\n\n        return result;\n    }\n\n    public synchronized boolean commit(final int commitLeastPages) {\n        boolean result = true;\n        MappedFile mappedFile = this.findMappedFileByOffset(this.getCommittedWhere(), this.getCommittedWhere() == 0);\n        if (mappedFile != null) {\n            int offset = mappedFile.commit(commitLeastPages);\n            long where = mappedFile.getFileFromOffset() + offset;\n            result = where == this.getCommittedWhere();\n            this.setCommittedWhere(where);\n        }\n\n        return result;\n    }\n\n    /**\n     * Finds a mapped file by offset.\n     *\n     * @param offset Offset.\n     * @param returnFirstOnNotFound If the mapped file is not found, then return the first one.\n     * @return Mapped file or null (when not found and returnFirstOnNotFound is <code>false</code>).\n     */\n    public MappedFile findMappedFileByOffset(final long offset, final boolean returnFirstOnNotFound) {\n        try {\n            MappedFile firstMappedFile = this.getFirstMappedFile();\n            MappedFile lastMappedFile = this.getLastMappedFile();\n            if (firstMappedFile != null && lastMappedFile != null) {\n                if (offset < firstMappedFile.getFileFromOffset() || offset >= lastMappedFile.getFileFromOffset() + this.mappedFileSize) {\n                    LOG_ERROR.warn(\"Offset not matched. Request offset: {}, firstOffset: {}, lastOffset: {}, mappedFileSize: {}, mappedFiles count: {}\",\n                        offset,\n                        firstMappedFile.getFileFromOffset(),\n                        lastMappedFile.getFileFromOffset() + this.mappedFileSize,\n                        this.mappedFileSize,\n                        this.mappedFiles.size());\n                } else {\n                    int index = (int) ((offset / this.mappedFileSize) - (firstMappedFile.getFileFromOffset() / this.mappedFileSize));\n                    MappedFile targetFile = null;\n                    try {\n                        targetFile = this.mappedFiles.get(index);\n                    } catch (Exception ignored) {\n                    }\n\n                    if (targetFile != null && offset >= targetFile.getFileFromOffset()\n                        && offset < targetFile.getFileFromOffset() + this.mappedFileSize) {\n                        return targetFile;\n                    }\n\n                    for (MappedFile tmpMappedFile : this.mappedFiles) {\n                        if (offset >= tmpMappedFile.getFileFromOffset()\n                            && offset < tmpMappedFile.getFileFromOffset() + this.mappedFileSize) {\n                            return tmpMappedFile;\n                        }\n                    }\n                }\n\n                if (returnFirstOnNotFound) {\n                    return firstMappedFile;\n                }\n            }\n        } catch (Exception e) {\n            log.error(\"findMappedFileByOffset Exception\", e);\n        }\n\n        return null;\n    }\n\n    public MappedFile getFirstMappedFile() {\n        MappedFile mappedFileFirst = null;\n\n        if (!this.mappedFiles.isEmpty()) {\n            try {\n                mappedFileFirst = this.mappedFiles.get(0);\n            } catch (IndexOutOfBoundsException e) {\n                //ignore\n            } catch (Exception e) {\n                log.error(\"getFirstMappedFile has exception.\", e);\n            }\n        }\n\n        return mappedFileFirst;\n    }\n\n    public MappedFile findMappedFileByOffset(final long offset) {\n        return findMappedFileByOffset(offset, false);\n    }\n\n    public long getMappedMemorySize() {\n        long size = 0;\n\n        Object[] mfs = this.copyMappedFiles(0);\n        if (mfs != null) {\n            for (Object mf : mfs) {\n                if (((ReferenceResource) mf).isAvailable()) {\n                    size += this.mappedFileSize;\n                }\n            }\n        }\n\n        return size;\n    }\n\n    public boolean retryDeleteFirstFile(final long intervalForcibly) {\n        MappedFile mappedFile = this.getFirstMappedFile();\n        if (mappedFile != null) {\n            if (!mappedFile.isAvailable()) {\n                log.warn(\"the mappedFile was destroyed once, but still alive, \" + mappedFile.getFileName());\n                boolean result = mappedFile.destroy(intervalForcibly);\n                if (result) {\n                    log.info(\"the mappedFile re delete OK, \" + mappedFile.getFileName());\n                    List<MappedFile> tmpFiles = new ArrayList<>();\n                    tmpFiles.add(mappedFile);\n                    this.deleteExpiredFile(tmpFiles);\n                } else {\n                    log.warn(\"the mappedFile re delete failed, \" + mappedFile.getFileName());\n                }\n\n                return result;\n            }\n        }\n\n        return false;\n    }\n\n    public void shutdown(final long intervalForcibly) {\n        for (MappedFile mf : this.mappedFiles) {\n            mf.shutdown(intervalForcibly);\n        }\n    }\n\n    public void cleanResourcesAll() {\n        for (MappedFile mf : this.mappedFiles) {\n            mf.cleanResources();\n        }\n    }\n\n    public void destroy() {\n        for (MappedFile mf : this.mappedFiles) {\n            mf.destroy(1000 * 3);\n        }\n        this.mappedFiles.clear();\n        this.setFlushedWhere(0);\n\n        // delete parent directory\n        File file = new File(storePath);\n        if (file.isDirectory()) {\n            file.delete();\n        }\n    }\n\n    @Override\n    public void swapMap(int reserveNum, long forceSwapIntervalMs, long normalSwapIntervalMs) {\n\n        if (mappedFiles.isEmpty()) {\n            return;\n        }\n\n        if (reserveNum < 3) {\n            reserveNum = 3;\n        }\n\n        Object[] mfs = this.copyMappedFiles(0);\n        if (null == mfs) {\n            return;\n        }\n\n        for (int i = mfs.length - reserveNum - 1; i >= 0; i--) {\n            MappedFile mappedFile = (MappedFile) mfs[i];\n            if (System.currentTimeMillis() - mappedFile.getRecentSwapMapTime() > forceSwapIntervalMs) {\n                mappedFile.swapMap();\n                continue;\n            }\n            if (System.currentTimeMillis() - mappedFile.getRecentSwapMapTime() > normalSwapIntervalMs\n                    && mappedFile.getMappedByteBufferAccessCountSinceLastSwap() > 0) {\n                mappedFile.swapMap();\n                continue;\n            }\n        }\n    }\n\n    @Override\n    public void cleanSwappedMap(long forceCleanSwapIntervalMs) {\n\n        if (mappedFiles.isEmpty()) {\n            return;\n        }\n\n        int reserveNum = 3;\n        Object[] mfs = this.copyMappedFiles(0);\n        if (null == mfs) {\n            return;\n        }\n\n        for (int i = mfs.length - reserveNum - 1; i >= 0; i--) {\n            MappedFile mappedFile = (MappedFile) mfs[i];\n            if (System.currentTimeMillis() - mappedFile.getRecentSwapMapTime() > forceCleanSwapIntervalMs) {\n                mappedFile.cleanSwapedMap(false);\n            }\n        }\n    }\n\n    public Object[] snapshot() {\n        // return a safe copy\n        return this.mappedFiles.toArray();\n    }\n\n    public Stream<MappedFile> stream() {\n        return this.mappedFiles.stream();\n    }\n\n    public Stream<MappedFile> reversedStream() {\n        return Lists.reverse(this.mappedFiles).stream();\n    }\n\n    public long getFlushedWhere() {\n        return flushedWhere;\n    }\n\n    public void setFlushedWhere(long flushedWhere) {\n        this.flushedWhere = flushedWhere;\n    }\n\n    public long getStoreTimestamp() {\n        return storeTimestamp;\n    }\n\n    public void setStoreTimestamp(long storeTimestamp) {\n        this.storeTimestamp = storeTimestamp;\n    }\n\n    public List<MappedFile> getMappedFiles() {\n        return mappedFiles;\n    }\n\n    public int getMappedFileSize() {\n        return mappedFileSize;\n    }\n\n    public long getCommittedWhere() {\n        return committedWhere;\n    }\n\n    public void setCommittedWhere(final long committedWhere) {\n        this.committedWhere = committedWhere;\n    }\n\n    public long getTotalFileSize() {\n        return (long) mappedFileSize * mappedFiles.size();\n    }\n\n    public String getStorePath() {\n        return storePath;\n    }\n\n    public List<MappedFile> range(final long from, final long to) {\n        Object[] mfs = copyMappedFiles(0);\n        if (null == mfs) {\n            return new ArrayList<>();\n        }\n\n        List<MappedFile> result = new ArrayList<>();\n        for (Object mf : mfs) {\n            MappedFile mappedFile = (MappedFile) mf;\n            if (mappedFile.getFileFromOffset() + mappedFile.getFileSize() <= from) {\n                continue;\n            }\n\n            if (to <= mappedFile.getFileFromOffset()) {\n                break;\n            }\n            result.add(mappedFile);\n        }\n\n        return result;\n    }\n\n    public MappedFile getEarliestMappedFile() {\n        MappedFile mappedFile = null;\n        while (!this.mappedFiles.isEmpty()) {\n            try {\n                mappedFile = this.mappedFiles.get(0);\n                break;\n            } catch (IndexOutOfBoundsException e) {\n                //continue;\n            } catch (Exception e) {\n                log.error(\"getEarliestMappedFile error: {}\", e.getMessage());\n                break;\n            }\n        }\n        return mappedFile;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/MessageArrivingListener.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport java.util.Map;\n\npublic interface MessageArrivingListener {\n\n    /**\n     * Notify that a new message arrives in a consume queue\n     * @param topic topic name\n     * @param queueId consume queue id\n     * @param logicOffset consume queue offset\n     * @param tagsCode message tags hash code\n     * @param msgStoreTime message store time\n     * @param filterBitMap message bloom filter\n     * @param properties message properties\n     */\n    void arriving(String topic, int queueId, long logicOffset, long tagsCode,\n        long msgStoreTime, byte[] filterBitMap, Map<String, String> properties);\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/MessageExtEncoder.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport io.netty.buffer.ByteBuf;\nimport io.netty.buffer.ByteBufAllocator;\nimport io.netty.buffer.UnpooledByteBufAllocator;\nimport java.nio.ByteBuffer;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExtBatch;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.message.MessageVersion;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\n\npublic class MessageExtEncoder {\n    protected static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private ByteBuf byteBuf;\n    // The maximum length of the message body.\n    private int maxMessageBodySize;\n    // The maximum length of the full message.\n    private int maxMessageSize;\n    private final int crc32ReservedLength;\n    private MessageStoreConfig messageStoreConfig;\n\n    public MessageExtEncoder(final int maxMessageBodySize, final MessageStoreConfig messageStoreConfig) {\n        this(messageStoreConfig);\n    }\n\n    public MessageExtEncoder(final MessageStoreConfig messageStoreConfig) {\n        ByteBufAllocator alloc = UnpooledByteBufAllocator.DEFAULT;\n        this.messageStoreConfig = messageStoreConfig;\n        this.maxMessageBodySize = messageStoreConfig.getMaxMessageSize();\n        //Reserve 64kb for encoding buffer outside body\n        int maxMessageSize = Integer.MAX_VALUE - maxMessageBodySize >= 64 * 1024 ?\n            maxMessageBodySize + 64 * 1024 : Integer.MAX_VALUE;\n        byteBuf = alloc.directBuffer(maxMessageSize);\n        this.maxMessageSize = maxMessageSize;\n        this.crc32ReservedLength = messageStoreConfig.isEnabledAppendPropCRC() ? CommitLog.CRC32_RESERVED_LEN : 0;\n    }\n\n    public static int calMsgLength(MessageVersion messageVersion,\n        int sysFlag, int bodyLength, int topicLength, int propertiesLength) {\n\n        int bornhostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20;\n        int storehostAddressLength = (sysFlag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 8 : 20;\n\n        return 4 //TOTALSIZE\n            + 4 //MAGICCODE\n            + 4 //BODYCRC\n            + 4 //QUEUEID\n            + 4 //FLAG\n            + 8 //QUEUEOFFSET\n            + 8 //PHYSICALOFFSET\n            + 4 //SYSFLAG\n            + 8 //BORNTIMESTAMP\n            + bornhostLength //BORNHOST\n            + 8 //STORETIMESTAMP\n            + storehostAddressLength //STOREHOSTADDRESS\n            + 4 //RECONSUMETIMES\n            + 8 //Prepared Transaction Offset\n            + 4 + (Math.max(bodyLength, 0)) //BODY\n            + messageVersion.getTopicLengthSize() + topicLength //TOPIC\n            + 2 + (Math.max(propertiesLength, 0)); //propertiesLength\n    }\n\n    public static int calMsgLengthNoProperties(MessageVersion messageVersion,\n                                               int sysFlag, int bodyLength, int topicLength) {\n\n        int bornhostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20;\n        int storehostAddressLength = (sysFlag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 8 : 20;\n\n        return 4 //TOTALSIZE\n                + 4 //MAGICCODE\n                + 4 //BODYCRC\n                + 4 //QUEUEID\n                + 4 //FLAG\n                + 8 //QUEUEOFFSET\n                + 8 //PHYSICALOFFSET\n                + 4 //SYSFLAG\n                + 8 //BORNTIMESTAMP\n                + bornhostLength //BORNHOST\n                + 8 //STORETIMESTAMP\n                + storehostAddressLength //STOREHOSTADDRESS\n                + 4 //RECONSUMETIMES\n                + 8 //Prepared Transaction Offset\n                + 4 + (Math.max(bodyLength, 0)) //BODY\n                + messageVersion.getTopicLengthSize() + topicLength; //TOPIC\n    }\n\n    public PutMessageResult encodeWithoutProperties(MessageExtBrokerInner msgInner) {\n\n        final byte[] topicData = msgInner.getTopic().getBytes(MessageDecoder.CHARSET_UTF8);\n        final int topicLength = topicData.length;\n\n        final int bodyLength = msgInner.getBody() == null ? 0 : msgInner.getBody().length;\n\n        // Exceeds the maximum message body\n        if (bodyLength > this.maxMessageBodySize) {\n            CommitLog.log.warn(\"message body size exceeded, msg body size: \" + bodyLength\n                    + \", maxMessageSize: \" + this.maxMessageBodySize);\n            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null);\n        }\n\n        final int msgLenNoProperties = calMsgLengthNoProperties(msgInner.getVersion(), msgInner.getSysFlag(), bodyLength, topicLength);\n\n        // 1 TOTALSIZE\n        this.byteBuf.writeInt(msgLenNoProperties);\n        // 2 MAGICCODE\n        this.byteBuf.writeInt(msgInner.getVersion().getMagicCode());\n        // 3 BODYCRC\n        this.byteBuf.writeInt(msgInner.getBodyCRC());\n        // 4 QUEUEID\n        this.byteBuf.writeInt(msgInner.getQueueId());\n        // 5 FLAG\n        this.byteBuf.writeInt(msgInner.getFlag());\n        // 6 QUEUEOFFSET, need update later\n        this.byteBuf.writeLong(0);\n        // 7 PHYSICALOFFSET, need update later\n        this.byteBuf.writeLong(0);\n        // 8 SYSFLAG\n        this.byteBuf.writeInt(msgInner.getSysFlag());\n        // 9 BORNTIMESTAMP\n        this.byteBuf.writeLong(msgInner.getBornTimestamp());\n\n        // 10 BORNHOST\n        ByteBuffer bornHostBytes = msgInner.getBornHostBytes();\n        this.byteBuf.writeBytes(bornHostBytes.array());\n\n        // 11 STORETIMESTAMP\n        this.byteBuf.writeLong(msgInner.getStoreTimestamp());\n\n        // 12 STOREHOSTADDRESS\n        ByteBuffer storeHostBytes = msgInner.getStoreHostBytes();\n        this.byteBuf.writeBytes(storeHostBytes.array());\n\n        // 13 RECONSUMETIMES\n        this.byteBuf.writeInt(msgInner.getReconsumeTimes());\n        // 14 Prepared Transaction Offset\n        this.byteBuf.writeLong(msgInner.getPreparedTransactionOffset());\n        // 15 BODY\n        this.byteBuf.writeInt(bodyLength);\n        if (bodyLength > 0)\n            this.byteBuf.writeBytes(msgInner.getBody());\n\n        // 16 TOPIC\n        if (MessageVersion.MESSAGE_VERSION_V2.equals(msgInner.getVersion())) {\n            this.byteBuf.writeShort((short) topicLength);\n        } else {\n            this.byteBuf.writeByte((byte) topicLength);\n        }\n        this.byteBuf.writeBytes(topicData);\n\n        return null;\n    }\n\n    public PutMessageResult encode(MessageExtBrokerInner msgInner) {\n        this.byteBuf.clear();\n\n        if (messageStoreConfig.isEnableLmq() && msgInner.needDispatchLMQ()) {\n            return encodeWithoutProperties(msgInner);\n        }\n\n        /*\n         * Serialize message\n         */\n        final byte[] propertiesData =\n            msgInner.getPropertiesString() == null ? null : msgInner.getPropertiesString().getBytes(MessageDecoder.CHARSET_UTF8);\n\n        boolean needAppendLastPropertySeparator = crc32ReservedLength > 0 && propertiesData != null && propertiesData.length > 0\n            && propertiesData[propertiesData.length - 1] != MessageDecoder.PROPERTY_SEPARATOR;\n\n        final int propertiesLength = (propertiesData == null ? 0 : propertiesData.length) + (needAppendLastPropertySeparator ? 1 : 0) + crc32ReservedLength;\n\n        if (propertiesLength > Short.MAX_VALUE) {\n            log.warn(\"putMessage message properties length too long. length={}\", propertiesLength);\n            return new PutMessageResult(PutMessageStatus.PROPERTIES_SIZE_EXCEEDED, null);\n        }\n\n        final byte[] topicData = msgInner.getTopic().getBytes(MessageDecoder.CHARSET_UTF8);\n        final int topicLength = topicData.length;\n\n        final int bodyLength = msgInner.getBody() == null ? 0 : msgInner.getBody().length;\n        final int msgLen = calMsgLength(\n            msgInner.getVersion(), msgInner.getSysFlag(), bodyLength, topicLength, propertiesLength);\n\n        // Exceeds the maximum message body\n        if (bodyLength > this.maxMessageBodySize) {\n            CommitLog.log.warn(\"message body size exceeded, msg total size: \" + msgLen + \", msg body size: \" + bodyLength\n                + \", maxMessageSize: \" + this.maxMessageBodySize);\n            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null);\n        }\n\n        final long queueOffset = msgInner.getQueueOffset();\n\n        // Exceeds the maximum message\n        if (msgLen > this.maxMessageSize) {\n            CommitLog.log.warn(\"message size exceeded, msg total size: \" + msgLen + \", msg body size: \" + bodyLength\n                + \", maxMessageSize: \" + this.maxMessageSize);\n            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null);\n        }\n\n        // 1 TOTALSIZE\n        this.byteBuf.writeInt(msgLen);\n        // 2 MAGICCODE\n        this.byteBuf.writeInt(msgInner.getVersion().getMagicCode());\n        // 3 BODYCRC\n        this.byteBuf.writeInt(msgInner.getBodyCRC());\n        // 4 QUEUEID\n        this.byteBuf.writeInt(msgInner.getQueueId());\n        // 5 FLAG\n        this.byteBuf.writeInt(msgInner.getFlag());\n        // 6 QUEUEOFFSET\n        this.byteBuf.writeLong(queueOffset);\n        // 7 PHYSICALOFFSET, need update later\n        this.byteBuf.writeLong(0);\n        // 8 SYSFLAG\n        this.byteBuf.writeInt(msgInner.getSysFlag());\n        // 9 BORNTIMESTAMP\n        this.byteBuf.writeLong(msgInner.getBornTimestamp());\n\n        // 10 BORNHOST\n        ByteBuffer bornHostBytes = msgInner.getBornHostBytes();\n        this.byteBuf.writeBytes(bornHostBytes.array());\n\n        // 11 STORETIMESTAMP\n        this.byteBuf.writeLong(msgInner.getStoreTimestamp());\n\n        // 12 STOREHOSTADDRESS\n        ByteBuffer storeHostBytes = msgInner.getStoreHostBytes();\n        this.byteBuf.writeBytes(storeHostBytes.array());\n\n        // 13 RECONSUMETIMES\n        this.byteBuf.writeInt(msgInner.getReconsumeTimes());\n        // 14 Prepared Transaction Offset\n        this.byteBuf.writeLong(msgInner.getPreparedTransactionOffset());\n        // 15 BODY\n        this.byteBuf.writeInt(bodyLength);\n        if (bodyLength > 0)\n            this.byteBuf.writeBytes(msgInner.getBody());\n\n        // 16 TOPIC\n        if (MessageVersion.MESSAGE_VERSION_V2.equals(msgInner.getVersion())) {\n            this.byteBuf.writeShort((short) topicLength);\n        } else {\n            this.byteBuf.writeByte((byte) topicLength);\n        }\n        this.byteBuf.writeBytes(topicData);\n\n        // 17 PROPERTIES\n        this.byteBuf.writeShort((short) propertiesLength);\n        if (propertiesLength > crc32ReservedLength) {\n            this.byteBuf.writeBytes(propertiesData);\n        }\n        if (needAppendLastPropertySeparator) {\n            this.byteBuf.writeByte((byte) MessageDecoder.PROPERTY_SEPARATOR);\n        }\n        // 18 CRC32\n        this.byteBuf.writerIndex(this.byteBuf.writerIndex() + crc32ReservedLength);\n\n        return null;\n    }\n\n    public ByteBuffer encode(final MessageExtBatch messageExtBatch, PutMessageContext putMessageContext) {\n        this.byteBuf.clear();\n\n        ByteBuffer messagesByteBuff = messageExtBatch.wrap();\n\n        int totalLength = messagesByteBuff.limit();\n        if (totalLength > this.maxMessageBodySize) {\n            CommitLog.log.warn(\"message body size exceeded, msg body size: \" + totalLength + \", maxMessageSize: \" + this.maxMessageBodySize);\n            throw new RuntimeException(\"message body size exceeded\");\n        }\n\n\n        int batchSize = 0;\n        while (messagesByteBuff.hasRemaining()) {\n            batchSize++;\n            // 1 TOTALSIZE\n            messagesByteBuff.getInt();\n            // 2 MAGICCODE\n            messagesByteBuff.getInt();\n            // 3 BODYCRC\n            messagesByteBuff.getInt();\n            // 4 FLAG\n            int flag = messagesByteBuff.getInt();\n            // 5 BODY\n            int bodyLen = messagesByteBuff.getInt();\n            int bodyPos = messagesByteBuff.position();\n            int bodyCrc = UtilAll.crc32(messagesByteBuff.array(), bodyPos, bodyLen);\n            messagesByteBuff.position(bodyPos + bodyLen);\n            // 6 properties\n            short propertiesLen = messagesByteBuff.getShort();\n            int propertiesPos = messagesByteBuff.position();\n            messagesByteBuff.position(propertiesPos + propertiesLen);\n\n            final byte[] topicData = messageExtBatch.getTopic().getBytes(MessageDecoder.CHARSET_UTF8);\n\n            final int topicLength = topicData.length;\n            int totalPropLen = propertiesLen;\n\n            // properties need to add crc32\n            totalPropLen += crc32ReservedLength;\n            final int msgLen = calMsgLength(\n                messageExtBatch.getVersion(), messageExtBatch.getSysFlag(), bodyLen, topicLength, totalPropLen);\n\n            // 1 TOTALSIZE\n            this.byteBuf.writeInt(msgLen);\n            // 2 MAGICCODE\n            this.byteBuf.writeInt(messageExtBatch.getVersion().getMagicCode());\n            // 3 BODYCRC\n            this.byteBuf.writeInt(bodyCrc);\n            // 4 QUEUEID\n            this.byteBuf.writeInt(messageExtBatch.getQueueId());\n            // 5 FLAG\n            this.byteBuf.writeInt(flag);\n            // 6 QUEUEOFFSET\n            this.byteBuf.writeLong(0);\n            // 7 PHYSICALOFFSET\n            this.byteBuf.writeLong(0);\n            // 8 SYSFLAG\n            this.byteBuf.writeInt(messageExtBatch.getSysFlag());\n            // 9 BORNTIMESTAMP\n            this.byteBuf.writeLong(messageExtBatch.getBornTimestamp());\n\n            // 10 BORNHOST\n            ByteBuffer bornHostBytes = messageExtBatch.getBornHostBytes();\n            this.byteBuf.writeBytes(bornHostBytes.array());\n\n            // 11 STORETIMESTAMP\n            this.byteBuf.writeLong(messageExtBatch.getStoreTimestamp());\n\n            // 12 STOREHOSTADDRESS\n            ByteBuffer storeHostBytes = messageExtBatch.getStoreHostBytes();\n            this.byteBuf.writeBytes(storeHostBytes.array());\n\n            // 13 RECONSUMETIMES\n            this.byteBuf.writeInt(messageExtBatch.getReconsumeTimes());\n            // 14 Prepared Transaction Offset, batch does not support transaction\n            this.byteBuf.writeLong(0);\n            // 15 BODY\n            this.byteBuf.writeInt(bodyLen);\n            if (bodyLen > 0)\n                this.byteBuf.writeBytes(messagesByteBuff.array(), bodyPos, bodyLen);\n\n            // 16 TOPIC\n            if (MessageVersion.MESSAGE_VERSION_V2.equals(messageExtBatch.getVersion())) {\n                this.byteBuf.writeShort((short) topicLength);\n            } else {\n                this.byteBuf.writeByte((byte) topicLength);\n            }\n            this.byteBuf.writeBytes(topicData);\n\n            // 17 PROPERTIES\n            this.byteBuf.writeShort((short) totalPropLen);\n            if (propertiesLen > 0) {\n                this.byteBuf.writeBytes(messagesByteBuff.array(), propertiesPos, propertiesLen);\n            }\n            this.byteBuf.writerIndex(this.byteBuf.writerIndex() + crc32ReservedLength);\n        }\n        putMessageContext.setBatchSize(batchSize);\n        putMessageContext.setPhyPos(new long[batchSize]);\n\n        return this.byteBuf.nioBuffer();\n    }\n\n    public ByteBuffer getEncoderBuffer() {\n        return this.byteBuf.nioBuffer(0, this.byteBuf.capacity());\n    }\n\n    public int getMaxMessageBodySize() {\n        return this.maxMessageBodySize;\n    }\n\n    public void updateEncoderBufferCapacity(int newMaxMessageBodySize) {\n        this.maxMessageBodySize = newMaxMessageBodySize;\n        //Reserve 64kb for encoding buffer outside body\n        this.maxMessageSize = Integer.MAX_VALUE - newMaxMessageBodySize >= 64 * 1024 ?\n            this.maxMessageBodySize + 64 * 1024 : Integer.MAX_VALUE;\n        this.byteBuf.capacity(this.maxMessageSize);\n    }\n\n    static class PutMessageThreadLocal {\n        private final MessageExtEncoder encoder;\n        private final StringBuilder keyBuilder;\n\n        PutMessageThreadLocal(MessageStoreConfig messageStoreConfig) {\n            encoder = new MessageExtEncoder(messageStoreConfig);\n            keyBuilder = new StringBuilder();\n        }\n\n        public MessageExtEncoder getEncoder() {\n            return encoder;\n        }\n\n        public StringBuilder getKeyBuilder() {\n            return keyBuilder;\n        }\n    }\n\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/MessageFilter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport java.nio.ByteBuffer;\nimport java.util.Map;\n\npublic interface MessageFilter {\n    /**\n     * match by tags code or filter bit map which is calculated when message received\n     * and stored in consume queue ext.\n     *\n     * @param tagsCode tagsCode\n     * @param cqExtUnit extend unit of consume queue\n     */\n    boolean isMatchedByConsumeQueue(final Long tagsCode,\n        final ConsumeQueueExt.CqExtUnit cqExtUnit);\n\n    /**\n     * match by message content which are stored in commit log.\n     * <br>{@code msgBuffer} and {@code properties} are not all null.If invoked in store,\n     * {@code properties} is null;If invoked in {@code PullRequestHoldService}, {@code msgBuffer} is null.\n     *\n     * @param msgBuffer message buffer in commit log, may be null if not invoked in store.\n     * @param properties message properties, should decode from buffer if null by yourself.\n     */\n    boolean isMatchedByCommitLog(final ByteBuffer msgBuffer,\n        final Map<String, String> properties);\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/MessageStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport io.opentelemetry.api.common.AttributesBuilder;\nimport io.opentelemetry.api.metrics.Meter;\nimport io.opentelemetry.sdk.metrics.InstrumentSelector;\nimport io.opentelemetry.sdk.metrics.ViewBuilder;\nimport java.nio.ByteBuffer;\nimport java.util.HashMap;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.function.Supplier;\nimport javax.annotation.Nonnull;\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.SystemClock;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBatch;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.ha.HAService;\nimport org.apache.rocketmq.store.hook.PutMessageHook;\nimport org.apache.rocketmq.store.hook.SendMessageBackHook;\nimport org.apache.rocketmq.store.logfile.MappedFile;\nimport org.apache.rocketmq.store.queue.ConsumeQueueInterface;\nimport org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface;\nimport org.apache.rocketmq.store.rocksdb.MessageRocksDBStorage;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.apache.rocketmq.store.timer.TimerMessageStore;\nimport org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStore;\nimport org.apache.rocketmq.store.transaction.TransMessageRocksDBStore;\nimport org.apache.rocketmq.store.util.PerfCounter;\nimport org.apache.rocketmq.store.metrics.StoreMetricsManager;\nimport org.rocksdb.RocksDBException;\n\n/**\n * This class defines contracting interfaces to implement, allowing third-party vendor to use customized message store.\n */\npublic interface MessageStore {\n\n    /**\n     * Load previously stored messages.\n     *\n     * @return true if success; false otherwise.\n     */\n    boolean load();\n\n    /**\n     * Launch this message store.\n     *\n     * @throws Exception if there is any error.\n     */\n    void start() throws Exception;\n\n    /**\n     * Shutdown this message store.\n     */\n    void shutdown();\n\n    /**\n     * Destroy this message store. Generally, all persistent files should be removed after invocation.\n     */\n    void destroy();\n\n    /**\n     * Store a message into store in async manner, the processor can process the next request rather than wait for\n     * result when result is completed, notify the client in async manner\n     *\n     * @param msg MessageInstance to store\n     * @return a CompletableFuture for the result of store operation\n     */\n    default CompletableFuture<PutMessageResult> asyncPutMessage(final MessageExtBrokerInner msg) {\n        return CompletableFuture.completedFuture(putMessage(msg));\n    }\n\n    /**\n     * Store a batch of messages in async manner\n     *\n     * @param messageExtBatch the message batch\n     * @return a CompletableFuture for the result of store operation\n     */\n    default CompletableFuture<PutMessageResult> asyncPutMessages(final MessageExtBatch messageExtBatch) {\n        return CompletableFuture.completedFuture(putMessages(messageExtBatch));\n    }\n\n    /**\n     * Store a message into store.\n     *\n     * @param msg Message instance to store\n     * @return result of store operation.\n     */\n    PutMessageResult putMessage(final MessageExtBrokerInner msg);\n\n    /**\n     * Store a batch of messages.\n     *\n     * @param messageExtBatch Message batch.\n     * @return result of storing batch messages.\n     */\n    PutMessageResult putMessages(final MessageExtBatch messageExtBatch);\n\n    /**\n     * Query at most <code>maxMsgNums</code> messages belonging to <code>topic</code> at <code>queueId</code> starting\n     * from given <code>offset</code>. Resulting messages will further be screened using provided message filter.\n     *\n     * @param group         Consumer group that launches this query.\n     * @param topic         Topic to query.\n     * @param queueId       Queue ID to query.\n     * @param offset        Logical offset to start from.\n     * @param maxMsgNums    Maximum count of messages to query.\n     * @param messageFilter Message filter used to screen desired messages.\n     * @return Matched messages.\n     */\n    GetMessageResult getMessage(final String group, final String topic, final int queueId,\n        final long offset, final int maxMsgNums, final MessageFilter messageFilter);\n\n    /**\n     * Asynchronous get message\n     * @see #getMessage(String, String, int, long, int, MessageFilter) getMessage\n     *\n     * @param group         Consumer group that launches this query.\n     * @param topic         Topic to query.\n     * @param queueId       Queue ID to query.\n     * @param offset        Logical offset to start from.\n     * @param maxMsgNums    Maximum count of messages to query.\n     * @param messageFilter Message filter used to screen desired messages.\n     * @return Matched messages.\n     */\n    CompletableFuture<GetMessageResult> getMessageAsync(final String group, final String topic, final int queueId,\n        final long offset, final int maxMsgNums, final MessageFilter messageFilter);\n\n    /**\n     * Query at most <code>maxMsgNums</code> messages belonging to <code>topic</code> at <code>queueId</code> starting\n     * from given <code>offset</code>. Resulting messages will further be screened using provided message filter.\n     *\n     * @param group           Consumer group that launches this query.\n     * @param topic           Topic to query.\n     * @param queueId         Queue ID to query.\n     * @param offset          Logical offset to start from.\n     * @param maxMsgNums      Maximum count of messages to query.\n     * @param maxTotalMsgSize Maximum total msg size of the messages\n     * @param messageFilter   Message filter used to screen desired messages.\n     * @return Matched messages.\n     */\n    GetMessageResult getMessage(final String group, final String topic, final int queueId,\n        final long offset, final int maxMsgNums, final int maxTotalMsgSize, final MessageFilter messageFilter);\n\n    /**\n     * Asynchronous get message\n     * @see #getMessage(String, String, int, long, int, int, MessageFilter) getMessage\n     *\n     * @param group           Consumer group that launches this query.\n     * @param topic           Topic to query.\n     * @param queueId         Queue ID to query.\n     * @param offset          Logical offset to start from.\n     * @param maxMsgNums      Maximum count of messages to query.\n     * @param maxTotalMsgSize Maximum total msg size of the messages\n     * @param messageFilter   Message filter used to screen desired messages.\n     * @return Matched messages.\n     */\n    CompletableFuture<GetMessageResult> getMessageAsync(final String group, final String topic, final int queueId,\n        final long offset, final int maxMsgNums, final int maxTotalMsgSize, final MessageFilter messageFilter);\n\n    /**\n     * Get maximum offset of the topic queue.\n     *\n     * @param topic   Topic name.\n     * @param queueId Queue ID.\n     * @return Maximum offset at present.\n     */\n    long getMaxOffsetInQueue(final String topic, final int queueId) throws ConsumeQueueException;\n\n    /**\n     * Get maximum offset of the topic queue.\n     *\n     * @param topic     Topic name.\n     * @param queueId   Queue ID.\n     * @param committed return the max offset in ConsumeQueue if true, or the max offset in CommitLog if false\n     * @return Maximum offset at present.\n     */\n    long getMaxOffsetInQueue(final String topic, final int queueId, final boolean committed) throws ConsumeQueueException;\n\n    /**\n     * Get the minimum offset of the topic queue.\n     *\n     * @param topic   Topic name.\n     * @param queueId Queue ID.\n     * @return Minimum offset at present.\n     */\n    long getMinOffsetInQueue(final String topic, final int queueId);\n\n    TimerMessageStore getTimerMessageStore();\n\n    TimerMessageRocksDBStore getTimerMessageRocksDBStore();\n\n    TransMessageRocksDBStore getTransMessageRocksDBStore();\n\n    void setTimerMessageStore(TimerMessageStore timerMessageStore);\n\n    void setTimerMessageRocksDBStore(TimerMessageRocksDBStore timerMessageRocksDBStore);\n\n    void setTransMessageRocksDBStore(TransMessageRocksDBStore transMessageRocksDBStore);\n\n    /**\n     * Get the offset of the message in the commit log, which is also known as physical offset.\n     *\n     * @param topic              Topic of the message to lookup.\n     * @param queueId            Queue ID.\n     * @param consumeQueueOffset offset of consume queue.\n     * @return physical offset.\n     */\n    long getCommitLogOffsetInQueue(final String topic, final int queueId, final long consumeQueueOffset);\n\n    /**\n     * Look up the physical offset of the message whose store timestamp is as specified.\n     *\n     * @param topic     Topic of the message.\n     * @param queueId   Queue ID.\n     * @param timestamp Timestamp to look up.\n     * @return physical offset which matches.\n     */\n    long getOffsetInQueueByTime(final String topic, final int queueId, final long timestamp);\n\n    /**\n     * Look up the physical offset of the message whose store timestamp is as specified with specific boundaryType.\n     *\n     * @param topic        Topic of the message.\n     * @param queueId      Queue ID.\n     * @param timestamp    Timestamp to look up.\n     * @param boundaryType Lower or Upper\n     * @return physical offset which matches.\n     */\n    long getOffsetInQueueByTime(final String topic, final int queueId, final long timestamp, final BoundaryType boundaryType);\n\n    /**\n     * Look up the message by given commit log offset.\n     *\n     * @param commitLogOffset physical offset.\n     * @return Message whose physical offset is as specified.\n     */\n    MessageExt lookMessageByOffset(final long commitLogOffset);\n\n    /**\n     * Look up the message by given commit log offset and size.\n     *\n     * @param commitLogOffset physical offset.\n     * @param size            message size\n     * @return Message whose physical offset is as specified.\n     */\n    MessageExt lookMessageByOffset(long commitLogOffset, int size);\n\n    /**\n     * Get one message from the specified commit log offset.\n     *\n     * @param commitLogOffset commit log offset.\n     * @return wrapped result of the message.\n     */\n    SelectMappedBufferResult selectOneMessageByOffset(final long commitLogOffset);\n\n    /**\n     * Get one message from the specified commit log offset.\n     *\n     * @param commitLogOffset commit log offset.\n     * @param msgSize         message size.\n     * @return wrapped result of the message.\n     */\n    SelectMappedBufferResult selectOneMessageByOffset(final long commitLogOffset, final int msgSize);\n\n    /**\n     * Get the running information of this store.\n     *\n     * @return message store running info.\n     */\n    String getRunningDataInfo();\n\n    long getTimingMessageCount(String topic);\n\n    /**\n     * Message store runtime information, which should generally contains various statistical information.\n     *\n     * @return runtime information of the message store in format of key-value pairs.\n     */\n    HashMap<String, String> getRuntimeInfo();\n\n    /**\n     * HA runtime information\n     * @return runtime information of ha\n     */\n    HARuntimeInfo getHARuntimeInfo();\n\n    /**\n     * Get the maximum commit log offset.\n     *\n     * @return maximum commit log offset.\n     */\n    long getMaxPhyOffset();\n\n    /**\n     * Get the minimum commit log offset.\n     *\n     * @return minimum commit log offset.\n     */\n    long getMinPhyOffset();\n\n    /**\n     * Get the store time of the earliest message in the given queue.\n     *\n     * @param topic   Topic of the messages to query.\n     * @param queueId Queue ID to find.\n     * @return store time of the earliest message.\n     */\n    long getEarliestMessageTime(final String topic, final int queueId);\n\n    /**\n     * Get the store time of the earliest message in this store.\n     *\n     * @return timestamp of the earliest message in this store.\n     */\n    long getEarliestMessageTime();\n\n    /**\n     * Asynchronous get the store time of the earliest message in this store.\n     * @see #getEarliestMessageTime() getEarliestMessageTime\n     *\n     * @return timestamp of the earliest message in this store.\n     */\n    CompletableFuture<Long> getEarliestMessageTimeAsync(final String topic, final int queueId);\n\n    /**\n     * Get the store time of the message specified.\n     *\n     * @param topic              message topic.\n     * @param queueId            queue ID.\n     * @param consumeQueueOffset consume queue offset.\n     * @return store timestamp of the message.\n     */\n    long getMessageStoreTimeStamp(final String topic, final int queueId, final long consumeQueueOffset);\n\n    /**\n     * Asynchronous get the store time of the message specified.\n     * @see #getMessageStoreTimeStamp(String, int, long) getMessageStoreTimeStamp\n     *\n     * @param topic              message topic.\n     * @param queueId            queue ID.\n     * @param consumeQueueOffset consume queue offset.\n     * @return store timestamp of the message.\n     */\n    CompletableFuture<Long> getMessageStoreTimeStampAsync(final String topic, final int queueId,\n        final long consumeQueueOffset);\n\n    /**\n     * Get the total number of the messages in the specified queue.\n     *\n     * @param topic   Topic\n     * @param queueId Queue ID.\n     * @return total number.\n     */\n    long getMessageTotalInQueue(final String topic, final int queueId);\n\n    /**\n     * Get the raw commit log data starting from the given offset, which should used for replication purpose.\n     *\n     * @param offset starting offset.\n     * @return commit log data.\n     */\n    SelectMappedBufferResult getCommitLogData(final long offset);\n\n    /**\n     * Get the raw commit log data starting from the given offset, across multiple mapped files.\n     *\n     * @param offset starting offset.\n     * @param size   size of data to get\n     * @return commit log data.\n     */\n    List<SelectMappedBufferResult> getBulkCommitLogData(final long offset, final int size);\n\n    /**\n     * Append data to commit log.\n     *\n     * @param startOffset starting offset.\n     * @param data        data to append.\n     * @param dataStart   the start index of data array\n     * @param dataLength  the length of data array\n     * @return true if success; false otherwise.\n     */\n    boolean appendToCommitLog(final long startOffset, final byte[] data, int dataStart, int dataLength);\n\n    /**\n     * Execute file deletion manually.\n     */\n    void executeDeleteFilesManually();\n\n    /**\n     * Query messages by given key.\n     *\n     * @param topic  topic of the message.\n     * @param key    message key.\n     * @param maxNum maximum number of the messages possible.\n     * @param begin  begin timestamp.\n     * @param end    end timestamp.\n     */\n    QueryMessageResult queryMessage(final String topic, final String key, final int maxNum, final long begin,\n        final long end);\n\n    QueryMessageResult queryMessage(final String topic, final String key, final int maxNum, final long begin, final long end, final String indexType, final String lastKey);\n\n    /**\n     * Asynchronous query messages by given key.\n     * @see #queryMessage(String, String, int, long, long) queryMessage\n     *\n     * @param topic  topic of the message.\n     * @param key    message key.\n     * @param maxNum maximum number of the messages possible.\n     * @param begin  begin timestamp.\n     * @param end    end timestamp.\n     */\n    CompletableFuture<QueryMessageResult> queryMessageAsync(final String topic, final String key, final int maxNum,\n        final long begin, final long end);\n\n    CompletableFuture<QueryMessageResult> queryMessageAsync(final String topic, final String key, final int maxNum, final long begin, final long end, final String indexType, final String lastKey);\n\n    /**\n     * Update HA master address.\n     *\n     * @param newAddr new address.\n     */\n    void updateHaMasterAddress(final String newAddr);\n\n    /**\n     * Update master address.\n     *\n     * @param newAddr new address.\n     */\n    void updateMasterAddress(final String newAddr);\n\n    /**\n     * Return how much the slave falls behind.\n     *\n     * @return number of bytes that slave falls behind.\n     */\n    long slaveFallBehindMuch();\n\n    /**\n     * Return the current timestamp of the store.\n     *\n     * @return current time in milliseconds since 1970-01-01.\n     */\n    long now();\n\n    /**\n     * Delete topic's consume queue file and unused stats.\n     * This interface allows user delete system topic.\n     *\n     * @param deleteTopics unused topic name set\n     * @return the number of the topics which has been deleted.\n     */\n    int deleteTopics(final Set<String> deleteTopics);\n\n    /**\n     * Clean unused topics which not in retain topic name set.\n     *\n     * @param retainTopics all valid topics.\n     * @return number of the topics deleted.\n     */\n    int cleanUnusedTopic(final Set<String> retainTopics);\n\n    /**\n     * Clean expired consume queues.\n     */\n    void cleanExpiredConsumerQueue();\n\n    /**\n     * Check if the given message has been swapped out of the memory.\n     *\n     * @param topic         topic.\n     * @param queueId       queue ID.\n     * @param consumeOffset consume queue offset.\n     * @return true if the message is no longer in memory; false otherwise.\n     * @deprecated As of RIP-57, replaced by {@link #checkInMemByConsumeOffset(String, int, long, int)}, see <a href=\"https://github.com/apache/rocketmq/issues/5837\">this issue</a> for more details\n     */\n    @Deprecated\n    boolean checkInDiskByConsumeOffset(final String topic, final int queueId, long consumeOffset);\n\n    /**\n     * Check if the given message is in the page cache.\n     *\n     * @param topic         topic.\n     * @param queueId       queue ID.\n     * @param consumeOffset consume queue offset.\n     * @return true if the message is in page cache; false otherwise.\n     */\n    boolean checkInMemByConsumeOffset(final String topic, final int queueId, long consumeOffset, int batchSize);\n\n    /**\n     * Check if the given message is in store.\n     *\n     * @param topic         topic.\n     * @param queueId       queue ID.\n     * @param consumeOffset consume queue offset.\n     * @return true if the message is in store; false otherwise.\n     */\n    boolean checkInStoreByConsumeOffset(final String topic, final int queueId, long consumeOffset);\n\n    /**\n     * Get number of the bytes that have been stored in commit log and not yet dispatched to consume queue.\n     *\n     * @return number of the bytes to dispatch.\n     */\n    long dispatchBehindBytes();\n\n    /**\n     * Get number of the bytes that have been stored in commit log and not yet flushed to disk.\n     *\n     * @return number of the bytes to flush.\n     */\n    long flushBehindBytes();\n\n    /**\n     * Get number of the milliseconds that have been stored in commit log and not yet dispatched to consume queue.\n     *\n     * @return number of the milliseconds to dispatch.\n     */\n    long dispatchBehindMilliseconds();\n\n    /**\n     * Flush the message store to persist all data.\n     *\n     * @return maximum offset flushed to persistent storage device.\n     */\n    long flush();\n\n    /**\n     * Get the current flushed offset.\n     *\n     * @return flushed offset\n     */\n    long getFlushedWhere();\n\n    /**\n     * Get confirm offset.\n     *\n     * @return confirm offset.\n     */\n    long getConfirmOffset();\n\n    /**\n     * Set confirm offset.\n     *\n     * @param phyOffset confirm offset to set.\n     */\n    void setConfirmOffset(long phyOffset);\n\n    /**\n     * Check if the operating system page cache is busy or not.\n     *\n     * @return true if the OS page cache is busy; false otherwise.\n     */\n    boolean isOSPageCacheBusy();\n\n    /**\n     * Get lock time in milliseconds of the store by far.\n     *\n     * @return lock time in milliseconds.\n     */\n    long lockTimeMills();\n\n    /**\n     * Check if the transient store pool is deficient.\n     *\n     * @return true if the transient store pool is running out; false otherwise.\n     */\n    boolean isTransientStorePoolDeficient();\n\n    /**\n     * Get the dispatcher list.\n     *\n     * @return list of the dispatcher.\n     */\n    LinkedList<CommitLogDispatcher> getDispatcherList();\n\n    /**\n     * Add dispatcher.\n     *\n     * @param dispatcher commit log dispatcher to add\n     */\n    void addDispatcher(CommitLogDispatcher dispatcher);\n\n    /**\n     * Get consume queue of the topic/queue. If consume queue not exist, will return null\n     *\n     * @param topic   Topic.\n     * @param queueId Queue ID.\n     * @return Consume queue.\n     */\n    ConsumeQueueInterface getConsumeQueue(String topic, int queueId);\n\n    /**\n     * Get consume queue of the topic/queue. If consume queue not exist, will create one then return it.\n     * @param topic   Topic.\n     * @param queueId Queue ID.\n     * @return Consume queue.\n     */\n    ConsumeQueueInterface findConsumeQueue(String topic, int queueId);\n\n    /**\n     * Get BrokerStatsManager of the messageStore.\n     *\n     * @return BrokerStatsManager.\n     */\n    BrokerStatsManager getBrokerStatsManager();\n\n    /**\n     * Will be triggered when a new message is appended to commit log.\n     *\n     * @param msg           the msg that is appended to commit log\n     * @param result        append message result\n     * @param commitLogFile commit log file\n     */\n    void onCommitLogAppend(MessageExtBrokerInner msg, AppendMessageResult result, MappedFile commitLogFile);\n\n    /**\n     * Will be triggered when a new dispatch request is sent to message store.\n     *\n     * @param dispatchRequest dispatch request\n     * @param doDispatch      do dispatch if true\n     * @param commitLogFile   commit log file\n     * @param isRecover       is from recover process\n     * @param isFileEnd       if the dispatch request represents 'file end'\n     * @throws RocksDBException      only in rocksdb mode\n     */\n    void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile,\n        boolean isRecover, boolean isFileEnd) throws RocksDBException;\n\n    /**\n     * Get the message store config\n     *\n     * @return the message store config\n     */\n    MessageStoreConfig getMessageStoreConfig();\n\n    /**\n     * Get the statistics service\n     *\n     * @return the statistics service\n     */\n    StoreStatsService getStoreStatsService();\n\n    /**\n     * Get the store checkpoint component\n     *\n     * @return the checkpoint component\n     */\n    StoreCheckpoint getStoreCheckpoint();\n\n    /**\n     * Get the system clock\n     *\n     * @return the system clock\n     */\n    SystemClock getSystemClock();\n\n    /**\n     * Get the commit log\n     *\n     * @return the commit log\n     */\n    CommitLog getCommitLog();\n\n    /**\n     * Get running flags\n     *\n     * @return running flags\n     */\n    RunningFlags getRunningFlags();\n\n    /**\n     * Get the transient store pool\n     *\n     * @return the transient store pool\n     */\n    TransientStorePool getTransientStorePool();\n\n    /**\n     * Get the HA service\n     *\n     * @return the HA service\n     */\n    HAService getHaService();\n\n    /**\n     * Get the allocate-mappedFile service\n     *\n     * @return the allocate-mappedFile service\n     */\n    AllocateMappedFileService getAllocateMappedFileService();\n\n    /**\n     * Truncate dirty logic files\n     *\n     * @param phyOffset physical offset\n     * @throws RocksDBException only in rocksdb mode\n     */\n    void truncateDirtyLogicFiles(long phyOffset) throws RocksDBException;\n\n    /**\n     * Unlock mappedFile\n     *\n     * @param unlockMappedFile the file that needs to be unlocked\n     */\n    void unlockMappedFile(MappedFile unlockMappedFile);\n\n    /**\n     * Get the perf counter component\n     *\n     * @return the perf counter component\n     */\n    PerfCounter.Ticks getPerfCounter();\n\n    /**\n     * Get the queue store\n     *\n     * @return the queue store\n     */\n    @Nonnull\n    ConsumeQueueStoreInterface getQueueStore();\n\n    /**\n     * If 'sync disk flush' is configured in this message store\n     *\n     * @return yes if true, no if false\n     */\n    boolean isSyncDiskFlush();\n\n    /**\n     * If this message store is sync master role\n     *\n     * @return yes if true, no if false\n     */\n    boolean isSyncMaster();\n\n    /**\n     * Assign a message to queue offset. If there is a race condition, you need to lock/unlock this method\n     * yourself.\n     *\n     * @param msg        message\n     * @throws RocksDBException\n     */\n    void assignOffset(MessageExtBrokerInner msg) throws RocksDBException;\n\n    /**\n     * Increase queue offset in memory table. If there is a race condition, you need to lock/unlock this method\n     *\n     * @param msg        message\n     * @param messageNum message num\n     */\n    void increaseOffset(MessageExtBrokerInner msg, short messageNum);\n\n    /**\n     * Get master broker message store in process in broker container\n     *\n     * @return\n     */\n    MessageStore getMasterStoreInProcess();\n\n    /**\n     * Set master broker message store in process\n     *\n     * @param masterStoreInProcess\n     */\n    void setMasterStoreInProcess(MessageStore masterStoreInProcess);\n\n    /**\n     * Use FileChannel to get data\n     *\n     * @param offset\n     * @param size\n     * @param byteBuffer\n     * @return\n     */\n    boolean getData(long offset, int size, ByteBuffer byteBuffer);\n\n    /**\n     * Set the number of alive replicas in group.\n     *\n     * @param aliveReplicaNums number of alive replicas\n     */\n    void setAliveReplicaNumInGroup(int aliveReplicaNums);\n\n    /**\n     * Get the number of alive replicas in group.\n     *\n     * @return number of alive replicas\n     */\n    int getAliveReplicaNumInGroup();\n\n    /**\n     * Wake up AutoRecoverHAClient to start HA connection.\n     */\n    void wakeupHAClient();\n\n    /**\n     * Get master flushed offset.\n     *\n     * @return master flushed offset\n     */\n    long getMasterFlushedOffset();\n\n    /**\n     * Get broker init max offset.\n     *\n     * @return broker max offset in startup\n     */\n    long getBrokerInitMaxOffset();\n\n    /**\n     * Set master flushed offset.\n     *\n     * @param masterFlushedOffset master flushed offset\n     */\n    void setMasterFlushedOffset(long masterFlushedOffset);\n\n    /**\n     * Set broker init max offset.\n     *\n     * @param brokerInitMaxOffset broker init max offset\n     */\n    void setBrokerInitMaxOffset(long brokerInitMaxOffset);\n\n    /**\n     * Calculate the checksum of a certain range of data.\n     *\n     * @param from begin offset\n     * @param to   end offset\n     * @return checksum\n     */\n    byte[] calcDeltaChecksum(long from, long to);\n\n    /**\n     * Truncate commitLog and consume queue to certain offset.\n     *\n     * @param offsetToTruncate offset to truncate\n     * @return true if truncate succeed, false otherwise\n     * @throws RocksDBException only in rocksdb mode\n     */\n    boolean truncateFiles(long offsetToTruncate) throws RocksDBException;\n\n    /**\n     * Check if the offset is aligned with one message.\n     *\n     * @param offset offset to check\n     * @return true if aligned, false otherwise\n     */\n    boolean isOffsetAligned(long offset);\n\n    /**\n     * Get put message hook list\n     *\n     * @return List of PutMessageHook\n     */\n    List<PutMessageHook> getPutMessageHookList();\n\n    /**\n     * Set send message back hook\n     *\n     * @param sendMessageBackHook\n     */\n    void setSendMessageBackHook(SendMessageBackHook sendMessageBackHook);\n\n    /**\n     * Get send message back hook\n     *\n     * @return SendMessageBackHook\n     */\n    SendMessageBackHook getSendMessageBackHook();\n\n    //The following interfaces are used for duplication mode\n\n    /**\n     * Get last mapped file and return lase file first Offset\n     *\n     * @return lastMappedFile first Offset\n     */\n    long getLastFileFromOffset();\n\n    /**\n     * Get last mapped file\n     *\n     * @param startOffset\n     * @return true when get the last mapped file, false when get null\n     */\n    boolean getLastMappedFile(long startOffset);\n\n    /**\n     * Set physical offset\n     *\n     * @param phyOffset\n     */\n    void setPhysicalOffset(long phyOffset);\n\n    /**\n     * Return whether mapped file is empty\n     *\n     * @return whether mapped file is empty\n     */\n    boolean isMappedFilesEmpty();\n\n    /**\n     * Get state machine version\n     *\n     * @return state machine version\n     */\n    long getStateMachineVersion();\n\n    /**\n     * Get store metrics manager\n     *\n     * @return store metrics manager\n     */\n    StoreMetricsManager getStoreMetricsManager();\n\n    /**\n     * Check message and return size\n     *\n     * @param byteBuffer\n     * @param checkCRC\n     * @param checkDupInfo\n     * @param readBody\n     * @return DispatchRequest\n     */\n    DispatchRequest checkMessageAndReturnSize(final ByteBuffer byteBuffer, final boolean checkCRC,\n        final boolean checkDupInfo, final boolean readBody);\n\n    /**\n     * Get remain transientStoreBuffer numbers\n     *\n     * @return remain transientStoreBuffer numbers\n     */\n    int remainTransientStoreBufferNumbs();\n\n    /**\n     * Get remain how many data to commit\n     *\n     * @return remain how many data to commit\n     */\n    long remainHowManyDataToCommit();\n\n    /**\n     * Get remain how many data to flush\n     *\n     * @return remain how many data to flush\n     */\n    long remainHowManyDataToFlush();\n\n    /**\n     * Get whether message store is shutdown\n     *\n     * @return whether shutdown\n     */\n    boolean isShutdown();\n\n    /**\n     * Estimate number of messages, within [from, to], which match given filter\n     *\n     * @param topic   Topic name\n     * @param queueId Queue ID\n     * @param from    Lower boundary of the range, inclusive.\n     * @param to      Upper boundary of the range, inclusive.\n     * @param filter  The message filter.\n     * @return Estimate number of messages matching given filter.\n     */\n    long estimateMessageCount(String topic, int queueId, long from, long to, MessageFilter filter);\n\n    /**\n     * Get metrics view of store\n     *\n     * @return List of metrics selector and view pair\n     */\n    List<Pair<InstrumentSelector, ViewBuilder>> getMetricsView();\n\n    /**\n     * Init store metrics\n     *\n     * @param meter                     opentelemetry meter\n     * @param attributesBuilderSupplier metrics attributes builder\n     */\n    void initMetrics(Meter meter, Supplier<AttributesBuilder> attributesBuilderSupplier);\n\n    /**\n     * Recover topic queue table\n     */\n    void recoverTopicQueueTable();\n\n    /**\n     * notify message arrive if necessary\n     */\n    void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest);\n\n    MessageStoreStateMachine getStateMachine();\n\n    MessageRocksDBStorage getMessageRocksDBStorage();\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/MessageStoreStateMachine.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class MessageStoreStateMachine {\n    protected final Logger log;\n\n    private MessageStoreState currentState;\n    private long lastStateChangeTimestamp;\n    private final long startTimestamp;\n\n    public enum MessageStoreState {\n        INIT(0),\n\n        LOAD_BEGIN(10),\n        LOAD_COMMITLOG_OK(11),\n        LOAD_CONSUME_QUEUE_OK(12),\n        LOAD_COMPACTION_OK(13),\n        LOAD_INDEX_OK(14),\n\n        RECOVER_BEGIN(20),\n        RECOVER_CONSUME_QUEUE_OK(21),\n        RECOVER_COMMITLOG_OK(22),\n        RECOVER_TOPIC_QUEUE_TABLE_OK(23),\n\n        RUNNING(30),\n\n        SHUTDOWN_BEGIN(40),\n        SHUTDOWN_OK(41);\n\n        final int order;\n\n        MessageStoreState(int order) {\n            this.order = order;\n        }\n\n        public int getOrder() {\n            return order;\n        }\n\n        public boolean isBefore(MessageStoreState storeState) {\n            return this.order < storeState.order;\n        }\n\n        public boolean isAfter(MessageStoreState storeState) {\n            return this.order > storeState.order;\n        }\n    }\n\n\n    public MessageStoreStateMachine(Logger log) {\n        this.log = log == null ? LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME) : log;\n        this.currentState = MessageStoreState.INIT;\n        this.startTimestamp = System.currentTimeMillis();\n        this.lastStateChangeTimestamp = startTimestamp;\n        logStateChange(null, currentState, true);\n    }\n\n    public void transitTo(MessageStoreState newState) {\n        transitTo(newState, true);\n    }\n\n    public void transitTo(MessageStoreState newState, boolean success) {\n        if (!newState.isAfter(currentState)) {\n            throw new IllegalStateException(\n                String.format(\"Invalid state transition from %s to %s. Can only move forward.\",\n                    currentState, newState)\n            );\n        }\n\n        logStateChange(currentState, newState, success);\n        if (success) {\n            this.currentState = newState;\n            this.lastStateChangeTimestamp = System.currentTimeMillis();\n        }\n    }\n\n    private void logStateChange(MessageStoreState fromState, MessageStoreState toState, boolean success) {\n        if (fromState == null && success) {\n            log.info(\"MessageStoreState initialized, state={}\", toState);\n        } else if (success) {\n            log.info(\"MessageStoreState transition from {} to {}; Time in previous state={}ms, Total time={}ms\",\n                fromState, toState, getCurrentStateRunningTimeMs(), getTotalRunningTimeMs());\n        } else {\n            log.warn(\"MessageStoreState transition from {} to {} failed; Time in previous state={}ms, Total \"\n                + \"time={}ms\", fromState, toState, getCurrentStateRunningTimeMs(), getTotalRunningTimeMs());\n        }\n    }\n\n    public MessageStoreState getCurrentState() {\n        return currentState;\n    }\n\n    public long getTotalRunningTimeMs() {\n        return System.currentTimeMillis() - startTimestamp;\n    }\n\n    public long getCurrentStateRunningTimeMs() {\n        return System.currentTimeMillis() - lastStateChangeTimestamp;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport java.util.Arrays;\nimport java.util.HashSet;\nimport java.util.Set;\nimport java.util.function.Supplier;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.logfile.MappedFile;\n\nimport java.io.File;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\n\npublic class MultiPathMappedFileQueue extends MappedFileQueue {\n\n    private final MessageStoreConfig config;\n    private final Supplier<Set<String>> fullStorePathsSupplier;\n\n    public MultiPathMappedFileQueue(MessageStoreConfig messageStoreConfig, int mappedFileSize,\n        AllocateMappedFileService allocateMappedFileService,\n        Supplier<Set<String>> fullStorePathsSupplier) {\n        this(messageStoreConfig, mappedFileSize, allocateMappedFileService, fullStorePathsSupplier, null);\n    }\n    public MultiPathMappedFileQueue(MessageStoreConfig messageStoreConfig, int mappedFileSize,\n                                    AllocateMappedFileService allocateMappedFileService,\n                                    Supplier<Set<String>> fullStorePathsSupplier, RunningFlags runningFlags) {\n        super(messageStoreConfig.getStorePathCommitLog(), mappedFileSize, allocateMappedFileService, runningFlags,\n              messageStoreConfig.isWriteWithoutMmap());\n        this.config = messageStoreConfig;\n        this.fullStorePathsSupplier = fullStorePathsSupplier;\n    }\n\n    private Set<String> getPaths() {\n        String[] paths = config.getStorePathCommitLog().trim().split(MixAll.MULTI_PATH_SPLITTER);\n        return new HashSet<>(Arrays.asList(paths));\n    }\n\n    private Set<String> getReadonlyPaths() {\n        String pathStr = config.getReadOnlyCommitLogStorePaths();\n        if (StringUtils.isBlank(pathStr)) {\n            return Collections.emptySet();\n        }\n        String[] paths = pathStr.trim().split(MixAll.MULTI_PATH_SPLITTER);\n        return new HashSet<>(Arrays.asList(paths));\n    }\n\n    @Override\n    public boolean load() {\n        Set<String> storePathSet = getPaths();\n        storePathSet.addAll(getReadonlyPaths());\n\n        List<File> files = new ArrayList<>();\n        for (String path : storePathSet) {\n            File dir = new File(path);\n            File[] ls = dir.listFiles();\n            if (ls != null) {\n                Collections.addAll(files, ls);\n            }\n        }\n\n        return doLoad(files);\n    }\n\n    @Override\n    public MappedFile tryCreateMappedFile(long createOffset) {\n        long fileIdx = createOffset / this.mappedFileSize;\n        Set<String> storePath = getPaths();\n        Set<String> readonlyPathSet = getReadonlyPaths();\n        Set<String> fullStorePaths =\n                fullStorePathsSupplier == null ? Collections.emptySet() : fullStorePathsSupplier.get();\n\n\n        HashSet<String> availableStorePath = new HashSet<>(storePath);\n        //do not create file in readonly store path.\n        availableStorePath.removeAll(readonlyPathSet);\n\n        //do not create file is space is nearly full.\n        availableStorePath.removeAll(fullStorePaths);\n\n        //if no store path left, fall back to writable store path.\n        if (availableStorePath.isEmpty()) {\n            availableStorePath = new HashSet<>(storePath);\n            availableStorePath.removeAll(readonlyPathSet);\n        }\n\n        String[] paths = availableStorePath.toArray(new String[]{});\n        Arrays.sort(paths);\n        String nextFilePath = paths[(int) (fileIdx % paths.length)] + File.separator\n                + UtilAll.offset2FileName(createOffset);\n        String nextNextFilePath = paths[(int) ((fileIdx + 1) % paths.length)] + File.separator\n                + UtilAll.offset2FileName(createOffset + this.mappedFileSize);\n        return doCreateMappedFile(nextFilePath, nextNextFilePath);\n    }\n\n    @Override\n    public void destroy() {\n        for (MappedFile mf : this.mappedFiles) {\n            mf.destroy(1000 * 3);\n        }\n        this.mappedFiles.clear();\n        this.setFlushedWhere(0);\n\n        Set<String> storePathSet = getPaths();\n        storePathSet.addAll(getReadonlyPaths());\n\n        for (String path : storePathSet) {\n            File file = new File(path);\n            if (file.isDirectory()) {\n                file.delete();\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/PutMessageContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\npublic class PutMessageContext {\n    private String topicQueueTableKey;\n    private long[] phyPos;\n    private int batchSize;\n\n    public PutMessageContext(String topicQueueTableKey) {\n        this.topicQueueTableKey = topicQueueTableKey;\n    }\n\n    public String getTopicQueueTableKey() {\n        return topicQueueTableKey;\n    }\n\n    public long[] getPhyPos() {\n        return phyPos;\n    }\n\n    public void setPhyPos(long[] phyPos) {\n        this.phyPos = phyPos;\n    }\n\n    public int getBatchSize() {\n        return batchSize;\n    }\n\n    public void setBatchSize(int batchSize) {\n        this.batchSize = batchSize;\n    }\n}"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/PutMessageLock.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\n/**\n * Used when trying to put message\n */\npublic interface PutMessageLock {\n    void lock();\n\n    void unlock();\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/PutMessageReentrantLock.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport java.util.concurrent.locks.ReentrantLock;\n\n/**\n * Exclusive lock implementation to put message\n */\npublic class PutMessageReentrantLock implements PutMessageLock {\n    private ReentrantLock putMessageNormalLock = new ReentrantLock(); // NonfairSync\n\n    @Override\n    public void lock() {\n        putMessageNormalLock.lock();\n    }\n\n    @Override\n    public void unlock() {\n        putMessageNormalLock.unlock();\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/PutMessageResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\npublic class PutMessageResult {\n    private PutMessageStatus putMessageStatus;\n    private AppendMessageResult appendMessageResult;\n    private boolean remotePut = false;\n\n    public PutMessageResult(PutMessageStatus putMessageStatus, AppendMessageResult appendMessageResult) {\n        this.putMessageStatus = putMessageStatus;\n        this.appendMessageResult = appendMessageResult;\n    }\n\n    public PutMessageResult(PutMessageStatus putMessageStatus, AppendMessageResult appendMessageResult,\n        boolean remotePut) {\n        this.putMessageStatus = putMessageStatus;\n        this.appendMessageResult = appendMessageResult;\n        this.remotePut = remotePut;\n    }\n\n    public boolean isOk() {\n        if (remotePut) {\n            return putMessageStatus == PutMessageStatus.PUT_OK || putMessageStatus == PutMessageStatus.FLUSH_DISK_TIMEOUT\n                || putMessageStatus == PutMessageStatus.FLUSH_SLAVE_TIMEOUT || putMessageStatus == PutMessageStatus.SLAVE_NOT_AVAILABLE;\n        } else {\n            return this.appendMessageResult != null && this.appendMessageResult.isOk();\n        }\n\n    }\n\n    public AppendMessageResult getAppendMessageResult() {\n        return appendMessageResult;\n    }\n\n    public void setAppendMessageResult(AppendMessageResult appendMessageResult) {\n        this.appendMessageResult = appendMessageResult;\n    }\n\n    public PutMessageStatus getPutMessageStatus() {\n        return putMessageStatus;\n    }\n\n    public void setPutMessageStatus(PutMessageStatus putMessageStatus) {\n        this.putMessageStatus = putMessageStatus;\n    }\n\n    public boolean isRemotePut() {\n        return remotePut;\n    }\n\n    public void setRemotePut(boolean remotePut) {\n        this.remotePut = remotePut;\n    }\n\n    @Override\n    public String toString() {\n        return \"PutMessageResult [putMessageStatus=\" + putMessageStatus + \", appendMessageResult=\"\n            + appendMessageResult + \", remotePut=\" + remotePut + \"]\";\n    }\n\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/PutMessageSpinLock.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport java.util.concurrent.atomic.AtomicBoolean;\n\n/**\n * Spin lock Implementation to put message, suggest using this with low race conditions\n */\npublic class PutMessageSpinLock implements PutMessageLock {\n    //true: Can lock, false : in lock.\n    private AtomicBoolean putMessageSpinLock = new AtomicBoolean(true);\n\n    @Override\n    public void lock() {\n        boolean flag;\n        do {\n            flag = this.putMessageSpinLock.compareAndSet(true, false);\n        }\n        while (!flag);\n    }\n\n    @Override\n    public void unlock() {\n        this.putMessageSpinLock.compareAndSet(false, true);\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/PutMessageStatus.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\npublic enum PutMessageStatus {\n    PUT_OK,\n    FLUSH_DISK_TIMEOUT,\n    FLUSH_SLAVE_TIMEOUT,\n    SLAVE_NOT_AVAILABLE,\n    SERVICE_NOT_AVAILABLE,\n    CREATE_MAPPED_FILE_FAILED,\n    MESSAGE_ILLEGAL,\n    PROPERTIES_SIZE_EXCEEDED,\n    OS_PAGE_CACHE_BUSY,\n    UNKNOWN_ERROR,\n    IN_SYNC_REPLICAS_NOT_ENOUGH,\n    PUT_TO_REMOTE_BROKER_FAIL,\n    LMQ_CONSUME_QUEUE_NUM_EXCEEDED,\n    WHEEL_TIMER_FLOW_CONTROL,\n    WHEEL_TIMER_MSG_ILLEGAL,\n    WHEEL_TIMER_NOT_ENABLE\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/QueryMessageResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.List;\n\npublic class QueryMessageResult {\n\n    private final List<SelectMappedBufferResult> messageMapedList =\n        new ArrayList<>(100);\n\n    private final List<ByteBuffer> messageBufferList = new ArrayList<>(100);\n    private long indexLastUpdateTimestamp;\n    private long indexLastUpdatePhyoffset;\n\n    private int bufferTotalSize = 0;\n\n    public void addMessage(final SelectMappedBufferResult mapedBuffer) {\n        this.messageMapedList.add(mapedBuffer);\n        this.messageBufferList.add(mapedBuffer.getByteBuffer());\n        this.bufferTotalSize += mapedBuffer.getSize();\n    }\n\n    public void release() {\n        for (SelectMappedBufferResult select : this.messageMapedList) {\n            select.release();\n        }\n    }\n\n    public long getIndexLastUpdateTimestamp() {\n        return indexLastUpdateTimestamp;\n    }\n\n    public void setIndexLastUpdateTimestamp(long indexLastUpdateTimestamp) {\n        this.indexLastUpdateTimestamp = indexLastUpdateTimestamp;\n    }\n\n    public long getIndexLastUpdatePhyoffset() {\n        return indexLastUpdatePhyoffset;\n    }\n\n    public void setIndexLastUpdatePhyoffset(long indexLastUpdatePhyoffset) {\n        this.indexLastUpdatePhyoffset = indexLastUpdatePhyoffset;\n    }\n\n    public List<ByteBuffer> getMessageBufferList() {\n        return messageBufferList;\n    }\n\n    public int getBufferTotalSize() {\n        return bufferTotalSize;\n    }\n\n    public List<SelectMappedBufferResult> getMessageMapedList() {\n        return messageMapedList;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ReferenceResource.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport java.util.concurrent.atomic.AtomicLong;\n\npublic abstract class ReferenceResource {\n    protected final AtomicLong refCount = new AtomicLong(1);\n    protected volatile boolean available = true;\n    protected volatile boolean cleanupOver = false;\n    private volatile long firstShutdownTimestamp = 0;\n\n    public synchronized boolean hold() {\n        if (this.isAvailable()) {\n            if (this.refCount.getAndIncrement() > 0) {\n                return true;\n            } else {\n                this.refCount.getAndDecrement();\n            }\n        }\n\n        return false;\n    }\n\n    public boolean isAvailable() {\n        return this.available;\n    }\n\n    public void shutdown(final long intervalForcibly) {\n        if (this.available) {\n            this.available = false;\n            this.firstShutdownTimestamp = System.currentTimeMillis();\n            this.release();\n        } else if (this.getRefCount() > 0) {\n            if ((System.currentTimeMillis() - this.firstShutdownTimestamp) >= intervalForcibly) {\n                this.refCount.set(-1000 - this.getRefCount());\n                this.release();\n            }\n        }\n    }\n\n    public void release() {\n        long value = this.refCount.decrementAndGet();\n        if (value > 0)\n            return;\n\n        synchronized (this) {\n\n            this.cleanupOver = this.cleanup(value);\n        }\n    }\n\n    public long getRefCount() {\n        return this.refCount.get();\n    }\n\n    public abstract boolean cleanup(final long currentRef);\n\n    public boolean isCleanupOver() {\n        return this.refCount.get() <= 0 && this.cleanupOver;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport java.io.IOException;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface;\nimport org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\n\npublic class RocksDBMessageStore extends DefaultMessageStore {\n\n    public RocksDBMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager,\n        final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig, final ConcurrentMap<String, TopicConfig> topicConfigTable) throws\n        IOException {\n        super(messageStoreConfig, brokerStatsManager, messageArrivingListener, brokerConfig, topicConfigTable);\n    }\n\n    @Override\n    public ConsumeQueueStoreInterface createConsumeQueueStore() {\n        return new RocksDBConsumeQueueStore(this);\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/RunningFlags.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\npublic class RunningFlags {\n\n    private static final int NOT_READABLE_BIT = 1;\n\n    private static final int NOT_WRITEABLE_BIT = 1 << 1;\n\n    private static final int WRITE_LOGICS_QUEUE_ERROR_BIT = 1 << 2;\n\n    private static final int WRITE_INDEX_FILE_ERROR_BIT = 1 << 3;\n\n    private static final int DISK_FULL_BIT = 1 << 4;\n\n    private static final int FENCED_BIT = 1 << 5;\n\n    private static final int LOGIC_DISK_FULL_BIT = 1 << 6;\n\n    private volatile int flagBits = 0;\n\n    public RunningFlags() {\n    }\n\n    public int getFlagBits() {\n        return flagBits;\n    }\n\n    public boolean getAndMakeReadable() {\n        boolean result = this.isReadable();\n        if (!result) {\n            this.flagBits &= ~NOT_READABLE_BIT;\n        }\n        return result;\n    }\n\n    public boolean isReadable() {\n        return (this.flagBits & NOT_READABLE_BIT) == 0;\n    }\n\n    public boolean isFenced() {\n        return (this.flagBits & FENCED_BIT) != 0;\n    }\n\n    public boolean getAndMakeNotReadable() {\n        boolean result = this.isReadable();\n        if (result) {\n            this.flagBits |= NOT_READABLE_BIT;\n        }\n        return result;\n    }\n\n    public void clearLogicsQueueError() {\n        this.flagBits &= ~WRITE_LOGICS_QUEUE_ERROR_BIT;\n    }\n\n    public boolean getAndMakeWriteable() {\n        boolean result = this.isWriteable();\n        if (!result) {\n            this.flagBits &= ~NOT_WRITEABLE_BIT;\n        }\n        return result;\n    }\n\n    public boolean isWriteable() {\n        if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | DISK_FULL_BIT | WRITE_INDEX_FILE_ERROR_BIT | FENCED_BIT | LOGIC_DISK_FULL_BIT)) == 0) {\n            return true;\n        }\n\n        return false;\n    }\n\n    public boolean isStoreWriteable() {\n        if ((this.flagBits & NOT_WRITEABLE_BIT) == 0) {\n            return true;\n        }\n\n        return false;\n    }\n\n\n    //for consume queue, just ignore the DISK_FULL_BIT\n    public boolean isCQWriteable() {\n        if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | WRITE_INDEX_FILE_ERROR_BIT | LOGIC_DISK_FULL_BIT)) == 0) {\n            return true;\n        }\n\n        return false;\n    }\n\n    public boolean getAndMakeStoreNotWriteable() {\n        boolean result = this.isWriteable();\n        if (result) {\n            this.flagBits |= NOT_WRITEABLE_BIT;\n        }\n        return result;\n    }\n\n    public void makeLogicsQueueError() {\n        this.flagBits |= WRITE_LOGICS_QUEUE_ERROR_BIT;\n    }\n\n    public void makeFenced(boolean fenced) {\n        if (fenced) {\n            this.flagBits |= FENCED_BIT;\n        } else {\n            this.flagBits &= ~FENCED_BIT;\n        }\n    }\n\n    public boolean isLogicsQueueError() {\n        if ((this.flagBits & WRITE_LOGICS_QUEUE_ERROR_BIT) == WRITE_LOGICS_QUEUE_ERROR_BIT) {\n            return true;\n        }\n\n        return false;\n    }\n\n    public void makeIndexFileError() {\n        this.flagBits |= WRITE_INDEX_FILE_ERROR_BIT;\n    }\n\n    public boolean isIndexFileError() {\n        if ((this.flagBits & WRITE_INDEX_FILE_ERROR_BIT) == WRITE_INDEX_FILE_ERROR_BIT) {\n            return true;\n        }\n\n        return false;\n    }\n\n    public boolean getAndMakeDiskFull() {\n        boolean result = !((this.flagBits & DISK_FULL_BIT) == DISK_FULL_BIT);\n        this.flagBits |= DISK_FULL_BIT;\n        return result;\n    }\n\n    public boolean getAndMakeDiskOK() {\n        boolean result = !((this.flagBits & DISK_FULL_BIT) == DISK_FULL_BIT);\n        this.flagBits &= ~DISK_FULL_BIT;\n        return result;\n    }\n\n    public boolean getAndMakeLogicDiskFull() {\n        boolean result = !((this.flagBits & LOGIC_DISK_FULL_BIT) == LOGIC_DISK_FULL_BIT);\n        this.flagBits |= LOGIC_DISK_FULL_BIT;\n        return result;\n    }\n\n    public boolean getAndMakeLogicDiskOK() {\n        boolean result = !((this.flagBits & LOGIC_DISK_FULL_BIT) == LOGIC_DISK_FULL_BIT);\n        this.flagBits &= ~LOGIC_DISK_FULL_BIT;\n        return result;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/SelectMappedBufferResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport java.nio.ByteBuffer;\nimport org.apache.rocketmq.store.logfile.MappedFile;\n\npublic class SelectMappedBufferResult {\n\n    private final long startOffset;\n\n    private final ByteBuffer byteBuffer;\n\n    private int size;\n\n    protected MappedFile mappedFile;\n\n    private boolean isInCache = true;\n\n    public SelectMappedBufferResult(long startOffset, ByteBuffer byteBuffer, int size, MappedFile mappedFile) {\n        this.startOffset = startOffset;\n        this.byteBuffer = byteBuffer;\n        this.size = size;\n        this.mappedFile = mappedFile;\n    }\n\n    public ByteBuffer getByteBuffer() {\n        return byteBuffer;\n    }\n\n    public int getSize() {\n        return size;\n    }\n\n    public void setSize(final int s) {\n        this.size = s;\n        this.byteBuffer.limit(this.size);\n    }\n\n    public MappedFile getMappedFile() {\n        return mappedFile;\n    }\n\n    public synchronized void release() {\n        if (this.mappedFile != null) {\n            this.mappedFile.release();\n            this.mappedFile = null;\n        }\n    }\n    public synchronized boolean hasReleased() {\n        return this.mappedFile == null;\n    }\n\n    public long getStartOffset() {\n        return startOffset;\n    }\n\n    public boolean isInMem() {\n        if (mappedFile == null) {\n            return true;\n        }\n        long pos = startOffset - mappedFile.getFileFromOffset();\n        return mappedFile.isLoaded(pos, size);\n    }\n\n    public boolean isInCache() {\n        return isInCache;\n    }\n\n    public void setInCache(boolean inCache) {\n        isInCache = inCache;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/SelectMappedFileResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport org.apache.rocketmq.store.logfile.MappedFile;\n\npublic class SelectMappedFileResult {\n\n    protected int size;\n\n    protected MappedFile mappedFile;\n\n    public SelectMappedFileResult(int size, MappedFile mappedFile) {\n        this.size = size;\n        this.mappedFile = mappedFile;\n    }\n\n    public int getSize() {\n        return size;\n    }\n\n    public void setSize(int size) {\n        this.size = size;\n    }\n\n    public MappedFile getMappedFile() {\n        return mappedFile;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/StoreCheckpoint.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.io.RandomAccessFile;\nimport java.nio.MappedByteBuffer;\nimport java.nio.channels.FileChannel;\nimport java.nio.channels.FileChannel.MapMode;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.logfile.DefaultMappedFile;\n\npublic class StoreCheckpoint {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private final RandomAccessFile randomAccessFile;\n    private final FileChannel fileChannel;\n    private final MappedByteBuffer mappedByteBuffer;\n    private volatile long tmpLogicsMsgTimestamp = 0;\n    private volatile long physicMsgTimestamp = 0;\n    private volatile long logicsMsgTimestamp = 0;\n    private volatile long tmpLogicsPhysicalOffset = 0;\n    private volatile long logicsPhysicalOffset = 0;\n    private volatile long indexMsgTimestamp = 0;\n    private volatile long masterFlushedOffset = 0;\n    private volatile long confirmPhyOffset = 0;\n\n    public StoreCheckpoint(final String scpPath) throws IOException {\n        File file = new File(scpPath);\n        UtilAll.ensureDirOK(file.getParent());\n        boolean fileExists = file.exists();\n\n        this.randomAccessFile = new RandomAccessFile(file, \"rw\");\n        this.fileChannel = this.randomAccessFile.getChannel();\n        this.mappedByteBuffer = fileChannel.map(MapMode.READ_WRITE, 0, DefaultMappedFile.OS_PAGE_SIZE);\n\n        if (fileExists) {\n            log.info(\"store checkpoint file exists, \" + scpPath);\n            this.physicMsgTimestamp = this.mappedByteBuffer.getLong(0);\n            this.logicsMsgTimestamp = this.mappedByteBuffer.getLong(8);\n            this.indexMsgTimestamp = this.mappedByteBuffer.getLong(16);\n            this.masterFlushedOffset = this.mappedByteBuffer.getLong(24);\n            this.confirmPhyOffset = this.mappedByteBuffer.getLong(32);\n            this.logicsPhysicalOffset = this.mappedByteBuffer.getLong(40);\n\n            log.info(\"store checkpoint file physicMsgTimestamp \" + this.physicMsgTimestamp + \", \"\n                + UtilAll.timeMillisToHumanString(this.physicMsgTimestamp));\n            log.info(\"store checkpoint file logicsMsgTimestamp \" + this.logicsMsgTimestamp + \", \"\n                + UtilAll.timeMillisToHumanString(this.logicsMsgTimestamp));\n            log.info(\"store checkpoint file indexMsgTimestamp \" + this.indexMsgTimestamp + \", \"\n                + UtilAll.timeMillisToHumanString(this.indexMsgTimestamp));\n            log.info(\"store checkpoint file masterFlushedOffset \" + this.masterFlushedOffset);\n            log.info(\"store checkpoint file confirmPhyOffset \" + this.confirmPhyOffset);\n            log.info(\"store checkpoint file logicsPhysicalOffset \" + this.logicsPhysicalOffset);\n        } else {\n            log.info(\"store checkpoint file not exists, \" + scpPath);\n        }\n    }\n\n    public void shutdown() {\n\n        this.flush();\n\n        // unmap mappedByteBuffer\n        UtilAll.cleanBuffer(this.mappedByteBuffer);\n\n        try {\n            this.fileChannel.close();\n        } catch (Throwable e) {\n            log.error(\"Failed to close file channel\", e);\n        }\n    }\n\n    public void flush() {\n        try {\n            this.mappedByteBuffer.putLong(0, this.physicMsgTimestamp);\n            this.mappedByteBuffer.putLong(8, this.logicsMsgTimestamp);\n            this.mappedByteBuffer.putLong(16, this.indexMsgTimestamp);\n            this.mappedByteBuffer.putLong(24, this.masterFlushedOffset);\n            this.mappedByteBuffer.putLong(32, this.confirmPhyOffset);\n            this.mappedByteBuffer.putLong(40, this.logicsPhysicalOffset);\n            this.mappedByteBuffer.force();\n        } catch (Throwable e) {\n            log.error(\"Failed to flush\", e);\n        }\n    }\n\n    public long getPhysicMsgTimestamp() {\n        return physicMsgTimestamp;\n    }\n\n    public void setPhysicMsgTimestamp(long physicMsgTimestamp) {\n        this.physicMsgTimestamp = physicMsgTimestamp;\n    }\n\n    public long getLogicsMsgTimestamp() {\n        return logicsMsgTimestamp;\n    }\n\n    public void setLogicsMsgTimestamp(long logicsMsgTimestamp) {\n        this.logicsMsgTimestamp = logicsMsgTimestamp;\n    }\n\n    public long getTmpLogicsMsgTimestamp() {\n        return tmpLogicsMsgTimestamp;\n    }\n\n    public void setTmpLogicsMsgTimestamp(long tmpLogicsMsgTimestamp) {\n        this.tmpLogicsMsgTimestamp = tmpLogicsMsgTimestamp;\n    }\n\n    public long getTmpLogicsPhysicalOffset() {\n        return tmpLogicsPhysicalOffset;\n    }\n\n    public void setTmpLogicsPhysicalOffset(long tmpLogicsPhysicalOffset) {\n        this.tmpLogicsPhysicalOffset = tmpLogicsPhysicalOffset;\n    }\n\n    public long getLogicsPhysicalOffset() {\n        return logicsPhysicalOffset;\n    }\n\n    public void setLogicsPhysicalOffset(long logicsPhysicalOffset) {\n        this.logicsPhysicalOffset = logicsPhysicalOffset;\n    }\n\n    public long getConfirmPhyOffset() {\n        return confirmPhyOffset;\n    }\n\n    public void setConfirmPhyOffset(long confirmPhyOffset) {\n        this.confirmPhyOffset = confirmPhyOffset;\n    }\n\n    public long getMinTimestampIndex() {\n        return Math.min(this.getMinTimestamp(), this.indexMsgTimestamp);\n    }\n\n    public long getMinTimestamp() {\n        long min = Math.min(this.physicMsgTimestamp, this.logicsMsgTimestamp);\n\n        min -= 1000 * 3;\n        if (min < 0) {\n            min = 0;\n        }\n\n        return min;\n    }\n\n    public long getIndexMsgTimestamp() {\n        return indexMsgTimestamp;\n    }\n\n    public void setIndexMsgTimestamp(long indexMsgTimestamp) {\n        this.indexMsgTimestamp = indexMsgTimestamp;\n    }\n\n    public long getMasterFlushedOffset() {\n        return masterFlushedOffset;\n    }\n\n    public void setMasterFlushedOffset(long masterFlushedOffset) {\n        this.masterFlushedOffset = masterFlushedOffset;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/StoreStatsService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport java.text.MessageFormat;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.TreeMap;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.concurrent.atomic.LongAdder;\nimport java.util.concurrent.locks.ReentrantLock;\nimport org.apache.rocketmq.common.BrokerIdentity;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class StoreStatsService extends ServiceThread {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n\n    private static final int FREQUENCY_OF_SAMPLING = 1000;\n\n    private static final int MAX_RECORDS_OF_SAMPLING = 60 * 10;\n    private static final String[] PUT_MESSAGE_ENTIRE_TIME_MAX_DESC = new String[] {\n        \"[<=0ms]\", \"[0~10ms]\", \"[10~50ms]\", \"[50~100ms]\", \"[100~200ms]\", \"[200~500ms]\", \"[500ms~1s]\", \"[1~2s]\", \"[2~3s]\", \"[3~4s]\", \"[4~5s]\", \"[5~10s]\", \"[10s~]\",\n    };\n\n    //The rule to define buckets\n    private static final Map<Integer/*interval step size*/, Integer/*times*/> PUT_MESSAGE_ENTIRE_TIME_BUCKETS = new TreeMap<>();\n    //buckets\n    private TreeMap<Long/*bucket*/, LongAdder/*times*/> buckets = new TreeMap<>();\n    private Map<Long/*bucket*/, LongAdder/*times*/> lastBuckets = new TreeMap<>();\n\n    private static int printTPSInterval = 60 * 1;\n\n    private final LongAdder putMessageFailedTimes = new LongAdder();\n\n    private final ConcurrentMap<String, LongAdder> putMessageTopicTimesTotal =\n        new ConcurrentHashMap<>(128);\n    private final ConcurrentMap<String, LongAdder> putMessageTopicSizeTotal =\n        new ConcurrentHashMap<>(128);\n\n    private final LongAdder getMessageTimesTotalFound = new LongAdder();\n    private final LongAdder getMessageTransferredMsgCount = new LongAdder();\n    private final LongAdder getMessageTimesTotalMiss = new LongAdder();\n    private final LinkedList<CallSnapshot> putTimesList = new LinkedList<>();\n\n    private final LinkedList<CallSnapshot> getTimesFoundList = new LinkedList<>();\n    private final LinkedList<CallSnapshot> getTimesMissList = new LinkedList<>();\n    private final LinkedList<CallSnapshot> transferredMsgCountList = new LinkedList<>();\n    private volatile LongAdder[] putMessageDistributeTime;\n    private volatile LongAdder[] lastPutMessageDistributeTime;\n    private long messageStoreBootTimestamp = System.currentTimeMillis();\n    private volatile long putMessageEntireTimeMax = 0;\n    private volatile long getMessageEntireTimeMax = 0;\n    // for putMessageEntireTimeMax\n    private ReentrantLock putLock = new ReentrantLock();\n    // for getMessageEntireTimeMax\n    private ReentrantLock getLock = new ReentrantLock();\n\n    private volatile long dispatchMaxBuffer = 0;\n\n    private ReentrantLock samplingLock = new ReentrantLock();\n    private long lastPrintTimestamp = System.currentTimeMillis();\n\n    private BrokerIdentity brokerIdentity;\n\n    public StoreStatsService(BrokerIdentity brokerIdentity) {\n        this();\n        this.brokerIdentity = brokerIdentity;\n    }\n\n    public StoreStatsService() {\n        PUT_MESSAGE_ENTIRE_TIME_BUCKETS.put(1,20);  //0-20\n        PUT_MESSAGE_ENTIRE_TIME_BUCKETS.put(2,15);  //20-50\n        PUT_MESSAGE_ENTIRE_TIME_BUCKETS.put(5,10);  //50-100\n        PUT_MESSAGE_ENTIRE_TIME_BUCKETS.put(10,10);  //100-200\n        PUT_MESSAGE_ENTIRE_TIME_BUCKETS.put(50,6);  //200-500\n        PUT_MESSAGE_ENTIRE_TIME_BUCKETS.put(100,5);  //500-1000\n        PUT_MESSAGE_ENTIRE_TIME_BUCKETS.put(1000,9);  //1s-10s\n\n        this.resetPutMessageTimeBuckets();\n        this.resetPutMessageDistributeTime();\n    }\n\n    private void resetPutMessageTimeBuckets() {\n        TreeMap<Long, LongAdder> nextBuckets = new TreeMap<>();\n        AtomicLong index = new AtomicLong(0);\n        PUT_MESSAGE_ENTIRE_TIME_BUCKETS.forEach((interval, times) -> {\n            for (int i = 0; i < times; i++) {\n                nextBuckets.put(index.addAndGet(interval), new LongAdder());\n            }\n        });\n        nextBuckets.put(Long.MAX_VALUE, new LongAdder());\n\n        this.lastBuckets = this.buckets;\n        this.buckets = nextBuckets;\n    }\n\n    public void incPutMessageEntireTime(long value) {\n        Map.Entry<Long, LongAdder> targetBucket = buckets.ceilingEntry(value);\n        if (targetBucket != null) {\n            targetBucket.getValue().add(1);\n        }\n    }\n\n    public double findPutMessageEntireTimePX(double px) {\n        Map<Long, LongAdder> lastBuckets = this.lastBuckets;\n        long start = System.currentTimeMillis();\n        double result = 0.0;\n        long totalRequest = lastBuckets.values().stream().mapToLong(LongAdder::longValue).sum();\n        long pxIndex = (long) (totalRequest * px);\n        long passCount = 0;\n        List<Long> bucketValue = new ArrayList<>(lastBuckets.keySet());\n        for (int i = 0; i < bucketValue.size(); i++) {\n            long count = lastBuckets.get(bucketValue.get(i)).longValue();\n            if (pxIndex <= passCount + count) {\n                long relativeIndex = pxIndex - passCount;\n                if (i == 0) {\n                    result = count == 0 ? 0 : bucketValue.get(i) * relativeIndex / (double)count;\n                } else {\n                    long lastBucket = bucketValue.get(i - 1);\n                    result = lastBucket + (count == 0 ? 0 : (bucketValue.get(i) - lastBucket) * relativeIndex / (double)count);\n                }\n                break;\n            } else {\n                passCount += count;\n            }\n        }\n        log.info(\"findPutMessageEntireTimePX {}={}ms cost {}ms\", px, String.format(\"%.2f\", result), System.currentTimeMillis() - start);\n        return result;\n    }\n\n    private LongAdder[] resetPutMessageDistributeTime() {\n        LongAdder[] next = new LongAdder[13];\n        for (int i = 0; i < next.length; i++) {\n            next[i] = new LongAdder();\n        }\n\n        this.lastPutMessageDistributeTime = this.putMessageDistributeTime;\n\n        this.putMessageDistributeTime = next;\n\n        return lastPutMessageDistributeTime;\n    }\n\n    public long getPutMessageEntireTimeMax() {\n        return putMessageEntireTimeMax;\n    }\n\n    public void setPutMessageEntireTimeMax(long value) {\n        this.incPutMessageEntireTime(value);\n        final LongAdder[] times = this.putMessageDistributeTime;\n\n        if (null == times)\n            return;\n\n        // us\n        if (value <= 0) {\n            times[0].add(1);\n        } else if (value < 10) {\n            times[1].add(1);\n        } else if (value < 50) {\n            times[2].add(1);\n        } else if (value < 100) {\n            times[3].add(1);\n        } else if (value < 200) {\n            times[4].add(1);\n        } else if (value < 500) {\n            times[5].add(1);\n        } else if (value < 1000) {\n            times[6].add(1);\n        }\n        // 2s\n        else if (value < 2000) {\n            times[7].add(1);\n        }\n        // 3s\n        else if (value < 3000) {\n            times[8].add(1);\n        }\n        // 4s\n        else if (value < 4000) {\n            times[9].add(1);\n        }\n        // 5s\n        else if (value < 5000) {\n            times[10].add(1);\n        }\n        // 10s\n        else if (value < 10000) {\n            times[11].add(1);\n        } else {\n            times[12].add(1);\n        }\n\n        if (value > this.putMessageEntireTimeMax) {\n            this.putLock.lock();\n            this.putMessageEntireTimeMax =\n                value > this.putMessageEntireTimeMax ? value : this.putMessageEntireTimeMax;\n            this.putLock.unlock();\n        }\n    }\n\n    public long getGetMessageEntireTimeMax() {\n        return getMessageEntireTimeMax;\n    }\n\n    public void setGetMessageEntireTimeMax(long value) {\n        if (value > this.getMessageEntireTimeMax) {\n            this.getLock.lock();\n            this.getMessageEntireTimeMax =\n                value > this.getMessageEntireTimeMax ? value : this.getMessageEntireTimeMax;\n            this.getLock.unlock();\n        }\n    }\n\n    public long getDispatchMaxBuffer() {\n        return dispatchMaxBuffer;\n    }\n\n    public void setDispatchMaxBuffer(long value) {\n        this.dispatchMaxBuffer = value > this.dispatchMaxBuffer ? value : this.dispatchMaxBuffer;\n    }\n\n    @Override\n    public String toString() {\n        final StringBuilder sb = new StringBuilder(1024);\n        Long totalTimes = getPutMessageTimesTotal();\n        if (0 == totalTimes) {\n            totalTimes = 1L;\n        }\n\n        sb.append(\"\\truntime: \" + this.getFormatRuntime() + \"\\r\\n\");\n        sb.append(\"\\tputMessageEntireTimeMax: \" + this.putMessageEntireTimeMax + \"\\r\\n\");\n        sb.append(\"\\tputMessageTimesTotal: \" + totalTimes + \"\\r\\n\");\n        sb.append(\"\\tgetPutMessageFailedTimes: \" + this.getPutMessageFailedTimes() + \"\\r\\n\");\n        sb.append(\"\\tputMessageSizeTotal: \" + this.getPutMessageSizeTotal() + \"\\r\\n\");\n        sb.append(\"\\tputMessageDistributeTime: \" + this.getPutMessageDistributeTimeStringInfo(totalTimes)\n            + \"\\r\\n\");\n        sb.append(\"\\tputMessageAverageSize: \" + (this.getPutMessageSizeTotal() / totalTimes.doubleValue())\n            + \"\\r\\n\");\n        sb.append(\"\\tdispatchMaxBuffer: \" + this.dispatchMaxBuffer + \"\\r\\n\");\n        sb.append(\"\\tgetMessageEntireTimeMax: \" + this.getMessageEntireTimeMax + \"\\r\\n\");\n        sb.append(\"\\tputTps: \" + this.getPutTps() + \"\\r\\n\");\n        sb.append(\"\\tgetFoundTps: \" + this.getGetFoundTps() + \"\\r\\n\");\n        sb.append(\"\\tgetMissTps: \" + this.getGetMissTps() + \"\\r\\n\");\n        sb.append(\"\\tgetTotalTps: \" + this.getGetTotalTps() + \"\\r\\n\");\n        sb.append(\"\\tgetTransferredTps: \" + this.getGetTransferredTps() + \"\\r\\n\");\n        return sb.toString();\n    }\n\n    public long getPutMessageTimesTotal() {\n        Map<String, LongAdder> map = putMessageTopicTimesTotal;\n        return map.values()\n                .parallelStream()\n                .mapToLong(LongAdder::longValue)\n                .sum();\n    }\n\n    private String getFormatRuntime() {\n        final long millisecond = 1;\n        final long second = 1000 * millisecond;\n        final long minute = 60 * second;\n        final long hour = 60 * minute;\n        final long day = 24 * hour;\n        final MessageFormat messageFormat = new MessageFormat(\"[ {0} days, {1} hours, {2} minutes, {3} seconds ]\");\n\n        long time = System.currentTimeMillis() - this.messageStoreBootTimestamp;\n        long days = time / day;\n        long hours = (time % day) / hour;\n        long minutes = (time % hour) / minute;\n        long seconds = (time % minute) / second;\n        return messageFormat.format(new Long[] {days, hours, minutes, seconds});\n    }\n\n    public long getPutMessageSizeTotal() {\n        Map<String, LongAdder> map = putMessageTopicSizeTotal;\n        return map.values()\n                .parallelStream()\n                .mapToLong(LongAdder::longValue)\n                .sum();\n    }\n\n    private String getPutMessageDistributeTimeStringInfo(Long total) {\n        return this.putMessageDistributeTimeToString();\n    }\n\n    private String getPutTps() {\n        StringBuilder sb = new StringBuilder();\n\n        sb.append(this.getPutTps(10));\n        sb.append(\" \");\n\n        sb.append(this.getPutTps(60));\n        sb.append(\" \");\n\n        sb.append(this.getPutTps(600));\n\n        return sb.toString();\n    }\n\n    private String getGetFoundTps() {\n        StringBuilder sb = new StringBuilder();\n\n        sb.append(this.getGetFoundTps(10));\n        sb.append(\" \");\n\n        sb.append(this.getGetFoundTps(60));\n        sb.append(\" \");\n\n        sb.append(this.getGetFoundTps(600));\n\n        return sb.toString();\n    }\n\n    private String getGetMissTps() {\n        StringBuilder sb = new StringBuilder();\n\n        sb.append(this.getGetMissTps(10));\n        sb.append(\" \");\n\n        sb.append(this.getGetMissTps(60));\n        sb.append(\" \");\n\n        sb.append(this.getGetMissTps(600));\n\n        return sb.toString();\n    }\n\n    private String getGetTotalTps() {\n        StringBuilder sb = new StringBuilder();\n\n        sb.append(this.getGetTotalTps(10));\n        sb.append(\" \");\n\n        sb.append(this.getGetTotalTps(60));\n        sb.append(\" \");\n\n        sb.append(this.getGetTotalTps(600));\n\n        return sb.toString();\n    }\n\n    private String getGetTransferredTps() {\n        StringBuilder sb = new StringBuilder();\n\n        sb.append(this.getGetTransferredTps(10));\n        sb.append(\" \");\n\n        sb.append(this.getGetTransferredTps(60));\n        sb.append(\" \");\n\n        sb.append(this.getGetTransferredTps(600));\n\n        return sb.toString();\n    }\n\n    private String putMessageDistributeTimeToString() {\n        final LongAdder[] times = this.lastPutMessageDistributeTime;\n        if (null == times)\n            return null;\n\n        final StringBuilder sb = new StringBuilder();\n        for (int i = 0; i < times.length; i++) {\n            long value = times[i].longValue();\n            sb.append(String.format(\"%s:%d\", PUT_MESSAGE_ENTIRE_TIME_MAX_DESC[i], value));\n            sb.append(\" \");\n        }\n\n        return sb.toString();\n    }\n\n    private String getPutTps(int time) {\n        String result = \"\";\n        this.samplingLock.lock();\n        try {\n            CallSnapshot last = this.putTimesList.getLast();\n\n            if (this.putTimesList.size() > time) {\n                CallSnapshot lastBefore = this.putTimesList.get(this.putTimesList.size() - (time + 1));\n                result += CallSnapshot.getTPS(lastBefore, last);\n            }\n\n        } finally {\n            this.samplingLock.unlock();\n        }\n        return result;\n    }\n\n    private String getGetFoundTps(int time) {\n        String result = \"\";\n        this.samplingLock.lock();\n        try {\n            CallSnapshot last = this.getTimesFoundList.getLast();\n\n            if (this.getTimesFoundList.size() > time) {\n                CallSnapshot lastBefore =\n                    this.getTimesFoundList.get(this.getTimesFoundList.size() - (time + 1));\n                result += CallSnapshot.getTPS(lastBefore, last);\n            }\n        } finally {\n            this.samplingLock.unlock();\n        }\n\n        return result;\n    }\n\n    private String getGetMissTps(int time) {\n        String result = \"\";\n        this.samplingLock.lock();\n        try {\n            CallSnapshot last = this.getTimesMissList.getLast();\n\n            if (this.getTimesMissList.size() > time) {\n                CallSnapshot lastBefore =\n                    this.getTimesMissList.get(this.getTimesMissList.size() - (time + 1));\n                result += CallSnapshot.getTPS(lastBefore, last);\n            }\n\n        } finally {\n            this.samplingLock.unlock();\n        }\n\n        return result;\n    }\n\n    private String getGetTotalTps(int time) {\n        this.samplingLock.lock();\n        double found = 0;\n        double miss = 0;\n        try {\n            {\n                CallSnapshot last = this.getTimesFoundList.getLast();\n\n                if (this.getTimesFoundList.size() > time) {\n                    CallSnapshot lastBefore =\n                        this.getTimesFoundList.get(this.getTimesFoundList.size() - (time + 1));\n                    found = CallSnapshot.getTPS(lastBefore, last);\n                }\n            }\n            {\n                CallSnapshot last = this.getTimesMissList.getLast();\n\n                if (this.getTimesMissList.size() > time) {\n                    CallSnapshot lastBefore =\n                        this.getTimesMissList.get(this.getTimesMissList.size() - (time + 1));\n                    miss = CallSnapshot.getTPS(lastBefore, last);\n                }\n            }\n\n        } finally {\n            this.samplingLock.unlock();\n        }\n\n        return Double.toString(found + miss);\n    }\n\n    private String getGetTransferredTps(int time) {\n        String result = \"\";\n        this.samplingLock.lock();\n        try {\n            CallSnapshot last = this.transferredMsgCountList.getLast();\n\n            if (this.transferredMsgCountList.size() > time) {\n                CallSnapshot lastBefore =\n                    this.transferredMsgCountList.get(this.transferredMsgCountList.size() - (time + 1));\n                result += CallSnapshot.getTPS(lastBefore, last);\n            }\n\n        } finally {\n            this.samplingLock.unlock();\n        }\n\n        return result;\n    }\n\n    public HashMap<String, String> getRuntimeInfo() {\n        HashMap<String, String> result = new HashMap<>(64);\n\n        Long totalTimes = getPutMessageTimesTotal();\n        if (0 == totalTimes) {\n            totalTimes = 1L;\n        }\n\n        result.put(\"bootTimestamp\", String.valueOf(this.messageStoreBootTimestamp));\n        result.put(\"runtime\", this.getFormatRuntime());\n        result.put(\"putMessageEntireTimeMax\", String.valueOf(this.putMessageEntireTimeMax));\n        result.put(\"putMessageTimesTotal\", String.valueOf(totalTimes));\n        result.put(\"putMessageFailedTimes\", String.valueOf(this.putMessageFailedTimes));\n        result.put(\"putMessageSizeTotal\", String.valueOf(this.getPutMessageSizeTotal()));\n        result.put(\"putMessageDistributeTime\",\n            String.valueOf(this.getPutMessageDistributeTimeStringInfo(totalTimes)));\n        result.put(\"putMessageAverageSize\",\n            String.valueOf(this.getPutMessageSizeTotal() / totalTimes.doubleValue()));\n        result.put(\"dispatchMaxBuffer\", String.valueOf(this.dispatchMaxBuffer));\n        result.put(\"getMessageEntireTimeMax\", String.valueOf(this.getMessageEntireTimeMax));\n        result.put(\"putTps\", this.getPutTps());\n        result.put(\"getFoundTps\", this.getGetFoundTps());\n        result.put(\"getMissTps\", this.getGetMissTps());\n        result.put(\"getTotalTps\", this.getGetTotalTps());\n        result.put(\"getTransferredTps\", this.getGetTransferredTps());\n        result.put(\"putLatency99\", String.format(\"%.2f\", this.findPutMessageEntireTimePX(0.99)));\n        result.put(\"putLatency999\", String.format(\"%.2f\", this.findPutMessageEntireTimePX(0.999)));\n\n        return result;\n    }\n\n    public void run() {\n        log.info(this.getServiceName() + \" service started\");\n\n        while (!this.isStopped()) {\n            try {\n                this.waitForRunning(FREQUENCY_OF_SAMPLING);\n\n                this.sampling();\n\n                this.printTps();\n            } catch (Exception e) {\n                log.warn(this.getServiceName() + \" service has exception. \", e);\n            }\n        }\n\n        log.info(this.getServiceName() + \" service end\");\n    }\n\n    @Override\n    public String getServiceName() {\n        if (this.brokerIdentity != null && this.brokerIdentity.isInBrokerContainer()) {\n            return brokerIdentity.getIdentifier() + StoreStatsService.class.getSimpleName();\n        }\n        return StoreStatsService.class.getSimpleName();\n    }\n\n    private void sampling() {\n        this.samplingLock.lock();\n        try {\n            this.putTimesList.add(new CallSnapshot(System.currentTimeMillis(), getPutMessageTimesTotal()));\n            if (this.putTimesList.size() > (MAX_RECORDS_OF_SAMPLING + 1)) {\n                this.putTimesList.removeFirst();\n            }\n\n            this.getTimesFoundList.add(new CallSnapshot(System.currentTimeMillis(),\n                this.getMessageTimesTotalFound.longValue()));\n            if (this.getTimesFoundList.size() > (MAX_RECORDS_OF_SAMPLING + 1)) {\n                this.getTimesFoundList.removeFirst();\n            }\n\n            this.getTimesMissList.add(new CallSnapshot(System.currentTimeMillis(),\n                this.getMessageTimesTotalMiss.longValue()));\n            if (this.getTimesMissList.size() > (MAX_RECORDS_OF_SAMPLING + 1)) {\n                this.getTimesMissList.removeFirst();\n            }\n\n            this.transferredMsgCountList.add(new CallSnapshot(System.currentTimeMillis(),\n                this.getMessageTransferredMsgCount.longValue()));\n            if (this.transferredMsgCountList.size() > (MAX_RECORDS_OF_SAMPLING + 1)) {\n                this.transferredMsgCountList.removeFirst();\n            }\n\n        } finally {\n            this.samplingLock.unlock();\n        }\n    }\n\n    private void printTps() {\n        if (System.currentTimeMillis() > (this.lastPrintTimestamp + printTPSInterval * 1000)) {\n            this.lastPrintTimestamp = System.currentTimeMillis();\n\n            log.info(\"[STORETPS] put_tps {} get_found_tps {} get_miss_tps {} get_transferred_tps {}\",\n                this.getPutTps(printTPSInterval),\n                this.getGetFoundTps(printTPSInterval),\n                this.getGetMissTps(printTPSInterval),\n                this.getGetTransferredTps(printTPSInterval)\n            );\n\n            final LongAdder[] times = this.resetPutMessageDistributeTime();\n            if (null == times)\n                return;\n\n            final StringBuilder sb = new StringBuilder();\n            long totalPut = 0;\n            for (int i = 0; i < times.length; i++) {\n                long value = times[i].longValue();\n                totalPut += value;\n                sb.append(String.format(\"%s:%d\", PUT_MESSAGE_ENTIRE_TIME_MAX_DESC[i], value));\n                sb.append(\" \");\n            }\n            this.resetPutMessageTimeBuckets();\n            this.findPutMessageEntireTimePX(0.99);\n            this.findPutMessageEntireTimePX(0.999);\n            log.info(\"[PAGECACHERT] TotalPut {}, PutMessageDistributeTime {}\", totalPut, sb.toString());\n        }\n    }\n\n    public LongAdder getGetMessageTimesTotalFound() {\n        return getMessageTimesTotalFound;\n    }\n\n    public LongAdder getGetMessageTimesTotalMiss() {\n        return getMessageTimesTotalMiss;\n    }\n\n    public LongAdder getGetMessageTransferredMsgCount() {\n        return getMessageTransferredMsgCount;\n    }\n\n    public LongAdder getPutMessageFailedTimes() {\n        return putMessageFailedTimes;\n    }\n\n    public LongAdder getSinglePutMessageTopicSizeTotal(String topic) {\n        LongAdder rs = putMessageTopicSizeTotal.get(topic);\n        if (null == rs) {\n            rs = new LongAdder();\n            LongAdder previous = putMessageTopicSizeTotal.putIfAbsent(topic, rs);\n            if (previous != null) {\n                rs = previous;\n            }\n        }\n        return rs;\n    }\n\n    public LongAdder getSinglePutMessageTopicTimesTotal(String topic) {\n        LongAdder rs = putMessageTopicTimesTotal.get(topic);\n        if (null == rs) {\n            rs = new LongAdder();\n            LongAdder previous = putMessageTopicTimesTotal.putIfAbsent(topic, rs);\n            if (previous != null) {\n                rs = previous;\n            }\n        }\n        return rs;\n    }\n\n    public Map<String, LongAdder> getPutMessageTopicTimesTotal() {\n        return putMessageTopicTimesTotal;\n    }\n\n    public Map<String, LongAdder> getPutMessageTopicSizeTotal() {\n        return putMessageTopicSizeTotal;\n    }\n\n    static class CallSnapshot {\n        public final long timestamp;\n        public final long callTimesTotal;\n\n        public CallSnapshot(long timestamp, long callTimesTotal) {\n            this.timestamp = timestamp;\n            this.callTimesTotal = callTimesTotal;\n        }\n\n        public static double getTPS(final CallSnapshot begin, final CallSnapshot end) {\n            long total = end.callTimesTotal - begin.callTimesTotal;\n            Long time = end.timestamp - begin.timestamp;\n\n            double tps = total / time.doubleValue();\n\n            return tps * 1000;\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/StoreType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.Objects;\nimport java.util.Set;\nimport java.util.stream.Collectors;\n\npublic enum StoreType {\n    DEFAULT(\"default\"),\n    DEFAULT_ROCKSDB(\"defaultRocksDB\");\n\n    private String storeType;\n\n    StoreType(String storeType) {\n        this.storeType = storeType;\n    }\n\n    public String getStoreType() {\n        return storeType;\n    }\n\n    /**\n     * convert string to set of StoreType\n     *\n     * @param str example \"default;defaultRocksDB\"\n     * @return set of StoreType\n     */\n    public static Set<StoreType> fromString(String str) {\n        if (str == null || str.trim().isEmpty()) {\n            return Collections.emptySet();\n        }\n\n        return Arrays.stream(str.split(\";\"))\n            .map(String::trim)\n            .filter(s -> !s.isEmpty())\n            .map(s -> Arrays.stream(StoreType.values())\n                .filter(type -> type.getStoreType().equalsIgnoreCase(s))\n                .findFirst()\n                .orElse(null))\n            .filter(Objects::nonNull)\n            .collect(Collectors.toSet());\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/StoreUtil.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport com.google.common.base.Preconditions;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.logfile.MappedFile;\n\nimport java.lang.management.ManagementFactory;\nimport java.lang.management.OperatingSystemMXBean;\nimport java.nio.ByteBuffer;\n\nimport static java.lang.String.format;\n\npublic class StoreUtil {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n\n    public static final long TOTAL_PHYSICAL_MEMORY_SIZE = getTotalPhysicalMemorySize();\n\n    @SuppressWarnings(\"restriction\")\n    public static long getTotalPhysicalMemorySize() {\n        long physicalTotal = 1024 * 1024 * 1024 * 24L;\n        OperatingSystemMXBean osmxb = ManagementFactory.getOperatingSystemMXBean();\n        if (osmxb instanceof com.sun.management.OperatingSystemMXBean) {\n            physicalTotal = ((com.sun.management.OperatingSystemMXBean) osmxb).getTotalPhysicalMemorySize();\n        }\n\n        return physicalTotal;\n    }\n\n    public static void fileAppend(MappedFile file, ByteBuffer data) {\n        boolean success = file.appendMessage(data);\n        if (!success) {\n            throw new RuntimeException(format(\"fileAppend failed for file: %s and data remaining: %d\", file, data.remaining()));\n        }\n    }\n\n    public static FileQueueSnapshot getFileQueueSnapshot(MappedFileQueue mappedFileQueue) {\n        return getFileQueueSnapshot(mappedFileQueue, mappedFileQueue.getLastMappedFile().getFileFromOffset());\n    }\n\n    public static FileQueueSnapshot getFileQueueSnapshot(MappedFileQueue mappedFileQueue, final long currentFile) {\n        try {\n            Preconditions.checkNotNull(mappedFileQueue, \"file queue shouldn't be null\");\n            MappedFile firstFile = mappedFileQueue.getFirstMappedFile();\n            MappedFile lastFile = mappedFileQueue.getLastMappedFile();\n            int mappedFileSize = mappedFileQueue.getMappedFileSize();\n            if (firstFile == null || lastFile == null) {\n                return new FileQueueSnapshot(firstFile, -1, lastFile, -1, currentFile, -1, 0, false);\n            }\n\n            long firstFileIndex = 0;\n            long lastFileIndex = (lastFile.getFileFromOffset() - firstFile.getFileFromOffset()) / mappedFileSize;\n            long currentFileIndex = (currentFile - firstFile.getFileFromOffset()) / mappedFileSize;\n            long behind = (lastFile.getFileFromOffset() - currentFile) / mappedFileSize;\n            boolean exist = firstFile.getFileFromOffset() <= currentFile && currentFile <= lastFile.getFileFromOffset();\n            return new FileQueueSnapshot(firstFile, firstFileIndex, lastFile, lastFileIndex, currentFile, currentFileIndex, behind, exist);\n        } catch (Exception e) {\n            log.error(\"[BUG] get file queue snapshot failed. fileQueue: {}, currentFile: {}\", mappedFileQueue, currentFile, e);\n        }\n        return new FileQueueSnapshot();\n    }\n\n    public static MessageExt getMessage(long offsetPy, int sizePy, MessageStore messageStore, ByteBuffer byteBuffer) {\n        try {\n            if (offsetPy < 0L || sizePy <= 0 || null == messageStore || null == byteBuffer) {\n                return null;\n            }\n            byteBuffer.position(0);\n            byteBuffer.limit(sizePy);\n            if (!messageStore.getData(offsetPy, sizePy, byteBuffer)) {\n                return null;\n            }\n            byteBuffer.flip();\n            return MessageDecoder.decode(byteBuffer, true, false, false);\n        } catch (Exception e) {\n            log.error(\"getMessage error, offsetPy: {}, sizePy: {}, error: {}\", offsetPy, sizePy, e.getMessage());\n        }\n        return null;\n    }\n\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/Swappable.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\n/**\n * Clean up page-table on super large disk\n */\npublic interface Swappable {\n    void swapMap(int reserveNum, long forceSwapIntervalMs, long normalSwapIntervalMs);\n    void cleanSwappedMap(long forceCleanSwapIntervalMs);\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/TopicQueueLock.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.locks.Lock;\nimport java.util.concurrent.locks.ReentrantLock;\n\npublic class TopicQueueLock {\n    private final int size;\n    private final List<Lock> lockList;\n\n    public TopicQueueLock() {\n        this.size = 32;\n        this.lockList = new ArrayList<>(32);\n        for (int i = 0; i < this.size; i++) {\n            this.lockList.add(new ReentrantLock());\n        }\n    }\n\n    public TopicQueueLock(int size) {\n        this.size = size;\n        this.lockList = new ArrayList<>(size);\n        for (int i = 0; i < this.size; i++) {\n            this.lockList.add(new ReentrantLock());\n        }\n    }\n\n    public void lock(String topicQueueKey) {\n        Lock lock = this.lockList.get((topicQueueKey.hashCode() & 0x7fffffff) % this.size);\n        lock.lock();\n    }\n\n    public void unlock(String topicQueueKey) {\n        Lock lock = this.lockList.get((topicQueueKey.hashCode() & 0x7fffffff) % this.size);\n        lock.unlock();\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport java.nio.ByteBuffer;\nimport java.util.Deque;\nimport java.util.concurrent.ConcurrentLinkedDeque;\n\nimport com.sun.jna.NativeLong;\nimport com.sun.jna.Pointer;\nimport io.netty.util.internal.PlatformDependent;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.util.LibC;\n\npublic class TransientStorePool {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n\n    private final int poolSize;\n    private final int fileSize;\n    private final Deque<ByteBuffer> availableBuffers;\n    private volatile boolean isRealCommit = true;\n\n    public TransientStorePool(final int poolSize, final int fileSize) {\n        this.poolSize = poolSize;\n        this.fileSize = fileSize;\n        this.availableBuffers = new ConcurrentLinkedDeque<>();\n    }\n\n    /**\n     * It's a heavy init method.\n     */\n    public void init() {\n        for (int i = 0; i < poolSize; i++) {\n            ByteBuffer byteBuffer = ByteBuffer.allocateDirect(fileSize);\n\n            final long address = PlatformDependent.directBufferAddress(byteBuffer);\n            Pointer pointer = new Pointer(address);\n            LibC.INSTANCE.mlock(pointer, new NativeLong(fileSize));\n\n            availableBuffers.offer(byteBuffer);\n        }\n    }\n\n    public void destroy() {\n        for (ByteBuffer byteBuffer : availableBuffers) {\n            final long address = PlatformDependent.directBufferAddress(byteBuffer);\n            Pointer pointer = new Pointer(address);\n            LibC.INSTANCE.munlock(pointer, new NativeLong(fileSize));\n        }\n    }\n\n    public void returnBuffer(ByteBuffer byteBuffer) {\n        byteBuffer.position(0);\n        byteBuffer.limit(fileSize);\n        this.availableBuffers.offerFirst(byteBuffer);\n    }\n\n    public ByteBuffer borrowBuffer() {\n        ByteBuffer buffer = availableBuffers.pollFirst();\n        if (availableBuffers.size() < poolSize * 0.4) {\n            log.warn(\"TransientStorePool only remain {} sheets.\", availableBuffers.size());\n        }\n        return buffer;\n    }\n\n    public int availableBufferNums() {\n        return availableBuffers.size();\n    }\n\n    public boolean isRealCommit() {\n        return isRealCommit;\n    }\n\n    public void setRealCommit(boolean realCommit) {\n        isRealCommit = realCommit;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/config/BrokerRole.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.config;\n\npublic enum BrokerRole {\n    ASYNC_MASTER,\n    SYNC_MASTER,\n    SLAVE;\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/config/FlushDiskType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.config;\n\npublic enum FlushDiskType {\n    SYNC_FLUSH,\n    ASYNC_FLUSH\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.config;\n\nimport java.io.File;\nimport org.apache.rocketmq.common.annotation.ImportantField;\nimport org.apache.rocketmq.store.ConsumeQueue;\nimport org.apache.rocketmq.store.StoreType;\nimport org.apache.rocketmq.store.queue.BatchConsumeQueue;\nimport org.rocksdb.CompressionType;\nimport org.rocksdb.util.SizeUnit;\n\npublic class MessageStoreConfig {\n\n    public static final String MULTI_PATH_SPLITTER = System.getProperty(\"rocketmq.broker.multiPathSplitter\", \",\");\n\n    //The root directory in which the log data is kept\n    @ImportantField\n    private String storePathRootDir = System.getProperty(\"user.home\") + File.separator + \"store\";\n\n    //The directory in which the commitlog is kept\n    @ImportantField\n    private String storePathCommitLog = null;\n\n    @ImportantField\n    private String storePathDLedgerCommitLog = null;\n\n    //The directory in which the epochFile is kept\n    @ImportantField\n    private String storePathEpochFile = null;\n\n    @ImportantField\n    private String storePathBrokerIdentity = null;\n\n    private String readOnlyCommitLogStorePaths = null;\n\n    // CommitLog file size,default is 1G\n    private int mappedFileSizeCommitLog = 1024 * 1024 * 1024;\n\n    // CompactionLog file size, default is 100M\n    private int compactionMappedFileSize = 100 * 1024 * 1024;\n\n    // CompactionLog consumeQueue file size, default is 10M\n    private int compactionCqMappedFileSize = 10 * 1024 * 1024;\n\n    private int compactionScheduleInternal = 15 * 60 * 1000;\n\n    private int maxOffsetMapSize = 100 * 1024 * 1024;\n\n    private int compactionThreadNum = 6;\n\n    private boolean enableCompaction = true;\n\n    // TimerLog file size, default is 100M\n    private int mappedFileSizeTimerLog = 100 * 1024 * 1024;\n\n    private int timerPrecisionMs = 1000;\n\n    private int timerRollWindowSlot = 3600 * 24 * 2;\n    private int timerFlushIntervalMs = 1000;\n    private int timerGetMessageThreadNum = 3;\n    private int timerPutMessageThreadNum = 3;\n\n    private boolean timerEnableDisruptor = false;\n\n    private boolean timerEnableCheckMetrics = true;\n    private boolean timerInterceptDelayLevel = false;\n    private int timerMaxDelaySec = 3600 * 24 * 3;\n    private boolean timerWheelSnapshotFlush = false;\n    private boolean timerWheelEnable = true;\n\n    /**\n     * 1. Register to broker after (startTime + disappearTimeAfterStart)\n     * 2. Internal msg exchange will start after (startTime + disappearTimeAfterStart)\n     * A. PopReviveService\n     * B. TimerDequeueGetService\n     */\n    @ImportantField\n    private int disappearTimeAfterStart = -1;\n\n    private boolean timerStopEnqueue = false;\n\n    private String timerCheckMetricsWhen = \"05\";\n\n    private boolean timerSkipUnknownError = false;\n    private boolean timerWarmEnable = false;\n    private boolean timerStopDequeue = false;\n    private boolean timerEnableRetryUntilSuccess = false;\n    private int timerCongestNumEachSlot = Integer.MAX_VALUE;\n\n    private int timerMetricSmallThreshold = 1000000;\n    private int timerProgressLogIntervalMs = 10 * 1000;\n    private int timerWheelSnapshotIntervalMs = 10 * 1000;\n\n    private int commitLogRecoverMaxNum = 10;\n    private boolean timerRocksDBEnable = false;\n    private boolean timerRocksDBStopScan = false;\n    private long timerRocksDBPrecisionMs = 1000L;\n    private double timerRocksDBRollMaxTps = 8000.0;\n    private double timerRocksDBTimeExpiredMaxTps = 200000.0;\n    private int timerRocksDBRollIntervalHours = 1;\n    private int timerRocksDBRollRangeHours = 2;\n    private boolean timerRecallToTimeWheelEnable = true;\n    private boolean timerRecallToTimelineEnable = true;\n    private int timerReputServiceCorePoolSize = 6;\n    private int timerReputServiceMaxPoolSize = 6;\n    private int timerReputServiceQueueCapacity = 10000;\n\n    private boolean transRocksDBEnable = false;\n    private boolean transWriteOriginTransHalfEnable = true;\n\n    private boolean indexRocksDBEnable = false;\n    private int maxRocksDBIndexQueryDays = 7;\n    private boolean indexFileWriteEnable = true;\n    private boolean indexFileReadEnable = true;\n\n    // default, defaultRocksDB\n    @ImportantField\n    private String storeType = StoreType.DEFAULT.getStoreType();\n\n    private boolean iteratorWhenUseRocksdbConsumeQueue = true;\n\n    // ConsumeQueue file size,default is 30W\n    private int mappedFileSizeConsumeQueue = 300000 * ConsumeQueue.CQ_STORE_UNIT_SIZE;\n    // enable consume queue ext\n    private boolean enableConsumeQueueExt = false;\n    // ConsumeQueue extend file size, 48M\n    private int mappedFileSizeConsumeQueueExt = 48 * 1024 * 1024;\n    private int mapperFileSizeBatchConsumeQueue = 300000 * BatchConsumeQueue.CQ_STORE_UNIT_SIZE;\n    // Bit count of filter bit map.\n    // this will be set by pipe of calculate filter bit map.\n    private int bitMapLengthConsumeQueueExt = 64;\n\n    // CommitLog flush interval\n    // flush data to disk\n    @ImportantField\n    private int flushIntervalCommitLog = 500;\n\n    // Only used if TransientStorePool enabled\n    // flush data to FileChannel\n    @ImportantField\n    private int commitIntervalCommitLog = 200;\n\n    private int maxRecoveryCommitlogFiles = 30;\n\n    private int diskSpaceWarningLevelRatio = 90;\n\n    private int diskSpaceCleanForciblyRatio = 85;\n\n    /**\n     * introduced since 4.0.x. Determine whether to use mutex reentrantLock when putting message.<br/>\n     */\n    private boolean useReentrantLockWhenPutMessage = true;\n\n    // Whether schedule flush\n    @ImportantField\n    private boolean flushCommitLogTimed = true;\n    // ConsumeQueue flush interval\n    private int flushIntervalConsumeQueue = 1000;\n    // Resource reclaim interval\n    private int cleanResourceInterval = 10000;\n    // CommitLog removal interval\n    private int deleteCommitLogFilesInterval = 100;\n    // ConsumeQueue removal interval\n    private int deleteConsumeQueueFilesInterval = 100;\n    private int destroyMapedFileIntervalForcibly = 1000 * 120;\n    private int redeleteHangedFileInterval = 1000 * 120;\n    // When to delete,default is at 4 am\n    @ImportantField\n    private String deleteWhen = \"04\";\n    private int diskMaxUsedSpaceRatio = 75;\n    // The number of hours to keep a log file before deleting it (in hours)\n    @ImportantField\n    private int fileReservedTime = 72;\n    @ImportantField\n    private int deleteFileBatchMax = 10;\n    // Flow control for ConsumeQueue\n    private int putMsgIndexHightWater = 600000;\n    // The maximum size of message body,default is 4M,4M only for body length,not include others.\n    private int maxMessageSize = 1024 * 1024 * 4;\n\n    // The maximum size of message body can be  set in config;count with maxMsgNums * CQ_STORE_UNIT_SIZE(20 || 46)\n    private int maxFilterMessageSize = 16000;\n    // Whether check the CRC32 of the records consumed.\n    // This ensures no on-the-wire or on-disk corruption to the messages occurred.\n    // This check adds some overhead,so it may be disabled in cases seeking extreme performance.\n    private boolean checkCRCOnRecover = true;\n    // Whether check the commitlog offset validity during abnormal recovery.\n    // This helps detect and truncate old file data that may pass CRC checks but contains invalid offsets.\n    private boolean checkCommitLogOffsetOnRecover = false;\n    // How many pages are to be flushed when flush CommitLog\n    private int flushCommitLogLeastPages = 4;\n    // How many pages are to be committed when commit data to file\n    private int commitCommitLogLeastPages = 4;\n    // Flush page size when the disk in warming state\n    private int flushLeastPagesWhenWarmMapedFile = 1024 / 4 * 16;\n    // How many pages are to be flushed when flush ConsumeQueue\n    private int flushConsumeQueueLeastPages = 2;\n    private int flushCommitLogThoroughInterval = 1000 * 10;\n    private int commitCommitLogThoroughInterval = 200;\n    private int flushConsumeQueueThoroughInterval = 1000 * 60;\n    @ImportantField\n    private int maxTransferBytesOnMessageInMemory = 1024 * 256;\n    @ImportantField\n    private int maxTransferCountOnMessageInMemory = 32;\n    @ImportantField\n    private int maxTransferBytesOnMessageInDisk = 1024 * 64;\n    @ImportantField\n    private int maxTransferCountOnMessageInDisk = 8;\n    @ImportantField\n    private int accessMessageInMemoryMaxRatio = 40;\n    @ImportantField\n    private boolean messageIndexEnable = true;\n    private int maxHashSlotNum = 5000000;\n    private int maxIndexNum = 5000000 * 4;\n    private int maxMsgsNumBatch = 64;\n    @ImportantField\n    private boolean messageIndexSafe = false;\n    private int haListenPort = 10912;\n    private int haSendHeartbeatInterval = 1000 * 5;\n    private int haHousekeepingInterval = 1000 * 20;\n    /**\n     * Maximum size of data to transfer to slave.\n     * NOTE: cannot be larger than HAClient.READ_MAX_BUFFER_SIZE\n     */\n    private int haTransferBatchSize = 1024 * 32;\n    @ImportantField\n    private String haMasterAddress = null;\n    private int haMaxGapNotInSync = 1024 * 1024 * 256;\n    @ImportantField\n    private volatile BrokerRole brokerRole = BrokerRole.ASYNC_MASTER;\n    @ImportantField\n    private FlushDiskType flushDiskType = FlushDiskType.ASYNC_FLUSH;\n    // Used by GroupTransferService to sync messages from master to slave\n    private int syncFlushTimeout = 1000 * 5;\n    // Used by PutMessage to wait messages be flushed to disk and synchronized in current broker member group.\n    private int putMessageTimeout = 1000 * 8;\n    private int slaveTimeout = 3000;\n    private String messageDelayLevel = \"1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h\";\n    private long flushDelayOffsetInterval = 1000 * 10;\n    @ImportantField\n    private boolean cleanFileForciblyEnable = true;\n    private boolean warmMapedFileEnable = false;\n    private boolean offsetCheckInSlave = false;\n    private boolean debugLockEnable = false;\n    private boolean duplicationEnable = false;\n    private boolean diskFallRecorded = true;\n    private long osPageCacheBusyTimeOutMills = 1000;\n    private int defaultQueryMaxNum = 32;\n\n    @ImportantField\n    private boolean transientStorePoolEnable = false;\n    private int transientStorePoolSize = 5;\n    private boolean fastFailIfNoBufferInStorePool = false;\n\n    /**\n     * When true, use RandomAccessFile for writing instead of MappedByteBuffer. This can be useful for certain scenarios\n     * where mmap is not desired.\n     *\n     * The configurations writeWithoutMmap and transientStorePoolEnable are mutually exclusive. When both are set to\n     * true, only writeWithoutMmap will be effective.\n     */\n    @ImportantField\n    private boolean writeWithoutMmap = false;\n\n    // DLedger message store config\n    private boolean enableDLegerCommitLog = false;\n    private String dLegerGroup;\n    private String dLegerPeers;\n    private String dLegerSelfId;\n    private String preferredLeaderId;\n    private boolean enableBatchPush = false;\n\n    private boolean enableScheduleMessageStats = true;\n\n    private boolean enableLmq = false;\n    private boolean enableMultiDispatch = false;\n    private int maxLmqConsumeQueueNum = 20000;\n    private boolean enableLmqQuota = false;\n\n    private boolean enableScheduleAsyncDeliver = false;\n    private int scheduleAsyncDeliverMaxPendingLimit = 2000;\n    private int scheduleAsyncDeliverMaxResendNum2Blocked = 3;\n\n    private int maxBatchDeleteFilesNum = 50;\n    //Polish dispatch\n    private int dispatchCqThreads = 10;\n    private int dispatchCqCacheNum = 1024 * 4;\n    private boolean enableAsyncReput = true;\n    //For recheck the reput\n    private boolean recheckReputOffsetFromCq = false;\n\n    // Maximum length of topic, it will be removed in the future release\n    @Deprecated\n    private int maxTopicLength = Byte.MAX_VALUE;\n\n    /**\n     * Use MessageVersion.MESSAGE_VERSION_V2 automatically if topic length larger than Bytes.MAX_VALUE.\n     * Otherwise, store use MESSAGE_VERSION_V1. Note: Client couldn't decode MESSAGE_VERSION_V2 version message.\n     * Enable this config to resolve this issue. https://github.com/apache/rocketmq/issues/5568\n     */\n    private boolean autoMessageVersionOnTopicLen = true;\n\n    /**\n     * Whether to use runningFlags when flushing data to disk.\n     * When disabled, runningFlags will be set to null during MappedFileQueue and MappedFile initialization.\n     */\n    @ImportantField\n    private boolean enableRunningFlagsInFlush = false;\n\n    /**\n     * It cannot be changed after the broker is started.\n     * Modifications need to be restarted to take effect.\n     */\n    private boolean enabledAppendPropCRC = false;\n    private boolean forceVerifyPropCRC = false;\n    private int travelCqFileNumWhenGetMessage = 1;\n    // Sleep interval between to corrections\n    private int correctLogicMinOffsetSleepInterval = 1;\n    // Force correct min offset interval\n    private int correctLogicMinOffsetForceInterval = 5 * 60 * 1000;\n    // swap\n    private boolean mappedFileSwapEnable = true;\n    private long commitLogForceSwapMapInterval = 12L * 60 * 60 * 1000;\n    private long commitLogSwapMapInterval = 1L * 60 * 60 * 1000;\n    private int commitLogSwapMapReserveFileNum = 100;\n    private long logicQueueForceSwapMapInterval = 12L * 60 * 60 * 1000;\n    private long logicQueueSwapMapInterval = 1L * 60 * 60 * 1000;\n    private long cleanSwapedMapInterval = 5L * 60 * 1000;\n    private int logicQueueSwapMapReserveFileNum = 20;\n\n    private boolean searchBcqByCacheEnable = true;\n\n    @ImportantField\n    private boolean dispatchFromSenderThread = false;\n\n    @ImportantField\n    private boolean wakeCommitWhenPutMessage = true;\n    @ImportantField\n    private boolean wakeFlushWhenPutMessage = false;\n\n    @ImportantField\n    private boolean enableCleanExpiredOffset = false;\n\n    private int maxAsyncPutMessageRequests = 5000;\n\n    private int pullBatchMaxMessageCount = 160;\n\n    @ImportantField\n    private int totalReplicas = 1;\n\n    /**\n     * Each message must be written successfully to at least in-sync replicas.\n     * The master broker is considered one of the in-sync replicas, and it's included in the count of total.\n     * If a master broker is ASYNC_MASTER, inSyncReplicas will be ignored.\n     * If enableControllerMode is true and ackAckInSyncStateSet is true, inSyncReplicas will be ignored.\n     */\n    @ImportantField\n    private int inSyncReplicas = 1;\n\n    /**\n     * Will be worked in auto multiple replicas mode, to provide minimum in-sync replicas.\n     * It is still valid in controller mode.\n     */\n    @ImportantField\n    private int minInSyncReplicas = 1;\n\n    /**\n     * Each message must be written successfully to all replicas in SyncStateSet.\n     */\n    @ImportantField\n    private boolean allAckInSyncStateSet = false;\n\n    /**\n     * Dynamically adjust in-sync replicas to provide higher availability, the real time in-sync replicas\n     * will smaller than inSyncReplicas config.\n     */\n    @ImportantField\n    private boolean enableAutoInSyncReplicas = false;\n\n    /**\n     * Enable or not ha flow control\n     */\n    @ImportantField\n    private boolean haFlowControlEnable = false;\n\n    /**\n     * The max speed for one slave when transfer data in ha\n     */\n    private long maxHaTransferByteInSecond = 100 * 1024 * 1024;\n\n    /**\n     * The max gap time that slave doesn't catch up to master.\n     */\n    private long haMaxTimeSlaveNotCatchup = 1000 * 15;\n\n    /**\n     * Sync flush offset from master when broker startup, used in upgrading from old version broker.\n     */\n    private boolean syncMasterFlushOffsetWhenStartup = false;\n\n    /**\n     * Max checksum range.\n     */\n    private long maxChecksumRange = 1024 * 1024 * 1024;\n\n    private int replicasPerDiskPartition = 1;\n\n    private double logicalDiskSpaceCleanForciblyThreshold = 0.8;\n\n    private long maxSlaveResendLength = 256 * 1024 * 1024;\n\n    /**\n     * Whether sync from lastFile when a new broker replicas(no data) join the master.\n     */\n    private boolean syncFromLastFile = false;\n\n    private boolean asyncLearner = false;\n\n    /**\n     * Number of records to scan before starting to estimate.\n     */\n    private int maxConsumeQueueScan = 20_000;\n\n    /**\n     * Number of matched records before starting to estimate.\n     */\n    private int sampleCountThreshold = 5000;\n\n    private boolean coldDataFlowControlEnable = false;\n    private boolean coldDataScanEnable = false;\n    private boolean dataReadAheadEnable = true;\n    private int timerColdDataCheckIntervalMs = 60 * 1000;\n    private int sampleSteps = 32;\n    private int accessMessageInMemoryHotRatio = 26;\n    /**\n     * Build ConsumeQueue concurrently with multi-thread\n     */\n    private boolean enableBuildConsumeQueueConcurrently = false;\n\n    private int batchDispatchRequestThreadPoolNums = 16;\n\n    // rocksdb mode\n    private long cleanRocksDBDirtyCQIntervalMin = 60;\n    private long statRocksDBCQIntervalSec = 10;\n    private long memTableFlushIntervalMs = 60 * 60 * 1000L;\n    private boolean realTimePersistRocksDBConfig = true;\n    private boolean enableRocksDBLog = false;\n\n    private int topicQueueLockNum = 32;\n\n    /**\n     * If readUnCommitted is true, the dispatch of the consume queue will exceed the confirmOffset, which may cause the client to read uncommitted messages.\n     * For example, reput offset exceeding the flush offset during synchronous disk flushing.\n     */\n    private boolean readUnCommitted = false;\n\n    private boolean putConsumeQueueDataByFileChannel = true;\n\n    private boolean rocksdbCQDoubleWriteEnable = false;\n\n    /**\n     * CombineConsumeQueueStore\n     * combineCQLoadingCQTypes is used to configure the loading types of CQ. load / recover / start order: [default -> defaultRocksDB]\n     * combineCQPreferCQType is used to configure the preferred CQ type when reading. Make sure the CQ type is included in combineCQLoadingCQTypes\n     * combineAssignOffsetCQType is used to configure the CQ type when assign offset. Make sure the CQ type is included in combineCQLoadingCQTypes\n     */\n    private String combineCQLoadingCQTypes = StoreType.DEFAULT.getStoreType() + \";\" + StoreType.DEFAULT_ROCKSDB.getStoreType();\n    private String combineCQPreferCQType = StoreType.DEFAULT.getStoreType();\n    private String combineAssignOffsetCQType = StoreType.DEFAULT.getStoreType();\n    private boolean combineCQEnableCheckSelf = false;\n    private int combineCQMaxExtraSearchCommitLogFiles = 3;\n\n    /**\n     * If ConsumeQueueStore is RocksDB based, this option is to configure bottom-most tier compression type.\n     * The following values are valid:\n     * <ul>\n     *     <li>snappy</li>\n     *     <li>z</li>\n     *     <li>bzip2</li>\n     *     <li>lz4</li>\n     *     <li>lz4hc</li>\n     *     <li>xpress</li>\n     *     <li>zstd</li>\n     * </ul>\n     *\n     * LZ4 is the recommended one.\n     */\n    private String bottomMostCompressionTypeForConsumeQueueStore = CompressionType.ZSTD_COMPRESSION.getLibraryName();\n\n    private String rocksdbCompressionType = CompressionType.LZ4_COMPRESSION.getLibraryName();\n\n    /**\n     * Flush RocksDB WAL frequency, aka, flush WAL every N write ops.\n     */\n    private int rocksdbFlushWalFrequency = 1024;\n\n    private long rocksdbWalFileRollingThreshold = SizeUnit.GB;\n\n    /**\n     * Note: For correctness, this switch should be enabled only if the previous startup was configured with SYNC_FLUSH.\n     * This switch is not recommended for normal use cases (include master-slave or controller mode).\n     */\n    private boolean enableAcceleratedRecovery = false;\n\n    // Shared byte buffer manager configuration\n    private int sharedByteBufferNum = 16;\n\n    public String getRocksdbCompressionType() {\n        return rocksdbCompressionType;\n    }\n\n    public void setRocksdbCompressionType(String compressionType) {\n        this.rocksdbCompressionType = compressionType;\n    }\n\n    /**\n     * Spin number in the retreat strategy of spin lock\n     * Default is 1000\n     */\n    private int spinLockCollisionRetreatOptimalDegree = 1000;\n\n    /**\n     * Use AdaptiveBackOffLock\n     **/\n    private boolean useABSLock = false;\n\n    private boolean enableLogConsumeQueueRepeatedlyBuildWhenRecover = false;\n\n    private boolean appendTopicForTimerDeleteKey = false;\n\n    public boolean isRocksdbCQDoubleWriteEnable() {\n        return rocksdbCQDoubleWriteEnable;\n    }\n\n    public void setRocksdbCQDoubleWriteEnable(boolean rocksdbWriteEnable) {\n        this.rocksdbCQDoubleWriteEnable = rocksdbWriteEnable;\n    }\n\n\n    public boolean isEnabledAppendPropCRC() {\n        return enabledAppendPropCRC;\n    }\n\n    public void setEnabledAppendPropCRC(boolean enabledAppendPropCRC) {\n        this.enabledAppendPropCRC = enabledAppendPropCRC;\n    }\n\n    public boolean isDebugLockEnable() {\n        return debugLockEnable;\n    }\n\n    public void setDebugLockEnable(final boolean debugLockEnable) {\n        this.debugLockEnable = debugLockEnable;\n    }\n\n    public boolean isDuplicationEnable() {\n        return duplicationEnable;\n    }\n\n    public void setDuplicationEnable(final boolean duplicationEnable) {\n        this.duplicationEnable = duplicationEnable;\n    }\n\n    public long getOsPageCacheBusyTimeOutMills() {\n        return osPageCacheBusyTimeOutMills;\n    }\n\n    public void setOsPageCacheBusyTimeOutMills(final long osPageCacheBusyTimeOutMills) {\n        this.osPageCacheBusyTimeOutMills = osPageCacheBusyTimeOutMills;\n    }\n\n    public boolean isDiskFallRecorded() {\n        return diskFallRecorded;\n    }\n\n    public void setDiskFallRecorded(final boolean diskFallRecorded) {\n        this.diskFallRecorded = diskFallRecorded;\n    }\n\n    public boolean isWarmMapedFileEnable() {\n        return warmMapedFileEnable;\n    }\n\n    public void setWarmMapedFileEnable(boolean warmMapedFileEnable) {\n        this.warmMapedFileEnable = warmMapedFileEnable;\n    }\n\n    public int getCompactionMappedFileSize() {\n        return compactionMappedFileSize;\n    }\n\n    public int getCompactionCqMappedFileSize() {\n        return compactionCqMappedFileSize;\n    }\n\n    public void setCompactionMappedFileSize(int compactionMappedFileSize) {\n        this.compactionMappedFileSize = compactionMappedFileSize;\n    }\n\n    public void setCompactionCqMappedFileSize(int compactionCqMappedFileSize) {\n        this.compactionCqMappedFileSize = compactionCqMappedFileSize;\n    }\n\n    public int getCompactionScheduleInternal() {\n        return compactionScheduleInternal;\n    }\n\n    public void setCompactionScheduleInternal(int compactionScheduleInternal) {\n        this.compactionScheduleInternal = compactionScheduleInternal;\n    }\n\n    public int getMaxOffsetMapSize() {\n        return maxOffsetMapSize;\n    }\n\n    public void setMaxOffsetMapSize(int maxOffsetMapSize) {\n        this.maxOffsetMapSize = maxOffsetMapSize;\n    }\n\n    public int getCompactionThreadNum() {\n        return compactionThreadNum;\n    }\n\n    public void setCompactionThreadNum(int compactionThreadNum) {\n        this.compactionThreadNum = compactionThreadNum;\n    }\n\n    public boolean isEnableCompaction() {\n        return enableCompaction;\n    }\n\n    public void setEnableCompaction(boolean enableCompaction) {\n        this.enableCompaction = enableCompaction;\n    }\n\n    public int getMappedFileSizeCommitLog() {\n        return mappedFileSizeCommitLog;\n    }\n\n    public void setMappedFileSizeCommitLog(int mappedFileSizeCommitLog) {\n        this.mappedFileSizeCommitLog = mappedFileSizeCommitLog;\n    }\n\n    public boolean isEnableRocksDBStore() {\n        return StoreType.DEFAULT_ROCKSDB.getStoreType().equalsIgnoreCase(this.storeType);\n    }\n\n    public String getStoreType() {\n        return storeType;\n    }\n\n    public void setStoreType(String storeType) {\n        this.storeType = storeType;\n    }\n\n    public boolean isIteratorWhenUseRocksdbConsumeQueue() {\n        return iteratorWhenUseRocksdbConsumeQueue;\n    }\n\n    public void setIteratorWhenUseRocksdbConsumeQueue(boolean iteratorWhenUseRocksdbConsumeQueue) {\n        this.iteratorWhenUseRocksdbConsumeQueue = iteratorWhenUseRocksdbConsumeQueue;\n    }\n\n    public int getMappedFileSizeConsumeQueue() {\n        int factor = (int) Math.ceil(this.mappedFileSizeConsumeQueue / (ConsumeQueue.CQ_STORE_UNIT_SIZE * 1.0));\n        return (int) (factor * ConsumeQueue.CQ_STORE_UNIT_SIZE);\n    }\n\n    public void setMappedFileSizeConsumeQueue(int mappedFileSizeConsumeQueue) {\n        this.mappedFileSizeConsumeQueue = mappedFileSizeConsumeQueue;\n    }\n\n    public boolean isEnableConsumeQueueExt() {\n        return enableConsumeQueueExt;\n    }\n\n    public void setEnableConsumeQueueExt(boolean enableConsumeQueueExt) {\n        this.enableConsumeQueueExt = enableConsumeQueueExt;\n    }\n\n    public int getMappedFileSizeConsumeQueueExt() {\n        return mappedFileSizeConsumeQueueExt;\n    }\n\n    public void setMappedFileSizeConsumeQueueExt(int mappedFileSizeConsumeQueueExt) {\n        this.mappedFileSizeConsumeQueueExt = mappedFileSizeConsumeQueueExt;\n    }\n\n    public int getBitMapLengthConsumeQueueExt() {\n        return bitMapLengthConsumeQueueExt;\n    }\n\n    public void setBitMapLengthConsumeQueueExt(int bitMapLengthConsumeQueueExt) {\n        this.bitMapLengthConsumeQueueExt = bitMapLengthConsumeQueueExt;\n    }\n\n    public int getFlushIntervalCommitLog() {\n        return flushIntervalCommitLog;\n    }\n\n    public void setFlushIntervalCommitLog(int flushIntervalCommitLog) {\n        this.flushIntervalCommitLog = flushIntervalCommitLog;\n    }\n\n    public int getFlushIntervalConsumeQueue() {\n        return flushIntervalConsumeQueue;\n    }\n\n    public void setFlushIntervalConsumeQueue(int flushIntervalConsumeQueue) {\n        this.flushIntervalConsumeQueue = flushIntervalConsumeQueue;\n    }\n\n    public int getPutMsgIndexHightWater() {\n        return putMsgIndexHightWater;\n    }\n\n    public void setPutMsgIndexHightWater(int putMsgIndexHightWater) {\n        this.putMsgIndexHightWater = putMsgIndexHightWater;\n    }\n\n    public int getCleanResourceInterval() {\n        return cleanResourceInterval;\n    }\n\n    public void setCleanResourceInterval(int cleanResourceInterval) {\n        this.cleanResourceInterval = cleanResourceInterval;\n    }\n\n    public int getMaxMessageSize() {\n        return maxMessageSize;\n    }\n\n    public void setMaxMessageSize(int maxMessageSize) {\n        this.maxMessageSize = maxMessageSize;\n    }\n\n    public int getMaxFilterMessageSize() {\n        return maxFilterMessageSize;\n    }\n\n    public void setMaxFilterMessageSize(int maxFilterMessageSize) {\n        this.maxFilterMessageSize = maxFilterMessageSize;\n    }\n\n    @Deprecated\n    public int getMaxTopicLength() {\n        return maxTopicLength;\n    }\n\n    @Deprecated\n    public void setMaxTopicLength(int maxTopicLength) {\n        this.maxTopicLength = maxTopicLength;\n    }\n\n    public boolean isAutoMessageVersionOnTopicLen() {\n        return autoMessageVersionOnTopicLen;\n    }\n\n    public void setAutoMessageVersionOnTopicLen(boolean autoMessageVersionOnTopicLen) {\n        this.autoMessageVersionOnTopicLen = autoMessageVersionOnTopicLen;\n    }\n\n    public int getTravelCqFileNumWhenGetMessage() {\n        return travelCqFileNumWhenGetMessage;\n    }\n\n    public void setTravelCqFileNumWhenGetMessage(int travelCqFileNumWhenGetMessage) {\n        this.travelCqFileNumWhenGetMessage = travelCqFileNumWhenGetMessage;\n    }\n\n    public int getCorrectLogicMinOffsetSleepInterval() {\n        return correctLogicMinOffsetSleepInterval;\n    }\n\n    public void setCorrectLogicMinOffsetSleepInterval(int correctLogicMinOffsetSleepInterval) {\n        this.correctLogicMinOffsetSleepInterval = correctLogicMinOffsetSleepInterval;\n    }\n\n    public int getCorrectLogicMinOffsetForceInterval() {\n        return correctLogicMinOffsetForceInterval;\n    }\n\n    public void setCorrectLogicMinOffsetForceInterval(int correctLogicMinOffsetForceInterval) {\n        this.correctLogicMinOffsetForceInterval = correctLogicMinOffsetForceInterval;\n    }\n\n    public boolean isCheckCRCOnRecover() {\n        return checkCRCOnRecover;\n    }\n\n    public boolean getCheckCRCOnRecover() {\n        return checkCRCOnRecover;\n    }\n\n    public void setCheckCRCOnRecover(boolean checkCRCOnRecover) {\n        this.checkCRCOnRecover = checkCRCOnRecover;\n    }\n\n    public boolean isCheckCommitLogOffsetOnRecover() {\n        return checkCommitLogOffsetOnRecover;\n    }\n\n    public void setCheckCommitLogOffsetOnRecover(boolean checkCommitLogOffsetOnRecover) {\n        this.checkCommitLogOffsetOnRecover = checkCommitLogOffsetOnRecover;\n    }\n\n    public boolean isForceVerifyPropCRC() {\n        return forceVerifyPropCRC;\n    }\n\n    public void setForceVerifyPropCRC(boolean forceVerifyPropCRC) {\n        this.forceVerifyPropCRC = forceVerifyPropCRC;\n    }\n\n    public String getStorePathCommitLog() {\n        if (storePathCommitLog == null) {\n            return storePathRootDir + File.separator + \"commitlog\";\n        }\n        return storePathCommitLog;\n    }\n\n    public void setStorePathCommitLog(String storePathCommitLog) {\n        this.storePathCommitLog = storePathCommitLog;\n    }\n\n    public String getStorePathDLedgerCommitLog() {\n        return storePathDLedgerCommitLog;\n    }\n\n    public void setStorePathDLedgerCommitLog(String storePathDLedgerCommitLog) {\n        this.storePathDLedgerCommitLog = storePathDLedgerCommitLog;\n    }\n\n    public String getStorePathEpochFile() {\n        if (storePathEpochFile == null) {\n            return storePathRootDir + File.separator + \"epochFileCheckpoint\";\n        }\n        return storePathEpochFile;\n    }\n\n    public void setStorePathEpochFile(String storePathEpochFile) {\n        this.storePathEpochFile = storePathEpochFile;\n    }\n\n    public String getStorePathBrokerIdentity() {\n        if (storePathBrokerIdentity == null) {\n            return storePathRootDir + File.separator + \"brokerIdentity\";\n        }\n        return storePathBrokerIdentity;\n    }\n\n    public void setStorePathBrokerIdentity(String storePathBrokerIdentity) {\n        this.storePathBrokerIdentity = storePathBrokerIdentity;\n    }\n\n    public String getDeleteWhen() {\n        return deleteWhen;\n    }\n\n    public void setDeleteWhen(String deleteWhen) {\n        this.deleteWhen = deleteWhen;\n    }\n\n    public int getDiskMaxUsedSpaceRatio() {\n        if (this.diskMaxUsedSpaceRatio < 10)\n            return 10;\n\n        if (this.diskMaxUsedSpaceRatio > 95)\n            return 95;\n\n        return diskMaxUsedSpaceRatio;\n    }\n\n    public void setDiskMaxUsedSpaceRatio(int diskMaxUsedSpaceRatio) {\n        this.diskMaxUsedSpaceRatio = diskMaxUsedSpaceRatio;\n    }\n\n    public int getDeleteCommitLogFilesInterval() {\n        return deleteCommitLogFilesInterval;\n    }\n\n    public void setDeleteCommitLogFilesInterval(int deleteCommitLogFilesInterval) {\n        this.deleteCommitLogFilesInterval = deleteCommitLogFilesInterval;\n    }\n\n    public int getDeleteConsumeQueueFilesInterval() {\n        return deleteConsumeQueueFilesInterval;\n    }\n\n    public void setDeleteConsumeQueueFilesInterval(int deleteConsumeQueueFilesInterval) {\n        this.deleteConsumeQueueFilesInterval = deleteConsumeQueueFilesInterval;\n    }\n\n    public int getMaxTransferBytesOnMessageInMemory() {\n        return maxTransferBytesOnMessageInMemory;\n    }\n\n    public void setMaxTransferBytesOnMessageInMemory(int maxTransferBytesOnMessageInMemory) {\n        this.maxTransferBytesOnMessageInMemory = maxTransferBytesOnMessageInMemory;\n    }\n\n    public int getMaxTransferCountOnMessageInMemory() {\n        return maxTransferCountOnMessageInMemory;\n    }\n\n    public void setMaxTransferCountOnMessageInMemory(int maxTransferCountOnMessageInMemory) {\n        this.maxTransferCountOnMessageInMemory = maxTransferCountOnMessageInMemory;\n    }\n\n    public int getMaxTransferBytesOnMessageInDisk() {\n        return maxTransferBytesOnMessageInDisk;\n    }\n\n    public void setMaxTransferBytesOnMessageInDisk(int maxTransferBytesOnMessageInDisk) {\n        this.maxTransferBytesOnMessageInDisk = maxTransferBytesOnMessageInDisk;\n    }\n\n    public int getMaxTransferCountOnMessageInDisk() {\n        return maxTransferCountOnMessageInDisk;\n    }\n\n    public void setMaxTransferCountOnMessageInDisk(int maxTransferCountOnMessageInDisk) {\n        this.maxTransferCountOnMessageInDisk = maxTransferCountOnMessageInDisk;\n    }\n\n    public int getFlushCommitLogLeastPages() {\n        return flushCommitLogLeastPages;\n    }\n\n    public void setFlushCommitLogLeastPages(int flushCommitLogLeastPages) {\n        this.flushCommitLogLeastPages = flushCommitLogLeastPages;\n    }\n\n    public int getFlushConsumeQueueLeastPages() {\n        return flushConsumeQueueLeastPages;\n    }\n\n    public void setFlushConsumeQueueLeastPages(int flushConsumeQueueLeastPages) {\n        this.flushConsumeQueueLeastPages = flushConsumeQueueLeastPages;\n    }\n\n    public int getFlushCommitLogThoroughInterval() {\n        return flushCommitLogThoroughInterval;\n    }\n\n    public void setFlushCommitLogThoroughInterval(int flushCommitLogThoroughInterval) {\n        this.flushCommitLogThoroughInterval = flushCommitLogThoroughInterval;\n    }\n\n    public int getFlushConsumeQueueThoroughInterval() {\n        return flushConsumeQueueThoroughInterval;\n    }\n\n    public void setFlushConsumeQueueThoroughInterval(int flushConsumeQueueThoroughInterval) {\n        this.flushConsumeQueueThoroughInterval = flushConsumeQueueThoroughInterval;\n    }\n\n    public int getDestroyMapedFileIntervalForcibly() {\n        return destroyMapedFileIntervalForcibly;\n    }\n\n    public void setDestroyMapedFileIntervalForcibly(int destroyMapedFileIntervalForcibly) {\n        this.destroyMapedFileIntervalForcibly = destroyMapedFileIntervalForcibly;\n    }\n\n    public int getFileReservedTime() {\n        return fileReservedTime;\n    }\n\n    public void setFileReservedTime(int fileReservedTime) {\n        this.fileReservedTime = fileReservedTime;\n    }\n\n    public int getRedeleteHangedFileInterval() {\n        return redeleteHangedFileInterval;\n    }\n\n    public void setRedeleteHangedFileInterval(int redeleteHangedFileInterval) {\n        this.redeleteHangedFileInterval = redeleteHangedFileInterval;\n    }\n\n    public int getAccessMessageInMemoryMaxRatio() {\n        return accessMessageInMemoryMaxRatio;\n    }\n\n    public void setAccessMessageInMemoryMaxRatio(int accessMessageInMemoryMaxRatio) {\n        this.accessMessageInMemoryMaxRatio = accessMessageInMemoryMaxRatio;\n    }\n\n    public boolean isMessageIndexEnable() {\n        return messageIndexEnable;\n    }\n\n    public void setMessageIndexEnable(boolean messageIndexEnable) {\n        this.messageIndexEnable = messageIndexEnable;\n    }\n\n    public int getMaxHashSlotNum() {\n        return maxHashSlotNum;\n    }\n\n    public void setMaxHashSlotNum(int maxHashSlotNum) {\n        this.maxHashSlotNum = maxHashSlotNum;\n    }\n\n    public int getMaxIndexNum() {\n        return maxIndexNum;\n    }\n\n    public void setMaxIndexNum(int maxIndexNum) {\n        this.maxIndexNum = maxIndexNum;\n    }\n\n    public int getMaxMsgsNumBatch() {\n        return maxMsgsNumBatch;\n    }\n\n    public void setMaxMsgsNumBatch(int maxMsgsNumBatch) {\n        this.maxMsgsNumBatch = maxMsgsNumBatch;\n    }\n\n    public int getHaListenPort() {\n        return haListenPort;\n    }\n\n    public void setHaListenPort(int haListenPort) {\n        if (haListenPort < 0) {\n            this.haListenPort = 0;\n            return;\n        }\n        this.haListenPort = haListenPort;\n    }\n\n    public int getHaSendHeartbeatInterval() {\n        return haSendHeartbeatInterval;\n    }\n\n    public void setHaSendHeartbeatInterval(int haSendHeartbeatInterval) {\n        this.haSendHeartbeatInterval = haSendHeartbeatInterval;\n    }\n\n    public int getHaHousekeepingInterval() {\n        return haHousekeepingInterval;\n    }\n\n    public void setHaHousekeepingInterval(int haHousekeepingInterval) {\n        this.haHousekeepingInterval = haHousekeepingInterval;\n    }\n\n    public BrokerRole getBrokerRole() {\n        return brokerRole;\n    }\n\n    public void setBrokerRole(BrokerRole brokerRole) {\n        this.brokerRole = brokerRole;\n    }\n\n    public void setBrokerRole(String brokerRole) {\n        this.brokerRole = BrokerRole.valueOf(brokerRole);\n    }\n\n    public int getHaTransferBatchSize() {\n        return haTransferBatchSize;\n    }\n\n    public void setHaTransferBatchSize(int haTransferBatchSize) {\n        this.haTransferBatchSize = haTransferBatchSize;\n    }\n\n    public int getHaMaxGapNotInSync() {\n        return haMaxGapNotInSync;\n    }\n\n    public void setHaMaxGapNotInSync(int haMaxGapNotInSync) {\n        this.haMaxGapNotInSync = haMaxGapNotInSync;\n    }\n\n    public FlushDiskType getFlushDiskType() {\n        return flushDiskType;\n    }\n\n    public void setFlushDiskType(FlushDiskType flushDiskType) {\n        this.flushDiskType = flushDiskType;\n    }\n\n    public void setFlushDiskType(String type) {\n        this.flushDiskType = FlushDiskType.valueOf(type);\n    }\n\n    public int getSyncFlushTimeout() {\n        return syncFlushTimeout;\n    }\n\n    public void setSyncFlushTimeout(int syncFlushTimeout) {\n        this.syncFlushTimeout = syncFlushTimeout;\n    }\n\n    public int getPutMessageTimeout() {\n        return putMessageTimeout;\n    }\n\n    public void setPutMessageTimeout(int putMessageTimeout) {\n        this.putMessageTimeout = putMessageTimeout;\n    }\n\n    public int getSlaveTimeout() {\n        return slaveTimeout;\n    }\n\n    public void setSlaveTimeout(int slaveTimeout) {\n        this.slaveTimeout = slaveTimeout;\n    }\n\n    public String getHaMasterAddress() {\n        return haMasterAddress;\n    }\n\n    public void setHaMasterAddress(String haMasterAddress) {\n        this.haMasterAddress = haMasterAddress;\n    }\n\n    public String getMessageDelayLevel() {\n        return messageDelayLevel;\n    }\n\n    public void setMessageDelayLevel(String messageDelayLevel) {\n        this.messageDelayLevel = messageDelayLevel;\n    }\n\n    public long getFlushDelayOffsetInterval() {\n        return flushDelayOffsetInterval;\n    }\n\n    public void setFlushDelayOffsetInterval(long flushDelayOffsetInterval) {\n        this.flushDelayOffsetInterval = flushDelayOffsetInterval;\n    }\n\n    public boolean isCleanFileForciblyEnable() {\n        return cleanFileForciblyEnable;\n    }\n\n    public void setCleanFileForciblyEnable(boolean cleanFileForciblyEnable) {\n        this.cleanFileForciblyEnable = cleanFileForciblyEnable;\n    }\n\n    public boolean isMessageIndexSafe() {\n        return messageIndexSafe;\n    }\n\n    public void setMessageIndexSafe(boolean messageIndexSafe) {\n        this.messageIndexSafe = messageIndexSafe;\n    }\n\n    public boolean isFlushCommitLogTimed() {\n        return flushCommitLogTimed;\n    }\n\n    public void setFlushCommitLogTimed(boolean flushCommitLogTimed) {\n        this.flushCommitLogTimed = flushCommitLogTimed;\n    }\n\n    public String getStorePathRootDir() {\n        return storePathRootDir;\n    }\n\n    public void setStorePathRootDir(String storePathRootDir) {\n        this.storePathRootDir = storePathRootDir;\n    }\n\n    public int getFlushLeastPagesWhenWarmMapedFile() {\n        return flushLeastPagesWhenWarmMapedFile;\n    }\n\n    public void setFlushLeastPagesWhenWarmMapedFile(int flushLeastPagesWhenWarmMapedFile) {\n        this.flushLeastPagesWhenWarmMapedFile = flushLeastPagesWhenWarmMapedFile;\n    }\n\n    public boolean isOffsetCheckInSlave() {\n        return offsetCheckInSlave;\n    }\n\n    public void setOffsetCheckInSlave(boolean offsetCheckInSlave) {\n        this.offsetCheckInSlave = offsetCheckInSlave;\n    }\n\n    public int getDefaultQueryMaxNum() {\n        return defaultQueryMaxNum;\n    }\n\n    public void setDefaultQueryMaxNum(int defaultQueryMaxNum) {\n        this.defaultQueryMaxNum = defaultQueryMaxNum;\n    }\n\n    public boolean isTransientStorePoolEnable() {\n        return transientStorePoolEnable;\n    }\n\n    public void setTransientStorePoolEnable(final boolean transientStorePoolEnable) {\n        this.transientStorePoolEnable = transientStorePoolEnable;\n    }\n\n    public boolean isWriteWithoutMmap() {\n        return writeWithoutMmap;\n    }\n\n    public void setWriteWithoutMmap(final boolean writeWithoutMmap) {\n        this.writeWithoutMmap = writeWithoutMmap;\n    }\n\n    public int getTransientStorePoolSize() {\n        return transientStorePoolSize;\n    }\n\n    public void setTransientStorePoolSize(final int transientStorePoolSize) {\n        this.transientStorePoolSize = transientStorePoolSize;\n    }\n\n    public int getCommitIntervalCommitLog() {\n        return commitIntervalCommitLog;\n    }\n\n    public void setCommitIntervalCommitLog(final int commitIntervalCommitLog) {\n        this.commitIntervalCommitLog = commitIntervalCommitLog;\n    }\n\n    public boolean isFastFailIfNoBufferInStorePool() {\n        return fastFailIfNoBufferInStorePool;\n    }\n\n    public void setFastFailIfNoBufferInStorePool(final boolean fastFailIfNoBufferInStorePool) {\n        this.fastFailIfNoBufferInStorePool = fastFailIfNoBufferInStorePool;\n    }\n\n    public boolean isUseReentrantLockWhenPutMessage() {\n        return useReentrantLockWhenPutMessage;\n    }\n\n    public void setUseReentrantLockWhenPutMessage(final boolean useReentrantLockWhenPutMessage) {\n        this.useReentrantLockWhenPutMessage = useReentrantLockWhenPutMessage;\n    }\n\n    public int getCommitCommitLogLeastPages() {\n        return commitCommitLogLeastPages;\n    }\n\n    public void setCommitCommitLogLeastPages(final int commitCommitLogLeastPages) {\n        this.commitCommitLogLeastPages = commitCommitLogLeastPages;\n    }\n\n    public int getCommitCommitLogThoroughInterval() {\n        return commitCommitLogThoroughInterval;\n    }\n\n    public void setCommitCommitLogThoroughInterval(final int commitCommitLogThoroughInterval) {\n        this.commitCommitLogThoroughInterval = commitCommitLogThoroughInterval;\n    }\n\n    public boolean isWakeCommitWhenPutMessage() {\n        return wakeCommitWhenPutMessage;\n    }\n\n    public void setWakeCommitWhenPutMessage(boolean wakeCommitWhenPutMessage) {\n        this.wakeCommitWhenPutMessage = wakeCommitWhenPutMessage;\n    }\n\n    public boolean isWakeFlushWhenPutMessage() {\n        return wakeFlushWhenPutMessage;\n    }\n\n    public void setWakeFlushWhenPutMessage(boolean wakeFlushWhenPutMessage) {\n        this.wakeFlushWhenPutMessage = wakeFlushWhenPutMessage;\n    }\n\n    public int getMapperFileSizeBatchConsumeQueue() {\n        return mapperFileSizeBatchConsumeQueue;\n    }\n\n    public void setMapperFileSizeBatchConsumeQueue(int mapperFileSizeBatchConsumeQueue) {\n        this.mapperFileSizeBatchConsumeQueue = mapperFileSizeBatchConsumeQueue;\n    }\n\n    public boolean isEnableCleanExpiredOffset() {\n        return enableCleanExpiredOffset;\n    }\n\n    public void setEnableCleanExpiredOffset(boolean enableCleanExpiredOffset) {\n        this.enableCleanExpiredOffset = enableCleanExpiredOffset;\n    }\n\n    public String getReadOnlyCommitLogStorePaths() {\n        return readOnlyCommitLogStorePaths;\n    }\n\n    public void setReadOnlyCommitLogStorePaths(String readOnlyCommitLogStorePaths) {\n        this.readOnlyCommitLogStorePaths = readOnlyCommitLogStorePaths;\n    }\n\n    public String getdLegerGroup() {\n        return dLegerGroup;\n    }\n\n    public void setdLegerGroup(String dLegerGroup) {\n        this.dLegerGroup = dLegerGroup;\n    }\n\n    public String getdLegerPeers() {\n        return dLegerPeers;\n    }\n\n    public void setdLegerPeers(String dLegerPeers) {\n        this.dLegerPeers = dLegerPeers;\n    }\n\n    public String getdLegerSelfId() {\n        return dLegerSelfId;\n    }\n\n    public void setdLegerSelfId(String dLegerSelfId) {\n        this.dLegerSelfId = dLegerSelfId;\n    }\n\n    public boolean isEnableDLegerCommitLog() {\n        return enableDLegerCommitLog;\n    }\n\n    public void setEnableDLegerCommitLog(boolean enableDLegerCommitLog) {\n        this.enableDLegerCommitLog = enableDLegerCommitLog;\n    }\n\n    public String getPreferredLeaderId() {\n        return preferredLeaderId;\n    }\n\n    public void setPreferredLeaderId(String preferredLeaderId) {\n        this.preferredLeaderId = preferredLeaderId;\n    }\n\n    public boolean isEnableBatchPush() {\n        return enableBatchPush;\n    }\n\n    public void setEnableBatchPush(boolean enableBatchPush) {\n        this.enableBatchPush = enableBatchPush;\n    }\n\n    public boolean isEnableScheduleMessageStats() {\n        return enableScheduleMessageStats;\n    }\n\n    public void setEnableScheduleMessageStats(boolean enableScheduleMessageStats) {\n        this.enableScheduleMessageStats = enableScheduleMessageStats;\n    }\n\n    public int getMaxAsyncPutMessageRequests() {\n        return maxAsyncPutMessageRequests;\n    }\n\n    public void setMaxAsyncPutMessageRequests(int maxAsyncPutMessageRequests) {\n        this.maxAsyncPutMessageRequests = maxAsyncPutMessageRequests;\n    }\n\n    public int getMaxRecoveryCommitlogFiles() {\n        return maxRecoveryCommitlogFiles;\n    }\n\n    public void setMaxRecoveryCommitlogFiles(final int maxRecoveryCommitlogFiles) {\n        this.maxRecoveryCommitlogFiles = maxRecoveryCommitlogFiles;\n    }\n\n    public boolean isDispatchFromSenderThread() {\n        return dispatchFromSenderThread;\n    }\n\n    public void setDispatchFromSenderThread(boolean dispatchFromSenderThread) {\n        this.dispatchFromSenderThread = dispatchFromSenderThread;\n    }\n\n    public int getDispatchCqThreads() {\n        return dispatchCqThreads;\n    }\n\n    public void setDispatchCqThreads(final int dispatchCqThreads) {\n        this.dispatchCqThreads = dispatchCqThreads;\n    }\n\n    public int getDispatchCqCacheNum() {\n        return dispatchCqCacheNum;\n    }\n\n    public void setDispatchCqCacheNum(final int dispatchCqCacheNum) {\n        this.dispatchCqCacheNum = dispatchCqCacheNum;\n    }\n\n    public boolean isEnableAsyncReput() {\n        return enableAsyncReput;\n    }\n\n    public void setEnableAsyncReput(final boolean enableAsyncReput) {\n        this.enableAsyncReput = enableAsyncReput;\n    }\n\n    public boolean isRecheckReputOffsetFromCq() {\n        return recheckReputOffsetFromCq;\n    }\n\n    public void setRecheckReputOffsetFromCq(final boolean recheckReputOffsetFromCq) {\n        this.recheckReputOffsetFromCq = recheckReputOffsetFromCq;\n    }\n\n    public long getCommitLogForceSwapMapInterval() {\n        return commitLogForceSwapMapInterval;\n    }\n\n    public void setCommitLogForceSwapMapInterval(long commitLogForceSwapMapInterval) {\n        this.commitLogForceSwapMapInterval = commitLogForceSwapMapInterval;\n    }\n\n    public int getCommitLogSwapMapReserveFileNum() {\n        return commitLogSwapMapReserveFileNum;\n    }\n\n    public void setCommitLogSwapMapReserveFileNum(int commitLogSwapMapReserveFileNum) {\n        this.commitLogSwapMapReserveFileNum = commitLogSwapMapReserveFileNum;\n    }\n\n    public long getLogicQueueForceSwapMapInterval() {\n        return logicQueueForceSwapMapInterval;\n    }\n\n    public void setLogicQueueForceSwapMapInterval(long logicQueueForceSwapMapInterval) {\n        this.logicQueueForceSwapMapInterval = logicQueueForceSwapMapInterval;\n    }\n\n    public int getLogicQueueSwapMapReserveFileNum() {\n        return logicQueueSwapMapReserveFileNum;\n    }\n\n    public void setLogicQueueSwapMapReserveFileNum(int logicQueueSwapMapReserveFileNum) {\n        this.logicQueueSwapMapReserveFileNum = logicQueueSwapMapReserveFileNum;\n    }\n\n    public long getCleanSwapedMapInterval() {\n        return cleanSwapedMapInterval;\n    }\n\n    public void setCleanSwapedMapInterval(long cleanSwapedMapInterval) {\n        this.cleanSwapedMapInterval = cleanSwapedMapInterval;\n    }\n\n    public long getCommitLogSwapMapInterval() {\n        return commitLogSwapMapInterval;\n    }\n\n    public void setCommitLogSwapMapInterval(long commitLogSwapMapInterval) {\n        this.commitLogSwapMapInterval = commitLogSwapMapInterval;\n    }\n\n    public long getLogicQueueSwapMapInterval() {\n        return logicQueueSwapMapInterval;\n    }\n\n    public void setLogicQueueSwapMapInterval(long logicQueueSwapMapInterval) {\n        this.logicQueueSwapMapInterval = logicQueueSwapMapInterval;\n    }\n\n    public int getMaxBatchDeleteFilesNum() {\n        return maxBatchDeleteFilesNum;\n    }\n\n    public void setMaxBatchDeleteFilesNum(int maxBatchDeleteFilesNum) {\n        this.maxBatchDeleteFilesNum = maxBatchDeleteFilesNum;\n    }\n\n    public boolean isSearchBcqByCacheEnable() {\n        return searchBcqByCacheEnable;\n    }\n\n    public void setSearchBcqByCacheEnable(boolean searchBcqByCacheEnable) {\n        this.searchBcqByCacheEnable = searchBcqByCacheEnable;\n    }\n\n    public int getDiskSpaceWarningLevelRatio() {\n        return diskSpaceWarningLevelRatio;\n    }\n\n    public void setDiskSpaceWarningLevelRatio(int diskSpaceWarningLevelRatio) {\n        this.diskSpaceWarningLevelRatio = diskSpaceWarningLevelRatio;\n    }\n\n    public int getDiskSpaceCleanForciblyRatio() {\n        return diskSpaceCleanForciblyRatio;\n    }\n\n    public void setDiskSpaceCleanForciblyRatio(int diskSpaceCleanForciblyRatio) {\n        this.diskSpaceCleanForciblyRatio = diskSpaceCleanForciblyRatio;\n    }\n\n    public boolean isMappedFileSwapEnable() {\n        return mappedFileSwapEnable;\n    }\n\n    public void setMappedFileSwapEnable(boolean mappedFileSwapEnable) {\n        this.mappedFileSwapEnable = mappedFileSwapEnable;\n    }\n\n    public int getPullBatchMaxMessageCount() {\n        return pullBatchMaxMessageCount;\n    }\n\n    public void setPullBatchMaxMessageCount(int pullBatchMaxMessageCount) {\n        this.pullBatchMaxMessageCount = pullBatchMaxMessageCount;\n    }\n\n    public int getDeleteFileBatchMax() {\n        return deleteFileBatchMax;\n    }\n\n    public void setDeleteFileBatchMax(int deleteFileBatchMax) {\n        this.deleteFileBatchMax = deleteFileBatchMax;\n    }\n\n    public int getTotalReplicas() {\n        return totalReplicas;\n    }\n\n    public void setTotalReplicas(int totalReplicas) {\n        this.totalReplicas = totalReplicas;\n    }\n\n    public int getInSyncReplicas() {\n        return inSyncReplicas;\n    }\n\n    public void setInSyncReplicas(int inSyncReplicas) {\n        this.inSyncReplicas = inSyncReplicas;\n    }\n\n    public int getMinInSyncReplicas() {\n        return minInSyncReplicas;\n    }\n\n    public void setMinInSyncReplicas(int minInSyncReplicas) {\n        this.minInSyncReplicas = minInSyncReplicas;\n    }\n\n    public boolean isAllAckInSyncStateSet() {\n        return allAckInSyncStateSet;\n    }\n\n    public void setAllAckInSyncStateSet(boolean allAckInSyncStateSet) {\n        this.allAckInSyncStateSet = allAckInSyncStateSet;\n    }\n\n    public boolean isEnableAutoInSyncReplicas() {\n        return enableAutoInSyncReplicas;\n    }\n\n    public void setEnableAutoInSyncReplicas(boolean enableAutoInSyncReplicas) {\n        this.enableAutoInSyncReplicas = enableAutoInSyncReplicas;\n    }\n\n    public boolean isHaFlowControlEnable() {\n        return haFlowControlEnable;\n    }\n\n    public void setHaFlowControlEnable(boolean haFlowControlEnable) {\n        this.haFlowControlEnable = haFlowControlEnable;\n    }\n\n    public long getMaxHaTransferByteInSecond() {\n        return maxHaTransferByteInSecond;\n    }\n\n    public void setMaxHaTransferByteInSecond(long maxHaTransferByteInSecond) {\n        this.maxHaTransferByteInSecond = maxHaTransferByteInSecond;\n    }\n\n    public long getHaMaxTimeSlaveNotCatchup() {\n        return haMaxTimeSlaveNotCatchup;\n    }\n\n    public void setHaMaxTimeSlaveNotCatchup(long haMaxTimeSlaveNotCatchup) {\n        this.haMaxTimeSlaveNotCatchup = haMaxTimeSlaveNotCatchup;\n    }\n\n    public boolean isSyncMasterFlushOffsetWhenStartup() {\n        return syncMasterFlushOffsetWhenStartup;\n    }\n\n    public void setSyncMasterFlushOffsetWhenStartup(boolean syncMasterFlushOffsetWhenStartup) {\n        this.syncMasterFlushOffsetWhenStartup = syncMasterFlushOffsetWhenStartup;\n    }\n\n    public long getMaxChecksumRange() {\n        return maxChecksumRange;\n    }\n\n    public void setMaxChecksumRange(long maxChecksumRange) {\n        this.maxChecksumRange = maxChecksumRange;\n    }\n\n    public int getReplicasPerDiskPartition() {\n        return replicasPerDiskPartition;\n    }\n\n    public void setReplicasPerDiskPartition(int replicasPerDiskPartition) {\n        this.replicasPerDiskPartition = replicasPerDiskPartition;\n    }\n\n    public double getLogicalDiskSpaceCleanForciblyThreshold() {\n        return logicalDiskSpaceCleanForciblyThreshold;\n    }\n\n    public void setLogicalDiskSpaceCleanForciblyThreshold(double logicalDiskSpaceCleanForciblyThreshold) {\n        this.logicalDiskSpaceCleanForciblyThreshold = logicalDiskSpaceCleanForciblyThreshold;\n    }\n\n    public int getDisappearTimeAfterStart() {\n        return disappearTimeAfterStart;\n    }\n\n    public void setDisappearTimeAfterStart(int disappearTimeAfterStart) {\n        this.disappearTimeAfterStart = disappearTimeAfterStart;\n    }\n\n    public long getMaxSlaveResendLength() {\n        return maxSlaveResendLength;\n    }\n\n    public void setMaxSlaveResendLength(long maxSlaveResendLength) {\n        this.maxSlaveResendLength = maxSlaveResendLength;\n    }\n\n    public boolean isSyncFromLastFile() {\n        return syncFromLastFile;\n    }\n\n    public void setSyncFromLastFile(boolean syncFromLastFile) {\n        this.syncFromLastFile = syncFromLastFile;\n    }\n\n    public boolean isEnableLmq() {\n        return enableLmq;\n    }\n\n    public void setEnableLmq(boolean enableLmq) {\n        this.enableLmq = enableLmq;\n    }\n\n    public boolean isEnableMultiDispatch() {\n        return enableMultiDispatch;\n    }\n\n    public void setEnableMultiDispatch(boolean enableMultiDispatch) {\n        this.enableMultiDispatch = enableMultiDispatch;\n    }\n\n    public int getMaxLmqConsumeQueueNum() {\n        return maxLmqConsumeQueueNum;\n    }\n\n    public void setMaxLmqConsumeQueueNum(int maxLmqConsumeQueueNum) {\n        this.maxLmqConsumeQueueNum = maxLmqConsumeQueueNum;\n    }\n\n    public boolean isEnableLmqQuota() {\n        return enableLmqQuota;\n    }\n\n    public void setEnableLmqQuota(boolean enableLmqQuota) {\n        this.enableLmqQuota = enableLmqQuota;\n    }\n\n    public boolean isEnableScheduleAsyncDeliver() {\n        return enableScheduleAsyncDeliver;\n    }\n\n    public void setEnableScheduleAsyncDeliver(boolean enableScheduleAsyncDeliver) {\n        this.enableScheduleAsyncDeliver = enableScheduleAsyncDeliver;\n    }\n\n    public int getScheduleAsyncDeliverMaxPendingLimit() {\n        return scheduleAsyncDeliverMaxPendingLimit;\n    }\n\n    public void setScheduleAsyncDeliverMaxPendingLimit(int scheduleAsyncDeliverMaxPendingLimit) {\n        this.scheduleAsyncDeliverMaxPendingLimit = scheduleAsyncDeliverMaxPendingLimit;\n    }\n\n    public int getScheduleAsyncDeliverMaxResendNum2Blocked() {\n        return scheduleAsyncDeliverMaxResendNum2Blocked;\n    }\n\n    public void setScheduleAsyncDeliverMaxResendNum2Blocked(int scheduleAsyncDeliverMaxResendNum2Blocked) {\n        this.scheduleAsyncDeliverMaxResendNum2Blocked = scheduleAsyncDeliverMaxResendNum2Blocked;\n    }\n\n    public boolean isAsyncLearner() {\n        return asyncLearner;\n    }\n\n    public void setAsyncLearner(boolean asyncLearner) {\n        this.asyncLearner = asyncLearner;\n    }\n\n    public int getMappedFileSizeTimerLog() {\n        return mappedFileSizeTimerLog;\n    }\n\n    public void setMappedFileSizeTimerLog(final int mappedFileSizeTimerLog) {\n        this.mappedFileSizeTimerLog = mappedFileSizeTimerLog;\n    }\n\n    public int getTimerPrecisionMs() {\n        return timerPrecisionMs;\n    }\n\n    public void setTimerPrecisionMs(int timerPrecisionMs) {\n        int[] candidates = {100, 200, 500, 1000};\n        for (int i = 1; i < candidates.length; i++) {\n            if (timerPrecisionMs < candidates[i]) {\n                this.timerPrecisionMs = candidates[i - 1];\n                return;\n            }\n        }\n        this.timerPrecisionMs = candidates[candidates.length - 1];\n    }\n\n    public int getTimerRollWindowSlot() {\n        return timerRollWindowSlot;\n    }\n\n    public int getTimerGetMessageThreadNum() {\n        return timerGetMessageThreadNum;\n    }\n\n    public void setTimerGetMessageThreadNum(int timerGetMessageThreadNum) {\n        this.timerGetMessageThreadNum = timerGetMessageThreadNum;\n    }\n\n    public int getTimerPutMessageThreadNum() {\n        return timerPutMessageThreadNum;\n    }\n\n    public void setTimerPutMessageThreadNum(int timerPutMessageThreadNum) {\n        this.timerPutMessageThreadNum = timerPutMessageThreadNum;\n    }\n\n    public boolean isTimerEnableDisruptor() {\n        return timerEnableDisruptor;\n    }\n\n    public boolean isTimerEnableCheckMetrics() {\n        return timerEnableCheckMetrics;\n    }\n\n    public void setTimerEnableCheckMetrics(boolean timerEnableCheckMetrics) {\n        this.timerEnableCheckMetrics = timerEnableCheckMetrics;\n    }\n\n    public boolean isTimerStopEnqueue() {\n        return timerStopEnqueue;\n    }\n\n    public void setTimerStopEnqueue(boolean timerStopEnqueue) {\n        this.timerStopEnqueue = timerStopEnqueue;\n    }\n\n    public String getTimerCheckMetricsWhen() {\n        return timerCheckMetricsWhen;\n    }\n\n    public boolean isTimerSkipUnknownError() {\n        return timerSkipUnknownError;\n    }\n\n    public void setTimerSkipUnknownError(boolean timerSkipUnknownError) {\n        this.timerSkipUnknownError = timerSkipUnknownError;\n    }\n\n    public boolean isTimerEnableRetryUntilSuccess() {\n        return timerEnableRetryUntilSuccess;\n    }\n\n    public void setTimerEnableRetryUntilSuccess(boolean timerEnableRetryUntilSuccess) {\n        this.timerEnableRetryUntilSuccess = timerEnableRetryUntilSuccess;\n    }\n\n    public boolean isTimerWarmEnable() {\n        return timerWarmEnable;\n    }\n\n    public boolean isTimerWheelSnapshotFlush() {\n        return timerWheelSnapshotFlush;\n    }\n\n    public void setTimerWheelSnapshotFlush(boolean timerWheelSnapshotFlush) {\n        this.timerWheelSnapshotFlush = timerWheelSnapshotFlush;\n    }\n\n    public boolean isTimerWheelEnable() {\n        return timerWheelEnable;\n    }\n\n    public void setTimerWheelEnable(boolean timerWheelEnable) {\n        this.timerWheelEnable = timerWheelEnable;\n    }\n\n    public boolean isTimerStopDequeue() {\n        return timerStopDequeue;\n    }\n\n    public int getTimerMetricSmallThreshold() {\n        return timerMetricSmallThreshold;\n    }\n\n    public void setTimerMetricSmallThreshold(int timerMetricSmallThreshold) {\n        this.timerMetricSmallThreshold = timerMetricSmallThreshold;\n    }\n\n    public int getTimerCongestNumEachSlot() {\n        return timerCongestNumEachSlot;\n    }\n\n    public void setTimerCongestNumEachSlot(int timerCongestNumEachSlot) {\n        // In order to get this value from messageStoreConfig properties file created before v4.4.1.\n        this.timerCongestNumEachSlot = timerCongestNumEachSlot;\n    }\n\n    public int getTimerFlushIntervalMs() {\n        return timerFlushIntervalMs;\n    }\n\n    public void setTimerFlushIntervalMs(final int timerFlushIntervalMs) {\n        this.timerFlushIntervalMs = timerFlushIntervalMs;\n    }\n\n    public void setTimerRollWindowSlot(final int timerRollWindowSlot) {\n        this.timerRollWindowSlot = timerRollWindowSlot;\n    }\n\n    public int getTimerProgressLogIntervalMs() {\n        return timerProgressLogIntervalMs;\n    }\n\n    public int getTimerWheelSnapshotIntervalMs() {\n        return timerWheelSnapshotIntervalMs;\n    }\n\n    public void setTimerWheelSnapshotIntervalMs(int timerWheelSnapshotIntervalMs) {\n        this.timerWheelSnapshotIntervalMs = timerWheelSnapshotIntervalMs;\n    }\n\n    public void setTimerProgressLogIntervalMs(final int timerProgressLogIntervalMs) {\n        this.timerProgressLogIntervalMs = timerProgressLogIntervalMs;\n    }\n\n    public boolean isTimerInterceptDelayLevel() {\n        return timerInterceptDelayLevel;\n    }\n\n    public void setTimerInterceptDelayLevel(boolean timerInterceptDelayLevel) {\n        this.timerInterceptDelayLevel = timerInterceptDelayLevel;\n    }\n\n    public int getTimerMaxDelaySec() {\n        return timerMaxDelaySec;\n    }\n\n    public void setTimerMaxDelaySec(final int timerMaxDelaySec) {\n        this.timerMaxDelaySec = timerMaxDelaySec;\n    }\n\n    public int getMaxConsumeQueueScan() {\n        return maxConsumeQueueScan;\n    }\n\n    public void setMaxConsumeQueueScan(int maxConsumeQueueScan) {\n        this.maxConsumeQueueScan = maxConsumeQueueScan;\n    }\n\n    public int getSampleCountThreshold() {\n        return sampleCountThreshold;\n    }\n\n    public void setSampleCountThreshold(int sampleCountThreshold) {\n        this.sampleCountThreshold = sampleCountThreshold;\n    }\n\n    public boolean isColdDataFlowControlEnable() {\n        return coldDataFlowControlEnable;\n    }\n\n    public void setColdDataFlowControlEnable(boolean coldDataFlowControlEnable) {\n        this.coldDataFlowControlEnable = coldDataFlowControlEnable;\n    }\n\n    public boolean isColdDataScanEnable() {\n        return coldDataScanEnable;\n    }\n\n    public void setColdDataScanEnable(boolean coldDataScanEnable) {\n        this.coldDataScanEnable = coldDataScanEnable;\n    }\n\n    public int getTimerColdDataCheckIntervalMs() {\n        return timerColdDataCheckIntervalMs;\n    }\n\n    public void setTimerColdDataCheckIntervalMs(int timerColdDataCheckIntervalMs) {\n        this.timerColdDataCheckIntervalMs = timerColdDataCheckIntervalMs;\n    }\n\n    public int getSampleSteps() {\n        return sampleSteps;\n    }\n\n    public void setSampleSteps(int sampleSteps) {\n        this.sampleSteps = sampleSteps;\n    }\n\n    public int getAccessMessageInMemoryHotRatio() {\n        return accessMessageInMemoryHotRatio;\n    }\n\n    public void setAccessMessageInMemoryHotRatio(int accessMessageInMemoryHotRatio) {\n        this.accessMessageInMemoryHotRatio = accessMessageInMemoryHotRatio;\n    }\n\n    public boolean isDataReadAheadEnable() {\n        return dataReadAheadEnable;\n    }\n\n    public void setDataReadAheadEnable(boolean dataReadAheadEnable) {\n        this.dataReadAheadEnable = dataReadAheadEnable;\n    }\n\n    public boolean isEnableBuildConsumeQueueConcurrently() {\n        return enableBuildConsumeQueueConcurrently;\n    }\n\n    public void setEnableBuildConsumeQueueConcurrently(boolean enableBuildConsumeQueueConcurrently) {\n        this.enableBuildConsumeQueueConcurrently = enableBuildConsumeQueueConcurrently;\n    }\n\n    public int getBatchDispatchRequestThreadPoolNums() {\n        return batchDispatchRequestThreadPoolNums;\n    }\n\n    public void setBatchDispatchRequestThreadPoolNums(int batchDispatchRequestThreadPoolNums) {\n        this.batchDispatchRequestThreadPoolNums = batchDispatchRequestThreadPoolNums;\n    }\n\n    public boolean isRealTimePersistRocksDBConfig() {\n        return realTimePersistRocksDBConfig;\n    }\n\n    public void setRealTimePersistRocksDBConfig(boolean realTimePersistRocksDBConfig) {\n        this.realTimePersistRocksDBConfig = realTimePersistRocksDBConfig;\n    }\n\n    public long getStatRocksDBCQIntervalSec() {\n        return statRocksDBCQIntervalSec;\n    }\n\n    public void setStatRocksDBCQIntervalSec(long statRocksDBCQIntervalSec) {\n        this.statRocksDBCQIntervalSec = statRocksDBCQIntervalSec;\n    }\n\n    public long getCleanRocksDBDirtyCQIntervalMin() {\n        return cleanRocksDBDirtyCQIntervalMin;\n    }\n\n    public void setCleanRocksDBDirtyCQIntervalMin(long cleanRocksDBDirtyCQIntervalMin) {\n        this.cleanRocksDBDirtyCQIntervalMin = cleanRocksDBDirtyCQIntervalMin;\n    }\n\n    public long getMemTableFlushIntervalMs() {\n        return memTableFlushIntervalMs;\n    }\n\n    public void setMemTableFlushIntervalMs(long memTableFlushIntervalMs) {\n        this.memTableFlushIntervalMs = memTableFlushIntervalMs;\n    }\n\n    public boolean isEnableRocksDBLog() {\n        return enableRocksDBLog;\n    }\n\n    public void setEnableRocksDBLog(boolean enableRocksDBLog) {\n        this.enableRocksDBLog = enableRocksDBLog;\n    }\n\n    public int getTopicQueueLockNum() {\n        return topicQueueLockNum;\n    }\n\n    public void setTopicQueueLockNum(int topicQueueLockNum) {\n        this.topicQueueLockNum = topicQueueLockNum;\n    }\n\n    public boolean isReadUnCommitted() {\n        return readUnCommitted;\n    }\n\n    public void setReadUnCommitted(boolean readUnCommitted) {\n        this.readUnCommitted = readUnCommitted;\n    }\n\n    public boolean isPutConsumeQueueDataByFileChannel() {\n        return putConsumeQueueDataByFileChannel;\n    }\n\n    public void setPutConsumeQueueDataByFileChannel(boolean putConsumeQueueDataByFileChannel) {\n        this.putConsumeQueueDataByFileChannel = putConsumeQueueDataByFileChannel;\n    }\n\n    public String getBottomMostCompressionTypeForConsumeQueueStore() {\n        return bottomMostCompressionTypeForConsumeQueueStore;\n    }\n\n    public void setBottomMostCompressionTypeForConsumeQueueStore(String bottomMostCompressionTypeForConsumeQueueStore) {\n        this.bottomMostCompressionTypeForConsumeQueueStore = bottomMostCompressionTypeForConsumeQueueStore;\n    }\n\n    public int getRocksdbFlushWalFrequency() {\n        return rocksdbFlushWalFrequency;\n    }\n\n    public void setRocksdbFlushWalFrequency(int rocksdbFlushWalFrequency) {\n        this.rocksdbFlushWalFrequency = rocksdbFlushWalFrequency;\n    }\n\n    public long getRocksdbWalFileRollingThreshold() {\n        return rocksdbWalFileRollingThreshold;\n    }\n\n    public void setRocksdbWalFileRollingThreshold(long rocksdbWalFileRollingThreshold) {\n        this.rocksdbWalFileRollingThreshold = rocksdbWalFileRollingThreshold;\n    }\n\n    public int getSpinLockCollisionRetreatOptimalDegree() {\n        return spinLockCollisionRetreatOptimalDegree;\n    }\n\n    public void setSpinLockCollisionRetreatOptimalDegree(int spinLockCollisionRetreatOptimalDegree) {\n        this.spinLockCollisionRetreatOptimalDegree = spinLockCollisionRetreatOptimalDegree;\n    }\n\n    public void setUseABSLock(boolean useABSLock) {\n        this.useABSLock = useABSLock;\n    }\n\n    public boolean getUseABSLock() {\n        return useABSLock;\n    }\n\n    public String getCombineCQPreferCQType() {\n        return combineCQPreferCQType;\n    }\n\n    public void setCombineCQPreferCQType(String combineCQPreferCQType) {\n        this.combineCQPreferCQType = combineCQPreferCQType;\n    }\n\n    public String getCombineCQLoadingCQTypes() {\n        return combineCQLoadingCQTypes;\n    }\n\n    public void setCombineCQLoadingCQTypes(String combineCQLoadingCQTypes) {\n        this.combineCQLoadingCQTypes = combineCQLoadingCQTypes;\n    }\n\n    public String getCombineAssignOffsetCQType() {\n        return combineAssignOffsetCQType;\n    }\n\n    public void setCombineAssignOffsetCQType(String combineAssignOffsetCQType) {\n        this.combineAssignOffsetCQType = combineAssignOffsetCQType;\n    }\n\n    public boolean isCombineCQEnableCheckSelf() {\n        return combineCQEnableCheckSelf;\n    }\n\n    public void setCombineCQEnableCheckSelf(boolean combineCQEnableCheckSelf) {\n        this.combineCQEnableCheckSelf = combineCQEnableCheckSelf;\n    }\n\n    public int getCombineCQMaxExtraSearchCommitLogFiles() {\n        return combineCQMaxExtraSearchCommitLogFiles;\n    }\n\n    public void setCombineCQMaxExtraSearchCommitLogFiles(int combineCQMaxExtraSearchCommitLogFiles) {\n        this.combineCQMaxExtraSearchCommitLogFiles = combineCQMaxExtraSearchCommitLogFiles;\n    }\n\n    public boolean isEnableLogConsumeQueueRepeatedlyBuildWhenRecover() {\n        return enableLogConsumeQueueRepeatedlyBuildWhenRecover;\n    }\n\n    public void setEnableLogConsumeQueueRepeatedlyBuildWhenRecover(\n        boolean enableLogConsumeQueueRepeatedlyBuildWhenRecover) {\n        this.enableLogConsumeQueueRepeatedlyBuildWhenRecover = enableLogConsumeQueueRepeatedlyBuildWhenRecover;\n    }\n\n    public boolean isEnableAcceleratedRecovery() {\n        return enableAcceleratedRecovery;\n    }\n\n    public void setEnableAcceleratedRecovery(boolean enableAcceleratedRecovery) {\n        this.enableAcceleratedRecovery = enableAcceleratedRecovery;\n    }\n\n    public boolean isEnableRunningFlagsInFlush() {\n        return enableRunningFlagsInFlush;\n    }\n\n    public void setEnableRunningFlagsInFlush(boolean enableRunningFlagsInFlush) {\n        this.enableRunningFlagsInFlush = enableRunningFlagsInFlush;\n    }\n\n    public boolean isTimerRocksDBEnable() {\n        return timerRocksDBEnable;\n    }\n\n    public void setTimerRocksDBEnable(boolean timerRocksDBEnable) {\n        this.timerRocksDBEnable = timerRocksDBEnable;\n    }\n\n    public double getTimerRocksDBRollMaxTps() {\n        return timerRocksDBRollMaxTps;\n    }\n\n    public void setTimerRocksDBRollMaxTps(double timerRocksDBRollMaxTps) {\n        this.timerRocksDBRollMaxTps = timerRocksDBRollMaxTps;\n    }\n\n    public double getTimerRocksDBTimeExpiredMaxTps() {\n        return timerRocksDBTimeExpiredMaxTps;\n    }\n\n    public void setTimerRocksDBTimeExpiredMaxTps(double timerRocksDBTimeExpiredMaxTps) {\n        this.timerRocksDBTimeExpiredMaxTps = timerRocksDBTimeExpiredMaxTps;\n    }\n\n    public boolean isTransRocksDBEnable() {\n        return transRocksDBEnable;\n    }\n\n    public void setTransRocksDBEnable(boolean transRocksDBEnable) {\n        this.transRocksDBEnable = transRocksDBEnable;\n    }\n\n    public boolean isIndexRocksDBEnable() {\n        return indexRocksDBEnable;\n    }\n\n    public void setIndexRocksDBEnable(boolean indexRocksDBEnable) {\n        this.indexRocksDBEnable = indexRocksDBEnable;\n    }\n\n    public int getMaxRocksDBIndexQueryDays() {\n        return maxRocksDBIndexQueryDays;\n    }\n\n    public void setMaxRocksDBIndexQueryDays(int maxRocksDBIndexQueryDays) {\n        this.maxRocksDBIndexQueryDays = maxRocksDBIndexQueryDays;\n    }\n\n    public boolean isTimerRocksDBStopScan() {\n        return timerRocksDBStopScan;\n    }\n\n    public void setTimerRocksDBStopScan(boolean timerRocksDBStopScan) {\n        this.timerRocksDBStopScan = timerRocksDBStopScan;\n    }\n\n    public long getTimerRocksDBPrecisionMs() {\n        return timerRocksDBPrecisionMs;\n    }\n\n    public void setTimerRocksDBPrecisionMs(long timerRocksDBPrecisionMs) {\n        this.timerRocksDBPrecisionMs = timerRocksDBPrecisionMs;\n    }\n\n    public boolean isIndexFileWriteEnable() {\n        return indexFileWriteEnable;\n    }\n\n    public void setIndexFileWriteEnable(boolean indexFileWriteEnable) {\n        this.indexFileWriteEnable = indexFileWriteEnable;\n    }\n\n    public boolean isIndexFileReadEnable() {\n        return indexFileReadEnable;\n    }\n\n    public void setIndexFileReadEnable(boolean indexFileReadEnable) {\n        this.indexFileReadEnable = indexFileReadEnable;\n    }\n\n    public boolean isTransWriteOriginTransHalfEnable() {\n        return transWriteOriginTransHalfEnable;\n    }\n\n    public void setTransWriteOriginTransHalfEnable(boolean transWriteOriginTransHalfEnable) {\n        this.transWriteOriginTransHalfEnable = transWriteOriginTransHalfEnable;\n    }\n\n    public boolean isTimerRecallToTimeWheelEnable() {\n        return timerRecallToTimeWheelEnable;\n    }\n\n    public void setTimerRecallToTimeWheelEnable(boolean timerRecallToTimeWheelEnable) {\n        this.timerRecallToTimeWheelEnable = timerRecallToTimeWheelEnable;\n    }\n\n    public boolean isTimerRecallToTimelineEnable() {\n        return timerRecallToTimelineEnable;\n    }\n\n    public void setTimerRecallToTimelineEnable(boolean timerRecallToTimelineEnable) {\n        this.timerRecallToTimelineEnable = timerRecallToTimelineEnable;\n    }\n\n    public void setTimerReputServiceCorePoolSize(int timerReputServiceCorePoolSize) {\n        this.timerReputServiceCorePoolSize = timerReputServiceCorePoolSize;\n    }\n\n    public int getTimerReputServiceCorePoolSize() {\n        return timerReputServiceCorePoolSize;\n    }\n\n    public void setTimerReputServiceMaxPoolSize(int timerReputServiceMaxPoolSize) {\n        this.timerReputServiceMaxPoolSize = timerReputServiceMaxPoolSize;\n    }\n\n    public int getTimerReputServiceMaxPoolSize() {\n        return timerReputServiceMaxPoolSize;\n    }\n\n    public void setTimerReputServiceQueueCapacity(int timerReputServiceQueueCapacity) {\n        this.timerReputServiceQueueCapacity = timerReputServiceQueueCapacity;\n    }\n\n    public int getTimerReputServiceQueueCapacity() {\n        return timerReputServiceQueueCapacity;\n    }\n\n    public int getTimerRocksDBRollIntervalHours() {\n        return timerRocksDBRollIntervalHours;\n    }\n\n    public void setTimerRocksDBRollIntervalHours(int timerRocksDBRollIntervalHours) {\n        this.timerRocksDBRollIntervalHours = timerRocksDBRollIntervalHours;\n    }\n\n    public int getTimerRocksDBRollRangeHours() {\n        return timerRocksDBRollRangeHours;\n    }\n\n    public void setTimerRocksDBRollRangeHours(int timerRocksDBRollRangeHours) {\n        this.timerRocksDBRollRangeHours = timerRocksDBRollRangeHours;\n    }\n\n    public int getCommitLogRecoverMaxNum() {\n        return commitLogRecoverMaxNum;\n    }\n\n    public void setCommitLogRecoverMaxNum(int commitLogRecoverMaxNum) {\n        this.commitLogRecoverMaxNum = commitLogRecoverMaxNum;\n    }\n\n    public int getSharedByteBufferNum() {\n        return sharedByteBufferNum;\n    }\n\n    public void setSharedByteBufferNum(int sharedByteBufferNum) {\n        this.sharedByteBufferNum = sharedByteBufferNum;\n    }\n\n    public boolean isAppendTopicForTimerDeleteKey() {\n        return appendTopicForTimerDeleteKey;\n    }\n\n    public void setAppendTopicForTimerDeleteKey(boolean appendTopicForTimerDeleteKey) {\n        this.appendTopicForTimerDeleteKey = appendTopicForTimerDeleteKey;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/config/StorePathConfigHelper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.config;\n\nimport java.io.File;\n\npublic class StorePathConfigHelper {\n\n    public static String getStorePathConsumeQueue(final String rootDir) {\n        return rootDir + File.separator + \"consumequeue\";\n    }\n\n    public static String getStorePathConsumeQueueExt(final String rootDir) {\n        return rootDir + File.separator + \"consumequeue_ext\";\n    }\n    public static String getStorePathBatchConsumeQueue(final String rootDir) {\n        return rootDir + File.separator + \"batchconsumequeue\";\n    }\n\n    public static String getStorePathIndex(final String rootDir) {\n        return rootDir + File.separator + \"index\";\n    }\n\n    public static String getStoreCheckpoint(final String rootDir) {\n        return rootDir + File.separator + \"checkpoint\";\n    }\n\n    public static String getAbortFile(final String rootDir) {\n        return rootDir + File.separator + \"abort\";\n    }\n\n    public static String getLockFile(final String rootDir) {\n        return rootDir + File.separator + \"lock\";\n    }\n\n    public static String getDelayOffsetStorePath(final String rootDir) {\n        return rootDir + File.separator + \"config\" + File.separator + \"delayOffset.json\";\n    }\n\n    public static String getTranStateTableStorePath(final String rootDir) {\n        return rootDir + File.separator + \"transaction\" + File.separator + \"statetable\";\n    }\n\n    public static String getTranRedoLogStorePath(final String rootDir) {\n        return rootDir + File.separator + \"transaction\" + File.separator + \"redolog\";\n    }\n\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.dledger;\n\nimport io.openmessaging.storage.dledger.AppendFuture;\nimport io.openmessaging.storage.dledger.BatchAppendFuture;\nimport io.openmessaging.storage.dledger.DLedgerConfig;\nimport io.openmessaging.storage.dledger.DLedgerServer;\nimport io.openmessaging.storage.dledger.entry.DLedgerEntry;\nimport io.openmessaging.storage.dledger.protocol.AppendEntryRequest;\nimport io.openmessaging.storage.dledger.protocol.AppendEntryResponse;\nimport io.openmessaging.storage.dledger.protocol.BatchAppendEntryRequest;\nimport io.openmessaging.storage.dledger.protocol.DLedgerResponseCode;\nimport io.openmessaging.storage.dledger.store.file.DLedgerMmapFileStore;\nimport io.openmessaging.storage.dledger.store.file.MmapFile;\nimport io.openmessaging.storage.dledger.store.file.MmapFileList;\nimport io.openmessaging.storage.dledger.store.file.SelectMmapBufferResult;\nimport io.openmessaging.storage.dledger.utils.DLedgerUtils;\nimport java.net.Inet6Address;\nimport java.net.InetSocketAddress;\nimport java.nio.ByteBuffer;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExtBatch;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.message.MessageVersion;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.store.AppendMessageResult;\nimport org.apache.rocketmq.store.AppendMessageStatus;\nimport org.apache.rocketmq.store.CommitLog;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.MessageExtEncoder;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.StoreStatsService;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.logfile.MappedFile;\nimport org.rocksdb.RocksDBException;\n\n/**\n * Store all metadata downtime for recovery, data protection reliability\n */\npublic class DLedgerCommitLog extends CommitLog {\n\n    static {\n        System.setProperty(\"dLedger.multiPath.Splitter\", MessageStoreConfig.MULTI_PATH_SPLITTER);\n    }\n\n    private final DLedgerServer dLedgerServer;\n    private final DLedgerConfig dLedgerConfig;\n    private final DLedgerMmapFileStore dLedgerFileStore;\n    private final MmapFileList dLedgerFileList;\n\n    //The id identifies the broker role, 0 means master, others means slave\n    private final int id;\n\n    private final MessageSerializer messageSerializer;\n    private volatile long beginTimeInDledgerLock = 0;\n\n    //This offset separate the old commitlog from dledger commitlog\n    private long dividedCommitlogOffset = -1;\n\n    private boolean isInrecoveringOldCommitlog = false;\n\n    private final StringBuilder msgIdBuilder = new StringBuilder();\n\n    public DLedgerCommitLog(final DefaultMessageStore defaultMessageStore) {\n        super(defaultMessageStore);\n        dLedgerConfig = new DLedgerConfig();\n        dLedgerConfig.setEnableDiskForceClean(defaultMessageStore.getMessageStoreConfig().isCleanFileForciblyEnable());\n        dLedgerConfig.setStoreType(DLedgerConfig.FILE);\n        dLedgerConfig.setSelfId(defaultMessageStore.getMessageStoreConfig().getdLegerSelfId());\n        dLedgerConfig.setGroup(defaultMessageStore.getMessageStoreConfig().getdLegerGroup());\n        dLedgerConfig.setPeers(defaultMessageStore.getMessageStoreConfig().getdLegerPeers());\n        dLedgerConfig.setStoreBaseDir(defaultMessageStore.getMessageStoreConfig().getStorePathRootDir());\n        dLedgerConfig.setDataStorePath(defaultMessageStore.getMessageStoreConfig().getStorePathDLedgerCommitLog());\n        dLedgerConfig.setReadOnlyDataStoreDirs(defaultMessageStore.getMessageStoreConfig().getReadOnlyCommitLogStorePaths());\n        dLedgerConfig.setMappedFileSizeForEntryData(defaultMessageStore.getMessageStoreConfig().getMappedFileSizeCommitLog());\n        dLedgerConfig.setDeleteWhen(defaultMessageStore.getMessageStoreConfig().getDeleteWhen());\n        dLedgerConfig.setFileReservedHours(defaultMessageStore.getMessageStoreConfig().getFileReservedTime() + 1);\n        dLedgerConfig.setPreferredLeaderId(defaultMessageStore.getMessageStoreConfig().getPreferredLeaderId());\n        dLedgerConfig.setEnableBatchPush(defaultMessageStore.getMessageStoreConfig().isEnableBatchPush());\n        dLedgerConfig.setDiskSpaceRatioToCheckExpired(defaultMessageStore.getMessageStoreConfig().getDiskMaxUsedSpaceRatio() / 100f);\n\n        id = Integer.parseInt(dLedgerConfig.getSelfId().substring(1)) + 1;\n        dLedgerServer = new DLedgerServer(dLedgerConfig);\n        dLedgerFileStore = (DLedgerMmapFileStore) dLedgerServer.getdLedgerStore();\n        DLedgerMmapFileStore.AppendHook appendHook = (entry, buffer, bodyOffset) -> {\n            assert bodyOffset == DLedgerEntry.BODY_OFFSET;\n            buffer.position(buffer.position() + bodyOffset + MessageDecoder.PHY_POS_POSITION);\n            buffer.putLong(entry.getPos() + bodyOffset);\n        };\n        dLedgerFileStore.addAppendHook(appendHook);\n        dLedgerFileList = dLedgerFileStore.getDataFileList();\n        this.messageSerializer = new MessageSerializer(defaultMessageStore.getMessageStoreConfig().getMaxMessageSize());\n\n    }\n\n    @Override\n    public boolean load() {\n        return super.load();\n    }\n\n    private void refreshConfig() {\n        dLedgerConfig.setEnableDiskForceClean(defaultMessageStore.getMessageStoreConfig().isCleanFileForciblyEnable());\n        dLedgerConfig.setDeleteWhen(defaultMessageStore.getMessageStoreConfig().getDeleteWhen());\n        dLedgerConfig.setFileReservedHours(defaultMessageStore.getMessageStoreConfig().getFileReservedTime() + 1);\n    }\n\n    private void disableDeleteDledger() {\n        dLedgerConfig.setEnableDiskForceClean(false);\n        dLedgerConfig.setFileReservedHours(24 * 365 * 10);\n    }\n\n    @Override\n    public void start() {\n        dLedgerServer.startup();\n    }\n\n    @Override\n    public void shutdown() {\n        dLedgerServer.shutdown();\n    }\n\n    @Override\n    public long flush() {\n        dLedgerFileStore.flush();\n        return dLedgerFileList.getFlushedWhere();\n    }\n\n    @Override\n    public long getMaxOffset() {\n        if (dLedgerFileStore.getCommittedPos() > 0) {\n            return dLedgerFileStore.getCommittedPos();\n        }\n        if (dLedgerFileList.getMinOffset() > 0) {\n            return dLedgerFileList.getMinOffset();\n        }\n        return 0;\n    }\n\n    @Override\n    public long getMinOffset() {\n        if (!mappedFileQueue.getMappedFiles().isEmpty()) {\n            return mappedFileQueue.getMinOffset();\n        }\n        for (MmapFile file : dLedgerFileList.getMappedFiles()) {\n            if (file.isAvailable()) {\n                return file.getFileFromOffset() + file.getStartPosition();\n            }\n        }\n        return 0;\n    }\n\n    @Override\n    public long getConfirmOffset() {\n        return this.getMaxOffset();\n    }\n\n    @Override\n    public void setConfirmOffset(long phyOffset) {\n        log.warn(\"Should not set confirm offset {} for dleger commitlog\", phyOffset);\n    }\n\n    @Override\n    public long remainHowManyDataToCommit() {\n        return dLedgerFileList.remainHowManyDataToCommit();\n    }\n\n    @Override\n    public long remainHowManyDataToFlush() {\n        return dLedgerFileList.remainHowManyDataToFlush();\n    }\n\n    @Override\n    public int deleteExpiredFile(\n        final long expiredTime,\n        final int deleteFilesInterval,\n        final long intervalForcibly,\n        final boolean cleanImmediately\n    ) {\n        if (mappedFileQueue.getMappedFiles().isEmpty()) {\n            refreshConfig();\n            //To prevent too much log in defaultMessageStore\n            return Integer.MAX_VALUE;\n        } else {\n            disableDeleteDledger();\n        }\n        int count = super.deleteExpiredFile(expiredTime, deleteFilesInterval, intervalForcibly, cleanImmediately);\n        if (count > 0 || mappedFileQueue.getMappedFiles().size() != 1) {\n            return count;\n        }\n        //the old logic will keep the last file, here to delete it\n        MappedFile mappedFile = mappedFileQueue.getLastMappedFile();\n        log.info(\"Try to delete the last old commitlog file {}\", mappedFile.getFileName());\n        long liveMaxTimestamp = mappedFile.getLastModifiedTimestamp() + expiredTime;\n        if (System.currentTimeMillis() >= liveMaxTimestamp || cleanImmediately) {\n            while (!mappedFile.destroy(10 * 1000)) {\n                DLedgerUtils.sleep(1000);\n            }\n            mappedFileQueue.getMappedFiles().remove(mappedFile);\n        }\n        return 1;\n    }\n\n    public SelectMappedBufferResult convertSbr(SelectMmapBufferResult sbr) {\n        if (sbr == null) {\n            return null;\n        } else {\n            return new DLedgerSelectMappedBufferResult(sbr);\n        }\n\n    }\n\n    public SelectMmapBufferResult truncate(SelectMmapBufferResult sbr) {\n        long committedPos = dLedgerFileStore.getCommittedPos();\n        if (sbr == null || sbr.getStartOffset() == committedPos) {\n            return null;\n        }\n        if (sbr.getStartOffset() + sbr.getSize() <= committedPos) {\n            return sbr;\n        } else {\n            sbr.setSize((int) (committedPos - sbr.getStartOffset()));\n            return sbr;\n        }\n    }\n\n    @Override\n    public SelectMappedBufferResult getData(final long offset) {\n        if (offset < dividedCommitlogOffset) {\n            return super.getData(offset);\n        }\n        return this.getData(offset, offset == 0);\n    }\n\n    @Override\n    public SelectMappedBufferResult getData(final long offset, final boolean returnFirstOnNotFound) {\n        if (offset < dividedCommitlogOffset) {\n            return super.getData(offset, returnFirstOnNotFound);\n        }\n        if (offset >= dLedgerFileStore.getCommittedPos()) {\n            return null;\n        }\n        int mappedFileSize = this.dLedgerServer.getdLedgerConfig().getMappedFileSizeForEntryData();\n        MmapFile mappedFile = this.dLedgerFileList.findMappedFileByOffset(offset, returnFirstOnNotFound);\n        if (mappedFile != null) {\n            int pos = (int) (offset % mappedFileSize);\n            SelectMmapBufferResult sbr = mappedFile.selectMappedBuffer(pos);\n            return convertSbr(truncate(sbr));\n        }\n\n        return null;\n    }\n\n    @Override\n    public boolean getData(final long offset, final int size, final ByteBuffer byteBuffer) {\n        if (offset < dividedCommitlogOffset) {\n            return super.getData(offset, size, byteBuffer);\n        }\n        if (offset >= dLedgerFileStore.getCommittedPos()) {\n            return false;\n        }\n        int mappedFileSize = this.dLedgerServer.getdLedgerConfig().getMappedFileSizeForEntryData();\n        MmapFile mappedFile = this.dLedgerFileList.findMappedFileByOffset(offset, offset == 0);\n        if (mappedFile != null) {\n            int pos = (int) (offset % mappedFileSize);\n            return mappedFile.getData(pos, size, byteBuffer);\n        }\n        return false;\n    }\n\n    private void dledgerRecoverNormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException {\n        dLedgerFileStore.load();\n        if (!dLedgerFileList.getMappedFiles().isEmpty()) {\n            dLedgerFileStore.recover();\n            dividedCommitlogOffset = dLedgerFileList.getFirstMappedFile().getFileFromOffset();\n            MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();\n            if (mappedFile != null) {\n                disableDeleteDledger();\n            }\n            long maxPhyOffset = dLedgerFileList.getMaxWrotePosition();\n            // Clear ConsumeQueue redundant data\n            if (maxPhyOffsetOfConsumeQueue >= maxPhyOffset) {\n                log.warn(\"[TruncateCQ]maxPhyOffsetOfConsumeQueue({}) >= processOffset({}), truncate dirty logic files\", maxPhyOffsetOfConsumeQueue, maxPhyOffset);\n                this.defaultMessageStore.truncateDirtyLogicFiles(maxPhyOffset);\n            }\n            return;\n        }\n        //Indicate that, it is the first time to load mixed commitlog, need to recover the old commitlog\n        isInrecoveringOldCommitlog = true;\n        super.recoverNormally(maxPhyOffsetOfConsumeQueue);\n        isInrecoveringOldCommitlog = false;\n\n        setRecoverPosition();\n\n    }\n\n    private void dledgerRecoverAbnormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException {\n        boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover();\n        boolean checkDupInfo = this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable();\n        dLedgerFileStore.load();\n        if (!dLedgerFileList.getMappedFiles().isEmpty()) {\n            dLedgerFileStore.recover();\n            dividedCommitlogOffset = dLedgerFileList.getFirstMappedFile().getFileFromOffset();\n            MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();\n            if (mappedFile != null) {\n                disableDeleteDledger();\n            }\n            List<MmapFile> mmapFiles = dLedgerFileList.getMappedFiles();\n            int index = mmapFiles.size() - 1;\n            MmapFile mmapFile = null;\n            for (; index >= 0; index--) {\n                mmapFile = mmapFiles.get(index);\n                if (isMmapFileMatchedRecover(mmapFile, false)) {\n                    log.info(\"dledger recover from this mappFile \" + mmapFile.getFileName());\n                    break;\n                }\n            }\n\n            if (index < 0) {\n                index = 0;\n                mmapFile = mmapFiles.get(index);\n            }\n\n            ByteBuffer byteBuffer = mmapFile.sliceByteBuffer();\n            long processOffset = mmapFile.getFileFromOffset();\n            long mmapFileOffset = 0;\n            while (true) {\n                DispatchRequest dispatchRequest = this.checkMessageAndReturnSize(byteBuffer, checkCRCOnRecover, checkDupInfo);\n                int size = dispatchRequest.getMsgSize();\n\n                if (dispatchRequest.isSuccess()) {\n                    if (size > 0) {\n                        mmapFileOffset += size;\n                        if (this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable()) {\n                            if (dispatchRequest.getCommitLogOffset() < this.defaultMessageStore.getConfirmOffset()) {\n                                this.defaultMessageStore.doDispatch(dispatchRequest);\n                            }\n                        } else {\n                            this.defaultMessageStore.doDispatch(dispatchRequest);\n                        }\n                    } else if (size == 0) {\n                        index++;\n                        if (index >= mmapFiles.size()) {\n                            log.info(\"dledger recover physics file over, last mapped file \" + mmapFile.getFileName());\n                            break;\n                        } else {\n                            mmapFile = mmapFiles.get(index);\n                            byteBuffer = mmapFile.sliceByteBuffer();\n                            processOffset = mmapFile.getFileFromOffset();\n                            mmapFileOffset = 0;\n                            log.info(\"dledger recover next physics file, \" + mmapFile.getFileName());\n                        }\n                    }\n                } else {\n                    log.info(\"dledger recover physics file end, \" + mmapFile.getFileName() + \" pos=\" + byteBuffer.position());\n                    break;\n                }\n            }\n\n            processOffset += mmapFileOffset;\n\n            if (maxPhyOffsetOfConsumeQueue >= processOffset) {\n                log.warn(\"dledger maxPhyOffsetOfConsumeQueue({}) >= processOffset({}), truncate dirty logic files\", maxPhyOffsetOfConsumeQueue, processOffset);\n                this.defaultMessageStore.truncateDirtyLogicFiles(processOffset);\n            }\n            return;\n        }\n        isInrecoveringOldCommitlog = true;\n        super.recoverAbnormally(maxPhyOffsetOfConsumeQueue);\n\n        isInrecoveringOldCommitlog = false;\n\n        setRecoverPosition();\n\n    }\n\n    private void setRecoverPosition() {\n        MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();\n        if (mappedFile == null) {\n            return;\n        }\n        ByteBuffer byteBuffer = mappedFile.sliceByteBuffer();\n        byteBuffer.position(mappedFile.getWrotePosition());\n        boolean needWriteMagicCode = true;\n        // 1 TOTAL SIZE\n        byteBuffer.getInt(); //size\n        int magicCode = byteBuffer.getInt();\n        if (magicCode == CommitLog.BLANK_MAGIC_CODE) {\n            needWriteMagicCode = false;\n        } else {\n            log.info(\"Recover old commitlog found a illegal magic code={}\", magicCode);\n        }\n        dLedgerConfig.setEnableDiskForceClean(false);\n        dividedCommitlogOffset = mappedFile.getFileFromOffset() + mappedFile.getFileSize();\n        log.info(\"Recover old commitlog needWriteMagicCode={} pos={} file={} dividedCommitlogOffset={}\", needWriteMagicCode, mappedFile.getFileFromOffset() + mappedFile.getWrotePosition(), mappedFile.getFileName(), dividedCommitlogOffset);\n        if (needWriteMagicCode) {\n            byteBuffer.position(mappedFile.getWrotePosition());\n            byteBuffer.putInt(mappedFile.getFileSize() - mappedFile.getWrotePosition());\n            byteBuffer.putInt(BLANK_MAGIC_CODE);\n            mappedFile.flush(0);\n        }\n        mappedFile.setWrotePosition(mappedFile.getFileSize());\n        mappedFile.setCommittedPosition(mappedFile.getFileSize());\n        mappedFile.setFlushedPosition(mappedFile.getFileSize());\n        dLedgerFileList.getLastMappedFile(dividedCommitlogOffset);\n        log.info(\"Will set the initial commitlog offset={} for dledger\", dividedCommitlogOffset);\n    }\n\n    private boolean isMmapFileMatchedRecover(final MmapFile mmapFile, boolean recoverNormally) throws RocksDBException {\n        ByteBuffer byteBuffer = mmapFile.sliceByteBuffer();\n\n        int magicCode = byteBuffer.getInt(DLedgerEntry.BODY_OFFSET + MessageDecoder.MESSAGE_MAGIC_CODE_POSITION);\n        if (magicCode != MESSAGE_MAGIC_CODE) {\n            return false;\n        }\n\n        int storeTimestampPosition;\n        int sysFlag = byteBuffer.getInt(DLedgerEntry.BODY_OFFSET + MessageDecoder.SYSFLAG_POSITION);\n        if ((sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0) {\n            storeTimestampPosition = MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION;\n        } else {\n            // v6 address is 12 byte larger than v4\n            storeTimestampPosition = MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION + 12;\n        }\n\n        long storeTimestamp = byteBuffer.getLong(DLedgerEntry.BODY_OFFSET + storeTimestampPosition);\n        if (storeTimestamp == 0) {\n            return false;\n        }\n        long phyOffset = byteBuffer.getLong(DLedgerEntry.BODY_OFFSET + MessageDecoder.MESSAGE_PHYSIC_OFFSET_POSITION);\n\n        if (this.defaultMessageStore.getMessageStoreConfig().isMessageIndexEnable()\n            && this.defaultMessageStore.getMessageStoreConfig().isMessageIndexSafe()) {\n            if (storeTimestamp > this.defaultMessageStore.getStoreCheckpoint().getIndexMsgTimestamp()) {\n                return false;\n            }\n            log.info(\"DLedgerCommitLog isMmapFileMatchedRecover find satisfied MmapFile for index, \" +\n                    \"MmapFile storeTimestamp={}, MmapFile phyOffset={}, indexMsgTimestamp={}\",\n                storeTimestamp, phyOffset, this.defaultMessageStore.getStoreCheckpoint().getIndexMsgTimestamp());\n        }\n        return this.defaultMessageStore.getQueueStore().isMappedFileMatchedRecover(phyOffset, storeTimestamp, recoverNormally);\n    }\n\n    @Override\n    public void recoverNormally(long dispatchFromPhyOffset) throws RocksDBException {\n        dledgerRecoverNormally(dispatchFromPhyOffset);\n    }\n\n    @Override\n    public void recoverAbnormally(long dispatchFromPhyOffset) throws RocksDBException {\n        dledgerRecoverAbnormally(dispatchFromPhyOffset);\n    }\n\n    @Override\n    public DispatchRequest checkMessageAndReturnSize(ByteBuffer byteBuffer, final boolean checkCRC,\n        final boolean checkDupInfo, final boolean readBody) {\n        if (isInrecoveringOldCommitlog) {\n            return super.checkMessageAndReturnSize(byteBuffer, checkCRC, checkDupInfo, readBody);\n        }\n        try {\n            int bodyOffset = DLedgerEntry.BODY_OFFSET;\n            int pos = byteBuffer.position();\n            int magic = byteBuffer.getInt();\n            //In dledger, this field is size, it must be gt 0, so it could prevent collision\n            int magicOld = byteBuffer.getInt();\n            if (magicOld == CommitLog.BLANK_MAGIC_CODE\n                || magicOld == MessageDecoder.MESSAGE_MAGIC_CODE\n                || magicOld == MessageDecoder.MESSAGE_MAGIC_CODE_V2) {\n                byteBuffer.position(pos);\n                return super.checkMessageAndReturnSize(byteBuffer, checkCRC, checkDupInfo, readBody);\n            }\n            if (magic == MmapFileList.BLANK_MAGIC_CODE) {\n                return new DispatchRequest(0, true);\n            }\n            byteBuffer.position(pos + bodyOffset);\n            DispatchRequest dispatchRequest = super.checkMessageAndReturnSize(byteBuffer, checkCRC, checkDupInfo, readBody);\n            if (dispatchRequest.isSuccess()) {\n                dispatchRequest.setBufferSize(dispatchRequest.getMsgSize() + bodyOffset);\n            } else if (dispatchRequest.getMsgSize() > 0) {\n                dispatchRequest.setBufferSize(dispatchRequest.getMsgSize() + bodyOffset);\n            }\n            return dispatchRequest;\n        } catch (Throwable ignored) {\n        }\n\n        return new DispatchRequest(-1, false /* success */);\n    }\n\n    @Override\n    public boolean resetOffset(long offset) {\n        //currently, it seems resetOffset has no use\n        return false;\n    }\n\n    @Override\n    public long getBeginTimeInLock() {\n        return beginTimeInDledgerLock;\n    }\n\n    private void setMessageInfo(MessageExtBrokerInner msg, int tranType) {\n        // Set the storage time\n        msg.setStoreTimestamp(System.currentTimeMillis());\n        // Set the message body BODY CRC (consider the most appropriate setting\n        // on the client)\n        msg.setBodyCRC(UtilAll.crc32(msg.getBody()));\n\n        InetSocketAddress bornSocketAddress = (InetSocketAddress) msg.getBornHost();\n        if (bornSocketAddress.getAddress() instanceof Inet6Address) {\n            msg.setBornHostV6Flag();\n        }\n\n        InetSocketAddress storeSocketAddress = (InetSocketAddress) msg.getStoreHost();\n        if (storeSocketAddress.getAddress() instanceof Inet6Address) {\n            msg.setStoreHostAddressV6Flag();\n        }\n    }\n\n    @Override\n    public CompletableFuture<PutMessageResult> asyncPutMessage(MessageExtBrokerInner msg) {\n\n        StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService();\n\n        final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag());\n\n        setMessageInfo(msg, tranType);\n\n        final String finalTopic = msg.getTopic();\n\n        msg.setVersion(MessageVersion.MESSAGE_VERSION_V1);\n        boolean autoMessageVersionOnTopicLen =\n            this.defaultMessageStore.getMessageStoreConfig().isAutoMessageVersionOnTopicLen();\n        if (autoMessageVersionOnTopicLen && msg.getTopic().length() > Byte.MAX_VALUE) {\n            msg.setVersion(MessageVersion.MESSAGE_VERSION_V2);\n        }\n\n        // Back to Results\n        AppendMessageResult appendResult;\n        AppendFuture<AppendEntryResponse> dledgerFuture;\n        EncodeResult encodeResult;\n\n        encodeResult = this.messageSerializer.serialize(msg);\n        if (encodeResult.status != AppendMessageStatus.PUT_OK) {\n            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult.status)));\n        }\n\n        String topicQueueKey = msg.getTopic() + \"-\" + msg.getQueueId();\n        topicQueueLock.lock(topicQueueKey);\n        try {\n            defaultMessageStore.assignOffset(msg);\n\n            putMessageLock.lock(); //spin or ReentrantLock ,depending on store config\n            long elapsedTimeInLock;\n            long queueOffset;\n            try {\n                beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now();\n                queueOffset = getQueueOffsetByKey(msg, tranType);\n                encodeResult.setQueueOffsetKey(queueOffset, false);\n                AppendEntryRequest request = new AppendEntryRequest();\n                request.setGroup(dLedgerConfig.getGroup());\n                request.setRemoteId(dLedgerServer.getMemberState().getSelfId());\n                request.setBody(encodeResult.getData());\n                dledgerFuture = (AppendFuture<AppendEntryResponse>) dLedgerServer.handleAppend(request);\n                if (dledgerFuture.getPos() == -1) {\n                    return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.OS_PAGE_CACHE_BUSY, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR)));\n                }\n                long wroteOffset = dledgerFuture.getPos() + DLedgerEntry.BODY_OFFSET;\n\n                int msgIdLength = (msg.getSysFlag() & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 + 4 + 8 : 16 + 4 + 8;\n                ByteBuffer buffer = ByteBuffer.allocate(msgIdLength);\n\n                String msgId = MessageDecoder.createMessageId(buffer, msg.getStoreHostBytes(), wroteOffset);\n                elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInDledgerLock;\n                appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, encodeResult.getData().length, msgId, System.currentTimeMillis(), queueOffset, elapsedTimeInLock);\n            } finally {\n                beginTimeInDledgerLock = 0;\n                putMessageLock.unlock();\n            }\n\n            if (elapsedTimeInLock > 500) {\n                log.warn(\"[NOTIFYME]putMessage in lock cost time(ms)={}, bodyLength={} AppendMessageResult={}\", elapsedTimeInLock, msg.getBody().length, appendResult);\n            }\n\n            defaultMessageStore.increaseOffset(msg, getMessageNum(msg));\n        } catch (Exception e) {\n            log.error(\"Put message error\", e);\n            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR)));\n        } finally {\n            topicQueueLock.unlock(topicQueueKey);\n        }\n\n        return dledgerFuture.thenApply(appendEntryResponse -> {\n            PutMessageStatus putMessageStatus = PutMessageStatus.UNKNOWN_ERROR;\n            switch (DLedgerResponseCode.valueOf(appendEntryResponse.getCode())) {\n                case SUCCESS:\n                    putMessageStatus = PutMessageStatus.PUT_OK;\n                    break;\n                case INCONSISTENT_LEADER:\n                case NOT_LEADER:\n                case LEADER_NOT_READY:\n                case DISK_FULL:\n                    putMessageStatus = PutMessageStatus.SERVICE_NOT_AVAILABLE;\n                    break;\n                case WAIT_QUORUM_ACK_TIMEOUT:\n                    //Do not return flush_slave_timeout to the client, for the client will ignore it.\n                    putMessageStatus = PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH;\n                    break;\n                case LEADER_PENDING_FULL:\n                    putMessageStatus = PutMessageStatus.OS_PAGE_CACHE_BUSY;\n                    break;\n            }\n            PutMessageResult putMessageResult = new PutMessageResult(putMessageStatus, appendResult);\n            if (putMessageStatus == PutMessageStatus.PUT_OK) {\n                // Statistics\n                storeStatsService.getSinglePutMessageTopicTimesTotal(finalTopic).add(1);\n                storeStatsService.getSinglePutMessageTopicSizeTotal(msg.getTopic()).add(appendResult.getWroteBytes());\n            }\n            return putMessageResult;\n        });\n    }\n\n    @Override\n    public CompletableFuture<PutMessageResult> asyncPutMessages(MessageExtBatch messageExtBatch) {\n        final int tranType = MessageSysFlag.getTransactionValue(messageExtBatch.getSysFlag());\n\n        if (tranType != MessageSysFlag.TRANSACTION_NOT_TYPE) {\n            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null));\n        }\n        if (messageExtBatch.getDelayTimeLevel() > 0) {\n            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null));\n        }\n\n        // Set the storage time\n        messageExtBatch.setStoreTimestamp(System.currentTimeMillis());\n\n        StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService();\n\n        InetSocketAddress bornSocketAddress = (InetSocketAddress) messageExtBatch.getBornHost();\n        if (bornSocketAddress.getAddress() instanceof Inet6Address) {\n            messageExtBatch.setBornHostV6Flag();\n        }\n\n        InetSocketAddress storeSocketAddress = (InetSocketAddress) messageExtBatch.getStoreHost();\n        if (storeSocketAddress.getAddress() instanceof Inet6Address) {\n            messageExtBatch.setStoreHostAddressV6Flag();\n        }\n\n        messageExtBatch.setVersion(MessageVersion.MESSAGE_VERSION_V1);\n        boolean autoMessageVersionOnTopicLen =\n            this.defaultMessageStore.getMessageStoreConfig().isAutoMessageVersionOnTopicLen();\n        if (autoMessageVersionOnTopicLen && messageExtBatch.getTopic().length() > Byte.MAX_VALUE) {\n            messageExtBatch.setVersion(MessageVersion.MESSAGE_VERSION_V2);\n        }\n\n        // Back to Results\n        AppendMessageResult appendResult;\n        BatchAppendFuture<AppendEntryResponse> dledgerFuture;\n        EncodeResult encodeResult;\n\n        encodeResult = this.messageSerializer.serialize(messageExtBatch);\n        if (encodeResult.status != AppendMessageStatus.PUT_OK) {\n            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult\n                .status)));\n        }\n\n        int batchNum = encodeResult.batchData.size();\n        topicQueueLock.lock(encodeResult.queueOffsetKey);\n        try {\n            defaultMessageStore.assignOffset(messageExtBatch);\n\n            putMessageLock.lock(); //spin or ReentrantLock ,depending on store config\n            msgIdBuilder.setLength(0);\n            long elapsedTimeInLock;\n            long queueOffset;\n            int msgNum = 0;\n            try {\n                beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now();\n                queueOffset = getQueueOffsetByKey(messageExtBatch, tranType);\n                encodeResult.setQueueOffsetKey(queueOffset, true);\n                BatchAppendEntryRequest request = new BatchAppendEntryRequest();\n                request.setGroup(dLedgerConfig.getGroup());\n                request.setRemoteId(dLedgerServer.getMemberState().getSelfId());\n                request.setBatchMsgs(encodeResult.batchData);\n                AppendFuture<AppendEntryResponse> appendFuture = (AppendFuture<AppendEntryResponse>) dLedgerServer.handleAppend(request);\n                if (appendFuture.getPos() == -1) {\n                    log.warn(\"HandleAppend return false due to error code {}\", appendFuture.get().getCode());\n                    return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.OS_PAGE_CACHE_BUSY, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR)));\n                }\n                dledgerFuture = (BatchAppendFuture<AppendEntryResponse>) appendFuture;\n\n                long wroteOffset = 0;\n\n                int msgIdLength = (messageExtBatch.getSysFlag() & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 + 4 + 8 : 16 + 4 + 8;\n                ByteBuffer buffer = ByteBuffer.allocate(msgIdLength);\n\n                boolean isFirstOffset = true;\n                long firstWroteOffset = 0;\n                for (long pos : dledgerFuture.getPositions()) {\n                    wroteOffset = pos + DLedgerEntry.BODY_OFFSET;\n                    if (isFirstOffset) {\n                        firstWroteOffset = wroteOffset;\n                        isFirstOffset = false;\n                    }\n                    String msgId = MessageDecoder.createMessageId(buffer, messageExtBatch.getStoreHostBytes(), wroteOffset);\n                    if (msgIdBuilder.length() > 0) {\n                        msgIdBuilder.append(',').append(msgId);\n                    } else {\n                        msgIdBuilder.append(msgId);\n                    }\n                    msgNum++;\n                }\n\n                elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInDledgerLock;\n                appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, firstWroteOffset, encodeResult.totalMsgLen,\n                    msgIdBuilder.toString(), System.currentTimeMillis(), queueOffset, elapsedTimeInLock);\n                appendResult.setMsgNum(msgNum);\n            } finally {\n                beginTimeInDledgerLock = 0;\n                putMessageLock.unlock();\n            }\n\n            if (elapsedTimeInLock > 500) {\n                log.warn(\"[NOTIFYME]putMessage in lock cost time(ms)={}, bodyLength={} AppendMessageResult={}\",\n                    elapsedTimeInLock, messageExtBatch.getBody().length, appendResult);\n            }\n\n            defaultMessageStore.increaseOffset(messageExtBatch, (short) batchNum);\n\n        } catch (Exception e) {\n            log.error(\"Put message error\", e);\n            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR)));\n        }  finally {\n            topicQueueLock.unlock(encodeResult.queueOffsetKey);\n        }\n\n        return dledgerFuture.thenApply(appendEntryResponse -> {\n            PutMessageStatus putMessageStatus = PutMessageStatus.UNKNOWN_ERROR;\n            switch (DLedgerResponseCode.valueOf(appendEntryResponse.getCode())) {\n                case SUCCESS:\n                    putMessageStatus = PutMessageStatus.PUT_OK;\n                    break;\n                case INCONSISTENT_LEADER:\n                case NOT_LEADER:\n                case LEADER_NOT_READY:\n                case DISK_FULL:\n                    putMessageStatus = PutMessageStatus.SERVICE_NOT_AVAILABLE;\n                    break;\n                case WAIT_QUORUM_ACK_TIMEOUT:\n                    //Do not return flush_slave_timeout to the client, for the client will ignore it.\n                    putMessageStatus = PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH;\n                    break;\n                case LEADER_PENDING_FULL:\n                    putMessageStatus = PutMessageStatus.OS_PAGE_CACHE_BUSY;\n                    break;\n            }\n            PutMessageResult putMessageResult = new PutMessageResult(putMessageStatus, appendResult);\n            if (putMessageStatus == PutMessageStatus.PUT_OK) {\n                // Statistics\n                storeStatsService.getSinglePutMessageTopicTimesTotal(messageExtBatch.getTopic()).add(appendResult.getMsgNum());\n                storeStatsService.getSinglePutMessageTopicSizeTotal(messageExtBatch.getTopic()).add(appendResult.getWroteBytes());\n            }\n            return putMessageResult;\n        });\n    }\n\n    @Override\n    public SelectMappedBufferResult getMessage(final long offset, final int size) {\n        if (offset < dividedCommitlogOffset) {\n            return super.getMessage(offset, size);\n        }\n        int mappedFileSize = this.dLedgerServer.getdLedgerConfig().getMappedFileSizeForEntryData();\n        MmapFile mappedFile = this.dLedgerFileList.findMappedFileByOffset(offset, offset == 0);\n        if (mappedFile != null) {\n            int pos = (int) (offset % mappedFileSize);\n            return convertSbr(mappedFile.selectMappedBuffer(pos, size));\n        }\n        return null;\n    }\n\n    @Override\n    public long rollNextFile(final long offset) {\n        int mappedFileSize = this.defaultMessageStore.getMessageStoreConfig().getMappedFileSizeCommitLog();\n        return offset + mappedFileSize - offset % mappedFileSize;\n    }\n\n    @Override\n    public void destroy() {\n        super.destroy();\n        dLedgerFileList.destroy();\n    }\n\n    @Override\n    public boolean appendData(long startOffset, byte[] data, int dataStart, int dataLength) {\n        //the old ha service will invoke method, here to prevent it\n        return false;\n    }\n\n    @Override\n    public void checkSelf() {\n        dLedgerFileList.checkSelf();\n    }\n\n    @Override\n    public long lockTimeMills() {\n        long diff = 0;\n        long begin = this.beginTimeInDledgerLock;\n        if (begin > 0) {\n            diff = this.defaultMessageStore.now() - begin;\n        }\n\n        if (diff < 0) {\n            diff = 0;\n        }\n\n        return diff;\n    }\n\n    private long getQueueOffsetByKey(MessageExtBrokerInner msg, int tranType) {\n        Long queueOffset = msg.getQueueOffset();\n\n        // Transaction messages that require special handling\n        switch (tranType) {\n            // Prepared and Rollback message is not consumed, will not enter the\n            // consumer queuec\n            case MessageSysFlag.TRANSACTION_PREPARED_TYPE:\n            case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:\n                queueOffset = 0L;\n                break;\n            case MessageSysFlag.TRANSACTION_NOT_TYPE:\n            case MessageSysFlag.TRANSACTION_COMMIT_TYPE:\n            default:\n                break;\n        }\n        return queueOffset;\n    }\n\n    class EncodeResult {\n        private String queueOffsetKey;\n        private ByteBuffer data;\n        private List<byte[]> batchData;\n        private AppendMessageStatus status;\n        private int totalMsgLen;\n\n        public EncodeResult(AppendMessageStatus status, ByteBuffer data, String queueOffsetKey) {\n            this.data = data;\n            this.status = status;\n            this.queueOffsetKey = queueOffsetKey;\n        }\n\n        public void setQueueOffsetKey(long offset, boolean isBatch) {\n            if (!isBatch) {\n                this.data.putLong(MessageDecoder.QUEUE_OFFSET_POSITION, offset);\n                return;\n            }\n\n            for (byte[] data : batchData) {\n                ByteBuffer.wrap(data).putLong(MessageDecoder.QUEUE_OFFSET_POSITION, offset++);\n            }\n        }\n\n        public byte[] getData() {\n            return data.array();\n        }\n\n        public EncodeResult(AppendMessageStatus status, String queueOffsetKey, List<byte[]> batchData,\n            int totalMsgLen) {\n            this.batchData = batchData;\n            this.status = status;\n            this.queueOffsetKey = queueOffsetKey;\n            this.totalMsgLen = totalMsgLen;\n        }\n    }\n\n    class MessageSerializer {\n\n        // The maximum length of the message body\n        private final int maxMessageBodySize;\n\n        MessageSerializer(final int size) {\n            this.maxMessageBodySize = size;\n        }\n\n        public EncodeResult serialize(final MessageExtBrokerInner msgInner) {\n            // STORETIMESTAMP + STOREHOSTADDRESS + OFFSET <br>\n\n            // PHY OFFSET\n            long wroteOffset = 0;\n\n            long queueOffset = 0;\n\n            int sysflag = msgInner.getSysFlag();\n\n            int bornHostLength = (sysflag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 4 + 4 : 16 + 4;\n            int storeHostLength = (sysflag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 + 4 : 16 + 4;\n            ByteBuffer bornHostHolder = ByteBuffer.allocate(bornHostLength);\n            ByteBuffer storeHostHolder = ByteBuffer.allocate(storeHostLength);\n\n            String key = msgInner.getTopic() + \"-\" + msgInner.getQueueId();\n\n            /**\n             * Serialize message\n             */\n            final byte[] propertiesData =\n                msgInner.getPropertiesString() == null ? null : msgInner.getPropertiesString().getBytes(MessageDecoder.CHARSET_UTF8);\n\n            final int propertiesLength = propertiesData == null ? 0 : propertiesData.length;\n\n            if (propertiesLength > Short.MAX_VALUE) {\n                log.warn(\"putMessage message properties length too long. length={}\", propertiesData.length);\n                return new EncodeResult(AppendMessageStatus.PROPERTIES_SIZE_EXCEEDED, null, key);\n            }\n\n            final byte[] topicData = msgInner.getTopic().getBytes(MessageDecoder.CHARSET_UTF8);\n            final int topicLength = topicData.length;\n\n            final int bodyLength = msgInner.getBody() == null ? 0 : msgInner.getBody().length;\n\n            final int msgLen = MessageExtEncoder.calMsgLength(msgInner.getVersion(), msgInner.getSysFlag(), bodyLength, topicLength, propertiesLength);\n\n            ByteBuffer msgStoreItemMemory = ByteBuffer.allocate(msgLen);\n\n            // Exceeds the maximum message\n            if (bodyLength > this.maxMessageBodySize) {\n                DLedgerCommitLog.log.warn(\"message body size exceeded, msg total size: \" + msgLen + \", msg body size: \" + bodyLength\n                    + \", maxMessageBodySize: \" + this.maxMessageBodySize);\n                return new EncodeResult(AppendMessageStatus.MESSAGE_SIZE_EXCEEDED, null, key);\n            }\n            // Initialization of storage space\n            this.resetByteBuffer(msgStoreItemMemory, msgLen);\n            // 1 TOTALSIZE\n            msgStoreItemMemory.putInt(msgLen);\n            // 2 MAGICCODE\n            msgStoreItemMemory.putInt(msgInner.getVersion().getMagicCode());\n            // 3 BODYCRC\n            msgStoreItemMemory.putInt(msgInner.getBodyCRC());\n            // 4 QUEUEID\n            msgStoreItemMemory.putInt(msgInner.getQueueId());\n            // 5 FLAG\n            msgStoreItemMemory.putInt(msgInner.getFlag());\n            // 6 QUEUEOFFSET\n            msgStoreItemMemory.putLong(queueOffset);\n            // 7 PHYSICALOFFSET\n            msgStoreItemMemory.putLong(wroteOffset);\n            // 8 SYSFLAG\n            msgStoreItemMemory.putInt(msgInner.getSysFlag());\n            // 9 BORNTIMESTAMP\n            msgStoreItemMemory.putLong(msgInner.getBornTimestamp());\n            // 10 BORNHOST\n            resetByteBuffer(bornHostHolder, bornHostLength);\n            msgStoreItemMemory.put(msgInner.getBornHostBytes(bornHostHolder));\n            // 11 STORETIMESTAMP\n            msgStoreItemMemory.putLong(msgInner.getStoreTimestamp());\n            // 12 STOREHOSTADDRESS\n            resetByteBuffer(storeHostHolder, storeHostLength);\n            msgStoreItemMemory.put(msgInner.getStoreHostBytes(storeHostHolder));\n            //this.msgBatchMemory.put(msgInner.getStoreHostBytes());\n            // 13 RECONSUMETIMES\n            msgStoreItemMemory.putInt(msgInner.getReconsumeTimes());\n            // 14 Prepared Transaction Offset\n            msgStoreItemMemory.putLong(msgInner.getPreparedTransactionOffset());\n            // 15 BODY\n            msgStoreItemMemory.putInt(bodyLength);\n            if (bodyLength > 0) {\n                msgStoreItemMemory.put(msgInner.getBody());\n            }\n            // 16 TOPIC\n            msgInner.getVersion().putTopicLength(msgStoreItemMemory, topicLength);\n            msgStoreItemMemory.put(topicData);\n            // 17 PROPERTIES\n            msgStoreItemMemory.putShort((short) propertiesLength);\n            if (propertiesLength > 0) {\n                msgStoreItemMemory.put(propertiesData);\n            }\n            return new EncodeResult(AppendMessageStatus.PUT_OK, msgStoreItemMemory, key);\n        }\n\n        public EncodeResult serialize(final MessageExtBatch messageExtBatch) {\n            String key = messageExtBatch.getTopic() + \"-\" + messageExtBatch.getQueueId();\n\n            int totalMsgLen = 0;\n            ByteBuffer messagesByteBuff = messageExtBatch.wrap();\n\n            int totalLength = messagesByteBuff.limit();\n            if (totalLength > this.maxMessageBodySize) {\n                CommitLog.log.warn(\"message body size exceeded, msg body size: \" + totalLength\n                    + \", maxMessageBodySize: \" + this.maxMessageBodySize);\n                throw new RuntimeException(\"message size exceeded\");\n            }\n\n            List<byte[]> batchBody = new LinkedList<>();\n\n            int sysFlag = messageExtBatch.getSysFlag();\n            int bornHostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 4 + 4 : 16 + 4;\n            int storeHostLength = (sysFlag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 + 4 : 16 + 4;\n            ByteBuffer bornHostHolder = ByteBuffer.allocate(bornHostLength);\n            ByteBuffer storeHostHolder = ByteBuffer.allocate(storeHostLength);\n\n            while (messagesByteBuff.hasRemaining()) {\n                // 1 TOTALSIZE\n                messagesByteBuff.getInt();\n                // 2 MAGICCODE\n                messagesByteBuff.getInt();\n                // 3 BODYCRC\n                messagesByteBuff.getInt();\n                // 4 FLAG\n                int flag = messagesByteBuff.getInt();\n                // 5 BODY\n                int bodyLen = messagesByteBuff.getInt();\n                int bodyPos = messagesByteBuff.position();\n                int bodyCrc = UtilAll.crc32(messagesByteBuff.array(), bodyPos, bodyLen);\n                messagesByteBuff.position(bodyPos + bodyLen);\n                // 6 properties\n                short propertiesLen = messagesByteBuff.getShort();\n                int propertiesPos = messagesByteBuff.position();\n                messagesByteBuff.position(propertiesPos + propertiesLen);\n\n                final byte[] topicData = messageExtBatch.getTopic().getBytes(MessageDecoder.CHARSET_UTF8);\n\n                final int topicLength = topicData.length;\n\n                final int msgLen = MessageExtEncoder.calMsgLength(messageExtBatch.getVersion(), messageExtBatch.getSysFlag(), bodyLen, topicLength, propertiesLen);\n                ByteBuffer msgStoreItemMemory = ByteBuffer.allocate(msgLen);\n\n                totalMsgLen += msgLen;\n\n                // Initialization of storage space\n                this.resetByteBuffer(msgStoreItemMemory, msgLen);\n                // 1 TOTALSIZE\n                msgStoreItemMemory.putInt(msgLen);\n                // 2 MAGICCODE\n                msgStoreItemMemory.putInt(messageExtBatch.getVersion().getMagicCode());\n                // 3 BODYCRC\n                msgStoreItemMemory.putInt(bodyCrc);\n                // 4 QUEUEID\n                msgStoreItemMemory.putInt(messageExtBatch.getQueueId());\n                // 5 FLAG\n                msgStoreItemMemory.putInt(flag);\n                // 6 QUEUEOFFSET\n                msgStoreItemMemory.putLong(0L);\n                // 7 PHYSICALOFFSET\n                msgStoreItemMemory.putLong(0);\n                // 8 SYSFLAG\n                msgStoreItemMemory.putInt(messageExtBatch.getSysFlag());\n                // 9 BORNTIMESTAMP\n                msgStoreItemMemory.putLong(messageExtBatch.getBornTimestamp());\n                // 10 BORNHOST\n                resetByteBuffer(bornHostHolder, bornHostLength);\n                msgStoreItemMemory.put(messageExtBatch.getBornHostBytes(bornHostHolder));\n                // 11 STORETIMESTAMP\n                msgStoreItemMemory.putLong(messageExtBatch.getStoreTimestamp());\n                // 12 STOREHOSTADDRESS\n                resetByteBuffer(storeHostHolder, storeHostLength);\n                msgStoreItemMemory.put(messageExtBatch.getStoreHostBytes(storeHostHolder));\n                // 13 RECONSUMETIMES\n                msgStoreItemMemory.putInt(messageExtBatch.getReconsumeTimes());\n                // 14 Prepared Transaction Offset\n                msgStoreItemMemory.putLong(0);\n                // 15 BODY\n                msgStoreItemMemory.putInt(bodyLen);\n                if (bodyLen > 0) {\n                    msgStoreItemMemory.put(messagesByteBuff.array(), bodyPos, bodyLen);\n                }\n                // 16 TOPIC\n                messageExtBatch.getVersion().putTopicLength(msgStoreItemMemory, topicLength);\n                msgStoreItemMemory.put(topicData);\n                // 17 PROPERTIES\n                msgStoreItemMemory.putShort(propertiesLen);\n                if (propertiesLen > 0) {\n                    msgStoreItemMemory.put(messagesByteBuff.array(), propertiesPos, propertiesLen);\n                }\n                byte[] data = new byte[msgLen];\n                msgStoreItemMemory.clear();\n                msgStoreItemMemory.get(data);\n                batchBody.add(data);\n            }\n\n            return new EncodeResult(AppendMessageStatus.PUT_OK, key, batchBody, totalMsgLen);\n        }\n\n        private void resetByteBuffer(final ByteBuffer byteBuffer, final int limit) {\n            byteBuffer.flip();\n            byteBuffer.limit(limit);\n        }\n    }\n\n    public static class DLedgerSelectMappedBufferResult extends SelectMappedBufferResult {\n\n        private SelectMmapBufferResult sbr;\n\n        public DLedgerSelectMappedBufferResult(SelectMmapBufferResult sbr) {\n            super(sbr.getStartOffset(), sbr.getByteBuffer(), sbr.getSize(), null);\n            this.sbr = sbr;\n        }\n\n        @Override\n        public synchronized void release() {\n            super.release();\n            if (sbr != null) {\n                sbr.release();\n            }\n        }\n\n    }\n\n    public DLedgerServer getdLedgerServer() {\n        return dLedgerServer;\n    }\n\n    public int getId() {\n        return id;\n    }\n\n    public long getDividedCommitlogOffset() {\n        return dividedCommitlogOffset;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/exception/ConsumeQueueException.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.exception;\n\npublic class ConsumeQueueException extends StoreException {\n    public ConsumeQueueException() {\n    }\n\n    public ConsumeQueueException(String message) {\n        super(message);\n    }\n\n    public ConsumeQueueException(String message, Throwable cause) {\n        super(message, cause);\n    }\n\n    public ConsumeQueueException(Throwable cause) {\n        super(cause);\n    }\n\n    public ConsumeQueueException(String message, Throwable cause, boolean enableSuppression,\n        boolean writableStackTrace) {\n        super(message, cause, enableSuppression, writableStackTrace);\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/exception/StoreException.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.exception;\n\npublic class StoreException extends Exception {\n    public StoreException() {\n    }\n\n    public StoreException(String message) {\n        super(message);\n    }\n\n    public StoreException(String message, Throwable cause) {\n        super(message, cause);\n    }\n\n    public StoreException(Throwable cause) {\n        super(cause);\n    }\n\n    public StoreException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {\n        super(message, cause, enableSuppression, writableStackTrace);\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/DefaultHAClient.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha;\n\nimport java.io.IOException;\nimport java.net.SocketAddress;\nimport java.nio.ByteBuffer;\nimport java.nio.channels.ClosedChannelException;\nimport java.nio.channels.SelectionKey;\nimport java.nio.channels.Selector;\nimport java.nio.channels.SocketChannel;\nimport java.util.concurrent.atomic.AtomicReference;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.store.DefaultMessageStore;\n\npublic class DefaultHAClient extends ServiceThread implements HAClient {\n\n    /**\n     * Report header buffer size. Schema: slaveMaxOffset. Format:\n     *\n     * <pre>\n     * ┌───────────────────────────────────────────────┐\n     * │                  slaveMaxOffset               │\n     * │                    (8bytes)                   │\n     * ├───────────────────────────────────────────────┤\n     * │                                               │\n     * │                  Report Header                │\n     * </pre>\n     * <p>\n     */\n    public static final int REPORT_HEADER_SIZE = 8;\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private static final int READ_MAX_BUFFER_SIZE = 1024 * 1024 * 4;\n    private final AtomicReference<String> masterHaAddress = new AtomicReference<>();\n    private final AtomicReference<String> masterAddress = new AtomicReference<>();\n    private final ByteBuffer reportOffset = ByteBuffer.allocate(REPORT_HEADER_SIZE);\n    private SocketChannel socketChannel;\n    private Selector selector;\n    /**\n     * last time that slave reads date from master.\n     */\n    private long lastReadTimestamp = System.currentTimeMillis();\n    /**\n     * last time that slave reports offset to master.\n     */\n    private long lastWriteTimestamp = System.currentTimeMillis();\n\n    private long currentReportedOffset = 0;\n    private int dispatchPosition = 0;\n    private ByteBuffer byteBufferRead = ByteBuffer.allocate(READ_MAX_BUFFER_SIZE);\n    private ByteBuffer byteBufferBackup = ByteBuffer.allocate(READ_MAX_BUFFER_SIZE);\n    private DefaultMessageStore defaultMessageStore;\n    private volatile HAConnectionState currentState = HAConnectionState.READY;\n    private FlowMonitor flowMonitor;\n\n    public DefaultHAClient(DefaultMessageStore defaultMessageStore) throws IOException {\n        this.selector = NetworkUtil.openSelector();\n        this.defaultMessageStore = defaultMessageStore;\n        this.flowMonitor = new FlowMonitor(defaultMessageStore.getMessageStoreConfig());\n    }\n\n    public void updateHaMasterAddress(final String newAddr) {\n        String currentAddr = this.masterHaAddress.get();\n        if (masterHaAddress.compareAndSet(currentAddr, newAddr)) {\n            log.info(\"update master ha address, OLD: \" + currentAddr + \" NEW: \" + newAddr);\n        }\n    }\n\n    public void updateMasterAddress(final String newAddr) {\n        String currentAddr = this.masterAddress.get();\n        if (masterAddress.compareAndSet(currentAddr, newAddr)) {\n            log.info(\"update master address, OLD: \" + currentAddr + \" NEW: \" + newAddr);\n        }\n    }\n\n    public String getHaMasterAddress() {\n        return this.masterHaAddress.get();\n    }\n\n    public String getMasterAddress() {\n        return this.masterAddress.get();\n    }\n\n    private boolean isTimeToReportOffset() {\n        long interval = defaultMessageStore.now() - this.lastWriteTimestamp;\n        return interval > defaultMessageStore.getMessageStoreConfig().getHaSendHeartbeatInterval();\n    }\n\n    private boolean reportSlaveMaxOffset(final long maxOffset) {\n        this.reportOffset.position(0);\n        this.reportOffset.limit(REPORT_HEADER_SIZE);\n        this.reportOffset.putLong(maxOffset);\n        this.reportOffset.position(0);\n        this.reportOffset.limit(REPORT_HEADER_SIZE);\n\n        for (int i = 0; i < 3 && this.reportOffset.hasRemaining(); i++) {\n            try {\n                this.socketChannel.write(this.reportOffset);\n            } catch (IOException e) {\n                log.error(this.getServiceName()\n                    + \"reportSlaveMaxOffset this.socketChannel.write exception\", e);\n                return false;\n            }\n        }\n        lastWriteTimestamp = this.defaultMessageStore.getSystemClock().now();\n        return !this.reportOffset.hasRemaining();\n    }\n\n    private void reallocateByteBuffer() {\n        int remain = READ_MAX_BUFFER_SIZE - this.dispatchPosition;\n        if (remain > 0) {\n            this.byteBufferRead.position(this.dispatchPosition);\n\n            this.byteBufferBackup.position(0);\n            this.byteBufferBackup.limit(READ_MAX_BUFFER_SIZE);\n            this.byteBufferBackup.put(this.byteBufferRead);\n        }\n\n        this.swapByteBuffer();\n\n        this.byteBufferRead.position(remain);\n        this.byteBufferRead.limit(READ_MAX_BUFFER_SIZE);\n        this.dispatchPosition = 0;\n    }\n\n    private void swapByteBuffer() {\n        ByteBuffer tmp = this.byteBufferRead;\n        this.byteBufferRead = this.byteBufferBackup;\n        this.byteBufferBackup = tmp;\n    }\n\n    private boolean processReadEvent() {\n        int readSizeZeroTimes = 0;\n        while (this.byteBufferRead.hasRemaining()) {\n            try {\n                int readSize = this.socketChannel.read(this.byteBufferRead);\n                if (readSize > 0) {\n                    flowMonitor.addByteCountTransferred(readSize);\n                    readSizeZeroTimes = 0;\n                    boolean result = this.dispatchReadRequest();\n                    if (!result) {\n                        log.error(\"HAClient, dispatchReadRequest error\");\n                        return false;\n                    }\n                    lastReadTimestamp = System.currentTimeMillis();\n                } else if (readSize == 0) {\n                    if (++readSizeZeroTimes >= 3) {\n                        break;\n                    }\n                } else {\n                    log.info(\"HAClient, processReadEvent read socket < 0\");\n                    return false;\n                }\n            } catch (IOException e) {\n                log.info(\"HAClient, processReadEvent read socket exception\", e);\n                return false;\n            }\n        }\n\n        return true;\n    }\n\n    private boolean dispatchReadRequest() {\n        int readSocketPos = this.byteBufferRead.position();\n\n        while (true) {\n            int diff = this.byteBufferRead.position() - this.dispatchPosition;\n            if (diff >= DefaultHAConnection.TRANSFER_HEADER_SIZE) {\n                long masterPhyOffset = this.byteBufferRead.getLong(this.dispatchPosition);\n                int bodySize = this.byteBufferRead.getInt(this.dispatchPosition + 8);\n\n                long slavePhyOffset = this.defaultMessageStore.getMaxPhyOffset();\n\n                if (slavePhyOffset != 0) {\n                    if (slavePhyOffset != masterPhyOffset) {\n                        log.error(\"master pushed offset not equal the max phy offset in slave, SLAVE: \"\n                            + slavePhyOffset + \" MASTER: \" + masterPhyOffset);\n                        return false;\n                    }\n                }\n\n                if (diff >= (DefaultHAConnection.TRANSFER_HEADER_SIZE + bodySize)) {\n                    byte[] bodyData = byteBufferRead.array();\n                    int dataStart = this.dispatchPosition + DefaultHAConnection.TRANSFER_HEADER_SIZE;\n\n                    this.defaultMessageStore.appendToCommitLog(\n                        masterPhyOffset, bodyData, dataStart, bodySize);\n\n                    this.byteBufferRead.position(readSocketPos);\n                    this.dispatchPosition += DefaultHAConnection.TRANSFER_HEADER_SIZE + bodySize;\n\n                    if (!reportSlaveMaxOffsetPlus()) {\n                        return false;\n                    }\n\n                    continue;\n                }\n            }\n\n            if (!this.byteBufferRead.hasRemaining()) {\n                this.reallocateByteBuffer();\n            }\n\n            break;\n        }\n\n        return true;\n    }\n\n    private boolean reportSlaveMaxOffsetPlus() {\n        boolean result = true;\n        long currentPhyOffset = this.defaultMessageStore.getMaxPhyOffset();\n        if (currentPhyOffset > this.currentReportedOffset) {\n            this.currentReportedOffset = currentPhyOffset;\n            result = this.reportSlaveMaxOffset(this.currentReportedOffset);\n            if (!result) {\n                this.closeMaster();\n                log.error(\"HAClient, reportSlaveMaxOffset error, \" + this.currentReportedOffset);\n            }\n        }\n\n        return result;\n    }\n\n    public void changeCurrentState(HAConnectionState currentState) {\n        log.info(\"change state to {}\", currentState);\n        this.currentState = currentState;\n    }\n\n    public boolean connectMaster() throws ClosedChannelException {\n        if (null == socketChannel) {\n            String addr = this.masterHaAddress.get();\n            if (addr != null) {\n                SocketAddress socketAddress = NetworkUtil.string2SocketAddress(addr);\n                this.socketChannel = RemotingHelper.connect(socketAddress);\n                if (this.socketChannel != null) {\n                    this.socketChannel.register(this.selector, SelectionKey.OP_READ);\n                    log.info(\"HAClient connect to master {}\", addr);\n                    this.changeCurrentState(HAConnectionState.TRANSFER);\n                }\n            }\n\n            this.currentReportedOffset = this.defaultMessageStore.getMaxPhyOffset();\n\n            this.lastReadTimestamp = System.currentTimeMillis();\n        }\n\n        return this.socketChannel != null;\n    }\n\n    public void closeMaster() {\n        if (null != this.socketChannel) {\n            try {\n\n                SelectionKey sk = this.socketChannel.keyFor(this.selector);\n                if (sk != null) {\n                    sk.cancel();\n                }\n\n                this.socketChannel.close();\n\n                this.socketChannel = null;\n\n                log.info(\"HAClient close connection with master {}\", this.masterHaAddress.get());\n                this.changeCurrentState(HAConnectionState.READY);\n            } catch (IOException e) {\n                log.warn(\"closeMaster exception. \", e);\n            }\n\n            this.lastReadTimestamp = 0;\n            this.dispatchPosition = 0;\n\n            this.byteBufferBackup.position(0);\n            this.byteBufferBackup.limit(READ_MAX_BUFFER_SIZE);\n\n            this.byteBufferRead.position(0);\n            this.byteBufferRead.limit(READ_MAX_BUFFER_SIZE);\n        }\n    }\n\n    @Override\n    public void run() {\n        log.info(this.getServiceName() + \" service started\");\n\n        this.flowMonitor.start();\n\n        while (!this.isStopped()) {\n            try {\n                switch (this.currentState) {\n                    case SHUTDOWN:\n                        this.flowMonitor.shutdown(true);\n                        return;\n                    case READY:\n                        if (!this.connectMaster()) {\n                            log.warn(\"HAClient connect to master {} failed\", this.masterHaAddress.get());\n                            this.waitForRunning(1000 * 5);\n                        }\n                        continue;\n                    case TRANSFER:\n                        if (!transferFromMaster()) {\n                            closeMasterAndWait();\n                            continue;\n                        }\n                        break;\n                    default:\n                        this.waitForRunning(1000 * 2);\n                        continue;\n                }\n                long interval = this.defaultMessageStore.now() - this.lastReadTimestamp;\n                if (interval > this.defaultMessageStore.getMessageStoreConfig().getHaHousekeepingInterval()) {\n                    log.warn(\"AutoRecoverHAClient, housekeeping, found this connection[\" + this.masterHaAddress\n                        + \"] expired, \" + interval);\n                    this.closeMaster();\n                    log.warn(\"AutoRecoverHAClient, master not response some time, so close connection\");\n                }\n            } catch (Exception e) {\n                log.warn(this.getServiceName() + \" service has exception. \", e);\n                this.closeMasterAndWait();\n            }\n        }\n\n        this.flowMonitor.shutdown(true);\n        log.info(this.getServiceName() + \" service end\");\n    }\n\n    private boolean transferFromMaster() throws IOException {\n        boolean result;\n        if (this.isTimeToReportOffset()) {\n            log.info(\"Slave report current offset {}\", this.currentReportedOffset);\n            result = this.reportSlaveMaxOffset(this.currentReportedOffset);\n            if (!result) {\n                return false;\n            }\n        }\n\n        this.selector.select(1000);\n\n        result = this.processReadEvent();\n        if (!result) {\n            return false;\n        }\n\n        return reportSlaveMaxOffsetPlus();\n    }\n\n    public void closeMasterAndWait() {\n        this.closeMaster();\n        this.waitForRunning(1000 * 5);\n    }\n\n    public long getLastWriteTimestamp() {\n        return this.lastWriteTimestamp;\n    }\n\n    public long getLastReadTimestamp() {\n        return lastReadTimestamp;\n    }\n\n    @Override\n    public HAConnectionState getCurrentState() {\n        return currentState;\n    }\n\n    @Override\n    public long getTransferredByteInSecond() {\n        return flowMonitor.getTransferredByteInSecond();\n    }\n\n    @Override\n    public void shutdown() {\n        this.changeCurrentState(HAConnectionState.SHUTDOWN);\n        this.flowMonitor.shutdown();\n        super.shutdown();\n\n        closeMaster();\n        try {\n            this.selector.close();\n        } catch (IOException e) {\n            log.warn(\"Close the selector of AutoRecoverHAClient error, \", e);\n        }\n    }\n\n    @Override\n    public String getServiceName() {\n        if (this.defaultMessageStore != null && this.defaultMessageStore.getBrokerConfig().isInBrokerContainer()) {\n            return this.defaultMessageStore.getBrokerIdentity().getIdentifier() + DefaultHAClient.class.getSimpleName();\n        }\n        return DefaultHAClient.class.getSimpleName();\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/DefaultHAConnection.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha;\n\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport java.nio.channels.SelectionKey;\nimport java.nio.channels.Selector;\nimport java.nio.channels.SocketChannel;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.netty.NettySystemConfig;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\n\npublic class DefaultHAConnection implements HAConnection {\n\n    /**\n     * Transfer Header buffer size. Schema: physic offset and body size. Format:\n     *\n     * <pre>\n     * ┌───────────────────────────────────────────────┬───────────────────────┐\n     * │                  physicOffset                 │         bodySize      │\n     * │                    (8bytes)                   │         (4bytes)      │\n     * ├───────────────────────────────────────────────┴───────────────────────┤\n     * │                                                                       │\n     * │                           Transfer Header                             │\n     * </pre>\n     * <p>\n     */\n    public static final int TRANSFER_HEADER_SIZE = 8 + 4;\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private final DefaultHAService haService;\n    private final SocketChannel socketChannel;\n    private final String clientAddress;\n    private WriteSocketService writeSocketService;\n    private ReadSocketService readSocketService;\n    private volatile HAConnectionState currentState = HAConnectionState.TRANSFER;\n    private volatile long slaveRequestOffset = -1;\n    private volatile long slaveAckOffset = -1;\n    private FlowMonitor flowMonitor;\n\n    public DefaultHAConnection(final DefaultHAService haService, final SocketChannel socketChannel) throws IOException {\n        this.haService = haService;\n        this.socketChannel = socketChannel;\n        this.clientAddress = this.socketChannel.socket().getRemoteSocketAddress().toString();\n        this.socketChannel.configureBlocking(false);\n        this.socketChannel.socket().setSoLinger(false, -1);\n        this.socketChannel.socket().setTcpNoDelay(true);\n        if (NettySystemConfig.socketSndbufSize > 0) {\n            this.socketChannel.socket().setReceiveBufferSize(NettySystemConfig.socketSndbufSize);\n        }\n        if (NettySystemConfig.socketRcvbufSize > 0) {\n            this.socketChannel.socket().setSendBufferSize(NettySystemConfig.socketRcvbufSize);\n        }\n        this.writeSocketService = new WriteSocketService(this.socketChannel);\n        this.readSocketService = new ReadSocketService(this.socketChannel);\n        this.haService.getConnectionCount().incrementAndGet();\n        this.flowMonitor = new FlowMonitor(haService.getDefaultMessageStore().getMessageStoreConfig());\n    }\n\n    public void start() {\n        changeCurrentState(HAConnectionState.TRANSFER);\n        this.flowMonitor.start();\n        this.readSocketService.start();\n        this.writeSocketService.start();\n    }\n\n    public void shutdown() {\n        changeCurrentState(HAConnectionState.SHUTDOWN);\n        this.writeSocketService.shutdown(true);\n        this.readSocketService.shutdown(true);\n        this.flowMonitor.shutdown(true);\n        this.close();\n    }\n\n    public void close() {\n        if (this.socketChannel != null) {\n            try {\n                this.socketChannel.close();\n            } catch (IOException e) {\n                log.error(\"\", e);\n            }\n        }\n    }\n\n    public SocketChannel getSocketChannel() {\n        return socketChannel;\n    }\n\n    public void changeCurrentState(HAConnectionState currentState) {\n        log.info(\"change state to {}\", currentState);\n        this.currentState = currentState;\n    }\n\n    @Override\n    public HAConnectionState getCurrentState() {\n        return currentState;\n    }\n\n    @Override\n    public String getClientAddress() {\n        return this.clientAddress;\n    }\n\n    @Override\n    public long getSlaveAckOffset() {\n        return slaveAckOffset;\n    }\n\n    public long getTransferredByteInSecond() {\n        return this.flowMonitor.getTransferredByteInSecond();\n    }\n\n    public long getTransferFromWhere() {\n        return writeSocketService.getNextTransferFromWhere();\n    }\n\n    class ReadSocketService extends ServiceThread {\n        private static final int READ_MAX_BUFFER_SIZE = 1024 * 1024;\n        private final Selector selector;\n        private final SocketChannel socketChannel;\n        private final ByteBuffer byteBufferRead = ByteBuffer.allocate(READ_MAX_BUFFER_SIZE);\n        private int processPosition = 0;\n        private volatile long lastReadTimestamp = System.currentTimeMillis();\n\n        public ReadSocketService(final SocketChannel socketChannel) throws IOException {\n            this.selector = NetworkUtil.openSelector();\n            this.socketChannel = socketChannel;\n            this.socketChannel.register(this.selector, SelectionKey.OP_READ);\n            this.setDaemon(true);\n        }\n\n        @Override\n        public void run() {\n            log.info(this.getServiceName() + \" service started\");\n\n            while (!this.isStopped()) {\n                try {\n                    this.selector.select(1000);\n                    boolean ok = this.processReadEvent();\n                    if (!ok) {\n                        log.error(\"processReadEvent error\");\n                        break;\n                    }\n\n                    long interval = DefaultHAConnection.this.haService.getDefaultMessageStore().getSystemClock().now() - this.lastReadTimestamp;\n                    if (interval > DefaultHAConnection.this.haService.getDefaultMessageStore().getMessageStoreConfig().getHaHousekeepingInterval()) {\n                        log.warn(\"ha housekeeping, found this connection[\" + DefaultHAConnection.this.clientAddress + \"] expired, \" + interval);\n                        break;\n                    }\n                } catch (Exception e) {\n                    log.error(this.getServiceName() + \" service has exception.\", e);\n                    break;\n                }\n            }\n\n            changeCurrentState(HAConnectionState.SHUTDOWN);\n\n            this.makeStop();\n\n            writeSocketService.makeStop();\n\n            haService.removeConnection(DefaultHAConnection.this);\n\n            DefaultHAConnection.this.haService.getConnectionCount().decrementAndGet();\n\n            SelectionKey sk = this.socketChannel.keyFor(this.selector);\n            if (sk != null) {\n                sk.cancel();\n            }\n\n            try {\n                this.selector.close();\n                this.socketChannel.close();\n            } catch (IOException e) {\n                log.error(\"\", e);\n            }\n\n            flowMonitor.shutdown(true);\n\n            log.info(this.getServiceName() + \" service end\");\n        }\n\n        @Override\n        public String getServiceName() {\n            if (haService.getDefaultMessageStore().getBrokerConfig().isInBrokerContainer()) {\n                return haService.getDefaultMessageStore().getBrokerIdentity().getIdentifier() + ReadSocketService.class.getSimpleName();\n            }\n            return ReadSocketService.class.getSimpleName();\n        }\n\n        private boolean processReadEvent() {\n            int readSizeZeroTimes = 0;\n\n            if (!this.byteBufferRead.hasRemaining()) {\n                this.byteBufferRead.flip();\n                this.processPosition = 0;\n            }\n\n            while (this.byteBufferRead.hasRemaining()) {\n                try {\n                    int readSize = this.socketChannel.read(this.byteBufferRead);\n                    if (readSize > 0) {\n                        readSizeZeroTimes = 0;\n                        this.lastReadTimestamp = DefaultHAConnection.this.haService.getDefaultMessageStore().getSystemClock().now();\n                        if ((this.byteBufferRead.position() - this.processPosition) >= DefaultHAClient.REPORT_HEADER_SIZE) {\n                            int pos = this.byteBufferRead.position() - (this.byteBufferRead.position() % DefaultHAClient.REPORT_HEADER_SIZE);\n                            long readOffset = this.byteBufferRead.getLong(pos - 8);\n                            this.processPosition = pos;\n\n                            DefaultHAConnection.this.slaveAckOffset = readOffset;\n                            if (DefaultHAConnection.this.slaveRequestOffset < 0) {\n                                DefaultHAConnection.this.slaveRequestOffset = readOffset;\n                                log.info(\"slave[\" + DefaultHAConnection.this.clientAddress + \"] request offset \" + readOffset);\n                            }\n\n                            DefaultHAConnection.this.haService.notifyTransferSome(DefaultHAConnection.this.slaveAckOffset);\n                        }\n                    } else if (readSize == 0) {\n                        if (++readSizeZeroTimes >= 3) {\n                            break;\n                        }\n                    } else {\n                        log.error(\"read socket[\" + DefaultHAConnection.this.clientAddress + \"] < 0\");\n                        return false;\n                    }\n                } catch (IOException e) {\n                    log.error(\"processReadEvent exception\", e);\n                    return false;\n                }\n            }\n\n            return true;\n        }\n    }\n\n    class WriteSocketService extends ServiceThread {\n        private final Selector selector;\n        private final SocketChannel socketChannel;\n\n        private final ByteBuffer byteBufferHeader = ByteBuffer.allocate(TRANSFER_HEADER_SIZE);\n        private long nextTransferFromWhere = -1;\n        private SelectMappedBufferResult selectMappedBufferResult;\n        private boolean lastWriteOver = true;\n        private long lastPrintTimestamp = System.currentTimeMillis();\n        private long lastWriteTimestamp = System.currentTimeMillis();\n\n        public WriteSocketService(final SocketChannel socketChannel) throws IOException {\n            this.selector = NetworkUtil.openSelector();\n            this.socketChannel = socketChannel;\n            this.socketChannel.register(this.selector, SelectionKey.OP_WRITE);\n            this.setDaemon(true);\n        }\n\n        @Override\n        public void run() {\n            log.info(this.getServiceName() + \" service started\");\n\n            while (!this.isStopped()) {\n                try {\n                    this.selector.select(1000);\n\n                    if (-1 == DefaultHAConnection.this.slaveRequestOffset) {\n                        Thread.sleep(10);\n                        continue;\n                    }\n\n                    if (-1 == this.nextTransferFromWhere) {\n                        if (0 == DefaultHAConnection.this.slaveRequestOffset) {\n                            long masterOffset = DefaultHAConnection.this.haService.getDefaultMessageStore().getCommitLog().getMaxOffset();\n                            masterOffset =\n                                masterOffset\n                                    - (masterOffset % DefaultHAConnection.this.haService.getDefaultMessageStore().getMessageStoreConfig()\n                                    .getMappedFileSizeCommitLog());\n\n                            if (masterOffset < 0) {\n                                masterOffset = 0;\n                            }\n\n                            this.nextTransferFromWhere = masterOffset;\n                        } else {\n                            this.nextTransferFromWhere = DefaultHAConnection.this.slaveRequestOffset;\n                        }\n\n                        log.info(\"master transfer data from \" + this.nextTransferFromWhere + \" to slave[\" + DefaultHAConnection.this.clientAddress\n                            + \"], and slave request \" + DefaultHAConnection.this.slaveRequestOffset);\n                    }\n\n                    if (this.lastWriteOver) {\n\n                        long interval =\n                            DefaultHAConnection.this.haService.getDefaultMessageStore().getSystemClock().now() - this.lastWriteTimestamp;\n\n                        if (interval > DefaultHAConnection.this.haService.getDefaultMessageStore().getMessageStoreConfig()\n                            .getHaSendHeartbeatInterval()) {\n\n                            // Build Header\n                            this.byteBufferHeader.position(0);\n                            this.byteBufferHeader.limit(TRANSFER_HEADER_SIZE);\n                            this.byteBufferHeader.putLong(this.nextTransferFromWhere);\n                            this.byteBufferHeader.putInt(0);\n                            this.byteBufferHeader.flip();\n\n                            this.lastWriteOver = this.transferData();\n                            if (!this.lastWriteOver)\n                                continue;\n                        }\n                    } else {\n                        this.lastWriteOver = this.transferData();\n                        if (!this.lastWriteOver)\n                            continue;\n                    }\n\n                    SelectMappedBufferResult selectResult =\n                        DefaultHAConnection.this.haService.getDefaultMessageStore().getCommitLogData(this.nextTransferFromWhere);\n                    if (selectResult != null) {\n                        int size = selectResult.getSize();\n                        if (size > DefaultHAConnection.this.haService.getDefaultMessageStore().getMessageStoreConfig().getHaTransferBatchSize()) {\n                            size = DefaultHAConnection.this.haService.getDefaultMessageStore().getMessageStoreConfig().getHaTransferBatchSize();\n                        }\n\n                        int canTransferMaxBytes = flowMonitor.canTransferMaxByteNum();\n                        if (size > canTransferMaxBytes) {\n                            if (System.currentTimeMillis() - lastPrintTimestamp > 1000) {\n                                log.warn(\"Trigger HA flow control, max transfer speed {}KB/s, current speed: {}KB/s\",\n                                    String.format(\"%.2f\", flowMonitor.maxTransferByteInSecond() / 1024.0),\n                                    String.format(\"%.2f\", flowMonitor.getTransferredByteInSecond() / 1024.0));\n                                lastPrintTimestamp = System.currentTimeMillis();\n                            }\n                            size = canTransferMaxBytes;\n                        }\n\n                        long thisOffset = this.nextTransferFromWhere;\n                        this.nextTransferFromWhere += size;\n\n                        selectResult.getByteBuffer().limit(size);\n                        this.selectMappedBufferResult = selectResult;\n\n                        // Build Header\n                        this.byteBufferHeader.position(0);\n                        this.byteBufferHeader.limit(TRANSFER_HEADER_SIZE);\n                        this.byteBufferHeader.putLong(thisOffset);\n                        this.byteBufferHeader.putInt(size);\n                        this.byteBufferHeader.flip();\n\n                        this.lastWriteOver = this.transferData();\n                    } else {\n\n                        DefaultHAConnection.this.haService.getWaitNotifyObject().allWaitForRunning(100);\n                    }\n                } catch (Exception e) {\n\n                    DefaultHAConnection.log.error(this.getServiceName() + \" service has exception.\", e);\n                    break;\n                }\n            }\n\n            DefaultHAConnection.this.haService.getWaitNotifyObject().removeFromWaitingThreadTable();\n\n            if (this.selectMappedBufferResult != null) {\n                this.selectMappedBufferResult.release();\n            }\n\n            changeCurrentState(HAConnectionState.SHUTDOWN);\n\n            this.makeStop();\n\n            readSocketService.makeStop();\n\n            haService.removeConnection(DefaultHAConnection.this);\n\n            SelectionKey sk = this.socketChannel.keyFor(this.selector);\n            if (sk != null) {\n                sk.cancel();\n            }\n\n            try {\n                this.selector.close();\n                this.socketChannel.close();\n            } catch (IOException e) {\n                DefaultHAConnection.log.error(\"\", e);\n            }\n\n            flowMonitor.shutdown(true);\n\n            DefaultHAConnection.log.info(this.getServiceName() + \" service end\");\n        }\n\n        private boolean transferData() throws Exception {\n            int writeSizeZeroTimes = 0;\n            // Write Header\n            while (this.byteBufferHeader.hasRemaining()) {\n                int writeSize = this.socketChannel.write(this.byteBufferHeader);\n                if (writeSize > 0) {\n                    flowMonitor.addByteCountTransferred(writeSize);\n                    writeSizeZeroTimes = 0;\n                    this.lastWriteTimestamp = DefaultHAConnection.this.haService.getDefaultMessageStore().getSystemClock().now();\n                } else if (writeSize == 0) {\n                    if (++writeSizeZeroTimes >= 3) {\n                        break;\n                    }\n                } else {\n                    throw new Exception(\"ha master write header error < 0\");\n                }\n            }\n\n            if (null == this.selectMappedBufferResult) {\n                return !this.byteBufferHeader.hasRemaining();\n            }\n\n            writeSizeZeroTimes = 0;\n\n            // Write Body\n            if (!this.byteBufferHeader.hasRemaining()) {\n                while (this.selectMappedBufferResult.getByteBuffer().hasRemaining()) {\n                    int writeSize = this.socketChannel.write(this.selectMappedBufferResult.getByteBuffer());\n                    if (writeSize > 0) {\n                        writeSizeZeroTimes = 0;\n                        this.lastWriteTimestamp = DefaultHAConnection.this.haService.getDefaultMessageStore().getSystemClock().now();\n                    } else if (writeSize == 0) {\n                        if (++writeSizeZeroTimes >= 3) {\n                            break;\n                        }\n                    } else {\n                        throw new Exception(\"ha master write body error < 0\");\n                    }\n                }\n            }\n\n            boolean result = !this.byteBufferHeader.hasRemaining() && !this.selectMappedBufferResult.getByteBuffer().hasRemaining();\n\n            if (!this.selectMappedBufferResult.getByteBuffer().hasRemaining()) {\n                this.selectMappedBufferResult.release();\n                this.selectMappedBufferResult = null;\n            }\n\n            return result;\n        }\n\n        @Override\n        public String getServiceName() {\n            if (haService.getDefaultMessageStore().getBrokerConfig().isInBrokerContainer()) {\n                return haService.getDefaultMessageStore().getBrokerIdentity().getIdentifier() + WriteSocketService.class.getSimpleName();\n            }\n            return WriteSocketService.class.getSimpleName();\n        }\n\n        @Override\n        public void shutdown() {\n            super.shutdown();\n        }\n\n        public long getNextTransferFromWhere() {\n            return nextTransferFromWhere;\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/DefaultHAService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha;\n\nimport java.io.IOException;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.nio.channels.SelectionKey;\nimport java.nio.channels.Selector;\nimport java.nio.channels.ServerSocketChannel;\nimport java.nio.channels.SocketChannel;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;\nimport org.apache.rocketmq.store.CommitLog;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\n\npublic class DefaultHAService implements HAService {\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n\n    protected final AtomicInteger connectionCount = new AtomicInteger(0);\n\n    protected final List<HAConnection> connectionList = new LinkedList<>();\n\n    protected AcceptSocketService acceptSocketService;\n\n    protected DefaultMessageStore defaultMessageStore;\n\n    protected WaitNotifyObject waitNotifyObject = new WaitNotifyObject();\n    protected AtomicLong push2SlaveMaxOffset = new AtomicLong(0);\n\n    protected GroupTransferService groupTransferService;\n\n    protected HAClient haClient;\n\n    protected HAConnectionStateNotificationService haConnectionStateNotificationService;\n\n    public DefaultHAService() {\n    }\n\n    @Override\n    public void init(final DefaultMessageStore defaultMessageStore) throws IOException {\n        this.defaultMessageStore = defaultMessageStore;\n        this.acceptSocketService = new DefaultAcceptSocketService(defaultMessageStore.getMessageStoreConfig());\n        this.groupTransferService = new GroupTransferService(this, defaultMessageStore);\n        if (this.defaultMessageStore.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) {\n            this.haClient = new DefaultHAClient(this.defaultMessageStore);\n        }\n        this.haConnectionStateNotificationService = new HAConnectionStateNotificationService(this, defaultMessageStore);\n    }\n\n    @Override\n    public void updateMasterAddress(final String newAddr) {\n        if (this.haClient != null) {\n            this.haClient.updateMasterAddress(newAddr);\n        }\n    }\n\n    @Override\n    public void updateHaMasterAddress(String newAddr) {\n        if (this.haClient != null) {\n            this.haClient.updateHaMasterAddress(newAddr);\n        }\n    }\n\n    @Override\n    public void putRequest(final CommitLog.GroupCommitRequest request) {\n        this.groupTransferService.putRequest(request);\n    }\n\n    @Override\n    public boolean isSlaveOK(final long masterPutWhere) {\n        boolean result = this.connectionCount.get() > 0;\n        result =\n            result\n                && masterPutWhere - this.push2SlaveMaxOffset.get() < this.defaultMessageStore\n                .getMessageStoreConfig().getHaMaxGapNotInSync();\n        return result;\n    }\n\n    public void notifyTransferSome(final long offset) {\n        for (long value = this.push2SlaveMaxOffset.get(); offset > value; ) {\n            boolean ok = this.push2SlaveMaxOffset.compareAndSet(value, offset);\n            if (ok) {\n                this.groupTransferService.notifyTransferSome();\n                break;\n            } else {\n                value = this.push2SlaveMaxOffset.get();\n            }\n        }\n    }\n\n    @Override\n    public AtomicInteger getConnectionCount() {\n        return connectionCount;\n    }\n\n    @Override\n    public void start() throws Exception {\n        this.acceptSocketService.beginAccept();\n        this.acceptSocketService.start();\n        this.groupTransferService.start();\n        this.haConnectionStateNotificationService.start();\n        if (haClient != null) {\n            this.haClient.start();\n        }\n    }\n\n    public void addConnection(final HAConnection conn) {\n        synchronized (this.connectionList) {\n            this.connectionList.add(conn);\n        }\n    }\n\n    public void removeConnection(final HAConnection conn) {\n        this.haConnectionStateNotificationService.checkConnectionStateAndNotify(conn);\n        synchronized (this.connectionList) {\n            this.connectionList.remove(conn);\n        }\n    }\n\n    @Override\n    public void shutdown() {\n        if (this.haClient != null) {\n            this.haClient.shutdown();\n        }\n        if (this.acceptSocketService != null) {\n            this.acceptSocketService.shutdown(true);\n        }\n        this.destroyConnections();\n        if (this.groupTransferService != null) {\n            groupTransferService.shutdown();\n        }\n\n        if (this.haConnectionStateNotificationService != null) {\n            this.haConnectionStateNotificationService.shutdown();\n        }\n    }\n\n    public void destroyConnections() {\n        synchronized (this.connectionList) {\n            for (HAConnection c : this.connectionList) {\n                c.shutdown();\n            }\n\n            this.connectionList.clear();\n        }\n    }\n\n    public DefaultMessageStore getDefaultMessageStore() {\n        return defaultMessageStore;\n    }\n\n    @Override\n    public WaitNotifyObject getWaitNotifyObject() {\n        return waitNotifyObject;\n    }\n\n    @Override\n    public AtomicLong getPush2SlaveMaxOffset() {\n        return push2SlaveMaxOffset;\n    }\n\n    @Override\n    public int inSyncReplicasNums(final long masterPutWhere) {\n        int inSyncNums = 1;\n        for (HAConnection conn : this.connectionList) {\n            if (this.isInSyncSlave(masterPutWhere, conn)) {\n                inSyncNums++;\n            }\n        }\n        return inSyncNums;\n    }\n\n    protected boolean isInSyncSlave(final long masterPutWhere, HAConnection conn) {\n        if (masterPutWhere - conn.getSlaveAckOffset() < this.defaultMessageStore.getMessageStoreConfig()\n            .getHaMaxGapNotInSync()) {\n            return true;\n        }\n        return false;\n    }\n\n    @Override\n    public void putGroupConnectionStateRequest(HAConnectionStateNotificationRequest request) {\n        this.haConnectionStateNotificationService.setRequest(request);\n    }\n\n    @Override\n    public List<HAConnection> getConnectionList() {\n        return connectionList;\n    }\n\n    @Override\n    public HAClient getHAClient() {\n        return this.haClient;\n    }\n\n    @Override\n    public HARuntimeInfo getRuntimeInfo(long masterPutWhere) {\n        HARuntimeInfo info = new HARuntimeInfo();\n\n        if (BrokerRole.SLAVE.equals(this.getDefaultMessageStore().getMessageStoreConfig().getBrokerRole())) {\n            info.setMaster(false);\n\n            info.getHaClientRuntimeInfo().setMasterAddr(this.haClient.getHaMasterAddress());\n            info.getHaClientRuntimeInfo().setMaxOffset(this.getDefaultMessageStore().getMaxPhyOffset());\n            info.getHaClientRuntimeInfo().setLastReadTimestamp(this.haClient.getLastReadTimestamp());\n            info.getHaClientRuntimeInfo().setLastWriteTimestamp(this.haClient.getLastWriteTimestamp());\n            info.getHaClientRuntimeInfo().setTransferredByteInSecond(this.haClient.getTransferredByteInSecond());\n            info.getHaClientRuntimeInfo().setMasterFlushOffset(this.defaultMessageStore.getMasterFlushedOffset());\n        } else {\n            info.setMaster(true);\n            int inSyncNums = 0;\n\n            info.setMasterCommitLogMaxOffset(masterPutWhere);\n\n            for (HAConnection conn : this.connectionList) {\n                HARuntimeInfo.HAConnectionRuntimeInfo cInfo = new HARuntimeInfo.HAConnectionRuntimeInfo();\n\n                long slaveAckOffset = conn.getSlaveAckOffset();\n                cInfo.setSlaveAckOffset(slaveAckOffset);\n                cInfo.setDiff(masterPutWhere - slaveAckOffset);\n                cInfo.setAddr(conn.getClientAddress().substring(1));\n                cInfo.setTransferredByteInSecond(conn.getTransferredByteInSecond());\n                cInfo.setTransferFromWhere(conn.getTransferFromWhere());\n\n                boolean isInSync = this.isInSyncSlave(masterPutWhere, conn);\n                if (isInSync) {\n                    inSyncNums++;\n                }\n                cInfo.setInSync(isInSync);\n\n                info.getHaConnectionInfo().add(cInfo);\n            }\n            info.setInSyncSlaveNums(inSyncNums);\n        }\n        return info;\n    }\n\n    class DefaultAcceptSocketService extends AcceptSocketService {\n\n        public DefaultAcceptSocketService(final MessageStoreConfig messageStoreConfig) {\n            super(messageStoreConfig);\n        }\n\n        @Override\n        protected HAConnection createConnection(SocketChannel sc) throws IOException {\n            return new DefaultHAConnection(DefaultHAService.this, sc);\n        }\n\n        @Override\n        public String getServiceName() {\n            if (defaultMessageStore.getBrokerConfig().isInBrokerContainer()) {\n                return defaultMessageStore.getBrokerConfig().getIdentifier() + AcceptSocketService.class.getSimpleName();\n            }\n            return DefaultAcceptSocketService.class.getSimpleName();\n        }\n    }\n\n    /**\n     * Listens to slave connections to create {@link HAConnection}.\n     */\n    protected abstract class AcceptSocketService extends ServiceThread {\n        private final SocketAddress socketAddressListen;\n        private ServerSocketChannel serverSocketChannel;\n        private Selector selector;\n\n        private final MessageStoreConfig messageStoreConfig;\n\n        public AcceptSocketService(final MessageStoreConfig messageStoreConfig) {\n            this.messageStoreConfig = messageStoreConfig;\n            this.socketAddressListen = new InetSocketAddress(messageStoreConfig.getHaListenPort());\n        }\n\n        /**\n         * Starts listening to slave connections.\n         *\n         * @throws Exception If fails.\n         */\n        public void beginAccept() throws Exception {\n            this.serverSocketChannel = ServerSocketChannel.open();\n            this.selector = NetworkUtil.openSelector();\n            this.serverSocketChannel.socket().setReuseAddress(true);\n            this.serverSocketChannel.socket().bind(this.socketAddressListen);\n            if (0 == messageStoreConfig.getHaListenPort()) {\n                messageStoreConfig.setHaListenPort(this.serverSocketChannel.socket().getLocalPort());\n                log.info(\"OS picked up {} to listen for HA\", messageStoreConfig.getHaListenPort());\n            }\n            this.serverSocketChannel.configureBlocking(false);\n            this.serverSocketChannel.register(this.selector, SelectionKey.OP_ACCEPT);\n        }\n\n        /**\n         * {@inheritDoc}\n         */\n        @Override\n        public void shutdown(final boolean interrupt) {\n            super.shutdown(interrupt);\n            try {\n                if (null != this.serverSocketChannel) {\n                    this.serverSocketChannel.close();\n                }\n\n                if (null != this.selector) {\n                    this.selector.close();\n                }\n            } catch (IOException e) {\n                log.error(\"AcceptSocketService shutdown exception\", e);\n            }\n        }\n\n        /**\n         * {@inheritDoc}\n         */\n        @Override\n        public void run() {\n            log.info(this.getServiceName() + \" service started\");\n\n            while (!this.isStopped()) {\n                try {\n                    this.selector.select(1000);\n                    Set<SelectionKey> selected = this.selector.selectedKeys();\n\n                    if (selected != null) {\n                        for (SelectionKey k : selected) {\n                            if (k.isAcceptable()) {\n                                SocketChannel sc = ((ServerSocketChannel) k.channel()).accept();\n\n                                if (sc != null) {\n                                    DefaultHAService.log.info(\"HAService receive new connection, \"\n                                        + sc.socket().getRemoteSocketAddress());\n                                    try {\n                                        HAConnection conn = createConnection(sc);\n                                        DefaultHAService.this.addConnection(conn);\n                                        conn.start();\n                                    } catch (Exception e) {\n                                        log.error(\"new HAConnection exception\", e);\n                                        sc.close();\n                                    }\n                                }\n                            } else {\n                                log.warn(\"Unexpected ops in select \" + k.readyOps());\n                            }\n                        }\n\n                        selected.clear();\n                    }\n                } catch (Exception e) {\n                    log.error(this.getServiceName() + \" service has exception.\", e);\n                }\n            }\n\n            log.info(this.getServiceName() + \" service end\");\n        }\n\n        /**\n         * Create ha connection\n         */\n        protected abstract HAConnection createConnection(final SocketChannel sc) throws IOException;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/FlowMonitor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha;\n\nimport java.util.concurrent.atomic.AtomicLong;\n\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\n\npublic class FlowMonitor extends ServiceThread {\n    private final AtomicLong transferredByte = new AtomicLong(0L);\n    private volatile long transferredByteInSecond;\n    protected MessageStoreConfig messageStoreConfig;\n\n    public FlowMonitor(MessageStoreConfig messageStoreConfig) {\n        this.messageStoreConfig = messageStoreConfig;\n    }\n\n    @Override\n    public void run() {\n        while (!this.isStopped()) {\n            this.waitForRunning(1 * 1000);\n            this.calculateSpeed();\n        }\n    }\n\n    public void calculateSpeed() {\n        this.transferredByteInSecond = this.transferredByte.get();\n        this.transferredByte.set(0);\n    }\n\n    public int canTransferMaxByteNum() {\n        // Flow control is not started at present\n        if (this.isFlowControlEnable()) {\n            long res = Math.max(this.maxTransferByteInSecond() - this.transferredByte.get(), 0);\n            return res > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) res;\n        }\n        return Integer.MAX_VALUE;\n    }\n\n    public void addByteCountTransferred(long count) {\n        this.transferredByte.addAndGet(count);\n    }\n\n    public long getTransferredByteInSecond() {\n        return this.transferredByteInSecond;\n    }\n\n    @Override\n    public String getServiceName() {\n        return FlowMonitor.class.getSimpleName();\n    }\n\n    protected boolean isFlowControlEnable() {\n        return this.messageStoreConfig.isHaFlowControlEnable();\n    }\n\n    public long maxTransferByteInSecond() {\n        return this.messageStoreConfig.getMaxHaTransferByteInSecond();\n    }\n}"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/GroupTransferService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha;\n\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Set;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.CommitLog;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.PutMessageSpinLock;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAConnection;\nimport org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService;\n\n/**\n * GroupTransferService Service\n */\npublic class GroupTransferService extends ServiceThread {\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n\n    private final WaitNotifyObject notifyTransferObject = new WaitNotifyObject();\n    private final PutMessageSpinLock lock = new PutMessageSpinLock();\n    private final DefaultMessageStore defaultMessageStore;\n    private final HAService haService;\n    private volatile List<CommitLog.GroupCommitRequest> requestsWrite = new LinkedList<>();\n    private volatile List<CommitLog.GroupCommitRequest> requestsRead = new LinkedList<>();\n\n    public GroupTransferService(final HAService haService, final DefaultMessageStore defaultMessageStore) {\n        this.haService = haService;\n        this.defaultMessageStore = defaultMessageStore;\n    }\n\n    public void putRequest(final CommitLog.GroupCommitRequest request) {\n        lock.lock();\n        try {\n            this.requestsWrite.add(request);\n        } finally {\n            lock.unlock();\n        }\n        wakeup();\n    }\n\n    public void notifyTransferSome() {\n        this.notifyTransferObject.wakeup();\n    }\n\n    private void swapRequests() {\n        lock.lock();\n        try {\n            List<CommitLog.GroupCommitRequest> tmp = this.requestsWrite;\n            this.requestsWrite = this.requestsRead;\n            this.requestsRead = tmp;\n        } finally {\n            lock.unlock();\n        }\n    }\n\n    private void doWaitTransfer() {\n        if (!this.requestsRead.isEmpty()) {\n            for (CommitLog.GroupCommitRequest req : this.requestsRead) {\n                boolean transferOK = false;\n\n                long deadLine = req.getDeadLine();\n                final boolean allAckInSyncStateSet = req.getAckNums() == MixAll.ALL_ACK_IN_SYNC_STATE_SET;\n\n                for (int i = 0; !transferOK && deadLine - System.nanoTime() > 0; i++) {\n                    if (i > 0) {\n                        this.notifyTransferObject.waitForRunning(1);\n                    }\n\n                    if (!allAckInSyncStateSet && req.getAckNums() <= 1) {\n                        transferOK = haService.getPush2SlaveMaxOffset().get() >= req.getNextOffset();\n                        continue;\n                    }\n\n                    if (allAckInSyncStateSet && this.haService instanceof AutoSwitchHAService) {\n                        // In this mode, we must wait for all replicas that in SyncStateSet.\n                        final AutoSwitchHAService autoSwitchHAService = (AutoSwitchHAService) this.haService;\n                        final Set<Long> syncStateSet = autoSwitchHAService.getSyncStateSet();\n                        if (syncStateSet.size() <= 1) {\n                            // Only master\n                            transferOK = true;\n                            break;\n                        }\n\n                        // Include master\n                        int ackNums = 1;\n                        for (HAConnection conn : haService.getConnectionList()) {\n                            final AutoSwitchHAConnection autoSwitchHAConnection = (AutoSwitchHAConnection) conn;\n                            if (syncStateSet.contains(autoSwitchHAConnection.getSlaveId()) && autoSwitchHAConnection.getSlaveAckOffset() >= req.getNextOffset()) {\n                                ackNums++;\n                            }\n                            if (ackNums >= syncStateSet.size()) {\n                                transferOK = true;\n                                break;\n                            }\n                        }\n                    } else {\n                        // Include master\n                        int ackNums = 1;\n                        for (HAConnection conn : haService.getConnectionList()) {\n                            // TODO: We must ensure every HAConnection represents a different slave\n                            // Solution: Consider assign a unique and fixed IP:ADDR for each different slave\n                            if (conn.getSlaveAckOffset() >= req.getNextOffset()) {\n                                ackNums++;\n                            }\n                            if (ackNums >= req.getAckNums()) {\n                                transferOK = true;\n                                break;\n                            }\n                        }\n                    }\n                }\n\n                if (!transferOK) {\n                    log.warn(\"transfer message to slave timeout, offset : {}, request acks: {}\",\n                        req.getNextOffset(), req.getAckNums());\n                }\n\n                req.wakeupCustomer(transferOK ? PutMessageStatus.PUT_OK : PutMessageStatus.FLUSH_SLAVE_TIMEOUT);\n            }\n\n            this.requestsRead = new LinkedList<>();\n        }\n    }\n\n    @Override\n    public void run() {\n        log.info(this.getServiceName() + \" service started\");\n\n        while (!this.isStopped()) {\n            try {\n                this.waitForRunning(10);\n                this.doWaitTransfer();\n            } catch (Exception e) {\n                log.warn(this.getServiceName() + \" service has exception. \", e);\n            }\n        }\n\n        log.info(this.getServiceName() + \" service end\");\n    }\n\n    @Override\n    protected void onWaitEnd() {\n        this.swapRequests();\n    }\n\n    @Override\n    public String getServiceName() {\n        if (defaultMessageStore != null && defaultMessageStore.getBrokerConfig().isInBrokerContainer()) {\n            return defaultMessageStore.getBrokerIdentity().getIdentifier() + GroupTransferService.class.getSimpleName();\n        }\n        return GroupTransferService.class.getSimpleName();\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/HAClient.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha;\n\npublic interface HAClient {\n\n    /**\n     * Start HAClient\n     */\n    void start();\n\n    /**\n     * Shutdown HAClient\n     */\n    void shutdown();\n\n    /**\n     * Wakeup HAClient\n     */\n    void wakeup();\n\n    /**\n     * Update master address\n     *\n     * @param newAddress\n     */\n    void updateMasterAddress(String newAddress);\n\n    /**\n     * Update master ha address\n     *\n     * @param newAddress\n     */\n    void updateHaMasterAddress(String newAddress);\n\n    /**\n     * Get master address\n     *\n     * @return master address\n     */\n    String getMasterAddress();\n\n    /**\n     * Get master ha address\n     *\n     * @return master ha address\n     */\n    String getHaMasterAddress();\n\n    /**\n     * Get HAClient last read timestamp\n     *\n     * @return last read timestamp\n     */\n    long getLastReadTimestamp();\n\n    /**\n     * Get HAClient last write timestamp\n     *\n     * @return last write timestamp\n     */\n    long getLastWriteTimestamp();\n\n    /**\n     * Get current state for ha connection\n     *\n     * @return HAConnectionState\n     */\n    HAConnectionState getCurrentState();\n\n    /**\n     * Change the current state for ha connection for testing\n     *\n     * @param haConnectionState\n     */\n    void changeCurrentState(HAConnectionState haConnectionState);\n\n    /**\n     * Disconnecting from the master for testing\n     */\n    void closeMaster();\n\n    /**\n     * Get the transfer rate per second\n     *\n     *  @return transfer bytes in second\n     */\n    long getTransferredByteInSecond();\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/HAConnection.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha;\n\nimport java.nio.channels.SocketChannel;\n\npublic interface HAConnection {\n    /**\n     * Start HA Connection\n     */\n    void start();\n\n    /**\n     * Shutdown HA Connection\n     */\n    void shutdown();\n\n    /**\n     * Close HA Connection\n     */\n    void close();\n\n    /**\n     * Get socket channel\n     */\n    SocketChannel getSocketChannel();\n\n    /**\n     * Get current state for ha connection\n     *\n     * @return HAConnectionState\n     */\n    HAConnectionState getCurrentState();\n\n    /**\n     * Get client address for ha connection\n     *\n     * @return client ip address\n     */\n    String getClientAddress();\n\n    /**\n     * Get the transfer rate per second\n     *\n     *  @return transfer bytes in second\n     */\n    long getTransferredByteInSecond();\n\n    /**\n     * Get the current transfer offset to the slave\n     *\n     * @return the current transfer offset to the slave\n     */\n    long getTransferFromWhere();\n\n    /**\n     * Get slave ack offset\n     *\n     * @return slave ack offset\n     */\n    long getSlaveAckOffset();\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/HAConnectionState.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha;\n\npublic enum HAConnectionState {\n    /**\n     * Ready to start connection.\n     */\n    READY,\n    /**\n     * CommitLog consistency checking.\n     */\n    HANDSHAKE,\n    /**\n     * Synchronizing data.\n     */\n    TRANSFER,\n    /**\n     * Temporarily stop transferring.\n     */\n    SUSPEND,\n    /**\n     * Connection shutdown.\n     */\n    SHUTDOWN,\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/HAConnectionStateNotificationRequest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha;\n\nimport java.util.concurrent.CompletableFuture;\n\npublic class HAConnectionStateNotificationRequest {\n    private final CompletableFuture<Boolean> requestFuture = new CompletableFuture<>();\n    private final HAConnectionState expectState;\n    private final String remoteAddr;\n    private final boolean notifyWhenShutdown;\n\n    public HAConnectionStateNotificationRequest(HAConnectionState expectState, String remoteAddr, boolean notifyWhenShutdown) {\n        this.expectState = expectState;\n        this.remoteAddr = remoteAddr;\n        this.notifyWhenShutdown = notifyWhenShutdown;\n    }\n\n    public CompletableFuture<Boolean> getRequestFuture() {\n        return requestFuture;\n    }\n\n    public String getRemoteAddr() {\n        return remoteAddr;\n    }\n\n    public boolean isNotifyWhenShutdown() {\n        return notifyWhenShutdown;\n    }\n\n    public HAConnectionState getExpectState() {\n        return expectState;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/HAConnectionStateNotificationService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha;\n\nimport java.net.InetSocketAddress;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.config.BrokerRole;\n\n/**\n * Service to periodically check and notify for certain connection state.\n */\npublic class HAConnectionStateNotificationService extends ServiceThread {\n\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n\n    private static final long CONNECTION_ESTABLISH_TIMEOUT = 10 * 1000;\n\n    private volatile HAConnectionStateNotificationRequest request;\n    private volatile long lastCheckTimeStamp = -1;\n    private HAService haService;\n    private DefaultMessageStore defaultMessageStore;\n\n    public HAConnectionStateNotificationService(HAService haService, DefaultMessageStore defaultMessageStore) {\n        this.haService = haService;\n        this.defaultMessageStore = defaultMessageStore;\n    }\n\n    @Override\n    public String getServiceName() {\n        if (defaultMessageStore != null && defaultMessageStore.getBrokerConfig().isInBrokerContainer()) {\n            return defaultMessageStore.getBrokerIdentity().getIdentifier() + HAConnectionStateNotificationService.class.getSimpleName();\n        }\n        return HAConnectionStateNotificationService.class.getSimpleName();\n    }\n\n    public synchronized void setRequest(HAConnectionStateNotificationRequest request) {\n        if (this.request != null) {\n            this.request.getRequestFuture().cancel(true);\n        }\n        this.request = request;\n        lastCheckTimeStamp = System.currentTimeMillis();\n    }\n\n    private synchronized void doWaitConnectionState() {\n        if (this.request == null || this.request.getRequestFuture().isDone()) {\n            return;\n        }\n\n        if (this.defaultMessageStore.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) {\n            if (haService.getHAClient().getCurrentState() == this.request.getExpectState()) {\n                this.request.getRequestFuture().complete(true);\n                this.request = null;\n            } else if (haService.getHAClient().getCurrentState() == HAConnectionState.READY) {\n                if ((System.currentTimeMillis() - lastCheckTimeStamp) > CONNECTION_ESTABLISH_TIMEOUT) {\n                    LOGGER.error(\"Wait HA connection establish with {} timeout\", this.request.getRemoteAddr());\n                    this.request.getRequestFuture().complete(false);\n                    this.request = null;\n                }\n            } else {\n                lastCheckTimeStamp = System.currentTimeMillis();\n            }\n        } else {\n            boolean connectionFound = false;\n            for (HAConnection connection : haService.getConnectionList()) {\n                if (checkConnectionStateAndNotify(connection)) {\n                    connectionFound = true;\n                }\n            }\n\n            if (connectionFound) {\n                lastCheckTimeStamp = System.currentTimeMillis();\n            }\n\n            if (!connectionFound && (System.currentTimeMillis() - lastCheckTimeStamp) > CONNECTION_ESTABLISH_TIMEOUT) {\n                LOGGER.error(\"Wait HA connection establish with {} timeout\", this.request.getRemoteAddr());\n                this.request.getRequestFuture().complete(false);\n                this.request = null;\n            }\n        }\n    }\n\n    /**\n     * Check if connection matched and notify request.\n     *\n     * @param connection connection to check.\n     * @return if connection remote address match request.\n     */\n    public synchronized boolean checkConnectionStateAndNotify(HAConnection connection) {\n        if (this.request == null || connection == null) {\n            return false;\n        }\n\n        String remoteAddress;\n        try {\n            remoteAddress = ((InetSocketAddress) connection.getSocketChannel().getRemoteAddress())\n                .getAddress().getHostAddress();\n            if (remoteAddress.equals(request.getRemoteAddr())) {\n                HAConnectionState connState = connection.getCurrentState();\n\n                if (connState == this.request.getExpectState()) {\n                    this.request.getRequestFuture().complete(true);\n                    this.request = null;\n                } else if (this.request.isNotifyWhenShutdown() && connState == HAConnectionState.SHUTDOWN) {\n                    this.request.getRequestFuture().complete(false);\n                    this.request = null;\n                }\n                return true;\n            }\n        } catch (Exception e) {\n            LOGGER.error(\"Check connection address exception: {}\", e);\n        }\n\n        return false;\n    }\n\n    @Override\n    public void run() {\n        LOGGER.info(this.getServiceName() + \" service started\");\n\n        while (!this.isStopped()) {\n            try {\n                this.waitForRunning(1000);\n                this.doWaitConnectionState();\n            } catch (Exception e) {\n                LOGGER.warn(this.getServiceName() + \" service has exception. \", e);\n            }\n        }\n\n        LOGGER.info(this.getServiceName() + \" service end\");\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/HAService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha;\n\nimport java.io.IOException;\nimport java.util.List;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;\nimport org.apache.rocketmq.store.CommitLog;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.rocksdb.RocksDBException;\n\npublic interface HAService {\n\n    /**\n     * Init HAService, must be called before other methods.\n     *\n     * @param defaultMessageStore\n     * @throws IOException\n     */\n    void init(DefaultMessageStore defaultMessageStore) throws IOException;\n\n    /**\n     * Start HA Service\n     *\n     * @throws Exception\n     */\n    void start() throws Exception;\n\n    /**\n     * Shutdown HA Service\n     */\n    void shutdown();\n\n    /**\n     * Change to master state\n     *\n     * @param masterEpoch the new masterEpoch\n     */\n    default boolean changeToMaster(int masterEpoch) throws RocksDBException {\n        return false;\n    }\n\n    /**\n     * Change to master state\n     *\n     * @param masterEpoch the new masterEpoch\n     */\n    default boolean changeToMasterWhenLastRoleIsMaster(int masterEpoch) {\n        return false;\n    }\n\n    /**\n     * Change to slave state\n     *\n     * @param newMasterAddr new master addr\n     * @param newMasterEpoch new masterEpoch\n     */\n    default boolean changeToSlave(String newMasterAddr, int newMasterEpoch, Long slaveId) {\n        return false;\n    }\n\n    /**\n     * Change to slave state\n     *\n     * @param newMasterAddr new master addr\n     * @param newMasterEpoch new masterEpoch\n     */\n    default boolean changeToSlaveWhenMasterNotChange(String newMasterAddr, int newMasterEpoch) {\n        return false;\n    }\n\n    /**\n     * Update master address\n     *\n     * @param newAddr\n     */\n    void updateMasterAddress(String newAddr);\n\n    /**\n     * Update ha master address\n     *\n     * @param newAddr\n     */\n    void updateHaMasterAddress(String newAddr);\n\n    /**\n     * Returns the number of replicas those commit log are not far behind the master. It includes master itself. Returns\n     * syncStateSet size if HAService instanceof AutoSwitchService\n     *\n     * @return the number of slaves\n     * @see MessageStoreConfig#getHaMaxGapNotInSync()\n     */\n    int inSyncReplicasNums(long masterPutWhere);\n\n    /**\n     * Get connection count\n     *\n     * @return the number of connection\n     */\n    AtomicInteger getConnectionCount();\n\n    /**\n     * Put request to handle HA\n     *\n     * @param request\n     */\n    void putRequest(final CommitLog.GroupCommitRequest request);\n\n    /**\n     * Put GroupConnectionStateRequest for preOnline\n     *\n     * @param request\n     */\n    void putGroupConnectionStateRequest(HAConnectionStateNotificationRequest request);\n\n    /**\n     * Get ha connection list\n     *\n     * @return List<HAConnection>\n     */\n    List<HAConnection> getConnectionList();\n\n    /**\n     * Get HAClient\n     *\n     * @return HAClient\n     */\n    HAClient getHAClient();\n\n    /**\n     * Get the max offset in all slaves\n     */\n    AtomicLong getPush2SlaveMaxOffset();\n\n    /**\n     * Get HA runtime info\n     */\n    HARuntimeInfo getRuntimeInfo(final long masterPutWhere);\n\n    /**\n     * Get WaitNotifyObject\n     */\n    WaitNotifyObject getWaitNotifyObject();\n\n    /**\n     * Judge whether the slave keeps up according to the masterPutWhere, If the offset gap exceeds haSlaveFallBehindMax,\n     * then slave is not OK\n     */\n    boolean isSlaveOK(long masterPutWhere);\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/WaitNotifyObject.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.ha;\n\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.ConcurrentHashMapUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.atomic.AtomicBoolean;\n\npublic class WaitNotifyObject {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n\n    protected final ConcurrentHashMap<Long/* thread id */, AtomicBoolean/* notified */> waitingThreadTable =\n        new ConcurrentHashMap<>(16);\n\n    protected AtomicBoolean hasNotified = new AtomicBoolean(false);\n\n    public void wakeup() {\n        boolean needNotify = hasNotified.compareAndSet(false, true);\n        if (needNotify) {\n            synchronized (this) {\n                this.notify();\n            }\n        }\n    }\n\n    protected void waitForRunning(long interval) {\n        if (this.hasNotified.compareAndSet(true, false)) {\n            this.onWaitEnd();\n            return;\n        }\n        synchronized (this) {\n            try {\n                if (this.hasNotified.compareAndSet(true, false)) {\n                    this.onWaitEnd();\n                    return;\n                }\n                this.wait(interval);\n            } catch (InterruptedException e) {\n                log.error(\"Interrupted\", e);\n            } finally {\n                this.hasNotified.set(false);\n                this.onWaitEnd();\n            }\n        }\n    }\n\n    protected void onWaitEnd() {\n    }\n\n    public void wakeupAll() {\n        boolean needNotify = false;\n        for (Map.Entry<Long, AtomicBoolean> entry : this.waitingThreadTable.entrySet()) {\n            if (entry.getValue().compareAndSet(false, true)) {\n                needNotify = true;\n            }\n        }\n        if (needNotify) {\n            synchronized (this) {\n                this.notifyAll();\n            }\n        }\n    }\n\n    public void allWaitForRunning(long interval) {\n        long currentThreadId = Thread.currentThread().getId();\n        AtomicBoolean notified = ConcurrentHashMapUtils.computeIfAbsent(this.waitingThreadTable, currentThreadId, k -> new AtomicBoolean(false));\n        if (notified.compareAndSet(true, false)) {\n            this.onWaitEnd();\n            return;\n        }\n        synchronized (this) {\n            try {\n                if (notified.compareAndSet(true, false)) {\n                    this.onWaitEnd();\n                    return;\n                }\n                this.wait(interval);\n            } catch (InterruptedException e) {\n                log.error(\"Interrupted\", e);\n            } finally {\n                notified.set(false);\n                this.onWaitEnd();\n            }\n        }\n    }\n\n    public void removeFromWaitingThreadTable() {\n        long currentThreadId = Thread.currentThread().getId();\n        synchronized (this) {\n            this.waitingThreadTable.remove(currentThreadId);\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAClient.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha.autoswitch;\n\nimport java.io.IOException;\nimport java.net.SocketAddress;\nimport java.nio.ByteBuffer;\nimport java.nio.channels.SelectionKey;\nimport java.nio.channels.Selector;\nimport java.nio.channels.SocketChannel;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.atomic.AtomicReference;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.protocol.EpochEntry;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.ha.FlowMonitor;\nimport org.apache.rocketmq.store.ha.HAClient;\nimport org.apache.rocketmq.store.ha.HAConnectionState;\nimport org.apache.rocketmq.store.ha.io.AbstractHAReader;\nimport org.apache.rocketmq.store.ha.io.HAWriter;\n\npublic class AutoSwitchHAClient extends ServiceThread implements HAClient {\n\n    /**\n     * Handshake header buffer size. Schema: state ordinal + Two flags + slaveBrokerId. Format:\n     *\n     * <pre>\n     *                   ┌──────────────────┬───────────────┐\n     *                   │isSyncFromLastFile│ isAsyncLearner│\n     *                   │     (2bytes)     │   (2bytes)    │\n     *                   └──────────────────┴───────────────┘\n     *                     \\                              /\n     *                      \\                            /\n     *                       ╲                          /\n     *                        ╲                        /\n     * ┌───────────────────────┬───────────────────────┬───────────────────────┐\n     * │      current state    │          Flags        │      slaveBrokerId    │\n     * │         (4bytes)      │         (4bytes)      │         (8bytes)      │\n     * ├───────────────────────┴───────────────────────┴───────────────────────┤\n     * │                                                                       │\n     * │                          HANDSHAKE  Header                            │\n     * </pre>\n     * <p>\n     * Flag: isSyncFromLastFile(short), isAsyncLearner(short)... we can add more flags in the future if needed\n     */\n    public static final int HANDSHAKE_HEADER_SIZE = 4 + 4 + 8;\n\n    /**\n     * Header + slaveAddress, Format:\n     * <pre>\n     *                   ┌──────────────────┬───────────────┐\n     *                   │isSyncFromLastFile│ isAsyncLearner│\n     *                   │     (2bytes)     │   (2bytes)    │\n     *                   └──────────────────┴───────────────┘\n     *                     \\                              /\n     *                      \\                            /\n     *                       ╲                          /\n     *                        ╲                        /\n     * ┌───────────────────────┬───────────────────────┬───────────────────────┬───────────────────────────────┐\n     * │      current state    │          Flags        │  slaveAddressLength   │          slaveAddress         │\n     * │         (4bytes)      │         (4bytes)      │         (4bytes)      │             (50bytes)         │\n     * ├───────────────────────┴───────────────────────┴───────────────────────┼───────────────────────────────┤\n     * │                                                                       │                               │\n     * │                        HANDSHAKE  Header                              │               body            │\n     * </pre>\n     */\n    @Deprecated\n    public static final int HANDSHAKE_SIZE = HANDSHAKE_HEADER_SIZE + 50;\n\n    /**\n     * Transfer header buffer size. Schema: state ordinal + maxOffset. Format:\n     * <pre>\n     * ┌───────────────────────┬───────────────────────┐\n     * │      current state    │        maxOffset      │\n     * │         (4bytes)      │         (8bytes)      │\n     * ├───────────────────────┴───────────────────────┤\n     * │                                               │\n     * │                TRANSFER  Header               │\n     * </pre>\n     */\n    public static final int TRANSFER_HEADER_SIZE = 4 + 8;\n    public static final int MIN_HEADER_SIZE = Math.min(HANDSHAKE_HEADER_SIZE, TRANSFER_HEADER_SIZE);\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private static final int READ_MAX_BUFFER_SIZE = 1024 * 1024 * 4;\n    private final AtomicReference<String> masterHaAddress = new AtomicReference<>();\n    private final AtomicReference<String> masterAddress = new AtomicReference<>();\n    private final ByteBuffer handshakeHeaderBuffer = ByteBuffer.allocate(HANDSHAKE_HEADER_SIZE);\n    private final ByteBuffer transferHeaderBuffer = ByteBuffer.allocate(TRANSFER_HEADER_SIZE);\n    private final AutoSwitchHAService haService;\n    private final ByteBuffer byteBufferRead = ByteBuffer.allocate(READ_MAX_BUFFER_SIZE);\n    private final DefaultMessageStore messageStore;\n    private final EpochFileCache epochCache;\n\n    private final Long brokerId;\n\n    private SocketChannel socketChannel;\n    private Selector selector;\n    private AbstractHAReader haReader;\n    private HAWriter haWriter;\n    private FlowMonitor flowMonitor;\n    /**\n     * last time that slave reads date from master.\n     */\n    private long lastReadTimestamp;\n    /**\n     * last time that slave reports offset to master.\n     */\n    private long lastWriteTimestamp;\n\n    private long currentReportedOffset;\n    private int processPosition;\n    private volatile HAConnectionState currentState;\n    /**\n     * Current epoch\n     */\n    private volatile int currentReceivedEpoch;\n\n    public AutoSwitchHAClient(AutoSwitchHAService haService, DefaultMessageStore defaultMessageStore,\n        EpochFileCache epochCache, Long brokerId) throws IOException {\n        this.haService = haService;\n        this.messageStore = defaultMessageStore;\n        this.epochCache = epochCache;\n        this.brokerId = brokerId;\n        init();\n    }\n\n    public void init() throws IOException {\n        this.selector = NetworkUtil.openSelector();\n        this.flowMonitor = new FlowMonitor(this.messageStore.getMessageStoreConfig());\n        this.haReader = new HAClientReader();\n        haReader.registerHook(readSize -> {\n            if (readSize > 0) {\n                AutoSwitchHAClient.this.flowMonitor.addByteCountTransferred(readSize);\n                lastReadTimestamp = System.currentTimeMillis();\n            }\n        });\n        this.haWriter = new HAWriter();\n        haWriter.registerHook(writeSize -> {\n            if (writeSize > 0) {\n                lastWriteTimestamp = System.currentTimeMillis();\n            }\n        });\n        changeCurrentState(HAConnectionState.READY);\n        this.currentReceivedEpoch = -1;\n        this.currentReportedOffset = 0;\n        this.processPosition = 0;\n        this.lastReadTimestamp = System.currentTimeMillis();\n        this.lastWriteTimestamp = System.currentTimeMillis();\n    }\n\n    public void reOpen() throws IOException {\n        shutdown();\n        init();\n    }\n\n    @Override\n    public String getServiceName() {\n        if (haService.getDefaultMessageStore().getBrokerConfig().isInBrokerContainer()) {\n            return haService.getDefaultMessageStore().getBrokerIdentity().getIdentifier() + AutoSwitchHAClient.class.getSimpleName();\n        }\n        return AutoSwitchHAClient.class.getSimpleName();\n    }\n\n    @Override\n    public void updateMasterAddress(String newAddress) {\n        String currentAddr = this.masterAddress.get();\n        if (!StringUtils.equals(newAddress, currentAddr) && masterAddress.compareAndSet(currentAddr, newAddress)) {\n            LOGGER.info(\"update master address, OLD: \" + currentAddr + \" NEW: \" + newAddress);\n        }\n    }\n\n    @Override\n    public void updateHaMasterAddress(String newAddress) {\n        String currentAddr = this.masterHaAddress.get();\n        if (!StringUtils.equals(newAddress, currentAddr) && masterHaAddress.compareAndSet(currentAddr, newAddress)) {\n            LOGGER.info(\"update master ha address, OLD: \" + currentAddr + \" NEW: \" + newAddress);\n            wakeup();\n        }\n    }\n\n    @Override\n    public String getMasterAddress() {\n        return this.masterAddress.get();\n    }\n\n    @Override\n    public String getHaMasterAddress() {\n        return this.masterHaAddress.get();\n    }\n\n    @Override\n    public long getLastReadTimestamp() {\n        return this.lastReadTimestamp;\n    }\n\n    @Override\n    public long getLastWriteTimestamp() {\n        return this.lastWriteTimestamp;\n    }\n\n    @Override\n    public HAConnectionState getCurrentState() {\n        return this.currentState;\n    }\n\n    @Override\n    public void changeCurrentState(HAConnectionState haConnectionState) {\n        LOGGER.info(\"change state to {}\", haConnectionState);\n        this.currentState = haConnectionState;\n    }\n\n    public void closeMasterAndWait() {\n        this.closeMaster();\n        this.waitForRunning(1000 * 5);\n    }\n\n    @Override\n    public void closeMaster() {\n        if (null != this.socketChannel) {\n            try {\n                SelectionKey sk = this.socketChannel.keyFor(this.selector);\n                if (sk != null) {\n                    sk.cancel();\n                }\n\n                this.socketChannel.close();\n                this.socketChannel = null;\n\n                LOGGER.info(\"AutoSwitchHAClient close connection with master {}\", this.masterHaAddress.get());\n                this.changeCurrentState(HAConnectionState.READY);\n            } catch (IOException e) {\n                LOGGER.warn(\"CloseMaster exception. \", e);\n            }\n\n            this.lastReadTimestamp = 0;\n            this.processPosition = 0;\n\n            this.byteBufferRead.position(0);\n            this.byteBufferRead.limit(READ_MAX_BUFFER_SIZE);\n        }\n    }\n\n    @Override\n    public long getTransferredByteInSecond() {\n        return this.flowMonitor.getTransferredByteInSecond();\n    }\n\n    @Override\n    public void shutdown() {\n        changeCurrentState(HAConnectionState.SHUTDOWN);\n        // Shutdown thread firstly\n        this.flowMonitor.shutdown();\n        super.shutdown();\n\n        closeMaster();\n        try {\n            this.selector.close();\n        } catch (IOException e) {\n            LOGGER.warn(\"Close the selector of AutoSwitchHAClient error, \", e);\n        }\n    }\n\n    private boolean isTimeToReportOffset() {\n        long interval = this.messageStore.now() - this.lastWriteTimestamp;\n        return interval > this.messageStore.getMessageStoreConfig().getHaSendHeartbeatInterval();\n    }\n\n    private boolean sendHandshakeHeader() throws IOException {\n        this.handshakeHeaderBuffer.position(0);\n        this.handshakeHeaderBuffer.limit(HANDSHAKE_HEADER_SIZE);\n        // Original state\n        this.handshakeHeaderBuffer.putInt(HAConnectionState.HANDSHAKE.ordinal());\n        // IsSyncFromLastFile\n        short isSyncFromLastFile = this.haService.getDefaultMessageStore().getMessageStoreConfig().isSyncFromLastFile() ? (short) 1 : (short) 0;\n        this.handshakeHeaderBuffer.putShort(isSyncFromLastFile);\n        // IsAsyncLearner role\n        short isAsyncLearner = this.haService.getDefaultMessageStore().getMessageStoreConfig().isAsyncLearner() ? (short) 1 : (short) 0;\n        this.handshakeHeaderBuffer.putShort(isAsyncLearner);\n        // Slave brokerId\n        this.handshakeHeaderBuffer.putLong(this.brokerId);\n\n        this.handshakeHeaderBuffer.flip();\n        return this.haWriter.write(this.socketChannel, this.handshakeHeaderBuffer);\n    }\n\n    private void handshakeWithMaster() throws IOException {\n        boolean result = this.sendHandshakeHeader();\n        if (!result) {\n            closeMasterAndWait();\n        }\n\n        this.selector.select(5000);\n\n        result = this.haReader.read(this.socketChannel, this.byteBufferRead);\n        if (!result) {\n            closeMasterAndWait();\n        }\n    }\n\n    private boolean reportSlaveOffset(HAConnectionState currentState, final long offsetToReport) throws IOException {\n        this.transferHeaderBuffer.position(0);\n        this.transferHeaderBuffer.limit(TRANSFER_HEADER_SIZE);\n        this.transferHeaderBuffer.putInt(currentState.ordinal());\n        this.transferHeaderBuffer.putLong(offsetToReport);\n        this.transferHeaderBuffer.flip();\n        return this.haWriter.write(this.socketChannel, this.transferHeaderBuffer);\n    }\n\n    private boolean reportSlaveMaxOffset(HAConnectionState currentState) throws IOException {\n        boolean result = true;\n        final long maxPhyOffset = this.messageStore.getMaxPhyOffset();\n        if (maxPhyOffset > this.currentReportedOffset) {\n            this.currentReportedOffset = maxPhyOffset;\n            result = reportSlaveOffset(currentState, this.currentReportedOffset);\n        }\n        return result;\n    }\n\n    public boolean connectMaster() throws IOException {\n        if (null == this.socketChannel) {\n            String addr = this.masterHaAddress.get();\n            if (StringUtils.isNotEmpty(addr)) {\n                SocketAddress socketAddress = NetworkUtil.string2SocketAddress(addr);\n                this.socketChannel = RemotingHelper.connect(socketAddress);\n                if (this.socketChannel != null) {\n                    this.socketChannel.register(this.selector, SelectionKey.OP_READ);\n                    LOGGER.info(\"AutoSwitchHAClient connect to master {}\", addr);\n                    changeCurrentState(HAConnectionState.HANDSHAKE);\n                }\n            }\n            this.currentReportedOffset = this.messageStore.getMaxPhyOffset();\n            this.lastReadTimestamp = System.currentTimeMillis();\n        }\n        return this.socketChannel != null;\n    }\n\n    private boolean transferFromMaster() throws IOException {\n        boolean result;\n        if (isTimeToReportOffset()) {\n            LOGGER.info(\"Slave report current offset {}\", this.currentReportedOffset);\n            result = reportSlaveOffset(HAConnectionState.TRANSFER, this.currentReportedOffset);\n            if (!result) {\n                return false;\n            }\n        }\n\n        this.selector.select(1000);\n\n        result = this.haReader.read(this.socketChannel, this.byteBufferRead);\n        if (!result) {\n            return false;\n        }\n\n        return this.reportSlaveMaxOffset(HAConnectionState.TRANSFER);\n    }\n\n    @Override\n    public void run() {\n        LOGGER.info(this.getServiceName() + \" service started\");\n\n        this.flowMonitor.start();\n        while (!this.isStopped()) {\n            try {\n                switch (this.currentState) {\n                    case SHUTDOWN:\n                        this.flowMonitor.shutdown(true);\n                        return;\n                    case READY:\n                        // Truncate invalid msg first\n                        final long truncateOffset = AutoSwitchHAClient.this.haService.truncateInvalidMsg();\n                        if (truncateOffset >= 0) {\n                            AutoSwitchHAClient.this.epochCache.truncateSuffixByOffset(truncateOffset);\n                        }\n                        if (!connectMaster()) {\n                            LOGGER.warn(\"AutoSwitchHAClient connect to master {} failed\", this.masterHaAddress.get());\n                            waitForRunning(1000 * 5);\n                        }\n                        continue;\n                    case HANDSHAKE:\n                        handshakeWithMaster();\n                        continue;\n                    case TRANSFER:\n                        if (!transferFromMaster()) {\n                            closeMasterAndWait();\n                            continue;\n                        }\n                        break;\n                    case SUSPEND:\n                    default:\n                        waitForRunning(1000 * 5);\n                        continue;\n                }\n                long interval = this.messageStore.now() - this.lastReadTimestamp;\n                if (interval > this.messageStore.getMessageStoreConfig().getHaHousekeepingInterval()) {\n                    LOGGER.warn(\"AutoSwitchHAClient, housekeeping, found this connection[\" + this.masterHaAddress\n                        + \"] expired, \" + interval);\n                    closeMaster();\n                    LOGGER.warn(\"AutoSwitchHAClient, master not response some time, so close connection\");\n                }\n            } catch (Exception e) {\n                LOGGER.warn(this.getServiceName() + \" service has exception. \", e);\n                closeMasterAndWait();\n            }\n        }\n\n        this.flowMonitor.shutdown(true);\n        LOGGER.info(this.getServiceName() + \" service end\");\n    }\n\n    /**\n     * Compare the master and slave's epoch file, find consistent point, do truncate.\n     */\n    private boolean doTruncate(List<EpochEntry> masterEpochEntries, long masterEndOffset) throws Exception {\n        if (this.epochCache.getEntrySize() == 0) {\n            // If epochMap is empty, means the broker is a new replicas\n            LOGGER.info(\"Slave local epochCache is empty, skip truncate log\");\n            changeCurrentState(HAConnectionState.TRANSFER);\n            this.currentReportedOffset = 0;\n        } else {\n            final EpochFileCache masterEpochCache = new EpochFileCache();\n            masterEpochCache.initCacheFromEntries(masterEpochEntries);\n            masterEpochCache.setLastEpochEntryEndOffset(masterEndOffset);\n            final List<EpochEntry> localEpochEntries = this.epochCache.getAllEntries();\n            final EpochFileCache localEpochCache = new EpochFileCache();\n            localEpochCache.initCacheFromEntries(localEpochEntries);\n            localEpochCache.setLastEpochEntryEndOffset(this.messageStore.getMaxPhyOffset());\n\n            LOGGER.info(\"master epoch entries is {}\", masterEpochCache.getAllEntries());\n            LOGGER.info(\"local epoch entries is {}\", localEpochEntries);\n\n            final long truncateOffset = localEpochCache.findConsistentPoint(masterEpochCache);\n\n            LOGGER.info(\"truncateOffset is {}\", truncateOffset);\n\n            if (truncateOffset < 0) {\n                // If truncateOffset < 0, means we can't find a consistent point\n                LOGGER.error(\"Failed to find a consistent point between masterEpoch:{} and slaveEpoch:{}\", masterEpochEntries, localEpochEntries);\n                return false;\n            }\n            if (!this.messageStore.truncateFiles(truncateOffset)) {\n                LOGGER.error(\"Failed to truncate slave log to {}\", truncateOffset);\n                return false;\n            }\n            final long maxPhyOffset = this.messageStore.getMaxPhyOffset();\n            if (truncateOffset < maxPhyOffset) {\n                this.epochCache.truncateSuffixByOffset(truncateOffset);\n                LOGGER.info(\"Truncate slave log to {} success, change to transfer state\", truncateOffset);\n            }\n            changeCurrentState(HAConnectionState.TRANSFER);\n            this.currentReportedOffset = truncateOffset;\n        }\n        if (!reportSlaveMaxOffset(HAConnectionState.TRANSFER)) {\n            LOGGER.error(\"AutoSwitchHAClient report max offset to master failed\");\n            return false;\n        }\n        return true;\n    }\n\n    class HAClientReader extends AbstractHAReader {\n\n        @Override\n        protected boolean processReadResult(ByteBuffer byteBufferRead) {\n            int readSocketPos = byteBufferRead.position();\n            try {\n                while (true) {\n                    int diff = byteBufferRead.position() - AutoSwitchHAClient.this.processPosition;\n                    if (diff >= AutoSwitchHAConnection.HANDSHAKE_HEADER_SIZE) {\n                        final int processPosition = AutoSwitchHAClient.this.processPosition;\n                        int masterState = byteBufferRead.getInt(processPosition + AutoSwitchHAConnection.HANDSHAKE_HEADER_SIZE - 20);\n                        int bodySize = byteBufferRead.getInt(processPosition + AutoSwitchHAConnection.HANDSHAKE_HEADER_SIZE - 16);\n                        long masterOffset = byteBufferRead.getLong(processPosition + AutoSwitchHAConnection.HANDSHAKE_HEADER_SIZE - 12);\n                        int masterEpoch = byteBufferRead.getInt(processPosition + AutoSwitchHAConnection.HANDSHAKE_HEADER_SIZE - 4);\n                        long masterEpochStartOffset = 0;\n                        long confirmOffset = 0;\n                        // If master send transfer header data, set masterEpochStartOffset and confirmOffset value.\n                        if (masterState == HAConnectionState.TRANSFER.ordinal() && diff >= AutoSwitchHAConnection.TRANSFER_HEADER_SIZE) {\n                            masterEpochStartOffset = byteBufferRead.getLong(processPosition + AutoSwitchHAConnection.TRANSFER_HEADER_SIZE - 16);\n                            confirmOffset = byteBufferRead.getLong(processPosition + AutoSwitchHAConnection.TRANSFER_HEADER_SIZE - 8);\n                        }\n                        if (masterState != AutoSwitchHAClient.this.currentState.ordinal()) {\n                            int headerSize = masterState == HAConnectionState.TRANSFER.ordinal() ? AutoSwitchHAConnection.TRANSFER_HEADER_SIZE : AutoSwitchHAConnection.HANDSHAKE_HEADER_SIZE;\n                            AutoSwitchHAClient.this.processPosition += headerSize + bodySize;\n                            AutoSwitchHAClient.this.waitForRunning(1);\n                            LOGGER.error(\"State not matched, masterState:{}, slaveState:{}, bodySize:{}, offset:{}, masterEpoch:{}, masterEpochStartOffset:{}, confirmOffset:{}\",\n                                HAConnectionState.values()[masterState], AutoSwitchHAClient.this.currentState, bodySize, masterOffset, masterEpoch, masterEpochStartOffset, confirmOffset);\n                            return false;\n                        }\n\n                        // Flag whether the received data is complete\n                        boolean isComplete = true;\n                        switch (AutoSwitchHAClient.this.currentState) {\n                            case HANDSHAKE: {\n                                if (diff < AutoSwitchHAConnection.HANDSHAKE_HEADER_SIZE + bodySize) {\n                                    // The received HANDSHAKE data is not complete\n                                    isComplete = false;\n                                    break;\n                                }\n                                AutoSwitchHAClient.this.processPosition += AutoSwitchHAConnection.HANDSHAKE_HEADER_SIZE;\n                                // Truncate log\n                                int entrySize = AutoSwitchHAConnection.EPOCH_ENTRY_SIZE;\n                                final int entryNums = bodySize / entrySize;\n                                final ArrayList<EpochEntry> epochEntries = new ArrayList<>(entryNums);\n                                for (int i = 0; i < entryNums; i++) {\n                                    int epoch = byteBufferRead.getInt(AutoSwitchHAClient.this.processPosition + i * entrySize);\n                                    long startOffset = byteBufferRead.getLong(AutoSwitchHAClient.this.processPosition + i * entrySize + 4);\n                                    epochEntries.add(new EpochEntry(epoch, startOffset));\n                                }\n                                byteBufferRead.position(readSocketPos);\n                                AutoSwitchHAClient.this.processPosition += bodySize;\n                                LOGGER.info(\"Receive handshake, masterMaxPosition {}, masterEpochEntries:{}, try truncate log\", masterOffset, epochEntries);\n                                if (!doTruncate(epochEntries, masterOffset)) {\n                                    waitForRunning(1000 * 2);\n                                    LOGGER.error(\"AutoSwitchHAClient truncate log failed in handshake state\");\n                                    return false;\n                                }\n                            }\n                            break;\n                            case TRANSFER: {\n                                if (diff < AutoSwitchHAConnection.TRANSFER_HEADER_SIZE + bodySize) {\n                                    // The received TRANSFER data is not complete\n                                    isComplete = false;\n                                    break;\n                                }\n                                byte[] bodyData = new byte[bodySize];\n                                byteBufferRead.position(AutoSwitchHAClient.this.processPosition + AutoSwitchHAConnection.TRANSFER_HEADER_SIZE);\n                                byteBufferRead.get(bodyData);\n                                byteBufferRead.position(readSocketPos);\n                                AutoSwitchHAClient.this.processPosition += AutoSwitchHAConnection.TRANSFER_HEADER_SIZE + bodySize;\n                                long slavePhyOffset = AutoSwitchHAClient.this.messageStore.getMaxPhyOffset();\n                                if (slavePhyOffset != 0) {\n                                    if (slavePhyOffset != masterOffset) {\n                                        LOGGER.error(\"master pushed offset not equal the max phy offset in slave, SLAVE: \"\n                                            + slavePhyOffset + \" MASTER: \" + masterOffset);\n                                        return false;\n                                    }\n                                }\n\n                                // If epoch changed\n                                if (masterEpoch != AutoSwitchHAClient.this.currentReceivedEpoch) {\n                                    AutoSwitchHAClient.this.currentReceivedEpoch = masterEpoch;\n                                    AutoSwitchHAClient.this.epochCache.appendEntry(new EpochEntry(masterEpoch, masterEpochStartOffset));\n                                }\n\n                                if (bodySize > 0) {\n                                    AutoSwitchHAClient.this.messageStore.appendToCommitLog(masterOffset, bodyData, 0, bodyData.length);\n                                }\n\n                                haService.getDefaultMessageStore().setConfirmOffset(Math.min(confirmOffset, messageStore.getMaxPhyOffset()));\n\n                                if (!reportSlaveMaxOffset(HAConnectionState.TRANSFER)) {\n                                    LOGGER.error(\"AutoSwitchHAClient report max offset to master failed\");\n                                    return false;\n                                }\n                                break;\n                            }\n                            default:\n                                break;\n                        }\n                        if (isComplete) {\n                            continue;\n                        }\n\n                    }\n\n                    if (!byteBufferRead.hasRemaining()) {\n                        byteBufferRead.position(AutoSwitchHAClient.this.processPosition);\n                        byteBufferRead.compact();\n                        AutoSwitchHAClient.this.processPosition = 0;\n                    }\n\n                    break;\n                }\n            } catch (final Exception e) {\n                LOGGER.error(\"Error when ha client process read request\", e);\n            }\n            return true;\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAConnection.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha.autoswitch;\n\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport java.nio.channels.SelectionKey;\nimport java.nio.channels.Selector;\nimport java.nio.channels.SocketChannel;\nimport java.util.List;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.netty.NettySystemConfig;\nimport org.apache.rocketmq.remoting.protocol.EpochEntry;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.ha.FlowMonitor;\nimport org.apache.rocketmq.store.ha.HAConnection;\nimport org.apache.rocketmq.store.ha.HAConnectionState;\nimport org.apache.rocketmq.store.ha.io.AbstractHAReader;\nimport org.apache.rocketmq.store.ha.io.HAWriter;\n\npublic class AutoSwitchHAConnection implements HAConnection {\n\n    /**\n     * Handshake data protocol in syncing msg from master. Format:\n     * <pre>\n     * ┌─────────────────┬───────────────┬───────────┬───────────┬────────────────────────────────────┐\n     * │  current state  │   body size   │   offset  │   epoch   │   EpochEntrySize * EpochEntryNums  │\n     * │     (4bytes)    │   (4bytes)    │  (8bytes) │  (4bytes) │      (12bytes * EpochEntryNums)    │\n     * ├─────────────────┴───────────────┴───────────┴───────────┼────────────────────────────────────┤\n     * │                       Header                            │             Body                   │\n     * │                                                         │                                    │\n     * </pre>\n     * Handshake Header protocol Format:\n     * current state + body size + offset + epoch\n     */\n    public static final int HANDSHAKE_HEADER_SIZE = 4 + 4 + 8 + 4;\n\n    /**\n     * Transfer data protocol in syncing msg from master. Format:\n     * <pre>\n     * ┌─────────────────┬───────────────┬───────────┬───────────┬─────────────────────┬──────────────────┬──────────────────┐\n     * │  current state  │   body size   │   offset  │   epoch   │   epochStartOffset  │   confirmOffset  │    log data      │\n     * │     (4bytes)    │   (4bytes)    │  (8bytes) │  (4bytes) │      (8bytes)       │      (8bytes)    │   (data size)    │\n     * ├─────────────────┴───────────────┴───────────┴───────────┴─────────────────────┴──────────────────┼──────────────────┤\n     * │                                               Header                                             │       Body       │\n     * │                                                                                                  │                  │\n     * </pre>\n     * Transfer Header protocol Format:\n     * current state + body size + offset + epoch  + epochStartOffset + additionalInfo(confirmOffset)\n     */\n    public static final int TRANSFER_HEADER_SIZE = HANDSHAKE_HEADER_SIZE + 8 + 8;\n    public static final int EPOCH_ENTRY_SIZE = 12;\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private final AutoSwitchHAService haService;\n    private final SocketChannel socketChannel;\n    private final String clientAddress;\n    private final EpochFileCache epochCache;\n    private final AbstractWriteSocketService writeSocketService;\n    private final ReadSocketService readSocketService;\n    private final FlowMonitor flowMonitor;\n\n    private volatile HAConnectionState currentState = HAConnectionState.HANDSHAKE;\n    private volatile long slaveRequestOffset = -1;\n    private volatile long slaveAckOffset = -1;\n    /**\n     * Whether the slave have already sent a handshake message\n     */\n    private volatile boolean isSlaveSendHandshake = false;\n    private volatile int currentTransferEpoch = -1;\n    private volatile long currentTransferEpochEndOffset = 0;\n    private volatile boolean isSyncFromLastFile = false;\n    private volatile boolean isAsyncLearner = false;\n    private volatile long slaveId = -1;\n\n    /**\n     * Last endOffset when master transfer data to slave\n     */\n    private volatile long lastMasterMaxOffset = -1;\n    /**\n     * Last time ms when transfer data to slave.\n     */\n    private volatile long lastTransferTimeMs = 0;\n\n    public AutoSwitchHAConnection(AutoSwitchHAService haService, SocketChannel socketChannel,\n        EpochFileCache epochCache) throws IOException {\n        this.haService = haService;\n        this.socketChannel = socketChannel;\n        this.epochCache = epochCache;\n        this.clientAddress = this.socketChannel.socket().getRemoteSocketAddress().toString();\n        this.socketChannel.configureBlocking(false);\n        this.socketChannel.socket().setSoLinger(false, -1);\n        this.socketChannel.socket().setTcpNoDelay(true);\n        if (NettySystemConfig.socketSndbufSize > 0) {\n            this.socketChannel.socket().setReceiveBufferSize(NettySystemConfig.socketSndbufSize);\n        }\n        if (NettySystemConfig.socketRcvbufSize > 0) {\n            this.socketChannel.socket().setSendBufferSize(NettySystemConfig.socketRcvbufSize);\n        }\n        this.writeSocketService = new WriteSocketService(this.socketChannel);\n        this.readSocketService = new ReadSocketService(this.socketChannel);\n        this.haService.getConnectionCount().incrementAndGet();\n        this.flowMonitor = new FlowMonitor(haService.getDefaultMessageStore().getMessageStoreConfig());\n    }\n\n    @Override\n    public void start() {\n        changeCurrentState(HAConnectionState.HANDSHAKE);\n        this.flowMonitor.start();\n        this.readSocketService.start();\n        this.writeSocketService.start();\n    }\n\n    @Override\n    public void shutdown() {\n        changeCurrentState(HAConnectionState.SHUTDOWN);\n        this.flowMonitor.shutdown(true);\n        this.writeSocketService.shutdown(true);\n        this.readSocketService.shutdown(true);\n        this.close();\n    }\n\n    @Override\n    public void close() {\n        if (this.socketChannel != null) {\n            try {\n                this.socketChannel.close();\n            } catch (final IOException e) {\n                LOGGER.error(\"\", e);\n            }\n        }\n    }\n\n    public void changeCurrentState(HAConnectionState connectionState) {\n        LOGGER.info(\"change state to {}\", connectionState);\n        this.currentState = connectionState;\n    }\n\n    public long getSlaveId() {\n        return slaveId;\n    }\n\n    @Override\n    public HAConnectionState getCurrentState() {\n        return currentState;\n    }\n\n    @Override\n    public SocketChannel getSocketChannel() {\n        return socketChannel;\n    }\n\n    @Override\n    public String getClientAddress() {\n        return clientAddress;\n    }\n\n    @Override\n    public long getSlaveAckOffset() {\n        return slaveAckOffset;\n    }\n\n    @Override\n    public long getTransferredByteInSecond() {\n        return flowMonitor.getTransferredByteInSecond();\n    }\n\n    @Override\n    public long getTransferFromWhere() {\n        return this.writeSocketService.getNextTransferFromWhere();\n    }\n\n    private void changeTransferEpochToNext(final EpochEntry entry) {\n        this.currentTransferEpoch = entry.getEpoch();\n        this.currentTransferEpochEndOffset = entry.getEndOffset();\n        if (entry.getEpoch() == this.epochCache.lastEpoch()) {\n            // Use -1 to stand for Long.max\n            this.currentTransferEpochEndOffset = -1;\n        }\n    }\n\n    public boolean isAsyncLearner() {\n        return isAsyncLearner;\n    }\n\n    public boolean isSyncFromLastFile() {\n        return isSyncFromLastFile;\n    }\n\n    private synchronized void updateLastTransferInfo() {\n        this.lastMasterMaxOffset = this.haService.getDefaultMessageStore().getMaxPhyOffset();\n        this.lastTransferTimeMs = System.currentTimeMillis();\n    }\n\n    private synchronized void maybeExpandInSyncStateSet(long slaveMaxOffset) {\n        if (!this.isAsyncLearner && slaveMaxOffset >= this.lastMasterMaxOffset) {\n            long caughtUpTimeMs = this.haService.getDefaultMessageStore().getMaxPhyOffset() == slaveMaxOffset ? System.currentTimeMillis() : this.lastTransferTimeMs;\n            this.haService.updateConnectionLastCaughtUpTime(this.slaveId, caughtUpTimeMs);\n            this.haService.maybeExpandInSyncStateSet(this.slaveId, slaveMaxOffset);\n        }\n    }\n\n    class ReadSocketService extends ServiceThread {\n        private static final int READ_MAX_BUFFER_SIZE = 1024 * 1024;\n        private final Selector selector;\n        private final SocketChannel socketChannel;\n        private final ByteBuffer byteBufferRead = ByteBuffer.allocate(READ_MAX_BUFFER_SIZE);\n        private final AbstractHAReader haReader;\n        private int processPosition = 0;\n        private volatile long lastReadTimestamp = System.currentTimeMillis();\n\n        public ReadSocketService(final SocketChannel socketChannel) throws IOException {\n            this.selector = NetworkUtil.openSelector();\n            this.socketChannel = socketChannel;\n            this.socketChannel.register(this.selector, SelectionKey.OP_READ);\n            this.setDaemon(true);\n            haReader = new HAServerReader();\n            haReader.registerHook(readSize -> {\n                if (readSize > 0) {\n                    ReadSocketService.this.lastReadTimestamp =\n                        haService.getDefaultMessageStore().getSystemClock().now();\n                }\n            });\n        }\n\n        @Override\n        public void run() {\n            LOGGER.info(this.getServiceName() + \" service started\");\n\n            while (!this.isStopped()) {\n                try {\n                    this.selector.select(1000);\n                    boolean ok = this.haReader.read(this.socketChannel, this.byteBufferRead);\n                    if (!ok) {\n                        AutoSwitchHAConnection.LOGGER.error(\"processReadEvent error\");\n                        break;\n                    }\n\n                    long interval = haService.getDefaultMessageStore().getSystemClock().now() - this.lastReadTimestamp;\n                    if (interval > haService.getDefaultMessageStore().getMessageStoreConfig().getHaHousekeepingInterval()) {\n                        LOGGER.warn(\"ha housekeeping, found this connection[\" + clientAddress + \"] expired, \" + interval);\n                        break;\n                    }\n                } catch (Exception e) {\n                    AutoSwitchHAConnection.LOGGER.error(this.getServiceName() + \" service has exception.\", e);\n                    break;\n                }\n            }\n\n            this.makeStop();\n\n            changeCurrentState(HAConnectionState.SHUTDOWN);\n\n            writeSocketService.makeStop();\n\n            haService.removeConnection(AutoSwitchHAConnection.this);\n\n            haService.getConnectionCount().decrementAndGet();\n\n            SelectionKey sk = this.socketChannel.keyFor(this.selector);\n            if (sk != null) {\n                sk.cancel();\n            }\n\n            try {\n                this.selector.close();\n                this.socketChannel.close();\n            } catch (IOException e) {\n                AutoSwitchHAConnection.LOGGER.error(\"\", e);\n            }\n\n            flowMonitor.shutdown(true);\n\n            AutoSwitchHAConnection.LOGGER.info(this.getServiceName() + \" service end\");\n        }\n\n        @Override\n        public String getServiceName() {\n            if (haService.getDefaultMessageStore().getBrokerConfig().isInBrokerContainer()) {\n                return haService.getDefaultMessageStore().getBrokerIdentity().getIdentifier() + ReadSocketService.class.getSimpleName();\n            }\n            return ReadSocketService.class.getSimpleName();\n        }\n\n        class HAServerReader extends AbstractHAReader {\n            @Override\n            protected boolean processReadResult(ByteBuffer byteBufferRead) {\n                while (true) {\n                    boolean processSuccess = true;\n                    int readSocketPos = byteBufferRead.position();\n                    int diff = byteBufferRead.position() - ReadSocketService.this.processPosition;\n                    if (diff >= AutoSwitchHAClient.MIN_HEADER_SIZE) {\n                        int readPosition = ReadSocketService.this.processPosition;\n                        HAConnectionState slaveState = HAConnectionState.values()[byteBufferRead.getInt(readPosition)];\n\n                        switch (slaveState) {\n                            case HANDSHAKE:\n                                // SlaveBrokerId\n                                Long slaveBrokerId = byteBufferRead.getLong(readPosition + AutoSwitchHAClient.HANDSHAKE_HEADER_SIZE - 8);\n                                AutoSwitchHAConnection.this.slaveId = slaveBrokerId;\n                                // Flag(isSyncFromLastFile)\n                                short syncFromLastFileFlag = byteBufferRead.getShort(readPosition + AutoSwitchHAClient.HANDSHAKE_HEADER_SIZE - 12);\n                                if (syncFromLastFileFlag == 1) {\n                                    AutoSwitchHAConnection.this.isSyncFromLastFile = true;\n                                }\n                                // Flag(isAsyncLearner role)\n                                short isAsyncLearner = byteBufferRead.getShort(readPosition + AutoSwitchHAClient.HANDSHAKE_HEADER_SIZE - 10);\n                                if (isAsyncLearner == 1) {\n                                    AutoSwitchHAConnection.this.isAsyncLearner = true;\n                                }\n\n                                isSlaveSendHandshake = true;\n                                byteBufferRead.position(readSocketPos);\n                                ReadSocketService.this.processPosition += AutoSwitchHAClient.HANDSHAKE_HEADER_SIZE;\n                                LOGGER.info(\"Receive slave handshake, slaveBrokerId:{}, isSyncFromLastFile:{}, isAsyncLearner:{}\",\n                                    AutoSwitchHAConnection.this.slaveId, AutoSwitchHAConnection.this.isSyncFromLastFile, AutoSwitchHAConnection.this.isAsyncLearner);\n                                break;\n                            case TRANSFER:\n                                long slaveMaxOffset = byteBufferRead.getLong(readPosition + 4);\n                                ReadSocketService.this.processPosition += AutoSwitchHAClient.TRANSFER_HEADER_SIZE;\n\n                                AutoSwitchHAConnection.this.slaveAckOffset = slaveMaxOffset;\n                                if (slaveRequestOffset < 0) {\n                                    slaveRequestOffset = slaveMaxOffset;\n                                }\n                                byteBufferRead.position(readSocketPos);\n                                maybeExpandInSyncStateSet(slaveMaxOffset);\n                                AutoSwitchHAConnection.this.haService.updateConfirmOffsetWhenSlaveAck(AutoSwitchHAConnection.this.slaveId);\n                                AutoSwitchHAConnection.this.haService.notifyTransferSome(AutoSwitchHAConnection.this.slaveAckOffset);\n                                break;\n                            default:\n                                LOGGER.error(\"Current state illegal {}\", currentState);\n                                return false;\n                        }\n\n                        if (!slaveState.equals(currentState)) {\n                            LOGGER.warn(\"Master change state from {} to {}\", currentState, slaveState);\n                            changeCurrentState(slaveState);\n                        }\n                        if (processSuccess) {\n                            continue;\n                        }\n                    }\n\n                    if (!byteBufferRead.hasRemaining()) {\n                        byteBufferRead.position(ReadSocketService.this.processPosition);\n                        byteBufferRead.compact();\n                        ReadSocketService.this.processPosition = 0;\n                    }\n                    break;\n                }\n\n                return true;\n            }\n        }\n    }\n\n    class WriteSocketService extends AbstractWriteSocketService {\n        private SelectMappedBufferResult selectMappedBufferResult;\n\n        public WriteSocketService(final SocketChannel socketChannel) throws IOException {\n            super(socketChannel);\n        }\n\n        @Override\n        protected int getNextTransferDataSize() {\n            SelectMappedBufferResult selectResult = haService.getDefaultMessageStore().getCommitLogData(this.nextTransferFromWhere);\n            if (selectResult == null || selectResult.getSize() <= 0) {\n                return 0;\n            }\n            this.selectMappedBufferResult = selectResult;\n            return selectResult.getSize();\n        }\n\n        @Override\n        protected void releaseData() {\n            this.selectMappedBufferResult.release();\n            this.selectMappedBufferResult = null;\n        }\n\n        @Override\n        protected boolean transferData(int maxTransferSize) throws Exception {\n\n            if (null != this.selectMappedBufferResult && maxTransferSize >= 0) {\n                this.selectMappedBufferResult.getByteBuffer().limit(maxTransferSize);\n            }\n\n            // Write Header\n            boolean result = haWriter.write(this.socketChannel, this.byteBufferHeader);\n\n            if (!result) {\n                return false;\n            }\n\n            if (null == this.selectMappedBufferResult) {\n                return true;\n            }\n\n            // Write Body\n            result = haWriter.write(this.socketChannel, this.selectMappedBufferResult.getByteBuffer());\n\n            if (result) {\n                releaseData();\n            }\n            return result;\n        }\n\n        @Override\n        protected void onStop() {\n            if (this.selectMappedBufferResult != null) {\n                this.selectMappedBufferResult.release();\n            }\n        }\n\n        @Override\n        public String getServiceName() {\n            if (haService.getDefaultMessageStore().getBrokerConfig().isInBrokerContainer()) {\n                return haService.getDefaultMessageStore().getBrokerIdentity().getIdentifier() + WriteSocketService.class.getSimpleName();\n            }\n            return WriteSocketService.class.getSimpleName();\n        }\n    }\n\n    abstract class AbstractWriteSocketService extends ServiceThread {\n        protected final Selector selector;\n        protected final SocketChannel socketChannel;\n        protected final HAWriter haWriter;\n\n        protected final ByteBuffer byteBufferHeader = ByteBuffer.allocate(TRANSFER_HEADER_SIZE);\n        // Store master epochFileCache: (Epoch + startOffset) * 1000\n        private final ByteBuffer handShakeBuffer = ByteBuffer.allocate(EPOCH_ENTRY_SIZE * 1000);\n        protected long nextTransferFromWhere = -1;\n        protected boolean lastWriteOver = true;\n        protected long lastWriteTimestamp = System.currentTimeMillis();\n        protected long lastPrintTimestamp = System.currentTimeMillis();\n        protected long transferOffset = 0;\n\n        public AbstractWriteSocketService(final SocketChannel socketChannel) throws IOException {\n            this.selector = NetworkUtil.openSelector();\n            this.socketChannel = socketChannel;\n            this.socketChannel.register(this.selector, SelectionKey.OP_WRITE);\n            this.setDaemon(true);\n            haWriter = new HAWriter();\n            haWriter.registerHook(writeSize -> {\n                flowMonitor.addByteCountTransferred(writeSize);\n                if (writeSize > 0) {\n                    AbstractWriteSocketService.this.lastWriteTimestamp =\n                        haService.getDefaultMessageStore().getSystemClock().now();\n                }\n            });\n        }\n\n        public long getNextTransferFromWhere() {\n            return this.nextTransferFromWhere;\n        }\n\n        private boolean buildHandshakeBuffer() {\n            final List<EpochEntry> epochEntries = AutoSwitchHAConnection.this.epochCache.getAllEntries();\n            final int lastEpoch = AutoSwitchHAConnection.this.epochCache.lastEpoch();\n            final long maxPhyOffset = AutoSwitchHAConnection.this.haService.getDefaultMessageStore().getMaxPhyOffset();\n            this.byteBufferHeader.position(0);\n            this.byteBufferHeader.limit(HANDSHAKE_HEADER_SIZE);\n            // State\n            this.byteBufferHeader.putInt(currentState.ordinal());\n            // Body size\n            this.byteBufferHeader.putInt(epochEntries.size() * EPOCH_ENTRY_SIZE);\n            // Offset\n            this.byteBufferHeader.putLong(maxPhyOffset);\n            // Epoch\n            this.byteBufferHeader.putInt(lastEpoch);\n            this.byteBufferHeader.flip();\n\n            // EpochEntries\n            this.handShakeBuffer.position(0);\n            this.handShakeBuffer.limit(EPOCH_ENTRY_SIZE * epochEntries.size());\n            for (final EpochEntry entry : epochEntries) {\n                if (entry != null) {\n                    this.handShakeBuffer.putInt(entry.getEpoch());\n                    this.handShakeBuffer.putLong(entry.getStartOffset());\n                }\n            }\n            this.handShakeBuffer.flip();\n            LOGGER.info(\"Master build handshake header: maxEpoch:{}, maxOffset:{}, epochEntries:{}\", lastEpoch, maxPhyOffset, epochEntries);\n            return true;\n        }\n\n        private boolean handshakeWithSlave() throws IOException {\n            // Write Header\n            boolean result = this.haWriter.write(this.socketChannel, this.byteBufferHeader);\n\n            if (!result) {\n                return false;\n            }\n\n            // Write Body\n            return this.haWriter.write(this.socketChannel, this.handShakeBuffer);\n        }\n\n        // Normal transfer method\n        private void buildTransferHeaderBuffer(long nextOffset, int bodySize) {\n\n            EpochEntry entry = AutoSwitchHAConnection.this.epochCache.getEntry(AutoSwitchHAConnection.this.currentTransferEpoch);\n\n            if (entry == null) {\n\n                // If broker is started on empty disk and no message entered (nextOffset = -1 and currentTransferEpoch = -1), do not output error log when sending heartbeat\n                if (nextOffset != -1 || currentTransferEpoch != -1 || bodySize > 0) {\n                    LOGGER.error(\"Failed to find epochEntry with epoch {} when build msg header\", AutoSwitchHAConnection.this.currentTransferEpoch);\n                }\n\n                if (bodySize > 0) {\n                    return;\n                }\n                // Maybe it's used for heartbeat\n                entry = AutoSwitchHAConnection.this.epochCache.firstEntry();\n            }\n            // Build Header\n            this.byteBufferHeader.position(0);\n            this.byteBufferHeader.limit(TRANSFER_HEADER_SIZE);\n            // State\n            this.byteBufferHeader.putInt(currentState.ordinal());\n            // Body size\n            this.byteBufferHeader.putInt(bodySize);\n            // Offset\n            this.byteBufferHeader.putLong(nextOffset);\n            // Epoch\n            this.byteBufferHeader.putInt(entry.getEpoch());\n            // EpochStartOffset\n            this.byteBufferHeader.putLong(entry.getStartOffset());\n            // Additional info(confirm offset)\n            final long confirmOffset = AutoSwitchHAConnection.this.haService.getDefaultMessageStore().getConfirmOffset();\n            this.byteBufferHeader.putLong(confirmOffset);\n            this.byteBufferHeader.flip();\n        }\n\n        private boolean sendHeartbeatIfNeeded() throws Exception {\n            long interval = haService.getDefaultMessageStore().getSystemClock().now() - this.lastWriteTimestamp;\n            if (interval > haService.getDefaultMessageStore().getMessageStoreConfig().getHaSendHeartbeatInterval()) {\n                buildTransferHeaderBuffer(this.nextTransferFromWhere, 0);\n                return this.transferData(0);\n            }\n            return true;\n        }\n\n        private void transferToSlave() throws Exception {\n            if (this.lastWriteOver) {\n                this.lastWriteOver = sendHeartbeatIfNeeded();\n            } else {\n                // maxTransferSize == -1 means to continue transfer remaining data.\n                this.lastWriteOver = this.transferData(-1);\n            }\n            if (!this.lastWriteOver) {\n                return;\n            }\n\n            int size = this.getNextTransferDataSize();\n            if (size > 0) {\n                if (size > haService.getDefaultMessageStore().getMessageStoreConfig().getHaTransferBatchSize()) {\n                    size = haService.getDefaultMessageStore().getMessageStoreConfig().getHaTransferBatchSize();\n                }\n                int canTransferMaxBytes = flowMonitor.canTransferMaxByteNum();\n                if (size > canTransferMaxBytes) {\n                    if (System.currentTimeMillis() - lastPrintTimestamp > 1000) {\n                        LOGGER.warn(\"Trigger HA flow control, max transfer speed {}KB/s, current speed: {}KB/s\",\n                            String.format(\"%.2f\", flowMonitor.maxTransferByteInSecond() / 1024.0),\n                            String.format(\"%.2f\", flowMonitor.getTransferredByteInSecond() / 1024.0));\n                        lastPrintTimestamp = System.currentTimeMillis();\n                    }\n                    size = canTransferMaxBytes;\n                }\n                if (size <= 0) {\n                    this.releaseData();\n                    this.waitForRunning(100);\n                    return;\n                }\n\n                // Check and update currentTransferEpochEndOffset\n                if (AutoSwitchHAConnection.this.currentTransferEpochEndOffset == -1) {\n                    EpochEntry currentEpochEntry = AutoSwitchHAConnection.this.epochCache.getEntry(AutoSwitchHAConnection.this.currentTransferEpoch);\n                    if (currentEpochEntry != null) {\n                        if (currentEpochEntry.getEndOffset() != EpochEntry.LAST_EPOCH_END_OFFSET) {\n                            LOGGER.info(\"Update currentTransferEpochEndOffset from -1 to {}\", currentEpochEntry.getEndOffset());\n                            AutoSwitchHAConnection.this.currentTransferEpochEndOffset = currentEpochEntry.getEndOffset();\n                        }\n                    } else {\n                        // we should never reach here\n                        LOGGER.warn(\"[BUG]Can't find currentTransferEpoch [{}] from epoch cache\", currentTransferEpoch);\n                    }\n                }\n\n                // We must ensure that the transmitted logs are within the same epoch\n                // If currentEpochEndOffset == -1, means that currentTransferEpoch = last epoch, so the endOffset = Long.max\n                final long currentEpochEndOffset = AutoSwitchHAConnection.this.currentTransferEpochEndOffset;\n                if (currentEpochEndOffset != -1 && this.nextTransferFromWhere + size > currentEpochEndOffset) {\n                    final EpochEntry epochEntry = AutoSwitchHAConnection.this.epochCache.nextEntry(AutoSwitchHAConnection.this.currentTransferEpoch);\n                    if (epochEntry == null) {\n                        LOGGER.error(\"Can't find a bigger epochEntry than epoch {}\", AutoSwitchHAConnection.this.currentTransferEpoch);\n                        waitForRunning(100);\n                        return;\n                    }\n                    size = (int) (currentEpochEndOffset - this.nextTransferFromWhere);\n                    changeTransferEpochToNext(epochEntry);\n                }\n\n                this.transferOffset = this.nextTransferFromWhere;\n                this.nextTransferFromWhere += size;\n                updateLastTransferInfo();\n\n                // Build Header\n                buildTransferHeaderBuffer(this.transferOffset, size);\n\n                this.lastWriteOver = this.transferData(size);\n            } else {\n                // If size == 0, we should update the lastCatchupTimeMs\n                AutoSwitchHAConnection.this.haService.updateConnectionLastCaughtUpTime(AutoSwitchHAConnection.this.slaveId, System.currentTimeMillis());\n                haService.getWaitNotifyObject().allWaitForRunning(100);\n            }\n        }\n\n        @Override\n        public void run() {\n            AutoSwitchHAConnection.LOGGER.info(this.getServiceName() + \" service started\");\n\n            while (!this.isStopped()) {\n                try {\n                    this.selector.select(1000);\n\n                    switch (currentState) {\n                        case HANDSHAKE:\n                            // Wait until the slave send it handshake msg to master.\n                            if (!isSlaveSendHandshake) {\n                                this.waitForRunning(10);\n                                continue;\n                            }\n\n                            if (this.lastWriteOver) {\n                                if (!buildHandshakeBuffer()) {\n                                    LOGGER.error(\"AutoSwitchHAConnection build handshake buffer failed\");\n                                    this.waitForRunning(5000);\n                                    continue;\n                                }\n                            }\n\n                            this.lastWriteOver = handshakeWithSlave();\n                            if (this.lastWriteOver) {\n                                // change flag to {false} to wait for slave notification\n                                isSlaveSendHandshake = false;\n                            }\n                            break;\n                        case TRANSFER:\n                            if (-1 == slaveRequestOffset) {\n                                this.waitForRunning(10);\n                                continue;\n                            }\n\n                            if (-1 == this.nextTransferFromWhere) {\n                                if (0 == slaveRequestOffset) {\n                                    // We must ensure that the starting point of syncing log\n                                    // must be the startOffset of a file (maybe the last file, or the minOffset)\n                                    final MessageStoreConfig config = haService.getDefaultMessageStore().getMessageStoreConfig();\n                                    if (AutoSwitchHAConnection.this.isSyncFromLastFile) {\n                                        long masterOffset = haService.getDefaultMessageStore().getCommitLog().getMaxOffset();\n                                        masterOffset = masterOffset - (masterOffset % config.getMappedFileSizeCommitLog());\n                                        if (masterOffset < 0) {\n                                            masterOffset = 0;\n                                        }\n                                        this.nextTransferFromWhere = masterOffset;\n                                    } else {\n                                        this.nextTransferFromWhere = haService.getDefaultMessageStore().getCommitLog().getMinOffset();\n                                    }\n                                } else {\n                                    this.nextTransferFromWhere = slaveRequestOffset;\n                                }\n\n                                // nextTransferFromWhere is not found. It may be empty disk and no message is entered\n                                if (this.nextTransferFromWhere == -1) {\n                                    sendHeartbeatIfNeeded();\n                                    waitForRunning(500);\n                                    break;\n                                }\n                                // Setup initial transferEpoch\n                                EpochEntry epochEntry = AutoSwitchHAConnection.this.epochCache.findEpochEntryByOffset(this.nextTransferFromWhere);\n                                if (epochEntry == null) {\n                                    LOGGER.error(\"Failed to find an epochEntry to match nextTransferFromWhere {}\", this.nextTransferFromWhere);\n                                    sendHeartbeatIfNeeded();\n                                    waitForRunning(500);\n                                    break;\n                                }\n                                changeTransferEpochToNext(epochEntry);\n                                LOGGER.info(\"Master transfer data to slave {}, from offset:{}, currentEpoch:{}\",\n                                    AutoSwitchHAConnection.this.clientAddress, this.nextTransferFromWhere, epochEntry);\n                            }\n                            transferToSlave();\n                            break;\n                        default:\n                            throw new Exception(\"unexpected state \" + currentState);\n                    }\n                } catch (Exception e) {\n                    AutoSwitchHAConnection.LOGGER.error(this.getServiceName() + \" service has exception.\", e);\n                    break;\n                }\n            }\n\n            this.onStop();\n\n            changeCurrentState(HAConnectionState.SHUTDOWN);\n\n            this.makeStop();\n\n            readSocketService.makeStop();\n\n            haService.removeConnection(AutoSwitchHAConnection.this);\n\n            SelectionKey sk = this.socketChannel.keyFor(this.selector);\n            if (sk != null) {\n                sk.cancel();\n            }\n\n            try {\n                this.selector.close();\n                this.socketChannel.close();\n            } catch (IOException e) {\n                AutoSwitchHAConnection.LOGGER.error(\"\", e);\n            }\n\n            flowMonitor.shutdown(true);\n\n            AutoSwitchHAConnection.LOGGER.info(this.getServiceName() + \" service end\");\n        }\n\n        abstract protected int getNextTransferDataSize();\n\n        abstract protected void releaseData();\n\n        abstract protected boolean transferData(int maxTransferSize) throws Exception;\n\n        abstract protected void onStop();\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha.autoswitch;\n\nimport java.io.IOException;\nimport java.nio.channels.SocketChannel;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.locks.Lock;\nimport java.util.concurrent.locks.ReadWriteLock;\nimport java.util.concurrent.locks.ReentrantReadWriteLock;\nimport java.util.function.Consumer;\nimport java.util.stream.Collectors;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.ConcurrentHashMapUtils;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.EpochEntry;\nimport org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.ha.DefaultHAService;\nimport org.apache.rocketmq.store.ha.GroupTransferService;\nimport org.apache.rocketmq.store.ha.HAClient;\nimport org.apache.rocketmq.store.ha.HAConnection;\nimport org.apache.rocketmq.store.ha.HAConnectionStateNotificationService;\nimport org.rocksdb.RocksDBException;\n\n/**\n * SwitchAble ha service, support switch role to master or slave.\n */\npublic class AutoSwitchHAService extends DefaultHAService {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private final ExecutorService executorService = ThreadUtils.newSingleThreadExecutor(new ThreadFactoryImpl(\"AutoSwitchHAService_Executor_\"));\n    private final ConcurrentHashMap<Long/*brokerId*/, Long/*lastCaughtUpTimestamp*/> connectionCaughtUpTimeTable = new ConcurrentHashMap<>();\n    private final List<Consumer<Set<Long/*brokerId*/>>> syncStateSetChangedListeners = new ArrayList<>();\n    private final Set<Long/*brokerId*/> syncStateSet = new HashSet<>();\n    private final Set<Long> remoteSyncStateSet = new HashSet<>();\n    private final ReadWriteLock syncStateSetReadWriteLock = new ReentrantReadWriteLock();\n    private final Lock readLock = syncStateSetReadWriteLock.readLock();\n    private final Lock writeLock = syncStateSetReadWriteLock.writeLock();\n\n    //  Indicate whether the syncStateSet is currently in the process of being synchronized to controller.\n    private volatile boolean isSynchronizingSyncStateSet = false;\n\n    private EpochFileCache epochCache;\n    private AutoSwitchHAClient haClient;\n\n    private Long localBrokerId = null;\n\n    public AutoSwitchHAService() {\n    }\n\n    @Override\n    public void init(final DefaultMessageStore defaultMessageStore) throws IOException {\n        this.epochCache = new EpochFileCache(defaultMessageStore.getMessageStoreConfig().getStorePathEpochFile());\n        this.epochCache.initCacheFromFile();\n        this.defaultMessageStore = defaultMessageStore;\n        this.acceptSocketService = new AutoSwitchAcceptSocketService(defaultMessageStore.getMessageStoreConfig());\n        this.groupTransferService = new GroupTransferService(this, defaultMessageStore);\n        this.haConnectionStateNotificationService = new HAConnectionStateNotificationService(this, defaultMessageStore);\n    }\n\n    @Override\n    public void shutdown() {\n        super.shutdown();\n        if (this.haClient != null) {\n            this.haClient.shutdown();\n        }\n        this.executorService.shutdown();\n    }\n\n    @Override\n    public void removeConnection(HAConnection conn) {\n        if (!defaultMessageStore.isShutdown()) {\n            Long slave = ((AutoSwitchHAConnection) conn).getSlaveId();\n            this.writeLock.lock();\n            try {\n                final Set<Long> newSyncStateSet = new HashSet<>(this.syncStateSet);\n                if (newSyncStateSet.contains(slave)) {\n                    newSyncStateSet.remove(slave);\n                    markSynchronizingSyncStateSet(newSyncStateSet);\n                    notifySyncStateSetChanged(newSyncStateSet);\n                    this.syncStateSet.clear();\n                    this.syncStateSet.addAll(newSyncStateSet);\n                }\n            } finally {\n                this.writeLock.unlock();\n            }\n        }\n        super.removeConnection(conn);\n    }\n\n    @Override\n    public boolean changeToMaster(int masterEpoch) throws RocksDBException {\n        final int lastEpoch = this.epochCache.lastEpoch();\n        if (masterEpoch < lastEpoch) {\n            LOGGER.warn(\"newMasterEpoch {} < lastEpoch {}, fail to change to master\", masterEpoch, lastEpoch);\n            return false;\n        }\n        destroyConnections();\n        // Stop ha client if needed\n        if (this.haClient != null) {\n            this.haClient.shutdown();\n        }\n\n        // Truncate dirty file\n        final long truncateOffset = truncateInvalidMsg();\n\n        this.defaultMessageStore.setConfirmOffset(computeConfirmOffset());\n\n        if (truncateOffset >= 0) {\n            this.epochCache.truncateSuffixByOffset(truncateOffset);\n        }\n\n        // Append new epoch to epochFile\n        final EpochEntry newEpochEntry = new EpochEntry(masterEpoch, this.defaultMessageStore.getMaxPhyOffset());\n        if (this.epochCache.lastEpoch() >= masterEpoch) {\n            this.epochCache.truncateSuffixByEpoch(masterEpoch);\n        }\n        this.epochCache.appendEntry(newEpochEntry);\n\n        // Waiting consume queue dispatch\n        while (defaultMessageStore.dispatchBehindBytes() > 0) {\n            try {\n                Thread.sleep(100);\n            } catch (Exception ignored) {\n\n            }\n        }\n\n        if (defaultMessageStore.isTransientStorePoolEnable()) {\n            waitingForAllCommit();\n            defaultMessageStore.getTransientStorePool().setRealCommit(true);\n        }\n\n        LOGGER.info(\"TruncateOffset is {}, confirmOffset is {}, maxPhyOffset is {}\", truncateOffset, this.defaultMessageStore.getConfirmOffset(), this.defaultMessageStore.getMaxPhyOffset());\n        this.defaultMessageStore.recoverTopicQueueTable();\n        this.defaultMessageStore.setStateMachineVersion(masterEpoch);\n        LOGGER.info(\"Change ha to master success, newMasterEpoch:{}, startOffset:{}\", masterEpoch, newEpochEntry.getStartOffset());\n        return true;\n    }\n\n    @Override\n    public boolean changeToSlave(String newMasterAddr, int newMasterEpoch, Long slaveId) {\n        final int lastEpoch = this.epochCache.lastEpoch();\n        if (newMasterEpoch < lastEpoch) {\n            LOGGER.warn(\"newMasterEpoch {} < lastEpoch {}, fail to change to slave\", newMasterEpoch, lastEpoch);\n            return false;\n        }\n        try {\n            destroyConnections();\n            if (this.haClient == null) {\n                this.haClient = new AutoSwitchHAClient(this, defaultMessageStore, this.epochCache, slaveId);\n            } else {\n                this.haClient.reOpen();\n            }\n            this.haClient.updateMasterAddress(newMasterAddr);\n            this.haClient.updateHaMasterAddress(null);\n            this.haClient.start();\n\n            if (defaultMessageStore.isTransientStorePoolEnable()) {\n                waitingForAllCommit();\n                defaultMessageStore.getTransientStorePool().setRealCommit(false);\n            }\n\n            this.defaultMessageStore.setStateMachineVersion(newMasterEpoch);\n\n            LOGGER.info(\"Change ha to slave success, newMasterAddress:{}, newMasterEpoch:{}\", newMasterAddr, newMasterEpoch);\n            return true;\n        } catch (final Exception e) {\n            LOGGER.error(\"Error happen when change ha to slave\", e);\n            return false;\n        }\n    }\n\n    @Override\n    public boolean changeToMasterWhenLastRoleIsMaster(int masterEpoch) {\n        final int lastEpoch = this.epochCache.lastEpoch();\n        if (masterEpoch < lastEpoch) {\n            LOGGER.warn(\"newMasterEpoch {} < lastEpoch {}, fail to change to master\", masterEpoch, lastEpoch);\n            return false;\n        }\n        // Append new epoch to epochFile\n        final EpochEntry newEpochEntry = new EpochEntry(masterEpoch, this.defaultMessageStore.getMaxPhyOffset());\n        if (this.epochCache.lastEpoch() >= masterEpoch) {\n            this.epochCache.truncateSuffixByEpoch(masterEpoch);\n        }\n        this.epochCache.appendEntry(newEpochEntry);\n\n        this.defaultMessageStore.setStateMachineVersion(masterEpoch);\n        LOGGER.info(\"Change ha to master success, last role is master, newMasterEpoch:{}, startOffset:{}\",\n            masterEpoch, newEpochEntry.getStartOffset());\n        return true;\n    }\n\n    @Override\n    public boolean changeToSlaveWhenMasterNotChange(String newMasterAddr, int newMasterEpoch) {\n        final int lastEpoch = this.epochCache.lastEpoch();\n        if (newMasterEpoch < lastEpoch) {\n            LOGGER.warn(\"newMasterEpoch {} < lastEpoch {}, fail to change to slave\", newMasterEpoch, lastEpoch);\n            return false;\n        }\n\n        this.defaultMessageStore.setStateMachineVersion(newMasterEpoch);\n        LOGGER.info(\"Change ha to slave success, master doesn't change, newMasterAddress:{}, newMasterEpoch:{}\",\n            newMasterAddr, newMasterEpoch);\n        return true;\n    }\n\n    public void waitingForAllCommit() {\n        while (getDefaultMessageStore().remainHowManyDataToCommit() > 0) {\n            getDefaultMessageStore().getCommitLog().getFlushManager().wakeUpCommit();\n            try {\n                Thread.sleep(100);\n            } catch (Exception e) {\n\n            }\n        }\n    }\n\n    @Override\n    public HAClient getHAClient() {\n        return this.haClient;\n    }\n\n    @Override\n    public void updateHaMasterAddress(String newAddr) {\n        if (this.haClient != null) {\n            this.haClient.updateHaMasterAddress(newAddr);\n        }\n    }\n\n    @Override\n    public void updateMasterAddress(String newAddr) {\n    }\n\n    public void registerSyncStateSetChangedListener(final Consumer<Set<Long>> listener) {\n        this.syncStateSetChangedListeners.add(listener);\n    }\n\n    public void notifySyncStateSetChanged(final Set<Long> newSyncStateSet) {\n        this.executorService.submit(() -> {\n            syncStateSetChangedListeners.forEach(listener -> listener.accept(newSyncStateSet));\n        });\n        LOGGER.info(\"Notify the syncStateSet has been changed into {}.\", newSyncStateSet);\n    }\n\n    /**\n     * Check and maybe shrink the SyncStateSet.\n     * A slave will be removed from SyncStateSet if (curTime - HaConnection.lastCaughtUpTime) > option(haMaxTimeSlaveNotCatchup)\n     */\n    public Set<Long> maybeShrinkSyncStateSet() {\n        final Set<Long> newSyncStateSet = getLocalSyncStateSet();\n        boolean isSyncStateSetChanged = false;\n        final long haMaxTimeSlaveNotCatchup = this.defaultMessageStore.getMessageStoreConfig().getHaMaxTimeSlaveNotCatchup();\n        for (Map.Entry<Long, Long> next : this.connectionCaughtUpTimeTable.entrySet()) {\n            final Long slaveBrokerId = next.getKey();\n            if (newSyncStateSet.contains(slaveBrokerId)) {\n                final Long lastCaughtUpTimeMs = next.getValue();\n                if ((System.currentTimeMillis() - lastCaughtUpTimeMs) > haMaxTimeSlaveNotCatchup) {\n                    newSyncStateSet.remove(slaveBrokerId);\n                    isSyncStateSetChanged = true;\n                }\n            }\n        }\n\n        // If the slaveBrokerId is in syncStateSet but not in connectionCaughtUpTimeTable,\n        // it means that the broker has not connected.\n        Iterator<Long> iterator = newSyncStateSet.iterator();\n        while (iterator.hasNext()) {\n            Long slaveBrokerId = iterator.next();\n            if (!Objects.equals(slaveBrokerId, this.localBrokerId) && !this.connectionCaughtUpTimeTable.containsKey(slaveBrokerId)) {\n                iterator.remove();\n                isSyncStateSetChanged = true;\n            }\n        }\n\n        if (isSyncStateSetChanged) {\n            markSynchronizingSyncStateSet(newSyncStateSet);\n        }\n        return newSyncStateSet;\n    }\n\n    /**\n     * Check and maybe add the slave to SyncStateSet. A slave will be added to SyncStateSet if its slaveMaxOffset >=\n     * current confirmOffset, and it is caught up to an offset within the current leader epoch.\n     */\n    public void maybeExpandInSyncStateSet(final Long slaveBrokerId, final long slaveMaxOffset) {\n        final Set<Long> currentSyncStateSet = getLocalSyncStateSet();\n        if (currentSyncStateSet.contains(slaveBrokerId)) {\n            return;\n        }\n        final long confirmOffset = this.defaultMessageStore.getConfirmOffset();\n        if (slaveMaxOffset >= confirmOffset) {\n            final EpochEntry currentLeaderEpoch = this.epochCache.lastEntry();\n            if (slaveMaxOffset >= currentLeaderEpoch.getStartOffset()) {\n                LOGGER.info(\"The slave {} has caught up, slaveMaxOffset: {}, confirmOffset: {}, epoch: {}, leader epoch startOffset: {}.\",\n                    slaveBrokerId, slaveMaxOffset, confirmOffset, currentLeaderEpoch.getEpoch(), currentLeaderEpoch.getStartOffset());\n                currentSyncStateSet.add(slaveBrokerId);\n                markSynchronizingSyncStateSet(currentSyncStateSet);\n                // Notify the upper layer that syncStateSet changed.\n                notifySyncStateSetChanged(currentSyncStateSet);\n            }\n        }\n    }\n\n    private void markSynchronizingSyncStateSet(final Set<Long> newSyncStateSet) {\n        this.writeLock.lock();\n        try {\n            this.isSynchronizingSyncStateSet = true;\n            this.remoteSyncStateSet.clear();\n            this.remoteSyncStateSet.addAll(newSyncStateSet);\n        } finally {\n            this.writeLock.unlock();\n        }\n    }\n\n    private void markSynchronizingSyncStateSetDone() {\n        // No need to lock, because the upper-level calling method has already locked write lock\n        this.isSynchronizingSyncStateSet = false;\n    }\n\n    public boolean isSynchronizingSyncStateSet() {\n        return isSynchronizingSyncStateSet;\n    }\n\n    public void updateConnectionLastCaughtUpTime(final Long slaveBrokerId, final long lastCaughtUpTimeMs) {\n        Long prevTime = ConcurrentHashMapUtils.computeIfAbsent(this.connectionCaughtUpTimeTable, slaveBrokerId, k -> 0L);\n        this.connectionCaughtUpTimeTable.put(slaveBrokerId, Math.max(prevTime, lastCaughtUpTimeMs));\n    }\n\n    public void updateConfirmOffsetWhenSlaveAck(final Long slaveBrokerId) {\n        this.readLock.lock();\n        try {\n            if (this.syncStateSet.contains(slaveBrokerId)) {\n                this.defaultMessageStore.setConfirmOffset(computeConfirmOffset());\n            }\n        } finally {\n            this.readLock.unlock();\n        }\n    }\n\n    @Override\n    public int inSyncReplicasNums(final long masterPutWhere) {\n        this.readLock.lock();\n        try {\n            if (this.isSynchronizingSyncStateSet) {\n                return Math.max(this.syncStateSet.size(), this.remoteSyncStateSet.size());\n            } else {\n                return this.syncStateSet.size();\n            }\n        } finally {\n            this.readLock.unlock();\n        }\n    }\n\n    @Override\n    public HARuntimeInfo getRuntimeInfo(long masterPutWhere) {\n        HARuntimeInfo info = new HARuntimeInfo();\n\n        if (BrokerRole.SLAVE.equals(this.getDefaultMessageStore().getMessageStoreConfig().getBrokerRole())) {\n            info.setMaster(false);\n\n            info.getHaClientRuntimeInfo().setMasterAddr(this.haClient.getHaMasterAddress());\n            info.getHaClientRuntimeInfo().setMaxOffset(this.getDefaultMessageStore().getMaxPhyOffset());\n            info.getHaClientRuntimeInfo().setLastReadTimestamp(this.haClient.getLastReadTimestamp());\n            info.getHaClientRuntimeInfo().setLastWriteTimestamp(this.haClient.getLastWriteTimestamp());\n            info.getHaClientRuntimeInfo().setTransferredByteInSecond(this.haClient.getTransferredByteInSecond());\n            info.getHaClientRuntimeInfo().setMasterFlushOffset(this.defaultMessageStore.getMasterFlushedOffset());\n        } else {\n            info.setMaster(true);\n\n            info.setMasterCommitLogMaxOffset(masterPutWhere);\n\n            Set<Long> localSyncStateSet = getLocalSyncStateSet();\n            for (HAConnection conn : this.connectionList) {\n                HARuntimeInfo.HAConnectionRuntimeInfo cInfo = new HARuntimeInfo.HAConnectionRuntimeInfo();\n\n                long slaveAckOffset = conn.getSlaveAckOffset();\n                cInfo.setSlaveAckOffset(slaveAckOffset);\n                cInfo.setDiff(masterPutWhere - slaveAckOffset);\n                cInfo.setAddr(conn.getClientAddress().substring(1));\n                cInfo.setTransferredByteInSecond(conn.getTransferredByteInSecond());\n                cInfo.setTransferFromWhere(conn.getTransferFromWhere());\n\n                cInfo.setInSync(localSyncStateSet.contains(((AutoSwitchHAConnection) conn).getSlaveId()));\n\n                info.getHaConnectionInfo().add(cInfo);\n            }\n            info.setInSyncSlaveNums(localSyncStateSet.size() - 1);\n        }\n        return info;\n    }\n\n    public long computeConfirmOffset() {\n        final Set<Long> currentSyncStateSet = getSyncStateSet();\n        long newConfirmOffset = this.defaultMessageStore.getMaxPhyOffset();\n        List<Long> idList = this.connectionList.stream().map(connection -> ((AutoSwitchHAConnection)connection).getSlaveId()).collect(Collectors.toList());\n\n        // To avoid the syncStateSet is not consistent with connectionList.\n        // Fix issue: https://github.com/apache/rocketmq/issues/6662\n        for (Long syncId : currentSyncStateSet) {\n            if (!idList.contains(syncId) && this.localBrokerId != null && !Objects.equals(syncId, this.localBrokerId)) {\n                LOGGER.warn(\"Slave {} is still in syncStateSet, but has lost its connection. So new offset can't be compute.\", syncId);\n                // Without check and re-compute, return the confirmOffset's value directly.\n                return this.defaultMessageStore.getConfirmOffsetDirectly();\n            }\n        }\n\n        for (HAConnection connection : this.connectionList) {\n            final Long slaveId = ((AutoSwitchHAConnection) connection).getSlaveId();\n            if (currentSyncStateSet.contains(slaveId) && connection.getSlaveAckOffset() > 0) {\n                newConfirmOffset = Math.min(newConfirmOffset, connection.getSlaveAckOffset());\n            }\n        }\n        return newConfirmOffset;\n    }\n\n    public void setSyncStateSet(final Set<Long> syncStateSet) {\n        this.writeLock.lock();\n        try {\n            markSynchronizingSyncStateSetDone();\n            this.syncStateSet.clear();\n            this.syncStateSet.addAll(syncStateSet);\n            this.defaultMessageStore.setConfirmOffset(computeConfirmOffset());\n        } finally {\n            this.writeLock.unlock();\n        }\n    }\n\n    /**\n     * Return the union of the local and remote syncStateSets\n     */\n    public Set<Long> getSyncStateSet() {\n        this.readLock.lock();\n        try {\n            if (this.isSynchronizingSyncStateSet) {\n                Set<Long> unionSyncStateSet = new HashSet<>(this.syncStateSet.size() + this.remoteSyncStateSet.size());\n                unionSyncStateSet.addAll(this.syncStateSet);\n                unionSyncStateSet.addAll(this.remoteSyncStateSet);\n                return unionSyncStateSet;\n            } else {\n                HashSet<Long> syncStateSet = new HashSet<>(this.syncStateSet.size());\n                syncStateSet.addAll(this.syncStateSet);\n                return syncStateSet;\n            }\n        } finally {\n            this.readLock.unlock();\n        }\n    }\n\n    public Set<Long> getLocalSyncStateSet() {\n        this.readLock.lock();\n        try {\n            HashSet<Long> localSyncStateSet = new HashSet<>(this.syncStateSet.size());\n            localSyncStateSet.addAll(this.syncStateSet);\n            return localSyncStateSet;\n        } finally {\n            this.readLock.unlock();\n        }\n    }\n\n    public void truncateEpochFilePrefix(final long offset) {\n        this.epochCache.truncatePrefixByOffset(offset);\n    }\n\n    public void truncateEpochFileSuffix(final long offset) {\n        this.epochCache.truncateSuffixByOffset(offset);\n    }\n\n    /**\n     * Try to truncate incomplete msg transferred from master.\n     */\n    public long truncateInvalidMsg() throws RocksDBException {\n        long dispatchBehind = this.defaultMessageStore.dispatchBehindBytes();\n        if (dispatchBehind <= 0) {\n            LOGGER.info(\"Dispatch complete, skip truncate\");\n            return -1;\n        }\n\n        boolean doNext = true;\n\n        // Here we could use reputFromOffset in DefaultMessageStore directly.\n        long reputFromOffset = this.defaultMessageStore.getReputFromOffset();\n        do {\n            SelectMappedBufferResult result = this.defaultMessageStore.getCommitLog().getData(reputFromOffset);\n            if (result == null) {\n                break;\n            }\n\n            try {\n                reputFromOffset = result.getStartOffset();\n\n                int readSize = 0;\n                while (readSize < result.getSize()) {\n                    DispatchRequest dispatchRequest = this.defaultMessageStore.getCommitLog().checkMessageAndReturnSize(result.getByteBuffer(), false, false);\n                    if (dispatchRequest.isSuccess()) {\n                        int size = dispatchRequest.getMsgSize();\n                        if (size > 0) {\n                            reputFromOffset += size;\n                            readSize += size;\n                        } else {\n                            reputFromOffset = this.defaultMessageStore.getCommitLog().rollNextFile(reputFromOffset);\n                            break;\n                        }\n                    } else {\n                        doNext = false;\n                        break;\n                    }\n                }\n            } finally {\n                result.release();\n            }\n        } while (reputFromOffset < this.defaultMessageStore.getMaxPhyOffset() && doNext);\n\n        LOGGER.info(\"Truncate commitLog to {}\", reputFromOffset);\n        this.defaultMessageStore.truncateDirtyFiles(reputFromOffset);\n        return reputFromOffset;\n    }\n\n    public int getLastEpoch() {\n        return this.epochCache.lastEpoch();\n    }\n\n    public List<EpochEntry> getEpochEntries() {\n        return this.epochCache.getAllEntries();\n    }\n\n    public Long getLocalBrokerId() {\n        return localBrokerId;\n    }\n\n    public void setLocalBrokerId(Long localBrokerId) {\n        this.localBrokerId = localBrokerId;\n    }\n\n    class AutoSwitchAcceptSocketService extends AcceptSocketService {\n\n        public AutoSwitchAcceptSocketService(final MessageStoreConfig messageStoreConfig) {\n            super(messageStoreConfig);\n        }\n\n        @Override\n        public String getServiceName() {\n            if (defaultMessageStore.getBrokerConfig().isInBrokerContainer()) {\n                return defaultMessageStore.getBrokerConfig().getIdentifier() + AcceptSocketService.class.getSimpleName();\n            }\n            return AutoSwitchAcceptSocketService.class.getSimpleName();\n        }\n\n        @Override\n        protected HAConnection createConnection(SocketChannel sc) throws IOException {\n            return new AutoSwitchHAConnection(AutoSwitchHAService.this, sc, AutoSwitchHAService.this.epochCache);\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/BrokerMetadata.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha.autoswitch;\n\nimport org.apache.commons.lang3.StringUtils;\n\nimport java.util.Objects;\n\npublic class BrokerMetadata extends MetadataFile {\n\n    protected String clusterName;\n\n    protected String brokerName;\n\n    protected Long brokerId;\n\n    public BrokerMetadata(String filePath) {\n        this.filePath = filePath;\n    }\n\n    public void updateAndPersist(String clusterName, String brokerName, Long brokerId) throws Exception {\n        this.clusterName = clusterName;\n        this.brokerName = brokerName;\n        this.brokerId = brokerId;\n        writeToFile();\n    }\n\n    @Override\n    public String encodeToStr() {\n        StringBuilder sb = new StringBuilder();\n        sb.append(clusterName).append(\"#\");\n        sb.append(brokerName).append(\"#\");\n        sb.append(brokerId);\n        return sb.toString();\n    }\n\n    @Override\n    public void decodeFromStr(String dataStr) {\n        if (dataStr == null) return;\n        String[] dataArr = dataStr.split(\"#\");\n        this.clusterName = dataArr[0];\n        this.brokerName = dataArr[1];\n        this.brokerId = Long.valueOf(dataArr[2]);\n    }\n\n    @Override\n    public boolean isLoaded() {\n        return StringUtils.isNotEmpty(this.clusterName) && StringUtils.isNotEmpty(this.brokerName) && brokerId != null;\n    }\n\n    @Override\n    public void clearInMem() {\n        this.clusterName = null;\n        this.brokerName = null;\n        this.brokerId = null;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public Long getBrokerId() {\n        return brokerId;\n    }\n\n    public String getClusterName() {\n        return clusterName;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) return true;\n        if (o == null || getClass() != o.getClass()) return false;\n        BrokerMetadata that = (BrokerMetadata) o;\n        return Objects.equals(clusterName, that.clusterName) && Objects.equals(brokerName, that.brokerName) && Objects.equals(brokerId, that.brokerId);\n    }\n\n    @Override\n    public int hashCode() {\n        return Objects.hash(clusterName, brokerName, brokerId);\n    }\n\n    @Override\n    public String toString() {\n        return \"BrokerMetadata{\" +\n                \"clusterName='\" + clusterName + '\\'' +\n                \", brokerName='\" + brokerName + '\\'' +\n                \", brokerId=\" + brokerId +\n                \", filePath='\" + filePath + '\\'' +\n                '}';\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/EpochFileCache.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha.autoswitch;\n\nimport java.io.IOException;\nimport java.util.ArrayList;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.TreeMap;\nimport java.util.concurrent.locks.Lock;\nimport java.util.concurrent.locks.ReadWriteLock;\nimport java.util.concurrent.locks.ReentrantReadWriteLock;\nimport java.util.function.Predicate;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.CheckpointFile;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.EpochEntry;\n\n/**\n * Cache for epochFile. Mapping (Epoch -> StartOffset)\n */\npublic class EpochFileCache {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();\n    private final Lock readLock = this.readWriteLock.readLock();\n    private final Lock writeLock = this.readWriteLock.writeLock();\n    private final TreeMap<Integer, EpochEntry> epochMap;\n    private CheckpointFile<EpochEntry> checkpoint;\n\n    public EpochFileCache() {\n        this.epochMap = new TreeMap<>();\n    }\n\n    public EpochFileCache(final String path) {\n        this.epochMap = new TreeMap<>();\n        this.checkpoint = new CheckpointFile<>(path, new EpochEntrySerializer());\n    }\n\n    public boolean initCacheFromFile() {\n        this.writeLock.lock();\n        try {\n            final List<EpochEntry> entries = this.checkpoint.read();\n            initEntries(entries);\n            return true;\n        } catch (final IOException e) {\n            log.error(\"Error happen when init epoch entries from epochFile\", e);\n            return false;\n        } finally {\n            this.writeLock.unlock();\n        }\n    }\n\n    public void initCacheFromEntries(final List<EpochEntry> entries) {\n        this.writeLock.lock();\n        try {\n            initEntries(entries);\n            flush();\n        } finally {\n            this.writeLock.unlock();\n        }\n    }\n\n    private void initEntries(final List<EpochEntry> entries) {\n        this.epochMap.clear();\n        EpochEntry preEntry = null;\n        for (final EpochEntry entry : entries) {\n            this.epochMap.put(entry.getEpoch(), entry);\n            if (preEntry != null) {\n                preEntry.setEndOffset(entry.getStartOffset());\n            }\n            preEntry = entry;\n        }\n    }\n\n    public int getEntrySize() {\n        this.readLock.lock();\n        try {\n            return this.epochMap.size();\n        } finally {\n            this.readLock.unlock();\n        }\n    }\n\n    public boolean appendEntry(final EpochEntry entry) {\n        this.writeLock.lock();\n        try {\n            if (!this.epochMap.isEmpty()) {\n                final EpochEntry lastEntry = this.epochMap.lastEntry().getValue();\n                if (lastEntry.getEpoch() >= entry.getEpoch() || lastEntry.getStartOffset() >= entry.getStartOffset()) {\n                    log.error(\"The appending entry's lastEpoch or endOffset {} is not bigger than lastEntry {}, append failed\", entry, lastEntry);\n                    return false;\n                }\n                lastEntry.setEndOffset(entry.getStartOffset());\n            }\n            this.epochMap.put(entry.getEpoch(), new EpochEntry(entry));\n            flush();\n            return true;\n        } finally {\n            this.writeLock.unlock();\n        }\n    }\n\n    /**\n     * Set endOffset for lastEpochEntry.\n     */\n    public void setLastEpochEntryEndOffset(final long endOffset) {\n        this.writeLock.lock();\n        try {\n            if (!this.epochMap.isEmpty()) {\n                final EpochEntry lastEntry = this.epochMap.lastEntry().getValue();\n                if (lastEntry.getStartOffset() <= endOffset) {\n                    lastEntry.setEndOffset(endOffset);\n                }\n            }\n        } finally {\n            this.writeLock.unlock();\n        }\n    }\n\n    public EpochEntry firstEntry() {\n        this.readLock.lock();\n        try {\n            if (this.epochMap.isEmpty()) {\n                return null;\n            }\n            return new EpochEntry(this.epochMap.firstEntry().getValue());\n        } finally {\n            this.readLock.unlock();\n        }\n    }\n\n    public EpochEntry lastEntry() {\n        this.readLock.lock();\n        try {\n            if (this.epochMap.isEmpty()) {\n                return null;\n            }\n            return new EpochEntry(this.epochMap.lastEntry().getValue());\n        } finally {\n            this.readLock.unlock();\n        }\n    }\n\n    public int lastEpoch() {\n        final EpochEntry entry = lastEntry();\n        if (entry != null) {\n            return entry.getEpoch();\n        }\n        return -1;\n    }\n\n    public EpochEntry getEntry(final int epoch) {\n        this.readLock.lock();\n        try {\n            if (this.epochMap.containsKey(epoch)) {\n                final EpochEntry entry = this.epochMap.get(epoch);\n                return new EpochEntry(entry);\n            }\n            return null;\n        } finally {\n            this.readLock.unlock();\n        }\n    }\n\n    public EpochEntry findEpochEntryByOffset(final long offset) {\n        this.readLock.lock();\n        try {\n            if (!this.epochMap.isEmpty()) {\n                for (Map.Entry<Integer, EpochEntry> entry : this.epochMap.entrySet()) {\n                    if (entry.getValue().getStartOffset() <= offset && entry.getValue().getEndOffset() > offset) {\n                        return new EpochEntry(entry.getValue());\n                    }\n                }\n            }\n            return null;\n        } finally {\n            this.readLock.unlock();\n        }\n    }\n\n    public EpochEntry nextEntry(final int epoch) {\n        this.readLock.lock();\n        try {\n            final Map.Entry<Integer, EpochEntry> entry = this.epochMap.ceilingEntry(epoch + 1);\n            if (entry != null) {\n                return new EpochEntry(entry.getValue());\n            }\n            return null;\n        } finally {\n            this.readLock.unlock();\n        }\n    }\n\n    public List<EpochEntry> getAllEntries() {\n        this.readLock.lock();\n        try {\n            final ArrayList<EpochEntry> result = new ArrayList<>(this.epochMap.size());\n            this.epochMap.forEach((key, value) -> result.add(new EpochEntry(value)));\n            return result;\n        } finally {\n            this.readLock.unlock();\n        }\n    }\n\n    /**\n     * Find the consistentPoint between compareCache and local.\n     *\n     * @return the consistent offset\n     */\n    public long findConsistentPoint(final EpochFileCache compareCache) {\n        this.readLock.lock();\n        try {\n            long consistentOffset = -1;\n            final Map<Integer, EpochEntry> descendingMap = new TreeMap<>(this.epochMap).descendingMap();\n            final Iterator<Map.Entry<Integer, EpochEntry>> iter = descendingMap.entrySet().iterator();\n            while (iter.hasNext()) {\n                final Map.Entry<Integer, EpochEntry> curLocalEntry = iter.next();\n                final EpochEntry compareEntry = compareCache.getEntry(curLocalEntry.getKey());\n                if (compareEntry != null && compareEntry.getStartOffset() == curLocalEntry.getValue().getStartOffset()) {\n                    consistentOffset = Math.min(curLocalEntry.getValue().getEndOffset(), compareEntry.getEndOffset());\n                    break;\n                }\n            }\n            return consistentOffset;\n        } finally {\n            this.readLock.unlock();\n        }\n    }\n\n    /**\n     * Remove epochEntries with epoch >= truncateEpoch.\n     */\n    public void truncateSuffixByEpoch(final int truncateEpoch) {\n        Predicate<EpochEntry> predict = entry -> entry.getEpoch() >= truncateEpoch;\n        doTruncateSuffix(predict);\n    }\n\n    /**\n     * Remove epochEntries with startOffset >= truncateOffset.\n     */\n    public void truncateSuffixByOffset(final long truncateOffset) {\n        Predicate<EpochEntry> predict = entry -> entry.getStartOffset() >= truncateOffset;\n        doTruncateSuffix(predict);\n    }\n\n    private void doTruncateSuffix(Predicate<EpochEntry> predict) {\n        this.writeLock.lock();\n        try {\n            this.epochMap.entrySet().removeIf(entry -> predict.test(entry.getValue()));\n            final EpochEntry entry = lastEntry();\n            if (entry != null) {\n                entry.setEndOffset(Long.MAX_VALUE);\n            }\n            flush();\n        } finally {\n            this.writeLock.unlock();\n        }\n    }\n\n    /**\n     * Remove epochEntries with endOffset <= truncateOffset.\n     */\n    public void truncatePrefixByOffset(final long truncateOffset) {\n        Predicate<EpochEntry> predict = entry -> entry.getEndOffset() <= truncateOffset;\n        this.writeLock.lock();\n        try {\n            this.epochMap.entrySet().removeIf(entry -> predict.test(entry.getValue()));\n            flush();\n        } finally {\n            this.writeLock.unlock();\n        }\n    }\n\n    private void flush() {\n        this.writeLock.lock();\n        try {\n            if (this.checkpoint != null) {\n                final ArrayList<EpochEntry> entries = new ArrayList<>(this.epochMap.values());\n                this.checkpoint.write(entries);\n            }\n        } catch (final IOException e) {\n            log.error(\"Error happen when flush epochEntries to epochCheckpointFile\", e);\n        } finally {\n            this.writeLock.unlock();\n        }\n    }\n\n    static class EpochEntrySerializer implements CheckpointFile.CheckpointSerializer<EpochEntry> {\n\n        @Override\n        public String toLine(EpochEntry entry) {\n            if (entry != null) {\n                return String.format(\"%d-%d\", entry.getEpoch(), entry.getStartOffset());\n            } else {\n                return null;\n            }\n        }\n\n        @Override\n        public EpochEntry fromLine(String line) {\n            final String[] arr = line.split(\"-\");\n            if (arr.length == 2) {\n                final int epoch = Integer.parseInt(arr[0]);\n                final long startOffset = Long.parseLong(arr[1]);\n                return new EpochEntry(epoch, startOffset);\n            }\n            return null;\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/MetadataFile.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha.autoswitch;\n\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\n\nimport java.io.File;\n\npublic abstract class MetadataFile {\n\n    protected String filePath;\n\n    public abstract String encodeToStr();\n\n    public abstract void decodeFromStr(String dataStr);\n\n    public abstract boolean isLoaded();\n\n    public abstract void clearInMem();\n\n    public void writeToFile() throws Exception {\n        UtilAll.deleteFile(new File(filePath));\n        MixAll.string2File(encodeToStr(), this.filePath);\n    }\n\n    public void readFromFile() throws Exception {\n        String dataStr = MixAll.file2String(filePath);\n        decodeFromStr(dataStr);\n    }\n    public boolean fileExists() {\n        File file = new File(filePath);\n        return file.exists();\n    }\n\n    public void clear() {\n        clearInMem();\n        UtilAll.deleteFile(new File(filePath));\n    }\n\n    public String getFilePath() {\n        return filePath;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/TempBrokerMetadata.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha.autoswitch;\n\nimport org.apache.commons.lang3.StringUtils;\n\npublic class TempBrokerMetadata extends BrokerMetadata {\n\n    private String registerCheckCode;\n\n    public TempBrokerMetadata(String filePath) {\n        this(filePath, null, null, null, null);\n    }\n\n    public TempBrokerMetadata(String filePath, String clusterName, String brokerName, Long brokerId, String registerCheckCode) {\n        super(filePath);\n        super.clusterName = clusterName;\n        super.brokerId = brokerId;\n        super.brokerName = brokerName;\n        this.registerCheckCode = registerCheckCode;\n    }\n\n    public void updateAndPersist(String clusterName, String brokerName, Long brokerId, String registerCheckCode) throws Exception {\n        super.clusterName = clusterName;\n        super.brokerName = brokerName;\n        super.brokerId = brokerId;\n        this.registerCheckCode = registerCheckCode;\n        writeToFile();\n    }\n\n    @Override\n    public String encodeToStr() {\n        StringBuilder sb = new StringBuilder();\n        sb.append(clusterName).append(\"#\");\n        sb.append(brokerName).append(\"#\");\n        sb.append(brokerId).append(\"#\");\n        sb.append(registerCheckCode);\n        return sb.toString();\n    }\n\n    @Override\n    public void decodeFromStr(String dataStr) {\n        if (dataStr == null) return;\n        String[] dataArr = dataStr.split(\"#\");\n        this.clusterName = dataArr[0];\n        this.brokerName = dataArr[1];\n        this.brokerId = Long.valueOf(dataArr[2]);\n        this.registerCheckCode = dataArr[3];\n    }\n\n    @Override\n    public boolean isLoaded() {\n        return super.isLoaded() && StringUtils.isNotEmpty(this.registerCheckCode);\n    }\n\n    @Override\n    public void clearInMem() {\n        super.clearInMem();\n        this.registerCheckCode = null;\n    }\n\n    public Long getBrokerId() {\n        return brokerId;\n    }\n\n    public String getRegisterCheckCode() {\n        return registerCheckCode;\n    }\n\n    @Override\n    public String toString() {\n        return \"TempBrokerMetadata{\" +\n                \"registerCheckCode='\" + registerCheckCode + '\\'' +\n                \", clusterName='\" + clusterName + '\\'' +\n                \", brokerName='\" + brokerName + '\\'' +\n                \", brokerId=\" + brokerId +\n                \", filePath='\" + filePath + '\\'' +\n                '}';\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/io/AbstractHAReader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha.io;\n\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport java.nio.channels.SocketChannel;\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic abstract class AbstractHAReader {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    protected final List<HAReadHook> readHookList = new ArrayList<>();\n\n    public boolean read(SocketChannel socketChannel, ByteBuffer byteBufferRead) {\n        int readSizeZeroTimes = 0;\n        while (byteBufferRead.hasRemaining()) {\n            try {\n                int readSize = socketChannel.read(byteBufferRead);\n                for (HAReadHook readHook : readHookList) {\n                    readHook.afterRead(readSize);\n                }\n                if (readSize > 0) {\n                    readSizeZeroTimes = 0;\n                    boolean result = processReadResult(byteBufferRead);\n                    if (!result) {\n                        LOGGER.error(\"Process read result failed\");\n                        return false;\n                    }\n                } else if (readSize == 0) {\n                    if (++readSizeZeroTimes >= 3) {\n                        break;\n                    }\n                } else {\n                    LOGGER.info(\"Read socket < 0\");\n                    return false;\n                }\n            } catch (IOException e) {\n                LOGGER.info(\"Read socket exception\", e);\n                return false;\n            }\n        }\n\n        return true;\n    }\n\n    public void registerHook(HAReadHook readHook) {\n        readHookList.add(readHook);\n    }\n\n    public void clearHook() {\n        readHookList.clear();\n    }\n\n    /**\n     * Process read result.\n     *\n     * @param byteBufferRead read result\n     * @return true if process succeed, false otherwise\n     */\n    protected abstract boolean processReadResult(ByteBuffer byteBufferRead);\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/io/HAReadHook.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha.io;\n\npublic interface HAReadHook {\n    void afterRead(int readSize);\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/io/HAWriteHook.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha.io;\n\npublic interface HAWriteHook {\n    void afterWrite(int writeSize);\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/ha/io/HAWriter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha.io;\n\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport java.nio.channels.SocketChannel;\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class HAWriter {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    protected final List<HAWriteHook> writeHookList = new ArrayList<>();\n\n    public boolean write(SocketChannel socketChannel, ByteBuffer byteBufferWrite) throws IOException {\n        int writeSizeZeroTimes = 0;\n        while (byteBufferWrite.hasRemaining()) {\n            int writeSize = socketChannel.write(byteBufferWrite);\n            for (HAWriteHook writeHook : writeHookList) {\n                writeHook.afterWrite(writeSize);\n            }\n            if (writeSize > 0) {\n                writeSizeZeroTimes = 0;\n            } else if (writeSize == 0) {\n                if (++writeSizeZeroTimes >= 3) {\n                    break;\n                }\n            } else {\n                LOGGER.info(\"Write socket < 0\");\n            }\n        }\n\n        return !byteBufferWrite.hasRemaining();\n    }\n\n    public void registerHook(HAWriteHook writeHook) {\n        writeHookList.add(writeHook);\n    }\n\n    public void clearHook() {\n        writeHookList.clear();\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/hook/PutMessageHook.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.hook;\n\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.store.PutMessageResult;\n\npublic interface PutMessageHook {\n\n    /**\n     * Name of the hook.\n     *\n     * @return name of the hook\n     */\n    String hookName();\n\n    /**\n     *  Execute before put message. For example, Message verification or special message transform\n     * @param msg\n     * @return\n     */\n    PutMessageResult executeBeforePutMessage(MessageExt msg);\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/hook/SendMessageBackHook.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.hook;\n\nimport java.util.List;\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic interface SendMessageBackHook {\n\n    /**\n     * Slave send message back to master at certain offset when HA handshake\n     *\n     * @param msgList\n     * @param brokerName\n     * @param brokerAddr\n     * @return\n     */\n    boolean executeSendMessageBack(List<MessageExt> msgList, String brokerName, String brokerAddr);\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/index/IndexFile.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.index;\n\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport java.nio.MappedByteBuffer;\nimport java.util.List;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.logfile.DefaultMappedFile;\nimport org.apache.rocketmq.store.logfile.MappedFile;\n\npublic class IndexFile {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private static int hashSlotSize = 4;\n    /**\n     * Each index's store unit. Format:\n     * <pre>\n     * ┌───────────────┬───────────────────────────────┬───────────────┬───────────────┐\n     * │ Key HashCode  │        Physical Offset        │   Time Diff   │ Next Index Pos│\n     * │   (4 Bytes)   │          (8 Bytes)            │   (4 Bytes)   │   (4 Bytes)   │\n     * ├───────────────┴───────────────────────────────┴───────────────┴───────────────┤\n     * │                                 Index Store Unit                              │\n     * │                                                                               │\n     * </pre>\n     * Each index's store unit. Size:\n     * Key HashCode(4) + Physical Offset(8) + Time Diff(4) + Next Index Pos(4) = 20 Bytes\n     */\n    private static int indexSize = 20;\n    private static int invalidIndex = 0;\n    private final int hashSlotNum;\n    private final int indexNum;\n    private final int fileTotalSize;\n    private final MappedFile mappedFile;\n    private final MappedByteBuffer mappedByteBuffer;\n    private final IndexHeader indexHeader;\n\n    public IndexFile(final String fileName, final int hashSlotNum, final int indexNum,\n        final long endPhyOffset, final long endTimestamp) throws IOException {\n        this.fileTotalSize =\n            IndexHeader.INDEX_HEADER_SIZE + (hashSlotNum * hashSlotSize) + (indexNum * indexSize);\n        this.mappedFile = new DefaultMappedFile(fileName, fileTotalSize);\n        this.mappedByteBuffer = this.mappedFile.getMappedByteBuffer();\n        this.hashSlotNum = hashSlotNum;\n        this.indexNum = indexNum;\n\n        ByteBuffer byteBuffer = this.mappedByteBuffer.slice();\n        this.indexHeader = new IndexHeader(byteBuffer);\n\n        if (endPhyOffset > 0) {\n            this.indexHeader.setBeginPhyOffset(endPhyOffset);\n            this.indexHeader.setEndPhyOffset(endPhyOffset);\n        }\n\n        if (endTimestamp > 0) {\n            this.indexHeader.setBeginTimestamp(endTimestamp);\n            this.indexHeader.setEndTimestamp(endTimestamp);\n        }\n    }\n\n    public String getFileName() {\n        return this.mappedFile.getFileName();\n    }\n\n    public int getFileSize() {\n        return this.fileTotalSize;\n    }\n\n    public void load() {\n        this.indexHeader.load();\n    }\n\n    public void shutdown() {\n        try {\n            this.flush();\n        } catch (Throwable e) {\n            log.error(\"flush error when shutdown\", e);\n        }\n        mappedFile.cleanResources();\n    }\n\n    public void flush() {\n        long beginTime = System.currentTimeMillis();\n        if (this.mappedFile.hold()) {\n            this.indexHeader.updateByteBuffer();\n            this.mappedByteBuffer.force();\n            this.mappedFile.release();\n            log.info(\"flush index file elapsed time(ms) \" + (System.currentTimeMillis() - beginTime));\n        }\n    }\n\n    public boolean isWriteFull() {\n        return this.indexHeader.getIndexCount() >= this.indexNum;\n    }\n\n    public boolean destroy(final long intervalForcibly) {\n        return this.mappedFile.destroy(intervalForcibly);\n    }\n\n    public boolean putKey(final String key, final long phyOffset, final long storeTimestamp) {\n        if (this.indexHeader.getIndexCount() < this.indexNum) {\n            int keyHash = indexKeyHashMethod(key);\n            int slotPos = keyHash % this.hashSlotNum;\n            int absSlotPos = IndexHeader.INDEX_HEADER_SIZE + slotPos * hashSlotSize;\n\n            try {\n\n                int slotValue = this.mappedByteBuffer.getInt(absSlotPos);\n                if (slotValue <= invalidIndex || slotValue > this.indexHeader.getIndexCount()) {\n                    slotValue = invalidIndex;\n                }\n\n                long timeDiff = storeTimestamp - this.indexHeader.getBeginTimestamp();\n\n                timeDiff = timeDiff / 1000;\n\n                if (this.indexHeader.getBeginTimestamp() <= 0) {\n                    timeDiff = 0;\n                } else if (timeDiff > Integer.MAX_VALUE) {\n                    timeDiff = Integer.MAX_VALUE;\n                } else if (timeDiff < 0) {\n                    timeDiff = 0;\n                }\n\n                int absIndexPos =\n                    IndexHeader.INDEX_HEADER_SIZE + this.hashSlotNum * hashSlotSize\n                        + this.indexHeader.getIndexCount() * indexSize;\n\n                this.mappedByteBuffer.putInt(absIndexPos, keyHash);\n                this.mappedByteBuffer.putLong(absIndexPos + 4, phyOffset);\n                this.mappedByteBuffer.putInt(absIndexPos + 4 + 8, (int) timeDiff);\n                this.mappedByteBuffer.putInt(absIndexPos + 4 + 8 + 4, slotValue);\n\n                this.mappedByteBuffer.putInt(absSlotPos, this.indexHeader.getIndexCount());\n\n                if (this.indexHeader.getIndexCount() <= 1) {\n                    this.indexHeader.setBeginPhyOffset(phyOffset);\n                    this.indexHeader.setBeginTimestamp(storeTimestamp);\n                }\n\n                if (invalidIndex == slotValue) {\n                    this.indexHeader.incHashSlotCount();\n                }\n                this.indexHeader.incIndexCount();\n                this.indexHeader.setEndPhyOffset(phyOffset);\n                this.indexHeader.setEndTimestamp(storeTimestamp);\n\n                return true;\n            } catch (Exception e) {\n                log.error(\"putKey exception, Key: \" + key + \" KeyHashCode: \" + key.hashCode(), e);\n            }\n        } else {\n            log.warn(\"Over index file capacity: index count = \" + this.indexHeader.getIndexCount()\n                + \"; index max num = \" + this.indexNum);\n        }\n\n        return false;\n    }\n\n    public int indexKeyHashMethod(final String key) {\n        int keyHash = key.hashCode();\n        int keyHashPositive = Math.abs(keyHash);\n        if (keyHashPositive < 0) {\n            keyHashPositive = 0;\n        }\n        return keyHashPositive;\n    }\n\n    public long getBeginTimestamp() {\n        return this.indexHeader.getBeginTimestamp();\n    }\n\n    public long getEndTimestamp() {\n        return this.indexHeader.getEndTimestamp();\n    }\n\n    public long getEndPhyOffset() {\n        return this.indexHeader.getEndPhyOffset();\n    }\n\n    public boolean isTimeMatched(final long begin, final long end) {\n        boolean result = begin < this.indexHeader.getBeginTimestamp() && end > this.indexHeader.getEndTimestamp();\n        result = result || begin >= this.indexHeader.getBeginTimestamp() && begin <= this.indexHeader.getEndTimestamp();\n        result = result || end >= this.indexHeader.getBeginTimestamp() && end <= this.indexHeader.getEndTimestamp();\n        return result;\n    }\n\n    public void selectPhyOffset(final List<Long> phyOffsets, final String key, final int maxNum,\n                                final long begin, final long end) {\n        if (this.mappedFile.hold()) {\n            int keyHash = indexKeyHashMethod(key);\n            int slotPos = keyHash % this.hashSlotNum;\n            int absSlotPos = IndexHeader.INDEX_HEADER_SIZE + slotPos * hashSlotSize;\n\n            try {\n                int slotValue = this.mappedByteBuffer.getInt(absSlotPos);\n                if (slotValue <= invalidIndex || slotValue > this.indexHeader.getIndexCount()\n                    || this.indexHeader.getIndexCount() <= 1) {\n                } else {\n                    for (int nextIndexToRead = slotValue; ; ) {\n                        if (phyOffsets.size() >= maxNum) {\n                            break;\n                        }\n\n                        int absIndexPos =\n                            IndexHeader.INDEX_HEADER_SIZE + this.hashSlotNum * hashSlotSize\n                                + nextIndexToRead * indexSize;\n\n                        int keyHashRead = this.mappedByteBuffer.getInt(absIndexPos);\n                        long phyOffsetRead = this.mappedByteBuffer.getLong(absIndexPos + 4);\n\n                        long timeDiff = this.mappedByteBuffer.getInt(absIndexPos + 4 + 8);\n                        int prevIndexRead = this.mappedByteBuffer.getInt(absIndexPos + 4 + 8 + 4);\n\n                        if (timeDiff < 0) {\n                            break;\n                        }\n\n                        timeDiff *= 1000L;\n\n                        long timeRead = this.indexHeader.getBeginTimestamp() + timeDiff;\n                        boolean timeMatched = timeRead >= begin && timeRead <= end;\n\n                        if (keyHash == keyHashRead && timeMatched) {\n                            phyOffsets.add(phyOffsetRead);\n                        }\n\n                        if (prevIndexRead <= invalidIndex\n                            || prevIndexRead > this.indexHeader.getIndexCount()\n                            || prevIndexRead == nextIndexToRead || timeRead < begin) {\n                            break;\n                        }\n\n                        nextIndexToRead = prevIndexRead;\n                    }\n                }\n            } catch (Exception e) {\n                log.error(\"selectPhyOffset exception \", e);\n            } finally {\n                this.mappedFile.release();\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/index/IndexHeader.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.index;\n\nimport java.nio.ByteBuffer;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicLong;\n\n/**\n * Index File Header. Format:\n * <pre>\n * ┌───────────────────────────────┬───────────────────────────────┬───────────────────────────────┬───────────────────────────────┬───────────────────┬───────────────────┐\n * │        Begin Timestamp        │          End Timestamp        │     Begin Physical Offset     │       End Physical Offset     │  Hash Slot Count  │    Index Count    │\n * │           (8 Bytes)           │            (8 Bytes)          │           (8 Bytes)           │           (8 Bytes)           │      (4 Bytes)    │      (4 Bytes)    │\n * ├───────────────────────────────┴───────────────────────────────┴───────────────────────────────┴───────────────────────────────┴───────────────────┴───────────────────┤\n * │                                                                      Index File Header                                                                                │\n * │\n * </pre>\n * Index File Header. Size:\n * Begin Timestamp(8) + End Timestamp(8) + Begin Physical Offset(8) + End Physical Offset(8) + Hash Slot Count(4) + Index Count(4) = 40 Bytes\n */\npublic class IndexHeader {\n    public static final int INDEX_HEADER_SIZE = 40;\n    private static int beginTimestampIndex = 0;\n    private static int endTimestampIndex = 8;\n    private static int beginPhyoffsetIndex = 16;\n    private static int endPhyoffsetIndex = 24;\n    private static int hashSlotcountIndex = 32;\n    private static int indexCountIndex = 36;\n    private final ByteBuffer byteBuffer;\n    private final AtomicLong beginTimestamp = new AtomicLong(0);\n    private final AtomicLong endTimestamp = new AtomicLong(0);\n    private final AtomicLong beginPhyOffset = new AtomicLong(0);\n    private final AtomicLong endPhyOffset = new AtomicLong(0);\n    private final AtomicInteger hashSlotCount = new AtomicInteger(0);\n    private final AtomicInteger indexCount = new AtomicInteger(1);\n\n    public IndexHeader(final ByteBuffer byteBuffer) {\n        this.byteBuffer = byteBuffer;\n    }\n\n    public void load() {\n        this.beginTimestamp.set(byteBuffer.getLong(beginTimestampIndex));\n        this.endTimestamp.set(byteBuffer.getLong(endTimestampIndex));\n        this.beginPhyOffset.set(byteBuffer.getLong(beginPhyoffsetIndex));\n        this.endPhyOffset.set(byteBuffer.getLong(endPhyoffsetIndex));\n\n        this.hashSlotCount.set(byteBuffer.getInt(hashSlotcountIndex));\n        this.indexCount.set(byteBuffer.getInt(indexCountIndex));\n\n        if (this.indexCount.get() <= 0) {\n            this.indexCount.set(1);\n        }\n    }\n\n    public void updateByteBuffer() {\n        this.byteBuffer.putLong(beginTimestampIndex, this.beginTimestamp.get());\n        this.byteBuffer.putLong(endTimestampIndex, this.endTimestamp.get());\n        this.byteBuffer.putLong(beginPhyoffsetIndex, this.beginPhyOffset.get());\n        this.byteBuffer.putLong(endPhyoffsetIndex, this.endPhyOffset.get());\n        this.byteBuffer.putInt(hashSlotcountIndex, this.hashSlotCount.get());\n        this.byteBuffer.putInt(indexCountIndex, this.indexCount.get());\n    }\n\n    public long getBeginTimestamp() {\n        return beginTimestamp.get();\n    }\n\n    public void setBeginTimestamp(long beginTimestamp) {\n        this.beginTimestamp.set(beginTimestamp);\n        this.byteBuffer.putLong(beginTimestampIndex, beginTimestamp);\n    }\n\n    public long getEndTimestamp() {\n        return endTimestamp.get();\n    }\n\n    public void setEndTimestamp(long endTimestamp) {\n        this.endTimestamp.set(endTimestamp);\n        this.byteBuffer.putLong(endTimestampIndex, endTimestamp);\n    }\n\n    public long getBeginPhyOffset() {\n        return beginPhyOffset.get();\n    }\n\n    public void setBeginPhyOffset(long beginPhyOffset) {\n        this.beginPhyOffset.set(beginPhyOffset);\n        this.byteBuffer.putLong(beginPhyoffsetIndex, beginPhyOffset);\n    }\n\n    public long getEndPhyOffset() {\n        return endPhyOffset.get();\n    }\n\n    public void setEndPhyOffset(long endPhyOffset) {\n        this.endPhyOffset.set(endPhyOffset);\n        this.byteBuffer.putLong(endPhyoffsetIndex, endPhyOffset);\n    }\n\n    public AtomicInteger getHashSlotCount() {\n        return hashSlotCount;\n    }\n\n    public void incHashSlotCount() {\n        int value = this.hashSlotCount.incrementAndGet();\n        this.byteBuffer.putInt(hashSlotcountIndex, value);\n    }\n\n    public int getIndexCount() {\n        return indexCount.get();\n    }\n\n    public void incIndexCount() {\n        int value = this.indexCount.incrementAndGet();\n        this.byteBuffer.putInt(indexCountIndex, value);\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/index/IndexService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.index;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.locks.ReadWriteLock;\nimport java.util.concurrent.locks.ReentrantReadWriteLock;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.store.CommitLogDispatchStore;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.config.StorePathConfigHelper;\nimport org.rocksdb.RocksDBException;\n\npublic class IndexService implements CommitLogDispatchStore {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    /**\n     * Maximum times to attempt index file creation.\n     */\n    private static final int MAX_TRY_IDX_CREATE = 3;\n    private final DefaultMessageStore defaultMessageStore;\n    private final int hashSlotNum;\n    private final int indexNum;\n    private final String storePath;\n    private final ArrayList<IndexFile> indexFileList = new ArrayList<>();\n    private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();\n\n    public IndexService(final DefaultMessageStore store) {\n        this.defaultMessageStore = store;\n        this.hashSlotNum = store.getMessageStoreConfig().getMaxHashSlotNum();\n        this.indexNum = store.getMessageStoreConfig().getMaxIndexNum();\n        this.storePath =\n            StorePathConfigHelper.getStorePathIndex(defaultMessageStore.getMessageStoreConfig().getStorePathRootDir());\n    }\n\n    public boolean load(final boolean lastExitOK) {\n        File dir = new File(this.storePath);\n        File[] files = dir.listFiles();\n        if (files != null) {\n            // ascending order\n            Arrays.sort(files);\n            for (File file : files) {\n                try {\n                    IndexFile f = new IndexFile(file.getPath(), this.hashSlotNum, this.indexNum, 0, 0);\n                    f.load();\n\n                    if (!lastExitOK) {\n                        if (f.getEndTimestamp() > this.defaultMessageStore.getStoreCheckpoint()\n                            .getIndexMsgTimestamp()) {\n                            f.destroy(0);\n                            continue;\n                        }\n                    }\n\n                    LOGGER.info(\"load index file OK, \" + f.getFileName());\n                    this.indexFileList.add(f);\n                } catch (IOException e) {\n                    LOGGER.error(\"load file {} error\", file, e);\n                    return false;\n                } catch (NumberFormatException e) {\n                    LOGGER.error(\"load file {} error\", file, e);\n                }\n            }\n        }\n\n        return true;\n    }\n\n    public long getTotalSize() {\n        if (indexFileList.isEmpty()) {\n            return 0;\n        }\n\n        return (long) indexFileList.get(0).getFileSize() * indexFileList.size();\n    }\n\n    public void deleteExpiredFile(long offset) {\n        Object[] files = null;\n        try {\n            this.readWriteLock.readLock().lock();\n            if (this.indexFileList.isEmpty()) {\n                return;\n            }\n\n            long endPhyOffset = this.indexFileList.get(0).getEndPhyOffset();\n            if (endPhyOffset < offset) {\n                files = this.indexFileList.toArray();\n            }\n        } catch (Exception e) {\n            LOGGER.error(\"destroy exception\", e);\n        } finally {\n            this.readWriteLock.readLock().unlock();\n        }\n\n        if (files != null) {\n            List<IndexFile> fileList = new ArrayList<>();\n            for (int i = 0; i < (files.length - 1); i++) {\n                IndexFile f = (IndexFile) files[i];\n                if (f.getEndPhyOffset() < offset) {\n                    fileList.add(f);\n                } else {\n                    break;\n                }\n            }\n\n            this.deleteExpiredFile(fileList);\n        }\n    }\n\n    private void deleteExpiredFile(List<IndexFile> files) {\n        if (!files.isEmpty()) {\n            try {\n                this.readWriteLock.writeLock().lock();\n                for (IndexFile file : files) {\n                    boolean destroyed = file.destroy(3000);\n                    destroyed = destroyed && this.indexFileList.remove(file);\n                    if (!destroyed) {\n                        LOGGER.error(\"deleteExpiredFile remove failed.\");\n                        break;\n                    }\n                }\n            } catch (Exception e) {\n                LOGGER.error(\"deleteExpiredFile has exception.\", e);\n            } finally {\n                this.readWriteLock.writeLock().unlock();\n            }\n        }\n    }\n\n    public void destroy() {\n        try {\n            this.readWriteLock.writeLock().lock();\n            for (IndexFile f : this.indexFileList) {\n                f.destroy(1000 * 3);\n            }\n            this.indexFileList.clear();\n        } catch (Exception e) {\n            LOGGER.error(\"destroy exception\", e);\n        } finally {\n            this.readWriteLock.writeLock().unlock();\n        }\n    }\n\n    public QueryOffsetResult queryOffset(String topic, String key, int maxNum, long begin, long end) {\n        long indexLastUpdateTimestamp = 0;\n        long indexLastUpdatePhyoffset = 0;\n        maxNum = Math.min(maxNum, this.defaultMessageStore.getMessageStoreConfig().getMaxMsgsNumBatch());\n        List<Long> phyOffsets = new ArrayList<>(maxNum);\n        try {\n            this.readWriteLock.readLock().lock();\n            if (!this.indexFileList.isEmpty()) {\n                for (int i = this.indexFileList.size(); i > 0; i--) {\n                    IndexFile f = this.indexFileList.get(i - 1);\n                    boolean lastFile = i == this.indexFileList.size();\n                    if (lastFile) {\n                        indexLastUpdateTimestamp = f.getEndTimestamp();\n                        indexLastUpdatePhyoffset = f.getEndPhyOffset();\n                    }\n\n                    if (f.isTimeMatched(begin, end)) {\n\n                        f.selectPhyOffset(phyOffsets, buildKey(topic, key), maxNum, begin, end);\n                    }\n\n                    if (f.getBeginTimestamp() < begin) {\n                        break;\n                    }\n\n                    if (phyOffsets.size() >= maxNum) {\n                        break;\n                    }\n                }\n            }\n        } catch (Exception e) {\n            LOGGER.error(\"queryMsg exception\", e);\n        } finally {\n            this.readWriteLock.readLock().unlock();\n        }\n\n        return new QueryOffsetResult(phyOffsets, indexLastUpdateTimestamp, indexLastUpdatePhyoffset);\n    }\n\n    public QueryOffsetResult queryOffset(String topic, String key, int maxNum, long begin, long end, String indexType) {\n        List<Long> phyOffsets = new ArrayList<>(maxNum);\n        long indexLastUpdateTimestamp = 0;\n        long indexLastUpdatePhyoffset = 0;\n        maxNum = Math.min(maxNum, this.defaultMessageStore.getMessageStoreConfig().getMaxMsgsNumBatch());\n        try {\n            this.readWriteLock.readLock().lock();\n            if (!this.indexFileList.isEmpty()) {\n                for (int i = this.indexFileList.size(); i > 0; i--) {\n                    IndexFile f = this.indexFileList.get(i - 1);\n                    boolean lastFile = i == this.indexFileList.size();\n                    if (lastFile) {\n                        indexLastUpdateTimestamp = f.getEndTimestamp();\n                        indexLastUpdatePhyoffset = f.getEndPhyOffset();\n                    }\n\n                    if (f.isTimeMatched(begin, end)) {\n                        String queryKey;\n                        if (!StringUtils.isEmpty(indexType) && MessageConst.INDEX_TAG_TYPE.equals(indexType)) {\n                            queryKey = buildKey(topic, key, MessageConst.INDEX_TAG_TYPE);\n                        } else {\n                            queryKey = buildKey(topic, key);\n                        }\n                        f.selectPhyOffset(phyOffsets, queryKey, maxNum, begin, end);\n                    }\n\n                    if (f.getBeginTimestamp() < begin) {\n                        break;\n                    }\n\n                    if (phyOffsets.size() >= maxNum) {\n                        break;\n                    }\n                }\n            }\n        } catch (Exception e) {\n            LOGGER.error(\"queryMsg queryOffset exception\", e);\n        } finally {\n            this.readWriteLock.readLock().unlock();\n        }\n\n        return new QueryOffsetResult(phyOffsets, indexLastUpdateTimestamp, indexLastUpdatePhyoffset);\n    }\n\n    private String buildKey(final String topic, final String key) {\n        return topic + \"#\" + key;\n    }\n    private String buildKey(final String topic, final String key, final String indexType) {\n        return topic + \"#\" + indexType + \"#\" + key;\n    }\n\n    public void buildIndex(DispatchRequest req) {\n        IndexFile indexFile = retryGetAndCreateIndexFile();\n        if (indexFile != null) {\n            long endPhyOffset = indexFile.getEndPhyOffset();\n            DispatchRequest msg = req;\n            String topic = msg.getTopic();\n            String keys = msg.getKeys();\n            if (msg.getCommitLogOffset() < endPhyOffset) {\n                return;\n            }\n\n            final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag());\n            switch (tranType) {\n                case MessageSysFlag.TRANSACTION_NOT_TYPE:\n                case MessageSysFlag.TRANSACTION_PREPARED_TYPE:\n                case MessageSysFlag.TRANSACTION_COMMIT_TYPE:\n                    break;\n                case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:\n                    return;\n            }\n\n            if (req.getUniqKey() != null) {\n                indexFile = putKey(indexFile, msg, buildKey(topic, req.getUniqKey()));\n                if (indexFile == null) {\n                    LOGGER.error(\"putKey error commitlog {} uniqkey {}\", req.getCommitLogOffset(), req.getUniqKey());\n                    return;\n                }\n            }\n\n            if (keys != null && keys.length() > 0) {\n                String[] keyset = keys.split(MessageConst.KEY_SEPARATOR);\n                for (int i = 0; i < keyset.length; i++) {\n                    String key = keyset[i];\n                    if (key.length() > 0) {\n                        indexFile = putKey(indexFile, msg, buildKey(topic, key));\n                        if (indexFile == null) {\n                            LOGGER.error(\"putKey error commitlog {} uniqkey {}\", req.getCommitLogOffset(), req.getUniqKey());\n                            return;\n                        }\n                    }\n                }\n            }\n\n            Map<String, String> propertiesMap = req.getPropertiesMap();\n            if (null != propertiesMap && propertiesMap.containsKey(MessageConst.PROPERTY_TAGS)) {\n                String tags = req.getPropertiesMap().get(MessageConst.PROPERTY_TAGS);\n                if (!StringUtils.isEmpty(tags)) {\n                    indexFile = putKey(indexFile, msg, buildKey(topic, tags, MessageConst.INDEX_TAG_TYPE));\n                    if (indexFile == null) {\n                        LOGGER.error(\"putKey error commitlog {} uniqkey {}\", req.getCommitLogOffset(), req.getUniqKey());\n                        return;\n                    }\n                }\n            }\n\n        } else {\n            LOGGER.error(\"build index error, stop building index\");\n        }\n    }\n\n    private IndexFile putKey(IndexFile indexFile, DispatchRequest msg, String idxKey) {\n        for (boolean ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp()); !ok; ) {\n            LOGGER.warn(\"Index file [\" + indexFile.getFileName() + \"] is full, trying to create another one\");\n\n            indexFile = retryGetAndCreateIndexFile();\n            if (null == indexFile) {\n                return null;\n            }\n\n            ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp());\n        }\n\n        return indexFile;\n    }\n\n    /**\n     * Retries to get or create index file.\n     *\n     * @return {@link IndexFile} or null on failure.\n     */\n    public IndexFile retryGetAndCreateIndexFile() {\n        IndexFile indexFile = null;\n\n        for (int times = 0; null == indexFile && times < MAX_TRY_IDX_CREATE; times++) {\n            indexFile = this.getAndCreateLastIndexFile();\n            if (null != indexFile) {\n                break;\n            }\n\n            try {\n                LOGGER.info(\"Tried to create index file \" + times + \" times\");\n                Thread.sleep(1000);\n            } catch (InterruptedException e) {\n                LOGGER.error(\"Interrupted\", e);\n            }\n        }\n\n        if (null == indexFile) {\n            this.defaultMessageStore.getRunningFlags().makeIndexFileError();\n            LOGGER.error(\"Mark index file cannot build flag\");\n        }\n\n        return indexFile;\n    }\n\n    public IndexFile getAndCreateLastIndexFile() {\n        IndexFile indexFile = null;\n        IndexFile prevIndexFile = null;\n        long lastUpdateEndPhyOffset = 0;\n        long lastUpdateIndexTimestamp = 0;\n\n        {\n            this.readWriteLock.readLock().lock();\n            if (!this.indexFileList.isEmpty()) {\n                IndexFile tmp = this.indexFileList.get(this.indexFileList.size() - 1);\n                if (!tmp.isWriteFull()) {\n                    indexFile = tmp;\n                } else {\n                    lastUpdateEndPhyOffset = tmp.getEndPhyOffset();\n                    lastUpdateIndexTimestamp = tmp.getEndTimestamp();\n                    prevIndexFile = tmp;\n                }\n            }\n\n            this.readWriteLock.readLock().unlock();\n        }\n\n        if (indexFile == null) {\n            try {\n                String fileName =\n                    this.storePath + File.separator\n                        + UtilAll.timeMillisToHumanString(System.currentTimeMillis());\n                indexFile =\n                    new IndexFile(fileName, this.hashSlotNum, this.indexNum, lastUpdateEndPhyOffset,\n                        lastUpdateIndexTimestamp);\n                this.readWriteLock.writeLock().lock();\n                this.indexFileList.add(indexFile);\n            } catch (Exception e) {\n                LOGGER.error(\"getLastIndexFile exception \", e);\n            } finally {\n                this.readWriteLock.writeLock().unlock();\n            }\n\n            if (indexFile != null) {\n                final IndexFile flushThisFile = prevIndexFile;\n\n                Thread flushThread = new Thread(new Runnable() {\n                    @Override\n                    public void run() {\n                        IndexService.this.flush(flushThisFile);\n                    }\n                }, \"FlushIndexFileThread\");\n\n                flushThread.setDaemon(true);\n                flushThread.start();\n            }\n        }\n\n        return indexFile;\n    }\n\n    public void flush(final IndexFile f) {\n        if (null == f) {\n            return;\n        }\n\n        long indexMsgTimestamp = 0;\n\n        if (f.isWriteFull()) {\n            indexMsgTimestamp = f.getEndTimestamp();\n        }\n\n        f.flush();\n\n        if (indexMsgTimestamp > 0) {\n            this.defaultMessageStore.getStoreCheckpoint().setIndexMsgTimestamp(indexMsgTimestamp);\n            this.defaultMessageStore.getStoreCheckpoint().flush();\n        }\n    }\n\n    public void start() {\n\n    }\n\n    public void shutdown() {\n        try {\n            this.readWriteLock.writeLock().lock();\n            for (IndexFile f : this.indexFileList) {\n                try {\n                    f.shutdown();\n                } catch (Exception e) {\n                    LOGGER.error(\"shutdown \" + f.getFileName() + \" exception\", e);\n                }\n            }\n            this.indexFileList.clear();\n        } catch (Exception e) {\n            LOGGER.error(\"shutdown exception\", e);\n        } finally {\n            this.readWriteLock.writeLock().unlock();\n        }\n    }\n\n    @Override\n    public Long getDispatchFromPhyOffset(boolean recoverNormally) throws RocksDBException {\n        return -1L;\n    }\n\n    @Override\n    public boolean isMappedFileMatchedRecover(long phyOffset, long storeTimestamp,\n        boolean recoverNormally) throws RocksDBException {\n        if (this.defaultMessageStore.getMessageStoreConfig().isMessageIndexEnable() &&\n            this.defaultMessageStore.getMessageStoreConfig().isMessageIndexSafe()) {\n            if (storeTimestamp > this.defaultMessageStore.getStoreCheckpoint().getIndexMsgTimestamp()) {\n                return false;\n            }\n            LOGGER.info(\"CommitLog isMmapFileMatchedRecover find satisfied MmapFile for index, \" +\n                    \"MmapFile storeTimestamp={}, MmapFile phyOffset={}, indexMsgTimestamp={}, recoverNormally={}\",\n                storeTimestamp, phyOffset, this.defaultMessageStore.getStoreCheckpoint().getIndexMsgTimestamp(), recoverNormally);\n        }\n        return true;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/index/QueryOffsetResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.index;\n\nimport java.util.List;\n\npublic class QueryOffsetResult {\n    private final List<Long> phyOffsets;\n    private final long indexLastUpdateTimestamp;\n    private final long indexLastUpdatePhyoffset;\n\n    public QueryOffsetResult(List<Long> phyOffsets, long indexLastUpdateTimestamp,\n        long indexLastUpdatePhyoffset) {\n        this.phyOffsets = phyOffsets;\n        this.indexLastUpdateTimestamp = indexLastUpdateTimestamp;\n        this.indexLastUpdatePhyoffset = indexLastUpdatePhyoffset;\n    }\n\n    public List<Long> getPhyOffsets() {\n        return phyOffsets;\n    }\n\n    public long getIndexLastUpdateTimestamp() {\n        return indexLastUpdateTimestamp;\n    }\n\n    public long getIndexLastUpdatePhyoffset() {\n        return indexLastUpdatePhyoffset;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/index/rocksdb/IndexRocksDBRecord.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.index.rocksdb;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageConst;\n\npublic class IndexRocksDBRecord {\n    public static final String KEY_SPLIT = \"@\";\n    public static final byte[] KEY_SPLIT_BYTES = KEY_SPLIT.getBytes(StandardCharsets.UTF_8);\n    private static final int VALUE_LENGTH = Long.BYTES;\n    private long storeTime;\n    private String topic;\n    private String key;\n    private String tag;\n    private String uniqKey;\n    private long offsetPy;\n\n    public IndexRocksDBRecord(String topic, String key, String tag, long storeTime, String uniqKey, long offsetPy) {\n        this.topic = topic;\n        this.key = key;\n        this.tag = tag;\n        this.storeTime = storeTime;\n        this.uniqKey = uniqKey;\n        this.offsetPy = offsetPy;\n    }\n\n    public byte[] getKeyBytes() {\n        if (StringUtils.isEmpty(topic) || StringUtils.isEmpty(uniqKey) || offsetPy < 0L || storeTime <= 0L) {\n            return null;\n        }\n        long storeTimeHour = MixAll.dealTimeToHourStamps(storeTime);\n        if (storeTimeHour <= 0L) {\n            return null;\n        }\n        String keyMiddleStr;\n        if (!StringUtils.isEmpty(key)) {\n            keyMiddleStr = KEY_SPLIT + topic + KEY_SPLIT + MessageConst.INDEX_KEY_TYPE + KEY_SPLIT + key + KEY_SPLIT + uniqKey + KEY_SPLIT;\n        } else if (!StringUtils.isEmpty(tag)) {\n            keyMiddleStr = KEY_SPLIT + topic + KEY_SPLIT + MessageConst.INDEX_TAG_TYPE + KEY_SPLIT + tag + KEY_SPLIT + uniqKey + KEY_SPLIT;\n        } else {\n            keyMiddleStr = KEY_SPLIT + topic + KEY_SPLIT + MessageConst.INDEX_UNIQUE_TYPE + KEY_SPLIT + uniqKey + KEY_SPLIT;\n        }\n        if (StringUtils.isEmpty(keyMiddleStr)) {\n            return null;\n        }\n        byte[] keyMiddleBytes = keyMiddleStr.getBytes(StandardCharsets.UTF_8);\n        int keyLength = Long.BYTES + keyMiddleBytes.length + Long.BYTES;\n        return ByteBuffer.allocate(keyLength).putLong(storeTimeHour).put(keyMiddleBytes).putLong(offsetPy).array();\n    }\n\n    public byte[] getValueBytes() {\n        if (storeTime <= 0L) {\n            return null;\n        }\n        return ByteBuffer.allocate(VALUE_LENGTH).putLong(storeTime).array();\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getKey() {\n        return key;\n    }\n\n    public void setKey(String key) {\n        this.key = key;\n    }\n\n    public long getStoreTime() {\n        return storeTime;\n    }\n\n    public void setStoreTime(long storeTime) {\n        this.storeTime = storeTime;\n    }\n\n    public String getUniqKey() {\n        return uniqKey;\n    }\n\n    public void setUniqKey(String uniqKey) {\n        this.uniqKey = uniqKey;\n    }\n\n    public long getOffsetPy() {\n        return offsetPy;\n    }\n\n    public void setOffsetPy(long offsetPy) {\n        this.offsetPy = offsetPy;\n    }\n\n    public String getTag() {\n        return tag;\n    }\n\n    public void setTag(String tag) {\n        this.tag = tag;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/index/rocksdb/IndexRocksDBStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.index.rocksdb;\nimport java.io.File;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.nio.file.attribute.BasicFileAttributes;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.LinkedBlockingDeque;\nimport java.util.concurrent.TimeUnit;\nimport java.util.stream.Collectors;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.CommitLogDispatchStore;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.index.QueryOffsetResult;\nimport org.apache.rocketmq.store.logfile.MappedFile;\nimport org.apache.rocketmq.store.rocksdb.MessageRocksDBStorage;\nimport org.rocksdb.RocksDB;\nimport org.rocksdb.RocksDBException;\nimport static org.apache.rocketmq.common.MixAll.dealTimeToHourStamps;\n\npublic class IndexRocksDBStore implements CommitLogDispatchStore {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private static final Logger logError = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);\n    private static final int DEFAULT_CAPACITY = 100000;\n    private static final int BATCH_SIZE = 1000;\n    private static final Set<String> INDEX_TYPE_SET = new HashSet<>();\n\n    static {\n        INDEX_TYPE_SET.add(MessageConst.INDEX_KEY_TYPE);\n        INDEX_TYPE_SET.add(MessageConst.INDEX_TAG_TYPE);\n        INDEX_TYPE_SET.add(MessageConst.INDEX_UNIQUE_TYPE);\n    }\n    private static final int INITIAL = 0, RUNNING = 1, SHUTDOWN = 2;\n    private volatile int state = INITIAL;\n\n    private final MessageStore messageStore;\n    private final MessageStoreConfig storeConfig;\n    private final MessageRocksDBStorage messageRocksDBStorage;\n    private volatile long lastDeleteIndexTime = 0L;\n    private IndexBuildService indexBuildService;\n    private BlockingQueue<IndexRocksDBRecord> originIndexMsgQueue;\n\n    public IndexRocksDBStore(MessageStore messageStore) {\n        this.messageStore = messageStore;\n        this.storeConfig = messageStore.getMessageStoreConfig();\n        this.messageRocksDBStorage = messageStore.getMessageRocksDBStorage();\n        if (this.storeConfig.isIndexRocksDBEnable()) {\n            this.initAndStart();\n        }\n    }\n\n    private void initAndStart() {\n        if (this.state == RUNNING) {\n            return;\n        }\n        this.indexBuildService = new IndexBuildService();\n        this.originIndexMsgQueue = new LinkedBlockingDeque<>(DEFAULT_CAPACITY);\n        this.indexBuildService.start();\n        this.state = RUNNING;\n        Long lastOffsetPy = messageRocksDBStorage.getLastOffsetPy(RocksDB.DEFAULT_COLUMN_FAMILY);\n        log.info(\"IndexRocksDBStore start success, lastOffsetPy: {}\", lastOffsetPy);\n    }\n\n    public void shutdown() {\n        if (this.state != RUNNING || this.state == SHUTDOWN) {\n            return;\n        }\n        if (null != this.indexBuildService) {\n            this.indexBuildService.shutdown();\n        }\n        this.state = SHUTDOWN;\n        log.info(\"IndexRocksDBStore shutdown success\");\n    }\n\n    public QueryOffsetResult queryOffset(String topic, String key, int maxNum, long beginTime, long endTime, String indexType, String lastKey) {\n        if (StringUtils.isEmpty(topic) || StringUtils.isEmpty(key) || maxNum <= 0 || beginTime < 0L || endTime <= 0L || beginTime > endTime || !StringUtils.isEmpty(indexType) && !INDEX_TYPE_SET.contains(indexType)) {\n            logError.error(\"IndexRocksDBStore queryOffset param error, topic: {}, key: {}, maxNum: {}, beginTime: {}, endTime: {}, indexType: {}, lastKey: {}\", topic, key, maxNum, beginTime, endTime, indexType, lastKey);\n            return null;\n        }\n        if (beginTime == 0L || Long.MAX_VALUE == endTime) {\n            endTime = System.currentTimeMillis();\n            beginTime = endTime - TimeUnit.DAYS.toMillis(storeConfig.getMaxRocksDBIndexQueryDays());\n        }\n        if ((endTime - beginTime) > (TimeUnit.DAYS.toMillis(storeConfig.getMaxRocksDBIndexQueryDays()))) {\n            logError.error(\"IndexRocksDBStore queryOffset index in rocksdb, can not query more than: {} days\", storeConfig.getMaxRocksDBIndexQueryDays());\n            return null;\n        }\n        long lastUpdateTime = 0L;\n        long lastOffsetPy = 0L;\n        maxNum = Math.min(maxNum, this.storeConfig.getMaxMsgsNumBatch());\n        List<Long> phyOffsets = null;\n        try {\n            lastUpdateTime = messageRocksDBStorage.getLastStoreTimeStampForIndex(RocksDB.DEFAULT_COLUMN_FAMILY);\n            lastOffsetPy = messageRocksDBStorage.getLastOffsetPy(RocksDB.DEFAULT_COLUMN_FAMILY);\n            //compact old client\n            if (StringUtils.isEmpty(indexType)) {\n                phyOffsets = messageRocksDBStorage.queryOffsetForIndex(RocksDB.DEFAULT_COLUMN_FAMILY, topic, MessageConst.INDEX_KEY_TYPE, key, beginTime, endTime, maxNum, null);\n                if (CollectionUtils.isEmpty(phyOffsets)) {\n                    phyOffsets = messageRocksDBStorage.queryOffsetForIndex(RocksDB.DEFAULT_COLUMN_FAMILY, topic, MessageConst.INDEX_UNIQUE_TYPE, key, beginTime, endTime, maxNum, null);\n                }\n            } else {\n                phyOffsets = messageRocksDBStorage.queryOffsetForIndex(RocksDB.DEFAULT_COLUMN_FAMILY, topic, indexType, key, beginTime, endTime, maxNum, lastKey);\n            }\n        } catch (Exception e) {\n            logError.error(\"IndexRocksDBStore queryOffset error, topic: {}, key: {}, maxNum: {}, beginTime: {}, endTime: {}, error: {}\", topic, key, maxNum, beginTime, endTime, e.getMessage());\n        }\n        return new QueryOffsetResult(phyOffsets, lastUpdateTime, lastOffsetPy);\n    }\n\n    public void buildIndex(DispatchRequest dispatchRequest) {\n        if (null == dispatchRequest || dispatchRequest.getCommitLogOffset() < 0L || dispatchRequest.getMsgSize() <= 0 || state != RUNNING || null == this.originIndexMsgQueue) {\n            logError.error(\"IndexRocksDBStore buildIndex error, dispatchRequest: {}, state: {}, originIndexMsgQueue: {}\", dispatchRequest, state, originIndexMsgQueue);\n            return;\n        }\n        try {\n            long reqOffsetPy = dispatchRequest.getCommitLogOffset();\n            long endOffsetPy = messageRocksDBStorage.getLastOffsetPy(RocksDB.DEFAULT_COLUMN_FAMILY);\n            if (reqOffsetPy < endOffsetPy) {\n                if (System.currentTimeMillis() % 1000 == 0) {\n                    log.warn(\"IndexRocksDBStore recover buildIndex, but ignore, build index offset reqOffsetPy: {}, endOffsetPy: {}\", reqOffsetPy, endOffsetPy);\n                }\n                return;\n            }\n            final int tranType = MessageSysFlag.getTransactionValue(dispatchRequest.getSysFlag());\n            switch (tranType) {\n                case MessageSysFlag.TRANSACTION_NOT_TYPE:\n                case MessageSysFlag.TRANSACTION_PREPARED_TYPE:\n                case MessageSysFlag.TRANSACTION_COMMIT_TYPE:\n                    break;\n                case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:\n                    return;\n            }\n            String topic = dispatchRequest.getTopic();\n            String uniqKey = dispatchRequest.getUniqKey();\n            long storeTime = dispatchRequest.getStoreTimestamp();\n            if (StringUtils.isEmpty(topic) || StringUtils.isEmpty(uniqKey) || storeTime <= 0L || reqOffsetPy < 0L) {\n                return;\n            }\n            String keys = dispatchRequest.getKeys();\n            if (!StringUtils.isEmpty(keys)) {\n                String[] keySplit = keys.split(MessageConst.KEY_SEPARATOR);\n                if (keySplit.length > 0) {\n                    Set<String> keySet = Arrays.stream(keySplit).filter(i -> !StringUtils.isEmpty(i)).collect(Collectors.toSet());\n                    for (String key : keySet) {\n                        try {\n                            while (!originIndexMsgQueue.offer(new IndexRocksDBRecord(topic, key, null, storeTime, uniqKey, reqOffsetPy), 3, TimeUnit.SECONDS)) {\n                                if (System.currentTimeMillis() % 1000 == 0) {\n                                    logError.error(\"IndexRocksDBStore buildIndex keys error, topic: {}, key: {}, storeTime: {}, uniqKey: {}, reqOffsetPy: {}\", topic, key, storeTime, uniqKey, reqOffsetPy);\n                                }\n                            }\n                        } catch (Exception e) {\n                            logError.error(\"IndexRocksDBStore buildIndex keys error, key: {}, uniqKey: {}, topic: {}, error: {}\", key, uniqKey, topic, e.getMessage());\n                        }\n                    }\n                }\n            }\n            Map<String, String> propertiesMap = dispatchRequest.getPropertiesMap();\n            if (null != propertiesMap && propertiesMap.containsKey(MessageConst.PROPERTY_TAGS)) {\n                String tags = propertiesMap.get(MessageConst.PROPERTY_TAGS);\n                if (!StringUtils.isEmpty(tags)) {\n                    try {\n                        while (!originIndexMsgQueue.offer(new IndexRocksDBRecord(topic, null, tags, storeTime, uniqKey, reqOffsetPy), 3, TimeUnit.SECONDS)) {\n                            if (System.currentTimeMillis() % 1000 == 0) {\n                                logError.error(\"IndexRocksDBStore buildIndex offer tags error, topic: {}, tags: {}, storeTime: {}, uniqKey: {}, reqOffsetPy: {}\", topic, tags, storeTime, uniqKey, reqOffsetPy);\n                            }\n                        }\n                    } catch (Exception e) {\n                        logError.error(\"IndexRocksDBStore buildIndex tags error, tags: {}, uniqKey: {}, topic: {}, error: {}\", tags, uniqKey, topic, e.getMessage());\n                    }\n                }\n            }\n            try {\n                while (!originIndexMsgQueue.offer(new IndexRocksDBRecord(topic, null, null, storeTime, uniqKey, reqOffsetPy), 3, TimeUnit.SECONDS)) {\n                    if (System.currentTimeMillis() % 1000 == 0) {\n                        logError.error(\"IndexRocksDBStore buildIndex uniqKey error, topic: {}, storeTime: {}, uniqKey: {}, reqOffsetPy: {}\", topic, storeTime, uniqKey, reqOffsetPy);\n                    }\n                }\n            } catch (Exception e) {\n                logError.error(\"IndexRocksDBStore buildIndex uniqKey error: {}\", e.getMessage());\n            }\n        } catch (Exception e) {\n            logError.error(\"IndexRocksDBStore buildIndex error: {}\", e.getMessage());\n        }\n    }\n\n    public void deleteExpiredIndex() {\n        try {\n            MappedFile mappedFile = messageStore.getCommitLog().getEarliestMappedFile();\n            if (null == mappedFile) {\n                logError.info(\"IndexRocksDBStore deleteExpiredIndex mappedFile is null\");\n                return;\n            }\n            File file = mappedFile.getFile();\n            if (null == file || StringUtils.isEmpty(file.getAbsolutePath())) {\n                logError.info(\"IndexRocksDBStore deleteExpiredIndex error, file is null\");\n                return;\n            }\n            Path path = Paths.get(file.getAbsolutePath());\n            BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class);\n            long deleteIndexFileTime = attrs.creationTime().toMillis() - TimeUnit.HOURS.toMillis(1);\n            long desDeleteTimeHour = dealTimeToHourStamps(deleteIndexFileTime);\n            if (desDeleteTimeHour != lastDeleteIndexTime) {\n                messageRocksDBStorage.deleteRecordsForIndex(RocksDB.DEFAULT_COLUMN_FAMILY, desDeleteTimeHour, 168);\n                lastDeleteIndexTime = desDeleteTimeHour;\n            } else {\n                log.info(\"IndexRocksDBStore ignore this delete, lastDeleteIndexTime: {}, desDeleteTimeHour: {}\", lastDeleteIndexTime, desDeleteTimeHour);\n            }\n        } catch (Exception e) {\n            logError.error(\"IndexRocksDBStore deleteExpiredIndex rocksdb error: {}\", e.getMessage());\n        }\n    }\n\n    public boolean isMappedFileMatchedRecover(long phyOffset, long storeTimestamp,\n        boolean recoverNormally) throws RocksDBException {\n        if (!storeConfig.isIndexRocksDBEnable()) {\n            return true;\n        }\n        Long lastOffsetPy = messageRocksDBStorage.getLastOffsetPy(RocksDB.DEFAULT_COLUMN_FAMILY);\n        log.info(\"index isMappedFileMatchedRecover lastOffsetPy: {}\", lastOffsetPy);\n        if (null != lastOffsetPy && phyOffset <= lastOffsetPy) {\n            log.info(\"isMappedFileMatchedRecover IndexRocksDBStore recover form this offset, phyOffset: {}, lastOffsetPy: {}\", phyOffset, lastOffsetPy);\n            return true;\n        }\n        return false;\n    }\n\n    public void destroy() {\n    }\n\n    @Override\n    public Long getDispatchFromPhyOffset(boolean recoverNormally) throws RocksDBException {\n        if (!storeConfig.isIndexRocksDBEnable()) {\n            return null;\n        }\n        Long dispatchFromIndexPhyOffset = messageRocksDBStorage.getLastOffsetPy(RocksDB.DEFAULT_COLUMN_FAMILY);\n        if (dispatchFromIndexPhyOffset != null && dispatchFromIndexPhyOffset > 0) {\n            return dispatchFromIndexPhyOffset;\n        }\n        return null;\n    }\n\n    private String getServiceThreadName() {\n        String brokerIdentifier = \"\";\n        if (this.messageStore instanceof DefaultMessageStore) {\n            DefaultMessageStore messageStore = (DefaultMessageStore) IndexRocksDBStore.this.messageStore;\n            if (messageStore.getBrokerConfig().isInBrokerContainer()) {\n                brokerIdentifier = messageStore.getBrokerConfig().getIdentifier();\n            }\n        }\n        return brokerIdentifier;\n    }\n\n    private class IndexBuildService extends ServiceThread {\n        private final Logger log = IndexRocksDBStore.log;\n        private List<IndexRocksDBRecord> irs;\n        @Override\n        public String getServiceName() {\n            return getServiceThreadName() + this.getClass().getSimpleName();\n        }\n\n        @Override\n        public void run() {\n            log.info(this.getServiceName() + \" service start\");\n            irs = new ArrayList<>(BATCH_SIZE);\n            while (!this.isStopped() || !originIndexMsgQueue.isEmpty()) {\n                try {\n                    pollAndPutIndexRequest();\n                } catch (Exception e) {\n                    irs.clear();\n                    logError.error(\"IndexRocksDBStore error occurred in: {}, error: {}\", getServiceName(), e.getMessage());\n                }\n            }\n            log.info(this.getServiceName() + \" service end\");\n        }\n\n        private void pollAndPutIndexRequest() {\n            pollIndexRecord();\n            if (CollectionUtils.isEmpty(irs)) {\n                return;\n            }\n            try {\n                messageRocksDBStorage.writeRecordsForIndex(RocksDB.DEFAULT_COLUMN_FAMILY, irs);\n            } catch (Exception e) {\n                logError.error(\"IndexRocksDBStore IndexBuildService pollAndPutIndexRequest error: {}\", e.getMessage());\n            }\n            irs.clear();\n        }\n\n        private void pollIndexRecord() {\n            try {\n                IndexRocksDBRecord firstReq = originIndexMsgQueue.poll(100, TimeUnit.MILLISECONDS);\n                if (firstReq == null) {\n                    return;\n                }\n                irs.add(firstReq);\n                originIndexMsgQueue.drainTo(irs, BATCH_SIZE - irs.size());\n                while (irs.size() < BATCH_SIZE) {\n                    IndexRocksDBRecord tmpReq = originIndexMsgQueue.poll(100, TimeUnit.MILLISECONDS);\n                    if (tmpReq == null) {\n                        break;\n                    }\n                    irs.add(tmpReq);\n                    originIndexMsgQueue.drainTo(irs, BATCH_SIZE - irs.size());\n                }\n            } catch (Exception e) {\n                logError.error(\"IndexRocksDBStore IndexBuildService error: {}\", e.getMessage());\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/kv/CommitLogDispatcherCompaction.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.kv;\n\nimport org.apache.rocketmq.store.CommitLogDispatcher;\nimport org.apache.rocketmq.store.DispatchRequest;\n\npublic class CommitLogDispatcherCompaction implements CommitLogDispatcher {\n    private final CompactionService cptService;\n\n    public CommitLogDispatcherCompaction(CompactionService srv) {\n        this.cptService = srv;\n    }\n\n    @Override\n    public void dispatch(DispatchRequest request) {\n        if (cptService != null) {\n            cptService.putRequest(request);\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/kv/CompactionLog.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.kv;\n\nimport com.google.common.collect.Lists;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.ArrayUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.AppendMessageResult;\nimport org.apache.rocketmq.store.AppendMessageStatus;\nimport org.apache.rocketmq.store.CompactionAppendMsgCallback;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.MappedFileQueue;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageLock;\nimport org.apache.rocketmq.store.PutMessageReentrantLock;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageSpinLock;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.StoreUtil;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.logfile.MappedFile;\nimport org.apache.rocketmq.store.queue.BatchConsumeQueue;\nimport org.apache.rocketmq.store.queue.CqUnit;\nimport org.apache.rocketmq.store.queue.ReferredIterator;\nimport org.apache.rocketmq.store.queue.SparseConsumeQueue;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Paths;\nimport java.security.DigestException;\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.util.Arrays;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicReference;\nimport java.util.stream.Collectors;\n\nimport static org.apache.rocketmq.common.message.MessageDecoder.BLANK_MAGIC_CODE;\n\npublic class CompactionLog {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n\n    private static final int END_FILE_MIN_BLANK_LENGTH = 4 + 4;\n    private static final int MAX_PULL_MSG_SIZE = 128 * 1024 * 1024;\n    public static final String COMPACTING_SUB_FOLDER = \"compacting\";\n    public static final String REPLICATING_SUB_FOLDER = \"replicating\";\n\n    private final int compactionLogMappedFileSize;\n    private final int compactionCqMappedFileSize;\n    private final String compactionLogFilePath;\n    private final String compactionCqFilePath;\n    private final MessageStore defaultMessageStore;\n    private final CompactionStore compactionStore;\n    private final MessageStoreConfig messageStoreConfig;\n    private final CompactionAppendMsgCallback endMsgCallback;\n    private final String topic;\n    private final int queueId;\n    private final int offsetMapMemorySize;\n    private final PutMessageLock putMessageLock;\n    private final PutMessageLock readMessageLock;\n    private TopicPartitionLog current;\n    private TopicPartitionLog compacting;\n    private TopicPartitionLog replicating;\n    private final CompactionPositionMgr positionMgr;\n    private final AtomicReference<State> state;\n\n    public CompactionLog(final MessageStore messageStore, final CompactionStore compactionStore, final String topic, final int queueId)\n        throws IOException {\n        this.topic = topic;\n        this.queueId = queueId;\n        this.defaultMessageStore = messageStore;\n        this.compactionStore = compactionStore;\n        this.messageStoreConfig = messageStore.getMessageStoreConfig();\n        this.offsetMapMemorySize = compactionStore.getOffsetMapSize();\n        this.compactionCqMappedFileSize =\n            messageStoreConfig.getCompactionCqMappedFileSize() / BatchConsumeQueue.CQ_STORE_UNIT_SIZE\n                * BatchConsumeQueue.CQ_STORE_UNIT_SIZE;\n        this.compactionLogMappedFileSize = getCompactionLogSize(compactionCqMappedFileSize,\n            messageStoreConfig.getCompactionMappedFileSize());\n        this.compactionLogFilePath = Paths.get(compactionStore.getCompactionLogPath(),\n            topic, String.valueOf(queueId)).toString();\n        this.compactionCqFilePath = compactionStore.getCompactionCqPath();        // batch consume queue already separated\n        this.positionMgr = compactionStore.getPositionMgr();\n\n        this.putMessageLock =\n            messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() :\n                new PutMessageSpinLock();\n        this.readMessageLock =\n            messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() :\n                new PutMessageSpinLock();\n        this.endMsgCallback = new CompactionAppendEndMsgCallback();\n        this.state = new AtomicReference<>(State.INITIALIZING);\n        log.info(\"CompactionLog {}:{} init completed.\", topic, queueId);\n    }\n\n    private int getCompactionLogSize(int cqSize, int origLogSize) {\n        int n = origLogSize / cqSize;\n        if (n < 5) {\n            return cqSize * 5;\n        }\n        int m = origLogSize % cqSize;\n        if (m > 0 && m < (cqSize >> 1)) {\n            return n * cqSize;\n        } else {\n            return (n + 1) * cqSize;\n        }\n    }\n\n    public void load(boolean exitOk) throws IOException, RuntimeException {\n        initLogAndCq(exitOk);\n        if (defaultMessageStore.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE\n            && getLog().isMappedFilesEmpty()) {\n            log.info(\"{}:{} load compactionLog from remote master\", topic, queueId);\n            loadFromRemoteAsync();\n        } else {\n            state.compareAndSet(State.INITIALIZING, State.NORMAL);\n        }\n    }\n\n    private void initLogAndCq(boolean exitOk) throws IOException, RuntimeException {\n        current = new TopicPartitionLog(this);\n        current.init(exitOk);\n    }\n\n\n    private boolean putMessageFromRemote(byte[] bytes) {\n        ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);\n        // split bytebuffer to avoid encode message again\n        while (byteBuffer.hasRemaining()) {\n            int mark = byteBuffer.position();\n            ByteBuffer bb = byteBuffer.slice();\n            int size = bb.getInt();\n            if (size < 0 || size > byteBuffer.capacity()) {\n                break;\n            } else {\n                bb.limit(size);\n                bb.rewind();\n            }\n\n            MessageExt messageExt = MessageDecoder.decode(bb, false, false);\n            long messageOffset = messageExt.getQueueOffset();\n            long minOffsetInQueue = getCQ().getMinOffsetInQueue();\n            if (getLog().isMappedFilesEmpty() || messageOffset < minOffsetInQueue) {\n                asyncPutMessage(bb, messageExt, replicating);\n            } else {\n                log.info(\"{}:{} message offset {} >= minOffsetInQueue {}, stop pull...\",\n                    topic, queueId, messageOffset, minOffsetInQueue);\n                return false;\n            }\n\n            byteBuffer.position(mark + size);\n        }\n\n        return true;\n\n    }\n\n    private void pullMessageFromMaster() throws Exception {\n\n        if (StringUtils.isBlank(compactionStore.getMasterAddr())) {\n            compactionStore.getCompactionSchedule().schedule(() -> {\n                try {\n                    pullMessageFromMaster();\n                } catch (Exception e) {\n                    log.error(\"pullMessageFromMaster exception: \", e);\n                }\n            }, 5, TimeUnit.SECONDS);\n            return;\n        }\n\n        replicating = new TopicPartitionLog(this, REPLICATING_SUB_FOLDER);\n        try (MessageFetcher messageFetcher = new MessageFetcher()) {\n            messageFetcher.pullMessageFromMaster(topic, queueId, getCQ().getMinOffsetInQueue(),\n                compactionStore.getMasterAddr(), (currOffset, response) -> {\n                    if (currOffset < 0) {\n                        log.info(\"{}:{} current offset {}, stop pull...\", topic, queueId, currOffset);\n                        return false;\n                    }\n                    return putMessageFromRemote(response.getBody());\n//                    positionMgr.setOffset(topic, queueId, currOffset);\n                });\n        }\n\n        // merge files\n        if (getLog().isMappedFilesEmpty()) {\n            replaceFiles(getLog().getMappedFiles(), current, replicating);\n        } else if (replicating.getLog().isMappedFilesEmpty()) {\n            log.info(\"replicating message is empty\");   //break\n        } else {\n            List<MappedFile> newFiles = Lists.newArrayList();\n            List<MappedFile> toCompactFiles = Lists.newArrayList(replicating.getLog().getMappedFiles());\n            putMessageLock.lock();\n            try {\n                // combine current and replicating to mappedFileList\n                newFiles = Lists.newArrayList(getLog().getMappedFiles());\n                toCompactFiles.addAll(newFiles);  //all from current\n                current.roll(toCompactFiles.size() * compactionLogMappedFileSize);\n            } catch (Throwable e) {\n                log.error(\"roll log and cq exception: \", e);\n            } finally {\n                putMessageLock.unlock();\n            }\n\n            try {\n                // doCompaction with current and replicating\n                compactAndReplace(new ProcessFileList(toCompactFiles, toCompactFiles));\n            } catch (Throwable e) {\n                log.error(\"do merge replicating and current exception: \", e);\n            }\n        }\n\n        // cleanReplicatingResource, force clean cq\n        replicating.clean(false, true);\n\n//        positionMgr.setOffset(topic, queueId, currentPullOffset);\n        state.compareAndSet(State.INITIALIZING, State.NORMAL);\n    }\n    private void loadFromRemoteAsync() {\n        compactionStore.getCompactionSchedule().submit(() -> {\n            try {\n                pullMessageFromMaster();\n            } catch (Exception e) {\n                log.error(\"fetch message from master exception: \", e);\n            }\n        });\n\n        // update (currentStatus) = LOADING\n\n        // request => get (start, end)\n        // pull message => current message offset > end\n        // done\n        // positionMgr.persist();\n\n        // update (currentStatus) = RUNNING\n    }\n\n    private long nextOffsetCorrection(long oldOffset, long newOffset) {\n        long nextOffset = oldOffset;\n        if (messageStoreConfig.getBrokerRole() != BrokerRole.SLAVE || messageStoreConfig.isOffsetCheckInSlave()) {\n            nextOffset = newOffset;\n        }\n        return nextOffset;\n    }\n\n    private boolean checkInDiskByCommitOffset(long offsetPy, long maxOffsetPy) {\n        long memory = (long) (StoreUtil.TOTAL_PHYSICAL_MEMORY_SIZE *\n            (this.messageStoreConfig.getAccessMessageInMemoryMaxRatio() / 100.0));\n        return (maxOffsetPy - offsetPy) > memory;\n    }\n\n    private boolean isTheBatchFull(int sizePy, int unitBatchNum, int maxMsgNums, long maxMsgSize,\n        int bufferTotal, int messageTotal, boolean isInDisk) {\n\n        if (0 == bufferTotal || 0 == messageTotal) {\n            return false;\n        }\n\n        if (messageTotal + unitBatchNum > maxMsgNums) {\n            return true;\n        }\n\n        if (bufferTotal + sizePy > maxMsgSize) {\n            return true;\n        }\n\n        if (isInDisk) {\n            if ((bufferTotal + sizePy) > this.messageStoreConfig.getMaxTransferBytesOnMessageInDisk()) {\n                return true;\n            }\n\n            if (messageTotal > this.messageStoreConfig.getMaxTransferCountOnMessageInDisk() - 1) {\n                return true;\n            }\n        } else {\n            if ((bufferTotal + sizePy) > this.messageStoreConfig.getMaxTransferBytesOnMessageInMemory()) {\n                return true;\n            }\n\n            if (messageTotal > this.messageStoreConfig.getMaxTransferCountOnMessageInMemory() - 1) {\n                return true;\n            }\n        }\n\n        return false;\n    }\n\n    public long rollNextFile(final long offset) {\n        return offset + compactionLogMappedFileSize - offset % compactionLogMappedFileSize;\n    }\n\n    boolean shouldRetainMsg(final MessageExt msgExt, final OffsetMap map) throws DigestException {\n        if (msgExt.getQueueOffset() > map.getLastOffset()) {\n            return true;\n        }\n\n        String key = msgExt.getKeys();\n        if (StringUtils.isNotBlank(key)) {\n            boolean keyNotExistOrOffsetBigger = msgExt.getQueueOffset() >= map.get(key);\n            boolean hasBody = ArrayUtils.isNotEmpty(msgExt.getBody());\n            return keyNotExistOrOffsetBigger && hasBody;\n        } else {\n            log.error(\"message has no keys\");\n            return false;\n        }\n    }\n\n    public void checkAndPutMessage(final SelectMappedBufferResult selectMappedBufferResult, final MessageExt msgExt,\n        final OffsetMap offsetMap, final TopicPartitionLog tpLog)\n        throws DigestException {\n        if (shouldRetainMsg(msgExt, offsetMap)) {\n            asyncPutMessage(selectMappedBufferResult.getByteBuffer(), msgExt, tpLog);\n        }\n    }\n\n    public CompletableFuture<PutMessageResult> asyncPutMessage(final SelectMappedBufferResult selectMappedBufferResult) {\n        return asyncPutMessage(selectMappedBufferResult, current);\n    }\n\n    public CompletableFuture<PutMessageResult> asyncPutMessage(final SelectMappedBufferResult selectMappedBufferResult,\n        final TopicPartitionLog tpLog) {\n        MessageExt msgExt = MessageDecoder.decode(selectMappedBufferResult.getByteBuffer(), false, false);\n        return asyncPutMessage(selectMappedBufferResult.getByteBuffer(), msgExt, tpLog);\n    }\n\n    public CompletableFuture<PutMessageResult> asyncPutMessage(final ByteBuffer msgBuffer,\n        final MessageExt msgExt, final TopicPartitionLog tpLog) {\n        return asyncPutMessage(msgBuffer, msgExt.getTopic(), msgExt.getQueueId(),\n            msgExt.getQueueOffset(), msgExt.getMsgId(), msgExt.getKeys(),\n            MessageExtBrokerInner.tagsString2tagsCode(msgExt.getTags()), msgExt.getStoreTimestamp(), tpLog);\n    }\n\n    public CompletableFuture<PutMessageResult> asyncPutMessage(final ByteBuffer msgBuffer,\n        final DispatchRequest dispatchRequest) {\n        return asyncPutMessage(msgBuffer, dispatchRequest.getTopic(), dispatchRequest.getQueueId(),\n            dispatchRequest.getConsumeQueueOffset(), dispatchRequest.getUniqKey(), dispatchRequest.getKeys(),\n            dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(), current);\n    }\n\n    public CompletableFuture<PutMessageResult> asyncPutMessage(final ByteBuffer msgBuffer,\n        final DispatchRequest dispatchRequest, final TopicPartitionLog tpLog) {\n        return asyncPutMessage(msgBuffer, dispatchRequest.getTopic(), dispatchRequest.getQueueId(),\n            dispatchRequest.getConsumeQueueOffset(), dispatchRequest.getUniqKey(), dispatchRequest.getKeys(),\n            dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(), tpLog);\n    }\n\n    public CompletableFuture<PutMessageResult> asyncPutMessage(final ByteBuffer msgBuffer,\n        String topic, int queueId, long queueOffset, String msgId, String keys, long tagsCode, long storeTimestamp, final TopicPartitionLog tpLog) {\n\n        // fix duplicate\n        if (tpLog.getCQ().getMaxOffsetInQueue() - 1 >= queueOffset) {\n            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null));\n        }\n\n        if (StringUtils.isBlank(keys)) {\n            log.warn(\"message {}-{}:{} have no key, will not put in compaction log\", topic, queueId, msgId);\n            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null));\n        }\n\n        putMessageLock.lock();\n        try {\n            long beginTime = System.nanoTime();\n\n            if (tpLog.isEmptyOrCurrentFileFull()) {\n                try {\n                    tpLog.roll();\n                } catch (IOException e) {\n                    log.error(\"create mapped file or consumerQueue exception: \", e);\n                    return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, null));\n                }\n            }\n\n            MappedFile mappedFile = tpLog.getLog().getLastMappedFile();\n\n            CompactionAppendMsgCallback callback = new CompactionAppendMessageCallback(topic, queueId, tagsCode, storeTimestamp, tpLog.getCQ());\n            AppendMessageResult result = mappedFile.appendMessage(msgBuffer, callback);\n\n            switch (result.getStatus()) {\n                case PUT_OK:\n                    break;\n                case END_OF_FILE:\n                    try {\n                        tpLog.roll();\n                    } catch (IOException e) {\n                        log.error(\"create mapped file2 error, topic: {}, msgId: {}\", topic, msgId);\n                        return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, result));\n                    }\n                    mappedFile = tpLog.getLog().getLastMappedFile();\n                    result = mappedFile.appendMessage(msgBuffer, callback);\n                    break;\n                default:\n                    return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result));\n            }\n\n            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK, result));\n        } finally {\n            putMessageLock.unlock();\n        }\n    }\n\n    private SelectMappedBufferResult getMessage(final long offset, final int size) {\n\n        MappedFile mappedFile = this.getLog().findMappedFileByOffset(offset, offset == 0);\n        if (mappedFile != null) {\n            int pos = (int) (offset % compactionLogMappedFileSize);\n            return mappedFile.selectMappedBuffer(pos, size);\n        }\n        return null;\n    }\n\n    private boolean validateCqUnit(CqUnit cqUnit) {\n        return cqUnit.getPos() >= 0\n            && cqUnit.getSize() > 0\n            && cqUnit.getQueueOffset() >= 0\n            && cqUnit.getBatchNum() > 0;\n    }\n\n    public GetMessageResult getMessage(final String group, final String topic, final int queueId, final long offset,\n        final int maxMsgNums, final int maxTotalMsgSize) {\n        readMessageLock.lock();\n        try {\n            long beginTime = System.nanoTime();\n\n            GetMessageStatus status;\n            long nextBeginOffset = offset;\n            long minOffset = 0;\n            long maxOffset = 0;\n\n            GetMessageResult getResult = new GetMessageResult();\n\n            final long maxOffsetPy = getLog().getMaxOffset();\n\n            SparseConsumeQueue consumeQueue = getCQ();\n            if (consumeQueue != null) {\n                minOffset = consumeQueue.getMinOffsetInQueue();\n                maxOffset = consumeQueue.getMaxOffsetInQueue();\n\n                if (maxOffset == 0) {\n                    status = GetMessageStatus.NO_MESSAGE_IN_QUEUE;\n                    nextBeginOffset = nextOffsetCorrection(offset, 0);\n                } else if (offset == maxOffset) {\n                    status = GetMessageStatus.OFFSET_OVERFLOW_ONE;\n                    nextBeginOffset = nextOffsetCorrection(offset, offset);\n                } else if (offset > maxOffset) {\n                    status = GetMessageStatus.OFFSET_OVERFLOW_BADLY;\n                    if (0 == minOffset) {\n                        nextBeginOffset = nextOffsetCorrection(offset, minOffset);\n                    } else {\n                        nextBeginOffset = nextOffsetCorrection(offset, maxOffset);\n                    }\n                } else {\n\n                    long maxPullSize = Math.max(maxTotalMsgSize, 100);\n                    if (maxPullSize > MAX_PULL_MSG_SIZE) {\n                        log.warn(\"The max pull size is too large maxPullSize={} topic={} queueId={}\",\n                            maxPullSize, topic, queueId);\n                        maxPullSize = MAX_PULL_MSG_SIZE;\n                    }\n                    status = GetMessageStatus.NO_MATCHED_MESSAGE;\n                    long maxPhyOffsetPulling = 0;\n                    int cqFileNum = 0;\n\n                    while (getResult.getBufferTotalSize() <= 0 && nextBeginOffset < maxOffset\n                        && cqFileNum++ < this.messageStoreConfig.getTravelCqFileNumWhenGetMessage()) {\n                        ReferredIterator<CqUnit> bufferConsumeQueue = consumeQueue.iterateFromOrNext(nextBeginOffset);\n\n                        if (bufferConsumeQueue == null) {\n                            status = GetMessageStatus.OFFSET_FOUND_NULL;\n                            nextBeginOffset = nextOffsetCorrection(nextBeginOffset, consumeQueue.rollNextFile(nextBeginOffset));\n                            log.warn(\"consumer request topic:{}, offset:{}, minOffset:{}, maxOffset:{}, \"\n                                    + \"but access logic queue failed. correct nextBeginOffset to {}\",\n                                topic, offset, minOffset, maxOffset, nextBeginOffset);\n                            break;\n                        }\n\n                        try {\n                            long nextPhyFileStartOffset = Long.MIN_VALUE;\n                            while (bufferConsumeQueue.hasNext() && nextBeginOffset < maxOffset) {\n                                CqUnit cqUnit = bufferConsumeQueue.next();\n                                if (!validateCqUnit(cqUnit)) {\n                                    break;\n                                }\n                                long offsetPy = cqUnit.getPos();\n                                int sizePy = cqUnit.getSize();\n\n                                boolean isInDisk = checkInDiskByCommitOffset(offsetPy, maxOffsetPy);\n\n                                if (isTheBatchFull(sizePy, cqUnit.getBatchNum(), maxMsgNums, maxPullSize,\n                                    getResult.getBufferTotalSize(), getResult.getMessageCount(), isInDisk)) {\n                                    break;\n                                }\n\n                                if (getResult.getBufferTotalSize() >= maxPullSize) {\n                                    break;\n                                }\n\n                                maxPhyOffsetPulling = offsetPy;\n\n                                //Be careful, here should before the isTheBatchFull\n                                nextBeginOffset = cqUnit.getQueueOffset() + cqUnit.getBatchNum();\n\n                                if (nextPhyFileStartOffset != Long.MIN_VALUE) {\n                                    if (offsetPy < nextPhyFileStartOffset) {\n                                        continue;\n                                    }\n                                }\n\n                                SelectMappedBufferResult selectResult = getMessage(offsetPy, sizePy);\n                                if (null == selectResult) {\n                                    if (getResult.getBufferTotalSize() == 0) {\n                                        status = GetMessageStatus.MESSAGE_WAS_REMOVING;\n                                    }\n\n                                    // nextPhyFileStartOffset = this.commitLog.rollNextFile(offsetPy);\n                                    nextPhyFileStartOffset = rollNextFile(offsetPy);\n                                    continue;\n                                }\n                                this.defaultMessageStore.getStoreStatsService().getGetMessageTransferredMsgCount().add(cqUnit.getBatchNum());\n                                getResult.addMessage(selectResult, cqUnit.getQueueOffset(), cqUnit.getBatchNum());\n                                status = GetMessageStatus.FOUND;\n                                nextPhyFileStartOffset = Long.MIN_VALUE;\n                            }\n                        } finally {\n                            bufferConsumeQueue.release();\n                        }\n                    }\n\n                    long diff = maxOffsetPy - maxPhyOffsetPulling;\n                    long memory = (long)(StoreUtil.TOTAL_PHYSICAL_MEMORY_SIZE * (this.messageStoreConfig.getAccessMessageInMemoryMaxRatio() / 100.0));\n                    getResult.setSuggestPullingFromSlave(diff > memory);\n                }\n            } else {\n                status = GetMessageStatus.NO_MATCHED_LOGIC_QUEUE;\n                nextBeginOffset = nextOffsetCorrection(offset, 0);\n            }\n\n            if (GetMessageStatus.FOUND == status) {\n                this.defaultMessageStore.getStoreStatsService().getGetMessageTimesTotalFound().add(getResult.getMessageCount());\n            } else {\n                this.defaultMessageStore.getStoreStatsService().getGetMessageTimesTotalMiss().add(getResult.getMessageCount());\n            }\n            long elapsedTime = this.defaultMessageStore.getSystemClock().now() - beginTime;\n            this.defaultMessageStore.getStoreStatsService().setGetMessageEntireTimeMax(elapsedTime);\n\n            getResult.setStatus(status);\n            getResult.setNextBeginOffset(nextBeginOffset);\n            getResult.setMaxOffset(maxOffset);\n            getResult.setMinOffset(minOffset);\n            return getResult;\n        } finally {\n            readMessageLock.unlock();\n        }\n    }\n\n    ProcessFileList getCompactionFile() {\n        List<MappedFile> mappedFileList = Lists.newArrayList(getLog().getMappedFiles());\n        if (mappedFileList.size() < 2) {\n            return null;\n        }\n\n        List<MappedFile> toCompactFiles = mappedFileList.subList(0, mappedFileList.size() - 1);\n\n        //exclude the last writing file\n        List<MappedFile> newFiles = Lists.newArrayList();\n        for (int i = 0; i < mappedFileList.size() - 1; i++) {\n            MappedFile mf = mappedFileList.get(i);\n            long maxQueueOffsetInFile = getCQ().getMaxMsgOffsetFromFile(mf.getFile().getName());\n            if (maxQueueOffsetInFile > positionMgr.getOffset(topic, queueId)) {\n                newFiles.add(mf);\n            }\n        }\n\n        if (newFiles.isEmpty()) {\n            return null;\n        }\n\n        return new ProcessFileList(toCompactFiles, newFiles);\n    }\n\n    void compactAndReplace(ProcessFileList compactFiles) throws Throwable {\n        if (compactFiles == null || compactFiles.isEmpty()) {\n            return;\n        }\n\n        long startTime = System.nanoTime();\n        OffsetMap offsetMap = getOffsetMap(compactFiles.newFiles);\n        compaction(compactFiles.toCompactFiles, offsetMap);\n        replaceFiles(compactFiles.toCompactFiles, current, compacting);\n        positionMgr.setOffset(topic, queueId, offsetMap.lastOffset);\n        positionMgr.persist();\n        compacting.clean(false, false);\n        log.info(\"this compaction elapsed {} milliseconds\",\n            TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime));\n\n    }\n\n    void doCompaction() {\n        if (!state.compareAndSet(State.NORMAL, State.COMPACTING)) {\n            log.warn(\"compactionLog state is {}, skip this time\", state.get());\n            return;\n        }\n\n        try {\n            compactAndReplace(getCompactionFile());\n        } catch (Throwable e) {\n            log.error(\"do compaction exception: \", e);\n        }\n        state.compareAndSet(State.COMPACTING, State.NORMAL);\n    }\n\n    protected OffsetMap getOffsetMap(List<MappedFile> mappedFileList) throws NoSuchAlgorithmException, DigestException {\n        OffsetMap offsetMap = new OffsetMap(offsetMapMemorySize);\n\n        for (MappedFile mappedFile : mappedFileList) {\n            Iterator<SelectMappedBufferResult> iterator = mappedFile.iterator(0);\n            while (iterator.hasNext()) {\n                SelectMappedBufferResult smb = null;\n                try {\n                    smb = iterator.next();\n                    //decode bytebuffer\n                    MessageExt msg = MessageDecoder.decode(smb.getByteBuffer(), true, false);\n                    if (msg != null) {\n                        ////get key & offset and put to offsetMap\n                        if (msg.getQueueOffset() > positionMgr.getOffset(topic, queueId)) {\n                            offsetMap.put(msg.getKeys(), msg.getQueueOffset());\n                        }\n                    } else {\n                        // msg is null indicate that file is end\n                        break;\n                    }\n                } catch (DigestException e) {\n                    log.error(\"offsetMap put exception: \", e);\n                    throw e;\n                } finally {\n                    if (smb != null) {\n                        smb.release();\n                    }\n                }\n            }\n        }\n        return offsetMap;\n    }\n\n    protected void putEndMessage(MappedFileQueue mappedFileQueue) {\n        MappedFile lastFile = mappedFileQueue.getLastMappedFile();\n        if (!lastFile.isFull()) {\n            lastFile.appendMessage(ByteBuffer.allocate(0), endMsgCallback);\n        }\n    }\n\n    protected void compaction(List<MappedFile> mappedFileList, OffsetMap offsetMap) throws DigestException {\n        compacting = new TopicPartitionLog(this, COMPACTING_SUB_FOLDER);\n\n        for (MappedFile mappedFile : mappedFileList) {\n            Iterator<SelectMappedBufferResult> iterator = mappedFile.iterator(0);\n            while (iterator.hasNext()) {\n                SelectMappedBufferResult smb = null;\n                try {\n                    smb = iterator.next();\n                    MessageExt msgExt = MessageDecoder.decode(smb.getByteBuffer(), true, true);\n                    if (msgExt == null) {\n                        // file end\n                        break;\n                    } else {\n                        checkAndPutMessage(smb, msgExt, offsetMap, compacting);\n                    }\n                } finally {\n                    if (smb != null) {\n                        smb.release();\n                    }\n                }\n            }\n        }\n        putEndMessage(compacting.getLog());\n    }\n\n    protected void replaceFiles(List<MappedFile> mappedFileList, TopicPartitionLog current,\n        TopicPartitionLog newLog) {\n\n        MappedFileQueue dest = current.getLog();\n        MappedFileQueue src = newLog.getLog();\n\n        long beginTime = System.nanoTime();\n//        List<String> fileNameToReplace = mappedFileList.stream()\n//            .map(m -> m.getFile().getName())\n//            .collect(Collectors.toList());\n\n        List<String> fileNameToReplace = dest.getMappedFiles().stream()\n            .filter(mappedFileList::contains)\n            .map(mf -> mf.getFile().getName())\n            .collect(Collectors.toList());\n\n        mappedFileList.forEach(MappedFile::renameToDelete);\n\n        src.getMappedFiles().forEach(mappedFile -> {\n            try {\n                mappedFile.flush(0);\n                mappedFile.moveToParent();\n            } catch (IOException e) {\n                log.error(\"move file {} to parent directory exception: \", mappedFile.getFileName());\n            }\n        });\n\n        dest.getMappedFiles().stream()\n            .filter(m -> !mappedFileList.contains(m))\n            .forEach(m -> src.getMappedFiles().add(m));\n\n        readMessageLock.lock();\n        try {\n            mappedFileList.forEach(mappedFile -> mappedFile.destroy(1000));\n\n            dest.getMappedFiles().clear();\n            dest.getMappedFiles().addAll(src.getMappedFiles());\n            src.getMappedFiles().clear();\n\n            replaceCqFiles(getCQ(), newLog.getCQ(), fileNameToReplace);\n\n            log.info(\"replace file elapsed {} milliseconds\",\n                TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - beginTime));\n        } finally {\n            readMessageLock.unlock();\n        }\n    }\n\n    protected void replaceCqFiles(SparseConsumeQueue currentBcq, SparseConsumeQueue compactionBcq,\n        List<String> fileNameToReplace) {\n        long beginTime = System.nanoTime();\n\n        MappedFileQueue currentMq = currentBcq.getMappedFileQueue();\n        MappedFileQueue compactMq = compactionBcq.getMappedFileQueue();\n        List<MappedFile> fileListToDelete = currentMq.getMappedFiles().stream().filter(m ->\n            fileNameToReplace.contains(m.getFile().getName())).collect(Collectors.toList());\n\n        fileListToDelete.forEach(MappedFile::renameToDelete);\n        compactMq.getMappedFiles().forEach(mappedFile -> {\n            try {\n                mappedFile.flush(0);\n                mappedFile.moveToParent();\n            } catch (IOException e) {\n                log.error(\"move consume queue file {} to parent directory exception: \", mappedFile.getFileName(), e);\n            }\n        });\n\n        currentMq.getMappedFiles().stream()\n            .filter(m -> !fileListToDelete.contains(m))\n            .forEach(m -> compactMq.getMappedFiles().add(m));\n\n        fileListToDelete.forEach(mappedFile -> mappedFile.destroy(1000));\n\n        currentMq.getMappedFiles().clear();\n        currentMq.getMappedFiles().addAll(compactMq.getMappedFiles());\n        compactMq.getMappedFiles().clear();\n\n        currentBcq.refresh();\n        log.info(\"replace consume queue file elapsed {} millsecs.\",\n            TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - beginTime));\n    }\n\n    public MappedFileQueue getLog() {\n        return current.mappedFileQueue;\n    }\n\n    public SparseConsumeQueue getCQ() {\n        return current.consumeQueue;\n    }\n\n//    public SparseConsumeQueue getCompactionScq() {\n//        return compactionScq;\n//    }\n\n    public void flush(int flushLeastPages) {\n        this.flushLog(flushLeastPages);\n        this.flushCQ(flushLeastPages);\n    }\n\n    public void flushLog(int flushLeastPages) {\n        getLog().flush(flushLeastPages);\n    }\n\n    public void flushCQ(int flushLeastPages) {\n        getCQ().flush(flushLeastPages);\n    }\n\n    static class CompactionAppendEndMsgCallback implements CompactionAppendMsgCallback {\n        @Override\n        public AppendMessageResult doAppend(ByteBuffer bbDest, long fileFromOffset, int maxBlank, ByteBuffer bbSrc) {\n            ByteBuffer endInfo = ByteBuffer.allocate(END_FILE_MIN_BLANK_LENGTH);\n            endInfo.putInt(maxBlank);\n            endInfo.putInt(BLANK_MAGIC_CODE);\n            return new AppendMessageResult(AppendMessageStatus.END_OF_FILE,\n                fileFromOffset + bbDest.position(), maxBlank, System.currentTimeMillis());\n        }\n    }\n\n    static class CompactionAppendMessageCallback implements CompactionAppendMsgCallback {\n        private final String topic;\n        private final int queueId;\n        private final long tagsCode;\n        private final long storeTimestamp;\n\n        private final SparseConsumeQueue bcq;\n        public CompactionAppendMessageCallback(MessageExt msgExt, SparseConsumeQueue bcq) {\n            this.topic = msgExt.getTopic();\n            this.queueId =  msgExt.getQueueId();\n            this.tagsCode = MessageExtBrokerInner.tagsString2tagsCode(msgExt.getTags());\n            this.storeTimestamp = msgExt.getStoreTimestamp();\n\n            this.bcq = bcq;\n        }\n        public CompactionAppendMessageCallback(String topic, int queueId, long tagsCode, long storeTimestamp, SparseConsumeQueue bcq) {\n            this.topic = topic;\n            this.queueId =  queueId;\n            this.tagsCode = tagsCode;\n            this.storeTimestamp = storeTimestamp;\n\n            this.bcq = bcq;\n        }\n\n        @Override\n        public AppendMessageResult doAppend(ByteBuffer bbDest, long fileFromOffset, int maxBlank, ByteBuffer bbSrc) {\n\n            final int msgLen = bbSrc.getInt(0);\n            MappedFile bcqMappedFile = bcq.getMappedFileQueue().getLastMappedFile();\n            if (bcqMappedFile.getWrotePosition() + BatchConsumeQueue.CQ_STORE_UNIT_SIZE >= bcqMappedFile.getFileSize()\n                || (msgLen + END_FILE_MIN_BLANK_LENGTH) > maxBlank) {      //bcq will full or log will full\n\n                bcq.putEndPositionInfo(bcqMappedFile);\n\n                bbDest.putInt(maxBlank);\n                bbDest.putInt(BLANK_MAGIC_CODE);\n                return new AppendMessageResult(AppendMessageStatus.END_OF_FILE,\n                    fileFromOffset + bbDest.position(), maxBlank, storeTimestamp);\n            }\n\n            //get logic offset and physical offset\n            int logicOffsetPos = 4 + 4 + 4 + 4 + 4;\n            long logicOffset = bbSrc.getLong(logicOffsetPos);\n            int destPos = bbDest.position();\n            long physicalOffset = fileFromOffset + bbDest.position();\n            bbSrc.rewind();\n            bbSrc.limit(msgLen);\n            bbDest.put(bbSrc);\n            bbDest.putLong(destPos + logicOffsetPos + 8, physicalOffset);       //replace physical offset\n\n            boolean result = bcq.putBatchMessagePositionInfo(physicalOffset, msgLen,\n                tagsCode, storeTimestamp, logicOffset, (short)1);\n            if (!result) {\n                log.error(\"put message {}-{} position info failed\", topic, queueId);\n            }\n            return new AppendMessageResult(AppendMessageStatus.PUT_OK, physicalOffset, msgLen, storeTimestamp);\n        }\n    }\n\n    static class OffsetMap {\n        private final ByteBuffer dataBytes;\n        private final int capacity;\n        private final int entrySize;\n        private int entryNum;\n        private final MessageDigest digest;\n        private final int hashSize;\n        private long lastOffset;\n        private final byte[] hash1;\n        private final byte[] hash2;\n\n        public OffsetMap(int memorySize) throws NoSuchAlgorithmException {\n            this(memorySize, MessageDigest.getInstance(\"MD5\"));\n        }\n\n        public OffsetMap(int memorySize, MessageDigest digest) {\n            this.hashSize = digest.getDigestLength();\n            this.entrySize = hashSize + (Long.SIZE / Byte.SIZE);\n            this.capacity = Math.max(memorySize / entrySize, 100);\n            this.dataBytes = ByteBuffer.allocate(capacity * entrySize);\n            this.hash1 = new byte[hashSize];\n            this.hash2 = new byte[hashSize];\n            this.entryNum = 0;\n            this.digest = digest;\n        }\n\n        public void put(String key, final long offset) throws DigestException {\n            if (entryNum >= capacity) {\n                throw new IllegalArgumentException(\"offset map is full\");\n            }\n            hashInto(key, hash1);\n            int tryNum = 0;\n            int index = indexOf(hash1, tryNum);\n            while (!isEmpty(index)) {\n                dataBytes.position(index);\n                dataBytes.get(hash2);\n                if (Arrays.equals(hash1, hash2)) {\n                    dataBytes.putLong(offset);\n                    lastOffset = offset;\n                    return;\n                }\n                tryNum++;\n                index = indexOf(hash1, tryNum);\n            }\n\n            dataBytes.position(index);\n            dataBytes.put(hash1);\n            dataBytes.putLong(offset);\n            lastOffset = offset;\n            entryNum += 1;\n        }\n\n        public long get(String key) throws DigestException {\n            hashInto(key, hash1);\n            int tryNum = 0;\n            int maxTryNum = entryNum + hashSize - 4;\n            int index = 0;\n            do {\n                if (tryNum >= maxTryNum) {\n                    return -1L;\n                }\n                index = indexOf(hash1, tryNum);\n                dataBytes.position(index);\n                if (isEmpty(index)) {\n                    return -1L;\n                }\n                dataBytes.get(hash2);\n                tryNum++;\n            } while (!Arrays.equals(hash1, hash2));\n            return dataBytes.getLong();\n        }\n\n        public long getLastOffset() {\n            return lastOffset;\n        }\n\n        private boolean isEmpty(int pos) {\n            return dataBytes.getLong(pos) == 0\n                && dataBytes.getLong(pos + 8) == 0\n                && dataBytes.getLong(pos + 16) == 0;\n        }\n\n        private int indexOf(byte[] hash, int tryNum) {\n            int index = readInt(hash, Math.min(tryNum, hashSize - 4)) + Math.max(0, tryNum - hashSize + 4);\n            int entry = Math.abs(index) % capacity;\n            return entry * entrySize;\n        }\n\n        private void hashInto(String key, byte[] buf) throws DigestException {\n            digest.update(key.getBytes(StandardCharsets.UTF_8));\n            digest.digest(buf, 0, hashSize);\n        }\n\n        private int readInt(byte[] buf, int offset) {\n            return ((buf[offset] & 0xFF) << 24) |\n                ((buf[offset + 1] & 0xFF) << 16) |\n                ((buf[offset + 2] & 0xFF) << 8) |\n                ((buf[offset + 3] & 0xFF));\n        }\n    }\n\n    static class TopicPartitionLog {\n        MappedFileQueue mappedFileQueue;\n        SparseConsumeQueue consumeQueue;\n\n        public TopicPartitionLog(CompactionLog compactionLog) {\n            this(compactionLog, null);\n        }\n        public TopicPartitionLog(CompactionLog compactionLog, String subFolder) {\n            if (StringUtils.isBlank(subFolder)) {\n                mappedFileQueue = new MappedFileQueue(compactionLog.compactionLogFilePath,\n                    compactionLog.compactionLogMappedFileSize, null);\n                consumeQueue = new SparseConsumeQueue(compactionLog.topic, compactionLog.queueId,\n                    compactionLog.compactionCqFilePath, compactionLog.compactionCqMappedFileSize,\n                    compactionLog.defaultMessageStore);\n            } else {\n                mappedFileQueue = new MappedFileQueue(compactionLog.compactionLogFilePath + File.separator + subFolder,\n                    compactionLog.compactionLogMappedFileSize, null);\n                consumeQueue = new SparseConsumeQueue(compactionLog.topic, compactionLog.queueId,\n                    compactionLog.compactionCqFilePath, compactionLog.compactionCqMappedFileSize,\n                    compactionLog.defaultMessageStore, subFolder);\n            }\n        }\n\n        public void shutdown() {\n            mappedFileQueue.shutdown(1000 * 30);\n            consumeQueue.getMappedFileQueue().shutdown(1000 * 30);\n        }\n\n        public void init(boolean exitOk) throws IOException, RuntimeException {\n            if (!mappedFileQueue.load()) {\n                shutdown();\n                throw new IOException(\"load log exception\");\n            }\n\n            if (!consumeQueue.load()) {\n                shutdown();\n                throw new IOException(\"load consume queue exception\");\n            }\n\n            try {\n                consumeQueue.recover();\n                recover();\n                sanityCheck();\n            } catch (Exception e) {\n                shutdown();\n                throw e;\n            }\n        }\n\n        private void recover() {\n            long maxCqPhysicOffset = consumeQueue.getMaxPhyOffsetInLog();\n            log.info(\"{}:{} max physical offset in compaction log is {}\",\n                consumeQueue.getTopic(), consumeQueue.getQueueId(), maxCqPhysicOffset);\n            if (maxCqPhysicOffset > 0) {\n                this.mappedFileQueue.setFlushedWhere(maxCqPhysicOffset);\n                this.mappedFileQueue.setCommittedWhere(maxCqPhysicOffset);\n                this.mappedFileQueue.truncateDirtyFiles(maxCqPhysicOffset);\n            }\n        }\n\n        void sanityCheck() throws RuntimeException {\n            List<MappedFile> mappedFileList = mappedFileQueue.getMappedFiles();\n            for (MappedFile file : mappedFileList) {\n                if (!consumeQueue.containsOffsetFile(Long.parseLong(file.getFile().getName()))) {\n                    throw new RuntimeException(\"log file mismatch with consumeQueue file \" + file.getFileName());\n                }\n            }\n\n            List<MappedFile> cqMappedFileList = consumeQueue.getMappedFileQueue().getMappedFiles();\n            for (MappedFile file: cqMappedFileList) {\n                if (mappedFileList.stream().noneMatch(m -> Objects.equals(m.getFile().getName(), file.getFile().getName()))) {\n                    throw new RuntimeException(\"consumeQueue file mismatch with log file \" + file.getFileName());\n                }\n            }\n        }\n\n        public synchronized void roll() throws IOException {\n            MappedFile mappedFile = mappedFileQueue.getLastMappedFile(0);\n            if (mappedFile == null) {\n                throw new IOException(\"create new file error\");\n            }\n            long baseOffset = mappedFile.getFileFromOffset();\n            MappedFile cqFile = consumeQueue.createFile(baseOffset);\n            if (cqFile == null) {\n                mappedFile.destroy(1000);\n                mappedFileQueue.getMappedFiles().remove(mappedFile);\n                throw new IOException(\"create new consumeQueue file error\");\n            }\n        }\n\n        public synchronized void roll(int baseOffset) throws IOException {\n\n            MappedFile mappedFile = mappedFileQueue.tryCreateMappedFile(baseOffset);\n            if (mappedFile == null) {\n                throw new IOException(\"create new file error\");\n            }\n\n            MappedFile cqFile = consumeQueue.createFile(baseOffset);\n            if (cqFile == null) {\n                mappedFile.destroy(1000);\n                mappedFileQueue.getMappedFiles().remove(mappedFile);\n                throw new IOException(\"create new consumeQueue file error\");\n            }\n        }\n\n        public boolean isEmptyOrCurrentFileFull() {\n            return mappedFileQueue.isEmptyOrCurrentFileFull() ||\n                consumeQueue.getMappedFileQueue().isEmptyOrCurrentFileFull();\n        }\n\n        public void clean(MappedFileQueue mappedFileQueue) throws IOException {\n            for (MappedFile mf : mappedFileQueue.getMappedFiles()) {\n                if (mf.getFile().exists()) {\n                    log.error(\"directory {} with {} not empty.\", mappedFileQueue.getStorePath(), mf.getFileName());\n                    throw new IOException(\"directory \" + mappedFileQueue.getStorePath() + \" not empty.\");\n                }\n            }\n\n            mappedFileQueue.destroy();\n        }\n\n        public void clean(boolean forceCleanLog, boolean forceCleanCq) throws IOException {\n            //clean and delete sub_folder\n            if (forceCleanLog) {\n                mappedFileQueue.destroy();\n            } else {\n                clean(mappedFileQueue);\n            }\n\n            if (forceCleanCq) {\n                consumeQueue.getMappedFileQueue().destroy();\n            } else {\n                clean(consumeQueue.getMappedFileQueue());\n            }\n        }\n\n        public MappedFileQueue getLog() {\n            return mappedFileQueue;\n        }\n\n        public SparseConsumeQueue getCQ() {\n            return consumeQueue;\n        }\n    }\n\n    enum State {\n        NORMAL,\n        INITIALIZING,\n        COMPACTING,\n    }\n\n    static class ProcessFileList {\n        List<MappedFile> newFiles;\n        List<MappedFile> toCompactFiles;\n        public ProcessFileList(List<MappedFile> toCompactFiles, List<MappedFile> newFiles) {\n            this.toCompactFiles = toCompactFiles;\n            this.newFiles = newFiles;\n        }\n\n        boolean isEmpty() {\n            return CollectionUtils.isEmpty(newFiles) || CollectionUtils.isEmpty(toCompactFiles);\n        }\n    }\n\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/kv/CompactionPositionMgr.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.kv;\n\nimport org.apache.rocketmq.common.ConfigManager;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\nimport java.io.File;\nimport java.util.concurrent.ConcurrentHashMap;\n\npublic class CompactionPositionMgr extends ConfigManager {\n\n    public static final String CHECKPOINT_FILE = \"position-checkpoint\";\n\n    private transient String compactionPath;\n    private transient String checkpointFileName;\n\n    private ConcurrentHashMap<String, Long> queueOffsetMap = new ConcurrentHashMap<>();\n\n    private CompactionPositionMgr() {\n\n    }\n\n    public CompactionPositionMgr(final String compactionPath) {\n        this.compactionPath = compactionPath;\n        this.checkpointFileName = compactionPath + File.separator + CHECKPOINT_FILE;\n        this.load();\n    }\n\n    public void setOffset(String topic, int queueId, final long offset) {\n        queueOffsetMap.put(topic + \"_\" + queueId, offset);\n    }\n\n    public long getOffset(String topic, int queueId) {\n        return queueOffsetMap.getOrDefault(topic + \"_\" + queueId, -1L);\n    }\n\n    public boolean isEmpty() {\n        return queueOffsetMap.isEmpty();\n    }\n\n    public boolean isCompaction(String topic, int queueId, long offset) {\n        return getOffset(topic, queueId) > offset;\n    }\n\n    @Override\n    public String configFilePath() {\n        return checkpointFileName;\n    }\n\n    @Override\n    public String encode() {\n        return this.encode(false);\n    }\n\n    @Override\n    public String encode(boolean prettyFormat) {\n        return RemotingSerializable.toJson(this, prettyFormat);\n    }\n\n    @Override\n    public void decode(String jsonString) {\n        if (jsonString != null) {\n            CompactionPositionMgr obj = RemotingSerializable.fromJson(jsonString, CompactionPositionMgr.class);\n            if (obj != null) {\n                this.queueOffsetMap = obj.queueOffsetMap;\n            }\n        }\n    }\n\n    public ConcurrentHashMap<String, Long> getQueueOffsetMap() {\n        return queueOffsetMap;\n    }\n\n    public void setQueueOffsetMap(ConcurrentHashMap<String, Long> queueOffsetMap) {\n        this.queueOffsetMap = queueOffsetMap;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/kv/CompactionService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.kv;\n\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.attribute.CleanupPolicy;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.CleanupPolicyUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.CommitLog;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\n\nimport java.util.Objects;\nimport java.util.Optional;\n\npublic class CompactionService {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n\n    private final CompactionStore compactionStore;\n    private final DefaultMessageStore defaultMessageStore;\n    private final CommitLog commitLog;\n\n    public CompactionService(CommitLog commitLog, DefaultMessageStore messageStore, CompactionStore compactionStore) {\n        this.commitLog = commitLog;\n        this.defaultMessageStore = messageStore;\n        this.compactionStore = compactionStore;\n    }\n\n    public void putRequest(DispatchRequest request) {\n        if (request == null) {\n            return;\n        }\n\n        String topic = request.getTopic();\n        Optional<TopicConfig> topicConfig = defaultMessageStore.getTopicConfig(topic);\n        CleanupPolicy policy = CleanupPolicyUtils.getDeletePolicy(topicConfig);\n        //check request topic flag\n        if (Objects.equals(policy, CleanupPolicy.COMPACTION)) {\n            SelectMappedBufferResult smr = null;\n            try {\n                smr = commitLog.getData(request.getCommitLogOffset());\n                if (smr != null) {\n                    compactionStore.doDispatch(request, smr);\n                }\n            } catch (Exception e) {\n                log.error(\"putMessage into {}:{} compactionLog exception: \", request.getTopic(), request.getQueueId(), e);\n            } finally {\n                if (smr != null) {\n                    smr.release();\n                }\n            }\n        } // else skip if message isn't compaction\n    }\n\n    public boolean load(boolean exitOK) {\n        try {\n            compactionStore.load(exitOK);\n            return true;\n        } catch (Exception e) {\n            log.error(\"load compaction store error \", e);\n            return false;\n        }\n    }\n\n//    @Override\n//    public void start() {\n//        compactionStore.load();\n//        super.start();\n//    }\n\n    public void shutdown() {\n        compactionStore.shutdown();\n    }\n\n    public void updateMasterAddress(String addr) {\n        compactionStore.updateMasterAddress(addr);\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/kv/CompactionStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.kv;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.nio.file.Files;\nimport java.nio.file.Paths;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Optional;\nimport java.util.Random;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.attribute.CleanupPolicy;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.CleanupPolicyUtils;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\n\npublic class CompactionStore {\n\n    public static final String COMPACTION_DIR = \"compaction\";\n    public static final String COMPACTION_LOG_DIR = \"compactionLog\";\n    public static final String COMPACTION_CQ_DIR = \"compactionCq\";\n\n    private final String compactionPath;\n    private final String compactionLogPath;\n    private final String compactionCqPath;\n    private final DefaultMessageStore defaultMessageStore;\n    private final CompactionPositionMgr positionMgr;\n    private final ConcurrentHashMap<String, CompactionLog> compactionLogTable;\n    private final ScheduledExecutorService compactionSchedule;\n    private final int scanInterval = 30000;\n    private final int compactionInterval;\n    private final int compactionThreadNum;\n    private final int offsetMapSize;\n    private String masterAddr;\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n\n    public CompactionStore(DefaultMessageStore defaultMessageStore) {\n        this.defaultMessageStore = defaultMessageStore;\n        this.compactionLogTable = new ConcurrentHashMap<>();\n        MessageStoreConfig config = defaultMessageStore.getMessageStoreConfig();\n        String storeRootPath = config.getStorePathRootDir();\n        this.compactionPath = Paths.get(storeRootPath, COMPACTION_DIR).toString();\n        this.compactionLogPath = Paths.get(compactionPath, COMPACTION_LOG_DIR).toString();\n        this.compactionCqPath = Paths.get(compactionPath, COMPACTION_CQ_DIR).toString();\n        this.positionMgr = new CompactionPositionMgr(compactionPath);\n        this.compactionThreadNum = Math.min(Runtime.getRuntime().availableProcessors(), Math.max(1, config.getCompactionThreadNum()));\n\n        this.compactionSchedule = ThreadUtils.newScheduledThreadPool(this.compactionThreadNum,\n            new ThreadFactoryImpl(\"compactionSchedule_\"));\n        this.offsetMapSize = config.getMaxOffsetMapSize() / compactionThreadNum;\n\n        this.compactionInterval = defaultMessageStore.getMessageStoreConfig().getCompactionScheduleInternal();\n    }\n\n    public void load(boolean exitOk) throws Exception {\n        File logRoot = new File(compactionLogPath);\n        File[] fileTopicList = logRoot.listFiles();\n        if (fileTopicList != null) {\n            for (File fileTopic : fileTopicList) {\n                if (!fileTopic.isDirectory()) {\n                    continue;\n                }\n\n                File[] fileQueueIdList = fileTopic.listFiles();\n                if (fileQueueIdList != null) {\n                    for (File fileQueueId : fileQueueIdList) {\n                        if (!fileQueueId.isDirectory()) {\n                            continue;\n                        }\n                        try {\n                            String topic = fileTopic.getName();\n                            int queueId = Integer.parseInt(fileQueueId.getName());\n\n                            if (Files.isDirectory(Paths.get(compactionCqPath, topic, String.valueOf(queueId)))) {\n                                loadAndGetClog(topic, queueId);\n                            } else {\n                                log.error(\"{}:{} compactionLog mismatch with compactionCq\", topic, queueId);\n                            }\n                        } catch (Exception e) {\n                            log.error(\"load compactionLog {}:{} exception: \",\n                                fileTopic.getName(), fileQueueId.getName(), e);\n                            throw new Exception(\"load compactionLog \" + fileTopic.getName()\n                                + \":\" + fileQueueId.getName() + \" exception: \" + e.getMessage());\n                        }\n                    }\n                }\n            }\n        }\n        log.info(\"compactionStore {}:{} load completed.\", compactionLogPath, compactionCqPath);\n\n        compactionSchedule.scheduleWithFixedDelay(this::scanAllTopicConfig, scanInterval, scanInterval, TimeUnit.MILLISECONDS);\n        log.info(\"loop to scan all topicConfig with fixed delay {}ms\", scanInterval);\n    }\n\n    private void scanAllTopicConfig() {\n        log.info(\"start to scan all topicConfig\");\n        try {\n            Iterator<Map.Entry<String, TopicConfig>> iterator = defaultMessageStore.getTopicConfigs().entrySet().iterator();\n            while (iterator.hasNext()) {\n                Map.Entry<String, TopicConfig> it = iterator.next();\n                TopicConfig topicConfig = it.getValue();\n                CleanupPolicy policy = CleanupPolicyUtils.getDeletePolicy(Optional.ofNullable(topicConfig));\n                //check topic flag\n                if (Objects.equals(policy, CleanupPolicy.COMPACTION)) {\n                    for (int queueId = 0; queueId < topicConfig.getWriteQueueNums(); queueId++) {\n                        loadAndGetClog(it.getKey(), queueId);\n                    }\n                }\n            }\n        } catch (Throwable ignore) {\n            // ignore\n        }\n        log.info(\"scan all topicConfig over\");\n    }\n\n    private CompactionLog loadAndGetClog(String topic, int queueId) {\n        CompactionLog clog = compactionLogTable.compute(topic + \"_\" + queueId, (k, v) -> {\n            if (v == null) {\n                try {\n                    v = new CompactionLog(defaultMessageStore, this, topic, queueId);\n                    v.load(true);\n                    int randomDelay = 1000 + new Random(System.currentTimeMillis()).nextInt(compactionInterval);\n                    compactionSchedule.scheduleWithFixedDelay(v::doCompaction, compactionInterval + randomDelay, compactionInterval + randomDelay, TimeUnit.MILLISECONDS);\n                } catch (IOException e) {\n                    log.error(\"create compactionLog exception: \", e);\n                    return null;\n                }\n            }\n            return v;\n        });\n        return clog;\n    }\n\n    public void putMessage(String topic, int queueId, SelectMappedBufferResult smr) throws Exception {\n        CompactionLog clog = loadAndGetClog(topic, queueId);\n\n        if (clog != null) {\n            clog.asyncPutMessage(smr);\n        }\n    }\n\n    public void doDispatch(DispatchRequest dispatchRequest, SelectMappedBufferResult smr) throws Exception {\n        CompactionLog clog = loadAndGetClog(dispatchRequest.getTopic(), dispatchRequest.getQueueId());\n\n        if (clog != null) {\n            clog.asyncPutMessage(smr.getByteBuffer(), dispatchRequest);\n        }\n    }\n\n    public GetMessageResult getMessage(final String group, final String topic, final int queueId, final long offset,\n        final int maxMsgNums, final int maxTotalMsgSize) {\n        CompactionLog log = compactionLogTable.get(topic + \"_\" + queueId);\n        if (log == null) {\n            return GetMessageResult.NO_MATCH_LOGIC_QUEUE;\n        } else {\n            return log.getMessage(group, topic, queueId, offset, maxMsgNums, maxTotalMsgSize);\n        }\n\n    }\n\n    public void flush(int flushLeastPages) {\n        compactionLogTable.values().forEach(log -> log.flush(flushLeastPages));\n    }\n\n    public void flushLog(int flushLeastPages) {\n        compactionLogTable.values().forEach(log -> log.flushLog(flushLeastPages));\n    }\n\n    public void flushCQ(int flushLeastPages) {\n        compactionLogTable.values().forEach(log -> log.flushCQ(flushLeastPages));\n    }\n\n    public void updateMasterAddress(String addr) {\n        this.masterAddr = addr;\n    }\n\n    public void shutdown() {\n        // close the thread pool first\n        compactionSchedule.shutdown();\n        try {\n            if (!compactionSchedule.awaitTermination(1000, TimeUnit.MILLISECONDS)) {\n                List<Runnable> droppedTasks = compactionSchedule.shutdownNow();\n                log.warn(\"compactionSchedule was abruptly shutdown. {} tasks will not be executed.\", droppedTasks.size());\n            }\n        } catch (InterruptedException e) {\n            log.warn(\"wait compaction schedule shutdown interrupted. \");\n        }\n        this.flush(0);\n        positionMgr.persist();\n    }\n\n    public ScheduledExecutorService getCompactionSchedule() {\n        return compactionSchedule;\n    }\n\n    public String getCompactionLogPath() {\n        return compactionLogPath;\n    }\n\n    public String getCompactionCqPath() {\n        return compactionCqPath;\n    }\n\n    public CompactionPositionMgr getPositionMgr() {\n        return positionMgr;\n    }\n\n    public int getOffsetMapSize() {\n        return offsetMapSize;\n    }\n\n    public String getMasterAddr() {\n        return masterAddr;\n    }\n\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/kv/MessageFetcher.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.kv;\n\nimport com.google.common.collect.Lists;\nimport com.google.common.collect.Sets;\n\nimport java.io.IOException;\nimport java.util.function.BiFunction;\n\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.sysflag.PullSysFlag;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.RemotingClient;\nimport org.apache.rocketmq.remoting.exception.RemotingCommandException;\nimport org.apache.rocketmq.remoting.exception.RemotingConnectException;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyRemotingClient;\nimport org.apache.rocketmq.remoting.protocol.LanguageCode;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.apache.rocketmq.remoting.protocol.header.HeartbeatRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PullMessageResponseHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UnregisterClientRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.ConsumerData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\n\npublic class MessageFetcher implements AutoCloseable {\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private final RemotingClient client;\n\n    public MessageFetcher() {\n        NettyClientConfig nettyClientConfig = new NettyClientConfig();\n        nettyClientConfig.setUseTLS(false);\n        this.client = new NettyRemotingClient(nettyClientConfig);\n        this.client.start();\n    }\n\n    @Override\n    public void close() throws IOException {\n        this.client.shutdown();\n    }\n\n    private PullMessageRequestHeader createPullMessageRequest(String topic, int queueId, long queueOffset, long subVersion) {\n        int sysFlag = PullSysFlag.buildSysFlag(false, false, false, false, true);\n\n        PullMessageRequestHeader requestHeader = new PullMessageRequestHeader();\n        requestHeader.setConsumerGroup(getConsumerGroup(topic, queueId));\n        requestHeader.setTopic(topic);\n        requestHeader.setQueueId(queueId);\n        requestHeader.setQueueOffset(queueOffset);\n        requestHeader.setMaxMsgNums(10);\n        requestHeader.setSysFlag(sysFlag);\n        requestHeader.setCommitOffset(0L);\n        requestHeader.setSuspendTimeoutMillis(20_000L);\n//        requestHeader.setSubscription(subExpression);\n        requestHeader.setSubVersion(subVersion);\n        requestHeader.setMaxMsgBytes(Integer.MAX_VALUE);\n//        requestHeader.setExpressionType(expressionType);\n        return requestHeader;\n    }\n\n    private String getConsumerGroup(String topic, int queueId) {\n        return String.join(\"-\", topic, String.valueOf(queueId), \"pull\", \"group\");\n    }\n\n    private String getClientId() {\n        return String.join(\"@\", NetworkUtil.getLocalAddress(), \"compactionIns\", \"compactionUnit\");\n    }\n\n    private boolean prepare(String masterAddr, String topic, String groupName, long subVersion)\n            throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException {\n        HeartbeatData heartbeatData = new HeartbeatData();\n\n        heartbeatData.setClientID(getClientId());\n\n        ConsumerData consumerData = new ConsumerData();\n        consumerData.setGroupName(groupName);\n        consumerData.setConsumeType(ConsumeType.CONSUME_ACTIVELY);\n        consumerData.setMessageModel(MessageModel.CLUSTERING);\n        consumerData.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n//        consumerData.setSubscriptionDataSet();\n        SubscriptionData subscriptionData = new SubscriptionData();\n        subscriptionData.setTopic(topic);\n        subscriptionData.setSubString(SubscriptionData.SUB_ALL);\n        subscriptionData.setSubVersion(subVersion);\n        consumerData.setSubscriptionDataSet(Sets.newHashSet(subscriptionData));\n\n        heartbeatData.getConsumerDataSet().add(consumerData);\n\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.HEART_BEAT, new HeartbeatRequestHeader());\n        request.setLanguage(LanguageCode.JAVA);\n        request.setBody(heartbeatData.encode());\n\n        RemotingCommand response = client.invokeSync(masterAddr, request, 1000 * 30L);\n        if (response != null && response.getCode() == ResponseCode.SUCCESS) {\n            return true;\n        }\n        return false;\n    }\n\n    private boolean pullDone(String masterAddr, String groupName)\n            throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException {\n        UnregisterClientRequestHeader requestHeader = new UnregisterClientRequestHeader();\n        requestHeader.setClientID(getClientId());\n        requestHeader.setProducerGroup(\"\");\n        requestHeader.setConsumerGroup(groupName);\n        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UNREGISTER_CLIENT, requestHeader);\n\n        RemotingCommand response = client.invokeSync(masterAddr, request, 1000 * 30L);\n        if (response != null && response.getCode() == ResponseCode.SUCCESS) {\n            return true;\n        }\n        return false;\n    }\n\n    private boolean stopPull(long currPullOffset, long endOffset) {\n        return currPullOffset >= endOffset && endOffset != -1;\n    }\n\n    public void pullMessageFromMaster(String topic, int queueId, long endOffset, String masterAddr,\n                                      BiFunction<Long, RemotingCommand, Boolean> responseHandler) throws Exception {\n        long currentPullOffset = 0;\n\n        try {\n            long subVersion = System.currentTimeMillis();\n            String groupName = getConsumerGroup(topic, queueId);\n            if (!prepare(masterAddr, topic, groupName, subVersion)) {\n                log.error(\"{}:{} prepare to {} pull message failed\", topic, queueId, masterAddr);\n                throw new RemotingCommandException(topic + \":\" + queueId + \" prepare to \" + masterAddr + \" pull message failed\");\n            }\n\n            boolean noNewMsg = false;\n            boolean keepPull = true;\n//            PullMessageRequestHeader requestHeader = createPullMessageRequest(topic, queueId, subVersion, currentPullOffset);\n            while (!stopPull(currentPullOffset, endOffset)) {\n//                requestHeader.setQueueOffset(currentPullOffset);\n                PullMessageRequestHeader requestHeader = createPullMessageRequest(topic, queueId, currentPullOffset, subVersion);\n\n                RemotingCommand\n                        request = RemotingCommand.createRequestCommand(RequestCode.LITE_PULL_MESSAGE, requestHeader);\n                RemotingCommand response = client.invokeSync(masterAddr, request, 1000 * 30L);\n\n                PullMessageResponseHeader responseHeader =\n                        (PullMessageResponseHeader) response.decodeCommandCustomHeader(PullMessageResponseHeader.class);\n                if (responseHeader == null) {\n                    log.error(\"{}:{} pull message responseHeader is null\", topic, queueId);\n                    throw new RemotingCommandException(topic + \":\" + queueId + \" pull message responseHeader is null\");\n                }\n\n                switch (response.getCode()) {\n                    case ResponseCode.SUCCESS:\n                        long curOffset = responseHeader.getNextBeginOffset() - 1;\n                        keepPull = responseHandler.apply(curOffset, response);\n                        currentPullOffset = responseHeader.getNextBeginOffset();\n                        break;\n                    case ResponseCode.PULL_NOT_FOUND:       // NO_NEW_MSG, need break loop\n                        log.info(\"PULL_NOT_FOUND, topic:{}, queueId:{}, pullOffset:{},\",\n                                topic, queueId, currentPullOffset);\n                        noNewMsg = true;\n                        break;\n                    case ResponseCode.PULL_RETRY_IMMEDIATELY:\n                        log.info(\"PULL_RETRY_IMMEDIATE, topic:{}, queueId:{}, pullOffset:{},\",\n                                topic, queueId, currentPullOffset);\n                        break;\n                    case ResponseCode.PULL_OFFSET_MOVED:\n                        log.info(\"PULL_OFFSET_MOVED, topic:{}, queueId:{}, pullOffset:{},\",\n                                topic, queueId, currentPullOffset);\n                        break;\n                    default:\n                        log.warn(\"Pull Message error, response code: {}, remark: {}\",\n                                response.getCode(), response.getRemark());\n                }\n\n                if (noNewMsg || !keepPull) {\n                    break;\n                }\n            }\n            pullDone(masterAddr, groupName);\n        } finally {\n            if (client != null) {\n                client.closeChannels(Lists.newArrayList(masterAddr));\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.lock;\n\nimport org.apache.rocketmq.store.PutMessageLock;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\n\npublic interface AdaptiveBackOffSpinLock extends PutMessageLock {\n    /**\n     * Configuration update\n     * @param messageStoreConfig\n     */\n    default void update(MessageStoreConfig messageStoreConfig) {\n    }\n\n    /**\n     * Locking mechanism switching\n     */\n    default void swap() {\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.lock;\n\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\n\nimport java.time.LocalTime;\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.concurrent.atomic.AtomicInteger;\n\npublic class AdaptiveBackOffSpinLockImpl implements AdaptiveBackOffSpinLock {\n    private AdaptiveBackOffSpinLock adaptiveLock;\n    //state\n    private AtomicBoolean state = new AtomicBoolean(true);\n\n    // Used to determine the switchover between a mutex lock and a spin lock\n    private final static float SWAP_SPIN_LOCK_RATIO = 0.8f;\n\n    // It is used to adjust the spin number K of the escape spin lock\n    // When (retreat number / TPS) <= (1 / BASE_SWAP_ADAPTIVE_RATIO * SPIN_LOCK_ADAPTIVE_RATIO), K is decreased\n    private final static int SPIN_LOCK_ADAPTIVE_RATIO = 4;\n\n    // It is used to adjust the spin number K of the escape spin lock\n    // When (retreat number / TPS) >= (1 / BASE_SWAP_ADAPTIVE_RATIO), K is increased\n    private final static int BASE_SWAP_LOCK_RATIO = 320;\n\n    private final static String BACK_OFF_SPIN_LOCK = \"SpinLock\";\n\n    private final static String REENTRANT_LOCK = \"ReentrantLock\";\n\n    private Map<String, AdaptiveBackOffSpinLock> locks;\n\n    private final List<AtomicInteger> tpsTable;\n\n    private final List<Set<Thread>> threadTable;\n\n    private int swapCriticalPoint;\n\n    private AtomicInteger currentThreadNum = new AtomicInteger(0);\n\n    private AtomicBoolean isOpen = new AtomicBoolean(true);\n\n    public AdaptiveBackOffSpinLockImpl() {\n        this.locks = new HashMap<>();\n        this.locks.put(REENTRANT_LOCK, new BackOffReentrantLock());\n        this.locks.put(BACK_OFF_SPIN_LOCK, new BackOffSpinLock());\n\n        this.threadTable = new ArrayList<>(2);\n        this.threadTable.add(ConcurrentHashMap.newKeySet());\n        this.threadTable.add(ConcurrentHashMap.newKeySet());\n\n        this.tpsTable = new ArrayList<>(2);\n        this.tpsTable.add(new AtomicInteger(0));\n        this.tpsTable.add(new AtomicInteger(0));\n\n        adaptiveLock = this.locks.get(BACK_OFF_SPIN_LOCK);\n    }\n\n    @Override\n    public void lock() {\n        int slot = LocalTime.now().getSecond() % 2;\n        this.threadTable.get(slot).add(Thread.currentThread());\n        this.tpsTable.get(slot).getAndIncrement();\n        boolean state;\n        do {\n            state = this.state.get();\n        } while (!state);\n\n        currentThreadNum.incrementAndGet();\n        this.adaptiveLock.lock();\n    }\n\n    @Override\n    public void unlock() {\n        this.adaptiveLock.unlock();\n        currentThreadNum.decrementAndGet();\n        if (isOpen.get()) {\n            swap();\n        }\n    }\n\n    @Override\n    public void update(MessageStoreConfig messageStoreConfig) {\n        this.adaptiveLock.update(messageStoreConfig);\n    }\n\n    @Override\n    public void swap() {\n        if (!this.state.get()) {\n            return;\n        }\n        boolean needSwap = false;\n        int slot = 1 - LocalTime.now().getSecond() % 2;\n        int tps = this.tpsTable.get(slot).get() + 1;\n        int threadNum = this.threadTable.get(slot).size();\n        this.tpsTable.get(slot).set(-1);\n        this.threadTable.get(slot).clear();\n        if (tps == 0) {\n            return;\n        }\n\n        if (this.adaptiveLock instanceof BackOffSpinLock) {\n            BackOffSpinLock lock = (BackOffSpinLock) this.adaptiveLock;\n            // Avoid frequent adjustment of K, and make a reasonable range through experiments\n            // reasonable range : (retreat number / TPS) > (1 / BASE_SWAP_ADAPTIVE_RATIO * SPIN_LOCK_ADAPTIVE_RATIO) &&\n            // (retreat number / TPS) < (1 / BASE_SWAP_ADAPTIVE_RATIO)\n            if (lock.getNumberOfRetreat(slot) * BASE_SWAP_LOCK_RATIO >= tps) {\n                if (lock.isAdapt()) {\n                    lock.adapt(true);\n                } else {\n                    // It is used to switch between mutex lock and spin lock\n                    this.swapCriticalPoint = tps * threadNum;\n                    needSwap = true;\n                }\n            } else if (lock.getNumberOfRetreat(slot) * BASE_SWAP_LOCK_RATIO * SPIN_LOCK_ADAPTIVE_RATIO <= tps) {\n                lock.adapt(false);\n            }\n            lock.setNumberOfRetreat(slot, 0);\n        } else {\n            if (tps * threadNum <= this.swapCriticalPoint * SWAP_SPIN_LOCK_RATIO) {\n                needSwap = true;\n            }\n        }\n\n        if (needSwap) {\n            if (this.state.compareAndSet(true, false)) {\n                // Ensures that no threads are in contention locks as well as in critical zones\n                int currentThreadNum;\n                do {\n                    currentThreadNum = this.currentThreadNum.get();\n                } while (currentThreadNum != 0);\n\n                try {\n                    if (this.adaptiveLock instanceof BackOffSpinLock) {\n                        this.adaptiveLock = this.locks.get(REENTRANT_LOCK);\n                    } else {\n                        this.adaptiveLock = this.locks.get(BACK_OFF_SPIN_LOCK);\n                        ((BackOffSpinLock) this.adaptiveLock).adapt(false);\n                    }\n                } catch (Exception e) {\n                    //ignore\n                } finally {\n                    this.state.compareAndSet(false, true);\n                }\n            }\n        }\n    }\n\n    public Collection<AdaptiveBackOffSpinLock> getLocks() {\n        return this.locks.values();\n    }\n\n    public void setLocks(Map<String, AdaptiveBackOffSpinLock> locks) {\n        this.locks = locks;\n    }\n\n    public boolean getState() {\n        return this.state.get();\n    }\n\n    public void setState(boolean state) {\n        this.state.set(state);\n    }\n\n    public AdaptiveBackOffSpinLock getAdaptiveLock() {\n        return adaptiveLock;\n    }\n\n    public List<AtomicInteger> getTpsTable() {\n        return tpsTable;\n    }\n\n    public void setSwapCriticalPoint(int swapCriticalPoint) {\n        this.swapCriticalPoint = swapCriticalPoint;\n    }\n\n    public int getSwapCriticalPoint() {\n        return swapCriticalPoint;\n    }\n\n    public boolean isOpen() {\n        return this.isOpen.get();\n    }\n\n    public void setOpen(boolean open) {\n        this.isOpen.set(open);\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.lock;\n\nimport java.util.concurrent.locks.ReentrantLock;\n\npublic class BackOffReentrantLock implements AdaptiveBackOffSpinLock {\n    private ReentrantLock putMessageNormalLock = new ReentrantLock(); // NonfairSync\n\n    @Override\n    public void lock() {\n        putMessageNormalLock.lock();\n    }\n\n    @Override\n    public void unlock() {\n        putMessageNormalLock.unlock();\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.lock;\n\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\n\nimport java.time.LocalTime;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.concurrent.atomic.AtomicInteger;\n\npublic class BackOffSpinLock implements AdaptiveBackOffSpinLock {\n\n    private AtomicBoolean putMessageSpinLock = new AtomicBoolean(true);\n\n    private int optimalDegree;\n\n    private final static int INITIAL_DEGREE = 1000;\n\n    private final static int MAX_OPTIMAL_DEGREE = 10000;\n\n    private final List<AtomicInteger> numberOfRetreat;\n\n    public BackOffSpinLock() {\n        this.optimalDegree = INITIAL_DEGREE;\n\n        numberOfRetreat = new ArrayList<>(2);\n        numberOfRetreat.add(new AtomicInteger(0));\n        numberOfRetreat.add(new AtomicInteger(0));\n    }\n\n    @Override\n    public void lock() {\n        int spinDegree = this.optimalDegree;\n        while (true) {\n            for (int i = 0; i < spinDegree; i++) {\n                if (this.putMessageSpinLock.compareAndSet(true, false)) {\n                    return;\n                }\n            }\n            numberOfRetreat.get(LocalTime.now().getSecond() % 2).getAndIncrement();\n            try {\n                Thread.sleep(0);\n            } catch (InterruptedException e) {\n                e.printStackTrace();\n            }\n        }\n    }\n\n    @Override\n    public void unlock() {\n        this.putMessageSpinLock.compareAndSet(false, true);\n    }\n\n    @Override\n    public void update(MessageStoreConfig messageStoreConfig) {\n        this.optimalDegree = messageStoreConfig.getSpinLockCollisionRetreatOptimalDegree();\n    }\n\n    public int getOptimalDegree() {\n        return this.optimalDegree;\n    }\n\n    public void setOptimalDegree(int optimalDegree) {\n        this.optimalDegree = optimalDegree;\n    }\n\n    public boolean isAdapt() {\n        return optimalDegree < MAX_OPTIMAL_DEGREE;\n    }\n\n    public synchronized void adapt(boolean isRise) {\n        if (isRise) {\n            if (optimalDegree * 2 <= MAX_OPTIMAL_DEGREE) {\n                optimalDegree *= 2;\n            } else {\n                if (optimalDegree + INITIAL_DEGREE <= MAX_OPTIMAL_DEGREE) {\n                    optimalDegree += INITIAL_DEGREE;\n                }\n            }\n        } else {\n            if (optimalDegree >= 2 * INITIAL_DEGREE) {\n                optimalDegree -= INITIAL_DEGREE;\n            }\n        }\n    }\n\n    public int getNumberOfRetreat(int pos) {\n        return numberOfRetreat.get(pos).get();\n    }\n\n    public void setNumberOfRetreat(int pos, int size) {\n        this.numberOfRetreat.get(pos).set(size);\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/logfile/AbstractMappedFile.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.logfile;\n\nimport org.apache.rocketmq.store.ReferenceResource;\n\npublic abstract class AbstractMappedFile extends ReferenceResource implements MappedFile {\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/logfile/DefaultMappedFile.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.logfile;\n\nimport com.sun.jna.NativeLong;\nimport com.sun.jna.Pointer;\nimport java.io.File;\nimport java.io.FileNotFoundException;\nimport java.io.IOException;\nimport java.io.RandomAccessFile;\nimport java.lang.reflect.Field;\nimport java.lang.reflect.Method;\nimport java.nio.ByteBuffer;\nimport java.nio.MappedByteBuffer;\nimport java.nio.channels.FileChannel;\nimport java.nio.channels.FileChannel.MapMode;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.nio.file.StandardCopyOption;\nimport java.util.Iterator;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicIntegerFieldUpdater;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.function.Consumer;\n\nimport io.netty.util.internal.PlatformDependent;\nimport org.apache.commons.lang3.SystemUtils;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBatch;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.AppendMessageCallback;\nimport org.apache.rocketmq.store.AppendMessageResult;\nimport org.apache.rocketmq.store.AppendMessageStatus;\nimport org.apache.rocketmq.store.CompactionAppendMsgCallback;\nimport org.apache.rocketmq.store.PutMessageContext;\nimport org.apache.rocketmq.store.RunningFlags;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.TransientStorePool;\nimport org.apache.rocketmq.store.config.FlushDiskType;\nimport org.apache.rocketmq.store.util.LibC;\nimport sun.misc.Unsafe;\n\n\npublic class DefaultMappedFile extends AbstractMappedFile {\n    public static final int OS_PAGE_SIZE = 1024 * 4;\n    public static final Unsafe UNSAFE = getUnsafe();\n    private static final Method IS_LOADED_METHOD;\n    public static final int UNSAFE_PAGE_SIZE = UNSAFE == null ? OS_PAGE_SIZE : UNSAFE.pageSize();\n\n    protected static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n\n    protected static final AtomicLong TOTAL_MAPPED_VIRTUAL_MEMORY = new AtomicLong(0);\n\n    protected static final AtomicInteger TOTAL_MAPPED_FILES = new AtomicInteger(0);\n\n    protected static final AtomicIntegerFieldUpdater<DefaultMappedFile> WROTE_POSITION_UPDATER;\n    protected static final AtomicIntegerFieldUpdater<DefaultMappedFile> COMMITTED_POSITION_UPDATER;\n    protected static final AtomicIntegerFieldUpdater<DefaultMappedFile> FLUSHED_POSITION_UPDATER;\n\n    protected volatile int wrotePosition;\n    protected volatile int committedPosition;\n    protected volatile int flushedPosition;\n    protected int fileSize;\n    protected FileChannel fileChannel;\n\n    /**\n     * Message will put to here first, and then reput to FileChannel if writeBuffer is not null.\n     */\n    protected ByteBuffer writeBuffer = null;\n    protected TransientStorePool transientStorePool = null;\n    /**\n     * Configuration flag to use RandomAccessFile instead of MappedByteBuffer for writing\n     */\n    protected boolean writeWithoutMmap = false;\n    protected String fileName;\n    protected long fileFromOffset;\n    protected File file;\n    protected MappedByteBuffer mappedByteBuffer;\n    protected volatile long storeTimestamp = 0;\n    protected boolean firstCreateInQueue = false;\n    private long lastFlushTime = -1L;\n\n    protected MappedByteBuffer mappedByteBufferWaitToClean = null;\n    protected long swapMapTime = 0L;\n    protected long mappedByteBufferAccessCountSinceLastSwap = 0L;\n\n    /**\n     * If this mapped file belongs to consume queue, this field stores store-timestamp of first message referenced by\n     * this logical queue.\n     */\n    private long startTimestamp = -1;\n\n    /**\n     * If this mapped file belongs to consume queue, this field stores store-timestamp of last message referenced by\n     * this logical queue.\n     */\n    private long stopTimestamp = -1;\n\n\n\n    protected RunningFlags runningFlags;\n\n\n\n    static {\n        WROTE_POSITION_UPDATER = AtomicIntegerFieldUpdater.newUpdater(DefaultMappedFile.class, \"wrotePosition\");\n        COMMITTED_POSITION_UPDATER = AtomicIntegerFieldUpdater.newUpdater(DefaultMappedFile.class, \"committedPosition\");\n        FLUSHED_POSITION_UPDATER = AtomicIntegerFieldUpdater.newUpdater(DefaultMappedFile.class, \"flushedPosition\");\n\n        Method isLoaded0method = null;\n        // On the windows platform and openjdk 11 method isLoaded0 always returns false.\n        // see https://github.com/AdoptOpenJDK/openjdk-jdk11/blob/19fb8f93c59dfd791f62d41f332db9e306bc1422/src/java.base/windows/native/libnio/MappedByteBuffer.c#L34\n        if (!SystemUtils.IS_OS_WINDOWS) {\n            try {\n                isLoaded0method = MappedByteBuffer.class.getDeclaredMethod(\"isLoaded0\", long.class, long.class, int.class);\n                isLoaded0method.setAccessible(true);\n            } catch (NoSuchMethodException ignore) {\n            }\n        }\n        IS_LOADED_METHOD = isLoaded0method;\n    }\n\n\n\n    public DefaultMappedFile() {\n    }\n\n    public DefaultMappedFile(final String fileName, final int fileSize) throws IOException {\n        this(fileName, fileSize, null);\n    }\n\n    public DefaultMappedFile(final String fileName, final int fileSize, boolean writeWithoutMmap) throws IOException {\n        this(fileName, fileSize, null, null, writeWithoutMmap);\n    }\n\n    public DefaultMappedFile(final String fileName, final int fileSize, RunningFlags runningFlags) throws IOException {\n        this(fileName, fileSize, runningFlags, null, false);\n    }\n\n    public DefaultMappedFile(final String fileName, final int fileSize, final RunningFlags runningFlags,\n        final TransientStorePool transientStorePool) throws IOException {\n        this(fileName, fileSize, runningFlags, transientStorePool, false);\n    }\n\n    public DefaultMappedFile(final String fileName, final int fileSize, final RunningFlags runningFlags,\n        final boolean writeWithoutMmap) throws IOException {\n        this(fileName, fileSize, runningFlags, null, writeWithoutMmap);\n    }\n\n    public DefaultMappedFile(final String fileName, final int fileSize,\n        final TransientStorePool transientStorePool, final boolean writeWithoutMmap) throws IOException {\n        this(fileName, fileSize, null, transientStorePool, writeWithoutMmap);\n    }\n\n    public DefaultMappedFile(final String fileName, final int fileSize, final RunningFlags runningFlags,\n        final TransientStorePool transientStorePool, final boolean writeWithoutMmap) throws IOException {\n        this.writeWithoutMmap = writeWithoutMmap;\n        init(fileName, fileSize, runningFlags, transientStorePool);\n    }\n\n    public static int getTotalMappedFiles() {\n        return TOTAL_MAPPED_FILES.get();\n    }\n\n    public static long getTotalMappedVirtualMemory() {\n        return TOTAL_MAPPED_VIRTUAL_MEMORY.get();\n    }\n\n    @Override\n    public void init(final String fileName, final int fileSize, final RunningFlags runningFlags,\n        final TransientStorePool transientStorePool) throws IOException {\n        init(fileName, fileSize, runningFlags);\n        if (transientStorePool != null) {\n            this.writeBuffer = transientStorePool.borrowBuffer();\n            this.transientStorePool = transientStorePool;\n        }\n    }\n\n    private void init(final String fileName, final int fileSize, final RunningFlags runningFlags) throws IOException {\n        this.fileName = fileName;\n        this.fileSize = fileSize;\n        this.file = new File(fileName);\n        this.fileFromOffset = Long.parseLong(this.file.getName());\n        this.runningFlags = runningFlags;\n        boolean ok = false;\n\n        UtilAll.ensureDirOK(this.file.getParent());\n\n        try {\n            this.fileChannel = new RandomAccessFile(this.file, \"rw\").getChannel();\n\n            if (writeWithoutMmap) {\n                // Still create MappedByteBuffer for reading operations\n                this.mappedByteBuffer = this.fileChannel.map(MapMode.READ_ONLY, 0, fileSize);\n            } else {\n                // Use MappedByteBuffer for both reading and writing (default behavior)\n                this.mappedByteBuffer = this.fileChannel.map(MapMode.READ_WRITE, 0, fileSize);\n            }\n\n            TOTAL_MAPPED_VIRTUAL_MEMORY.addAndGet(fileSize);\n            TOTAL_MAPPED_FILES.incrementAndGet();\n            ok = true;\n        } catch (FileNotFoundException e) {\n            log.error(\"Failed to create file \" + this.fileName, e);\n            throw e;\n        } catch (IOException e) {\n            log.error(\"Failed to map file \" + this.fileName, e);\n            throw e;\n        } finally {\n            if (!ok && this.fileChannel != null) {\n                this.fileChannel.close();\n            }\n        }\n    }\n\n    @Override\n    public boolean renameTo(String fileName) {\n        File newFile = new File(fileName);\n        boolean rename = file.renameTo(newFile);\n        if (rename) {\n            this.fileName = fileName;\n            this.file = newFile;\n        }\n        return rename;\n    }\n\n    @Override\n    public long getLastModifiedTimestamp() {\n        return this.file.lastModified();\n    }\n\n    public boolean getData(int pos, int size, ByteBuffer byteBuffer) {\n        if (byteBuffer.remaining() < size) {\n            return false;\n        }\n\n        int readPosition = getReadPosition();\n        if ((pos + size) <= readPosition) {\n\n            if (this.hold()) {\n                try {\n                    int readNum = fileChannel.read(byteBuffer, pos);\n                    return size == readNum;\n                } catch (Throwable t) {\n                    log.warn(\"Get data failed pos:{} size:{} fileFromOffset:{}\", pos, size, this.fileFromOffset);\n                    return false;\n                } finally {\n                    this.release();\n                }\n            } else {\n                log.debug(\"matched, but hold failed, request pos: \" + pos + \", fileFromOffset: \"\n                    + this.fileFromOffset);\n            }\n        } else {\n            log.warn(\"selectMappedBuffer request pos invalid, request pos: \" + pos + \", size: \" + size\n                + \", fileFromOffset: \" + this.fileFromOffset);\n        }\n\n        return false;\n    }\n\n    @Override\n    public int getFileSize() {\n        return fileSize;\n    }\n\n    @Override\n    public FileChannel getFileChannel() {\n        return fileChannel;\n    }\n\n    public AppendMessageResult appendMessage(final ByteBuffer byteBufferMsg, final CompactionAppendMsgCallback cb) {\n        assert byteBufferMsg != null;\n        assert cb != null;\n\n        int currentPos = WROTE_POSITION_UPDATER.get(this);\n        long fileFromOffset = this.getFileFromOffset();\n\n        if (currentPos < this.fileSize) {\n            SharedByteBufferManager.SharedByteBuffer sharedByteBuffer = null;\n            ByteBuffer byteBuffer;\n            if (writeWithoutMmap) {\n                sharedByteBuffer = SharedByteBufferManager.getInstance().borrowSharedByteBuffer();\n                byteBuffer = sharedByteBuffer.acquire();\n                byteBuffer.position(0).limit(byteBuffer.capacity());\n                fileFromOffset += currentPos;\n            } else {\n                byteBuffer = appendMessageBuffer().slice();\n                byteBuffer.position(currentPos);\n            }\n\n            try {\n                AppendMessageResult result = cb.doAppend(byteBuffer, fileFromOffset, this.fileSize - currentPos, byteBufferMsg);\n\n                if (sharedByteBuffer != null) {\n                    try {\n                        this.fileChannel.position(currentPos);\n                        byteBuffer.position(0).limit(result.getWroteBytes());\n                        this.fileChannel.write(byteBuffer);\n                    } catch (Throwable t) {\n                        log.error(\"Failed to write to mappedFile {}\", this.fileName, t);\n                        return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);\n                    }\n                }\n\n                WROTE_POSITION_UPDATER.addAndGet(this, result.getWroteBytes());\n                this.storeTimestamp = result.getStoreTimestamp();\n                return result;\n            } finally {\n                if (sharedByteBuffer != null) {\n                    sharedByteBuffer.release();\n                }\n            }\n        }\n        log.error(\"MappedFile.appendMessage return null, wrotePosition: {} fileSize: {}\", currentPos, this.fileSize);\n        return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);\n    }\n\n    @Override\n    public AppendMessageResult appendMessage(final MessageExtBrokerInner msg, final AppendMessageCallback cb,\n        PutMessageContext putMessageContext) {\n        return appendMessagesInner(msg, cb, putMessageContext);\n    }\n\n    @Override\n    public AppendMessageResult appendMessages(final MessageExtBatch messageExtBatch, final AppendMessageCallback cb,\n        PutMessageContext putMessageContext) {\n        return appendMessagesInner(messageExtBatch, cb, putMessageContext);\n    }\n\n    public AppendMessageResult appendMessagesInner(final MessageExt messageExt, final AppendMessageCallback cb,\n        PutMessageContext putMessageContext) {\n        assert messageExt != null;\n        assert cb != null;\n\n        int currentPos = WROTE_POSITION_UPDATER.get(this);\n        long fileFromOffset = this.getFileFromOffset();\n\n        if (currentPos < this.fileSize) {\n            SharedByteBufferManager.SharedByteBuffer sharedByteBuffer = null;\n            ByteBuffer byteBuffer;\n            if (writeWithoutMmap) {\n                sharedByteBuffer = SharedByteBufferManager.getInstance().borrowSharedByteBuffer();\n                byteBuffer = sharedByteBuffer.acquire();\n                byteBuffer.position(0).limit(byteBuffer.capacity());\n                fileFromOffset += currentPos;\n            } else {\n                byteBuffer = appendMessageBuffer().slice();\n                byteBuffer.position(currentPos);\n            }\n\n            AppendMessageResult result;\n            try {\n                if (messageExt instanceof MessageExtBatch && !((MessageExtBatch) messageExt).isInnerBatch()) {\n                    // traditional batch message\n                    result = cb.doAppend(fileFromOffset, byteBuffer, this.fileSize - currentPos,\n                        (MessageExtBatch) messageExt, putMessageContext);\n                } else if (messageExt instanceof MessageExtBrokerInner) {\n                    // traditional single message or newly introduced inner-batch message\n                    result = cb.doAppend(fileFromOffset, byteBuffer, this.fileSize - currentPos,\n                        (MessageExtBrokerInner) messageExt, putMessageContext);\n                } else {\n                    return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);\n                }\n\n                if (sharedByteBuffer != null) {\n                    try {\n                        int msgLen = result.getWroteBytes();\n                        int endpos = currentPos + msgLen;\n                        // alignment end position\n                        int extraAppendSize = UNSAFE_PAGE_SIZE - endpos % UNSAFE_PAGE_SIZE;\n                        if (extraAppendSize == UNSAFE_PAGE_SIZE) {\n                            extraAppendSize = 0;\n                        }\n                        int actualAppendSize = msgLen + extraAppendSize;\n\n                        this.fileChannel.position(currentPos);\n                        // commitlog can contain dirty data at the end.\n                        if (byteBuffer.capacity() >= actualAppendSize) {\n                            byteBuffer.position(0).limit(actualAppendSize);\n                        } else {\n                            byteBuffer.position(0).limit(msgLen);\n                        }\n                        this.fileChannel.write(byteBuffer);\n                    } catch (Throwable t) {\n                        log.error(\"Failed to write to mappedFile {}\", this.fileName, t);\n                        return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);\n                    }\n                }\n            } finally {\n                if (sharedByteBuffer != null) {\n                    sharedByteBuffer.release();\n                }\n            }\n\n            WROTE_POSITION_UPDATER.addAndGet(this, result.getWroteBytes());\n            this.storeTimestamp = result.getStoreTimestamp();\n            return result;\n        }\n        log.error(\"MappedFile.appendMessage return null, wrotePosition: {} fileSize: {}\", currentPos, this.fileSize);\n        return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);\n    }\n    protected ByteBuffer appendMessageBuffer() {\n        this.mappedByteBufferAccessCountSinceLastSwap++;\n        return writeBuffer != null ? writeBuffer : this.mappedByteBuffer;\n    }\n\n    @Override\n    public long getFileFromOffset() {\n        return this.fileFromOffset;\n    }\n\n    @Override\n    public boolean appendMessage(final byte[] data) {\n        return appendMessage(data, 0, data.length);\n    }\n\n    @Override\n    public boolean appendMessage(ByteBuffer data) {\n        int currentPos = WROTE_POSITION_UPDATER.get(this);\n        int remaining = data.remaining();\n\n        if ((currentPos + remaining) <= this.fileSize) {\n            try {\n                if (writeWithoutMmap) {\n                    // Use FileChannel for writing\n                    this.fileChannel.position(currentPos);\n                    byte[] buffer = new byte[remaining];\n                    data.get(buffer);\n                    ByteBuffer writeBuffer = ByteBuffer.wrap(buffer);\n                    this.fileChannel.write(writeBuffer);\n                } else {\n                    // Use FileChannel for writing (default behavior)\n                    this.fileChannel.position(currentPos);\n                    while (data.hasRemaining()) {\n                        this.fileChannel.write(data);\n                    }\n                }\n                WROTE_POSITION_UPDATER.addAndGet(this, remaining);\n                return true;\n            } catch (Throwable e) {\n                log.error(\"Error occurred when append message to mappedFile.\", e);\n                return false;\n            }\n        }\n        return false;\n    }\n\n    /**\n     * Content of data from offset to offset + length will be written to file.\n     *\n     * @param offset The offset of the subarray to be used.\n     * @param length The length of the subarray to be used.\n     */\n    @Override\n    public boolean appendMessage(final byte[] data, final int offset, final int length) {\n        int currentPos = WROTE_POSITION_UPDATER.get(this);\n\n        if ((currentPos + length) <= this.fileSize) {\n            try {\n                if (writeWithoutMmap) {\n                    // Use FileChannel for writing\n                    this.fileChannel.position(currentPos);\n                    ByteBuffer writeBuffer = ByteBuffer.wrap(data, offset, length);\n                    this.fileChannel.write(writeBuffer);\n                } else {\n                    // Use MappedByteBuffer for writing (default behavior)\n                    ByteBuffer buf = this.mappedByteBuffer.slice();\n                    buf.position(currentPos);\n                    buf.put(data, offset, length);\n                }\n                WROTE_POSITION_UPDATER.addAndGet(this, length);\n                return true;\n            } catch (Throwable e) {\n                log.error(\"Error occurred when append message to mappedFile.\", e);\n                return false;\n            }\n        }\n\n        return false;\n    }\n\n    @Override\n    public boolean appendMessageUsingFileChannel(byte[] data) {\n        int currentPos = WROTE_POSITION_UPDATER.get(this);\n\n        if ((currentPos + data.length) <= this.fileSize) {\n            try {\n                this.fileChannel.position(currentPos);\n                this.fileChannel.write(ByteBuffer.wrap(data, 0, data.length));\n                WROTE_POSITION_UPDATER.addAndGet(this, data.length);\n                return true;\n            } catch (Throwable e) {\n                log.error(\"Error occurred when append message to mappedFile.\", e);\n                return false;\n            }\n        }\n\n        return false;\n    }\n\n    /**\n     * @return The current flushed position\n     */\n    @Override\n    public int flush(final int flushLeastPages) {\n        if (!isWriteable()) {\n            return this.getFlushedPosition();\n        }\n        if (this.isAbleToFlush(flushLeastPages)) {\n            if (this.hold()) {\n                int value = getReadPosition();\n\n                try {\n                    this.mappedByteBufferAccessCountSinceLastSwap++;\n\n                    //We only append data to fileChannel or mappedByteBuffer, never both.\n                    if (writeWithoutMmap || writeBuffer != null || this.fileChannel.position() != 0) {\n                        this.fileChannel.force(false);\n                    } else {\n                        this.mappedByteBuffer.force();\n                    }\n\n                    this.lastFlushTime = System.currentTimeMillis();\n                    FLUSHED_POSITION_UPDATER.set(this, value);\n                } catch (Throwable e) {\n                    if (e instanceof IOException) {\n                        getAndMakeNotWriteable();\n                    }\n                    log.error(\"Error occurred when force data to disk.\", e);\n                }\n                this.release();\n            } else {\n                log.warn(\"in flush, hold failed, flush offset = \" + FLUSHED_POSITION_UPDATER.get(this));\n                FLUSHED_POSITION_UPDATER.set(this, getReadPosition());\n            }\n        }\n        return this.getFlushedPosition();\n    }\n\n    @Override\n    public int commit(final int commitLeastPages) {\n        if (writeBuffer == null) {\n            //no need to commit data to file channel, so just regard wrotePosition as committedPosition.\n            return WROTE_POSITION_UPDATER.get(this);\n        }\n\n        //no need to commit data to file channel, so just set committedPosition to wrotePosition.\n        if (transientStorePool != null && !transientStorePool.isRealCommit()) {\n            COMMITTED_POSITION_UPDATER.set(this, WROTE_POSITION_UPDATER.get(this));\n        } else if (this.isAbleToCommit(commitLeastPages)) {\n            if (this.hold()) {\n                commit0();\n                this.release();\n            } else {\n                log.warn(\"in commit, hold failed, commit offset = \" + COMMITTED_POSITION_UPDATER.get(this));\n            }\n        }\n\n        // All dirty data has been committed to FileChannel.\n        if (writeBuffer != null && this.transientStorePool != null && this.fileSize == COMMITTED_POSITION_UPDATER.get(this)) {\n            this.transientStorePool.returnBuffer(writeBuffer);\n            this.writeBuffer = null;\n        }\n\n        return COMMITTED_POSITION_UPDATER.get(this);\n    }\n\n    protected void commit0() {\n        int writePos = WROTE_POSITION_UPDATER.get(this);\n        int lastCommittedPosition = COMMITTED_POSITION_UPDATER.get(this);\n\n        if (writePos - lastCommittedPosition > 0) {\n            try {\n                ByteBuffer byteBuffer = writeBuffer.slice();\n                byteBuffer.position(lastCommittedPosition);\n                byteBuffer.limit(writePos);\n                this.fileChannel.position(lastCommittedPosition);\n                this.fileChannel.write(byteBuffer);\n                COMMITTED_POSITION_UPDATER.set(this, writePos);\n            } catch (Throwable e) {\n                log.error(\"Error occurred when commit data to FileChannel.\", e);\n            }\n        }\n    }\n\n    public boolean getAndMakeNotWriteable() {\n        if (runningFlags == null) {\n            return false;\n        }\n        return runningFlags.getAndMakeStoreNotWriteable();\n    }\n\n    public boolean isWriteable() {\n        if (runningFlags == null) {\n            return true;\n        }\n        return runningFlags.isWriteable();\n    }\n\n    private boolean isAbleToFlush(final int flushLeastPages) {\n        int flush = FLUSHED_POSITION_UPDATER.get(this);\n        int write = getReadPosition();\n\n        if (this.isFull()) {\n            return true;\n        }\n\n        if (flushLeastPages > 0) {\n            return ((write / OS_PAGE_SIZE) - (flush / OS_PAGE_SIZE)) >= flushLeastPages;\n        }\n\n        return write > flush;\n    }\n\n    protected boolean isAbleToCommit(final int commitLeastPages) {\n        int commit = COMMITTED_POSITION_UPDATER.get(this);\n        int write = WROTE_POSITION_UPDATER.get(this);\n\n        if (this.isFull()) {\n            return true;\n        }\n\n        if (commitLeastPages > 0) {\n            return ((write / OS_PAGE_SIZE) - (commit / OS_PAGE_SIZE)) >= commitLeastPages;\n        }\n\n        return write > commit;\n    }\n\n    @Override\n    public int getFlushedPosition() {\n        return FLUSHED_POSITION_UPDATER.get(this);\n    }\n\n    @Override\n    public void setFlushedPosition(int pos) {\n        FLUSHED_POSITION_UPDATER.set(this, pos);\n    }\n\n    @Override\n    public boolean isFull() {\n        return this.fileSize == WROTE_POSITION_UPDATER.get(this);\n    }\n\n    @Override\n    public SelectMappedBufferResult selectMappedBuffer(int pos, int size) {\n        int readPosition = getReadPosition();\n        if ((pos + size) <= readPosition) {\n            if (this.hold()) {\n                this.mappedByteBufferAccessCountSinceLastSwap++;\n\n                ByteBuffer byteBuffer = this.mappedByteBuffer.slice();\n                byteBuffer.position(pos);\n                ByteBuffer byteBufferNew = byteBuffer.slice();\n                byteBufferNew.limit(size);\n                return new SelectMappedBufferResult(this.fileFromOffset + pos, byteBufferNew, size, this);\n            } else {\n                log.warn(\"matched, but hold failed, request pos: \" + pos + \", fileFromOffset: \"\n                    + this.fileFromOffset);\n            }\n        } else {\n            log.warn(\"selectMappedBuffer request pos invalid, request pos: \" + pos + \", size: \" + size\n                + \", fileFromOffset: \" + this.fileFromOffset);\n        }\n\n        return null;\n    }\n\n    @Override\n    public SelectMappedBufferResult selectMappedBuffer(int pos) {\n        int readPosition = getReadPosition();\n        if (pos < readPosition && pos >= 0) {\n            if (this.hold()) {\n                this.mappedByteBufferAccessCountSinceLastSwap++;\n                ByteBuffer byteBuffer = this.mappedByteBuffer.slice();\n                byteBuffer.position(pos);\n                int size = readPosition - pos;\n                ByteBuffer byteBufferNew = byteBuffer.slice();\n                byteBufferNew.limit(size);\n                return new SelectMappedBufferResult(this.fileFromOffset + pos, byteBufferNew, size, this);\n            }\n        }\n\n        return null;\n    }\n\n    @Override\n    public boolean cleanup(final long currentRef) {\n        if (this.isAvailable()) {\n            log.error(\"this file[REF:\" + currentRef + \"] \" + this.fileName\n                + \" have not shutdown, stop unmapping.\");\n            return false;\n        }\n\n        if (this.isCleanupOver()) {\n            log.error(\"this file[REF:\" + currentRef + \"] \" + this.fileName\n                + \" have cleanup, do not do it again.\");\n            return true;\n        }\n\n        cleanResources();\n\n        log.info(\"unmap file[REF:\" + currentRef + \"] \" + this.fileName + \" OK\");\n\n        return true;\n    }\n\n    @Override\n    public void cleanResources() {\n        UtilAll.cleanBuffer(this.mappedByteBuffer);\n        UtilAll.cleanBuffer(this.mappedByteBufferWaitToClean);\n        this.mappedByteBufferWaitToClean = null;\n        TOTAL_MAPPED_VIRTUAL_MEMORY.addAndGet(this.fileSize * (-1));\n        TOTAL_MAPPED_FILES.decrementAndGet();\n        try {\n            fileChannel.close();\n        } catch (Throwable e) {\n            log.warn(\"close file channel {\" + this.fileName + \"} failed when cleanup\", e);\n        }\n    }\n\n    @Override\n    public boolean destroy(final long intervalForcibly) {\n        this.shutdown(intervalForcibly);\n\n        if (this.isCleanupOver()) {\n            try {\n                long lastModified = getLastModifiedTimestamp();\n                this.fileChannel.close();\n                log.info(\"close file channel \" + this.fileName + \" OK\");\n\n                long beginTime = System.currentTimeMillis();\n                boolean result = this.file.delete();\n                log.info(\"delete file[REF:\" + this.getRefCount() + \"] \" + this.fileName\n                    + (result ? \" OK, \" : \" Failed, \") + \"W:\" + this.getWrotePosition() + \" M:\"\n                    + this.getFlushedPosition() + \", \"\n                    + UtilAll.computeElapsedTimeMilliseconds(beginTime)\n                    + \",\" + (System.currentTimeMillis() - lastModified));\n            } catch (Exception e) {\n                log.warn(\"close file channel \" + this.fileName + \" Failed. \", e);\n            }\n\n            return true;\n        } else {\n            log.warn(\"destroy mapped file[REF:\" + this.getRefCount() + \"] \" + this.fileName\n                + \" Failed. cleanupOver: \" + this.cleanupOver);\n        }\n\n        return false;\n    }\n\n    @Override\n    public int getWrotePosition() {\n        return WROTE_POSITION_UPDATER.get(this);\n    }\n\n    @Override\n    public void setWrotePosition(int pos) {\n        WROTE_POSITION_UPDATER.set(this, pos);\n    }\n\n    /**\n     * @return The max position which have valid data\n     */\n    @Override\n    public int getReadPosition() {\n        return transientStorePool == null || !transientStorePool.isRealCommit() ? WROTE_POSITION_UPDATER.get(this) : COMMITTED_POSITION_UPDATER.get(this);\n    }\n\n    @Override\n    public void setCommittedPosition(int pos) {\n        COMMITTED_POSITION_UPDATER.set(this, pos);\n    }\n\n    @Override\n    public void warmMappedFile(FlushDiskType type, int pages) {\n        this.mappedByteBufferAccessCountSinceLastSwap++;\n\n        long beginTime = System.currentTimeMillis();\n        ByteBuffer byteBuffer = this.mappedByteBuffer.slice();\n        long flush = 0;\n        // long time = System.currentTimeMillis();\n        for (long i = 0, j = 0; i < this.fileSize; i += DefaultMappedFile.OS_PAGE_SIZE, j++) {\n            byteBuffer.put((int) i, (byte) 0);\n            // force flush when flush disk type is sync\n            if (type == FlushDiskType.SYNC_FLUSH) {\n                if ((i / OS_PAGE_SIZE) - (flush / OS_PAGE_SIZE) >= pages) {\n                    flush = i;\n                    mappedByteBuffer.force();\n                }\n            }\n\n            // prevent gc\n            // if (j % 1000 == 0) {\n            //     log.info(\"j={}, costTime={}\", j, System.currentTimeMillis() - time);\n            //     time = System.currentTimeMillis();\n            //     try {\n            //         Thread.sleep(0);\n            //     } catch (InterruptedException e) {\n            //         log.error(\"Interrupted\", e);\n            //     }\n            // }\n        }\n\n        // force flush when prepare load finished\n        if (type == FlushDiskType.SYNC_FLUSH) {\n            log.info(\"mapped file warm-up done, force to disk, mappedFile={}, costTime={}\",\n                this.getFileName(), System.currentTimeMillis() - beginTime);\n            mappedByteBuffer.force();\n        }\n        log.info(\"mapped file warm-up done. mappedFile={}, costTime={}\", this.getFileName(),\n            System.currentTimeMillis() - beginTime);\n\n        this.mlock();\n    }\n\n    @Override\n    public boolean swapMap() {\n        if (getRefCount() == 1 && this.mappedByteBufferWaitToClean == null) {\n\n            if (!hold()) {\n                log.warn(\"in swapMap, hold failed, fileName: \" + this.fileName);\n                return false;\n            }\n            try {\n                this.mappedByteBufferWaitToClean = this.mappedByteBuffer;\n                this.mappedByteBuffer = this.fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, fileSize);\n                this.mappedByteBufferAccessCountSinceLastSwap = 0L;\n                this.swapMapTime = System.currentTimeMillis();\n                log.info(\"swap file \" + this.fileName + \" success.\");\n                return true;\n            } catch (Exception e) {\n                log.error(\"swapMap file \" + this.fileName + \" Failed. \", e);\n            } finally {\n                this.release();\n            }\n        } else {\n            log.info(\"Will not swap file: \" + this.fileName + \", ref=\" + getRefCount());\n        }\n        return false;\n    }\n\n    @Override\n    public void cleanSwapedMap(boolean force) {\n        try {\n            if (this.mappedByteBufferWaitToClean == null) {\n                return;\n            }\n            long minGapTime = 120 * 1000L;\n            long gapTime = System.currentTimeMillis() - this.swapMapTime;\n            if (!force && gapTime < minGapTime) {\n                Thread.sleep(minGapTime - gapTime);\n            }\n            UtilAll.cleanBuffer(this.mappedByteBufferWaitToClean);\n            mappedByteBufferWaitToClean = null;\n            log.info(\"cleanSwapedMap file \" + this.fileName + \" success.\");\n        } catch (Exception e) {\n            log.error(\"cleanSwapedMap file \" + this.fileName + \" Failed. \", e);\n        }\n    }\n\n    @Override\n    public long getRecentSwapMapTime() {\n        return 0;\n    }\n\n    @Override\n    public long getMappedByteBufferAccessCountSinceLastSwap() {\n        return this.mappedByteBufferAccessCountSinceLastSwap;\n    }\n\n    @Override\n    public long getLastFlushTime() {\n        return this.lastFlushTime;\n    }\n\n    @Override\n    public String getFileName() {\n        return fileName;\n    }\n\n    @Override\n    public MappedByteBuffer getMappedByteBuffer() {\n        this.mappedByteBufferAccessCountSinceLastSwap++;\n        return mappedByteBuffer;\n    }\n\n    @Override\n    public ByteBuffer sliceByteBuffer() {\n        this.mappedByteBufferAccessCountSinceLastSwap++;\n        return this.mappedByteBuffer.slice();\n    }\n\n    @Override\n    public long getStoreTimestamp() {\n        return storeTimestamp;\n    }\n\n    @Override\n    public boolean isFirstCreateInQueue() {\n        return firstCreateInQueue;\n    }\n\n    @Override\n    public void setFirstCreateInQueue(boolean firstCreateInQueue) {\n        this.firstCreateInQueue = firstCreateInQueue;\n    }\n\n    @Override\n    public void mlock() {\n        final long beginTime = System.currentTimeMillis();\n        final long address = PlatformDependent.directBufferAddress(this.mappedByteBuffer);\n        Pointer pointer = new Pointer(address);\n        {\n            int ret = LibC.INSTANCE.mlock(pointer, new NativeLong(this.fileSize));\n            log.info(\"mlock {} {} {} ret = {} time consuming = {}\", address, this.fileName, this.fileSize, ret, System.currentTimeMillis() - beginTime);\n        }\n\n        {\n            int ret = LibC.INSTANCE.madvise(pointer, new NativeLong(this.fileSize), LibC.MADV_WILLNEED);\n            log.info(\"madvise {} {} {} ret = {} time consuming = {}\", address, this.fileName, this.fileSize, ret, System.currentTimeMillis() - beginTime);\n        }\n    }\n\n    @Override\n    public void munlock() {\n        final long beginTime = System.currentTimeMillis();\n        final long address = PlatformDependent.directBufferAddress(this.mappedByteBuffer);\n        Pointer pointer = new Pointer(address);\n        int ret = LibC.INSTANCE.munlock(pointer, new NativeLong(this.fileSize));\n        log.info(\"munlock {} {} {} ret = {} time consuming = {}\", address, this.fileName, this.fileSize, ret, System.currentTimeMillis() - beginTime);\n    }\n\n    @Override\n    public File getFile() {\n        return this.file;\n    }\n\n    @Override\n    public void renameToDelete() {\n        //use Files.move\n        if (!fileName.endsWith(\".delete\")) {\n            String newFileName = this.fileName + \".delete\";\n            try {\n                Path newFilePath = Paths.get(newFileName);\n                // https://bugs.openjdk.org/browse/JDK-4724038\n                // https://bugs.java.com/bugdatabase/view_bug.do?bug_id=4715154\n                // Windows can't move the file when mmapped.\n                if (NetworkUtil.isWindowsPlatform() && mappedByteBuffer != null) {\n                    long position = this.fileChannel.position();\n                    UtilAll.cleanBuffer(this.mappedByteBuffer);\n                    this.fileChannel.close();\n                    Files.move(Paths.get(fileName), newFilePath, StandardCopyOption.ATOMIC_MOVE);\n                    try (RandomAccessFile file = new RandomAccessFile(newFileName, \"rw\")) {\n                        this.fileChannel = file.getChannel();\n                        this.fileChannel.position(position);\n                        this.mappedByteBuffer = this.fileChannel.map(MapMode.READ_WRITE, 0, fileSize);\n                    }\n                } else {\n                    Files.move(Paths.get(fileName), newFilePath, StandardCopyOption.ATOMIC_MOVE);\n                }\n                this.fileName = newFileName;\n                this.file = new File(newFileName);\n            } catch (IOException e) {\n                log.error(\"move file {} failed\", fileName, e);\n            }\n        }\n    }\n\n    @Override\n    public void moveToParent() throws IOException {\n        Path currentPath = Paths.get(fileName);\n        String baseName = currentPath.getFileName().toString();\n        Path parentPath = currentPath.getParent().getParent().resolve(baseName);\n        // https://bugs.openjdk.org/browse/JDK-4724038\n        // https://bugs.java.com/bugdatabase/view_bug.do?bug_id=4715154\n        // Windows can't move the file when mmapped.\n        if (NetworkUtil.isWindowsPlatform() && mappedByteBuffer != null) {\n            long position = this.fileChannel.position();\n            UtilAll.cleanBuffer(this.mappedByteBuffer);\n            this.fileChannel.close();\n            Files.move(Paths.get(fileName), parentPath, StandardCopyOption.ATOMIC_MOVE);\n            try (RandomAccessFile file = new RandomAccessFile(parentPath.toFile(), \"rw\")) {\n                this.fileChannel = file.getChannel();\n                this.fileChannel.position(position);\n                this.mappedByteBuffer = this.fileChannel.map(MapMode.READ_WRITE, 0, fileSize);\n            }\n        } else {\n            Files.move(Paths.get(fileName), parentPath, StandardCopyOption.ATOMIC_MOVE);\n        }\n        this.file = parentPath.toFile();\n        this.fileName = parentPath.toString();\n    }\n\n    @Override\n    public String toString() {\n        return this.fileName;\n    }\n\n    public long getStartTimestamp() {\n        return startTimestamp;\n    }\n\n    public void setStartTimestamp(long startTimestamp) {\n        this.startTimestamp = startTimestamp;\n    }\n\n    public long getStopTimestamp() {\n        return stopTimestamp;\n    }\n\n    public void setStopTimestamp(long stopTimestamp) {\n        this.stopTimestamp = stopTimestamp;\n    }\n\n    public Iterator<SelectMappedBufferResult> iterator(int startPos) {\n        return new Itr(startPos);\n    }\n\n    public static Unsafe getUnsafe() {\n        try {\n            Field f = Unsafe.class.getDeclaredField(\"theUnsafe\");\n            f.setAccessible(true);\n            return (Unsafe) f.get(null);\n        } catch (Exception ignore) {\n\n        }\n        return null;\n    }\n\n    public static long mappingAddr(long addr) {\n        long offset = addr % UNSAFE_PAGE_SIZE;\n        offset = (offset >= 0) ? offset : (UNSAFE_PAGE_SIZE + offset);\n        return addr - offset;\n    }\n\n    public static int pageCount(long size) {\n        return (int) (size + (long) UNSAFE_PAGE_SIZE - 1L) / UNSAFE_PAGE_SIZE;\n    }\n\n    @Override\n    public boolean isLoaded(long position, int size) {\n        if (IS_LOADED_METHOD == null) {\n            return true;\n        }\n        try {\n            long addr = PlatformDependent.directBufferAddress(mappedByteBuffer) + position;\n            return (boolean) IS_LOADED_METHOD.invoke(mappedByteBuffer, mappingAddr(addr), size, pageCount(size));\n        } catch (Exception e) {\n            log.info(\"invoke isLoaded0 of file {} error:\", file.getAbsolutePath(), e);\n        }\n        return true;\n    }\n\n    private class Itr implements Iterator<SelectMappedBufferResult> {\n        private int start;\n        private int current;\n        private ByteBuffer buf;\n\n        public Itr(int pos) {\n            this.start = pos;\n            this.current = pos;\n            this.buf = mappedByteBuffer.slice();\n            this.buf.position(start);\n        }\n\n        @Override\n        public boolean hasNext() {\n            return current < getReadPosition();\n        }\n\n        @Override\n        public SelectMappedBufferResult next() {\n            int readPosition = getReadPosition();\n            if (current < readPosition && current >= 0) {\n                if (hold()) {\n                    ByteBuffer byteBuffer = buf.slice();\n                    byteBuffer.position(current);\n                    int size = byteBuffer.getInt(current);\n                    ByteBuffer bufferResult = byteBuffer.slice();\n                    bufferResult.limit(size);\n                    current += size;\n                    return new SelectMappedBufferResult(fileFromOffset + current, bufferResult, size,\n                        DefaultMappedFile.this);\n                }\n            }\n            return null;\n        }\n\n        @Override\n        public void forEachRemaining(Consumer<? super SelectMappedBufferResult> action) {\n            Iterator.super.forEachRemaining(action);\n        }\n\n        @Override\n        public void remove() {\n            throw new UnsupportedOperationException();\n        }\n    }\n\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/logfile/MappedFile.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.logfile;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport java.nio.MappedByteBuffer;\nimport java.nio.channels.FileChannel;\nimport java.util.Iterator;\nimport org.apache.rocketmq.common.message.MessageExtBatch;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.AppendMessageCallback;\nimport org.apache.rocketmq.store.AppendMessageResult;\nimport org.apache.rocketmq.store.CompactionAppendMsgCallback;\nimport org.apache.rocketmq.store.PutMessageContext;\nimport org.apache.rocketmq.store.RunningFlags;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.TransientStorePool;\nimport org.apache.rocketmq.store.config.FlushDiskType;\n\npublic interface MappedFile {\n    /**\n     * Returns the file name of the {@code MappedFile}.\n     *\n     * @return the file name\n     */\n    String getFileName();\n\n    /**\n     * Change the file name of the {@code MappedFile}.\n     *\n     * @param fileName the new file name\n     */\n    boolean renameTo(String fileName);\n\n    /**\n     * Returns the file size of the {@code MappedFile}.\n     *\n     * @return the file size\n     */\n    int getFileSize();\n\n    /**\n     * Returns the {@code FileChannel} behind the {@code MappedFile}.\n     *\n     * @return the file channel\n     */\n    FileChannel getFileChannel();\n\n    /**\n     * Returns true if this {@code MappedFile} is full and no new messages can be added.\n     *\n     * @return true if the file is full\n     */\n    boolean isFull();\n\n    /**\n     * Returns true if this {@code MappedFile} is available.\n     * <p>\n     * The mapped file will be not available if it's shutdown or destroyed.\n     *\n     * @return true if the file is available\n     */\n    boolean isAvailable();\n\n    /**\n     * Appends a message object to the current {@code MappedFile} with a specific call back.\n     *\n     * @param message a message to append\n     * @param messageCallback the specific call back to execute the real append action\n     * @param putMessageContext\n     * @return the append result\n     */\n    AppendMessageResult appendMessage(MessageExtBrokerInner message, AppendMessageCallback messageCallback, PutMessageContext putMessageContext);\n\n    /**\n     * Appends a batch message object to the current {@code MappedFile} with a specific call back.\n     *\n     * @param message a message to append\n     * @param messageCallback the specific call back to execute the real append action\n     * @param putMessageContext\n     * @return the append result\n     */\n    AppendMessageResult appendMessages(MessageExtBatch message, AppendMessageCallback messageCallback, PutMessageContext putMessageContext);\n\n    AppendMessageResult appendMessage(final ByteBuffer byteBufferMsg, final CompactionAppendMsgCallback cb);\n\n    /**\n     * Appends a raw message data represents by a byte array to the current {@code MappedFile}.\n     * Using mappedByteBuffer\n     *\n     * @param data the byte array to append\n     * @return true if success; false otherwise.\n     */\n    boolean appendMessage(byte[] data);\n\n\n    /**\n     * Appends a raw message data represents by a byte array to the current {@code MappedFile}.\n     * Using fileChannel\n     *\n     * @param data the byte array to append\n     * @return true if success; false otherwise.\n     */\n    boolean appendMessageUsingFileChannel(byte[] data);\n\n    /**\n     * Appends a raw message data represents by a byte array to the current {@code MappedFile}.\n     *\n     * @param data the byte buffer to append\n     * @return true if success; false otherwise.\n     */\n    boolean appendMessage(ByteBuffer data);\n\n    /**\n     * Appends a raw message data represents by a byte array to the current {@code MappedFile},\n     * starting at the given offset in the array.\n     *\n     * @param data the byte array to append\n     * @param offset the offset within the array of the first byte to be read\n     * @param length the number of bytes to be read from the given array\n     * @return true if success; false otherwise.\n     */\n    boolean appendMessage(byte[] data, int offset, int length);\n\n    /**\n     * Returns the global offset of the current {code MappedFile}, it's a long value of the file name.\n     *\n     * @return the offset of this file\n     */\n    long getFileFromOffset();\n\n    /**\n     * Flushes the data in cache to disk immediately.\n     *\n     * @param flushLeastPages the least pages to flush\n     * @return the flushed position after the method call\n     */\n    int flush(int flushLeastPages);\n\n    /**\n     * Flushes the data in the secondary cache to page cache or disk immediately.\n     *\n     * @param commitLeastPages the least pages to commit\n     * @return the committed position after the method call\n     */\n    int commit(int commitLeastPages);\n\n    /**\n     * Selects a slice of the mapped byte buffer's sub-region behind the mapped file,\n     * starting at the given position.\n     *\n     * @param pos the given position\n     * @param size the size of the returned sub-region\n     * @return a {@code SelectMappedBufferResult} instance contains the selected slice\n     */\n    SelectMappedBufferResult selectMappedBuffer(int pos, int size);\n\n    /**\n     * Selects a slice of the mapped byte buffer's sub-region behind the mapped file,\n     * starting at the given position.\n     *\n     * @param pos the given position\n     * @return a {@code SelectMappedBufferResult} instance contains the selected slice\n     */\n    SelectMappedBufferResult selectMappedBuffer(int pos);\n\n    /**\n     * Returns the mapped byte buffer behind the mapped file.\n     *\n     * @return the mapped byte buffer\n     */\n    MappedByteBuffer getMappedByteBuffer();\n\n    /**\n     * Returns a slice of the mapped byte buffer behind the mapped file.\n     *\n     * @return the slice of the mapped byte buffer\n     */\n    ByteBuffer sliceByteBuffer();\n\n    /**\n     * Returns the store timestamp of the last message.\n     *\n     * @return the store timestamp\n     */\n    long getStoreTimestamp();\n\n    /**\n     * Returns the last modified timestamp of the file.\n     *\n     * @return the last modified timestamp\n     */\n    long getLastModifiedTimestamp();\n\n    /**\n     * Get data from a certain pos offset with size byte\n     *\n     * @param pos a certain pos offset to get data\n     * @param size the size of data\n     * @param byteBuffer the data\n     * @return true if with data; false if no data;\n     */\n    boolean getData(int pos, int size, ByteBuffer byteBuffer);\n\n    /**\n     * Destroys the file and delete it from the file system.\n     *\n     * @param intervalForcibly The time interval in milliseconds after which any remaining references will be forcibly released during destroy\n     * @return true if success; false otherwise.\n     */\n    boolean destroy(long intervalForcibly);\n\n    /**\n     * Shutdowns the file and mark it unavailable.\n     *\n     * @param intervalForcibly The time interval in milliseconds after which any remaining references will be forcibly released during shutdown\n     */\n    void shutdown(long intervalForcibly);\n\n    /**\n     * Decreases the reference count by {@code 1} and clean up the mapped file if the reference count reaches at\n     * {@code 0}.\n     */\n    void release();\n\n    /**\n     * Increases the reference count by {@code 1}.\n     *\n     * @return true if success; false otherwise.\n     */\n    boolean hold();\n\n    /**\n     * Returns true if the current file is first mapped file of some consume queue.\n     *\n     * @return true or false\n     */\n    boolean isFirstCreateInQueue();\n\n    /**\n     * Sets the flag whether the current file is first mapped file of some consume queue.\n     *\n     * @param firstCreateInQueue true or false\n     */\n    void setFirstCreateInQueue(boolean firstCreateInQueue);\n\n    /**\n     * Returns the flushed position of this mapped file.\n     *\n     * @return the flushed posotion\n     */\n    int getFlushedPosition();\n\n    /**\n     * Sets the flushed position of this mapped file.\n     *\n     * @param flushedPosition the specific flushed position\n     */\n    void setFlushedPosition(int flushedPosition);\n\n    /**\n     * Returns the wrote position of this mapped file.\n     *\n     * @return the wrote position\n     */\n    int getWrotePosition();\n\n    /**\n     * Sets the wrote position of this mapped file.\n     *\n     * @param wrotePosition the specific wrote position\n     */\n    void setWrotePosition(int wrotePosition);\n\n    /**\n     * Returns the current max readable position of this mapped file.\n     *\n     * @return the max readable position\n     */\n    int getReadPosition();\n\n    /**\n     * Sets the committed position of this mapped file.\n     *\n     * @param committedPosition the specific committed position\n     */\n    void setCommittedPosition(int committedPosition);\n\n    /**\n     * Lock the mapped bytebuffer\n     */\n    void mlock();\n\n    /**\n     * Unlock the mapped bytebuffer\n     */\n    void munlock();\n\n    /**\n     * Warm up the mapped bytebuffer\n     * @param type\n     * @param pages\n     */\n    void warmMappedFile(FlushDiskType type, int pages);\n\n    /**\n     * Swap map\n     */\n    boolean swapMap();\n\n    /**\n     * Clean pageTable\n     */\n    void cleanSwapedMap(boolean force);\n\n    void cleanResources();\n\n    /**\n     * Get recent swap map time\n     */\n    long getRecentSwapMapTime();\n\n    /**\n     * Get recent MappedByteBuffer access count since last swap\n     */\n    long getMappedByteBufferAccessCountSinceLastSwap();\n\n    /**\n     * Get the underlying file\n     * @return\n     */\n    File getFile();\n\n    /**\n     * rename file to add \".delete\" suffix\n     */\n    void renameToDelete();\n\n    /**\n     * move the file to the parent directory\n     * @throws IOException\n     */\n    void moveToParent() throws IOException;\n\n    /**\n     * Get the last flush time\n     * @return\n     */\n    long getLastFlushTime();\n\n    /**\n     * Init mapped file\n     * @param fileName file name\n     * @param fileSize file size\n     * @param transientStorePool transient store pool\n     * @throws IOException\n     */\n    void init(String fileName, int fileSize, RunningFlags runningFlags, TransientStorePool transientStorePool) throws IOException;\n\n    Iterator<SelectMappedBufferResult> iterator(int pos);\n\n    /**\n     * Check mapped file is loaded to memory with given position and size\n     * @param position start offset of data\n     * @param size data size\n     * @return data is resided in memory or not\n     */\n    boolean isLoaded(long position, int size);\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/logfile/SharedByteBufferManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.logfile;\n\nimport java.nio.ByteBuffer;\nimport java.util.concurrent.ThreadLocalRandom;\nimport java.util.concurrent.locks.ReentrantLock;\n\n/**\n * Shared byte buffer manager for managing some shared ByteBuffers Buffer size is set based on MessageStoreConfig's\n * maxMessageSize\n */\npublic class SharedByteBufferManager {\n\n    private static volatile SharedByteBufferManager instance;\n    private static final Object LOCK = new Object();\n\n    private SharedByteBuffer[] sharedByteBuffers;\n    private int bufferSize;\n    private int maxSharedNum;\n    private volatile boolean initialized = false;\n\n    private SharedByteBufferManager() {\n        // Private constructor\n    }\n\n    /**\n     * Get singleton instance\n     */\n    public static SharedByteBufferManager getInstance() {\n        if (instance == null) {\n            synchronized (LOCK) {\n                if (instance == null) {\n                    instance = new SharedByteBufferManager();\n                }\n            }\n        }\n        return instance;\n    }\n\n    /**\n     * Initialize shared buffers with specified messageSize size and shared buffer number\n     *\n     * @param maxMessageSize max messageSize size\n     * @param sharedBufferNum number of shared buffers\n     */\n    public synchronized void init(int maxMessageSize, int sharedBufferNum) {\n        if (!initialized) {\n            //Reserve 64kb for encoding buffer outside body\n            bufferSize = Integer.MAX_VALUE - maxMessageSize >= 64 * 1024 ?\n                maxMessageSize + 64 * 1024 : Integer.MAX_VALUE;\n\n            this.maxSharedNum = sharedBufferNum;\n            this.sharedByteBuffers = new SharedByteBuffer[maxSharedNum];\n            for (int i = 0; i < maxSharedNum; i++) {\n                this.sharedByteBuffers[i] = new SharedByteBuffer(bufferSize);\n            }\n            this.initialized = true;\n        }\n    }\n\n    /**\n     * Borrow a shared buffer\n     *\n     * @return Shared buffer\n     */\n    public SharedByteBuffer borrowSharedByteBuffer() {\n        if (!initialized) {\n            throw new IllegalStateException(\"SharedByteBufferManager not initialized\");\n        }\n        int idx = ThreadLocalRandom.current().nextInt(maxSharedNum);\n        return sharedByteBuffers[idx];\n    }\n\n    /**\n     * Get current buffer size\n     *\n     * @return Buffer size\n     */\n    public int getBufferSize() {\n        return bufferSize;\n    }\n\n    /**\n     * Check if initialized\n     *\n     * @return Whether initialized\n     */\n    public boolean isInitialized() {\n        return initialized;\n    }\n\n    /**\n     * Shared byte buffer class\n     */\n    public static class SharedByteBuffer {\n        private final ReentrantLock lock;\n        private final ByteBuffer buffer;\n\n        public SharedByteBuffer(int size) {\n            this.lock = new ReentrantLock();\n            this.buffer = ByteBuffer.allocateDirect(size);\n        }\n\n        public void release() {\n            this.lock.unlock();\n        }\n\n        public ByteBuffer acquire() {\n            this.lock.lock();\n            return buffer;\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsConstant.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.metrics;\n\npublic class DefaultStoreMetricsConstant {\n    public static final String GAUGE_STORAGE_SIZE = \"rocketmq_storage_size\";\n    public static final String GAUGE_STORAGE_FLUSH_BEHIND = \"rocketmq_storage_flush_behind_bytes\";\n    public static final String GAUGE_STORAGE_DISPATCH_BEHIND = \"rocketmq_storage_dispatch_behind_bytes\";\n    public static final String GAUGE_STORAGE_MESSAGE_RESERVE_TIME = \"rocketmq_storage_message_reserve_time\";\n\n    public static final String GAUGE_TIMER_ENQUEUE_LAG = \"rocketmq_timer_enqueue_lag\";\n    public static final String GAUGE_TIMER_ENQUEUE_LATENCY = \"rocketmq_timer_enqueue_latency\";\n    public static final String GAUGE_TIMER_DEQUEUE_LAG = \"rocketmq_timer_dequeue_lag\";\n    public static final String GAUGE_TIMER_DEQUEUE_LATENCY = \"rocketmq_timer_dequeue_latency\";\n    public static final String GAUGE_TIMING_MESSAGES = \"rocketmq_timing_messages\";\n\n    public static final String COUNTER_TIMER_ENQUEUE_TOTAL = \"rocketmq_timer_enqueue_total\";\n    public static final String COUNTER_TIMER_DEQUEUE_TOTAL = \"rocketmq_timer_dequeue_total\";\n    public static final String GAUGE_TIMER_MESSAGE_SNAPSHOT = \"rocketmq_timer_message_snapshot\";\n    public static final String HISTOGRAM_DELAY_MSG_LATENCY = \"rocketmq_delay_message_latency\";\n\n    public static final String LABEL_STORAGE_TYPE = \"storage_type\";\n    public static final String DEFAULT_STORAGE_TYPE = \"local\";\n    public static final String LABEL_STORAGE_MEDIUM = \"storage_medium\";\n    public static final String DEFAULT_STORAGE_MEDIUM = \"disk\";\n    public static final String LABEL_TOPIC = \"topic\";\n    public static final String LABEL_TIMING_BOUND = \"timer_bound_s\";\n    public static final String GAUGE_BYTES_ROCKSDB_WRITTEN = \"rocketmq_rocksdb_bytes_written\";\n    public static final String GAUGE_BYTES_ROCKSDB_READ = \"rocketmq_rocksdb_bytes_read\";\n\n    public static final String GAUGE_TIMES_ROCKSDB_WRITTEN_SELF = \"rocketmq_rocksdb_times_written_self\";\n    public static final String GAUGE_TIMES_ROCKSDB_WRITTEN_OTHER = \"rocketmq_rocksdb_times_written_other\";\n    public static final String GAUGE_RATE_ROCKSDB_CACHE_HIT = \"rocketmq_rocksdb_rate_cache_hit\";\n    public static final String GAUGE_TIMES_ROCKSDB_COMPRESSED = \"rocketmq_rocksdb_times_compressed\";\n    public static final String GAUGE_BYTES_READ_AMPLIFICATION = \"rocketmq_rocksdb_read_amplification_bytes\";\n    public static final String GAUGE_TIMES_ROCKSDB_READ = \"rocketmq_rocksdb_times_read\";\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.metrics;\n\nimport com.google.common.collect.Lists;\nimport io.opentelemetry.api.common.Attributes;\nimport io.opentelemetry.api.common.AttributesBuilder;\nimport io.opentelemetry.api.metrics.LongCounter;\nimport io.opentelemetry.api.metrics.LongHistogram;\nimport io.opentelemetry.api.metrics.Meter;\nimport io.opentelemetry.api.metrics.ObservableLongGauge;\nimport io.opentelemetry.sdk.metrics.Aggregation;\nimport io.opentelemetry.sdk.metrics.InstrumentSelector;\nimport io.opentelemetry.sdk.metrics.InstrumentType;\nimport io.opentelemetry.sdk.metrics.View;\nimport io.opentelemetry.sdk.metrics.ViewBuilder;\nimport java.io.File;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.function.Supplier;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.metrics.NopLongCounter;\nimport org.apache.rocketmq.common.metrics.NopLongHistogram;\nimport org.apache.rocketmq.common.metrics.NopObservableLongGauge;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.timer.Slot;\nimport org.apache.rocketmq.store.timer.TimerMessageStore;\nimport org.apache.rocketmq.store.timer.TimerMetrics;\nimport org.apache.rocketmq.store.timer.TimerWheel;\n\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.COUNTER_TIMER_DEQUEUE_TOTAL;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.COUNTER_TIMER_ENQUEUE_TOTAL;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.DEFAULT_STORAGE_MEDIUM;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.DEFAULT_STORAGE_TYPE;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.GAUGE_STORAGE_DISPATCH_BEHIND;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.GAUGE_STORAGE_FLUSH_BEHIND;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.GAUGE_STORAGE_MESSAGE_RESERVE_TIME;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.GAUGE_STORAGE_SIZE;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.GAUGE_TIMER_DEQUEUE_LAG;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.GAUGE_TIMER_DEQUEUE_LATENCY;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.GAUGE_TIMER_ENQUEUE_LAG;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.GAUGE_TIMER_ENQUEUE_LATENCY;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.GAUGE_TIMER_MESSAGE_SNAPSHOT;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.GAUGE_TIMING_MESSAGES;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.HISTOGRAM_DELAY_MSG_LATENCY;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.LABEL_STORAGE_MEDIUM;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.LABEL_STORAGE_TYPE;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.LABEL_TIMING_BOUND;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.LABEL_TOPIC;\n\npublic class DefaultStoreMetricsManager implements StoreMetricsManager {\n    private Supplier<AttributesBuilder> attributesBuilderSupplier;\n    private MessageStoreConfig messageStoreConfig;\n\n    private ObservableLongGauge storageSize = new NopObservableLongGauge();\n    private ObservableLongGauge flushBehind = new NopObservableLongGauge();\n    private ObservableLongGauge dispatchBehind = new NopObservableLongGauge();\n    private ObservableLongGauge messageReserveTime = new NopObservableLongGauge();\n\n    private ObservableLongGauge timerEnqueueLag = new NopObservableLongGauge();\n    private ObservableLongGauge timerEnqueueLatency = new NopObservableLongGauge();\n    private ObservableLongGauge timerDequeueLag = new NopObservableLongGauge();\n    private ObservableLongGauge timerDequeueLatency = new NopObservableLongGauge();\n    private ObservableLongGauge timingMessages = new NopObservableLongGauge();\n\n    private LongCounter timerDequeueTotal = new NopLongCounter();\n    private LongCounter timerEnqueueTotal = new NopLongCounter();\n    private ObservableLongGauge timerMessageSnapshot = new NopObservableLongGauge();\n    private LongHistogram timerMessageSetLatency = new NopLongHistogram();\n\n    private RocksDBStoreMetricsManager rocksDBStoreMetricsManager;\n\n    public DefaultStoreMetricsManager() {\n        this.rocksDBStoreMetricsManager = new RocksDBStoreMetricsManager();\n    }\n\n    public List<Pair<InstrumentSelector, ViewBuilder>> getMetricsView() {\n        List<Double> rpcCostTimeBuckets = Arrays.asList(\n                // day * hour * min * second\n                1d * 1 * 1 * 60, // 60 second\n                1d * 1 * 10 * 60, // 10 min\n                1d * 1 * 60 * 60, // 1 hour\n                1d * 12 * 60 * 60, // 12 hour\n                1d * 24 * 60 * 60, // 1 day\n                3d * 24 * 60 * 60 // 3 day\n        );\n        InstrumentSelector selector = InstrumentSelector.builder()\n                .setType(InstrumentType.HISTOGRAM)\n                .setName(HISTOGRAM_DELAY_MSG_LATENCY)\n                .build();\n        ViewBuilder viewBuilder = View.builder()\n                .setAggregation(Aggregation.explicitBucketHistogram(rpcCostTimeBuckets));\n        return Lists.newArrayList(new Pair<>(selector, viewBuilder));\n    }\n\n    public void init(Meter meter, Supplier<AttributesBuilder> attributesBuilderSupplier,\n        MessageStore messageStore) {\n\n        // Also add some metrics for rocksdb's monitoring.\n        this.rocksDBStoreMetricsManager.init(meter, attributesBuilderSupplier, messageStore.getQueueStore());\n\n        this.attributesBuilderSupplier = attributesBuilderSupplier;\n        this.messageStoreConfig = messageStore.getMessageStoreConfig();\n\n        this.storageSize = meter.gaugeBuilder(GAUGE_STORAGE_SIZE)\n            .setDescription(\"Broker storage size\")\n            .setUnit(\"bytes\")\n            .ofLongs()\n            .buildWithCallback(measurement -> {\n                File storeDir = new File(this.messageStoreConfig.getStorePathRootDir());\n                if (storeDir.exists() && storeDir.isDirectory()) {\n                    long totalSpace = storeDir.getTotalSpace();\n                    if (totalSpace > 0) {\n                        measurement.record(totalSpace - storeDir.getFreeSpace(), this.newAttributesBuilder().build());\n                    }\n                }\n            });\n\n        this.flushBehind = meter.gaugeBuilder(GAUGE_STORAGE_FLUSH_BEHIND)\n            .setDescription(\"Broker flush behind bytes\")\n            .setUnit(\"bytes\")\n            .ofLongs()\n            .buildWithCallback(measurement -> measurement.record(messageStore.flushBehindBytes(), this.newAttributesBuilder().build()));\n\n        this.dispatchBehind = meter.gaugeBuilder(GAUGE_STORAGE_DISPATCH_BEHIND)\n            .setDescription(\"Broker dispatch behind bytes\")\n            .setUnit(\"bytes\")\n            .ofLongs()\n            .buildWithCallback(measurement -> measurement.record(messageStore.dispatchBehindBytes(), this.newAttributesBuilder().build()));\n\n        this.messageReserveTime = meter.gaugeBuilder(GAUGE_STORAGE_MESSAGE_RESERVE_TIME)\n            .setDescription(\"Broker message reserve time\")\n            .setUnit(\"milliseconds\")\n            .ofLongs()\n            .buildWithCallback(measurement -> {\n                long earliestMessageTime = messageStore.getEarliestMessageTime();\n                if (earliestMessageTime <= 0) {\n                    return;\n                }\n                measurement.record(System.currentTimeMillis() - earliestMessageTime, this.newAttributesBuilder().build());\n            });\n\n        if (messageStore.getMessageStoreConfig().isTimerWheelEnable()) {\n            this.timerEnqueueLag = meter.gaugeBuilder(GAUGE_TIMER_ENQUEUE_LAG)\n                .setDescription(\"Timer enqueue messages lag\")\n                .ofLongs()\n                .buildWithCallback(measurement -> {\n                    TimerMessageStore timerMessageStore = messageStore.getTimerMessageStore();\n                    measurement.record(timerMessageStore.getEnqueueBehindMessages(), this.newAttributesBuilder().build());\n                });\n\n            this.timerEnqueueLatency = meter.gaugeBuilder(GAUGE_TIMER_ENQUEUE_LATENCY)\n                .setDescription(\"Timer enqueue latency\")\n                .setUnit(\"milliseconds\")\n                .ofLongs()\n                .buildWithCallback(measurement -> {\n                    TimerMessageStore timerMessageStore = messageStore.getTimerMessageStore();\n                    measurement.record(timerMessageStore.getEnqueueBehindMillis(), this.newAttributesBuilder().build());\n                });\n            this.timerDequeueLag = meter.gaugeBuilder(GAUGE_TIMER_DEQUEUE_LAG)\n                .setDescription(\"Timer dequeue messages lag\")\n                .ofLongs()\n                .buildWithCallback(measurement -> {\n                    TimerMessageStore timerMessageStore = messageStore.getTimerMessageStore();\n                    measurement.record(timerMessageStore.getDequeueBehindMessages(), this.newAttributesBuilder().build());\n                });\n            this.timerDequeueLatency = meter.gaugeBuilder(GAUGE_TIMER_DEQUEUE_LATENCY)\n                .setDescription(\"Timer dequeue latency\")\n                .setUnit(\"milliseconds\")\n                .ofLongs()\n                .buildWithCallback(measurement -> {\n                    TimerMessageStore timerMessageStore = messageStore.getTimerMessageStore();\n                    measurement.record(timerMessageStore.getDequeueBehindMillis(), this.newAttributesBuilder().build());\n                });\n            this.timingMessages = meter.gaugeBuilder(GAUGE_TIMING_MESSAGES)\n                .setDescription(\"Current message number in timing\")\n                .ofLongs()\n                .buildWithCallback(measurement -> {\n                    TimerMessageStore timerMessageStore = messageStore.getTimerMessageStore();\n                    timerMessageStore.getTimerMetrics()\n                        .getTimingCount()\n                        .forEach((topic, metric) -> {\n                            measurement.record(\n                                metric.getCount().get(),\n                                this.newAttributesBuilder().put(LABEL_TOPIC, topic).build()\n                            );\n                        });\n                });\n            this.timerDequeueTotal = meter.counterBuilder(COUNTER_TIMER_DEQUEUE_TOTAL)\n                .setDescription(\"Total number of timer dequeue\")\n                .build();\n            this.timerEnqueueTotal = meter.counterBuilder(COUNTER_TIMER_ENQUEUE_TOTAL)\n                .setDescription(\"Total number of timer enqueue\")\n                .build();\n            this.timerMessageSnapshot = meter.gaugeBuilder(GAUGE_TIMER_MESSAGE_SNAPSHOT)\n                .setDescription(\"Timer message distribution snapshot, only count timing messages in 24h.\")\n                .ofLongs()\n                .buildWithCallback(measurement -> {\n                    TimerMetrics timerMetrics = messageStore.getTimerMessageStore().getTimerMetrics();\n                    TimerWheel timerWheel = messageStore.getTimerMessageStore().getTimerWheel();\n                    int precisionMs = this.messageStoreConfig.getTimerPrecisionMs();\n                    List<Integer> timerDist = timerMetrics.getTimerDistList();\n                    long currTime = System.currentTimeMillis() / precisionMs * precisionMs;\n                    for (int i = 0; i < timerDist.size(); i++) {\n                        int slotBeforeNum = i == 0 ? 0 : timerDist.get(i - 1) * 1000 / precisionMs;\n                        int slotTotalNum = timerDist.get(i) * 1000 / precisionMs;\n                        int periodTotal = 0;\n                        for (int j = slotBeforeNum; j < slotTotalNum; j++) {\n                            Slot slotEach = timerWheel.getSlot(currTime + (long) j * precisionMs);\n                            periodTotal += slotEach.num;\n                        }\n                        measurement.record(periodTotal, this.newAttributesBuilder().put(LABEL_TIMING_BOUND, timerDist.get(i).toString()).build());\n                    }\n                });\n            this.timerMessageSetLatency = meter.histogramBuilder(HISTOGRAM_DELAY_MSG_LATENCY)\n                    .setDescription(\"Timer message set latency distribution\")\n                    .setUnit(\"seconds\")\n                    .ofLongs()\n                    .build();\n        }\n    }\n\n    public void incTimerDequeueCount(String topic) {\n        this.timerDequeueTotal.add(1, this.newAttributesBuilder()\n            .put(LABEL_TOPIC, topic)\n            .build());\n    }\n\n    public void incTimerEnqueueCount(String topic) {\n        AttributesBuilder attributesBuilder = this.newAttributesBuilder();\n        if (topic != null) {\n            attributesBuilder.put(LABEL_TOPIC, topic);\n        }\n        this.timerEnqueueTotal.add(1, attributesBuilder.build());\n    }\n\n    public AttributesBuilder newAttributesBuilder() {\n        if (this.attributesBuilderSupplier == null) {\n            return Attributes.builder();\n        }\n        return this.attributesBuilderSupplier.get()\n            .put(LABEL_STORAGE_TYPE, DEFAULT_STORAGE_TYPE)\n            .put(LABEL_STORAGE_MEDIUM, DEFAULT_STORAGE_MEDIUM);\n    }\n\n    // Getter methods for external access\n    public Supplier<AttributesBuilder> getAttributesBuilderSupplier() {\n        return attributesBuilderSupplier;\n    }\n\n    public MessageStoreConfig getMessageStoreConfig() {\n        return messageStoreConfig;\n    }\n\n    public ObservableLongGauge getStorageSize() {\n        return storageSize;\n    }\n\n    public ObservableLongGauge getFlushBehind() {\n        return flushBehind;\n    }\n\n    public ObservableLongGauge getDispatchBehind() {\n        return dispatchBehind;\n    }\n\n    public ObservableLongGauge getMessageReserveTime() {\n        return messageReserveTime;\n    }\n\n    public ObservableLongGauge getTimerEnqueueLag() {\n        return timerEnqueueLag;\n    }\n\n    public ObservableLongGauge getTimerEnqueueLatency() {\n        return timerEnqueueLatency;\n    }\n\n    public ObservableLongGauge getTimerDequeueLag() {\n        return timerDequeueLag;\n    }\n\n    public ObservableLongGauge getTimerDequeueLatency() {\n        return timerDequeueLatency;\n    }\n\n    public ObservableLongGauge getTimingMessages() {\n        return timingMessages;\n    }\n\n    public LongCounter getTimerDequeueTotal() {\n        return timerDequeueTotal;\n    }\n\n    public LongCounter getTimerEnqueueTotal() {\n        return timerEnqueueTotal;\n    }\n\n    public ObservableLongGauge getTimerMessageSnapshot() {\n        return timerMessageSnapshot;\n    }\n\n    public LongHistogram getTimerMessageSetLatency() {\n        return timerMessageSetLatency;\n    }\n\n    // Setter methods for testing\n    public void setAttributesBuilderSupplier(Supplier<AttributesBuilder> attributesBuilderSupplier) {\n        this.attributesBuilderSupplier = attributesBuilderSupplier;\n    }\n\n    public RocksDBStoreMetricsManager getRocksDBStoreMetricsManager() {\n        return rocksDBStoreMetricsManager;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/metrics/RocksDBStoreMetricsManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.metrics;\n\nimport com.google.common.collect.Lists;\nimport io.opentelemetry.api.common.Attributes;\nimport io.opentelemetry.api.common.AttributesBuilder;\nimport io.opentelemetry.api.metrics.Meter;\nimport io.opentelemetry.api.metrics.ObservableDoubleGauge;\nimport io.opentelemetry.api.metrics.ObservableLongGauge;\nimport io.opentelemetry.sdk.metrics.InstrumentSelector;\nimport io.opentelemetry.sdk.metrics.ViewBuilder;\nimport java.util.List;\nimport java.util.function.Supplier;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.metrics.NopObservableDoubleGauge;\nimport org.apache.rocketmq.common.metrics.NopObservableLongGauge;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.queue.CombineConsumeQueueStore;\nimport org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface;\nimport org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore;\nimport org.rocksdb.TickerType;\n\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.DEFAULT_STORAGE_MEDIUM;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.DEFAULT_STORAGE_TYPE;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.GAUGE_BYTES_ROCKSDB_READ;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.GAUGE_BYTES_ROCKSDB_WRITTEN;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.LABEL_STORAGE_MEDIUM;\nimport static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.LABEL_STORAGE_TYPE;\n\npublic class RocksDBStoreMetricsManager {\n    private Supplier<AttributesBuilder> attributesBuilderSupplier;\n    private MessageStoreConfig messageStoreConfig;\n\n    // The cumulative number of bytes read from the database.\n    private ObservableLongGauge bytesRocksdbRead = new NopObservableLongGauge();\n\n    // The cumulative number of bytes written to the database.\n    private ObservableLongGauge bytesRocksdbWritten = new NopObservableLongGauge();\n\n    // The cumulative number of read operations performed.\n    private ObservableLongGauge timesRocksdbRead = new NopObservableLongGauge();\n\n    // The cumulative number of write operations performed.\n    private ObservableLongGauge timesRocksdbWrittenSelf = new NopObservableLongGauge();\n    private ObservableLongGauge timesRocksdbWrittenOther = new NopObservableLongGauge();\n\n    // The cumulative number of compressions that have occurred.\n    private ObservableLongGauge timesRocksdbCompressed = new NopObservableLongGauge();\n\n    // The ratio of the amount of data actually written to the storage medium to the amount of data written by the application.\n    private ObservableDoubleGauge bytesRocksdbAmplificationRead = new NopObservableDoubleGauge();\n\n    // The rate at which cache lookups were served from the cache rather than needing to be fetched from disk.\n    private ObservableDoubleGauge rocksdbCacheHitRate = new NopObservableDoubleGauge();\n\n    private volatile long blockCacheHitTimes = 0;\n    private volatile long blockCacheMissTimes = 0;\n\n    public RocksDBStoreMetricsManager() {\n    }\n\n\n\n    public List<Pair<InstrumentSelector, ViewBuilder>> getMetricsView() {\n        return Lists.newArrayList();\n    }\n\n    public void init(Meter meter, Supplier<AttributesBuilder> attributesBuilderSupplier,\n        ConsumeQueueStoreInterface consumeQueueStore) {\n\n        final RocksDBConsumeQueueStore rocksDBMessageStore;\n        if (consumeQueueStore instanceof RocksDBConsumeQueueStore) {\n            rocksDBMessageStore = (RocksDBConsumeQueueStore) consumeQueueStore;\n        } else if (consumeQueueStore instanceof CombineConsumeQueueStore) {\n            rocksDBMessageStore = ((CombineConsumeQueueStore) consumeQueueStore).getRocksDBConsumeQueueStore();\n        } else {\n            rocksDBMessageStore = null;\n        }\n\n        if (rocksDBMessageStore == null) {\n            return;\n        }\n\n        this.attributesBuilderSupplier = attributesBuilderSupplier;\n        this.bytesRocksdbWritten = meter.gaugeBuilder(GAUGE_BYTES_ROCKSDB_WRITTEN)\n                .setDescription(\"The cumulative number of bytes written to the database.\")\n                .ofLongs()\n                .buildWithCallback(measurement -> {\n                    measurement.record(rocksDBMessageStore\n                            .getStatistics().getTickerCount(TickerType.BYTES_WRITTEN), this.newAttributesBuilder().put(\"type\", \"consume_queue\").build());\n                });\n        this.bytesRocksdbRead = meter.gaugeBuilder(GAUGE_BYTES_ROCKSDB_READ)\n                .setDescription(\"The cumulative number of bytes read from the database.\")\n                .ofLongs()\n                .buildWithCallback(measurement -> {\n                    measurement.record(rocksDBMessageStore\n                            .getStatistics().getTickerCount(TickerType.BYTES_READ), this.newAttributesBuilder().put(\"type\", \"consume_queue\").build());\n                });\n        this.timesRocksdbWrittenSelf = meter.gaugeBuilder(DefaultStoreMetricsConstant.GAUGE_TIMES_ROCKSDB_WRITTEN_SELF)\n                .setDescription(\"The cumulative number of write operations performed by self.\")\n                .ofLongs()\n                .buildWithCallback(measurement -> {\n                    measurement.record(rocksDBMessageStore\n                            .getStatistics().getTickerCount(TickerType.WRITE_DONE_BY_SELF), this.newAttributesBuilder().put(\"type\", \"consume_queue\").build());\n                });\n        this.timesRocksdbWrittenOther = meter.gaugeBuilder(DefaultStoreMetricsConstant.GAUGE_TIMES_ROCKSDB_WRITTEN_OTHER)\n                .setDescription(\"The cumulative number of write operations performed by other.\")\n                .ofLongs()\n                .buildWithCallback(measurement -> {\n                    measurement.record(rocksDBMessageStore\n                            .getStatistics().getTickerCount(TickerType.WRITE_DONE_BY_OTHER), this.newAttributesBuilder().put(\"type\", \"consume_queue\").build());\n                });\n        this.timesRocksdbRead = meter.gaugeBuilder(DefaultStoreMetricsConstant.GAUGE_TIMES_ROCKSDB_READ)\n                .setDescription(\"The cumulative number of write operations performed by other.\")\n                .ofLongs()\n                .buildWithCallback(measurement -> {\n                    measurement.record(rocksDBMessageStore\n                            .getStatistics().getTickerCount(TickerType.NUMBER_KEYS_READ), this.newAttributesBuilder().put(\"type\", \"consume_queue\").build());\n                });\n        this.rocksdbCacheHitRate = meter.gaugeBuilder(DefaultStoreMetricsConstant.GAUGE_RATE_ROCKSDB_CACHE_HIT)\n                .setDescription(\"The rate at which cache lookups were served from the cache rather than needing to be fetched from disk.\")\n                .buildWithCallback(measurement -> {\n                    long newHitTimes = rocksDBMessageStore.getStatistics().getTickerCount(TickerType.BLOCK_CACHE_HIT);\n                    long newMissTimes = rocksDBMessageStore.getStatistics().getTickerCount(TickerType.BLOCK_CACHE_MISS);\n                    long totalPeriod = newHitTimes - this.blockCacheHitTimes + newMissTimes - this.blockCacheMissTimes;\n                    double hitRate = totalPeriod == 0 ? 0 : (double)(newHitTimes - this.blockCacheHitTimes) / totalPeriod;\n                    this.blockCacheHitTimes = newHitTimes;\n                    this.blockCacheMissTimes = newMissTimes;\n                    measurement.record(hitRate, this.newAttributesBuilder().put(\"type\", \"consume_queue\").build());\n                });\n        this.timesRocksdbCompressed = meter.gaugeBuilder(DefaultStoreMetricsConstant.GAUGE_TIMES_ROCKSDB_COMPRESSED)\n                .setDescription(\"The cumulative number of compressions that have occurred.\")\n                .ofLongs()\n                .buildWithCallback(measurement -> {\n                    measurement.record(rocksDBMessageStore\n                            .getStatistics().getTickerCount(TickerType.NUMBER_BLOCK_COMPRESSED), this.newAttributesBuilder().put(\"type\", \"consume_queue\").build());\n                });\n        this.bytesRocksdbAmplificationRead = meter.gaugeBuilder(DefaultStoreMetricsConstant.GAUGE_BYTES_READ_AMPLIFICATION)\n                .setDescription(\"The rate at which cache lookups were served from the cache rather than needing to be fetched from disk.\")\n                .buildWithCallback(measurement -> {\n                    measurement.record(rocksDBMessageStore\n                            .getStatistics().getTickerCount(TickerType.READ_AMP_TOTAL_READ_BYTES), this.newAttributesBuilder().put(\"type\", \"consume_queue\").build());\n                });\n    }\n\n    public AttributesBuilder newAttributesBuilder() {\n        if (this.attributesBuilderSupplier == null) {\n            return Attributes.builder();\n        }\n        return this.attributesBuilderSupplier.get()\n            .put(LABEL_STORAGE_TYPE, DEFAULT_STORAGE_TYPE)\n            .put(LABEL_STORAGE_MEDIUM, DEFAULT_STORAGE_MEDIUM);\n    }\n\n    // Getter methods for external access\n    public Supplier<AttributesBuilder> getAttributesBuilderSupplier() {\n        return attributesBuilderSupplier;\n    }\n\n    public MessageStoreConfig getMessageStoreConfig() {\n        return messageStoreConfig;\n    }\n\n    public ObservableLongGauge getBytesRocksdbRead() {\n        return bytesRocksdbRead;\n    }\n\n    public ObservableLongGauge getBytesRocksdbWritten() {\n        return bytesRocksdbWritten;\n    }\n\n    public ObservableLongGauge getTimesRocksdbRead() {\n        return timesRocksdbRead;\n    }\n\n    public ObservableLongGauge getTimesRocksdbWrittenSelf() {\n        return timesRocksdbWrittenSelf;\n    }\n\n    public ObservableLongGauge getTimesRocksdbWrittenOther() {\n        return timesRocksdbWrittenOther;\n    }\n\n    public ObservableLongGauge getTimesRocksdbCompressed() {\n        return timesRocksdbCompressed;\n    }\n\n    public ObservableDoubleGauge getBytesRocksdbAmplificationRead() {\n        return bytesRocksdbAmplificationRead;\n    }\n\n    public ObservableDoubleGauge getRocksdbCacheHitRate() {\n        return rocksdbCacheHitRate;\n    }\n\n    public long getBlockCacheHitTimes() {\n        return blockCacheHitTimes;\n    }\n\n    public long getBlockCacheMissTimes() {\n        return blockCacheMissTimes;\n    }\n\n    // Setter methods for testing\n    public void setAttributesBuilderSupplier(Supplier<AttributesBuilder> attributesBuilderSupplier) {\n        this.attributesBuilderSupplier = attributesBuilderSupplier;\n    }\n\n    public void setMessageStoreConfig(MessageStoreConfig messageStoreConfig) {\n        this.messageStoreConfig = messageStoreConfig;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/metrics/StoreMetricsManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.metrics;\n\nimport io.opentelemetry.api.common.AttributesBuilder;\nimport java.util.List;\nimport java.util.function.Supplier;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.store.MessageStore;\nimport io.opentelemetry.api.metrics.Meter;\nimport io.opentelemetry.sdk.metrics.InstrumentSelector;\nimport io.opentelemetry.sdk.metrics.ViewBuilder;\n\n/**\n * Store metrics manager interface for different message store implementations.\n * This interface provides a unified way to access metrics functionality\n * regardless of the underlying message store type.\n */\npublic interface StoreMetricsManager {\n\n    /**\n     * Initialize metrics with the given meter and attributes builder supplier.\n     *\n     * @param meter                     OpenTelemetry meter\n     * @param attributesBuilderSupplier Metrics attributes builder supplier\n     * @param messageStore             The message store instance\n     */\n    void init(Meter meter, Supplier<AttributesBuilder> attributesBuilderSupplier, MessageStore messageStore);\n\n    /**\n     * Get metrics view configuration.\n     *\n     * @return List of instrument selector and view builder pairs\n     */\n    List<Pair<InstrumentSelector, ViewBuilder>> getMetricsView();\n\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.plugin;\n\nimport io.opentelemetry.api.common.AttributesBuilder;\nimport io.opentelemetry.api.metrics.Meter;\nimport io.opentelemetry.sdk.metrics.InstrumentSelector;\nimport io.opentelemetry.sdk.metrics.ViewBuilder;\nimport java.nio.ByteBuffer;\nimport java.util.HashMap;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.function.Supplier;\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.SystemClock;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBatch;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;\nimport org.apache.rocketmq.store.AllocateMappedFileService;\nimport org.apache.rocketmq.store.AppendMessageResult;\nimport org.apache.rocketmq.store.CommitLog;\nimport org.apache.rocketmq.store.CommitLogDispatcher;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.MessageFilter;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.MessageStoreStateMachine;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.QueryMessageResult;\nimport org.apache.rocketmq.store.RunningFlags;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.StoreCheckpoint;\nimport org.apache.rocketmq.store.StoreStatsService;\nimport org.apache.rocketmq.store.TransientStorePool;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.ha.HAService;\nimport org.apache.rocketmq.store.hook.PutMessageHook;\nimport org.apache.rocketmq.store.hook.SendMessageBackHook;\nimport org.apache.rocketmq.store.logfile.MappedFile;\nimport org.apache.rocketmq.store.queue.ConsumeQueueInterface;\nimport org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.apache.rocketmq.store.timer.TimerMessageStore;\nimport org.apache.rocketmq.store.util.PerfCounter;\nimport org.apache.rocketmq.store.metrics.StoreMetricsManager;\nimport org.rocksdb.RocksDBException;\n\npublic abstract class AbstractPluginMessageStore implements MessageStore {\n    protected MessageStore next;\n    protected MessageStorePluginContext context;\n\n    public AbstractPluginMessageStore(MessageStorePluginContext context, MessageStore next) {\n        this.next = next;\n        this.context = context;\n    }\n\n    @Override\n    public long getEarliestMessageTime() {\n        return next.getEarliestMessageTime();\n    }\n\n    @Override\n    public long lockTimeMills() {\n        return next.lockTimeMills();\n    }\n\n    @Override\n    public boolean isOSPageCacheBusy() {\n        return next.isOSPageCacheBusy();\n    }\n\n    @Override\n    public boolean isTransientStorePoolDeficient() {\n        return next.isTransientStorePoolDeficient();\n    }\n\n    @Override\n    public boolean load() {\n        return next.load();\n    }\n\n    @Override\n    public void start() throws Exception {\n        next.start();\n    }\n\n    @Override\n    public void shutdown() {\n        next.shutdown();\n    }\n\n    @Override\n    public void destroy() {\n        next.destroy();\n    }\n\n    @Override\n    public PutMessageResult putMessage(MessageExtBrokerInner msg) {\n        return next.putMessage(msg);\n    }\n\n    @Override\n    public CompletableFuture<PutMessageResult> asyncPutMessage(MessageExtBrokerInner msg) {\n        return next.asyncPutMessage(msg);\n    }\n\n    @Override\n    public CompletableFuture<PutMessageResult> asyncPutMessages(MessageExtBatch messageExtBatch) {\n        return next.asyncPutMessages(messageExtBatch);\n    }\n\n    @Override\n    public GetMessageResult getMessage(String group, String topic, int queueId, long offset,\n        int maxMsgNums, final MessageFilter messageFilter) {\n        return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter);\n    }\n\n    @Override\n    public CompletableFuture<GetMessageResult> getMessageAsync(String group, String topic,\n        int queueId, long offset, int maxMsgNums, MessageFilter messageFilter) {\n        return next.getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter);\n    }\n\n    @Override\n    public long getMaxOffsetInQueue(String topic, int queueId) throws ConsumeQueueException {\n        return next.getMaxOffsetInQueue(topic, queueId);\n    }\n\n    @Override\n    public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) throws ConsumeQueueException {\n        return next.getMaxOffsetInQueue(topic, queueId, committed);\n    }\n\n    @Override\n    public long getMinOffsetInQueue(String topic, int queueId) {\n        return next.getMinOffsetInQueue(topic, queueId);\n    }\n\n    @Override\n    public long getCommitLogOffsetInQueue(String topic, int queueId, long consumeQueueOffset) {\n        return next.getCommitLogOffsetInQueue(topic, queueId, consumeQueueOffset);\n    }\n\n    @Override\n    public long getOffsetInQueueByTime(String topic, int queueId, long timestamp) {\n        return next.getOffsetInQueueByTime(topic, queueId, timestamp);\n    }\n\n    @Override\n    public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) {\n        return next.getOffsetInQueueByTime(topic, queueId, timestamp, boundaryType);\n    }\n\n    @Override\n    public MessageExt lookMessageByOffset(long commitLogOffset) {\n        return next.lookMessageByOffset(commitLogOffset);\n    }\n\n    @Override\n    public SelectMappedBufferResult selectOneMessageByOffset(long commitLogOffset) {\n        return next.selectOneMessageByOffset(commitLogOffset);\n    }\n\n    @Override\n    public SelectMappedBufferResult selectOneMessageByOffset(long commitLogOffset, int msgSize) {\n        return next.selectOneMessageByOffset(commitLogOffset, msgSize);\n    }\n\n    @Override\n    public String getRunningDataInfo() {\n        return next.getRunningDataInfo();\n    }\n\n    @Override\n    public HashMap<String, String> getRuntimeInfo() {\n        return next.getRuntimeInfo();\n    }\n\n    @Override\n    public long getMaxPhyOffset() {\n        return next.getMaxPhyOffset();\n    }\n\n    @Override\n    public long getMinPhyOffset() {\n        return next.getMinPhyOffset();\n    }\n\n    @Override\n    public long getEarliestMessageTime(String topic, int queueId) {\n        return next.getEarliestMessageTime(topic, queueId);\n    }\n\n    @Override\n    public CompletableFuture<Long> getEarliestMessageTimeAsync(String topic, int queueId) {\n        return next.getEarliestMessageTimeAsync(topic, queueId);\n    }\n\n    @Override\n    public long getMessageStoreTimeStamp(String topic, int queueId, long consumeQueueOffset) {\n        return next.getMessageStoreTimeStamp(topic, queueId, consumeQueueOffset);\n    }\n\n    @Override\n    public CompletableFuture<Long> getMessageStoreTimeStampAsync(String topic, int queueId,\n        long consumeQueueOffset) {\n        return next.getMessageStoreTimeStampAsync(topic, queueId, consumeQueueOffset);\n    }\n\n    @Override\n    public long getMessageTotalInQueue(String topic, int queueId) {\n        return next.getMessageTotalInQueue(topic, queueId);\n    }\n\n    @Override\n    public SelectMappedBufferResult getCommitLogData(long offset) {\n        return next.getCommitLogData(offset);\n    }\n\n    @Override\n    public boolean appendToCommitLog(long startOffset, byte[] data, int dataStart, int dataLength) {\n        return next.appendToCommitLog(startOffset, data, dataStart, dataLength);\n    }\n\n    @Override\n    public void executeDeleteFilesManually() {\n        next.executeDeleteFilesManually();\n    }\n\n    @Override\n    public QueryMessageResult queryMessage(String topic, String key, int maxNum, long begin,\n        long end) {\n        return next.queryMessage(topic, key, maxNum, begin, end);\n    }\n\n    @Override\n    public CompletableFuture<QueryMessageResult> queryMessageAsync(String topic, String key,\n        int maxNum, long begin, long end) {\n        return next.queryMessageAsync(topic, key, maxNum, begin, end);\n    }\n\n    @Override\n    public long now() {\n        return next.now();\n    }\n\n    @Override\n    public int deleteTopics(final Set<String> deleteTopics) {\n        return next.deleteTopics(deleteTopics);\n    }\n\n    @Override\n    public int cleanUnusedTopic(final Set<String> retainTopics) {\n        return next.cleanUnusedTopic(retainTopics);\n    }\n\n    @Override\n    public void cleanExpiredConsumerQueue() {\n        next.cleanExpiredConsumerQueue();\n    }\n\n    @Override\n    @Deprecated\n    public boolean checkInDiskByConsumeOffset(String topic, int queueId, long consumeOffset) {\n        return next.checkInDiskByConsumeOffset(topic, queueId, consumeOffset);\n    }\n\n    @Override\n    public boolean checkInMemByConsumeOffset(String topic, int queueId, long consumeOffset, int batchSize) {\n        return next.checkInMemByConsumeOffset(topic, queueId, consumeOffset, batchSize);\n    }\n\n    @Override\n    public boolean checkInStoreByConsumeOffset(String topic, int queueId, long consumeOffset) {\n        return next.checkInStoreByConsumeOffset(topic, queueId, consumeOffset);\n    }\n\n    @Override\n    public long dispatchBehindBytes() {\n        return next.dispatchBehindBytes();\n    }\n\n    @Override\n    public long flushBehindBytes() {\n        return next.flushBehindBytes();\n    }\n\n    @Override\n    public long dispatchBehindMilliseconds() {\n        return next.dispatchBehindMilliseconds();\n    }\n\n    @Override\n    public long flush() {\n        return next.flush();\n    }\n\n    @Override\n    public long getConfirmOffset() {\n        return next.getConfirmOffset();\n    }\n\n    @Override\n    public void setConfirmOffset(long phyOffset) {\n        next.setConfirmOffset(phyOffset);\n    }\n\n    @Override\n    public LinkedList<CommitLogDispatcher> getDispatcherList() {\n        return next.getDispatcherList();\n    }\n\n    @Override\n    public void addDispatcher(CommitLogDispatcher dispatcher) {\n        next.addDispatcher(dispatcher);\n    }\n\n    @Override\n    public ConsumeQueueInterface getConsumeQueue(String topic, int queueId) {\n        return next.getConsumeQueue(topic, queueId);\n    }\n\n    @Override\n    public ConsumeQueueInterface findConsumeQueue(String topic, int queueId) {\n        return next.findConsumeQueue(topic, queueId);\n    }\n\n    @Override\n    public BrokerStatsManager getBrokerStatsManager() {\n        return next.getBrokerStatsManager();\n    }\n\n    @Override\n    public int remainTransientStoreBufferNumbs() {\n        return next.remainTransientStoreBufferNumbs();\n    }\n\n    @Override\n    public long remainHowManyDataToCommit() {\n        return next.remainHowManyDataToCommit();\n    }\n\n    @Override\n    public long remainHowManyDataToFlush() {\n        return next.remainHowManyDataToFlush();\n    }\n\n    @Override\n    public DispatchRequest checkMessageAndReturnSize(final ByteBuffer byteBuffer, final boolean checkCRC,\n        final boolean checkDupInfo, final boolean readBody) {\n        return next.checkMessageAndReturnSize(byteBuffer, checkCRC, checkDupInfo, readBody);\n    }\n\n    @Override\n    public long getStateMachineVersion() {\n        return next.getStateMachineVersion();\n    }\n\n    @Override\n    public PutMessageResult putMessages(MessageExtBatch messageExtBatch) {\n        return next.putMessages(messageExtBatch);\n    }\n\n    @Override\n    public HARuntimeInfo getHARuntimeInfo() {\n        return next.getHARuntimeInfo();\n    }\n\n    @Override\n    public boolean getLastMappedFile(long startOffset) {\n        return next.getLastMappedFile(startOffset);\n    }\n\n    @Override\n    public void updateHaMasterAddress(String newAddr) {\n        next.updateHaMasterAddress(newAddr);\n    }\n\n    @Override\n    public void updateMasterAddress(String newAddr) {\n        next.updateMasterAddress(newAddr);\n    }\n\n    @Override\n    public long slaveFallBehindMuch() {\n        return next.slaveFallBehindMuch();\n    }\n\n    @Override\n    public long getFlushedWhere() {\n        return next.getFlushedWhere();\n    }\n\n    @Override\n    public MessageStore getMasterStoreInProcess() {\n        return next.getMasterStoreInProcess();\n    }\n\n    @Override\n    public void setMasterStoreInProcess(MessageStore masterStoreInProcess) {\n        next.setMasterStoreInProcess(masterStoreInProcess);\n    }\n\n    @Override\n    public boolean getData(long offset, int size, ByteBuffer byteBuffer) {\n        return next.getData(offset, size, byteBuffer);\n    }\n\n    @Override\n    public void setAliveReplicaNumInGroup(int aliveReplicaNums) {\n        next.setAliveReplicaNumInGroup(aliveReplicaNums);\n    }\n\n    @Override\n    public int getAliveReplicaNumInGroup() {\n        return next.getAliveReplicaNumInGroup();\n    }\n\n    @Override\n    public void wakeupHAClient() {\n        next.wakeupHAClient();\n    }\n\n    @Override\n    public long getMasterFlushedOffset() {\n        return next.getMasterFlushedOffset();\n    }\n\n    @Override\n    public long getBrokerInitMaxOffset() {\n        return next.getBrokerInitMaxOffset();\n    }\n\n    @Override\n    public void setMasterFlushedOffset(long masterFlushedOffset) {\n        next.setMasterFlushedOffset(masterFlushedOffset);\n    }\n\n    @Override\n    public void setBrokerInitMaxOffset(long brokerInitMaxOffset) {\n        next.setBrokerInitMaxOffset(brokerInitMaxOffset);\n    }\n\n    @Override\n    public byte[] calcDeltaChecksum(long from, long to) {\n        return next.calcDeltaChecksum(from, to);\n    }\n\n    @Override\n    public HAService getHaService() {\n        return next.getHaService();\n    }\n\n    @Override\n    public boolean truncateFiles(long offsetToTruncate) throws RocksDBException {\n        return next.truncateFiles(offsetToTruncate);\n    }\n\n    @Override\n    public boolean isOffsetAligned(long offset) {\n        return next.isOffsetAligned(offset);\n    }\n\n    @Override\n    public RunningFlags getRunningFlags() {\n        return next.getRunningFlags();\n    }\n\n    @Override\n    public void setSendMessageBackHook(SendMessageBackHook sendMessageBackHook) {\n        next.setSendMessageBackHook(sendMessageBackHook);\n    }\n\n    @Override\n    public SendMessageBackHook getSendMessageBackHook() {\n        return next.getSendMessageBackHook();\n    }\n\n    @Override\n    public GetMessageResult getMessage(String group, String topic, int queueId, long offset,\n        int maxMsgNums, int maxTotalMsgSize, MessageFilter messageFilter) {\n        return next.getMessage(group, topic, queueId, offset, maxMsgNums, maxTotalMsgSize, messageFilter);\n    }\n\n    @Override\n    public CompletableFuture<GetMessageResult> getMessageAsync(String group, String topic,\n        int queueId, long offset, int maxMsgNums, int maxTotalMsgSize,\n        MessageFilter messageFilter) {\n        return next.getMessageAsync(group, topic, queueId, offset, maxMsgNums, maxTotalMsgSize, messageFilter);\n    }\n\n    @Override\n    public MessageExt lookMessageByOffset(long commitLogOffset, int size) {\n        return next.lookMessageByOffset(commitLogOffset, size);\n    }\n\n    @Override\n    public List<SelectMappedBufferResult> getBulkCommitLogData(long offset, int size) {\n        return next.getBulkCommitLogData(offset, size);\n    }\n\n    @Override\n    public void onCommitLogAppend(MessageExtBrokerInner msg, AppendMessageResult result, MappedFile commitLogFile) {\n        next.onCommitLogAppend(msg, result, commitLogFile);\n    }\n\n    @Override\n    public void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile,\n        boolean isRecover, boolean isFileEnd) throws RocksDBException {\n        next.onCommitLogDispatch(dispatchRequest, doDispatch, commitLogFile, isRecover, isFileEnd);\n    }\n\n    @Override\n    public MessageStoreConfig getMessageStoreConfig() {\n        return next.getMessageStoreConfig();\n    }\n\n    @Override\n    public StoreStatsService getStoreStatsService() {\n        return next.getStoreStatsService();\n    }\n\n    @Override\n    public StoreCheckpoint getStoreCheckpoint() {\n        return next.getStoreCheckpoint();\n    }\n\n    @Override\n    public SystemClock getSystemClock() {\n        return next.getSystemClock();\n    }\n\n    @Override\n    public CommitLog getCommitLog() {\n        return next.getCommitLog();\n    }\n\n    @Override\n    public TransientStorePool getTransientStorePool() {\n        return next.getTransientStorePool();\n    }\n\n    @Override\n    public AllocateMappedFileService getAllocateMappedFileService() {\n        return next.getAllocateMappedFileService();\n    }\n\n    @Override\n    public void truncateDirtyLogicFiles(long phyOffset) throws RocksDBException {\n        next.truncateDirtyLogicFiles(phyOffset);\n    }\n\n    @Override\n    public void unlockMappedFile(MappedFile unlockMappedFile) {\n        next.unlockMappedFile(unlockMappedFile);\n    }\n\n    @Override\n    public PerfCounter.Ticks getPerfCounter() {\n        return next.getPerfCounter();\n    }\n\n    @Override\n    public ConsumeQueueStoreInterface getQueueStore() {\n        return next.getQueueStore();\n    }\n\n    @Override\n    public boolean isSyncDiskFlush() {\n        return next.isSyncDiskFlush();\n    }\n\n    @Override\n    public boolean isSyncMaster() {\n        return next.isSyncMaster();\n    }\n\n    @Override\n    public void assignOffset(MessageExtBrokerInner msg) throws RocksDBException {\n        next.assignOffset(msg);\n    }\n\n    @Override\n    public void increaseOffset(MessageExtBrokerInner msg, short messageNum) {\n        next.increaseOffset(msg, messageNum);\n    }\n\n    @Override\n    public List<PutMessageHook> getPutMessageHookList() {\n        return next.getPutMessageHookList();\n    }\n\n    @Override\n    public long getLastFileFromOffset() {\n        return next.getLastFileFromOffset();\n    }\n\n    @Override\n    public void setPhysicalOffset(long phyOffset) {\n        next.setPhysicalOffset(phyOffset);\n    }\n\n    @Override\n    public boolean isMappedFilesEmpty() {\n        return next.isMappedFilesEmpty();\n    }\n\n    @Override\n    public TimerMessageStore getTimerMessageStore() {\n        return next.getTimerMessageStore();\n    }\n\n    @Override\n    public void setTimerMessageStore(TimerMessageStore timerMessageStore) {\n        next.setTimerMessageStore(timerMessageStore);\n    }\n\n    @Override\n    public long getTimingMessageCount(String topic) {\n        return next.getTimingMessageCount(topic);\n    }\n\n    @Override\n    public boolean isShutdown() {\n        return next.isShutdown();\n    }\n\n    @Override\n    public long estimateMessageCount(String topic, int queueId, long from, long to, MessageFilter filter) {\n        return next.estimateMessageCount(topic, queueId, from, to, filter);\n    }\n\n    @Override\n    public List<Pair<InstrumentSelector, ViewBuilder>> getMetricsView() {\n        return next.getMetricsView();\n    }\n\n    @Override\n    public void initMetrics(Meter meter, Supplier<AttributesBuilder> attributesBuilderSupplier) {\n        next.initMetrics(meter, attributesBuilderSupplier);\n    }\n\n    @Override\n    public void recoverTopicQueueTable() {\n        next.recoverTopicQueueTable();\n    }\n\n    @Override\n    public void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest) {\n        next.notifyMessageArriveIfNecessary(dispatchRequest);\n    }\n\n    public MessageStore getNext() {\n        return next;\n    }\n\n    @Override\n    public MessageStoreStateMachine getStateMachine() {\n        return next.getStateMachine();\n    }\n\n    @Override\n    public StoreMetricsManager getStoreMetricsManager() {\n        return next.getStoreMetricsManager();\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/plugin/MessageStoreFactory.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.plugin;\n\nimport java.io.IOException;\nimport java.lang.reflect.Constructor;\nimport org.apache.rocketmq.store.MessageStore;\n\npublic final class MessageStoreFactory {\n    public static MessageStore build(MessageStorePluginContext context,\n        MessageStore messageStore) throws IOException {\n        String plugin = context.getBrokerConfig().getMessageStorePlugIn();\n        if (plugin != null && plugin.trim().length() != 0) {\n            String[] pluginClasses = plugin.split(\",\");\n            for (int i = pluginClasses.length - 1; i >= 0; --i) {\n                String pluginClass = pluginClasses[i];\n                try {\n                    @SuppressWarnings(\"unchecked\")\n                    Class<AbstractPluginMessageStore> clazz = (Class<AbstractPluginMessageStore>) Class.forName(pluginClass);\n                    Constructor<AbstractPluginMessageStore> construct = clazz.getConstructor(MessageStorePluginContext.class, MessageStore.class);\n                    AbstractPluginMessageStore pluginMessageStore = construct.newInstance(context, messageStore);\n                    messageStore = pluginMessageStore;\n                } catch (Throwable e) {\n                    throw new RuntimeException(\"Initialize plugin's class: \" + pluginClass + \" not found!\", e);\n                }\n            }\n        }\n        return messageStore;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/plugin/MessageStorePluginContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.plugin;\n\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.remoting.Configuration;\nimport org.apache.rocketmq.store.MessageArrivingListener;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\n\npublic class MessageStorePluginContext {\n    private MessageStoreConfig messageStoreConfig;\n    private BrokerStatsManager brokerStatsManager;\n    private MessageArrivingListener messageArrivingListener;\n    private BrokerConfig brokerConfig;\n    private final Configuration configuration;\n\n    public MessageStorePluginContext(MessageStoreConfig messageStoreConfig,\n        BrokerStatsManager brokerStatsManager, MessageArrivingListener messageArrivingListener,\n        BrokerConfig brokerConfig, Configuration configuration) {\n        super();\n        this.messageStoreConfig = messageStoreConfig;\n        this.brokerStatsManager = brokerStatsManager;\n        this.messageArrivingListener = messageArrivingListener;\n        this.brokerConfig = brokerConfig;\n        this.configuration = configuration;\n    }\n\n    public MessageStoreConfig getMessageStoreConfig() {\n        return messageStoreConfig;\n    }\n\n    public BrokerStatsManager getBrokerStatsManager() {\n        return brokerStatsManager;\n    }\n\n    public MessageArrivingListener getMessageArrivingListener() {\n        return messageArrivingListener;\n    }\n\n    public BrokerConfig getBrokerConfig() {\n        return brokerConfig;\n    }\n\n    public void registerConfiguration(Object config) {\n        MixAll.properties2Object(configuration.getAllConfigs(), config);\n        configuration.registerConfig(config);\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/pop/AckMsg.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.pop;\n\nimport com.alibaba.fastjson2.annotation.JSONField;\n\npublic class AckMsg {\n\n    @JSONField(name = \"ao\", alternateNames = {\"ackOffset\"})\n    private long ackOffset;\n\n    @JSONField(name = \"so\", alternateNames = {\"startOffset\"})\n    private long startOffset;\n\n    @JSONField(name = \"c\", alternateNames = {\"consumerGroup\"})\n    private String consumerGroup;\n\n    @JSONField(name = \"t\", alternateNames = {\"topic\"})\n    private String topic;\n\n    @JSONField(name = \"q\", alternateNames = {\"queueId\"})\n    private int queueId;\n\n    @JSONField(name = \"pt\", alternateNames = {\"popTime\"})\n    private long popTime;\n\n    @JSONField(name = \"bn\", alternateNames = {\"brokerName\"})\n    private String brokerName;\n\n    public long getPopTime() {\n        return popTime;\n    }\n\n    public void setPopTime(long popTime) {\n        this.popTime = popTime;\n    }\n\n    public void setQueueId(int queueId) {\n        this.queueId = queueId;\n    }\n\n    public int getQueueId() {\n        return queueId;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public long getAckOffset() {\n        return ackOffset;\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public void setAckOffset(long ackOffset) {\n        this.ackOffset = ackOffset;\n    }\n\n    public long getStartOffset() {\n        return startOffset;\n    }\n\n    public void setStartOffset(long startOffset) {\n        this.startOffset = startOffset;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    @Override\n    public String toString() {\n        final StringBuilder sb = new StringBuilder(\"AckMsg{\");\n        sb.append(\"ackOffset=\").append(ackOffset);\n        sb.append(\", startOffset=\").append(startOffset);\n        sb.append(\", consumerGroup='\").append(consumerGroup).append('\\'');\n        sb.append(\", topic='\").append(topic).append('\\'');\n        sb.append(\", queueId=\").append(queueId);\n        sb.append(\", popTime=\").append(popTime);\n        sb.append(\", brokerName=\").append(brokerName);\n        sb.append('}');\n        return sb.toString();\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/pop/BatchAckMsg.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.pop;\n\nimport com.alibaba.fastjson2.annotation.JSONField;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\n\npublic class BatchAckMsg extends AckMsg {\n    @JSONField(name = \"aol\", alternateNames = {\"ackOffsetList\"})\n    private List<Long> ackOffsetList = new ArrayList(32);\n\n\n    public List<Long> getAckOffsetList() {\n        return ackOffsetList;\n    }\n\n    public void setAckOffsetList(List<Long> ackOffsetList) {\n        this.ackOffsetList = ackOffsetList;\n    }\n\n    @Override\n    public String toString() {\n        final StringBuilder sb = new StringBuilder(\"BatchAckMsg{\");\n        sb.append(\"ackOffsetList=\").append(ackOffsetList);\n        sb.append(\", startOffset=\").append(getStartOffset());\n        sb.append(\", consumerGroup='\").append(getConsumerGroup()).append('\\'');\n        sb.append(\", topic='\").append(getTopic()).append('\\'');\n        sb.append(\", queueId=\").append(getQueueId());\n        sb.append(\", popTime=\").append(getPopTime());\n        sb.append(\", brokerName=\").append(getBrokerName());\n        sb.append('}');\n        return sb.toString();\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/pop/PopCheckPoint.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.pop;\n\nimport com.alibaba.fastjson2.annotation.JSONField;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\npublic class PopCheckPoint implements Comparable<PopCheckPoint> {\n    @JSONField(name = \"so\")\n    private long startOffset;\n    @JSONField(name = \"pt\")\n    private long popTime;\n    @JSONField(name = \"it\")\n    private long invisibleTime;\n    @JSONField(name = \"bm\")\n    private int bitMap;\n    @JSONField(name = \"n\")\n    private byte num;\n    @JSONField(name = \"q\")\n    private int queueId;\n    @JSONField(name = \"t\")\n    private String topic;\n    private String cid;\n    @JSONField(name = \"ro\")\n    private long reviveOffset;\n    @JSONField(name = \"d\")\n    private List<Integer> queueOffsetDiff;\n    @JSONField(name = \"bn\")\n    String brokerName;\n    @JSONField(name = \"rp\")\n    String rePutTimes; // ck rePut times\n    @JSONField(name = \"sp\")\n    private boolean suspend; // nack without inc reconsume times, false default.\n\n    public long getReviveOffset() {\n        return reviveOffset;\n    }\n\n    public void setReviveOffset(long reviveOffset) {\n        this.reviveOffset = reviveOffset;\n    }\n\n    public long getStartOffset() {\n        return startOffset;\n    }\n\n    public void setStartOffset(long startOffset) {\n        this.startOffset = startOffset;\n    }\n\n    public void setPopTime(long popTime) {\n        this.popTime = popTime;\n    }\n\n    public void setInvisibleTime(long invisibleTime) {\n        this.invisibleTime = invisibleTime;\n    }\n\n    public long getPopTime() {\n        return popTime;\n    }\n\n    public long getInvisibleTime() {\n        return invisibleTime;\n    }\n\n    public long getReviveTime() {\n        return popTime + invisibleTime;\n    }\n\n    public int getBitMap() {\n        return bitMap;\n    }\n\n    public void setBitMap(int bitMap) {\n        this.bitMap = bitMap;\n    }\n\n    public byte getNum() {\n        return num;\n    }\n\n    public void setNum(byte num) {\n        this.num = num;\n    }\n\n    public int getQueueId() {\n        return queueId;\n    }\n\n    public void setQueueId(int queueId) {\n        this.queueId = queueId;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    @JSONField(name = \"c\")\n    public String getCId() {\n        return cid;\n    }\n\n    @JSONField(name = \"c\")\n    public void setCId(String cid) {\n        this.cid = cid;\n    }\n\n    public List<Integer> getQueueOffsetDiff() {\n        return queueOffsetDiff;\n    }\n\n    public void setQueueOffsetDiff(List<Integer> queueOffsetDiff) {\n        this.queueOffsetDiff = queueOffsetDiff;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public String getRePutTimes() {\n        return rePutTimes;\n    }\n\n    public void setRePutTimes(String rePutTimes) {\n        this.rePutTimes = rePutTimes;\n    }\n\n    public boolean isSuspend() {\n        return suspend;\n    }\n\n    public void setSuspend(boolean suspend) {\n        this.suspend = suspend;\n    }\n\n    public void addDiff(int diff) {\n        if (this.queueOffsetDiff == null) {\n            this.queueOffsetDiff = new ArrayList<>(8);\n        }\n        this.queueOffsetDiff.add(diff);\n    }\n\n    public int indexOfAck(long ackOffset) {\n        if (ackOffset < startOffset) {\n            return -1;\n        }\n\n        // old version of checkpoint\n        if (queueOffsetDiff == null || queueOffsetDiff.isEmpty()) {\n\n            if (ackOffset - startOffset < num) {\n                return (int) (ackOffset - startOffset);\n            }\n\n            return -1;\n        }\n\n        // new version of checkpoint\n        return queueOffsetDiff.indexOf((int) (ackOffset - startOffset));\n    }\n\n    public long ackOffsetByIndex(byte index) {\n        // old version of checkpoint\n        if (queueOffsetDiff == null || queueOffsetDiff.isEmpty()) {\n            return startOffset + index;\n        }\n\n        return startOffset + queueOffsetDiff.get(index);\n    }\n\n    public int parseRePutTimes() {\n        if (null == rePutTimes) {\n            return 0;\n        }\n        try {\n            return Integer.parseInt(rePutTimes);\n        } catch (Exception e) {\n        }\n        return Byte.MAX_VALUE;\n    }\n\n    @Override\n    public String toString() {\n        return \"PopCheckPoint [topic=\" + topic + \", cid=\" + cid + \", queueId=\" + queueId + \", startOffset=\" + startOffset + \", bitMap=\" + bitMap + \", num=\" + num + \", reviveTime=\" + getReviveTime()\n            + \", reviveOffset=\" + reviveOffset + \", diff=\" + queueOffsetDiff + \", brokerName=\" + brokerName + \", rePutTimes=\" + rePutTimes + \", suspend=\" + suspend + \"]\";\n    }\n\n    @Override\n    public int compareTo(PopCheckPoint o) {\n        return (int) (this.getStartOffset() - o.getStartOffset());\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.queue;\n\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.rocksdb.RocksDBException;\n\npublic abstract class AbstractConsumeQueueStore implements ConsumeQueueStoreInterface {\n    protected static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n\n    protected final DefaultMessageStore messageStore;\n    protected final MessageStoreConfig messageStoreConfig;\n    protected final QueueOffsetOperator queueOffsetOperator = new QueueOffsetOperator();\n    protected final ConcurrentMap<String/* topic */, ConcurrentMap<Integer/* queueId */, ConsumeQueueInterface>> consumeQueueTable;\n\n    public AbstractConsumeQueueStore(DefaultMessageStore messageStore) {\n        this.messageStore = messageStore;\n        this.messageStoreConfig = messageStore.getMessageStoreConfig();\n        if (messageStoreConfig.isEnableLmq()) {\n            this.consumeQueueTable = new ConcurrentHashMap<>(32_768);\n        } else {\n            this.consumeQueueTable = new ConcurrentHashMap<>(32);\n        }\n    }\n\n    public void putMessagePositionInfoWrapper(ConsumeQueueInterface consumeQueue, DispatchRequest request) {\n        consumeQueue.putMessagePositionInfoWrapper(request);\n    }\n\n    @Override\n    public Long getMaxOffset(String topic, int queueId) throws ConsumeQueueException {\n        return this.queueOffsetOperator.currentQueueOffset(topic + \"-\" + queueId);\n    }\n\n    public void setTopicQueueTable(ConcurrentMap<String, Long> topicQueueTable) {\n        this.queueOffsetOperator.setTopicQueueTable(topicQueueTable);\n        this.queueOffsetOperator.setLmqTopicQueueTable(topicQueueTable);\n    }\n\n    @Override\n    public void assignQueueOffset(MessageExtBrokerInner msg) throws RocksDBException {\n        ConsumeQueueInterface consumeQueue = findOrCreateConsumeQueue(msg.getTopic(), msg.getQueueId());\n        consumeQueue.assignQueueOffset(this.queueOffsetOperator, msg);\n    }\n\n    @Override\n    public void increaseQueueOffset(MessageExtBrokerInner msg, short messageNum) {\n        ConsumeQueueInterface consumeQueue = findOrCreateConsumeQueue(msg.getTopic(), msg.getQueueId());\n        consumeQueue.increaseQueueOffset(this.queueOffsetOperator, msg, messageNum);\n    }\n\n    @Override\n    public void increaseLmqOffset(String topic, int queueId, short delta) throws ConsumeQueueException {\n        queueOffsetOperator.increaseLmqOffset(topic, queueId, delta);\n    }\n\n    @Override\n    public long getLmqQueueOffset(String topic, int queueId) throws ConsumeQueueException {\n        return queueOffsetOperator.getLmqOffset(topic, queueId, (t, q) -> 0L);\n    }\n\n    public void removeTopicQueueTable(String topic, Integer queueId) {\n        this.queueOffsetOperator.remove(topic, queueId);\n    }\n\n    @Override\n    public ConcurrentMap<String, ConcurrentMap<Integer, ConsumeQueueInterface>> getConsumeQueueTable() {\n        return this.consumeQueueTable;\n    }\n\n    public long getStoreTime(CqUnit cqUnit) {\n        if (cqUnit != null) {\n            try {\n                final long phyOffset = cqUnit.getPos();\n                final int size = cqUnit.getSize();\n                return this.messageStore.getCommitLog().pickupStoreTimestamp(phyOffset, size);\n            } catch (Exception e) {\n                log.error(\"Failed to getStoreTime\", e);\n            }\n        }\n        return -1;\n    }\n\n    /**\n     * get max physic offset in consumeQueue\n     *\n     * @return the max physic offset in consumeQueue\n     * @throws RocksDBException only in rocksdb mode\n     */\n    public abstract long getMaxPhyOffsetInConsumeQueue() throws RocksDBException;\n\n    /**\n     * destroy the specific consumeQueue\n     *\n     * @param consumeQueue consumeQueue to be destroyed\n     * @throws RocksDBException only in rocksdb mode\n     */\n    protected abstract void destroy(ConsumeQueueInterface consumeQueue) throws RocksDBException;\n\n    @Override\n    public boolean deleteTopic(String topic) {\n        ConcurrentMap<Integer, ConsumeQueueInterface> queueTable = this.consumeQueueTable.get(topic);\n\n        if (queueTable == null || queueTable.isEmpty()) {\n            return false;\n        }\n\n        for (ConsumeQueueInterface cq : queueTable.values()) {\n            try {\n                destroy(cq);\n            } catch (RocksDBException e) {\n                log.error(\"DeleteTopic: ConsumeQueue cleans error!, topic={}, queueId={}\", cq.getTopic(), cq.getQueueId(), e);\n            }\n            log.info(\"DeleteTopic: ConsumeQueue has been cleaned, topic={}, queueId={}\", cq.getTopic(), cq.getQueueId());\n            removeTopicQueueTable(cq.getTopic(), cq.getQueueId());\n        }\n\n        // remove topic from cq table\n        this.consumeQueueTable.remove(topic);\n        return true;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.queue;\n\nimport java.io.File;\nimport java.nio.ByteBuffer;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentSkipListMap;\nimport java.util.function.Function;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.MappedFileQueue;\nimport org.apache.rocketmq.store.MessageFilter;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.logfile.MappedFile;\n\npublic class BatchConsumeQueue implements ConsumeQueueInterface {\n    protected static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n\n    /**\n     * BatchConsumeQueue's store unit. Format:\n     * <pre>\n     * ┌─────────────────────────┬───────────┬────────────┬──────────┬─────────────┬─────────┬───────────────┬─────────┐\n     * │CommitLog Physical Offset│ Body Size │Tag HashCode│Store time│msgBaseOffset│batchSize│compactedOffset│reserved │\n     * │        (8 Bytes)        │ (4 Bytes) │ (8 Bytes)  │(8 Bytes) │(8 Bytes)    │(2 Bytes)│   (4 Bytes)   │(4 Bytes)│\n     * ├─────────────────────────┴───────────┴────────────┴──────────┴─────────────┴─────────┴───────────────┴─────────┤\n     * │                                                  Store Unit                                                   │\n     * │                                                                                                               │\n     * </pre>\n     * BatchConsumeQueue's store unit. Size:\n     * CommitLog Physical Offset(8) + Body Size(4) + Tag HashCode(8) + Store time(8) +\n     * msgBaseOffset(8) + batchSize(2) + compactedOffset(4) + reserved(4)= 46 Bytes\n     */\n    public static final int CQ_STORE_UNIT_SIZE = 46;\n    public static final int MSG_TAG_OFFSET_INDEX = 12;\n    public static final int MSG_STORE_TIME_OFFSET_INDEX = 20;\n    public static final int MSG_BASE_OFFSET_INDEX = 28;\n    public static final int MSG_BATCH_SIZE_INDEX = 36;\n    public static final int MSG_COMPACT_OFFSET_INDEX = 38;\n    private static final int MSG_COMPACT_OFFSET_LENGTH = 4;\n    public static final int INVALID_POS = -1;\n    protected final MappedFileQueue mappedFileQueue;\n    protected MessageStore messageStore;\n    protected ConsumeQueueStore consumeQueueStore;\n    protected final String topic;\n    protected final int queueId;\n    protected final ByteBuffer byteBufferItem;\n\n    protected final String storePath;\n    protected final int mappedFileSize;\n    protected volatile long maxMsgPhyOffsetInCommitLog = -1;\n\n    protected volatile long minLogicOffset = 0;\n\n    protected volatile long maxOffsetInQueue = 0;\n    protected volatile long minOffsetInQueue = -1;\n    protected final int commitLogSize;\n\n    protected ConcurrentSkipListMap<Long, MappedFile> offsetCache = new ConcurrentSkipListMap<>();\n    protected ConcurrentSkipListMap<Long, MappedFile> timeCache = new ConcurrentSkipListMap<>();\n\n    public BatchConsumeQueue(\n        final String topic,\n        final int queueId,\n        final String storePath,\n        final int mappedFileSize,\n        final MessageStore messageStore,\n        final ConsumeQueueStore consumeQueueStore,\n        final String subfolder) {\n        this.storePath = storePath;\n        this.mappedFileSize = mappedFileSize;\n        this.messageStore = messageStore;\n        this.consumeQueueStore = consumeQueueStore;\n        this.commitLogSize = messageStore.getCommitLog().getCommitLogSize();\n\n        this.topic = topic;\n        this.queueId = queueId;\n\n        boolean writeWithoutMmap = false;\n        if (messageStore.getMessageStoreConfig() != null) {\n            writeWithoutMmap = messageStore.getMessageStoreConfig().isWriteWithoutMmap();\n        }\n\n        if (StringUtils.isBlank(subfolder)) {\n            String queueDir = this.storePath + File.separator + topic + File.separator + queueId;\n            this.mappedFileQueue = new MappedFileQueue(queueDir, mappedFileSize, null,\n                writeWithoutMmap);\n        } else {\n            String queueDir = this.storePath + File.separator + topic + File.separator + queueId + File.separator + subfolder;\n            this.mappedFileQueue = new MappedFileQueue(queueDir, mappedFileSize, null,\n                writeWithoutMmap);\n        }\n\n        this.byteBufferItem = ByteBuffer.allocate(CQ_STORE_UNIT_SIZE);\n    }\n\n    public BatchConsumeQueue(\n        final String topic,\n        final int queueId,\n        final String storePath,\n        final int mappedFileSize,\n        final MessageStore messageStore,\n        final String subfolder) {\n        this(topic, queueId, storePath, mappedFileSize, messageStore, (ConsumeQueueStore) messageStore.getQueueStore(), subfolder);\n    }\n\n    public BatchConsumeQueue(\n        final String topic,\n        final int queueId,\n        final String storePath,\n        final int mappedFileSize,\n        final MessageStore defaultMessageStore) {\n        this(topic, queueId, storePath, mappedFileSize, defaultMessageStore, StringUtils.EMPTY);\n    }\n\n    @Override\n    public boolean load() {\n        boolean result = this.mappedFileQueue.load();\n        log.info(\"Load batch consume queue {}-{} {} {}\", topic, queueId, result ? \"OK\" : \"Failed\", mappedFileQueue.getMappedFiles().size());\n        return result;\n    }\n\n    protected void doRefreshCache(Function<MappedFile, BatchOffsetIndex> offsetFunction) {\n        if (!this.messageStore.getMessageStoreConfig().isSearchBcqByCacheEnable()) {\n            return;\n        }\n        ConcurrentSkipListMap<Long, MappedFile> newOffsetCache = new ConcurrentSkipListMap<>();\n        ConcurrentSkipListMap<Long, MappedFile> newTimeCache = new ConcurrentSkipListMap<>();\n\n        List<MappedFile> mappedFiles = mappedFileQueue.getMappedFiles();\n        // iterate all BCQ files\n        for (int i = 0; i < mappedFiles.size(); i++) {\n            MappedFile bcq = mappedFiles.get(i);\n            if (isNewFile(bcq)) {\n                continue;\n            }\n\n            BatchOffsetIndex offset = offsetFunction.apply(bcq);\n            if (offset == null) {\n                continue;\n            }\n            newOffsetCache.put(offset.getMsgOffset(), offset.getMappedFile());\n            newTimeCache.put(offset.getStoreTimestamp(), offset.getMappedFile());\n        }\n\n        this.offsetCache = newOffsetCache;\n        this.timeCache = newTimeCache;\n\n        log.info(\"refreshCache for BCQ [Topic: {}, QueueId: {}].\" +\n                \"offsetCacheSize: {}, minCachedMsgOffset: {}, maxCachedMsgOffset: {}, \" +\n                \"timeCacheSize: {}, minCachedTime: {}, maxCachedTime: {}\", this.topic, this.queueId,\n            this.offsetCache.size(), this.offsetCache.firstEntry(), this.offsetCache.lastEntry(),\n            this.timeCache.size(), this.timeCache.firstEntry(), this.timeCache.lastEntry());\n    }\n\n    protected void refreshCache() {\n        doRefreshCache(m -> getMinMsgOffset(m, false, true));\n    }\n\n    private void destroyCache() {\n        this.offsetCache.clear();\n        this.timeCache.clear();\n\n        log.info(\"BCQ [Topic: {}, QueueId: {}]. Cache destroyed\", this.topic, this.queueId);\n    }\n\n    protected void cacheBcq(MappedFile bcq) {\n        try {\n            BatchOffsetIndex min = getMinMsgOffset(bcq, false, true);\n            this.offsetCache.put(min.getMsgOffset(), min.getMappedFile());\n            this.timeCache.put(min.getStoreTimestamp(), min.getMappedFile());\n        } catch (Exception e) {\n            log.error(\"Failed caching offset and time on BCQ [Topic: {}, QueueId: {}, File: {}]\", this.topic, this.queueId, bcq);\n        }\n    }\n\n    protected boolean isNewFile(MappedFile mappedFile) {\n        return mappedFile.getReadPosition() < CQ_STORE_UNIT_SIZE;\n    }\n\n    protected MappedFile searchOffsetFromCache(long msgOffset) {\n        Map.Entry<Long, MappedFile> floorEntry = this.offsetCache.floorEntry(msgOffset);\n        if (floorEntry == null) {\n            // the offset is too small.\n            return null;\n        } else {\n            return floorEntry.getValue();\n        }\n    }\n\n    private MappedFile searchTimeFromCache(long time) {\n        Map.Entry<Long, MappedFile> floorEntry = this.timeCache.floorEntry(time);\n        if (floorEntry == null) {\n            // the timestamp is too small. so we decide to result first BCQ file.\n            return this.mappedFileQueue.getFirstMappedFile();\n        } else {\n            return floorEntry.getValue();\n        }\n    }\n\n    @Override\n    public void recover() {\n        final List<MappedFile> mappedFiles = this.mappedFileQueue.getMappedFiles();\n        if (!mappedFiles.isEmpty()) {\n            int index = mappedFiles.size() - 3;\n            if (index < 0)\n                index = 0;\n\n            int mappedFileSizeLogics = this.mappedFileSize;\n            MappedFile mappedFile = mappedFiles.get(index);\n            ByteBuffer byteBuffer = mappedFile.sliceByteBuffer();\n            long processOffset = mappedFile.getFileFromOffset();\n            long mappedFileOffset = 0;\n            while (true) {\n                for (int i = 0; i < mappedFileSizeLogics; i += CQ_STORE_UNIT_SIZE) {\n                    byteBuffer.position(i);\n                    long offset = byteBuffer.getLong();\n                    int size = byteBuffer.getInt();\n                    byteBuffer.getLong();//tagscode\n                    byteBuffer.getLong();//timestamp\n                    long msgBaseOffset = byteBuffer.getLong();\n                    short batchSize = byteBuffer.getShort();\n                    if (offset >= 0 && size > 0 && msgBaseOffset >= 0 && batchSize > 0) {\n                        mappedFileOffset = i + CQ_STORE_UNIT_SIZE;\n                        this.maxMsgPhyOffsetInCommitLog = offset;\n                    } else {\n                        log.info(\"Recover current batch consume queue file over, file:{} offset:{} size:{} msgBaseOffset:{} batchSize:{} mappedFileOffset:{}\",\n                            mappedFile.getFileName(), offset, size, msgBaseOffset, batchSize, mappedFileOffset);\n                        break;\n                    }\n                }\n\n                if (mappedFileOffset == mappedFileSizeLogics) {\n                    index++;\n                    if (index >= mappedFiles.size()) {\n                        log.info(\"Recover last batch consume queue file over, last mapped file:{} \", mappedFile.getFileName());\n                        break;\n                    } else {\n                        mappedFile = mappedFiles.get(index);\n                        byteBuffer = mappedFile.sliceByteBuffer();\n                        processOffset = mappedFile.getFileFromOffset();\n                        mappedFileOffset = 0;\n                        log.info(\"Recover next batch consume queue file: \" + mappedFile.getFileName());\n                    }\n                } else {\n                    log.info(\"Recover current batch consume queue file over:{} processOffset:{}\", mappedFile.getFileName(), processOffset + mappedFileOffset);\n                    break;\n                }\n            }\n            processOffset += mappedFileOffset;\n            this.mappedFileQueue.setFlushedWhere(processOffset);\n            this.mappedFileQueue.setCommittedWhere(processOffset);\n            this.mappedFileQueue.truncateDirtyFiles(processOffset);\n            reviseMaxAndMinOffsetInQueue();\n        }\n    }\n\n    void reviseMinOffsetInQueue() {\n        MappedFile firstMappedFile = this.mappedFileQueue.getFirstMappedFile();\n        if (null == firstMappedFile) {\n            maxOffsetInQueue = 0;\n            minOffsetInQueue = -1;\n            minLogicOffset = -1;\n            log.info(\"reviseMinOffsetInQueue found firstMappedFile null, topic:{} queue:{}\", topic, queueId);\n            return;\n        }\n        minLogicOffset = firstMappedFile.getFileFromOffset();\n        BatchOffsetIndex min = getMinMsgOffset(firstMappedFile, false, false);\n        minOffsetInQueue = null == min ? -1 : min.getMsgOffset();\n    }\n\n    void reviseMaxOffsetInQueue() {\n        MappedFile lastMappedFile = this.mappedFileQueue.getLastMappedFile();\n        BatchOffsetIndex max = getMaxMsgOffset(lastMappedFile, true, false);\n        if (null == max && this.mappedFileQueue.getMappedFiles().size() >= 2) {\n            MappedFile lastTwoMappedFile = this.mappedFileQueue.getMappedFiles().get(this.mappedFileQueue.getMappedFiles().size() - 2);\n            max = getMaxMsgOffset(lastTwoMappedFile, true, false);\n        }\n        maxOffsetInQueue = (null == max) ? 0 : max.getMsgOffset() + max.getBatchSize();\n    }\n\n    void reviseMaxAndMinOffsetInQueue() {\n        reviseMinOffsetInQueue();\n        reviseMaxOffsetInQueue();\n    }\n\n    @Override\n    public long getMaxPhysicOffset() {\n        return maxMsgPhyOffsetInCommitLog;\n    }\n\n    @Override\n    public long getMinLogicOffset() {\n        return minLogicOffset;\n    }\n\n    @Override\n    public ReferredIterator<CqUnit> iterateFrom(long startOffset) {\n        SelectMappedBufferResult sbr = getBatchMsgIndexBuffer(startOffset);\n        if (sbr == null) {\n            return null;\n        }\n        return new BatchConsumeQueueIterator(sbr);\n    }\n\n    @Override\n    public ReferredIterator<CqUnit> iterateFrom(long startIndex, int count) {\n        return iterateFrom(startIndex);\n    }\n\n    @Override\n    public CqUnit get(long offset) {\n        ReferredIterator<CqUnit> it = iterateFrom(offset);\n        if (it == null) {\n            return null;\n        }\n        return it.nextAndRelease();\n    }\n\n    @Override\n    public Pair<CqUnit, Long> getCqUnitAndStoreTime(long index) {\n        CqUnit cqUnit = get(index);\n        Long messageStoreTime = this.consumeQueueStore.getStoreTime(cqUnit);\n        return new Pair<>(cqUnit, messageStoreTime);\n    }\n\n    @Override\n    public Pair<CqUnit, Long> getEarliestUnitAndStoreTime() {\n        CqUnit cqUnit = getEarliestUnit();\n        Long messageStoreTime = this.consumeQueueStore.getStoreTime(cqUnit);\n        return new Pair<>(cqUnit, messageStoreTime);\n    }\n\n    @Override\n    public CqUnit getEarliestUnit() {\n        return get(minOffsetInQueue);\n    }\n\n    @Override\n    public CqUnit getLatestUnit() {\n        return get(maxOffsetInQueue - 1);\n    }\n\n    @Override\n    public long getLastOffset() {\n        CqUnit latestUnit = getLatestUnit();\n        return latestUnit.getPos() + latestUnit.getSize();\n    }\n\n    @Override\n    public boolean isFirstFileAvailable() {\n        MappedFile mappedFile = this.mappedFileQueue.getFirstMappedFile();\n        if (mappedFile != null) {\n            return mappedFile.isAvailable();\n        }\n        return false;\n    }\n\n    @Override\n    public boolean isFirstFileExist() {\n        MappedFile mappedFile = this.mappedFileQueue.getFirstMappedFile();\n        return mappedFile != null;\n    }\n\n    @Override\n    public void truncateDirtyLogicFiles(long phyOffset) {\n\n        long oldMinOffset = minOffsetInQueue;\n        long oldMaxOffset = maxOffsetInQueue;\n\n        int logicFileSize = this.mappedFileSize;\n\n        this.maxMsgPhyOffsetInCommitLog = phyOffset - 1;\n        boolean stop = false;\n        while (!stop) {\n            MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();\n            if (mappedFile != null) {\n                ByteBuffer byteBuffer = mappedFile.sliceByteBuffer();\n\n                mappedFile.setWrotePosition(0);\n                mappedFile.setCommittedPosition(0);\n                mappedFile.setFlushedPosition(0);\n\n                for (int i = 0; i < logicFileSize; i += CQ_STORE_UNIT_SIZE) {\n                    byteBuffer.position(i);\n                    long offset = byteBuffer.getLong();\n                    int size = byteBuffer.getInt();\n                    byteBuffer.getLong();//tagscode\n                    byteBuffer.getLong();//timestamp\n                    long msgBaseOffset = byteBuffer.getLong();\n                    short batchSize = byteBuffer.getShort();\n\n                    if (0 == i) {\n                        if (offset >= phyOffset) {\n                            this.mappedFileQueue.deleteLastMappedFile();\n                            break;\n                        } else {\n                            int pos = i + CQ_STORE_UNIT_SIZE;\n                            mappedFile.setWrotePosition(pos);\n                            mappedFile.setCommittedPosition(pos);\n                            mappedFile.setFlushedPosition(pos);\n                            this.maxMsgPhyOffsetInCommitLog = offset;\n                        }\n                    } else {\n                        if (offset >= 0 && size > 0 && msgBaseOffset >= 0 && batchSize > 0) {\n                            if (offset >= phyOffset) {\n                                stop = true;\n                                break;\n                            }\n\n                            int pos = i + CQ_STORE_UNIT_SIZE;\n                            mappedFile.setWrotePosition(pos);\n                            mappedFile.setCommittedPosition(pos);\n                            mappedFile.setFlushedPosition(pos);\n                            this.maxMsgPhyOffsetInCommitLog = offset;\n                            if (pos == logicFileSize) {\n                                stop = true;\n                                break;\n                            }\n                        } else {\n                            stop = true;\n                            break;\n                        }\n                    }\n                }\n            } else {\n                break;\n            }\n        }\n        reviseMaxAndMinOffsetInQueue();\n        log.info(\"Truncate batch logic file topic={} queue={} oldMinOffset={} oldMaxOffset={} minOffset={} maxOffset={} maxPhyOffsetHere={} maxPhyOffsetThere={}\",\n            topic, queueId, oldMinOffset, oldMaxOffset, minOffsetInQueue, maxOffsetInQueue, maxMsgPhyOffsetInCommitLog, phyOffset);\n    }\n\n    @Override\n    public boolean flush(final int flushLeastPages) {\n        boolean result = this.mappedFileQueue.flush(flushLeastPages);\n        return result;\n    }\n\n    @Override\n    public int deleteExpiredFile(long minCommitLogPos) {\n        int cnt = this.mappedFileQueue.deleteExpiredFileByOffset(minCommitLogPos, CQ_STORE_UNIT_SIZE);\n        this.correctMinOffset(minCommitLogPos);\n        return cnt;\n    }\n\n    @Override\n    public void correctMinOffset(long phyMinOffset) {\n        reviseMinOffsetInQueue();\n        refreshCache();\n        long oldMinOffset = minOffsetInQueue;\n        MappedFile mappedFile = this.mappedFileQueue.getFirstMappedFile();\n        if (mappedFile != null) {\n            SelectMappedBufferResult result = mappedFile.selectMappedBuffer(0);\n            if (result != null) {\n                try {\n                    int startPos = result.getByteBuffer().position();\n                    for (int i = 0; i < result.getSize(); i += BatchConsumeQueue.CQ_STORE_UNIT_SIZE) {\n                        result.getByteBuffer().position(startPos + i);\n                        long offsetPy = result.getByteBuffer().getLong();\n                        result.getByteBuffer().getInt(); //size\n                        result.getByteBuffer().getLong();//tagscode\n                        result.getByteBuffer().getLong();//timestamp\n                        long msgBaseOffset = result.getByteBuffer().getLong();\n                        short batchSize = result.getByteBuffer().getShort();\n\n                        if (offsetPy < phyMinOffset) {\n                            this.minOffsetInQueue = msgBaseOffset + batchSize;\n                        } else {\n                            break;\n                        }\n                    }\n                } catch (Exception e) {\n                    log.error(\"Exception thrown when correctMinOffset\", e);\n                } finally {\n                    result.release();\n                }\n            } else {\n                /**\n                 *  It will go to here under two conditions:\n                 1. the files number is 1, and it has no data\n                 2. the pull process hold the cq reference, and release it just the moment\n                 */\n                log.warn(\"Correct min offset found null cq file topic:{} queue:{} files:{} minOffset:{} maxOffset:{}\",\n                    topic, queueId, this.mappedFileQueue.getMappedFiles().size(), minOffsetInQueue, maxOffsetInQueue);\n            }\n        }\n        if (oldMinOffset != this.minOffsetInQueue) {\n            log.info(\"BatchCQ Compute new minOffset:{} oldMinOffset{} topic:{} queue:{}\", minOffsetInQueue, oldMinOffset, topic, queueId);\n        }\n    }\n\n    @Override\n    public void putMessagePositionInfoWrapper(DispatchRequest request) {\n        final int maxRetries = 30;\n        boolean canWrite = this.messageStore.getRunningFlags().isCQWriteable();\n        if (request.getMsgBaseOffset() < 0 || request.getBatchSize() < 0) {\n            log.warn(\"[NOTIFYME]unexpected dispatch request in batch consume queue topic:{} queue:{} offset:{}\", topic, queueId, request.getCommitLogOffset());\n            return;\n        }\n        for (int i = 0; i < maxRetries && canWrite; i++) {\n            boolean result = this.putBatchMessagePositionInfo(request.getCommitLogOffset(),\n                request.getMsgSize(), request.getTagsCode(),\n                request.getStoreTimestamp(), request.getMsgBaseOffset(), request.getBatchSize());\n            if (result) {\n                if (BrokerRole.SLAVE == this.messageStore.getMessageStoreConfig().getBrokerRole()) {\n                    this.messageStore.getStoreCheckpoint().setPhysicMsgTimestamp(request.getStoreTimestamp());\n                }\n                this.messageStore.getStoreCheckpoint().setTmpLogicsMsgTimestamp(request.getStoreTimestamp());\n                this.messageStore.getStoreCheckpoint().setTmpLogicsPhysicalOffset(request.getCommitLogOffset());\n                return;\n            } else {\n                // XXX: warn and notify me\n                log.warn(\"[NOTIFYME]put commit log position info to batch consume queue \" + topic + \":\" + queueId + \" \" + request.getCommitLogOffset()\n                    + \" failed, retry \" + i + \" times\");\n\n                try {\n                    Thread.sleep(1000);\n                } catch (InterruptedException e) {\n                    log.warn(\"\", e);\n                }\n            }\n        }\n        // XXX: warn and notify me\n        log.error(\"[NOTIFYME]batch consume queue can not write, {} {}\", this.topic, this.queueId);\n        this.messageStore.getRunningFlags().makeLogicsQueueError();\n    }\n\n    @Override\n    public void assignQueueOffset(QueueOffsetOperator queueOffsetOperator, MessageExtBrokerInner msg) {\n        String topicQueueKey = getTopic() + \"-\" + getQueueId();\n\n        long queueOffset = queueOffsetOperator.getBatchQueueOffset(topicQueueKey);\n\n        if (MessageSysFlag.check(msg.getSysFlag(), MessageSysFlag.INNER_BATCH_FLAG)) {\n            MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_BASE, String.valueOf(queueOffset));\n            msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n        }\n        msg.setQueueOffset(queueOffset);\n    }\n\n    @Override\n    public void increaseQueueOffset(QueueOffsetOperator queueOffsetOperator, MessageExtBrokerInner msg,\n        short messageNum) {\n        String topicQueueKey = getTopic() + \"-\" + getQueueId();\n        queueOffsetOperator.increaseBatchQueueOffset(topicQueueKey, messageNum);\n    }\n\n    public boolean putBatchMessagePositionInfo(final long offset, final int size, final long tagsCode,\n        final long storeTime,\n        final long msgBaseOffset, final short batchSize) {\n\n        if (offset <= this.maxMsgPhyOffsetInCommitLog) {\n            if (System.currentTimeMillis() % 1000 == 0) {\n                log.warn(\"Build batch consume queue repeatedly, maxMsgPhyOffsetInCommitLog:{} offset:{} Topic: {} QID: {}\",\n                    maxMsgPhyOffsetInCommitLog, offset, this.topic, this.queueId);\n            }\n            return true;\n        }\n\n        long behind = System.currentTimeMillis() - storeTime;\n        if (behind > 10000 && System.currentTimeMillis() % 10000 == 0) {\n            String flag = \"LEVEL\" + (behind / 10000);\n            log.warn(\"Reput behind {} topic:{} queue:{} offset:{} behind:{}\", flag, topic, queueId, offset, behind);\n        }\n\n        this.byteBufferItem.flip();\n        this.byteBufferItem.limit(CQ_STORE_UNIT_SIZE);\n        this.byteBufferItem.putLong(offset);\n        this.byteBufferItem.putInt(size);\n        this.byteBufferItem.putLong(tagsCode);\n        this.byteBufferItem.putLong(storeTime);\n        this.byteBufferItem.putLong(msgBaseOffset);\n        this.byteBufferItem.putShort(batchSize);\n        this.byteBufferItem.putInt(INVALID_POS);\n        this.byteBufferItem.putInt(0); // 4 bytes reserved\n\n        MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(this.mappedFileQueue.getMaxOffset());\n        if (mappedFile != null) {\n            boolean isNewFile = isNewFile(mappedFile);\n            boolean appendRes;\n            if (messageStore.getMessageStoreConfig().isPutConsumeQueueDataByFileChannel()) {\n                appendRes = mappedFile.appendMessageUsingFileChannel(this.byteBufferItem.array());\n            } else {\n                appendRes = mappedFile.appendMessage(this.byteBufferItem.array());\n            }\n            if (appendRes) {\n                maxMsgPhyOffsetInCommitLog = offset;\n                maxOffsetInQueue = msgBaseOffset + batchSize;\n                //only the first time need to correct the minOffsetInQueue\n                //the other correctness is done in correctLogicMinoffsetService\n                if (mappedFile.isFirstCreateInQueue() && minOffsetInQueue == -1) {\n                    reviseMinOffsetInQueue();\n                }\n                if (isNewFile) {\n                    // cache new file\n                    this.cacheBcq(mappedFile);\n                }\n            }\n            return appendRes;\n        }\n        return false;\n    }\n\n    protected BatchOffsetIndex getMinMsgOffset(MappedFile mappedFile, boolean getBatchSize, boolean getStoreTime) {\n        if (mappedFile.getReadPosition() < CQ_STORE_UNIT_SIZE) {\n            return null;\n        }\n        return getBatchOffsetIndexByPos(mappedFile, 0, getBatchSize, getStoreTime);\n    }\n\n    protected BatchOffsetIndex getBatchOffsetIndexByPos(MappedFile mappedFile, int pos, boolean getBatchSize,\n        boolean getStoreTime) {\n        SelectMappedBufferResult sbr = mappedFile.selectMappedBuffer(pos);\n        try {\n            return new BatchOffsetIndex(mappedFile, pos, sbr.getByteBuffer().getLong(MSG_BASE_OFFSET_INDEX),\n                getBatchSize ? sbr.getByteBuffer().getShort(MSG_BATCH_SIZE_INDEX) : 0,\n                getStoreTime ? sbr.getByteBuffer().getLong(MSG_STORE_TIME_OFFSET_INDEX) : 0);\n        } finally {\n            if (sbr != null) {\n                sbr.release();\n            }\n        }\n    }\n\n    protected BatchOffsetIndex getMaxMsgOffset(MappedFile mappedFile, boolean getBatchSize, boolean getStoreTime) {\n        if (mappedFile == null || mappedFile.getReadPosition() < CQ_STORE_UNIT_SIZE) {\n            return null;\n        }\n        int pos = mappedFile.getReadPosition() - CQ_STORE_UNIT_SIZE;\n        return getBatchOffsetIndexByPos(mappedFile, pos, getBatchSize, getStoreTime);\n    }\n\n    private static int ceil(int pos) {\n        return (pos / CQ_STORE_UNIT_SIZE) * CQ_STORE_UNIT_SIZE;\n    }\n\n    /**\n     * Gets SelectMappedBufferResult by batch-message offset\n     * Node: the caller is responsible for the release of SelectMappedBufferResult\n     * @param msgOffset\n     * @return SelectMappedBufferResult\n     */\n    public SelectMappedBufferResult getBatchMsgIndexBuffer(final long msgOffset) {\n        if (msgOffset >= maxOffsetInQueue) {\n            return null;\n        }\n        MappedFile targetBcq;\n        BatchOffsetIndex targetMinOffset;\n\n        // first check the last bcq file\n        MappedFile lastBcq = mappedFileQueue.getLastMappedFile();\n        BatchOffsetIndex minForLastBcq = getMinMsgOffset(lastBcq, false, false);\n        if (null != minForLastBcq && minForLastBcq.getMsgOffset() <= msgOffset) {\n            // found, it's the last bcq.\n            targetBcq = lastBcq;\n            targetMinOffset = minForLastBcq;\n        } else {\n            boolean searchBcqByCacheEnable = this.messageStore.getMessageStoreConfig().isSearchBcqByCacheEnable();\n            if (searchBcqByCacheEnable) {\n                // it's not the last BCQ file, so search it through cache.\n                targetBcq = this.searchOffsetFromCache(msgOffset);\n                // not found in cache\n                if (targetBcq == null) {\n                    MappedFile firstBcq = mappedFileQueue.getFirstMappedFile();\n                    BatchOffsetIndex minForFirstBcq = getMinMsgOffset(firstBcq, false, false);\n                    if (minForFirstBcq != null && minForFirstBcq.getMsgOffset() <= msgOffset && msgOffset < minForLastBcq.getMsgOffset()) {\n                        // old search logic\n                        targetBcq = this.searchOffsetFromFiles(msgOffset);\n                    }\n                    log.warn(\"cache is not working on BCQ [Topic: {}, QueueId: {}] for msgOffset: {}, targetBcq: {}\", this.topic, this.queueId, msgOffset, targetBcq);\n                }\n            } else {\n                // old search logic\n                targetBcq = this.searchOffsetFromFiles(msgOffset);\n            }\n\n            if (targetBcq == null) {\n                return null;\n            }\n\n            targetMinOffset = getMinMsgOffset(targetBcq, false, false);\n        }\n\n        BatchOffsetIndex targetMaxOffset = getMaxMsgOffset(targetBcq, false, false);\n        if (null == targetMinOffset || null == targetMaxOffset) {\n            return null;\n        }\n\n        // then use binary search to find the indexed position\n        SelectMappedBufferResult sbr = targetMinOffset.getMappedFile().selectMappedBuffer(0);\n        try {\n            ByteBuffer byteBuffer = sbr.getByteBuffer();\n            int left = targetMinOffset.getIndexPos(), right = targetMaxOffset.getIndexPos();\n            int mid = binarySearch(byteBuffer, left, right, CQ_STORE_UNIT_SIZE, MSG_BASE_OFFSET_INDEX, msgOffset);\n            if (mid != -1) {\n                // return a buffer that needs to be released manually.\n                return targetMinOffset.getMappedFile().selectMappedBuffer(mid);\n            }\n        } finally {\n            sbr.release();\n        }\n        return null;\n    }\n\n    public MappedFile searchOffsetFromFiles(long msgOffset) {\n        MappedFile targetBcq = null;\n        // find the mapped file one by one reversely\n        int mappedFileNum = this.mappedFileQueue.getMappedFiles().size();\n        for (int i = mappedFileNum - 1; i >= 0; i--) {\n            MappedFile mappedFile = mappedFileQueue.getMappedFiles().get(i);\n            BatchOffsetIndex tmpMinMsgOffset = getMinMsgOffset(mappedFile, false, false);\n            if (null != tmpMinMsgOffset && tmpMinMsgOffset.getMsgOffset() <= msgOffset) {\n                targetBcq = mappedFile;\n                break;\n            }\n        }\n\n        return targetBcq;\n    }\n\n    /**\n     * Find the message whose timestamp is the smallest, greater than or equal to the given time.\n     *\n     * @param timestamp\n     * @return\n     */\n    @Deprecated\n    @Override\n    public long getOffsetInQueueByTime(final long timestamp) {\n        return getOffsetInQueueByTime(timestamp, BoundaryType.LOWER);\n    }\n\n    @Override\n    public long getOffsetInQueueByTime(long timestamp, BoundaryType boundaryType) {\n        MappedFile targetBcq;\n        BatchOffsetIndex targetMinOffset;\n\n        // first check the last bcq\n        MappedFile lastBcq = mappedFileQueue.getLastMappedFile();\n        BatchOffsetIndex minForLastBcq = getMinMsgOffset(lastBcq, false, true);\n        if (null != minForLastBcq && minForLastBcq.getStoreTimestamp() <= timestamp) {\n            // found, it's the last bcq.\n            targetBcq = lastBcq;\n            targetMinOffset = minForLastBcq;\n        } else {\n            boolean searchBcqByCacheEnable = this.messageStore.getMessageStoreConfig().isSearchBcqByCacheEnable();\n            if (searchBcqByCacheEnable) {\n                // it's not the last BCQ file, so search it through cache.\n                targetBcq = this.searchTimeFromCache(timestamp);\n                if (targetBcq == null) {\n                    // not found in cache\n                    MappedFile firstBcq = mappedFileQueue.getFirstMappedFile();\n                    BatchOffsetIndex minForFirstBcq = getMinMsgOffset(firstBcq, false, true);\n                    if (minForFirstBcq != null && minForFirstBcq.getStoreTimestamp() <= timestamp && timestamp < minForLastBcq.getStoreTimestamp()) {\n                        // old search logic\n                        targetBcq = this.searchTimeFromFiles(timestamp);\n                    }\n                    log.warn(\"cache is not working on BCQ [Topic: {}, QueueId: {}] for timestamp: {}, targetBcq: {}\", this.topic, this.queueId, timestamp, targetBcq);\n                }\n            } else {\n                // old search logic\n                targetBcq = this.searchTimeFromFiles(timestamp);\n            }\n\n            if (targetBcq == null) {\n                return -1;\n            }\n            targetMinOffset = getMinMsgOffset(targetBcq, false, true);\n        }\n\n        BatchOffsetIndex targetMaxOffset = getMaxMsgOffset(targetBcq, false, true);\n        if (null == targetMinOffset || null == targetMaxOffset) {\n            return -1;\n        }\n\n        //then use binary search to find the indexed position\n        SelectMappedBufferResult sbr = targetMinOffset.getMappedFile().selectMappedBuffer(0);\n        try {\n            ByteBuffer byteBuffer = sbr.getByteBuffer();\n            int left = targetMinOffset.getIndexPos(), right = targetMaxOffset.getIndexPos();\n            long maxQueueTimestamp = byteBuffer.getLong(right + MSG_STORE_TIME_OFFSET_INDEX);\n            if (timestamp >= maxQueueTimestamp) {\n                return byteBuffer.getLong(right + MSG_BASE_OFFSET_INDEX);\n            }\n            int mid = binarySearchRight(byteBuffer, left, right, CQ_STORE_UNIT_SIZE, MSG_STORE_TIME_OFFSET_INDEX, timestamp, boundaryType);\n            if (mid != -1) {\n                return byteBuffer.getLong(mid + MSG_BASE_OFFSET_INDEX);\n            }\n        } finally {\n            sbr.release();\n        }\n\n        return -1;\n    }\n\n    private MappedFile searchTimeFromFiles(long timestamp) {\n        MappedFile targetBcq = null;\n\n        int mappedFileNum = this.mappedFileQueue.getMappedFiles().size();\n        for (int i = mappedFileNum - 1; i >= 0; i--) {\n            MappedFile mappedFile = mappedFileQueue.getMappedFiles().get(i);\n            BatchOffsetIndex tmpMinMsgOffset = getMinMsgOffset(mappedFile, false, true);\n            if (tmpMinMsgOffset == null) {\n                //Maybe the new file\n                continue;\n            }\n            BatchOffsetIndex tmpMaxMsgOffset = getMaxMsgOffset(mappedFile, false, true);\n            //Here should not be null\n            if (tmpMaxMsgOffset == null) {\n                break;\n            }\n            if (tmpMaxMsgOffset.getStoreTimestamp() >= timestamp) {\n                if (tmpMinMsgOffset.getStoreTimestamp() <= timestamp) {\n                    targetBcq = mappedFile;\n                    break;\n                } else {\n                    if (i - 1 < 0) {\n                        //This is the first file\n                        targetBcq = mappedFile;\n                        break;\n                    } else {\n                        //The min timestamp of this file is larger than the given timestamp, so check the next file\n                        continue;\n                    }\n                }\n            } else {\n                //The max timestamp of this file is smaller than the given timestamp, so double check the previous file\n                if (i + 1 <= mappedFileNum - 1) {\n                    mappedFile = mappedFileQueue.getMappedFiles().get(i + 1);\n                    targetBcq = mappedFile;\n                    break;\n                } else {\n                    //There is no timestamp larger than the given timestamp\n                    break;\n                }\n            }\n        }\n\n        return targetBcq;\n    }\n\n    /**\n     * Find the offset of which the value is equal or larger than the given targetValue.\n     * If there are many values equal to the target, then return the lowest offset if boundaryType is LOWER while\n     * return the highest offset if boundaryType is UPPER.\n     */\n    public static int binarySearchRight(ByteBuffer byteBuffer, int left, int right, final int unitSize,\n        final int unitShift, long targetValue, BoundaryType boundaryType) {\n        int mid = -1;\n        while (left <= right) {\n            mid = ceil((left + right) / 2);\n            long tmpValue = byteBuffer.getLong(mid + unitShift);\n            if (mid == right) {\n                //Means left and the right are the same\n                if (tmpValue >= targetValue) {\n                    return mid;\n                } else {\n                    return -1;\n                }\n            } else if (mid == left) {\n                //Means the left + unitSize = right\n                if (tmpValue >= targetValue) {\n                    return mid;\n                } else {\n                    left = mid + unitSize;\n                }\n            } else {\n                //mid is actually in the mid\n                switch (boundaryType) {\n                    case LOWER:\n                        if (tmpValue < targetValue) {\n                            left = mid + unitSize;\n                        } else {\n                            right = mid;\n                        }\n                        break;\n                    case UPPER:\n                        if (tmpValue <= targetValue) {\n                            left = mid;\n                        } else {\n                            right = mid - unitSize;\n                        }\n                        break;\n                    default:\n                        log.warn(\"Unknown boundary type\");\n                        return -1;\n                }\n            }\n        }\n        return -1;\n    }\n\n    /**\n     * Here is vulnerable, the min value of the bytebuffer must be smaller or equal then the given value.\n     * Otherwise, it may get -1\n     */\n    protected int binarySearch(ByteBuffer byteBuffer, int left, int right, final int unitSize, final int unitShift,\n        long targetValue) {\n        int maxRight = right;\n        int mid = -1;\n        while (left <= right) {\n            mid = ceil((left + right) / 2);\n            long tmpValue = byteBuffer.getLong(mid + unitShift);\n            if (tmpValue == targetValue) {\n                return mid;\n            }\n            if (tmpValue > targetValue) {\n                right = mid - unitSize;\n            } else {\n                if (mid == left) {\n                    //the binary search is converging to the left, so maybe the one on the right of mid is the exactly correct one\n                    if (mid + unitSize <= maxRight\n                        && byteBuffer.getLong(mid + unitSize + unitShift) <= targetValue) {\n                        return mid + unitSize;\n                    } else {\n                        return mid;\n                    }\n                } else {\n                    left = mid;\n                }\n            }\n        }\n        return -1;\n    }\n\n    static class BatchConsumeQueueIterator implements ReferredIterator<CqUnit> {\n        private SelectMappedBufferResult sbr;\n        private int relativePos = 0;\n\n        public BatchConsumeQueueIterator(SelectMappedBufferResult sbr) {\n            this.sbr = sbr;\n            if (sbr != null && sbr.getByteBuffer() != null) {\n                relativePos = sbr.getByteBuffer().position();\n            }\n        }\n\n        @Override\n        public boolean hasNext() {\n            if (sbr == null || sbr.getByteBuffer() == null) {\n                return false;\n            }\n\n            return sbr.getByteBuffer().hasRemaining();\n        }\n\n        @Override\n        public CqUnit next() {\n            if (!hasNext()) {\n                return null;\n            }\n            ByteBuffer tmpBuffer = sbr.getByteBuffer().slice();\n            tmpBuffer.position(MSG_COMPACT_OFFSET_INDEX);\n            ByteBuffer compactOffsetStoreBuffer = tmpBuffer.slice();\n            compactOffsetStoreBuffer.limit(MSG_COMPACT_OFFSET_LENGTH);\n\n            int relativePos = sbr.getByteBuffer().position();\n            long offsetPy = sbr.getByteBuffer().getLong();\n            int sizePy = sbr.getByteBuffer().getInt();\n            long tagsCode = sbr.getByteBuffer().getLong(); //tagscode\n            sbr.getByteBuffer().getLong();//timestamp\n            long msgBaseOffset = sbr.getByteBuffer().getLong();\n            short batchSize = sbr.getByteBuffer().getShort();\n            int compactedOffset = sbr.getByteBuffer().getInt();\n            sbr.getByteBuffer().position(relativePos + CQ_STORE_UNIT_SIZE);\n\n            return new CqUnit(msgBaseOffset, offsetPy, sizePy, tagsCode, batchSize, compactedOffset, compactOffsetStoreBuffer);\n        }\n\n        @Override\n        public void remove() {\n            throw new UnsupportedOperationException(\"remove\");\n        }\n\n        @Override\n        public void release() {\n            if (sbr != null) {\n                sbr.release();\n                sbr = null;\n            }\n        }\n\n        @Override\n        public CqUnit nextAndRelease() {\n            try {\n                return next();\n            } finally {\n                release();\n            }\n        }\n    }\n\n    @Override\n    public String getTopic() {\n        return topic;\n    }\n\n    @Override\n    public int getQueueId() {\n        return queueId;\n    }\n\n    @Override\n    public CQType getCQType() {\n        return CQType.BatchCQ;\n    }\n\n    @Override\n    public long getTotalSize() {\n        return this.mappedFileQueue.getTotalFileSize();\n    }\n\n    @Override\n    public int getUnitSize() {\n        return CQ_STORE_UNIT_SIZE;\n    }\n\n    @Override\n    public void destroy() {\n        this.maxMsgPhyOffsetInCommitLog = -1;\n        this.minOffsetInQueue = -1;\n        this.maxOffsetInQueue = 0;\n        this.mappedFileQueue.destroy();\n        this.destroyCache();\n    }\n\n    @Override\n    public long getMessageTotalInQueue() {\n        return this.getMaxOffsetInQueue() - this.getMinOffsetInQueue();\n    }\n\n    @Override\n    public long rollNextFile(long nextBeginOffset) {\n        return 0;\n    }\n\n    /**\n     * Batch msg offset (deep logic offset)\n     *\n     * @return max deep offset\n     */\n    @Override\n    public long getMaxOffsetInQueue() {\n        return maxOffsetInQueue;\n    }\n\n    @Override\n    public long getMinOffsetInQueue() {\n        return minOffsetInQueue;\n    }\n\n    @Override\n    public void checkSelf() {\n        mappedFileQueue.checkSelf();\n    }\n\n    @Override\n    public void swapMap(int reserveNum, long forceSwapIntervalMs, long normalSwapIntervalMs) {\n        mappedFileQueue.swapMap(reserveNum, forceSwapIntervalMs, normalSwapIntervalMs);\n    }\n\n    @Override\n    public void cleanSwappedMap(long forceCleanSwapIntervalMs) {\n        mappedFileQueue.cleanSwappedMap(forceCleanSwapIntervalMs);\n    }\n\n    public MappedFileQueue getMappedFileQueue() {\n        return mappedFileQueue;\n    }\n\n    @Override\n    public long estimateMessageCount(long from, long to, MessageFilter filter) {\n        // transfer message offset to physical offset\n        SelectMappedBufferResult firstMappedFileBuffer = getBatchMsgIndexBuffer(from);\n        if (firstMappedFileBuffer == null) {\n            return -1;\n        }\n        long physicalOffsetFrom = firstMappedFileBuffer.getStartOffset();\n\n        SelectMappedBufferResult lastMappedFileBuffer = getBatchMsgIndexBuffer(to);\n        if (lastMappedFileBuffer == null) {\n            return -1;\n        }\n        long physicalOffsetTo = lastMappedFileBuffer.getStartOffset();\n\n        List<MappedFile> mappedFiles = mappedFileQueue.range(physicalOffsetFrom, physicalOffsetTo);\n        if (mappedFiles.isEmpty()) {\n            return -1;\n        }\n\n        boolean sample = false;\n        long match = 0;\n        long matchCqUnitCount = 0;\n        long raw = 0;\n        long scanCqUnitCount = 0;\n\n        for (MappedFile mappedFile : mappedFiles) {\n            int start = 0;\n            int len = mappedFile.getFileSize();\n\n            // calculate start and len for first segment and last segment to reduce scanning\n            // first file segment\n            if (mappedFile.getFileFromOffset() <= physicalOffsetFrom) {\n                start = (int) (physicalOffsetFrom - mappedFile.getFileFromOffset());\n                if (mappedFile.getFileFromOffset() + mappedFile.getFileSize() >= physicalOffsetTo) {\n                    // current mapped file covers search range completely.\n                    len = (int) (physicalOffsetTo - physicalOffsetFrom);\n                } else {\n                    len = mappedFile.getFileSize() - start;\n                }\n            }\n\n            // last file segment\n            if (0 == start && mappedFile.getFileFromOffset() + mappedFile.getFileSize() > physicalOffsetTo) {\n                len = (int) (physicalOffsetTo - mappedFile.getFileFromOffset());\n            }\n\n            // select partial data to scan\n            SelectMappedBufferResult slice = mappedFile.selectMappedBuffer(start, len);\n            if (null != slice) {\n                try {\n                    ByteBuffer buffer = slice.getByteBuffer();\n                    int current = 0;\n                    while (current < len) {\n                        // skip physicalOffset and message length fields.\n                        buffer.position(current + MSG_TAG_OFFSET_INDEX);\n                        long tagCode = buffer.getLong();\n                        buffer.position(current + MSG_BATCH_SIZE_INDEX);\n                        long batchSize = buffer.getShort();\n                        if (filter.isMatchedByConsumeQueue(tagCode, null)) {\n                            match += batchSize;\n                            matchCqUnitCount++;\n                        }\n                        raw += batchSize;\n                        scanCqUnitCount++;\n                        current += CQ_STORE_UNIT_SIZE;\n\n                        if (scanCqUnitCount >= messageStore.getMessageStoreConfig().getMaxConsumeQueueScan()) {\n                            sample = true;\n                            break;\n                        }\n\n                        if (matchCqUnitCount > messageStore.getMessageStoreConfig().getSampleCountThreshold()) {\n                            sample = true;\n                            break;\n                        }\n                    }\n                } finally {\n                    slice.release();\n                }\n            }\n            // we have scanned enough entries, now is the time to return an educated guess.\n            if (sample) {\n                break;\n            }\n        }\n\n        long result = match;\n        if (sample) {\n            if (0 == raw) {\n                log.error(\"[BUG]. Raw should NOT be 0\");\n                return 0;\n            }\n            result = (long) (match * (to - from) * 1.0 / raw);\n        }\n        log.debug(\"Result={}, raw={}, match={}, sample={}\", result, raw, match, sample);\n        return result;\n    }\n\n    @Override\n    public void initializeWithOffset(long offset, long minPhyOffset) {\n        // not support now\n    }\n\n    @Override\n    public boolean shutdown() {\n        return true;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/BatchOffsetIndex.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.queue;\n\nimport org.apache.rocketmq.store.logfile.MappedFile;\n\npublic class BatchOffsetIndex {\n\n    private final MappedFile mappedFile;\n    private final int indexPos;\n    private final long msgOffset;\n    private final short batchSize;\n    private final long storeTimestamp;\n\n    public BatchOffsetIndex(MappedFile file, int pos, long msgOffset, short size, long storeTimestamp) {\n        mappedFile = file;\n        indexPos = pos;\n        this.msgOffset = msgOffset;\n        batchSize = size;\n        this.storeTimestamp = storeTimestamp;\n    }\n\n    public MappedFile getMappedFile() {\n        return mappedFile;\n    }\n\n    public int getIndexPos() {\n        return indexPos;\n    }\n\n    public long getMsgOffset() {\n        return msgOffset;\n    }\n\n    public short getBatchSize() {\n        return batchSize;\n    }\n\n    public long getStoreTimestamp() {\n        return storeTimestamp;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/CombineConsumeQueueStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.queue;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.google.common.annotations.VisibleForTesting;\nimport java.util.Arrays;\nimport java.util.LinkedList;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.CheckRocksdbCqWriteResult;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.StoreType;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.exception.StoreException;\nimport org.rocksdb.RocksDBException;\n\npublic class CombineConsumeQueueStore implements ConsumeQueueStoreInterface {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private static final Logger BROKER_LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n\n    private final DefaultMessageStore messageStore;\n    private final MessageStoreConfig messageStoreConfig;\n\n    // Inner consume queue store.\n    private final LinkedList<AbstractConsumeQueueStore> innerConsumeQueueStoreList = new LinkedList<>();\n    private final ConsumeQueueStore consumeQueueStore;\n    private final RocksDBConsumeQueueStore rocksDBConsumeQueueStore;\n\n    // current read consume queue store.\n    private final AbstractConsumeQueueStore currentReadStore;\n    // consume queue store for assign offset and increase offset.\n    private final AbstractConsumeQueueStore assignOffsetStore;\n\n\n    /**\n     * ConsumeQueueStore recovers through commitlog dispatch, so it needs to search which file in the commitLog to\n     * start recovery from. It might not be possible for all inner consumeQueueStores in CombineConsumeQueueStore to\n     * fully recover (for example, a newly consumeQueueStore needs to start dispatch from the first file, which\n     * could be very time-consuming).\n     * <p>\n     * However, we need to ensure that assignOffsetStore can be fully recovered to guarantee the correctness of\n     * commitlog. When assignOffsetStore can be fully recovered but other stores cannot, we need to use\n     * extraSearchCommitLogFilesForRecovery to control whether to continue searching forward for positions that might\n     * satisfy the recovery of other stores.\n     */\n\n    private final AtomicInteger extraSearchCommitLogFilesForRecovery;\n\n    public CombineConsumeQueueStore(DefaultMessageStore messageStore) {\n        this.messageStore = messageStore;\n        this.messageStoreConfig = messageStore.getMessageStoreConfig();\n        extraSearchCommitLogFilesForRecovery =\n            new AtomicInteger(messageStoreConfig.getCombineCQMaxExtraSearchCommitLogFiles());\n\n        Set<StoreType> loadingConsumeQueueTypeSet = StoreType.fromString(messageStoreConfig.getCombineCQLoadingCQTypes());\n        if (loadingConsumeQueueTypeSet.isEmpty()) {\n            throw new IllegalArgumentException(\"CombineConsumeQueueStore loadingCQTypes is empty\");\n        }\n\n        if (loadingConsumeQueueTypeSet.contains(StoreType.DEFAULT)) {\n            this.consumeQueueStore = new ConsumeQueueStore(messageStore);\n            this.innerConsumeQueueStoreList.add(consumeQueueStore);\n        } else {\n            this.consumeQueueStore = null;\n        }\n\n        if (loadingConsumeQueueTypeSet.contains(StoreType.DEFAULT_ROCKSDB)) {\n            this.rocksDBConsumeQueueStore = new RocksDBConsumeQueueStore(messageStore);\n            this.innerConsumeQueueStoreList.add(rocksDBConsumeQueueStore);\n        } else {\n            this.rocksDBConsumeQueueStore = null;\n        }\n\n        if (innerConsumeQueueStoreList.isEmpty()) {\n            throw new IllegalArgumentException(\"CombineConsumeQueueStore loadingCQTypes is empty\");\n        }\n\n        assignOffsetStore = getInnerStoreByString(messageStoreConfig.getCombineAssignOffsetCQType());\n        if (assignOffsetStore == null) {\n            log.error(\"CombineConsumeQueueStore chooseAssignOffsetStore fail, config={}\",\n                messageStoreConfig.getCombineAssignOffsetCQType());\n            throw new IllegalArgumentException(\"CombineConsumeQueue chooseAssignOffsetStore fail\");\n        }\n\n        currentReadStore = getInnerStoreByString(messageStoreConfig.getCombineCQPreferCQType());\n        if (currentReadStore == null) {\n            log.error(\"CombineConsumeQueueStore choosePreferCQ fail, config={}\",\n                messageStoreConfig.getCombineCQPreferCQType());\n            throw new IllegalArgumentException(\"CombineConsumeQueue choosePreferCQ fail\");\n        }\n\n        log.info(\"CombineConsumeQueueStore init, consumeQueueStoreList={}, currentReadStore={}, assignOffsetStore={}\",\n            innerConsumeQueueStoreList, currentReadStore.getClass().getSimpleName(), assignOffsetStore.getClass().getSimpleName());\n    }\n\n    @Override\n    public boolean load() {\n        for (AbstractConsumeQueueStore store : innerConsumeQueueStoreList) {\n            if (!store.load()) {\n                log.error(\"CombineConsumeQueueStore load fail, loadType={}\", store.getClass().getSimpleName());\n                return false;\n            }\n        }\n        log.info(\"CombineConsumeQueueStore load success\");\n        return true;\n    }\n\n    @Override\n    public void recover(boolean concurrently) throws RocksDBException {\n        for (AbstractConsumeQueueStore store : innerConsumeQueueStoreList) {\n            store.recover(concurrently);\n        }\n        log.info(\"CombineConsumeQueueStore recover success, concurrently={}\", concurrently);\n    }\n\n    @Override\n    public boolean isMappedFileMatchedRecover(long phyOffset, long storeTimestamp,\n        boolean recoverNormally) throws RocksDBException {\n        // make sure assignOffsetStore can be fully recovered\n        if (!assignOffsetStore.isMappedFileMatchedRecover(phyOffset, storeTimestamp, recoverNormally)) {\n            return false;\n        }\n\n        for (AbstractConsumeQueueStore store : innerConsumeQueueStoreList) {\n            if (store == assignOffsetStore || store.isMappedFileMatchedRecover(phyOffset, storeTimestamp, recoverNormally)) {\n                continue;\n            }\n            // if other store is not matched for fully recovery, extraSearchCommitLogFilesForRecovery will minus 1\n            if (extraSearchCommitLogFilesForRecovery.getAndDecrement() <= 0) {\n                // extraSearchCommitLogFilesForRecovery <= 0, only can read from assignOffsetStore\n                if (assignOffsetStore != currentReadStore) {\n                    log.error(\"CombineConsumeQueueStore currentReadStore not satisfied readable conditions, assignOffsetStore={}, currentReadStore={}\",\n                        assignOffsetStore.getClass().getSimpleName(), currentReadStore.getClass().getSimpleName());\n                    throw new IllegalArgumentException(store.getClass().getSimpleName() + \" not satisfied readable conditions, only can read from \" + assignOffsetStore.getClass().getSimpleName());\n                }\n                log.warn(\"CombineConsumeQueueStore can not recover all inner store, maybe some inner store start haven’t started before, store={}\",\n                    store.getClass().getSimpleName());\n                return true;\n            } else {\n                return false;\n            }\n        }\n        return true;\n    }\n\n    @Override\n    public Long getDispatchFromPhyOffset(boolean recoverNormally) throws RocksDBException {\n        Long dispatchFromPhyOffset = assignOffsetStore.getDispatchFromPhyOffset(recoverNormally);\n        for (AbstractConsumeQueueStore store : innerConsumeQueueStoreList) {\n            if (store == assignOffsetStore) {\n                continue;\n            }\n            Long storeOffset = store.getDispatchFromPhyOffset(recoverNormally);\n            if (storeOffset != null && dispatchFromPhyOffset != null && storeOffset < dispatchFromPhyOffset) {\n                dispatchFromPhyOffset = storeOffset;\n            }\n        }\n        return dispatchFromPhyOffset;\n    }\n\n    @Override\n    public void start() {\n        boolean success = false;\n        try {\n            success = verifyAndInitOffsetForAllStore(true);\n        } catch (RocksDBException e) {\n            log.error(\"CombineConsumeQueueStore checkAssignOffsetStore fail\", e);\n        }\n\n        if (!success && assignOffsetStore != currentReadStore) {\n            log.error(\"CombineConsumeQueueStore currentReadStore not satisfied readable conditions, \" +\n                    \"checkAssignOffsetResult={}, assignOffsetStore={}, currentReadStore={}\",\n                success, assignOffsetStore.getClass().getSimpleName(), currentReadStore.getClass().getSimpleName());\n            throw new RuntimeException(\"CombineConsumeQueueStore currentReadStore not satisfied readable conditions\");\n        }\n\n        for (AbstractConsumeQueueStore store : innerConsumeQueueStoreList) {\n            store.start();\n        }\n    }\n\n    public boolean verifyAndInitOffsetForAllStore(boolean initializeOffset) throws RocksDBException {\n        if (innerConsumeQueueStoreList.size() <= 1) {\n            return true;\n        }\n\n        boolean result = true;\n        long minPhyOffset = this.messageStore.getCommitLog().getMinOffset();\n        // for each topic and queueId in assignOffsetStore\n        for (Map.Entry<String, ConcurrentMap<Integer, ConsumeQueueInterface>> entry : assignOffsetStore.getConsumeQueueTable().entrySet()) {\n            for (Map.Entry<Integer, ConsumeQueueInterface> entry0 : entry.getValue().entrySet()) {\n                String topic = entry.getKey();\n                int queueId = entry0.getKey();\n                long maxOffsetInAssign = entry0.getValue().getMaxOffsetInQueue();\n\n                for (AbstractConsumeQueueStore abstractConsumeQueueStore : innerConsumeQueueStoreList) {\n                    // skip compare self\n                    if (abstractConsumeQueueStore == assignOffsetStore) {\n                        continue;\n                    }\n\n                    ConsumeQueueInterface queue = abstractConsumeQueueStore.findOrCreateConsumeQueue(topic, queueId);\n                    long maxOffset0 = queue.getMaxOffsetInQueue();\n\n                    if (maxOffsetInAssign == maxOffset0 || maxOffsetInAssign <= 0 && maxOffset0 <= 0) {\n                        continue;\n                    }\n\n                    if (maxOffset0 > 0) {\n                        log.error(\"CombineConsumeQueueStore checkAssignOffsetStore fail, topic={}, queueId={}, maxOffsetInAssign={}, otherCQ={}, maxOffset0={}\",\n                            topic, queueId, maxOffsetInAssign, abstractConsumeQueueStore.getClass().getSimpleName(), maxOffset0);\n                        result = false;\n                    }\n\n                    if (initializeOffset) {\n                        queue.initializeWithOffset(maxOffsetInAssign, minPhyOffset);\n                        log.info(\"CombineConsumeQueueStore initialize offset in queue, topic={}, queueId={}, maxOffsetInAssign={}, otherCQ={}, maxOffset0={}, maxOffsetNew={}\",\n                            topic, queueId, maxOffsetInAssign, abstractConsumeQueueStore.getClass().getSimpleName(), maxOffset0, queue.getMaxOffsetInQueue());\n                    }\n                }\n            }\n        }\n        return result;\n    }\n\n    @Override\n    public boolean shutdown() {\n        boolean result = true;\n        for (AbstractConsumeQueueStore store : innerConsumeQueueStoreList) {\n            if (!store.shutdown()) {\n                result = false;\n            }\n        }\n        return result;\n    }\n\n    @Override\n    public void destroy(boolean loadAfterDestroy) {\n        for (AbstractConsumeQueueStore store : innerConsumeQueueStoreList) {\n            store.destroy(loadAfterDestroy);\n        }\n    }\n\n    @Override\n    public boolean deleteTopic(String topic) {\n        boolean result = false;\n        for (AbstractConsumeQueueStore store : innerConsumeQueueStoreList) {\n            if (store.deleteTopic(topic)) {\n                result = true;\n            }\n        }\n        return result;\n    }\n\n    @Override\n    public void flush() throws StoreException {\n        for (AbstractConsumeQueueStore store : innerConsumeQueueStoreList) {\n            store.flush();\n        }\n    }\n\n    @Override\n    public void cleanExpired(long minCommitLogOffset) {\n        for (AbstractConsumeQueueStore store : innerConsumeQueueStoreList) {\n            store.cleanExpired(minCommitLogOffset);\n        }\n    }\n\n    @Override\n    public void checkSelf() {\n        for (AbstractConsumeQueueStore store : innerConsumeQueueStoreList) {\n            store.checkSelf();\n        }\n\n        if (messageStoreConfig.isCombineCQEnableCheckSelf()) {\n            try {\n                verifyAndInitOffsetForAllStore(false);\n            } catch (RocksDBException e) {\n                log.error(\"CombineConsumeQueueStore checkAssignOffsetStore fail in checkSelf\", e);\n            }\n            CheckRocksdbCqWriteResult checkResult = doCheckCqWriteProgress(null, System.currentTimeMillis() - 10 * 60 * 1000, StoreType.DEFAULT, StoreType.DEFAULT_ROCKSDB);\n            BROKER_LOG.info(\"checkRocksdbCqWriteProgress result: {}\", JSON.toJSONString(checkResult));\n        }\n    }\n\n    @Override\n    public void truncateDirty(long offsetToTruncate) throws RocksDBException {\n        for (AbstractConsumeQueueStore store : innerConsumeQueueStoreList) {\n            store.truncateDirty(offsetToTruncate);\n        }\n    }\n\n    @Override\n    public void putMessagePositionInfoWrapper(DispatchRequest request) throws RocksDBException {\n        for (AbstractConsumeQueueStore store : innerConsumeQueueStoreList) {\n            store.putMessagePositionInfoWrapper(request);\n        }\n    }\n\n    @Override\n    public ConcurrentMap<String, ConcurrentMap<Integer, ConsumeQueueInterface>> getConsumeQueueTable() {\n        return currentReadStore.getConsumeQueueTable();\n    }\n\n    @Override\n    public void assignQueueOffset(MessageExtBrokerInner msg) throws RocksDBException {\n        assignOffsetStore.assignQueueOffset(msg);\n    }\n\n    @Override\n    public void increaseQueueOffset(MessageExtBrokerInner msg, short messageNum) {\n        assignOffsetStore.increaseQueueOffset(msg, messageNum);\n    }\n\n    @Override\n    public void increaseLmqOffset(String topic, int queueId, short delta) throws ConsumeQueueException {\n        assignOffsetStore.increaseLmqOffset(topic, queueId, delta);\n    }\n\n    @Override\n    public long getLmqQueueOffset(String topic, int queueId) throws ConsumeQueueException {\n        return assignOffsetStore.getLmqQueueOffset(topic, queueId);\n    }\n\n    @Override\n    public void recoverOffsetTable(long minPhyOffset) {\n        for (AbstractConsumeQueueStore store : innerConsumeQueueStoreList) {\n            store.recoverOffsetTable(minPhyOffset);\n        }\n    }\n\n    @Override\n    public Long getMaxOffset(String topic, int queueId) throws ConsumeQueueException {\n        return currentReadStore.getMaxOffset(topic, queueId);\n    }\n\n    @Override\n    public long getMinOffsetInQueue(String topic, int queueId) throws RocksDBException {\n        return currentReadStore.getMinOffsetInQueue(topic, queueId);\n    }\n\n    @Override\n    public long getOffsetInQueueByTime(String topic, int queueId, long timestamp,\n        BoundaryType boundaryType) throws RocksDBException {\n        return currentReadStore.getOffsetInQueueByTime(topic, queueId, timestamp, boundaryType);\n    }\n\n    @Override\n    public ConsumeQueueInterface findOrCreateConsumeQueue(String topic, int queueId) {\n        return currentReadStore.findOrCreateConsumeQueue(topic, queueId);\n    }\n\n    @Override\n    public ConsumeQueueInterface getConsumeQueue(String topic, int queueId) {\n        return currentReadStore.getConsumeQueue(topic, queueId);\n    }\n\n    @Override\n    public long getTotalSize() {\n        long result = 0;\n        for (AbstractConsumeQueueStore store : innerConsumeQueueStoreList) {\n            result += store.getTotalSize();\n        }\n        return result;\n    }\n\n    @Override\n    public int getLmqNum() {\n        return currentReadStore.getLmqNum();\n    }\n\n    @Override\n    public boolean isLmqExist(String lmqTopic) {\n        return currentReadStore.isLmqExist(lmqTopic);\n    }\n\n    public RocksDBConsumeQueueStore getRocksDBConsumeQueueStore() {\n        return rocksDBConsumeQueueStore;\n    }\n\n    @VisibleForTesting\n    public ConsumeQueueStore getConsumeQueueStore() {\n        return consumeQueueStore;\n    }\n\n    @VisibleForTesting\n    public AbstractConsumeQueueStore getCurrentReadStore() {\n        return currentReadStore;\n    }\n\n    @VisibleForTesting\n    public AbstractConsumeQueueStore getAssignOffsetStore() {\n        return assignOffsetStore;\n    }\n\n    public CheckRocksdbCqWriteResult doCheckCqWriteProgress(String requestTopic, long checkStoreTime,\n        StoreType baseStoreType, StoreType compareStoreType) {\n        CheckRocksdbCqWriteResult result = new CheckRocksdbCqWriteResult();\n        AbstractConsumeQueueStore baseStore = getInnerStoreByStoreType(baseStoreType);\n        AbstractConsumeQueueStore compareStore = getInnerStoreByStoreType(compareStoreType);\n\n        if (baseStore == null || compareStore == null) {\n            result.setCheckResult(\"baseStore or compareStore is null, no need check\");\n            result.setCheckStatus(CheckRocksdbCqWriteResult.CheckStatus.CHECK_OK.getValue());\n            return result;\n        }\n\n        ConcurrentMap<String, ConcurrentMap<Integer, ConsumeQueueInterface>> cqTable = baseStore.getConsumeQueueTable();\n        StringBuilder diffResult = new StringBuilder();\n        try {\n            if (StringUtils.isNotBlank(requestTopic)) {\n                boolean checkResult = processConsumeQueuesForTopic(cqTable.get(requestTopic), requestTopic, compareStore, diffResult, true, checkStoreTime);\n                result.setCheckResult(diffResult.toString());\n                result.setCheckStatus(checkResult ? CheckRocksdbCqWriteResult.CheckStatus.CHECK_OK.getValue() : CheckRocksdbCqWriteResult.CheckStatus.CHECK_NOT_OK.getValue());\n                return result;\n            }\n            int successNum = 0;\n            int checkSize = 0;\n            for (Map.Entry<String, ConcurrentMap<Integer, ConsumeQueueInterface>> topicEntry : cqTable.entrySet()) {\n                boolean checkResult = processConsumeQueuesForTopic(topicEntry.getValue(), topicEntry.getKey(), compareStore, diffResult, false, checkStoreTime);\n                successNum += checkResult ? 1 : 0;\n                checkSize++;\n            }\n            // check all topic finish, all topic is ready, checkSize: 100, currentQueueNum: 110      -> ready  (The currentQueueNum means when we do checking, new topics are added.)\n            // check all topic finish, success/all : 89/100, currentQueueNum: 110                    -> not ready\n            boolean checkReady = successNum == checkSize;\n            String checkResultString = checkReady ? String.format(\"all topic is ready, checkSize: %s, currentQueueNum: %s\", checkSize, cqTable.size()) :\n                String.format(\"success/all : %s/%s, currentQueueNum: %s\", successNum, checkSize, cqTable.size());\n            diffResult.append(\"check all topic finish, \").append(checkResultString);\n            result.setCheckResult(diffResult.toString());\n            result.setCheckStatus(checkReady ? CheckRocksdbCqWriteResult.CheckStatus.CHECK_OK.getValue() : CheckRocksdbCqWriteResult.CheckStatus.CHECK_NOT_OK.getValue());\n        } catch (Exception e) {\n            log.error(\"CheckRocksdbCqWriteProgressCommand error\", e);\n            result.setCheckResult(e.getMessage() + Arrays.toString(e.getStackTrace()));\n            result.setCheckStatus(CheckRocksdbCqWriteResult.CheckStatus.CHECK_ERROR.getValue());\n        }\n        return result;\n    }\n\n    private boolean processConsumeQueuesForTopic(ConcurrentMap<Integer, ConsumeQueueInterface> queueMap, String topic,\n        AbstractConsumeQueueStore abstractConsumeQueueStore, StringBuilder diffResult, boolean printDetail,\n        long checkpointByStoreTime) {\n        boolean processResult = true;\n        for (Map.Entry<Integer, ConsumeQueueInterface> queueEntry : queueMap.entrySet()) {\n            Integer queueId = queueEntry.getKey();\n            ConsumeQueueInterface baseCQ = queueEntry.getValue();\n            ConsumeQueueInterface compareCQ = abstractConsumeQueueStore.findOrCreateConsumeQueue(topic, queueId);\n            if (printDetail) {\n                String format = String.format(\"[topic: %s, queue:  %s] \\n  kvEarliest : %s |  kvLatest : %s \\n fileEarliest: %s | fileEarliest: %s \",\n                    topic, queueId, compareCQ.getEarliestUnit(), compareCQ.getLatestUnit(), baseCQ.getEarliestUnit(), baseCQ.getLatestUnit());\n                diffResult.append(format).append(\"\\n\");\n            }\n\n            long minOffsetByTime = 0L;\n            try {\n                minOffsetByTime = abstractConsumeQueueStore.getOffsetInQueueByTime(topic, queueId, checkpointByStoreTime, BoundaryType.UPPER);\n            } catch (Exception e) {\n                // ignore\n            }\n            long minOffsetInQueue = compareCQ.getMinOffsetInQueue();\n            long checkFrom = Math.max(minOffsetInQueue, minOffsetByTime);\n            long checkTo = baseCQ.getMaxOffsetInQueue() - 1;\n            /*\n                                                            checkTo(maxOffsetInQueue - 1)\n                                                                        v\n        baseCQ   +------------------------------------------------------+\n        compareCQ        +----------------------------------------------+\n                         ^                ^\n                   minOffsetInQueue   minOffsetByTime\n                                          ^\n                        checkFrom = max(minOffsetInQueue, minOffsetByTime)\n             */\n            // The latest message is earlier than the check time\n            Pair<CqUnit, Long> fileLatestCq = baseCQ.getCqUnitAndStoreTime(checkTo);\n            if (fileLatestCq != null) {\n                if (fileLatestCq.getObject2() < checkpointByStoreTime) {\n                    continue;\n                }\n            }\n            for (long i = checkFrom; i <= checkTo; i++) {\n                Pair<CqUnit, Long> baseCqUnit = baseCQ.getCqUnitAndStoreTime(i);\n                Pair<CqUnit, Long> compareCqUnit = compareCQ.getCqUnitAndStoreTime(i);\n                if (baseCqUnit == null || compareCqUnit == null || !checkCqUnitEqual(compareCqUnit.getObject1(), baseCqUnit.getObject1())) {\n                    log.error(String.format(\"[topic: %s, queue: %s, offset: %s] \\n file : %s  \\n  kv : %s \\n\",\n                        topic, queueId, i, compareCqUnit != null ? compareCqUnit.getObject1() : \"null\", baseCqUnit != null ? baseCqUnit.getObject1() : \"null\"));\n                    processResult = false;\n                    break;\n                }\n            }\n        }\n        return processResult;\n    }\n\n    private boolean checkCqUnitEqual(CqUnit cqUnit1, CqUnit cqUnit2) {\n        if (cqUnit1.getQueueOffset() != cqUnit2.getQueueOffset()) {\n            return false;\n        }\n        if (cqUnit1.getSize() != cqUnit2.getSize()) {\n            return false;\n        }\n        if (cqUnit1.getPos() != cqUnit2.getPos()) {\n            return false;\n        }\n        if (cqUnit1.getBatchNum() != cqUnit2.getBatchNum()) {\n            return false;\n        }\n        return cqUnit1.getTagsCode() == cqUnit2.getTagsCode();\n    }\n\n    private AbstractConsumeQueueStore getInnerStoreByString(String storeTypeString) {\n        if (StoreType.DEFAULT.getStoreType().equalsIgnoreCase(storeTypeString)) {\n            return consumeQueueStore;\n        } else if (StoreType.DEFAULT_ROCKSDB.getStoreType().equalsIgnoreCase(storeTypeString)) {\n            return rocksDBConsumeQueueStore;\n        } else {\n            return null;\n        }\n    }\n\n    private AbstractConsumeQueueStore getInnerStoreByStoreType(StoreType storeType) {\n        switch (storeType) {\n            case DEFAULT:\n                return consumeQueueStore;\n            case DEFAULT_ROCKSDB:\n                return rocksDBConsumeQueueStore;\n            default:\n                return null;\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueInterface.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.queue;\n\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.MessageFilter;\nimport org.rocksdb.RocksDBException;\n\npublic interface ConsumeQueueInterface extends FileQueueLifeCycle {\n    /**\n     * Get the topic name\n     * @return the topic this cq belongs to.\n     */\n    String getTopic();\n\n    /**\n     * Get queue id\n     * @return the queue id this cq belongs to.\n     */\n    int getQueueId();\n\n    /**\n     * Get the units from the start offset.\n     *\n     * @param startIndex start index\n     * @return the unit iterateFrom\n     */\n    ReferredIterator<CqUnit> iterateFrom(long startIndex);\n\n    /**\n     * Get the units from the start offset.\n     *\n     * @param startIndex start index\n     * @param count the unit counts will be iterated\n     * @return the unit iterateFrom\n     * @throws RocksDBException only in rocksdb mode\n     */\n    ReferredIterator<CqUnit> iterateFrom(long startIndex, int count) throws RocksDBException;\n\n    /**\n     * Get cq unit at specified index\n     * @param index index\n     * @return the cq unit at index\n     */\n    CqUnit get(long index);\n\n    /**\n     * Get earliest cq unit\n     * @return the cq unit and message storeTime at index\n     */\n    Pair<CqUnit, Long> getCqUnitAndStoreTime(long index);\n\n    /**\n     * Get earliest cq unit\n     * @return earliest cq unit and message storeTime\n     */\n    Pair<CqUnit, Long> getEarliestUnitAndStoreTime();\n\n    /**\n     * Get earliest cq unit\n     * @return earliest cq unit\n     */\n    CqUnit getEarliestUnit();\n\n    /**\n     * Get last cq unit\n     * @return last cq unit\n     */\n    CqUnit getLatestUnit();\n\n    /**\n     * Get last commit log offset\n     * @return last commit log offset\n     */\n    long getLastOffset();\n\n    /**\n     * Get min offset(index) in queue\n     * @return the min offset(index) in queue\n     */\n    long getMinOffsetInQueue();\n\n    /**\n     * Get max offset(index) in queue\n     * @return the max offset(index) in queue\n     */\n    long getMaxOffsetInQueue();\n\n    /**\n     * Get total message count\n     * @return total message count\n     */\n    long getMessageTotalInQueue();\n\n    /**\n     * Get the message whose timestamp is the smallest, greater than or equal to the given time.\n     * @param timestamp timestamp\n     * @return the offset(index)\n     */\n    long getOffsetInQueueByTime(final long timestamp);\n\n    /**\n     * Get the message whose timestamp is the smallest, greater than or equal to the given time and when there are more\n     * than one message satisfy the condition, decide which one to return based on boundaryType.\n     * @param timestamp    timestamp\n     * @param boundaryType Lower or Upper\n     * @return the offset(index)\n     */\n    long getOffsetInQueueByTime(final long timestamp, final BoundaryType boundaryType);\n\n    /**\n     * The max physical offset of commitlog has been dispatched to this queue.\n     * It should be exclusive.\n     *\n     * @return the max physical offset point to commitlog\n     */\n    long getMaxPhysicOffset();\n\n    /**\n     * Usually, the cq files are not exactly consistent with the commitlog, there maybe some redundant data in the first\n     * cq file.\n     *\n     * @return the minimal effective pos of the cq file.\n     */\n    long getMinLogicOffset();\n\n    /**\n     * Get cq type\n     * @return cq type\n     */\n    CQType getCQType();\n\n    /**\n     * Gets the occupied size of CQ file on disk\n     * @return total size\n     */\n    long getTotalSize();\n\n    /**\n     * Get the unit size of this CQ which is different in different CQ impl\n     * @return cq unit size\n     */\n    int getUnitSize();\n\n    /**\n     * Correct min offset by min commit log offset.\n     * @param minCommitLogOffset min commit log offset\n     */\n    void correctMinOffset(long minCommitLogOffset);\n\n    /**\n     * Do dispatch.\n     * @param request the request containing dispatch information.\n     */\n    void putMessagePositionInfoWrapper(DispatchRequest request);\n\n    /**\n     * Assign queue offset.\n     * @param queueOffsetAssigner the delegated queue offset assigner\n     * @param msg message itself\n     * @throws RocksDBException only in rocksdb mode\n     */\n    void assignQueueOffset(QueueOffsetOperator queueOffsetAssigner, MessageExtBrokerInner msg) throws RocksDBException;\n\n    /**\n     * Increase queue offset.\n     * @param queueOffsetAssigner the delegated queue offset assigner\n     * @param msg message itself\n     * @param messageNum message number\n     */\n    void increaseQueueOffset(QueueOffsetOperator queueOffsetAssigner, MessageExtBrokerInner msg, short messageNum);\n\n    /**\n     * Estimate number of records matching given filter.\n     *\n     * @param from Lower boundary, inclusive.\n     * @param to Upper boundary, inclusive.\n     * @param filter Specified filter criteria\n     * @return Number of matching records.\n     */\n    long estimateMessageCount(long from, long to, MessageFilter filter);\n\n    /**\n     * Initialize cq and set max offset and min offset to given offset\n     *\n     * @param offset       set max and min offset to given offset\n     * @param minPhyOffset min physical offset, used to correct min offset\n     */\n    void initializeWithOffset(long offset, long minPhyOffset);\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.queue;\n\nimport java.io.File;\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Map.Entry;\nimport java.util.Objects;\nimport java.util.Optional;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.FutureTask;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicInteger;\n\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.common.utils.QueueTypeUtils;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.store.CommitLog;\nimport org.apache.rocketmq.store.ConsumeQueue;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.exception.StoreException;\nimport org.rocksdb.RocksDBException;\n\nimport static java.lang.String.format;\nimport static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathBatchConsumeQueue;\nimport static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathConsumeQueue;\n\npublic class ConsumeQueueStore extends AbstractConsumeQueueStore {\n    private final FlushConsumeQueueService flushConsumeQueueService;\n    private final CorrectLogicOffsetService correctLogicOffsetService;\n    private final CleanConsumeQueueService cleanConsumeQueueService;\n    private final AtomicInteger lmqCounter = new AtomicInteger(0);\n\n    public ConsumeQueueStore(DefaultMessageStore messageStore) {\n        super(messageStore);\n        this.flushConsumeQueueService = new FlushConsumeQueueService();\n        this.correctLogicOffsetService = new CorrectLogicOffsetService();\n        this.cleanConsumeQueueService = new CleanConsumeQueueService();\n    }\n\n    @Override\n    public void start() {\n        this.flushConsumeQueueService.start();\n        messageStore.getScheduledCleanQueueExecutorService().scheduleWithFixedDelay(this::cleanQueueFilesPeriodically,\n            1000 * 60, this.messageStoreConfig.getCleanResourceInterval(), TimeUnit.MILLISECONDS);\n        log.info(\"Default ConsumeQueueStore start!\");\n    }\n\n    private void cleanQueueFilesPeriodically() {\n        this.correctLogicOffsetService.run();\n        this.cleanConsumeQueueService.run();\n    }\n\n    @Override\n    public boolean load() {\n        boolean cqLoadResult = loadConsumeQueues(getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), CQType.SimpleCQ);\n        boolean bcqLoadResult = loadConsumeQueues(getStorePathBatchConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), CQType.BatchCQ);\n        return cqLoadResult && bcqLoadResult;\n    }\n\n    @Override\n    public void recover(boolean concurrently) {\n        log.info(\"Start to recover consume queue concurrently={}\", concurrently);\n        if (concurrently) {\n            recoverConcurrently();\n        } else {\n            for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : this.consumeQueueTable.values()) {\n                for (ConsumeQueueInterface logic : maps.values()) {\n                    this.recover(logic);\n                }\n            }\n        }\n    }\n\n    /**\n     * Implementation of CommitLogDispatchStore.getDispatchFromPhyOffset() (inherited from ConsumeQueueStoreInterface).\n     * When recoverNormally is false, returns checkpoint's logicsPhysicalOffset so commitlog abnormal recovery starts\n     * from it.\n     */\n    @Override\n    public Long getDispatchFromPhyOffset(boolean recoverNormally) throws RocksDBException {\n        if (recoverNormally) {\n            return getMaxPhyOffsetInConsumeQueue();\n        } else {\n            long fromCheckpoint = this.messageStore.getStoreCheckpoint().getLogicsPhysicalOffset();\n            long physicMsgTimestamp = this.messageStore.getStoreCheckpoint().getPhysicMsgTimestamp();\n            if (physicMsgTimestamp > 0 && fromCheckpoint <= 0 && messageStoreConfig.isEnableAcceleratedRecovery()) {\n                throw new RuntimeException(\"Accelerated recovery is enabled but checkpoint's logicsPhysicalOffset is invalid\");\n            }\n            return fromCheckpoint;\n        }\n    }\n\n    public boolean recoverConcurrently() {\n        int count = 0;\n        for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : this.consumeQueueTable.values()) {\n            count += maps.values().size();\n        }\n        final CountDownLatch countDownLatch = new CountDownLatch(count);\n        BlockingQueue<Runnable> recoverQueue = new LinkedBlockingQueue<>();\n        final ExecutorService executor = buildExecutorService(recoverQueue, \"RecoverConsumeQueueThread_\");\n        List<FutureTask<Boolean>> result = new ArrayList<>(count);\n        try {\n            for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : this.consumeQueueTable.values()) {\n                for (final ConsumeQueueInterface logic : maps.values()) {\n                    FutureTask<Boolean> futureTask = new FutureTask<>(() -> {\n                        boolean ret = true;\n                        try {\n                            logic.recover();\n                        } catch (Throwable e) {\n                            ret = false;\n                            log.error(\"Exception occurs while recover consume queue concurrently, \" +\n                                \"topic={}, queueId={}\", logic.getTopic(), logic.getQueueId(), e);\n                        } finally {\n                            countDownLatch.countDown();\n                        }\n                        return ret;\n                    });\n\n                    result.add(futureTask);\n                    executor.submit(futureTask);\n                }\n            }\n            countDownLatch.await();\n            for (FutureTask<Boolean> task : result) {\n                if (task != null && task.isDone()) {\n                    if (!task.get()) {\n                        return false;\n                    }\n                }\n            }\n        } catch (Exception e) {\n            log.error(\"Exception occurs while recover consume queue concurrently\", e);\n            return false;\n        } finally {\n            executor.shutdown();\n        }\n        return true;\n    }\n\n    @Override\n    public boolean shutdown() {\n        try {\n            flush();\n            this.flushConsumeQueueService.shutdown();\n        } catch (StoreException e) {\n            log.error(\"Failed to flush all consume queues\", e);\n            return false;\n        }\n\n        return true;\n    }\n\n    public void correctMinOffset(ConsumeQueueInterface consumeQueue, long minCommitLogOffset) {\n        consumeQueue.correctMinOffset(minCommitLogOffset);\n    }\n\n    public void putMessagePositionInfoWrapper(DispatchRequest dispatchRequest) {\n        ConsumeQueueInterface cq = this.findOrCreateConsumeQueue(dispatchRequest.getTopic(), dispatchRequest.getQueueId());\n        this.putMessagePositionInfoWrapper(cq, dispatchRequest);\n    }\n\n    @Override\n    public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) {\n        ConsumeQueueInterface logic = findOrCreateConsumeQueue(topic, queueId);\n        if (logic != null) {\n            long resultOffset = logic.getOffsetInQueueByTime(timestamp, boundaryType);\n            // Make sure the result offset is in valid range.\n            resultOffset = Math.max(resultOffset, logic.getMinOffsetInQueue());\n            resultOffset = Math.min(resultOffset, logic.getMaxOffsetInQueue());\n            return resultOffset;\n        }\n        return 0;\n    }\n\n    private FileQueueLifeCycle getLifeCycle(String topic, int queueId) {\n        return findOrCreateConsumeQueue(topic, queueId);\n    }\n\n    public boolean load(ConsumeQueueInterface consumeQueue) {\n        FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId());\n        return fileQueueLifeCycle.load();\n    }\n\n    private boolean loadConsumeQueues(String storePath, CQType cqType) {\n        File dirLogic = new File(storePath);\n        File[] fileTopicList = dirLogic.listFiles();\n        if (fileTopicList != null) {\n\n            for (File fileTopic : fileTopicList) {\n                String topic = fileTopic.getName();\n\n                File[] fileQueueIdList = fileTopic.listFiles();\n                if (fileQueueIdList != null) {\n                    for (File fileQueueId : fileQueueIdList) {\n                        int queueId;\n                        try {\n                            queueId = Integer.parseInt(fileQueueId.getName());\n                        } catch (NumberFormatException e) {\n                            continue;\n                        }\n\n                        queueTypeShouldBe(topic, cqType);\n\n                        ConsumeQueueInterface logic = createConsumeQueueByType(cqType, topic, queueId, storePath);\n                        this.putConsumeQueue(topic, queueId, logic);\n                        if (!this.load(logic)) {\n                            return false;\n                        }\n                    }\n                }\n            }\n        }\n\n        log.info(\"load {} all over, OK\", cqType);\n\n        return true;\n    }\n\n    private ConsumeQueueInterface createConsumeQueueByType(CQType cqType, String topic, int queueId, String storePath) {\n        if (Objects.equals(CQType.SimpleCQ, cqType)) {\n            return new ConsumeQueue(\n                topic,\n                queueId,\n                storePath,\n                this.messageStoreConfig.getMappedFileSizeConsumeQueue(),\n                this.messageStore,\n                this);\n        } else if (Objects.equals(CQType.BatchCQ, cqType)) {\n            return new BatchConsumeQueue(\n                topic,\n                queueId,\n                storePath,\n                this.messageStoreConfig.getMapperFileSizeBatchConsumeQueue(),\n                this.messageStore);\n        } else {\n            throw new RuntimeException(format(\"queue type %s is not supported.\", cqType.toString()));\n        }\n    }\n\n    private void queueTypeShouldBe(String topic, CQType cqTypeExpected) {\n        Optional<TopicConfig> topicConfig = this.messageStore.getTopicConfig(topic);\n\n        CQType cqTypeActual = QueueTypeUtils.getCQType(topicConfig);\n\n        if (!Objects.equals(cqTypeExpected, cqTypeActual)) {\n            throw new RuntimeException(format(\"The queue type of topic: %s should be %s, but is %s\", topic, cqTypeExpected, cqTypeActual));\n        }\n    }\n\n    private ExecutorService buildExecutorService(BlockingQueue<Runnable> blockingQueue, String threadNamePrefix) {\n        return ThreadUtils.newThreadPoolExecutor(\n            this.messageStore.getBrokerConfig().getRecoverThreadPoolNums(),\n            this.messageStore.getBrokerConfig().getRecoverThreadPoolNums(),\n            1000 * 60,\n            TimeUnit.MILLISECONDS,\n            blockingQueue,\n            new ThreadFactoryImpl(threadNamePrefix));\n    }\n\n    public void recover(ConsumeQueueInterface consumeQueue) {\n        FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId());\n        fileQueueLifeCycle.recover();\n    }\n\n    @Override\n    public long getMaxPhyOffsetInConsumeQueue() {\n        long maxPhysicOffset = -1L;\n        for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : this.consumeQueueTable.values()) {\n            for (ConsumeQueueInterface logic : maps.values()) {\n                if (logic.getMaxPhysicOffset() > maxPhysicOffset) {\n                    maxPhysicOffset = logic.getMaxPhysicOffset();\n                }\n            }\n        }\n        return maxPhysicOffset;\n    }\n\n    @Override\n    public long getMinOffsetInQueue(String topic, int queueId) {\n        ConsumeQueueInterface logic = findOrCreateConsumeQueue(topic, queueId);\n        if (logic != null) {\n            return logic.getMinOffsetInQueue();\n        }\n\n        return -1;\n    }\n\n    public void checkSelf(ConsumeQueueInterface consumeQueue) {\n        FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId());\n        fileQueueLifeCycle.checkSelf();\n    }\n\n    @Override\n    public void checkSelf() {\n        for (Map.Entry<String, ConcurrentMap<Integer, ConsumeQueueInterface>> topicEntry : this.consumeQueueTable.entrySet()) {\n            for (Map.Entry<Integer, ConsumeQueueInterface> cqEntry : topicEntry.getValue().entrySet()) {\n                this.checkSelf(cqEntry.getValue());\n            }\n        }\n    }\n\n    public boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages) {\n        FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId());\n        return fileQueueLifeCycle.flush(flushLeastPages);\n    }\n\n    public void flush() throws StoreException {\n        for (Map.Entry<String, ConcurrentMap<Integer, ConsumeQueueInterface>> topicEntry : this.consumeQueueTable.entrySet()) {\n            for (Map.Entry<Integer, ConsumeQueueInterface> cqEntry : topicEntry.getValue().entrySet()) {\n                flush(cqEntry.getValue(), 0);\n            }\n        }\n    }\n\n    @Override\n    public void destroy(ConsumeQueueInterface consumeQueue) {\n        FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId());\n        fileQueueLifeCycle.destroy();\n        if (MixAll.isLmq(consumeQueue.getTopic())) {\n            lmqCounter.decrementAndGet();\n        }\n    }\n\n    public int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogPos) {\n        FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId());\n        return fileQueueLifeCycle.deleteExpiredFile(minCommitLogPos);\n    }\n\n    public void truncateDirtyLogicFiles(ConsumeQueueInterface consumeQueue, long phyOffset) {\n        FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId());\n        fileQueueLifeCycle.truncateDirtyLogicFiles(phyOffset);\n    }\n\n    public void swapMap(ConsumeQueueInterface consumeQueue, int reserveNum, long forceSwapIntervalMs,\n        long normalSwapIntervalMs) {\n        FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId());\n        fileQueueLifeCycle.swapMap(reserveNum, forceSwapIntervalMs, normalSwapIntervalMs);\n    }\n\n    public void cleanSwappedMap(ConsumeQueueInterface consumeQueue, long forceCleanSwapIntervalMs) {\n        FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId());\n        fileQueueLifeCycle.cleanSwappedMap(forceCleanSwapIntervalMs);\n    }\n\n    public boolean isFirstFileAvailable(ConsumeQueueInterface consumeQueue) {\n        FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId());\n        return fileQueueLifeCycle.isFirstFileAvailable();\n    }\n\n    public boolean isFirstFileExist(ConsumeQueueInterface consumeQueue) {\n        FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId());\n        return fileQueueLifeCycle.isFirstFileExist();\n    }\n\n    @Override\n    public ConsumeQueueInterface findOrCreateConsumeQueue(String topic, int queueId) {\n        ConcurrentMap<Integer, ConsumeQueueInterface> map = consumeQueueTable.get(topic);\n        if (null == map) {\n            ConcurrentMap<Integer, ConsumeQueueInterface> newMap = new ConcurrentHashMap<>(128);\n            ConcurrentMap<Integer, ConsumeQueueInterface> oldMap = consumeQueueTable.putIfAbsent(topic, newMap);\n            if (oldMap != null) {\n                map = oldMap;\n            } else {\n                map = newMap;\n            }\n        }\n\n        ConsumeQueueInterface logic = map.get(queueId);\n        if (logic != null) {\n            return logic;\n        }\n\n        ConsumeQueueInterface newLogic;\n\n        Optional<TopicConfig> topicConfig = this.messageStore.getTopicConfig(topic);\n        // TODO maybe the topic has been deleted.\n        if (Objects.equals(CQType.BatchCQ, QueueTypeUtils.getCQType(topicConfig))) {\n            newLogic = new BatchConsumeQueue(\n                topic,\n                queueId,\n                getStorePathBatchConsumeQueue(this.messageStoreConfig.getStorePathRootDir()),\n                this.messageStoreConfig.getMapperFileSizeBatchConsumeQueue(),\n                this.messageStore);\n        } else {\n            newLogic = new ConsumeQueue(\n                topic,\n                queueId,\n                getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir()),\n                this.messageStoreConfig.getMappedFileSizeConsumeQueue(),\n                this.messageStore, this);\n        }\n\n        ConsumeQueueInterface oldLogic = map.putIfAbsent(queueId, newLogic);\n        if (oldLogic != null) {\n            logic = oldLogic;\n        } else {\n            logic = newLogic;\n            if (MixAll.isLmq(topic)) {\n                lmqCounter.incrementAndGet();\n            }\n        }\n\n        return logic;\n    }\n\n    @Override\n    public ConsumeQueueInterface getConsumeQueue(String topic, int queueId) {\n        ConcurrentMap<Integer, ConsumeQueueInterface> map = this.getConsumeQueueTable().get(topic);\n        if (map == null) {\n            return null;\n        }\n        return map.get(queueId);\n    }\n\n    public void setBatchTopicQueueTable(ConcurrentMap<String, Long> batchTopicQueueTable) {\n        this.queueOffsetOperator.setBatchTopicQueueTable(batchTopicQueueTable);\n    }\n\n    public void updateQueueOffset(String topic, int queueId, long offset) {\n        String topicQueueKey = topic + \"-\" + queueId;\n        this.queueOffsetOperator.updateQueueOffset(topicQueueKey, offset);\n    }\n\n    private void putConsumeQueue(final String topic, final int queueId, final ConsumeQueueInterface consumeQueue) {\n        ConcurrentMap<Integer/* queueId */, ConsumeQueueInterface> map = this.consumeQueueTable.get(topic);\n        if (null == map) {\n            map = new ConcurrentHashMap<>();\n            map.put(queueId, consumeQueue);\n            this.consumeQueueTable.put(topic, map);\n            if (MixAll.isLmq(topic)) {\n                lmqCounter.incrementAndGet();\n            }\n        } else {\n            ConsumeQueueInterface prev = map.put(queueId, consumeQueue);\n            if (null == prev && MixAll.isLmq(topic)) {\n                lmqCounter.incrementAndGet();\n            }\n        }\n    }\n\n    @Override\n    public void recoverOffsetTable(long minPhyOffset) {\n        ConcurrentMap<String, Long> cqOffsetTable = new ConcurrentHashMap<>(1024);\n        ConcurrentMap<String, Long> bcqOffsetTable = new ConcurrentHashMap<>(1024);\n\n        for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : this.consumeQueueTable.values()) {\n            for (ConsumeQueueInterface logic : maps.values()) {\n                String key = logic.getTopic() + \"-\" + logic.getQueueId();\n\n                long maxOffsetInQueue = logic.getMaxOffsetInQueue();\n                if (Objects.equals(CQType.BatchCQ, logic.getCQType())) {\n                    bcqOffsetTable.put(key, maxOffsetInQueue);\n                } else {\n                    cqOffsetTable.put(key, maxOffsetInQueue);\n                }\n\n                this.correctMinOffset(logic, minPhyOffset);\n            }\n        }\n\n        // Correct unSubmit consumeOffset\n        if (messageStoreConfig.isDuplicationEnable() || messageStore.getBrokerConfig().isEnableControllerMode()) {\n            compensateForHA(cqOffsetTable);\n        }\n\n        this.setTopicQueueTable(cqOffsetTable);\n        this.setBatchTopicQueueTable(bcqOffsetTable);\n    }\n\n    private void compensateForHA(ConcurrentMap<String, Long> cqOffsetTable) {\n        SelectMappedBufferResult lastBuffer = null;\n        long startReadOffset = messageStore.getCommitLog().getConfirmOffset() == -1 ? 0 : messageStore.getCommitLog().getConfirmOffset();\n        log.info(\"Correct unsubmitted offset...StartReadOffset = {}\", startReadOffset);\n        while ((lastBuffer = messageStore.selectOneMessageByOffset(startReadOffset)) != null) {\n            try {\n                if (lastBuffer.getStartOffset() > startReadOffset) {\n                    startReadOffset = lastBuffer.getStartOffset();\n                    continue;\n                }\n\n                ByteBuffer bb = lastBuffer.getByteBuffer();\n                int magicCode = bb.getInt(bb.position() + 4);\n                if (magicCode == CommitLog.BLANK_MAGIC_CODE) {\n                    startReadOffset += bb.getInt(bb.position());\n                    continue;\n                } else if (magicCode != MessageDecoder.MESSAGE_MAGIC_CODE) {\n                    throw new RuntimeException(\"Unknown magicCode: \" + magicCode);\n                }\n\n                lastBuffer.getByteBuffer().mark();\n                DispatchRequest dispatchRequest = messageStore.getCommitLog().checkMessageAndReturnSize(lastBuffer.getByteBuffer(), true, messageStoreConfig.isDuplicationEnable(), true);\n                if (!dispatchRequest.isSuccess())\n                    break;\n                lastBuffer.getByteBuffer().reset();\n\n                MessageExt msg = MessageDecoder.decode(lastBuffer.getByteBuffer(), true, false, false, false, true);\n                if (msg == null)\n                    break;\n\n                String key = msg.getTopic() + \"-\" + msg.getQueueId();\n                cqOffsetTable.put(key, msg.getQueueOffset() + 1);\n                startReadOffset += msg.getStoreSize();\n                log.info(\"Correcting. Key:{}, start read Offset: {}\", key, startReadOffset);\n            } finally {\n                if (lastBuffer != null)\n                    lastBuffer.release();\n            }\n        }\n    }\n\n    /**\n     * @param loadAfterDestroy file version cq do not need reload, so ignore\n     */\n    @Override\n    public void destroy(boolean loadAfterDestroy) {\n        for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : this.consumeQueueTable.values()) {\n            for (ConsumeQueueInterface logic : maps.values()) {\n                this.destroy(logic);\n            }\n        }\n    }\n\n    @Override\n    public void cleanExpired(long minCommitLogOffset) {\n        Iterator<Entry<String, ConcurrentMap<Integer, ConsumeQueueInterface>>> it = this.consumeQueueTable.entrySet().iterator();\n        while (it.hasNext()) {\n            Map.Entry<String, ConcurrentMap<Integer, ConsumeQueueInterface>> next = it.next();\n            String topic = next.getKey();\n            if (!TopicValidator.isSystemTopic(topic)) {\n                ConcurrentMap<Integer, ConsumeQueueInterface> queueTable = next.getValue();\n                Iterator<Map.Entry<Integer, ConsumeQueueInterface>> itQT = queueTable.entrySet().iterator();\n                while (itQT.hasNext()) {\n                    Map.Entry<Integer, ConsumeQueueInterface> nextQT = itQT.next();\n                    long maxCLOffsetInConsumeQueue = nextQT.getValue().getLastOffset();\n\n                    if (maxCLOffsetInConsumeQueue == -1) {\n                        log.warn(\"maybe ConsumeQueue was created just now. topic={} queueId={} maxPhysicOffset={} minLogicOffset={}.\",\n                            nextQT.getValue().getTopic(),\n                            nextQT.getValue().getQueueId(),\n                            nextQT.getValue().getMaxPhysicOffset(),\n                            nextQT.getValue().getMinLogicOffset());\n                    } else if (maxCLOffsetInConsumeQueue < minCommitLogOffset) {\n                        log.info(\n                            \"cleanExpiredConsumerQueue: {} {} consumer queue destroyed, minCommitLogOffset: {} maxCLOffsetInConsumeQueue: {}\",\n                            topic,\n                            nextQT.getKey(),\n                            minCommitLogOffset,\n                            maxCLOffsetInConsumeQueue);\n\n                        removeTopicQueueTable(nextQT.getValue().getTopic(),\n                            nextQT.getValue().getQueueId());\n\n                        this.destroy(nextQT.getValue());\n                        itQT.remove();\n                    }\n                }\n\n                if (queueTable.isEmpty()) {\n                    log.info(\"cleanExpiredConsumerQueue: {},topic destroyed\", topic);\n                    it.remove();\n                }\n            }\n        }\n    }\n\n    @Override\n    public void truncateDirty(long offsetToTruncate) {\n        long maxPhyOffsetOfConsumeQueue = getMaxPhyOffsetInConsumeQueue();\n        if (maxPhyOffsetOfConsumeQueue >= offsetToTruncate) {\n            log.warn(\"maxPhyOffsetOfConsumeQueue({}) >= processOffset({}), truncate dirty logic files\", maxPhyOffsetOfConsumeQueue, offsetToTruncate);\n            for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : this.consumeQueueTable.values()) {\n                for (ConsumeQueueInterface logic : maps.values()) {\n                    this.truncateDirtyLogicFiles(logic, offsetToTruncate);\n                }\n            }\n        }\n    }\n\n    @Override\n    public long getTotalSize() {\n        long totalSize = 0;\n        for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : this.consumeQueueTable.values()) {\n            for (ConsumeQueueInterface logic : maps.values()) {\n                totalSize += logic.getTotalSize();\n            }\n        }\n        return totalSize;\n    }\n\n    @Override\n    public boolean isMappedFileMatchedRecover(long phyOffset, long storeTimestamp,\n        boolean recoverNormally) throws RocksDBException {\n        if (!recoverNormally && this.messageStore.getStoreCheckpoint().getLogicsPhysicalOffset() <= 0) { // for the sake of compatibility\n            return storeTimestamp <= this.messageStore.getStoreCheckpoint().getLogicsMsgTimestamp();\n        }\n        return phyOffset <= getDispatchFromPhyOffset(recoverNormally);\n    }\n\n    @Override\n    public int getLmqNum() {\n        return lmqCounter.get();\n    }\n\n    @Override\n    public boolean isLmqExist(String lmqTopic) {\n        return getConsumeQueue(lmqTopic, 0) != null;\n    }\n\n    public class FlushConsumeQueueService extends ServiceThread {\n        private static final int RETRY_TIMES_OVER = 3;\n        private long lastFlushTimestamp = 0;\n\n        private void doFlush(int retryTimes) {\n            int flushConsumeQueueLeastPages = messageStoreConfig.getFlushConsumeQueueLeastPages();\n\n            if (retryTimes == RETRY_TIMES_OVER) {\n                flushConsumeQueueLeastPages = 0;\n            }\n\n            long logicsMsgTimestamp = 0;\n            long logicsPhysicalOffset = 0;\n\n            int flushConsumeQueueThoroughInterval = messageStoreConfig.getFlushConsumeQueueThoroughInterval();\n            long currentTimeMillis = System.currentTimeMillis();\n            if (currentTimeMillis >= (this.lastFlushTimestamp + flushConsumeQueueThoroughInterval)) {\n                this.lastFlushTimestamp = currentTimeMillis;\n                flushConsumeQueueLeastPages = 0;\n                logicsMsgTimestamp = messageStore.getStoreCheckpoint().getTmpLogicsMsgTimestamp();\n                logicsPhysicalOffset = messageStore.getStoreCheckpoint().getTmpLogicsPhysicalOffset();\n            }\n\n            for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : consumeQueueTable.values()) {\n                for (ConsumeQueueInterface cq : maps.values()) {\n                    boolean result = false;\n                    for (int i = 0; i < retryTimes && !result; i++) {\n                        result = flush(cq, flushConsumeQueueLeastPages);\n                    }\n                }\n            }\n\n            if (messageStoreConfig.isEnableCompaction()) {\n                messageStore.getCompactionStore().flush(flushConsumeQueueLeastPages);\n            }\n\n            if (0 == flushConsumeQueueLeastPages) {\n                if (logicsMsgTimestamp > 0) {\n                    messageStore.getStoreCheckpoint().setLogicsMsgTimestamp(logicsMsgTimestamp);\n                }\n                if (logicsPhysicalOffset > 0) {\n                    messageStore.getStoreCheckpoint().setLogicsPhysicalOffset(logicsPhysicalOffset);\n                }\n                messageStore.getStoreCheckpoint().flush();\n            }\n        }\n\n        @Override\n        public void run() {\n            log.info(this.getServiceName() + \" service started\");\n\n            while (!this.isStopped()) {\n                try {\n                    int interval = messageStoreConfig.getFlushIntervalConsumeQueue();\n                    this.waitForRunning(interval);\n                    this.doFlush(1);\n                } catch (Exception e) {\n                    log.warn(this.getServiceName() + \" service has exception. \", e);\n                }\n            }\n\n            this.doFlush(RETRY_TIMES_OVER);\n\n            log.info(this.getServiceName() + \" service end\");\n        }\n\n        @Override\n        public String getServiceName() {\n            if (messageStore.getBrokerConfig().isInBrokerContainer()) {\n                return messageStore.getBrokerIdentity().getIdentifier() + FlushConsumeQueueService.class.getSimpleName();\n            }\n            return FlushConsumeQueueService.class.getSimpleName();\n        }\n\n        @Override\n        public long getJoinTime() {\n            return 1000 * 60;\n        }\n    }\n\n    class CorrectLogicOffsetService {\n        private long lastForceCorrectTime = -1L;\n\n        public void run() {\n            try {\n                this.correctLogicMinOffset();\n            } catch (Throwable e) {\n                log.warn(this.getServiceName() + \" service has exception. \", e);\n            }\n        }\n\n        private boolean needCorrect(ConsumeQueueInterface logic, long minPhyOffset, long lastForeCorrectTimeCurRun) {\n            if (logic == null) {\n                return false;\n            }\n            // If first exist and not available, it means first file may destroy failed, delete it.\n            if (isFirstFileExist(logic) && !isFirstFileAvailable(logic)) {\n                log.error(\"CorrectLogicOffsetService.needCorrect. first file not available, trigger correct.\" +\n                        \" topic:{}, queue:{}, maxPhyOffset in queue:{}, minPhyOffset \" +\n                        \"in commit log:{}, minOffset in queue:{}, maxOffset in queue:{}, cqType:{}\"\n                    , logic.getTopic(), logic.getQueueId(), logic.getMaxPhysicOffset()\n                    , minPhyOffset, logic.getMinOffsetInQueue(), logic.getMaxOffsetInQueue(), logic.getCQType());\n                return true;\n            }\n\n            // logic.getMaxPhysicOffset() or minPhyOffset = -1\n            // means there is no message in current queue, so no need to correct.\n            if (logic.getMaxPhysicOffset() == -1 || minPhyOffset == -1) {\n                return false;\n            }\n\n            if (logic.getMaxPhysicOffset() < minPhyOffset) {\n                if (logic.getMinOffsetInQueue() < logic.getMaxOffsetInQueue()) {\n                    log.error(\"CorrectLogicOffsetService.needCorrect. logic max phy offset: {} is less than min phy offset: {}, \" +\n                            \"but min offset: {} is less than max offset: {}. topic:{}, queue:{}, cqType:{}.\"\n                        , logic.getMaxPhysicOffset(), minPhyOffset, logic.getMinOffsetInQueue()\n                        , logic.getMaxOffsetInQueue(), logic.getTopic(), logic.getQueueId(), logic.getCQType());\n                    return true;\n                } else if (logic.getMinOffsetInQueue() == logic.getMaxOffsetInQueue()) {\n                    return false;\n                } else {\n                    log.error(\"CorrectLogicOffsetService.needCorrect. It should not happen, logic max phy offset: {} is less than min phy offset: {},\" +\n                            \" but min offset: {} is larger than max offset: {}. topic:{}, queue:{}, cqType:{}\"\n                        , logic.getMaxPhysicOffset(), minPhyOffset, logic.getMinOffsetInQueue()\n                        , logic.getMaxOffsetInQueue(), logic.getTopic(), logic.getQueueId(), logic.getCQType());\n                    return false;\n                }\n            }\n            //the logic.getMaxPhysicOffset() >= minPhyOffset\n            int forceCorrectInterval = messageStoreConfig.getCorrectLogicMinOffsetForceInterval();\n            if ((System.currentTimeMillis() - lastForeCorrectTimeCurRun) > forceCorrectInterval) {\n                lastForceCorrectTime = System.currentTimeMillis();\n                CqUnit cqUnit = logic.getEarliestUnit();\n                if (cqUnit == null) {\n                    if (logic.getMinOffsetInQueue() == logic.getMaxOffsetInQueue()) {\n                        return false;\n                    } else {\n                        log.error(\"CorrectLogicOffsetService.needCorrect. cqUnit is null, logic max phy offset: {} is greater than min phy offset: {}, \" +\n                                \"but min offset: {} is not equal to max offset: {}. topic:{}, queue:{}, cqType:{}.\"\n                            , logic.getMaxPhysicOffset(), minPhyOffset, logic.getMinOffsetInQueue()\n                            , logic.getMaxOffsetInQueue(), logic.getTopic(), logic.getQueueId(), logic.getCQType());\n                        return true;\n                    }\n                }\n\n                if (cqUnit.getPos() < minPhyOffset) {\n                    log.error(\"CorrectLogicOffsetService.needCorrect. logic max phy offset: {} is greater than min phy offset: {}, \" +\n                            \"but minPhyPos in cq is: {}. min offset in queue: {}, max offset in queue: {}, topic:{}, queue:{}, cqType:{}.\"\n                        , logic.getMaxPhysicOffset(), minPhyOffset, cqUnit.getPos(), logic.getMinOffsetInQueue()\n                        , logic.getMaxOffsetInQueue(), logic.getTopic(), logic.getQueueId(), logic.getCQType());\n                    return true;\n                }\n\n                if (cqUnit.getPos() >= minPhyOffset) {\n\n                    // Normal case, do not need to correct.\n                    return false;\n                }\n            }\n\n            return false;\n        }\n\n        private void correctLogicMinOffset() {\n\n            long lastForeCorrectTimeCurRun = lastForceCorrectTime;\n            long minPhyOffset = messageStore.getMinPhyOffset();\n            for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : consumeQueueTable.values()) {\n                for (ConsumeQueueInterface logic : maps.values()) {\n                    if (Objects.equals(CQType.SimpleCQ, logic.getCQType())) {\n                        // cq is not supported for now.\n                        continue;\n                    }\n                    if (needCorrect(logic, minPhyOffset, lastForeCorrectTimeCurRun)) {\n                        doCorrect(logic, minPhyOffset);\n                    }\n                }\n            }\n        }\n\n        private void doCorrect(ConsumeQueueInterface logic, long minPhyOffset) {\n            deleteExpiredFile(logic, minPhyOffset);\n            int sleepIntervalWhenCorrectMinOffset = messageStoreConfig.getCorrectLogicMinOffsetSleepInterval();\n            if (sleepIntervalWhenCorrectMinOffset > 0) {\n                try {\n                    Thread.sleep(sleepIntervalWhenCorrectMinOffset);\n                } catch (InterruptedException ignored) {\n                }\n            }\n        }\n\n        public String getServiceName() {\n            if (messageStore.getBrokerConfig().isInBrokerContainer()) {\n                return messageStore.getBrokerConfig().getIdentifier() + CorrectLogicOffsetService.class.getSimpleName();\n            }\n            return CorrectLogicOffsetService.class.getSimpleName();\n        }\n    }\n\n    public class CleanConsumeQueueService {\n        protected long lastPhysicalMinOffset = 0;\n\n        public void run() {\n            try {\n                this.deleteExpiredFiles();\n            } catch (Throwable e) {\n                log.warn(this.getServiceName() + \" service has exception. \", e);\n            }\n        }\n\n        protected void deleteExpiredFiles() {\n            int deleteLogicsFilesInterval = messageStoreConfig.getDeleteConsumeQueueFilesInterval();\n\n            long minOffset = messageStore.getCommitLog().getMinOffset();\n            if (minOffset > this.lastPhysicalMinOffset) {\n                this.lastPhysicalMinOffset = minOffset;\n\n                for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : consumeQueueTable.values()) {\n                    for (ConsumeQueueInterface logic : maps.values()) {\n                        int deleteCount = deleteExpiredFile(logic, minOffset);\n                        if (deleteCount > 0 && deleteLogicsFilesInterval > 0) {\n                            try {\n                                Thread.sleep(deleteLogicsFilesInterval);\n                            } catch (InterruptedException ignored) {\n                            }\n                        }\n                    }\n                }\n\n                messageStore.getIndexService().deleteExpiredFile(minOffset);\n                if (messageStoreConfig.isIndexRocksDBEnable() && null != messageStore.getIndexRocksDBStore()) {\n                    messageStore.getIndexRocksDBStore().deleteExpiredIndex();\n                }\n            }\n        }\n\n        public String getServiceName() {\n            return messageStore.getBrokerConfig().getIdentifier() + CleanConsumeQueueService.class.getSimpleName();\n        }\n    }\n\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.queue;\n\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.CommitLogDispatchStore;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.exception.StoreException;\nimport org.rocksdb.RocksDBException;\n\npublic interface ConsumeQueueStoreInterface extends CommitLogDispatchStore {\n\n    /**\n     * Load from file.\n     *\n     * @return true if loaded successfully.\n     */\n    boolean load();\n\n    /**\n     * Recover from file.\n     * @param concurrently whether to recover concurrently\n     */\n    void recover(boolean concurrently) throws RocksDBException;\n\n    /**\n     * Start the consumeQueueStore\n     */\n    void start();\n\n    /**\n     * Shutdown the consumeQueueStore\n     * @return true if shutdown successfully.\n     */\n    boolean shutdown();\n\n    /**\n     * destroy all consumeQueues\n     * @param loadAfterDestroy reload store after destroy, only used in RocksDB mode\n     */\n    void destroy(boolean loadAfterDestroy);\n\n    /**\n     * delete topic\n     */\n    boolean deleteTopic(String topic);\n\n    /**\n     * Flush all nested consume queues to disk\n     *\n     * @throws StoreException if there is an error during flush\n     */\n    void flush() throws StoreException;\n\n    /**\n     * clean expired data from minCommitLogOffset\n     * @param minCommitLogOffset Minimum commit log offset\n     */\n    void cleanExpired(long minCommitLogOffset);\n\n    /**\n     * Check files.\n     */\n    void checkSelf();\n\n    /**\n     * truncate dirty data\n     * @param offsetToTruncate\n     * @throws RocksDBException only in rocksdb mode\n     */\n    void truncateDirty(long offsetToTruncate) throws RocksDBException;\n\n    /**\n     * Apply the dispatched request. This function should be idempotent.\n     *\n     * @param request dispatch request\n     * @throws RocksDBException only in rocksdb mode will throw exception\n     */\n    void putMessagePositionInfoWrapper(DispatchRequest request) throws RocksDBException;\n\n    /**\n     * get consumeQueue table\n     * @return the consumeQueue table\n     */\n    ConcurrentMap<String, ConcurrentMap<Integer, ConsumeQueueInterface>> getConsumeQueueTable();\n\n    /**\n     * Assign queue offset.\n     * @param msg message itself\n     * @throws RocksDBException only in rocksdb mode\n     */\n    void assignQueueOffset(MessageExtBrokerInner msg) throws RocksDBException;\n\n    /**\n     * Increase queue offset.\n     * @param msg message itself\n     * @param messageNum message number\n     */\n    void increaseQueueOffset(MessageExtBrokerInner msg, short messageNum);\n\n    /**\n     * Increase lmq offset\n     * @param topic Topic/Queue name\n     * @param queueId Queue ID\n     * @param delta amount to increase\n     */\n    void increaseLmqOffset(String topic, int queueId, short delta) throws ConsumeQueueException;\n\n    /**\n     * get lmq queue offset\n     * @param topic\n     * @param queueId\n     * @return\n     */\n    long getLmqQueueOffset(String topic, int queueId) throws ConsumeQueueException;\n\n    /**\n     * recover topicQueue table by minPhyOffset\n     * @param minPhyOffset\n     */\n    void recoverOffsetTable(long minPhyOffset);\n\n    /**\n     * get maxOffset of specific topic-queueId in topicQueue table\n     *\n     * @param topic Topic name\n     * @param queueId Queue identifier\n     * @return the max offset in QueueOffsetOperator\n     * @throws ConsumeQueueException if there is an error while retrieving max consume queue offset\n     */\n    Long getMaxOffset(String topic, int queueId) throws ConsumeQueueException;\n\n    /**\n     * get min logic offset of specific topic-queueId in consumeQueue\n     * @param topic\n     * @param queueId\n     * @return the min logic offset of specific topic-queueId in consumeQueue\n     * @throws RocksDBException only in rocksdb mode\n     */\n    long getMinOffsetInQueue(final String topic, final int queueId) throws RocksDBException;\n\n    /**\n     * Get the message whose timestamp is the smallest, greater than or equal to the given time and when there are more\n     * than one message satisfy the condition, decide which one to return based on boundaryType.\n     * @param timestamp    timestamp\n     * @param boundaryType Lower or Upper\n     * @return the offset(index)\n     * @throws RocksDBException only in rocksdb mode\n     */\n    long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) throws RocksDBException;\n\n    /**\n     * find or create the consumeQueue\n     * @param topic\n     * @param queueId\n     * @return the consumeQueue\n     */\n    ConsumeQueueInterface findOrCreateConsumeQueue(String topic, int queueId);\n\n    /**\n     * only find consumeQueue\n     *\n     * @param topic\n     * @param queueId\n     * @return the consumeQueue\n     */\n    ConsumeQueueInterface getConsumeQueue(String topic, int queueId);\n\n    /**\n     * get the total size of all consumeQueue\n     * @return the total size of all consumeQueue\n     */\n    long getTotalSize();\n\n    /**\n     * get lmq consume queue count\n     * @return the count of lmq\n     */\n    int getLmqNum();\n\n    /**\n     * Check if the LMQ exists, this is different from getConsumeQueue()\n     * @param lmqTopic\n     * @return exist or not\n     */\n    boolean isLmqExist(String lmqTopic);\n\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/CqUnit.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.queue;\n\nimport org.apache.rocketmq.store.ConsumeQueueExt;\n\nimport java.nio.ByteBuffer;\n\npublic class CqUnit {\n    private final long queueOffset;\n    private final int size;\n    private final long pos;\n    private final short batchNum;\n    /**\n     * Be careful, the tagsCode is reused as an address for extent file. To prevent accident mistake, we follow the\n     * rules: 1. If the cqExtUnit is not null, make tagsCode == cqExtUnit.getTagsCode() 2. If the cqExtUnit is null, and\n     * the tagsCode is smaller than 0, it is an invalid tagsCode, which means failed to get cqExtUnit by address\n     */\n    private long tagsCode;\n    private ConsumeQueueExt.CqExtUnit cqExtUnit;\n    private final ByteBuffer nativeBuffer;\n    private final int compactedOffset;\n\n    public CqUnit(long queueOffset, long pos, int size, long tagsCode) {\n        this(queueOffset, pos, size, tagsCode, (short) 1, 0, null);\n    }\n\n    public CqUnit(long queueOffset, long pos, int size, long tagsCode, short batchNum, int compactedOffset, ByteBuffer buffer) {\n        this.queueOffset = queueOffset;\n        this.pos = pos;\n        this.size = size;\n        this.tagsCode = tagsCode;\n        this.batchNum = batchNum;\n\n        this.nativeBuffer = buffer;\n        this.compactedOffset = compactedOffset;\n    }\n\n    public int getSize() {\n        return size;\n    }\n\n    public long getPos() {\n        return pos;\n    }\n\n    public long getTagsCode() {\n        return tagsCode;\n    }\n\n    public Long getValidTagsCodeAsLong() {\n        if (!isTagsCodeValid()) {\n            return null;\n        }\n        return tagsCode;\n    }\n\n    public boolean isTagsCodeValid() {\n        return !ConsumeQueueExt.isExtAddr(tagsCode);\n    }\n\n    public ConsumeQueueExt.CqExtUnit getCqExtUnit() {\n        return cqExtUnit;\n    }\n\n    public void setCqExtUnit(ConsumeQueueExt.CqExtUnit cqExtUnit) {\n        this.cqExtUnit = cqExtUnit;\n    }\n\n    public void setTagsCode(long tagsCode) {\n        this.tagsCode = tagsCode;\n    }\n\n    public long getQueueOffset() {\n        return queueOffset;\n    }\n\n    public short getBatchNum() {\n        return batchNum;\n    }\n\n    public void correctCompactOffset(int correctedOffset) {\n        this.nativeBuffer.putInt(correctedOffset);\n    }\n\n    public int getCompactedOffset() {\n        return compactedOffset;\n    }\n\n    @Override\n    public String toString() {\n        return \"CqUnit{\" +\n                \"queueOffset=\" + queueOffset +\n                \", size=\" + size +\n                \", pos=\" + pos +\n                \", batchNum=\" + batchNum +\n                \", tagsCode=\" + tagsCode +\n                \", compactedOffset=\" + compactedOffset +\n                '}';\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/DispatchEntry.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.queue;\n\nimport java.nio.charset.StandardCharsets;\nimport javax.annotation.Nonnull;\nimport org.apache.rocketmq.store.DispatchRequest;\n\n/**\n * Use Record when Java 16 is available\n */\npublic class DispatchEntry {\n    public byte[] topic;\n    public int queueId;\n    public long queueOffset;\n    public long commitLogOffset;\n    public int messageSize;\n    public long tagCode;\n    public long storeTimestamp;\n\n    public static DispatchEntry from(@Nonnull DispatchRequest request) {\n        DispatchEntry entry = new DispatchEntry();\n        entry.topic = request.getTopic().getBytes(StandardCharsets.UTF_8);\n        entry.queueId = request.getQueueId();\n        entry.queueOffset = request.getConsumeQueueOffset();\n        entry.commitLogOffset = request.getCommitLogOffset();\n        entry.messageSize = request.getMsgSize();\n        entry.tagCode = request.getTagsCode();\n        entry.storeTimestamp = request.getStoreTimestamp();\n        return entry;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/FileQueueLifeCycle.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.queue;\n\nimport org.apache.rocketmq.store.Swappable;\n\n/**\n * FileQueueLifeCycle contains life cycle methods of ConsumerQueue that is directly implemented by FILE.\n */\npublic interface FileQueueLifeCycle extends Swappable {\n    /**\n     * Load from file.\n     * @return true if loaded successfully.\n     */\n    boolean load();\n\n    /**\n     * Recover from file.\n     */\n    void recover();\n\n    /**\n     * Check files.\n     */\n    void checkSelf();\n\n    /**\n     * Flush cache to file.\n     * @param flushLeastPages  the minimum number of pages to be flushed\n     * @return true if any data has been flushed.\n     */\n    boolean flush(int flushLeastPages);\n\n    /**\n     * Destroy files.\n     */\n    void destroy();\n\n    /**\n     * Truncate dirty logic files starting at max commit log position.\n     * @param maxCommitLogPos max commit log position\n     */\n    void truncateDirtyLogicFiles(long maxCommitLogPos);\n\n    /**\n     * Delete expired files ending at min commit log position.\n     * @param minCommitLogPos min commit log position\n     * @return deleted file numbers.\n     */\n    int deleteExpiredFile(long minCommitLogPos);\n\n    /**\n     * Roll to next file.\n     * @param nextBeginOffset next begin offset\n     * @return the beginning offset of the next file\n     */\n    long rollNextFile(final long nextBeginOffset);\n\n    /**\n     * Is the first file available?\n     * @return true if it's available\n     */\n    boolean isFirstFileAvailable();\n\n    /**\n     * Does the first file exist?\n     *\n     * @return true if it exists\n     */\n    boolean isFirstFileExist();\n\n    boolean shutdown();\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/MultiDispatchUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.queue;\n\nimport java.util.Map;\n\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\n\npublic class MultiDispatchUtils {\n\n    public static String lmqQueueKey(String queueName) {\n        StringBuilder keyBuilder = new StringBuilder();\n        keyBuilder.append(queueName);\n        keyBuilder.append('-');\n        int queueId = 0;\n        keyBuilder.append(queueId);\n        return keyBuilder.toString();\n    }\n\n    public static boolean isNeedHandleMultiDispatch(MessageStoreConfig messageStoreConfig, String topic) {\n        return messageStoreConfig.isEnableMultiDispatch()\n            && !topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)\n            && !topic.startsWith(TopicValidator.SYSTEM_TOPIC_PREFIX)\n            && !topic.equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC);\n    }\n\n    public static boolean checkMultiDispatchQueue(MessageStoreConfig messageStoreConfig, DispatchRequest dispatchRequest) {\n        if (!isNeedHandleMultiDispatch(messageStoreConfig, dispatchRequest.getTopic())) {\n            return false;\n        }\n        Map<String, String> prop = dispatchRequest.getPropertiesMap();\n        if (prop == null || prop.isEmpty()) {\n            return false;\n        }\n        String multiDispatchQueue = prop.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);\n        String multiQueueOffset = prop.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET);\n        if (StringUtils.isBlank(multiDispatchQueue) || StringUtils.isBlank(multiQueueOffset)) {\n            return false;\n        }\n        return true;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.queue;\n\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\n\npublic interface OffsetInitializer {\n    long maxConsumeQueueOffset(String topic, int queueId) throws ConsumeQueueException;\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializerRocksDBImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.queue;\n\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.rocksdb.RocksDBException;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class OffsetInitializerRocksDBImpl implements OffsetInitializer {\n\n    private static final Logger LOGGER = LoggerFactory.getLogger(OffsetInitializerRocksDBImpl.class);\n\n    private final RocksDBConsumeQueueStore consumeQueueStore;\n\n    public OffsetInitializerRocksDBImpl(RocksDBConsumeQueueStore consumeQueueStore) {\n        this.consumeQueueStore = consumeQueueStore;\n    }\n\n    @Override\n    public long maxConsumeQueueOffset(String topic, int queueId) throws ConsumeQueueException {\n        try {\n            long offset = consumeQueueStore.getMaxOffsetInQueue(topic, queueId);\n            LOGGER.info(\"Look up RocksDB for max-offset of LMQ[{}:{}]: {}\", topic, queueId, offset);\n            return offset;\n        } catch (RocksDBException e) {\n            throw new ConsumeQueueException(e);\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.store.queue;\n\nimport com.google.common.base.Preconditions;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\n\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.utils.ConcurrentHashMapUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\n\n/**\n * QueueOffsetOperator is a component for operating offsets for queues.\n */\npublic class QueueOffsetOperator {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n\n    private ConcurrentMap<String, Long> topicQueueTable = new ConcurrentHashMap<>(1024);\n    private ConcurrentMap<String, Long> batchTopicQueueTable = new ConcurrentHashMap<>(1024);\n\n    /**\n     * {TOPIC}-{QUEUE_ID} --> NEXT Consume Queue Offset\n     */\n    private ConcurrentMap<String/* topic-queue-id */, Long/* offset */> lmqTopicQueueTable = new ConcurrentHashMap<>(1024);\n\n    public long getQueueOffset(String topicQueueKey) {\n        return ConcurrentHashMapUtils.computeIfAbsent(this.topicQueueTable, topicQueueKey, k -> 0L);\n    }\n\n    public Long getTopicQueueNextOffset(String topicQueueKey) {\n        return this.topicQueueTable.get(topicQueueKey);\n    }\n\n    public void increaseQueueOffset(String topicQueueKey, short messageNum) {\n        Long queueOffset = ConcurrentHashMapUtils.computeIfAbsent(this.topicQueueTable, topicQueueKey, k -> 0L);\n        topicQueueTable.put(topicQueueKey, queueOffset + messageNum);\n    }\n\n    public void updateQueueOffset(String topicQueueKey, long offset) {\n        this.topicQueueTable.put(topicQueueKey, offset);\n    }\n\n    public long getBatchQueueOffset(String topicQueueKey) {\n        return ConcurrentHashMapUtils.computeIfAbsent(this.batchTopicQueueTable, topicQueueKey, k -> 0L);\n    }\n\n    public void increaseBatchQueueOffset(String topicQueueKey, short messageNum) {\n        Long batchQueueOffset = ConcurrentHashMapUtils.computeIfAbsent(this.batchTopicQueueTable, topicQueueKey, k -> 0L);\n        this.batchTopicQueueTable.put(topicQueueKey, batchQueueOffset + messageNum);\n    }\n\n    public long getLmqOffset(String topic, int queueId, OffsetInitializer callback) throws ConsumeQueueException {\n        Preconditions.checkNotNull(callback, \"ConsumeQueueOffsetCallback cannot be null\");\n        String topicQueue = topic + \"-\" + queueId;\n        if (!lmqTopicQueueTable.containsKey(topicQueue)) {\n            // Load from RocksDB on cache miss.\n            Long prev = lmqTopicQueueTable.putIfAbsent(topicQueue, callback.maxConsumeQueueOffset(topic, queueId));\n            if (null != prev) {\n                log.error(\"[BUG] Data racing, lmqTopicQueueTable should NOT contain key={}\", topicQueue);\n            }\n        }\n        return lmqTopicQueueTable.get(topicQueue);\n    }\n\n    public void increaseLmqOffset(String topic, int queueId, short delta) throws ConsumeQueueException {\n        String topicQueue = topic + \"-\" + queueId;\n        if (!this.lmqTopicQueueTable.containsKey(topicQueue)) {\n            throw new ConsumeQueueException(String.format(\"Max offset of Queue[name=%s, id=%d] should have existed\", topic, queueId));\n        }\n        long prev = lmqTopicQueueTable.get(topicQueue);\n        this.lmqTopicQueueTable.compute(topicQueue, (k, offset) -> offset + delta);\n        long current = lmqTopicQueueTable.get(topicQueue);\n        log.debug(\"Max offset of LMQ[{}:{}] increased: {} --> {}\", topic, queueId, prev, current);\n    }\n\n    public long currentQueueOffset(String topicQueueKey) {\n        Long currentQueueOffset = this.topicQueueTable.get(topicQueueKey);\n        return currentQueueOffset == null ? 0L : currentQueueOffset;\n    }\n\n    public synchronized void remove(String topic, Integer queueId) {\n        String topicQueueKey = topic + \"-\" + queueId;\n        // Beware of thread-safety\n        this.topicQueueTable.remove(topicQueueKey);\n        this.batchTopicQueueTable.remove(topicQueueKey);\n        this.lmqTopicQueueTable.remove(topicQueueKey);\n\n        log.info(\"removeQueueFromTopicQueueTable OK Topic: {} QueueId: {}\", topic, queueId);\n    }\n\n    public void setTopicQueueTable(ConcurrentMap<String, Long> topicQueueTable) {\n        this.topicQueueTable = topicQueueTable;\n    }\n\n    public void setLmqTopicQueueTable(ConcurrentMap<String, Long> lmqTopicQueueTable) {\n        ConcurrentMap<String, Long> table = new ConcurrentHashMap<String, Long>(1024);\n        for (Map.Entry<String, Long> entry : lmqTopicQueueTable.entrySet()) {\n            if (MixAll.isLmq(entry.getKey())) {\n                table.put(entry.getKey(), entry.getValue());\n            }\n        }\n        this.lmqTopicQueueTable = table;\n    }\n\n    public ConcurrentMap<String, Long> getTopicQueueTable() {\n        return topicQueueTable;\n    }\n\n    public void setBatchTopicQueueTable(ConcurrentMap<String, Long> batchTopicQueueTable) {\n        this.batchTopicQueueTable = batchTopicQueueTable;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/ReferredIterator.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.queue;\n\nimport java.util.Iterator;\n\npublic interface ReferredIterator<T> extends Iterator<T> {\n\n    /**\n     * Release the referred resources.\n     */\n    void release();\n\n    T nextAndRelease();\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.queue;\n\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.ConsumeQueue;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.MessageFilter;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.rocksdb.RocksDBException;\n\npublic class RocksDBConsumeQueue implements ConsumeQueueInterface {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);\n\n    private final MessageStoreConfig messageStoreConfig;\n    private final RocksDBConsumeQueueStore consumeQueueStore;\n    private final String topic;\n    private final int queueId;\n\n    public RocksDBConsumeQueue(final MessageStoreConfig messageStoreConfig,\n        final RocksDBConsumeQueueStore consumeQueueStore,\n        final String topic, final int queueId) {\n        this.messageStoreConfig = messageStoreConfig;\n        this.consumeQueueStore = consumeQueueStore;\n        this.topic = topic;\n        this.queueId = queueId;\n    }\n\n    /**\n     * Only used to pass parameters when calling the destroy method\n     *\n     * @see RocksDBConsumeQueueStore#destroy(ConsumeQueueInterface)\n     */\n    public RocksDBConsumeQueue(final String topic, final int queueId) {\n        this(null, null, topic, queueId);\n    }\n\n    @Override\n    public boolean load() {\n        return true;\n    }\n\n    @Override\n    public void recover() {\n        // ignore\n    }\n\n    @Override\n    public void checkSelf() {\n        // ignore\n    }\n\n    @Override\n    public boolean flush(final int flushLeastPages) {\n        return true;\n    }\n\n    @Override\n    public void destroy() {\n        // ignore\n    }\n\n    @Override\n    public void truncateDirtyLogicFiles(long maxCommitLogPos) {\n        // ignored\n    }\n\n    @Override\n    public int deleteExpiredFile(long minCommitLogPos) {\n        return 0;\n    }\n\n    @Override\n    public long rollNextFile(long nextBeginOffset) {\n        return 0;\n    }\n\n    @Override\n    public boolean isFirstFileAvailable() {\n        return true;\n    }\n\n    @Override\n    public boolean isFirstFileExist() {\n        return true;\n    }\n\n    @Override\n    public void swapMap(int reserveNum, long forceSwapIntervalMs, long normalSwapIntervalMs) {\n        // ignore\n    }\n\n    @Override\n    public void cleanSwappedMap(long forceCleanSwapIntervalMs) {\n        // ignore\n    }\n\n    @Override\n    public long getMaxOffsetInQueue() {\n        try {\n            return this.consumeQueueStore.getMaxOffsetInQueue(topic, queueId);\n        } catch (RocksDBException e) {\n            ERROR_LOG.error(\"getMaxOffsetInQueue Failed. topic: {}, queueId: {}\", topic, queueId, e);\n            return 0;\n        }\n    }\n\n    @Override\n    public long getMessageTotalInQueue() {\n        try {\n            long maxOffsetInQueue = this.consumeQueueStore.getMaxOffsetInQueue(topic, queueId);\n            long minOffsetInQueue = this.consumeQueueStore.getMinOffsetInQueue(topic, queueId);\n            return maxOffsetInQueue - minOffsetInQueue;\n        } catch (RocksDBException e) {\n            ERROR_LOG.error(\"getMessageTotalInQueue Failed. topic: {}, queueId: {}, {}\", topic, queueId, e);\n        }\n        return -1;\n    }\n\n    /**\n     * We already implement it in RocksDBConsumeQueueStore.\n     * @see RocksDBConsumeQueueStore#getOffsetInQueueByTime\n     * @param timestamp timestamp\n     * @return\n     */\n    @Override\n    public long getOffsetInQueueByTime(long timestamp) {\n        return 0;\n    }\n\n    /**\n     * We already implement it in RocksDBConsumeQueueStore.\n     * @see RocksDBConsumeQueueStore#getOffsetInQueueByTime\n     * @param timestamp    timestamp\n     * @param boundaryType Lower or Upper\n     * @return\n     */\n    @Override\n    public long getOffsetInQueueByTime(long timestamp, BoundaryType boundaryType) {\n        return 0;\n    }\n\n    @Override\n    public long getMaxPhysicOffset() {\n        Long maxPhyOffset = this.consumeQueueStore.getMaxPhyOffsetInConsumeQueue(topic, queueId);\n        return maxPhyOffset == null ? -1 : maxPhyOffset;\n    }\n\n    @Override\n    public long getMinLogicOffset() {\n        return 0;\n    }\n\n    @Override\n    public CQType getCQType() {\n        return CQType.RocksDBCQ;\n    }\n\n    @Override\n    public long getTotalSize() {\n        // ignored\n        return 0;\n    }\n\n    @Override\n    public int getUnitSize() {\n        // attention: unitSize should equal to 'ConsumeQueue.CQ_STORE_UNIT_SIZE'\n        return ConsumeQueue.CQ_STORE_UNIT_SIZE;\n    }\n\n    /**\n     * Ignored, we already implement this method\n     * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable#getMinCqOffset(String, int)\n     */\n    @Override\n    public void correctMinOffset(long minCommitLogOffset) {\n\n    }\n\n    /**\n     * Ignored, in rocksdb mode, we build cq in RocksDBConsumeQueueStore\n     */\n    @Override\n    public void putMessagePositionInfoWrapper(DispatchRequest request) {\n\n    }\n\n    @Override\n    public void assignQueueOffset(QueueOffsetOperator queueOffsetOperator, MessageExtBrokerInner msg) throws RocksDBException {\n        String topicQueueKey = getTopic() + \"-\" + getQueueId();\n        Long queueOffset = queueOffsetOperator.getTopicQueueNextOffset(topicQueueKey);\n        if (queueOffset == null) {\n            // we will recover topic queue table from rocksdb when we use it.\n            queueOffset = this.consumeQueueStore.getMaxOffsetInQueue(topic, queueId);\n            queueOffsetOperator.updateQueueOffset(topicQueueKey, queueOffset);\n        }\n        msg.setQueueOffset(queueOffset);\n    }\n\n    @Override\n    public void increaseQueueOffset(QueueOffsetOperator queueOffsetOperator, MessageExtBrokerInner msg, short messageNum) {\n        String topicQueueKey = getTopic() + \"-\" + getQueueId();\n        queueOffsetOperator.increaseQueueOffset(topicQueueKey, messageNum);\n    }\n\n    /**\n     * It is CPU-intensive with many offline group\n     * Optimize by caching their estimated info\n     */\n    @Override\n    public long estimateMessageCount(long from, long to, MessageFilter filter) {\n\n        Pair<CqUnit, Long> fromUnit = getCqUnitAndStoreTime(from);\n        if (fromUnit == null || from >= to) {\n            return -1L;\n        }\n\n        if (to > getMaxOffsetInQueue()) {\n            to = getMaxOffsetInQueue();\n        }\n\n        int sampleCount = 0;\n        int sampleTotal = Math.min((int) (to - from), messageStoreConfig.getMaxConsumeQueueScan());\n\n        int matchCount = 0;\n        int matchTotal = messageStoreConfig.getSampleCountThreshold();\n\n        try {\n            ReferredIterator<CqUnit> iterator = this.iterateFrom(from, matchTotal);\n            while (iterator != null && iterator.hasNext() && sampleCount++ < sampleTotal) {\n                CqUnit cqUnit = iterator.next();\n                if (filter.isMatchedByConsumeQueue(\n                    cqUnit.getTagsCode(), cqUnit.getCqExtUnit())) {\n                    if (++matchCount > matchTotal) {\n                        sampleTotal = sampleCount;\n                        break;\n                    }\n                }\n            }\n        } catch (Throwable t) {\n            log.error(\"EstimateLag error, from={}, to={}\", from, to, t);\n        }\n\n        long result = sampleTotal == 0 ? 0 :\n            (long) ((to - from) * (matchCount / (sampleTotal * 1.0)));\n\n        if (log.isTraceEnabled()) {\n            log.trace(\"EstimateLag, topic={}, queueId={}, offset={}-{}, total={}, hit rate={}/{}({}%), result={}\",\n                topic, queueId, from, to, to - from,\n                matchCount, sampleCount, String.format(\"%.1f\", (double) matchCount * 100.0 / sampleCount), result);\n        }\n\n        return result;\n    }\n\n\n    @Override\n    public long getMinOffsetInQueue() {\n        try {\n            return this.consumeQueueStore.getMinOffsetInQueue(topic, queueId);\n        } catch (RocksDBException e) {\n            ERROR_LOG.error(\"getMinOffsetInQueue Failed. topic: {}, queueId: {}\", topic, queueId, e);\n            return -1;\n        }\n    }\n\n    private int pullNum(long cqOffset, long maxCqOffset) {\n        long diffLong = maxCqOffset - cqOffset;\n        if (diffLong < Integer.MAX_VALUE) {\n            return (int) diffLong;\n        }\n        return Integer.MAX_VALUE;\n    }\n\n    @Override\n    public ReferredIterator<CqUnit> iterateFrom(final long startIndex) {\n        long maxCqOffset = getMaxOffsetInQueue();\n        if (startIndex < maxCqOffset && startIndex >= 0) {\n            int num = pullNum(startIndex, maxCqOffset);\n            return new LargeRocksDBConsumeQueueIterator(startIndex, num);\n        }\n        return null;\n    }\n\n    @Override\n    public ReferredIterator<CqUnit> iterateFrom(long startIndex, int count) throws RocksDBException {\n        long maxCqOffset = getMaxOffsetInQueue();\n        if (startIndex < maxCqOffset) {\n            int num = Math.min((int)(maxCqOffset - startIndex), count);\n            return iterateFrom0(startIndex, num, maxCqOffset);\n        }\n        return null;\n    }\n\n    @Override\n    public CqUnit get(long index) {\n        Pair<CqUnit, Long> pair = getCqUnitAndStoreTime(index);\n        return pair == null ? null : pair.getObject1();\n    }\n\n    @Override\n    public Pair<CqUnit, Long> getCqUnitAndStoreTime(long index) {\n        ByteBuffer byteBuffer;\n        try {\n            byteBuffer = this.consumeQueueStore.get(topic, queueId, index);\n        } catch (RocksDBException e) {\n            ERROR_LOG.error(\"getUnitAndStoreTime Failed. topic: {}, queueId: {}\", topic, queueId, e);\n            return null;\n        }\n        if (byteBuffer == null || byteBuffer.remaining() < RocksDBConsumeQueueTable.CQ_UNIT_SIZE) {\n            return null;\n        }\n        long phyOffset = byteBuffer.getLong();\n        int size = byteBuffer.getInt();\n        long tagCode = byteBuffer.getLong();\n        long messageStoreTime = byteBuffer.getLong();\n        return new Pair<>(new CqUnit(index, phyOffset, size, tagCode), messageStoreTime);\n    }\n\n    @Override\n    public Pair<CqUnit, Long> getEarliestUnitAndStoreTime() {\n        try {\n            long minOffset = this.consumeQueueStore.getMinOffsetInQueue(topic, queueId);\n            return getCqUnitAndStoreTime(minOffset);\n        } catch (RocksDBException e) {\n            ERROR_LOG.error(\"getEarliestUnitAndStoreTime Failed. topic: {}, queueId: {}\", topic, queueId, e);\n        }\n        return null;\n    }\n\n    @Override\n    public CqUnit getEarliestUnit() {\n        Pair<CqUnit, Long> pair = getEarliestUnitAndStoreTime();\n        return pair == null ? null : pair.getObject1();\n    }\n\n    @Override\n    public CqUnit getLatestUnit() {\n        try {\n            long maxOffset = this.consumeQueueStore.getMaxOffsetInQueue(topic, queueId);\n            return get(maxOffset > 0 ? maxOffset - 1 : maxOffset);\n        } catch (RocksDBException e) {\n            ERROR_LOG.error(\"getLatestUnit Failed. topic: {}, queueId: {}, {}\", topic, queueId, e.getMessage());\n        }\n        return null;\n    }\n\n    @Override\n    public long getLastOffset() {\n        return getMaxPhysicOffset();\n    }\n\n    private ReferredIterator<CqUnit> iterateFrom0(\n        final long startIndex, final int count, final long maxOffset) throws RocksDBException {\n\n        if (messageStoreConfig.isIteratorWhenUseRocksdbConsumeQueue()) {\n            return new RocksDBReusableIterator(topic, queueId, startIndex, count, maxOffset);\n        }\n\n        List<ByteBuffer> byteBufferList = this.consumeQueueStore.rangeQuery(topic, queueId, startIndex, count);\n        if (byteBufferList == null || byteBufferList.isEmpty()) {\n            if (this.messageStoreConfig.isEnableRocksDBLog()) {\n                log.warn(\"iterateFrom0 - find nothing, startIndex:{}, count:{}\", startIndex, count);\n            }\n            return null;\n        }\n        return new RocksDBConsumeQueueIterator(byteBufferList, startIndex);\n    }\n\n    @Override\n    public String getTopic() {\n        return topic;\n    }\n\n    @Override\n    public int getQueueId() {\n        return queueId;\n    }\n\n    private class RocksDBReusableIterator implements ReferredIterator<CqUnit> {\n\n        private final String topic;\n        private final int queueId;\n        private long offset;\n        private final int count;\n        private final long maxOffset;\n\n        private int bufferIndex;\n        private List<ByteBuffer> buffers;\n\n        // offset + count <= max offset\n        public RocksDBReusableIterator(String topic, int queueId, long offset, int count, long maxOffset) {\n            this.topic = topic;\n            this.queueId = queueId;\n            this.offset = offset;\n            this.count = count;\n            this.maxOffset = maxOffset;\n\n            this.bufferIndex = 0;\n            this.buffers = new ArrayList<>(count);\n        }\n\n        @Override\n        public void release() {\n        }\n\n        @Override\n        public CqUnit nextAndRelease() {\n            try {\n                return next();\n            } finally {\n                release();\n            }\n        }\n\n        @Override\n        public boolean hasNext() {\n            return offset < maxOffset;\n        }\n\n        @Override\n        public CqUnit next() {\n            try {\n                if (buffers.isEmpty() || bufferIndex >= buffers.size()) {\n                    int batchSize = (int) Math.min(count, maxOffset - offset);\n                    if (batchSize == 0) {\n                        return null;\n                    } else {\n                        bufferIndex = 0;\n                        buffers = consumeQueueStore.rangeQuery(topic, queueId, offset, batchSize);\n                    }\n                }\n                if (bufferIndex < buffers.size()) {\n                    ByteBuffer buffer = buffers.get(bufferIndex++);\n                    return new CqUnit(offset++, buffer.getLong(), buffer.getInt(), buffer.getLong());\n                }\n            } catch (Throwable t) {\n                log.error(\"RocksDB reusable iterator search error, \" +\n                    \"topic={}, queueId={}, offset={}, count={}\", topic, queueId, offset, count, maxOffset, t);\n            }\n            return null;\n        }\n    }\n\n    private class RocksDBConsumeQueueIterator implements ReferredIterator<CqUnit> {\n        private final List<ByteBuffer> byteBufferList;\n        private final long startIndex;\n        private final int totalCount;\n        private int currentIndex;\n\n        public RocksDBConsumeQueueIterator(final List<ByteBuffer> byteBufferList, final long startIndex) {\n            this.byteBufferList = byteBufferList;\n            this.startIndex = startIndex;\n            this.totalCount = byteBufferList.size();\n            this.currentIndex = 0;\n        }\n\n        @Override\n        public boolean hasNext() {\n            return this.currentIndex < this.totalCount;\n        }\n\n        @Override\n        public CqUnit next() {\n            if (!hasNext()) {\n                return null;\n            }\n            final int currentIndex = this.currentIndex;\n            final ByteBuffer byteBuffer = this.byteBufferList.get(currentIndex);\n            CqUnit cqUnit = new CqUnit(this.startIndex + currentIndex, byteBuffer.getLong(), byteBuffer.getInt(), byteBuffer.getLong());\n            this.currentIndex++;\n            return cqUnit;\n        }\n\n        @Override\n        public void remove() {\n            throw new UnsupportedOperationException(\"remove\");\n        }\n\n        @Override\n        public void release() {\n        }\n\n        @Override\n        public CqUnit nextAndRelease() {\n            try {\n                return next();\n            } finally {\n                release();\n            }\n        }\n    }\n\n    private class LargeRocksDBConsumeQueueIterator implements ReferredIterator<CqUnit> {\n        private final long startIndex;\n        private final int totalCount;\n        private int currentIndex;\n\n        public LargeRocksDBConsumeQueueIterator(final long startIndex, final int num) {\n            this.startIndex = startIndex;\n            this.totalCount = num;\n            this.currentIndex = 0;\n        }\n\n        @Override\n        public boolean hasNext() {\n            return this.currentIndex < this.totalCount;\n        }\n\n\n        @Override\n        public CqUnit next() {\n            if (!hasNext()) {\n                return null;\n            }\n\n            final ByteBuffer byteBuffer;\n            try {\n                byteBuffer = consumeQueueStore.get(topic, queueId, startIndex + currentIndex);\n            } catch (RocksDBException e) {\n                ERROR_LOG.error(\"get cq from rocksdb failed. topic: {}, queueId: {}\", topic, queueId, e);\n                return null;\n            }\n            if (byteBuffer == null || byteBuffer.remaining() < RocksDBConsumeQueueTable.CQ_UNIT_SIZE) {\n                return null;\n            }\n            CqUnit cqUnit = new CqUnit(this.startIndex + currentIndex, byteBuffer.getLong(), byteBuffer.getInt(), byteBuffer.getLong());\n            this.currentIndex++;\n            return cqUnit;\n        }\n\n        @Override\n        public void remove() {\n            throw new UnsupportedOperationException(\"remove\");\n        }\n\n        @Override\n        public void release() {\n        }\n\n        @Override\n        public CqUnit nextAndRelease() {\n            try {\n                return next();\n            } finally {\n                release();\n            }\n        }\n    }\n\n    public void initializeWithOffset(long offset, long minPhyOffset) {\n        log.info(\"RocksDBConsumeQueue initializeWithOffset topic={}, queueId={}, offset={}, oldMax={}, oldMin={}\",\n            topic, queueId, offset, getMaxOffsetInQueue(), getMinOffsetInQueue());\n        try {\n            // clean the expired cqUnit and offset\n            consumeQueueStore.cleanExpired(minPhyOffset);\n\n            // update the max and min offset\n            this.consumeQueueStore.updateCqOffset(topic, queueId, 0L, offset - 1, true);\n            // set phyOffset to 0, min cq offset will be lazy corrected to max cq Offset + 1\n            this.consumeQueueStore.updateCqOffset(topic, queueId, 0L, offset, false);\n        } catch (RocksDBException e) {\n            ERROR_LOG.error(\"RocksDBConsumeQueue initializeWithOffset Failed. topic={}, queueId={}, offset={}\", topic, queueId, offset, e);\n        }\n    }\n\n    @Override\n    public boolean shutdown() {\n        return true;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.queue;\n\nimport static org.apache.rocketmq.common.config.AbstractRocksDBStorage.CTRL_1;\n\nimport io.netty.util.internal.PlatformDependent;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.function.Consumer;\nimport java.util.function.Function;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.ConsumeQueue;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.queue.offset.OffsetEntry;\nimport org.apache.rocketmq.store.queue.offset.OffsetEntryType;\nimport org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage;\nimport org.rocksdb.ColumnFamilyHandle;\nimport org.rocksdb.RocksDBException;\nimport org.rocksdb.RocksIterator;\nimport org.rocksdb.WriteBatch;\n\npublic class RocksDBConsumeQueueOffsetTable {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);\n    private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME);\n\n    private static final byte[] MAX_BYTES = \"max\".getBytes(StandardCharsets.UTF_8);\n    private static final byte[] MIN_BYTES = \"min\".getBytes(StandardCharsets.UTF_8);\n\n    /**\n     * Rocksdb ConsumeQueue's Offset unit. Format:\n     *\n     * <pre>\n     * ┌─────────────────────────┬───────────┬───────────────────────┬───────────┬───────────┬───────────┬─────────────┐\n     * │ Topic Bytes Array Size  │  CTRL_1   │   Topic Bytes Array   │  CTRL_1   │  Max(Min) │  CTRL_1   │   QueueId   │\n     * │        (4 Bytes)        │ (1 Bytes) │       (n Bytes)       │ (1 Bytes) │ (3 Bytes) │ (1 Bytes) │  (4 Bytes)  │\n     * ├─────────────────────────┴───────────┴───────────────────────┴───────────┴───────────┴───────────┴─────────────┤\n     * │                                                    Key Unit                                                   │\n     * │                                                                                                               │\n     * </pre>\n     *\n     * <pre>\n     * ┌─────────────────────────────┬────────────────────────┐\n     * │  CommitLog Physical Offset  │   ConsumeQueue Offset  │\n     * │        (8 Bytes)            │    (8 Bytes)           │\n     * ├─────────────────────────────┴────────────────────────┤\n     * │                     Value Unit                       │\n     * │                                                      │\n     * </pre>\n     * ConsumeQueue's Offset unit. Size: CommitLog Physical Offset(8) + ConsumeQueue Offset(8) =  16 Bytes\n     */\n    static final int OFFSET_PHY_OFFSET = 0;\n    static final int OFFSET_CQ_OFFSET = 8;\n    /**\n     * ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────┬─────────────┐\n     * │ Topic Bytes Array Size  │  CTRL_1   │  CTRL_1   │  Max(Min) │  CTRL_1   │   QueueId   │\n     * │        (4 Bytes)        │ (1 Bytes) │ (1 Bytes) │ (3 Bytes) │ (1 Bytes) │  (4 Bytes)  │\n     * ├─────────────────────────┴───────────┴───────────┴───────────┴───────────┴─────────────┤\n     */\n    public static final int OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES = 4 + 1 + 1 + 3 + 1 + 4;\n    private static final int OFFSET_VALUE_LENGTH = 8 + 8;\n\n    /**\n     * ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────┐\n     * │ Topic Bytes Array Size  │  CTRL_1   │  CTRL_1   │  Max(Min) │  CTRL_1   │\n     * │        (4 Bytes)        │ (1 Bytes) │ (1 Bytes) │ (3 Bytes) │ (1 Bytes) │\n     * ├─────────────────────────┴───────────┴───────────┴───────────┴───────────┤\n     */\n    public static final int OFFSET_KEY_LENGTH_WITHOUT_TOPIC_QUEUE_ID_BYTES = 4 + 1 + 1 + 3 + 1;\n\n    /**\n     * We use a new system topic='CHECKPOINT_TOPIC' to record the maxPhyOffset built by CQ dispatch thread.\n     *\n     * @see ConsumeQueueStore#getMaxPhyOffsetInConsumeQueue(), we use it to find the maxPhyOffset built by CQ dispatch thread.\n     * If we do not record the maxPhyOffset, it may take us a long time to start traversing from the head of\n     * RocksDBConsumeQueueOffsetTable to find it.\n     */\n    private static final String MAX_PHYSICAL_OFFSET_CHECKPOINT = TopicValidator.RMQ_SYS_ROCKSDB_OFFSET_TOPIC;\n    private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES = MAX_PHYSICAL_OFFSET_CHECKPOINT.getBytes(StandardCharsets.UTF_8);\n    private static final int INNER_CHECKPOINT_TOPIC_LEN = OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES.length;\n    private static final ByteBuffer INNER_CHECKPOINT_TOPIC = ByteBuffer.allocateDirect(INNER_CHECKPOINT_TOPIC_LEN);\n    private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_KEY = new byte[INNER_CHECKPOINT_TOPIC_LEN];\n    private final ByteBuffer maxPhyOffsetBB;\n\n    static {\n        buildOffsetKeyByteBuffer0(INNER_CHECKPOINT_TOPIC, MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES, 0, true);\n        INNER_CHECKPOINT_TOPIC.position(0).limit(INNER_CHECKPOINT_TOPIC_LEN);\n        INNER_CHECKPOINT_TOPIC.get(MAX_PHYSICAL_OFFSET_CHECKPOINT_KEY);\n    }\n\n    private final RocksDBConsumeQueueTable rocksDBConsumeQueueTable;\n    private final ConsumeQueueRocksDBStorage rocksDBStorage;\n    private final DefaultMessageStore messageStore;\n\n    private ColumnFamilyHandle offsetCFH;\n\n    /**\n     * Although we have already put max(min) consumeQueueOffset and physicalOffset in rocksdb, we still hope to get them\n     * from heap to avoid accessing rocksdb.\n     *\n     * @see ConsumeQueue#getMaxPhysicOffset(), maxPhysicOffset  --> topicQueueMaxCqOffset\n     * @see ConsumeQueue#getMinLogicOffset(),   minLogicOffset  --> topicQueueMinOffset\n     */\n    private final ConcurrentMap<String/* topic-queueId */, PhyAndCQOffset> topicQueueMinOffset;\n    private final ConcurrentMap<String/* topic-queueId */, Long> topicQueueMaxCqOffset;\n    private final AtomicInteger lmqCounter = new AtomicInteger(0);\n\n    public RocksDBConsumeQueueOffsetTable(RocksDBConsumeQueueTable rocksDBConsumeQueueTable,\n        ConsumeQueueRocksDBStorage rocksDBStorage, DefaultMessageStore messageStore) {\n        this.rocksDBConsumeQueueTable = rocksDBConsumeQueueTable;\n        this.rocksDBStorage = rocksDBStorage;\n        this.messageStore = messageStore;\n        this.topicQueueMinOffset = new ConcurrentHashMap<>(1024);\n        this.topicQueueMaxCqOffset = new ConcurrentHashMap<>(1024);\n\n        this.maxPhyOffsetBB = ByteBuffer.allocateDirect(8);\n    }\n\n    public void load() {\n        this.offsetCFH = this.rocksDBStorage.getOffsetCFHandle();\n        loadMaxConsumeQueueOffsets();\n    }\n\n    public Set<Integer> scanAllQueueIdInTopic(String topic) throws RocksDBException {\n        Set<Integer> queueIdSet = new HashSet<>();\n        byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);\n        ByteBuffer byteBuffer = ByteBuffer.allocate(OFFSET_KEY_LENGTH_WITHOUT_TOPIC_QUEUE_ID_BYTES + topicBytes.length);\n        byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1).put(MAX_BYTES).put(CTRL_1);\n        byteBuffer.flip();\n        byte[] prefix = byteBuffer.array();\n        rocksDBStorage.iterate(offsetCFH, prefix, (keyBytes, unused) -> {\n            ByteBuffer keyBuffer = ByteBuffer.wrap(keyBytes);\n            keyBuffer.position(prefix.length);\n            int queueId = keyBuffer.getInt();\n            queueIdSet.add(queueId);\n        });\n        return queueIdSet;\n    }\n\n    private void loadMaxConsumeQueueOffsets() {\n        lmqCounter.set(0);\n        Function<OffsetEntry, Boolean> predicate = entry -> entry.type == OffsetEntryType.MAXIMUM;\n        Consumer<OffsetEntry> fn = entry -> {\n            topicQueueMaxCqOffset.putIfAbsent(entry.topic + \"-\" + entry.queueId, entry.offset);\n            if (MixAll.isLmq(entry.topic)) {\n                lmqCounter.incrementAndGet();\n            }\n            log.info(\"LoadMaxConsumeQueueOffsets Max {}:{} --> {}|{}\", entry.topic, entry.queueId, entry.offset, entry.commitLogOffset);\n        };\n        try {\n            forEach(predicate, fn);\n            log.info(\"lmq count from maxConsumeQueueOffset table. {}\", lmqCounter.get());\n        } catch (RocksDBException e) {\n            log.error(\"Failed to maximum consume queue offset\", e);\n        }\n    }\n\n    public void forEach(Function<OffsetEntry, Boolean> predicate, Consumer<OffsetEntry> fn) throws RocksDBException {\n        try (RocksIterator iterator = this.rocksDBStorage.seekOffsetCF()) {\n            if (null == iterator) {\n                return;\n            }\n\n            int keyBufferCapacity = 256;\n            iterator.seekToFirst();\n            ByteBuffer keyBuffer = ByteBuffer.allocateDirect(keyBufferCapacity);\n            ByteBuffer valueBuffer = ByteBuffer.allocateDirect(16);\n            while (iterator.isValid()) {\n                // parse key buffer according to key layout\n                keyBuffer.clear(); // clear position and limit before reuse\n                int total = iterator.key(keyBuffer);\n                if (total > keyBufferCapacity) {\n                    keyBufferCapacity = total;\n                    PlatformDependent.freeDirectBuffer(keyBuffer);\n                    keyBuffer = ByteBuffer.allocateDirect(keyBufferCapacity);\n                    continue;\n                }\n\n                if (keyBuffer.remaining() <= OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES) {\n                    iterator.next();\n                    ROCKSDB_LOG.warn(\"Malformed Key/Value pair\");\n                    continue;\n                }\n\n                int topicLength = keyBuffer.getInt();\n                byte ctrl1 = keyBuffer.get();\n                assert ctrl1 == CTRL_1;\n\n                byte[] topicBytes = new byte[topicLength];\n                keyBuffer.get(topicBytes);\n                ctrl1 = keyBuffer.get();\n                assert ctrl1 == CTRL_1;\n                String topic = new String(topicBytes, StandardCharsets.UTF_8);\n\n                byte[] minMax = new byte[3];\n                keyBuffer.get(minMax);\n                OffsetEntryType entryType;\n                if (Arrays.equals(minMax, MAX_BYTES)) {\n                    entryType = OffsetEntryType.MAXIMUM;\n                } else {\n                    entryType = OffsetEntryType.MINIMUM;\n                }\n                ctrl1 = keyBuffer.get();\n                assert ctrl1 == CTRL_1;\n\n                assert keyBuffer.remaining() == Integer.BYTES;\n                int queueId = keyBuffer.getInt();\n\n                // Read and parse value buffer according to value layout\n                valueBuffer.clear(); // clear position and limit before reuse\n                total = iterator.value(valueBuffer);\n                if (total != Long.BYTES + Long.BYTES) {\n                    // Skip system checkpoint topic as its value is only 8 bytes\n                    iterator.next();\n                    continue;\n                }\n                long commitLogOffset = valueBuffer.getLong();\n                long consumeOffset = valueBuffer.getLong();\n\n                OffsetEntry entry = new OffsetEntry();\n                entry.topic = topic;\n                entry.queueId = queueId;\n                entry.type = entryType;\n                entry.offset = consumeOffset;\n                entry.commitLogOffset = commitLogOffset;\n                if (predicate.apply(entry)) {\n                    fn.accept(entry);\n                }\n                iterator.next();\n            }\n            // clean up direct buffers\n            PlatformDependent.freeDirectBuffer(keyBuffer);\n            PlatformDependent.freeDirectBuffer(valueBuffer);\n        }\n    }\n\n    public void putMaxPhyAndCqOffset(final Map<ByteBuffer, Pair<ByteBuffer, DispatchEntry>> tempTopicQueueMaxOffsetMap,\n        final WriteBatch writeBatch, final long maxPhyOffset) throws RocksDBException {\n        for (Map.Entry<ByteBuffer, Pair<ByteBuffer, DispatchEntry>> entry : tempTopicQueueMaxOffsetMap.entrySet()) {\n            writeBatch.put(this.offsetCFH, entry.getKey(), entry.getValue().getObject1());\n        }\n\n        appendMaxPhyOffset(writeBatch, maxPhyOffset);\n    }\n\n    public void putHeapMaxCqOffset(final Map<ByteBuffer, Pair<ByteBuffer, DispatchEntry>> tempTopicQueueMaxOffsetMap) {\n        for (Map.Entry<ByteBuffer, Pair<ByteBuffer, DispatchEntry>> entry : tempTopicQueueMaxOffsetMap.entrySet()) {\n            DispatchEntry dispatchEntry = entry.getValue().getObject2();\n            String topic = new String(dispatchEntry.topic, StandardCharsets.UTF_8);\n            putHeapMaxCqOffset(topic, dispatchEntry.queueId, dispatchEntry.queueOffset);\n        }\n    }\n\n    /**\n     * When topic is deleted, we clean up its offset info in rocksdb.\n     *\n     * @param topic\n     * @param queueId\n     * @throws RocksDBException\n     */\n    public void destroyOffset(String topic, int queueId, WriteBatch writeBatch) throws RocksDBException {\n        final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);\n        final ByteBuffer minOffsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, false);\n        byte[] minOffsetBytes = this.rocksDBStorage.getOffset(minOffsetKey.array());\n        Long startCQOffset = (minOffsetBytes != null) ? ByteBuffer.wrap(minOffsetBytes).getLong(OFFSET_CQ_OFFSET) : null;\n\n        final ByteBuffer maxOffsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, true);\n        byte[] maxOffsetBytes = this.rocksDBStorage.getOffset(maxOffsetKey.array());\n        Long endCQOffset = (maxOffsetBytes != null) ? ByteBuffer.wrap(maxOffsetBytes).getLong(OFFSET_CQ_OFFSET) : null;\n\n        writeBatch.delete(this.offsetCFH, minOffsetKey.array());\n        writeBatch.delete(this.offsetCFH, maxOffsetKey.array());\n\n        String topicQueueId = buildTopicQueueId(topic, queueId);\n        removeHeapMinCqOffset(topicQueueId);\n        removeHeapMaxCqOffset(topicQueueId);\n\n        log.info(\"RocksDB offset table delete topic: {}, queueId: {}, minOffset: {}, maxOffset: {}\", topic, queueId,\n            startCQOffset, endCQOffset);\n    }\n\n    private void appendMaxPhyOffset(final WriteBatch writeBatch, final long maxPhyOffset) throws RocksDBException {\n        final ByteBuffer maxPhyOffsetBB = this.maxPhyOffsetBB;\n        maxPhyOffsetBB.position(0).limit(8);\n        maxPhyOffsetBB.putLong(maxPhyOffset);\n        maxPhyOffsetBB.flip();\n\n        INNER_CHECKPOINT_TOPIC.position(0).limit(INNER_CHECKPOINT_TOPIC_LEN);\n        writeBatch.put(this.offsetCFH, INNER_CHECKPOINT_TOPIC, maxPhyOffsetBB);\n    }\n\n    public long getMaxPhyOffset() throws RocksDBException {\n        byte[] valueBytes = this.rocksDBStorage.getOffset(MAX_PHYSICAL_OFFSET_CHECKPOINT_KEY);\n        if (valueBytes == null) {\n            return 0;\n        }\n        ByteBuffer valueBB = ByteBuffer.wrap(valueBytes);\n        return valueBB.getLong(0);\n    }\n\n    /**\n     * Traverse the offset table to find dirty topic\n     *\n     * @param existTopicSet\n     * @return\n     */\n    public Map<String, Set<Integer>> iterateOffsetTable2FindDirty(final Set<String> existTopicSet) {\n        Map<String/* topic */, Set<Integer/* queueId */>> topicQueueIdToBeDeletedMap = new HashMap<>();\n\n        try (RocksIterator iterator = rocksDBStorage.seekOffsetCF()) {\n            if (iterator == null) {\n                return topicQueueIdToBeDeletedMap;\n            }\n            for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) {\n                byte[] key = iterator.key();\n                byte[] value = iterator.value();\n                if (key == null || key.length <= OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES\n                    || value == null || value.length != OFFSET_VALUE_LENGTH) {\n                    continue;\n                }\n                ByteBuffer keyBB = ByteBuffer.wrap(key);\n                int topicLen = keyBB.getInt(0);\n                byte[] topicBytes = new byte[topicLen];\n                /*\n                 * \"Topic Bytes Array Size\" + \"CTRL_1\" = 4 + 1\n                 */\n                keyBB.position(4 + 1);\n                keyBB.get(topicBytes);\n                String topic = new String(topicBytes, StandardCharsets.UTF_8);\n                if (TopicValidator.isSystemTopic(topic)) {\n                    continue;\n                }\n\n                // LMQ topic offsets should NOT be removed\n                if (MixAll.isLmq(topic)) {\n                    continue;\n                }\n\n                /*\n                 * \"Topic Bytes Array Size\" + \"CTRL_1\" + \"Topic Bytes Array\" + \"CTRL_1\"  + \"Max(min)\" + \"CTRL_1\"\n                 *  = 4 + 1 + topicLen + 1 + 3 + 1\n                 */\n                int queueId = keyBB.getInt(4 + 1 + topicLen + 1 + 3 + 1);\n\n                if (!existTopicSet.contains(topic)) {\n                    ByteBuffer valueBB = ByteBuffer.wrap(value);\n                    long cqOffset = valueBB.getLong(OFFSET_CQ_OFFSET);\n\n                    Set<Integer> topicQueueIdSet = topicQueueIdToBeDeletedMap.get(topic);\n                    if (topicQueueIdSet == null) {\n                        Set<Integer> newSet = new HashSet<>();\n                        newSet.add(queueId);\n                        topicQueueIdToBeDeletedMap.put(topic, newSet);\n                    } else {\n                        topicQueueIdSet.add(queueId);\n                    }\n                    ERROR_LOG.info(\"RocksDBConsumeQueueOffsetTable has dirty cqOffset. topic: {}, queueId: {}, cqOffset: {}\",\n                        topic, queueId, cqOffset);\n                }\n            }\n        } catch (Exception e) {\n            ERROR_LOG.error(\"iterateOffsetTable2MarkDirtyCQ Failed.\", e);\n        }\n        return topicQueueIdToBeDeletedMap;\n    }\n\n    public Long getMaxCqOffset(String topic, int queueId) throws RocksDBException {\n        Long maxCqOffset = getHeapMaxCqOffset(topic, queueId);\n\n        if (maxCqOffset == null) {\n            final ByteBuffer byteBuffer = getMaxPhyAndCqOffsetInKV(topic, queueId);\n            maxCqOffset = (byteBuffer != null) ? byteBuffer.getLong(OFFSET_CQ_OFFSET) : null;\n            String topicQueueId = buildTopicQueueId(topic, queueId);\n            long offset = maxCqOffset != null ? maxCqOffset : -1L;\n            Long prev = this.topicQueueMaxCqOffset.putIfAbsent(topicQueueId, offset);\n            if (null == prev) {\n                ROCKSDB_LOG.info(\"Max offset of {} is initialized to {} according to RocksDB\", topicQueueId, offset);\n            }\n            if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) {\n                ROCKSDB_LOG.warn(\"updateMaxOffsetInQueue. {}, {}\", topicQueueId, offset);\n            }\n        }\n\n        return maxCqOffset;\n    }\n\n    /**\n     * truncate dirty offset in rocksdb\n     *\n     * @param offsetToTruncate\n     * @throws RocksDBException\n     */\n    public void truncateDirty(long offsetToTruncate) throws RocksDBException {\n        correctMaxPyhOffset(offsetToTruncate);\n\n        Function<OffsetEntry, Boolean> predicate = entry -> {\n            if (entry.type == OffsetEntryType.MINIMUM) {\n                return false;\n            }\n            // Normal entry offset MUST have the following inequality\n            // entry commit-log offset + message-size-in-bytes <= offsetToTruncate;\n            // otherwise, the consume queue contains dirty records to truncate;\n            //\n            // If the broker node is configured to use async-flush, it's possible consume queues contain\n            //  pointers to message records that is not flushed and lost during restart.\n            return entry.commitLogOffset >= offsetToTruncate;\n        };\n\n        Consumer<OffsetEntry> fn = entry -> {\n            try {\n                truncateDirtyOffset(entry.topic, entry.queueId);\n            } catch (RocksDBException e) {\n                log.error(\"Failed to truncate maximum offset of consume queue[topic={}, queue-id={}]\",\n                    entry.topic, entry.queueId, e);\n            }\n        };\n\n        forEach(predicate, fn);\n    }\n\n    private Pair<Boolean, Long> isMinOffsetOk(final String topic, final int queueId,\n        final long minPhyOffset) throws RocksDBException {\n        PhyAndCQOffset phyAndCQOffset = getHeapMinOffset(topic, queueId);\n        if (phyAndCQOffset != null) {\n            final long phyOffset = phyAndCQOffset.getPhyOffset();\n            final long cqOffset = phyAndCQOffset.getCqOffset();\n\n            return (phyOffset >= minPhyOffset) ? new Pair<>(true, cqOffset) : new Pair<>(false, cqOffset);\n        }\n        ByteBuffer byteBuffer = getMinPhyAndCqOffsetInKV(topic, queueId);\n        if (byteBuffer == null) {\n            return new Pair<>(false, 0L);\n        }\n        final long phyOffset = byteBuffer.getLong(OFFSET_PHY_OFFSET);\n        final long cqOffset = byteBuffer.getLong(OFFSET_CQ_OFFSET);\n        if (phyOffset >= minPhyOffset) {\n            String topicQueueId = buildTopicQueueId(topic, queueId);\n            PhyAndCQOffset newPhyAndCQOffset = new PhyAndCQOffset(phyOffset, cqOffset);\n            this.topicQueueMinOffset.putIfAbsent(topicQueueId, newPhyAndCQOffset);\n            if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) {\n                ROCKSDB_LOG.warn(\"updateMinOffsetInQueue. {}, {}\", topicQueueId, newPhyAndCQOffset);\n            }\n            return new Pair<>(true, cqOffset);\n        }\n        return new Pair<>(false, cqOffset);\n    }\n\n    private void truncateDirtyOffset(String topic, int queueId) throws RocksDBException {\n        final ByteBuffer byteBuffer = getMaxPhyAndCqOffsetInKV(topic, queueId);\n        if (byteBuffer == null) {\n            return;\n        }\n\n        long maxPhyOffset = byteBuffer.getLong(OFFSET_PHY_OFFSET);\n        long maxCqOffset = byteBuffer.getLong(OFFSET_CQ_OFFSET);\n        long maxPhyOffsetInCQ = getMaxPhyOffset();\n\n        if (maxPhyOffset >= maxPhyOffsetInCQ) {\n            correctMaxCqOffset(topic, queueId, maxCqOffset, maxPhyOffsetInCQ);\n            Long newMaxCqOffset = getHeapMaxCqOffset(topic, queueId);\n            ROCKSDB_LOG.warn(\"truncateDirtyLogicFile topic: {}, queueId: {} from {} to {}\", topic, queueId,\n                maxPhyOffset, newMaxCqOffset);\n        }\n    }\n\n    private void correctMaxPyhOffset(long maxPhyOffset) throws RocksDBException {\n        if (!this.rocksDBStorage.hold()) {\n            return;\n        }\n        try (WriteBatch writeBatch = new WriteBatch()) {\n            long oldMaxPhyOffset = getMaxPhyOffset();\n            if (oldMaxPhyOffset <= maxPhyOffset) {\n                return;\n            }\n            log.info(\"correctMaxPyhOffset, oldMaxPhyOffset={}, newMaxPhyOffset={}\", oldMaxPhyOffset, maxPhyOffset);\n            appendMaxPhyOffset(writeBatch, maxPhyOffset);\n            this.rocksDBStorage.batchPut(writeBatch);\n        } catch (RocksDBException e) {\n            ERROR_LOG.error(\"correctMaxPyhOffset Failed.\", e);\n            throw e;\n        } finally {\n            this.rocksDBStorage.release();\n        }\n    }\n\n    public long getMinCqOffset(String topic, int queueId) throws RocksDBException {\n        final long minPhyOffset = this.messageStore.getMinPhyOffset();\n        Pair<Boolean, Long> pair = isMinOffsetOk(topic, queueId, minPhyOffset);\n        final long cqOffset = pair.getObject2();\n        if (!pair.getObject1() && correctMinCqOffset(topic, queueId, cqOffset, minPhyOffset)) {\n            PhyAndCQOffset phyAndCQOffset = getHeapMinOffset(topic, queueId);\n            if (phyAndCQOffset != null) {\n                if (this.messageStore.getMessageStoreConfig().isEnableRocksDBLog()) {\n                    ROCKSDB_LOG.warn(\"getMinOffsetInQueue miss heap. topic: {}, queueId: {}, old: {}, new: {}\",\n                        topic, queueId, cqOffset, phyAndCQOffset);\n                }\n                return phyAndCQOffset.getCqOffset();\n            }\n        }\n        return cqOffset;\n    }\n\n    public Long getMaxPhyOffset(String topic, int queueId) {\n        try {\n            ByteBuffer byteBuffer = getMaxPhyAndCqOffsetInKV(topic, queueId);\n            if (byteBuffer != null) {\n                return byteBuffer.getLong(OFFSET_PHY_OFFSET);\n            }\n        } catch (Exception e) {\n            ERROR_LOG.info(\"getMaxPhyOffset error. topic: {}, queueId: {}\", topic, queueId);\n        }\n        return null;\n    }\n\n    private ByteBuffer getMinPhyAndCqOffsetInKV(String topic, int queueId) throws RocksDBException {\n        return getPhyAndCqOffsetInKV(topic, queueId, false);\n    }\n\n    private ByteBuffer getMaxPhyAndCqOffsetInKV(String topic, int queueId) throws RocksDBException {\n        return getPhyAndCqOffsetInKV(topic, queueId, true);\n    }\n\n    private ByteBuffer getPhyAndCqOffsetInKV(String topic, int queueId, boolean max) throws RocksDBException {\n        final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);\n        final ByteBuffer keyBB = buildOffsetKeyByteBuffer(topicBytes, queueId, max);\n\n        byte[] value = this.rocksDBStorage.getOffset(keyBB.array());\n        return (value != null) ? ByteBuffer.wrap(value) : null;\n    }\n\n    private String buildTopicQueueId(final String topic, final int queueId) {\n        return topic + \"-\" + queueId;\n    }\n\n    private void putHeapMinCqOffset(final String topic, final int queueId, final long minPhyOffset,\n        final long minCQOffset) {\n        String topicQueueId = buildTopicQueueId(topic, queueId);\n        PhyAndCQOffset phyAndCQOffset = new PhyAndCQOffset(minPhyOffset, minCQOffset);\n        this.topicQueueMinOffset.put(topicQueueId, phyAndCQOffset);\n    }\n\n    private void putHeapMaxCqOffset(final String topic, final int queueId, final long maxOffset) {\n        String topicQueueId = buildTopicQueueId(topic, queueId);\n        Long prev = this.topicQueueMaxCqOffset.put(topicQueueId, maxOffset);\n        if (prev != null && prev > maxOffset) {\n            ERROR_LOG.error(\"Max offset of consume-queue[topic={}, queue-id={}] regressed. prev-max={}, current-max={}\",\n                topic, queueId, prev, maxOffset);\n        }\n        if (prev != null && prev == -1 && MixAll.isLmq(topic)) {\n            lmqCounter.incrementAndGet();\n        }\n        if (null == prev && MixAll.isLmq(topic)) {\n            // this usually happens when broker exits abnormally, do nothing here and wait for the next scan to delete it.\n            ERROR_LOG.error(\"probably recover a lmq which was already deleted. lmq:{}, maxOffset:{}\", topic, maxOffset);\n            lmqCounter.incrementAndGet();\n        }\n    }\n\n    private PhyAndCQOffset getHeapMinOffset(final String topic, final int queueId) {\n        return this.topicQueueMinOffset.get(buildTopicQueueId(topic, queueId));\n    }\n\n    private Long getHeapMaxCqOffset(final String topic, final int queueId) {\n        String topicQueueId = buildTopicQueueId(topic, queueId);\n        return this.topicQueueMaxCqOffset.get(topicQueueId);\n    }\n\n    private PhyAndCQOffset removeHeapMinCqOffset(String topicQueueId) {\n        return this.topicQueueMinOffset.remove(topicQueueId);\n    }\n\n    private Long removeHeapMaxCqOffset(String topicQueueId) {\n        Long prev = this.topicQueueMaxCqOffset.remove(topicQueueId);\n        if (prev != null && topicQueueId.startsWith(MixAll.LMQ_PREFIX)) {\n            lmqCounter.decrementAndGet();\n        }\n        return prev;\n    }\n\n    public void updateCqOffset(final String topic, final int queueId, final long phyOffset,\n        final long cqOffset, boolean max) throws RocksDBException {\n        if (!this.rocksDBStorage.hold()) {\n            return;\n        }\n        try (WriteBatch writeBatch = new WriteBatch()) {\n            final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);\n            final ByteBuffer offsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, max);\n\n            final ByteBuffer offsetValue = buildOffsetValueByteBuffer(phyOffset, cqOffset);\n            writeBatch.put(this.offsetCFH, offsetKey.array(), offsetValue.array());\n            this.rocksDBStorage.batchPut(writeBatch);\n\n            if (max) {\n                putHeapMaxCqOffset(topic, queueId, cqOffset);\n            } else {\n                putHeapMinCqOffset(topic, queueId, phyOffset, cqOffset);\n            }\n        } catch (RocksDBException e) {\n            ERROR_LOG.error(\"updateCqOffset({}) failed.\", max ? \"max\" : \"min\", e);\n            throw e;\n        } finally {\n            this.rocksDBStorage.release();\n            if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) {\n                ROCKSDB_LOG.warn(\"updateCqOffset({}). topic: {}, queueId: {}, phyOffset: {}, cqOffset: {}\",\n                    max ? \"max\" : \"min\", topic, queueId, phyOffset, cqOffset);\n            }\n        }\n    }\n\n    public int getLmqNum() {\n        return lmqCounter.get();\n    }\n\n    public boolean isLmqExist(String lmqTopic) {\n        return this.topicQueueMaxCqOffset.containsKey(buildTopicQueueId(lmqTopic, 0));\n    }\n\n    private boolean correctMaxCqOffset(final String topic, final int queueId, final long maxCQOffset,\n        final long maxPhyOffsetInCQ) throws RocksDBException {\n        // 'getMinOffsetInQueue' may correct minCqOffset and put it into heap\n        long minCQOffset = getMinCqOffset(topic, queueId);\n        PhyAndCQOffset minPhyAndCQOffset = getHeapMinOffset(topic, queueId);\n        if (minPhyAndCQOffset == null\n            || minPhyAndCQOffset.getCqOffset() != minCQOffset\n            || minPhyAndCQOffset.getPhyOffset() > maxPhyOffsetInCQ) {\n            ROCKSDB_LOG.info(\"[BUG] correctMaxCqOffset error! topic: {}, queueId: {}, maxPhyOffsetInCQ: {}, \"\n                    + \"minCqOffset: {}, phyAndCQOffset: {}\",\n                topic, queueId, maxPhyOffsetInCQ, minCQOffset, minPhyAndCQOffset);\n            throw new RocksDBException(\"correctMaxCqOffset error\");\n        }\n\n        PhyAndCQOffset targetPhyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, maxCQOffset,\n            minCQOffset, maxPhyOffsetInCQ, false);\n\n        long targetCQOffset = targetPhyAndCQOffset.getCqOffset();\n        long targetPhyOffset = targetPhyAndCQOffset.getPhyOffset();\n\n        if (targetCQOffset == -1) {\n            if (maxCQOffset != minCQOffset) {\n                updateCqOffset(topic, queueId, minPhyAndCQOffset.getPhyOffset(), minCQOffset, true);\n            }\n            if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) {\n                ROCKSDB_LOG.warn(\"correct error. {}, {}, {}, {}, {}\", topic, queueId, minCQOffset, maxCQOffset, minPhyAndCQOffset.getPhyOffset());\n            }\n            return false;\n        } else {\n            updateCqOffset(topic, queueId, targetPhyOffset, targetCQOffset, true);\n            return true;\n        }\n    }\n\n    private boolean correctMinCqOffset(final String topic, final int queueId,\n        final long minCQOffset, final long minPhyOffset) throws RocksDBException {\n        final ByteBuffer maxBB = getMaxPhyAndCqOffsetInKV(topic, queueId);\n        if (maxBB == null) {\n            updateCqOffset(topic, queueId, minPhyOffset, 0L, false);\n            return true;\n        }\n        final long maxPhyOffset = maxBB.getLong(OFFSET_PHY_OFFSET);\n        final long maxCQOffset = maxBB.getLong(OFFSET_CQ_OFFSET);\n\n        if (maxPhyOffset < minPhyOffset) {\n            updateCqOffset(topic, queueId, minPhyOffset, maxCQOffset + 1, false);\n            return true;\n        }\n\n        PhyAndCQOffset phyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, maxCQOffset,\n            minCQOffset, minPhyOffset, true);\n        long targetCQOffset = phyAndCQOffset.getCqOffset();\n        long targetPhyOffset = phyAndCQOffset.getPhyOffset();\n\n        if (targetCQOffset == -1) {\n            if (maxCQOffset != minCQOffset) {\n                updateCqOffset(topic, queueId, maxPhyOffset, maxCQOffset, false);\n            }\n            if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) {\n                ROCKSDB_LOG.warn(\"correct error. {}, {}, {}, {}, {}\", topic, queueId, minCQOffset, maxCQOffset, minPhyOffset);\n            }\n            return false;\n        } else {\n            updateCqOffset(topic, queueId, targetPhyOffset, targetCQOffset, false);\n            return true;\n        }\n    }\n\n    public static Pair<ByteBuffer, ByteBuffer> getOffsetByteBufferPair() {\n        ByteBuffer offsetKey = ByteBuffer.allocateDirect(RocksDBConsumeQueueStore.MAX_KEY_LEN);\n        ByteBuffer offsetValue = ByteBuffer.allocateDirect(OFFSET_VALUE_LENGTH);\n        return new Pair<>(offsetKey, offsetValue);\n    }\n\n    static void buildOffsetKeyAndValueByteBuffer(final Pair<ByteBuffer, ByteBuffer> offsetBBPair,\n        final DispatchEntry entry) {\n        final ByteBuffer offsetKey = offsetBBPair.getObject1();\n        buildOffsetKeyByteBuffer(offsetKey, entry.topic, entry.queueId, true);\n\n        final ByteBuffer offsetValue = offsetBBPair.getObject2();\n        buildOffsetValueByteBuffer(offsetValue, entry.commitLogOffset, entry.queueOffset);\n    }\n\n    private static ByteBuffer buildOffsetKeyByteBuffer(final byte[] topicBytes, final int queueId, final boolean max) {\n        ByteBuffer byteBuffer = ByteBuffer.allocate(OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length);\n        buildOffsetKeyByteBuffer0(byteBuffer, topicBytes, queueId, max);\n        return byteBuffer;\n    }\n\n    public static void buildOffsetKeyByteBuffer(final ByteBuffer byteBuffer, final byte[] topicBytes,\n        final int queueId, final boolean max) {\n        byteBuffer.position(0).limit(OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length);\n        buildOffsetKeyByteBuffer0(byteBuffer, topicBytes, queueId, max);\n    }\n\n    private static void buildOffsetKeyByteBuffer0(final ByteBuffer byteBuffer, final byte[] topicBytes,\n        final int queueId, final boolean max) {\n        byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1);\n        if (max) {\n            byteBuffer.put(MAX_BYTES);\n        } else {\n            byteBuffer.put(MIN_BYTES);\n        }\n        byteBuffer.put(CTRL_1).putInt(queueId);\n        byteBuffer.flip();\n    }\n\n    private static void buildOffsetValueByteBuffer(final ByteBuffer byteBuffer, final long phyOffset,\n        final long cqOffset) {\n        byteBuffer.position(0).limit(OFFSET_VALUE_LENGTH);\n        buildOffsetValueByteBuffer0(byteBuffer, phyOffset, cqOffset);\n    }\n\n    private static ByteBuffer buildOffsetValueByteBuffer(final long phyOffset, final long cqOffset) {\n        final ByteBuffer byteBuffer = ByteBuffer.allocate(OFFSET_VALUE_LENGTH);\n        buildOffsetValueByteBuffer0(byteBuffer, phyOffset, cqOffset);\n        return byteBuffer;\n    }\n\n    private static void buildOffsetValueByteBuffer0(final ByteBuffer byteBuffer, final long phyOffset,\n        final long cqOffset) {\n        byteBuffer.putLong(phyOffset).putLong(cqOffset);\n        byteBuffer.flip();\n    }\n\n    static class PhyAndCQOffset {\n        private final long phyOffset;\n        private final long cqOffset;\n\n        public PhyAndCQOffset(final long phyOffset, final long cqOffset) {\n            this.phyOffset = phyOffset;\n            this.cqOffset = cqOffset;\n        }\n\n        public long getPhyOffset() {\n            return this.phyOffset;\n        }\n\n        public long getCqOffset() {\n            return this.cqOffset;\n        }\n\n        @Override\n        public String toString() {\n            return \"[cqOffset=\" + cqOffset + \", phyOffset=\" + phyOffset + \"]\";\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.queue;\n\nimport java.io.File;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicReference;\nimport javax.annotation.Nonnull;\nimport org.apache.commons.io.FileUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.ServiceState;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.config.StorePathConfigHelper;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.exception.StoreException;\nimport org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage;\nimport org.rocksdb.FlushOptions;\nimport org.rocksdb.RocksDBException;\nimport org.rocksdb.Statistics;\nimport org.rocksdb.WriteBatch;\n\npublic class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore {\n    private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);\n    private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME);\n\n    private static final int DEFAULT_BYTE_BUFFER_CAPACITY = 16;\n\n    public static final int MAX_KEY_LEN = 300;\n\n    private final ScheduledExecutorService scheduledExecutorService;\n    private final String storePath;\n\n    /**\n     * we use two tables with different ColumnFamilyHandle, called RocksDBConsumeQueueTable and RocksDBConsumeQueueOffsetTable.\n     * 1.RocksDBConsumeQueueTable uses to store CqUnit[physicalOffset, msgSize, tagHashCode, msgStoreTime]\n     * 2.RocksDBConsumeQueueOffsetTable uses to store physicalOffset and consumeQueueOffset(@see PhyAndCQOffset) of topic-queueId\n     */\n    private final ConsumeQueueRocksDBStorage rocksDBStorage;\n    private final RocksDBConsumeQueueTable rocksDBConsumeQueueTable;\n    private final RocksDBConsumeQueueOffsetTable rocksDBConsumeQueueOffsetTable;\n\n    private final List<Pair<ByteBuffer, ByteBuffer>> cqBBPairList;\n    private final List<Pair<ByteBuffer, ByteBuffer>> offsetBBPairList;\n    private final Map<ByteBuffer, Pair<ByteBuffer, DispatchEntry>> tempTopicQueueMaxOffsetMap;\n    private volatile boolean isCQError = false;\n\n    private int consumeQueueByteBufferCacheIndex;\n    private int offsetBufferCacheIndex;\n\n    private final OffsetInitializer offsetInitializer;\n\n    private final RocksGroupCommitService groupCommitService;\n\n    private final AtomicReference<ServiceState> serviceState = new AtomicReference<>(ServiceState.CREATE_JUST);\n\n    private final RocksDBCleanConsumeQueueService cleanConsumeQueueService;\n\n    private long dispatchFromPhyOffset;\n\n    /**\n     * there are two threads to notify longPolling when build cq successfully\n     *\n     * @see DefaultMessageStore.ReputMessageService#doReput()\n     * @see RocksGroupCommitService#groupCommit()\n     * <p>\n     * RocksDB CQ is build by RocksGroupCommitService, so we do not need to notify longPolling in\n     * ReputMessageService\n     */\n    public RocksDBConsumeQueueStore(DefaultMessageStore messageStore) {\n        super(messageStore);\n        messageStore.setNotifyMessageArriveInBatch(true);\n\n        this.storePath = StorePathConfigHelper.getStorePathConsumeQueue(messageStoreConfig.getStorePathRootDir());\n        this.rocksDBStorage = new ConsumeQueueRocksDBStorage(messageStore, storePath);\n        this.rocksDBConsumeQueueTable = new RocksDBConsumeQueueTable(rocksDBStorage, messageStore);\n        this.rocksDBConsumeQueueOffsetTable = new RocksDBConsumeQueueOffsetTable(rocksDBConsumeQueueTable, rocksDBStorage, messageStore);\n\n        this.offsetInitializer = new OffsetInitializerRocksDBImpl(this);\n        this.groupCommitService = new RocksGroupCommitService(this);\n        this.cqBBPairList = new ArrayList<>(16);\n        this.offsetBBPairList = new ArrayList<>(DEFAULT_BYTE_BUFFER_CAPACITY);\n        for (int i = 0; i < DEFAULT_BYTE_BUFFER_CAPACITY; i++) {\n            this.cqBBPairList.add(RocksDBConsumeQueueTable.getCQByteBufferPair());\n            this.offsetBBPairList.add(RocksDBConsumeQueueOffsetTable.getOffsetByteBufferPair());\n        }\n\n        this.tempTopicQueueMaxOffsetMap = new HashMap<>();\n        this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(\n            new ThreadFactoryImpl(\"RocksDBConsumeQueueStoreScheduledThread\", messageStore.getBrokerIdentity()));\n        this.cleanConsumeQueueService = new RocksDBCleanConsumeQueueService();\n    }\n\n    private Pair<ByteBuffer, ByteBuffer> getCQByteBufferPair() {\n        int idx = consumeQueueByteBufferCacheIndex++;\n        if (idx >= cqBBPairList.size()) {\n            this.cqBBPairList.add(RocksDBConsumeQueueTable.getCQByteBufferPair());\n        }\n        return cqBBPairList.get(idx);\n    }\n\n    private Pair<ByteBuffer, ByteBuffer> getOffsetByteBufferPair() {\n        int idx = offsetBufferCacheIndex++;\n        if (idx >= offsetBBPairList.size()) {\n            this.offsetBBPairList.add(RocksDBConsumeQueueOffsetTable.getOffsetByteBufferPair());\n        }\n        return offsetBBPairList.get(idx);\n    }\n\n    @Override\n    public void start() {\n        if (serviceState.compareAndSet(ServiceState.CREATE_JUST, ServiceState.RUNNING)) {\n            log.info(\"RocksDB ConsumeQueueStore start!\");\n            this.groupCommitService.start();\n            this.scheduledExecutorService.scheduleAtFixedRate(() -> {\n                this.rocksDBStorage.statRocksdb(ROCKSDB_LOG);\n            }, 10, this.messageStoreConfig.getStatRocksDBCQIntervalSec(), TimeUnit.SECONDS);\n\n            this.scheduledExecutorService.scheduleWithFixedDelay(() -> {\n                cleanDirty(messageStore.getTopicConfigs().keySet());\n            }, 10, this.messageStoreConfig.getCleanRocksDBDirtyCQIntervalMin(), TimeUnit.MINUTES);\n\n            messageStore.getScheduledCleanQueueExecutorService().scheduleAtFixedRate(this.cleanConsumeQueueService::run,\n                1000 * 60, this.messageStoreConfig.getCleanResourceInterval(), TimeUnit.MILLISECONDS);\n        }\n    }\n\n    private void cleanDirty(final Set<String> existTopicSet) {\n        try {\n            Map<String, Set<Integer>> topicQueueIdToBeDeletedMap =\n                this.rocksDBConsumeQueueOffsetTable.iterateOffsetTable2FindDirty(existTopicSet);\n\n            for (Map.Entry<String, Set<Integer>> entry : topicQueueIdToBeDeletedMap.entrySet()) {\n                String topic = entry.getKey();\n                for (int queueId : entry.getValue()) {\n                    destroy(new RocksDBConsumeQueue(topic, queueId));\n                }\n            }\n        } catch (Exception e) {\n            log.error(\"cleanUnusedTopic Failed.\", e);\n        }\n    }\n\n    @Override\n    public boolean load() {\n        boolean result = this.rocksDBStorage.start();\n        this.rocksDBConsumeQueueTable.load();\n        this.rocksDBConsumeQueueOffsetTable.load();\n        log.info(\"load rocksdb consume queue {}.\", result ? \"OK\" : \"Failed\");\n        return result;\n    }\n    @Override\n    public void recover(boolean concurrently) throws RocksDBException {\n        start();\n        this.dispatchFromPhyOffset = getMaxPhyOffsetInConsumeQueue();\n    }\n\n    @Override\n    public Long getDispatchFromPhyOffset(boolean recoverNormally) throws RocksDBException {\n        return dispatchFromPhyOffset;\n    }\n\n    @Override\n    public boolean shutdown() {\n        if (serviceState.compareAndSet(ServiceState.RUNNING, ServiceState.SHUTDOWN_ALREADY)) {\n            if (this.groupCommitService != null) {\n                this.groupCommitService.shutdown();\n            }\n\n            if (this.scheduledExecutorService != null) {\n                this.scheduledExecutorService.shutdown();\n            }\n            return shutdownInner();\n        }\n        return true;\n    }\n\n    private boolean shutdownInner() {\n        if (this.rocksDBStorage != null) {\n            return this.rocksDBStorage.shutdown();\n        }\n        return true;\n    }\n\n    @Override\n    public void putMessagePositionInfoWrapper(DispatchRequest request) throws RocksDBException {\n        if (null == request) {\n            return;\n        }\n\n        try {\n            groupCommitService.putRequest(request);\n        } catch (InterruptedException e) {\n            Thread.currentThread().interrupt();\n        }\n    }\n\n    public void putMessagePosition(List<DispatchRequest> requests) throws RocksDBException {\n        final int maxRetries = 30;\n        for (int i = 0; i < maxRetries; i++) {\n            if (putMessagePosition0(requests)) {\n                if (this.isCQError) {\n                    this.messageStore.getRunningFlags().clearLogicsQueueError();\n                    this.isCQError = false;\n                }\n                return;\n            } else {\n                ERROR_LOG.warn(\"Put cq Failed. retryTime: {}\", i);\n                try {\n                    Thread.sleep(100);\n                } catch (InterruptedException ignored) {\n                }\n            }\n        }\n        if (!this.isCQError) {\n            ERROR_LOG.error(\"[BUG] put CQ Failed.\");\n            this.messageStore.getRunningFlags().makeLogicsQueueError();\n            this.isCQError = true;\n        }\n        throw new RocksDBException(\"put CQ Failed\");\n    }\n\n    private boolean putMessagePosition0(List<DispatchRequest> requests) {\n        if (!this.rocksDBStorage.hold()) {\n            return false;\n        }\n\n        try (WriteBatch writeBatch = new WriteBatch()) {\n            final int size = requests.size();\n            if (size == 0) {\n                return true;\n            }\n            long maxPhyOffset = 0;\n            for (int i = size - 1; i >= 0; i--) {\n                final DispatchRequest request = requests.get(i);\n                DispatchEntry entry = DispatchEntry.from(request);\n                dispatch(entry, writeBatch);\n                dispatchLMQ(request, writeBatch);\n\n                final int msgSize = request.getMsgSize();\n                final long phyOffset = request.getCommitLogOffset();\n                if (phyOffset + msgSize >= maxPhyOffset) {\n                    maxPhyOffset = phyOffset + msgSize;\n                }\n            }\n\n            this.rocksDBConsumeQueueOffsetTable.putMaxPhyAndCqOffset(tempTopicQueueMaxOffsetMap, writeBatch, maxPhyOffset);\n\n            this.rocksDBStorage.batchPut(writeBatch);\n\n            this.rocksDBConsumeQueueOffsetTable.putHeapMaxCqOffset(tempTopicQueueMaxOffsetMap);\n            notifyMessageArriveAndClear(requests);\n            return true;\n        } catch (Exception e) {\n            ERROR_LOG.error(\"putMessagePosition0 failed.\", e);\n            return false;\n        } finally {\n            tempTopicQueueMaxOffsetMap.clear();\n            consumeQueueByteBufferCacheIndex = 0;\n            offsetBufferCacheIndex = 0;\n            this.rocksDBStorage.release();\n        }\n    }\n\n    private void dispatch(@Nonnull DispatchEntry entry, @Nonnull final WriteBatch writeBatch) throws RocksDBException {\n        this.rocksDBConsumeQueueTable.buildAndPutCQByteBuffer(getCQByteBufferPair(), entry, writeBatch);\n        updateTempTopicQueueMaxOffset(getOffsetByteBufferPair(), entry);\n    }\n\n    private void updateTempTopicQueueMaxOffset(final Pair<ByteBuffer, ByteBuffer> offsetBBPair,\n        final DispatchEntry entry) {\n        RocksDBConsumeQueueOffsetTable.buildOffsetKeyAndValueByteBuffer(offsetBBPair, entry);\n        ByteBuffer topicQueueId = offsetBBPair.getObject1();\n        ByteBuffer maxOffsetBB = offsetBBPair.getObject2();\n        Pair<ByteBuffer, DispatchEntry> old = tempTopicQueueMaxOffsetMap.get(topicQueueId);\n        if (old == null) {\n            tempTopicQueueMaxOffsetMap.put(topicQueueId, new Pair<>(maxOffsetBB, entry));\n        } else {\n            long oldMaxOffset = old.getObject1().getLong(RocksDBConsumeQueueOffsetTable.OFFSET_CQ_OFFSET);\n            long maxOffset = maxOffsetBB.getLong(RocksDBConsumeQueueOffsetTable.OFFSET_CQ_OFFSET);\n            if (maxOffset >= oldMaxOffset) {\n                ERROR_LOG.error(\"cqOffset invalid1. old: {}, now: {}\", oldMaxOffset, maxOffset);\n            }\n        }\n    }\n\n    private void dispatchLMQ(@Nonnull DispatchRequest request, @Nonnull final WriteBatch writeBatch)\n        throws RocksDBException {\n        if (!messageStoreConfig.isEnableLmq() || !request.containsLMQ()) {\n            return;\n        }\n        Map<String, String> map = request.getPropertiesMap();\n        String lmqNames = map.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);\n        String lmqOffsets = map.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET);\n        String[] queues = lmqNames.split(MixAll.LMQ_DISPATCH_SEPARATOR);\n        String[] queueOffsets = lmqOffsets.split(MixAll.LMQ_DISPATCH_SEPARATOR);\n        if (queues.length != queueOffsets.length) {\n            ERROR_LOG.error(\"[bug] queues.length!=queueOffsets.length \", request.getTopic());\n            return;\n        }\n        for (int i = 0; i < queues.length; i++) {\n            String queueName = queues[i];\n            DispatchEntry entry = DispatchEntry.from(request);\n            long queueOffset = Long.parseLong(queueOffsets[i]);\n            int queueId = request.getQueueId();\n            if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) {\n                queueId = MixAll.LMQ_QUEUE_ID;\n            }\n            entry.queueId = queueId;\n            entry.queueOffset = queueOffset;\n            entry.topic = queueName.getBytes(StandardCharsets.UTF_8);\n            log.debug(\"Dispatch LMQ[{}:{}]:{} --> {}\", queueName, queueId, queueOffset, entry.commitLogOffset);\n            dispatch(entry, writeBatch);\n        }\n    }\n\n    private void notifyMessageArriveAndClear(List<DispatchRequest> requests) {\n        try {\n            for (DispatchRequest dp : requests) {\n                this.messageStore.notifyMessageArriveIfNecessary(dp);\n            }\n            requests.clear();\n        } catch (Exception e) {\n            ERROR_LOG.error(\"notifyMessageArriveAndClear Failed.\", e);\n        }\n    }\n\n    public Statistics getStatistics() {\n        return rocksDBStorage.getStatistics();\n    }\n\n    public List<ByteBuffer> rangeQuery(final String topic, final int queueId, final long startIndex,\n        final int num) throws RocksDBException {\n        return this.rocksDBConsumeQueueTable.rangeQuery(topic, queueId, startIndex, num);\n    }\n\n    public ByteBuffer get(final String topic, final int queueId, final long cqOffset) throws RocksDBException {\n        return this.rocksDBConsumeQueueTable.getCQInKV(topic, queueId, cqOffset);\n    }\n\n    /**\n     * Try to set topicQueueTable = new HashMap<>(), otherwise it will cause bug when broker role changes.\n     * And unlike method in DefaultMessageStore, we don't need to really recover topic queue table advance,\n     * because we can recover topic queue table from rocksdb when we need to use it.\n     * @see RocksDBConsumeQueue#assignQueueOffset\n     *\n     * @see RocksDBConsumeQueue#increaseQueueOffset(QueueOffsetOperator, MessageExtBrokerInner, short)\n     * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable#getMinCqOffset(String, int)\n     */\n    @Override\n    public void recoverOffsetTable(long minPhyOffset) {\n        this.setTopicQueueTable(new ConcurrentHashMap<>());\n    }\n\n    @Override\n    public void destroy(boolean loadAfterDestroy) {\n        try {\n            shutdownInner();\n            FileUtils.deleteDirectory(new File(this.storePath));\n        } catch (Exception e) {\n            ERROR_LOG.error(\"destroy cq Failed. {}\", this.storePath, e);\n        }\n        if (loadAfterDestroy) {\n            load();\n        }\n    }\n\n    @Override\n    public void destroy(ConsumeQueueInterface consumeQueue) throws RocksDBException {\n        String topic = consumeQueue.getTopic();\n        int queueId = consumeQueue.getQueueId();\n        if (StringUtils.isEmpty(topic) || queueId < 0 || !this.rocksDBStorage.hold()) {\n            return;\n        }\n\n        try (WriteBatch writeBatch = new WriteBatch()) {\n            this.rocksDBConsumeQueueTable.destroyCQ(topic, queueId, writeBatch);\n            this.rocksDBConsumeQueueOffsetTable.destroyOffset(topic, queueId, writeBatch);\n\n            this.rocksDBStorage.batchPut(writeBatch);\n        } catch (RocksDBException e) {\n            ERROR_LOG.error(\"kv deleteTopic {} Failed.\", topic, e);\n            throw e;\n        } finally {\n            this.rocksDBStorage.release();\n        }\n    }\n\n    /**\n     * ConsumerQueueTable, as an in-memory data structure, uses lazy loading mechanism in RocksDBConsumeQueueStore.\n     * This means that when the broker restarts, it may not be able to retrieve all ConsumerQueues from the table.\n     * Therefore, before deleting a topic, we need to attempt to build all ConsumerQueues under that topic to ensure\n     * the completeness of the deletion operation.\n     */\n    @Override\n    public boolean deleteTopic(String topic) {\n        try {\n            Set<Integer> queueIds = rocksDBConsumeQueueOffsetTable.scanAllQueueIdInTopic(topic);\n            queueIds.forEach(queueId -> findOrCreateConsumeQueue(topic, queueId));\n        } catch (RocksDBException e) {\n            ERROR_LOG.error(\"Failed to scan queueIds for topic. topic={}\", topic, e);\n        }\n        return super.deleteTopic(topic);\n    }\n\n    @Override\n    public void flush() throws StoreException {\n        try (FlushOptions flushOptions = new FlushOptions()) {\n            flushOptions.setWaitForFlush(true);\n            flushOptions.setAllowWriteStall(true);\n            this.rocksDBStorage.flush(flushOptions);\n        } catch (RocksDBException e) {\n            throw new StoreException(e);\n        }\n    }\n\n    @Override\n    public void checkSelf() {\n        // ignored\n    }\n\n    /**\n     * We do not need to truncate dirty CQ in RocksDBConsumeQueueTable,  Because dirty CQ in RocksDBConsumeQueueTable\n     * will be rewritten by new KV when new messages are appended or will be cleaned up when topics are deleted.\n     * But dirty offset info in RocksDBConsumeQueueOffsetTable must be truncated, because we use offset info in\n     * RocksDBConsumeQueueOffsetTable to rebuild topicQueueTable(@see RocksDBConsumeQueue#increaseQueueOffset).\n     *\n     * @param offsetToTruncate CommitLog offset to truncate to\n     * @throws RocksDBException If there is any error.\n     */\n    @Override\n    public void truncateDirty(long offsetToTruncate) throws RocksDBException {\n        long maxPhyOffsetInRocksdb = getMaxPhyOffsetInConsumeQueue();\n        if (offsetToTruncate >= maxPhyOffsetInRocksdb) {\n            return;\n        }\n\n        this.rocksDBConsumeQueueOffsetTable.truncateDirty(offsetToTruncate);\n    }\n\n    @Override\n    public void cleanExpired(final long minPhyOffset) {\n        this.rocksDBStorage.manualCompaction(minPhyOffset);\n    }\n\n    @Override\n    public long getOffsetInQueueByTime(String topic, int queueId, long timestamp,\n        BoundaryType boundaryType) throws RocksDBException {\n        final long minPhysicOffset = this.messageStore.getMinPhyOffset();\n        long low = this.rocksDBConsumeQueueOffsetTable.getMinCqOffset(topic, queueId);\n        Long high = this.rocksDBConsumeQueueOffsetTable.getMaxCqOffset(topic, queueId);\n        if (high == null || high == -1) {\n            return 0;\n        }\n        return this.rocksDBConsumeQueueTable.binarySearchInCQByTime(topic, queueId, high, low, timestamp,\n            minPhysicOffset, boundaryType);\n    }\n\n    /**\n     * This method actually returns NEXT slot index to use, starting from 0. For example, if the queue is empty,\n     * it returns 0, pointing to the first slot of the 0-based queue;\n     *\n     * @param topic   Topic name\n     * @param queueId Queue ID\n     * @return Index of the next slot to push into\n     * @throws RocksDBException if RocksDB fails to fulfill the request.\n     */\n    public long getMaxOffsetInQueue(String topic, int queueId) throws RocksDBException {\n        Long maxOffset = this.rocksDBConsumeQueueOffsetTable.getMaxCqOffset(topic, queueId);\n        return (maxOffset != null) ? maxOffset + 1 : 0;\n    }\n\n    @Override\n    public long getMinOffsetInQueue(String topic, int queueId) throws RocksDBException {\n        return this.rocksDBConsumeQueueOffsetTable.getMinCqOffset(topic, queueId);\n    }\n\n    public Long getMaxPhyOffsetInConsumeQueue(String topic, int queueId) {\n        return this.rocksDBConsumeQueueOffsetTable.getMaxPhyOffset(topic, queueId);\n    }\n\n    @Override\n    public long getMaxPhyOffsetInConsumeQueue() throws RocksDBException {\n        return this.rocksDBConsumeQueueOffsetTable.getMaxPhyOffset();\n    }\n\n    @Override\n    public ConsumeQueueInterface findOrCreateConsumeQueue(String topic, int queueId) {\n        ConcurrentMap<Integer, ConsumeQueueInterface> map = this.consumeQueueTable.get(topic);\n        if (null == map) {\n            ConcurrentMap<Integer, ConsumeQueueInterface> newMap;\n            if (MixAll.isLmq(topic)) {\n                // For LMQ, no need to over allocate internal hashtable\n                newMap = new ConcurrentHashMap<>(1, 1.0F);\n            } else {\n                newMap = new ConcurrentHashMap<>(8);\n            }\n            ConcurrentMap<Integer, ConsumeQueueInterface> oldMap = this.consumeQueueTable.putIfAbsent(topic, newMap);\n            if (oldMap != null) {\n                map = oldMap;\n            } else {\n                map = newMap;\n            }\n        }\n\n        ConsumeQueueInterface logic = map.get(queueId);\n        if (logic != null) {\n            return logic;\n        }\n\n        ConsumeQueueInterface newLogic = new RocksDBConsumeQueue(this.messageStore.getMessageStoreConfig(), this, topic, queueId);\n        ConsumeQueueInterface oldLogic = map.putIfAbsent(queueId, newLogic);\n\n        return oldLogic != null ? oldLogic : newLogic;\n    }\n\n    @Override\n    public ConsumeQueueInterface getConsumeQueue(String topic, int queueId) {\n        return findOrCreateConsumeQueue(topic, queueId);\n    }\n\n    @Override\n    public long getTotalSize() {\n        return 0;\n    }\n\n    @Override\n    public boolean isMappedFileMatchedRecover(long phyOffset, long storeTimestamp,\n        boolean recoverNormally) {\n        return phyOffset <= dispatchFromPhyOffset;\n    }\n\n    @Override\n    public long getLmqQueueOffset(String topic, int queueId) throws ConsumeQueueException {\n        return queueOffsetOperator.getLmqOffset(topic, queueId, offsetInitializer);\n    }\n\n    @Override\n    public Long getMaxOffset(String topic, int queueId) throws ConsumeQueueException {\n        if (MixAll.isLmq(topic)) {\n            return getLmqQueueOffset(topic, queueId);\n        }\n        return super.getMaxOffset(topic, queueId);\n    }\n\n    @Override\n    public int getLmqNum() {\n        return this.rocksDBConsumeQueueOffsetTable.getLmqNum();\n    }\n\n    @Override\n    public boolean isLmqExist(String lmqTopic) {\n        return MixAll.isLmq(lmqTopic) ? this.rocksDBConsumeQueueOffsetTable.isLmqExist(lmqTopic) : false;\n    }\n\n    public boolean isStopped() {\n        return ServiceState.SHUTDOWN_ALREADY == serviceState.get();\n    }\n\n    public void updateCqOffset(final String topic, final int queueId, final long phyOffset,\n        final long cqOffset, boolean max) throws RocksDBException {\n        this.rocksDBConsumeQueueOffsetTable.updateCqOffset(topic, queueId, phyOffset, cqOffset, max);\n    }\n\n    class RocksDBCleanConsumeQueueService {\n        protected long lastPhysicalMinOffset = 0;\n\n        private final double diskSpaceWarningLevelRatio =\n            Double.parseDouble(System.getProperty(\"rocketmq.broker.diskSpaceWarningLevelRatio\", \"0.90\"));\n\n        private final double diskSpaceCleanForciblyRatio =\n            Double.parseDouble(System.getProperty(\"rocketmq.broker.diskSpaceCleanForciblyRatio\", \"0.85\"));\n\n        public void run() {\n            try {\n                this.deleteExpiredFiles();\n            } catch (Throwable e) {\n                log.warn(this.getServiceName() + \" service has exception. \", e);\n            }\n        }\n\n        public String getServiceName() {\n            return messageStore.getBrokerConfig().getIdentifier() + ConsumeQueueStore.CleanConsumeQueueService.class.getSimpleName();\n        }\n\n        protected void deleteExpiredFiles() {\n            long minOffset = messageStore.getCommitLog().getMinOffset();\n            if (minOffset > this.lastPhysicalMinOffset) {\n                this.lastPhysicalMinOffset = minOffset;\n\n                boolean spaceFull = isSpaceToDelete();\n                boolean timeUp = messageStore.isTimeToDelete();\n                if (spaceFull || timeUp) {\n                    // To delete the CQ Units whose physical offset is smaller min physical offset in commitLog.\n                    cleanExpired(minOffset);\n                }\n\n                messageStore.getIndexService().deleteExpiredFile(minOffset);\n                if (messageStoreConfig.isIndexRocksDBEnable() && null != messageStore.getIndexRocksDBStore()) {\n                    messageStore.getIndexRocksDBStore().deleteExpiredIndex();\n                }\n            }\n        }\n\n        private boolean isSpaceToDelete() {\n            double ratio = messageStoreConfig.getDiskMaxUsedSpaceRatio() / 100.0;\n\n            String storePathLogics = StorePathConfigHelper\n                .getStorePathConsumeQueue(messageStoreConfig.getStorePathRootDir());\n            double logicsRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathLogics);\n            if (logicsRatio > diskSpaceWarningLevelRatio) {\n                boolean diskMaybeFull = messageStore.getRunningFlags().getAndMakeLogicDiskFull();\n                if (diskMaybeFull) {\n                    log.error(\"logics disk maybe full soon \" + logicsRatio + \", so mark disk full\");\n                }\n            } else if (logicsRatio > diskSpaceCleanForciblyRatio) {\n            } else {\n                boolean diskOk = messageStore.getRunningFlags().getAndMakeLogicDiskOK();\n                if (!diskOk) {\n                    log.info(\"logics disk space OK \" + logicsRatio + \", so mark disk ok\");\n                }\n            }\n\n            if (logicsRatio < 0 || logicsRatio > ratio) {\n                log.info(\"logics disk maybe full soon, so reclaim space, \" + logicsRatio);\n                return true;\n            }\n\n            return false;\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.queue;\n\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.List;\n\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable.PhyAndCQOffset;\nimport org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage;\nimport org.rocksdb.ColumnFamilyHandle;\nimport org.rocksdb.RocksDBException;\nimport org.rocksdb.WriteBatch;\n\nimport static org.apache.rocketmq.common.config.AbstractRocksDBStorage.CTRL_0;\nimport static org.apache.rocketmq.common.config.AbstractRocksDBStorage.CTRL_1;\nimport static org.apache.rocketmq.common.config.AbstractRocksDBStorage.CTRL_2;\n\n/**\n * We use RocksDBConsumeQueueTable to store cqUnit.\n */\npublic class RocksDBConsumeQueueTable {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME);\n    private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);\n\n    /**\n     * Rocksdb ConsumeQueue's store unit. Format:\n     *\n     * <pre>\n     * ┌─────────────────────────┬───────────┬───────────────────────┬───────────┬───────────┬───────────┬───────────────────────┐\n     * │ Topic Bytes Array Size  │  CTRL_1   │   Topic Bytes Array   │  CTRL_1   │  QueueId  │  CTRL_1   │  ConsumeQueue Offset  │\n     * │        (4 Bytes)        │ (1 Bytes) │       (n Bytes)       │ (1 Bytes) │ (4 Bytes) │ (1 Bytes) │     (8 Bytes)         │\n     * ├─────────────────────────┴───────────┴───────────────────────┴───────────┴───────────┴───────────┴───────────────────────┤\n     * │                                                    Key Unit                                                             │\n     * │                                                                                                                         │\n     * </pre>\n     *\n     * <pre>\n     * ┌─────────────────────────────┬───────────────────┬──────────────────┬──────────────────┐\n     * │  CommitLog Physical Offset  │      Body Size    │   Tag HashCode   │  Msg Store Time  │\n     * │        (8 Bytes)            │      (4 Bytes)    │    (8 Bytes)     │    (8 Bytes)     │\n     * ├─────────────────────────────┴───────────────────┴──────────────────┴──────────────────┤\n     * │                                                    Value Unit                         │\n     * │                                                                                       │\n     * </pre>\n     * ConsumeQueue's store unit. Size:\n     * CommitLog Physical Offset(8) + Body Size(4) + Tag HashCode(8) + Msg Store Time(8) = 28 Bytes\n     */\n    private static final int PHY_OFFSET_OFFSET = 0;\n    private static final int PHY_MSG_LEN_OFFSET = 8;\n    private static final int MSG_TAG_HASHCODE_OFFSET = 12;\n    private static final int MSG_STORE_TIME_SIZE_OFFSET = 20;\n    public static final int CQ_UNIT_SIZE = 8 + 4 + 8 + 8;\n\n    /**\n     * ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────┬───────────────────────┐\n     * │ Topic Bytes Array Size  │  CTRL_1   │  CTRL_1   │  QueueId  │  CTRL_1   │  ConsumeQueue Offset  │\n     * │        (4 Bytes)        │ (1 Bytes) │ (1 Bytes) │ (4 Bytes) │ (1 Bytes) │     (8 Bytes)         │\n     * ├─────────────────────────┴───────────┴───────────┴───────────┴───────────┴───────────────────────┤\n     */\n    private static final int CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES = 4 + 1 + 1 + 4 + 1 + 8;\n\n    /**\n     * ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────────────┐\n     * │ Topic Bytes Array Size  │  CTRL_1   │  CTRL_1   │  QueueId  │  CTRL_0(CTRL_2)   │\n     * │        (4 Bytes)        │ (1 Bytes) │ (1 Bytes) │ (4 Bytes) │     (1 Bytes)     │\n     * ├─────────────────────────┴───────────┴───────────┴───────────┴───────────────────┤\n     */\n    private static final int DELETE_CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES = 4 + 1 + 1 + 4 + 1;\n\n    private final ConsumeQueueRocksDBStorage rocksDBStorage;\n    private final DefaultMessageStore messageStore;\n\n    private ColumnFamilyHandle defaultCFH;\n\n    public RocksDBConsumeQueueTable(ConsumeQueueRocksDBStorage rocksDBStorage, DefaultMessageStore messageStore) {\n        this.rocksDBStorage = rocksDBStorage;\n        this.messageStore = messageStore;\n    }\n\n    public void load() {\n        this.defaultCFH = this.rocksDBStorage.getDefaultCFHandle();\n    }\n\n    public void buildAndPutCQByteBuffer(final Pair<ByteBuffer, ByteBuffer> cqBBPair, final DispatchEntry request,\n        final WriteBatch writeBatch) throws RocksDBException {\n        final ByteBuffer cqKey = cqBBPair.getObject1();\n        buildCQKeyByteBuffer(cqKey, request.topic, request.queueId, request.queueOffset);\n\n        final ByteBuffer cqValue = cqBBPair.getObject2();\n        buildCQValueByteBuffer(cqValue, request.commitLogOffset, request.messageSize, request.tagCode, request.storeTimestamp);\n\n        writeBatch.put(this.defaultCFH, cqKey, cqValue);\n    }\n\n    public ByteBuffer getCQInKV(final String topic, final int queueId, final long cqOffset) throws RocksDBException {\n        final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);\n        final ByteBuffer keyBB = buildCQKeyByteBuffer(topicBytes, queueId, cqOffset);\n        byte[] value = this.rocksDBStorage.getCQ(keyBB.array());\n        return (value != null) ? ByteBuffer.wrap(value) : null;\n    }\n\n    public List<ByteBuffer> rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException {\n        final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);\n        final List<ColumnFamilyHandle> defaultCFHList = new ArrayList<>(num);\n        final ByteBuffer[] resultList = new ByteBuffer[num];\n        final List<Integer> kvIndexList = new ArrayList<>(num);\n        final List<byte[]> kvKeyList = new ArrayList<>(num);\n        for (int i = 0; i < num; i++) {\n            final ByteBuffer keyBB = buildCQKeyByteBuffer(topicBytes, queueId, startIndex + i);\n            kvIndexList.add(i);\n            kvKeyList.add(keyBB.array());\n            defaultCFHList.add(this.defaultCFH);\n        }\n        int keyNum = kvIndexList.size();\n        if (keyNum > 0) {\n            List<byte[]> kvValueList = this.rocksDBStorage.multiGet(defaultCFHList, kvKeyList);\n            final int valueNum = kvValueList.size();\n            if (keyNum != valueNum) {\n                throw new RocksDBException(\"rocksdb bug, multiGet\");\n            }\n            for (int i = 0; i < valueNum; i++) {\n                byte[] value = kvValueList.get(i);\n                if (value == null) {\n                    continue;\n                }\n                ByteBuffer byteBuffer = ByteBuffer.wrap(value);\n                resultList[kvIndexList.get(i)] = byteBuffer;\n            }\n        }\n\n        final int resultSize = resultList.length;\n        List<ByteBuffer> bbValueList = new ArrayList<>(resultSize);\n        for (int i = 0; i < resultSize; i++) {\n            ByteBuffer byteBuffer = resultList[i];\n            if (byteBuffer == null) {\n                break;\n            }\n            bbValueList.add(byteBuffer);\n        }\n        return bbValueList;\n    }\n\n    /**\n     * When topic is deleted, we clean up its CqUnit in rocksdb.\n     * @param topic\n     * @param queueId\n     * @throws RocksDBException\n     */\n    public void destroyCQ(final String topic, final int queueId, WriteBatch writeBatch) throws RocksDBException {\n        final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);\n        final ByteBuffer cqStartKey = buildDeleteCQKey(true, topicBytes, queueId);\n        final ByteBuffer cqEndKey = buildDeleteCQKey(false, topicBytes, queueId);\n\n        writeBatch.deleteRange(this.defaultCFH, cqStartKey.array(), cqEndKey.array());\n\n        log.info(\"Rocksdb consumeQueue table delete topic. {}, {}\", topic, queueId);\n    }\n\n    public long binarySearchInCQByTime(String topic, int queueId, long high, long low, long timestamp,\n        long minPhysicOffset, BoundaryType boundaryType) throws RocksDBException {\n        long result = -1L;\n        long targetOffset = -1L, leftOffset = -1L, rightOffset = -1L;\n        long ceiling = high, floor = low;\n        // Handle the following corner cases first:\n        // 1. store time of (high) < timestamp\n        ByteBuffer buffer = getCQInKV(topic, queueId, ceiling);\n        if (buffer != null) {\n            long storeTime = buffer.getLong(MSG_STORE_TIME_SIZE_OFFSET);\n            if (storeTime < timestamp) {\n                switch (boundaryType) {\n                    case LOWER:\n                        return ceiling + 1;\n                    case UPPER:\n                        return ceiling;\n                    default:\n                        log.warn(\"Unknown boundary type\");\n                        break;\n                }\n            }\n        }\n        // 2. store time of (low) > timestamp\n        buffer = getCQInKV(topic, queueId, floor);\n        if (buffer != null) {\n            long storeTime = buffer.getLong(MSG_STORE_TIME_SIZE_OFFSET);\n            if (storeTime > timestamp) {\n                switch (boundaryType) {\n                    case LOWER:\n                        return floor;\n                    case UPPER:\n                        return 0;\n                    default:\n                        log.warn(\"Unknown boundary type\");\n                        break;\n                }\n            }\n        }\n        while (high >= low) {\n            long midOffset = low + ((high - low) >>> 1);\n            ByteBuffer byteBuffer = getCQInKV(topic, queueId, midOffset);\n            if (byteBuffer == null) {\n                ERROR_LOG.warn(\"binarySearchInCQByTimeStamp Failed. topic: {}, queueId: {}, timestamp: {}, result: null\",\n                    topic, queueId, timestamp);\n                low = midOffset + 1;\n                continue;\n            }\n\n            long phyOffset = byteBuffer.getLong(PHY_OFFSET_OFFSET);\n            if (phyOffset < minPhysicOffset) {\n                low = midOffset + 1;\n                leftOffset = midOffset;\n                continue;\n            }\n            long storeTime = byteBuffer.getLong(MSG_STORE_TIME_SIZE_OFFSET);\n            if (storeTime < 0) {\n                return 0;\n            } else if (storeTime == timestamp) {\n                targetOffset = midOffset;\n                break;\n            } else if (storeTime > timestamp) {\n                high = midOffset - 1;\n                rightOffset = midOffset;\n            } else {\n                low = midOffset + 1;\n                leftOffset = midOffset;\n            }\n        }\n        if (targetOffset != -1) {\n            // offset next to it might also share the same store-timestamp.\n            switch (boundaryType) {\n                case LOWER: {\n                    while (true) {\n                        long nextOffset = targetOffset - 1;\n                        if (nextOffset < floor) {\n                            break;\n                        }\n                        ByteBuffer byteBuffer = getCQInKV(topic, queueId, nextOffset);\n                        long storeTime = byteBuffer.getLong(MSG_STORE_TIME_SIZE_OFFSET);\n                        if (storeTime != timestamp) {\n                            break;\n                        }\n                        targetOffset = nextOffset;\n                    }\n                    break;\n                }\n                case UPPER: {\n                    while (true) {\n                        long nextOffset = targetOffset + 1;\n                        if (nextOffset > ceiling) {\n                            break;\n                        }\n                        ByteBuffer byteBuffer = getCQInKV(topic, queueId, nextOffset);\n                        long storeTime = byteBuffer.getLong(MSG_STORE_TIME_SIZE_OFFSET);\n                        if (storeTime != timestamp) {\n                            break;\n                        }\n                        targetOffset = nextOffset;\n                    }\n                    break;\n                }\n                default: {\n                    log.warn(\"Unknown boundary type\");\n                    break;\n                }\n            }\n            result = targetOffset;\n        } else {\n            switch (boundaryType) {\n                case LOWER: {\n                    result = rightOffset;\n                    break;\n                }\n                case UPPER: {\n                    result = leftOffset;\n                    break;\n                }\n                default: {\n                    log.warn(\"Unknown boundary type\");\n                    break;\n                }\n            }\n        }\n        return result;\n    }\n\n    public PhyAndCQOffset binarySearchInCQ(String topic, int queueId, long high, long low, long targetPhyOffset,\n        boolean min) throws RocksDBException {\n        long resultCQOffset = -1L;\n        long resultPhyOffset = -1L;\n        while (high >= low) {\n            long midCQOffset = low + ((high - low) >>> 1);\n            ByteBuffer byteBuffer = getCQInKV(topic, queueId, midCQOffset);\n            if (this.messageStore.getMessageStoreConfig().isEnableRocksDBLog()) {\n                ROCKSDB_LOG.warn(\"binarySearchInCQ. {}, {}, {}, {}, {}\", topic, queueId, midCQOffset, low, high);\n            }\n            if (byteBuffer == null) {\n                low = midCQOffset + 1;\n                continue;\n            }\n\n            final long phyOffset = byteBuffer.getLong(PHY_OFFSET_OFFSET);\n            if (phyOffset == targetPhyOffset) {\n                if (min) {\n                    resultCQOffset =  midCQOffset;\n                    resultPhyOffset = phyOffset;\n                }\n                break;\n            } else if (phyOffset > targetPhyOffset) {\n                high = midCQOffset - 1;\n                if (min) {\n                    resultCQOffset = midCQOffset;\n                    resultPhyOffset = phyOffset;\n                }\n            } else {\n                low = midCQOffset + 1;\n                if (!min) {\n                    resultCQOffset = midCQOffset;\n                    resultPhyOffset = phyOffset;\n                }\n            }\n        }\n        return new PhyAndCQOffset(resultPhyOffset, resultCQOffset);\n    }\n\n    public static Pair<ByteBuffer, ByteBuffer> getCQByteBufferPair() {\n        ByteBuffer cqKey = ByteBuffer.allocateDirect(RocksDBConsumeQueueStore.MAX_KEY_LEN);\n        ByteBuffer cqValue = ByteBuffer.allocateDirect(CQ_UNIT_SIZE);\n        return new Pair<>(cqKey, cqValue);\n    }\n\n    private ByteBuffer buildCQKeyByteBuffer(final byte[] topicBytes, final int queueId, final long cqOffset) {\n        final ByteBuffer byteBuffer = ByteBuffer.allocate(CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length);\n        buildCQKeyByteBuffer0(byteBuffer, topicBytes, queueId, cqOffset);\n        return byteBuffer;\n    }\n\n    private void buildCQKeyByteBuffer(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, final long cqOffset) {\n        byteBuffer.position(0).limit(CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length);\n        buildCQKeyByteBuffer0(byteBuffer, topicBytes, queueId, cqOffset);\n    }\n\n    private void buildCQKeyByteBuffer0(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, final long cqOffset) {\n        byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1).putInt(queueId).put(CTRL_1).putLong(cqOffset);\n        byteBuffer.flip();\n    }\n\n    private void buildCQValueByteBuffer(final ByteBuffer byteBuffer, final long phyOffset, final int msgSize, final long tagsCode, final long storeTimestamp) {\n        byteBuffer.position(0).limit(CQ_UNIT_SIZE);\n        buildCQValueByteBuffer0(byteBuffer, phyOffset, msgSize, tagsCode, storeTimestamp);\n    }\n\n    private void buildCQValueByteBuffer0(final ByteBuffer byteBuffer, final long phyOffset, final int msgSize,\n        final long tagsCode, final long storeTimestamp) {\n        byteBuffer.putLong(phyOffset).putInt(msgSize).putLong(tagsCode).putLong(storeTimestamp);\n        byteBuffer.flip();\n    }\n\n    private ByteBuffer buildDeleteCQKey(final boolean start, final byte[] topicBytes, final int queueId) {\n        final ByteBuffer byteBuffer = ByteBuffer.allocate(DELETE_CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length);\n\n        byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1).putInt(queueId).put(start ? CTRL_0 : CTRL_2);\n        byteBuffer.flip();\n        return byteBuffer;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/RocksGroupCommitService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.queue;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.rocksdb.RocksDBException;\n\npublic class RocksGroupCommitService extends ServiceThread {\n\n    private static final int MAX_BUFFER_SIZE = 100_000;\n\n    private static final int PREFERRED_DISPATCH_REQUEST_COUNT = 256;\n\n    private final LinkedBlockingQueue<DispatchRequest> buffer;\n\n    private final RocksDBConsumeQueueStore store;\n\n    private final List<DispatchRequest> requests = new ArrayList<>(PREFERRED_DISPATCH_REQUEST_COUNT);\n\n    public RocksGroupCommitService(RocksDBConsumeQueueStore store) {\n        this.store = store;\n        this.buffer = new LinkedBlockingQueue<>(MAX_BUFFER_SIZE);\n    }\n\n    @Override\n    public String getServiceName() {\n        return \"RocksGroupCommit\";\n    }\n\n    @Override\n    public void run() {\n        log.info(\"{} service started\", this.getServiceName());\n        while (!this.isStopped()) {\n            try {\n                this.waitForRunning(10);\n                this.doCommit();\n            } catch (Exception e) {\n                log.warn(\"{} service has exception. \", this.getServiceName(), e);\n            }\n        }\n        log.info(\"{} service end\", this.getServiceName());\n    }\n\n    public void putRequest(final DispatchRequest request) throws InterruptedException {\n        while (!buffer.offer(request, 3, TimeUnit.SECONDS)) {\n            log.warn(\"RocksGroupCommitService#buffer is full, 3s elapsed before space becomes available\");\n        }\n        this.wakeup();\n    }\n\n    private void doCommit() {\n        while (!buffer.isEmpty()) {\n            while (true) {\n                DispatchRequest dispatchRequest = buffer.poll();\n                if (null != dispatchRequest) {\n                    requests.add(dispatchRequest);\n                }\n\n                if (requests.isEmpty()) {\n                    // buffer has been drained\n                    break;\n                }\n\n                if (null == dispatchRequest || requests.size() >= PREFERRED_DISPATCH_REQUEST_COUNT) {\n                    groupCommit();\n                }\n            }\n        }\n    }\n\n    private void groupCommit() {\n        while (!store.isStopped()) {\n            try {\n                // putMessagePosition will clear requests after consume queue building completion\n                store.putMessagePosition(requests);\n                break;\n            } catch (RocksDBException e) {\n                log.error(\"Failed to build consume queue in RocksDB\", e);\n            }\n        }\n    }\n\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/SparseConsumeQueue.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.queue;\n\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.logfile.MappedFile;\n\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.function.Function;\n\npublic class SparseConsumeQueue extends BatchConsumeQueue {\n\n    public SparseConsumeQueue(\n        final String topic,\n        final int queueId,\n        final String storePath,\n        final int mappedFileSize,\n        final MessageStore defaultMessageStore) {\n        super(topic, queueId, storePath, mappedFileSize, defaultMessageStore);\n    }\n\n    public SparseConsumeQueue(\n        final String topic,\n        final int queueId,\n        final String storePath,\n        final int mappedFileSize,\n        final MessageStore defaultMessageStore,\n        final String subfolder) {\n        super(topic, queueId, storePath, mappedFileSize, defaultMessageStore, subfolder);\n    }\n\n    @Override\n    public void recover() {\n        final List<MappedFile> mappedFiles = this.mappedFileQueue.getMappedFiles();\n        if (!mappedFiles.isEmpty()) {\n            int index = mappedFiles.size() - 3;\n            if (index < 0) {\n                index = 0;\n            }\n\n            MappedFile mappedFile = mappedFiles.get(index);\n            ByteBuffer byteBuffer = mappedFile.sliceByteBuffer();\n            int mappedFileOffset = 0;\n            long processOffset = mappedFile.getFileFromOffset();\n            while (true) {\n                for (int i = 0; i < mappedFileSize; i += CQ_STORE_UNIT_SIZE) {\n                    byteBuffer.position(i);\n                    long offset = byteBuffer.getLong();\n                    int size = byteBuffer.getInt();\n                    byteBuffer.getLong();   //tagscode\n                    byteBuffer.getLong();   //timestamp\n                    long msgBaseOffset = byteBuffer.getLong();\n                    short batchSize = byteBuffer.getShort();\n                    if (offset >= 0 && size > 0 && msgBaseOffset >= 0 && batchSize > 0) {\n                        mappedFileOffset += CQ_STORE_UNIT_SIZE;\n                        this.maxMsgPhyOffsetInCommitLog = offset;\n                    } else {\n                        log.info(\"Recover current batch consume queue file over, \" + \"file:{} offset:{} size:{} msgBaseOffset:{} batchSize:{} mappedFileOffset:{}\",\n                            mappedFile.getFileName(), offset, size, msgBaseOffset, batchSize, mappedFileOffset);\n\n                        if (mappedFileOffset != mappedFileSize) {\n                            mappedFile.setWrotePosition(mappedFileOffset);\n                            mappedFile.setFlushedPosition(mappedFileOffset);\n                            mappedFile.setCommittedPosition(mappedFileOffset);\n                        }\n\n                        break;\n                    }\n                }\n\n                index++;\n                if (index >= mappedFiles.size()) {\n                    log.info(\"Recover last batch consume queue file over, last mapped file:{} \", mappedFile.getFileName());\n                    break;\n                } else {\n                    mappedFile = mappedFiles.get(index);\n                    byteBuffer = mappedFile.sliceByteBuffer();\n                    processOffset = mappedFile.getFileFromOffset();\n                    mappedFileOffset = 0;\n                    log.info(\"Recover next batch consume queue file: \" + mappedFile.getFileName());\n                }\n            }\n\n            processOffset += mappedFileOffset;\n            mappedFileQueue.setFlushedWhere(processOffset);\n            mappedFileQueue.setCommittedWhere(processOffset);\n            mappedFileQueue.truncateDirtyFiles(processOffset);\n            reviseMaxAndMinOffsetInQueue();\n        }\n    }\n\n    public ReferredIterator<CqUnit> iterateFromOrNext(long startOffset) {\n        SelectMappedBufferResult sbr = getBatchMsgIndexOrNextBuffer(startOffset);\n        if (sbr == null) {\n            return null;\n        }\n        return new BatchConsumeQueueIterator(sbr);\n    }\n\n    /**\n     * Gets SelectMappedBufferResult by batch-message offset, if not found will return the next valid offset buffer\n     * Node: the caller is responsible for the release of SelectMappedBufferResult\n     * @param msgOffset\n     * @return SelectMappedBufferResult\n     */\n    public SelectMappedBufferResult getBatchMsgIndexOrNextBuffer(final long msgOffset) {\n\n        MappedFile targetBcq;\n\n        if (msgOffset <= minOffsetInQueue) {\n            targetBcq = mappedFileQueue.getFirstMappedFile();\n        } else {\n            targetBcq = searchFileByOffsetOrRight(msgOffset);\n        }\n\n        if (targetBcq == null) {\n            return null;\n        }\n\n        BatchOffsetIndex minOffset = getMinMsgOffset(targetBcq, false, false);\n        BatchOffsetIndex maxOffset = getMaxMsgOffset(targetBcq, false, false);\n        if (null == minOffset || null == maxOffset) {\n            return null;\n        }\n\n        SelectMappedBufferResult sbr = minOffset.getMappedFile().selectMappedBuffer(0);\n        try {\n            ByteBuffer byteBuffer = sbr.getByteBuffer();\n            int left = minOffset.getIndexPos();\n            int right = maxOffset.getIndexPos();\n            int mid = binarySearchRight(byteBuffer, left, right, CQ_STORE_UNIT_SIZE, MSG_BASE_OFFSET_INDEX, msgOffset, BoundaryType.LOWER);\n            if (mid != -1) {\n                return minOffset.getMappedFile().selectMappedBuffer(mid);\n            }\n        } finally {\n            sbr.release();\n        }\n\n        return null;\n    }\n\n    protected MappedFile searchOffsetFromCacheOrRight(long msgOffset) {\n        Map.Entry<Long, MappedFile> ceilingEntry = this.offsetCache.ceilingEntry(msgOffset);\n        if (ceilingEntry == null) {\n            return null;\n        } else {\n            return ceilingEntry.getValue();\n        }\n    }\n\n    protected MappedFile searchFileByOffsetOrRight(long msgOffset) {\n        MappedFile targetBcq = null;\n        boolean searchBcqByCacheEnable = this.messageStore.getMessageStoreConfig().isSearchBcqByCacheEnable();\n        if (searchBcqByCacheEnable) {\n            // it's not the last BCQ file, so search it through cache.\n            targetBcq = this.searchOffsetFromCacheOrRight(msgOffset);\n            // not found in cache\n            if (targetBcq == null) {\n                MappedFile firstBcq = mappedFileQueue.getFirstMappedFile();\n                BatchOffsetIndex minForFirstBcq = getMinMsgOffset(firstBcq, false, false);\n                if (minForFirstBcq != null && minForFirstBcq.getMsgOffset() <= msgOffset && msgOffset < maxOffsetInQueue) {\n                    // old search logic\n                    targetBcq = this.searchOffsetFromFilesOrRight(msgOffset);\n                }\n                log.warn(\"cache is not working on BCQ [Topic: {}, QueueId: {}] for msgOffset: {}, targetBcq: {}\", this.topic, this.queueId, msgOffset, targetBcq);\n            }\n        } else {\n            // old search logic\n            targetBcq = this.searchOffsetFromFilesOrRight(msgOffset);\n        }\n\n        return targetBcq;\n    }\n\n    public MappedFile searchOffsetFromFilesOrRight(long msgOffset) {\n        MappedFile targetBcq = null;\n        // find the mapped file one by one reversely\n        int mappedFileNum = this.mappedFileQueue.getMappedFiles().size();\n        for (int i = mappedFileNum - 1; i >= 0; i--) {\n            MappedFile mappedFile = mappedFileQueue.getMappedFiles().get(i);\n            BatchOffsetIndex tmpMinMsgOffset = getMinMsgOffset(mappedFile, false, false);\n            BatchOffsetIndex tmpMaxMsgOffset = getMaxMsgOffset(mappedFile, false, false);\n            if (null != tmpMaxMsgOffset && tmpMaxMsgOffset.getMsgOffset() < msgOffset) {\n                if (i != mappedFileNum - 1) {   //not the last mapped file max msg offset\n                    targetBcq = mappedFileQueue.getMappedFiles().get(i + 1);\n                    break;\n                }\n            }\n\n            if (null != tmpMinMsgOffset && tmpMinMsgOffset.getMsgOffset() <= msgOffset\n                && null != tmpMaxMsgOffset && msgOffset <= tmpMaxMsgOffset.getMsgOffset()) {\n                targetBcq = mappedFile;\n                break;\n            }\n        }\n\n        return targetBcq;\n    }\n\n    private MappedFile getPreFile(MappedFile file) {\n        int index = mappedFileQueue.getMappedFiles().indexOf(file);\n        if (index < 1) {\n            // indicate that this is the first file or not found\n            return null;\n        } else {\n            return mappedFileQueue.getMappedFiles().get(index - 1);\n        }\n    }\n\n    private void cacheOffset(MappedFile file, Function<MappedFile, BatchOffsetIndex> offsetGetFunc) {\n        try {\n            BatchOffsetIndex offset = offsetGetFunc.apply(file);\n            if (offset != null) {\n                this.offsetCache.put(offset.getMsgOffset(), offset.getMappedFile());\n                this.timeCache.put(offset.getStoreTimestamp(), offset.getMappedFile());\n            }\n        } catch (Exception e) {\n            log.error(\"Failed caching offset and time on BCQ [Topic: {}, QueueId: {}, File: {}]\",\n                this.topic, this.queueId, file);\n        }\n    }\n\n    @Override\n    protected void cacheBcq(MappedFile bcq) {\n        MappedFile file = getPreFile(bcq);\n        if (file != null) {\n            cacheOffset(file, m -> getMaxMsgOffset(m, false, true));\n        }\n    }\n\n    public void putEndPositionInfo(MappedFile mappedFile) {\n        // cache max offset\n        if (!mappedFile.isFull()) {\n            this.byteBufferItem.flip();\n            this.byteBufferItem.limit(CQ_STORE_UNIT_SIZE);\n            this.byteBufferItem.putLong(-1);\n            this.byteBufferItem.putInt(0);\n            this.byteBufferItem.putLong(0);\n            this.byteBufferItem.putLong(0);\n            this.byteBufferItem.putLong(0);\n            this.byteBufferItem.putShort((short)0);\n            this.byteBufferItem.putInt(INVALID_POS);\n            this.byteBufferItem.putInt(0); // 4 bytes reserved\n\n            boolean appendRes;\n\n            if (messageStore.getMessageStoreConfig().isPutConsumeQueueDataByFileChannel()) {\n                appendRes = mappedFile.appendMessageUsingFileChannel(this.byteBufferItem.array());\n            } else {\n                appendRes = mappedFile.appendMessage(this.byteBufferItem.array());\n            }\n\n            if (!appendRes) {\n                log.error(\"append end position info into {} failed\", mappedFile.getFileName());\n            }\n        }\n\n        cacheOffset(mappedFile, m -> getMaxMsgOffset(m, false, true));\n    }\n\n    public MappedFile createFile(final long physicalOffset) throws IOException {\n        // cache max offset\n        return mappedFileQueue.tryCreateMappedFile(physicalOffset);\n    }\n\n    public boolean isLastFileFull() {\n        if (mappedFileQueue.getLastMappedFile() != null) {\n            return mappedFileQueue.getLastMappedFile().isFull();\n        } else {\n            return true;\n        }\n    }\n\n    public boolean shouldRoll() {\n        if (mappedFileQueue.getLastMappedFile() == null) {\n            return true;\n        }\n        if (mappedFileQueue.getLastMappedFile().isFull()) {\n            return true;\n        }\n        if (mappedFileQueue.getLastMappedFile().getWrotePosition() + BatchConsumeQueue.CQ_STORE_UNIT_SIZE\n            > mappedFileQueue.getMappedFileSize()) {\n            return true;\n        }\n\n        return false;\n    }\n\n    public boolean containsOffsetFile(final long physicalOffset) {\n        String fileName = UtilAll.offset2FileName(physicalOffset);\n        return mappedFileQueue.getMappedFiles().stream()\n            .anyMatch(mf -> Objects.equals(mf.getFile().getName(), fileName));\n    }\n\n    public long getMaxPhyOffsetInLog() {\n        MappedFile lastMappedFile = mappedFileQueue.getLastMappedFile();\n        Long maxOffsetInLog = getMax(lastMappedFile, b -> b.getLong(0) + b.getInt(8));\n        if (maxOffsetInLog != null) {\n            return maxOffsetInLog;\n        } else {\n            return -1;\n        }\n    }\n\n    private <T> T getMax(MappedFile mappedFile, Function<ByteBuffer, T> function) {\n        if (mappedFile == null || mappedFile.getReadPosition() < CQ_STORE_UNIT_SIZE) {\n            return null;\n        }\n\n        ByteBuffer byteBuffer = mappedFile.sliceByteBuffer();\n        for (int i = mappedFile.getReadPosition() - CQ_STORE_UNIT_SIZE; i >= 0; i -= CQ_STORE_UNIT_SIZE) {\n            byteBuffer.position(i);\n            long offset = byteBuffer.getLong();\n            int size = byteBuffer.getInt();\n            long tagsCode = byteBuffer.getLong();   //tagscode\n            long timestamp = byteBuffer.getLong();  //timestamp\n            long msgBaseOffset = byteBuffer.getLong();\n            short batchSize = byteBuffer.getShort();\n            if (offset >= 0 && size > 0 && msgBaseOffset >= 0 && batchSize > 0) {\n                byteBuffer.position(i);     //reset position\n                return function.apply(byteBuffer.slice());\n            }\n        }\n\n        return null;\n    }\n\n    @Override\n    protected BatchOffsetIndex getMaxMsgOffset(MappedFile mappedFile, boolean getBatchSize, boolean getStoreTime) {\n        if (mappedFile == null || mappedFile.getReadPosition() < CQ_STORE_UNIT_SIZE) {\n            return null;\n        }\n\n        ByteBuffer byteBuffer = mappedFile.sliceByteBuffer();\n        for (int i = mappedFile.getReadPosition() - CQ_STORE_UNIT_SIZE; i >= 0; i -= CQ_STORE_UNIT_SIZE) {\n            byteBuffer.position(i);\n            long offset = byteBuffer.getLong();\n            int size = byteBuffer.getInt();\n            byteBuffer.getLong();   //tagscode\n            long timestamp = byteBuffer.getLong();//timestamp\n            long msgBaseOffset = byteBuffer.getLong();\n            short batchSize = byteBuffer.getShort();\n            if (offset >= 0 && size > 0 && msgBaseOffset >= 0 && batchSize > 0) {\n//                mappedFile.setWrotePosition(i + CQ_STORE_UNIT_SIZE);\n//                mappedFile.setFlushedPosition(i + CQ_STORE_UNIT_SIZE);\n//                mappedFile.setCommittedPosition(i + CQ_STORE_UNIT_SIZE);\n                return new BatchOffsetIndex(mappedFile, i, msgBaseOffset, batchSize, timestamp);\n            }\n        }\n\n        return null;\n    }\n\n    public long getMaxMsgOffsetFromFile(String simpleFileName) {\n        MappedFile mappedFile = mappedFileQueue.getMappedFiles().stream()\n            .filter(m -> Objects.equals(m.getFile().getName(), simpleFileName))\n            .findFirst()\n            .orElse(null);\n\n        if (mappedFile == null) {\n            return -1;\n        }\n\n        BatchOffsetIndex max = getMaxMsgOffset(mappedFile, false, false);\n        if (max == null) {\n            return -1;\n        }\n        return max.getMsgOffset();\n    }\n\n    private void refreshMaxCache() {\n        doRefreshCache(m -> getMaxMsgOffset(m, false, true));\n    }\n\n    @Override\n    protected void refreshCache() {\n        refreshMaxCache();\n    }\n\n    public void refresh() {\n        reviseMaxAndMinOffsetInQueue();\n        refreshCache();\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntry.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.queue.offset;\n\npublic class OffsetEntry {\n    /**\n     * Topic identifier. For now, it's topic name directly. In the future, we should use fixed length topic identifier.\n     */\n    public String topic;\n\n    /**\n     * Queue ID\n     */\n    public int queueId;\n\n    /**\n     * Flag if the entry is for maximum or minimum\n     */\n    public OffsetEntryType type;\n\n    /**\n     * Maximum or minimum consume-queue offset.\n     */\n    public long offset;\n\n    /**\n     * Maximum or minimum commit-log offset.\n     */\n    public long commitLogOffset;\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntryType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.queue.offset;\n\npublic enum OffsetEntryType {\n    MAXIMUM,\n    MINIMUM\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueCompactionFilterFactory.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.rocksdb;\n\nimport java.util.function.LongSupplier;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.rocksdb.AbstractCompactionFilter;\nimport org.rocksdb.AbstractCompactionFilterFactory;\nimport org.rocksdb.RemoveConsumeQueueCompactionFilter;\n\npublic class ConsumeQueueCompactionFilterFactory extends AbstractCompactionFilterFactory<RemoveConsumeQueueCompactionFilter> {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME);\n    private final LongSupplier minPhyOffsetSupplier;\n\n    public ConsumeQueueCompactionFilterFactory(final LongSupplier minPhyOffsetSupplier) {\n        this.minPhyOffsetSupplier = minPhyOffsetSupplier;\n    }\n\n    @Override\n    public String name() {\n        return \"ConsumeQueueCompactionFilterFactory\";\n    }\n\n    @Override\n    public RemoveConsumeQueueCompactionFilter createCompactionFilter(final AbstractCompactionFilter.Context context) {\n        long minPhyOffset = this.minPhyOffsetSupplier.getAsLong();\n        LOGGER.info(\"manualCompaction minPhyOffset: {}, isFull: {}, isManual: {}\",\n                minPhyOffset, context.isFullCompaction(), context.isManualCompaction());\n        return new RemoveConsumeQueueCompactionFilter(minPhyOffset);\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.rocksdb;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.config.AbstractRocksDBStorage;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.rocksdb.ColumnFamilyDescriptor;\nimport org.rocksdb.ColumnFamilyHandle;\nimport org.rocksdb.ColumnFamilyOptions;\nimport org.rocksdb.ReadOptions;\nimport org.rocksdb.RocksDB;\nimport org.rocksdb.RocksDBException;\nimport org.rocksdb.RocksIterator;\nimport org.rocksdb.WriteBatch;\n\npublic class ConsumeQueueRocksDBStorage extends AbstractRocksDBStorage {\n\n    public static final byte[] OFFSET_COLUMN_FAMILY = \"offset\".getBytes(StandardCharsets.UTF_8);\n\n    private final MessageStore messageStore;\n    private volatile ColumnFamilyHandle offsetCFHandle;\n\n    private ConsumeQueueCompactionFilterFactory compactionFilterFactory;\n\n    public ConsumeQueueRocksDBStorage(final MessageStore messageStore, final String dbPath) {\n        super(dbPath);\n        this.messageStore = messageStore;\n        this.readOnly = false;\n    }\n\n    protected void initOptions() {\n        this.options = RocksDBOptionsFactory.createDBOptions();\n        super.initOptions();\n    }\n\n    @Override\n    protected void initTotalOrderReadOptions() {\n        this.totalOrderReadOptions = new ReadOptions();\n        this.totalOrderReadOptions.setPrefixSameAsStart(false);\n        this.totalOrderReadOptions.setTotalOrderSeek(false);\n    }\n\n    @Override\n    protected boolean postLoad() {\n        try {\n            UtilAll.ensureDirOK(this.dbPath);\n\n            initOptions();\n\n            final List<ColumnFamilyDescriptor> cfDescriptors = new ArrayList<>();\n\n            this.compactionFilterFactory = new ConsumeQueueCompactionFilterFactory(messageStore::getMinPhyOffset);\n\n            ColumnFamilyOptions cqCfOptions = RocksDBOptionsFactory.createCQCFOptions(this.messageStore, this.compactionFilterFactory);\n            this.cfOptions.add(cqCfOptions);\n            cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cqCfOptions));\n\n            ColumnFamilyOptions offsetCfOptions = RocksDBOptionsFactory.createOffsetCFOptions();\n            this.cfOptions.add(offsetCfOptions);\n            cfDescriptors.add(new ColumnFamilyDescriptor(OFFSET_COLUMN_FAMILY, offsetCfOptions));\n            open(cfDescriptors);\n            this.defaultCFHandle = cfHandles.get(0);\n            this.offsetCFHandle = cfHandles.get(1);\n        } catch (final Exception e) {\n            LOGGER.error(\"postLoad Failed. {}\", this.dbPath, e);\n            return false;\n        }\n        return true;\n    }\n\n    @Override\n    protected void preShutdown() {\n        if (this.offsetCFHandle != null) {\n            this.offsetCFHandle.close();\n        }\n\n        if (this.compactionFilterFactory != null) {\n            this.compactionFilterFactory.close();\n        }\n\n    }\n\n    public byte[] getCQ(final byte[] keyBytes) throws RocksDBException {\n        return get(this.defaultCFHandle, this.totalOrderReadOptions, keyBytes);\n    }\n\n    public byte[] getOffset(final byte[] keyBytes) throws RocksDBException {\n        return get(this.offsetCFHandle, this.totalOrderReadOptions, keyBytes);\n    }\n\n    public List<byte[]> multiGet(final List<ColumnFamilyHandle> cfhList,\n        final List<byte[]> keys) throws RocksDBException {\n        return multiGet(this.totalOrderReadOptions, cfhList, keys);\n    }\n\n    public void batchPut(final WriteBatch batch) throws RocksDBException {\n        batchPut(this.writeOptions, batch);\n    }\n\n    public void manualCompaction(final long minPhyOffset) {\n        try {\n            manualCompaction(minPhyOffset, this.compactRangeOptions);\n        } catch (Exception e) {\n            LOGGER.error(\"manualCompaction Failed. minPhyOffset: {}\", minPhyOffset, e);\n        }\n    }\n\n    public RocksIterator seekOffsetCF() {\n        return this.db.newIterator(this.offsetCFHandle, this.totalOrderReadOptions);\n    }\n\n    public ColumnFamilyHandle getOffsetCFHandle() {\n        return this.offsetCFHandle;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/rocksdb/MessageRocksDBStorage.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.rocksdb;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Paths;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport com.google.common.cache.Cache;\nimport com.google.common.cache.CacheBuilder;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.config.AbstractRocksDBStorage;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.index.rocksdb.IndexRocksDBRecord;\nimport org.apache.rocketmq.store.timer.rocksdb.TimerRocksDBRecord;\nimport org.apache.rocketmq.store.transaction.TransRocksDBRecord;\nimport org.rocksdb.ColumnFamilyDescriptor;\nimport org.rocksdb.ColumnFamilyHandle;\nimport org.rocksdb.ColumnFamilyOptions;\nimport org.rocksdb.ReadOptions;\nimport org.rocksdb.RocksDB;\nimport org.rocksdb.RocksIterator;\nimport org.rocksdb.Slice;\nimport org.rocksdb.WriteBatch;\nimport static org.apache.rocketmq.common.MixAll.dealTimeToHourStamps;\nimport static org.apache.rocketmq.common.MixAll.getHours;\nimport static org.apache.rocketmq.common.MixAll.isHourTime;\nimport static org.apache.rocketmq.store.index.rocksdb.IndexRocksDBRecord.KEY_SPLIT;\nimport static org.apache.rocketmq.store.index.rocksdb.IndexRocksDBRecord.KEY_SPLIT_BYTES;\nimport static org.apache.rocketmq.store.timer.rocksdb.TimerRocksDBRecord.TIMER_ROCKSDB_DELETE;\nimport static org.apache.rocketmq.store.timer.rocksdb.TimerRocksDBRecord.TIMER_ROCKSDB_PUT;\nimport static org.apache.rocketmq.store.timer.rocksdb.TimerRocksDBRecord.TIMER_ROCKSDB_UPDATE;\n\npublic class MessageRocksDBStorage extends AbstractRocksDBStorage {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private static final Logger logError = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);\n    private static final String ROCKSDB_MESSAGE_DIRECTORY = \"rocksdbstore\";\n\n    public static final byte[] TIMER_COLUMN_FAMILY = \"timer\".getBytes(StandardCharsets.UTF_8);\n    public static final byte[] TRANS_COLUMN_FAMILY = \"trans\".getBytes(StandardCharsets.UTF_8);\n    private static final byte[] LAST_OFFSET_PY = \"lastOffsetPy\".getBytes(StandardCharsets.UTF_8);\n    private static final byte[] LAST_STORE_TIMESTAMP = \"lastStoreTimeStamp\".getBytes(StandardCharsets.UTF_8);\n    private static final byte[] END_SUFFIX_BYTES = new byte[512];\n    static {\n        Arrays.fill(END_SUFFIX_BYTES, (byte) 0xFF);\n    }\n    private static final Set<byte[]> COMMON_CHECK_POINT_KEY_SET_FOR_TIMER = new HashSet<>();\n    public static final byte[] SYS_TOPIC_SCAN_OFFSET_CHECK_POINT = \"sys_topic_scan_offset_checkpoint\".getBytes(StandardCharsets.UTF_8);\n    public static final byte[] TIMELINE_CHECK_POINT = \"timeline_checkpoint\".getBytes(StandardCharsets.UTF_8);\n    static {\n        COMMON_CHECK_POINT_KEY_SET_FOR_TIMER.add(SYS_TOPIC_SCAN_OFFSET_CHECK_POINT);\n        COMMON_CHECK_POINT_KEY_SET_FOR_TIMER.add(TIMELINE_CHECK_POINT);\n    }\n    private static final byte[] DELETE_VAL_FLAG = new byte[] {(byte)0xFF};\n    private static final int LAST_OFFSET_PY_LENGTH = LAST_OFFSET_PY.length;\n\n    private volatile ColumnFamilyHandle timerCFHandle;\n    private volatile ColumnFamilyHandle transCFHandle;\n\n    private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);\n    private static final Cache<byte[], byte[]> DELETE_KEY_CACHE_FOR_TIMER = CacheBuilder.newBuilder()\n        .maximumSize(10000)\n        .expireAfterWrite(60, TimeUnit.MINUTES)\n        .build();\n\n    public MessageRocksDBStorage(MessageStoreConfig messageStoreConfig) {\n        super(Paths.get(messageStoreConfig.getStorePathRootDir(), ROCKSDB_MESSAGE_DIRECTORY).toString());\n        this.start();\n    }\n\n    @Override\n    protected boolean postLoad() {\n        try {\n            UtilAll.ensureDirOK(this.dbPath);\n            initOptions();\n            ColumnFamilyOptions indexCFOptions = RocksDBOptionsFactory.createIndexCFOptions();\n            ColumnFamilyOptions timerCFOptions = RocksDBOptionsFactory.createTimerCFOptions();\n            ColumnFamilyOptions transCFOptions = RocksDBOptionsFactory.createTransCFOptions();\n            this.cfOptions.add(indexCFOptions);\n            this.cfOptions.add(timerCFOptions);\n            this.cfOptions.add(transCFOptions);\n\n            List<ColumnFamilyDescriptor> cfDescriptors = new ArrayList<>();\n            cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, indexCFOptions));\n            cfDescriptors.add(new ColumnFamilyDescriptor(TIMER_COLUMN_FAMILY, timerCFOptions));\n            cfDescriptors.add(new ColumnFamilyDescriptor(TRANS_COLUMN_FAMILY, transCFOptions));\n            this.open(cfDescriptors);\n            this.defaultCFHandle = cfHandles.get(0);\n            this.timerCFHandle = cfHandles.get(1);\n            this.transCFHandle = cfHandles.get(2);\n            scheduler.scheduleAtFixedRate(() -> {\n                try {\n                    db.flush(flushOptions, timerCFHandle);\n                    log.info(\"MessageRocksDBStorage flush timer wal success\");\n                } catch (Exception e) {\n                    logError.error(\"MessageRocksDBStorage flush timer wal failed, error: {}\", e.getMessage());\n                }\n            }, 5, 5, TimeUnit.MINUTES);\n\n            log.info(\"MessageRocksDBStorage init success, dbPath: {}\", this.dbPath);\n        } catch (final Exception e) {\n            logError.error(\"MessageRocksDBStorage init error, dbPath: {}, error: {}\", this.dbPath, e.getMessage());\n            return false;\n        }\n        return true;\n    }\n\n    protected void initOptions() {\n        this.options = RocksDBOptionsFactory.createDBOptions();\n        super.initOptions();\n    }\n\n    public String getFilePath() {\n        return this.dbPath;\n    }\n\n    @Override\n    protected void preShutdown() {\n        log.info(\"MessageRocksDBStorage pre shutdown success, dbPath: {}\", this.dbPath);\n    }\n\n    public List<Long> queryOffsetForIndex(byte[] columnFamily, String topic, String indexType, String key, long beginTime, long endTime, int maxNum, String lastKey) {\n        ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily);\n        if (null == cfHandle || StringUtils.isEmpty(topic) || StringUtils.isEmpty(indexType) || StringUtils.isEmpty(key) || beginTime < 0L || endTime <= 0L || beginTime > endTime || maxNum <= 0) {\n            logError.error(\"MessageRocksDBStorage queryOffsetForIndex param error, cfHandle: {}, topic: {}, indexType: {}, key: {}, beginTime: {}, endTime: {}, maxNum: {}\", cfHandle, topic, indexType, key, beginTime, endTime, maxNum);\n            return null;\n        }\n        Long lastIndexTime = getLastIndexTimeForIndex(lastKey);\n        if (!StringUtils.isEmpty(lastKey) && (null == lastIndexTime || lastIndexTime <= 0L || !isHourTime(lastIndexTime))) {\n            logError.error(\"MessageRocksDBStorage queryOffsetForIndex parse and check lastIndexTime error, lastIndexTime: {}, lastKey: {}\", lastIndexTime, lastKey);\n            return null;\n        }\n        List<Long> hours = getHours(beginTime, endTime);\n        if (CollectionUtils.isEmpty(hours)) {\n            logError.error(\"MessageRocksDBStorage queryOffsetForIndex param error, hours is empty, beginTime: {}, endTime: {}\", beginTime, endTime);\n            return null;\n        }\n        List<Long> offsetPyList = new ArrayList<>(maxNum);\n        String keyMiddleStr = KEY_SPLIT + topic + KEY_SPLIT + indexType + KEY_SPLIT + key + KEY_SPLIT;\n        byte[] keyMiddleBytes = keyMiddleStr.getBytes(StandardCharsets.UTF_8);\n        for (Long hour : hours) {\n            if (null == hour || null != lastIndexTime && hour < lastIndexTime) {\n                continue;\n            }\n            byte[] seekKeyBytes = null;\n            byte[] lastKeyBytes = null;\n            byte[] keyPrefixBytes = ByteBuffer.allocate(Long.BYTES + keyMiddleBytes.length).putLong(hour).put(keyMiddleBytes).array();\n            if (!StringUtils.isEmpty(lastKey) && hour.equals(lastIndexTime)) {\n                seekKeyBytes = lastKeyToBytes(lastKey);\n                lastKeyBytes = seekKeyBytes;\n            } else {\n                seekKeyBytes = keyPrefixBytes;\n            }\n            if (null == seekKeyBytes) {\n                logError.error(\"MessageRocksDBStorage queryOffsetForIndex error, seekKeyBytes is null\");\n                return null;\n            }\n            try (RocksIterator iterator = db.newIterator(cfHandle, readOptions)) {\n                for (iterator.seek(seekKeyBytes); iterator.isValid(); iterator.next()) {\n                    try {\n                        byte[] currentKeyBytes = iterator.key();\n                        if (null == currentKeyBytes || currentKeyBytes.length == 0) {\n                            break;\n                        }\n                        if (null != lastKeyBytes && currentKeyBytes.length == lastKeyBytes.length && MixAll.isByteArrayEqual(currentKeyBytes, 0, currentKeyBytes.length, lastKeyBytes, 0, lastKeyBytes.length)) {\n                            continue;\n                        }\n                        if (currentKeyBytes.length < keyPrefixBytes.length || !MixAll.isByteArrayEqual(currentKeyBytes, 0, keyPrefixBytes.length, keyPrefixBytes, 0, keyPrefixBytes.length)) {\n                            break;\n                        }\n                        ByteBuffer valueBuffer = ByteBuffer.wrap(iterator.value());\n                        long storeTime = valueBuffer.getLong();\n                        if (storeTime >= beginTime && storeTime <= endTime) {\n                            byte[] indexKey = iterator.key();\n                            if (null == indexKey || indexKey.length < Long.BYTES) {\n                                continue;\n                            }\n                            byte[] bytes = Arrays.copyOfRange(indexKey, indexKey.length - Long.BYTES, indexKey.length);\n                            long offset = ByteBuffer.wrap(bytes).getLong();\n                            offsetPyList.add(offset);\n                            if (offsetPyList.size() >= maxNum) {\n                                return offsetPyList;\n                            }\n                        }\n                    } catch (Exception e) {\n                        logError.error(\"MessageRocksDBStorage queryOffsetForIndex iterator error: {}\", e.getMessage());\n                    }\n                }\n            } catch (Exception e) {\n                logError.error(\"MessageRocksDBStorage queryOffsetForIndex error: {}\", e.getMessage());\n            }\n        }\n        return offsetPyList;\n    }\n\n    private byte[] lastKeyToBytes(String lastKey) {\n        if (StringUtils.isEmpty(lastKey)) {\n            return null;\n        }\n        String[] split = lastKey.split(KEY_SPLIT);\n        if (split.length != 6) {\n            log.error(\"MessageRocksDBStorage lastKeyToBytes split error, lastKey: {}\", lastKey);\n            return null;\n        }\n        try {\n            long storeTimeHour = Long.parseLong(split[0]);\n            long offsetPy = Long.parseLong(split[split.length - 1]);\n            StringBuilder stringBuilder = new StringBuilder();\n            for (int i = 1; i < split.length - 1; i++) {\n                stringBuilder.append(KEY_SPLIT).append(split[i]);\n            }\n            byte[] middleKeyBytes = stringBuilder.append(KEY_SPLIT).toString().getBytes(StandardCharsets.UTF_8);\n            return ByteBuffer.allocate(Long.BYTES + middleKeyBytes.length + Long.BYTES).putLong(storeTimeHour).put(middleKeyBytes).putLong(offsetPy).array();\n        } catch (Exception e) {\n            log.error(\"MessageRocksDBStorage lastKeyToBytes error, lastKey: {}, error: {}\", lastKey, e.getMessage());\n            return null;\n        }\n    }\n\n    public void deleteRecordsForIndex(byte[] columnFamily, long storeTime, int hours) {\n        ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily);\n        if (null == cfHandle || storeTime < 0L || hours <= 0) {\n            logError.error(\"MessageRocksDBStorage deleteRecordsForIndex param error, storeTime: {}, hours: {}\", storeTime, hours);\n            return;\n        }\n        long endTime = dealTimeToHourStamps(storeTime);\n        long startTime = endTime - TimeUnit.HOURS.toMillis(hours);\n        try {\n            byte[] startKey = ByteBuffer.allocate(Long.BYTES + KEY_SPLIT_BYTES.length).putLong(startTime).put(KEY_SPLIT_BYTES).array();\n            byte[] endKey = ByteBuffer.allocate(Long.BYTES + KEY_SPLIT_BYTES.length + END_SUFFIX_BYTES.length).putLong(endTime).put(KEY_SPLIT_BYTES).put(END_SUFFIX_BYTES).array();\n            rangeDelete(cfHandle, ableWalWriteOptions, startKey, endKey);\n            log.info(\"MessageRocksDBStorage deleteRecordsForIndex delete success, storeTime: {}, hours: {}\", storeTime, hours);\n        } catch (Exception e) {\n            logError.error(\"MessageRocksDBStorage deleteRecordsForIndex delete error, storeTime: {}, hours: {}, error: {}\", storeTime, hours, e.getMessage());\n        }\n    }\n\n    public void writeRecordsForIndex(byte[] columnFamily, List<IndexRocksDBRecord> recordList) {\n        ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily);\n        if (null == cfHandle || CollectionUtils.isEmpty(recordList)) {\n            return;\n        }\n        try (WriteBatch writeBatch = new WriteBatch()) {\n            for (IndexRocksDBRecord record : recordList) {\n                try {\n                    if (null == record) {\n                        logError.warn(\"MessageRocksDBStorage writeRecordsForIndex error, record is null\");\n                        continue;\n                    }\n                    byte[] keyBytes = record.getKeyBytes();\n                    byte[] valueBytes = record.getValueBytes();\n                    if (null == keyBytes || keyBytes.length == 0 || null == valueBytes || valueBytes.length == 0) {\n                        logError.error(\"MessageRocksDBStorage writeRecordsForIndex param error, keyBytes: {}, valueBytes: {}\", keyBytes, valueBytes);\n                        continue;\n                    }\n                    writeBatch.put(cfHandle, keyBytes, valueBytes);\n                } catch (Exception e) {\n                    logError.error(\"MessageRocksDBStorage writeRecordsForIndex error: {}\", e.getMessage());\n                }\n            }\n            IndexRocksDBRecord lastRecord = recordList.get(recordList.size() - 1);\n            if (null != lastRecord && StringUtils.isEmpty(lastRecord.getKey()) && StringUtils.isEmpty(lastRecord.getTag())) {\n                long offset = lastRecord.getOffsetPy();\n                Long lastOffsetPy = getLastOffsetPy(columnFamily);\n                if (null == lastOffsetPy || offset > lastOffsetPy) {\n                    writeBatch.put(cfHandle, LAST_OFFSET_PY, ByteBuffer.allocate(Long.BYTES).putLong(offset).array());\n                }\n                long storeTime = lastRecord.getStoreTime();\n                Long lastStoreTimeStamp = getLastStoreTimeStampForIndex(columnFamily);\n                if (null == lastStoreTimeStamp || storeTime > lastStoreTimeStamp) {\n                    writeBatch.put(cfHandle, LAST_STORE_TIMESTAMP, ByteBuffer.allocate(Long.BYTES).putLong(storeTime).array());\n                }\n            }\n            batchPut(ableWalWriteOptions, writeBatch);\n        } catch (Exception e) {\n            logError.error(\"MessageRocksDBStorage writeRecordsForIndex error: {}\", e.getMessage());\n        }\n    }\n\n    public Long getLastStoreTimeStampForIndex(byte[] columnFamily) {\n        ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily);\n        if (null == cfHandle) {\n            return null;\n        }\n        try {\n            byte[] storeTime = get(cfHandle, readOptions, LAST_STORE_TIMESTAMP);\n            return null == storeTime ? 0L : ByteBuffer.wrap(storeTime).getLong();\n        } catch (Exception e) {\n            logError.error(\"MessageRocksDBStorage getLastStoreTimeStampForIndex error: {}\", e.getMessage());\n            return null;\n        }\n    }\n\n    private static Long getLastIndexTimeForIndex(String lastKey) {\n        if (StringUtils.isEmpty(lastKey)) {\n            return null;\n        }\n        try {\n            String[] split = lastKey.split(KEY_SPLIT);\n            if (split.length > 0) {\n                return Long.valueOf(split[0]);\n            }\n        } catch (Exception e) {\n            logError.error(\"MessageRocksDBStorage getLastIndexTimeForIndex error lastKey: {}, e: {}\", lastKey, e.getMessage());\n        }\n        return null;\n    }\n\n    public void writeRecordsForTimer(byte[] columnFamily, List<TimerRocksDBRecord> recordList) {\n        ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily);\n        if (null == cfHandle || CollectionUtils.isEmpty(recordList)) {\n            return;\n        }\n        try (WriteBatch writeBatch = new WriteBatch()) {\n            for (TimerRocksDBRecord record : recordList) {\n                if (null == record) {\n                    logError.error(\"MessageRocksDBStorage writeRecordsForTimer error, record is null\");\n                    continue;\n                }\n                try {\n                    byte[] keyBytes = record.getKeyBytes();\n                    byte[] valueBytes = record.getValueBytes();\n                    if (null == keyBytes || keyBytes.length == 0 || null == valueBytes || valueBytes.length == 0) {\n                        logError.error(\"MessageRocksDBStorage writeRecordsForTimer param error, keyBytes: {}, valueBytes: {}\", keyBytes, valueBytes);\n                        continue;\n                    }\n                    if (record.getActionFlag() == TIMER_ROCKSDB_PUT) {\n                        writeBatch.put(cfHandle, keyBytes, valueBytes);\n                    } else if (record.getActionFlag() == TIMER_ROCKSDB_DELETE) {\n                        writeBatch.delete(cfHandle, keyBytes);\n                        DELETE_KEY_CACHE_FOR_TIMER.put(keyBytes, DELETE_VAL_FLAG);\n                    } else if (record.getActionFlag() == TIMER_ROCKSDB_UPDATE) {\n                        byte[] deleteByte = DELETE_KEY_CACHE_FOR_TIMER.getIfPresent(keyBytes);\n                        if (null == deleteByte) {\n                            writeBatch.put(cfHandle, keyBytes, valueBytes);\n                        }\n                    } else {\n                        logError.error(\"MessageRocksDBStorage writeRecordsForTimer record actionFlag error, actionFlag: {}\", record.getActionFlag());\n                    }\n                } catch (Exception e) {\n                    logError.error(\"MessageRocksDBStorage writeRecordsForTimer error: {}\", e.getMessage());\n                }\n            }\n            batchPut(ableWalWriteOptions, writeBatch);\n        } catch (Exception e) {\n            logError.error(\"MessageRocksDBStorage writeRecordsForTimer error: {}\", e.getMessage());\n        }\n    }\n\n    public List<TimerRocksDBRecord> scanRecordsForTimer(byte[] columnFamily, long lowerTime, long upperTime, int size, byte[] startKey) {\n        ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily);\n        if (null == cfHandle || lowerTime <= 0L || upperTime <= 0L || lowerTime > upperTime || size <= 0) {\n            return null;\n        }\n        RocksIterator iterator = null;\n        try (ReadOptions readOptions = new ReadOptions()\n            .setIterateLowerBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(lowerTime).array()))\n            .setIterateUpperBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(upperTime).array()))) {\n            iterator = db.newIterator(cfHandle, readOptions);\n            if (null == startKey || startKey.length == 0) {\n                iterator.seek(ByteBuffer.allocate(Long.BYTES).putLong(lowerTime).array());\n            } else {\n                iterator.seek(startKey);\n                iterator.next();\n            }\n            List<TimerRocksDBRecord> records = new ArrayList<>();\n            for (; iterator.isValid(); iterator.next()) {\n                try {\n                    TimerRocksDBRecord timerRocksDBRecord = TimerRocksDBRecord.decode(iterator.key(), iterator.value());\n                    if (null == timerRocksDBRecord) {\n                        logError.error(\"MessageRocksDBStorage scanRecordsForTimer error, decode timerRocksDBRecord is null\");\n                        continue;\n                    }\n                    records.add(timerRocksDBRecord);\n                    if (records.size() >= size) {\n                        break;\n                    }\n                } catch (Exception e) {\n                    logError.error(\"MessageRocksDBStorage scanRecordsForTimer iterator error: {}\", e.getMessage());\n                }\n            }\n            return records;\n        } catch (Exception e) {\n            logError.error(\"MessageRocksDBStorage scanRecordsForTimer error: {}\", e.getMessage());\n        } finally {\n            if (null != iterator) {\n                iterator.close();\n            }\n        }\n        return null;\n    }\n\n    public void deleteRecordsForTimer(byte[] columnFamily, long lowerTime, long upperTime) {\n        ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily);\n        if (null == cfHandle || lowerTime <= 0L || upperTime <= 0L || lowerTime > upperTime) {\n            logError.error(\"MessageRocksDBStorage deleteRecordsForTimer param error, cfHandle: {}, lowerTime: {}, upperTime: {}\", cfHandle, lowerTime, upperTime);\n            return;\n        }\n        byte[] startKey = ByteBuffer.allocate(Long.BYTES).putLong(lowerTime).array();\n        byte[] endKey = ByteBuffer.allocate(Long.BYTES + END_SUFFIX_BYTES.length).putLong(upperTime).put(END_SUFFIX_BYTES).array();\n        try {\n            rangeDelete(cfHandle, ableWalWriteOptions, startKey, endKey);\n            log.info(\"MessageRocksDBStorage deleteRecordsForTimer success, lowerTime: {}, upperTime: {}\", lowerTime, upperTime);\n        } catch (Exception e) {\n            logError.error(\"MessageRocksDBStorage deleteRecordsForTimer param error, lowerTime: {}, upperTime: {}, error: {}\", lowerTime, upperTime, e.getMessage());\n        }\n    }\n\n    public void writeCheckPointForTimer(byte[] columnFamily, byte[] key, long value) {\n        ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily);\n        if (null == cfHandle || !COMMON_CHECK_POINT_KEY_SET_FOR_TIMER.contains(key) || value < 0L) {\n            logError.error(\"MessageRocksDBStorage writeCheckPointForTimer param error, cfHandle: {}, key: {}, value: {}\", cfHandle, key, value);\n            return;\n        }\n        try {\n            byte[] valueBytes = ByteBuffer.allocate(Long.BYTES).putLong(value).array();\n            put(cfHandle, ableWalWriteOptions, key, key.length, valueBytes, valueBytes.length);\n        } catch (Exception e) {\n            logError.error(\"MessageRocksDBStorage writeCheckPointForTimer error: {}\", e.getMessage());\n        }\n    }\n\n    public long getCheckpointForTimer(byte[] columnFamily, byte[] key) {\n        ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily);\n        if (null == cfHandle || !COMMON_CHECK_POINT_KEY_SET_FOR_TIMER.contains(key)) {\n            logError.error(\"MessageRocksDBStorage getCheckpointForTimer error, cfHandle: {}, key: {}\", cfHandle, key);\n            return 0L;\n        }\n        try {\n            byte[] checkpoint = get(cfHandle, readOptions, key);\n            if (null == checkpoint && Arrays.equals(key, TIMELINE_CHECK_POINT)) {\n                return (System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(10)) / TimeUnit.SECONDS.toMillis(1) * TimeUnit.SECONDS.toMillis(1);\n            }\n            return checkpoint == null ? 0L : ByteBuffer.wrap(checkpoint).getLong();\n        } catch (Exception e) {\n            logError.error(\"MessageRocksDBStorage getCheckpointForTimer error: {}\", e.getMessage());\n            return 0L;\n        }\n    }\n\n    public void deleteCheckPointForTimer(byte[] columnFamily, byte[] key) {\n        ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily);\n        if (null == cfHandle || !COMMON_CHECK_POINT_KEY_SET_FOR_TIMER.contains(key)) {\n            logError.error(\"MessageRocksDBStorage deleteCheckPointForTimer error, cfHandle: {}, key: {}\", cfHandle, key);\n            return;\n        }\n        try {\n            delete(cfHandle, ableWalWriteOptions, key);\n        } catch (Exception e) {\n            logError.error(\"MessageRocksDBStorage deleteCheckPointForTimer error: {}\", e.getMessage());\n            throw new RuntimeException(\"MessageRocksDBStorage deleteCheckPointForTimer error\", e);\n        }\n    }\n\n    public void writeRecordsForTrans(byte[] columnFamily, List<TransRocksDBRecord> recordList) {\n        ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily);\n        if (null == cfHandle || CollectionUtils.isEmpty(recordList)) {\n            return;\n        }\n        long lastOffsetPy = 0L;\n        try (WriteBatch writeBatch = new WriteBatch()) {\n            for (TransRocksDBRecord record : recordList) {\n                if (null == record) {\n                    logError.error(\"MessageRocksDBStorage writeRecordsForTrans error, record is null\");\n                    continue;\n                }\n                byte[] keyBytes = record.getKeyBytes();\n                if (null == keyBytes || keyBytes.length == 0) {\n                    logError.error(\"MessageRocksDBStorage writeRecordsForTrans param error, keyBytes: {}\", keyBytes);\n                    continue;\n                }\n                if (record.isOp()) {\n                    writeBatch.delete(cfHandle, record.getKeyBytes());\n                } else {\n                    byte[] valueBytes = record.getValueBytes();\n                    if (null == valueBytes || valueBytes.length == 0) {\n                        logError.error(\"MessageRocksDBStorage writeRecordsForTrans param error, valueBytes: {}\", valueBytes);\n                        continue;\n                    }\n                    writeBatch.put(cfHandle, keyBytes, valueBytes);\n                    lastOffsetPy = Math.max(lastOffsetPy, record.getOffsetPy());\n                }\n            }\n            if (lastOffsetPy > 0L) {\n                Long lastOffsetPyStore = getLastOffsetPy(columnFamily);\n                if (null == lastOffsetPyStore || lastOffsetPy > lastOffsetPyStore) {\n                    writeBatch.put(cfHandle, LAST_OFFSET_PY, ByteBuffer.allocate(Long.BYTES).putLong(lastOffsetPy).array());\n                }\n            }\n            batchPut(ableWalWriteOptions, writeBatch);\n        } catch (Exception e) {\n            logError.error(\"MessageRocksDBStorage writeRecordsForTrans error: {}\", e.getMessage());\n        }\n    }\n\n    public void updateRecordsForTrans(byte[] columnFamily, List<TransRocksDBRecord> recordList) {\n        ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily);\n        if (null == cfHandle || CollectionUtils.isEmpty(recordList)) {\n            return;\n        }\n        try (WriteBatch writeBatch = new WriteBatch()) {\n            for (TransRocksDBRecord record : recordList) {\n                if (null == record) {\n                    logError.error(\"MessageRocksDBStorage updateRecordsForTrans error, record is null\");\n                    continue;\n                }\n                byte[] keyBytes = record.getKeyBytes();\n                byte[] valueBytes = record.getValueBytes();\n                if (null == keyBytes || keyBytes.length == 0 || null == valueBytes || valueBytes.length == 0) {\n                    logError.error(\"MessageRocksDBStorage updateRecordsForTrans param error, keyBytes: {}, valueBytes: {}\", keyBytes, valueBytes);\n                    continue;\n                }\n                if (record.isDelete()) {\n                    writeBatch.delete(cfHandle, keyBytes);\n                } else {\n                    writeBatch.put(cfHandle, keyBytes, valueBytes);\n                }\n            }\n            batchPut(ableWalWriteOptions, writeBatch);\n        } catch (Exception e) {\n            logError.error(\"MessageRocksDBStorage updateRecordsForTrans error: {}\", e.getMessage());\n        }\n    }\n\n    public List<TransRocksDBRecord> scanRecordsForTrans(byte[] columnFamily, int size, byte[] startKey) {\n        ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily);\n        if (null == cfHandle || size <= 0) {\n            return null;\n        }\n        RocksIterator iterator = null;\n        try {\n            iterator = db.newIterator(cfHandle);\n            if (null == startKey || startKey.length == 0) {\n                iterator.seekToFirst();\n            } else {\n                iterator.seek(startKey);\n                iterator.next();\n            }\n            List<TransRocksDBRecord> records = new ArrayList<>();\n            for (; iterator.isValid(); iterator.next()) {\n                byte[] key = iterator.key();\n                if (null == key || key.length == 0 || key.length == LAST_OFFSET_PY_LENGTH && Arrays.equals(key, LAST_OFFSET_PY)) {\n                    continue;\n                }\n                TransRocksDBRecord transRocksDBRecord = null;\n                try {\n                    transRocksDBRecord = TransRocksDBRecord.decode(key, iterator.value());\n                } catch (Exception e) {\n                    logError.error(\"MessageRocksDBStorage scanRecordsForTrans error: {}\", e.getMessage());\n                }\n                if (null != transRocksDBRecord) {\n                    records.add(transRocksDBRecord);\n                }\n                if (records.size() >= size) {\n                    break;\n                }\n            }\n            return records;\n        } catch (Exception e) {\n            logError.error(\"MessageRocksDBStorage scanRecordsForTrans error: {}\", e.getMessage());\n        } finally {\n            if (null != iterator) {\n                iterator.close();\n            }\n        }\n        return null;\n    }\n\n    public TransRocksDBRecord getRecordForTrans(byte[] columnFamily, TransRocksDBRecord transRocksDBRecord) {\n        ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily);\n        if (null == cfHandle || null == transRocksDBRecord) {\n            return null;\n        }\n        try {\n            byte[] keyBytes = transRocksDBRecord.getKeyBytes();\n            if (null == keyBytes) {\n                return null;\n            }\n            byte[] valueBytes = get(cfHandle, readOptions, keyBytes);\n            if (null == valueBytes || valueBytes.length != TransRocksDBRecord.VALUE_LENGTH) {\n                return null;\n            }\n            return TransRocksDBRecord.decode(keyBytes, valueBytes);\n        } catch (Exception e) {\n            logError.error(\"MessageRocksDBStorage getRecordForTrans error: {}\", e.getMessage());\n            return null;\n        }\n    }\n\n    public Long getLastOffsetPy(byte[] columnFamily) {\n        ColumnFamilyHandle cfHandle = getColumnFamily(columnFamily);\n        if (null == cfHandle) {\n            return null;\n        }\n        try {\n            byte[] offsetBytes = get(cfHandle, readOptions, LAST_OFFSET_PY);\n            return offsetBytes == null ? 0L : ByteBuffer.wrap(offsetBytes).getLong();\n        } catch (Exception e) {\n            logError.error(\"MessageRocksDBStorage getLastOffsetPy error: {}\", e.getMessage());\n            return null;\n        }\n    }\n\n    @Override\n    public synchronized boolean shutdown() {\n        try {\n            boolean result = super.shutdown();\n            log.info(\"shutdown MessageRocksDBStorage result: {}\", result);\n            return result;\n        } catch (Exception e) {\n            logError.error(\"shutdown MessageRocksDBStorage error : {}\", e.getMessage());\n            return false;\n        }\n    }\n\n    private ColumnFamilyHandle getColumnFamily(byte[] columnFamily) {\n        if (Arrays.equals(columnFamily, RocksDB.DEFAULT_COLUMN_FAMILY)) {\n            return this.defaultCFHandle;\n        } else if (Arrays.equals(columnFamily, TIMER_COLUMN_FAMILY)) {\n            return this.timerCFHandle;\n        } else if (Arrays.equals(columnFamily, TRANS_COLUMN_FAMILY)) {\n            return this.transCFHandle;\n        }\n        throw new RuntimeException(\"Unknown column family\");\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.rocksdb;\n\nimport org.apache.rocketmq.common.config.ConfigHelper;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.rocksdb.BlockBasedTableConfig;\nimport org.rocksdb.BloomFilter;\nimport org.rocksdb.ColumnFamilyOptions;\nimport org.rocksdb.CompactionOptionsUniversal;\nimport org.rocksdb.CompactionPriority;\nimport org.rocksdb.CompactionStopStyle;\nimport org.rocksdb.CompactionStyle;\nimport org.rocksdb.CompressionType;\nimport org.rocksdb.DBOptions;\nimport org.rocksdb.DataBlockIndexType;\nimport org.rocksdb.IndexType;\nimport org.rocksdb.InfoLogLevel;\nimport org.rocksdb.LRUCache;\nimport org.rocksdb.RateLimiter;\nimport org.rocksdb.SkipListMemTableConfig;\nimport org.rocksdb.Statistics;\nimport org.rocksdb.StatsLevel;\nimport org.rocksdb.StringAppendOperator;\nimport org.rocksdb.WALRecoveryMode;\nimport org.rocksdb.util.SizeUnit;\n\npublic class RocksDBOptionsFactory {\n\n    public static ColumnFamilyOptions createCQCFOptions(final MessageStore messageStore,\n        ConsumeQueueCompactionFilterFactory consumeQueueCompactionFilterFactory) {\n        BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig().\n                setFormatVersion(5).\n                setIndexType(IndexType.kBinarySearch).\n                setDataBlockIndexType(DataBlockIndexType.kDataBlockBinaryAndHash).\n                setDataBlockHashTableUtilRatio(0.75).\n                setBlockSize(32 * SizeUnit.KB).\n                setMetadataBlockSize(4 * SizeUnit.KB).\n                setFilterPolicy(new BloomFilter(16, false)).\n                setCacheIndexAndFilterBlocks(false).\n                setCacheIndexAndFilterBlocksWithHighPriority(true).\n                setPinL0FilterAndIndexBlocksInCache(false).\n                setPinTopLevelIndexAndFilter(true).\n                setBlockCache(new LRUCache(1024 * SizeUnit.MB, 8, false)).\n                setWholeKeyFiltering(true);\n\n        ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions();\n        CompactionOptionsUniversal compactionOption = new CompactionOptionsUniversal();\n        compactionOption.setSizeRatio(100).\n                setMaxSizeAmplificationPercent(25).\n                setAllowTrivialMove(true).\n                setMinMergeWidth(2).\n                setMaxMergeWidth(Integer.MAX_VALUE).\n                setStopStyle(CompactionStopStyle.CompactionStopStyleTotalSize).\n                setCompressionSizePercent(-1);\n        String bottomMostCompressionTypeOpt = messageStore.getMessageStoreConfig()\n            .getBottomMostCompressionTypeForConsumeQueueStore();\n        String compressionTypeOpt = messageStore.getMessageStoreConfig()\n            .getRocksdbCompressionType();\n        CompressionType bottomMostCompressionType = CompressionType.getCompressionType(bottomMostCompressionTypeOpt);\n        CompressionType compressionType = CompressionType.getCompressionType(compressionTypeOpt);\n        return columnFamilyOptions.setMaxWriteBufferNumber(4).\n                setWriteBufferSize(128 * SizeUnit.MB).\n                setMinWriteBufferNumberToMerge(1).\n                setTableFormatConfig(blockBasedTableConfig).\n                setMemTableConfig(new SkipListMemTableConfig()).\n                setCompressionType(compressionType).\n                setBottommostCompressionType(bottomMostCompressionType).\n                setNumLevels(7).\n                setCompactionPriority(CompactionPriority.MinOverlappingRatio).\n                setCompactionStyle(CompactionStyle.UNIVERSAL).\n                setCompactionOptionsUniversal(compactionOption).\n                setMaxCompactionBytes(100 * SizeUnit.GB).\n                setSoftPendingCompactionBytesLimit(100 * SizeUnit.GB).\n                setHardPendingCompactionBytesLimit(256 * SizeUnit.GB).\n                setLevel0FileNumCompactionTrigger(2).\n                setLevel0SlowdownWritesTrigger(8).\n                setLevel0StopWritesTrigger(10).\n                setTargetFileSizeBase(256 * SizeUnit.MB).\n                setTargetFileSizeMultiplier(2).\n                setMergeOperator(new StringAppendOperator()).\n                setCompactionFilterFactory(consumeQueueCompactionFilterFactory).\n                setReportBgIoStats(true).\n                setOptimizeFiltersForHits(true);\n    }\n\n    public static ColumnFamilyOptions createOffsetCFOptions() {\n        BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig().\n                setFormatVersion(5).\n                setIndexType(IndexType.kBinarySearch).\n                setDataBlockIndexType(DataBlockIndexType.kDataBlockBinarySearch).\n                setBlockSize(32 * SizeUnit.KB).\n                setFilterPolicy(new BloomFilter(16, false)).\n                setCacheIndexAndFilterBlocks(false).\n                setCacheIndexAndFilterBlocksWithHighPriority(true).\n                setPinL0FilterAndIndexBlocksInCache(false).\n                setPinTopLevelIndexAndFilter(true).\n                setBlockCache(new LRUCache(128 * SizeUnit.MB, 8, false)).\n                setWholeKeyFiltering(true);\n\n        ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions();\n        return columnFamilyOptions.setMaxWriteBufferNumber(4).\n                setWriteBufferSize(64 * SizeUnit.MB).\n                setMinWriteBufferNumberToMerge(1).\n                setTableFormatConfig(blockBasedTableConfig).\n                setMemTableConfig(new SkipListMemTableConfig()).\n                setCompressionType(CompressionType.NO_COMPRESSION).\n                setNumLevels(7).\n                setCompactionStyle(CompactionStyle.LEVEL).\n                setLevel0FileNumCompactionTrigger(2).\n                setLevel0SlowdownWritesTrigger(8).\n                setLevel0StopWritesTrigger(10).\n                setTargetFileSizeBase(64 * SizeUnit.MB).\n                setTargetFileSizeMultiplier(2).\n                setMaxBytesForLevelBase(256 * SizeUnit.MB).\n                setMaxBytesForLevelMultiplier(2).\n                setMergeOperator(new StringAppendOperator()).\n                setInplaceUpdateSupport(true);\n    }\n\n    public static ColumnFamilyOptions createPopCFOptions() {\n        BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig()\n            .setFormatVersion(5)\n            .setIndexType(IndexType.kBinarySearch)\n            .setDataBlockIndexType(DataBlockIndexType.kDataBlockBinaryAndHash)\n            .setDataBlockHashTableUtilRatio(0.75)\n            .setBlockSize(32 * SizeUnit.KB)\n            .setMetadataBlockSize(4 * SizeUnit.KB)\n            .setFilterPolicy(new BloomFilter(16, false))\n            .setCacheIndexAndFilterBlocks(false)\n            .setCacheIndexAndFilterBlocksWithHighPriority(true)\n            .setPinL0FilterAndIndexBlocksInCache(false)\n            .setPinTopLevelIndexAndFilter(true)\n            .setBlockCache(new LRUCache(1024 * SizeUnit.MB, 8, false))\n            .setWholeKeyFiltering(true);\n\n        CompactionOptionsUniversal compactionOption = new CompactionOptionsUniversal()\n            .setSizeRatio(100)\n            .setMaxSizeAmplificationPercent(25)\n            .setAllowTrivialMove(true)\n            .setMinMergeWidth(2)\n            .setMaxMergeWidth(Integer.MAX_VALUE)\n            .setStopStyle(CompactionStopStyle.CompactionStopStyleTotalSize)\n            .setCompressionSizePercent(-1);\n\n        //noinspection resource\n        return new ColumnFamilyOptions()\n            .setMaxWriteBufferNumber(4)\n            .setWriteBufferSize(128 * SizeUnit.MB)\n            .setMinWriteBufferNumberToMerge(1)\n            .setTableFormatConfig(blockBasedTableConfig)\n            .setMemTableConfig(new SkipListMemTableConfig())\n            .setCompressionType(CompressionType.NO_COMPRESSION)\n            .setBottommostCompressionType(CompressionType.NO_COMPRESSION)\n            .setNumLevels(7)\n            .setCompactionPriority(CompactionPriority.MinOverlappingRatio)\n            .setCompactionStyle(CompactionStyle.UNIVERSAL)\n            .setCompactionOptionsUniversal(compactionOption)\n            .setMaxCompactionBytes(100 * SizeUnit.GB)\n            .setSoftPendingCompactionBytesLimit(100 * SizeUnit.GB)\n            .setHardPendingCompactionBytesLimit(256 * SizeUnit.GB)\n            .setLevel0FileNumCompactionTrigger(2)\n            .setLevel0SlowdownWritesTrigger(8)\n            .setLevel0StopWritesTrigger(10)\n            .setTargetFileSizeBase(256 * SizeUnit.MB)\n            .setTargetFileSizeMultiplier(2)\n            .setMergeOperator(new StringAppendOperator())\n            .setReportBgIoStats(true)\n            .setOptimizeFiltersForHits(true);\n    }\n\n    /**\n     * Create a rocksdb db options, the user must take care to close it after closing db.\n     * @return\n     */\n    public static DBOptions createDBOptions() {\n        //Turn based on https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide\n        // and http://gitlab.alibaba-inc.com/aloha/aloha/blob/branch_2_5_0/jstorm-core/src/main/java/com/alibaba/jstorm/cache/rocksdb/RocksDbOptionsFactory.java\n        DBOptions options = new DBOptions();\n        Statistics statistics = new Statistics();\n        statistics.setStatsLevel(StatsLevel.EXCEPT_DETAILED_TIMERS);\n        return options.\n                setDbLogDir(ConfigHelper.getDBLogDir()).\n                setInfoLogLevel(InfoLogLevel.INFO_LEVEL).\n                setWalRecoveryMode(WALRecoveryMode.PointInTimeRecovery).\n                setManualWalFlush(true).\n                setCreateIfMissing(true).\n                setBytesPerSync(SizeUnit.MB).\n                setCreateMissingColumnFamilies(true).\n                setMaxOpenFiles(-1).\n                setMaxLogFileSize(SizeUnit.GB).\n                setKeepLogFileNum(5).\n                setMaxManifestFileSize(SizeUnit.GB).\n                setAllowConcurrentMemtableWrite(false).\n                setStatistics(statistics).\n                setAtomicFlush(true).\n                setCompactionReadaheadSize(4 * SizeUnit.MB).\n                setMaxBackgroundJobs(32).\n                setMaxSubcompactions(8).\n                setParanoidChecks(true).\n                setDelayedWriteRate(16 * SizeUnit.MB).\n                setRateLimiter(new RateLimiter(100 * SizeUnit.MB)).\n                setUseDirectIoForFlushAndCompaction(false).\n                setUseDirectReads(false);\n    }\n\n    public static ColumnFamilyOptions createTimerCFOptions() {\n        BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig()\n            .setFormatVersion(5)\n            .setIndexType(IndexType.kBinarySearch)\n            .setDataBlockIndexType(DataBlockIndexType.kDataBlockBinaryAndHash)\n            .setDataBlockHashTableUtilRatio(0.75)\n            .setBlockSize(128 * SizeUnit.KB)\n            .setMetadataBlockSize(4 * SizeUnit.KB)\n            .setFilterPolicy(new BloomFilter(16, false))\n            .setCacheIndexAndFilterBlocks(false)\n            .setCacheIndexAndFilterBlocksWithHighPriority(true)\n            .setPinL0FilterAndIndexBlocksInCache(false)\n            .setPinTopLevelIndexAndFilter(true)\n            .setBlockCache(new LRUCache(2048 * SizeUnit.MB, 8, false))\n            .setWholeKeyFiltering(true);\n\n        //noinspection resource\n        return new ColumnFamilyOptions()\n            .setMaxWriteBufferNumber(6)\n            .setWriteBufferSize(256 * SizeUnit.MB)\n            .setMinWriteBufferNumberToMerge(1)\n            .setTableFormatConfig(blockBasedTableConfig)\n            .setMemTableConfig(new SkipListMemTableConfig())\n            .setCompressionType(CompressionType.ZSTD_COMPRESSION)\n            .setBottommostCompressionType(CompressionType.NO_COMPRESSION)\n            .setNumLevels(7)\n            .setCompactionPriority(CompactionPriority.MinOverlappingRatio)\n            .setCompactionStyle(CompactionStyle.LEVEL)\n            .setMaxCompactionBytes(256 * SizeUnit.MB)\n            .setSoftPendingCompactionBytesLimit(100 * SizeUnit.GB)\n            .setHardPendingCompactionBytesLimit(256 * SizeUnit.GB)\n            .setLevel0FileNumCompactionTrigger(2)\n            .setLevel0SlowdownWritesTrigger(8)\n            .setLevel0StopWritesTrigger(10)\n            .setTargetFileSizeBase(256 * SizeUnit.MB)\n            .setTargetFileSizeMultiplier(2)\n            .setMergeOperator(new StringAppendOperator())\n            .setReportBgIoStats(true)\n            .setOptimizeFiltersForHits(true)\n            .setMaxBytesForLevelBase(512 * SizeUnit.MB);\n    }\n\n    public static ColumnFamilyOptions createTransCFOptions() {\n        BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig()\n            .setFormatVersion(5)\n            .setIndexType(IndexType.kBinarySearch)\n            .setDataBlockIndexType(DataBlockIndexType.kDataBlockBinaryAndHash)\n            .setDataBlockHashTableUtilRatio(0.75)\n            .setBlockSize(128 * SizeUnit.KB)\n            .setMetadataBlockSize(4 * SizeUnit.KB)\n            .setFilterPolicy(new BloomFilter(16, false))\n            .setCacheIndexAndFilterBlocks(false)\n            .setCacheIndexAndFilterBlocksWithHighPriority(true)\n            .setPinL0FilterAndIndexBlocksInCache(false)\n            .setPinTopLevelIndexAndFilter(true)\n            .setBlockCache(new LRUCache(1024 * SizeUnit.MB, 8, false))\n            .setWholeKeyFiltering(true);\n\n        CompactionOptionsUniversal compactionOption = new CompactionOptionsUniversal()\n            .setSizeRatio(100)\n            .setMaxSizeAmplificationPercent(25)\n            .setAllowTrivialMove(true)\n            .setMinMergeWidth(2)\n            .setMaxMergeWidth(Integer.MAX_VALUE)\n            .setStopStyle(CompactionStopStyle.CompactionStopStyleTotalSize)\n            .setCompressionSizePercent(-1);\n\n        //noinspection resource\n        return new ColumnFamilyOptions()\n            .setMaxWriteBufferNumber(6)\n            .setWriteBufferSize(128 * SizeUnit.MB)\n            .setMinWriteBufferNumberToMerge(1)\n            .setTableFormatConfig(blockBasedTableConfig)\n            .setMemTableConfig(new SkipListMemTableConfig())\n            .setCompressionType(CompressionType.NO_COMPRESSION)\n            .setBottommostCompressionType(CompressionType.NO_COMPRESSION)\n            .setNumLevels(7)\n            .setCompactionPriority(CompactionPriority.MinOverlappingRatio)\n            .setCompactionStyle(CompactionStyle.UNIVERSAL)\n            .setCompactionOptionsUniversal(compactionOption)\n            .setMaxCompactionBytes(100 * SizeUnit.GB)\n            .setSoftPendingCompactionBytesLimit(100 * SizeUnit.GB)\n            .setHardPendingCompactionBytesLimit(256 * SizeUnit.GB)\n            .setLevel0FileNumCompactionTrigger(2)\n            .setLevel0SlowdownWritesTrigger(8)\n            .setLevel0StopWritesTrigger(10)\n            .setTargetFileSizeBase(256 * SizeUnit.MB)\n            .setTargetFileSizeMultiplier(2)\n            .setMergeOperator(new StringAppendOperator())\n            .setReportBgIoStats(true)\n            .setOptimizeFiltersForHits(true);\n    }\n\n    public static ColumnFamilyOptions createIndexCFOptions() {\n        BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig()\n            .setFormatVersion(5)\n            .setIndexType(IndexType.kBinarySearch)\n            .setDataBlockIndexType(DataBlockIndexType.kDataBlockBinaryAndHash)\n            .setDataBlockHashTableUtilRatio(0.75)\n            .setBlockSize(128 * SizeUnit.KB)\n            .setMetadataBlockSize(4 * SizeUnit.KB)\n            .setFilterPolicy(new BloomFilter(16, false))\n            .setCacheIndexAndFilterBlocks(false)\n            .setCacheIndexAndFilterBlocksWithHighPriority(true)\n            .setPinL0FilterAndIndexBlocksInCache(false)\n            .setPinTopLevelIndexAndFilter(true)\n            .setBlockCache(new LRUCache(1024 * SizeUnit.MB, 8, false))\n            .setWholeKeyFiltering(true);\n\n        CompactionOptionsUniversal compactionOption = new CompactionOptionsUniversal()\n            .setSizeRatio(100)\n            .setMaxSizeAmplificationPercent(25)\n            .setAllowTrivialMove(true)\n            .setMinMergeWidth(2)\n            .setMaxMergeWidth(Integer.MAX_VALUE)\n            .setStopStyle(CompactionStopStyle.CompactionStopStyleTotalSize)\n            .setCompressionSizePercent(-1);\n\n        //noinspection resource\n        return new ColumnFamilyOptions()\n            .setMaxWriteBufferNumber(6)\n            .setWriteBufferSize(128 * SizeUnit.MB)\n            .setMinWriteBufferNumberToMerge(1)\n            .setTableFormatConfig(blockBasedTableConfig)\n            .setMemTableConfig(new SkipListMemTableConfig())\n            .setCompressionType(CompressionType.NO_COMPRESSION)\n            .setBottommostCompressionType(CompressionType.NO_COMPRESSION)\n            .setNumLevels(7)\n            .setCompactionPriority(CompactionPriority.MinOverlappingRatio)\n            .setCompactionStyle(CompactionStyle.UNIVERSAL)\n            .setCompactionOptionsUniversal(compactionOption)\n            .setMaxCompactionBytes(256 * SizeUnit.MB)\n            .setSoftPendingCompactionBytesLimit(100 * SizeUnit.GB)\n            .setHardPendingCompactionBytesLimit(256 * SizeUnit.GB)\n            .setLevel0FileNumCompactionTrigger(8)\n            .setLevel0SlowdownWritesTrigger(8)\n            .setLevel0StopWritesTrigger(20)\n            .setTargetFileSizeBase(256 * SizeUnit.MB)\n            .setTargetFileSizeMultiplier(2)\n            .setMergeOperator(new StringAppendOperator())\n            .setReportBgIoStats(true)\n            .setOptimizeFiltersForHits(true);\n    }\n\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/stats/BrokerStats.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.stats;\n\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.MessageStore;\n\npublic class BrokerStats {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n\n    private final MessageStore defaultMessageStore;\n\n    private volatile long msgPutTotalYesterdayMorning;\n\n    private volatile long msgPutTotalTodayMorning;\n\n    private volatile long msgGetTotalYesterdayMorning;\n\n    private volatile long msgGetTotalTodayMorning;\n\n    public BrokerStats(MessageStore defaultMessageStore) {\n        this.defaultMessageStore = defaultMessageStore;\n    }\n\n    public void record() {\n        this.msgPutTotalYesterdayMorning = this.msgPutTotalTodayMorning;\n        this.msgGetTotalYesterdayMorning = this.msgGetTotalTodayMorning;\n\n        this.msgPutTotalTodayMorning =\n            this.defaultMessageStore.getBrokerStatsManager().getBrokerPutNumsWithoutSystemTopic();\n        this.msgGetTotalTodayMorning =\n            this.defaultMessageStore.getBrokerStatsManager().getBrokerGetNumsWithoutSystemTopic();\n\n        log.info(\"yesterday put message total: {}\", msgPutTotalTodayMorning - msgPutTotalYesterdayMorning);\n        log.info(\"yesterday get message total: {}\", msgGetTotalTodayMorning - msgGetTotalYesterdayMorning);\n    }\n\n    public long getMsgPutTotalYesterdayMorning() {\n        return msgPutTotalYesterdayMorning;\n    }\n\n    public void setMsgPutTotalYesterdayMorning(long msgPutTotalYesterdayMorning) {\n        this.msgPutTotalYesterdayMorning = msgPutTotalYesterdayMorning;\n    }\n\n    public long getMsgPutTotalTodayMorning() {\n        return msgPutTotalTodayMorning;\n    }\n\n    public void setMsgPutTotalTodayMorning(long msgPutTotalTodayMorning) {\n        this.msgPutTotalTodayMorning = msgPutTotalTodayMorning;\n    }\n\n    public long getMsgGetTotalYesterdayMorning() {\n        return msgGetTotalYesterdayMorning;\n    }\n\n    public void setMsgGetTotalYesterdayMorning(long msgGetTotalYesterdayMorning) {\n        this.msgGetTotalYesterdayMorning = msgGetTotalYesterdayMorning;\n    }\n\n    public long getMsgGetTotalTodayMorning() {\n        return msgGetTotalTodayMorning;\n    }\n\n    public void setMsgGetTotalTodayMorning(long msgGetTotalTodayMorning) {\n        this.msgGetTotalTodayMorning = msgGetTotalTodayMorning;\n    }\n\n    public long getMsgPutTotalTodayNow() {\n        return this.defaultMessageStore.getBrokerStatsManager().getBrokerPutNumsWithoutSystemTopic();\n    }\n\n    public long getMsgGetTotalTodayNow() {\n        return this.defaultMessageStore.getBrokerStatsManager().getBrokerGetNumsWithoutSystemTopic();\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.stats;\n\nimport java.util.HashMap;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\n\nimport org.apache.commons.lang3.tuple.Pair;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.statistics.StatisticsItem;\nimport org.apache.rocketmq.common.statistics.StatisticsItemFormatter;\nimport org.apache.rocketmq.common.statistics.StatisticsItemPrinter;\nimport org.apache.rocketmq.common.statistics.StatisticsItemScheduledIncrementPrinter;\nimport org.apache.rocketmq.common.statistics.StatisticsItemScheduledPrinter;\nimport org.apache.rocketmq.common.statistics.StatisticsItemStateGetter;\nimport org.apache.rocketmq.common.statistics.StatisticsKindMeta;\nimport org.apache.rocketmq.common.statistics.StatisticsManager;\nimport org.apache.rocketmq.common.stats.MomentStatsItemSet;\nimport org.apache.rocketmq.common.stats.Stats;\nimport org.apache.rocketmq.common.stats.StatsItem;\nimport org.apache.rocketmq.common.stats.StatsItemSet;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class BrokerStatsManager {\n\n    @Deprecated public static final String QUEUE_PUT_NUMS = Stats.QUEUE_PUT_NUMS;\n    @Deprecated public static final String QUEUE_PUT_SIZE = Stats.QUEUE_PUT_SIZE;\n    @Deprecated public static final String QUEUE_GET_NUMS = Stats.QUEUE_GET_NUMS;\n    @Deprecated public static final String QUEUE_GET_SIZE = Stats.QUEUE_GET_SIZE;\n    @Deprecated public static final String TOPIC_PUT_NUMS = Stats.TOPIC_PUT_NUMS;\n    @Deprecated public static final String TOPIC_PUT_SIZE = Stats.TOPIC_PUT_SIZE;\n\n    @Deprecated public static final String GROUP_GET_NUMS = Stats.GROUP_GET_NUMS;\n    @Deprecated public static final String GROUP_GET_SIZE = Stats.GROUP_GET_SIZE;\n\n    @Deprecated public static final String SNDBCK_PUT_NUMS = Stats.SNDBCK_PUT_NUMS;\n    @Deprecated public static final String BROKER_PUT_NUMS = Stats.BROKER_PUT_NUMS;\n    @Deprecated public static final String BROKER_GET_NUMS = Stats.BROKER_GET_NUMS;\n    @Deprecated public static final String GROUP_GET_FROM_DISK_NUMS = Stats.GROUP_GET_FROM_DISK_NUMS;\n    @Deprecated public static final String GROUP_GET_FROM_DISK_SIZE = Stats.GROUP_GET_FROM_DISK_SIZE;\n    @Deprecated public static final String BROKER_GET_FROM_DISK_NUMS = Stats.BROKER_GET_FROM_DISK_NUMS;\n    @Deprecated public static final String BROKER_GET_FROM_DISK_SIZE = Stats.BROKER_GET_FROM_DISK_SIZE;\n    // For commercial\n    @Deprecated public static final String COMMERCIAL_SEND_TIMES = Stats.COMMERCIAL_SEND_TIMES;\n    @Deprecated public static final String COMMERCIAL_SNDBCK_TIMES = Stats.COMMERCIAL_SNDBCK_TIMES;\n    @Deprecated public static final String COMMERCIAL_RCV_TIMES = Stats.COMMERCIAL_RCV_TIMES;\n    @Deprecated public static final String COMMERCIAL_RCV_EPOLLS = Stats.COMMERCIAL_RCV_EPOLLS;\n    @Deprecated public static final String COMMERCIAL_SEND_SIZE = Stats.COMMERCIAL_SEND_SIZE;\n    @Deprecated public static final String COMMERCIAL_RCV_SIZE = Stats.COMMERCIAL_RCV_SIZE;\n    @Deprecated public static final String COMMERCIAL_PERM_FAILURES = Stats.COMMERCIAL_PERM_FAILURES;\n\n    // Send message latency\n    @Deprecated public static final String TOPIC_PUT_LATENCY = \"TOPIC_PUT_LATENCY\";\n    @Deprecated public static final String GROUP_ACK_NUMS = \"GROUP_ACK_NUMS\";\n    @Deprecated public static final String GROUP_CK_NUMS = \"GROUP_CK_NUMS\";\n    public static final String DLQ_PUT_NUMS = \"DLQ_PUT_NUMS\";\n    public static final String BROKER_ACK_NUMS = \"BROKER_ACK_NUMS\";\n    public static final String BROKER_CK_NUMS = \"BROKER_CK_NUMS\";\n    public static final String BROKER_GET_NUMS_WITHOUT_SYSTEM_TOPIC = \"BROKER_GET_NUMS_WITHOUT_SYSTEM_TOPIC\";\n    public static final String BROKER_PUT_NUMS_WITHOUT_SYSTEM_TOPIC = \"BROKER_PUT_NUMS_WITHOUT_SYSTEM_TOPIC\";\n    public static final String SNDBCK2DLQ_TIMES = \"SNDBCK2DLQ_TIMES\";\n\n    public static final String COMMERCIAL_OWNER = \"Owner\";\n\n    public static final String ACCOUNT_OWNER_PARENT = \"OWNER_PARENT\";\n    public static final String ACCOUNT_OWNER_SELF = \"OWNER_SELF\";\n\n    public static final long ACCOUNT_STAT_INVERTAL = 60 * 1000;\n    public static final String ACCOUNT_AUTH_TYPE = \"AUTH_TYPE\";\n\n    public static final String ACCOUNT_SEND = \"SEND\";\n    public static final String ACCOUNT_RCV = \"RCV\";\n    public static final String ACCOUNT_SEND_BACK = \"SEND_BACK\";\n    public static final String ACCOUNT_SEND_BACK_TO_DLQ = \"SEND_BACK_TO_DLQ\";\n    public static final String ACCOUNT_AUTH_FAILED = \"AUTH_FAILED\";\n    public static final String ACCOUNT_SEND_REJ = \"SEND_REJ\";\n    public static final String ACCOUNT_REV_REJ = \"RCV_REJ\";\n\n    public static final String MSG_NUM = \"MSG_NUM\";\n    public static final String MSG_SIZE = \"MSG_SIZE\";\n    public static final String SUCCESS_MSG_NUM = \"SUCCESS_MSG_NUM\";\n    public static final String FAILURE_MSG_NUM = \"FAILURE_MSG_NUM\";\n    public static final String COMMERCIAL_MSG_NUM = \"COMMERCIAL_MSG_NUM\";\n    public static final String SUCCESS_REQ_NUM = \"SUCCESS_REQ_NUM\";\n    public static final String FAILURE_REQ_NUM = \"FAILURE_REQ_NUM\";\n    public static final String SUCCESS_MSG_SIZE = \"SUCCESS_MSG_SIZE\";\n    public static final String FAILURE_MSG_SIZE = \"FAILURE_MSG_SIZE\";\n    public static final String RT = \"RT\";\n    public static final String INNER_RT = \"INNER_RT\";\n\n    @Deprecated public static final String GROUP_GET_FALL_SIZE = Stats.GROUP_GET_FALL_SIZE;\n    @Deprecated public static final String GROUP_GET_FALL_TIME = Stats.GROUP_GET_FALL_TIME;\n    // Pull Message Latency\n    @Deprecated public static final String GROUP_GET_LATENCY = Stats.GROUP_GET_LATENCY;\n\n    // Consumer Register Time\n    public static final String CONSUMER_REGISTER_TIME = \"CONSUMER_REGISTER_TIME\";\n    // Producer Register Time\n    public static final String PRODUCER_REGISTER_TIME = \"PRODUCER_REGISTER_TIME\";\n    public static final String CHANNEL_ACTIVITY = \"CHANNEL_ACTIVITY\";\n    public static final String CHANNEL_ACTIVITY_CONNECT = \"CONNECT\";\n    public static final String CHANNEL_ACTIVITY_IDLE = \"IDLE\";\n    public static final String CHANNEL_ACTIVITY_EXCEPTION = \"EXCEPTION\";\n    public static final String CHANNEL_ACTIVITY_CLOSE = \"CLOSE\";\n    private static final String[] NEED_CLEAN_STATS_SET =\n            new String[] {TOPIC_PUT_NUMS, TOPIC_PUT_SIZE, GROUP_GET_NUMS, GROUP_GET_SIZE, SNDBCK_PUT_NUMS, GROUP_GET_LATENCY};\n\n    /**\n     * read disk follow stats\n     */\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_STATS_LOGGER_NAME);\n    private static final Logger COMMERCIAL_LOG = LoggerFactory.getLogger(\n        LoggerName.COMMERCIAL_LOGGER_NAME);\n    private static final Logger ACCOUNT_LOG = LoggerFactory.getLogger(LoggerName.ACCOUNT_LOGGER_NAME);\n    private static final Logger DLQ_STAT_LOG = LoggerFactory.getLogger(\n        LoggerName.DLQ_STATS_LOGGER_NAME);\n    private ScheduledExecutorService scheduledExecutorService;\n    private ScheduledExecutorService commercialExecutor;\n    private ScheduledExecutorService accountExecutor;\n    private ScheduledExecutorService cleanResourceExecutor;\n\n    private final HashMap<String, StatsItemSet> statsTable = new HashMap<>();\n    private final String clusterName;\n    private final boolean enableQueueStat;\n    private MomentStatsItemSet momentStatsItemSetFallSize;\n    private MomentStatsItemSet momentStatsItemSetFallTime;\n\n    private final StatisticsManager accountStatManager = new StatisticsManager();\n    private StateGetter producerStateGetter;\n    private StateGetter consumerStateGetter;\n\n    private BrokerConfig brokerConfig;\n\n    public BrokerStatsManager(BrokerConfig brokerConfig) {\n        this.brokerConfig = brokerConfig;\n        this.enableQueueStat = brokerConfig.isEnableDetailStat();\n        initScheduleService();\n        this.clusterName = brokerConfig.getBrokerClusterName();\n        init();\n    }\n\n    public BrokerStatsManager(String clusterName, boolean enableQueueStat) {\n        this.clusterName = clusterName;\n        this.enableQueueStat = enableQueueStat;\n        initScheduleService();\n        init();\n    }\n\n    public void init() {\n        momentStatsItemSetFallSize = new MomentStatsItemSet(GROUP_GET_FALL_SIZE,\n            scheduledExecutorService, log);\n\n        momentStatsItemSetFallTime = new MomentStatsItemSet(GROUP_GET_FALL_TIME,\n            scheduledExecutorService, log);\n\n        if (enableQueueStat) {\n            this.statsTable.put(Stats.QUEUE_PUT_NUMS, new StatsItemSet(Stats.QUEUE_PUT_NUMS, this.scheduledExecutorService, log));\n            this.statsTable.put(Stats.QUEUE_PUT_SIZE, new StatsItemSet(Stats.QUEUE_PUT_SIZE, this.scheduledExecutorService, log));\n            this.statsTable.put(Stats.QUEUE_GET_NUMS, new StatsItemSet(Stats.QUEUE_GET_NUMS, this.scheduledExecutorService, log));\n            this.statsTable.put(Stats.QUEUE_GET_SIZE, new StatsItemSet(Stats.QUEUE_GET_SIZE, this.scheduledExecutorService, log));\n        }\n        this.statsTable.put(Stats.TOPIC_PUT_NUMS, new StatsItemSet(Stats.TOPIC_PUT_NUMS, this.scheduledExecutorService, log));\n        this.statsTable.put(Stats.TOPIC_PUT_SIZE, new StatsItemSet(Stats.TOPIC_PUT_SIZE, this.scheduledExecutorService, log));\n        this.statsTable.put(Stats.GROUP_GET_NUMS, new StatsItemSet(Stats.GROUP_GET_NUMS, this.scheduledExecutorService, log));\n        this.statsTable.put(Stats.GROUP_GET_SIZE, new StatsItemSet(Stats.GROUP_GET_SIZE, this.scheduledExecutorService, log));\n        this.statsTable.put(Stats.GROUP_ACK_NUMS, new StatsItemSet(Stats.GROUP_ACK_NUMS, this.scheduledExecutorService, log));\n        this.statsTable.put(Stats.GROUP_CK_NUMS, new StatsItemSet(Stats.GROUP_CK_NUMS, this.scheduledExecutorService, log));\n        this.statsTable.put(Stats.GROUP_GET_LATENCY, new StatsItemSet(Stats.GROUP_GET_LATENCY, this.scheduledExecutorService, log));\n        this.statsTable.put(Stats.TOPIC_PUT_LATENCY, new StatsItemSet(Stats.TOPIC_PUT_LATENCY, this.scheduledExecutorService, log));\n        this.statsTable.put(Stats.SNDBCK_PUT_NUMS, new StatsItemSet(Stats.SNDBCK_PUT_NUMS, this.scheduledExecutorService, log));\n        this.statsTable.put(DLQ_PUT_NUMS, new StatsItemSet(DLQ_PUT_NUMS, this.scheduledExecutorService, log));\n        this.statsTable.put(Stats.BROKER_PUT_NUMS, new StatsItemSet(Stats.BROKER_PUT_NUMS, this.scheduledExecutorService, log));\n        this.statsTable.put(Stats.BROKER_GET_NUMS, new StatsItemSet(Stats.BROKER_GET_NUMS, this.scheduledExecutorService, log));\n        this.statsTable.put(BROKER_ACK_NUMS, new StatsItemSet(BROKER_ACK_NUMS, this.scheduledExecutorService, log));\n        this.statsTable.put(BROKER_CK_NUMS, new StatsItemSet(BROKER_CK_NUMS, this.scheduledExecutorService, log));\n        this.statsTable.put(BROKER_GET_NUMS_WITHOUT_SYSTEM_TOPIC,\n            new StatsItemSet(BROKER_GET_NUMS_WITHOUT_SYSTEM_TOPIC, this.scheduledExecutorService, log));\n        this.statsTable.put(BROKER_PUT_NUMS_WITHOUT_SYSTEM_TOPIC,\n            new StatsItemSet(BROKER_PUT_NUMS_WITHOUT_SYSTEM_TOPIC, this.scheduledExecutorService, log));\n        this.statsTable.put(Stats.GROUP_GET_FROM_DISK_NUMS,\n            new StatsItemSet(Stats.GROUP_GET_FROM_DISK_NUMS, this.scheduledExecutorService, log));\n        this.statsTable.put(Stats.GROUP_GET_FROM_DISK_SIZE,\n            new StatsItemSet(Stats.GROUP_GET_FROM_DISK_SIZE, this.scheduledExecutorService, log));\n        this.statsTable.put(Stats.BROKER_GET_FROM_DISK_NUMS,\n            new StatsItemSet(Stats.BROKER_GET_FROM_DISK_NUMS, this.scheduledExecutorService, log));\n        this.statsTable.put(Stats.BROKER_GET_FROM_DISK_SIZE,\n            new StatsItemSet(Stats.BROKER_GET_FROM_DISK_SIZE, this.scheduledExecutorService, log));\n\n        this.statsTable.put(SNDBCK2DLQ_TIMES,\n            new StatsItemSet(SNDBCK2DLQ_TIMES, this.scheduledExecutorService, DLQ_STAT_LOG));\n\n        this.statsTable.put(Stats.COMMERCIAL_SEND_TIMES,\n            new StatsItemSet(Stats.COMMERCIAL_SEND_TIMES, this.commercialExecutor, COMMERCIAL_LOG));\n        this.statsTable.put(Stats.COMMERCIAL_RCV_TIMES,\n            new StatsItemSet(Stats.COMMERCIAL_RCV_TIMES, this.commercialExecutor, COMMERCIAL_LOG));\n        this.statsTable.put(Stats.COMMERCIAL_SEND_SIZE,\n            new StatsItemSet(Stats.COMMERCIAL_SEND_SIZE, this.commercialExecutor, COMMERCIAL_LOG));\n        this.statsTable.put(Stats.COMMERCIAL_RCV_SIZE,\n            new StatsItemSet(Stats.COMMERCIAL_RCV_SIZE, this.commercialExecutor, COMMERCIAL_LOG));\n        this.statsTable.put(Stats.COMMERCIAL_RCV_EPOLLS,\n            new StatsItemSet(Stats.COMMERCIAL_RCV_EPOLLS, this.commercialExecutor, COMMERCIAL_LOG));\n        this.statsTable.put(Stats.COMMERCIAL_SNDBCK_TIMES,\n            new StatsItemSet(Stats.COMMERCIAL_SNDBCK_TIMES, this.commercialExecutor, COMMERCIAL_LOG));\n        this.statsTable.put(Stats.COMMERCIAL_PERM_FAILURES,\n            new StatsItemSet(Stats.COMMERCIAL_PERM_FAILURES, this.commercialExecutor, COMMERCIAL_LOG));\n\n        this.statsTable.put(CONSUMER_REGISTER_TIME,\n            new StatsItemSet(CONSUMER_REGISTER_TIME, this.scheduledExecutorService, log));\n        this.statsTable.put(PRODUCER_REGISTER_TIME,\n            new StatsItemSet(PRODUCER_REGISTER_TIME, this.scheduledExecutorService, log));\n\n        this.statsTable.put(CHANNEL_ACTIVITY, new StatsItemSet(CHANNEL_ACTIVITY, this.scheduledExecutorService, log));\n\n        StatisticsItemFormatter formatter = new StatisticsItemFormatter();\n        accountStatManager.setBriefMeta(new Pair[] {\n            Pair.of(RT, new long[][] {{50, 50}, {100, 10}, {1000, 10}}),\n            Pair.of(INNER_RT, new long[][] {{10, 10}, {100, 10}, {1000, 10}})});\n        String[] itemNames = new String[] {\n            MSG_NUM, SUCCESS_MSG_NUM, FAILURE_MSG_NUM, COMMERCIAL_MSG_NUM,\n            SUCCESS_REQ_NUM, FAILURE_REQ_NUM,\n            MSG_SIZE, SUCCESS_MSG_SIZE, FAILURE_MSG_SIZE,\n            RT, INNER_RT};\n        this.accountStatManager.addStatisticsKindMeta(createStatisticsKindMeta(\n            ACCOUNT_SEND, itemNames, this.accountExecutor, formatter, ACCOUNT_LOG, ACCOUNT_STAT_INVERTAL));\n        this.accountStatManager.addStatisticsKindMeta(createStatisticsKindMeta(\n            ACCOUNT_RCV, itemNames, this.accountExecutor, formatter, ACCOUNT_LOG, ACCOUNT_STAT_INVERTAL));\n        this.accountStatManager.addStatisticsKindMeta(createStatisticsKindMeta(\n            ACCOUNT_SEND_BACK, itemNames, this.accountExecutor, formatter, ACCOUNT_LOG, ACCOUNT_STAT_INVERTAL));\n        this.accountStatManager.addStatisticsKindMeta(createStatisticsKindMeta(\n            ACCOUNT_SEND_BACK_TO_DLQ, itemNames, this.accountExecutor, formatter, ACCOUNT_LOG, ACCOUNT_STAT_INVERTAL));\n        this.accountStatManager.addStatisticsKindMeta(createStatisticsKindMeta(\n            ACCOUNT_SEND_REJ, itemNames, this.accountExecutor, formatter, ACCOUNT_LOG, ACCOUNT_STAT_INVERTAL));\n        this.accountStatManager.addStatisticsKindMeta(createStatisticsKindMeta(\n            ACCOUNT_REV_REJ, itemNames, this.accountExecutor, formatter, ACCOUNT_LOG, ACCOUNT_STAT_INVERTAL));\n        this.accountStatManager.setStatisticsItemStateGetter(new StatisticsItemStateGetter() {\n            @Override\n            public boolean online(StatisticsItem item) {\n                String[] strArr = null;\n                try {\n                    strArr = splitAccountStatKey(item.getStatObject());\n                } catch (Exception e) {\n                    log.warn(\"parse account stat key failed, key: {}\", item.getStatObject());\n                    return false;\n                }\n\n                // TODO ugly\n                if (strArr == null || strArr.length < 4) {\n                    return false;\n                }\n\n                String instanceId = strArr[1];\n                String topic = strArr[2];\n                String group = strArr[3];\n\n                String kind = item.getStatKind();\n                if (ACCOUNT_SEND.equals(kind) || ACCOUNT_SEND_REJ.equals(kind)) {\n                    return producerStateGetter.online(instanceId, group, topic);\n                } else if (ACCOUNT_RCV.equals(kind) || ACCOUNT_SEND_BACK.equals(kind) || ACCOUNT_SEND_BACK_TO_DLQ.equals(kind) || ACCOUNT_REV_REJ.equals(kind)) {\n                    return consumerStateGetter.online(instanceId, group, topic);\n                }\n                return false;\n            }\n        });\n        cleanResourceExecutor.scheduleWithFixedDelay(new Runnable() {\n            @Override\n            public void run() {\n                cleanAllResource();\n            }\n        }, 10, 10, TimeUnit.MINUTES);\n    }\n\n    private void initScheduleService() {\n        this.scheduledExecutorService =\n            ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(\"BrokerStatsThread\", true, brokerConfig));\n        this.commercialExecutor =\n            ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(\"CommercialStatsThread\", true, brokerConfig));\n        this.accountExecutor =\n            ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(\"AccountStatsThread\", true, brokerConfig));\n        this.cleanResourceExecutor =\n                ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(\"CleanStatsResourceThread\", true, brokerConfig));\n    }\n\n    public MomentStatsItemSet getMomentStatsItemSetFallSize() {\n        return momentStatsItemSetFallSize;\n    }\n\n    public MomentStatsItemSet getMomentStatsItemSetFallTime() {\n        return momentStatsItemSetFallTime;\n    }\n\n    public StateGetter getProducerStateGetter() {\n        return producerStateGetter;\n    }\n\n    public void setProducerStateGetter(StateGetter producerStateGetter) {\n        this.producerStateGetter = producerStateGetter;\n    }\n\n    public StateGetter getConsumerStateGetter() {\n        return consumerStateGetter;\n    }\n\n    public void setConsumerStateGetter(StateGetter consumerStateGetter) {\n        this.consumerStateGetter = consumerStateGetter;\n    }\n\n    public void start() {\n    }\n\n    public void shutdown() {\n        this.scheduledExecutorService.shutdown();\n        this.commercialExecutor.shutdown();\n        this.accountExecutor.shutdown();\n        this.cleanResourceExecutor.shutdown();\n        this.accountStatManager.shutdown();\n    }\n\n    public StatsItem getStatsItem(final String statsName, final String statsKey) {\n        try {\n            return this.statsTable.get(statsName).getStatsItem(statsKey);\n        } catch (Exception e) {\n        }\n\n        return null;\n    }\n\n    public void onTopicDeleted(final String topic) {\n        this.statsTable.get(Stats.TOPIC_PUT_NUMS).delValue(topic);\n        this.statsTable.get(Stats.TOPIC_PUT_SIZE).delValue(topic);\n        if (enableQueueStat) {\n            this.statsTable.get(Stats.QUEUE_PUT_NUMS).delValueByPrefixKey(topic, \"@\");\n            this.statsTable.get(Stats.QUEUE_PUT_SIZE).delValueByPrefixKey(topic, \"@\");\n        }\n        this.statsTable.get(Stats.GROUP_GET_NUMS).delValueByPrefixKey(topic, \"@\");\n        this.statsTable.get(Stats.GROUP_GET_SIZE).delValueByPrefixKey(topic, \"@\");\n        this.statsTable.get(Stats.GROUP_CK_NUMS).delValueByPrefixKey(topic, \"@\");\n        this.statsTable.get(Stats.GROUP_ACK_NUMS).delValueByPrefixKey(topic, \"@\");\n        this.statsTable.get(Stats.QUEUE_GET_NUMS).delValueByPrefixKey(topic, \"@\");\n        this.statsTable.get(Stats.QUEUE_GET_SIZE).delValueByPrefixKey(topic, \"@\");\n        this.statsTable.get(Stats.SNDBCK_PUT_NUMS).delValueByPrefixKey(topic, \"@\");\n        this.statsTable.get(Stats.GROUP_GET_LATENCY).delValueByInfixKey(topic, \"@\");\n        this.statsTable.get(Stats.TOPIC_PUT_LATENCY).delValueBySuffixKey(topic, \"@\");\n        this.momentStatsItemSetFallSize.delValueByInfixKey(topic, \"@\");\n        this.momentStatsItemSetFallTime.delValueByInfixKey(topic, \"@\");\n    }\n\n    public void onGroupDeleted(final String group) {\n        this.statsTable.get(Stats.GROUP_GET_NUMS).delValueBySuffixKey(group, \"@\");\n        this.statsTable.get(Stats.GROUP_GET_SIZE).delValueBySuffixKey(group, \"@\");\n        this.statsTable.get(Stats.GROUP_CK_NUMS).delValueBySuffixKey(group, \"@\");\n        this.statsTable.get(Stats.GROUP_ACK_NUMS).delValueBySuffixKey(group, \"@\");\n        if (enableQueueStat) {\n            this.statsTable.get(Stats.QUEUE_GET_NUMS).delValueBySuffixKey(group, \"@\");\n            this.statsTable.get(Stats.QUEUE_GET_SIZE).delValueBySuffixKey(group, \"@\");\n        }\n        this.statsTable.get(Stats.SNDBCK_PUT_NUMS).delValueBySuffixKey(group, \"@\");\n        this.statsTable.get(Stats.GROUP_GET_LATENCY).delValueBySuffixKey(group, \"@\");\n        this.momentStatsItemSetFallSize.delValueBySuffixKey(group, \"@\");\n        this.momentStatsItemSetFallTime.delValueBySuffixKey(group, \"@\");\n    }\n\n    public void incQueuePutNums(final String topic, final Integer queueId) {\n        if (enableQueueStat) {\n            this.statsTable.get(Stats.QUEUE_PUT_NUMS).addValue(buildStatsKey(topic, queueId), 1, 1);\n        }\n    }\n\n    public void incQueuePutNums(final String topic, final Integer queueId, int num, int times) {\n        if (enableQueueStat) {\n            this.statsTable.get(Stats.QUEUE_PUT_NUMS).addValue(buildStatsKey(topic, queueId), num, times);\n        }\n    }\n\n    public void incQueuePutSize(final String topic, final Integer queueId, final int size) {\n        if (enableQueueStat) {\n            this.statsTable.get(Stats.QUEUE_PUT_SIZE).addValue(buildStatsKey(topic, queueId), size, 1);\n        }\n    }\n\n    public void incQueueGetNums(final String group, final String topic, final Integer queueId, final int incValue) {\n        if (enableQueueStat) {\n            final String statsKey = buildStatsKey(topic, queueId, group);\n            this.statsTable.get(Stats.QUEUE_GET_NUMS).addValue(statsKey, incValue, 1);\n        }\n    }\n\n    public void incQueueGetSize(final String group, final String topic, final Integer queueId, final int incValue) {\n        if (enableQueueStat) {\n            final String statsKey = buildStatsKey(topic, queueId, group);\n            this.statsTable.get(Stats.QUEUE_GET_SIZE).addValue(statsKey, incValue, 1);\n        }\n    }\n\n    public void incConsumerRegisterTime(final int incValue) {\n        this.statsTable.get(CONSUMER_REGISTER_TIME).addValue(this.clusterName, incValue, 1);\n    }\n\n    public void incProducerRegisterTime(final int incValue) {\n        this.statsTable.get(PRODUCER_REGISTER_TIME).addValue(this.clusterName, incValue, 1);\n    }\n\n    public void incChannelConnectNum() {\n        this.statsTable.get(CHANNEL_ACTIVITY).addValue(CHANNEL_ACTIVITY_CONNECT, 1, 1);\n    }\n\n    public void incChannelCloseNum() {\n        this.statsTable.get(CHANNEL_ACTIVITY).addValue(CHANNEL_ACTIVITY_CLOSE, 1, 1);\n    }\n\n    public void incChannelExceptionNum() {\n        this.statsTable.get(CHANNEL_ACTIVITY).addValue(CHANNEL_ACTIVITY_EXCEPTION, 1, 1);\n    }\n\n    public void incChannelIdleNum() {\n        this.statsTable.get(CHANNEL_ACTIVITY).addValue(CHANNEL_ACTIVITY_IDLE, 1, 1);\n    }\n\n    public void incTopicPutNums(final String topic) {\n        this.statsTable.get(Stats.TOPIC_PUT_NUMS).addValue(topic, 1, 1);\n    }\n\n    public void incTopicPutNums(final String topic, int num, int times) {\n        this.statsTable.get(Stats.TOPIC_PUT_NUMS).addValue(topic, num, times);\n    }\n\n    public void incTopicPutSize(final String topic, final int size) {\n        this.statsTable.get(Stats.TOPIC_PUT_SIZE).addValue(topic, size, 1);\n    }\n\n    public void incGroupGetNums(final String group, final String topic, final int incValue) {\n        final String statsKey = buildStatsKey(topic, group);\n        this.statsTable.get(Stats.GROUP_GET_NUMS).addValue(statsKey, incValue, 1);\n    }\n\n    public void incGroupCkNums(final String group, final String topic, final int incValue) {\n        final String statsKey = buildStatsKey(topic, group);\n        this.statsTable.get(Stats.GROUP_CK_NUMS).addValue(statsKey, incValue, 1);\n    }\n\n    public void incGroupAckNums(final String group, final String topic, final int incValue) {\n        final String statsKey = buildStatsKey(topic, group);\n        this.statsTable.get(Stats.GROUP_ACK_NUMS).addValue(statsKey, incValue, 1);\n    }\n\n    public String buildStatsKey(String topic, String group) {\n        StringBuilder strBuilder;\n        if (topic != null && group != null) {\n            strBuilder = new StringBuilder(topic.length() + group.length() + 1);\n        } else {\n            strBuilder = new StringBuilder();\n        }\n        strBuilder.append(topic).append(\"@\").append(group);\n        return strBuilder.toString();\n    }\n\n    public String buildStatsKey(String topic, int queueId) {\n        StringBuilder strBuilder;\n        if (topic != null) {\n            strBuilder = new StringBuilder(topic.length() + 5);\n        } else {\n            strBuilder = new StringBuilder();\n        }\n        strBuilder.append(topic).append(\"@\").append(queueId);\n        return strBuilder.toString();\n    }\n\n    public String buildStatsKey(String topic, int queueId, String group) {\n        StringBuilder strBuilder;\n        if (topic != null && group != null) {\n            strBuilder = new StringBuilder(topic.length() + group.length() + 6);\n        } else {\n            strBuilder = new StringBuilder();\n        }\n        strBuilder.append(topic).append(\"@\").append(queueId).append(\"@\").append(group);\n        return strBuilder.toString();\n    }\n\n    public String buildStatsKey(int queueId, String topic, String group) {\n        StringBuilder strBuilder;\n        if (topic != null && group != null) {\n            strBuilder = new StringBuilder(topic.length() + group.length() + 6);\n        } else {\n            strBuilder = new StringBuilder();\n        }\n        strBuilder.append(queueId).append(\"@\").append(topic).append(\"@\").append(group);\n        return strBuilder.toString();\n    }\n\n    public void incGroupGetSize(final String group, final String topic, final int incValue) {\n        final String statsKey = buildStatsKey(topic, group);\n        this.statsTable.get(Stats.GROUP_GET_SIZE).addValue(statsKey, incValue, 1);\n    }\n\n    public void incGroupGetLatency(final String group, final String topic, final int queueId, final int incValue) {\n        String statsKey;\n        if (enableQueueStat) {\n            statsKey = buildStatsKey(queueId, topic, group);\n        } else {\n            statsKey = buildStatsKey(topic, group);\n        }\n        this.statsTable.get(Stats.GROUP_GET_LATENCY).addRTValue(statsKey, incValue, 1);\n    }\n\n    public void incTopicPutLatency(final String topic, final int queueId, final int incValue) {\n        StringBuilder statsKey;\n        if (topic != null) {\n            statsKey = new StringBuilder(topic.length() + 6);\n        } else {\n            statsKey = new StringBuilder(6);\n        }\n        statsKey.append(queueId).append(\"@\").append(topic);\n        this.statsTable.get(Stats.TOPIC_PUT_LATENCY).addValue(statsKey.toString(), incValue, 1);\n    }\n    public void incBrokerPutNums() {\n        this.statsTable.get(Stats.BROKER_PUT_NUMS).getAndCreateStatsItem(this.clusterName).getValue().add(1);\n    }\n\n    public void incBrokerPutNums(final String topic, final int incValue) {\n        this.statsTable.get(Stats.BROKER_PUT_NUMS).getAndCreateStatsItem(this.clusterName).getValue().add(incValue);\n        incBrokerPutNumsWithoutSystemTopic(topic, incValue);\n    }\n\n    public void incBrokerGetNums(final String topic, final int incValue) {\n        this.statsTable.get(Stats.BROKER_GET_NUMS).getAndCreateStatsItem(this.clusterName).getValue().add(incValue);\n        this.incBrokerGetNumsWithoutSystemTopic(topic, incValue);\n    }\n\n    public void incBrokerAckNums(final int incValue) {\n        this.statsTable.get(BROKER_ACK_NUMS).getAndCreateStatsItem(this.clusterName).getValue().add(incValue);\n    }\n\n    public void incBrokerCkNums(final int incValue) {\n        this.statsTable.get(BROKER_CK_NUMS).getAndCreateStatsItem(this.clusterName).getValue().add(incValue);\n    }\n\n    public void incBrokerGetNumsWithoutSystemTopic(final String topic, final int incValue) {\n        if (TopicValidator.isSystemTopic(topic)) {\n            return;\n        }\n        this.statsTable.get(BROKER_GET_NUMS_WITHOUT_SYSTEM_TOPIC).getAndCreateStatsItem(this.clusterName).getValue().add(incValue);\n    }\n\n    public void incBrokerPutNumsWithoutSystemTopic(final String topic, final int incValue) {\n        if (TopicValidator.isSystemTopic(topic)) {\n            return;\n        }\n        this.statsTable.get(BROKER_PUT_NUMS_WITHOUT_SYSTEM_TOPIC).getAndCreateStatsItem(this.clusterName).getValue().add(incValue);\n    }\n\n    public long getBrokerGetNumsWithoutSystemTopic() {\n        final StatsItemSet statsItemSet = this.statsTable.get(BROKER_GET_NUMS_WITHOUT_SYSTEM_TOPIC);\n        if (statsItemSet == null) {\n            return 0;\n        }\n        final StatsItem statsItem = statsItemSet.getStatsItem(this.clusterName);\n        if (statsItem == null) {\n            return 0;\n        }\n        return statsItem.getValue().longValue();\n    }\n\n    public long getBrokerPutNumsWithoutSystemTopic() {\n        final StatsItemSet statsItemSet = this.statsTable.get(BROKER_PUT_NUMS_WITHOUT_SYSTEM_TOPIC);\n        if (statsItemSet == null) {\n            return 0;\n        }\n        final StatsItem statsItem = statsItemSet.getStatsItem(this.clusterName);\n        if (statsItem == null) {\n            return 0;\n        }\n        return statsItem.getValue().longValue();\n    }\n\n    public void incSendBackNums(final String group, final String topic) {\n        final String statsKey = buildStatsKey(topic, group);\n        this.statsTable.get(Stats.SNDBCK_PUT_NUMS).addValue(statsKey, 1, 1);\n    }\n\n    public double tpsTopicPutNums(final String topic) {\n        return this.statsTable.get(TOPIC_PUT_NUMS).getStatsDataInMinute(topic).getTps();\n    }\n\n    public double tpsGroupGetNums(final String group, final String topic) {\n        final String statsKey = buildStatsKey(topic, group);\n        return this.statsTable.get(Stats.GROUP_GET_NUMS).getStatsDataInMinute(statsKey).getTps();\n    }\n\n    public void recordDiskFallBehindTime(final String group, final String topic, final int queueId,\n        final long fallBehind) {\n        final String statsKey = buildStatsKey(queueId, topic, group);\n        this.momentStatsItemSetFallTime.setValue(statsKey, fallBehind);\n    }\n\n    public void recordDiskFallBehindSize(final String group, final String topic, final int queueId,\n        final long fallBehind) {\n        final String statsKey = buildStatsKey(queueId, topic, group);\n        this.momentStatsItemSetFallSize.setValue(statsKey, fallBehind);\n    }\n\n    public void incDLQStatValue(final String key, final String owner, final String group,\n        final String topic, final String type, final int incValue) {\n        final String statsKey = buildCommercialStatsKey(owner, topic, group, type);\n        this.statsTable.get(key).addValue(statsKey, incValue, 1);\n    }\n\n    public void incCommercialValue(final String key, final String owner, final String group,\n        final String topic, final String type, final int incValue) {\n        final String statsKey = buildCommercialStatsKey(owner, topic, group, type);\n        this.statsTable.get(key).addValue(statsKey, incValue, 1);\n    }\n\n    public void incAccountValue(final String key, final String accountOwnerParent, final String accountOwnerSelf,\n        final String instanceId, final String group, final String topic,\n        final String msgType, final int incValue) {\n        final String statsKey = buildAccountStatsKey(accountOwnerParent, accountOwnerSelf, instanceId, topic, group,\n            msgType);\n        this.statsTable.get(key).addValue(statsKey, incValue, 1);\n    }\n\n    public void incAccountValue(final String key, final String accountOwnerParent, final String accountOwnerSelf,\n        final String instanceId, final String group, final String topic,\n        final String msgType, final String flowlimitThreshold, final int incValue) {\n        final String statsKey = buildAccountStatsKey(accountOwnerParent, accountOwnerSelf, instanceId, topic, group,\n            msgType, flowlimitThreshold);\n        this.statsTable.get(key).addValue(statsKey, incValue, 1);\n    }\n\n    public void incAccountValue(final String statType, final String owner, final String instanceId, final String topic,\n        final String group, final String msgType,\n        final long... incValues) {\n        final String key = buildAccountStatKey(owner, instanceId, topic, group, msgType);\n        this.accountStatManager.inc(statType, key, incValues);\n    }\n\n    public void incAccountValue(final String statType, final String owner, final String instanceId, final String topic,\n        final String group, final String msgType, final String flowlimitThreshold,\n        final long... incValues) {\n        final String key = buildAccountStatKey(owner, instanceId, topic, group, msgType, flowlimitThreshold);\n        this.accountStatManager.inc(statType, key, incValues);\n    }\n\n    public String buildCommercialStatsKey(String owner, String topic, String group, String type) {\n        StringBuilder strBuilder = new StringBuilder();\n        strBuilder.append(owner);\n        strBuilder.append(\"@\");\n        strBuilder.append(topic);\n        strBuilder.append(\"@\");\n        strBuilder.append(group);\n        strBuilder.append(\"@\");\n        strBuilder.append(type);\n        return strBuilder.toString();\n    }\n\n    public String buildAccountStatsKey(String accountOwnerParent, String accountOwnerSelf, String instanceId,\n        String topic, String group, String msgType) {\n        StringBuffer strBuilder = new StringBuffer();\n        strBuilder.append(accountOwnerParent);\n        strBuilder.append(\"@\");\n        strBuilder.append(accountOwnerSelf);\n        strBuilder.append(\"@\");\n        strBuilder.append(instanceId);\n        strBuilder.append(\"@\");\n        strBuilder.append(topic);\n        strBuilder.append(\"@\");\n        strBuilder.append(group);\n        strBuilder.append(\"@\");\n        strBuilder.append(msgType);\n        return strBuilder.toString();\n    }\n\n    public String buildAccountStatsKey(String accountOwnerParent, String accountOwnerSelf, String instanceId,\n        String topic, String group, String msgType, String flowlimitThreshold) {\n        StringBuffer strBuilder = new StringBuffer();\n        strBuilder.append(accountOwnerParent);\n        strBuilder.append(\"@\");\n        strBuilder.append(accountOwnerSelf);\n        strBuilder.append(\"@\");\n        strBuilder.append(instanceId);\n        strBuilder.append(\"@\");\n        strBuilder.append(topic);\n        strBuilder.append(\"@\");\n        strBuilder.append(group);\n        strBuilder.append(\"@\");\n        strBuilder.append(msgType);\n        strBuilder.append(\"@\");\n        strBuilder.append(flowlimitThreshold);\n        return strBuilder.toString();\n    }\n\n    public String buildAccountStatKey(final String owner, final String instanceId,\n        final String topic, final String group,\n        final String msgType) {\n        final String sep = \"|\";\n        StringBuffer strBuilder = new StringBuffer();\n        strBuilder.append(owner).append(sep);\n        strBuilder.append(instanceId).append(sep);\n        strBuilder.append(topic).append(sep);\n        strBuilder.append(group).append(sep);\n        strBuilder.append(msgType);\n        return strBuilder.toString();\n    }\n\n    public String buildAccountStatKey(final String owner, final String instanceId,\n        final String topic, final String group,\n        final String msgType, String flowlimitThreshold) {\n        final String sep = \"|\";\n        StringBuffer strBuilder = new StringBuffer();\n        strBuilder.append(owner).append(sep);\n        strBuilder.append(instanceId).append(sep);\n        strBuilder.append(topic).append(sep);\n        strBuilder.append(group).append(sep);\n        strBuilder.append(msgType).append(sep);\n        strBuilder.append(flowlimitThreshold);\n        return strBuilder.toString();\n    }\n\n    public String[] splitAccountStatKey(final String accountStatKey) {\n        final String sep = \"\\\\|\";\n        return accountStatKey.split(sep);\n    }\n\n    private StatisticsKindMeta createStatisticsKindMeta(String name,\n        String[] itemNames,\n        ScheduledExecutorService executorService,\n        StatisticsItemFormatter formatter,\n        Logger log,\n        long interval) {\n        final BrokerConfig brokerConfig = this.brokerConfig;\n        StatisticsItemPrinter printer = new StatisticsItemPrinter(formatter, log);\n        StatisticsKindMeta kindMeta = new StatisticsKindMeta();\n        kindMeta.setName(name);\n        kindMeta.setItemNames(itemNames);\n        kindMeta.setScheduledPrinter(\n            new StatisticsItemScheduledIncrementPrinter(\n                \"Stat In One Minute: \",\n                printer,\n                executorService,\n                new StatisticsItemScheduledPrinter.InitialDelay() {\n                    @Override\n                    public long get() {\n                        return Math.abs(UtilAll.computeNextMinutesTimeMillis() - System.currentTimeMillis());\n                    }\n                },\n                interval,\n                new String[] {MSG_NUM},\n                new StatisticsItemScheduledIncrementPrinter.Valve() {\n                    @Override\n                    public boolean enabled() {\n                        return brokerConfig != null ? brokerConfig.isAccountStatsEnable() : true;\n                    }\n\n                    @Override\n                    public boolean printZeroLine() {\n                        return brokerConfig != null ? brokerConfig.isAccountStatsPrintZeroValues() : true;\n                    }\n                }\n            )\n        );\n        return kindMeta;\n    }\n\n    public interface StateGetter {\n        boolean online(String instanceId, String group, String topic);\n    }\n\n\n    private void cleanAllResource() {\n        try {\n            int maxStatsIdleTimeInMinutes = brokerConfig != null ? brokerConfig.getMaxStatsIdleTimeInMinutes() : -1;\n            if (maxStatsIdleTimeInMinutes < 0) {\n                COMMERCIAL_LOG.info(\"[BrokerStatsManager#cleanAllResource] maxStatsIdleTimeInMinutes={}, no need to clean resource\", maxStatsIdleTimeInMinutes);\n                return;\n            }\n            if (maxStatsIdleTimeInMinutes <= 10 && maxStatsIdleTimeInMinutes >= 0) {\n                maxStatsIdleTimeInMinutes = 30;\n            }\n            for (String statsKind : NEED_CLEAN_STATS_SET) {\n                StatsItemSet statsItemSet = this.statsTable.get(statsKind);\n                if (null == statsItemSet) {\n                    continue;\n                }\n                statsItemSet.cleanResource(maxStatsIdleTimeInMinutes);\n            }\n            momentStatsItemSetFallSize.cleanResource(maxStatsIdleTimeInMinutes);\n            momentStatsItemSetFallTime.cleanResource(maxStatsIdleTimeInMinutes);\n        } catch (Throwable throwable) {\n            COMMERCIAL_LOG.error(\"[BrokerStatsManager#cleanAllResource] clean resource error\", throwable);\n        }\n    }\n\n    public enum StatsType {\n        SEND_SUCCESS,\n        SEND_FAILURE,\n\n        RCV_SUCCESS,\n        RCV_EPOLLS,\n        SEND_BACK,\n        SEND_BACK_TO_DLQ,\n\n        SEND_ORDER,\n        SEND_TIMER,\n        SEND_TRANSACTION,\n\n        PERM_FAILURE\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/stats/LmqBrokerStatsManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.stats;\n\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\n\npublic class LmqBrokerStatsManager extends BrokerStatsManager {\n\n    private final BrokerConfig brokerConfig;\n\n    public LmqBrokerStatsManager(BrokerConfig brokerConfig) {\n        super(brokerConfig.getBrokerClusterName(), brokerConfig.isEnableDetailStat());\n        this.brokerConfig = brokerConfig;\n    }\n\n    @Override\n    public void incGroupGetNums(final String group, final String topic, final int incValue) {\n        super.incGroupGetNums(getAdjustedGroup(group), getAdjustedTopic(topic), incValue);\n    }\n\n    @Override\n    public void incGroupGetSize(final String group, final String topic, final int incValue) {\n        super.incGroupGetSize(getAdjustedGroup(group), getAdjustedTopic(topic), incValue);\n    }\n\n    @Override\n    public void incGroupAckNums(final String group, final String topic, final int incValue) {\n        super.incGroupAckNums(getAdjustedGroup(group), getAdjustedTopic(topic), incValue);\n    }\n\n    @Override\n    public void incGroupCkNums(final String group, final String topic, final int incValue) {\n        super.incGroupCkNums(getAdjustedGroup(group), getAdjustedTopic(topic), incValue);\n    }\n\n    @Override\n    public void incGroupGetLatency(final String group, final String topic, final int queueId, final int incValue) {\n        super.incGroupGetLatency(getAdjustedGroup(group), getAdjustedTopic(topic), queueId, incValue);\n    }\n\n    @Override\n    public void incSendBackNums(final String group, final String topic) {\n        super.incSendBackNums(getAdjustedGroup(group), getAdjustedTopic(topic));\n    }\n\n    @Override\n    public double tpsGroupGetNums(final String group, final String topic) {\n        return super.tpsGroupGetNums(getAdjustedGroup(group), getAdjustedTopic(topic));\n    }\n\n    @Override\n    public void recordDiskFallBehindTime(final String group, final String topic, final int queueId,\n        final long fallBehind) {\n        super.recordDiskFallBehindTime(getAdjustedGroup(group), getAdjustedTopic(topic), queueId, fallBehind);\n    }\n\n    @Override\n    public void recordDiskFallBehindSize(final String group, final String topic, final int queueId,\n        final long fallBehind) {\n        super.recordDiskFallBehindSize(getAdjustedGroup(group), getAdjustedTopic(topic), queueId, fallBehind);\n    }\n\n    private String getAdjustedGroup(String group) {\n        return !brokerConfig.isEnableLmqStats() && MixAll.isLmq(group) ? MixAll.LMQ_PREFIX : group;\n    }\n\n    private String getAdjustedTopic(String topic) {\n        return !brokerConfig.isEnableLmqStats() && MixAll.isLmq(topic) ? MixAll.LMQ_PREFIX : topic;\n    }\n\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/timer/Slot.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.timer;\n\n/**\n * Represents a slot of timing wheel. Format:\n * ┌────────────┬───────────┬───────────┬───────────┬───────────┐\n * │delayed time│ first pos │ last pos  │    num    │   magic   │\n * ├────────────┼───────────┼───────────┼───────────┼───────────┤\n * │   8bytes   │   8bytes  │  8bytes   │   4bytes  │   4bytes  │\n * └────────────┴───────────┴───────────┴───────────┴───────────┘\n */\npublic class Slot {\n    public static final short SIZE = 32;\n    public final long timeMs; //delayed time\n    public final long firstPos;\n    public final long lastPos;\n    public final int num;\n    public final int magic; //no use now, just keep it\n\n    public Slot(long timeMs, long firstPos, long lastPos) {\n        this.timeMs = timeMs;\n        this.firstPos = firstPos;\n        this.lastPos = lastPos;\n        this.num = 0;\n        this.magic = 0;\n    }\n\n    public Slot(long timeMs, long firstPos, long lastPos, int num, int magic) {\n        this.timeMs = timeMs;\n        this.firstPos = firstPos;\n        this.lastPos = lastPos;\n        this.num = num;\n        this.magic = magic;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/timer/TimerCheckpoint.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.timer;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.io.RandomAccessFile;\nimport java.nio.ByteBuffer;\nimport java.nio.MappedByteBuffer;\nimport java.nio.channels.FileChannel;\nimport java.nio.channels.FileChannel.MapMode;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.store.logfile.DefaultMappedFile;\n\npublic class TimerCheckpoint {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private final RandomAccessFile randomAccessFile;\n    private final FileChannel fileChannel;\n    private final MappedByteBuffer mappedByteBuffer;\n    private volatile long lastReadTimeMs = 0; //if it is slave, need to read from master\n    private volatile long lastTimerLogFlushPos = 0;\n    private volatile long lastTimerQueueOffset = 0;\n    private volatile long masterTimerQueueOffset = 0; // read from master\n    private final DataVersion dataVersion = new DataVersion();\n\n    public TimerCheckpoint() {\n        this.randomAccessFile = null;\n        this.fileChannel = null;\n        this.mappedByteBuffer = null;\n    }\n\n    public TimerCheckpoint(final String scpPath) throws IOException {\n        File file = new File(scpPath);\n        UtilAll.ensureDirOK(file.getParent());\n        boolean fileExists = file.exists();\n\n        this.randomAccessFile = new RandomAccessFile(file, \"rw\");\n        this.fileChannel = this.randomAccessFile.getChannel();\n        this.mappedByteBuffer = fileChannel.map(MapMode.READ_WRITE, 0, DefaultMappedFile.OS_PAGE_SIZE);\n\n        if (fileExists) {\n            log.info(\"timer checkpoint file exists, \" + scpPath);\n            this.lastReadTimeMs = this.mappedByteBuffer.getLong(0);\n            this.lastTimerLogFlushPos = this.mappedByteBuffer.getLong(8);\n            this.lastTimerQueueOffset = this.mappedByteBuffer.getLong(16);\n            this.masterTimerQueueOffset = this.mappedByteBuffer.getLong(24);\n            // new add to record dataVersion\n            if (this.mappedByteBuffer.hasRemaining()) {\n                dataVersion.setStateVersion(this.mappedByteBuffer.getLong(32));\n                dataVersion.setTimestamp(this.mappedByteBuffer.getLong(40));\n                dataVersion.setCounter(new AtomicLong(this.mappedByteBuffer.getLong(48)));\n            }\n\n            log.info(\"timer checkpoint file lastReadTimeMs \" + this.lastReadTimeMs + \", \"\n                + UtilAll.timeMillisToHumanString(this.lastReadTimeMs));\n            log.info(\"timer checkpoint file lastTimerLogFlushPos \" + this.lastTimerLogFlushPos);\n            log.info(\"timer checkpoint file lastTimerQueueOffset \" + this.lastTimerQueueOffset);\n            log.info(\"timer checkpoint file masterTimerQueueOffset \" + this.masterTimerQueueOffset);\n            log.info(\"timer checkpoint file data version state version \" + this.dataVersion.getStateVersion());\n            log.info(\"timer checkpoint file data version timestamp \" + this.dataVersion.getTimestamp());\n            log.info(\"timer checkpoint file data version counter \" + this.dataVersion.getCounter());\n        } else {\n            log.info(\"timer checkpoint file not exists, \" + scpPath);\n        }\n    }\n\n    public void shutdown() {\n\n        try {\n            this.flush();\n        } catch (Throwable e) {\n            log.error(\"Shutdown error in timer check point\", e);\n        }\n\n        if (null != this.mappedByteBuffer) {\n            // unmap mappedByteBuffer\n            UtilAll.cleanBuffer(this.mappedByteBuffer);\n        }\n\n        if (null != this.fileChannel) {\n            try {\n                this.fileChannel.close();\n            } catch (Throwable e) {\n                log.error(\"Shutdown error in timer check point\", e);\n            }\n        }\n    }\n\n    public void flush() {\n        if (null == this.mappedByteBuffer) {\n            return;\n        }\n        this.mappedByteBuffer.putLong(0, this.lastReadTimeMs);\n        this.mappedByteBuffer.putLong(8, this.lastTimerLogFlushPos);\n        this.mappedByteBuffer.putLong(16, this.lastTimerQueueOffset);\n        this.mappedByteBuffer.putLong(24, this.masterTimerQueueOffset);\n        // new add to record dataVersion\n        this.mappedByteBuffer.putLong(32, this.dataVersion.getStateVersion());\n        this.mappedByteBuffer.putLong(40, this.dataVersion.getTimestamp());\n        this.mappedByteBuffer.putLong(48, this.dataVersion.getCounter().get());\n        this.mappedByteBuffer.force();\n    }\n\n    public long getLastReadTimeMs() {\n        return lastReadTimeMs;\n    }\n\n    public static ByteBuffer encode(TimerCheckpoint another) {\n        ByteBuffer byteBuffer = ByteBuffer.allocate(56);\n        byteBuffer.putLong(another.getLastReadTimeMs());\n        byteBuffer.putLong(another.getLastTimerLogFlushPos());\n        byteBuffer.putLong(another.getLastTimerQueueOffset());\n        byteBuffer.putLong(another.getMasterTimerQueueOffset());\n        // new add to record dataVersion\n        byteBuffer.putLong(another.getDataVersion().getStateVersion());\n        byteBuffer.putLong(another.getDataVersion().getTimestamp());\n        byteBuffer.putLong(another.getDataVersion().getCounter().get());\n        byteBuffer.flip();\n        return byteBuffer;\n    }\n\n    public static TimerCheckpoint decode(ByteBuffer byteBuffer) {\n        TimerCheckpoint tmp = new TimerCheckpoint();\n        tmp.setLastReadTimeMs(byteBuffer.getLong());\n        tmp.setLastTimerLogFlushPos(byteBuffer.getLong());\n        tmp.setLastTimerQueueOffset(byteBuffer.getLong());\n        tmp.setMasterTimerQueueOffset(byteBuffer.getLong());\n        // new add to record dataVersion\n        if (byteBuffer.hasRemaining()) {\n            tmp.getDataVersion().setStateVersion(byteBuffer.getLong());\n            tmp.getDataVersion().setTimestamp(byteBuffer.getLong());\n            tmp.getDataVersion().setCounter(new AtomicLong(byteBuffer.getLong()));\n        }\n        return tmp;\n    }\n\n    public void setLastReadTimeMs(long lastReadTimeMs) {\n        this.lastReadTimeMs = lastReadTimeMs;\n    }\n\n    public long getLastTimerLogFlushPos() {\n        return lastTimerLogFlushPos;\n    }\n\n    public void setLastTimerLogFlushPos(long lastTimerLogFlushPos) {\n        this.lastTimerLogFlushPos = lastTimerLogFlushPos;\n    }\n\n    public long getLastTimerQueueOffset() {\n        return lastTimerQueueOffset;\n    }\n\n    public void setLastTimerQueueOffset(long lastTimerQueueOffset) {\n        this.lastTimerQueueOffset = lastTimerQueueOffset;\n    }\n\n    public long getMasterTimerQueueOffset() {\n        return masterTimerQueueOffset;\n    }\n\n    public void setMasterTimerQueueOffset(final long masterTimerQueueOffset) {\n        this.masterTimerQueueOffset = masterTimerQueueOffset;\n    }\n\n    public void updateDataVersion(long stateVersion) {\n        dataVersion.nextVersion(stateVersion);\n    }\n\n    public DataVersion getDataVersion() {\n        return dataVersion;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/timer/TimerLog.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.timer;\n\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.RunningFlags;\nimport org.apache.rocketmq.store.logfile.MappedFile;\nimport org.apache.rocketmq.store.MappedFileQueue;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\n\nimport java.nio.ByteBuffer;\n\npublic class TimerLog {\n    private static Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    public final static int BLANK_MAGIC_CODE = 0xBBCCDDEE ^ 1880681586 + 8;\n    private final static int MIN_BLANK_LEN = 4 + 8 + 4;\n    public final static int UNIT_SIZE = 4  //size\n            + 8 //prev pos\n            + 4 //magic value\n            + 8 //curr write time, for trace\n            + 4 //delayed time, for check\n            + 8 //offsetPy\n            + 4 //sizePy\n            + 4 //hash code of real topic\n            + 8; //reserved value, just in case of\n    public final static int UNIT_PRE_SIZE_FOR_MSG = 28;\n    public final static int UNIT_PRE_SIZE_FOR_METRIC = 40;\n    private final MappedFileQueue mappedFileQueue;\n\n    private final int fileSize;\n\n    public TimerLog(final String storePath, final int fileSize) {\n        this(storePath, fileSize, null, false);\n    }\n\n    public TimerLog(final String storePath, final int fileSize, RunningFlags runningFlags, boolean writeWithoutMmap) {\n        this.fileSize = fileSize;\n        this.mappedFileQueue = new MappedFileQueue(storePath, fileSize, null, runningFlags, writeWithoutMmap);\n    }\n\n    public boolean load() {\n        return this.mappedFileQueue.load();\n    }\n\n    public long append(byte[] data) {\n        return append(data, 0, data.length);\n    }\n\n    public long append(byte[] data, int pos, int len) {\n        MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();\n        if (null == mappedFile || mappedFile.isFull()) {\n            mappedFile = this.mappedFileQueue.getLastMappedFile(0);\n        }\n        if (null == mappedFile) {\n            log.error(\"Create mapped file1 error for timer log\");\n            return -1;\n        }\n        if (len + MIN_BLANK_LEN > mappedFile.getFileSize() - mappedFile.getWrotePosition()) {\n            ByteBuffer byteBuffer = ByteBuffer.allocate(MIN_BLANK_LEN);\n            byteBuffer.putInt(mappedFile.getFileSize() - mappedFile.getWrotePosition());\n            byteBuffer.putLong(0);\n            byteBuffer.putInt(BLANK_MAGIC_CODE);\n            if (mappedFile.appendMessage(byteBuffer.array())) {\n                //need to set the wrote position\n                mappedFile.setWrotePosition(mappedFile.getFileSize());\n            } else {\n                log.error(\"Append blank error for timer log\");\n                return -1;\n            }\n            mappedFile = this.mappedFileQueue.getLastMappedFile(0);\n            if (null == mappedFile) {\n                log.error(\"create mapped file2 error for timer log\");\n                return -1;\n            }\n        }\n        long currPosition = mappedFile.getFileFromOffset() + mappedFile.getWrotePosition();\n        if (!mappedFile.appendMessage(data, pos, len)) {\n            log.error(\"Append error for timer log\");\n            return -1;\n        }\n        return currPosition;\n    }\n\n    public SelectMappedBufferResult getTimerMessage(long offsetPy) {\n        MappedFile mappedFile = mappedFileQueue.findMappedFileByOffset(offsetPy);\n        if (null == mappedFile)\n            return null;\n        return mappedFile.selectMappedBuffer((int) (offsetPy % mappedFile.getFileSize()));\n    }\n\n    public SelectMappedBufferResult getWholeBuffer(long offsetPy) {\n        MappedFile mappedFile = mappedFileQueue.findMappedFileByOffset(offsetPy);\n        if (null == mappedFile)\n            return null;\n        return mappedFile.selectMappedBuffer(0);\n    }\n\n    public MappedFileQueue getMappedFileQueue() {\n        return mappedFileQueue;\n    }\n\n    public void shutdown() {\n        try {\n            this.mappedFileQueue.flush(0);\n        } catch (Throwable e) {\n            log.error(\"flush error when shutdown\", e);\n        }\n\n        this.mappedFileQueue.cleanResourcesAll();\n    }\n\n    // be careful.\n    // if the format of timerlog changed, this offset has to be changed too\n    // so does the batch writing\n    public int getOffsetForLastUnit() {\n\n        return fileSize - (fileSize - MIN_BLANK_LEN) % UNIT_SIZE - MIN_BLANK_LEN - UNIT_SIZE;\n    }\n\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.timer;\n\nimport com.conversantmedia.util.concurrent.DisruptorBlockingQueue;\nimport io.opentelemetry.api.common.Attributes;\nimport java.io.File;\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport java.sql.Timestamp;\nimport java.text.SimpleDateFormat;\nimport java.util.ArrayList;\nimport java.util.Date;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Random;\nimport java.util.Set;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.ConcurrentSkipListSet;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.LinkedBlockingDeque;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.function.Function;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.math.NumberUtils;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.TopicFilterType;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.RunningFlags;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.StoreUtil;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.logfile.MappedFile;\nimport org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant;\nimport org.apache.rocketmq.store.metrics.DefaultStoreMetricsManager;\nimport org.apache.rocketmq.store.metrics.StoreMetricsManager;\nimport org.apache.rocketmq.store.queue.ConsumeQueueInterface;\nimport org.apache.rocketmq.store.queue.CqUnit;\nimport org.apache.rocketmq.store.queue.ReferredIterator;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.apache.rocketmq.store.util.PerfCounter;\n\npublic class TimerMessageStore {\n\n    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n\n    public static final int INITIAL = 0, RUNNING = 1, HAULT = 2, SHUTDOWN = 3;\n    private volatile int state = INITIAL;\n\n    public static final String TIMER_TOPIC = TopicValidator.SYSTEM_TOPIC_PREFIX + \"wheel_timer\";\n    public static final String TIMER_OUT_MS = MessageConst.PROPERTY_TIMER_OUT_MS;\n    public static final String TIMER_ENQUEUE_MS = MessageConst.PROPERTY_TIMER_ENQUEUE_MS;\n    public static final String TIMER_DEQUEUE_MS = MessageConst.PROPERTY_TIMER_DEQUEUE_MS;\n    public static final String TIMER_ROLL_TIMES = MessageConst.PROPERTY_TIMER_ROLL_TIMES;\n    public static final String TIMER_DELETE_UNIQUE_KEY = MessageConst.PROPERTY_TIMER_DEL_UNIQKEY;\n\n    public static final Random RANDOM = new Random();\n    public static final int PUT_OK = 0, PUT_NEED_RETRY = 1, PUT_NO_RETRY = 2;\n    public static final int DAY_SECS = 24 * 3600;\n    public static final int DEFAULT_CAPACITY = 1024;\n\n    // The total days in the timer wheel when precision is 1000ms.\n    // If the broker shutdown last more than the configured days, will cause message loss\n    public static final int TIMER_WHEEL_TTL_DAY = 7;\n    public static final int TIMER_BLANK_SLOTS = 60;\n    public static final int MAGIC_DEFAULT = 1;\n    public static final int MAGIC_ROLL = 1 << 1;\n    public static final int MAGIC_DELETE = 1 << 2;\n    public boolean debug = false;\n\n    protected static final String ENQUEUE_PUT = \"enqueue_put\";\n    protected static final String DEQUEUE_PUT = \"dequeue_put\";\n    protected final PerfCounter.Ticks perfCounterTicks = new PerfCounter.Ticks(LOGGER);\n\n    protected final BlockingQueue<TimerRequest> enqueuePutQueue;\n    protected final BlockingQueue<List<TimerRequest>> dequeueGetQueue;\n    protected final BlockingQueue<TimerRequest> dequeuePutQueue;\n\n    private final ByteBuffer timerLogBuffer = ByteBuffer.allocate(4 * 1024);\n    private final ThreadLocal<ByteBuffer> bufferLocal;\n    private final ScheduledExecutorService scheduler;\n\n    private final MessageStore messageStore;\n    private final TimerWheel timerWheel;\n    private final TimerLog timerLog;\n    private final TimerCheckpoint timerCheckpoint;\n\n    private TimerEnqueueGetService enqueueGetService;\n    private TimerEnqueuePutService enqueuePutService;\n    private TimerDequeueWarmService dequeueWarmService;\n    private TimerDequeueGetService dequeueGetService;\n    private TimerDequeuePutMessageService[] dequeuePutMessageServices;\n    private TimerDequeueGetMessageService[] dequeueGetMessageServices;\n    private TimerFlushService timerFlushService;\n\n    protected volatile long currReadTimeMs;\n    protected volatile long currWriteTimeMs;\n    protected volatile long preReadTimeMs;\n    protected volatile long commitReadTimeMs;\n    protected volatile long currQueueOffset; //only one queue that is 0\n    protected volatile long commitQueueOffset;\n    protected volatile long lastCommitReadTimeMs;\n    protected volatile long lastCommitQueueOffset;\n\n    private long lastEnqueueButExpiredTime;\n    private long lastEnqueueButExpiredStoreTime;\n\n    private final int commitLogFileSize;\n    private final int timerLogFileSize;\n    private final int timerRollWindowSlots;\n    private final int slotsTotal;\n\n    protected final int precisionMs;\n    protected final MessageStoreConfig storeConfig;\n    protected TimerMetrics timerMetrics;\n    protected long lastTimeOfCheckMetrics = System.currentTimeMillis();\n    protected AtomicInteger frequency = new AtomicInteger(0);\n\n    private volatile BrokerRole lastBrokerRole = BrokerRole.SLAVE;\n    //the dequeue is an asynchronous process, use this flag to track if the status has changed\n    private boolean dequeueStatusChangeFlag = false;\n    private long shouldStartTime;\n\n    // True if current store is master or current brokerId is equal to the minimum brokerId of the replica group in slaveActingMaster mode.\n    protected volatile boolean shouldRunningDequeue;\n    private final BrokerStatsManager brokerStatsManager;\n    private Function<MessageExtBrokerInner, PutMessageResult> escapeBridgeHook;\n\n    private final Object lockWhenFlush = new Object();\n\n    public TimerMessageStore(final MessageStore messageStore, final MessageStoreConfig storeConfig,\n        TimerCheckpoint timerCheckpoint, TimerMetrics timerMetrics,\n        final BrokerStatsManager brokerStatsManager) throws IOException {\n\n        this.messageStore = messageStore;\n        this.storeConfig = storeConfig;\n        this.commitLogFileSize = storeConfig.getMappedFileSizeCommitLog();\n        this.timerLogFileSize = storeConfig.getMappedFileSizeTimerLog();\n        this.precisionMs = storeConfig.getTimerPrecisionMs();\n\n        // TimerWheel contains the fixed number of slots regardless of precision.\n        this.slotsTotal = TIMER_WHEEL_TTL_DAY * DAY_SECS;\n\n        String timerWheelPath = getTimerWheelPath(storeConfig.getStorePathRootDir());\n        long snapOffset = -1;\n        if (storeConfig.isTimerWheelSnapshotFlush()) {\n            snapOffset = TimerWheel.getMaxSnapshotFlag(timerWheelPath);\n            if (snapOffset > 0) {\n                // correct recover offset\n                timerCheckpoint.setLastTimerLogFlushPos(snapOffset);\n                LOGGER.info(\"found timerWheel snapshot offset {}\", snapOffset);\n            } else {\n                LOGGER.info(\"not found timerWheel snapshot\", snapOffset);\n            }\n        }\n\n        RunningFlags runningFlags = null;\n        if (storeConfig.isEnableRunningFlagsInFlush() && messageStore != null) {\n            runningFlags = messageStore.getRunningFlags();\n        }\n\n        this.timerWheel = new TimerWheel(\n            timerWheelPath, this.slotsTotal, precisionMs, snapOffset);\n        this.timerLog = new TimerLog(getTimerLogPath(storeConfig.getStorePathRootDir()), timerLogFileSize,\n            runningFlags, storeConfig.isWriteWithoutMmap());\n        this.timerMetrics = timerMetrics;\n        this.timerCheckpoint = timerCheckpoint;\n        this.lastBrokerRole = storeConfig.getBrokerRole();\n\n        if (messageStore instanceof DefaultMessageStore) {\n            scheduler = ThreadUtils.newSingleThreadScheduledExecutor(\n                new ThreadFactoryImpl(\"TimerScheduledThread\",\n                    ((DefaultMessageStore) messageStore).getBrokerIdentity()));\n        } else {\n            scheduler = ThreadUtils.newSingleThreadScheduledExecutor(\n                new ThreadFactoryImpl(\"TimerScheduledThread\"));\n        }\n\n        // timerRollWindow contains the fixed number of slots regardless of precision.\n        if (storeConfig.getTimerRollWindowSlot() > slotsTotal - TIMER_BLANK_SLOTS\n            || storeConfig.getTimerRollWindowSlot() < 2) {\n            this.timerRollWindowSlots = slotsTotal - TIMER_BLANK_SLOTS;\n        } else {\n            this.timerRollWindowSlots = storeConfig.getTimerRollWindowSlot();\n        }\n\n        bufferLocal = new ThreadLocal<ByteBuffer>() {\n            @Override\n            protected ByteBuffer initialValue() {\n                return ByteBuffer.allocateDirect(storeConfig.getMaxMessageSize() + 100);\n            }\n        };\n\n        if (storeConfig.isTimerEnableDisruptor()) {\n            enqueuePutQueue = new DisruptorBlockingQueue<>(DEFAULT_CAPACITY);\n            dequeueGetQueue = new DisruptorBlockingQueue<>(DEFAULT_CAPACITY);\n            dequeuePutQueue = new DisruptorBlockingQueue<>(DEFAULT_CAPACITY);\n        } else {\n            enqueuePutQueue = new LinkedBlockingDeque<>(DEFAULT_CAPACITY);\n            dequeueGetQueue = new LinkedBlockingDeque<>(DEFAULT_CAPACITY);\n            dequeuePutQueue = new LinkedBlockingDeque<>(DEFAULT_CAPACITY);\n        }\n        this.brokerStatsManager = brokerStatsManager;\n    }\n\n    public void initService() {\n        enqueueGetService = new TimerEnqueueGetService();\n        enqueuePutService = new TimerEnqueuePutService();\n        dequeueWarmService = new TimerDequeueWarmService();\n        dequeueGetService = new TimerDequeueGetService();\n        timerFlushService = new TimerFlushService();\n\n        int getThreadNum = Math.max(storeConfig.getTimerGetMessageThreadNum(), 1);\n        dequeueGetMessageServices = new TimerDequeueGetMessageService[getThreadNum];\n        for (int i = 0; i < dequeueGetMessageServices.length; i++) {\n            dequeueGetMessageServices[i] = new TimerDequeueGetMessageService();\n        }\n\n        int putThreadNum = Math.max(storeConfig.getTimerPutMessageThreadNum(), 1);\n        dequeuePutMessageServices = new TimerDequeuePutMessageService[putThreadNum];\n        for (int i = 0; i < dequeuePutMessageServices.length; i++) {\n            dequeuePutMessageServices[i] = new TimerDequeuePutMessageService();\n        }\n    }\n\n    public boolean load() {\n        this.initService();\n        boolean load = timerLog.load();\n        load = load && this.timerMetrics.load();\n        recover();\n        calcTimerDistribution();\n        return load;\n    }\n\n    public static String getTimerWheelPath(final String rootDir) {\n        return rootDir + File.separator + \"timerwheel\";\n    }\n\n    public static String getTimerLogPath(final String rootDir) {\n        return rootDir + File.separator + \"timerlog\";\n    }\n\n    private void calcTimerDistribution() {\n        long startTime = System.currentTimeMillis();\n        List<Integer> timerDist = this.timerMetrics.getTimerDistList();\n        long currTime = System.currentTimeMillis() / precisionMs * precisionMs;\n        for (int i = 0; i < timerDist.size(); i++) {\n            int slotBeforeNum = i == 0 ? 0 : timerDist.get(i - 1) * 1000 / precisionMs;\n            int slotTotalNum = timerDist.get(i) * 1000 / precisionMs;\n            int periodTotal = 0;\n            for (int j = slotBeforeNum; j < slotTotalNum; j++) {\n                Slot slotEach = timerWheel.getSlot(currTime + (long) j * precisionMs);\n                periodTotal += slotEach.num;\n            }\n            LOGGER.debug(\"{} period's total num: {}\", timerDist.get(i), periodTotal);\n            this.timerMetrics.updateDistPair(timerDist.get(i), periodTotal);\n        }\n        long endTime = System.currentTimeMillis();\n        LOGGER.debug(\"Total cost Time: {}\", endTime - startTime);\n    }\n\n    @SuppressWarnings(\"NonAtomicOperationOnVolatileField\")\n    public void recover() {\n        //recover timerLog\n        long lastFlushPos = timerCheckpoint.getLastTimerLogFlushPos();\n        MappedFile lastFile = timerLog.getMappedFileQueue().getLastMappedFile();\n        if (null != lastFile) {\n            lastFlushPos = lastFlushPos - lastFile.getFileSize();\n        }\n        if (lastFlushPos < 0) {\n            lastFlushPos = 0;\n        }\n        long processOffset = recoverAndRevise(lastFlushPos, true);\n\n        timerLog.getMappedFileQueue().setFlushedWhere(processOffset);\n        //revise queue offset\n        long queueOffset = reviseQueueOffset(processOffset);\n        if (-1 == queueOffset) {\n            currQueueOffset = timerCheckpoint.getLastTimerQueueOffset();\n        } else {\n            currQueueOffset = queueOffset + 1;\n        }\n        currQueueOffset = Math.min(currQueueOffset, timerCheckpoint.getMasterTimerQueueOffset());\n        if (storeConfig.isTimerRocksDBEnable()) {\n            long commitOffsetInRocksDB = messageStore.getTimerMessageRocksDBStore().getCommitOffsetInRocksDB();\n            LOGGER.info(\"recover time wheel, currQueueOffset: {}, commitOffsetInRocksDB: {}\", currQueueOffset, commitOffsetInRocksDB);\n            currQueueOffset = Math.max(currQueueOffset, commitOffsetInRocksDB);\n        }\n        ConsumeQueueInterface cq = this.messageStore.getConsumeQueue(TIMER_TOPIC, 0);\n\n        // Correction based consume queue\n        if (cq != null && currQueueOffset < cq.getMinOffsetInQueue()) {\n            LOGGER.warn(\"Timer currQueueOffset:{} is smaller than minOffsetInQueue:{}\",\n                currQueueOffset, cq.getMinOffsetInQueue());\n            currQueueOffset = cq.getMinOffsetInQueue();\n        } else if (cq != null && currQueueOffset > cq.getMaxOffsetInQueue()) {\n            LOGGER.warn(\"Timer currQueueOffset:{} is larger than maxOffsetInQueue:{}\",\n                currQueueOffset, cq.getMaxOffsetInQueue());\n            currQueueOffset = cq.getMaxOffsetInQueue();\n        }\n\n        //check timer wheel\n        currReadTimeMs = timerCheckpoint.getLastReadTimeMs();\n        long nextReadTimeMs = formatTimeMs(\n            System.currentTimeMillis()) - (long) slotsTotal * precisionMs + (long) TIMER_BLANK_SLOTS * precisionMs;\n        if (currReadTimeMs < nextReadTimeMs) {\n            currReadTimeMs = nextReadTimeMs;\n        }\n        //the timer wheel may contain physical offset bigger than timerLog\n        //This will only happen when the timerLog is damaged\n        //hard to test\n        long minFirst = timerWheel.checkPhyPos(currReadTimeMs, processOffset);\n        if (debug) {\n            minFirst = 0;\n        }\n        if (minFirst < processOffset) {\n            LOGGER.warn(\"Timer recheck because of minFirst:{} processOffset:{}\", minFirst, processOffset);\n            recoverAndRevise(minFirst, false);\n        }\n        LOGGER.info(\"Timer recover ok currReadTimerMs:{} currQueueOffset:{} checkQueueOffset:{} processOffset:{}\",\n            currReadTimeMs, currQueueOffset, timerCheckpoint.getLastTimerQueueOffset(), processOffset);\n\n        commitReadTimeMs = currReadTimeMs;\n        commitQueueOffset = currQueueOffset;\n\n        prepareTimerCheckPoint();\n    }\n\n    public long reviseQueueOffset(long processOffset) {\n        SelectMappedBufferResult selectRes = timerLog.getTimerMessage(processOffset - (TimerLog.UNIT_SIZE - TimerLog.UNIT_PRE_SIZE_FOR_MSG));\n        if (null == selectRes) {\n            return -1;\n        }\n        try {\n            long offsetPy = selectRes.getByteBuffer().getLong();\n            int sizePy = selectRes.getByteBuffer().getInt();\n            MessageExt messageExt = getMessageByCommitOffset(offsetPy, sizePy);\n            if (null == messageExt) {\n                return -1;\n            }\n\n            // check offset in msg is equal to offset of cq.\n            // if not, use cq offset.\n            long msgQueueOffset = messageExt.getQueueOffset();\n            int queueId = messageExt.getQueueId();\n            ConsumeQueueInterface cq = this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId);\n            if (null == cq) {\n                return msgQueueOffset;\n            }\n            long cqOffset = msgQueueOffset;\n            long tmpOffset = msgQueueOffset;\n            int maxCount = 20000;\n            while (maxCount-- > 0) {\n                if (tmpOffset < 0) {\n                    LOGGER.warn(\"reviseQueueOffset check cq offset fail, msg in cq is not found.{}, {}\",\n                        offsetPy, sizePy);\n                    break;\n                }\n                ReferredIterator<CqUnit> iterator = null;\n                try {\n                    iterator = cq.iterateFrom(tmpOffset);\n                    CqUnit cqUnit = null;\n                    if (null == iterator || (cqUnit = iterator.next()) == null) {\n                        // offset in msg may be greater than offset of cq.\n                        tmpOffset -= 1;\n                        continue;\n                    }\n\n                    long offsetPyTemp = cqUnit.getPos();\n                    int sizePyTemp = cqUnit.getSize();\n                    if (offsetPyTemp == offsetPy && sizePyTemp == sizePy) {\n                        LOGGER.info(\"reviseQueueOffset check cq offset ok. {}, {}, {}\",\n                            tmpOffset, offsetPyTemp, sizePyTemp);\n                        cqOffset = tmpOffset;\n                        break;\n                    }\n                    tmpOffset -= 1;\n                } catch (Throwable e) {\n                    LOGGER.error(\"reviseQueueOffset check cq offset error.\", e);\n                } finally {\n                    if (iterator != null) {\n                        iterator.release();\n                    }\n                }\n            }\n\n            return cqOffset;\n        } finally {\n            selectRes.release();\n        }\n    }\n\n    //recover timerLog and revise timerWheel\n    //return process offset\n    private long recoverAndRevise(long beginOffset, boolean checkTimerLog) {\n        LOGGER.info(\"Begin to recover timerLog offset:{} check:{}\", beginOffset, checkTimerLog);\n        MappedFile lastFile = timerLog.getMappedFileQueue().getLastMappedFile();\n        if (null == lastFile) {\n            return 0;\n        }\n\n        List<MappedFile> mappedFiles = timerLog.getMappedFileQueue().getMappedFiles();\n        int index = mappedFiles.size() - 1;\n        for (; index >= 0; index--) {\n            MappedFile mappedFile = mappedFiles.get(index);\n            if (beginOffset >= mappedFile.getFileFromOffset()) {\n                break;\n            }\n        }\n        if (index < 0) {\n            index = 0;\n        }\n        long checkOffset = mappedFiles.get(index).getFileFromOffset();\n        for (; index < mappedFiles.size(); index++) {\n            MappedFile mappedFile = mappedFiles.get(index);\n            SelectMappedBufferResult sbr = mappedFile.selectMappedBuffer(0, checkTimerLog ? mappedFiles.get(index).getFileSize() : mappedFile.getReadPosition());\n            ByteBuffer bf = sbr.getByteBuffer();\n            int position = 0;\n            boolean stopCheck = false;\n            for (; position < sbr.getSize(); position += TimerLog.UNIT_SIZE) {\n                try {\n                    bf.position(position);\n                    int size = bf.getInt();//size\n                    bf.getLong();//prev pos\n                    int magic = bf.getInt();\n                    if (magic == TimerLog.BLANK_MAGIC_CODE) {\n                        break;\n                    }\n                    if (checkTimerLog && (!isMagicOK(magic) || TimerLog.UNIT_SIZE != size)) {\n                        stopCheck = true;\n                        break;\n                    }\n                    long delayTime = bf.getLong() + bf.getInt();\n                    if (TimerLog.UNIT_SIZE == size && isMagicOK(magic)) {\n                        timerWheel.reviseSlot(delayTime, TimerWheel.IGNORE, sbr.getStartOffset() + position, true);\n                    }\n                } catch (Exception e) {\n                    LOGGER.error(\"Recover timerLog error\", e);\n                    stopCheck = true;\n                    break;\n                }\n            }\n            sbr.release();\n            checkOffset = mappedFiles.get(index).getFileFromOffset() + position;\n            if (stopCheck) {\n                break;\n            }\n        }\n        if (checkTimerLog) {\n            timerLog.getMappedFileQueue().truncateDirtyFiles(checkOffset);\n        }\n        return checkOffset;\n    }\n\n    public static boolean isMagicOK(int magic) {\n        return (magic | 0xF) == 0xF;\n    }\n\n    public void start() {\n        this.shouldStartTime = storeConfig.getDisappearTimeAfterStart() + System.currentTimeMillis();\n        maybeMoveWriteTime();\n        enqueueGetService.start();\n        enqueuePutService.start();\n        dequeueWarmService.start();\n        dequeueGetService.start();\n        for (int i = 0; i < dequeueGetMessageServices.length; i++) {\n            dequeueGetMessageServices[i].start();\n        }\n        for (int i = 0; i < dequeuePutMessageServices.length; i++) {\n            dequeuePutMessageServices[i].start();\n        }\n        timerFlushService.start();\n\n        scheduler.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    long minPy = messageStore.getMinPhyOffset();\n                    int checkOffset = timerLog.getOffsetForLastUnit();\n                    timerLog.getMappedFileQueue()\n                        .deleteExpiredFileByOffsetForTimerLog(minPy, checkOffset, TimerLog.UNIT_SIZE);\n                } catch (Exception e) {\n                    LOGGER.error(\"Error in cleaning timerLog\", e);\n                }\n            }\n        }, 30, 30, TimeUnit.SECONDS);\n\n        scheduler.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    if (storeConfig.isTimerEnableCheckMetrics()) {\n                        String when = storeConfig.getTimerCheckMetricsWhen();\n                        if (!UtilAll.isItTimeToDo(when)) {\n                            return;\n                        }\n                        long curr = System.currentTimeMillis();\n                        if (curr - lastTimeOfCheckMetrics > 70 * 60 * 1000) {\n                            lastTimeOfCheckMetrics = curr;\n                            checkAndReviseMetrics();\n                            LOGGER.info(\"[CheckAndReviseMetrics]Timer do check timer metrics cost {} ms\",\n                                System.currentTimeMillis() - curr);\n                        }\n                    }\n                } catch (Exception e) {\n                    LOGGER.error(\"Error in cleaning timerLog\", e);\n                }\n            }\n        }, 45, 45, TimeUnit.MINUTES);\n\n        state = RUNNING;\n        LOGGER.info(\"Timer start ok currReadTimerMs:[{}] queueOffset:[{}]\", new Timestamp(currReadTimeMs), currQueueOffset);\n    }\n\n    public void start(boolean shouldRunningDequeue) {\n        this.shouldRunningDequeue = shouldRunningDequeue;\n        this.start();\n    }\n\n    public void shutdown() {\n        if (SHUTDOWN == state) {\n            return;\n        }\n        state = SHUTDOWN;\n\n        if (this.scheduler != null) {\n            List<Runnable> remainingTasks = this.scheduler.shutdownNow();\n            if (!remainingTasks.isEmpty()) {\n                LOGGER.info(\"Timer scheduler shutdown interrupted {} tasks\", remainingTasks.size());\n            }\n\n            try {\n                if (!this.scheduler.awaitTermination(30, TimeUnit.SECONDS)) {\n                    LOGGER.warn(\"Timer scheduler did not terminate gracefully\");\n                }\n            } catch (InterruptedException e) {\n                LOGGER.warn(\"Interrupted while waiting for scheduler termination\", e);\n            }\n        }\n\n        //first save checkpoint\n        if (timerCheckpoint != null) {\n            prepareTimerCheckPoint();\n        }\n\n        if (timerFlushService != null) {\n            timerFlushService.shutdown();\n        }\n\n        if (timerCheckpoint != null) {\n            timerCheckpoint.shutdown();\n        }\n\n        if (enqueuePutQueue != null) {\n            enqueuePutQueue.clear(); //avoid blocking\n        }\n\n        if (dequeueGetQueue != null) {\n            dequeueGetQueue.clear(); //avoid blocking\n        }\n\n        if (dequeuePutQueue != null) {\n            dequeuePutQueue.clear(); //avoid blocking\n        }\n\n        if (enqueueGetService != null) {\n            enqueueGetService.shutdown();\n        }\n\n        if (enqueuePutService != null) {\n            enqueuePutService.shutdown();\n        }\n\n        if (dequeueWarmService != null) {\n            dequeueWarmService.shutdown();\n        }\n\n        if (dequeueGetService != null) {\n            dequeueGetService.shutdown();\n        }\n\n        if (dequeueGetMessageServices != null) {\n            for (TimerDequeueGetMessageService dequeueGetMessageServices : dequeueGetMessageServices) {\n                if (dequeueGetMessageServices != null) {\n                    dequeueGetMessageServices.shutdown();\n                }\n            }\n        }\n\n        if (dequeuePutMessageServices != null) {\n            for (TimerDequeuePutMessageService dequeuePutMessageServices : dequeuePutMessageServices) {\n                if (dequeuePutMessageServices != null) {\n                    dequeuePutMessageServices.shutdown();\n                }\n            }\n        }\n\n        if (timerWheel != null) {\n            timerWheel.shutdown(false);\n        }\n\n        if (timerLog != null) {\n            timerLog.shutdown();\n        }\n\n        if (this.bufferLocal != null) {\n            UtilAll.cleanBuffer(this.bufferLocal.get());\n            this.bufferLocal.remove();\n        }\n    }\n\n    protected void maybeMoveWriteTime() {\n        if (currWriteTimeMs < formatTimeMs(System.currentTimeMillis())) {\n            currWriteTimeMs = formatTimeMs(System.currentTimeMillis());\n        }\n    }\n\n    private void moveReadTime() {\n        currReadTimeMs = currReadTimeMs + precisionMs;\n        commitReadTimeMs = currReadTimeMs;\n    }\n\n    private boolean isRunning() {\n        return RUNNING == state;\n    }\n\n    private void checkBrokerRole() {\n        BrokerRole currRole = storeConfig.getBrokerRole();\n        if (lastBrokerRole != currRole) {\n            synchronized (lastBrokerRole) {\n                LOGGER.info(\"Broker role change from {} to {}\", lastBrokerRole, currRole);\n                //if change to master, do something\n                if (BrokerRole.SLAVE != currRole) {\n                    currQueueOffset = Math.min(currQueueOffset, timerCheckpoint.getMasterTimerQueueOffset());\n                    commitQueueOffset = currQueueOffset;\n                    prepareTimerCheckPoint();\n                    try {\n                        timerCheckpoint.flush();\n                    } catch (Throwable e) {\n                        LOGGER.error(\"Error in flush timerCheckpoint\", e);\n                    }\n                    currReadTimeMs = timerCheckpoint.getLastReadTimeMs();\n                    commitReadTimeMs = currReadTimeMs;\n                }\n                //if change to slave, just let it go\n                lastBrokerRole = currRole;\n            }\n        }\n    }\n\n    private boolean isRunningEnqueue() {\n        checkBrokerRole();\n        if (!shouldRunningDequeue && !isMaster() && currQueueOffset >= timerCheckpoint.getMasterTimerQueueOffset()) {\n            return false;\n        }\n\n        return isRunning();\n    }\n\n    private boolean isRunningDequeue() {\n        if (!this.shouldRunningDequeue) {\n            syncLastReadTimeMs();\n            return false;\n        }\n        return isRunning();\n    }\n\n    public void syncLastReadTimeMs() {\n        currReadTimeMs = timerCheckpoint.getLastReadTimeMs();\n        commitReadTimeMs = currReadTimeMs;\n    }\n\n    public void setShouldRunningDequeue(final boolean shouldRunningDequeue) {\n        this.shouldRunningDequeue = shouldRunningDequeue;\n    }\n\n    public boolean isShouldRunningDequeue() {\n        return shouldRunningDequeue;\n    }\n\n    public void addMetric(MessageExt msg, int value) {\n        try {\n            if (null == msg || null == msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC)) {\n                return;\n            }\n            if (msg.getProperty(TIMER_ENQUEUE_MS) != null\n                && NumberUtils.toLong(msg.getProperty(TIMER_ENQUEUE_MS)) == Long.MAX_VALUE) {\n                return;\n            }\n            // pass msg into addAndGet, for further more judgement extension.\n            timerMetrics.addAndGet(msg, value);\n        } catch (Throwable t) {\n            if (frequency.incrementAndGet() % 1000 == 0) {\n                LOGGER.error(\"error in adding metric\", t);\n            }\n        }\n\n    }\n\n    public void holdMomentForUnknownError(long ms) {\n        try {\n            Thread.sleep(ms);\n        } catch (Exception ignored) {\n\n        }\n    }\n\n    public void holdMomentForUnknownError() {\n        holdMomentForUnknownError(50);\n    }\n\n    public boolean enqueue(int queueId) {\n        if (storeConfig.isTimerStopEnqueue()) {\n            return false;\n        }\n        if (!isRunningEnqueue()) {\n            return false;\n        }\n        ConsumeQueueInterface cq = this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId);\n        if (null == cq) {\n            return false;\n        }\n        if (currQueueOffset < cq.getMinOffsetInQueue()) {\n            LOGGER.warn(\"Timer currQueueOffset:{} is smaller than minOffsetInQueue:{}\",\n                currQueueOffset, cq.getMinOffsetInQueue());\n            currQueueOffset = cq.getMinOffsetInQueue();\n        }\n        long offset = currQueueOffset;\n        ReferredIterator<CqUnit> iterator = null;\n        try {\n            iterator = cq.iterateFrom(offset);\n            if (null == iterator) {\n                return false;\n            }\n\n            int i = 0;\n            while (iterator.hasNext()) {\n                i++;\n                perfCounterTicks.startTick(\"enqueue_get\");\n                try {\n                    CqUnit cqUnit = iterator.next();\n                    long offsetPy = cqUnit.getPos();\n                    int sizePy = cqUnit.getSize();\n                    cqUnit.getTagsCode(); //tags code\n                    MessageExt msgExt = getMessageByCommitOffset(offsetPy, sizePy);\n                    if (null == msgExt) {\n                        perfCounterTicks.getCounter(\"enqueue_get_miss\");\n                    } else {\n                        lastEnqueueButExpiredTime = System.currentTimeMillis();\n                        lastEnqueueButExpiredStoreTime = msgExt.getStoreTimestamp();\n                        long delayedTime = Long.parseLong(msgExt.getProperty(TIMER_OUT_MS));\n                        // use CQ offset, not offset in Message\n                        msgExt.setQueueOffset(offset + i);\n                        TimerRequest timerRequest = new TimerRequest(offsetPy, sizePy, delayedTime, System.currentTimeMillis(), MAGIC_DEFAULT, msgExt);\n                        // System.out.printf(\"build enqueue request, %s%n\", timerRequest);\n                        while (!enqueuePutQueue.offer(timerRequest, 3, TimeUnit.SECONDS)) {\n                            if (!isRunningEnqueue()) {\n                                return false;\n                            }\n                        }\n                        // Record timer message set latency\n                        StoreMetricsManager metricsManager = messageStore.getStoreMetricsManager();\n                        if (metricsManager instanceof DefaultStoreMetricsManager) {\n                            DefaultStoreMetricsManager defaultMetricsManager = (DefaultStoreMetricsManager) metricsManager;\n                            Attributes attributes = defaultMetricsManager.newAttributesBuilder()\n                                .put(DefaultStoreMetricsConstant.LABEL_TOPIC, msgExt.getProperty(MessageConst.PROPERTY_REAL_TOPIC)).build();\n                            defaultMetricsManager.getTimerMessageSetLatency().record((delayedTime - msgExt.getBornTimestamp()) / 1000, attributes);\n                        }\n                    }\n                } catch (Exception e) {\n                    // here may cause the message loss\n                    if (storeConfig.isTimerSkipUnknownError()) {\n                        LOGGER.warn(\"Unknown error in skipped in enqueuing\", e);\n                    } else {\n                        holdMomentForUnknownError();\n                        throw e;\n                    }\n                } finally {\n                    perfCounterTicks.endTick(\"enqueue_get\");\n                }\n                // if broker role changes, ignore last enqueue\n                if (!isRunningEnqueue()) {\n                    return false;\n                }\n                currQueueOffset = offset + i;\n            }\n            currQueueOffset = offset + i;\n            return i > 0;\n        } catch (Exception e) {\n            LOGGER.error(\"Unknown exception in enqueuing\", e);\n        } finally {\n            if (iterator != null) {\n                iterator.release();\n            }\n        }\n        return false;\n    }\n\n    public boolean doEnqueue(long offsetPy, int sizePy, long delayedTime, MessageExt messageExt, boolean isFromTimeline) {\n        LOGGER.debug(\"Do enqueue [{}] [{}]\", new Timestamp(delayedTime), messageExt);\n        //copy the value first, avoid concurrent problem\n        long tmpWriteTimeMs = currWriteTimeMs;\n        boolean needRoll = delayedTime - tmpWriteTimeMs >= (long) timerRollWindowSlots * precisionMs;\n        int magic = MAGIC_DEFAULT;\n        if (needRoll) {\n            magic = magic | MAGIC_ROLL;\n            if (delayedTime - tmpWriteTimeMs - (long) timerRollWindowSlots * precisionMs < (long) timerRollWindowSlots / 3 * precisionMs) {\n                //give enough time to next roll\n                delayedTime = tmpWriteTimeMs + (long) (timerRollWindowSlots / 2) * precisionMs;\n            } else {\n                delayedTime = tmpWriteTimeMs + (long) timerRollWindowSlots * precisionMs;\n            }\n        }\n        boolean isDelete = messageExt.getProperty(TIMER_DELETE_UNIQUE_KEY) != null;\n        if (isDelete) {\n            magic = magic | MAGIC_DELETE;\n            if (!isFromTimeline) {\n                recallToTimeline(delayedTime, offsetPy, sizePy, messageExt);\n            }\n        }\n        String realTopic = messageExt.getProperty(MessageConst.PROPERTY_REAL_TOPIC);\n        Slot slot = timerWheel.getSlot(delayedTime);\n        ByteBuffer tmpBuffer = timerLogBuffer;\n        tmpBuffer.clear();\n        tmpBuffer.putInt(TimerLog.UNIT_SIZE); //size\n        tmpBuffer.putLong(slot.lastPos); //prev pos\n        tmpBuffer.putInt(magic); //magic\n        tmpBuffer.putLong(tmpWriteTimeMs); //currWriteTime\n        tmpBuffer.putInt((int) (delayedTime - tmpWriteTimeMs)); //delayTime\n        tmpBuffer.putLong(offsetPy); //offset\n        tmpBuffer.putInt(sizePy); //size\n        tmpBuffer.putInt(hashTopicForMetrics(realTopic)); //hashcode of real topic\n        tmpBuffer.putLong(0); //reserved value, just set to 0 now\n        long ret = timerLog.append(tmpBuffer.array(), 0, TimerLog.UNIT_SIZE);\n        if (-1 != ret) {\n            // If it's a delete message, then slot's total num -1\n            // TODO: check if the delete msg is in the same slot with \"the msg to be deleted\".\n            timerWheel.putSlot(delayedTime, slot.firstPos == -1 ? ret : slot.firstPos, ret,\n                isDelete ? slot.num - 1 : slot.num + 1, slot.magic);\n            addMetric(messageExt, isDelete ? -1 : 1);\n        }\n        return -1 != ret;\n    }\n\n    @SuppressWarnings(\"NonAtomicOperationOnVolatileField\")\n    public int warmDequeue() {\n        if (!isRunningDequeue()) {\n            return -1;\n        }\n        if (!storeConfig.isTimerWarmEnable()) {\n            return -1;\n        }\n        if (preReadTimeMs <= currReadTimeMs) {\n            preReadTimeMs = currReadTimeMs + precisionMs;\n        }\n        if (preReadTimeMs >= currWriteTimeMs) {\n            return -1;\n        }\n        if (preReadTimeMs >= currReadTimeMs + 3L * precisionMs) {\n            return -1;\n        }\n        Slot slot = timerWheel.getSlot(preReadTimeMs);\n        if (-1 == slot.timeMs) {\n            preReadTimeMs = preReadTimeMs + precisionMs;\n            return 0;\n        }\n        long currOffsetPy = slot.lastPos;\n        LinkedList<SelectMappedBufferResult> sbrs = new LinkedList<>();\n        SelectMappedBufferResult timeSbr = null;\n        SelectMappedBufferResult msgSbr = null;\n        try {\n            //read the msg one by one\n            while (currOffsetPy != -1) {\n                if (!isRunning()) {\n                    break;\n                }\n                perfCounterTicks.startTick(\"warm_dequeue\");\n                if (null == timeSbr || timeSbr.getStartOffset() > currOffsetPy) {\n                    timeSbr = timerLog.getWholeBuffer(currOffsetPy);\n                    if (null != timeSbr) {\n                        sbrs.add(timeSbr);\n                    }\n                }\n                if (null == timeSbr) {\n                    break;\n                }\n                long prevPos = -1;\n                try {\n                    int position = (int) (currOffsetPy % timerLogFileSize);\n                    timeSbr.getByteBuffer().position(position);\n                    timeSbr.getByteBuffer().getInt(); //size\n                    prevPos = timeSbr.getByteBuffer().getLong();\n                    timeSbr.getByteBuffer().position(position + TimerLog.UNIT_PRE_SIZE_FOR_MSG);\n                    long offsetPy = timeSbr.getByteBuffer().getLong();\n                    int sizePy = timeSbr.getByteBuffer().getInt();\n                    if (null == msgSbr || msgSbr.getStartOffset() > offsetPy) {\n                        msgSbr = messageStore.getCommitLogData(offsetPy - offsetPy % commitLogFileSize);\n                        if (null != msgSbr) {\n                            sbrs.add(msgSbr);\n                        }\n                    }\n                    if (null != msgSbr) {\n                        ByteBuffer bf = msgSbr.getByteBuffer();\n                        int firstPos = (int) (offsetPy % commitLogFileSize);\n                        for (int pos = firstPos; pos < firstPos + sizePy; pos += 4096) {\n                            bf.position(pos);\n                            bf.get();\n                        }\n                    }\n                } catch (Exception e) {\n                    LOGGER.error(\"Unexpected error in warm\", e);\n                } finally {\n                    currOffsetPy = prevPos;\n                    perfCounterTicks.endTick(\"warm_dequeue\");\n                }\n            }\n            for (SelectMappedBufferResult sbr : sbrs) {\n                if (null != sbr) {\n                    sbr.release();\n                }\n            }\n        } finally {\n            preReadTimeMs = preReadTimeMs + precisionMs;\n        }\n        return 1;\n    }\n\n    public boolean checkStateForPutMessages(int state) {\n        for (AbstractStateService service : dequeuePutMessageServices) {\n            if (!service.isState(state)) {\n                return false;\n            }\n        }\n        return true;\n    }\n\n    public boolean checkStateForGetMessages(int state) {\n        for (AbstractStateService service : dequeueGetMessageServices) {\n            if (!service.isState(state)) {\n                return false;\n            }\n        }\n        return true;\n    }\n\n    public void checkDequeueLatch(CountDownLatch latch, long delayedTime) throws Exception {\n        if (latch.await(1, TimeUnit.SECONDS)) {\n            return;\n        }\n        int checkNum = 0;\n        while (true) {\n            if (!isRunningDequeue()) {\n                LOGGER.info(\"Not Running dequeue, skip checkDequeueLatch for delayedTime:{}\", delayedTime);\n                break;\n            }\n\n            if (dequeuePutQueue.size() > 0\n                || !checkStateForGetMessages(AbstractStateService.WAITING)\n                || !checkStateForPutMessages(AbstractStateService.WAITING)) {\n                //let it go\n            } else {\n                checkNum++;\n                if (checkNum >= 2) {\n                    break;\n                }\n            }\n            if (latch.await(1, TimeUnit.SECONDS)) {\n                break;\n            }\n        }\n        if (!latch.await(1, TimeUnit.SECONDS)) {\n            LOGGER.warn(\"Check latch failed delayedTime:{}\", delayedTime);\n        }\n    }\n\n    public int dequeue() throws Exception {\n        if (storeConfig.isTimerStopDequeue()) {\n            return -1;\n        }\n        if (!isRunningDequeue()) {\n            return -1;\n        }\n        if (currReadTimeMs >= currWriteTimeMs) {\n            return -1;\n        }\n\n        Slot slot = timerWheel.getSlot(currReadTimeMs);\n        if (-1 == slot.timeMs) {\n            moveReadTime();\n            return 0;\n        }\n        try {\n            //clear the flag\n            dequeueStatusChangeFlag = false;\n\n            long currOffsetPy = slot.lastPos;\n            Set<String> deleteUniqKeys = new ConcurrentSkipListSet<>();\n            LinkedList<TimerRequest> normalMsgStack = new LinkedList<>();\n            LinkedList<TimerRequest> deleteMsgStack = new LinkedList<>();\n            LinkedList<SelectMappedBufferResult> sbrs = new LinkedList<>();\n            SelectMappedBufferResult timeSbr = null;\n            //read the timer log one by one\n            while (currOffsetPy != -1) {\n                perfCounterTicks.startTick(\"dequeue_read_timerlog\");\n                if (null == timeSbr || timeSbr.getStartOffset() > currOffsetPy) {\n                    timeSbr = timerLog.getWholeBuffer(currOffsetPy);\n                    if (null != timeSbr) {\n                        sbrs.add(timeSbr);\n                    }\n                }\n                if (null == timeSbr) {\n                    break;\n                }\n                long prevPos = -1;\n                try {\n                    int position = (int) (currOffsetPy % timerLogFileSize);\n                    timeSbr.getByteBuffer().position(position);\n                    timeSbr.getByteBuffer().getInt(); //size\n                    prevPos = timeSbr.getByteBuffer().getLong();\n                    int magic = timeSbr.getByteBuffer().getInt();\n                    long enqueueTime = timeSbr.getByteBuffer().getLong();\n                    long delayedTime = timeSbr.getByteBuffer().getInt() + enqueueTime;\n                    long offsetPy = timeSbr.getByteBuffer().getLong();\n                    int sizePy = timeSbr.getByteBuffer().getInt();\n                    TimerRequest timerRequest = new TimerRequest(offsetPy, sizePy, delayedTime, enqueueTime, magic);\n                    timerRequest.setDeleteList(deleteUniqKeys);\n                    if (needDelete(magic) && !needRoll(magic)) {\n                        deleteMsgStack.add(timerRequest);\n                    } else {\n                        normalMsgStack.addFirst(timerRequest);\n                    }\n                } catch (Exception e) {\n                    LOGGER.error(\"Error in dequeue_read_timerlog\", e);\n                } finally {\n                    currOffsetPy = prevPos;\n                    perfCounterTicks.endTick(\"dequeue_read_timerlog\");\n                }\n            }\n            if (deleteMsgStack.size() == 0 && normalMsgStack.size() == 0) {\n                LOGGER.warn(\"dequeue time:{} but read nothing from timerLog\", currReadTimeMs);\n            }\n            for (SelectMappedBufferResult sbr : sbrs) {\n                if (null != sbr) {\n                    sbr.release();\n                }\n            }\n            if (!isRunningDequeue()) {\n                return -1;\n            }\n            CountDownLatch deleteLatch = new CountDownLatch(deleteMsgStack.size());\n            //read the delete msg: the msg used to mark another msg is deleted\n            for (List<TimerRequest> deleteList : splitIntoLists(deleteMsgStack)) {\n                for (TimerRequest tr : deleteList) {\n                    tr.setLatch(deleteLatch);\n                }\n                dequeueGetQueue.put(deleteList);\n            }\n            //do we need to use loop with tryAcquire\n            checkDequeueLatch(deleteLatch, currReadTimeMs);\n\n            CountDownLatch normalLatch = new CountDownLatch(normalMsgStack.size());\n            //read the normal msg\n            for (List<TimerRequest> normalList : splitIntoLists(normalMsgStack)) {\n                for (TimerRequest tr : normalList) {\n                    tr.setLatch(normalLatch);\n                }\n                dequeueGetQueue.put(normalList);\n            }\n            checkDequeueLatch(normalLatch, currReadTimeMs);\n            // if master -> slave -> master, then the read time move forward, and messages will be lossed\n            if (dequeueStatusChangeFlag) {\n                return -1;\n            }\n            if (!isRunningDequeue()) {\n                return -1;\n            }\n            moveReadTime();\n        } catch (Throwable t) {\n            LOGGER.error(\"Unknown error in dequeue process\", t);\n            if (storeConfig.isTimerSkipUnknownError()) {\n                moveReadTime();\n            }\n        }\n        return 1;\n    }\n\n    private List<List<TimerRequest>> splitIntoLists(List<TimerRequest> origin) {\n        //this method assume that the origin is not null;\n        List<List<TimerRequest>> lists = new LinkedList<>();\n        if (origin.size() < 100) {\n            lists.add(origin);\n            return lists;\n        }\n        List<TimerRequest> currList = null;\n        int fileIndexPy = -1;\n        int msgIndex = 0;\n        for (TimerRequest tr : origin) {\n            if (fileIndexPy != tr.getOffsetPy() / commitLogFileSize) {\n                msgIndex = 0;\n                if (null != currList && currList.size() > 0) {\n                    lists.add(currList);\n                }\n                currList = new LinkedList<>();\n                currList.add(tr);\n                fileIndexPy = (int) (tr.getOffsetPy() / commitLogFileSize);\n            } else {\n                currList.add(tr);\n                if (++msgIndex % 2000 == 0) {\n                    lists.add(currList);\n                    currList = new ArrayList<>();\n                }\n            }\n        }\n        if (null != currList && currList.size() > 0) {\n            lists.add(currList);\n        }\n        return lists;\n    }\n\n    private MessageExt getMessageByCommitOffset(long offsetPy, int sizePy) {\n        for (int i = 0; i < 3; i++) {\n            MessageExt msgExt = StoreUtil.getMessage(offsetPy, sizePy, messageStore, bufferLocal.get());\n            if (null == msgExt) {\n                LOGGER.warn(\"Fail to read msg from commitLog offsetPy:{} sizePy:{}\", offsetPy, sizePy);\n            } else {\n                return msgExt;\n            }\n        }\n        return null;\n    }\n\n    public MessageExtBrokerInner convert(MessageExt messageExt, long enqueueTime, boolean needRoll) {\n        if (enqueueTime != -1) {\n            MessageAccessor.putProperty(messageExt, TIMER_ENQUEUE_MS, enqueueTime + \"\");\n        }\n        if (needRoll) {\n            if (messageExt.getProperty(TIMER_ROLL_TIMES) != null) {\n                MessageAccessor.putProperty(messageExt, TIMER_ROLL_TIMES, Integer.parseInt(messageExt.getProperty(TIMER_ROLL_TIMES)) + 1 + \"\");\n            } else {\n                MessageAccessor.putProperty(messageExt, TIMER_ROLL_TIMES, 1 + \"\");\n            }\n        }\n        MessageAccessor.putProperty(messageExt, TIMER_DEQUEUE_MS, System.currentTimeMillis() + \"\");\n        MessageExtBrokerInner message = convertMessage(messageExt, needRoll);\n        return message;\n    }\n\n    //0 succ; 1 fail, need retry; 2 fail, do not retry;\n    public int doPut(MessageExtBrokerInner message, boolean roll) throws Exception {\n\n        if (!roll && null != message.getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY)) {\n            LOGGER.warn(\"Trying do put delete timer msg:[{}] roll:[{}]\", message, roll);\n            return PUT_NO_RETRY;\n        }\n\n        PutMessageResult putMessageResult = null;\n        if (escapeBridgeHook != null) {\n            putMessageResult = escapeBridgeHook.apply(message);\n        } else {\n            putMessageResult = messageStore.putMessage(message);\n        }\n\n        if (putMessageResult != null && putMessageResult.getPutMessageStatus() != null) {\n            switch (putMessageResult.getPutMessageStatus()) {\n                case PUT_OK:\n                    if (brokerStatsManager != null) {\n                        brokerStatsManager.incTopicPutNums(message.getTopic(), 1, 1);\n                        if (putMessageResult.getAppendMessageResult() != null) {\n                            brokerStatsManager.incTopicPutSize(message.getTopic(), putMessageResult.getAppendMessageResult().getWroteBytes());\n                        }\n                        brokerStatsManager.incBrokerPutNums(message.getTopic(), 1);\n                    }\n                    return PUT_OK;\n\n                case MESSAGE_ILLEGAL:\n                case PROPERTIES_SIZE_EXCEEDED:\n                case WHEEL_TIMER_NOT_ENABLE:\n                case WHEEL_TIMER_MSG_ILLEGAL:\n                    return PUT_NO_RETRY;\n\n                case SERVICE_NOT_AVAILABLE:\n                case FLUSH_DISK_TIMEOUT:\n                case FLUSH_SLAVE_TIMEOUT:\n                case OS_PAGE_CACHE_BUSY:\n                case CREATE_MAPPED_FILE_FAILED:\n                case SLAVE_NOT_AVAILABLE:\n                    return PUT_NEED_RETRY;\n\n                case UNKNOWN_ERROR:\n                default:\n                    if (storeConfig.isTimerSkipUnknownError()) {\n                        LOGGER.warn(\"Skipping message due to unknown error, msg: {}\", message);\n                        return PUT_NO_RETRY;\n                    } else {\n                        holdMomentForUnknownError();\n                        return PUT_NEED_RETRY;\n                    }\n            }\n        }\n        return PUT_NEED_RETRY;\n    }\n\n    public MessageExtBrokerInner convertMessage(MessageExt msgExt, boolean needRoll) {\n        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n        msgInner.setBody(msgExt.getBody());\n        msgInner.setFlag(msgExt.getFlag());\n        MessageAccessor.setProperties(msgInner, MessageAccessor.deepCopyProperties(msgExt.getProperties()));\n        TopicFilterType topicFilterType = MessageExt.parseTopicFilterType(msgInner.getSysFlag());\n        long tagsCodeValue =\n            MessageExtBrokerInner.tagsString2tagsCode(topicFilterType, msgInner.getTags());\n        msgInner.setTagsCode(tagsCodeValue);\n        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties()));\n\n        msgInner.setSysFlag(msgExt.getSysFlag());\n        msgInner.setBornTimestamp(msgExt.getBornTimestamp());\n        msgInner.setBornHost(msgExt.getBornHost());\n        msgInner.setStoreHost(msgExt.getStoreHost());\n        msgInner.setReconsumeTimes(msgExt.getReconsumeTimes());\n\n        msgInner.setWaitStoreMsgOK(false);\n\n        if (needRoll) {\n            msgInner.setTopic(msgExt.getTopic());\n            msgInner.setQueueId(msgExt.getQueueId());\n        } else {\n            msgInner.setTopic(msgInner.getProperty(MessageConst.PROPERTY_REAL_TOPIC));\n            msgInner.setQueueId(Integer.parseInt(msgInner.getProperty(MessageConst.PROPERTY_REAL_QUEUE_ID)));\n            MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_REAL_TOPIC);\n            MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_REAL_QUEUE_ID);\n        }\n        return msgInner;\n    }\n\n    protected String getRealTopic(MessageExt msgExt) {\n        if (msgExt == null) {\n            return null;\n        }\n        return msgExt.getProperty(MessageConst.PROPERTY_REAL_TOPIC);\n    }\n\n    private long formatTimeMs(long timeMs) {\n        return timeMs / precisionMs * precisionMs;\n    }\n\n    public int hashTopicForMetrics(String topic) {\n        return null == topic ? 0 : topic.hashCode();\n    }\n\n    public void checkAndReviseMetrics() {\n        Map<String, TimerMetrics.Metric> smallOnes = new HashMap<>();\n        Map<String, TimerMetrics.Metric> bigOnes = new HashMap<>();\n        Map<Integer, String> smallHashs = new HashMap<>();\n        Set<Integer> smallHashCollisions = new HashSet<>();\n        for (Map.Entry<String, TimerMetrics.Metric> entry : timerMetrics.getTimingCount().entrySet()) {\n            if (entry.getValue().getCount().get() < storeConfig.getTimerMetricSmallThreshold()) {\n                smallOnes.put(entry.getKey(), entry.getValue());\n                int hash = hashTopicForMetrics(entry.getKey());\n                if (smallHashs.containsKey(hash)) {\n                    LOGGER.warn(\"[CheckAndReviseMetrics]Metric hash collision between small-small code:{} small topic:{}{} small topic:{}{}\", hash,\n                        entry.getKey(), entry.getValue(),\n                        smallHashs.get(hash), smallOnes.get(smallHashs.get(hash)));\n                    smallHashCollisions.add(hash);\n                }\n                smallHashs.put(hash, entry.getKey());\n            } else {\n                bigOnes.put(entry.getKey(), entry.getValue());\n            }\n        }\n        //check the hash collision between small ons and big ons\n        for (Map.Entry<String, TimerMetrics.Metric> bjgEntry : bigOnes.entrySet()) {\n            if (smallHashs.containsKey(hashTopicForMetrics(bjgEntry.getKey()))) {\n                Iterator<Map.Entry<String, TimerMetrics.Metric>> smallIt = smallOnes.entrySet().iterator();\n                while (smallIt.hasNext()) {\n                    Map.Entry<String, TimerMetrics.Metric> smallEntry = smallIt.next();\n                    if (hashTopicForMetrics(smallEntry.getKey()) == hashTopicForMetrics(bjgEntry.getKey())) {\n                        LOGGER.warn(\"[CheckAndReviseMetrics]Metric hash collision between small-big code:{} small topic:{}{} big topic:{}{}\", hashTopicForMetrics(smallEntry.getKey()),\n                            smallEntry.getKey(), smallEntry.getValue(),\n                            bjgEntry.getKey(), bjgEntry.getValue());\n                        smallIt.remove();\n                    }\n                }\n            }\n        }\n        //refresh\n        smallHashs.clear();\n        Map<String, TimerMetrics.Metric> newSmallOnes = new HashMap<>();\n        for (String topic : smallOnes.keySet()) {\n            newSmallOnes.put(topic, new TimerMetrics.Metric());\n            smallHashs.put(hashTopicForMetrics(topic), topic);\n        }\n\n        //travel the timer log\n        long readTimeMs = currReadTimeMs;\n        long currOffsetPy = timerWheel.checkPhyPos(readTimeMs, 0);\n        LinkedList<SelectMappedBufferResult> sbrs = new LinkedList<>();\n        boolean hasError = false;\n        try {\n            while (true) {\n                SelectMappedBufferResult timeSbr = timerLog.getWholeBuffer(currOffsetPy);\n                if (timeSbr == null) {\n                    break;\n                } else {\n                    sbrs.add(timeSbr);\n                }\n                ByteBuffer bf = timeSbr.getByteBuffer();\n                for (int position = 0; position < timeSbr.getSize(); position += TimerLog.UNIT_SIZE) {\n                    bf.position(position);\n                    bf.getInt();//size\n                    bf.getLong();//prev pos\n                    int magic = bf.getInt(); //magic\n                    if (magic == TimerLog.BLANK_MAGIC_CODE) {\n                        break;\n                    }\n                    long enqueueTime = bf.getLong();\n                    long delayedTime = bf.getInt() + enqueueTime;\n                    long offsetPy = bf.getLong();\n                    int sizePy = bf.getInt();\n                    int hashCode = bf.getInt();\n                    if (delayedTime < readTimeMs) {\n                        continue;\n                    }\n                    if (!smallHashs.containsKey(hashCode)) {\n                        continue;\n                    }\n                    String topic = null;\n                    if (smallHashCollisions.contains(hashCode)) {\n                        MessageExt messageExt = getMessageByCommitOffset(offsetPy, sizePy);\n                        if (null != messageExt) {\n                            topic = messageExt.getProperty(MessageConst.PROPERTY_REAL_TOPIC);\n                        }\n                    } else {\n                        topic = smallHashs.get(hashCode);\n                    }\n                    if (null != topic && newSmallOnes.containsKey(topic)) {\n                        newSmallOnes.get(topic).getCount().addAndGet(needDelete(magic) ? -1 : 1);\n                    } else {\n                        LOGGER.warn(\"[CheckAndReviseMetrics]Unexpected topic in checking timer metrics topic:{} code:{} offsetPy:{} size:{}\", topic, hashCode, offsetPy, sizePy);\n                    }\n                }\n                if (timeSbr.getSize() < timerLogFileSize) {\n                    break;\n                } else {\n                    currOffsetPy = currOffsetPy + timerLogFileSize;\n                }\n            }\n\n        } catch (Exception e) {\n            hasError = true;\n            LOGGER.error(\"[CheckAndReviseMetrics]Unknown error in checkAndReviseMetrics and abort\", e);\n        } finally {\n            for (SelectMappedBufferResult sbr : sbrs) {\n                if (null != sbr) {\n                    sbr.release();\n                }\n            }\n        }\n\n        if (!hasError) {\n            //update\n            for (String topic : newSmallOnes.keySet()) {\n                LOGGER.info(\"[CheckAndReviseMetrics]Revise metric for topic {} from {} to {}\", topic, smallOnes.get(topic), newSmallOnes.get(topic));\n            }\n            timerMetrics.getTimingCount().putAll(newSmallOnes);\n        }\n\n    }\n\n    public class TimerEnqueueGetService extends ServiceThread {\n\n        @Override\n        public String getServiceName() {\n            return getServiceThreadName() + this.getClass().getSimpleName();\n        }\n\n        @Override\n        public void run() {\n            TimerMessageStore.LOGGER.info(this.getServiceName() + \" service start\");\n            while (!this.isStopped()) {\n                try {\n                    if (storeConfig.isTimerRocksDBEnable() && !storeConfig.isTimerRocksDBStopScan()) {\n                        LOGGER.info(\"now timer use rocksdb to driver, so will not enqueue in timer wheel\");\n                        waitForRunning(10 * 1000L);\n                    } else if (!TimerMessageStore.this.enqueue(0)) {\n                        waitForRunning(100L * precisionMs / 1000);\n                    }\n                } catch (Throwable e) {\n                    TimerMessageStore.LOGGER.error(\"Error occurred in \" + getServiceName(), e);\n                }\n            }\n            TimerMessageStore.LOGGER.info(this.getServiceName() + \" service end\");\n        }\n    }\n\n    public String getServiceThreadName() {\n        String brokerIdentifier = \"\";\n        if (TimerMessageStore.this.messageStore instanceof DefaultMessageStore) {\n            DefaultMessageStore messageStore = (DefaultMessageStore) TimerMessageStore.this.messageStore;\n            if (messageStore.getBrokerConfig().isInBrokerContainer()) {\n                brokerIdentifier = messageStore.getBrokerConfig().getIdentifier();\n            }\n        }\n        return brokerIdentifier;\n    }\n\n    public class TimerEnqueuePutService extends ServiceThread {\n\n        @Override\n        public String getServiceName() {\n            return getServiceThreadName() + this.getClass().getSimpleName();\n        }\n\n        /**\n         * collect the requests\n         */\n        protected List<TimerRequest> fetchTimerRequests() throws InterruptedException {\n            List<TimerRequest> trs = null;\n            TimerRequest firstReq = enqueuePutQueue.poll(10, TimeUnit.MILLISECONDS);\n            if (null != firstReq) {\n                trs = new ArrayList<>(16);\n                trs.add(firstReq);\n                while (true) {\n                    TimerRequest tmpReq = enqueuePutQueue.poll(3, TimeUnit.MILLISECONDS);\n                    if (null == tmpReq) {\n                        break;\n                    }\n                    trs.add(tmpReq);\n                    if (trs.size() > 10) {\n                        break;\n                    }\n                }\n            }\n            return trs;\n        }\n\n        protected void putMessageToTimerWheel(TimerRequest req) {\n            try {\n                perfCounterTicks.startTick(ENQUEUE_PUT);\n                StoreMetricsManager metricsManager = messageStore.getStoreMetricsManager();\n                if (metricsManager instanceof DefaultStoreMetricsManager) {\n                    ((DefaultStoreMetricsManager) metricsManager).incTimerEnqueueCount(getRealTopic(req.getMsg()));\n                }\n                if (shouldRunningDequeue && req.getDelayTime() < currWriteTimeMs) {\n                    req.setEnqueueTime(Long.MAX_VALUE);\n                    dequeuePutQueue.put(req);\n                } else {\n                    boolean doEnqueueRes = doEnqueue(\n                        req.getOffsetPy(), req.getSizePy(), req.getDelayTime(), req.getMsg(), false);\n                    req.idempotentRelease(doEnqueueRes || storeConfig.isTimerSkipUnknownError());\n                }\n                perfCounterTicks.endTick(ENQUEUE_PUT);\n            } catch (Throwable t) {\n                LOGGER.error(\"Unknown error\", t);\n                if (storeConfig.isTimerSkipUnknownError()) {\n                    req.idempotentRelease(true);\n                } else {\n                    holdMomentForUnknownError();\n                }\n            }\n        }\n\n        protected void fetchAndPutTimerRequest() throws Exception {\n            long tmpCommitQueueOffset = currQueueOffset;\n            List<TimerRequest> trs = this.fetchTimerRequests();\n            if (CollectionUtils.isEmpty(trs)) {\n                commitQueueOffset = tmpCommitQueueOffset;\n                maybeMoveWriteTime();\n                return;\n            }\n\n            while (!isStopped()) {\n                CountDownLatch latch = new CountDownLatch(trs.size());\n                for (TimerRequest req : trs) {\n                    req.setLatch(latch);\n                    if (storeConfig.isTimerWheelSnapshotFlush()) {\n                        synchronized (lockWhenFlush) {\n                            this.putMessageToTimerWheel(req);\n                        }\n                    } else {\n                        this.putMessageToTimerWheel(req);\n                    }\n                }\n                checkDequeueLatch(latch, -1);\n                boolean allSuccess = trs.stream().allMatch(TimerRequest::isSucc);\n                if (allSuccess) {\n                    break;\n                } else {\n                    holdMomentForUnknownError();\n                }\n            }\n            commitQueueOffset = trs.get(trs.size() - 1).getMsg().getQueueOffset();\n            maybeMoveWriteTime();\n        }\n\n        @Override\n        public void run() {\n            TimerMessageStore.LOGGER.info(this.getServiceName() + \" service start\");\n            while (!this.isStopped() || enqueuePutQueue.size() != 0) {\n                try {\n                    fetchAndPutTimerRequest();\n                } catch (Throwable e) {\n                    TimerMessageStore.LOGGER.error(\"Unknown error\", e);\n                }\n            }\n            TimerMessageStore.LOGGER.info(this.getServiceName() + \" service end\");\n        }\n    }\n\n    public class TimerDequeueGetService extends ServiceThread {\n\n        @Override\n        public String getServiceName() {\n            return getServiceThreadName() + this.getClass().getSimpleName();\n        }\n\n        @Override\n        public void run() {\n            TimerMessageStore.LOGGER.info(this.getServiceName() + \" service start\");\n            while (!this.isStopped()) {\n                try {\n                    if (System.currentTimeMillis() < shouldStartTime) {\n                        TimerMessageStore.LOGGER.info(\"TimerDequeueGetService ready to run after {}.\", shouldStartTime);\n                        waitForRunning(1000);\n                        continue;\n                    }\n                    if (-1 == TimerMessageStore.this.dequeue()) {\n                        waitForRunning(100L * precisionMs / 1000);\n                    }\n                } catch (Throwable e) {\n                    TimerMessageStore.LOGGER.error(\"Error occurred in \" + getServiceName(), e);\n                }\n            }\n            TimerMessageStore.LOGGER.info(this.getServiceName() + \" service end\");\n        }\n    }\n\n    abstract class AbstractStateService extends ServiceThread {\n        public static final int INITIAL = -1, START = 0, WAITING = 1, RUNNING = 2, END = 3;\n        protected int state = INITIAL;\n\n        protected void setState(int state) {\n            this.state = state;\n        }\n\n        protected boolean isState(int state) {\n            return this.state == state;\n        }\n    }\n\n    public class TimerDequeuePutMessageService extends AbstractStateService {\n        @Override\n        public String getServiceName() {\n            String brokerIdentifier = \"\";\n            if (TimerMessageStore.this.messageStore instanceof DefaultMessageStore) {\n                try {\n                    DefaultMessageStore defaultStore = (DefaultMessageStore) TimerMessageStore.this.messageStore;\n                    if (defaultStore.getBrokerConfig().isInBrokerContainer()) {\n                        brokerIdentifier = defaultStore.getBrokerConfig().getIdentifier();\n                    }\n                } catch (Exception e) {\n                    LOGGER.warn(\"Failed to get broker identifier\", e);\n                }\n            }\n            return brokerIdentifier + this.getClass().getSimpleName();\n        }\n\n        @Override\n        public void run() {\n            setState(AbstractStateService.START);\n            TimerMessageStore.LOGGER.info(this.getServiceName() + \" service start\");\n\n            while (!this.isStopped() || dequeuePutQueue.size() != 0) {\n                try {\n                    setState(AbstractStateService.WAITING);\n                    TimerRequest tr = dequeuePutQueue.poll(10, TimeUnit.MILLISECONDS);\n                    if (null == tr) {\n                        continue;\n                    }\n\n                    setState(AbstractStateService.RUNNING);\n                    boolean tmpDequeueChangeFlag = false;\n\n                    try {\n                        while (!isStopped()) {\n                            if (!isRunningDequeue()) {\n                                dequeueStatusChangeFlag = true;\n                                tmpDequeueChangeFlag = true;\n                                break;\n                            }\n\n                            try {\n                                perfCounterTicks.startTick(DEQUEUE_PUT);\n\n                                MessageExt msgExt = tr.getMsg();\n                                StoreMetricsManager metricsManager = messageStore.getStoreMetricsManager();\n                                if (metricsManager instanceof DefaultStoreMetricsManager) {\n                                    ((DefaultStoreMetricsManager) metricsManager).incTimerDequeueCount(getRealTopic(msgExt));\n                                }\n\n                                if (tr.getEnqueueTime() == Long.MAX_VALUE) {\n                                    // Never enqueue, mark it.\n                                    MessageAccessor.putProperty(msgExt, TIMER_ENQUEUE_MS, String.valueOf(Long.MAX_VALUE));\n                                }\n\n                                addMetric(msgExt, -1);\n                                MessageExtBrokerInner msg = convert(msgExt, tr.getEnqueueTime(), needRoll(tr.getMagic()));\n\n                                boolean processed = false;\n                                int retryCount = 0;\n\n                                while (!processed && !isStopped()) {\n                                    int result = doPut(msg, needRoll(tr.getMagic()));\n\n                                    if (result == PUT_OK) {\n                                        processed = true;\n                                    } else if (result == PUT_NO_RETRY) {\n                                        TimerMessageStore.LOGGER.warn(\"Skipping message due to unrecoverable error. Msg: {}\", msg);\n                                        processed = true;\n                                    } else {\n                                        retryCount++;\n                                        // Without enabling TimerEnableRetryUntilSuccess, messages will retry up to 3 times before being discarded\n                                        if (!storeConfig.isTimerEnableRetryUntilSuccess() && retryCount >= 3) {\n                                            TimerMessageStore.LOGGER.error(\"Message processing failed after {} retries. Msg: {}\", retryCount, msg);\n                                            processed = true;\n                                        } else {\n                                            Thread.sleep(500L * precisionMs / 1000);\n                                            TimerMessageStore.LOGGER.warn(\"Retrying to process message. Retry count: {}, Msg: {}\", retryCount, msg);\n                                        }\n                                    }\n                                }\n\n                                perfCounterTicks.endTick(DEQUEUE_PUT);\n                                break;\n\n                            } catch (Throwable t) {\n                                TimerMessageStore.LOGGER.info(\"Unknown error\", t);\n                                if (storeConfig.isTimerSkipUnknownError()) {\n                                    break;\n                                } else {\n                                    holdMomentForUnknownError();\n                                }\n                            }\n                        }\n                    } finally {\n                        tr.idempotentRelease(!tmpDequeueChangeFlag);\n                    }\n                } catch (Throwable e) {\n                    TimerMessageStore.LOGGER.error(\"Error occurred in \" + getServiceName(), e);\n                }\n            }\n            TimerMessageStore.LOGGER.info(this.getServiceName() + \" service end\");\n            setState(AbstractStateService.END);\n        }\n    }\n\n    public class TimerDequeueGetMessageService extends AbstractStateService {\n\n        @Override\n        public String getServiceName() {\n            return getServiceThreadName() + this.getClass().getSimpleName();\n        }\n\n        @Override\n        public void run() {\n            setState(AbstractStateService.START);\n            TimerMessageStore.LOGGER.info(this.getServiceName() + \" service start\");\n            while (!this.isStopped()) {\n                try {\n                    setState(AbstractStateService.WAITING);\n                    List<TimerRequest> trs = dequeueGetQueue.poll(100L * precisionMs / 1000, TimeUnit.MILLISECONDS);\n                    if (null == trs || trs.size() == 0) {\n                        continue;\n                    }\n                    setState(AbstractStateService.RUNNING);\n                    for (int i = 0; i < trs.size(); ) {\n                        TimerRequest tr = trs.get(i);\n                        boolean doRes = false;\n                        try {\n                            long start = System.currentTimeMillis();\n                            MessageExt msgExt = getMessageByCommitOffset(tr.getOffsetPy(), tr.getSizePy());\n                            if (null != msgExt) {\n                                if (needDelete(tr.getMagic()) && !needRoll(tr.getMagic())) {\n                                    if (msgExt.getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY) != null && tr.getDeleteList() != null) {\n                                        //Execute metric plus one for messages that fail to be deleted\n                                        addMetric(msgExt, 1);\n                                        tr.getDeleteList().add(msgExt.getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY));\n                                    }\n                                    tr.idempotentRelease();\n                                    doRes = true;\n                                } else {\n                                    String uniqueKey = MessageClientIDSetter.getUniqID(msgExt);\n                                    if (null == uniqueKey) {\n                                        LOGGER.warn(\"No uniqueKey for msg:{}\", msgExt);\n                                    }\n                                    if (null != uniqueKey && tr.getDeleteList() != null && tr.getDeleteList().size() > 0 && tr.getDeleteList().contains(buildDeleteKey(getRealTopic(msgExt), uniqueKey, storeConfig.isAppendTopicForTimerDeleteKey()))) {\n                                        //Normally, it cancels out with the +1 above\n                                        addMetric(msgExt, -1);\n                                        doRes = true;\n                                        tr.idempotentRelease();\n                                        perfCounterTicks.getCounter(\"dequeue_delete\").flow(1);\n                                    } else {\n                                        tr.setMsg(msgExt);\n                                        while (!isStopped() && !doRes) {\n                                            doRes = dequeuePutQueue.offer(tr, 3, TimeUnit.SECONDS);\n                                        }\n                                    }\n                                }\n                                perfCounterTicks.getCounter(\"dequeue_get_msg\").flow(System.currentTimeMillis() - start);\n                            } else {\n                                //the tr will never be processed afterwards, so idempotentRelease it\n                                tr.idempotentRelease();\n                                doRes = true;\n                                perfCounterTicks.getCounter(\"dequeue_get_msg_miss\").flow(System.currentTimeMillis() - start);\n                            }\n                        } catch (Throwable e) {\n                            LOGGER.error(\"Unknown exception\", e);\n                            if (storeConfig.isTimerSkipUnknownError()) {\n                                tr.idempotentRelease();\n                                doRes = true;\n                            } else {\n                                holdMomentForUnknownError();\n                            }\n                        } finally {\n                            if (doRes) {\n                                i++;\n                            }\n                        }\n                    }\n                    trs.clear();\n                } catch (Throwable e) {\n                    TimerMessageStore.LOGGER.error(\"Error occurred in \" + getServiceName(), e);\n                }\n            }\n            TimerMessageStore.LOGGER.info(this.getServiceName() + \" service end\");\n            setState(AbstractStateService.END);\n        }\n    }\n\n    public class TimerDequeueWarmService extends ServiceThread {\n\n        @Override\n        public String getServiceName() {\n            String brokerIdentifier = \"\";\n            if (TimerMessageStore.this.messageStore instanceof DefaultMessageStore && ((DefaultMessageStore) TimerMessageStore.this.messageStore).getBrokerConfig().isInBrokerContainer()) {\n                brokerIdentifier = ((DefaultMessageStore) TimerMessageStore.this.messageStore).getBrokerConfig().getIdentifier();\n            }\n            return brokerIdentifier + this.getClass().getSimpleName();\n        }\n\n        @Override\n        public void run() {\n            TimerMessageStore.LOGGER.info(this.getServiceName() + \" service start\");\n            while (!this.isStopped()) {\n                try {\n                    //if (!storeConfig.isTimerWarmEnable() || -1 == TimerMessageStore.this.warmDequeue()) {\n                    waitForRunning(50);\n                    //}\n                } catch (Throwable e) {\n                    TimerMessageStore.LOGGER.error(\"Error occurred in \" + getServiceName(), e);\n                }\n            }\n            TimerMessageStore.LOGGER.info(this.getServiceName() + \" service end\");\n        }\n    }\n\n    public boolean needRoll(int magic) {\n        return (magic & MAGIC_ROLL) != 0;\n    }\n\n    public boolean needDelete(int magic) {\n        return (magic & MAGIC_DELETE) != 0;\n    }\n\n    public class TimerFlushService extends ServiceThread {\n        private final SimpleDateFormat sdf = new SimpleDateFormat(\"MM-dd HH:mm:ss\");\n\n        @Override\n        public String getServiceName() {\n            String brokerIdentifier = \"\";\n            if (TimerMessageStore.this.messageStore instanceof DefaultMessageStore && ((DefaultMessageStore) TimerMessageStore.this.messageStore).getBrokerConfig().isInBrokerContainer()) {\n                brokerIdentifier = ((DefaultMessageStore) TimerMessageStore.this.messageStore).getBrokerConfig().getIdentifier();\n            }\n            return brokerIdentifier + this.getClass().getSimpleName();\n        }\n\n        private String format(long time) {\n            return sdf.format(new Date(time));\n        }\n\n        @Override\n        public void run() {\n            TimerMessageStore.LOGGER.info(this.getServiceName() + \" service start\");\n            while (!this.isStopped()) {\n                try {\n                    this.flush();\n                } catch (Throwable e) {\n                    TimerMessageStore.LOGGER.error(\"Error occurred in \" + getServiceName(), e);\n                }\n                try {\n                    waitForRunning(storeConfig.getTimerFlushIntervalMs());\n                } catch (Throwable e) {\n                    // ignore interrupt\n                }\n            }\n            TimerMessageStore.LOGGER.info(this.getServiceName() + \" service end\");\n        }\n\n        long start = System.currentTimeMillis();\n        long lastSnapshotTime = System.currentTimeMillis();\n\n        public void flush() throws IOException {\n            if (storeConfig.isTimerWheelSnapshotFlush()) {\n                synchronized (lockWhenFlush) {\n                    prepareTimerCheckPoint();\n                    timerLog.getMappedFileQueue().flush(0);\n                    if (System.currentTimeMillis() - lastSnapshotTime > storeConfig.getTimerWheelSnapshotIntervalMs()) {\n                        lastSnapshotTime = System.currentTimeMillis();\n                        timerWheel.backup(timerLog.getMappedFileQueue().getFlushedWhere());\n                    }\n                    timerCheckpoint.flush();\n                }\n            } else {\n                prepareTimerCheckPoint();\n                timerLog.getMappedFileQueue().flush(0);\n                timerWheel.flush();\n                timerCheckpoint.flush();\n            }\n            if (System.currentTimeMillis() - start > storeConfig.getTimerProgressLogIntervalMs()) {\n                start = System.currentTimeMillis();\n                long tmpQueueOffset = currQueueOffset;\n                ConsumeQueueInterface cq = messageStore.getConsumeQueue(TIMER_TOPIC, 0);\n                long maxOffsetInQueue = cq == null ? 0 : cq.getMaxOffsetInQueue();\n                TimerMessageStore.LOGGER.info(\"[{}]Timer progress-check commitRead:[{}] currRead:[{}] currWrite:[{}] readBehind:{} currReadOffset:{} offsetBehind:{} behindMaster:{} \" +\n                        \"enqPutQueue:{} deqGetQueue:{} deqPutQueue:{} allCongestNum:{} enqExpiredStoreTime:{}\",\n                    storeConfig.getBrokerRole(),\n                    format(commitReadTimeMs), format(currReadTimeMs), format(currWriteTimeMs), getDequeueBehind(),\n                    tmpQueueOffset, maxOffsetInQueue - tmpQueueOffset, timerCheckpoint.getMasterTimerQueueOffset() - tmpQueueOffset,\n                    enqueuePutQueue.size(), dequeueGetQueue.size(), dequeuePutQueue.size(), getAllCongestNum(), format(lastEnqueueButExpiredStoreTime));\n            }\n            timerMetrics.persist();\n        }\n    }\n\n    public long getAllCongestNum() {\n        return timerWheel.getAllNum(currReadTimeMs);\n    }\n\n    public long getCongestNum(long deliverTimeMs) {\n        return timerWheel.getNum(deliverTimeMs);\n    }\n\n    public boolean isReject(long deliverTimeMs) {\n        long congestNum = timerWheel.getNum(deliverTimeMs);\n        if (congestNum <= storeConfig.getTimerCongestNumEachSlot()) {\n            return false;\n        }\n        if (congestNum >= storeConfig.getTimerCongestNumEachSlot() * 2L) {\n            return true;\n        }\n        if (RANDOM.nextInt(1000) > 1000 * (congestNum - storeConfig.getTimerCongestNumEachSlot()) / (storeConfig.getTimerCongestNumEachSlot() + 0.1)) {\n            return true;\n        }\n        return false;\n    }\n\n    public long getEnqueueBehindMessages() {\n        long tmpQueueOffset = currQueueOffset;\n        ConsumeQueueInterface cq = messageStore.getConsumeQueue(TIMER_TOPIC, 0);\n        long maxOffsetInQueue = cq == null ? 0 : cq.getMaxOffsetInQueue();\n        return maxOffsetInQueue - tmpQueueOffset;\n    }\n\n    public long getEnqueueBehindMillis() {\n        if (System.currentTimeMillis() - lastEnqueueButExpiredTime < 2000) {\n            return System.currentTimeMillis() - lastEnqueueButExpiredStoreTime;\n        }\n        return 0;\n    }\n\n    public long getEnqueueBehind() {\n        return getEnqueueBehindMillis() / 1000;\n    }\n\n    public long getDequeueBehindMessages() {\n        return timerWheel.getAllNum(currReadTimeMs);\n    }\n\n    public long getDequeueBehindMillis() {\n        return System.currentTimeMillis() - currReadTimeMs;\n    }\n\n    public long getDequeueBehind() {\n        return getDequeueBehindMillis() / 1000;\n    }\n\n    public float getEnqueueTps() {\n        return perfCounterTicks.getCounter(ENQUEUE_PUT).getLastTps();\n    }\n\n    public float getDequeueTps() {\n        return perfCounterTicks.getCounter(\"dequeue_put\").getLastTps();\n    }\n\n    public void prepareTimerCheckPoint() {\n        timerCheckpoint.setLastTimerLogFlushPos(timerLog.getMappedFileQueue().getFlushedWhere());\n        timerCheckpoint.setLastReadTimeMs(commitReadTimeMs);\n        if (shouldRunningDequeue) {\n            timerCheckpoint.setMasterTimerQueueOffset(commitQueueOffset);\n            if (commitReadTimeMs != lastCommitReadTimeMs || commitQueueOffset != lastCommitQueueOffset) {\n                timerCheckpoint.updateDataVersion(messageStore.getStateMachineVersion());\n                lastCommitReadTimeMs = commitReadTimeMs;\n                lastCommitQueueOffset = commitQueueOffset;\n            }\n        }\n        timerCheckpoint.setLastTimerQueueOffset(Math.min(commitQueueOffset, timerCheckpoint.getMasterTimerQueueOffset()));\n    }\n\n    public void registerEscapeBridgeHook(Function<MessageExtBrokerInner, PutMessageResult> escapeBridgeHook) {\n        this.escapeBridgeHook = escapeBridgeHook;\n    }\n\n    public boolean isMaster() {\n        return BrokerRole.SLAVE != lastBrokerRole;\n    }\n\n    public long getCurrReadTimeMs() {\n        return this.currReadTimeMs;\n    }\n\n    public long getQueueOffset() {\n        return currQueueOffset;\n    }\n\n    public long getCommitQueueOffset() {\n        return this.commitQueueOffset;\n    }\n\n    public long getCommitReadTimeMs() {\n        return this.commitReadTimeMs;\n    }\n\n    public MessageStore getMessageStore() {\n        return messageStore;\n    }\n\n    public TimerWheel getTimerWheel() {\n        return timerWheel;\n    }\n\n    public TimerLog getTimerLog() {\n        return timerLog;\n    }\n\n    public TimerMetrics getTimerMetrics() {\n        return this.timerMetrics;\n    }\n\n    public int getPrecisionMs() {\n        return precisionMs;\n    }\n\n    public TimerEnqueueGetService getEnqueueGetService() {\n        return enqueueGetService;\n    }\n\n    public void setEnqueueGetService(TimerEnqueueGetService enqueueGetService) {\n        this.enqueueGetService = enqueueGetService;\n    }\n\n    public TimerEnqueuePutService getEnqueuePutService() {\n        return enqueuePutService;\n    }\n\n    public void setEnqueuePutService(TimerEnqueuePutService enqueuePutService) {\n        this.enqueuePutService = enqueuePutService;\n    }\n\n    public TimerDequeueWarmService getDequeueWarmService() {\n        return dequeueWarmService;\n    }\n\n    public void setDequeueWarmService(\n        TimerDequeueWarmService dequeueWarmService) {\n        this.dequeueWarmService = dequeueWarmService;\n    }\n\n    public TimerDequeueGetService getDequeueGetService() {\n        return dequeueGetService;\n    }\n\n    public void setDequeueGetService(TimerDequeueGetService dequeueGetService) {\n        this.dequeueGetService = dequeueGetService;\n    }\n\n    public TimerDequeuePutMessageService[] getDequeuePutMessageServices() {\n        return dequeuePutMessageServices;\n    }\n\n    public void setDequeuePutMessageServices(\n        TimerDequeuePutMessageService[] dequeuePutMessageServices) {\n        this.dequeuePutMessageServices = dequeuePutMessageServices;\n    }\n\n    public TimerDequeueGetMessageService[] getDequeueGetMessageServices() {\n        return dequeueGetMessageServices;\n    }\n\n    public void setDequeueGetMessageServices(\n        TimerDequeueGetMessageService[] dequeueGetMessageServices) {\n        this.dequeueGetMessageServices = dequeueGetMessageServices;\n    }\n\n    public void setTimerMetrics(TimerMetrics timerMetrics) {\n        this.timerMetrics = timerMetrics;\n    }\n\n    public AtomicInteger getFrequency() {\n        return frequency;\n    }\n\n    public void setFrequency(AtomicInteger frequency) {\n        this.frequency = frequency;\n    }\n\n    public TimerCheckpoint getTimerCheckpoint() {\n        return timerCheckpoint;\n    }\n\n    // identify a message by topic or topic + uk(like query operation)\n    public static String buildDeleteKey(String realTopic, String uniqueKey, Boolean appendTopicForTimerDeleteKey) {\n        return appendTopicForTimerDeleteKey ? (realTopic + \"+\" + uniqueKey) : uniqueKey;\n    }\n\n    private void recallToTimeline(long delayTime, long offsetPy, int sizePy, MessageExt messageExt) {\n        if (!storeConfig.isTimerRecallToTimelineEnable() || !storeConfig.isTimerRocksDBEnable()) {\n            return;\n        }\n        if (delayTime < 0L || offsetPy < 0L || sizePy <= 0 || null == messageExt) {\n            LOGGER.error(\"recallToTimeline param error, delayTime: {}, offsetPy: {}, sizePy: {}, messageExt: {}\", delayTime, offsetPy, sizePy, messageExt);\n            return;\n        }\n        if (null == messageStore.getTimerMessageRocksDBStore() || null == messageStore.getTimerMessageRocksDBStore().getTimeline()) {\n            LOGGER.error(\"recallToTimeline error, timerRocksDBStore is null or timeline is null\");\n            return;\n        }\n        try {\n            messageStore.getTimerMessageRocksDBStore().getTimeline().putDeleteRecord(delayTime, messageExt.getMsgId(), offsetPy, sizePy, messageExt.getQueueOffset(), messageExt);\n        } catch (Exception e) {\n            LOGGER.error(\"recallToTimeline error: {}\", e.getMessage());\n        }\n    }\n\n    public boolean restart() {\n        try {\n            if (this.state != RUNNING) {\n                LOGGER.info(\"TimerMessageStore restart operation just support for running state\");\n                return false;\n            }\n            this.storeConfig.setTimerRocksDBStopScan(true);\n            if (this.state == RUNNING && !this.storeConfig.isTimerStopEnqueue()) {\n                LOGGER.info(\"restart TimerMessageStore has been running\");\n                return true;\n            }\n            long commitOffsetRocksDB = this.messageStore.getTimerMessageRocksDBStore().getCommitOffsetInRocksDB();\n            long commitOffsetFile = this.messageStore.getTimerMessageStore().getCommitQueueOffset();\n            long maxCommitOffset = Math.max(commitOffsetFile, commitOffsetRocksDB);\n            currQueueOffset = maxCommitOffset;\n            this.storeConfig.setTimerStopEnqueue(false);\n            LOGGER.info(\"TimerMessageStore restart commitOffsetRocksDB: {}, commitOffsetFile: {}, currQueueOffset: {}\", commitOffsetRocksDB, commitOffsetFile, currQueueOffset);\n            return true;\n        } catch (Exception e) {\n            LOGGER.error(\"TimerMessageStore restart error: {}\", e.getMessage());\n            return false;\n        }\n    }\n\n    public TimerFlushService getTimerFlushService() {\n        return timerFlushService;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.timer;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.alibaba.fastjson2.JSONWriter;\nimport org.apache.rocketmq.common.ConfigManager;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.DataVersion;\nimport org.apache.rocketmq.remoting.protocol.RemotingSerializable;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.io.RandomAccessFile;\nimport java.io.StringWriter;\nimport java.io.Writer;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.nio.file.Paths;\nimport java.nio.file.StandardCopyOption;\nimport java.util.ArrayList;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.concurrent.locks.Lock;\nimport java.util.concurrent.locks.ReentrantLock;\n\npublic class TimerMetrics extends ConfigManager {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);\n    private static final long LOCK_TIMEOUT_MILLIS = 3000;\n    private transient final Lock lock = new ReentrantLock();\n\n    private final ConcurrentMap<String, Metric> timingCount = new ConcurrentHashMap<>(1024);\n\n    private final ConcurrentMap<Integer, Metric> timingDistribution = new ConcurrentHashMap<>(1024);\n\n    @SuppressWarnings(\"DoubleBraceInitialization\")\n    public List<Integer> timerDist = new ArrayList<Integer>() {{\n            add(5);\n            add(60);\n            add(300); // 5s, 1min, 5min\n            add(900);\n            add(3600);\n            add(14400); // 15min, 1h, 4h\n            add(28800);\n            add(86400); // 8h, 24h\n        }};\n    private final DataVersion dataVersion = new DataVersion();\n\n    private final String configPath;\n\n    public TimerMetrics(String configPath) {\n        this.configPath = configPath;\n    }\n\n    public long updateDistPair(int period, int value) {\n        Metric distPair = getDistPair(period);\n        return distPair.getCount().addAndGet(value);\n    }\n\n    public long addAndGet(MessageExt msg, int value) {\n        String topic = msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC);\n        Metric pair = getTopicPair(topic);\n        getDataVersion().nextVersion();\n        pair.setTimeStamp(System.currentTimeMillis());\n        return pair.getCount().addAndGet(value);\n    }\n\n    public Metric getDistPair(Integer period) {\n        Metric pair = timingDistribution.get(period);\n        if (null != pair) {\n            return pair;\n        }\n        pair = new Metric();\n        final Metric previous = timingDistribution.putIfAbsent(period, pair);\n        if (null != previous) {\n            return previous;\n        }\n        return pair;\n    }\n\n    public Metric getTopicPair(String topic) {\n        Metric pair = timingCount.get(topic);\n        if (null != pair) {\n            return pair;\n        }\n        pair = new Metric();\n        final Metric previous = timingCount.putIfAbsent(topic, pair);\n        if (null != previous) {\n            return previous;\n        }\n        return pair;\n    }\n\n    public List<Integer> getTimerDistList() {\n        return this.timerDist;\n    }\n\n    public void setTimerDistList(List<Integer> timerDist) {\n        this.timerDist = timerDist;\n    }\n\n    public long getTimingCount(String topic) {\n        Metric pair = timingCount.get(topic);\n        if (null == pair) {\n            return 0;\n        } else {\n            return pair.getCount().get();\n        }\n    }\n\n    public Map<String, Metric> getTimingCount() {\n        return timingCount;\n    }\n\n    protected void write0(Writer writer) throws IOException {\n        TimerMetricsSerializeWrapper wrapper = new TimerMetricsSerializeWrapper();\n        wrapper.setTimingCount(timingCount);\n        wrapper.setDataVersion(dataVersion);\n        writer.write(JSON.toJSONString(wrapper, JSONWriter.Feature.BrowserCompatible));\n    }\n\n    @Override public String encode() {\n        return encode(false);\n    }\n\n    @Override public String configFilePath() {\n        return configPath;\n    }\n\n    @Override public void decode(String jsonString) {\n        if (jsonString != null) {\n            TimerMetricsSerializeWrapper timerMetricsSerializeWrapper = TimerMetricsSerializeWrapper.fromJson(jsonString, TimerMetricsSerializeWrapper.class);\n            if (timerMetricsSerializeWrapper != null) {\n                this.timingCount.putAll(timerMetricsSerializeWrapper.getTimingCount());\n                this.dataVersion.assignNewOne(timerMetricsSerializeWrapper.getDataVersion());\n            }\n        }\n    }\n\n    @Override public String encode(boolean prettyFormat) {\n        TimerMetricsSerializeWrapper metricsSerializeWrapper = new TimerMetricsSerializeWrapper();\n        metricsSerializeWrapper.setDataVersion(this.dataVersion);\n        metricsSerializeWrapper.setTimingCount(this.timingCount);\n        return metricsSerializeWrapper.toJson(prettyFormat);\n    }\n\n    public DataVersion getDataVersion() {\n        return dataVersion;\n    }\n\n    public void cleanMetrics(Set<String> topics) {\n        if (topics == null || topics.isEmpty()) {\n            return;\n        }\n        Iterator<Map.Entry<String, Metric>> iterator = timingCount.entrySet().iterator();\n        while (iterator.hasNext()) {\n            Map.Entry<String, Metric> entry = iterator.next();\n            final String topic = entry.getKey();\n            if (topic.startsWith(TopicValidator.SYSTEM_TOPIC_PREFIX) || topic.startsWith(MixAll.LMQ_PREFIX)) {\n                continue;\n            }\n            if (topics.contains(topic)) {\n                continue;\n            }\n\n            iterator.remove();\n            log.info(\"clean timer metrics, because not in topic config, {}\", topic);\n        }\n    }\n\n    public boolean removeTimingCount(String topic) {\n        try {\n            timingCount.remove(topic);\n        } catch (Exception e) {\n            log.error(\"removeTimingCount error\", e);\n            return false;\n        }\n        return true;\n    }\n\n    public static class TimerMetricsSerializeWrapper extends RemotingSerializable {\n        private ConcurrentMap<String, Metric> timingCount = new ConcurrentHashMap<>(1024);\n        private DataVersion dataVersion = new DataVersion();\n\n        public ConcurrentMap<String, Metric> getTimingCount() {\n            return timingCount;\n        }\n\n        public void setTimingCount(ConcurrentMap<String, Metric> timingCount) {\n            this.timingCount = timingCount;\n        }\n\n        public DataVersion getDataVersion() {\n            return dataVersion;\n        }\n\n        public void setDataVersion(DataVersion dataVersion) {\n            this.dataVersion = dataVersion;\n        }\n    }\n\n    @Override public synchronized void persist() {\n        try {\n            // bak metrics file\n            String config = configFilePath();\n            String backup = config + \".bak\";\n            File configFile = new File(config);\n            File bakFile = new File(backup);\n\n            if (configFile.exists()) {\n                // atomic move\n                Files.move(configFile.toPath(), bakFile.toPath(), StandardCopyOption.ATOMIC_MOVE);\n\n                // sync the directory, ensure that the bak file is visible\n                MixAll.fsyncDirectory(Paths.get(bakFile.getParent()));\n            }\n\n            File dir = new File(configFile.getParent());\n            if (!dir.exists()) {\n                Files.createDirectories(dir.toPath());\n            }\n\n            // persist metrics file\n            StringWriter stringWriter = new StringWriter();\n            write0(stringWriter);\n            try (RandomAccessFile randomAccessFile = new RandomAccessFile(config, \"rw\")) {\n                randomAccessFile.write(stringWriter.toString().getBytes(StandardCharsets.UTF_8));\n                randomAccessFile.getChannel().force(true);\n                // sync the directory, ensure that the config file is visible\n                MixAll.fsyncDirectory(Paths.get(configFile.getParent()));\n            }\n        } catch (Throwable t) {\n            log.error(\"Failed to persist\", t);\n        }\n    }\n\n    public static class Metric {\n        private AtomicLong count;\n        private long timeStamp;\n\n        public Metric() {\n            count = new AtomicLong(0);\n            timeStamp = System.currentTimeMillis();\n        }\n\n        public AtomicLong getCount() {\n            return count;\n        }\n\n        public void setCount(AtomicLong count) {\n            this.count = count;\n        }\n\n        public long getTimeStamp() {\n            return timeStamp;\n        }\n\n        public void setTimeStamp(long timeStamp) {\n            this.timeStamp = timeStamp;\n        }\n\n        @Override public String toString() {\n            return String.format(\"[%d,%d]\", count.get(), timeStamp);\n        }\n    }\n\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/timer/TimerRequest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.timer;\n\nimport org.apache.rocketmq.common.message.MessageExt;\n\nimport java.util.Set;\nimport java.util.concurrent.CountDownLatch;\n\npublic class TimerRequest {\n\n    private final long offsetPy;\n    private final int sizePy;\n    private final long delayTime;\n\n    private final int magic;\n\n    private long enqueueTime;\n    private MessageExt msg;\n\n\n    //optional would be a good choice, but it relies on JDK 8\n    private CountDownLatch latch;\n\n    private boolean released;\n\n    //whether the operation is successful\n    private boolean succ;\n\n    private Set<String> deleteList;\n\n    public TimerRequest(long offsetPy, int sizePy, long delayTime, long enqueueTime, int magic) {\n        this(offsetPy, sizePy, delayTime, enqueueTime, magic, null);\n    }\n\n    public TimerRequest(long offsetPy, int sizePy, long delayTime, long enqueueTime, int magic, MessageExt msg) {\n        this.offsetPy = offsetPy;\n        this.sizePy = sizePy;\n        this.delayTime = delayTime;\n        this.enqueueTime = enqueueTime;\n        this.magic = magic;\n        this.msg = msg;\n    }\n\n    public long getOffsetPy() {\n        return offsetPy;\n    }\n\n    public int getSizePy() {\n        return sizePy;\n    }\n\n    public long getDelayTime() {\n        return delayTime;\n    }\n\n    public long getEnqueueTime() {\n        return enqueueTime;\n    }\n\n    public MessageExt getMsg() {\n        return msg;\n    }\n\n    public void setMsg(MessageExt msg) {\n        this.msg = msg;\n    }\n\n    public int getMagic() {\n        return magic;\n    }\n\n    public Set<String> getDeleteList() {\n        return deleteList;\n    }\n\n    public void setDeleteList(Set<String> deleteList) {\n        this.deleteList = deleteList;\n    }\n\n    public void setLatch(CountDownLatch latch) {\n        this.latch = latch;\n    }\n    public void setEnqueueTime(long enqueueTime) {\n        this.enqueueTime = enqueueTime;\n    }\n    public void idempotentRelease() {\n        idempotentRelease(true);\n    }\n\n    public void idempotentRelease(boolean succ) {\n        this.succ = succ;\n        if (!released && latch != null) {\n            released = true;\n            latch.countDown();\n        }\n    }\n\n    public boolean isSucc() {\n        return succ;\n    }\n\n    @Override\n    public String toString() {\n        return \"TimerRequest{\" +\n            \"offsetPy=\" + offsetPy +\n            \", sizePy=\" + sizePy +\n            \", delayTime=\" + delayTime +\n            \", enqueueTime=\" + enqueueTime +\n            \", magic=\" + magic +\n            \", msg=\" + msg +\n            \", latch=\" + latch +\n            \", released=\" + released +\n            \", succ=\" + succ +\n            \", deleteList=\" + deleteList +\n            '}';\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/timer/TimerWheel.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.timer;\n\nimport java.nio.file.Files;\nimport java.nio.file.Paths;\nimport java.nio.file.StandardCopyOption;\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\nimport java.io.File;\nimport java.io.FileNotFoundException;\nimport java.io.IOException;\nimport java.io.RandomAccessFile;\nimport java.nio.ByteBuffer;\nimport java.nio.MappedByteBuffer;\nimport java.nio.channels.FileChannel;\n\npublic class TimerWheel {\n\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    public static final String TIMER_WHEEL_FILE_NAME = \"timerwheel\";\n    public static final int BLANK = -1, IGNORE = -2;\n    public final int slotsTotal;\n    public final int precisionMs;\n    private final String fileName;\n    private final MappedByteBuffer mappedByteBuffer;\n    private final RandomAccessFile randomAccessFile;\n    private final FileChannel fileChannel;\n    private final ByteBuffer byteBuffer;\n    private final ThreadLocal<ByteBuffer> localBuffer = new ThreadLocal<ByteBuffer>() {\n        @Override\n        protected ByteBuffer initialValue() {\n            return byteBuffer.duplicate();\n        }\n    };\n    private final int wheelLength;\n\n    private long snapOffset;\n\n    public TimerWheel(String fileName, int slotsTotal, int precisionMs) throws IOException {\n        this(fileName, slotsTotal, precisionMs, -1);\n    }\n    public TimerWheel(String fileName, int slotsTotal, int precisionMs, long snapOffset) throws IOException {\n        this.slotsTotal = slotsTotal;\n        this.precisionMs = precisionMs;\n        this.fileName = fileName;\n        this.wheelLength = this.slotsTotal * 2 * Slot.SIZE;\n        this.snapOffset = snapOffset;\n\n        String finalFileName = selectSnapshotByFlag(snapOffset);\n        File file = new File(finalFileName);\n        UtilAll.ensureDirOK(file.getParent());\n\n        try {\n            randomAccessFile = new RandomAccessFile(finalFileName, \"rw\");\n            if (file.exists() && randomAccessFile.length() != 0 &&\n                randomAccessFile.length() != wheelLength) {\n                throw new RuntimeException(String.format(\"Timer wheel length:%d != expected:%s\",\n                    randomAccessFile.length(), wheelLength));\n            }\n            randomAccessFile.setLength(wheelLength);\n            if (snapOffset < 0) {\n                fileChannel = randomAccessFile.getChannel();\n                mappedByteBuffer = fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, wheelLength);\n                assert wheelLength == mappedByteBuffer.remaining();\n            } else {\n                fileChannel = null;\n                mappedByteBuffer = null;\n                randomAccessFile.close();\n            }\n            this.byteBuffer = ByteBuffer.allocateDirect(wheelLength);\n            this.byteBuffer.put(Files.readAllBytes(file.toPath()));\n        } catch (FileNotFoundException e) {\n            log.error(\"create file channel \" + finalFileName + \" Failed. \", e);\n            throw e;\n        } catch (IOException e) {\n            log.error(\"map file \" + finalFileName + \" Failed. \", e);\n            throw e;\n        }\n    }\n\n    public void shutdown() {\n        shutdown(true);\n    }\n\n    public void shutdown(boolean flush) {\n        if (flush) {\n            try {\n                this.flush();\n            } catch (Throwable e) {\n                log.error(\"flush error when shutdown\", e);\n            }\n        }\n\n        // unmap mappedByteBuffer\n        UtilAll.cleanBuffer(this.mappedByteBuffer);\n        UtilAll.cleanBuffer(this.byteBuffer);\n        localBuffer.remove();\n\n        try {\n            this.fileChannel.close();\n        } catch (Throwable t) {\n            log.error(\"Shutdown error in timer wheel\", t);\n        }\n    }\n\n    public void flush() {\n        if (mappedByteBuffer == null) {\n            return;\n        }\n        ByteBuffer bf = localBuffer.get();\n        bf.position(0);\n        bf.limit(wheelLength);\n        mappedByteBuffer.position(0);\n        mappedByteBuffer.limit(wheelLength);\n        for (int i = 0; i < wheelLength; i++) {\n            if (bf.get(i) != mappedByteBuffer.get(i)) {\n                mappedByteBuffer.put(i, bf.get(i));\n            }\n        }\n        this.mappedByteBuffer.force();\n    }\n\n    /**\n     * Perform backup operation.\n     * <p>\n     * Select snapshot file based on the provided flag, write current buffer content to a temporary file,\n     * then rename the temporary file to the formal snapshot file. If rename fails, delete the temporary file.\n     * Finally clean up expired snapshot files.\n     *\n     * @param flushWhere Flag used to select snapshot file.\n     * @throws IOException If I/O error occurs during backup process.\n     */\n    public void backup(long flushWhere) throws IOException {\n        // Get current local buffer and position it to the beginning\n        ByteBuffer bf = localBuffer.get();\n        bf.position(0);\n        bf.limit(wheelLength);\n\n        // Select snapshot file name based on flag\n        String fileName = selectSnapshotByFlag(flushWhere);\n        File bakFile = new File(fileName);\n        // Create or open temporary file for snapshot, ready for writing\n        File tmpFile = new File(fileName + \".tmp\");\n        // Delete if exists first\n        Files.deleteIfExists(tmpFile.toPath());\n        try (RandomAccessFile randomAccessFile = new RandomAccessFile(tmpFile, \"rw\")) {\n            try (FileChannel fileChannel = randomAccessFile.getChannel()) {\n                fileChannel.write(bf);\n                fileChannel.force(true);\n            }\n        }\n\n        if (tmpFile.exists()) {\n            // atomic move\n            Files.move(tmpFile.toPath(), bakFile.toPath(), StandardCopyOption.ATOMIC_MOVE);\n\n            // sync the directory, ensure that the bak file is visible\n            MixAll.fsyncDirectory(Paths.get(bakFile.getParent()));\n        }\n        cleanExpiredSnapshot(); // Clean up expired snapshot files\n    }\n\n    /**\n     * Select snapshot file name based on flag.\n     *\n     * @param flag Flag used to select or identify snapshot file.\n     * @return Name of the snapshot file.\n     */\n    private String selectSnapshotByFlag(long flag) {\n        if (flag < 0) {\n            return this.fileName; // If flag is less than 0, return default file name\n        }\n        return this.fileName + \".\" + flag; // Otherwise, return file name with flag suffix\n    }\n\n    /**\n     * Clean up expired snapshot files.\n     * <p>\n     * This method will find and delete all snapshot files with flags smaller than the specified value\n     * under the current file name, keeping the two snapshot files with the largest flags.\n     */\n    public void cleanExpiredSnapshot() {\n        File dir = new File(this.fileName).getParentFile();\n        File[] files = dir.listFiles();\n        if (files == null) {\n            return;\n        }\n\n        // Collect all snapshot files and their flags\n        List<FileWithFlag> snapshotFiles = new ArrayList<>();\n        for (File file : files) {\n            String fileName = file.getName();\n            if (fileName.startsWith(TIMER_WHEEL_FILE_NAME + \".\")) {\n                long flag = UtilAll.asLong(fileName.substring(TIMER_WHEEL_FILE_NAME.length() + 1), -1);\n                if (flag >= 0) {\n                    snapshotFiles.add(new FileWithFlag(file, flag));\n                }\n            }\n        }\n\n        // Sort by flag in descending order\n        snapshotFiles.sort((a, b) -> Long.compare(b.flag, a.flag));\n\n        // Delete all files except the first two\n        for (int i = 2; i < snapshotFiles.size(); i++) {\n            UtilAll.deleteFile(snapshotFiles.get(i).file);\n        }\n    }\n\n    /**\n     * Get the maximum flag from existing snapshot files.\n     *\n     * @return The maximum flag value, or -1 if no snapshot files exist\n     */\n    public static long getMaxSnapshotFlag(String timerWheelPath) {\n        File dir = new File(timerWheelPath).getParentFile();\n        File[] files = dir.listFiles();\n        if (files == null) {\n            return -1;\n        }\n\n        long maxFlag = -1;\n        for (File file : files) {\n            String fileName = file.getName();\n            if (fileName.startsWith(TIMER_WHEEL_FILE_NAME + \".\")) {\n                long flag = UtilAll.asLong(fileName.substring(TIMER_WHEEL_FILE_NAME.length() + 1), -1);\n                if (flag > maxFlag) {\n                    maxFlag = flag;\n                }\n            }\n        }\n        return maxFlag;\n    }\n\n    /**\n     * Wrapper class for file and flag\n     */\n    private static class FileWithFlag {\n        final File file;\n        final long flag;\n\n        FileWithFlag(File file, long flag) {\n            this.file = file;\n            this.flag = flag;\n        }\n    }\n\n    public Slot getSlot(long timeMs) {\n        Slot slot = getRawSlot(timeMs);\n        if (slot.timeMs != timeMs / precisionMs * precisionMs) {\n            return new Slot(-1, -1, -1);\n        }\n        return slot;\n    }\n\n    //testable\n    public Slot getRawSlot(long timeMs) {\n        localBuffer.get().position(getSlotIndex(timeMs) * Slot.SIZE);\n        return new Slot(localBuffer.get().getLong() * precisionMs,\n            localBuffer.get().getLong(), localBuffer.get().getLong(), localBuffer.get().getInt(), localBuffer.get().getInt());\n    }\n\n    public int getSlotIndex(long timeMs) {\n        return (int) (timeMs / precisionMs % (slotsTotal * 2));\n    }\n\n    public void putSlot(long timeMs, long firstPos, long lastPos) {\n        localBuffer.get().position(getSlotIndex(timeMs) * Slot.SIZE);\n        // To be compatible with previous version.\n        // The previous version's precision is fixed at 1000ms and it store timeMs / 1000 in slot.\n        localBuffer.get().putLong(timeMs / precisionMs);\n        localBuffer.get().putLong(firstPos);\n        localBuffer.get().putLong(lastPos);\n    }\n\n    public void putSlot(long timeMs, long firstPos, long lastPos, int num, int magic) {\n        localBuffer.get().position(getSlotIndex(timeMs) * Slot.SIZE);\n        localBuffer.get().putLong(timeMs / precisionMs);\n        localBuffer.get().putLong(firstPos);\n        localBuffer.get().putLong(lastPos);\n        localBuffer.get().putInt(num);\n        localBuffer.get().putInt(magic);\n    }\n\n    public void reviseSlot(long timeMs, long firstPos, long lastPos, boolean force) {\n        localBuffer.get().position(getSlotIndex(timeMs) * Slot.SIZE);\n\n        if (timeMs / precisionMs != localBuffer.get().getLong()) {\n            if (force) {\n                putSlot(timeMs, firstPos != IGNORE ? firstPos : lastPos, lastPos);\n            }\n        } else {\n            if (IGNORE != firstPos) {\n                localBuffer.get().putLong(firstPos);\n            } else {\n                localBuffer.get().getLong();\n            }\n            if (IGNORE != lastPos) {\n                localBuffer.get().putLong(lastPos);\n            }\n        }\n    }\n\n    //check the timerwheel to see if its stored offset > maxOffset in timerlog\n    public long checkPhyPos(long timeStartMs, long maxOffset) {\n        long minFirst = Long.MAX_VALUE;\n        int firstSlotIndex = getSlotIndex(timeStartMs);\n        for (int i = 0; i < slotsTotal * 2; i++) {\n            int slotIndex = (firstSlotIndex + i) % (slotsTotal * 2);\n            localBuffer.get().position(slotIndex * Slot.SIZE);\n            if ((timeStartMs + i * precisionMs) / precisionMs != localBuffer.get().getLong()) {\n                continue;\n            }\n            long first = localBuffer.get().getLong();\n            long last = localBuffer.get().getLong();\n            if (last > maxOffset) {\n                if (first < minFirst) {\n                    minFirst = first;\n                }\n            }\n        }\n        return minFirst;\n    }\n\n    public long getNum(long timeMs) {\n        return getSlot(timeMs).num;\n    }\n\n    public long getAllNum(long timeStartMs) {\n        int allNum = 0;\n        int firstSlotIndex = getSlotIndex(timeStartMs);\n        for (int i = 0; i < slotsTotal * 2; i++) {\n            int slotIndex = (firstSlotIndex + i) % (slotsTotal * 2);\n            localBuffer.get().position(slotIndex * Slot.SIZE);\n            if ((timeStartMs + i * precisionMs) / precisionMs == localBuffer.get().getLong()) {\n                localBuffer.get().getLong(); //first pos\n                localBuffer.get().getLong(); //last pos\n                allNum = allNum + localBuffer.get().getInt();\n            }\n        }\n        return allNum;\n    }\n\n    public String getFileName() {\n        return fileName;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/Timeline.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.timer.rocksdb;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.LinkedBlockingDeque;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\nimport com.conversantmedia.util.concurrent.DisruptorBlockingQueue;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.commons.lang3.math.NumberUtils;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.rocksdb.MessageRocksDBStorage;\nimport org.apache.rocketmq.store.timer.TimerMessageStore;\nimport org.apache.rocketmq.store.timer.TimerMetrics;\nimport static org.apache.rocketmq.common.message.MessageConst.PROPERTY_TIMER_ROLL_LABEL;\nimport static org.apache.rocketmq.store.rocksdb.MessageRocksDBStorage.TIMER_COLUMN_FAMILY;\nimport static org.apache.rocketmq.store.timer.rocksdb.TimerRocksDBRecord.TIMER_ROCKSDB_DELETE;\nimport static org.apache.rocketmq.store.timer.rocksdb.TimerRocksDBRecord.TIMER_ROCKSDB_PUT;\nimport static org.apache.rocketmq.store.timer.rocksdb.TimerRocksDBRecord.TIMER_ROCKSDB_UPDATE;\n\npublic class Timeline {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private static final Logger logError = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);\n    private static final String DELETE_KEY_SPLIT = \"+\";\n    private static final int ORIGIN_CAPACITY = 100000;\n    private static final int BATCH_SIZE = 1000, MAX_BATCH_SIZE_FROM_ROCKSDB = 8000;\n    private static final int INITIAL = 0, RUNNING = 1, SHUTDOWN = 2;\n    private volatile int state = INITIAL;\n    private final AtomicLong commitOffset = new AtomicLong(0);\n    private final MessageStore messageStore;\n    private final MessageStoreConfig storeConfig;\n    private final TimerMessageStore timerMessageStore;\n    private final MessageRocksDBStorage messageRocksDBStorage;\n    private final TimerMessageRocksDBStore timerMessageRocksDBStore;\n    private final long precisionMs;\n    private final TimerMetrics timerMetrics;\n\n    private TimelineIndexBuildService timelineIndexBuildService;\n    private TimelineForwardService timelineForwardService;\n    private TimelineRollService timelineRollService;\n    private TimelineDeleteService timelineDeleteService;\n    private BlockingQueue<TimerRocksDBRecord> originTimerMsgQueue;\n\n    public Timeline(final MessageStore messageStore, final MessageRocksDBStorage messageRocksDBStorage, final TimerMessageRocksDBStore timerMessageRocksDBStore, final TimerMetrics timerMetrics) {\n        this.messageStore = messageStore;\n        this.storeConfig = messageStore.getMessageStoreConfig();\n        this.timerMessageStore = messageStore.getTimerMessageStore();\n        this.messageRocksDBStorage = messageRocksDBStorage;\n        this.timerMessageRocksDBStore = timerMessageRocksDBStore;\n        this.precisionMs = timerMessageRocksDBStore.precisionMs;\n        this.timerMetrics = timerMetrics;\n        initService();\n    }\n\n    private void initService() {\n        this.timelineIndexBuildService = new TimelineIndexBuildService();\n        this.timelineForwardService = new TimelineForwardService();\n        if (storeConfig.isTimerEnableDisruptor()) {\n            this.originTimerMsgQueue = new DisruptorBlockingQueue<>(ORIGIN_CAPACITY);\n        } else {\n            this.originTimerMsgQueue = new LinkedBlockingDeque<>(ORIGIN_CAPACITY);\n        }\n        this.timelineRollService = new TimelineRollService();\n        this.timelineDeleteService = new TimelineDeleteService();\n    }\n\n    public void start() {\n        if (this.state == RUNNING) {\n            return;\n        }\n        this.commitOffset.set(this.timerMessageRocksDBStore.getReadOffset().get());\n        this.timelineIndexBuildService.start();\n        this.timelineForwardService.start();\n        this.timelineRollService.start();\n        this.timelineDeleteService.start();\n        this.state = RUNNING;\n        log.info(\"Timeline start success, start commitOffset: {}\", this.commitOffset.get());\n    }\n\n    public void shutDown() {\n        if (this.state != RUNNING || this.state == SHUTDOWN) {\n            return;\n        }\n        if (null != this.timelineIndexBuildService) {\n            this.timelineIndexBuildService.shutdown();\n        }\n        if (null != this.timelineForwardService) {\n            this.timelineForwardService.shutdown();\n        }\n        if (null != this.timelineRollService) {\n            this.timelineRollService.shutdown();\n        }\n        if (null != this.timelineDeleteService) {\n            this.timelineDeleteService.shutdown();\n        }\n        this.state = SHUTDOWN;\n        log.info(\"Timeline shutdown success\");\n    }\n\n    public void putRecord(TimerRocksDBRecord timerRecord) throws InterruptedException {\n        if (null == timerRecord) {\n            return;\n        }\n        while (!originTimerMsgQueue.offer(timerRecord, 3, TimeUnit.SECONDS)) {\n            if (null != timerRecord.getMessageExt()) {\n                logError.error(\"Timeline originTimerMsgQueue put record failed, topic: {}, uniqKey: {}\", timerRecord.getMessageExt().getTopic(), timerRecord.getUniqKey());\n            } else {\n                logError.error(\"Timeline originTimerMsgQueue put record failed, uniqKey: {}\", timerRecord.getUniqKey());\n            }\n        }\n    }\n\n    public void putDeleteRecord(long delayTime, String uniqKey, long offsetPy, int sizePy, long queueOffset, MessageExt messageExt) throws InterruptedException {\n        if (delayTime <= 0L || StringUtils.isEmpty(uniqKey) || offsetPy < 0L || sizePy < 0 || queueOffset < 0L || null == messageExt) {\n            log.info(\"Timeline putDeleteRecord param error, delayTime: {}, uniqKey: {}, offsetPy: {}, sizePy: {}, queueOffset: {}, messageExt: {}\", delayTime, uniqKey, offsetPy, sizePy, queueOffset, messageExt);\n            return;\n        }\n        while (!originTimerMsgQueue.offer(new TimerRocksDBRecord(delayTime, uniqKey, offsetPy, sizePy, queueOffset, messageExt), 3, TimeUnit.SECONDS)) {\n            log.error(\"Timeline putDeleteRecord originTimerMsgQueue put delete record failed, uniqKey: {}\", uniqKey);\n        }\n    }\n\n    public void addMetric(MessageExt msg, int value) {\n        if (null == msg || null == msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC)) {\n            return;\n        }\n        if (null != msg.getProperty(MessageConst.PROPERTY_TIMER_ENQUEUE_MS) && NumberUtils.toLong(msg.getProperty(MessageConst.PROPERTY_TIMER_ENQUEUE_MS)) == Long.MAX_VALUE) {\n            return;\n        }\n        timerMetrics.addAndGet(msg, value);\n    }\n\n    private String getServiceThreadName() {\n        String brokerIdentifier = \"\";\n        if (Timeline.this.messageStore instanceof DefaultMessageStore) {\n            DefaultMessageStore messageStore = (DefaultMessageStore) Timeline.this.messageStore;\n            if (messageStore.getBrokerConfig().isInBrokerContainer()) {\n                brokerIdentifier = messageStore.getBrokerConfig().getIdentifier();\n            }\n        }\n        return brokerIdentifier;\n    }\n\n    private void recallToTimeWheel(TimerRocksDBRecord tr) {\n        if (!messageStore.getMessageStoreConfig().isTimerRecallToTimeWheelEnable()) {\n            return;\n        }\n        if (null == tr || null == tr.getMessageExt()) {\n            return;\n        }\n        try {\n            timerMessageStore.doEnqueue(tr.getOffsetPy(), tr.getSizePy(), tr.getDelayTime(), tr.getMessageExt(), true);\n        } catch (Exception e) {\n            log.error(\"Timeline recallToTimeWheel error: {}\", e.getMessage());\n        }\n    }\n\n    private boolean scanRecordsToQueue(long checkpoint, long checkRange, BlockingQueue<List<TimerRocksDBRecord>> queue) {\n        if (checkpoint <= 0L || checkRange <= 0L || null == queue) {\n            logError.error(\"Timeline scanRecordsToQueue param error, checkpoint: {}, checkRange: {}, queue: {}\", checkpoint, checkRange, queue);\n            return false;\n        }\n        if (storeConfig.isTimerStopDequeue()) {\n            logError.info(\"Timeline scanRecordsToQueue storeConfig isTimerStopDequeue is true, stop to scan records to queue\");\n            return false;\n        }\n        long count = 0;\n        byte[] lastKey = null;\n        while (true) {\n            try {\n                List<TimerRocksDBRecord> trs = messageRocksDBStorage.scanRecordsForTimer(TIMER_COLUMN_FAMILY, checkpoint, checkpoint + checkRange, MAX_BATCH_SIZE_FROM_ROCKSDB, lastKey);\n                if (null == trs || CollectionUtils.isEmpty(trs)) {\n                    break;\n                }\n                count += trs.size();\n                boolean hasMoreData = trs.size() >= MAX_BATCH_SIZE_FROM_ROCKSDB;\n                lastKey = hasMoreData ? trs.get(trs.size() - 1).getKeyBytes() : null;\n                if (null == lastKey) {\n                    trs.get(trs.size() - 1).setCheckPoint(checkpoint + checkRange);\n                }\n                while (!queue.offer(trs, 3, TimeUnit.SECONDS)) {\n                    log.debug(\"Timeline scanRecordsToQueue offer to queue error, queue size: {}, records size: {}\", queue.size(), trs.size());\n                }\n                if (!hasMoreData) {\n                    break;\n                }\n            } catch (Exception e) {\n                logError.error(\"Timeline scanRecordsToQueue error: {}\", e.getMessage());\n                return false;\n            }\n        }\n        log.info(\"Timeline scan records from rocksdb, checkpoint: {}, records size: {}\", checkpoint, count);\n        return true;\n    }\n\n    public class TimelineIndexBuildService extends ServiceThread {\n        private final Logger log = Timeline.log;\n        private List<TimerRocksDBRecord> trs;\n\n        @Override\n        public String getServiceName() {\n            return getServiceThreadName() + this.getClass().getSimpleName();\n        }\n\n        @Override\n        public void run() {\n            log.info(this.getServiceName() + \" service start\");\n            trs = new ArrayList<>(BATCH_SIZE);\n            while (!this.isStopped() || !originTimerMsgQueue.isEmpty()) {\n                try {\n                    buildTimelineIndex();\n                } catch (Exception e) {\n                    logError.error(\"Timeline error occurred in: {}, error: {}\", getServiceName(), e.getMessage());\n                }\n            }\n            log.info(this.getServiceName() + \" service end\");\n        }\n\n        private void buildTimelineIndex() throws InterruptedException {\n            pollTimerMessageRecords();\n            if (CollectionUtils.isEmpty(trs)) {\n                return;\n            }\n            List<TimerRocksDBRecord> cudlist = new ArrayList<>();\n            for (TimerRocksDBRecord tr : trs) {\n                try {\n                    MessageExt messageExt = tr.getMessageExt();\n                    if (null == messageExt) {\n                        logError.error(\"Timeline TimelineIndexBuildService buildTimelineIndex error, messageExt is null\");\n                        continue;\n                    }\n                    String timerDelUniqKey = messageExt.getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY);\n                    if (!StringUtils.isEmpty(timerDelUniqKey)) {\n                        tr.setUniqKey(extractUniqKey(timerDelUniqKey));\n                        tr.setActionFlag(TIMER_ROCKSDB_DELETE);\n                        cudlist.add(tr);\n                        addMetric(messageExt, -1);\n                        recallToTimeWheel(tr);\n                    } else if (TimerMessageRocksDBStore.isExpired(tr.getDelayTime())) {\n                        timerMessageRocksDBStore.putRealTopicMessage(tr.getMessageExt());\n                    } else if (!StringUtils.isEmpty(messageExt.getProperty(PROPERTY_TIMER_ROLL_LABEL))) {\n                        tr.setActionFlag(TIMER_ROCKSDB_UPDATE);\n                        cudlist.add(tr);\n                    } else {\n                        tr.setActionFlag(TIMER_ROCKSDB_PUT);\n                        cudlist.add(tr);\n                        addMetric(messageExt, 1);\n                    }\n                } catch (Exception e) {\n                    logError.error(\"Timeline TimelineIndexBuildService buildTimelineIndex deal trs error\", e);\n                }\n            }\n            if (!CollectionUtils.isEmpty(cudlist)) {\n                messageRocksDBStorage.writeRecordsForTimer(TIMER_COLUMN_FAMILY, cudlist);\n            }\n            synCommitOffset(trs);\n            trs.clear();\n        }\n\n        private String extractUniqKey(String deleteKey) throws IllegalArgumentException {\n            if (StringUtils.isEmpty(deleteKey)) {\n                throw new IllegalArgumentException(\"deleteKey is empty\");\n            }\n            int separatorIndex = deleteKey.indexOf(DELETE_KEY_SPLIT);\n            if (separatorIndex == -1) {\n                return deleteKey;\n            }\n            return deleteKey.substring(separatorIndex + 1);\n        }\n\n        private void pollTimerMessageRecords() throws InterruptedException {\n            TimerRocksDBRecord firstReq = originTimerMsgQueue.poll(100, TimeUnit.MILLISECONDS);\n            if (null != firstReq) {\n                trs.add(firstReq);\n                while (true) {\n                    TimerRocksDBRecord tmpReq = originTimerMsgQueue.poll(100, TimeUnit.MILLISECONDS);\n                    if (null == tmpReq) {\n                        break;\n                    }\n                    trs.add(tmpReq);\n                    if (trs.size() >= BATCH_SIZE) {\n                        break;\n                    }\n                }\n            }\n        }\n\n        private void synCommitOffset(List<TimerRocksDBRecord> trs) {\n            if (CollectionUtils.isEmpty(trs)) {\n                return;\n            }\n            long lastQueueOffset = messageRocksDBStorage.getCheckpointForTimer(TIMER_COLUMN_FAMILY, MessageRocksDBStorage.SYS_TOPIC_SCAN_OFFSET_CHECK_POINT);\n            long queueOffset = trs.get(trs.size() - 1).getQueueOffset() + 1L;\n            if (queueOffset > lastQueueOffset) {\n                commitOffset.set(queueOffset);\n                messageRocksDBStorage.writeCheckPointForTimer(TIMER_COLUMN_FAMILY, MessageRocksDBStorage.SYS_TOPIC_SCAN_OFFSET_CHECK_POINT, commitOffset.get());\n            }\n        }\n    }\n\n    private class TimelineForwardService extends ServiceThread {\n        private final Logger log = Timeline.log;\n        @Override\n        public String getServiceName() {\n            return getServiceThreadName() + this.getClass().getSimpleName();\n        }\n\n        @Override\n        public void run() {\n            long checkpoint = messageRocksDBStorage.getCheckpointForTimer(TIMER_COLUMN_FAMILY, MessageRocksDBStorage.TIMELINE_CHECK_POINT);\n            log.info(this.getServiceName() + \" service start, checkpoint: {}\", checkpoint);\n            while (!this.isStopped()) {\n                try {\n                    if (!timelineForward(checkpoint, precisionMs)) {\n                        waitForRunning(100L);\n                    } else {\n                        checkpoint += precisionMs;\n                    }\n                } catch (Exception e) {\n                    logError.error(\"Timeline error occurred in \" + getServiceName(), e);\n                }\n            }\n            log.info(this.getServiceName() + \" service end\");\n        }\n\n        private boolean timelineForward(long checkpoint, long checkRange) {\n            if (checkpoint > System.currentTimeMillis()) {\n                return false;\n            }\n            try {\n                long begin = System.currentTimeMillis();\n                boolean result = scanRecordsToQueue(checkpoint, checkRange, timerMessageRocksDBStore.getExpiredMessageQueue());\n                log.info(\"Timeline TimelineForwardService timelineForward scanRecordsToQueue end, result: {}, checkpoint: {}, checkRange: {}, checkDelay: {}, cost: {}\", result, checkpoint, checkRange, System.currentTimeMillis() - checkpoint, System.currentTimeMillis() - begin);\n                return result;\n            } catch (Exception e) {\n                logError.error(\"Timeline TimelineForwardService timelineForward error: {}\", e.getMessage());\n                return false;\n            }\n        }\n    }\n\n    private class TimelineRollService extends ServiceThread {\n        private final Logger log = Timeline.log;\n        @Override\n        public String getServiceName() {\n            return getServiceThreadName() + this.getClass().getSimpleName();\n        }\n\n        @Override\n        public void run() {\n            log.info(this.getServiceName() + \" service start\");\n            while (!this.isStopped()) {\n                int rollIntervalHour = 1;\n                int rollRangeHour = 2;\n                try {\n                    if (storeConfig.getTimerRocksDBRollIntervalHours() > 0) {\n                        rollIntervalHour = storeConfig.getTimerRocksDBRollIntervalHours();\n                    }\n                    if (storeConfig.getTimerRocksDBRollRangeHours() > 0) {\n                        rollRangeHour = storeConfig.getTimerRocksDBRollRangeHours();\n                    }\n                    this.waitForRunning(TimeUnit.HOURS.toMillis(rollIntervalHour));\n                    if (stopped) {\n                        log.info(this.getServiceName() + \" service end\");\n                        return;\n                    }\n                } catch (Exception e) {\n                    logError.error(\"Timeline TimelineRollService wait error: {}\", e.getMessage());\n                }\n                long rollCheckpoint = System.currentTimeMillis();\n                try {\n                    log.info(\"Timeline TimelineRollService start roll rollCheckpoint: {}\", rollCheckpoint);\n                    while (!scanRecordsToQueue(rollCheckpoint + TimeUnit.HOURS.toMillis(rollRangeHour),\n                            TimeUnit.SECONDS.toMillis(storeConfig.getTimerMaxDelaySec()),\n                            timerMessageRocksDBStore.getRollMessageQueue())) {\n                        logError.error(\"Timeline TimelineRollService scanRecordsToQueue error.\");\n                        Thread.sleep(200);\n                    }\n                    log.info(\"Timeline TimelineRollService roll records success, lastRollTime: {}, rollCheckpoint: {}, cost: {}\", rollCheckpoint, rollCheckpoint, System.currentTimeMillis() - rollCheckpoint);\n                } catch (Exception e) {\n                    logError.error(\"Timeline TimelineRollService failed error: {}\", e.getMessage());\n                }\n            }\n            log.info(this.getServiceName() + \" service end\");\n        }\n    }\n\n    private class TimelineDeleteService extends ServiceThread {\n        private final Logger log = Timeline.log;\n        private long lastDeleteCheckPoint = 0L;\n        @Override\n        public String getServiceName() {\n            return getServiceThreadName() + this.getClass().getSimpleName();\n        }\n        @Override\n        public void run() {\n            log.info(this.getServiceName() + \" service start\");\n            while (!this.isStopped()) {\n                try {\n                    this.waitForRunning(TimeUnit.MINUTES.toMillis(30));\n                    if (stopped) {\n                        log.info(this.getServiceName() + \" service end\");\n                        return;\n                    }\n                } catch (Exception e) {\n                    logError.error(\"Timeline TimelineDeleteService wait error: {}\", e.getMessage());\n                }\n                try {\n                    long checkpoint = messageRocksDBStorage.getCheckpointForTimer(TIMER_COLUMN_FAMILY, MessageRocksDBStorage.TIMELINE_CHECK_POINT);\n                    if (lastDeleteCheckPoint == checkpoint) {\n                        continue;\n                    }\n                    messageRocksDBStorage.deleteRecordsForTimer(TIMER_COLUMN_FAMILY, checkpoint - TimeUnit.HOURS.toMillis(168), checkpoint - TimeUnit.MINUTES.toMillis(30));\n                    lastDeleteCheckPoint = checkpoint;\n                } catch (Exception e) {\n                    logError.error(\"Timeline TimelineDeleteService delete failed, lastDeleteCheckPoint: {} error: {}\", lastDeleteCheckPoint, e.getMessage());\n                }\n            }\n            log.info(this.getServiceName() + \" service end\");\n        }\n    }\n\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerMessageRocksDBStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.timer.rocksdb;\n\nimport java.nio.ByteBuffer;\nimport java.util.List;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.Callable;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.LinkedBlockingDeque;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.function.Function;\nimport com.conversantmedia.util.concurrent.DisruptorBlockingQueue;\nimport com.google.common.util.concurrent.RateLimiter;\nimport io.opentelemetry.api.common.Attributes;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.TopicFilterType;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.StoreUtil;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant;\nimport org.apache.rocketmq.store.metrics.DefaultStoreMetricsManager;\nimport org.apache.rocketmq.store.metrics.StoreMetricsManager;\nimport org.apache.rocketmq.store.queue.ConsumeQueueInterface;\nimport org.apache.rocketmq.store.queue.CqUnit;\nimport org.apache.rocketmq.store.queue.ReferredIterator;\nimport org.apache.rocketmq.store.rocksdb.MessageRocksDBStorage;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.apache.rocketmq.store.timer.TimerMetrics;\nimport org.apache.rocketmq.store.util.PerfCounter;\nimport static org.apache.rocketmq.common.message.MessageConst.PROPERTY_TIMER_ROLL_LABEL;\nimport static org.apache.rocketmq.store.rocksdb.MessageRocksDBStorage.TIMER_COLUMN_FAMILY;\nimport static org.apache.rocketmq.store.timer.TimerMessageStore.TIMER_TOPIC;\n\npublic class TimerMessageRocksDBStore {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private static final Logger logError = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);\n    private static final String SCAN_SYS_TOPIC = \"scanSysTopic\";\n    private static final String SCAN_SYS_TOPIC_MISS = \"scanSysTopicMiss\";\n    private static final String OUT_BIZ_MESSAGE = \"outBizMsg\";\n    private static final String ROLL_LABEL = \"R\";\n    private static final int PUT_OK = 0, PUT_NEED_RETRY = 1, PUT_NO_RETRY = 2;\n    private static final int MAX_GET_MSG_TIMES = 3, MAX_PUT_MSG_TIMES = 3;\n    private static final int TIME_UP_CAPACITY = 100, ROLL_CAPACITY = 50;\n    private static final int INITIAL = 0, RUNNING = 1, SHUTDOWN = 2;\n    private volatile int state = INITIAL;\n    private static long expirationThresholdMillis = 999L;\n    private final AtomicLong readOffset = new AtomicLong(0);//timerSysTopic read offset\n    private final MessageStore messageStore;\n    private final TimerMetrics timerMetrics;\n    private final MessageStoreConfig storeConfig;\n    private final BrokerStatsManager brokerStatsManager;\n    private final MessageRocksDBStorage messageRocksDBStorage;\n    private Timeline timeline;\n    private TimerSysTopicScanService timerSysTopicScanService;\n    private TimerMessageReputService expiredMessageReputService;\n    private TimerMessageReputService rollMessageReputService;\n    protected long precisionMs;\n    private BlockingQueue<List<TimerRocksDBRecord>> expiredMessageQueue;\n    private BlockingQueue<List<TimerRocksDBRecord>> rollMessageQueue;\n    protected final PerfCounter.Ticks perfCounterTicks = new PerfCounter.Ticks(log);\n    private Function<MessageExtBrokerInner, PutMessageResult> escapeBridgeHook;\n    private ThreadLocal<ByteBuffer> bufferLocal = null;\n\n    public TimerMessageRocksDBStore(final MessageStore messageStore, final TimerMetrics timerMetrics,\n        final BrokerStatsManager brokerStatsManager) {\n        this.messageStore = messageStore;\n        this.storeConfig = messageStore.getMessageStoreConfig();\n        this.precisionMs = storeConfig.getTimerRocksDBPrecisionMs() < 100L ? 1000L : storeConfig.getTimerRocksDBPrecisionMs();\n        expirationThresholdMillis = precisionMs - 1L;\n        this.messageRocksDBStorage = messageStore.getMessageRocksDBStorage();\n        this.timerMetrics = timerMetrics;\n        this.brokerStatsManager = brokerStatsManager;\n        bufferLocal = ThreadLocal.withInitial(() -> ByteBuffer.allocate(storeConfig.getMaxMessageSize()));\n    }\n\n    public synchronized boolean load() {\n        initService();\n        boolean result = this.timerMetrics.load();\n        log.info(\"TimerMessageRocksDBStore load result: {}\", result);\n        return result;\n    }\n\n    public synchronized void start() {\n        if (this.state == RUNNING) {\n            return;\n        }\n        long commitOffsetFile = null != this.messageStore.getTimerMessageStore() ? this.messageStore.getTimerMessageStore().getCommitQueueOffset() : 0L;\n        long commitOffsetRocksDB = messageRocksDBStorage.getCheckpointForTimer(TIMER_COLUMN_FAMILY, MessageRocksDBStorage.SYS_TOPIC_SCAN_OFFSET_CHECK_POINT);\n        long maxCommitOffset = Math.max(commitOffsetFile, commitOffsetRocksDB);\n        this.readOffset.set(maxCommitOffset);\n        this.expiredMessageReputService.start();\n        this.rollMessageReputService.start();\n        this.timeline.start();\n        this.timerSysTopicScanService.start();\n        this.state = RUNNING;\n        log.info(\"TimerMessageRocksDBStore start success, start commitOffsetFile: {}, commitOffsetRocksDB: {}, readOffset: {}\", commitOffsetFile, commitOffsetRocksDB, this.readOffset.get());\n    }\n\n    public synchronized boolean restart() {\n        try {\n            this.storeConfig.setTimerStopEnqueue(true);\n            if (this.state == RUNNING && !this.storeConfig.isTimerRocksDBStopScan()) {\n                log.info(\"restart TimerMessageRocksDBStore has been running\");\n                return true;\n            }\n            if (this.state == RUNNING && this.storeConfig.isTimerRocksDBStopScan()) {\n                long commitOffsetFile = null != this.messageStore.getTimerMessageStore() ? this.messageStore.getTimerMessageStore().getCommitQueueOffset() : 0L;\n                long commitOffsetRocksDB = messageRocksDBStorage.getCheckpointForTimer(TIMER_COLUMN_FAMILY, MessageRocksDBStorage.SYS_TOPIC_SCAN_OFFSET_CHECK_POINT);\n                long maxCommitOffset = Math.max(commitOffsetFile, commitOffsetRocksDB);\n                this.readOffset.set(maxCommitOffset);\n                log.info(\"restart TimerMessageRocksDBStore has been recover running, commitOffsetFile: {}, commitOffsetRocksDB: {}, readOffset: {}\", commitOffsetFile, commitOffsetRocksDB, readOffset.get());\n            } else {\n                this.load();\n                this.start();\n            }\n            this.storeConfig.setTimerRocksDBEnable(true);\n            this.storeConfig.setTimerRocksDBStopScan(false);\n            return true;\n        } catch (Exception e) {\n            logError.error(\"TimerMessageRocksDBStore restart error: {}\", e.getMessage());\n            return false;\n        }\n    }\n\n    public void shutdown() {\n        if (this.state != RUNNING || this.state == SHUTDOWN) {\n            return;\n        }\n        if (null != this.timerSysTopicScanService) {\n            this.timerSysTopicScanService.shutdown();\n        }\n        if (null != this.timeline) {\n            this.timeline.shutDown();\n        }\n        if (null != this.expiredMessageReputService) {\n            this.expiredMessageReputService.shutdown();\n        }\n        if (null != this.rollMessageReputService) {\n            this.rollMessageReputService.shutdown();\n        }\n        this.state = SHUTDOWN;\n        log.info(\"TimerMessageRocksDBStore shutdown success\");\n    }\n\n    public void putRealTopicMessage(MessageExt msg) {\n        if (null == msg) {\n            logError.error(\"putRealTopicMessage msg is null\");\n            return;\n        }\n        MessageExtBrokerInner messageExtBrokerInner = convertMessage(msg);\n        if (null == messageExtBrokerInner) {\n            logError.error(\"putRealTopicMessage error, messageExtBrokerInner is null\");\n            return;\n        }\n        doPut(messageExtBrokerInner);\n    }\n\n    public MessageStore getMessageStore() {\n        return messageStore;\n    }\n\n    public TimerMetrics getTimerMetrics() {\n        return timerMetrics;\n    }\n\n    public BrokerStatsManager getBrokerStatsManager() {\n        return brokerStatsManager;\n    }\n\n    public AtomicLong getReadOffset() {\n        return readOffset;\n    }\n\n    public BlockingQueue<List<TimerRocksDBRecord>> getExpiredMessageQueue() {\n        return expiredMessageQueue;\n    }\n\n    public BlockingQueue<List<TimerRocksDBRecord>> getRollMessageQueue() {\n        return rollMessageQueue;\n    }\n\n    public long getCommitOffsetInRocksDB() {\n        if (null == messageRocksDBStorage || !storeConfig.isTimerRocksDBEnable()) {\n            return 0L;\n        }\n        return messageRocksDBStorage.getCheckpointForTimer(TIMER_COLUMN_FAMILY, MessageRocksDBStorage.SYS_TOPIC_SCAN_OFFSET_CHECK_POINT);\n    }\n\n    public Timeline getTimeline() {\n        return timeline;\n    }\n\n    private void initService() {\n        if (storeConfig.isTimerEnableDisruptor()) {\n            this.expiredMessageQueue = new DisruptorBlockingQueue<>(TIME_UP_CAPACITY);\n            this.rollMessageQueue = new DisruptorBlockingQueue<>(ROLL_CAPACITY);\n        } else {\n            this.expiredMessageQueue = new LinkedBlockingDeque<>(TIME_UP_CAPACITY);\n            this.rollMessageQueue = new LinkedBlockingDeque<>(ROLL_CAPACITY);\n        }\n        this.expiredMessageReputService = new TimerMessageReputService(expiredMessageQueue, storeConfig.getTimerRocksDBTimeExpiredMaxTps(), true);\n        this.rollMessageReputService = new TimerMessageReputService(rollMessageQueue, storeConfig.getTimerRocksDBRollMaxTps(), false);\n        this.timeline = new Timeline(messageStore, messageRocksDBStorage, this, timerMetrics);\n        this.timerSysTopicScanService = new TimerSysTopicScanService();\n    }\n\n    private MessageExtBrokerInner convertMessage(MessageExt msgExt) {\n        if (null == msgExt) {\n            logError.error(\"convertMessage msgExt is null\");\n            return null;\n        }\n        MessageExtBrokerInner msgInner = null;\n        try {\n            msgInner = new MessageExtBrokerInner();\n            msgInner.setBody(msgExt.getBody());\n            msgInner.setFlag(msgExt.getFlag());\n            msgInner.setTags(msgExt.getTags());\n            msgInner.setSysFlag(msgExt.getSysFlag());\n            TopicFilterType topicFilterType = MessageExt.parseTopicFilterType(msgInner.getSysFlag());\n            long tagsCodeValue = MessageExtBrokerInner.tagsString2tagsCode(topicFilterType, msgInner.getTags());\n            msgInner.setTagsCode(tagsCodeValue);\n            msgInner.setBornTimestamp(msgExt.getBornTimestamp());\n            msgInner.setBornHost(msgExt.getBornHost());\n            msgInner.setStoreHost(msgExt.getStoreHost());\n            msgInner.setReconsumeTimes(msgExt.getReconsumeTimes());\n            msgInner.setWaitStoreMsgOK(false);\n            MessageAccessor.setProperties(msgInner, MessageAccessor.deepCopyProperties(msgExt.getProperties()));\n            if (isNeedRoll(msgInner)) {\n                msgInner.setTopic(msgExt.getTopic());\n                msgInner.setQueueId(msgExt.getQueueId());\n                msgInner.getProperties().put(PROPERTY_TIMER_ROLL_LABEL, ROLL_LABEL);\n            } else {\n                msgInner.setTopic(msgInner.getProperty(MessageConst.PROPERTY_REAL_TOPIC));\n                msgInner.setQueueId(Integer.parseInt(msgInner.getProperty(MessageConst.PROPERTY_REAL_QUEUE_ID)));\n                MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_REAL_TOPIC);\n                MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_REAL_QUEUE_ID);\n                MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_TIMER_ROLL_LABEL);\n            }\n            msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));\n        } catch (Exception e) {\n            logError.error(\"convertMessage error: {}\", e.getMessage());\n            return null;\n        }\n        return msgInner;\n    }\n\n    private MessageExt getMessageByCommitOffset(long offsetPy, int sizePy) {\n        if (offsetPy < 0L || sizePy <= 0 || sizePy > storeConfig.getMaxMessageSize()) {\n            logError.error(\"getMessageByCommitOffset param error, offsetPy: {}, sizePy: {}, maxMsgSize: {}\", offsetPy, sizePy, storeConfig.getMaxMessageSize());\n            return null;\n        }\n        if (sizePy > bufferLocal.get().limit()) {\n            bufferLocal.remove();\n            bufferLocal.set(ByteBuffer.allocate(sizePy));\n        }\n        for (int i = 0; i < MAX_GET_MSG_TIMES; i++) {\n            MessageExt msgExt = StoreUtil.getMessage(offsetPy, sizePy, messageStore, bufferLocal.get());\n            if (null == msgExt) {\n                log.warn(\"Fail to read msg from commitLog offsetPy: {} sizePy: {}\", offsetPy, sizePy);\n            } else {\n                return msgExt;\n            }\n        }\n        return null;\n    }\n\n    private boolean isNeedRoll(MessageExt messageExt) {\n        try {\n            String property = messageExt.getProperty(MessageConst.PROPERTY_TIMER_OUT_MS);\n            if (StringUtils.isEmpty(property)) {\n                return false;\n            }\n            if (!isExpired(Long.parseLong(property))) {\n                return true;\n            }\n        } catch (Exception e) {\n            logError.error(\"isNeedRoll error : {}\", e.getMessage());\n        }\n        return false;\n    }\n\n    private Long getDelayTime(MessageExt msgExt) {\n        if (null == msgExt) {\n            logError.error(\"getDelayTime msgExt is null\");\n            return null;\n        }\n        String delayTimeStr = msgExt.getProperty(MessageConst.PROPERTY_TIMER_OUT_MS);\n        if (StringUtils.isEmpty(delayTimeStr)) {\n            logError.error(\"getDelayTime is empty, queueOffset: {}, delayTimeStr: {}\", msgExt.getQueueOffset(), delayTimeStr);\n            return null;\n        }\n        try {\n            return Long.parseLong(delayTimeStr);\n        } catch (Exception e) {\n            logError.error(\"getDelayTime parse to long error : {}\", e.getMessage());\n        }\n        return null;\n    }\n\n    private int doPut(MessageExtBrokerInner message) {\n        if (null == message) {\n            logError.error(\"doPut message is null\");\n            return PUT_NO_RETRY;\n        }\n        if (null != message.getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY)) {\n            logError.warn(\"Trying do put delete timer msg:[{}]\", message);\n            return PUT_NO_RETRY;\n        }\n        PutMessageResult putMessageResult = null;\n        if (escapeBridgeHook != null) {\n            putMessageResult = escapeBridgeHook.apply(message);\n        } else {\n            putMessageResult = messageStore.putMessage(message);\n        }\n        if (null != putMessageResult && null != putMessageResult.getPutMessageStatus()) {\n            switch (putMessageResult.getPutMessageStatus()) {\n                case PUT_OK:\n                    if (null != brokerStatsManager) {\n                        brokerStatsManager.incTopicPutNums(message.getTopic(), 1, 1);\n                        if (null != putMessageResult.getAppendMessageResult()) {\n                            brokerStatsManager.incTopicPutSize(message.getTopic(), putMessageResult.getAppendMessageResult().getWroteBytes());\n                        }\n                        brokerStatsManager.incBrokerPutNums(message.getTopic(), 1);\n                    }\n                    return PUT_OK;\n\n                case MESSAGE_ILLEGAL:\n                case PROPERTIES_SIZE_EXCEEDED:\n                case WHEEL_TIMER_NOT_ENABLE:\n                case WHEEL_TIMER_MSG_ILLEGAL:\n                    return PUT_NO_RETRY;\n\n                case SERVICE_NOT_AVAILABLE:\n                case FLUSH_DISK_TIMEOUT:\n                case FLUSH_SLAVE_TIMEOUT:\n                case OS_PAGE_CACHE_BUSY:\n                case CREATE_MAPPED_FILE_FAILED:\n                case SLAVE_NOT_AVAILABLE:\n                    return PUT_NEED_RETRY;\n\n                case UNKNOWN_ERROR:\n                default:\n                    if (storeConfig.isTimerSkipUnknownError()) {\n                        logError.warn(\"Skipping message due to unknown error, msg: {}\", message);\n                        return PUT_NO_RETRY;\n                    } else {\n                        return PUT_NEED_RETRY;\n                    }\n            }\n        }\n        return PUT_NEED_RETRY;\n    }\n\n    public static boolean isExpired(long delayedTime) {\n        return delayedTime <= System.currentTimeMillis() + expirationThresholdMillis;\n    }\n\n    public void registerEscapeBridgeHook(Function<MessageExtBrokerInner, PutMessageResult> escapeBridgeHook) {\n        this.escapeBridgeHook = escapeBridgeHook;\n    }\n\n    private String getServiceThreadName() {\n        String brokerIdentifier = \"\";\n        if (TimerMessageRocksDBStore.this.messageStore instanceof DefaultMessageStore) {\n            DefaultMessageStore messageStore = (DefaultMessageStore)TimerMessageRocksDBStore.this.messageStore;\n            if (messageStore.getBrokerConfig().isInBrokerContainer()) {\n                brokerIdentifier = messageStore.getBrokerConfig().getIdentifier();\n            }\n        }\n        return brokerIdentifier;\n    }\n\n    private class TimerSysTopicScanService extends ServiceThread {\n        private final Logger log = TimerMessageRocksDBStore.log;\n\n        @Override\n        public String getServiceName() {\n            return getServiceThreadName() + this.getClass().getSimpleName();\n        }\n\n        @Override\n        public void run() {\n            log.info(this.getServiceName() + \" service start\");\n            long waitTime;\n            while (!this.isStopped()) {\n                try {\n                    if (!storeConfig.isTimerRocksDBEnable() || storeConfig.isTimerRocksDBStopScan()) {\n                        waitTime = TimeUnit.SECONDS.toMillis(10);\n                    } else {\n                        scanSysTimerTopic();\n                        waitTime = 100L;\n                    }\n                    waitForRunning(waitTime);\n                } catch (Exception e) {\n                    logError.error(\"TimerMessageRocksDBStore error occurred in: {}, error: {}\", getServiceName(),\n                        e.getMessage());\n                }\n            }\n            log.info(this.getServiceName() + \" service end\");\n        }\n\n        private void scanSysTimerTopic() {\n            ConsumeQueueInterface cq = messageStore.getConsumeQueue(TIMER_TOPIC, 0);\n            if (null == cq) {\n                return;\n            }\n            if (readOffset.get() < cq.getMinOffsetInQueue()) {\n                logError.warn(\"scanSysTimerTopic readOffset: {} is smaller than minOffsetInQueue: {}, use minOffsetInQueue to scan timer sysTimerTopic\", readOffset.get(), cq.getMinOffsetInQueue());\n                readOffset.set(cq.getMinOffsetInQueue());\n            } else if (readOffset.get() > cq.getMaxOffsetInQueue()) {\n                logError.warn(\"scanSysTimerTopic readOffset: {} is bigger than maxOffsetInQueue: {}, use maxOffsetInQueue to scan timer sysTimerTopic\", readOffset.get(), cq.getMaxOffsetInQueue());\n                readOffset.set(cq.getMaxOffsetInQueue());\n            }\n            ReferredIterator<CqUnit> iterator = null;\n            try {\n                iterator = cq.iterateFrom(readOffset.get());\n                if (null == iterator) {\n                    return;\n                }\n                while (iterator.hasNext()) {\n                    perfCounterTicks.startTick(SCAN_SYS_TOPIC);\n                    try {\n                        CqUnit cqUnit = iterator.next();\n                        if (null == cqUnit) {\n                            logError.error(\"scanSysTimerTopic cqUnit is null, readOffset: {}\", readOffset.get());\n                            break;\n                        }\n                        long offsetPy = cqUnit.getPos();\n                        int sizePy = cqUnit.getSize();\n                        long queueOffset = cqUnit.getQueueOffset();\n                        MessageExt msgExt = getMessageByCommitOffset(offsetPy, sizePy);\n                        if (null == msgExt) {\n                            perfCounterTicks.getCounter(SCAN_SYS_TOPIC_MISS);\n                            break;\n                        }\n                        Long delayedTime = getDelayTime(msgExt);\n                        if (null == delayedTime) {\n                            readOffset.incrementAndGet();\n                            continue;\n                        }\n                        if (isExpired(delayedTime)) {\n                            putRealTopicMessage(msgExt);\n                            readOffset.incrementAndGet();\n                            continue;\n                        }\n                        TimerRocksDBRecord timerRecord = new TimerRocksDBRecord(delayedTime, MessageClientIDSetter.getUniqID(msgExt), offsetPy, sizePy, queueOffset, msgExt);\n                        timeline.putRecord(timerRecord);\n                        readOffset.incrementAndGet();\n\n                        StoreMetricsManager metricsManager = messageStore.getStoreMetricsManager();\n                        if (metricsManager instanceof DefaultStoreMetricsManager) {\n                            DefaultStoreMetricsManager defaultMetricsManager = (DefaultStoreMetricsManager)metricsManager;\n                            Attributes attributes = defaultMetricsManager.newAttributesBuilder().put(DefaultStoreMetricsConstant.LABEL_TOPIC, msgExt.getProperty(MessageConst.PROPERTY_REAL_TOPIC)).build();\n                            defaultMetricsManager.getTimerMessageSetLatency().record((delayedTime - msgExt.getBornTimestamp()) / 1000, attributes);\n                        }\n                    } catch (Exception e) {\n                        logError.error(\"Unknown error in scan the system topic error: {}\", e.getMessage());\n                    } finally {\n                        perfCounterTicks.endTick(SCAN_SYS_TOPIC);\n                    }\n                }\n            } catch (Exception e) {\n                logError.error(\"scanSysTimerTopic throw Unknown exception. {}\", e.getMessage());\n            } finally {\n                if (iterator != null) {\n                    iterator.release();\n                }\n            }\n        }\n    }\n\n    private class TimerMessageReputService extends ServiceThread {\n        private final Logger log = TimerMessageRocksDBStore.log;\n        private final BlockingQueue<List<TimerRocksDBRecord>> queue;\n        private final RateLimiter rateLimiter;\n        private final boolean writeCheckPoint;\n        private final ExecutorService executor =\n                ThreadUtils.newThreadPoolExecutor(\n                        storeConfig.getTimerReputServiceCorePoolSize(),\n                        storeConfig.getTimerReputServiceMaxPoolSize(),\n                        60L,\n                        TimeUnit.SECONDS,\n                        new LinkedBlockingQueue<>(storeConfig.getTimerReputServiceQueueCapacity()),\n                        ThreadUtils.newGenericThreadFactory(\"TimerMessageReputService\", false),\n                        new ThreadPoolExecutor.CallerRunsPolicy()\n                );\n\n        public TimerMessageReputService(BlockingQueue<List<TimerRocksDBRecord>> queue, double maxTps, boolean writeCheckPoint) {\n            this.queue = queue;\n            this.rateLimiter = RateLimiter.create(maxTps);\n            this.writeCheckPoint = writeCheckPoint;\n        }\n\n        @Override\n        public String getServiceName() {\n            return getServiceThreadName() + this.getClass().getSimpleName();\n        }\n\n        @Override\n        public void run() {\n            log.info(this.getServiceName() + \" service start\");\n            while (!this.isStopped() || !queue.isEmpty()) {\n                try {\n                    List<TimerRocksDBRecord> trs = queue.poll(100L, TimeUnit.MILLISECONDS);\n                    if (CollectionUtils.isEmpty(trs)) {\n                        continue;\n                    }\n                    long start = System.currentTimeMillis();\n                    CountDownLatch countDownLatch = new CountDownLatch(trs.size());\n                    for (TimerRocksDBRecord record : trs) {\n                        executor.submit(new Task(countDownLatch, record));\n                    }\n                    countDownLatch.await();\n                    log.info(\"TimerMessageReputService reput messages to commitlog, cost: {}, trs size: {}, checkPoint: {}\", System.currentTimeMillis() - start, trs.size(), trs.get(trs.size() - 1).getCheckPoint());\n                    if (this.writeCheckPoint && !CollectionUtils.isEmpty(trs) && trs.get(trs.size() - 1).getCheckPoint() > 0L) {\n                        log.info(\"TimerMessageReputService reput messages to commitlog, checkPoint: {}\", trs.get(trs.size() - 1).getCheckPoint());\n                        messageRocksDBStorage.writeCheckPointForTimer(TIMER_COLUMN_FAMILY, MessageRocksDBStorage.TIMELINE_CHECK_POINT, trs.get(trs.size() - 1).getCheckPoint());\n                    }\n                } catch (Exception e) {\n                    logError.error(\"TimerMessageReputService error: {}\", e.getMessage());\n                }\n            }\n            log.info(this.getServiceName() + \" service end\");\n        }\n\n        private void putMsgWithRetry(MessageExtBrokerInner msg) throws InterruptedException {\n            if (null == msg) {\n                return;\n            }\n            for (int retryCount = 0; !isStopped() && retryCount <= MAX_PUT_MSG_TIMES; retryCount++) {\n                int result = doPut(msg);\n                switch (result) {\n                    case PUT_OK:\n                        return;\n                    case PUT_NO_RETRY:\n                        logError.warn(\"Skipping message due to unrecoverable error. Msg: {}\", msg);\n                        return;\n                    default:\n                        if (retryCount == MAX_PUT_MSG_TIMES) {\n                            logError.error(\"Message processing failed after {} retries. Msg: {}\", retryCount, msg);\n                            return;\n                        } else {\n                            Thread.sleep(100L);\n                            logError.warn(\"Retrying to process message. Retry count: {}, Msg: {}\", retryCount, msg);\n                        }\n                }\n            }\n        }\n\n        class Task implements Callable<Void> {\n            private CountDownLatch countDownLatch;\n            private TimerRocksDBRecord record;\n\n            public Task(CountDownLatch countDownLatch, TimerRocksDBRecord record) {\n                this.countDownLatch = countDownLatch;\n                this.record = record;\n            }\n\n            @Override\n            public Void call() throws Exception {\n                try {\n                    perfCounterTicks.startTick(OUT_BIZ_MESSAGE);\n                    MessageExt messageExt = record.getMessageExt();\n                    if (null == messageExt) {\n                        messageExt = getMessageByCommitOffset(record.getOffsetPy(), record.getSizePy());\n                        if (null == messageExt) {\n                            return null;\n                        }\n                    }\n                    MessageExtBrokerInner msg = convertMessage(messageExt);\n                    if (null == msg) {\n                        return null;\n                    }\n                    record.setUniqKey(MessageClientIDSetter.getUniqID(msg));\n                    putMsgWithRetry(msg);\n                    timeline.addMetric(msg, -1);\n                    perfCounterTicks.endTick(OUT_BIZ_MESSAGE);\n                    rateLimiter.acquire();\n                } catch (Exception e) {\n                    logError.error(\"TimerMessageReputService running error: {}\", e.getMessage());\n                } finally {\n                    countDownLatch.countDown();\n                }\n                return null;\n            }\n        }\n\n        @Override\n        public void shutdown() {\n            super.shutdown();\n            ThreadUtils.shutdownGracefully(executor, 5, TimeUnit.SECONDS);\n        }\n    }\n\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/timer/rocksdb/TimerRocksDBRecord.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.timer.rocksdb;\n\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class TimerRocksDBRecord {\n    private static final Logger logError = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);\n    public static final byte TIMER_ROCKSDB_PUT = (byte)0;\n    public static final byte TIMER_ROCKSDB_DELETE = (byte)1;\n    public static final byte TIMER_ROCKSDB_UPDATE = (byte)2;\n    private static final int VALUE_LENGTH = Integer.BYTES + Long.BYTES;\n\n    private long delayTime;\n    private String uniqKey;\n    private int sizePy;\n    private long offsetPy;\n    private long queueOffset;\n    private long checkPoint;\n    private byte actionFlag;\n    private MessageExt messageExt;\n\n    public TimerRocksDBRecord() {}\n\n    public TimerRocksDBRecord(long delayTime, String uniqKey, long offsetPy, int sizePy, long queueOffset, MessageExt messageExt) {\n        this.delayTime = delayTime;\n        this.uniqKey = uniqKey;\n        this.offsetPy = offsetPy;\n        this.sizePy = sizePy;\n        this.messageExt = messageExt;\n        this.queueOffset = queueOffset;\n    }\n\n    public byte[] getKeyBytes() {\n        if (StringUtils.isEmpty(uniqKey) || delayTime <= 0L) {\n            return null;\n        }\n        try {\n            byte[] uniqKeyBytes = uniqKey.getBytes(StandardCharsets.UTF_8);\n            int keyLength = Long.BYTES + uniqKeyBytes.length;\n            return ByteBuffer.allocate(keyLength).putLong(delayTime).put(uniqKeyBytes).array();\n        } catch (Exception e) {\n            logError.error(\"TimerRocksDBRecord getKeyBytes error: {}\", e.getMessage());\n            return null;\n        }\n    }\n\n    public byte[] getValueBytes() {\n        if (sizePy <= 0 || offsetPy < 0L) {\n            return null;\n        }\n        try {\n            return ByteBuffer.allocate(VALUE_LENGTH).putInt(sizePy).putLong(offsetPy).array();\n        } catch (Exception e) {\n            logError.error(\"TimerRocksDBRecord getValueBytes error: {}\", e.getMessage());\n            return null;\n        }\n    }\n\n    public static TimerRocksDBRecord decode(byte[] key, byte[] value) {\n        if (null == key || key.length < Long.BYTES || null == value || value.length != VALUE_LENGTH) {\n            return null;\n        }\n        try {\n            TimerRocksDBRecord rocksDBRecord = new TimerRocksDBRecord();\n            ByteBuffer keyBuffer = ByteBuffer.wrap(key);\n            rocksDBRecord.setDelayTime(keyBuffer.getLong());\n            byte[] uniqKey = new byte[key.length - Long.BYTES];\n            keyBuffer.get(uniqKey);\n            rocksDBRecord.setUniqKey(new String(uniqKey, StandardCharsets.UTF_8));\n            ByteBuffer valueByteBuffer = ByteBuffer.wrap(value);\n            rocksDBRecord.setSizePy(valueByteBuffer.getInt());\n            rocksDBRecord.setOffsetPy(valueByteBuffer.getLong());\n            return rocksDBRecord;\n        } catch (Exception e) {\n            logError.error(\"TimerRocksDBRecord decode error: {}\", e.getMessage());\n            return null;\n        }\n    }\n\n    public void setDelayTime(long delayTime) {\n        this.delayTime = delayTime;\n    }\n\n    public void setOffsetPy(long offsetPy) {\n        this.offsetPy = offsetPy;\n    }\n\n    public void setSizePy(int sizePy) {\n        this.sizePy = sizePy;\n    }\n\n    public int getSizePy() {\n        return sizePy;\n    }\n\n    public long getDelayTime() {\n        return delayTime;\n    }\n\n    public long getOffsetPy() {\n        return offsetPy;\n    }\n\n    public MessageExt getMessageExt() {\n        return messageExt;\n    }\n\n    public void setMessageExt(MessageExt messageExt) {\n        this.messageExt = messageExt;\n    }\n\n    public String getUniqKey() {\n        return uniqKey;\n    }\n\n    public void setUniqKey(String uniqKey) {\n        this.uniqKey = uniqKey;\n    }\n\n    public void setCheckPoint(long checkPoint) {\n        this.checkPoint = checkPoint;\n    }\n\n    public long getCheckPoint() {\n        return checkPoint;\n    }\n\n    public long getQueueOffset() {\n        return queueOffset;\n    }\n\n    public void setQueueOffset(long queueOffset) {\n        this.queueOffset = queueOffset;\n    }\n\n    public byte getActionFlag() {\n        return actionFlag;\n    }\n\n    public void setActionFlag(byte actionFlag) {\n        this.actionFlag = actionFlag;\n    }\n\n    @Override\n    public String toString() {\n        return \"TimerRocksDBRecord{\" +\n            \"delayTime=\" + delayTime +\n            \", uniqKey=\" + uniqKey +\n            \", sizePy=\" + sizePy +\n            \", offsetPy=\" + offsetPy +\n            \", queueOffset=\" + queueOffset +\n            \", checkPoint=\" + checkPoint +\n            '}';\n    }\n\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/transaction/TransMessageRocksDBStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.transaction;\nimport java.net.SocketAddress;\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.LinkedBlockingDeque;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.CommitLogDispatchStore;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.StoreUtil;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.rocksdb.MessageRocksDBStorage;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.rocksdb.RocksDBException;\nimport static org.apache.rocketmq.store.rocksdb.MessageRocksDBStorage.TRANS_COLUMN_FAMILY;\n\npublic class TransMessageRocksDBStore implements CommitLogDispatchStore {\n    private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);\n    private static final Logger logError = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);\n    private static final String REMOVE_TAG = \"d\";\n    private static final byte[] FILL_BYTE = new byte[] {(byte) 0};\n    private static final int DEFAULT_CAPACITY = 100000;\n    private static final int BATCH_SIZE = 1000;\n    private static final int MAX_GET_MSG_TIMES = 3;\n    private static final int INITIAL = 0, RUNNING = 1, SHUTDOWN = 2;\n    private volatile int state = INITIAL;\n\n    private final MessageStore messageStore;\n    private final MessageStoreConfig storeConfig;\n    private final MessageRocksDBStorage messageRocksDBStorage;\n    private final BrokerStatsManager brokerStatsManager;\n    private final SocketAddress storeHost;\n    private ThreadLocal<ByteBuffer> bufferLocal = null;\n    private TransIndexBuildService transIndexBuildService;\n    protected BlockingQueue<TransRocksDBRecord> originTransMsgQueue;\n\n    public TransMessageRocksDBStore(final MessageStore messageStore, final BrokerStatsManager brokerStatsManager, final SocketAddress storeHost) {\n        this.messageStore = messageStore;\n        this.storeConfig = messageStore.getMessageStoreConfig();\n        this.messageRocksDBStorage = messageStore.getMessageRocksDBStorage();\n        this.brokerStatsManager = brokerStatsManager;\n        this.storeHost = storeHost;\n        bufferLocal = ThreadLocal.withInitial(() -> ByteBuffer.allocate(storeConfig.getMaxMessageSize()));\n        if (storeConfig.isTransRocksDBEnable()) {\n            init();\n        }\n    }\n\n    private void init() {\n        if (this.state == RUNNING) {\n            return;\n        }\n        this.transIndexBuildService = new TransIndexBuildService();\n        this.originTransMsgQueue = new LinkedBlockingDeque<>(DEFAULT_CAPACITY);\n        this.transIndexBuildService.start();\n        this.state = RUNNING;\n        Long lastOffsetPy = messageRocksDBStorage.getLastOffsetPy(TRANS_COLUMN_FAMILY);\n        log.info(\"TransMessageRocksDBStore start success, lastOffsetPy: {}\", lastOffsetPy);\n    }\n\n    public void shutdown() {\n        if (this.state != RUNNING || this.state == SHUTDOWN) {\n            return;\n        }\n        if (null != this.transIndexBuildService) {\n            this.transIndexBuildService.shutdown();\n        }\n        this.state = SHUTDOWN;\n        log.info(\"TransMessageRocksDBStore shutdown success\");\n    }\n\n    public void buildTransIndex(DispatchRequest dispatchRequest) {\n        if (null == dispatchRequest || dispatchRequest.getCommitLogOffset() < 0L || dispatchRequest.getMsgSize() <= 0 || state != RUNNING || null == this.originTransMsgQueue) {\n            logError.error(\"TransMessageRocksDBStore buildTransIndex error, dispatchRequest: {}, state: {}, originTransMsgQueue: {}\", dispatchRequest, state, originTransMsgQueue);\n            return;\n        }\n        long reqOffsetPy = dispatchRequest.getCommitLogOffset();\n        long endOffsetPy = messageRocksDBStorage.getLastOffsetPy(TRANS_COLUMN_FAMILY);\n        if (reqOffsetPy < endOffsetPy) {\n            if (System.currentTimeMillis() % 1000 == 0) {\n                log.warn(\"TransMessageRocksDBStore buildTransIndex recover, but ignore, reqOffsetPy: {}, endOffsetPy: {}\", reqOffsetPy, endOffsetPy);\n            }\n            return;\n        }\n        int reqMsgSize = dispatchRequest.getMsgSize();\n        try {\n            MessageExt msgInner = getMessage(reqOffsetPy, reqMsgSize);\n            if (null == msgInner) {\n                logError.error(\"TransMessageRocksDBStore buildTransIndex error, msgInner is not found, reqOffsetPy: {}, reqMsgSize: {}\", reqOffsetPy, reqMsgSize);\n                return;\n            }\n            String topic = msgInner.getUserProperty(MessageConst.PROPERTY_REAL_TOPIC);\n            String uniqKey = msgInner.getUserProperty(MessageConst.PROPERTY_TRANSACTION_ID);\n            if (StringUtils.isEmpty(topic) || StringUtils.isEmpty(uniqKey)) {\n                logError.error(\"TransMessageRocksDBStore buildTransIndex error, uniqKey: {}, topic: {}\", uniqKey, topic);\n                return;\n            }\n            TransRocksDBRecord transRocksDBRecord = null;\n            String reqTopic = dispatchRequest.getTopic();\n            if (TopicValidator.RMQ_SYS_ROCKSDB_TRANS_HALF_TOPIC.equals(reqTopic)) {\n                transRocksDBRecord = new TransRocksDBRecord(reqOffsetPy, topic, uniqKey, reqMsgSize, 0);\n            } else if (TopicValidator.RMQ_SYS_ROCKSDB_TRANS_OP_HALF_TOPIC.equals(reqTopic)) {\n                long offsetPy = -1L;\n                String transOffsetPy = null;\n                try {\n                    transOffsetPy = msgInner.getUserProperty(MessageConst.PROPERTY_TRANS_OFFSET);\n                    if (!StringUtils.isEmpty(transOffsetPy)) {\n                        offsetPy = Long.parseLong(transOffsetPy);\n                    }\n                    if (offsetPy >= 0L) {\n                        transRocksDBRecord = new TransRocksDBRecord(offsetPy, topic, uniqKey, true);\n                    }\n                } catch (Exception e) {\n                    logError.error(\"TransMessageRocksDBStore buildTransIndex error, transOffsetPy: {}, error: {}\", transOffsetPy, e.getMessage());\n                }\n            }\n            if (null != transRocksDBRecord) {\n                while (!originTransMsgQueue.offer(transRocksDBRecord, 3, TimeUnit.SECONDS)) {\n                    if (System.currentTimeMillis() % 1000 == 0) {\n                        logError.error(\"TransMessageRocksDBStore buildTransStatus offer transRocksDBRecord error, topic: {}, uniqKey: {}, reqOffsetPy: {}\", topic, uniqKey, reqOffsetPy);\n                    }\n                }\n            }\n        } catch (Exception e) {\n            logError.error(\"TransMessageRocksDBStore buildTransStatus error: {}\", e.getMessage());\n        }\n    }\n\n    public void deletePrepareMessage(MessageExt messageExt) {\n        if (null == messageExt) {\n            logError.error(\"TransMessageRocksDBStore deletePrepareMessage error, messageExt is null\");\n            return;\n        }\n        try {\n            MessageExtBrokerInner msgInner = makeOpMessageInner(messageExt);\n            if (null == msgInner) {\n                logError.error(\"TransMessageRocksDBStore deletePrepareMessage msgInner is null\");\n                return;\n            }\n            PutMessageResult result = messageStore.putMessage(msgInner);\n            if (result != null && result.getPutMessageStatus() == PutMessageStatus.PUT_OK) {\n                this.brokerStatsManager.incTopicPutNums(msgInner.getTopic());\n                this.brokerStatsManager.incTopicPutSize(msgInner.getTopic(), result.getAppendMessageResult().getWroteBytes());\n                this.brokerStatsManager.incBrokerPutNums();\n                return;\n            }\n            logError.error(\"TransMessageRocksDBStore deletePrepareMessage put op msg failed, result: {}\", result);\n        } catch (Exception e) {\n            logError.error(\"TransMessageRocksDBStore deletePrepareMessage error: {}\", e.getMessage());\n        }\n    }\n\n    public MessageExt getMessage(long offsetPy, int sizePy) {\n        if (offsetPy < 0L || sizePy <= 0 || sizePy > storeConfig.getMaxMessageSize()) {\n            logError.error(\"TransMessageRocksDBStore getMessage param error, offsetPy: {}, sizePy: {}, maxMsgSizeConfig: {}\", offsetPy, sizePy, storeConfig.getMaxMessageSize());\n            return null;\n        }\n        ByteBuffer byteBuffer = bufferLocal.get();\n        if (sizePy > byteBuffer.limit()) {\n            bufferLocal.remove();\n            byteBuffer = ByteBuffer.allocate(sizePy);\n            bufferLocal.set(byteBuffer);\n        }\n        for (int i = 0; i < MAX_GET_MSG_TIMES; i++) {\n            try {\n                MessageExt msgExt = StoreUtil.getMessage(offsetPy, sizePy, messageStore, bufferLocal.get());\n                if (null == msgExt) {\n                    log.warn(\"Fail to read msg from commitLog offsetPy:{} sizePy:{}\", offsetPy, sizePy);\n                } else {\n                    return msgExt;\n                }\n            } catch (Exception e) {\n                logError.error(\"TransMessageRocksDBStore getMessage error, offsetPy: {}, sizePy: {}, error: {}\", offsetPy, sizePy, e.getMessage());\n            }\n        }\n        return null;\n    }\n\n    public MessageRocksDBStorage getMessageRocksDBStorage() {\n        return messageRocksDBStorage;\n    }\n\n    private MessageExtBrokerInner makeOpMessageInner(MessageExt messageExt) {\n        if (null == messageExt) {\n            logError.error(\"TransMessageRocksDBStore makeOpMessageInner messageExt is null\");\n            return null;\n        }\n        try {\n            MessageExtBrokerInner msgInner = new MessageExtBrokerInner();\n            msgInner.setTopic(TopicValidator.RMQ_SYS_ROCKSDB_TRANS_OP_HALF_TOPIC);\n            msgInner.setBody(FILL_BYTE);\n            msgInner.setQueueId(0);\n            msgInner.setTags(REMOVE_TAG);\n            msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(msgInner.getTags()));\n            msgInner.setSysFlag(0);\n            msgInner.setBornTimestamp(System.currentTimeMillis());\n            msgInner.setBornHost(this.storeHost);\n            msgInner.setStoreHost(this.storeHost);\n            msgInner.setWaitStoreMsgOK(false);\n            MessageClientIDSetter.setUniqID(msgInner);\n            String uniqKey = MessageClientIDSetter.getUniqID(messageExt);\n            if (!StringUtils.isEmpty(uniqKey)) {\n                MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_ID, uniqKey);\n            }\n            MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRANS_OFFSET, String.valueOf(messageExt.getCommitLogOffset()));\n            MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_REAL_TOPIC, messageExt.getUserProperty(MessageConst.PROPERTY_REAL_TOPIC));\n            msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));\n            return msgInner;\n        } catch (Exception e) {\n            logError.error(\"TransMessageRocksDBStore makeOpMessageInner error: {}\", e.getMessage());\n            return null;\n        }\n    }\n\n    public Integer getCheckTimes(String topic, String uniqKey, Long offsetPy) {\n        if (StringUtils.isEmpty(topic) || StringUtils.isEmpty(uniqKey) || null == offsetPy || offsetPy < 0L) {\n            return null;\n        }\n        try {\n            TransRocksDBRecord record = messageRocksDBStorage.getRecordForTrans(TRANS_COLUMN_FAMILY, new TransRocksDBRecord(offsetPy, topic, uniqKey, false));\n            if (null == record) {\n                return null;\n            }\n            return record.getCheckTimes();\n        } catch (Exception e) {\n            logError.error(\"TransMessageRocksDBStore getCheckTimes error, topic: {}, uniqKey: {}, offsetPy: {}, error: {}\", topic, uniqKey, offsetPy, e.getMessage());\n            return null;\n        }\n    }\n\n    public boolean isMappedFileMatchedRecover(long phyOffset, long storeTimestamp,\n        boolean recoverNormally) throws RocksDBException {\n        if (!storeConfig.isTransRocksDBEnable()) {\n            return true;\n        }\n        Long lastOffsetPy = messageRocksDBStorage.getLastOffsetPy(TRANS_COLUMN_FAMILY);\n        log.info(\"trans isMappedFileMatchedRecover lastOffsetPy: {}\", lastOffsetPy);\n        if (null != lastOffsetPy && phyOffset <= lastOffsetPy) {\n            log.info(\"isMappedFileMatchedRecover TransMessageRocksDBStore recover form this offset, phyOffset: {}, lastOffsetPy: {}\", phyOffset, lastOffsetPy);\n            return true;\n        }\n        return false;\n    }\n\n    private String getServiceThreadName() {\n        String brokerIdentifier = \"\";\n        if (TransMessageRocksDBStore.this.messageStore instanceof DefaultMessageStore) {\n            DefaultMessageStore messageStore = (DefaultMessageStore) TransMessageRocksDBStore.this.messageStore;\n            if (messageStore.getBrokerConfig().isInBrokerContainer()) {\n                brokerIdentifier = messageStore.getBrokerConfig().getIdentifier();\n            }\n        }\n        return brokerIdentifier;\n    }\n\n    public class TransIndexBuildService extends ServiceThread {\n        private final Logger log = TransMessageRocksDBStore.log;\n        private List<TransRocksDBRecord> trs;\n        @Override\n        public String getServiceName() {\n            return getServiceThreadName() + this.getClass().getSimpleName();\n        }\n\n        @Override\n        public void run() {\n            log.info(this.getServiceName() + \"service start\");\n            trs = new ArrayList<>(BATCH_SIZE);\n            while (!this.isStopped() || !originTransMsgQueue.isEmpty()) {\n                try {\n                    buildTransIndex();\n                } catch (Exception e) {\n                    trs.clear();\n                    logError.error(\"TransMessageRocksDBStore error occurred in: {}, error: {}\", getServiceName(), e.getMessage());\n                }\n            }\n            log.info(this.getServiceName() + \" service end\");\n        }\n\n        protected void buildTransIndex() {\n            pollTransMessageRecords();\n            if (CollectionUtils.isEmpty(trs)) {\n                return;\n            }\n            try {\n                messageRocksDBStorage.writeRecordsForTrans(TRANS_COLUMN_FAMILY, trs);\n            } catch (Exception e) {\n                logError.error(\"TransMessageRocksDBStore pollAndPutTransRequest writeRecords error: {}\", e.getMessage());\n            }\n            trs.clear();\n        }\n\n        protected void pollTransMessageRecords() {\n            try {\n                TransRocksDBRecord firstReq = originTransMsgQueue.poll(100, TimeUnit.MILLISECONDS);\n                if (null != firstReq) {\n                    trs.add(firstReq);\n                    while (true) {\n                        TransRocksDBRecord tmpReq = originTransMsgQueue.poll(100, TimeUnit.MILLISECONDS);\n                        if (null == tmpReq) {\n                            break;\n                        }\n                        trs.add(tmpReq);\n                        if (trs.size() >= BATCH_SIZE) {\n                            break;\n                        }\n                    }\n                }\n            } catch (Exception e) {\n                logError.error(\"TransMessageRocksDBStore fetchTransMessageRecord error: {}\", e.getMessage());\n            }\n        }\n    }\n\n    @Override\n    public Long getDispatchFromPhyOffset(boolean recoverNormally) throws RocksDBException {\n        if (!storeConfig.isTransRocksDBEnable()) {\n            return null;\n        }\n        Long dispatchFromTransPhyOffset = messageRocksDBStorage.getLastOffsetPy(TRANS_COLUMN_FAMILY);\n        if (dispatchFromTransPhyOffset != null && dispatchFromTransPhyOffset > 0) {\n            return dispatchFromTransPhyOffset;\n        }\n        return null;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/transaction/TransRocksDBRecord.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.transaction;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class TransRocksDBRecord {\n    private static final Logger logError = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);\n    public static final int VALUE_LENGTH = Integer.BYTES + Integer.BYTES;\n    private static final String KEY_SPLIT = \"@\";\n    protected long offsetPy;\n    private String topic;\n    private String uniqKey;\n    private int checkTimes = 0;\n    private int sizePy;\n    private boolean isOp;\n    private boolean delete;\n    private MessageExt messageExt;\n\n    public TransRocksDBRecord(long offsetPy, String topic, String uniqKey, int sizePy, int checkTimes) {\n        this.offsetPy = offsetPy;\n        this.topic = topic;\n        this.uniqKey = uniqKey;\n        this.sizePy = sizePy;\n        this.checkTimes = checkTimes;\n    }\n\n    public TransRocksDBRecord(long offsetPy, String topic, String uniqKey, boolean isOp) {\n        this.offsetPy = offsetPy;\n        this.topic = topic;\n        this.uniqKey = uniqKey;\n        this.isOp = isOp;\n    }\n\n    public TransRocksDBRecord() {}\n\n    public byte[] getKeyBytes() {\n        if (offsetPy < 0L || StringUtils.isEmpty(topic) || StringUtils.isEmpty(uniqKey)) {\n            return null;\n        }\n        byte[] keySuffixBytes = (KEY_SPLIT + topic + KEY_SPLIT + uniqKey).getBytes(StandardCharsets.UTF_8);\n        int keyLength = Long.BYTES + keySuffixBytes.length;\n        return ByteBuffer.allocate(keyLength).putLong(offsetPy).put(keySuffixBytes).array();\n    }\n\n    public byte[] getValueBytes() {\n        if (checkTimes < 0 || sizePy <= 0) {\n            logError.error(\"TransRocksDBRecord getValueBytes error, checkTimes: {}, sizePy: {}\", checkTimes, sizePy);\n            return null;\n        }\n        return ByteBuffer.allocate(VALUE_LENGTH).putInt(checkTimes).putInt(sizePy).array();\n    }\n\n    public static TransRocksDBRecord decode(byte[] key, byte[] value) {\n        if (null == key || key.length <= Long.BYTES || null == value || value.length != VALUE_LENGTH) {\n            logError.error(\"TransRocksDBRecord decode param error, key: {}, value: {}\", key, value);\n            return null;\n        }\n        TransRocksDBRecord transRocksDBRecord = null;\n        try {\n            transRocksDBRecord = new TransRocksDBRecord();\n            ByteBuffer keyByteBuffer = ByteBuffer.wrap(key);\n            transRocksDBRecord.setOffsetPy(keyByteBuffer.getLong());\n            byte[] keySuffix = new byte[key.length - Long.BYTES];\n            keyByteBuffer.get(keySuffix);\n            String[] keySuffixSplit = new String(keySuffix, StandardCharsets.UTF_8).split(KEY_SPLIT);\n            if (keySuffixSplit.length != 3) {\n                logError.error(\"TransRocksDBRecord decode keySuffixSplit parse error\");\n                return null;\n            }\n            transRocksDBRecord.setTopic(keySuffixSplit[1]);\n            transRocksDBRecord.setUniqKey(keySuffixSplit[2]);\n            ByteBuffer valueByteBuffer = ByteBuffer.wrap(value);\n            transRocksDBRecord.setCheckTimes(valueByteBuffer.getInt());\n            transRocksDBRecord.setSizePy(valueByteBuffer.getInt());\n        } catch (Exception e) {\n            logError.error(\"TransRocksDBRecord decode error, valueLength: {}, error: {}\", value.length, e.getMessage());\n            return null;\n        }\n        return transRocksDBRecord;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getUniqKey() {\n        return uniqKey;\n    }\n\n    public void setUniqKey(String uniqKey) {\n        this.uniqKey = uniqKey;\n    }\n\n    public int getCheckTimes() {\n        return checkTimes;\n    }\n\n    public void setCheckTimes(int checkTimes) {\n        this.checkTimes = checkTimes;\n    }\n\n    public int getSizePy() {\n        return sizePy;\n    }\n\n    public void setSizePy(int sizePy) {\n        this.sizePy = sizePy;\n    }\n\n    public long getOffsetPy() {\n        return offsetPy;\n    }\n\n    public void setOffsetPy(long offsetPy) {\n        this.offsetPy = offsetPy;\n    }\n\n    public MessageExt getMessageExt() {\n        return messageExt;\n    }\n\n    public void setMessageExt(MessageExt messageExt) {\n        this.messageExt = messageExt;\n    }\n\n    public boolean isOp() {\n        return isOp;\n    }\n\n    public void setOp(boolean op) {\n        isOp = op;\n    }\n\n    public boolean isDelete() {\n        return delete;\n    }\n\n    public void setDelete(boolean delete) {\n        this.delete = delete;\n    }\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/util/LibC.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.util;\n\nimport com.sun.jna.Library;\nimport com.sun.jna.Native;\nimport com.sun.jna.NativeLong;\nimport com.sun.jna.Platform;\nimport com.sun.jna.Pointer;\n\npublic interface LibC extends Library {\n    LibC INSTANCE = (LibC) Native.loadLibrary(Platform.isWindows() ? \"msvcrt\" : \"c\", LibC.class);\n\n    int MADV_NORMAL = 0;\n    int MADV_RANDOM = 1;\n    int MADV_WILLNEED = 3;\n    int MADV_DONTNEED = 4;\n\n    int MCL_CURRENT = 1;\n    int MCL_FUTURE = 2;\n    int MCL_ONFAULT = 4;\n\n    /* sync memory asynchronously */\n    int MS_ASYNC = 0x0001;\n    /* invalidate mappings & caches */\n    int MS_INVALIDATE = 0x0002;\n    /* synchronous memory sync */\n    int MS_SYNC = 0x0004;\n\n    int mlock(Pointer var1, NativeLong var2);\n\n    int munlock(Pointer var1, NativeLong var2);\n\n    int madvise(Pointer var1, NativeLong var2, int var3);\n\n    Pointer memset(Pointer p, int v, long len);\n\n    int mlockall(int flags);\n\n    int msync(Pointer p, NativeLong length, int flags);\n\n    int mincore(Pointer p, NativeLong length, byte[] vec);\n\n    int getpagesize();\n}\n"
  },
  {
    "path": "store/src/main/java/org/apache/rocketmq/store/util/PerfCounter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.util;\n\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\n\nimport java.sql.Timestamp;\nimport java.util.Iterator;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicLong;\n\npublic class PerfCounter {\n\n    private long last = System.currentTimeMillis();\n    private float lastTps = 0.0f;\n\n    private final ThreadLocal<AtomicLong> lastTickMs = new ThreadLocal<AtomicLong>() {\n        @Override\n        protected AtomicLong initialValue() {\n            return new AtomicLong(System.currentTimeMillis());\n        }\n    };\n\n    private final Logger logger;\n    private String prefix = \"DEFAULT\";\n\n    public float getLastTps() {\n        if (System.currentTimeMillis() - last <= maxTimeMsPerCount  + 3000) {\n            return lastTps;\n        }\n        return 0.0f;\n    }\n\n    //1000 * ms, 1000 * 10 ms, then 100ms every slots\n    private final AtomicInteger[] count;\n    private final AtomicLong allCount;\n    private final int maxNumPerCount;\n    private final int maxTimeMsPerCount;\n\n\n    public PerfCounter() {\n        this(5001, null, null, 1000 * 1000, 10 * 1000);\n    }\n\n    public PerfCounter(int slots, Logger logger, String prefix, int maxNumPerCount, int maxTimeMsPerCount) {\n        if (slots < 3000) {\n            throw new RuntimeException(\"slots must bigger than 3000, but:%s\" + slots);\n        }\n        count = new AtomicInteger[slots];\n        allCount = new AtomicLong(0);\n        this.logger = logger;\n        if (prefix != null) {\n            this.prefix = prefix;\n        }\n        this.maxNumPerCount = maxNumPerCount;\n        this.maxTimeMsPerCount = maxTimeMsPerCount;\n        reset();\n    }\n\n    public void flow(long cost) {\n        flow(cost, 1);\n    }\n\n    public void flow(long cost, int num) {\n        if (cost < 0) return;\n        allCount.addAndGet(num);\n        count[getIndex(cost)].addAndGet(num);\n        if (allCount.get() >= maxNumPerCount\n            || System.currentTimeMillis() - last >= maxTimeMsPerCount) {\n            synchronized (allCount) {\n                if (allCount.get() < maxNumPerCount\n                    && System.currentTimeMillis() - last < maxTimeMsPerCount) {\n                    return;\n                }\n                print();\n                this.reset();\n            }\n        }\n    }\n\n    public void print() {\n        int min = this.getMin();\n        int max = this.getMax();\n        int tp50 = this.getTPValue(0.5f);\n        int tp80 = this.getTPValue(0.8f);\n        int tp90 = this.getTPValue(0.9f);\n        int tp99 = this.getTPValue(0.99f);\n        int tp999 = this.getTPValue(0.999f);\n        long count0t1 = this.getCount(0, 1);\n        long count2t5 = this.getCount(2, 5);\n        long count6t10 = this.getCount(6, 10);\n        long count11t50 = this.getCount(11, 50);\n        long count51t100 = this.getCount(51, 100);\n        long count101t500 = this.getCount(101, 500);\n        long count501t999 = this.getCount(501, 999);\n        long count1000t = this.getCount(1000, 100000000);\n        long elapsed = System.currentTimeMillis() - last;\n        lastTps = (allCount.get() + 0.1f) * 1000 / elapsed;\n        String str = String.format(\"PERF_COUNTER_%s[%s] num:%d cost:%d tps:%.4f min:%d max:%d tp50:%d tp80:%d tp90:%d tp99:%d tp999:%d \" +\n                \"0_1:%d 2_5:%d 6_10:%d 11_50:%d 51_100:%d 101_500:%d 501_999:%d 1000_:%d\",\n            prefix, new Timestamp(System.currentTimeMillis()), allCount.get(), elapsed, lastTps,\n            min, max, tp50, tp80, tp90, tp99, tp999,\n            count0t1, count2t5, count6t10, count11t50, count51t100, count101t500, count501t999, count1000t);\n        if (logger != null) {\n            logger.info(str);\n        }\n    }\n\n    private int getIndex(long cost) {\n        if (cost < 1000) {\n            return (int) cost;\n        }\n        if (cost >= 1000 && cost < 1000 + 1000 * 10) {\n            int units = (int) ((cost - 1000) / 10);\n            return 1000 + units;\n        }\n        int units = (int) ((cost - 1000 - 1000 * 10) / 100);\n        units = 2000 + units;\n        if (units >= count.length) {\n            units = count.length - 1;\n        }\n        return units;\n    }\n\n    private int convert(int index) {\n        if (index < 1000) {\n            return index;\n        } else if (index >= 1000 && index < 2000) {\n            return (index - 1000) * 10 + 1000;\n        } else {\n            return (index - 2000) * 100 + 1000 * 10 + 1000;\n        }\n    }\n\n    public float getRate(int from, int to) {\n        long tmp = getCount(from, to);\n        return ((tmp + 0.0f) * 100) / (allCount.get() + 1);\n    }\n\n    public long getCount(int from, int to) {\n        from = getIndex(from);\n        to = getIndex(to);\n        long tmp = 0;\n        for (int i = from; i <= to && i < count.length; i++) {\n            tmp = tmp + count[i].get();\n        }\n        return tmp;\n    }\n\n    public int getTPValue(float ratio) {\n        if (ratio <= 0 || ratio >= 1) {\n            ratio = 0.99f;\n        }\n        long num = (long) (allCount.get() * (1 - ratio));\n        int tmp = 0;\n        for (int i = count.length - 1; i > 0; i--) {\n            tmp += count[i].get();\n            if (tmp > num) {\n                return convert(i);\n            }\n        }\n        return 0;\n    }\n\n    public int getMin() {\n        for (int i = 0; i < count.length; i++) {\n            if (count[i].get() > 0) {\n                return convert(i);\n            }\n        }\n        return 0;\n    }\n\n    public int getMax() {\n        for (int i = count.length - 1; i > 0; i--) {\n            if (count[i].get() > 0) {\n                return convert(i);\n            }\n        }\n        return 99999999;\n    }\n\n    public void reset() {\n        for (int i = 0; i < count.length; i++) {\n            if (count[i] == null) {\n                count[i] = new AtomicInteger(0);\n            } else {\n                count[i].set(0);\n            }\n        }\n        allCount.set(0);\n        last = System.currentTimeMillis();\n    }\n\n    public void startTick() {\n        lastTickMs.get().set(System.currentTimeMillis());\n    }\n\n    public void endTick() {\n        flow(System.currentTimeMillis() - lastTickMs.get().get());\n    }\n\n    public static class Ticks extends ServiceThread {\n        private final Logger logger;\n        private final Map<String, PerfCounter> perfs = new ConcurrentHashMap<>();\n        private final Map<String, AtomicLong> keyFreqs = new ConcurrentHashMap<>();\n        private final PerfCounter defaultPerf;\n        private final AtomicLong defaultTime = new AtomicLong(System.currentTimeMillis());\n\n        private final int maxKeyNumPerf;\n        private final int maxKeyNumDebug;\n\n        private final int maxNumPerCount;\n        private final int maxTimeMsPerCount;\n\n        public Ticks() {\n            this(null, 1000 * 1000, 10 * 1000, 20 * 1000, 100 * 1000);\n        }\n\n        public Ticks(Logger logger) {\n            this(logger, 1000 * 1000, 10 * 1000, 20 * 1000, 100 * 1000);\n        }\n\n        @Override\n        public String getServiceName() {\n            return this.getClass().getName();\n        }\n\n        public Ticks(Logger logger, int maxNumPerCount, int maxTimeMsPerCount, int maxKeyNumPerf, int maxKeyNumDebug) {\n            this.logger = logger;\n            this.maxNumPerCount = maxNumPerCount;\n            this.maxTimeMsPerCount = maxTimeMsPerCount;\n            this.maxKeyNumPerf =  maxKeyNumPerf;\n            this.maxKeyNumDebug =  maxKeyNumDebug;\n            this.defaultPerf = new PerfCounter(3001, logger, null, maxNumPerCount, maxTimeMsPerCount);\n\n        }\n\n        private PerfCounter makeSureExists(String key) {\n            if (perfs.get(key) == null) {\n                if (perfs.size() >= maxKeyNumPerf + 100) {\n                    return defaultPerf;\n                }\n                perfs.put(key, new PerfCounter(3001, logger, key, maxNumPerCount, maxTimeMsPerCount));\n            }\n            return perfs.getOrDefault(key, defaultPerf);\n        }\n\n        public void startTick(String key) {\n            try {\n                makeSureExists(key).startTick();\n            } catch (Throwable ignored) {\n\n            }\n        }\n\n        public void endTick(String key) {\n            try {\n                makeSureExists(key).endTick();\n            } catch (Throwable ignored) {\n\n            }\n        }\n\n        public void flowOnce(String key, int cost) {\n            try {\n                makeSureExists(key).flow(cost);\n            } catch (Throwable ignored) {\n\n            }\n        }\n\n        public PerfCounter getCounter(String key) {\n            try {\n                return makeSureExists(key);\n            } catch (Throwable ignored) {\n                return defaultPerf;\n            }\n        }\n\n        private AtomicLong makeSureDebugKeyExists(String key) {\n            AtomicLong lastTimeMs = keyFreqs.get(key);\n            if (null == lastTimeMs) {\n                if (keyFreqs.size() >= maxKeyNumDebug + 100) {\n                    return defaultTime;\n                }\n                lastTimeMs = new AtomicLong(0);\n                keyFreqs.put(key, lastTimeMs);\n            }\n            return keyFreqs.getOrDefault(key, defaultTime);\n        }\n        public boolean shouldDebugKeyAndTimeMs(String key, int intervalMs) {\n            try {\n                AtomicLong lastTimeMs =  makeSureDebugKeyExists(key);\n                if (System.currentTimeMillis() - lastTimeMs.get() > intervalMs) {\n                    lastTimeMs.set(System.currentTimeMillis());\n                    return true;\n                }\n                return false;\n            } catch (Throwable ignored) {\n\n            }\n            return false;\n        }\n\n        @Override\n        public void run() {\n            logger.info(\"{} get started\", getServiceName());\n            while (!this.isStopped()) {\n                try {\n                    long maxLiveTimeMs = maxTimeMsPerCount * 2 + 1000;\n                    this.waitForRunning(maxLiveTimeMs);\n                    if (perfs.size() >= maxKeyNumPerf\n                            || keyFreqs.size() >= maxKeyNumDebug) {\n                        logger.warn(\"The key is full {}-{} {}-{}\", perfs.size(), maxKeyNumPerf, keyFreqs.size(), maxKeyNumDebug);\n                    }\n                    {\n                        Iterator<Map.Entry<String, PerfCounter>> it = perfs.entrySet().iterator();\n                        while (it.hasNext()) {\n                            Map.Entry<String, PerfCounter> entry = it.next();\n                            PerfCounter value = entry.getValue();\n                            // May have concurrency problem, but it has no effect, we can ignore it.\n                            if (System.currentTimeMillis() - value.last > maxLiveTimeMs) {\n                                it.remove();\n                            }\n                        }\n                    }\n\n                    {\n                        Iterator<Map.Entry<String, AtomicLong>> it = keyFreqs.entrySet().iterator();\n                        while (it.hasNext()) {\n                            Map.Entry<String, AtomicLong> entry = it.next();\n                            AtomicLong value = entry.getValue();\n                            // May have concurrency problem, but it has no effect, we can ignore it.\n                            if (System.currentTimeMillis() - value.get() > maxLiveTimeMs) {\n                                it.remove();\n                            }\n                        }\n                    }\n\n                } catch (Exception e) {\n                    logger.error(\"{} get unknown error\", getServiceName(), e);\n                    try {\n                        Thread.sleep(1000);\n                    } catch (Throwable ignored) {\n\n                    }\n                }\n            }\n            logger.info(\"{} get stopped\", getServiceName());\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/AppendCallbackTest.java",
    "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\n\nimport java.io.File;\nimport java.net.InetSocketAddress;\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBatch;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\npublic class AppendCallbackTest {\n\n    AppendMessageCallback callback;\n\n    MessageExtEncoder batchEncoder;\n\n    @Before\n    public void init() throws Exception {\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setMappedFileSizeCommitLog(1024 * 8);\n        messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 4);\n        messageStoreConfig.setMaxHashSlotNum(100);\n        messageStoreConfig.setMaxIndexNum(100 * 10);\n        messageStoreConfig.setMaxMessageSize(10 * 1024 * 1024);\n        messageStoreConfig.setStorePathRootDir(System.getProperty(\"java.io.tmpdir\") + File.separator + \"unitteststore\");\n        messageStoreConfig.setStorePathCommitLog(System.getProperty(\"java.io.tmpdir\") + File.separator + \"unitteststore\" + File.separator + \"commitlog\");\n        //too much reference\n        DefaultMessageStore messageStore = new DefaultMessageStore(messageStoreConfig, null, null, new BrokerConfig(), new ConcurrentHashMap<>());\n        CommitLog commitLog = new CommitLog(messageStore);\n        callback = commitLog.new DefaultAppendMessageCallback(messageStoreConfig);\n        batchEncoder = new MessageExtEncoder(messageStoreConfig);\n    }\n\n    @After\n    public void destroy() {\n        UtilAll.deleteFile(new File(System.getProperty(\"java.io.tmpdir\") + File.separator + \"unitteststore\"));\n    }\n\n    @Test\n    public void testAppendMessageBatchEndOfFile() throws Exception {\n        List<Message> messages = new ArrayList<>();\n        String topic = \"test-topic\";\n        int queue = 0;\n        for (int i = 0; i < 10; i++) {\n            Message msg = new Message();\n            msg.setBody(\"body\".getBytes());\n            msg.setTopic(topic);\n            msg.setTags(\"abc\");\n            messages.add(msg);\n        }\n        MessageExtBatch messageExtBatch = new MessageExtBatch();\n        messageExtBatch.setTopic(topic);\n        messageExtBatch.setQueueId(queue);\n        messageExtBatch.setBornTimestamp(System.currentTimeMillis());\n        messageExtBatch.setBornHost(new InetSocketAddress(\"127.0.0.1\", 123));\n        messageExtBatch.setStoreHost(new InetSocketAddress(\"127.0.0.1\", 124));\n        messageExtBatch.setBody(MessageDecoder.encodeMessages(messages));\n\n        PutMessageContext putMessageContext = new PutMessageContext(topic + \"-\" + queue);\n        messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch, putMessageContext));\n        ByteBuffer buff = ByteBuffer.allocate(1024 * 10);\n        //encounter end of file when append half of the data\n        AppendMessageResult result =\n                callback.doAppend(0, buff, 1000, messageExtBatch, putMessageContext);\n        assertEquals(AppendMessageStatus.END_OF_FILE, result.getStatus());\n        assertEquals(0, result.getWroteOffset());\n        assertEquals(0, result.getLogicsOffset());\n        assertEquals(1000, result.getWroteBytes());\n        assertEquals(8, buff.position()); //write blank size and magic value\n\n        assertTrue(result.getMsgId().length() > 0); //should have already constructed some message ids\n    }\n\n    @Test\n    public void testAppendIPv6HostMessageBatchEndOfFile() throws Exception {\n        List<Message> messages = new ArrayList<>();\n        String topic = \"test-topic\";\n        int queue = 0;\n        for (int i = 0; i < 10; i++) {\n            Message msg = new Message();\n            msg.setBody(\"body\".getBytes());\n            msg.setTopic(topic);\n            msg.setTags(\"abc\");\n            messages.add(msg);\n        }\n        MessageExtBatch messageExtBatch = new MessageExtBatch();\n        messageExtBatch.setTopic(topic);\n        messageExtBatch.setQueueId(queue);\n        messageExtBatch.setBornTimestamp(System.currentTimeMillis());\n        messageExtBatch.setMsgId(\"24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0\");\n        messageExtBatch.setSysFlag(0);\n        messageExtBatch.setBornHostV6Flag();\n        messageExtBatch.setStoreHostAddressV6Flag();\n        messageExtBatch.setBornHost(new InetSocketAddress(\"1050:0000:0000:0000:0005:0600:300c:326b\", 123));\n        messageExtBatch.setStoreHost(new InetSocketAddress(\"::1\", 124));\n        messageExtBatch.setBody(MessageDecoder.encodeMessages(messages));\n\n        PutMessageContext putMessageContext = new PutMessageContext(topic + \"-\" + queue);\n        messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch, putMessageContext));\n        ByteBuffer buff = ByteBuffer.allocate(1024 * 10);\n        //encounter end of file when append half of the data\n        AppendMessageResult result =\n                callback.doAppend(0, buff, 1000, messageExtBatch, putMessageContext);\n        assertEquals(AppendMessageStatus.END_OF_FILE, result.getStatus());\n        assertEquals(0, result.getWroteOffset());\n        assertEquals(0, result.getLogicsOffset());\n        assertEquals(1000, result.getWroteBytes());\n        assertEquals(8, buff.position()); //write blank size and magic value\n\n        assertTrue(result.getMsgId().length() > 0); //should have already constructed some message ids\n    }\n\n    @Test\n    public void testAppendMessageBatchSucc() throws Exception {\n        List<Message> messages = new ArrayList<>();\n        String topic = \"test-topic\";\n        int queue = 0;\n        for (int i = 0; i < 10; i++) {\n            Message msg = new Message();\n            msg.setBody(\"body\".getBytes());\n            msg.setTopic(topic);\n            msg.setTags(\"abc\");\n            messages.add(msg);\n        }\n        MessageExtBatch messageExtBatch = new MessageExtBatch();\n        messageExtBatch.setTopic(topic);\n        messageExtBatch.setQueueId(queue);\n        messageExtBatch.setBornTimestamp(System.currentTimeMillis());\n        messageExtBatch.setBornHost(new InetSocketAddress(\"127.0.0.1\", 123));\n        messageExtBatch.setStoreHost(new InetSocketAddress(\"127.0.0.1\", 124));\n        messageExtBatch.setBody(MessageDecoder.encodeMessages(messages));\n\n        PutMessageContext putMessageContext = new PutMessageContext(topic + \"-\" + queue);\n        messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch, putMessageContext));\n        ByteBuffer buff = ByteBuffer.allocate(1024 * 10);\n        AppendMessageResult allresult =\n                callback.doAppend(0, buff, 1024 * 10, messageExtBatch, putMessageContext);\n\n        assertEquals(AppendMessageStatus.PUT_OK, allresult.getStatus());\n        assertEquals(0, allresult.getWroteOffset());\n        assertEquals(0, allresult.getLogicsOffset());\n        assertEquals(buff.position(), allresult.getWroteBytes());\n\n        assertEquals(messages.size(), allresult.getMsgNum());\n\n        Set<String> msgIds = new HashSet<>();\n        for (String msgId : allresult.getMsgId().split(\",\")) {\n            assertEquals(32, msgId.length());\n            msgIds.add(msgId);\n        }\n        assertEquals(messages.size(), msgIds.size());\n\n        List<MessageExt> decodeMsgs = MessageDecoder.decodes((ByteBuffer) buff.flip());\n        assertEquals(decodeMsgs.size(), decodeMsgs.size());\n        long queueOffset = decodeMsgs.get(0).getQueueOffset();\n        long storeTimeStamp = decodeMsgs.get(0).getStoreTimestamp();\n        for (int i = 0; i < messages.size(); i++) {\n            assertEquals(messages.get(i).getTopic(), decodeMsgs.get(i).getTopic());\n            assertEquals(new String(messages.get(i).getBody()), new String(decodeMsgs.get(i).getBody()));\n            assertEquals(messages.get(i).getTags(), decodeMsgs.get(i).getTags());\n\n            assertEquals(messageExtBatch.getBornHostNameString(), decodeMsgs.get(i).getBornHostNameString());\n\n            assertEquals(messageExtBatch.getBornTimestamp(), decodeMsgs.get(i).getBornTimestamp());\n            assertEquals(storeTimeStamp, decodeMsgs.get(i).getStoreTimestamp());\n            assertEquals(queueOffset++, decodeMsgs.get(i).getQueueOffset());\n        }\n\n    }\n\n    @Test\n    public void testAppendIPv6HostMessageBatchSucc() throws Exception {\n        List<Message> messages = new ArrayList<>();\n        String topic = \"test-topic\";\n        int queue = 0;\n        for (int i = 0; i < 10; i++) {\n            Message msg = new Message();\n            msg.setBody(\"body\".getBytes());\n            msg.setTopic(topic);\n            msg.setTags(\"abc\");\n            messages.add(msg);\n        }\n        MessageExtBatch messageExtBatch = new MessageExtBatch();\n        messageExtBatch.setTopic(topic);\n        messageExtBatch.setQueueId(queue);\n        messageExtBatch.setBornTimestamp(System.currentTimeMillis());\n        messageExtBatch.setMsgId(\"24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0\");\n        messageExtBatch.setSysFlag(0);\n        messageExtBatch.setBornHostV6Flag();\n        messageExtBatch.setStoreHostAddressV6Flag();\n        messageExtBatch.setBornHost(new InetSocketAddress(\"1050:0000:0000:0000:0005:0600:300c:326b\", 123));\n        messageExtBatch.setStoreHost(new InetSocketAddress(\"::1\", 124));\n        messageExtBatch.setBody(MessageDecoder.encodeMessages(messages));\n\n        PutMessageContext putMessageContext = new PutMessageContext(topic + \"-\" + queue);\n        messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch, putMessageContext));\n        ByteBuffer buff = ByteBuffer.allocate(1024 * 10);\n        AppendMessageResult allresult =\n                callback.doAppend(0, buff, 1024 * 10, messageExtBatch, putMessageContext);\n\n        assertEquals(AppendMessageStatus.PUT_OK, allresult.getStatus());\n        assertEquals(0, allresult.getWroteOffset());\n        assertEquals(0, allresult.getLogicsOffset());\n        assertEquals(buff.position(), allresult.getWroteBytes());\n\n        assertEquals(messages.size(), allresult.getMsgNum());\n\n        Set<String> msgIds = new HashSet<>();\n        for (String msgId : allresult.getMsgId().split(\",\")) {\n            assertEquals(56, msgId.length());\n            msgIds.add(msgId);\n        }\n        assertEquals(messages.size(), msgIds.size());\n\n        List<MessageExt> decodeMsgs = MessageDecoder.decodes((ByteBuffer) buff.flip());\n        assertEquals(decodeMsgs.size(), decodeMsgs.size());\n        long queueOffset = decodeMsgs.get(0).getQueueOffset();\n        long storeTimeStamp = decodeMsgs.get(0).getStoreTimestamp();\n        for (int i = 0; i < messages.size(); i++) {\n            assertEquals(messages.get(i).getTopic(), decodeMsgs.get(i).getTopic());\n            assertEquals(new String(messages.get(i).getBody()), new String(decodeMsgs.get(i).getBody()));\n            assertEquals(messages.get(i).getTags(), decodeMsgs.get(i).getTags());\n\n            assertEquals(messageExtBatch.getBornHostNameString(), decodeMsgs.get(i).getBornHostNameString());\n\n            assertEquals(messageExtBatch.getBornTimestamp(), decodeMsgs.get(i).getBornTimestamp());\n            assertEquals(storeTimeStamp, decodeMsgs.get(i).getStoreTimestamp());\n            assertEquals(queueOffset++, decodeMsgs.get(i).getQueueOffset());\n        }\n\n    }\n\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/AppendPropCRCTest.java",
    "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport java.io.File;\nimport java.net.InetSocketAddress;\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBatch;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertTrue;\n\npublic class AppendPropCRCTest {\n\n    AppendMessageCallback callback;\n\n    MessageExtEncoder encoder;\n\n    CommitLog commitLog;\n\n    @Before\n    public void init() throws Exception {\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setMappedFileSizeCommitLog(1024 * 8);\n        messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 4);\n        messageStoreConfig.setMaxHashSlotNum(100);\n        messageStoreConfig.setMaxIndexNum(100 * 10);\n        messageStoreConfig.setMaxMessageSize(10 * 1024 * 1024);\n        messageStoreConfig.setStorePathRootDir(System.getProperty(\"java.io.tmpdir\") + File.separator + \"unitteststore\");\n        messageStoreConfig.setStorePathCommitLog(System.getProperty(\"java.io.tmpdir\") + File.separator + \"unitteststore\" + File.separator + \"commitlog\");\n        messageStoreConfig.setForceVerifyPropCRC(true);\n        messageStoreConfig.setEnabledAppendPropCRC(true);\n        //too much reference\n        DefaultMessageStore messageStore = new DefaultMessageStore(messageStoreConfig, null, null, new BrokerConfig(), new ConcurrentHashMap<>());\n        commitLog = new CommitLog(messageStore);\n        encoder = new MessageExtEncoder(messageStoreConfig);\n        callback = commitLog.new DefaultAppendMessageCallback(messageStoreConfig);\n    }\n\n    @After\n    public void destroy() {\n        UtilAll.deleteFile(new File(System.getProperty(\"user.home\") + File.separator + \"unitteststore\"));\n    }\n\n    @Test\n    public void testAppendMessageSucc() throws Exception {\n        String topic = \"test-topic\";\n        int queue = 0;\n        int msgNum = 10;\n        int propertiesLen = 0;\n        Message msg = new Message();\n        msg.setBody(\"body\".getBytes());\n        msg.setTopic(topic);\n        msg.setTags(\"abc\");\n        msg.putUserProperty(\"a\", \"aaaaaaaa\");\n        msg.putUserProperty(\"b\", \"bbbbbbbb\");\n        msg.putUserProperty(\"c\", \"cccccccc\");\n        msg.putUserProperty(\"d\", \"dddddddd\");\n        msg.putUserProperty(\"e\", \"eeeeeeee\");\n        msg.putUserProperty(\"f\", \"ffffffff\");\n\n        MessageExtBrokerInner messageExtBrokerInner = new MessageExtBrokerInner();\n        messageExtBrokerInner.setTopic(topic);\n        messageExtBrokerInner.setQueueId(queue);\n        messageExtBrokerInner.setBornTimestamp(System.currentTimeMillis());\n        messageExtBrokerInner.setBornHost(new InetSocketAddress(\"127.0.0.1\", 123));\n        messageExtBrokerInner.setStoreHost(new InetSocketAddress(\"127.0.0.1\", 124));\n        messageExtBrokerInner.setBody(msg.getBody());\n        messageExtBrokerInner.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n        propertiesLen = messageExtBrokerInner.getPropertiesString().length();\n\n        ByteBuffer buff = ByteBuffer.allocate(1024 * 10);\n        for (int i = 0; i < msgNum; i++) {\n            encoder.encode(messageExtBrokerInner);\n            messageExtBrokerInner.setEncodedBuff(encoder.getEncoderBuffer());\n            AppendMessageResult allresult = callback.doAppend(0, buff, 1024 * 10, messageExtBrokerInner, null);\n            assertEquals(AppendMessageStatus.PUT_OK, allresult.getStatus());\n        }\n        // Expected to pass when message is not modified\n        buff.flip();\n        for (int i = 0; i < msgNum - 1; i++) {\n            DispatchRequest request = commitLog.checkMessageAndReturnSize(buff, true, false);\n            assertTrue(request.isSuccess());\n        }\n        // Modify the properties of the last message and expect the verification to fail.\n        int idx = buff.limit() - (propertiesLen / 2);\n        buff.put(idx, (byte) (buff.get(idx) + 1));\n        DispatchRequest request = commitLog.checkMessageAndReturnSize(buff, true, false);\n        assertFalse(request.isSuccess());\n    }\n\n    @Test\n    public void testAppendMessageBatchSucc() throws Exception {\n        List<Message> messages = new ArrayList<>();\n        String topic = \"test-topic\";\n        int queue = 0;\n        int propertiesLen = 0;\n        for (int i = 0; i < 10; i++) {\n            Message msg = new Message();\n            msg.setBody(\"body\".getBytes());\n            msg.setTopic(topic);\n            msg.setTags(\"abc\");\n            msg.putUserProperty(\"a\", \"aaaaaaaa\");\n            msg.putUserProperty(\"b\", \"bbbbbbbb\");\n            msg.putUserProperty(\"c\", \"cccccccc\");\n            msg.putUserProperty(\"d\", \"dddddddd\");\n            msg.putUserProperty(\"e\", \"eeeeeeee\");\n            msg.putUserProperty(\"f\", \"ffffffff\");\n            String propertiesString = MessageDecoder.messageProperties2String(msg.getProperties());\n            propertiesLen = propertiesString.length();\n            messages.add(msg);\n        }\n        MessageExtBatch messageExtBatch = new MessageExtBatch();\n        messageExtBatch.setTopic(topic);\n        messageExtBatch.setQueueId(queue);\n        messageExtBatch.setBornTimestamp(System.currentTimeMillis());\n        messageExtBatch.setBornHost(new InetSocketAddress(\"127.0.0.1\", 123));\n        messageExtBatch.setStoreHost(new InetSocketAddress(\"127.0.0.1\", 124));\n        messageExtBatch.setBody(MessageDecoder.encodeMessages(messages));\n\n        PutMessageContext putMessageContext = new PutMessageContext(topic + \"-\" + queue);\n        messageExtBatch.setEncodedBuff(encoder.encode(messageExtBatch, putMessageContext));\n        ByteBuffer buff = ByteBuffer.allocate(1024 * 10);\n        //encounter end of file when append half of the data\n        AppendMessageResult allresult =\n            callback.doAppend(0, buff, 1024 * 10, messageExtBatch, putMessageContext);\n\n        assertEquals(AppendMessageStatus.PUT_OK, allresult.getStatus());\n        assertEquals(0, allresult.getWroteOffset());\n        assertEquals(0, allresult.getLogicsOffset());\n        assertEquals(buff.position(), allresult.getWroteBytes());\n\n        assertEquals(messages.size(), allresult.getMsgNum());\n\n        Set<String> msgIds = new HashSet<>();\n        for (String msgId : allresult.getMsgId().split(\",\")) {\n            assertEquals(32, msgId.length());\n            msgIds.add(msgId);\n        }\n        assertEquals(messages.size(), msgIds.size());\n\n        List<MessageExt> decodeMsgs = MessageDecoder.decodes((ByteBuffer) buff.flip());\n        assertEquals(decodeMsgs.size(), decodeMsgs.size());\n        long queueOffset = decodeMsgs.get(0).getQueueOffset();\n        long storeTimeStamp = decodeMsgs.get(0).getStoreTimestamp();\n        for (int i = 0; i < messages.size(); i++) {\n            assertEquals(messages.get(i).getTopic(), decodeMsgs.get(i).getTopic());\n            assertEquals(new String(messages.get(i).getBody()), new String(decodeMsgs.get(i).getBody()));\n            assertEquals(messages.get(i).getTags(), decodeMsgs.get(i).getTags());\n\n            assertEquals(messageExtBatch.getBornHostNameString(), decodeMsgs.get(i).getBornHostNameString());\n\n            assertEquals(messageExtBatch.getBornTimestamp(), decodeMsgs.get(i).getBornTimestamp());\n            assertEquals(storeTimeStamp, decodeMsgs.get(i).getStoreTimestamp());\n            assertEquals(queueOffset++, decodeMsgs.get(i).getQueueOffset());\n        }\n\n        // Expected to pass when message is not modified\n        buff.flip();\n        for (int i = 0; i < messages.size() - 1; i++) {\n            DispatchRequest request = commitLog.checkMessageAndReturnSize(buff, true, false);\n            assertTrue(request.isSuccess());\n        }\n        // Modify the properties of the last message and expect the verification to fail.\n        int idx = buff.limit() - (propertiesLen / 2);\n        buff.put(idx, (byte) (buff.get(idx) + 1));\n        DispatchRequest request = commitLog.checkMessageAndReturnSize(buff, true, false);\n        assertFalse(request.isSuccess());\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/BatchPutMessageTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport static java.util.concurrent.TimeUnit.SECONDS;\nimport static org.apache.rocketmq.common.message.MessageDecoder.messageProperties2String;\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertTrue;\n\nimport java.io.File;\nimport java.net.InetSocketAddress;\nimport java.nio.charset.Charset;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBatch;\nimport org.apache.rocketmq.store.config.FlushDiskType;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\npublic class BatchPutMessageTest {\n\n    private MessageStore messageStore;\n\n    public final static Charset CHARSET_UTF8 = StandardCharsets.UTF_8;\n\n    @Before\n    public void init() throws Exception {\n        messageStore = buildMessageStore();\n        boolean load = messageStore.load();\n        assertTrue(load);\n        messageStore.start();\n    }\n\n    @After\n    public void destroy() {\n        messageStore.shutdown();\n        messageStore.destroy();\n\n        UtilAll.deleteFile(new File(System.getProperty(\"java.io.tmpdir\") + File.separator + \"putmessagesteststore\"));\n    }\n\n    private MessageStore buildMessageStore() throws Exception {\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setMappedFileSizeCommitLog(1024 * 8);\n        messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 4);\n        messageStoreConfig.setMaxHashSlotNum(100);\n        messageStoreConfig.setMaxIndexNum(100 * 10);\n        messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);\n        messageStoreConfig.setFlushIntervalConsumeQueue(1);\n        messageStoreConfig.setStorePathRootDir(System.getProperty(\"java.io.tmpdir\") + File.separator + \"putmessagesteststore\");\n        messageStoreConfig.setStorePathCommitLog(System.getProperty(\"java.io.tmpdir\") + File.separator\n            + \"putmessagesteststore\" + File.separator + \"commitlog\");\n        messageStoreConfig.setHaListenPort(0);\n        return new DefaultMessageStore(messageStoreConfig, new BrokerStatsManager(\"simpleTest\", true), new MyMessageArrivingListener(), new BrokerConfig(), new ConcurrentHashMap<>());\n    }\n\n    @Test\n    public void testPutMessages() throws Exception {\n        String batchPropK = \"extraKey\";\n        String batchPropV = \"extraValue\";\n        Map<String, String> batchProp = new HashMap<>(1);\n        batchProp.put(batchPropK, batchPropV);\n        short batchPropLen = (short) messageProperties2String(batchProp).getBytes(MessageDecoder.CHARSET_UTF8).length;\n\n        List<Message> messages = new ArrayList<>();\n        String topic = \"batch-write-topic\";\n        int queue = 0;\n        int[] msgLengthArr = new int[11];\n        msgLengthArr[0] = 0;\n        int j = 1;\n        for (int i = 0; i < 10; i++) {\n            Message msg = new Message();\n            msg.setBody((\"body\" + i).getBytes());\n            msg.setTopic(topic);\n            msg.setTags(\"TAG1\");\n            msg.setKeys(String.valueOf(System.currentTimeMillis()));\n            messages.add(msg);\n            String properties = messageProperties2String(msg.getProperties());\n            byte[] propertiesBytes = properties.getBytes(CHARSET_UTF8);\n            short propertiesLength = (short) propertiesBytes.length;\n            final byte[] topicData = msg.getTopic().getBytes(MessageDecoder.CHARSET_UTF8);\n            final int topicLength = topicData.length;\n            msgLengthArr[j] = calMsgLength(msg.getBody().length, topicLength, propertiesLength) + msgLengthArr[j - 1];\n            j++;\n        }\n        byte[] batchMessageBody = MessageDecoder.encodeMessages(messages);\n        MessageExtBatch messageExtBatch = new MessageExtBatch();\n        messageExtBatch.setTopic(topic);\n        messageExtBatch.setQueueId(queue);\n        messageExtBatch.setBody(batchMessageBody);\n        messageExtBatch.putUserProperty(batchPropK, batchPropV);\n        messageExtBatch.setBornTimestamp(System.currentTimeMillis());\n        messageExtBatch.setStoreHost(new InetSocketAddress(\"127.0.0.1\", 125));\n        messageExtBatch.setBornHost(new InetSocketAddress(\"127.0.0.1\", 126));\n\n        PutMessageResult putMessageResult = messageStore.putMessages(messageExtBatch);\n        assertThat(putMessageResult.isOk()).isTrue();\n\n        for (long i = 0; i < 10; i++) {\n            final long index = i;\n            Boolean exist = await().atMost(3, SECONDS).until(() -> {\n                MessageExt messageExt = messageStore.lookMessageByOffset(msgLengthArr[(int) index]);\n                if (messageExt == null) {\n                    return false;\n                }\n                GetMessageResult result = messageStore.getMessage(\"batch_write_group\", topic, queue, index, 1024 * 1024, null);\n                if (result == null) {\n                    return false;\n                }\n                boolean equals = GetMessageStatus.FOUND.equals(result.getStatus());\n                result.release();\n                return equals;\n            }, item -> item);\n            assertTrue(exist);\n        }\n\n    }\n\n    @Test\n    public void testPutIPv6HostMessages() throws Exception {\n        List<Message> messages = new ArrayList<>();\n        String topic = \"batch-write-topic\";\n        int queue = 0;\n        int[] msgLengthArr = new int[11];\n        msgLengthArr[0] = 0;\n        int j = 1;\n        for (int i = 0; i < 10; i++) {\n            Message msg = new Message();\n            msg.setBody((\"body\" + i).getBytes());\n            msg.setTopic(topic);\n            msg.setTags(\"TAG1\");\n            msg.setKeys(String.valueOf(System.currentTimeMillis()));\n            messages.add(msg);\n            String properties = messageProperties2String(msg.getProperties());\n            byte[] propertiesBytes = properties.getBytes(CHARSET_UTF8);\n            short propertiesLength = (short) propertiesBytes.length;\n            final byte[] topicData = msg.getTopic().getBytes(MessageDecoder.CHARSET_UTF8);\n            final int topicLength = topicData.length;\n            msgLengthArr[j] = calIPv6HostMsgLength(msg.getBody().length, topicLength, propertiesLength) + msgLengthArr[j - 1];\n            j++;\n        }\n        byte[] batchMessageBody = MessageDecoder.encodeMessages(messages);\n        MessageExtBatch messageExtBatch = new MessageExtBatch();\n        messageExtBatch.setTopic(topic);\n        messageExtBatch.setQueueId(queue);\n        messageExtBatch.setBody(batchMessageBody);\n        messageExtBatch.setMsgId(\"24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0\");\n        messageExtBatch.setBornTimestamp(System.currentTimeMillis());\n        messageExtBatch.setSysFlag(0);\n        messageExtBatch.setBornHostV6Flag();\n        messageExtBatch.setStoreHostAddressV6Flag();\n        messageExtBatch.setStoreHost(new InetSocketAddress(\"1050:0000:0000:0000:0005:0600:300c:326b\", 125));\n        messageExtBatch.setBornHost(new InetSocketAddress(\"::1\", 126));\n\n        PutMessageResult putMessageResult = messageStore.putMessages(messageExtBatch);\n        assertThat(putMessageResult.isOk()).isTrue();\n\n        for (long i = 0; i < 10; i++) {\n            final long index = i;\n            Boolean exist = await().atMost(3, SECONDS).until(() -> {\n                MessageExt messageExt = messageStore.lookMessageByOffset(msgLengthArr[(int) index]);\n                if (messageExt == null) {\n                    return false;\n                }\n                GetMessageResult result = messageStore.getMessage(\"batch_write_group\", topic, queue, index, 1024 * 1024, null);\n                if (result == null) {\n                    return false;\n                }\n                boolean equals = GetMessageStatus.FOUND.equals(result.getStatus());\n                result.release();\n                return equals;\n            }, item -> item);\n            assertTrue(exist);\n        }\n\n    }\n\n    private String generateKey(StringBuilder keyBuilder, MessageExt messageExt) {\n        keyBuilder.setLength(0);\n        keyBuilder.append(messageExt.getTopic());\n        keyBuilder.append('-');\n        keyBuilder.append(messageExt.getQueueId());\n        return keyBuilder.toString();\n    }\n\n    private int calMsgLength(int bodyLength, int topicLength, int propertiesLength) {\n        final int msgLen = 4 //TOTALSIZE\n            + 4 //MAGICCODE\n            + 4 //BODYCRC\n            + 4 //QUEUEID\n            + 4 //FLAG\n            + 8 //QUEUEOFFSET\n            + 8 //PHYSICALOFFSET\n            + 4 //SYSFLAG\n            + 8 //BORNTIMESTAMP\n            + 8 //BORNHOST\n            + 8 //STORETIMESTAMP\n            + 8 //STOREHOSTADDRESS\n            + 4 //RECONSUMETIMES\n            + 8 //Prepared Transaction Offset\n            + 4 + (bodyLength > 0 ? bodyLength : 0) //BODY\n            + 1 + topicLength //TOPIC\n            + 2 + (propertiesLength > 0 ? propertiesLength : 0) //propertiesLength\n            + 0;\n        return msgLen;\n    }\n\n    private int calIPv6HostMsgLength(int bodyLength, int topicLength, int propertiesLength) {\n        final int msgLen = 4 //TOTALSIZE\n            + 4 //MAGICCODE\n            + 4 //BODYCRC\n            + 4 //QUEUEID\n            + 4 //FLAG\n            + 8 //QUEUEOFFSET\n            + 8 //PHYSICALOFFSET\n            + 4 //SYSFLAG\n            + 8 //BORNTIMESTAMP\n            + 20 //BORNHOST\n            + 8 //STORETIMESTAMP\n            + 20 //STOREHOSTADDRESS\n            + 4 //RECONSUMETIMES\n            + 8 //Prepared Transaction Offset\n            + 4 + (bodyLength > 0 ? bodyLength : 0) //BODY\n            + 1 + topicLength //TOPIC\n            + 2 + (propertiesLength > 0 ? propertiesLength : 0) //propertiesLength\n            + 0;\n        return msgLen;\n    }\n\n    private class MyMessageArrivingListener implements MessageArrivingListener {\n        @Override\n        public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime,\n            byte[] filterBitMap, Map<String, String> properties) {\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/ConsumeQueueExtTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport java.io.File;\nimport java.util.Random;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.junit.After;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class ConsumeQueueExtTest {\n\n    private static final String TOPIC = \"abc\";\n    private static final int QUEUE_ID = 0;\n    private static final String STORE_PATH = System.getProperty(\"java.io.tmpdir\") + File.separator + \"unit_test_store\";\n    private static final int BIT_MAP_LENGTH = 64;\n    private static final int UNIT_SIZE_WITH_BIT_MAP = ConsumeQueueExt.CqExtUnit.MIN_EXT_UNIT_SIZE + BIT_MAP_LENGTH / Byte.SIZE;\n    private static final int CQ_EXT_FILE_SIZE = 10 * UNIT_SIZE_WITH_BIT_MAP;\n    private static final int UNIT_COUNT = 20;\n\n    protected ConsumeQueueExt genExt() {\n        return new ConsumeQueueExt(\n            TOPIC, QUEUE_ID, STORE_PATH, CQ_EXT_FILE_SIZE, BIT_MAP_LENGTH\n        );\n    }\n\n    protected byte[] genBitMap(int bitMapLength) {\n        byte[] bytes = new byte[bitMapLength / Byte.SIZE];\n\n        Random random = new Random(System.currentTimeMillis());\n        random.nextBytes(bytes);\n\n        return bytes;\n    }\n\n    protected ConsumeQueueExt.CqExtUnit genUnit(boolean hasBitMap) {\n        ConsumeQueueExt.CqExtUnit cqExtUnit = new ConsumeQueueExt.CqExtUnit();\n\n        cqExtUnit.setTagsCode(Math.abs((new Random(System.currentTimeMillis())).nextInt()));\n        cqExtUnit.setMsgStoreTime(System.currentTimeMillis());\n        if (hasBitMap) {\n            cqExtUnit.setFilterBitMap(genBitMap(BIT_MAP_LENGTH));\n        }\n\n        return cqExtUnit;\n    }\n\n    protected void putSth(ConsumeQueueExt consumeQueueExt, boolean getAfterPut,\n        boolean unitSameSize, int unitCount) {\n        for (int i = 0; i < unitCount; i++) {\n            ConsumeQueueExt.CqExtUnit putUnit =\n                unitSameSize ? genUnit(true) : genUnit(i % 2 == 0);\n\n            long addr = consumeQueueExt.put(putUnit);\n            assertThat(addr).isLessThan(0);\n\n            if (getAfterPut) {\n                ConsumeQueueExt.CqExtUnit getUnit = consumeQueueExt.get(addr);\n\n                assertThat(getUnit).isNotNull();\n                assertThat(putUnit).isEqualTo(getUnit);\n            }\n\n            try {\n                Thread.sleep(10);\n            } catch (InterruptedException e) {\n                e.printStackTrace();\n                assertThat(false).isTrue();\n            }\n        }\n    }\n\n    @Test\n    public void testPut() {\n        ConsumeQueueExt consumeQueueExt = genExt();\n\n        try {\n            putSth(consumeQueueExt, true, false, UNIT_COUNT);\n        } finally {\n            consumeQueueExt.destroy();\n            UtilAll.deleteFile(new File(STORE_PATH));\n        }\n    }\n\n    @Test\n    public void testGet() {\n        ConsumeQueueExt consumeQueueExt = genExt();\n\n        putSth(consumeQueueExt, false, false, UNIT_COUNT);\n\n        try {\n            // from start.\n            long addr = consumeQueueExt.decorate(0);\n\n            ConsumeQueueExt.CqExtUnit unit = new ConsumeQueueExt.CqExtUnit();\n            while (true) {\n                boolean ret = consumeQueueExt.get(addr, unit);\n\n                if (!ret) {\n                    break;\n                }\n\n                assertThat(unit.getSize()).isGreaterThanOrEqualTo(ConsumeQueueExt.CqExtUnit.MIN_EXT_UNIT_SIZE);\n\n                addr += unit.getSize();\n            }\n        } finally {\n            consumeQueueExt.destroy();\n            UtilAll.deleteFile(new File(STORE_PATH));\n        }\n    }\n\n    @Test\n    public void testGet_invalidAddress() {\n        ConsumeQueueExt consumeQueueExt = genExt();\n\n        putSth(consumeQueueExt, false, true, UNIT_COUNT);\n\n        try {\n            ConsumeQueueExt.CqExtUnit unit = consumeQueueExt.get(0);\n\n            assertThat(unit).isNull();\n\n            long addr = (CQ_EXT_FILE_SIZE / UNIT_SIZE_WITH_BIT_MAP) * UNIT_SIZE_WITH_BIT_MAP;\n            addr += UNIT_SIZE_WITH_BIT_MAP;\n\n            unit = consumeQueueExt.get(addr);\n            assertThat(unit).isNull();\n        } finally {\n            consumeQueueExt.destroy();\n            UtilAll.deleteFile(new File(STORE_PATH));\n        }\n    }\n\n    @Test\n    public void testRecovery() {\n        ConsumeQueueExt putCqExt = genExt();\n\n        putSth(putCqExt, false, true, UNIT_COUNT);\n\n        ConsumeQueueExt loadCqExt = genExt();\n\n        loadCqExt.load();\n\n        loadCqExt.recover();\n\n        try {\n            assertThat(loadCqExt.getMinAddress()).isEqualTo(Long.MIN_VALUE);\n\n            // same unit size.\n            int countPerFile = (CQ_EXT_FILE_SIZE - ConsumeQueueExt.END_BLANK_DATA_LENGTH) / UNIT_SIZE_WITH_BIT_MAP;\n\n            int lastFileUnitCount = UNIT_COUNT % countPerFile;\n\n            int fileCount = UNIT_COUNT / countPerFile + 1;\n            if (lastFileUnitCount == 0) {\n                fileCount -= 1;\n            }\n\n            if (lastFileUnitCount == 0) {\n                assertThat(loadCqExt.unDecorate(loadCqExt.getMaxAddress()) % CQ_EXT_FILE_SIZE).isEqualTo(0);\n            } else {\n                assertThat(loadCqExt.unDecorate(loadCqExt.getMaxAddress()))\n                    .isEqualTo(lastFileUnitCount * UNIT_SIZE_WITH_BIT_MAP + (fileCount - 1) * CQ_EXT_FILE_SIZE);\n            }\n        } finally {\n            putCqExt.destroy();\n            loadCqExt.destroy();\n            UtilAll.deleteFile(new File(STORE_PATH));\n        }\n    }\n\n    @Test\n    public void testTruncateByMinOffset() {\n        ConsumeQueueExt consumeQueueExt = genExt();\n\n        putSth(consumeQueueExt, false, true, UNIT_COUNT * 2);\n\n        try {\n            // truncate first one file.\n            long address = consumeQueueExt.decorate((long) (CQ_EXT_FILE_SIZE * 1.5));\n\n            long expectMinAddress = consumeQueueExt.decorate(CQ_EXT_FILE_SIZE);\n\n            consumeQueueExt.truncateByMinAddress(address);\n\n            long minAddress = consumeQueueExt.getMinAddress();\n\n            assertThat(expectMinAddress).isEqualTo(minAddress);\n        } finally {\n            consumeQueueExt.destroy();\n            UtilAll.deleteFile(new File(STORE_PATH));\n        }\n    }\n\n    @Test\n    public void testTruncateByMaxOffset() {\n        ConsumeQueueExt consumeQueueExt = genExt();\n\n        putSth(consumeQueueExt, false, true, UNIT_COUNT * 2);\n\n        try {\n            // truncate, only first 3 files exist.\n            long address = consumeQueueExt.decorate(CQ_EXT_FILE_SIZE * 2 + UNIT_SIZE_WITH_BIT_MAP);\n\n            long expectMaxAddress = address + UNIT_SIZE_WITH_BIT_MAP;\n\n            consumeQueueExt.truncateByMaxAddress(address);\n\n            long maxAddress = consumeQueueExt.getMaxAddress();\n\n            assertThat(expectMaxAddress).isEqualTo(maxAddress);\n        } finally {\n            consumeQueueExt.destroy();\n            UtilAll.deleteFile(new File(STORE_PATH));\n        }\n    }\n\n    @After\n    public void destroy() {\n        UtilAll.deleteFile(new File(STORE_PATH));\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/ConsumeQueueTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.lang.reflect.Method;\nimport java.net.InetAddress;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.net.UnknownHostException;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.io.FileUtils;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.queue.ConsumeQueueInterface;\nimport org.apache.rocketmq.store.queue.CqUnit;\nimport org.apache.rocketmq.store.queue.ReferredIterator;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.awaitility.Awaitility;\nimport org.junit.Assert;\nimport org.junit.Assume;\nimport org.junit.Test;\nimport org.mockito.Mockito;\n\nimport static java.util.concurrent.TimeUnit.SECONDS;\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.awaitility.Awaitility.await;\n\npublic class ConsumeQueueTest {\n\n    private static final String MSG = \"Once, there was a chance for me!\";\n    private static final byte[] MSG_BODY = MSG.getBytes();\n\n    private static final String TOPIC = \"abc\";\n    private static final int QUEUE_ID = 0;\n    private static final String STORE_PATH = System.getProperty(\"java.io.tmpdir\") + File.separator + \"unit_test_store\";\n    private static final int COMMIT_LOG_FILE_SIZE = 1024 * 8;\n    private static final int CQ_FILE_SIZE = 10 * 20;\n    private static final int CQ_EXT_FILE_SIZE = 10 * (ConsumeQueueExt.CqExtUnit.MIN_EXT_UNIT_SIZE + 64);\n\n    private static SocketAddress bornHost;\n\n    private static SocketAddress storeHost;\n\n    static {\n        try {\n            storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123);\n        } catch (UnknownHostException e) {\n            e.printStackTrace();\n        }\n        try {\n            bornHost = new InetSocketAddress(InetAddress.getByName(\"127.0.0.1\"), 0);\n        } catch (UnknownHostException e) {\n            e.printStackTrace();\n        }\n    }\n\n    public MessageExtBrokerInner buildMessage() {\n        MessageExtBrokerInner msg = new MessageExtBrokerInner();\n        msg.setTopic(TOPIC);\n        msg.setTags(\"TAG1\");\n        msg.setKeys(\"Hello\");\n        msg.setBody(MSG_BODY);\n        msg.setKeys(String.valueOf(System.currentTimeMillis()));\n        msg.setQueueId(QUEUE_ID);\n        msg.setSysFlag(0);\n        msg.setBornTimestamp(System.currentTimeMillis());\n        msg.setStoreHost(storeHost);\n        msg.setBornHost(bornHost);\n        for (int i = 0; i < 1; i++) {\n            msg.putUserProperty(String.valueOf(i), \"imagoodperson\" + i);\n        }\n        msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n\n        return msg;\n    }\n\n    public MessageExtBrokerInner buildIPv6HostMessage() {\n        MessageExtBrokerInner msg = new MessageExtBrokerInner();\n        msg.setTopic(TOPIC);\n        msg.setTags(\"TAG1\");\n        msg.setKeys(\"Hello\");\n        msg.setBody(MSG_BODY);\n        msg.setMsgId(\"24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0\");\n        msg.setKeys(String.valueOf(System.currentTimeMillis()));\n        msg.setQueueId(QUEUE_ID);\n        msg.setSysFlag(0);\n        msg.setBornHostV6Flag();\n        msg.setStoreHostAddressV6Flag();\n        msg.setBornTimestamp(System.currentTimeMillis());\n        msg.setBornHost(new InetSocketAddress(\"1050:0000:0000:0000:0005:0600:300c:326b\", 123));\n        msg.setStoreHost(new InetSocketAddress(\"::1\", 124));\n        for (int i = 0; i < 1; i++) {\n            msg.putUserProperty(String.valueOf(i), \"imagoodperson\" + i);\n        }\n        msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n\n        return msg;\n    }\n\n    public MessageStoreConfig buildStoreConfig(int commitLogFileSize, int cqFileSize,\n        boolean enableCqExt, int cqExtFileSize) {\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setMappedFileSizeCommitLog(commitLogFileSize);\n        messageStoreConfig.setMappedFileSizeConsumeQueue(cqFileSize);\n        messageStoreConfig.setMappedFileSizeConsumeQueueExt(cqExtFileSize);\n        messageStoreConfig.setMessageIndexEnable(false);\n        messageStoreConfig.setEnableConsumeQueueExt(enableCqExt);\n        messageStoreConfig.setHaListenPort(0);\n        messageStoreConfig.setStorePathRootDir(STORE_PATH);\n        messageStoreConfig.setStorePathCommitLog(STORE_PATH + File.separator + \"commitlog\");\n\n        return messageStoreConfig;\n    }\n\n    protected DefaultMessageStore gen() throws Exception {\n        MessageStoreConfig messageStoreConfig = buildStoreConfig(\n                COMMIT_LOG_FILE_SIZE, CQ_FILE_SIZE, true, CQ_EXT_FILE_SIZE\n        );\n\n        BrokerConfig brokerConfig = new BrokerConfig();\n\n        DefaultMessageStore master = new DefaultMessageStore(\n            messageStoreConfig,\n            new BrokerStatsManager(brokerConfig.getBrokerClusterName(), brokerConfig.isEnableDetailStat()),\n            new MessageArrivingListener() {\n                @Override\n                public void arriving(String topic, int queueId, long logicOffset, long tagsCode,\n                    long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {\n                }\n            }\n            , brokerConfig, new ConcurrentHashMap<>());\n\n        assertThat(master.load()).isTrue();\n\n        master.start();\n\n        return master;\n    }\n\n    protected DefaultMessageStore genForMultiQueue() throws Exception {\n        MessageStoreConfig messageStoreConfig = buildStoreConfig(\n                COMMIT_LOG_FILE_SIZE, CQ_FILE_SIZE, true, CQ_EXT_FILE_SIZE\n        );\n\n        messageStoreConfig.setEnableLmq(true);\n        messageStoreConfig.setEnableMultiDispatch(true);\n\n        BrokerConfig brokerConfig = new BrokerConfig();\n\n        DefaultMessageStore master = new DefaultMessageStore(\n            messageStoreConfig,\n            new BrokerStatsManager(brokerConfig.getBrokerClusterName(), brokerConfig.isEnableDetailStat()),\n            new MessageArrivingListener() {\n                @Override\n                public void arriving(String topic, int queueId, long logicOffset, long tagsCode,\n                    long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {\n                }\n            }\n            , brokerConfig, new ConcurrentHashMap<>());\n\n        assertThat(master.load()).isTrue();\n\n        master.start();\n\n        return master;\n    }\n\n    protected void putMsg(DefaultMessageStore master) {\n        long totalMsgs = 200;\n\n        for (long i = 0; i < totalMsgs; i++) {\n            if (i < totalMsgs / 2) {\n                master.putMessage(buildMessage());\n            } else {\n                master.putMessage(buildIPv6HostMessage());\n            }\n        }\n    }\n\n    protected void putMsgMultiQueue(DefaultMessageStore master) {\n        for (long i = 0; i < 1; i++) {\n            master.putMessage(buildMessageMultiQueue());\n        }\n    }\n\n    private MessageExtBrokerInner buildMessageMultiQueue() {\n        MessageExtBrokerInner msg = new MessageExtBrokerInner();\n        msg.setTopic(TOPIC);\n        msg.setTags(\"TAG1\");\n        msg.setKeys(\"Hello\");\n        msg.setBody(MSG_BODY);\n        msg.setKeys(String.valueOf(System.currentTimeMillis()));\n        msg.setQueueId(QUEUE_ID);\n        msg.setSysFlag(0);\n        msg.setBornTimestamp(System.currentTimeMillis());\n        msg.setStoreHost(storeHost);\n        msg.setBornHost(bornHost);\n        for (int i = 0; i < 1; i++) {\n            MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_MULTI_DISPATCH, \"%LMQ%123,%LMQ%456\");\n            msg.putUserProperty(String.valueOf(i), \"imagoodperson\" + i);\n        }\n        msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n\n        return msg;\n    }\n\n    protected void deleteDirectory(String rootPath) {\n        File file = new File(rootPath);\n        deleteFile(file);\n    }\n\n    protected void deleteFile(File file) {\n        File[] subFiles = file.listFiles();\n        if (subFiles != null) {\n            for (File sub : subFiles) {\n                deleteFile(sub);\n            }\n        }\n\n        file.delete();\n    }\n\n    @Test\n    public void testPutMessagePositionInfo_buildCQRepeatedly() throws Exception {\n        DefaultMessageStore messageStore = null;\n        try {\n\n            messageStore = gen();\n\n            int totalMessages = 10;\n\n            for (int i = 0; i < totalMessages; i++) {\n                putMsg(messageStore);\n            }\n\n\n            // Wait consume queue build finish.\n            final MessageStore store = messageStore;\n            Awaitility.with().pollInterval(100, TimeUnit.MILLISECONDS).await().timeout(1, TimeUnit.MINUTES).until(() -> {\n                return store.dispatchBehindBytes() == 0;\n            });\n\n            ConsumeQueueInterface cq = messageStore.getConsumeQueueTable().get(TOPIC).get(QUEUE_ID);\n            Method method = cq.getClass().getDeclaredMethod(\"putMessagePositionInfo\", long.class, int.class, long.class, long.class);\n\n            assertThat(method).isNotNull();\n\n            method.setAccessible(true);\n\n            SelectMappedBufferResult result = messageStore.getCommitLog().getData(0);\n            assertThat(result != null).isTrue();\n\n            DispatchRequest dispatchRequest = messageStore.getCommitLog().checkMessageAndReturnSize(result.getByteBuffer(), false, false);\n\n            assertThat(cq).isNotNull();\n\n            Object dispatchResult = method.invoke(cq, dispatchRequest.getCommitLogOffset(),\n                dispatchRequest.getMsgSize(), dispatchRequest.getTagsCode(), dispatchRequest.getConsumeQueueOffset());\n\n            assertThat(Boolean.parseBoolean(dispatchResult.toString())).isTrue();\n\n        } finally {\n            if (messageStore != null) {\n                messageStore.shutdown();\n                messageStore.destroy();\n            }\n            deleteDirectory(STORE_PATH);\n        }\n\n    }\n\n    @Test\n    public void testPutMessagePositionInfoWrapper_MultiQueue() throws Exception {\n        Assume.assumeFalse(MixAll.isWindows());\n        DefaultMessageStore messageStore = null;\n        try {\n            messageStore = genForMultiQueue();\n\n            int totalMessages = 10;\n\n            for (int i = 0; i < totalMessages; i++) {\n                putMsgMultiQueue(messageStore);\n            }\n\n            // Wait consume queue build finish.\n            final MessageStore store = messageStore;\n            Awaitility.with().pollInterval(100, TimeUnit.MILLISECONDS).await().timeout(1, TimeUnit.MINUTES).until(() -> {\n                return store.dispatchBehindBytes() == 0;\n            });\n\n            ConsumeQueueInterface cq = messageStore.getConsumeQueueTable().get(TOPIC).get(QUEUE_ID);\n            Method method = ((ConsumeQueue) cq).getClass().getDeclaredMethod(\"putMessagePositionInfoWrapper\", DispatchRequest.class);\n\n            assertThat(method).isNotNull();\n\n            method.setAccessible(true);\n\n            SelectMappedBufferResult result = messageStore.getCommitLog().getData(0);\n            assertThat(result != null).isTrue();\n\n            DispatchRequest dispatchRequest = messageStore.getCommitLog().checkMessageAndReturnSize(result.getByteBuffer(), false, false);\n\n            assertThat(cq).isNotNull();\n\n            Object dispatchResult = method.invoke(cq, dispatchRequest);\n\n            ConsumeQueueInterface lmqCq1 = messageStore.getConsumeQueueTable().get(\"%LMQ%123\").get(0);\n\n            ConsumeQueueInterface lmqCq2 = messageStore.getConsumeQueueTable().get(\"%LMQ%456\").get(0);\n\n            assertThat(lmqCq1).isNotNull();\n\n            assertThat(lmqCq2).isNotNull();\n\n        } finally {\n            if (messageStore != null) {\n                messageStore.shutdown();\n                messageStore.destroy();\n            }\n            deleteDirectory(STORE_PATH);\n        }\n\n    }\n\n    @Test\n    public void testPutMessagePositionInfoMultiQueue() throws Exception {\n        DefaultMessageStore messageStore = null;\n        try {\n\n            messageStore = genForMultiQueue();\n\n            int totalMessages = 10;\n\n            for (int i = 0; i < totalMessages; i++) {\n                putMsgMultiQueue(messageStore);\n            }\n\n            // Wait consume queue build finish.\n            final MessageStore store = messageStore;\n            Awaitility.with().pollInterval(100, TimeUnit.MILLISECONDS).await().timeout(1, TimeUnit.MINUTES).until(() -> {\n                return store.dispatchBehindBytes() == 0;\n            });\n\n            ConsumeQueueInterface cq = messageStore.getConsumeQueueTable().get(TOPIC).get(QUEUE_ID);\n\n            ConsumeQueueInterface lmqCq1 = messageStore.getConsumeQueueTable().get(\"%LMQ%123\").get(0);\n\n            ConsumeQueueInterface lmqCq2 = messageStore.getConsumeQueueTable().get(\"%LMQ%456\").get(0);\n\n            assertThat(cq).isNotNull();\n\n            assertThat(lmqCq1).isNotNull();\n\n            assertThat(lmqCq2).isNotNull();\n\n        } finally {\n            if (messageStore != null) {\n                messageStore.shutdown();\n                messageStore.destroy();\n            }\n            deleteDirectory(STORE_PATH);\n        }\n    }\n\n    @Test\n    public void testConsumeQueueWithExtendData() {\n        DefaultMessageStore master = null;\n        try {\n            master = gen();\n        } catch (Exception e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n\n        master.getDispatcherList().addFirst(new CommitLogDispatcher() {\n\n            @Override\n            public void dispatch(DispatchRequest request) {\n                runCount++;\n            }\n\n            private int runCount = 0;\n        });\n\n        try {\n\n            putMsg(master);\n            final DefaultMessageStore master1 = master;\n            ConsumeQueueInterface cq = await().atMost(3, SECONDS).until(() -> {\n                ConcurrentMap<Integer, ConsumeQueueInterface> map = master1.getConsumeQueueTable().get(TOPIC);\n                if (map == null) {\n                    return null;\n                }\n                ConsumeQueueInterface anInterface = map.get(QUEUE_ID);\n                return anInterface;\n            }, item -> null != item);\n\n            assertThat(cq).isNotNull();\n\n            ReferredIterator<CqUnit> bufferResult = cq.iterateFrom(0);\n\n            assertThat(bufferResult).isNotNull();\n\n            Assert.assertTrue(bufferResult.hasNext());\n\n            try {\n                while (bufferResult.hasNext()) {\n                    CqUnit cqUnit = bufferResult.next();\n                    Assert.assertNotNull(cqUnit);\n                    long phyOffset = cqUnit.getPos();\n                    int size = cqUnit.getSize();\n                    long tagsCode = cqUnit.getTagsCode();\n\n                    assertThat(phyOffset).isGreaterThanOrEqualTo(0);\n                    assertThat(size).isGreaterThan(0);\n                    assertThat(tagsCode).isGreaterThan(0);\n\n                    ConsumeQueueExt.CqExtUnit cqExtUnit = cqUnit.getCqExtUnit();\n                    assertThat(cqExtUnit).isNotNull();\n                    assertThat(tagsCode).isEqualTo(cqExtUnit.getTagsCode());\n                    assertThat(cqExtUnit.getSize()).isGreaterThan((short) 0);\n                    assertThat(cqExtUnit.getMsgStoreTime()).isGreaterThan(0);\n                    assertThat(cqExtUnit.getTagsCode()).isGreaterThan(0);\n                }\n\n            } finally {\n                bufferResult.release();\n            }\n\n        } finally {\n            master.shutdown();\n            master.destroy();\n            UtilAll.deleteFile(new File(STORE_PATH));\n        }\n    }\n\n    @Test\n    public void testCorrectMinOffset() {\n        String topic = \"T1\";\n        int queueId = 0;\n        MessageStoreConfig storeConfig = new MessageStoreConfig();\n        File tmpDir = new File(System.getProperty(\"java.io.tmpdir\"), \"test_correct_min_offset\");\n        tmpDir.deleteOnExit();\n        storeConfig.setStorePathRootDir(tmpDir.getAbsolutePath());\n        storeConfig.setEnableConsumeQueueExt(false);\n        DefaultMessageStore messageStore = Mockito.mock(DefaultMessageStore.class);\n        Mockito.when(messageStore.getMessageStoreConfig()).thenReturn(storeConfig);\n\n        RunningFlags runningFlags = new RunningFlags();\n        Mockito.when(messageStore.getRunningFlags()).thenReturn(runningFlags);\n\n        StoreCheckpoint storeCheckpoint = Mockito.mock(StoreCheckpoint.class);\n        Mockito.when(messageStore.getStoreCheckpoint()).thenReturn(storeCheckpoint);\n\n        ConsumeQueue consumeQueue = new ConsumeQueue(topic, queueId, storeConfig.getStorePathRootDir(),\n            storeConfig.getMappedFileSizeConsumeQueue(), messageStore);\n\n        int max = 10000;\n        int messageSize = 100;\n        for (int i = 0; i < max; ++i) {\n            DispatchRequest dispatchRequest = new DispatchRequest(topic, queueId, messageSize * i, messageSize, 0, 0, i, null, null, 0, 0, null);\n            consumeQueue.putMessagePositionInfoWrapper(dispatchRequest);\n        }\n\n        consumeQueue.setMinLogicOffset(0L);\n        consumeQueue.correctMinOffset(0L);\n        Assert.assertEquals(0, consumeQueue.getMinOffsetInQueue());\n\n        consumeQueue.setMinLogicOffset(100);\n        consumeQueue.correctMinOffset(2000);\n        Assert.assertEquals(20, consumeQueue.getMinOffsetInQueue());\n\n        consumeQueue.setMinLogicOffset((max - 1) * ConsumeQueue.CQ_STORE_UNIT_SIZE);\n        consumeQueue.correctMinOffset(max * messageSize);\n        Assert.assertEquals(max * ConsumeQueue.CQ_STORE_UNIT_SIZE, consumeQueue.getMinLogicOffset());\n\n        consumeQueue.setMinLogicOffset(max * ConsumeQueue.CQ_STORE_UNIT_SIZE);\n        consumeQueue.correctMinOffset(max * messageSize);\n        Assert.assertEquals(max * ConsumeQueue.CQ_STORE_UNIT_SIZE, consumeQueue.getMinLogicOffset());\n        consumeQueue.destroy();\n    }\n\n    @Test\n    public void testFillBankThenCorrectMinOffset() throws IOException {\n        String topic = \"T1\";\n        int queueId = 0;\n        MessageStoreConfig storeConfig = new MessageStoreConfig();\n        File tmpDir = new File(System.getProperty(\"java.io.tmpdir\"), \"testFillBankThenCorrectMinOffset\");\n        FileUtils.deleteDirectory(tmpDir);\n        storeConfig.setStorePathRootDir(tmpDir.getAbsolutePath());\n        storeConfig.setEnableConsumeQueueExt(false);\n        DefaultMessageStore messageStore = Mockito.mock(DefaultMessageStore.class);\n        Mockito.when(messageStore.getMessageStoreConfig()).thenReturn(storeConfig);\n\n        RunningFlags runningFlags = new RunningFlags();\n        Mockito.when(messageStore.getRunningFlags()).thenReturn(runningFlags);\n\n        StoreCheckpoint storeCheckpoint = Mockito.mock(StoreCheckpoint.class);\n        Mockito.when(messageStore.getStoreCheckpoint()).thenReturn(storeCheckpoint);\n\n        {\n            ConsumeQueue consumeQueue = new ConsumeQueue(topic, queueId, storeConfig.getStorePathRootDir(),\n                storeConfig.getMappedFileSizeConsumeQueue(), messageStore);\n            Assert.assertTrue(consumeQueue.load());\n            consumeQueue.recover();\n            consumeQueue.initializeWithOffset(100, 100);\n            Assert.assertEquals(100, consumeQueue.getMinOffsetInQueue());\n            Assert.assertEquals(100, consumeQueue.getMaxOffsetInQueue());\n        }\n\n        {\n            ConsumeQueue consumeQueue = new ConsumeQueue(topic, queueId, storeConfig.getStorePathRootDir(),\n                storeConfig.getMappedFileSizeConsumeQueue(), messageStore);\n            Assert.assertTrue(consumeQueue.load());\n            consumeQueue.recover();\n            consumeQueue.correctMinOffset(1L);\n            Assert.assertEquals(100, consumeQueue.getMinOffsetInQueue());\n            Assert.assertEquals(100, consumeQueue.getMaxOffsetInQueue());\n        }\n\n//        {\n//            ConsumeQueue consumeQueue = new ConsumeQueue(topic, queueId, storeConfig.getStorePathRootDir(),\n//                storeConfig.getMappedFileSizeConsumeQueue(), messageStore);\n//            Assert.assertTrue(consumeQueue.load());\n//            consumeQueue.recover();\n//            consumeQueue.correctMinOffset(0L);\n//            Assert.assertEquals(100, consumeQueue.getMinOffsetInQueue());\n//            Assert.assertEquals(100, consumeQueue.getMaxOffsetInQueue());\n//        }\n\n        ConsumeQueue consumeQueue0 = new ConsumeQueue(topic, queueId, storeConfig.getStorePathRootDir(),\n            storeConfig.getMappedFileSizeConsumeQueue(), messageStore);\n        consumeQueue0.destroy();\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreCleanFilesTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport java.io.File;\nimport java.lang.reflect.Field;\nimport java.net.InetAddress;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.util.ArrayList;\nimport java.util.Calendar;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.UUID;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.config.FlushDiskType;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.index.IndexFile;\nimport org.apache.rocketmq.store.index.IndexService;\nimport org.apache.rocketmq.store.logfile.MappedFile;\nimport org.apache.rocketmq.store.queue.ConsumeQueueInterface;\nimport org.apache.rocketmq.store.queue.ConsumeQueueStore;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.apache.rocketmq.common.message.MessageDecoder.CHARSET_UTF8;\nimport static org.apache.rocketmq.store.ConsumeQueue.CQ_STORE_UNIT_SIZE;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.when;\n\n/**\n * Test case for DefaultMessageStore.CleanCommitLogService and DefaultMessageStore.CleanConsumeQueueService\n */\npublic class DefaultMessageStoreCleanFilesTest {\n    private DefaultMessageStore messageStore;\n    private DefaultMessageStore.CleanCommitLogService cleanCommitLogService;\n    private ConsumeQueueStore.CleanConsumeQueueService cleanConsumeQueueService;\n\n    private SocketAddress bornHost;\n    private SocketAddress storeHost;\n\n    private String topic = \"test\";\n    private String keys = \"hello\";\n    private int queueId = 0;\n    private int fileCountCommitLog = 55;\n    // exactly one message per CommitLog file.\n    private int msgCount = fileCountCommitLog;\n    private int mappedFileSize = 128;\n    private int fileReservedTime = 1;\n\n    @Before\n    public void init() throws Exception {\n        storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123);\n        bornHost = new InetSocketAddress(InetAddress.getByName(\"127.0.0.1\"), 0);\n    }\n\n    @Test\n    public void testIsSpaceFullFunctionEmpty2Full() throws Exception {\n        String deleteWhen = \"04\";\n        // the min value of diskMaxUsedSpaceRatio.\n        int diskMaxUsedSpaceRatio = 1;\n        // used to  set disk-full flag\n        double diskSpaceCleanForciblyRatio = 0.01D;\n        initMessageStore(deleteWhen, diskMaxUsedSpaceRatio, diskSpaceCleanForciblyRatio);\n        // build and put 55 messages, exactly one message per CommitLog file.\n        buildAndPutMessagesToMessageStore(msgCount);\n        MappedFileQueue commitLogQueue = getMappedFileQueueCommitLog();\n        assertEquals(fileCountCommitLog, commitLogQueue.getMappedFiles().size());\n        int fileCountConsumeQueue = getFileCountConsumeQueue();\n        MappedFileQueue consumeQueue = getMappedFileQueueConsumeQueue();\n        assertEquals(fileCountConsumeQueue, consumeQueue.getMappedFiles().size());\n        cleanCommitLogService.isSpaceFull();\n        assertEquals(1 << 4, messageStore.getRunningFlags().getFlagBits() & (1 << 4));\n    }\n\n    @Test\n    public void testIsSpaceFullMultiCommitLogStorePath() throws Exception {\n        String deleteWhen = \"04\";\n        // the min value of diskMaxUsedSpaceRatio.\n        int diskMaxUsedSpaceRatio = 1;\n        // used to  set disk-full flag\n        double diskSpaceCleanForciblyRatio = 0.01D;\n        MessageStoreConfig config = genMessageStoreConfig(deleteWhen, diskMaxUsedSpaceRatio);\n        String storePath = config.getStorePathCommitLog();\n        StringBuilder storePathBuilder = new StringBuilder();\n        for (int i = 0; i < 3; i++) {\n            storePathBuilder.append(storePath).append(i).append(MixAll.MULTI_PATH_SPLITTER);\n        }\n        config.setStorePathCommitLog(storePathBuilder.toString());\n        String[] paths = config.getStorePathCommitLog().trim().split(MixAll.MULTI_PATH_SPLITTER);\n        assertEquals(3, paths.length);\n        initMessageStore(config, diskSpaceCleanForciblyRatio);\n\n\n\n        // build and put 55 messages, exactly one message per CommitLog file.\n        buildAndPutMessagesToMessageStore(msgCount);\n        MappedFileQueue commitLogQueue = getMappedFileQueueCommitLog();\n        assertEquals(fileCountCommitLog, commitLogQueue.getMappedFiles().size());\n        int fileCountConsumeQueue = getFileCountConsumeQueue();\n        MappedFileQueue consumeQueue = getMappedFileQueueConsumeQueue();\n        assertEquals(fileCountConsumeQueue, consumeQueue.getMappedFiles().size());\n        cleanCommitLogService.isSpaceFull();\n\n        assertEquals(1 << 4, messageStore.getRunningFlags().getFlagBits() & (1 << 4));\n    }\n\n    @Test\n    public void testIsSpaceFullFunctionFull2Empty() throws Exception {\n        String deleteWhen = \"04\";\n        // the min value of diskMaxUsedSpaceRatio.\n        int diskMaxUsedSpaceRatio = 1;\n        //use to reset disk-full flag\n        double diskSpaceCleanForciblyRatio = 0.999D;\n        initMessageStore(deleteWhen, diskMaxUsedSpaceRatio, diskSpaceCleanForciblyRatio);\n        //set disk full\n        messageStore.getRunningFlags().getAndMakeDiskFull();\n\n        cleanCommitLogService.isSpaceFull();\n        assertEquals(0, messageStore.getRunningFlags().getFlagBits() & (1 << 4));\n    }\n\n    @Test\n    public void testDeleteExpiredFilesByTimeUp() throws Exception {\n        String deleteWhen = Calendar.getInstance().get(Calendar.HOUR_OF_DAY) + \"\";\n        // the max value of diskMaxUsedSpaceRatio\n        int diskMaxUsedSpaceRatio = 99;\n        // used to ensure that automatic file deletion is not triggered\n        double diskSpaceCleanForciblyRatio = 0.999D;\n        initMessageStore(deleteWhen, diskMaxUsedSpaceRatio, diskSpaceCleanForciblyRatio);\n\n        // build and put 55 messages, exactly one message per CommitLog file.\n        buildAndPutMessagesToMessageStore(msgCount);\n\n        // undo comment out the code below, if want to debug this case rather than just run it.\n        // Thread.sleep(1000 * 60 + 100);\n\n        MappedFileQueue commitLogQueue = getMappedFileQueueCommitLog();\n        assertEquals(fileCountCommitLog, commitLogQueue.getMappedFiles().size());\n\n        int fileCountConsumeQueue = getFileCountConsumeQueue();\n        MappedFileQueue consumeQueue = getMappedFileQueueConsumeQueue();\n        assertEquals(fileCountConsumeQueue, consumeQueue.getMappedFiles().size());\n\n        int fileCountIndexFile = getFileCountIndexFile();\n        assertEquals(fileCountIndexFile, getIndexFileList().size());\n\n        int expireFileCount = 15;\n        expireFiles(commitLogQueue, expireFileCount);\n\n        // magic code 10 reference to MappedFileQueue#DELETE_FILES_BATCH_MAX\n        for (int a = 1, fileCount = expireFileCount; a <= (int) Math.ceil((double) expireFileCount / 10); a++, fileCount -= 10) {\n            cleanCommitLogService.run();\n            cleanConsumeQueueService.run();\n\n            int expectDeletedCount = fileCount >= 10 ? a * 10 : ((a - 1) * 10 + fileCount);\n            assertEquals(fileCountCommitLog - expectDeletedCount, commitLogQueue.getMappedFiles().size());\n\n            int msgCountPerFile = getMsgCountPerConsumeQueueMappedFile();\n            int expectDeleteCountConsumeQueue = (int) Math.floor((double) expectDeletedCount / msgCountPerFile);\n            assertEquals(fileCountConsumeQueue - expectDeleteCountConsumeQueue, consumeQueue.getMappedFiles().size());\n\n            int msgCountPerIndexFile = getMsgCountPerIndexFile();\n            int expectDeleteCountIndexFile = (int) Math.floor((double) expectDeletedCount / msgCountPerIndexFile);\n            assertEquals(fileCountIndexFile - expectDeleteCountIndexFile, getIndexFileList().size());\n        }\n    }\n\n    @Test\n    public void testDeleteExpiredFilesBySpaceFull() throws Exception {\n        String deleteWhen = \"04\";\n        // the min value of diskMaxUsedSpaceRatio.\n        int diskMaxUsedSpaceRatio = 1;\n        // used to ensure that automatic file deletion is not triggered\n        double diskSpaceCleanForciblyRatio = 0.999D;\n        initMessageStore(deleteWhen, diskMaxUsedSpaceRatio, diskSpaceCleanForciblyRatio);\n\n        // build and put 55 messages, exactly one message per CommitLog file.\n        buildAndPutMessagesToMessageStore(msgCount);\n\n        // undo comment out the code below, if want to debug this case rather than just run it.\n        // Thread.sleep(1000 * 60 + 100);\n\n        MappedFileQueue commitLogQueue = getMappedFileQueueCommitLog();\n        assertEquals(fileCountCommitLog, commitLogQueue.getMappedFiles().size());\n\n        int fileCountConsumeQueue = getFileCountConsumeQueue();\n        MappedFileQueue consumeQueue = getMappedFileQueueConsumeQueue();\n        assertEquals(fileCountConsumeQueue, consumeQueue.getMappedFiles().size());\n\n        int fileCountIndexFile = getFileCountIndexFile();\n        assertEquals(fileCountIndexFile, getIndexFileList().size());\n\n        int expireFileCount = 15;\n        expireFiles(commitLogQueue, expireFileCount);\n\n        // magic code 10 reference to MappedFileQueue#DELETE_FILES_BATCH_MAX\n        for (int a = 1, fileCount = expireFileCount; a <= (int) Math.ceil((double) expireFileCount / 10); a++, fileCount -= 10) {\n            cleanCommitLogService.run();\n            cleanConsumeQueueService.run();\n\n            int expectDeletedCount = fileCount >= 10 ? a * 10 : ((a - 1) * 10 + fileCount);\n            assertEquals(fileCountCommitLog - expectDeletedCount, commitLogQueue.getMappedFiles().size());\n\n            int msgCountPerFile = getMsgCountPerConsumeQueueMappedFile();\n            int expectDeleteCountConsumeQueue = (int) Math.floor((double) expectDeletedCount / msgCountPerFile);\n            assertEquals(fileCountConsumeQueue - expectDeleteCountConsumeQueue, consumeQueue.getMappedFiles().size());\n\n            int msgCountPerIndexFile = getMsgCountPerIndexFile();\n            int expectDeleteCountIndexFile = (int) Math.floor((double) expectDeletedCount / msgCountPerIndexFile);\n            assertEquals(fileCountIndexFile - expectDeleteCountIndexFile, getIndexFileList().size());\n        }\n    }\n\n    @Test\n    public void testDeleteFilesImmediatelyBySpaceFull() throws Exception {\n        String deleteWhen = \"04\";\n        // the min value of diskMaxUsedSpaceRatio.\n        int diskMaxUsedSpaceRatio = 1;\n        // make sure to trigger the automatic file deletion feature\n        double diskSpaceCleanForciblyRatio = 0.01D;\n        initMessageStore(deleteWhen, diskMaxUsedSpaceRatio, diskSpaceCleanForciblyRatio);\n\n        // build and put 55 messages, exactly one message per CommitLog file.\n        buildAndPutMessagesToMessageStore(msgCount);\n\n        // undo comment out the code below, if want to debug this case rather than just run it.\n        // Thread.sleep(1000 * 60 + 100);\n\n        MappedFileQueue commitLogQueue = getMappedFileQueueCommitLog();\n        assertEquals(fileCountCommitLog, commitLogQueue.getMappedFiles().size());\n\n        int fileCountConsumeQueue = getFileCountConsumeQueue();\n        MappedFileQueue consumeQueue = getMappedFileQueueConsumeQueue();\n        assertEquals(fileCountConsumeQueue, consumeQueue.getMappedFiles().size());\n\n        int fileCountIndexFile = getFileCountIndexFile();\n        assertEquals(fileCountIndexFile, getIndexFileList().size());\n\n        // In this case, there is no need to expire the files.\n        // int expireFileCount = 15;\n        // expireFiles(commitLogQueue, expireFileCount);\n\n        // magic code 10 reference to MappedFileQueue#DELETE_FILES_BATCH_MAX\n        for (int a = 1, fileCount = fileCountCommitLog;\n             a <= (int) Math.ceil((double) fileCountCommitLog / 10) && fileCount >= 10;\n             a++, fileCount -= 10) {\n            cleanCommitLogService.run();\n            cleanConsumeQueueService.run();\n\n            assertEquals(fileCountCommitLog - 10 * a, commitLogQueue.getMappedFiles().size());\n\n            int msgCountPerFile = getMsgCountPerConsumeQueueMappedFile();\n            int expectDeleteCountConsumeQueue = (int) Math.floor((double) (a * 10) / msgCountPerFile);\n            assertEquals(fileCountConsumeQueue - expectDeleteCountConsumeQueue, consumeQueue.getMappedFiles().size());\n\n            int msgCountPerIndexFile = getMsgCountPerIndexFile();\n            int expectDeleteCountIndexFile = (int) Math.floor((double) (a * 10) / msgCountPerIndexFile);\n            assertEquals(fileCountIndexFile - expectDeleteCountIndexFile, getIndexFileList().size());\n        }\n    }\n\n    @Test\n    public void testDeleteExpiredFilesManually() throws Exception {\n        String deleteWhen = \"04\";\n        // the max value of diskMaxUsedSpaceRatio\n        int diskMaxUsedSpaceRatio = 99;\n        // used to ensure that automatic file deletion is not triggered\n        double diskSpaceCleanForciblyRatio = 0.999D;\n        initMessageStore(deleteWhen, diskMaxUsedSpaceRatio, diskSpaceCleanForciblyRatio);\n\n        messageStore.executeDeleteFilesManually();\n\n        // build and put 55 messages, exactly one message per CommitLog file.\n        buildAndPutMessagesToMessageStore(msgCount);\n\n        // undo comment out the code below, if want to debug this case rather than just run it.\n        // Thread.sleep(1000 * 60 + 100);\n\n        MappedFileQueue commitLogQueue = getMappedFileQueueCommitLog();\n        assertEquals(fileCountCommitLog, commitLogQueue.getMappedFiles().size());\n\n        int fileCountConsumeQueue = getFileCountConsumeQueue();\n        MappedFileQueue consumeQueue = getMappedFileQueueConsumeQueue();\n        assertEquals(fileCountConsumeQueue, consumeQueue.getMappedFiles().size());\n\n        int fileCountIndexFile = getFileCountIndexFile();\n        assertEquals(fileCountIndexFile, getIndexFileList().size());\n\n        int expireFileCount = 15;\n        expireFiles(commitLogQueue, expireFileCount);\n\n        // magic code 10 reference to MappedFileQueue#DELETE_FILES_BATCH_MAX\n        for (int a = 1, fileCount = expireFileCount; a <= (int) Math.ceil((double) expireFileCount / 10); a++, fileCount -= 10) {\n            cleanCommitLogService.run();\n            cleanConsumeQueueService.run();\n\n            int expectDeletedCount = fileCount >= 10 ? a * 10 : ((a - 1) * 10 + fileCount);\n            assertEquals(fileCountCommitLog - expectDeletedCount, commitLogQueue.getMappedFiles().size());\n\n            int msgCountPerFile = getMsgCountPerConsumeQueueMappedFile();\n            int expectDeleteCountConsumeQueue = (int) Math.floor((double) expectDeletedCount / msgCountPerFile);\n            assertEquals(fileCountConsumeQueue - expectDeleteCountConsumeQueue, consumeQueue.getMappedFiles().size());\n\n            int msgCountPerIndexFile = getMsgCountPerIndexFile();\n            int expectDeleteCountIndexFile = (int) Math.floor((double) (a * 10) / msgCountPerIndexFile);\n            assertEquals(fileCountIndexFile - expectDeleteCountIndexFile, getIndexFileList().size());\n        }\n    }\n\n    private DefaultMessageStore.CleanCommitLogService getCleanCommitLogService()\n            throws Exception {\n        Field serviceField = messageStore.getClass().getDeclaredField(\"cleanCommitLogService\");\n        serviceField.setAccessible(true);\n        DefaultMessageStore.CleanCommitLogService cleanCommitLogService =\n                (DefaultMessageStore.CleanCommitLogService) serviceField.get(messageStore);\n        serviceField.setAccessible(false);\n\n        return cleanCommitLogService;\n    }\n\n    private ConsumeQueueStore.CleanConsumeQueueService getCleanConsumeQueueService()\n            throws Exception {\n        Field serviceField = messageStore.getQueueStore().getClass().getDeclaredField(\"cleanConsumeQueueService\");\n        serviceField.setAccessible(true);\n        ConsumeQueueStore.CleanConsumeQueueService cleanConsumeQueueService =\n            (ConsumeQueueStore.CleanConsumeQueueService) serviceField.get(messageStore.getQueueStore());\n        serviceField.setAccessible(false);\n        return cleanConsumeQueueService;\n    }\n\n    private MappedFileQueue getMappedFileQueueConsumeQueue()\n            throws Exception {\n        ConsumeQueueInterface consumeQueue = messageStore.getConsumeQueueTable().get(topic).get(queueId);\n        Field queueField = consumeQueue.getClass().getDeclaredField(\"mappedFileQueue\");\n        queueField.setAccessible(true);\n        MappedFileQueue fileQueue = (MappedFileQueue) queueField.get(consumeQueue);\n        queueField.setAccessible(false);\n        return fileQueue;\n    }\n\n    private MappedFileQueue getMappedFileQueueCommitLog() throws Exception {\n        CommitLog commitLog = messageStore.getCommitLog();\n        Field queueField = commitLog.getClass().getDeclaredField(\"mappedFileQueue\");\n        queueField.setAccessible(true);\n        MappedFileQueue fileQueue = (MappedFileQueue) queueField.get(commitLog);\n        queueField.setAccessible(false);\n        return fileQueue;\n    }\n\n    private ArrayList<IndexFile> getIndexFileList() throws Exception {\n        Field indexServiceField = messageStore.getClass().getDeclaredField(\"indexService\");\n        indexServiceField.setAccessible(true);\n        IndexService indexService = (IndexService) indexServiceField.get(messageStore);\n\n        Field indexFileListField = indexService.getClass().getDeclaredField(\"indexFileList\");\n        indexFileListField.setAccessible(true);\n        ArrayList<IndexFile> indexFileList = (ArrayList<IndexFile>) indexFileListField.get(indexService);\n\n        return indexFileList;\n    }\n\n    private int getFileCountConsumeQueue() {\n        int countPerFile = getMsgCountPerConsumeQueueMappedFile();\n        double fileCount = (double) msgCount / countPerFile;\n        return (int) Math.ceil(fileCount);\n    }\n\n    private int getFileCountIndexFile() {\n        int countPerFile = getMsgCountPerIndexFile();\n        double fileCount = (double) msgCount / countPerFile;\n        return (int) Math.ceil(fileCount);\n    }\n\n    private int getMsgCountPerConsumeQueueMappedFile() {\n        int size = messageStore.getMessageStoreConfig().getMappedFileSizeConsumeQueue();\n        return size / CQ_STORE_UNIT_SIZE;// 7 in this case\n    }\n\n    private int getMsgCountPerIndexFile() {\n        // 7 in this case\n        return messageStore.getMessageStoreConfig().getMaxIndexNum() - 1;\n    }\n\n    private void buildAndPutMessagesToMessageStore(int msgCount) throws Exception {\n        int msgLen = topic.getBytes(CHARSET_UTF8).length + 91;\n        Map<String, String> properties = new HashMap<>(4);\n        properties.put(MessageConst.PROPERTY_KEYS, keys);\n        String s = MessageDecoder.messageProperties2String(properties);\n        int propertiesLen = s.getBytes(CHARSET_UTF8).length;\n        int commitLogEndFileMinBlankLength = 4 + 4;\n        int singleMsgBodyLen = mappedFileSize - msgLen - propertiesLen - commitLogEndFileMinBlankLength;\n\n        for (int i = 0; i < msgCount; i++) {\n            MessageExtBrokerInner msg = new MessageExtBrokerInner();\n            msg.setTopic(topic);\n            msg.setBody(new byte[singleMsgBodyLen]);\n            msg.setKeys(keys);\n            msg.setQueueId(queueId);\n            msg.setSysFlag(0);\n            msg.setBornTimestamp(System.currentTimeMillis());\n            msg.setStoreHost(storeHost);\n            msg.setBornHost(bornHost);\n            msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n            PutMessageResult result = messageStore.putMessage(msg);\n            assertTrue(result != null && result.isOk());\n        }\n\n        StoreTestUtil.waitCommitLogReput(messageStore);\n        StoreTestUtil.flushConsumeQueue(messageStore);\n        StoreTestUtil.flushConsumeIndex(messageStore);\n    }\n\n    private void expireFiles(MappedFileQueue commitLogQueue, int expireCount) {\n        for (int i = 0; i < commitLogQueue.getMappedFiles().size(); i++) {\n            MappedFile mappedFile = commitLogQueue.getMappedFiles().get(i);\n            int reservedTime = fileReservedTime * 60 * 60 * 1000;\n            if (i < expireCount) {\n                boolean modified = mappedFile.getFile().setLastModified(System.currentTimeMillis() - reservedTime * 2);\n                assertTrue(modified);\n            }\n        }\n    }\n\n    private void initMessageStore(String deleteWhen, int diskMaxUsedSpaceRatio, double diskSpaceCleanForciblyRatio) throws Exception {\n        initMessageStore(genMessageStoreConfig(deleteWhen,diskMaxUsedSpaceRatio), diskSpaceCleanForciblyRatio);\n    }\n\n    private MessageStoreConfig genMessageStoreConfig(String deleteWhen, int diskMaxUsedSpaceRatio) {\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfigForTest();\n        messageStoreConfig.setMappedFileSizeCommitLog(mappedFileSize);\n        messageStoreConfig.setMappedFileSizeConsumeQueue(mappedFileSize);\n        messageStoreConfig.setMaxHashSlotNum(100);\n        messageStoreConfig.setMaxIndexNum(8);\n        messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);\n        messageStoreConfig.setFlushIntervalConsumeQueue(1);\n\n        // Invalidate DefaultMessageStore`s scheduled task of cleaning expired files.\n        // work with the code 'Thread.sleep(1000 * 60 + 100)' behind.\n        messageStoreConfig.setCleanResourceInterval(Integer.MAX_VALUE);\n\n        messageStoreConfig.setFileReservedTime(fileReservedTime);\n        messageStoreConfig.setDeleteWhen(deleteWhen);\n        messageStoreConfig.setDiskMaxUsedSpaceRatio(diskMaxUsedSpaceRatio);\n\n        String storePathRootDir = System.getProperty(\"java.io.tmpdir\") + File.separator\n                + \"DefaultMessageStoreCleanFilesTest-\" + UUID.randomUUID();\n        String storePathCommitLog = storePathRootDir + File.separator + \"commitlog\";\n        messageStoreConfig.setStorePathRootDir(storePathRootDir);\n        messageStoreConfig.setStorePathCommitLog(storePathCommitLog);\n        return messageStoreConfig;\n    }\n\n    private void initMessageStore(MessageStoreConfig messageStoreConfig, double diskSpaceCleanForciblyRatio) throws Exception {\n        messageStore = new DefaultMessageStore(messageStoreConfig,\n                new BrokerStatsManager(\"test\", true), new MyMessageArrivingListener(), new BrokerConfig(), new ConcurrentHashMap<>());\n\n        cleanCommitLogService = getCleanCommitLogService();\n        cleanConsumeQueueService = getCleanConsumeQueueService();\n\n        assertTrue(messageStore.load());\n        messageStore.start();\n\n        // partially mock a real obj\n        cleanCommitLogService = spy(cleanCommitLogService);\n        when(cleanCommitLogService.getDiskSpaceWarningLevelRatio()).thenReturn(diskSpaceCleanForciblyRatio);\n        when(cleanCommitLogService.getDiskSpaceCleanForciblyRatio()).thenReturn(diskSpaceCleanForciblyRatio);\n\n        putFiledBackToMessageStore(cleanCommitLogService);\n    }\n\n    private void putFiledBackToMessageStore(DefaultMessageStore.CleanCommitLogService cleanCommitLogService) throws Exception {\n        Field cleanCommitLogServiceField = DefaultMessageStore.class.getDeclaredField(\"cleanCommitLogService\");\n        cleanCommitLogServiceField.setAccessible(true);\n        cleanCommitLogServiceField.set(messageStore, cleanCommitLogService);\n        cleanCommitLogServiceField.setAccessible(false);\n    }\n\n    private class MyMessageArrivingListener implements MessageArrivingListener {\n        @Override\n        public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime,\n                             byte[] filterBitMap, Map<String, String> properties) {\n        }\n    }\n\n    @After\n    public void destroy() {\n\n        messageStore.shutdown();\n        messageStore.destroy();\n\n        if (messageStore != null) {\n            MessageStoreConfig messageStoreConfig = messageStore.getMessageStoreConfig();\n            File file = new File(messageStoreConfig.getStorePathRootDir());\n            UtilAll.deleteFile(file);\n        }\n    }\n\n    private class MessageStoreConfigForTest extends MessageStoreConfig {\n        @Override\n        public int getDiskMaxUsedSpaceRatio() {\n            try {\n                Field diskMaxUsedSpaceRatioField = this.getClass().getSuperclass().getDeclaredField(\"diskMaxUsedSpaceRatio\");\n                diskMaxUsedSpaceRatioField.setAccessible(true);\n                int ratio = (int) diskMaxUsedSpaceRatioField.get(this);\n                diskMaxUsedSpaceRatioField.setAccessible(false);\n                return ratio;\n            } catch (Exception ignored) {\n            }\n            return super.getDiskMaxUsedSpaceRatio();\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreShutDownTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport java.io.File;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.store.config.FlushDiskType;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.config.StorePathConfigHelper;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class DefaultMessageStoreShutDownTest {\n    private DefaultMessageStore messageStore;\n\n    @Before\n    public void init() throws Exception {\n        DefaultMessageStore store = buildMessageStore();\n        boolean load = store.load();\n        assertTrue(load);\n        store.start();\n        messageStore = spy(store);\n        when(messageStore.dispatchBehindBytes()).thenReturn(100L);\n    }\n\n    @Test\n    public void testDispatchBehindWhenShutdown() {\n        messageStore.shutdown();\n        assertTrue(!messageStore.shutDownNormal);\n        File file = new File(StorePathConfigHelper.getAbortFile(messageStore.getMessageStoreConfig().getStorePathRootDir()));\n        assertTrue(file.exists());\n    }\n\n    @After\n    public void destroy() {\n        messageStore.destroy();\n        File file = new File(messageStore.getMessageStoreConfig().getStorePathRootDir());\n        UtilAll.deleteFile(file);\n    }\n\n    public DefaultMessageStore buildMessageStore() throws Exception {\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setMappedFileSizeCommitLog(1024 * 1024 * 10);\n        messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 1024 * 10);\n        messageStoreConfig.setMaxHashSlotNum(10000);\n        messageStoreConfig.setMaxIndexNum(100 * 100);\n        messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);\n        messageStoreConfig.setHaListenPort(0);\n        String storeRootPath = System.getProperty(\"java.io.tmpdir\") + File.separator + \"store\";\n        messageStoreConfig.setStorePathRootDir(storeRootPath);\n        messageStoreConfig.setHaListenPort(0);\n        return new DefaultMessageStore(messageStoreConfig, new BrokerStatsManager(\"simpleTest\", true), null, new BrokerConfig(), new ConcurrentHashMap<>());\n    }\n\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.Mockito.atLeastOnce;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.spy;\nimport static org.mockito.Mockito.verify;\n\nimport org.mockito.ArgumentCaptor;\n\nimport com.google.common.collect.Sets;\nimport java.io.File;\nimport java.io.RandomAccessFile;\nimport java.lang.reflect.InvocationTargetException;\nimport java.lang.reflect.Method;\nimport java.net.InetAddress;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.net.UnknownHostException;\nimport java.nio.MappedByteBuffer;\nimport java.nio.channels.FileChannel;\nimport java.nio.channels.OverlappingFileLockException;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Properties;\nimport java.util.Random;\nimport java.util.UUID;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.MessageBatch;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBatch;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.config.FlushDiskType;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.config.StorePathConfigHelper;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.queue.ConsumeQueueInterface;\nimport org.apache.rocketmq.store.queue.CqUnit;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.assertj.core.util.Strings;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.junit.MockitoJUnitRunner;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class DefaultMessageStoreTest {\n    private final String storeMessage = \"Once, there was a chance for me!\";\n    private final String messageTopic = \"FooBar\";\n    private int queueTotal = 100;\n    private AtomicInteger queueId = new AtomicInteger(0);\n    private SocketAddress bornHost;\n    private SocketAddress storeHost;\n    private byte[] messageBody;\n    private MessageStore messageStore;\n\n    @Before\n    public void init() throws Exception {\n        storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123);\n        bornHost = new InetSocketAddress(InetAddress.getByName(\"127.0.0.1\"), 0);\n\n        messageStore = buildMessageStore();\n        boolean load = messageStore.load();\n        assertTrue(load);\n        messageStore.start();\n    }\n\n    @Test(expected = OverlappingFileLockException.class)\n    public void test_repeat_restart() throws Exception {\n        queueTotal = 1;\n        messageBody = storeMessage.getBytes();\n\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setMappedFileSizeCommitLog(1024 * 8);\n        messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 4);\n        messageStoreConfig.setMaxHashSlotNum(100);\n        messageStoreConfig.setMaxIndexNum(100 * 10);\n        messageStoreConfig.setStorePathRootDir(System.getProperty(\"java.io.tmpdir\") + File.separator + \"store\");\n        messageStoreConfig.setHaListenPort(0);\n        MessageStore master = new DefaultMessageStore(messageStoreConfig, null, new MyMessageArrivingListener(), new BrokerConfig(), new ConcurrentHashMap<>());\n\n        boolean load = master.load();\n        assertTrue(load);\n\n        try {\n            master.start();\n            master.start();\n        } finally {\n            master.shutdown();\n            master.destroy();\n        }\n    }\n\n    @After\n    public void destroy() {\n        messageStore.shutdown();\n        messageStore.destroy();\n\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        File file = new File(messageStoreConfig.getStorePathRootDir());\n        UtilAll.deleteFile(file);\n    }\n\n    private MessageStore buildMessageStore() throws Exception {\n        return buildMessageStore(null);\n    }\n\n    private MessageStore buildMessageStore(String storePathRootDir) throws Exception {\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setMappedFileSizeCommitLog(1024 * 1024 * 10);\n        messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 1024 * 10);\n        messageStoreConfig.setMaxHashSlotNum(10000);\n        messageStoreConfig.setMaxIndexNum(100 * 100);\n        messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);\n        messageStoreConfig.setFlushIntervalConsumeQueue(1);\n        messageStoreConfig.setHaListenPort(0);\n        if (Strings.isNullOrEmpty(storePathRootDir)) {\n            UUID uuid = UUID.randomUUID();\n            storePathRootDir = System.getProperty(\"java.io.tmpdir\") + File.separator + \"store-\" + uuid.toString();\n        }\n        messageStoreConfig.setStorePathRootDir(storePathRootDir);\n        return new DefaultMessageStore(messageStoreConfig,\n            new BrokerStatsManager(\"simpleTest\", true),\n            new MyMessageArrivingListener(),\n            new BrokerConfig(), new ConcurrentHashMap<>());\n    }\n\n    @Test\n    public void testWriteAndRead() {\n        long ipv4HostMsgs = 10;\n        long ipv6HostMsgs = 10;\n        long totalMsgs = ipv4HostMsgs + ipv6HostMsgs;\n        queueTotal = 1;\n        messageBody = storeMessage.getBytes();\n        for (long i = 0; i < ipv4HostMsgs; i++) {\n            messageStore.putMessage(buildMessage());\n        }\n\n        for (long i = 0; i < ipv6HostMsgs; i++) {\n            messageStore.putMessage(buildIPv6HostMessage());\n        }\n\n        StoreTestUtil.waitCommitLogReput((DefaultMessageStore) messageStore);\n\n        for (long i = 0; i < totalMsgs; i++) {\n            GetMessageResult result = messageStore.getMessage(\"GROUP_A\", \"FooBar\", 0, i, 1024 * 1024, null);\n            assertThat(result).isNotNull();\n            result.release();\n        }\n        verifyThatMasterIsFunctional(totalMsgs, messageStore);\n    }\n\n    @Test\n    public void testLookMessageByOffset_OffsetIsFirst() {\n        final int totalCount = 10;\n        int queueId = new Random().nextInt(10);\n        String topic = \"FooBar\";\n        int firstOffset = 0;\n        AppendMessageResult[] appendMessageResultArray = putMessages(totalCount, topic, queueId);\n        AppendMessageResult firstResult = appendMessageResultArray[0];\n\n        MessageExt messageExt = messageStore.lookMessageByOffset(firstResult.getWroteOffset());\n        MessageExt messageExt1 = getDefaultMessageStore().lookMessageByOffset(firstResult.getWroteOffset(), firstResult.getWroteBytes());\n\n        assertThat(new String(messageExt.getBody())).isEqualTo(buildMessageBodyByOffset(storeMessage, firstOffset));\n        assertThat(new String(messageExt1.getBody())).isEqualTo(buildMessageBodyByOffset(storeMessage, firstOffset));\n    }\n\n    @Test\n    public void testLookMessageByOffset_OffsetIsLast() {\n        final int totalCount = 10;\n        int queueId = new Random().nextInt(10);\n        String topic = \"FooBar\";\n        AppendMessageResult[] appendMessageResultArray = putMessages(totalCount, topic, queueId);\n        int lastIndex = totalCount - 1;\n        AppendMessageResult lastResult = appendMessageResultArray[lastIndex];\n\n        MessageExt messageExt = getDefaultMessageStore().lookMessageByOffset(lastResult.getWroteOffset(), lastResult.getWroteBytes());\n\n        assertThat(new String(messageExt.getBody())).isEqualTo(buildMessageBodyByOffset(storeMessage, lastIndex));\n    }\n\n    @Test\n    public void testLookMessageByOffset_OffsetIsOutOfBound() {\n        final int totalCount = 10;\n        int queueId = new Random().nextInt(10);\n        String topic = \"FooBar\";\n        AppendMessageResult[] appendMessageResultArray = putMessages(totalCount, topic, queueId);\n        long lastOffset = getMaxOffset(appendMessageResultArray);\n\n        MessageExt messageExt = getDefaultMessageStore().lookMessageByOffset(lastOffset);\n\n        assertThat(messageExt).isNull();\n    }\n\n    @Test\n    public void testGetOffsetInQueueByTime() {\n        final int totalCount = 10;\n        int queueId = 0;\n        String topic = \"FooBar\";\n        AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, true);\n        //Thread.sleep(10);\n        StoreTestUtil.waitCommitLogReput((DefaultMessageStore) messageStore);\n\n        ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId);\n        for (AppendMessageResult appendMessageResult : appendMessageResults) {\n            long offset = messageStore.getOffsetInQueueByTime(topic, queueId, appendMessageResult.getStoreTimestamp());\n            CqUnit cqUnit = consumeQueue.get(offset);\n            assertThat(cqUnit.getPos()).isEqualTo(appendMessageResult.getWroteOffset());\n            assertThat(cqUnit.getSize()).isEqualTo(appendMessageResult.getWroteBytes());\n        }\n    }\n\n    @Test\n    public void testGetOffsetInQueueByTime_TimestampIsSkewing() {\n        final int totalCount = 10;\n        int queueId = 0;\n        String topic = \"FooBar\";\n        AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, true);\n        //Thread.sleep(10);\n        StoreTestUtil.waitCommitLogReput((DefaultMessageStore) messageStore);\n        int skewing = 2;\n\n        ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId);\n        for (AppendMessageResult appendMessageResult : appendMessageResults) {\n            long offset = messageStore.getOffsetInQueueByTime(topic, queueId, appendMessageResult.getStoreTimestamp() - skewing);\n            CqUnit cqUnit = consumeQueue.get(offset);\n            assertThat(cqUnit.getPos()).isEqualTo(appendMessageResult.getWroteOffset());\n            assertThat(cqUnit.getSize()).isEqualTo(appendMessageResult.getWroteBytes());\n        }\n    }\n\n    @Test\n    public void testGetOffsetInQueueByTime_TimestampSkewingIsLarge() {\n        final int totalCount = 10;\n        int queueId = 0;\n        String topic = \"FooBar\";\n        AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, true);\n        //Thread.sleep(10);\n        StoreTestUtil.waitCommitLogReput((DefaultMessageStore) messageStore);\n        int skewing = 20000;\n\n        ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId);\n        for (AppendMessageResult appendMessageResult : appendMessageResults) {\n            long offset = messageStore.getOffsetInQueueByTime(topic, queueId, appendMessageResult.getStoreTimestamp() - skewing);\n            CqUnit cqUnit = consumeQueue.get(offset);\n            assertThat(cqUnit.getPos()).isEqualTo(appendMessageResults[0].getWroteOffset());\n            assertThat(cqUnit.getSize()).isEqualTo(appendMessageResults[0].getWroteBytes());\n        }\n    }\n\n    @Test\n    public void testGetOffsetInQueueByTime_ConsumeQueueNotFound1() {\n        final int totalCount = 10;\n        int queueId = 0;\n        int wrongQueueId = 1;\n        String topic = \"FooBar\";\n        AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, false);\n        //Thread.sleep(10);\n\n        StoreTestUtil.waitCommitLogReput((DefaultMessageStore) messageStore);\n\n        long offset = messageStore.getOffsetInQueueByTime(topic, wrongQueueId, appendMessageResults[0].getStoreTimestamp());\n\n        assertThat(offset).isEqualTo(0);\n    }\n\n    @Test\n    public void testGetOffsetInQueueByTime_ConsumeQueueNotFound2() {\n        final int totalCount = 10;\n        int queueId = 0;\n        int wrongQueueId = 1;\n        String topic = \"FooBar\";\n        putMessages(totalCount, topic, queueId, false);\n        //Thread.sleep(10);\n        StoreTestUtil.waitCommitLogReput((DefaultMessageStore) messageStore);\n\n        long messageStoreTimeStamp = messageStore.getMessageStoreTimeStamp(topic, wrongQueueId, 0);\n\n        assertThat(messageStoreTimeStamp).isEqualTo(-1);\n    }\n\n    @Test\n    public void testGetOffsetInQueueByTime_ConsumeQueueOffsetNotExist() {\n        final int totalCount = 10;\n        int queueId = 0;\n        int wrongQueueId = 1;\n        String topic = \"FooBar\";\n        putMessages(totalCount, topic, queueId, true);\n        //Thread.sleep(10);\n\n        StoreTestUtil.waitCommitLogReput((DefaultMessageStore) messageStore);\n\n        long messageStoreTimeStamp = messageStore.getMessageStoreTimeStamp(topic, wrongQueueId, -1);\n\n        assertThat(messageStoreTimeStamp).isEqualTo(-1);\n    }\n\n    @Test\n    public void testGetMessageStoreTimeStamp() {\n        final int totalCount = 10;\n        int queueId = 0;\n        String topic = \"FooBar\";\n        AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, false);\n        //Thread.sleep(10);\n        StoreTestUtil.waitCommitLogReput((DefaultMessageStore) messageStore);\n\n        ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId);\n        int minOffsetInQueue = (int) consumeQueue.getMinOffsetInQueue();\n        for (int i = minOffsetInQueue; i < consumeQueue.getMaxOffsetInQueue(); i++) {\n            long messageStoreTimeStamp = messageStore.getMessageStoreTimeStamp(topic, queueId, i);\n            assertThat(messageStoreTimeStamp).isEqualTo(appendMessageResults[i].getStoreTimestamp());\n        }\n    }\n\n    @Test\n    public void testGetStoreTime_ParamIsNull() {\n        long storeTime = getStoreTime(null);\n\n        assertThat(storeTime).isEqualTo(-1);\n    }\n\n    @Test\n    public void testGetStoreTime_EverythingIsOk() {\n        final int totalCount = 10;\n        int queueId = 0;\n        String topic = \"FooBar\";\n        AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, false);\n        //Thread.sleep(10);\n        StoreTestUtil.waitCommitLogReput((DefaultMessageStore) messageStore);\n        ConsumeQueueInterface consumeQueue = messageStore.getConsumeQueue(topic, queueId);\n\n        for (int i = 0; i < totalCount; i++) {\n            CqUnit cqUnit = consumeQueue.get(i);\n            long storeTime = getStoreTime(cqUnit);\n            assertThat(storeTime).isEqualTo(appendMessageResults[i].getStoreTimestamp());\n        }\n    }\n\n    @Test\n    public void testGetStoreTime_PhyOffsetIsLessThanCommitLogMinOffset() {\n        long phyOffset = -10;\n        int size = 138;\n        CqUnit cqUnit = new CqUnit(0, phyOffset, size, 0);\n        long storeTime = getStoreTime(cqUnit);\n\n        assertThat(storeTime).isEqualTo(-1);\n    }\n\n    @Test\n    public void testPutMessage_whenMessagePropertyIsTooLong() throws ConsumeQueueException {\n        String topicName = \"messagePropertyIsTooLongTest\";\n        MessageExtBrokerInner illegalMessage = buildSpecifyLengthPropertyMessage(\"123\".getBytes(StandardCharsets.UTF_8), topicName, Short.MAX_VALUE + 1);\n        assertEquals(messageStore.putMessage(illegalMessage).getPutMessageStatus(), PutMessageStatus.PROPERTIES_SIZE_EXCEEDED);\n        assertEquals(0L, messageStore.getQueueStore().getMaxOffset(topicName, 0).longValue());\n        MessageExtBrokerInner normalMessage = buildSpecifyLengthPropertyMessage(\"123\".getBytes(StandardCharsets.UTF_8), topicName, 100);\n        assertEquals(messageStore.putMessage(normalMessage).getPutMessageStatus(), PutMessageStatus.PUT_OK);\n        assertEquals(1L, messageStore.getQueueStore().getMaxOffset(topicName, 0).longValue());\n    }\n\n    private DefaultMessageStore getDefaultMessageStore() {\n        return (DefaultMessageStore) this.messageStore;\n    }\n\n    private AppendMessageResult[] putMessages(int totalCount, String topic, int queueId) {\n        return putMessages(totalCount, topic, queueId, false);\n    }\n\n    private AppendMessageResult[] putMessages(int totalCount, String topic, int queueId, boolean interval) {\n        AppendMessageResult[] appendMessageResultArray = new AppendMessageResult[totalCount];\n        for (int i = 0; i < totalCount; i++) {\n            String messageBody = buildMessageBodyByOffset(storeMessage, i);\n\n            MessageExtBrokerInner msgInner =\n                i < totalCount / 2 ? buildMessage(messageBody.getBytes(), topic) : buildIPv6HostMessage(messageBody.getBytes(), topic);\n            msgInner.setQueueId(queueId);\n            PutMessageResult result = messageStore.putMessage(msgInner);\n            appendMessageResultArray[i] = result.getAppendMessageResult();\n            assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK);\n            if (interval) {\n                try {\n                    Thread.sleep(10);\n                } catch (InterruptedException e) {\n                    throw new RuntimeException(\"Thread sleep ERROR\");\n                }\n            }\n        }\n        return appendMessageResultArray;\n    }\n\n    private long getMaxOffset(AppendMessageResult[] appendMessageResultArray) {\n        if (appendMessageResultArray == null) {\n            return 0;\n        }\n        AppendMessageResult last = appendMessageResultArray[appendMessageResultArray.length - 1];\n        return last.getWroteOffset() + last.getWroteBytes();\n    }\n\n    private String buildMessageBodyByOffset(String message, long i) {\n        return String.format(\"%s offset %d\", message, i);\n    }\n\n    private long getStoreTime(CqUnit cqUnit) {\n        try {\n            Class abstractConsumeQueueStore = getDefaultMessageStore().getQueueStore().getClass().getSuperclass();\n            Method getStoreTime = abstractConsumeQueueStore.getDeclaredMethod(\"getStoreTime\", CqUnit.class);\n            getStoreTime.setAccessible(true);\n            return (long) getStoreTime.invoke(getDefaultMessageStore().getQueueStore(), cqUnit);\n        } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    private MessageExtBrokerInner buildMessage(byte[] messageBody, String topic) {\n        MessageExtBrokerInner msg = new MessageExtBrokerInner();\n        msg.setTopic(topic);\n        msg.setTags(\"TAG1\");\n        msg.setKeys(\"Hello\");\n        msg.setBody(messageBody);\n        msg.setKeys(String.valueOf(System.currentTimeMillis()));\n        msg.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal);\n        msg.setSysFlag(0);\n        msg.setBornTimestamp(System.currentTimeMillis());\n        msg.setStoreHost(storeHost);\n        msg.setBornHost(bornHost);\n        msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n        return msg;\n    }\n\n    private MessageExtBrokerInner buildSpecifyLengthPropertyMessage(byte[] messageBody, String topic, int length) {\n        StringBuilder stringBuilder = new StringBuilder();\n        Random random = new Random();\n        for (int i = 0; i < length; i++) {\n            stringBuilder.append(random.nextInt(10));\n        }\n        MessageExtBrokerInner msg = new MessageExtBrokerInner();\n        msg.putUserProperty(\"test\", stringBuilder.toString());\n        msg.setTopic(topic);\n        msg.setTags(\"TAG1\");\n        msg.setKeys(\"Hello\");\n        msg.setBody(messageBody);\n        msg.setQueueId(0);\n        msg.setBornTimestamp(System.currentTimeMillis());\n        msg.setStoreHost(storeHost);\n        msg.setBornHost(bornHost);\n        msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n        return msg;\n    }\n\n    private MessageExtBrokerInner buildIPv6HostMessage(byte[] messageBody, String topic) {\n        MessageExtBrokerInner msg = new MessageExtBrokerInner();\n        msg.setTopic(topic);\n        msg.setTags(\"TAG1\");\n        msg.setKeys(\"Hello\");\n        msg.setBody(messageBody);\n        msg.setMsgId(\"24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0\");\n        msg.setKeys(String.valueOf(System.currentTimeMillis()));\n        msg.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal);\n        msg.setSysFlag(0);\n        msg.setBornHostV6Flag();\n        msg.setStoreHostAddressV6Flag();\n        msg.setBornTimestamp(System.currentTimeMillis());\n        try {\n            msg.setBornHost(new InetSocketAddress(InetAddress.getByName(\"1050:0000:0000:0000:0005:0600:300c:326b\"), 0));\n        } catch (UnknownHostException e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n\n        try {\n            msg.setStoreHost(new InetSocketAddress(InetAddress.getByName(\"::1\"), 0));\n        } catch (UnknownHostException e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n        return msg;\n    }\n\n    private MessageExtBrokerInner buildMessage() {\n        return buildMessage(messageBody, messageTopic);\n    }\n\n    public MessageExtBatch buildMessageBatch(MessageBatch msgBatch) {\n        MessageExtBatch msgExtBatch = new MessageExtBatch();\n        msgExtBatch.setTopic(messageTopic);\n        msgExtBatch.setTags(\"TAG1\");\n        msgExtBatch.setKeys(\"Hello\");\n        msgExtBatch.setBody(msgBatch.getBody());\n        msgExtBatch.setKeys(String.valueOf(System.currentTimeMillis()));\n        msgExtBatch.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal);\n        msgExtBatch.setSysFlag(0);\n        msgExtBatch.setBornTimestamp(System.currentTimeMillis());\n        msgExtBatch.setStoreHost(storeHost);\n        msgExtBatch.setBornHost(bornHost);\n        return msgExtBatch;\n    }\n\n    @Test\n    public void testGroupCommit() throws Exception {\n        long totalMsgs = 10;\n        queueTotal = 1;\n        messageBody = storeMessage.getBytes();\n        for (long i = 0; i < totalMsgs; i++) {\n            messageStore.putMessage(buildMessage());\n        }\n\n        for (long i = 0; i < totalMsgs; i++) {\n            GetMessageResult result = messageStore.getMessage(\"GROUP_A\", \"TOPIC_A\", 0, i, 1024 * 1024, null);\n            assertThat(result).isNotNull();\n            result.release();\n        }\n        verifyThatMasterIsFunctional(totalMsgs, messageStore);\n    }\n\n    @Test\n    public void testMaxOffset() throws InterruptedException, ConsumeQueueException {\n        int firstBatchMessages = 3;\n        int queueId = 0;\n        messageBody = storeMessage.getBytes();\n\n        assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId)).isEqualTo(0);\n\n        for (int i = 0; i < firstBatchMessages; i++) {\n            final MessageExtBrokerInner msg = buildMessage();\n            msg.setQueueId(queueId);\n            messageStore.putMessage(msg);\n        }\n\n        while (messageStore.dispatchBehindBytes() != 0) {\n            TimeUnit.MILLISECONDS.sleep(1);\n        }\n\n        assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId)).isEqualTo(firstBatchMessages);\n\n        // Disable the dispatcher\n        messageStore.getDispatcherList().clear();\n\n        int secondBatchMessages = 2;\n\n        for (int i = 0; i < secondBatchMessages; i++) {\n            final MessageExtBrokerInner msg = buildMessage();\n            msg.setQueueId(queueId);\n            messageStore.putMessage(msg);\n        }\n\n        assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId)).isEqualTo(firstBatchMessages);\n        assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId, true)).isEqualTo(firstBatchMessages);\n        assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId, false)).isEqualTo(firstBatchMessages + secondBatchMessages);\n    }\n\n    private MessageExtBrokerInner buildIPv6HostMessage() {\n        return buildIPv6HostMessage(messageBody, \"FooBar\");\n    }\n\n    private void verifyThatMasterIsFunctional(long totalMsgs, MessageStore master) {\n        for (long i = 0; i < totalMsgs; i++) {\n            master.putMessage(buildMessage());\n        }\n\n        StoreTestUtil.waitCommitLogReput((DefaultMessageStore) messageStore);\n\n        for (long i = 0; i < totalMsgs; i++) {\n            GetMessageResult result = master.getMessage(\"GROUP_A\", \"FooBar\", 0, i, 1024 * 1024, null);\n            assertThat(result).isNotNull();\n            result.release();\n\n        }\n    }\n\n    @Test\n    public void testPullSize() throws Exception {\n        String topic = \"pullSizeTopic\";\n\n        for (int i = 0; i < 32; i++) {\n            MessageExtBrokerInner messageExtBrokerInner = buildMessage();\n            messageExtBrokerInner.setTopic(topic);\n            messageExtBrokerInner.setQueueId(0);\n            messageStore.putMessage(messageExtBrokerInner);\n        }\n        // wait for consume queue build\n        // the sleep time should be great than consume queue flush interval\n        //Thread.sleep(100);\n        StoreTestUtil.waitCommitLogReput((DefaultMessageStore) messageStore);\n        String group = \"simple\";\n        GetMessageResult getMessageResult32 = messageStore.getMessage(group, topic, 0, 0, 32, null);\n        assertThat(getMessageResult32.getMessageBufferList().size()).isEqualTo(32);\n        getMessageResult32.release();\n\n        GetMessageResult getMessageResult20 = messageStore.getMessage(group, topic, 0, 0, 20, null);\n        assertThat(getMessageResult20.getMessageBufferList().size()).isEqualTo(20);\n\n        getMessageResult20.release();\n        GetMessageResult getMessageResult45 = messageStore.getMessage(group, topic, 0, 0, 10, null);\n        assertThat(getMessageResult45.getMessageBufferList().size()).isEqualTo(10);\n        getMessageResult45.release();\n\n    }\n\n    @Test\n    public void testRecover() throws Exception {\n        String topic = \"recoverTopic\";\n        messageBody = storeMessage.getBytes();\n        for (int i = 0; i < 100; i++) {\n            MessageExtBrokerInner messageExtBrokerInner = buildMessage();\n            messageExtBrokerInner.setTopic(topic);\n            messageExtBrokerInner.setQueueId(0);\n            messageStore.putMessage(messageExtBrokerInner);\n        }\n\n        // Thread.sleep(100);//wait for build consumer queue\n        StoreTestUtil.waitCommitLogReput((DefaultMessageStore) messageStore);\n\n        long maxPhyOffset = messageStore.getMaxPhyOffset();\n        long maxCqOffset = messageStore.getMaxOffsetInQueue(topic, 0);\n\n        //1.just reboot\n        messageStore.shutdown();\n        String storeRootDir = ((DefaultMessageStore) messageStore).getMessageStoreConfig().getStorePathRootDir();\n        messageStore = buildMessageStore(storeRootDir);\n        boolean load = messageStore.load();\n        assertTrue(load);\n        messageStore.start();\n        assertTrue(maxPhyOffset == messageStore.getMaxPhyOffset());\n        assertTrue(maxCqOffset == messageStore.getMaxOffsetInQueue(topic, 0));\n\n        //2.damage commit-log and reboot normal\n        for (int i = 0; i < 100; i++) {\n            MessageExtBrokerInner messageExtBrokerInner = buildMessage();\n            messageExtBrokerInner.setTopic(topic);\n            messageExtBrokerInner.setQueueId(0);\n            messageStore.putMessage(messageExtBrokerInner);\n        }\n        //Thread.sleep(100);\n        StoreTestUtil.waitCommitLogReput((DefaultMessageStore) messageStore);\n        long secondLastPhyOffset = messageStore.getMaxPhyOffset();\n        long secondLastCqOffset = messageStore.getMaxOffsetInQueue(topic, 0);\n\n        MessageExtBrokerInner messageExtBrokerInner = buildMessage();\n        messageExtBrokerInner.setTopic(topic);\n        messageExtBrokerInner.setQueueId(0);\n        messageStore.putMessage(messageExtBrokerInner);\n\n        messageStore.shutdown();\n\n        //damage last message\n        damageCommitLog((DefaultMessageStore) messageStore, secondLastPhyOffset);\n\n        //reboot\n        messageStore = buildMessageStore(storeRootDir);\n        load = messageStore.load();\n        assertTrue(load);\n        messageStore.start();\n        assertTrue(secondLastPhyOffset == messageStore.getMaxPhyOffset());\n        assertTrue(secondLastCqOffset == messageStore.getMaxOffsetInQueue(topic, 0));\n\n        //3.damage commitlog and reboot abnormal\n        for (int i = 0; i < 100; i++) {\n            messageExtBrokerInner = buildMessage();\n            messageExtBrokerInner.setTopic(topic);\n            messageExtBrokerInner.setQueueId(0);\n            messageStore.putMessage(messageExtBrokerInner);\n        }\n        //Thread.sleep(100);\n        StoreTestUtil.waitCommitLogReput((DefaultMessageStore) messageStore);\n        secondLastPhyOffset = messageStore.getMaxPhyOffset();\n        secondLastCqOffset = messageStore.getMaxOffsetInQueue(topic, 0);\n\n        messageExtBrokerInner = buildMessage();\n        messageExtBrokerInner.setTopic(topic);\n        messageExtBrokerInner.setQueueId(0);\n        messageStore.putMessage(messageExtBrokerInner);\n        messageStore.shutdown();\n\n        //damage last message\n        damageCommitLog((DefaultMessageStore) messageStore, secondLastPhyOffset);\n        //add abort file\n        String fileName = StorePathConfigHelper.getAbortFile(((DefaultMessageStore) messageStore).getMessageStoreConfig().getStorePathRootDir());\n        File file = new File(fileName);\n        UtilAll.ensureDirOK(file.getParent());\n        file.createNewFile();\n\n        messageStore = buildMessageStore(storeRootDir);\n        load = messageStore.load();\n        assertTrue(load);\n        messageStore.start();\n        assertTrue(secondLastPhyOffset == messageStore.getMaxPhyOffset());\n        assertTrue(secondLastCqOffset == messageStore.getMaxOffsetInQueue(topic, 0));\n\n        //message write again\n        for (int i = 0; i < 100; i++) {\n            messageExtBrokerInner = buildMessage();\n            messageExtBrokerInner.setTopic(topic);\n            messageExtBrokerInner.setQueueId(0);\n            messageStore.putMessage(messageExtBrokerInner);\n        }\n    }\n\n    @Test\n    public void testStorePathOK() {\n        if (messageStore instanceof DefaultMessageStore) {\n            assertTrue(fileExists(((DefaultMessageStore) messageStore).getStorePathPhysic()));\n            assertTrue(fileExists(((DefaultMessageStore) messageStore).getStorePathLogic()));\n        }\n    }\n\n    private boolean fileExists(String path) {\n        if (path != null) {\n            File f = new File(path);\n            return f.exists();\n        }\n        return false;\n    }\n\n    private void damageCommitLog(DefaultMessageStore store, long offset) throws Exception {\n        assertThat(store).isNotNull();\n        MessageStoreConfig messageStoreConfig = store.getMessageStoreConfig();\n        File file = new File(messageStoreConfig.getStorePathCommitLog() + File.separator + \"00000000000000000000\");\n        try (RandomAccessFile raf = new RandomAccessFile(file, \"rw\");\n             FileChannel fileChannel = raf.getChannel()) {\n            MappedByteBuffer mappedByteBuffer = fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, 1024 * 1024 * 10);\n            int bodyLen = mappedByteBuffer.getInt((int) offset + 84);\n            int topicLenIndex = (int) offset + 84 + bodyLen + 4;\n            mappedByteBuffer.position(topicLenIndex);\n            mappedByteBuffer.putInt(0);\n            mappedByteBuffer.putInt(0);\n            mappedByteBuffer.putInt(0);\n            mappedByteBuffer.putInt(0);\n            mappedByteBuffer.force();\n            fileChannel.force(true);\n        }\n    }\n\n    @Test\n    public void testPutMsgExceedsMaxLength() {\n        messageBody = new byte[4 * 1024 * 1024 + 1];\n        MessageExtBrokerInner msg = buildMessage();\n\n        PutMessageResult result = messageStore.putMessage(msg);\n        assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.MESSAGE_ILLEGAL);\n    }\n\n    @Test\n    public void testPutMsgBatchExceedsMaxLength() {\n        messageBody = new byte[4 * 1024 * 1024 + 1];\n        MessageExtBrokerInner msg1 = buildMessage();\n        MessageExtBrokerInner msg2 = buildMessage();\n        MessageExtBrokerInner msg3 = buildMessage();\n\n        MessageBatch msgBatch = MessageBatch.generateFromList(Arrays.asList(msg1, msg2, msg3));\n        msgBatch.setBody(msgBatch.encode());\n\n        MessageExtBatch msgExtBatch = buildMessageBatch(msgBatch);\n\n        try {\n            PutMessageResult result = this.messageStore.putMessages(msgExtBatch);\n        } catch (Exception e) {\n            assertThat(e.getMessage()).contains(\"message body size exceeded\");\n        }\n    }\n\n    @Test\n    public void testPutMsgWhenReplicasNotEnough() {\n        MessageStoreConfig messageStoreConfig = ((DefaultMessageStore) this.messageStore).getMessageStoreConfig();\n        messageStoreConfig.setBrokerRole(BrokerRole.SYNC_MASTER);\n        messageStoreConfig.setTotalReplicas(2);\n        messageStoreConfig.setInSyncReplicas(2);\n        messageStoreConfig.setEnableAutoInSyncReplicas(false);\n        ((DefaultMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(true);\n        this.messageStore.setAliveReplicaNumInGroup(1);\n\n        MessageExtBrokerInner msg = buildMessage();\n        PutMessageResult result = this.messageStore.putMessage(msg);\n        assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH);\n        ((DefaultMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(false);\n    }\n\n    @Test\n    public void testPutMsgWhenAdaptiveDegradation() {\n        MessageStoreConfig messageStoreConfig = ((DefaultMessageStore) this.messageStore).getMessageStoreConfig();\n        messageStoreConfig.setBrokerRole(BrokerRole.SYNC_MASTER);\n        messageStoreConfig.setTotalReplicas(2);\n        messageStoreConfig.setInSyncReplicas(2);\n        messageStoreConfig.setEnableAutoInSyncReplicas(true);\n        ((DefaultMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(true);\n        this.messageStore.setAliveReplicaNumInGroup(1);\n\n        MessageExtBrokerInner msg = buildMessage();\n        PutMessageResult result = this.messageStore.putMessage(msg);\n        assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK);\n        ((DefaultMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(false);\n        messageStoreConfig.setEnableAutoInSyncReplicas(false);\n    }\n\n    @Test\n    public void testGetBulkCommitLogData() {\n        DefaultMessageStore defaultMessageStore = (DefaultMessageStore) messageStore;\n\n        messageBody = new byte[2 * 1024 * 1024];\n\n        for (int i = 0; i < 10; i++) {\n            MessageExtBrokerInner msg1 = buildMessage();\n            messageStore.putMessage(msg1);\n        }\n\n        System.out.printf(\"%d%n\", defaultMessageStore.getMaxPhyOffset());\n\n        List<SelectMappedBufferResult> bufferResultList = defaultMessageStore.getBulkCommitLogData(0, (int) defaultMessageStore.getMaxPhyOffset());\n        List<MessageExt> msgList = new ArrayList<>();\n        for (SelectMappedBufferResult bufferResult : bufferResultList) {\n            msgList.addAll(MessageDecoder.decodesBatch(bufferResult.getByteBuffer(), true, false, false));\n            bufferResult.release();\n        }\n\n        assertThat(msgList.size()).isEqualTo(10);\n    }\n\n    @Test\n    public void testPutLongMessage() throws Exception {\n        MessageExtBrokerInner messageExtBrokerInner = buildMessage();\n        CommitLog commitLog = ((DefaultMessageStore) messageStore).getCommitLog();\n        MessageStoreConfig messageStoreConfig = ((DefaultMessageStore) messageStore).getMessageStoreConfig();\n        MessageExtEncoder.PutMessageThreadLocal putMessageThreadLocal = commitLog.getPutMessageThreadLocal().get();\n\n        //body size, topic size, properties size exactly equal to max size\n        messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]);\n        messageExtBrokerInner.setTopic(new String(new byte[127]));\n        messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE]));\n        PutMessageResult encodeResult1 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);\n        assertTrue(encodeResult1 == null);\n\n        //body size exactly more than max message body size\n        messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize() + 1]);\n        PutMessageResult encodeResult2 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);\n        assertTrue(encodeResult2.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL);\n\n        //body size exactly equal to max message size\n        messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize() + 64 * 1024]);\n        PutMessageResult encodeResult3 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);\n        assertTrue(encodeResult3.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL);\n\n        //message properties length more than properties maxSize\n        messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]);\n        messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE + 1]));\n        PutMessageResult encodeResult4 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);\n        assertTrue(encodeResult4.getPutMessageStatus() == PutMessageStatus.PROPERTIES_SIZE_EXCEEDED);\n\n        //message length more than buffer length capacity\n        messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]);\n        messageExtBrokerInner.setTopic(new String(new byte[Short.MAX_VALUE]));\n        messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE]));\n        PutMessageResult encodeResult5 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);\n        assertTrue(encodeResult5.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL);\n    }\n\n    @Test\n    public void testDynamicMaxMessageSize() {\n        MessageExtBrokerInner messageExtBrokerInner = buildMessage();\n        MessageStoreConfig messageStoreConfig = ((DefaultMessageStore) messageStore).getMessageStoreConfig();\n        int originMaxMessageSize = messageStoreConfig.getMaxMessageSize();\n\n        messageExtBrokerInner.setBody(new byte[originMaxMessageSize + 10]);\n        PutMessageResult putMessageResult = messageStore.putMessage(messageExtBrokerInner);\n        assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL);\n\n        int newMaxMessageSize = originMaxMessageSize + 10;\n        messageStoreConfig.setMaxMessageSize(newMaxMessageSize);\n        putMessageResult = messageStore.putMessage(messageExtBrokerInner);\n        assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK);\n\n        messageStoreConfig.setMaxMessageSize(10);\n        putMessageResult = messageStore.putMessage(messageExtBrokerInner);\n        assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL);\n\n        messageStoreConfig.setMaxMessageSize(originMaxMessageSize);\n    }\n\n    @Test\n    public void testDeleteTopics() {\n        MessageStoreConfig messageStoreConfig = messageStore.getMessageStoreConfig();\n        ConcurrentMap<String, ConcurrentMap<Integer, ConsumeQueueInterface>> consumeQueueTable =\n            ((DefaultMessageStore) messageStore).getConsumeQueueTable();\n        for (int i = 0; i < 10; i++) {\n            ConcurrentMap<Integer, ConsumeQueueInterface> cqTable = new ConcurrentHashMap<>();\n            String topicName = \"topic-\" + i;\n            for (int j = 0; j < 4; j++) {\n                ConsumeQueue consumeQueue = new ConsumeQueue(topicName, j, messageStoreConfig.getStorePathRootDir(),\n                    messageStoreConfig.getMappedFileSizeConsumeQueue(), (DefaultMessageStore) messageStore);\n                cqTable.put(j, consumeQueue);\n            }\n            consumeQueueTable.put(topicName, cqTable);\n        }\n        Assert.assertEquals(consumeQueueTable.size(), 10);\n        HashSet<String> resultSet = Sets.newHashSet(\"topic-3\", \"topic-5\");\n        messageStore.deleteTopics(Sets.difference(consumeQueueTable.keySet(), resultSet));\n        Assert.assertEquals(consumeQueueTable.size(), 2);\n        Assert.assertEquals(resultSet, consumeQueueTable.keySet());\n    }\n\n    @Test\n    public void testCleanUnusedTopic() {\n        MessageStoreConfig messageStoreConfig = messageStore.getMessageStoreConfig();\n        ConcurrentMap<String, ConcurrentMap<Integer, ConsumeQueueInterface>> consumeQueueTable =\n            ((DefaultMessageStore) messageStore).getConsumeQueueTable();\n        for (int i = 0; i < 10; i++) {\n            ConcurrentMap<Integer, ConsumeQueueInterface> cqTable = new ConcurrentHashMap<>();\n            String topicName = \"topic-\" + i;\n            for (int j = 0; j < 4; j++) {\n                ConsumeQueue consumeQueue = new ConsumeQueue(topicName, j, messageStoreConfig.getStorePathRootDir(),\n                    messageStoreConfig.getMappedFileSizeConsumeQueue(), (DefaultMessageStore) messageStore);\n                cqTable.put(j, consumeQueue);\n            }\n            consumeQueueTable.put(topicName, cqTable);\n        }\n        Assert.assertEquals(consumeQueueTable.size(), 10);\n        HashSet<String> resultSet = Sets.newHashSet(\"topic-3\", \"topic-5\");\n        messageStore.cleanUnusedTopic(resultSet);\n        Assert.assertEquals(consumeQueueTable.size(), 2);\n        Assert.assertEquals(resultSet, consumeQueueTable.keySet());\n    }\n\n    @Test\n    public void testChangeStoreConfig() {\n        Properties properties = new Properties();\n        properties.setProperty(\"enableBatchPush\", \"true\");\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        MixAll.properties2Object(properties, messageStoreConfig);\n        assertThat(messageStoreConfig.isEnableBatchPush()).isTrue();\n    }\n\n    @Test\n    public void testRecoverWithRocksDBOffsets() throws Exception {\n        // Test that recovery process considers RocksDB offsets when IndexRocksDBEnable or TransRocksDBEnable is enabled\n        UUID uuid = UUID.randomUUID();\n        String storePathRootDir = System.getProperty(\"java.io.tmpdir\") + File.separator + \"store-recover-test-\" + uuid.toString();\n\n        try {\n            // Test case 1: IndexRocksDBEnable enabled with valid offset\n            // index offset: 500L, expected: min(consumeQueueOffset, 500L)\n            testRecoverWithRocksDBOffset(storePathRootDir + \"-1\", true, false, 500L, null);\n\n            // Test case 2: TransRocksDBEnable enabled with valid offset\n            // trans offset: 600L, expected: min(consumeQueueOffset, 600L)\n            testRecoverWithRocksDBOffset(storePathRootDir + \"-2\", false, true, null, 600L);\n\n            // Test case 3: Both enabled, take minimum value\n            // index offset: 500L, trans offset: 300L, expected: min(consumeQueueOffset, 500L, 300L)\n            testRecoverWithRocksDBOffset(storePathRootDir + \"-3\", true, true, 500L, 300L);\n        } finally {\n            // Clean up all test directories\n            for (int i = 1; i <= 3; i++) {\n                UtilAll.deleteFile(new File(storePathRootDir + \"-\" + i));\n            }\n        }\n    }\n\n    private void testRecoverWithRocksDBOffset(String storePathRootDir, boolean indexEnable,\n        boolean transEnable, Long indexOffset, Long transOffset) throws Exception {\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setMappedFileSizeCommitLog(1024 * 1024 * 10);\n        messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 1024 * 10);\n        messageStoreConfig.setMaxHashSlotNum(10000);\n        messageStoreConfig.setMaxIndexNum(100 * 100);\n        messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);\n        messageStoreConfig.setHaListenPort(0);\n        messageStoreConfig.setStorePathRootDir(storePathRootDir);\n        messageStoreConfig.setIndexRocksDBEnable(indexEnable);\n        messageStoreConfig.setTransRocksDBEnable(transEnable);\n\n        DefaultMessageStore store = new DefaultMessageStore(messageStoreConfig,\n            new BrokerStatsManager(\"test\", true),\n            new MyMessageArrivingListener(),\n            new BrokerConfig(), new ConcurrentHashMap<>());\n\n        // Get the actual consumeQueueStore dispatchFromPhyOffset before loading (normal recovery)\n        long consumeQueueOffset = store.getQueueStore().getDispatchFromPhyOffset(true);\n\n        // Calculate expected value: min of consumeQueueOffset and RocksDB offsets\n        long calculatedExpected = consumeQueueOffset;\n        if (indexEnable && indexOffset != null && indexOffset > 0) {\n            calculatedExpected = Math.min(calculatedExpected, indexOffset);\n        }\n        if (transEnable && transOffset != null && transOffset > 0) {\n            calculatedExpected = Math.min(calculatedExpected, transOffset);\n        }\n\n        // Mock messageRocksDBStorage\n        java.lang.reflect.Field field = DefaultMessageStore.class.getDeclaredField(\"messageRocksDBStorage\");\n        field.setAccessible(true);\n        org.apache.rocketmq.store.rocksdb.MessageRocksDBStorage mockStorage =\n            mock(org.apache.rocketmq.store.rocksdb.MessageRocksDBStorage.class);\n        field.set(store, mockStorage);\n\n        // Spy commitLog to verify invocation and capture the dispatchFromPhyOffset value\n        java.lang.reflect.Field commitLogField = DefaultMessageStore.class.getDeclaredField(\"commitLog\");\n        commitLogField.setAccessible(true);\n        CommitLog commitLog = (CommitLog) commitLogField.get(store);\n        CommitLog spyCommitLog = spy(commitLog);\n        commitLogField.set(store, spyCommitLog);\n\n        // Use ArgumentCaptor to capture the dispatchFromPhyOffset value\n        ArgumentCaptor<Long> offsetCaptor = ArgumentCaptor.forClass(Long.class);\n\n        // Load store, which will call recover method\n        boolean loadResult = store.load();\n        assertTrue(loadResult);\n\n        // Verify recoverNormally or recoverAbnormally is called and capture the argument\n        // Since it's a new store (no abort file), it should call recoverNormally\n        verify(spyCommitLog, atLeastOnce()).recoverNormally(offsetCaptor.capture());\n\n        // Verify the dispatchFromPhyOffset value is correct (should be the minimum)\n        Long actualDispatchFromPhyOffset = offsetCaptor.getValue();\n        assertThat(actualDispatchFromPhyOffset).isEqualTo(calculatedExpected);\n\n        // Clean up resources\n        store.shutdown();\n        store.destroy();\n    }\n\n    private class MyMessageArrivingListener implements MessageArrivingListener {\n        @Override\n        public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime,\n            byte[] filterBitMap, Map<String, String> properties) {\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/FlushDiskWatcherTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport org.apache.rocketmq.store.CommitLog.GroupCommitRequest;\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport java.util.LinkedList;\nimport java.util.List;\n\npublic class FlushDiskWatcherTest {\n\n    private final long timeoutMill = 5000;\n\n    @Test\n    public void testTimeout() throws Exception {\n        FlushDiskWatcher flushDiskWatcher = new FlushDiskWatcher();\n        flushDiskWatcher.setDaemon(true);\n        flushDiskWatcher.start();\n\n        int count = 100;\n        List<GroupCommitRequest> requestList = new LinkedList<>();\n        for (int i = 0; i < count; i++) {\n            GroupCommitRequest groupCommitRequest =\n                    new GroupCommitRequest(0, timeoutMill);\n            requestList.add(groupCommitRequest);\n            flushDiskWatcher.add(groupCommitRequest);\n        }\n        Thread.sleep(2 * timeoutMill);\n\n        for (GroupCommitRequest request : requestList) {\n            request.wakeupCustomer(PutMessageStatus.PUT_OK);\n        }\n\n        for (GroupCommitRequest request : requestList) {\n            Assert.assertTrue(request.future().isDone());\n            Assert.assertEquals(request.future().get(), PutMessageStatus.FLUSH_DISK_TIMEOUT);\n        }\n        Assert.assertEquals(flushDiskWatcher.queueSize(), 0);\n        flushDiskWatcher.shutdown();\n    }\n\n    @Test\n    public void testWatcher() throws Exception {\n        FlushDiskWatcher flushDiskWatcher = new FlushDiskWatcher();\n        flushDiskWatcher.setDaemon(true);\n        flushDiskWatcher.start();\n\n        int count = 100;\n        List<GroupCommitRequest> requestList = new LinkedList<>();\n        for (int i = 0; i < count; i++) {\n            GroupCommitRequest groupCommitRequest =\n                    new GroupCommitRequest(0, timeoutMill);\n            requestList.add(groupCommitRequest);\n            flushDiskWatcher.add(groupCommitRequest);\n            groupCommitRequest.wakeupCustomer(PutMessageStatus.PUT_OK);\n        }\n        Thread.sleep((timeoutMill << 20) / 1000000);\n        for (GroupCommitRequest request : requestList) {\n            Assert.assertTrue(request.future().isDone());\n            Assert.assertEquals(request.future().get(), PutMessageStatus.PUT_OK);\n        }\n        Assert.assertEquals(flushDiskWatcher.queueSize(), 0);\n        flushDiskWatcher.shutdown();\n    }\n\n\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/GetMessageResultTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class GetMessageResultTest {\n\n    @Test\n    public void testAddMessage() {\n        GetMessageResult getMessageResult = new GetMessageResult();\n        SelectMappedBufferResult mappedBufferResult1 = new SelectMappedBufferResult(0, null, 4 * 1024, null);\n        getMessageResult.addMessage(mappedBufferResult1);\n\n        SelectMappedBufferResult mappedBufferResult2 = new SelectMappedBufferResult(0, null, 2 * 4 * 1024, null);\n        getMessageResult.addMessage(mappedBufferResult2, 0);\n\n        SelectMappedBufferResult mappedBufferResult3 = new SelectMappedBufferResult(0, null, 4 * 4 * 1024, null);\n        getMessageResult.addMessage(mappedBufferResult3, 0, 2);\n\n        Assert.assertEquals(getMessageResult.getMessageQueueOffset().size(), 2);\n        Assert.assertEquals(getMessageResult.getMessageBufferList().size(), 3);\n        Assert.assertEquals(getMessageResult.getMessageMapedList().size(), 3);\n        Assert.assertEquals(getMessageResult.getMessageCount(), 4);\n        Assert.assertEquals(getMessageResult.getMsgCount4Commercial(), 1 + 2 + 4);\n        Assert.assertEquals(getMessageResult.getBufferTotalSize(), (1 + 2 + 4) * 4 * 1024);\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/HATest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport java.io.File;\nimport java.lang.reflect.Field;\nimport java.lang.reflect.InvocationTargetException;\nimport java.lang.reflect.Method;\nimport java.net.InetAddress;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.time.Duration;\nimport java.util.UUID;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.config.FlushDiskType;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.ha.HAConnectionState;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.junit.After;\nimport org.junit.Assume;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static java.util.concurrent.TimeUnit.SECONDS;\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertArrayEquals;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertTrue;\n\npublic class HATest {\n    private final String storeMessage = \"Once, there was a chance for me!\";\n    private int queueTotal = 100;\n    private AtomicInteger queueId = new AtomicInteger(0);\n    private SocketAddress bornHost;\n    private SocketAddress storeHost;\n    private byte[] messageBody;\n\n    private MessageStore messageStore;\n    private MessageStore slaveMessageStore;\n    private MessageStoreConfig masterMessageStoreConfig;\n    private MessageStoreConfig slaveStoreConfig;\n    private BrokerStatsManager brokerStatsManager = new BrokerStatsManager(\"simpleTest\", true);\n    private String storePathRootParentDir = System.getProperty(\"java.io.tmpdir\") + File.separator + UUID.randomUUID();\n    private String storePathRootDir = storePathRootParentDir + File.separator + \"store\";\n\n    @Before\n    public void init() throws Exception {\n        storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123);\n        bornHost = new InetSocketAddress(InetAddress.getByName(\"127.0.0.1\"), 0);\n        masterMessageStoreConfig = new MessageStoreConfig();\n        masterMessageStoreConfig.setBrokerRole(BrokerRole.SYNC_MASTER);\n        masterMessageStoreConfig.setStorePathRootDir(storePathRootDir + File.separator + \"master\");\n        masterMessageStoreConfig.setStorePathCommitLog(storePathRootDir + File.separator + \"master\" + File.separator + \"commitlog\");\n        masterMessageStoreConfig.setHaListenPort(0);\n        masterMessageStoreConfig.setTotalReplicas(2);\n        masterMessageStoreConfig.setInSyncReplicas(2);\n        masterMessageStoreConfig.setHaHousekeepingInterval(2 * 1000);\n        masterMessageStoreConfig.setHaSendHeartbeatInterval(1000);\n        buildMessageStoreConfig(masterMessageStoreConfig);\n        slaveStoreConfig = new MessageStoreConfig();\n        slaveStoreConfig.setBrokerRole(BrokerRole.SLAVE);\n        slaveStoreConfig.setStorePathRootDir(storePathRootDir + File.separator + \"slave\");\n        slaveStoreConfig.setStorePathCommitLog(storePathRootDir + File.separator + \"slave\" + File.separator + \"commitlog\");\n        slaveStoreConfig.setHaListenPort(0);\n        slaveStoreConfig.setTotalReplicas(2);\n        slaveStoreConfig.setInSyncReplicas(2);\n        slaveStoreConfig.setHaHousekeepingInterval(2 * 1000);\n        slaveStoreConfig.setHaSendHeartbeatInterval(1000);\n        buildMessageStoreConfig(slaveStoreConfig);\n        messageStore = buildMessageStore(masterMessageStoreConfig, 0L);\n        slaveMessageStore = buildMessageStore(slaveStoreConfig, 1L);\n        boolean load = messageStore.load();\n        boolean slaveLoad = slaveMessageStore.load();\n        assertTrue(load);\n        assertTrue(slaveLoad);\n        messageStore.start();\n\n        slaveMessageStore.updateHaMasterAddress(\"127.0.0.1:\" + masterMessageStoreConfig.getHaListenPort());\n        slaveMessageStore.start();\n        slaveMessageStore.updateHaMasterAddress(\"127.0.0.1:\" + masterMessageStoreConfig.getHaListenPort());\n        await().atMost(6, SECONDS).until(() -> slaveMessageStore.getHaService().getHAClient().getCurrentState() == HAConnectionState.TRANSFER);\n    }\n\n    @Test\n    public void testHandleHA() {\n        long totalMsgs = 10;\n        queueTotal = 1;\n        messageBody = storeMessage.getBytes();\n        for (long i = 0; i < totalMsgs; i++) {\n            messageStore.putMessage(buildMessage());\n        }\n        for (long i = 0; i < totalMsgs; i++) {\n            final long index = i;\n            Boolean exist = await().atMost(Duration.ofSeconds(5)).until(() -> {\n                GetMessageResult result = slaveMessageStore.getMessage(\"GROUP_A\", \"FooBar\", 0, index, 1024 * 1024, null);\n                if (result == null) {\n                    return false;\n                }\n                boolean flag = GetMessageStatus.FOUND == result.getStatus();\n                result.release();\n                return flag;\n\n            }, item -> item);\n            assertTrue(exist);\n        }\n    }\n\n    @Test\n    public void testSemiSyncReplica() throws Exception {\n        long totalMsgs = 5;\n        queueTotal = 1;\n        messageBody = storeMessage.getBytes();\n        for (long i = 0; i < totalMsgs; i++) {\n            MessageExtBrokerInner msg = buildMessage();\n            CompletableFuture<PutMessageResult> putResultFuture = messageStore.asyncPutMessage(msg);\n            PutMessageResult result = putResultFuture.get();\n            assertEquals(PutMessageStatus.PUT_OK, result.getPutMessageStatus());\n            //message has been replicated to slave's commitLog, but maybe not dispatch to ConsumeQueue yet\n            //so direct read from commitLog by physical offset\n            MessageExt slaveMsg = slaveMessageStore.lookMessageByOffset(result.getAppendMessageResult().getWroteOffset());\n            assertNotNull(slaveMsg);\n            assertArrayEquals(msg.getBody(), slaveMsg.getBody());\n            assertEquals(msg.getTopic(), slaveMsg.getTopic());\n            assertEquals(msg.getTags(), slaveMsg.getTags());\n            assertEquals(msg.getKeys(), slaveMsg.getKeys());\n        }\n        //shutdown slave, putMessage should return FLUSH_SLAVE_TIMEOUT\n        slaveMessageStore.shutdown();\n\n        //wait to let master clean the slave's connection\n        await().atMost(Duration.ofSeconds(3)).until(() -> messageStore.getHaService().getConnectionCount().get() == 0);\n        for (long i = 0; i < totalMsgs; i++) {\n            CompletableFuture<PutMessageResult> putResultFuture = messageStore.asyncPutMessage(buildMessage());\n            PutMessageResult result = putResultFuture.get();\n            assertEquals(PutMessageStatus.FLUSH_SLAVE_TIMEOUT, result.getPutMessageStatus());\n        }\n    }\n\n    @Test\n    public void testSemiSyncReplicaWhenSlaveActingMaster() throws Exception {\n        // SKip MacOS\n        Assume.assumeFalse(MixAll.isMac());\n        long totalMsgs = 5;\n        queueTotal = 1;\n        messageBody = storeMessage.getBytes();\n        ((DefaultMessageStore) messageStore).getBrokerConfig().setEnableSlaveActingMaster(true);\n        for (long i = 0; i < totalMsgs; i++) {\n            MessageExtBrokerInner msg = buildMessage();\n            CompletableFuture<PutMessageResult> putResultFuture = messageStore.asyncPutMessage(msg);\n            PutMessageResult result = putResultFuture.get();\n            assertEquals(PutMessageStatus.PUT_OK, result.getPutMessageStatus());\n            //message has been replicated to slave's commitLog, but maybe not dispatch to ConsumeQueue yet\n            //so direct read from commitLog by physical offset\n            MessageExt slaveMsg = slaveMessageStore.lookMessageByOffset(result.getAppendMessageResult().getWroteOffset());\n            assertNotNull(slaveMsg);\n            assertArrayEquals(msg.getBody(), slaveMsg.getBody());\n            assertEquals(msg.getTopic(), slaveMsg.getTopic());\n            assertEquals(msg.getTags(), slaveMsg.getTags());\n            assertEquals(msg.getKeys(), slaveMsg.getKeys());\n        }\n\n        //shutdown slave, putMessage should return IN_SYNC_REPLICAS_NOT_ENOUGH\n        slaveMessageStore.shutdown();\n        messageStore.setAliveReplicaNumInGroup(1);\n\n        //wait to let master clean the slave's connection\n        Thread.sleep(masterMessageStoreConfig.getHaHousekeepingInterval() + 500);\n        for (long i = 0; i < totalMsgs; i++) {\n            CompletableFuture<PutMessageResult> putResultFuture = messageStore.asyncPutMessage(buildMessage());\n            PutMessageResult result = putResultFuture.get();\n            assertEquals(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH, result.getPutMessageStatus());\n        }\n\n        ((DefaultMessageStore) messageStore).getBrokerConfig().setEnableSlaveActingMaster(false);\n    }\n\n    @Test\n    public void testSemiSyncReplicaWhenAdaptiveDegradation() throws Exception {\n        long totalMsgs = 5;\n        queueTotal = 1;\n        messageBody = storeMessage.getBytes();\n        ((DefaultMessageStore) messageStore).getBrokerConfig().setEnableSlaveActingMaster(true);\n        messageStore.getMessageStoreConfig().setEnableAutoInSyncReplicas(true);\n        for (long i = 0; i < totalMsgs; i++) {\n            MessageExtBrokerInner msg = buildMessage();\n            CompletableFuture<PutMessageResult> putResultFuture = messageStore.asyncPutMessage(msg);\n            PutMessageResult result = putResultFuture.get();\n            assertEquals(PutMessageStatus.PUT_OK, result.getPutMessageStatus());\n            //message has been replicated to slave's commitLog, but maybe not dispatch to ConsumeQueue yet\n            //so direct read from commitLog by physical offset\n            final MessageExt[] slaveMsg = {null};\n            await().atMost(Duration.ofSeconds(3)).until(() -> {\n                slaveMsg[0] = slaveMessageStore.lookMessageByOffset(result.getAppendMessageResult().getWroteOffset());\n                return slaveMsg[0] != null;\n            });\n            assertArrayEquals(msg.getBody(), slaveMsg[0].getBody());\n            assertEquals(msg.getTopic(), slaveMsg[0].getTopic());\n            assertEquals(msg.getTags(), slaveMsg[0].getTags());\n            assertEquals(msg.getKeys(), slaveMsg[0].getKeys());\n        }\n\n        //shutdown slave, putMessage should return IN_SYNC_REPLICAS_NOT_ENOUGH\n        slaveMessageStore.shutdown();\n        messageStore.setAliveReplicaNumInGroup(1);\n\n        //wait to let master clean the slave's connection\n        await().atMost(Duration.ofSeconds(3)).until(() -> messageStore.getHaService().getConnectionCount().get() == 0);\n        for (long i = 0; i < totalMsgs; i++) {\n            CompletableFuture<PutMessageResult> putResultFuture = messageStore.asyncPutMessage(buildMessage());\n            PutMessageResult result = putResultFuture.get();\n            assertEquals(PutMessageStatus.PUT_OK, result.getPutMessageStatus());\n        }\n\n        ((DefaultMessageStore) messageStore).getBrokerConfig().setEnableSlaveActingMaster(false);\n        messageStore.getMessageStoreConfig().setEnableAutoInSyncReplicas(false);\n    }\n\n    @After\n    public void destroy() throws Exception {\n\n        slaveMessageStore.shutdown();\n        slaveMessageStore.destroy();\n        messageStore.shutdown();\n        messageStore.destroy();\n        File file = new File(storePathRootParentDir);\n        UtilAll.deleteFile(file);\n    }\n\n    private MessageStore buildMessageStore(MessageStoreConfig messageStoreConfig, long brokerId) throws Exception {\n        BrokerConfig brokerConfig = new BrokerConfig();\n        brokerConfig.setBrokerId(brokerId);\n        return new DefaultMessageStore(messageStoreConfig, brokerStatsManager, null, brokerConfig, new ConcurrentHashMap<>());\n    }\n\n    private void buildMessageStoreConfig(MessageStoreConfig messageStoreConfig) {\n        messageStoreConfig.setMappedFileSizeCommitLog(1024 * 1024 * 10);\n        messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 1024 * 10);\n        messageStoreConfig.setMaxHashSlotNum(10000);\n        messageStoreConfig.setMaxIndexNum(100 * 100);\n        messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);\n        messageStoreConfig.setFlushIntervalConsumeQueue(1);\n    }\n\n    private MessageExtBrokerInner buildMessage() {\n        MessageExtBrokerInner msg = new MessageExtBrokerInner();\n        msg.setTopic(\"FooBar\");\n        msg.setTags(\"TAG1\");\n        msg.setBody(messageBody);\n        msg.setKeys(String.valueOf(System.currentTimeMillis()));\n        msg.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal);\n        msg.setSysFlag(0);\n        msg.setBornTimestamp(System.currentTimeMillis());\n        msg.setStoreHost(storeHost);\n        msg.setBornHost(bornHost);\n        msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n        return msg;\n    }\n\n    private boolean isCommitLogAvailable(DefaultMessageStore store) {\n        try {\n            Field serviceField = store.getClass().getDeclaredField(\"reputMessageService\");\n            serviceField.setAccessible(true);\n            DefaultMessageStore.ReputMessageService reputService =\n                (DefaultMessageStore.ReputMessageService) serviceField.get(store);\n\n            Method method = DefaultMessageStore.ReputMessageService.class.getDeclaredMethod(\"isCommitLogAvailable\");\n            method.setAccessible(true);\n            return (boolean) method.invoke(reputService);\n        } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | NoSuchFieldException e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/MappedFileQueueTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport java.util.concurrent.CountDownLatch;\nimport org.apache.commons.lang3.RandomStringUtils;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.store.logfile.DefaultMappedFile;\nimport org.apache.rocketmq.store.logfile.MappedFile;\nimport org.assertj.core.util.Lists;\nimport org.junit.After;\nimport org.junit.Assume;\nimport org.junit.Test;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicBoolean;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class MappedFileQueueTest {\n\n    private String storePath = System.getProperty(\"java.io.tmpdir\") + File.separator + \"unit_test_store\";\n\n    @Test\n    public void testGetLastMappedFile() {\n        final String fixedMsg = \"0123456789abcdef\";\n\n        MappedFileQueue mappedFileQueue =\n            new MappedFileQueue(storePath + File.separator + \"a/\", 1024, null);\n\n        for (int i = 0; i < 1024; i++) {\n            MappedFile mappedFile = mappedFileQueue.getLastMappedFile(0);\n            assertThat(mappedFile).isNotNull();\n            assertThat(mappedFile.appendMessage(fixedMsg.getBytes())).isTrue();\n        }\n\n        mappedFileQueue.shutdown(1000);\n        mappedFileQueue.destroy();\n    }\n\n    @Test\n    public void testFindMappedFileByOffset() {\n        // four-byte string.\n        final String fixedMsg = \"abcd\";\n\n        MappedFileQueue mappedFileQueue =\n            new MappedFileQueue(storePath + File.separator + \"b/\", 1024, null);\n\n        for (int i = 0; i < 1024; i++) {\n            MappedFile mappedFile = mappedFileQueue.getLastMappedFile(0);\n            assertThat(mappedFile).isNotNull();\n            assertThat(mappedFile.appendMessage(fixedMsg.getBytes())).isTrue();\n        }\n\n        assertThat(mappedFileQueue.getMappedMemorySize()).isEqualTo(fixedMsg.getBytes().length * 1024);\n\n        MappedFile mappedFile = mappedFileQueue.findMappedFileByOffset(0);\n        assertThat(mappedFile).isNotNull();\n        assertThat(mappedFile.getFileFromOffset()).isEqualTo(0);\n\n        mappedFile = mappedFileQueue.findMappedFileByOffset(100);\n        assertThat(mappedFile).isNotNull();\n        assertThat(mappedFile.getFileFromOffset()).isEqualTo(0);\n\n        mappedFile = mappedFileQueue.findMappedFileByOffset(1024);\n        assertThat(mappedFile).isNotNull();\n        assertThat(mappedFile.getFileFromOffset()).isEqualTo(1024);\n\n        mappedFile = mappedFileQueue.findMappedFileByOffset(1024 + 100);\n        assertThat(mappedFile).isNotNull();\n        assertThat(mappedFile.getFileFromOffset()).isEqualTo(1024);\n\n        mappedFile = mappedFileQueue.findMappedFileByOffset(1024 * 2);\n        assertThat(mappedFile).isNotNull();\n        assertThat(mappedFile.getFileFromOffset()).isEqualTo(1024 * 2);\n\n        mappedFile = mappedFileQueue.findMappedFileByOffset(1024 * 2 + 100);\n        assertThat(mappedFile).isNotNull();\n        assertThat(mappedFile.getFileFromOffset()).isEqualTo(1024 * 2);\n\n        // over mapped memory size.\n        mappedFile = mappedFileQueue.findMappedFileByOffset(1024 * 4);\n        assertThat(mappedFile).isNull();\n\n        mappedFile = mappedFileQueue.findMappedFileByOffset(1024 * 4 + 100);\n        assertThat(mappedFile).isNull();\n\n        mappedFileQueue.shutdown(1000);\n        mappedFileQueue.destroy();\n    }\n\n    @Test\n    public void testFindMappedFileByOffset_StartOffsetIsNonZero() {\n        MappedFileQueue mappedFileQueue =\n            new MappedFileQueue(storePath + File.separator + \"b/\", 1024, null);\n\n        //Start from a non-zero offset\n        MappedFile mappedFile = mappedFileQueue.getLastMappedFile(1024);\n        assertThat(mappedFile).isNotNull();\n\n        assertThat(mappedFileQueue.findMappedFileByOffset(1025)).isEqualTo(mappedFile);\n\n        assertThat(mappedFileQueue.findMappedFileByOffset(0)).isNull();\n        assertThat(mappedFileQueue.findMappedFileByOffset(123, false)).isNull();\n        assertThat(mappedFileQueue.findMappedFileByOffset(123, true)).isEqualTo(mappedFile);\n\n        assertThat(mappedFileQueue.findMappedFileByOffset(0, false)).isNull();\n        assertThat(mappedFileQueue.findMappedFileByOffset(0, true)).isEqualTo(mappedFile);\n\n        mappedFileQueue.shutdown(1000);\n        mappedFileQueue.destroy();\n    }\n\n    @Test\n    public void testAppendMessage() {\n        final String fixedMsg = \"0123456789abcdef\";\n\n        MappedFileQueue mappedFileQueue =\n            new MappedFileQueue(storePath + File.separator + \"c/\", 1024, null);\n\n        for (int i = 0; i < 1024; i++) {\n            MappedFile mappedFile = mappedFileQueue.getLastMappedFile(0);\n            assertThat(mappedFile).isNotNull();\n            assertThat(mappedFile.appendMessage(fixedMsg.getBytes())).isTrue();\n        }\n\n        assertThat(mappedFileQueue.flush(0)).isFalse();\n        assertThat(mappedFileQueue.getFlushedWhere()).isEqualTo(1024);\n\n        assertThat(mappedFileQueue.flush(0)).isFalse();\n        assertThat(mappedFileQueue.getFlushedWhere()).isEqualTo(1024 * 2);\n\n        assertThat(mappedFileQueue.flush(0)).isFalse();\n        assertThat(mappedFileQueue.getFlushedWhere()).isEqualTo(1024 * 3);\n\n        assertThat(mappedFileQueue.flush(0)).isFalse();\n        assertThat(mappedFileQueue.getFlushedWhere()).isEqualTo(1024 * 4);\n\n        assertThat(mappedFileQueue.flush(0)).isFalse();\n        assertThat(mappedFileQueue.getFlushedWhere()).isEqualTo(1024 * 5);\n\n        assertThat(mappedFileQueue.flush(0)).isFalse();\n        assertThat(mappedFileQueue.getFlushedWhere()).isEqualTo(1024 * 6);\n\n        mappedFileQueue.shutdown(1000);\n        mappedFileQueue.destroy();\n    }\n\n    @Test\n    public void testGetMappedMemorySize() {\n        final String fixedMsg = \"abcd\";\n\n        MappedFileQueue mappedFileQueue =\n            new MappedFileQueue(storePath + File.separator + \"d/\", 1024, null);\n\n        for (int i = 0; i < 1024; i++) {\n            MappedFile mappedFile = mappedFileQueue.getLastMappedFile(0);\n            assertThat(mappedFile).isNotNull();\n            assertThat(mappedFile.appendMessage(fixedMsg.getBytes())).isTrue();\n        }\n\n        assertThat(mappedFileQueue.getMappedMemorySize()).isEqualTo(fixedMsg.length() * 1024);\n        mappedFileQueue.shutdown(1000);\n        mappedFileQueue.destroy();\n    }\n\n    @Test\n    public void testDeleteExpiredFileByOffset() {\n        MappedFileQueue mappedFileQueue =\n            new MappedFileQueue(storePath + File.separator + \"e/\", 5120, null);\n\n        for (int i = 0; i < 2048; i++) {\n            MappedFile mappedFile = mappedFileQueue.getLastMappedFile(0);\n            assertThat(mappedFile).isNotNull();\n            ByteBuffer byteBuffer = ByteBuffer.allocate(ConsumeQueue.CQ_STORE_UNIT_SIZE);\n            byteBuffer.putLong(i);\n            byte[] padding = new byte[12];\n            Arrays.fill(padding, (byte) '0');\n            byteBuffer.put(padding);\n            byteBuffer.flip();\n\n            assertThat(mappedFile.appendMessage(byteBuffer.array())).isTrue();\n        }\n\n        MappedFile first = mappedFileQueue.getFirstMappedFile();\n        first.hold();\n\n        assertThat(mappedFileQueue.deleteExpiredFileByOffset(20480, ConsumeQueue.CQ_STORE_UNIT_SIZE)).isEqualTo(0);\n        first.release();\n\n        assertThat(mappedFileQueue.deleteExpiredFileByOffset(20480, ConsumeQueue.CQ_STORE_UNIT_SIZE)).isGreaterThan(0);\n        first = mappedFileQueue.getFirstMappedFile();\n        assertThat(first.getFileFromOffset()).isGreaterThan(0);\n\n        mappedFileQueue.shutdown(1000);\n        mappedFileQueue.destroy();\n    }\n\n    @Test\n    public void testDeleteExpiredFileByTime() throws Exception {\n        MappedFileQueue mappedFileQueue =\n            new MappedFileQueue(storePath + File.separator + \"f/\", 1024, null);\n\n        for (int i = 0; i < 100; i++) {\n            MappedFile mappedFile = mappedFileQueue.getLastMappedFile(0);\n            assertThat(mappedFile).isNotNull();\n            byte[] bytes = new byte[512];\n            assertThat(mappedFile.appendMessage(bytes)).isTrue();\n        }\n\n        assertThat(mappedFileQueue.getMappedFiles().size()).isEqualTo(50);\n        long expiredTime = 100 * 1000;\n        for (int i = 0; i < mappedFileQueue.getMappedFiles().size(); i++) {\n            DefaultMappedFile mappedFile = (DefaultMappedFile) mappedFileQueue.getMappedFiles().get(i);\n            if (i < 5) {\n                mappedFile.getFile().setLastModified(System.currentTimeMillis() - expiredTime * 2);\n            }\n            if (i > 20) {\n                mappedFile.getFile().setLastModified(System.currentTimeMillis() - expiredTime * 2);\n            }\n        }\n        int maxBatchDeleteFilesNum = 50;\n        mappedFileQueue.deleteExpiredFileByTime(expiredTime, 0, 0, false, maxBatchDeleteFilesNum);\n        assertThat(mappedFileQueue.getMappedFiles().size()).isEqualTo(45);\n    }\n\n    @Test\n    public void testFindMappedFile_ByIteration() {\n        MappedFileQueue mappedFileQueue =\n            new MappedFileQueue(storePath + File.separator + \"g/\", 1024, null);\n        for (int i = 0; i < 3; i++) {\n            MappedFile mappedFile = mappedFileQueue.getLastMappedFile(1024 * i);\n            mappedFile.setWrotePosition(1024);\n        }\n\n        assertThat(mappedFileQueue.findMappedFileByOffset(1028).getFileFromOffset()).isEqualTo(1024);\n\n        // Switch two MappedFiles and verify findMappedFileByOffset method\n        MappedFile tmpFile = mappedFileQueue.getMappedFiles().get(1);\n        mappedFileQueue.getMappedFiles().set(1, mappedFileQueue.getMappedFiles().get(2));\n        mappedFileQueue.getMappedFiles().set(2, tmpFile);\n        assertThat(mappedFileQueue.findMappedFileByOffset(1028).getFileFromOffset()).isEqualTo(1024);\n    }\n\n    @Test\n    public void testMappedFile_SwapMap() {\n        // four-byte string.\n        final String fixedMsg = \"abcdefgh\";\n        final int mappedFileSize = 102400;\n\n        MappedFileQueue mappedFileQueue =\n            new MappedFileQueue(storePath + File.separator + \"b/\", mappedFileSize, null);\n\n        ThreadPoolExecutor executor = new ThreadPoolExecutor(3, 3, 1000 * 60,\n            TimeUnit.MILLISECONDS,\n            new LinkedBlockingQueue<>(),\n            new ThreadFactoryImpl(\"testThreadPool\"));\n\n        for (int i = 0; i < mappedFileSize; i++) {\n            MappedFile mappedFile = mappedFileQueue.getLastMappedFile(0);\n            assertThat(mappedFile).isNotNull();\n            assertThat(mappedFile.appendMessage(fixedMsg.getBytes())).isTrue();\n        }\n        assertThat(mappedFileQueue.getMappedMemorySize()).isEqualTo(fixedMsg.getBytes().length * mappedFileSize);\n\n        AtomicBoolean readOver = new AtomicBoolean(false);\n        AtomicBoolean hasException = new AtomicBoolean(false);\n\n        executor.submit(() -> {\n                try {\n                    while (!readOver.get()) {\n                        for (MappedFile mappedFile : mappedFileQueue.getMappedFiles()) {\n                            mappedFile.swapMap();\n                            Thread.sleep(10);\n                            mappedFile.cleanSwapedMap(true);\n                        }\n                    }\n                } catch (Throwable t) {\n                    hasException.set(true);\n                }\n            }\n        );\n        long start = System.currentTimeMillis();\n        long maxReadTimeMs = 60 * 1000;\n        try {\n            while (System.currentTimeMillis() - start <= maxReadTimeMs) {\n                for (int i = 0; i < mappedFileSize && !readOver.get(); i++) {\n                    MappedFile mappedFile = null;\n                    int retryTime = 0;\n                    while (mappedFile == null && retryTime < 10000) {\n                        mappedFile = mappedFileQueue.findMappedFileByOffset(i * fixedMsg.getBytes().length);\n                        retryTime++;\n                        if (mappedFile == null) {\n                            Thread.sleep(1);\n                        }\n                    }\n                    assertThat(mappedFile != null).isTrue();\n                    retryTime = 0;\n                    int pos = (i * fixedMsg.getBytes().length) % mappedFileSize;\n                    while ((pos + fixedMsg.getBytes().length) > mappedFile.getReadPosition() && retryTime < 10000) {\n                        retryTime++;\n                        if ((pos + fixedMsg.getBytes().length) > mappedFile.getReadPosition()) {\n                            Thread.sleep(1);\n                        }\n                    }\n                    assertThat((pos + fixedMsg.getBytes().length) <= mappedFile.getReadPosition()).isTrue();\n                    SelectMappedBufferResult ret = mappedFile.selectMappedBuffer(pos, fixedMsg.getBytes().length);\n                    byte[] readRes = new byte[fixedMsg.getBytes().length];\n                    ret.getByteBuffer().get(readRes);\n                    String readStr = new String(readRes, StandardCharsets.UTF_8);\n                    assertThat(readStr.equals(fixedMsg)).isTrue();\n                }\n            }\n            readOver.set(true);\n        } catch (Throwable e) {\n            hasException.set(true);\n            readOver.set(true);\n        }\n        assertThat(readOver.get()).isTrue();\n        assertThat(hasException.get()).isFalse();\n\n    }\n\n    @Test\n    public void testMappedFile_CleanSwapedMap() throws InterruptedException {\n        // four-byte string.\n        final String fixedMsg = \"abcd\";\n        final int mappedFileSize = 1024000;\n\n        MappedFileQueue mappedFileQueue =\n            new MappedFileQueue(storePath + File.separator + \"b/\", mappedFileSize, null);\n\n        ThreadPoolExecutor executor = new ThreadPoolExecutor(5, 5, 1000 * 60,\n            TimeUnit.MILLISECONDS,\n            new LinkedBlockingQueue<>(),\n            new ThreadFactoryImpl(\"testThreadPool\"));\n        for (int i = 0; i < mappedFileSize; i++) {\n            MappedFile mappedFile = mappedFileQueue.getLastMappedFile(0);\n            assertThat(mappedFile).isNotNull();\n            assertThat(mappedFile.appendMessage(fixedMsg.getBytes())).isTrue();\n        }\n\n        for (MappedFile mappedFile : mappedFileQueue.getMappedFiles()) {\n            mappedFile.swapMap();\n        }\n        AtomicBoolean hasException = new AtomicBoolean(false);\n        CountDownLatch downLatch = new CountDownLatch(5);\n        for (int i = 0; i < 5; i++) {\n            executor.submit(() -> {\n                try {\n                    for (MappedFile mappedFile : mappedFileQueue.getMappedFiles()) {\n                        mappedFile.cleanSwapedMap(true);\n                        mappedFile.cleanSwapedMap(true);\n                    }\n                } catch (Exception e) {\n                    hasException.set(true);\n                } finally {\n                    downLatch.countDown();\n                }\n            });\n        }\n\n        downLatch.await(10, TimeUnit.SECONDS);\n        assertThat(hasException.get()).isFalse();\n    }\n\n    @Test\n    public void testMappedFile_Rename() throws IOException, InterruptedException {\n        Assume.assumeFalse(MixAll.isWindows());\n        final String fixedMsg = RandomStringUtils.randomAlphanumeric(128);\n        final byte[] msgByteArr = fixedMsg.getBytes(StandardCharsets.UTF_8);\n        final int mappedFileSize = 5 * 1024 * 1024;\n\n        MappedFileQueue mappedFileQueue =\n            new MappedFileQueue(\"target/unit_test_store\", mappedFileSize, null);\n\n        int currentSize = 0;\n        while (currentSize <= 2 * mappedFileSize) {\n            MappedFile mappedFile = mappedFileQueue.getLastMappedFile(0);\n            mappedFile.appendMessage(msgByteArr);\n            currentSize += fixedMsg.length();\n        }\n\n        assertThat(mappedFileQueue.getMappedFiles().size()).isEqualTo(3);\n\n        ScheduledExecutorService ses = Executors.newSingleThreadScheduledExecutor();\n        ses.scheduleWithFixedDelay(() -> {\n            MappedFile mappedFile = mappedFileQueue.getLastMappedFile(0);\n            mappedFile.appendMessage(msgByteArr);\n        }, 1,1, TimeUnit.MILLISECONDS);\n\n        List<MappedFile> mappedFileList = Lists.newArrayList(mappedFileQueue.getMappedFiles());\n        mappedFileList.remove(mappedFileList.size() - 1);\n\n        MappedFileQueue compactingMappedFileQueue =\n            new MappedFileQueue(\"target/unit_test_store/compacting\", mappedFileSize, null);\n\n        currentSize = 0;\n        while (currentSize < (2 * mappedFileSize - mappedFileSize / 2)) {\n            MappedFile mappedFile = compactingMappedFileQueue.getLastMappedFile(0);\n            mappedFile.appendMessage(msgByteArr);\n            currentSize += fixedMsg.length();\n        }\n\n\n        mappedFileList.forEach(MappedFile::renameToDelete);\n        assertThat(mappedFileQueue.getFirstMappedFile().getFileName()).endsWith(\".delete\");\n        assertThat(mappedFileQueue.findMappedFileByOffset(mappedFileSize + fixedMsg.length()).getFileName()).endsWith(\".delete\");\n\n        SelectMappedBufferResult sbr = mappedFileList.get(mappedFileList.size() - 1).selectMappedBuffer(0, msgByteArr.length);\n        assertThat(sbr).isNotNull();\n        try {\n            assertThat(sbr.getMappedFile().getFileName().endsWith(\".delete\")).isTrue();\n            if (sbr.getByteBuffer().hasArray()) {\n                assertThat(sbr.getByteBuffer().array()).isEqualTo(msgByteArr);\n            } else {\n                for (int i = 0; i < msgByteArr.length; i++) {\n                    assertThat(sbr.getByteBuffer().get(i)).isEqualTo(msgByteArr[i]);\n                }\n            }\n        } finally {\n            sbr.release();\n        }\n\n\n        compactingMappedFileQueue.getMappedFiles().forEach(mappedFile -> {\n            try {\n                mappedFile.moveToParent();\n            } catch (IOException e) {\n                e.printStackTrace();\n            }\n        });\n\n        mappedFileQueue.getMappedFiles().stream()\n            .filter(m -> !mappedFileList.contains(m))\n            .forEach(m -> compactingMappedFileQueue.getMappedFiles().add(m));\n\n        int wrotePosition = mappedFileQueue.getLastMappedFile().getWrotePosition();\n\n        mappedFileList.forEach(mappedFile -> {\n            mappedFile.destroy(1000);\n        });\n\n        TimeUnit.SECONDS.sleep(3);\n        ses.shutdown();\n\n        mappedFileQueue.getMappedFiles().clear();\n        mappedFileQueue.getMappedFiles().addAll(compactingMappedFileQueue.getMappedFiles());\n\n        TimeUnit.SECONDS.sleep(3);\n    }\n\n    @Test\n    public void testReset() {\n        final String fixedMsg = \"0123456789abcdef\";\n        MappedFileQueue mappedFileQueue =\n                new MappedFileQueue(storePath + File.separator + \"a/\", 64, null);\n        for (int i = 0; i < 8; i++) {\n            MappedFile mappedFile = mappedFileQueue.getLastMappedFile(0);\n            assertThat(mappedFile).isNotNull();\n            assertThat(mappedFile.appendMessage(fixedMsg.getBytes())).isTrue();\n        }\n        assertThat(mappedFileQueue.getMappedFiles().size()).isEqualTo(2);\n        assertThat(mappedFileQueue.resetOffset(0)).isTrue();\n        assertThat(mappedFileQueue.getMappedFiles().size()).isEqualTo(1);\n    }\n\n    @After\n    public void destroy() {\n        File file = new File(storePath);\n        UtilAll.deleteFile(file);\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/MappedFileTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: MappedFileTest.java 1831 2013-05-16 01:39:51Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.store;\n\nimport java.io.File;\nimport java.io.IOException;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.store.logfile.DefaultMappedFile;\nimport org.junit.After;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class MappedFileTest {\n    private final String storeMessage = \"Once, there was a chance for me!\";\n\n    @Test\n    public void testSelectMappedBuffer() throws IOException {\n        DefaultMappedFile mappedFile = new DefaultMappedFile(\"target/unit_test_store/MappedFileTest/000\", 1024 * 64);\n        boolean result = mappedFile.appendMessage(storeMessage.getBytes());\n        assertThat(result).isTrue();\n\n        SelectMappedBufferResult selectMappedBufferResult = mappedFile.selectMappedBuffer(0);\n        byte[] data = new byte[storeMessage.length()];\n        selectMappedBufferResult.getByteBuffer().get(data);\n        String readString = new String(data);\n\n        assertThat(readString).isEqualTo(storeMessage);\n\n        mappedFile.shutdown(1000);\n        assertThat(mappedFile.isAvailable()).isFalse();\n        selectMappedBufferResult.release();\n        assertThat(mappedFile.isCleanupOver()).isTrue();\n        assertThat(mappedFile.destroy(1000)).isTrue();\n    }\n\n    @After\n    public void destroy() {\n        File file = new File(\"target/unit_test_store\");\n        UtilAll.deleteFile(file);\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/MessageExtBrokerInnerTest.java",
    "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class MessageExtBrokerInnerTest {\n    @Test\n    public void testDeleteProperty() {\n        MessageExtBrokerInner messageExtBrokerInner = new MessageExtBrokerInner();\n        String propertiesString = \"\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"\");\n\n        propertiesString = \"KeyA\\u0001ValueA\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"\");\n\n        propertiesString = \"KeyA\\u0001ValueA\\u0002\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"\");\n\n        propertiesString = \"KeyA\\u0001ValueA\\u0002KeyA\\u0001ValueA\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"\");\n\n        propertiesString = \"KeyA\\u0001ValueA\\u0002KeyA\\u0001ValueA\\u0002\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"\");\n\n        propertiesString = \"KeyB\\u0001ValueB\\u0002KeyA\\u0001ValueA\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"KeyB\\u0001ValueB\\u0002\");\n\n        propertiesString = \"KeyB\\u0001ValueB\\u0002KeyA\\u0001ValueA\\u0002\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"KeyB\\u0001ValueB\\u0002\");\n\n        propertiesString = \"KeyB\\u0001ValueB\\u0002KeyA\\u0001ValueA\\u0002KeyB\\u0001ValueB\\u0002\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"KeyB\\u0001ValueB\\u0002KeyB\\u0001ValueB\\u0002\");\n\n        propertiesString = \"KeyA\\u0001ValueA\\u0002KeyB\\u0001ValueB\\u0002\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"KeyB\\u0001ValueB\\u0002\");\n\n        propertiesString = \"KeyA\\u0001ValueA\\u0002KeyB\\u0001ValueB\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"KeyB\\u0001ValueB\");\n\n        propertiesString = \"KeyA\\u0001ValueA\\u0002KeyB\\u0001ValueBKeyA\\u0001ValueA\\u0002\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"KeyB\\u0001ValueBKeyA\\u0001ValueA\\u0002\");\n\n        propertiesString = \"KeyA\\u0001ValueA\\u0002KeyB\\u0001ValueBKeyA\\u0001\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"KeyB\\u0001ValueBKeyA\\u0001\");\n\n        propertiesString = \"KeyA\\u0001ValueA\\u0002KeyB\\u0001ValueBKeyA\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"KeyA\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(\"KeyB\\u0001ValueBKeyA\");\n\n        propertiesString = \"__CRC32#\\u0001\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"__CRC32#\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEmpty();\n\n        propertiesString = \"__CRC32#\";\n        messageExtBrokerInner.setPropertiesString(propertiesString);\n        messageExtBrokerInner.deleteProperty(\"__CRC32#\");\n        assertThat(messageExtBrokerInner.getPropertiesString()).isEqualTo(propertiesString);\n    }\n\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/MessageStoreStateMachineTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\nimport static org.junit.jupiter.api.Assertions.assertThrows;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.ArgumentMatchers.eq;\nimport static org.mockito.Mockito.anyLong;\nimport static org.mockito.Mockito.verify;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.store.MessageStoreStateMachine.MessageStoreState;\nimport org.junit.Test;\nimport org.junit.Before;\nimport org.mockito.Mockito;\n\npublic class MessageStoreStateMachineTest {\n\n    private Logger mockLogger;\n    private MessageStoreStateMachine stateMachine;\n\n    @Before\n    public void setUp() {\n        // Mock Logger\n        mockLogger = Mockito.mock(Logger.class);\n\n        // Initialize StateMachine\n        stateMachine = new MessageStoreStateMachine(mockLogger);\n    }\n\n    /**\n     * Test the constructor of MessageStoreStateMachine.\n     */\n    @Test\n    public void testConstructor() {\n        // Verify initial state\n        assertEquals(MessageStoreState.INIT, stateMachine.getCurrentState());\n\n        // Verify logger was called for initialization\n        verify(mockLogger).info(anyString(), eq(MessageStoreState.INIT));\n    }\n\n    /**\n     * Test valid state transition in transitTo method.\n     */\n    @Test\n    public void testValidStateTransition() {\n        // Perform a valid state transition\n        stateMachine.transitTo(MessageStoreState.LOAD_COMMITLOG_OK);\n\n        // Verify the current state is updated\n        assertEquals(MessageStoreState.LOAD_COMMITLOG_OK, stateMachine.getCurrentState());\n\n        // Verify logger was called for state transition\n        verify(mockLogger).info(anyString(), eq(MessageStoreState.INIT), eq(MessageStoreState.LOAD_COMMITLOG_OK),\n            anyLong(), anyLong());\n    }\n\n    /**\n     * Test fail state transition in transitTo method.\n     */\n    @Test\n    public void testValidFailStateTransition() {\n        stateMachine.transitTo(MessageStoreState.LOAD_COMMITLOG_OK, false);\n        assertEquals(MessageStoreState.INIT, stateMachine.getCurrentState());\n        verify(mockLogger).warn(anyString(), eq(MessageStoreState.INIT), eq(MessageStoreState.LOAD_COMMITLOG_OK),\n            anyLong(), anyLong());\n    }\n\n    /**\n     * Test invalid state transition in transitTo method.\n     */\n    @Test\n    public void testInvalidStateTransition() {\n        // Perform an invalid state transition\n        Exception exception = assertThrows(IllegalStateException.class, () -> {\n            stateMachine.transitTo(MessageStoreState.INIT);\n        });\n\n        // Verify the exception message\n        String expectedMessage = \"Invalid state transition from INIT to INIT. Can only move forward.\";\n        assertEquals(expectedMessage, exception.getMessage());\n    }\n\n    /**\n     * Test getCurrentState method.\n     */\n    @Test\n    public void testGetCurrentState() {\n        // Verify the current state\n        assertEquals(MessageStoreState.INIT, stateMachine.getCurrentState());\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/MultiPathMappedFileQueueTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport java.util.HashSet;\nimport java.util.Set;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.logfile.MappedFile;\nimport org.junit.Test;\n\n\npublic class MultiPathMappedFileQueueTest {\n\n    @Test\n    public void testGetLastMappedFile() {\n        final byte[] fixedMsg = new byte[1024];\n\n        MessageStoreConfig config = new MessageStoreConfig();\n        config.setStorePathCommitLog(\"target/unit_test_store/a/\" + MixAll.MULTI_PATH_SPLITTER\n                + \"target/unit_test_store/b/\" + MixAll.MULTI_PATH_SPLITTER\n                + \"target/unit_test_store/c/\");\n        MappedFileQueue mappedFileQueue = new MultiPathMappedFileQueue(config, 1024, null, null);\n        String[] storePaths = config.getStorePathCommitLog().trim().split(MixAll.MULTI_PATH_SPLITTER);\n        for (int i = 0; i < 1024; i++) {\n            MappedFile mappedFile = mappedFileQueue.getLastMappedFile(fixedMsg.length * i);\n            assertThat(mappedFile).isNotNull();\n            assertThat(mappedFile.appendMessage(fixedMsg)).isTrue();\n            int idx = i % storePaths.length;\n            assertThat(mappedFile.getFileName().startsWith(storePaths[idx])).isTrue();\n        }\n        mappedFileQueue.shutdown(1000);\n        mappedFileQueue.destroy();\n    }\n\n    @Test\n    public void testLoadReadOnlyMappedFiles() {\n        {\n            //create old mapped files\n            final byte[] fixedMsg = new byte[1024];\n            MessageStoreConfig config = new MessageStoreConfig();\n            config.setStorePathCommitLog(\"target/unit_test_store/a/\" + MixAll.MULTI_PATH_SPLITTER\n                    + \"target/unit_test_store/b/\" + MixAll.MULTI_PATH_SPLITTER\n                    + \"target/unit_test_store/c/\");\n            MappedFileQueue mappedFileQueue = new MultiPathMappedFileQueue(config, 1024, null, null);\n            String[] storePaths = config.getStorePathCommitLog().trim().split(MixAll.MULTI_PATH_SPLITTER);\n            for (int i = 0; i < 1024; i++) {\n                MappedFile mappedFile = mappedFileQueue.getLastMappedFile(fixedMsg.length * i);\n                assertThat(mappedFile).isNotNull();\n                assertThat(mappedFile.appendMessage(fixedMsg)).isTrue();\n                int idx = i % storePaths.length;\n                assertThat(mappedFile.getFileName().startsWith(storePaths[idx])).isTrue();\n            }\n            mappedFileQueue.shutdown(1000);\n        }\n\n        // test load and readonly\n        MessageStoreConfig config = new MessageStoreConfig();\n        config.setStorePathCommitLog(\"target/unit_test_store/b/\");\n        config.setReadOnlyCommitLogStorePaths(\"target/unit_test_store/a\" + MixAll.MULTI_PATH_SPLITTER\n                + \"target/unit_test_store/c\");\n        MultiPathMappedFileQueue mappedFileQueue = new MultiPathMappedFileQueue(config, 1024, null, null);\n\n        mappedFileQueue.load();\n\n        assertThat(mappedFileQueue.mappedFiles.size()).isEqualTo(1024);\n        for (int i = 0; i < 1024; i++) {\n            assertThat(mappedFileQueue.mappedFiles.get(i).getFile().getName())\n                    .isEqualTo(UtilAll.offset2FileName(1024 * i));\n        }\n        mappedFileQueue.destroy();\n\n    }\n\n    @Test\n    public void testUpdatePathsOnline() {\n        final byte[] fixedMsg = new byte[1024];\n\n        MessageStoreConfig config = new MessageStoreConfig();\n        config.setStorePathCommitLog(\"target/unit_test_store/a/\" + MixAll.MULTI_PATH_SPLITTER\n                + \"target/unit_test_store/b/\" + MixAll.MULTI_PATH_SPLITTER\n                + \"target/unit_test_store/c/\");\n        MappedFileQueue mappedFileQueue = new MultiPathMappedFileQueue(config, 1024, null, null);\n        String[] storePaths = config.getStorePathCommitLog().trim().split(MixAll.MULTI_PATH_SPLITTER);\n        for (int i = 0; i < 1024; i++) {\n            MappedFile mappedFile = mappedFileQueue.getLastMappedFile(fixedMsg.length * i);\n            assertThat(mappedFile).isNotNull();\n            assertThat(mappedFile.appendMessage(fixedMsg)).isTrue();\n            int idx = i % storePaths.length;\n            assertThat(mappedFile.getFileName().startsWith(storePaths[idx])).isTrue();\n\n            if (i == 500) {\n                config.setStorePathCommitLog(\"target/unit_test_store/a/\" + MixAll.MULTI_PATH_SPLITTER\n                        + \"target/unit_test_store/b/\");\n                storePaths = config.getStorePathCommitLog().trim().split(MixAll.MULTI_PATH_SPLITTER);\n            }\n        }\n        mappedFileQueue.shutdown(1000);\n        mappedFileQueue.destroy();\n    }\n\n    @Test\n    public void testFullStorePath() {\n        final byte[] fixedMsg = new byte[1024];\n\n        Set<String> fullStorePath = new HashSet<>();\n        MessageStoreConfig config = new MessageStoreConfig();\n        config.setStorePathCommitLog(\"target/unit_test_store/a/\" + MixAll.MULTI_PATH_SPLITTER\n                + \"target/unit_test_store/b/\" + MixAll.MULTI_PATH_SPLITTER\n                + \"target/unit_test_store/c/\");\n        MappedFileQueue mappedFileQueue = new MultiPathMappedFileQueue(config, 1024, null, () -> fullStorePath);\n        String[] storePaths = config.getStorePathCommitLog().trim().split(MixAll.MULTI_PATH_SPLITTER);\n        assertThat(storePaths.length).isEqualTo(3);\n\n        MappedFile mappedFile = mappedFileQueue.getLastMappedFile(0);\n        assertThat(mappedFile).isNotNull();\n        assertThat(mappedFile.appendMessage(fixedMsg)).isTrue();\n        assertThat(mappedFile.getFileName().startsWith(storePaths[0])).isTrue();\n\n        mappedFile = mappedFileQueue.getLastMappedFile(fixedMsg.length);\n        assertThat(mappedFile.getFileName().startsWith(storePaths[1])).isTrue();\n        assertThat(mappedFile.appendMessage(fixedMsg)).isTrue();\n        mappedFile = mappedFileQueue.getLastMappedFile(fixedMsg.length * 2);\n        assertThat(mappedFile.appendMessage(fixedMsg)).isTrue();\n        assertThat(mappedFile.getFileName().startsWith(storePaths[2])).isTrue();\n\n        fullStorePath.add(\"target/unit_test_store/b/\");\n        mappedFile = mappedFileQueue.getLastMappedFile(fixedMsg.length * 3);\n        assertThat(mappedFile.appendMessage(fixedMsg)).isTrue();\n        assertThat(mappedFile.getFileName().startsWith(storePaths[2])).isTrue();\n\n        mappedFile = mappedFileQueue.getLastMappedFile(fixedMsg.length * 4);\n        assertThat(mappedFile.appendMessage(fixedMsg)).isTrue();\n        assertThat(mappedFile.getFileName().startsWith(storePaths[0])).isTrue();\n\n        mappedFileQueue.shutdown(1000);\n        mappedFileQueue.destroy();\n    }\n}"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/ReputMessageServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport java.lang.reflect.Field;\nimport java.net.InetAddress;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.util.UUID;\nimport java.io.File;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.MessageDecoder;\n\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.config.FlushDiskType;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertTrue;\nimport static org.junit.jupiter.api.Assertions.assertEquals;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\nimport static org.mockito.ArgumentMatchers.any;\n\npublic class ReputMessageServiceTest {\n    private DefaultMessageStore syncFlushMessageStore;\n    private DefaultMessageStore asyncFlushMessageStore;\n    private final String topic = \"FooBar\";\n    private final String tmpdir = System.getProperty(\"java.io.tmpdir\");\n    private final String storePathRootParentDir = (StringUtils.endsWith(tmpdir, File.separator) ? tmpdir : tmpdir + File.separator) + UUID.randomUUID();\n    private SocketAddress bornHost;\n    private SocketAddress storeHost;\n\n    @Before\n    public void init() throws Exception {\n        File file = new File(storePathRootParentDir);\n        UtilAll.deleteFile(file);\n        syncFlushMessageStore = buildMessageStore(FlushDiskType.SYNC_FLUSH);\n        asyncFlushMessageStore = buildMessageStore(FlushDiskType.ASYNC_FLUSH);\n        assertTrue(syncFlushMessageStore.load());\n        assertTrue(asyncFlushMessageStore.load());\n        syncFlushMessageStore.start();\n        asyncFlushMessageStore.start();\n        storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123);\n        bornHost = new InetSocketAddress(InetAddress.getByName(\"127.0.0.1\"), 0);\n    }\n\n    private DefaultMessageStore buildMessageStore(FlushDiskType flushDiskType) throws Exception {\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setHaListenPort(0);\n        messageStoreConfig.setFlushDiskType(flushDiskType);\n        messageStoreConfig.setStorePathRootDir(storePathRootParentDir + File.separator + flushDiskType);\n        messageStoreConfig.setStorePathCommitLog(storePathRootParentDir + File.separator + flushDiskType + File.separator + \"commitlog\");\n        BrokerConfig brokerConfig = new BrokerConfig();\n        brokerConfig.setLongPollingEnable(false);\n        DefaultMessageStore messageStore = new DefaultMessageStore(messageStoreConfig, mock(BrokerStatsManager.class), null, brokerConfig, null);\n        // Mock flush disk service\n        Field field = CommitLog.class.getDeclaredField(\"flushManager\");\n        field.setAccessible(true);\n        FlushManager flushManager = mock(FlushManager.class);\n        CompletableFuture<PutMessageStatus> completableFuture = new CompletableFuture<>();\n        completableFuture.complete(PutMessageStatus.PUT_OK);\n        when(flushManager.handleDiskFlush(any(AppendMessageResult.class), any(MessageExt.class))).thenReturn(completableFuture);\n        field.set(messageStore.getCommitLog(), flushManager);\n        return messageStore;\n    }\n\n    @Test\n    public void testReputEndOffset_whenSyncFlush() throws Exception {\n        for (int i = 0; i < 10; i++) {\n            assertEquals(PutMessageStatus.PUT_OK, syncFlushMessageStore.putMessage(buildMessage()).getPutMessageStatus());\n        }\n        assertEquals(1580, syncFlushMessageStore.getMaxPhyOffset());\n        assertEquals(0, syncFlushMessageStore.getCommitLog().getFlushedWhere());\n        // wait for cq dispatch\n        Thread.sleep(3000);\n        assertEquals(0, syncFlushMessageStore.getCommitLog().getFlushedWhere());\n        assertEquals(0, syncFlushMessageStore.getMaxOffsetInQueue(topic, 0));\n        GetMessageResult getMessageResult = syncFlushMessageStore.getMessage(\"testGroup\", topic, 0, 0, 32, null);\n        assertEquals(GetMessageStatus.NO_MESSAGE_IN_QUEUE, getMessageResult.getStatus());\n    }\n\n    @Test\n    public void testReputEndOffset_whenAsyncFlush() throws Exception {\n        for (int i = 0; i < 10; i++) {\n            assertEquals(PutMessageStatus.PUT_OK, asyncFlushMessageStore.putMessage(buildMessage()).getPutMessageStatus());\n        }\n        assertEquals(1580, asyncFlushMessageStore.getMaxPhyOffset());\n        assertEquals(0, asyncFlushMessageStore.getCommitLog().getFlushedWhere());\n        // wait for cq dispatch\n        Thread.sleep(3000);\n        assertEquals(0, asyncFlushMessageStore.getCommitLog().getFlushedWhere());\n        assertEquals(10, asyncFlushMessageStore.getMaxOffsetInQueue(topic, 0));\n        GetMessageResult getMessageResult = asyncFlushMessageStore.getMessage(\"testGroup\", topic, 0, 0, 32, null);\n        assertEquals(10, getMessageResult.getMessageCount());\n    }\n\n    private MessageExtBrokerInner buildMessage() {\n        MessageExtBrokerInner msg = new MessageExtBrokerInner();\n        msg.setTopic(topic);\n        msg.setTags(\"TAG1\");\n        msg.setBody(\"Once, there was a chance for me!\".getBytes());\n        msg.setKeys(String.valueOf(System.currentTimeMillis()));\n        msg.setQueueId(0);\n        msg.setSysFlag(0);\n        msg.setBornTimestamp(System.currentTimeMillis());\n        msg.setStoreHost(storeHost);\n        msg.setBornHost(bornHost);\n        msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n        return msg;\n    }\n\n    @After\n    public void destroy() throws Exception {\n        if (this.syncFlushMessageStore != null) {\n            syncFlushMessageStore.shutdown();\n            syncFlushMessageStore.destroy();\n        }\n        if (this.asyncFlushMessageStore != null) {\n            asyncFlushMessageStore.shutdown();\n            asyncFlushMessageStore.destroy();\n        }\n        File file = new File(storePathRootParentDir);\n        UtilAll.deleteFile(file);\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store;\n\nimport com.google.common.collect.Sets;\nimport java.io.File;\nimport java.io.RandomAccessFile;\nimport java.lang.reflect.InvocationTargetException;\nimport java.lang.reflect.Method;\nimport java.net.InetAddress;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.net.UnknownHostException;\nimport java.nio.MappedByteBuffer;\nimport java.nio.channels.FileChannel;\nimport java.nio.channels.OverlappingFileLockException;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Random;\nimport java.util.UUID;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.MessageBatch;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBatch;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.config.FlushDiskType;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.config.StorePathConfigHelper;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.queue.ConsumeQueueInterface;\nimport org.apache.rocketmq.store.queue.CqUnit;\nimport org.apache.rocketmq.store.queue.RocksDBConsumeQueue;\nimport org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.assertj.core.util.Strings;\nimport org.awaitility.Awaitility;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.assertj.core.api.Assertions.fail;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertSame;\nimport static org.junit.Assert.assertTrue;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class RocksDBMessageStoreTest {\n    private final String storeMessage = \"Once, there was a chance for me!\";\n    private final String messageTopic = \"FooBar\";\n    private final String storeType = StoreType.DEFAULT_ROCKSDB.getStoreType();\n    private int queueTotal = 100;\n    private final AtomicInteger queueId = new AtomicInteger(0);\n    private SocketAddress bornHost;\n    private SocketAddress storeHost;\n    private byte[] messageBody;\n    private MessageStore messageStore;\n\n    @Before\n    public void init() throws Exception {\n        if (notExecuted()) {\n            return;\n        }\n        storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123);\n        bornHost = new InetSocketAddress(InetAddress.getByName(\"127.0.0.1\"), 0);\n\n        messageStore = buildMessageStore();\n        boolean load = messageStore.load();\n        assertTrue(load);\n        messageStore.start();\n    }\n\n    @Test(expected = OverlappingFileLockException.class)\n    public void test_repeat_restart() throws Exception {\n        if (notExecuted()) {\n            throw new OverlappingFileLockException();\n        }\n        queueTotal = 1;\n        messageBody = storeMessage.getBytes();\n\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setMappedFileSizeCommitLog(1024 * 8);\n        messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 4);\n        messageStoreConfig.setMaxHashSlotNum(100);\n        messageStoreConfig.setMaxIndexNum(100 * 10);\n        messageStoreConfig.setStorePathRootDir(System.getProperty(\"java.io.tmpdir\") + File.separator + \"store\");\n        messageStoreConfig.setHaListenPort(0);\n        MessageStore master = new RocksDBMessageStore(messageStoreConfig, null, new MyMessageArrivingListener(), new BrokerConfig(), new ConcurrentHashMap<>());\n\n        boolean load = master.load();\n        assertTrue(load);\n\n        try {\n            master.start();\n            master.start();\n        } finally {\n            master.shutdown();\n            master.destroy();\n        }\n    }\n\n    @After\n    public void destroy() {\n        if (notExecuted()) {\n            return;\n        }\n        messageStore.shutdown();\n        messageStore.destroy();\n\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        File file = new File(messageStoreConfig.getStorePathRootDir());\n        UtilAll.deleteFile(file);\n    }\n\n    private MessageStore buildMessageStore() throws Exception {\n        return buildMessageStore(null, \"\");\n    }\n\n    private MessageStore buildMessageStore(String storePathRootDir, String topic) throws Exception {\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setMappedFileSizeCommitLog(1024 * 1024 * 10);\n        messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 1024 * 10);\n        messageStoreConfig.setMaxHashSlotNum(10000);\n        messageStoreConfig.setMaxIndexNum(100 * 100);\n        messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);\n        messageStoreConfig.setFlushIntervalConsumeQueue(1);\n        messageStoreConfig.setStoreType(storeType);\n        messageStoreConfig.setHaListenPort(0);\n        if (Strings.isNullOrEmpty(storePathRootDir)) {\n            UUID uuid = UUID.randomUUID();\n            storePathRootDir = System.getProperty(\"java.io.tmpdir\") + File.separator + \"store-\" + uuid.toString();\n        }\n        messageStoreConfig.setStorePathRootDir(storePathRootDir);\n        ConcurrentMap<String, TopicConfig> topicConfigTable = new ConcurrentHashMap<>();\n        topicConfigTable.put(topic, new TopicConfig(topic, 1, 1));\n        return new RocksDBMessageStore(messageStoreConfig,\n            new BrokerStatsManager(\"simpleTest\", true),\n            new MyMessageArrivingListener(),\n            new BrokerConfig(), topicConfigTable);\n    }\n\n    @Test\n    public void testWriteAndRead() {\n        if (notExecuted()) {\n            return;\n        }\n        long ipv4HostMessages = 10;\n        long ipv6HostMessages = 10;\n        long totalMessages = ipv4HostMessages + ipv6HostMessages;\n        queueTotal = 1;\n        messageBody = storeMessage.getBytes();\n        for (long i = 0; i < ipv4HostMessages; i++) {\n            messageStore.putMessage(buildMessage());\n        }\n\n        for (long i = 0; i < ipv6HostMessages; i++) {\n            messageStore.putMessage(buildIPv6HostMessage());\n        }\n\n        StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);\n\n        for (long i = 0; i < totalMessages; i++) {\n            GetMessageResult result = messageStore.getMessage(\"GROUP_A\", \"FooBar\", 0, i, 1024 * 1024, null);\n            assertThat(result).isNotNull();\n            result.release();\n        }\n        verifyThatMasterIsFunctional(totalMessages, messageStore);\n    }\n\n    @Test\n    public void testLookMessageByOffset_OffsetIsFirst() {\n        if (notExecuted()) {\n            return;\n        }\n        final int totalCount = 10;\n        int queueId = new Random().nextInt(10);\n        String topic = \"FooBar\";\n        int firstOffset = 0;\n        AppendMessageResult[] appendMessageResultArray = putMessages(totalCount, topic, queueId);\n        AppendMessageResult firstResult = appendMessageResultArray[0];\n\n        MessageExt messageExt = messageStore.lookMessageByOffset(firstResult.getWroteOffset());\n        MessageExt messageExt1 = getDefaultMessageStore().lookMessageByOffset(firstResult.getWroteOffset(), firstResult.getWroteBytes());\n\n        assertThat(new String(messageExt.getBody())).isEqualTo(buildMessageBodyByOffset(storeMessage, firstOffset));\n        assertThat(new String(messageExt1.getBody())).isEqualTo(buildMessageBodyByOffset(storeMessage, firstOffset));\n    }\n\n    @Test\n    public void testLookMessageByOffset_OffsetIsLast() {\n        if (notExecuted()) {\n            return;\n        }\n        final int totalCount = 10;\n        int queueId = new Random().nextInt(10);\n        String topic = \"FooBar\";\n        AppendMessageResult[] appendMessageResultArray = putMessages(totalCount, topic, queueId);\n        int lastIndex = totalCount - 1;\n        AppendMessageResult lastResult = appendMessageResultArray[lastIndex];\n\n        MessageExt messageExt = getDefaultMessageStore().lookMessageByOffset(lastResult.getWroteOffset(), lastResult.getWroteBytes());\n\n        assertThat(new String(messageExt.getBody())).isEqualTo(buildMessageBodyByOffset(storeMessage, lastIndex));\n    }\n\n    @Test\n    public void testLookMessageByOffset_OffsetIsOutOfBound() {\n        if (notExecuted()) {\n            return;\n        }\n        final int totalCount = 10;\n        int queueId = new Random().nextInt(10);\n        String topic = \"FooBar\";\n        AppendMessageResult[] appendMessageResultArray = putMessages(totalCount, topic, queueId);\n        long lastOffset = getMaxOffset(appendMessageResultArray);\n\n        MessageExt messageExt = getDefaultMessageStore().lookMessageByOffset(lastOffset);\n\n        assertThat(messageExt).isNull();\n    }\n\n    @Test\n    public void testGetOffsetInQueueByTime() {\n        if (notExecuted()) {\n            return;\n        }\n        final int totalCount = 10;\n        int queueId = 0;\n        String topic = \"FooBar\";\n        AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, true);\n        //Thread.sleep(10);\n        StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);\n\n        ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId);\n        for (AppendMessageResult appendMessageResult : appendMessageResults) {\n            long offset = messageStore.getOffsetInQueueByTime(topic, queueId, appendMessageResult.getStoreTimestamp());\n            CqUnit cqUnit = consumeQueue.get(offset);\n            assertThat(cqUnit.getPos()).isEqualTo(appendMessageResult.getWroteOffset());\n            assertThat(cqUnit.getSize()).isEqualTo(appendMessageResult.getWroteBytes());\n        }\n    }\n\n    @Test\n    public void testGetOffsetInQueueByTime_TimestampIsSkewing() {\n        if (notExecuted()) {\n            return;\n        }\n        final int totalCount = 10;\n        int queueId = 0;\n        String topic = \"FooBar\";\n        AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, true);\n        //Thread.sleep(10);\n        StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);\n        int skewing = 2;\n\n        ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId);\n        for (AppendMessageResult appendMessageResult : appendMessageResults) {\n            long offset = messageStore.getOffsetInQueueByTime(topic, queueId, appendMessageResult.getStoreTimestamp() - skewing);\n            CqUnit cqUnit = consumeQueue.get(offset);\n            assertThat(cqUnit.getPos()).isEqualTo(appendMessageResult.getWroteOffset());\n            assertThat(cqUnit.getSize()).isEqualTo(appendMessageResult.getWroteBytes());\n        }\n    }\n\n    @Test\n    public void testGetOffsetInQueueByTime_TimestampSkewingIsLarge() {\n        if (notExecuted()) {\n            return;\n        }\n        final int totalCount = 10;\n        int queueId = 0;\n        String topic = \"FooBar\";\n        AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, true);\n        //Thread.sleep(10);\n        StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);\n        int skewing = 20000;\n\n        ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId);\n        for (AppendMessageResult appendMessageResult : appendMessageResults) {\n            long offset = messageStore.getOffsetInQueueByTime(topic, queueId, appendMessageResult.getStoreTimestamp() - skewing);\n            CqUnit cqUnit = consumeQueue.get(offset);\n            assertThat(cqUnit.getPos()).isEqualTo(appendMessageResults[0].getWroteOffset());\n            assertThat(cqUnit.getSize()).isEqualTo(appendMessageResults[0].getWroteBytes());\n        }\n    }\n\n    @Test\n    public void testGetOffsetInQueueByTime_ConsumeQueueNotFound1() {\n        if (notExecuted()) {\n            return;\n        }\n        final int totalCount = 10;\n        int queueId = 0;\n        int wrongQueueId = 1;\n        String topic = \"FooBar\";\n        AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, false);\n        //Thread.sleep(10);\n\n        StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);\n\n        long offset = messageStore.getOffsetInQueueByTime(topic, wrongQueueId, appendMessageResults[0].getStoreTimestamp());\n\n        assertThat(offset).isEqualTo(0);\n    }\n\n    @Test\n    public void testGetOffsetInQueueByTime_ConsumeQueueNotFound2() {\n        if (notExecuted()) {\n            return;\n        }\n        final int totalCount = 10;\n        int queueId = 0;\n        int wrongQueueId = 1;\n        String topic = \"FooBar\";\n        putMessages(totalCount, topic, queueId, false);\n        //Thread.sleep(10);\n        StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);\n\n        long messageStoreTimeStamp = messageStore.getMessageStoreTimeStamp(topic, wrongQueueId, 0);\n\n        assertThat(messageStoreTimeStamp).isEqualTo(-1);\n    }\n\n    @Test\n    public void testGetOffsetInQueueByTime_ConsumeQueueOffsetNotExist() {\n        if (notExecuted()) {\n            return;\n        }\n        final int totalCount = 10;\n        int queueId = 0;\n        int wrongQueueId = 1;\n        String topic = \"FooBar\";\n        putMessages(totalCount, topic, queueId, true);\n        //Thread.sleep(10);\n\n        StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);\n\n        long messageStoreTimeStamp = messageStore.getMessageStoreTimeStamp(topic, wrongQueueId, -1);\n\n        assertThat(messageStoreTimeStamp).isEqualTo(-1);\n    }\n\n    @Test\n    public void testGetMessageStoreTimeStamp() {\n        if (notExecuted()) {\n            return;\n        }\n        final int totalCount = 10;\n        int queueId = 0;\n        String topic = \"FooBar\";\n        AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, false);\n        //Thread.sleep(10);\n        StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);\n\n        ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId);\n        int minOffsetInQueue = (int) consumeQueue.getMinOffsetInQueue();\n        for (int i = minOffsetInQueue; i < consumeQueue.getMaxOffsetInQueue(); i++) {\n            long messageStoreTimeStamp = messageStore.getMessageStoreTimeStamp(topic, queueId, i);\n            assertThat(messageStoreTimeStamp).isEqualTo(appendMessageResults[i].getStoreTimestamp());\n        }\n    }\n\n    @Test\n    public void testGetStoreTime_ParamIsNull() {\n        if (notExecuted()) {\n            return;\n        }\n        long storeTime = getStoreTime(null);\n\n        assertThat(storeTime).isEqualTo(-1);\n    }\n\n    @Test\n    public void testGetStoreTime_EverythingIsOk() {\n        if (notExecuted()) {\n            return;\n        }\n        final int totalCount = 10;\n        int queueId = 0;\n        String topic = \"FooBar\";\n        AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, false);\n        //Thread.sleep(10);\n        StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);\n        ConsumeQueueInterface consumeQueue = messageStore.getConsumeQueue(topic, queueId);\n\n        for (int i = 0; i < totalCount; i++) {\n            CqUnit cqUnit = consumeQueue.get(i);\n            long storeTime = getStoreTime(cqUnit);\n            assertThat(storeTime).isEqualTo(appendMessageResults[i].getStoreTimestamp());\n        }\n    }\n\n    @Test\n    public void testGetStoreTime_PhyOffsetIsLessThanCommitLogMinOffset() {\n        if (notExecuted()) {\n            return;\n        }\n        long phyOffset = -10;\n        int size = 138;\n        CqUnit cqUnit = new CqUnit(0, phyOffset, size, 0);\n        long storeTime = getStoreTime(cqUnit);\n\n        assertThat(storeTime).isEqualTo(-1);\n    }\n\n    @Test\n    public void testPutMessage_whenMessagePropertyIsTooLong() throws ConsumeQueueException {\n        if (notExecuted()) {\n            return;\n        }\n        String topicName = \"messagePropertyIsTooLongTest\";\n        MessageExtBrokerInner illegalMessage = buildSpecifyLengthPropertyMessage(\"123\".getBytes(StandardCharsets.UTF_8), topicName, Short.MAX_VALUE + 1);\n        assertEquals(messageStore.putMessage(illegalMessage).getPutMessageStatus(), PutMessageStatus.PROPERTIES_SIZE_EXCEEDED);\n        assertEquals(0L, messageStore.getQueueStore().getMaxOffset(topicName, 0).longValue());\n        MessageExtBrokerInner normalMessage = buildSpecifyLengthPropertyMessage(\"123\".getBytes(StandardCharsets.UTF_8), topicName, 100);\n        assertEquals(messageStore.putMessage(normalMessage).getPutMessageStatus(), PutMessageStatus.PUT_OK);\n        assertEquals(1L, messageStore.getQueueStore().getMaxOffset(topicName, 0).longValue());\n    }\n\n    private RocksDBMessageStore getDefaultMessageStore() {\n        return (RocksDBMessageStore) this.messageStore;\n    }\n\n    private AppendMessageResult[] putMessages(int totalCount, String topic, int queueId) {\n        return putMessages(totalCount, topic, queueId, false);\n    }\n\n    private AppendMessageResult[] putMessages(int totalCount, String topic, int queueId, boolean interval) {\n        AppendMessageResult[] appendMessageResultArray = new AppendMessageResult[totalCount];\n        for (int i = 0; i < totalCount; i++) {\n            String messageBody = buildMessageBodyByOffset(storeMessage, i);\n\n            MessageExtBrokerInner msgInner =\n                i < totalCount / 2 ? buildMessage(messageBody.getBytes(), topic) : buildIPv6HostMessage(messageBody.getBytes(), topic);\n            msgInner.setQueueId(queueId);\n            PutMessageResult result = messageStore.putMessage(msgInner);\n            appendMessageResultArray[i] = result.getAppendMessageResult();\n            assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK);\n            if (interval) {\n                try {\n                    Thread.sleep(10);\n                } catch (InterruptedException e) {\n                    throw new RuntimeException(\"Thread sleep ERROR\");\n                }\n            }\n        }\n        return appendMessageResultArray;\n    }\n\n    private long getMaxOffset(AppendMessageResult[] appendMessageResultArray) {\n        if (appendMessageResultArray == null) {\n            return 0;\n        }\n        AppendMessageResult last = appendMessageResultArray[appendMessageResultArray.length - 1];\n        return last.getWroteOffset() + last.getWroteBytes();\n    }\n\n    private String buildMessageBodyByOffset(String message, long i) {\n        return String.format(\"%s offset %d\", message, i);\n    }\n\n    private long getStoreTime(CqUnit cqUnit) {\n        try {\n            Class abstractConsumeQueueStore = getDefaultMessageStore().getQueueStore().getClass().getSuperclass();\n            Method getStoreTime = abstractConsumeQueueStore.getDeclaredMethod(\"getStoreTime\", CqUnit.class);\n            getStoreTime.setAccessible(true);\n            return (long) getStoreTime.invoke(getDefaultMessageStore().getQueueStore(), cqUnit);\n        } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    private MessageExtBrokerInner buildMessage(byte[] messageBody, String topic) {\n        MessageExtBrokerInner msg = new MessageExtBrokerInner();\n        msg.setTopic(topic);\n        msg.setTags(\"TAG1\");\n        msg.setKeys(\"Hello\");\n        msg.setBody(messageBody);\n        msg.setKeys(String.valueOf(System.currentTimeMillis()));\n        msg.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal);\n        msg.setSysFlag(0);\n        msg.setBornTimestamp(System.currentTimeMillis());\n        msg.setStoreHost(storeHost);\n        msg.setBornHost(bornHost);\n        msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n        return msg;\n    }\n\n    private MessageExtBrokerInner buildSpecifyLengthPropertyMessage(byte[] messageBody, String topic, int length) {\n        StringBuilder stringBuilder = new StringBuilder();\n        Random random = new Random();\n        for (int i = 0; i < length; i++) {\n            stringBuilder.append(random.nextInt(10));\n        }\n        MessageExtBrokerInner msg = new MessageExtBrokerInner();\n        msg.putUserProperty(\"test\", stringBuilder.toString());\n        msg.setTopic(topic);\n        msg.setTags(\"TAG1\");\n        msg.setKeys(\"Hello\");\n        msg.setBody(messageBody);\n        msg.setQueueId(0);\n        msg.setBornTimestamp(System.currentTimeMillis());\n        msg.setStoreHost(storeHost);\n        msg.setBornHost(bornHost);\n        msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n        return msg;\n    }\n\n    private MessageExtBrokerInner buildIPv6HostMessage(byte[] messageBody, String topic) {\n        MessageExtBrokerInner msg = new MessageExtBrokerInner();\n        msg.setTopic(topic);\n        msg.setTags(\"TAG1\");\n        msg.setKeys(\"Hello\");\n        msg.setBody(messageBody);\n        msg.setMsgId(\"24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0\");\n        msg.setKeys(String.valueOf(System.currentTimeMillis()));\n        msg.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal);\n        msg.setSysFlag(0);\n        msg.setBornHostV6Flag();\n        msg.setStoreHostAddressV6Flag();\n        msg.setBornTimestamp(System.currentTimeMillis());\n        try {\n            msg.setBornHost(new InetSocketAddress(InetAddress.getByName(\"1050:0000:0000:0000:0005:0600:300c:326b\"), 0));\n        } catch (UnknownHostException e) {\n            fail(\"build IPv6 host message error\", e);\n        }\n\n        try {\n            msg.setStoreHost(new InetSocketAddress(InetAddress.getByName(\"::1\"), 0));\n        } catch (UnknownHostException e) {\n            fail(\"build IPv6 host message error\", e);\n        }\n        return msg;\n    }\n\n    private MessageExtBrokerInner buildMessage() {\n        return buildMessage(messageBody, messageTopic);\n    }\n\n    public MessageExtBatch buildMessageBatch(MessageBatch msgBatch) {\n        MessageExtBatch msgExtBatch = new MessageExtBatch();\n        msgExtBatch.setTopic(messageTopic);\n        msgExtBatch.setTags(\"TAG1\");\n        msgExtBatch.setKeys(\"Hello\");\n        msgExtBatch.setBody(msgBatch.getBody());\n        msgExtBatch.setKeys(String.valueOf(System.currentTimeMillis()));\n        msgExtBatch.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal);\n        msgExtBatch.setSysFlag(0);\n        msgExtBatch.setBornTimestamp(System.currentTimeMillis());\n        msgExtBatch.setStoreHost(storeHost);\n        msgExtBatch.setBornHost(bornHost);\n        return msgExtBatch;\n    }\n\n    @Test\n    public void testGroupCommit() {\n        if (notExecuted()) {\n            return;\n        }\n        long totalMessages = 10;\n        queueTotal = 1;\n        messageBody = storeMessage.getBytes();\n        for (long i = 0; i < totalMessages; i++) {\n            messageStore.putMessage(buildMessage());\n        }\n\n        for (long i = 0; i < totalMessages; i++) {\n            GetMessageResult result = messageStore.getMessage(\"GROUP_A\", \"TOPIC_A\", 0, i, 1024 * 1024, null);\n            assertThat(result).isNotNull();\n            result.release();\n        }\n        verifyThatMasterIsFunctional(totalMessages, messageStore);\n    }\n\n    @Test\n    public void testMaxOffset() throws ConsumeQueueException {\n        if (notExecuted()) {\n            return;\n        }\n        int firstBatchMessages = 3;\n        int queueId = 0;\n        messageBody = storeMessage.getBytes();\n\n        assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId)).isEqualTo(0);\n\n        for (int i = 0; i < firstBatchMessages; i++) {\n            final MessageExtBrokerInner msg = buildMessage();\n            msg.setQueueId(queueId);\n            messageStore.putMessage(msg);\n        }\n\n        Awaitility.await()\n                .with()\n                .atMost(3, TimeUnit.SECONDS)\n                .pollInterval(1, TimeUnit.MILLISECONDS)\n                .until(() -> messageStore.getMaxOffsetInQueue(messageTopic, queueId) == firstBatchMessages);\n\n        // Disable the dispatcher\n        messageStore.getDispatcherList().clear();\n\n        int secondBatchMessages = 2;\n\n        for (int i = 0; i < secondBatchMessages; i++) {\n            final MessageExtBrokerInner msg = buildMessage();\n            msg.setQueueId(queueId);\n            messageStore.putMessage(msg);\n        }\n\n        assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId)).isEqualTo(firstBatchMessages);\n        assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId, true)).isEqualTo(firstBatchMessages);\n        assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId, false)).isEqualTo(firstBatchMessages + secondBatchMessages);\n    }\n\n    private MessageExtBrokerInner buildIPv6HostMessage() {\n        return buildIPv6HostMessage(messageBody, \"FooBar\");\n    }\n\n    private void verifyThatMasterIsFunctional(long totalMessages, MessageStore master) {\n        for (long i = 0; i < totalMessages; i++) {\n            master.putMessage(buildMessage());\n        }\n\n        StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore);\n\n        for (long i = 0; i < totalMessages; i++) {\n            GetMessageResult result = master.getMessage(\"GROUP_A\", \"FooBar\", 0, i, 1024 * 1024, null);\n            assertThat(result).isNotNull();\n            result.release();\n\n        }\n    }\n\n    @Test\n    public void testPullSize() {\n        if (notExecuted()) {\n            return;\n        }\n        String topic = \"pullSizeTopic\";\n\n        for (int i = 0; i < 32; i++) {\n            MessageExtBrokerInner messageExtBrokerInner = buildMessage();\n            messageExtBrokerInner.setTopic(topic);\n            messageExtBrokerInner.setQueueId(0);\n            messageStore.putMessage(messageExtBrokerInner);\n        }\n        // wait for consume queue build\n        Awaitility.await().atMost(10, TimeUnit.SECONDS)\n                .with()\n                .pollInterval(10, TimeUnit.MILLISECONDS)\n                .until(() -> messageStore.getMaxOffsetInQueue(topic, 0) >= 32);\n\n        String group = \"simple\";\n        GetMessageResult getMessageResult32 = messageStore.getMessage(group, topic, 0, 0, 32, null);\n        assertThat(getMessageResult32.getMessageBufferList().size()).isEqualTo(32);\n        getMessageResult32.release();\n\n        GetMessageResult getMessageResult20 = messageStore.getMessage(group, topic, 0, 0, 20, null);\n        assertThat(getMessageResult20.getMessageBufferList().size()).isEqualTo(20);\n\n        getMessageResult20.release();\n        GetMessageResult getMessageResult45 = messageStore.getMessage(group, topic, 0, 0, 10, null);\n        assertThat(getMessageResult45.getMessageBufferList().size()).isEqualTo(10);\n        getMessageResult45.release();\n\n    }\n\n    @Test\n    public void testRecover() throws Exception {\n        if (notExecuted()) {\n            return;\n        }\n        String topic = \"recoverTopic\";\n        messageBody = storeMessage.getBytes();\n        for (int i = 0; i < 100; i++) {\n            MessageExtBrokerInner messageExtBrokerInner = buildMessage();\n            messageExtBrokerInner.setTopic(topic);\n            messageExtBrokerInner.setQueueId(0);\n            messageStore.putMessage(messageExtBrokerInner);\n        }\n\n        // wait for build consumer queue\n        Awaitility.await()\n                .with()\n                .pollInterval(100, TimeUnit.MILLISECONDS)\n                .atMost(10, TimeUnit.SECONDS)\n                .until(() -> messageStore.getMaxOffsetInQueue(topic, 0) >= 100);\n\n        long maxPhyOffset = messageStore.getMaxPhyOffset();\n        long maxCqOffset = messageStore.getMaxOffsetInQueue(topic, 0);\n\n        //1.just reboot\n        messageStore.shutdown();\n        String storeRootDir = messageStore.getMessageStoreConfig().getStorePathRootDir();\n        messageStore = buildMessageStore(storeRootDir, topic);\n        boolean load = messageStore.load();\n        assertTrue(load);\n        messageStore.start();\n        assertEquals(maxPhyOffset, messageStore.getMaxPhyOffset());\n        assertEquals(maxCqOffset, messageStore.getMaxOffsetInQueue(topic, 0));\n\n        //2.damage commit-log and reboot normal\n        for (int i = 0; i < 100; i++) {\n            MessageExtBrokerInner messageExtBrokerInner = buildMessage();\n            messageExtBrokerInner.setTopic(topic);\n            messageExtBrokerInner.setQueueId(0);\n            messageStore.putMessage(messageExtBrokerInner);\n        }\n\n        Awaitility.await()\n                .with()\n                .pollInterval(100, TimeUnit.MILLISECONDS)\n                .atMost(10, TimeUnit.SECONDS)\n                .until(() -> messageStore.getMaxOffsetInQueue(topic, 0) >= 200);\n\n        long secondLastPhyOffset = messageStore.getMaxPhyOffset();\n        long secondLastCqOffset = messageStore.getMaxOffsetInQueue(topic, 0);\n\n        // Append a message to corrupt\n        MessageExtBrokerInner messageExtBrokerInner = buildMessage();\n        messageExtBrokerInner.setTopic(topic);\n        messageExtBrokerInner.setQueueId(0);\n        messageStore.putMessage(messageExtBrokerInner);\n\n        messageStore.shutdown();\n\n        // Corrupt the last message\n        damageCommitLog((RocksDBMessageStore) messageStore, secondLastPhyOffset);\n\n        //reboot\n        messageStore = buildMessageStore(storeRootDir, topic);\n        load = messageStore.load();\n        assertTrue(load);\n        messageStore.start();\n        assertEquals(secondLastPhyOffset, messageStore.getMaxPhyOffset());\n        assertEquals(secondLastCqOffset, messageStore.getMaxOffsetInQueue(topic, 0));\n\n        //3.Corrupt commit-log and reboot abnormal\n        for (int i = 0; i < 100; i++) {\n            messageExtBrokerInner = buildMessage();\n            messageExtBrokerInner.setTopic(topic);\n            messageExtBrokerInner.setQueueId(0);\n            messageStore.putMessage(messageExtBrokerInner);\n        }\n\n        Awaitility.await()\n                .with()\n                .pollInterval(100, TimeUnit.MILLISECONDS)\n                .atMost(10, TimeUnit.SECONDS)\n                .until(() -> messageStore.getMaxOffsetInQueue(topic, 0) >= 300);\n\n        secondLastPhyOffset = messageStore.getMaxPhyOffset();\n        secondLastCqOffset = messageStore.getMaxOffsetInQueue(topic, 0);\n\n        messageExtBrokerInner = buildMessage();\n        messageExtBrokerInner.setTopic(topic);\n        messageExtBrokerInner.setQueueId(0);\n        messageStore.putMessage(messageExtBrokerInner);\n        messageStore.shutdown();\n\n        //Corrupt the last message\n        damageCommitLog((RocksDBMessageStore) messageStore, secondLastPhyOffset);\n        //add abort file\n        String fileName = StorePathConfigHelper.getAbortFile(messageStore.getMessageStoreConfig().getStorePathRootDir());\n        File file = new File(fileName);\n        UtilAll.ensureDirOK(file.getParent());\n        assertTrue(file.createNewFile());\n\n        messageStore = buildMessageStore(storeRootDir, topic);\n        load = messageStore.load();\n        assertTrue(load);\n        messageStore.start();\n        assertEquals(secondLastPhyOffset, messageStore.getMaxPhyOffset());\n        assertEquals(secondLastCqOffset, messageStore.getMaxOffsetInQueue(topic, 0));\n\n        //message write again\n        for (int i = 0; i < 100; i++) {\n            messageExtBrokerInner = buildMessage();\n            messageExtBrokerInner.setTopic(topic);\n            messageExtBrokerInner.setQueueId(0);\n            messageStore.putMessage(messageExtBrokerInner);\n        }\n    }\n\n    @Test\n    public void testStorePathOK() {\n        if (notExecuted()) {\n            return;\n        }\n        if (messageStore instanceof RocksDBMessageStore) {\n            assertTrue(fileExists(((RocksDBMessageStore) messageStore).getStorePathPhysic()));\n            assertTrue(fileExists(((RocksDBMessageStore) messageStore).getStorePathLogic()));\n        }\n    }\n\n    private boolean fileExists(String path) {\n        if (path != null) {\n            File f = new File(path);\n            return f.exists();\n        }\n        return false;\n    }\n\n    private void damageCommitLog(RocksDBMessageStore store, long offset) throws Exception {\n        assertThat(store).isNotNull();\n        MessageStoreConfig messageStoreConfig = store.getMessageStoreConfig();\n        File file = new File(messageStoreConfig.getStorePathCommitLog() + File.separator + \"00000000000000000000\");\n        try (RandomAccessFile raf = new RandomAccessFile(file, \"rw\");\n            FileChannel fileChannel = raf.getChannel()) {\n            MappedByteBuffer mappedByteBuffer = fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, 1024 * 1024 * 10);\n            int bodyLen = mappedByteBuffer.getInt((int) offset + 84);\n            int topicLenIndex = (int) offset + 84 + bodyLen + 4;\n            mappedByteBuffer.position(topicLenIndex);\n            mappedByteBuffer.putInt(0);\n            mappedByteBuffer.putInt(0);\n            mappedByteBuffer.putInt(0);\n            mappedByteBuffer.putInt(0);\n            mappedByteBuffer.force();\n            fileChannel.force(true);\n        }\n    }\n\n    @Test\n    public void testPutMsgExceedsMaxLength() {\n        if (notExecuted()) {\n            return;\n        }\n        messageBody = new byte[4 * 1024 * 1024 + 1];\n        MessageExtBrokerInner msg = buildMessage();\n\n        PutMessageResult result = messageStore.putMessage(msg);\n        assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.MESSAGE_ILLEGAL);\n    }\n\n    @Test\n    public void testPutMsgBatchExceedsMaxLength() {\n        if (notExecuted()) {\n            return;\n        }\n        messageBody = new byte[4 * 1024 * 1024 + 1];\n        MessageExtBrokerInner msg1 = buildMessage();\n        MessageExtBrokerInner msg2 = buildMessage();\n        MessageExtBrokerInner msg3 = buildMessage();\n\n        MessageBatch msgBatch = MessageBatch.generateFromList(Arrays.asList(msg1, msg2, msg3));\n        msgBatch.setBody(msgBatch.encode());\n\n        MessageExtBatch msgExtBatch = buildMessageBatch(msgBatch);\n\n        try {\n            this.messageStore.putMessages(msgExtBatch);\n            fail(\"Should have raised an exception\");\n        } catch (Exception e) {\n            assertThat(e.getMessage()).contains(\"message body size exceeded\");\n        }\n    }\n\n    @Test\n    public void testPutMsgWhenReplicasNotEnough() {\n        if (notExecuted()) {\n            return;\n        }\n        MessageStoreConfig messageStoreConfig = this.messageStore.getMessageStoreConfig();\n        messageStoreConfig.setBrokerRole(BrokerRole.SYNC_MASTER);\n        messageStoreConfig.setTotalReplicas(2);\n        messageStoreConfig.setInSyncReplicas(2);\n        messageStoreConfig.setEnableAutoInSyncReplicas(false);\n        ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(true);\n        this.messageStore.setAliveReplicaNumInGroup(1);\n\n        MessageExtBrokerInner msg = buildMessage();\n        PutMessageResult result = this.messageStore.putMessage(msg);\n        assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH);\n        ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(false);\n    }\n\n    @Test\n    public void testPutMsgWhenAdaptiveDegradation() {\n        if (notExecuted()) {\n            return;\n        }\n        MessageStoreConfig messageStoreConfig = this.messageStore.getMessageStoreConfig();\n        messageStoreConfig.setBrokerRole(BrokerRole.SYNC_MASTER);\n        messageStoreConfig.setTotalReplicas(2);\n        messageStoreConfig.setInSyncReplicas(2);\n        messageStoreConfig.setEnableAutoInSyncReplicas(true);\n        ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(true);\n        this.messageStore.setAliveReplicaNumInGroup(1);\n\n        MessageExtBrokerInner msg = buildMessage();\n        PutMessageResult result = this.messageStore.putMessage(msg);\n        assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK);\n        ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(false);\n        messageStoreConfig.setEnableAutoInSyncReplicas(false);\n    }\n\n    @Test\n    public void testGetBulkCommitLogData() {\n        if (notExecuted()) {\n            return;\n        }\n        RocksDBMessageStore defaultMessageStore = (RocksDBMessageStore) messageStore;\n\n        messageBody = new byte[2 * 1024 * 1024];\n\n        for (int i = 0; i < 10; i++) {\n            MessageExtBrokerInner msg1 = buildMessage();\n            messageStore.putMessage(msg1);\n        }\n\n        List<SelectMappedBufferResult> bufferResultList = defaultMessageStore.getBulkCommitLogData(0, (int) defaultMessageStore.getMaxPhyOffset());\n        List<MessageExt> msgList = new ArrayList<>();\n        for (SelectMappedBufferResult bufferResult : bufferResultList) {\n            msgList.addAll(MessageDecoder.decodesBatch(bufferResult.getByteBuffer(), true, false, false));\n            bufferResult.release();\n        }\n\n        assertThat(msgList.size()).isEqualTo(10);\n    }\n\n    @Test\n    public void testPutLongMessage() {\n        if (notExecuted()) {\n            return;\n        }\n        MessageExtBrokerInner messageExtBrokerInner = buildMessage();\n        CommitLog commitLog = messageStore.getCommitLog();\n        MessageStoreConfig messageStoreConfig = messageStore.getMessageStoreConfig();\n        MessageExtEncoder.PutMessageThreadLocal putMessageThreadLocal = commitLog.getPutMessageThreadLocal().get();\n\n        //body size, topic size, properties size exactly equal to max size\n        messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]);\n        messageExtBrokerInner.setTopic(new String(new byte[127]));\n        messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE]));\n        PutMessageResult encodeResult1 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);\n        assertNull(encodeResult1);\n\n        //body size exactly more than max message body size\n        messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize() + 1]);\n        PutMessageResult encodeResult2 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);\n        assertSame(encodeResult2.getPutMessageStatus(), PutMessageStatus.MESSAGE_ILLEGAL);\n\n        //body size exactly equal to max message size\n        messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize() + 64 * 1024]);\n        PutMessageResult encodeResult3 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);\n        assertSame(encodeResult3.getPutMessageStatus(), PutMessageStatus.MESSAGE_ILLEGAL);\n\n        //message properties length more than properties maxSize\n        messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]);\n        messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE + 1]));\n        PutMessageResult encodeResult4 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);\n        assertSame(encodeResult4.getPutMessageStatus(), PutMessageStatus.PROPERTIES_SIZE_EXCEEDED);\n\n        //message length more than buffer length capacity\n        messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]);\n        messageExtBrokerInner.setTopic(new String(new byte[Short.MAX_VALUE]));\n        messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE]));\n        PutMessageResult encodeResult5 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);\n        assertSame(encodeResult5.getPutMessageStatus(), PutMessageStatus.MESSAGE_ILLEGAL);\n    }\n\n    @Test\n    public void testDynamicMaxMessageSize() {\n        if (notExecuted()) {\n            return;\n        }\n        MessageExtBrokerInner messageExtBrokerInner = buildMessage();\n        MessageStoreConfig messageStoreConfig = messageStore.getMessageStoreConfig();\n        int originMaxMessageSize = messageStoreConfig.getMaxMessageSize();\n\n        messageExtBrokerInner.setBody(new byte[originMaxMessageSize + 10]);\n        PutMessageResult putMessageResult = messageStore.putMessage(messageExtBrokerInner);\n        assertSame(putMessageResult.getPutMessageStatus(), PutMessageStatus.MESSAGE_ILLEGAL);\n\n        int newMaxMessageSize = originMaxMessageSize + 10;\n        messageStoreConfig.setMaxMessageSize(newMaxMessageSize);\n        putMessageResult = messageStore.putMessage(messageExtBrokerInner);\n        assertSame(putMessageResult.getPutMessageStatus(), PutMessageStatus.PUT_OK);\n\n        messageStoreConfig.setMaxMessageSize(10);\n        putMessageResult = messageStore.putMessage(messageExtBrokerInner);\n        assertSame(putMessageResult.getPutMessageStatus(), PutMessageStatus.MESSAGE_ILLEGAL);\n\n        messageStoreConfig.setMaxMessageSize(originMaxMessageSize);\n    }\n\n    @Test\n    public void testDeleteTopics() {\n        if (notExecuted()) {\n            return;\n        }\n        MessageStoreConfig messageStoreConfig = messageStore.getMessageStoreConfig();\n        ConcurrentMap<String, ConcurrentMap<Integer, ConsumeQueueInterface>> consumeQueueTable =\n            ((RocksDBMessageStore) messageStore).getConsumeQueueTable();\n        for (int i = 0; i < 10; i++) {\n            ConcurrentMap<Integer, ConsumeQueueInterface> cqTable = new ConcurrentHashMap<>();\n            String topicName = \"topic-\" + i;\n            for (int j = 0; j < 4; j++) {\n                RocksDBConsumeQueue consumeQueue = new RocksDBConsumeQueue(messageStoreConfig,\n                    (RocksDBConsumeQueueStore) messageStore.getQueueStore(), topicName, i);\n                cqTable.put(j, consumeQueue);\n            }\n            consumeQueueTable.put(topicName, cqTable);\n        }\n        assertEquals(consumeQueueTable.size(), 10);\n        HashSet<String> resultSet = Sets.newHashSet(\"topic-3\", \"topic-5\");\n        messageStore.deleteTopics(Sets.difference(consumeQueueTable.keySet(), resultSet));\n        assertEquals(consumeQueueTable.size(), 2);\n        assertEquals(resultSet, consumeQueueTable.keySet());\n    }\n\n    @Test\n    public void testCleanUnusedTopic() {\n        if (notExecuted()) {\n            return;\n        }\n        MessageStoreConfig messageStoreConfig = messageStore.getMessageStoreConfig();\n        ConcurrentMap<String, ConcurrentMap<Integer, ConsumeQueueInterface>> consumeQueueTable =\n            ((RocksDBMessageStore) messageStore).getConsumeQueueTable();\n        for (int i = 0; i < 10; i++) {\n            ConcurrentMap<Integer, ConsumeQueueInterface> cqTable = new ConcurrentHashMap<>();\n            String topicName = \"topic-\" + i;\n            for (int j = 0; j < 4; j++) {\n                RocksDBConsumeQueue consumeQueue = new RocksDBConsumeQueue(messageStoreConfig,\n                    (RocksDBConsumeQueueStore) messageStore.getQueueStore(), topicName, i);\n                cqTable.put(j, consumeQueue);\n            }\n            consumeQueueTable.put(topicName, cqTable);\n        }\n        assertEquals(consumeQueueTable.size(), 10);\n        HashSet<String> resultSet = Sets.newHashSet(\"topic-3\", \"topic-5\");\n        messageStore.cleanUnusedTopic(resultSet);\n        assertEquals(consumeQueueTable.size(), 2);\n        assertEquals(resultSet, consumeQueueTable.keySet());\n    }\n\n    private static class MyMessageArrivingListener implements MessageArrivingListener {\n        @Override\n        public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime,\n            byte[] filterBitMap, Map<String, String> properties) {\n        }\n    }\n\n    private boolean notExecuted() {\n        return MixAll.isMac();\n    }\n}\n\n\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/StoreCheckpointTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: StoreCheckpointTest.java 1831 2013-05-16 01:39:51Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.store;\n\nimport java.io.File;\nimport java.io.IOException;\n\nimport org.apache.rocketmq.common.UtilAll;\nimport org.junit.After;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class StoreCheckpointTest {\n    @Test\n    public void testWriteAndRead() throws IOException {\n        StoreCheckpoint storeCheckpoint = new StoreCheckpoint(\"target/checkpoint_test/0000\");\n        long physicMsgTimestamp = 0xAABB;\n        long logicsMsgTimestamp = 0xCCDD;\n        long logicsPhysicalOffset = 0x1000L;\n        storeCheckpoint.setPhysicMsgTimestamp(physicMsgTimestamp);\n        storeCheckpoint.setLogicsMsgTimestamp(logicsMsgTimestamp);\n        storeCheckpoint.setLogicsPhysicalOffset(logicsPhysicalOffset);\n        storeCheckpoint.flush();\n\n        long diff = physicMsgTimestamp - storeCheckpoint.getMinTimestamp();\n        assertThat(diff).isEqualTo(3000);\n        storeCheckpoint.shutdown();\n        storeCheckpoint = new StoreCheckpoint(\"target/checkpoint_test/0000\");\n        assertThat(storeCheckpoint.getPhysicMsgTimestamp()).isEqualTo(physicMsgTimestamp);\n        assertThat(storeCheckpoint.getLogicsMsgTimestamp()).isEqualTo(logicsMsgTimestamp);\n        assertThat(storeCheckpoint.getLogicsPhysicalOffset()).isEqualTo(logicsPhysicalOffset);\n    }\n\n    @After\n    public void destroy() {\n        File file = new File(\"target/checkpoint_test\");\n        UtilAll.deleteFile(file);\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/StoreStatsServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport java.lang.reflect.InvocationTargetException;\nimport java.lang.reflect.Method;\nimport java.util.concurrent.BrokenBarrierException;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.CyclicBarrier;\nimport java.util.concurrent.atomic.AtomicReference;\nimport java.util.concurrent.atomic.LongAdder;\n\nimport org.junit.Test;\n\npublic class StoreStatsServiceTest {\n\n    @Test\n    public void getSinglePutMessageTopicSizeTotal() throws Exception {\n        final StoreStatsService storeStatsService = new StoreStatsService();\n        int num = Runtime.getRuntime().availableProcessors() * 2;\n        for (int j = 0; j < 100; j++) {\n            final AtomicReference<LongAdder> reference = new AtomicReference<>(null);\n            final CountDownLatch latch = new CountDownLatch(num);\n            final CyclicBarrier barrier = new CyclicBarrier(num);\n            for (int i = 0; i < num; i++) {\n                new Thread(new Runnable() {\n                    @Override\n                    public void run() {\n                        try {\n                            barrier.await();\n                            LongAdder longAdder = storeStatsService.getSinglePutMessageTopicSizeTotal(\"test\");\n                            if (reference.compareAndSet(null, longAdder)) {\n                            } else if (reference.get() != longAdder) {\n                                throw new RuntimeException(\"Reference should be same!\");\n                            }\n                        } catch (InterruptedException | BrokenBarrierException e) {\n                            e.printStackTrace();\n                        } finally {\n                            latch.countDown();\n                        }\n                    }\n                }).start();\n            }\n            latch.await();\n        }\n    }\n\n    @Test\n    public void getSinglePutMessageTopicTimesTotal() throws Exception {\n        final StoreStatsService storeStatsService = new StoreStatsService();\n        int num = Runtime.getRuntime().availableProcessors() * 2;\n        for (int j = 0; j < 100; j++) {\n            final AtomicReference<LongAdder> reference = new AtomicReference<>(null);\n            final CountDownLatch latch = new CountDownLatch(num);\n            final CyclicBarrier barrier = new CyclicBarrier(num);\n            for (int i = 0; i < num; i++) {\n                new Thread(new Runnable() {\n                    @Override\n                    public void run() {\n                        try {\n                            barrier.await();\n                            LongAdder longAdder = storeStatsService.getSinglePutMessageTopicTimesTotal(\"test\");\n                            if (reference.compareAndSet(null, longAdder)) {\n                            } else if (reference.get() != longAdder) {\n                                throw new RuntimeException(\"Reference should be same!\");\n                            }\n                        } catch (InterruptedException | BrokenBarrierException e) {\n                            e.printStackTrace();\n                        } finally {\n                            latch.countDown();\n                        }\n                    }\n                }).start();\n            }\n            latch.await();\n        }\n    }\n\n    @Test\n    public void findPutMessageEntireTimePXTest() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException {\n        final StoreStatsService storeStatsService = new StoreStatsService();\n        for (int i = 1; i <= 1000; i++) {\n            for (int j = 0; j < i; j++) {\n                storeStatsService.incPutMessageEntireTime(i);\n            }\n        }\n        Method method = StoreStatsService.class.getDeclaredMethod(\"resetPutMessageTimeBuckets\");\n        method.setAccessible(true);\n        method.invoke(storeStatsService);\n    }\n\n}"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExtBatch;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.junit.After;\n\nimport java.io.File;\nimport java.net.InetAddress;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.net.UnknownHostException;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.UUID;\nimport java.util.concurrent.atomic.AtomicInteger;\n\npublic class StoreTestBase {\n\n    private static final int QUEUE_TOTAL = 100;\n    private AtomicInteger queueId = new AtomicInteger(0);\n    protected SocketAddress bornHost = new InetSocketAddress(\"127.0.0.1\", 8123);\n    protected SocketAddress storeHost = bornHost;\n    private byte[] messageBody = new byte[1024];\n\n    protected Set<String> baseDirs = new HashSet<>();\n\n    private static AtomicInteger port = new AtomicInteger(30000);\n\n    public static synchronized int nextPort() {\n        return port.addAndGet(5);\n    }\n\n    protected MessageExtBatch buildBatchMessage(int size) {\n        MessageExtBatch messageExtBatch = new MessageExtBatch();\n        messageExtBatch.setTopic(\"StoreTest\");\n        messageExtBatch.setTags(\"TAG1\");\n        messageExtBatch.setKeys(\"Hello\");\n        messageExtBatch.setQueueId(Math.abs(queueId.getAndIncrement()) % QUEUE_TOTAL);\n        messageExtBatch.setSysFlag(0);\n\n        messageExtBatch.setBornTimestamp(System.currentTimeMillis());\n        messageExtBatch.setBornHost(bornHost);\n        messageExtBatch.setStoreHost(storeHost);\n\n        List<Message> messageList = new ArrayList<>(size);\n        for (int i = 0; i < size; i++) {\n            messageList.add(buildMessage());\n        }\n\n        messageExtBatch.setBody(MessageDecoder.encodeMessages(messageList));\n\n        return messageExtBatch;\n    }\n\n    protected MessageExtBrokerInner buildMessage() {\n        MessageExtBrokerInner msg = new MessageExtBrokerInner();\n        msg.setTopic(\"StoreTest\");\n        msg.setTags(\"TAG1\");\n        msg.setKeys(\"Hello\");\n        msg.setBody(messageBody);\n        msg.setKeys(String.valueOf(System.currentTimeMillis()));\n        msg.setQueueId(Math.abs(queueId.getAndIncrement()) % QUEUE_TOTAL);\n        msg.setSysFlag(0);\n        msg.setBornTimestamp(System.currentTimeMillis());\n        msg.setStoreHost(storeHost);\n        msg.setBornHost(bornHost);\n        return msg;\n    }\n\n    protected MessageExtBatch buildIPv6HostBatchMessage(int size) {\n        MessageExtBatch messageExtBatch = new MessageExtBatch();\n        messageExtBatch.setTopic(\"StoreTest\");\n        messageExtBatch.setTags(\"TAG1\");\n        messageExtBatch.setKeys(\"Hello\");\n        messageExtBatch.setBody(messageBody);\n        messageExtBatch.setMsgId(\"24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0\");\n        messageExtBatch.setKeys(String.valueOf(System.currentTimeMillis()));\n        messageExtBatch.setQueueId(Math.abs(queueId.getAndIncrement()) % QUEUE_TOTAL);\n        messageExtBatch.setSysFlag(0);\n        messageExtBatch.setBornHostV6Flag();\n        messageExtBatch.setStoreHostAddressV6Flag();\n        messageExtBatch.setBornTimestamp(System.currentTimeMillis());\n        try {\n            messageExtBatch.setBornHost(new InetSocketAddress(InetAddress.getByName(\"1050:0000:0000:0000:0005:0600:300c:326b\"), 8123));\n        } catch (UnknownHostException e) {\n            e.printStackTrace();\n        }\n\n        try {\n            messageExtBatch.setStoreHost(new InetSocketAddress(InetAddress.getByName(\"::1\"), 8123));\n        } catch (UnknownHostException e) {\n            e.printStackTrace();\n        }\n\n        List<Message> messageList = new ArrayList<>(size);\n        for (int i = 0; i < size; i++) {\n            messageList.add(buildIPv6HostMessage());\n        }\n\n        messageExtBatch.setBody(MessageDecoder.encodeMessages(messageList));\n        return messageExtBatch;\n    }\n\n    protected MessageExtBrokerInner buildIPv6HostMessage() {\n        MessageExtBrokerInner msg = new MessageExtBrokerInner();\n        msg.setTopic(\"StoreTest\");\n        msg.setTags(\"TAG1\");\n        msg.setKeys(\"Hello\");\n        msg.setBody(messageBody);\n        msg.setMsgId(\"24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0\");\n        msg.setKeys(String.valueOf(System.currentTimeMillis()));\n        msg.setQueueId(Math.abs(queueId.getAndIncrement()) % QUEUE_TOTAL);\n        msg.setSysFlag(0);\n        msg.setBornHostV6Flag();\n        msg.setStoreHostAddressV6Flag();\n        msg.setBornTimestamp(System.currentTimeMillis());\n        try {\n            msg.setBornHost(new InetSocketAddress(InetAddress.getByName(\"1050:0000:0000:0000:0005:0600:300c:326b\"), 8123));\n        } catch (UnknownHostException e) {\n            e.printStackTrace();\n        }\n\n        try {\n            msg.setStoreHost(new InetSocketAddress(InetAddress.getByName(\"::1\"), 8123));\n        } catch (UnknownHostException e) {\n            e.printStackTrace();\n        }\n        return msg;\n    }\n\n    public static String createBaseDir() {\n        String baseDir = System.getProperty(\"java.io.tmpdir\") + File.separator + \"unitteststore\" + File.separator + UUID.randomUUID();\n        final File file = new File(baseDir);\n        if (file.exists()) {\n            System.exit(1);\n        }\n        return baseDir;\n    }\n\n    public static boolean makeSureFileExists(String fileName) throws Exception {\n        File file = new File(fileName);\n        UtilAll.ensureDirOK(file.getParent());\n        return file.createNewFile();\n    }\n\n    public static void deleteFile(String fileName) {\n        deleteFile(new File(fileName));\n    }\n\n    public static void deleteFile(File file) {\n        UtilAll.deleteFile(file);\n    }\n\n    @After\n    public void clear() {\n        for (String baseDir : baseDirs) {\n            deleteFile(baseDir);\n        }\n    }\n\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/StoreTestUtil.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store;\n\nimport io.openmessaging.storage.dledger.store.file.DefaultMmapFile;\nimport io.openmessaging.storage.dledger.store.file.MmapFile;\nimport java.io.IOException;\nimport java.lang.reflect.Field;\nimport java.lang.reflect.InvocationTargetException;\nimport java.lang.reflect.Method;\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.commons.lang3.SystemUtils;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.index.IndexFile;\nimport org.apache.rocketmq.store.index.IndexService;\nimport org.apache.rocketmq.store.queue.ConsumeQueueStore;\n\npublic class StoreTestUtil {\n\n    private static final Logger log = LoggerFactory.getLogger(StoreTestUtil.class);\n\n    public static boolean isCommitLogAvailable(DefaultMessageStore store) {\n        try {\n            Field serviceField = null;\n            if (store instanceof RocksDBMessageStore) {\n                serviceField = store.getClass().getSuperclass().getDeclaredField(\"reputMessageService\");\n            } else {\n                serviceField = store.getClass().getDeclaredField(\"reputMessageService\");\n            }\n\n            serviceField.setAccessible(true);\n            DefaultMessageStore.ReputMessageService reputService =\n                (DefaultMessageStore.ReputMessageService) serviceField.get(store);\n\n            Method method = DefaultMessageStore.ReputMessageService.class.getDeclaredMethod(\"isCommitLogAvailable\");\n            method.setAccessible(true);\n            return (boolean) method.invoke(reputService);\n        } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | NoSuchFieldException e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    public static void flushConsumeQueue(DefaultMessageStore store) throws Exception {\n        Field field = store.getQueueStore().getClass().getDeclaredField(\"flushConsumeQueueService\");\n        field.setAccessible(true);\n        ConsumeQueueStore.FlushConsumeQueueService flushService = (ConsumeQueueStore.FlushConsumeQueueService) field.get(store.getQueueStore());\n\n        final int retryTimesOver = 3;\n        Method method = ConsumeQueueStore.FlushConsumeQueueService.class.getDeclaredMethod(\"doFlush\", int.class);\n        method.setAccessible(true);\n        method.invoke(flushService, retryTimesOver);\n    }\n\n\n    public static void waitCommitLogReput(DefaultMessageStore store) {\n        for (int i = 0; i < 500 && isCommitLogAvailable(store); i++) {\n            try {\n                Thread.sleep(100);\n            } catch (InterruptedException ignored) {\n            }\n        }\n\n        if (isCommitLogAvailable(store)) {\n            log.warn(\"isCommitLogAvailable expected false ,but true\");\n        }\n    }\n\n\n    public static void flushConsumeIndex(DefaultMessageStore store) throws NoSuchFieldException, Exception {\n        Field field = store.getClass().getDeclaredField(\"indexService\");\n        field.setAccessible(true);\n        IndexService indexService = (IndexService) field.get(store);\n\n        Field field2 = indexService.getClass().getDeclaredField(\"indexFileList\");\n        field2.setAccessible(true);\n        ArrayList<IndexFile> indexFileList = (ArrayList<IndexFile>) field2.get(indexService);\n\n        for (IndexFile f : indexFileList) {\n            indexService.flush(f);\n        }\n    }\n\n    public static void releaseMmapFilesOnWindows(List<MmapFile> mappedFiles) throws IOException {\n        if (!SystemUtils.IS_OS_WINDOWS) {\n            return;\n        }\n        for (final MmapFile mappedFile : mappedFiles) {\n            DefaultMmapFile.clean(mappedFile.getMappedByteBuffer());\n            mappedFile.getFileChannel().close();\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.dledger;\n\nimport io.openmessaging.storage.dledger.DLedgerServer;\nimport io.openmessaging.storage.dledger.store.file.DLedgerMmapFileStore;\nimport io.openmessaging.storage.dledger.store.file.MmapFileList;\n\nimport java.io.File;\nimport java.nio.ByteBuffer;\nimport java.time.Duration;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.UUID;\nimport java.util.concurrent.Callable;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBatch;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.StoreCheckpoint;\nimport org.apache.rocketmq.store.config.StorePathConfigHelper;\nimport org.junit.Assert;\nimport org.junit.BeforeClass;\nimport org.junit.Ignore;\nimport org.junit.Test;\nimport org.junit.Assume;\nimport org.apache.rocketmq.common.MixAll;\n\nimport static java.util.concurrent.TimeUnit.SECONDS;\nimport static org.apache.rocketmq.store.StoreTestUtil.releaseMmapFilesOnWindows;\nimport static org.awaitility.Awaitility.await;\n\npublic class DLedgerCommitlogTest extends MessageStoreTestBase {\n\n    @BeforeClass\n    public static void beforeClass() {\n        // Temporarily skip those tests on the macOS as they are flaky\n        Assume.assumeFalse(MixAll.isMac());\n    }\n\n    @Ignore\n    @Test\n    public void testTruncateCQ() throws Exception {\n        String base = createBaseDir();\n        String peers = String.format(\"n0-localhost:%d\", nextPort());\n        String group = UUID.randomUUID().toString();\n        String topic = UUID.randomUUID().toString();\n        {\n            DefaultMessageStore messageStore = createDledgerMessageStore(base, group, \"n0\", peers, null, false, 0);\n            DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog();\n            DLedgerServer dLedgerServer = dLedgerCommitLog.getdLedgerServer();\n            DLedgerMmapFileStore dLedgerMmapFileStore = (DLedgerMmapFileStore) dLedgerServer.getdLedgerStore();\n            MmapFileList mmapFileList = dLedgerMmapFileStore.getDataFileList();\n            Boolean success = await().atMost(Duration.ofSeconds(4)).until(() -> dLedgerCommitLog.getdLedgerServer().getMemberState().isLeader(), item -> item);\n            Assert.assertTrue(success);\n            doPutMessages(messageStore, topic, 0, 2000, 0);\n            await().atMost(Duration.ofSeconds(10)).until(() -> 2000 == messageStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(24, mmapFileList.getMappedFiles().size());\n            Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0));\n            Assert.assertEquals(2000, messageStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, messageStore.dispatchBehindBytes());\n            doGetMessages(messageStore, topic, 0, 2000, 0);\n            messageStore.shutdown();\n            releaseMmapFilesOnWindows(dLedgerMmapFileStore.getDataFileList().getMappedFiles());\n        }\n\n        {\n            //Abnormal recover, left some commitlogs\n            DefaultMessageStore messageStore = createDledgerMessageStore(base, group, \"n0\", peers, null, true, 4);\n            DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog();\n            DLedgerServer dLedgerServer = dLedgerCommitLog.getdLedgerServer();\n            DLedgerMmapFileStore dLedgerMmapFileStore = (DLedgerMmapFileStore) dLedgerServer.getdLedgerStore();\n            MmapFileList mmapFileList = dLedgerMmapFileStore.getDataFileList();\n            Boolean success = await().atMost(Duration.ofSeconds(4)).until(() -> dLedgerCommitLog.getdLedgerServer().getMemberState().isLeader(), item -> item);\n            Assert.assertTrue(success);\n            Assert.assertEquals(20, mmapFileList.getMappedFiles().size());\n            Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0));\n            Assert.assertEquals(1700, messageStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, messageStore.dispatchBehindBytes());\n            doGetMessages(messageStore, topic, 0, 1700, 0);\n            messageStore.shutdown();\n            releaseMmapFilesOnWindows(dLedgerMmapFileStore.getDataFileList().getMappedFiles());\n        }\n        {\n            //Abnormal recover, left none commitlogs\n            DefaultMessageStore messageStore = createDledgerMessageStore(base, group, \"n0\", peers, null, true, 20);\n            DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog();\n            DLedgerServer dLedgerServer = dLedgerCommitLog.getdLedgerServer();\n            DLedgerMmapFileStore dLedgerMmapFileStore = (DLedgerMmapFileStore) dLedgerServer.getdLedgerStore();\n            MmapFileList mmapFileList = dLedgerMmapFileStore.getDataFileList();\n            Boolean success = await().atMost(Duration.ofSeconds(4)).until(() -> dLedgerCommitLog.getdLedgerServer().getMemberState().isLeader(), item -> item);\n            Assert.assertTrue(success);\n            Assert.assertEquals(0, mmapFileList.getMappedFiles().size());\n            Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, messageStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, messageStore.dispatchBehindBytes());\n            messageStore.shutdown();\n        }\n    }\n\n    @Test\n    public void testRecover() throws Exception {\n        String base = createBaseDir();\n        String peers = String.format(\"n0-localhost:%d\", nextPort());\n        String group = UUID.randomUUID().toString();\n        String topic = UUID.randomUUID().toString();\n        {\n            DefaultMessageStore messageStore = createDledgerMessageStore(base, group, \"n0\", peers, null, false, 0);\n            DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog();\n            Boolean success = await().atMost(Duration.ofSeconds(4)).until(() -> dLedgerCommitLog.getdLedgerServer().getMemberState().isLeader(), item -> item);\n            Assert.assertTrue(success);\n            doPutMessages(messageStore, topic, 0, 1000, 0);\n            await().atMost(Duration.ofSeconds(10)).until(() -> 1000 == messageStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0));\n            Assert.assertEquals(1000, messageStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, messageStore.dispatchBehindBytes());\n            doGetMessages(messageStore, topic, 0, 1000, 0);\n            messageStore.shutdown();\n        }\n\n        {\n            //normal recover\n            DefaultMessageStore messageStore = createDledgerMessageStore(base, group, \"n0\", peers, null, false, 0);\n            Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0));\n            Assert.assertEquals(1000, messageStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, messageStore.dispatchBehindBytes());\n            doGetMessages(messageStore, topic, 0, 1000, 0);\n            messageStore.shutdown();\n        }\n\n        {\n            //abnormal recover\n            DefaultMessageStore messageStore = createDledgerMessageStore(base, group, \"n0\", peers, null, true, 0);\n            Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0));\n            Assert.assertEquals(1000, messageStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, messageStore.dispatchBehindBytes());\n            doGetMessages(messageStore, topic, 0, 1000, 0);\n            messageStore.shutdown();\n        }\n    }\n    @Test\n    public void testDLedgerAbnormallyRecover() throws Exception {\n        String base = createBaseDir();\n        String peers = String.format(\"n0-localhost:%d\", nextPort());\n        String group = UUID.randomUUID().toString();\n        String topic = UUID.randomUUID().toString();\n\n        int messageNumPerQueue = 100;\n\n        DefaultMessageStore messageStore = createDledgerMessageStore(base, group, \"n0\", peers, null, false, 0);\n        Thread.sleep(1000);\n        doPutMessages(messageStore, topic, 0, messageNumPerQueue, 0);\n        doPutMessages(messageStore, topic, 1, messageNumPerQueue, 0);\n        Thread.sleep(1000);\n        Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0));\n        Assert.assertEquals(messageNumPerQueue, messageStore.getMaxOffsetInQueue(topic, 0));\n        Assert.assertEquals(0, messageStore.dispatchBehindBytes());\n        doGetMessages(messageStore, topic, 0, messageNumPerQueue, 0);\n        StoreCheckpoint storeCheckpoint = messageStore.getStoreCheckpoint();\n        storeCheckpoint.setPhysicMsgTimestamp(0);\n        storeCheckpoint.setLogicsMsgTimestamp(0);\n        messageStore.shutdown();\n\n        String fileName = StorePathConfigHelper.getAbortFile(base);\n        makeSureFileExists(fileName);\n\n        File file = new File(base + File.separator + \"consumequeue\" + File.separator + topic + File.separator + \"0\" + File.separator + \"00000000000000001040\");\n        file.delete();\n//        truncateAllConsumeQueue(base + File.separator + \"consumequeue\" + File.separator + topic + File.separator);\n        messageStore = createDledgerMessageStore(base, group, \"n0\", peers, null, false, 0);\n        Thread.sleep(1000);\n        doGetMessages(messageStore, topic, 0, messageNumPerQueue, 0);\n        doGetMessages(messageStore, topic, 1, messageNumPerQueue, 0);\n        messageStore.shutdown();\n\n    }\n\n    @Test\n    public void testPutAndGetMessage() throws Exception {\n        String base = createBaseDir();\n        String peers = String.format(\"n0-localhost:%d\", nextPort());\n        String group = UUID.randomUUID().toString();\n        DefaultMessageStore messageStore = createDledgerMessageStore(base, group, \"n0\", peers, null, false, 0);\n        DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog();\n        Boolean success = await().atMost(Duration.ofSeconds(4)).until(() -> dLedgerCommitLog.getdLedgerServer().getMemberState().isLeader(), item -> item);\n        Assert.assertTrue(success);\n        String topic = UUID.randomUUID().toString();\n\n        List<PutMessageResult> results = new ArrayList<>();\n        for (int i = 0; i < 10; i++) {\n            MessageExtBrokerInner msgInner =\n                i < 5 ? buildMessage() : buildIPv6HostMessage();\n            msgInner.setTopic(topic);\n            msgInner.setQueueId(0);\n            PutMessageResult putMessageResult = messageStore.putMessage(msgInner);\n            results.add(putMessageResult);\n            Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n            Assert.assertEquals(i, putMessageResult.getAppendMessageResult().getLogicsOffset());\n        }\n        await().atMost(Duration.ofSeconds(10)).until(() -> 10 == messageStore.getMaxOffsetInQueue(topic, 0));\n        Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0));\n        Assert.assertEquals(10, messageStore.getMaxOffsetInQueue(topic, 0));\n        Assert.assertEquals(0, messageStore.dispatchBehindBytes());\n        GetMessageResult getMessageResult = messageStore.getMessage(\"group\", topic, 0, 0, 32, null);\n        Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());\n\n        Assert.assertEquals(10, getMessageResult.getMessageBufferList().size());\n        Assert.assertEquals(10, getMessageResult.getMessageMapedList().size());\n\n        for (int i = 0; i < results.size(); i++) {\n            ByteBuffer buffer = getMessageResult.getMessageBufferList().get(i);\n            MessageExt messageExt = MessageDecoder.decode(buffer);\n            Assert.assertEquals(i, messageExt.getQueueOffset());\n            Assert.assertEquals(results.get(i).getAppendMessageResult().getMsgId(), messageExt.getMsgId());\n            Assert.assertEquals(results.get(i).getAppendMessageResult().getWroteOffset(), messageExt.getCommitLogOffset());\n        }\n        messageStore.destroy();\n        messageStore.shutdown();\n    }\n\n    @Test\n    public void testBatchPutAndGetMessage() throws Exception {\n        String base = createBaseDir();\n        String peers = String.format(\"n0-localhost:%d\", nextPort());\n        String group = UUID.randomUUID().toString();\n        DefaultMessageStore messageStore = createDledgerMessageStore(base, group, \"n0\", peers, null, false, 0);\n        DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog();\n        Boolean success = await().atMost(Duration.ofSeconds(4)).until(() -> dLedgerCommitLog.getdLedgerServer().getMemberState().isLeader(), item -> item);\n        Assert.assertTrue(success);\n        String topic = UUID.randomUUID().toString();\n        // should be less than 4\n        int batchMessageSize = 2;\n        int repeat = 10;\n        List<PutMessageResult> results = new ArrayList<>();\n        for (int i = 0; i < repeat; i++) {\n            MessageExtBatch messageExtBatch =\n                i < repeat / 10 ? buildBatchMessage(batchMessageSize) : buildIPv6HostBatchMessage(batchMessageSize);\n            messageExtBatch.setTopic(topic);\n            messageExtBatch.setQueueId(0);\n            PutMessageResult putMessageResult = messageStore.putMessages(messageExtBatch);\n            results.add(putMessageResult);\n            Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n            Assert.assertEquals(i * batchMessageSize, putMessageResult.getAppendMessageResult().getLogicsOffset());\n        }\n        await().atMost(Duration.ofSeconds(10)).until(() -> repeat * batchMessageSize == messageStore.getMaxOffsetInQueue(topic, 0));\n        Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0));\n        Assert.assertEquals(repeat * batchMessageSize, messageStore.getMaxOffsetInQueue(topic, 0));\n        Assert.assertEquals(0, messageStore.dispatchBehindBytes());\n        GetMessageResult getMessageResult = messageStore.getMessage(\"group\", topic, 0, 0, 100, null);\n        Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());\n\n        Assert.assertEquals(repeat * batchMessageSize > 32 ? 32 : repeat * batchMessageSize, getMessageResult.getMessageBufferList().size());\n        Assert.assertEquals(repeat * batchMessageSize > 32 ? 32 : repeat * batchMessageSize, getMessageResult.getMessageMapedList().size());\n        Assert.assertEquals(repeat * batchMessageSize, getMessageResult.getMaxOffset());\n\n        for (int i = 0; i < results.size(); i++) {\n            ByteBuffer buffer = getMessageResult.getMessageBufferList().get(i * batchMessageSize);\n            MessageExt messageExt = MessageDecoder.decode(buffer);\n            Assert.assertEquals(i * batchMessageSize, messageExt.getQueueOffset());\n            Assert.assertEquals(results.get(i).getAppendMessageResult().getMsgId().split(\",\").length, batchMessageSize);\n            Assert.assertEquals(results.get(i).getAppendMessageResult().getWroteOffset(), messageExt.getCommitLogOffset());\n        }\n        messageStore.destroy();\n        messageStore.shutdown();\n    }\n\n    @Test\n    public void testAsyncPutAndGetMessage() throws Exception {\n        Assume.assumeFalse(MixAll.isWindows());\n        String base = createBaseDir();\n        String peers = String.format(\"n0-localhost:%d\", nextPort());\n        String group = UUID.randomUUID().toString();\n        DefaultMessageStore messageStore = createDledgerMessageStore(base, group, \"n0\", peers, null, false, 0);\n        DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog();\n        Boolean success = await().atMost(Duration.ofSeconds(4)).until(() -> dLedgerCommitLog.getdLedgerServer().getMemberState().isLeader(), item -> item);\n        Assert.assertTrue(success);\n        String topic = UUID.randomUUID().toString();\n\n        List<PutMessageResult> results = new ArrayList<>();\n        for (int i = 0; i < 10; i++) {\n            MessageExtBrokerInner msgInner =\n                i < 5 ? buildMessage() : buildIPv6HostMessage();\n            msgInner.setTopic(topic);\n            msgInner.setQueueId(0);\n            CompletableFuture<PutMessageResult> futureResult = messageStore.asyncPutMessage(msgInner);\n            PutMessageResult putMessageResult = futureResult.get(3000, TimeUnit.MILLISECONDS);\n            results.add(putMessageResult);\n            Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n            Assert.assertEquals(i, putMessageResult.getAppendMessageResult().getLogicsOffset());\n        }\n        await().atMost(Duration.ofSeconds(10)).until(() -> 10 == messageStore.getMaxOffsetInQueue(topic, 0));\n        Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0));\n        Assert.assertEquals(10, messageStore.getMaxOffsetInQueue(topic, 0));\n        Assert.assertEquals(0, messageStore.dispatchBehindBytes());\n        GetMessageResult getMessageResult = messageStore.getMessage(\"group\", topic, 0, 0, 32, null);\n        Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());\n\n        Assert.assertEquals(10, getMessageResult.getMessageBufferList().size());\n        Assert.assertEquals(10, getMessageResult.getMessageMapedList().size());\n\n        for (int i = 0; i < results.size(); i++) {\n            ByteBuffer buffer = getMessageResult.getMessageBufferList().get(i);\n            MessageExt messageExt = MessageDecoder.decode(buffer);\n            Assert.assertEquals(i, messageExt.getQueueOffset());\n            Assert.assertEquals(results.get(i).getAppendMessageResult().getMsgId(), messageExt.getMsgId());\n            Assert.assertEquals(results.get(i).getAppendMessageResult().getWroteOffset(), messageExt.getCommitLogOffset());\n        }\n        messageStore.destroy();\n        messageStore.shutdown();\n    }\n\n    @Test\n    public void testAsyncBatchPutAndGetMessage() throws Exception {\n        String base = createBaseDir();\n        String peers = String.format(\"n0-localhost:%d\", nextPort());\n        String group = UUID.randomUUID().toString();\n        DefaultMessageStore messageStore = createDledgerMessageStore(base, group, \"n0\", peers, null, false, 0);\n        DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog();\n        Boolean success = await().atMost(Duration.ofSeconds(4)).until(() -> dLedgerCommitLog.getdLedgerServer().getMemberState().isLeader(), item -> item);\n        Assert.assertTrue(success);\n        String topic = UUID.randomUUID().toString();\n        // should be less than 4\n        int batchMessageSize = 2;\n        int repeat = 10;\n\n        List<PutMessageResult> results = new ArrayList<>();\n        for (int i = 0; i < repeat; i++) {\n            MessageExtBatch messageExtBatch =\n                i < 5 ? buildBatchMessage(batchMessageSize) : buildIPv6HostBatchMessage(batchMessageSize);\n            messageExtBatch.setTopic(topic);\n            messageExtBatch.setQueueId(0);\n            CompletableFuture<PutMessageResult> futureResult = messageStore.asyncPutMessages(messageExtBatch);\n            PutMessageResult putMessageResult = futureResult.get(3000, TimeUnit.MILLISECONDS);\n            results.add(putMessageResult);\n            Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n            Assert.assertEquals(i * batchMessageSize, putMessageResult.getAppendMessageResult().getLogicsOffset());\n        }\n        await().atMost(Duration.ofSeconds(10)).until(() -> repeat * batchMessageSize == messageStore.getMaxOffsetInQueue(topic, 0));\n        Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0));\n        Assert.assertEquals(repeat * batchMessageSize, messageStore.getMaxOffsetInQueue(topic, 0));\n        Assert.assertEquals(0, messageStore.dispatchBehindBytes());\n        GetMessageResult getMessageResult = messageStore.getMessage(\"group\", topic, 0, 0, 32, null);\n        Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());\n\n        Assert.assertEquals(repeat * batchMessageSize > 32 ? 32 : repeat * batchMessageSize, getMessageResult.getMessageBufferList().size());\n        Assert.assertEquals(repeat * batchMessageSize > 32 ? 32 : repeat * batchMessageSize, getMessageResult.getMessageMapedList().size());\n        Assert.assertEquals(repeat * batchMessageSize, getMessageResult.getMaxOffset());\n\n        for (int i = 0; i < results.size(); i++) {\n            ByteBuffer buffer = getMessageResult.getMessageBufferList().get(i * batchMessageSize);\n            MessageExt messageExt = MessageDecoder.decode(buffer);\n            Assert.assertEquals(i * batchMessageSize, messageExt.getQueueOffset());\n            Assert.assertEquals(results.get(i).getAppendMessageResult().getMsgId().split(\",\").length, batchMessageSize);\n            Assert.assertEquals(results.get(i).getAppendMessageResult().getWroteOffset(), messageExt.getCommitLogOffset());\n        }\n        messageStore.destroy();\n        messageStore.shutdown();\n    }\n\n    @Test\n    public void testCommittedPos() throws Exception {\n        String peers = String.format(\"n0-localhost:%d;n1-localhost:%d\", nextPort(), nextPort());\n        String group = UUID.randomUUID().toString();\n        DefaultMessageStore leaderStore = createDledgerMessageStore(createBaseDir(), group, \"n0\", peers, \"n0\", false, 0);\n\n        String topic = UUID.randomUUID().toString();\n        MessageExtBrokerInner msgInner = buildMessage();\n        msgInner.setTopic(topic);\n        msgInner.setQueueId(0);\n        PutMessageResult putMessageResult = leaderStore.putMessage(msgInner);\n        Assert.assertEquals(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH, putMessageResult.getPutMessageStatus());\n\n        Assert.assertEquals(0, leaderStore.getCommitLog().getMaxOffset());\n        Assert.assertEquals(0, leaderStore.getMaxOffsetInQueue(topic, 0));\n\n        DefaultMessageStore followerStore = createDledgerMessageStore(createBaseDir(), group, \"n1\", peers, \"n0\", false, 0);\n        await().atMost(10, SECONDS).until(followerCatchesUp(followerStore, topic));\n\n        Assert.assertEquals(1, leaderStore.getMaxOffsetInQueue(topic, 0));\n        Assert.assertTrue(leaderStore.getCommitLog().getMaxOffset() > 0);\n\n        leaderStore.destroy();\n        followerStore.destroy();\n\n        leaderStore.shutdown();\n        followerStore.shutdown();\n    }\n\n    @Test\n    public void testIPv6HostMsgCommittedPos() throws Exception {\n        String peers = String.format(\"n0-localhost:%d;n1-localhost:%d\", nextPort(), nextPort());\n        String group = UUID.randomUUID().toString();\n        DefaultMessageStore leaderStore = createDledgerMessageStore(createBaseDir(), group, \"n0\", peers, \"n0\", false, 0);\n\n        String topic = UUID.randomUUID().toString();\n        MessageExtBrokerInner msgInner = buildIPv6HostMessage();\n        msgInner.setTopic(topic);\n        msgInner.setQueueId(0);\n        PutMessageResult putMessageResult = leaderStore.putMessage(msgInner);\n        Assert.assertEquals(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH, putMessageResult.getPutMessageStatus());\n\n        //Thread.sleep(1000);\n\n        Assert.assertEquals(0, leaderStore.getCommitLog().getMaxOffset());\n        Assert.assertEquals(0, leaderStore.getMaxOffsetInQueue(topic, 0));\n\n        DefaultMessageStore followerStore = createDledgerMessageStore(createBaseDir(), group, \"n1\", peers, \"n0\", false, 0);\n        await().atMost(10, SECONDS).until(followerCatchesUp(followerStore, topic));\n\n        Assert.assertEquals(1, leaderStore.getMaxOffsetInQueue(topic, 0));\n        Assert.assertTrue(leaderStore.getCommitLog().getMaxOffset() > 0);\n\n        leaderStore.destroy();\n        followerStore.destroy();\n\n        leaderStore.shutdown();\n        followerStore.shutdown();\n    }\n\n    private Callable<Boolean> followerCatchesUp(DefaultMessageStore followerStore, String topic) {\n        return () -> followerStore.getMaxOffsetInQueue(topic, 0) == 1;\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerMultiPathTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.dledger;\n\nimport java.io.File;\nimport java.time.Duration;\nimport java.util.Objects;\nimport java.util.UUID;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.config.FlushDiskType;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.junit.Assert;\nimport org.junit.Test;\nimport org.junit.Assume;\n\nimport static org.awaitility.Awaitility.await;\n\npublic class DLedgerMultiPathTest extends MessageStoreTestBase {\n\n\n    @Test\n    public void multiDirsStorageTest() throws Exception {\n        Assume.assumeFalse(MixAll.isMac());\n        Assume.assumeFalse(MixAll.isWindows());\n        String base = createBaseDir();\n        String topic = UUID.randomUUID().toString();\n        String peers = String.format(\"n0-localhost:%d\", nextPort());\n        String group = UUID.randomUUID().toString();\n        String multiStorePath =\n            base + \"/multi/a/\" + MessageStoreConfig.MULTI_PATH_SPLITTER +\n                base + \"/multi/b/\" + MessageStoreConfig.MULTI_PATH_SPLITTER +\n                base + \"/multi/c/\" + MessageStoreConfig.MULTI_PATH_SPLITTER;\n        {\n\n            DefaultMessageStore dLedgerStore = createDLedgerMessageStore(base, group, \"n0\", peers, multiStorePath, null);\n            DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) dLedgerStore.getCommitLog();\n            Boolean success = await().atMost(Duration.ofSeconds(4)).until(() -> dLedgerCommitLog.getdLedgerServer().getMemberState().isLeader(), item -> item);\n            Assert.assertTrue(success);\n            doPutMessages(dLedgerStore, topic, 0, 1000, 0);\n            await().atMost(Duration.ofSeconds(10)).until(() -> 1000 == dLedgerStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(11, dLedgerStore.getMaxPhyOffset() / dLedgerStore.getMessageStoreConfig().getMappedFileSizeCommitLog());\n            Assert.assertEquals(0, dLedgerStore.getMinOffsetInQueue(topic, 0));\n            Assert.assertEquals(1000, dLedgerStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, dLedgerStore.dispatchBehindBytes());\n            doGetMessages(dLedgerStore, topic, 0, 1000, 0);\n            dLedgerStore.shutdown();\n        }\n        {\n            String readOnlyPath =\n                base + \"/multi/a/\" + MessageStoreConfig.MULTI_PATH_SPLITTER +\n                    base + \"/multi/b/\" + MessageStoreConfig.MULTI_PATH_SPLITTER;\n            multiStorePath =\n                base + \"/multi/c/\" + MessageStoreConfig.MULTI_PATH_SPLITTER +\n                    base + \"/multi/d/\" + MessageStoreConfig.MULTI_PATH_SPLITTER;\n\n            DefaultMessageStore dLedgerStore = createDLedgerMessageStore(base, group, \"n0\", peers, multiStorePath, readOnlyPath);\n            DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) dLedgerStore.getCommitLog();\n            Boolean success = await().atMost(Duration.ofSeconds(4)).until(() -> dLedgerCommitLog.getdLedgerServer().getMemberState().isLeader(), item -> item);\n            Assert.assertTrue(success);\n            doGetMessages(dLedgerStore, topic, 0, 1000, 0);\n            long beforeSize = Objects.requireNonNull(new File(base + \"/multi/a/\").listFiles()).length;\n            doPutMessages(dLedgerStore, topic, 0, 1000, 1000);\n            await().atMost(Duration.ofSeconds(10)).until(() -> 2000 == dLedgerStore.getMaxOffsetInQueue(topic, 0));\n            long afterSize = Objects.requireNonNull(new File(base + \"/multi/a/\").listFiles()).length;\n            Assert.assertEquals(beforeSize, afterSize);\n            Assert.assertEquals(0, dLedgerStore.getMinOffsetInQueue(topic, 0));\n            Assert.assertEquals(2000, dLedgerStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, dLedgerStore.dispatchBehindBytes());\n\n            dLedgerStore.shutdown();\n        }\n\n    }\n\n    protected DefaultMessageStore createDLedgerMessageStore(String base, String group, String selfId, String peers,\n        String dLedgerCommitLogPath, String readOnlyPath) throws Exception {\n        MessageStoreConfig storeConfig = new MessageStoreConfig();\n        storeConfig.setMappedFileSizeCommitLog(1024 * 100);\n        storeConfig.setMappedFileSizeConsumeQueue(1024);\n        storeConfig.setMaxHashSlotNum(100);\n        storeConfig.setMaxIndexNum(100 * 10);\n        storeConfig.setStorePathRootDir(base);\n        storeConfig.setStorePathDLedgerCommitLog(dLedgerCommitLogPath);\n        storeConfig.setReadOnlyCommitLogStorePaths(readOnlyPath);\n        storeConfig.setFlushDiskType(FlushDiskType.ASYNC_FLUSH);\n\n        storeConfig.setEnableDLegerCommitLog(true);\n        storeConfig.setdLegerGroup(group);\n        storeConfig.setdLegerPeers(peers);\n        storeConfig.setdLegerSelfId(selfId);\n        DefaultMessageStore defaultMessageStore = new DefaultMessageStore(storeConfig, new BrokerStatsManager(\"DLedgerCommitLogTest\", true), (topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties) -> {\n\n        }, new BrokerConfig(), new ConcurrentHashMap<>());\n        Assert.assertTrue(defaultMessageStore.load());\n        defaultMessageStore.start();\n        return defaultMessageStore;\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.dledger;\n\nimport com.google.common.util.concurrent.RateLimiter;\nimport io.openmessaging.storage.dledger.DLedgerConfig;\nimport io.openmessaging.storage.dledger.DLedgerServer;\nimport java.io.File;\nimport java.net.UnknownHostException;\nimport java.util.Arrays;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.StoreTestBase;\nimport org.apache.rocketmq.store.config.FlushDiskType;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.config.StorePathConfigHelper;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.junit.Assert;\n\npublic class MessageStoreTestBase extends StoreTestBase {\n\n    protected DefaultMessageStore createDledgerMessageStore(String base, String group, String selfId, String peers, String leaderId, boolean createAbort, int deleteFileNum) throws Exception {\n        System.setProperty(\"dledger.disk.ratio.check\", \"0.95\");\n        System.setProperty(\"dledger.disk.ratio.clean\", \"0.95\");\n        baseDirs.add(base);\n        MessageStoreConfig storeConfig = new MessageStoreConfig();\n        storeConfig.setMappedFileSizeCommitLog(1024 * 100);\n        storeConfig.setMappedFileSizeConsumeQueue(1024);\n        storeConfig.setMaxHashSlotNum(100);\n        storeConfig.setMaxIndexNum(100 * 10);\n        storeConfig.setStorePathRootDir(base);\n        storeConfig.setStorePathCommitLog(base + File.separator + \"commitlog\");\n        storeConfig.setFlushDiskType(FlushDiskType.ASYNC_FLUSH);\n\n        storeConfig.setEnableDLegerCommitLog(true);\n        storeConfig.setdLegerGroup(group);\n        storeConfig.setdLegerPeers(peers);\n        storeConfig.setdLegerSelfId(selfId);\n\n        storeConfig.setRecheckReputOffsetFromCq(true);\n        DefaultMessageStore defaultMessageStore = new DefaultMessageStore(storeConfig,  new BrokerStatsManager(\"DLedgerCommitlogTest\", true), (topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties) -> {\n\n        }, new BrokerConfig(), new ConcurrentHashMap<>());\n        DLedgerServer dLegerServer = ((DLedgerCommitLog) defaultMessageStore.getCommitLog()).getdLedgerServer();\n        if (leaderId != null) {\n            dLegerServer.getdLedgerConfig().setEnableLeaderElector(false);\n            if (selfId.equals(leaderId)) {\n                dLegerServer.getMemberState().changeToLeader(0);\n            } else {\n                dLegerServer.getMemberState().changeToFollower(0, leaderId);\n            }\n        }\n        if (createAbort) {\n            String fileName = StorePathConfigHelper.getAbortFile(storeConfig.getStorePathRootDir());\n            makeSureFileExists(fileName);\n        }\n        if (deleteFileNum > 0) {\n            DLedgerConfig config = dLegerServer.getdLedgerConfig();\n            if (deleteFileNum > 0) {\n                File dir = new File(config.getDataStorePath());\n                File[] files = dir.listFiles();\n                if (files != null) {\n                    Arrays.sort(files);\n                    for (int i = files.length - 1; i >= 0; i--) {\n                        File file = files[i];\n                        file.delete();\n                        if (files.length - i >= deleteFileNum) {\n                            break;\n                        }\n                    }\n                }\n            }\n        }\n        Assert.assertTrue(defaultMessageStore.load());\n        defaultMessageStore.start();\n        return defaultMessageStore;\n    }\n\n\n    protected DefaultMessageStore createMessageStore(String base, boolean createAbort) throws Exception {\n        baseDirs.add(base);\n        MessageStoreConfig storeConfig = new MessageStoreConfig();\n        storeConfig.setMappedFileSizeCommitLog(1024 * 100);\n        storeConfig.setMappedFileSizeConsumeQueue(1024);\n        storeConfig.setMaxHashSlotNum(100);\n        storeConfig.setMaxIndexNum(100 * 10);\n        storeConfig.setStorePathRootDir(base);\n        storeConfig.setStorePathCommitLog(base + File.separator + \"commitlog\");\n        storeConfig.setFlushDiskType(FlushDiskType.ASYNC_FLUSH);\n        DefaultMessageStore defaultMessageStore = new DefaultMessageStore(storeConfig,  new BrokerStatsManager(\"CommitlogTest\", true), (topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties) -> {\n\n        }, new BrokerConfig(), new ConcurrentHashMap<>());\n\n        if (createAbort) {\n            String fileName = StorePathConfigHelper.getAbortFile(storeConfig.getStorePathRootDir());\n            makeSureFileExists(fileName);\n        }\n        Assert.assertTrue(defaultMessageStore.load());\n        defaultMessageStore.start();\n        return defaultMessageStore;\n    }\n\n    protected void doPutMessages(MessageStore messageStore, String topic, int queueId, int num, long beginLogicsOffset) throws UnknownHostException {\n        RateLimiter rateLimiter = RateLimiter.create(100);\n        MessageStoreConfig storeConfig = messageStore.getMessageStoreConfig();\n        boolean limitAppendRate = storeConfig.isEnableDLegerCommitLog();\n        for (int i = 0; i < num; i++) {\n            if (limitAppendRate) {\n                rateLimiter.acquire();\n            }\n            MessageExtBrokerInner msgInner = buildMessage();\n            msgInner.setTopic(topic);\n            msgInner.setQueueId(queueId);\n            PutMessageResult putMessageResult = messageStore.putMessage(msgInner);\n            Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n            Assert.assertEquals(beginLogicsOffset + i, putMessageResult.getAppendMessageResult().getLogicsOffset());\n        }\n    }\n\n    protected void doGetMessages(MessageStore messageStore, String topic, int queueId, int num, long beginLogicsOffset) {\n        for (int i = 0; i < num; i++) {\n            GetMessageResult getMessageResult =  messageStore.getMessage(\"group\", topic, queueId, beginLogicsOffset + i, 3, null);\n            Assert.assertNotNull(getMessageResult);\n            Assert.assertTrue(!getMessageResult.getMessageBufferList().isEmpty());\n            MessageExt messageExt = MessageDecoder.decode(getMessageResult.getMessageBufferList().get(0));\n            Assert.assertEquals(beginLogicsOffset + i, messageExt.getQueueOffset());\n            getMessageResult.release();\n        }\n    }\n\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.dledger;\n\nimport java.time.Duration;\nimport java.util.UUID;\n\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.StoreTestBase;\nimport org.apache.rocketmq.store.config.StorePathConfigHelper;\nimport org.junit.Assert;\nimport org.junit.Assume;\nimport org.junit.Ignore;\nimport org.junit.Test;\n\nimport static org.awaitility.Awaitility.await;\n\npublic class MixCommitlogTest extends MessageStoreTestBase {\n\n    @Ignore\n    @Test\n    public void testFallBehindCQ() throws Exception {\n        Assume.assumeFalse(MixAll.isWindows());\n        Assume.assumeFalse(MixAll.isMac());\n        String base = createBaseDir();\n        String topic = UUID.randomUUID().toString();\n        String peers = String.format(\"n0-localhost:%d\", nextPort());\n        String group = UUID.randomUUID().toString();\n        {\n            DefaultMessageStore originalStore = createMessageStore(base, false);\n            doPutMessages(originalStore, topic, 0, 1000, 0);\n            await().atMost(Duration.ofSeconds(10)).until(() -> 1000 == originalStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(11, originalStore.getMaxPhyOffset() / originalStore.getMessageStoreConfig().getMappedFileSizeCommitLog());\n            Assert.assertEquals(0, originalStore.getMinOffsetInQueue(topic, 0));\n            Assert.assertEquals(1000, originalStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, originalStore.dispatchBehindBytes());\n            doGetMessages(originalStore, topic, 0, 1000, 0);\n            originalStore.shutdown();\n        }\n        //delete the cq files\n        {\n            StoreTestBase.deleteFile(StorePathConfigHelper.getStorePathConsumeQueue(base));\n        }\n        {\n            DefaultMessageStore dledgerStore = createDledgerMessageStore(base, group, \"n0\", peers, null, true, 0);\n            DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) dledgerStore.getCommitLog();\n            Boolean success = await().atMost(Duration.ofSeconds(4)).until(() -> dLedgerCommitLog.getdLedgerServer().getMemberState().isLeader(), item -> item);\n            Assert.assertTrue(success);\n            await().atMost(Duration.ofSeconds(10)).until(() -> 1000 == dledgerStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, dledgerStore.getMinOffsetInQueue(topic, 0));\n            Assert.assertEquals(1000, dledgerStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, dledgerStore.dispatchBehindBytes());\n            doGetMessages(dledgerStore, topic, 0, 1000, 0);\n            doPutMessages(dledgerStore, topic, 0, 1000, 1000);\n            await().atMost(Duration.ofSeconds(10)).until(() -> 2000 == dledgerStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, dledgerStore.getMinOffsetInQueue(topic, 0));\n            Assert.assertEquals(2000, dledgerStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, dledgerStore.dispatchBehindBytes());\n            doGetMessages(dledgerStore, topic, 0, 2000, 0);\n            dledgerStore.shutdown();\n        }\n    }\n\n    @Test\n    public void testPutAndGet() throws Exception {\n        Assume.assumeFalse(MixAll.isMac());\n        String base = createBaseDir();\n        String topic = UUID.randomUUID().toString();\n        String peers = String.format(\"n0-localhost:%d\", nextPort());\n        String group = UUID.randomUUID().toString();\n\n        long dividedOffset;\n        {\n            DefaultMessageStore originalStore = createMessageStore(base, false);\n            doPutMessages(originalStore, topic, 0, 1000, 0);\n            await().atMost(Duration.ofSeconds(10)).until(() -> 1000 == originalStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, originalStore.getMinOffsetInQueue(topic, 0));\n            Assert.assertEquals(1000, originalStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, originalStore.dispatchBehindBytes());\n            dividedOffset = originalStore.getCommitLog().getMaxOffset();\n            dividedOffset = dividedOffset - dividedOffset % originalStore.getMessageStoreConfig().getMappedFileSizeCommitLog() + originalStore.getMessageStoreConfig().getMappedFileSizeCommitLog();\n            doGetMessages(originalStore, topic, 0, 1000, 0);\n            originalStore.shutdown();\n        }\n        {\n            DefaultMessageStore recoverOriginalStore = createMessageStore(base, true);\n            await().atMost(Duration.ofSeconds(10)).until(() -> 1000 == recoverOriginalStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, recoverOriginalStore.getMinOffsetInQueue(topic, 0));\n            Assert.assertEquals(1000, recoverOriginalStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, recoverOriginalStore.dispatchBehindBytes());\n            doGetMessages(recoverOriginalStore, topic, 0, 1000, 0);\n            recoverOriginalStore.shutdown();\n        }\n        {\n            DefaultMessageStore dledgerStore = createDledgerMessageStore(base, group, \"n0\", peers, null, true, 0);\n            DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) dledgerStore.getCommitLog();\n            Assert.assertFalse(dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().isEnableDiskForceClean());\n            Assert.assertEquals(dividedOffset, dLedgerCommitLog.getDividedCommitlogOffset());\n            Assert.assertEquals(0, dledgerStore.dispatchBehindBytes());\n            Assert.assertEquals(dividedOffset, dLedgerCommitLog.getMaxOffset());\n            Boolean success = await().atMost(Duration.ofSeconds(4)).until(() -> dLedgerCommitLog.getdLedgerServer().getMemberState().isLeader(), item -> item);\n            Assert.assertTrue(success);\n            doPutMessages(dledgerStore, topic, 0, 1000, 1000);\n            await().atMost(Duration.ofSeconds(10)).until(() -> 2000 == dledgerStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, dledgerStore.getMinOffsetInQueue(topic, 0));\n            Assert.assertEquals(2000, dledgerStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, dledgerStore.dispatchBehindBytes());\n            doGetMessages(dledgerStore, topic, 0, 2000, 0);\n            dledgerStore.shutdown();\n        }\n        {\n            DefaultMessageStore recoverDledgerStore = createDledgerMessageStore(base, group, \"n0\", peers, null, true, 0);\n            DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) recoverDledgerStore.getCommitLog();\n            Assert.assertFalse(dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().isEnableDiskForceClean());\n            Assert.assertEquals(dividedOffset, dLedgerCommitLog.getDividedCommitlogOffset());\n            Boolean success = await().atMost(Duration.ofSeconds(4)).until(() -> dLedgerCommitLog.getdLedgerServer().getMemberState().isLeader(), item -> item);\n            Assert.assertTrue(success);\n            doPutMessages(recoverDledgerStore, topic, 0, 1000, 2000);\n            await().atMost(Duration.ofSeconds(10)).until(() -> 3000 == recoverDledgerStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, recoverDledgerStore.getMinOffsetInQueue(topic, 0));\n            Assert.assertEquals(3000, recoverDledgerStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, recoverDledgerStore.dispatchBehindBytes());\n            doGetMessages(recoverDledgerStore, topic, 0, 3000, 0);\n            recoverDledgerStore.shutdown();\n        }\n    }\n\n    @Test\n    public void testDeleteExpiredFiles() throws Exception {\n        Assume.assumeFalse(MixAll.isMac());\n        String base = createBaseDir();\n        String topic = UUID.randomUUID().toString();\n        String peers = String.format(\"n0-localhost:%d\", nextPort());\n        String group = UUID.randomUUID().toString();\n\n        long dividedOffset;\n        {\n            DefaultMessageStore originalStore = createMessageStore(base, false);\n            doPutMessages(originalStore, topic, 0, 1000, 0);\n            await().atMost(Duration.ofSeconds(10)).until(() -> 1000 == originalStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, originalStore.getMinOffsetInQueue(topic, 0));\n            Assert.assertEquals(1000, originalStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, originalStore.dispatchBehindBytes());\n            dividedOffset = originalStore.getCommitLog().getMaxOffset();\n            dividedOffset = dividedOffset - dividedOffset % originalStore.getMessageStoreConfig().getMappedFileSizeCommitLog() + originalStore.getMessageStoreConfig().getMappedFileSizeCommitLog();\n            originalStore.shutdown();\n        }\n        long maxPhysicalOffset;\n        {\n            DefaultMessageStore dledgerStore = createDledgerMessageStore(base, group, \"n0\", peers, null, true, 0);\n            DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) dledgerStore.getCommitLog();\n            Assert.assertTrue(dledgerStore.getMessageStoreConfig().isCleanFileForciblyEnable());\n            Assert.assertFalse(dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().isEnableDiskForceClean());\n            Assert.assertEquals(dividedOffset, dLedgerCommitLog.getDividedCommitlogOffset());\n            Boolean success = await().atMost(Duration.ofSeconds(4)).until(() -> dLedgerCommitLog.getdLedgerServer().getMemberState().isLeader(), item -> item);\n            Assert.assertTrue(success);\n            doPutMessages(dledgerStore, topic, 0, 1000, 1000);\n            await().atMost(Duration.ofSeconds(10)).until(() -> 2000 == dledgerStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, dledgerStore.getMinOffsetInQueue(topic, 0));\n            Assert.assertEquals(2000, dledgerStore.getMaxOffsetInQueue(topic, 0));\n            Assert.assertEquals(0, dledgerStore.dispatchBehindBytes());\n            Assert.assertEquals(0, dledgerStore.getMinPhyOffset());\n            maxPhysicalOffset = dledgerStore.getMaxPhyOffset();\n            Assert.assertTrue(maxPhysicalOffset > 0);\n\n            doGetMessages(dledgerStore, topic, 0, 2000, 0);\n\n            for (int i = 0; i < 100; i++) {\n                dledgerStore.getCommitLog().deleteExpiredFile(System.currentTimeMillis(), 0, 0, true);\n            }\n            Assert.assertEquals(dividedOffset, dledgerStore.getMinPhyOffset());\n            Assert.assertEquals(maxPhysicalOffset, dledgerStore.getMaxPhyOffset());\n            for (int i = 0; i < 100; i++) {\n                Assert.assertEquals(Integer.MAX_VALUE, dledgerStore.getCommitLog().deleteExpiredFile(System.currentTimeMillis(), 0, 0, true));\n            }\n            Assert.assertEquals(dividedOffset, dledgerStore.getMinPhyOffset());\n            Assert.assertEquals(maxPhysicalOffset, dledgerStore.getMaxPhyOffset());\n\n            Assert.assertTrue(dledgerStore.getMessageStoreConfig().isCleanFileForciblyEnable());\n            Assert.assertTrue(dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().isEnableDiskForceClean());\n\n            //Test fresh\n            dledgerStore.getMessageStoreConfig().setCleanFileForciblyEnable(false);\n            for (int i = 0; i < 100; i++) {\n                Assert.assertEquals(Integer.MAX_VALUE, dledgerStore.getCommitLog().deleteExpiredFile(System.currentTimeMillis(), 0, 0, true));\n            }\n            Assert.assertFalse(dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().isEnableDiskForceClean());\n            doGetMessages(dledgerStore, topic, 0, 1000, 1000);\n            dledgerStore.shutdown();\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/ha/FlowMonitorTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha;\n\nimport java.time.Duration;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport static org.awaitility.Awaitility.await;\n\npublic class FlowMonitorTest {\n\n    @Test\n    public void testLimit() throws Exception {\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setHaFlowControlEnable(true);\n        messageStoreConfig.setMaxHaTransferByteInSecond(10);\n\n        FlowMonitor flowMonitor = new FlowMonitor(messageStoreConfig);\n        flowMonitor.start();\n\n        flowMonitor.addByteCountTransferred(3);\n        Boolean flag = await().atMost(Duration.ofSeconds(2)).until(() -> 7 == flowMonitor.canTransferMaxByteNum(), item -> item);\n        flag &= await().atMost(Duration.ofSeconds(2)).until(() -> 10 == flowMonitor.canTransferMaxByteNum(), item -> item);\n        Assert.assertTrue(flag);\n\n        flowMonitor.shutdown();\n    }\n\n    @Test\n    public void testSpeed() throws Exception {\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setHaFlowControlEnable(true);\n        messageStoreConfig.setMaxHaTransferByteInSecond(10);\n\n        FlowMonitor flowMonitor = new FlowMonitor(messageStoreConfig);\n\n        flowMonitor.addByteCountTransferred(3);\n        flowMonitor.calculateSpeed();\n        Assert.assertEquals(3, flowMonitor.getTransferredByteInSecond());\n\n        flowMonitor.addByteCountTransferred(5);\n        flowMonitor.calculateSpeed();\n        Assert.assertEquals(5, flowMonitor.getTransferredByteInSecond());\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/ha/HAClientTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha;\n\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class HAClientTest {\n    private HAClient haClient;\n\n    @Mock\n    private DefaultMessageStore messageStore;\n\n    @Before\n    public void setUp() throws Exception {\n//        when(messageStore.getMessageStoreConfig()).thenReturn(new MessageStoreConfig());\n        when(messageStore.getBrokerConfig()).thenReturn(new BrokerConfig());\n        this.haClient = new DefaultHAClient(this.messageStore);\n    }\n\n    @After\n    public void tearDown() throws Exception {\n        this.haClient.shutdown();\n    }\n\n    @Test\n    public void updateMasterAddress() {\n        assertThat(this.haClient.getMasterAddress()).isNull();\n        this.haClient.updateMasterAddress(\"127.0.0.1:10911\");\n        assertThat(this.haClient.getMasterAddress()).isEqualTo(\"127.0.0.1:10911\");\n\n        this.haClient.updateMasterAddress(\"127.0.0.1:10912\");\n        assertThat(this.haClient.getMasterAddress()).isEqualTo(\"127.0.0.1:10912\");\n    }\n\n    @Test\n    public void updateHaMasterAddress() {\n        assertThat(this.haClient.getHaMasterAddress()).isNull();\n        this.haClient.updateHaMasterAddress(\"127.0.0.1:10911\");\n        assertThat(this.haClient.getHaMasterAddress()).isEqualTo(\"127.0.0.1:10911\");\n\n        this.haClient.updateHaMasterAddress(\"127.0.0.1:10912\");\n        assertThat(this.haClient.getHaMasterAddress()).isEqualTo(\"127.0.0.1:10912\");\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/ha/HAServerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha;\n\nimport java.io.IOException;\nimport java.time.Duration;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Random;\nimport java.util.concurrent.Callable;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.TimeoutException;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.SystemClock;\nimport org.apache.rocketmq.store.CommitLog;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.junit.MockitoJUnitRunner;\nimport org.rocksdb.RocksDBException;\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.awaitility.Awaitility.await;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.Mockito.doAnswer;\nimport static org.mockito.Mockito.doReturn;\nimport static org.mockito.Mockito.mock;\n\n@RunWith(MockitoJUnitRunner.Silent.class)\npublic class HAServerTest {\n    private DefaultMessageStore defaultMessageStore;\n    private MessageStoreConfig storeConfig;\n    private HAService haService;\n    private Random random = new Random();\n    private List<HAClient> haClientList = new ArrayList<>();\n\n    @Before\n    public void setUp() throws Exception {\n        this.storeConfig = new MessageStoreConfig();\n        this.storeConfig.setHaListenPort(9000 + random.nextInt(1000));\n        this.storeConfig.setHaSendHeartbeatInterval(10);\n\n        this.defaultMessageStore = mockMessageStore();\n        this.haService = new DefaultHAService();\n        this.haService.init(defaultMessageStore);\n        this.haService.start();\n    }\n\n    @After\n    public void tearDown() {\n        tearDownAllHAClient();\n\n        await().atMost(Duration.ofMinutes(1)).until(new Callable<Boolean>() {\n            @Override\n            public Boolean call() throws Exception {\n                return HAServerTest.this.haService.getConnectionCount().get() == 0;\n            }\n        });\n\n        this.haService.shutdown();\n    }\n\n    @Test\n    public void testConnectionList_OneHAClient() throws IOException {\n        setUpOneHAClient();\n\n        await().atMost(Duration.ofMinutes(1)).until(new Callable<Boolean>() {\n            @Override\n            public Boolean call() {\n                return HAServerTest.this.haService.getConnectionCount().get() == 1;\n            }\n        });\n    }\n\n    @Test\n    public void testConnectionList_MultipleHAClient() throws IOException {\n        setUpOneHAClient();\n        setUpOneHAClient();\n        setUpOneHAClient();\n\n        await().atMost(Duration.ofMinutes(1)).until(new Callable<Boolean>() {\n            @Override\n            public Boolean call() {\n                return HAServerTest.this.haService.getConnectionCount().get() == 3;\n            }\n        });\n\n        tearDownOneHAClient();\n\n        await().atMost(Duration.ofMinutes(1)).until(new Callable<Boolean>() {\n            @Override\n            public Boolean call() {\n                return HAServerTest.this.haService.getConnectionCount().get() == 2;\n            }\n        });\n    }\n\n    @Test\n    public void inSyncReplicasNums() throws IOException, RocksDBException {\n        DefaultMessageStore messageStore = mockMessageStore();\n        doReturn(123L).when(messageStore).getMaxPhyOffset();\n        doReturn(123L).when(messageStore).getMasterFlushedOffset();\n        setUpOneHAClient(messageStore);\n\n        messageStore = mockMessageStore();\n        doReturn(124L).when(messageStore).getMaxPhyOffset();\n        doReturn(124L).when(messageStore).getMasterFlushedOffset();\n        setUpOneHAClient(messageStore);\n\n        messageStore = mockMessageStore();\n        doReturn(123L).when(messageStore).getMaxPhyOffset();\n        doReturn(123L).when(messageStore).getMasterFlushedOffset();\n        setUpOneHAClient(messageStore);\n\n        messageStore = mockMessageStore();\n        doReturn(125L).when(messageStore).getMaxPhyOffset();\n        doReturn(125L).when(messageStore).getMasterFlushedOffset();\n        setUpOneHAClient(messageStore);\n\n        final int haSlaveFallbehindMax = this.defaultMessageStore.getMessageStoreConfig().getHaMaxGapNotInSync();\n\n        await().atMost(Duration.ofMinutes(1)).until(new Callable<Boolean>() {\n            @Override\n            public Boolean call() throws Exception {\n                return HAServerTest.this.haService.inSyncReplicasNums(haSlaveFallbehindMax) == 5;\n            }\n        });\n\n        assertThat(HAServerTest.this.haService.inSyncReplicasNums(123L + haSlaveFallbehindMax)).isEqualTo(3);\n        assertThat(HAServerTest.this.haService.inSyncReplicasNums(124L + haSlaveFallbehindMax)).isEqualTo(2);\n        assertThat(HAServerTest.this.haService.inSyncReplicasNums(125L + haSlaveFallbehindMax)).isEqualTo(1);\n    }\n\n    @Test\n    public void isSlaveOK() throws IOException, RocksDBException {\n        DefaultMessageStore messageStore = mockMessageStore();\n        doReturn(123L).when(messageStore).getMaxPhyOffset();\n        doReturn(123L).when(messageStore).getMasterFlushedOffset();\n        setUpOneHAClient(messageStore);\n\n        messageStore = mockMessageStore();\n        doReturn(124L).when(messageStore).getMaxPhyOffset();\n        doReturn(124L).when(messageStore).getMasterFlushedOffset();\n        setUpOneHAClient(messageStore);\n\n        final int haSlaveFallbehindMax = this.defaultMessageStore.getMessageStoreConfig().getHaMaxGapNotInSync();\n\n        await().atMost(Duration.ofMinutes(1)).until(new Callable<Boolean>() {\n            @Override\n            public Boolean call() throws Exception {\n                return HAServerTest.this.haService.isSlaveOK(haSlaveFallbehindMax + 123);\n            }\n        });\n\n        assertThat(HAServerTest.this.haService.isSlaveOK(122L + haSlaveFallbehindMax)).isTrue();\n        assertThat(HAServerTest.this.haService.isSlaveOK(124L + haSlaveFallbehindMax)).isFalse();\n    }\n\n    @Test\n    public void putRequest_SingleAck()\n        throws IOException, ExecutionException, InterruptedException, TimeoutException, RocksDBException {\n        CommitLog.GroupCommitRequest request = new CommitLog.GroupCommitRequest(124, 4000, 1);\n        this.haService.putRequest(request);\n\n        assertThat(request.future().get()).isEqualTo(PutMessageStatus.FLUSH_SLAVE_TIMEOUT);\n\n        DefaultMessageStore messageStore = mockMessageStore();\n        doReturn(124L).when(messageStore).getMaxPhyOffset();\n        doReturn(124L).when(messageStore).getMasterFlushedOffset();\n        setUpOneHAClient(messageStore);\n\n        request = new CommitLog.GroupCommitRequest(124, 4000, 1);\n        this.haService.putRequest(request);\n        assertThat(request.future().get()).isEqualTo(PutMessageStatus.PUT_OK);\n    }\n\n    @Test\n    public void putRequest_MultipleAckAndRequests()\n        throws IOException, ExecutionException, InterruptedException, RocksDBException {\n        CommitLog.GroupCommitRequest oneAck = new CommitLog.GroupCommitRequest(124, 4000, 2);\n        this.haService.putRequest(oneAck);\n\n        CommitLog.GroupCommitRequest twoAck = new CommitLog.GroupCommitRequest(124, 4000, 3);\n        this.haService.putRequest(twoAck);\n\n        DefaultMessageStore messageStore = mockMessageStore();\n        doReturn(125L).when(messageStore).getMaxPhyOffset();\n        doReturn(125L).when(messageStore).getMasterFlushedOffset();\n        setUpOneHAClient(messageStore);\n\n        assertThat(oneAck.future().get()).isEqualTo(PutMessageStatus.PUT_OK);\n        assertThat(twoAck.future().get()).isEqualTo(PutMessageStatus.FLUSH_SLAVE_TIMEOUT);\n\n        messageStore = mockMessageStore();\n        doReturn(128L).when(messageStore).getMaxPhyOffset();\n        doReturn(128L).when(messageStore).getMasterFlushedOffset();\n        setUpOneHAClient(messageStore);\n\n        twoAck = new CommitLog.GroupCommitRequest(124, 4000, 3);\n        this.haService.putRequest(twoAck);\n        assertThat(twoAck.future().get()).isEqualTo(PutMessageStatus.PUT_OK);\n    }\n\n    @Test\n    public void getPush2SlaveMaxOffset() throws IOException, RocksDBException {\n        DefaultMessageStore messageStore = mockMessageStore();\n        doReturn(123L).when(messageStore).getMaxPhyOffset();\n        doReturn(123L).when(messageStore).getMasterFlushedOffset();\n        setUpOneHAClient(messageStore);\n\n        messageStore = mockMessageStore();\n        doReturn(124L).when(messageStore).getMaxPhyOffset();\n        doReturn(124L).when(messageStore).getMasterFlushedOffset();\n        setUpOneHAClient(messageStore);\n\n        messageStore = mockMessageStore();\n        doReturn(125L).when(messageStore).getMaxPhyOffset();\n        doReturn(125L).when(messageStore).getMasterFlushedOffset();\n        setUpOneHAClient(messageStore);\n\n        await().atMost(Duration.ofMinutes(1)).until(new Callable<Boolean>() {\n            @Override\n            public Boolean call() throws Exception {\n                return HAServerTest.this.haService.getPush2SlaveMaxOffset().get() == 125L;\n            }\n        });\n    }\n\n    private void setUpOneHAClient(DefaultMessageStore defaultMessageStore) throws IOException {\n        HAClient haClient = new DefaultHAClient(defaultMessageStore);\n        haClient.updateHaMasterAddress(\"127.0.0.1:\" + this.storeConfig.getHaListenPort());\n        haClient.start();\n        this.haClientList.add(haClient);\n    }\n\n    private void setUpOneHAClient() throws IOException {\n        HAClient haClient = new DefaultHAClient(this.defaultMessageStore);\n        haClient.updateHaMasterAddress(\"127.0.0.1:\" + this.storeConfig.getHaListenPort());\n        haClient.start();\n        this.haClientList.add(haClient);\n    }\n\n    private DefaultMessageStore mockMessageStore() throws IOException, RocksDBException {\n        DefaultMessageStore messageStore = mock(DefaultMessageStore.class);\n        BrokerConfig brokerConfig = mock(BrokerConfig.class);\n\n        doReturn(true).when(brokerConfig).isInBrokerContainer();\n        doReturn(\"mock\").when(brokerConfig).getIdentifier();\n        doReturn(brokerConfig).when(messageStore).getBrokerConfig();\n        doReturn(new SystemClock()).when(messageStore).getSystemClock();\n        doAnswer(invocation -> System.currentTimeMillis()).when(messageStore).now();\n        doReturn(this.storeConfig).when(messageStore).getMessageStoreConfig();\n        doReturn(new BrokerConfig()).when(messageStore).getBrokerConfig();\n        doReturn(true).when(messageStore).isOffsetAligned(anyLong());\n//        doReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))).when(messageStore).sendMsgBack(anyLong());\n        doReturn(true).when(messageStore).truncateFiles(anyLong());\n\n        DefaultMessageStore masterStore = mock(DefaultMessageStore.class);\n        doReturn(Long.MAX_VALUE).when(masterStore).getFlushedWhere();\n        doReturn(masterStore).when(messageStore).getMasterStoreInProcess();\n\n        CommitLog commitLog = new CommitLog(messageStore);\n        doReturn(commitLog).when(messageStore).getCommitLog();\n        return messageStore;\n    }\n\n    private void tearDownOneHAClient() {\n        final HAClient haClient = this.haClientList.remove(0);\n        haClient.shutdown();\n    }\n\n    private void tearDownAllHAClient() {\n        for (final HAClient client : this.haClientList) {\n            client.shutdown();\n        }\n        this.haClientList.clear();\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/ha/WaitNotifyObjectTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class WaitNotifyObjectTest {\n    @Test\n    public void removeFromWaitingThreadTable() throws Exception {\n        final WaitNotifyObject waitNotifyObject = new WaitNotifyObject();\n        for (int i = 0; i < 5; i++) {\n            Thread t = new Thread(new Runnable() {\n                @Override\n                public void run() {\n                    waitNotifyObject.allWaitForRunning(100);\n                    waitNotifyObject.removeFromWaitingThreadTable();\n                }\n            });\n            t.start();\n            t.join();\n        }\n        Assert.assertEquals(0, waitNotifyObject.waitingThreadTable.size());\n    }\n\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.ha.autoswitch;\n\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.MappedFileQueue;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.StoreCheckpoint;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.config.FlushDiskType;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.logfile.MappedFile;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Assume;\nimport org.junit.Ignore;\nimport org.junit.Test;\nimport org.rocksdb.RocksDBException;\n\nimport java.io.File;\nimport java.net.InetAddress;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.HashSet;\nimport java.util.Random;\nimport java.util.Set;\nimport java.util.UUID;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicReference;\n\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertTrue;\n\npublic class AutoSwitchHATest {\n    private final String storeMessage = \"Once, there was a chance for me!\";\n    private final int defaultMappedFileSize = 1024 * 1024;\n    private int queueTotal = 100;\n    private AtomicInteger queueId = new AtomicInteger(0);\n    private SocketAddress bornHost;\n    private SocketAddress storeHost;\n    private byte[] messageBody;\n\n    private DefaultMessageStore messageStore1;\n    private DefaultMessageStore messageStore2;\n    private DefaultMessageStore messageStore3;\n    private MessageStoreConfig storeConfig1;\n    private MessageStoreConfig storeConfig2;\n    private MessageStoreConfig storeConfig3;\n    private String store1HaAddress;\n    private String store2HaAddress;\n\n    private BrokerStatsManager brokerStatsManager = new BrokerStatsManager(\"simpleTest\", true);\n    private String tmpdir = System.getProperty(\"java.io.tmpdir\");\n    private String storePathRootParentDir = (StringUtils.endsWith(tmpdir, File.separator) ? tmpdir : tmpdir + File.separator) + UUID.randomUUID();\n    private String storePathRootDir = storePathRootParentDir + File.separator + \"store\";\n    private Random random = new Random();\n\n    public void init(int mappedFileSize) throws Exception {\n        String brokerName = \"AutoSwitchHATest_\" + random.nextInt(65535);\n        queueTotal = 1;\n        messageBody = storeMessage.getBytes();\n        storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123);\n        bornHost = new InetSocketAddress(InetAddress.getByName(\"127.0.0.1\"), 0);\n        storeConfig1 = new MessageStoreConfig();\n        storeConfig1.setBrokerRole(BrokerRole.SYNC_MASTER);\n        storeConfig1.setHaSendHeartbeatInterval(1000);\n        storeConfig1.setStorePathRootDir(storePathRootDir + File.separator + brokerName + \"#1\");\n        storeConfig1.setStorePathCommitLog(storePathRootDir + File.separator + brokerName + \"#1\" + File.separator + \"commitlog\");\n        storeConfig1.setStorePathEpochFile(storePathRootDir + File.separator + brokerName + \"#1\" + File.separator + \"EpochFileCache\");\n        storeConfig1.setTotalReplicas(3);\n        storeConfig1.setInSyncReplicas(2);\n        buildMessageStoreConfig(storeConfig1, mappedFileSize);\n        this.store1HaAddress = \"127.0.0.1:10912\";\n\n        storeConfig2 = new MessageStoreConfig();\n        storeConfig2.setBrokerRole(BrokerRole.SLAVE);\n        storeConfig2.setHaSendHeartbeatInterval(1000);\n        storeConfig2.setStorePathRootDir(storePathRootDir + File.separator + brokerName + \"#2\");\n        storeConfig2.setStorePathCommitLog(storePathRootDir + File.separator + brokerName + \"#2\" + File.separator + \"commitlog\");\n        storeConfig2.setStorePathEpochFile(storePathRootDir + File.separator + brokerName + \"#2\" + File.separator + \"EpochFileCache\");\n        storeConfig2.setHaListenPort(10943);\n        storeConfig2.setTotalReplicas(3);\n        storeConfig2.setInSyncReplicas(2);\n        buildMessageStoreConfig(storeConfig2, mappedFileSize);\n        this.store2HaAddress = \"127.0.0.1:10943\";\n\n        messageStore1 = buildMessageStore(storeConfig1, 1L);\n        messageStore2 = buildMessageStore(storeConfig2, 2L);\n\n        storeConfig3 = new MessageStoreConfig();\n        storeConfig3.setBrokerRole(BrokerRole.SLAVE);\n        storeConfig3.setHaSendHeartbeatInterval(1000);\n        storeConfig3.setStorePathRootDir(storePathRootDir + File.separator + brokerName + \"#3\");\n        storeConfig3.setStorePathCommitLog(storePathRootDir + File.separator + brokerName + \"#3\" + File.separator + \"commitlog\");\n        storeConfig3.setStorePathEpochFile(storePathRootDir + File.separator + brokerName + \"#3\" + File.separator + \"EpochFileCache\");\n        storeConfig3.setHaListenPort(10980);\n        storeConfig3.setTotalReplicas(3);\n        storeConfig3.setInSyncReplicas(2);\n        buildMessageStoreConfig(storeConfig3, mappedFileSize);\n        messageStore3 = buildMessageStore(storeConfig3, 3L);\n\n        assertTrue(messageStore1.load());\n        assertTrue(messageStore2.load());\n        assertTrue(messageStore3.load());\n        messageStore1.start();\n        messageStore2.start();\n        messageStore3.start();\n\n//        ((AutoSwitchHAService) this.messageStore1.getHaService()).(\"127.0.0.1:8000\");\n//        ((AutoSwitchHAService) this.messageStore2.getHaService()).setLocalAddress(\"127.0.0.1:8001\");\n//        ((AutoSwitchHAService) this.messageStore3.getHaService()).setLocalAddress(\"127.0.0.1:8002\");\n    }\n\n    public void init(int mappedFileSize, boolean allAckInSyncStateSet) throws Exception {\n        String brokerName = \"AutoSwitchHATest_\" + random.nextInt(65535);\n        queueTotal = 1;\n        messageBody = storeMessage.getBytes();\n        storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123);\n        bornHost = new InetSocketAddress(InetAddress.getByName(\"127.0.0.1\"), 0);\n        storeConfig1 = new MessageStoreConfig();\n        storeConfig1.setBrokerRole(BrokerRole.SYNC_MASTER);\n        storeConfig1.setStorePathRootDir(storePathRootDir + File.separator + brokerName + \"#1\");\n        storeConfig1.setStorePathCommitLog(storePathRootDir + File.separator + brokerName + \"#1\" + File.separator + \"commitlog\");\n        storeConfig1.setStorePathEpochFile(storePathRootDir + File.separator + brokerName + \"#1\" + File.separator + \"EpochFileCache\");\n        storeConfig1.setAllAckInSyncStateSet(allAckInSyncStateSet);\n        buildMessageStoreConfig(storeConfig1, mappedFileSize);\n        this.store1HaAddress = \"127.0.0.1:10912\";\n\n        storeConfig2 = new MessageStoreConfig();\n        storeConfig2.setBrokerRole(BrokerRole.SLAVE);\n        storeConfig2.setStorePathRootDir(storePathRootDir + File.separator + brokerName + \"#2\");\n        storeConfig2.setStorePathCommitLog(storePathRootDir + File.separator + brokerName + \"#2\" + File.separator + \"commitlog\");\n        storeConfig2.setStorePathEpochFile(storePathRootDir + File.separator + brokerName + \"#2\" + File.separator + \"EpochFileCache\");\n        storeConfig2.setHaListenPort(10943);\n        storeConfig2.setAllAckInSyncStateSet(allAckInSyncStateSet);\n        buildMessageStoreConfig(storeConfig2, mappedFileSize);\n        this.store2HaAddress = \"127.0.0.1:10943\";\n\n        messageStore1 = buildMessageStore(storeConfig1, 1L);\n        messageStore2 = buildMessageStore(storeConfig2, 2L);\n\n        assertTrue(messageStore1.load());\n        assertTrue(messageStore2.load());\n        messageStore1.start();\n        messageStore2.start();\n\n//        ((AutoSwitchHAService) this.messageStore1.getHaService()).setLocalAddress(\"127.0.0.1:8000\");\n//        ((AutoSwitchHAService) this.messageStore2.getHaService()).setLocalAddress(\"127.0.0.1:8001\");\n    }\n\n    private boolean changeMasterAndPutMessage(DefaultMessageStore master, MessageStoreConfig masterConfig,\n        DefaultMessageStore slave, long slaveId, MessageStoreConfig slaveConfig, int epoch, String masterHaAddress,\n        int totalPutMessageNums) throws RocksDBException {\n\n        boolean flag = true;\n        // Change role\n        slaveConfig.setBrokerRole(BrokerRole.SLAVE);\n        masterConfig.setBrokerRole(BrokerRole.SYNC_MASTER);\n        flag &= slave.getHaService().changeToSlave(\"\", epoch, slaveId);\n        slave.getHaService().updateHaMasterAddress(masterHaAddress);\n        flag &= master.getHaService().changeToMaster(epoch);\n        // Put message on master\n        for (int i = 0; i < totalPutMessageNums; i++) {\n            PutMessageResult result = master.putMessage(buildMessage());\n            flag &= result.isOk();\n        }\n        return flag;\n    }\n\n    private void checkMessage(final DefaultMessageStore messageStore, int totalNums, int startOffset) {\n        await().atMost(30, TimeUnit.SECONDS)\n            .until(() -> {\n                GetMessageResult result = messageStore.getMessage(\"GROUP_A\", \"FooBar\", 0, startOffset, 1024, null);\n//                System.out.printf(result + \"%n\");\n                return result != null && result.getStatus() == GetMessageStatus.FOUND && result.getMessageCount() >= totalNums;\n            });\n    }\n\n    @Test\n    public void testConfirmOffset() throws Exception {\n        init(defaultMappedFileSize, true);\n        // Step1, set syncStateSet, if both broker1 and broker2 are in syncStateSet, the confirmOffset will be computed as the min slaveAckOffset(broker2's ack)\n        ((AutoSwitchHAService) this.messageStore1.getHaService()).setSyncStateSet(new HashSet<>(Arrays.asList(1L, 2L)));\n        boolean masterAndPutMessage = changeMasterAndPutMessage(this.messageStore1, this.storeConfig1, this.messageStore2, 2, this.storeConfig2, 1, store1HaAddress, 10);\n        assertTrue(masterAndPutMessage);\n        checkMessage(this.messageStore2, 10, 0);\n\n        final long confirmOffset = this.messageStore1.getConfirmOffset();\n\n        // Step2, shutdown store2\n        this.messageStore2.shutdown();\n\n        // Put message, which should succeed because slave is removed from syncStateSet, only master remains\n        final PutMessageResult putMessageResult = this.messageStore1.putMessage(buildMessage());\n        assertEquals(PutMessageStatus.PUT_OK,putMessageResult.getPutMessageStatus());\n\n        // The confirmOffset should update because syncStateSet only contains master after slave shutdown\n        assertTrue(this.messageStore1.getConfirmOffset() >= confirmOffset);\n\n        // Step3, shutdown store1, start store2, change store2 to master, epoch = 2\n        this.messageStore1.shutdown();\n\n        storeConfig2.setBrokerRole(BrokerRole.SYNC_MASTER);\n        messageStore2 = buildMessageStore(storeConfig2, 2L);\n        messageStore2.getRunningFlags().makeFenced(true);\n        assertTrue(messageStore2.load());\n        messageStore2.start();\n        messageStore2.getHaService().changeToMaster(2);\n        messageStore2.getRunningFlags().makeFenced(false);\n        ((AutoSwitchHAService) messageStore2.getHaService()).setSyncStateSet(new HashSet<>(Collections.singletonList(2L)));\n\n        // Put message on master\n        for (int i = 0; i < 10; i++) {\n            messageStore2.putMessage(buildMessage());\n        }\n\n        // Step4, start store1, it should truncate dirty logs and syncLog from store2\n        storeConfig1.setBrokerRole(BrokerRole.SLAVE);\n        messageStore1 = buildMessageStore(storeConfig1, 1L);\n        assertTrue(messageStore1.load());\n        messageStore1.start();\n        messageStore1.getHaService().changeToSlave(\"\", 2, 1L);\n        messageStore1.getHaService().updateHaMasterAddress(this.store2HaAddress);\n\n        checkMessage(this.messageStore1, 20, 0);\n    }\n\n    @Test\n    public void testAsyncLearnerBrokerRole() throws Exception {\n        init(defaultMappedFileSize);\n        ((AutoSwitchHAService) this.messageStore1.getHaService()).setSyncStateSet(new HashSet<>(Collections.singletonList(1L)));\n\n        storeConfig1.setBrokerRole(BrokerRole.SYNC_MASTER);\n        storeConfig2.setBrokerRole(BrokerRole.SLAVE);\n        storeConfig2.setAsyncLearner(true);\n        messageStore1.getHaService().changeToMaster(1);\n        messageStore2.getHaService().changeToSlave(\"\", 1, 2L);\n        messageStore2.getHaService().updateHaMasterAddress(store1HaAddress);\n        // Put message on master\n        for (int i = 0; i < 10; i++) {\n            messageStore1.putMessage(buildMessage());\n        }\n        checkMessage(messageStore2, 10, 0);\n        final Set<Long> syncStateSet = ((AutoSwitchHAService) this.messageStore1.getHaService()).getSyncStateSet();\n        assertFalse(syncStateSet.contains(2L));\n    }\n\n    @Test\n    public void testOptionAllAckInSyncStateSet() throws Exception {\n        init(defaultMappedFileSize, true);\n        AtomicReference<Set<Long>> syncStateSet = new AtomicReference<>();\n        ((AutoSwitchHAService) this.messageStore1.getHaService()).setSyncStateSet(new HashSet<>(Collections.singletonList(1L)));\n        ((AutoSwitchHAService) this.messageStore1.getHaService()).registerSyncStateSetChangedListener(newSyncStateSet -> {\n            syncStateSet.set(newSyncStateSet);\n        });\n\n        changeMasterAndPutMessage(this.messageStore1, this.storeConfig1, this.messageStore2, 2, this.storeConfig2, 1, store1HaAddress, 10);\n        checkMessage(this.messageStore2, 10, 0);\n        // Check syncStateSet\n        final Set<Long> result = syncStateSet.get();\n        assertTrue(result.contains(1L));\n        assertTrue(result.contains(2L));\n\n        // Now, shutdown store2\n        this.messageStore2.shutdown();\n        this.messageStore2.destroy();\n\n        // Wait for connection to be removed and syncStateSet to be updated by removeConnection\n        await().atMost(10, TimeUnit.SECONDS).until(() -> {\n            AutoSwitchHAService haService = (AutoSwitchHAService) this.messageStore1.getHaService();\n            return haService.getConnectionCount().get() == 0\n                && haService.getLocalSyncStateSet().size() == 1;\n        });\n\n        // Now manually set syncStateSet back to {1, 2} to test the scenario where\n        // syncStateSet contains a disconnected slave\n        ((AutoSwitchHAService) this.messageStore1.getHaService()).setSyncStateSet(result);\n\n        final PutMessageResult putMessageResult = this.messageStore1.putMessage(buildMessage());\n        assertEquals(PutMessageStatus.FLUSH_SLAVE_TIMEOUT,putMessageResult.getPutMessageStatus());\n    }\n\n    @Ignore\n    @Test\n    public void testChangeRoleManyTimes() throws Exception {\n\n        // Skip MacOSX platform for now as this test case is not stable on it.\n        Assume.assumeFalse(MixAll.isMac());\n\n        // Step1, change store1 to master, store2 to follower\n        init(defaultMappedFileSize);\n        ((AutoSwitchHAService) this.messageStore1.getHaService()).setSyncStateSet(new HashSet<>(Collections.singletonList(1L)));\n        changeMasterAndPutMessage(this.messageStore1, this.storeConfig1, this.messageStore2, 2, this.storeConfig2, 1, store1HaAddress, 10);\n        checkMessage(this.messageStore2, 10, 0);\n\n        // Step2, change store1 to follower, store2 to master, epoch = 2\n        ((AutoSwitchHAService) this.messageStore2.getHaService()).setSyncStateSet(new HashSet<>(Collections.singletonList(2L)));\n        changeMasterAndPutMessage(this.messageStore2, this.storeConfig2, this.messageStore1, 1, this.storeConfig1, 2, store2HaAddress, 10);\n        checkMessage(this.messageStore1, 20, 0);\n\n        // Step3, change store2 to follower, store1 to master, epoch = 3\n        changeMasterAndPutMessage(this.messageStore1, this.storeConfig1, this.messageStore2, 2, this.storeConfig2, 3, store1HaAddress, 10);\n        checkMessage(this.messageStore2, 30, 0);\n    }\n\n    @Test\n    public void testAddBroker() throws Exception {\n        // Step1: broker1 as leader, broker2 as follower\n        init(defaultMappedFileSize);\n        ((AutoSwitchHAService) this.messageStore1.getHaService()).setSyncStateSet(new HashSet<>(Collections.singletonList(1L)));\n\n        changeMasterAndPutMessage(this.messageStore1, this.storeConfig1, this.messageStore2, 2, this.storeConfig2, 1, store1HaAddress, 10);\n        checkMessage(this.messageStore2, 10, 0);\n\n        // Step2: add new broker3, link to broker1\n        messageStore3.getHaService().changeToSlave(\"\", 1, 3L);\n        messageStore3.getHaService().updateHaMasterAddress(store1HaAddress);\n        checkMessage(messageStore3, 10, 0);\n    }\n\n    @Test\n    public void testTruncateEpochLogAndAddBroker() throws Exception {\n        // Noted that 10 msg 's total size = 1570, and if init the mappedFileSize = 1700, one file only be used to store 10 msg.\n        init(1700);\n\n        // Step1: broker1 as leader, broker2 as follower, append 2 epoch, each epoch will be stored on one file(Because fileSize = 1700, which only can hold 10 msgs);\n        // Master: <Epoch1, 0, 1570> <Epoch2, 1570, 3270>\n\n        ((AutoSwitchHAService) this.messageStore1.getHaService()).setSyncStateSet(new HashSet<>(Collections.singletonList(1L)));\n        changeMasterAndPutMessage(this.messageStore1, this.storeConfig1, this.messageStore2, 2, this.storeConfig2, 1, store1HaAddress, 10);\n        checkMessage(this.messageStore2, 10, 0);\n\n        changeMasterAndPutMessage(this.messageStore1, this.storeConfig1, this.messageStore2, 2, this.storeConfig2, 2, store1HaAddress, 10);\n        checkMessage(this.messageStore2, 20, 0);\n\n        // Step2: Check file position, each epoch will be stored on one file(Because fileSize = 1700, which equal to 10 msg size);\n        // So epoch1 was stored in firstFile, epoch2 was stored in second file, the lastFile was empty.\n        final MappedFileQueue fileQueue = this.messageStore1.getCommitLog().getMappedFileQueue();\n        assertEquals(2, fileQueue.getTotalFileSize() / 1700);\n\n        // Step3: truncate epoch1's log (truncateEndOffset = 1570), which means we should delete the first file directly.\n        final MappedFile firstFile = this.messageStore1.getCommitLog().getMappedFileQueue().getFirstMappedFile();\n        firstFile.shutdown(1000);\n        fileQueue.retryDeleteFirstFile(1000);\n        assertEquals(this.messageStore1.getCommitLog().getMinOffset(), 1700);\n        checkMessage(this.messageStore1, 10, 10);\n\n        final AutoSwitchHAService haService = (AutoSwitchHAService) this.messageStore1.getHaService();\n        haService.truncateEpochFilePrefix(1570);\n\n        // Step4: add broker3 as slave, only have 10 msg from offset 10;\n        messageStore3.getHaService().changeToSlave(\"\", 2, 3L);\n        messageStore3.getHaService().updateHaMasterAddress(store1HaAddress);\n\n        checkMessage(messageStore3, 10, 10);\n    }\n\n    @Test\n    public void testTruncateEpochLogAndChangeMaster() throws Exception {\n        // Noted that 10 msg 's total size = 1570, and if init the mappedFileSize = 1700, one file only be used to store 10 msg.\n        init(1700);\n\n        // Step1: broker1 as leader, broker2 as follower, append 2 epoch, each epoch will be stored on one file(Because fileSize = 1700, which only can hold 10 msgs);\n        // Master: <Epoch1, 0, 1570> <Epoch2, 1570, 3270>\n\n        ((AutoSwitchHAService) this.messageStore1.getHaService()).setSyncStateSet(new HashSet<>(Collections.singletonList(1L)));\n        changeMasterAndPutMessage(this.messageStore1, this.storeConfig1, this.messageStore2, 2, this.storeConfig2, 1, store1HaAddress, 10);\n        checkMessage(this.messageStore2, 10, 0);\n        changeMasterAndPutMessage(this.messageStore1, this.storeConfig1, this.messageStore2, 2, this.storeConfig2, 2, store1HaAddress, 10);\n        checkMessage(this.messageStore2, 20, 0);\n\n        // Step2: Check file position, each epoch will be stored on one file(Because fileSize = 1700, which equal to 10 msg size);\n        // So epoch1 was stored in firstFile, epoch2 was stored in second file, the lastFile was empty.\n        final MappedFileQueue fileQueue = this.messageStore1.getCommitLog().getMappedFileQueue();\n        assertEquals(2, fileQueue.getTotalFileSize() / 1700);\n\n        // Step3: truncate epoch1's log (truncateEndOffset = 1570), which means we should delete the first file directly.\n        final MappedFile firstFile = this.messageStore1.getCommitLog().getMappedFileQueue().getFirstMappedFile();\n        firstFile.shutdown(1000);\n        fileQueue.retryDeleteFirstFile(1000);\n        assertEquals(this.messageStore1.getCommitLog().getMinOffset(), 1700);\n\n        final AutoSwitchHAService haService = (AutoSwitchHAService) this.messageStore1.getHaService();\n        haService.truncateEpochFilePrefix(1570);\n        checkMessage(this.messageStore1, 10, 10);\n\n        // Step4: add broker3 as slave\n        messageStore3.getHaService().changeToSlave(\"\", 2, 3L);\n        messageStore3.getHaService().updateHaMasterAddress(store1HaAddress);\n\n        checkMessage(messageStore3, 10, 10);\n\n        // Step5: change broker2 as leader, broker3 as follower\n        ((AutoSwitchHAService) this.messageStore2.getHaService()).setSyncStateSet(new HashSet<>(Collections.singletonList(2L)));\n        changeMasterAndPutMessage(this.messageStore2, this.storeConfig2, this.messageStore3, 3, this.storeConfig3, 3, this.store2HaAddress, 10);\n        checkMessage(messageStore3, 20, 10);\n\n        // Step6, let broker1 link to broker2, it should sync log from epoch3.\n        this.storeConfig1.setBrokerRole(BrokerRole.SLAVE);\n        this.messageStore1.getHaService().changeToSlave(\"\", 3, 1L);\n        this.messageStore1.getHaService().updateHaMasterAddress(this.store2HaAddress);\n\n        checkMessage(messageStore1, 20, 0);\n    }\n\n    @Test\n    public void testAddBrokerAndSyncFromLastFile() throws Exception {\n        init(1700);\n\n        // Step1: broker1 as leader, broker2 as follower, append 2 epoch, each epoch will be stored on one file(Because fileSize = 1700, which only can hold 10 msgs);\n        // Master: <Epoch1, 0, 1570> <Epoch2, 1570, 3270>\n        ((AutoSwitchHAService) this.messageStore1.getHaService()).setSyncStateSet(new HashSet<>(Collections.singletonList(1L)));\n        changeMasterAndPutMessage(this.messageStore1, this.storeConfig1, this.messageStore2, 2, this.storeConfig2, 1, store1HaAddress, 10);\n        checkMessage(this.messageStore2, 10, 0);\n        changeMasterAndPutMessage(this.messageStore1, this.storeConfig1, this.messageStore2, 2, this.storeConfig2, 2, store1HaAddress, 10);\n        checkMessage(this.messageStore2, 20, 0);\n\n        // Step2: restart broker3\n        messageStore3.shutdown();\n        messageStore3.destroy();\n\n        storeConfig3.setSyncFromLastFile(true);\n        messageStore3 = buildMessageStore(storeConfig3, 3L);\n        assertTrue(messageStore3.load());\n        messageStore3.start();\n\n        // Step2: add new broker3, link to broker1. because broker3 request sync from lastFile, so it only synced 10 msg from offset 10;\n        messageStore3.getHaService().changeToSlave(\"\", 2, 3L);\n        messageStore3.getHaService().updateHaMasterAddress(\"127.0.0.1:10912\");\n\n        checkMessage(messageStore3, 10, 10);\n    }\n\n    @Test\n    @SuppressWarnings(\"DoubleBraceInitialization\")\n    public void testCheckSynchronizingSyncStateSetFlag() throws Exception {\n        // Step1: broker1 as leader, broker2 as follower\n        init(defaultMappedFileSize);\n        ((AutoSwitchHAService) this.messageStore1.getHaService()).setSyncStateSet(new HashSet<>(Collections.singletonList(1L)));\n\n        changeMasterAndPutMessage(this.messageStore1, this.storeConfig1, this.messageStore2, 2, this.storeConfig2, 1, store1HaAddress, 10);\n        checkMessage(this.messageStore2, 10, 0);\n        AutoSwitchHAService masterHAService = (AutoSwitchHAService) this.messageStore1.getHaService();\n\n        // Step2: check flag SynchronizingSyncStateSet\n        Assert.assertTrue(masterHAService.isSynchronizingSyncStateSet());\n        Assert.assertEquals(this.messageStore1.getConfirmOffset(), 1580);\n        Set<Long> syncStateSet = masterHAService.getSyncStateSet();\n        Assert.assertEquals(syncStateSet.size(), 2);\n        Assert.assertTrue(syncStateSet.contains(1L));\n\n        // Step3: set new syncStateSet\n        HashSet<Long> newSyncStateSet = new HashSet<Long>() {{\n                add(1L);\n                add(2L);\n                }};\n        masterHAService.setSyncStateSet(newSyncStateSet);\n        Assert.assertFalse(masterHAService.isSynchronizingSyncStateSet());\n    }\n\n    @Test\n    public void testBuildConsumeQueueNotExceedConfirmOffset() throws Exception {\n        init(defaultMappedFileSize);\n        ((AutoSwitchHAService) this.messageStore1.getHaService()).setSyncStateSet(new HashSet<>(Collections.singletonList(1L)));\n        changeMasterAndPutMessage(this.messageStore1, this.storeConfig1, this.messageStore2, 2, this.storeConfig2, 1, store1HaAddress, 10);\n        checkMessage(this.messageStore2, 10, 0);\n\n        long tmpConfirmOffset = this.messageStore2.getConfirmOffset();\n        long setConfirmOffset = this.messageStore2.getConfirmOffset() - this.messageStore2.getConfirmOffset() / 2;\n        messageStore2.shutdown();\n        StoreCheckpoint storeCheckpoint = new StoreCheckpoint(storeConfig2.getStorePathRootDir() + File.separator + \"checkpoint\");\n        assertEquals(tmpConfirmOffset, storeCheckpoint.getConfirmPhyOffset());\n        storeCheckpoint.setConfirmPhyOffset(setConfirmOffset);\n        storeCheckpoint.shutdown();\n        messageStore2 = buildMessageStore(storeConfig2, 2L);\n        messageStore2.getRunningFlags().makeFenced(true);\n        assertTrue(messageStore2.load());\n        messageStore2.start();\n        messageStore2.getRunningFlags().makeFenced(false);\n        assertEquals(setConfirmOffset, messageStore2.getConfirmOffset());\n        checkMessage(this.messageStore2, 5, 0);\n    }\n\n    @After\n    public void destroy() throws Exception {\n        if (this.messageStore2 != null) {\n            messageStore2.shutdown();\n            messageStore2.destroy();\n        }\n        if (this.messageStore1 != null) {\n            messageStore1.shutdown();\n            messageStore1.destroy();\n        }\n        if (this.messageStore3 != null) {\n            messageStore3.shutdown();\n            messageStore3.destroy();\n        }\n        File file = new File(storePathRootParentDir);\n        UtilAll.deleteFile(file);\n    }\n\n    private DefaultMessageStore buildMessageStore(MessageStoreConfig messageStoreConfig,\n        long brokerId) throws Exception {\n        BrokerConfig brokerConfig = new BrokerConfig();\n        brokerConfig.setBrokerId(brokerId);\n        brokerConfig.setEnableControllerMode(true);\n        return new DefaultMessageStore(messageStoreConfig, brokerStatsManager, null, brokerConfig, new ConcurrentHashMap<>());\n    }\n\n    private void buildMessageStoreConfig(MessageStoreConfig messageStoreConfig, int mappedFileSize) {\n        messageStoreConfig.setMappedFileSizeCommitLog(mappedFileSize);\n        messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 1024);\n        messageStoreConfig.setMaxHashSlotNum(10000);\n        messageStoreConfig.setMaxIndexNum(100 * 100);\n        messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);\n        messageStoreConfig.setFlushIntervalConsumeQueue(1);\n    }\n\n    private MessageExtBrokerInner buildMessage() {\n        MessageExtBrokerInner msg = new MessageExtBrokerInner();\n        msg.setTopic(\"FooBar\");\n        msg.setTags(\"TAG1\");\n        msg.setBody(messageBody);\n        msg.setKeys(String.valueOf(System.currentTimeMillis()));\n        msg.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal);\n        msg.setSysFlag(0);\n        msg.setBornTimestamp(System.currentTimeMillis());\n        msg.setStoreHost(storeHost);\n        msg.setBornHost(bornHost);\n        msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n        return msg;\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/ha/autoswitch/EpochFileCacheTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.ha.autoswitch;\n\nimport java.io.File;\nimport java.nio.file.Paths;\nimport org.apache.rocketmq.remoting.protocol.EpochEntry;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertTrue;\n\npublic class EpochFileCacheTest {\n    private EpochFileCache epochCache;\n    private EpochFileCache epochCache2;\n    private String path;\n    private String path2;\n\n    @Before\n    public void setup() {\n        this.path = Paths.get(File.separator + \"tmp\", \"EpochCheckpoint\").toString();\n        this.epochCache = new EpochFileCache(path);\n        assertTrue(this.epochCache.appendEntry(new EpochEntry(1, 100)));\n        assertTrue(this.epochCache.appendEntry(new EpochEntry(2, 300)));\n        assertTrue(this.epochCache.appendEntry(new EpochEntry(3, 500)));\n        final EpochEntry entry = this.epochCache.getEntry(2);\n        assertEquals(entry.getEpoch(), 2);\n        assertEquals(entry.getStartOffset(), 300);\n        assertEquals(entry.getEndOffset(), 500);\n    }\n\n    @After\n    public void shutdown() {\n        new File(this.path).delete();\n        if (this.path2 != null) {\n            new File(this.path2).delete();\n        }\n    }\n\n    @Test\n    public void testInitFromFile() {\n        // Remove entries, init from file\n        assertTrue(this.epochCache.initCacheFromFile());\n        final EpochEntry entry = this.epochCache.getEntry(2);\n        assertEquals(entry.getEpoch(), 2);\n        assertEquals(entry.getStartOffset(), 300);\n        assertEquals(entry.getEndOffset(), 500);\n    }\n\n    @Test\n    public void testTruncate() {\n        this.epochCache.truncateSuffixByOffset(150);\n        assertNotNull(this.epochCache.getEntry(1));\n        assertNull(this.epochCache.getEntry(2));\n    }\n\n    @Test\n    public void testFindEpochEntryByOffset() {\n        final EpochEntry entry = this.epochCache.findEpochEntryByOffset(350);\n        assertEquals(entry.getEpoch(), 2);\n        assertEquals(entry.getStartOffset(), 300);\n        assertEquals(entry.getEndOffset(), 500);\n    }\n\n    @Test\n    public void testFindConsistentPointSample1() {\n        this.path2 = Paths.get(File.separator + \"tmp\", \"EpochCheckpoint2\").toString();\n        this.epochCache2 = new EpochFileCache(path2);\n        assertTrue(this.epochCache2.appendEntry(new EpochEntry(1, 100)));\n        assertTrue(this.epochCache2.appendEntry(new EpochEntry(2, 300)));\n        assertTrue(this.epochCache2.appendEntry(new EpochEntry(3, 450)));\n        /**\n         *  cache1: <Epoch1, 100>, <Epoch2, 300>, <Epoch3, 500>\n         *  cache2: <Epoch1, 100>, <Epoch2, 300>, <Epoch3, 450>\n         *  The consistent point should be 450\n         */\n        final long consistentPoint = this.epochCache.findConsistentPoint(this.epochCache2);\n        assertEquals(consistentPoint, 450);\n    }\n\n    @Test\n    public void testFindConsistentPointSample2() {\n        this.path2 = Paths.get(File.separator + \"tmp\", \"EpochCheckpoint2\").toString();\n        this.epochCache2 = new EpochFileCache(path2);\n        assertTrue(this.epochCache2.appendEntry(new EpochEntry(1, 100)));\n        assertTrue(this.epochCache2.appendEntry(new EpochEntry(2, 300)));\n        assertTrue(this.epochCache2.appendEntry(new EpochEntry(3, 500)));\n        /**\n         *  cache1: <Epoch1, 100>, <Epoch2, 300>, <Epoch3, 500, 700>\n         *  cache2: <Epoch1, 100>, <Epoch2, 300>, <Epoch3, 500, 600>\n         *  The consistent point should be 600\n         */\n        this.epochCache.setLastEpochEntryEndOffset(700);\n        this.epochCache2.setLastEpochEntryEndOffset(600);\n        final long consistentPoint = this.epochCache.findConsistentPoint(this.epochCache2);\n        assertEquals(consistentPoint, 600);\n    }\n\n    @Test\n    public void testFindConsistentPointSample3() {\n        this.path2 = Paths.get(File.separator + \"tmp\", \"EpochCheckpoint2\").toString();\n        this.epochCache2 = new EpochFileCache(path2);\n        assertTrue(this.epochCache2.appendEntry(new EpochEntry(1, 200)));\n        assertTrue(this.epochCache2.appendEntry(new EpochEntry(2, 500)));\n        /**\n         *  cache1: <Epoch1, 100>, <Epoch2, 300>, <Epoch3, 500, 700>\n         *  cache2: <Epoch1, 200>, <Epoch2, 500>\n         *  The consistent point should be -1\n         */\n        final long consistentPoint = this.epochCache.findConsistentPoint(this.epochCache2);\n        assertEquals(consistentPoint, -1);\n    }\n\n    @Test\n    public void testFindConsistentPointSample4() {\n        this.path2 = Paths.get(File.separator + \"tmp\", \"EpochCheckpoint2\").toString();\n        this.epochCache2 = new EpochFileCache(path2);\n        assertTrue(this.epochCache2.appendEntry(new EpochEntry(1, 100)));\n        assertTrue(this.epochCache2.appendEntry(new EpochEntry(2, 300)));\n        assertTrue(this.epochCache2.appendEntry(new EpochEntry(3, 500)));\n        assertTrue(this.epochCache2.appendEntry(new EpochEntry(4, 800)));\n        /**\n         *  cache1: <Epoch1, 100>, <Epoch2, 300>, <Epoch3, 500, 700>\n         *  cache2: <Epoch1, 100>, <Epoch2, 300>, <Epoch3, 500>, <Epoch4, 800>\n         *  The consistent point should be 700\n         */\n        this.epochCache.setLastEpochEntryEndOffset(700);\n        final long consistentPoint = this.epochCache2.findConsistentPoint(this.epochCache);\n        assertEquals(consistentPoint, 700);\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/index/IndexFileTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * $Id: IndexFileTest.java 1831 2013-05-16 01:39:51Z vintagewang@apache.org $\n */\npackage org.apache.rocketmq.store.index;\n\nimport java.io.File;\nimport java.util.ArrayList;\nimport java.util.List;\n\nimport org.apache.rocketmq.common.UtilAll;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class IndexFileTest {\n    private static final int HASH_SLOT_NUM = 100;\n    private static final int INDEX_NUM = 400;\n\n    @Test\n    public void testPutKey() throws Exception {\n        IndexFile indexFile = new IndexFile(\"100\", HASH_SLOT_NUM, INDEX_NUM, 0, 0);\n        for (long i = 0; i < (INDEX_NUM - 1); i++) {\n            boolean putResult = indexFile.putKey(Long.toString(i), i, System.currentTimeMillis());\n            assertThat(putResult).isTrue();\n        }\n\n        // put over index file capacity.\n        boolean putResult = indexFile.putKey(Long.toString(400), 400, System.currentTimeMillis());\n        assertThat(putResult).isFalse();\n        indexFile.destroy(0);\n        File file = new File(\"100\");\n        UtilAll.deleteFile(file);\n    }\n\n    @Test\n    public void testSelectPhyOffset() throws Exception {\n        IndexFile indexFile = new IndexFile(\"200\", HASH_SLOT_NUM, INDEX_NUM, 0, 0);\n\n        for (long i = 0; i < (INDEX_NUM - 1); i++) {\n            boolean putResult = indexFile.putKey(Long.toString(i), i, System.currentTimeMillis());\n            assertThat(putResult).isTrue();\n        }\n\n        // put over index file capacity.\n        boolean putResult = indexFile.putKey(Long.toString(400), 400, System.currentTimeMillis());\n        assertThat(putResult).isFalse();\n\n        final List<Long> phyOffsets = new ArrayList<>();\n        indexFile.selectPhyOffset(phyOffsets, \"60\", 10, 0, Long.MAX_VALUE);\n        assertThat(phyOffsets).isNotEmpty();\n        assertThat(phyOffsets.size()).isEqualTo(1);\n        indexFile.destroy(0);\n        File file = new File(\"200\");\n        UtilAll.deleteFile(file);\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/index/IndexServiceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.index;\n\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.junit.Test;\n\nimport java.util.concurrent.ConcurrentHashMap;\n\nimport static org.junit.jupiter.api.Assertions.assertDoesNotThrow;\n\n\npublic class IndexServiceTest {\n\n    @Test\n    public void testQueryOffsetThrow() throws Exception {\n        assertDoesNotThrow(() -> {\n            DefaultMessageStore store = new DefaultMessageStore(\n                    new MessageStoreConfig(),\n                    new BrokerStatsManager(new BrokerConfig()),\n                    null,\n                    new BrokerConfig(),\n                    new ConcurrentHashMap<>()\n            );\n\n            IndexService indexService = new IndexService(store);\n            indexService.queryOffset(\"test\", \"\", Integer.MAX_VALUE, 10, 100);\n        });\n    }\n\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/kv/CompactionLogTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.kv;\n\nimport com.google.common.collect.Lists;\nimport org.apache.commons.lang3.RandomStringUtils;\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.AppendMessageResult;\nimport org.apache.rocketmq.store.AppendMessageStatus;\nimport org.apache.rocketmq.store.CommitLog;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.MappedFileQueue;\nimport org.apache.rocketmq.store.MessageExtEncoder;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageSpinLock;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.logfile.DefaultMappedFile;\nimport org.apache.rocketmq.store.logfile.MappedFile;\nimport org.apache.rocketmq.store.queue.SparseConsumeQueue;\nimport org.junit.Assume;\nimport org.junit.Before;\nimport org.junit.Rule;\nimport org.junit.Test;\nimport org.junit.rules.TemporaryFolder;\nimport org.mockito.stubbing.Answer;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.net.InetAddress;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.net.UnknownHostException;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.nio.file.Paths;\nimport java.nio.file.StandardOpenOption;\nimport java.security.DigestException;\nimport java.security.NoSuchAlgorithmException;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\n\nimport static org.apache.rocketmq.store.kv.CompactionLog.COMPACTING_SUB_FOLDER;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyList;\nimport static org.mockito.Mockito.doCallRealMethod;\nimport static org.mockito.Mockito.doNothing;\nimport static org.mockito.Mockito.doReturn;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\npublic class CompactionLogTest {\n    CompactionLog clog;\n    MessageStoreConfig storeConfig;\n    MessageStore defaultMessageStore;\n    CompactionPositionMgr positionMgr;\n    String topic = \"ctopic\";\n    int queueId = 0;\n    int offsetMemorySize = 1024;\n    int compactionFileSize = 10240;\n    int compactionCqFileSize = 1024;\n\n\n    private static MessageExtEncoder encoder = new MessageExtEncoder(1024, new MessageStoreConfig());\n    private static SocketAddress storeHost;\n    private static SocketAddress bornHost;\n\n    @Rule\n    public TemporaryFolder tmpFolder = new TemporaryFolder();\n    String logPath;\n    String cqPath;\n\n    static {\n        try {\n            storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123);\n        } catch (UnknownHostException e) {\n        }\n        try {\n            bornHost = new InetSocketAddress(InetAddress.getByName(\"127.0.0.1\"), 0);\n        } catch (UnknownHostException e) {\n        }\n    }\n\n    @Before\n    public void setUp() throws IOException {\n        File file = tmpFolder.newFolder(\"compaction\");\n        logPath = Paths.get(file.getAbsolutePath(), \"compactionLog\").toString();\n        cqPath = Paths.get(file.getAbsolutePath(), \"compactionCq\").toString();\n\n        storeConfig = mock(MessageStoreConfig.class);\n        doReturn(compactionFileSize).when(storeConfig).getCompactionMappedFileSize();\n        doReturn(compactionCqFileSize).when(storeConfig).getCompactionCqMappedFileSize();\n        defaultMessageStore = mock(DefaultMessageStore.class);\n        doReturn(storeConfig).when(defaultMessageStore).getMessageStoreConfig();\n        positionMgr = mock(CompactionPositionMgr.class);\n        doReturn(-1L).when(positionMgr).getOffset(topic, queueId);\n    }\n\n    static int queueOffset = 0;\n    static int keyCount = 10;\n    public static ByteBuffer buildMessage() {\n        MessageExtBrokerInner msg = new MessageExtBrokerInner();\n        msg.setTopic(\"ctopic\");\n        msg.setTags(System.currentTimeMillis() + \"TAG\");\n        msg.setKeys(String.valueOf(queueOffset % keyCount));\n        msg.setBody(RandomStringUtils.randomAlphabetic(100).getBytes(StandardCharsets.UTF_8));\n        msg.setQueueId(0);\n        msg.setSysFlag(0);\n        msg.setBornTimestamp(System.currentTimeMillis());\n        msg.setStoreHost(storeHost);\n        msg.setBornHost(bornHost);\n        msg.setQueueOffset(queueOffset);\n        queueOffset++;\n        for (int i = 1; i < 3; i++) {\n            msg.putUserProperty(String.valueOf(i), \"xxx\" + i);\n        }\n        msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n        encoder.encode(msg);\n        return encoder.getEncoderBuffer();\n    }\n\n\n    @Test\n    public void testCheck() throws IllegalAccessException {\n        MappedFileQueue mfq = mock(MappedFileQueue.class);\n        MappedFileQueue smfq = mock(MappedFileQueue.class);\n        SparseConsumeQueue scq = mock(SparseConsumeQueue.class);\n        doReturn(smfq).when(scq).getMappedFileQueue();\n        CompactionLog.TopicPartitionLog tpLog = mock(CompactionLog.TopicPartitionLog.class);\n        FieldUtils.writeField(tpLog, \"mappedFileQueue\", mfq, true);\n        FieldUtils.writeField(tpLog, \"consumeQueue\", scq, true);\n\n        doReturn(Lists.newArrayList()).when(mfq).getMappedFiles();\n        doReturn(Lists.newArrayList()).when(smfq).getMappedFiles();\n\n        doCallRealMethod().when(tpLog).sanityCheck();\n        tpLog.sanityCheck();\n    }\n\n    @Test(expected = RuntimeException.class)\n    public void testCheckWithException() throws IllegalAccessException, IOException {\n        MappedFileQueue mfq = mock(MappedFileQueue.class);\n        MappedFileQueue smfq = mock(MappedFileQueue.class);\n        SparseConsumeQueue scq = mock(SparseConsumeQueue.class);\n        doReturn(smfq).when(scq).getMappedFileQueue();\n        CompactionLog.TopicPartitionLog tpLog = mock(CompactionLog.TopicPartitionLog.class);\n        FieldUtils.writeField(tpLog, \"mappedFileQueue\", mfq, true);\n        FieldUtils.writeField(tpLog, \"consumeQueue\", scq, true);\n\n        Files.createDirectories(Paths.get(logPath, topic, String.valueOf(queueId)));\n        Files.write(Paths.get(logPath, topic, String.valueOf(queueId), \"102400\"),\n            RandomStringUtils.randomAlphanumeric(compactionFileSize).getBytes(StandardCharsets.UTF_8),\n            StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE);\n        MappedFile mappedFile = new DefaultMappedFile(\n            Paths.get(logPath, topic, String.valueOf(queueId), \"102400\").toFile().getAbsolutePath(),\n            compactionFileSize);\n        doReturn(Lists.newArrayList(mappedFile)).when(mfq).getMappedFiles();\n        doReturn(Lists.newArrayList()).when(smfq).getMappedFiles();\n\n        doCallRealMethod().when(tpLog).sanityCheck();\n        tpLog.sanityCheck();\n    }\n\n    @Test\n    public void testCompaction() throws DigestException, NoSuchAlgorithmException, IllegalAccessException {\n        Iterator<SelectMappedBufferResult> iterator = mock(Iterator.class);\n        SelectMappedBufferResult smb = mock(SelectMappedBufferResult.class);\n        when(iterator.hasNext()).thenAnswer((Answer<Boolean>)invocationOnMock -> queueOffset < 1024);\n        when(iterator.next()).thenAnswer((Answer<SelectMappedBufferResult>)invocation ->\n            new SelectMappedBufferResult(0, buildMessage(), 0, null));\n\n        MappedFile mf = mock(MappedFile.class);\n        List<MappedFile> mappedFileList = Lists.newArrayList(mf);\n        doReturn(iterator).when(mf).iterator(0);\n\n        MessageStore messageStore = mock(DefaultMessageStore.class);\n        CommitLog commitLog = mock(CommitLog.class);\n        when(messageStore.getCommitLog()).thenReturn(commitLog);\n        when(commitLog.getCommitLogSize()).thenReturn(1024 * 1024);\n        CompactionLog clog = mock(CompactionLog.class);\n        FieldUtils.writeField(clog, \"defaultMessageStore\", messageStore, true);\n        doCallRealMethod().when(clog).getOffsetMap(any());\n        FieldUtils.writeField(clog, \"positionMgr\", positionMgr, true);\n\n        queueOffset = 0;\n        CompactionLog.OffsetMap offsetMap = clog.getOffsetMap(mappedFileList);\n        assertEquals(1023, offsetMap.getLastOffset());\n\n        doCallRealMethod().when(clog).compaction(any(List.class), any(CompactionLog.OffsetMap.class));\n        doNothing().when(clog).putEndMessage(any(MappedFileQueue.class));\n        doCallRealMethod().when(clog).checkAndPutMessage(any(SelectMappedBufferResult.class),\n            any(MessageExt.class), any(CompactionLog.OffsetMap.class), any(CompactionLog.TopicPartitionLog.class));\n        doCallRealMethod().when(clog).shouldRetainMsg(any(MessageExt.class), any(CompactionLog.OffsetMap.class));\n        List<MessageExt> compactResult = Lists.newArrayList();\n        when(clog.asyncPutMessage(any(ByteBuffer.class), any(MessageExt.class),\n            any(CompactionLog.TopicPartitionLog.class)))\n            .thenAnswer((Answer<CompletableFuture<PutMessageResult>>)invocation -> {\n                compactResult.add(invocation.getArgument(1));\n                return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK,\n                    new AppendMessageResult(AppendMessageStatus.PUT_OK)));\n            });\n        queueOffset = 0;\n        clog.compaction(mappedFileList, offsetMap);\n        assertEquals(keyCount, compactResult.size());\n        assertEquals(1014, compactResult.stream().mapToLong(MessageExt::getQueueOffset).min().orElse(1024));\n        assertEquals(1023, compactResult.stream().mapToLong(MessageExt::getQueueOffset).max().orElse(0));\n    }\n\n    @Test\n    public void testReplaceFiles() throws IOException, IllegalAccessException {\n        Assume.assumeFalse(MixAll.isWindows());\n        CompactionLog clog = mock(CompactionLog.class);\n        doCallRealMethod().when(clog).replaceFiles(anyList(), any(CompactionLog.TopicPartitionLog.class),\n            any(CompactionLog.TopicPartitionLog.class));\n        doCallRealMethod().when(clog).replaceCqFiles(any(SparseConsumeQueue.class),\n            any(SparseConsumeQueue.class), anyList());\n\n        CompactionLog.TopicPartitionLog dest = mock(CompactionLog.TopicPartitionLog.class);\n        MappedFileQueue destMFQ = mock(MappedFileQueue.class);\n        when(dest.getLog()).thenReturn(destMFQ);\n        List<MappedFile> destFiles = Lists.newArrayList();\n        when(destMFQ.getMappedFiles()).thenReturn(destFiles);\n\n        List<MappedFile> srcFiles = Lists.newArrayList();\n        String fileName = logPath + File.separator + COMPACTING_SUB_FOLDER + File.separator + String.format(\"%010d\", 0);\n        MappedFile mf = new DefaultMappedFile(fileName, 1024);\n        srcFiles.add(mf);\n        MappedFileQueue srcMFQ = mock(MappedFileQueue.class);\n        when(srcMFQ.getMappedFiles()).thenReturn(srcFiles);\n        CompactionLog.TopicPartitionLog src = mock(CompactionLog.TopicPartitionLog.class);\n        when(src.getLog()).thenReturn(srcMFQ);\n\n        FieldUtils.writeField(clog, \"readMessageLock\", new PutMessageSpinLock(), true);\n\n        clog.replaceFiles(Lists.newArrayList(), dest, src);\n        assertEquals(destFiles.size(), 1);\n        destFiles.forEach(f -> {\n            assertFalse(f.getFileName().contains(COMPACTING_SUB_FOLDER));\n        });\n    }\n\n}"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/kv/CompactionPositionMgrTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.kv;\n\nimport org.junit.Before;\nimport org.junit.Rule;\nimport org.junit.Test;\nimport org.junit.rules.TemporaryFolder;\n\nimport java.io.File;\nimport java.io.IOException;\n\nimport static org.junit.Assert.assertEquals;\n\npublic class CompactionPositionMgrTest {\n\n    @Rule\n    public TemporaryFolder tmpFolder = new TemporaryFolder();\n\n    File file;\n\n    @Before\n    public void setUp() throws IOException  {\n        file = tmpFolder.newFolder(\"compaction\");\n    }\n\n    @Test\n    public void testGetAndSet() {\n        CompactionPositionMgr mgr = new CompactionPositionMgr(file.getAbsolutePath());\n        mgr.setOffset(\"topic1\", 1, 1);\n        assertEquals(1, mgr.getOffset(\"topic1\", 1));\n        mgr.setOffset(\"topic1\", 1, 2);\n        assertEquals(2, mgr.getOffset(\"topic1\", 1));\n        mgr.setOffset(\"topic1\", 2, 1);\n        assertEquals(1, mgr.getOffset(\"topic1\", 2));\n    }\n\n    @Test\n    public void testLoadAndPersist() throws IOException {\n        CompactionPositionMgr mgr = new CompactionPositionMgr(file.getAbsolutePath());\n        mgr.setOffset(\"topic1\", 1, 2);\n        mgr.setOffset(\"topic1\", 2, 1);\n        mgr.persist();\n        mgr = null;\n\n        CompactionPositionMgr mgr2 = new CompactionPositionMgr(file.getAbsolutePath());\n        mgr2.load();\n        assertEquals(2, mgr2.getOffset(\"topic1\", 1));\n        assertEquals(1, mgr2.getOffset(\"topic1\", 2));\n    }\n}"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/kv/OffsetMapTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.kv;\n\nimport org.apache.rocketmq.store.kv.CompactionLog.OffsetMap;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotEquals;\nimport static org.junit.Assert.assertThrows;\n\npublic class OffsetMapTest {\n\n    @Test\n    public void testPutAndGet() throws Exception {\n        OffsetMap offsetMap = new OffsetMap(0);     //min 100 entry\n        offsetMap.put(\"abcde\", 1);\n        offsetMap.put(\"abc\", 3);\n        offsetMap.put(\"cde\", 4);\n        offsetMap.put(\"abcde\", 9);\n        assertEquals(offsetMap.get(\"abcde\"), 9);\n        assertEquals(offsetMap.get(\"cde\"), 4);\n        assertEquals(offsetMap.get(\"not_exist\"), -1);\n        assertEquals(offsetMap.getLastOffset(), 9);\n    }\n\n    @Test\n    public void testFull() throws Exception {\n        OffsetMap offsetMap = new OffsetMap(0);     //min 100 entry\n        for (int i = 0; i < 100; i++) {\n            offsetMap.put(String.valueOf(i), i);\n        }\n\n        assertEquals(offsetMap.get(\"66\"), 66);\n        assertNotEquals(offsetMap.get(\"55\"), 56);\n        assertEquals(offsetMap.getLastOffset(), 99);\n        assertThrows(IllegalArgumentException.class, () -> offsetMap.put(String.valueOf(100), 100));\n    }\n}"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImplTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.lock;\n\nimport org.junit.Test;\n\nimport java.util.Collection;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\n\npublic class AdaptiveBackOffSpinLockImplTest {\n\n    @Test\n    public void testGetLocks() {\n        AdaptiveBackOffSpinLockImpl lockImpl = new AdaptiveBackOffSpinLockImpl();\n        Collection<AdaptiveBackOffSpinLock> locks = lockImpl.getLocks();\n        assertEquals(2, locks.size());\n        for (AdaptiveBackOffSpinLock lock : locks) {\n            assertNotNull(lock);\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.lock;\n\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport java.util.concurrent.CountDownLatch;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\n\npublic class AdaptiveLockTest {\n\n    AdaptiveBackOffSpinLockImpl adaptiveLock;\n\n    @Before\n    public void init() {\n        adaptiveLock = new AdaptiveBackOffSpinLockImpl();\n    }\n\n    @Test\n    public void testAdaptiveLock() throws InterruptedException {\n        assertTrue(adaptiveLock.getAdaptiveLock() instanceof BackOffSpinLock);\n        CountDownLatch countDownLatch = new CountDownLatch(1);\n        adaptiveLock.lock();\n        new Thread(new Runnable() {\n            @Override\n            public void run() {\n                adaptiveLock.lock();\n                try {\n                    Thread.sleep(1);\n                } catch (InterruptedException e) {\n                    //ignore\n                }\n                adaptiveLock.unlock();\n                countDownLatch.countDown();\n            }\n        }).start();\n        Thread.sleep(1000);\n        adaptiveLock.unlock();\n        assertEquals(2000, ((BackOffSpinLock) adaptiveLock.getAdaptiveLock()).getOptimalDegree());\n        countDownLatch.await();\n\n        for (int i = 0; i <= 5; i++) {\n            CountDownLatch countDownLatch1 = new CountDownLatch(1);\n            adaptiveLock.lock();\n            new Thread(new Runnable() {\n                @Override\n                public void run() {\n                    adaptiveLock.lock();\n                    try {\n                        Thread.sleep(1);\n                    } catch (InterruptedException e) {\n                        //ignore\n                    }\n                    adaptiveLock.unlock();\n                    countDownLatch1.countDown();\n                }\n            }).start();\n            Thread.sleep(1000);\n            adaptiveLock.unlock();\n            countDownLatch1.await();\n        }\n        assertTrue(adaptiveLock.getAdaptiveLock() instanceof BackOffReentrantLock);\n\n        for (int i = 0; i <= 2; i++) {\n            adaptiveLock.lock();\n            adaptiveLock.unlock();\n            Thread.sleep(1000);\n        }\n        assertTrue(adaptiveLock.getAdaptiveLock() instanceof BackOffSpinLock);\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/logfile/DefaultMappedFileConcurrencyTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.logfile;\n\nimport java.io.File;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\npublic class DefaultMappedFileConcurrencyTest {\n\n    private String storePath;\n    private String fileName;\n    private int fileSize = 1024 * 1024; // 1MB\n    private static final int THREAD_COUNT = 10;\n    private static final int OPERATIONS_PER_THREAD = 100;\n\n    @Before\n    public void setUp() throws Exception {\n        storePath = System.getProperty(\"user.home\") + File.separator + \"unitteststore\" + System.currentTimeMillis();\n        fileName = storePath + File.separator + \"00000000000000000000\";\n        UtilAll.ensureDirOK(storePath);\n\n        // Initialize SharedByteBufferManager for tests\n        SharedByteBufferManager.getInstance().init(4 * 1024 * 1024, 16); // 4MB default, 16 shared buffers\n    }\n\n    @After\n    public void tearDown() throws Exception {\n        UtilAll.deleteFile(new File(storePath));\n    }\n\n    @Test\n    public void testConcurrentWriteWithoutMmap() throws Exception {\n        DefaultMappedFile mappedFile = new DefaultMappedFile(fileName, fileSize, true);\n        \n        try {\n            ExecutorService executor = Executors.newFixedThreadPool(THREAD_COUNT);\n            CountDownLatch latch = new CountDownLatch(THREAD_COUNT);\n            AtomicInteger successCount = new AtomicInteger(0);\n            AtomicInteger errorCount = new AtomicInteger(0);\n            \n            for (int i = 0; i < THREAD_COUNT; i++) {\n                final int threadId = i;\n                executor.submit(() -> {\n                    try {\n                        for (int j = 0; j < OPERATIONS_PER_THREAD; j++) {\n                            String data = String.format(\"Thread-%d-Operation-%d\", threadId, j);\n                            byte[] bytes = data.getBytes();\n                            \n                            boolean result = mappedFile.appendMessage(bytes);\n                            if (result) {\n                                successCount.incrementAndGet();\n                            } else {\n                                errorCount.incrementAndGet();\n                            }\n                        }\n                    } catch (Exception e) {\n                        errorCount.incrementAndGet();\n                        e.printStackTrace();\n                    } finally {\n                        latch.countDown();\n                    }\n                });\n            }\n            \n            latch.await();\n            executor.shutdown();\n            \n            // Success count: successCount.get()\n            // Error count: errorCount.get()\n            // Final wrote position: mappedFile.getWrotePosition()\n            \n            // All operations should succeed\n            Assert.assertEquals(\"All write operations should succeed\", \n                THREAD_COUNT * OPERATIONS_PER_THREAD, successCount.get());\n            Assert.assertEquals(\"No errors should occur\", 0, errorCount.get());\n            \n        } finally {\n            mappedFile.destroy(0);\n        }\n    }\n\n    @Test\n    public void testConcurrentWriteWithMmap() throws Exception {\n        DefaultMappedFile mappedFile = new DefaultMappedFile(fileName, fileSize, false);\n        \n        try {\n            ExecutorService executor = Executors.newFixedThreadPool(THREAD_COUNT);\n            CountDownLatch latch = new CountDownLatch(THREAD_COUNT);\n            AtomicInteger successCount = new AtomicInteger(0);\n            AtomicInteger errorCount = new AtomicInteger(0);\n            \n            for (int i = 0; i < THREAD_COUNT; i++) {\n                final int threadId = i;\n                executor.submit(() -> {\n                    try {\n                        for (int j = 0; j < OPERATIONS_PER_THREAD; j++) {\n                            String data = String.format(\"Thread-%d-Operation-%d\", threadId, j);\n                            byte[] bytes = data.getBytes();\n                            \n                            boolean result = mappedFile.appendMessage(bytes);\n                            if (result) {\n                                successCount.incrementAndGet();\n                            } else {\n                                errorCount.incrementAndGet();\n                            }\n                        }\n                    } catch (Exception e) {\n                        errorCount.incrementAndGet();\n                        e.printStackTrace();\n                    } finally {\n                        latch.countDown();\n                    }\n                });\n            }\n            \n            latch.await();\n            executor.shutdown();\n            \n            // Success count: successCount.get()\n            // Error count: errorCount.get()\n            // Final wrote position: mappedFile.getWrotePosition()\n            \n            // All operations should succeed\n            Assert.assertEquals(\"All write operations should succeed\", \n                THREAD_COUNT * OPERATIONS_PER_THREAD, successCount.get());\n            Assert.assertEquals(\"No errors should occur\", 0, errorCount.get());\n            \n        } finally {\n            mappedFile.destroy(0);\n        }\n    }\n\n    @Test\n    public void testConcurrentFlush() throws Exception {\n        DefaultMappedFile mappedFile = new DefaultMappedFile(fileName, fileSize, true);\n        \n        try {\n            // Write some data first\n            for (int i = 0; i < 100; i++) {\n                String data = \"Test data \" + i;\n                mappedFile.appendMessage(data.getBytes());\n            }\n            \n            ExecutorService executor = Executors.newFixedThreadPool(5);\n            CountDownLatch latch = new CountDownLatch(5);\n            AtomicInteger flushCount = new AtomicInteger(0);\n            \n            for (int i = 0; i < 5; i++) {\n                executor.submit(() -> {\n                    try {\n                        int flushed = mappedFile.flush(0);\n                        if (flushed > 0) {\n                            flushCount.incrementAndGet();\n                        }\n                    } catch (Exception e) {\n                        e.printStackTrace();\n                    } finally {\n                        latch.countDown();\n                    }\n                });\n            }\n            \n            latch.await();\n            executor.shutdown();\n            \n            Assert.assertTrue(\"At least one flush should succeed\", flushCount.get() > 0);\n            \n        } finally {\n            mappedFile.destroy(0);\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/logfile/DefaultMappedFileErrorHandlingTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.logfile;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.message.MessageExtBatch;\nimport org.apache.rocketmq.store.AppendMessageCallback;\nimport org.apache.rocketmq.store.AppendMessageResult;\nimport org.apache.rocketmq.store.AppendMessageStatus;\nimport org.apache.rocketmq.store.CompactionAppendMsgCallback;\nimport org.apache.rocketmq.store.PutMessageContext;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\npublic class DefaultMappedFileErrorHandlingTest {\n\n    private String storePath;\n    private String fileName;\n    private int fileSize = 1024 * 1024; // 1MB\n\n    @Before\n    public void setUp() throws Exception {\n        storePath = System.getProperty(\"user.home\") + File.separator + \"unitteststore\" + System.currentTimeMillis();\n        fileName = storePath + File.separator + \"00000000000000000000\";\n        UtilAll.ensureDirOK(storePath);\n\n        // Initialize SharedByteBufferManager for tests\n        SharedByteBufferManager.getInstance().init(4 * 1024 * 1024, 16); // 4MB default, 16 shared buffers\n    }\n\n    @After\n    public void tearDown() throws Exception {\n        UtilAll.deleteFile(new File(storePath));\n    }\n\n    @Test\n    public void testAppendMessageCallbackErrorHandling() throws IOException {\n        DefaultMappedFile mappedFile = new DefaultMappedFile(fileName, fileSize, true);\n        \n        try {\n            // Test with a callback that returns an error\n            AppendMessageCallback errorCallback = new AppendMessageCallback() {\n                @Override\n                public AppendMessageResult doAppend(long fileFromOffset, ByteBuffer byteBuffer, \n                    int maxBlank, MessageExtBrokerInner msg, \n                    PutMessageContext putMessageContext) {\n                    return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);\n                }\n                \n                @Override\n                public AppendMessageResult doAppend(long fileFromOffset, ByteBuffer byteBuffer, \n                    int maxBlank, MessageExtBatch messageExtBatch, \n                    PutMessageContext putMessageContext) {\n                    return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);\n                }\n            };\n            \n            // Create a mock message\n            MessageExtBrokerInner msg = new MessageExtBrokerInner();\n            msg.setBody(\"test message\".getBytes());\n            \n            AppendMessageResult result = mappedFile.appendMessage(msg, errorCallback, new PutMessageContext(\"test-topic\"));\n            \n            Assert.assertEquals(\"Should return error status\", \n                AppendMessageStatus.UNKNOWN_ERROR, result.getStatus());\n            \n        } finally {\n            mappedFile.destroy(0);\n        }\n    }\n\n    @Test\n    public void testCompactionAppendMsgCallbackErrorHandling() throws IOException {\n        DefaultMappedFile mappedFile = new DefaultMappedFile(fileName, fileSize, true);\n        \n        try {\n            // Test with a callback that returns an error\n            CompactionAppendMsgCallback errorCallback = new CompactionAppendMsgCallback() {\n                @Override\n                public AppendMessageResult doAppend(ByteBuffer bbDest, long fileFromOffset, \n                    int maxBlank, ByteBuffer bbSrc) {\n                    return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);\n                }\n            };\n            \n            ByteBuffer testBuffer = ByteBuffer.wrap(\"test data\".getBytes());\n            AppendMessageResult result = mappedFile.appendMessage(testBuffer, errorCallback);\n            \n            Assert.assertEquals(\"Should return error status\", \n                AppendMessageStatus.UNKNOWN_ERROR, result.getStatus());\n            \n        } finally {\n            mappedFile.destroy(0);\n        }\n    }\n\n    @Test\n    public void testWriteWithoutMmapWithNullRandomAccessFile() throws IOException {\n        DefaultMappedFile mappedFile = new DefaultMappedFile(fileName, fileSize, true);\n        \n        try {\n            // Simulate the case where randomAccessFile is null\n            // This should fall back to normal behavior\n            byte[] testData = \"test data\".getBytes();\n            boolean result = mappedFile.appendMessage(testData);\n            \n            // Should still work, but using MappedByteBuffer\n            Assert.assertTrue(\"Should still work with null RandomAccessFile\", result);\n            \n        } finally {\n            mappedFile.destroy(0);\n        }\n    }\n\n    @Test\n    public void testLargeDataWrite() throws IOException {\n        DefaultMappedFile mappedFile = new DefaultMappedFile(fileName, fileSize, true);\n        \n        try {\n            // Test writing data that's close to the file size limit\n            byte[] largeData = new byte[fileSize - 100]; // Leave some space\n            for (int i = 0; i < largeData.length; i++) {\n                largeData[i] = (byte) (i % 256);\n            }\n            \n            boolean result = mappedFile.appendMessage(largeData);\n            Assert.assertTrue(\"Should successfully write large data\", result);\n            Assert.assertEquals(\"Wrote position should match data size\", \n                largeData.length, mappedFile.getWrotePosition());\n            \n        } finally {\n            mappedFile.destroy(0);\n        }\n    }\n\n    @Test\n    public void testWriteBeyondFileSize() throws IOException {\n        DefaultMappedFile mappedFile = new DefaultMappedFile(fileName, fileSize, true);\n        \n        try {\n            // Fill the file almost completely\n            byte[] data = new byte[fileSize - 10];\n            boolean result = mappedFile.appendMessage(data);\n            Assert.assertTrue(\"Should successfully write data\", result);\n            \n            // Try to write more data than remaining space\n            byte[] overflowData = new byte[20]; // More than remaining 10 bytes\n            result = mappedFile.appendMessage(overflowData);\n            Assert.assertFalse(\"Should fail to write beyond file size\", result);\n            \n        } finally {\n            mappedFile.destroy(0);\n        }\n    }\n\n    @Test\n    public void testFlushErrorHandling() throws IOException {\n        DefaultMappedFile mappedFile = new DefaultMappedFile(fileName, fileSize, true);\n        \n        try {\n            // Write some data\n            byte[] testData = \"test data for flush\".getBytes();\n            mappedFile.appendMessage(testData);\n            \n            // Flush should succeed\n            int flushedPosition = mappedFile.flush(0);\n            Assert.assertTrue(\"Flush should succeed\", flushedPosition > 0);\n            \n        } finally {\n            mappedFile.destroy(0);\n        }\n    }\n\n    @Test\n    public void testAppendMessageWithOffset() throws IOException {\n        DefaultMappedFile mappedFile = new DefaultMappedFile(fileName, fileSize, true);\n        \n        try {\n            byte[] testData = \"Hello, RocketMQ!\".getBytes();\n            \n            // Test with valid offset\n            boolean result = mappedFile.appendMessage(testData, 0, testData.length);\n            Assert.assertTrue(\"Should successfully append with valid offset\", result);\n            \n            // Test with invalid offset (beyond array length)\n            result = mappedFile.appendMessage(testData, testData.length + 1, 1);\n            Assert.assertFalse(\"Should fail with invalid offset\", result);\n            \n        } finally {\n            mappedFile.destroy(0);\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/logfile/DefaultMappedFilePerformanceTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.logfile;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\npublic class DefaultMappedFilePerformanceTest {\n\n    private String storePath;\n    private String fileName;\n    private int fileSize = 10 * 1024 * 1024; // 10MB\n    private static final int WRITE_COUNT = 10000;\n    private static final int DATA_SIZE = 1024; // 1KB per write\n\n    @Before\n    public void setUp() throws Exception {\n        storePath = System.getProperty(\"user.home\") + File.separator + \"unitteststore\" + System.currentTimeMillis();\n        fileName = storePath + File.separator + \"00000000000000000000\";\n        UtilAll.ensureDirOK(storePath);\n\n        // Initialize SharedByteBufferManager for tests\n        SharedByteBufferManager.getInstance().init(4 * 1024 * 1024, 16); // 4MB default, 16 shared buffers\n    }\n\n    @After\n    public void tearDown() throws Exception {\n        UtilAll.deleteFile(new File(storePath));\n    }\n\n    @Test\n    public void testWriteWithoutMmapPerformance() throws IOException {\n        DefaultMappedFile mappedFile = new DefaultMappedFile(fileName, fileSize, true);\n        \n        try {\n            byte[] testData = new byte[DATA_SIZE];\n            for (int i = 0; i < testData.length; i++) {\n                testData[i] = (byte) (i % 256);\n            }\n            \n            long startTime = System.currentTimeMillis();\n            \n            for (int i = 0; i < WRITE_COUNT; i++) {\n                boolean result = mappedFile.appendMessage(testData);\n                Assert.assertTrue(\"Write should succeed\", result);\n            }\n            \n            long endTime = System.currentTimeMillis();\n            long duration = endTime - startTime;\n            \n            // WriteWithoutMmap Performance:\n            //   Writes: WRITE_COUNT\n            //   Data size per write: DATA_SIZE bytes\n            //   Total data: (WRITE_COUNT * DATA_SIZE / 1024) KB\n            //   Duration: duration ms\n            //   Throughput: (WRITE_COUNT * DATA_SIZE / 1024.0 / duration * 1000) KB/s\n            \n            Assert.assertEquals(\"Wrote position should match expected\", \n                WRITE_COUNT * DATA_SIZE, mappedFile.getWrotePosition());\n            \n        } finally {\n            mappedFile.destroy(0);\n        }\n    }\n\n    @Test\n    public void testWriteWithMmapPerformance() throws IOException {\n        DefaultMappedFile mappedFile = new DefaultMappedFile(fileName, fileSize, false);\n        \n        try {\n            byte[] testData = new byte[DATA_SIZE];\n            for (int i = 0; i < testData.length; i++) {\n                testData[i] = (byte) (i % 256);\n            }\n            \n            long startTime = System.currentTimeMillis();\n            \n            for (int i = 0; i < WRITE_COUNT; i++) {\n                boolean result = mappedFile.appendMessage(testData);\n                Assert.assertTrue(\"Write should succeed\", result);\n            }\n            \n            long endTime = System.currentTimeMillis();\n            long duration = endTime - startTime;\n            \n            // WriteWithMmap Performance:\n            //   Writes: WRITE_COUNT\n            //   Data size per write: DATA_SIZE bytes\n            //   Total data: (WRITE_COUNT * DATA_SIZE / 1024) KB\n            //   Duration: duration ms\n            //   Throughput: (WRITE_COUNT * DATA_SIZE / 1024.0 / duration * 1000) KB/s\n            \n            Assert.assertEquals(\"Wrote position should match expected\", \n                WRITE_COUNT * DATA_SIZE, mappedFile.getWrotePosition());\n            \n        } finally {\n            mappedFile.destroy(0);\n        }\n    }\n\n    @Test\n    public void testFlushPerformance() throws IOException {\n        DefaultMappedFile mappedFile = new DefaultMappedFile(fileName, fileSize, true);\n        \n        try {\n            // Write some data first\n            byte[] testData = new byte[DATA_SIZE];\n            for (int i = 0; i < testData.length; i++) {\n                testData[i] = (byte) (i % 256);\n            }\n            \n            for (int i = 0; i < 1000; i++) {\n                mappedFile.appendMessage(testData);\n            }\n            \n            long startTime = System.currentTimeMillis();\n            \n            int flushedPosition = mappedFile.flush(0);\n            \n            long endTime = System.currentTimeMillis();\n            long duration = endTime - startTime;\n            \n            // Flush Performance:\n            //   Flushed position: flushedPosition\n            //   Duration: duration ms\n            \n            Assert.assertTrue(\"Flush should succeed\", flushedPosition > 0);\n            \n        } finally {\n            mappedFile.destroy(0);\n        }\n    }\n\n    @Test\n    public void testByteBufferWritePerformance() throws IOException {\n        DefaultMappedFile mappedFile = new DefaultMappedFile(fileName, fileSize, true);\n        \n        try {\n            ByteBuffer testBuffer = ByteBuffer.allocate(DATA_SIZE);\n            for (int i = 0; i < DATA_SIZE; i++) {\n                testBuffer.put((byte) (i % 256));\n            }\n            \n            long startTime = System.currentTimeMillis();\n            \n            for (int i = 0; i < WRITE_COUNT; i++) {\n                testBuffer.rewind();\n                boolean result = mappedFile.appendMessage(testBuffer);\n                Assert.assertTrue(\"Write should succeed\", result);\n            }\n            \n            long endTime = System.currentTimeMillis();\n            long duration = endTime - startTime;\n            \n            // ByteBuffer Write Performance:\n            //   Writes: WRITE_COUNT\n            //   Data size per write: DATA_SIZE bytes\n            //   Total data: (WRITE_COUNT * DATA_SIZE / 1024) KB\n            //   Duration: duration ms\n            //   Throughput: (WRITE_COUNT * DATA_SIZE / 1024.0 / duration * 1000) KB/s\n            \n            Assert.assertEquals(\"Wrote position should match expected\", \n                WRITE_COUNT * DATA_SIZE, mappedFile.getWrotePosition());\n            \n        } finally {\n            mappedFile.destroy(0);\n        }\n    }\n\n    @Test\n    public void testMixedWriteOperations() throws IOException {\n        DefaultMappedFile mappedFile = new DefaultMappedFile(fileName, fileSize, true);\n        \n        try {\n            byte[] testData = new byte[DATA_SIZE];\n            for (int i = 0; i < testData.length; i++) {\n                testData[i] = (byte) (i % 256);\n            }\n            \n            long startTime = System.currentTimeMillis();\n            \n            // Mix of different write operations\n            for (int i = 0; i < WRITE_COUNT / 4; i++) {\n                // appendMessage(byte[])\n                boolean result1 = mappedFile.appendMessage(testData);\n                Assert.assertTrue(\"Write should succeed\", result1);\n                \n                // appendMessage(byte[], offset, length)\n                boolean result2 = mappedFile.appendMessage(testData, 0, testData.length);\n                Assert.assertTrue(\"Write should succeed\", result2);\n                \n                // appendMessage(ByteBuffer)\n                ByteBuffer buffer = ByteBuffer.wrap(testData);\n                boolean result3 = mappedFile.appendMessage(buffer);\n                Assert.assertTrue(\"Write should succeed\", result3);\n                \n                // appendMessageUsingFileChannel(byte[])\n                boolean result4 = mappedFile.appendMessageUsingFileChannel(testData);\n                Assert.assertTrue(\"Write should succeed\", result4);\n            }\n            \n            long endTime = System.currentTimeMillis();\n            long duration = endTime - startTime;\n            \n            // Mixed Write Operations Performance:\n            //   Total operations: WRITE_COUNT\n            //   Data size per operation: DATA_SIZE bytes\n            //   Total data: (WRITE_COUNT * DATA_SIZE / 1024) KB\n            //   Duration: duration ms\n            //   Throughput: (WRITE_COUNT * DATA_SIZE / 1024.0 / duration * 1000) KB/s\n            \n            Assert.assertEquals(\"Wrote position should match expected\", \n                WRITE_COUNT * DATA_SIZE, mappedFile.getWrotePosition());\n            \n        } finally {\n            mappedFile.destroy(0);\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/logfile/DefaultMappedFileTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.logfile;\n\nimport org.junit.Before;\nimport org.junit.Rule;\nimport org.junit.Test;\nimport org.junit.rules.TemporaryFolder;\n\nimport java.io.IOException;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.nio.file.Paths;\nimport java.nio.file.StandardOpenOption;\nimport java.util.List;\n\nimport static org.junit.Assert.assertEquals;\n\npublic class DefaultMappedFileTest {\n\n    @Rule\n    public TemporaryFolder tmpFolder = new TemporaryFolder();\n\n    String path;\n\n    @Before\n    public void setUp() throws IOException  {\n        path = tmpFolder.newFolder(\"compaction\").getAbsolutePath();\n    }\n\n    @Test\n    public void testWriteFile() throws IOException  {\n        Files.write(Paths.get(path,\"test.file\"), \"111\".getBytes(StandardCharsets.UTF_8),\n            StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);\n\n        Files.write(Paths.get(path,\"test.file\"), \"111\".getBytes(StandardCharsets.UTF_8),\n            StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);\n\n        List<String> positions = Files.readAllLines(Paths.get(path, \"test.file\"), StandardCharsets.UTF_8);\n        int p = Integer.parseInt(positions.stream().findFirst().orElse(\"0\"));\n        assertEquals(111, p);\n\n        Files.write(Paths.get(path,\"test.file\"), \"222\".getBytes(StandardCharsets.UTF_8),\n            StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);\n        positions = Files.readAllLines(Paths.get(path,\"test.file\"), StandardCharsets.UTF_8);\n        p = Integer.parseInt(positions.stream().findFirst().orElse(\"0\"));\n        assertEquals(222, p);\n    }\n\n}"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/logfile/DefaultMappedFileWriteWithoutMmapTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.logfile;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.store.TransientStorePool;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\npublic class DefaultMappedFileWriteWithoutMmapTest {\n\n    private String storePath;\n    private String fileName;\n    private int fileSize = 1024 * 1024; // 1MB\n\n    @Before\n    public void setUp() throws Exception {\n        storePath = System.getProperty(\"user.home\") + File.separator + \"unitteststore\" + System.currentTimeMillis();\n        fileName = storePath + File.separator + \"00000000000000000000\";\n        UtilAll.ensureDirOK(storePath);\n\n        // Initialize SharedByteBufferManager for tests\n        SharedByteBufferManager.getInstance().init(4 * 1024 * 1024, 16); // 4MB default, 16 shared buffers\n    }\n\n    @After\n    public void tearDown() throws Exception {\n        UtilAll.deleteFile(new File(storePath));\n    }\n\n    @Test\n    public void testWriteWithoutMmapEnabled() throws IOException {\n        // Test with writeWithoutMmap = true\n        DefaultMappedFile mappedFile = new DefaultMappedFile(fileName, fileSize, true);\n        \n        try {\n            // Test appendMessage with byte array\n            byte[] testData = \"Hello, RocketMQ!\".getBytes();\n            boolean result = mappedFile.appendMessage(testData);\n            Assert.assertTrue(\"Should successfully append message\", result);\n            Assert.assertEquals(\"Wrote position should be updated\", testData.length, mappedFile.getWrotePosition());\n            \n            // Test appendMessage with ByteBuffer\n            ByteBuffer buffer = ByteBuffer.wrap(\"Test ByteBuffer\".getBytes());\n            result = mappedFile.appendMessage(buffer);\n            Assert.assertTrue(\"Should successfully append ByteBuffer\", result);\n            Assert.assertEquals(\"Wrote position should be updated\", testData.length + \"Test ByteBuffer\".length(), mappedFile.getWrotePosition());\n            \n            // Test flush\n            int flushedPosition = mappedFile.flush(0);\n            Assert.assertTrue(\"Flush should succeed\", flushedPosition > 0);\n            \n        } finally {\n            mappedFile.destroy(0);\n        }\n    }\n\n    @Test\n    public void testWriteWithoutMmapDisabled() throws IOException {\n        // Test with writeWithoutMmap = false (default behavior)\n        DefaultMappedFile mappedFile = new DefaultMappedFile(fileName, fileSize, false);\n        \n        try {\n            // Test appendMessage with byte array\n            byte[] testData = \"Hello, RocketMQ!\".getBytes();\n            boolean result = mappedFile.appendMessage(testData);\n            Assert.assertTrue(\"Should successfully append message\", result);\n            Assert.assertEquals(\"Wrote position should be updated\", testData.length, mappedFile.getWrotePosition());\n            \n            // Test appendMessage with ByteBuffer\n            ByteBuffer buffer = ByteBuffer.wrap(\"Test ByteBuffer\".getBytes());\n            result = mappedFile.appendMessage(buffer);\n            Assert.assertTrue(\"Should successfully append ByteBuffer\", result);\n            Assert.assertEquals(\"Wrote position should be updated\", testData.length + \"Test ByteBuffer\".length(), mappedFile.getWrotePosition());\n            \n            // Test flush\n            int flushedPosition = mappedFile.flush(0);\n            Assert.assertTrue(\"Flush should succeed\", flushedPosition > 0);\n            \n        } finally {\n            mappedFile.destroy(0);\n        }\n    }\n\n    @Test\n    public void testWriteWithoutMmapWithTransientStorePool() throws IOException {\n        // Test with writeWithoutMmap = true and TransientStorePool\n        TransientStorePool transientStorePool = new TransientStorePool(5, fileSize);\n        DefaultMappedFile mappedFile = new DefaultMappedFile(fileName, fileSize, transientStorePool, true);\n        \n        try {\n            // Test appendMessage with byte array\n            byte[] testData = \"Hello, RocketMQ with TransientStorePool!\".getBytes();\n            boolean result = mappedFile.appendMessage(testData);\n            Assert.assertTrue(\"Should successfully append message\", result);\n            Assert.assertEquals(\"Wrote position should be updated\", testData.length, mappedFile.getWrotePosition());\n            \n        } finally {\n            mappedFile.destroy(0);\n        }\n    }\n\n    @Test\n    public void testAppendMessageWithOffset() throws IOException {\n        DefaultMappedFile mappedFile = new DefaultMappedFile(fileName, fileSize, true);\n        \n        try {\n            byte[] testData = \"Hello, RocketMQ with offset!\".getBytes();\n            boolean result = mappedFile.appendMessage(testData, 0, testData.length);\n            Assert.assertTrue(\"Should successfully append message with offset\", result);\n            Assert.assertEquals(\"Wrote position should be updated\", testData.length, mappedFile.getWrotePosition());\n            \n        } finally {\n            mappedFile.destroy(0);\n        }\n    }\n\n    @Test\n    public void testAppendMessageUsingFileChannel() throws IOException {\n        DefaultMappedFile mappedFile = new DefaultMappedFile(fileName, fileSize, true);\n        \n        try {\n            byte[] testData = \"Hello, RocketMQ using FileChannel!\".getBytes();\n            boolean result = mappedFile.appendMessageUsingFileChannel(testData);\n            Assert.assertTrue(\"Should successfully append message using FileChannel\", result);\n            Assert.assertEquals(\"Wrote position should be updated\", testData.length, mappedFile.getWrotePosition());\n            \n        } finally {\n            mappedFile.destroy(0);\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/pop/AckMsgTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.pop;\n\nimport com.alibaba.fastjson2.JSON;\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class AckMsgTest {\n\n    @Test\n    public void testSerializeAndDeSerialize() {\n        String longString = \"{\\\"ackOffset\\\":100,\\\"brokerName\\\":\\\"brokerName\\\",\\\"consumerGroup\\\":\\\"group\\\",\" +\n            \"\\\"popTime\\\":1670212915531,\\\"queueId\\\":3,\\\"startOffset\\\":200,\\\"topic\\\":\\\"topic\\\"}\";\n\n        AckMsg ackMsg = new AckMsg();\n        ackMsg.setBrokerName(\"brokerName\");\n        ackMsg.setTopic(\"topic\");\n        ackMsg.setConsumerGroup(\"group\");\n        ackMsg.setQueueId(3);\n        ackMsg.setStartOffset(200L);\n        ackMsg.setAckOffset(100L);\n        ackMsg.setPopTime(1670212915531L);\n        String jsonString = JSON.toJSONString(ackMsg);\n        AckMsg ackMsg1 = JSON.parseObject(jsonString, AckMsg.class);\n        AckMsg ackMsg2 = JSON.parseObject(longString, AckMsg.class);\n\n        Assert.assertEquals(ackMsg1.getBrokerName(), ackMsg2.getBrokerName());\n        Assert.assertEquals(ackMsg1.getTopic(), ackMsg2.getTopic());\n        Assert.assertEquals(ackMsg1.getConsumerGroup(), ackMsg2.getConsumerGroup());\n        Assert.assertEquals(ackMsg1.getQueueId(), ackMsg2.getQueueId());\n        Assert.assertEquals(ackMsg1.getStartOffset(), ackMsg2.getStartOffset());\n        Assert.assertEquals(ackMsg1.getAckOffset(), ackMsg2.getAckOffset());\n        Assert.assertEquals(ackMsg1.getPopTime(), ackMsg2.getPopTime());\n    }\n}"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/pop/BatchAckMsgTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.pop;\n\nimport com.alibaba.fastjson2.JSON;\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\npublic class BatchAckMsgTest {\n\n    @Test\n    public void testSerializeAndDeSerialize() {\n        String longString = \"{\\\"ackOffsetList\\\":[100, 101],\\\"consumerGroup\\\":\\\"group\\\",\" +\n                \"\\\"popTime\\\":1679454922000,\\\"queueId\\\":3,\\\"startOffset\\\":200,\\\"topic\\\":\\\"topic\\\"}\";\n\n        BatchAckMsg batchAckMsg = new BatchAckMsg();\n        List<Long> aol = new ArrayList<>(32);\n        aol.add(100L);\n        aol.add(101L);\n\n        batchAckMsg.setAckOffsetList(aol);\n        batchAckMsg.setStartOffset(200L);\n        batchAckMsg.setConsumerGroup(\"group\");\n        batchAckMsg.setTopic(\"topic\");\n        batchAckMsg.setQueueId(3);\n        batchAckMsg.setPopTime(1679454922000L);\n\n        String jsonString = JSON.toJSONString(batchAckMsg);\n        BatchAckMsg batchAckMsg1 = JSON.parseObject(jsonString, BatchAckMsg.class);\n        BatchAckMsg batchAckMsg2 = JSON.parseObject(longString, BatchAckMsg.class);\n\n        Assert.assertEquals(batchAckMsg1.getAckOffsetList(), batchAckMsg2.getAckOffsetList());\n        Assert.assertEquals(batchAckMsg1.getTopic(), batchAckMsg2.getTopic());\n        Assert.assertEquals(batchAckMsg1.getConsumerGroup(), batchAckMsg2.getConsumerGroup());\n        Assert.assertEquals(batchAckMsg1.getQueueId(), batchAckMsg2.getQueueId());\n        Assert.assertEquals(batchAckMsg1.getStartOffset(), batchAckMsg2.getStartOffset());\n        Assert.assertEquals(batchAckMsg1.getPopTime(), batchAckMsg2.getPopTime());\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/queue/BatchConsumeMessageTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.queue;\n\nimport java.io.File;\nimport java.nio.ByteBuffer;\nimport java.util.ArrayDeque;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Queue;\nimport java.util.Random;\nimport java.util.UUID;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.common.utils.QueueTypeUtils;\nimport org.apache.rocketmq.store.ConsumeQueueExt;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.MessageFilter;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static java.util.concurrent.TimeUnit.SECONDS;\nimport static org.awaitility.Awaitility.await;\n\npublic class BatchConsumeMessageTest extends QueueTestBase {\n    private static final int BATCH_NUM = 10;\n    private static final int TOTAL_MSGS = 200;\n    private DefaultMessageStore messageStore;\n    private ConcurrentMap<String, TopicConfig> topicConfigTableMap;\n\n    @Before\n    public void init() throws Exception {\n        this.topicConfigTableMap = new ConcurrentHashMap<>();\n        messageStore = (DefaultMessageStore) createMessageStore(null, true, this.topicConfigTableMap);\n        messageStore.load();\n        messageStore.start();\n    }\n\n    @After\n    public void destroy() {\n        messageStore.shutdown();\n        messageStore.destroy();\n\n        File file = new File(messageStore.getMessageStoreConfig().getStorePathRootDir());\n        UtilAll.deleteFile(file);\n    }\n\n    @Test\n    public void testSendMessagesToCqTopic() {\n        String topic = UUID.randomUUID().toString();\n        ConcurrentMap<String, TopicConfig>  topicConfigTable = createTopicConfigTable(topic, CQType.SimpleCQ);\n        this.topicConfigTableMap.putAll(topicConfigTable);\n\n//        int batchNum = 10;\n\n        // case 1 has PROPERTY_INNER_NUM but has no INNER_BATCH_FLAG\n//        MessageExtBrokerInner messageExtBrokerInner = buildMessage(topic, batchNum);\n//        messageExtBrokerInner.setSysFlag(0);\n//        PutMessageResult putMessageResult = messageStore.putMessage(messageExtBrokerInner);\n//        Assert.assertEquals(PutMessageStatus.MESSAGE_ILLEGAL, putMessageResult.getPutMessageStatus());\n\n        // case 2 has PROPERTY_INNER_NUM and has INNER_BATCH_FLAG, but is not a batchCq\n//        MessageExtBrokerInner messageExtBrokerInner = buildMessage(topic, 1);\n//        PutMessageResult putMessageResult = messageStore.putMessage(messageExtBrokerInner);\n//        Assert.assertEquals(PutMessageStatus.MESSAGE_ILLEGAL, putMessageResult.getPutMessageStatus());\n\n        // case 3 has neither PROPERTY_INNER_NUM nor INNER_BATCH_FLAG.\n        MessageExtBrokerInner messageExtBrokerInner = buildMessage(topic, -1);\n        PutMessageResult putMessageResult = messageStore.putMessage(messageExtBrokerInner);\n        Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n    }\n\n    @Test\n    public void testSendMessagesToBcqTopic() {\n        String topic = UUID.randomUUID().toString();\n        ConcurrentMap<String, TopicConfig>  topicConfigTable = createTopicConfigTable(topic, CQType.BatchCQ);\n        this.topicConfigTableMap.putAll(topicConfigTable);\n\n        // case 1 has PROPERTY_INNER_NUM but has no INNER_BATCH_FLAG\n//        MessageExtBrokerInner messageExtBrokerInner = buildMessage(topic, 1);\n//        PutMessageResult putMessageResult = messageStore.putMessage(messageExtBrokerInner);\n//        Assert.assertEquals(PutMessageStatus.MESSAGE_ILLEGAL, putMessageResult.getPutMessageStatus());\n\n        // case 2 has neither PROPERTY_INNER_NUM nor INNER_BATCH_FLAG.\n        MessageExtBrokerInner messageExtBrokerInner = buildMessage(topic, -1);\n        PutMessageResult putMessageResult = messageStore.putMessage(messageExtBrokerInner);\n        Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n\n        // case 3 has INNER_BATCH_FLAG but has no PROPERTY_INNER_NUM.\n        messageExtBrokerInner = buildMessage(topic, 1);\n        MessageAccessor.clearProperty(messageExtBrokerInner, MessageConst.PROPERTY_INNER_NUM);\n        messageExtBrokerInner.setSysFlag(MessageSysFlag.INNER_BATCH_FLAG);\n        putMessageResult = messageStore.putMessage(messageExtBrokerInner);\n        Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n    }\n\n    @Test\n    public void testConsumeBatchMessage() {\n        String topic = UUID.randomUUID().toString();\n        ConcurrentMap<String, TopicConfig>  topicConfigTable = createTopicConfigTable(topic, CQType.BatchCQ);\n        this.topicConfigTableMap.putAll(topicConfigTable);\n        int batchNum = 10;\n\n        MessageExtBrokerInner messageExtBrokerInner = buildMessage(topic, batchNum);\n        PutMessageResult putMessageResult = messageStore.putMessage(messageExtBrokerInner);\n        Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n\n        await().atMost(5, SECONDS).until(fullyDispatched(messageStore));\n\n        List<GetMessageResult> results = new ArrayList<>();\n        for (int i = 0; i < batchNum; i++) {\n            GetMessageResult result = messageStore.getMessage(\"whatever\", topic, 0, i, Integer.MAX_VALUE, Integer.MAX_VALUE, null);\n            try {\n                Assert.assertEquals(GetMessageStatus.FOUND, result.getStatus());\n                results.add(result);\n            } finally {\n                result.release();\n            }\n        }\n\n        for (GetMessageResult result : results) {\n            Assert.assertEquals(0, result.getMinOffset());\n            Assert.assertEquals(batchNum, result.getMaxOffset());\n        }\n\n    }\n\n    @Test\n    public void testNextBeginOffsetConsumeBatchMessage() {\n        String topic = UUID.randomUUID().toString();\n        ConcurrentMap<String, TopicConfig>  topicConfigTable = createTopicConfigTable(topic, CQType.BatchCQ);\n        this.topicConfigTableMap.putAll(topicConfigTable);\n        Random random = new Random();\n        int putMessageCount = 1000;\n\n        Queue<Integer> queue = new ArrayDeque<>();\n        for (int i = 0; i < putMessageCount; i++) {\n            int batchNum = random.nextInt(1000) + 2;\n            MessageExtBrokerInner messageExtBrokerInner = buildMessage(topic, batchNum);\n            PutMessageResult putMessageResult = messageStore.putMessage(messageExtBrokerInner);\n            Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n            queue.add(batchNum);\n        }\n\n        await().atMost(5, SECONDS).until(fullyDispatched(messageStore));\n\n        long pullOffset = 0L;\n        int getMessageCount = 0;\n        int atMostMsgNum = 1;\n        while (true) {\n            GetMessageResult getMessageResult = messageStore.getMessage(\"group\", topic, 0, pullOffset, atMostMsgNum, null);\n            if (Objects.equals(getMessageResult.getStatus(), GetMessageStatus.OFFSET_OVERFLOW_ONE)) {\n                break;\n            }\n            Assert.assertEquals(1, getMessageResult.getMessageQueueOffset().size());\n            Long baseOffset = getMessageResult.getMessageQueueOffset().get(0);\n            Integer batchNum = queue.poll();\n            Assert.assertNotNull(batchNum);\n            Assert.assertEquals(baseOffset + batchNum, getMessageResult.getNextBeginOffset());\n            pullOffset = getMessageResult.getNextBeginOffset();\n            getMessageCount++;\n        }\n        Assert.assertEquals(putMessageCount, getMessageCount);\n    }\n\n    @Test\n    public void testGetOffsetInQueueByTime() throws Exception {\n        String topic = \"testGetOffsetInQueueByTime\";\n\n        ConcurrentMap<String, TopicConfig>  topicConfigTable = createTopicConfigTable(topic, CQType.BatchCQ);\n        this.topicConfigTableMap.putAll(topicConfigTable);\n        Assert.assertTrue(QueueTypeUtils.isBatchCq(messageStore.getTopicConfig(topic)));\n\n        // The initial min max offset, before and after the creation of consume queue\n        Assert.assertEquals(0, messageStore.getMaxOffsetInQueue(topic, 0));\n        Assert.assertEquals(-1, messageStore.getMinOffsetInQueue(topic, 0));\n\n        int batchNum = 10;\n        long timeMid = -1;\n        for (int i = 0; i < 19; i++) {\n            PutMessageResult putMessageResult = messageStore.putMessage(buildMessage(topic, batchNum));\n            Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n            Thread.sleep(2);\n            if (i == 7)\n                timeMid = System.currentTimeMillis();\n        }\n\n        await().atMost(5, SECONDS).until(fullyDispatched(messageStore));\n\n        Assert.assertEquals(80, messageStore.getOffsetInQueueByTime(topic, 0, timeMid));\n        Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0));\n        Assert.assertEquals(190, messageStore.getMaxOffsetInQueue(topic, 0));\n\n        int maxBatchDeleteFilesNum = messageStore.getMessageStoreConfig().getMaxBatchDeleteFilesNum();\n        messageStore.getCommitLog().deleteExpiredFile(1L, 100, 12000, true, maxBatchDeleteFilesNum);\n        Assert.assertEquals(80, messageStore.getOffsetInQueueByTime(topic, 0, timeMid));\n\n        // can set periodic interval for executing  DefaultMessageStore.this.cleanFilesPeriodically() method, we can execute following code.\n        // default periodic interval is 60s, This code snippet will take 60 seconds.\n        /*final long a = timeMid;\n        await().atMost(Duration.ofMinutes(2)).until(()->{\n            long time = messageStore.getOffsetInQueueByTime(topic, 0, a);\n            return 180 ==time;\n        });\n        Assert.assertEquals(180, messageStore.getOffsetInQueueByTime(topic, 0, timeMid));*/\n    }\n\n    @Test\n    public void testDispatchNormalConsumeQueue() throws Exception {\n        String topic = \"TestDispatchBuildConsumeQueue\";\n        ConcurrentMap<String, TopicConfig>  topicConfigTable = createTopicConfigTable(topic, CQType.SimpleCQ);\n        this.topicConfigTableMap.putAll(topicConfigTable);\n\n        long timeStart = -1;\n        long timeMid = -1;\n        long commitLogMid = -1;\n\n        for (int i = 0; i < 100; i++) {\n            MessageExtBrokerInner messageExtBrokerInner = buildMessage(topic, -1);\n            PutMessageResult putMessageResult = messageStore.putMessage(messageExtBrokerInner);\n            Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n\n            Thread.sleep(2);\n            if (i == 0) {\n                timeStart = putMessageResult.getAppendMessageResult().getStoreTimestamp();\n            }\n            if (i == 50) {\n                timeMid = putMessageResult.getAppendMessageResult().getStoreTimestamp();\n                commitLogMid = putMessageResult.getAppendMessageResult().getWroteOffset();\n            }\n\n        }\n\n        await().atMost(5, SECONDS).until(fullyDispatched(messageStore));\n\n        ConsumeQueueInterface consumeQueue = messageStore.getConsumeQueue(topic, 0);\n        Assert.assertEquals(CQType.SimpleCQ, consumeQueue.getCQType());\n        //check the consume queue\n        Assert.assertEquals(0, consumeQueue.getMinOffsetInQueue());\n        Assert.assertEquals(0, consumeQueue.getOffsetInQueueByTime(0));\n        Assert.assertEquals(50, consumeQueue.getOffsetInQueueByTime(timeMid));\n        Assert.assertEquals(100, consumeQueue.getOffsetInQueueByTime(timeMid + Integer.MAX_VALUE));\n        Assert.assertEquals(100, consumeQueue.getMaxOffsetInQueue());\n        //check the messagestore\n        Assert.assertEquals(100, messageStore.getMessageTotalInQueue(topic, 0));\n        Assert.assertEquals(consumeQueue.getMinOffsetInQueue(), messageStore.getMinOffsetInQueue(topic, 0));\n        Assert.assertEquals(consumeQueue.getMaxOffsetInQueue(), messageStore.getMaxOffsetInQueue(topic, 0));\n        for (int i = -100; i < 100; i += 20) {\n            Assert.assertEquals(consumeQueue.getOffsetInQueueByTime(timeMid + i), messageStore.getOffsetInQueueByTime(topic, 0, timeMid + i));\n        }\n\n        //check the message time\n        long earliestMessageTime = messageStore.getEarliestMessageTime(topic, 0);\n        Assert.assertEquals(timeStart, earliestMessageTime);\n        long messageStoreTime = messageStore.getMessageStoreTimeStamp(topic, 0, 50);\n        Assert.assertEquals(timeMid, messageStoreTime);\n        long commitLogOffset = messageStore.getCommitLogOffsetInQueue(topic, 0, 50);\n        Assert.assertTrue(commitLogOffset >= messageStore.getMinPhyOffset());\n        Assert.assertTrue(commitLogOffset <= messageStore.getMaxPhyOffset());\n        Assert.assertEquals(commitLogMid, commitLogOffset);\n\n        Assert.assertTrue(messageStore.checkInMemByConsumeOffset(topic, 0, 50, 1));\n    }\n\n    @Test\n    public void testDispatchBuildBatchConsumeQueue() throws Exception {\n        String topic = \"testDispatchBuildBatchConsumeQueue\";\n        int batchNum = 10;\n        long timeStart = -1;\n        long timeMid = -1;\n\n        ConcurrentMap<String, TopicConfig>  topicConfigTable = createTopicConfigTable(topic, CQType.BatchCQ);\n        this.topicConfigTableMap.putAll(topicConfigTable);\n\n        for (int i = 0; i < 100; i++) {\n            PutMessageResult putMessageResult = messageStore.putMessage(buildMessage(topic, batchNum));\n            Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n            Thread.sleep(2);\n            if (i == 0) {\n                timeStart = putMessageResult.getAppendMessageResult().getStoreTimestamp();\n            }\n            if (i == 30) {\n                timeMid = putMessageResult.getAppendMessageResult().getStoreTimestamp();\n            }\n\n        }\n\n        await().atMost(5, SECONDS).until(fullyDispatched(messageStore));\n\n        ConsumeQueueInterface consumeQueue = messageStore.getConsumeQueue(topic, 0);\n        Assert.assertEquals(CQType.BatchCQ, consumeQueue.getCQType());\n\n        Assert.assertEquals(0, consumeQueue.getMinOffsetInQueue());\n        Assert.assertEquals(1000, consumeQueue.getMaxOffsetInQueue());\n\n        //check the message store\n        Assert.assertEquals(1000, messageStore.getMessageTotalInQueue(topic, 0));\n        Assert.assertEquals(consumeQueue.getMinOffsetInQueue(), messageStore.getMinOffsetInQueue(topic, 0));\n        Assert.assertEquals(consumeQueue.getMaxOffsetInQueue(), messageStore.getMaxOffsetInQueue(topic, 0));\n        for (int i = -100; i < 100; i += 20) {\n            Assert.assertEquals(consumeQueue.getOffsetInQueueByTime(timeMid + i), messageStore.getOffsetInQueueByTime(topic, 0, timeMid + i));\n        }\n\n        //check the message time\n        long earliestMessageTime = messageStore.getEarliestMessageTime(topic, 0);\n        Assert.assertEquals(earliestMessageTime, timeStart);\n        long messageStoreTime = messageStore.getMessageStoreTimeStamp(topic, 0, 300);\n        Assert.assertEquals(messageStoreTime, timeMid);\n        long commitLogOffset = messageStore.getCommitLogOffsetInQueue(topic, 0, 300);\n        Assert.assertTrue(commitLogOffset >= messageStore.getMinPhyOffset());\n        Assert.assertTrue(commitLogOffset <= messageStore.getMaxPhyOffset());\n\n        Assert.assertTrue(messageStore.checkInMemByConsumeOffset(topic, 0, 300, 1));\n\n        //get the message Normally\n        GetMessageResult getMessageResult = messageStore.getMessage(\"group\", topic, 0, 0, 10 * batchNum, null);\n        Assert.assertEquals(10, getMessageResult.getMessageMapedList().size());\n        for (int i = 0; i < 10; i++) {\n            SelectMappedBufferResult sbr = getMessageResult.getMessageMapedList().get(i);\n            MessageExt messageExt = MessageDecoder.decode(sbr.getByteBuffer());\n            short tmpBatchNum = Short.parseShort(messageExt.getProperty(MessageConst.PROPERTY_INNER_NUM));\n            Assert.assertEquals(i * batchNum, Long.parseLong(messageExt.getProperty(MessageConst.PROPERTY_INNER_BASE)));\n            Assert.assertEquals(batchNum, tmpBatchNum);\n        }\n    }\n\n    @Test\n    public void testGetBatchMessageWithinNumber() {\n        String topic = UUID.randomUUID().toString();\n\n        ConcurrentMap<String, TopicConfig>  topicConfigTable = createTopicConfigTable(topic, CQType.BatchCQ);\n        this.topicConfigTableMap.putAll(topicConfigTable);\n\n        int batchNum = 20;\n        for (int i = 0; i < 200; i++) {\n            PutMessageResult putMessageResult = messageStore.putMessage(buildMessage(topic, batchNum));\n            Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n            Assert.assertEquals(i * batchNum, putMessageResult.getAppendMessageResult().getLogicsOffset());\n            Assert.assertEquals(batchNum, putMessageResult.getAppendMessageResult().getMsgNum());\n        }\n\n        await().atMost(5, SECONDS).until(fullyDispatched(messageStore));\n\n        ConsumeQueueInterface consumeQueue = messageStore.getConsumeQueue(topic, 0);\n        Assert.assertEquals(CQType.BatchCQ, consumeQueue.getCQType());\n        Assert.assertEquals(0, consumeQueue.getMinOffsetInQueue());\n        Assert.assertEquals(200 * batchNum, consumeQueue.getMaxOffsetInQueue());\n\n        {\n            GetMessageResult getMessageResult = messageStore.getMessage(\"group\", topic, 0, 5, 1, Integer.MAX_VALUE, null);\n            Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());\n            Assert.assertEquals(batchNum, getMessageResult.getNextBeginOffset());\n            Assert.assertEquals(1, getMessageResult.getMessageMapedList().size());\n            Assert.assertEquals(batchNum, getMessageResult.getMessageCount());\n            SelectMappedBufferResult sbr = getMessageResult.getMessageMapedList().get(0);\n            MessageExt messageExt = MessageDecoder.decode(sbr.getByteBuffer());\n            short tmpBatchNum = Short.parseShort(messageExt.getProperty(MessageConst.PROPERTY_INNER_NUM));\n            Assert.assertEquals(0, messageExt.getQueueOffset());\n            Assert.assertEquals(batchNum, tmpBatchNum);\n        }\n        {\n            GetMessageResult getMessageResult = messageStore.getMessage(\"group\", topic, 0, 5, 39, Integer.MAX_VALUE, null);\n            Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());\n            Assert.assertEquals(1, getMessageResult.getMessageMapedList().size());\n            Assert.assertEquals(batchNum, getMessageResult.getNextBeginOffset());\n            Assert.assertEquals(batchNum, getMessageResult.getMessageCount());\n\n        }\n\n        {\n            GetMessageResult getMessageResult = messageStore.getMessage(\"group\", topic, 0, 5, 60, Integer.MAX_VALUE, null);\n            Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());\n            Assert.assertEquals(3, getMessageResult.getMessageMapedList().size());\n            Assert.assertEquals(3 * batchNum, getMessageResult.getNextBeginOffset());\n            Assert.assertEquals(3 * batchNum, getMessageResult.getMessageCount());\n            for (int i = 0; i < getMessageResult.getMessageBufferList().size(); i++) {\n                Assert.assertFalse(getMessageResult.getMessageMapedList().get(i).hasReleased());\n                SelectMappedBufferResult sbr = getMessageResult.getMessageMapedList().get(i);\n                MessageExt messageExt = MessageDecoder.decode(sbr.getByteBuffer());\n                Assert.assertNotNull(messageExt);\n                short innerBatchNum = Short.parseShort(messageExt.getProperty(MessageConst.PROPERTY_INNER_NUM));\n                Assert.assertEquals(i * batchNum, Long.parseLong(messageExt.getProperty(MessageConst.PROPERTY_INNER_BASE)));\n                Assert.assertEquals(batchNum, innerBatchNum);\n\n            }\n        }\n    }\n\n    @Test\n    public void testGetBatchMessageWithinSize() {\n        String topic = UUID.randomUUID().toString();\n        ConcurrentMap<String, TopicConfig>  topicConfigTable = createTopicConfigTable(topic, CQType.BatchCQ);\n        this.topicConfigTableMap.putAll(topicConfigTable);\n\n        int batchNum = 10;\n        for (int i = 0; i < 100; i++) {\n            PutMessageResult putMessageResult = messageStore.putMessage(buildMessage(topic, batchNum));\n            Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n            Assert.assertEquals(i * 10, putMessageResult.getAppendMessageResult().getLogicsOffset());\n            Assert.assertEquals(batchNum, putMessageResult.getAppendMessageResult().getMsgNum());\n        }\n\n        await().atMost(5, SECONDS).until(fullyDispatched(messageStore));\n\n        ConsumeQueueInterface consumeQueue = messageStore.getConsumeQueue(topic, 0);\n        Assert.assertEquals(CQType.BatchCQ, consumeQueue.getCQType());\n        Assert.assertEquals(0, consumeQueue.getMinOffsetInQueue());\n        Assert.assertEquals(1000, consumeQueue.getMaxOffsetInQueue());\n\n        {\n            GetMessageResult getMessageResult = messageStore.getMessage(\"group\", topic, 0, 5, Integer.MAX_VALUE, 100, null);\n            Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());\n            Assert.assertEquals(10, getMessageResult.getNextBeginOffset());\n            Assert.assertEquals(1, getMessageResult.getMessageMapedList().size());\n            SelectMappedBufferResult sbr = getMessageResult.getMessageMapedList().get(0);\n            MessageExt messageExt = MessageDecoder.decode(sbr.getByteBuffer());\n            short tmpBatchNum = Short.parseShort(messageExt.getProperty(MessageConst.PROPERTY_INNER_NUM));\n            Assert.assertEquals(0, messageExt.getQueueOffset());\n            Assert.assertEquals(batchNum, tmpBatchNum);\n        }\n        {\n            GetMessageResult getMessageResult = messageStore.getMessage(\"group\", topic, 0, 5, Integer.MAX_VALUE, 2048, null);\n            Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());\n            Assert.assertEquals(1, getMessageResult.getMessageMapedList().size());\n            Assert.assertEquals(10, getMessageResult.getNextBeginOffset());\n\n        }\n\n        {\n            GetMessageResult getMessageResult = messageStore.getMessage(\"group\", topic, 0, 5, Integer.MAX_VALUE, 4096, null);\n            Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());\n            Assert.assertEquals(3, getMessageResult.getMessageMapedList().size());\n            Assert.assertEquals(30, getMessageResult.getNextBeginOffset());\n            for (int i = 0; i < getMessageResult.getMessageBufferList().size(); i++) {\n                Assert.assertFalse(getMessageResult.getMessageMapedList().get(i).hasReleased());\n                SelectMappedBufferResult sbr = getMessageResult.getMessageMapedList().get(i);\n                MessageExt messageExt = MessageDecoder.decode(sbr.getByteBuffer());\n                short tmpBatchNum = Short.parseShort(messageExt.getProperty(MessageConst.PROPERTY_INNER_NUM));\n                Assert.assertEquals(i * batchNum, Long.parseLong(messageExt.getProperty(MessageConst.PROPERTY_INNER_BASE)));\n                Assert.assertEquals(batchNum, tmpBatchNum);\n\n            }\n        }\n    }\n\n    protected void putMsg(String topic) {\n        ConcurrentMap<String, TopicConfig>  topicConfigTable = createTopicConfigTable(topic, CQType.BatchCQ);\n        this.topicConfigTableMap.putAll(topicConfigTable);\n\n        for (int i = 0; i < TOTAL_MSGS; i++) {\n            MessageExtBrokerInner message = buildMessage(topic, BATCH_NUM * (i % 2 + 1));\n            switch (i % 3) {\n                case 0:\n                    message.setTags(\"TagA\");\n                    break;\n\n                case 1:\n                    message.setTags(\"TagB\");\n                    break;\n            }\n            message.setTagsCode(message.getTags().hashCode());\n            message.setPropertiesString(MessageDecoder.messageProperties2String(message.getProperties()));\n            PutMessageResult putMessageResult = messageStore.putMessage(message);\n            Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n        }\n\n        await().atMost(5, SECONDS).until(fullyDispatched(messageStore));\n    }\n\n    @Test\n    public void testEstimateMessageCountInEmptyConsumeQueue() {\n        String topic = UUID.randomUUID().toString();\n        ConsumeQueueInterface consumeQueue = messageStore.findConsumeQueue(topic, 0);\n        MessageFilter filter = new MessageFilter() {\n            @Override\n            public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {\n                return tagsCode == \"TagA\".hashCode();\n            }\n\n            @Override\n            public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map<String, String> properties) {\n                return false;\n            }\n        };\n        long estimation = consumeQueue.estimateMessageCount(0, 0, filter);\n        Assert.assertEquals(-1, estimation);\n\n        // test for illegal offset\n        estimation = consumeQueue.estimateMessageCount(0, 100, filter);\n        Assert.assertEquals(-1, estimation);\n        estimation = consumeQueue.estimateMessageCount(100, 1000, filter);\n        Assert.assertEquals(-1, estimation);\n    }\n\n    @Test\n    public void testEstimateMessageCount() {\n        String topic = UUID.randomUUID().toString();\n        putMsg(topic);\n        ConsumeQueueInterface cq = messageStore.findConsumeQueue(topic, 0);\n        MessageFilter filter = new MessageFilter() {\n            @Override\n            public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {\n                return tagsCode == \"TagA\".hashCode();\n            }\n\n            @Override\n            public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map<String, String> properties) {\n                return false;\n            }\n        };\n        long estimation = cq.estimateMessageCount(0, 2999, filter);\n        Assert.assertEquals(1000, estimation);\n\n        // test for illegal offset\n        estimation = cq.estimateMessageCount(0, Long.MAX_VALUE, filter);\n        Assert.assertEquals(-1, estimation);\n        estimation = cq.estimateMessageCount(100000, 1000000, filter);\n        Assert.assertEquals(-1, estimation);\n        estimation = cq.estimateMessageCount(100, 0, filter);\n        Assert.assertEquals(-1, estimation);\n    }\n\n    @Test\n    public void testEstimateMessageCountSample() {\n        String topic = UUID.randomUUID().toString();\n        putMsg(topic);\n        messageStore.getMessageStoreConfig().setSampleCountThreshold(10);\n        messageStore.getMessageStoreConfig().setMaxConsumeQueueScan(20);\n        ConsumeQueueInterface cq = messageStore.findConsumeQueue(topic, 0);\n        MessageFilter filter = new MessageFilter() {\n            @Override\n            public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {\n                return tagsCode == \"TagA\".hashCode();\n            }\n\n            @Override\n            public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map<String, String> properties) {\n                return false;\n            }\n        };\n        long estimation = cq.estimateMessageCount(1000, 2000, filter);\n        Assert.assertEquals(300, estimation);\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/queue/BatchConsumeQueueTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.queue;\n\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.store.ConsumeQueue;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.StoreTestBase;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport java.io.File;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Random;\n\nimport static java.lang.String.format;\n\npublic class BatchConsumeQueueTest extends StoreTestBase {\n\n    List<BatchConsumeQueue> batchConsumeQueues = new ArrayList<>();\n\n    private BatchConsumeQueue createBatchConsume(String path) {\n        if (path == null) {\n            path = createBaseDir();\n        }\n        baseDirs.add(path);\n        MessageStore messageStore = null;\n        try {\n            messageStore = createMessageStore(null);\n        } catch (Exception e) {\n            Assert.fail();\n        }\n        BatchConsumeQueue batchConsumeQueue = new BatchConsumeQueue(\"topic\", 0, path, fileSize, messageStore);\n        batchConsumeQueues.add(batchConsumeQueue);\n        return batchConsumeQueue;\n    }\n\n    private int fileSize = BatchConsumeQueue.CQ_STORE_UNIT_SIZE * 20;\n\n    @Test(timeout = 20000)\n    public void testBuildAndIterateBatchConsumeQueue() {\n        BatchConsumeQueue batchConsumeQueue = createBatchConsume(null);\n        batchConsumeQueue.load();\n        short batchNum = 10;\n        int unitNum = 10000;\n        int initialMsgOffset = 1000;\n        for (int i = 0; i < unitNum; i++) {\n            batchConsumeQueue.putBatchMessagePositionInfo(i, 1024, 111, i * batchNum, i * batchNum + initialMsgOffset, batchNum);\n        }\n        Assert.assertEquals(500, getBcqFileSize(batchConsumeQueue));\n        Assert.assertEquals(initialMsgOffset + batchNum * unitNum, batchConsumeQueue.getMaxOffsetInQueue());\n        Assert.assertEquals(initialMsgOffset, batchConsumeQueue.getMinOffsetInQueue());\n\n        {\n            CqUnit first = batchConsumeQueue.getEarliestUnit();\n            Assert.assertNotNull(first);\n            Assert.assertEquals(initialMsgOffset, first.getQueueOffset());\n            Assert.assertEquals(batchNum, first.getBatchNum());\n        }\n\n        {\n            CqUnit last = batchConsumeQueue.getLatestUnit();\n            Assert.assertNotNull(last);\n            Assert.assertEquals(initialMsgOffset + batchNum * unitNum - batchNum, last.getQueueOffset());\n            Assert.assertEquals(batchNum, last.getBatchNum());\n        }\n\n        for (int i = 0; i < initialMsgOffset + batchNum * unitNum + 10; i++) {\n            ReferredIterator<CqUnit> it = batchConsumeQueue.iterateFrom(i);\n            if (i < initialMsgOffset || i >= initialMsgOffset + batchNum * unitNum) {\n                Assert.assertNull(it);\n                continue;\n            }\n            Assert.assertNotNull(it);\n            CqUnit cqUnit = it.nextAndRelease();\n            Assert.assertNotNull(cqUnit);\n\n            long baseOffset = (i / batchNum) * batchNum;\n            Assert.assertEquals(baseOffset, cqUnit.getQueueOffset());\n            Assert.assertEquals(batchNum, cqUnit.getBatchNum());\n\n            Assert.assertEquals((i - initialMsgOffset) / batchNum, cqUnit.getPos());\n            Assert.assertEquals(1024, cqUnit.getSize());\n            Assert.assertEquals(111, cqUnit.getTagsCode());\n            Assert.assertNull(cqUnit.getCqExtUnit());\n        }\n        batchConsumeQueue.destroy();\n    }\n\n    @Test(timeout = 20000)\n    public void testBuildAndSearchBatchConsumeQueue() {\n        // Preparing the data may take some time\n        BatchConsumeQueue batchConsumeQueue = createBatchConsume(null);\n        batchConsumeQueue.load();\n        short batchSize = 10;\n        int unitNum = 20000;\n        for (int i = 0; i < unitNum; i++) {\n            batchConsumeQueue.putBatchMessagePositionInfo(i, 100, 0, i * batchSize, i * batchSize + 1, batchSize);\n        }\n        Assert.assertEquals(1000, getBcqFileSize(batchConsumeQueue));\n        Assert.assertEquals(unitNum * batchSize + 1, batchConsumeQueue.getMaxOffsetInQueue());\n        Assert.assertEquals(1, batchConsumeQueue.getMinOffsetInQueue());\n        batchConsumeQueue.reviseMaxAndMinOffsetInQueue();\n        Assert.assertEquals(unitNum * batchSize + 1, batchConsumeQueue.getMaxOffsetInQueue());\n        Assert.assertEquals(1, batchConsumeQueue.getMinOffsetInQueue());\n\n        // test search the offset\n        // lower bounds\n        Assert.assertFalse(ableToFindResult(batchConsumeQueue, 0));\n        Assert.assertTrue(ableToFindResult(batchConsumeQueue, 1));\n        // upper bounds\n        Assert.assertFalse(ableToFindResult(batchConsumeQueue, unitNum * batchSize + 1));\n        Assert.assertTrue(ableToFindResult(batchConsumeQueue, unitNum * batchSize));\n        // iterate every possible batch-msg offset\n        for (int i = 1; i <= unitNum * batchSize; i++) {\n            int expectedValue = ((i - 1) / batchSize) * batchSize + 1;\n            SelectMappedBufferResult batchMsgIndexBuffer = batchConsumeQueue.getBatchMsgIndexBuffer(i);\n            Assert.assertEquals(expectedValue, batchMsgIndexBuffer.getByteBuffer().getLong(BatchConsumeQueue.MSG_BASE_OFFSET_INDEX));\n            batchMsgIndexBuffer.release();\n        }\n        SelectMappedBufferResult sbr = batchConsumeQueue.getBatchMsgIndexBuffer(501);\n        Assert.assertEquals(501, sbr.getByteBuffer().getLong(BatchConsumeQueue.MSG_BASE_OFFSET_INDEX));\n        Assert.assertEquals(10, sbr.getByteBuffer().getShort(BatchConsumeQueue.MSG_BASE_OFFSET_INDEX + 8));\n        sbr.release();\n\n        // test search the storeTime\n        Assert.assertEquals(1, batchConsumeQueue.getOffsetInQueueByTime(-100));\n        Assert.assertEquals(1, batchConsumeQueue.getOffsetInQueueByTime(0));\n        Assert.assertEquals(11, batchConsumeQueue.getOffsetInQueueByTime(1));\n        for (int i = 0; i < unitNum; i++) {\n            int storeTime = i * batchSize;\n            int expectedOffset = storeTime + 1;\n            long offset = batchConsumeQueue.getOffsetInQueueByTime(storeTime);\n            Assert.assertEquals(expectedOffset, offset);\n        }\n        Assert.assertEquals(199991, batchConsumeQueue.getOffsetInQueueByTime(System.currentTimeMillis()));\n        batchConsumeQueue.destroy();\n    }\n\n    @Test(timeout = 20000)\n    public void testBuildAndRecoverBatchConsumeQueue() {\n        String tmpPath = createBaseDir();\n        short batchSize = 10;\n        {\n            BatchConsumeQueue batchConsumeQueue = createBatchConsume(tmpPath);\n            batchConsumeQueue.load();\n            for (int i = 0; i < 100; i++) {\n                batchConsumeQueue.putBatchMessagePositionInfo(i, 100, 0, i * batchSize, i * batchSize + 1, batchSize);\n            }\n            Assert.assertEquals(5, getBcqFileSize(batchConsumeQueue));\n            Assert.assertEquals(1001, batchConsumeQueue.getMaxOffsetInQueue());\n            Assert.assertEquals(1, batchConsumeQueue.getMinOffsetInQueue());\n\n            for (int i = 0; i < 10; i++) {\n                batchConsumeQueue.flush(0);\n            }\n        }\n        {\n            BatchConsumeQueue recover = createBatchConsume(tmpPath);\n            recover.load();\n            recover.recover();\n            Assert.assertEquals(5, getBcqFileSize(recover));\n            Assert.assertEquals(1001, recover.getMaxOffsetInQueue());\n            Assert.assertEquals(1, recover.getMinOffsetInQueue());\n            for (int i = 1; i <= 1000; i++) {\n                int expectedValue = ((i - 1) / batchSize) * batchSize + 1;\n                SelectMappedBufferResult batchMsgIndexBuffer = recover.getBatchMsgIndexBuffer(i);\n                Assert.assertEquals(expectedValue, batchMsgIndexBuffer.getByteBuffer().getLong(BatchConsumeQueue.MSG_BASE_OFFSET_INDEX));\n                batchMsgIndexBuffer.release();\n            }\n        }\n    }\n\n    @Test(timeout = 20000)\n    public void testTruncateBatchConsumeQueue() {\n        String tmpPath = createBaseDir();\n        BatchConsumeQueue batchConsumeQueue = createBatchConsume(tmpPath);\n        batchConsumeQueue.load();\n        short batchSize = 10;\n        int unitNum = 20000;\n        for (int i = 0; i < unitNum; i++) {\n            batchConsumeQueue.putBatchMessagePositionInfo(i, 100, 0, i * batchSize, i * batchSize + 1, batchSize);\n        }\n        Assert.assertEquals(1000, getBcqFileSize(batchConsumeQueue));\n        Assert.assertEquals(unitNum * batchSize + 1, batchConsumeQueue.getMaxOffsetInQueue());\n        Assert.assertEquals(1, batchConsumeQueue.getMinOffsetInQueue());\n\n        int truncatePhyOffset = new Random().nextInt(unitNum);\n        batchConsumeQueue.truncateDirtyLogicFiles(truncatePhyOffset);\n\n        for (int i = 1; i < unitNum; i++) {\n            long msgOffset = i * batchSize + 1;\n            if (i < truncatePhyOffset) {\n                int expectedValue = ((i - 1) / batchSize) * batchSize + 1;\n                SelectMappedBufferResult batchMsgIndexBuffer = batchConsumeQueue.getBatchMsgIndexBuffer(i);\n                Assert.assertEquals(expectedValue, batchMsgIndexBuffer.getByteBuffer().getLong(BatchConsumeQueue.MSG_BASE_OFFSET_INDEX));\n                batchMsgIndexBuffer.release();\n            } else {\n                Assert.assertNull(format(\"i: %d, truncatePhyOffset: %d\", i, truncatePhyOffset), batchConsumeQueue.getBatchMsgIndexBuffer(msgOffset));\n            }\n        }\n    }\n\n    @Test\n    public void testTruncateAndDeleteBatchConsumeQueue() {\n        String tmpPath = createBaseDir();\n        BatchConsumeQueue batchConsumeQueue = createBatchConsume(tmpPath);\n        batchConsumeQueue.load();\n        short batchSize = 10;\n        for (int i = 0; i < 100; i++) {\n            batchConsumeQueue.putBatchMessagePositionInfo(i, 100, 0, i * batchSize, i * batchSize + 1, batchSize);\n        }\n        Assert.assertEquals(5, batchConsumeQueue.mappedFileQueue.getMappedFiles().size());\n        Assert.assertEquals(1001, batchConsumeQueue.getMaxOffsetInQueue());\n        Assert.assertEquals(1, batchConsumeQueue.getMinOffsetInQueue());\n\n        batchConsumeQueue.truncateDirtyLogicFiles(80);\n\n        Assert.assertEquals(4, batchConsumeQueue.mappedFileQueue.getMappedFiles().size());\n        Assert.assertEquals(801, batchConsumeQueue.getMaxOffsetInQueue());\n        Assert.assertEquals(1, batchConsumeQueue.getMinOffsetInQueue());\n\n        //test\n        batchConsumeQueue.deleteExpiredFile(30);\n        Assert.assertEquals(3, batchConsumeQueue.mappedFileQueue.getMappedFiles().size());\n        Assert.assertEquals(801, batchConsumeQueue.getMaxOffsetInQueue());\n        Assert.assertEquals(301, batchConsumeQueue.getMinOffsetInQueue());\n\n    }\n\n    @After\n    @Override\n    public void clear() {\n        super.clear();\n        for (BatchConsumeQueue batchConsumeQueue : batchConsumeQueues) {\n            batchConsumeQueue.destroy();\n        }\n    }\n\n    private int getBcqFileSize(BatchConsumeQueue batchConsumeQueue) {\n        return batchConsumeQueue.mappedFileQueue.getMappedFiles().size();\n    }\n\n    private boolean ableToFindResult(BatchConsumeQueue batchConsumeQueue, long msgOffset) {\n        SelectMappedBufferResult batchMsgIndexBuffer = batchConsumeQueue.getBatchMsgIndexBuffer(msgOffset);\n        try {\n            return batchMsgIndexBuffer != null;\n        } finally {\n            if (batchMsgIndexBuffer != null) {\n                batchMsgIndexBuffer.release();\n            }\n        }\n    }\n\n    protected MessageStore createMessageStore(String baseDir) throws Exception {\n        if (baseDir == null) {\n            baseDir = createBaseDir();\n        }\n        baseDirs.add(baseDir);\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setMappedFileSizeCommitLog(1024 * 8);\n        messageStoreConfig.setMappedFileSizeConsumeQueue(100 * ConsumeQueue.CQ_STORE_UNIT_SIZE);\n        messageStoreConfig.setMapperFileSizeBatchConsumeQueue(20 * BatchConsumeQueue.CQ_STORE_UNIT_SIZE);\n        messageStoreConfig.setMappedFileSizeConsumeQueueExt(1024);\n        messageStoreConfig.setMaxIndexNum(100 * 10);\n        messageStoreConfig.setEnableConsumeQueueExt(false);\n        messageStoreConfig.setStorePathRootDir(baseDir);\n        messageStoreConfig.setStorePathCommitLog(baseDir + File.separator + \"commitlog\");\n        messageStoreConfig.setHaListenPort(nextPort());\n        messageStoreConfig.setMaxTransferBytesOnMessageInDisk(1024 * 1024);\n        messageStoreConfig.setMaxTransferBytesOnMessageInMemory(1024 * 1024);\n        messageStoreConfig.setMaxTransferCountOnMessageInDisk(1024);\n        messageStoreConfig.setMaxTransferCountOnMessageInMemory(1024);\n        messageStoreConfig.setSearchBcqByCacheEnable(true);\n\n        return new DefaultMessageStore(\n            messageStoreConfig,\n            new BrokerStatsManager(\"simpleTest\", true),\n            (topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties) -> {\n            },\n            new BrokerConfig(), new ConcurrentHashMap<>());\n    }\n\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/queue/CombineConsumeQueueStoreTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.queue;\n\nimport java.io.File;\nimport java.net.InetSocketAddress;\nimport java.util.UUID;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.io.FileUtils;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.constant.PermName;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.ConsumeQueue;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.apache.rocketmq.common.TopicFilterType.SINGLE_TAG;\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertTrue;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class CombineConsumeQueueStoreTest extends QueueTestBase {\n    private DefaultMessageStore messageStore;\n    private MessageStoreConfig messageStoreConfig;\n    private ConcurrentMap<String, TopicConfig> topicConfigTableMap;\n\n    String topic = UUID.randomUUID().toString();\n    final int queueId = 0;\n    final int msgNum = 100;\n    final int msgSize = 1000;\n\n    @Before\n    public void init() throws Exception {\n        this.topicConfigTableMap = new ConcurrentHashMap<>();\n        messageStoreConfig = new MessageStoreConfig();\n    }\n\n    @After\n    public void destroy() {\n        if (!messageStore.isShutdown()) {\n            messageStore.shutdown();\n        }\n        messageStore.destroy();\n\n        File file = new File(messageStore.getMessageStoreConfig().getStorePathRootDir());\n        UtilAll.deleteFile(file);\n    }\n\n    @Test(expected = IllegalArgumentException.class)\n    public void CombineConsumeQueueStore_EmptyLoadingCQTypes_ThrowsException() throws Exception {\n        messageStore = (DefaultMessageStore) createMessageStore(null, false, topicConfigTableMap, messageStoreConfig);\n\n        messageStoreConfig.setCombineCQLoadingCQTypes(\"\");\n        new CombineConsumeQueueStore(messageStore);\n    }\n\n    @Test\n    public void CombineConsumeQueueStore_InitializesConsumeQueueStore() throws Exception {\n        messageStore = (DefaultMessageStore) createMessageStore(null, false, topicConfigTableMap, messageStoreConfig);\n        {\n            messageStoreConfig.setCombineCQLoadingCQTypes(\"default\");\n            messageStoreConfig.setCombineCQPreferCQType(\"default\");\n            CombineConsumeQueueStore store = new CombineConsumeQueueStore(messageStore);\n            assertNotNull(store.getConsumeQueueStore());\n            assertNull(store.getRocksDBConsumeQueueStore());\n        }\n\n        {\n            messageStoreConfig.setCombineCQLoadingCQTypes(\"defaultRocksDB\");\n            messageStoreConfig.setCombineCQPreferCQType(\"defaultRocksDB\");\n            messageStoreConfig.setCombineAssignOffsetCQType(\"defaultRocksDB\");\n            CombineConsumeQueueStore store = new CombineConsumeQueueStore(messageStore);\n            assertNull(store.getConsumeQueueStore());\n            assertNotNull(store.getRocksDBConsumeQueueStore());\n            assertTrue(store.getCurrentReadStore() instanceof RocksDBConsumeQueueStore);\n        }\n\n        {\n            messageStoreConfig.setCombineCQLoadingCQTypes(\";;default;defaultRocksDB;\");\n            messageStoreConfig.setCombineCQPreferCQType(\"defaultRocksDB\");\n            CombineConsumeQueueStore store = new CombineConsumeQueueStore(messageStore);\n            assertNotNull(store.getConsumeQueueStore());\n            assertNotNull(store.getRocksDBConsumeQueueStore());\n            assertTrue(store.getCurrentReadStore() instanceof RocksDBConsumeQueueStore);\n        }\n\n        {\n            messageStoreConfig.setCombineCQLoadingCQTypes(\"default;defaultRocksDB\");\n            messageStoreConfig.setCombineCQPreferCQType(\"defaultRocksDB\");\n            CombineConsumeQueueStore store = new CombineConsumeQueueStore(messageStore);\n            assertTrue(store.getCurrentReadStore() instanceof RocksDBConsumeQueueStore);\n        }\n\n    }\n\n    @Test\n    public void testIterator() throws Exception {\n        messageStoreConfig.setRocksdbCQDoubleWriteEnable(true);\n        messageStore = (DefaultMessageStore) createMessageStore(null, false, topicConfigTableMap, messageStoreConfig);\n        messageStore.load();\n        messageStore.start();\n\n        //The initial min max offset, before and after the creation of consume queue\n        Assert.assertEquals(0, messageStore.getMaxOffsetInQueue(topic, queueId));\n        Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, queueId));\n\n        ConsumeQueueInterface consumeQueue = messageStore.getConsumeQueue(topic, queueId);\n        Assert.assertEquals(CQType.SimpleCQ, consumeQueue.getCQType());\n        Assert.assertEquals(0, consumeQueue.getMaxOffsetInQueue());\n        Assert.assertEquals(0, consumeQueue.getMinOffsetInQueue());\n        Assert.assertEquals(0, messageStore.getMaxOffsetInQueue(topic, queueId));\n        Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, queueId));\n\n        for (int i = 0; i < msgNum; i++) {\n            DispatchRequest request = new DispatchRequest(topic, queueId, i * msgSize, msgSize, i,\n                System.currentTimeMillis(), i, null, null, 0, 0, null);\n            messageStore.getQueueStore().putMessagePositionInfoWrapper(request);\n        }\n\n        await().atMost(1, TimeUnit.SECONDS).untilAsserted(() -> {\n            checkCQ(consumeQueue, msgNum, msgSize);\n\n            CombineConsumeQueueStore combineConsumeQueueStore = (CombineConsumeQueueStore) messageStore.getQueueStore();\n            ConsumeQueueInterface rocksDBConsumeQueue = combineConsumeQueueStore.getRocksDBConsumeQueueStore().getConsumeQueue(topic, queueId);\n            Assert.assertEquals(CQType.RocksDBCQ, rocksDBConsumeQueue.getCQType());\n            Assert.assertEquals(msgNum, rocksDBConsumeQueue.getMaxOffsetInQueue());\n            checkCQ(rocksDBConsumeQueue, msgNum, msgSize);\n        });\n    }\n\n    private void checkCQ(ConsumeQueueInterface consumeQueue, int msgNum,\n        int msgSize) {\n        Assert.assertEquals(0, consumeQueue.getMinLogicOffset());\n        Assert.assertEquals(0, consumeQueue.getMinOffsetInQueue());\n        Assert.assertEquals(msgNum, consumeQueue.getMaxOffsetInQueue());\n        Assert.assertEquals(msgNum, consumeQueue.getMessageTotalInQueue());\n\n        assertNull(consumeQueue.iterateFrom(-1));\n        assertNull(consumeQueue.iterateFrom(msgNum));\n\n        {\n            CqUnit first = consumeQueue.getEarliestUnit();\n            assertNotNull(first);\n            Assert.assertEquals(0, first.getQueueOffset());\n            Assert.assertEquals(msgSize, first.getSize());\n            assertTrue(first.isTagsCodeValid());\n        }\n        {\n            CqUnit last = consumeQueue.getLatestUnit();\n            assertNotNull(last);\n            Assert.assertEquals(msgNum - 1, last.getQueueOffset());\n            Assert.assertEquals(msgSize, last.getSize());\n            assertTrue(last.isTagsCodeValid());\n        }\n\n        for (int i = 0; i < msgNum; i++) {\n            ReferredIterator<CqUnit> iterator = consumeQueue.iterateFrom(i);\n            assertNotNull(iterator);\n            long queueOffset = i;\n            while (iterator.hasNext()) {\n                CqUnit cqUnit = iterator.next();\n                Assert.assertEquals(queueOffset, cqUnit.getQueueOffset());\n                Assert.assertEquals(queueOffset * msgSize, cqUnit.getPos());\n                Assert.assertEquals(msgSize, cqUnit.getSize());\n                assertTrue(cqUnit.isTagsCodeValid());\n                Assert.assertEquals(queueOffset, cqUnit.getTagsCode());\n                Assert.assertEquals(queueOffset, cqUnit.getValidTagsCodeAsLong().longValue());\n                Assert.assertEquals(1, cqUnit.getBatchNum());\n                assertNull(cqUnit.getCqExtUnit());\n                queueOffset++;\n            }\n            Assert.assertEquals(msgNum, queueOffset);\n        }\n    }\n\n    @Test\n    public void testInitializeWithOffset() throws Exception {\n        final String path = createBaseDir();\n        FileUtils.deleteDirectory(new File(path));\n        topicConfigTableMap.put(topic, new TopicConfig(topic, 1, 1, PermName.PERM_WRITE | PermName.PERM_READ));\n\n        {\n            messageStoreConfig.setRocksdbCQDoubleWriteEnable(true);\n            messageStore = (DefaultMessageStore) createMessageStore(path, false, topicConfigTableMap, messageStoreConfig);\n            messageStore.load();\n\n            ConsumeQueueStoreInterface consumeQueueStoreInterface = messageStore.getQueueStore();\n            assertTrue(consumeQueueStoreInterface instanceof CombineConsumeQueueStore);\n            CombineConsumeQueueStore combineConsumeQueueStore = (CombineConsumeQueueStore) consumeQueueStoreInterface;\n            ConsumeQueueStore consumeQueueStore = combineConsumeQueueStore.getConsumeQueueStore();\n            RocksDBConsumeQueueStore rocksDBConsumeQueueStore = combineConsumeQueueStore.getRocksDBConsumeQueueStore();\n            assertNotNull(consumeQueueStore);\n            assertNotNull(rocksDBConsumeQueueStore);\n\n            ConsumeQueueInterface rocksDBConsumeQueue = rocksDBConsumeQueueStore.findOrCreateConsumeQueue(topic, queueId);\n            consumeQueueStore.findOrCreateConsumeQueue(topic, queueId);\n            rocksDBConsumeQueue.initializeWithOffset(100, 0);\n\n            Assert.assertEquals(100, rocksDBConsumeQueue.getMaxOffsetInQueue());\n            Assert.assertEquals(100, rocksDBConsumeQueue.getMinOffsetInQueue());\n\n            rocksDBConsumeQueue.initializeWithOffset(200, 0);\n\n            Assert.assertEquals(200, rocksDBConsumeQueue.getMaxOffsetInQueue());\n            Assert.assertEquals(200, rocksDBConsumeQueue.getMinOffsetInQueue());\n\n            messageStore.start();\n\n            Assert.assertEquals(0, rocksDBConsumeQueue.getMaxOffsetInQueue());\n            Assert.assertEquals(0, rocksDBConsumeQueue.getMinOffsetInQueue());\n\n            ConsumeQueue consumeQueue = (ConsumeQueue) consumeQueueStore.findOrCreateConsumeQueue(topic, queueId);\n\n            for (int i = 0; i < msgNum; i++) {\n                MessageExtBrokerInner msg = new MessageExtBrokerInner();\n\n                msg.setQueueId(queueId);\n                msg.setBody(new byte[msgSize]);\n                msg.setTopic(topic);\n                msg.setTags(\"TAG1\");\n\n                msg.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(SINGLE_TAG, msg.getTags()));\n                msg.setBornTimestamp(System.currentTimeMillis());\n                msg.setReconsumeTimes(0);\n\n                msg.setBornHost(new InetSocketAddress(9999));\n                msg.setStoreHost(new InetSocketAddress(8888));\n\n                messageStore.putMessage(msg);\n            }\n\n            await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> {\n                Assert.assertEquals(msgNum, consumeQueue.getMaxOffsetInQueue());\n                Assert.assertEquals(0, consumeQueue.getMinOffsetInQueue());\n            });\n\n            messageStore.shutdown();\n        }\n\n        {\n            messageStoreConfig.setRocksdbCQDoubleWriteEnable(true);\n            messageStore = (DefaultMessageStore) createMessageStore(path, false, topicConfigTableMap, messageStoreConfig);\n            messageStore.load();\n\n            ConsumeQueueStoreInterface consumeQueueStoreInterface = messageStore.getQueueStore();\n            assertTrue(consumeQueueStoreInterface instanceof CombineConsumeQueueStore);\n            CombineConsumeQueueStore combineConsumeQueueStore = (CombineConsumeQueueStore) consumeQueueStoreInterface;\n            ConsumeQueueStore consumeQueueStore = combineConsumeQueueStore.getConsumeQueueStore();\n            RocksDBConsumeQueueStore rocksDBConsumeQueueStore = combineConsumeQueueStore.getRocksDBConsumeQueueStore();\n            assertNotNull(consumeQueueStore);\n            assertNotNull(rocksDBConsumeQueueStore);\n\n            consumeQueueStore.findOrCreateConsumeQueue(topic, queueId).initializeWithOffset(200, 0);\n\n            ConsumeQueueInterface cq = rocksDBConsumeQueueStore.findOrCreateConsumeQueue(topic, queueId);\n            Assert.assertEquals(msgNum, cq.getMaxOffsetInQueue());\n            Assert.assertEquals(0, cq.getMinOffsetInQueue());\n\n            combineConsumeQueueStore.verifyAndInitOffsetForAllStore(true);\n\n            Assert.assertEquals(200, cq.getMaxOffsetInQueue());\n            Assert.assertEquals(200, cq.getMinOffsetInQueue());\n\n            messageStore.shutdown();\n        }\n    }\n\n    @Test\n    public void testVerifyAndInitOffsetForAllStore() throws Exception {\n        final String path = createBaseDir();\n        topicConfigTableMap.put(topic, new TopicConfig(topic, 1, 1, PermName.PERM_WRITE | PermName.PERM_READ));\n\n        {\n            messageStoreConfig.setRocksdbCQDoubleWriteEnable(false);\n            messageStore = (DefaultMessageStore) createMessageStore(path, false, topicConfigTableMap, messageStoreConfig);\n            messageStore.load();\n            messageStore.start();\n\n            assertTrue(messageStore.getQueueStore() instanceof ConsumeQueueStore);\n\n            for (int i = 0; i < msgNum; i++) {\n                MessageExtBrokerInner msg = new MessageExtBrokerInner();\n\n                msg.setQueueId(queueId);\n                msg.setBody(new byte[msgSize]);\n                msg.setTopic(topic);\n                msg.setTags(\"TAG1\");\n\n                msg.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(SINGLE_TAG, msg.getTags()));\n                msg.setBornTimestamp(System.currentTimeMillis());\n                msg.setReconsumeTimes(0);\n\n                msg.setBornHost(new InetSocketAddress(9999));\n                msg.setStoreHost(new InetSocketAddress(8888));\n\n                messageStore.putMessage(msg);\n            }\n\n            await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> {\n                File cq = new File(path + File.separator + \"consumequeue\" + File.separator + topic + File.separator + queueId + File.separator + \"00000000000000000000\");\n                assertTrue(cq.exists());\n                Assert.assertEquals(msgNum, (long) messageStore.getQueueStore().getMaxOffset(topic, queueId));\n                Assert.assertEquals(0, messageStore.getQueueStore().getMinOffsetInQueue(topic, queueId));\n            });\n\n            messageStore.shutdown();\n        }\n\n        {\n            messageStoreConfig.setRocksdbCQDoubleWriteEnable(true);\n            messageStore = (DefaultMessageStore) createMessageStore(path, false, topicConfigTableMap, messageStoreConfig);\n            messageStore.load();\n            await().atMost(1, TimeUnit.SECONDS).untilAsserted(() -> {\n                assertTrue(((CombineConsumeQueueStore) messageStore.getQueueStore()).verifyAndInitOffsetForAllStore(false));\n            });\n            messageStore.start();\n            messageStore.shutdown();\n        }\n\n        {\n            messageStoreConfig.setRocksdbCQDoubleWriteEnable(true);\n            messageStore = (DefaultMessageStore) createMessageStore(path, false, topicConfigTableMap, messageStoreConfig);\n            messageStore.load();\n            await().atMost(1, TimeUnit.SECONDS).untilAsserted(() -> {\n                assertTrue(((CombineConsumeQueueStore) messageStore.getQueueStore()).verifyAndInitOffsetForAllStore(false));\n            });\n            messageStore.start();\n            messageStore.shutdown();\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.queue;\n\nimport java.util.List;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\n\nimport com.google.common.collect.Sets;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.LmqDispatch;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport java.io.File;\nimport java.util.UUID;\nimport java.util.stream.IntStream;\n\nimport static java.util.concurrent.TimeUnit.SECONDS;\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\n\npublic class ConsumeQueueStoreTest extends QueueTestBase {\n\n    private MessageStore messageStore;\n    private ConcurrentMap<String, TopicConfig> topicConfigTableMap;\n\n    @Before\n    public void init() throws Exception {\n        this.topicConfigTableMap = new ConcurrentHashMap<>();\n        messageStore = createMessageStore(null, true, topicConfigTableMap);\n        messageStore.load();\n        messageStore.start();\n    }\n\n    @After\n    public void destroy() {\n        messageStore.shutdown();\n        messageStore.destroy();\n\n        File file = new File(messageStore.getMessageStoreConfig().getStorePathRootDir());\n        UtilAll.deleteFile(file);\n    }\n\n    @Test\n    public void testLoadConsumeQueuesWithWrongAttribute() {\n        String normalTopic = UUID.randomUUID().toString();\n        ConcurrentMap<String, TopicConfig> topicConfigTable = createTopicConfigTable(normalTopic, CQType.SimpleCQ);\n        this.topicConfigTableMap.putAll(topicConfigTable);\n\n        for (int i = 0; i < 10; i++) {\n            PutMessageResult putMessageResult = messageStore.putMessage(buildMessage(normalTopic, -1));\n            assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n        }\n\n        await().atMost(5, SECONDS).until(fullyDispatched(messageStore));\n\n        // simulate delete topic but with files left.\n        this.topicConfigTableMap.clear();\n\n        topicConfigTable = createTopicConfigTable(normalTopic, CQType.BatchCQ);\n        this.topicConfigTableMap.putAll(topicConfigTable);\n\n        RuntimeException runtimeException = Assert.assertThrows(RuntimeException.class, () -> messageStore.getQueueStore().load());\n        Assert.assertTrue(runtimeException.getMessage().endsWith(\"should be SimpleCQ, but is BatchCQ\"));\n    }\n\n    @Test\n    public void testLoadBatchConsumeQueuesWithWrongAttribute() {\n        String batchTopic = UUID.randomUUID().toString();\n        ConcurrentMap<String, TopicConfig>  topicConfigTable = createTopicConfigTable(batchTopic, CQType.BatchCQ);\n        this.topicConfigTableMap.putAll(topicConfigTable);\n\n        for (int i = 0; i < 10; i++) {\n            PutMessageResult putMessageResult = messageStore.putMessage(buildMessage(batchTopic, 10));\n            assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n        }\n\n        await().atMost(5, SECONDS).until(fullyDispatched(messageStore));\n\n        // simulate delete topic but with files left.\n        this.topicConfigTableMap.clear();\n\n        topicConfigTable = createTopicConfigTable(batchTopic, CQType.SimpleCQ);\n        this.topicConfigTableMap.putAll(topicConfigTable);\n        messageStore.shutdown();\n\n        RuntimeException runtimeException = Assert.assertThrows(RuntimeException.class, () -> messageStore.getQueueStore().load());\n        Assert.assertTrue(runtimeException.getMessage().endsWith(\"should be BatchCQ, but is SimpleCQ\"));\n    }\n\n    @Test\n    public void testLmqCounter_running() throws ConsumeQueueException {\n        messageStore.getMessageStoreConfig().setEnableMultiDispatch(true);\n        messageStore.getMessageStoreConfig().setEnableLmq(true);\n        messageStore.getMessageStoreConfig().setEnableCompaction(false);\n        int num = 5;\n        String topic = \"topic\";\n        List<String> lmqNameList = IntStream.range(0, num)\n            .mapToObj(i -> MixAll.LMQ_PREFIX + UUID.randomUUID())\n            .collect(java.util.stream.Collectors.toList());\n        assertEquals(0, messageStore.getQueueStore().getLmqNum());\n\n        lmqNameList.forEach(lmqName -> assertNull(messageStore.getConsumeQueue(lmqName, 0)));\n        assertEquals(0, messageStore.getQueueStore().getLmqNum());\n\n        for (String lmqName : lmqNameList) {\n            MessageExtBrokerInner message = buildMessage(topic, -1);\n            MessageAccessor.putProperty(message, MessageConst.PROPERTY_INNER_MULTI_DISPATCH, lmqName);\n            LmqDispatch.wrapLmqDispatch(messageStore, message);\n            PutMessageResult putMessageResult = messageStore.putMessage(message);\n            assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n        }\n        await().atMost(5, SECONDS).until(fullyDispatched(messageStore));\n\n        lmqNameList.forEach(lmqName -> assertNotNull(messageStore.getConsumeQueue(lmqName, 0)));\n        assertEquals(num, messageStore.getQueueStore().getLmqNum());\n\n        lmqNameList.forEach(lmqName -> messageStore.deleteTopics(Sets.newHashSet(lmqName)));\n        assertEquals(0, messageStore.getQueueStore().getLmqNum());\n    }\n\n    @Test\n    public void testLmqCounter_reload() throws Exception {\n        messageStore.getMessageStoreConfig().setEnableMultiDispatch(true);\n        messageStore.getMessageStoreConfig().setEnableLmq(true);\n        int num = 5;\n        String topic = \"topic\";\n        List<String> lmqNameList = IntStream.range(0, num)\n            .mapToObj(i -> MixAll.LMQ_PREFIX + UUID.randomUUID())\n            .collect(java.util.stream.Collectors.toList());\n        assertEquals(0, messageStore.getQueueStore().getLmqNum());\n\n        for (String lmqName : lmqNameList) {\n            MessageExtBrokerInner message = buildMessage(topic, -1);\n            MessageAccessor.putProperty(message, MessageConst.PROPERTY_INNER_MULTI_DISPATCH, lmqName);\n            LmqDispatch.wrapLmqDispatch(messageStore, message);\n            PutMessageResult putMessageResult = messageStore.putMessage(message);\n            assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n        }\n        await().atMost(5, SECONDS).until(fullyDispatched(messageStore));\n        assertEquals(num, messageStore.getQueueStore().getLmqNum());\n        messageStore.shutdown();\n\n        // create new one based on current store\n        MessageStore newStore = createMessageStore(messageStore.getMessageStoreConfig().getStorePathRootDir(),\n            true, topicConfigTableMap, messageStore.getMessageStoreConfig());\n        newStore.load();\n        newStore.start();\n\n        assertEquals(num, newStore.getQueueStore().getLmqNum());\n        lmqNameList.forEach(lmqName -> assertNotNull(newStore.getConsumeQueue(lmqName, 0)));\n        newStore.shutdown();\n    }\n\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/queue/ConsumeQueueTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.queue;\n\nimport java.io.File;\nimport java.nio.ByteBuffer;\nimport java.util.Map;\nimport java.util.UUID;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.ConsumeQueueExt;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.MessageFilter;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.RocksDBMessageStore;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport static java.util.concurrent.TimeUnit.SECONDS;\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.assertj.core.api.Assertions.fail;\nimport static org.awaitility.Awaitility.await;\n\npublic class ConsumeQueueTest extends QueueTestBase {\n\n    private static final String TOPIC = \"StoreTest\";\n    private static final int QUEUE_ID = 0;\n    private static final String STORE_PATH = \".\" + File.separator + \"unit_test_store\";\n    private static final int COMMIT_LOG_FILE_SIZE = 1024 * 8;\n    private static final int CQ_FILE_SIZE = 10 * 20;\n    private static final int CQ_EXT_FILE_SIZE = 10 * (ConsumeQueueExt.CqExtUnit.MIN_EXT_UNIT_SIZE + 64);\n\n    public MessageStoreConfig buildStoreConfig(int commitLogFileSize, int cqFileSize,\n        boolean enableCqExt, int cqExtFileSize) {\n        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();\n        messageStoreConfig.setMappedFileSizeCommitLog(commitLogFileSize);\n        messageStoreConfig.setMappedFileSizeConsumeQueue(cqFileSize);\n        messageStoreConfig.setMappedFileSizeConsumeQueueExt(cqExtFileSize);\n        messageStoreConfig.setMessageIndexEnable(false);\n        messageStoreConfig.setEnableConsumeQueueExt(enableCqExt);\n\n        messageStoreConfig.setStorePathRootDir(STORE_PATH);\n        messageStoreConfig.setStorePathCommitLog(STORE_PATH + File.separator + \"commitlog\");\n\n        return messageStoreConfig;\n    }\n\n    protected DefaultMessageStore gen() throws Exception {\n        MessageStoreConfig messageStoreConfig = buildStoreConfig(\n            COMMIT_LOG_FILE_SIZE, CQ_FILE_SIZE, true, CQ_EXT_FILE_SIZE\n        );\n\n        BrokerConfig brokerConfig = new BrokerConfig();\n\n        DefaultMessageStore master = new DefaultMessageStore(\n            messageStoreConfig, new BrokerStatsManager(brokerConfig),\n            (topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties) -> {\n            }, brokerConfig, new ConcurrentHashMap<>());\n\n        assertThat(master.load()).isTrue();\n\n        master.start();\n\n        return master;\n    }\n\n    protected RocksDBMessageStore genRocksdbMessageStore() throws Exception {\n        MessageStoreConfig messageStoreConfig = buildStoreConfig(\n            COMMIT_LOG_FILE_SIZE, CQ_FILE_SIZE, true, CQ_EXT_FILE_SIZE\n        );\n\n        BrokerConfig brokerConfig = new BrokerConfig();\n\n        RocksDBMessageStore master = new RocksDBMessageStore(\n            messageStoreConfig, new BrokerStatsManager(brokerConfig),\n            (topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties) -> {\n            }, brokerConfig, new ConcurrentHashMap<>());\n\n        assertThat(master.load()).isTrue();\n\n        master.start();\n\n        return master;\n    }\n\n    protected void putMsg(MessageStore messageStore) {\n        int totalMsgs = 200;\n        for (int i = 0; i < totalMsgs; i++) {\n            MessageExtBrokerInner message = buildMessage();\n            message.setQueueId(0);\n            switch (i % 3) {\n                case 0:\n                    message.setTags(\"TagA\");\n                    break;\n\n                case 1:\n                    message.setTags(\"TagB\");\n                    break;\n\n                case 2:\n                    message.setTags(\"TagC\");\n                    break;\n            }\n            message.setTagsCode(message.getTags().hashCode());\n            message.setPropertiesString(MessageDecoder.messageProperties2String(message.getProperties()));\n            messageStore.putMessage(message);\n        }\n        await().atMost(5, SECONDS).until(fullyDispatched(messageStore));\n    }\n\n    @Test\n    public void testIterator() throws Exception {\n        final int msgNum = 100;\n        final int msgSize = 1000;\n        MessageStore messageStore =  createMessageStore(null, true, null);\n        messageStore.load();\n        String topic = UUID.randomUUID().toString();\n        //The initial min max offset, before and after the creation of consume queue\n        Assert.assertEquals(0, messageStore.getMaxOffsetInQueue(topic, 0));\n        Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0));\n\n        ConsumeQueueInterface consumeQueue = messageStore.getConsumeQueue(topic, 0);\n        Assert.assertEquals(CQType.SimpleCQ, consumeQueue.getCQType());\n        Assert.assertEquals(0, consumeQueue.getMaxOffsetInQueue());\n        Assert.assertEquals(0, consumeQueue.getMinOffsetInQueue());\n        Assert.assertEquals(0, messageStore.getMaxOffsetInQueue(topic, 0));\n        Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0));\n        for (int i = 0; i < msgNum; i++) {\n            DispatchRequest request = new DispatchRequest(consumeQueue.getTopic(), consumeQueue.getQueueId(), i * msgSize, msgSize, i,\n                System.currentTimeMillis(), i, null, null, 0, 0, null);\n            request.setBitMap(new byte[10]);\n            ((AbstractConsumeQueueStore) messageStore.getQueueStore()).putMessagePositionInfoWrapper(consumeQueue, request);\n        }\n        Assert.assertEquals(0, consumeQueue.getMinLogicOffset());\n        Assert.assertEquals(0, consumeQueue.getMinOffsetInQueue());\n        Assert.assertEquals(msgNum, consumeQueue.getMaxOffsetInQueue());\n        Assert.assertEquals(msgNum, consumeQueue.getMessageTotalInQueue());\n        //TO DO Should test it\n        //Assert.assertEquals(100 * 100, consumeQueue.getMaxPhysicOffset());\n\n\n        Assert.assertNull(consumeQueue.iterateFrom(-1));\n        Assert.assertNull(consumeQueue.iterateFrom(msgNum));\n\n        {\n            CqUnit first = consumeQueue.getEarliestUnit();\n            Assert.assertNotNull(first);\n            Assert.assertEquals(0, first.getQueueOffset());\n            Assert.assertEquals(msgSize, first.getSize());\n            Assert.assertTrue(first.isTagsCodeValid());\n        }\n        {\n            CqUnit last = consumeQueue.getLatestUnit();\n            Assert.assertNotNull(last);\n            Assert.assertEquals(msgNum - 1, last.getQueueOffset());\n            Assert.assertEquals(msgSize, last.getSize());\n            Assert.assertTrue(last.isTagsCodeValid());\n        }\n\n        for (int i = 0; i < msgNum; i++) {\n            ReferredIterator<CqUnit> iterator = consumeQueue.iterateFrom(i);\n            Assert.assertNotNull(iterator);\n            long queueOffset = i;\n            while (iterator.hasNext()) {\n                CqUnit cqUnit =  iterator.next();\n                Assert.assertEquals(queueOffset, cqUnit.getQueueOffset());\n                Assert.assertEquals(queueOffset * msgSize, cqUnit.getPos());\n                Assert.assertEquals(msgSize, cqUnit.getSize());\n                Assert.assertTrue(cqUnit.isTagsCodeValid());\n                Assert.assertEquals(queueOffset, cqUnit.getTagsCode());\n                Assert.assertEquals(queueOffset, cqUnit.getValidTagsCodeAsLong().longValue());\n                Assert.assertEquals(1, cqUnit.getBatchNum());\n                Assert.assertNotNull(cqUnit.getCqExtUnit());\n                ConsumeQueueExt.CqExtUnit cqExtUnit =  cqUnit.getCqExtUnit();\n                Assert.assertEquals(queueOffset, cqExtUnit.getTagsCode());\n                Assert.assertArrayEquals(new byte[10], cqExtUnit.getFilterBitMap());\n                queueOffset++;\n            }\n            Assert.assertEquals(msgNum, queueOffset);\n        }\n        ((AbstractConsumeQueueStore) messageStore.getQueueStore()).destroy(consumeQueue);\n    }\n\n    @Test\n    public void testEstimateMessageCountInEmptyConsumeQueue() {\n        DefaultMessageStore messageStore = null;\n        try {\n            messageStore = gen();\n            doTestEstimateMessageCountInEmptyConsumeQueue(messageStore);\n        } catch (Exception e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n    }\n\n    @Test\n    public void testEstimateRocksdbMessageCountInEmptyConsumeQueue() {\n        if (notExecuted()) {\n            return;\n        }\n        DefaultMessageStore messageStore = null;\n        try {\n            messageStore = genRocksdbMessageStore();\n            doTestEstimateMessageCountInEmptyConsumeQueue(messageStore);\n        } catch (Exception e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n    }\n\n    public void doTestEstimateMessageCountInEmptyConsumeQueue(MessageStore master) {\n        try {\n            ConsumeQueueInterface consumeQueue = master.findConsumeQueue(TOPIC, QUEUE_ID);\n            MessageFilter filter = new MessageFilter() {\n                @Override\n                public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {\n                    return tagsCode == \"TagA\".hashCode();\n                }\n\n                @Override\n                public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map<String, String> properties) {\n                    return false;\n                }\n            };\n            long estimation = consumeQueue.estimateMessageCount(0, 0, filter);\n            Assert.assertEquals(-1, estimation);\n\n            // test for illegal offset\n            estimation = consumeQueue.estimateMessageCount(0, 100, filter);\n            Assert.assertEquals(-1, estimation);\n            estimation = consumeQueue.estimateMessageCount(100, 1000, filter);\n            Assert.assertEquals(-1, estimation);\n        } catch (Exception e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        } finally {\n            if (master != null) {\n                master.shutdown();\n                master.destroy();\n            }\n            UtilAll.deleteFile(new File(STORE_PATH));\n        }\n    }\n\n    @Test\n    public void testEstimateRocksdbMessageCount() {\n        if (notExecuted()) {\n            return;\n        }\n        DefaultMessageStore messageStore = null;\n        try {\n            messageStore = genRocksdbMessageStore();\n            doTestEstimateMessageCount(messageStore);\n        } catch (Exception e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n    }\n\n    @Test\n    public void testEstimateMessageCount() {\n        DefaultMessageStore messageStore = null;\n        try {\n            messageStore = gen();\n            doTestEstimateMessageCount(messageStore);\n        } catch (Exception e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n    }\n\n    public void doTestEstimateMessageCount(MessageStore messageStore) {\n        try {\n            try {\n                putMsg(messageStore);\n            } catch (Exception e) {\n                fail(\"Failed to put message\", e);\n            }\n\n            ConsumeQueueInterface cq = messageStore.findConsumeQueue(TOPIC, QUEUE_ID);\n            MessageFilter filter = new MessageFilter() {\n                @Override\n                public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {\n                    return tagsCode == \"TagA\".hashCode();\n                }\n\n                @Override\n                public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map<String, String> properties) {\n                    return false;\n                }\n            };\n            long estimation = cq.estimateMessageCount(0, 199, filter);\n            Assert.assertEquals(67, estimation);\n\n            // test for illegal offset\n            estimation = cq.estimateMessageCount(0, 1000, filter);\n            Assert.assertEquals(67, estimation);\n            estimation = cq.estimateMessageCount(1000, 10000, filter);\n            Assert.assertEquals(-1, estimation);\n            estimation = cq.estimateMessageCount(100, 0, filter);\n            Assert.assertEquals(-1, estimation);\n        } finally {\n            messageStore.shutdown();\n            messageStore.destroy();\n            UtilAll.deleteFile(new File(STORE_PATH));\n        }\n    }\n\n    @Test\n    public void testEstimateRocksdbMessageCountSample() {\n        if (notExecuted()) {\n            return;\n        }\n        DefaultMessageStore messageStore = null;\n        try {\n            messageStore = genRocksdbMessageStore();\n            doTestEstimateMessageCountSample(messageStore);\n        } catch (Exception e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n    }\n\n    @Test\n    public void testEstimateMessageCountSample() {\n        DefaultMessageStore messageStore = null;\n        try {\n            messageStore = gen();\n            doTestEstimateMessageCountSample(messageStore);\n        } catch (Exception e) {\n            e.printStackTrace();\n            assertThat(Boolean.FALSE).isTrue();\n        }\n    }\n\n    public void doTestEstimateMessageCountSample(MessageStore messageStore) {\n\n        try {\n            try {\n                putMsg(messageStore);\n            } catch (Exception e) {\n                fail(\"Failed to put message\", e);\n            }\n            messageStore.getMessageStoreConfig().setSampleCountThreshold(10);\n            messageStore.getMessageStoreConfig().setMaxConsumeQueueScan(20);\n            ConsumeQueueInterface cq = messageStore.findConsumeQueue(TOPIC, QUEUE_ID);\n            MessageFilter filter = new MessageFilter() {\n                @Override\n                public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {\n                    return tagsCode == \"TagA\".hashCode();\n                }\n\n                @Override\n                public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map<String, String> properties) {\n                    return false;\n                }\n            };\n            long estimation = cq.estimateMessageCount(100, 150, filter);\n            Assert.assertEquals(15, estimation);\n        } finally {\n            messageStore.shutdown();\n            messageStore.destroy();\n            UtilAll.deleteFile(new File(STORE_PATH));\n        }\n    }\n\n    private boolean notExecuted() {\n        return MixAll.isMac();\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/queue/QueueTestBase.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.queue;\n\nimport java.io.File;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.concurrent.Callable;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.TopicAttributes;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.apache.rocketmq.store.ConsumeQueue;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.RocksDBMessageStore;\nimport org.apache.rocketmq.store.StoreTestBase;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\n\npublic class QueueTestBase extends StoreTestBase {\n\n    protected ConcurrentMap<String, TopicConfig> createTopicConfigTable(String topic, CQType cqType) {\n        ConcurrentMap<String, TopicConfig> topicConfigTable = new ConcurrentHashMap<>();\n        TopicConfig topicConfigToBeAdded = new TopicConfig();\n\n        Map<String, String> attributes = new HashMap<>();\n        attributes.put(TopicAttributes.QUEUE_TYPE_ATTRIBUTE.getName(), cqType.toString());\n        topicConfigToBeAdded.setTopicName(topic);\n        topicConfigToBeAdded.setAttributes(attributes);\n\n        topicConfigTable.put(topic, topicConfigToBeAdded);\n        return topicConfigTable;\n    }\n\n    protected Callable<Boolean> fullyDispatched(MessageStore messageStore) {\n        return () -> messageStore.dispatchBehindBytes() == 0;\n    }\n\n    protected MessageStore createMessageStore(String baseDir, boolean extent,\n        ConcurrentMap<String, TopicConfig> topicConfigTable) throws Exception {\n        return createMessageStore(baseDir, extent, topicConfigTable, new MessageStoreConfig());\n    }\n\n    protected MessageStore createMessageStore(String baseDir, boolean extent,\n        ConcurrentMap<String, TopicConfig> topicConfigTable, MessageStoreConfig messageStoreConfig) throws Exception {\n        if (baseDir == null) {\n            baseDir = createBaseDir();\n        }\n        baseDirs.add(baseDir);\n        messageStoreConfig.setMappedFileSizeCommitLog(1024 * 8);\n        messageStoreConfig.setMappedFileSizeConsumeQueue(100 * ConsumeQueue.CQ_STORE_UNIT_SIZE);\n        messageStoreConfig.setMapperFileSizeBatchConsumeQueue(20 * BatchConsumeQueue.CQ_STORE_UNIT_SIZE);\n        messageStoreConfig.setMappedFileSizeConsumeQueueExt(1024);\n        messageStoreConfig.setMaxIndexNum(100 * 10);\n        messageStoreConfig.setEnableConsumeQueueExt(extent);\n        messageStoreConfig.setStorePathRootDir(baseDir);\n        messageStoreConfig.setStorePathCommitLog(baseDir + File.separator + \"commitlog\");\n        messageStoreConfig.setHaListenPort(nextPort());\n        messageStoreConfig.setMaxTransferBytesOnMessageInDisk(1024 * 1024);\n        messageStoreConfig.setMaxTransferBytesOnMessageInMemory(1024 * 1024);\n        messageStoreConfig.setMaxTransferCountOnMessageInDisk(1024);\n        messageStoreConfig.setMaxTransferCountOnMessageInMemory(1024);\n\n        messageStoreConfig.setFlushIntervalCommitLog(1);\n        messageStoreConfig.setFlushCommitLogThoroughInterval(2);\n\n\n        MessageStore messageStore;\n        if (messageStoreConfig.isEnableRocksDBStore()) {\n            messageStore = new RocksDBMessageStore(\n                messageStoreConfig,\n                new BrokerStatsManager(\"simpleTest\", true),\n                (topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties) -> {\n                },\n                new BrokerConfig(), topicConfigTable);\n        } else {\n            messageStore = new DefaultMessageStore(\n                messageStoreConfig,\n                new BrokerStatsManager(\"simpleTest\", true),\n                (topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties) -> {\n                },\n                new BrokerConfig(), topicConfigTable);\n        }\n        return messageStore;\n    }\n\n    public MessageExtBrokerInner buildMessage(String topic, int batchNum) {\n        MessageExtBrokerInner msg = new MessageExtBrokerInner();\n        msg.setTopic(topic);\n        msg.setTags(\"TAG1\");\n        msg.setKeys(\"Hello\");\n        msg.setBody(new byte[1024]);\n        msg.setKeys(String.valueOf(System.currentTimeMillis()));\n        msg.setQueueId(0);\n        msg.setSysFlag(0);\n        msg.setBornTimestamp(System.currentTimeMillis());\n        msg.setStoreHost(storeHost);\n        msg.setBornHost(storeHost);\n        MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_NUM, String.valueOf(batchNum));\n        msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n        if (batchNum > 1) {\n            msg.setSysFlag(MessageSysFlag.INNER_BATCH_FLAG);\n        }\n        if (batchNum == -1) {\n            MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_INNER_NUM);\n        }\n        return msg;\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTableTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.queue;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport java.nio.charset.StandardCharsets;\nimport java.util.UUID;\nimport java.util.concurrent.atomic.AtomicBoolean;\n\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.queue.offset.OffsetEntryType;\nimport org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage;\nimport org.junit.AfterClass;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.BeforeClass;\nimport org.junit.Test;\nimport org.junit.rules.TemporaryFolder;\nimport org.junit.runner.RunWith;\nimport org.mockito.Mock;\nimport org.mockito.Mockito;\nimport org.mockito.junit.MockitoJUnitRunner;\nimport org.rocksdb.Options;\nimport org.rocksdb.RocksDB;\nimport org.rocksdb.RocksDBException;\nimport org.rocksdb.RocksIterator;\nimport org.rocksdb.WriteBatch;\nimport org.rocksdb.WriteOptions;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class RocksDBConsumeQueueOffsetTableTest {\n\n    private RocksDBConsumeQueueOffsetTable offsetTable;\n\n    @Mock\n    private ConsumeQueueRocksDBStorage rocksDBStorage;\n\n    @Mock\n    private RocksDBConsumeQueueTable consumeQueueTable;\n\n    @Mock\n    private DefaultMessageStore messageStore;\n\n    private static RocksDB db;\n\n    private static File dbPath;\n\n    private static String topicName;\n\n    @BeforeClass\n    public static void initDB() throws IOException, RocksDBException {\n        TemporaryFolder tempFolder = new TemporaryFolder();\n        tempFolder.create();\n        dbPath = tempFolder.newFolder();\n\n        db = RocksDB.open(dbPath.getAbsolutePath());\n        StringBuilder topicBuilder = new StringBuilder();\n        for (int i = 0; i < 100; i++) {\n            topicBuilder.append(\"topic\");\n        }\n        topicName = topicBuilder.toString();\n        writeOffset(topicName, 1, 100, 2, true);\n    }\n\n    @AfterClass\n    public static void tearDownDB() throws RocksDBException {\n        db.closeE();\n        RocksDB.destroyDB(dbPath.getAbsolutePath(), new Options());\n    }\n\n    @Before\n    public void setUp() {\n        RocksIterator iterator = db.newIterator();\n        Mockito.doReturn(iterator).when(rocksDBStorage).seekOffsetCF();\n        offsetTable = new RocksDBConsumeQueueOffsetTable(consumeQueueTable, rocksDBStorage, messageStore);\n    }\n\n    /**\n     * Verify forEach can expand key-buffer properly and works well for long topic names.\n     *\n     * @throws RocksDBException If there is an RocksDB error.\n     */\n    @Test\n    public void testForEach() throws RocksDBException {\n        AtomicBoolean called = new AtomicBoolean(false);\n        offsetTable.forEach(entry -> true, entry -> {\n            called.set(true);\n            Assert.assertEquals(topicName, entry.topic);\n            Assert.assertTrue(topicName.length() > 256);\n            Assert.assertEquals(1, entry.queueId);\n            Assert.assertEquals(100, entry.commitLogOffset);\n            Assert.assertEquals(2, entry.offset);\n            Assert.assertEquals(OffsetEntryType.MAXIMUM, entry.type);\n        });\n        Assert.assertTrue(called.get());\n    }\n\n    @Test\n    public void testLmqCounter() throws RocksDBException {\n        Assert.assertEquals(0, offsetTable.getLmqNum());\n        offsetTable.load();\n        int initCount = offsetTable.getLmqNum();\n        int lmqCount = 2;\n        int repeatCount = 3;\n        for (int i = 0; i < lmqCount; i++) {\n            String lmqName = MixAll.LMQ_PREFIX + UUID.randomUUID();\n            String normalTopic = UUID.randomUUID().toString();\n            for (int j = 0; j < repeatCount; j++) {\n                writeOffset(lmqName, 0, 100, j, true);\n                writeOffset(lmqName, 0, 100, j, false);\n                writeOffset(normalTopic, 0, 100, j, true);\n                writeOffset(normalTopic, 0, 100, j, false);\n            }\n        }\n\n        Mockito.doReturn(db.newIterator()).when(rocksDBStorage).seekOffsetCF();\n        offsetTable.load();\n        Assert.assertEquals(initCount + lmqCount, offsetTable.getLmqNum());\n    }\n\n    private static void writeOffset(String topic, int queueId, long phyOffset,\n        long cqOffset, boolean max) throws RocksDBException {\n        byte[] topicInBytes = topic.getBytes(StandardCharsets.UTF_8);\n        ByteBuffer keyBuffer = ByteBuffer.allocateDirect(\n            RocksDBConsumeQueueOffsetTable.OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicInBytes.length);\n        RocksDBConsumeQueueOffsetTable.buildOffsetKeyByteBuffer(keyBuffer, topicInBytes, 1, max);\n        Assert.assertEquals(0, keyBuffer.position());\n        Assert.assertEquals(RocksDBConsumeQueueOffsetTable.OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES\n            + topicInBytes.length, keyBuffer.limit());\n\n        ByteBuffer valueBuffer = ByteBuffer.allocateDirect(Long.BYTES + Long.BYTES);\n        valueBuffer.putLong(phyOffset);\n        valueBuffer.putLong(cqOffset);\n        valueBuffer.flip();\n\n        try (WriteBatch writeBatch = new WriteBatch();\n             WriteOptions writeOptions = new WriteOptions()) {\n            writeOptions.setDisableWAL(false);\n            writeOptions.setSync(true);\n            writeBatch.put(keyBuffer, valueBuffer);\n            db.write(writeOptions, writeBatch);\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTableTest.java",
    "content": "/*\r\n * Licensed to the Apache Software Foundation (ASF) under one or more\r\n * contributor license agreements.  See the NOTICE file distributed with\r\n * this work for additional information regarding copyright ownership.\r\n * The ASF licenses this file to You under the Apache License, Version 2.0\r\n * (the \"License\"); you may not use this file except in compliance with\r\n * the License.  You may obtain a copy of the License at\r\n *\r\n *     http://www.apache.org/licenses/LICENSE-2.0\r\n *\r\n * Unless required by applicable law or agreed to in writing, software\r\n * distributed under the License is distributed on an \"AS IS\" BASIS,\r\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r\n * See the License for the specific language governing permissions and\r\n * limitations under the License.\r\n */\r\npackage org.apache.rocketmq.store.queue;\r\n\r\nimport org.apache.rocketmq.common.BoundaryType;\r\nimport org.apache.rocketmq.common.MixAll;\r\nimport org.apache.rocketmq.store.DefaultMessageStore;\r\nimport org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage;\r\nimport org.junit.Test;\r\nimport org.mockito.stubbing.Answer;\r\nimport org.rocksdb.RocksDBException;\r\n\r\nimport java.nio.ByteBuffer;\r\n\r\nimport static org.apache.rocketmq.store.queue.RocksDBConsumeQueueTable.CQ_UNIT_SIZE;\r\nimport static org.junit.Assert.assertEquals;\r\nimport static org.mockito.ArgumentMatchers.any;\r\nimport static org.mockito.Mockito.doAnswer;\r\nimport static org.mockito.Mockito.mock;\r\n\r\npublic class RocksDBConsumeQueueTableTest {\r\n\r\n    @Test\r\n    public void testBinarySearchInCQByTime() throws RocksDBException {\r\n        if (MixAll.isMac()) {\r\n            return;\r\n        }\r\n        ConsumeQueueRocksDBStorage rocksDBStorage = mock(ConsumeQueueRocksDBStorage.class);\r\n        DefaultMessageStore store = mock(DefaultMessageStore.class);\r\n        RocksDBConsumeQueueTable table = new RocksDBConsumeQueueTable(rocksDBStorage, store);\r\n        doAnswer((Answer<byte[]>) mock -> {\r\n            /*\r\n             * queueOffset timestamp\r\n             * 100         1000\r\n             * 200         2000\r\n             * 201         2010\r\n             * 1000        10000\r\n             */\r\n            byte[] keyBytes = mock.getArgument(0);\r\n            ByteBuffer keyBuffer = ByteBuffer.wrap(keyBytes);\r\n            int len = keyBuffer.getInt(0);\r\n            long offset = keyBuffer.getLong(4 + 1 + len + 1 + 4 + 1);\r\n            long phyOffset = offset;\r\n            long timestamp = offset * 10;\r\n            final ByteBuffer byteBuffer = ByteBuffer.allocate(CQ_UNIT_SIZE);\r\n            byteBuffer.putLong(phyOffset);\r\n            byteBuffer.putInt(1);\r\n            byteBuffer.putLong(0);\r\n            byteBuffer.putLong(timestamp);\r\n            return byteBuffer.array();\r\n        }).when(rocksDBStorage).getCQ(any());\r\n        assertEquals(1001, table.binarySearchInCQByTime(\"topic\", 0, 1000, 100, 20000, 0, BoundaryType.LOWER));\r\n        assertEquals(1000, table.binarySearchInCQByTime(\"topic\", 0, 1000, 100, 20000, 0, BoundaryType.UPPER));\r\n        assertEquals(100, table.binarySearchInCQByTime(\"topic\", 0, 1000, 100, 1, 0, BoundaryType.LOWER));\r\n        assertEquals(0, table.binarySearchInCQByTime(\"topic\", 0, 1000, 100, 1, 0, BoundaryType.UPPER));\r\n        assertEquals(201, table.binarySearchInCQByTime(\"topic\", 0, 1000, 100, 2001, 0, BoundaryType.LOWER));\r\n        assertEquals(200, table.binarySearchInCQByTime(\"topic\", 0, 1000, 100, 2001, 0, BoundaryType.UPPER));\r\n        assertEquals(200, table.binarySearchInCQByTime(\"topic\", 0, 1000, 100, 2000, 0, BoundaryType.LOWER));\r\n        assertEquals(200, table.binarySearchInCQByTime(\"topic\", 0, 1000, 100, 2000, 0, BoundaryType.UPPER));\r\n    }\r\n}"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTest.java",
    "content": "/*\r\n * Licensed to the Apache Software Foundation (ASF) under one or more\r\n * contributor license agreements.  See the NOTICE file distributed with\r\n * this work for additional information regarding copyright ownership.\r\n * The ASF licenses this file to You under the Apache License, Version 2.0\r\n * (the \"License\"); you may not use this file except in compliance with\r\n * the License.  You may obtain a copy of the License at\r\n *\r\n *     http://www.apache.org/licenses/LICENSE-2.0\r\n *\r\n * Unless required by applicable law or agreed to in writing, software\r\n * distributed under the License is distributed on an \"AS IS\" BASIS,\r\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r\n * See the License for the specific language governing permissions and\r\n * limitations under the License.\r\n */\r\npackage org.apache.rocketmq.store.queue;\r\n\r\nimport java.io.File;\r\nimport java.nio.ByteBuffer;\r\nimport java.util.List;\r\nimport java.util.UUID;\r\nimport java.util.concurrent.ConcurrentHashMap;\r\nimport java.util.concurrent.ConcurrentMap;\r\nimport java.util.stream.IntStream;\r\n\r\nimport com.google.common.collect.Sets;\r\nimport org.apache.rocketmq.common.MixAll;\r\nimport org.apache.rocketmq.common.TopicConfig;\r\nimport org.apache.rocketmq.common.UtilAll;\r\nimport org.apache.rocketmq.common.message.MessageAccessor;\r\nimport org.apache.rocketmq.common.message.MessageConst;\r\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\r\nimport org.apache.rocketmq.store.DefaultMessageStore;\r\nimport org.apache.rocketmq.store.LmqDispatch;\r\nimport org.apache.rocketmq.store.MessageStore;\r\nimport org.apache.rocketmq.store.PutMessageResult;\r\nimport org.apache.rocketmq.store.PutMessageStatus;\r\nimport org.apache.rocketmq.store.StoreType;\r\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\r\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\r\nimport org.junit.After;\r\nimport org.junit.Before;\r\nimport org.junit.Test;\r\nimport org.mockito.invocation.InvocationOnMock;\r\nimport org.mockito.stubbing.Answer;\r\n\r\nimport static java.util.concurrent.TimeUnit.SECONDS;\r\nimport static org.apache.rocketmq.store.queue.RocksDBConsumeQueueTable.CQ_UNIT_SIZE;\r\nimport static org.awaitility.Awaitility.await;\r\nimport static org.junit.Assert.assertEquals;\r\nimport static org.junit.Assert.assertFalse;\r\nimport static org.junit.Assert.assertNotNull;\r\nimport static org.junit.Assert.assertNull;\r\nimport static org.junit.Assert.assertTrue;\r\nimport static org.mockito.ArgumentMatchers.anyInt;\r\nimport static org.mockito.ArgumentMatchers.anyLong;\r\nimport static org.mockito.ArgumentMatchers.anyString;\r\nimport static org.mockito.Mockito.mock;\r\nimport static org.mockito.Mockito.when;\r\n\r\npublic class RocksDBConsumeQueueTest extends QueueTestBase {\r\n\r\n    private MessageStore messageStore;\r\n    private ConcurrentMap<String, TopicConfig> topicConfigTableMap;\r\n\r\n    @Before\r\n    public void init() throws Exception {\r\n        MessageStoreConfig storeConfig = new MessageStoreConfig();\r\n        storeConfig.setStoreType(StoreType.DEFAULT_ROCKSDB.getStoreType());\r\n        storeConfig.setEnableCompaction(false);\r\n        this.topicConfigTableMap = new ConcurrentHashMap<>();\r\n\r\n        messageStore = createMessageStore(null, true, topicConfigTableMap, storeConfig);\r\n        messageStore.load();\r\n        messageStore.start();\r\n    }\r\n\r\n    @After\r\n    public void destroy() {\r\n        messageStore.shutdown();\r\n        messageStore.destroy();\r\n\r\n        File file = new File(messageStore.getMessageStoreConfig().getStorePathRootDir());\r\n        UtilAll.deleteFile(file);\r\n    }\r\n\r\n    @Test\r\n    public void testIterator() throws Exception {\r\n        if (MixAll.isMac()) {\r\n            return;\r\n        }\r\n        DefaultMessageStore messageStore = mock(DefaultMessageStore.class);\r\n        RocksDBConsumeQueueStore rocksDBConsumeQueueStore = mock(RocksDBConsumeQueueStore.class);\r\n        when(messageStore.getQueueStore()).thenReturn(rocksDBConsumeQueueStore);\r\n        when(rocksDBConsumeQueueStore.getMaxOffsetInQueue(anyString(), anyInt())).thenReturn(10000L);\r\n        when(rocksDBConsumeQueueStore.get(anyString(), anyInt(), anyLong())).then(new Answer<ByteBuffer>() {\r\n            @Override\r\n            public ByteBuffer answer(InvocationOnMock mock) throws Throwable {\r\n                long startIndex = mock.getArgument(2);\r\n                final ByteBuffer byteBuffer = ByteBuffer.allocate(CQ_UNIT_SIZE);\r\n                long phyOffset = startIndex * 10;\r\n                byteBuffer.putLong(phyOffset);\r\n                byteBuffer.putInt(1);\r\n                byteBuffer.putLong(0);\r\n                byteBuffer.putLong(0);\r\n                byteBuffer.flip();\r\n                return byteBuffer;\r\n            }\r\n        });\r\n\r\n        RocksDBConsumeQueue consumeQueue = new RocksDBConsumeQueue(messageStore.getMessageStoreConfig(), rocksDBConsumeQueueStore, \"topic\", 0);\r\n        ReferredIterator<CqUnit> it = consumeQueue.iterateFrom(9000);\r\n        for (int i = 0; i < 1000; i++) {\r\n            assertTrue(it.hasNext());\r\n            CqUnit next = it.next();\r\n            assertEquals(9000 + i, next.getQueueOffset());\r\n            assertEquals(10 * (9000 + i), next.getPos());\r\n        }\r\n        assertFalse(it.hasNext());\r\n    }\r\n\r\n    @Test\r\n    public void testLmqCounter_running() throws ConsumeQueueException {\r\n        messageStore.getMessageStoreConfig().setEnableMultiDispatch(true);\r\n        messageStore.getMessageStoreConfig().setEnableLmq(true);\r\n        int num = 5;\r\n        String topic = \"topic\";\r\n        List<String> lmqNameList = IntStream.range(0, num)\r\n            .mapToObj(i -> MixAll.LMQ_PREFIX + UUID.randomUUID())\r\n            .collect(java.util.stream.Collectors.toList());\r\n        assertEquals(0, messageStore.getQueueStore().getLmqNum());\r\n\r\n        lmqNameList.forEach(lmqName -> assertNotNull(messageStore.getConsumeQueue(lmqName, 0))); // create if not exist\r\n        assertEquals(0, messageStore.getQueueStore().getLmqNum());\r\n\r\n        for (String lmqName : lmqNameList) {\r\n            MessageExtBrokerInner message = buildMessage(topic, -1);\r\n            MessageAccessor.putProperty(message, MessageConst.PROPERTY_INNER_MULTI_DISPATCH, lmqName);\r\n            LmqDispatch.wrapLmqDispatch(messageStore, message);\r\n            PutMessageResult putMessageResult = messageStore.putMessage(message);\r\n            assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\r\n        }\r\n        await().atMost(5, SECONDS).until(fullyDispatched(messageStore));\r\n\r\n        lmqNameList.forEach(lmqName -> assertNotNull(messageStore.getConsumeQueue(lmqName, 0)));\r\n        assertEquals(num, messageStore.getQueueStore().getLmqNum());\r\n\r\n        lmqNameList.forEach(lmqName -> messageStore.deleteTopics(Sets.newHashSet(lmqName)));\r\n        assertEquals(0, messageStore.getQueueStore().getLmqNum());\r\n    }\r\n\r\n    @Test\r\n    public void testLmqCounter_reload() throws Exception {\r\n        messageStore.getMessageStoreConfig().setEnableMultiDispatch(true);\r\n        messageStore.getMessageStoreConfig().setEnableLmq(true);\r\n        int num = 5;\r\n        String topic = \"topic\";\r\n        List<String> lmqNameList = IntStream.range(0, num)\r\n            .mapToObj(i -> MixAll.LMQ_PREFIX + UUID.randomUUID())\r\n            .collect(java.util.stream.Collectors.toList());\r\n        assertEquals(0, messageStore.getQueueStore().getLmqNum());\r\n\r\n        for (String lmqName : lmqNameList) {\r\n            MessageExtBrokerInner message = buildMessage(topic, -1);\r\n            MessageAccessor.putProperty(message, MessageConst.PROPERTY_INNER_MULTI_DISPATCH, lmqName);\r\n            LmqDispatch.wrapLmqDispatch(messageStore, message);\r\n            PutMessageResult putMessageResult = messageStore.putMessage(message);\r\n            assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\r\n        }\r\n        await().atMost(5, SECONDS).until(fullyDispatched(messageStore));\r\n        assertEquals(num, messageStore.getQueueStore().getLmqNum());\r\n        messageStore.shutdown();\r\n\r\n        // create new one based on current store\r\n        MessageStore newStore = createMessageStore(messageStore.getMessageStoreConfig().getStorePathRootDir(),\r\n            true, topicConfigTableMap, messageStore.getMessageStoreConfig());\r\n        newStore.load();\r\n        newStore.start();\r\n\r\n        assertEquals(num, newStore.getQueueStore().getLmqNum());\r\n        lmqNameList.forEach(lmqName -> assertNull(newStore.getQueueStore().getConsumeQueueTable().get(lmqName))); // not in consumeQueueTable\r\n        newStore.shutdown();\r\n    }\r\n}"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/queue/SparseConsumeQueueTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.queue;\n\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.store.CommitLog;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Before;\nimport org.junit.Rule;\nimport org.junit.Test;\nimport org.junit.rules.TemporaryFolder;\n\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport java.nio.file.Files;\nimport java.nio.file.Paths;\nimport java.nio.file.StandardOpenOption;\nimport java.util.concurrent.ThreadLocalRandom;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.Mockito.doReturn;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\npublic class SparseConsumeQueueTest {\n    @Rule\n    public TemporaryFolder tempFolder = new TemporaryFolder();\n    String path;\n\n    MessageStore defaultMessageStore;\n    SparseConsumeQueue scq;\n\n    String topic = \"topic1\";\n    int queueId = 1;\n\n    @Before\n    public void setUp() throws IOException {\n        path = tempFolder.newFolder(\"scq\").getAbsolutePath();\n        defaultMessageStore = mock(DefaultMessageStore.class);\n        CommitLog commitLog = mock(CommitLog.class);\n        when(defaultMessageStore.getCommitLog()).thenReturn(commitLog);\n        when(commitLog.getCommitLogSize()).thenReturn(10 * 1024 * 1024);\n        MessageStoreConfig config = mock(MessageStoreConfig.class);\n        doReturn(config).when(defaultMessageStore).getMessageStoreConfig();\n        doReturn(true).when(config).isSearchBcqByCacheEnable();\n    }\n\n    private void fillByteBuf(ByteBuffer bb, long phyOffset, long queueOffset) {\n        bb.putLong(phyOffset);\n        bb.putInt(\"size\".length());\n        bb.putLong(\"tagsCode\".length());\n        bb.putLong(System.currentTimeMillis());\n        bb.putLong(queueOffset);\n        bb.putShort((short)1);\n        bb.putInt(0);\n        bb.putInt(0); // 4 bytes reserved\n    }\n\n    @Test\n    public void testLoad() throws IOException {\n        scq = new SparseConsumeQueue(topic, queueId, path, BatchConsumeQueue.CQ_STORE_UNIT_SIZE, defaultMessageStore);\n\n        String file1 = UtilAll.offset2FileName(111111);\n        String file2 = UtilAll.offset2FileName(222222);\n\n        long phyOffset = 10;\n        long queueOffset = 1;\n        ByteBuffer bb = ByteBuffer.allocate(BatchConsumeQueue.CQ_STORE_UNIT_SIZE);\n        fillByteBuf(bb, phyOffset, queueOffset);\n        Files.createDirectories(Paths.get(path, topic, String.valueOf(queueId)));\n        Files.write(Paths.get(path, topic, String.valueOf(queueId), file1), bb.array(),\n            StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);\n        bb.clear();\n        fillByteBuf(bb, phyOffset + 1, queueOffset + 1);\n        Files.write(Paths.get(path, topic, String.valueOf(queueId), file2), bb.array(),\n            StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);\n\n        scq.load();\n        scq.recover();\n        assertEquals(scq.get(queueOffset + 1).getPos(), phyOffset + 1);\n    }\n\n    private void fillByteBufSeq(ByteBuffer bb, int circle, long basePhyOffset, long baseQueueOffset) {\n        long phyOffset = basePhyOffset;\n        long queueOffset = baseQueueOffset;\n\n        for (int i = 0; i < circle; i++) {\n            fillByteBuf(bb, phyOffset, queueOffset);\n            phyOffset++;\n            queueOffset++;\n        }\n    }\n\n    @Test\n    public void testSearch() throws IOException {\n        int fileSize = 10 * BatchConsumeQueue.CQ_STORE_UNIT_SIZE;\n        scq = new SparseConsumeQueue(topic, queueId, path, fileSize, defaultMessageStore);\n\n        ByteBuffer bb = ByteBuffer.allocate(fileSize);\n        long basePhyOffset = 101;\n        long baseQueueOffset = 101;\n\n        /* 101 -> 101 ... 110 -> 110\n           201 -> 201 ... 210 -> 210\n           301 -> 301 ... 310 -> 310\n           ...\n         */\n        for (int i = 0; i < 5; i++) {\n            String fileName = UtilAll.offset2FileName(i * fileSize);\n            fillByteBufSeq(bb, 10, basePhyOffset, baseQueueOffset);\n            Files.createDirectories(Paths.get(path, topic, String.valueOf(queueId)));\n            Files.write(Paths.get(path, topic, String.valueOf(queueId), fileName), bb.array(),\n                StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);\n            basePhyOffset = i * 100 + 1;\n            baseQueueOffset = i * 100 + 1;\n            bb.clear();\n        }\n\n        scq.load();\n        scq.recover();\n\n        ReferredIterator<CqUnit> bufferConsumeQueue = scq.iterateFromOrNext(105);   //in the file\n        assertNotNull(bufferConsumeQueue);\n        assertTrue(bufferConsumeQueue.hasNext());\n        assertEquals(bufferConsumeQueue.next().getQueueOffset(), 105);\n        bufferConsumeQueue.release();\n\n        bufferConsumeQueue = scq.iterateFromOrNext(120);    // in the next file\n        assertNotNull(bufferConsumeQueue);\n        assertTrue(bufferConsumeQueue.hasNext());\n        assertEquals(bufferConsumeQueue.next().getQueueOffset(), 201);\n        bufferConsumeQueue.release();\n\n        bufferConsumeQueue = scq.iterateFromOrNext(600);       // not in the file\n        assertNull(bufferConsumeQueue);\n    }\n\n    @Test\n    public void testCreateFile() throws IOException {\n        scq = new SparseConsumeQueue(topic, queueId, path, BatchConsumeQueue.CQ_STORE_UNIT_SIZE, defaultMessageStore);\n        long physicalOffset = Math.abs(ThreadLocalRandom.current().nextLong());\n        String formatName = UtilAll.offset2FileName(physicalOffset);\n        scq.createFile(physicalOffset);\n\n        assertTrue(Files.exists(Paths.get(path, topic, String.valueOf(queueId), formatName)));\n        scq.putBatchMessagePositionInfo(5,4,3,2,1,(short)1);\n        assertEquals(4, scq.get(1).getSize());\n    }\n}"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactoryTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.rocksdb;\n\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.junit.Assert;\nimport org.junit.Test;\nimport org.rocksdb.CompressionType;\n\npublic class RocksDBOptionsFactoryTest {\n\n    @Test\n    public void testBottomMostCompressionType() {\n        MessageStoreConfig config = new MessageStoreConfig();\n        Assert.assertEquals(CompressionType.ZSTD_COMPRESSION,\n            CompressionType.getCompressionType(config.getBottomMostCompressionTypeForConsumeQueueStore()));\n        Assert.assertEquals(CompressionType.LZ4_COMPRESSION, CompressionType.getCompressionType(\"lz4\"));\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/stats/BrokerStatsManagerTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.stats;\n\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.apache.rocketmq.common.stats.Stats.BROKER_PUT_NUMS;\nimport static org.apache.rocketmq.common.stats.Stats.GROUP_ACK_NUMS;\nimport static org.apache.rocketmq.common.stats.Stats.GROUP_CK_NUMS;\nimport static org.apache.rocketmq.common.stats.Stats.GROUP_GET_FALL_SIZE;\nimport static org.apache.rocketmq.common.stats.Stats.GROUP_GET_FALL_TIME;\nimport static org.apache.rocketmq.common.stats.Stats.GROUP_GET_LATENCY;\nimport static org.apache.rocketmq.common.stats.Stats.GROUP_GET_NUMS;\nimport static org.apache.rocketmq.common.stats.Stats.GROUP_GET_SIZE;\nimport static org.apache.rocketmq.common.stats.Stats.QUEUE_GET_NUMS;\nimport static org.apache.rocketmq.common.stats.Stats.QUEUE_GET_SIZE;\nimport static org.apache.rocketmq.common.stats.Stats.QUEUE_PUT_NUMS;\nimport static org.apache.rocketmq.common.stats.Stats.QUEUE_PUT_SIZE;\nimport static org.apache.rocketmq.common.stats.Stats.SNDBCK_PUT_NUMS;\nimport static org.apache.rocketmq.common.stats.Stats.TOPIC_PUT_LATENCY;\nimport static org.apache.rocketmq.common.stats.Stats.TOPIC_PUT_NUMS;\nimport static org.apache.rocketmq.common.stats.Stats.TOPIC_PUT_SIZE;\nimport static org.assertj.core.api.Assertions.assertThat;\n\npublic class BrokerStatsManagerTest {\n    private BrokerStatsManager brokerStatsManager;\n\n    private static final String TOPIC = \"TOPIC_TEST\";\n    private static final Integer QUEUE_ID = 0;\n    private static final String GROUP_NAME = \"GROUP_TEST\";\n    private static final String CLUSTER_NAME = \"DefaultCluster\";\n\n    @Before\n    public void init() {\n        brokerStatsManager = new BrokerStatsManager(CLUSTER_NAME, true);\n        brokerStatsManager.start();\n    }\n\n    @After\n    public void destroy() {\n        brokerStatsManager.shutdown();\n    }\n\n    @Test\n    public void testGetStatsItem() {\n        assertThat(brokerStatsManager.getStatsItem(\"TEST\", \"TEST\")).isNull();\n    }\n\n    @Test\n    public void testIncQueuePutNums() {\n        brokerStatsManager.incQueuePutNums(TOPIC, QUEUE_ID);\n        String statsKey = brokerStatsManager.buildStatsKey(TOPIC, String.valueOf(QUEUE_ID));\n        assertThat(brokerStatsManager.getStatsItem(QUEUE_PUT_NUMS, statsKey).getTimes().doubleValue()).isEqualTo(1L);\n        brokerStatsManager.incQueuePutNums(TOPIC, QUEUE_ID, 2, 2);\n        assertThat(brokerStatsManager.getStatsItem(QUEUE_PUT_NUMS, statsKey).getValue().doubleValue()).isEqualTo(3L);\n    }\n\n    @Test\n    public void testIncQueuePutSize() {\n        brokerStatsManager.incQueuePutSize(TOPIC, QUEUE_ID, 2);\n        String statsKey = brokerStatsManager.buildStatsKey(TOPIC, String.valueOf(QUEUE_ID));\n        assertThat(brokerStatsManager.getStatsItem(QUEUE_PUT_SIZE, statsKey).getValue().doubleValue()).isEqualTo(2L);\n    }\n\n    @Test\n    public void testIncQueueGetNums() {\n        brokerStatsManager.incQueueGetNums(GROUP_NAME, TOPIC, QUEUE_ID, 1);\n        final String statsKey = brokerStatsManager.buildStatsKey(brokerStatsManager.buildStatsKey(TOPIC, String.valueOf(QUEUE_ID)), GROUP_NAME);\n        assertThat(brokerStatsManager.getStatsItem(QUEUE_GET_NUMS, statsKey).getValue().doubleValue()).isEqualTo(1L);\n    }\n\n    @Test\n    public void testIncQueueGetSize() {\n        brokerStatsManager.incQueueGetSize(GROUP_NAME, TOPIC, QUEUE_ID, 1);\n        final String statsKey = brokerStatsManager.buildStatsKey(brokerStatsManager.buildStatsKey(TOPIC, String.valueOf(QUEUE_ID)), GROUP_NAME);\n        assertThat(brokerStatsManager.getStatsItem(QUEUE_GET_SIZE, statsKey).getValue().doubleValue()).isEqualTo(1L);\n    }\n\n    @Test\n    public void testIncTopicPutNums() {\n        brokerStatsManager.incTopicPutNums(TOPIC);\n        assertThat(brokerStatsManager.getStatsItem(TOPIC_PUT_NUMS, TOPIC).getTimes().doubleValue()).isEqualTo(1L);\n        brokerStatsManager.incTopicPutNums(TOPIC, 2, 2);\n        assertThat(brokerStatsManager.getStatsItem(TOPIC_PUT_NUMS, TOPIC).getValue().doubleValue()).isEqualTo(3L);\n    }\n\n    @Test\n    public void testIncTopicPutSize() {\n        brokerStatsManager.incTopicPutSize(TOPIC, 2);\n        assertThat(brokerStatsManager.getStatsItem(TOPIC_PUT_SIZE, TOPIC).getValue().doubleValue()).isEqualTo(2L);\n    }\n\n    @Test\n    public void testIncGroupGetNums() {\n        brokerStatsManager.incGroupGetNums(GROUP_NAME, TOPIC, 1);\n        String statsKey = brokerStatsManager.buildStatsKey(TOPIC, GROUP_NAME);\n        assertThat(brokerStatsManager.getStatsItem(GROUP_GET_NUMS, statsKey).getValue().doubleValue()).isEqualTo(1L);\n    }\n\n    @Test\n    public void testIncGroupGetSize() {\n        brokerStatsManager.incGroupGetSize(GROUP_NAME, TOPIC, 1);\n        String statsKey = brokerStatsManager.buildStatsKey(TOPIC, GROUP_NAME);\n        assertThat(brokerStatsManager.getStatsItem(GROUP_GET_SIZE, statsKey).getValue().doubleValue()).isEqualTo(1L);\n    }\n\n    @Test\n    public void testIncGroupGetLatency() {\n        brokerStatsManager.incGroupGetLatency(GROUP_NAME, TOPIC, 1, 1);\n        String statsKey = String.format(\"%d@%s@%s\", 1, TOPIC, GROUP_NAME);\n        assertThat(brokerStatsManager.getStatsItem(GROUP_GET_LATENCY, statsKey).getValue().doubleValue()).isEqualTo(1L);\n    }\n\n    @Test\n    public void testIncBrokerPutNums() {\n        brokerStatsManager.incBrokerPutNums();\n        assertThat(brokerStatsManager.getStatsItem(BROKER_PUT_NUMS, CLUSTER_NAME).getValue().doubleValue()).isEqualTo(1L);\n    }\n\n    @Test\n    public void testOnTopicDeleted() {\n        brokerStatsManager.incTopicPutNums(TOPIC);\n        brokerStatsManager.incTopicPutSize(TOPIC, 100);\n        brokerStatsManager.incQueuePutNums(TOPIC, QUEUE_ID);\n        brokerStatsManager.incQueuePutSize(TOPIC, QUEUE_ID, 100);\n        brokerStatsManager.incTopicPutLatency(TOPIC, QUEUE_ID, 10);\n        brokerStatsManager.incGroupGetNums(GROUP_NAME, TOPIC, 1);\n        brokerStatsManager.incGroupGetSize(GROUP_NAME, TOPIC, 100);\n        brokerStatsManager.incGroupCkNums(GROUP_NAME, TOPIC, 1);\n        brokerStatsManager.incGroupAckNums(GROUP_NAME, TOPIC, 1);\n        brokerStatsManager.incQueueGetNums(GROUP_NAME, TOPIC, QUEUE_ID, 1);\n        brokerStatsManager.incQueueGetSize(GROUP_NAME, TOPIC, QUEUE_ID, 100);\n        brokerStatsManager.incSendBackNums(GROUP_NAME, TOPIC);\n        brokerStatsManager.incGroupGetLatency(GROUP_NAME, TOPIC, 1, 1);\n        brokerStatsManager.recordDiskFallBehindTime(GROUP_NAME, TOPIC, 1, 11L);\n        brokerStatsManager.recordDiskFallBehindSize(GROUP_NAME, TOPIC, 1, 11L);\n\n        brokerStatsManager.onTopicDeleted(TOPIC);\n\n        Assert.assertNull(brokerStatsManager.getStatsItem(TOPIC_PUT_NUMS, TOPIC));\n        Assert.assertNull(brokerStatsManager.getStatsItem(TOPIC_PUT_SIZE, TOPIC));\n        Assert.assertNull(brokerStatsManager.getStatsItem(QUEUE_PUT_NUMS, TOPIC + \"@\" + QUEUE_ID));\n        Assert.assertNull(brokerStatsManager.getStatsItem(QUEUE_PUT_SIZE, TOPIC + \"@\" + QUEUE_ID));\n        Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_SIZE, TOPIC + \"@\" + GROUP_NAME));\n        Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_NUMS, TOPIC + \"@\" + GROUP_NAME));\n        Assert.assertNull(brokerStatsManager.getStatsItem(QUEUE_GET_SIZE, TOPIC + \"@\" + QUEUE_ID + \"@\" + GROUP_NAME));\n        Assert.assertNull(brokerStatsManager.getStatsItem(QUEUE_GET_NUMS, TOPIC + \"@\" + QUEUE_ID + \"@\" + GROUP_NAME));\n        Assert.assertNull(brokerStatsManager.getStatsItem(SNDBCK_PUT_NUMS, TOPIC + \"@\" + GROUP_NAME));\n        Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_LATENCY, \"1@\" + TOPIC + \"@\" + GROUP_NAME));\n        Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_FALL_SIZE, \"1@\" + TOPIC + \"@\" + GROUP_NAME));\n        Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_FALL_TIME, \"1@\" + TOPIC + \"@\" + GROUP_NAME));\n        Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_CK_NUMS, TOPIC + \"@\" + GROUP_NAME));\n        Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_ACK_NUMS, TOPIC + \"@\" + GROUP_NAME));\n        Assert.assertNull(brokerStatsManager.getStatsItem(TOPIC_PUT_LATENCY, QUEUE_ID + \"@\" + TOPIC));\n    }\n\n    @Test\n    public void testOnGroupDeleted() {\n        brokerStatsManager.incGroupGetNums(GROUP_NAME, TOPIC, 1);\n        brokerStatsManager.incGroupGetSize(GROUP_NAME, TOPIC, 100);\n        brokerStatsManager.incQueueGetNums(GROUP_NAME, TOPIC, QUEUE_ID, 1);\n        brokerStatsManager.incQueueGetSize(GROUP_NAME, TOPIC, QUEUE_ID, 100);\n        brokerStatsManager.incSendBackNums(GROUP_NAME, TOPIC);\n        brokerStatsManager.incGroupGetLatency(GROUP_NAME, TOPIC, 1, 1);\n        brokerStatsManager.recordDiskFallBehindTime(GROUP_NAME, TOPIC, 1, 11L);\n        brokerStatsManager.recordDiskFallBehindSize(GROUP_NAME, TOPIC, 1, 11L);\n        brokerStatsManager.incGroupCkNums(GROUP_NAME, TOPIC, 1);\n        brokerStatsManager.incGroupAckNums(GROUP_NAME, TOPIC, 1);\n\n        brokerStatsManager.onGroupDeleted(GROUP_NAME);\n\n        Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_SIZE, TOPIC + \"@\" + GROUP_NAME));\n        Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_NUMS, TOPIC + \"@\" + GROUP_NAME));\n        Assert.assertNull(brokerStatsManager.getStatsItem(QUEUE_GET_SIZE, TOPIC + \"@\" + QUEUE_ID + \"@\" + GROUP_NAME));\n        Assert.assertNull(brokerStatsManager.getStatsItem(QUEUE_GET_NUMS, TOPIC + \"@\" + QUEUE_ID + \"@\" + GROUP_NAME));\n        Assert.assertNull(brokerStatsManager.getStatsItem(SNDBCK_PUT_NUMS, TOPIC + \"@\" + GROUP_NAME));\n        Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_LATENCY, \"1@\" + TOPIC + \"@\" + GROUP_NAME));\n        Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_FALL_SIZE, \"1@\" + TOPIC + \"@\" + GROUP_NAME));\n        Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_FALL_TIME, \"1@\" + TOPIC + \"@\" + GROUP_NAME));\n        Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_CK_NUMS, TOPIC + \"@\" + GROUP_NAME));\n        Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_ACK_NUMS, TOPIC + \"@\" + GROUP_NAME));\n    }\n\n    @Test\n    public void testIncBrokerGetNumsWithoutSystemTopic() {\n        brokerStatsManager.incBrokerGetNumsWithoutSystemTopic(TOPIC, 1);\n        assertThat(brokerStatsManager.getStatsItem(BrokerStatsManager.BROKER_GET_NUMS_WITHOUT_SYSTEM_TOPIC, CLUSTER_NAME)\n            .getValue().doubleValue()).isEqualTo(1L);\n        assertThat(brokerStatsManager.getBrokerGetNumsWithoutSystemTopic()).isEqualTo(1L);\n\n        brokerStatsManager.incBrokerGetNumsWithoutSystemTopic(TopicValidator.RMQ_SYS_TRACE_TOPIC, 1);\n        assertThat(brokerStatsManager.getStatsItem(BrokerStatsManager.BROKER_GET_NUMS_WITHOUT_SYSTEM_TOPIC, CLUSTER_NAME)\n            .getValue().doubleValue()).isEqualTo(1L);\n        assertThat(brokerStatsManager.getBrokerGetNumsWithoutSystemTopic()).isEqualTo(1L);\n    }\n\n    @Test\n    public void testIncBrokerPutNumsWithoutSystemTopic() {\n        brokerStatsManager.incBrokerPutNumsWithoutSystemTopic(TOPIC, 1);\n        assertThat(brokerStatsManager.getStatsItem(BrokerStatsManager.BROKER_PUT_NUMS_WITHOUT_SYSTEM_TOPIC, CLUSTER_NAME)\n            .getValue().doubleValue()).isEqualTo(1L);\n        assertThat(brokerStatsManager.getBrokerPutNumsWithoutSystemTopic()).isEqualTo(1L);\n\n        brokerStatsManager.incBrokerPutNumsWithoutSystemTopic(TopicValidator.RMQ_SYS_TRACE_TOPIC, 1);\n        assertThat(brokerStatsManager.getStatsItem(BrokerStatsManager.BROKER_PUT_NUMS_WITHOUT_SYSTEM_TOPIC, CLUSTER_NAME)\n            .getValue().doubleValue()).isEqualTo(1L);\n        assertThat(brokerStatsManager.getBrokerPutNumsWithoutSystemTopic()).isEqualTo(1L);\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/timer/StoreTestUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.timer;\n\nimport java.io.File;\nimport java.util.UUID;\n\npublic class StoreTestUtils {\n    public static String createBaseDir() {\n        String baseDir = System.getProperty(\"java.io.tmpdir\") + File.separator + \"unitteststore-\" + UUID.randomUUID();\n        final File file = new File(baseDir);\n        if (file.exists()) {\n            System.exit(1);\n        }\n        return baseDir;\n    }\n\n    public static void deleteFile(String fileName) {\n        deleteFile(new File(fileName));\n    }\n\n    public static void deleteFile(File file) {\n        if (!file.exists()) {\n            return;\n        }\n        if (file.isFile()) {\n            file.delete();\n        } else if (file.isDirectory()) {\n            File[] files = file.listFiles();\n            for (File file1 : files) {\n                deleteFile(file1);\n            }\n            file.delete();\n        }\n    }\n\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/timer/TimerCheckPointTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.timer;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\n\npublic class TimerCheckPointTest {\n\n    private String baseDir;\n\n    @Before\n    public void init() throws IOException {\n        baseDir = StoreTestUtils.createBaseDir();\n    }\n\n    @Test\n    public void testCheckPoint() throws IOException {\n        String baseSrc = baseDir + File.separator + \"timercheck\";\n        TimerCheckpoint first = new TimerCheckpoint(baseSrc);\n        assertEquals(0, first.getLastReadTimeMs());\n        assertEquals(0, first.getLastTimerLogFlushPos());\n        assertEquals(0, first.getLastTimerQueueOffset());\n        assertEquals(0, first.getMasterTimerQueueOffset());\n        first.setLastReadTimeMs(1000);\n        first.setLastTimerLogFlushPos(1100);\n        first.setLastTimerQueueOffset(1200);\n        first.setMasterTimerQueueOffset(1300);\n        first.shutdown();\n        TimerCheckpoint second = new TimerCheckpoint(baseSrc);\n        assertEquals(1000, second.getLastReadTimeMs());\n        assertEquals(1100, second.getLastTimerLogFlushPos());\n        assertEquals(1200, second.getLastTimerQueueOffset());\n        assertEquals(1300, second.getMasterTimerQueueOffset());\n    }\n\n    @Test\n    public void testNewCheckPoint() throws IOException {\n        String baseSrc = baseDir + File.separator + \"timercheck2\";\n        TimerCheckpoint first = new TimerCheckpoint(baseSrc);\n        assertEquals(0, first.getLastReadTimeMs());\n        assertEquals(0, first.getLastTimerLogFlushPos());\n        assertEquals(0, first.getLastTimerQueueOffset());\n        assertEquals(0, first.getMasterTimerQueueOffset());\n        assertEquals(0, first.getDataVersion().getStateVersion());\n        assertEquals(0, first.getDataVersion().getCounter().get());\n        first.setLastReadTimeMs(1000);\n        first.setLastTimerLogFlushPos(1100);\n        first.setLastTimerQueueOffset(1200);\n        first.setMasterTimerQueueOffset(1300);\n        first.getDataVersion().setStateVersion(1400);\n        first.getDataVersion().setTimestamp(1500);\n        first.getDataVersion().setCounter(new AtomicLong(1600));\n        first.shutdown();\n        TimerCheckpoint second = new TimerCheckpoint(baseSrc);\n        assertEquals(1000, second.getLastReadTimeMs());\n        assertEquals(1100, second.getLastTimerLogFlushPos());\n        assertEquals(1200, second.getLastTimerQueueOffset());\n        assertEquals(1300, second.getMasterTimerQueueOffset());\n        assertEquals(1400, second.getDataVersion().getStateVersion());\n        assertEquals(1500, second.getDataVersion().getTimestamp());\n        assertEquals(1600, second.getDataVersion().getCounter().get());\n    }\n\n    @Test\n    public void testEncodeDecode() throws IOException {\n        TimerCheckpoint first = new TimerCheckpoint();\n        first.setLastReadTimeMs(1000);\n        first.setLastTimerLogFlushPos(1100);\n        first.setLastTimerQueueOffset(1200);\n        first.setMasterTimerQueueOffset(1300);\n\n        TimerCheckpoint second = TimerCheckpoint.decode(TimerCheckpoint.encode(first));\n        assertEquals(first.getLastReadTimeMs(), second.getLastReadTimeMs());\n        assertEquals(first.getLastTimerLogFlushPos(), second.getLastTimerLogFlushPos());\n        assertEquals(first.getLastTimerQueueOffset(), second.getLastTimerQueueOffset());\n        assertEquals(first.getMasterTimerQueueOffset(), second.getMasterTimerQueueOffset());\n    }\n\n    @Test\n    public void testNewEncodeDecode() throws IOException {\n        TimerCheckpoint first = new TimerCheckpoint();\n        first.setLastReadTimeMs(1000);\n        first.setLastTimerLogFlushPos(1100);\n        first.setLastTimerQueueOffset(1200);\n        first.setMasterTimerQueueOffset(1300);\n        first.getDataVersion().setStateVersion(1400);\n        first.getDataVersion().setTimestamp(1500);\n        first.getDataVersion().setCounter(new AtomicLong(1600));\n        TimerCheckpoint second = TimerCheckpoint.decode(TimerCheckpoint.encode(first));\n        assertEquals(first.getLastReadTimeMs(), second.getLastReadTimeMs());\n        assertEquals(first.getLastTimerLogFlushPos(), second.getLastTimerLogFlushPos());\n        assertEquals(first.getLastTimerQueueOffset(), second.getLastTimerQueueOffset());\n        assertEquals(first.getMasterTimerQueueOffset(), second.getMasterTimerQueueOffset());\n        assertEquals(first.getDataVersion().getStateVersion(), 1400);\n        assertEquals(first.getDataVersion().getTimestamp(), 1500);\n        assertEquals(first.getDataVersion().getCounter().get(), 1600);\n    }\n\n    @After\n    public void shutdown() {\n        if (null != baseDir) {\n            StoreTestUtils.deleteFile(baseDir);\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/timer/TimerLogTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.timer;\n\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.junit.After;\nimport org.junit.Test;\n\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\n\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertArrayEquals;\n\npublic class TimerLogTest {\n\n    private final Set<String> baseDirs = new HashSet<>();\n    private final List<TimerLog> timerLogs = new ArrayList<>();\n\n    public TimerLog createTimerLog(String baseDir) {\n        if (null == baseDir) {\n            baseDir = StoreTestUtils.createBaseDir();\n        }\n        TimerLog timerLog = new TimerLog(baseDir, 1024);\n        timerLogs.add(timerLog);\n        baseDirs.add(baseDir);\n        timerLog.load();\n        return timerLog;\n    }\n\n    @Test\n    public void testAppendRollSelectDelete() throws Exception {\n        TimerLog timerLog = createTimerLog(null);\n        ByteBuffer byteBuffer = ByteBuffer.allocate(TimerLog.UNIT_SIZE);\n        byteBuffer.putInt(TimerLog.UNIT_SIZE);\n        byteBuffer.putLong(Long.MAX_VALUE);\n        byteBuffer.putInt(0);\n        byteBuffer.putLong(Long.MAX_VALUE);\n        byteBuffer.putInt(0);\n        byteBuffer.putLong(1000);\n        byteBuffer.putInt(10);\n        byteBuffer.putInt(123);\n        byteBuffer.putInt(0);\n        long ret = -1;\n        for (int i = 0; i < 10; i++) {\n            ret = timerLog.append(byteBuffer.array(), 0, TimerLog.UNIT_SIZE);\n            assertEquals(i * TimerLog.UNIT_SIZE, ret);\n        }\n        for (int i = 0; i < 100; i++) {\n            timerLog.append(byteBuffer.array());\n        }\n        assertEquals(6, timerLog.getMappedFileQueue().getMappedFiles().size());\n        SelectMappedBufferResult sbr = timerLog.getTimerMessage(ret);\n        assertNotNull(sbr);\n        assertEquals(TimerLog.UNIT_SIZE, sbr.getByteBuffer().getInt());\n        sbr.release();\n        SelectMappedBufferResult wholeSbr = timerLog.getWholeBuffer(ret);\n        assertEquals(0, wholeSbr.getStartOffset());\n        wholeSbr.release();\n        timerLog.getMappedFileQueue().deleteExpiredFileByOffsetForTimerLog(1024, timerLog.getOffsetForLastUnit(), TimerLog.UNIT_SIZE);\n        assertEquals(1, timerLog.getMappedFileQueue().getMappedFiles().size());\n    }\n\n    @Test\n    public void testRecovery() throws Exception {\n        String basedir = StoreTestUtils.createBaseDir();\n        TimerLog first = createTimerLog(basedir);\n        first.append(new byte[512]);\n        first.append(new byte[510]);\n        byte[] data = \"Hello Recovery\".getBytes();\n        first.append(data);\n        first.shutdown();\n        TimerLog second = createTimerLog(basedir);\n        assertEquals(2, second.getMappedFileQueue().getMappedFiles().size());\n        second.getMappedFileQueue().truncateDirtyFiles(1204 + 1000);\n        SelectMappedBufferResult sbr = second.getTimerMessage(1024 + 510);\n        byte[] expect = new byte[data.length];\n        sbr.getByteBuffer().get(expect);\n        assertArrayEquals(expect, data);\n    }\n\n    @Test\n    public void testAppendBlankByteBuffer() throws Exception {\n        TimerLog timerLog = createTimerLog(null);\n        ByteBuffer byteBuffer = ByteBuffer.allocate(TimerLog.UNIT_SIZE);\n        byteBuffer.putInt(TimerLog.UNIT_SIZE);\n        byteBuffer.putLong(Long.MAX_VALUE);\n        byteBuffer.putInt(0);\n        byteBuffer.putLong(Long.MAX_VALUE);\n        byteBuffer.putInt(0);\n        byteBuffer.putLong(1000);\n        byteBuffer.putInt(10);\n        byteBuffer.putInt(123);\n        byteBuffer.putInt(0);\n        int maxAppend = 1024 / TimerLog.UNIT_SIZE + 1;\n        for (int i = 0; i < maxAppend; i++) {\n            timerLog.append(byteBuffer.array(), 0, TimerLog.UNIT_SIZE);\n        }\n        SelectMappedBufferResult sbr = timerLog.getWholeBuffer(0);\n        ByteBuffer bf = sbr.getByteBuffer();\n        for (int position = 0; position < sbr.getSize(); position += TimerLog.UNIT_SIZE) {\n            bf.position(position);\n            bf.getInt();\n            bf.getLong();\n            int magic = bf.getInt();\n            if (position / TimerLog.UNIT_SIZE == maxAppend - 1) {\n                assertEquals(TimerLog.BLANK_MAGIC_CODE, magic);\n                continue;\n            }\n            bf.getLong();\n            bf.getInt();\n            bf.getLong();\n            bf.getInt();\n            bf.getInt();\n        }\n    }\n\n    @After\n    public void shutdown() {\n        for (TimerLog timerLog : timerLogs) {\n            timerLog.shutdown();\n            timerLog.getMappedFileQueue().destroy();\n        }\n        for (String baseDir : baseDirs) {\n            StoreTestUtils.deleteFile(baseDir);\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/timer/TimerMessageStoreTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.store.timer;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.lang.reflect.Field;\nimport java.net.InetAddress;\nimport java.net.InetSocketAddress;\nimport java.net.SocketAddress;\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.Callable;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.TopicFilterType;\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageResult;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.config.FlushDiskType;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.stats.BrokerStatsManager;\nimport org.apache.rocketmq.store.ConsumeQueue;\nimport org.apache.rocketmq.store.AppendMessageStatus;\nimport org.apache.rocketmq.store.AppendMessageResult;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.MessageArrivingListener;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Assume;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.Mockito;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotEquals;\nimport static org.junit.Assert.assertNotNull;\nimport static org.junit.Assert.assertNull;\nimport static org.junit.Assert.assertTrue;\nimport static org.junit.Assert.assertFalse;\nimport static org.mockito.Mockito.when;\nimport static org.mockito.Mockito.any;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.times;\n\npublic class TimerMessageStoreTest {\n    private final byte[] msgBody = new byte[1024];\n    private static MessageStore messageStore;\n    private MessageStore mockMessageStore;\n    private SocketAddress bornHost;\n    private SocketAddress storeHost;\n\n    private final int precisionMs = 500;\n\n    private final Set<String> baseDirs = new HashSet<>();\n    private final List<TimerMessageStore> timerStores = new ArrayList<>();\n    private final AtomicInteger counter = new AtomicInteger(0);\n\n    public static MessageStoreConfig storeConfig;\n\n    @Before\n    public void init() throws Exception {\n        String baseDir = StoreTestUtils.createBaseDir();\n        baseDirs.add(baseDir);\n\n        storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123);\n        bornHost = new InetSocketAddress(InetAddress.getByName(\"127.0.0.1\"), 0);\n\n        storeConfig = new MessageStoreConfig();\n        storeConfig.setMappedFileSizeCommitLog(1024 * 1024 * 1024);\n        storeConfig.setMappedFileSizeTimerLog(1024 * 1024 * 1024);\n        storeConfig.setMappedFileSizeConsumeQueue(10240);\n        storeConfig.setMaxHashSlotNum(10000);\n        storeConfig.setMaxIndexNum(100 * 1000);\n        storeConfig.setStorePathRootDir(baseDir);\n        storeConfig.setStorePathCommitLog(baseDir + File.separator + \"commitlog\");\n        storeConfig.setFlushDiskType(FlushDiskType.ASYNC_FLUSH);\n        storeConfig.setTimerInterceptDelayLevel(true);\n        storeConfig.setTimerPrecisionMs(precisionMs);\n        storeConfig.setAppendTopicForTimerDeleteKey(false); // reset default value\n\n        mockMessageStore = Mockito.mock(MessageStore.class);\n        messageStore = new DefaultMessageStore(storeConfig, new BrokerStatsManager(\"TimerTest\",false), new MyMessageArrivingListener(), new BrokerConfig(), new ConcurrentHashMap<>());\n        boolean load = messageStore.load();\n        assertTrue(load);\n        messageStore.start();\n    }\n\n    public TimerMessageStore createTimerMessageStore(String rootDir , boolean needMock) throws IOException {\n        if (null == rootDir) {\n            rootDir = StoreTestUtils.createBaseDir();\n        }\n\n        TimerCheckpoint timerCheckpoint = new TimerCheckpoint(rootDir + File.separator + \"config\" + File.separator + \"timercheck\");\n        TimerMetrics timerMetrics = new TimerMetrics(rootDir + File.separator + \"config\" + File.separator + \"timermetrics\");\n        MessageStore ms = needMock ? mockMessageStore : messageStore;\n        TimerMessageStore timerMessageStore = new TimerMessageStore(ms, storeConfig, timerCheckpoint, timerMetrics, null);\n        ms.setTimerMessageStore(timerMessageStore);\n\n        baseDirs.add(rootDir);\n        timerStores.add(timerMessageStore);\n\n        return timerMessageStore;\n    }\n\n    private static PutMessageResult transformTimerMessage(TimerMessageStore timerMessageStore, MessageExtBrokerInner msg) {\n        //do transform\n        int delayLevel = msg.getDelayTimeLevel();\n        long deliverMs;\n\n        try {\n            if (msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC) != null) {\n                deliverMs = System.currentTimeMillis() + Integer.parseInt(msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC)) * 1000;\n            } else if (msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_MS) != null) {\n                deliverMs = System.currentTimeMillis() + Integer.parseInt(msg.getProperty(MessageConst.PROPERTY_TIMER_DELAY_MS));\n            } else {\n                deliverMs = Long.parseLong(msg.getProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS));\n            }\n        } catch (Exception e) {\n            return new PutMessageResult(PutMessageStatus.WHEEL_TIMER_MSG_ILLEGAL, null);\n        }\n        if (deliverMs > System.currentTimeMillis()) {\n            if (delayLevel <= 0 && deliverMs - System.currentTimeMillis() > storeConfig.getTimerMaxDelaySec() * 1000L) {\n                return new PutMessageResult(PutMessageStatus.WHEEL_TIMER_MSG_ILLEGAL, null);\n            }\n\n            int timerPrecisionMs = storeConfig.getTimerPrecisionMs();\n            if (deliverMs % timerPrecisionMs == 0) {\n                deliverMs -= timerPrecisionMs;\n            } else {\n                deliverMs = deliverMs / timerPrecisionMs * timerPrecisionMs;\n            }\n\n            if (timerMessageStore.isReject(deliverMs)) {\n                return new PutMessageResult(PutMessageStatus.WHEEL_TIMER_FLOW_CONTROL, null);\n            }\n            MessageAccessor.putProperty(msg, MessageConst.PROPERTY_TIMER_OUT_MS, deliverMs + \"\");\n            MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, msg.getTopic());\n            MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_QUEUE_ID, String.valueOf(msg.getQueueId()));\n            msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n            msg.setTopic(TimerMessageStore.TIMER_TOPIC);\n            msg.setQueueId(0);\n        } else if (null != msg.getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY)) {\n            return new PutMessageResult(PutMessageStatus.WHEEL_TIMER_MSG_ILLEGAL, null);\n        }\n        return null;\n    }\n\n    @Test\n    public void testPutTimerMessage() throws Exception {\n        Assume.assumeFalse(MixAll.isWindows());\n        String topic = \"TimerTest_testPutTimerMessage\";\n\n        final TimerMessageStore timerMessageStore = createTimerMessageStore(null , false);\n        timerMessageStore.load();\n        timerMessageStore.start(true);\n\n        long curr = System.currentTimeMillis() / precisionMs * precisionMs;\n        long delayMs = curr + 3000;\n        for (int i = 0; i < 10; i++) {\n            for (int j = 0; j < 5; j++) {\n                MessageExtBrokerInner inner = buildMessage((i % 2 == 0) ? 3000 : delayMs, topic + i, i % 2 == 0);\n                transformTimerMessage(timerMessageStore,inner);\n                PutMessageResult putMessageResult = messageStore.putMessage(inner);\n                assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n            }\n        }\n\n        // Wait until messages have been wrote to TimerLog but the slot (delayMs) hasn't expired.\n        await().atMost(2000, TimeUnit.MILLISECONDS).until(new Callable<Boolean>() {\n            @Override\n            public Boolean call() {\n                return timerMessageStore.getCommitQueueOffset() == 10 * 5;\n            }\n        });\n\n        for (int i = 0; i < 10; i++) {\n            Assert.assertEquals(5, timerMessageStore.getTimerMetrics().getTimingCount(topic + i));\n        }\n\n        for (int i = 0; i < 10; i++) {\n            for (int j = 0; j < 5; j++) {\n                ByteBuffer msgBuff = getOneMessage(topic + i, 0, j, 4000);\n                assertNotNull(msgBuff);\n                MessageExt msgExt = MessageDecoder.decode(msgBuff);\n                assertNotNull(msgExt);\n                assertEquals(topic + i, msgExt.getTopic());\n                // assertThat(System.currentTimeMillis()).isLessThan(delayMs + precisionMs * 2);\n            }\n        }\n        for (int i = 0; i < 10; i++) {\n            Assert.assertEquals(0, timerMessageStore.getTimerMetrics().getTimingCount(topic + i));\n        }\n    }\n\n    @Test\n    public void testRetryUntilSuccess() throws Exception {\n        storeConfig.setTimerEnableRetryUntilSuccess(true);\n        TimerMessageStore timerMessageStore = createTimerMessageStore(null , true);\n        timerMessageStore.load();\n        timerMessageStore.setShouldRunningDequeue(true);\n        Field stateField = TimerMessageStore.class.getDeclaredField(\"state\");\n        stateField.setAccessible(true);\n        stateField.set(timerMessageStore, TimerMessageStore.RUNNING);\n\n        MessageExtBrokerInner msg = buildMessage(3000L, \"TestRetry\", true);\n        transformTimerMessage(timerMessageStore, msg);\n        TimerRequest timerRequest = new TimerRequest(100, 200, 3000, System.currentTimeMillis(), 0, msg);\n        boolean offered = timerMessageStore.dequeuePutQueue.offer(timerRequest);\n        assertTrue(offered);\n        assertFalse(timerMessageStore.dequeuePutQueue.isEmpty());\n\n        // If enableRetryUntilSuccess is set and putMessage return NEED_RETRY type, the message should be retried until success.\n        when(mockMessageStore.putMessage(any(MessageExtBrokerInner.class)))\n                .thenReturn(new PutMessageResult(PutMessageStatus.FLUSH_DISK_TIMEOUT, null))\n                .thenReturn(new PutMessageResult(PutMessageStatus.FLUSH_SLAVE_TIMEOUT, null))\n                .thenReturn(new PutMessageResult(PutMessageStatus.OS_PAGE_CACHE_BUSY, null))\n                .thenReturn(new PutMessageResult(PutMessageStatus.OS_PAGE_CACHE_BUSY, null))\n                .thenReturn(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, null))\n                .thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));\n\n        final CountDownLatch latch = new CountDownLatch(1);\n        new Thread(() -> {\n            try {\n                timerMessageStore.getDequeuePutMessageServices()[0].run();\n            } finally {\n                latch.countDown();\n            }\n        }).start();\n        latch.await(5, TimeUnit.SECONDS);\n        assertTrue(timerMessageStore.dequeuePutQueue.isEmpty());\n        verify(mockMessageStore, times(6)).putMessage(any(MessageExtBrokerInner.class));\n    }\n\n    @Test\n    public void testTimerFlowControl() throws Exception {\n        String topic = \"TimerTest_testTimerFlowControl\";\n\n        storeConfig.setTimerCongestNumEachSlot(100);\n        TimerMessageStore timerMessageStore = createTimerMessageStore(null , false);\n        timerMessageStore.load();\n        timerMessageStore.start(true);\n\n        long curr = System.currentTimeMillis() / precisionMs * precisionMs;\n        // Make sure delayMs won't be over.\n        long delayMs = curr + 100000;\n\n        int passFlowControlNum = 0;\n        for (int i = 0; i < 500; i++) {\n            MessageExtBrokerInner inner = buildMessage(delayMs, topic, false);\n\n            PutMessageResult putMessageResult = transformTimerMessage(timerMessageStore,inner);\n            if (putMessageResult == null || !putMessageResult.getPutMessageStatus().equals(PutMessageStatus.WHEEL_TIMER_FLOW_CONTROL)) {\n                putMessageResult = messageStore.putMessage(inner);\n            }\n            else {\n                putMessageResult = new PutMessageResult(PutMessageStatus.WHEEL_TIMER_FLOW_CONTROL,null);\n            }\n\n            // Message with delayMs in getSlotIndex(delayMs - precisionMs).\n            long congestNum = timerMessageStore.getCongestNum(delayMs - precisionMs);\n            assertTrue(congestNum <= 220);\n            if (congestNum < 100) {\n                assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n            } else {\n                Assert.assertTrue(PutMessageStatus.PUT_OK == putMessageResult.getPutMessageStatus()\n                        || PutMessageStatus.WHEEL_TIMER_FLOW_CONTROL == putMessageResult.getPutMessageStatus());\n                if (PutMessageStatus.PUT_OK == putMessageResult.getPutMessageStatus()) {\n                    passFlowControlNum++;\n                }\n            }\n            //wait reput\n            Thread.sleep(5);\n        }\n        assertThat(passFlowControlNum).isGreaterThan(0).isLessThan(120);\n    }\n\n\n    @Test\n    public void testPutExpiredTimerMessage() throws Exception {\n        // Skip on Mac to make CI pass\n        Assume.assumeFalse(MixAll.isMac());\n        Assume.assumeFalse(MixAll.isWindows());\n\n        String topic = \"TimerTest_testPutExpiredTimerMessage\";\n\n        TimerMessageStore timerMessageStore = createTimerMessageStore(null ,false);\n        timerMessageStore.load();\n        timerMessageStore.start(true);\n\n        long delayMs = System.currentTimeMillis() - 2 * precisionMs;\n        for (int i = 0; i < 10; i++) {\n            MessageExtBrokerInner inner = buildMessage(delayMs, topic, false);\n            transformTimerMessage(timerMessageStore,inner);\n            PutMessageResult putMessageResult = messageStore.putMessage(inner);\n            assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n        }\n\n        long curr = System.currentTimeMillis();\n        for (int i = 0; i < 10; i++) {\n            ByteBuffer msgBuff = getOneMessage(topic, 0, i, 1000);\n            assertNotNull(msgBuff);\n            assertTrue(System.currentTimeMillis() - curr < 200);\n        }\n    }\n\n    @Test\n    public void testDeleteTimerMessage() throws Exception {\n        String topic = \"TimerTest_testDeleteTimerMessage\";\n\n        TimerMessageStore timerMessageStore = createTimerMessageStore(null ,false);\n        timerMessageStore.load();\n        timerMessageStore.start(true);\n\n        long curr = System.currentTimeMillis() / precisionMs * precisionMs;\n        long delayMs = curr + 1000;\n        String uniqKey = null;\n        for (int i = 0; i < 5; i++) {\n            MessageExtBrokerInner inner = buildMessage(delayMs, topic, false);\n            transformTimerMessage(timerMessageStore,inner);\n            if (null == uniqKey) {\n                uniqKey = MessageClientIDSetter.getUniqID(inner);\n            }\n            assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(inner).getPutMessageStatus());\n        }\n\n        MessageExtBrokerInner delMsg = buildMessage(delayMs, topic, false);\n        transformTimerMessage(timerMessageStore,delMsg);\n        MessageAccessor.putProperty(delMsg, TimerMessageStore.TIMER_DELETE_UNIQUE_KEY, TimerMessageStore.buildDeleteKey(topic, uniqKey, false));\n        delMsg.setPropertiesString(MessageDecoder.messageProperties2String(delMsg.getProperties()));\n        assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(delMsg).getPutMessageStatus());\n\n        // The first one should have been deleted.\n        ByteBuffer msgBuff = getOneMessage(topic, 0, 0, 3000);\n        assertNotNull(msgBuff);\n        MessageExt msgExt = MessageDecoder.decode(msgBuff);\n        assertNotNull(msgExt);\n        assertNotEquals(uniqKey, MessageClientIDSetter.getUniqID(msgExt));\n\n        // The last one should be null.\n        assertNull(getOneMessage(topic, 0, 4, 500));\n    }\n\n    @Test\n    public void testDeleteTimerMessage_ukCollision() throws Exception {\n        storeConfig.setAppendTopicForTimerDeleteKey(true); // append topic as namespace\n        String topic = \"TimerTest_testDeleteTimerMessage\";\n        String collisionTopic = \"TimerTest_testDeleteTimerMessage_collision\";\n\n        TimerMessageStore timerMessageStore = createTimerMessageStore(null , false);\n        timerMessageStore.load();\n        timerMessageStore.start(true);\n\n        long curr = System.currentTimeMillis() / precisionMs * precisionMs;\n        long delayMs = curr + 1000;\n\n        MessageExtBrokerInner inner = buildMessage(delayMs, topic, false);\n        transformTimerMessage(timerMessageStore, inner);\n        String firstUniqKey = MessageClientIDSetter.getUniqID(inner);\n        assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(inner).getPutMessageStatus());\n\n        inner = buildMessage(delayMs, topic, false);\n        transformTimerMessage(timerMessageStore, inner);\n        String secondUniqKey = MessageClientIDSetter.getUniqID(inner);\n        assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(inner).getPutMessageStatus());\n\n        MessageExtBrokerInner delMsg = buildMessage(delayMs, \"whatever\", false);\n        transformTimerMessage(timerMessageStore, delMsg);\n        MessageAccessor.putProperty(delMsg, TimerMessageStore.TIMER_DELETE_UNIQUE_KEY, TimerMessageStore.buildDeleteKey(topic, firstUniqKey, true));\n        delMsg.setPropertiesString(MessageDecoder.messageProperties2String(delMsg.getProperties()));\n        assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(delMsg).getPutMessageStatus());\n\n        delMsg = buildMessage(delayMs, \"whatever\", false);\n        transformTimerMessage(timerMessageStore, delMsg);\n        MessageAccessor.putProperty(delMsg, TimerMessageStore.TIMER_DELETE_UNIQUE_KEY, TimerMessageStore.buildDeleteKey(collisionTopic, secondUniqKey, true));\n        delMsg.setPropertiesString(MessageDecoder.messageProperties2String(delMsg.getProperties()));\n        assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(delMsg).getPutMessageStatus());\n\n        // The first one should have been deleted, the second one should not be deleted.\n        ByteBuffer msgBuff = getOneMessage(topic, 0, 0, 3000);\n        assertNotNull(msgBuff);\n        MessageExt msgExt = MessageDecoder.decode(msgBuff);\n        assertNotNull(msgExt);\n        assertNotEquals(firstUniqKey, MessageClientIDSetter.getUniqID(msgExt));\n        assertEquals(secondUniqKey, MessageClientIDSetter.getUniqID(msgExt));\n    }\n\n    @Test\n    public void testPutDeleteTimerMessage() throws Exception {\n        String topic = \"TimerTest_testPutDeleteTimerMessage\";\n\n        final TimerMessageStore timerMessageStore = createTimerMessageStore(null , false);\n        timerMessageStore.load();\n        timerMessageStore.start(true);\n\n        long curr = System.currentTimeMillis() / precisionMs * precisionMs;\n        final long delayMs = curr + 1000;\n        for (int i = 0; i < 5; i++) {\n            MessageExtBrokerInner inner = buildMessage(delayMs, topic, false);\n            transformTimerMessage(timerMessageStore,inner);\n            assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(inner).getPutMessageStatus());\n        }\n\n        MessageExtBrokerInner delMsg = buildMessage(delayMs, topic, false);\n        transformTimerMessage(timerMessageStore,delMsg);\n        MessageAccessor.putProperty(delMsg, TimerMessageStore.TIMER_DELETE_UNIQUE_KEY, \"XXX\");\n        delMsg.setPropertiesString(MessageDecoder.messageProperties2String(delMsg.getProperties()));\n        assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(delMsg).getPutMessageStatus());\n\n        // Wait until currReadTimeMs catches up current time and delayMs is over.\n        await().atMost(5000, TimeUnit.MILLISECONDS).until(new Callable<Boolean>() {\n            @Override\n            public Boolean call() {\n                long curr = System.currentTimeMillis() / precisionMs * precisionMs;\n                return curr >= delayMs\n                        && (timerMessageStore.getCurrReadTimeMs() == curr || timerMessageStore.getCurrReadTimeMs() == curr + precisionMs);\n            }\n        });\n\n        for (int i = 0; i < 5; i++) {\n            ByteBuffer msgBuff = getOneMessage(topic, 0, i, 1000);\n            assertNotNull(msgBuff);\n            // assertThat(System.currentTimeMillis()).isLessThan(delayMs + precisionMs);\n        }\n        assertNull(getOneMessage(topic, 0, 5, 1000));\n\n        // Test put expired delete msg.\n        MessageExtBrokerInner expiredInner = buildMessage(System.currentTimeMillis() - 100, topic, false);\n        MessageAccessor.putProperty(expiredInner, TimerMessageStore.TIMER_DELETE_UNIQUE_KEY, \"XXX\");\n        PutMessageResult putMessageResult = transformTimerMessage(timerMessageStore,expiredInner);\n        assertEquals(PutMessageStatus.WHEEL_TIMER_MSG_ILLEGAL, putMessageResult.getPutMessageStatus());\n    }\n\n    @Test\n    public void testStateAndRecover() throws Exception {\n        final String topic = \"TimerTest_testStateAndRecover\";\n\n        String base = StoreTestUtils.createBaseDir();\n        final TimerMessageStore first = createTimerMessageStore(base , false);\n        first.load();\n        first.start(true);\n\n        final int msgNum = 250;\n        long curr = System.currentTimeMillis() / precisionMs * precisionMs;\n        final long delayMs = curr + 5000;\n        for (int i = 0; i < msgNum; i++) {\n            MessageExtBrokerInner inner = buildMessage((i % 2 == 0) ? 5000 : delayMs, topic, i % 2 == 0);\n            transformTimerMessage(first,inner);\n            PutMessageResult putMessageResult = messageStore.putMessage(inner);\n            long cqOffset = first.getCommitQueueOffset();\n            assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());\n        }\n\n        // Wait until messages have written to TimerLog and currReadTimeMs catches up current time.\n        await().atMost(5000, TimeUnit.MILLISECONDS).until(new Callable<Boolean>() {\n            @Override\n            public Boolean call() {\n                long curr = System.currentTimeMillis() / precisionMs * precisionMs;\n                long cqOffset = first.getCommitQueueOffset();\n                return first.getCommitQueueOffset() == msgNum\n                        && (first.getCurrReadTimeMs() == curr || first.getCurrReadTimeMs() == curr + precisionMs);\n            }\n        });\n        assertThat(first.getTimerLog().getMappedFileQueue().getMappedFiles().size())\n                .isGreaterThanOrEqualTo(msgNum / (storeConfig.getMappedFileSizeTimerLog() / TimerLog.UNIT_SIZE));\n        assertThat(first.getQueueOffset()).isEqualTo(msgNum);\n        assertThat(first.getCommitQueueOffset()).isEqualTo(first.getQueueOffset());\n        assertThat(first.getCommitReadTimeMs()).isEqualTo(first.getCurrReadTimeMs());\n        curr = System.currentTimeMillis() / precisionMs * precisionMs;\n        assertThat(first.getCurrReadTimeMs()).isLessThanOrEqualTo(curr + precisionMs);\n\n        for (int i = 0; i <= first.getTimerLog().getMappedFileQueue().getMappedFiles().size() + 10; i++) {\n            first.getTimerLog().getMappedFileQueue().flush(0);\n            Thread.sleep(10);\n        }\n\n        // Damage the timer wheel, trigger the check physical pos.\n        Slot slot = first.getTimerWheel().getSlot(delayMs - precisionMs);\n        assertNotEquals(-1, slot.timeMs);\n        first.getTimerWheel().putSlot(slot.timeMs, -1, Long.MAX_VALUE, slot.num, slot.magic);\n        first.getTimerWheel().flush();\n        first.shutdown();\n\n        final TimerMessageStore second = createTimerMessageStore(base , false);\n        second.debug = true;\n        assertTrue(second.load());\n        assertEquals(msgNum, second.getQueueOffset());\n        assertEquals(second.getCommitQueueOffset(), second.getQueueOffset());\n        assertEquals(second.getCurrReadTimeMs(), second.getCommitReadTimeMs());\n        assertEquals(first.getCommitReadTimeMs(), second.getCommitReadTimeMs());\n        second.start(true);\n\n        // Wait until all messages have been written back to commitLog and consumeQueue.\n        await().atMost(30000, TimeUnit.MILLISECONDS).until(new Callable<Boolean>() {\n            @Override\n            public Boolean call() {\n                ConsumeQueue cq = (ConsumeQueue) messageStore.getConsumeQueue(topic, 0);\n                return cq != null && cq.getMaxOffsetInQueue() >= msgNum - 1;\n            }\n        });\n\n        for (int i = 0; i < msgNum; i++) {\n            ByteBuffer msgBuff = getOneMessage(topic, 0, i, 2000);\n            assertThat(msgBuff).isNotNull();\n        }\n        second.shutdown();\n    }\n\n    @Test\n    public void testMaxDelaySec() throws Exception {\n        String topic = \"TimerTest_testMaxDelaySec\";\n\n        TimerMessageStore first = createTimerMessageStore(null , false);\n        first.load();\n        first.start(true);\n\n        long curr = System.currentTimeMillis() / precisionMs * precisionMs;\n        long delaySec = storeConfig.getTimerMaxDelaySec() + 20;\n\n        MessageExtBrokerInner absolute = buildMessage(curr + delaySec * 1000, topic, false);\n        PutMessageResult putMessageResult = transformTimerMessage(first,absolute);\n        assertEquals(PutMessageStatus.WHEEL_TIMER_MSG_ILLEGAL, putMessageResult.getPutMessageStatus());\n\n        MessageExtBrokerInner relative = buildMessage(delaySec * 1000, topic, true);\n        putMessageResult = transformTimerMessage(first,relative);\n        assertEquals(PutMessageStatus.WHEEL_TIMER_MSG_ILLEGAL, putMessageResult.getPutMessageStatus());\n    }\n\n\n    @Test\n    public void testRollMessage() throws Exception {\n        storeConfig.setTimerRollWindowSlot(2);\n        String topic = \"TimerTest_testRollMessage\";\n\n        TimerMessageStore timerMessageStore = createTimerMessageStore(null , false);\n        timerMessageStore.load();\n        timerMessageStore.start(true);\n\n        long curr = System.currentTimeMillis() / precisionMs * precisionMs;\n        long delayMs = curr + 4 * precisionMs;\n        MessageExtBrokerInner inner = buildMessage(delayMs, topic, false);\n        transformTimerMessage(timerMessageStore,inner);\n        assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(inner).getPutMessageStatus());\n\n        ByteBuffer msgBuff = getOneMessage(topic, 0, 0, 5000);\n        assertNotNull(msgBuff);\n        MessageExt msgExt = MessageDecoder.decode(msgBuff);\n        assertNotNull(msgExt);\n        assertEquals(1, Integer.valueOf(msgExt.getProperty(MessageConst.PROPERTY_TIMER_ROLL_TIMES)).intValue());\n        storeConfig.setTimerRollWindowSlot(Integer.MAX_VALUE);\n    }\n\n    public ByteBuffer getOneMessage(String topic, int queue, long offset, int timeout) throws Exception {\n        int retry = timeout / 100;\n        while (retry-- > 0) {\n            GetMessageResult getMessageResult = messageStore.getMessage(\"TimerGroup\", topic, queue, offset, 1, null);\n            if (null != getMessageResult && GetMessageStatus.FOUND == getMessageResult.getStatus()) {\n                return getMessageResult.getMessageBufferList().get(0);\n            }\n            Thread.sleep(100);\n        }\n        return null;\n    }\n\n    public MessageExtBrokerInner buildMessage(long delayedMs, String topic, boolean relative) {\n        MessageExtBrokerInner msg = new MessageExtBrokerInner();\n        msg.setTopic(topic);\n        msg.setQueueId(0);\n        msg.setTags(counter.incrementAndGet() + \"\");\n        msg.setKeys(\"timer\");\n        if (relative) {\n            MessageAccessor.putProperty(msg, MessageConst.PROPERTY_TIMER_DELAY_SEC, delayedMs / 1000 + \"\");\n        } else {\n            MessageAccessor.putProperty(msg, MessageConst.PROPERTY_TIMER_DELIVER_MS, delayedMs + \"\");\n        }\n        msg.setBody(msgBody);\n        msg.setKeys(String.valueOf(System.currentTimeMillis()));\n        msg.setQueueId(0);\n        msg.setBornTimestamp(System.currentTimeMillis());\n        msg.setBornHost(bornHost);\n        msg.setStoreHost(storeHost);\n        MessageClientIDSetter.setUniqID(msg);\n        TopicFilterType topicFilterType = MessageExt.parseTopicFilterType(msg.getSysFlag());\n        long tagsCodeValue =\n                MessageExtBrokerInner.tagsString2tagsCode(topicFilterType, msg.getTags());\n        msg.setTagsCode(tagsCodeValue);\n        msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n        return msg;\n    }\n\n    private class MyMessageArrivingListener implements MessageArrivingListener {\n        @Override\n        public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime,\n                             byte[] filterBitMap, Map<String, String> properties) {\n        }\n    }\n\n    @After\n    public void clear() {\n        for (TimerMessageStore store : timerStores) {\n            store.shutdown();\n        }\n        for (String baseDir : baseDirs) {\n            StoreTestUtils.deleteFile(baseDir);\n        }\n        if (null != messageStore) {\n            messageStore.shutdown();\n            messageStore.destroy();\n        }\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/timer/TimerMetricsTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.timer;\n\nimport org.apache.rocketmq.common.message.MessageAccessor;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\npublic class TimerMetricsTest {\n\n\n    @Test\n    public void testTimingCount() {\n        String baseDir = StoreTestUtils.createBaseDir();\n\n        TimerMetrics first = new TimerMetrics(baseDir);\n        Assert.assertTrue(first.load());\n        MessageExt msg = new MessageExt();\n        MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, \"AAA\");\n        first.addAndGet(msg, 1000);\n        MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, \"BBB\");\n        first.addAndGet(msg, 2000);\n        Assert.assertEquals(1000, first.getTimingCount(\"AAA\"));\n        Assert.assertEquals(2000, first.getTimingCount(\"BBB\"));\n        long curr = System.currentTimeMillis();\n        Assert.assertTrue(first.getTopicPair(\"AAA\").getTimeStamp() > curr - 10);\n        Assert.assertTrue(first.getTopicPair(\"AAA\").getTimeStamp() <= curr);\n        first.persist();\n\n        TimerMetrics second = new TimerMetrics(baseDir);\n        Assert.assertTrue(second.load());\n        Assert.assertEquals(1000, second.getTimingCount(\"AAA\"));\n        Assert.assertEquals(2000, second.getTimingCount(\"BBB\"));\n        Assert.assertTrue(second.getTopicPair(\"BBB\").getTimeStamp() > curr - 100);\n        Assert.assertTrue(second.getTopicPair(\"BBB\").getTimeStamp() <= curr);\n        second.persist();\n        StoreTestUtils.deleteFile(baseDir);\n    }\n\n    @Test\n    @SuppressWarnings(\"DoubleBraceInitialization\")\n    public void testTimingDistribution() {\n        String baseDir = StoreTestUtils.createBaseDir();\n        TimerMetrics first = new TimerMetrics(baseDir);\n        List<Integer> timerDist = new ArrayList<Integer>() {{\n                add(5);\n                add(60);\n                add(300); // 5s, 1min, 5min\n                add(900);\n                add(3600);\n                add(14400); // 15min, 1h, 4h\n                add(28800);\n                add(86400); // 8h, 24h\n            }};\n        for (int period : timerDist) {\n            first.updateDistPair(period, period);\n        }\n\n        int temp = 0;\n\n        for (int j = 0; j < 50; j++) {\n            for (int period : timerDist) {\n                Assert.assertEquals(first.getDistPair(period).getCount().get(),period + temp);\n                first.updateDistPair(period, j);\n            }\n            temp += j;\n        }\n\n        StoreTestUtils.deleteFile(baseDir);\n    }\n}\n"
  },
  {
    "path": "store/src/test/java/org/apache/rocketmq/store/timer/TimerWheelTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.store.timer;\n\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport java.io.IOException;\n\nimport static org.junit.Assert.assertEquals;\n\npublic class TimerWheelTest {\n\n    private String baseDir;\n\n    private final int slotsTotal = 30;\n    private final int precisionMs = 500;\n    private TimerWheel timerWheel;\n\n    private final long defaultDelay = System.currentTimeMillis() / precisionMs * precisionMs;\n\n    @Before\n    public void init() throws IOException {\n        baseDir = StoreTestUtils.createBaseDir();\n        timerWheel = new TimerWheel(baseDir, slotsTotal, precisionMs);\n    }\n\n    @Test\n    public void testPutGet() {\n        long delayedTime = defaultDelay + precisionMs;\n\n        Slot first = timerWheel.getSlot(delayedTime);\n        assertEquals(-1, first.timeMs);\n        assertEquals(-1, first.firstPos);\n        assertEquals(-1, first.lastPos);\n\n        timerWheel.putSlot(delayedTime, 1, 2, 3, 4);\n        Slot second = timerWheel.getSlot(delayedTime);\n        assertEquals(delayedTime, second.timeMs);\n        assertEquals(1, second.firstPos);\n        assertEquals(2, second.lastPos);\n        assertEquals(3, second.num);\n        assertEquals(4, second.magic);\n    }\n\n    @Test\n    public void testGetNum() {\n        long delayedTime = defaultDelay + precisionMs;\n\n        timerWheel.putSlot(delayedTime, 1, 2, 3, 4);\n        assertEquals(3, timerWheel.getNum(delayedTime));\n        assertEquals(3, timerWheel.getAllNum(delayedTime));\n\n        timerWheel.putSlot(delayedTime + 5 * precisionMs, 5, 6, 7, 8);\n        assertEquals(7, timerWheel.getNum(delayedTime + 5 * precisionMs));\n        assertEquals(10, timerWheel.getAllNum(delayedTime));\n    }\n\n    @Test\n    public void testCheckPhyPos() {\n        long delayedTime = defaultDelay + precisionMs;\n        timerWheel.putSlot(delayedTime, 1, 100, 1, 0);\n        timerWheel.putSlot(delayedTime + 5 * precisionMs, 2, 200, 2, 0);\n        timerWheel.putSlot(delayedTime + 10 * precisionMs, 3, 300, 3, 0);\n\n        assertEquals(1, timerWheel.checkPhyPos(delayedTime, 50));\n        assertEquals(2, timerWheel.checkPhyPos(delayedTime, 100));\n        assertEquals(3, timerWheel.checkPhyPos(delayedTime, 200));\n        assertEquals(Long.MAX_VALUE, timerWheel.checkPhyPos(delayedTime, 300));\n        assertEquals(Long.MAX_VALUE, timerWheel.checkPhyPos(delayedTime, 400));\n\n        assertEquals(2, timerWheel.checkPhyPos(delayedTime + 5 * precisionMs, 50));\n        assertEquals(2, timerWheel.checkPhyPos(delayedTime + 5 * precisionMs, 100));\n        assertEquals(3, timerWheel.checkPhyPos(delayedTime + 5 * precisionMs, 200));\n        assertEquals(Long.MAX_VALUE, timerWheel.checkPhyPos(delayedTime + 5 * precisionMs, 300));\n        assertEquals(Long.MAX_VALUE, timerWheel.checkPhyPos(delayedTime + 5 * precisionMs, 400));\n    }\n\n    @Test\n    public void testPutRevise() {\n        long delayedTime = System.currentTimeMillis() / precisionMs * precisionMs + 3 * precisionMs;\n        timerWheel.putSlot(delayedTime, 1, 2);\n\n        timerWheel.reviseSlot(delayedTime + 5 * precisionMs, 3, 4, false);\n        Slot second = timerWheel.getSlot(delayedTime);\n        assertEquals(delayedTime, second.timeMs);\n        assertEquals(1, second.firstPos);\n        assertEquals(2, second.lastPos);\n\n        timerWheel.reviseSlot(delayedTime, TimerWheel.IGNORE, 4, false);\n        Slot three = timerWheel.getSlot(delayedTime);\n        assertEquals(1, three.firstPos);\n        assertEquals(4, three.lastPos);\n\n        timerWheel.reviseSlot(delayedTime, 3, TimerWheel.IGNORE, false);\n        Slot four = timerWheel.getSlot(delayedTime);\n        assertEquals(3, four.firstPos);\n        assertEquals(4, four.lastPos);\n\n        timerWheel.reviseSlot(delayedTime + 2 * slotsTotal * precisionMs, TimerWheel.IGNORE, 5, true);\n        Slot five = timerWheel.getRawSlot(delayedTime);\n        assertEquals(delayedTime + 2 * slotsTotal * precisionMs, five.timeMs);\n        assertEquals(5, five.firstPos);\n        assertEquals(5, five.lastPos);\n    }\n\n    @Test\n    public void testRecoveryData() throws Exception {\n        long delayedTime = System.currentTimeMillis() / precisionMs * precisionMs + 5 * precisionMs;\n        timerWheel.putSlot(delayedTime, 1, 2, 3, 4);\n        timerWheel.flush();\n\n        TimerWheel tmpWheel = new TimerWheel(baseDir, slotsTotal, precisionMs);\n        Slot slot = tmpWheel.getSlot(delayedTime);\n        assertEquals(delayedTime, slot.timeMs);\n        assertEquals(1, slot.firstPos);\n        assertEquals(2, slot.lastPos);\n        assertEquals(3, slot.num);\n        assertEquals(4, slot.magic);\n\n        tmpWheel.shutdown();\n    }\n\n    @Test(expected = RuntimeException.class)\n    public void testRecoveryFixedTTL() throws Exception {\n        timerWheel.flush();\n        TimerWheel tmpWheel = new TimerWheel(baseDir, slotsTotal + 1, precisionMs);\n    }\n\n    @After\n    public void shutdown() {\n        if (null != timerWheel) {\n            timerWheel.shutdown();\n        }\n        if (null != baseDir) {\n            StoreTestUtils.deleteFile(baseDir);\n        }\n    }\n\n\n}\n"
  },
  {
    "path": "store/src/test/resources/rmq.logback-test.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n<configuration>\n\n    <appender name=\"CONSOLE\" class=\"ch.qos.logback.core.ConsoleAppender\">\n        <layout class=\"ch.qos.logback.classic.PatternLayout\">\n            <Pattern>\n                %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n\n            </Pattern>\n        </layout>\n    </appender>\n\n    <logger name=\"org.apache.rocketmq\" level=\"error\" additivity=\"false\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </logger>\n\n    <root level=\"error\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </root>\n\n</configuration>"
  },
  {
    "path": "style/copyright/Apache.xml",
    "content": "<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n<component name=\"CopyrightManager\">\n    <copyright>\n        <option name=\"myName\" value=\"Apache\" />\n        <option name=\"notice\" value=\"Licensed to the Apache Software Foundation (ASF) under one or more&#10;contributor license agreements.  See the NOTICE file distributed with&#10;this work for additional information regarding copyright ownership.&#10;The ASF licenses this file to You under the Apache License, Version 2.0&#10;(the &quot;License&quot;); you may not use this file except in compliance with&#10;the License.  You may obtain a copy of the License at&#10;&#10;    http://www.apache.org/licenses/LICENSE-2.0&#10;&#10;Unless required by applicable law or agreed to in writing, software&#10;distributed under the License is distributed on an &quot;AS IS&quot; BASIS,&#10;WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.&#10;See the License for the specific language governing permissions and&#10;limitations under the License.\" />\n    </copyright>\n</component>"
  },
  {
    "path": "style/copyright/profiles_settings.xml",
    "content": "<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n<component name=\"CopyrightManager\">\n    <settings default=\"Apache\">\n        <module2copyright>\n            <element module=\"All\" copyright=\"Apache\"/>\n        </module2copyright>\n        <LanguageOptions name=\"GSP\">\n            <option name=\"fileTypeOverride\" value=\"3\"/>\n            <option name=\"prefixLines\" value=\"false\"/>\n        </LanguageOptions>\n        <LanguageOptions name=\"HTML\">\n            <option name=\"fileTypeOverride\" value=\"3\"/>\n            <option name=\"prefixLines\" value=\"false\"/>\n        </LanguageOptions>\n        <LanguageOptions name=\"JAVA\">\n            <option name=\"fileTypeOverride\" value=\"3\" />\n            <option name=\"addBlankAfter\" value=\"false\" />\n        </LanguageOptions>\n        <LanguageOptions name=\"JSP\">\n            <option name=\"fileTypeOverride\" value=\"3\"/>\n            <option name=\"prefixLines\" value=\"false\"/>\n        </LanguageOptions>\n        <LanguageOptions name=\"JSPX\">\n            <option name=\"fileTypeOverride\" value=\"3\"/>\n            <option name=\"prefixLines\" value=\"false\"/>\n        </LanguageOptions>\n        <LanguageOptions name=\"MXML\">\n            <option name=\"fileTypeOverride\" value=\"3\"/>\n            <option name=\"prefixLines\" value=\"false\"/>\n        </LanguageOptions>\n        <LanguageOptions name=\"Properties\">\n            <option name=\"fileTypeOverride\" value=\"3\"/>\n            <option name=\"block\" value=\"false\"/>\n        </LanguageOptions>\n        <LanguageOptions name=\"SPI\">\n            <option name=\"fileTypeOverride\" value=\"3\"/>\n            <option name=\"block\" value=\"false\"/>\n        </LanguageOptions>\n        <LanguageOptions name=\"XML\">\n            <option name=\"fileTypeOverride\" value=\"3\"/>\n            <option name=\"prefixLines\" value=\"false\"/>\n        </LanguageOptions>\n        <LanguageOptions name=\"__TEMPLATE__\">\n            <option name=\"separateBefore\" value=\"true\"/>\n            <option name=\"lenBefore\" value=\"1\"/>\n        </LanguageOptions>\n    </settings>\n</component>"
  },
  {
    "path": "style/rmq_checkstyle.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n<!DOCTYPE module PUBLIC\n    \"-//Puppy Crawl//DTD Check Configuration 1.3//EN\"\n    \"http://www.puppycrawl.com/dtds/configuration_1_3.dtd\">\n<!--Refer http://checkstyle.sourceforge.net/reports/google-java-style.html#s2.2-file-encoding -->\n<module name=\"Checker\">\n\n    <property name=\"localeLanguage\" value=\"en\"/>\n\n    <!--To configure the check to report on the first instance in each file-->\n    <module name=\"FileTabCharacter\"/>\n\n    <!-- header -->\n    <module name=\"RegexpHeader\">\n        <property name=\"header\" value=\"/\\*\\nLicensed to the Apache Software Foundation*\"/>\n        <property name=\"fileExtensions\" value=\"java\"/>\n    </module>\n\n    <module name=\"RegexpHeader\">\n        <property name=\"header\" value=\"#[\\s]*Licensed to the Apache Software Foundation*\"/>\n        <property name=\"fileExtensions\" value=\"properties\"/>\n    </module>\n\n    <module name=\"RegexpSingleline\">\n        <property name=\"format\" value=\"System\\.out\\.println\"/>\n        <property name=\"message\" value=\"Prohibit invoking System.out.println in source code !\"/>\n    </module>\n\n    <module name=\"RegexpSingleline\">\n        <property name=\"format\" value=\"//FIXME\"/>\n        <property name=\"message\" value=\"Recommended fix FIXME task !\"/>\n    </module>\n\n    <module name=\"RegexpSingleline\">\n        <property name=\"format\" value=\"//TODO\"/>\n        <property name=\"message\" value=\"Recommended fix TODO task !\"/>\n    </module>\n\n    <module name=\"RegexpSingleline\">\n        <property name=\"format\" value=\"@alibaba\"/>\n        <property name=\"message\" value=\"Recommended remove @alibaba keyword!\"/>\n    </module>\n    <module name=\"RegexpSingleline\">\n        <property name=\"format\" value=\"@taobao\"/>\n        <property name=\"message\" value=\"Recommended remove @taobao keyword!\"/>\n    </module>\n    <module name=\"RegexpSingleline\">\n        <property name=\"format\" value=\"@author\"/>\n        <property name=\"message\" value=\"Recommended remove @author tag in javadoc!\"/>\n    </module>\n\n    <module name=\"RegexpSingleline\">\n        <property name=\"format\"\n                  value=\".*[\\u3400-\\u4DB5\\u4E00-\\u9FA5\\u9FA6-\\u9FBB\\uF900-\\uFA2D\\uFA30-\\uFA6A\\uFA70-\\uFAD9\\uFF00-\\uFFEF\\u2E80-\\u2EFF\\u3000-\\u303F\\u31C0-\\u31EF]+.*\"/>\n        <property name=\"message\" value=\"Not allow chinese character !\"/>\n    </module>\n\n    <module name=\"FileLength\">\n        <property name=\"max\" value=\"5000\"/>\n    </module>\n\n    <module name=\"TreeWalker\">\n\n        <module name=\"UnusedImports\">\n            <property name=\"processJavadoc\" value=\"true\"/>\n        </module>\n        <module name=\"RedundantImport\"/>\n\n        <!--<module name=\"IllegalImport\" />-->\n\n        <!--Checks that classes that override equals() also override hashCode()-->\n        <module name=\"EqualsHashCode\"/>\n        <!--Checks for over-complicated boolean expressions. Currently finds code like if (topic == true), topic || true, !false, etc.-->\n        <module name=\"SimplifyBooleanExpression\"/>\n        <module name=\"OneStatementPerLine\"/>\n        <module name=\"UnnecessaryParentheses\"/>\n        <!--Checks for over-complicated boolean return statements. For example the following code-->\n        <module name=\"SimplifyBooleanReturn\"/>\n\n        <!--Check that the default is after all the cases in producerGroup switch statement-->\n        <module name=\"DefaultComesLast\"/>\n        <!--Detects empty statements (standalone \";\" semicolon)-->\n        <module name=\"EmptyStatement\"/>\n        <!--Checks that long constants are defined with an upper ell-->\n        <module name=\"UpperEll\"/>\n        <module name=\"ConstantName\">\n            <property name=\"format\" value=\"(^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$)|(^log[a-zA-Z0-9]*$)\"/>\n        </module>\n        <!--Checks that local, non-final variable names conform to producerGroup format specified by the format property-->\n        <module name=\"LocalVariableName\"/>\n        <!--Validates identifiers for local, final variables, including catch parameters-->\n        <module name=\"LocalFinalVariableName\"/>\n        <!--Validates identifiers for non-static fields-->\n        <module name=\"MemberName\"/>\n        <!--Validates identifiers for class type parameters-->\n        <module name=\"ClassTypeParameterName\">\n            <property name=\"format\" value=\"^[A-Z0-9]*$\"/>\n        </module>\n        <!--Validates identifiers for method type parameters-->\n        <module name=\"MethodTypeParameterName\">\n            <property name=\"format\" value=\"^[A-Z0-9]*$\"/>\n        </module>\n        <module name=\"PackageName\"/>\n        <module name=\"ParameterName\"/>\n        <module name=\"StaticVariableName\"/>\n        <module name=\"TypeName\"/>\n        <!--Checks that there are no import statements that use the * notation-->\n        <module name=\"AvoidStarImport\"/>\n\n        <!--whitespace-->\n        <module name=\"GenericWhitespace\"/>\n        <!--<module name=\"NoWhitespaceBefore\"/>-->\n        <!--<module name=\"NoWhitespaceAfter\"/>-->\n        <module name=\"WhitespaceAround\">\n            <property name=\"allowEmptyConstructors\" value=\"true\"/>\n            <property name=\"allowEmptyMethods\" value=\"true\"/>\n        </module>\n        <module name=\"Indentation\"/>\n        <module name=\"MethodParamPad\"/>\n        <module name=\"ParenPad\"/>\n        <module name=\"TypecastParenPad\"/>\n    </module>\n</module>\n"
  },
  {
    "path": "style/rmq_codeStyle.xml",
    "content": "<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n<code_scheme name=\"rocketmq\">\n    <option name=\"USE_SAME_INDENTS\" value=\"true\"/>\n    <option name=\"IGNORE_SAME_INDENTS_FOR_LANGUAGES\" value=\"true\"/>\n    <option name=\"OTHER_INDENT_OPTIONS\">\n        <value>\n            <option name=\"INDENT_SIZE\" value=\"4\"/>\n            <option name=\"CONTINUATION_INDENT_SIZE\" value=\"4\"/>\n            <option name=\"TAB_SIZE\" value=\"4\"/>\n            <option name=\"USE_TAB_CHARACTER\" value=\"false\"/>\n            <option name=\"SMART_TABS\" value=\"false\"/>\n            <option name=\"LABEL_INDENT_SIZE\" value=\"0\"/>\n            <option name=\"LABEL_INDENT_ABSOLUTE\" value=\"false\"/>\n            <option name=\"USE_RELATIVE_INDENTS\" value=\"false\"/>\n        </value>\n    </option>\n    <option name=\"PREFER_LONGER_NAMES\" value=\"false\"/>\n    <option name=\"CLASS_COUNT_TO_USE_IMPORT_ON_DEMAND\" value=\"1000\"/>\n    <option name=\"NAMES_COUNT_TO_USE_IMPORT_ON_DEMAND\" value=\"1000\"/>\n    <option name=\"PACKAGES_TO_USE_IMPORT_ON_DEMAND\">\n        <value/>\n    </option>\n    <option name=\"IMPORT_LAYOUT_TABLE\">\n        <value>\n            <package name=\"\" withSubpackages=\"true\" static=\"false\"/>\n            <emptyLine/>\n            <package name=\"\" withSubpackages=\"true\" static=\"true\"/>\n        </value>\n    </option>\n    <option name=\"JD_ALIGN_PARAM_COMMENTS\" value=\"false\"/>\n    <option name=\"JD_ALIGN_EXCEPTION_COMMENTS\" value=\"false\"/>\n    <option name=\"JD_P_AT_EMPTY_LINES\" value=\"false\"/>\n    <option name=\"JD_KEEP_INVALID_TAGS\" value=\"false\"/>\n    <option name=\"JD_DO_NOT_WRAP_ONE_LINE_COMMENTS\" value=\"true\"/>\n    <option name=\"KEEP_CONTROL_STATEMENT_IN_ONE_LINE\" value=\"false\"/>\n    <option name=\"KEEP_BLANK_LINES_IN_DECLARATIONS\" value=\"1\"/>\n    <option name=\"KEEP_BLANK_LINES_IN_CODE\" value=\"1\"/>\n    <option name=\"KEEP_BLANK_LINES_BEFORE_RBRACE\" value=\"1\"/>\n    <option name=\"WHILE_ON_NEW_LINE\" value=\"true\"/>\n    <option name=\"ALIGN_MULTILINE_PARAMETERS\" value=\"false\"/>\n    <option name=\"ALIGN_MULTILINE_FOR\" value=\"false\"/>\n    <option name=\"SPACE_AFTER_TYPE_CAST\" value=\"true\"/>\n    <option name=\"SPACE_BEFORE_ARRAY_INITIALIZER_LBRACE\" value=\"true\"/>\n    <option name=\"METHOD_PARAMETERS_WRAP\" value=\"1\"/>\n    <option name=\"ARRAY_INITIALIZER_LBRACE_ON_NEXT_LINE\" value=\"true\"/>\n    <option name=\"LABELED_STATEMENT_WRAP\" value=\"1\"/>\n    <option name=\"WRAP_COMMENTS\" value=\"true\"/>\n    <option name=\"METHOD_ANNOTATION_WRAP\" value=\"1\"/>\n    <option name=\"CLASS_ANNOTATION_WRAP\" value=\"1\"/>\n    <option name=\"FIELD_ANNOTATION_WRAP\" value=\"1\"/>\n    <JavaCodeStyleSettings>\n        <option name=\"CLASS_NAMES_IN_JAVADOC\" value=\"3\"/>\n    </JavaCodeStyleSettings>\n    <XML>\n        <option name=\"XML_LEGACY_SETTINGS_IMPORTED\" value=\"true\"/>\n    </XML>\n    <ADDITIONAL_INDENT_OPTIONS fileType=\"haml\">\n        <option name=\"INDENT_SIZE\" value=\"2\"/>\n    </ADDITIONAL_INDENT_OPTIONS>\n    <codeStyleSettings language=\"Groovy\">\n        <option name=\"KEEP_CONTROL_STATEMENT_IN_ONE_LINE\" value=\"false\"/>\n        <option name=\"KEEP_BLANK_LINES_IN_DECLARATIONS\" value=\"1\"/>\n        <option name=\"KEEP_BLANK_LINES_IN_CODE\" value=\"1\"/>\n        <option name=\"KEEP_BLANK_LINES_BEFORE_RBRACE\" value=\"1\"/>\n        <option name=\"ALIGN_MULTILINE_PARAMETERS\" value=\"false\"/>\n        <option name=\"ALIGN_MULTILINE_FOR\" value=\"false\"/>\n        <option name=\"METHOD_PARAMETERS_WRAP\" value=\"1\"/>\n        <option name=\"METHOD_ANNOTATION_WRAP\" value=\"1\"/>\n        <option name=\"CLASS_ANNOTATION_WRAP\" value=\"1\"/>\n        <option name=\"FIELD_ANNOTATION_WRAP\" value=\"1\"/>\n        <option name=\"PARENT_SETTINGS_INSTALLED\" value=\"true\"/>\n        <indentOptions>\n            <option name=\"CONTINUATION_INDENT_SIZE\" value=\"4\"/>\n        </indentOptions>\n    </codeStyleSettings>\n    <codeStyleSettings language=\"HOCON\">\n        <option name=\"KEEP_BLANK_LINES_BEFORE_RBRACE\" value=\"1\"/>\n        <option name=\"PARENT_SETTINGS_INSTALLED\" value=\"true\"/>\n    </codeStyleSettings>\n    <codeStyleSettings language=\"JAVA\">\n        <option name=\"KEEP_CONTROL_STATEMENT_IN_ONE_LINE\" value=\"false\"/>\n        <option name=\"KEEP_BLANK_LINES_IN_DECLARATIONS\" value=\"1\"/>\n        <option name=\"KEEP_BLANK_LINES_IN_CODE\" value=\"1\"/>\n        <option name=\"KEEP_BLANK_LINES_BEFORE_RBRACE\" value=\"1\"/>\n        <option name=\"WHILE_ON_NEW_LINE\" value=\"true\"/>\n        <option name=\"ALIGN_MULTILINE_PARAMETERS\" value=\"false\"/>\n        <option name=\"ALIGN_MULTILINE_FOR\" value=\"false\"/>\n        <option name=\"SPACE_BEFORE_ARRAY_INITIALIZER_LBRACE\" value=\"true\"/>\n        <option name=\"METHOD_PARAMETERS_WRAP\" value=\"1\"/>\n        <option name=\"ARRAY_INITIALIZER_LBRACE_ON_NEXT_LINE\" value=\"true\"/>\n        <option name=\"LABELED_STATEMENT_WRAP\" value=\"1\"/>\n        <option name=\"METHOD_ANNOTATION_WRAP\" value=\"2\"/>\n        <option name=\"CLASS_ANNOTATION_WRAP\" value=\"1\"/>\n        <option name=\"FIELD_ANNOTATION_WRAP\" value=\"1\"/>\n        <option name=\"PARENT_SETTINGS_INSTALLED\" value=\"true\"/>\n        <indentOptions>\n            <option name=\"CONTINUATION_INDENT_SIZE\" value=\"4\"/>\n        </indentOptions>\n    </codeStyleSettings>\n    <codeStyleSettings language=\"JSON\">\n        <option name=\"KEEP_BLANK_LINES_IN_CODE\" value=\"1\"/>\n        <option name=\"PARENT_SETTINGS_INSTALLED\" value=\"true\"/>\n    </codeStyleSettings>\n    <codeStyleSettings language=\"Scala\">\n        <option name=\"KEEP_BLANK_LINES_IN_DECLARATIONS\" value=\"1\"/>\n        <option name=\"KEEP_BLANK_LINES_IN_CODE\" value=\"1\"/>\n        <option name=\"KEEP_BLANK_LINES_BEFORE_RBRACE\" value=\"1\"/>\n        <option name=\"WHILE_ON_NEW_LINE\" value=\"true\"/>\n        <option name=\"ALIGN_MULTILINE_PARAMETERS\" value=\"false\"/>\n        <option name=\"ALIGN_MULTILINE_FOR\" value=\"false\"/>\n        <option name=\"METHOD_PARAMETERS_WRAP\" value=\"1\"/>\n        <option name=\"METHOD_ANNOTATION_WRAP\" value=\"1\"/>\n        <option name=\"CLASS_ANNOTATION_WRAP\" value=\"1\"/>\n        <option name=\"FIELD_ANNOTATION_WRAP\" value=\"1\"/>\n        <option name=\"PARENT_SETTINGS_INSTALLED\" value=\"true\"/>\n        <indentOptions>\n            <option name=\"INDENT_SIZE\" value=\"4\"/>\n            <option name=\"CONTINUATION_INDENT_SIZE\" value=\"4\"/>\n            <option name=\"TAB_SIZE\" value=\"4\"/>\n        </indentOptions>\n    </codeStyleSettings>\n    <codeStyleSettings language=\"XML\">\n        <indentOptions>\n            <option name=\"CONTINUATION_INDENT_SIZE\" value=\"4\"/>\n        </indentOptions>\n    </codeStyleSettings>\n</code_scheme>"
  },
  {
    "path": "style/spotbugs-suppressions.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n      http://www.apache.org/licenses/LICENSE-2.0\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n<FindBugsFilter>\n    <Match>\n        <Class name=\"org.apache.rocketmq.remoting.netty.TlsSystemConfig\"/>\n        <Field name=\"tlsConfigFile\"/>\n        <Bug pattern=\"MS_SHOULD_BE_FINAL\"/>\n    </Match>\n    <Match>\n        <Class name=\"org.apache.rocketmq.remoting.netty.TlsSystemConfig\"/>\n        <Field name=\"tlsMode\"/>\n        <Bug pattern=\"MS_SHOULD_BE_FINAL\"/>\n    </Match>\n    <Match>\n        <Class name=\"org.apache.rocketmq.store.index.IndexFile\"/>\n        <Method name=\"indexKeyHashMethod\" />\n        <Bug pattern=\"RV_ABSOLUTE_VALUE_OF_HASHCODE\"/>\n    </Match>\n    <Match>\n        <Class name=\"org.apache.rocketmq.tieredstore.index.TieredIndexFile\"/>\n        <Method name=\"indexKeyHashMethod\" />\n        <Bug pattern=\"RV_ABSOLUTE_VALUE_OF_HASHCODE\"/>\n    </Match>\n    <Match>\n        <Class name=\"org.apache.rocketmq.broker.transaction.queue.DefaultTransactionalMessageCheckListener\"/>\n        <Method name=\"toMessageExtBrokerInner\" />\n        <Bug pattern=\"INT_BAD_REM_BY_1\"/>\n    </Match>\n    <Match>\n        <Class name=\"org.apache.rocketmq.tools.command.cluster.ClusterListSubCommand\"/>\n        <Method name=\"execute\" />\n        <Bug pattern=\"IL_INFINITE_LOOP\"/>\n    </Match>\n</FindBugsFilter>\n"
  },
  {
    "path": "test/BUILD.bazel",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\nload(\"//bazel:GenTestRules.bzl\", \"GenTestRules\")\n\njava_library(\n    name = \"test\",\n    srcs = glob([\"src/main/java/**/*.java\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \"//client\",\n        \"//common\",\n        \"//remoting\",\n        \"//srvutil\",\n        \"//tools\",\n        \"@maven//:ch_qos_logback_logback_classic\",\n        \"@maven//:ch_qos_logback_logback_core\",\n        \"@maven//:com_github_luben_zstd_jni\",\n        \"@maven//:com_google_guava_guava\",\n        \"@maven//:com_google_protobuf_protobuf_java_util\",\n        \"@maven//:com_google_truth_truth\",\n        \"@maven//:commons_cli_commons_cli\",\n        \"@maven//:commons_validator_commons_validator\",\n        \"@maven//:io_netty_netty_all\",\n        \"@maven//:org_apache_tomcat_annotations_api\",\n        \"@maven//:org_apache_commons_commons_lang3\",\n        \"@maven//:org_awaitility_awaitility\",\n        \"@maven//:org_lz4_lz4_java\",\n        \"@maven//:org_reflections_reflections\",\n        \"@maven//:org_slf4j_slf4j_api\",\n        \"@maven//:io_github_aliyunmq_rocketmq_slf4j_api\",\n        \"@maven//:io_github_aliyunmq_rocketmq_logback_classic\",\n    ],\n)\n\njava_library(\n    name = \"tests\",\n    srcs = glob([\"src/test/java/**/*.java\"]),\n    resources = [\n        \"src/test/resources/rmq-proxy-home/conf/broker.conf\",\n        \"src/test/resources/rmq-proxy-home/conf/logback_proxy.xml\",\n        \"src/test/resources/rmq-proxy-home/conf/rmq-proxy.json\",\n        \"src/test/resources/rmq.logback-test.xml\",\n    ] + glob([\"src/test/resources/schema/**/*.schema\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \":test\",\n        \"//:test_deps\",\n        \"//broker\",\n        \"//client\",\n        \"//common\",\n        \"//container\",\n        \"//controller\",\n        \"//namesrv\",\n        \"//proxy\",\n        \"//remoting\",\n        \"//store\",\n        \"//tools\",\n        \"@maven//:com_google_guava_guava\",\n        \"@maven//:com_google_protobuf_protobuf_java\",\n        \"@maven//:com_google_protobuf_protobuf_java_util\",\n        \"@maven//:com_google_truth_truth\",\n        \"@maven//:io_grpc_grpc_api\",\n        \"@maven//:io_grpc_grpc_context\",\n        \"@maven//:io_grpc_grpc_netty_shaded\",\n        \"@maven//:io_grpc_grpc_stub\",\n        \"@maven//:io_grpc_grpc_testing\",\n        \"@maven//:io_netty_netty_all\",\n        \"@maven//:org_apache_commons_commons_lang3\",\n        \"@maven//:org_apache_rocketmq_rocketmq_proto\",\n        \"@maven//:io_github_aliyunmq_rocketmq_slf4j_api\",\n    ],\n)\n\nGenTestRules(\n    name = \"GeneratedTestRules\",\n    default_test_size = \"medium\",\n    exclude_tests = [\n        \"src/test/java/org/apache/rocketmq/test/grpc/v2/GrpcBaseIT\",\n        # Following tests are found flaky\n        \"src/test/java/org/apache/rocketmq/test/statictopic/StaticTopicIT\",\n        \"src/test/java/org/apache/rocketmq/test/client/consumer/topic/MulConsumerMulTopicIT\",\n        \"src/test/java/org/apache/rocketmq/test/client/consumer/tag/MulTagSubIT\",\n        \"src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWith1ConsumerIT\",\n        \"src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWithMulConsumerIT\",\n        \"src/test/java/org/apache/rocketmq/test/client/consumer/topic/OneConsumerMulTopicIT\",\n        \"src/test/java/org/apache/rocketmq/test/client/producer/order/OrderMsgDynamicRebalanceIT\",\n        \"src/test/java/org/apache/rocketmq/test/client/producer/order/OrderMsgIT\",\n        \"src/test/java/org/apache/rocketmq/test/client/producer/order/OrderMsgWithTagIT\",\n        \"src/test/java/org/apache/rocketmq/test/client/consumer/cluster/DynamicAddAndCrashIT\",\n        \"src/test/java/org/apache/rocketmq/test/client/consumer/cluster/DynamicAddConsumerIT\",\n        \"src/test/java/org/apache/rocketmq/test/client/consumer/cluster/DynamicCrashConsumerIT\",\n        \"src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWithSameGroupConsumerIT\",\n        \"src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueIT\",\n        \"src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueSelectorIT\",\n        \"src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithOnlySendCallBackIT\",\n        \"src/test/java/org/apache/rocketmq/test/client/producer/exception/msg/MessageUserPropIT\",\n        \"src/test/java/org/apache/rocketmq/test/client/producer/oneway/OneWaySendIT\",\n        \"src/test/java/org/apache/rocketmq/test/client/producer/oneway/OneWaySendWithMQIT\",\n        \"src/test/java/org/apache/rocketmq/test/offset/OffsetNotFoundIT\",\n        \"src/test/java/org/apache/rocketmq/test/recall/RecallWithTraceIT\",\n        \"src/test/java/org/apache/rocketmq/test/recall/SendAndRecallDelayMessageIT\",\n        \"src/test/java/org/apache/rocketmq/test/client/producer/querymsg/QueryMsgByIdIT\",\n        \"src/test/java/org/apache/rocketmq/test/client/producer/oneway/OneWaySendWithSelectorIT\",\n        \"src/test/java/org/apache/rocketmq/test/smoke/NormalMessageSendAndRecvIT\",\n        \"src/test/java/org/apache/rocketmq/test/base/dledger/DLedgerProduceAndConsumeIT\",\n    \t\"src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopOrderlyIT\",\n        \"src/test/java/org/apache/rocketmq/test/delay/NormalMsgDelayIT\",\n        \"src/test/java/org/apache/rocketmq/test/client/producer/querymsg/QueryMsgByIdExceptionIT\",\n        \"src/test/java/org/apache/rocketmq/test/offset/LagCalculationIT\",\n        \"src/test/java/org/apache/rocketmq/test/grpc/v2/ClusterGrpcIT\",\n    ],\n    flaky_tests = [\n       \"src/test/java/org/apache/rocketmq/test/client/producer/querymsg/QueryMsgByKeyIT\", \n    ],\n    test_files = glob([\"src/test/java/**/*IT.java\"]),\n    deps = [\n        \":tests\",\n    ],\n)\n"
  },
  {
    "path": "test/pom.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one or more\n  ~ contributor license agreements.  See the NOTICE file distributed with\n  ~ this work for additional information regarding copyright ownership.\n  ~ The ASF licenses this file to You under the Apache License, Version 2.0\n  ~ (the \"License\"); you may not use this file except in compliance with\n  ~ the License.  You may obtain a copy of the License at\n  ~\n  ~     http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~  Unless required by applicable law or agreed to in writing, software\n  ~  distributed under the License is distributed on an \"AS IS\" BASIS,\n  ~  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  ~  See the License for the specific language governing permissions and\n  ~  limitations under the License.\n  -->\n\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n    <parent>\n        <artifactId>rocketmq-all</artifactId>\n        <groupId>org.apache.rocketmq</groupId>\n        <version>${revision}</version>\n    </parent>\n\n    <modelVersion>4.0.0</modelVersion>\n    <artifactId>rocketmq-test</artifactId>\n    <name>rocketmq-test ${project.version}</name>\n\n    <properties>\n        <project.root>${basedir}/..</project.root>\n    </properties>\n\n    <dependencies>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-proto</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>com.google.protobuf</groupId>\n            <artifactId>protobuf-java-util</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-proxy</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-broker</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-namesrv</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-container</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.apache.tomcat</groupId>\n            <artifactId>annotations-api</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>com.google.truth</groupId>\n            <artifactId>truth</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>junit</groupId>\n            <artifactId>junit</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-client</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>${project.groupId}</groupId>\n            <artifactId>rocketmq-tools</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.grpc</groupId>\n            <artifactId>grpc-testing</artifactId>\n            <scope>test</scope>\n        </dependency>\n        <dependency>\n            <groupId>org.awaitility</groupId>\n            <artifactId>awaitility</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.reflections</groupId>\n            <artifactId>reflections</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.github.aliyunmq</groupId>\n            <artifactId>rocketmq-slf4j-api</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>io.github.aliyunmq</groupId>\n            <artifactId>rocketmq-logback-classic</artifactId>\n        </dependency>\n        <dependency>\n            <groupId>org.slf4j</groupId>\n            <artifactId>slf4j-api</artifactId>\n            <scope>test</scope>\n        </dependency>\n    </dependencies>\n\n    <build>\n        <plugins>\n            <plugin>\n                <groupId>org.apache.maven.plugins</groupId>\n                <artifactId>maven-jar-plugin</artifactId>\n                <executions>\n                    <execution>\n                        <goals>\n                            <goal>test-jar</goal>\n                        </goals>\n                    </execution>\n                </executions>\n            </plugin>\n        </plugins>\n    </build>\n</project>\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/client/mq/MQAsyncProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.mq;\n\nimport java.util.concurrent.atomic.AtomicBoolean;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.clientinterface.AbstractMQProducer;\nimport org.apache.rocketmq.test.util.TestUtil;\n\npublic class MQAsyncProducer {\n    private static Logger logger = LoggerFactory.getLogger(MQAsyncProducer.class);\n    private AbstractMQProducer producer = null;\n    private long msgNum;\n    private int intervalMills;\n    private Thread sendT;\n    private AtomicBoolean bPause = new AtomicBoolean(false);\n\n    public MQAsyncProducer(final AbstractMQProducer producer, final long msgNum,\n        final int intervalMills) {\n        this.producer = producer;\n        this.msgNum = msgNum;\n        this.intervalMills = intervalMills;\n\n        sendT = new Thread(new Runnable() {\n            public void run() {\n                for (int i = 0; i < msgNum; i++) {\n                    if (!bPause.get()) {\n                        producer.send();\n                        TestUtil.waitForMonment(intervalMills);\n                    } else {\n                        while (true) {\n                            if (bPause.get()) {\n                                TestUtil.waitForMonment(10);\n                            } else\n                                break;\n                        }\n                    }\n\n                }\n            }\n        });\n\n    }\n\n    public void start() {\n        sendT.start();\n    }\n\n    public void waitSendAll(int waitMills) {\n        long startTime = System.currentTimeMillis();\n        while ((producer.getAllMsgBody().size() + producer.getSendErrorMsg().size()) < msgNum) {\n            if (System.currentTimeMillis() - startTime < waitMills) {\n                TestUtil.waitForMonment(200);\n            } else {\n                logger.error(String.format(\"time elapse:%s, but the message sending has not finished\",\n                    System.currentTimeMillis() - startTime));\n                break;\n            }\n        }\n    }\n\n    public void pauseProducer() {\n        bPause.set(true);\n    }\n\n    public void notifyProducer() {\n        bPause.set(false);\n    }\n\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQAsyncSendProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.rmq;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.MessageQueueSelector;\nimport org.apache.rocketmq.client.producer.SendCallback;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.clientinterface.AbstractMQProducer;\nimport org.apache.rocketmq.test.sendresult.ResultWrapper;\nimport org.apache.rocketmq.test.util.RandomUtil;\nimport org.apache.rocketmq.test.util.TestUtil;\n\npublic class RMQAsyncSendProducer extends AbstractMQProducer {\n    private static Logger logger = LoggerFactory\n        .getLogger(RMQAsyncSendProducer.class);\n    private String nsAddr = null;\n    private DefaultMQProducer producer = null;\n    private SendCallback sendCallback = null;\n    private List<SendResult> successSendResult = Collections.synchronizedList(new ArrayList<SendResult>());\n    private AtomicInteger exceptionMsgCount = new AtomicInteger(0);\n    private int msgSize = 0;\n\n    public RMQAsyncSendProducer(String nsAddr, String topic) {\n        super(topic);\n        this.nsAddr = nsAddr;\n        sendCallback = new SendCallback() {\n            @Override\n            public void onSuccess(SendResult sendResult) {\n                successSendResult.add(sendResult);\n            }\n            @Override\n            public void onException(Throwable throwable) {\n                exceptionMsgCount.getAndIncrement();\n            }\n        };\n\n        create();\n        start();\n    }\n\n    public int getSuccessMsgCount() {\n        return successSendResult.size();\n    }\n\n    public List<SendResult> getSuccessSendResult() {\n        return successSendResult;\n    }\n\n    public int getExceptionMsgCount() {\n        return exceptionMsgCount.get();\n    }\n\n    private void create() {\n        producer = new DefaultMQProducer();\n        producer.setProducerGroup(RandomUtil.getStringByUUID());\n        producer.setInstanceName(RandomUtil.getStringByUUID());\n\n        if (nsAddr != null) {\n            producer.setNamesrvAddr(nsAddr);\n        }\n\n    }\n\n    private void start() {\n        try {\n            producer.start();\n        } catch (MQClientException e) {\n            logger.error(\"producer start failed!\");\n            e.printStackTrace();\n        }\n    }\n\n    @Override\n    public ResultWrapper send(Object msg, Object arg) {\n        return null;\n    }\n\n    @Override\n    public void shutdown() {\n        producer.shutdown();\n    }\n\n    public void asyncSend(Object msg) {\n        Message metaqMsg = (Message) msg;\n        try {\n            producer.send(metaqMsg, sendCallback);\n            msgBodys.addData(new String(metaqMsg.getBody(), StandardCharsets.UTF_8));\n            originMsgs.addData(msg);\n        } catch (Exception e) {\n            e.printStackTrace();\n        }\n    }\n\n    public void asyncSend(int msgSize) {\n        this.msgSize = msgSize;\n\n        for (int i = 0; i < msgSize; i++) {\n            Message msg = new Message(topic, RandomUtil.getStringByUUID().getBytes(StandardCharsets.UTF_8));\n            this.asyncSend(msg);\n        }\n    }\n\n    public void asyncSend(Object msg, MessageQueueSelector selector, Object arg) {\n        Message metaqMsg = (Message) msg;\n        try {\n            producer.send(metaqMsg, selector, arg, sendCallback);\n            msgBodys.addData(new String(metaqMsg.getBody(), StandardCharsets.UTF_8));\n            originMsgs.addData(msg);\n        } catch (Exception e) {\n            e.printStackTrace();\n        }\n    }\n\n    public void asyncSend(int msgSize, MessageQueueSelector selector) {\n        this.msgSize = msgSize;\n        for (int i = 0; i < msgSize; i++) {\n            Message msg = new Message(topic, RandomUtil.getStringByUUID().getBytes(StandardCharsets.UTF_8));\n            this.asyncSend(msg, selector, i);\n        }\n    }\n\n    public void asyncSend(Object msg, MessageQueue mq) {\n        Message metaqMsg = (Message) msg;\n        try {\n            producer.send(metaqMsg, mq, sendCallback);\n            msgBodys.addData(new String(metaqMsg.getBody(), StandardCharsets.UTF_8));\n            originMsgs.addData(msg);\n        } catch (Exception e) {\n            e.printStackTrace();\n        }\n    }\n\n    public void asyncSend(int msgSize, MessageQueue mq) {\n        this.msgSize = msgSize;\n        for (int i = 0; i < msgSize; i++) {\n            Message msg = new Message(topic, RandomUtil.getStringByUUID().getBytes(StandardCharsets.UTF_8));\n            this.asyncSend(msg, mq);\n        }\n    }\n\n    public void waitForResponse(int timeoutMills) {\n        long startTime = System.currentTimeMillis();\n        while (this.successSendResult.size() != this.msgSize) {\n            if (System.currentTimeMillis() - startTime < timeoutMills) {\n                TestUtil.waitForMonment(100);\n            } else {\n                logger.info(\"timeout but still not recv all response!\");\n                break;\n            }\n        }\n    }\n\n    public void sendOneWay(Object msg) {\n        Message metaqMsg = (Message) msg;\n        try {\n            producer.sendOneway(metaqMsg);\n            msgBodys.addData(new String(metaqMsg.getBody(), StandardCharsets.UTF_8));\n            originMsgs.addData(msg);\n        } catch (Exception e) {\n            e.printStackTrace();\n        }\n    }\n\n    public void sendOneWay(int msgSize) {\n        for (int i = 0; i < msgSize; i++) {\n            Message msg = new Message(topic, RandomUtil.getStringByUUID().getBytes(StandardCharsets.UTF_8));\n            this.sendOneWay(msg);\n        }\n    }\n\n    public void sendOneWay(Object msg, MessageQueue mq) {\n        Message metaqMsg = (Message) msg;\n        try {\n            producer.sendOneway(metaqMsg, mq);\n            msgBodys.addData(new String(metaqMsg.getBody(), StandardCharsets.UTF_8));\n            originMsgs.addData(msg);\n        } catch (Exception e) {\n            e.printStackTrace();\n        }\n    }\n\n    public void sendOneWay(int msgSize, MessageQueue mq) {\n        for (int i = 0; i < msgSize; i++) {\n            Message msg = new Message(topic, RandomUtil.getStringByUUID().getBytes(StandardCharsets.UTF_8));\n            this.sendOneWay(msg, mq);\n        }\n    }\n\n    public void sendOneWay(Object msg, MessageQueueSelector selector, Object arg) {\n        Message metaqMsg = (Message) msg;\n        try {\n            producer.sendOneway(metaqMsg, selector, arg);\n            msgBodys.addData(new String(metaqMsg.getBody(), StandardCharsets.UTF_8));\n            originMsgs.addData(msg);\n        } catch (Exception e) {\n            e.printStackTrace();\n        }\n    }\n\n    public void sendOneWay(int msgSize, MessageQueueSelector selector) {\n        for (int i = 0; i < msgSize; i++) {\n            Message msg = new Message(topic, RandomUtil.getStringByUUID().getBytes(StandardCharsets.UTF_8));\n            this.sendOneWay(msg, selector, i);\n        }\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQBroadCastConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.rmq;\n\nimport org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.listener.AbstractListener;\n\npublic class RMQBroadCastConsumer extends RMQNormalConsumer {\n    private static Logger logger = LoggerFactory.getLogger(RMQBroadCastConsumer.class);\n\n    public RMQBroadCastConsumer(String nsAddr, String topic, String subExpression,\n        String consumerGroup, AbstractListener listener) {\n        super(nsAddr, topic, subExpression, consumerGroup, listener);\n    }\n\n    @Override\n    public void create() {\n        super.create();\n        consumer.setMessageModel(MessageModel.BROADCASTING);\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQNormalConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.rmq;\n\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.clientinterface.AbstractMQConsumer;\nimport org.apache.rocketmq.test.listener.AbstractListener;\nimport org.apache.rocketmq.test.util.RandomUtil;\n\npublic class RMQNormalConsumer extends AbstractMQConsumer {\n\n    private static final Logger LOGGER = LoggerFactory.getLogger(RMQNormalConsumer.class);\n    protected DefaultMQPushConsumer consumer = null;\n\n    public RMQNormalConsumer(String nsAddr, String topic, String subExpression,\n        String consumerGroup, AbstractListener listener) {\n        super(nsAddr, topic, subExpression, consumerGroup, listener);\n    }\n\n    @Override\n    public AbstractListener getListener() {\n        return listener;\n    }\n\n    @Override\n    public void setListener(AbstractListener listener) {\n        this.listener = listener;\n    }\n\n    @Override\n    public void create() {\n        create(false);\n    }\n\n    @Override\n    public void create(boolean useTLS) {\n        consumer = new DefaultMQPushConsumer(consumerGroup);\n        consumer.setInstanceName(RandomUtil.getStringByUUID());\n        consumer.setNamesrvAddr(nsAddr);\n        consumer.setPollNameServerInterval(100);\n        try {\n            consumer.subscribe(topic, subExpression);\n        } catch (MQClientException e) {\n            LOGGER.error(\"consumer subscribe failed!\");\n            e.printStackTrace();\n        }\n        consumer.setMessageListener(listener);\n        consumer.setUseTLS(useTLS);\n    }\n\n    @Override\n    public void start() {\n        try {\n            consumer.start();\n            LOGGER.info(String.format(\"consumer[%s] started!\", consumer.getConsumerGroup()));\n        } catch (MQClientException e) {\n            LOGGER.error(\"consumer start failed!\");\n            e.printStackTrace();\n        }\n    }\n\n    public void subscribe(String topic, String subExpression) {\n        try {\n            consumer.subscribe(topic, subExpression);\n        } catch (MQClientException e) {\n            LOGGER.error(\"consumer subscribe failed!\");\n            e.printStackTrace();\n        }\n    }\n\n    @Override\n    public void shutdown() {\n        consumer.shutdown();\n    }\n\n    @Override\n    public void clearMsg() {\n        this.listener.clearMsg();\n    }\n\n    public void restart() {\n        consumer.shutdown();\n        create();\n        start();\n    }\n\n    public DefaultMQPushConsumer getConsumer() {\n        return consumer;\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQNormalProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.rmq;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.List;\nimport java.util.Map;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.clientinterface.AbstractMQProducer;\nimport org.apache.rocketmq.test.sendresult.ResultWrapper;\n\npublic class RMQNormalProducer extends AbstractMQProducer {\n    private static Logger logger = LoggerFactory.getLogger(RMQNormalProducer.class);\n    private DefaultMQProducer producer = null;\n    private String nsAddr = null;\n\n    public RMQNormalProducer(String nsAddr, String topic) {\n        this(nsAddr, topic, false);\n    }\n\n    public RMQNormalProducer(String nsAddr, String topic, boolean useTLS) {\n        super(topic);\n        this.nsAddr = nsAddr;\n        create(useTLS);\n        start();\n    }\n\n    public RMQNormalProducer(String nsAddr, String topic, String producerGroupName,\n        String producerInstanceName) {\n        this(nsAddr, topic, producerGroupName, producerInstanceName, false);\n    }\n\n    public RMQNormalProducer(String nsAddr, String topic, String producerGroupName,\n        String producerInstanceName, boolean useTLS) {\n        super(topic);\n        this.producerGroupName = producerGroupName;\n        this.producerInstanceName = producerInstanceName;\n        this.nsAddr = nsAddr;\n\n        create(useTLS);\n        start();\n    }\n\n    public DefaultMQProducer getProducer() {\n        return producer;\n    }\n\n    public void setProducer(DefaultMQProducer producer) {\n        this.producer = producer;\n    }\n\n    protected void create(boolean useTLS) {\n        producer = new DefaultMQProducer();\n        producer.setProducerGroup(getProducerGroupName());\n        producer.setInstanceName(getProducerInstanceName());\n        producer.setUseTLS(useTLS);\n        producer.setPollNameServerInterval(100);\n\n        if (nsAddr != null) {\n            producer.setNamesrvAddr(nsAddr);\n        }\n    }\n\n\n    public void start() {\n        try {\n            producer.start();\n            super.setStartSuccess(true);\n        } catch (MQClientException e) {\n            super.setStartSuccess(false);\n            logger.error(\"producer start failed!\");\n            e.printStackTrace();\n        }\n    }\n\n    public ResultWrapper send(Object msg, Object orderKey) {\n        org.apache.rocketmq.client.producer.SendResult internalSendResult = null;\n        Message message = (Message) msg;\n        try {\n            long start = System.currentTimeMillis();\n            internalSendResult = producer.send(message);\n            this.msgRTs.addData(System.currentTimeMillis() - start);\n            if (isDebug) {\n                logger.info(\"SendResult: {}\", internalSendResult);\n            }\n            sendResult.setMsgId(internalSendResult.getMsgId());\n            sendResult.setSendResult(internalSendResult.getSendStatus().equals(SendStatus.SEND_OK));\n            sendResult.setBrokerIp(internalSendResult.getMessageQueue().getBrokerName());\n            msgBodys.addData(new String(message.getBody(), StandardCharsets.UTF_8));\n            originMsgs.addData(msg);\n            originMsgIndex.put(new String(message.getBody(), StandardCharsets.UTF_8), internalSendResult);\n            sendResult.setSendResultObj(internalSendResult);\n        } catch (Exception e) {\n            if (isDebug) {\n                e.printStackTrace();\n            }\n\n            sendResult.setSendResult(false);\n            sendResult.setSendException(e);\n            errorMsgs.addData(msg);\n        }\n\n        return sendResult;\n    }\n\n    public void send(Map<MessageQueue, List<Object>> msgs) {\n        for (MessageQueue mq : msgs.keySet()) {\n            send(msgs.get(mq), mq);\n        }\n    }\n\n    public void send(List<Object> msgs, MessageQueue mq) {\n        for (Object msg : msgs) {\n            sendMQ((Message) msg, mq);\n        }\n    }\n\n    public void send(int num, MessageQueue mq) {\n        for (int i = 0; i < num; i++) {\n            sendMQ((Message) getMessageByTag(null), mq);\n        }\n    }\n\n    public ResultWrapper sendMQ(Message msg, MessageQueue mq) {\n        org.apache.rocketmq.client.producer.SendResult internalSendResult = null;\n        try {\n            long start = System.currentTimeMillis();\n            internalSendResult = producer.send(msg, mq);\n            this.msgRTs.addData(System.currentTimeMillis() - start);\n            if (isDebug) {\n                logger.info(\"SendResult: {}\", internalSendResult);\n            }\n            sendResult.setMsgId(internalSendResult.getMsgId());\n            sendResult.setSendResult(internalSendResult.getSendStatus().equals(SendStatus.SEND_OK));\n            sendResult.setBrokerIp(internalSendResult.getMessageQueue().getBrokerName());\n            msgBodys.addData(new String(msg.getBody(), StandardCharsets.UTF_8));\n            originMsgs.addData(msg);\n            originMsgIndex.put(new String(msg.getBody(), StandardCharsets.UTF_8), internalSendResult);\n        } catch (Exception e) {\n            if (isDebug) {\n                e.printStackTrace();\n            }\n\n            sendResult.setSendResult(false);\n            sendResult.setSendException(e);\n            errorMsgs.addData(msg);\n        }\n\n        return sendResult;\n    }\n\n    public void shutdown() {\n        producer.shutdown();\n    }\n\n    @Override\n    public List<MessageQueue> getMessageQueue() {\n        List<MessageQueue> mqs = null;\n        try {\n            mqs = producer.fetchPublishMessageQueues(topic);\n        } catch (MQClientException e) {\n            e.printStackTrace();\n        }\n        return mqs;\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopClient.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.rmq;\n\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.consumer.AckCallback;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.PopCallback;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.impl.ClientRemotingProcessor;\nimport org.apache.rocketmq.client.impl.mqclient.MQClientAPIExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;\nimport org.apache.rocketmq.remoting.protocol.header.NotificationRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;\nimport org.apache.rocketmq.test.clientinterface.MQConsumer;\nimport org.apache.rocketmq.test.util.RandomUtil;\n\npublic class RMQPopClient implements MQConsumer {\n\n    private static final long DEFAULT_TIMEOUT = 3000;\n\n    private MQClientAPIExt mqClientAPI;\n\n    @Override\n    public void create() {\n        create(false);\n    }\n\n    @Override\n    public void create(boolean useTLS) {\n        ClientConfig clientConfig = new ClientConfig();\n        clientConfig.setInstanceName(RandomUtil.getStringByUUID());\n\n        NettyClientConfig nettyClientConfig = new NettyClientConfig();\n        nettyClientConfig.setUseTLS(useTLS);\n        this.mqClientAPI = new MQClientAPIExt(\n            clientConfig, nettyClientConfig, new ClientRemotingProcessor(null), null);\n    }\n\n    @Override\n    public void start() {\n        this.mqClientAPI.start();\n    }\n\n    @Override\n    public void shutdown() {\n        this.mqClientAPI.shutdown();\n    }\n\n    public CompletableFuture<PopResult> popMessageAsync(String brokerAddr, MessageQueue mq, long invisibleTime,\n        int maxNums, String consumerGroup, long timeout, boolean poll, int initMode, boolean order,\n        String expressionType, String expression) {\n        return popMessageAsync(brokerAddr, mq, invisibleTime, maxNums, consumerGroup, timeout, poll, initMode, order, expressionType, expression, null);\n    }\n\n    public CompletableFuture<PopResult> popMessageAsync(String brokerAddr, MessageQueue mq, long invisibleTime,\n        int maxNums, String consumerGroup, long timeout, boolean poll, int initMode, boolean order,\n        String expressionType, String expression, String attemptId) {\n        PopMessageRequestHeader requestHeader = new PopMessageRequestHeader();\n        requestHeader.setConsumerGroup(consumerGroup);\n        requestHeader.setTopic(mq.getTopic());\n        requestHeader.setQueueId(mq.getQueueId());\n        requestHeader.setMaxMsgNums(maxNums);\n        requestHeader.setInvisibleTime(invisibleTime);\n        requestHeader.setInitMode(initMode);\n        requestHeader.setExpType(expressionType);\n        requestHeader.setExp(expression);\n        requestHeader.setOrder(order);\n        requestHeader.setAttemptId(attemptId);\n        if (poll) {\n            requestHeader.setPollTime(timeout);\n            requestHeader.setBornTime(System.currentTimeMillis());\n            timeout += 10 * 1000;\n        }\n        CompletableFuture<PopResult> future = new CompletableFuture<>();\n        try {\n            this.mqClientAPI.popMessageAsync(mq.getBrokerName(), brokerAddr, requestHeader, timeout, new PopCallback() {\n                @Override\n                public void onSuccess(PopResult popResult) {\n                    future.complete(popResult);\n                }\n\n                @Override\n                public void onException(Throwable e) {\n                    future.completeExceptionally(e);\n                }\n            });\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    public CompletableFuture<AckResult> ackMessageAsync(\n        String brokerAddr, String topic, String consumerGroup, String extraInfo) {\n\n        String[] extraInfoStrs = ExtraInfoUtil.split(extraInfo);\n        AckMessageRequestHeader requestHeader = new AckMessageRequestHeader();\n        requestHeader.setTopic(ExtraInfoUtil.getRealTopic(extraInfoStrs, topic, consumerGroup));\n        requestHeader.setQueueId(ExtraInfoUtil.getQueueId(extraInfoStrs));\n        requestHeader.setOffset(ExtraInfoUtil.getQueueOffset(extraInfoStrs));\n        requestHeader.setConsumerGroup(consumerGroup);\n        requestHeader.setExtraInfo(extraInfo);\n        CompletableFuture<AckResult> future = new CompletableFuture<>();\n        try {\n            this.mqClientAPI.ackMessageAsync(brokerAddr, DEFAULT_TIMEOUT, new AckCallback() {\n                @Override\n                public void onSuccess(AckResult ackResult) {\n                    future.complete(ackResult);\n                }\n\n                @Override\n                public void onException(Throwable e) {\n                    future.completeExceptionally(e);\n                }\n            }, requestHeader);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    public CompletableFuture<AckResult> batchAckMessageAsync(String brokerAddr, String topic, String consumerGroup,\n        List<String> extraInfoList) {\n        CompletableFuture<AckResult> future = new CompletableFuture<>();\n        try {\n            this.mqClientAPI.batchAckMessageAsync(brokerAddr, DEFAULT_TIMEOUT, new AckCallback() {\n                @Override\n                public void onSuccess(AckResult ackResult) {\n                    future.complete(ackResult);\n                }\n\n                @Override\n                public void onException(Throwable e) {\n                    future.completeExceptionally(e);\n                }\n            }, topic, consumerGroup, extraInfoList);\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    public CompletableFuture<AckResult> changeInvisibleTimeAsync(String brokerAddr, String brokerName, String topic,\n        String consumerGroup, String extraInfo, long invisibleTime) {\n        String[] extraInfoStrs = ExtraInfoUtil.split(extraInfo);\n        ChangeInvisibleTimeRequestHeader requestHeader = new ChangeInvisibleTimeRequestHeader();\n        requestHeader.setTopic(ExtraInfoUtil.getRealTopic(extraInfoStrs, topic, consumerGroup));\n        requestHeader.setQueueId(ExtraInfoUtil.getQueueId(extraInfoStrs));\n        requestHeader.setOffset(ExtraInfoUtil.getQueueOffset(extraInfoStrs));\n        requestHeader.setConsumerGroup(consumerGroup);\n        requestHeader.setExtraInfo(extraInfo);\n        requestHeader.setInvisibleTime(invisibleTime);\n\n        CompletableFuture<AckResult> future = new CompletableFuture<>();\n        try {\n            this.mqClientAPI.changeInvisibleTimeAsync(brokerName, brokerAddr, requestHeader, DEFAULT_TIMEOUT, new AckCallback() {\n                @Override\n                public void onSuccess(AckResult ackResult) {\n                    future.complete(ackResult);\n                }\n\n                @Override\n                public void onException(Throwable e) {\n                    future.completeExceptionally(e);\n                }\n            });\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    public CompletableFuture<Boolean> notification(String brokerAddr, String topic,\n        String consumerGroup, int queueId, long pollTime, long bornTime, long timeoutMillis) {\n        return notification(brokerAddr, topic, consumerGroup, queueId, null, null, pollTime, bornTime, timeoutMillis);\n    }\n\n    public CompletableFuture<Boolean> notification(String brokerAddr, String topic,\n        String consumerGroup, int queueId, Boolean order, String attemptId, long pollTime, long bornTime, long timeoutMillis) {\n        return notification(brokerAddr, topic, consumerGroup, queueId, order, attemptId, pollTime, bornTime, timeoutMillis, null, null);\n    }\n\n\n    public CompletableFuture<Boolean> notification(String brokerAddr, String topic,\n        String consumerGroup, int queueId, Boolean order, String attemptId, long pollTime, long bornTime, long timeoutMillis, String expType, String exp) {\n        NotificationRequestHeader requestHeader = new NotificationRequestHeader();\n        requestHeader.setConsumerGroup(consumerGroup);\n        requestHeader.setTopic(topic);\n        requestHeader.setQueueId(queueId);\n        requestHeader.setPollTime(pollTime);\n        requestHeader.setBornTime(bornTime);\n        requestHeader.setOrder(order);\n        requestHeader.setAttemptId(attemptId);\n        requestHeader.setExpType(expType);\n        requestHeader.setExp(exp);\n        return this.mqClientAPI.notification(brokerAddr, requestHeader, timeoutMillis);\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.rmq;\n\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.TimeoutException;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.constant.ConsumeInitMode;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.factory.ConsumerFactory;\nimport org.apache.rocketmq.test.listener.AbstractListener;\n\npublic class RMQPopConsumer extends RMQNormalConsumer {\n\n    private static final Logger log = LoggerFactory.getLogger(RMQPopConsumer.class);\n\n    public static final long POP_TIMEOUT = 3000;\n    public static final long DEFAULT_INVISIBLE_TIME = 30000;\n\n    private RMQPopClient client;\n\n    private int maxNum = 16;\n\n    public RMQPopConsumer(String nsAddr, String topic, String subExpression,\n        String consumerGroup, AbstractListener listener) {\n        super(nsAddr, topic, subExpression, consumerGroup, listener);\n    }\n\n    public RMQPopConsumer(String nsAddr, String topic, String subExpression,\n        String consumerGroup, AbstractListener listener, int maxNum) {\n        super(nsAddr, topic, subExpression, consumerGroup, listener);\n        this.maxNum = maxNum;\n    }\n\n    @Override\n    public void start() {\n        client = ConsumerFactory.getRMQPopClient();\n        log.info(\"consumer[{}] started!\", consumerGroup);\n    }\n\n    @Override\n    public void shutdown() {\n        client.shutdown();\n    }\n\n    public PopResult pop(String brokerAddr, MessageQueue mq) throws Exception {\n        return this.pop(brokerAddr, mq, DEFAULT_INVISIBLE_TIME, 5000);\n    }\n\n    public PopResult pop(String brokerAddr, MessageQueue mq, long invisibleTime, long timeout)\n        throws InterruptedException, RemotingException, MQClientException, MQBrokerException,\n        ExecutionException, TimeoutException {\n\n        CompletableFuture<PopResult> future = this.client.popMessageAsync(\n            brokerAddr, mq, invisibleTime, maxNum, consumerGroup, timeout, true,\n            ConsumeInitMode.MIN, false, ExpressionType.TAG, \"*\");\n\n        return future.get();\n    }\n\n    public PopResult popOrderly(String brokerAddr, MessageQueue mq) throws Exception {\n        return this.popOrderly(brokerAddr, mq, DEFAULT_INVISIBLE_TIME, 5000);\n    }\n\n    public PopResult popOrderly(String brokerAddr, MessageQueue mq, long invisibleTime, long timeout)\n        throws InterruptedException, ExecutionException {\n\n        CompletableFuture<PopResult> future = this.client.popMessageAsync(\n            brokerAddr, mq, invisibleTime, maxNum, consumerGroup, timeout, true,\n            ConsumeInitMode.MIN, true, ExpressionType.TAG, \"*\");\n\n        return future.get();\n    }\n\n    public CompletableFuture<AckResult> ackAsync(String brokerAddr, String extraInfo) {\n        return this.client.ackMessageAsync(brokerAddr, topic, consumerGroup, extraInfo);\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQSqlConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.rmq;\n\nimport org.apache.rocketmq.client.consumer.MessageSelector;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.listener.AbstractListener;\n\npublic class RMQSqlConsumer extends RMQNormalConsumer {\n    private static Logger logger = LoggerFactory.getLogger(RMQSqlConsumer.class);\n    private MessageSelector selector;\n\n    public RMQSqlConsumer(String nsAddr, String topic, MessageSelector selector,\n        String consumerGroup, AbstractListener listener) {\n        super(nsAddr, topic, \"*\", consumerGroup, listener);\n        this.selector = selector;\n    }\n\n    @Override\n    public void create() {\n        super.create();\n        try {\n            consumer.subscribe(topic, selector);\n        } catch (Exception e) {\n            logger.error(\"Subscribe Sql Errored\", e);\n        }\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQTransactionalProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.rmq;\n\nimport java.nio.charset.StandardCharsets;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.LocalTransactionState;\nimport org.apache.rocketmq.client.producer.TransactionListener;\nimport org.apache.rocketmq.client.producer.TransactionMQProducer;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.clientinterface.AbstractMQProducer;\nimport org.apache.rocketmq.test.sendresult.ResultWrapper;\n\npublic class RMQTransactionalProducer extends AbstractMQProducer {\n    private static Logger logger  = LoggerFactory.getLogger(RMQTransactionalProducer.class);\n    private TransactionMQProducer producer = null;\n    private String nsAddr = null;\n\n    public RMQTransactionalProducer(String nsAddr, String topic, TransactionListener transactionListener) {\n        this(nsAddr, topic, false, transactionListener);\n    }\n\n    public RMQTransactionalProducer(String nsAddr, String topic, boolean useTLS, TransactionListener transactionListener) {\n        super(topic);\n        this.nsAddr = nsAddr;\n        create(useTLS, transactionListener);\n        start();\n    }\n\n    protected void create(boolean useTLS, TransactionListener transactionListener) {\n        producer = new TransactionMQProducer();\n        producer.setProducerGroup(getProducerGroupName());\n        producer.setInstanceName(getProducerInstanceName());\n        producer.setTransactionListener(transactionListener);\n        producer.setUseTLS(useTLS);\n\n        if (nsAddr != null) {\n            producer.setNamesrvAddr(nsAddr);\n        }\n    }\n\n    public void start() {\n        try {\n            producer.start();\n            super.setStartSuccess(true);\n        } catch (MQClientException e) {\n            super.setStartSuccess(false);\n            logger.error(\"\", e);\n            e.printStackTrace();\n        }\n    }\n\n    @Override\n    public ResultWrapper send(Object msg, Object arg) {\n        boolean commitMsg = ((Pair<Boolean, LocalTransactionState>) arg).getObject2() == LocalTransactionState.COMMIT_MESSAGE;\n        org.apache.rocketmq.client.producer.SendResult metaqResult = null;\n        Message message = (Message) msg;\n        try {\n            long start = System.currentTimeMillis();\n            metaqResult = producer.sendMessageInTransaction(message, arg);\n            this.msgRTs.addData(System.currentTimeMillis() - start);\n            if (isDebug) {\n                logger.info(\"SendResult: {}\", metaqResult);\n            }\n            sendResult.setMsgId(metaqResult.getMsgId());\n            sendResult.setSendResult(true);\n            sendResult.setBrokerIp(metaqResult.getMessageQueue().getBrokerName());\n            if (commitMsg) {\n                msgBodys.addData(new String(message.getBody(), StandardCharsets.UTF_8));\n            }\n            originMsgs.addData(msg);\n            originMsgIndex.put(new String(message.getBody(), StandardCharsets.UTF_8), metaqResult);\n        } catch (MQClientException e) {\n            if (isDebug) {\n                e.printStackTrace();\n            }\n\n            sendResult.setSendResult(false);\n            sendResult.setSendException(e);\n            errorMsgs.addData(msg);\n        }\n        return sendResult;\n    }\n\n    @Override\n    public void shutdown() {\n        producer.shutdown();\n    }\n\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/clientinterface/AbstractMQConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.clientinterface;\n\nimport org.apache.rocketmq.test.listener.AbstractListener;\n\npublic abstract class AbstractMQConsumer implements MQConsumer {\n    protected AbstractListener listener = null;\n    protected String nsAddr = null;\n    protected String topic = null;\n    protected String subExpression = null;\n    protected String consumerGroup = null;\n    protected boolean isDebug = false;\n\n    public AbstractMQConsumer() {\n    }\n\n    public AbstractMQConsumer(String nsAddr, String topic, String subExpression,\n        String consumerGroup, AbstractListener listener) {\n        this.topic = topic;\n        this.subExpression = subExpression;\n        this.consumerGroup = consumerGroup;\n        this.listener = listener;\n        this.nsAddr = nsAddr;\n    }\n\n    public AbstractMQConsumer(String topic, String subExpression) {\n        this.topic = topic;\n        this.subExpression = subExpression;\n    }\n\n    public void setDebug() {\n        if (listener != null) {\n            listener.setDebug(true);\n        }\n\n        isDebug = true;\n    }\n\n    public void setDebug(boolean isDebug) {\n        if (listener != null) {\n            listener.setDebug(isDebug);\n        }\n\n        this.isDebug = isDebug;\n    }\n\n    public void setSubscription(String topic, String subExpression) {\n        this.topic = topic;\n        this.subExpression = subExpression;\n    }\n\n    public AbstractListener getListener() {\n        return listener;\n    }\n\n    public void setListener(AbstractListener listener) {\n        this.listener = listener;\n    }\n\n    public String getNsAddr() {\n        return nsAddr;\n    }\n\n    public void setNsAddr(String nsAddr) {\n        this.nsAddr = nsAddr;\n    }\n\n    public String getTopic() {\n        return topic;\n    }\n\n    public void setTopic(String topic) {\n        this.topic = topic;\n    }\n\n    public String getSubExpression() {\n        return subExpression;\n    }\n\n    public void setSubExpression(String subExpression) {\n        this.subExpression = subExpression;\n    }\n\n    public String getConsumerGroup() {\n        return consumerGroup;\n    }\n\n    public void setConsumerGroup(String consumerGroup) {\n        this.consumerGroup = consumerGroup;\n    }\n\n    public void clearMsg() {\n        listener.clearMsg();\n    }\n\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/clientinterface/AbstractMQProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.clientinterface;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.Date;\nimport java.util.List;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.sendresult.ResultWrapper;\nimport org.apache.rocketmq.test.util.RandomUtil;\nimport org.apache.rocketmq.test.util.TestUtil;\n\npublic abstract class AbstractMQProducer extends MQCollector implements MQProducer {\n    protected String topic = null;\n\n    protected ResultWrapper sendResult = new ResultWrapper();\n    protected boolean startSuccess = false;\n    protected String producerGroupName = null;\n    protected String producerInstanceName = null;\n    protected boolean isDebug = false;\n\n\n    public AbstractMQProducer(String topic) {\n        super();\n        producerGroupName = RandomUtil.getStringByUUID();\n        producerInstanceName = RandomUtil.getStringByUUID();\n        this.topic = topic;\n\n    }\n\n    public AbstractMQProducer(String topic, String originMsgCollector, String msgBodyCollector) {\n        super(originMsgCollector, msgBodyCollector);\n        producerGroupName = RandomUtil.getStringByUUID();\n        producerInstanceName = RandomUtil.getStringByUUID();\n        this.topic = topic;\n    }\n\n    public boolean isStartSuccess() {\n        return startSuccess;\n    }\n\n    public void setStartSuccess(boolean startSuccess) {\n        this.startSuccess = startSuccess;\n    }\n\n    public String getProducerInstanceName() {\n        return producerInstanceName;\n    }\n\n    public void setProducerInstanceName(String producerInstanceName) {\n        this.producerInstanceName = producerInstanceName;\n    }\n\n    public String getProducerGroupName() {\n        return producerGroupName;\n    }\n\n    public void setProducerGroupName(String producerGroupName) {\n        this.producerGroupName = producerGroupName;\n    }\n\n    public void setDebug() {\n        isDebug = true;\n    }\n\n    public void setDebug(boolean isDebug) {\n        this.isDebug = isDebug;\n    }\n\n    public void setRun() {\n        isDebug = false;\n    }\n\n    public List<MessageQueue> getMessageQueue() {\n        return null;\n    }\n\n    private Object getMessage() {\n        return this.getMessageByTag(null);\n    }\n\n    public Object getMessageByTag(String tag) {\n        Object objMsg = null;\n        if (this instanceof RMQNormalProducer) {\n            org.apache.rocketmq.common.message.Message msg = new org.apache.rocketmq.common.message.Message(\n                topic, (RandomUtil.getStringByUUID() + \".\" + new Date()).getBytes(StandardCharsets.UTF_8));\n            objMsg = msg;\n            if (tag != null) {\n                msg.setTags(tag);\n            }\n        }\n        return objMsg;\n    }\n\n    public void send() {\n        Object msg = getMessage();\n        send(msg, null);\n    }\n\n    public void send(Object msg) {\n        send(msg, null);\n    }\n\n    public void send(long msgNum) {\n        for (int i = 0; i < msgNum; i++) {\n            this.send();\n        }\n    }\n\n    public void send(long msgNum, int intervalMills) {\n        for (int i = 0; i < msgNum; i++) {\n            this.send();\n            TestUtil.waitForMonment(intervalMills);\n        }\n    }\n\n    public void send(String tag, int msgSize) {\n        for (int i = 0; i < msgSize; i++) {\n            Object msg = getMessageByTag(tag);\n            send(msg, null);\n        }\n    }\n\n    public void send(String tag, int msgSize, int intervalMills) {\n        for (int i = 0; i < msgSize; i++) {\n            Object msg = getMessageByTag(tag);\n            send(msg, null);\n            TestUtil.waitForMonment(intervalMills);\n        }\n    }\n\n    public void send(List<Object> msgs) {\n        for (Object msg : msgs) {\n            this.send(msg, null);\n        }\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/clientinterface/MQCollector.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.clientinterface;\n\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.test.util.RandomUtil;\nimport org.apache.rocketmq.test.util.data.collect.DataCollector;\nimport org.apache.rocketmq.test.util.data.collect.DataCollectorManager;\n\npublic abstract class MQCollector {\n    protected DataCollector msgBodys = null;\n    protected DataCollector originMsgs = null;\n    protected DataCollector errorMsgs = null;\n    protected Map<Object, Object> originMsgIndex = null;\n    protected Collection<Object> msgBodysCopy = null;\n\n    protected DataCollector msgRTs = null;\n\n    public MQCollector() {\n        msgBodys = DataCollectorManager.getInstance()\n            .fetchListDataCollector(RandomUtil.getStringByUUID());\n        originMsgs = DataCollectorManager.getInstance()\n            .fetchListDataCollector(RandomUtil.getStringByUUID());\n        errorMsgs = DataCollectorManager.getInstance()\n            .fetchListDataCollector(RandomUtil.getStringByUUID());\n        originMsgIndex = new ConcurrentHashMap<Object, Object>();\n        msgRTs = DataCollectorManager.getInstance()\n            .fetchListDataCollector(RandomUtil.getStringByUUID());\n    }\n\n    public MQCollector(String originMsgCollector, String msgBodyCollector) {\n        originMsgs = DataCollectorManager.getInstance().fetchDataCollector(originMsgCollector);\n        msgBodys = DataCollectorManager.getInstance().fetchDataCollector(msgBodyCollector);\n    }\n\n    public Collection<Object> getAllMsgBody() {\n        return msgBodys.getAllData();\n    }\n\n    public Collection<Object> getAllOriginMsg() {\n        return originMsgs.getAllData();\n    }\n\n    public Object getFirstMsg() {\n        return ((List<Object>) originMsgs.getAllData()).get(0);\n    }\n\n    public Collection<Object> getAllUndupMsgBody() {\n        return msgBodys.getAllDataWithoutDuplicate();\n    }\n\n    public Collection<Object> getAllUndupOriginMsg() {\n        return originMsgs.getAllData();\n    }\n\n    public Collection<Object> getSendErrorMsg() {\n        return errorMsgs.getAllData();\n    }\n\n    public Collection<Object> getMsgRTs() {\n        return msgRTs.getAllData();\n    }\n\n    public Map<Object, Object> getOriginMsgIndex() {\n        return originMsgIndex;\n    }\n\n    public Collection<Object> getMsgBodysCopy() {\n        msgBodysCopy = new ArrayList<Object>();\n        msgBodysCopy.addAll(msgBodys.getAllData());\n        return msgBodysCopy;\n    }\n\n    public void clearMsg() {\n        if (msgBodys != null) {\n            msgBodys.resetData();\n        }\n        if (originMsgs != null) {\n            originMsgs.resetData();\n        }\n        if (originMsgs != null) {\n            errorMsgs.resetData();\n        }\n        if (originMsgIndex != null) {\n            originMsgIndex.clear();\n        }\n        if (msgRTs != null) {\n            msgRTs.resetData();\n        }\n    }\n\n    public void lockCollectors() {\n        msgBodys.lockIncrement();\n        originMsgs.lockIncrement();\n        errorMsgs.lockIncrement();\n        msgRTs.lockIncrement();\n\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/clientinterface/MQConsumer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.clientinterface;\n\npublic interface MQConsumer {\n    void create();\n\n    void create(boolean useTLS);\n\n    void start();\n\n    void shutdown();\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/clientinterface/MQProducer.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.clientinterface;\n\nimport org.apache.rocketmq.test.sendresult.ResultWrapper;\n\npublic interface MQProducer {\n    ResultWrapper send(Object msg, Object arg);\n\n    void setDebug();\n\n    void setRun();\n\n    void shutdown();\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.factory;\n\nimport java.util.UUID;\nimport org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;\nimport org.apache.rocketmq.client.consumer.MessageSelector;\nimport org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQPopClient;\nimport org.apache.rocketmq.test.client.rmq.RMQPopConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQSqlConsumer;\nimport org.apache.rocketmq.test.listener.AbstractListener;\n\npublic class ConsumerFactory {\n\n    public static RMQNormalConsumer getRMQNormalConsumer(String nsAddr, String consumerGroup,\n        String topic, String subExpression,\n        AbstractListener listener) {\n        return getRMQNormalConsumer(nsAddr, consumerGroup, topic, subExpression, listener, false);\n    }\n\n    public static RMQNormalConsumer getRMQNormalConsumer(String nsAddr, String consumerGroup,\n        String topic, String subExpression,\n        AbstractListener listener, boolean useTLS) {\n        RMQNormalConsumer consumer = new RMQNormalConsumer(nsAddr, topic, subExpression,\n            consumerGroup, listener);\n        consumer.create(useTLS);\n        consumer.start();\n        return consumer;\n    }\n\n    public static RMQBroadCastConsumer getRMQBroadCastConsumer(String nsAddr, String consumerGroup,\n        String topic, String subExpression,\n        AbstractListener listner) {\n        RMQBroadCastConsumer consumer = new RMQBroadCastConsumer(nsAddr, topic, subExpression,\n            consumerGroup, listner);\n        consumer.create();\n        consumer.start();\n        return consumer;\n    }\n\n    public static RMQSqlConsumer getRMQSqlConsumer(String nsAddr, String consumerGroup,\n        String topic, MessageSelector selector,\n        AbstractListener listner) {\n        RMQSqlConsumer consumer = new RMQSqlConsumer(nsAddr, topic, selector,\n            consumerGroup, listner);\n        consumer.create();\n        consumer.start();\n        return consumer;\n    }\n\n    public static RMQPopConsumer getRMQPopConsumer(String nsAddr, String consumerGroup,\n        String topic, String subExpression, AbstractListener listener) {\n\n        RMQPopConsumer consumer = new RMQPopConsumer(nsAddr, topic, subExpression, consumerGroup, listener);\n        consumer.create();\n        consumer.start();\n        return consumer;\n    }\n\n    public static RMQPopClient getRMQPopClient() {\n        RMQPopClient client = new RMQPopClient();\n        client.create();\n        client.start();\n        return client;\n    }\n\n    public static DefaultMQPullConsumer getRMQPullConsumer(String nsAddr, String consumerGroup) throws Exception {\n        DefaultMQPullConsumer defaultMQPullConsumer = new DefaultMQPullConsumer(consumerGroup);\n        defaultMQPullConsumer.setInstanceName(UUID.randomUUID().toString());\n        defaultMQPullConsumer.setNamesrvAddr(nsAddr);\n        defaultMQPullConsumer.start();\n        return defaultMQPullConsumer;\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/factory/MQMessageFactory.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.factory;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collection;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.test.util.RandomUtil;\n\npublic class MQMessageFactory {\n    private static Integer index = 0;\n\n    public static List<Object> getRMQMessage(String tag, String topic, int msgSize) {\n        List<Object> msgs = new ArrayList<Object>();\n        for (int i = 0; i < msgSize; i++) {\n            msgs.add(new Message(topic, tag, RandomUtil.getStringByUUID().getBytes(StandardCharsets.UTF_8)));\n        }\n\n        return msgs;\n    }\n\n    public static List<Object> getRMQMessage(List<String> tags, String topic, int msgSize) {\n        List<Object> msgs = new ArrayList<Object>();\n        for (int i = 0; i < msgSize; i++) {\n            for (String tag : tags) {\n                msgs.add(new Message(topic, tag, RandomUtil.getStringByUUID().getBytes(StandardCharsets.UTF_8)));\n            }\n        }\n        return msgs;\n    }\n\n    public static List<Object> getMessageBody(List<Object> msgs) {\n        List<Object> msgBodys = new ArrayList<Object>();\n        for (Object msg : msgs) {\n            msgBodys.add(new String(((Message) msg).getBody(), StandardCharsets.UTF_8));\n        }\n\n        return msgBodys;\n    }\n\n    public static Collection<Object> getMessage(Collection<Object>... msgs) {\n        Collection<Object> allMsgs = new ArrayList<Object>();\n        for (Collection<Object> msg : msgs) {\n            allMsgs.addAll(msg);\n        }\n        return allMsgs;\n    }\n\n    public static List<Object> getDelayMsg(String topic, int delayLevel, int msgSize) {\n        List<Object> msgs = new ArrayList<Object>();\n        for (int i = 0; i < msgSize; i++) {\n            Message msg = new Message(topic, RandomUtil.getStringByUUID().getBytes(StandardCharsets.UTF_8));\n            msg.setDelayTimeLevel(delayLevel);\n            msgs.add(msg);\n        }\n        return msgs;\n    }\n\n    public static List<Object> getKeyMsg(String topic, String key, int msgSize) {\n        List<Object> msgs = new ArrayList<Object>();\n        for (int i = 0; i < msgSize; i++) {\n            Message msg = new Message(topic, null, key, RandomUtil.getStringByUUID().getBytes(StandardCharsets.UTF_8));\n            msgs.add(msg);\n        }\n        return msgs;\n    }\n\n    public static Map<MessageQueue, List<Object>> getMsgByMQ(MessageQueue mq, int msgSize) {\n        List<MessageQueue> mqs = new ArrayList<MessageQueue>();\n        mqs.add(mq);\n        return getMsgByMQ(mqs, msgSize);\n    }\n\n    public static Map<MessageQueue, List<Object>> getMsgByMQ(List<MessageQueue> mqs, int msgSize) {\n        return getMsgByMQ(mqs, msgSize, null);\n    }\n\n    public static Map<MessageQueue, List<Object>> getMsgByMQ(List<MessageQueue> mqs, int msgSize,\n        String tag) {\n        Map<MessageQueue, List<Object>> msgs = new HashMap<MessageQueue, List<Object>>();\n        for (MessageQueue mq : mqs) {\n            msgs.put(mq, getMsg(mq.getTopic(), msgSize, tag));\n        }\n        return msgs;\n    }\n\n    public static List<Object> getMsg(String topic, int msgSize) {\n        return getMsg(topic, msgSize, null);\n    }\n\n    public static List<Object> getMsg(String topic, int msgSize, String tag) {\n        List<Object> msgs = new ArrayList<Object>();\n        while (msgSize > 0) {\n            Message msg = new Message(topic, (index++).toString().getBytes(StandardCharsets.UTF_8));\n            if (tag != null) {\n                msg.setTags(tag);\n            }\n            msgs.add(msg);\n            msgSize--;\n        }\n\n        return msgs;\n    }\n\n    public static List<MessageQueue> getMessageQueues(MessageQueue... mq) {\n        return Arrays.asList(mq);\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/factory/MessageFactory.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.factory;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.List;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.test.util.RandomUtils;\n\npublic class MessageFactory {\n\n    public static Message getRandomMessage(String topic) {\n        return getStringMessage(topic, RandomUtils.getStringByUUID());\n    }\n\n    public static Message getStringMessage(String topic, String body) {\n        return new Message(topic, body.getBytes(StandardCharsets.UTF_8));\n    }\n\n    public static Message getStringMessageByTag(String topic, String tags, String body) {\n        return new Message(topic, tags, body.getBytes(StandardCharsets.UTF_8));\n    }\n\n    public static Message getRandomMessageByTag(String topic, String tags) {\n        return getStringMessageByTag(topic, tags, RandomUtils.getStringByUUID());\n    }\n\n    public static Collection<Message> getRandomMessageList(String topic, int size) {\n        List<Message> msgList = new ArrayList<Message>();\n        for (int i = 0; i < size; i++) {\n            msgList.add(getRandomMessage(topic));\n        }\n        return msgList;\n    }\n\n    public static Collection<Message> getRandomMessageListByTag(String topic, String tags, int size) {\n        List<Message> msgList = new ArrayList<Message>();\n        for (int i = 0; i < size; i++) {\n            msgList.add(getRandomMessageByTag(topic, tags));\n        }\n        return msgList;\n    }\n\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/factory/ProducerFactory.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.factory;\n\nimport java.util.UUID;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.test.util.RandomUtil;\n\npublic class ProducerFactory {\n\n    public static DefaultMQProducer getRMQProducer(String ns) {\n        DefaultMQProducer producer = new DefaultMQProducer(RandomUtil.getStringByUUID());\n        producer.setInstanceName(UUID.randomUUID().toString());\n        producer.setNamesrvAddr(ns);\n        try {\n            producer.start();\n        } catch (MQClientException e) {\n            e.printStackTrace();\n        }\n\n        return producer;\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/factory/SendCallBackFactory.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.factory;\n\nimport org.apache.rocketmq.client.producer.SendCallback;\nimport org.apache.rocketmq.client.producer.SendResult;\n\npublic class SendCallBackFactory {\n    public static SendCallback getSendCallBack() {\n        return new SendCallback() {\n            @Override\n            public void onSuccess(SendResult sendResult) {\n            }\n\n            @Override\n            public void onException(Throwable throwable) {\n            }\n        };\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/factory/TagMessage.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.factory;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\npublic class TagMessage {\n    private List<String> tags = null;\n    private String topic = null;\n    private int msgSize = 0;\n    private Map<String, List<Object>> rmqMsgs = new HashMap<String, List<Object>>();\n\n    public TagMessage(String tag, String topic, int msgSize) {\n        String[] tags = {tag};\n        this.tags = Arrays.asList(tags);\n        this.topic = topic;\n        this.msgSize = msgSize;\n\n        init();\n    }\n\n    public TagMessage(String[] tags, String topic, int msgSize) {\n        this(Arrays.asList(tags), topic, msgSize);\n    }\n\n    public TagMessage(List<String> tags, String topic, int msgSize) {\n        this.tags = tags;\n        this.topic = topic;\n        this.msgSize = msgSize;\n\n        init();\n    }\n\n    private void init() {\n        for (String tag : tags) {\n            List<Object> tagMsgs = MQMessageFactory.getRMQMessage(tag, topic, msgSize);\n            rmqMsgs.put(tag, tagMsgs);\n        }\n    }\n\n    public List<Object> getMessageByTag(String tag) {\n        if (tags.contains(tag)) {\n            return rmqMsgs.get(tag);\n        } else {\n            return new ArrayList<Object>();\n        }\n    }\n\n    public List<Object> getMixedTagMessages() {\n        List<Object> mixedMsgs = new ArrayList<Object>();\n        for (int i = 0; i < msgSize; i++) {\n            for (String tag : tags) {\n                mixedMsgs.add(rmqMsgs.get(tag).get(i));\n            }\n        }\n\n        return mixedMsgs;\n    }\n\n    public List<Object> getMessageBodyByTag(String tag) {\n        if (tags.contains(tag)) {\n            return MQMessageFactory.getMessageBody(rmqMsgs.get(tag));\n        } else {\n            return new ArrayList<Object>();\n        }\n    }\n\n    public List<Object> getMessageBodyByTag(String... tag) {\n        return this.getMessageBodyByTag(Arrays.asList(tag));\n    }\n\n    public List<Object> getMessageBodyByTag(List<String> tags) {\n        List<Object> msgBodys = new ArrayList<Object>();\n        for (String tag : tags) {\n            msgBodys.addAll(MQMessageFactory.getMessageBody(rmqMsgs.get(tag)));\n        }\n        return msgBodys;\n    }\n\n    public List<Object> getAllTagMessageBody() {\n        List<Object> msgs = new ArrayList<Object>();\n        for (String tag : tags) {\n            msgs.addAll(MQMessageFactory.getMessageBody(rmqMsgs.get(tag)));\n        }\n\n        return msgs;\n    }\n\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/listener/AbstractListener.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.listener;\n\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.List;\nimport java.util.Map;\nimport org.apache.rocketmq.client.consumer.listener.MessageListener;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.clientinterface.MQCollector;\nimport org.apache.rocketmq.test.util.TestUtil;\n\npublic class AbstractListener extends MQCollector implements MessageListener {\n    public static final Logger LOGGER = LoggerFactory.getLogger(AbstractListener.class);\n    protected boolean isDebug = true;\n    protected String listenerName = null;\n    protected Collection<Object> allSendMsgs = null;\n\n    public AbstractListener() {\n        super();\n    }\n\n    public AbstractListener(String listenerName) {\n        super();\n        this.listenerName = listenerName;\n    }\n\n    public AbstractListener(String originMsgCollector, String msgBodyCollector) {\n        super(originMsgCollector, msgBodyCollector);\n    }\n\n    public boolean isDebug() {\n        return isDebug;\n    }\n\n    public void setDebug(boolean debug) {\n        isDebug = debug;\n    }\n\n    public void waitForMessageConsume(int timeoutMills) {\n        TestUtil.waitForMonment(timeoutMills);\n    }\n\n    public void stopRecv() {\n        super.lockCollectors();\n    }\n\n    public Collection<Object> waitForMessageConsume(Collection<Object> allSendMessages, int timeoutMills) {\n        this.allSendMsgs = allSendMessages;\n        List<Object> sendMessages = new ArrayList<>(allSendMessages);\n\n        long curTime = System.currentTimeMillis();\n        while (!sendMessages.isEmpty()) {\n            sendMessages.removeIf(msg -> msgBodys.getAllData().contains(msg));\n            if (sendMessages.isEmpty()) {\n                break;\n            } else {\n                if (System.currentTimeMillis() - curTime >= timeoutMills) {\n                    LOGGER.error(String.format(\"timeout but [%s] not recv all send messages!\", listenerName));\n                    break;\n                } else {\n                    LOGGER.info(String.format(\"[%s] still [%s] msg not recv!\", listenerName, sendMessages.size()));\n                    TestUtil.waitForMonment(500);\n                }\n            }\n        }\n        return sendMessages;\n    }\n\n    public long waitForMessageConsume(int size, int timeoutMills) {\n        long curTime = System.currentTimeMillis();\n        while (true) {\n            if (msgBodys.getDataSize() >= size) {\n                break;\n            }\n            if (System.currentTimeMillis() - curTime >= timeoutMills) {\n                LOGGER.error(String.format(\"timeout but  [%s]  not recv all send messages!\", listenerName));\n                break;\n            } else {\n                LOGGER.info(String.format(\"[%s] still [%s] msg not recv!\",\n                    listenerName, size - msgBodys.getDataSize()));\n                TestUtil.waitForMonment(500);\n            }\n        }\n\n        return msgBodys.getDataSize();\n    }\n\n    public void waitForMessageConsume(Map<Object, Object> sendMsgIndex, int timeoutMills) {\n        Collection<Object> notRecvMsgs = waitForMessageConsume(sendMsgIndex.keySet(), timeoutMills);\n        for (Object object : notRecvMsgs) {\n            LOGGER.info(\"{}\", sendMsgIndex.get(object));\n        }\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/listener/rmq/concurrent/RMQBlockListener.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.listener.rmq.concurrent;\n\nimport java.util.List;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic class RMQBlockListener extends RMQNormalListener {\n    private volatile boolean block = true;\n    private volatile boolean inBlock = true;\n\n    public RMQBlockListener() {\n        super();\n    }\n\n    public RMQBlockListener(boolean block) {\n        super();\n        this.block = block;\n    }\n\n    public boolean isBlocked() {\n        return inBlock;\n    }\n\n    public void setBlock(boolean block) {\n        this.block = block;\n    }\n\n    @Override\n    public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {\n        ConsumeConcurrentlyStatus status = super.consumeMessage(msgs, context);\n\n        try {\n            while (block) {\n                inBlock = true;\n                Thread.sleep(100);\n            }\n        } catch (InterruptedException ignore) {\n        }\n\n        return status;\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/listener/rmq/concurrent/RMQDelayListener.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.listener.rmq.concurrent;\n\nimport java.nio.charset.StandardCharsets;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.test.listener.AbstractListener;\nimport org.apache.rocketmq.test.util.RandomUtil;\nimport org.apache.rocketmq.test.util.data.collect.DataCollector;\nimport org.apache.rocketmq.test.util.data.collect.DataCollectorManager;\n\nimport java.util.Collection;\nimport java.util.List;\n\npublic class RMQDelayListener extends AbstractListener implements MessageListenerConcurrently {\n    private DataCollector msgDelayTimes = null;\n\n    public RMQDelayListener() {\n        msgDelayTimes = DataCollectorManager.getInstance()\n            .fetchDataCollector(RandomUtil.getStringByUUID());\n    }\n\n    public Collection<Object> getMsgDelayTimes() {\n        return msgDelayTimes.getAllData();\n    }\n\n    public void resetMsgDelayTimes() {\n        msgDelayTimes.resetData();\n    }\n\n    public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,\n        ConsumeConcurrentlyContext consumeConcurrentlyContext) {\n        long recvTime = System.currentTimeMillis();\n        for (MessageExt msg : msgs) {\n            if (isDebug) {\n                LOGGER.info(listenerName + \":\" + msg);\n            }\n\n            msgBodys.addData(new String(msg.getBody(), StandardCharsets.UTF_8));\n            originMsgs.addData(msg);\n            msgDelayTimes.addData(Math.abs(recvTime - msg.getBornTimestamp()));\n        }\n        return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/listener/rmq/concurrent/RMQNormalListener.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.listener.rmq.concurrent;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.List;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.test.listener.AbstractListener;\n\npublic class RMQNormalListener extends AbstractListener implements MessageListenerConcurrently {\n\n    private ConsumeConcurrentlyStatus consumeStatus = ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n\n    private final AtomicInteger msgIndex = new AtomicInteger(0);\n\n    public RMQNormalListener() {\n        super();\n    }\n\n    public RMQNormalListener(String listenerName) {\n        super(listenerName);\n    }\n\n    public RMQNormalListener(ConsumeConcurrentlyStatus consumeStatus) {\n        super();\n        this.consumeStatus = consumeStatus;\n    }\n\n    public RMQNormalListener(String originMsgCollector, String msgBodyCollector) {\n        super(originMsgCollector, msgBodyCollector);\n    }\n\n    public AtomicInteger getMsgIndex() {\n        return msgIndex;\n    }\n\n    @Override\n    public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,\n        ConsumeConcurrentlyContext consumeConcurrentlyContext) {\n        for (MessageExt msg : msgs) {\n            msgIndex.getAndIncrement();\n            if (isDebug) {\n                if (listenerName != null && !listenerName.isEmpty()) {\n                    LOGGER.info(listenerName + \":\" + msgIndex.get() + \":\"\n                        + String.format(\"msgid:%s broker:%s queueId:%s offset:%s\",\n                        msg.getMsgId(), msg.getStoreHost(), msg.getQueueId(),\n                        msg.getQueueOffset()));\n                } else {\n                    LOGGER.info(\"{}\", msg);\n                }\n            }\n\n            msgBodys.addData(new String(msg.getBody(), StandardCharsets.UTF_8));\n            originMsgs.addData(msg);\n            if (originMsgIndex != null) {\n                originMsgIndex.put(new String(msg.getBody(), StandardCharsets.UTF_8), msg);\n            }\n        }\n        return consumeStatus;\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/listener/rmq/order/RMQOrderListener.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.listener.rmq.order;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.test.listener.AbstractListener;\n\npublic class RMQOrderListener extends AbstractListener implements MessageListenerOrderly {\n    private Map<String/* brokerId_brokerIp */, Collection<Object>> msgs = new ConcurrentHashMap<String, Collection<Object>>();\n\n    public RMQOrderListener() {\n        super();\n    }\n\n    public RMQOrderListener(String listnerName) {\n        super(listnerName);\n    }\n\n    public RMQOrderListener(String originMsgCollector, String msgBodyCollector) {\n        super(originMsgCollector, msgBodyCollector);\n    }\n\n    public Collection<Collection<Object>> getMsgs() {\n        return msgs.values();\n    }\n\n    private void putMsg(MessageExt msg) {\n        Collection<Object> msgQueue = null;\n        String key = getKey(msg.getQueueId(), msg.getStoreHost().toString());\n        if (!msgs.containsKey(key)) {\n            msgQueue = new ArrayList<Object>();\n        } else {\n            msgQueue = msgs.get(key);\n        }\n\n        msgQueue.add(new String(msg.getBody(), StandardCharsets.UTF_8));\n        msgs.put(key, msgQueue);\n    }\n\n    private String getKey(int queueId, String brokerIp) {\n        return String.format(\"%s_%s\", queueId, brokerIp);\n    }\n\n    @Override\n    public ConsumeOrderlyStatus consumeMessage(List<MessageExt> msgs,\n                                               ConsumeOrderlyContext context) {\n        for (MessageExt msg : msgs) {\n            if (isDebug) {\n                if (listenerName != null && listenerName != \"\") {\n                    LOGGER.info(listenerName + \": \" + msg);\n                } else {\n                    LOGGER.info(\"{}\", msg);\n                }\n            }\n\n            putMsg(msg);\n            msgBodys.addData(new String(msg.getBody(), StandardCharsets.UTF_8));\n            originMsgs.addData(msg);\n        }\n\n        return ConsumeOrderlyStatus.SUCCESS;\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/lmq/benchmark/BenchLmqStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.test.lmq.benchmark;\n\nimport com.google.common.math.IntMath;\nimport com.google.common.math.LongMath;\nimport java.nio.charset.StandardCharsets;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.ScheduledThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;\nimport org.apache.rocketmq.client.consumer.PullCallback;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.util.StatUtil;\n\npublic class BenchLmqStore {\n    private static Logger logger = LoggerFactory.getLogger(BenchLmqStore.class);\n    private static String namesrv = System.getProperty(\"namesrv\", \"127.0.0.1:9876\");\n    private static String lmqTopic = System.getProperty(\"lmqTopic\", \"lmqTestTopic\");\n    private static boolean enableSub = Boolean.parseBoolean(System.getProperty(\"enableSub\", \"true\"));\n    private static String queuePrefix = System.getProperty(\"queuePrefix\", \"lmqTest\");\n    private static int tps = Integer.parseInt(System.getProperty(\"tps\", \"1\"));\n    private static int lmqNum = Integer.parseInt(System.getProperty(\"lmqNum\", \"1\"));\n    private static int sendThreadNum = Integer.parseInt(System.getProperty(\"sendThreadNum\", \"64\"));\n    private static int consumerThreadNum = Integer.parseInt(System.getProperty(\"consumerThreadNum\", \"64\"));\n    private static String brokerName = System.getProperty(\"brokerName\", \"broker-a\");\n    private static int size = Integer.parseInt(System.getProperty(\"size\", \"128\"));\n    private static int suspendTime = Integer.parseInt(System.getProperty(\"suspendTime\", \"2000\"));\n    private static final boolean RETRY_NO_MATCHED_MSG = Boolean.parseBoolean(System.getProperty(\"retry_no_matched_msg\", \"false\"));\n    private static boolean benchOffset = Boolean.parseBoolean(System.getProperty(\"benchOffset\", \"false\"));\n    private static int benchOffsetNum = Integer.parseInt(System.getProperty(\"benchOffsetNum\", \"1\"));\n    private static Map<MessageQueue, Long> offsetMap = new ConcurrentHashMap<>(256);\n    private static Map<MessageQueue, Boolean> pullStatus = new ConcurrentHashMap<>(256);\n    private static Map<Integer, Map<MessageQueue, Long>> pullEvent = new ConcurrentHashMap<>(256);\n    public static DefaultMQProducer defaultMQProducer;\n    private static int pullConsumerNum = Integer.parseInt(System.getProperty(\"pullConsumerNum\", \"8\"));\n    public static DefaultMQPullConsumer[] defaultMQPullConsumers = new DefaultMQPullConsumer[pullConsumerNum];\n    private static AtomicLong rid = new AtomicLong();\n    private static final String LMQ_PREFIX = \"%LMQ%\";\n\n    public static void main(String[] args) throws InterruptedException, MQClientException, MQBrokerException,\n        RemotingException {\n        defaultMQProducer = new DefaultMQProducer();\n        defaultMQProducer.setProducerGroup(\"PID_LMQ_TEST\");\n        defaultMQProducer.setVipChannelEnabled(false);\n        defaultMQProducer.setNamesrvAddr(namesrv);\n        defaultMQProducer.start();\n        //defaultMQProducer.createTopic(lmqTopic, lmqTopic, 8);\n        for (int i = 0; i < pullConsumerNum; i++) {\n            DefaultMQPullConsumer defaultMQPullConsumer = new DefaultMQPullConsumer();\n            defaultMQPullConsumers[i] = defaultMQPullConsumer;\n            defaultMQPullConsumer.setNamesrvAddr(namesrv);\n            defaultMQPullConsumer.setVipChannelEnabled(false);\n            defaultMQPullConsumer.setConsumerGroup(\"CID_RMQ_SYS_LMQ_TEST_\" + i);\n            defaultMQPullConsumer.setInstanceName(\"CID_RMQ_SYS_LMQ_TEST_\" + i);\n            defaultMQPullConsumer.setRegisterTopics(new HashSet<>(Collections.singletonList(lmqTopic)));\n            defaultMQPullConsumer.setBrokerSuspendMaxTimeMillis(suspendTime);\n            defaultMQPullConsumer.setConsumerTimeoutMillisWhenSuspend(suspendTime + 1000);\n            defaultMQPullConsumer.start();\n        }\n        Thread.sleep(3000L);\n        if (benchOffset) {\n            doBenchOffset();\n            return;\n        }\n        ScheduledThreadPoolExecutor consumerPool = new ScheduledThreadPoolExecutor(consumerThreadNum, new ThreadFactoryImpl(\"test\"));\n        for (int i = 0; i < consumerThreadNum; i++) {\n            final int idx = i;\n            consumerPool.scheduleWithFixedDelay(() -> {\n                try {\n                    Map<MessageQueue, Long> map = pullEvent.get(idx);\n                    if (map == null) {\n                        return;\n                    }\n                    for (Map.Entry<MessageQueue, Long> entry : map.entrySet()) {\n                        try {\n                            Boolean status = pullStatus.get(entry.getKey());\n                            if (Boolean.TRUE.equals(status)) {\n                                continue;\n                            }\n                            doPull(map, entry.getKey(), entry.getValue());\n                        } catch (Exception e) {\n                            logger.error(\"pull broker msg error\", e);\n                        }\n                    }\n                } catch (Exception e) {\n                    logger.error(\"exec doPull task error\", e);\n                }\n            }, 1, 1, TimeUnit.MILLISECONDS);\n        }\n        // init queue sub\n        if (enableSub && lmqNum > 0 && StringUtils.isNotBlank(brokerName)) {\n            for (int i = 0; i < lmqNum; i++) {\n                long idx = rid.incrementAndGet();\n                String queue = LMQ_PREFIX + queuePrefix + LongMath.mod(idx, lmqNum);\n                MessageQueue mq = new MessageQueue(queue, brokerName, 0);\n                int queueHash = IntMath.mod(queue.hashCode(), consumerThreadNum);\n                pullEvent.putIfAbsent(queueHash, new ConcurrentHashMap<>());\n                pullEvent.get(queueHash).put(mq, idx);\n            }\n        }\n        Thread.sleep(5000L);\n        doSend();\n    }\n\n    public static void doSend() {\n        StringBuilder sb = new StringBuilder();\n        for (int j = 0; j < size; j += 10) {\n            sb.append(\"hello baby\");\n        }\n        byte[] body = sb.toString().getBytes(StandardCharsets.UTF_8);\n        String pubKey = \"pub\";\n        ExecutorService sendPool = Executors.newFixedThreadPool(sendThreadNum);\n        for (int i = 0; i < sendThreadNum; i++) {\n            sendPool.execute(() -> {\n                while (true) {\n                    if (StatUtil.isOverFlow(pubKey, tps)) {\n                        try {\n                            Thread.sleep(100L);\n                        } catch (InterruptedException e) {\n                            e.printStackTrace();\n                        }\n                    }\n                    long start = System.currentTimeMillis();\n                    try {\n                        long idx = rid.incrementAndGet();\n                        Message message = new Message(lmqTopic, body);\n                        String queue = lmqTopic;\n                        if (lmqNum > 0) {\n                            queue = LMQ_PREFIX + queuePrefix + idx % lmqNum;\n                            message.putUserProperty(\"INNER_MULTI_DISPATCH\", queue);\n                        }\n                        SendResult sendResult = defaultMQProducer.send(message);\n                        StatUtil.addInvoke(pubKey, System.currentTimeMillis() - start);\n                        if (StatUtil.nowTps(pubKey) < 10) {\n                            logger.warn(\"pub: {} \", sendResult.getMsgId());\n                        }\n                        if (enableSub && null != sendResult.getMessageQueue()) {\n                            MessageQueue mq = new MessageQueue(queue, sendResult.getMessageQueue().getBrokerName(),\n                                lmqNum > 0 ? 0 : sendResult.getMessageQueue().getQueueId());\n                            int queueHash = IntMath.mod(queue.hashCode(), consumerThreadNum);\n                            pullEvent.putIfAbsent(queueHash, new ConcurrentHashMap<>());\n                            pullEvent.get(queueHash).put(mq, idx);\n                        }\n                    } catch (Exception e) {\n                        logger.error(\"\", e);\n                        StatUtil.addInvoke(pubKey, System.currentTimeMillis() - start, false);\n                    }\n                }\n            });\n        }\n    }\n\n    public static void doPull(Map<MessageQueue, Long> eventMap, MessageQueue mq,\n        Long eventId) throws RemotingException, InterruptedException, MQClientException {\n        if (!enableSub) {\n            eventMap.remove(mq, eventId);\n            pullStatus.remove(mq);\n            return;\n        }\n        DefaultMQPullConsumer defaultMQPullConsumer = defaultMQPullConsumers[(int) (eventId % pullConsumerNum)];\n        Long offset = offsetMap.get(mq);\n        if (offset == null) {\n            long start = System.currentTimeMillis();\n            offset = defaultMQPullConsumer.maxOffset(mq);\n            StatUtil.addInvoke(\"maxOffset\", System.currentTimeMillis() - start);\n            offsetMap.put(mq, offset);\n        }\n        long start = System.currentTimeMillis();\n        if (null != pullStatus.putIfAbsent(mq, true)) {\n            return;\n        }\n        defaultMQPullConsumer.pullBlockIfNotFound(\n            mq, \"*\", offset, 32,\n            new PullCallback() {\n                @Override\n                public void onSuccess(PullResult pullResult) {\n                    StatUtil.addInvoke(pullResult.getPullStatus().name(), System.currentTimeMillis() - start);\n                    eventMap.remove(mq, eventId);\n                    pullStatus.remove(mq);\n                    offsetMap.put(mq, pullResult.getNextBeginOffset());\n                    StatUtil.addInvoke(\"doPull\", System.currentTimeMillis() - start);\n                    if (PullStatus.NO_MATCHED_MSG.equals(pullResult.getPullStatus()) && RETRY_NO_MATCHED_MSG) {\n                        long idx = rid.incrementAndGet();\n                        eventMap.put(mq, idx);\n                    }\n                    List<MessageExt> list = pullResult.getMsgFoundList();\n                    if (list == null || list.isEmpty()) {\n                        StatUtil.addInvoke(\"NoMsg\", System.currentTimeMillis() - start);\n                        return;\n                    }\n                    for (MessageExt messageExt : list) {\n                        StatUtil.addInvoke(\"sub\", System.currentTimeMillis() - messageExt.getBornTimestamp());\n                        if (StatUtil.nowTps(\"sub\") < 10) {\n                            logger.warn(\"sub: {}\", messageExt.getMsgId());\n                        }\n                    }\n                }\n\n                @Override\n                public void onException(Throwable e) {\n                    eventMap.remove(mq, eventId);\n                    pullStatus.remove(mq);\n                    logger.error(\"\", e);\n                    StatUtil.addInvoke(\"doPull\", System.currentTimeMillis() - start, false);\n                }\n            });\n    }\n\n    public static void doBenchOffset() throws RemotingException, InterruptedException, MQClientException {\n        ExecutorService sendPool = Executors.newFixedThreadPool(sendThreadNum);\n        Map<String, Long> offsetMap = new ConcurrentHashMap<>();\n        String statKey = \"benchOffset\";\n        TopicRouteData topicRouteData = defaultMQPullConsumers[0].getDefaultMQPullConsumerImpl().\n            getRebalanceImpl().getmQClientFactory().getMQClientAPIImpl().\n            getTopicRouteInfoFromNameServer(lmqTopic, 3000);\n        HashMap<Long, String> brokerMap = topicRouteData.getBrokerDatas().get(0).getBrokerAddrs();\n        if (brokerMap == null || brokerMap.isEmpty()) {\n            return;\n        }\n        String brokerAddress = brokerMap.get(MixAll.MASTER_ID);\n        for (int i = 0; i < sendThreadNum; i++) {\n            final int flag = i;\n            sendPool.execute(new Runnable() {\n                @Override\n                public void run() {\n                    while (true) {\n                        try {\n                            if (StatUtil.isOverFlow(statKey, tps)) {\n                                Thread.sleep(100L);\n                            }\n                            long start = System.currentTimeMillis();\n                            long id = rid.incrementAndGet();\n                            int index = (Integer.MAX_VALUE & (int) id) % defaultMQPullConsumers.length;\n                            DefaultMQPullConsumer defaultMQPullConsumer = defaultMQPullConsumers[index];\n                            String lmq = LMQ_PREFIX + queuePrefix + id % benchOffsetNum;\n                            String lmqCid = LMQ_PREFIX + \"GID_LMQ@@c\" + flag + \"-\" + id % benchOffsetNum;\n                            offsetMap.putIfAbsent(lmq, 0L);\n                            long newOffset1 = offsetMap.get(lmq) + 1;\n                            UpdateConsumerOffsetRequestHeader updateHeader = new UpdateConsumerOffsetRequestHeader();\n                            updateHeader.setTopic(lmq);\n                            updateHeader.setConsumerGroup(lmqCid);\n                            updateHeader.setQueueId(0);\n                            updateHeader.setCommitOffset(newOffset1);\n                            defaultMQPullConsumer\n                                .getDefaultMQPullConsumerImpl()\n                                .getRebalanceImpl()\n                                .getmQClientFactory()\n                                .getMQClientAPIImpl().updateConsumerOffset(brokerAddress, updateHeader, 1000);\n                            QueryConsumerOffsetRequestHeader queryHeader = new QueryConsumerOffsetRequestHeader();\n                            queryHeader.setTopic(lmq);\n                            queryHeader.setConsumerGroup(lmqCid);\n                            queryHeader.setQueueId(0);\n                            long newOffset2 = defaultMQPullConsumer\n                                .getDefaultMQPullConsumerImpl()\n                                .getRebalanceImpl()\n                                .getmQClientFactory()\n                                .getMQClientAPIImpl()\n                                .queryConsumerOffset(brokerAddress, queryHeader, 1000);\n                            offsetMap.put(lmq, newOffset2);\n                            if (newOffset1 != newOffset2) {\n                                StatUtil.addInvoke(\"ErrorOffset\", 1);\n                            }\n                            StatUtil.addInvoke(statKey, System.currentTimeMillis() - start);\n                        } catch (Exception e) {\n                            logger.error(\"\", e);\n                        }\n                    }\n                }\n            });\n        }\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/message/MessageQueueMsg.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.message;\n\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Map.Entry;\n\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.test.factory.MQMessageFactory;\n\npublic class MessageQueueMsg {\n    private Map<MessageQueue, List<Object>> msgsWithMQ = null;\n    private Map<Integer, List<Object>> msgsWithMQId = null;\n    private Collection<Object> msgBodys = null;\n\n    public MessageQueueMsg(List<MessageQueue> mqs, int msgSize) {\n        this(mqs, msgSize, null);\n    }\n\n    public MessageQueueMsg(List<MessageQueue> mqs, int msgSize, String tag) {\n        msgsWithMQ = MQMessageFactory.getMsgByMQ(mqs, msgSize, tag);\n        msgsWithMQId = new HashMap<Integer, List<Object>>();\n        msgBodys = new ArrayList<Object>();\n        init();\n    }\n\n    public Map<MessageQueue, List<Object>> getMsgsWithMQ() {\n        return msgsWithMQ;\n    }\n\n    public Map<Integer, List<Object>> getMsgWithMQId() {\n        return msgsWithMQId;\n    }\n\n    public Collection<Object> getMsgBodys() {\n        return msgBodys;\n    }\n\n    private void init() {\n        for (Entry<MessageQueue, List<Object>> mqEntry : msgsWithMQ.entrySet()) {\n            msgsWithMQId.put(mqEntry.getKey().getQueueId(), mqEntry.getValue());\n            msgBodys.addAll(MQMessageFactory.getMessageBody(mqEntry.getValue()));\n        }\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/schema/SchemaDefiner.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\npackage org.apache.rocketmq.test.schema;\n\nimport com.google.common.collect.Sets;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;\nimport org.apache.rocketmq.client.consumer.DefaultLitePullConsumer;\nimport org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.PullCallback;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListener;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.MessageQueueSelector;\nimport org.apache.rocketmq.client.producer.SendCallback;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.CommandCustomHeader;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.tools.admin.DefaultMQAdminExt;\nimport org.reflections.Reflections;\n\npublic class SchemaDefiner {\n    public static final Map<Class<?>, Set<String>> IGNORED_FIELDS = new HashMap<>();\n    //Use name as the key instead of X.class directly. X.class is not equal to field.getType().\n    public static final Set<String> FIELD_CLASS_NAMES = new HashSet<>();\n    public static final List<Class<?>> API_CLASS_LIST = new ArrayList<>();\n    public static final List<Class<?>> PROTOCOL_CLASS_LIST = new ArrayList<>();\n\n    public static void doLoad() {\n        {\n            IGNORED_FIELDS.put(ClientConfig.class, Sets.newHashSet(\"namesrvAddr\", \"clientIP\", \"clientCallbackExecutorThreads\"));\n            IGNORED_FIELDS.put(DefaultLitePullConsumer.class, Sets.newHashSet(\"consumeTimestamp\"));\n            IGNORED_FIELDS.put(DefaultMQPushConsumer.class, Sets.newHashSet(\"consumeTimestamp\"));\n            FIELD_CLASS_NAMES.add(String.class.getName());\n            FIELD_CLASS_NAMES.add(Long.class.getName());\n            FIELD_CLASS_NAMES.add(Integer.class.getName());\n            FIELD_CLASS_NAMES.add(Short.class.getName());\n            FIELD_CLASS_NAMES.add(Byte.class.getName());\n            FIELD_CLASS_NAMES.add(Double.class.getName());\n            FIELD_CLASS_NAMES.add(Float.class.getName());\n            FIELD_CLASS_NAMES.add(Boolean.class.getName());\n        }\n        {\n            //basic\n            API_CLASS_LIST.add(DefaultMQPushConsumer.class);\n            API_CLASS_LIST.add(DefaultMQProducer.class);\n            API_CLASS_LIST.add(DefaultMQPullConsumer.class);\n            API_CLASS_LIST.add(DefaultLitePullConsumer.class);\n            API_CLASS_LIST.add(DefaultMQAdminExt.class);\n\n            //argument\n            API_CLASS_LIST.add(Message.class);\n            API_CLASS_LIST.add(MessageQueue.class);\n            API_CLASS_LIST.add(SendCallback.class);\n            API_CLASS_LIST.add(PullCallback.class);\n            API_CLASS_LIST.add(MessageQueueSelector.class);\n            API_CLASS_LIST.add(AllocateMessageQueueStrategy.class);\n            //result\n            API_CLASS_LIST.add(MessageExt.class);\n            API_CLASS_LIST.add(SendResult.class);\n            API_CLASS_LIST.add(SendStatus.class);\n            API_CLASS_LIST.add(PullResult.class);\n            API_CLASS_LIST.add(PullStatus.class);\n            //listener and context\n            API_CLASS_LIST.add(MessageListener.class);\n            API_CLASS_LIST.add(MessageListenerConcurrently.class);\n            API_CLASS_LIST.add(ConsumeConcurrentlyContext.class);\n            API_CLASS_LIST.add(ConsumeConcurrentlyStatus.class);\n            API_CLASS_LIST.add(MessageListenerOrderly.class);\n            API_CLASS_LIST.add(ConsumeOrderlyContext.class);\n            API_CLASS_LIST.add(ConsumeOrderlyStatus.class);\n            //hook and context\n            API_CLASS_LIST.add(RPCHook.class);\n            API_CLASS_LIST.add(org.apache.rocketmq.client.hook.FilterMessageHook.class);\n            API_CLASS_LIST.add(org.apache.rocketmq.client.hook.SendMessageHook.class);\n            API_CLASS_LIST.add(org.apache.rocketmq.client.hook.CheckForbiddenHook.class);\n            API_CLASS_LIST.add(org.apache.rocketmq.client.hook.ConsumeMessageHook.class);\n            API_CLASS_LIST.add(org.apache.rocketmq.client.hook.EndTransactionHook.class);\n            API_CLASS_LIST.add(org.apache.rocketmq.client.hook.FilterMessageContext.class);\n            API_CLASS_LIST.add(org.apache.rocketmq.client.hook.SendMessageContext.class);\n            API_CLASS_LIST.add(org.apache.rocketmq.client.hook.ConsumeMessageContext.class);\n            API_CLASS_LIST.add(org.apache.rocketmq.client.hook.ConsumeMessageContext.class);\n            API_CLASS_LIST.add(org.apache.rocketmq.client.hook.EndTransactionContext.class);\n\n        }\n        {\n            PROTOCOL_CLASS_LIST.add(RequestCode.class);\n            Reflections reflections = new Reflections(\"org.apache.rocketmq\");\n            for (Class<?> protocolClass: reflections.getSubTypesOf(CommandCustomHeader.class)) {\n                PROTOCOL_CLASS_LIST.add(protocolClass);\n            }\n        }\n\n    }\n\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/schema/SchemaTools.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\npackage org.apache.rocketmq.test.schema;\n\nimport java.io.BufferedReader;\nimport java.io.File;\nimport java.io.FileInputStream;\nimport java.io.FileOutputStream;\nimport java.io.InputStreamReader;\nimport java.io.OutputStreamWriter;\nimport java.io.Writer;\nimport java.lang.reflect.Constructor;\nimport java.lang.reflect.Field;\nimport java.lang.reflect.Method;\nimport java.lang.reflect.Modifier;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Comparator;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.TreeMap;\nimport java.util.stream.Collectors;\n\nimport static org.apache.rocketmq.test.schema.SchemaDefiner.FIELD_CLASS_NAMES;\nimport static org.apache.rocketmq.test.schema.SchemaDefiner.IGNORED_FIELDS;\n\npublic class SchemaTools {\n    public static final String PATH_API = \"api\";\n    public static final String PATH_PROTOCOL = \"protocol\";\n\n    public static String isPublicOrPrivate(int modifiers) {\n        if (Modifier.isPublic(modifiers)) {\n            return \"public\";\n        } else if (Modifier.isProtected(modifiers)) {\n            return \"protected\";\n        } else {\n            return \"private\";\n        }\n    }\n    public static TreeMap<String, String> buildSchemaOfFields(Class apiClass) throws Exception {\n        List<Field> fields = new ArrayList<>();\n        Class current = apiClass;\n        do {\n            fields.addAll(Arrays.asList(current.getDeclaredFields()));\n        } while ((current = current.getSuperclass()) != null\n            && current != Object.class);\n        Object obj = null;\n        if (!apiClass.isInterface()\n            && !Modifier.isAbstract(apiClass.getModifiers())\n            && !apiClass.isEnum()) {\n            Constructor<?> constructor = null;\n            for (Constructor<?> tmp : apiClass.getConstructors()) {\n                if (constructor == null) {\n                    constructor = tmp;\n                }\n                if (tmp.getParameterCount() < constructor.getParameterCount()) {\n                    constructor = tmp;\n                }\n            }\n            assert constructor != null;\n            constructor.setAccessible(true);\n            final String msg = constructor.getName();\n            try {\n                obj = constructor.newInstance(Arrays.stream(constructor.getParameterTypes()).map(x -> {\n                    try {\n                        if (x.isEnum()) {\n                            return x.getEnumConstants()[0];\n                        } if (x == boolean.class) {\n                            return false;\n                        } else if (x == char.class) {\n                            return \"\";\n                        } else if (x.isPrimitive()) {\n                            return 0;\n                        } else {\n                            return x.newInstance();\n                        }\n                    } catch (InstantiationException instantiationException) {\n                        return x.cast(null);\n                    } catch (Exception e) {\n                        throw new RuntimeException(msg + \" \" + x.getName(), e);\n                    }\n                }).toArray());\n            } catch (Exception e) {\n                throw new RuntimeException(msg, e);\n            }\n        }\n        TreeMap<String, String> map = new TreeMap<>();\n        if (apiClass.isEnum()) {\n            for (Object enumObject: apiClass.getEnumConstants()) {\n                String name = ((Enum<?>)enumObject).name();\n                int ordinal = ((Enum<?>)enumObject).ordinal();\n                String key = String.format(\"Field %s\", name);\n                String value = String.format(\"%s %s %s\", \"public\", \"int\", \"\" + ordinal);\n                map.put(key, value);\n            }\n            return map;\n        }\n        for (Field field: fields) {\n            if (field.getName().startsWith(\"$\")) {\n                //inner fields\n                continue;\n            }\n            String key = String.format(\"Field %s\", field.getName());\n            boolean ignore = false;\n            for (Class<?> tmpClass: IGNORED_FIELDS.keySet()) {\n                if (tmpClass.isAssignableFrom(apiClass)\n                    && IGNORED_FIELDS.get(tmpClass).contains(field.getName())) {\n                    ignore = true;\n                    //System.out.printf(\"Ignore AAA:%s %s %s\\n\", apiClass.getName(), field.getName(), field.getType().getName());\n                    break;\n                }\n            }\n            if (!field.getType().isEnum()\n                && !field.getType().isPrimitive()\n                && !FIELD_CLASS_NAMES.contains(field.getType().getName())) {\n                //System.out.printf(\"Ignore BBB:%s %s %s\\n\", apiClass.getName(), field.getName(), field.getType().getName());\n                ignore = true;\n            }\n            field.setAccessible(true);\n            Object fieldValue = \"null\";\n            try {\n                fieldValue = field.get(obj);\n            } catch (Exception e) {\n                throw new RuntimeException(apiClass.getName() + \" \" + field.getName(), e);\n            }\n            if (ignore) {\n                //System.out.printf(\"Ignore:%s %s %s\\n\", apiClass.getName(), field.getName(), field.getType().getName());\n                fieldValue = \"null\";\n            }\n            String value = String.format(\"%s %s %s\", isPublicOrPrivate(field.getModifiers()), field.getType().getName(), fieldValue);\n            map.put(key, value);\n        }\n        return map;\n    }\n\n    public static TreeMap<String, String> buildSchemaOfMethods(Class apiClass) throws Exception {\n        List<Method> methods = new ArrayList<>();\n        Class current = apiClass;\n        do {\n            methods.addAll(Arrays.asList(current.getDeclaredMethods()));\n        } while ((current = current.getSuperclass()) != null\n            && current != Object.class);\n        TreeMap<String, String> map = new TreeMap<>();\n        if (apiClass.isEnum()) {\n            return map;\n        }\n        for (Method method: methods) {\n            if (!Modifier.isPublic(method.getModifiers())) {\n                //only care for the public methods\n                continue;\n            }\n            Class<?>[] parameterTypes = method.getParameterTypes();\n            Arrays.sort(parameterTypes, Comparator.comparing(Class::getName));\n            Class<?>[] exceptionTypes = method.getExceptionTypes();\n            Arrays.sort(exceptionTypes, Comparator.comparing(Class::getName));\n            String key = String.format(\"Method %s(%s)\", method.getName(), Arrays.stream(parameterTypes).map(Class::getName).collect(Collectors.joining(\",\")));\n            String value = String.format(\"%s throws (%s): %s\",\n                isPublicOrPrivate(method.getModifiers()),\n                method.getReturnType().getName(),\n                Arrays.stream(exceptionTypes).map(Class::getName).collect(Collectors.joining(\",\")));\n            map.put(key, value);\n        }\n        return map;\n    }\n\n\n    public static Map<String, TreeMap<String, String>> generate(List<Class<?>> classList) throws Exception {\n        Map<String, TreeMap<String, String>> schemaMap = new HashMap<>();\n        for (Class<?> apiClass : classList) {\n            TreeMap<String, String> map = new TreeMap<>();\n            map.putAll(buildSchemaOfFields(apiClass));\n            map.putAll(buildSchemaOfMethods(apiClass));\n            schemaMap.put(apiClass.getName().replace(\"org.apache.rocketmq.\", \"\"), map);\n        }\n        return schemaMap;\n    }\n\n    public static void write(Map<String, TreeMap<String, String>> schemaMap, String base, String label) throws Exception {\n        for (Map.Entry<String, TreeMap<String, String>> entry : schemaMap.entrySet()) {\n            TreeMap<String, String> map = entry.getValue();\n            final String fileName = String.format(\"%s/%s/%s.schema\", base, label, entry.getKey());\n            File file = new File(fileName);\n            FileOutputStream fileStream = new FileOutputStream(file);\n            Writer writer = new OutputStreamWriter(fileStream, StandardCharsets.UTF_8);\n            writer.write(\"/*\\n\" +\n                \" * Licensed to the Apache Software Foundation (ASF) under one or more\\n\" +\n                \" * contributor license agreements.  See the NOTICE file distributed with\\n\" +\n                \" * this work for additional information regarding copyright ownership.\\n\" +\n                \" * The ASF licenses this file to You under the Apache License, Version 2.0\\n\" +\n                \" * (the \\\"License\\\"); you may not use this file except in compliance with\\n\" +\n                \" * the License.  You may obtain a copy of the License at\\n\" +\n                \" *\\n\" +\n                \" *     http://www.apache.org/licenses/LICENSE-2.0\\n\" +\n                \" *\\n\" +\n                \" * Unless required by applicable law or agreed to in writing, software\\n\" +\n                \" * distributed under the License is distributed on an \\\"AS IS\\\" BASIS,\\n\" +\n                \" * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\\n\" +\n                \" * See the License for the specific language governing permissions and\\n\" +\n                \" * limitations under the License.\\n\" +\n                \" */\\n\\n\\n\");\n            for (Map.Entry<String, String> kv: map.entrySet()) {\n                writer.append(String.format(\"%s : %s\\n\", kv.getKey(), kv.getValue()));\n            }\n            writer.close();\n        }\n    }\n\n    public static Map<String, TreeMap<String, String>> load(String base, String label) throws Exception {\n        File dir = new File(String.format(\"%s/%s\", base, label));\n        Map<String, TreeMap<String, String>> schemaMap = new TreeMap<>();\n        for (File file: dir.listFiles()) {\n            BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(file), StandardCharsets.UTF_8));\n            String line = null;\n            TreeMap<String, String> kvs = new TreeMap<>();\n            while ((line = br.readLine()) != null) {\n                if (line.contains(\"*\")) {\n                    continue;\n                }\n                if (line.trim().isEmpty()) {\n                    continue;\n                }\n                String[] items = line.split(\":\");\n                kvs.put(items[0].trim(), items[1].trim());\n            }\n            br.close();\n            schemaMap.put(file.getName().replace(\".schema\", \"\"), kvs);\n        }\n        return schemaMap;\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/sendresult/ResultWrapper.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.sendresult;\n\nimport org.apache.rocketmq.client.producer.SendResult;\n\npublic class ResultWrapper {\n    private boolean sendResult = false;\n    private String msgId = null;\n    private Exception sendException = null;\n    private String brokerIp = null;\n    private SendResult sendResultObj = null;\n\n    public String getBrokerIp() {\n        return brokerIp;\n    }\n\n    public void setBrokerIp(String brokerIp) {\n        this.brokerIp = brokerIp;\n    }\n\n    public boolean isSendResult() {\n        return sendResult;\n    }\n\n    public void setSendResult(boolean sendResult) {\n        this.sendResult = sendResult;\n    }\n\n    public String getMsgId() {\n        return msgId;\n    }\n\n    public void setMsgId(String msgId) {\n        this.msgId = msgId;\n    }\n\n    public Exception getSendException() {\n        return sendException;\n    }\n\n    public void setSendException(Exception sendException) {\n        this.sendException = sendException;\n    }\n\n    public SendResult getSendResultObj() {\n        return sendResultObj;\n    }\n    public void setSendResultObj(SendResult sendResultObj) {\n        this.sendResultObj = sendResultObj;\n    }\n\n    @Override\n    public String toString() {\n        return String.format(\"sendstatus:%s msgId:%s\", sendResult, msgId);\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/util/Condition.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.util;\n\npublic interface Condition {\n    boolean meetCondition();\n}\n\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/util/DuplicateMessageInfo.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.util;\n\nimport java.io.File;\nimport java.io.FileOutputStream;\nimport java.io.IOException;\nimport java.io.OutputStream;\nimport java.nio.charset.StandardCharsets;\nimport java.text.DecimalFormat;\nimport java.util.HashMap;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Map;\n\npublic class DuplicateMessageInfo<T> {\n\n    public void checkDuplicatedMessageInfo(boolean bPrintLog,\n        List<List<T>> lQueueList) throws IOException {\n        int msgListSize = lQueueList.size();\n        int maxmsgList = 0;\n        Map<T, Integer> msgIdMap = new HashMap<T, Integer>();\n        Map<Integer, Integer> dupMsgMap = new HashMap<Integer, Integer>();\n\n        for (int i = 0; i < msgListSize; i++) {\n            if (maxmsgList < lQueueList.get(i).size())\n                maxmsgList = lQueueList.get(i).size();\n        }\n\n        List<StringBuilder> strBQueue = new LinkedList<StringBuilder>();\n        for (int i = 0; i < msgListSize; i++)\n            strBQueue.add(new StringBuilder());\n\n        for (int msgListIndex = 0; msgListIndex < maxmsgList; msgListIndex++) {\n            for (int msgQueueListIndex = 0; msgQueueListIndex < msgListSize; msgQueueListIndex++) {\n                if (msgListIndex < lQueueList.get(msgQueueListIndex).size()) {\n                    if (msgIdMap.containsKey(lQueueList.get(msgQueueListIndex).get(msgListIndex))) {\n                        if (dupMsgMap.containsKey(msgQueueListIndex)) {\n                            int dupMsgCount = dupMsgMap.get(msgQueueListIndex);\n                            dupMsgCount++;\n                            dupMsgMap.remove(msgQueueListIndex);\n                            dupMsgMap.put(msgQueueListIndex, dupMsgCount);\n                        } else {\n                            dupMsgMap.put(msgQueueListIndex, 1);\n                        }\n\n                        strBQueue.get(msgQueueListIndex).append(\"\" + msgQueueListIndex + \"\\t\" +\n                            msgIdMap.get(lQueueList.get(msgQueueListIndex).get(msgListIndex)) + \"\\t\"\n                            + lQueueList.get(msgQueueListIndex).get(msgListIndex) + \"\\r\\n\");\n                    } else {\n                        msgIdMap.put(lQueueList.get(msgQueueListIndex).get(msgListIndex), msgQueueListIndex);\n                    }\n                }\n            }\n        }\n\n        int msgTotalNum = getMsgTotalNumber(lQueueList);\n        int msgTotalDupNum = getDuplicateMsgNum(dupMsgMap);\n        int msgNoDupNum = msgTotalNum - msgTotalDupNum;\n        float msgDupRate = ((float) msgTotalDupNum / (float) msgTotalNum) * 100.0f;\n        StringBuilder strBuilder = new StringBuilder();\n\n        strBuilder.append(\"msgTotalNum:\" + msgTotalNum + \"\\r\\n\");\n        strBuilder.append(\"msgTotalDupNum:\" + msgTotalDupNum + \"\\r\\n\");\n        strBuilder.append(\"msgNoDupNum:\" + msgNoDupNum + \"\\r\\n\");\n        strBuilder.append(\"msgDupRate\" + getFloatNumString(msgDupRate) + \"%\\r\\n\");\n\n        strBuilder.append(\"queue\\tmsg(dupNum/dupRate)\\tdupRate\\r\\n\");\n        for (int i = 0; i < dupMsgMap.size(); i++) {\n            int msgDupNum = dupMsgMap.get(i);\n            int msgNum = lQueueList.get(i).size();\n            float msgQueueDupRate = ((float) msgDupNum / (float) msgTotalDupNum) * 100.0f;\n            float msgQueueInnerDupRate = ((float) msgDupNum / (float) msgNum) * 100.0f;\n\n            strBuilder.append(i + \"\\t\" + msgDupNum + \"/\" + getFloatNumString(msgQueueDupRate) + \"%\" + \"\\t\\t\" +\n                getFloatNumString(msgQueueInnerDupRate) + \"%\\r\\n\");\n        }\n\n        System.out.print(strBuilder);\n        String titleString = \"queue\\tdupQueue\\tdupMsg\\r\\n\";\n        System.out.print(titleString);\n\n        for (int i = 0; i < msgListSize; i++)\n            System.out.print(strBQueue.get(i).toString());\n\n        if (bPrintLog) {\n            String logFileNameStr = \"D:\" + File.separator + \"checkDuplicatedMessageInfo.txt\";\n            File logFileNameFile = new File(logFileNameStr);\n            try (OutputStream out = new FileOutputStream(logFileNameFile, true)) {\n\n                String strToWrite;\n                byte[] byteToWrite;\n                strToWrite = strBuilder + titleString;\n                for (int i = 0; i < msgListSize; i++)\n                    strToWrite += strBQueue.get(i).toString() + \"\\r\\n\";\n\n                byteToWrite = strToWrite.getBytes(StandardCharsets.UTF_8);\n                out.write(byteToWrite);\n            }\n        }\n    }\n\n    private int getMsgTotalNumber(List<List<T>> lQueueList) {\n        int msgTotalNum = 0;\n        for (int i = 0; i < lQueueList.size(); i++) {\n            msgTotalNum += lQueueList.get(i).size();\n        }\n        return msgTotalNum;\n    }\n\n    private int getDuplicateMsgNum(Map<Integer, Integer> msgDupMap) {\n        int msgDupNum = 0;\n        for (int i = 0; i < msgDupMap.size(); i++) {\n            msgDupNum += msgDupMap.get(i);\n        }\n        return msgDupNum;\n    }\n\n    private String getFloatNumString(float fNum) {\n        DecimalFormat dcmFmt = new DecimalFormat(\"0.00\");\n        return dcmFmt.format(fNum);\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/util/FileUtil.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.util;\n\nimport java.io.File;\nimport java.io.FileOutputStream;\nimport java.io.IOException;\nimport java.io.OutputStreamWriter;\nimport java.io.Writer;\nimport java.nio.charset.StandardCharsets;\nimport java.util.Map.Entry;\nimport java.util.Properties;\n\npublic class FileUtil {\n    private static String lineSeparator = System.getProperty(\"line.separator\");\n\n    private String filePath = \"\";\n    private String fileName = \"\";\n\n    public FileUtil(String filePath, String fileName) {\n        this.filePath = filePath;\n        this.fileName = fileName;\n    }\n\n    public static void main(String[] args) {\n        String filePath = FileUtil.class.getResource(\"/\").getPath();\n        String fileName = \"test.txt\";\n        FileUtil fileUtil = new FileUtil(filePath, fileName);\n        Properties properties = new Properties();\n        properties.put(\"xx\", \"yy\");\n        properties.put(\"yy\", \"xx\");\n        fileUtil.writeProperties(properties);\n    }\n\n    public void deleteFile() {\n        File file = new File(filePath + File.separator + fileName);\n        if (file.exists()) {\n            file.delete();\n        }\n    }\n\n    public void appendFile(String content) {\n        File file = openFile();\n        String newContent = lineSeparator + content;\n        writeFile(file, newContent, true);\n    }\n\n    public void coverFile(String content) {\n        File file = openFile();\n        writeFile(file, content, false);\n    }\n\n    public void writeProperties(Properties properties) {\n        String content = getPropertiesAsString(properties);\n        this.coverFile(content);\n    }\n\n    private String getPropertiesAsString(Properties properties) {\n        StringBuilder sb = new StringBuilder();\n        for (Entry<Object, Object> keyEnty : properties.entrySet()) {\n            sb.append(keyEnty.getKey()).append(\"=\").append((String) keyEnty.getValue())\n                    .append(lineSeparator);\n        }\n        return sb.toString();\n    }\n\n    private void writeFile(File file, String content, boolean append) {\n        Writer writer = null;\n        try {\n            FileOutputStream fileStream = new FileOutputStream(file, append);\n            writer = new OutputStreamWriter(fileStream, StandardCharsets.UTF_8);\n            writer.write(content);\n            writer.flush();\n        } catch (IOException e) {\n            e.printStackTrace();\n        } finally {\n            if (writer != null) {\n                try {\n                    writer.close();\n                } catch (IOException e) {\n                    e.printStackTrace();\n                }\n            }\n        }\n    }\n\n    private File openFile() {\n        File file = new File(filePath + File.separator + fileName);\n        if (!file.exists()) {\n            try {\n                file.createNewFile();\n            } catch (IOException e) {\n                e.printStackTrace();\n            }\n        }\n        return file;\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/util/MQAdminTestUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.util;\n\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.UUID;\nimport java.util.concurrent.ForkJoinPool;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.cli.CommandLine;\nimport org.apache.commons.cli.DefaultParser;\nimport org.apache.commons.cli.Options;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.admin.ConsumeStats;\nimport org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;\nimport org.apache.rocketmq.remoting.protocol.body.ClusterInfo;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicConfigAndQueueMapping;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingOne;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingUtils;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicRemappingDetailWrapper;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.remoting.rpc.ClientMetadata;\nimport org.apache.rocketmq.srvutil.ServerUtil;\nimport org.apache.rocketmq.tools.admin.DefaultMQAdminExt;\nimport org.apache.rocketmq.tools.admin.MQAdminUtils;\nimport org.apache.rocketmq.tools.command.CommandUtil;\nimport org.apache.rocketmq.tools.command.topic.RemappingStaticTopicSubCommand;\nimport org.apache.rocketmq.tools.command.topic.UpdateStaticTopicSubCommand;\n\nimport static org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingUtils.getMappingDetailFromConfig;\nimport static org.awaitility.Awaitility.await;\n\npublic class MQAdminTestUtils {\n    private static Logger log = LoggerFactory.getLogger(MQAdminTestUtils.class);\n\n    private static DefaultMQAdminExt mqAdminExt;\n\n    public static void startAdmin(String nameSrvAddr) throws MQClientException {\n        mqAdminExt = new DefaultMQAdminExt();\n        mqAdminExt.setNamesrvAddr(nameSrvAddr);\n        mqAdminExt.start();\n    }\n\n    public static void shutdownAdmin() {\n        mqAdminExt.shutdown();\n    }\n\n    public static boolean createTopic(String nameSrvAddr, String clusterName, String topic,\n                                      int queueNum, Map<String, String> attributes) {\n        int defaultWaitTime = 30;\n        return createTopic(nameSrvAddr, clusterName, topic, queueNum, attributes, defaultWaitTime);\n    }\n\n    public static boolean createTopic(String nameSrvAddr, String clusterName, String topic,\n                                      int queueNum, Map<String, String> attributes, int waitTimeSec) {\n        DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt();\n        mqAdminExt.setInstanceName(UUID.randomUUID().toString());\n        mqAdminExt.setNamesrvAddr(nameSrvAddr);\n        try {\n            mqAdminExt.start();\n            mqAdminExt.createTopic(clusterName, topic, queueNum, attributes);\n        } catch (Exception e) {\n        }\n\n        await().atMost(waitTimeSec, TimeUnit.SECONDS).until(() -> checkTopicExist(mqAdminExt, topic));\n        ForkJoinPool.commonPool().execute(mqAdminExt::shutdown);\n        return true;\n    }\n\n    public static boolean checkTopicExist(DefaultMQAdminExt mqAdminExt, String topic) {\n        boolean createResult = false;\n        try {\n            TopicStatsTable topicInfo = mqAdminExt.examineTopicStats(topic);\n            createResult = !topicInfo.getOffsetTable().isEmpty();\n        } catch (Exception e) {\n        }\n\n        return createResult;\n    }\n\n    public static boolean createSub(String nameSrvAddr, String clusterName, SubscriptionGroupConfig config) {\n        boolean createResult = true;\n        DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt();\n        mqAdminExt.setNamesrvAddr(nameSrvAddr);\n        try {\n            mqAdminExt.start();\n            Set<String> masterSet = CommandUtil.fetchMasterAddrByClusterName(mqAdminExt,\n                    clusterName);\n            for (String addr : masterSet) {\n                try {\n                    mqAdminExt.createAndUpdateSubscriptionGroupConfig(addr, config);\n                    log.info(\"create subscription group {} to {} success.\", config.getGroupName(), addr);\n                } catch (Exception e) {\n                    e.printStackTrace();\n                    Thread.sleep(1000 * 1);\n                }\n            }\n        } catch (Exception e) {\n            createResult = false;\n            e.printStackTrace();\n        }\n        ForkJoinPool.commonPool().execute(mqAdminExt::shutdown);\n        return createResult;\n    }\n\n    public static ClusterInfo getCluster(String nameSrvAddr) {\n        DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt();\n        mqAdminExt.setNamesrvAddr(nameSrvAddr);\n        ClusterInfo clusterInfo = null;\n        try {\n            mqAdminExt.start();\n            clusterInfo = mqAdminExt.examineBrokerClusterInfo();\n        } catch (Exception e) {\n            e.printStackTrace();\n        }\n        ForkJoinPool.commonPool().execute(mqAdminExt::shutdown);\n        return clusterInfo;\n    }\n\n    public static boolean isBrokerExist(String ns, String ip) {\n        ClusterInfo clusterInfo = getCluster(ns);\n        if (clusterInfo == null) {\n            return false;\n        } else {\n            Map<String, BrokerData> brokers = clusterInfo.getBrokerAddrTable();\n            for (String brokerName : brokers.keySet()) {\n                HashMap<Long, String> brokerIps = brokers.get(brokerName).getBrokerAddrs();\n                for (long brokerId : brokerIps.keySet()) {\n                    if (brokerIps.get(brokerId).contains(ip))\n                        return true;\n                }\n            }\n        }\n\n        return false;\n    }\n\n\n    public static boolean awaitStaticTopicMs(long timeMs, String topic, DefaultMQAdminExt defaultMQAdminExt, MQClientInstance clientInstance) throws Exception {\n        long start = System.currentTimeMillis();\n        while (System.currentTimeMillis() - start <= timeMs) {\n            if (checkStaticTopic(topic, defaultMQAdminExt, clientInstance)) {\n                return true;\n            }\n            Thread.sleep(100);\n        }\n        return false;\n    }\n\n    //Check if the client metadata is consistent with server metadata\n    public static boolean checkStaticTopic(String topic, DefaultMQAdminExt defaultMQAdminExt, MQClientInstance clientInstance) throws Exception {\n        Map<String, TopicConfigAndQueueMapping> brokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);\n        assert !brokerConfigMap.isEmpty();\n        TopicQueueMappingUtils.checkPhysicalQueueConsistence(brokerConfigMap);\n        TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap);\n        Map<Integer, TopicQueueMappingOne> globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(getMappingDetailFromConfig(brokerConfigMap.values()), false, true);\n        for (int i = 0; i < globalIdMap.size(); i++) {\n            TopicQueueMappingOne mappingOne = globalIdMap.get(i);\n            String mockBrokerName = TopicQueueMappingUtils.getMockBrokerName(mappingOne.getMappingDetail().getScope());\n            String bnameFromRoute = clientInstance.getBrokerNameFromMessageQueue(new MessageQueue(topic, mockBrokerName, mappingOne.getGlobalId()));\n            if (!mappingOne.getBname().equals(bnameFromRoute)) {\n                return false;\n            }\n        }\n        return  true;\n    }\n\n    //should only be test, if some middle operation failed, it does not backup the brokerConfigMap\n    public static Map<String, TopicConfigAndQueueMapping> createStaticTopic(String topic, int queueNum, Set<String> targetBrokers, DefaultMQAdminExt defaultMQAdminExt) throws Exception {\n        Map<String, TopicConfigAndQueueMapping> brokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);\n        assert brokerConfigMap.isEmpty();\n        TopicQueueMappingUtils.createTopicConfigMapping(topic, queueNum, targetBrokers, brokerConfigMap);\n        MQAdminUtils.completeNoTargetBrokers(brokerConfigMap, defaultMQAdminExt);\n        MQAdminUtils.updateTopicConfigMappingAll(brokerConfigMap, defaultMQAdminExt, false);\n        return brokerConfigMap;\n    }\n\n    //should only be test, if some middle operation failed, it does not backup the brokerConfigMap\n    public static void remappingStaticTopic(String topic, Set<String> targetBrokers, DefaultMQAdminExt defaultMQAdminExt) throws Exception {\n        Map<String, TopicConfigAndQueueMapping> brokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);\n        assert !brokerConfigMap.isEmpty();\n        TopicRemappingDetailWrapper wrapper = TopicQueueMappingUtils.remappingStaticTopic(topic, brokerConfigMap, targetBrokers);\n        MQAdminUtils.completeNoTargetBrokers(brokerConfigMap, defaultMQAdminExt);\n        MQAdminUtils.remappingStaticTopic(topic, wrapper.getBrokerToMapIn(), wrapper.getBrokerToMapOut(), brokerConfigMap, TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE, false, defaultMQAdminExt);\n    }\n\n\n    //for test only\n    public static void remappingStaticTopicWithNegativeLogicOffset(String topic, Set<String> targetBrokers, DefaultMQAdminExt defaultMQAdminExt) throws Exception {\n        Map<String, TopicConfigAndQueueMapping> brokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);\n        assert !brokerConfigMap.isEmpty();\n        TopicRemappingDetailWrapper wrapper = TopicQueueMappingUtils.remappingStaticTopic(topic, brokerConfigMap, targetBrokers);\n        MQAdminUtils.completeNoTargetBrokers(brokerConfigMap, defaultMQAdminExt);\n        remappingStaticTopicWithNegativeLogicOffset(topic, wrapper.getBrokerToMapIn(), wrapper.getBrokerToMapOut(), brokerConfigMap, TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE, false, defaultMQAdminExt);\n    }\n\n    //for test only\n    public static void remappingStaticTopicWithNegativeLogicOffset(String topic, Set<String> brokersToMapIn, Set<String> brokersToMapOut, Map<String, TopicConfigAndQueueMapping> brokerConfigMap, int blockSeqSize, boolean force, DefaultMQAdminExt defaultMQAdminExt) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {\n\n        ClientMetadata clientMetadata = MQAdminUtils.getBrokerMetadata(defaultMQAdminExt);\n        MQAdminUtils.checkIfMasterAlive(brokerConfigMap.keySet(), defaultMQAdminExt, clientMetadata);\n        // now do the remapping\n        //Step1: let the new leader can be write without the logicOffset\n        for (String broker : brokersToMapIn) {\n            String addr = clientMetadata.findMasterBrokerAddr(broker);\n            TopicConfigAndQueueMapping configMapping = brokerConfigMap.get(broker);\n            defaultMQAdminExt.createStaticTopic(addr, defaultMQAdminExt.getCreateTopicKey(), configMapping, configMapping.getMappingDetail(), force);\n        }\n        //Step2: forbid the write of old leader\n        for (String broker : brokersToMapOut) {\n            String addr = clientMetadata.findMasterBrokerAddr(broker);\n            TopicConfigAndQueueMapping configMapping = brokerConfigMap.get(broker);\n            defaultMQAdminExt.createStaticTopic(addr, defaultMQAdminExt.getCreateTopicKey(), configMapping, configMapping.getMappingDetail(), force);\n        }\n\n        //Step5: write the non-target brokers\n        for (String broker : brokerConfigMap.keySet()) {\n            if (brokersToMapIn.contains(broker) || brokersToMapOut.contains(broker)) {\n                continue;\n            }\n            String addr = clientMetadata.findMasterBrokerAddr(broker);\n            TopicConfigAndQueueMapping configMapping = brokerConfigMap.get(broker);\n            defaultMQAdminExt.createStaticTopic(addr, defaultMQAdminExt.getCreateTopicKey(), configMapping, configMapping.getMappingDetail(), force);\n        }\n    }\n\n    public static void createStaticTopicWithCommand(String topic, int queueNum, Set<String> brokers, String cluster, String nameservers) throws Exception {\n        UpdateStaticTopicSubCommand cmd = new UpdateStaticTopicSubCommand();\n        Options options = ServerUtil.buildCommandlineOptions(new Options());\n        String[] args;\n        if (cluster != null) {\n            args = new String[]{\n                \"-c\", cluster,\n                \"-t\", topic,\n                \"-qn\", String.valueOf(queueNum),\n                \"-n\", nameservers\n            };\n        } else {\n            String brokerStr = String.join(\",\", brokers);\n            args = new String[]{\n                \"-b\", brokerStr,\n                \"-t\", topic,\n                \"-qn\", String.valueOf(queueNum),\n                \"-n\", nameservers\n            };\n        }\n        final CommandLine commandLine = ServerUtil.parseCmdLine(\"mqadmin \" + cmd.commandName(), args, cmd.buildCommandlineOptions(options), new DefaultParser());\n        if (null == commandLine) {\n            return;\n        }\n        if (commandLine.hasOption('n')) {\n            String namesrvAddr = commandLine.getOptionValue('n');\n            System.setProperty(MixAll.NAMESRV_ADDR_PROPERTY, namesrvAddr);\n        }\n        cmd.execute(commandLine, options, null);\n    }\n\n    public static void remappingStaticTopicWithCommand(String topic, Set<String> brokers, String cluster, String nameservers) throws Exception {\n        RemappingStaticTopicSubCommand cmd = new RemappingStaticTopicSubCommand();\n        Options options = ServerUtil.buildCommandlineOptions(new Options());\n        String[] args;\n        if (cluster != null) {\n            args = new String[]{\n                \"-c\", cluster,\n                \"-t\", topic,\n                \"-n\", nameservers\n            };\n        } else {\n            String brokerStr = String.join(\",\", brokers);\n            args = new String[]{\n                \"-b\", brokerStr,\n                \"-t\", topic,\n                \"-n\", nameservers\n            };\n        }\n        final CommandLine commandLine = ServerUtil.parseCmdLine(\"mqadmin \" + cmd.commandName(), args, cmd.buildCommandlineOptions(options), new DefaultParser());\n        if (null == commandLine) {\n            return;\n        }\n        if (commandLine.hasOption('n')) {\n            String namesrvAddr = commandLine.getOptionValue('n');\n            System.setProperty(MixAll.NAMESRV_ADDR_PROPERTY, namesrvAddr);\n        }\n        cmd.execute(commandLine, options, null);\n    }\n\n    public static ConsumeStats examineConsumeStats(String brokerAddr, String topic, String group) {\n        ConsumeStats consumeStats = null;\n        try {\n            consumeStats = mqAdminExt.examineConsumeStats(brokerAddr, group, topic, 3000);\n        } catch (Exception ignored) {\n        }\n        return consumeStats;\n    }\n\n    /**\n     * Delete topic from broker only without cleaning route info from name server forwardly\n     *\n     * @param nameSrvAddr the namesrv addr to connect\n     * @param brokerName the specific broker\n     * @param topic the specific topic to delete\n     */\n    public static void deleteTopicFromBrokerOnly(String nameSrvAddr, String brokerName, String topic) {\n        DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt();\n        mqAdminExt.setNamesrvAddr(nameSrvAddr);\n\n        try {\n            mqAdminExt.start();\n            String brokerAddr = CommandUtil.fetchMasterAddrByBrokerName(mqAdminExt, brokerName);\n            mqAdminExt.deleteTopicInBroker(Collections.singleton(brokerAddr), topic);\n        } catch (Exception ignored) {\n        } finally {\n            mqAdminExt.shutdown();\n        }\n    }\n\n    public static TopicRouteData examineTopicRouteInfo(String nameSrvAddr, String topicName) {\n        DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt();\n        mqAdminExt.setNamesrvAddr(nameSrvAddr);\n        TopicRouteData route = null;\n        try {\n            mqAdminExt.start();\n            route = mqAdminExt.examineTopicRouteInfo(topicName);\n        } catch (Exception ignored) {\n        } finally {\n            mqAdminExt.shutdown();\n        }\n        return route;\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/util/MQRandomUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.util;\n\npublic class MQRandomUtils {\n    public static String getRandomTopic() {\n        return RandomUtils.getStringByUUID();\n    }\n\n    public static String getRandomConsumerGroup() {\n        return RandomUtils.getStringByUUID();\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/util/MQWait.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.util;\n\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.Collections;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.listener.AbstractListener;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class MQWait {\n    private static Logger logger = LoggerFactory.getLogger(MQWait.class);\n\n    public static boolean waitConsumeAll(int timeoutMills, Collection<Object> allSendMsgs,\n        AbstractListener... listeners) {\n        boolean recvAll = false;\n        long startTime = System.currentTimeMillis();\n        Collection<Object> noDupMsgs = new ArrayList<Object>();\n        while (!recvAll) {\n            if ((System.currentTimeMillis() - startTime) < timeoutMills) {\n                noDupMsgs.clear();\n                try {\n                    for (AbstractListener listener : listeners) {\n                        Collection<Object> recvMsgs = Collections\n                            .synchronizedCollection(listener.getAllUndupMsgBody());\n                        noDupMsgs.addAll(VerifyUtils.getFilterdMessage(allSendMsgs, recvMsgs));\n                    }\n                } catch (Exception e) {\n                    e.printStackTrace();\n                }\n\n                try {\n                    assertThat(noDupMsgs).containsAllIn(allSendMsgs);\n                    recvAll = true;\n                    break;\n                } catch (Throwable e) {\n                }\n                TestUtil.waitForMonment(500);\n            } else {\n                logger.error(String.format(\n                    \"timeout but still not receive all messages,expectSize[%s],realSize[%s]\",\n                    allSendMsgs.size(), noDupMsgs.size()));\n                break;\n            }\n        }\n\n        return recvAll;\n    }\n\n    public static void setCondition(Condition condition, int waitTimeMills, int intervalMills) {\n        long startTime = System.currentTimeMillis();\n        while (!condition.meetCondition()) {\n            if (System.currentTimeMillis() - startTime > waitTimeMills) {\n                logger.error(\"time out,but contidion still not meet!\");\n                break;\n            } else {\n                TestUtil.waitForMonment(intervalMills);\n            }\n        }\n    }\n\n    public static void main(String[] args) {\n\n        long start = System.currentTimeMillis();\n        MQWait.setCondition(new Condition() {\n            int i = 0;\n\n            public boolean meetCondition() {\n                i++;\n                return i == 100;\n            }\n        }, 10 * 1000, 200);\n\n    }\n\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/util/RandomUtil.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.util;\n\nimport java.util.Collection;\nimport java.util.HashSet;\nimport java.util.Random;\nimport java.util.Set;\nimport java.util.UUID;\n\npublic final class RandomUtil {\n\n    private static final int UNICODE_START = '\\u4E00';\n    private static final int UNICODE_END = '\\u9FA0';\n    private static Random rd = new Random();\n\n    private RandomUtil() {\n\n    }\n\n    public static long getLong() {\n        return rd.nextLong();\n    }\n\n    public static long getLongMoreThanZero() {\n        long res = rd.nextLong();\n        while (res <= 0) {\n            res = rd.nextLong();\n        }\n        return res;\n    }\n\n    public static long getLongLessThan(long n) {\n        long res = rd.nextLong();\n        return res % n;\n    }\n\n    public static long getLongMoreThanZeroLessThan(long n) {\n        long res = getLongLessThan(n);\n        while (res <= 0) {\n            res = getLongLessThan(n);\n        }\n        return res;\n    }\n\n    public static long getLongBetween(long n, long m) {\n        if (m <= n) {\n            return n;\n        }\n        long res = getLongMoreThanZero();\n        return n + res % (m - n);\n    }\n\n    public static int getInteger() {\n        return rd.nextInt();\n    }\n\n    public static int getIntegerMoreThanZero() {\n        int res = rd.nextInt();\n        while (res <= 0) {\n            res = rd.nextInt();\n        }\n        return res;\n    }\n\n    public static int getIntegerLessThan(int n) {\n        int res = rd.nextInt();\n        return res % n;\n    }\n\n    public static int getIntegerMoreThanZeroLessThan(int n) {\n        int res = rd.nextInt(n);\n        while (res == 0) {\n            res = rd.nextInt(n);\n        }\n        return res;\n    }\n\n    public static int getIntegerBetween(int n, int m)// m��ֵ����Ϊ���أ�\n    {\n        if (m == n) {\n            return n;\n        }\n        int res = getIntegerMoreThanZero();\n        return n + res % (m - n);\n    }\n\n    private static char getChar(int[] arg) {\n        int size = arg.length;\n        int c = rd.nextInt(size / 2);\n        c = c * 2;\n        return (char) (getIntegerBetween(arg[c], arg[c + 1]));\n    }\n\n    private static String getString(int n, int[] arg) {\n        StringBuilder res = new StringBuilder();\n        for (int i = 0; i < n; i++) {\n            res.append(getChar(arg));\n        }\n        return res.toString();\n    }\n\n    public static String getStringWithCharacter(int n) {\n        int[] arg = new int[] {'a', 'z' + 1, 'A', 'Z' + 1};\n        return getString(n, arg);\n    }\n\n    public static String getStringWithNumber(int n) {\n        int[] arg = new int[] {'0', '9' + 1};\n        return getString(n, arg);\n    }\n\n    public static String getStringWithNumAndCha(int n) {\n        int[] arg = new int[] {'a', 'z' + 1, 'A', 'Z' + 1, '0', '9' + 1};\n        return getString(n, arg);\n    }\n\n    public static String getStringShortenThan(int n) {\n        int len = getIntegerMoreThanZeroLessThan(n);\n        return getStringWithCharacter(len);\n    }\n\n    public static String getStringWithNumAndChaShortenThan(int n) {\n        int len = getIntegerMoreThanZeroLessThan(n);\n        return getStringWithNumAndCha(len);\n    }\n\n    public static String getStringBetween(int n, int m) {\n        int len = getIntegerBetween(n, m);\n        return getStringWithCharacter(len);\n    }\n\n    public static String getStringWithNumAndChaBetween(int n, int m) {\n        int len = getIntegerBetween(n, m);\n        return getStringWithNumAndCha(len);\n    }\n\n    public static String getStringWithPrefix(int n, String prefix) {\n        int len = prefix.length();\n        if (n <= len)\n            return prefix;\n        else {\n            len = n - len;\n            StringBuilder res = new StringBuilder(prefix);\n            res.append(getStringWithCharacter(len));\n            return res.toString();\n        }\n    }\n\n    public static String getStringWithSuffix(int n, String suffix) {\n\n        int len = suffix.length();\n        if (n <= len)\n            return suffix;\n        else {\n            len = n - len;\n            StringBuilder res = new StringBuilder();\n            res.append(getStringWithCharacter(len));\n            res.append(suffix);\n            return res.toString();\n        }\n    }\n\n    public static String getStringWithBoth(int n, String prefix, String suffix) {\n        int len = prefix.length() + suffix.length();\n        StringBuilder res = new StringBuilder(prefix);\n        if (n <= len)\n            return res.append(suffix).toString();\n        else {\n            len = n - len;\n            res.append(getStringWithCharacter(len));\n            res.append(suffix);\n            return res.toString();\n        }\n    }\n\n    public static String getCheseWordWithPrifix(int n, String prefix) {\n        int len = prefix.length();\n        if (n <= len)\n            return prefix;\n        else {\n            len = n - len;\n            StringBuilder res = new StringBuilder(prefix);\n            res.append(getCheseWord(len));\n            return res.toString();\n        }\n    }\n\n    public static String getCheseWordWithSuffix(int n, String suffix) {\n\n        int len = suffix.length();\n        if (n <= len)\n            return suffix;\n        else {\n            len = n - len;\n            StringBuilder res = new StringBuilder();\n            res.append(getCheseWord(len));\n            res.append(suffix);\n            return res.toString();\n        }\n    }\n\n    public static String getCheseWordWithBoth(int n, String prefix, String suffix) {\n        int len = prefix.length() + suffix.length();\n        StringBuilder res = new StringBuilder(prefix);\n        if (n <= len)\n            return res.append(suffix).toString();\n        else {\n            len = n - len;\n            res.append(getCheseWord(len));\n            res.append(suffix);\n            return res.toString();\n        }\n    }\n\n    public static String getCheseWord(int len) {\n        StringBuilder res = new StringBuilder();\n        for (int i = 0; i < len; i++) {\n            char str = getCheseChar();\n            res.append(str);\n        }\n        return res.toString();\n    }\n\n    private static char getCheseChar() {\n        return (char) (UNICODE_START + rd.nextInt(UNICODE_END - UNICODE_START));\n    }\n\n    public static boolean getBoolean() {\n        return getIntegerMoreThanZeroLessThan(3) == 1;\n    }\n\n    public static String getStringByUUID() {\n        return UUID.randomUUID().toString();\n    }\n\n    public static int[] getRandomArray(int min, int max, int n) {\n        int len = max - min + 1;\n\n        if (max < min || n > len) {\n            return null;\n        }\n\n        int[] source = new int[len];\n        for (int i = min; i < min + len; i++) {\n            source[i - min] = i;\n        }\n\n        int[] result = new int[n];\n        Random rd = new Random();\n        int index = 0;\n        for (int i = 0; i < result.length; i++) {\n            index = rd.nextInt(len--);\n            result[i] = source[index];\n            source[index] = source[len];\n        }\n        return result;\n    }\n\n    public static Collection<Integer> getRandomCollection(int min, int max, int n) {\n        Set<Integer> res = new HashSet<Integer>();\n        int mx = max;\n        int mn = min;\n        if (n == (max + 1 - min)) {\n            for (int i = 1; i <= n; i++) {\n                res.add(i);\n            }\n            return res;\n        }\n        for (int i = 0; i < n; i++) {\n            int v = getIntegerBetween(mn, mx);\n            if (v == mx) {\n                mx--;\n            }\n            if (v == mn) {\n                mn++;\n            }\n            while (res.contains(v)) {\n                v = getIntegerBetween(mn, mx);\n                if (v == mx) {\n                    mx = v;\n                }\n                if (v == mn) {\n                    mn = v;\n                }\n            }\n            res.add(v);\n        }\n        return res;\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/util/RandomUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.util;\n\nimport java.util.Random;\nimport java.util.UUID;\n\npublic class RandomUtils {\n    private static final int UNICODE_START = '\\u4E00';\n    private static final int UNICODE_END = '\\u9FA0';\n    private static Random rd = new Random();\n\n    private RandomUtils() {\n\n    }\n\n    public static String getStringByUUID() {\n        return UUID.randomUUID().toString();\n    }\n\n    public static String getCheseWord(int len) {\n        StringBuilder res = new StringBuilder();\n\n        for (int i = 0; i < len; ++i) {\n            char str = getCheseChar();\n            res.append(str);\n        }\n\n        return res.toString();\n    }\n\n    public static String getStringWithNumber(int n) {\n        int[] arg = new int[] {'0', '9' + 1};\n        return getString(n, arg);\n    }\n\n    public static String getStringWithCharacter(int n) {\n        int[] arg = new int[] {'a', 'z' + 1, 'A', 'Z' + 1};\n        return getString(n, arg);\n    }\n\n    private static String getString(int n, int[] arg) {\n        StringBuilder res = new StringBuilder();\n        for (int i = 0; i < n; i++) {\n            res.append(getChar(arg));\n        }\n        return res.toString();\n    }\n\n    private static char getChar(int[] arg) {\n        int size = arg.length;\n        int c = rd.nextInt(size / 2);\n        c = c * 2;\n        return (char) (getIntegerBetween(arg[c], arg[c + 1]));\n    }\n\n    public static int getIntegerBetween(int n, int m) {\n        if (m == n) {\n            return n;\n        }\n        int res = getIntegerMoreThanZero();\n        return n + res % (m - n);\n    }\n\n    public static int getIntegerMoreThanZero() {\n        int res = rd.nextInt();\n        while (res <= 0) {\n            res = rd.nextInt();\n        }\n        return res;\n    }\n\n    private static char getCheseChar() {\n        return (char) (UNICODE_START + rd.nextInt(UNICODE_END - UNICODE_START));\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/util/StatUtil.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.test.util;\n\nimport java.math.BigDecimal;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicLong;\nimport javax.annotation.Generated;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\nimport static java.math.BigDecimal.ROUND_HALF_UP;\n\n@Generated(\"StatUtil\")\npublic class StatUtil {\n    private static Logger sysLogger = LoggerFactory.getLogger(StatUtil.class);\n    private static Logger logger = LoggerFactory.getLogger(\"StatLogger\");\n    private static final int MAX_KEY_NUM = Integer.parseInt(System.getProperty(\"stat.util.key.max.num\", \"10000\"));\n    private static volatile ConcurrentMap<String, Invoke> invokeCache = new ConcurrentHashMap<>(64);\n    private static volatile ConcurrentMap<String, Map<Long, SecondInvoke>> secondInvokeCache = new ConcurrentHashMap<>(\n        64);\n\n    private static final int STAT_WINDOW_SECONDS = Integer.parseInt(System.getProperty(\"stat.win.seconds\", \"60\"));\n    private static final String SPLITTER = \"|\";\n    private static ScheduledExecutorService daemon = Executors.newSingleThreadScheduledExecutor();\n\n    static class Invoke {\n        AtomicLong totalPv = new AtomicLong();\n        AtomicLong failPv = new AtomicLong();\n        AtomicLong sumRt = new AtomicLong();\n        AtomicLong maxRt = new AtomicLong();\n        AtomicLong minRt = new AtomicLong();\n        AtomicInteger topSecondPv = new AtomicInteger();\n        AtomicInteger secondPv = new AtomicInteger();\n        AtomicLong second = new AtomicLong(System.currentTimeMillis() / 1000L);\n    }\n\n    static class SecondInvoke implements Comparable<SecondInvoke> {\n        AtomicLong total = new AtomicLong();\n        AtomicLong fail = new AtomicLong();\n        AtomicLong sumRt = new AtomicLong();\n        AtomicLong maxRt = new AtomicLong();\n        AtomicLong minRt = new AtomicLong();\n        Long second = nowSecond();\n\n        @Override\n        public int compareTo(SecondInvoke o) {\n            return o.second.compareTo(second);\n        }\n    }\n\n    static {\n        daemon.scheduleAtFixedRate(new Runnable() {\n            @Override\n            public void run() {\n                try {\n                    printInvokeStat();\n                    printSecondInvokeStat();\n                } catch (Exception e) {\n                    logger.error(\"\", e);\n                }\n            }\n        }, STAT_WINDOW_SECONDS, STAT_WINDOW_SECONDS, TimeUnit.SECONDS);\n    }\n\n    private static void printInvokeStat() {\n        Map<String, Invoke> tmp = invokeCache;\n        invokeCache = new ConcurrentHashMap<>(64);\n\n        sysLogger.warn(\"printInvokeStat key count:{}\", tmp.size());\n        for (Map.Entry<String, Invoke> entry : tmp.entrySet()) {\n            String key = entry.getKey();\n            Invoke invoke = entry.getValue();\n            logger.warn(\"{}\",\n                buildLog(key, invoke.topSecondPv.get(), invoke.totalPv.get(), invoke.failPv.get(), invoke.minRt.get(),\n                    invoke.maxRt.get(), invoke.sumRt.get()));\n        }\n    }\n\n    private static void printSecondInvokeStat() {\n        sysLogger.warn(\"printSecondInvokeStat key count:{}\", secondInvokeCache.size());\n        for (Map.Entry<String, Map<Long, SecondInvoke>> entry : secondInvokeCache.entrySet()) {\n            String key = entry.getKey();\n            Map<Long, SecondInvoke> secondInvokeMap = entry.getValue();\n            long totalPv = 0L;\n            long failPv = 0L;\n            long topSecondPv = 0L;\n            long sumRt = 0L;\n            long maxRt = 0L;\n            long minRt = 0L;\n\n            for (Map.Entry<Long, SecondInvoke> invokeEntry : secondInvokeMap.entrySet()) {\n                long second = invokeEntry.getKey();\n                SecondInvoke secondInvoke = invokeEntry.getValue();\n                if (nowSecond() - second >= STAT_WINDOW_SECONDS) {\n                    secondInvokeMap.remove(second);\n                    continue;\n                }\n                long secondPv = secondInvoke.total.get();\n                totalPv += secondPv;\n                failPv += secondInvoke.fail.get();\n                sumRt += secondInvoke.sumRt.get();\n                if (maxRt < secondInvoke.maxRt.get()) {\n                    maxRt = secondInvoke.maxRt.get();\n                }\n                if (minRt > secondInvoke.minRt.get()) {\n                    minRt = secondInvoke.minRt.get();\n                }\n                if (topSecondPv < secondPv) {\n                    topSecondPv = secondPv;\n                }\n            }\n            if (secondInvokeMap.isEmpty()) {\n                secondInvokeCache.remove(key);\n                continue;\n            }\n            logger.warn(\"{}\", buildLog(key, topSecondPv, totalPv, failPv, minRt, maxRt, sumRt));\n        }\n    }\n\n    private static String buildLog(String key, long topSecondPv, long totalPv, long failPv, long minRt, long maxRt,\n        long sumRt) {\n        StringBuilder sb = new StringBuilder();\n        sb.append(SPLITTER);\n        sb.append(key);\n        sb.append(SPLITTER);\n        sb.append(topSecondPv);\n        sb.append(SPLITTER);\n        int tps = new BigDecimal(totalPv).divide(new BigDecimal(STAT_WINDOW_SECONDS),\n            ROUND_HALF_UP).intValue();\n        sb.append(tps);\n        sb.append(SPLITTER);\n        sb.append(totalPv);\n        sb.append(SPLITTER);\n        sb.append(failPv);\n        sb.append(SPLITTER);\n        sb.append(minRt);\n        sb.append(SPLITTER);\n        long avg = new BigDecimal(sumRt).divide(new BigDecimal(totalPv),\n            ROUND_HALF_UP).longValue();\n        sb.append(avg);\n        sb.append(SPLITTER);\n        sb.append(maxRt);\n        return sb.toString();\n    }\n\n    public static String buildKey(String... keys) {\n        if (keys == null || keys.length <= 0) {\n            return null;\n        }\n        StringBuilder sb = new StringBuilder();\n        for (String key : keys) {\n            sb.append(key);\n            sb.append(\",\");\n        }\n        sb.deleteCharAt(sb.length() - 1);\n        return sb.toString();\n    }\n\n    public static void addInvoke(String key, long rt) {\n        addInvoke(key, rt, true);\n    }\n\n    private static Invoke getAndSetInvoke(String key) {\n        Invoke invoke = invokeCache.get(key);\n        if (invoke == null) {\n            invokeCache.putIfAbsent(key, new Invoke());\n        }\n        return invokeCache.get(key);\n    }\n\n    public static void addInvoke(String key, int num, long rt, boolean success) {\n        if (invokeCache.size() > MAX_KEY_NUM || secondInvokeCache.size() > MAX_KEY_NUM) {\n            return;\n        }\n        Invoke invoke = getAndSetInvoke(key);\n        if (invoke == null) {\n            return;\n        }\n\n        invoke.totalPv.getAndAdd(num);\n        if (!success) {\n            invoke.failPv.getAndAdd(num);\n        }\n        long now = nowSecond();\n        AtomicLong oldSecond = invoke.second;\n        if (oldSecond.get() == now) {\n            invoke.secondPv.getAndAdd(num);\n        } else {\n            if (oldSecond.compareAndSet(oldSecond.get(), now)) {\n                if (invoke.secondPv.get() > invoke.topSecondPv.get()) {\n                    invoke.topSecondPv.set(invoke.secondPv.get());\n                }\n                invoke.secondPv.set(num);\n            } else {\n                invoke.secondPv.getAndAdd(num);\n            }\n        }\n\n        invoke.sumRt.addAndGet(rt);\n        if (invoke.maxRt.get() < rt) {\n            invoke.maxRt.set(rt);\n        }\n        if (invoke.minRt.get() > rt) {\n            invoke.minRt.set(rt);\n        }\n    }\n\n    public static void addInvoke(String key, long rt, boolean success) {\n        if (invokeCache.size() > MAX_KEY_NUM || secondInvokeCache.size() > MAX_KEY_NUM) {\n            return;\n        }\n        Invoke invoke = getAndSetInvoke(key);\n        if (invoke == null) {\n            return;\n        }\n\n        invoke.totalPv.getAndIncrement();\n        if (!success) {\n            invoke.failPv.getAndIncrement();\n        }\n        long now = nowSecond();\n        AtomicLong oldSecond = invoke.second;\n        if (oldSecond.get() == now) {\n            invoke.secondPv.getAndIncrement();\n        } else {\n            if (oldSecond.compareAndSet(oldSecond.get(), now)) {\n                if (invoke.secondPv.get() > invoke.topSecondPv.get()) {\n                    invoke.topSecondPv.set(invoke.secondPv.get());\n                }\n                invoke.secondPv.set(1);\n            } else {\n                invoke.secondPv.getAndIncrement();\n            }\n        }\n\n        invoke.sumRt.addAndGet(rt);\n        if (invoke.maxRt.get() < rt) {\n            invoke.maxRt.set(rt);\n        }\n        if (invoke.minRt.get() > rt) {\n            invoke.minRt.set(rt);\n        }\n    }\n\n    public static SecondInvoke getAndSetSecondInvoke(String key) {\n        if (!secondInvokeCache.containsKey(key)) {\n            secondInvokeCache.putIfAbsent(key, new ConcurrentHashMap<>(STAT_WINDOW_SECONDS));\n        }\n        Map<Long, SecondInvoke> secondInvokeMap = secondInvokeCache.get(key);\n        if (secondInvokeMap == null) {\n            return null;\n        }\n        long second = nowSecond();\n        if (!secondInvokeMap.containsKey(second)) {\n            secondInvokeMap.putIfAbsent(second, new SecondInvoke());\n        }\n        return secondInvokeMap.get(second);\n    }\n\n    public static void addSecondInvoke(String key, long rt) {\n        addSecondInvoke(key, rt, true);\n    }\n\n    public static void addSecondInvoke(String key, long rt, boolean success) {\n        if (invokeCache.size() > MAX_KEY_NUM || secondInvokeCache.size() > MAX_KEY_NUM) {\n            return;\n        }\n        SecondInvoke secondInvoke = getAndSetSecondInvoke(key);\n        if (secondInvoke == null) {\n            return;\n        }\n        secondInvoke.total.addAndGet(1);\n        if (!success) {\n            secondInvoke.fail.addAndGet(1);\n        }\n        secondInvoke.sumRt.addAndGet(rt);\n        if (secondInvoke.maxRt.get() < rt) {\n            secondInvoke.maxRt.set(rt);\n        }\n        if (secondInvoke.minRt.get() > rt) {\n            secondInvoke.minRt.set(rt);\n        }\n    }\n\n    public static void addPv(String key, long totalPv) {\n        addPv(key, totalPv, true);\n    }\n\n    public static void addPv(String key, long totalPv, boolean success) {\n        if (invokeCache.size() > MAX_KEY_NUM || secondInvokeCache.size() > MAX_KEY_NUM) {\n            return;\n        }\n        if (totalPv <= 0) {\n            return;\n        }\n        Invoke invoke = getAndSetInvoke(key);\n        if (invoke == null) {\n            return;\n        }\n        invoke.totalPv.addAndGet(totalPv);\n        if (!success) {\n            invoke.failPv.addAndGet(totalPv);\n        }\n        long now = nowSecond();\n        AtomicLong oldSecond = invoke.second;\n        if (oldSecond.get() == now) {\n            invoke.secondPv.addAndGet((int)totalPv);\n        } else {\n            if (oldSecond.compareAndSet(oldSecond.get(), now)) {\n                if (invoke.secondPv.get() > invoke.topSecondPv.get()) {\n                    invoke.topSecondPv.set(invoke.secondPv.get());\n                }\n                invoke.secondPv.set((int)totalPv);\n            } else {\n                invoke.secondPv.addAndGet((int)totalPv);\n            }\n        }\n    }\n\n    public static void addSecondPv(String key, long totalPv) {\n        addSecondPv(key, totalPv, true);\n    }\n\n    public static void addSecondPv(String key, long totalPv, boolean success) {\n        if (invokeCache.size() > MAX_KEY_NUM || secondInvokeCache.size() > MAX_KEY_NUM) {\n            return;\n        }\n        if (totalPv <= 0) {\n            return;\n        }\n        SecondInvoke secondInvoke = getAndSetSecondInvoke(key);\n        if (secondInvoke == null) {\n            return;\n        }\n        secondInvoke.total.addAndGet(totalPv);\n        if (!success) {\n            secondInvoke.fail.addAndGet(totalPv);\n        }\n    }\n\n    public static boolean isOverFlow(String key, int tps) {\n        return nowTps(key) >= tps;\n    }\n\n    public static int nowTps(String key) {\n        Map<Long, SecondInvoke> secondInvokeMap = secondInvokeCache.get(key);\n        if (secondInvokeMap != null) {\n            SecondInvoke secondInvoke = secondInvokeMap.get(nowSecond());\n            if (secondInvoke != null) {\n                return (int)secondInvoke.total.get();\n            }\n        }\n        Invoke invoke = invokeCache.get(key);\n        if (invoke == null) {\n            return 0;\n        }\n        AtomicLong oldSecond = invoke.second;\n        if (oldSecond.get() == nowSecond()) {\n            return invoke.secondPv.get();\n        }\n        return 0;\n    }\n\n    public static int totalPvInWindow(String key, int windowSeconds) {\n        List<SecondInvoke> list = secondInvokeList(key, windowSeconds);\n        long totalPv = 0;\n        for (int i = 0; i < windowSeconds && i < list.size(); i++) {\n            totalPv += list.get(i).total.get();\n        }\n        return (int)totalPv;\n    }\n\n    public static int failPvInWindow(String key, int windowSeconds) {\n        List<SecondInvoke> list = secondInvokeList(key, windowSeconds);\n        long failPv = 0;\n        for (int i = 0; i < windowSeconds && i < list.size(); i++) {\n            failPv += list.get(i).fail.get();\n        }\n        return (int)failPv;\n    }\n\n    public static int topTpsInWindow(String key, int windowSeconds) {\n        List<SecondInvoke> list = secondInvokeList(key, windowSeconds);\n        long topTps = 0;\n        for (int i = 0; i < windowSeconds && i < list.size(); i++) {\n            long secondPv = list.get(i).total.get();\n            if (topTps < secondPv) {\n                topTps = secondPv;\n            }\n        }\n        return (int)topTps;\n    }\n\n    public static int avgRtInWindow(String key, int windowSeconds) {\n        List<SecondInvoke> list = secondInvokeList(key, windowSeconds);\n        long sumRt = 0;\n        long totalPv = 0;\n        for (int i = 0; i < windowSeconds && i < list.size(); i++) {\n            sumRt += list.get(i).sumRt.get();\n            totalPv += list.get(i).total.get();\n        }\n        if (totalPv <= 0) {\n            return 0;\n        }\n        long avg = new BigDecimal(sumRt).divide(new BigDecimal(totalPv),\n            ROUND_HALF_UP).longValue();\n        return (int)avg;\n    }\n\n    public static int maxRtInWindow(String key, int windowSeconds) {\n        List<SecondInvoke> list = secondInvokeList(key, windowSeconds);\n        long maxRt = 0;\n        long totalPv = 0;\n        for (int i = 0; i < windowSeconds && i < list.size(); i++) {\n            if (maxRt < list.get(i).maxRt.get()) {\n                maxRt = list.get(i).maxRt.get();\n            }\n        }\n        return (int)maxRt;\n    }\n\n    public static int minRtInWindow(String key, int windowSeconds) {\n        List<SecondInvoke> list = secondInvokeList(key, windowSeconds);\n        long minRt = 0;\n        long totalPv = 0;\n        for (int i = 0; i < windowSeconds && i < list.size(); i++) {\n            if (minRt < list.get(i).minRt.get()) {\n                minRt = list.get(i).minRt.get();\n            }\n        }\n        return (int)minRt;\n    }\n\n    private static List<SecondInvoke> secondInvokeList(String key, int windowSeconds) {\n        if (windowSeconds > STAT_WINDOW_SECONDS || windowSeconds <= 0) {\n            throw new IllegalArgumentException(\"windowSeconds Must Not be great than \" + STAT_WINDOW_SECONDS);\n        }\n        Map<Long, SecondInvoke> secondInvokeMap = secondInvokeCache.get(key);\n        if (secondInvokeMap == null || secondInvokeMap.isEmpty()) {\n            return new ArrayList<>();\n        }\n        List<SecondInvoke> list = new ArrayList<>();\n        list.addAll(secondInvokeMap.values());\n        Collections.sort(list);\n        return list;\n    }\n\n    private static long nowSecond() {\n        return System.currentTimeMillis() / 1000L;\n    }\n\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/util/TestUtil.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.util;\n\nimport java.io.IOException;\nimport java.nio.charset.StandardCharsets;\nimport java.util.LinkedHashMap;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.TimeUnit;\n\npublic final class TestUtil {\n\n    private TestUtil() {\n    }\n\n    public static Long parseStringToLong(String s, Long defval) {\n        Long val = defval;\n        try {\n            val = Long.parseLong(s);\n        } catch (NumberFormatException e) {\n            val = defval;\n        }\n        return val;\n    }\n\n    public static Integer parseStringToInteger(String s, Integer defval) {\n        Integer val = defval;\n        try {\n            val = Integer.parseInt(s);\n        } catch (NumberFormatException e) {\n            val = defval;\n        }\n        return val;\n    }\n\n    public static String addQuoteToParamater(String param) {\n        StringBuilder sb = new StringBuilder(\"'\");\n        sb.append(param).append(\"'\");\n        return sb.toString();\n    }\n\n    public static void waitForMonment(long time) {\n        try {\n            Thread.sleep(time);\n        } catch (InterruptedException e) {\n            e.printStackTrace();\n        }\n    }\n\n    public static void waitForSeconds(long time) {\n        try {\n            TimeUnit.SECONDS.sleep(time);\n        } catch (InterruptedException e) {\n            e.printStackTrace();\n        }\n    }\n\n    public static void waitForMinutes(long time) {\n        try {\n            TimeUnit.MINUTES.sleep(time);\n        } catch (InterruptedException e) {\n            e.printStackTrace();\n        }\n    }\n\n    public static void waitForInputQuit() {\n        waitForInput(\"quit\");\n    }\n\n    public static void waitForInput(String keyWord) {\n        waitForInput(keyWord,\n            String.format(\"The thread will wait until you input stop command[%s]:\", keyWord));\n    }\n\n    public static void waitForInput(String keyWord, String info) {\n        try {\n            byte[] b = new byte[1024];\n            int n = System.in.read(b);\n            String s = new String(b, 0, n - 1, StandardCharsets.UTF_8).replace(\"\\r\", \"\").replace(\"\\n\", \"\");\n            while (!s.equals(keyWord)) {\n                n = System.in.read(b);\n                s = new String(b, 0, n - 1, StandardCharsets.UTF_8);\n            }\n        } catch (IOException e) {\n            e.printStackTrace();\n        }\n    }\n\n    public static <K, V extends Comparable<? super V>> Map<K, V> sortByValue(Map<K, V> map) {\n        List<Map.Entry<K, V>> list = new LinkedList<Map.Entry<K, V>>(map.entrySet());\n        list.sort(Map.Entry.comparingByValue());\n\n        Map<K, V> result = new LinkedHashMap<K, V>();\n        for (Map.Entry<K, V> entry : list) {\n            result.put(entry.getKey(), entry.getValue());\n        }\n        return result;\n    }\n\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/util/TestUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.util;\n\nimport java.util.concurrent.TimeUnit;\n\npublic class TestUtils {\n    public static void waitForMoment(long time) {\n        try {\n            Thread.sleep(time);\n        } catch (InterruptedException var3) {\n            var3.printStackTrace();\n        }\n\n    }\n\n    public static void waitForSeconds(long time) {\n        try {\n            TimeUnit.SECONDS.sleep(time);\n        } catch (InterruptedException var3) {\n            var3.printStackTrace();\n        }\n\n    }\n\n    public static void waitForMinutes(long time) {\n        try {\n            TimeUnit.MINUTES.sleep(time);\n        } catch (InterruptedException var3) {\n            var3.printStackTrace();\n        }\n\n    }\n\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/util/VerifyUtils.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.util;\n\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.Collections;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\n\npublic class VerifyUtils {\n    private static Logger logger = LoggerFactory.getLogger(VerifyUtils.class);\n\n    public static int verify(Collection<Object> sendMsgs, Collection<Object> recvMsgs) {\n        int miss = 0;\n        for (Object msg : sendMsgs) {\n            if (!recvMsgs.contains(msg)) {\n                miss++;\n            }\n        }\n\n        return miss;\n    }\n\n    public static Collection<Object> getFilterdMessage(Collection<Object> sendMsgs,\n        Collection<Object> recvMsgs) {\n        Collection<Object> recvMsgsSync = Collections.synchronizedCollection(recvMsgs);\n        Collection<Object> filteredMsgs = new ArrayList<Object>();\n        int filterNum = 0;\n        for (Object msg : recvMsgsSync) {\n            if (sendMsgs.contains(msg)) {\n                filteredMsgs.add(msg);\n            } else {\n                filterNum++;\n            }\n        }\n\n        logger.info(String.format(\"[%s] messages is filtered!\", filterNum));\n        return filteredMsgs;\n    }\n\n    public static int verifyUserProperty(Collection<Object> sendMsgs, Collection<Object> recvMsgs) {\n        return 0;\n    }\n\n    public static void verifyMessageQueueId(int expectId, Collection<Object> msgs) {\n        for (Object msg : msgs) {\n            MessageExt msgEx = (MessageExt) msg;\n            assert expectId == msgEx.getQueueId();\n        }\n    }\n\n    public static boolean verifyBalance(int msgSize, float error, int... recvSize) {\n        boolean balance = true;\n        int evenSize = msgSize / recvSize.length;\n        for (int size : recvSize) {\n            if (Math.abs(size - evenSize) > error * evenSize) {\n                balance = false;\n                break;\n            }\n        }\n        return balance;\n    }\n\n    public static boolean verifyBalance(int msgSize, int... recvSize) {\n        return verifyBalance(msgSize, 0.1f, recvSize);\n    }\n\n    public static boolean verifyDelay(long delayTimeMills, long nextLevelDelayTimeMills,\n        Collection<Object> recvMsgTimes) {\n        boolean delay = true;\n        for (Object timeObj : recvMsgTimes) {\n            long time = (Long) timeObj;\n            if (time < delayTimeMills || time > nextLevelDelayTimeMills) {\n                delay = false;\n                logger.info(String.format(\"delay error:%s\", Math.abs(time - delayTimeMills)));\n                break;\n            }\n        }\n        return delay;\n    }\n\n    public static boolean verifyOrder(Collection<Collection<Object>> queueMsgs) {\n        for (Collection<Object> msgs : queueMsgs) {\n            if (!verifyOrderMsg(msgs)) {\n                return false;\n            }\n        }\n        return true;\n\n    }\n\n    public static boolean verifyOrderMsg(Collection<Object> msgs) {\n        int min = Integer.MIN_VALUE;\n        int curr;\n        if (msgs.size() == 0 || msgs.size() == 1) {\n            return true;\n        } else {\n            for (Object msg : msgs) {\n                curr = Integer.parseInt((String) msg);\n                if (curr < min) {\n                    return false;\n                } else {\n                    min = curr;\n                }\n            }\n        }\n        return true;\n    }\n\n    public static boolean verifyRT(Collection<Object> rts, long maxRTMills) {\n        boolean rtExpect = true;\n        for (Object obj : rts) {\n            long rt = (Long) obj;\n            if (rt > maxRTMills) {\n                rtExpect = false;\n                logger.info(String.format(\"%s greater thran maxtRT:%s!\", rt, maxRTMills));\n\n            }\n        }\n        return rtExpect;\n    }\n\n    public static void main(String[] args) {\n        verifyBalance(400, 0.1f, 230, 190);\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/util/data/collect/DataCollector.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.util.data.collect;\n\nimport java.util.Collection;\n\npublic interface DataCollector {\n\n    void resetData();\n\n    Collection<Object> getAllData();\n\n    Collection<Object> getAllDataWithoutDuplicate();\n\n    void addData(Object data);\n\n    long getDataSizeWithoutDuplicate();\n\n    long getDataSize();\n\n    boolean isRepeatedData(Object data);\n\n    int getRepeatedTimeForData(Object data);\n\n    void removeData(Object data);\n\n    void lockIncrement();\n\n    void unlockIncrement();\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/util/data/collect/DataCollectorManager.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.util.data.collect;\n\nimport java.util.HashMap;\nimport java.util.Map;\nimport org.apache.rocketmq.test.util.data.collect.impl.ListDataCollectorImpl;\nimport org.apache.rocketmq.test.util.data.collect.impl.MapDataCollectorImpl;\n\npublic final class DataCollectorManager {\n    private static DataCollectorManager instance = new DataCollectorManager();\n    private Map<String, DataCollector> collectMap = new HashMap<String, DataCollector>();\n    private Object lock = new Object();\n\n    private DataCollectorManager() {\n    }\n\n    public static DataCollectorManager getInstance() {\n        return instance;\n    }\n\n    public DataCollector fetchDataCollector(String key) {\n        String realKey = key;\n        if (!collectMap.containsKey(realKey)) {\n            synchronized (lock) {\n                if (!collectMap.containsKey(realKey)) {\n                    DataCollector collect = (DataCollector) new MapDataCollectorImpl();\n                    collectMap.put(realKey, collect);\n                }\n            }\n        }\n        return collectMap.get(realKey);\n    }\n\n    public DataCollector fetchMapDataCollector(String key) {\n        String realKey = key;\n        if (!collectMap.containsKey(realKey)\n            || collectMap.get(realKey) instanceof ListDataCollectorImpl) {\n            synchronized (lock) {\n                if (!collectMap.containsKey(realKey)\n                    || collectMap.get(realKey) instanceof ListDataCollectorImpl) {\n                    DataCollector collect = null;\n                    if (collectMap.containsKey(realKey)) {\n                        DataCollector src = collectMap.get(realKey);\n                        collect = new MapDataCollectorImpl(src.getAllData());\n                    } else {\n                        collect = new MapDataCollectorImpl();\n                    }\n                    collectMap.put(realKey, collect);\n\n                }\n            }\n        }\n        return collectMap.get(realKey);\n    }\n\n    public DataCollector fetchListDataCollector(String key) {\n        String realKey = key;\n        if (!collectMap.containsKey(realKey)\n            || collectMap.get(realKey) instanceof MapDataCollectorImpl) {\n            synchronized (lock) {\n                if (!collectMap.containsKey(realKey)\n                    || collectMap.get(realKey) instanceof MapDataCollectorImpl) {\n                    DataCollector collect = null;\n                    if (collectMap.containsKey(realKey)) {\n                        DataCollector src = collectMap.get(realKey);\n                        collect = new ListDataCollectorImpl(src.getAllData());\n                    } else {\n                        collect = new ListDataCollectorImpl();\n                    }\n                    collectMap.put(realKey, collect);\n                }\n            }\n        }\n        return collectMap.get(realKey);\n    }\n\n    public void resetDataCollect(String key) {\n        if (collectMap.containsKey(key)) {\n            collectMap.get(key).resetData();\n        }\n    }\n\n    public void resetAll() {\n        for (Map.Entry<String, DataCollector> entry : collectMap.entrySet()) {\n            entry.getValue().resetData();\n        }\n    }\n\n    public void removeDataCollect(String key) {\n        collectMap.remove(key);\n    }\n\n    public void removeAll() {\n        collectMap.clear();\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/util/data/collect/DataFilter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.util.data.collect;\n\npublic interface DataFilter {\n\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/ListDataCollectorImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.util.data.collect.impl;\n\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.Collections;\nimport java.util.HashSet;\nimport java.util.List;\nimport org.apache.rocketmq.test.util.data.collect.DataCollector;\n\npublic class ListDataCollectorImpl implements DataCollector {\n\n    private List<Object> datas = new ArrayList<Object>();\n    private boolean lock = false;\n\n    public ListDataCollectorImpl() {\n\n    }\n\n    public ListDataCollectorImpl(Collection<Object> datas) {\n        for (Object data : datas) {\n            addData(data);\n        }\n    }\n\n    @Override\n    public Collection<Object> getAllData() {\n        return datas;\n    }\n\n    @Override\n    public synchronized void resetData() {\n        datas.clear();\n        unlockIncrement();\n    }\n\n    @Override\n    public long getDataSizeWithoutDuplicate() {\n        return getAllDataWithoutDuplicate().size();\n    }\n\n    @Override\n    public synchronized void addData(Object data) {\n        if (lock) {\n            return;\n        }\n        datas.add(data);\n    }\n\n    @Override\n    public long getDataSize() {\n        return datas.size();\n    }\n\n    @Override\n    public boolean isRepeatedData(Object data) {\n        return Collections.frequency(datas, data) == 1;\n    }\n\n    @Override\n    public synchronized Collection<Object> getAllDataWithoutDuplicate() {\n        return new HashSet<Object>(datas);\n    }\n\n    @Override\n    public int getRepeatedTimeForData(Object data) {\n        int res = 0;\n        for (Object obj : datas) {\n            if (obj.equals(data)) {\n                res++;\n            }\n        }\n        return res;\n    }\n\n    @Override\n    public synchronized void removeData(Object data) {\n        datas.remove(data);\n    }\n\n    @Override\n    public void lockIncrement() {\n        lock = true;\n    }\n\n    @Override\n    public void unlockIncrement() {\n        lock = false;\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/MapDataCollectorImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.util.data.collect.impl;\n\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Map.Entry;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.rocketmq.test.util.data.collect.DataCollector;\n\npublic class MapDataCollectorImpl implements DataCollector {\n\n    private Map<Object, AtomicInteger> datas = new ConcurrentHashMap<Object, AtomicInteger>();\n    private boolean lock = false;\n\n    public MapDataCollectorImpl() {\n\n    }\n\n    public MapDataCollectorImpl(Collection<Object> datas) {\n        for (Object data : datas) {\n            addData(data);\n        }\n    }\n\n    @Override\n    public synchronized void addData(Object data) {\n        if (lock) {\n            return;\n        }\n        if (datas.containsKey(data)) {\n            datas.get(data).addAndGet(1);\n        } else {\n            datas.put(data, new AtomicInteger(1));\n        }\n    }\n\n    @Override\n    public Collection<Object> getAllData() {\n        List<Object> lst = new ArrayList<Object>();\n        for (Entry<Object, AtomicInteger> entry : datas.entrySet()) {\n            for (int i = 0; i < entry.getValue().get(); i++) {\n                lst.add(entry.getKey());\n            }\n        }\n        return lst;\n    }\n\n    @Override\n    public long getDataSizeWithoutDuplicate() {\n        return datas.keySet().size();\n    }\n\n    @Override\n    public void resetData() {\n        datas.clear();\n        unlockIncrement();\n    }\n\n    @Override\n    public long getDataSize() {\n        long sum = 0;\n        for (AtomicInteger count : datas.values()) {\n            sum = sum + count.get();\n        }\n        return sum;\n    }\n\n    @Override\n    public boolean isRepeatedData(Object data) {\n        if (datas.containsKey(data)) {\n            return datas.get(data).get() == 1;\n        }\n        return false;\n    }\n\n    @Override\n    public Collection<Object> getAllDataWithoutDuplicate() {\n        return datas.keySet();\n    }\n\n    @Override\n    public int getRepeatedTimeForData(Object data) {\n        if (datas.containsKey(data)) {\n            return datas.get(data).intValue();\n        }\n        return 0;\n    }\n\n    @Override\n    public void removeData(Object data) {\n        datas.remove(data);\n    }\n\n    @Override\n    public void lockIncrement() {\n        lock = true;\n    }\n\n    @Override\n    public void unlockIncrement() {\n        lock = false;\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/util/parallel/ParallelTask.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.util.parallel;\n\nimport java.util.concurrent.CountDownLatch;\n\npublic abstract class ParallelTask extends Thread {\n    private CountDownLatch latch = null;\n\n    public CountDownLatch getLatch() {\n        return latch;\n    }\n\n    public void setLatch(CountDownLatch latch) {\n        this.latch = latch;\n    }\n\n    public abstract void execute();\n\n    @Override\n    public void run() {\n        this.execute();\n\n        if (latch != null) {\n            latch.countDown();\n        }\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/util/parallel/ParallelTaskExecutor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.util.parallel;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\n\npublic class ParallelTaskExecutor {\n    public List<ParallelTask> tasks = new ArrayList<ParallelTask>();\n    public ExecutorService cachedThreadPool = Executors.newCachedThreadPool();\n    public CountDownLatch latch = null;\n\n    public ParallelTaskExecutor() {\n\n    }\n\n    public void pushTask(ParallelTask task) {\n        tasks.add(task);\n    }\n\n    public void startBlock() {\n        init();\n        startTask();\n        try {\n            latch.await();\n        } catch (InterruptedException e) {\n            e.printStackTrace();\n        }\n    }\n\n    public void startNoBlock() {\n        for (ParallelTask task : tasks) {\n            cachedThreadPool.execute(task);\n        }\n    }\n\n    private void init() {\n        latch = new CountDownLatch(tasks.size());\n        for (ParallelTask task : tasks) {\n            task.setLatch(latch);\n        }\n    }\n\n    private void startTask() {\n        for (ParallelTask task : tasks) {\n            task.start();\n        }\n    }\n}\n"
  },
  {
    "path": "test/src/main/java/org/apache/rocketmq/test/util/parallel/Task4Test.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.util.parallel;\n\npublic class Task4Test extends ParallelTask {\n    private String name = \"\";\n\n    public Task4Test(String name) {\n        this.name = name;\n    }\n\n    @Override\n    public void execute() {\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/autoswitchrole/AutoSwitchRoleBase.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.autoswitchrole;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.net.InetAddress;\nimport java.net.InetSocketAddress;\nimport java.net.ServerSocket;\nimport java.net.SocketAddress;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Random;\nimport java.util.UUID;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.message.MessageDecoder;\nimport org.apache.rocketmq.common.message.MessageExtBrokerInner;\nimport org.apache.rocketmq.common.ControllerConfig;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.PutMessageStatus;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.config.FlushDiskType;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\n\nimport static org.awaitility.Awaitility.await;\n\nimport static org.junit.Assert.assertSame;\nimport static org.junit.Assert.assertTrue;\n\npublic class AutoSwitchRoleBase {\n\n    protected static final String STORE_PATH_ROOT_PARENT_DIR = System.getProperty(\"user.home\") + File.separator +\n        UUID.randomUUID().toString().replace(\"-\", \"\");\n    private static final String STORE_PATH_ROOT_DIR = STORE_PATH_ROOT_PARENT_DIR + File.separator + \"store\";\n    private static final String STORE_MESSAGE = \"Once, there was a chance for me!\";\n    private static final byte[] MESSAGE_BODY = STORE_MESSAGE.getBytes();\n    protected static List<BrokerController> brokerList;\n    private static SocketAddress bornHost;\n    private static SocketAddress storeHost;\n    private static int number = 0;\n\n    protected static void initialize() {\n        brokerList = new ArrayList<>();\n        try {\n            storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123);\n            bornHost = new InetSocketAddress(InetAddress.getByName(\"127.0.0.1\"), 0);\n        } catch (Exception ignored) {\n        }\n    }\n\n    public static int nextPort() throws IOException {\n        return nextPort(1001, 9999);\n    }\n\n    public static int nextPort(int minPort, int maxPort) throws IOException {\n\n        Random random = new Random();\n        int tempPort;\n        int port;\n        while (true) {\n            try {\n                tempPort = random.nextInt(maxPort) % (maxPort - minPort + 1) + minPort;\n                ServerSocket serverSocket = new ServerSocket(tempPort);\n                port = serverSocket.getLocalPort();\n                serverSocket.close();\n                break;\n            } catch (IOException ignored) {\n                if (number > 200) {\n                    throw new IOException(\"This server's open ports are temporarily full!\");\n                }\n                ++number;\n            }\n        }\n        number = 0;\n        return port;\n    }\n\n    public BrokerController startBroker(String namesrvAddress, String controllerAddress, String brokerName,\n        int brokerId, int haPort,\n        int brokerListenPort,\n        int nettyListenPort, BrokerRole expectedRole, int mappedFileSize) throws Exception {\n        final MessageStoreConfig storeConfig = buildMessageStoreConfig(brokerName + \"#\" + brokerId, haPort, mappedFileSize);\n        storeConfig.setHaMaxTimeSlaveNotCatchup(3 * 1000);\n        final BrokerConfig brokerConfig = new BrokerConfig();\n        brokerConfig.setListenPort(brokerListenPort);\n        brokerConfig.setNamesrvAddr(namesrvAddress);\n        brokerConfig.setControllerAddr(controllerAddress);\n        brokerConfig.setSyncBrokerMetadataPeriod(2 * 1000);\n        brokerConfig.setCheckSyncStateSetPeriod(2 * 1000);\n        brokerConfig.setBrokerName(brokerName);\n        brokerConfig.setEnableControllerMode(true);\n\n        final NettyServerConfig nettyServerConfig = new NettyServerConfig();\n        nettyServerConfig.setListenPort(nettyListenPort);\n\n        final BrokerController brokerController = new BrokerController(brokerConfig, nettyServerConfig, new NettyClientConfig(), storeConfig);\n        assertTrue(brokerController.initialize());\n        brokerController.start();\n        brokerList.add(brokerController);\n        await().atMost(20, TimeUnit.SECONDS).until(() -> (expectedRole == BrokerRole.SYNC_MASTER) == brokerController.getReplicasManager().isMasterState());\n        return brokerController;\n    }\n\n    protected MessageStoreConfig buildMessageStoreConfig(final String brokerDir, final int haPort,\n        final int mappedFileSize) {\n        MessageStoreConfig storeConfig = new MessageStoreConfig();\n        storeConfig.setHaSendHeartbeatInterval(1000);\n        storeConfig.setBrokerRole(BrokerRole.SLAVE);\n        storeConfig.setHaListenPort(haPort);\n        storeConfig.setStorePathRootDir(STORE_PATH_ROOT_DIR + File.separator + brokerDir);\n        storeConfig.setStorePathCommitLog(STORE_PATH_ROOT_DIR + File.separator + brokerDir + File.separator + \"commitlog\");\n        storeConfig.setStorePathEpochFile(STORE_PATH_ROOT_DIR + File.separator + brokerDir + File.separator + \"EpochFileCache\");\n        storeConfig.setStorePathBrokerIdentity(STORE_PATH_ROOT_DIR + File.separator + brokerDir + File.separator + \"brokerIdentity\");\n        storeConfig.setTotalReplicas(3);\n        storeConfig.setInSyncReplicas(2);\n\n        storeConfig.setMappedFileSizeCommitLog(mappedFileSize);\n        storeConfig.setMappedFileSizeConsumeQueue(1024 * 1024);\n        storeConfig.setMaxHashSlotNum(10000);\n        storeConfig.setMaxIndexNum(100 * 100);\n        storeConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);\n        storeConfig.setFlushIntervalConsumeQueue(1);\n        return storeConfig;\n    }\n\n    protected static ControllerConfig buildControllerConfig(final String id, final String peers) {\n        final ControllerConfig config = new ControllerConfig();\n        config.setControllerDLegerGroup(\"group1\");\n        config.setControllerDLegerPeers(peers);\n        config.setControllerDLegerSelfId(id);\n        config.setMappedFileSize(1024 * 1024);\n        config.setControllerStorePath(STORE_PATH_ROOT_DIR + File.separator + \"namesrv\" + id + File.separator + \"DLedgerController\");\n        return config;\n    }\n\n    protected MessageExtBrokerInner buildMessage(String topic) {\n        MessageExtBrokerInner msg = new MessageExtBrokerInner();\n        msg.setTopic(topic);\n        msg.setTags(\"TAG1\");\n        msg.setBody(MESSAGE_BODY);\n        msg.setKeys(String.valueOf(System.currentTimeMillis()));\n        msg.setQueueId(0);\n        msg.setSysFlag(0);\n        msg.setBornTimestamp(System.currentTimeMillis());\n        msg.setStoreHost(storeHost);\n        msg.setBornHost(bornHost);\n        msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));\n        return msg;\n    }\n\n    protected void putMessage(MessageStore messageStore, String topic) {\n        // Put message on master\n        for (int i = 0; i < 10; i++) {\n            assertSame(messageStore.putMessage(buildMessage(topic)).getPutMessageStatus(), PutMessageStatus.PUT_OK);\n        }\n    }\n\n    protected void checkMessage(final MessageStore messageStore, String topic, int totalNums, int startOffset) {\n        await().atMost(30, TimeUnit.SECONDS)\n            .until(() -> {\n                GetMessageResult result = messageStore.getMessage(\"GROUP_A\", topic, 0, startOffset, 1024, null);\n//                System.out.printf(result + \"%n\");\n//                System.out.printf(\"maxPhyOffset=\" + messageStore.getMaxPhyOffset() + \"%n\");\n//                System.out.printf(\"confirmOffset=\" + messageStore.getConfirmOffset() + \"%n\");\n                return result != null && result.getStatus() == GetMessageStatus.FOUND && result.getMessageCount() >= totalNums;\n            });\n    }\n\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/autoswitchrole/AutoSwitchRoleIntegrationTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.autoswitchrole;\n\nimport java.io.File;\nimport java.util.Random;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.controller.ReplicasManager;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.ControllerConfig;\nimport org.apache.rocketmq.common.namesrv.NamesrvConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.body.SyncStateSet;\nimport org.apache.rocketmq.controller.ControllerManager;\nimport org.apache.rocketmq.namesrv.NamesrvController;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoResponseHeader;\nimport org.apache.rocketmq.store.MappedFileQueue;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.ha.HAClient;\nimport org.apache.rocketmq.store.ha.HAConnectionState;\nimport org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService;\nimport org.apache.rocketmq.store.logfile.MappedFile;\nimport org.junit.AfterClass;\nimport org.junit.BeforeClass;\nimport org.junit.Ignore;\nimport org.junit.Test;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertTrue;\n\n@Ignore\npublic class AutoSwitchRoleIntegrationTest extends AutoSwitchRoleBase {\n\n    private static final int DEFAULT_FILE_SIZE = 1024 * 1024;\n    private static NamesrvController namesrvController;\n    private static ControllerManager controllerManager;\n    private static String nameserverAddress;\n    private static String controllerAddress;\n\n    private static ControllerConfig controllerConfig;\n\n    private BrokerController brokerController1;\n    private BrokerController brokerController2;\n    private Random random = new Random();\n\n    @BeforeClass\n    public static void init() throws Exception {\n        initialize();\n\n        int controllerPort = nextPort();\n        final String peers = String.format(\"n0-localhost:%d\", controllerPort);\n\n        final NettyServerConfig serverConfig = new NettyServerConfig();\n        int namesrvPort = nextPort();\n        serverConfig.setListenPort(namesrvPort);\n\n        controllerConfig = buildControllerConfig(\"n0\", peers);\n        namesrvController = new NamesrvController(new NamesrvConfig(), serverConfig, new NettyClientConfig());\n        assertTrue(namesrvController.initialize());\n        namesrvController.start();\n\n        initAndStartControllerManager();\n\n        nameserverAddress = \"127.0.0.1:\" + namesrvPort + \";\";\n        controllerAddress = \"127.0.0.1:\" + controllerPort + \";\";\n    }\n\n    private static void initAndStartControllerManager() {\n        controllerManager = new ControllerManager(controllerConfig, new NettyServerConfig(), new NettyClientConfig());\n        assertTrue(controllerManager.initialize());\n        controllerManager.start();\n    }\n\n    public void initBroker(int mappedFileSize, String brokerName) throws Exception {\n\n        this.brokerController1 = startBroker(nameserverAddress, controllerAddress, brokerName, 1, nextPort(), nextPort(), nextPort(), BrokerRole.SYNC_MASTER, mappedFileSize);\n        this.brokerController2 = startBroker(nameserverAddress, controllerAddress, brokerName, 2, nextPort(), nextPort(), nextPort(), BrokerRole.SLAVE, mappedFileSize);\n        // Wait slave connecting to master\n        assertTrue(waitSlaveReady(this.brokerController2.getMessageStore()));\n        Thread.sleep(1000);\n    }\n\n    public void mockData(String topic) throws Exception {\n        final MessageStore messageStore = brokerController1.getMessageStore();\n        putMessage(messageStore, topic);\n        // Check slave message\n        checkMessage(brokerController2.getMessageStore(), topic, 10, 0);\n    }\n\n    public boolean waitSlaveReady(MessageStore messageStore) throws InterruptedException {\n        int tryTimes = 0;\n        while (tryTimes < 100) {\n            final HAClient haClient = messageStore.getHaService().getHAClient();\n            if (haClient != null && haClient.getCurrentState().equals(HAConnectionState.TRANSFER)) {\n                return true;\n            } else {\n                Thread.sleep(2000);\n                tryTimes++;\n            }\n        }\n        return false;\n    }\n\n    @Test\n    public void testCheckSyncStateSet() throws Exception {\n        String topic = \"Topic-\" + AutoSwitchRoleIntegrationTest.class.getSimpleName() + random.nextInt(65535);\n        String brokerName = \"Broker-\" + AutoSwitchRoleIntegrationTest.class.getSimpleName() + random.nextInt(65535);\n        initBroker(DEFAULT_FILE_SIZE, brokerName);\n\n        mockData(topic);\n\n        // Check SyncStateSet\n        final ReplicasManager replicasManager = brokerController1.getReplicasManager();\n        SyncStateSet syncStateSet = replicasManager.getSyncStateSet();\n        assertEquals(2, syncStateSet.getSyncStateSet().size());\n\n        // Shutdown controller2\n        ScheduledExecutorService singleThread = Executors.newSingleThreadScheduledExecutor();\n        while (!singleThread.awaitTermination(6 * 1000, TimeUnit.MILLISECONDS)) {\n            this.brokerController2.shutdown();\n            singleThread.shutdown();\n        }\n\n        syncStateSet = replicasManager.getSyncStateSet();\n        shutdownAndClearBroker();\n        assertEquals(1, syncStateSet.getSyncStateSet().size());\n    }\n\n    @Test\n    public void testChangeMaster() throws Exception {\n        String topic = \"Topic-\" + AutoSwitchRoleIntegrationTest.class.getSimpleName() + random.nextInt(65535);\n        String brokerName = \"Broker-\" + AutoSwitchRoleIntegrationTest.class.getSimpleName() + random.nextInt(65535);\n        initBroker(DEFAULT_FILE_SIZE, brokerName);\n        int listenPort = brokerController1.getBrokerConfig().getListenPort();\n        int nettyPort = brokerController1.getNettyServerConfig().getListenPort();\n        mockData(topic);\n\n        // Let master shutdown\n        brokerController1.shutdown();\n        brokerList.remove(this.brokerController1);\n        Thread.sleep(6000);\n\n        // The slave should change to master\n        assertTrue(brokerController2.getReplicasManager().isMasterState());\n        assertEquals(brokerController2.getReplicasManager().getMasterEpoch(), 2);\n\n        // Restart old master, it should be slave\n        brokerController1 = startBroker(nameserverAddress, controllerAddress, brokerName, 1, nextPort(), listenPort, nettyPort, BrokerRole.SLAVE, DEFAULT_FILE_SIZE);\n        waitSlaveReady(brokerController1.getMessageStore());\n\n        assertFalse(brokerController1.getReplicasManager().isMasterState());\n        assertEquals(brokerController1.getReplicasManager().getMasterAddress(), brokerController2.getReplicasManager().getBrokerAddress());\n\n        // Put another batch messages\n        final MessageStore messageStore = brokerController2.getMessageStore();\n        putMessage(messageStore, topic);\n\n        // Check slave message\n        checkMessage(brokerController1.getMessageStore(), topic, 20, 0);\n        shutdownAndClearBroker();\n    }\n\n\n    @Test\n    public void testRestartWithChangedAddress() throws Exception {\n        String topic = \"Topic-\" + AutoSwitchRoleIntegrationTest.class.getSimpleName() + random.nextInt(65535);\n        String brokerName = \"Broker-\" + AutoSwitchRoleIntegrationTest.class.getSimpleName() + random.nextInt(65535);\n        int oldPort = nextPort();\n        this.brokerController1 = startBroker(nameserverAddress, controllerAddress, brokerName, 1, nextPort(), oldPort, oldPort, BrokerRole.SYNC_MASTER, DEFAULT_FILE_SIZE);\n        Thread.sleep(1000);\n        assertTrue(brokerController1.getReplicasManager().isMasterState());\n        assertEquals(brokerController1.getReplicasManager().getMasterEpoch(), 1);\n\n        // Let master shutdown\n        brokerController1.shutdown();\n        brokerList.remove(this.brokerController1);\n        Thread.sleep(6000);\n\n        // Restart with changed address\n        int newPort = nextPort();\n        this.brokerController1 = startBroker(nameserverAddress, controllerAddress, brokerName, 1, nextPort(), newPort, newPort, BrokerRole.SYNC_MASTER, DEFAULT_FILE_SIZE);\n        Thread.sleep(1000);\n\n        // Check broker id\n        assertEquals(1, brokerController1.getReplicasManager().getBrokerControllerId().longValue());\n        // Check role\n        assertTrue(brokerController1.getReplicasManager().isMasterState());\n\n        // check ip address\n        RemotingCommand remotingCommand = controllerManager.getController().getReplicaInfo(new GetReplicaInfoRequestHeader(brokerName)).get(500, TimeUnit.MILLISECONDS);\n        GetReplicaInfoResponseHeader resp = (GetReplicaInfoResponseHeader) remotingCommand.readCustomHeader();\n        assertEquals(1, resp.getMasterBrokerId().longValue());\n        assertTrue(resp.getMasterAddress().contains(String.valueOf(newPort)));\n        shutdownAndClearBroker();\n    }\n\n    @Test\n    public void testBasicWorkWhenControllerShutdown() throws Exception {\n        String topic = \"Foobar\";\n        String brokerName = \"Broker-\" + AutoSwitchRoleIntegrationTest.class.getSimpleName() + random.nextInt();\n        initBroker(DEFAULT_FILE_SIZE, brokerName);\n        // Put message from 0 to 9\n        putMessage(this.brokerController1.getMessageStore(), topic);\n        checkMessage(this.brokerController2.getMessageStore(), topic, 10, 0);\n\n        // Shutdown Controller\n        controllerManager.shutdown();\n\n        // Put message from 10 to 19\n        putMessage(this.brokerController1.getMessageStore(), topic);\n        checkMessage(this.brokerController2.getMessageStore(), topic, 20, 0);\n\n        initAndStartControllerManager();\n    }\n\n    @Test\n    public void testAddBroker() throws Exception {\n        String topic = \"Topic-\" + AutoSwitchRoleIntegrationTest.class.getSimpleName() + random.nextInt(65535);\n        String brokerName = \"Broker-\" + AutoSwitchRoleIntegrationTest.class.getSimpleName() + random.nextInt(65535);\n        initBroker(DEFAULT_FILE_SIZE, brokerName);\n        mockData(topic);\n\n        BrokerController broker3 = startBroker(nameserverAddress, controllerAddress, brokerName, 3, nextPort(), nextPort(), nextPort(), BrokerRole.SLAVE, DEFAULT_FILE_SIZE);\n        waitSlaveReady(broker3.getMessageStore());\n        checkMessage(broker3.getMessageStore(), topic, 10, 0);\n        putMessage(this.brokerController1.getMessageStore(), topic);\n        checkMessage(broker3.getMessageStore(), topic, 20, 0);\n        shutdownAndClearBroker();\n    }\n\n    @Test\n    public void testTruncateEpochLogAndChangeMaster() throws Exception {\n        shutdownAndClearBroker();\n        String topic = \"FooBar\";\n        String brokerName = \"Broker-\" + AutoSwitchRoleIntegrationTest.class.getSimpleName() + random.nextInt(65535);\n        // Noted that 10 msg 's total size = 1570, and if init the mappedFileSize = 1700, one file only be used to store 10 msg.\n        initBroker(1700, brokerName);\n        // Step1: Put message\n        putMessage(this.brokerController1.getMessageStore(), topic);\n        checkMessage(this.brokerController2.getMessageStore(), topic, 10, 0);\n\n        // Step2: shutdown broker1, broker2 as master\n        brokerController1.shutdown();\n        brokerList.remove(brokerController1);\n        Thread.sleep(5000);\n\n        assertTrue(brokerController2.getReplicasManager().isMasterState());\n        assertEquals(brokerController2.getReplicasManager().getMasterEpoch(), 2);\n\n        // Step3: add broker3\n        BrokerController broker3 = startBroker(nameserverAddress, controllerAddress, brokerName, 3, nextPort(), nextPort(), nextPort(), BrokerRole.SLAVE, 1700);\n        waitSlaveReady(broker3.getMessageStore());\n        checkMessage(broker3.getMessageStore(), topic, 10, 0);\n\n        // Step4: put another batch message\n        // Master: <Epoch1, 0, 1570> <Epoch2, 1570, 3270>\n        putMessage(this.brokerController2.getMessageStore(), topic);\n        checkMessage(broker3.getMessageStore(), topic, 20, 0);\n\n        // Step5: Check file position, each epoch will be stored on one file(Because fileSize = 1700, which equal to 10 msg size);\n        // So epoch1 was stored in firstFile, epoch2 was stored in second file, the lastFile was empty.\n        final MessageStore broker2MessageStore = this.brokerController2.getMessageStore();\n        final MappedFileQueue fileQueue = broker2MessageStore.getCommitLog().getMappedFileQueue();\n        assertEquals(2, fileQueue.getTotalFileSize() / 1700);\n\n        // Truncate epoch1's log (truncateEndOffset = 1570), which means we should delete the first file directly.\n        final MappedFile firstFile = broker2MessageStore.getCommitLog().getMappedFileQueue().getFirstMappedFile();\n        firstFile.shutdown(1000);\n        fileQueue.retryDeleteFirstFile(1000);\n        assertEquals(broker2MessageStore.getCommitLog().getMinOffset(), 1700);\n\n        final AutoSwitchHAService haService = (AutoSwitchHAService) this.brokerController2.getMessageStore().getHaService();\n        haService.truncateEpochFilePrefix(1570);\n        checkMessage(broker2MessageStore, topic, 10, 10);\n\n        // Step6, start broker4, link to broker2, it should sync msg from epoch2(offset = 1700).\n        BrokerController broker4 = startBroker(nameserverAddress, controllerAddress, brokerName, 4, nextPort(), nextPort(), nextPort(), BrokerRole.SLAVE, 1700);\n        waitSlaveReady(broker4.getMessageStore());\n        checkMessage(broker4.getMessageStore(), topic, 10, 10);\n        shutdownAndClearBroker();\n    }\n\n    public void shutdownAndClearBroker() throws InterruptedException {\n        for (BrokerController controller : brokerList) {\n            controller.shutdown();\n            UtilAll.deleteFile(new File(controller.getMessageStoreConfig().getStorePathRootDir()));\n        }\n        brokerList.clear();\n    }\n\n    @AfterClass\n    public static void destroy() {\n        if (namesrvController != null) {\n            namesrvController.shutdown();\n        }\n        if (controllerManager != null) {\n            controllerManager.shutdown();\n        }\n        File file = new File(STORE_PATH_ROOT_PARENT_DIR);\n        UtilAll.deleteFile(file);\n    }\n\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.base;\n\nimport com.google.common.collect.ImmutableList;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ForkJoinPool;\nimport java.util.concurrent.TimeUnit;\nimport java.util.function.Function;\nimport java.util.stream.Collectors;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.client.consumer.MQPullConsumer;\nimport org.apache.rocketmq.client.consumer.MQPushConsumer;\nimport org.apache.rocketmq.client.producer.MQProducer;\nimport org.apache.rocketmq.client.producer.TransactionListener;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.namesrv.NamesrvController;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.test.client.rmq.RMQAsyncSendProducer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.client.rmq.RMQTransactionalProducer;\nimport org.apache.rocketmq.test.clientinterface.AbstractMQConsumer;\nimport org.apache.rocketmq.test.clientinterface.AbstractMQProducer;\nimport org.apache.rocketmq.test.clientinterface.MQConsumer;\nimport org.apache.rocketmq.test.factory.ConsumerFactory;\nimport org.apache.rocketmq.test.listener.AbstractListener;\nimport org.apache.rocketmq.test.util.MQAdminTestUtils;\nimport org.apache.rocketmq.test.util.MQRandomUtils;\nimport org.apache.rocketmq.tools.admin.DefaultMQAdminExt;\nimport org.apache.rocketmq.tools.admin.MQAdminExt;\nimport org.junit.Assert;\n\nimport static org.apache.rocketmq.test.base.IntegrationTestBase.initMQAdmin;\nimport static org.awaitility.Awaitility.await;\n\npublic class BaseConf {\n\n    private final static Logger log = LoggerFactory.getLogger(BaseConf.class);\n\n    public final static String NAMESRV_ADDR;\n\n    //the logic queue test need at least three brokers\n    protected final static String CLUSTER_NAME;\n    protected final static String BROKER1_NAME;\n    protected final static String BROKER2_NAME;\n    protected final static String BROKER3_NAME;\n\n    protected final static int BROKER_NUM = 3;\n    protected final static int WAIT_TIME = 5;\n    protected final static int CONSUME_TIME = 2 * 60 * 1000;\n    protected final static int QUEUE_NUMBERS = 8;\n\n    protected static NamesrvController namesrvController;\n    protected static BrokerController brokerController1;\n    protected static BrokerController brokerController2;\n    protected static BrokerController brokerController3;\n    protected static List<BrokerController> brokerControllerList;\n    protected static Map<String, BrokerController> brokerControllerMap;\n\n    protected static List<Object> mqClients = new ArrayList<Object>();\n    protected static boolean debug = false;\n\n    static {\n        System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION));\n        namesrvController = IntegrationTestBase.createAndStartNamesrv();\n        NAMESRV_ADDR = \"127.0.0.1:\" + namesrvController.getNettyServerConfig().getListenPort();\n        log.debug(\"Name server started, listening: {}\", NAMESRV_ADDR);\n\n        brokerController1 = IntegrationTestBase.createAndStartBroker(NAMESRV_ADDR);\n        log.debug(\"Broker {} started, listening: {}\", brokerController1.getBrokerConfig().getBrokerName(),\n            brokerController1.getBrokerConfig().getListenPort());\n\n        brokerController2 = IntegrationTestBase.createAndStartBroker(NAMESRV_ADDR);\n        log.debug(\"Broker {} started, listening: {}\", brokerController2.getBrokerConfig().getBrokerName(),\n            brokerController2.getBrokerConfig().getListenPort());\n\n        brokerController3 = IntegrationTestBase.createAndStartBroker(NAMESRV_ADDR);\n        log.debug(\"Broker {} started, listening: {}\", brokerController3.getBrokerConfig().getBrokerName(),\n            brokerController3.getBrokerConfig().getListenPort());\n\n        CLUSTER_NAME = brokerController1.getBrokerConfig().getBrokerClusterName();\n        BROKER1_NAME = brokerController1.getBrokerConfig().getBrokerName();\n        BROKER2_NAME = brokerController2.getBrokerConfig().getBrokerName();\n        BROKER3_NAME = brokerController3.getBrokerConfig().getBrokerName();\n        brokerControllerList = ImmutableList.of(brokerController1, brokerController2, brokerController3);\n        brokerControllerMap = brokerControllerList.stream().collect(\n            Collectors.toMap(input -> input.getBrokerConfig().getBrokerName(), Function.identity()));\n        initMQAdmin(NAMESRV_ADDR);\n    }\n\n    public BaseConf() {\n        // Add waitBrokerRegistered to BaseConf constructor to make it default for all subclasses.\n        waitBrokerRegistered(NAMESRV_ADDR, CLUSTER_NAME, BROKER_NUM);\n    }\n\n    // This method can't be placed in the static block of BaseConf, which seems to lead to a strange dead lock.\n    public static void waitBrokerRegistered(final String nsAddr, final String clusterName, final int expectedBrokerNum) {\n        final DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt(500);\n        mqAdminExt.setNamesrvAddr(nsAddr);\n        try {\n            mqAdminExt.start();\n            await().atMost(30, TimeUnit.SECONDS).until(() -> {\n                List<BrokerData> brokerDatas;\n                try {\n                    brokerDatas = mqAdminExt.examineTopicRouteInfo(clusterName).getBrokerDatas();\n                } catch (Exception e) {\n                    return false;\n                }\n                return brokerDatas.size() == expectedBrokerNum;\n            });\n            for (BrokerController brokerController: brokerControllerList) {\n                brokerController.getBrokerOuterAPI().refreshMetadata();\n            }\n        } catch (Exception e) {\n            log.error(\"init failed, please check BaseConf\", e);\n            Assert.fail(e.getMessage());\n        }\n        ForkJoinPool.commonPool().execute(mqAdminExt::shutdown);\n    }\n\n    public boolean awaitDispatchMs(long timeMs) throws Exception {\n        long start = System.currentTimeMillis();\n        while (System.currentTimeMillis() - start <= timeMs) {\n            boolean allOk = true;\n            for (BrokerController brokerController: brokerControllerList) {\n                if (brokerController.getMessageStore().dispatchBehindBytes() != 0) {\n                    allOk = false;\n                    break;\n                }\n            }\n            if (allOk) {\n                return true;\n            }\n            Thread.sleep(100);\n        }\n        return false;\n    }\n\n\n    public static String initTopic() {\n        String topic = MQRandomUtils.getRandomTopic();\n        return initTopicWithName(topic);\n    }\n\n    public static String initTopic(TopicMessageType topicMessageType) {\n        String topic = MQRandomUtils.getRandomTopic();\n        return initTopicWithName(topic, topicMessageType);\n    }\n\n    public static String initTopicOnSampleTopicBroker(String sampleTopic) {\n        String topic = MQRandomUtils.getRandomTopic();\n        return initTopicOnSampleTopicBroker(topic, sampleTopic);\n    }\n\n    public static String initTopicOnSampleTopicBroker(String sampleTopic, TopicMessageType topicMessageType) {\n        String topic = MQRandomUtils.getRandomTopic();\n        return initTopicOnSampleTopicBroker(topic, sampleTopic, topicMessageType);\n    }\n\n    public static String initTopicWithName(String topicName) {\n        IntegrationTestBase.initTopic(topicName, NAMESRV_ADDR, CLUSTER_NAME, CQType.SimpleCQ);\n        return topicName;\n    }\n\n    public static String initTopicWithName(String topicName, TopicMessageType topicMessageType) {\n        IntegrationTestBase.initTopic(topicName, NAMESRV_ADDR, CLUSTER_NAME, topicMessageType);\n        return topicName;\n    }\n\n    public static String initTopicOnSampleTopicBroker(String topicName, String sampleTopic) {\n        IntegrationTestBase.initTopic(topicName, NAMESRV_ADDR, sampleTopic, CQType.SimpleCQ);\n        return topicName;\n    }\n\n    public static String initTopicOnSampleTopicBroker(String topicName, String sampleTopic, TopicMessageType topicMessageType) {\n        IntegrationTestBase.initTopic(topicName, NAMESRV_ADDR, sampleTopic, topicMessageType);\n        return topicName;\n    }\n\n    public static String initConsumerGroup() {\n        return initConsumerGroup(MQRandomUtils.getRandomConsumerGroup());\n    }\n\n    public static String initConsumerGroup(String group) {\n        SubscriptionGroupConfig config = new SubscriptionGroupConfig();\n        config.setGroupName(group);\n        MQAdminTestUtils.createSub(NAMESRV_ADDR, CLUSTER_NAME, config);\n        return group;\n    }\n\n    public static String initConsumerGroup(SubscriptionGroupConfig config) {\n        MQAdminTestUtils.createSub(NAMESRV_ADDR, CLUSTER_NAME, config);\n        return config.getGroupName();\n    }\n\n    public static DefaultMQAdminExt getAdmin(String nsAddr) {\n        final DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt(3 * 1000);\n        mqAdminExt.setNamesrvAddr(nsAddr);\n        mqAdminExt.setPollNameServerInterval(100);\n        mqClients.add(mqAdminExt);\n        return mqAdminExt;\n    }\n\n\n    public static RMQNormalProducer getProducer(String nsAddr, String topic) {\n        return getProducer(nsAddr, topic, false);\n    }\n\n    public static RMQNormalProducer getProducer(String nsAddr, String topic, boolean useTLS) {\n        RMQNormalProducer producer = new RMQNormalProducer(nsAddr, topic, useTLS);\n        if (debug) {\n            producer.setDebug();\n        }\n        mqClients.add(producer);\n        return producer;\n    }\n\n    public static RMQTransactionalProducer getTransactionalProducer(String nsAddr, String topic, TransactionListener transactionListener) {\n        RMQTransactionalProducer producer = new RMQTransactionalProducer(nsAddr, topic, false, transactionListener);\n        if (debug) {\n            producer.setDebug();\n        }\n        mqClients.add(producer);\n        return producer;\n    }\n\n    public static RMQNormalProducer getProducer(String nsAddr, String topic, String producerGoup,\n                                                String instanceName) {\n        RMQNormalProducer producer = new RMQNormalProducer(nsAddr, topic, producerGoup,\n                instanceName);\n        if (debug) {\n            producer.setDebug();\n        }\n        mqClients.add(producer);\n        return producer;\n    }\n\n    public static RMQAsyncSendProducer getAsyncProducer(String nsAddr, String topic) {\n        RMQAsyncSendProducer producer = new RMQAsyncSendProducer(nsAddr, topic);\n        if (debug) {\n            producer.setDebug();\n        }\n        mqClients.add(producer);\n        return producer;\n    }\n\n    public static RMQNormalConsumer getConsumer(String nsAddr, String topic, String subExpression,\n                                                AbstractListener listener) {\n        return getConsumer(nsAddr, topic, subExpression, listener, false);\n    }\n\n    public static RMQNormalConsumer getConsumer(String nsAddr, String topic, String subExpression,\n                                                AbstractListener listener, boolean useTLS) {\n        String consumerGroup = initConsumerGroup();\n        return getConsumer(nsAddr, consumerGroup, topic, subExpression, listener, useTLS);\n    }\n\n    public static RMQNormalConsumer getConsumer(String nsAddr, String consumerGroup, String topic,\n                                                String subExpression, AbstractListener listener) {\n        return getConsumer(nsAddr, consumerGroup, topic, subExpression, listener, false);\n    }\n\n    public static RMQNormalConsumer getConsumer(String nsAddr, String consumerGroup, String topic,\n                                                String subExpression, AbstractListener listener, boolean useTLS) {\n        RMQNormalConsumer consumer = ConsumerFactory.getRMQNormalConsumer(nsAddr, consumerGroup,\n                topic, subExpression, listener, useTLS);\n        if (debug) {\n            consumer.setDebug();\n        }\n        mqClients.add(consumer);\n        log.info(\"consumer[{}] start,topic[{}],subExpression[{}]\", consumerGroup, topic, subExpression);\n        return consumer;\n    }\n\n    public static void shutdown() {\n        ImmutableList<Object> mqClients = ImmutableList.copyOf(BaseConf.mqClients);\n        BaseConf.mqClients.clear();\n        shutdown(mqClients);\n    }\n\n    public static Set<String> getBrokers() {\n        Set<String> brokers = new HashSet<>();\n        brokers.add(BROKER1_NAME);\n        brokers.add(BROKER2_NAME);\n        brokers.add(BROKER3_NAME);\n        return brokers;\n    }\n\n    public static void shutdown(List<Object> mqClients) {\n        mqClients.forEach(mqClient -> ForkJoinPool.commonPool().execute(() -> {\n            if (mqClient instanceof AbstractMQProducer) {\n                ((AbstractMQProducer) mqClient).shutdown();\n            } else if (mqClient instanceof AbstractMQConsumer) {\n                ((AbstractMQConsumer) mqClient).shutdown();\n            } else if (mqClient instanceof MQAdminExt) {\n                ((MQAdminExt) mqClient).shutdown();\n            } else if (mqClient instanceof MQProducer) {\n                ((MQProducer) mqClient).shutdown();\n            } else if (mqClient instanceof MQPullConsumer) {\n                ((MQPullConsumer) mqClient).shutdown();\n            } else if (mqClient instanceof MQPushConsumer) {\n                ((MQPushConsumer) mqClient).shutdown();\n            } else if (mqClient instanceof MQConsumer) {\n                ((MQConsumer) mqClient).shutdown();\n            }\n        }));\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.base;\n\nimport com.google.common.truth.Truth;\nimport java.io.File;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.UUID;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport io.grpc.protobuf.services.ChannelzService;\nimport io.grpc.protobuf.services.ProtoReflectionService;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.TopicAttributes;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.namesrv.NamesrvConfig;\nimport org.apache.rocketmq.common.thread.ThreadPoolMonitor;\nimport org.apache.rocketmq.common.utils.AbstractStartAndShutdown;\nimport org.apache.rocketmq.common.utils.StartAndShutdown;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.namesrv.NamesrvController;\nimport org.apache.rocketmq.proxy.ProxyMode;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.config.ProxyConfig;\nimport org.apache.rocketmq.proxy.grpc.GrpcServer;\nimport org.apache.rocketmq.proxy.grpc.GrpcServerBuilder;\nimport org.apache.rocketmq.proxy.grpc.v2.GrpcMessagingApplication;\nimport org.apache.rocketmq.proxy.processor.DefaultMessagingProcessor;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.proxy.service.cert.TlsCertificateManager;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.test.util.MQAdminTestUtils;\n\nimport static org.apache.rocketmq.test.base.BaseConf.brokerController1;\n\npublic class IntegrationTestBase {\n    public static Logger logger = LoggerFactory.getLogger(IntegrationTestBase.class);\n\n    protected static final String SEP = File.separator;\n    protected static final String BROKER_NAME_PREFIX = \"TestBrokerName_\";\n    protected static final AtomicInteger BROKER_INDEX = new AtomicInteger(0);\n    protected static final List<File> TMPE_FILES = new ArrayList<>();\n    protected static final List<BrokerController> BROKER_CONTROLLERS = new ArrayList<>();\n    protected static final List<NamesrvController> NAMESRV_CONTROLLERS = new ArrayList<>();\n    protected static int topicCreateTime = (int) TimeUnit.SECONDS.toSeconds(30);\n    public static volatile int commitLogSize = 1024 * 1024 * 100;\n    protected static final int INDEX_NUM = 1000;\n\n    static {\n\n        System.setProperty(\"rocketmq.client.logRoot\", System.getProperty(\"java.io.tmpdir\"));\n\n        Runtime.getRuntime().addShutdownHook(new Thread() {\n            @Override\n            public void run() {\n                try {\n                    for (BrokerController brokerController : BROKER_CONTROLLERS) {\n                        if (brokerController != null) {\n                            brokerController.shutdown();\n                        }\n                    }\n\n                    // should destroy message store, otherwise could not delete the temp files.\n                    for (BrokerController brokerController : BROKER_CONTROLLERS) {\n                        if (brokerController != null) {\n                            brokerController.getMessageStore().destroy();\n                        }\n                    }\n\n                    for (NamesrvController namesrvController : NAMESRV_CONTROLLERS) {\n                        if (namesrvController != null) {\n                            namesrvController.shutdown();\n                        }\n                    }\n                    for (File file : TMPE_FILES) {\n                        UtilAll.deleteFile(file);\n                    }\n                    MQAdminTestUtils.shutdownAdmin();\n                } catch (Exception e) {\n                    logger.error(\"Shutdown error\", e);\n                }\n            }\n        });\n\n    }\n\n    public static String createBaseDir() {\n        String baseDir = System.getProperty(\"java.io.tmpdir\") + SEP + \"unitteststore-\" + UUID.randomUUID();\n        final File file = new File(baseDir);\n        if (file.exists()) {\n            logger.info(String.format(\"[%s] has already existed, please back up and remove it for integration tests\", baseDir));\n            System.exit(1);\n        }\n        TMPE_FILES.add(file);\n        return baseDir;\n    }\n\n    public static NamesrvController createAndStartNamesrv() {\n        String baseDir = createBaseDir();\n        NamesrvConfig namesrvConfig = new NamesrvConfig();\n        NettyServerConfig nameServerNettyServerConfig = new NettyServerConfig();\n        namesrvConfig.setKvConfigPath(baseDir + SEP + \"namesrv\" + SEP + \"kvConfig.json\");\n        namesrvConfig.setConfigStorePath(baseDir + SEP + \"namesrv\" + SEP + \"namesrv.properties\");\n\n        nameServerNettyServerConfig.setListenPort(0);\n        NamesrvController namesrvController = new NamesrvController(namesrvConfig, nameServerNettyServerConfig);\n        try {\n            Truth.assertThat(namesrvController.initialize()).isTrue();\n            logger.info(\"Name Server Start:{}\", nameServerNettyServerConfig.getListenPort());\n            namesrvController.start();\n        } catch (Exception e) {\n            logger.info(\"Name Server start failed\", e);\n            System.exit(1);\n        }\n        NAMESRV_CONTROLLERS.add(namesrvController);\n        return namesrvController;\n\n    }\n\n    public static BrokerController createAndStartBroker(String nsAddr) {\n        String baseDir = createBaseDir();\n        BrokerConfig brokerConfig = new BrokerConfig();\n        MessageStoreConfig storeConfig = new MessageStoreConfig();\n        brokerConfig.setBrokerName(BROKER_NAME_PREFIX + BROKER_INDEX.incrementAndGet());\n        brokerConfig.setBrokerIP1(\"127.0.0.1\");\n        brokerConfig.setNamesrvAddr(nsAddr);\n        brokerConfig.setEnablePropertyFilter(true);\n        brokerConfig.setEnableCalcFilterBitMap(true);\n        brokerConfig.setAppendAckAsync(true);\n        brokerConfig.setAppendCkAsync(true);\n        brokerConfig.setRecallMessageEnable(true);\n        storeConfig.setEnableConsumeQueueExt(true);\n        brokerConfig.setLoadBalancePollNameServerInterval(500);\n        brokerConfig.setPopConsumerKVServiceInit(true);\n        brokerConfig.setConfigManagerVersion(System.getProperty(\"configManagerVersion\", \"v1\"));\n        storeConfig.setStorePathRootDir(baseDir);\n        storeConfig.setStorePathCommitLog(baseDir + SEP + \"commitlog\");\n        storeConfig.setMappedFileSizeCommitLog(commitLogSize);\n        storeConfig.setMaxIndexNum(INDEX_NUM);\n        storeConfig.setMaxHashSlotNum(INDEX_NUM * 4);\n        storeConfig.setDeleteWhen(\"01;02;03;04;05;06;07;08;09;10;11;12;13;14;15;16;17;18;19;20;21;22;23;00\");\n        storeConfig.setMaxTransferCountOnMessageInMemory(1024);\n        storeConfig.setMaxTransferCountOnMessageInDisk(1024);\n        storeConfig.setEnableLmq(Boolean.valueOf(System.getProperty(\"enableLmq\", \"false\")));\n        storeConfig.setEnableMultiDispatch(Boolean.valueOf(System.getProperty(\"enableMultiDispatch\", \"false\")));\n        storeConfig.setStoreType(System.getProperty(\"storeType\", \"default\"));\n        return createAndStartBroker(storeConfig, brokerConfig);\n    }\n\n    public static void createAndStartProxy(String nsAddr) {\n        try {\n            ProxyStartAndShutdown startAndShutdown = new ProxyStartAndShutdown();\n            ConfigurationManager.initConfig();\n            ProxyConfig config = ConfigurationManager.getProxyConfig();\n            config.setNamesrvAddr(nsAddr);\n            config.setEnableTopicMessageTypeCheck(false);\n            ThreadPoolExecutor executor = ThreadPoolMonitor.createAndMonitor(\n                config.getGrpcThreadPoolNums(),\n                config.getGrpcThreadPoolNums(),\n                1, TimeUnit.MINUTES,\n                \"GrpcRequestExecutorThread\",\n                config.getGrpcThreadPoolQueueCapacity()\n            );\n            startAndShutdown.appendShutdown(executor::shutdown);\n\n            String proxyModeStr = ConfigurationManager.getProxyConfig().getProxyMode();\n            MessagingProcessor messagingProcessor;\n            if (ProxyMode.isClusterMode(proxyModeStr)) {\n                messagingProcessor = DefaultMessagingProcessor.createForClusterMode();\n            } else {\n                messagingProcessor = DefaultMessagingProcessor.createForLocalMode(brokerController1);\n            }\n            startAndShutdown.appendStartAndShutdown(messagingProcessor);\n\n            TlsCertificateManager tlsCertificateManager = new TlsCertificateManager();\n            startAndShutdown.appendStartAndShutdown(tlsCertificateManager);\n\n            GrpcMessagingApplication application = GrpcMessagingApplication.create(messagingProcessor);\n            startAndShutdown.appendStartAndShutdown(application);\n\n            GrpcServer grpcServer = GrpcServerBuilder.newBuilder(executor,\n                    ConfigurationManager.getProxyConfig().getGrpcServerPort(), tlsCertificateManager)\n                .addService(application)\n                .addService(ChannelzService.newInstance(100))\n                .addService(ProtoReflectionService.newInstance())\n                .configInterceptor()\n                .shutdownTime(ConfigurationManager.getProxyConfig().getGrpcShutdownTimeSeconds(), TimeUnit.SECONDS)\n                .build();\n            startAndShutdown.appendStartAndShutdown(grpcServer);\n\n            startAndShutdown.start();\n            Runtime.getRuntime().addShutdownHook(new Thread(() -> {\n                try {\n                    startAndShutdown.preShutdown();\n                    startAndShutdown.shutdown();\n                } catch (Exception e) {\n                }\n            }));\n        } catch (Throwable e) {\n            logger.error(\"proxy start failed, will exit\", e);\n            System.exit(1);\n        }\n    }\n\n    public static BrokerController createAndStartBroker(MessageStoreConfig storeConfig, BrokerConfig brokerConfig) {\n        NettyServerConfig nettyServerConfig = new NettyServerConfig();\n        NettyClientConfig nettyClientConfig = new NettyClientConfig();\n        nettyServerConfig.setListenPort(0);\n        storeConfig.setHaListenPort(0);\n        BrokerController brokerController = new BrokerController(brokerConfig, nettyServerConfig, nettyClientConfig, storeConfig);\n        try {\n            Truth.assertThat(brokerController.initialize()).isTrue();\n            logger.info(\"Broker Start name:{} addr:{}\", brokerConfig.getBrokerName(), brokerController.getBrokerAddr());\n            brokerController.start();\n        } catch (Throwable t) {\n            logger.error(\"Broker start failed, will exit\", t);\n            System.exit(1);\n        }\n        BROKER_CONTROLLERS.add(brokerController);\n        return brokerController;\n    }\n\n    public static boolean initTopic(String topic, String nsAddr, String clusterName, int queueNumbers, CQType cqType) {\n        return initTopic(topic, nsAddr, clusterName, queueNumbers, cqType, TopicMessageType.NORMAL, null);\n    }\n\n    public static boolean initTopic(String topic, String nsAddr, String clusterName, int queueNumbers,\n        CQType cqType, TopicMessageType topicMessageType) {\n        return initTopic(topic, nsAddr, clusterName, queueNumbers, cqType, topicMessageType, null);\n    }\n\n    public static boolean initTopic(String topic, String nsAddr, String clusterName, int queueNumbers,\n        CQType cqType, TopicMessageType topicMessageType, Long liteTtl) {\n        boolean createResult;\n        Map<String, String> attributes = new HashMap<>();\n        if (!Objects.equals(CQType.SimpleCQ, cqType)) {\n            attributes.put(\"+\" + TopicAttributes.QUEUE_TYPE_ATTRIBUTE.getName(), cqType.toString());\n        }\n        if (!Objects.equals(TopicMessageType.NORMAL, topicMessageType)) {\n            attributes.put(\"+\" + TopicAttributes.TOPIC_MESSAGE_TYPE_ATTRIBUTE.getName(), topicMessageType.toString());\n        }\n        if (Objects.equals(TopicMessageType.LITE, topicMessageType)) {\n            attributes.put(\"+\" + TopicAttributes.LITE_EXPIRATION_ATTRIBUTE.getName(), liteTtl.toString());\n        }\n        createResult = MQAdminTestUtils.createTopic(nsAddr, clusterName, topic, queueNumbers, attributes, topicCreateTime);\n        return createResult;\n    }\n\n    public static boolean initTopic(String topic, String nsAddr, String clusterName, CQType cqType) {\n        return initTopic(topic, nsAddr, clusterName, BaseConf.QUEUE_NUMBERS, cqType, TopicMessageType.NORMAL, null);\n    }\n\n    public static boolean initTopic(String topic, String nsAddr, String clusterName,\n        TopicMessageType topicMessageType) {\n        return initTopic(topic, nsAddr, clusterName, BaseConf.QUEUE_NUMBERS, CQType.SimpleCQ, topicMessageType, null);\n    }\n\n    public static void deleteFile(File file) {\n        if (!file.exists()) {\n            return;\n        }\n        UtilAll.deleteFile(file);\n    }\n\n    public static void initMQAdmin(String nsAddr) {\n        try {\n            MQAdminTestUtils.startAdmin(nsAddr);\n        } catch (MQClientException e) {\n            logger.info(\"MQAdmin start failed\");\n            System.exit(1);\n        }\n    }\n\n    private static class ProxyStartAndShutdown extends AbstractStartAndShutdown {\n        @Override\n        public void appendStartAndShutdown(StartAndShutdown startAndShutdown) {\n            super.appendStartAndShutdown(startAndShutdown);\n        }\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgDynamicBalanceIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.balance;\n\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.MQWait;\nimport org.apache.rocketmq.test.util.TestUtils;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class NormalMsgDynamicBalanceIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(NormalMsgStaticBalanceIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s !\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testTwoConsumerAndCrashOne() {\n        int msgSize = 400;\n        RMQNormalConsumer consumer1 = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n        RMQNormalConsumer consumer2 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic,\n            \"*\", new RMQNormalListener());\n        TestUtils.waitForSeconds(WAIT_TIME);\n\n        producer.send(msgSize);\n\n        MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(), consumer1.getListener(),\n            consumer2.getListener());\n        consumer2.shutdown();\n\n        producer.send(msgSize);\n        Assert.assertEquals(\"Not all are sent\", msgSize * 2, producer.getAllUndupMsgBody().size());\n\n        boolean recvAll = MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(),\n            consumer1.getListener(), consumer2.getListener());\n        assertThat(recvAll).isEqualTo(true);\n\n        boolean balance = VerifyUtils.verifyBalance(msgSize,\n            VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n                consumer1.getListener().getAllUndupMsgBody()).size() - msgSize,\n            VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n                consumer2.getListener().getAllUndupMsgBody()).size());\n        assertThat(balance).isEqualTo(true);\n    }\n\n    @Test\n    public void test3ConsumerAndCrashOne() {\n        int msgSize = 400;\n        RMQNormalConsumer consumer1 = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n        RMQNormalConsumer consumer2 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic,\n            \"*\", new RMQNormalListener());\n        RMQNormalConsumer consumer3 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic,\n            \"*\", new RMQNormalListener());\n        TestUtils.waitForSeconds(WAIT_TIME);\n\n        producer.send(msgSize);\n\n        MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(), consumer1.getListener(),\n            consumer2.getListener(), consumer3.getListener());\n        consumer3.shutdown();\n        TestUtils.waitForSeconds(WAIT_TIME);\n\n        producer.clearMsg();\n        consumer1.clearMsg();\n        consumer2.clearMsg();\n\n        producer.send(msgSize);\n        Assert.assertEquals(\"Not all are sent\", msgSize, producer.getAllUndupMsgBody().size());\n\n        boolean recvAll = MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(),\n            consumer1.getListener(), consumer2.getListener());\n        assertThat(recvAll).isEqualTo(true);\n\n        boolean balance = VerifyUtils.verifyBalance(msgSize,\n            VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n                consumer1.getListener().getAllUndupMsgBody()).size(),\n            VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n                consumer2.getListener().getAllUndupMsgBody()).size());\n        assertThat(balance).isEqualTo(true);\n    }\n\n    @Test\n    public void testMessageQueueListener() throws InterruptedException {\n        final CountDownLatch latch = new CountDownLatch(1);\n\n        RMQNormalConsumer consumer1 = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n        // Register message queue listener\n        consumer1.getConsumer().setMessageQueueListener((topic, mqAll, mqAssigned) -> latch.countDown());\n\n        // Without message queue listener\n        RMQNormalConsumer consumer2 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic,\n            \"*\", new RMQNormalListener());\n\n        Assert.assertTrue(latch.await(30, TimeUnit.SECONDS));\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgStaticBalanceIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.balance;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.MQWait;\nimport org.apache.rocketmq.test.util.TestUtils;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class NormalMsgStaticBalanceIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(NormalMsgStaticBalanceIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s !\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testTwoConsumersBalance() {\n        int msgSize = 400;\n        RMQNormalConsumer consumer1 = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n        RMQNormalConsumer consumer2 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic,\n            \"*\", new RMQNormalListener());\n        TestUtils.waitForSeconds(WAIT_TIME);\n\n        producer.send(msgSize);\n        Assert.assertEquals(\"Not all are sent\", msgSize, producer.getAllUndupMsgBody().size());\n\n        boolean recvAll = MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(),\n            consumer1.getListener(), consumer2.getListener());\n        assertThat(recvAll).isEqualTo(true);\n\n        boolean balance = VerifyUtils.verifyBalance(msgSize,\n            VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n                consumer1.getListener().getAllUndupMsgBody()).size(),\n            VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n                consumer2.getListener().getAllUndupMsgBody()).size());\n        assertThat(balance).isEqualTo(true);\n    }\n\n    @Test\n    public void testFourConsumersBalance() {\n        int msgSize = 600;\n        String consumerGroup = initConsumerGroup();\n        logger.info(\"use group: {}\", consumerGroup);\n        RMQNormalConsumer consumer1 = getConsumer(NAMESRV_ADDR, consumerGroup, topic, \"*\", new RMQNormalListener());\n        RMQNormalConsumer consumer2 = getConsumer(NAMESRV_ADDR, consumerGroup, topic, \"*\", new RMQNormalListener());\n        RMQNormalConsumer consumer3 = getConsumer(NAMESRV_ADDR, consumerGroup, topic, \"*\", new RMQNormalListener());\n        RMQNormalConsumer consumer4 = getConsumer(NAMESRV_ADDR, consumerGroup, topic, \"*\", new RMQNormalListener());\n        TestUtils.waitForSeconds(WAIT_TIME);\n\n        producer.send(msgSize);\n        Assert.assertEquals(\"Not all are sent\", msgSize, producer.getAllUndupMsgBody().size());\n\n        boolean recvAll = MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(),\n            consumer1.getListener(), consumer2.getListener(), consumer3.getListener(),\n            consumer4.getListener());\n        assertThat(recvAll).isEqualTo(true);\n\n        boolean balance = VerifyUtils\n            .verifyBalance(msgSize,\n                VerifyUtils\n                    .getFilterdMessage(producer.getAllMsgBody(),\n                        consumer1.getListener().getAllUndupMsgBody())\n                    .size(),\n                VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n                    consumer2.getListener().getAllUndupMsgBody()).size(),\n                VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n                    consumer3.getListener().getAllUndupMsgBody()).size(),\n                VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n                    consumer4.getListener().getAllUndupMsgBody()).size());\n        assertThat(balance).isEqualTo(true);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/BaseBroadcast.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.broadcast;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;\nimport org.apache.rocketmq.test.factory.ConsumerFactory;\nimport org.apache.rocketmq.test.listener.AbstractListener;\n\npublic class BaseBroadcast extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(BaseBroadcast.class);\n\n    public static RMQBroadCastConsumer getBroadCastConsumer(String nsAddr, String topic,\n        String subExpression,\n        AbstractListener listener) {\n        String consumerGroup = initConsumerGroup();\n        return getBroadCastConsumer(nsAddr, consumerGroup, topic, subExpression, listener);\n    }\n\n    public static RMQBroadCastConsumer getBroadCastConsumer(String nsAddr, String consumerGroup,\n        String topic, String subExpression,\n        AbstractListener listener) {\n        RMQBroadCastConsumer consumer = ConsumerFactory.getRMQBroadCastConsumer(nsAddr,\n            consumerGroup, topic, subExpression, listener);\n\n        consumer.setDebug();\n\n        mqClients.add(consumer);\n        logger.info(String.format(\"consumer[%s] start,topic[%s],subExpression[%s]\", consumerGroup,\n            topic, subExpression));\n        return consumer;\n    }\n\n    public void printSeparator() {\n        for (int i = 0; i < 3; i++) {\n            logger.info(\n                \"<<<<<<<<================================================================================>>>>>>>>\");\n        }\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/normal/BroadcastNormalMsgNotReceiveIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.broadcast.normal;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadcast;\nimport org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class BroadcastNormalMsgNotReceiveIT extends BaseBroadcast {\n    private static Logger logger = LoggerFactory\n        .getLogger(NormalMsgTwoSameGroupConsumerIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        printSeparator();\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s;\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testNotConsumeAfterConsume() throws Exception {\n        int msgSize = 16;\n\n        String group = initConsumerGroup();\n        RMQBroadCastConsumer consumer1 = getBroadCastConsumer(NAMESRV_ADDR, group, topic, \"*\",\n            new RMQNormalListener(group + \"_1\"));\n        Thread.sleep(3000);\n        producer.send(msgSize);\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize, producer.getAllUndupMsgBody().size());\n\n        consumer1.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer1.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n\n        RMQBroadCastConsumer consumer2 = getBroadCastConsumer(NAMESRV_ADDR,\n            consumer1.getConsumerGroup(), topic, \"*\", new RMQNormalListener(group + \"_2\"));\n        consumer2.getListener().waitForMessageConsume(producer.getAllMsgBody(), WAIT_TIME);\n        assertThat(consumer2.getListener().getAllMsgBody().size()).isEqualTo(0);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/normal/BroadcastNormalMsgRecvCrashIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.broadcast.normal;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadcast;\nimport org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.TestUtils;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class BroadcastNormalMsgRecvCrashIT extends BaseBroadcast {\n    private static Logger logger = LoggerFactory\n        .getLogger(NormalMsgTwoSameGroupConsumerIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        printSeparator();\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s;\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testStartTwoAndCrashOneLater() {\n        int msgSize = 16;\n\n        String group = initConsumerGroup();\n        RMQBroadCastConsumer consumer1 = getBroadCastConsumer(NAMESRV_ADDR, group, topic, \"*\",\n            new RMQNormalListener(group + \"_1\"));\n        RMQBroadCastConsumer consumer2 = getBroadCastConsumer(NAMESRV_ADDR,\n            consumer1.getConsumerGroup(), topic, \"*\", new RMQNormalListener(group + \"_2\"));\n        TestUtils.waitForSeconds(WAIT_TIME);\n\n        producer.send(msgSize);\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize, producer.getAllUndupMsgBody().size());\n\n        consumer1.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        consumer2.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer1.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer2.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n\n        consumer2.shutdown();\n\n        producer.clearMsg();\n        consumer1.clearMsg();\n\n        producer.send(msgSize);\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize, producer.getAllUndupMsgBody().size());\n\n        consumer1.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer1.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/normal/BroadcastNormalMsgRecvFailIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.broadcast.normal;\n\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadcast;\nimport org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Ignore;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class BroadcastNormalMsgRecvFailIT extends BaseBroadcast {\n    private static Logger logger = LoggerFactory\n        .getLogger(NormalMsgTwoSameGroupConsumerIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        printSeparator();\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s;\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Ignore\n    @Test\n    public void testStartTwoConsumerAndOneConsumerFail() {\n        int msgSize = 16;\n\n        RMQBroadCastConsumer consumer1 = getBroadCastConsumer(NAMESRV_ADDR, topic, \"*\",\n            new RMQNormalListener());\n        RMQBroadCastConsumer consumer2 = getBroadCastConsumer(NAMESRV_ADDR,\n            consumer1.getConsumerGroup(), topic, \"*\",\n            new RMQNormalListener(ConsumeConcurrentlyStatus.RECONSUME_LATER));\n\n        producer.send(msgSize);\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize, producer.getAllUndupMsgBody().size());\n\n        consumer1.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer1.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/normal/BroadcastNormalMsgRecvStartLaterIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.broadcast.normal;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadcast;\nimport org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.TestUtils;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class BroadcastNormalMsgRecvStartLaterIT extends BaseBroadcast {\n    private static Logger logger = LoggerFactory\n        .getLogger(NormalMsgTwoSameGroupConsumerIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        printSeparator();\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s;\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testStartOneAndStartAnotherLater() {\n        int msgSize = 16;\n\n        String group = initConsumerGroup();\n        RMQBroadCastConsumer consumer1 = getBroadCastConsumer(NAMESRV_ADDR, group, topic, \"*\",\n            new RMQNormalListener(group + \"_1\"));\n        TestUtils.waitForSeconds(WAIT_TIME);\n\n        producer.send(msgSize);\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize, producer.getAllUndupMsgBody().size());\n\n        consumer1.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer1.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n\n        producer.clearMsg();\n        consumer1.clearMsg();\n\n        RMQBroadCastConsumer consumer2 = getBroadCastConsumer(NAMESRV_ADDR,\n            consumer1.getConsumerGroup(), topic, \"*\", new RMQNormalListener(group + \"_2\"));\n        TestUtils.waitForSeconds(WAIT_TIME);\n        producer.send(msgSize);\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize, producer.getAllUndupMsgBody().size());\n\n        consumer1.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        consumer2.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer1.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer2.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/normal/BroadcastNormalMsgTwoDiffGroupRecvIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.broadcast.normal;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadcast;\nimport org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.TestUtils;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class BroadcastNormalMsgTwoDiffGroupRecvIT extends BaseBroadcast {\n    private static Logger logger = LoggerFactory\n        .getLogger(NormalMsgTwoSameGroupConsumerIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        printSeparator();\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s;\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testStartDiffSameGroupConsumer() {\n        int msgSize = 16;\n\n        String group1 = initConsumerGroup();\n        String group2 = initConsumerGroup();\n        RMQBroadCastConsumer consumer1 = getBroadCastConsumer(NAMESRV_ADDR, group1, topic, \"*\",\n            new RMQNormalListener(group1 + \"_1\"));\n        RMQBroadCastConsumer consumer2 = getBroadCastConsumer(NAMESRV_ADDR, group2, topic, \"*\",\n            new RMQNormalListener(group2 + \"_2\"));\n        TestUtils.waitForSeconds(WAIT_TIME);\n\n        producer.send(msgSize);\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize, producer.getAllUndupMsgBody().size());\n\n        consumer1.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        consumer2.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer1.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer2.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/normal/NormalMsgTwoSameGroupConsumerIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.broadcast.normal;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadcast;\nimport org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.TestUtils;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class NormalMsgTwoSameGroupConsumerIT extends BaseBroadcast {\n    private static Logger logger = LoggerFactory\n        .getLogger(NormalMsgTwoSameGroupConsumerIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        printSeparator();\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s;\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testStartTwoSameGroupConsumer() {\n        int msgSize = 16;\n\n        String group = initConsumerGroup();\n        RMQBroadCastConsumer consumer1 = getBroadCastConsumer(NAMESRV_ADDR, group, topic, \"*\",\n            new RMQNormalListener(group + \"_1\"));\n        RMQBroadCastConsumer consumer2 = getBroadCastConsumer(NAMESRV_ADDR,\n            consumer1.getConsumerGroup(), topic, \"*\", new RMQNormalListener(group + \"_2\"));\n        TestUtils.waitForSeconds(WAIT_TIME);\n\n        producer.send(msgSize);\n        Assert.assertEquals(\"Not all are sent\", msgSize, producer.getAllUndupMsgBody().size());\n\n        consumer1.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        consumer2.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer1.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer2.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/order/OrderMsgBroadcastIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.broadcast.order;\n\nimport java.util.List;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadcast;\nimport org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.order.RMQOrderListener;\nimport org.apache.rocketmq.test.message.MessageQueueMsg;\nimport org.apache.rocketmq.test.util.TestUtils;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Ignore;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\n/**\n * Currently, does not support the ordered broadcast message\n */\n@Ignore\npublic class OrderMsgBroadcastIT extends BaseBroadcast {\n    private static Logger logger = LoggerFactory.getLogger(OrderMsgBroadcastIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    private int broadcastConsumeTime = 1 * 60 * 1000;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s;\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testTwoConsumerSubTag() {\n        int msgSize = 10;\n\n        RMQBroadCastConsumer consumer1 = getBroadCastConsumer(NAMESRV_ADDR, topic, \"*\",\n            new RMQOrderListener());\n        RMQBroadCastConsumer consumer2 = getBroadCastConsumer(NAMESRV_ADDR,\n            consumer1.getConsumerGroup(), topic, \"*\", new RMQOrderListener());\n        TestUtils.waitForSeconds(WAIT_TIME);\n\n        List<MessageQueue> mqs = producer.getMessageQueue();\n        MessageQueueMsg mqMsgs = new MessageQueueMsg(mqs, msgSize);\n        producer.send(mqMsgs.getMsgsWithMQ());\n        consumer1.getListener().waitForMessageConsume(producer.getAllMsgBody(), broadcastConsumeTime);\n        consumer2.getListener().waitForMessageConsume(producer.getAllMsgBody(), broadcastConsumeTime);\n\n        assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer1.getListener()).getMsgs()))\n            .isEqualTo(true);\n        assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer2.getListener()).getMsgs()))\n            .isEqualTo(true);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/tag/BroadcastTwoConsumerFilterIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.broadcast.tag;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadcast;\nimport org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.TestUtils;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class BroadcastTwoConsumerFilterIT extends BaseBroadcast {\n    private static Logger logger = LoggerFactory.getLogger(BroadcastTwoConsumerSubTagIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s;\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testTwoConsumerFilter() {\n        int msgSize = 40;\n        String tag1 = \"jueyin_tag_1\";\n        String tag2 = \"jueyin_tag_2\";\n\n        RMQBroadCastConsumer consumer1 = getBroadCastConsumer(NAMESRV_ADDR, topic, tag1,\n            new RMQNormalListener());\n        RMQBroadCastConsumer consumer2 = getBroadCastConsumer(NAMESRV_ADDR,\n            consumer1.getConsumerGroup(), topic, tag1, new RMQNormalListener());\n        TestUtils.waitForSeconds(WAIT_TIME);\n\n        producer.send(tag2, msgSize);\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize, producer.getAllUndupMsgBody().size());\n        producer.clearMsg();\n        producer.send(tag1, msgSize);\n\n        consumer1.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        consumer2.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer1.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer2.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/tag/BroadcastTwoConsumerSubDiffTagIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.broadcast.tag;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadcast;\nimport org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.TestUtils;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class BroadcastTwoConsumerSubDiffTagIT extends BaseBroadcast {\n    private static Logger logger = LoggerFactory.getLogger(BroadcastTwoConsumerSubTagIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s;\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testTwoConsumerSubDiffTag() {\n        int msgSize = 40;\n        String tag = \"jueyin_tag\";\n\n        RMQBroadCastConsumer consumer1 = getBroadCastConsumer(NAMESRV_ADDR, topic, \"*\",\n            new RMQNormalListener());\n        RMQBroadCastConsumer consumer2 = getBroadCastConsumer(NAMESRV_ADDR,\n            consumer1.getConsumerGroup(), topic, tag, new RMQNormalListener());\n        TestUtils.waitForSeconds(WAIT_TIME);\n\n        producer.send(tag, msgSize);\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize, producer.getAllUndupMsgBody().size());\n\n        consumer1.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        consumer2.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer1.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer2.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/tag/BroadcastTwoConsumerSubTagIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.broadcast.tag;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.client.consumer.broadcast.BaseBroadcast;\nimport org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.TestUtils;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class BroadcastTwoConsumerSubTagIT extends BaseBroadcast {\n    private static Logger logger = LoggerFactory.getLogger(BroadcastTwoConsumerSubTagIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s;\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testTwoConsumerSubTag() {\n        int msgSize = 20;\n        String tag = \"jueyin_tag\";\n\n        RMQBroadCastConsumer consumer1 = getBroadCastConsumer(NAMESRV_ADDR, topic, tag,\n            new RMQNormalListener());\n        RMQBroadCastConsumer consumer2 = getBroadCastConsumer(NAMESRV_ADDR,\n            consumer1.getConsumerGroup(), topic, tag, new RMQNormalListener());\n        TestUtils.waitForSeconds(WAIT_TIME);\n\n        producer.send(tag, msgSize);\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize, producer.getAllUndupMsgBody().size());\n\n        consumer1.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        consumer2.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer1.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer2.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/cluster/DynamicAddAndCrashIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.cluster;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.consumer.balance.NormalMsgStaticBalanceIT;\nimport org.apache.rocketmq.test.client.mq.MQAsyncProducer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.MQWait;\nimport org.apache.rocketmq.test.util.TestUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class DynamicAddAndCrashIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(NormalMsgStaticBalanceIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s !\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testAddOneConsumerAndCrashAfterWhile() {\n        int msgSize = 150;\n        RMQNormalConsumer consumer1 = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n\n        MQAsyncProducer asyncDefaultMQProducer = new MQAsyncProducer(producer, msgSize, 100);\n        asyncDefaultMQProducer.start();\n        TestUtils.waitForSeconds(WAIT_TIME);\n\n        RMQNormalConsumer consumer2 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic,\n            \"*\", new RMQNormalListener());\n        TestUtils.waitForSeconds(WAIT_TIME);\n        consumer2.shutdown();\n\n        asyncDefaultMQProducer.waitSendAll(WAIT_TIME * 6);\n\n        MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(), consumer1.getListener(),\n            consumer2.getListener());\n\n        boolean recvAll = MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(),\n            consumer1.getListener(), consumer2.getListener());\n        assertThat(recvAll).isEqualTo(true);\n    }\n\n    @Test\n    public void testAddTwoConsumerAndCrashAfterWhile() {\n        int msgSize = 150;\n        RMQNormalConsumer consumer1 = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n\n        MQAsyncProducer asyncDefaultMQProducer = new MQAsyncProducer(producer, msgSize, 100);\n        asyncDefaultMQProducer.start();\n        TestUtils.waitForSeconds(WAIT_TIME);\n\n        RMQNormalConsumer consumer2 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic,\n            \"*\", new RMQNormalListener());\n        RMQNormalConsumer consumer3 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic,\n            \"*\", new RMQNormalListener());\n        TestUtils.waitForSeconds(WAIT_TIME);\n\n        consumer2.shutdown();\n        consumer3.shutdown();\n\n        asyncDefaultMQProducer.waitSendAll(WAIT_TIME * 6);\n\n        MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(), consumer1.getListener(),\n            consumer2.getListener(), consumer3.getListener());\n\n        boolean recvAll = MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(),\n            consumer1.getListener(), consumer2.getListener(), consumer3.getListener());\n        assertThat(recvAll).isEqualTo(true);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/cluster/DynamicAddConsumerIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.cluster;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.consumer.balance.NormalMsgStaticBalanceIT;\nimport org.apache.rocketmq.test.client.mq.MQAsyncProducer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.MQWait;\nimport org.apache.rocketmq.test.util.TestUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class DynamicAddConsumerIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(NormalMsgStaticBalanceIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s !\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testAddOneConsumer() {\n        int msgSize = 100;\n        RMQNormalConsumer consumer1 = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n\n        MQAsyncProducer asyncDefaultMQProducer = new MQAsyncProducer(producer, msgSize, 100);\n        asyncDefaultMQProducer.start();\n        TestUtils.waitForSeconds(WAIT_TIME);\n\n        RMQNormalConsumer consumer2 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic,\n            \"*\", new RMQNormalListener());\n\n        asyncDefaultMQProducer.waitSendAll(WAIT_TIME * 6);\n\n        MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(), consumer1.getListener(),\n            consumer2.getListener());\n\n        boolean recvAll = MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(),\n            consumer1.getListener(), consumer2.getListener());\n        assertThat(recvAll).isEqualTo(true);\n    }\n\n    @Test\n    public void testAddTwoConsumer() {\n        int msgSize = 100;\n        RMQNormalConsumer consumer1 = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n\n        MQAsyncProducer asyncDefaultMQProducer = new MQAsyncProducer(producer, msgSize, 100);\n        asyncDefaultMQProducer.start();\n        TestUtils.waitForSeconds(WAIT_TIME);\n\n        RMQNormalConsumer consumer2 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic,\n            \"*\", new RMQNormalListener());\n        RMQNormalConsumer consumer3 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic,\n            \"*\", new RMQNormalListener());\n\n        asyncDefaultMQProducer.waitSendAll(WAIT_TIME * 6);\n\n        MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(), consumer1.getListener(),\n            consumer2.getListener(), consumer3.getListener());\n\n        boolean recvAll = MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(),\n            consumer1.getListener(), consumer2.getListener(), consumer3.getListener());\n        assertThat(recvAll).isEqualTo(true);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/cluster/DynamicCrashConsumerIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.cluster;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.consumer.balance.NormalMsgStaticBalanceIT;\nimport org.apache.rocketmq.test.client.mq.MQAsyncProducer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.MQWait;\nimport org.apache.rocketmq.test.util.TestUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class DynamicCrashConsumerIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(NormalMsgStaticBalanceIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s !\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testAddOneConsumer() {\n        int msgSize = 100;\n        RMQNormalConsumer consumer1 = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n        RMQNormalConsumer consumer2 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic,\n            \"*\", new RMQNormalListener());\n\n        MQAsyncProducer asyncDefaultMQProducer = new MQAsyncProducer(producer, msgSize, 100);\n        asyncDefaultMQProducer.start();\n        TestUtils.waitForSeconds(WAIT_TIME);\n\n        consumer2.shutdown();\n\n        asyncDefaultMQProducer.waitSendAll(WAIT_TIME * 6);\n\n        MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(), consumer1.getListener(),\n            consumer2.getListener());\n\n        boolean recvAll = MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(),\n            consumer1.getListener(), consumer2.getListener());\n        assertThat(recvAll).isEqualTo(true);\n    }\n\n    @Test\n    public void testAddTwoConsumer() {\n        int msgSize = 100;\n        RMQNormalConsumer consumer1 = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n        RMQNormalConsumer consumer2 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic,\n            \"*\", new RMQNormalListener());\n        RMQNormalConsumer consumer3 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic,\n            \"*\", new RMQNormalListener());\n\n        MQAsyncProducer asyncDefaultMQProducer = new MQAsyncProducer(producer, msgSize, 100);\n        asyncDefaultMQProducer.start();\n        TestUtils.waitForSeconds(WAIT_TIME);\n\n        consumer2.shutdown();\n        consumer3.shutdown();\n\n        asyncDefaultMQProducer.waitSendAll(WAIT_TIME * 6);\n\n        MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(), consumer1.getListener(),\n            consumer2.getListener(), consumer3.getListener());\n\n        boolean recvAll = MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(),\n            consumer1.getListener(), consumer2.getListener(), consumer3.getListener());\n        assertThat(recvAll).isEqualTo(true);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/filter/SqlFilterIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.filter;\n\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;\nimport org.apache.rocketmq.client.consumer.MessageSelector;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.client.rmq.RMQSqlConsumer;\nimport org.apache.rocketmq.test.factory.ConsumerFactory;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class SqlFilterIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(SqlFilterIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n    private static final Map<MessageQueue, Long> OFFSE_TABLE = new HashMap<MessageQueue, Long>();\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s;\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n        OFFSE_TABLE.clear();\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testFilterConsumer() throws Exception {\n        int msgSize = 16;\n\n        String group = initConsumerGroup();\n        MessageSelector selector = MessageSelector.bySql(\"(TAGS is not null and TAGS in ('TagA', 'TagB'))\");\n        RMQSqlConsumer consumer = ConsumerFactory.getRMQSqlConsumer(NAMESRV_ADDR, group, topic, selector, new RMQNormalListener(group + \"_1\"));\n        Thread.sleep(3000);\n        producer.send(\"TagA\", msgSize);\n        producer.send(\"TagB\", msgSize);\n        producer.send(\"TagC\", msgSize);\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize * 3, producer.getAllUndupMsgBody().size());\n        consumer.getListener().waitForMessageConsume(msgSize * 2, CONSUME_TIME);\n        assertThat(producer.getAllMsgBody())\n            .containsAllIn(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n                consumer.getListener().getAllMsgBody()));\n\n        assertThat(consumer.getListener().getAllMsgBody().size()).isEqualTo(msgSize * 2);\n    }\n\n    @Test\n    public void testFilterPullConsumer() throws Exception {\n        int msgSize = 16;\n\n        String group = initConsumerGroup();\n        MessageSelector selector = MessageSelector.bySql(\"(TAGS is not null and TAGS in ('TagA', 'TagB'))\");\n        DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(group);\n        consumer.setNamesrvAddr(NAMESRV_ADDR);\n        consumer.start();\n        Thread.sleep(3000);\n        producer.send(\"TagA\", msgSize);\n        producer.send(\"TagB\", msgSize);\n        producer.send(\"TagC\", msgSize);\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize * 3, producer.getAllUndupMsgBody().size());\n\n        List<String> receivedMessage = new ArrayList<>(2);\n        Set<MessageQueue> mqs = consumer.fetchSubscribeMessageQueues(topic);\n        for (MessageQueue mq : mqs) {\n            SINGLE_MQ:\n            while (true) {\n                try {\n                    PullResult pullResult =\n                        consumer.pull(mq, selector, getMessageQueueOffset(mq), 32);\n                    putMessageQueueOffset(mq, pullResult.getNextBeginOffset());\n                    switch (pullResult.getPullStatus()) {\n                        case FOUND:\n                            List<MessageExt> msgs = pullResult.getMsgFoundList();\n                            for (MessageExt msg : msgs) {\n                                receivedMessage.add(new String(msg.getBody()));\n                            }\n                            break;\n                        case NO_MATCHED_MSG:\n                            break;\n                        case NO_NEW_MSG:\n                            break SINGLE_MQ;\n                        case OFFSET_ILLEGAL:\n                            break;\n                        default:\n                            break;\n                    }\n                } catch (Exception e) {\n                    e.printStackTrace();\n                }\n            }\n        }\n\n        assertThat(receivedMessage.size()).isEqualTo(msgSize * 2);\n    }\n\n    private static long getMessageQueueOffset(MessageQueue mq) {\n        Long offset = OFFSE_TABLE.get(mq);\n        if (offset != null)\n            return offset;\n\n        return 0;\n    }\n\n    private static void putMessageQueueOffset(MessageQueue mq, long offset) {\n        OFFSE_TABLE.put(mq, offset);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePop.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.pop;\n\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQPopClient;\nimport org.apache.rocketmq.test.factory.ConsumerFactory;\n\npublic class BasePop extends BaseConf {\n\n    public RMQPopClient getRMQPopClient() {\n        RMQPopClient client = ConsumerFactory.getRMQPopClient();\n        mqClients.add(client);\n        return client;\n    }\n\n    protected static class MsgRcv {\n        public final long rcvTime;\n        public final MessageExt messageExt;\n\n        public MsgRcv(long rcvTime, MessageExt messageExt) {\n            this.rcvTime = rcvTime;\n            this.messageExt = messageExt;\n        }\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePopNormally.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.pop;\n\nimport java.util.concurrent.CompletableFuture;\n\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.ConsumeInitMode;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.test.base.IntegrationTestBase;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.client.rmq.RMQPopClient;\nimport org.apache.rocketmq.test.util.MQRandomUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Ignore;\n\n@Ignore\npublic class BasePopNormally extends BasePop {\n\n    protected String topic;\n    protected String group;\n    protected RMQNormalProducer producer = null;\n    protected RMQPopClient client = null;\n    protected String brokerAddr;\n    protected MessageQueue messageQueue;\n\n    @Before\n    public void setUp() {\n        brokerAddr = brokerController1.getBrokerAddr();\n        topic = MQRandomUtils.getRandomTopic();\n        group = initConsumerGroup();\n        IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, 8, CQType.SimpleCQ, TopicMessageType.NORMAL);\n        producer = getProducer(NAMESRV_ADDR, topic);\n        client = getRMQPopClient();\n        messageQueue = new MessageQueue(topic, BROKER1_NAME, -1);\n    }\n\n    @After\n    public void tearDown() {\n        shutdown();\n    }\n\n    protected CompletableFuture<PopResult> popMessageAsync(long invisibleTime, int maxNums, long timeout) {\n        return client.popMessageAsync(\n            brokerAddr, messageQueue, invisibleTime, maxNums, group, timeout, true,\n            ConsumeInitMode.MIN, false, ExpressionType.TAG, \"*\");\n    }\n\n    protected CompletableFuture<PopResult> popMessageAsync(long invisibleTime, int maxNums) {\n        return client.popMessageAsync(\n            brokerAddr, messageQueue, invisibleTime, maxNums, group, 3000, false,\n            ConsumeInitMode.MIN, false, ExpressionType.TAG, \"*\");\n    }\n\n    protected CompletableFuture<AckResult> ackMessageAsync(MessageExt messageExt) {\n        return client.ackMessageAsync(brokerAddr, topic, group, messageExt.getProperty(MessageConst.PROPERTY_POP_CK));\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePopOrderly.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.pop;\n\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.CopyOnWriteArrayList;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.ConsumeInitMode;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.test.base.IntegrationTestBase;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.client.rmq.RMQPopClient;\nimport org.apache.rocketmq.test.message.MessageQueueMsg;\nimport org.apache.rocketmq.test.util.MQRandomUtils;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.assertj.core.util.Lists;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Ignore;\n\nimport static org.junit.Assert.assertEquals;\n\n@Ignore\npublic class BasePopOrderly extends BasePop {\n    protected String topic;\n    protected String group;\n    protected RMQNormalProducer producer = null;\n    protected RMQPopClient client = null;\n    protected String brokerAddr;\n    protected MessageQueue messageQueue;\n    protected final Map<String, List<MsgRcv>> msgRecv = new ConcurrentHashMap<>();\n    protected final List<String> msgRecvSequence = new CopyOnWriteArrayList<>();\n    protected final List<Object> msgDataRecv = new CopyOnWriteArrayList<>();\n\n    @Before\n    public void setUp() {\n        brokerController1.getBrokerConfig().setEnableNotifyAfterPopOrderLockRelease(true);\n        brokerAddr = brokerController1.getBrokerAddr();\n        topic = MQRandomUtils.getRandomTopic();\n        group = initConsumerGroup();\n        IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, 1, CQType.SimpleCQ, TopicMessageType.FIFO);\n        producer = getProducer(NAMESRV_ADDR, topic);\n        client = getRMQPopClient();\n        messageQueue = new MessageQueue(topic, BROKER1_NAME, -1);\n    }\n\n    @After\n    public void tearDown() {\n        shutdown();\n    }\n\n    protected void sendMessage(int num) {\n        MessageQueueMsg mqMsgs = new MessageQueueMsg(Lists.newArrayList(messageQueue), num);\n        producer.send(mqMsgs.getMsgsWithMQ());\n    }\n\n    protected void assertMessageRecvOrder() {\n        VerifyUtils.verifyOrderMsg(msgDataRecv);\n    }\n\n    protected void assertMsgRecv(int seqId, int expectNum) {\n        String msgId = msgRecvSequence.get(seqId);\n        List<MsgRcv> msgRcvList = msgRecv.get(msgId);\n        assertEquals(expectNum, msgRcvList.size());\n        assertConsumeTimes(msgRcvList);\n    }\n\n    protected void assertConsumeTimes(List<MsgRcv> msgRcvList) {\n        for (int i = 0; i < msgRcvList.size(); i++) {\n            assertEquals(i, msgRcvList.get(i).messageExt.getReconsumeTimes());\n        }\n    }\n\n    protected void assertMsgRecv(int seqId, int expectNum, List<Integer> expectReconsumeTimes) {\n        String msgId = msgRecvSequence.get(seqId);\n        List<MsgRcv> msgRcvList = msgRecv.get(msgId);\n        assertEquals(expectNum, msgRcvList.size());\n        assertConsumeTimes(msgRcvList, expectReconsumeTimes);\n    }\n\n    protected void assertConsumeTimes(List<MsgRcv> msgRcvList, List<Integer> expectReconsumeTimes) {\n        for (int i = 0; i < msgRcvList.size(); i++) {\n            assertEquals(expectReconsumeTimes.get(i).intValue(), msgRcvList.get(i).messageExt.getReconsumeTimes());\n        }\n    }\n\n    protected void onRecvNewMessage(MessageExt messageExt) {\n        msgDataRecv.add(new String(messageExt.getBody()));\n        msgRecvSequence.add(messageExt.getMsgId());\n        msgRecv.compute(messageExt.getMsgId(), (k, msgRcvList) -> {\n            if (msgRcvList == null) {\n                msgRcvList = new CopyOnWriteArrayList<>();\n            }\n            msgRcvList.add(new MsgRcv(System.currentTimeMillis(), messageExt));\n            return msgRcvList;\n        });\n    }\n\n    protected CompletableFuture<PopResult> popMessageOrderlyAsync(long invisibleTime, int maxNums, long timeout) {\n        return popMessageOrderlyAsync(invisibleTime, maxNums, timeout, null);\n    }\n\n    protected CompletableFuture<PopResult> popMessageOrderlyAsync(long invisibleTime, int maxNums, long timeout, String attemptId) {\n        return client.popMessageAsync(\n            brokerAddr, messageQueue, invisibleTime, maxNums, group, timeout, true,\n            ConsumeInitMode.MIN, true, ExpressionType.TAG, \"*\", attemptId);\n    }\n\n    protected CompletableFuture<AckResult> ackMessageAsync(MessageExt messageExt) {\n        return client.ackMessageAsync(brokerAddr, topic, group, messageExt.getProperty(MessageConst.PROPERTY_POP_CK));\n    }\n\n    protected CompletableFuture<AckResult> changeInvisibleTimeAsync(MessageExt messageExt, long invisibleTime) {\n        return client.changeInvisibleTimeAsync(\n            brokerAddr, BROKER1_NAME, topic, group,\n            messageExt.getProperty(MessageConst.PROPERTY_POP_CK), invisibleTime);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BatchAckIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.pop;\n\nimport java.time.Duration;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.function.Supplier;\nimport org.apache.rocketmq.client.consumer.AckResult;\nimport org.apache.rocketmq.client.consumer.AckStatus;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PopStatus;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.ConsumeInitMode;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.test.base.IntegrationTestBase;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.client.rmq.RMQPopClient;\nimport org.apache.rocketmq.test.util.MQRandomUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertEquals;\n\npublic class BatchAckIT extends BasePop {\n\n    protected String topic;\n    protected String group;\n    protected RMQNormalProducer producer = null;\n    protected RMQPopClient client = null;\n    protected String brokerAddr;\n    protected MessageQueue messageQueue;\n\n    @Before\n    public void setUp() {\n        brokerAddr = brokerController1.getBrokerAddr();\n        topic = MQRandomUtils.getRandomTopic();\n        group = initConsumerGroup();\n        IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, 8, CQType.SimpleCQ, TopicMessageType.NORMAL);\n        producer = getProducer(NAMESRV_ADDR, topic);\n        client = getRMQPopClient();\n        messageQueue = new MessageQueue(topic, BROKER1_NAME, -1);\n    }\n\n    @After\n    public void tearDown() {\n        shutdown();\n    }\n\n    @Test\n    public void testBatchAckNormallyWithPopBuffer() throws Throwable {\n        brokerController1.getBrokerConfig().setEnablePopBufferMerge(true);\n        brokerController2.getBrokerConfig().setEnablePopBufferMerge(true);\n\n        testBatchAck(() -> {\n            try {\n                return popMessageAsync().get();\n            } catch (Exception e) {\n                throw new RuntimeException(e);\n            }\n        });\n    }\n\n    @Test\n    public void testBatchAckNormallyWithOutPopBuffer() throws Throwable {\n        brokerController1.getBrokerConfig().setEnablePopBufferMerge(false);\n        brokerController2.getBrokerConfig().setEnablePopBufferMerge(false);\n\n        testBatchAck(() -> {\n            try {\n                return popMessageAsync().get();\n            } catch (Exception e) {\n                throw new RuntimeException(e);\n            }\n        });\n    }\n\n    @Test\n    public void testBatchAckOrderly() throws Throwable {\n        testBatchAck(() -> {\n            try {\n                return popMessageOrderlyAsync().get();\n            } catch (Exception e) {\n                throw new RuntimeException(e);\n            }\n        });\n    }\n\n    public void testBatchAck(Supplier<PopResult> popResultSupplier) throws Throwable {\n        // Send 10 messages but do not ack, let them enter the retry topic\n        producer.send(10);\n        AtomicInteger firstMsgRcvNum = new AtomicInteger();\n        await().atMost(Duration.ofSeconds(3)).untilAsserted(() -> {\n            PopResult popResult = popResultSupplier.get();\n            if (popResult.getPopStatus().equals(PopStatus.FOUND)) {\n                firstMsgRcvNum.addAndGet(popResult.getMsgFoundList().size());\n            }\n            assertEquals(10, firstMsgRcvNum.get());\n        });\n        // sleep 6s, expect messages to enter the retry topic\n        TimeUnit.SECONDS.sleep(6);\n\n        producer.send(20);\n        List<String> extraInfoList = new ArrayList<>();\n        await().atMost(Duration.ofSeconds(3)).untilAsserted(() -> {\n            PopResult popResult = popResultSupplier.get();\n            if (popResult.getPopStatus().equals(PopStatus.FOUND)) {\n                for (MessageExt messageExt : popResult.getMsgFoundList()) {\n                    extraInfoList.add(messageExt.getProperty(MessageConst.PROPERTY_POP_CK));\n                }\n            }\n            assertEquals(30, extraInfoList.size());\n        });\n\n        AckResult ackResult = client.batchAckMessageAsync(brokerAddr, topic, group, extraInfoList).get();\n        assertEquals(AckStatus.OK, ackResult.getStatus());\n\n        // sleep 6s, expected that messages that have been acked will not be re-consumed\n        TimeUnit.SECONDS.sleep(6);\n        PopResult popResult = popResultSupplier.get();\n        assertEquals(PopStatus.POLLING_NOT_FOUND, popResult.getPopStatus());\n    }\n\n    private CompletableFuture<PopResult> popMessageAsync() {\n        return client.popMessageAsync(\n            brokerAddr, messageQueue, Duration.ofSeconds(3).toMillis(), 30, group, 3000, false,\n            ConsumeInitMode.MIN, false, ExpressionType.TAG, \"*\");\n    }\n\n    private CompletableFuture<PopResult> popMessageOrderlyAsync() {\n        return client.popMessageAsync(\n            brokerAddr, messageQueue, Duration.ofSeconds(3).toMillis(), 30, group, 3000, false,\n            ConsumeInitMode.MIN, true, ExpressionType.TAG, \"*\", null);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/NotificationIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.pop;\n\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.ConsumeInitMode;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.test.base.IntegrationTestBase;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.client.rmq.RMQPopClient;\nimport org.apache.rocketmq.test.message.MessageQueueMsg;\nimport org.apache.rocketmq.test.util.MQRandomUtils;\nimport org.assertj.core.util.Lists;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.Ignore;\n\nimport static org.assertj.core.api.AssertionsForClassTypes.assertThat;\n\npublic class NotificationIT extends BasePop {\n    protected String topic;\n    protected String group;\n    protected RMQNormalProducer producer = null;\n    protected RMQPopClient client = null;\n    protected String brokerAddr;\n    protected MessageQueue messageQueue;\n\n    @Before\n    public void setUp() {\n        brokerAddr = brokerController1.getBrokerAddr();\n        topic = MQRandomUtils.getRandomTopic();\n        group = initConsumerGroup();\n        IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, 8, CQType.SimpleCQ, TopicMessageType.NORMAL);\n        producer = getProducer(NAMESRV_ADDR, topic);\n        client = getRMQPopClient();\n        messageQueue = new MessageQueue(topic, BROKER1_NAME, -1);\n    }\n\n    @Test\n    @Ignore\n    public void testNotification() throws Exception {\n        long pollTime = 500;\n        CompletableFuture<Boolean> future1 = client.notification(brokerAddr, topic, group, messageQueue.getQueueId(), pollTime, System.currentTimeMillis(), 5000);\n        CompletableFuture<Boolean> future2 = client.notification(brokerAddr, topic, group, messageQueue.getQueueId(), pollTime, System.currentTimeMillis(), 5000);\n        sendMessage(1);\n        Boolean result2 = future2.get();\n        assertThat(result2).isTrue();\n        client.popMessageAsync(brokerAddr, messageQueue, 10000, 1, group, 1000, false,\n            ConsumeInitMode.MIN, false, null, null).get();\n        Boolean result1 = future1.get();\n        assertThat(result1).isFalse();\n    }\n\n    @Test\n    public void testNotificationOrderly() throws Exception {\n        long pollTime = 500;\n        String attemptId = \"attemptId\";\n        CompletableFuture<Boolean> future1 = client.notification(brokerAddr, topic, group, messageQueue.getQueueId(), true, attemptId, pollTime, System.currentTimeMillis(), 5000);\n        CompletableFuture<Boolean> future2 = client.notification(brokerAddr, topic, group, messageQueue.getQueueId(), true, attemptId, pollTime, System.currentTimeMillis(), 5000);\n        sendMessage(1);\n        Boolean result1 = future1.get();\n        assertThat(result1).isTrue();\n        client.popMessageAsync(brokerAddr, messageQueue, 10000, 1, group, 1000, false,\n            ConsumeInitMode.MIN, true, null, null, attemptId).get();\n        Boolean result2 = future2.get();\n        assertThat(result2).isTrue();\n\n        String attemptId2 = \"attemptId2\";\n        CompletableFuture<Boolean> future3 = client.notification(brokerAddr, topic, group, messageQueue.getQueueId(), true, attemptId2, pollTime, System.currentTimeMillis(), 5000);\n        assertThat(future3.get()).isFalse();\n    }\n\n    protected void sendMessage(int num) {\n        MessageQueueMsg mqMsgs = new MessageQueueMsg(Lists.newArrayList(messageQueue), num);\n        producer.send(mqMsgs.getMsgsWithMQ());\n    }\n\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopBigMessageIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.pop;\n\nimport java.io.IOException;\nimport java.time.Duration;\nimport java.util.concurrent.atomic.AtomicReference;\nimport org.apache.rocketmq.client.ClientConfig;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PopStatus;\nimport org.apache.rocketmq.common.compression.Compressor;\nimport org.apache.rocketmq.common.compression.CompressorFactory;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.sysflag.MessageSysFlag;\nimport org.junit.Test;\n\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertEquals;\n\npublic class PopBigMessageIT extends BasePopNormally {\n\n    private static final int BODY_LEN = 3 * 1024 * 1024;\n\n    static {\n        System.setProperty(ClientConfig.DECODE_DECOMPRESS_BODY, \"false\");\n    }\n\n    private Message createBigMessage() {\n        byte[] bytes = new byte[BODY_LEN];\n        return new Message(topic, bytes);\n    }\n\n    @Test\n    public void testSendAndRecvBigMsgWhenDisablePopBufferMerge() throws Throwable {\n        brokerController1.getBrokerConfig().setEnablePopBufferMerge(false);\n        brokerController2.getBrokerConfig().setEnablePopBufferMerge(false);\n\n        this.testSendAndRecvBigMsg();\n    }\n\n    @Test\n    public void testSendAndRecvBigMsgWhenEnablePopBufferMerge() throws Throwable {\n        brokerController1.getBrokerConfig().setEnablePopBufferMerge(true);\n        brokerController2.getBrokerConfig().setEnablePopBufferMerge(true);\n\n        this.testSendAndRecvBigMsg();\n    }\n\n    /**\n     * set DECODE_DECOMPRESS_BODY to false, then pop message from broker and not ack\n     * <p>\n     * expect when re-consume this message, the message is not decompressed\n     */\n    private void testSendAndRecvBigMsg() {\n        Message message = createBigMessage();\n        producer.send(message);\n\n        AtomicReference<MessageExt> firstMessageExtRef = new AtomicReference<>();\n        await().atMost(Duration.ofSeconds(3)).untilAsserted(() -> {\n            PopResult popResult = popMessageAsync(Duration.ofSeconds(3).toMillis(), 1, 5000).get();\n            assertEquals(PopStatus.FOUND, popResult.getPopStatus());\n\n            firstMessageExtRef.set(popResult.getMsgFoundList().get(0));\n            MessageExt messageExt = firstMessageExtRef.get();\n            assertMessageRecv(messageExt);\n        });\n\n        // no ack, msg will put into pop retry topic\n        await().atMost(Duration.ofSeconds(60)).untilAsserted(() -> {\n            PopResult retryPopResult = popMessageAsync(Duration.ofSeconds(3).toMillis(), 1, 5000).get();\n            assertEquals(PopStatus.FOUND, retryPopResult.getPopStatus());\n\n            MessageExt retryMessageExt = retryPopResult.getMsgFoundList().get(0);\n            assertMessageRecv(retryMessageExt);\n            assertEquals(firstMessageExtRef.get().getBody().length, retryMessageExt.getBody().length);\n        });\n    }\n\n    private void assertMessageRecv(MessageExt messageExt) throws IOException {\n        assertEquals(MessageSysFlag.COMPRESSED_FLAG, messageExt.getSysFlag() & MessageSysFlag.COMPRESSED_FLAG);\n        Compressor compressor = CompressorFactory.getCompressor(MessageSysFlag.getCompressionType(messageExt.getSysFlag()));\n        assertEquals(BODY_LEN, compressor.decompress(messageExt.getBody()).length);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopMessageAndForwardingIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.pop;\n\nimport java.time.Duration;\nimport java.util.concurrent.atomic.AtomicReference;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PopStatus;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.constant.ConsumeInitMode;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;\nimport org.apache.rocketmq.test.base.IntegrationTestBase;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.client.rmq.RMQPopClient;\nimport org.apache.rocketmq.test.util.MQRandomUtils;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotEquals;\n\npublic class PopMessageAndForwardingIT extends BasePop {\n\n    protected String topic;\n    protected String group;\n    protected RMQNormalProducer producer = null;\n    protected RMQPopClient client = null;\n    protected String broker1Addr;\n    protected MessageQueue broker1MessageQueue;\n    protected String broker2Addr;\n    protected MessageQueue broker2MessageQueue;\n\n    @Before\n    public void setUp() {\n        broker1Addr = brokerController1.getBrokerAddr();\n        broker2Addr = brokerController2.getBrokerAddr();\n        topic = MQRandomUtils.getRandomTopic();\n        group = initConsumerGroup();\n        IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, 8, CQType.SimpleCQ, TopicMessageType.NORMAL);\n        IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER2_NAME, 8, CQType.SimpleCQ, TopicMessageType.NORMAL);\n        producer = getProducer(NAMESRV_ADDR, topic);\n        client = getRMQPopClient();\n        broker1MessageQueue = new MessageQueue(topic, BROKER1_NAME, -1);\n        broker2MessageQueue = new MessageQueue(topic, BROKER2_NAME, -1);\n    }\n\n    @Test\n    public void test() {\n        producer.send(1, broker1MessageQueue);\n\n        AtomicReference<MessageExt> firstMessageExtRef = new AtomicReference<>();\n        await().atMost(Duration.ofSeconds(3)).until(() -> {\n            PopResult popResult = client.popMessageAsync(broker1Addr, broker1MessageQueue, 3000, 32, group, 1000,\n                true, ConsumeInitMode.MIN, false, ExpressionType.TAG, \"*\").get();\n            if (!popResult.getPopStatus().equals(PopStatus.FOUND)) {\n                return false;\n            }\n            firstMessageExtRef.set(popResult.getMsgFoundList().get(0));\n            return true;\n        });\n\n        producer.sendMQ(firstMessageExtRef.get(), broker2MessageQueue);\n        AtomicReference<MessageExt> secondMessageExtRef = new AtomicReference<>();\n        await().atMost(Duration.ofSeconds(3)).until(() -> {\n            PopResult popResult = client.popMessageAsync(broker2Addr, broker2MessageQueue, 3000, 32, group, 1000,\n                true, ConsumeInitMode.MIN, false, ExpressionType.TAG, \"*\").get();\n            if (!popResult.getPopStatus().equals(PopStatus.FOUND)) {\n                return false;\n            }\n            secondMessageExtRef.set(popResult.getMsgFoundList().get(0));\n            return true;\n        });\n\n        assertEquals(firstMessageExtRef.get().getMsgId(), secondMessageExtRef.get().getMsgId());\n        String firstPopCk = firstMessageExtRef.get().getProperty(MessageConst.PROPERTY_POP_CK);\n        String secondPopCk = secondMessageExtRef.get().getProperty(MessageConst.PROPERTY_POP_CK);\n        assertNotEquals(firstPopCk, secondPopCk);\n        assertEquals(BROKER1_NAME, ExtraInfoUtil.getBrokerName(ExtraInfoUtil.split(firstPopCk)));\n        assertEquals(BROKER2_NAME, ExtraInfoUtil.getBrokerName(ExtraInfoUtil.split(secondPopCk)));\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopOrderlyIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.pop;\n\nimport java.time.Duration;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.assertj.core.util.Lists;\nimport org.junit.Test;\n\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertEquals;\n\npublic class PopOrderlyIT extends BasePopOrderly {\n\n    /**\n     * send 10 messages, pop one message orderly at a time\n     * <p>\n     * expect receive this 10 messages in order\n     */\n    @Test\n    public void testPopOrderly() {\n        sendMessage(10);\n        await().atMost(Duration.ofSeconds(10)).until(() -> {\n            popMessageOrderly().get();\n            return msgRecv.size() == 10;\n        });\n\n        assertMessageRecvOrder();\n    }\n\n    private CompletableFuture<Void> popMessageOrderly() {\n        CompletableFuture<PopResult> future = popMessageOrderlyAsync(TimeUnit.SECONDS.toMillis(3), 1, TimeUnit.SECONDS.toMillis(30));\n        CompletableFuture<Void> resultFuture = new CompletableFuture<>();\n        future.whenComplete((popResult, throwable) -> {\n            if (throwable != null) {\n                resultFuture.completeExceptionally(throwable);\n                return;\n            }\n            if (popResult.getMsgFoundList() == null || popResult.getMsgFoundList().isEmpty()) {\n                resultFuture.complete(null);\n                return;\n            }\n            try {\n                for (MessageExt messageExt : popResult.getMsgFoundList()) {\n                    onRecvNewMessage(messageExt);\n                    // ack later\n                    // expect when the lock is free, pop message request can receive messages immediately after ack\n                    new Thread(() -> {\n                        try {\n                            TimeUnit.MILLISECONDS.sleep(100);\n                        } catch (InterruptedException ignored) {\n                        }\n                        ackMessageAsync(messageExt);\n                    }).start();\n                }\n                resultFuture.complete(null);\n            } catch (Throwable t) {\n                resultFuture.completeExceptionally(t);\n            }\n        });\n        return resultFuture;\n    }\n\n    /**\n     * send 10 messages, pop five messages orderly at a time\n     * <p>\n     * expect only receive the first five messages\n     */\n    @Test\n    public void testPopOrderlyThenNoAck() {\n        sendMessage(10);\n        await().atMost(Duration.ofSeconds(5)).until(() -> {\n            popOrderlyThenNoAck().get();\n            return msgRecvSequence.size() == 10;\n        });\n        assertEquals(5, msgRecv.size());\n        for (Map.Entry<String, List<MsgRcv>> entry : msgRecv.entrySet()) {\n            assertEquals(2, entry.getValue().size());\n            for (int i = 0; i < entry.getValue().size(); i++) {\n                assertEquals(i, entry.getValue().get(i).messageExt.getReconsumeTimes());\n            }\n        }\n        assertMessageRecvOrder();\n    }\n\n    private CompletableFuture<Void> popOrderlyThenNoAck() {\n        CompletableFuture<PopResult> future = popMessageOrderlyAsync(TimeUnit.SECONDS.toMillis(3), 5, TimeUnit.SECONDS.toMillis(30));\n        CompletableFuture<Void> resultFuture = new CompletableFuture<>();\n        future.whenComplete((popResult, throwable) -> {\n            if (throwable != null) {\n                resultFuture.completeExceptionally(throwable);\n                return;\n            }\n            if (popResult.getMsgFoundList() == null || popResult.getMsgFoundList().isEmpty()) {\n                resultFuture.complete(null);\n                return;\n            }\n            try {\n                for (MessageExt messageExt : popResult.getMsgFoundList()) {\n                    onRecvNewMessage(messageExt);\n                }\n                resultFuture.complete(null);\n            } catch (Throwable t) {\n                resultFuture.completeExceptionally(t);\n            }\n        });\n        return resultFuture;\n    }\n\n    /**\n     * send one message, changeInvisibleTime to 5s later at the first time\n     * <p>\n     * expect receive two times\n     */\n    @Test\n    public void testPopMessageOrderlyThenChangeInvisibleTime() {\n        sendMessage(1);\n\n        await().atMost(Duration.ofSeconds(15)).until(() -> {\n            popMessageOrderlyThenChangeInvisibleTime().get();\n            return msgRecvSequence.size() == 2;\n        });\n\n        assertMsgRecv(1, 2);\n        assertMessageRecvOrder();\n    }\n\n    private CompletableFuture<Void> popMessageOrderlyThenChangeInvisibleTime() {\n        CompletableFuture<PopResult> future = popMessageOrderlyAsync(TimeUnit.SECONDS.toMillis(3), 1, TimeUnit.SECONDS.toMillis(30));\n        CompletableFuture<Void> resultFuture = new CompletableFuture<>();\n        future.whenComplete((popResult, throwable) -> {\n            if (throwable != null) {\n                resultFuture.completeExceptionally(throwable);\n                return;\n            }\n            if (popResult.getMsgFoundList() == null || popResult.getMsgFoundList().isEmpty()) {\n                resultFuture.complete(null);\n                return;\n            }\n            try {\n                for (MessageExt messageExt : popResult.getMsgFoundList()) {\n                    onRecvNewMessage(messageExt);\n                    if (msgRecvSequence.size() == 1) {\n                        try {\n                            TimeUnit.MILLISECONDS.sleep(1);\n                            changeInvisibleTimeAsync(messageExt, 5000).get();\n                        } catch (Exception e) {\n                            resultFuture.completeExceptionally(e);\n                            return;\n                        }\n                    } else {\n                        try {\n                            ackMessageAsync(messageExt).get();\n                        } catch (Exception e) {\n                            resultFuture.completeExceptionally(e);\n                            return;\n                        }\n                    }\n                }\n                resultFuture.complete(null);\n            } catch (Throwable t) {\n                resultFuture.completeExceptionally(t);\n            }\n        });\n        return resultFuture;\n    }\n\n    /**\n     * send three messages (msg1, msg2, msg3, msg4) and the max message num of pop request is three\n     * <p>\n     * ack msg1 and msg3, changeInvisibleTime msg2\n     * <p>\n     * expect the sequence of messages received is: msg1, msg2, msg3, msg2, msg3, msg4\n     */\n    @Test\n    public void testPopMessageOrderlyThenChangeInvisibleTimeMidMessage() {\n        producer.send(4);\n\n        await().atMost(Duration.ofSeconds(5)).until(() -> {\n            popMessageOrderlyThenChangeInvisibleTimeMidMessage().get();\n            return msgRecvSequence.size() == 6;\n        });\n\n        assertMsgRecv(0, 1);\n        assertMsgRecv(1, 2);\n        assertMsgRecv(2, 2);\n        assertMsgRecv(5, 1);\n\n        assertEquals(msgRecvSequence.get(1), msgRecvSequence.get(3));\n        assertEquals(msgRecvSequence.get(2), msgRecvSequence.get(4));\n    }\n\n    private CompletableFuture<Void> popMessageOrderlyThenChangeInvisibleTimeMidMessage() {\n        CompletableFuture<PopResult> future = popMessageOrderlyAsync(5000, 3, TimeUnit.SECONDS.toMillis(30));\n        CompletableFuture<Void> resultFuture = new CompletableFuture<>();\n        future.whenComplete((popResult, throwable) -> {\n            if (throwable != null) {\n                resultFuture.completeExceptionally(throwable);\n                return;\n            }\n            if (popResult.getMsgFoundList() == null || popResult.getMsgFoundList().isEmpty()) {\n                resultFuture.complete(null);\n                return;\n            }\n            try {\n                for (MessageExt messageExt : popResult.getMsgFoundList()) {\n                    onRecvNewMessage(messageExt);\n                    if (msgRecv.size() != 2) {\n                        try {\n                            ackMessageAsync(messageExt).get();\n                        } catch (Exception e) {\n                            resultFuture.completeExceptionally(e);\n                            return;\n                        }\n                    } else {\n                        try {\n                            TimeUnit.MILLISECONDS.sleep(1);\n                            changeInvisibleTimeAsync(messageExt, 3000).get();\n                        } catch (Exception e) {\n                            resultFuture.completeExceptionally(e);\n                            return;\n                        }\n                    }\n                }\n                resultFuture.complete(null);\n            } catch (Throwable t) {\n                resultFuture.completeExceptionally(t);\n            }\n        });\n        return resultFuture;\n    }\n\n    @Test\n    public void testReentrant() {\n        producer.send(1);\n\n        popMessageForReentrant(null).join();\n        assertMsgRecv(0, 1, Lists.newArrayList(0));\n\n        String attemptId01 = \"attemptId-01\";\n        popMessageForReentrant(attemptId01).join();\n        assertMsgRecv(0, 2, Lists.newArrayList(0, 1));\n        popMessageForReentrant(attemptId01).join();\n        assertMsgRecv(0, 3, Lists.newArrayList(0, 1, 1));\n\n        String attemptId02 = \"attemptId-02\";\n        await().atLeast(Duration.ofSeconds(5)).atMost(Duration.ofSeconds(15)).until(() -> {\n            popMessageForReentrant(attemptId02).join();\n            return msgRecvSequence.size() == 4;\n        });\n        popMessageForReentrant(attemptId02).join();\n        assertMsgRecv(0, 5, Lists.newArrayList(0, 1, 1, 2, 2));\n\n        await().atLeast(Duration.ofSeconds(5)).atMost(Duration.ofSeconds(15)).until(() -> {\n            popMessageForReentrant(null).join();\n            return msgRecvSequence.size() == 6;\n        });\n        assertMsgRecv(0, 6, Lists.newArrayList(0, 1, 1, 2, 2, 3));\n    }\n\n    private CompletableFuture<Void> popMessageForReentrant(String attemptId) {\n        return popMessageOrderlyAsync(TimeUnit.SECONDS.toMillis(10), 3, TimeUnit.SECONDS.toMillis(30), attemptId)\n            .thenAccept(popResult -> {\n                for (MessageExt messageExt : popResult.getMsgFoundList()) {\n                    onRecvNewMessage(messageExt);\n                }\n            });\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopPriorityIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.pop;\n\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PopStatus;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.attribute.AttributeParser;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.test.base.IntegrationTestBase;\nimport org.apache.rocketmq.test.util.TestUtil;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.junit.runners.Parameterized;\n\nimport java.time.Duration;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Random;\nimport java.util.Set;\nimport java.util.concurrent.TimeUnit;\n\nimport static org.apache.rocketmq.common.SubscriptionGroupAttributes.PRIORITY_FACTOR_ATTRIBUTE;\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertTrue;\n\n@RunWith(Parameterized.class)\npublic class PopPriorityIT extends BasePopNormally {\n\n    private final boolean popConsumerKVServiceEnable;\n    private final boolean priorityOrderAsc;\n    private int writeQueueNum = 8;\n\n    public PopPriorityIT(boolean popConsumerKVServiceEnable, boolean priorityOrderAsc) {\n        this.popConsumerKVServiceEnable = popConsumerKVServiceEnable;\n        this.priorityOrderAsc = priorityOrderAsc;\n    }\n\n    @Parameterized.Parameters\n    public static List<Object[]> params() {\n        List<Object[]> result = new ArrayList<>();\n        result.add(new Object[] {false, true});\n        result.add(new Object[] {false, false});\n        result.add(new Object[] {true, true});\n        result.add(new Object[] {true, false});\n        return result;\n    }\n\n    @Before\n    public void setUp() {\n        super.setUp();\n        // reset default config if changed\n        writeQueueNum = 8;\n        brokerController1.getBrokerConfig().setPopFromRetryProbabilityForPriority(0);\n        brokerController1.getBrokerConfig().setUseSeparateRetryQueue(false);\n        brokerController1.getBrokerConfig().setPopConsumerKVServiceEnable(popConsumerKVServiceEnable);\n        brokerController1.getBrokerConfig().setPriorityOrderAsc(priorityOrderAsc);\n        IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, writeQueueNum, CQType.SimpleCQ, TopicMessageType.PRIORITY);\n    }\n\n    @After\n    public void tearDown() {\n        super.tearDown();\n    }\n\n    @Test\n    public void test_normal_send() {\n        int priority = -1; // normal message\n        Set<Integer> queueIdSet = new HashSet<>();\n        for (int i = 0; i < 32; i++) {\n            Message message = mockMessage(topic, priority, \"\");\n            SendResult sendResult = producer.send(message, null).getSendResultObj();\n            queueIdSet.add(sendResult.getMessageQueue().getQueueId());\n        }\n        assertTrue(queueIdSet.size() > 1);\n    }\n\n    @Test\n    public void test_priority_send() {\n        final int priority = 0; // priority message\n        for (int i = 0; i < 32; i++) {\n            Message message = mockMessage(topic, priority, \"\");\n            SendResult sendResult = producer.send(message, null).getSendResultObj();\n            assertEquals(priority, sendResult.getMessageQueue().getQueueId());\n        }\n    }\n\n    @Test\n    public void test_priority_consume_always_high_priority() throws Exception {\n        int msgNumPerQueue = 20;\n        final int maxPriority = priorityOrderAsc ? writeQueueNum - 1 : 0;\n        for (int i = 0; i < writeQueueNum; i++) {\n            Message message = mockMessage(topic, i, String.valueOf(i));\n            for (int j = 0; j < msgNumPerQueue; j++) {\n                producer.send(message);\n            }\n        }\n        Assert.assertTrue(awaitDispatchMs(2000));\n        for (int i = 0; i < msgNumPerQueue; i++) {\n            PopResult popResult = popMessageAsync(Duration.ofSeconds(600).toMillis(), 1, 30000).get();\n            TestUtil.waitForMonment(20); // wait lock release\n            assertEquals(PopStatus.FOUND, popResult.getPopStatus());\n            MessageExt message = popResult.getMsgFoundList().get(0);\n            assertEquals(maxPriority, message.getPriority()); // not a coincidence\n        }\n    }\n\n    @Test\n    public void test_priority_consume_from_high_to_low() throws Exception {\n        for (int i = 0; i < writeQueueNum; i++) {\n            Message message = mockMessage(topic, i, String.valueOf(i));\n            producer.send(message);\n        }\n        Assert.assertTrue(awaitDispatchMs(2000));\n        for (int i = 0; i < writeQueueNum; i++) {\n            PopResult popResult = popMessageAsync(Duration.ofSeconds(30).toMillis(), 1, 30000).get();\n            assertEquals(PopStatus.FOUND, popResult.getPopStatus());\n            MessageExt message = popResult.getMsgFoundList().get(0);\n            int expectPriority = priorityOrderAsc ? writeQueueNum - 1 - i : i;\n            assertEquals(0, message.getQueueOffset());\n            assertEquals(expectPriority, message.getQueueId());\n            assertEquals(expectPriority, message.getPriority());\n        }\n    }\n\n    @Test\n    public void test_priority_consume_disable() throws Exception {\n        SubscriptionGroupConfig config = new SubscriptionGroupConfig();\n        config.setGroupName(group);\n        config.setAttributes(AttributeParser.parseToMap(\"+\" + PRIORITY_FACTOR_ATTRIBUTE.getName() + \"=0\"));\n        initConsumerGroup(config);\n\n        int msgNumPerQueue = 200;\n        for (int i = 0; i < writeQueueNum; i++) {\n            Message message = mockMessage(topic, i, String.valueOf(i));\n            for (int j = 0; j < msgNumPerQueue; j++) {\n                producer.send(message);\n            }\n        }\n        Assert.assertTrue(awaitDispatchMs(2000));\n        int sampleCount = 800;\n        int[] queueIdCount = new int[writeQueueNum];\n        for (int i = 0; i < sampleCount; i++) {\n            PopResult popResult = popMessageAsync(Duration.ofSeconds(600).toMillis(), 1, 30000).get();\n            TestUtil.waitForMonment(10); // wait lock release\n            assertEquals(PopStatus.FOUND, popResult.getPopStatus());\n            MessageExt message = popResult.getMsgFoundList().get(0);\n            queueIdCount[message.getQueueId()] = queueIdCount[message.getQueueId()] + 1;\n        }\n\n        double expectAverage = (double) sampleCount / writeQueueNum;\n        for (int count : queueIdCount) {\n            assertTrue(Math.abs(count - expectAverage) < expectAverage * 0.4);\n        }\n    }\n\n    @Test\n    public void test_priority_consume_retry_as_lowest() throws Exception {\n        // retry as lowest by default\n        int count = 100;\n        for (int i = 0; i < count; i++) {\n            Message message = mockMessage(topic, new Random().nextInt(writeQueueNum), String.valueOf(i));\n            producer.send(message);\n        }\n        int invisibleTime = 3;\n        PopResult popResult = popMessageAsync(Duration.ofSeconds(invisibleTime).toMillis(), 1, 30000).get();\n        assertEquals(PopStatus.FOUND, popResult.getPopStatus());\n        String retryId = popResult.getMsgFoundList().get(0).getMsgId();\n        TestUtil.waitForSeconds(invisibleTime + 3);\n        Assert.assertTrue(awaitDispatchMs(2000));\n\n        List<MessageExt> collect = new ArrayList<>();\n        await()\n            .pollInterval(1, TimeUnit.SECONDS)\n            .atMost(35, TimeUnit.SECONDS)\n            .until(() -> {\n                PopResult result = popMessageAsync(Duration.ofSeconds(600).toMillis(), 32, 5000).get();\n                if (PopStatus.FOUND.equals(result.getPopStatus())) {\n                    collect.addAll(result.getMsgFoundList());\n                    return false;\n                }\n                return true;\n            });\n\n        assertEquals(count, collect.size());\n        assertEquals(1, collect.get(collect.size() - 1).getReconsumeTimes());\n        assertEquals(retryId, collect.get(collect.size() - 1).getMsgId());\n    }\n\n    @Test\n    public void test_priority_consume_retry_as_highest() throws Exception {\n        brokerController1.getBrokerConfig().setPopFromRetryProbabilityForPriority(100);\n        int count = 100;\n        for (int i = 0; i < count; i++) {\n            Message message = mockMessage(topic, new Random().nextInt(writeQueueNum), String.valueOf(i));\n            producer.send(message);\n        }\n        int invisibleTime = 3;\n        PopResult popResult = popMessageAsync(Duration.ofSeconds(invisibleTime).toMillis(), 1, 30000).get();\n        assertEquals(PopStatus.FOUND, popResult.getPopStatus());\n        String retryId = popResult.getMsgFoundList().get(0).getMsgId();\n        TestUtil.waitForSeconds(invisibleTime + 3);\n        Assert.assertTrue(awaitDispatchMs(2000));\n\n        List<MessageExt> collect = new ArrayList<>();\n        await()\n            .pollInterval(1, TimeUnit.SECONDS)\n            .atMost(35, TimeUnit.SECONDS)\n            .until(() -> {\n                PopResult result = popMessageAsync(Duration.ofSeconds(600).toMillis(), 32, 5000).get();\n                if (PopStatus.FOUND.equals(result.getPopStatus())) {\n                    collect.addAll(result.getMsgFoundList());\n                    return false;\n                }\n                return true;\n            });\n\n        assertEquals(count, collect.size());\n        assertEquals(1, collect.get(0).getReconsumeTimes());\n        assertEquals(retryId, collect.get(0).getMsgId());\n    }\n\n    @Test\n    public void test_priority_consume_use_separate_retry_queue() throws Exception {\n        brokerController1.getBrokerConfig().setUseSeparateRetryQueue(true);\n        brokerController1.getBrokerConfig().setPopFromRetryProbabilityForPriority(100);\n        for (int i = 0; i < writeQueueNum; i++) {\n            Message message = mockMessage(topic, i, String.valueOf(i));\n            producer.send(message);\n        }\n        Assert.assertTrue(awaitDispatchMs(2000));\n        int invisibleTime = 3;\n        PopResult popResult = popMessageAsync(Duration.ofSeconds(invisibleTime).toMillis(), writeQueueNum, 30000).get();\n        assertEquals(PopStatus.FOUND, popResult.getPopStatus());\n        assertEquals(writeQueueNum, popResult.getMsgFoundList().size());\n        TestUtil.waitForSeconds(invisibleTime + 3);\n\n        popResult = popMessageAsync(Duration.ofSeconds(600).toMillis(), 32, 10000).get();\n        assertEquals(PopStatus.FOUND, popResult.getPopStatus());\n        assertEquals(writeQueueNum, popResult.getMsgFoundList().size());\n        for (int i = 0; i < writeQueueNum; i++) {\n            MessageExt message = popResult.getMsgFoundList().get(i);\n            assertEquals(0, message.getQueueOffset()); // means a separate retry queue\n            assertEquals(1, message.getReconsumeTimes());\n            int expectPriority = priorityOrderAsc ? writeQueueNum - 1 - i : i;\n            assertEquals(expectPriority, message.getQueueId());\n            assertEquals(expectPriority, message.getPriority());\n        }\n    }\n\n    @Test\n    public void test_priority_consume_use_separate_retry_queue_with_queue_expansion() throws Exception {\n        // retry as lowest by default\n        brokerController1.getBrokerConfig().setUseSeparateRetryQueue(true);\n        for (int i = 0; i < writeQueueNum; i++) {\n            Message message = mockMessage(topic, i, String.valueOf(i));\n            producer.send(message);\n        }\n        Assert.assertTrue(awaitDispatchMs(2000));\n        int invisibleTime = 3;\n        PopResult popResult = popMessageAsync(Duration.ofSeconds(invisibleTime).toMillis(), writeQueueNum, 30000).get();\n        assertEquals(PopStatus.FOUND, popResult.getPopStatus());\n        assertEquals(writeQueueNum, popResult.getMsgFoundList().size());\n        TestUtil.waitForSeconds(invisibleTime + 3); // wait retry created\n\n        writeQueueNum = writeQueueNum * 2;\n        IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, writeQueueNum, CQType.SimpleCQ, TopicMessageType.PRIORITY);\n        for (int i = writeQueueNum / 2; i < writeQueueNum; i++) {\n            Message message = mockMessage(topic, i, String.valueOf(i));\n            producer.send(message);\n        }\n        Assert.assertTrue(awaitDispatchMs(2000));\n\n        popResult = popMessageAsync(Duration.ofSeconds(invisibleTime).toMillis(), 32, 5000).get();\n        List<MessageExt> msgList = popResult.getMsgFoundList();\n        // asc == true, collect: [15 -> 8, 7 -> 0]\n        // asc == false, collect: [8 -> 15, 0 -> 7]\n        assertEquals(writeQueueNum, msgList.size());\n        assertEquals(priorityOrderAsc ? writeQueueNum - 1 : writeQueueNum / 2, msgList.get(0).getQueueId());\n        assertEquals(priorityOrderAsc ? writeQueueNum - 1 : writeQueueNum / 2, msgList.get(0).getPriority());\n        assertEquals(priorityOrderAsc ? 0 : writeQueueNum / 2 - 1, msgList.get(msgList.size() - 1).getQueueId());\n        assertEquals(priorityOrderAsc ? 0 : writeQueueNum / 2 - 1, msgList.get(msgList.size() - 1).getPriority());\n        assertEquals(1, msgList.get(msgList.size() - 1).getReconsumeTimes());\n        assertEquals(0, msgList.get(msgList.size() - 1).getQueueOffset()); // means a separate retry queue\n    }\n\n    private static Message mockMessage(String topic, int priority, String key) {\n        Message msg = new Message(topic, \"HW\".getBytes());\n        if (priority >= 0) {\n            msg.setPriority(priority);\n        }\n        msg.setKeys(key);\n        return msg;\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopSubCheckIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.pop;\n\nimport org.apache.rocketmq.common.message.MessageClientExt;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageRequestMode;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.client.rmq.RMQPopConsumer;\nimport org.apache.rocketmq.test.factory.ConsumerFactory;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.RandomUtil;\nimport org.apache.rocketmq.test.util.TestUtils;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.apache.rocketmq.tools.admin.DefaultMQAdminExt;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Ignore;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class PopSubCheckIT extends BaseConf {\n    private static final Logger log = LoggerFactory.getLogger(PopSubCheckIT.class);\n    private String group;\n\n    private DefaultMQAdminExt defaultMQAdminExt;\n\n    @Before\n    public void setUp() throws Exception {\n        group = initConsumerGroup();\n\n        defaultMQAdminExt = new DefaultMQAdminExt();\n        defaultMQAdminExt.setInstanceName(RandomUtil.getStringByUUID());\n        defaultMQAdminExt.start();\n    }\n\n    @After\n    public void tearDown() {\n        defaultMQAdminExt.shutdown();\n        super.shutdown();\n    }\n\n    @Ignore\n    @Test\n    public void testNormalPopAck() throws Exception {\n        String topic = initTopic();\n        log.info(\"use topic: {}; group: {} !\", topic, group);\n\n        RMQNormalProducer producer = getProducer(NAMESRV_ADDR, topic);\n        producer.getProducer().setCompressMsgBodyOverHowmuch(Integer.MAX_VALUE);\n\n        for (String brokerAddr : new String[]{brokerController1.getBrokerAddr(), brokerController2.getBrokerAddr()}) {\n            defaultMQAdminExt.setMessageRequestMode(brokerAddr, topic, group, MessageRequestMode.POP, 8, 60_000);\n        }\n\n        RMQPopConsumer consumer = ConsumerFactory.getRMQPopConsumer(NAMESRV_ADDR, group,\n            topic, \"*\", new RMQNormalListener());\n        mqClients.add(consumer);\n\n        int msgNum = 1;\n        producer.send(msgNum);\n        Assert.assertEquals(\"Not all sent succeeded\", msgNum, producer.getAllUndupMsgBody().size());\n        log.info(producer.getFirstMsg().toString());\n\n        TestUtils.waitForSeconds(10);\n\n        consumer.getListener().waitForMessageConsume(msgNum, 30_000);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(), consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n        for (Object o : consumer.getListener().getAllOriginMsg()) {\n            MessageClientExt msg = (MessageClientExt) o;\n            assertThat(msg.getProperty(MessageConst.PROPERTY_POP_CK)).named(\"check pop meta\").isNotEmpty();\n        }\n\n        consumer.getListener().waitForMessageConsume(msgNum, 3_000 * 9);\n        assertThat(consumer.getListener().getAllOriginMsg().size()).isEqualTo(msgNum);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/MulTagSubIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.tag;\n\nimport java.util.List;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.factory.MQMessageFactory;\nimport org.apache.rocketmq.test.factory.TagMessage;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.TestUtils;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class MulTagSubIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(TagMessageWith1ConsumerIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        String consumerId = initConsumerGroup();\n        logger.info(String.format(\"use topic: %s; consumerId: %s !\", topic, consumerId));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testSubTwoTabMessageOnsTag() {\n        String tag = \"jueyin1\";\n        String subExpress = String.format(\"%s||jueyin2\", tag);\n        int msgSize = 10;\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, subExpress,\n            new RMQNormalListener());\n        producer.send(tag, msgSize);\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize, producer.getAllUndupMsgBody().size());\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n\n    @Test\n    public void testSubTwoTabAndMatchOne() {\n        String tag1 = \"jueyin1\";\n        String tag2 = \"jueyin2\";\n        String subExpress = String.format(\"%s||noExistTag\", tag2);\n        int msgSize = 10;\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, subExpress,\n            new RMQNormalListener());\n\n        producer.send(tag1, msgSize);\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize, producer.getAllUndupMsgBody().size());\n        List<Object> tag2Msgs = MQMessageFactory.getRMQMessage(tag2, topic, msgSize);\n        producer.send(tag2Msgs);\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize * 2, producer.getAllUndupMsgBody().size());\n\n        consumer.getListener().waitForMessageConsume(MQMessageFactory.getMessageBody(tag2Msgs),\n            CONSUME_TIME);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(MQMessageFactory.getMessageBody(tag2Msgs));\n    }\n\n    @Test\n    public void testSubTwoTabAndMatchTwo() {\n        String[] tags = {\"jueyin1\", \"jueyin2\"};\n        String subExpress = String.format(\"%s||%s\", tags[0], tags[1]);\n        int msgSize = 10;\n\n        TagMessage tagMessage = new TagMessage(tags, topic, msgSize);\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, subExpress,\n            new RMQNormalListener());\n\n        producer.send(tagMessage.getMixedTagMessages());\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize * tags.length,\n            producer.getAllUndupMsgBody().size());\n\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(tagMessage.getAllTagMessageBody());\n    }\n\n    @Test\n    public void testSubThreeTabAndMatchTwo() {\n        String[] tags = {\"jueyin1\", \"jueyin2\", \"jueyin3\"};\n        String subExpress = String.format(\"%s||%s\", tags[0], tags[1]);\n        int msgSize = 10;\n\n        TagMessage tagMessage = new TagMessage(tags, topic, msgSize);\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, subExpress,\n            new RMQNormalListener());\n\n        producer.send(tagMessage.getMixedTagMessages());\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize * tags.length,\n            producer.getAllUndupMsgBody().size());\n\n        consumer.getListener().waitForMessageConsume(\n            tagMessage.getMessageBodyByTag(tags[0], tags[1]), CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody())).containsExactlyElementsIn(\n            tagMessage.getMessageBodyByTag(tags[0], tags[1]));\n    }\n\n    @Test\n    public void testNoMatch() {\n        String[] tags = {\"jueyin1\", \"jueyin2\", \"jueyin3\"};\n        String subExpress = \"no_match\";\n        int msgSize = 10;\n\n        TagMessage tagMessage = new TagMessage(tags, topic, msgSize);\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, subExpress,\n            new RMQNormalListener());\n\n        producer.send(tagMessage.getMixedTagMessages());\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize * tags.length,\n            producer.getAllUndupMsgBody().size());\n\n        TestUtils.waitForSeconds(5);\n\n        assertThat(VerifyUtils\n            .getFilterdMessage(producer.getAllMsgBody(), consumer.getListener().getAllMsgBody())\n            .size()).isEqualTo(0);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWith1ConsumerIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.tag;\n\nimport java.util.List;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.factory.MQMessageFactory;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class TagMessageWith1ConsumerIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(TagMessageWith1ConsumerIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        String consumerId = initConsumerGroup();\n        logger.info(String.format(\"use topic: %s; consumerId: %s !\", topic, consumerId));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testTagSmoke() {\n        String tag = \"jueyin\";\n        int msgSize = 10;\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, tag, new RMQNormalListener());\n        producer.send(tag, msgSize);\n        Assert.assertEquals(\"Not all are sent\", msgSize, producer.getAllUndupMsgBody().size());\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n\n    @Test\n    public void testSubAllMessageNoTag() {\n        String subExprress = \"*\";\n        int msgSize = 10;\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, subExprress,\n            new RMQNormalListener());\n        producer.send(msgSize);\n        Assert.assertEquals(\"Not all are sent\", msgSize, producer.getAllUndupMsgBody().size());\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n\n    @Test\n    public void testSubAllMessageWithTag() {\n        String tag = \"jueyin\";\n        String subExpress = \"*\";\n        int msgSize = 10;\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, subExpress,\n            new RMQNormalListener());\n        producer.send(tag, msgSize);\n        Assert.assertEquals(\"Not all are sent\", msgSize, producer.getAllUndupMsgBody().size());\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n\n    @Test\n    public void testSubAllMessageWithNullTag() {\n        String tag = null;\n        String subExpress = \"*\";\n        int msgSize = 10;\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, subExpress,\n            new RMQNormalListener());\n        producer.send(tag, msgSize);\n        Assert.assertEquals(\"Not all are sent\", msgSize, producer.getAllUndupMsgBody().size());\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n\n    @Test\n    public void testSubNullWithTagNull() {\n        String tag = null;\n        String subExpress = null;\n        int msgSize = 10;\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, subExpress,\n            new RMQNormalListener());\n        producer.send(tag, msgSize);\n        Assert.assertEquals(\"Not all are sent\", msgSize, producer.getAllUndupMsgBody().size());\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n\n    @Test\n    public void testSubAllWithKindsOfMessage() {\n        String tag1 = null;\n        String tag2 = \"jueyin\";\n        String subExpress = \"*\";\n        int msgSize = 10;\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, subExpress,\n            new RMQNormalListener());\n\n        List<Object> tag1Msgs = MQMessageFactory.getRMQMessage(tag1, topic, msgSize);\n        List<Object> tag2Msgs = MQMessageFactory.getRMQMessage(tag2, topic, msgSize);\n\n        producer.send(tag1Msgs);\n        producer.send(tag2Msgs);\n        producer.send(10);\n        Assert.assertEquals(\"Not all are sent\", msgSize * 3, producer.getAllUndupMsgBody().size());\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n\n    @Test\n    public void testSubNullWithKindsOfMessage() {\n        String tag1 = null;\n        String tag2 = \"jueyin\";\n        String subExpress = null;\n        int msgSize = 10;\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, subExpress,\n            new RMQNormalListener());\n\n        List<Object> tag1Msgs = MQMessageFactory.getRMQMessage(tag1, topic, msgSize);\n        List<Object> tag2Msgs = MQMessageFactory.getRMQMessage(tag2, topic, msgSize);\n\n        producer.send(tag1Msgs);\n        producer.send(tag2Msgs);\n        Assert.assertEquals(\"Not all are sent\", msgSize * 2, producer.getAllUndupMsgBody().size());\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n\n    @Test\n    public void testSubTagWithKindsOfMessage() {\n        String tag1 = null;\n        String tag2 = \"jueyin\";\n        String subExpress = tag2;\n        int msgSize = 10;\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, subExpress,\n            new RMQNormalListener());\n\n        List<Object> tag1Msgs = MQMessageFactory.getRMQMessage(tag1, topic, msgSize);\n        List<Object> tag2Msgs = MQMessageFactory.getRMQMessage(tag2, topic, msgSize);\n\n        producer.send(tag1Msgs);\n        producer.send(tag2Msgs);\n        producer.send(10);\n        Assert.assertEquals(\"Not all are sent\", msgSize * 3, producer.getAllUndupMsgBody().size());\n        consumer.getListener().waitForMessageConsume(MQMessageFactory.getMessageBody(tag2Msgs),\n            CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(MQMessageFactory.getMessageBody(tag2Msgs));\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWithMulConsumerIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.tag;\n\nimport java.util.Collection;\nimport java.util.List;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.factory.MQMessageFactory;\nimport org.apache.rocketmq.test.factory.TagMessage;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class TagMessageWithMulConsumerIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(TagMessageWith1ConsumerIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        String consumerId = initConsumerGroup();\n        logger.info(String.format(\"use topic: %s; consumerId: %s !\", topic, consumerId));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testSendTwoTag() {\n        String tag1 = \"jueyin1\";\n        String tag2 = \"jueyin2\";\n        int msgSize = 10;\n        RMQNormalConsumer consumerTag1 = getConsumer(NAMESRV_ADDR, topic, tag1,\n            new RMQNormalListener());\n        RMQNormalConsumer consumerTag2 = getConsumer(NAMESRV_ADDR, topic, tag2,\n            new RMQNormalListener());\n\n        List<Object> tag1Msgs = MQMessageFactory.getRMQMessage(tag1, topic, msgSize);\n        producer.send(tag1Msgs);\n        Assert.assertEquals(\"Not all are sent\", msgSize, producer.getAllUndupMsgBody().size());\n        List<Object> tag2Msgs = MQMessageFactory.getRMQMessage(tag2, topic, msgSize);\n        producer.send(tag2Msgs);\n        Assert.assertEquals(\"Not all are sent\", msgSize * 2, producer.getAllUndupMsgBody().size());\n\n        consumerTag1.getListener().waitForMessageConsume(MQMessageFactory.getMessageBody(tag1Msgs),\n            CONSUME_TIME);\n        consumerTag2.getListener().waitForMessageConsume(MQMessageFactory.getMessageBody(tag2Msgs),\n            CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumerTag1.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(MQMessageFactory.getMessageBody(tag1Msgs));\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumerTag2.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(MQMessageFactory.getMessageBody(tag2Msgs));\n    }\n\n    @Test\n    public void testSendMessagesWithTwoTag() {\n        String[] tags = {\"jueyin1\", \"jueyin2\"};\n        int msgSize = 10;\n\n        TagMessage tagMessage = new TagMessage(tags, topic, msgSize);\n        RMQNormalConsumer consumerTag1 = getConsumer(NAMESRV_ADDR, topic, tags[0],\n            new RMQNormalListener());\n        RMQNormalConsumer consumerTag2 = getConsumer(NAMESRV_ADDR, topic, tags[1],\n            new RMQNormalListener());\n\n        List<Object> tagMsgs = tagMessage.getMixedTagMessages();\n        producer.send(tagMsgs);\n        Assert.assertEquals(\"Not all are sent\", msgSize * tags.length,\n            producer.getAllUndupMsgBody().size());\n\n        consumerTag1.getListener().waitForMessageConsume(tagMessage.getMessageBodyByTag(tags[0]),\n            CONSUME_TIME);\n        consumerTag2.getListener().waitForMessageConsume(tagMessage.getMessageBodyByTag(tags[1]),\n            CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumerTag1.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(tagMessage.getMessageBodyByTag(tags[0]));\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumerTag2.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(tagMessage.getMessageBodyByTag(tags[1]));\n    }\n\n    @Test\n    public void testTwoConsumerOneMatchOneOtherMatchAll() {\n        String[] tags = {\"jueyin1\", \"jueyin2\"};\n        String sub1 = String.format(\"%s||%s\", tags[0], tags[1]);\n        String sub2 = String.format(\"%s|| noExist\", tags[0]);\n        int msgSize = 10;\n\n        TagMessage tagMessage = new TagMessage(tags, topic, msgSize);\n        RMQNormalConsumer consumerTag1 = getConsumer(NAMESRV_ADDR, topic, sub1,\n            new RMQNormalListener());\n        RMQNormalConsumer consumerTag2 = getConsumer(NAMESRV_ADDR, topic, sub2,\n            new RMQNormalListener());\n\n        List<Object> tagMsgs = tagMessage.getMixedTagMessages();\n        producer.send(tagMsgs);\n        Assert.assertEquals(\"Not all are sent\", msgSize * tags.length,\n            producer.getAllUndupMsgBody().size());\n\n        consumerTag1.getListener().waitForMessageConsume(tagMessage.getMessageBodyByTag(tags),\n            CONSUME_TIME);\n        consumerTag2.getListener().waitForMessageConsume(tagMessage.getMessageBodyByTag(tags[0]),\n            CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumerTag1.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(tagMessage.getAllTagMessageBody());\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumerTag2.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(tagMessage.getMessageBodyByTag(tags[0]));\n    }\n\n    @Test\n    public void testSubKindsOf() {\n        String[] tags = {\"jueyin1\", \"jueyin2\"};\n        String sub1 = String.format(\"%s||%s\", tags[0], tags[1]);\n        String sub2 = String.format(\"%s|| noExist\", tags[0]);\n        String sub3 = tags[0];\n        String sub4 = \"*\";\n        int msgSize = 10;\n\n        RMQNormalConsumer consumerSubTwoMatchAll = getConsumer(NAMESRV_ADDR, topic, sub1,\n            new RMQNormalListener());\n        RMQNormalConsumer consumerSubTwoMachieOne = getConsumer(NAMESRV_ADDR, topic, sub2,\n            new RMQNormalListener());\n        RMQNormalConsumer consumerSubTag1 = getConsumer(NAMESRV_ADDR, topic, sub3,\n            new RMQNormalListener());\n        RMQNormalConsumer consumerSubAll = getConsumer(NAMESRV_ADDR, topic, sub4,\n            new RMQNormalListener());\n\n        producer.send(msgSize);\n        Assert.assertEquals(\"Not all are sent\", msgSize, producer.getAllUndupMsgBody().size());\n        Collection<Object> msgsWithNoTag = producer.getMsgBodysCopy();\n\n        TagMessage tagMessage = new TagMessage(tags, topic, msgSize);\n        List<Object> tagMsgs = tagMessage.getMixedTagMessages();\n        producer.send(tagMsgs);\n        Assert.assertEquals(\"Not all are sent\", msgSize * 3, producer.getAllUndupMsgBody().size());\n\n        consumerSubTwoMatchAll.getListener()\n            .waitForMessageConsume(tagMessage.getMessageBodyByTag(tags), CONSUME_TIME);\n        consumerSubTwoMachieOne.getListener()\n            .waitForMessageConsume(tagMessage.getMessageBodyByTag(tags[0]), CONSUME_TIME);\n        consumerSubTag1.getListener().waitForMessageConsume(tagMessage.getMessageBodyByTag(tags[0]),\n            CONSUME_TIME);\n        consumerSubAll.getListener().waitForMessageConsume(\n            MQMessageFactory.getMessage(msgsWithNoTag, tagMessage.getAllTagMessageBody()),\n            CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumerSubTwoMatchAll.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(tagMessage.getAllTagMessageBody());\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumerSubTwoMachieOne.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(tagMessage.getMessageBodyByTag(tags[0]));\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumerSubTag1.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(tagMessage.getMessageBodyByTag(tags[0]));\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumerSubAll.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(MQMessageFactory.getMessage(msgsWithNoTag,\n                tagMessage.getAllTagMessageBody()));\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWithSameGroupConsumerIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.tag;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.RandomUtils;\nimport org.apache.rocketmq.test.util.TestUtils;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class TagMessageWithSameGroupConsumerIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(TagMessageWith1ConsumerIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n    private String tag = \"tag\";\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s !\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testTwoConsumerWithSameGroup() {\n        int msgSize = 20;\n        String originMsgDCName = RandomUtils.getStringByUUID();\n        String msgBodyDCName = RandomUtils.getStringByUUID();\n        RMQNormalConsumer consumer1 = getConsumer(NAMESRV_ADDR, topic, tag,\n            new RMQNormalListener(originMsgDCName, msgBodyDCName));\n        getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), tag,\n            new RMQNormalListener(originMsgDCName, msgBodyDCName));\n        producer.send(tag, msgSize);\n        Assert.assertEquals(\"Not all are sent\", msgSize, producer.getAllUndupMsgBody().size());\n        consumer1.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer1.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n\n    @Test\n    public void testConsumerStartWithInterval() {\n        int msgSize = 100;\n        String originMsgDCName = RandomUtils.getStringByUUID();\n        String msgBodyDCName = RandomUtils.getStringByUUID();\n\n        RMQNormalConsumer consumer1 = getConsumer(NAMESRV_ADDR, topic, tag,\n            new RMQNormalListener(originMsgDCName, msgBodyDCName));\n        producer.send(tag, msgSize, 100);\n        TestUtils.waitForMoment(5);\n        getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), tag,\n            new RMQNormalListener(originMsgDCName, msgBodyDCName));\n        TestUtils.waitForMoment(5);\n\n        consumer1.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer1.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n\n    @Test\n    public void testConsumerStartTwoAndCrashOneAfterWhile() {\n        int msgSize = 100;\n        String originMsgDCName = RandomUtils.getStringByUUID();\n        String msgBodyDCName = RandomUtils.getStringByUUID();\n\n        RMQNormalConsumer consumer1 = getConsumer(NAMESRV_ADDR, topic, tag,\n            new RMQNormalListener(originMsgDCName, msgBodyDCName));\n        RMQNormalConsumer consumer2 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), tag,\n            new RMQNormalListener(originMsgDCName, msgBodyDCName));\n\n        producer.send(tag, msgSize, 100);\n        TestUtils.waitForMoment(5);\n        consumer2.shutdown();\n        mqClients.remove(1);\n        TestUtils.waitForMoment(5);\n\n        consumer1.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer1.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/topic/MulConsumerMulTopicIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.topic;\n\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.factory.MQMessageFactory;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.MQWait;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class MulConsumerMulTopicIT extends BaseConf {\n    private RMQNormalProducer producer = null;\n\n    @Before\n    public void setUp() {\n        producer = getProducer(NAMESRV_ADDR, null);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testSynSendMessage() {\n        int msgSize = 10;\n        String topic1 = initTopic();\n        String topic2 = initTopic();\n        RMQNormalConsumer consumer1 = getConsumer(NAMESRV_ADDR, topic1, \"*\", new RMQNormalListener());\n        consumer1.subscribe(topic2, \"*\");\n        RMQNormalConsumer consumer2 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic1,\n            \"*\", new RMQNormalListener());\n        consumer2.subscribe(topic2, \"*\");\n\n        producer.send(MQMessageFactory.getMsg(topic1, msgSize));\n        producer.send(MQMessageFactory.getMsg(topic2, msgSize));\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize * 2, producer.getAllUndupMsgBody().size());\n\n        boolean recvAll = MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(),\n            consumer1.getListener(), consumer2.getListener());\n        assertThat(recvAll).isEqualTo(true);\n    }\n\n    @Test\n    public void testConsumeWithDiffTag() {\n        int msgSize = 10;\n        String topic1 = initTopic();\n        String topic2 = initTopic();\n        String tag = \"jueyin_tag\";\n        RMQNormalConsumer consumer1 = getConsumer(NAMESRV_ADDR, topic1, \"*\", new RMQNormalListener());\n        consumer1.subscribe(topic2, tag);\n        RMQNormalConsumer consumer2 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic1,\n            \"*\", new RMQNormalListener());\n        consumer2.subscribe(topic2, tag);\n\n        producer.send(MQMessageFactory.getMsg(topic1, msgSize));\n        producer.send(MQMessageFactory.getMsg(topic2, msgSize, tag));\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize * 2, producer.getAllUndupMsgBody().size());\n\n        boolean recvAll = MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(),\n            consumer1.getListener(), consumer2.getListener());\n        assertThat(recvAll).isEqualTo(true);\n    }\n\n    @Test\n    public void testConsumeWithDiffTagAndFilter() {\n        int msgSize = 10;\n        String topic1 = initTopic();\n        String topic2 = initTopic();\n        String tag1 = \"jueyin_tag_1\";\n        String tag2 = \"jueyin_tag_2\";\n        RMQNormalConsumer consumer1 = getConsumer(NAMESRV_ADDR, topic1, \"*\", new RMQNormalListener());\n        consumer1.subscribe(topic2, tag1);\n        RMQNormalConsumer consumer2 = getConsumer(NAMESRV_ADDR, topic1, \"*\", new RMQNormalListener());\n        consumer2.subscribe(topic2, tag1);\n\n        producer.send(MQMessageFactory.getMsg(topic2, msgSize, tag2));\n        producer.clearMsg();\n        producer.send(MQMessageFactory.getMsg(topic1, msgSize));\n        producer.send(MQMessageFactory.getMsg(topic2, msgSize, tag1));\n\n        boolean recvAll = MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(),\n            consumer1.getListener(), consumer2.getListener());\n        assertThat(recvAll).isEqualTo(true);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/consumer/topic/OneConsumerMulTopicIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.consumer.topic;\n\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.factory.MQMessageFactory;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class OneConsumerMulTopicIT extends BaseConf {\n    private RMQNormalProducer producer = null;\n\n    @Before\n    public void setUp() {\n        producer = getProducer(NAMESRV_ADDR, null);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testSynSendMessage() {\n        int msgSize = 10;\n        String topic1 = initTopic();\n        String topic2 = initTopic();\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic1, \"*\", new RMQNormalListener());\n        consumer.subscribe(topic2, \"*\");\n\n        producer.send(MQMessageFactory.getMsg(topic1, msgSize));\n        producer.send(MQMessageFactory.getMsg(topic2, msgSize));\n\n        Assert.assertEquals(\"Not all are sent\", msgSize * 2, producer.getAllUndupMsgBody().size());\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n\n    @Test\n    public void testConsumeWithDiffTag() {\n        int msgSize = 10;\n        String topic1 = initTopic();\n        String topic2 = initTopic();\n        String tag = \"jueyin_tag\";\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic1, \"*\", new RMQNormalListener());\n        consumer.subscribe(topic2, tag);\n\n        producer.send(MQMessageFactory.getMsg(topic1, msgSize));\n        producer.send(MQMessageFactory.getMsg(topic2, msgSize, tag));\n\n        Assert.assertEquals(\"Not all are sent\", msgSize * 2, producer.getAllUndupMsgBody().size());\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n\n    @Test\n    public void testConsumeWithDiffTagAndFilter() {\n        int msgSize = 10;\n        String topic1 = initTopic();\n        String topic2 = initTopic();\n        String tag1 = \"jueyin_tag_1\";\n        String tag2 = \"jueyin_tag_2\";\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic1, \"*\", new RMQNormalListener());\n        consumer.subscribe(topic2, tag1);\n\n        producer.send(MQMessageFactory.getMsg(topic2, msgSize, tag2));\n        producer.clearMsg();\n        producer.send(MQMessageFactory.getMsg(topic1, msgSize));\n        producer.send(MQMessageFactory.getMsg(topic2, msgSize, tag1));\n\n        Assert.assertEquals(\"Not all are sent\", msgSize * 2, producer.getAllUndupMsgBody().size());\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendExceptionIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.producer.async;\n\nimport java.util.List;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.MessageQueueSelector;\nimport org.apache.rocketmq.client.producer.SendCallback;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.consumer.tag.TagMessageWith1ConsumerIT;\nimport org.apache.rocketmq.test.factory.ProducerFactory;\nimport org.apache.rocketmq.test.factory.SendCallBackFactory;\nimport org.apache.rocketmq.test.util.RandomUtils;\nimport org.apache.rocketmq.test.util.TestUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class AsyncSendExceptionIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(TagMessageWith1ConsumerIT.class);\n    private static boolean sendFail = false;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"user topic[%s]!\", topic));\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testSendCallBackNull() throws Exception {\n        Message msg = new Message(topic, RandomUtils.getStringByUUID().getBytes());\n        DefaultMQProducer producer = ProducerFactory.getRMQProducer(NAMESRV_ADDR);\n        SendCallback sendCallback = null;\n        producer.send(msg, sendCallback);\n    }\n\n    @Test\n    public void testSendMQNull() throws Exception {\n        Message msg = new Message(topic, RandomUtils.getStringByUUID().getBytes());\n        DefaultMQProducer producer = ProducerFactory.getRMQProducer(NAMESRV_ADDR);\n        MessageQueue messageQueue = null;\n        producer.send(msg, messageQueue, SendCallBackFactory.getSendCallBack());\n    }\n\n    @Test\n    public void testSendSelectorNull() throws Exception {\n        Message msg = new Message(topic, RandomUtils.getStringByUUID().getBytes());\n        DefaultMQProducer producer = ProducerFactory.getRMQProducer(NAMESRV_ADDR);\n        MessageQueueSelector selector = null;\n        producer.send(msg, selector, 100, SendCallBackFactory.getSendCallBack());\n    }\n\n    @Test\n    public void testSelectorThrowsException() throws Exception {\n        Message msg = new Message(topic, RandomUtils.getStringByUUID().getBytes());\n        DefaultMQProducer producer = ProducerFactory.getRMQProducer(NAMESRV_ADDR);\n        producer.send(msg, new MessageQueueSelector() {\n            @Override\n            public MessageQueue select(List<MessageQueue> list, Message message, Object o) {\n                String str = null;\n                return list.get(str.length());\n            }\n        }, null, SendCallBackFactory.getSendCallBack());\n    }\n\n    @Test\n    public void testQueueIdBigThanQueueNum() throws Exception {\n        int queueId = 100;\n        sendFail = false;\n        MessageQueue mq = new MessageQueue(topic, BROKER1_NAME, queueId);\n        Message msg = new Message(topic, RandomUtils.getStringByUUID().getBytes());\n        DefaultMQProducer producer = ProducerFactory.getRMQProducer(NAMESRV_ADDR);\n\n        producer.send(msg, mq, new SendCallback() {\n            @Override\n            public void onSuccess(SendResult sendResult) {\n            }\n\n            @Override\n            public void onException(Throwable throwable) {\n                sendFail = true;\n            }\n        });\n\n        int checkNum = 50;\n        while (!sendFail && checkNum > 0) {\n            checkNum--;\n            TestUtils.waitForMoment(100);\n        }\n        producer.shutdown();\n        assertThat(sendFail).isEqualTo(true);\n    }\n\n    @Test\n    public void testQueueIdSmallZero() throws Exception {\n        int queueId = -100;\n        sendFail = true;\n        MessageQueue mq = new MessageQueue(topic, BROKER1_NAME, queueId);\n        Message msg = new Message(topic, RandomUtils.getStringByUUID().getBytes());\n        DefaultMQProducer producer = ProducerFactory.getRMQProducer(NAMESRV_ADDR);\n\n        producer.send(msg, mq, new SendCallback() {\n            @Override\n            public void onSuccess(SendResult sendResult) {\n                sendFail = false;\n            }\n\n            @Override\n            public void onException(Throwable throwable) {\n                sendFail = true;\n            }\n        });\n\n        int checkNum = 50;\n        while (sendFail && checkNum > 0) {\n            checkNum--;\n            TestUtils.waitForMoment(100);\n        }\n        producer.shutdown();\n        assertThat(sendFail).isEqualTo(false);\n    }\n\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.producer.async;\n\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.consumer.tag.TagMessageWith1ConsumerIT;\nimport org.apache.rocketmq.test.client.rmq.RMQAsyncSendProducer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class AsyncSendWithMessageQueueIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(TagMessageWith1ConsumerIT.class);\n    private RMQAsyncSendProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"user topic[%s]!\", topic));\n        producer = getAsyncProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testAsyncSendWithMQ() {\n        int msgSize = 20;\n        int queueId = 0;\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n        MessageQueue mq = new MessageQueue(topic, BROKER1_NAME, queueId);\n\n        producer.asyncSend(msgSize, mq);\n        producer.waitForResponse(10 * 1000);\n        assertThat(producer.getSuccessMsgCount()).isEqualTo(msgSize);\n\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n\n        VerifyUtils.verifyMessageQueueId(queueId, consumer.getListener().getAllOriginMsg());\n\n        producer.clearMsg();\n        consumer.clearMsg();\n        producer.getSuccessSendResult().clear();\n        mq = new MessageQueue(topic, BROKER2_NAME, queueId);\n        producer.asyncSend(msgSize, mq);\n        producer.waitForResponse(10 * 1000);\n        assertThat(producer.getSuccessMsgCount()).isEqualTo(msgSize);\n\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n\n        VerifyUtils.verifyMessageQueueId(queueId, consumer.getListener().getAllOriginMsg());\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithMessageQueueSelectorIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.producer.async;\n\nimport java.util.List;\nimport org.apache.rocketmq.client.producer.MessageQueueSelector;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.consumer.tag.TagMessageWith1ConsumerIT;\nimport org.apache.rocketmq.test.client.rmq.RMQAsyncSendProducer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class AsyncSendWithMessageQueueSelectorIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(TagMessageWith1ConsumerIT.class);\n    private RMQAsyncSendProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"user topic[%s]!\", topic));\n        producer = getAsyncProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testSendWithSelector() {\n        int msgSize = 20;\n        final int queueId = 0;\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n\n        producer.asyncSend(msgSize, new MessageQueueSelector() {\n            @Override\n            public MessageQueue select(List<MessageQueue> list, Message message, Object o) {\n                for (MessageQueue mq : list) {\n                    if (mq.getQueueId() == queueId && mq.getBrokerName().equals(BROKER1_NAME)) {\n                        return mq;\n                    }\n                }\n                return list.get(0);\n            }\n        });\n        producer.waitForResponse(5 * 1000);\n        assertThat(producer.getSuccessMsgCount()).isEqualTo(msgSize);\n\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n\n        VerifyUtils.verifyMessageQueueId(queueId, consumer.getListener().getAllOriginMsg());\n\n        producer.clearMsg();\n        consumer.clearMsg();\n        producer.getSuccessSendResult().clear();\n\n        producer.asyncSend(msgSize, new MessageQueueSelector() {\n            @Override\n            public MessageQueue select(List<MessageQueue> list, Message message, Object o) {\n                for (MessageQueue mq : list) {\n                    if (mq.getQueueId() == queueId && mq.getBrokerName().equals(BROKER2_NAME)) {\n                        return mq;\n                    }\n                }\n                return list.get(8);\n            }\n        });\n        producer.waitForResponse(5 * 1000);\n        assertThat(producer.getSuccessMsgCount()).isEqualTo(msgSize);\n\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n\n        VerifyUtils.verifyMessageQueueId(queueId, consumer.getListener().getAllOriginMsg());\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/producer/async/AsyncSendWithOnlySendCallBackIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.producer.async;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.consumer.tag.TagMessageWith1ConsumerIT;\nimport org.apache.rocketmq.test.client.rmq.RMQAsyncSendProducer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class AsyncSendWithOnlySendCallBackIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(TagMessageWith1ConsumerIT.class);\n    private RMQAsyncSendProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"user topic[%s]!\", topic));\n        producer = getAsyncProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testSendWithOnlyCallBack() {\n        int msgSize = 20;\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n        producer.asyncSend(msgSize);\n        producer.waitForResponse(10 * 1000);\n        assertThat(producer.getSuccessMsgCount()).isEqualTo(msgSize);\n\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/producer/batch/BatchSendIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.producer.batch;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Random;\nimport java.util.UUID;\n\nimport org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.client.hook.SendMessageContext;\nimport org.apache.rocketmq.client.hook.SendMessageHook;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.TopicAttributes;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageBatch;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.base.IntegrationTestBase;\nimport org.apache.rocketmq.test.client.consumer.tag.TagMessageWith1ConsumerIT;\nimport org.apache.rocketmq.test.factory.ConsumerFactory;\nimport org.apache.rocketmq.test.factory.ProducerFactory;\nimport org.apache.rocketmq.test.util.RandomUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\npublic class BatchSendIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(TagMessageWith1ConsumerIT.class);\n    private String topic = null;\n    private Random random = new Random();\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"user topic[%s]!\", topic));\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testBatchSend_ViewMessage() throws Exception {\n        Assert.assertTrue(brokerController1.getMessageStore() instanceof DefaultMessageStore);\n        Assert.assertTrue(brokerController2.getMessageStore() instanceof DefaultMessageStore);\n\n        List<Message> messageList = new ArrayList<>();\n        int batchNum = 100;\n        for (int i = 0; i < batchNum; i++) {\n            messageList.add(new Message(topic, RandomUtils.getStringByUUID().getBytes()));\n        }\n\n        DefaultMQProducer producer = ProducerFactory.getRMQProducer(NAMESRV_ADDR);\n        removeBatchUniqueId(producer);\n\n        SendResult sendResult = producer.send(messageList);\n        Assert.assertEquals(SendStatus.SEND_OK, sendResult.getSendStatus());\n\n        String[] offsetIds = sendResult.getOffsetMsgId().split(\",\");\n        String[] msgIds = sendResult.getMsgId().split(\",\");\n        Assert.assertEquals(messageList.size(), offsetIds.length);\n        Assert.assertEquals(messageList.size(), msgIds.length);\n\n        Thread.sleep(2000);\n\n        for (int i = 0; i < 3; i++) {\n            producer.viewMessage(topic, offsetIds[random.nextInt(batchNum)]);\n        }\n        for (int i = 0; i < 3; i++) {\n            producer.viewMessage(topic, msgIds[random.nextInt(batchNum)]);\n        }\n    }\n\n    @Test\n    public void testBatchSend_SysInnerBatch() throws Exception {\n        waitBrokerRegistered(NAMESRV_ADDR, CLUSTER_NAME, BROKER_NUM);\n\n        String batchTopic = UUID.randomUUID().toString();\n        IntegrationTestBase.initTopic(batchTopic, NAMESRV_ADDR, CLUSTER_NAME, CQType.BatchCQ);\n\n        Assert.assertEquals(CQType.BatchCQ.toString(), brokerController1.getTopicConfigManager().getTopicConfigTable().get(batchTopic).getAttributes().get(TopicAttributes.QUEUE_TYPE_ATTRIBUTE.getName()));\n        Assert.assertEquals(CQType.BatchCQ.toString(), brokerController2.getTopicConfigManager().getTopicConfigTable().get(batchTopic).getAttributes().get(TopicAttributes.QUEUE_TYPE_ATTRIBUTE.getName()));\n        Assert.assertEquals(CQType.BatchCQ.toString(), brokerController3.getTopicConfigManager().getTopicConfigTable().get(batchTopic).getAttributes().get(TopicAttributes.QUEUE_TYPE_ATTRIBUTE.getName()));\n        Assert.assertEquals(8, brokerController1.getTopicConfigManager().getTopicConfigTable().get(batchTopic).getReadQueueNums());\n        Assert.assertEquals(8, brokerController2.getTopicConfigManager().getTopicConfigTable().get(batchTopic).getReadQueueNums());\n        Assert.assertEquals(8, brokerController3.getTopicConfigManager().getTopicConfigTable().get(batchTopic).getReadQueueNums());\n        Assert.assertEquals(-1, brokerController1.getMessageStore().getMinOffsetInQueue(batchTopic, 0));\n        Assert.assertEquals(-1, brokerController2.getMessageStore().getMinOffsetInQueue(batchTopic, 0));\n        Assert.assertEquals(-1, brokerController3.getMessageStore().getMinOffsetInQueue(batchTopic, 0));\n        Assert.assertEquals(0, brokerController1.getMessageStore().getMaxOffsetInQueue(batchTopic, 0));\n        Assert.assertEquals(0, brokerController2.getMessageStore().getMaxOffsetInQueue(batchTopic, 0));\n        Assert.assertEquals(0, brokerController3.getMessageStore().getMaxOffsetInQueue(batchTopic, 0));\n\n        DefaultMQProducer producer = ProducerFactory.getRMQProducer(NAMESRV_ADDR);\n        MessageQueue messageQueue = producer.fetchPublishMessageQueues(batchTopic).iterator().next();\n\n        int batchCount = 10;\n        int batchNum = 10;\n        for (int i = 0; i < batchCount; i++) {\n            List<Message> messageList = new ArrayList<>();\n            for (int j = 0; j < batchNum; j++) {\n                messageList.add(new Message(batchTopic, RandomUtils.getStringByUUID().getBytes()));\n            }\n            SendResult sendResult = producer.send(messageList, messageQueue);\n            Assert.assertEquals(SendStatus.SEND_OK, sendResult.getSendStatus());\n            Assert.assertEquals(messageQueue.getQueueId(), sendResult.getMessageQueue().getQueueId());\n            Assert.assertEquals(i * batchNum, sendResult.getQueueOffset());\n            Assert.assertEquals(1, sendResult.getMsgId().split(\",\").length);\n        }\n        Thread.sleep(300);\n        {\n            DefaultMQPullConsumer defaultMQPullConsumer = ConsumerFactory.getRMQPullConsumer(NAMESRV_ADDR, \"group\");\n\n            PullResult pullResult = defaultMQPullConsumer.pullBlockIfNotFound(messageQueue, \"*\", 5, batchCount * batchNum);\n            Assert.assertEquals(PullStatus.FOUND, pullResult.getPullStatus());\n            Assert.assertEquals(0, pullResult.getMinOffset());\n            Assert.assertEquals(batchCount * batchNum, pullResult.getMaxOffset());\n            Assert.assertEquals(batchCount * batchNum, pullResult.getMsgFoundList().size());\n            MessageExt first = pullResult.getMsgFoundList().get(0);\n            for (int i = 0; i < pullResult.getMsgFoundList().size(); i++) {\n                MessageExt messageExt = pullResult.getMsgFoundList().get(i);\n                if (i % batchNum == 0) {\n                    first = messageExt;\n                }\n                Assert.assertEquals(i, messageExt.getQueueOffset());\n                Assert.assertEquals(batchTopic, messageExt.getTopic());\n                Assert.assertEquals(messageQueue.getQueueId(), messageExt.getQueueId());\n                Assert.assertEquals(first.getBornHostString(), messageExt.getBornHostString());\n                Assert.assertEquals(first.getBornHostNameString(), messageExt.getBornHostNameString());\n                Assert.assertEquals(first.getBornTimestamp(), messageExt.getBornTimestamp());\n                Assert.assertEquals(first.getStoreTimestamp(), messageExt.getStoreTimestamp());\n            }\n        }\n    }\n\n    @Test\n    public void testBatchSend_SysOuterBatch() throws Exception {\n        Assert.assertTrue(brokerController1.getMessageStore() instanceof DefaultMessageStore);\n        Assert.assertTrue(brokerController2.getMessageStore() instanceof DefaultMessageStore);\n        Assert.assertTrue(brokerController3.getMessageStore() instanceof DefaultMessageStore);\n\n        String batchTopic = UUID.randomUUID().toString();\n        IntegrationTestBase.initTopic(batchTopic, NAMESRV_ADDR, CLUSTER_NAME, CQType.SimpleCQ);\n        Assert.assertEquals(8, brokerController1.getTopicConfigManager().getTopicConfigTable().get(batchTopic).getReadQueueNums());\n        Assert.assertEquals(8, brokerController2.getTopicConfigManager().getTopicConfigTable().get(batchTopic).getReadQueueNums());\n        Assert.assertEquals(8, brokerController3.getTopicConfigManager().getTopicConfigTable().get(batchTopic).getReadQueueNums());\n        Assert.assertEquals(0, brokerController1.getMessageStore().getMinOffsetInQueue(batchTopic, 0));\n        Assert.assertEquals(0, brokerController2.getMessageStore().getMinOffsetInQueue(batchTopic, 0));\n        Assert.assertEquals(0, brokerController3.getMessageStore().getMinOffsetInQueue(batchTopic, 0));\n        Assert.assertEquals(0, brokerController1.getMessageStore().getMaxOffsetInQueue(batchTopic, 0));\n        Assert.assertEquals(0, brokerController2.getMessageStore().getMaxOffsetInQueue(batchTopic, 0));\n        Assert.assertEquals(0, brokerController3.getMessageStore().getMaxOffsetInQueue(batchTopic, 0));\n\n        DefaultMQProducer producer = ProducerFactory.getRMQProducer(NAMESRV_ADDR);\n        MessageQueue messageQueue = producer.fetchPublishMessageQueues(batchTopic).iterator().next();\n\n        int batchCount = 10;\n        int batchNum = 10;\n        for (int i = 0; i < batchCount; i++) {\n            List<Message> messageList = new ArrayList<>();\n            for (int j = 0; j < batchNum; j++) {\n                messageList.add(new Message(batchTopic, RandomUtils.getStringByUUID().getBytes()));\n            }\n            SendResult sendResult = producer.send(messageList, messageQueue);\n            Assert.assertEquals(SendStatus.SEND_OK, sendResult.getSendStatus());\n            Assert.assertEquals(messageQueue.getQueueId(), sendResult.getMessageQueue().getQueueId());\n            Assert.assertEquals(i * batchNum, sendResult.getQueueOffset());\n            Assert.assertEquals(10, sendResult.getMsgId().split(\",\").length);\n        }\n        Thread.sleep(300);\n        {\n            DefaultMQPullConsumer defaultMQPullConsumer = ConsumerFactory.getRMQPullConsumer(NAMESRV_ADDR, \"group\");\n\n            long startOffset = 5;\n            PullResult pullResult = defaultMQPullConsumer.pullBlockIfNotFound(messageQueue, \"*\", startOffset, batchCount * batchNum);\n            Assert.assertEquals(PullStatus.FOUND, pullResult.getPullStatus());\n            Assert.assertEquals(0, pullResult.getMinOffset());\n            Assert.assertEquals(batchCount * batchNum, pullResult.getMaxOffset());\n            Assert.assertEquals(batchCount * batchNum - startOffset, pullResult.getMsgFoundList().size());\n            for (int i = 0; i < pullResult.getMsgFoundList().size(); i++) {\n                MessageExt messageExt = pullResult.getMsgFoundList().get(i);\n                Assert.assertEquals(i + startOffset, messageExt.getQueueOffset());\n                Assert.assertEquals(batchTopic, messageExt.getTopic());\n                Assert.assertEquals(messageQueue.getQueueId(), messageExt.getQueueId());\n            }\n        }\n    }\n\n    @Test\n    public void testBatchSend_CheckProperties() throws Exception {\n        List<Message> messageList = new ArrayList<>();\n        Message message = new Message();\n        message.setTopic(topic);\n        message.setKeys(\"keys123\");\n        message.setTags(\"tags123\");\n        message.setWaitStoreMsgOK(false);\n        message.setBuyerId(\"buyerid123\");\n        message.setFlag(123);\n        message.setBody(\"body\".getBytes());\n        messageList.add(message);\n\n        DefaultMQProducer producer = ProducerFactory.getRMQProducer(NAMESRV_ADDR);\n        removeBatchUniqueId(producer);\n\n        SendResult sendResult = producer.send(messageList);\n        Assert.assertEquals(SendStatus.SEND_OK, sendResult.getSendStatus());\n\n        String[] offsetIds = sendResult.getOffsetMsgId().split(\",\");\n        String[] msgIds = sendResult.getMsgId().split(\",\");\n        Assert.assertEquals(messageList.size(), offsetIds.length);\n        Assert.assertEquals(messageList.size(), msgIds.length);\n\n        Thread.sleep(2000);\n\n        Message messageByOffset = producer.viewMessage(topic, offsetIds[0]);\n        Message messageByMsgId = producer.viewMessage(topic, msgIds[0]);\n\n        Assert.assertEquals(message.getTopic(), messageByMsgId.getTopic());\n        Assert.assertEquals(message.getTopic(), messageByOffset.getTopic());\n\n        Assert.assertEquals(message.getKeys(), messageByOffset.getKeys());\n        Assert.assertEquals(message.getKeys(), messageByMsgId.getKeys());\n\n        Assert.assertEquals(message.getTags(), messageByOffset.getTags());\n        Assert.assertEquals(message.getTags(), messageByMsgId.getTags());\n\n        Assert.assertEquals(message.isWaitStoreMsgOK(), messageByOffset.isWaitStoreMsgOK());\n        Assert.assertEquals(message.isWaitStoreMsgOK(), messageByMsgId.isWaitStoreMsgOK());\n\n        Assert.assertEquals(message.getBuyerId(), messageByOffset.getBuyerId());\n        Assert.assertEquals(message.getBuyerId(), messageByMsgId.getBuyerId());\n\n        Assert.assertEquals(message.getFlag(), messageByOffset.getFlag());\n        Assert.assertEquals(message.getFlag(), messageByMsgId.getFlag());\n    }\n\n    // simulate legacy batch message send.\n    private void removeBatchUniqueId(DefaultMQProducer producer) {\n        producer.getDefaultMQProducerImpl().registerSendMessageHook(new SendMessageHook() {\n            @Override\n            public String hookName() {\n                return null;\n            }\n\n            @Override\n            public void sendMessageBefore(SendMessageContext context) {\n                MessageBatch messageBatch = (MessageBatch) context.getMessage();\n                if (messageBatch.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX) != null) {\n                    messageBatch.getProperties().remove(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX);\n                }\n            }\n\n            @Override\n            public void sendMessageAfter(SendMessageContext context) {\n            }\n        });\n    }\n\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/producer/exception/msg/ChinaPropIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.producer.exception.msg;\n\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.factory.MessageFactory;\nimport org.apache.rocketmq.test.factory.ProducerFactory;\nimport org.apache.rocketmq.test.util.RandomUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class ChinaPropIT extends BaseConf {\n    private static DefaultMQProducer producer = null;\n    private static String topic = null;\n\n    @Before\n    public void setUp() {\n        producer = ProducerFactory.getRMQProducer(NAMESRV_ADDR);\n        topic = initTopic();\n    }\n\n    @After\n    public void tearDown() {\n        producer.shutdown();\n    }\n\n    /**\n     * @since version3.4.6\n     */\n    @Test(expected = org.apache.rocketmq.client.exception.MQBrokerException.class)\n    public void testSend20kChinaPropMsg() throws Exception {\n        Message msg = MessageFactory.getRandomMessage(topic);\n        msg.putUserProperty(\"key\", RandomUtils.getCheseWord(32 * 1024 + 1));\n        producer.send(msg);\n    }\n\n    /**\n     * @since version3.4.6\n     */\n    @Test\n    public void testSend10kChinaPropMsg() {\n\n        Message msg = MessageFactory.getRandomMessage(topic);\n        msg.putUserProperty(\"key\", RandomUtils.getCheseWord(10 * 1024));\n        SendResult sendResult = null;\n        try {\n            sendResult = producer.send(msg);\n        } catch (Exception e) {\n        }\n        assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/producer/exception/msg/MessageExceptionIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.producer.exception.msg;\n\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.factory.MessageFactory;\nimport org.apache.rocketmq.test.factory.ProducerFactory;\nimport org.apache.rocketmq.test.util.RandomUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class MessageExceptionIT extends BaseConf {\n    private static DefaultMQProducer producer = null;\n    private static String topic = null;\n\n    @Before\n    public void setUp() {\n        producer = ProducerFactory.getRMQProducer(NAMESRV_ADDR);\n        topic = initTopic();\n    }\n\n    @After\n    public void tearDown() {\n        producer.shutdown();\n    }\n\n    @Test\n    public void testProducerSmoke() {\n        Message msg = new Message(topic, RandomUtils.getStringByUUID().getBytes());\n        SendResult sendResult = null;\n        try {\n            sendResult = producer.send(msg);\n        } catch (Exception e) {\n        }\n\n        assertThat(sendResult).isNotEqualTo(null);\n        assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n    }\n\n    @Test(expected = java.lang.NullPointerException.class)\n    public void testSynSendNullMessage() throws Exception {\n        producer.send((Message) null);\n    }\n\n    @Test(expected = org.apache.rocketmq.client.exception.MQClientException.class)\n    public void testSynSendNullBodyMessage() throws Exception {\n        Message msg = new Message(topic, RandomUtils.getStringByUUID().getBytes());\n        msg.setBody(null);\n        producer.send(msg);\n    }\n\n    @Test(expected = org.apache.rocketmq.client.exception.MQClientException.class)\n    public void testSynSendZeroSizeBodyMessage() throws Exception {\n        Message msg = new Message(topic, RandomUtils.getStringByUUID().getBytes());\n        msg.setBody(new byte[0]);\n        producer.send(msg);\n    }\n\n    @Test(expected = org.apache.rocketmq.client.exception.MQClientException.class)\n    public void testSynSendOutOfSizeBodyMessage() throws Exception {\n        Message msg = new Message(topic, RandomUtils.getStringByUUID().getBytes());\n        msg.setBody(new byte[1024 * 1024 * 4 + 1]);\n        producer.send(msg);\n    }\n\n    @Test(expected = org.apache.rocketmq.client.exception.MQClientException.class)\n    public void testSynSendNullTopicMessage() throws Exception {\n        Message msg = new Message(null, RandomUtils.getStringByUUID().getBytes());\n        producer.send(msg);\n    }\n\n    @Test(expected = org.apache.rocketmq.client.exception.MQClientException.class)\n    public void testSynSendBlankTopicMessage() throws Exception {\n        Message msg = new Message(\"\", RandomUtils.getStringByUUID().getBytes());\n        producer.send(msg);\n    }\n\n    @Test(expected = org.apache.rocketmq.client.exception.MQClientException.class)\n    public void testSend128kMsg() throws Exception {\n        Message msg = new Message(topic,\n            RandomUtils.getStringWithNumber(1024 * 1024 * 4 + 1).getBytes());\n        producer.send(msg);\n    }\n\n    @Test\n    public void testSendLess128kMsg() {\n        Message msg = new Message(topic, RandomUtils.getStringWithNumber(128 * 1024).getBytes());\n        SendResult sendResult = null;\n        try {\n            sendResult = producer.send(msg);\n        } catch (Exception e) {\n        }\n        assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n    }\n\n    @Test\n    public void testSendMsgWithUserProperty() {\n        Message msg = MessageFactory.getRandomMessage(topic);\n        msg.putUserProperty(\"key\", RandomUtils.getCheseWord(10 * 1024));\n        SendResult sendResult = null;\n        try {\n            sendResult = producer.send(msg);\n        } catch (Exception e) {\n        }\n        assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/producer/exception/msg/MessageUserPropIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.producer.exception.msg;\n\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.consumer.balance.NormalMsgStaticBalanceIT;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.factory.MessageFactory;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class MessageUserPropIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(NormalMsgStaticBalanceIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s !\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    /**\n     * @since version3.4.6\n     */\n    @Test\n    public void testSendEnglishUserProp() {\n        Message msg = MessageFactory.getRandomMessage(topic);\n        String msgKey = \"jueyinKey\";\n        String msgValue = \"jueyinValue\";\n        msg.putUserProperty(msgKey, msgValue);\n\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n\n        producer.send(msg, null);\n        assertThat(producer.getAllMsgBody().size()).isEqualTo(1);\n\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        Message sendMsg = (Message) producer.getFirstMsg();\n        Message recvMsg = (Message) consumer.getListener().getFirstMsg();\n        assertThat(recvMsg.getUserProperty(msgKey)).isEqualTo(sendMsg.getUserProperty(msgKey));\n    }\n\n    /**\n     * @since version3.4.6\n     */\n    @Test\n    public void testSendChinaUserProp() {\n        Message msg = MessageFactory.getRandomMessage(topic);\n        String msgKey = \"jueyinKey\";\n        String msgValue = \"jueyinzhi\";\n        msg.putUserProperty(msgKey, msgValue);\n\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n\n        producer.send(msg, null);\n        assertThat(producer.getAllMsgBody().size()).isEqualTo(1);\n\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        Message sendMsg = (Message) producer.getFirstMsg();\n        Message recvMsg = (Message) consumer.getListener().getFirstMsg();\n        assertThat(recvMsg.getUserProperty(msgKey)).isEqualTo(sendMsg.getUserProperty(msgKey));\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/producer/exception/producer/ProducerGroupAndInstanceNameValidityIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.producer.exception.producer;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.util.RandomUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class ProducerGroupAndInstanceNameValidityIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(ProducerGroupAndInstanceNameValidityIT.class);\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s !\", topic));\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    /**\n     * @since version3.4.6\n     */\n    @Test\n    public void testTwoProducerSameGroupAndInstanceName() {\n        RMQNormalProducer producer1 = getProducer(NAMESRV_ADDR, topic);\n        assertThat(producer1.isStartSuccess()).isEqualTo(true);\n        RMQNormalProducer producer2 = getProducer(NAMESRV_ADDR, topic,\n            producer1.getProducerGroupName(), producer1.getProducerInstanceName());\n        assertThat(producer2.isStartSuccess()).isEqualTo(false);\n    }\n\n    /**\n     * @since version3.4.6\n     */\n    @Test\n    public void testTwoProducerSameGroup() {\n        RMQNormalProducer producer1 = getProducer(NAMESRV_ADDR, topic);\n        assertThat(producer1.isStartSuccess()).isEqualTo(true);\n        RMQNormalProducer producer2 = getProducer(NAMESRV_ADDR, topic,\n            producer1.getProducerGroupName(), RandomUtils.getStringByUUID());\n        assertThat(producer2.isStartSuccess()).isEqualTo(true);\n    }\n\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/producer/oneway/OneWaySendExceptionIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.producer.oneway;\n\nimport java.util.List;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.MessageQueueSelector;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.consumer.tag.TagMessageWith1ConsumerIT;\nimport org.apache.rocketmq.test.factory.ProducerFactory;\nimport org.apache.rocketmq.test.util.RandomUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\npublic class OneWaySendExceptionIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(TagMessageWith1ConsumerIT.class);\n    private static boolean sendFail = false;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"user topic[%s]!\", topic));\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test(expected = java.lang.NullPointerException.class)\n    public void testSendMQNull() throws Exception {\n        Message msg = new Message(topic, RandomUtils.getStringByUUID().getBytes());\n        DefaultMQProducer producer = ProducerFactory.getRMQProducer(NAMESRV_ADDR);\n        MessageQueue messageQueue = null;\n        producer.sendOneway(msg, messageQueue);\n    }\n\n    @Test(expected = org.apache.rocketmq.client.exception.MQClientException.class)\n    public void testSendSelectorNull() throws Exception {\n        Message msg = new Message(topic, RandomUtils.getStringByUUID().getBytes());\n        DefaultMQProducer producer = ProducerFactory.getRMQProducer(NAMESRV_ADDR);\n        MessageQueueSelector selector = null;\n        producer.sendOneway(msg, selector, 100);\n    }\n\n    @Test(expected = org.apache.rocketmq.client.exception.MQClientException.class)\n    public void testSelectorThrowsException() throws Exception {\n        Message msg = new Message(topic, RandomUtils.getStringByUUID().getBytes());\n        DefaultMQProducer producer = ProducerFactory.getRMQProducer(NAMESRV_ADDR);\n        producer.sendOneway(msg, new MessageQueueSelector() {\n            @Override\n            public MessageQueue select(List<MessageQueue> list, Message message, Object o) {\n                String str = null;\n                return list.get(str.length());\n            }\n        }, null);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/producer/oneway/OneWaySendIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.producer.oneway;\n\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.consumer.tag.TagMessageWith1ConsumerIT;\nimport org.apache.rocketmq.test.client.rmq.RMQAsyncSendProducer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class OneWaySendIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(TagMessageWith1ConsumerIT.class);\n    private RMQAsyncSendProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"user topic[%s]!\", topic));\n        producer = getAsyncProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testOneWaySendWithOnlyMsgAsParam() {\n        int msgSize = 20;\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n\n        producer.sendOneWay(msgSize);\n        producer.waitForResponse(5 * 1000);\n        assertThat(producer.getAllMsgBody().size()).isEqualTo(msgSize);\n\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/producer/oneway/OneWaySendWithMQIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.producer.oneway;\n\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.consumer.tag.TagMessageWith1ConsumerIT;\nimport org.apache.rocketmq.test.client.rmq.RMQAsyncSendProducer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class OneWaySendWithMQIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(TagMessageWith1ConsumerIT.class);\n    private static boolean sendFail = false;\n    private RMQAsyncSendProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"user topic[%s]!\", topic));\n        producer = getAsyncProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testAsyncSendWithMQ() {\n        int msgSize = 20;\n        int queueId = 0;\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n        MessageQueue mq = new MessageQueue(topic, BROKER1_NAME, queueId);\n\n        producer.sendOneWay(msgSize, mq);\n        producer.waitForResponse(5 * 1000);\n\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n\n\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/producer/oneway/OneWaySendWithSelectorIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.producer.oneway;\n\nimport java.util.List;\nimport org.apache.rocketmq.client.producer.MessageQueueSelector;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.consumer.tag.TagMessageWith1ConsumerIT;\nimport org.apache.rocketmq.test.client.rmq.RMQAsyncSendProducer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class OneWaySendWithSelectorIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(TagMessageWith1ConsumerIT.class);\n    private static boolean sendFail = false;\n    private RMQAsyncSendProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"user topic[%s]!\", topic));\n        producer = getAsyncProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testSendWithSelector() {\n        int msgSize = 20;\n        final int queueId = 0;\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n\n        producer.sendOneWay(msgSize, new MessageQueueSelector() {\n            @Override\n            public MessageQueue select(List<MessageQueue> list, Message message, Object o) {\n                for (MessageQueue mq : list) {\n                    if (mq.getQueueId() == queueId && mq.getBrokerName().equals(BROKER1_NAME)) {\n                        return mq;\n                    }\n                }\n                return list.get(0);\n            }\n        });\n        assertThat(producer.getAllMsgBody().size()).isEqualTo(msgSize);\n\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n\n        VerifyUtils.verifyMessageQueueId(queueId, consumer.getListener().getAllOriginMsg());\n\n        producer.clearMsg();\n        consumer.clearMsg();\n\n        producer.sendOneWay(msgSize, new MessageQueueSelector() {\n            @Override\n            public MessageQueue select(List<MessageQueue> list, Message message, Object o) {\n                for (MessageQueue mq : list) {\n                    if (mq.getQueueId() == queueId && mq.getBrokerName().equals(BROKER2_NAME)) {\n                        return mq;\n                    }\n                }\n                return list.get(8);\n            }\n        });\n        assertThat(producer.getAllMsgBody().size()).isEqualTo(msgSize);\n\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n\n        VerifyUtils.verifyMessageQueueId(queueId, consumer.getListener().getAllOriginMsg());\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/producer/order/OrderMsgDynamicRebalanceIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.producer.order;\n\nimport java.util.List;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.consumer.balance.NormalMsgStaticBalanceIT;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.order.RMQOrderListener;\nimport org.apache.rocketmq.test.message.MessageQueueMsg;\nimport org.apache.rocketmq.test.util.MQWait;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class OrderMsgDynamicRebalanceIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(NormalMsgStaticBalanceIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s !\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testTwoConsumerAndCrashOne() {\n        int msgSize = 10;\n        RMQNormalConsumer consumer1 = getConsumer(NAMESRV_ADDR, topic, \"*\",\n            new RMQOrderListener(\"1\"));\n        RMQNormalConsumer consumer2 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic,\n            \"*\", new RMQOrderListener(\"2\"));\n\n        List<MessageQueue> mqs = producer.getMessageQueue();\n        MessageQueueMsg mqMsgs = new MessageQueueMsg(mqs, msgSize);\n        producer.send(mqMsgs.getMsgsWithMQ());\n\n        MQWait.waitConsumeAll(30 * 1000, producer.getAllMsgBody(), consumer1.getListener(),\n            consumer2.getListener());\n        consumer2.shutdown();\n\n        mqMsgs = new MessageQueueMsg(mqs, msgSize);\n        producer.send(mqMsgs.getMsgsWithMQ());\n\n        boolean recvAll = MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(),\n            consumer1.getListener(), consumer2.getListener());\n        assertThat(recvAll).isEqualTo(true);\n\n        assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer1.getListener()).getMsgs()))\n            .isEqualTo(true);\n        assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer2.getListener()).getMsgs()))\n            .isEqualTo(true);\n    }\n\n    @Test\n    public void testThreeConsumerAndCrashOne() {\n        int msgSize = 10;\n        RMQNormalConsumer consumer1 = getConsumer(NAMESRV_ADDR, topic, \"*\",\n            new RMQOrderListener(\"1\"));\n        RMQNormalConsumer consumer2 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic,\n            \"*\", new RMQOrderListener(\"2\"));\n        RMQNormalConsumer consumer3 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic,\n            \"*\", new RMQOrderListener(\"3\"));\n\n        List<MessageQueue> mqs = producer.getMessageQueue();\n        MessageQueueMsg mqMsgs = new MessageQueueMsg(mqs, msgSize);\n        producer.send(mqMsgs.getMsgsWithMQ());\n\n        MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(), consumer1.getListener(),\n            consumer2.getListener(), consumer3.getListener());\n        consumer3.shutdown();\n\n        mqMsgs = new MessageQueueMsg(mqs, msgSize);\n        producer.send(mqMsgs.getMsgsWithMQ());\n\n        boolean recvAll = MQWait.waitConsumeAll(30 * 1000, producer.getAllMsgBody(),\n            consumer1.getListener(), consumer2.getListener(), consumer3.getListener());\n        assertThat(recvAll).isEqualTo(true);\n\n        assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer1.getListener()).getMsgs()))\n            .isEqualTo(true);\n        assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer2.getListener()).getMsgs()))\n            .isEqualTo(true);\n        assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer3.getListener()).getMsgs()))\n            .isEqualTo(true);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/producer/order/OrderMsgIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.producer.order;\n\nimport java.util.List;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.factory.MQMessageFactory;\nimport org.apache.rocketmq.test.listener.rmq.order.RMQOrderListener;\nimport org.apache.rocketmq.test.message.MessageQueueMsg;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class OrderMsgIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(OrderMsgIT.class);\n    private RMQNormalProducer producer = null;\n    private RMQNormalConsumer consumer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s;\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n        consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQOrderListener());\n    }\n\n    @After\n    public void tearDown() {\n        shutdown();\n    }\n\n    @Test\n    public void testOrderMsg() {\n        int msgSize = 10;\n        List<MessageQueue> mqs = producer.getMessageQueue();\n        MessageQueueMsg mqMsgs = new MessageQueueMsg(mqs, msgSize);\n        producer.send(mqMsgs.getMsgsWithMQ());\n\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(mqMsgs.getMsgBodys());\n\n        assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer.getListener()).getMsgs()))\n            .isEqualTo(true);\n    }\n\n    @Test\n    public void testSendOneQueue() {\n        int msgSize = 20;\n        List<MessageQueue> mqs = producer.getMessageQueue();\n        MessageQueueMsg mqMsgs = new MessageQueueMsg(MQMessageFactory.getMessageQueues(mqs.get(0)),\n            msgSize);\n        producer.send(mqMsgs.getMsgsWithMQ());\n\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(mqMsgs.getMsgBodys());\n\n        assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer.getListener()).getMsgs()))\n            .isEqualTo(true);\n    }\n\n    @Test\n    public void testSendRandomQueues() {\n        int msgSize = 10;\n        List<MessageQueue> mqs = producer.getMessageQueue();\n        MessageQueueMsg mqMsgs = new MessageQueueMsg(\n            MQMessageFactory.getMessageQueues(mqs.get(0), mqs.get(1), mqs.get(mqs.size() - 1)),\n            msgSize);\n        producer.send(mqMsgs.getMsgsWithMQ());\n\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(mqMsgs.getMsgBodys());\n\n        assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer.getListener()).getMsgs()))\n            .isEqualTo(true);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/producer/order/OrderMsgRebalanceIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.producer.order;\n\nimport java.util.List;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.order.RMQOrderListener;\nimport org.apache.rocketmq.test.message.MessageQueueMsg;\nimport org.apache.rocketmq.test.util.MQWait;\nimport org.apache.rocketmq.test.util.TestUtils;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class OrderMsgRebalanceIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(OrderMsgRebalanceIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s !\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testTwoConsumersBalance() {\n        int msgSize = 10;\n        RMQNormalConsumer consumer1 = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQOrderListener());\n        RMQNormalConsumer consumer2 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic,\n            \"*\", new RMQOrderListener());\n        TestUtils.waitForSeconds(WAIT_TIME);\n\n        List<MessageQueue> mqs = producer.getMessageQueue();\n        MessageQueueMsg mqMsgs = new MessageQueueMsg(mqs, msgSize);\n        producer.send(mqMsgs.getMsgsWithMQ());\n\n        boolean recvAll = MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(),\n            consumer1.getListener(), consumer2.getListener());\n        assertThat(recvAll).isEqualTo(true);\n\n        boolean balance = VerifyUtils.verifyBalance(producer.getAllMsgBody().size(),\n            VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n                consumer1.getListener().getAllUndupMsgBody()).size(),\n            VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n                consumer2.getListener().getAllUndupMsgBody()).size());\n        assertThat(balance).isEqualTo(true);\n\n        assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer1.getListener()).getMsgs()))\n            .isEqualTo(true);\n        assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer2.getListener()).getMsgs()))\n            .isEqualTo(true);\n    }\n\n    @Test\n    public void testFourConsumerBalance() {\n        int msgSize = 20;\n        RMQNormalConsumer consumer1 = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQOrderListener());\n        RMQNormalConsumer consumer2 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic,\n            \"*\", new RMQOrderListener());\n        RMQNormalConsumer consumer3 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic,\n            \"*\", new RMQOrderListener());\n        RMQNormalConsumer consumer4 = getConsumer(NAMESRV_ADDR, consumer1.getConsumerGroup(), topic,\n            \"*\", new RMQOrderListener());\n        TestUtils.waitForSeconds(WAIT_TIME);\n\n        List<MessageQueue> mqs = producer.getMessageQueue();\n        MessageQueueMsg mqMsgs = new MessageQueueMsg(mqs, msgSize);\n        producer.send(mqMsgs.getMsgsWithMQ());\n\n        boolean recvAll = MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(),\n            consumer1.getListener(), consumer2.getListener(), consumer3.getListener(),\n            consumer4.getListener());\n        assertThat(recvAll).isEqualTo(true);\n\n        boolean balance = VerifyUtils\n            .verifyBalance(producer.getAllMsgBody().size(),\n                VerifyUtils\n                    .getFilterdMessage(producer.getAllMsgBody(),\n                        consumer1.getListener().getAllUndupMsgBody())\n                    .size(),\n                VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n                    consumer2.getListener().getAllUndupMsgBody()).size(),\n                VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n                    consumer3.getListener().getAllUndupMsgBody()).size(),\n                VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n                    consumer4.getListener().getAllUndupMsgBody()).size());\n        logger.info(String.format(\"consumer1:%s;consumer2:%s;consumer3:%s,consumer4:%s\",\n            consumer1.getListener().getAllMsgBody().size(),\n            consumer2.getListener().getAllMsgBody().size(),\n            consumer3.getListener().getAllMsgBody().size(),\n            consumer4.getListener().getAllMsgBody().size()));\n        assertThat(balance).isEqualTo(true);\n\n        assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer1.getListener()).getMsgs()))\n            .isEqualTo(true);\n        assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer2.getListener()).getMsgs()))\n            .isEqualTo(true);\n        assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer3.getListener()).getMsgs()))\n            .isEqualTo(true);\n        assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer4.getListener()).getMsgs()))\n            .isEqualTo(true);\n    }\n\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/producer/order/OrderMsgWithTagIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.producer.order;\n\nimport java.util.List;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.order.RMQOrderListener;\nimport org.apache.rocketmq.test.message.MessageQueueMsg;\nimport org.apache.rocketmq.test.util.MQWait;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class OrderMsgWithTagIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(OrderMsgIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s;\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        shutdown();\n    }\n\n    @Test\n    public void testOrderMsgWithTagSubAll() {\n        int msgSize = 10;\n        String tag = \"jueyin_tag\";\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQOrderListener());\n\n        List<MessageQueue> mqs = producer.getMessageQueue();\n        MessageQueueMsg mqMsgs = new MessageQueueMsg(mqs, msgSize, tag);\n        producer.send(mqMsgs.getMsgsWithMQ());\n\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(mqMsgs.getMsgBodys());\n\n        assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer.getListener()).getMsgs()))\n            .isEqualTo(true);\n    }\n\n    @Test\n    public void testOrderMsgWithTagSubTag() {\n        int msgSize = 5;\n        String tag = \"jueyin_tag\";\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, tag, new RMQOrderListener());\n\n        List<MessageQueue> mqs = producer.getMessageQueue();\n        MessageQueueMsg mqMsgs = new MessageQueueMsg(mqs, msgSize, tag);\n        producer.send(mqMsgs.getMsgsWithMQ());\n\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(mqMsgs.getMsgBodys());\n\n        assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer.getListener()).getMsgs()))\n            .isEqualTo(true);\n    }\n\n    @Test\n    public void testOrderMsgWithTag1AndTag2SubTag1() {\n        int msgSize = 5;\n        String tag1 = \"jueyin_tag_1\";\n        String tag2 = \"jueyin_tag_2\";\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, tag1, new RMQOrderListener());\n\n        List<MessageQueue> mqs = producer.getMessageQueue();\n\n        MessageQueueMsg mqMsgs = new MessageQueueMsg(mqs, msgSize, tag2);\n        producer.send(mqMsgs.getMsgsWithMQ());\n        producer.clearMsg();\n\n        mqMsgs = new MessageQueueMsg(mqs, msgSize, tag1);\n        producer.send(mqMsgs.getMsgsWithMQ());\n\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(mqMsgs.getMsgBodys());\n\n        assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer.getListener()).getMsgs()))\n            .isEqualTo(true);\n    }\n\n    @Test\n    public void testTwoConsumerSubTag() {\n        int msgSize = 10;\n        String tag1 = \"jueyin_tag_1\";\n        String tag2 = \"jueyin_tag_2\";\n        RMQNormalConsumer consumer1 = getConsumer(NAMESRV_ADDR, topic, tag1,\n            new RMQOrderListener(\"consumer1\"));\n        RMQNormalConsumer consumer2 = getConsumer(NAMESRV_ADDR, topic, tag2,\n            new RMQOrderListener(\"consumer2\"));\n        List<MessageQueue> mqs = producer.getMessageQueue();\n\n        MessageQueueMsg mqMsgs = new MessageQueueMsg(mqs, msgSize, tag1);\n        producer.send(mqMsgs.getMsgsWithMQ());\n\n        mqMsgs = new MessageQueueMsg(mqs, msgSize, tag2);\n        producer.send(mqMsgs.getMsgsWithMQ());\n\n        boolean recvAll = MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(),\n            consumer1.getListener(), consumer2.getListener());\n        assertThat(recvAll).isEqualTo(true);\n\n        assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer1.getListener()).getMsgs()))\n            .isEqualTo(true);\n        assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer2.getListener()).getMsgs()))\n            .isEqualTo(true);\n    }\n\n    @Test\n    public void testConsumeTwoTag() {\n        int msgSize = 10;\n        String tag1 = \"jueyin_tag_1\";\n        String tag2 = \"jueyin_tag_2\";\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic,\n            String.format(\"%s||%s\", tag1, tag2), new RMQOrderListener());\n\n        List<MessageQueue> mqs = producer.getMessageQueue();\n\n        MessageQueueMsg mqMsgs = new MessageQueueMsg(mqs, msgSize, tag1);\n        producer.send(mqMsgs.getMsgsWithMQ());\n\n        mqMsgs = new MessageQueueMsg(mqs, msgSize, tag2);\n        producer.send(mqMsgs.getMsgsWithMQ());\n\n        boolean recvAll = MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(),\n            consumer.getListener());\n        assertThat(recvAll).isEqualTo(true);\n\n        assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer.getListener()).getMsgs()))\n            .isEqualTo(true);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/producer/querymsg/QueryMsgByIdExceptionIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.producer.querymsg;\n\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.junit.AfterClass;\nimport org.junit.Assert;\nimport org.junit.BeforeClass;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class QueryMsgByIdExceptionIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(QueryMsgByKeyIT.class);\n    private static RMQNormalProducer producer = null;\n    private static String topic = null;\n\n    @BeforeClass\n    public static void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s;\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @AfterClass\n    public static void tearDown() {\n        shutdown();\n    }\n\n    @Test\n    public void testQueryMsgByErrorMsgId() {\n        producer.clearMsg();\n        int msgSize = 20;\n        String errorMsgId = \"errorMsgId\";\n        producer.send(msgSize);\n        Assert.assertEquals(\"Not all are sent\", msgSize, producer.getAllUndupMsgBody().size());\n\n        MessageExt queryMsg = null;\n        try {\n            queryMsg = producer.getProducer().viewMessage(\"topic\", errorMsgId);\n        } catch (Exception e) {\n        }\n\n        assertThat(queryMsg).isNull();\n    }\n\n    @Test\n    public void testQueryMsgByNullMsgId() {\n        producer.clearMsg();\n        int msgSize = 20;\n        String errorMsgId = null;\n        producer.send(msgSize);\n        Assert.assertEquals(\"Not all are sent\", msgSize, producer.getAllUndupMsgBody().size());\n\n        MessageExt queryMsg = null;\n        try {\n            queryMsg = producer.getProducer().viewMessage(\"topic\", errorMsgId);\n        } catch (Exception e) {\n        }\n\n        assertThat(queryMsg).isNull();\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/producer/querymsg/QueryMsgByIdIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.producer.querymsg;\n\nimport org.apache.rocketmq.common.message.MessageClientExt;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.TestUtils;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class QueryMsgByIdIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(QueryMsgByIdIT.class);\n    private RMQNormalProducer producer = null;\n    private RMQNormalConsumer consumer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s;\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n        consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n    }\n\n    @After\n    public void tearDown() {\n        shutdown();\n    }\n\n    @Test\n    public void testQueryMsg() {\n        int msgSize = 20;\n        producer.send(msgSize);\n        Assert.assertEquals(\"Not all are sent\", msgSize, producer.getAllUndupMsgBody().size());\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        Assert.assertEquals(\"Not all are consumed\", 0, VerifyUtils.verify(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()));\n\n        MessageExt recvMsg = (MessageExt) consumer.getListener().getFirstMsg();\n        MessageExt queryMsg = null;\n        try {\n            TestUtils.waitForMoment(3000);\n            queryMsg = producer.getProducer().viewMessage(recvMsg.getTopic(), ((MessageClientExt) recvMsg).getOffsetMsgId());\n        } catch (Exception e) {\n        }\n\n        assertThat(queryMsg).isNotNull();\n        assertThat(new String(queryMsg.getBody())).isEqualTo(new String(recvMsg.getBody()));\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/producer/querymsg/QueryMsgByKeyIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.producer.querymsg;\n\nimport java.util.List;\n\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.factory.MQMessageFactory;\nimport org.apache.rocketmq.test.util.TestUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class QueryMsgByKeyIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(QueryMsgByKeyIT.class);\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s;\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        shutdown();\n    }\n\n    @Test\n    public void testQueryMsg() {\n        int msgSize = 20;\n        String key = \"jueyin\";\n        long begin = System.currentTimeMillis();\n        List<Object> msgs = MQMessageFactory.getKeyMsg(topic, key, msgSize);\n        producer.send(msgs);\n        Assert.assertEquals(\"Not all are sent\", msgSize, producer.getAllUndupMsgBody().size());\n\n        List<MessageExt> queryMsgs = null;\n        try {\n            TestUtils.waitForMoment(500 * 3);\n            queryMsgs = producer.getProducer().queryMessage(topic, key, msgSize, begin - 5000,\n                System.currentTimeMillis() + 5000).getMessageList();\n        } catch (Exception e) {\n        }\n\n        assertThat(queryMsgs).isNotNull();\n        assertThat(queryMsgs.size()).isEqualTo(msgSize);\n    }\n\n    @Test\n    public void testQueryMax() {\n        int msgSize = 500;\n        int max = 64 * BROKER_NUM;\n        String key = \"jueyin\";\n        long begin = System.currentTimeMillis();\n        List<Object> msgs = MQMessageFactory.getKeyMsg(topic, key, msgSize);\n        producer.send(msgs);\n        Assert.assertEquals(\"Not all are sent\", msgSize, producer.getAllUndupMsgBody().size());\n\n        List<MessageExt> queryMsgs = null;\n        try {\n            queryMsgs = producer.getProducer().queryMessage(topic, key, msgSize, begin - 15000,\n                System.currentTimeMillis() + 15000).getMessageList();\n\n            int i = 3;\n            while (queryMsgs == null || queryMsgs.size() != BROKER_NUM) {\n                i--;\n                queryMsgs = producer.getProducer().queryMessage(topic, key, msgSize, begin - 15000,\n                    System.currentTimeMillis() + 15000).getMessageList();\n                TestUtils.waitForMoment(1000);\n\n                if (i == 0 || queryMsgs != null && queryMsgs.size() == max) {\n                    break;\n                }\n            }\n        } catch (Exception e) {\n        }\n\n        assertThat(queryMsgs).isNotNull();\n        assertThat(queryMsgs.size()).isEqualTo(max);\n    }\n\n\n    @Test(expected = MQClientException.class)\n    public void testQueryMsgWithSameHash1() throws Exception {\n        int msgSize = 1;\n        String topicA = \"AaTopic\";\n        String keyA = \"Aa\";\n        String topicB = \"BBTopic\";\n        String keyB = \"BB\";\n\n        initTopicWithName(topicA);\n        initTopicWithName(topicB);\n\n        RMQNormalProducer producerA = getProducer(NAMESRV_ADDR, topicA);\n        RMQNormalProducer producerB = getProducer(NAMESRV_ADDR, topicB);\n\n        List<Object> msgA = MQMessageFactory.getKeyMsg(topicA, keyA, msgSize);\n        List<Object> msgB = MQMessageFactory.getKeyMsg(topicB, keyB, msgSize);\n\n        producerA.send(msgA);\n        producerB.send(msgB);\n\n        long begin = System.currentTimeMillis() - 500000;\n        long end = System.currentTimeMillis() + 500000;\n        producerA.getProducer().queryMessage(topicA, keyB, msgSize * 10, begin, end).getMessageList();\n    }\n\n\n    @Test\n    public void testQueryMsgWithSameHash2() throws Exception {\n        int msgSize = 1;\n        String topicA = \"AaAaTopic\";\n        String keyA = \"Aa\";\n        String topicB = \"BBBBTopic\";\n        String keyB = \"Aa\";\n\n        initTopicWithName(topicA);\n        initTopicWithName(topicB);\n\n        RMQNormalProducer producerA = getProducer(NAMESRV_ADDR, topicA);\n        RMQNormalProducer producerB = getProducer(NAMESRV_ADDR, topicB);\n\n        List<Object> msgA = MQMessageFactory.getKeyMsg(topicA, keyA, msgSize);\n        List<Object> msgB = MQMessageFactory.getKeyMsg(topicB, keyB, msgSize);\n\n        producerA.send(msgA);\n        producerB.send(msgB);\n\n        long begin = System.currentTimeMillis() - 500000;\n        long end = System.currentTimeMillis() + 500000;\n        List<MessageExt> list = producerA.getProducer().queryMessage(topicA, keyA, msgSize * 10, begin, end).getMessageList();\n\n        assertThat(list).isNotNull();\n        assertThat(list.size()).isEqualTo(1);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/client/producer/transaction/TransactionalMsgIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.client.producer.transaction;\n\nimport org.apache.rocketmq.client.producer.LocalTransactionState;\nimport org.apache.rocketmq.client.producer.TransactionListener;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQTransactionalProducer;\nimport org.apache.rocketmq.test.factory.MQMessageFactory;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.MQWait;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\nimport static com.google.common.truth.Truth.assertThat;\n\nimport java.util.List;\nimport java.util.concurrent.ConcurrentHashMap;\n\npublic class TransactionalMsgIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(TransactionalMsgIT.class);\n    private RMQTransactionalProducer producer = null;\n    private RMQNormalConsumer consumer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s;\", topic));\n        producer = getTransactionalProducer(NAMESRV_ADDR, topic, new TransactionListenerImpl());\n        consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testMessageVisibility() throws Exception {\n        Thread.sleep(3000);\n        int msgSize = 120;\n        List<Object> msgs = MQMessageFactory.getMsg(topic, msgSize);\n        for (int i = 0; i < msgSize; i++) {\n            producer.send(msgs.get(i), getTransactionHandle(i));\n        }\n        boolean recvAll = MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(), consumer.getListener());\n        assertThat(recvAll).isEqualTo(true);\n    }\n\n    static Pair<Boolean, LocalTransactionState> getTransactionHandle(int msgIndex) {\n        switch (msgIndex % 5) {\n            case 0:\n                //commit immediately\n                return new Pair<>(true, LocalTransactionState.COMMIT_MESSAGE);\n            case 1:\n                //rollback immediately\n                return new Pair<>(true, LocalTransactionState.ROLLBACK_MESSAGE);\n            case 2:\n                //commit in check\n                return new Pair<>(false, LocalTransactionState.COMMIT_MESSAGE);\n            case 3:\n                //rollback in check\n                return new Pair<>(false, LocalTransactionState.ROLLBACK_MESSAGE);\n            case 4:\n            default:\n                return new Pair<>(false, LocalTransactionState.UNKNOW);\n\n        }\n    }\n\n    static private class TransactionListenerImpl implements TransactionListener {\n        ConcurrentHashMap<String, LocalTransactionState> checkStatus = new ConcurrentHashMap<>();\n\n        @Override\n        public LocalTransactionState executeLocalTransaction(Message msg, Object arg) {\n            Pair<Boolean, LocalTransactionState> transactionHandle = (Pair<Boolean,LocalTransactionState>) arg;\n            if (transactionHandle.getObject1()) {\n                return transactionHandle.getObject2();\n            } else {\n                checkStatus.put(msg.getTransactionId(), transactionHandle.getObject2());\n                return LocalTransactionState.UNKNOW;\n            }\n        }\n\n        @Override\n        public LocalTransactionState checkLocalTransaction(MessageExt msg) {\n            LocalTransactionState state = checkStatus.get(msg.getTransactionId());\n            if (state == null) {\n                return LocalTransactionState.UNKNOW;\n            } else {\n                return state;\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/container/AddAndRemoveBrokerIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.container;\n\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.container.BrokerContainer;\nimport org.apache.rocketmq.remoting.exception.RemotingConnectException;\nimport org.apache.rocketmq.remoting.exception.RemotingSendRequestException;\nimport org.apache.rocketmq.remoting.exception.RemotingTimeoutException;\nimport org.apache.rocketmq.remoting.protocol.ResponseCode;\nimport org.junit.AfterClass;\nimport org.junit.BeforeClass;\nimport org.junit.Ignore;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\n\n@Ignore\npublic class AddAndRemoveBrokerIT extends ContainerIntegrationTestBase {\n    private static BrokerContainer brokerContainer4;\n\n    @BeforeClass\n    public static void beforeClass() {\n        brokerContainer4 = createAndStartBrokerContainer(nsAddr);\n    }\n\n    @AfterClass\n    public static void afterClass() {\n        brokerContainer4.shutdown();\n    }\n\n    @Test\n    public void addBrokerTest()\n        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException,\n        RemotingConnectException {\n        String remark = null;\n        int code = 0;\n        try {\n            defaultMQAdminExt.addBrokerToContainer(brokerContainer4.getBrokerContainerAddr(), \"\");\n        } catch (MQBrokerException e) {\n            code = e.getResponseCode();\n            remark = e.getErrorMessage();\n        }\n        assertThat(code).isEqualTo(ResponseCode.SYSTEM_ERROR);\n        assertThat(remark).isEqualTo(\"addBroker properties empty\");\n    }\n\n    @Test\n    public void removeBrokerTest()\n        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {\n\n        boolean exceptionCaught = false;\n\n        try {\n            defaultMQAdminExt.removeBrokerFromContainer(brokerContainer1.getBrokerContainerAddr(),\n                master3With3Replicas.getBrokerConfig().getBrokerClusterName(),\n                master3With3Replicas.getBrokerConfig().getBrokerName(), 1);\n        } catch (MQBrokerException e) {\n            exceptionCaught = true;\n        }\n\n        assertThat(exceptionCaught).isFalse();\n        assertThat(brokerContainer1.getSlaveBrokers().size()).isEqualTo(1);\n\n        createAndAddSlave(1, brokerContainer1, master3With3Replicas);\n        awaitUntilSlaveOK();\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/container/BrokerFailoverIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.container;\n\nimport java.time.Duration;\nimport org.apache.rocketmq.container.InnerSalveBrokerController;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.junit.Ignore;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.awaitility.Awaitility.await;\n\n@Ignore\npublic class BrokerFailoverIT extends ContainerIntegrationTestBase {\n\n    @Test\n    public void testBrokerFailoverWithoutCompatible() {\n        changeCompatibleMode(false);\n        awaitUntilSlaveOK();\n        testBrokerFailover(false);\n    }\n\n    @Test\n    public void testBrokerFailoverWithCompatible() {\n        changeCompatibleMode(true);\n        awaitUntilSlaveOK();\n        testBrokerFailover(true);\n    }\n\n    private void testBrokerFailover(boolean compatibleMode) {\n        await().atMost(Duration.ofSeconds(10)).until(() ->\n            master1With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 3\n                && master2With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 3\n                && master3With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 3);\n\n        InnerSalveBrokerController targetSlave = getSlaveFromContainerByName(brokerContainer2, master1With3Replicas.getBrokerConfig().getBrokerName());\n\n        assertThat(targetSlave).isNotNull();\n\n        brokerContainer1.registerClientRPCHook(new RPCHook() {\n            @Override\n            public void doBeforeRequest(String remoteAddr, RemotingCommand request) {\n                if (request.getCode() == (compatibleMode ? RequestCode.QUERY_DATA_VERSION : RequestCode.BROKER_HEARTBEAT)) {\n                    request.setCode(-1);\n                }\n            }\n\n            @Override\n            public void doAfterResponse(String remoteAddr, RemotingCommand request,\n                RemotingCommand response) {\n\n            }\n        });\n\n        InnerSalveBrokerController finalTargetSlave = targetSlave;\n        await().atMost(Duration.ofSeconds(60)).until(() ->\n            finalTargetSlave.getMessageStore().getAliveReplicaNumInGroup() == 2\n                && master2With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 2\n                && master3With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 2);\n\n        brokerContainer1.clearClientRPCHook();\n\n        await().atMost(Duration.ofSeconds(60)).until(() ->\n            master1With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 3\n                && master2With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 3\n                && master3With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 3);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/container/BrokerMemberGroupIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.container;\n\nimport java.time.Duration;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;\nimport org.junit.Ignore;\nimport org.junit.Test;\n\nimport static org.awaitility.Awaitility.await;\n\n@Ignore\npublic class BrokerMemberGroupIT extends ContainerIntegrationTestBase {\n    @Test\n    public void testSyncBrokerMemberGroup() throws Exception {\n        await().atMost(Duration.ofSeconds(5)).until(() -> {\n            final BrokerConfig brokerConfig = master1With3Replicas.getBrokerConfig();\n            final BrokerMemberGroup memberGroup = master1With3Replicas.getBrokerOuterAPI()\n                .syncBrokerMemberGroup(brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName());\n\n            return memberGroup.getBrokerAddrs().size() == 3;\n        });\n\n        await().atMost(Duration.ofSeconds(5)).until(() -> {\n            final BrokerConfig brokerConfig = master3With3Replicas.getBrokerConfig();\n            final BrokerMemberGroup memberGroup = master3With3Replicas.getBrokerOuterAPI()\n                .syncBrokerMemberGroup(brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName());\n\n            return memberGroup.getBrokerAddrs().size() == 3;\n        });\n\n        removeSlaveBroker(1, brokerContainer1, master3With3Replicas);\n        removeSlaveBroker(1, brokerContainer2, master1With3Replicas);\n\n        await().atMost(Duration.ofSeconds(5)).until(() -> {\n            final BrokerConfig brokerConfig = master1With3Replicas.getBrokerConfig();\n            final BrokerMemberGroup memberGroup = master1With3Replicas.getBrokerOuterAPI()\n                .syncBrokerMemberGroup(brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName());\n\n            return memberGroup.getBrokerAddrs().size() == 2 && memberGroup.getBrokerAddrs().get(1L) == null;\n        });\n\n        await().atMost(Duration.ofSeconds(5)).until(() -> {\n            final BrokerConfig brokerConfig = master3With3Replicas.getBrokerConfig();\n            final BrokerMemberGroup memberGroup = master3With3Replicas.getBrokerOuterAPI()\n                .syncBrokerMemberGroup(brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName());\n            return memberGroup.getBrokerAddrs().size() == 2 && memberGroup.getBrokerAddrs().get(1L) == null;\n        });\n\n        createAndAddSlave(1, brokerContainer2, master1With3Replicas);\n        createAndAddSlave(1, brokerContainer1, master3With3Replicas);\n\n        awaitUntilSlaveOK();\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/container/ContainerIntegrationTestBase.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.container;\n\nimport io.netty.channel.ChannelHandlerContext;\nimport java.io.File;\nimport java.io.IOException;\nimport java.nio.file.Files;\nimport java.time.Duration;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Random;\nimport java.util.Set;\nimport java.util.UUID;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.commons.lang3.builder.EqualsBuilder;\nimport org.apache.commons.lang3.builder.HashCodeBuilder;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.ConfigContext;\nimport org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.TransactionCheckListener;\nimport org.apache.rocketmq.client.producer.TransactionListener;\nimport org.apache.rocketmq.client.producer.TransactionMQProducer;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.BrokerIdentity;\nimport org.apache.rocketmq.common.MQVersion;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.namesrv.NamesrvConfig;\nimport org.apache.rocketmq.container.BrokerContainer;\nimport org.apache.rocketmq.container.BrokerContainerConfig;\nimport org.apache.rocketmq.container.InnerSalveBrokerController;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.namesrv.NamesrvController;\nimport org.apache.rocketmq.remoting.netty.NettyClientConfig;\nimport org.apache.rocketmq.remoting.netty.NettyRequestProcessor;\nimport org.apache.rocketmq.remoting.netty.NettyServerConfig;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.store.ha.HAConnection;\nimport org.apache.rocketmq.store.ha.HAConnectionState;\nimport org.apache.rocketmq.tools.admin.DefaultMQAdminExt;\nimport org.junit.Assert;\nimport org.junit.BeforeClass;\n\nimport static org.awaitility.Awaitility.await;\n\n/**\n * ContainerIntegrationTestBase will setup a rocketmq ha cluster contains two broker group:\n * <li>BrokerA contains two replicas</li>\n * <li>BrokerB contains three replicas</li>\n */\npublic class ContainerIntegrationTestBase {\n    private static final AtomicBoolean CLUSTER_SET_UP = new AtomicBoolean(false);\n    private static final List<File> TMP_FILE_LIST = new ArrayList<>();\n    private static final Random RANDOM = new Random();\n    protected static String nsAddr;\n\n    protected static final String THREE_REPLICAS_TOPIC = \"SEND_MESSAGE_TEST_TOPIC_THREE_REPLICAS\";\n\n    protected static List<BrokerContainer> brokerContainerList = new ArrayList<>();\n    protected static List<NamesrvController> namesrvControllers = new ArrayList<>();\n\n    protected static final String BROKER_NAME_PREFIX = \"TestBrokerName_\";\n    protected static final int COMMIT_LOG_SIZE = 128 * 1024;\n    protected static final int INDEX_NUM = 1000;\n    protected static final AtomicInteger BROKER_INDEX = new AtomicInteger(0);\n\n    protected static BrokerContainer brokerContainer1;\n    protected static BrokerContainer brokerContainer2;\n    protected static BrokerContainer brokerContainer3;\n    protected static BrokerController master1With3Replicas;\n    protected static BrokerController master2With3Replicas;\n    protected static BrokerController master3With3Replicas;\n    protected static NamesrvController namesrvController;\n\n    protected static DefaultMQAdminExt defaultMQAdminExt;\n\n    private final static Logger LOG = LoggerFactory.getLogger(ContainerIntegrationTestBase.class);\n    private static ConcurrentMap<BrokerConfig, MessageStoreConfig> slaveStoreConfigCache = new ConcurrentHashMap<>();\n\n    protected static ConcurrentMap<BrokerConfigLite, BrokerController> isolatedBrokers = new ConcurrentHashMap<>();\n    private static final Set<Integer> PORTS_IN_USE = new HashSet<>();\n\n    @BeforeClass\n    public static void setUp() throws Exception {\n        if (CLUSTER_SET_UP.compareAndSet(false, true)) {\n            System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION));\n            System.setProperty(\"rocketmq.broker.diskSpaceCleanForciblyRatio\", \"0.99\");\n            System.setProperty(\"rocketmq.broker.diskSpaceWarningLevelRatio\", \"0.99\");\n\n            setUpCluster();\n            setUpTopic();\n            registerCleaner();\n\n            System.out.printf(\"cluster setup complete%n\");\n        }\n    }\n\n    private static void setUpTopic() {\n        createTopic(THREE_REPLICAS_TOPIC);\n    }\n\n    private static void createTopic(String topic) {\n        createTopicTo(master1With3Replicas, topic);\n        createTopicTo(master2With3Replicas, topic);\n        createTopicTo(master3With3Replicas, topic);\n    }\n\n    private static void setUpCluster() throws Exception {\n        namesrvController = createAndStartNamesrv();\n        nsAddr = \"127.0.0.1:\" + namesrvController.getNettyServerConfig().getListenPort();\n        System.out.printf(\"namesrv addr: %s%n\", nsAddr);\n\n        /*\n         *     BrokerContainer1      |      BrokerContainer2      |      BrokerContainer3\n         *\n         *   master1With3Replicas(m)      master2With3Replicas(m)      master3With3Replicas(m)\n         *   master3With3Replicas(s0)     master1With3Replicas(s0)     master2With3Replicas(s0)\n         *   master2With3Replicas(s1)     master3With3Replicas(s1)     master1With3Replicas(s1)\n         */\n\n        brokerContainer1 = createAndStartBrokerContainer(nsAddr);\n        brokerContainer2 = createAndStartBrokerContainer(nsAddr);\n        brokerContainer3 = createAndStartBrokerContainer(nsAddr);\n        // Create three broker groups, two contains two replicas, another contains three replicas\n        master1With3Replicas = createAndAddMaster(brokerContainer1, new BrokerGroupConfig(), BROKER_INDEX.getAndIncrement());\n        master2With3Replicas = createAndAddMaster(brokerContainer2, new BrokerGroupConfig(), BROKER_INDEX.getAndIncrement());\n        master3With3Replicas = createAndAddMaster(brokerContainer3, new BrokerGroupConfig(), BROKER_INDEX.getAndIncrement());\n\n        createAndAddSlave(1, brokerContainer1, master3With3Replicas);\n        createAndAddSlave(1, brokerContainer2, master1With3Replicas);\n        createAndAddSlave(1, brokerContainer3, master2With3Replicas);\n\n        createAndAddSlave(2, brokerContainer1, master2With3Replicas);\n        createAndAddSlave(2, brokerContainer2, master3With3Replicas);\n        createAndAddSlave(2, brokerContainer3, master1With3Replicas);\n\n        awaitUntilSlaveOK();\n\n        defaultMQAdminExt = new DefaultMQAdminExt(\"HATest_Admin_Group\");\n        defaultMQAdminExt.setNamesrvAddr(nsAddr);\n        defaultMQAdminExt.start();\n    }\n\n    protected static void createTopicTo(BrokerController masterBroker, String topicName, int rqn, int wqn) {\n        try {\n            TopicConfig topicConfig = new TopicConfig(topicName, rqn, wqn, 6, 0);\n            defaultMQAdminExt.createAndUpdateTopicConfig(masterBroker.getBrokerAddr(), topicConfig);\n            triggerSlaveSync(masterBroker.getBrokerConfig().getBrokerName(), brokerContainer1);\n            triggerSlaveSync(masterBroker.getBrokerConfig().getBrokerName(), brokerContainer2);\n            triggerSlaveSync(masterBroker.getBrokerConfig().getBrokerName(), brokerContainer3);\n        } catch (Exception e) {\n            e.printStackTrace();\n            throw new RuntimeException(\"Create topic to broker failed\", e);\n        }\n    }\n\n    protected static void createGroup(BrokerController masterBroker, String groupName) {\n        try {\n            SubscriptionGroupConfig config = new SubscriptionGroupConfig();\n            config.setGroupName(groupName);\n\n            masterBroker.getSubscriptionGroupManager().updateSubscriptionGroupConfig(config);\n\n            triggerSlaveSync(masterBroker.getBrokerConfig().getBrokerName(), brokerContainer1);\n            triggerSlaveSync(masterBroker.getBrokerConfig().getBrokerName(), brokerContainer2);\n            triggerSlaveSync(masterBroker.getBrokerConfig().getBrokerName(), brokerContainer3);\n        } catch (Exception e) {\n            e.printStackTrace();\n            throw new RuntimeException(\"Create group to broker failed\", e);\n        }\n    }\n\n    private static void triggerSlaveSync(String brokerName, BrokerContainer brokerContainer) {\n        for (InnerSalveBrokerController slaveBroker : brokerContainer.getSlaveBrokers()) {\n            if (slaveBroker.getBrokerConfig().getBrokerName().equals(brokerName)) {\n                slaveBroker.getSlaveSynchronize().syncAll();\n                slaveBroker.registerBrokerAll(true, false, true);\n            }\n        }\n    }\n\n    protected static void createTopicTo(BrokerController brokerController, String topicName) {\n        createTopicTo(brokerController, topicName, 8, 8);\n    }\n\n    private static void registerCleaner() {\n        Runtime.getRuntime().addShutdownHook(new Thread(() -> {\n            if (CLUSTER_SET_UP.compareAndSet(true, false)) {\n                System.out.printf(\"clean up%n\");\n                defaultMQAdminExt.shutdown();\n\n                for (final BrokerContainer brokerContainer : brokerContainerList) {\n                    brokerContainer.shutdown();\n                    for (BrokerController brokerController : brokerContainer.getBrokerControllers()) {\n                        brokerController.getMessageStore().destroy();\n                    }\n                }\n\n                for (final NamesrvController namesrvController : namesrvControllers) {\n                    namesrvController.shutdown();\n                }\n\n                for (final File file : TMP_FILE_LIST) {\n                    UtilAll.deleteFile(file);\n                }\n            }\n        }));\n    }\n\n    private static File createBaseDir(String prefix) {\n        final File file;\n        try {\n            file = Files.createTempDirectory(prefix).toFile();\n            TMP_FILE_LIST.add(file);\n            System.out.printf(\"create file at %s%n\", file.getAbsolutePath());\n            return file;\n        } catch (IOException e) {\n            throw new RuntimeException(\"Couldn't create tmp folder\", e);\n        }\n    }\n\n    public static NamesrvController createAndStartNamesrv() {\n        String baseDir = createBaseDir(\"test-cluster-namesrv\").getAbsolutePath();\n        NamesrvConfig namesrvConfig = new NamesrvConfig();\n        NettyServerConfig nameServerNettyServerConfig = new NettyServerConfig();\n        namesrvConfig.setKvConfigPath(baseDir + File.separator + \"namesrv\" + File.separator + \"kvConfig.json\");\n        namesrvConfig.setConfigStorePath(baseDir + File.separator + \"namesrv\" + File.separator + \"namesrv.properties\");\n        namesrvConfig.setSupportActingMaster(true);\n        namesrvConfig.setScanNotActiveBrokerInterval(1000);\n\n        nameServerNettyServerConfig.setListenPort(generatePort(10000, 10000));\n        NamesrvController namesrvController = new NamesrvController(namesrvConfig, nameServerNettyServerConfig);\n        try {\n            Assert.assertTrue(namesrvController.initialize());\n            LOG.info(\"Name Server Start:{}\", nameServerNettyServerConfig.getListenPort());\n            namesrvController.start();\n        } catch (Exception e) {\n            LOG.info(\"Name Server start failed\");\n            e.printStackTrace();\n            System.exit(1);\n        }\n\n        namesrvController.getRemotingServer().registerProcessor(RequestCode.REGISTER_BROKER, new NettyRequestProcessor() {\n            @Override\n            public RemotingCommand processRequest(final ChannelHandlerContext ctx,\n                final RemotingCommand request) throws Exception {\n                final RegisterBrokerRequestHeader requestHeader = (RegisterBrokerRequestHeader) request.decodeCommandCustomHeader(RegisterBrokerRequestHeader.class);\n                final BrokerConfigLite liteConfig = new BrokerConfigLite(requestHeader.getClusterName(),\n                    requestHeader.getBrokerName(),\n                    requestHeader.getBrokerAddr(),\n                    requestHeader.getBrokerId());\n                if (isolatedBrokers.containsKey(liteConfig)) {\n                    // return response with SYSTEM_ERROR\n                    return RemotingCommand.createResponseCommand(null);\n                }\n                return namesrvController.getRemotingServer().getDefaultProcessorPair().getObject1().processRequest(ctx, request);\n            }\n\n            @Override\n            public boolean rejectRequest() {\n                return false;\n            }\n        }, null);\n\n        namesrvControllers.add(namesrvController);\n        return namesrvController;\n\n    }\n\n    public static BrokerContainer createAndStartBrokerContainer(String nsAddr) {\n        BrokerContainerConfig brokerContainerConfig = new BrokerContainerConfig();\n        NettyServerConfig nettyServerConfig = new NettyServerConfig();\n        NettyClientConfig nettyClientConfig = new NettyClientConfig();\n        brokerContainerConfig.setNamesrvAddr(nsAddr);\n\n        nettyServerConfig.setListenPort(generatePort(20000, 10000));\n        BrokerContainer brokerContainer = new BrokerContainer(brokerContainerConfig, nettyServerConfig, nettyClientConfig);\n        try {\n            Assert.assertTrue(brokerContainer.initialize());\n            LOG.info(\"Broker container Start, listen on {}.\", nettyServerConfig.getListenPort());\n            brokerContainer.start();\n        } catch (Exception e) {\n            LOG.info(\"Broker container start failed\", e);\n            e.printStackTrace();\n            System.exit(1);\n        }\n        brokerContainerList.add(brokerContainer);\n        return brokerContainer;\n    }\n\n    private static int generatePort(int base, int range) {\n        int result = base + RANDOM.nextInt(range);\n        while (PORTS_IN_USE.contains(result) || PORTS_IN_USE.contains(result - 2)) {\n            result = base + RANDOM.nextInt(range);\n        }\n        PORTS_IN_USE.add(result);\n        PORTS_IN_USE.add(result - 2);\n        return result;\n    }\n\n    public static BrokerController createAndAddMaster(BrokerContainer brokerContainer,\n        BrokerGroupConfig brokerGroupConfig, int brokerIndex) throws Exception {\n        BrokerConfig brokerConfig = new BrokerConfig();\n        MessageStoreConfig storeConfig = new MessageStoreConfig();\n        brokerConfig.setBrokerName(BROKER_NAME_PREFIX + brokerIndex);\n        brokerConfig.setBrokerIP1(\"127.0.0.1\");\n        brokerConfig.setBrokerIP2(\"127.0.0.1\");\n        brokerConfig.setBrokerId(0);\n        brokerConfig.setEnablePropertyFilter(true);\n        brokerConfig.setEnableSlaveActingMaster(brokerGroupConfig.enableSlaveActingMaster);\n        brokerConfig.setEnableRemoteEscape(brokerGroupConfig.enableRemoteEscape);\n        brokerConfig.setSlaveReadEnable(brokerGroupConfig.slaveReadEnable);\n        brokerConfig.setLockInStrictMode(true);\n        brokerConfig.setConsumerOffsetUpdateVersionStep(10);\n        brokerConfig.setDelayOffsetUpdateVersionStep(10);\n        brokerConfig.setCompatibleWithOldNameSrv(false);\n        brokerConfig.setListenPort(generatePort(brokerContainer.getRemotingServer().localListenPort(), 10000));\n\n        String baseDir = createBaseDir(brokerConfig.getBrokerName() + \"_\" + brokerConfig.getBrokerId()).getAbsolutePath();\n        storeConfig.setStorePathRootDir(baseDir);\n        storeConfig.setStorePathCommitLog(baseDir + File.separator + \"commitlog\");\n        storeConfig.setHaListenPort(generatePort(30000, 10000));\n        storeConfig.setMappedFileSizeCommitLog(COMMIT_LOG_SIZE);\n        storeConfig.setMaxIndexNum(INDEX_NUM);\n        storeConfig.setMaxHashSlotNum(INDEX_NUM * 4);\n        storeConfig.setTotalReplicas(brokerGroupConfig.totalReplicas);\n        storeConfig.setInSyncReplicas(brokerGroupConfig.inSyncReplicas);\n        storeConfig.setMinInSyncReplicas(brokerGroupConfig.minReplicas);\n        storeConfig.setEnableAutoInSyncReplicas(brokerGroupConfig.autoReplicas);\n        storeConfig.setBrokerRole(BrokerRole.SYNC_MASTER);\n        storeConfig.setSyncFlushTimeout(10 * 1000);\n\n        System.out.printf(\"start master %s with port %d-%d%n\", brokerConfig.getCanonicalName(), brokerConfig.getListenPort(), storeConfig.getHaListenPort());\n        BrokerController brokerController = null;\n        try {\n            brokerController = brokerContainer.addBroker(buildConfigContext(brokerConfig, storeConfig));\n            Assert.assertNotNull(brokerController);\n            brokerController.start();\n            TMP_FILE_LIST.add(new File(brokerController.getTopicConfigManager().configFilePath()));\n            TMP_FILE_LIST.add(new File(brokerController.getSubscriptionGroupManager().configFilePath()));\n            LOG.info(\"Broker Start name:{} addr:{}\", brokerConfig.getBrokerName(), brokerController.getBrokerAddr());\n        } catch (Exception e) {\n            LOG.info(\"Broker start failed\", e);\n            e.printStackTrace();\n            System.exit(1);\n        }\n\n        return brokerController;\n    }\n\n    protected static DefaultMQProducer createProducer(String producerGroup) {\n        DefaultMQProducer producer = new DefaultMQProducer(producerGroup);\n        producer.setInstanceName(UUID.randomUUID().toString());\n        producer.setNamesrvAddr(nsAddr);\n        return producer;\n    }\n\n    protected static TransactionMQProducer createTransactionProducer(String producerGroup,\n        TransactionCheckListener transactionCheckListener) {\n        TransactionMQProducer producer = new TransactionMQProducer(producerGroup);\n        producer.setInstanceName(UUID.randomUUID().toString());\n        producer.setNamesrvAddr(nsAddr);\n        producer.setTransactionCheckListener(transactionCheckListener);\n        return producer;\n    }\n\n    protected static TransactionMQProducer createTransactionProducer(String producerGroup,\n        TransactionListener transactionListener) {\n        TransactionMQProducer producer = new TransactionMQProducer(producerGroup);\n        producer.setInstanceName(UUID.randomUUID().toString());\n        producer.setNamesrvAddr(nsAddr);\n        producer.setTransactionListener(transactionListener);\n        return producer;\n    }\n\n    protected static DefaultMQPullConsumer createPullConsumer(String consumerGroup) {\n        DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(consumerGroup);\n        consumer.setInstanceName(UUID.randomUUID().toString());\n        consumer.setNamesrvAddr(nsAddr);\n        return consumer;\n    }\n\n    protected static DefaultMQPushConsumer createPushConsumer(String consumerGroup) {\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(consumerGroup);\n        consumer.setInstanceName(UUID.randomUUID().toString());\n        consumer.setNamesrvAddr(nsAddr);\n        return consumer;\n    }\n\n    protected static void createAndAddSlave(int slaveBrokerId, BrokerContainer brokerContainer,\n        BrokerController master) {\n        BrokerConfig slaveBrokerConfig = new BrokerConfig();\n        slaveBrokerConfig.setBrokerName(master.getBrokerConfig().getBrokerName());\n        slaveBrokerConfig.setBrokerId(slaveBrokerId);\n        slaveBrokerConfig.setBrokerClusterName(master.getBrokerConfig().getBrokerClusterName());\n        slaveBrokerConfig.setCompatibleWithOldNameSrv(false);\n        slaveBrokerConfig.setBrokerIP1(\"127.0.0.1\");\n        slaveBrokerConfig.setBrokerIP2(\"127.0.0.1\");\n        slaveBrokerConfig.setEnablePropertyFilter(true);\n        slaveBrokerConfig.setSlaveReadEnable(true);\n        slaveBrokerConfig.setEnableSlaveActingMaster(true);\n        slaveBrokerConfig.setEnableRemoteEscape(true);\n        slaveBrokerConfig.setLockInStrictMode(true);\n        slaveBrokerConfig.setListenPort(generatePort(brokerContainer.getRemotingServer().localListenPort(), 10000));\n        slaveBrokerConfig.setConsumerOffsetUpdateVersionStep(10);\n        slaveBrokerConfig.setDelayOffsetUpdateVersionStep(10);\n\n        MessageStoreConfig storeConfig = slaveStoreConfigCache.get(slaveBrokerConfig);\n\n        if (storeConfig == null) {\n            storeConfig = new MessageStoreConfig();\n            String baseDir = createBaseDir(slaveBrokerConfig.getBrokerName() + \"_\" + slaveBrokerConfig.getBrokerId()).getAbsolutePath();\n            storeConfig.setStorePathRootDir(baseDir);\n            storeConfig.setStorePathCommitLog(baseDir + File.separator + \"commitlog\");\n            storeConfig.setHaListenPort(generatePort(master.getMessageStoreConfig().getHaListenPort(), 10000));\n            storeConfig.setMappedFileSizeCommitLog(COMMIT_LOG_SIZE);\n            storeConfig.setMaxIndexNum(INDEX_NUM);\n            storeConfig.setMaxHashSlotNum(INDEX_NUM * 4);\n            storeConfig.setTotalReplicas(master.getMessageStoreConfig().getTotalReplicas());\n            storeConfig.setInSyncReplicas(master.getMessageStoreConfig().getInSyncReplicas());\n            storeConfig.setMinInSyncReplicas(master.getMessageStoreConfig().getMinInSyncReplicas());\n            storeConfig.setBrokerRole(BrokerRole.SLAVE);\n            slaveStoreConfigCache.put(slaveBrokerConfig, storeConfig);\n        }\n\n        System.out.printf(\"start slave %s with port %d-%d%n\", slaveBrokerConfig.getCanonicalName(), slaveBrokerConfig.getListenPort(), storeConfig.getHaListenPort());\n\n        try {\n            BrokerController brokerController = brokerContainer.addBroker(buildConfigContext(slaveBrokerConfig, storeConfig));\n            Assert.assertNotNull(brokerContainer);\n            brokerController.start();\n            TMP_FILE_LIST.add(new File(brokerController.getTopicConfigManager().configFilePath()));\n            TMP_FILE_LIST.add(new File(brokerController.getSubscriptionGroupManager().configFilePath()));\n            LOG.info(\"Add slave name:{} addr:{}\", slaveBrokerConfig.getBrokerName(), brokerController.getBrokerAddr());\n        } catch (Exception e) {\n            e.printStackTrace();\n            throw new RuntimeException(\"Couldn't add slave broker\", e);\n        }\n    }\n\n    protected static void removeSlaveBroker(int slaveBrokerId, BrokerContainer brokerContainer,\n        BrokerController master) throws Exception {\n        BrokerIdentity brokerIdentity = new BrokerIdentity(master.getBrokerConfig().getBrokerClusterName(),\n            master.getBrokerConfig().getBrokerName(), slaveBrokerId);\n\n        brokerContainer.removeBroker(brokerIdentity);\n    }\n\n    protected static void awaitUntilSlaveOK() {\n        await().atMost(100, TimeUnit.SECONDS)\n            .until(() -> {\n                boolean isOk = master1With3Replicas.getMessageStore().getHaService().getConnectionCount().get() == 2\n                    && master1With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 3;\n                for (HAConnection haConnection : master1With3Replicas.getMessageStore().getHaService().getConnectionList()) {\n                    isOk &= haConnection.getCurrentState().equals(HAConnectionState.TRANSFER);\n                }\n                return isOk;\n            });\n\n        await().atMost(100, TimeUnit.SECONDS)\n            .until(() -> {\n                boolean isOk = master2With3Replicas.getMessageStore().getHaService().getConnectionCount().get() == 2\n                    && master2With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 3;\n                for (HAConnection haConnection : master2With3Replicas.getMessageStore().getHaService().getConnectionList()) {\n                    isOk &= haConnection.getCurrentState().equals(HAConnectionState.TRANSFER);\n                }\n                return isOk;\n            });\n\n        await().atMost(100, TimeUnit.SECONDS)\n            .until(() -> {\n                boolean isOk = master3With3Replicas.getMessageStore().getHaService().getConnectionCount().get() == 2\n                    && master3With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 3;\n                for (HAConnection haConnection : master3With3Replicas.getMessageStore().getHaService().getConnectionList()) {\n                    isOk &= haConnection.getCurrentState().equals(HAConnectionState.TRANSFER);\n                }\n                return isOk;\n            });\n\n        try {\n            Thread.sleep(2000);\n        } catch (InterruptedException e) {\n            e.printStackTrace();\n        }\n    }\n\n    protected static void isolateBroker(BrokerController brokerController) {\n        final BrokerConfig config = brokerController.getBrokerConfig();\n\n        BrokerConfigLite liteConfig = new BrokerConfigLite(config.getBrokerClusterName(),\n            config.getBrokerName(),\n            brokerController.getBrokerAddr(),\n            config.getBrokerId());\n\n        // Reject register requests from the specific broker\n        isolatedBrokers.putIfAbsent(liteConfig, brokerController);\n\n        // UnRegister the specific broker immediately\n        namesrvController.getRouteInfoManager().unregisterBroker(liteConfig.getClusterName(),\n            liteConfig.getBrokerAddr(),\n            liteConfig.getBrokerName(),\n            liteConfig.getBrokerId());\n    }\n\n    protected static void cancelIsolatedBroker(BrokerController brokerController) {\n        final BrokerConfig config = brokerController.getBrokerConfig();\n\n        BrokerConfigLite liteConfig = new BrokerConfigLite(config.getBrokerClusterName(),\n            config.getBrokerName(),\n            brokerController.getBrokerAddr(),\n            config.getBrokerId());\n\n        isolatedBrokers.remove(liteConfig);\n        brokerController.registerBrokerAll(true, false, true);\n\n        await().atMost(Duration.ofMinutes(1)).until(() -> namesrvController.getRouteInfoManager()\n            .getBrokerMemberGroup(liteConfig.getClusterName(), liteConfig.brokerName).getBrokerAddrs()\n            .containsKey(liteConfig.getBrokerId()));\n    }\n\n    protected static InnerSalveBrokerController getSlaveFromContainerByName(BrokerContainer brokerContainer,\n        String brokerName) {\n        InnerSalveBrokerController targetSlave = null;\n        for (InnerSalveBrokerController slave : brokerContainer.getSlaveBrokers()) {\n            if (slave.getBrokerConfig().getBrokerName().equals(brokerName)) {\n                targetSlave = slave;\n            }\n        }\n\n        return targetSlave;\n    }\n\n    protected static void changeCompatibleMode(boolean compatibleMode) {\n        brokerContainer1.getBrokerControllers().forEach(brokerController -> brokerController.getBrokerConfig().setCompatibleWithOldNameSrv(compatibleMode));\n        brokerContainer2.getBrokerControllers().forEach(brokerController -> brokerController.getBrokerConfig().setCompatibleWithOldNameSrv(compatibleMode));\n        brokerContainer3.getBrokerControllers().forEach(brokerController -> brokerController.getBrokerConfig().setCompatibleWithOldNameSrv(compatibleMode));\n    }\n\n    protected static Set<MessageQueue> filterMessageQueue(Set<MessageQueue> mqSet, String topic) {\n        Set<MessageQueue> targetMqSet = new HashSet<>();\n        if (topic != null) {\n            for (MessageQueue mq : mqSet) {\n                if (mq.getTopic().equals(topic)) {\n                    targetMqSet.add(mq);\n                }\n            }\n        }\n\n        return targetMqSet;\n    }\n\n    public static class BrokerGroupConfig {\n        int totalReplicas = 3;\n        int minReplicas = 1;\n        int inSyncReplicas = 2;\n        boolean autoReplicas = true;\n        boolean enableSlaveActingMaster = true;\n        boolean enableRemoteEscape = true;\n        boolean slaveReadEnable = true;\n\n        public BrokerGroupConfig() {\n        }\n\n        public BrokerGroupConfig(final int totalReplicas, final int minReplicas, final int inSyncReplicas,\n            final boolean autoReplicas, boolean enableSlaveActingMaster, boolean slaveReadEnable) {\n            this.totalReplicas = totalReplicas;\n            this.minReplicas = minReplicas;\n            this.inSyncReplicas = inSyncReplicas;\n            this.autoReplicas = autoReplicas;\n            this.enableSlaveActingMaster = enableSlaveActingMaster;\n            this.slaveReadEnable = slaveReadEnable;\n        }\n    }\n\n    static class BrokerConfigLite {\n        private String clusterName;\n        private String brokerName;\n        private String brokerAddr;\n        private long brokerId;\n\n        public BrokerConfigLite(final String clusterName, final String brokerName, final String brokerAddr,\n            final long brokerId) {\n            this.clusterName = clusterName;\n            this.brokerName = brokerName;\n            this.brokerAddr = brokerAddr;\n            this.brokerId = brokerId;\n        }\n\n        public String getClusterName() {\n            return clusterName;\n        }\n\n        public String getBrokerName() {\n            return brokerName;\n        }\n\n        public String getBrokerAddr() {\n            return brokerAddr;\n        }\n\n        public long getBrokerId() {\n            return brokerId;\n        }\n\n        @Override\n        public boolean equals(final Object o) {\n            if (this == o)\n                return true;\n\n            if (o == null || getClass() != o.getClass())\n                return false;\n\n            final BrokerConfigLite lite = (BrokerConfigLite) o;\n\n            return new EqualsBuilder()\n                .append(clusterName, lite.clusterName)\n                .append(brokerName, lite.brokerName)\n                .append(brokerAddr, lite.brokerAddr)\n                .append(brokerId, lite.brokerId)\n                .isEquals();\n        }\n\n        @Override\n        public int hashCode() {\n            return new HashCodeBuilder(17, 37)\n                .append(clusterName)\n                .append(brokerName)\n                .append(brokerAddr)\n                .append(brokerId)\n                .toHashCode();\n        }\n    }\n\n    public static ConfigContext buildConfigContext(BrokerConfig brokerConfig, MessageStoreConfig messageStoreConfig) {\n        return new ConfigContext.Builder()\n            .brokerConfig(brokerConfig)\n            .messageStoreConfig(messageStoreConfig)\n            .build();\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/container/GetMaxOffsetFromSlaveIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.container;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.producer.TopicPublishInfo;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.junit.AfterClass;\nimport org.junit.BeforeClass;\nimport org.junit.Ignore;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Java6Assertions.assertThat;\nimport static org.awaitility.Awaitility.await;\n\n@Ignore\npublic class GetMaxOffsetFromSlaveIT extends ContainerIntegrationTestBase {\n    private static DefaultMQProducer mqProducer;\n\n    private static final String MSG = \"Hello RocketMQ \";\n    private static final byte[] MESSAGE_BODY = MSG.getBytes(StandardCharsets.UTF_8);\n\n    public GetMaxOffsetFromSlaveIT() {\n    }\n\n    @BeforeClass\n    public static void beforeClass() throws MQClientException {\n        mqProducer = createProducer(GetMaxOffsetFromSlaveIT.class.getSimpleName() + \"_Producer\");\n        mqProducer.start();\n    }\n\n    @AfterClass\n    public static void afterClass() {\n        if (mqProducer != null) {\n            mqProducer.shutdown();\n        }\n    }\n\n    @Test\n    public void testGetMaxOffsetFromSlave() throws InterruptedException, RemotingException, MQClientException, MQBrokerException {\n        awaitUntilSlaveOK();\n        mqProducer.getDefaultMQProducerImpl().getmQClientFactory().updateTopicRouteInfoFromNameServer(THREE_REPLICAS_TOPIC);\n\n        for (int i = 0; i < 100; i++) {\n            Message msg = new Message(THREE_REPLICAS_TOPIC, MESSAGE_BODY);\n            SendResult sendResult = mqProducer.send(msg, 10000);\n            assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n        }\n\n        Map<Integer, Long> maxOffsetMap = new HashMap<>();\n        TopicPublishInfo publishInfo = mqProducer.getDefaultMQProducerImpl().getTopicPublishInfoTable().get(THREE_REPLICAS_TOPIC);\n        assertThat(publishInfo).isNotNull();\n        for (MessageQueue mq : publishInfo.getMessageQueueList()) {\n            maxOffsetMap.put(mq.getQueueId(), mqProducer.getDefaultMQProducerImpl().\n                maxOffset(new MessageQueue(THREE_REPLICAS_TOPIC, master3With3Replicas.getBrokerConfig().getBrokerName(), mq.getQueueId())));\n        }\n\n        isolateBroker(master3With3Replicas);\n\n        mqProducer.getDefaultMQProducerImpl().getmQClientFactory().updateTopicRouteInfoFromNameServer(THREE_REPLICAS_TOPIC);\n        assertThat(mqProducer.getDefaultMQProducerImpl().getmQClientFactory().findBrokerAddressInPublish(\n            master3With3Replicas.getBrokerConfig().getBrokerName())).isNotNull();\n\n        for (MessageQueue mq : publishInfo.getMessageQueueList()) {\n            assertThat(mqProducer.getDefaultMQProducerImpl().maxOffset(\n                new MessageQueue(THREE_REPLICAS_TOPIC, master3With3Replicas.getBrokerConfig().getBrokerName(), mq.getQueueId())))\n                .isEqualTo(maxOffsetMap.get(mq.getQueueId()));\n        }\n\n        cancelIsolatedBroker(master3With3Replicas);\n        await().atMost(100, TimeUnit.SECONDS)\n            .until(() -> ((DefaultMessageStore) master3With3Replicas.getMessageStore()).getHaService().getConnectionCount().get() == 2);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/container/GetMetadataReverseIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.container;\n\nimport java.time.Duration;\nimport java.util.Map;\nimport java.util.Random;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.rocketmq.container.InnerSalveBrokerController;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.BrokerIdentity;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.junit.AfterClass;\nimport org.junit.BeforeClass;\nimport org.junit.Ignore;\nimport org.junit.Test;\n\nimport static org.awaitility.Awaitility.await;\n\n@Ignore\npublic class GetMetadataReverseIT extends ContainerIntegrationTestBase {\n\n    private static DefaultMQProducer producer;\n\n    private static final String CONSUMER_GROUP = GetMetadataReverseIT.class.getSimpleName() + \"_Consumer\";\n\n    private static final int MESSAGE_COUNT = 32;\n\n    private final Random random = new Random();\n\n    public GetMetadataReverseIT() {\n\n    }\n\n    @BeforeClass\n    public static void beforeClass() throws Throwable {\n        producer = createProducer(PushMultipleReplicasIT.class.getSimpleName() + \"_PRODUCER\");\n        producer.setSendMsgTimeout(15 * 1000);\n        producer.start();\n    }\n\n    @AfterClass\n    public static void afterClass() throws Exception {\n        producer.shutdown();\n    }\n\n    @Test\n    public void testGetMetadataReverse_consumerOffset() throws Exception {\n        String topic = GetMetadataReverseIT.class.getSimpleName() + \"_consumerOffset\" + random.nextInt(65535);\n        createTopicTo(master1With3Replicas, topic, 1, 1);\n        // Wait topic synchronization\n        await().atMost(Duration.ofMinutes(1)).until(() -> {\n            InnerSalveBrokerController slaveBroker = brokerContainer2.getSlaveBrokers().iterator().next();\n            return slaveBroker.getTopicConfigManager().selectTopicConfig(topic) != null;\n        });\n\n        int sendSuccess = 0;\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            Message msg = new Message(topic, Integer.toString(i).getBytes());\n            SendResult sendResult = producer.send(msg);\n            if (sendResult.getSendStatus() == SendStatus.SEND_OK) {\n                sendSuccess++;\n            }\n        }\n        final int finalSendSuccess = sendSuccess;\n        await().atMost(Duration.ofMinutes(1)).until(() -> finalSendSuccess >= MESSAGE_COUNT);\n\n        isolateBroker(master1With3Replicas);\n        brokerContainer1.removeBroker(new BrokerIdentity(\n            master1With3Replicas.getBrokerConfig().getBrokerClusterName(),\n            master1With3Replicas.getBrokerConfig().getBrokerName(),\n            master1With3Replicas.getBrokerConfig().getBrokerId()));\n\n        DefaultMQPushConsumer pushConsumer = createPushConsumer(CONSUMER_GROUP);\n        pushConsumer.subscribe(topic, \"*\");\n        pushConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        AtomicInteger receivedMsgCount = new AtomicInteger(0);\n        pushConsumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            receivedMsgCount.addAndGet(msgs.size());\n            msgs.forEach(x -> System.out.printf(x + \"%n\"));\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        pushConsumer.start();\n        await().atMost(Duration.ofMinutes(3)).until(() -> receivedMsgCount.get() >= MESSAGE_COUNT);\n\n        await().atMost(Duration.ofMinutes(1)).until(() -> {\n            pushConsumer.getDefaultMQPushConsumerImpl().persistConsumerOffset();\n            Map<Integer, Long> slaveOffsetTable = null;\n            for (InnerSalveBrokerController slave : brokerContainer2.getSlaveBrokers()) {\n                if (slave.getBrokerConfig().getBrokerName().equals(master1With3Replicas.getBrokerConfig().getBrokerName())) {\n                    slaveOffsetTable = slave.getConsumerOffsetManager().queryOffset(CONSUMER_GROUP, topic);\n                }\n            }\n\n            if (slaveOffsetTable != null) {\n                long totalOffset = 0;\n                for (final Long offset : slaveOffsetTable.values()) {\n                    totalOffset += offset;\n                }\n\n                return totalOffset >= MESSAGE_COUNT;\n            }\n            return false;\n        });\n\n        //Add back master\n        master1With3Replicas = brokerContainer1.addBroker(buildConfigContext(master1With3Replicas.getBrokerConfig(), master1With3Replicas.getMessageStoreConfig()));\n        master1With3Replicas.start();\n        cancelIsolatedBroker(master1With3Replicas);\n\n        awaitUntilSlaveOK();\n\n        await().atMost(Duration.ofMinutes(1)).until(() -> {\n            Map<Integer, Long> offsetTable = master1With3Replicas.getConsumerOffsetManager().queryOffset(CONSUMER_GROUP, topic);\n            long totalOffset = 0;\n            if (offsetTable != null) {\n                for (final Long offset : offsetTable.values()) {\n                    totalOffset += offset;\n                }\n            }\n            return totalOffset >= MESSAGE_COUNT;\n        });\n\n        pushConsumer.shutdown();\n    }\n\n    @Test\n    public void testGetMetadataReverse_delayOffset() throws Exception {\n        String topic = GetMetadataReverseIT.class.getSimpleName() + \"_delayOffset\" + random.nextInt(65535);\n        createTopicTo(master1With3Replicas, topic, 1, 1);\n        createTopicTo(master2With3Replicas, topic, 1, 1);\n        createTopicTo(master3With3Replicas, topic, 1, 1);\n        // Wait topic synchronization\n        await().atMost(Duration.ofMinutes(1)).until(() -> {\n            InnerSalveBrokerController slaveBroker = brokerContainer2.getSlaveBrokers().iterator().next();\n            return slaveBroker.getTopicConfigManager().selectTopicConfig(topic) != null;\n        });\n        int delayLevel = 4;\n\n        DefaultMQPushConsumer pushConsumer = createPushConsumer(CONSUMER_GROUP);\n        pushConsumer.subscribe(topic, \"*\");\n        AtomicInteger receivedMsgCount = new AtomicInteger(0);\n        pushConsumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            receivedMsgCount.addAndGet(msgs.size());\n            msgs.forEach(x -> System.out.printf(x + \"%n\"));\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        pushConsumer.start();\n\n        MessageQueue messageQueue = new MessageQueue(topic, master1With3Replicas.getBrokerConfig().getBrokerName(), 0);\n        int sendSuccess = 0;\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            Message msg = new Message(topic, Integer.toString(i).getBytes());\n            msg.setDelayTimeLevel(delayLevel);\n            SendResult sendResult = producer.send(msg, messageQueue);\n            if (sendResult.getSendStatus() == SendStatus.SEND_OK) {\n                sendSuccess++;\n            }\n        }\n        final int finalSendSuccess = sendSuccess;\n        await().atMost(Duration.ofMinutes(1)).until(() -> finalSendSuccess >= MESSAGE_COUNT);\n\n        isolateBroker(master1With3Replicas);\n        brokerContainer1.removeBroker(new BrokerIdentity(\n            master1With3Replicas.getBrokerConfig().getBrokerClusterName(),\n            master1With3Replicas.getBrokerConfig().getBrokerName(),\n            master1With3Replicas.getBrokerConfig().getBrokerId()));\n\n        await().atMost(Duration.ofMinutes(1)).until(() -> receivedMsgCount.get() >= MESSAGE_COUNT);\n\n        await().atMost(Duration.ofMinutes(1)).until(() -> {\n            pushConsumer.getDefaultMQPushConsumerImpl().persistConsumerOffset();\n            Map<Integer, Long> offsetTable = master2With3Replicas.getConsumerOffsetManager().queryOffset(CONSUMER_GROUP, topic);\n            if (offsetTable != null) {\n                long totalOffset = 0;\n                for (final Long offset : offsetTable.values()) {\n                    totalOffset += offset;\n                }\n                return totalOffset >= MESSAGE_COUNT;\n\n            }\n            return false;\n        });\n\n        //Add back master\n        master1With3Replicas = brokerContainer1.addBroker(buildConfigContext(master1With3Replicas.getBrokerConfig(), master1With3Replicas.getMessageStoreConfig()));\n        master1With3Replicas.start();\n        cancelIsolatedBroker(master1With3Replicas);\n\n        awaitUntilSlaveOK();\n\n        await().atMost(Duration.ofMinutes(1)).until(() -> {\n            Map<Integer, Long> offsetTable = master1With3Replicas.getScheduleMessageService().getOffsetTable();\n            return offsetTable.get(delayLevel) >= MESSAGE_COUNT;\n        });\n\n        pushConsumer.shutdown();\n    }\n\n    @Test\n    public void testGetMetadataReverse_timerCheckPoint() throws Exception {\n        String topic = GetMetadataReverseIT.class.getSimpleName() + \"_timerCheckPoint\" + random.nextInt(65535);\n        createTopicTo(master1With3Replicas, topic, 1, 1);\n        createTopicTo(master2With3Replicas, topic, 1, 1);\n        createTopicTo(master3With3Replicas, topic, 1, 1);\n        // Wait topic synchronization\n        await().atMost(Duration.ofMinutes(1)).until(() -> {\n            InnerSalveBrokerController slaveBroker = brokerContainer2.getSlaveBrokers().iterator().next();\n            return slaveBroker.getTopicConfigManager().selectTopicConfig(topic) != null;\n        });\n\n        DefaultMQPushConsumer pushConsumer = createPushConsumer(CONSUMER_GROUP);\n        pushConsumer.subscribe(topic, \"*\");\n        AtomicInteger receivedMsgCount = new AtomicInteger(0);\n        pushConsumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            receivedMsgCount.addAndGet(msgs.size());\n            msgs.forEach(x -> System.out.printf(x + \"%n\"));\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        pushConsumer.start();\n\n        MessageQueue messageQueue = new MessageQueue(topic, master1With3Replicas.getBrokerConfig().getBrokerName(), 0);\n        int sendSuccess = 0;\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            Message msg = new Message(topic, Integer.toString(i).getBytes());\n            msg.setDelayTimeSec(30);\n            SendResult sendResult = producer.send(msg, messageQueue);\n            if (sendResult.getSendStatus() == SendStatus.SEND_OK) {\n                sendSuccess++;\n            }\n        }\n        final int finalSendSuccess = sendSuccess;\n        await().atMost(Duration.ofMinutes(1)).until(() -> finalSendSuccess >= MESSAGE_COUNT);\n\n        isolateBroker(master1With3Replicas);\n        brokerContainer1.removeBroker(new BrokerIdentity(\n            master1With3Replicas.getBrokerConfig().getBrokerClusterName(),\n            master1With3Replicas.getBrokerConfig().getBrokerName(),\n            master1With3Replicas.getBrokerConfig().getBrokerId()));\n\n        await().atMost(Duration.ofMinutes(1)).until(() -> receivedMsgCount.get() >= MESSAGE_COUNT);\n\n        await().atMost(Duration.ofMinutes(1)).until(() -> {\n            pushConsumer.getDefaultMQPushConsumerImpl().persistConsumerOffset();\n            Map<Integer, Long> offsetTable = master2With3Replicas.getConsumerOffsetManager().queryOffset(CONSUMER_GROUP, topic);\n            if (offsetTable != null) {\n                long totalOffset = 0;\n                for (final Long offset : offsetTable.values()) {\n                    totalOffset += offset;\n                }\n                return totalOffset >= MESSAGE_COUNT;\n            }\n            return false;\n        });\n\n        //Add back master\n        master1With3Replicas = brokerContainer1.addBroker(buildConfigContext(master1With3Replicas.getBrokerConfig(), master1With3Replicas.getMessageStoreConfig()));\n        master1With3Replicas.start();\n        cancelIsolatedBroker(master1With3Replicas);\n\n        awaitUntilSlaveOK();\n\n        await().atMost(Duration.ofMinutes(1)).until(() -> master1With3Replicas.getTimerCheckpoint().getMasterTimerQueueOffset() >= MESSAGE_COUNT);\n\n        pushConsumer.shutdown();\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/container/PopSlaveActingMasterIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.container;\n\nimport org.apache.commons.lang3.RandomStringUtils;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.BrokerIdentity;\nimport org.apache.rocketmq.common.KeyBuilder;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.message.MessageRequestMode;\nimport org.apache.rocketmq.container.BrokerContainer;\nimport org.apache.rocketmq.container.InnerBrokerController;\nimport org.apache.rocketmq.container.InnerSalveBrokerController;\nimport org.junit.AfterClass;\nimport org.junit.BeforeClass;\nimport org.junit.Ignore;\nimport org.junit.Test;\n\nimport java.nio.charset.StandardCharsets;\nimport java.time.Duration;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Random;\nimport java.util.Set;\nimport java.util.concurrent.CopyOnWriteArrayList;\nimport java.util.concurrent.atomic.AtomicInteger;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.awaitility.Awaitility.await;\n\n@Ignore\npublic class PopSlaveActingMasterIT extends ContainerIntegrationTestBase {\n    private static final String CONSUME_GROUP = PopSlaveActingMasterIT.class.getSimpleName() + \"_Consumer\";\n    private final static int MESSAGE_COUNT = 16;\n    private final Random random = new Random();\n    private static DefaultMQProducer producer;\n    private final static String MESSAGE_STRING = RandomStringUtils.random(1024);\n    private static final byte[] MESSAGE_BODY = MESSAGE_STRING.getBytes(StandardCharsets.UTF_8);\n    private final BrokerConfig brokerConfig = new BrokerConfig();\n\n    public PopSlaveActingMasterIT() {\n    }\n\n    void createTopic(String topic) {\n        createTopicTo(master1With3Replicas, topic, 1, 1);\n        createTopicTo(master2With3Replicas, topic, 1, 1);\n        createTopicTo(master3With3Replicas, topic, 1, 1);\n    }\n\n    @BeforeClass\n    public static void beforeClass() throws Throwable {\n        producer = createProducer(PopSlaveActingMasterIT.class.getSimpleName() + \"_PRODUCER\");\n        producer.setSendMsgTimeout(5000);\n        producer.start();\n    }\n\n    @AfterClass\n    public static void afterClass() throws Exception {\n        producer.shutdown();\n    }\n\n\n    @Test\n    public void testLocalActing_ackSlave() throws Exception {\n        String topic = PopSlaveActingMasterIT.class.getSimpleName() + random.nextInt(65535);\n        createTopic(topic);\n        String retryTopic = KeyBuilder.buildPopRetryTopic(topic, CONSUME_GROUP, brokerConfig.isEnableRetryTopicV2());\n        createTopic(retryTopic);\n\n        this.switchPop(topic);\n\n        producer.getDefaultMQProducerImpl().getmQClientFactory().updateTopicRouteInfoFromNameServer(topic);\n\n        MessageQueue messageQueue = new MessageQueue(topic, master1With3Replicas.getBrokerConfig().getBrokerName(), 0);\n        int sendSuccess = 0;\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            Message msg = new Message(topic, MESSAGE_BODY);\n            SendResult sendResult = producer.send(msg, messageQueue);\n            if (sendResult.getSendStatus() == SendStatus.SEND_OK) {\n                sendSuccess++;\n            }\n        }\n\n        final int finalSendSuccess = sendSuccess;\n        await().atMost(Duration.ofMinutes(1)).until(() -> finalSendSuccess >= MESSAGE_COUNT);\n\n        isolateBroker(master1With3Replicas);\n\n        DefaultMQPushConsumer consumer = createPushConsumer(CONSUME_GROUP);\n        consumer.subscribe(topic, \"*\");\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        List<String> consumedMessages = new CopyOnWriteArrayList<>();\n        consumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            msgs.forEach(msg -> {\n                consumedMessages.add(msg.getMsgId());\n            });\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        consumer.setClientRebalance(false);\n        consumer.start();\n\n        await().atMost(Duration.ofMinutes(1)).until(() -> consumedMessages.size() >= MESSAGE_COUNT);\n\n        consumer.shutdown();\n\n        List<String> retryMsgList = new CopyOnWriteArrayList<>();\n        DefaultMQPushConsumer pushConsumer = createPushConsumer(CONSUME_GROUP);\n        pushConsumer.subscribe(retryTopic, \"*\");\n        pushConsumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            for (MessageExt msg : msgs) {\n                retryMsgList.add(new String(msg.getBody()));\n            }\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        pushConsumer.start();\n\n        Thread.sleep(10000L);\n\n        assertThat(retryMsgList.size()).isEqualTo(0);\n\n        cancelIsolatedBroker(master1With3Replicas);\n        awaitUntilSlaveOK();\n\n        pushConsumer.shutdown();\n    }\n\n    @Test\n    public void testLocalActing_notAckSlave() throws Exception {\n        String topic = PopSlaveActingMasterIT.class.getSimpleName() + random.nextInt(65535);\n        createTopic(topic);\n        String retryTopic = KeyBuilder.buildPopRetryTopic(topic, CONSUME_GROUP, brokerConfig.isEnableRetryTopicV2());\n        createTopic(retryTopic);\n\n        this.switchPop(topic);\n\n        producer.getDefaultMQProducerImpl().getmQClientFactory().updateTopicRouteInfoFromNameServer(topic);\n\n        Set<String> sendToIsolateMsgSet = new HashSet<>();\n        MessageQueue messageQueue = new MessageQueue(topic, master1With3Replicas.getBrokerConfig().getBrokerName(), 0);\n        int sendSuccess = 0;\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            Message msg = new Message(topic, MESSAGE_BODY);\n            SendResult sendResult = producer.send(msg, messageQueue);\n            if (sendResult.getSendStatus() == SendStatus.SEND_OK) {\n                sendToIsolateMsgSet.add(new String(msg.getBody()));\n                sendSuccess++;\n            }\n        }\n\n        System.out.printf(\"send success %d%n\", sendSuccess);\n        final int finalSendSuccess = sendSuccess;\n        await().atMost(Duration.ofMinutes(1)).until(() -> finalSendSuccess >= MESSAGE_COUNT);\n\n        isolateBroker(master1With3Replicas);\n        System.out.printf(\"isolate master1%n\");\n\n        DefaultMQPushConsumer consumer = createPushConsumer(CONSUME_GROUP);\n        consumer.subscribe(topic, \"*\");\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        consumer.setPopInvisibleTime(5000L);\n        List<String> consumedMessages = new CopyOnWriteArrayList<>();\n        consumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            msgs.forEach(msg -> {\n                msg.setReconsumeTimes(0);\n                consumedMessages.add(msg.getMsgId());\n            });\n            return ConsumeConcurrentlyStatus.RECONSUME_LATER;\n        });\n        consumer.setClientRebalance(false);\n        consumer.start();\n\n        await().atMost(Duration.ofMinutes(1)).until(() -> consumedMessages.size() >= MESSAGE_COUNT);\n        consumer.shutdown();\n\n        List<String> retryMsgList = new CopyOnWriteArrayList<>();\n        DefaultMQPushConsumer pushConsumer = createPushConsumer(CONSUME_GROUP);\n        pushConsumer.subscribe(retryTopic, \"*\");\n        pushConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        pushConsumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            for (MessageExt msg : msgs) {\n                retryMsgList.add(new String(msg.getBody()));\n            }\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        pushConsumer.start();\n\n        AtomicInteger failCnt = new AtomicInteger(0);\n        await().atMost(Duration.ofMinutes(3)).pollInterval(Duration.ofSeconds(10)).until(() -> {\n            if (retryMsgList.size() < MESSAGE_COUNT) {\n                return false;\n            }\n\n            for (String msgBodyString : retryMsgList) {\n                if (!sendToIsolateMsgSet.contains(msgBodyString)) {\n                    return false;\n                }\n            }\n            return true;\n        });\n\n        cancelIsolatedBroker(master1With3Replicas);\n        awaitUntilSlaveOK();\n\n        pushConsumer.shutdown();\n    }\n\n    @Test\n    public void testRemoteActing_ackSlave() throws Exception {\n        String topic = PopSlaveActingMasterIT.class.getSimpleName() + random.nextInt(65535);\n        createTopic(topic);\n        String retryTopic = KeyBuilder.buildPopRetryTopic(topic, CONSUME_GROUP, brokerConfig.isEnableRetryTopicV2());\n        createTopic(retryTopic);\n\n        switchPop(topic);\n\n        producer.getDefaultMQProducerImpl().getmQClientFactory().updateTopicRouteInfoFromNameServer(topic);\n\n        MessageQueue messageQueue = new MessageQueue(topic, master1With3Replicas.getBrokerConfig().getBrokerName(), 0);\n        int sendSuccess = 0;\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            Message msg = new Message(topic, MESSAGE_BODY);\n            SendResult sendResult = producer.send(msg, messageQueue);\n            if (sendResult.getSendStatus() == SendStatus.SEND_OK) {\n                sendSuccess++;\n            }\n        }\n\n        final int finalSendSuccess = sendSuccess;\n        await().atMost(Duration.ofMinutes(1)).until(() -> finalSendSuccess >= MESSAGE_COUNT);\n\n        isolateBroker(master1With3Replicas);\n\n        isolateBroker(master2With3Replicas);\n        brokerContainer2.removeBroker(new BrokerIdentity(\n                master2With3Replicas.getBrokerConfig().getBrokerClusterName(),\n                master2With3Replicas.getBrokerConfig().getBrokerName(),\n                master2With3Replicas.getBrokerConfig().getBrokerId()));\n\n        DefaultMQPushConsumer consumer = createPushConsumer(CONSUME_GROUP);\n        consumer.subscribe(topic, \"*\");\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        List<String> consumedMessages = new CopyOnWriteArrayList<>();\n        consumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            msgs.forEach(msg -> {\n                consumedMessages.add(msg.getMsgId());\n            });\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        consumer.setClientRebalance(false);\n        consumer.start();\n\n        await().atMost(Duration.ofMinutes(2)).until(() -> consumedMessages.size() >= MESSAGE_COUNT);\n        consumer.shutdown();\n\n        List<String> retryMsgList = new CopyOnWriteArrayList<>();\n        DefaultMQPushConsumer pushConsumer = createPushConsumer(CONSUME_GROUP);\n        pushConsumer.subscribe(retryTopic, \"*\");\n        pushConsumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            for (MessageExt msg : msgs) {\n                retryMsgList.add(new String(msg.getBody()));\n            }\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        pushConsumer.start();\n\n        Thread.sleep(10000);\n\n        assertThat(retryMsgList.size()).isEqualTo(0);\n\n        cancelIsolatedBroker(master1With3Replicas);\n\n        //Add back master\n        master2With3Replicas = brokerContainer2.addBroker(buildConfigContext(master2With3Replicas.getBrokerConfig(), master2With3Replicas.getMessageStoreConfig()));\n        master2With3Replicas.start();\n        cancelIsolatedBroker(master2With3Replicas);\n\n        awaitUntilSlaveOK();\n\n        Thread.sleep(10000);\n\n        assertThat(retryMsgList.size()).isEqualTo(0);\n\n        pushConsumer.shutdown();\n    }\n\n    @Test\n    public void testRemoteActing_notAckSlave_getFromLocal() throws Exception {\n        String topic = PopSlaveActingMasterIT.class.getSimpleName() + random.nextInt(65535);\n        createTopic(topic);\n        this.switchPop(topic);\n\n        String retryTopic = KeyBuilder.buildPopRetryTopic(topic, CONSUME_GROUP, brokerConfig.isEnableRetryTopicV2());\n        createTopic(retryTopic);\n\n        producer.getDefaultMQProducerImpl().getmQClientFactory().updateTopicRouteInfoFromNameServer(topic);\n\n        Set<String> sendToIsolateMsgSet = new HashSet<>();\n        MessageQueue messageQueue = new MessageQueue(topic, master1With3Replicas.getBrokerConfig().getBrokerName(), 0);\n        int sendSuccess = 0;\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            Message msg = new Message(topic, MESSAGE_BODY);\n            SendResult sendResult = producer.send(msg, messageQueue);\n            if (sendResult.getSendStatus() == SendStatus.SEND_OK) {\n                sendToIsolateMsgSet.add(new String(msg.getBody()));\n                sendSuccess++;\n            }\n        }\n\n        final int finalSendSuccess = sendSuccess;\n        await().atMost(Duration.ofMinutes(1)).until(() -> finalSendSuccess >= MESSAGE_COUNT);\n\n        isolateBroker(master1With3Replicas);\n\n        isolateBroker(master2With3Replicas);\n        brokerContainer2.removeBroker(new BrokerIdentity(\n                master2With3Replicas.getBrokerConfig().getBrokerClusterName(),\n                master2With3Replicas.getBrokerConfig().getBrokerName(),\n                master2With3Replicas.getBrokerConfig().getBrokerId()));\n\n\n        DefaultMQPushConsumer consumer = createPushConsumer(CONSUME_GROUP);\n        consumer.subscribe(topic, \"*\");\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        List<String> consumedMessages = new CopyOnWriteArrayList<>();\n        consumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            msgs.forEach(msg -> {\n                consumedMessages.add(msg.getMsgId());\n            });\n            return ConsumeConcurrentlyStatus.RECONSUME_LATER;\n        });\n        consumer.setClientRebalance(false);\n        consumer.start();\n\n        await().atMost(Duration.ofMinutes(3)).until(() -> consumedMessages.size() >= MESSAGE_COUNT);\n        consumer.shutdown();\n\n\n        List<String> retryMsgList = new CopyOnWriteArrayList<>();\n        DefaultMQPushConsumer pushConsumer = createPushConsumer(CONSUME_GROUP);\n        pushConsumer.subscribe(retryTopic, \"*\");\n        pushConsumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            for (MessageExt msg : msgs) {\n                retryMsgList.add(new String(msg.getBody()));\n            }\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        pushConsumer.start();\n\n\n        await().atMost(Duration.ofMinutes(1)).until(() -> {\n            if (retryMsgList.size() < MESSAGE_COUNT) {\n                return false;\n            }\n\n            for (String msgBodyString : retryMsgList) {\n                if (!sendToIsolateMsgSet.contains(msgBodyString)) {\n                    return false;\n                }\n            }\n            return true;\n        });\n\n        cancelIsolatedBroker(master1With3Replicas);\n\n        //Add back master\n        master2With3Replicas = brokerContainer2.addBroker(buildConfigContext(master2With3Replicas.getBrokerConfig(), master2With3Replicas.getMessageStoreConfig()));\n        master2With3Replicas.start();\n        cancelIsolatedBroker(master2With3Replicas);\n\n        awaitUntilSlaveOK();\n        pushConsumer.shutdown();\n    }\n\n    @Test\n    public void testRemoteActing_notAckSlave_getFromRemote() throws Exception {\n        String topic = PopSlaveActingMasterIT.class.getSimpleName() + random.nextInt(65535);\n        createTopic(topic);\n        this.switchPop(topic);\n        String retryTopic = KeyBuilder.buildPopRetryTopic(topic, CONSUME_GROUP, brokerConfig.isEnableRetryTopicV2());\n        createTopic(retryTopic);\n\n        producer.getDefaultMQProducerImpl().getmQClientFactory().updateTopicRouteInfoFromNameServer(topic);\n\n        Set<String> sendToIsolateMsgSet = new HashSet<>();\n        MessageQueue messageQueue = new MessageQueue(topic, master1With3Replicas.getBrokerConfig().getBrokerName(), 0);\n        int sendSuccess = 0;\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            Message msg = new Message(topic, MESSAGE_BODY);\n            SendResult sendResult = producer.send(msg, messageQueue);\n            if (sendResult.getSendStatus() == SendStatus.SEND_OK) {\n                sendToIsolateMsgSet.add(new String(msg.getBody()));\n                sendSuccess++;\n            }\n        }\n\n        final int finalSendSuccess = sendSuccess;\n        await().atMost(Duration.ofMinutes(1)).until(() -> finalSendSuccess >= MESSAGE_COUNT);\n\n        isolateBroker(master1With3Replicas);\n\n        isolateBroker(master2With3Replicas);\n        brokerContainer2.removeBroker(new BrokerIdentity(\n                master2With3Replicas.getBrokerConfig().getBrokerClusterName(),\n                master2With3Replicas.getBrokerConfig().getBrokerName(),\n                master2With3Replicas.getBrokerConfig().getBrokerId()));\n\n        BrokerController slave1InBrokerContainer3 = getSlaveFromContainerByName(brokerContainer3, master1With3Replicas.getBrokerConfig().getBrokerName());\n        isolateBroker(slave1InBrokerContainer3);\n        brokerContainer3.removeBroker(new BrokerIdentity(\n                slave1InBrokerContainer3.getBrokerConfig().getBrokerClusterName(),\n                slave1InBrokerContainer3.getBrokerConfig().getBrokerName(),\n                slave1InBrokerContainer3.getBrokerConfig().getBrokerId()));\n\n        DefaultMQPushConsumer consumer = createPushConsumer(CONSUME_GROUP);\n        consumer.subscribe(topic, \"*\");\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        List<String> consumedMessages = new CopyOnWriteArrayList<>();\n        consumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            msgs.forEach(msg -> {\n                consumedMessages.add(msg.getMsgId());\n            });\n            return ConsumeConcurrentlyStatus.RECONSUME_LATER;\n        });\n        consumer.setClientRebalance(false);\n        consumer.start();\n\n        await().atMost(Duration.ofMinutes(1)).until(() -> consumedMessages.size() >= MESSAGE_COUNT);\n        consumer.shutdown();\n\n\n        List<String> retryMsgList = new CopyOnWriteArrayList<>();\n        DefaultMQPushConsumer pushConsumer = createPushConsumer(CONSUME_GROUP);\n        pushConsumer.subscribe(retryTopic, \"*\");\n        pushConsumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            for (MessageExt msg : msgs) {\n                retryMsgList.add(new String(msg.getBody()));\n            }\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        pushConsumer.start();\n\n        Thread.sleep(10000);\n\n        await().atMost(Duration.ofMinutes(1)).until(() -> {\n            if (retryMsgList.size() < MESSAGE_COUNT) {\n                return false;\n            }\n\n            for (String msgBodyString : retryMsgList) {\n                if (!sendToIsolateMsgSet.contains(msgBodyString)) {\n                    return false;\n                }\n            }\n            return true;\n        });\n\n        cancelIsolatedBroker(master1With3Replicas);\n\n        //Add back master\n        master2With3Replicas = brokerContainer2.addBroker(buildConfigContext(master2With3Replicas.getBrokerConfig(), master2With3Replicas.getMessageStoreConfig()));\n        master2With3Replicas.start();\n        cancelIsolatedBroker(master2With3Replicas);\n\n        //Add back slave1 to container3\n        slave1InBrokerContainer3 = brokerContainer3.addBroker(buildConfigContext(slave1InBrokerContainer3.getBrokerConfig(), slave1InBrokerContainer3.getMessageStoreConfig()));\n        slave1InBrokerContainer3.start();\n        cancelIsolatedBroker(slave1InBrokerContainer3);\n\n        awaitUntilSlaveOK();\n        pushConsumer.shutdown();\n    }\n\n    private void switchPop(String topic) throws Exception {\n        for (BrokerContainer brokerContainer : brokerContainerList) {\n            for (InnerBrokerController master : brokerContainer.getMasterBrokers()) {\n                String brokerAddr = master.getBrokerAddr();\n                defaultMQAdminExt.setMessageRequestMode(brokerAddr, topic, CONSUME_GROUP, MessageRequestMode.POP, 8, 60_000);\n            }\n            for (InnerSalveBrokerController slave : brokerContainer.getSlaveBrokers()) {\n                defaultMQAdminExt.setMessageRequestMode(slave.getBrokerAddr(), topic, CONSUME_GROUP, MessageRequestMode.POP, 8, 60_000);\n            }\n        }\n\n    }\n\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/container/PullMultipleReplicasIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.container;\n\nimport java.io.UnsupportedEncodingException;\nimport java.lang.reflect.Field;\nimport java.net.InetSocketAddress;\nimport java.nio.charset.StandardCharsets;\nimport java.time.Duration;\nimport java.util.List;\n\nimport org.apache.commons.lang3.RandomStringUtils;\nimport org.apache.rocketmq.client.impl.consumer.DefaultMQPullConsumerImpl;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.container.InnerSalveBrokerController;\nimport org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.junit.AfterClass;\nimport org.junit.BeforeClass;\nimport org.junit.Ignore;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Java6Assertions.assertThat;\nimport static org.awaitility.Awaitility.await;\n\n@Ignore\npublic class PullMultipleReplicasIT extends ContainerIntegrationTestBase {\n    private static DefaultMQPullConsumer pullConsumer;\n    private static DefaultMQProducer producer;\n    private static MQClientInstance mqClientInstance;\n\n    private static final String MESSAGE_STRING = RandomStringUtils.random(1024);\n    private static final byte[] MESSAGE_BODY = MESSAGE_STRING.getBytes(StandardCharsets.UTF_8);\n\n    public PullMultipleReplicasIT() {\n    }\n\n    @BeforeClass\n    public static void beforeClass() throws Exception {\n\n        pullConsumer = createPullConsumer(PullMultipleReplicasIT.class.getSimpleName() + \"_Consumer\");\n        pullConsumer.start();\n\n        Field field = DefaultMQPullConsumerImpl.class.getDeclaredField(\"mQClientFactory\");\n        field.setAccessible(true);\n        mqClientInstance = (MQClientInstance) field.get(pullConsumer.getDefaultMQPullConsumerImpl());\n\n        producer = createProducer(PullMultipleReplicasIT.class.getSimpleName() + \"_Producer\");\n        producer.setSendMsgTimeout(15 * 1000);\n        producer.start();\n    }\n\n    @AfterClass\n    public static void afterClass() {\n        producer.shutdown();\n        pullConsumer.shutdown();\n    }\n\n    @Test\n    public void testPullMessageFromSlave() throws InterruptedException, RemotingException, MQClientException, MQBrokerException, UnsupportedEncodingException {\n        awaitUntilSlaveOK();\n\n        Message msg = new Message(THREE_REPLICAS_TOPIC, MESSAGE_BODY);\n        SendResult sendResult = producer.send(msg);\n        assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n\n        final MessageQueue messageQueue = sendResult.getMessageQueue();\n        final long queueOffset = sendResult.getQueueOffset();\n\n        final PullResult[] pullResult = {null};\n        await().atMost(Duration.ofSeconds(5)).until(() -> {\n            pullResult[0] = pullConsumer.pull(messageQueue, \"*\", queueOffset, 1);\n            return pullResult[0].getPullStatus() == PullStatus.FOUND;\n        });\n\n        List<MessageExt> msgFoundList = pullResult[0].getMsgFoundList();\n        assertThat(msgFoundList.size()).isEqualTo(1);\n        assertThat(new String(msgFoundList.get(0).getBody(), RemotingHelper.DEFAULT_CHARSET)).isEqualTo(MESSAGE_STRING);\n\n        // Pull the same message from the slave broker\n        pullConsumer.getDefaultMQPullConsumerImpl().getPullAPIWrapper().updatePullFromWhichNode(messageQueue, 1);\n\n        await().atMost(Duration.ofSeconds(5)).until(() -> {\n            pullResult[0] = pullConsumer.pull(messageQueue, \"*\", queueOffset, 1);\n            return pullResult[0].getPullStatus() == PullStatus.FOUND;\n        });\n\n        msgFoundList = pullResult[0].getMsgFoundList();\n        assertThat(msgFoundList.size()).isEqualTo(1);\n        assertThat(new String(msgFoundList.get(0).getBody(), RemotingHelper.DEFAULT_CHARSET)).isEqualTo(MESSAGE_STRING);\n\n        // Pull the same message from the slave broker\n        pullConsumer.getDefaultMQPullConsumerImpl().getPullAPIWrapper().updatePullFromWhichNode(messageQueue, 2);\n\n        await().atMost(Duration.ofSeconds(5)).until(() -> {\n            pullResult[0] = pullConsumer.pull(messageQueue, \"*\", queueOffset, 1);\n            return pullResult[0].getPullStatus() == PullStatus.FOUND;\n        });\n\n        msgFoundList = pullResult[0].getMsgFoundList();\n        assertThat(msgFoundList.size()).isEqualTo(1);\n        assertThat(new String(msgFoundList.get(0).getBody(), RemotingHelper.DEFAULT_CHARSET)).isEqualTo(MESSAGE_STRING);\n\n        pullConsumer.getDefaultMQPullConsumerImpl().getPullAPIWrapper().updatePullFromWhichNode(messageQueue, 0);\n    }\n\n    @Test\n    public void testSendMessageBackToSlave() throws InterruptedException, RemotingException, MQClientException, MQBrokerException, UnsupportedEncodingException {\n        awaitUntilSlaveOK();\n\n        String clusterTopic = \"TOPIC_ON_BROKER2_AND_BROKER3_FOR_MESSAGE_BACK\";\n        createTopicTo(master1With3Replicas, clusterTopic);\n        createTopicTo(master3With3Replicas, clusterTopic);\n\n        Message msg = new Message(clusterTopic, MESSAGE_BODY);\n        producer.setSendMsgTimeout(10 * 1000);\n\n        final MessageQueue[] selectedQueue = new MessageQueue[1];\n        await().atMost(Duration.ofSeconds(5)).until(() -> {\n            for (final MessageQueue queue : producer.fetchPublishMessageQueues(clusterTopic)) {\n                if (queue.getBrokerName().equals(master3With3Replicas.getBrokerConfig().getBrokerName())) {\n                    selectedQueue[0] = queue;\n                }\n            }\n            return selectedQueue[0] != null;\n        });\n\n        SendResult sendResult = producer.send(msg, selectedQueue[0]);\n        assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n\n        final MessageQueue messageQueue = sendResult.getMessageQueue();\n        final long queueOffset = sendResult.getQueueOffset();\n\n        final PullResult[] pullResult = {null};\n        await().atMost(Duration.ofSeconds(60)).until(() -> {\n            pullResult[0] = pullConsumer.pull(messageQueue, \"*\", queueOffset, 1);\n            return pullResult[0].getPullStatus() == PullStatus.FOUND;\n        });\n\n        await().atMost(Duration.ofSeconds(60)).until(() -> {\n            DefaultMessageStore messageStore = (DefaultMessageStore) master3With3Replicas.getMessageStore();\n            return messageStore.getHaService().inSyncReplicasNums(messageStore.getMaxPhyOffset()) == 3;\n        });\n\n        InnerSalveBrokerController slaveBroker = null;\n        for (InnerSalveBrokerController slave : brokerContainer1.getSlaveBrokers()) {\n            if (slave.getBrokerConfig().getBrokerName().equals(master3With3Replicas.getBrokerConfig().getBrokerName())) {\n                slaveBroker = slave;\n            }\n        }\n\n        assertThat(slaveBroker).isNotNull();\n\n        MessageExt backMessage = pullResult[0].getMsgFoundList().get(0);\n\n        // Message will be sent to the master broker(master1With3Replicas) beside a slave broker of master3With3Replicas\n        backMessage.setStoreHost(new InetSocketAddress(slaveBroker.getBrokerConfig().getBrokerIP1(), slaveBroker.getBrokerConfig().getListenPort()));\n        pullConsumer.sendMessageBack(backMessage, 0);\n\n        String retryTopic = MixAll.getRetryTopic(pullConsumer.getConsumerGroup());\n        // Retry topic only has one queue by default\n        MessageQueue newMsgQueue = new MessageQueue(retryTopic, master1With3Replicas.getBrokerConfig().getBrokerName(), 0);\n        await().atMost(Duration.ofSeconds(60)).until(() -> {\n            pullResult[0] = pullConsumer.pull(newMsgQueue, \"*\", 0, 1);\n            return pullResult[0].getPullStatus() == PullStatus.FOUND;\n        });\n\n        List<MessageExt> msgFoundList = pullResult[0].getMsgFoundList();\n        assertThat(msgFoundList.size()).isEqualTo(1);\n        assertThat(new String(msgFoundList.get(0).getBody(), RemotingHelper.DEFAULT_CHARSET)).isEqualTo(MESSAGE_STRING);\n\n        awaitUntilSlaveOK();\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/container/PushMultipleReplicasIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.container;\n\nimport java.io.UnsupportedEncodingException;\nimport java.time.Duration;\nimport java.util.Map;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.rocketmq.container.InnerSalveBrokerController;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.Message;\nimport org.junit.AfterClass;\nimport org.junit.BeforeClass;\nimport org.junit.Ignore;\nimport org.junit.Test;\n\nimport static org.awaitility.Awaitility.await;\n\n@Ignore\npublic class PushMultipleReplicasIT extends ContainerIntegrationTestBase {\n    private static DefaultMQProducer producer;\n\n    private static final String TOPIC = PushMultipleReplicasIT.class.getSimpleName() + \"_TOPIC\";\n    private static final String REDIRECT_TOPIC = PushMultipleReplicasIT.class.getSimpleName() + \"_REDIRECT_TOPIC\";\n    private static final String CONSUMER_GROUP = PushMultipleReplicasIT.class.getSimpleName() + \"_Consumer\";\n    private static final int MESSAGE_COUNT = 32;\n\n    public PushMultipleReplicasIT() throws UnsupportedEncodingException {\n    }\n\n    @BeforeClass\n    public static void beforeClass() throws Throwable {\n        createTopicTo(master1With3Replicas, TOPIC,1, 1);\n        producer = createProducer(PushMultipleReplicasIT.class.getSimpleName() + \"_PRODUCER\");\n        producer.setSendMsgTimeout(15 * 1000);\n        producer.start();\n\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            producer.send(new Message(TOPIC, Integer.toString(i).getBytes()));\n        }\n\n        createTopicTo(master3With3Replicas, REDIRECT_TOPIC, 1, 1);\n    }\n\n    @AfterClass\n    public static void afterClass() throws Exception {\n        producer.shutdown();\n    }\n\n    @Test\n    public void consumeMessageFromSlave_PushConsumer() throws MQClientException {\n        // Wait topic synchronization\n        await().atMost(Duration.ofMinutes(1)).until(() -> {\n            InnerSalveBrokerController slaveBroker = brokerContainer2.getSlaveBrokers().iterator().next();\n            return slaveBroker.getTopicConfigManager().selectTopicConfig(TOPIC) != null;\n        });\n        isolateBroker(master1With3Replicas);\n        DefaultMQPushConsumer pushConsumer = createPushConsumer(CONSUMER_GROUP);\n        pushConsumer.subscribe(TOPIC, \"*\");\n        pushConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n        AtomicInteger receivedMsgCount = new AtomicInteger(0);\n        pushConsumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            receivedMsgCount.addAndGet(msgs.size());\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        pushConsumer.start();\n        await().atMost(Duration.ofMinutes(5)).until(() -> receivedMsgCount.get() >= MESSAGE_COUNT);\n\n        await().atMost(Duration.ofMinutes(1)).until(() -> {\n            pushConsumer.getDefaultMQPushConsumerImpl().persistConsumerOffset();\n            Map<Integer, Long> slaveOffsetTable = null;\n            for (InnerSalveBrokerController slave : brokerContainer2.getSlaveBrokers()) {\n                if (slave.getBrokerConfig().getBrokerName().equals(master1With3Replicas.getBrokerConfig().getBrokerName())) {\n                    slaveOffsetTable = slave.getConsumerOffsetManager().queryOffset(CONSUMER_GROUP, TOPIC);\n                }\n            }\n\n            if (slaveOffsetTable != null) {\n                long totalOffset = 0;\n                for (final Long offset : slaveOffsetTable.values()) {\n                    totalOffset += offset;\n                }\n\n                return totalOffset >= MESSAGE_COUNT;\n            }\n            return false;\n        });\n\n        pushConsumer.shutdown();\n        cancelIsolatedBroker(master1With3Replicas);\n\n        awaitUntilSlaveOK();\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/container/RebalanceLockOnSlaveIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.container;\n\nimport java.time.Duration;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.FindBrokerResult;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.ServiceState;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.junit.AfterClass;\nimport org.junit.BeforeClass;\nimport org.junit.Ignore;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.awaitility.Awaitility.await;\n\n/**\n * Test lock on slave when acting master enabled\n */\n@Ignore\npublic class RebalanceLockOnSlaveIT extends ContainerIntegrationTestBase {\n    private static final String THREE_REPLICA_CONSUMER_GROUP = \"SyncConsumerOffsetIT_ConsumerThreeReplica\";\n\n    private static DefaultMQProducer mqProducer;\n    private static DefaultMQPushConsumer mqConsumerThreeReplica1;\n    private static DefaultMQPushConsumer mqConsumerThreeReplica2;\n    private static DefaultMQPushConsumer mqConsumerThreeReplica3;\n\n    public RebalanceLockOnSlaveIT() {\n    }\n\n    @BeforeClass\n    public static void beforeClass() throws Exception {\n        mqProducer = createProducer(\"SyncConsumerOffsetIT_Producer\");\n        mqProducer.start();\n\n        mqConsumerThreeReplica1 = createPushConsumer(THREE_REPLICA_CONSUMER_GROUP);\n        mqConsumerThreeReplica1.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);\n        mqConsumerThreeReplica1.subscribe(THREE_REPLICAS_TOPIC, \"*\");\n\n        mqConsumerThreeReplica2 = createPushConsumer(THREE_REPLICA_CONSUMER_GROUP);\n        mqConsumerThreeReplica2.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);\n        mqConsumerThreeReplica2.subscribe(THREE_REPLICAS_TOPIC, \"*\");\n\n        mqConsumerThreeReplica3 = createPushConsumer(THREE_REPLICA_CONSUMER_GROUP);\n        mqConsumerThreeReplica3.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);\n        mqConsumerThreeReplica3.subscribe(THREE_REPLICAS_TOPIC, \"*\");\n    }\n\n    @AfterClass\n    public static void afterClass() {\n        if (mqProducer != null) {\n            mqProducer.shutdown();\n        }\n    }\n\n    @Test\n    public void lockFromSlave() throws Exception {\n        awaitUntilSlaveOK();\n\n        mqConsumerThreeReplica3.registerMessageListener((MessageListenerOrderly) (msgs, context) -> ConsumeOrderlyStatus.SUCCESS);\n        mqConsumerThreeReplica3.start();\n\n        final Set<MessageQueue> mqSet = mqConsumerThreeReplica3.fetchSubscribeMessageQueues(THREE_REPLICAS_TOPIC);\n\n        assertThat(targetTopicMqCount(mqSet, THREE_REPLICAS_TOPIC)).isEqualTo(24);\n\n        for (MessageQueue mq : mqSet) {\n            await().atMost(Duration.ofSeconds(60)).until(() -> mqConsumerThreeReplica3.getDefaultMQPushConsumerImpl().getRebalanceImpl().lock(mq));\n        }\n\n        isolateBroker(master3With3Replicas);\n\n        mqConsumerThreeReplica3.getDefaultMQPushConsumerImpl().getmQClientFactory().updateTopicRouteInfoFromNameServer(THREE_REPLICAS_TOPIC);\n        FindBrokerResult result = mqConsumerThreeReplica3.getDefaultMQPushConsumerImpl().getmQClientFactory().findBrokerAddressInSubscribe(\n            master3With3Replicas.getBrokerConfig().getBrokerName(), MixAll.MASTER_ID, true);\n        assertThat(result).isNotNull();\n\n        for (MessageQueue mq : mqSet) {\n            if (mq.getBrokerName().equals(master3With3Replicas.getBrokerConfig().getBrokerName())) {\n                await().atMost(Duration.ofSeconds(60)).until(() -> mqConsumerThreeReplica3.getDefaultMQPushConsumerImpl().getRebalanceImpl().lock(mq));\n            }\n        }\n\n        removeSlaveBroker(1, brokerContainer1, master3With3Replicas);\n        assertThat(brokerContainer1.getSlaveBrokers().size()).isEqualTo(1);\n\n        mqConsumerThreeReplica3.getDefaultMQPushConsumerImpl().getmQClientFactory().updateTopicRouteInfoFromNameServer(THREE_REPLICAS_TOPIC);\n\n        for (MessageQueue mq : mqSet) {\n            if (mq.getBrokerName().equals(master3With3Replicas.getBrokerConfig().getBrokerName())) {\n                await().atMost(Duration.ofSeconds(60)).until(() -> !mqConsumerThreeReplica3.getDefaultMQPushConsumerImpl().getRebalanceImpl().lock(mq));\n            }\n        }\n\n        cancelIsolatedBroker(master3With3Replicas);\n        createAndAddSlave(1, brokerContainer1, master3With3Replicas);\n        awaitUntilSlaveOK();\n\n        mqConsumerThreeReplica3.shutdown();\n        await().atMost(100, TimeUnit.SECONDS).until(() -> mqConsumerThreeReplica3.getDefaultMQPushConsumerImpl().getServiceState() == ServiceState.SHUTDOWN_ALREADY);\n    }\n\n    @Ignore\n    @Test\n    public void multiConsumerLockFromSlave() throws MQClientException, InterruptedException {\n        awaitUntilSlaveOK();\n\n        mqConsumerThreeReplica1.registerMessageListener((MessageListenerOrderly) (msgs, context) -> ConsumeOrderlyStatus.SUCCESS);\n        mqConsumerThreeReplica1.start();\n\n        mqConsumerThreeReplica1.getDefaultMQPushConsumerImpl().doRebalance();\n        Set<MessageQueue> mqSet1 = filterMessageQueue(mqConsumerThreeReplica1.getDefaultMQPushConsumerImpl().getRebalanceImpl().getProcessQueueTable().keySet(), THREE_REPLICAS_TOPIC);\n\n        assertThat(mqSet1.size()).isEqualTo(24);\n\n        isolateBroker(master3With3Replicas);\n\n        System.out.printf(\"%s isolated%n\", master3With3Replicas.getBrokerConfig().getCanonicalName());\n\n        Thread.sleep(5000);\n\n        mqConsumerThreeReplica2.registerMessageListener((MessageListenerOrderly) (msgs, context) -> ConsumeOrderlyStatus.SUCCESS);\n        mqConsumerThreeReplica2.start();\n\n        Thread.sleep(5000);\n\n        mqConsumerThreeReplica1.getDefaultMQPushConsumerImpl().getmQClientFactory().updateTopicRouteInfoFromNameServer(THREE_REPLICAS_TOPIC);\n        mqConsumerThreeReplica2.getDefaultMQPushConsumerImpl().getmQClientFactory().updateTopicRouteInfoFromNameServer(THREE_REPLICAS_TOPIC);\n\n        assertThat(mqConsumerThreeReplica1.getDefaultMQPushConsumerImpl().getmQClientFactory().findBrokerAddressInSubscribe(\n            master3With3Replicas.getBrokerConfig().getBrokerName(), MixAll.MASTER_ID, true)).isNotNull();\n\n        mqConsumerThreeReplica2.getDefaultMQPushConsumerImpl().getmQClientFactory().findBrokerAddressInSubscribe(\n            master3With3Replicas.getBrokerConfig().getBrokerName(), MixAll.MASTER_ID, true);\n        assertThat(mqConsumerThreeReplica2.getDefaultMQPushConsumerImpl().getmQClientFactory().findBrokerAddressInSubscribe(\n            master3With3Replicas.getBrokerConfig().getBrokerName(), MixAll.MASTER_ID, true)).isNotNull();\n\n        mqConsumerThreeReplica1.getDefaultMQPushConsumerImpl().doRebalance();\n        mqConsumerThreeReplica2.getDefaultMQPushConsumerImpl().doRebalance();\n\n        Set<MessageQueue> mqSet2 = filterMessageQueue(mqConsumerThreeReplica2.getDefaultMQPushConsumerImpl().getRebalanceImpl().getProcessQueueTable().keySet(), THREE_REPLICAS_TOPIC);\n\n        mqSet1 = filterMessageQueue(mqConsumerThreeReplica1.getDefaultMQPushConsumerImpl().getRebalanceImpl().getProcessQueueTable().keySet(), THREE_REPLICAS_TOPIC);\n\n        List<MessageQueue> mqList = new ArrayList<>();\n\n        for (MessageQueue mq : mqSet2) {\n            if (mq.getTopic().equals(THREE_REPLICAS_TOPIC)) {\n                mqList.add(mq);\n            }\n        }\n\n        for (MessageQueue mq : mqSet1) {\n            if (mq.getTopic().equals(THREE_REPLICAS_TOPIC)) {\n                mqList.add(mq);\n            }\n        }\n\n        await().atMost(Duration.ofSeconds(30)).until(() -> mqList.size() == 24);\n\n        cancelIsolatedBroker(master3With3Replicas);\n        awaitUntilSlaveOK();\n\n        mqConsumerThreeReplica1.shutdown();\n        mqConsumerThreeReplica2.shutdown();\n\n        await().atMost(100, TimeUnit.SECONDS).until(() ->\n            mqConsumerThreeReplica1.getDefaultMQPushConsumerImpl().getServiceState() == ServiceState.SHUTDOWN_ALREADY &&\n                mqConsumerThreeReplica2.getDefaultMQPushConsumerImpl().getServiceState() == ServiceState.SHUTDOWN_ALREADY\n        );\n    }\n\n    private static int targetTopicMqCount(Set<MessageQueue> mqSet, String topic) {\n        int count = 0;\n        for (MessageQueue mq : mqSet) {\n            if (mq.getTopic().equals(topic)) {\n                count++;\n            }\n        }\n        return count;\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/container/ScheduleSlaveActingMasterIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.container;\n\nimport java.nio.charset.StandardCharsets;\nimport java.time.Duration;\nimport java.util.Random;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.commons.lang3.RandomStringUtils;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.BrokerIdentity;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.junit.AfterClass;\nimport org.junit.BeforeClass;\nimport org.junit.Ignore;\nimport org.junit.Test;\n\nimport static org.awaitility.Awaitility.await;\n\n//The test is correct, but it takes too much time and not core functions, so it is ignored for the time being\n@Ignore\npublic class ScheduleSlaveActingMasterIT extends ContainerIntegrationTestBase {\n\n    private static final String CONSUME_GROUP = ScheduleSlaveActingMasterIT.class.getSimpleName() + \"_Consumer\";\n    private static final int MESSAGE_COUNT = 32;\n    private final Random random = new Random();\n    private static DefaultMQProducer producer;\n    private static final String MESSAGE_STRING = RandomStringUtils.random(1024);\n    private static final byte[] MESSAGE_BODY = MESSAGE_STRING.getBytes(StandardCharsets.UTF_8);\n\n    void createTopic(String topic) {\n        createTopicTo(master1With3Replicas, topic, 1, 1);\n        createTopicTo(master2With3Replicas, topic, 1, 1);\n        createTopicTo(master3With3Replicas, topic, 1, 1);\n    }\n\n    @BeforeClass\n    public static void beforeClass() throws Throwable {\n        producer = createProducer(ScheduleSlaveActingMasterIT.class.getSimpleName() + \"_PRODUCER\");\n        producer.setSendMsgTimeout(5000);\n        producer.start();\n    }\n\n    @AfterClass\n    public static void afterClass() throws Exception {\n        producer.shutdown();\n    }\n\n    @Test\n    public void testLocalActing_delayMsg() throws Exception {\n        awaitUntilSlaveOK();\n        String topic = ScheduleSlaveActingMasterIT.class.getSimpleName() + random.nextInt(65535);\n        createTopic(topic);\n        DefaultMQPushConsumer pushConsumer = createPushConsumer(CONSUME_GROUP);\n        pushConsumer.subscribe(topic, \"*\");\n        AtomicInteger receivedMsgCount = new AtomicInteger(0);\n        AtomicInteger inTimeMsgCount = new AtomicInteger(0);\n        pushConsumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            long period = System.currentTimeMillis() - msgs.get(0).getBornTimestamp();\n            if (Math.abs(period - 30000) <= 4000) {\n                inTimeMsgCount.addAndGet(msgs.size());\n            }\n            receivedMsgCount.addAndGet(msgs.size());\n            msgs.forEach(x -> System.out.printf(x + \"%n\"));\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        pushConsumer.start();\n\n        MessageQueue messageQueue = new MessageQueue(topic, master1With3Replicas.getBrokerConfig().getBrokerName(), 0);\n        int sendSuccess = 0;\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            Message msg = new Message(topic, MESSAGE_BODY);\n            msg.setDelayTimeLevel(4);\n            SendResult sendResult = producer.send(msg, messageQueue);\n            if (sendResult.getSendStatus() == SendStatus.SEND_OK) {\n                sendSuccess++;\n            }\n        }\n        final int finalSendSuccess = sendSuccess;\n        await().atMost(Duration.ofMinutes(1)).until(() -> finalSendSuccess >= MESSAGE_COUNT);\n        System.out.printf(\"send success%n\");\n\n        isolateBroker(master1With3Replicas);\n        brokerContainer1.removeBroker(new BrokerIdentity(\n            master1With3Replicas.getBrokerConfig().getBrokerClusterName(),\n            master1With3Replicas.getBrokerConfig().getBrokerName(),\n            master1With3Replicas.getBrokerConfig().getBrokerId()));\n\n        System.out.printf(\"Remove master1%n\");\n\n        await().atMost(Duration.ofMinutes(1)).until(() -> receivedMsgCount.get() >= MESSAGE_COUNT && inTimeMsgCount.get() >= MESSAGE_COUNT * 0.95);\n\n        System.out.printf(\"consumer received %d msg, %d in time%n\", receivedMsgCount.get(), inTimeMsgCount.get());\n\n        pushConsumer.shutdown();\n\n        //Add back master\n        master1With3Replicas = brokerContainer1.addBroker(buildConfigContext(master1With3Replicas.getBrokerConfig(), master1With3Replicas.getMessageStoreConfig()));\n        master1With3Replicas.start();\n        cancelIsolatedBroker(master1With3Replicas);\n        System.out.printf(\"Add back master1%n\");\n\n        awaitUntilSlaveOK();\n        // sleep a while to recover\n        Thread.sleep(30000);\n    }\n\n    @Test\n    public void testLocalActing_timerMsg() throws Exception {\n        awaitUntilSlaveOK();\n        String topic = ScheduleSlaveActingMasterIT.class.getSimpleName() + random.nextInt(65535);\n        createTopic(topic);\n        DefaultMQPushConsumer pushConsumer = createPushConsumer(CONSUME_GROUP);\n        pushConsumer.subscribe(topic, \"*\");\n        AtomicInteger receivedMsgCount = new AtomicInteger(0);\n        AtomicInteger inTimeMsgCount = new AtomicInteger(0);\n        pushConsumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            long period = System.currentTimeMillis() - msgs.get(0).getBornTimestamp();\n            if (Math.abs(period - 30000) <= 1000) {\n                inTimeMsgCount.addAndGet(msgs.size());\n            }\n            receivedMsgCount.addAndGet(msgs.size());\n            msgs.forEach(x -> System.out.printf(x + \"%n\"));\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        pushConsumer.start();\n\n        MessageQueue messageQueue = new MessageQueue(topic, master1With3Replicas.getBrokerConfig().getBrokerName(), 0);\n        int sendSuccess = 0;\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            Message msg = new Message(topic, MESSAGE_BODY);\n            msg.setDelayTimeSec(30);\n            SendResult sendResult = producer.send(msg, messageQueue);\n            if (sendResult.getSendStatus() == SendStatus.SEND_OK) {\n                sendSuccess++;\n            }\n        }\n        final int finalSendSuccess = sendSuccess;\n        await().atMost(Duration.ofMinutes(2)).until(() -> finalSendSuccess >= MESSAGE_COUNT);\n        System.out.printf(\"send success%n\");\n\n        isolateBroker(master1With3Replicas);\n        brokerContainer1.removeBroker(new BrokerIdentity(\n            master1With3Replicas.getBrokerConfig().getBrokerClusterName(),\n            master1With3Replicas.getBrokerConfig().getBrokerName(),\n            master1With3Replicas.getBrokerConfig().getBrokerId()));\n\n        System.out.printf(\"Remove master1%n\");\n\n        await().atMost(Duration.ofMinutes(1)).until(() -> receivedMsgCount.get() >= MESSAGE_COUNT && inTimeMsgCount.get() >= MESSAGE_COUNT * 0.95);\n\n        System.out.printf(\"consumer received %d msg, %d in time%n\", receivedMsgCount.get(), inTimeMsgCount.get());\n\n        pushConsumer.shutdown();\n\n        //Add back master\n        master1With3Replicas = brokerContainer1.addBroker(buildConfigContext(master1With3Replicas.getBrokerConfig(), master1With3Replicas.getMessageStoreConfig()));\n        master1With3Replicas.start();\n        cancelIsolatedBroker(master1With3Replicas);\n        System.out.printf(\"Add back master1%n\");\n\n        awaitUntilSlaveOK();\n        // sleep a while to recover\n        Thread.sleep(20000);\n    }\n\n    @Test\n    public void testRemoteActing_delayMsg() throws Exception {\n        awaitUntilSlaveOK();\n\n        String topic = ScheduleSlaveActingMasterIT.class.getSimpleName() + random.nextInt(65535);\n        createTopic(topic);\n        AtomicInteger receivedMsgCount = new AtomicInteger(0);\n        AtomicInteger inTimeMsgCount = new AtomicInteger(0);\n        AtomicInteger master3MsgCount = new AtomicInteger(0);\n\n        MessageQueue messageQueue = new MessageQueue(topic, master1With3Replicas.getBrokerConfig().getBrokerName(), 0);\n        int sendSuccess = 0;\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            Message msg = new Message(topic, MESSAGE_BODY);\n            msg.setDelayTimeLevel(4);\n            SendResult sendResult = producer.send(msg, messageQueue);\n            if (sendResult.getSendStatus() == SendStatus.SEND_OK) {\n                sendSuccess++;\n            }\n        }\n        final int finalSendSuccess = sendSuccess;\n        await().atMost(Duration.ofMinutes(1)).until(() -> finalSendSuccess >= MESSAGE_COUNT);\n        long sendCompleteTimeStamp = System.currentTimeMillis();\n        System.out.printf(\"send success%n\");\n\n        DefaultMQPushConsumer pushConsumer = createPushConsumer(CONSUME_GROUP);\n        pushConsumer.subscribe(topic, \"*\");\n        pushConsumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            long period = System.currentTimeMillis() - sendCompleteTimeStamp;\n            // Remote Acting lead to born timestamp, msgId changed, it need to polish.\n            if (Math.abs(period - 30000) <= 4000) {\n                inTimeMsgCount.addAndGet(msgs.size());\n            }\n            if (msgs.get(0).getBrokerName().equals(master3With3Replicas.getBrokerConfig().getBrokerName())) {\n                master3MsgCount.addAndGet(msgs.size());\n            }\n            receivedMsgCount.addAndGet(msgs.size());\n            msgs.forEach(x -> System.out.printf(\"cost \" + period + \" \" + x + \"%n\"));\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        pushConsumer.start();\n\n        isolateBroker(master1With3Replicas);\n        BrokerIdentity master1BrokerIdentity = new BrokerIdentity(\n            master1With3Replicas.getBrokerConfig().getBrokerClusterName(),\n            master1With3Replicas.getBrokerConfig().getBrokerName(),\n            master1With3Replicas.getBrokerConfig().getBrokerId());\n\n        brokerContainer1.removeBroker(master1BrokerIdentity);\n        System.out.printf(\"Remove master1%n\");\n\n        isolateBroker(master2With3Replicas);\n        BrokerIdentity master2BrokerIdentity = new BrokerIdentity(\n            master2With3Replicas.getBrokerConfig().getBrokerClusterName(),\n            master2With3Replicas.getBrokerConfig().getBrokerName(),\n            master2With3Replicas.getBrokerConfig().getBrokerId());\n        brokerContainer2.removeBroker(master2BrokerIdentity);\n        System.out.printf(\"Remove master2%n\");\n\n        await().atMost(Duration.ofMinutes(2)).until(() -> receivedMsgCount.get() >= MESSAGE_COUNT && master3MsgCount.get() >= MESSAGE_COUNT && inTimeMsgCount.get() >= MESSAGE_COUNT * 0.95);\n\n        System.out.printf(\"consumer received %d msg, %d in time%n\", receivedMsgCount.get(), inTimeMsgCount.get());\n\n        pushConsumer.shutdown();\n\n        //Add back master\n        master1With3Replicas = brokerContainer1.addBroker(buildConfigContext(master1With3Replicas.getBrokerConfig(), master1With3Replicas.getMessageStoreConfig()));\n        master1With3Replicas.start();\n        cancelIsolatedBroker(master1With3Replicas);\n        System.out.printf(\"Add back master1%n\");\n\n        //Add back master\n        master2With3Replicas = brokerContainer2.addBroker(buildConfigContext(master2With3Replicas.getBrokerConfig(), master2With3Replicas.getMessageStoreConfig()));\n        master2With3Replicas.start();\n        cancelIsolatedBroker(master2With3Replicas);\n        System.out.printf(\"Add back master2%n\");\n\n        awaitUntilSlaveOK();\n        // sleep a while to recover\n        Thread.sleep(30000);\n    }\n\n    @Test\n    public void testRemoteActing_timerMsg() throws Exception {\n        awaitUntilSlaveOK();\n\n        String topic = ScheduleSlaveActingMasterIT.class.getSimpleName() + random.nextInt(65535);\n        createTopic(topic);\n        AtomicInteger receivedMsgCount = new AtomicInteger(0);\n        AtomicInteger inTimeMsgCount = new AtomicInteger(0);\n        AtomicInteger master3MsgCount = new AtomicInteger(0);\n\n        MessageQueue messageQueue = new MessageQueue(topic, master1With3Replicas.getBrokerConfig().getBrokerName(), 0);\n        int sendSuccess = 0;\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            Message msg = new Message(topic, MESSAGE_BODY);\n            msg.setDelayTimeSec(30);\n            SendResult sendResult = producer.send(msg, messageQueue);\n            if (sendResult.getSendStatus() == SendStatus.SEND_OK) {\n                sendSuccess++;\n            }\n        }\n        final int finalSendSuccess = sendSuccess;\n        await().atMost(Duration.ofMinutes(1)).until(() -> finalSendSuccess >= MESSAGE_COUNT);\n        long sendCompleteTimeStamp = System.currentTimeMillis();\n        System.out.printf(\"send success%n\");\n\n        DefaultMQPushConsumer pushConsumer = createPushConsumer(CONSUME_GROUP);\n        pushConsumer.subscribe(topic, \"*\");\n        pushConsumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            long period = System.currentTimeMillis() - sendCompleteTimeStamp;\n            // Remote Acting lead to born timestamp, msgId changed, it need to polish.\n            if (Math.abs(period - 30000) <= 3000) {\n                inTimeMsgCount.addAndGet(msgs.size());\n            }\n            if (msgs.get(0).getBrokerName().equals(master3With3Replicas.getBrokerConfig().getBrokerName())) {\n                master3MsgCount.addAndGet(msgs.size());\n            }\n            receivedMsgCount.addAndGet(msgs.size());\n            msgs.forEach(x -> System.out.printf(\"cost \" + period + \" \" + x + \"%n\"));\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        pushConsumer.start();\n\n        isolateBroker(master1With3Replicas);\n        brokerContainer1.removeBroker(new BrokerIdentity(\n            master1With3Replicas.getBrokerConfig().getBrokerClusterName(),\n            master1With3Replicas.getBrokerConfig().getBrokerName(),\n            master1With3Replicas.getBrokerConfig().getBrokerId()));\n        System.out.printf(\"Remove master1%n\");\n\n        isolateBroker(master2With3Replicas);\n        brokerContainer2.removeBroker(new BrokerIdentity(\n            master2With3Replicas.getBrokerConfig().getBrokerClusterName(),\n            master2With3Replicas.getBrokerConfig().getBrokerName(),\n            master2With3Replicas.getBrokerConfig().getBrokerId()));\n        System.out.printf(\"Remove master2%n\");\n\n        await().atMost(Duration.ofMinutes(1)).until(() -> receivedMsgCount.get() >= MESSAGE_COUNT && master3MsgCount.get() >= MESSAGE_COUNT && inTimeMsgCount.get() >= MESSAGE_COUNT * 0.95);\n\n        System.out.printf(\"consumer received %d msg, %d in time%n\", receivedMsgCount.get(), inTimeMsgCount.get());\n\n        pushConsumer.shutdown();\n\n        //Add back master\n        master1With3Replicas = brokerContainer1.addBroker(buildConfigContext(master1With3Replicas.getBrokerConfig(), master1With3Replicas.getMessageStoreConfig()));\n        master1With3Replicas.start();\n        cancelIsolatedBroker(master1With3Replicas);\n        System.out.printf(\"Add back master1%n\");\n\n        //Add back master\n        master2With3Replicas = brokerContainer2.addBroker(buildConfigContext(master2With3Replicas.getBrokerConfig(), master2With3Replicas.getMessageStoreConfig()));\n        master2With3Replicas.start();\n        cancelIsolatedBroker(master2With3Replicas);\n        System.out.printf(\"Add back master2%n\");\n\n        awaitUntilSlaveOK();\n        // sleep a while to recover\n        Thread.sleep(20000);\n    }\n\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/container/ScheduledMessageIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.container;\n\nimport java.io.UnsupportedEncodingException;\nimport java.nio.charset.StandardCharsets;\nimport java.time.Duration;\nimport java.util.Random;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.commons.lang3.RandomStringUtils;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.junit.AfterClass;\nimport org.junit.BeforeClass;\nimport org.junit.Ignore;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Java6Assertions.assertThat;\nimport static org.awaitility.Awaitility.await;\n\n@Ignore\npublic class ScheduledMessageIT extends ContainerIntegrationTestBase {\n    private static DefaultMQProducer producer;\n\n    private static final String CONSUME_GROUP = ScheduledMessageIT.class.getSimpleName() + \"_Consumer\";\n    private static final String MESSAGE_STRING = RandomStringUtils.random(1024);\n    private static final byte[] MESSAGE_BODY = MESSAGE_STRING.getBytes(StandardCharsets.UTF_8);\n\n    private static final String TOPIC_PREFIX = ScheduledMessageIT.class.getSimpleName() + \"_TOPIC\";\n    private final Random random = new Random();\n    private static final int MESSAGE_COUNT = 128;\n\n    public ScheduledMessageIT() throws UnsupportedEncodingException {\n    }\n\n    void createTopic(String topic) {\n        createTopicTo(master1With3Replicas, topic, 1, 1);\n        createTopicTo(master2With3Replicas, topic, 1, 1);\n        createTopicTo(master3With3Replicas, topic, 1, 1);\n    }\n\n    @BeforeClass\n    public static void beforeClass() throws Throwable {\n        producer = createProducer(ScheduledMessageIT.class.getSimpleName() + \"_PRODUCER\");\n        producer.setSendMsgTimeout(5000);\n        producer.start();\n    }\n\n    @AfterClass\n    public static void afterClass() throws Exception {\n        producer.shutdown();\n    }\n\n    @Ignore\n    @Test\n    public void consumeScheduledMsg() throws MQClientException, RemotingException, InterruptedException, MQBrokerException {\n        String topic = TOPIC_PREFIX + random.nextInt(65535);\n        createTopic(topic);\n        DefaultMQPushConsumer pushConsumer = createPushConsumer(CONSUME_GROUP + random.nextInt(65535));\n        pushConsumer.subscribe(topic, \"*\");\n        AtomicInteger receivedMsgCount = new AtomicInteger(0);\n        AtomicInteger inTimeMsgCount = new AtomicInteger(0);\n        pushConsumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            long period = System.currentTimeMillis() - msgs.get(0).getBornTimestamp();\n            if (Math.abs(period - 5000) <= 1000) {\n                inTimeMsgCount.addAndGet(msgs.size());\n            }\n            receivedMsgCount.addAndGet(msgs.size());\n            msgs.forEach(x -> System.out.printf(receivedMsgCount.get() + \" cost \" + period + \" \" + x + \"%n\"));\n\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        pushConsumer.start();\n\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            Message msg = new Message(topic, MESSAGE_BODY);\n            msg.setDelayTimeLevel(2);\n            producer.send(msg);\n        }\n\n        await().atMost(Duration.ofSeconds(MESSAGE_COUNT * 2)).until(() -> receivedMsgCount.get() >= MESSAGE_COUNT && inTimeMsgCount.get() >= MESSAGE_COUNT * 0.95);\n\n        System.out.printf(\"consumer received %d msg, %d in time%n\", receivedMsgCount.get(), inTimeMsgCount.get());\n\n        pushConsumer.shutdown();\n    }\n\n    @Test\n    public void consumeScheduledMsgFromSlave() throws MQClientException, RemotingException, InterruptedException, MQBrokerException {\n        String topic = TOPIC_PREFIX + random.nextInt(65535);\n        createTopic(topic);\n        DefaultMQPushConsumer pushConsumer = createPushConsumer(CONSUME_GROUP + random.nextInt(65535));\n        pushConsumer.subscribe(topic, \"*\");\n        AtomicInteger receivedMsgCount = new AtomicInteger(0);\n        pushConsumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            receivedMsgCount.addAndGet(msgs.size());\n            msgs.forEach(x -> System.out.printf(x + \"%n\"));\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            Message msg = new Message(topic, String.valueOf(i).getBytes());\n            msg.setDelayTimeLevel(2);\n            producer.send(msg);\n        }\n\n        isolateBroker(master1With3Replicas);\n\n        producer.getDefaultMQProducerImpl().getmQClientFactory().updateTopicRouteInfoFromNameServer(topic);\n        assertThat(producer.getDefaultMQProducerImpl().getmQClientFactory().findBrokerAddressInPublish(topic)).isNull();\n\n        pushConsumer.start();\n\n        await().atMost(Duration.ofSeconds(MESSAGE_COUNT * 2)).until(() -> receivedMsgCount.get() >= MESSAGE_COUNT);\n\n        pushConsumer.shutdown();\n        cancelIsolatedBroker(master1With3Replicas);\n\n        await().atMost(100, TimeUnit.SECONDS)\n            .until(() -> ((DefaultMessageStore) master1With3Replicas.getMessageStore()).getHaService().getConnectionCount().get() == 2);\n    }\n\n    @Test\n    public void consumeTimerMsgFromSlave() throws MQClientException, RemotingException, InterruptedException, MQBrokerException {\n        String topic = TOPIC_PREFIX + random.nextInt(65535);\n        createTopic(topic);\n        DefaultMQPushConsumer pushConsumer = createPushConsumer(CONSUME_GROUP);\n        pushConsumer.subscribe(topic, \"*\");\n        AtomicInteger receivedMsgCount = new AtomicInteger(0);\n        pushConsumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            receivedMsgCount.addAndGet(msgs.size());\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            Message msg = new Message(topic, String.valueOf(i).getBytes());\n            msg.setDelayTimeSec(3);\n            producer.send(msg);\n        }\n\n        isolateBroker(master1With3Replicas);\n\n        producer.getDefaultMQProducerImpl().getmQClientFactory().updateTopicRouteInfoFromNameServer(topic);\n        assertThat(producer.getDefaultMQProducerImpl().getmQClientFactory().findBrokerAddressInPublish(topic)).isNull();\n\n        pushConsumer.start();\n\n        await().atMost(Duration.ofSeconds(MESSAGE_COUNT * 2)).until(() -> receivedMsgCount.get() >= MESSAGE_COUNT);\n\n        pushConsumer.shutdown();\n        cancelIsolatedBroker(master1With3Replicas);\n\n        await().atMost(100, TimeUnit.SECONDS)\n            .until(() -> ((DefaultMessageStore) master1With3Replicas.getMessageStore()).getHaService().getConnectionCount().get() == 2);\n    }\n\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/container/SendMultipleReplicasIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.container;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.List;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.junit.AfterClass;\nimport org.junit.BeforeClass;\nimport org.junit.Ignore;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Java6Assertions.assertThat;\nimport static org.awaitility.Awaitility.await;\n\n@Ignore\npublic class SendMultipleReplicasIT extends ContainerIntegrationTestBase {\n    private static DefaultMQProducer mqProducer;\n    private static final String MSG = \"Hello RocketMQ \";\n    private static final byte[] MESSAGE_BODY = MSG.getBytes(StandardCharsets.UTF_8);\n\n    public SendMultipleReplicasIT() {\n    }\n\n    @BeforeClass\n    public static void beforeClass() throws Exception {\n        mqProducer = createProducer(\"SendMultipleReplicasMessageIT_Producer\");\n        mqProducer.setSendMsgTimeout(15 * 1000);\n        mqProducer.start();\n    }\n\n    @AfterClass\n    public static void afterClass() {\n        if (mqProducer != null) {\n            mqProducer.shutdown();\n        }\n    }\n\n    @Test\n    public void sendMessageToBrokerGroup() throws MQClientException, RemotingException, InterruptedException, MQBrokerException {\n        awaitUntilSlaveOK();\n\n        // Send message to broker group with three replicas\n        Message msg = new Message(THREE_REPLICAS_TOPIC, MESSAGE_BODY);\n        SendResult sendResult = mqProducer.send(msg);\n        assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n    }\n\n    @Test\n    public void sendMessage_Auto_Replicas_Success() throws Exception {\n        await().atMost(100, TimeUnit.SECONDS)\n            .until(() -> ((DefaultMessageStore) master1With3Replicas.getMessageStore()).getHaService().getConnectionCount().get() == 2\n                && master1With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 3);\n        // Broker with 3 replicas configured as 3-2-1 auto replicas mode\n        Message msg = new Message(THREE_REPLICAS_TOPIC, MESSAGE_BODY);\n        SendResult sendResult = mqProducer.send(msg);\n        assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n\n        // Remove two slave broker\n        removeSlaveBroker(1, brokerContainer2, master1With3Replicas);\n        removeSlaveBroker(2, brokerContainer3, master1With3Replicas);\n        await().atMost(100, TimeUnit.SECONDS)\n            .until(() ->\n                ((DefaultMessageStore) master1With3Replicas.getMessageStore()).getHaService().getConnectionCount().get() == 0\n                    && master1With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 1);\n\n        master1With3Replicas.getMessageStoreConfig().setEnableAutoInSyncReplicas(true);\n        List<MessageQueue> mqList = mqProducer.getDefaultMQProducerImpl().fetchPublishMessageQueues(THREE_REPLICAS_TOPIC);\n        MessageQueue targetMq = null;\n        for (MessageQueue mq : mqList) {\n            if (mq.getBrokerName().equals(master1With3Replicas.getBrokerConfig().getBrokerName())) {\n                targetMq = mq;\n            }\n        }\n\n        assertThat(targetMq).isNotNull();\n        // Although this broker group only has one slave broker, send will be success in auto mode.\n        msg = new Message(THREE_REPLICAS_TOPIC, MESSAGE_BODY);\n        sendResult = mqProducer.send(msg, targetMq);\n        assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n\n        // Recover the cluster state\n        createAndAddSlave(1, brokerContainer2, master1With3Replicas);\n        createAndAddSlave(2, brokerContainer3, master1With3Replicas);\n        await().atMost(100, TimeUnit.SECONDS)\n            .until(() -> ((DefaultMessageStore) master1With3Replicas.getMessageStore()).getHaService().getConnectionCount().get() == 2\n                && master1With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 3);\n    }\n\n    @Test\n    public void sendMessage_Auto_Replicas_Failed()\n        throws Exception {\n        await().atMost(100, TimeUnit.SECONDS)\n            .until(() -> ((DefaultMessageStore) master1With3Replicas.getMessageStore()).getHaService().getConnectionCount().get() == 2\n                && master1With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 3);\n        // Broker with 3 replicas configured as 3-2-1 auto replicas mode\n        // Remove two slave broker\n        removeSlaveBroker(1, brokerContainer2, master1With3Replicas);\n        removeSlaveBroker(2, brokerContainer3, master1With3Replicas);\n        await().atMost(100, TimeUnit.SECONDS)\n            .until(() ->\n                ((DefaultMessageStore) master1With3Replicas.getMessageStore()).getHaService().getConnectionCount().get() == 0\n                    && master1With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 1);\n\n        // Disable the auto mode\n        master1With3Replicas.getMessageStoreConfig().setEnableAutoInSyncReplicas(false);\n\n        List<MessageQueue> mqList = mqProducer.getDefaultMQProducerImpl().fetchPublishMessageQueues(THREE_REPLICAS_TOPIC);\n        MessageQueue targetMq = null;\n        for (MessageQueue mq : mqList) {\n            if (mq.getBrokerName().equals(master1With3Replicas.getBrokerConfig().getBrokerName())) {\n                targetMq = mq;\n            }\n        }\n\n        assertThat(targetMq).isNotNull();\n\n        Message msg = new Message(THREE_REPLICAS_TOPIC, MESSAGE_BODY);\n        boolean exceptionCaught = false;\n        try {\n            mqProducer.send(msg, targetMq);\n        } catch (MQBrokerException e) {\n            exceptionCaught = true;\n        }\n\n        assertThat(exceptionCaught).isTrue();\n        // Recover the cluster state\n        createAndAddSlave(1, brokerContainer2, master1With3Replicas);\n        createAndAddSlave(2, brokerContainer3, master1With3Replicas);\n        await().atMost(100, TimeUnit.SECONDS)\n            .until(() -> ((DefaultMessageStore) master1With3Replicas.getMessageStore()).getHaService().getConnectionCount().get() == 2\n                && master1With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 3);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/container/SlaveBrokerIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.container;\n\nimport java.time.Duration;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.remoting.protocol.body.ClusterInfo;\nimport org.apache.rocketmq.store.DefaultMessageStore;\nimport org.junit.Ignore;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.awaitility.Awaitility.await;\n\n@Ignore\npublic class SlaveBrokerIT extends ContainerIntegrationTestBase {\n    @Test\n    public void reAddSlaveBroker() throws Exception {\n        await().atMost(Duration.ofMinutes(1)).until(() -> {\n            ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo();\n\n            if (clusterInfo.getClusterAddrTable().get(master1With3Replicas.getBrokerConfig().getBrokerClusterName()).size() != 3) {\n                return false;\n            }\n\n            if (clusterInfo.getBrokerAddrTable().get(master1With3Replicas.getBrokerConfig().getBrokerName()).getBrokerAddrs().size() != 3) {\n                return false;\n            }\n\n            if (clusterInfo.getBrokerAddrTable().get(master2With3Replicas.getBrokerConfig().getBrokerName()).getBrokerAddrs().size() != 3) {\n                return false;\n            }\n\n            if (clusterInfo.getBrokerAddrTable().get(master3With3Replicas.getBrokerConfig().getBrokerName()).getBrokerAddrs().size() != 3) {\n                return false;\n            }\n\n            return true;\n        });\n\n        // Remove one replicas from each broker group\n        removeSlaveBroker(1, brokerContainer1, master3With3Replicas);\n        removeSlaveBroker(1, brokerContainer2, master1With3Replicas);\n        removeSlaveBroker(1, brokerContainer3, master2With3Replicas);\n\n        await().atMost(Duration.ofMinutes(1)).until(() -> {\n            // Test cluster info again\n            ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo();\n            assertThat(clusterInfo.getBrokerAddrTable().get(master1With3Replicas.getBrokerConfig().getBrokerName()).getBrokerAddrs().size())\n                .isEqualTo(2);\n\n            assertThat(clusterInfo.getBrokerAddrTable().get(master2With3Replicas.getBrokerConfig().getBrokerName()).getBrokerAddrs().size())\n                .isEqualTo(2);\n\n            assertThat(clusterInfo.getBrokerAddrTable().get(master3With3Replicas.getBrokerConfig().getBrokerName()).getBrokerAddrs().size())\n                .isEqualTo(2);\n            return true;\n        });\n\n        // ReAdd the slave broker\n        createAndAddSlave(1, brokerContainer1, master3With3Replicas);\n        createAndAddSlave(1, brokerContainer2, master1With3Replicas);\n        createAndAddSlave(1, brokerContainer3, master2With3Replicas);\n\n        // Trigger a register action\n        //for (final SlaveBrokerController slaveBrokerController : brokerContainer1.getSlaveBrokers()) {\n        //    slaveBrokerController.registerBrokerAll(false, false, true);\n        //}\n        //\n        //for (final SlaveBrokerController slaveBrokerController : brokerContainer2.getSlaveBrokers()) {\n        //    slaveBrokerController.registerBrokerAll(false, false, true);\n        //}\n\n        await().atMost(Duration.ofMinutes(1)).until(() -> {\n            ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo();\n\n            return clusterInfo.getBrokerAddrTable()\n                .get(master1With3Replicas.getBrokerConfig().getBrokerName()).getBrokerAddrs().size() == 3\n                && clusterInfo.getBrokerAddrTable()\n                .get(master2With3Replicas.getBrokerConfig().getBrokerName()).getBrokerAddrs().size() == 3\n                && clusterInfo.getBrokerAddrTable()\n                .get(master2With3Replicas.getBrokerConfig().getBrokerName()).getBrokerAddrs().size() == 3;\n        });\n    }\n\n    @Test\n    public void reAddSlaveBroker_ConnectionCheck() throws Exception {\n        await().atMost(100, TimeUnit.SECONDS)\n            .until(() -> ((DefaultMessageStore) master3With3Replicas.getMessageStore()).getHaService().getConnectionCount().get() == 2);\n\n        removeSlaveBroker(1, brokerContainer1, master3With3Replicas);\n        createAndAddSlave(1, brokerContainer1, master3With3Replicas);\n\n        await().atMost(100, TimeUnit.SECONDS)\n            .until(() -> ((DefaultMessageStore) master3With3Replicas.getMessageStore()).getHaService().getConnectionCount().get() == 2);\n\n        await().atMost(100, TimeUnit.SECONDS)\n            .until(() -> ((DefaultMessageStore) master3With3Replicas.getMessageStore()).getHaService().inSyncReplicasNums(0) == 3);\n\n        Thread.sleep(1000 * 101);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/container/SyncConsumerOffsetIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.container;\n\nimport java.nio.charset.StandardCharsets;\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.CountDownLatch;\nimport java.util.concurrent.TimeUnit;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.container.BrokerContainer;\nimport org.apache.rocketmq.container.InnerSalveBrokerController;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.junit.AfterClass;\nimport org.junit.BeforeClass;\nimport org.junit.Ignore;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Java6Assertions.assertThat;\nimport static org.awaitility.Awaitility.await;\n\n@Ignore\npublic class SyncConsumerOffsetIT extends ContainerIntegrationTestBase {\n    private static final String THREE_REPLICA_CONSUMER_GROUP = \"SyncConsumerOffsetIT_ConsumerThreeReplica\";\n    private static final String TEST_SYNC_TOPIC = SyncConsumerOffsetIT.class.getSimpleName() + \"_topic\";\n\n    private static DefaultMQProducer mqProducer;\n    private static DefaultMQPushConsumer mqConsumerThreeReplica;\n    private static final String MSG = \"Hello RocketMQ \";\n    private static final byte[] MESSAGE_BODY = MSG.getBytes(StandardCharsets.UTF_8);\n\n    public SyncConsumerOffsetIT() {\n    }\n\n    @BeforeClass\n    public static void beforeClass() throws Exception {\n        createTopicTo(master3With3Replicas, TEST_SYNC_TOPIC);\n\n        mqProducer = createProducer(\"SyncConsumerOffsetIT_Producer\");\n        mqProducer.setSendMsgTimeout(15 * 1000);\n        mqProducer.start();\n\n        mqConsumerThreeReplica = createPushConsumer(THREE_REPLICA_CONSUMER_GROUP);\n        mqConsumerThreeReplica.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);\n        mqConsumerThreeReplica.subscribe(TEST_SYNC_TOPIC, \"*\");\n    }\n\n    @AfterClass\n    public static void afterClass() {\n        mqProducer.shutdown();\n        mqConsumerThreeReplica.shutdown();\n    }\n\n    @Test\n    public void syncConsumerOffsetWith3Replicas() throws InterruptedException, RemotingException, MQClientException, MQBrokerException {\n        syncConsumeOffsetInner(TEST_SYNC_TOPIC, mqConsumerThreeReplica,\n            master3With3Replicas, Arrays.asList(brokerContainer1, brokerContainer2));\n    }\n\n    private void syncConsumeOffsetInner(String topic, DefaultMQPushConsumer consumer, BrokerController master,\n        List<BrokerContainer> slaveContainers) throws InterruptedException, RemotingException, MQClientException, MQBrokerException {\n        awaitUntilSlaveOK();\n        String group = THREE_REPLICA_CONSUMER_GROUP;\n\n        int msgCount = 100;\n        for (int i = 0; i < msgCount; i++) {\n            Message msg = new Message(topic, MESSAGE_BODY);\n            SendResult sendResult = mqProducer.send(msg);\n            assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);\n        }\n\n        CountDownLatch countDownLatch = new CountDownLatch(msgCount);\n        consumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            countDownLatch.countDown();\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        consumer.start();\n        boolean ok = countDownLatch.await(100, TimeUnit.SECONDS);\n        assertThat(ok).isEqualTo(true);\n        System.out.printf(\"consume complete%n\");\n\n        final Set<MessageQueue> mqSet = filterMessageQueue(consumer.fetchSubscribeMessageQueues(topic), topic);\n\n        await().atMost(120, TimeUnit.SECONDS).until(() -> {\n            Map<Integer, Long> consumerOffsetMap = new HashMap<>();\n            long offsetTotal = 0L;\n            for (MessageQueue mq : mqSet) {\n                long queueOffset = master.getConsumerOffsetManager().queryOffset(group, topic, mq.getQueueId());\n                if (queueOffset < 0) {\n                    continue;\n                }\n                offsetTotal += queueOffset;\n                consumerOffsetMap.put(mq.getQueueId(), queueOffset);\n            }\n\n            if (offsetTotal < 100) {\n                return false;\n            }\n            boolean syncOk = true;\n\n            for (BrokerContainer brokerContainer : slaveContainers) {\n                for (InnerSalveBrokerController slave : brokerContainer.getSlaveBrokers()) {\n                    if (!slave.getBrokerConfig().getBrokerName().equals(master.getBrokerConfig().getBrokerName())) {\n                        continue;\n                    }\n                    for (MessageQueue mq : mqSet) {\n                        long slaveOffset = slave.getConsumerOffsetManager().queryOffset(group, topic, mq.getQueueId());\n                        boolean check = slaveOffset == consumerOffsetMap.get(mq.getQueueId());\n                        syncOk &= check;\n                    }\n                }\n            }\n\n            return syncOk;\n        });\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/container/TransactionListenerImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.container;\n\nimport org.apache.rocketmq.client.producer.LocalTransactionState;\nimport org.apache.rocketmq.client.producer.TransactionListener;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageExt;\n\npublic class TransactionListenerImpl implements TransactionListener {\n    private boolean shouldReturnUnknownState = false;\n\n\n\n    public TransactionListenerImpl(boolean shouldReturnUnknownState) {\n        this.shouldReturnUnknownState = shouldReturnUnknownState;\n    }\n\n    public void setShouldReturnUnknownState(boolean shouldReturnUnknownState) {\n        this.shouldReturnUnknownState = shouldReturnUnknownState;\n    }\n\n    @Override\n    public LocalTransactionState executeLocalTransaction(Message msg, Object arg) {\n        if (shouldReturnUnknownState) {\n            return LocalTransactionState.UNKNOW;\n        } else {\n            return LocalTransactionState.COMMIT_MESSAGE;\n        }\n    }\n\n    @Override\n    public LocalTransactionState checkLocalTransaction(MessageExt msg) {\n        if (shouldReturnUnknownState) {\n            return LocalTransactionState.UNKNOW;\n        } else {\n            return LocalTransactionState.COMMIT_MESSAGE;\n        }\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/container/TransactionMessageIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.container;\n\nimport java.io.UnsupportedEncodingException;\nimport java.time.Duration;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.UUID;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.commons.lang3.RandomStringUtils;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.LocalTransactionState;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.TransactionMQProducer;\nimport org.apache.rocketmq.client.producer.TransactionSendResult;\nimport org.apache.rocketmq.common.BrokerIdentity;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.junit.Ignore;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.awaitility.Awaitility.await;\n\n@Ignore\npublic class TransactionMessageIT extends ContainerIntegrationTestBase {\n\n    private static final String MESSAGE_STRING = RandomStringUtils.random(1024);\n    private static byte[] messageBody;\n\n    static {\n        try {\n            messageBody = MESSAGE_STRING.getBytes(RemotingHelper.DEFAULT_CHARSET);\n        } catch (UnsupportedEncodingException ignored) {\n        }\n    }\n\n    private static final int MESSAGE_COUNT = 16;\n\n    public TransactionMessageIT() {\n    }\n\n    private static String generateGroup() {\n        return \"GID-\" + TransactionMessageIT.class.getSimpleName() + RandomStringUtils.randomNumeric(5);\n    }\n\n    @Test\n    public void consumeTransactionMsg() throws MQClientException {\n        final String topic = generateTopic();\n        createTopicTo(master1With3Replicas, topic, 1, 1);\n\n        final String group = generateGroup();\n        DefaultMQPushConsumer pushConsumer = createPushConsumer(group);\n        pushConsumer.subscribe(topic, \"*\");\n        AtomicInteger receivedMsgCount = new AtomicInteger(0);\n        pushConsumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            receivedMsgCount.addAndGet(msgs.size());\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        pushConsumer.start();\n\n        TransactionMQProducer producer = createTransactionProducer(group, new TransactionListenerImpl(false));\n        producer.start();\n\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            Message msg = new Message(topic, messageBody);\n            TransactionSendResult result = producer.sendMessageInTransaction(msg, null);\n            assertThat(result.getLocalTransactionState()).isEqualTo(LocalTransactionState.COMMIT_MESSAGE);\n        }\n\n        System.out.printf(\"send message complete%n\");\n\n        await().atMost(Duration.ofSeconds(MESSAGE_COUNT * 2)).until(() -> receivedMsgCount.get() >= MESSAGE_COUNT);\n\n        System.out.printf(\"consumer received %d msg%n\", receivedMsgCount.get());\n\n        pushConsumer.shutdown();\n        producer.shutdown();\n    }\n\n    private static String generateTopic() {\n        return TransactionMessageIT.class.getSimpleName() + RandomStringUtils.randomNumeric(5);\n    }\n\n    @Test\n    public void consumeTransactionMsgLocalEscape() throws Exception {\n        final String topic = generateTopic();\n        createTopicTo(master1With3Replicas, topic, 1, 1);\n\n        final String group = generateGroup();\n        DefaultMQPushConsumer pushConsumer = createPushConsumer(group);\n        pushConsumer.subscribe(topic, \"*\");\n        AtomicInteger receivedMsgCount = new AtomicInteger(0);\n        Map<String, Message> msgSentMap = new HashMap<>();\n        pushConsumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            for (MessageExt msg : msgs) {\n                if (msgSentMap.containsKey(msg.getMsgId())) {\n                    receivedMsgCount.incrementAndGet();\n                }\n            }\n\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        pushConsumer.start();\n\n        TransactionListenerImpl transactionCheckListener = new TransactionListenerImpl(true);\n        TransactionMQProducer producer = createTransactionProducer(group, transactionCheckListener);\n        producer.start();\n\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            Message msg = new Message(topic, messageBody);\n            msg.setKeys(UUID.randomUUID().toString());\n            SendResult result = producer.sendMessageInTransaction(msg, null);\n            String msgId = result.getMsgId();\n\n            msgSentMap.put(msgId, msg);\n        }\n\n        isolateBroker(master1With3Replicas);\n        brokerContainer1.removeBroker(new BrokerIdentity(master1With3Replicas.getBrokerIdentity().getBrokerClusterName(),\n            master1With3Replicas.getBrokerIdentity().getBrokerName(),\n            master1With3Replicas.getBrokerIdentity().getBrokerId()));\n        System.out.printf(\"=========\" + master1With3Replicas.getBrokerIdentity().getBrokerName() + \"-\"\n            + master1With3Replicas.getBrokerIdentity().getBrokerId() + \" removed%n\");\n        createTopicTo(master2With3Replicas, topic, 1, 1);\n\n        transactionCheckListener.setShouldReturnUnknownState(false);\n        producer.getDefaultMQProducerImpl().getmQClientFactory().updateTopicRouteInfoFromNameServer(topic);\n\n        System.out.printf(\"Wait for consuming%n\");\n\n        await().atMost(Duration.ofSeconds(300)).until(() -> receivedMsgCount.get() >= MESSAGE_COUNT);\n\n        System.out.printf(\"consumer received %d msg%n\", receivedMsgCount.get());\n\n        pushConsumer.shutdown();\n        producer.shutdown();\n\n        master1With3Replicas = brokerContainer1.addBroker(buildConfigContext(master1With3Replicas.getBrokerConfig(), master1With3Replicas.getMessageStoreConfig()));\n        master1With3Replicas.start();\n        cancelIsolatedBroker(master1With3Replicas);\n        awaitUntilSlaveOK();\n\n        receivedMsgCount.set(0);\n        DefaultMQPushConsumer pushConsumer2 = createPushConsumer(group);\n        pushConsumer2.subscribe(topic, \"*\");\n        pushConsumer2.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            for (MessageExt msg : msgs) {\n                if (msgSentMap.containsKey(msg.getMsgId())) {\n                    receivedMsgCount.incrementAndGet();\n                }\n            }\n\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        pushConsumer2.start();\n        System.out.printf(\"Wait for checking...%n\");\n        Thread.sleep(10000L);\n\n    }\n\n    @Test\n    public void consumeTransactionMsgRemoteEscape() throws Exception {\n        final String topic = generateTopic();\n        createTopicTo(master1With3Replicas, topic, 1, 1);\n\n        final String group = generateGroup();\n\n        AtomicInteger receivedMsgCount = new AtomicInteger(0);\n        Map<String, Message> msgSentMap = new HashMap<>();\n        DefaultMQPushConsumer pushConsumer = createPushConsumer(group);\n        pushConsumer.subscribe(topic, \"*\");\n        pushConsumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            for (MessageExt msg : msgs) {\n                if (msgSentMap.containsKey(msg.getMsgId())) {\n                    receivedMsgCount.incrementAndGet();\n                }\n            }\n\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        pushConsumer.start();\n\n        TransactionListenerImpl transactionCheckListener = new TransactionListenerImpl(true);\n        TransactionMQProducer producer = createTransactionProducer(group, transactionCheckListener);\n        producer.start();\n\n        for (int i = 0; i < MESSAGE_COUNT; i++) {\n            Message msg = new Message(topic, messageBody);\n            msg.setKeys(UUID.randomUUID().toString());\n            SendResult result = producer.sendMessageInTransaction(msg, null);\n            String msgId = result.getMsgId();\n\n            msgSentMap.put(msgId, msg);\n        }\n\n        isolateBroker(master1With3Replicas);\n        brokerContainer1.removeBroker(new BrokerIdentity(master1With3Replicas.getBrokerIdentity().getBrokerClusterName(),\n            master1With3Replicas.getBrokerIdentity().getBrokerName(),\n            master1With3Replicas.getBrokerIdentity().getBrokerId()));\n        System.out.printf(\"=========\" + master1With3Replicas.getBrokerIdentity().getBrokerName() + \"-\"\n            + master1With3Replicas.getBrokerIdentity().getBrokerId() + \" removed%n\");\n\n        createTopicTo(master2With3Replicas, topic, 1, 1);\n        createTopicTo(master3With3Replicas, topic, 1, 1);\n        //isolateBroker(master2With3Replicas);\n        brokerContainer2.removeBroker(new BrokerIdentity(master2With3Replicas.getBrokerIdentity().getBrokerClusterName(),\n            master2With3Replicas.getBrokerIdentity().getBrokerName(),\n            master2With3Replicas.getBrokerIdentity().getBrokerId()));\n        System.out.printf(\"=========\" + master2With3Replicas.getBrokerIdentity().getBrokerClusterName() + \"-\"\n            + master2With3Replicas.getBrokerIdentity().getBrokerName()\n            + \"-\" + master2With3Replicas.getBrokerIdentity().getBrokerId() + \" removed%n\");\n\n        pushConsumer.getDefaultMQPushConsumerImpl().getRebalanceImpl().doRebalance(false);\n        transactionCheckListener.setShouldReturnUnknownState(false);\n        producer.getDefaultMQProducerImpl().getmQClientFactory().updateTopicRouteInfoFromNameServer(topic);\n\n        System.out.printf(\"Wait for consuming%n\");\n\n        await().atMost(Duration.ofSeconds(180)).until(() -> receivedMsgCount.get() >= MESSAGE_COUNT);\n\n        System.out.printf(\"consumer received %d msg%n\", receivedMsgCount.get());\n\n        pushConsumer.shutdown();\n        producer.shutdown();\n\n        master1With3Replicas = brokerContainer1.addBroker(buildConfigContext(master1With3Replicas.getBrokerConfig(), master1With3Replicas.getMessageStoreConfig()));\n        master1With3Replicas.start();\n        cancelIsolatedBroker(master1With3Replicas);\n\n        master2With3Replicas = brokerContainer2.addBroker(buildConfigContext(master2With3Replicas.getBrokerConfig(),\n            master2With3Replicas.getMessageStoreConfig()));\n        master2With3Replicas.start();\n        cancelIsolatedBroker(master2With3Replicas);\n\n        awaitUntilSlaveOK();\n\n        receivedMsgCount.set(0);\n        DefaultMQPushConsumer pushConsumer2 = createPushConsumer(group);\n        pushConsumer2.subscribe(topic, \"*\");\n        pushConsumer2.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            for (MessageExt msg : msgs) {\n                if (msgSentMap.containsKey(msg.getMsgId())) {\n                    receivedMsgCount.incrementAndGet();\n                }\n            }\n\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        pushConsumer2.start();\n        System.out.printf(\"Wait for checking...%n\");\n        Thread.sleep(10000L);\n        assertThat(receivedMsgCount.get()).isEqualTo(0);\n        pushConsumer2.shutdown();\n\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/delay/DelayConf.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.delay;\n\nimport org.apache.rocketmq.test.base.BaseConf;\n\npublic class DelayConf extends BaseConf {\n    protected static final int[] DELAY_LEVEL = {\n        1, 5, 10, 30, 1 * 60, 5 * 60, 10 * 60,\n        30 * 60, 1 * 3600, 2 * 3600, 6 * 3600, 12 * 3600, 1 * 24 * 3600};\n\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/delay/NormalMsgDelayIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.delay;\n\nimport java.util.List;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.factory.MQMessageFactory;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQDelayListener;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\npublic class NormalMsgDelayIT extends DelayConf {\n    private static Logger logger = LoggerFactory.getLogger(NormalMsgDelayIT.class);\n    protected int msgSize = 100;\n    private RMQNormalProducer producer = null;\n    private RMQNormalConsumer consumer = null;\n    private String topic = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        logger.info(String.format(\"use topic: %s;\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n        consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQDelayListener());\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testDelayLevel1() throws Exception {\n        Thread.sleep(3000);\n        int delayLevel = 1;\n        List<Object> delayMsgs = MQMessageFactory.getDelayMsg(topic, delayLevel, msgSize);\n        producer.send(delayMsgs);\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize, producer.getAllUndupMsgBody().size());\n\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        Assert.assertEquals(\"Not all are consumed\", 0, VerifyUtils.verify(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()));\n        Assert.assertEquals(\"Timer is not correct\", true,\n            VerifyUtils.verifyDelay(DELAY_LEVEL[delayLevel - 1] * 1000, DELAY_LEVEL[delayLevel] * 1000,\n                ((RMQDelayListener) consumer.getListener()).getMsgDelayTimes()));\n    }\n\n    @Test\n    public void testDelayLevel2() {\n        int delayLevel = 2;\n        List<Object> delayMsgs = MQMessageFactory.getDelayMsg(topic, delayLevel, msgSize);\n        producer.send(delayMsgs);\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize, producer.getAllUndupMsgBody().size());\n\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(),\n            DELAY_LEVEL[delayLevel - 1] * 1000 * 2);\n        Assert.assertEquals(\"Not all are consumed\", 0, VerifyUtils.verify(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()));\n        Assert.assertEquals(\"Timer is not correct\", true,\n            VerifyUtils.verifyDelay(DELAY_LEVEL[delayLevel - 1] * 1000, DELAY_LEVEL[delayLevel] * 1000,\n                ((RMQDelayListener) consumer.getListener()).getMsgDelayTimes()));\n    }\n\n    @Test\n    public void testDelayLevel3() {\n        int delayLevel = 3;\n        List<Object> delayMsgs = MQMessageFactory.getDelayMsg(topic, delayLevel, msgSize);\n        producer.send(delayMsgs);\n        Assert.assertEquals(\"Not all are sent\", msgSize, producer.getAllUndupMsgBody().size());\n\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(),\n            DELAY_LEVEL[delayLevel - 1] * 1000 * 2);\n        Assert.assertEquals(\"Not all are consumed\", 0, VerifyUtils.verify(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()));\n        Assert.assertEquals(\"Timer is not correct\", true,\n            VerifyUtils.verifyDelay(DELAY_LEVEL[delayLevel - 1] * 1000, DELAY_LEVEL[delayLevel] * 1000,\n                ((RMQDelayListener) consumer.getListener()).getMsgDelayTimes()));\n    }\n\n    @Test\n    public void testDelayLevel4() {\n        int delayLevel = 4;\n        List<Object> delayMsgs = MQMessageFactory.getDelayMsg(topic, delayLevel, msgSize);\n        producer.send(delayMsgs);\n        Assert.assertEquals(\"Not all are sent\", msgSize, producer.getAllUndupMsgBody().size());\n\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(),\n            DELAY_LEVEL[delayLevel - 1] * 1000 * 2);\n        Assert.assertEquals(\"Not all are consumed\", 0, VerifyUtils.verify(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()));\n        Assert.assertEquals(\"Timer is not correct\", true,\n            VerifyUtils.verifyDelay(DELAY_LEVEL[delayLevel - 1] * 1000, DELAY_LEVEL[delayLevel] * 1000,\n                ((RMQDelayListener) consumer.getListener()).getMsgDelayTimes()));\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/dledger/DLedgerProduceAndConsumeIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\npackage org.apache.rocketmq.test.dledger;\n\nimport java.util.UUID;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.store.config.BrokerRole;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.base.IntegrationTestBase;\nimport org.apache.rocketmq.test.factory.ConsumerFactory;\nimport org.apache.rocketmq.test.factory.ProducerFactory;\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class DLedgerProduceAndConsumeIT {\n\n    public BrokerConfig buildBrokerConfig(String cluster, String brokerName) {\n        BrokerConfig brokerConfig =  new BrokerConfig();\n        brokerConfig.setBrokerClusterName(cluster);\n        brokerConfig.setBrokerName(brokerName);\n        brokerConfig.setBrokerIP1(\"127.0.0.1\");\n        brokerConfig.setNamesrvAddr(BaseConf.NAMESRV_ADDR);\n        return brokerConfig;\n    }\n\n    public MessageStoreConfig buildStoreConfig(String brokerName, String peers, String selfId) {\n        MessageStoreConfig storeConfig = new MessageStoreConfig();\n        String baseDir =  IntegrationTestBase.createBaseDir();\n        storeConfig.setStorePathRootDir(baseDir);\n        storeConfig.setStorePathCommitLog(baseDir + \"_\" + \"commitlog\");\n        storeConfig.setHaListenPort(0);\n        storeConfig.setMappedFileSizeCommitLog(10 * 1024 * 1024);\n        storeConfig.setEnableDLegerCommitLog(true);\n        storeConfig.setdLegerGroup(brokerName);\n        storeConfig.setdLegerSelfId(selfId);\n        storeConfig.setdLegerPeers(peers);\n        return storeConfig;\n    }\n\n    @Test\n    public void testProduceAndConsume() throws Exception {\n        String cluster = UUID.randomUUID().toString();\n        String brokerName = UUID.randomUUID().toString();\n        String selfId = \"n0\";\n        // TODO: We need to acquire the actual listening port after the peer has started.\n        String peers = String.format(\"n0-localhost:%d\", 0);\n        BrokerConfig brokerConfig = buildBrokerConfig(cluster, brokerName);\n        MessageStoreConfig storeConfig = buildStoreConfig(brokerName, peers, selfId);\n        BrokerController brokerController = IntegrationTestBase.createAndStartBroker(storeConfig, brokerConfig);\n        BaseConf.waitBrokerRegistered(BaseConf.NAMESRV_ADDR, brokerConfig.getBrokerName(), 1);\n\n        Assert.assertEquals(BrokerRole.SYNC_MASTER, storeConfig.getBrokerRole());\n\n\n        String topic = UUID.randomUUID().toString();\n        String consumerGroup = UUID.randomUUID().toString();\n        IntegrationTestBase.initTopic(topic, BaseConf.NAMESRV_ADDR, cluster, 1, CQType.SimpleCQ);\n        DefaultMQProducer producer = ProducerFactory.getRMQProducer(BaseConf.NAMESRV_ADDR);\n        DefaultMQPullConsumer consumer = ConsumerFactory.getRMQPullConsumer(BaseConf.NAMESRV_ADDR, consumerGroup);\n\n        for (int i = 0; i < 10; i++) {\n            Message message = new Message();\n            message.setTopic(topic);\n            message.setBody((\"Hello\" + i).getBytes());\n            SendResult sendResult = producer.send(message);\n            Assert.assertEquals(SendStatus.SEND_OK, sendResult.getSendStatus());\n            Assert.assertEquals(0, sendResult.getMessageQueue().getQueueId());\n            Assert.assertEquals(brokerName, sendResult.getMessageQueue().getBrokerName());\n            Assert.assertEquals(i, sendResult.getQueueOffset());\n            Assert.assertNotNull(sendResult.getMsgId());\n            Assert.assertNotNull(sendResult.getOffsetMsgId());\n        }\n\n        Thread.sleep(500);\n        Assert.assertEquals(0, brokerController.getMessageStore().getMinOffsetInQueue(topic, 0));\n        Assert.assertEquals(10, brokerController.getMessageStore().getMaxOffsetInQueue(topic, 0));\n\n        MessageQueue messageQueue = new MessageQueue(topic, brokerName, 0);\n        PullResult pullResult = consumer.pull(messageQueue, \"*\", 0, 32);\n        Assert.assertEquals(PullStatus.FOUND, pullResult.getPullStatus());\n        Assert.assertEquals(10, pullResult.getMsgFoundList().size());\n\n        for (int i = 0; i < 10; i++) {\n            MessageExt messageExt = pullResult.getMsgFoundList().get(i);\n            Assert.assertEquals(i, messageExt.getQueueOffset());\n            Assert.assertArrayEquals((\"Hello\" + i).getBytes(), messageExt.getBody());\n        }\n\n        producer.shutdown();\n        consumer.shutdown();\n        brokerController.shutdown();\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/grpc/v2/ClusterGrpcIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.grpc.v2;\n\nimport apache.rocketmq.v2.QueryRouteResponse;\nimport java.time.Duration;\nimport java.util.Map;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.grpc.v2.GrpcMessagingApplication;\nimport org.apache.rocketmq.proxy.processor.DefaultMessagingProcessor;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.test.util.MQAdminTestUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.FixMethodOrder;\nimport org.junit.Test;\nimport org.junit.Ignore;\nimport org.junit.runners.MethodSorters;\n\nimport static org.awaitility.Awaitility.await;\n\n@FixMethodOrder(value = MethodSorters.NAME_ASCENDING)\npublic class ClusterGrpcIT extends GrpcBaseIT {\n\n    private MessagingProcessor messagingProcessor;\n    private GrpcMessagingApplication grpcMessagingApplication;\n\n    @Before\n    public void setUp() throws Exception {\n        super.setUp();\n        ConfigurationManager.getProxyConfig().setTransactionHeartbeatPeriodSecond(3);\n        messagingProcessor = DefaultMessagingProcessor.createForClusterMode();\n        messagingProcessor.start();\n        grpcMessagingApplication = GrpcMessagingApplication.create(messagingProcessor);\n        grpcMessagingApplication.start();\n        setUpServer(grpcMessagingApplication, 0, true);\n\n        await().atMost(Duration.ofSeconds(40)).until(() -> {\n            Map<String, BrokerData> brokerDataMap = MQAdminTestUtils.getCluster(NAMESRV_ADDR).getBrokerAddrTable();\n            return brokerDataMap.size() == BROKER_NUM;\n        });\n    }\n\n    @After\n    public void tearDown() throws Exception {\n        messagingProcessor.shutdown();\n        grpcMessagingApplication.shutdown();\n        shutdown();\n    }\n\n    @Test\n    public void testQueryRoute() throws Exception {\n        String topic = initTopic();\n\n        QueryRouteResponse response = blockingStub.queryRoute(buildQueryRouteRequest(topic));\n        assertQueryRoute(response, BROKER_NUM * DEFAULT_QUEUE_NUMS);\n    }\n\n    @Test\n    public void testQueryAssignment() throws Exception {\n        super.testQueryAssignment();\n    }\n\n    @Test\n    public void testQueryFifoAssignment() throws Exception {\n        super.testQueryFifoAssignment();\n    }\n\n    @Test\n    @Ignore\n    public void testTransactionCheckThenCommit() {\n        super.testTransactionCheckThenCommit();\n    }\n\n    @Test\n    @Ignore\n    public void testSimpleConsumerSendAndRecvDelayMessage() throws Exception {\n        super.testSimpleConsumerSendAndRecvDelayMessage();\n    }\n\n    @Test\n    public void testSimpleConsumerSendAndRecallDelayMessage() throws Exception {\n        super.testSimpleConsumerSendAndRecallDelayMessage();\n    }\n\n    @Test\n    public void testSimpleConsumerSendAndRecvBigMessage() throws Exception {\n        super.testSimpleConsumerSendAndRecvBigMessage();\n    }\n\n    @Test\n    public void testSimpleConsumerSendAndRecv() throws Exception {\n        super.testSimpleConsumerSendAndRecv();\n    }\n\n    @Test\n    public void testSimpleConsumerToDLQ() throws Exception {\n        super.testSimpleConsumerToDLQ();\n    }\n\n    @Test\n    public void testConsumeOrderly() throws Exception {\n        super.testConsumeOrderly();\n    }\n\n    @Test\n    public void testSimpleConsumerSendAndRecvPriorityMessage() throws Exception {\n        super.testSimpleConsumerSendAndRecvPriorityMessage();\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/grpc/v2/GrpcBaseIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.grpc.v2;\n\nimport apache.rocketmq.v2.AckMessageEntry;\nimport apache.rocketmq.v2.AckMessageRequest;\nimport apache.rocketmq.v2.AckMessageResponse;\nimport apache.rocketmq.v2.AckMessageResultEntry;\nimport apache.rocketmq.v2.Address;\nimport apache.rocketmq.v2.AddressScheme;\nimport apache.rocketmq.v2.ChangeInvisibleDurationRequest;\nimport apache.rocketmq.v2.ChangeInvisibleDurationResponse;\nimport apache.rocketmq.v2.ClientType;\nimport apache.rocketmq.v2.Code;\nimport apache.rocketmq.v2.Encoding;\nimport apache.rocketmq.v2.EndTransactionRequest;\nimport apache.rocketmq.v2.EndTransactionResponse;\nimport apache.rocketmq.v2.Endpoints;\nimport apache.rocketmq.v2.HeartbeatRequest;\nimport apache.rocketmq.v2.Message;\nimport apache.rocketmq.v2.MessageQueue;\nimport apache.rocketmq.v2.MessageType;\nimport apache.rocketmq.v2.MessagingServiceGrpc;\nimport apache.rocketmq.v2.Publishing;\nimport apache.rocketmq.v2.QueryAssignmentRequest;\nimport apache.rocketmq.v2.QueryAssignmentResponse;\nimport apache.rocketmq.v2.QueryRouteRequest;\nimport apache.rocketmq.v2.QueryRouteResponse;\nimport apache.rocketmq.v2.RecallMessageRequest;\nimport apache.rocketmq.v2.RecallMessageResponse;\nimport apache.rocketmq.v2.ReceiveMessageRequest;\nimport apache.rocketmq.v2.ReceiveMessageResponse;\nimport apache.rocketmq.v2.RecoverOrphanedTransactionCommand;\nimport apache.rocketmq.v2.Resource;\nimport apache.rocketmq.v2.RetryPolicy;\nimport apache.rocketmq.v2.SendMessageRequest;\nimport apache.rocketmq.v2.SendMessageResponse;\nimport apache.rocketmq.v2.Settings;\nimport apache.rocketmq.v2.Subscription;\nimport apache.rocketmq.v2.SystemProperties;\nimport apache.rocketmq.v2.TelemetryCommand;\nimport apache.rocketmq.v2.TransactionResolution;\nimport apache.rocketmq.v2.TransactionSource;\nimport com.google.protobuf.ByteString;\nimport com.google.protobuf.Duration;\nimport com.google.protobuf.util.Durations;\nimport com.google.protobuf.util.Timestamps;\nimport io.grpc.Channel;\nimport io.grpc.Metadata;\nimport io.grpc.Server;\nimport io.grpc.ServerInterceptors;\nimport io.grpc.ServerServiceDefinition;\nimport io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder;\nimport io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder;\nimport io.grpc.netty.shaded.io.netty.handler.ssl.ApplicationProtocolConfig;\nimport io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder;\nimport io.grpc.netty.shaded.io.netty.handler.ssl.SslProvider;\nimport io.grpc.stub.MetadataUtils;\nimport io.grpc.stub.StreamObserver;\nimport io.grpc.testing.GrpcCleanupRule;\nimport io.netty.handler.ssl.ApplicationProtocolNames;\nimport io.netty.handler.ssl.util.InsecureTrustManagerFactory;\nimport io.netty.handler.ssl.util.SelfSignedCertificate;\nimport java.io.IOException;\nimport java.net.URL;\nimport java.security.cert.CertificateException;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.UUID;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.concurrent.atomic.AtomicReference;\nimport java.util.stream.Collectors;\nimport javax.net.ssl.SSLException;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;\nimport org.apache.rocketmq.client.consumer.PullResult;\nimport org.apache.rocketmq.client.consumer.PullStatus;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.utils.NetworkUtil;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.grpc.interceptor.ContextInterceptor;\nimport org.apache.rocketmq.proxy.grpc.interceptor.HeaderInterceptor;\nimport org.apache.rocketmq.common.constant.GrpcConstants;\nimport org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder;\nimport org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.util.MQRandomUtils;\nimport org.apache.rocketmq.test.util.RandomUtils;\nimport org.junit.Rule;\n\nimport static org.apache.rocketmq.common.message.MessageClientIDSetter.createUniqID;\nimport static org.apache.rocketmq.proxy.config.ConfigurationManager.RMQ_PROXY_HOME;\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.awaitility.Awaitility.await;\n\npublic class GrpcBaseIT extends BaseConf {\n\n    /**\n     * Let OS pick up an available port.\n     */\n    private int port = 0;\n\n    /**\n     * This rule manages automatic graceful shutdown for the registered servers and channels at the end of test.\n     */\n    @Rule\n    public final GrpcCleanupRule grpcCleanup = new GrpcCleanupRule();\n\n    protected MessagingServiceGrpc.MessagingServiceBlockingStub blockingStub;\n    protected MessagingServiceGrpc.MessagingServiceStub stub;\n    protected final Metadata header = new Metadata();\n\n    protected static final int DEFAULT_QUEUE_NUMS = 8;\n\n    public void setUp() throws Exception {\n        brokerController1.getBrokerConfig().setTransactionCheckInterval(1 * 1000);\n        brokerController2.getBrokerConfig().setTransactionCheckInterval(1 * 1000);\n        brokerController3.getBrokerConfig().setTransactionCheckInterval(1 * 1000);\n\n        header.put(GrpcConstants.CLIENT_ID, \"client-id\" + UUID.randomUUID());\n        header.put(GrpcConstants.LANGUAGE, \"JAVA\");\n\n        String mockProxyHome = \"/mock/rmq/proxy/home\";\n        URL mockProxyHomeURL = getClass().getClassLoader().getResource(\"rmq-proxy-home\");\n        if (mockProxyHomeURL != null) {\n            mockProxyHome = mockProxyHomeURL.toURI().getPath();\n        }\n\n        if (null != mockProxyHome) {\n            System.setProperty(RMQ_PROXY_HOME, mockProxyHome);\n        }\n\n        ConfigurationManager.initEnv();\n        ConfigurationManager.initConfig();\n        ConfigurationManager.getProxyConfig().setNamesrvAddr(NAMESRV_ADDR);\n        // Set LongPollingReserveTimeInMillis to 500ms to reserve more time for IT\n        ConfigurationManager.getProxyConfig().setLongPollingReserveTimeInMillis(500);\n        ConfigurationManager.getProxyConfig().setRocketMQClusterName(brokerController1.getBrokerConfig().getBrokerClusterName());\n        ConfigurationManager.getProxyConfig().setHeartbeatSyncerTopicClusterName(brokerController1.getBrokerConfig().getBrokerClusterName());\n        ConfigurationManager.getProxyConfig().setMinInvisibleTimeMillsForRecv(3);\n        ConfigurationManager.getProxyConfig().setGrpcClientConsumerMinLongPollingTimeoutMillis(0);\n    }\n\n    protected MessagingServiceGrpc.MessagingServiceStub createStub(Channel channel) {\n        MessagingServiceGrpc.MessagingServiceStub stub = MessagingServiceGrpc.newStub(channel);\n        return stub.withInterceptors(MetadataUtils.newAttachHeadersInterceptor(header));\n    }\n\n    protected MessagingServiceGrpc.MessagingServiceBlockingStub createBlockingStub(Channel channel) {\n        MessagingServiceGrpc.MessagingServiceBlockingStub stub = MessagingServiceGrpc.newBlockingStub(channel);\n        return stub.withInterceptors(MetadataUtils.newAttachHeadersInterceptor(header));\n    }\n\n    protected CompletableFuture<Settings> sendClientSettings(MessagingServiceGrpc.MessagingServiceStub stub,\n        Settings clientSettings) {\n        CompletableFuture<Settings> future = new CompletableFuture<>();\n        StreamObserver<TelemetryCommand> requestStreamObserver = stub.telemetry(new DefaultTelemetryCommandStreamObserver() {\n            @Override\n            public void onNext(TelemetryCommand value) {\n                TelemetryCommand.CommandCase commandCase = value.getCommandCase();\n                if (TelemetryCommand.CommandCase.SETTINGS.equals(commandCase)) {\n                    future.complete(value.getSettings());\n                }\n            }\n        });\n        requestStreamObserver.onNext(TelemetryCommand.newBuilder()\n            .setSettings(clientSettings)\n            .build());\n        future.whenComplete((settings, throwable) -> requestStreamObserver.onCompleted());\n        return future;\n    }\n\n    protected void setUpServer(MessagingServiceGrpc.MessagingServiceImplBase serverImpl,\n        int port, boolean enableInterceptor) throws IOException, CertificateException {\n        SelfSignedCertificate selfSignedCertificate = new SelfSignedCertificate();\n        ServerServiceDefinition serviceDefinition = ServerInterceptors.intercept(serverImpl);\n        if (enableInterceptor) {\n            serviceDefinition = ServerInterceptors.intercept(serverImpl, new ContextInterceptor(), new HeaderInterceptor());\n        }\n        Server server = NettyServerBuilder.forPort(port)\n            .directExecutor()\n            .addService(serviceDefinition)\n            .useTransportSecurity(selfSignedCertificate.certificate(), selfSignedCertificate.privateKey())\n            .build()\n            .start();\n        this.port = server.getPort();\n        // Create a server, add service, start, and register for automatic graceful shutdown.\n        grpcCleanup.register(server);\n\n        ConfigurationManager.getProxyConfig().setGrpcServerPort(this.port);\n        blockingStub = createBlockingStub(createChannel(ConfigurationManager.getProxyConfig().getGrpcServerPort()));\n        stub = createStub(createChannel(ConfigurationManager.getProxyConfig().getGrpcServerPort()));\n    }\n\n    protected Channel createChannel(int port) throws SSLException {\n        return grpcCleanup.register(NettyChannelBuilder.forAddress(\"127.0.0.1\", port)\n            .directExecutor()\n            .sslContext(SslContextBuilder\n                .forClient()\n                .sslProvider(SslProvider.OPENSSL)\n                .trustManager(InsecureTrustManagerFactory.INSTANCE)\n                .applicationProtocolConfig(new ApplicationProtocolConfig(\n                    ApplicationProtocolConfig.Protocol.ALPN,\n                    ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE,\n                    ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT,\n                    ApplicationProtocolNames.HTTP_2))\n                .build()\n            )\n            .build());\n    }\n\n    public void testQueryAssignment() throws Exception {\n        String topic = initTopic();\n        String group = \"group\";\n\n        QueryAssignmentResponse response = blockingStub.queryAssignment(buildQueryAssignmentRequest(topic, group));\n\n        assertQueryAssignment(response, BROKER_NUM);\n    }\n\n    public void testQueryFifoAssignment() throws Exception {\n        String topic = initTopic(TopicMessageType.FIFO);\n        String group = MQRandomUtils.getRandomConsumerGroup();\n        SubscriptionGroupConfig groupConfig = brokerController1.getSubscriptionGroupManager().findSubscriptionGroupConfig(group);\n        groupConfig.setConsumeMessageOrderly(true);\n        brokerController1.getSubscriptionGroupManager().updateSubscriptionGroupConfig(groupConfig);\n        brokerController2.getSubscriptionGroupManager().updateSubscriptionGroupConfig(groupConfig);\n        brokerController3.getSubscriptionGroupManager().updateSubscriptionGroupConfig(groupConfig);\n\n        QueryAssignmentResponse response = blockingStub.queryAssignment(buildQueryAssignmentRequest(topic, group));\n\n        assertQueryAssignment(response, BROKER_NUM * QUEUE_NUMBERS);\n    }\n\n    public void testTransactionCheckThenCommit() {\n        String topic = initTopicOnSampleTopicBroker(BROKER1_NAME, TopicMessageType.TRANSACTION);\n        String group = MQRandomUtils.getRandomConsumerGroup();\n        initConsumerGroup(group);\n\n        AtomicReference<TelemetryCommand> telemetryCommandRef = new AtomicReference<>(null);\n        StreamObserver<TelemetryCommand> requestStreamObserver = stub.telemetry(new DefaultTelemetryCommandStreamObserver() {\n            @Override\n            public void onNext(TelemetryCommand value) {\n                telemetryCommandRef.set(value);\n            }\n        });\n\n        try {\n            requestStreamObserver.onNext(TelemetryCommand.newBuilder()\n                .setSettings(buildPushConsumerClientSettings(group))\n                .build());\n            await().atMost(java.time.Duration.ofSeconds(3)).until(() -> {\n                if (telemetryCommandRef.get() == null) {\n                    return false;\n                }\n                if (telemetryCommandRef.get().getCommandCase() != TelemetryCommand.CommandCase.SETTINGS) {\n                    return false;\n                }\n                return telemetryCommandRef.get() != null;\n            });\n            telemetryCommandRef.set(null);\n            // init consumer offset\n            receiveMessage(blockingStub, topic, group, 1);\n\n            requestStreamObserver.onNext(TelemetryCommand.newBuilder()\n                .setSettings(buildProducerClientSettings(topic))\n                .build());\n            blockingStub.heartbeat(buildHeartbeatRequest(group));\n            await().atMost(java.time.Duration.ofSeconds(3)).until(() -> {\n                if (telemetryCommandRef.get() == null) {\n                    blockingStub.heartbeat(buildHeartbeatRequest(group));\n                    return false;\n                }\n                if (telemetryCommandRef.get().getCommandCase() != TelemetryCommand.CommandCase.SETTINGS) {\n                    blockingStub.heartbeat(buildHeartbeatRequest(group));\n                    return false;\n                }\n                return telemetryCommandRef.get() != null;\n            });\n            telemetryCommandRef.set(null);\n\n            String messageId = createUniqID();\n            SendMessageResponse sendResponse = blockingStub.sendMessage(buildTransactionSendMessageRequest(topic, messageId));\n            assertSendMessage(sendResponse, messageId);\n\n            await().atMost(java.time.Duration.ofMinutes(2)).until(() -> {\n                if (telemetryCommandRef.get() == null) {\n                    blockingStub.heartbeat(buildHeartbeatRequest(group));\n                    return false;\n                }\n                if (telemetryCommandRef.get().getCommandCase() != TelemetryCommand.CommandCase.RECOVER_ORPHANED_TRANSACTION_COMMAND) {\n                    blockingStub.heartbeat(buildHeartbeatRequest(group));\n                    return false;\n                }\n                return telemetryCommandRef.get() != null;\n            });\n            RecoverOrphanedTransactionCommand recoverOrphanedTransactionCommand = telemetryCommandRef.get().getRecoverOrphanedTransactionCommand();\n            assertRecoverOrphanedTransactionCommand(recoverOrphanedTransactionCommand, messageId);\n\n            EndTransactionResponse endTransactionResponse = blockingStub.endTransaction(\n                buildEndTransactionRequest(topic, messageId, recoverOrphanedTransactionCommand.getTransactionId(), TransactionResolution.COMMIT));\n            assertEndTransactionResponse(endTransactionResponse);\n\n            requestStreamObserver.onNext(TelemetryCommand.newBuilder()\n                .setSettings(buildPushConsumerClientSettings(group))\n                .build());\n\n            await().atMost(java.time.Duration.ofSeconds(30)).until(() -> {\n                List<Message> retryMessageList = getMessageFromReceiveMessageResponse(receiveMessage(blockingStub, topic, group));\n                if (retryMessageList.isEmpty()) {\n                    return false;\n                }\n                return retryMessageList.get(0).getSystemProperties()\n                    .getMessageId().equals(messageId);\n            });\n        } finally {\n            requestStreamObserver.onCompleted();\n        }\n    }\n\n    public HeartbeatRequest buildHeartbeatRequest(String group) {\n        return HeartbeatRequest.newBuilder()\n            .setGroup(Resource.newBuilder()\n                .setName(group)\n                .build())\n            .build();\n    }\n\n    public void testSimpleConsumerSendAndRecvDelayMessage() throws Exception {\n        String topic = initTopicOnSampleTopicBroker(BROKER1_NAME, TopicMessageType.DELAY);\n        String group = MQRandomUtils.getRandomConsumerGroup();\n        long delayTime = TimeUnit.SECONDS.toMillis(5);\n        initConsumerGroup(group);\n\n        // init consumer offset\n        this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get();\n        receiveMessage(blockingStub, topic, group, 1);\n\n        this.sendClientSettings(stub, buildProducerClientSettings(topic)).get();\n        String messageId = createUniqID();\n        SendMessageResponse sendResponse = blockingStub.sendMessage(SendMessageRequest.newBuilder()\n            .addMessages(Message.newBuilder()\n                .setTopic(Resource.newBuilder()\n                    .setName(topic)\n                    .build())\n                .setSystemProperties(SystemProperties.newBuilder()\n                    .setMessageId(messageId)\n                    .setQueueId(0)\n                    .setMessageType(MessageType.NORMAL)\n                    .setBodyEncoding(Encoding.GZIP)\n                    .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                    .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                    .setDeliveryTimestamp(Timestamps.fromMillis(System.currentTimeMillis() + delayTime))\n                    .build())\n                .setBody(ByteString.copyFromUtf8(\"hello\"))\n                .build())\n            .build());\n        long sendTime = System.currentTimeMillis();\n        assertSendMessage(sendResponse, messageId);\n\n        this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get();\n\n        AtomicLong recvTime = new AtomicLong();\n        AtomicReference<Message> recvMessage = new AtomicReference<>();\n        await().atMost(java.time.Duration.ofSeconds(10)).until(() -> {\n            List<Message> messageList = getMessageFromReceiveMessageResponse(receiveMessage(blockingStub, topic, group));\n            if (messageList.isEmpty()) {\n                return false;\n            }\n            recvTime.set(System.currentTimeMillis());\n            recvMessage.set(messageList.get(0));\n            return messageList.get(0).getSystemProperties().getMessageId().equals(messageId);\n        });\n\n        assertThat(Math.abs(recvTime.get() - sendTime - delayTime) < 2 * 1000).isTrue();\n    }\n\n    public void testSimpleConsumerSendAndRecallDelayMessage() throws Exception {\n        String topic = initTopicOnSampleTopicBroker(BROKER1_NAME, TopicMessageType.DELAY);\n        String group = MQRandomUtils.getRandomConsumerGroup();\n        initConsumerGroup(group);\n        long delayTime = TimeUnit.SECONDS.toMillis(5);\n\n        // init consumer offset\n        this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get();\n        receiveMessage(blockingStub, topic, group, 1);\n\n        this.sendClientSettings(stub, buildProducerClientSettings(topic)).get();\n        String messageId = createUniqID();\n        SendMessageResponse sendResponse = blockingStub.sendMessage(SendMessageRequest.newBuilder()\n            .addMessages(Message.newBuilder()\n                .setTopic(Resource.newBuilder()\n                    .setName(topic)\n                    .build())\n                .setSystemProperties(SystemProperties.newBuilder()\n                    .setMessageId(messageId)\n                    .setQueueId(0)\n                    .setMessageType(MessageType.DELAY)\n                    .setBodyEncoding(Encoding.GZIP)\n                    .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                    .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                    .setDeliveryTimestamp(Timestamps.fromMillis(System.currentTimeMillis() + delayTime))\n                    .build())\n                .setBody(ByteString.copyFromUtf8(\"hello\"))\n                .build())\n            .build());\n        long sendTime = System.currentTimeMillis();\n        assertSendMessage(sendResponse, messageId);\n        String recallHandle = sendResponse.getEntries(0).getRecallHandle();\n        assertThat(recallHandle).isNotEmpty();\n\n        RecallMessageRequest recallRequest = RecallMessageRequest.newBuilder()\n            .setRecallHandle(recallHandle)\n            .setTopic(Resource.newBuilder().setResourceNamespace(\"\").setName(topic).build())\n            .build();\n        RecallMessageResponse recallResponse =\n            blockingStub.withDeadlineAfter(2, TimeUnit.SECONDS).recallMessage(recallRequest);\n        assertThat(recallResponse.getStatus()).isEqualTo(\n            ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()));\n        assertThat(recallResponse.getMessageId()).isEqualTo(messageId);\n\n        this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get();\n\n        AtomicLong recvTime = new AtomicLong();\n        AtomicReference<Message> recvMessage = new AtomicReference<>();\n        try {\n            await().atMost(java.time.Duration.ofSeconds(10)).until(() -> {\n                List<Message> messageList = getMessageFromReceiveMessageResponse(receiveMessage(blockingStub, topic, group));\n                if (messageList.isEmpty()) {\n                    return false;\n                }\n                recvTime.set(System.currentTimeMillis());\n                recvMessage.set(messageList.get(0));\n                return messageList.get(0).getSystemProperties().getMessageId().equals(messageId);\n            });\n        } catch (Exception e) {\n        }\n        assertThat(recvTime.get()).isEqualTo(0L);\n        assertThat(recvMessage.get()).isNull();\n    }\n\n    public void testSimpleConsumerSendAndRecvBigMessage() throws Exception {\n        String topic = initTopicOnSampleTopicBroker(BROKER1_NAME);\n        String group = MQRandomUtils.getRandomConsumerGroup();\n        initConsumerGroup(group);\n\n        int bodySize = 4 * 1024;\n\n        // init consumer offset\n        this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get();\n        receiveMessage(blockingStub, topic, group, 1);\n\n        this.sendClientSettings(stub, buildProducerClientSettings(topic)).get();\n        String messageId = createUniqID();\n        SendMessageResponse sendResponse = blockingStub.sendMessage(buildSendBigMessageRequest(topic, messageId, bodySize));\n        assertSendMessage(sendResponse, messageId);\n\n        this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get();\n\n        Message message = assertAndGetReceiveMessage(receiveMessage(blockingStub, topic, group), messageId);\n        assertThat(message.getSystemProperties().getBodyEncoding()).isEqualTo(Encoding.GZIP);\n        assertThat(message.getBody().size()).isEqualTo(bodySize);\n    }\n\n    public void testSimpleConsumerSendAndRecv() throws Exception {\n        String topic = initTopicOnSampleTopicBroker(BROKER1_NAME);\n        String group = MQRandomUtils.getRandomConsumerGroup();\n        initConsumerGroup(group);\n\n        // init consumer offset\n        this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get();\n        receiveMessage(blockingStub, topic, group, 1);\n\n        this.sendClientSettings(stub, buildProducerClientSettings(topic)).get();\n        String messageId = createUniqID();\n        SendMessageResponse sendResponse = blockingStub.sendMessage(buildSendMessageRequest(topic, messageId));\n        assertSendMessage(sendResponse, messageId);\n        assertThat(sendResponse.getEntries(0).getRecallHandle()).isNullOrEmpty();\n\n        this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get();\n\n        Message message = assertAndGetReceiveMessage(receiveMessage(blockingStub, topic, group), messageId);\n\n        String receiptHandle = message.getSystemProperties().getReceiptHandle();\n        ChangeInvisibleDurationResponse changeResponse = blockingStub.changeInvisibleDuration(buildChangeInvisibleDurationRequest(topic, group, receiptHandle, 5));\n        assertChangeInvisibleDurationResponse(changeResponse, receiptHandle);\n\n        List<String> ackHandles = new ArrayList<>();\n        ackHandles.add(changeResponse.getReceiptHandle());\n\n        await().atMost(java.time.Duration.ofSeconds(20)).until(() -> {\n            List<Message> retryMessageList = getMessageFromReceiveMessageResponse(receiveMessage(blockingStub, topic, group));\n            if (retryMessageList.isEmpty()) {\n                return false;\n            }\n            if (retryMessageList.get(0).getSystemProperties()\n                .getMessageId().equals(messageId)) {\n                ackHandles.add(retryMessageList.get(0).getSystemProperties().getReceiptHandle());\n                return true;\n            }\n            return false;\n        });\n\n        assertThat(ackHandles.size()).isEqualTo(2);\n        AckMessageResponse ackMessageResponse = blockingStub.ackMessage(buildAckMessageRequest(topic, group,\n            AckMessageEntry.newBuilder().setMessageId(messageId).setReceiptHandle(ackHandles.get(0)).build(),\n            AckMessageEntry.newBuilder().setMessageId(messageId).setReceiptHandle(ackHandles.get(1)).build()));\n        assertThat(ackMessageResponse.getStatus().getCode()).isEqualTo(Code.MULTIPLE_RESULTS);\n        int okNum = 0;\n        int expireNum = 0;\n        for (AckMessageResultEntry entry : ackMessageResponse.getEntriesList()) {\n            if (entry.getStatus().getCode().equals(Code.OK)) {\n                okNum++;\n            } else if (entry.getStatus().getCode().equals(Code.INVALID_RECEIPT_HANDLE)) {\n                expireNum++;\n            }\n        }\n        assertThat(okNum).isEqualTo(1);\n        assertThat(expireNum).isEqualTo(1);\n    }\n\n    public void testSimpleConsumerToDLQ() throws Exception {\n        String topic = initTopicOnSampleTopicBroker(BROKER1_NAME);\n        String group = MQRandomUtils.getRandomConsumerGroup();\n        initConsumerGroup(group);\n        int maxDeliveryAttempts = 2;\n\n        SubscriptionGroupConfig groupConfig = brokerController1.getSubscriptionGroupManager().findSubscriptionGroupConfig(group);\n        groupConfig.setRetryMaxTimes(maxDeliveryAttempts - 1);\n        brokerController1.getSubscriptionGroupManager().updateSubscriptionGroupConfig(groupConfig);\n        brokerController2.getSubscriptionGroupManager().updateSubscriptionGroupConfig(groupConfig);\n        brokerController3.getSubscriptionGroupManager().updateSubscriptionGroupConfig(groupConfig);\n\n        // init consumer offset\n        this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get();\n        receiveMessage(blockingStub, topic, group, 1);\n\n        this.sendClientSettings(stub, buildProducerClientSettings(topic)).get();\n        String messageId = createUniqID();\n        SendMessageResponse sendResponse = blockingStub.sendMessage(buildSendMessageRequest(topic, messageId));\n        assertSendMessage(sendResponse, messageId);\n\n        this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get();\n\n        AtomicInteger receiveMessageCount = new AtomicInteger(0);\n\n        assertAndGetReceiveMessage(receiveMessage(blockingStub, topic, group), messageId);\n        receiveMessageCount.incrementAndGet();\n\n        DefaultMQPullConsumer defaultMQPullConsumer = new DefaultMQPullConsumer(group);\n        defaultMQPullConsumer.start();\n        org.apache.rocketmq.common.message.MessageQueue dlqMQ = new org.apache.rocketmq.common.message.MessageQueue(MixAll.getDLQTopic(group), BROKER1_NAME, 0);\n        await().atMost(java.time.Duration.ofSeconds(30)).until(() -> {\n            try {\n                List<Message> messageList = getMessageFromReceiveMessageResponse(receiveMessage(blockingStub, topic, group, 1));\n                receiveMessageCount.addAndGet(messageList.size());\n\n                PullResult pullResult = defaultMQPullConsumer.pull(dlqMQ, \"*\", 0L, 1);\n                if (!PullStatus.FOUND.equals(pullResult.getPullStatus())) {\n                    return false;\n                }\n                MessageExt messageExt = pullResult.getMsgFoundList().get(0);\n                return messageId.equals(messageExt.getMsgId());\n            } catch (Throwable ignore) {\n                return false;\n            }\n        });\n\n        assertThat(receiveMessageCount.get()).isEqualTo(maxDeliveryAttempts);\n    }\n\n    public void testConsumeOrderly() throws Exception {\n        String topic = initTopicOnSampleTopicBroker(BROKER1_NAME, TopicMessageType.FIFO);\n        String group = MQRandomUtils.getRandomConsumerGroup();\n\n        SubscriptionGroupConfig groupConfig = brokerController1.getSubscriptionGroupManager().findSubscriptionGroupConfig(group);\n        groupConfig.setConsumeMessageOrderly(true);\n        brokerController1.getSubscriptionGroupManager().updateSubscriptionGroupConfig(groupConfig);\n        brokerController2.getSubscriptionGroupManager().updateSubscriptionGroupConfig(groupConfig);\n        brokerController3.getSubscriptionGroupManager().updateSubscriptionGroupConfig(groupConfig);\n\n        this.sendClientSettings(stub, buildPushConsumerClientSettings(group)).get();\n        receiveMessage(blockingStub, topic, group, 1);\n\n        String messageGroup = \"group\";\n        this.sendClientSettings(stub, buildProducerClientSettings(topic)).get();\n        List<String> messageIdList = new ArrayList<>();\n        for (int i = 0; i < 3; i++) {\n            String messageId = createUniqID();\n            messageIdList.add(messageId);\n            SendMessageResponse sendResponse = blockingStub.sendMessage(buildSendOrderMessageRequest(topic, messageId, messageGroup));\n            assertSendMessage(sendResponse, messageId);\n        }\n\n        List<String> messageRecvList = new ArrayList<>();\n        this.sendClientSettings(stub, buildPushConsumerClientSettings(group)).get();\n        await().atMost(java.time.Duration.ofSeconds(20)).until(() -> {\n            List<Message> retryMessageList = getMessageFromReceiveMessageResponse(receiveMessage(blockingStub, topic, group));\n            if (retryMessageList.isEmpty()) {\n                return false;\n            }\n            for (Message message : retryMessageList) {\n                String messageId = message.getSystemProperties().getMessageId();\n                messageRecvList.add(messageId);\n                blockingStub.ackMessage(buildAckMessageRequest(topic, group,\n                    AckMessageEntry.newBuilder().setMessageId(messageId).setReceiptHandle(message.getSystemProperties().getReceiptHandle()).build()));\n            }\n            return messageRecvList.size() == messageIdList.size();\n        });\n\n        for (int i = 0; i < messageIdList.size(); i++) {\n            assertThat(messageRecvList.get(i)).isEqualTo(messageIdList.get(i));\n        }\n    }\n\n    public void testSimpleConsumerSendAndRecvPriorityMessage() throws Exception {\n        brokerController1.getBrokerConfig().setPriorityOrderAsc(true);\n        String topic = initTopicOnSampleTopicBroker(BROKER1_NAME, TopicMessageType.PRIORITY);\n        String group = MQRandomUtils.getRandomConsumerGroup();\n        initConsumerGroup(group);\n\n        // init consumer offset\n        this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get();\n        receiveMessage(blockingStub, topic, group, 1);\n\n        this.sendClientSettings(stub, buildProducerClientSettings(topic)).get();\n        for (int i = 0; i < BaseConf.QUEUE_NUMBERS; i++) {\n            String messageId = createUniqID();\n            SendMessageResponse sendResponse = blockingStub.sendMessage(SendMessageRequest.newBuilder()\n                .addMessages(Message.newBuilder()\n                    .setTopic(Resource.newBuilder()\n                        .setName(topic)\n                        .build())\n                    .setSystemProperties(SystemProperties.newBuilder()\n                        .setMessageId(messageId)\n                        .setQueueId(0)\n                        .setMessageType(MessageType.PRIORITY)\n                        .setBodyEncoding(Encoding.GZIP)\n                        .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                        .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                        .setPriority(i)\n                        .build())\n                    .setBody(ByteString.copyFromUtf8(\"hello\"))\n                    .build())\n                .build());\n            assertSendMessage(sendResponse, messageId);\n        }\n\n        this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get();\n        List<Message> recvList = new ArrayList<>();\n        try {\n            await().atMost(java.time.Duration.ofSeconds(10)).until(() -> {\n                List<Message> messageList = getMessageFromReceiveMessageResponse(receiveMessage(blockingStub, topic, group));\n                if (messageList.isEmpty()) {\n                    return false;\n                }\n                recvList.addAll(messageList);\n                return recvList.size() == BaseConf.QUEUE_NUMBERS;\n            });\n        } catch (Exception e) {\n        }\n        for (int i = 0; i < BaseConf.QUEUE_NUMBERS; i++) {\n            // default priority order: 0 as lowest priority\n            assertThat(recvList.get(i).getSystemProperties().getPriority()).isEqualTo(BaseConf.QUEUE_NUMBERS - i - 1);\n        }\n    }\n\n    public List<ReceiveMessageResponse> receiveMessage(MessagingServiceGrpc.MessagingServiceBlockingStub stub,\n        String topic, String group) {\n        return receiveMessage(stub, topic, group, 15);\n    }\n\n    public List<ReceiveMessageResponse> receiveMessage(MessagingServiceGrpc.MessagingServiceBlockingStub stub,\n        String topic, String group, int timeSeconds) {\n        List<ReceiveMessageResponse> responseList = new ArrayList<>();\n        Iterator<ReceiveMessageResponse> responseIterator = stub.withDeadlineAfter(timeSeconds, TimeUnit.SECONDS)\n            .receiveMessage(buildReceiveMessageRequest(topic, group));\n        while (responseIterator.hasNext()) {\n            responseList.add(responseIterator.next());\n        }\n        return responseList;\n    }\n\n    public List<Message> getMessageFromReceiveMessageResponse(List<ReceiveMessageResponse> responseList) {\n        List<Message> messageList = new ArrayList<>();\n        for (ReceiveMessageResponse response : responseList) {\n            if (response.hasMessage()) {\n                messageList.add(response.getMessage());\n            }\n        }\n        return messageList;\n    }\n\n    public QueryRouteRequest buildQueryRouteRequest(String topic) {\n        return QueryRouteRequest.newBuilder()\n            .setEndpoints(buildEndpoints(port))\n            .setTopic(Resource.newBuilder()\n                .setName(topic)\n                .build())\n            .build();\n    }\n\n    public QueryAssignmentRequest buildQueryAssignmentRequest(String topic, String group) {\n        return QueryAssignmentRequest.newBuilder()\n            .setEndpoints(buildEndpoints(port))\n            .setTopic(Resource.newBuilder().setName(topic).build())\n            .setGroup(Resource.newBuilder().setName(group).build())\n            .build();\n    }\n\n    public SendMessageRequest buildSendMessageRequest(String topic, String messageId) {\n        return SendMessageRequest.newBuilder()\n            .addMessages(Message.newBuilder()\n                .setTopic(Resource.newBuilder()\n                    .setName(topic)\n                    .build())\n                .setSystemProperties(SystemProperties.newBuilder()\n                    .setMessageId(messageId)\n                    .setQueueId(0)\n                    .setMessageType(MessageType.NORMAL)\n                    .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                    .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                    .build())\n                .setBody(ByteString.copyFromUtf8(\"123\"))\n                .build())\n            .build();\n    }\n\n    public SendMessageRequest buildSendOrderMessageRequest(String topic, String messageId, String messageGroup) {\n        return SendMessageRequest.newBuilder()\n            .addMessages(Message.newBuilder()\n                .setTopic(Resource.newBuilder()\n                    .setName(topic)\n                    .build())\n                .setSystemProperties(SystemProperties.newBuilder()\n                    .setMessageId(messageId)\n                    .setQueueId(0)\n                    .setMessageType(MessageType.FIFO)\n                    .setMessageGroup(messageGroup)\n                    .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                    .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                    .build())\n                .setBody(ByteString.copyFromUtf8(\"123\"))\n                .build())\n            .build();\n    }\n\n    public SendMessageRequest buildSendBigMessageRequest(String topic, String messageId, int messageSize) {\n        return SendMessageRequest.newBuilder()\n            .addMessages(Message.newBuilder()\n                .setTopic(Resource.newBuilder()\n                    .setName(topic)\n                    .build())\n                .setSystemProperties(SystemProperties.newBuilder()\n                    .setMessageId(messageId)\n                    .setQueueId(0)\n                    .setMessageType(MessageType.NORMAL)\n                    .setBodyEncoding(Encoding.GZIP)\n                    .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                    .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                    .build())\n                .setBody(ByteString.copyFromUtf8(RandomUtils.getStringWithCharacter(messageSize)))\n                .build())\n            .build();\n    }\n\n    public SendMessageRequest buildTransactionSendMessageRequest(String topic, String messageId) {\n        return SendMessageRequest.newBuilder()\n            .addMessages(Message.newBuilder()\n                .setTopic(Resource.newBuilder()\n                    .setName(topic)\n                    .build())\n                .setSystemProperties(SystemProperties.newBuilder()\n                    .setMessageId(messageId)\n                    .setQueueId(0)\n                    .setMessageType(MessageType.TRANSACTION)\n                    .setOrphanedTransactionRecoveryDuration(Duration.newBuilder().setSeconds(10))\n                    .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))\n                    .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), \"127.0.0.1:1234\"))\n                    .build())\n                .setBody(ByteString.copyFromUtf8(\"123\"))\n                .build())\n            .build();\n    }\n\n    public ReceiveMessageRequest buildReceiveMessageRequest(String topic, String group) {\n        return ReceiveMessageRequest.newBuilder()\n            .setGroup(Resource.newBuilder()\n                .setName(group)\n                .build())\n            .setMessageQueue(MessageQueue.newBuilder()\n                .setTopic(Resource.newBuilder()\n                    .setName(topic)\n                    .build())\n                .setId(-1)\n                .build())\n            .setBatchSize(1)\n            .setAutoRenew(false)\n            .setInvisibleDuration(Duration.newBuilder()\n                .setSeconds(3)\n                .build())\n            .build();\n    }\n\n    public AckMessageRequest buildAckMessageRequest(String topic, String group, AckMessageEntry... entry) {\n        return AckMessageRequest.newBuilder()\n            .setGroup(Resource.newBuilder()\n                .setName(group)\n                .build())\n            .setTopic(Resource.newBuilder()\n                .setName(topic)\n                .build())\n            .addAllEntries(Arrays.stream(entry).collect(Collectors.toList()))\n            .build();\n    }\n\n    public EndTransactionRequest buildEndTransactionRequest(String topic, String messageId, String transactionId,\n        TransactionResolution resolution) {\n        return EndTransactionRequest.newBuilder()\n            .setMessageId(messageId)\n            .setTopic(Resource.newBuilder()\n                .setName(topic)\n                .build())\n            .setTransactionId(transactionId)\n            .setResolution(resolution)\n            .setSource(TransactionSource.SOURCE_SERVER_CHECK)\n            .build();\n    }\n\n    public ChangeInvisibleDurationRequest buildChangeInvisibleDurationRequest(String topic, String group,\n        String receiptHandle, int second) {\n        return ChangeInvisibleDurationRequest.newBuilder()\n            .setTopic(Resource.newBuilder().setName(topic).build())\n            .setGroup(Resource.newBuilder().setName(group).build())\n            .setInvisibleDuration(Durations.fromSeconds(second))\n            .setReceiptHandle(receiptHandle)\n            .build();\n    }\n\n    public void assertQueryRoute(QueryRouteResponse response, int messageQueueSize) {\n        assertThat(response.getStatus()).isEqualTo(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()));\n        assertThat(response.getMessageQueuesList().size()).isEqualTo(messageQueueSize);\n        assertThat(response.getMessageQueues(0).getBroker().getEndpoints().getAddresses(0).getPort()).isEqualTo(ConfigurationManager.getProxyConfig().getGrpcServerPort());\n    }\n\n    public void assertQueryAssignment(QueryAssignmentResponse response, int assignmentCount) {\n        assertThat(response.getStatus()).isEqualTo(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()));\n        assertThat(response.getAssignmentsCount()).isEqualTo(assignmentCount);\n        assertThat(response.getAssignments(0).getMessageQueue().getBroker().getEndpoints().getAddresses(0).getPort()).isEqualTo(ConfigurationManager.getProxyConfig().getGrpcServerPort());\n    }\n\n    public void assertSendMessage(SendMessageResponse response, String messageId) {\n        assertThat(response.getStatus()).isEqualTo(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()));\n        assertThat(response.getEntries(0).getMessageId()).isEqualTo(messageId);\n    }\n\n    public Message assertAndGetReceiveMessage(List<ReceiveMessageResponse> response, String messageId) {\n        assertThat(response.get(0).hasStatus()).isTrue();\n        assertThat(response.get(0).getStatus()\n            .getCode()).isEqualTo(Code.OK);\n        assertThat(response.get(1).getMessage()\n            .getSystemProperties()\n            .getMessageId()).isEqualTo(messageId);\n        return response.get(1).getMessage();\n    }\n\n    public void assertRecoverOrphanedTransactionCommand(RecoverOrphanedTransactionCommand command, String messageId) {\n        assertThat(command.getTransactionId()).isNotBlank();\n    }\n\n    public void assertEndTransactionResponse(EndTransactionResponse response) {\n        assertThat(response.getStatus().getCode()).isEqualTo(Code.OK);\n    }\n\n    public void assertChangeInvisibleDurationResponse(ChangeInvisibleDurationResponse response, String prevHandle) {\n        assertThat(response.getStatus().getCode()).isEqualTo(Code.OK);\n        assertThat(response.getReceiptHandle()).isNotEqualTo(prevHandle);\n    }\n\n    public Endpoints buildEndpoints(int port) {\n        return Endpoints.newBuilder()\n            .setScheme(AddressScheme.IPv4)\n            .addAddresses(Address.newBuilder()\n                .setHost(\"127.0.0.1\")\n                .setPort(port)\n                .build())\n            .build();\n    }\n\n    public Settings buildSimpleConsumerClientSettings(String group) {\n        return Settings.newBuilder()\n            .setClientType(ClientType.SIMPLE_CONSUMER)\n            .setRequestTimeout(Durations.fromSeconds(3))\n            .setSubscription(Subscription.newBuilder()\n                .setGroup(Resource.newBuilder().setName(group).build())\n                .build())\n            .build();\n    }\n\n    public Settings buildPushConsumerClientSettings(String group) {\n        return buildPushConsumerClientSettings(2, group);\n    }\n\n    public Settings buildPushConsumerClientSettings(int maxDeliveryAttempts, String group) {\n        return Settings.newBuilder()\n            .setClientType(ClientType.PUSH_CONSUMER)\n            .setRequestTimeout(Durations.fromSeconds(3))\n            .setBackoffPolicy(RetryPolicy.newBuilder()\n                .setMaxAttempts(maxDeliveryAttempts)\n                .build())\n            .setSubscription(Subscription.newBuilder()\n                .setGroup(Resource.newBuilder().setName(group).build())\n                .build())\n            .build();\n    }\n\n    public Settings buildProducerClientSettings(String... topics) {\n        List<Resource> topicResources = Arrays.stream(topics).map(topic -> Resource.newBuilder().setName(topic).build())\n            .collect(Collectors.toList());\n        return Settings.newBuilder()\n            .setClientType(ClientType.PRODUCER)\n            .setPublishing(Publishing.newBuilder()\n                .addAllTopics(topicResources)\n                .build())\n            .build();\n    }\n\n    protected static class DefaultTelemetryCommandStreamObserver implements StreamObserver<TelemetryCommand> {\n\n        @Override\n        public void onNext(TelemetryCommand value) {\n\n        }\n\n        @Override\n        public void onError(Throwable t) {\n\n        }\n\n        @Override\n        public void onCompleted() {\n\n        }\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/grpc/v2/LocalGrpcIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.grpc.v2;\n\nimport apache.rocketmq.v2.QueryRouteResponse;\nimport org.apache.rocketmq.proxy.config.ConfigurationManager;\nimport org.apache.rocketmq.proxy.grpc.v2.GrpcMessagingApplication;\nimport org.apache.rocketmq.proxy.processor.DefaultMessagingProcessor;\nimport org.apache.rocketmq.proxy.processor.MessagingProcessor;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.FixMethodOrder;\nimport org.junit.Test;\nimport org.junit.Ignore;\nimport org.junit.runners.MethodSorters;\n\n@FixMethodOrder(value = MethodSorters.NAME_ASCENDING)\npublic class LocalGrpcIT extends GrpcBaseIT {\n\n    private MessagingProcessor messagingProcessor;\n    private GrpcMessagingApplication grpcMessagingApplication;\n\n    @Before\n    public void setUp() throws Exception {\n        super.setUp();\n        messagingProcessor = DefaultMessagingProcessor.createForLocalMode(brokerController1);\n        messagingProcessor.start();\n        grpcMessagingApplication = GrpcMessagingApplication.create(messagingProcessor);\n        grpcMessagingApplication.start();\n        setUpServer(grpcMessagingApplication, ConfigurationManager.getProxyConfig().getGrpcServerPort(), true);\n    }\n\n    @After\n    public void clean() throws Exception {\n        messagingProcessor.shutdown();\n        grpcMessagingApplication.shutdown();\n        shutdown();\n    }\n\n    @Test\n    public void testQueryRoute() throws Exception {\n        String topic = initTopic();\n\n        QueryRouteResponse response = blockingStub.queryRoute(buildQueryRouteRequest(topic));\n        assertQueryRoute(response, brokerControllerList.size() * DEFAULT_QUEUE_NUMS);\n    }\n\n    @Test\n    public void testQueryAssignment() throws Exception {\n        super.testQueryAssignment();\n    }\n\n    @Test\n    public void testQueryFifoAssignment() throws Exception {\n        super.testQueryFifoAssignment();\n    }\n\n    @Test\n    @Ignore\n    public void testTransactionCheckThenCommit() {\n        super.testTransactionCheckThenCommit();\n    }\n\n    @Test\n    @Ignore\n    public void testSimpleConsumerSendAndRecvDelayMessage() throws Exception {\n        super.testSimpleConsumerSendAndRecvDelayMessage();\n    }\n\n    @Test\n    public void testSimpleConsumerSendAndRecallDelayMessage() throws Exception {\n        super.testSimpleConsumerSendAndRecallDelayMessage();\n    }\n\n    @Test\n    public void testSimpleConsumerSendAndRecvBigMessage() throws Exception {\n        super.testSimpleConsumerSendAndRecvBigMessage();\n    }\n\n    @Test\n    public void testSimpleConsumerSendAndRecv() throws Exception {\n        super.testSimpleConsumerSendAndRecv();\n    }\n\n    @Test\n    public void testSimpleConsumerToDLQ() throws Exception {\n        super.testSimpleConsumerToDLQ();\n    }\n\n    @Test\n    public void testConsumeOrderly() throws Exception {\n        super.testConsumeOrderly();\n    }\n\n    @Test\n    public void testSimpleConsumerSendAndRecvPriorityMessage() throws Exception {\n        super.testSimpleConsumerSendAndRecvPriorityMessage();\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/lmq/TestBenchLmqStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.test.lmq;\n\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.concurrent.ConcurrentHashMap;\nimport org.apache.commons.lang3.reflect.FieldUtils;\nimport org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;\nimport org.apache.rocketmq.client.consumer.PullCallback;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.impl.MQClientAPIImpl;\nimport org.apache.rocketmq.client.impl.consumer.DefaultMQPullConsumerImpl;\nimport org.apache.rocketmq.client.impl.consumer.RebalanceImpl;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.apache.rocketmq.test.lmq.benchmark.BenchLmqStore;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.mockito.junit.MockitoJUnitRunner;\n\nimport static org.mockito.ArgumentMatchers.any;\nimport static org.mockito.ArgumentMatchers.anyInt;\nimport static org.mockito.ArgumentMatchers.anyLong;\nimport static org.mockito.ArgumentMatchers.anyString;\nimport static org.mockito.Mockito.atLeastOnce;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.verify;\nimport static org.mockito.Mockito.when;\n\n@RunWith(MockitoJUnitRunner.class)\npublic class TestBenchLmqStore {\n    @Test\n    public void test() throws MQBrokerException, RemotingException, InterruptedException, MQClientException {\n        System.setProperty(\"sendThreadNum\", \"1\");\n        System.setProperty(\"pullConsumerNum\", \"1\");\n        System.setProperty(\"consumerThreadNum\", \"1\");\n        BenchLmqStore.defaultMQProducer = mock(DefaultMQProducer.class);\n//        SendResult sendResult = new SendResult();\n//        when(BenchLmqStore.defaultMQProducer.send(any(Message.class))).thenReturn(sendResult);\n        BenchLmqStore.doSend();\n        Thread.sleep(100L);\n        //verify(BenchLmqStore.defaultMQProducer, atLeastOnce()).send(any(Message.class));\n        BenchLmqStore.defaultMQPullConsumers = new DefaultMQPullConsumer[1];\n        BenchLmqStore.defaultMQPullConsumers[0] = mock(DefaultMQPullConsumer.class);\n        BenchLmqStore.doPull(new ConcurrentHashMap<>(), new MessageQueue(), 1L);\n        verify(BenchLmqStore.defaultMQPullConsumers[0], atLeastOnce()).pullBlockIfNotFound(any(MessageQueue.class), anyString(), anyLong(), anyInt(), any(\n            PullCallback.class));\n    }\n\n    @Test\n    public void testOffset() throws RemotingException, InterruptedException, MQClientException, MQBrokerException, IllegalAccessException {\n        System.setProperty(\"sendThreadNum\", \"1\");\n        DefaultMQPullConsumer defaultMQPullConsumer = mock(DefaultMQPullConsumer.class);\n        BenchLmqStore.defaultMQPullConsumers = new DefaultMQPullConsumer[1];\n        BenchLmqStore.defaultMQPullConsumers[0] = defaultMQPullConsumer;\n        DefaultMQPullConsumerImpl defaultMQPullConsumerImpl = mock(DefaultMQPullConsumerImpl.class);\n        when(defaultMQPullConsumer.getDefaultMQPullConsumerImpl()).thenReturn(defaultMQPullConsumerImpl);\n        RebalanceImpl rebalanceImpl = mock(RebalanceImpl.class);\n        when(defaultMQPullConsumerImpl.getRebalanceImpl()).thenReturn(rebalanceImpl);\n        MQClientInstance mqClientInstance = mock(MQClientInstance.class);\n        when(rebalanceImpl.getmQClientFactory()).thenReturn(mqClientInstance);\n        MQClientAPIImpl mqClientAPI = mock(MQClientAPIImpl.class);\n        when(mqClientInstance.getMQClientAPIImpl()).thenReturn(mqClientAPI);\n        TopicRouteData topicRouteData = new TopicRouteData();\n        HashMap<Long, String> brokerAddrs = new HashMap<>();\n        brokerAddrs.put(MixAll.MASTER_ID, \"test\");\n        List<BrokerData> brokerData = Collections.singletonList(new BrokerData(\"test\", \"test\", brokerAddrs));\n        topicRouteData.setBrokerDatas(brokerData);\n        FieldUtils.writeStaticField(BenchLmqStore.class, \"lmqTopic\", \"test\", true);\n        when(mqClientAPI.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(topicRouteData);\n        BenchLmqStore.doBenchOffset();\n        Thread.sleep(100L);\n        verify(mqClientAPI, atLeastOnce()).queryConsumerOffset(anyString(), any(QueryConsumerOffsetRequestHeader.class), anyLong());\n        verify(mqClientAPI, atLeastOnce()).updateConsumerOffset(anyString(), any(UpdateConsumerOffsetRequestHeader.class), anyLong());\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/offset/LagCalculationIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.offset;\n\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.broker.filter.ConsumerFilterData;\nimport org.apache.rocketmq.broker.filter.ExpressionMessageFilter;\nimport org.apache.rocketmq.client.consumer.MessageSelector;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.filter.ExpressionType;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.admin.ConsumeStats;\nimport org.apache.rocketmq.remoting.protocol.admin.OffsetWrapper;\nimport org.apache.rocketmq.remoting.protocol.filter.FilterAPI;\nimport org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;\nimport org.apache.rocketmq.store.DefaultMessageFilter;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.client.rmq.RMQSqlConsumer;\nimport org.apache.rocketmq.test.factory.ConsumerFactory;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQBlockListener;\nimport org.apache.rocketmq.test.message.MessageQueueMsg;\nimport org.apache.rocketmq.test.util.MQAdminTestUtils;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.FixMethodOrder;\nimport org.junit.Test;\nimport org.junit.runners.MethodSorters;\n\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertEquals;\n\n@FixMethodOrder(MethodSorters.NAME_ASCENDING)\npublic class LagCalculationIT extends BaseConf {\n    private static final Logger LOGGER = LoggerFactory.getLogger(LagCalculationIT.class);\n    private RMQNormalProducer producer = null;\n    private RMQNormalConsumer consumer = null;\n    private String topic = null;\n    private RMQBlockListener blockListener = null;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        LOGGER.info(String.format(\"use topic: %s;\", topic));\n        for (BrokerController controller : brokerControllerList) {\n            controller.getBrokerConfig().setLongPollingEnable(false);\n            controller.getBrokerConfig().setShortPollingTimeMills(500);\n            controller.getBrokerConfig().setEstimateAccumulation(true);\n        }\n        producer = getProducer(NAMESRV_ADDR, topic);\n        blockListener = new RMQBlockListener(false);\n        consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", blockListener);\n    }\n\n    @After\n    public void tearDown() {\n        shutdown();\n    }\n\n    private Pair<Long, Long> getLag(List<MessageQueue> mqs) throws ConsumeQueueException {\n        long lag = 0;\n        long pullLag = 0;\n        for (BrokerController controller : brokerControllerList) {\n            ConsumeStats consumeStats = MQAdminTestUtils.examineConsumeStats(controller.getBrokerAddr(), topic, consumer.getConsumerGroup());\n            Map<MessageQueue, OffsetWrapper> offsetTable = consumeStats.getOffsetTable();\n            for (MessageQueue mq : mqs) {\n                if (mq.getBrokerName().equals(controller.getBrokerConfig().getBrokerName())) {\n                    long brokerOffset = controller.getMessageStore().getMaxOffsetInQueue(topic, mq.getQueueId());\n\n                    long consumerOffset = controller.getConsumerOffsetManager().queryOffset(consumer.getConsumerGroup(),\n                        topic, mq.getQueueId());\n                    long pullOffset =\n                        controller.getConsumerOffsetManager().queryPullOffset(consumer.getConsumerGroup(),\n                            topic, mq.getQueueId());\n                    OffsetWrapper offsetWrapper = offsetTable.get(mq);\n                    assertEquals(brokerOffset, offsetWrapper.getBrokerOffset());\n                    if (offsetWrapper.getConsumerOffset() != consumerOffset || offsetWrapper.getPullOffset() != pullOffset) {\n                        return new Pair<>(-1L, -1L);\n                    }\n                    lag += brokerOffset - consumerOffset;\n                    pullLag += brokerOffset - pullOffset;\n                }\n            }\n        }\n        return new Pair<>(lag, pullLag);\n    }\n\n    public void waitForFullyDispatched() {\n        await().atMost(5, TimeUnit.SECONDS).until(() -> {\n            for (BrokerController controller : brokerControllerList) {\n                if (controller.getMessageStore().dispatchBehindBytes() != 0) {\n                    return false;\n                }\n            }\n            return true;\n        });\n    }\n\n    @Test\n    public void testCalculateLag() throws ConsumeQueueException {\n        int msgSize = 10;\n        List<MessageQueue> mqs = producer.getMessageQueue();\n        MessageQueueMsg mqMsgs = new MessageQueueMsg(mqs, msgSize);\n\n        producer.send(mqMsgs.getMsgsWithMQ());\n        waitForFullyDispatched();\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        consumer.getConsumer().getDefaultMQPushConsumerImpl().persistConsumerOffset();\n\n        // wait for consume all msgs\n        await().atMost(5, TimeUnit.SECONDS).until(() -> {\n            Pair<Long, Long> lag = getLag(mqs);\n            return lag.getObject1() == 0 && lag.getObject2() == 0;\n        });\n\n        blockListener.setBlock(true);\n        consumer.clearMsg();\n        producer.clearMsg();\n        producer.send(mqMsgs.getMsgsWithMQ());\n        waitForFullyDispatched();\n\n        // wait for pull all msgs\n        await().atMost(5, TimeUnit.SECONDS).until(() -> {\n            Pair<Long, Long> lag = getLag(mqs);\n            return lag.getObject1() == producer.getAllMsgBody().size() && lag.getObject2() == 0;\n        });\n\n        blockListener.setBlock(false);\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        consumer.shutdown();\n        producer.clearMsg();\n        producer.send(mqMsgs.getMsgsWithMQ());\n        waitForFullyDispatched();\n\n        Pair<Long, Long> lag = getLag(mqs);\n        assertEquals(producer.getAllMsgBody().size(), (long) lag.getObject1());\n        assertEquals(producer.getAllMsgBody().size(), (long) lag.getObject2());\n    }\n\n    @Test\n    public void testEstimateLag() throws Exception {\n        int msgNoTagSize = 80;\n        int msgWithTagSize = 20;\n        int repeat = 2;\n        String tag = \"TAG_FOR_TEST_ESTIMATE\";\n        String sql = \"TAGS = 'TAG_FOR_TEST_ESTIMATE' And value < \" + repeat / 2;\n        MessageSelector selector = MessageSelector.bySql(sql);\n        RMQBlockListener sqlListener = new RMQBlockListener(true);\n        RMQSqlConsumer sqlConsumer = ConsumerFactory.getRMQSqlConsumer(NAMESRV_ADDR, initConsumerGroup(), topic, selector, sqlListener);\n        RMQBlockListener tagListener = new RMQBlockListener(true);\n        RMQNormalConsumer tagConsumer = getConsumer(NAMESRV_ADDR, topic, tag, tagListener);\n\n        //init subscriptionData & consumerFilterData for sql\n        SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(topic, sql, ExpressionType.SQL92);\n        for (BrokerController controller : brokerControllerList) {\n            controller.getConsumerFilterManager().register(topic, sqlConsumer.getConsumerGroup(), sql, ExpressionType.SQL92, subscriptionData.getSubVersion());\n        }\n\n        // wait for building filter data\n        await().atMost(5, TimeUnit.SECONDS).until(() -> sqlListener.isBlocked() && tagListener.isBlocked());\n\n        List<MessageQueue> mqs = producer.getMessageQueue();\n        for (int i = 0; i < repeat; i++) {\n            MessageQueueMsg mqMsgs = new MessageQueueMsg(mqs, msgNoTagSize);\n            Map<MessageQueue, List<Object>> msgMap = mqMsgs.getMsgsWithMQ();\n            mqMsgs = new MessageQueueMsg(mqs, msgWithTagSize, tag);\n            Map<MessageQueue, List<Object>> msgWithTagMap = mqMsgs.getMsgsWithMQ();\n            int finalI = i;\n            msgMap.forEach((mq, msgList) -> {\n                List<Object> msgWithTagList = msgWithTagMap.get(mq);\n                for (Object o : msgWithTagList) {\n                    ((Message) o).putUserProperty(\"value\", String.valueOf(finalI));\n                }\n                msgList.addAll(msgWithTagList);\n                Collections.shuffle(msgList);\n            });\n            producer.send(msgMap);\n        }\n\n        // test lag estimation for tag consumer\n        for (BrokerController controller : brokerControllerList) {\n            for (MessageQueue mq : mqs) {\n                if (mq.getBrokerName().equals(controller.getBrokerConfig().getBrokerName())) {\n                    long brokerOffset = controller.getMessageStore().getMaxOffsetInQueue(topic, mq.getQueueId());\n                    long estimateMessageCount = controller.getMessageStore()\n                        .estimateMessageCount(topic, mq.getQueueId(), 0, brokerOffset,\n                            new DefaultMessageFilter(FilterAPI.buildSubscriptionData(topic, tag)));\n                    assertEquals(repeat * msgWithTagSize, estimateMessageCount);\n                }\n            }\n        }\n\n        // test lag estimation for sql consumer\n        for (BrokerController controller : brokerControllerList) {\n            for (MessageQueue mq : mqs) {\n                if (mq.getBrokerName().equals(controller.getBrokerConfig().getBrokerName())) {\n                    long brokerOffset = controller.getMessageStore().getMaxOffsetInQueue(topic, mq.getQueueId());\n                    ConsumerFilterData consumerFilterData = controller.getConsumerFilterManager().get(topic, sqlConsumer.getConsumerGroup());\n                    long estimateMessageCount = controller.getMessageStore()\n                        .estimateMessageCount(topic, mq.getQueueId(), 0, brokerOffset,\n                            new ExpressionMessageFilter(subscriptionData, consumerFilterData, controller.getConsumerFilterManager()));\n                    assertEquals(repeat / 2 * msgWithTagSize, estimateMessageCount);\n                }\n            }\n        }\n\n        sqlConsumer.shutdown();\n        tagConsumer.shutdown();\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/offset/OffsetNotFoundIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.offset;\n\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.remoting.RPCHook;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class OffsetNotFoundIT extends BaseConf {\n\n    private OffsetRpcHook offsetRpcHook = new OffsetRpcHook();\n\n    static class OffsetRpcHook implements RPCHook {\n\n        private boolean throwException = false;\n\n        private boolean addSetZeroOfNotFound = false;\n\n        @Override\n        public void doBeforeRequest(String remoteAddr, RemotingCommand request) {\n\n            if (request.getCode() == RequestCode.QUERY_CONSUMER_OFFSET) {\n                if (throwException) {\n                    throw new RuntimeException(\"Stop by rpc hook\");\n                }\n                if (addSetZeroOfNotFound) {\n                    request.getExtFields().put(\"setZeroIfNotFound\", \"false\");\n                }\n            }\n        }\n\n        @Override\n        public void doAfterResponse(String remoteAddr, RemotingCommand request,\n            RemotingCommand response) {\n\n        }\n    }\n\n    @Before\n    public void setUp() {\n        for (BrokerController brokerController: brokerControllerList) {\n            brokerController.registerServerRPCHook(offsetRpcHook);\n        }\n\n\n    }\n\n    @After\n    public void tearDown() {\n        super.shutdown();\n    }\n\n    @Test\n    public void testConsumeStopAndResume() {\n        String topic = initTopic();\n        RMQNormalProducer producer = getProducer(NAMESRV_ADDR, topic);\n        int msgSize = 10;\n        producer.send(msgSize);\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize, producer.getAllUndupMsgBody().size());\n        try {\n            offsetRpcHook.throwException = true;\n            RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n            consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), 15000);\n            Assert.assertEquals(0, consumer.getListener().getAllMsgBody().size());\n            consumer.shutdown();\n        } finally {\n            offsetRpcHook.throwException = false;\n        }\n        //test the normal\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), 15000);\n        Assert.assertEquals(producer.getAllMsgBody().size(), consumer.getListener().getAllMsgBody().size());\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n        consumer.shutdown();\n    }\n\n\n    @Test\n    public void testOffsetNotFoundException() {\n        String topic = initTopic();\n        String group = initConsumerGroup();\n        RMQNormalProducer producer = getProducer(NAMESRV_ADDR, topic);\n        int msgSize = 10;\n        producer.send(msgSize);\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize, producer.getAllUndupMsgBody().size());\n        try {\n            offsetRpcHook.addSetZeroOfNotFound = true;\n            //test the normal\n            RMQNormalConsumer consumer = new RMQNormalConsumer(NAMESRV_ADDR, topic, \"*\", group, new RMQNormalListener());\n            consumer.create(false);\n            consumer.getConsumer().setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);\n            consumer.start();\n            consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), 15000);\n            Assert.assertEquals(producer.getAllMsgBody().size(), consumer.getListener().getAllMsgBody().size());\n            assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n                consumer.getListener().getAllMsgBody()))\n                .containsExactlyElementsIn(producer.getAllMsgBody());\n            consumer.shutdown();\n        } finally {\n            offsetRpcHook.addSetZeroOfNotFound = false;\n        }\n\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/offset/OffsetResetForPopIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.offset;\n\nimport com.google.common.collect.Lists;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.admin.ConsumeStats;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.client.rmq.RMQPopConsumer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.MQAdminTestUtils;\nimport org.apache.rocketmq.test.util.MQRandomUtils;\nimport org.apache.rocketmq.tools.admin.DefaultMQAdminExt;\nimport org.apache.rocketmq.tools.command.CommandUtil;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.awaitility.Awaitility.await;\n\npublic class OffsetResetForPopIT extends BaseConf {\n\n    private static final Logger LOGGER = LoggerFactory.getLogger(OffsetResetForPopIT.class);\n\n    private String topic;\n    private String group;\n    private RMQNormalProducer producer = null;\n    private RMQPopConsumer consumer = null;\n    private DefaultMQAdminExt adminExt;\n\n    @Before\n    public void setUp() throws Exception {\n        // reset pop offset rely on server side offset\n        brokerController1.getBrokerConfig().setUseServerSideResetOffset(true);\n        brokerController1.getBrokerConfig().setPopConsumerKVServiceEnable(false); // force disable before fifo resetOffset issue fixed\n\n        adminExt = BaseConf.getAdmin(NAMESRV_ADDR);\n        adminExt.start();\n\n        topic = MQRandomUtils.getRandomTopic();\n        this.createAndWaitTopicRegister(BROKER1_NAME, topic);\n        group = initConsumerGroup();\n        LOGGER.info(String.format(\"use topic: %s, group: %s\", topic, group));\n        producer = getProducer(NAMESRV_ADDR, topic);\n    }\n\n    @After\n    public void tearDown() {\n        shutdown();\n    }\n\n    private void createAndWaitTopicRegister(String brokerName, String topic) throws Exception {\n        String brokerAddress = CommandUtil.fetchMasterAddrByBrokerName(adminExt, brokerName);\n        TopicConfig topicConfig = new TopicConfig(topic);\n        topicConfig.setReadQueueNums(1);\n        topicConfig.setWriteQueueNums(1);\n        adminExt.createAndUpdateTopicConfig(brokerAddress, topicConfig);\n\n        await().atMost(30, TimeUnit.SECONDS).until(\n            () -> MQAdminTestUtils.checkTopicExist(adminExt, topic));\n    }\n\n    private void resetOffsetInner(long resetOffset) {\n        try {\n            // reset offset by queue\n            adminExt.resetOffsetByQueueId(brokerController1.getBrokerAddr(),\n                consumer.getConsumerGroup(), consumer.getTopic(), 0, resetOffset);\n        } catch (Exception ignore) {\n        }\n    }\n\n    private void ackMessageSync(MessageExt messageExt) {\n        try {\n            consumer.ackAsync(brokerController1.getBrokerAddr(),\n                messageExt.getProperty(MessageConst.PROPERTY_POP_CK)).get();\n        } catch (Exception e) {\n            e.printStackTrace();\n        }\n    }\n\n    private void ackMessageSync(List<MessageExt> messageExtList) {\n        if (messageExtList != null) {\n            messageExtList.forEach(this::ackMessageSync);\n        }\n    }\n\n    @Test\n    public void testResetOffsetAfterPop() throws Exception {\n        int messageCount = 10;\n        int resetOffset = 4;\n        producer.send(messageCount);\n        consumer = new RMQPopConsumer(NAMESRV_ADDR, topic, \"*\", group, new RMQNormalListener());\n        consumer.start();\n\n        MessageQueue mq = new MessageQueue(topic, BROKER1_NAME, 0);\n        PopResult popResult = consumer.pop(brokerController1.getBrokerAddr(), mq);\n        Assert.assertEquals(10, popResult.getMsgFoundList().size());\n\n        resetOffsetInner(resetOffset);\n        popResult = consumer.pop(brokerController1.getBrokerAddr(), mq);\n        Assert.assertTrue(popResult != null && popResult.getMsgFoundList() != null);\n        Assert.assertEquals(messageCount - resetOffset, popResult.getMsgFoundList().size());\n    }\n\n    @Test\n    public void testResetOffsetThenAckOldForPopOrderly() throws Exception {\n        int messageCount = 10;\n        int resetOffset = 2;\n        producer.send(messageCount);\n        consumer = new RMQPopConsumer(NAMESRV_ADDR, topic, \"*\", group, new RMQNormalListener());\n        consumer.start();\n\n        MessageQueue mq = new MessageQueue(topic, BROKER1_NAME, 0);\n        PopResult popResult1 = consumer.popOrderly(brokerController1.getBrokerAddr(), mq);\n        Assert.assertEquals(10, popResult1.getMsgFoundList().size());\n\n        resetOffsetInner(resetOffset);\n        ConsumeStats consumeStats = adminExt.examineConsumeStats(group, topic);\n        Assert.assertEquals(resetOffset, consumeStats.getOffsetTable().get(mq).getConsumerOffset());\n\n        PopResult popResult2 = consumer.popOrderly(brokerController1.getBrokerAddr(), mq);\n        Assert.assertTrue(popResult2 != null && popResult2.getMsgFoundList() != null);\n        Assert.assertEquals(messageCount - resetOffset, popResult2.getMsgFoundList().size());\n\n        // ack old msg, expect has no effect\n        ackMessageSync(popResult1.getMsgFoundList());\n        Assert.assertTrue(brokerController1.getConsumerOrderInfoManager()\n            .checkBlock(null, topic, group, 0, RMQPopConsumer.DEFAULT_INVISIBLE_TIME));\n\n        // ack new msg\n        ackMessageSync(popResult2.getMsgFoundList());\n        Assert.assertFalse(brokerController1.getConsumerOrderInfoManager()\n            .checkBlock(null, topic, group, 0, RMQPopConsumer.DEFAULT_INVISIBLE_TIME));\n    }\n\n    @Test\n    public void testRestOffsetToSkipMsgForPopOrderly() throws Exception {\n        int messageCount = 10;\n        int resetOffset = 4;\n        producer.send(messageCount);\n        consumer = new RMQPopConsumer(NAMESRV_ADDR, topic, \"*\", group, new RMQNormalListener());\n        resetOffsetInner(resetOffset);\n        consumer.start();\n\n        MessageQueue mq = new MessageQueue(topic, BROKER1_NAME, 0);\n        PopResult popResult = consumer.popOrderly(brokerController1.getBrokerAddr(), mq);\n        Assert.assertEquals(messageCount - resetOffset, popResult.getMsgFoundList().size());\n        Assert.assertTrue(brokerController1.getConsumerOrderInfoManager()\n            .checkBlock(null, topic, group, 0, RMQPopConsumer.DEFAULT_INVISIBLE_TIME));\n\n        ackMessageSync(popResult.getMsgFoundList());\n        TimeUnit.SECONDS.sleep(1);\n        Assert.assertFalse(brokerController1.getConsumerOrderInfoManager()\n            .checkBlock(null, topic, group, 0, RMQPopConsumer.DEFAULT_INVISIBLE_TIME));\n    }\n\n    @Test\n    public void testResetOffsetAfterPopWhenOpenBufferAndWait() throws Exception {\n        int messageCount = 10;\n        int resetOffset = 4;\n        brokerController1.getBrokerConfig().setEnablePopBufferMerge(true);\n        producer.send(messageCount);\n        consumer = new RMQPopConsumer(NAMESRV_ADDR, topic, \"*\", group, new RMQNormalListener());\n        consumer.start();\n\n        MessageQueue mq = new MessageQueue(topic, BROKER1_NAME, 0);\n        PopResult popResult = consumer.pop(brokerController1.getBrokerAddr(), mq);\n        Assert.assertEquals(10, popResult.getMsgFoundList().size());\n\n        resetOffsetInner(resetOffset);\n        TimeUnit.MILLISECONDS.sleep(brokerController1.getBrokerConfig().getPopCkStayBufferTimeOut());\n\n        popResult = consumer.pop(brokerController1.getBrokerAddr(), mq);\n        Assert.assertTrue(popResult != null && popResult.getMsgFoundList() != null);\n        Assert.assertEquals(messageCount - resetOffset, popResult.getMsgFoundList().size());\n    }\n\n    @Test\n    public void testResetOffsetWhilePopWhenOpenBuffer() {\n        testResetOffsetWhilePop(8, false, false, 5);\n    }\n\n    @Test\n    public void testResetOffsetWhilePopWhenOpenBufferAndAck() {\n        testResetOffsetWhilePop(8, false, true, 5);\n    }\n\n    @Test\n    public void testMultipleResetOffsetWhilePopWhenOpenBufferAndAck() {\n        testResetOffsetWhilePop(8, false, true, 3, 5);\n    }\n\n    @Test\n    public void testResetFutureOffsetWhilePopWhenOpenBufferAndAck() {\n        testResetOffsetWhilePop(2, true, true, 8);\n    }\n\n    @Test\n    public void testMultipleResetFutureOffsetWhilePopWhenOpenBufferAndAck() {\n        testResetOffsetWhilePop(2, true, true, 5, 8);\n    }\n\n    private void testResetOffsetWhilePop(int targetCount, boolean resetFuture, boolean needAck,\n        int... resetOffset) {\n        brokerController1.getBrokerConfig().setEnablePopBufferMerge(true);\n        producer.send(10);\n\n        // max pop one message per request\n        consumer =\n            new RMQPopConsumer(NAMESRV_ADDR, topic, \"*\", group, new RMQNormalListener(), 1);\n\n        MessageQueue mq = new MessageQueue(topic, BROKER1_NAME, 0);\n        AtomicInteger counter = new AtomicInteger(0);\n        consumer.start();\n        Executors.newSingleThreadScheduledExecutor().execute(() -> {\n            long start = System.currentTimeMillis();\n            while (System.currentTimeMillis() - start <= 30 * 1000L) {\n                try {\n                    PopResult popResult = consumer.pop(brokerController1.getBrokerAddr(), mq);\n                    if (popResult == null || popResult.getMsgFoundList() == null) {\n                        continue;\n                    }\n\n                    int count = counter.addAndGet(popResult.getMsgFoundList().size());\n                    if (needAck) {\n                        ackMessageSync(popResult.getMsgFoundList());\n                    }\n                    if (count == targetCount) {\n                        for (int offset : resetOffset) {\n                            resetOffsetInner(offset);\n                        }\n                    }\n                } catch (Exception e) {\n                    e.printStackTrace();\n                }\n            }\n        });\n\n        await().atMost(10, TimeUnit.SECONDS).until(() -> {\n            boolean result = true;\n            if (resetFuture) {\n                result = counter.get() < 10;\n            }\n            result &= counter.get() >= targetCount + 10 - resetOffset[resetOffset.length - 1];\n            return result;\n        });\n    }\n\n    @Test\n    public void testResetFutureOffsetWhilePopOrderlyAndAck() {\n        testResetOffsetWhilePopOrderly(1,\n            Lists.newArrayList(0, 5, 6, 7, 8, 9), Lists.newArrayList(5), 6);\n    }\n\n    @Test\n    public void testMultipleResetFutureOffsetWhilePopOrderlyAndAck() {\n        testResetOffsetWhilePopOrderly(1,\n            Lists.newArrayList(0, 5, 6, 7, 8, 9), Lists.newArrayList(3, 5), 6);\n    }\n\n    @Test\n    public void testResetOffsetWhilePopOrderlyAndAck() {\n        testResetOffsetWhilePopOrderly(5,\n            Lists.newArrayList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9),\n            Lists.newArrayList(3), 12);\n    }\n\n    @Test\n    public void testMultipleResetOffsetWhilePopOrderlyAndAck() {\n        testResetOffsetWhilePopOrderly(5,\n            Lists.newArrayList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9),\n            Lists.newArrayList(3, 1), 14);\n    }\n\n    private void testResetOffsetWhilePopOrderly(int targetCount, List<Integer> expectMsgReceive,\n        List<Integer> resetOffset, int expectCount) {\n        brokerController1.getBrokerConfig().setEnablePopBufferMerge(true);\n        for (int i = 0; i < 10; i++) {\n            Message msg = new Message(topic, (String.valueOf(i)).getBytes());\n            producer.send(msg);\n        }\n        consumer = new RMQPopConsumer(NAMESRV_ADDR, topic, \"*\", group, new RMQNormalListener(), 1);\n        MessageQueue mq = new MessageQueue(topic, BROKER1_NAME, 0);\n        Set<Integer> msgReceive = Collections.newSetFromMap(new ConcurrentHashMap<>());\n        AtomicInteger counter = new AtomicInteger(0);\n        consumer.start();\n\n        Executors.newSingleThreadScheduledExecutor().execute(() -> {\n            long start = System.currentTimeMillis();\n            while (System.currentTimeMillis() - start <= 30 * 1000L) {\n                try {\n                    PopResult popResult = consumer.popOrderly(brokerController1.getBrokerAddr(), mq);\n                    if (popResult == null || popResult.getMsgFoundList() == null) {\n                        continue;\n                    }\n                    int count = counter.addAndGet(popResult.getMsgFoundList().size());\n                    for (MessageExt messageExt : popResult.getMsgFoundList()) {\n                        msgReceive.add(Integer.valueOf(new String(messageExt.getBody())));\n                        ackMessageSync(messageExt);\n                    }\n                    if (count == targetCount) {\n                        for (int offset : resetOffset) {\n                            resetOffsetInner(offset);\n                        }\n                    }\n                } catch (Exception e) {\n                    // do nothing;\n                }\n            }\n        });\n\n        await().atMost(10, TimeUnit.SECONDS).until(() -> {\n            boolean result = true;\n            if (expectMsgReceive.size() != msgReceive.size()) {\n                return false;\n            }\n            if (counter.get() != expectCount) {\n                return false;\n            }\n            for (Integer expectMsg : expectMsgReceive) {\n                result &= msgReceive.contains(expectMsg);\n            }\n            return result;\n        });\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/offset/OffsetResetIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.offset;\n\nimport java.time.Duration;\nimport java.util.List;\nimport java.util.Map;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.RemotingCommand;\nimport org.apache.rocketmq.remoting.protocol.RequestCode;\nimport org.apache.rocketmq.remoting.protocol.admin.ConsumeStats;\nimport org.apache.rocketmq.remoting.protocol.admin.OffsetWrapper;\nimport org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.message.MessageQueueMsg;\nimport org.apache.rocketmq.tools.admin.DefaultMQAdminExt;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.FixMethodOrder;\nimport org.junit.Test;\nimport org.junit.runners.MethodSorters;\n\nimport static org.awaitility.Awaitility.await;\n\n@FixMethodOrder(MethodSorters.NAME_ASCENDING)\npublic class OffsetResetIT extends BaseConf {\n\n    private static final Logger LOGGER = LoggerFactory.getLogger(OffsetResetIT.class);\n\n    private RMQNormalListener listener = null;\n    private RMQNormalProducer producer = null;\n    private RMQNormalConsumer consumer = null;\n    private DefaultMQAdminExt defaultMQAdminExt = null;\n    private String topic = null;\n\n    @Before\n    public void init() throws MQClientException {\n        topic = initTopic();\n        LOGGER.info(String.format(\"use topic: %s;\", topic));\n\n        for (BrokerController controller : brokerControllerList) {\n            controller.getBrokerConfig().setLongPollingEnable(false);\n            controller.getBrokerConfig().setShortPollingTimeMills(500);\n            controller.getBrokerConfig().setUseServerSideResetOffset(true);\n        }\n\n        listener = new RMQNormalListener();\n        producer = getProducer(NAMESRV_ADDR, topic);\n        consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", listener);\n\n        defaultMQAdminExt = BaseConf.getAdmin(NAMESRV_ADDR);\n        defaultMQAdminExt.start();\n    }\n\n    @After\n    public void tearDown() {\n        shutdown();\n    }\n\n    @Test\n    public void testEncodeOffsetHeader() {\n        ResetOffsetRequestHeader requestHeader = new ResetOffsetRequestHeader();\n        requestHeader.setTopic(topic);\n        requestHeader.setGroup(consumer.getConsumerGroup());\n        requestHeader.setTimestamp(System.currentTimeMillis());\n        requestHeader.setForce(false);\n        RemotingCommand.createRequestCommand(RequestCode.INVOKE_BROKER_TO_RESET_OFFSET, requestHeader);\n    }\n\n    /**\n     * use mq admin tool to query remote offset\n     */\n    private long getConsumerLag(String topic, String group) throws Exception {\n        long consumerLag = 0L;\n        for (BrokerController controller : brokerControllerList) {\n            ConsumeStats consumeStats = defaultMQAdminExt.getDefaultMQAdminExtImpl()\n                .getMqClientInstance().getMQClientAPIImpl()\n                .getConsumeStats(controller.getBrokerAddr(), group, topic, 3000);\n            Map<MessageQueue, OffsetWrapper> offsetTable = consumeStats.getOffsetTable();\n\n            for (Map.Entry<MessageQueue, OffsetWrapper> entry : offsetTable.entrySet()) {\n                MessageQueue messageQueue = entry.getKey();\n                OffsetWrapper offsetWrapper = entry.getValue();\n\n                Assert.assertEquals(messageQueue.getBrokerName(), controller.getBrokerConfig().getBrokerName());\n                long brokerOffset = controller.getMessageStore().getMaxOffsetInQueue(topic, messageQueue.getQueueId());\n                long consumerOffset = controller.getConsumerOffsetManager().queryOffset(\n                    consumer.getConsumerGroup(), topic, messageQueue.getQueueId());\n                Assert.assertEquals(brokerOffset, offsetWrapper.getBrokerOffset());\n                Assert.assertEquals(consumerOffset, offsetWrapper.getConsumerOffset());\n\n                consumerLag += brokerOffset - consumerOffset;\n            }\n        }\n        return consumerLag;\n    }\n\n    @Test\n    public void testResetOffsetSingleQueue() throws Exception {\n        int msgSize = 100;\n        List<MessageQueue> mqs = producer.getMessageQueue();\n        MessageQueueMsg messageQueueMsg = new MessageQueueMsg(mqs, msgSize);\n\n        producer.send(messageQueueMsg.getMsgsWithMQ());\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        await().pollInterval(Duration.ofSeconds(1)).atMost(Duration.ofMinutes(3)).until(\n            () -> 0L == this.getConsumerLag(topic, consumer.getConsumerGroup()));\n\n        for (BrokerController controller : brokerControllerList) {\n            defaultMQAdminExt.resetOffsetByQueueId(controller.getBrokerAddr(),\n                consumer.getConsumerGroup(), consumer.getTopic(), 3, 0);\n        }\n\n        int hasConsumeBefore = listener.getMsgIndex().get();\n        int expectAfterReset = brokerControllerList.size() * msgSize;\n        await().pollInterval(Duration.ofSeconds(1)).atMost(Duration.ofMinutes(3)).until(() -> {\n            long receive = listener.getMsgIndex().get();\n            long expect = hasConsumeBefore + expectAfterReset;\n            return receive >= expect;\n        });\n    }\n\n    @Test\n    public void testResetOffsetTotal() throws Exception {\n        int msgSize = 100;\n        long start = System.currentTimeMillis();\n        List<MessageQueue> mqs = producer.getMessageQueue();\n        MessageQueueMsg messageQueueMsg = new MessageQueueMsg(mqs, msgSize);\n\n        producer.send(messageQueueMsg.getMsgsWithMQ());\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        await().pollInterval(Duration.ofSeconds(1)).atMost(Duration.ofMinutes(3)).until(\n            () -> 0L == this.getConsumerLag(topic, consumer.getConsumerGroup()));\n\n        for (BrokerController controller : brokerControllerList) {\n            defaultMQAdminExt.getDefaultMQAdminExtImpl().getMqClientInstance().getMQClientAPIImpl()\n                .invokeBrokerToResetOffset(controller.getBrokerAddr(),\n                    consumer.getTopic(), consumer.getConsumerGroup(), start, true, 3 * 1000);\n        }\n\n        int hasConsumeBefore = listener.getMsgIndex().get();\n        int expectAfterReset = mqs.size() * msgSize;\n        await().pollInterval(Duration.ofSeconds(1)).atMost(Duration.ofMinutes(3)).until(() -> {\n            long receive = listener.getMsgIndex().get();\n            long expect = hasConsumeBefore + expectAfterReset;\n            return receive >= expect;\n        });\n    }\n\n    @Test\n    public void testPullOffsetTotal() throws Exception {\n        int msgSize = 100;\n        List<MessageQueue> mqs = producer.getMessageQueue();\n        MessageQueueMsg messageQueueMsg = new MessageQueueMsg(mqs, msgSize);\n\n        producer.send(messageQueueMsg.getMsgsWithMQ());\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n\n        await().pollInterval(Duration.ofSeconds(1)).atMost(Duration.ofMinutes(3)).until(\n            () -> 0L == this.getConsumerLag(topic, consumer.getConsumerGroup()));\n\n        long expectInflight = 0L;\n        for (BrokerController controller : brokerControllerList) {\n            ConsumeStats consumeStats = defaultMQAdminExt.getDefaultMQAdminExtImpl().getMqClientInstance()\n                .getMQClientAPIImpl().getConsumeStats(controller.getBrokerAddr(),\n                    consumer.getConsumerGroup(), consumer.getTopic(), 3 * 1000);\n            expectInflight += consumeStats.computeInflightTotalDiff();\n        }\n        Assert.assertEquals(0L, expectInflight);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/recall/RecallWithTraceIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.recall;\n\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PopStatus;\nimport org.apache.rocketmq.client.exception.MQBrokerException;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.trace.TraceContext;\nimport org.apache.rocketmq.client.trace.TraceDataEncoder;\nimport org.apache.rocketmq.client.trace.TraceType;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.message.MessageClientIDSetter;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.producer.RecallMessageHandle;\nimport org.apache.rocketmq.remoting.exception.RemotingException;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.base.IntegrationTestBase;\nimport org.apache.rocketmq.test.client.rmq.RMQPopConsumer;\nimport org.apache.rocketmq.test.factory.ConsumerFactory;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.MQRandomUtils;\nimport org.junit.AfterClass;\nimport org.junit.Assert;\nimport org.junit.BeforeClass;\nimport org.junit.Test;\n\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicReference;\n\nimport static org.awaitility.Awaitility.await;\n\npublic class RecallWithTraceIT extends BaseConf {\n    private static String topic;\n    private static String group;\n    private static DefaultMQProducer producer;\n    private static RMQPopConsumer popConsumer;\n\n    @BeforeClass\n    public static void init() throws MQClientException {\n        System.setProperty(\"com.rocketmq.recall.default.trace.enable\", Boolean.TRUE.toString());\n        topic = MQRandomUtils.getRandomTopic();\n        IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, 1, CQType.SimpleCQ, TopicMessageType.NORMAL);\n        group = initConsumerGroup();\n        producer = new DefaultMQProducer(group, true, topic);\n        producer.setNamesrvAddr(NAMESRV_ADDR);\n        producer.start();\n        popConsumer = ConsumerFactory.getRMQPopConsumer(NAMESRV_ADDR, group, topic, \"*\", new RMQNormalListener());\n        mqClients.add(popConsumer);\n        mqClients.add(producer);\n    }\n\n    @AfterClass\n    public static void tearDown() {\n        shutdown();\n    }\n\n    @Test\n    public void testRecallTrace() throws MQBrokerException, RemotingException, InterruptedException, MQClientException {\n        String msgId = MessageClientIDSetter.createUniqID();\n        String recallHandle = RecallMessageHandle.HandleV1.buildHandle(topic, BROKER1_NAME,\n            String.valueOf(System.currentTimeMillis() + 30000), msgId);\n        producer.recallMessage(topic, recallHandle);\n\n        MessageQueue messageQueue = new MessageQueue(topic, BROKER1_NAME, 0);\n        String brokerAddress = brokerController1.getBrokerAddr();\n        AtomicReference<MessageExt> traceMessage = new AtomicReference();\n        await()\n            .pollInterval(1, TimeUnit.SECONDS)\n            .atMost(15, TimeUnit.SECONDS)\n            .until(() -> {\n                PopResult popResult = popConsumer.pop(brokerAddress, messageQueue, 60 * 1000, -1);\n                boolean found = popResult.getPopStatus().equals(PopStatus.FOUND);\n                traceMessage.set(found ? popResult.getMsgFoundList().get(0) : null);\n                return found;\n            });\n\n        Assert.assertNotNull(traceMessage.get());\n        TraceContext context =\n            TraceDataEncoder.decoderFromTraceDataString(new String(traceMessage.get().getBody())).get(0);\n        Assert.assertEquals(TraceType.Recall, context.getTraceType());\n        Assert.assertEquals(group, context.getGroupName());\n        Assert.assertTrue(context.isSuccess());\n        Assert.assertEquals(msgId, context.getTraceBeans().get(0).getMsgId());\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/recall/SendAndRecallDelayMessageIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.recall;\n\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertEquals;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.client.consumer.PopResult;\nimport org.apache.rocketmq.client.consumer.PopStatus;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.attribute.TopicMessageType;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.producer.RecallMessageHandle;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.base.IntegrationTestBase;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.client.rmq.RMQPopConsumer;\nimport org.apache.rocketmq.test.factory.ConsumerFactory;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.MQRandomUtils;\nimport org.junit.AfterClass;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\nimport org.junit.runners.Parameterized;\n\n@RunWith(Parameterized.class)\npublic class SendAndRecallDelayMessageIT extends BaseConf {\n\n    private static String initTopic;\n    private static String consumerGroup;\n    private static RMQNormalProducer producer;\n    private static RMQPopConsumer popConsumer;\n\n    private final boolean appendTopicForTimerDeleteKey;\n\n    public SendAndRecallDelayMessageIT(boolean appendTopicForTimerDeleteKey) {\n        this.appendTopicForTimerDeleteKey = appendTopicForTimerDeleteKey;\n    }\n\n    @Parameterized.Parameters\n    public static List<Object[]> params() {\n        List<Object[]> result = new ArrayList<>();\n        result.add(new Object[] {false});\n        result.add(new Object[] {true});\n        return result;\n    }\n\n    @Before\n    public void init() {\n        brokerController1.getMessageStoreConfig().setAppendTopicForTimerDeleteKey(appendTopicForTimerDeleteKey);\n        initTopic = initTopic();\n        consumerGroup = initConsumerGroup();\n        producer = getProducer(NAMESRV_ADDR, initTopic);\n        popConsumer = ConsumerFactory.getRMQPopConsumer(NAMESRV_ADDR, consumerGroup, initTopic, \"*\", new RMQNormalListener());\n        mqClients.add(popConsumer);\n    }\n\n    @AfterClass\n    public static void tearDown() {\n        shutdown();\n    }\n\n    @Test\n    public void testSendAndRecv() throws Exception {\n        int delaySecond = 1;\n        String topic = MQRandomUtils.getRandomTopic();\n        IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, 1, CQType.SimpleCQ, TopicMessageType.DELAY);\n        MessageQueue messageQueue = new MessageQueue(topic, BROKER1_NAME, 0);\n        String brokerAddress = brokerController1.getBrokerAddr();\n\n        List<Message> sendList = buildSendMessageList(topic, delaySecond);\n        List<Message> recvList = new ArrayList<>();\n\n        for (Message message : sendList) {\n            producer.getProducer().send(message);\n        }\n\n        await()\n            .pollInterval(1, TimeUnit.SECONDS)\n            .atMost(delaySecond + 15, TimeUnit.SECONDS)\n            .until(() -> {\n                PopResult popResult = popConsumer.pop(brokerAddress, messageQueue, 60 * 1000, -1);\n                processPopResult(recvList, popResult);\n                return recvList.size() == sendList.size();\n            });\n    }\n\n    @Test\n    public void testSendAndRecall() throws Exception {\n        int delaySecond = 5;\n        String topic = MQRandomUtils.getRandomTopic();\n        IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, 1, CQType.SimpleCQ, TopicMessageType.DELAY);\n        MessageQueue messageQueue = new MessageQueue(topic, BROKER1_NAME, 0);\n        String brokerAddress = brokerController1.getBrokerAddr();\n\n        List<Message> sendList = buildSendMessageList(topic, delaySecond);\n        List<Message> recvList = new ArrayList<>();\n        int recallCount = 0;\n\n        for (Message message : sendList) {\n            SendResult sendResult = producer.getProducer().send(message);\n            if (sendResult.getRecallHandle() != null) {\n                String messageId = producer.getProducer().recallMessage(topic, sendResult.getRecallHandle());\n                assertEquals(sendResult.getMsgId(), messageId);\n                recallCount += 1;\n            }\n        }\n        assertEquals(sendList.size() - 2, recallCount); // one normal and one delay-level message\n        try {\n            await()\n                .pollInterval(1, TimeUnit.SECONDS)\n                .atMost(delaySecond + 15, TimeUnit.SECONDS)\n                .until(() -> {\n                    PopResult popResult = popConsumer.pop(brokerAddress, messageQueue, 60 * 1000, -1);\n                    processPopResult(recvList, popResult);\n                    return recvList.size() == sendList.size();\n                });\n        } catch (Exception e) {\n        }\n        assertEquals(sendList.size() - recallCount, recvList.size());\n    }\n\n    @Test\n    public void testSendAndRecall_ukCollision() throws Exception {\n        if (!appendTopicForTimerDeleteKey) { // skip\n            return;\n        }\n        int delaySecond = 5;\n        String topic = MQRandomUtils.getRandomTopic();\n        String collisionTopic = MQRandomUtils.getRandomTopic();\n        IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, 1, CQType.SimpleCQ, TopicMessageType.DELAY);\n        IntegrationTestBase.initTopic(collisionTopic, NAMESRV_ADDR, BROKER1_NAME, 1, CQType.SimpleCQ, TopicMessageType.DELAY);\n        MessageQueue messageQueue = new MessageQueue(topic, BROKER1_NAME, 0);\n        String brokerAddress = brokerController1.getBrokerAddr();\n\n        List<Message> sendList = buildSendMessageList(topic, delaySecond);\n        List<Message> recvList = new ArrayList<>();\n        int recallCount = 0;\n\n        for (Message message : sendList) {\n            SendResult sendResult = producer.getProducer().send(message);\n            if (sendResult.getRecallHandle() != null) {\n                RecallMessageHandle.HandleV1 handleEntity =\n                    (RecallMessageHandle.HandleV1) RecallMessageHandle.decodeHandle(sendResult.getRecallHandle());\n                String collisionHandle = RecallMessageHandle.HandleV1.buildHandle(collisionTopic,\n                    handleEntity.getBrokerName(), handleEntity.getTimestampStr(), handleEntity.getMessageId());\n                String messageId = producer.getProducer().recallMessage(collisionTopic, collisionHandle);\n                assertEquals(sendResult.getMsgId(), messageId);\n                recallCount += 1;\n            }\n        }\n        assertEquals(sendList.size() - 2, recallCount); // one normal and one delay-level message\n\n        try {\n            await()\n                .pollInterval(1, TimeUnit.SECONDS)\n                .atMost(delaySecond + 15, TimeUnit.SECONDS)\n                .until(() -> {\n                    PopResult popResult = popConsumer.pop(brokerAddress, messageQueue, 60 * 1000, -1);\n                    processPopResult(recvList, popResult);\n                    return recvList.size() == sendList.size();\n                });\n        } catch (Exception e) {\n        }\n        assertEquals(sendList.size(), recvList.size());\n    }\n\n    private void processPopResult(List<Message> recvList, PopResult popResult) {\n        if (popResult.getPopStatus() == PopStatus.FOUND && popResult.getMsgFoundList() != null) {\n            recvList.addAll(popResult.getMsgFoundList());\n        }\n    }\n\n    private List<Message> buildSendMessageList(String topic, int delaySecond) {\n        Message msg0 = new Message(topic, \"tag\", \"Hello RocketMQ\".getBytes()); // not supported\n\n        Message msg1 = new Message(topic, \"tag\", \"Hello RocketMQ\".getBytes()); // not supported\n        msg1.setDelayTimeLevel(2);\n\n        Message msg2 = new Message(topic, \"tag\", \"Hello RocketMQ\".getBytes());\n        msg2.setDelayTimeMs(delaySecond * 1000L);\n\n        Message msg3 = new Message(topic, \"tag\", \"Hello RocketMQ\".getBytes());\n        msg3.setDelayTimeSec(delaySecond);\n\n        Message msg4 = new Message(topic, \"tag\", \"Hello RocketMQ\".getBytes());\n        msg4.setDeliverTimeMs(System.currentTimeMillis() + delaySecond * 1000L);\n\n        return Arrays.asList(msg0, msg1, msg2, msg3, msg4);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/retry/PopConsumerRetryIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.retry;\n\nimport java.util.Set;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.stream.Collectors;\nimport org.apache.commons.lang3.RandomStringUtils;\nimport org.apache.rocketmq.client.AccessChannel;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;\nimport org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.client.producer.SendResult;\nimport org.apache.rocketmq.client.producer.SendStatus;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.common.attribute.CQType;\nimport org.apache.rocketmq.common.consumer.ConsumeFromWhere;\nimport org.apache.rocketmq.common.message.Message;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageRequestMode;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.common.RemotingHelper;\nimport org.apache.rocketmq.remoting.protocol.body.ClusterInfo;\nimport org.apache.rocketmq.remoting.protocol.route.BrokerData;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.base.IntegrationTestBase;\nimport org.apache.rocketmq.test.offset.OffsetResetIT;\nimport org.apache.rocketmq.tools.admin.DefaultMQAdminExt;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport static org.awaitility.Awaitility.await;\n\npublic class PopConsumerRetryIT extends BaseConf {\n\n    private static final Logger LOGGER = LoggerFactory.getLogger(OffsetResetIT.class);\n\n    private DefaultMQAdminExt defaultMQAdminExt = null;\n    private String topicName = null;\n    private String groupName = null;\n\n    @Before\n    public void init() throws MQClientException {\n        topicName = \"topic-\" + RandomStringUtils.randomAlphabetic(72).toUpperCase();\n        groupName = \"group-\" + RandomStringUtils.randomAlphabetic(72).toUpperCase();\n        LOGGER.info(String.format(\"use topic: %s, group: %s\", topicName, groupName));\n        IntegrationTestBase.initTopic(topicName, NAMESRV_ADDR, CLUSTER_NAME, CQType.SimpleCQ);\n        defaultMQAdminExt = getAdmin(NAMESRV_ADDR);\n        defaultMQAdminExt.start();\n    }\n\n    @After\n    public void tearDown() {\n        shutdown();\n    }\n\n    private void switchPop(String groupName, String topicName) throws Exception {\n        ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo();\n        Set<String> brokerAddrs = clusterInfo.getBrokerAddrTable().values()\n            .stream().map(BrokerData::selectBrokerAddr).collect(Collectors.toSet());\n        for (String brokerAddr : brokerAddrs) {\n            TopicConfig topicConfig = new TopicConfig(topicName, 1, 1, 6);\n            defaultMQAdminExt.createAndUpdateTopicConfig(brokerAddr, topicConfig);\n            defaultMQAdminExt.setMessageRequestMode(brokerAddr, topicName, groupName,\n                MessageRequestMode.POP, 8, 3000L);\n        }\n    }\n\n    @Test\n    public void testNormalMessageUseMessageVersionV2() throws Exception {\n        switchPop(groupName, topicName);\n\n        AtomicInteger successCount = new AtomicInteger();\n        AtomicInteger retryCount = new AtomicInteger();\n\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(groupName);\n        consumer.subscribe(topicName, \"*\");\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);\n        consumer.setConsumeThreadMin(1);\n        consumer.setConsumeThreadMax(1);\n        consumer.setConsumeMessageBatchMaxSize(1);\n        consumer.setNamesrvAddr(NAMESRV_ADDR);\n        consumer.setClientRebalance(false);\n        consumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {\n            for (MessageExt message : msgs) {\n                LOGGER.debug(String.format(\"messageId: %s, times: %d, topic: %s\",\n                    message.getMsgId(), message.getReconsumeTimes(), message.getTopic()));\n                if (message.getReconsumeTimes() < 2) {\n                    retryCount.incrementAndGet();\n                    return ConsumeConcurrentlyStatus.RECONSUME_LATER;\n                } else {\n                    successCount.incrementAndGet();\n                    return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n                }\n            }\n            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;\n        });\n        consumer.start();\n        LOGGER.info(\"Consumer Started...\");\n\n        DefaultMQProducer producer = new DefaultMQProducer(\"PID-1\", false, null);\n        producer.setAccessChannel(AccessChannel.CLOUD);\n        producer.setNamesrvAddr(NAMESRV_ADDR);\n        producer.start();\n        LOGGER.info(\"Producer Started...%n\");\n\n        // wait pop client register\n        TimeUnit.SECONDS.sleep(3);\n\n        int total = 10;\n        for (int i = 0; i < total; i++) {\n            Message msg = new Message(\n                topicName, \"*\", \"Hello world\".getBytes(RemotingHelper.DEFAULT_CHARSET));\n            SendResult sendResult = producer.send(msg);\n            Assert.assertEquals(SendStatus.SEND_OK, sendResult.getSendStatus());\n        }\n\n        await().pollInterval(1, TimeUnit.SECONDS).atMost(90, TimeUnit.SECONDS)\n            .until(() -> {\n                LOGGER.debug(String.format(\"retry: %d, success: %d\", retryCount.get(), successCount.get()));\n                return retryCount.get() == total * 2 && successCount.get() == total;\n            });\n    }\n\n    @Test\n    public void testFIFOMessageUseMessageVersionV2() throws Exception {\n        switchPop(groupName, topicName);\n\n        AtomicInteger successCount = new AtomicInteger();\n        AtomicInteger retryCount = new AtomicInteger();\n\n        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(groupName);\n        consumer.subscribe(topicName, \"*\");\n        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);\n        consumer.setConsumeThreadMin(1);\n        consumer.setConsumeThreadMax(1);\n        consumer.setConsumeMessageBatchMaxSize(1);\n        consumer.setNamesrvAddr(NAMESRV_ADDR);\n        consumer.setClientRebalance(false);\n        consumer.registerMessageListener((MessageListenerOrderly) (msgs, context) -> {\n            for (MessageExt message : msgs) {\n                LOGGER.debug(String.format(\"messageId: %s, times: %d, topic: %s\",\n                    message.getMsgId(), message.getReconsumeTimes(), message.getTopic()));\n                if (message.getReconsumeTimes() < 2) {\n                    retryCount.incrementAndGet();\n                    return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT;\n                } else {\n                    successCount.incrementAndGet();\n                    return ConsumeOrderlyStatus.SUCCESS;\n                }\n            }\n            return ConsumeOrderlyStatus.SUCCESS;\n        });\n        consumer.start();\n        LOGGER.info(\"Consumer Started...\");\n\n        DefaultMQProducer producer = new DefaultMQProducer(\"PID-1\", false, null);\n        producer.setAccessChannel(AccessChannel.CLOUD);\n        producer.setNamesrvAddr(NAMESRV_ADDR);\n        producer.start();\n        LOGGER.info(\"Producer Started...%n\");\n\n        // wait pop client register\n        TimeUnit.SECONDS.sleep(3);\n\n        int total = 10;\n        for (int i = 0; i < total; i++) {\n            Message msg = new Message(\n                topicName, \"*\", \"Hello world\".getBytes(RemotingHelper.DEFAULT_CHARSET));\n            SendResult sendResult = producer.send(msg);\n            Assert.assertEquals(SendStatus.SEND_OK, sendResult.getSendStatus());\n        }\n\n        await().pollInterval(1, TimeUnit.SECONDS).atMost(90, TimeUnit.SECONDS)\n            .until(() -> {\n                LOGGER.debug(String.format(\"retry: %d, success: %d\", retryCount.get(), successCount.get()));\n                return retryCount.get() == total * 2 && successCount.get() == total;\n            });\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.route;\n\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.common.TopicConfig;\nimport org.apache.rocketmq.remoting.protocol.route.TopicRouteData;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.util.MQAdminTestUtils;\nimport org.junit.Ignore;\nimport org.junit.Test;\n\nimport static org.assertj.core.api.Assertions.assertThat;\nimport static org.awaitility.Awaitility.await;\n\npublic class CreateAndUpdateTopicIT extends BaseConf {\n\n    @Test\n    public void testCreateOrUpdateTopic_EnableSingleTopicRegistration() {\n        String topic = \"test-topic-without-broker-registration\";\n        brokerController1.getBrokerConfig().setEnableSingleTopicRegister(true);\n        brokerController2.getBrokerConfig().setEnableSingleTopicRegister(true);\n        brokerController3.getBrokerConfig().setEnableSingleTopicRegister(true);\n\n        final boolean createResult = MQAdminTestUtils.createTopic(NAMESRV_ADDR, CLUSTER_NAME, topic, 8, null);\n        assertThat(createResult).isTrue();\n\n        TopicRouteData route = MQAdminTestUtils.examineTopicRouteInfo(NAMESRV_ADDR, topic);\n        assertThat(route.getBrokerDatas()).hasSize(3);\n        assertThat(route.getQueueDatas()).hasSize(3);\n\n        brokerController1.getBrokerConfig().setEnableSingleTopicRegister(false);\n        brokerController2.getBrokerConfig().setEnableSingleTopicRegister(false);\n        brokerController3.getBrokerConfig().setEnableSingleTopicRegister(false);\n\n    }\n\n    // Temporarily ignore the fact that this test cannot pass in the integration test pipeline due to unknown reasons\n    @Ignore\n    @Test\n    public void testDeleteTopicFromNameSrvWithBrokerRegistration() {\n        namesrvController.getNamesrvConfig().setDeleteTopicWithBrokerRegistration(true);\n        brokerController1.getBrokerConfig().setEnableSingleTopicRegister(true);\n        brokerController2.getBrokerConfig().setEnableSingleTopicRegister(true);\n        brokerController3.getBrokerConfig().setEnableSingleTopicRegister(true);\n\n        String testTopic1 = \"test-topic-keep-route\";\n        String testTopic2 = \"test-topic-delete-route\";\n\n        boolean createResult = MQAdminTestUtils.createTopic(NAMESRV_ADDR, CLUSTER_NAME, testTopic1, 8, null);\n        assertThat(createResult).isTrue();\n\n        createResult = MQAdminTestUtils.createTopic(NAMESRV_ADDR, CLUSTER_NAME, testTopic2, 8, null);\n        assertThat(createResult).isTrue();\n\n        TopicRouteData route = MQAdminTestUtils.examineTopicRouteInfo(NAMESRV_ADDR, testTopic2);\n        assertThat(route.getBrokerDatas()).hasSize(3);\n\n        MQAdminTestUtils.deleteTopicFromBrokerOnly(NAMESRV_ADDR, BROKER1_NAME, testTopic2);\n\n        // Deletion is lazy, trigger broker registration\n        brokerController1.registerBrokerAll(false, false, true);\n\n        await().atMost(10, TimeUnit.SECONDS).until(() -> {\n            // The route info of testTopic2 will be removed from broker1 after the registration\n            TopicRouteData finalRoute = MQAdminTestUtils.examineTopicRouteInfo(NAMESRV_ADDR, testTopic2);\n            return finalRoute.getBrokerDatas().size() == 2\n                && finalRoute.getQueueDatas().get(0).getBrokerName().equals(BROKER2_NAME)\n                && finalRoute.getQueueDatas().get(1).getBrokerName().equals(BROKER3_NAME);\n        });\n\n        brokerController1.getBrokerConfig().setEnableSingleTopicRegister(false);\n        brokerController2.getBrokerConfig().setEnableSingleTopicRegister(false);\n        brokerController3.getBrokerConfig().setEnableSingleTopicRegister(false);\n        namesrvController.getNamesrvConfig().setDeleteTopicWithBrokerRegistration(false);\n    }\n\n    @Test\n    public void testStaticTopicNotAffected() throws Exception {\n        namesrvController.getNamesrvConfig().setDeleteTopicWithBrokerRegistration(true);\n        brokerController1.getBrokerConfig().setEnableSingleTopicRegister(true);\n        brokerController2.getBrokerConfig().setEnableSingleTopicRegister(true);\n        brokerController3.getBrokerConfig().setEnableSingleTopicRegister(true);\n\n        String testTopic = \"test-topic-not-affected\";\n        String testStaticTopic = \"test-static-topic\";\n\n        boolean createResult = MQAdminTestUtils.createTopic(NAMESRV_ADDR, CLUSTER_NAME, testTopic, 8, null);\n        assertThat(createResult).isTrue();\n\n        TopicRouteData route = MQAdminTestUtils.examineTopicRouteInfo(NAMESRV_ADDR, testTopic);\n        assertThat(route.getBrokerDatas()).hasSize(3);\n        assertThat(route.getQueueDatas()).hasSize(3);\n\n        MQAdminTestUtils.createStaticTopicWithCommand(testStaticTopic, 10, null, CLUSTER_NAME, NAMESRV_ADDR);\n\n        assertThat(route.getBrokerDatas()).hasSize(3);\n        assertThat(route.getQueueDatas()).hasSize(3);\n\n        brokerController1.getBrokerConfig().setEnableSingleTopicRegister(false);\n        brokerController2.getBrokerConfig().setEnableSingleTopicRegister(false);\n        brokerController3.getBrokerConfig().setEnableSingleTopicRegister(false);\n        namesrvController.getNamesrvConfig().setDeleteTopicWithBrokerRegistration(false);\n    }\n\n    @Test\n    public void testCreateOrUpdateTopic_EnableSplitRegistration() {\n        brokerController1.getBrokerConfig().setEnableSplitRegistration(true);\n        brokerController2.getBrokerConfig().setEnableSplitRegistration(true);\n        brokerController3.getBrokerConfig().setEnableSplitRegistration(true);\n\n        String testTopic = \"test-topic-\";\n\n        for (int i = 0; i < 10; i++) {\n            TopicConfig topicConfig = new TopicConfig(testTopic + i, 8, 8);\n            brokerController1.getTopicConfigManager().updateTopicConfig(topicConfig);\n            brokerController2.getTopicConfigManager().updateTopicConfig(topicConfig);\n            brokerController3.getTopicConfigManager().updateTopicConfig(topicConfig);\n        }\n\n        brokerController1.registerBrokerAll(false, true, true);\n        brokerController2.registerBrokerAll(false, true, true);\n        brokerController3.registerBrokerAll(false, true, true);\n\n        for (int i = 0; i < 10; i++) {\n            TopicRouteData route = MQAdminTestUtils.examineTopicRouteInfo(NAMESRV_ADDR, testTopic + i);\n            assertThat(route.getBrokerDatas()).hasSize(3);\n            assertThat(route.getQueueDatas()).hasSize(3);\n        }\n\n        brokerController1.getBrokerConfig().setEnableSplitRegistration(false);\n        brokerController2.getBrokerConfig().setEnableSplitRegistration(false);\n        brokerController3.getBrokerConfig().setEnableSplitRegistration(false);\n    }\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/schema/SchemaTest.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\npackage org.apache.rocketmq.test.schema;\n\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.TreeMap;\nimport org.junit.Assert;\nimport org.junit.Ignore;\nimport org.junit.Test;\n\n\n\npublic class SchemaTest {\n    private static final String BASE_SCHEMA_PATH = \"src/test/resources/schema\";\n    private static final String ADD = \"ADD\";\n    private static final String DELETE = \"DELETE\";\n    private static final String CHANGE = \"CHANGE\";\n\n\n\n    public void generate() throws Exception {\n        SchemaDefiner.doLoad();\n        SchemaTools.write(SchemaTools.generate(SchemaDefiner.API_CLASS_LIST), BASE_SCHEMA_PATH, \"api\");\n        SchemaTools.write(SchemaTools.generate(SchemaDefiner.PROTOCOL_CLASS_LIST), BASE_SCHEMA_PATH, \"protocol\");\n    }\n\n    @Test\n    @Ignore\n    public void checkSchema() throws Exception {\n        SchemaDefiner.doLoad();\n        Map<String, Map<String, String>> schemaFromFile = new HashMap<>();\n        {\n            schemaFromFile.putAll(SchemaTools.load(BASE_SCHEMA_PATH, SchemaTools.PATH_API));\n            schemaFromFile.putAll(SchemaTools.load(BASE_SCHEMA_PATH, SchemaTools.PATH_PROTOCOL));\n        }\n        Map<String, Map<String, String>> schemaFromCode = new HashMap<>();\n        {\n            schemaFromCode.putAll(SchemaTools.generate(SchemaDefiner.API_CLASS_LIST));\n            schemaFromCode.putAll(SchemaTools.generate(SchemaDefiner.PROTOCOL_CLASS_LIST));\n        }\n\n        Map<String, String> fileChanges = new TreeMap<>();\n        schemaFromFile.keySet().forEach(x -> {\n            if (!schemaFromCode.containsKey(x)) {\n                fileChanges.put(x, DELETE);\n            }\n        });\n        schemaFromCode.keySet().forEach(x -> {\n            if (!schemaFromFile.containsKey(x)) {\n                fileChanges.put(x, ADD);\n            }\n        });\n\n        Map<String, Map<String, String>> changesByFile = new HashMap<>();\n        schemaFromFile.forEach((file, oldSchema) -> {\n            Map<String, String> newSchema = schemaFromCode.get(file);\n            Map<String, String> schemaChanges = new TreeMap<>();\n            oldSchema.forEach((k, v) -> {\n                if (!newSchema.containsKey(k)) {\n                    schemaChanges.put(k, DELETE);\n                } else if (!newSchema.get(k).equals(v)) {\n                    schemaChanges.put(k, CHANGE);\n                }\n            });\n\n            newSchema.forEach((k, v) -> {\n                if (!oldSchema.containsKey(k)) {\n                    schemaChanges.put(k, ADD);\n                }\n            });\n            if (!schemaChanges.isEmpty()) {\n                changesByFile.put(file, schemaChanges);\n            }\n        });\n\n        fileChanges.forEach((k,v) -> {\n            System.out.printf(\"%s file %s\\n\", v, k);\n        });\n\n        changesByFile.forEach((k, v) -> {\n            System.out.printf(\"%s file %s:\\n\", CHANGE, k);\n            v.forEach((kk, vv) -> {\n                System.out.printf(\"\\t%s %s\\n\", vv, kk);\n            });\n        });\n\n        String message = \"The schema test failed, which means you have changed the API or Protocol defined in org.apache.rocketmq.test.schema.SchemaDefiner.\\n\" +\n            \"Please submit a pr only contains the API/Protocol changes and request at least one PMC Member's review.\\n\" +\n            \"For original motivation of this test, please refer to https://github.com/apache/rocketmq/pull/4565 .\";\n        Assert.assertTrue(message, fileChanges.isEmpty() && changesByFile.isEmpty());\n    }\n\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/smoke/NormalMessageSendAndRecvIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.apache.rocketmq.test.smoke;\n\nimport com.google.common.collect.ImmutableList;\nimport org.apache.rocketmq.client.exception.MQClientException;\nimport org.apache.rocketmq.common.BrokerConfig;\nimport org.apache.rocketmq.common.message.MessageClientExt;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.remoting.protocol.admin.ConsumeStats;\nimport org.apache.rocketmq.store.config.MessageStoreConfig;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.base.IntegrationTestBase;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.apache.rocketmq.tools.admin.DefaultMQAdminExt;\nimport org.awaitility.Awaitility;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.Test;\nimport java.time.Duration;\nimport java.util.List;\nimport java.util.concurrent.atomic.AtomicReference;\nimport java.util.function.Function;\nimport java.util.stream.Collectors;\n\nimport static com.google.common.truth.Truth.assertThat;\n\npublic class NormalMessageSendAndRecvIT extends BaseConf {\n    private static Logger logger = LoggerFactory.getLogger(NormalMessageSendAndRecvIT.class);\n    private RMQNormalConsumer consumer = null;\n    private RMQNormalProducer producer = null;\n    private String topic = null;\n    private String group = null;\n    private DefaultMQAdminExt defaultMQAdminExt;\n\n    @Before\n    public void setUp() throws Exception {\n        topic = initTopic();\n        group = initConsumerGroup();\n        logger.info(String.format(\"use topic: %s;\", topic));\n        producer = getProducer(NAMESRV_ADDR, topic);\n        consumer = getConsumer(NAMESRV_ADDR, group, topic, \"*\", new RMQNormalListener());\n        defaultMQAdminExt = getAdmin(NAMESRV_ADDR);\n        defaultMQAdminExt.start();\n    }\n\n    @After\n    public void tearDown() {\n        BaseConf.shutdown();\n    }\n\n    @Test\n    public void testSynSendMessage() throws Exception {\n        AtomicReference<List<MessageQueue>> messageQueueList = new AtomicReference<>();\n        AtomicReference<ConsumeStats> consumeStats = new AtomicReference<>();\n        Awaitility.await().atMost(Duration.ofSeconds(120))\n            .until(() -> {\n                try {\n                    consumeStats.set(defaultMQAdminExt.examineConsumeStats(group));\n                    messageQueueList.set(producer.getProducer().fetchPublishMessageQueues(topic));\n                    return !messageQueueList.get().isEmpty() && null != consumeStats.get()\n                        && consumeStats.get().getOffsetTable().keySet().containsAll(messageQueueList.get());\n                } catch (MQClientException e) {\n                    logger.debug(\"Exception raised while checking producer and consumer are started\", e);\n                }\n                return false;\n            });\n\n        int msgSize = 10;\n        for (MessageQueue messageQueue : messageQueueList.get()) {\n            producer.send(msgSize, messageQueue);\n        }\n        Assert.assertEquals(\"Not all sent succeeded\", msgSize * messageQueueList.get().size(),\n            producer.getAllUndupMsgBody().size());\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), CONSUME_TIME);\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n            consumer.getListener().getAllMsgBody()))\n            .containsExactlyElementsIn(producer.getAllMsgBody());\n\n        for (Object o : consumer.getListener().getAllOriginMsg()) {\n            MessageClientExt msg = (MessageClientExt) o;\n            assertThat(msg.getProperty(MessageConst.PROPERTY_POP_CK)).isNull();\n        }\n        //shutdown to persist the offset\n        consumer.getConsumer().shutdown();\n        consumeStats.set(defaultMQAdminExt.examineConsumeStats(group));\n        //+1 for the retry topic\n        for (MessageQueue messageQueue : messageQueueList.get()) {\n            Assert.assertTrue(consumeStats.get().getOffsetTable().containsKey(messageQueue));\n            Assert.assertEquals(msgSize, consumeStats.get().getOffsetTable().get(messageQueue).getConsumerOffset());\n            Assert.assertEquals(msgSize, consumeStats.get().getOffsetTable().get(messageQueue).getBrokerOffset());\n        }\n\n    }\n\n    @Test\n    public void testSynSendMessageWhenEnableBuildConsumeQueueConcurrently() throws Exception {\n        resetStoreConfigWithEnableBuildConsumeQueueConcurrently(true);\n\n        testSynSendMessage();\n\n        resetStoreConfigWithEnableBuildConsumeQueueConcurrently(false);\n    }\n\n    void resetStoreConfigWithEnableBuildConsumeQueueConcurrently(boolean enableBuildConsumeQueueConcurrently) {\n        {\n            brokerController1.shutdown();\n            MessageStoreConfig storeConfig = brokerController1.getMessageStoreConfig();\n            BrokerConfig brokerConfig = brokerController1.getBrokerConfig();\n            storeConfig.setEnableBuildConsumeQueueConcurrently(enableBuildConsumeQueueConcurrently);\n            brokerController1 = IntegrationTestBase.createAndStartBroker(storeConfig, brokerConfig);\n        }\n        {\n            brokerController2.shutdown();\n            MessageStoreConfig storeConfig = brokerController2.getMessageStoreConfig();\n            BrokerConfig brokerConfig = brokerController2.getBrokerConfig();\n            storeConfig.setEnableBuildConsumeQueueConcurrently(enableBuildConsumeQueueConcurrently);\n            brokerController2 = IntegrationTestBase.createAndStartBroker(storeConfig, brokerConfig);\n        }\n        {\n            brokerController3.shutdown();\n            MessageStoreConfig storeConfig = brokerController3.getMessageStoreConfig();\n            BrokerConfig brokerConfig = brokerController3.getBrokerConfig();\n            storeConfig.setEnableBuildConsumeQueueConcurrently(enableBuildConsumeQueueConcurrently);\n            brokerController3 = IntegrationTestBase.createAndStartBroker(storeConfig, brokerConfig);\n        }\n        brokerControllerList = ImmutableList.of(brokerController1, brokerController2, brokerController3);\n        brokerControllerMap = brokerControllerList.stream().collect(\n                Collectors.toMap(input -> input.getBrokerConfig().getBrokerName(), Function.identity()));\n    }\n\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/statictopic/StaticTopicIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.statictopic;\n\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.ImmutableSet;\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport org.apache.rocketmq.broker.BrokerController;\nimport org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;\nimport org.apache.rocketmq.client.impl.factory.MQClientInstance;\nimport org.apache.rocketmq.client.producer.DefaultMQProducer;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageExt;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.remoting.protocol.admin.ConsumeStats;\nimport org.apache.rocketmq.remoting.protocol.admin.OffsetWrapper;\nimport org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;\nimport org.apache.rocketmq.remoting.protocol.statictopic.LogicQueueMappingItem;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicConfigAndQueueMapping;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingOne;\nimport org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingUtils;\nimport org.apache.rocketmq.remoting.rpc.ClientMetadata;\nimport org.apache.rocketmq.logging.org.slf4j.Logger;\nimport org.apache.rocketmq.logging.org.slf4j.LoggerFactory;\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.MQAdminTestUtils;\nimport org.apache.rocketmq.test.util.MQRandomUtils;\nimport org.apache.rocketmq.test.util.TestUtils;\nimport org.apache.rocketmq.test.util.VerifyUtils;\nimport org.apache.rocketmq.tools.admin.DefaultMQAdminExt;\nimport org.apache.rocketmq.tools.admin.MQAdminUtils;\nimport org.junit.After;\nimport org.junit.Assert;\nimport org.junit.Before;\nimport org.junit.FixMethodOrder;\nimport org.junit.Test;\n\nimport static com.google.common.truth.Truth.assertThat;\nimport static org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingUtils.getMappingDetailFromConfig;\n\n@FixMethodOrder\npublic class StaticTopicIT extends BaseConf {\n\n    private static Logger logger = LoggerFactory.getLogger(StaticTopicIT.class);\n    private DefaultMQAdminExt defaultMQAdminExt;\n\n    @Before\n    public void setUp() throws Exception {\n        System.setProperty(\"rocketmq.client.rebalance.waitInterval\", \"500\");\n        defaultMQAdminExt = getAdmin(NAMESRV_ADDR);\n        waitBrokerRegistered(NAMESRV_ADDR, CLUSTER_NAME, BROKER_NUM);\n        defaultMQAdminExt.start();\n    }\n\n\n    @Test\n    public void testCommandsWithCluster() throws Exception {\n        //This case is used to mock the env to test the command manually\n        String topic = \"static\" + MQRandomUtils.getRandomTopic();\n        RMQNormalProducer producer = getProducer(NAMESRV_ADDR, topic);\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n        int queueNum = 10;\n        int msgEachQueue = 100;\n\n        {\n            MQAdminTestUtils.createStaticTopicWithCommand(topic, queueNum, null, CLUSTER_NAME, NAMESRV_ADDR);\n            sendMessagesAndCheck(producer, getBrokers(), topic, queueNum, msgEachQueue, 0);\n            //consume and check\n            consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 1);\n        }\n        {\n            MQAdminTestUtils.remappingStaticTopicWithCommand(topic, null, CLUSTER_NAME, NAMESRV_ADDR);\n            awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), consumer.getConsumer(), defaultMQAdminExt);\n            sendMessagesAndCheck(producer, getBrokers(), topic, queueNum, msgEachQueue, msgEachQueue);\n        }\n    }\n\n    @Test\n    public void testCommandsWithBrokers() throws Exception {\n        //This case is used to mock the env to test the command manually\n        String topic = \"static\" + MQRandomUtils.getRandomTopic();\n        RMQNormalProducer producer = getProducer(NAMESRV_ADDR, topic);\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n        int queueNum = 10;\n        int msgEachQueue = 10;\n        {\n            Set<String> brokers = ImmutableSet.of(BROKER1_NAME);\n            MQAdminTestUtils.createStaticTopicWithCommand(topic, queueNum, brokers, null, NAMESRV_ADDR);\n            sendMessagesAndCheck(producer, brokers, topic, queueNum, msgEachQueue, 0);\n            //consume and check\n            consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 1);\n        }\n        {\n            Set<String> brokers = ImmutableSet.of(BROKER2_NAME);\n            MQAdminTestUtils.remappingStaticTopicWithCommand(topic, brokers, null, NAMESRV_ADDR);\n            awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), consumer.getConsumer(), defaultMQAdminExt);\n            sendMessagesAndCheck(producer, brokers, topic, queueNum, msgEachQueue, TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE);\n            consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 2);\n        }\n    }\n\n    @Test\n    public void testNoTargetBrokers() throws Exception {\n        String topic = \"static\" + MQRandomUtils.getRandomTopic();\n        int queueNum = 10;\n        {\n            Set<String> targetBrokers = new HashSet<>();\n            targetBrokers.add(BROKER1_NAME);\n            MQAdminTestUtils.createStaticTopic(topic, queueNum, targetBrokers, defaultMQAdminExt);\n            Map<String, TopicConfigAndQueueMapping> remoteBrokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);\n            Assert.assertEquals(BROKER_NUM, remoteBrokerConfigMap.size());\n            TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, remoteBrokerConfigMap);\n            Map<Integer, TopicQueueMappingOne>  globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(getMappingDetailFromConfig(remoteBrokerConfigMap.values())), false, true);\n            Assert.assertEquals(queueNum, globalIdMap.size());\n            TopicConfigAndQueueMapping configMapping = remoteBrokerConfigMap.get(BROKER2_NAME);\n            Assert.assertEquals(0, configMapping.getWriteQueueNums());\n            Assert.assertEquals(0, configMapping.getReadQueueNums());\n            Assert.assertEquals(0, configMapping.getMappingDetail().getHostedQueues().size());\n        }\n\n        {\n            Set<String> targetBrokers = new HashSet<>();\n            targetBrokers.add(BROKER2_NAME);\n            MQAdminTestUtils.remappingStaticTopic(topic, targetBrokers, defaultMQAdminExt);\n            Map<String, TopicConfigAndQueueMapping> remoteBrokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);\n            Assert.assertEquals(BROKER_NUM, remoteBrokerConfigMap.size());\n            TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, remoteBrokerConfigMap);\n            Map<Integer, TopicQueueMappingOne>  globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(getMappingDetailFromConfig(remoteBrokerConfigMap.values())), false, true);\n            Assert.assertEquals(queueNum, globalIdMap.size());\n        }\n\n    }\n\n    private void sendMessagesAndCheck(RMQNormalProducer producer, Set<String> targetBrokers, String topic, int queueNum, int msgEachQueue, long baseOffset) throws Exception {\n        ClientMetadata clientMetadata = MQAdminUtils.getBrokerAndTopicMetadata(topic, defaultMQAdminExt);\n        List<MessageQueue> messageQueueList = producer.getMessageQueue();\n        Assert.assertEquals(queueNum, messageQueueList.size());\n        for (int i = 0; i < queueNum; i++) {\n            MessageQueue messageQueue = messageQueueList.get(i);\n            Assert.assertEquals(topic, messageQueue.getTopic());\n            Assert.assertEquals(TopicQueueMappingUtils.getMockBrokerName(MixAll.METADATA_SCOPE_GLOBAL), messageQueue.getBrokerName());\n            Assert.assertEquals(i, messageQueue.getQueueId());\n            String destBrokerName = clientMetadata.getBrokerNameFromMessageQueue(messageQueue);\n            Assert.assertTrue(targetBrokers.contains(destBrokerName));\n        }\n        for (MessageQueue messageQueue: messageQueueList) {\n            producer.send(msgEachQueue, messageQueue);\n        }\n        Assert.assertEquals(0, producer.getSendErrorMsg().size());\n        //leave the time to build the cq\n        Assert.assertTrue(awaitDispatchMs(500));\n        for (MessageQueue messageQueue : messageQueueList) {\n            Assert.assertEquals(0, defaultMQAdminExt.minOffset(messageQueue));\n            Assert.assertEquals(msgEachQueue + baseOffset, defaultMQAdminExt.maxOffset(messageQueue));\n        }\n        TopicStatsTable topicStatsTable = defaultMQAdminExt.examineTopicStats(topic);\n        for (MessageQueue messageQueue : messageQueueList) {\n            Assert.assertEquals(0, topicStatsTable.getOffsetTable().get(messageQueue).getMinOffset());\n            Assert.assertEquals(msgEachQueue + baseOffset, topicStatsTable.getOffsetTable().get(messageQueue).getMaxOffset());\n        }\n    }\n\n    private Map<Integer, List<MessageExt>> computeMessageByQueue(Collection<Object> msgs) {\n        Map<Integer, List<MessageExt>> messagesByQueue = new HashMap<>();\n        for (Object object : msgs) {\n            MessageExt messageExt = (MessageExt) object;\n            if (!messagesByQueue.containsKey(messageExt.getQueueId())) {\n                messagesByQueue.put(messageExt.getQueueId(), new ArrayList<>());\n            }\n            messagesByQueue.get(messageExt.getQueueId()).add(messageExt);\n        }\n        for (List<MessageExt> msgEachQueue : messagesByQueue.values()) {\n            msgEachQueue.sort((o1, o2) -> (int) (o1.getQueueOffset() - o2.getQueueOffset()));\n        }\n        return messagesByQueue;\n    }\n\n    private void consumeMessagesAndCheck(RMQNormalProducer producer, RMQNormalConsumer consumer, String topic, int queueNum, int msgEachQueue, int startGen, int genNum) {\n        consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), 60000);\n\n        Assert.assertEquals(producer.getAllMsgBody().size(), consumer.getListener().getAllMsgBody().size());\n        assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),\n                consumer.getListener().getAllMsgBody()))\n                .containsExactlyElementsIn(producer.getAllMsgBody());\n        Map<Integer, List<MessageExt>> messagesByQueue = computeMessageByQueue(consumer.getListener().getAllOriginMsg());\n        Assert.assertEquals(queueNum, messagesByQueue.size());\n        for (int i = 0; i < queueNum; i++) {\n            List<MessageExt> messageExts = messagesByQueue.get(i);\n\n            int totalEachQueue = msgEachQueue * genNum;\n            Assert.assertEquals(totalEachQueue, messageExts.size());\n            for (int j = 0; j < totalEachQueue; j++) {\n                MessageExt messageExt = messageExts.get(j);\n                int currGen = startGen + j / msgEachQueue;\n                Assert.assertEquals(topic, messageExt.getTopic());\n                Assert.assertEquals(TopicQueueMappingUtils.getMockBrokerName(MixAll.METADATA_SCOPE_GLOBAL), messageExt.getBrokerName());\n                Assert.assertEquals(i, messageExt.getQueueId());\n                Assert.assertEquals((j % msgEachQueue) + currGen * TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE, messageExt.getQueueOffset());\n            }\n        }\n    }\n\n\n    @Test\n    public void testCreateProduceConsumeStaticTopic() throws Exception {\n        String topic = \"static\" + MQRandomUtils.getRandomTopic();\n        RMQNormalProducer producer = getProducer(NAMESRV_ADDR, topic);\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n\n        int queueNum = 10;\n        int msgEachQueue = 10;\n        //create static topic\n        Map<String, TopicConfigAndQueueMapping> localBrokerConfigMap = MQAdminTestUtils.createStaticTopic(topic, queueNum, getBrokers(), defaultMQAdminExt);\n        //check the static topic config\n        {\n            Map<String, TopicConfigAndQueueMapping> remoteBrokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);\n            Assert.assertEquals(BROKER_NUM, remoteBrokerConfigMap.size());\n            for (Map.Entry<String, TopicConfigAndQueueMapping> entry: remoteBrokerConfigMap.entrySet())  {\n                String broker = entry.getKey();\n                TopicConfigAndQueueMapping configMapping = entry.getValue();\n                TopicConfigAndQueueMapping localConfigMapping = localBrokerConfigMap.get(broker);\n                Assert.assertNotNull(localConfigMapping);\n                Assert.assertEquals(configMapping, localConfigMapping);\n            }\n            TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, remoteBrokerConfigMap);\n            Map<Integer, TopicQueueMappingOne>  globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(getMappingDetailFromConfig(remoteBrokerConfigMap.values())), false, true);\n            Assert.assertEquals(queueNum, globalIdMap.size());\n        }\n        //send and check\n        sendMessagesAndCheck(producer, getBrokers(), topic, queueNum, msgEachQueue, 0);\n        //consume and check\n        consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 1);\n    }\n\n\n    @Test\n    public void testRemappingProduceConsumeStaticTopic() throws Exception {\n        String topic = \"static\" + MQRandomUtils.getRandomTopic();\n        RMQNormalProducer producer = getProducer(NAMESRV_ADDR, topic);\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener());\n\n        int queueNum = 1;\n        int msgEachQueue = 10;\n        //create send consume\n        {\n            Set<String> targetBrokers = ImmutableSet.of(BROKER1_NAME);\n            MQAdminTestUtils.createStaticTopic(topic, queueNum, targetBrokers, defaultMQAdminExt);\n            sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 0);\n            consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 1);\n        }\n        //remapping the static topic\n        {\n            Set<String> targetBrokers = ImmutableSet.of(BROKER2_NAME);\n            MQAdminTestUtils.remappingStaticTopic(topic, targetBrokers, defaultMQAdminExt);\n            Map<String, TopicConfigAndQueueMapping> remoteBrokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);\n            TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, remoteBrokerConfigMap);\n            Map<Integer, TopicQueueMappingOne>  globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(getMappingDetailFromConfig(remoteBrokerConfigMap.values())), false, true);\n            Assert.assertEquals(queueNum, globalIdMap.size());\n            for (TopicQueueMappingOne mappingOne: globalIdMap.values()) {\n                Assert.assertEquals(BROKER2_NAME, mappingOne.getBname());\n                Assert.assertEquals(TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE, mappingOne.getItems().get(mappingOne.getItems().size() - 1).getLogicOffset());\n            }\n            awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), consumer.getConsumer(), defaultMQAdminExt);\n            sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE);\n            consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 2);\n        }\n    }\n\n\n    public boolean awaitRefreshStaticTopicMetadata(long timeMs, String topic, DefaultMQProducer producer, DefaultMQPushConsumer consumer, DefaultMQAdminExt adminExt) throws Exception {\n        long start = System.currentTimeMillis();\n        MQClientInstance currentInstance = null;\n        while (System.currentTimeMillis() - start <= timeMs) {\n            boolean allOk = true;\n            if (producer != null) {\n                currentInstance = producer.getDefaultMQProducerImpl().getmQClientFactory();\n                currentInstance.updateTopicRouteInfoFromNameServer(topic);\n                if (!MQAdminTestUtils.checkStaticTopic(topic, adminExt, currentInstance)) {\n                    allOk = false;\n                }\n            }\n            if (consumer != null) {\n                currentInstance = consumer.getDefaultMQPushConsumerImpl().getmQClientFactory();\n                currentInstance.updateTopicRouteInfoFromNameServer(topic);\n                if (!MQAdminTestUtils.checkStaticTopic(topic, adminExt, currentInstance)) {\n                    allOk = false;\n                }\n            }\n            if (adminExt != null) {\n                currentInstance = adminExt.getDefaultMQAdminExtImpl().getMqClientInstance();\n                currentInstance.updateTopicRouteInfoFromNameServer(topic);\n                if (!MQAdminTestUtils.checkStaticTopic(topic, adminExt, currentInstance)) {\n                    allOk = false;\n                }\n            }\n            if (allOk) {\n                return true;\n            }\n            Thread.sleep(100);\n        }\n        return false;\n    }\n\n\n    @Test\n    public void testDoubleReadCheckConsumerOffset() throws Exception {\n        String topic = \"static\" + MQRandomUtils.getRandomTopic();\n        String group = initConsumerGroup();\n        RMQNormalProducer producer = getProducer(NAMESRV_ADDR, topic);\n        RMQNormalConsumer consumer = getConsumer(NAMESRV_ADDR, group, topic, \"*\", new RMQNormalListener());\n        long start = System.currentTimeMillis();\n\n        int queueNum = 5;\n        int msgEachQueue = 10;\n        //create static topic\n        {\n            Set<String> targetBrokers = ImmutableSet.of(BROKER1_NAME);\n            MQAdminTestUtils.createStaticTopic(topic, queueNum, targetBrokers, defaultMQAdminExt);\n            sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 0);\n            consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 1);\n        }\n        producer.shutdown();\n        consumer.shutdown();\n        //use a new producer\n        producer = getProducer(NAMESRV_ADDR, topic);\n\n        ConsumeStats consumeStats = defaultMQAdminExt.examineConsumeStats(group);\n        List<MessageQueue> messageQueues = producer.getMessageQueue();\n        for (MessageQueue queue: messageQueues) {\n            OffsetWrapper wrapper = consumeStats.getOffsetTable().get(queue);\n            Assert.assertNotNull(wrapper);\n            Assert.assertEquals(msgEachQueue, wrapper.getBrokerOffset());\n            Assert.assertEquals(msgEachQueue, wrapper.getConsumerOffset());\n            Assert.assertTrue(wrapper.getLastTimestamp() > start);\n        }\n\n        List<String> brokers = ImmutableList.of(BROKER2_NAME, BROKER3_NAME, BROKER1_NAME);\n        for (int i = 0; i < brokers.size(); i++) {\n            Set<String> targetBrokers = ImmutableSet.of(brokers.get(i));\n            MQAdminTestUtils.remappingStaticTopic(topic, targetBrokers, defaultMQAdminExt);\n            //make the metadata\n            awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), null, defaultMQAdminExt);\n            sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, (i + 1) * TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE);\n        }\n\n        TestUtils.waitForSeconds(1);\n        consumeStats = defaultMQAdminExt.examineConsumeStats(group);\n\n        messageQueues = producer.getMessageQueue();\n        for (MessageQueue queue: messageQueues) {\n            OffsetWrapper wrapper = consumeStats.getOffsetTable().get(queue);\n            Assert.assertNotNull(wrapper);\n            Assert.assertEquals(msgEachQueue + brokers.size() * TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE, wrapper.getBrokerOffset());\n            Assert.assertEquals(msgEachQueue, wrapper.getConsumerOffset());\n            Assert.assertTrue(wrapper.getLastTimestamp() > start);\n        }\n        consumer = getConsumer(NAMESRV_ADDR, group, topic, \"*\", new RMQNormalListener());\n        consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 1, brokers.size());\n    }\n\n\n\n\n    @Test\n    public void testRemappingAndClear() throws Exception {\n        String topic = \"static\" + MQRandomUtils.getRandomTopic();\n        RMQNormalProducer producer = getProducer(NAMESRV_ADDR, topic);\n        int queueNum = 10;\n        int msgEachQueue = 100;\n        //create to broker1Name\n        {\n            Set<String> targetBrokers = ImmutableSet.of(BROKER1_NAME);\n            MQAdminTestUtils.createStaticTopic(topic, queueNum, targetBrokers, defaultMQAdminExt);\n            //leave the time to refresh the metadata\n            awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), null, defaultMQAdminExt);\n            sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 0);\n        }\n\n        //remapping to broker2Name\n        {\n            Set<String> targetBrokers = ImmutableSet.of(BROKER2_NAME);\n            MQAdminTestUtils.remappingStaticTopic(topic, targetBrokers, defaultMQAdminExt);\n            //leave the time to refresh the metadata\n            awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), null, defaultMQAdminExt);\n            sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 1 * TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE);\n        }\n\n        //remapping to broker3Name\n        {\n            Set<String> targetBrokers = ImmutableSet.of(BROKER3_NAME);\n            MQAdminTestUtils.remappingStaticTopic(topic, targetBrokers, defaultMQAdminExt);\n            //leave the time to refresh the metadata\n            awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), null, defaultMQAdminExt);\n            sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 2 * TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE);\n        }\n\n        // 1 -> 2 -> 3, currently 1 should not have any mappings\n\n        {\n            for (int i = 0; i < 10; i++) {\n                for (BrokerController brokerController: brokerControllerList) {\n                    brokerController.getTopicQueueMappingCleanService().wakeup();\n                }\n                Thread.sleep(100);\n            }\n            Map<String, TopicConfigAndQueueMapping> brokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);\n            Assert.assertEquals(BROKER_NUM, brokerConfigMap.size());\n            TopicConfigAndQueueMapping config1 = brokerConfigMap.get(BROKER1_NAME);\n            TopicConfigAndQueueMapping config2 = brokerConfigMap.get(BROKER2_NAME);\n            TopicConfigAndQueueMapping config3 = brokerConfigMap.get(BROKER3_NAME);\n            Assert.assertEquals(0, config1.getMappingDetail().getHostedQueues().size());\n            Assert.assertEquals(queueNum, config2.getMappingDetail().getHostedQueues().size());\n\n            Assert.assertEquals(queueNum, config3.getMappingDetail().getHostedQueues().size());\n\n        }\n        {\n            Set<String> topics =  new HashSet<>(brokerController1.getTopicConfigManager().getTopicConfigTable().keySet());\n            topics.remove(topic);\n            brokerController1.getMessageStore().cleanUnusedTopic(topics);\n            brokerController2.getMessageStore().cleanUnusedTopic(topics);\n            for (int i = 0; i < 10; i++) {\n                for (BrokerController brokerController: brokerControllerList) {\n                    brokerController.getTopicQueueMappingCleanService().wakeup();\n                }\n                Thread.sleep(100);\n            }\n\n            Map<String, TopicConfigAndQueueMapping> brokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);\n            Assert.assertEquals(BROKER_NUM, brokerConfigMap.size());\n            TopicConfigAndQueueMapping config1 = brokerConfigMap.get(BROKER1_NAME);\n            TopicConfigAndQueueMapping config2 = brokerConfigMap.get(BROKER2_NAME);\n            TopicConfigAndQueueMapping config3 = brokerConfigMap.get(BROKER3_NAME);\n            Assert.assertEquals(0, config1.getMappingDetail().getHostedQueues().size());\n            Assert.assertEquals(queueNum, config2.getMappingDetail().getHostedQueues().size());\n            Assert.assertEquals(queueNum, config3.getMappingDetail().getHostedQueues().size());\n            //The first leader will clear it\n            for (List<LogicQueueMappingItem> items : config1.getMappingDetail().getHostedQueues().values()) {\n                Assert.assertEquals(3, items.size());\n            }\n            //The second leader do nothing\n            for (List<LogicQueueMappingItem> items : config3.getMappingDetail().getHostedQueues().values()) {\n                Assert.assertEquals(1, items.size());\n            }\n        }\n    }\n\n\n    @Test\n    public void testRemappingWithNegativeLogicOffset() throws Exception {\n        String topic = \"static\" + MQRandomUtils.getRandomTopic();\n        RMQNormalProducer producer = getProducer(NAMESRV_ADDR, topic);\n        int queueNum = 10;\n        int msgEachQueue = 100;\n        //create and send\n        {\n            Set<String> targetBrokers = ImmutableSet.of(BROKER1_NAME);\n            MQAdminTestUtils.createStaticTopic(topic, queueNum, targetBrokers, defaultMQAdminExt);\n            sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 0);\n        }\n\n        //remapping the static topic with -1 logic offset\n        {\n            Set<String> targetBrokers = ImmutableSet.of(BROKER2_NAME);\n            MQAdminTestUtils.remappingStaticTopicWithNegativeLogicOffset(topic, targetBrokers, defaultMQAdminExt);\n            Map<String, TopicConfigAndQueueMapping> remoteBrokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);\n            TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, remoteBrokerConfigMap);\n            Map<Integer, TopicQueueMappingOne>  globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(getMappingDetailFromConfig(remoteBrokerConfigMap.values())), false, true);\n            Assert.assertEquals(queueNum, globalIdMap.size());\n            for (TopicQueueMappingOne mappingOne: globalIdMap.values()) {\n                Assert.assertEquals(BROKER2_NAME, mappingOne.getBname());\n                Assert.assertEquals(-1, mappingOne.getItems().get(mappingOne.getItems().size() - 1).getLogicOffset());\n            }\n            //leave the time to refresh the metadata\n            awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), null, defaultMQAdminExt);\n            //here the gen should be 0\n            sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 0);\n        }\n    }\n\n\n    @After\n    public void tearDown() {\n        System.setProperty(\"rocketmq.client.rebalance.waitInterval\", \"20000\");\n        super.shutdown();\n    }\n\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/tls/TlsIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.tls;\n\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.MQWait;\nimport org.assertj.core.api.Assertions;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\npublic class TlsIT extends BaseConf {\n\n    private RMQNormalProducer producer;\n    private RMQNormalConsumer consumer;\n\n    private String topic;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        // Send messages via TLS\n        producer = getProducer(NAMESRV_ADDR, topic, true);\n        // Receive messages via TLS\n        consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener(), true);\n    }\n\n    @After\n    public void tearDown() {\n        shutdown();\n    }\n\n    @Test\n    public void testSendAndReceiveMessageOverTLS() {\n        int numberOfMessagesToSend = 16;\n        producer.send(numberOfMessagesToSend);\n\n        boolean consumedAll = MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(), consumer.getListener());\n        Assertions.assertThat(consumedAll).isEqualTo(true);\n    }\n\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/tls/TlsMix2IT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.tls;\n\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.MQWait;\nimport org.assertj.core.api.Assertions;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\npublic class TlsMix2IT extends BaseConf {\n\n    private RMQNormalProducer producer;\n    private RMQNormalConsumer consumer;\n\n    private String topic;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n        // send message via TLS\n        producer = getProducer(NAMESRV_ADDR, topic, true);\n\n        // Receive message without TLS.\n        consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener(), false);\n    }\n\n    @After\n    public void tearDown() {\n        shutdown();\n    }\n\n    @Test\n    public void testSendAndReceiveMessageOverTLS() {\n        int numberOfMessagesToSend = 16;\n        producer.send(numberOfMessagesToSend);\n\n        boolean consumedAll = MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(), consumer.getListener());\n        Assertions.assertThat(consumedAll).isEqualTo(true);\n    }\n\n}\n"
  },
  {
    "path": "test/src/test/java/org/apache/rocketmq/test/tls/TlsMixIT.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.test.tls;\n\nimport org.apache.rocketmq.test.base.BaseConf;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;\nimport org.apache.rocketmq.test.client.rmq.RMQNormalProducer;\nimport org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;\nimport org.apache.rocketmq.test.util.MQWait;\nimport org.assertj.core.api.Assertions;\nimport org.junit.After;\nimport org.junit.Before;\nimport org.junit.Test;\n\npublic class TlsMixIT extends BaseConf {\n\n    private RMQNormalProducer producer;\n    private RMQNormalConsumer consumer;\n\n    private String topic;\n\n    @Before\n    public void setUp() {\n        topic = initTopic();\n\n        // send message without TLS\n        producer = getProducer(NAMESRV_ADDR, topic);\n\n        // Receive message via TLS\n        consumer = getConsumer(NAMESRV_ADDR, topic, \"*\", new RMQNormalListener(), true);\n    }\n\n    @After\n    public void tearDown() {\n        shutdown();\n    }\n\n    @Test\n    public void testSendAndReceiveMessageOverTLS() {\n        int numberOfMessagesToSend = 16;\n        producer.send(numberOfMessagesToSend);\n\n        boolean consumedAll = MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(), consumer.getListener());\n        Assertions.assertThat(consumedAll).isEqualTo(true);\n    }\n\n}\n"
  },
  {
    "path": "test/src/test/resources/rmq-proxy-home/conf/broker.conf",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n#  Unless required by applicable law or agreed to in writing, software\n#  distributed under the License is distributed on an \"AS IS\" BASIS,\n#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#  See the License for the specific language governing permissions and\n#  limitations under the License.\n\nbrokerClusterName = DefaultCluster\nbrokerName = broker-a\nbrokerId = 0\ndeleteWhen = 04\nfileReservedTime = 48\nbrokerRole = ASYNC_MASTER\nflushDiskType = ASYNC_FLUSH\n"
  },
  {
    "path": "test/src/test/resources/rmq-proxy-home/conf/logback_proxy.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n<configuration>\n\n    <appender name=\"RocketmqProxyAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}proxy.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}proxy.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>20</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>128MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqProxyAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqProxyAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqGrpcAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}grpc.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}grpc.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>20</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>128MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqGrpcAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqGrpcAppender_inner\"/>\n    </appender>\n\n    <!-- Below is the logger configuration for broker-->\n    <appender name=\"DefaultAppender\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}broker_default.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}broker_default.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n\n    <appender name=\"RocketmqBrokerAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}broker.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}broker.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>20</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>128MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqBrokerAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqBrokerAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqProtectionAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}protection.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}protection.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqProtectionAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqProtectionAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqWaterMarkAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}watermark.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}watermark.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqWaterMarkAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqWaterMarkAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqStoreAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}store.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}store.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>128MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqStoreAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqStoreAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqRemotingAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}remoting.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}remoting.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqRemotingAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqRemotingAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqStoreErrorAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}storeerror.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}storeerror.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqStoreErrorAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqStoreErrorAppender_inner\"/>\n    </appender>\n\n\n    <appender name=\"RocketmqTransactionAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}transaction.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}transaction.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqTransactionAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqTransactionAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqRebalanceLockAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}lock.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}log${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}lock.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>5</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqRebalanceLockAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqRebalanceLockAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqFilterAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}filter.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}filter.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n    <appender name=\"RocketmqFilterAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqFilterAppender_inner\"/>\n    </appender>\n\n    <appender name=\"RocketmqStatsAppender\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}stats.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}stats.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>5</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>100MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n\n    <appender name=\"RocketmqCommercialAppender\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}commercial.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}commercial.%i.log.gz</fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>10</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>500MB</maxFileSize>\n        </triggeringPolicy>\n    </appender>\n\n    <appender name=\"RocketmqPopAppender_inner\"\n              class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}pop.log</file>\n        <append>true</append>\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.FixedWindowRollingPolicy\">\n            <fileNamePattern>${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}qpop.%i.log\n            </fileNamePattern>\n            <minIndex>1</minIndex>\n            <maxIndex>20</maxIndex>\n        </rollingPolicy>\n        <triggeringPolicy\n            class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n            <maxFileSize>128MB</maxFileSize>\n        </triggeringPolicy>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n\n    <appender name=\"RocketmqPopAppender\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"RocketmqPopAppender_inner\"/>\n    </appender>\n\n    <appender name=\"STDOUT\" class=\"ch.qos.logback.core.ConsoleAppender\">\n        <append>true</append>\n        <encoder>\n            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>\n            <charset class=\"java.nio.charset.Charset\">UTF-8</charset>\n        </encoder>\n    </appender>\n\n    <logger name=\"RocketmqBroker\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqBrokerAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqProtection\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqProtectionAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqWaterMark\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqWaterMarkAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqCommon\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqBrokerAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqStore\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqStoreAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqStoreError\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqStoreErrorAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqTransaction\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqTransactionAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqRebalanceLock\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqRebalanceLockAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqRemoting\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqRemotingAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqStats\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqStatsAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqCommercial\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqCommercialAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqFilter\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"RocketmqFilterAppender\"/>\n    </logger>\n\n    <logger name=\"RocketmqConsole\" additivity=\"false\">\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"STDOUT\"/>\n    </logger>\n\n    <logger name=\"RocketmqPop\" additivity=\"false\">\n        <level value=\"INFO\" />\n        <appender-ref ref=\"RocketmqPopAppender\" />\n    </logger>\n\n    <logger name=\"RocketmqProxy\" additivity=\"false\">\n        <level value=\"INFO\" />\n        <appender-ref ref=\"RocketmqProxyAppender\" />\n    </logger>\n\n    <logger name=\"RocketmqGrpc\" additivity=\"false\">\n        <level value=\"INFO\" />\n        <appender-ref ref=\"RocketmqGrpcAppender\" />\n    </logger>\n\n    <root>\n        <level value=\"INFO\"/>\n        <appender-ref ref=\"DefaultAppender\"/>\n    </root>\n</configuration>\n"
  },
  {
    "path": "test/src/test/resources/rmq-proxy-home/conf/rmq-proxy.json",
    "content": "{\n  \"proxyMode\": \"cluster\"\n}"
  },
  {
    "path": "test/src/test/resources/rmq.logback-test.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n<configuration>\n\n    <appender name=\"CONSOLE\" class=\"ch.qos.logback.core.ConsoleAppender\">\n        <layout class=\"ch.qos.logback.classic.PatternLayout\">\n            <Pattern>\n                %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n\n            </Pattern>\n        </layout>\n    </appender>\n\n    <logger name=\"org.apache.rocketmq\" level=\"error\" additivity=\"false\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </logger>\n\n    <root level=\"error\">\n        <appender-ref ref=\"CONSOLE\"/>\n    </root>\n\n</configuration>"
  },
  {
    "path": "test/src/test/resources/schema/api/client.consumer.AllocateMessageQueueStrategy.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nMethod allocate(java.lang.String,java.lang.String,java.util.List,java.util.List) : public throws (java.util.List)\nMethod getName() : public throws (java.lang.String)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.consumer.DefaultLitePullConsumer.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField MIN_AUTOCOMMIT_INTERVAL_MILLIS : private long 1000\nField SEND_MESSAGE_WITH_VIP_CHANNEL_PROPERTY : public java.lang.String com.rocketmq.sendMessageWithVIPChannel\nField accessChannel : protected org.apache.rocketmq.client.AccessChannel LOCAL\nField allocateMessageQueueStrategy : private org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy null\nField autoCommit : private boolean true\nField autoCommitIntervalMillis : private long 5000\nField brokerSuspendMaxTimeMillis : private long 20000\nField clientCallbackExecutorThreads : private int null\nField clientIP : private java.lang.String null\nField consumeFromWhere : private org.apache.rocketmq.common.consumer.ConsumeFromWhere CONSUME_FROM_LAST_OFFSET\nField consumeMaxSpan : private int 2000\nField consumeTimestamp : private java.lang.String null\nField consumerGroup : private java.lang.String DEFAULT_CONSUMER\nField consumerPullTimeoutMillis : private long 10000\nField consumerTimeoutMillisWhenSuspend : private long 30000\nField customizedTraceTopic : private java.lang.String null\nField defaultLitePullConsumerImpl : private org.apache.rocketmq.client.impl.consumer.DefaultLitePullConsumerImpl null\nField enableMsgTrace : private boolean false\nField enableStreamRequestType : protected boolean true\nField heartbeatBrokerInterval : private int 30000\nField instanceName : private java.lang.String DEFAULT\nField language : private org.apache.rocketmq.remoting.protocol.LanguageCode JAVA\nField log : private org.apache.rocketmq.logging.InternalLogger null\nField messageModel : private org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel CLUSTERING\nField messageQueueListener : private org.apache.rocketmq.client.consumer.MessageQueueListener null\nField mqClientApiTimeout : private int 3000\nField namespace : protected java.lang.String null\nField namespaceInitialized : private boolean false\nField namesrvAddr : private java.lang.String null\nField offsetStore : private org.apache.rocketmq.client.consumer.store.OffsetStore null\nField persistConsumerOffsetInterval : private int 5000\nField pollNameServerInterval : private int 30000\nField pollTimeoutMillis : private long 5000\nField pullBatchSize : private int 10\nField pullThreadNums : private int 20\nField pullThresholdForAll : private long 10000\nField pullThresholdForQueue : private int 1000\nField pullThresholdSizeForQueue : private int 100\nField pullTimeDelayMillsWhenException : private long 1000\nField topicMetadataCheckIntervalMillis : private long 30000\nField traceDispatcher : private org.apache.rocketmq.client.trace.TraceDispatcher null\nField unitMode : private boolean false\nField unitName : private java.lang.String null\nField useTLS : private boolean false\nField vipChannelEnabled : private boolean false\nMethod assign(java.util.Collection) : public throws (void)\nMethod buildMQClientId() : public throws (java.lang.String)\nMethod changeInstanceNameToPID() : public throws (void)\nMethod cloneClientConfig() : public throws (org.apache.rocketmq.client.ClientConfig)\nMethod commit(boolean,java.util.Set) : public throws (void)\nMethod commitSync() : public throws (void)\nMethod committed(org.apache.rocketmq.common.message.MessageQueue) : public throws (java.lang.Long)\nMethod fetchMessageQueues(java.lang.String) : public throws (java.util.Collection)\nMethod getAccessChannel() : public throws (org.apache.rocketmq.client.AccessChannel)\nMethod getAllocateMessageQueueStrategy() : public throws (org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy)\nMethod getAutoCommitIntervalMillis() : public throws (long)\nMethod getBrokerSuspendMaxTimeMillis() : public throws (long)\nMethod getClientCallbackExecutorThreads() : public throws (int)\nMethod getClientIP() : public throws (java.lang.String)\nMethod getConsumeFromWhere() : public throws (org.apache.rocketmq.common.consumer.ConsumeFromWhere)\nMethod getConsumeMaxSpan() : public throws (int)\nMethod getConsumeTimestamp() : public throws (java.lang.String)\nMethod getConsumerGroup() : public throws (java.lang.String)\nMethod getConsumerPullTimeoutMillis() : public throws (long)\nMethod getConsumerTimeoutMillisWhenSuspend() : public throws (long)\nMethod getCustomizedTraceTopic() : public throws (java.lang.String)\nMethod getDefaultBrokerId() : public throws (long)\nMethod getHeartbeatBrokerInterval() : public throws (int)\nMethod getInstanceName() : public throws (java.lang.String)\nMethod getLanguage() : public throws (org.apache.rocketmq.remoting.protocol.LanguageCode)\nMethod getMessageModel() : public throws (org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel)\nMethod getMessageQueueListener() : public throws (org.apache.rocketmq.client.consumer.MessageQueueListener)\nMethod getMqClientApiTimeout() : public throws (int)\nMethod getNamespace() : public throws (java.lang.String)\nMethod getNamesrvAddr() : public throws (java.lang.String)\nMethod getOffsetStore() : public throws (org.apache.rocketmq.client.consumer.store.OffsetStore)\nMethod getPersistConsumerOffsetInterval() : public throws (int)\nMethod getPollNameServerInterval() : public throws (int)\nMethod getPollTimeoutMillis() : public throws (long)\nMethod getPullBatchSize() : public throws (int)\nMethod getPullThreadNums() : public throws (int)\nMethod getPullThresholdForAll() : public throws (long)\nMethod getPullThresholdForQueue() : public throws (int)\nMethod getPullThresholdSizeForQueue() : public throws (int)\nMethod getPullTimeDelayMillsWhenException() : public throws (long)\nMethod getTopicMetadataCheckIntervalMillis() : public throws (long)\nMethod getTraceDispatcher() : public throws (org.apache.rocketmq.client.trace.TraceDispatcher)\nMethod getUnitName() : public throws (java.lang.String)\nMethod isAutoCommit() : public throws (boolean)\nMethod isConnectBrokerByUser() : public throws (boolean)\nMethod isEnableMsgTrace() : public throws (boolean)\nMethod isEnableStreamRequestType() : public throws (boolean)\nMethod isRunning() : public throws (boolean)\nMethod isUnitMode() : public throws (boolean)\nMethod isUseTLS() : public throws (boolean)\nMethod isVipChannelEnabled() : public throws (boolean)\nMethod offsetForTimestamp(java.lang.Long,org.apache.rocketmq.common.message.MessageQueue) : public throws (java.lang.Long)\nMethod pause(java.util.Collection) : public throws (void)\nMethod poll() : public throws (java.util.List)\nMethod poll(long) : public throws (java.util.List)\nMethod queueWithNamespace(org.apache.rocketmq.common.message.MessageQueue) : public throws (org.apache.rocketmq.common.message.MessageQueue)\nMethod queuesWithNamespace(java.util.Collection) : public throws (java.util.Collection)\nMethod registerTopicMessageQueueChangeListener(java.lang.String,org.apache.rocketmq.client.consumer.TopicMessageQueueChangeListener) : public throws (void)\nMethod resetClientConfig(org.apache.rocketmq.client.ClientConfig) : public throws (void)\nMethod resume(java.util.Collection) : public throws (void)\nMethod seek(long,org.apache.rocketmq.common.message.MessageQueue) : public throws (void)\nMethod seekToBegin(org.apache.rocketmq.common.message.MessageQueue) : public throws (void)\nMethod seekToEnd(org.apache.rocketmq.common.message.MessageQueue) : public throws (void)\nMethod setAccessChannel(org.apache.rocketmq.client.AccessChannel) : public throws (void)\nMethod setAllocateMessageQueueStrategy(org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy) : public throws (void)\nMethod setAutoCommit(boolean) : public throws (void)\nMethod setAutoCommitIntervalMillis(long) : public throws (void)\nMethod setClientCallbackExecutorThreads(int) : public throws (void)\nMethod setClientIP(java.lang.String) : public throws (void)\nMethod setConnectBrokerByUser(boolean) : public throws (void)\nMethod setConsumeFromWhere(org.apache.rocketmq.common.consumer.ConsumeFromWhere) : public throws (void)\nMethod setConsumeMaxSpan(int) : public throws (void)\nMethod setConsumeTimestamp(java.lang.String) : public throws (void)\nMethod setConsumerGroup(java.lang.String) : public throws (void)\nMethod setConsumerPullTimeoutMillis(long) : public throws (void)\nMethod setConsumerTimeoutMillisWhenSuspend(long) : public throws (void)\nMethod setCustomizedTraceTopic(java.lang.String) : public throws (void)\nMethod setDefaultBrokerId(long) : public throws (void)\nMethod setEnableMsgTrace(boolean) : public throws (void)\nMethod setEnableStreamRequestType(boolean) : public throws (void)\nMethod setHeartbeatBrokerInterval(int) : public throws (void)\nMethod setInstanceName(java.lang.String) : public throws (void)\nMethod setLanguage(org.apache.rocketmq.remoting.protocol.LanguageCode) : public throws (void)\nMethod setMessageModel(org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel) : public throws (void)\nMethod setMessageQueueListener(org.apache.rocketmq.client.consumer.MessageQueueListener) : public throws (void)\nMethod setMqClientApiTimeout(int) : public throws (void)\nMethod setNamespace(java.lang.String) : public throws (void)\nMethod setNamesrvAddr(java.lang.String) : public throws (void)\nMethod setOffsetStore(org.apache.rocketmq.client.consumer.store.OffsetStore) : public throws (void)\nMethod setPersistConsumerOffsetInterval(int) : public throws (void)\nMethod setPollNameServerInterval(int) : public throws (void)\nMethod setPollTimeoutMillis(long) : public throws (void)\nMethod setPullBatchSize(int) : public throws (void)\nMethod setPullThreadNums(int) : public throws (void)\nMethod setPullThresholdForAll(long) : public throws (void)\nMethod setPullThresholdForQueue(int) : public throws (void)\nMethod setPullThresholdSizeForQueue(int) : public throws (void)\nMethod setPullTimeDelayMillsWhenException(long) : public throws (void)\nMethod setTopicMetadataCheckIntervalMillis(long) : public throws (void)\nMethod setUnitMode(boolean) : public throws (void)\nMethod setUnitName(java.lang.String) : public throws (void)\nMethod setUseTLS(boolean) : public throws (void)\nMethod setVipChannelEnabled(boolean) : public throws (void)\nMethod shutdown() : public throws (void)\nMethod start() : public throws (void)\nMethod subscribe(java.lang.String,java.lang.String) : public throws (void)\nMethod subscribe(java.lang.String,org.apache.rocketmq.client.consumer.MessageSelector) : public throws (void)\nMethod toString() : public throws (java.lang.String)\nMethod unsubscribe(java.lang.String) : public throws (void)\nMethod updateNameServerAddress(java.lang.String) : public throws (void)\nMethod withNamespace(java.lang.String) : public throws (java.lang.String)\nMethod withNamespace(java.util.Set) : public throws (java.util.Set)\nMethod withoutNamespace(java.lang.String) : public throws (java.lang.String)\nMethod withoutNamespace(java.util.Set) : public throws (java.util.Set)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.consumer.DefaultMQPullConsumer.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField SEND_MESSAGE_WITH_VIP_CHANNEL_PROPERTY : public java.lang.String com.rocketmq.sendMessageWithVIPChannel\nField accessChannel : protected org.apache.rocketmq.client.AccessChannel LOCAL\nField allocateMessageQueueStrategy : private org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy null\nField brokerSuspendMaxTimeMillis : private long 20000\nField clientCallbackExecutorThreads : private int null\nField clientIP : private java.lang.String null\nField consumerGroup : private java.lang.String DEFAULT_CONSUMER\nField consumerPullTimeoutMillis : private long 10000\nField consumerTimeoutMillisWhenSuspend : private long 30000\nField defaultMQPullConsumerImpl : protected org.apache.rocketmq.client.impl.consumer.DefaultMQPullConsumerImpl null\nField enableStreamRequestType : protected boolean true\nField heartbeatBrokerInterval : private int 30000\nField instanceName : private java.lang.String DEFAULT\nField language : private org.apache.rocketmq.remoting.protocol.LanguageCode JAVA\nField maxReconsumeTimes : private int 16\nField messageModel : private org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel CLUSTERING\nField messageQueueListener : private org.apache.rocketmq.client.consumer.MessageQueueListener null\nField mqClientApiTimeout : private int 3000\nField namespace : protected java.lang.String null\nField namespaceInitialized : private boolean false\nField namesrvAddr : private java.lang.String null\nField offsetStore : private org.apache.rocketmq.client.consumer.store.OffsetStore null\nField persistConsumerOffsetInterval : private int 5000\nField pollNameServerInterval : private int 30000\nField pullTimeDelayMillsWhenException : private long 1000\nField registerTopics : private java.util.Set null\nField unitMode : private boolean false\nField unitName : private java.lang.String null\nField useTLS : private boolean false\nField vipChannelEnabled : private boolean false\nMethod buildMQClientId() : public throws (java.lang.String)\nMethod changeInstanceNameToPID() : public throws (void)\nMethod cloneClientConfig() : public throws (org.apache.rocketmq.client.ClientConfig)\nMethod createTopic(int,int,java.lang.String,java.lang.String) : public throws (void)\nMethod createTopic(int,java.lang.String,java.lang.String) : public throws (void)\nMethod earliestMsgStoreTime(org.apache.rocketmq.common.message.MessageQueue) : public throws (long)\nMethod fetchConsumeOffset(boolean,org.apache.rocketmq.common.message.MessageQueue) : public throws (long)\nMethod fetchMessageQueuesInBalance(java.lang.String) : public throws (java.util.Set)\nMethod fetchSubscribeMessageQueues(java.lang.String) : public throws (java.util.Set)\nMethod getAccessChannel() : public throws (org.apache.rocketmq.client.AccessChannel)\nMethod getAllocateMessageQueueStrategy() : public throws (org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy)\nMethod getBrokerSuspendMaxTimeMillis() : public throws (long)\nMethod getClientCallbackExecutorThreads() : public throws (int)\nMethod getClientIP() : public throws (java.lang.String)\nMethod getConsumerGroup() : public throws (java.lang.String)\nMethod getConsumerPullTimeoutMillis() : public throws (long)\nMethod getConsumerTimeoutMillisWhenSuspend() : public throws (long)\nMethod getDefaultMQPullConsumerImpl() : public throws (org.apache.rocketmq.client.impl.consumer.DefaultMQPullConsumerImpl)\nMethod getHeartbeatBrokerInterval() : public throws (int)\nMethod getInstanceName() : public throws (java.lang.String)\nMethod getLanguage() : public throws (org.apache.rocketmq.remoting.protocol.LanguageCode)\nMethod getMaxReconsumeTimes() : public throws (int)\nMethod getMessageModel() : public throws (org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel)\nMethod getMessageQueueListener() : public throws (org.apache.rocketmq.client.consumer.MessageQueueListener)\nMethod getMqClientApiTimeout() : public throws (int)\nMethod getNamespace() : public throws (java.lang.String)\nMethod getNamesrvAddr() : public throws (java.lang.String)\nMethod getOffsetStore() : public throws (org.apache.rocketmq.client.consumer.store.OffsetStore)\nMethod getPersistConsumerOffsetInterval() : public throws (int)\nMethod getPollNameServerInterval() : public throws (int)\nMethod getPullTimeDelayMillsWhenException() : public throws (long)\nMethod getRegisterTopics() : public throws (java.util.Set)\nMethod getUnitName() : public throws (java.lang.String)\nMethod isEnableStreamRequestType() : public throws (boolean)\nMethod isUnitMode() : public throws (boolean)\nMethod isUseTLS() : public throws (boolean)\nMethod isVipChannelEnabled() : public throws (boolean)\nMethod maxOffset(org.apache.rocketmq.common.message.MessageQueue) : public throws (long)\nMethod minOffset(org.apache.rocketmq.common.message.MessageQueue) : public throws (long)\nMethod pull(int,java.lang.String,long,long,org.apache.rocketmq.client.consumer.PullCallback,org.apache.rocketmq.common.message.MessageQueue) : public throws (void)\nMethod pull(int,java.lang.String,long,long,org.apache.rocketmq.common.message.MessageQueue) : public throws (org.apache.rocketmq.client.consumer.PullResult)\nMethod pull(int,java.lang.String,long,org.apache.rocketmq.client.consumer.PullCallback,org.apache.rocketmq.common.message.MessageQueue) : public throws (void)\nMethod pull(int,java.lang.String,long,org.apache.rocketmq.common.message.MessageQueue) : public throws (org.apache.rocketmq.client.consumer.PullResult)\nMethod pull(int,long,long,org.apache.rocketmq.client.consumer.MessageSelector,org.apache.rocketmq.client.consumer.PullCallback,org.apache.rocketmq.common.message.MessageQueue) : public throws (void)\nMethod pull(int,long,long,org.apache.rocketmq.client.consumer.MessageSelector,org.apache.rocketmq.common.message.MessageQueue) : public throws (org.apache.rocketmq.client.consumer.PullResult)\nMethod pull(int,long,org.apache.rocketmq.client.consumer.MessageSelector,org.apache.rocketmq.client.consumer.PullCallback,org.apache.rocketmq.common.message.MessageQueue) : public throws (void)\nMethod pull(int,long,org.apache.rocketmq.client.consumer.MessageSelector,org.apache.rocketmq.common.message.MessageQueue) : public throws (org.apache.rocketmq.client.consumer.PullResult)\nMethod pullBlockIfNotFound(int,java.lang.String,long,org.apache.rocketmq.client.consumer.PullCallback,org.apache.rocketmq.common.message.MessageQueue) : public throws (void)\nMethod pullBlockIfNotFound(int,java.lang.String,long,org.apache.rocketmq.common.message.MessageQueue) : public throws (org.apache.rocketmq.client.consumer.PullResult)\nMethod queryMessage(int,java.lang.String,java.lang.String,long,long) : public throws (org.apache.rocketmq.client.QueryResult)\nMethod queueWithNamespace(org.apache.rocketmq.common.message.MessageQueue) : public throws (org.apache.rocketmq.common.message.MessageQueue)\nMethod queuesWithNamespace(java.util.Collection) : public throws (java.util.Collection)\nMethod registerMessageQueueListener(java.lang.String,org.apache.rocketmq.client.consumer.MessageQueueListener) : public throws (void)\nMethod resetClientConfig(org.apache.rocketmq.client.ClientConfig) : public throws (void)\nMethod searchOffset(long,org.apache.rocketmq.common.message.MessageQueue) : public throws (long)\nMethod sendMessageBack(int,java.lang.String,java.lang.String,org.apache.rocketmq.common.message.MessageExt) : public throws (void)\nMethod sendMessageBack(int,java.lang.String,org.apache.rocketmq.common.message.MessageExt) : public throws (void)\nMethod sendMessageBack(int,org.apache.rocketmq.common.message.MessageExt) : public throws (void)\nMethod setAccessChannel(org.apache.rocketmq.client.AccessChannel) : public throws (void)\nMethod setAllocateMessageQueueStrategy(org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy) : public throws (void)\nMethod setBrokerSuspendMaxTimeMillis(long) : public throws (void)\nMethod setClientCallbackExecutorThreads(int) : public throws (void)\nMethod setClientIP(java.lang.String) : public throws (void)\nMethod setConsumerGroup(java.lang.String) : public throws (void)\nMethod setConsumerPullTimeoutMillis(long) : public throws (void)\nMethod setConsumerTimeoutMillisWhenSuspend(long) : public throws (void)\nMethod setEnableStreamRequestType(boolean) : public throws (void)\nMethod setHeartbeatBrokerInterval(int) : public throws (void)\nMethod setInstanceName(java.lang.String) : public throws (void)\nMethod setLanguage(org.apache.rocketmq.remoting.protocol.LanguageCode) : public throws (void)\nMethod setMaxReconsumeTimes(int) : public throws (void)\nMethod setMessageModel(org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel) : public throws (void)\nMethod setMessageQueueListener(org.apache.rocketmq.client.consumer.MessageQueueListener) : public throws (void)\nMethod setMqClientApiTimeout(int) : public throws (void)\nMethod setNamespace(java.lang.String) : public throws (void)\nMethod setNamesrvAddr(java.lang.String) : public throws (void)\nMethod setOffsetStore(org.apache.rocketmq.client.consumer.store.OffsetStore) : public throws (void)\nMethod setPersistConsumerOffsetInterval(int) : public throws (void)\nMethod setPollNameServerInterval(int) : public throws (void)\nMethod setPullTimeDelayMillsWhenException(long) : public throws (void)\nMethod setRegisterTopics(java.util.Set) : public throws (void)\nMethod setUnitMode(boolean) : public throws (void)\nMethod setUnitName(java.lang.String) : public throws (void)\nMethod setUseTLS(boolean) : public throws (void)\nMethod setVipChannelEnabled(boolean) : public throws (void)\nMethod shutdown() : public throws (void)\nMethod start() : public throws (void)\nMethod toString() : public throws (java.lang.String)\nMethod updateConsumeOffset(long,org.apache.rocketmq.common.message.MessageQueue) : public throws (void)\nMethod viewMessage(java.lang.String) : public throws (org.apache.rocketmq.common.message.MessageExt)\nMethod viewMessage(java.lang.String,java.lang.String) : public throws (org.apache.rocketmq.common.message.MessageExt)\nMethod withNamespace(java.lang.String) : public throws (java.lang.String)\nMethod withNamespace(java.util.Set) : public throws (java.util.Set)\nMethod withoutNamespace(java.lang.String) : public throws (java.lang.String)\nMethod withoutNamespace(java.util.Set) : public throws (java.util.Set)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.consumer.DefaultMQPushConsumer.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField SEND_MESSAGE_WITH_VIP_CHANNEL_PROPERTY : public java.lang.String com.rocketmq.sendMessageWithVIPChannel\nField accessChannel : protected org.apache.rocketmq.client.AccessChannel LOCAL\nField adjustThreadPoolNumsThreshold : private long 100000\nField allocateMessageQueueStrategy : private org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy null\nField awaitTerminationMillisWhenShutdown : private long 0\nField clientCallbackExecutorThreads : private int null\nField clientIP : private java.lang.String null\nField consumeConcurrentlyMaxSpan : private int 2000\nField consumeFromWhere : private org.apache.rocketmq.common.consumer.ConsumeFromWhere CONSUME_FROM_LAST_OFFSET\nField consumeMessageBatchMaxSize : private int 1\nField consumeThreadMax : private int 20\nField consumeThreadMin : private int 20\nField consumeTimeout : private long 15\nField consumeTimestamp : private java.lang.String null\nField consumerGroup : private java.lang.String DEFAULT_CONSUMER\nField defaultMQPushConsumerImpl : protected org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl null\nField enableStreamRequestType : protected boolean false\nField heartbeatBrokerInterval : private int 30000\nField instanceName : private java.lang.String DEFAULT\nField language : private org.apache.rocketmq.remoting.protocol.LanguageCode JAVA\nField log : private org.apache.rocketmq.logging.InternalLogger null\nField maxReconsumeTimes : private int -1\nField messageListener : private org.apache.rocketmq.client.consumer.listener.MessageListener null\nField messageModel : private org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel CLUSTERING\nField mqClientApiTimeout : private int 3000\nField namespace : protected java.lang.String null\nField namespaceInitialized : private boolean false\nField namesrvAddr : private java.lang.String null\nField offsetStore : private org.apache.rocketmq.client.consumer.store.OffsetStore null\nField persistConsumerOffsetInterval : private int 5000\nField pollNameServerInterval : private int 30000\nField postSubscriptionWhenPull : private boolean false\nField pullBatchSize : private int 32\nField pullInterval : private long 0\nField pullThresholdForQueue : private int 1000\nField pullThresholdForTopic : private int -1\nField pullThresholdSizeForQueue : private int 100\nField pullThresholdSizeForTopic : private int -1\nField pullTimeDelayMillsWhenException : private long 1000\nField subscription : private java.util.Map null\nField suspendCurrentQueueTimeMillis : private long 1000\nField traceDispatcher : private org.apache.rocketmq.client.trace.TraceDispatcher null\nField unitMode : private boolean false\nField unitName : private java.lang.String null\nField useTLS : private boolean false\nField vipChannelEnabled : private boolean false\nMethod buildMQClientId() : public throws (java.lang.String)\nMethod changeInstanceNameToPID() : public throws (void)\nMethod cloneClientConfig() : public throws (org.apache.rocketmq.client.ClientConfig)\nMethod createTopic(int,int,java.lang.String,java.lang.String) : public throws (void)\nMethod createTopic(int,java.lang.String,java.lang.String) : public throws (void)\nMethod earliestMsgStoreTime(org.apache.rocketmq.common.message.MessageQueue) : public throws (long)\nMethod fetchSubscribeMessageQueues(java.lang.String) : public throws (java.util.Set)\nMethod getAccessChannel() : public throws (org.apache.rocketmq.client.AccessChannel)\nMethod getAdjustThreadPoolNumsThreshold() : public throws (long)\nMethod getAllocateMessageQueueStrategy() : public throws (org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy)\nMethod getAwaitTerminationMillisWhenShutdown() : public throws (long)\nMethod getClientCallbackExecutorThreads() : public throws (int)\nMethod getClientIP() : public throws (java.lang.String)\nMethod getConsumeConcurrentlyMaxSpan() : public throws (int)\nMethod getConsumeFromWhere() : public throws (org.apache.rocketmq.common.consumer.ConsumeFromWhere)\nMethod getConsumeMessageBatchMaxSize() : public throws (int)\nMethod getConsumeThreadMax() : public throws (int)\nMethod getConsumeThreadMin() : public throws (int)\nMethod getConsumeTimeout() : public throws (long)\nMethod getConsumeTimestamp() : public throws (java.lang.String)\nMethod getConsumerGroup() : public throws (java.lang.String)\nMethod getDefaultMQPushConsumerImpl() : public throws (org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl)\nMethod getHeartbeatBrokerInterval() : public throws (int)\nMethod getInstanceName() : public throws (java.lang.String)\nMethod getLanguage() : public throws (org.apache.rocketmq.remoting.protocol.LanguageCode)\nMethod getMaxReconsumeTimes() : public throws (int)\nMethod getMessageListener() : public throws (org.apache.rocketmq.client.consumer.listener.MessageListener)\nMethod getMessageModel() : public throws (org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel)\nMethod getMqClientApiTimeout() : public throws (int)\nMethod getNamespace() : public throws (java.lang.String)\nMethod getNamesrvAddr() : public throws (java.lang.String)\nMethod getOffsetStore() : public throws (org.apache.rocketmq.client.consumer.store.OffsetStore)\nMethod getPersistConsumerOffsetInterval() : public throws (int)\nMethod getPollNameServerInterval() : public throws (int)\nMethod getPullBatchSize() : public throws (int)\nMethod getPullInterval() : public throws (long)\nMethod getPullThresholdForQueue() : public throws (int)\nMethod getPullThresholdForTopic() : public throws (int)\nMethod getPullThresholdSizeForQueue() : public throws (int)\nMethod getPullThresholdSizeForTopic() : public throws (int)\nMethod getPullTimeDelayMillsWhenException() : public throws (long)\nMethod getSubscription() : public throws (java.util.Map)\nMethod getSuspendCurrentQueueTimeMillis() : public throws (long)\nMethod getTraceDispatcher() : public throws (org.apache.rocketmq.client.trace.TraceDispatcher)\nMethod getUnitName() : public throws (java.lang.String)\nMethod isEnableStreamRequestType() : public throws (boolean)\nMethod isPostSubscriptionWhenPull() : public throws (boolean)\nMethod isUnitMode() : public throws (boolean)\nMethod isUseTLS() : public throws (boolean)\nMethod isVipChannelEnabled() : public throws (boolean)\nMethod maxOffset(org.apache.rocketmq.common.message.MessageQueue) : public throws (long)\nMethod minOffset(org.apache.rocketmq.common.message.MessageQueue) : public throws (long)\nMethod queryMessage(int,java.lang.String,java.lang.String,long,long) : public throws (org.apache.rocketmq.client.QueryResult)\nMethod queueWithNamespace(org.apache.rocketmq.common.message.MessageQueue) : public throws (org.apache.rocketmq.common.message.MessageQueue)\nMethod queuesWithNamespace(java.util.Collection) : public throws (java.util.Collection)\nMethod registerMessageListener(org.apache.rocketmq.client.consumer.listener.MessageListener) : public throws (void)\nMethod registerMessageListener(org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently) : public throws (void)\nMethod registerMessageListener(org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly) : public throws (void)\nMethod resetClientConfig(org.apache.rocketmq.client.ClientConfig) : public throws (void)\nMethod resume() : public throws (void)\nMethod searchOffset(long,org.apache.rocketmq.common.message.MessageQueue) : public throws (long)\nMethod sendMessageBack(int,java.lang.String,org.apache.rocketmq.common.message.MessageExt) : public throws (void)\nMethod sendMessageBack(int,org.apache.rocketmq.common.message.MessageExt) : public throws (void)\nMethod setAccessChannel(org.apache.rocketmq.client.AccessChannel) : public throws (void)\nMethod setAdjustThreadPoolNumsThreshold(long) : public throws (void)\nMethod setAllocateMessageQueueStrategy(org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy) : public throws (void)\nMethod setAwaitTerminationMillisWhenShutdown(long) : public throws (void)\nMethod setClientCallbackExecutorThreads(int) : public throws (void)\nMethod setClientIP(java.lang.String) : public throws (void)\nMethod setConsumeConcurrentlyMaxSpan(int) : public throws (void)\nMethod setConsumeFromWhere(org.apache.rocketmq.common.consumer.ConsumeFromWhere) : public throws (void)\nMethod setConsumeMessageBatchMaxSize(int) : public throws (void)\nMethod setConsumeThreadMax(int) : public throws (void)\nMethod setConsumeThreadMin(int) : public throws (void)\nMethod setConsumeTimeout(long) : public throws (void)\nMethod setConsumeTimestamp(java.lang.String) : public throws (void)\nMethod setConsumerGroup(java.lang.String) : public throws (void)\nMethod setEnableStreamRequestType(boolean) : public throws (void)\nMethod setHeartbeatBrokerInterval(int) : public throws (void)\nMethod setInstanceName(java.lang.String) : public throws (void)\nMethod setLanguage(org.apache.rocketmq.remoting.protocol.LanguageCode) : public throws (void)\nMethod setMaxReconsumeTimes(int) : public throws (void)\nMethod setMessageListener(org.apache.rocketmq.client.consumer.listener.MessageListener) : public throws (void)\nMethod setMessageModel(org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel) : public throws (void)\nMethod setMqClientApiTimeout(int) : public throws (void)\nMethod setNamespace(java.lang.String) : public throws (void)\nMethod setNamesrvAddr(java.lang.String) : public throws (void)\nMethod setOffsetStore(org.apache.rocketmq.client.consumer.store.OffsetStore) : public throws (void)\nMethod setPersistConsumerOffsetInterval(int) : public throws (void)\nMethod setPollNameServerInterval(int) : public throws (void)\nMethod setPostSubscriptionWhenPull(boolean) : public throws (void)\nMethod setPullBatchSize(int) : public throws (void)\nMethod setPullInterval(long) : public throws (void)\nMethod setPullThresholdForQueue(int) : public throws (void)\nMethod setPullThresholdForTopic(int) : public throws (void)\nMethod setPullThresholdSizeForQueue(int) : public throws (void)\nMethod setPullThresholdSizeForTopic(int) : public throws (void)\nMethod setPullTimeDelayMillsWhenException(long) : public throws (void)\nMethod setSubscription(java.util.Map) : public throws (void)\nMethod setSuspendCurrentQueueTimeMillis(long) : public throws (void)\nMethod setUnitMode(boolean) : public throws (void)\nMethod setUnitName(java.lang.String) : public throws (void)\nMethod setUseTLS(boolean) : public throws (void)\nMethod setVipChannelEnabled(boolean) : public throws (void)\nMethod shutdown() : public throws (void)\nMethod start() : public throws (void)\nMethod subscribe(java.lang.String,java.lang.String) : public throws (void)\nMethod subscribe(java.lang.String,java.lang.String,java.lang.String) : public throws (void)\nMethod subscribe(java.lang.String,org.apache.rocketmq.client.consumer.MessageSelector) : public throws (void)\nMethod suspend() : public throws (void)\nMethod toString() : public throws (java.lang.String)\nMethod unsubscribe(java.lang.String) : public throws (void)\nMethod updateCorePoolSize(int) : public throws (void)\nMethod viewMessage(java.lang.String) : public throws (org.apache.rocketmq.common.message.MessageExt)\nMethod viewMessage(java.lang.String,java.lang.String) : public throws (org.apache.rocketmq.common.message.MessageExt)\nMethod withNamespace(java.lang.String) : public throws (java.lang.String)\nMethod withNamespace(java.util.Set) : public throws (java.util.Set)\nMethod withoutNamespace(java.lang.String) : public throws (java.lang.String)\nMethod withoutNamespace(java.util.Set) : public throws (java.util.Set)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.consumer.PullCallback.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nMethod onException(java.lang.Throwable) : public throws (void)\nMethod onSuccess(org.apache.rocketmq.client.consumer.PullResult) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.consumer.PullResult.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField maxOffset : private long 0\nField minOffset : private long 0\nField msgFoundList : private java.util.List null\nField nextBeginOffset : private long 0\nField pullStatus : private org.apache.rocketmq.client.consumer.PullStatus FOUND\nMethod getMaxOffset() : public throws (long)\nMethod getMinOffset() : public throws (long)\nMethod getMsgFoundList() : public throws (java.util.List)\nMethod getNextBeginOffset() : public throws (long)\nMethod getPullStatus() : public throws (org.apache.rocketmq.client.consumer.PullStatus)\nMethod setMsgFoundList(java.util.List) : public throws (void)\nMethod toString() : public throws (java.lang.String)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.consumer.PullStatus.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField FOUND : public int 0\nField NO_MATCHED_MSG : public int 2\nField NO_NEW_MSG : public int 1\nField OFFSET_ILLEGAL : public int 3\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.consumer.listener.ConsumeConcurrentlyContext.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField ackIndex : private int 2147483647\nField delayLevelWhenNextConsume : private int 0\nField messageQueue : private org.apache.rocketmq.common.message.MessageQueue null\nMethod getAckIndex() : public throws (int)\nMethod getDelayLevelWhenNextConsume() : public throws (int)\nMethod getMessageQueue() : public throws (org.apache.rocketmq.common.message.MessageQueue)\nMethod setAckIndex(int) : public throws (void)\nMethod setDelayLevelWhenNextConsume(int) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.consumer.listener.ConsumeConcurrentlyStatus.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField CONSUME_SUCCESS : public int 0\nField RECONSUME_LATER : public int 1\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.consumer.listener.ConsumeOrderlyContext.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField autoCommit : private boolean true\nField messageQueue : private org.apache.rocketmq.common.message.MessageQueue null\nField suspendCurrentQueueTimeMillis : private long -1\nMethod getMessageQueue() : public throws (org.apache.rocketmq.common.message.MessageQueue)\nMethod getSuspendCurrentQueueTimeMillis() : public throws (long)\nMethod isAutoCommit() : public throws (boolean)\nMethod setAutoCommit(boolean) : public throws (void)\nMethod setSuspendCurrentQueueTimeMillis(long) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.consumer.listener.ConsumeOrderlyStatus.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField COMMIT : public int 2\nField ROLLBACK : public int 1\nField SUCCESS : public int 0\nField SUSPEND_CURRENT_QUEUE_A_MOMENT : public int 3\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.consumer.listener.MessageListener.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.consumer.listener.MessageListenerConcurrently.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nMethod consumeMessage(java.util.List,org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext) : public throws (org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.consumer.listener.MessageListenerOrderly.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nMethod consumeMessage(java.util.List,org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext) : public throws (org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.hook.CheckForbiddenHook.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nMethod checkForbidden(org.apache.rocketmq.client.hook.CheckForbiddenContext) : public throws (void)\nMethod hookName() : public throws (java.lang.String)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.hook.ConsumeMessageContext.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField consumerGroup : private java.lang.String null\nField mq : private org.apache.rocketmq.common.message.MessageQueue null\nField mqTraceContext : private java.lang.Object null\nField msgList : private java.util.List null\nField namespace : private java.lang.String null\nField props : private java.util.Map null\nField status : private java.lang.String null\nField success : private boolean false\nMethod getConsumerGroup() : public throws (java.lang.String)\nMethod getMq() : public throws (org.apache.rocketmq.common.message.MessageQueue)\nMethod getMqTraceContext() : public throws (java.lang.Object)\nMethod getMsgList() : public throws (java.util.List)\nMethod getNamespace() : public throws (java.lang.String)\nMethod getProps() : public throws (java.util.Map)\nMethod getStatus() : public throws (java.lang.String)\nMethod isSuccess() : public throws (boolean)\nMethod setConsumerGroup(java.lang.String) : public throws (void)\nMethod setMq(org.apache.rocketmq.common.message.MessageQueue) : public throws (void)\nMethod setMqTraceContext(java.lang.Object) : public throws (void)\nMethod setMsgList(java.util.List) : public throws (void)\nMethod setNamespace(java.lang.String) : public throws (void)\nMethod setProps(java.util.Map) : public throws (void)\nMethod setStatus(java.lang.String) : public throws (void)\nMethod setSuccess(boolean) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.hook.ConsumeMessageHook.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nMethod consumeMessageAfter(org.apache.rocketmq.client.hook.ConsumeMessageContext) : public throws (void)\nMethod consumeMessageBefore(org.apache.rocketmq.client.hook.ConsumeMessageContext) : public throws (void)\nMethod hookName() : public throws (java.lang.String)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.hook.EndTransactionContext.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField brokerAddr : private java.lang.String null\nField fromTransactionCheck : private boolean false\nField message : private org.apache.rocketmq.common.message.Message null\nField msgId : private java.lang.String null\nField producerGroup : private java.lang.String null\nField transactionId : private java.lang.String null\nField transactionState : private org.apache.rocketmq.client.producer.LocalTransactionState null\nMethod getBrokerAddr() : public throws (java.lang.String)\nMethod getMessage() : public throws (org.apache.rocketmq.common.message.Message)\nMethod getMsgId() : public throws (java.lang.String)\nMethod getProducerGroup() : public throws (java.lang.String)\nMethod getTransactionId() : public throws (java.lang.String)\nMethod getTransactionState() : public throws (org.apache.rocketmq.client.producer.LocalTransactionState)\nMethod isFromTransactionCheck() : public throws (boolean)\nMethod setBrokerAddr(java.lang.String) : public throws (void)\nMethod setFromTransactionCheck(boolean) : public throws (void)\nMethod setMessage(org.apache.rocketmq.common.message.Message) : public throws (void)\nMethod setMsgId(java.lang.String) : public throws (void)\nMethod setProducerGroup(java.lang.String) : public throws (void)\nMethod setTransactionId(java.lang.String) : public throws (void)\nMethod setTransactionState(org.apache.rocketmq.client.producer.LocalTransactionState) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.hook.EndTransactionHook.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nMethod endTransaction(org.apache.rocketmq.client.hook.EndTransactionContext) : public throws (void)\nMethod hookName() : public throws (java.lang.String)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.hook.FilterMessageContext.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField arg : private java.lang.Object null\nField consumerGroup : private java.lang.String null\nField mq : private org.apache.rocketmq.common.message.MessageQueue null\nField msgList : private java.util.List null\nField unitMode : private boolean false\nMethod getArg() : public throws (java.lang.Object)\nMethod getConsumerGroup() : public throws (java.lang.String)\nMethod getMq() : public throws (org.apache.rocketmq.common.message.MessageQueue)\nMethod getMsgList() : public throws (java.util.List)\nMethod isUnitMode() : public throws (boolean)\nMethod setArg(java.lang.Object) : public throws (void)\nMethod setConsumerGroup(java.lang.String) : public throws (void)\nMethod setMq(org.apache.rocketmq.common.message.MessageQueue) : public throws (void)\nMethod setMsgList(java.util.List) : public throws (void)\nMethod setUnitMode(boolean) : public throws (void)\nMethod toString() : public throws (java.lang.String)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.hook.FilterMessageHook.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nMethod filterMessage(org.apache.rocketmq.client.hook.FilterMessageContext) : public throws (void)\nMethod hookName() : public throws (java.lang.String)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.hook.SendMessageContext.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField bornHost : private java.lang.String null\nField brokerAddr : private java.lang.String null\nField communicationMode : private org.apache.rocketmq.client.impl.CommunicationMode null\nField exception : private java.lang.Exception null\nField message : private org.apache.rocketmq.common.message.Message null\nField mq : private org.apache.rocketmq.common.message.MessageQueue null\nField mqTraceContext : private java.lang.Object null\nField msgType : private org.apache.rocketmq.common.message.MessageType Normal_Msg\nField namespace : private java.lang.String null\nField producer : private org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl null\nField producerGroup : private java.lang.String null\nField props : private java.util.Map null\nField sendResult : private org.apache.rocketmq.client.producer.SendResult null\nMethod getBornHost() : public throws (java.lang.String)\nMethod getBrokerAddr() : public throws (java.lang.String)\nMethod getCommunicationMode() : public throws (org.apache.rocketmq.client.impl.CommunicationMode)\nMethod getException() : public throws (java.lang.Exception)\nMethod getMessage() : public throws (org.apache.rocketmq.common.message.Message)\nMethod getMq() : public throws (org.apache.rocketmq.common.message.MessageQueue)\nMethod getMqTraceContext() : public throws (java.lang.Object)\nMethod getMsgType() : public throws (org.apache.rocketmq.common.message.MessageType)\nMethod getNamespace() : public throws (java.lang.String)\nMethod getProducer() : public throws (org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl)\nMethod getProducerGroup() : public throws (java.lang.String)\nMethod getProps() : public throws (java.util.Map)\nMethod getSendResult() : public throws (org.apache.rocketmq.client.producer.SendResult)\nMethod setBornHost(java.lang.String) : public throws (void)\nMethod setBrokerAddr(java.lang.String) : public throws (void)\nMethod setCommunicationMode(org.apache.rocketmq.client.impl.CommunicationMode) : public throws (void)\nMethod setException(java.lang.Exception) : public throws (void)\nMethod setMessage(org.apache.rocketmq.common.message.Message) : public throws (void)\nMethod setMq(org.apache.rocketmq.common.message.MessageQueue) : public throws (void)\nMethod setMqTraceContext(java.lang.Object) : public throws (void)\nMethod setMsgType(org.apache.rocketmq.common.message.MessageType) : public throws (void)\nMethod setNamespace(java.lang.String) : public throws (void)\nMethod setProducer(org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl) : public throws (void)\nMethod setProducerGroup(java.lang.String) : public throws (void)\nMethod setProps(java.util.Map) : public throws (void)\nMethod setSendResult(org.apache.rocketmq.client.producer.SendResult) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.hook.SendMessageHook.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nMethod hookName() : public throws (java.lang.String)\nMethod sendMessageAfter(org.apache.rocketmq.client.hook.SendMessageContext) : public throws (void)\nMethod sendMessageBefore(org.apache.rocketmq.client.hook.SendMessageContext) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.producer.DefaultMQProducer.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField SEND_MESSAGE_WITH_VIP_CHANNEL_PROPERTY : public java.lang.String com.rocketmq.sendMessageWithVIPChannel\nField accessChannel : protected org.apache.rocketmq.client.AccessChannel LOCAL\nField clientCallbackExecutorThreads : private int null\nField clientIP : private java.lang.String null\nField compressMsgBodyOverHowmuch : private int 4096\nField createTopicKey : private java.lang.String TBW102\nField defaultMQProducerImpl : protected org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl null\nField defaultTopicQueueNums : private int 4\nField enableStreamRequestType : protected boolean false\nField heartbeatBrokerInterval : private int 30000\nField instanceName : private java.lang.String DEFAULT\nField language : private org.apache.rocketmq.remoting.protocol.LanguageCode JAVA\nField log : private org.apache.rocketmq.logging.InternalLogger null\nField maxMessageSize : private int 4194304\nField mqClientApiTimeout : private int 3000\nField namespace : protected java.lang.String null\nField namespaceInitialized : private boolean false\nField namesrvAddr : private java.lang.String null\nField persistConsumerOffsetInterval : private int 5000\nField pollNameServerInterval : private int 30000\nField producerGroup : private java.lang.String DEFAULT_PRODUCER\nField pullTimeDelayMillsWhenException : private long 1000\nField retryAnotherBrokerWhenNotStoreOK : private boolean false\nField retryResponseCodes : private java.util.Set null\nField retryTimesWhenSendAsyncFailed : private int 2\nField retryTimesWhenSendFailed : private int 2\nField sendMsgTimeout : private int 3000\nField traceDispatcher : private org.apache.rocketmq.client.trace.TraceDispatcher null\nField unitMode : private boolean false\nField unitName : private java.lang.String null\nField useTLS : private boolean false\nField vipChannelEnabled : private boolean false\nMethod addRetryResponseCode(int) : public throws (void)\nMethod buildMQClientId() : public throws (java.lang.String)\nMethod changeInstanceNameToPID() : public throws (void)\nMethod cloneClientConfig() : public throws (org.apache.rocketmq.client.ClientConfig)\nMethod createTopic(int,int,java.lang.String,java.lang.String) : public throws (void)\nMethod createTopic(int,java.lang.String,java.lang.String) : public throws (void)\nMethod earliestMsgStoreTime(org.apache.rocketmq.common.message.MessageQueue) : public throws (long)\nMethod fetchPublishMessageQueues(java.lang.String) : public throws (java.util.List)\nMethod getAccessChannel() : public throws (org.apache.rocketmq.client.AccessChannel)\nMethod getClientCallbackExecutorThreads() : public throws (int)\nMethod getClientIP() : public throws (java.lang.String)\nMethod getCompressMsgBodyOverHowmuch() : public throws (int)\nMethod getCreateTopicKey() : public throws (java.lang.String)\nMethod getDefaultMQProducerImpl() : public throws (org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl)\nMethod getDefaultTopicQueueNums() : public throws (int)\nMethod getHeartbeatBrokerInterval() : public throws (int)\nMethod getInstanceName() : public throws (java.lang.String)\nMethod getLanguage() : public throws (org.apache.rocketmq.remoting.protocol.LanguageCode)\nMethod getLatencyMax() : public throws ([J)\nMethod getMaxMessageSize() : public throws (int)\nMethod getMqClientApiTimeout() : public throws (int)\nMethod getNamespace() : public throws (java.lang.String)\nMethod getNamesrvAddr() : public throws (java.lang.String)\nMethod getNotAvailableDuration() : public throws ([J)\nMethod getPersistConsumerOffsetInterval() : public throws (int)\nMethod getPollNameServerInterval() : public throws (int)\nMethod getProducerGroup() : public throws (java.lang.String)\nMethod getPullTimeDelayMillsWhenException() : public throws (long)\nMethod getRetryResponseCodes() : public throws (java.util.Set)\nMethod getRetryTimesWhenSendAsyncFailed() : public throws (int)\nMethod getRetryTimesWhenSendFailed() : public throws (int)\nMethod getSendMsgTimeout() : public throws (int)\nMethod getTraceDispatcher() : public throws (org.apache.rocketmq.client.trace.TraceDispatcher)\nMethod getUnitName() : public throws (java.lang.String)\nMethod isEnableStreamRequestType() : public throws (boolean)\nMethod isRetryAnotherBrokerWhenNotStoreOK() : public throws (boolean)\nMethod isSendLatencyFaultEnable() : public throws (boolean)\nMethod isSendMessageWithVIPChannel() : public throws (boolean)\nMethod isUnitMode() : public throws (boolean)\nMethod isUseTLS() : public throws (boolean)\nMethod isVipChannelEnabled() : public throws (boolean)\nMethod maxOffset(org.apache.rocketmq.common.message.MessageQueue) : public throws (long)\nMethod minOffset(org.apache.rocketmq.common.message.MessageQueue) : public throws (long)\nMethod queryMessage(int,java.lang.String,java.lang.String,long,long) : public throws (org.apache.rocketmq.client.QueryResult)\nMethod queueWithNamespace(org.apache.rocketmq.common.message.MessageQueue) : public throws (org.apache.rocketmq.common.message.MessageQueue)\nMethod queuesWithNamespace(java.util.Collection) : public throws (java.util.Collection)\nMethod request(java.lang.Object,long,org.apache.rocketmq.client.producer.MessageQueueSelector,org.apache.rocketmq.client.producer.RequestCallback,org.apache.rocketmq.common.message.Message) : public throws (void)\nMethod request(java.lang.Object,long,org.apache.rocketmq.client.producer.MessageQueueSelector,org.apache.rocketmq.common.message.Message) : public throws (org.apache.rocketmq.common.message.Message)\nMethod request(long,org.apache.rocketmq.client.producer.RequestCallback,org.apache.rocketmq.common.message.Message) : public throws (void)\nMethod request(long,org.apache.rocketmq.client.producer.RequestCallback,org.apache.rocketmq.common.message.Message,org.apache.rocketmq.common.message.MessageQueue) : public throws (void)\nMethod request(long,org.apache.rocketmq.common.message.Message) : public throws (org.apache.rocketmq.common.message.Message)\nMethod request(long,org.apache.rocketmq.common.message.Message,org.apache.rocketmq.common.message.MessageQueue) : public throws (org.apache.rocketmq.common.message.Message)\nMethod resetClientConfig(org.apache.rocketmq.client.ClientConfig) : public throws (void)\nMethod searchOffset(long,org.apache.rocketmq.common.message.MessageQueue) : public throws (long)\nMethod send(java.lang.Object,long,org.apache.rocketmq.client.producer.MessageQueueSelector,org.apache.rocketmq.client.producer.SendCallback,org.apache.rocketmq.common.message.Message) : public throws (void)\nMethod send(java.lang.Object,long,org.apache.rocketmq.client.producer.MessageQueueSelector,org.apache.rocketmq.common.message.Message) : public throws (org.apache.rocketmq.client.producer.SendResult)\nMethod send(java.lang.Object,org.apache.rocketmq.client.producer.MessageQueueSelector,org.apache.rocketmq.client.producer.SendCallback,org.apache.rocketmq.common.message.Message) : public throws (void)\nMethod send(java.lang.Object,org.apache.rocketmq.client.producer.MessageQueueSelector,org.apache.rocketmq.common.message.Message) : public throws (org.apache.rocketmq.client.producer.SendResult)\nMethod send(java.util.Collection) : public throws (org.apache.rocketmq.client.producer.SendResult)\nMethod send(java.util.Collection,long) : public throws (org.apache.rocketmq.client.producer.SendResult)\nMethod send(java.util.Collection,long,org.apache.rocketmq.client.producer.SendCallback) : public throws (void)\nMethod send(java.util.Collection,long,org.apache.rocketmq.client.producer.SendCallback,org.apache.rocketmq.common.message.MessageQueue) : public throws (void)\nMethod send(java.util.Collection,long,org.apache.rocketmq.common.message.MessageQueue) : public throws (org.apache.rocketmq.client.producer.SendResult)\nMethod send(java.util.Collection,org.apache.rocketmq.client.producer.SendCallback) : public throws (void)\nMethod send(java.util.Collection,org.apache.rocketmq.client.producer.SendCallback,org.apache.rocketmq.common.message.MessageQueue) : public throws (void)\nMethod send(java.util.Collection,org.apache.rocketmq.common.message.MessageQueue) : public throws (org.apache.rocketmq.client.producer.SendResult)\nMethod send(long,org.apache.rocketmq.client.producer.SendCallback,org.apache.rocketmq.common.message.Message) : public throws (void)\nMethod send(long,org.apache.rocketmq.client.producer.SendCallback,org.apache.rocketmq.common.message.Message,org.apache.rocketmq.common.message.MessageQueue) : public throws (void)\nMethod send(long,org.apache.rocketmq.common.message.Message) : public throws (org.apache.rocketmq.client.producer.SendResult)\nMethod send(long,org.apache.rocketmq.common.message.Message,org.apache.rocketmq.common.message.MessageQueue) : public throws (org.apache.rocketmq.client.producer.SendResult)\nMethod send(org.apache.rocketmq.client.producer.SendCallback,org.apache.rocketmq.common.message.Message) : public throws (void)\nMethod send(org.apache.rocketmq.client.producer.SendCallback,org.apache.rocketmq.common.message.Message,org.apache.rocketmq.common.message.MessageQueue) : public throws (void)\nMethod send(org.apache.rocketmq.common.message.Message) : public throws (org.apache.rocketmq.client.producer.SendResult)\nMethod send(org.apache.rocketmq.common.message.Message,org.apache.rocketmq.common.message.MessageQueue) : public throws (org.apache.rocketmq.client.producer.SendResult)\nMethod sendMessageInTransaction(java.lang.Object,org.apache.rocketmq.common.message.Message) : public throws (org.apache.rocketmq.client.producer.TransactionSendResult)\nMethod sendOneway(java.lang.Object,org.apache.rocketmq.client.producer.MessageQueueSelector,org.apache.rocketmq.common.message.Message) : public throws (void)\nMethod sendOneway(org.apache.rocketmq.common.message.Message) : public throws (void)\nMethod sendOneway(org.apache.rocketmq.common.message.Message,org.apache.rocketmq.common.message.MessageQueue) : public throws (void)\nMethod setAccessChannel(org.apache.rocketmq.client.AccessChannel) : public throws (void)\nMethod setAsyncSenderExecutor(java.util.concurrent.ExecutorService) : public throws (void)\nMethod setCallbackExecutor(java.util.concurrent.ExecutorService) : public throws (void)\nMethod setClientCallbackExecutorThreads(int) : public throws (void)\nMethod setClientIP(java.lang.String) : public throws (void)\nMethod setCompressMsgBodyOverHowmuch(int) : public throws (void)\nMethod setCreateTopicKey(java.lang.String) : public throws (void)\nMethod setDefaultTopicQueueNums(int) : public throws (void)\nMethod setEnableStreamRequestType(boolean) : public throws (void)\nMethod setHeartbeatBrokerInterval(int) : public throws (void)\nMethod setInstanceName(java.lang.String) : public throws (void)\nMethod setLanguage(org.apache.rocketmq.remoting.protocol.LanguageCode) : public throws (void)\nMethod setLatencyMax([J) : public throws (void)\nMethod setMaxMessageSize(int) : public throws (void)\nMethod setMqClientApiTimeout(int) : public throws (void)\nMethod setNamespace(java.lang.String) : public throws (void)\nMethod setNamesrvAddr(java.lang.String) : public throws (void)\nMethod setNotAvailableDuration([J) : public throws (void)\nMethod setPersistConsumerOffsetInterval(int) : public throws (void)\nMethod setPollNameServerInterval(int) : public throws (void)\nMethod setProducerGroup(java.lang.String) : public throws (void)\nMethod setPullTimeDelayMillsWhenException(long) : public throws (void)\nMethod setRetryAnotherBrokerWhenNotStoreOK(boolean) : public throws (void)\nMethod setRetryTimesWhenSendAsyncFailed(int) : public throws (void)\nMethod setRetryTimesWhenSendFailed(int) : public throws (void)\nMethod setSendLatencyFaultEnable(boolean) : public throws (void)\nMethod setSendMessageWithVIPChannel(boolean) : public throws (void)\nMethod setSendMsgTimeout(int) : public throws (void)\nMethod setUnitMode(boolean) : public throws (void)\nMethod setUnitName(java.lang.String) : public throws (void)\nMethod setUseTLS(boolean) : public throws (void)\nMethod setVipChannelEnabled(boolean) : public throws (void)\nMethod shutdown() : public throws (void)\nMethod start() : public throws (void)\nMethod toString() : public throws (java.lang.String)\nMethod viewMessage(java.lang.String) : public throws (org.apache.rocketmq.common.message.MessageExt)\nMethod viewMessage(java.lang.String,java.lang.String) : public throws (org.apache.rocketmq.common.message.MessageExt)\nMethod withNamespace(java.lang.String) : public throws (java.lang.String)\nMethod withNamespace(java.util.Set) : public throws (java.util.Set)\nMethod withoutNamespace(java.lang.String) : public throws (java.lang.String)\nMethod withoutNamespace(java.util.Set) : public throws (java.util.Set)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.producer.MessageQueueSelector.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nMethod select(java.lang.Object,java.util.List,org.apache.rocketmq.common.message.Message) : public throws (org.apache.rocketmq.common.message.MessageQueue)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.producer.SendCallback.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nMethod onException(java.lang.Throwable) : public throws (void)\nMethod onSuccess(org.apache.rocketmq.client.producer.SendResult) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.producer.SendResult.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField messageQueue : private org.apache.rocketmq.common.message.MessageQueue null\nField msgId : private java.lang.String null\nField offsetMsgId : private java.lang.String null\nField queueOffset : private long 0\nField regionId : private java.lang.String null\nField sendStatus : private org.apache.rocketmq.client.producer.SendStatus null\nField traceOn : private boolean true\nField transactionId : private java.lang.String null\nMethod decoderSendResultFromJson(java.lang.String) : public throws (org.apache.rocketmq.client.producer.SendResult)\nMethod encoderSendResultToJson(java.lang.Object) : public throws (java.lang.String)\nMethod getMessageQueue() : public throws (org.apache.rocketmq.common.message.MessageQueue)\nMethod getMsgId() : public throws (java.lang.String)\nMethod getOffsetMsgId() : public throws (java.lang.String)\nMethod getQueueOffset() : public throws (long)\nMethod getRegionId() : public throws (java.lang.String)\nMethod getSendStatus() : public throws (org.apache.rocketmq.client.producer.SendStatus)\nMethod getTransactionId() : public throws (java.lang.String)\nMethod isTraceOn() : public throws (boolean)\nMethod setMessageQueue(org.apache.rocketmq.common.message.MessageQueue) : public throws (void)\nMethod setMsgId(java.lang.String) : public throws (void)\nMethod setOffsetMsgId(java.lang.String) : public throws (void)\nMethod setQueueOffset(long) : public throws (void)\nMethod setRegionId(java.lang.String) : public throws (void)\nMethod setSendStatus(org.apache.rocketmq.client.producer.SendStatus) : public throws (void)\nMethod setTraceOn(boolean) : public throws (void)\nMethod setTransactionId(java.lang.String) : public throws (void)\nMethod toString() : public throws (java.lang.String)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/client.producer.SendStatus.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField FLUSH_DISK_TIMEOUT : public int 1\nField FLUSH_SLAVE_TIMEOUT : public int 2\nField SEND_OK : public int 0\nField SLAVE_NOT_AVAILABLE : public int 3\n"
  },
  {
    "path": "test/src/test/resources/schema/api/common.message.Message.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField body : private [B null\nField flag : private int 0\nField properties : private java.util.Map null\nField serialVersionUID : private long 8445773977080406428\nField topic : private java.lang.String null\nField transactionId : private java.lang.String null\nMethod getBody() : public throws ([B)\nMethod getBuyerId() : public throws (java.lang.String)\nMethod getDelayTimeLevel() : public throws (int)\nMethod getFlag() : public throws (int)\nMethod getKeys() : public throws (java.lang.String)\nMethod getProperties() : public throws (java.util.Map)\nMethod getProperty(java.lang.String) : public throws (java.lang.String)\nMethod getTags() : public throws (java.lang.String)\nMethod getTopic() : public throws (java.lang.String)\nMethod getTransactionId() : public throws (java.lang.String)\nMethod getUserProperty(java.lang.String) : public throws (java.lang.String)\nMethod isWaitStoreMsgOK() : public throws (boolean)\nMethod putUserProperty(java.lang.String,java.lang.String) : public throws (void)\nMethod setBody([B) : public throws (void)\nMethod setBuyerId(java.lang.String) : public throws (void)\nMethod setDelayTimeLevel(int) : public throws (void)\nMethod setFlag(int) : public throws (void)\nMethod setInstanceId(java.lang.String) : public throws (void)\nMethod setKeys(java.lang.String) : public throws (void)\nMethod setKeys(java.util.Collection) : public throws (void)\nMethod setTags(java.lang.String) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\nMethod setTransactionId(java.lang.String) : public throws (void)\nMethod setWaitStoreMsgOK(boolean) : public throws (void)\nMethod toString() : public throws (java.lang.String)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/common.message.MessageExt.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField body : private [B null\nField bodyCRC : private int 0\nField bornHost : private java.net.SocketAddress null\nField bornTimestamp : private long 0\nField brokerName : private java.lang.String null\nField commitLogOffset : private long 0\nField flag : private int 0\nField msgId : private java.lang.String null\nField preparedTransactionOffset : private long 0\nField properties : private java.util.Map null\nField queueId : private int 0\nField queueOffset : private long 0\nField reconsumeTimes : private int 0\nField serialVersionUID : private long 8445773977080406428\nField storeHost : private java.net.SocketAddress null\nField storeSize : private int 0\nField storeTimestamp : private long 0\nField sysFlag : private int 0\nField topic : private java.lang.String null\nField transactionId : private java.lang.String null\nMethod getBody() : public throws ([B)\nMethod getBodyCRC() : public throws (int)\nMethod getBornHost() : public throws (java.net.SocketAddress)\nMethod getBornHostBytes() : public throws (java.nio.ByteBuffer)\nMethod getBornHostBytes(java.nio.ByteBuffer) : public throws (java.nio.ByteBuffer)\nMethod getBornHostNameString() : public throws (java.lang.String)\nMethod getBornHostString() : public throws (java.lang.String)\nMethod getBornTimestamp() : public throws (long)\nMethod getBrokerName() : public throws (java.lang.String)\nMethod getBuyerId() : public throws (java.lang.String)\nMethod getCommitLogOffset() : public throws (long)\nMethod getDelayTimeLevel() : public throws (int)\nMethod getFlag() : public throws (int)\nMethod getKeys() : public throws (java.lang.String)\nMethod getMsgId() : public throws (java.lang.String)\nMethod getPreparedTransactionOffset() : public throws (long)\nMethod getProperties() : public throws (java.util.Map)\nMethod getProperty(java.lang.String) : public throws (java.lang.String)\nMethod getQueueId() : public throws (int)\nMethod getQueueOffset() : public throws (long)\nMethod getReconsumeTimes() : public throws (int)\nMethod getStoreHost() : public throws (java.net.SocketAddress)\nMethod getStoreHostBytes() : public throws (java.nio.ByteBuffer)\nMethod getStoreHostBytes(java.nio.ByteBuffer) : public throws (java.nio.ByteBuffer)\nMethod getStoreSize() : public throws (int)\nMethod getStoreTimestamp() : public throws (long)\nMethod getSysFlag() : public throws (int)\nMethod getTags() : public throws (java.lang.String)\nMethod getTopic() : public throws (java.lang.String)\nMethod getTransactionId() : public throws (java.lang.String)\nMethod getUserProperty(java.lang.String) : public throws (java.lang.String)\nMethod isWaitStoreMsgOK() : public throws (boolean)\nMethod parseTopicFilterType(int) : public throws (org.apache.rocketmq.common.TopicFilterType)\nMethod putUserProperty(java.lang.String,java.lang.String) : public throws (void)\nMethod setBody([B) : public throws (void)\nMethod setBodyCRC(int) : public throws (void)\nMethod setBornHost(java.net.SocketAddress) : public throws (void)\nMethod setBornHostV6Flag() : public throws (void)\nMethod setBornTimestamp(long) : public throws (void)\nMethod setBrokerName(java.lang.String) : public throws (void)\nMethod setBuyerId(java.lang.String) : public throws (void)\nMethod setCommitLogOffset(long) : public throws (void)\nMethod setDelayTimeLevel(int) : public throws (void)\nMethod setFlag(int) : public throws (void)\nMethod setInstanceId(java.lang.String) : public throws (void)\nMethod setKeys(java.lang.String) : public throws (void)\nMethod setKeys(java.util.Collection) : public throws (void)\nMethod setMsgId(java.lang.String) : public throws (void)\nMethod setPreparedTransactionOffset(long) : public throws (void)\nMethod setQueueId(int) : public throws (void)\nMethod setQueueOffset(long) : public throws (void)\nMethod setReconsumeTimes(int) : public throws (void)\nMethod setStoreHost(java.net.SocketAddress) : public throws (void)\nMethod setStoreHostAddressV6Flag() : public throws (void)\nMethod setStoreSize(int) : public throws (void)\nMethod setStoreTimestamp(long) : public throws (void)\nMethod setSysFlag(int) : public throws (void)\nMethod setTags(java.lang.String) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\nMethod setTransactionId(java.lang.String) : public throws (void)\nMethod setWaitStoreMsgOK(boolean) : public throws (void)\nMethod socketAddress2ByteBuffer(java.net.SocketAddress) : public throws (java.nio.ByteBuffer)\nMethod socketAddress2ByteBuffer(java.net.SocketAddress,java.nio.ByteBuffer) : public throws (java.nio.ByteBuffer)\nMethod toString() : public throws (java.lang.String)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/common.message.MessageQueue.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField brokerName : private java.lang.String null\nField queueId : private int 0\nField serialVersionUID : private long 6191200464116433425\nField topic : private java.lang.String null\nMethod compareTo(java.lang.Object) : public throws (int)\nMethod compareTo(org.apache.rocketmq.common.message.MessageQueue) : public throws (int)\nMethod equals(java.lang.Object) : public throws (boolean)\nMethod getBrokerName() : public throws (java.lang.String)\nMethod getQueueId() : public throws (int)\nMethod getTopic() : public throws (java.lang.String)\nMethod hashCode() : public throws (int)\nMethod setBrokerName(java.lang.String) : public throws (void)\nMethod setQueueId(int) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\nMethod toString() : public throws (java.lang.String)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/remoting.RPCHook.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nMethod doAfterResponse(java.lang.String,org.apache.rocketmq.remoting.protocol.RemotingCommand,org.apache.rocketmq.remoting.protocol.RemotingCommand) : public throws (void)\nMethod doBeforeRequest(java.lang.String,org.apache.rocketmq.remoting.protocol.RemotingCommand) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/api/tools.admin.DefaultMQAdminExt.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField SEND_MESSAGE_WITH_VIP_CHANNEL_PROPERTY : public java.lang.String com.rocketmq.sendMessageWithVIPChannel\nField accessChannel : protected org.apache.rocketmq.client.AccessChannel LOCAL\nField adminExtGroup : private java.lang.String admin_ext_group\nField clientCallbackExecutorThreads : private int null\nField clientIP : private java.lang.String null\nField createTopicKey : private java.lang.String TBW102\nField defaultMQAdminExtImpl : private org.apache.rocketmq.tools.admin.DefaultMQAdminExtImpl null\nField enableStreamRequestType : protected boolean false\nField heartbeatBrokerInterval : private int 30000\nField instanceName : private java.lang.String DEFAULT\nField language : private org.apache.rocketmq.remoting.protocol.LanguageCode JAVA\nField mqClientApiTimeout : private int 3000\nField namespace : protected java.lang.String null\nField namespaceInitialized : private boolean false\nField namesrvAddr : private java.lang.String null\nField persistConsumerOffsetInterval : private int 5000\nField pollNameServerInterval : private int 30000\nField pullTimeDelayMillsWhenException : private long 1000\nField timeoutMillis : private long 5000\nField unitMode : private boolean false\nField unitName : private java.lang.String null\nField useTLS : private boolean false\nField vipChannelEnabled : private boolean false\nMethod addWritePermOfBroker(java.lang.String,java.lang.String) : public throws (int)\nMethod buildMQClientId() : public throws (java.lang.String)\nMethod changeInstanceNameToPID() : public throws (void)\nMethod cleanExpiredConsumerQueue(java.lang.String) : public throws (boolean)\nMethod cleanExpiredConsumerQueueByAddr(java.lang.String) : public throws (boolean)\nMethod cleanUnusedTopic(java.lang.String) : public throws (boolean)\nMethod cleanUnusedTopicByAddr(java.lang.String) : public throws (boolean)\nMethod cloneClientConfig() : public throws (org.apache.rocketmq.client.ClientConfig)\nMethod cloneGroupOffset(boolean,java.lang.String,java.lang.String,java.lang.String) : public throws (void)\nMethod consumeMessageDirectly(java.lang.String,java.lang.String,java.lang.String) : public throws (org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult)\nMethod consumeMessageDirectly(java.lang.String,java.lang.String,java.lang.String,java.lang.String) : public throws (org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult)\nMethod createAndUpdateKvConfig(java.lang.String,java.lang.String,java.lang.String) : public throws (void)\nMethod createAndUpdatePlainAccessConfig(java.lang.String,org.apache.rocketmq.auth.migration.plain.PlainAccessConfig) : public throws (void)\nMethod createAndUpdateSubscriptionGroupConfig(java.lang.String,org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig) : public throws (void)\nMethod createAndUpdateTopicConfig(java.lang.String,org.apache.rocketmq.common.TopicConfig) : public throws (void)\nMethod createOrUpdateOrderConf(boolean,java.lang.String,java.lang.String) : public throws (void)\nMethod createTopic(int,int,java.lang.String,java.lang.String) : public throws (void)\nMethod createTopic(int,java.lang.String,java.lang.String) : public throws (void)\nMethod deleteExpiredCommitLog(java.lang.String) : public throws (boolean)\nMethod deleteExpiredCommitLogByAddr(java.lang.String) : public throws (boolean)\nMethod deleteKvConfig(java.lang.String,java.lang.String) : public throws (void)\nMethod deletePlainAccessConfig(java.lang.String,java.lang.String) : public throws (void)\nMethod deleteSubscriptionGroup(boolean,java.lang.String,java.lang.String) : public throws (void)\nMethod deleteSubscriptionGroup(java.lang.String,java.lang.String) : public throws (void)\nMethod deleteTopicInBroker(java.lang.String,java.util.Set) : public throws (void)\nMethod deleteTopicInNameServer(java.lang.String,java.lang.String,java.util.Set) : public throws (void)\nMethod earliestMsgStoreTime(org.apache.rocketmq.common.message.MessageQueue) : public throws (long)\nMethod examineBrokerClusterAclConfig(java.lang.String) : public throws (org.apache.rocketmq.auth.migration.plain.AclConfig)\nMethod examineBrokerClusterAclVersionInfo(java.lang.String) : public throws (org.apache.rocketmq.remoting.protocol.body.ClusterAclVersionInfo)\nMethod examineBrokerClusterInfo() : public throws (org.apache.rocketmq.remoting.protocol.body.ClusterInfo)\nMethod examineConsumeStats(java.lang.String) : public throws (org.apache.rocketmq.remoting.protocol.admin.ConsumeStats)\nMethod examineConsumeStats(java.lang.String,java.lang.String) : public throws (org.apache.rocketmq.remoting.protocol.admin.ConsumeStats)\nMethod examineConsumerConnectionInfo(java.lang.String) : public throws (org.apache.rocketmq.remoting.protocol.body.ConsumerConnection)\nMethod examineConsumerConnectionInfo(java.lang.String,java.lang.String) : public throws (org.apache.rocketmq.remoting.protocol.body.ConsumerConnection)\nMethod examineProducerConnectionInfo(java.lang.String,java.lang.String) : public throws (org.apache.rocketmq.remoting.protocol.body.ProducerConnection)\nMethod examineSubscriptionGroupConfig(java.lang.String,java.lang.String) : public throws (org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig)\nMethod examineTopicConfig(java.lang.String,java.lang.String) : public throws (org.apache.rocketmq.common.TopicConfig)\nMethod examineTopicRouteInfo(java.lang.String) : public throws (org.apache.rocketmq.remoting.protocol.route.TopicRouteData)\nMethod examineTopicStats(java.lang.String) : public throws (org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable)\nMethod fetchAllTopicList() : public throws (org.apache.rocketmq.remoting.protocol.body.TopicList)\nMethod fetchBrokerRuntimeStats(java.lang.String) : public throws (org.apache.rocketmq.remoting.protocol.body.KVTable)\nMethod fetchConsumeStatsInBroker(boolean,java.lang.String,long) : public throws (org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList)\nMethod fetchTopicsByCLuster(java.lang.String) : public throws (org.apache.rocketmq.remoting.protocol.body.TopicList)\nMethod getAccessChannel() : public throws (org.apache.rocketmq.client.AccessChannel)\nMethod getAdminExtGroup() : public throws (java.lang.String)\nMethod getAllProducerInfo(java.lang.String) : public throws (org.apache.rocketmq.remoting.protocol.body.ProducerTableInfo)\nMethod getAllSubscriptionGroup(java.lang.String,long) : public throws (org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper)\nMethod getAllTopicConfig(java.lang.String,long) : public throws (org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper)\nMethod getBrokerConfig(java.lang.String) : public throws (java.util.Properties)\nMethod getClientCallbackExecutorThreads() : public throws (int)\nMethod getClientIP() : public throws (java.lang.String)\nMethod getClusterList(java.lang.String) : public throws (java.util.Set)\nMethod getConsumeStatus(java.lang.String,java.lang.String,java.lang.String) : public throws (java.util.Map)\nMethod getConsumerRunningInfo(boolean,java.lang.String,java.lang.String) : public throws (org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo)\nMethod getCreateTopicKey() : public throws (java.lang.String)\nMethod getHeartbeatBrokerInterval() : public throws (int)\nMethod getInstanceName() : public throws (java.lang.String)\nMethod getKVConfig(java.lang.String,java.lang.String) : public throws (java.lang.String)\nMethod getKVListByNamespace(java.lang.String) : public throws (org.apache.rocketmq.remoting.protocol.body.KVTable)\nMethod getLanguage() : public throws (org.apache.rocketmq.remoting.protocol.LanguageCode)\nMethod getMqClientApiTimeout() : public throws (int)\nMethod getNameServerAddressList() : public throws (java.util.List)\nMethod getNameServerConfig(java.util.List) : public throws (java.util.Map)\nMethod getNamespace() : public throws (java.lang.String)\nMethod getNamesrvAddr() : public throws (java.lang.String)\nMethod getPersistConsumerOffsetInterval() : public throws (int)\nMethod getPollNameServerInterval() : public throws (int)\nMethod getPullTimeDelayMillsWhenException() : public throws (long)\nMethod getTopicClusterList(java.lang.String) : public throws (java.util.Set)\nMethod getUnitName() : public throws (java.lang.String)\nMethod getUserSubscriptionGroup(java.lang.String,long) : public throws (org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper)\nMethod getUserTopicConfig(boolean,java.lang.String,long) : public throws (org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper)\nMethod isEnableStreamRequestType() : public throws (boolean)\nMethod isUnitMode() : public throws (boolean)\nMethod isUseTLS() : public throws (boolean)\nMethod isVipChannelEnabled() : public throws (boolean)\nMethod maxOffset(org.apache.rocketmq.common.message.MessageQueue) : public throws (long)\nMethod messageTrackDetail(org.apache.rocketmq.common.message.MessageExt) : public throws (java.util.List)\nMethod minOffset(org.apache.rocketmq.common.message.MessageQueue) : public throws (long)\nMethod putKVConfig(java.lang.String,java.lang.String,java.lang.String) : public throws (void)\nMethod queryConsumeQueue(int,int,java.lang.String,java.lang.String,java.lang.String,long) : public throws (org.apache.rocketmq.remoting.protocol.body.QueryConsumeQueueResponseBody)\nMethod queryConsumeTimeSpan(java.lang.String,java.lang.String) : public throws (java.util.List)\nMethod queryMessage(int,java.lang.String,java.lang.String,long,long) : public throws (org.apache.rocketmq.client.QueryResult)\nMethod queryMessageByUniqKey(int,java.lang.String,java.lang.String,long,long) : public throws (org.apache.rocketmq.client.QueryResult)\nMethod queryTopicConsumeByWho(java.lang.String) : public throws (org.apache.rocketmq.remoting.protocol.body.GroupList)\nMethod queueWithNamespace(org.apache.rocketmq.common.message.MessageQueue) : public throws (org.apache.rocketmq.common.message.MessageQueue)\nMethod queuesWithNamespace(java.util.Collection) : public throws (java.util.Collection)\nMethod resetClientConfig(org.apache.rocketmq.client.ClientConfig) : public throws (void)\nMethod resetOffsetByTimestamp(boolean,boolean,java.lang.String,java.lang.String,long) : public throws (java.util.Map)\nMethod resetOffsetByTimestamp(boolean,java.lang.String,java.lang.String,long) : public throws (java.util.Map)\nMethod resetOffsetByTimestampOld(boolean,java.lang.String,java.lang.String,long) : public throws (java.util.List)\nMethod resetOffsetNew(java.lang.String,java.lang.String,long) : public throws (void)\nMethod resumeCheckHalfMessage(java.lang.String) : public throws (boolean)\nMethod resumeCheckHalfMessage(java.lang.String,java.lang.String) : public throws (boolean)\nMethod searchOffset(long,org.apache.rocketmq.common.message.MessageQueue) : public throws (long)\nMethod setAccessChannel(org.apache.rocketmq.client.AccessChannel) : public throws (void)\nMethod setAdminExtGroup(java.lang.String) : public throws (void)\nMethod setClientCallbackExecutorThreads(int) : public throws (void)\nMethod setClientIP(java.lang.String) : public throws (void)\nMethod setCreateTopicKey(java.lang.String) : public throws (void)\nMethod setEnableStreamRequestType(boolean) : public throws (void)\nMethod setHeartbeatBrokerInterval(int) : public throws (void)\nMethod setInstanceName(java.lang.String) : public throws (void)\nMethod setLanguage(org.apache.rocketmq.remoting.protocol.LanguageCode) : public throws (void)\nMethod setMqClientApiTimeout(int) : public throws (void)\nMethod setNamespace(java.lang.String) : public throws (void)\nMethod setNamesrvAddr(java.lang.String) : public throws (void)\nMethod setPersistConsumerOffsetInterval(int) : public throws (void)\nMethod setPollNameServerInterval(int) : public throws (void)\nMethod setPullTimeDelayMillsWhenException(long) : public throws (void)\nMethod setUnitMode(boolean) : public throws (void)\nMethod setUnitName(java.lang.String) : public throws (void)\nMethod setUseTLS(boolean) : public throws (void)\nMethod setVipChannelEnabled(boolean) : public throws (void)\nMethod shutdown() : public throws (void)\nMethod start() : public throws (void)\nMethod toString() : public throws (java.lang.String)\nMethod updateBrokerConfig(java.lang.String,java.util.Properties) : public throws (void)\nMethod updateConsumeOffset(java.lang.String,java.lang.String,long,org.apache.rocketmq.common.message.MessageQueue) : public throws (void)\nMethod updateGlobalWhiteAddrConfig(java.lang.String,java.lang.String) : public throws (void)\nMethod updateGlobalWhiteAddrConfig(java.lang.String,java.lang.String,java.lang.String) : public throws (void)\nMethod updateNameServerConfig(java.util.List,java.util.Properties) : public throws (void)\nMethod viewBrokerStatsData(java.lang.String,java.lang.String,java.lang.String) : public throws (org.apache.rocketmq.remoting.protocol.body.BrokerStatsData)\nMethod viewMessage(java.lang.String) : public throws (org.apache.rocketmq.common.message.MessageExt)\nMethod viewMessage(java.lang.String,java.lang.String) : public throws (org.apache.rocketmq.common.message.MessageExt)\nMethod wipeWritePermOfBroker(java.lang.String,java.lang.String) : public throws (int)\nMethod withNamespace(java.lang.String) : public throws (java.lang.String)\nMethod withNamespace(java.util.Set) : public throws (java.util.Set)\nMethod withoutNamespace(java.lang.String) : public throws (java.lang.String)\nMethod withoutNamespace(java.util.Set) : public throws (java.util.Set)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.RequestCode.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField ADD_WRITE_PERM_OF_BROKER : public int 327\nField ADJUST_CONSUMER_THREAD_POOL : public int 213\nField CHECK_CLIENT_CONFIG : public int 46\nField CHECK_TRANSACTION_STATE : public int 39\nField CLEAN_EXPIRED_CONSUMEQUEUE : public int 306\nField CLEAN_UNUSED_TOPIC : public int 316\nField CLONE_GROUP_OFFSET : public int 314\nField CONSUMER_SEND_MSG_BACK : public int 36\nField CONSUME_MESSAGE_DIRECTLY : public int 309\nField DELETE_ACL_CONFIG : public int 51\nField DELETE_EXPIRED_COMMITLOG : public int 329\nField DELETE_KV_CONFIG : public int 102\nField DELETE_SUBSCRIPTIONGROUP : public int 207\nField DELETE_TOPIC_IN_BROKER : public int 215\nField DELETE_TOPIC_IN_NAMESRV : public int 216\nField END_TRANSACTION : public int 37\nField GET_ALL_CONSUMER_OFFSET : public int 43\nField GET_ALL_DELAY_OFFSET : public int 45\nField GET_ALL_PRODUCER_INFO : public int 328\nField GET_ALL_SUBSCRIPTIONGROUP_CONFIG : public int 201\nField GET_ALL_TOPIC_CONFIG : public int 21\nField GET_ALL_TOPIC_LIST_FROM_NAMESERVER : public int 206\nField GET_BROKER_CLUSTER_ACL_CONFIG : public int 54\nField GET_BROKER_CLUSTER_ACL_INFO : public int 52\nField GET_BROKER_CLUSTER_INFO : public int 106\nField GET_BROKER_CONFIG : public int 26\nField GET_BROKER_CONSUME_STATS : public int 317\nField GET_BROKER_RUNTIME_INFO : public int 28\nField GET_CONSUMER_CONNECTION_LIST : public int 203\nField GET_CONSUMER_LIST_BY_GROUP : public int 38\nField GET_CONSUMER_RUNNING_INFO : public int 307\nField GET_CONSUMER_STATUS_FROM_CLIENT : public int 221\nField GET_CONSUME_STATS : public int 208\nField GET_EARLIEST_MSG_STORETIME : public int 32\nField GET_HAS_UNIT_SUB_TOPIC_LIST : public int 312\nField GET_HAS_UNIT_SUB_UNUNIT_TOPIC_LIST : public int 313\nField GET_KVLIST_BY_NAMESPACE : public int 219\nField GET_KV_CONFIG : public int 101\nField GET_MAX_OFFSET : public int 30\nField GET_MIN_OFFSET : public int 31\nField GET_NAMESRV_CONFIG : public int 319\nField GET_PRODUCER_CONNECTION_LIST : public int 204\nField GET_ROUTEINFO_BY_TOPIC : public int 105\nField GET_SYSTEM_TOPIC_LIST_FROM_BROKER : public int 305\nField GET_SYSTEM_TOPIC_LIST_FROM_NS : public int 304\nField GET_TOPICS_BY_CLUSTER : public int 224\nField GET_TOPIC_CONFIG_LIST : public int 22\nField GET_TOPIC_NAME_LIST : public int 23\nField GET_TOPIC_STATS_INFO : public int 202\nField GET_UNIT_TOPIC_LIST : public int 311\nField HEART_BEAT : public int 34\nField INVOKE_BROKER_TO_GET_CONSUMER_STATUS : public int 223\nField INVOKE_BROKER_TO_RESET_OFFSET : public int 222\nField LOCK_BATCH_MQ : public int 41\nField NOTIFY_CONSUMER_IDS_CHANGED : public int 40\nField PULL_MESSAGE : public int 11\nField PUSH_REPLY_MESSAGE_TO_CLIENT : public int 326\nField PUT_KV_CONFIG : public int 100\nField QUERY_BROKER_OFFSET : public int 13\nField QUERY_CONSUMER_OFFSET : public int 14\nField QUERY_CONSUME_QUEUE : public int 321\nField QUERY_CONSUME_TIME_SPAN : public int 303\nField QUERY_CORRECTION_OFFSET : public int 308\nField QUERY_DATA_VERSION : public int 322\nField QUERY_MESSAGE : public int 12\nField QUERY_TOPIC_CONSUME_BY_WHO : public int 300\nField REGISTER_BROKER : public int 103\nField REGISTER_FILTER_SERVER : public int 301\nField REGISTER_MESSAGE_FILTER_CLASS : public int 302\nField RESET_CONSUMER_CLIENT_OFFSET : public int 220\nField RESET_CONSUMER_OFFSET_IN_BROKER : public int 212\nField RESET_CONSUMER_OFFSET_IN_CONSUMER : public int 211\nField RESUME_CHECK_HALF_MESSAGE : public int 323\nField RESUME_CONSUMER : public int 210\nField SEARCH_OFFSET_BY_TIMESTAMP : public int 29\nField SEND_BATCH_MESSAGE : public int 320\nField SEND_MESSAGE : public int 10\nField SEND_MESSAGE_V2 : public int 310\nField SEND_REPLY_MESSAGE : public int 324\nField SEND_REPLY_MESSAGE_V2 : public int 325\nField SUSPEND_CONSUMER : public int 209\nField TRIGGER_DELETE_FILES : public int 27\nField UNLOCK_BATCH_MQ : public int 42\nField UNREGISTER_BROKER : public int 104\nField UNREGISTER_CLIENT : public int 35\nField UPDATE_AND_CREATE_ACL_CONFIG : public int 50\nField UPDATE_AND_CREATE_SUBSCRIPTIONGROUP : public int 200\nField UPDATE_AND_CREATE_TOPIC : public int 17\nField UPDATE_BROKER_CONFIG : public int 25\nField UPDATE_CONSUMER_OFFSET : public int 15\nField UPDATE_GLOBAL_WHITE_ADDRS_CONFIG : public int 53\nField UPDATE_NAMESRV_CONFIG : public int 318\nField VIEW_BROKER_STATS_DATA : public int 315\nField VIEW_MESSAGE_BY_ID : public int 33\nField WHO_CONSUME_THE_MESSAGE : public int 214\nField WIPE_WRITE_PERM_OF_BROKER : public int 205\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.CheckTransactionStateRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField commitLogOffset : private java.lang.Long null\nField msgId : private java.lang.String null\nField offsetMsgId : private java.lang.String null\nField tranStateTableOffset : private java.lang.Long null\nField transactionId : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getCommitLogOffset() : public throws (java.lang.Long)\nMethod getMsgId() : public throws (java.lang.String)\nMethod getOffsetMsgId() : public throws (java.lang.String)\nMethod getTranStateTableOffset() : public throws (java.lang.Long)\nMethod getTransactionId() : public throws (java.lang.String)\nMethod setCommitLogOffset(java.lang.Long) : public throws (void)\nMethod setMsgId(java.lang.String) : public throws (void)\nMethod setOffsetMsgId(java.lang.String) : public throws (void)\nMethod setTranStateTableOffset(java.lang.Long) : public throws (void)\nMethod setTransactionId(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.CheckTransactionStateResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField commitLogOffset : private java.lang.Long null\nField commitOrRollback : private java.lang.Integer null\nField producerGroup : private java.lang.String null\nField tranStateTableOffset : private java.lang.Long null\nMethod checkFields() : public throws (void)\nMethod getCommitLogOffset() : public throws (java.lang.Long)\nMethod getCommitOrRollback() : public throws (java.lang.Integer)\nMethod getProducerGroup() : public throws (java.lang.String)\nMethod getTranStateTableOffset() : public throws (java.lang.Long)\nMethod setCommitLogOffset(java.lang.Long) : public throws (void)\nMethod setCommitOrRollback(java.lang.Integer) : public throws (void)\nMethod setProducerGroup(java.lang.String) : public throws (void)\nMethod setTranStateTableOffset(java.lang.Long) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.CloneGroupOffsetRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField destGroup : private java.lang.String null\nField offline : private boolean false\nField srcGroup : private java.lang.String null\nField topic : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getDestGroup() : public throws (java.lang.String)\nMethod getSrcGroup() : public throws (java.lang.String)\nMethod getTopic() : public throws (java.lang.String)\nMethod isOffline() : public throws (boolean)\nMethod setDestGroup(java.lang.String) : public throws (void)\nMethod setOffline(boolean) : public throws (void)\nMethod setSrcGroup(java.lang.String) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.ConsumeMessageDirectlyResultRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField brokerName : private java.lang.String null\nField clientId : private java.lang.String null\nField consumerGroup : private java.lang.String null\nField msgId : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getBrokerName() : public throws (java.lang.String)\nMethod getClientId() : public throws (java.lang.String)\nMethod getConsumerGroup() : public throws (java.lang.String)\nMethod getMsgId() : public throws (java.lang.String)\nMethod setBrokerName(java.lang.String) : public throws (void)\nMethod setClientId(java.lang.String) : public throws (void)\nMethod setConsumerGroup(java.lang.String) : public throws (void)\nMethod setMsgId(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.ConsumerSendMsgBackRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField delayLevel : private java.lang.Integer null\nField group : private java.lang.String null\nField maxReconsumeTimes : private java.lang.Integer null\nField offset : private java.lang.Long null\nField originMsgId : private java.lang.String null\nField originTopic : private java.lang.String null\nField unitMode : private boolean false\nMethod checkFields() : public throws (void)\nMethod getDelayLevel() : public throws (java.lang.Integer)\nMethod getGroup() : public throws (java.lang.String)\nMethod getMaxReconsumeTimes() : public throws (java.lang.Integer)\nMethod getOffset() : public throws (java.lang.Long)\nMethod getOriginMsgId() : public throws (java.lang.String)\nMethod getOriginTopic() : public throws (java.lang.String)\nMethod isUnitMode() : public throws (boolean)\nMethod setDelayLevel(java.lang.Integer) : public throws (void)\nMethod setGroup(java.lang.String) : public throws (void)\nMethod setMaxReconsumeTimes(java.lang.Integer) : public throws (void)\nMethod setOffset(java.lang.Long) : public throws (void)\nMethod setOriginMsgId(java.lang.String) : public throws (void)\nMethod setOriginTopic(java.lang.String) : public throws (void)\nMethod setUnitMode(boolean) : public throws (void)\nMethod toString() : public throws (java.lang.String)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.CreateAccessConfigRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField accessKey : private java.lang.String null\nField admin : private boolean false\nField defaultGroupPerm : private java.lang.String null\nField defaultTopicPerm : private java.lang.String null\nField groupPerms : private java.lang.String null\nField secretKey : private java.lang.String null\nField topicPerms : private java.lang.String null\nField whiteRemoteAddress : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getAccessKey() : public throws (java.lang.String)\nMethod getDefaultGroupPerm() : public throws (java.lang.String)\nMethod getDefaultTopicPerm() : public throws (java.lang.String)\nMethod getGroupPerms() : public throws (java.lang.String)\nMethod getSecretKey() : public throws (java.lang.String)\nMethod getTopicPerms() : public throws (java.lang.String)\nMethod getWhiteRemoteAddress() : public throws (java.lang.String)\nMethod isAdmin() : public throws (boolean)\nMethod setAccessKey(java.lang.String) : public throws (void)\nMethod setAdmin(boolean) : public throws (void)\nMethod setDefaultGroupPerm(java.lang.String) : public throws (void)\nMethod setDefaultTopicPerm(java.lang.String) : public throws (void)\nMethod setGroupPerms(java.lang.String) : public throws (void)\nMethod setSecretKey(java.lang.String) : public throws (void)\nMethod setTopicPerms(java.lang.String) : public throws (void)\nMethod setWhiteRemoteAddress(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.CreateTopicRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField defaultTopic : private java.lang.String null\nField order : private java.lang.Boolean false\nField perm : private java.lang.Integer null\nField readQueueNums : private java.lang.Integer null\nField topic : private java.lang.String null\nField topicFilterType : private java.lang.String null\nField topicSysFlag : private java.lang.Integer null\nField writeQueueNums : private java.lang.Integer null\nMethod checkFields() : public throws (void)\nMethod getDefaultTopic() : public throws (java.lang.String)\nMethod getOrder() : public throws (java.lang.Boolean)\nMethod getPerm() : public throws (java.lang.Integer)\nMethod getReadQueueNums() : public throws (java.lang.Integer)\nMethod getTopic() : public throws (java.lang.String)\nMethod getTopicFilterType() : public throws (java.lang.String)\nMethod getTopicFilterTypeEnum() : public throws (org.apache.rocketmq.common.TopicFilterType)\nMethod getTopicSysFlag() : public throws (java.lang.Integer)\nMethod getWriteQueueNums() : public throws (java.lang.Integer)\nMethod setDefaultTopic(java.lang.String) : public throws (void)\nMethod setOrder(java.lang.Boolean) : public throws (void)\nMethod setPerm(java.lang.Integer) : public throws (void)\nMethod setReadQueueNums(java.lang.Integer) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\nMethod setTopicFilterType(java.lang.String) : public throws (void)\nMethod setTopicSysFlag(java.lang.Integer) : public throws (void)\nMethod setWriteQueueNums(java.lang.Integer) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.DeleteAccessConfigRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField accessKey : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getAccessKey() : public throws (java.lang.String)\nMethod setAccessKey(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.DeleteSubscriptionGroupRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField groupName : private java.lang.String null\nField removeOffset : private boolean false\nMethod checkFields() : public throws (void)\nMethod getGroupName() : public throws (java.lang.String)\nMethod isRemoveOffset() : public throws (boolean)\nMethod setGroupName(java.lang.String) : public throws (void)\nMethod setRemoveOffset(boolean) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.DeleteTopicRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField topic : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getTopic() : public throws (java.lang.String)\nMethod setTopic(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.EndTransactionRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField commitLogOffset : private java.lang.Long null\nField commitOrRollback : private java.lang.Integer null\nField fromTransactionCheck : private java.lang.Boolean false\nField msgId : private java.lang.String null\nField producerGroup : private java.lang.String null\nField tranStateTableOffset : private java.lang.Long null\nField transactionId : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getCommitLogOffset() : public throws (java.lang.Long)\nMethod getCommitOrRollback() : public throws (java.lang.Integer)\nMethod getFromTransactionCheck() : public throws (java.lang.Boolean)\nMethod getMsgId() : public throws (java.lang.String)\nMethod getProducerGroup() : public throws (java.lang.String)\nMethod getTranStateTableOffset() : public throws (java.lang.Long)\nMethod getTransactionId() : public throws (java.lang.String)\nMethod setCommitLogOffset(java.lang.Long) : public throws (void)\nMethod setCommitOrRollback(java.lang.Integer) : public throws (void)\nMethod setFromTransactionCheck(java.lang.Boolean) : public throws (void)\nMethod setMsgId(java.lang.String) : public throws (void)\nMethod setProducerGroup(java.lang.String) : public throws (void)\nMethod setTranStateTableOffset(java.lang.Long) : public throws (void)\nMethod setTransactionId(java.lang.String) : public throws (void)\nMethod toString() : public throws (java.lang.String)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.EndTransactionResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nMethod checkFields() : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.GetAllProducerInfoRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nMethod checkFields() : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.GetAllTopicConfigResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nMethod checkFields() : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.GetBrokerAclConfigResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField allAclFileVersion : private java.lang.String null\nField brokerAddr : private java.lang.String null\nField brokerName : private java.lang.String null\nField clusterName : private java.lang.String null\nField version : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getAllAclFileVersion() : public throws (java.lang.String)\nMethod getBrokerAddr() : public throws (java.lang.String)\nMethod getBrokerName() : public throws (java.lang.String)\nMethod getClusterName() : public throws (java.lang.String)\nMethod getVersion() : public throws (java.lang.String)\nMethod setAllAclFileVersion(java.lang.String) : public throws (void)\nMethod setBrokerAddr(java.lang.String) : public throws (void)\nMethod setBrokerName(java.lang.String) : public throws (void)\nMethod setClusterName(java.lang.String) : public throws (void)\nMethod setVersion(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.GetBrokerClusterAclConfigResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField plainAccessConfigs : private java.util.List null\nMethod checkFields() : public throws (void)\nMethod getPlainAccessConfigs() : public throws (java.util.List)\nMethod setPlainAccessConfigs(java.util.List) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.GetBrokerConfigResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField version : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getVersion() : public throws (java.lang.String)\nMethod setVersion(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.GetConsumeStatsInBrokerHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField isOrder : private boolean false\nMethod checkFields() : public throws (void)\nMethod isOrder() : public throws (boolean)\nMethod setIsOrder(boolean) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.GetConsumeStatsRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField consumerGroup : private java.lang.String null\nField topic : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getConsumerGroup() : public throws (java.lang.String)\nMethod getTopic() : public throws (java.lang.String)\nMethod setConsumerGroup(java.lang.String) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.GetConsumerConnectionListRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField consumerGroup : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getConsumerGroup() : public throws (java.lang.String)\nMethod setConsumerGroup(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.GetConsumerListByGroupRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField consumerGroup : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getConsumerGroup() : public throws (java.lang.String)\nMethod setConsumerGroup(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.GetConsumerListByGroupResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nMethod checkFields() : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.GetConsumerRunningInfoRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField clientId : private java.lang.String null\nField consumerGroup : private java.lang.String null\nField jstackEnable : private boolean false\nMethod checkFields() : public throws (void)\nMethod getClientId() : public throws (java.lang.String)\nMethod getConsumerGroup() : public throws (java.lang.String)\nMethod isJstackEnable() : public throws (boolean)\nMethod setClientId(java.lang.String) : public throws (void)\nMethod setConsumerGroup(java.lang.String) : public throws (void)\nMethod setJstackEnable(boolean) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.GetConsumerStatusRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField clientAddr : private java.lang.String null\nField group : private java.lang.String null\nField topic : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getClientAddr() : public throws (java.lang.String)\nMethod getGroup() : public throws (java.lang.String)\nMethod getTopic() : public throws (java.lang.String)\nMethod setClientAddr(java.lang.String) : public throws (void)\nMethod setGroup(java.lang.String) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.GetEarliestMsgStoretimeRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField queueId : private java.lang.Integer null\nField topic : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getQueueId() : public throws (java.lang.Integer)\nMethod getTopic() : public throws (java.lang.String)\nMethod setQueueId(java.lang.Integer) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.GetEarliestMsgStoretimeResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField timestamp : private java.lang.Long null\nMethod checkFields() : public throws (void)\nMethod getTimestamp() : public throws (java.lang.Long)\nMethod setTimestamp(java.lang.Long) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.GetMaxOffsetRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField queueId : private java.lang.Integer null\nField topic : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getQueueId() : public throws (java.lang.Integer)\nMethod getTopic() : public throws (java.lang.String)\nMethod setQueueId(java.lang.Integer) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.GetMaxOffsetResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField offset : private java.lang.Long null\nMethod checkFields() : public throws (void)\nMethod getOffset() : public throws (java.lang.Long)\nMethod setOffset(java.lang.Long) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.GetMinOffsetRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField queueId : private java.lang.Integer null\nField topic : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getQueueId() : public throws (java.lang.Integer)\nMethod getTopic() : public throws (java.lang.String)\nMethod setQueueId(java.lang.Integer) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.GetMinOffsetResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField offset : private java.lang.Long null\nMethod checkFields() : public throws (void)\nMethod getOffset() : public throws (java.lang.Long)\nMethod setOffset(java.lang.Long) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.GetProducerConnectionListRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField producerGroup : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getProducerGroup() : public throws (java.lang.String)\nMethod setProducerGroup(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.GetTopicStatsInfoRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField topic : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getTopic() : public throws (java.lang.String)\nMethod setTopic(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.GetTopicsByClusterRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField cluster : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getCluster() : public throws (java.lang.String)\nMethod setCluster(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.NotifyConsumerIdsChangedRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField consumerGroup : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getConsumerGroup() : public throws (java.lang.String)\nMethod setConsumerGroup(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.PullMessageRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField commitOffset : private java.lang.Long null\nField consumerGroup : private java.lang.String null\nField expressionType : private java.lang.String null\nField maxMsgNums : private java.lang.Integer null\nField queueId : private java.lang.Integer null\nField queueOffset : private java.lang.Long null\nField subVersion : private java.lang.Long null\nField subscription : private java.lang.String null\nField suspendTimeoutMillis : private java.lang.Long null\nField sysFlag : private java.lang.Integer null\nField topic : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod decode(java.util.HashMap) : public throws (void)\nMethod encode(io.netty.buffer.ByteBuf) : public throws (void)\nMethod getCommitOffset() : public throws (java.lang.Long)\nMethod getConsumerGroup() : public throws (java.lang.String)\nMethod getExpressionType() : public throws (java.lang.String)\nMethod getMaxMsgNums() : public throws (java.lang.Integer)\nMethod getQueueId() : public throws (java.lang.Integer)\nMethod getQueueOffset() : public throws (java.lang.Long)\nMethod getSubVersion() : public throws (java.lang.Long)\nMethod getSubscription() : public throws (java.lang.String)\nMethod getSuspendTimeoutMillis() : public throws (java.lang.Long)\nMethod getSysFlag() : public throws (java.lang.Integer)\nMethod getTopic() : public throws (java.lang.String)\nMethod setCommitOffset(java.lang.Long) : public throws (void)\nMethod setConsumerGroup(java.lang.String) : public throws (void)\nMethod setExpressionType(java.lang.String) : public throws (void)\nMethod setMaxMsgNums(java.lang.Integer) : public throws (void)\nMethod setQueueId(java.lang.Integer) : public throws (void)\nMethod setQueueOffset(java.lang.Long) : public throws (void)\nMethod setSubVersion(java.lang.Long) : public throws (void)\nMethod setSubscription(java.lang.String) : public throws (void)\nMethod setSuspendTimeoutMillis(java.lang.Long) : public throws (void)\nMethod setSysFlag(java.lang.Integer) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.PullMessageResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField maxOffset : private java.lang.Long null\nField minOffset : private java.lang.Long null\nField nextBeginOffset : private java.lang.Long null\nField suggestWhichBrokerId : private java.lang.Long null\nMethod checkFields() : public throws (void)\nMethod decode(java.util.HashMap) : public throws (void)\nMethod encode(io.netty.buffer.ByteBuf) : public throws (void)\nMethod getMaxOffset() : public throws (java.lang.Long)\nMethod getMinOffset() : public throws (java.lang.Long)\nMethod getNextBeginOffset() : public throws (java.lang.Long)\nMethod getSuggestWhichBrokerId() : public throws (java.lang.Long)\nMethod setMaxOffset(java.lang.Long) : public throws (void)\nMethod setMinOffset(java.lang.Long) : public throws (void)\nMethod setNextBeginOffset(java.lang.Long) : public throws (void)\nMethod setSuggestWhichBrokerId(java.lang.Long) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.QueryConsumeQueueRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField consumerGroup : private java.lang.String null\nField count : private int 0\nField index : private long 0\nField queueId : private int 0\nField topic : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getConsumerGroup() : public throws (java.lang.String)\nMethod getCount() : public throws (int)\nMethod getIndex() : public throws (long)\nMethod getQueueId() : public throws (int)\nMethod getTopic() : public throws (java.lang.String)\nMethod setConsumerGroup(java.lang.String) : public throws (void)\nMethod setCount(int) : public throws (void)\nMethod setIndex(long) : public throws (void)\nMethod setQueueId(int) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.QueryConsumeTimeSpanRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField group : private java.lang.String null\nField topic : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getGroup() : public throws (java.lang.String)\nMethod getTopic() : public throws (java.lang.String)\nMethod setGroup(java.lang.String) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.QueryConsumerOffsetRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField consumerGroup : private java.lang.String null\nField queueId : private java.lang.Integer null\nField topic : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getConsumerGroup() : public throws (java.lang.String)\nMethod getQueueId() : public throws (java.lang.Integer)\nMethod getTopic() : public throws (java.lang.String)\nMethod setConsumerGroup(java.lang.String) : public throws (void)\nMethod setQueueId(java.lang.Integer) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.QueryConsumerOffsetResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField offset : private java.lang.Long null\nMethod checkFields() : public throws (void)\nMethod getOffset() : public throws (java.lang.Long)\nMethod setOffset(java.lang.Long) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.QueryCorrectionOffsetHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField compareGroup : private java.lang.String null\nField filterGroups : private java.lang.String null\nField topic : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getCompareGroup() : public throws (java.lang.String)\nMethod getFilterGroups() : public throws (java.lang.String)\nMethod getTopic() : public throws (java.lang.String)\nMethod setCompareGroup(java.lang.String) : public throws (void)\nMethod setFilterGroups(java.lang.String) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.QueryMessageRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField beginTimestamp : private java.lang.Long null\nField endTimestamp : private java.lang.Long null\nField key : private java.lang.String null\nField maxNum : private java.lang.Integer null\nField topic : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getBeginTimestamp() : public throws (java.lang.Long)\nMethod getEndTimestamp() : public throws (java.lang.Long)\nMethod getKey() : public throws (java.lang.String)\nMethod getMaxNum() : public throws (java.lang.Integer)\nMethod getTopic() : public throws (java.lang.String)\nMethod setBeginTimestamp(java.lang.Long) : public throws (void)\nMethod setEndTimestamp(java.lang.Long) : public throws (void)\nMethod setKey(java.lang.String) : public throws (void)\nMethod setMaxNum(java.lang.Integer) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.QueryMessageResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField indexLastUpdatePhyoffset : private java.lang.Long null\nField indexLastUpdateTimestamp : private java.lang.Long null\nMethod checkFields() : public throws (void)\nMethod getIndexLastUpdatePhyoffset() : public throws (java.lang.Long)\nMethod getIndexLastUpdateTimestamp() : public throws (java.lang.Long)\nMethod setIndexLastUpdatePhyoffset(java.lang.Long) : public throws (void)\nMethod setIndexLastUpdateTimestamp(java.lang.Long) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.QueryTopicConsumeByWhoRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField topic : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getTopic() : public throws (java.lang.String)\nMethod setTopic(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.ReplyMessageRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField bornHost : private java.lang.String null\nField bornTimestamp : private java.lang.Long null\nField defaultTopic : private java.lang.String null\nField defaultTopicQueueNums : private java.lang.Integer null\nField flag : private java.lang.Integer null\nField producerGroup : private java.lang.String null\nField properties : private java.lang.String null\nField queueId : private java.lang.Integer null\nField reconsumeTimes : private java.lang.Integer null\nField storeHost : private java.lang.String null\nField storeTimestamp : private long 0\nField sysFlag : private java.lang.Integer null\nField topic : private java.lang.String null\nField unitMode : private boolean false\nMethod checkFields() : public throws (void)\nMethod getBornHost() : public throws (java.lang.String)\nMethod getBornTimestamp() : public throws (java.lang.Long)\nMethod getDefaultTopic() : public throws (java.lang.String)\nMethod getDefaultTopicQueueNums() : public throws (java.lang.Integer)\nMethod getFlag() : public throws (java.lang.Integer)\nMethod getProducerGroup() : public throws (java.lang.String)\nMethod getProperties() : public throws (java.lang.String)\nMethod getQueueId() : public throws (java.lang.Integer)\nMethod getReconsumeTimes() : public throws (java.lang.Integer)\nMethod getStoreHost() : public throws (java.lang.String)\nMethod getStoreTimestamp() : public throws (long)\nMethod getSysFlag() : public throws (java.lang.Integer)\nMethod getTopic() : public throws (java.lang.String)\nMethod isUnitMode() : public throws (boolean)\nMethod setBornHost(java.lang.String) : public throws (void)\nMethod setBornTimestamp(java.lang.Long) : public throws (void)\nMethod setDefaultTopic(java.lang.String) : public throws (void)\nMethod setDefaultTopicQueueNums(java.lang.Integer) : public throws (void)\nMethod setFlag(java.lang.Integer) : public throws (void)\nMethod setProducerGroup(java.lang.String) : public throws (void)\nMethod setProperties(java.lang.String) : public throws (void)\nMethod setQueueId(java.lang.Integer) : public throws (void)\nMethod setReconsumeTimes(java.lang.Integer) : public throws (void)\nMethod setStoreHost(java.lang.String) : public throws (void)\nMethod setStoreTimestamp(long) : public throws (void)\nMethod setSysFlag(java.lang.Integer) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\nMethod setUnitMode(boolean) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.ResetOffsetRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField group : private java.lang.String null\nField isForce : private boolean false\nField timestamp : private long 0\nField topic : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getGroup() : public throws (java.lang.String)\nMethod getTimestamp() : public throws (long)\nMethod getTopic() : public throws (java.lang.String)\nMethod isForce() : public throws (boolean)\nMethod setForce(boolean) : public throws (void)\nMethod setGroup(java.lang.String) : public throws (void)\nMethod setTimestamp(long) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.ResumeCheckHalfMessageRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField msgId : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getMsgId() : public throws (java.lang.String)\nMethod setMsgId(java.lang.String) : public throws (void)\nMethod toString() : public throws (java.lang.String)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.SearchOffsetRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField queueId : private java.lang.Integer null\nField timestamp : private java.lang.Long null\nField topic : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getQueueId() : public throws (java.lang.Integer)\nMethod getTimestamp() : public throws (java.lang.Long)\nMethod getTopic() : public throws (java.lang.String)\nMethod setQueueId(java.lang.Integer) : public throws (void)\nMethod setTimestamp(java.lang.Long) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.SearchOffsetResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField offset : private java.lang.Long null\nMethod checkFields() : public throws (void)\nMethod getOffset() : public throws (java.lang.Long)\nMethod setOffset(java.lang.Long) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.SendMessageRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField batch : private boolean false\nField bornTimestamp : private java.lang.Long null\nField defaultTopic : private java.lang.String null\nField defaultTopicQueueNums : private java.lang.Integer null\nField flag : private java.lang.Integer null\nField maxReconsumeTimes : private java.lang.Integer null\nField producerGroup : private java.lang.String null\nField properties : private java.lang.String null\nField queueId : private java.lang.Integer null\nField reconsumeTimes : private java.lang.Integer null\nField sysFlag : private java.lang.Integer null\nField topic : private java.lang.String null\nField unitMode : private boolean false\nMethod checkFields() : public throws (void)\nMethod getBornTimestamp() : public throws (java.lang.Long)\nMethod getDefaultTopic() : public throws (java.lang.String)\nMethod getDefaultTopicQueueNums() : public throws (java.lang.Integer)\nMethod getFlag() : public throws (java.lang.Integer)\nMethod getMaxReconsumeTimes() : public throws (java.lang.Integer)\nMethod getProducerGroup() : public throws (java.lang.String)\nMethod getProperties() : public throws (java.lang.String)\nMethod getQueueId() : public throws (java.lang.Integer)\nMethod getReconsumeTimes() : public throws (java.lang.Integer)\nMethod getSysFlag() : public throws (java.lang.Integer)\nMethod getTopic() : public throws (java.lang.String)\nMethod isBatch() : public throws (boolean)\nMethod isUnitMode() : public throws (boolean)\nMethod setBatch(boolean) : public throws (void)\nMethod setBornTimestamp(java.lang.Long) : public throws (void)\nMethod setDefaultTopic(java.lang.String) : public throws (void)\nMethod setDefaultTopicQueueNums(java.lang.Integer) : public throws (void)\nMethod setFlag(java.lang.Integer) : public throws (void)\nMethod setMaxReconsumeTimes(java.lang.Integer) : public throws (void)\nMethod setProducerGroup(java.lang.String) : public throws (void)\nMethod setProperties(java.lang.String) : public throws (void)\nMethod setQueueId(java.lang.Integer) : public throws (void)\nMethod setReconsumeTimes(java.lang.Integer) : public throws (void)\nMethod setSysFlag(java.lang.Integer) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\nMethod setUnitMode(boolean) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.SendMessageRequestHeaderV2.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField a : private java.lang.String null\nField b : private java.lang.String null\nField c : private java.lang.String null\nField d : private java.lang.Integer null\nField e : private java.lang.Integer null\nField f : private java.lang.Integer null\nField g : private java.lang.Long null\nField h : private java.lang.Integer null\nField i : private java.lang.String null\nField j : private java.lang.Integer null\nField k : private boolean false\nField l : private java.lang.Integer null\nField m : private boolean false\nMethod checkFields() : public throws (void)\nMethod createSendMessageRequestHeaderV1(org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2) : public throws (org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader)\nMethod createSendMessageRequestHeaderV2(org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader) : public throws (org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2)\nMethod decode(java.util.HashMap) : public throws (void)\nMethod encode(io.netty.buffer.ByteBuf) : public throws (void)\nMethod getA() : public throws (java.lang.String)\nMethod getB() : public throws (java.lang.String)\nMethod getC() : public throws (java.lang.String)\nMethod getD() : public throws (java.lang.Integer)\nMethod getE() : public throws (java.lang.Integer)\nMethod getF() : public throws (java.lang.Integer)\nMethod getG() : public throws (java.lang.Long)\nMethod getH() : public throws (java.lang.Integer)\nMethod getI() : public throws (java.lang.String)\nMethod getJ() : public throws (java.lang.Integer)\nMethod getL() : public throws (java.lang.Integer)\nMethod isK() : public throws (boolean)\nMethod isM() : public throws (boolean)\nMethod setA(java.lang.String) : public throws (void)\nMethod setB(java.lang.String) : public throws (void)\nMethod setC(java.lang.String) : public throws (void)\nMethod setD(java.lang.Integer) : public throws (void)\nMethod setE(java.lang.Integer) : public throws (void)\nMethod setF(java.lang.Integer) : public throws (void)\nMethod setG(java.lang.Long) : public throws (void)\nMethod setH(java.lang.Integer) : public throws (void)\nMethod setI(java.lang.String) : public throws (void)\nMethod setJ(java.lang.Integer) : public throws (void)\nMethod setK(boolean) : public throws (void)\nMethod setL(java.lang.Integer) : public throws (void)\nMethod setM(boolean) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.SendMessageResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField msgId : private java.lang.String null\nField queueId : private java.lang.Integer null\nField queueOffset : private java.lang.Long null\nField transactionId : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod decode(java.util.HashMap) : public throws (void)\nMethod encode(io.netty.buffer.ByteBuf) : public throws (void)\nMethod getMsgId() : public throws (java.lang.String)\nMethod getQueueId() : public throws (java.lang.Integer)\nMethod getQueueOffset() : public throws (java.lang.Long)\nMethod getTransactionId() : public throws (java.lang.String)\nMethod setMsgId(java.lang.String) : public throws (void)\nMethod setQueueId(java.lang.Integer) : public throws (void)\nMethod setQueueOffset(java.lang.Long) : public throws (void)\nMethod setTransactionId(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.UnregisterClientRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField clientID : private java.lang.String null\nField consumerGroup : private java.lang.String null\nField producerGroup : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getClientID() : public throws (java.lang.String)\nMethod getConsumerGroup() : public throws (java.lang.String)\nMethod getProducerGroup() : public throws (java.lang.String)\nMethod setClientID(java.lang.String) : public throws (void)\nMethod setConsumerGroup(java.lang.String) : public throws (void)\nMethod setProducerGroup(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.UnregisterClientResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nMethod checkFields() : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.UpdateConsumerOffsetRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField commitOffset : private java.lang.Long null\nField consumerGroup : private java.lang.String null\nField queueId : private java.lang.Integer null\nField topic : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getCommitOffset() : public throws (java.lang.Long)\nMethod getConsumerGroup() : public throws (java.lang.String)\nMethod getQueueId() : public throws (java.lang.Integer)\nMethod getTopic() : public throws (java.lang.String)\nMethod setCommitOffset(java.lang.Long) : public throws (void)\nMethod setConsumerGroup(java.lang.String) : public throws (void)\nMethod setQueueId(java.lang.Integer) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.UpdateConsumerOffsetResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nMethod checkFields() : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.UpdateGlobalWhiteAddrsConfigRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField aclFileFullPath : private java.lang.String null\nField globalWhiteAddrs : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getAclFileFullPath() : public throws (java.lang.String)\nMethod getGlobalWhiteAddrs() : public throws (java.lang.String)\nMethod setAclFileFullPath(java.lang.String) : public throws (void)\nMethod setGlobalWhiteAddrs(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.ViewBrokerStatsDataRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField statsKey : private java.lang.String null\nField statsName : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getStatsKey() : public throws (java.lang.String)\nMethod getStatsName() : public throws (java.lang.String)\nMethod setStatsKey(java.lang.String) : public throws (void)\nMethod setStatsName(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.ViewMessageRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField offset : private java.lang.Long null\nMethod checkFields() : public throws (void)\nMethod getOffset() : public throws (java.lang.Long)\nMethod setOffset(java.lang.Long) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.ViewMessageResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nMethod checkFields() : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.filtersrv.RegisterFilterServerRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField filterServerAddr : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getFilterServerAddr() : public throws (java.lang.String)\nMethod setFilterServerAddr(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.filtersrv.RegisterFilterServerResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField brokerId : private long 0\nField brokerName : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getBrokerId() : public throws (long)\nMethod getBrokerName() : public throws (java.lang.String)\nMethod setBrokerId(long) : public throws (void)\nMethod setBrokerName(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.filtersrv.RegisterMessageFilterClassRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField classCRC : private java.lang.Integer null\nField className : private java.lang.String null\nField consumerGroup : private java.lang.String null\nField topic : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getClassCRC() : public throws (java.lang.Integer)\nMethod getClassName() : public throws (java.lang.String)\nMethod getConsumerGroup() : public throws (java.lang.String)\nMethod getTopic() : public throws (java.lang.String)\nMethod setClassCRC(java.lang.Integer) : public throws (void)\nMethod setClassName(java.lang.String) : public throws (void)\nMethod setConsumerGroup(java.lang.String) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.namesrv.AddWritePermOfBrokerRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField brokerName : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getBrokerName() : public throws (java.lang.String)\nMethod setBrokerName(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.namesrv.AddWritePermOfBrokerResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField addTopicCount : private java.lang.Integer null\nMethod checkFields() : public throws (void)\nMethod getAddTopicCount() : public throws (java.lang.Integer)\nMethod setAddTopicCount(java.lang.Integer) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.namesrv.DeleteKVConfigRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField key : private java.lang.String null\nField namespace : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getKey() : public throws (java.lang.String)\nMethod getNamespace() : public throws (java.lang.String)\nMethod setKey(java.lang.String) : public throws (void)\nMethod setNamespace(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.namesrv.DeleteTopicFromNamesrvRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField clusterName : private java.lang.String null\nField topic : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getClusterName() : public throws (java.lang.String)\nMethod getTopic() : public throws (java.lang.String)\nMethod setClusterName(java.lang.String) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.namesrv.GetKVConfigRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField key : private java.lang.String null\nField namespace : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getKey() : public throws (java.lang.String)\nMethod getNamespace() : public throws (java.lang.String)\nMethod setKey(java.lang.String) : public throws (void)\nMethod setNamespace(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.namesrv.GetKVConfigResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField value : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getValue() : public throws (java.lang.String)\nMethod setValue(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.namesrv.GetKVListByNamespaceRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField namespace : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getNamespace() : public throws (java.lang.String)\nMethod setNamespace(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.namesrv.GetRouteInfoRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField acceptStandardJsonOnly : private java.lang.Boolean null\nField topic : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getAcceptStandardJsonOnly() : public throws (java.lang.Boolean)\nMethod getTopic() : public throws (java.lang.String)\nMethod setAcceptStandardJsonOnly(java.lang.Boolean) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.namesrv.PutKVConfigRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField key : private java.lang.String null\nField namespace : private java.lang.String null\nField value : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getKey() : public throws (java.lang.String)\nMethod getNamespace() : public throws (java.lang.String)\nMethod getValue() : public throws (java.lang.String)\nMethod setKey(java.lang.String) : public throws (void)\nMethod setNamespace(java.lang.String) : public throws (void)\nMethod setValue(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.namesrv.QueryDataVersionRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField brokerAddr : private java.lang.String null\nField brokerId : private java.lang.Long null\nField brokerName : private java.lang.String null\nField clusterName : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getBrokerAddr() : public throws (java.lang.String)\nMethod getBrokerId() : public throws (java.lang.Long)\nMethod getBrokerName() : public throws (java.lang.String)\nMethod getClusterName() : public throws (java.lang.String)\nMethod setBrokerAddr(java.lang.String) : public throws (void)\nMethod setBrokerId(java.lang.Long) : public throws (void)\nMethod setBrokerName(java.lang.String) : public throws (void)\nMethod setClusterName(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.namesrv.QueryDataVersionResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField changed : private java.lang.Boolean null\nMethod checkFields() : public throws (void)\nMethod getChanged() : public throws (java.lang.Boolean)\nMethod setChanged(java.lang.Boolean) : public throws (void)\nMethod toString() : public throws (java.lang.String)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.namesrv.RegisterBrokerRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField bodyCrc32 : private java.lang.Integer 0\nField brokerAddr : private java.lang.String null\nField brokerId : private java.lang.Long null\nField brokerName : private java.lang.String null\nField clusterName : private java.lang.String null\nField compressed : private boolean false\nField haServerAddr : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getBodyCrc32() : public throws (java.lang.Integer)\nMethod getBrokerAddr() : public throws (java.lang.String)\nMethod getBrokerId() : public throws (java.lang.Long)\nMethod getBrokerName() : public throws (java.lang.String)\nMethod getClusterName() : public throws (java.lang.String)\nMethod getHaServerAddr() : public throws (java.lang.String)\nMethod isCompressed() : public throws (boolean)\nMethod setBodyCrc32(java.lang.Integer) : public throws (void)\nMethod setBrokerAddr(java.lang.String) : public throws (void)\nMethod setBrokerId(java.lang.Long) : public throws (void)\nMethod setBrokerName(java.lang.String) : public throws (void)\nMethod setClusterName(java.lang.String) : public throws (void)\nMethod setCompressed(boolean) : public throws (void)\nMethod setHaServerAddr(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.namesrv.RegisterBrokerResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField haServerAddr : private java.lang.String null\nField masterAddr : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getHaServerAddr() : public throws (java.lang.String)\nMethod getMasterAddr() : public throws (java.lang.String)\nMethod setHaServerAddr(java.lang.String) : public throws (void)\nMethod setMasterAddr(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.namesrv.RegisterOrderTopicRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField orderTopicString : private java.lang.String null\nField topic : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getOrderTopicString() : public throws (java.lang.String)\nMethod getTopic() : public throws (java.lang.String)\nMethod setOrderTopicString(java.lang.String) : public throws (void)\nMethod setTopic(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.namesrv.UnRegisterBrokerRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField brokerAddr : private java.lang.String null\nField brokerId : private java.lang.Long null\nField brokerName : private java.lang.String null\nField clusterName : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getBrokerAddr() : public throws (java.lang.String)\nMethod getBrokerId() : public throws (java.lang.Long)\nMethod getBrokerName() : public throws (java.lang.String)\nMethod getClusterName() : public throws (java.lang.String)\nMethod setBrokerAddr(java.lang.String) : public throws (void)\nMethod setBrokerId(java.lang.Long) : public throws (void)\nMethod setBrokerName(java.lang.String) : public throws (void)\nMethod setClusterName(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.namesrv.WipeWritePermOfBrokerRequestHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField brokerName : private java.lang.String null\nMethod checkFields() : public throws (void)\nMethod getBrokerName() : public throws (java.lang.String)\nMethod setBrokerName(java.lang.String) : public throws (void)\n"
  },
  {
    "path": "test/src/test/resources/schema/protocol/common.protocol.header.namesrv.WipeWritePermOfBrokerResponseHeader.schema",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nField wipeTopicCount : private java.lang.Integer null\nMethod checkFields() : public throws (void)\nMethod getWipeTopicCount() : public throws (java.lang.Integer)\nMethod setWipeTopicCount(java.lang.Integer) : public throws (void)\n"
  },
  {
    "path": "tieredstore/BUILD.bazel",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\nload(\"//bazel:GenTestRules.bzl\", \"GenTestRules\")\n\njava_library(\n    name = \"tieredstore\",\n    srcs = glob([\"src/main/java/**/*.java\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \"//common\",\n        \"//remoting\",\n        \"//store\",\n        \"@maven//:com_github_ben_manes_caffeine_caffeine\",\n        \"@maven//:com_google_code_findbugs_jsr305\",\n        \"@maven//:com_google_guava_guava\",\n        \"@maven//:io_github_aliyunmq_rocketmq_slf4j_api\",\n        \"@maven//:io_github_aliyunmq_rocketmq_logback_classic\",\n        \"@maven//:io_opentelemetry_opentelemetry_api\",\n        \"@maven//:io_opentelemetry_opentelemetry_context\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_otlp\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_prometheus\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk_common\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk_metrics\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp\",\n        \"@maven//:org_apache_commons_commons_lang3\",\n        \"@maven//:org_apache_tomcat_annotations_api\",\n        \"@maven//:com_alibaba_fastjson2_fastjson2\",\n        \"@maven//:org_apache_rocketmq_rocketmq_rocksdb\",\n        \"@maven//:commons_collections_commons_collections\",\n        \"@maven//:org_slf4j_slf4j_api\",\n    ],\n)\n\njava_library(\n    name = \"tests\",\n    srcs = glob([\"src/test/java/**/*.java\"]),\n    resources = glob([\"src/test/resources/certs/*.pem\"]) + glob([\"src/test/resources/certs/*.key\"]),\n    visibility = [\"//visibility:public\"],\n    deps = [\n        \":tieredstore\",\n        \"//:test_deps\",\n        \"//common\",\n        \"//remoting\",\n        \"//store\",\n        \"@maven//:com_alibaba_fastjson2_fastjson2\",\n        \"@maven//:commons_io_commons_io\",\n        \"@maven//:io_opentelemetry_opentelemetry_api\",\n        \"@maven//:io_opentelemetry_opentelemetry_context\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_otlp\",\n        \"@maven//:io_opentelemetry_opentelemetry_exporter_prometheus\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk_common\",\n        \"@maven//:io_opentelemetry_opentelemetry_sdk_metrics\",\n        \"@maven//:org_apache_commons_commons_lang3\",\n        \"@maven//:com_google_guava_guava\",\n        \"@maven//:io_github_aliyunmq_rocketmq_slf4j_api\",\n        \"@maven//:io_github_aliyunmq_rocketmq_logback_classic\",\n        \"@maven//:net_java_dev_jna_jna\",\n        \"@maven//:org_slf4j_slf4j_api\",\n    ],\n)\n\nGenTestRules(\n    name = \"GeneratedTestRules\",\n    exclude_tests = [\n    ],\n    medium_tests = [\n    ],\n    test_files = glob([\"src/test/java/**/*Test.java\"]),\n    deps = [\n        \":tests\",\n    ],\n)\n"
  },
  {
    "path": "tieredstore/README.md",
    "content": "# Tiered storage for RocketMQ (Technical preview)\n\nRocketMQ tiered storage allows users to offload message data from the local disk to other cheaper and larger storage mediums. So that users can extend the message reserve time at a lower cost. And different topics can flexibly specify different TTL as needed.\n\nThis article is a cookbook for RocketMQ tiered storage.\n\n## Architecture\n\n![Tiered storage architecture](tiered_storage_arch.png)\n\n## Quick start\n\nUse the following steps to easily use tiered storage\n\n1. Change `messageStorePlugIn` to `org.apache.rocketmq.tieredstore.TieredMessageStore` in your `broker.conf`.\n2. Configure your backend service provider. Change `tieredBackendServiceProvider` to your storage medium implementation. We provide a default implementation: POSIX provider, and you need to change `tieredStoreFilePath` to the mount point of the storage medium for tiered storage.\n3. Start the broker and enjoy!\n\n## Configuration\n\nThe following are some core configurations, for more details, see [TieredMessageStoreConfig](https://github.com/apache/rocketmq/blob/develop/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredMessageStoreConfig.java)\n\n| Configuration                   | Default value                                                 | Unit        | Function                                                                        |\n| ------------------------------- |---------------------------------------------------------------| ----------- |---------------------------------------------------------------------------------|\n| messageStorePlugIn              |                                                               |             | Set to org.apache.rocketmq.tieredstore.TieredMessageStore to use tiered storage |\n| tieredMetadataServiceProvider   | org.apache.rocketmq.tieredstore.metadata.DefaultMetadataStore |             | Select your metadata provider                                                   |\n| tieredBackendServiceProvider    | org.apache.rocketmq.tieredstore.provider.PosixFileSegment     |             | Select your backend service provider                                            |\n| tieredStoreFilePath             |                                                               |             | Select the directory used for tiered storage, only for POSIX provider.          |\n| tieredStorageLevel              | NOT_IN_DISK                                                   |             | The options are DISABLE, NOT_IN_DISK, NOT_IN_MEM, FORCE                         |\n| tieredStoreFileReservedTime     | 72                                                            | hour        | Default topic TTL in tiered storage                                             |\n| tieredStoreGroupCommitCount     | 2500                                                          |             | The number of messages that trigger one batch transfer                          |\n| tieredStoreGroupCommitSize      | 33554432                                                      | byte        | The size of messages that trigger one batch transfer, 32M by default            |\n| tieredStoreMaxGroupCommitCount  | 10000                                                         |             | The maximum number of messages waiting to be transferred per queue              |\n| readAheadCacheExpireDuration    | 1000                                                          | millisecond | Read-ahead cache expiration time                                                |\n| readAheadCacheSizeThresholdRate | 0.3                                                           |             | The maximum heap space occupied by the read-ahead cache                         |\n\n## Metrics\n\nTiered storage provides some useful metrics, see [RIP-46](https://github.com/apache/rocketmq/wiki/RIP-46-Observability-improvement-for-RocketMQ) for details.\n\n| Type      | Name                                                | Unit         |\n| --------- | --------------------------------------------------- | ------------ |\n| Histogram | rocketmq_tiered_store_api_latency                   | milliseconds |\n| Histogram | rocketmq_tiered_store_provider_rpc_latency          | milliseconds |\n| Histogram | rocketmq_tiered_store_provider_upload_bytes         | byte         |\n| Histogram | rocketmq_tiered_store_provider_download_bytes       | byte         |\n| Gauge     | rocketmq_tiered_store_dispatch_behind               |              |\n| Gauge     | rocketmq_tiered_store_dispatch_latency              | milliseconds |\n| Counter   | rocketmq_tiered_store_messages_dispatch_total       |              |\n| Counter   | rocketmq_tiered_store_messages_out_total            |              |\n| Counter   | rocketmq_tiered_store_get_message_fallback_total    |              |\n| Gauge     | rocketmq_tiered_store_read_ahead_cache_count        |              |\n| Gauge     | rocketmq_tiered_store_read_ahead_cache_bytes        | bytes        |\n| Counter   | rocketmq_tiered_store_read_ahead_cache_access_total |              |\n| Counter   | rocketmq_tiered_store_read_ahead_cache_hit_total    |              |\n| Gauge     | rocketmq_storage_message_reserve_time               | milliseconds |\n\n## How to contribute\n\nWe need community participation to add more backend service providers for tiered storage. [PosixFileSegment](https://github.com/apache/rocketmq/blob/develop/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/PosixFileSegment.java), the implementation provided by default is just an example. People who want to contribute can follow it to implement their own providers, such as S3FileSegment, OSSFileSegment, and MinIOFileSegment. Here are some guidelines:\n\n1. Extend [FileSegment](https://github.com/apache/rocketmq/blob/develop/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegment.java) and implement the methods of [FileSegmentProvider](https://github.com/apache/rocketmq/blob/develop/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegmentProvider.java) interface.\n2. Record metrics where appropriate. See `rocketmq_tiered_store_provider_rpc_latency`, `rocketmq_tiered_store_provider_upload_bytes`, and `rocketmq_tiered_store_provider_download_bytes`\n3. No need to maintain your own cache and avoid polluting the page cache. It already has the read-ahead cache.\n"
  },
  {
    "path": "tieredstore/pom.xml",
    "content": "<!--\n  Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license agreements.  See the NOTICE file distributed with\n  this work for additional information regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License, Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n  -->\n\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n    <parent>\n        <groupId>org.apache.rocketmq</groupId>\n        <artifactId>rocketmq-all</artifactId>\n        <version>${revision}</version>\n    </parent>\n\n    <modelVersion>4.0.0</modelVersion>\n    <packaging>jar</packaging>\n    <artifactId>rocketmq-tiered-store</artifactId>\n    <name>rocketmq-tiered-store ${project.version}</name>\n\n    <properties>\n        <project.root>${basedir}/..</project.root>\n    </properties>\n\n    <dependencies>\n        <dependency>\n            <groupId>org.apache.rocketmq</groupId>\n            <artifactId>rocketmq-store</artifactId>\n        </dependency>\n\n        <dependency>\n            <groupId>com.github.ben-manes.caffeine</groupId>\n            <artifactId>caffeine</artifactId>\n            <exclusions>\n                <exclusion>\n                    <groupId>org.checkerframework</groupId>\n                    <artifactId>checker-qual</artifactId>\n                </exclusion>\n            </exclusions>\n        </dependency>\n\n        <dependency>\n            <groupId>commons-io</groupId>\n            <artifactId>commons-io</artifactId>\n            <scope>test</scope>\n        </dependency>\n\n        <dependency>\n            <groupId>org.openjdk.jmh</groupId>\n            <artifactId>jmh-core</artifactId>\n            <version>1.36</version>\n            <scope>provided</scope>\n        </dependency>\n\n        <dependency>\n            <groupId>org.openjdk.jmh</groupId>\n            <artifactId>jmh-generator-annprocess</artifactId>\n            <version>1.36</version>\n            <scope>provided</scope>\n        </dependency>\n    </dependencies>\n</project>\n"
  },
  {
    "path": "tieredstore/src/main/java/org/apache/rocketmq/tieredstore/MessageStoreConfig.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.tieredstore;\n\nimport java.io.File;\nimport java.net.InetAddress;\nimport java.net.UnknownHostException;\nimport java.time.Duration;\n\npublic class MessageStoreConfig {\n\n    private String brokerName = localHostName();\n    private String brokerClusterName = \"DefaultCluster\";\n    private TieredStorageLevel tieredStorageLevel = TieredStorageLevel.NOT_IN_DISK;\n\n    /**\n     * All fetch requests are judged against this level first,\n     * and if the message cannot be read from the TiredMessageStore,\n     * these requests will still go to the next store for fallback processing.\n     */\n    public enum TieredStorageLevel {\n        /**\n         * Disable tiered storage, all fetch request will be handled by default message store.\n         */\n        DISABLE(0),\n        /**\n         * Only fetch request with offset not in disk will be handled by tiered storage.\n         */\n        NOT_IN_DISK(1),\n        /**\n         * Only fetch request with offset not in memory(page cache) will be handled by tiered storage.\n         */\n        NOT_IN_MEM(2),\n        /**\n         * All fetch request will be handled by tiered storage.\n         */\n        FORCE(3);\n\n        private final int value;\n\n        TieredStorageLevel(int value) {\n            this.value = value;\n        }\n\n        public int getValue() {\n            return value;\n        }\n\n        @SuppressWarnings(\"DuplicatedCode\")\n        public static TieredStorageLevel valueOf(int value) {\n            switch (value) {\n                case 1:\n                    return NOT_IN_DISK;\n                case 2:\n                    return NOT_IN_MEM;\n                case 3:\n                    return FORCE;\n                default:\n                    return DISABLE;\n            }\n        }\n\n        public boolean isEnable() {\n            return this.value > 0;\n        }\n\n        public boolean check(TieredStorageLevel targetLevel) {\n            return this.value >= targetLevel.value;\n        }\n    }\n\n    private String storePathRootDir = System.getProperty(\"user.home\") + File.separator + \"store\";\n    private boolean messageIndexEnable = true;\n    private boolean recordGetMessageResult = false;\n\n    // CommitLog file size, default is 1G\n    private long tieredStoreCommitLogMaxSize = 1024 * 1024 * 1024;\n    // ConsumeQueue file size, default is 100M\n    private long tieredStoreConsumeQueueMaxSize = 100 * 1024 * 1024;\n    private int tieredStoreIndexFileMaxHashSlotNum = 5000000;\n    private int tieredStoreIndexFileMaxIndexNum = 5000000 * 4;\n\n    private String tieredMetadataServiceProvider = \"org.apache.rocketmq.tieredstore.metadata.DefaultMetadataStore\";\n    private String tieredBackendServiceProvider = \"org.apache.rocketmq.tieredstore.provider.MemoryFileSegment\";\n\n    // file reserved time, default is 72 hour\n    private boolean tieredStoreDeleteFileEnable = true;\n    private int tieredStoreFileReservedTime = 72;\n    private long tieredStoreDeleteFileInterval = Duration.ofHours(1).toMillis();\n\n    // time of forcing commitLog to roll to next file, default is 24 hour\n    private int commitLogRollingInterval = 24;\n    private int commitLogRollingMinimumSize = 16 * 1024 * 1024;\n\n    private boolean tieredStoreGroupCommit = true;\n    private int tieredStoreGroupCommitTimeout = 30 * 1000;\n    // Cached message count larger than this value will trigger async commit. default is 4096\n    private int tieredStoreGroupCommitCount = 4 * 1024;\n    // Cached message size larger than this value will trigger async commit. default is 4M\n    private int tieredStoreGroupCommitSize = 4 * 1024 * 1024;\n    // Cached message count larger than this value will suspend append. default is 10000\n    private int tieredStoreMaxGroupCommitCount = 10000;\n\n    private boolean readAheadCacheEnable = true;\n    private int readAheadMessageCountThreshold = 4096;\n    private int readAheadMessageSizeThreshold = 16 * 1024 * 1024;\n    private long readAheadCacheExpireDuration = 15 * 1000;\n    private double readAheadCacheSizeThresholdRate = 0.3;\n\n    private int tieredStoreMaxPendingLimit = 10000;\n    private boolean tieredStoreCrcCheckEnable = false;\n\n    private String tieredStoreFilePath = \"\";\n    private String objectStoreEndpoint = \"\";\n    private String objectStoreBucket = \"\";\n    private String objectStoreAccessKey = \"\";\n    private String objectStoreSecretKey = \"\";\n    private boolean writeWithoutMmap = false;\n\n    public static String localHostName() {\n        try {\n            return InetAddress.getLocalHost().getHostName();\n        } catch (UnknownHostException ignore) {\n        }\n\n        return \"DEFAULT_BROKER\";\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public void setBrokerName(String brokerName) {\n        this.brokerName = brokerName;\n    }\n\n    public String getBrokerClusterName() {\n        return brokerClusterName;\n    }\n\n    public void setBrokerClusterName(String brokerClusterName) {\n        this.brokerClusterName = brokerClusterName;\n    }\n\n    public TieredStorageLevel getTieredStorageLevel() {\n        return tieredStorageLevel;\n    }\n\n    public void setTieredStorageLevel(TieredStorageLevel tieredStorageLevel) {\n        this.tieredStorageLevel = tieredStorageLevel;\n    }\n\n    public void setTieredStorageLevel(int tieredStorageLevel) {\n        this.tieredStorageLevel = TieredStorageLevel.valueOf(tieredStorageLevel);\n    }\n\n    public void setTieredStorageLevel(String tieredStorageLevel) {\n        this.tieredStorageLevel = TieredStorageLevel.valueOf(tieredStorageLevel);\n    }\n\n    public String getStorePathRootDir() {\n        return storePathRootDir;\n    }\n\n    public void setStorePathRootDir(String storePathRootDir) {\n        this.storePathRootDir = storePathRootDir;\n    }\n\n    public boolean isMessageIndexEnable() {\n        return messageIndexEnable;\n    }\n\n    public void setMessageIndexEnable(boolean messageIndexEnable) {\n        this.messageIndexEnable = messageIndexEnable;\n    }\n\n    public boolean isRecordGetMessageResult() {\n        return recordGetMessageResult;\n    }\n\n    public void setRecordGetMessageResult(boolean recordGetMessageResult) {\n        this.recordGetMessageResult = recordGetMessageResult;\n    }\n\n    public long getTieredStoreCommitLogMaxSize() {\n        return tieredStoreCommitLogMaxSize;\n    }\n\n    public void setTieredStoreCommitLogMaxSize(long tieredStoreCommitLogMaxSize) {\n        this.tieredStoreCommitLogMaxSize = tieredStoreCommitLogMaxSize;\n    }\n\n    public long getTieredStoreConsumeQueueMaxSize() {\n        return tieredStoreConsumeQueueMaxSize;\n    }\n\n    public void setTieredStoreConsumeQueueMaxSize(long tieredStoreConsumeQueueMaxSize) {\n        this.tieredStoreConsumeQueueMaxSize = tieredStoreConsumeQueueMaxSize;\n    }\n\n    public int getTieredStoreIndexFileMaxHashSlotNum() {\n        return tieredStoreIndexFileMaxHashSlotNum;\n    }\n\n    public void setTieredStoreIndexFileMaxHashSlotNum(int tieredStoreIndexFileMaxHashSlotNum) {\n        this.tieredStoreIndexFileMaxHashSlotNum = tieredStoreIndexFileMaxHashSlotNum;\n    }\n\n    public int getTieredStoreIndexFileMaxIndexNum() {\n        return tieredStoreIndexFileMaxIndexNum;\n    }\n\n    public void setTieredStoreIndexFileMaxIndexNum(int tieredStoreIndexFileMaxIndexNum) {\n        this.tieredStoreIndexFileMaxIndexNum = tieredStoreIndexFileMaxIndexNum;\n    }\n\n    public String getTieredMetadataServiceProvider() {\n        return tieredMetadataServiceProvider;\n    }\n\n    public void setTieredMetadataServiceProvider(String tieredMetadataServiceProvider) {\n        this.tieredMetadataServiceProvider = tieredMetadataServiceProvider;\n    }\n\n    public String getTieredBackendServiceProvider() {\n        return tieredBackendServiceProvider;\n    }\n\n    public void setTieredBackendServiceProvider(String tieredBackendServiceProvider) {\n        this.tieredBackendServiceProvider = tieredBackendServiceProvider;\n    }\n\n    public boolean isTieredStoreDeleteFileEnable() {\n        return tieredStoreDeleteFileEnable;\n    }\n\n    public void setTieredStoreDeleteFileEnable(boolean tieredStoreDeleteFileEnable) {\n        this.tieredStoreDeleteFileEnable = tieredStoreDeleteFileEnable;\n    }\n\n    public int getTieredStoreFileReservedTime() {\n        return tieredStoreFileReservedTime;\n    }\n\n    public void setTieredStoreFileReservedTime(int tieredStoreFileReservedTime) {\n        this.tieredStoreFileReservedTime = tieredStoreFileReservedTime;\n    }\n\n    public long getTieredStoreDeleteFileInterval() {\n        return tieredStoreDeleteFileInterval;\n    }\n\n    public void setTieredStoreDeleteFileInterval(long tieredStoreDeleteFileInterval) {\n        this.tieredStoreDeleteFileInterval = tieredStoreDeleteFileInterval;\n    }\n\n    public int getCommitLogRollingInterval() {\n        return commitLogRollingInterval;\n    }\n\n    public void setCommitLogRollingInterval(int commitLogRollingInterval) {\n        this.commitLogRollingInterval = commitLogRollingInterval;\n    }\n\n    public int getCommitLogRollingMinimumSize() {\n        return commitLogRollingMinimumSize;\n    }\n\n    public void setCommitLogRollingMinimumSize(int commitLogRollingMinimumSize) {\n        this.commitLogRollingMinimumSize = commitLogRollingMinimumSize;\n    }\n\n    public boolean isTieredStoreGroupCommit() {\n        return tieredStoreGroupCommit;\n    }\n\n    public void setTieredStoreGroupCommit(boolean tieredStoreGroupCommit) {\n        this.tieredStoreGroupCommit = tieredStoreGroupCommit;\n    }\n\n    public int getTieredStoreGroupCommitTimeout() {\n        return tieredStoreGroupCommitTimeout;\n    }\n\n    public void setTieredStoreGroupCommitTimeout(int tieredStoreGroupCommitTimeout) {\n        this.tieredStoreGroupCommitTimeout = tieredStoreGroupCommitTimeout;\n    }\n\n    public int getTieredStoreGroupCommitCount() {\n        return tieredStoreGroupCommitCount;\n    }\n\n    public void setTieredStoreGroupCommitCount(int tieredStoreGroupCommitCount) {\n        this.tieredStoreGroupCommitCount = tieredStoreGroupCommitCount;\n    }\n\n    public int getTieredStoreGroupCommitSize() {\n        return tieredStoreGroupCommitSize;\n    }\n\n    public void setTieredStoreGroupCommitSize(int tieredStoreGroupCommitSize) {\n        this.tieredStoreGroupCommitSize = tieredStoreGroupCommitSize;\n    }\n\n    public int getTieredStoreMaxGroupCommitCount() {\n        return tieredStoreMaxGroupCommitCount;\n    }\n\n    public void setTieredStoreMaxGroupCommitCount(int tieredStoreMaxGroupCommitCount) {\n        this.tieredStoreMaxGroupCommitCount = tieredStoreMaxGroupCommitCount;\n    }\n\n    public boolean isReadAheadCacheEnable() {\n        return readAheadCacheEnable;\n    }\n\n    public void setReadAheadCacheEnable(boolean readAheadCacheEnable) {\n        this.readAheadCacheEnable = readAheadCacheEnable;\n    }\n\n    public int getReadAheadMessageCountThreshold() {\n        return readAheadMessageCountThreshold;\n    }\n\n    public void setReadAheadMessageCountThreshold(int readAheadMessageCountThreshold) {\n        this.readAheadMessageCountThreshold = readAheadMessageCountThreshold;\n    }\n\n    public int getReadAheadMessageSizeThreshold() {\n        return readAheadMessageSizeThreshold;\n    }\n\n    public void setReadAheadMessageSizeThreshold(int readAheadMessageSizeThreshold) {\n        this.readAheadMessageSizeThreshold = readAheadMessageSizeThreshold;\n    }\n\n    public long getReadAheadCacheExpireDuration() {\n        return readAheadCacheExpireDuration;\n    }\n\n    public void setReadAheadCacheExpireDuration(long duration) {\n        this.readAheadCacheExpireDuration = duration;\n    }\n\n    public double getReadAheadCacheSizeThresholdRate() {\n        return readAheadCacheSizeThresholdRate;\n    }\n\n    public void setReadAheadCacheSizeThresholdRate(double rate) {\n        this.readAheadCacheSizeThresholdRate = rate;\n    }\n\n    public int getTieredStoreMaxPendingLimit() {\n        return tieredStoreMaxPendingLimit;\n    }\n\n    public void setTieredStoreMaxPendingLimit(int tieredStoreMaxPendingLimit) {\n        this.tieredStoreMaxPendingLimit = tieredStoreMaxPendingLimit;\n    }\n\n    public boolean isTieredStoreCrcCheckEnable() {\n        return tieredStoreCrcCheckEnable;\n    }\n\n    public void setTieredStoreCrcCheckEnable(boolean tieredStoreCrcCheckEnable) {\n        this.tieredStoreCrcCheckEnable = tieredStoreCrcCheckEnable;\n    }\n\n    public String getTieredStoreFilePath() {\n        return tieredStoreFilePath;\n    }\n\n    public void setTieredStoreFilePath(String tieredStoreFilePath) {\n        this.tieredStoreFilePath = tieredStoreFilePath;\n    }\n\n    public void setObjectStoreEndpoint(String objectStoreEndpoint) {\n        this.objectStoreEndpoint = objectStoreEndpoint;\n    }\n\n    public String getObjectStoreBucket() {\n        return objectStoreBucket;\n    }\n\n    public void setObjectStoreBucket(String objectStoreBucket) {\n        this.objectStoreBucket = objectStoreBucket;\n    }\n\n    public String getObjectStoreAccessKey() {\n        return objectStoreAccessKey;\n    }\n\n    public void setObjectStoreAccessKey(String objectStoreAccessKey) {\n        this.objectStoreAccessKey = objectStoreAccessKey;\n    }\n\n    public String getObjectStoreSecretKey() {\n        return objectStoreSecretKey;\n    }\n\n    public void setObjectStoreSecretKey(String objectStoreSecretKey) {\n        this.objectStoreSecretKey = objectStoreSecretKey;\n    }\n\n    public String getObjectStoreEndpoint() {\n        return objectStoreEndpoint;\n    }\n\n    public boolean isWriteWithoutMmap() {\n        return writeWithoutMmap;\n    }\n\n    public void setWriteWithoutMmap(boolean writeWithoutMmap) {\n        this.writeWithoutMmap = writeWithoutMmap;\n    }\n}\n"
  },
  {
    "path": "tieredstore/src/main/java/org/apache/rocketmq/tieredstore/MessageStoreExecutor.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.tieredstore;\n\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.common.ThreadFactoryImpl;\nimport org.apache.rocketmq.common.utils.ThreadUtils;\n\npublic class MessageStoreExecutor {\n\n    public final BlockingQueue<Runnable> bufferCommitThreadPoolQueue;\n    public final BlockingQueue<Runnable> bufferFetchThreadPoolQueue;\n    public final BlockingQueue<Runnable> fileRecyclingThreadPoolQueue;\n\n    public final ScheduledExecutorService commonExecutor;\n    public final ExecutorService bufferCommitExecutor;\n    public final ExecutorService bufferFetchExecutor;\n    public final ExecutorService fileRecyclingExecutor;\n\n    private static class SingletonHolder {\n        private static final MessageStoreExecutor INSTANCE = new MessageStoreExecutor();\n    }\n\n    public static MessageStoreExecutor getInstance() {\n        return SingletonHolder.INSTANCE;\n    }\n\n    public MessageStoreExecutor() {\n        this(10000);\n    }\n\n    public MessageStoreExecutor(int maxQueueCapacity) {\n\n        this.commonExecutor = ThreadUtils.newScheduledThreadPool(\n            Math.max(4, Runtime.getRuntime().availableProcessors()),\n            new ThreadFactoryImpl(\"TieredCommonExecutor_\"));\n\n        this.bufferCommitThreadPoolQueue = new LinkedBlockingQueue<>(maxQueueCapacity);\n        this.bufferCommitExecutor = ThreadUtils.newThreadPoolExecutor(\n            Math.max(16, Runtime.getRuntime().availableProcessors() * 4),\n            Math.max(16, Runtime.getRuntime().availableProcessors() * 4),\n            TimeUnit.MINUTES.toMillis(1), TimeUnit.MILLISECONDS,\n            this.bufferCommitThreadPoolQueue,\n            new ThreadFactoryImpl(\"BufferCommitExecutor_\"));\n\n        this.bufferFetchThreadPoolQueue = new LinkedBlockingQueue<>(maxQueueCapacity);\n        this.bufferFetchExecutor = ThreadUtils.newThreadPoolExecutor(\n            Math.max(16, Runtime.getRuntime().availableProcessors() * 4),\n            Math.max(16, Runtime.getRuntime().availableProcessors() * 4),\n            TimeUnit.MINUTES.toMillis(1), TimeUnit.MILLISECONDS,\n            this.bufferFetchThreadPoolQueue,\n            new ThreadFactoryImpl(\"BufferFetchExecutor_\"));\n\n        this.fileRecyclingThreadPoolQueue = new LinkedBlockingQueue<>(maxQueueCapacity);\n        this.fileRecyclingExecutor = ThreadUtils.newThreadPoolExecutor(\n            Math.max(4, Runtime.getRuntime().availableProcessors()),\n            Math.max(4, Runtime.getRuntime().availableProcessors()),\n            TimeUnit.MINUTES.toMillis(1), TimeUnit.MILLISECONDS,\n            this.fileRecyclingThreadPoolQueue,\n            new ThreadFactoryImpl(\"BufferFetchExecutor_\"));\n    }\n\n    private void shutdownExecutor(ExecutorService executor) {\n        if (executor != null) {\n            executor.shutdown();\n        }\n    }\n\n    public void shutdown() {\n        this.shutdownExecutor(this.commonExecutor);\n        this.shutdownExecutor(this.bufferCommitExecutor);\n        this.shutdownExecutor(this.bufferFetchExecutor);\n        this.shutdownExecutor(this.fileRecyclingExecutor);\n    }\n}\n"
  },
  {
    "path": "tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.tieredstore;\n\nimport com.google.common.base.Stopwatch;\nimport com.google.common.collect.Sets;\nimport io.opentelemetry.api.common.Attributes;\nimport io.opentelemetry.api.common.AttributesBuilder;\nimport io.opentelemetry.api.metrics.Meter;\nimport io.opentelemetry.sdk.metrics.InstrumentSelector;\nimport io.opentelemetry.sdk.metrics.ViewBuilder;\nimport java.lang.reflect.Constructor;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.TimeUnit;\nimport java.util.function.Supplier;\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.Pair;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.MessageFilter;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.QueryMessageResult;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.plugin.AbstractPluginMessageStore;\nimport org.apache.rocketmq.store.plugin.MessageStorePluginContext;\nimport org.apache.rocketmq.store.rocksdb.MessageRocksDBStorage;\nimport org.apache.rocketmq.store.timer.rocksdb.TimerMessageRocksDBStore;\nimport org.apache.rocketmq.store.transaction.TransMessageRocksDBStore;\nimport org.apache.rocketmq.tieredstore.core.MessageStoreDispatcher;\nimport org.apache.rocketmq.tieredstore.core.MessageStoreDispatcherImpl;\nimport org.apache.rocketmq.tieredstore.core.MessageStoreFetcher;\nimport org.apache.rocketmq.tieredstore.core.MessageStoreFetcherImpl;\nimport org.apache.rocketmq.tieredstore.core.MessageStoreFilter;\nimport org.apache.rocketmq.tieredstore.core.MessageStoreTopicFilter;\nimport org.apache.rocketmq.tieredstore.file.FlatFileStore;\nimport org.apache.rocketmq.tieredstore.file.FlatMessageFile;\nimport org.apache.rocketmq.tieredstore.index.IndexService;\nimport org.apache.rocketmq.tieredstore.index.IndexStoreService;\nimport org.apache.rocketmq.tieredstore.metadata.MetadataStore;\nimport org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant;\nimport org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager;\nimport org.apache.rocketmq.tieredstore.util.MessageStoreUtil;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\npublic class TieredMessageStore extends AbstractPluginMessageStore {\n\n    protected static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME);\n    protected static final long MIN_STORE_TIME = -1L;\n\n    protected final String brokerName;\n    protected final MessageStore defaultStore;\n    protected final MessageStoreConfig storeConfig;\n    protected final MessageStorePluginContext context;\n\n    protected final MetadataStore metadataStore;\n    protected final MessageStoreExecutor storeExecutor;\n    protected final IndexService indexService;\n    protected final FlatFileStore flatFileStore;\n    protected final MessageStoreFilter topicFilter;\n    protected final MessageStoreFetcher fetcher;\n    protected final MessageStoreDispatcher dispatcher;\n    protected final MessageRocksDBStorage messageRocksDBStorage;\n    protected TimerMessageRocksDBStore timerMessageRocksDBStore;\n    protected TransMessageRocksDBStore transMessageRocksDBStore;\n\n    public TieredMessageStore(MessageStorePluginContext context, MessageStore next) {\n        super(context, next);\n\n        this.storeConfig = new MessageStoreConfig();\n        this.context = context;\n        this.context.registerConfiguration(this.storeConfig);\n        this.storeConfig.setWriteWithoutMmap(context.getMessageStoreConfig().isWriteWithoutMmap());\n        this.brokerName = this.storeConfig.getBrokerName();\n        this.defaultStore = next;\n        this.messageRocksDBStorage = defaultStore.getMessageRocksDBStorage();\n        this.metadataStore = this.getMetadataStore(this.storeConfig);\n        this.topicFilter = new MessageStoreTopicFilter(this.storeConfig);\n        this.storeExecutor = new MessageStoreExecutor();\n        this.flatFileStore = new FlatFileStore(this.storeConfig, this.metadataStore, this.storeExecutor);\n        this.indexService = new IndexStoreService(this.flatFileStore.getFlatFileFactory(),\n            MessageStoreUtil.getIndexFilePath(this.storeConfig.getBrokerName()));\n        this.fetcher = new MessageStoreFetcherImpl(this);\n        this.dispatcher = new MessageStoreDispatcherImpl(this);\n        next.addDispatcher(dispatcher);\n    }\n\n    @Override\n    public boolean load() {\n        boolean loadFlatFile = flatFileStore.load();\n        boolean loadNextStore = next.load();\n        boolean result = loadFlatFile && loadNextStore;\n        if (result) {\n            indexService.start();\n            dispatcher.start();\n            storeExecutor.commonExecutor.scheduleWithFixedDelay(\n                flatFileStore::scheduleDeleteExpireFile, storeConfig.getTieredStoreDeleteFileInterval(),\n                storeConfig.getTieredStoreDeleteFileInterval(), TimeUnit.MILLISECONDS);\n        }\n        return result;\n    }\n\n    public String getBrokerName() {\n        return brokerName;\n    }\n\n    public MessageStoreConfig getStoreConfig() {\n        return storeConfig;\n    }\n\n    public MessageStore getDefaultStore() {\n        return defaultStore;\n    }\n\n    private MetadataStore getMetadataStore(MessageStoreConfig storeConfig) {\n        try {\n            Class<? extends MetadataStore> clazz =\n                Class.forName(storeConfig.getTieredMetadataServiceProvider()).asSubclass(MetadataStore.class);\n            Constructor<? extends MetadataStore> constructor = clazz.getConstructor(MessageStoreConfig.class);\n            return constructor.newInstance(storeConfig);\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    public MetadataStore getMetadataStore() {\n        return metadataStore;\n    }\n\n    public MessageStoreFilter getTopicFilter() {\n        return topicFilter;\n    }\n\n    public MessageStoreExecutor getStoreExecutor() {\n        return storeExecutor;\n    }\n\n    public FlatFileStore getFlatFileStore() {\n        return flatFileStore;\n    }\n\n    public IndexService getIndexService() {\n        return indexService;\n    }\n\n    public boolean fetchFromCurrentStore(String topic, int queueId, long offset) {\n        return fetchFromCurrentStore(topic, queueId, offset, 1);\n    }\n\n    @SuppressWarnings(\"all\")\n    public boolean fetchFromCurrentStore(String topic, int queueId, long offset, int batchSize) {\n        MessageStoreConfig.TieredStorageLevel storageLevel = storeConfig.getTieredStorageLevel();\n\n        if (storageLevel.check(MessageStoreConfig.TieredStorageLevel.FORCE)) {\n            return true;\n        }\n\n        if (!storageLevel.isEnable()) {\n            return false;\n        }\n\n        FlatMessageFile flatFile = flatFileStore.getFlatFile(new MessageQueue(topic, brokerName, queueId));\n        if (flatFile == null) {\n            return false;\n        }\n\n        if (offset >= flatFile.getConsumeQueueCommitOffset()) {\n            return false;\n        }\n\n        // determine whether tiered storage path conditions are met\n        if (storageLevel.check(MessageStoreConfig.TieredStorageLevel.NOT_IN_DISK)) {\n            // return true to read from tiered storage if the CommitLog is empty\n            if (next != null && next.getCommitLog() != null &&\n                next.getCommitLog().getMinOffset() < 0L) {\n                return true;\n            }\n            if (!next.checkInStoreByConsumeOffset(topic, queueId, offset)) {\n                return true;\n            }\n        }\n\n        if (storageLevel.check(MessageStoreConfig.TieredStorageLevel.NOT_IN_MEM)\n            && !next.checkInMemByConsumeOffset(topic, queueId, offset, batchSize)) {\n            return true;\n        }\n        return false;\n    }\n\n    @Override\n    public GetMessageResult getMessage(String group, String topic, int queueId, long offset, int maxMsgNums,\n        MessageFilter messageFilter) {\n        return getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter).join();\n    }\n\n    @Override\n    public CompletableFuture<GetMessageResult> getMessageAsync(String group, String topic,\n        int queueId, long offset, int maxMsgNums, MessageFilter messageFilter) {\n\n        // for system topic, force reading from local store\n        if (topicFilter.filterTopic(topic)) {\n            return next.getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter);\n        }\n\n        if (fetchFromCurrentStore(topic, queueId, offset, maxMsgNums)) {\n            log.trace(\"GetMessageAsync from remote store, \" +\n                \"topic: {}, queue: {}, offset: {}, maxCount: {}\", topic, queueId, offset, maxMsgNums);\n        } else {\n            log.trace(\"GetMessageAsync from next store, \" +\n                \"topic: {}, queue: {}, offset: {}, maxCount: {}\", topic, queueId, offset, maxMsgNums);\n            return next.getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter);\n        }\n\n        Stopwatch stopwatch = Stopwatch.createStarted();\n        return fetcher\n            .getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter)\n            .thenApply(result -> {\n\n                Attributes latencyAttributes = TieredStoreMetricsManager.newAttributesBuilder()\n                    .put(TieredStoreMetricsConstant.LABEL_OPERATION, TieredStoreMetricsConstant.OPERATION_API_GET_MESSAGE)\n                    .put(TieredStoreMetricsConstant.LABEL_TOPIC, topic)\n                    .put(TieredStoreMetricsConstant.LABEL_GROUP, group)\n                    .build();\n                TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes);\n\n                if (result.getStatus() == GetMessageStatus.OFFSET_FOUND_NULL ||\n                    result.getStatus() == GetMessageStatus.NO_MATCHED_LOGIC_QUEUE) {\n\n                    if (next.checkInStoreByConsumeOffset(topic, queueId, offset)) {\n                        TieredStoreMetricsManager.fallbackTotal.add(1, latencyAttributes);\n                        log.debug(\"GetMessageAsync not found, then back to next store, result: {}, \" +\n                                \"topic: {}, queue: {}, queue offset: {}, offset range: {}-{}\",\n                            result.getStatus(), topic, queueId, offset, result.getMinOffset(), result.getMaxOffset());\n                        return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter);\n                    }\n                }\n\n                if (result.getStatus() != GetMessageStatus.FOUND &&\n                    result.getStatus() != GetMessageStatus.NO_MESSAGE_IN_QUEUE &&\n                    result.getStatus() != GetMessageStatus.NO_MATCHED_LOGIC_QUEUE &&\n                    result.getStatus() != GetMessageStatus.OFFSET_TOO_SMALL &&\n                    result.getStatus() != GetMessageStatus.OFFSET_OVERFLOW_ONE &&\n                    result.getStatus() != GetMessageStatus.OFFSET_OVERFLOW_BADLY) {\n                    log.warn(\"GetMessageAsync not found and message is not in next store, result: {}, \" +\n                            \"topic: {}, queue: {}, queue offset: {}, offset range: {}-{}\",\n                        result.getStatus(), topic, queueId, offset, result.getMinOffset(), result.getMaxOffset());\n                }\n\n                if (result.getStatus() == GetMessageStatus.FOUND) {\n                    Attributes messagesOutAttributes = TieredStoreMetricsManager.newAttributesBuilder()\n                        .put(TieredStoreMetricsConstant.LABEL_TOPIC, topic)\n                        .put(TieredStoreMetricsConstant.LABEL_GROUP, group)\n                        .build();\n                    TieredStoreMetricsManager.messagesOutTotal.add(result.getMessageCount(), messagesOutAttributes);\n\n                    if (next.getStoreStatsService() != null) {\n                        next.getStoreStatsService().getGetMessageTransferredMsgCount().add(result.getMessageCount());\n                    }\n                }\n\n                // Fix min or max offset according next store at last\n                long minOffsetInQueue = next.getMinOffsetInQueue(topic, queueId);\n                if (minOffsetInQueue >= 0 && minOffsetInQueue < result.getMinOffset()) {\n                    result.setMinOffset(minOffsetInQueue);\n                }\n\n                // In general, the local cq offset is slightly greater than the commit offset in read message,\n                // so there is no need to update the maximum offset to the local cq offset here,\n                // otherwise it will cause repeated consumption after next start offset over commit offset.\n\n                if (storeConfig.isRecordGetMessageResult()) {\n                    log.info(\"GetMessageAsync result, {}, group: {}, topic: {}, queueId: {}, offset: {}, count:{}\",\n                        result, group, topic, queueId, offset, maxMsgNums);\n                }\n\n                return result;\n            }).exceptionally(e -> {\n                log.error(\"GetMessageAsync from tiered store failed\", e);\n                return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter);\n            });\n    }\n\n    @Override\n    public long getMinOffsetInQueue(String topic, int queueId) {\n        long minOffsetInNextStore = next.getMinOffsetInQueue(topic, queueId);\n        FlatMessageFile flatFile = flatFileStore.getFlatFile(new MessageQueue(topic, brokerName, queueId));\n        if (flatFile == null) {\n            return minOffsetInNextStore;\n        }\n        long minOffsetInTieredStore = flatFile.getConsumeQueueMinOffset();\n        if (minOffsetInTieredStore < 0) {\n            return minOffsetInNextStore;\n        }\n        return Math.min(minOffsetInNextStore, minOffsetInTieredStore);\n    }\n\n    @Override\n    public TimerMessageRocksDBStore getTimerMessageRocksDBStore() {\n        return timerMessageRocksDBStore;\n    }\n\n    @Override\n    public TransMessageRocksDBStore getTransMessageRocksDBStore() {\n        return transMessageRocksDBStore;\n    }\n\n    @Override\n    public void setTimerMessageRocksDBStore(TimerMessageRocksDBStore timerMessageRocksDBStore) {\n        this.timerMessageRocksDBStore = timerMessageRocksDBStore;\n    }\n\n    @Override\n    public void setTransMessageRocksDBStore(TransMessageRocksDBStore transMessageRocksDBStore) {\n        this.transMessageRocksDBStore = transMessageRocksDBStore;\n    }\n\n    @Override\n    public long getEarliestMessageTime(String topic, int queueId) {\n        return getEarliestMessageTimeAsync(topic, queueId).join();\n    }\n\n    /**\n     * In the original design, getting the earliest time of the first message\n     * would generate two RPC requests. However, using the timestamp stored in the metadata\n     * avoids these requests, although this approach might introduce some level of inaccuracy.\n     */\n    @Override\n    public CompletableFuture<Long> getEarliestMessageTimeAsync(String topic, int queueId) {\n        long localMinTime = next.getEarliestMessageTime(topic, queueId);\n        return fetcher.getEarliestMessageTimeAsync(topic, queueId)\n            .thenApply(remoteMinTime -> {\n                if (localMinTime > MIN_STORE_TIME && remoteMinTime > MIN_STORE_TIME) {\n                    return Math.min(localMinTime, remoteMinTime);\n                }\n                return localMinTime > MIN_STORE_TIME ? localMinTime :\n                    (remoteMinTime > MIN_STORE_TIME ? remoteMinTime : MIN_STORE_TIME);\n            });\n    }\n\n    @Override\n    public long getMessageStoreTimeStamp(String topic, int queueId, long consumeQueueOffset) {\n        return getMessageStoreTimeStampAsync(topic, queueId, consumeQueueOffset).join();\n    }\n\n    @Override\n    public CompletableFuture<Long> getMessageStoreTimeStampAsync(String topic, int queueId, long consumeQueueOffset) {\n        if (fetchFromCurrentStore(topic, queueId, consumeQueueOffset)) {\n            Stopwatch stopwatch = Stopwatch.createStarted();\n            return fetcher.getMessageStoreTimeStampAsync(topic, queueId, consumeQueueOffset)\n                .thenApply(time -> {\n                    Attributes latencyAttributes = TieredStoreMetricsManager.newAttributesBuilder()\n                        .put(TieredStoreMetricsConstant.LABEL_OPERATION,\n                            TieredStoreMetricsConstant.OPERATION_API_GET_TIME_BY_OFFSET)\n                        .put(TieredStoreMetricsConstant.LABEL_TOPIC, topic)\n                        .build();\n                    TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes);\n                    return time;\n                });\n        }\n        return next.getMessageStoreTimeStampAsync(topic, queueId, consumeQueueOffset);\n    }\n\n    @Override\n    public long getOffsetInQueueByTime(String topic, int queueId, long timestamp) {\n        return getOffsetInQueueByTime(topic, queueId, timestamp, BoundaryType.LOWER);\n    }\n\n    @Override\n    public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) {\n        boolean isForce = storeConfig.getTieredStorageLevel() == MessageStoreConfig.TieredStorageLevel.FORCE;\n        if (timestamp < next.getEarliestMessageTime() || isForce) {\n            Stopwatch stopwatch = Stopwatch.createStarted();\n            long offsetInTieredStore = fetcher.getOffsetInQueueByTime(topic, queueId, timestamp, boundaryType);\n            Attributes latencyAttributes = TieredStoreMetricsManager.newAttributesBuilder()\n                .put(TieredStoreMetricsConstant.LABEL_OPERATION, TieredStoreMetricsConstant.OPERATION_API_GET_OFFSET_BY_TIME)\n                .put(TieredStoreMetricsConstant.LABEL_TOPIC, topic)\n                .build();\n            TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes);\n            if (offsetInTieredStore == -1L && !isForce) {\n                return next.getOffsetInQueueByTime(topic, queueId, timestamp, boundaryType);\n            }\n            return offsetInTieredStore;\n        }\n        return next.getOffsetInQueueByTime(topic, queueId, timestamp, boundaryType);\n    }\n\n    @Override\n    public QueryMessageResult queryMessage(String topic, String key, int maxNum, long begin, long end) {\n        return queryMessageAsync(topic, key, maxNum, begin, end).join();\n    }\n\n    @Override\n    public QueryMessageResult queryMessage(String topic, String key, int maxNum, long begin,\n        long end, String keyType, String lastKey) {\n        return queryMessageAsync(topic, key, maxNum, begin, end, keyType, lastKey).join();\n    }\n\n    @Override\n    public CompletableFuture<QueryMessageResult> queryMessageAsync(String topic, String key,\n        int maxNum, long begin, long end) {\n        long earliestTimeInNextStore = next.getEarliestMessageTime();\n        if (earliestTimeInNextStore <= 0) {\n            log.warn(\"TieredMessageStore#queryMessageAsync: get earliest message time in next store failed: {}\", earliestTimeInNextStore);\n        }\n        boolean isForce = storeConfig.getTieredStorageLevel() == MessageStoreConfig.TieredStorageLevel.FORCE;\n        QueryMessageResult result = end < earliestTimeInNextStore || isForce ?\n            new QueryMessageResult() :\n            next.queryMessage(topic, key, maxNum, begin, end);\n        int resultSize = result.getMessageBufferList().size();\n        if (resultSize < maxNum && begin < earliestTimeInNextStore || isForce) {\n            Stopwatch stopwatch = Stopwatch.createStarted();\n            try {\n                return fetcher.queryMessageAsync(topic, key, maxNum - resultSize, begin, isForce ? end : earliestTimeInNextStore)\n                    .thenApply(tieredStoreResult -> {\n                        Attributes latencyAttributes = TieredStoreMetricsManager.newAttributesBuilder()\n                            .put(TieredStoreMetricsConstant.LABEL_OPERATION, TieredStoreMetricsConstant.OPERATION_API_QUERY_MESSAGE)\n                            .put(TieredStoreMetricsConstant.LABEL_TOPIC, topic)\n                            .build();\n                        TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes);\n                        for (SelectMappedBufferResult msg : tieredStoreResult.getMessageMapedList()) {\n                            result.addMessage(msg);\n                        }\n                        return result;\n                    });\n            } catch (Exception e) {\n                log.error(\"TieredMessageStore#queryMessageAsync: query message in tiered store failed\", e);\n                return CompletableFuture.completedFuture(result);\n            }\n        }\n        return CompletableFuture.completedFuture(result);\n    }\n\n    @Override\n    public CompletableFuture<QueryMessageResult> queryMessageAsync(String topic, String key, int maxNum, long begin, long end, String indexType, String lastKey) {\n        long earliestTimeInNextStore = next.getEarliestMessageTime();\n        if (earliestTimeInNextStore <= 0) {\n            log.warn(\"TieredMessageStore queryMessageAsync: get earliest message time in next store failed: {}\", earliestTimeInNextStore);\n        }\n        boolean isForce = storeConfig.getTieredStorageLevel() == MessageStoreConfig.TieredStorageLevel.FORCE;\n        QueryMessageResult result = end < earliestTimeInNextStore || isForce ? new QueryMessageResult() : next.queryMessage(topic, key, maxNum, begin, end, indexType, lastKey);\n        int resultSize = result.getMessageBufferList().size();\n        if (resultSize < maxNum && begin < earliestTimeInNextStore || isForce) {\n            Stopwatch stopwatch = Stopwatch.createStarted();\n            try {\n                return fetcher.queryMessageAsync(topic, key, maxNum - resultSize, begin, isForce ? end : earliestTimeInNextStore)\n                    .thenApply(tieredStoreResult -> {\n                        Attributes latencyAttributes = TieredStoreMetricsManager.newAttributesBuilder()\n                            .put(TieredStoreMetricsConstant.LABEL_OPERATION, TieredStoreMetricsConstant.OPERATION_API_QUERY_MESSAGE)\n                            .put(TieredStoreMetricsConstant.LABEL_TOPIC, topic)\n                            .build();\n                        TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes);\n                        for (SelectMappedBufferResult msg : tieredStoreResult.getMessageMapedList()) {\n                            result.addMessage(msg);\n                        }\n                        return result;\n                    });\n            } catch (Exception e) {\n                log.error(\"TieredMessageStore#queryMessageAsync: query message in tiered store failed\", e);\n                return CompletableFuture.completedFuture(result);\n            }\n        }\n        return CompletableFuture.completedFuture(result);\n    }\n\n    @Override\n    public List<Pair<InstrumentSelector, ViewBuilder>> getMetricsView() {\n        List<Pair<InstrumentSelector, ViewBuilder>> res = super.getMetricsView();\n        res.addAll(TieredStoreMetricsManager.getMetricsView());\n        return res;\n    }\n\n    @Override\n    public void initMetrics(Meter meter, Supplier<AttributesBuilder> attributesBuilderSupplier) {\n        super.initMetrics(meter, attributesBuilderSupplier);\n        TieredStoreMetricsManager.init(meter, attributesBuilderSupplier, storeConfig, fetcher, flatFileStore, next);\n    }\n\n    @Override\n    public MessageRocksDBStorage getMessageRocksDBStorage() {\n        return messageRocksDBStorage;\n    }\n\n    @Override\n    public int cleanUnusedTopic(Set<String> retainTopics) {\n        metadataStore.iterateTopic(topicMetadata -> {\n            String topic = topicMetadata.getTopic();\n            if (retainTopics.contains(topic) ||\n                TopicValidator.isSystemTopic(topic) ||\n                MixAll.isLmq(topic)) {\n                return;\n            }\n            this.deleteTopics(Sets.newHashSet(topicMetadata.getTopic()));\n        });\n        return next.cleanUnusedTopic(retainTopics);\n    }\n\n    @Override\n    public int deleteTopics(Set<String> deleteTopics) {\n        for (String topic : deleteTopics) {\n            metadataStore.iterateQueue(topic, queueMetadata -> {\n                flatFileStore.destroyFile(queueMetadata.getQueue());\n            });\n            metadataStore.deleteTopic(topic);\n            log.info(\"MessageStore delete topic success, topicName={}\", topic);\n        }\n        return next.deleteTopics(deleteTopics);\n    }\n\n    @Override\n    public synchronized void shutdown() {\n        if (next != null) {\n            next.shutdown();\n        }\n        if (dispatcher != null) {\n            dispatcher.shutdown();\n        }\n        if (indexService != null) {\n            if (defaultStore.getRunningFlags() != null && defaultStore.getRunningFlags().isStoreWriteable()) {\n                indexService.shutdown();\n            } else {\n                indexService.forceShutdown();\n            }\n        }\n\n        if (flatFileStore != null) {\n            flatFileStore.shutdown();\n        }\n        if (storeExecutor != null) {\n            storeExecutor.shutdown();\n        }\n    }\n\n    @Override\n    public void destroy() {\n        if (next != null) {\n            next.destroy();\n        }\n        if (indexService != null) {\n            indexService.destroy();\n        }\n        if (flatFileStore != null) {\n            flatFileStore.destroy();\n        }\n        if (metadataStore != null) {\n            metadataStore.destroy();\n        }\n    }\n}\n"
  },
  {
    "path": "tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/AppendResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.tieredstore.common;\n\npublic enum AppendResult {\n\n    /**\n     * The append operation was successful.\n     */\n    SUCCESS,\n\n    /**\n     * The buffer used for the append operation is full.\n     */\n    BUFFER_FULL,\n\n    /**\n     * The file is full and cannot accept more data.\n     */\n    FILE_FULL,\n\n    /**\n     * The file is closed and cannot accept more data.\n     */\n    FILE_CLOSED,\n\n    /**\n     * An unknown error occurred during the append operation.\n     */\n    UNKNOWN_ERROR\n}\n"
  },
  {
    "path": "tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/FileSegmentType.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.tieredstore.common;\n\nimport java.util.Arrays;\n\npublic enum FileSegmentType {\n\n    COMMIT_LOG(0),\n\n    CONSUME_QUEUE(1),\n\n    INDEX(2);\n\n    private final int code;\n\n    FileSegmentType(int code) {\n        this.code = code;\n    }\n\n    public int getCode() {\n        return code;\n    }\n\n    public static FileSegmentType valueOf(int fileType) {\n        return Arrays.stream(FileSegmentType.values())\n            .filter(segmentType -> segmentType.getCode() == fileType)\n            .findFirst()\n            .orElse(COMMIT_LOG);\n    }\n}"
  },
  {
    "path": "tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/GetMessageResultExt.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.tieredstore.common;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.MessageFilter;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\n\npublic class GetMessageResultExt extends GetMessageResult {\n\n    private final List<Long> tagCodeList;\n\n    public GetMessageResultExt() {\n        this.tagCodeList = new ArrayList<>();\n    }\n\n    public void addMessageExt(SelectMappedBufferResult bufferResult, long queueOffset, long tagCode) {\n        super.addMessage(bufferResult, queueOffset);\n        this.tagCodeList.add(tagCode);\n    }\n\n    public List<Long> getTagCodeList() {\n        return tagCodeList;\n    }\n\n    /**\n     * Due to the message fetched from the object storage is sequential,\n     * do message filtering occurs after the data retrieval.\n     */\n    public GetMessageResult doFilterMessage(MessageFilter messageFilter) {\n        if (GetMessageStatus.FOUND != super.getStatus() || messageFilter == null) {\n            return this;\n        }\n\n        GetMessageResult result = new GetMessageResult();\n        result.setStatus(GetMessageStatus.FOUND);\n        result.setMinOffset(this.getMinOffset());\n        result.setMaxOffset(this.getMaxOffset());\n        result.setNextBeginOffset(this.getNextBeginOffset());\n\n        for (int i = 0; i < this.getMessageMapedList().size(); i++) {\n            if (!messageFilter.isMatchedByConsumeQueue(this.tagCodeList.get(i), null)) {\n                continue;\n            }\n\n            SelectMappedBufferResult bufferResult = this.getMessageMapedList().get(i);\n            if (!messageFilter.isMatchedByCommitLog(bufferResult.getByteBuffer().slice(), null)) {\n                continue;\n            }\n\n            long offset = this.getMessageQueueOffset().get(i);\n            result.addMessage(new SelectMappedBufferResult(bufferResult.getStartOffset(),\n                bufferResult.getByteBuffer().asReadOnlyBuffer(), bufferResult.getSize(), null), offset);\n        }\n\n        if (result.getBufferTotalSize() == 0) {\n            result.setStatus(GetMessageStatus.NO_MATCHED_MESSAGE);\n        }\n        return result;\n    }\n}\n"
  },
  {
    "path": "tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/GroupCommitContext.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.tieredstore.common;\n\nimport java.util.List;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\n\npublic class GroupCommitContext {\n\n    private long endOffset;\n\n    private List<SelectMappedBufferResult> bufferList;\n\n    private List<DispatchRequest> dispatchRequests;\n\n    public long getEndOffset() {\n        return endOffset;\n    }\n\n    public void setEndOffset(long endOffset) {\n        this.endOffset = endOffset;\n    }\n\n    public List<SelectMappedBufferResult> getBufferList() {\n        return bufferList;\n    }\n\n    public void setBufferList(List<SelectMappedBufferResult> bufferList) {\n        this.bufferList = bufferList;\n    }\n\n    public List<DispatchRequest> getDispatchRequests() {\n        return dispatchRequests;\n    }\n\n    public void setDispatchRequests(List<DispatchRequest> dispatchRequests) {\n        this.dispatchRequests = dispatchRequests;\n    }\n\n    public void release() {\n        if (bufferList != null) {\n            for (SelectMappedBufferResult bufferResult : bufferList) {\n                bufferResult.release();\n            }\n            bufferList.clear();\n            bufferList = null;\n        }\n        if (dispatchRequests != null) {\n            dispatchRequests.clear();\n            dispatchRequests = null;\n        }\n\n    }\n}\n"
  },
  {
    "path": "tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/SelectBufferResult.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.tieredstore.common;\n\nimport java.nio.ByteBuffer;\nimport java.util.concurrent.atomic.AtomicLong;\n\npublic class SelectBufferResult {\n\n    private final ByteBuffer byteBuffer;\n    private final long startOffset;\n    private final int size;\n    private final long tagCode;\n    private final AtomicLong accessCount;\n\n    public SelectBufferResult(ByteBuffer byteBuffer, long startOffset, int size, long tagCode) {\n        this.startOffset = startOffset;\n        this.byteBuffer = byteBuffer;\n        this.size = size;\n        this.tagCode = tagCode;\n        this.accessCount = new AtomicLong();\n    }\n\n    public ByteBuffer getByteBuffer() {\n        return byteBuffer;\n    }\n\n    public long getStartOffset() {\n        return startOffset;\n    }\n\n    public int getSize() {\n        return size;\n    }\n\n    public long getTagCode() {\n        return tagCode;\n    }\n\n    public AtomicLong getAccessCount() {\n        return accessCount;\n    }\n}\n"
  },
  {
    "path": "tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.tieredstore.core;\n\nimport com.google.common.annotations.VisibleForTesting;\nimport io.opentelemetry.api.common.Attributes;\nimport java.nio.ByteBuffer;\nimport java.time.Duration;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.Semaphore;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.message.MessageConst;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.MessageStore;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.store.exception.ConsumeQueueException;\nimport org.apache.rocketmq.store.queue.ConsumeQueueInterface;\nimport org.apache.rocketmq.store.queue.CqUnit;\nimport org.apache.rocketmq.tieredstore.MessageStoreConfig;\nimport org.apache.rocketmq.tieredstore.MessageStoreExecutor;\nimport org.apache.rocketmq.tieredstore.TieredMessageStore;\nimport org.apache.rocketmq.tieredstore.common.AppendResult;\nimport org.apache.rocketmq.tieredstore.common.FileSegmentType;\nimport org.apache.rocketmq.tieredstore.common.GroupCommitContext;\nimport org.apache.rocketmq.tieredstore.file.FlatFileInterface;\nimport org.apache.rocketmq.tieredstore.file.FlatFileStore;\nimport org.apache.rocketmq.tieredstore.index.IndexService;\nimport org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant;\nimport org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager;\nimport org.apache.rocketmq.tieredstore.util.MessageFormatUtil;\nimport org.apache.rocketmq.tieredstore.util.MessageStoreUtil;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\npublic class MessageStoreDispatcherImpl extends ServiceThread implements MessageStoreDispatcher {\n\n    protected static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME);\n\n    protected final String brokerName;\n    protected final MessageStore defaultStore;\n    protected final MessageStoreConfig storeConfig;\n    protected final TieredMessageStore messageStore;\n    protected final FlatFileStore flatFileStore;\n    protected final MessageStoreExecutor storeExecutor;\n    protected final MessageStoreFilter topicFilter;\n    protected final Semaphore semaphore;\n    protected final IndexService indexService;\n    protected final Map<FlatFileInterface, GroupCommitContext> failedGroupCommitMap;\n\n    public MessageStoreDispatcherImpl(TieredMessageStore messageStore) {\n        this.messageStore = messageStore;\n        this.storeConfig = messageStore.getStoreConfig();\n        this.defaultStore = messageStore.getDefaultStore();\n        this.brokerName = storeConfig.getBrokerName();\n        this.semaphore = new Semaphore(\n            this.storeConfig.getTieredStoreMaxPendingLimit() / 4);\n        this.topicFilter = messageStore.getTopicFilter();\n        this.flatFileStore = messageStore.getFlatFileStore();\n        this.storeExecutor = messageStore.getStoreExecutor();\n        this.indexService = messageStore.getIndexService();\n        this.failedGroupCommitMap = new ConcurrentHashMap<>();\n    }\n\n    @Override\n    public String getServiceName() {\n        return MessageStoreDispatcher.class.getSimpleName();\n    }\n\n    @VisibleForTesting\n    public Map<FlatFileInterface, GroupCommitContext> getFailedGroupCommitMap() {\n        return failedGroupCommitMap;\n    }\n\n    public void dispatchWithSemaphore(FlatFileInterface flatFile) {\n        try {\n            if (stopped) {\n                return;\n            }\n            semaphore.acquire();\n            this.doScheduleDispatch(flatFile, false)\n                .whenComplete((future, throwable) -> semaphore.release());\n        } catch (Throwable t) {\n            semaphore.release();\n            log.error(\"MessageStore dispatch error, topic={}, queueId={}\",\n                flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId(), t);\n        }\n    }\n\n    @Override\n    public void dispatch(DispatchRequest request) {\n        if (stopped || topicFilter != null && topicFilter.filterTopic(request.getTopic())) {\n            return;\n        }\n        flatFileStore.computeIfAbsent(\n            new MessageQueue(request.getTopic(), brokerName, request.getQueueId()));\n    }\n\n    @Override\n    public CompletableFuture<Boolean> doScheduleDispatch(FlatFileInterface flatFile, boolean force) {\n        if (stopped) {\n            return CompletableFuture.completedFuture(true);\n        }\n\n        String topic = flatFile.getMessageQueue().getTopic();\n        int queueId = flatFile.getMessageQueue().getQueueId();\n\n        // For test scenarios, we set the 'force' variable to true to\n        // ensure that the data in the cache is directly committed successfully.\n        force = !storeConfig.isTieredStoreGroupCommit() || force;\n        if (force) {\n            flatFile.getFileLock().lock();\n        } else {\n            if (!flatFile.getFileLock().tryLock()) {\n                return CompletableFuture.completedFuture(false);\n            }\n        }\n\n        try {\n            if (topicFilter != null && topicFilter.filterTopic(flatFile.getMessageQueue().getTopic())) {\n                flatFileStore.destroyFile(flatFile.getMessageQueue());\n                return CompletableFuture.completedFuture(false);\n            }\n\n            long currentOffset = flatFile.getConsumeQueueMaxOffset();\n            long commitOffset = flatFile.getConsumeQueueCommitOffset();\n            long minOffsetInQueue = defaultStore.getMinOffsetInQueue(topic, queueId);\n            long maxOffsetInQueue = defaultStore.getMaxOffsetInQueue(topic, queueId);\n\n            // If set to max offset here, some written messages may be lost\n            if (!flatFile.isFlatFileInit()) {\n                currentOffset = defaultStore.getOffsetInQueueByTime(\n                    topic, queueId, System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(2));\n                currentOffset = Math.max(currentOffset, minOffsetInQueue);\n                currentOffset = Math.min(currentOffset, maxOffsetInQueue);\n                flatFile.initOffset(currentOffset);\n                log.warn(\"MessageDispatcher#dispatch init, topic={}, queueId={}, offset={}-{}, current={}\",\n                    topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset);\n                return CompletableFuture.completedFuture(true);\n            }\n\n            // If the previous commit fails, attempt to trigger a commit directly.\n            if (commitOffset < currentOffset) {\n                this.commitAsync(flatFile).whenComplete((result, throwable) -> {\n                    if (throwable != null) {\n                        log.error(\"MessageDispatcher#flatFile commitOffset less than currentOffset, commitAsync again failed. topic: {}, queueId: {} \", topic, queueId, throwable);\n                    }\n                });\n                return CompletableFuture.completedFuture(false);\n            }\n\n            if (failedGroupCommitMap.containsKey(flatFile)) {\n                GroupCommitContext failedCommit = failedGroupCommitMap.get(flatFile);\n                if (failedCommit.getEndOffset() <= commitOffset) {\n                    failedGroupCommitMap.remove(flatFile);\n                    constructIndexFile(flatFile.getTopicId(), failedCommit);\n                }\n            }\n\n            if (currentOffset < minOffsetInQueue) {\n                log.warn(\"MessageDispatcher#dispatch, current offset is too small, topic={}, queueId={}, offset={}-{}, current={}\",\n                    topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset);\n                flatFileStore.destroyFile(flatFile.getMessageQueue());\n                flatFileStore.computeIfAbsent(new MessageQueue(topic, brokerName, queueId));\n                return CompletableFuture.completedFuture(true);\n            }\n\n            if (currentOffset > maxOffsetInQueue) {\n                log.warn(\"MessageDispatcher#dispatch, current offset is too large, topic={}, queueId={}, offset={}-{}, current={}\",\n                    topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset);\n                return CompletableFuture.completedFuture(false);\n            }\n\n            long interval = TimeUnit.HOURS.toMillis(storeConfig.getCommitLogRollingInterval());\n            if (flatFile.rollingFile(interval)) {\n                log.info(\"MessageDispatcher#dispatch, rolling file, topic={}, queueId={}, offset={}-{}, current={}\",\n                    topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset);\n            }\n\n            if (currentOffset == maxOffsetInQueue) {\n                return CompletableFuture.completedFuture(false);\n            }\n\n            long bufferSize = 0L;\n            long groupCommitSize = storeConfig.getTieredStoreGroupCommitSize();\n            long groupCommitCount = storeConfig.getTieredStoreGroupCommitCount();\n            long targetOffset = Math.min(currentOffset + groupCommitCount, maxOffsetInQueue);\n\n            ConsumeQueueInterface consumeQueue = defaultStore.getConsumeQueue(topic, queueId);\n            CqUnit cqUnit = consumeQueue.get(currentOffset);\n            if (cqUnit == null) {\n                log.warn(\"MessageDispatcher#dispatch cq not found, topic={}, queueId={}, offset={}-{}, current={}, remain={}\",\n                    topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset, maxOffsetInQueue - currentOffset);\n                return CompletableFuture.completedFuture(false);\n            }\n\n            SelectMappedBufferResult message =\n                defaultStore.selectOneMessageByOffset(cqUnit.getPos(), cqUnit.getSize());\n            if (message == null) {\n                log.warn(\"MessageDispatcher#dispatch message not found, topic={}, queueId={}, offset={}-{}, current={}, remain={}\",\n                    topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset, maxOffsetInQueue - currentOffset);\n                return CompletableFuture.completedFuture(false);\n            }\n\n            boolean timeout = MessageFormatUtil.getStoreTimeStamp(message.getByteBuffer()) +\n                storeConfig.getTieredStoreGroupCommitTimeout() < System.currentTimeMillis();\n            boolean bufferFull = maxOffsetInQueue - currentOffset > storeConfig.getTieredStoreGroupCommitCount();\n\n            if (!timeout && !bufferFull && !force) {\n                log.debug(\"MessageDispatcher#dispatch hold, topic={}, queueId={}, offset={}-{}, current={}, remain={}\",\n                    topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset, maxOffsetInQueue - currentOffset);\n                message.release();\n                return CompletableFuture.completedFuture(false);\n            } else {\n                if (MessageFormatUtil.getStoreTimeStamp(message.getByteBuffer()) +\n                    TimeUnit.MINUTES.toMillis(5) < System.currentTimeMillis()) {\n                    log.warn(\"MessageDispatcher#dispatch behind too much, topic={}, queueId={}, offset={}-{}, current={}, remain={}\",\n                        topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset, maxOffsetInQueue - currentOffset);\n                } else {\n                    log.info(\"MessageDispatcher#dispatch success, topic={}, queueId={}, offset={}-{}, current={}, remain={}\",\n                        topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset, maxOffsetInQueue - currentOffset);\n                }\n                message.release();\n            }\n\n            long offset = currentOffset;\n            List<SelectMappedBufferResult> appendingBufferList = new ArrayList<>();\n            List<DispatchRequest> dispatchRequestList = new ArrayList<>();\n            for (; offset < targetOffset; offset++) {\n                cqUnit = consumeQueue.get(offset);\n                bufferSize += cqUnit.getSize();\n                if (bufferSize >= groupCommitSize) {\n                    break;\n                }\n                message = defaultStore.selectOneMessageByOffset(cqUnit.getPos(), cqUnit.getSize());\n                appendingBufferList.add(message);\n\n                ByteBuffer byteBuffer = message.getByteBuffer();\n                AppendResult result = flatFile.appendCommitLog(message);\n                if (!AppendResult.SUCCESS.equals(result)) {\n                    break;\n                }\n\n                long mappedCommitLogOffset = flatFile.getCommitLogMaxOffset() - byteBuffer.remaining();\n                Map<String, String> properties = MessageFormatUtil.getProperties(byteBuffer);\n\n                DispatchRequest dispatchRequest = new DispatchRequest(topic, queueId, mappedCommitLogOffset,\n                    cqUnit.getSize(), cqUnit.getTagsCode(), MessageFormatUtil.getStoreTimeStamp(byteBuffer),\n                    cqUnit.getQueueOffset(), properties.getOrDefault(MessageConst.PROPERTY_KEYS, \"\"),\n                    properties.getOrDefault(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, \"\"),\n                    0, 0, new HashMap<>());\n                dispatchRequest.setOffsetId(MessageFormatUtil.getOffsetId(byteBuffer));\n\n                result = flatFile.appendConsumeQueue(dispatchRequest);\n                if (!AppendResult.SUCCESS.equals(result)) {\n                    break;\n                } else {\n                    dispatchRequestList.add(dispatchRequest);\n                }\n            }\n\n            GroupCommitContext groupCommitContext = new GroupCommitContext();\n            groupCommitContext.setEndOffset(offset);\n            groupCommitContext.setBufferList(appendingBufferList);\n            groupCommitContext.setDispatchRequests(dispatchRequestList);\n\n            // If there are many messages waiting to be uploaded, call the upload logic immediately.\n            boolean repeat = timeout || maxOffsetInQueue - offset > storeConfig.getTieredStoreGroupCommitCount();\n\n            if (!dispatchRequestList.isEmpty()) {\n                Attributes attributes = TieredStoreMetricsManager.newAttributesBuilder()\n                    .put(TieredStoreMetricsConstant.LABEL_TOPIC, topic)\n                    .put(TieredStoreMetricsConstant.LABEL_QUEUE_ID, queueId)\n                    .put(TieredStoreMetricsConstant.LABEL_FILE_TYPE, FileSegmentType.COMMIT_LOG.name().toLowerCase())\n                    .build();\n                TieredStoreMetricsManager.messagesDispatchTotal.add(offset - currentOffset, attributes);\n\n                this.commitAsync(flatFile).whenComplete((success, throwable) -> {\n                        if (success) {\n                            constructIndexFile(flatFile.getTopicId(), groupCommitContext);\n                        }\n                        else {\n                            //next commit async,execute constructIndexFile.\n                            GroupCommitContext oldCommit = failedGroupCommitMap.put(flatFile, groupCommitContext);\n                            if (oldCommit != null) {\n                                log.warn(\"MessageDispatcher#commitAsync failed,flatFile old failed commit context not release, topic={}, queueId={}  \", topic, queueId);\n                                oldCommit.release();\n                            }\n                        }\n                        if (success && repeat) {\n                            storeExecutor.commonExecutor.submit(() -> dispatchWithSemaphore(flatFile));\n                        }\n                    }\n                );\n            }\n        } catch (ConsumeQueueException e) {\n            CompletableFuture<Boolean> future = new CompletableFuture<>();\n            future.completeExceptionally(e);\n            return future;\n        } finally {\n            flatFile.getFileLock().unlock();\n        }\n        return CompletableFuture.completedFuture(false);\n    }\n\n    public CompletableFuture<Boolean> commitAsync(FlatFileInterface flatFile) {\n        return flatFile.commitAsync();\n    }\n\n    public void constructIndexFile(long topicId, GroupCommitContext groupCommitContext) {\n        MessageStoreExecutor.getInstance().bufferCommitExecutor.submit(() -> {\n            if (storeConfig.isMessageIndexEnable()) {\n                try {\n                    groupCommitContext.getDispatchRequests().forEach(request -> constructIndexFile0(topicId, request));\n                }\n                catch (Throwable e) {\n                    log.error(\"constructIndexFile error {}\", topicId, e);\n                }\n            }\n            groupCommitContext.release();\n        });\n    }\n\n    /**\n     * Building indexes with offsetId is no longer supported because offsetId has changed in tiered storage\n     */\n    public void constructIndexFile0(long topicId, DispatchRequest request) {\n        Set<String> keySet = new HashSet<>();\n        if (StringUtils.isNotBlank(request.getUniqKey())) {\n            keySet.add(request.getUniqKey());\n        }\n        if (StringUtils.isNotBlank(request.getKeys())) {\n            keySet.addAll(Arrays.asList(request.getKeys().split(MessageConst.KEY_SEPARATOR)));\n        }\n        indexService.putKey(request.getTopic(), (int) topicId, request.getQueueId(), keySet,\n            request.getCommitLogOffset(), request.getMsgSize(), request.getStoreTimestamp());\n    }\n\n    public void releaseClosedPendingGroupCommit() {\n        Iterator<Map.Entry<FlatFileInterface, GroupCommitContext>> iterator = failedGroupCommitMap.entrySet().iterator();\n        while (iterator.hasNext()) {\n            Map.Entry<FlatFileInterface, GroupCommitContext> entry = iterator.next();\n            if (entry.getKey().isClosed()) {\n                entry.getValue().release();\n                iterator.remove();\n            }\n        }\n    }\n\n\n    @Override\n    public void run() {\n        log.info(\"{} service started\", this.getServiceName());\n        while (!this.isStopped()) {\n            try {\n                flatFileStore.deepCopyFlatFileToList().forEach(this::dispatchWithSemaphore);\n\n                releaseClosedPendingGroupCommit();\n\n                this.waitForRunning(Duration.ofSeconds(20).toMillis());\n            } catch (Throwable t) {\n                log.error(\"MessageStore dispatch error\", t);\n            }\n        }\n        log.info(\"{} service shutdown\", this.getServiceName());\n    }\n}\n"
  },
  {
    "path": "tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcher.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.tieredstore.core;\n\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.MessageFilter;\nimport org.apache.rocketmq.store.QueryMessageResult;\n\npublic interface MessageStoreFetcher {\n\n    /**\n     * Asynchronous get the store time of the earliest message in this store.\n     *\n     * @return timestamp of the earliest message in this store.\n     */\n    CompletableFuture<Long> getEarliestMessageTimeAsync(String topic, int queueId);\n\n    /**\n     * Asynchronous get the store time of the message specified.\n     *\n     * @param topic              Message topic.\n     * @param queueId            Queue ID.\n     * @param consumeQueueOffset Consume queue offset.\n     * @return store timestamp of the message.\n     */\n    CompletableFuture<Long> getMessageStoreTimeStampAsync(String topic, int queueId, long consumeQueueOffset);\n\n    /**\n     * Look up the physical offset of the message whose store timestamp is as specified.\n     *\n     * @param topic     Topic of the message.\n     * @param queueId   Queue ID.\n     * @param timestamp Timestamp to look up.\n     * @return physical offset which matches.\n     */\n    long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType type);\n\n    /**\n     * Asynchronous get message\n     *\n     * @param group         Consumer group that launches this query.\n     * @param topic         Topic to query.\n     * @param queueId       Queue ID to query.\n     * @param offset        Logical offset to start from.\n     * @param maxCount      Maximum count of messages to query.\n     * @param messageFilter Message filter used to screen desired messages.\n     * @return Matched messages.\n     */\n    CompletableFuture<GetMessageResult> getMessageAsync(\n        String group, String topic, int queueId, long offset, int maxCount, MessageFilter messageFilter);\n\n    /**\n     * Asynchronous query messages by given key.\n     *\n     * @param topic    Topic of the message.\n     * @param key      Message key.\n     * @param maxCount Maximum count of the messages possible.\n     * @param begin    Begin timestamp.\n     * @param end      End timestamp.\n     */\n    CompletableFuture<QueryMessageResult> queryMessageAsync(\n        String topic, String key, int maxCount, long begin, long end);\n}\n"
  },
  {
    "path": "tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcherImpl.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.tieredstore.core;\n\nimport com.github.benmanes.caffeine.cache.Cache;\nimport com.github.benmanes.caffeine.cache.Caffeine;\nimport com.github.benmanes.caffeine.cache.Scheduler;\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.TimeUnit;\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.store.GetMessageResult;\nimport org.apache.rocketmq.store.GetMessageStatus;\nimport org.apache.rocketmq.store.MessageFilter;\nimport org.apache.rocketmq.store.QueryMessageResult;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.tieredstore.MessageStoreConfig;\nimport org.apache.rocketmq.tieredstore.TieredMessageStore;\nimport org.apache.rocketmq.tieredstore.common.GetMessageResultExt;\nimport org.apache.rocketmq.tieredstore.common.SelectBufferResult;\nimport org.apache.rocketmq.tieredstore.exception.TieredStoreException;\nimport org.apache.rocketmq.tieredstore.file.FlatFileStore;\nimport org.apache.rocketmq.tieredstore.file.FlatMessageFile;\nimport org.apache.rocketmq.tieredstore.index.IndexItem;\nimport org.apache.rocketmq.tieredstore.index.IndexService;\nimport org.apache.rocketmq.tieredstore.metadata.MetadataStore;\nimport org.apache.rocketmq.tieredstore.metadata.entity.TopicMetadata;\nimport org.apache.rocketmq.tieredstore.util.MessageFormatUtil;\nimport org.apache.rocketmq.tieredstore.util.MessageStoreUtil;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\npublic class MessageStoreFetcherImpl implements MessageStoreFetcher {\n\n    private static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME);\n\n    protected static final String CACHE_KEY_FORMAT = \"%s@%d@%d\";\n    protected static final String FETCHER_GROUP_NAME = MixAll.CID_RMQ_SYS_PREFIX + \"FETCHER_TIMESTAMP\";\n\n    private final String brokerName;\n    private final MetadataStore metadataStore;\n    private final MessageStoreConfig storeConfig;\n    private final TieredMessageStore messageStore;\n    private final IndexService indexService;\n    private final FlatFileStore flatFileStore;\n    private final long memoryMaxSize;\n    private final Cache<String /* topic@queueId@offset */, SelectBufferResult> fetcherCache;\n\n    public MessageStoreFetcherImpl(TieredMessageStore messageStore) {\n        this(messageStore, messageStore.getStoreConfig(),\n            messageStore.getFlatFileStore(), messageStore.getIndexService());\n    }\n\n    public MessageStoreFetcherImpl(TieredMessageStore messageStore, MessageStoreConfig storeConfig,\n        FlatFileStore flatFileStore, IndexService indexService) {\n\n        this.storeConfig = storeConfig;\n        this.brokerName = storeConfig.getBrokerName();\n        this.flatFileStore = flatFileStore;\n        this.messageStore = messageStore;\n        this.indexService = indexService;\n        this.metadataStore = flatFileStore.getMetadataStore();\n        this.memoryMaxSize =\n            (long) (Runtime.getRuntime().maxMemory() * storeConfig.getReadAheadCacheSizeThresholdRate());\n        this.fetcherCache = this.initCache(storeConfig);\n        log.info(\"MessageStoreFetcher init success, brokerName={}\", storeConfig.getBrokerName());\n    }\n\n    private Cache<String, SelectBufferResult> initCache(MessageStoreConfig storeConfig) {\n\n        return Caffeine.newBuilder()\n            .scheduler(Scheduler.systemScheduler())\n            // Clients may repeatedly request messages at the same offset in tiered storage,\n            // causing the request queue to become full. Using expire after read or write policy\n            // to refresh the cache expiration time.\n            .expireAfterAccess(storeConfig.getReadAheadCacheExpireDuration(), TimeUnit.MILLISECONDS)\n            .maximumWeight(memoryMaxSize)\n            // Using the buffer size of messages to calculate memory usage\n            .weigher((String key, SelectBufferResult buffer) -> buffer.getSize())\n            .recordStats()\n            .build();\n    }\n\n    public Cache<String, SelectBufferResult> getFetcherCache() {\n        return fetcherCache;\n    }\n\n    protected void putMessageToCache(FlatMessageFile flatFile, long offset, SelectBufferResult result) {\n        MessageQueue mq = flatFile.getMessageQueue();\n        this.fetcherCache.put(String.format(CACHE_KEY_FORMAT, mq.getTopic(), mq.getQueueId(), offset), result);\n    }\n\n    protected SelectBufferResult getMessageFromCache(FlatMessageFile flatFile, long offset) {\n        MessageQueue mq = flatFile.getMessageQueue();\n        SelectBufferResult buffer = this.fetcherCache.getIfPresent(\n            String.format(CACHE_KEY_FORMAT, mq.getTopic(), mq.getQueueId(), offset));\n        // return duplicate buffer here\n        if (buffer == null) {\n            return null;\n        }\n        long count = buffer.getAccessCount().incrementAndGet();\n        if (count % 1000L == 0L) {\n            log.warn(\"MessageFetcher fetch same offset message too many times, \" +\n                \"topic={}, queueId={}, offset={}, count={}\", mq.getTopic(), mq.getQueueId(), offset, count);\n        }\n        return new SelectBufferResult(\n            buffer.getByteBuffer().asReadOnlyBuffer(), buffer.getStartOffset(), buffer.getSize(), buffer.getTagCode());\n    }\n\n    protected GetMessageResultExt getMessageFromCache(\n        FlatMessageFile flatFile, long offset, int maxCount, MessageFilter messageFilter) {\n        GetMessageResultExt result = new GetMessageResultExt();\n        int interval = storeConfig.getReadAheadMessageCountThreshold();\n        for (long current = offset, end = offset + interval; current < end; current++) {\n            SelectBufferResult buffer = getMessageFromCache(flatFile, current);\n            if (buffer == null) {\n                result.setNextBeginOffset(current);\n                break;\n            }\n            result.setNextBeginOffset(current + 1);\n            if (messageFilter != null) {\n                if (!messageFilter.isMatchedByConsumeQueue(buffer.getTagCode(), null)) {\n                    continue;\n                }\n                if (!messageFilter.isMatchedByCommitLog(buffer.getByteBuffer().slice(), null)) {\n                    continue;\n                }\n            }\n            SelectMappedBufferResult bufferResult = new SelectMappedBufferResult(\n                buffer.getStartOffset(), buffer.getByteBuffer(), buffer.getSize(), null);\n            result.addMessageExt(bufferResult, current, buffer.getTagCode());\n            if (result.getMessageCount() == maxCount) {\n                break;\n            }\n            long maxTransferBytes = messageStore.getMessageStoreConfig().getMaxTransferBytesOnMessageInMemory();\n            if (result.getBufferTotalSize() >= maxTransferBytes) {\n                break;\n            }\n        }\n        result.setStatus(result.getMessageCount() > 0 ?\n            GetMessageStatus.FOUND : GetMessageStatus.NO_MATCHED_MESSAGE);\n        result.setMinOffset(flatFile.getConsumeQueueMinOffset());\n        result.setMaxOffset(flatFile.getConsumeQueueCommitOffset());\n        return result;\n    }\n\n    protected CompletableFuture<Long> fetchMessageThenPutToCache(\n        FlatMessageFile flatFile, long queueOffset, int batchSize) {\n\n        MessageQueue mq = flatFile.getMessageQueue();\n        return this.getMessageFromTieredStoreAsync(flatFile, queueOffset, batchSize)\n            .thenApply(result -> {\n                if (result.getStatus() == GetMessageStatus.OFFSET_OVERFLOW_ONE ||\n                    result.getStatus() == GetMessageStatus.OFFSET_OVERFLOW_BADLY) {\n                    return -1L;\n                }\n                if (result.getStatus() != GetMessageStatus.FOUND) {\n                    log.warn(\"MessageFetcher prefetch message then put to cache failed, result={}, \" +\n                            \"topic={}, queue={}, queue offset={}, batch size={}\",\n                        result.getStatus(), mq.getTopic(), mq.getQueueId(), queueOffset, batchSize);\n                    return -1L;\n                }\n                List<Long> offsetList = result.getMessageQueueOffset();\n                List<Long> tagCodeList = result.getTagCodeList();\n                List<SelectMappedBufferResult> msgList = result.getMessageMapedList();\n                for (int i = 0; i < offsetList.size(); i++) {\n                    SelectMappedBufferResult msg = msgList.get(i);\n                    SelectBufferResult bufferResult = new SelectBufferResult(\n                        msg.getByteBuffer(), msg.getStartOffset(), msg.getSize(), tagCodeList.get(i));\n                    this.putMessageToCache(flatFile, queueOffset + i, bufferResult);\n                }\n                return offsetList.get(offsetList.size() - 1);\n            });\n    }\n\n    public CompletableFuture<GetMessageResult> getMessageFromCacheAsync(\n        FlatMessageFile flatFile, String group, long queueOffset, int maxCount, MessageFilter messageFilter) {\n\n        MessageQueue mq = flatFile.getMessageQueue();\n        GetMessageResultExt result = getMessageFromCache(flatFile, queueOffset, maxCount, messageFilter);\n\n        if (GetMessageStatus.FOUND.equals(result.getStatus())) {\n            log.debug(\"MessageFetcher cache hit, group={}, topic={}, queueId={}, offset={}, maxCount={}, resultSize={}, lag={}\",\n                group, mq.getTopic(), mq.getQueueId(), queueOffset, maxCount,\n                result.getMessageCount(), result.getMaxOffset() - result.getNextBeginOffset());\n            return CompletableFuture.completedFuture(result);\n        }\n\n        // If cache miss, pull messages immediately\n        log.debug(\"MessageFetcher cache miss, group={}, topic={}, queueId={}, offset={}, maxCount={}, lag={}\",\n            group, mq.getTopic(), mq.getQueueId(), queueOffset, maxCount, result.getMaxOffset() - result.getNextBeginOffset());\n\n        // To optimize the performance of pop consumption\n        // Pop revive will cause a large number of random reads,\n        // so the amount of pre-fetch message num needs to be reduced.\n        int fetchSize = maxCount == 1 ? 32 : storeConfig.getReadAheadMessageCountThreshold();\n\n        // In the current design, when the min offset cache expires,\n        // this method may trigger an RPC call, causing buffer fetch thread starvation\n        return fetchMessageThenPutToCache(flatFile, queueOffset, fetchSize)\n            .thenApplyAsync(maxOffset -> getMessageFromCache(flatFile, queueOffset, maxCount, messageFilter),\n                messageStore.getStoreExecutor().commonExecutor);\n    }\n\n    public CompletableFuture<GetMessageResultExt> getMessageFromTieredStoreAsync(\n        FlatMessageFile flatFile, long queueOffset, int batchSize) {\n\n        GetMessageResultExt result = new GetMessageResultExt();\n        result.setMinOffset(flatFile.getConsumeQueueMinOffset());\n        result.setMaxOffset(flatFile.getConsumeQueueCommitOffset());\n\n        if (queueOffset < result.getMinOffset()) {\n            result.setStatus(GetMessageStatus.OFFSET_TOO_SMALL);\n            result.setNextBeginOffset(result.getMinOffset());\n            return CompletableFuture.completedFuture(result);\n        } else if (queueOffset == result.getMaxOffset()) {\n            result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_ONE);\n            result.setNextBeginOffset(queueOffset);\n            return CompletableFuture.completedFuture(result);\n        } else if (queueOffset > result.getMaxOffset()) {\n            result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_BADLY);\n            result.setNextBeginOffset(result.getMaxOffset());\n            return CompletableFuture.completedFuture(result);\n        }\n\n        if (queueOffset < result.getMaxOffset()) {\n            batchSize = Math.min(batchSize, (int) Math.min(\n                result.getMaxOffset() - queueOffset, storeConfig.getReadAheadMessageCountThreshold()));\n        }\n\n        CompletableFuture<ByteBuffer> readConsumeQueueFuture;\n        try {\n            readConsumeQueueFuture = flatFile.getConsumeQueueAsync(queueOffset, batchSize);\n        } catch (TieredStoreException e) {\n            switch (e.getErrorCode()) {\n                case ILLEGAL_PARAM:\n                case ILLEGAL_OFFSET:\n                default:\n                    result.setStatus(GetMessageStatus.OFFSET_FOUND_NULL);\n                    result.setNextBeginOffset(queueOffset);\n                    return CompletableFuture.completedFuture(result);\n            }\n        }\n\n        int finalBatchSize = batchSize;\n        CompletableFuture<ByteBuffer> readCommitLogFuture = readConsumeQueueFuture.thenCompose(cqBuffer -> {\n\n            long firstCommitLogOffset = MessageFormatUtil.getCommitLogOffsetFromItem(cqBuffer);\n            cqBuffer.position(cqBuffer.remaining() - MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE);\n            long lastCommitLogOffset = MessageFormatUtil.getCommitLogOffsetFromItem(cqBuffer);\n            if (lastCommitLogOffset < firstCommitLogOffset) {\n                log.error(\"MessageFetcher#getMessageFromTieredStoreAsync, last offset is smaller than first offset, \" +\n                        \"topic={} queueId={}, offset={}, firstOffset={}, lastOffset={}\",\n                    flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId(), queueOffset,\n                    firstCommitLogOffset, lastCommitLogOffset);\n                return CompletableFuture.completedFuture(ByteBuffer.allocate(0));\n            }\n\n            // Get at least one message\n            // Reducing the length limit of cq to prevent OOM\n            long length = lastCommitLogOffset - firstCommitLogOffset + MessageFormatUtil.getSizeFromItem(cqBuffer);\n            while (cqBuffer.limit() > MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE &&\n                length > storeConfig.getReadAheadMessageSizeThreshold()) {\n                cqBuffer.limit(cqBuffer.position());\n                cqBuffer.position(cqBuffer.limit() - MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE);\n                length = MessageFormatUtil.getCommitLogOffsetFromItem(cqBuffer)\n                    - firstCommitLogOffset + MessageFormatUtil.getSizeFromItem(cqBuffer);\n            }\n            int messageCount = cqBuffer.position() / MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE + 1;\n\n            log.info(\"MessageFetcher#getMessageFromTieredStoreAsync, \" +\n                    \"topic={}, queueId={}, broker offset={}-{}, offset={}, expect={}, actually={}, lag={}\",\n                flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId(),\n                result.getMinOffset(), result.getMaxOffset(), queueOffset, finalBatchSize,\n                messageCount, result.getMaxOffset() - queueOffset);\n\n            return flatFile.getCommitLogAsync(firstCommitLogOffset, (int) length);\n        });\n\n        return readConsumeQueueFuture.thenCombine(readCommitLogFuture, (cqBuffer, msgBuffer) -> {\n            List<SelectBufferResult> bufferList = MessageFormatUtil.splitMessageBuffer(cqBuffer, msgBuffer);\n            int requestSize = cqBuffer.remaining() / MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE;\n\n            // not use buffer list size to calculate next offset to prevent split error\n            if (bufferList.isEmpty()) {\n                result.setStatus(GetMessageStatus.NO_MATCHED_MESSAGE);\n                result.setNextBeginOffset(queueOffset + requestSize);\n            } else {\n                result.setStatus(GetMessageStatus.FOUND);\n                result.setNextBeginOffset(queueOffset + requestSize);\n\n                for (SelectBufferResult bufferResult : bufferList) {\n                    ByteBuffer slice = bufferResult.getByteBuffer().slice();\n                    slice.limit(bufferResult.getSize());\n                    SelectMappedBufferResult msg = new SelectMappedBufferResult(bufferResult.getStartOffset(),\n                        bufferResult.getByteBuffer(), bufferResult.getSize(), null);\n                    result.addMessageExt(msg, MessageFormatUtil.getQueueOffset(slice), bufferResult.getTagCode());\n                }\n            }\n            return result;\n        }).exceptionally(e -> {\n            MessageQueue mq = flatFile.getMessageQueue();\n            log.warn(\"MessageFetcher#getMessageFromTieredStoreAsync failed, \" +\n                \"topic={} queueId={}, offset={}, batchSize={}\", mq.getTopic(), mq.getQueueId(), queueOffset, finalBatchSize, e);\n            result.setStatus(GetMessageStatus.OFFSET_FOUND_NULL);\n            result.setNextBeginOffset(queueOffset);\n            return result;\n        });\n    }\n\n    @Override\n    public CompletableFuture<GetMessageResult> getMessageAsync(\n        String group, String topic, int queueId, long queueOffset, int maxCount, final MessageFilter messageFilter) {\n\n        GetMessageResult result = new GetMessageResult();\n        FlatMessageFile flatFile = flatFileStore.getFlatFile(new MessageQueue(topic, brokerName, queueId));\n\n        if (flatFile == null) {\n            result.setNextBeginOffset(queueOffset);\n            result.setStatus(GetMessageStatus.NO_MATCHED_LOGIC_QUEUE);\n            return CompletableFuture.completedFuture(result);\n        }\n\n        // Max queue offset means next message put position\n        result.setMinOffset(flatFile.getConsumeQueueMinOffset());\n        result.setMaxOffset(flatFile.getConsumeQueueCommitOffset());\n\n        // Fill result according file offset.\n        // Offset range  | Result           | Fix to\n        // (-oo, 0]      | no message       | current offset\n        // (0, min)      | too small        | min offset\n        // [min, max)    | correct          |\n        // [max, max]    | overflow one     | max offset\n        // (max, +oo)    | overflow badly   | max offset\n\n        if (result.getMaxOffset() <= 0) {\n            result.setStatus(GetMessageStatus.NO_MESSAGE_IN_QUEUE);\n            result.setNextBeginOffset(queueOffset);\n            return CompletableFuture.completedFuture(result);\n        } else if (queueOffset < result.getMinOffset()) {\n            result.setStatus(GetMessageStatus.OFFSET_TOO_SMALL);\n            result.setNextBeginOffset(result.getMinOffset());\n            return CompletableFuture.completedFuture(result);\n        } else if (queueOffset == result.getMaxOffset()) {\n            result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_ONE);\n            result.setNextBeginOffset(result.getMaxOffset());\n            return CompletableFuture.completedFuture(result);\n        } else if (queueOffset > result.getMaxOffset()) {\n            result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_BADLY);\n            result.setNextBeginOffset(result.getMaxOffset());\n            return CompletableFuture.completedFuture(result);\n        }\n\n        boolean cacheBusy = fetcherCache.estimatedSize() > memoryMaxSize * 0.8;\n        if (storeConfig.isReadAheadCacheEnable() && !cacheBusy) {\n            return getMessageFromCacheAsync(flatFile, group, queueOffset, maxCount, messageFilter);\n        } else {\n            return getMessageFromTieredStoreAsync(flatFile, queueOffset, maxCount)\n                .thenApply(messageResultExt -> messageResultExt.doFilterMessage(messageFilter));\n        }\n    }\n\n    @Override\n    public CompletableFuture<Long> getEarliestMessageTimeAsync(String topic, int queueId) {\n        FlatMessageFile flatFile = flatFileStore.getFlatFile(new MessageQueue(topic, brokerName, queueId));\n        return CompletableFuture.completedFuture(flatFile == null ? -1L : flatFile.getMinStoreTimestamp());\n    }\n\n    @Override\n    public CompletableFuture<Long> getMessageStoreTimeStampAsync(String topic, int queueId, long queueOffset) {\n        FlatMessageFile flatFile = flatFileStore.getFlatFile(new MessageQueue(topic, brokerName, queueId));\n        if (flatFile == null) {\n            return CompletableFuture.completedFuture(-1L);\n        }\n\n        // The Metrics thread frequently retrieves the storage timestamp of the latest message;\n        // as an alternative, return the queue's saved timestamp here.\n        if (queueOffset + 1L == flatFile.getConsumeQueueCommitOffset()) {\n            long timestamp = flatFile.getMaxStoreTimestamp();\n            return CompletableFuture.completedFuture(timestamp == Long.MAX_VALUE ? -1L : timestamp);\n        }\n\n        CompletableFuture<Long> future = new CompletableFuture<>();\n        try {\n            this.getMessageAsync(FETCHER_GROUP_NAME, topic, queueId, queueOffset, 1, null)\n                .whenComplete((result, e) -> {\n                    if (e != null) {\n                        log.error(\"MessageStoreFetcherImpl#getMessageStoreTimeStampAsync: \" +\n                            \"Get or decode message failed, topic={}, queue={}, offset={}\", topic, queueId, queueOffset, e);\n                        future.completeExceptionally(e);\n                        return;\n                    }\n                    if (result != null && result.getMessageBufferList() != null\n                        && !result.getMessageBufferList().isEmpty()) {\n                        long timestamp = MessageFormatUtil.getStoreTimeStamp(result.getMessageBufferList().get(0));\n                        log.info(\"MessageStoreFetcherImpl#getMessageStoreTimeStampAsync: \" +\n                            \"topic={}, queue={}, offset={}, timestamp={}\", topic, queueId, queueOffset, timestamp);\n                        future.complete(timestamp);\n                    } else {\n                        future.complete(-1L);\n                    }\n                });\n        } catch (Throwable t) {\n            future.completeExceptionally(t);\n        }\n        return future;\n    }\n\n    @Override\n    public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType type) {\n        FlatMessageFile flatFile = flatFileStore.getFlatFile(new MessageQueue(topic, brokerName, queueId));\n        if (flatFile == null) {\n            return -1L;\n        }\n        return flatFile.getQueueOffsetByTimeAsync(timestamp, type).join();\n    }\n\n    @Override\n    public CompletableFuture<QueryMessageResult> queryMessageAsync(\n        String topic, String key, int maxCount, long begin, long end) {\n\n        long topicId;\n        try {\n            TopicMetadata topicMetadata = metadataStore.getTopic(topic);\n            if (topicMetadata == null) {\n                log.info(\"MessageFetcher#queryMessageAsync, topic metadata not found, topic={}\", topic);\n                return CompletableFuture.completedFuture(new QueryMessageResult());\n            }\n            topicId = topicMetadata.getTopicId();\n        } catch (Exception e) {\n            log.error(\"MessageFetcher#queryMessageAsync, get topic id failed, topic={}\", topic, e);\n            return CompletableFuture.completedFuture(new QueryMessageResult());\n        }\n\n        CompletableFuture<List<IndexItem>> future = indexService.queryAsync(topic, key, maxCount, begin, end);\n\n        return future.thenCompose(indexItemList -> {\n            List<CompletableFuture<SelectMappedBufferResult>> futureList = new ArrayList<>(maxCount);\n            for (IndexItem indexItem : indexItemList) {\n                if (topicId != indexItem.getTopicId()) {\n                    continue;\n                }\n                FlatMessageFile flatFile =\n                    flatFileStore.getFlatFile(new MessageQueue(topic, brokerName, indexItem.getQueueId()));\n                if (flatFile == null) {\n                    continue;\n                }\n                if (indexItem.getOffset() < flatFile.getCommitLogMinOffset() ||\n                    indexItem.getOffset() > flatFile.getCommitLogMaxOffset()) {\n                    continue;\n                }\n                CompletableFuture<SelectMappedBufferResult> getMessageFuture = flatFile\n                    .getCommitLogAsync(indexItem.getOffset(), indexItem.getSize())\n                    .thenApply(messageBuffer -> new SelectMappedBufferResult(\n                        indexItem.getOffset(), messageBuffer, indexItem.getSize(), null));\n                futureList.add(getMessageFuture);\n                if (futureList.size() >= maxCount) {\n                    break;\n                }\n            }\n            return CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).thenApply(v -> {\n                QueryMessageResult result = new QueryMessageResult();\n                futureList.forEach(f -> f.thenAccept(result::addMessage));\n                return result;\n            });\n        }).whenComplete((result, throwable) -> {\n            if (result != null) {\n                log.info(\"MessageFetcher#queryMessageAsync, \" +\n                        \"query result={}, topic={}, topicId={}, key={}, maxCount={}, timestamp={}-{}\",\n                    result.getMessageBufferList().size(), topic, topicId, key, maxCount, begin, end);\n            }\n        });\n    }\n}\n"
  },
  {
    "path": "tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreFilter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.tieredstore.core;\n\npublic interface MessageStoreFilter {\n\n    boolean filterTopic(String topicName);\n\n    void addTopicToBlackList(String topicName);\n}\n"
  },
  {
    "path": "tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreTopicFilter.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.tieredstore.core;\n\nimport java.util.HashSet;\nimport java.util.Set;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.MixAll;\nimport org.apache.rocketmq.common.PopAckConstants;\nimport org.apache.rocketmq.common.topic.TopicValidator;\nimport org.apache.rocketmq.tieredstore.MessageStoreConfig;\n\npublic class MessageStoreTopicFilter implements MessageStoreFilter {\n\n    private final Set<String> topicBlackSet;\n\n    public MessageStoreTopicFilter(MessageStoreConfig storeConfig) {\n        this.topicBlackSet = new HashSet<>();\n        this.topicBlackSet.add(storeConfig.getBrokerClusterName());\n        this.topicBlackSet.add(storeConfig.getBrokerName());\n    }\n\n    @Override\n    public boolean filterTopic(String topicName) {\n        if (StringUtils.isBlank(topicName)) {\n            return true;\n        }\n        return TopicValidator.isSystemTopic(topicName) ||\n            PopAckConstants.isStartWithRevivePrefix(topicName) ||\n            this.topicBlackSet.contains(topicName) ||\n            MixAll.isLmq(topicName);\n    }\n\n    @Override\n    public void addTopicToBlackList(String topicName) {\n        this.topicBlackSet.add(topicName);\n    }\n}\n"
  },
  {
    "path": "tieredstore/src/main/java/org/apache/rocketmq/tieredstore/exception/TieredStoreErrorCode.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.tieredstore.exception;\n\npublic enum TieredStoreErrorCode {\n\n    /**\n     * Error code for an invalid offset.\n     */\n    ILLEGAL_OFFSET,\n\n    /**\n     * Error code for an invalid parameter.\n     */\n    ILLEGAL_PARAM,\n\n    /**\n     * Error code for an incorrect download length.\n     */\n    DOWNLOAD_LENGTH_NOT_CORRECT,\n\n    /**\n     * Error code for no new data found in the storage system.\n     */\n    NO_NEW_DATA,\n\n    /**\n     * Error code for a storage provider error.\n     */\n    STORAGE_PROVIDER_ERROR,\n\n    /**\n     * Error code for an input/output error.\n     */\n    IO_ERROR,\n\n    /**\n     * Segment has been sealed\n     */\n    SEGMENT_SEALED,\n\n    /**\n     * Error code for an unknown error.\n     */\n    UNKNOWN\n}"
  },
  {
    "path": "tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileInterface.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.tieredstore.file;\n\nimport java.nio.ByteBuffer;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.locks.Lock;\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.tieredstore.common.AppendResult;\n\npublic interface FlatFileInterface {\n\n    long getTopicId();\n\n    Lock getFileLock();\n\n    MessageQueue getMessageQueue();\n\n    boolean isFlatFileInit();\n\n    void initOffset(long offset);\n\n    boolean rollingFile(long interval);\n\n    /**\n     * Appends a message to the commit log file\n     *\n     * @param message thByteBuffere message to append\n     * @return append result\n     */\n    AppendResult appendCommitLog(ByteBuffer message);\n\n    AppendResult appendCommitLog(SelectMappedBufferResult message);\n\n    /**\n     * Append message to consume queue file, but does not commit it immediately\n     *\n     * @param request the dispatch request\n     * @return append result\n     */\n    AppendResult appendConsumeQueue(DispatchRequest request);\n\n    void release();\n\n    long getMinStoreTimestamp();\n\n    long getMaxStoreTimestamp();\n\n    long getFirstMessageOffset();\n\n    long getCommitLogMinOffset();\n\n    long getCommitLogMaxOffset();\n\n    long getCommitLogCommitOffset();\n\n    long getConsumeQueueMinOffset();\n\n    long getConsumeQueueMaxOffset();\n\n    long getConsumeQueueCommitOffset();\n\n    /**\n     * Persist commit log file and consume queue file\n     */\n    CompletableFuture<Boolean> commitAsync();\n\n    /**\n     * Asynchronously retrieves the message at the specified consume queue offset\n     *\n     * @param consumeQueueOffset consume queue offset.\n     * @return the message inner object serialized content\n     */\n    CompletableFuture<ByteBuffer> getMessageAsync(long consumeQueueOffset);\n\n    /**\n     * Get message from commitLog file at specified offset and length\n     *\n     * @param offset the offset\n     * @param length the length\n     * @return the message inner object serialized content\n     */\n    CompletableFuture<ByteBuffer> getCommitLogAsync(long offset, int length);\n\n    /**\n     * Asynchronously retrieves the consume queue message at the specified queue offset\n     *\n     * @param consumeQueueOffset consume queue offset.\n     * @return the consumer queue unit serialized content\n     */\n    CompletableFuture<ByteBuffer> getConsumeQueueAsync(long consumeQueueOffset);\n\n    /**\n     * Asynchronously reads the message body from the consume queue file at the specified offset and count\n     *\n     * @param consumeQueueOffset the message offset\n     * @param count              the number of messages to read\n     * @return the consumer queue unit serialized content\n     */\n    CompletableFuture<ByteBuffer> getConsumeQueueAsync(long consumeQueueOffset, int count);\n\n    /**\n     * Gets the start offset in the consume queue based on the timestamp and boundary type.\n     * The consume queues consist of ordered units, and their storage times are non-decreasing\n     * sequence. If the specified message exists, it returns the offset of either the first\n     * or last message, depending on the boundary type. If the specified message does not exist,\n     * it returns the offset of the next message as the pull offset. For example:\n     * ------------------------------------------------------------\n     *   store time   : 40, 50, 50, 50, 60, 60, 70\n     *   queue offset : 10, 11, 12, 13, 14, 15, 16\n     * ------------------------------------------------------------\n     *   query timestamp | boundary | result (reason)\n     *         35        |    -     |   10 (minimum offset)\n     *         45        |    -     |   11 (next offset)\n     *         50        |   lower  |   11\n     *         50        |   upper  |   13\n     *         60        |    -     |   14 (default to lower)\n     *         75        |    -     |   17 (maximum offset + 1)\n     * ------------------------------------------------------------\n     * @param timestamp    The search time\n     * @param boundaryType 'lower' or 'upper' to determine the boundary\n     * @return Returns the offset of the message in the consume queue\n     */\n    CompletableFuture<Long> getQueueOffsetByTimeAsync(long timestamp, BoundaryType boundaryType);\n\n    boolean isClosed();\n\n    /**\n     * Shutdown process\n     */\n    void shutdown();\n\n    /**\n     * Destroys expired files\n     */\n    void destroyExpiredFile(long timestamp);\n\n    /**\n     * Delete file\n     */\n    void destroy();\n}\n"
  },
  {
    "path": "tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileStore.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.tieredstore.file;\n\nimport com.google.common.base.Stopwatch;\nimport com.google.common.collect.ImmutableList;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.Semaphore;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.rocketmq.common.constant.LoggerName;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.tieredstore.MessageStoreConfig;\nimport org.apache.rocketmq.tieredstore.MessageStoreExecutor;\nimport org.apache.rocketmq.tieredstore.metadata.MetadataStore;\nimport org.apache.rocketmq.tieredstore.metadata.entity.TopicMetadata;\nimport org.apache.rocketmq.tieredstore.util.MessageStoreUtil;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\npublic class FlatFileStore {\n\n    private static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME);\n\n    private final MetadataStore metadataStore;\n    private final MessageStoreConfig storeConfig;\n    private final MessageStoreExecutor executor;\n    private final FlatFileFactory flatFileFactory;\n    private final ConcurrentMap<MessageQueue, FlatMessageFile> flatFileConcurrentMap;\n\n    public FlatFileStore(MessageStoreConfig storeConfig, MetadataStore metadataStore, MessageStoreExecutor executor) {\n        this.storeConfig = storeConfig;\n        this.metadataStore = metadataStore;\n        this.executor = executor;\n        this.flatFileFactory = new FlatFileFactory(metadataStore, storeConfig, executor);\n        this.flatFileConcurrentMap = new ConcurrentHashMap<>();\n    }\n\n    public boolean load() {\n        Stopwatch stopwatch = Stopwatch.createStarted();\n        try {\n            this.flatFileConcurrentMap.clear();\n            this.recover();\n            log.info(\"FlatFileStore recover finished, total cost={}ms\", stopwatch.elapsed(TimeUnit.MILLISECONDS));\n        } catch (Exception e) {\n            long costTime = stopwatch.elapsed(TimeUnit.MILLISECONDS);\n            log.info(\"FlatFileStore recover error, total cost={}ms\", costTime);\n            LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME)\n                .error(\"FlatFileStore recover error, total cost={}ms\", costTime, e);\n            return false;\n        }\n        return true;\n    }\n\n    public void recover() {\n        Semaphore semaphore = new Semaphore(storeConfig.getTieredStoreMaxPendingLimit() / 4);\n        List<CompletableFuture<Void>> futures = new ArrayList<>();\n        metadataStore.iterateTopic(topicMetadata -> {\n            semaphore.acquireUninterruptibly();\n            futures.add(this.recoverAsync(topicMetadata)\n                .whenComplete((unused, throwable) -> {\n                    if (throwable != null) {\n                        log.error(\"FlatFileStore recover file error, topic={}\", topicMetadata.getTopic(), throwable);\n                    }\n                    semaphore.release();\n                }));\n        });\n        CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();\n    }\n\n    public CompletableFuture<Void> recoverAsync(TopicMetadata topicMetadata) {\n        return CompletableFuture.runAsync(() -> {\n            Stopwatch stopwatch = Stopwatch.createStarted();\n            AtomicLong queueCount = new AtomicLong();\n            metadataStore.iterateQueue(topicMetadata.getTopic(), queueMetadata -> {\n                FlatMessageFile flatFile = this.computeIfAbsent(new MessageQueue(\n                    topicMetadata.getTopic(), storeConfig.getBrokerName(), queueMetadata.getQueue().getQueueId()));\n                queueCount.incrementAndGet();\n                log.debug(\"FlatFileStore recover file, topicId={}, topic={}, queueId={}, cost={}ms\",\n                    flatFile.getTopicId(), flatFile.getMessageQueue().getTopic(),\n                    flatFile.getMessageQueue().getQueueId(), stopwatch.elapsed(TimeUnit.MILLISECONDS));\n            });\n            log.info(\"FlatFileStore recover file, topic={}, total={}, cost={}ms\",\n                topicMetadata.getTopic(), queueCount.get(), stopwatch.elapsed(TimeUnit.MILLISECONDS));\n        }, executor.bufferCommitExecutor);\n    }\n\n    public void scheduleDeleteExpireFile() {\n        if (!storeConfig.isTieredStoreDeleteFileEnable()) {\n            return;\n        }\n        Stopwatch stopwatch = Stopwatch.createStarted();\n        ImmutableList<FlatMessageFile> fileList = this.deepCopyFlatFileToList();\n        for (FlatMessageFile flatFile : fileList) {\n            flatFile.getFileLock().lock();\n            try {\n                flatFile.destroyExpiredFile(System.currentTimeMillis() -\n                    TimeUnit.HOURS.toMillis(flatFile.getFileReservedHours()));\n            } catch (Exception e) {\n                log.error(\"FlatFileStore delete expire file error\", e);\n            } finally {\n                flatFile.getFileLock().unlock();\n            }\n        }\n        log.info(\"FlatFileStore schedule delete expired file, count={}, cost={}ms\",\n            fileList.size(), stopwatch.elapsed(TimeUnit.MILLISECONDS));\n    }\n\n    public MetadataStore getMetadataStore() {\n        return metadataStore;\n    }\n\n    public MessageStoreConfig getStoreConfig() {\n        return storeConfig;\n    }\n\n    public FlatFileFactory getFlatFileFactory() {\n        return flatFileFactory;\n    }\n\n    public FlatMessageFile computeIfAbsent(MessageQueue messageQueue) {\n        return flatFileConcurrentMap.computeIfAbsent(messageQueue,\n            mq -> new FlatMessageFile(flatFileFactory, mq.getTopic(), mq.getQueueId()));\n    }\n\n    public FlatMessageFile getFlatFile(MessageQueue messageQueue) {\n        return flatFileConcurrentMap.get(messageQueue);\n    }\n\n    public ImmutableList<FlatMessageFile> deepCopyFlatFileToList() {\n        return ImmutableList.copyOf(flatFileConcurrentMap.values());\n    }\n\n    public void shutdown() {\n        flatFileConcurrentMap.values().forEach(FlatMessageFile::shutdown);\n    }\n\n    public void destroyFile(MessageQueue mq) {\n        if (mq == null) {\n            return;\n        }\n\n        FlatMessageFile flatFile = flatFileConcurrentMap.remove(mq);\n        if (flatFile != null) {\n            flatFile.shutdown();\n            flatFile.destroy();\n        }\n        log.info(\"FlatFileStore destroy file, topic={}, queueId={}\", mq.getTopic(), mq.getQueueId());\n    }\n\n    public void destroy() {\n        this.shutdown();\n        flatFileConcurrentMap.values().forEach(FlatMessageFile::destroy);\n        flatFileConcurrentMap.clear();\n    }\n}\n"
  },
  {
    "path": "tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatMessageFile.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.tieredstore.file;\n\nimport com.alibaba.fastjson2.JSON;\nimport com.google.common.annotations.VisibleForTesting;\nimport java.nio.ByteBuffer;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.Semaphore;\nimport java.util.concurrent.locks.Lock;\nimport java.util.concurrent.locks.ReentrantLock;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.BoundaryType;\nimport org.apache.rocketmq.common.message.MessageQueue;\nimport org.apache.rocketmq.store.DispatchRequest;\nimport org.apache.rocketmq.store.SelectMappedBufferResult;\nimport org.apache.rocketmq.tieredstore.MessageStoreConfig;\nimport org.apache.rocketmq.tieredstore.common.AppendResult;\nimport org.apache.rocketmq.tieredstore.metadata.MetadataStore;\nimport org.apache.rocketmq.tieredstore.metadata.entity.QueueMetadata;\nimport org.apache.rocketmq.tieredstore.metadata.entity.TopicMetadata;\nimport org.apache.rocketmq.tieredstore.provider.FileSegment;\nimport org.apache.rocketmq.tieredstore.util.MessageFormatUtil;\nimport org.apache.rocketmq.tieredstore.util.MessageStoreUtil;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\npublic class FlatMessageFile implements FlatFileInterface {\n\n    protected static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME);\n    protected volatile boolean closed = false;\n\n    protected TopicMetadata topicMetadata;\n    protected QueueMetadata queueMetadata;\n\n    protected final String filePath;\n    protected final ReentrantLock fileLock;\n    protected final Semaphore commitLock = new Semaphore(1);\n    protected final MessageStoreConfig storeConfig;\n    protected final MetadataStore metadataStore;\n    protected final FlatCommitLogFile commitLog;\n    protected final FlatConsumeQueueFile consumeQueue;\n\n    protected final ConcurrentMap<String, CompletableFuture<?>> inFlightRequestMap;\n\n    public FlatMessageFile(FlatFileFactory fileFactory, String topic, int queueId) {\n        this(fileFactory, MessageStoreUtil.toFilePath(\n            new MessageQueue(topic, fileFactory.getStoreConfig().getBrokerName(), queueId)));\n        this.topicMetadata = this.recoverTopicMetadata(topic);\n        this.queueMetadata = this.recoverQueueMetadata(topic, queueId);\n    }\n\n    public FlatMessageFile(FlatFileFactory fileFactory, String filePath) {\n        this.filePath = filePath;\n        this.fileLock = new ReentrantLock(false);\n        this.storeConfig = fileFactory.getStoreConfig();\n        this.metadataStore = fileFactory.getMetadataStore();\n        this.commitLog = fileFactory.createFlatFileForCommitLog(filePath);\n        this.consumeQueue = fileFactory.createFlatFileForConsumeQueue(filePath);\n        this.inFlightRequestMap = new ConcurrentHashMap<>();\n    }\n\n    @Override\n    public long getTopicId() {\n        return topicMetadata.getTopicId();\n    }\n\n    @Override\n    public MessageQueue getMessageQueue() {\n        return queueMetadata != null ? queueMetadata.getQueue() : null;\n    }\n\n    @Override\n    public boolean isFlatFileInit() {\n        return !this.consumeQueue.fileSegmentTable.isEmpty();\n    }\n\n    public TopicMetadata recoverTopicMetadata(String topic) {\n        TopicMetadata topicMetadata = this.metadataStore.getTopic(topic);\n        if (topicMetadata == null) {\n            topicMetadata = this.metadataStore.addTopic(topic, -1L);\n        }\n        return topicMetadata;\n    }\n\n    public QueueMetadata recoverQueueMetadata(String topic, int queueId) {\n        MessageQueue mq = new MessageQueue(topic, storeConfig.getBrokerName(), queueId);\n        QueueMetadata queueMetadata = this.metadataStore.getQueue(mq);\n        if (queueMetadata == null) {\n            queueMetadata = this.metadataStore.addQueue(mq, -1L);\n        }\n        return queueMetadata;\n    }\n\n    public void flushMetadata() {\n        if (queueMetadata != null) {\n            queueMetadata.setMinOffset(this.getConsumeQueueMinOffset());\n            queueMetadata.setMaxOffset(this.getConsumeQueueCommitOffset());\n            queueMetadata.setUpdateTimestamp(System.currentTimeMillis());\n            metadataStore.updateQueue(queueMetadata);\n        }\n    }\n\n    @Override\n    public Lock getFileLock() {\n        return this.fileLock;\n    }\n\n    @VisibleForTesting\n    public Semaphore getCommitLock() {\n        return commitLock;\n    }\n\n    @Override\n    public boolean rollingFile(long interval) {\n        return this.commitLog.tryRollingFile(interval);\n    }\n\n    @Override\n    public void initOffset(long offset) {\n        fileLock.lock();\n        try {\n            this.commitLog.initOffset(0L);\n            this.consumeQueue.initOffset(offset * MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE);\n        } finally {\n            fileLock.unlock();\n        }\n    }\n\n    @Override\n    public AppendResult appendCommitLog(ByteBuffer message) {\n        if (closed) {\n            return AppendResult.FILE_CLOSED;\n        }\n        return commitLog.append(message, MessageFormatUtil.getStoreTimeStamp(message));\n    }\n\n    @Override\n    public AppendResult appendCommitLog(SelectMappedBufferResult message) {\n        if (closed) {\n            return AppendResult.FILE_CLOSED;\n        }\n        return this.appendCommitLog(message.getByteBuffer());\n    }\n\n    @Override\n    public AppendResult appendConsumeQueue(DispatchRequest request) {\n        if (closed) {\n            return AppendResult.FILE_CLOSED;\n        }\n\n        ByteBuffer buffer = ByteBuffer.allocate(MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE);\n        buffer.putLong(request.getCommitLogOffset());\n        buffer.putInt(request.getMsgSize());\n        buffer.putLong(request.getTagsCode());\n        buffer.flip();\n\n        return consumeQueue.append(buffer, request.getStoreTimestamp());\n    }\n\n    @Override\n    public void release() {\n    }\n\n    @Override\n    public long getMinStoreTimestamp() {\n        long minStoreTime = -1L;\n        if (Long.MAX_VALUE != commitLog.getMinTimestamp()) {\n            minStoreTime = Math.max(minStoreTime, commitLog.getMinTimestamp());\n        }\n        if (Long.MAX_VALUE != consumeQueue.getMinTimestamp()) {\n            minStoreTime = Math.max(minStoreTime, consumeQueue.getMinTimestamp());\n        }\n        return minStoreTime;\n    }\n\n    @Override\n    public long getMaxStoreTimestamp() {\n        return commitLog.getMaxTimestamp();\n    }\n\n    @Override\n    public long getFirstMessageOffset() {\n        return commitLog.getMinOffsetFromFile();\n    }\n\n    @Override\n    public long getCommitLogMinOffset() {\n        return commitLog.getMinOffset();\n    }\n\n    @Override\n    public long getCommitLogMaxOffset() {\n        return commitLog.getAppendOffset();\n    }\n\n    @Override\n    public long getCommitLogCommitOffset() {\n        return commitLog.getCommitOffset();\n    }\n\n    @Override\n    public long getConsumeQueueMinOffset() {\n        long cqOffset = consumeQueue.getMinOffset() / MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE;\n        long effectiveOffset = this.commitLog.getMinOffsetFromFile();\n        return Math.max(cqOffset, effectiveOffset);\n    }\n\n    @Override\n    public long getConsumeQueueMaxOffset() {\n        return consumeQueue.getAppendOffset() / MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE;\n    }\n\n    @Override\n    public long getConsumeQueueCommitOffset() {\n        return consumeQueue.getCommitOffset() / MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE;\n    }\n\n    @Override\n    public CompletableFuture<Boolean> commitAsync() {\n        // acquire lock\n        if (commitLock.drainPermits() <= 0) {\n            return CompletableFuture.completedFuture(false);\n        }\n\n        return this.commitLog.commitAsync()\n            .thenCompose(result -> {\n                if (result) {\n                    return consumeQueue.commitAsync();\n                }\n                return CompletableFuture.completedFuture(false);\n            }).whenComplete((result, throwable) -> commitLock.release());\n    }\n\n    @Override\n    public CompletableFuture<ByteBuffer> getMessageAsync(long queueOffset) {\n        return getConsumeQueueAsync(queueOffset).thenCompose(cqBuffer -> {\n            long commitLogOffset = MessageFormatUtil.getCommitLogOffsetFromItem(cqBuffer);\n            int length = MessageFormatUtil.getSizeFromItem(cqBuffer);\n            return getCommitLogAsync(commitLogOffset, length);\n        });\n    }\n\n    @Override\n    public CompletableFuture<ByteBuffer> getCommitLogAsync(long offset, int length) {\n        return commitLog.readAsync(offset, length);\n    }\n\n    @Override\n    public CompletableFuture<ByteBuffer> getConsumeQueueAsync(long queueOffset) {\n        return this.getConsumeQueueAsync(queueOffset, 1);\n    }\n\n    @Override\n    public CompletableFuture<ByteBuffer> getConsumeQueueAsync(long queueOffset, int count) {\n        return consumeQueue.readAsync(\n            queueOffset * MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE,\n            count * MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE);\n    }\n\n    @Override\n    public CompletableFuture<Long> getQueueOffsetByTimeAsync(long timestamp, BoundaryType boundaryType) {\n        long cqMin = getConsumeQueueMinOffset();\n        long cqMax = getConsumeQueueCommitOffset() - 1;\n        if (cqMax == -1 || cqMax < cqMin) {\n            return CompletableFuture.completedFuture(cqMin);\n        }\n\n        ByteBuffer buffer = getMessageAsync(cqMax).join();\n        long storeTime = MessageFormatUtil.getStoreTimeStamp(buffer);\n        if (storeTime < timestamp) {\n            log.info(\"FlatMessageFile getQueueOffsetByTimeAsync, exceeded maximum time, \" +\n                \"filePath={}, timestamp={}, result={}\", filePath, timestamp, cqMax + 1);\n            return CompletableFuture.completedFuture(cqMax + 1);\n        }\n\n        buffer = getMessageAsync(cqMin).join();\n        storeTime = MessageFormatUtil.getStoreTimeStamp(buffer);\n        if (storeTime > timestamp) {\n            log.info(\"FlatMessageFile getQueueOffsetByTimeAsync, less than minimum time, \" +\n                \"filePath={}, timestamp={}, result={}\", filePath, timestamp, cqMin);\n            return CompletableFuture.completedFuture(cqMin);\n        }\n\n        // get correct consume queue file by binary search\n        List<FileSegment> consumeQueueFileList = this.consumeQueue.getFileSegmentList();\n        int low = 0, high = consumeQueueFileList.size() - 1;\n        int mid = low + (high - low) / 2;\n        while (low <= high) {\n            FileSegment fileSegment = consumeQueueFileList.get(mid);\n            if (fileSegment.getMinTimestamp() <= timestamp && timestamp <= fileSegment.getMaxTimestamp()) {\n                break;\n            } else if (timestamp < fileSegment.getMinTimestamp()) {\n                high = mid - 1;\n            } else {\n                low = mid + 1;\n            }\n            mid = low + (high - low) / 2;\n        }\n        FileSegment target = consumeQueueFileList.get(mid);\n\n        // binary search lower bound index in a sorted array\n        long minOffset = target.getBaseOffset() / MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE;\n        long maxOffset = target.getCommitOffset() / MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE - 1;\n        List<String> queryLog = new ArrayList<>();\n        while (minOffset < maxOffset) {\n            long middle = minOffset + (maxOffset - minOffset) / 2;\n            buffer = this.getMessageAsync(middle).join();\n            storeTime = MessageFormatUtil.getStoreTimeStamp(buffer);\n            queryLog.add(String.format(\"(range=%d-%d, middle=%d, timestamp=%d, diff=%dms)\",\n                minOffset, maxOffset, middle, storeTime, timestamp - storeTime));\n            if (storeTime < timestamp) {\n                minOffset = middle + 1;\n            } else {\n                maxOffset = middle;\n            }\n        }\n\n        long offset = minOffset;\n        if (boundaryType == BoundaryType.UPPER) {\n            while (true) {\n                long next = offset + 1;\n                if (next > cqMax) {\n                    break;\n                }\n                buffer = this.getMessageAsync(next).join();\n                storeTime = MessageFormatUtil.getStoreTimeStamp(buffer);\n                if (storeTime == timestamp) {\n                    offset = next;\n                } else {\n                    break;\n                }\n            }\n        }\n\n        log.info(\"FlatMessageFile getQueueOffsetByTimeAsync, filePath={}, timestamp={}, result={}, log={}\",\n            filePath, timestamp, offset, JSON.toJSONString(queryLog));\n        return CompletableFuture.completedFuture(offset);\n    }\n\n    @Override\n    public int hashCode() {\n        return filePath.hashCode();\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj) {\n            return true;\n        }\n        if (obj == null) {\n            return false;\n        }\n        if (getClass() != obj.getClass()) {\n            return false;\n        }\n        return StringUtils.equals(filePath, ((FlatMessageFile) obj).filePath);\n    }\n\n    @Override\n    public boolean isClosed() {\n        return closed;\n    }\n\n    @Override\n    public void shutdown() {\n        closed = true;\n        fileLock.lock();\n        try {\n            consumeQueue.shutdown();\n            commitLog.shutdown();\n        } finally {\n            fileLock.unlock();\n        }\n    }\n\n    @Override\n    public void destroyExpiredFile(long timestamp) {\n        fileLock.lock();\n        try {\n            consumeQueue.destroyExpiredFile(timestamp);\n            commitLog.destroyExpiredFile(timestamp);\n        } finally {\n            fileLock.unlock();\n        }\n    }\n\n    public void destroy() {\n        this.shutdown();\n        fileLock.lock();\n        try {\n            consumeQueue.destroyExpiredFile(Long.MAX_VALUE);\n            commitLog.destroyExpiredFile(Long.MAX_VALUE);\n            if (queueMetadata != null) {\n                metadataStore.deleteQueue(queueMetadata.getQueue());\n            }\n        } finally {\n            fileLock.unlock();\n        }\n    }\n\n    public long getFileReservedHours() {\n        if (topicMetadata.getReserveTime() > 0) {\n            return topicMetadata.getReserveTime();\n        }\n        return storeConfig.getTieredStoreFileReservedTime();\n    }\n}\n"
  },
  {
    "path": "tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.rocketmq.tieredstore.index;\n\nimport java.util.List;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport org.apache.rocketmq.tieredstore.common.AppendResult;\n\npublic interface IndexService {\n\n    void start();\n\n    /**\n     * Puts a key into the index.\n     *\n     * @param topic     The topic of the key.\n     * @param topicId   The ID of the topic.\n     * @param queueId   The ID of the queue.\n     * @param keySet    The set of keys to be indexed.\n     * @param offset    The offset value of the key.\n     * @param size      The size of the key.\n     * @param timestamp The timestamp of the key.\n     * @return The result of the put operation.\n     */\n    AppendResult putKey(\n        String topic, int topicId, int queueId, Set<String> keySet, long offset, int size, long timestamp);\n\n    /**\n     * Asynchronously queries the index for a specific key within a given time range.\n     *\n     * @param topic     The topic of the key.\n     * @param key       The key to be queried.\n     * @param beginTime The start time of the query range.\n     * @param endTime   The end time of the query range.\n     * @return A CompletableFuture that holds the list of IndexItems matching the query.\n     */\n    CompletableFuture<List<IndexItem>> queryAsync(String topic, String key, int maxCount, long beginTime, long endTime);\n\n    default void forceUpload() {\n    }\n\n    /**\n     * Shutdown the index service.\n     */\n    void shutdown();\n\n    /**\n     * Force shutdown the index service.\n     */\n    default void forceShutdown() {\n        shutdown();\n    };\n\n    /**\n     * Destroys the index service and releases all resources.\n     */\n    void destroy();\n}\n"
  },
  {
    "path": "tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreService.java",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.apache.rocketmq.tieredstore.index;\n\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.base.Stopwatch;\nimport java.io.File;\nimport java.nio.ByteBuffer;\nimport java.nio.file.Paths;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Comparator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ConcurrentNavigableMap;\nimport java.util.concurrent.ConcurrentSkipListMap;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.concurrent.locks.ReadWriteLock;\nimport java.util.concurrent.locks.ReentrantReadWriteLock;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.rocketmq.common.ServiceThread;\nimport org.apache.rocketmq.common.UtilAll;\nimport org.apache.rocketmq.store.logfile.DefaultMappedFile;\nimport org.apache.rocketmq.store.logfile.MappedFile;\nimport org.apache.rocketmq.tieredstore.MessageStoreConfig;\nimport org.apache.rocketmq.tieredstore.common.AppendResult;\nimport org.apache.rocketmq.tieredstore.file.FlatAppendFile;\nimport org.apache.rocketmq.tieredstore.file.FlatFileFactory;\nimport org.apache.rocketmq.tieredstore.provider.FileSegment;\nimport org.apache.rocketmq.tieredstore.util.MessageStoreUtil;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\npublic class IndexStoreService extends ServiceThread implements IndexService {\n\n    private static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME);\n\n    public static final String FILE_DIRECTORY_NAME = \"tiered_index_file\";\n    public static final String FILE_COMPACTED_DIRECTORY_NAME = \"compacting\";\n\n    /**\n     * File status in table example:\n     * upload, upload, upload, sealed, sealed, unsealed\n     */\n    private final MessageStoreConfig storeConfig;\n    private final ConcurrentSkipListMap<Long /* timestamp */, IndexFile> timeStoreTable;\n    private final ReadWriteLock readWriteLock;\n    private final AtomicLong compactTimestamp;\n    private final String filePath;\n    private final FlatFileFactory fileAllocator;\n    private final boolean autoCreateNewFile;\n\n    private volatile IndexFile currentWriteFile;\n    private volatile FlatAppendFile flatAppendFile;\n\n    public IndexStoreService(FlatFileFactory flatFileFactory, String filePath) {\n        this(flatFileFactory, filePath, true);\n    }\n\n    public IndexStoreService(FlatFileFactory flatFileFactory, String filePath, boolean autoCreateNewFile) {\n        this.storeConfig = flatFileFactory.getStoreConfig();\n        this.filePath = filePath;\n        this.fileAllocator = flatFileFactory;\n        this.timeStoreTable = new ConcurrentSkipListMap<>();\n        this.compactTimestamp = new AtomicLong(0L);\n        this.readWriteLock = new ReentrantReadWriteLock();\n        this.autoCreateNewFile = autoCreateNewFile;\n    }\n\n    @Override\n    public void start() {\n        this.recover();\n        super.start();\n    }\n\n    private void doConvertOldFormatFile(String filePath) {\n        try {\n            File file = new File(filePath);\n            if (!file.exists()) {\n                return;\n            }\n            MappedFile mappedFile = new DefaultMappedFile(file.getPath(), (int) file.length());\n            long timestamp = mappedFile.getMappedByteBuffer().getLong(IndexStoreFile.INDEX_BEGIN_TIME_STAMP);\n            if (timestamp <= 0) {\n                mappedFile.destroy(TimeUnit.SECONDS.toMillis(10));\n            } else {\n                mappedFile.renameTo(String.valueOf(new File(file.getParent(), String.valueOf(timestamp))));\n                mappedFile.shutdown(TimeUnit.SECONDS.toMillis(10));\n            }\n        } catch (Exception e) {\n            log.error(\"IndexStoreService do convert old format error, file: {}\", filePath, e);\n        }\n    }\n\n    private void recover() {\n        Stopwatch stopwatch = Stopwatch.createStarted();\n\n        // delete compact file directory\n        UtilAll.deleteFile(new File(Paths.get(storeConfig.getStorePathRootDir(),\n            FILE_DIRECTORY_NAME, FILE_COMPACTED_DIRECTORY_NAME).toString()));\n\n        // recover local\n        File dir = new File(Paths.get(storeConfig.getStorePathRootDir(), FILE_DIRECTORY_NAME).toString());\n        this.doConvertOldFormatFile(Paths.get(dir.getPath(), \"0000\").toString());\n        this.doConvertOldFormatFile(Paths.get(dir.getPath(), \"1111\").toString());\n        File[] files = dir.listFiles();\n\n        if (files != null) {\n            List<File> fileList = Arrays.asList(files);\n            fileList.sort(Comparator.comparing(File::getName));\n\n            for (File file : fileList) {\n                if (file.isDirectory() || !StringUtils.isNumeric(file.getName())) {\n                    continue;\n                }\n\n                try {\n                    IndexFile indexFile = new IndexStoreFile(storeConfig, Long.parseLong(file.getName()));\n                    timeStoreTable.put(indexFile.getTimestamp(), indexFile);\n                    log.info(\"IndexStoreService recover load local file, timestamp: {}\", indexFile.getTimestamp());\n                } catch (Exception e) {\n                    log.error(\"IndexStoreService recover, load local file error\", e);\n                }\n            }\n        }\n\n        if (this.autoCreateNewFile && this.timeStoreTable.isEmpty()) {\n            this.createNewIndexFile(System.currentTimeMillis());\n        }\n\n        if (this.timeStoreTable.isEmpty()) {\n            this.setCompactTimestamp(Long.MAX_VALUE);\n        } else {\n            this.currentWriteFile = this.timeStoreTable.lastEntry().getValue();\n            this.setCompactTimestamp(this.timeStoreTable.firstKey() - 1);\n        }\n\n        // recover remote\n        this.flatAppendFile = fileAllocator.createFlatFileForIndexFile(filePath);\n\n        for (FileSegment fileSegment : flatAppendFile.getFileSegmentList()) {\n            IndexFile indexFile = new IndexStoreFile(storeConfig, fileSegment);\n            IndexFile localFile = timeStoreTable.get(indexFile.getTimestamp());\n            if (localFile != null) {\n                localFile.destroy();\n            }\n            timeStoreTable.put(indexFile.getTimestamp(), indexFile);\n            log.info(\"IndexStoreService recover load remote file, timestamp: {}, end timestamp: {}\",\n                indexFile.getTimestamp(), indexFile.getEndTimestamp());\n        }\n\n        log.info(\"IndexStoreService recover finished, total: {}, cost: {}ms, directory: {}\",\n            timeStoreTable.size(), stopwatch.elapsed(TimeUnit.MILLISECONDS), dir.getAbsolutePath());\n    }\n\n    public void createNewIndexFile(long timestamp) {\n        try {\n            this.readWriteLock.writeLock().lock();\n            IndexFile indexFile = this.currentWriteFile;\n            if (this.timeStoreTable.containsKey(timestamp) ||\n                indexFile != null && IndexFile.IndexStatusEnum.UNSEALED.equals(indexFile.getFileStatus())) {\n                return;\n            }\n            IndexStoreFile newStoreFile = new IndexStoreFile(storeConfig, timestamp);\n            this.timeStoreTable.put(timestamp, newStoreFile);\n            this.currentWriteFile = newStoreFile;\n            log.info(\"IndexStoreService construct next file, timestamp: {}\", timestamp);\n        } catch (Exception e) {\n            log.error(\"IndexStoreService construct next file, timestamp: {}\", timestamp, e);\n        } finally {\n            this.readWriteLock.writeLock().unlock();\n        }\n    }\n\n    @VisibleForTesting\n    public ConcurrentSkipListMap<Long, IndexFile> getTimeStoreTable() {\n        return timeStoreTable;\n    }\n\n    @Override\n    public AppendResult putKey(\n        String topic, int topicId, int queueId, Set<String> keySet, long offset, int size, long timestamp) {\n\n        if (StringUtils.isBlank(topic)) {\n            return AppendResult.UNKNOWN_ERROR;\n        }\n\n        if (keySet == null || keySet.isEmpty()) {\n            return AppendResult.SUCCESS;\n        }\n\n        for (int i = 0; i < 3; i++) {\n            AppendResult result = this.currentWriteFile.putKey(\n                topic, topicId, queueId, keySet, offset, size, timestamp);\n\n            if (AppendResult.SUCCESS.equals(result)) {\n                return AppendResult.SUCCESS;\n            } else if (AppendResult.FILE_FULL.equals(result)) {\n                // use current time to ensure the order of file\n                this.createNewIndexFile(System.currentTimeMillis());\n            }\n        }\n\n        log.error(\"IndexStoreService put key three times return error, topic: {}, topicId: {}, \" +\n            \"queueId: {}, keySize: {}, timestamp: {}\", topic, topicId, queueId, keySet.size(), timestamp);\n        return AppendResult.SUCCESS;\n    }\n\n    @Override\n    public CompletableFuture<List<IndexItem>> queryAsync(\n        String topic, String key, int maxCount, long beginTime, long endTime) {\n\n        if (beginTime > endTime) {\n            return CompletableFuture.completedFuture(new ArrayList<>());\n        }\n\n        CompletableFuture<List<IndexItem>> future = new CompletableFuture<>();\n        try {\n            readWriteLock.readLock().lock();\n            ConcurrentNavigableMap<Long, IndexFile> pendingMap =\n                this.timeStoreTable.subMap(beginTime, true, endTime, true);\n            List<CompletableFuture<Void>> futureList = new ArrayList<>(pendingMap.size());\n            ConcurrentSkipListMap<String /* queueId-offset */, IndexItem> result = new ConcurrentSkipListMap<>();\n\n            for (Map.Entry<Long, IndexFile> entry : pendingMap.descendingMap().entrySet()) {\n                CompletableFuture<Void> completableFuture = entry.getValue()\n                    .queryAsync(topic, key, maxCount, beginTime, endTime)\n                    .thenAccept(itemList -> itemList.forEach(indexItem -> {\n                        if (result.size() < maxCount) {\n                            result.put(String.format(\n                                \"%d-%20d\", indexItem.getQueueId(), indexItem.getOffset()), indexItem);\n                        }\n                    }));\n                futureList.add(completableFuture);\n            }\n\n            CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0]))\n                .whenComplete((v, t) -> {\n                    // Try to return the query results as much as possible here\n                    // rather than directly throwing exceptions\n                    if (t != null) {\n                        log.error(\"IndexStoreService#queryAsync, topicId={}, key={}, maxCount={}, timestamp={}-{}\",\n                            topic, key, maxCount, beginTime, endTime, t);\n                    }\n                    List<IndexItem> resultList = new ArrayList<>(result.values());\n                    future.complete(resultList.subList(0, Math.min(resultList.size(), maxCount)));\n                });\n        } catch (Exception e) {\n            log.error(\"IndexStoreService#queryAsync, topicId={}, key={}, maxCount={}, timestamp={}-{}\",\n                topic, key, maxCount, beginTime, endTime, e);\n            future.completeExceptionally(e);\n        } finally {\n            readWriteLock.readLock().unlock();\n        }\n        return future;\n    }\n\n    @Override\n    public void forceUpload() {\n        try {\n            readWriteLock.writeLock().lock();\n            while (true) {\n                Map.Entry<Long, IndexFile> entry =\n                    this.timeStoreTable.higherEntry(this.compactTimestamp.get());\n                if (entry == null) {\n                    break;\n                }\n                if (this.doCompactThenUploadFile(entry.getValue())) {\n                    this.setCompactTimestamp(entry.getValue().getTimestamp());\n                    // The total number of files will not too much, prevent io too fast.\n                    TimeUnit.MILLISECONDS.sleep(50);\n                }\n            }\n        } catch (Exception e) {\n            log.error(\"IndexStoreService force upload error\", e);\n            throw new RuntimeException(e);\n        } finally {\n            readWriteLock.writeLock().unlock();\n        }\n    }\n\n    public boolean doCompactThenUploadFile(IndexFile indexFile) {\n        if (IndexFile.IndexStatusEnum.UPLOAD.equals(indexFile.getFileStatus())) {\n            log.error(\"IndexStoreService file status not correct, so skip, timestamp: {}, status: {}\",\n                indexFile.getTimestamp(), indexFile.getFileStatus());\n            indexFile.destroy();\n            return true;\n        }\n\n        Stopwatch stopwatch = Stopwatch.createStarted();\n        if (flatAppendFile.getCommitOffset() == flatAppendFile.getAppendOffset()) {\n            ByteBuffer byteBuffer = indexFile.doCompaction();\n            if (byteBuffer == null) {\n                log.error(\"IndexStoreService found compaction buffer is null, timestamp: {}\", indexFile.getTimestamp());\n                return false;\n            }\n            flatAppendFile.rollingNewFile(Math.max(0L, flatAppendFile.getAppendOffset()));\n            flatAppendFile.append(byteBuffer, indexFile.getTimestamp());\n            flatAppendFile.getFileToWrite().setMinTimestamp(indexFile.getTimestamp());\n            flatAppendFile.getFileToWrite().setMaxTimestamp(indexFile.getEndTimestamp());\n        }\n        boolean result = flatAppendFile.commitAsync().join();\n\n        List<FileSegment> fileSegmentList = flatAppendFile.getFileSegmentList();\n        FileSegment fileSegment = fileSegmentList.get(fileSegmentList.size() - 1);\n        if (!result || fileSegment == null || fileSegment.getMinTimestamp() != indexFile.getTimestamp()) {\n            log.warn(\"IndexStoreService upload compacted file error, timestamp: {}\", indexFile.getTimestamp());\n            return false;\n        } else {\n            log.info(\"IndexStoreService upload compacted file success, timestamp: {}\", indexFile.getTimestamp());\n        }\n\n        readWriteLock.writeLock().lock();\n        try {\n            IndexFile storeFile = new IndexStoreFile(storeConfig, fileSegment);\n            timeStoreTable.put(storeFile.getTimestamp(), storeFile);\n            indexFile.destroy();\n        } catch (Exception e) {\n            log.error(\"IndexStoreService rolling file error, timestamp: {}, cost: {}ms\",\n                indexFile.getTimestamp(), stopwatch.elapsed(TimeUnit.MILLISECONDS), e);\n        } finally {\n            readWriteLock.writeLock().unlock();\n        }\n        return true;\n    }\n\n    public void destroyExpiredFile(long expireTimestamp) {\n        // delete file in time store table\n        readWriteLock.writeLock().lock();\n        try {\n            flatAppendFile.destroyExpiredFile(expireTimestamp);\n            timeStoreTable.entrySet().removeIf(entry ->\n                IndexFile.IndexStatusEnum.UPLOAD.equals(entry.getValue().getFileStatus()) &&\n                    (flatAppendFile.getFileSegmentList().isEmpty() ||\n                        entry.getKey() < flatAppendFile.getMinTimestamp()));\n            int tableSize = (int) timeStoreTable.entrySet().stream()\n                .filter(entry -> IndexFile.IndexStatusEnum.UPLOAD.equals(entry.getValue().getFileStatus()))\n                .count();\n            log.debug(\"IndexStoreService delete file, timestamp={}, remote={}, table={}, all={}\",\n                expireTimestamp, flatAppendFile.getFileSegmentList().size(), tableSize, timeStoreTable.size());\n        } finally {\n            readWriteLock.writeLock().unlock();\n        }\n    }\n\n    public void destroy() {\n        readWriteLock.writeLock().lock();\n        try {\n            // delete local store file\n            for (Map.Entry<Long, IndexFile> entry : timeStoreTable.entrySet()) {\n                IndexFile indexFile = entry.getValue();\n                if (IndexFile.IndexStatusEnum.UPLOAD.equals(indexFile.getFileStatus())) {\n                    continue;\n                }\n                indexFile.destroy();\n            }\n            // delete remote\n            if (flatAppendFile != null) {\n                flatAppendFile.destroy();\n            }\n        } catch (Exception e) {\n            log.error(\"IndexStoreService destroy all file error\", e);\n        } finally {\n            readWriteLock.writeLock().unlock();\n        }\n    }\n\n    @Override\n    public String getServiceName() {\n        return IndexStoreService.class.getSimpleName() + \"_\" + this.storeConfig.getBrokerName();\n    }\n\n    public void setCompactTimestamp(long timestamp) {\n        this.compactTimestamp.set(timestamp);\n        log.debug(\"IndexStoreService set compact timestamp to: {}\", timestamp);\n    }\n\n    protected IndexFile getNextSealedFile() {\n        Map.Entry<Long, IndexFile> entry =\n            this.timeStoreTable.higherEntry(this.compactTimestamp.get());\n        if (entry != null && entry.getKey() < this.timeStoreTable.lastKey()) {\n            return entry.getValue();\n        }\n        return null;\n    }\n\n    @Override\n    public void shutdown() {\n        super.shutdown();\n        // Wait index service upload then clear time store table\n        while (!this.timeStoreTable.isEmpty()) {\n            try {\n                TimeUnit.MILLISECONDS.sleep(50);\n            } catch (InterruptedException e) {\n                throw new RuntimeException(e);\n            }\n        }\n    }\n\n    @Override\n    public void forceShutdown() {\n        super.shutdown();\n    }\n\n    @Override\n    public void run() {\n        while (!this.isStopped()) {\n            try {\n                long expireTimestamp = System.currentTimeMillis()\n                    - TimeUnit.HOURS.toMillis(storeConfig.getTieredStoreFileReservedTime());\n                this.destroyExpiredFile(expireTimestamp);\n                IndexFile indexFile = this.getNextSealedFile();\n                if (indexFile != null) {\n                    if (this.doCompactThenUploadFile(indexFile)) {\n                        this.setCompactTimestamp(indexFile.getTimestamp());\n                        continue;\n                    }\n                }\n            } catch (Throwable e) {\n                log.error(\"IndexStoreService running error\", e);\n            }\n            this.waitForRunning(TimeUnit.SECONDS.toMillis(10));\n        }\n        readWriteLock.writeLock().lock();\n        try {\n            if (autoCreateNewFile) {\n                this.forceUpload();\n            }\n        } catch (Exception e) {\n            log.error(\"IndexStoreService shutdown error\", e);\n        } finally {\n            this.timeStoreTable.forEach((timestamp, file) -> file.shutdown());\n            this.timeStoreTable.clear();\n            readWriteLock.writeLock().unlock();\n        }\n\n        log.info(this.getServiceName() + \" service shutdown\");\n    }\n}\n"
  }
]